2023-10-09T08:39:50.855 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T08:39:50.855 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607 --description upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --owner scheduled_teuthology@teuthology --verbose -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607/orig.config.yaml 2023-10-09T08:39:50.915 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T08:39:51.002 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' first_in_suite: false job_id: '7417607' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 kernel: hwe: true workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 3687 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi163.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= smithi181.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= tasks: - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T08:39:51.004 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T08:39:51.005 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T08:39:51.005 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T08:39:51.006 INFO:teuthology.task.internal:Checking packages... 2023-10-09T08:39:51.034 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T08:39:51.035 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T08:39:51.035 INFO:teuthology.packaging:ref: None 2023-10-09T08:39:51.035 INFO:teuthology.packaging:tag: None 2023-10-09T08:39:51.035 INFO:teuthology.packaging:branch: reef 2023-10-09T08:39:51.035 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:39:51.035 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=reef 2023-10-09T08:39:51.242 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610-g397de1b0-1focal 2023-10-09T08:39:51.242 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T08:39:51.249 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T08:39:51.249 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T08:39:51.264 INFO:teuthology.task.internal:Saving configuration 2023-10-09T08:39:51.286 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T08:39:51.292 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T08:39:51.316 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi163.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 08:33:16.710457', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-09T08:39:51.337 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi181.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 08:33:16.713244', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-09T08:39:51.337 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T08:39:51.343 INFO:teuthology.task.internal:roles: ubuntu@smithi163.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T08:39:51.343 INFO:teuthology.task.internal:roles: ubuntu@smithi181.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T08:39:51.343 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T08:39:51.413 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f61ee8021f0>, signals=[15]) 2023-10-09T08:39:51.414 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T08:39:51.420 INFO:teuthology.task.internal:Opening connections... 2023-10-09T08:39:51.420 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi163.front.sepia.ceph.com 2023-10-09T08:39:51.422 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:39:51.467 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi181.front.sepia.ceph.com 2023-10-09T08:39:51.467 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:39:51.514 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T08:39:51.521 DEBUG:teuthology.orchestra.run.smithi163:> uname -m 2023-10-09T08:39:51.548 INFO:teuthology.orchestra.run.smithi163.stdout:x86_64 2023-10-09T08:39:51.548 DEBUG:teuthology.orchestra.run.smithi163:> cat /etc/os-release 2023-10-09T08:39:51.598 INFO:teuthology.orchestra.run.smithi163.stdout:NAME="Ubuntu" 2023-10-09T08:39:51.598 INFO:teuthology.orchestra.run.smithi163.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-09T08:39:51.599 INFO:teuthology.orchestra.run.smithi163.stdout:ID=ubuntu 2023-10-09T08:39:51.599 INFO:teuthology.orchestra.run.smithi163.stdout:ID_LIKE=debian 2023-10-09T08:39:51.599 INFO:teuthology.orchestra.run.smithi163.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-09T08:39:51.599 INFO:teuthology.orchestra.run.smithi163.stdout:VERSION_ID="20.04" 2023-10-09T08:39:51.599 INFO:teuthology.orchestra.run.smithi163.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-09T08:39:51.599 INFO:teuthology.orchestra.run.smithi163.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-09T08:39:51.599 INFO:teuthology.orchestra.run.smithi163.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-09T08:39:51.599 INFO:teuthology.orchestra.run.smithi163.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-09T08:39:51.599 INFO:teuthology.orchestra.run.smithi163.stdout:VERSION_CODENAME=focal 2023-10-09T08:39:51.599 INFO:teuthology.orchestra.run.smithi163.stdout:UBUNTU_CODENAME=focal 2023-10-09T08:39:51.600 INFO:teuthology.lock.ops:Updating smithi163.front.sepia.ceph.com on lock server 2023-10-09T08:39:51.621 DEBUG:teuthology.orchestra.run.smithi181:> uname -m 2023-10-09T08:39:51.628 INFO:teuthology.orchestra.run.smithi181.stdout:x86_64 2023-10-09T08:39:51.628 DEBUG:teuthology.orchestra.run.smithi181:> cat /etc/os-release 2023-10-09T08:39:51.678 INFO:teuthology.orchestra.run.smithi181.stdout:NAME="Ubuntu" 2023-10-09T08:39:51.679 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-09T08:39:51.679 INFO:teuthology.orchestra.run.smithi181.stdout:ID=ubuntu 2023-10-09T08:39:51.679 INFO:teuthology.orchestra.run.smithi181.stdout:ID_LIKE=debian 2023-10-09T08:39:51.679 INFO:teuthology.orchestra.run.smithi181.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-09T08:39:51.679 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION_ID="20.04" 2023-10-09T08:39:51.679 INFO:teuthology.orchestra.run.smithi181.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-09T08:39:51.679 INFO:teuthology.orchestra.run.smithi181.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-09T08:39:51.679 INFO:teuthology.orchestra.run.smithi181.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-09T08:39:51.679 INFO:teuthology.orchestra.run.smithi181.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-09T08:39:51.679 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION_CODENAME=focal 2023-10-09T08:39:51.680 INFO:teuthology.orchestra.run.smithi181.stdout:UBUNTU_CODENAME=focal 2023-10-09T08:39:51.680 INFO:teuthology.lock.ops:Updating smithi181.front.sepia.ceph.com on lock server 2023-10-09T08:39:51.704 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T08:39:51.713 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T08:39:51.718 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T08:39:51.719 DEBUG:teuthology.orchestra.run.smithi163:> test '!' -e /home/ubuntu/cephtest 2023-10-09T08:39:51.721 DEBUG:teuthology.orchestra.run.smithi181:> test '!' -e /home/ubuntu/cephtest 2023-10-09T08:39:51.727 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T08:39:51.734 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T08:39:51.734 DEBUG:teuthology.orchestra.run.smithi163:> test -z $(ls -A /var/lib/ceph) 2023-10-09T08:39:51.770 DEBUG:teuthology.orchestra.run.smithi181:> test -z $(ls -A /var/lib/ceph) 2023-10-09T08:39:51.788 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T08:39:51.849 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T08:39:51.865 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T08:39:51.865 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2023-10-09T08:39:51.865 DEBUG:teuthology.task.kernel:normalized overrides {'mon.a': {'hwe': True}, 'mon.c': {'hwe': True}, 'mgr.y': {'hwe': True}, 'osd.0': {'hwe': True}, 'osd.1': {'hwe': True}, 'osd.2': {'hwe': True}, 'osd.3': {'hwe': True}, 'client.0': {'hwe': True}, 'node-exporter.a': {'hwe': True}, 'alertmanager.a': {'hwe': True}, 'mon.b': {'hwe': True}, 'mgr.x': {'hwe': True}, 'osd.4': {'hwe': True}, 'osd.5': {'hwe': True}, 'osd.6': {'hwe': True}, 'osd.7': {'hwe': True}, 'client.1': {'hwe': True}, 'prometheus.a': {'hwe': True}, 'grafana.a': {'hwe': True}, 'node-exporter.b': {'hwe': True}} 2023-10-09T08:39:51.866 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'mon.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2023-10-09T08:39:51.866 DEBUG:teuthology.orchestra.run.smithi163:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T08:39:51.867 DEBUG:teuthology.orchestra.run.smithi181:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T08:39:51.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:39:51.872 DEBUG:teuthology.orchestra.run.smithi163:> uname -r 2023-10-09T08:39:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T08:39:51.873 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2023-10-09T08:39:51.922 INFO:teuthology.orchestra.run.smithi163.stdout:5.4.0-124-generic 2023-10-09T08:39:51.922 INFO:teuthology.task.kernel:Running kernel on smithi163: 5.4.0-124-generic 2023-10-09T08:39:51.922 DEBUG:teuthology.orchestra.run.smithi163:> sudo apt-get clean 2023-10-09T08:39:51.923 INFO:teuthology.orchestra.run.smithi181.stdout:5.4.0-124-generic 2023-10-09T08:39:51.923 INFO:teuthology.task.kernel:Running kernel on smithi181: 5.4.0-124-generic 2023-10-09T08:39:51.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-get clean 2023-10-09T08:39:52.033 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-get update 2023-10-09T08:39:52.041 DEBUG:teuthology.orchestra.run.smithi163:> sudo apt-get update 2023-10-09T08:39:52.342 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:39:52.347 INFO:teuthology.orchestra.run.smithi181.stdout:Get:2 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T08:39:52.362 INFO:teuthology.orchestra.run.smithi163.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:39:52.363 INFO:teuthology.orchestra.run.smithi163.stdout:Get:2 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T08:39:52.401 INFO:teuthology.orchestra.run.smithi181.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T08:39:52.411 INFO:teuthology.orchestra.run.smithi163.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T08:39:52.746 INFO:teuthology.orchestra.run.smithi181.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T08:39:52.756 INFO:teuthology.orchestra.run.smithi163.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T08:39:52.821 INFO:teuthology.orchestra.run.smithi181.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,500 kB] 2023-10-09T08:39:52.835 INFO:teuthology.orchestra.run.smithi163.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,500 kB] 2023-10-09T08:39:52.968 INFO:teuthology.orchestra.run.smithi181.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T08:39:52.978 INFO:teuthology.orchestra.run.smithi163.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T08:39:53.333 INFO:teuthology.orchestra.run.smithi163.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T08:39:53.334 INFO:teuthology.orchestra.run.smithi181.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [660 kB] 2023-10-09T08:39:53.334 INFO:teuthology.orchestra.run.smithi163.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [660 kB] 2023-10-09T08:39:53.348 INFO:teuthology.orchestra.run.smithi163.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [390 kB] 2023-10-09T08:39:53.349 INFO:teuthology.orchestra.run.smithi181.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [390 kB] 2023-10-09T08:39:53.357 INFO:teuthology.orchestra.run.smithi163.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-09T08:39:53.357 INFO:teuthology.orchestra.run.smithi181.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-09T08:39:53.357 INFO:teuthology.orchestra.run.smithi181.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,267 kB] 2023-10-09T08:39:53.358 INFO:teuthology.orchestra.run.smithi163.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,267 kB] 2023-10-09T08:39:53.433 INFO:teuthology.orchestra.run.smithi163.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-09T08:39:53.434 INFO:teuthology.orchestra.run.smithi163.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [317 kB] 2023-10-09T08:39:53.434 INFO:teuthology.orchestra.run.smithi181.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-09T08:39:53.434 INFO:teuthology.orchestra.run.smithi181.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [317 kB] 2023-10-09T08:39:53.435 INFO:teuthology.orchestra.run.smithi181.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T08:39:53.440 INFO:teuthology.orchestra.run.smithi163.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T08:39:53.440 INFO:teuthology.orchestra.run.smithi163.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [616 kB] 2023-10-09T08:39:53.441 INFO:teuthology.orchestra.run.smithi181.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T08:39:53.441 INFO:teuthology.orchestra.run.smithi181.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [890 kB] 2023-10-09T08:39:53.450 INFO:teuthology.orchestra.run.smithi181.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [473 kB] 2023-10-09T08:39:53.453 INFO:teuthology.orchestra.run.smithi163.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [890 kB] 2023-10-09T08:39:53.460 INFO:teuthology.orchestra.run.smithi181.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-09T08:39:53.460 INFO:teuthology.orchestra.run.smithi181.stdout:Get:18 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [616 kB] 2023-10-09T08:39:53.461 INFO:teuthology.orchestra.run.smithi181.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-09T08:39:53.461 INFO:teuthology.orchestra.run.smithi181.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,383 kB] 2023-10-09T08:39:53.508 INFO:teuthology.orchestra.run.smithi163.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [473 kB] 2023-10-09T08:39:53.521 INFO:teuthology.orchestra.run.smithi163.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-09T08:39:53.521 INFO:teuthology.orchestra.run.smithi163.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,383 kB] 2023-10-09T08:39:53.536 INFO:teuthology.orchestra.run.smithi181.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [334 kB] 2023-10-09T08:39:53.542 INFO:teuthology.orchestra.run.smithi181.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T08:39:53.592 INFO:teuthology.orchestra.run.smithi163.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-09T08:39:53.592 INFO:teuthology.orchestra.run.smithi163.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [334 kB] 2023-10-09T08:39:53.597 INFO:teuthology.orchestra.run.smithi163.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T08:39:53.636 INFO:teuthology.orchestra.run.smithi163.stdout:Get:23 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [186 kB] 2023-10-09T08:39:53.637 INFO:teuthology.orchestra.run.smithi181.stdout:Get:23 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [186 kB] 2023-10-09T08:39:53.706 INFO:teuthology.orchestra.run.smithi181.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T08:39:53.761 INFO:teuthology.orchestra.run.smithi163.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T08:39:54.051 INFO:teuthology.orchestra.run.smithi163.stdout:Get:25 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-09T08:39:54.053 INFO:teuthology.orchestra.run.smithi181.stdout:Get:25 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-09T08:39:54.057 INFO:teuthology.orchestra.run.smithi163.stdout:Get:26 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-09T08:39:54.059 INFO:teuthology.orchestra.run.smithi181.stdout:Get:26 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-09T08:39:54.061 INFO:teuthology.orchestra.run.smithi181.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-09T08:39:54.064 INFO:teuthology.orchestra.run.smithi163.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-09T08:39:54.067 INFO:teuthology.orchestra.run.smithi163.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-09T08:39:54.069 INFO:teuthology.orchestra.run.smithi163.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-09T08:39:54.069 INFO:teuthology.orchestra.run.smithi181.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-09T08:39:54.071 INFO:teuthology.orchestra.run.smithi181.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-09T08:39:54.339 INFO:teuthology.orchestra.run.smithi181.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T08:39:54.348 INFO:teuthology.orchestra.run.smithi163.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T08:39:54.377 INFO:teuthology.orchestra.run.smithi181.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [268 kB] 2023-10-09T08:39:54.391 INFO:teuthology.orchestra.run.smithi181.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-09T08:39:54.392 INFO:teuthology.orchestra.run.smithi181.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-09T08:39:54.393 INFO:teuthology.orchestra.run.smithi181.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-09T08:39:54.394 INFO:teuthology.orchestra.run.smithi181.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-09T08:39:54.395 INFO:teuthology.orchestra.run.smithi181.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-09T08:39:54.395 INFO:teuthology.orchestra.run.smithi181.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-09T08:39:54.420 INFO:teuthology.orchestra.run.smithi181.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-09T08:39:54.422 INFO:teuthology.orchestra.run.smithi181.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-09T08:39:54.458 INFO:teuthology.orchestra.run.smithi163.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [268 kB] 2023-10-09T08:39:54.479 INFO:teuthology.orchestra.run.smithi163.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-09T08:39:54.480 INFO:teuthology.orchestra.run.smithi163.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-09T08:39:54.481 INFO:teuthology.orchestra.run.smithi163.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-09T08:39:54.482 INFO:teuthology.orchestra.run.smithi163.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-09T08:39:54.483 INFO:teuthology.orchestra.run.smithi163.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-09T08:39:54.483 INFO:teuthology.orchestra.run.smithi163.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-09T08:39:54.485 INFO:teuthology.orchestra.run.smithi163.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-09T08:39:54.489 INFO:teuthology.orchestra.run.smithi163.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-09T08:39:54.586 INFO:teuthology.orchestra.run.smithi181.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-09T08:39:54.653 INFO:teuthology.orchestra.run.smithi163.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-09T08:39:54.770 INFO:teuthology.orchestra.run.smithi181.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-09T08:39:54.792 INFO:teuthology.orchestra.run.smithi181.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-09T08:39:54.837 INFO:teuthology.orchestra.run.smithi163.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-09T08:39:54.859 INFO:teuthology.orchestra.run.smithi163.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-09T08:39:57.210 INFO:teuthology.orchestra.run.smithi181.stdout:Fetched 17.6 MB in 3s (5,646 kB/s) 2023-10-09T08:39:57.261 INFO:teuthology.orchestra.run.smithi163.stdout:Fetched 17.6 MB in 3s (5,676 kB/s) 2023-10-09T08:39:58.671 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2023-10-09T08:39:58.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T08:39:58.723 INFO:teuthology.orchestra.run.smithi163.stdout:Reading package lists... 2023-10-09T08:39:58.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T08:39:58.783 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2023-10-09T08:39:58.836 INFO:teuthology.orchestra.run.smithi163.stdout:Reading package lists... 2023-10-09T08:39:58.985 INFO:teuthology.orchestra.run.smithi181.stdout:Building dependency tree... 2023-10-09T08:39:58.985 INFO:teuthology.orchestra.run.smithi181.stdout:Reading state information... 2023-10-09T08:39:59.036 INFO:teuthology.orchestra.run.smithi163.stdout:Building dependency tree... 2023-10-09T08:39:59.036 INFO:teuthology.orchestra.run.smithi163.stdout:Reading state information... 2023-10-09T08:39:59.158 INFO:teuthology.orchestra.run.smithi181.stdout:The following additional packages will be installed: 2023-10-09T08:39:59.158 INFO:teuthology.orchestra.run.smithi181.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T08:39:59.158 INFO:teuthology.orchestra.run.smithi181.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-modules-5.15.0-86-generic 2023-10-09T08:39:59.158 INFO:teuthology.orchestra.run.smithi181.stdout: linux-modules-extra-5.15.0-86-generic thermald upower usbmuxd 2023-10-09T08:39:59.159 INFO:teuthology.orchestra.run.smithi181.stdout:Suggested packages: 2023-10-09T08:39:59.159 INFO:teuthology.orchestra.run.smithi181.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-09T08:39:59.159 INFO:teuthology.orchestra.run.smithi181.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-86-generic 2023-10-09T08:39:59.205 INFO:teuthology.orchestra.run.smithi181.stdout:The following NEW packages will be installed: 2023-10-09T08:39:59.206 INFO:teuthology.orchestra.run.smithi181.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T08:39:59.206 INFO:teuthology.orchestra.run.smithi181.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-image-generic-hwe-20.04 2023-10-09T08:39:59.206 INFO:teuthology.orchestra.run.smithi181.stdout: linux-modules-5.15.0-86-generic linux-modules-extra-5.15.0-86-generic 2023-10-09T08:39:59.206 INFO:teuthology.orchestra.run.smithi181.stdout: thermald upower usbmuxd 2023-10-09T08:39:59.214 INFO:teuthology.orchestra.run.smithi163.stdout:The following additional packages will be installed: 2023-10-09T08:39:59.215 INFO:teuthology.orchestra.run.smithi163.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T08:39:59.215 INFO:teuthology.orchestra.run.smithi163.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-modules-5.15.0-86-generic 2023-10-09T08:39:59.215 INFO:teuthology.orchestra.run.smithi163.stdout: linux-modules-extra-5.15.0-86-generic thermald upower usbmuxd 2023-10-09T08:39:59.215 INFO:teuthology.orchestra.run.smithi163.stdout:Suggested packages: 2023-10-09T08:39:59.215 INFO:teuthology.orchestra.run.smithi163.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-09T08:39:59.215 INFO:teuthology.orchestra.run.smithi163.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-86-generic 2023-10-09T08:39:59.265 INFO:teuthology.orchestra.run.smithi181.stdout:0 upgraded, 13 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T08:39:59.265 INFO:teuthology.orchestra.run.smithi181.stdout:Need to get 90.9 MB of archives. 2023-10-09T08:39:59.265 INFO:teuthology.orchestra.run.smithi181.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-09T08:39:59.266 INFO:teuthology.orchestra.run.smithi181.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-09T08:39:59.266 INFO:teuthology.orchestra.run.smithi163.stdout:The following NEW packages will be installed: 2023-10-09T08:39:59.267 INFO:teuthology.orchestra.run.smithi163.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T08:39:59.267 INFO:teuthology.orchestra.run.smithi163.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-image-generic-hwe-20.04 2023-10-09T08:39:59.267 INFO:teuthology.orchestra.run.smithi163.stdout: linux-modules-5.15.0-86-generic linux-modules-extra-5.15.0-86-generic 2023-10-09T08:39:59.267 INFO:teuthology.orchestra.run.smithi163.stdout: thermald upower usbmuxd 2023-10-09T08:39:59.331 INFO:teuthology.orchestra.run.smithi181.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-09T08:39:59.337 INFO:teuthology.orchestra.run.smithi181.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-09T08:39:59.342 INFO:teuthology.orchestra.run.smithi181.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-09T08:39:59.360 INFO:teuthology.orchestra.run.smithi181.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-09T08:39:59.368 INFO:teuthology.orchestra.run.smithi181.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [20.9 MB] 2023-10-09T08:39:59.449 INFO:teuthology.orchestra.run.smithi163.stdout:0 upgraded, 13 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T08:39:59.449 INFO:teuthology.orchestra.run.smithi163.stdout:Need to get 90.9 MB of archives. 2023-10-09T08:39:59.449 INFO:teuthology.orchestra.run.smithi163.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-09T08:39:59.450 INFO:teuthology.orchestra.run.smithi163.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-09T08:39:59.720 INFO:teuthology.orchestra.run.smithi163.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-09T08:39:59.723 INFO:teuthology.orchestra.run.smithi181.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [11.4 MB] 2023-10-09T08:39:59.748 INFO:teuthology.orchestra.run.smithi163.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-09T08:39:59.766 INFO:teuthology.orchestra.run.smithi163.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-09T08:39:59.836 INFO:teuthology.orchestra.run.smithi181.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [58.0 MB] 2023-10-09T08:39:59.840 INFO:teuthology.orchestra.run.smithi163.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-09T08:39:59.875 INFO:teuthology.orchestra.run.smithi163.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [20.9 MB] 2023-10-09T08:40:00.746 INFO:teuthology.orchestra.run.smithi163.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [11.4 MB] 2023-10-09T08:40:00.795 INFO:teuthology.orchestra.run.smithi181.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.86.96~20.04.44 [2,648 B] 2023-10-09T08:40:00.796 INFO:teuthology.orchestra.run.smithi181.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-09T08:40:00.797 INFO:teuthology.orchestra.run.smithi181.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-09T08:40:00.802 INFO:teuthology.orchestra.run.smithi181.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-09T08:40:00.804 INFO:teuthology.orchestra.run.smithi181.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-09T08:40:01.055 INFO:teuthology.orchestra.run.smithi163.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [58.0 MB] 2023-10-09T08:40:01.478 INFO:teuthology.orchestra.run.smithi181.stdout:Fetched 90.9 MB in 2s (57.2 MB/s) 2023-10-09T08:40:01.693 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-09T08:40:02.934 INFO:teuthology.orchestra.run.smithi163.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.86.96~20.04.44 [2,648 B] 2023-10-09T08:40:02.934 INFO:teuthology.orchestra.run.smithi163.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-09T08:40:02.935 INFO:teuthology.orchestra.run.smithi163.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-09T08:40:03.105 INFO:teuthology.orchestra.run.smithi163.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-09T08:40:03.449 INFO:teuthology.orchestra.run.smithi163.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-09T08:40:03.474 INFO:teuthology.orchestra.run.smithi181.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-09T08:40:03.479 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-09T08:40:03.657 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T08:40:04.044 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-09T08:40:04.057 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-09T08:40:04.102 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T08:40:04.236 INFO:teuthology.orchestra.run.smithi163.stdout:Fetched 90.9 MB in 4s (21.7 MB/s) 2023-10-09T08:40:04.388 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-09T08:40:04.401 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-09T08:40:04.430 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T08:40:04.452 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-09T08:40:04.741 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-09T08:40:04.756 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-09T08:40:04.791 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T08:40:05.144 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-09T08:40:05.159 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-09T08:40:05.202 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T08:40:05.530 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package linux-modules-5.15.0-86-generic. 2023-10-09T08:40:05.546 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../05-linux-modules-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:40:05.605 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:06.211 INFO:teuthology.orchestra.run.smithi163.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-09T08:40:06.216 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-09T08:40:06.390 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T08:40:06.744 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-09T08:40:06.759 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-09T08:40:06.802 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T08:40:07.154 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-09T08:40:07.168 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-09T08:40:07.213 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T08:40:07.532 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-09T08:40:07.547 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-09T08:40:07.590 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T08:40:07.952 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-09T08:40:07.965 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-09T08:40:08.026 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T08:40:08.214 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package linux-image-5.15.0-86-generic. 2023-10-09T08:40:08.228 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../06-linux-image-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:40:08.330 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:08.363 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package linux-modules-5.15.0-86-generic. 2023-10-09T08:40:08.378 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../05-linux-modules-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:40:08.421 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:09.059 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-86-generic. 2023-10-09T08:40:09.073 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:40:09.117 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:10.962 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package linux-image-5.15.0-86-generic. 2023-10-09T08:40:10.979 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../06-linux-image-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:40:11.082 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:11.607 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-86-generic. 2023-10-09T08:40:11.621 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T08:40:11.665 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:16.111 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-09T08:40:16.127 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.86.96~20.04.44_amd64.deb ... 2023-10-09T08:40:16.169 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T08:40:16.513 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-09T08:40:16.526 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-09T08:40:16.571 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T08:40:16.957 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package thermald. 2023-10-09T08:40:16.971 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-09T08:40:17.016 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T08:40:17.621 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package upower. 2023-10-09T08:40:17.634 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-09T08:40:17.697 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-09T08:40:18.108 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package usbmuxd. 2023-10-09T08:40:18.124 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-09T08:40:18.202 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T08:40:18.499 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T08:40:18.638 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T08:40:18.764 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T08:40:18.915 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T08:40:19.041 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T08:40:19.102 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-09T08:40:19.121 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.86.96~20.04.44_amd64.deb ... 2023-10-09T08:40:19.162 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T08:40:19.167 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T08:40:19.335 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-09T08:40:19.506 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-09T08:40:19.521 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-09T08:40:19.556 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T08:40:19.873 INFO:teuthology.orchestra.run.smithi181.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-09T08:40:19.918 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package thermald. 2023-10-09T08:40:19.934 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-09T08:40:19.956 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T08:40:19.976 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T08:40:20.150 INFO:teuthology.orchestra.run.smithi181.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-09T08:40:20.165 INFO:teuthology.orchestra.run.smithi181.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-09T08:40:20.165 INFO:teuthology.orchestra.run.smithi181.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-09T08:40:20.549 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package upower. 2023-10-09T08:40:20.562 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-09T08:40:20.625 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-09T08:40:20.690 INFO:teuthology.orchestra.run.smithi181.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-09T08:40:20.912 INFO:teuthology.orchestra.run.smithi181.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-09T08:40:20.952 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T08:40:21.061 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package usbmuxd. 2023-10-09T08:40:21.075 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-09T08:40:21.120 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T08:40:21.276 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T08:40:21.276 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T08:40:21.451 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T08:40:21.599 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T08:40:21.726 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T08:40:21.740 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:21.852 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T08:40:22.003 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T08:40:22.130 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T08:40:22.280 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-09T08:40:22.838 INFO:teuthology.orchestra.run.smithi163.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-09T08:40:22.893 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T08:40:23.062 INFO:teuthology.orchestra.run.smithi181.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:40:23.062 INFO:teuthology.orchestra.run.smithi181.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-86-generic 2023-10-09T08:40:23.103 INFO:teuthology.orchestra.run.smithi163.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-09T08:40:23.118 INFO:teuthology.orchestra.run.smithi163.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-09T08:40:23.118 INFO:teuthology.orchestra.run.smithi163.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-09T08:40:23.235 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:23.682 INFO:teuthology.orchestra.run.smithi163.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-09T08:40:23.914 INFO:teuthology.orchestra.run.smithi163.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-09T08:40:24.508 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T08:40:24.819 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T08:40:24.819 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T08:40:25.013 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:25.262 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:26.406 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T08:40:26.531 INFO:teuthology.orchestra.run.smithi181.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T08:40:26.581 INFO:teuthology.orchestra.run.smithi163.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:40:26.581 INFO:teuthology.orchestra.run.smithi163.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-86-generic 2023-10-09T08:40:26.774 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:26.820 INFO:teuthology.orchestra.run.smithi181.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T08:40:27.623 INFO:teuthology.orchestra.run.smithi181.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-09T08:40:27.723 INFO:teuthology.orchestra.run.smithi181.stdout:Processing triggers for linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:27.830 INFO:teuthology.orchestra.run.smithi181.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-09T08:40:27.831 INFO:teuthology.orchestra.run.smithi181.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:40:28.176 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:30.053 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T08:40:30.704 INFO:teuthology.orchestra.run.smithi163.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T08:40:30.994 INFO:teuthology.orchestra.run.smithi163.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T08:40:31.763 INFO:teuthology.orchestra.run.smithi163.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-09T08:40:31.897 INFO:teuthology.orchestra.run.smithi163.stdout:Processing triggers for linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T08:40:31.996 INFO:teuthology.orchestra.run.smithi163.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-09T08:40:31.996 INFO:teuthology.orchestra.run.smithi163.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:40:53.536 INFO:teuthology.orchestra.run.smithi181.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-09T08:40:53.537 INFO:teuthology.orchestra.run.smithi181.stdout:Sourcing file `/etc/default/grub' 2023-10-09T08:40:53.551 INFO:teuthology.orchestra.run.smithi181.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T08:40:53.588 INFO:teuthology.orchestra.run.smithi181.stdout:Generating grub configuration file ... 2023-10-09T08:40:53.913 INFO:teuthology.orchestra.run.smithi181.stdout:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:40:53.937 INFO:teuthology.orchestra.run.smithi181.stdout:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:40:54.345 INFO:teuthology.orchestra.run.smithi181.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T08:40:54.356 INFO:teuthology.orchestra.run.smithi181.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T08:40:56.427 INFO:teuthology.orchestra.run.smithi181.stdout:done 2023-10-09T08:40:56.866 DEBUG:teuthology.orchestra.run.smithi181:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T08:40:56.906 INFO:teuthology.orchestra.run.smithi181.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T08:40:56.907 INFO:teuthology.orchestra.run.smithi181.stdout:Status: install ok installed 2023-10-09T08:40:56.907 INFO:teuthology.orchestra.run.smithi181.stdout:Priority: optional 2023-10-09T08:40:56.907 INFO:teuthology.orchestra.run.smithi181.stdout:Section: kernel 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout:Installed-Size: 20 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout:Architecture: amd64 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout:Source: linux-meta-hwe-5.15 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout:Recommends: thermald 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout:Description: Generic Linux kernel image 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout: This package will always depend on the latest generic kernel image 2023-10-09T08:40:56.908 INFO:teuthology.orchestra.run.smithi181.stdout: available. 2023-10-09T08:40:56.909 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-86-generic 2023-10-09T08:40:56.909 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T08:40:56.909 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T08:40:56.909 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-10-09T08:40:56.909 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi181.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T08:40:56.909 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-get clean 2023-10-09T08:40:56.987 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-get update 2023-10-09T08:40:57.228 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:40:57.272 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T08:40:57.297 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:3 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T08:40:57.330 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T08:40:57.491 INFO:teuthology.orchestra.run.smithi163.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-09T08:40:57.491 INFO:teuthology.orchestra.run.smithi163.stdout:Sourcing file `/etc/default/grub' 2023-10-09T08:40:57.512 INFO:teuthology.orchestra.run.smithi163.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T08:40:57.541 INFO:teuthology.orchestra.run.smithi163.stdout:Generating grub configuration file ... 2023-10-09T08:40:57.884 INFO:teuthology.orchestra.run.smithi163.stdout:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:40:57.911 INFO:teuthology.orchestra.run.smithi163.stdout:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:40:58.333 INFO:teuthology.orchestra.run.smithi163.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T08:40:58.345 INFO:teuthology.orchestra.run.smithi163.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T08:40:58.990 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2023-10-09T08:40:59.015 DEBUG:teuthology.orchestra.run.smithi181:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T08:40:59.082 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2023-10-09T08:40:59.284 INFO:teuthology.orchestra.run.smithi181.stdout:Building dependency tree... 2023-10-09T08:40:59.285 INFO:teuthology.orchestra.run.smithi181.stdout:Reading state information... 2023-10-09T08:40:59.499 INFO:teuthology.orchestra.run.smithi181.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.86.96~20.04.44). 2023-10-09T08:40:59.499 INFO:teuthology.orchestra.run.smithi181.stdout:0 upgraded, 0 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T08:40:59.501 DEBUG:teuthology.orchestra.run.smithi181:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T08:40:59.520 INFO:teuthology.orchestra.run.smithi181.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T08:40:59.520 INFO:teuthology.orchestra.run.smithi181.stdout:Status: install ok installed 2023-10-09T08:40:59.520 INFO:teuthology.orchestra.run.smithi181.stdout:Priority: optional 2023-10-09T08:40:59.520 INFO:teuthology.orchestra.run.smithi181.stdout:Section: kernel 2023-10-09T08:40:59.520 INFO:teuthology.orchestra.run.smithi181.stdout:Installed-Size: 20 2023-10-09T08:40:59.521 INFO:teuthology.orchestra.run.smithi181.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T08:40:59.521 INFO:teuthology.orchestra.run.smithi181.stdout:Architecture: amd64 2023-10-09T08:40:59.521 INFO:teuthology.orchestra.run.smithi181.stdout:Source: linux-meta-hwe-5.15 2023-10-09T08:40:59.521 INFO:teuthology.orchestra.run.smithi181.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T08:40:59.521 INFO:teuthology.orchestra.run.smithi181.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T08:40:59.521 INFO:teuthology.orchestra.run.smithi181.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T08:40:59.521 INFO:teuthology.orchestra.run.smithi181.stdout:Recommends: thermald 2023-10-09T08:40:59.521 INFO:teuthology.orchestra.run.smithi181.stdout:Description: Generic Linux kernel image 2023-10-09T08:40:59.521 INFO:teuthology.orchestra.run.smithi181.stdout: This package will always depend on the latest generic kernel image 2023-10-09T08:40:59.521 INFO:teuthology.orchestra.run.smithi181.stdout: available. 2023-10-09T08:40:59.522 DEBUG:teuthology.orchestra.run.smithi181:> mktemp 2023-10-09T08:40:59.528 INFO:teuthology.orchestra.run.smithi181.stdout:/tmp/tmp.Is483JGpdL 2023-10-09T08:40:59.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo cp /boot/grub/grub.cfg /tmp/tmp.Is483JGpdL 2023-10-09T08:40:59.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 0666 /tmp/tmp.Is483JGpdL 2023-10-09T08:40:59.612 DEBUG:teuthology.orchestra.remote:smithi181:/tmp/tmp.Is483JGpdL is 9KB 2023-10-09T08:40:59.626 DEBUG:teuthology.orchestra.run.smithi181:> rm -fr /tmp/tmp.Is483JGpdL 2023-10-09T08:40:59.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-09T08:40:59.687 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:40:59.687 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-09T08:40:59.687 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-09T08:40:59.754 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-86-generic 2023-10-09T08:40:59.754 DEBUG:teuthology.orchestra.run.smithi181:> sudo update-grub 2023-10-09T08:41:00.391 INFO:teuthology.orchestra.run.smithi163.stdout:done 2023-10-09T08:41:00.850 DEBUG:teuthology.orchestra.run.smithi163:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T08:41:00.907 INFO:teuthology.orchestra.run.smithi163.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Status: install ok installed 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Priority: optional 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Section: kernel 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Installed-Size: 20 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Architecture: amd64 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Source: linux-meta-hwe-5.15 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Recommends: thermald 2023-10-09T08:41:00.908 INFO:teuthology.orchestra.run.smithi163.stdout:Description: Generic Linux kernel image 2023-10-09T08:41:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: This package will always depend on the latest generic kernel image 2023-10-09T08:41:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: available. 2023-10-09T08:41:00.909 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-86-generic 2023-10-09T08:41:00.909 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T08:41:00.909 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T08:41:00.909 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-10-09T08:41:00.909 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi163.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T08:41:00.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo apt-get clean 2023-10-09T08:41:01.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo apt-get update 2023-10-09T08:41:01.135 INFO:teuthology.orchestra.run.smithi163.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:41:01.142 INFO:teuthology.orchestra.run.smithi163.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T08:41:01.159 INFO:teuthology.orchestra.run.smithi163.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T08:41:01.162 INFO:teuthology.orchestra.run.smithi181.stderr:Sourcing file `/etc/default/grub' 2023-10-09T08:41:01.166 INFO:teuthology.orchestra.run.smithi181.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T08:41:01.179 INFO:teuthology.orchestra.run.smithi181.stderr:Generating grub configuration file ... 2023-10-09T08:41:01.255 INFO:teuthology.orchestra.run.smithi163.stdout:Hit:4 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T08:41:01.445 INFO:teuthology.orchestra.run.smithi181.stderr:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:41:01.457 INFO:teuthology.orchestra.run.smithi181.stderr:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:41:01.813 INFO:teuthology.orchestra.run.smithi181.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T08:41:01.825 INFO:teuthology.orchestra.run.smithi181.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T08:41:02.835 INFO:teuthology.orchestra.run.smithi181.stderr:done 2023-10-09T08:41:02.837 DEBUG:teuthology.orchestra.run.smithi181:> sudo shutdown -r now 2023-10-09T08:41:02.857 INFO:teuthology.orchestra.run.smithi163.stdout:Reading package lists... 2023-10-09T08:41:02.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T08:41:02.955 INFO:teuthology.orchestra.run.smithi163.stdout:Reading package lists... 2023-10-09T08:41:03.133 INFO:teuthology.orchestra.run.smithi163.stdout:Building dependency tree... 2023-10-09T08:41:03.134 INFO:teuthology.orchestra.run.smithi163.stdout:Reading state information... 2023-10-09T08:41:03.340 INFO:teuthology.orchestra.run.smithi163.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.86.96~20.04.44). 2023-10-09T08:41:03.340 INFO:teuthology.orchestra.run.smithi163.stdout:0 upgraded, 0 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T08:41:03.342 DEBUG:teuthology.orchestra.run.smithi163:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T08:41:03.361 INFO:teuthology.orchestra.run.smithi163.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T08:41:03.361 INFO:teuthology.orchestra.run.smithi163.stdout:Status: install ok installed 2023-10-09T08:41:03.362 INFO:teuthology.orchestra.run.smithi163.stdout:Priority: optional 2023-10-09T08:41:03.362 INFO:teuthology.orchestra.run.smithi163.stdout:Section: kernel 2023-10-09T08:41:03.362 INFO:teuthology.orchestra.run.smithi163.stdout:Installed-Size: 20 2023-10-09T08:41:03.362 INFO:teuthology.orchestra.run.smithi163.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T08:41:03.362 INFO:teuthology.orchestra.run.smithi163.stdout:Architecture: amd64 2023-10-09T08:41:03.362 INFO:teuthology.orchestra.run.smithi163.stdout:Source: linux-meta-hwe-5.15 2023-10-09T08:41:03.362 INFO:teuthology.orchestra.run.smithi163.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T08:41:03.362 INFO:teuthology.orchestra.run.smithi163.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T08:41:03.362 INFO:teuthology.orchestra.run.smithi163.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T08:41:03.362 INFO:teuthology.orchestra.run.smithi163.stdout:Recommends: thermald 2023-10-09T08:41:03.363 INFO:teuthology.orchestra.run.smithi163.stdout:Description: Generic Linux kernel image 2023-10-09T08:41:03.363 INFO:teuthology.orchestra.run.smithi163.stdout: This package will always depend on the latest generic kernel image 2023-10-09T08:41:03.363 INFO:teuthology.orchestra.run.smithi163.stdout: available. 2023-10-09T08:41:03.363 DEBUG:teuthology.orchestra.run.smithi163:> mktemp 2023-10-09T08:41:03.369 INFO:teuthology.orchestra.run.smithi163.stdout:/tmp/tmp.n3xaXbX7q9 2023-10-09T08:41:03.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo cp /boot/grub/grub.cfg /tmp/tmp.n3xaXbX7q9 2023-10-09T08:41:03.382 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod 0666 /tmp/tmp.n3xaXbX7q9 2023-10-09T08:41:03.468 DEBUG:teuthology.orchestra.remote:smithi163:/tmp/tmp.n3xaXbX7q9 is 9KB 2023-10-09T08:41:03.482 DEBUG:teuthology.orchestra.run.smithi163:> rm -fr /tmp/tmp.n3xaXbX7q9 2023-10-09T08:41:03.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-09T08:41:03.543 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:41:03.543 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-09T08:41:03.543 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-09T08:41:03.576 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-86-generic 2023-10-09T08:41:03.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo update-grub 2023-10-09T08:41:05.019 INFO:teuthology.orchestra.run.smithi163.stderr:Sourcing file `/etc/default/grub' 2023-10-09T08:41:05.027 INFO:teuthology.orchestra.run.smithi163.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T08:41:05.043 INFO:teuthology.orchestra.run.smithi163.stderr:Generating grub configuration file ... 2023-10-09T08:41:05.293 INFO:teuthology.orchestra.run.smithi163.stderr:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T08:41:05.305 INFO:teuthology.orchestra.run.smithi163.stderr:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T08:41:05.660 INFO:teuthology.orchestra.run.smithi163.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T08:41:05.672 INFO:teuthology.orchestra.run.smithi163.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T08:41:06.662 INFO:teuthology.orchestra.run.smithi163.stderr:done 2023-10-09T08:41:06.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo shutdown -r now 2023-10-09T08:41:32.862 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-10-09T08:41:32.866 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2023-10-09T08:41:32.867 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:41:36.671 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-10-09T08:41:36.671 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2023-10-09T08:41:36.671 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:41:55.066 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.163 2023-10-09T08:42:04.075 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2023-10-09T08:42:04.076 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:42:07.134 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.163 2023-10-09T08:42:07.358 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.181 2023-10-09T08:42:16.366 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2023-10-09T08:42:16.367 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:42:19.138 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2023-10-09T08:42:19.139 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:42:22.202 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.163 2023-10-09T08:42:34.746 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.181 2023-10-09T08:42:37.206 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2023-10-09T08:42:37.206 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:42:46.754 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2023-10-09T08:42:46.755 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:42:47.539 DEBUG:teuthology.orchestra.run.smithi181:> true 2023-10-09T08:42:48.596 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2023-10-09T08:42:48.596 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-86-generic"... 2023-10-09T08:42:48.597 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2023-10-09T08:42:48.647 INFO:teuthology.orchestra.run.smithi181.stdout:5.15.0-86-generic 2023-10-09T08:42:48.647 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-86-generic vs 5.15.0-86-generic 2023-10-09T08:42:48.647 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T08:42:48.648 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-09T08:42:49.649 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T08:42:49.649 DEBUG:teuthology.orchestra.run.smithi181:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T08:42:49.731 INFO:teuthology.orchestra.run.smithi181.stdout:ttyS1 2023-10-09T08:42:49.771 DEBUG:teuthology.parallel:result is None 2023-10-09T08:43:08.798 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.163 2023-10-09T08:43:26.800 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2023-10-09T08:43:26.800 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:43:27.053 DEBUG:teuthology.orchestra.run.smithi163:> true 2023-10-09T08:43:27.558 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi163.front.sepia.ceph.com' 2023-10-09T08:43:27.558 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-86-generic"... 2023-10-09T08:43:27.559 DEBUG:teuthology.orchestra.run.smithi163:> uname -r 2023-10-09T08:43:27.608 INFO:teuthology.orchestra.run.smithi163.stdout:5.15.0-86-generic 2023-10-09T08:43:27.608 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-86-generic vs 5.15.0-86-generic 2023-10-09T08:43:27.609 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T08:43:27.609 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-09T08:43:28.609 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T08:43:28.610 DEBUG:teuthology.orchestra.run.smithi163:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T08:43:28.976 INFO:teuthology.orchestra.run.smithi163.stdout:ttyS1 2023-10-09T08:43:29.001 DEBUG:teuthology.parallel:result is None 2023-10-09T08:43:29.002 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T08:43:29.114 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T08:43:29.115 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T08:43:29.117 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T08:43:29.126 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T08:43:29.157 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T08:43:29.169 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T08:43:29.169 DEBUG:teuthology.orchestra.run.smithi163:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T08:43:29.173 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T08:43:29.281 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T08:43:29.321 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T08:43:29.321 DEBUG:teuthology.orchestra.run.smithi163:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T08:43:29.329 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T08:43:29.344 INFO:teuthology.orchestra.run.smithi163.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T08:43:29.351 INFO:teuthology.orchestra.run.smithi163.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T08:43:29.359 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T08:43:29.367 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T08:43:29.369 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T08:43:29.384 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T08:43:29.385 DEBUG:teuthology.orchestra.run.smithi163:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T08:43:29.397 DEBUG:teuthology.orchestra.run.smithi181:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T08:43:29.431 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T08:43:29.449 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T08:43:29.449 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T08:43:29.459 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T08:43:29.486 DEBUG:teuthology.orchestra.run.smithi163:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T08:43:29.514 DEBUG:teuthology.orchestra.run.smithi163:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T08:43:29.564 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:43:29.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T08:43:29.633 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T08:43:29.642 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T08:43:29.692 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:43:29.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T08:43:29.755 DEBUG:teuthology.orchestra.run.smithi163:> sudo service rsyslog restart 2023-10-09T08:43:29.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo service rsyslog restart 2023-10-09T08:43:29.844 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T08:43:29.855 INFO:teuthology.task.internal:Starting timer... 2023-10-09T08:43:29.855 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T08:43:29.867 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T08:43:29.888 DEBUG:teuthology.task.selinux:Excluding smithi163: OS 'ubuntu' does not support SELinux 2023-10-09T08:43:29.888 DEBUG:teuthology.task.selinux:Excluding smithi181: OS 'ubuntu' does not support SELinux 2023-10-09T08:43:29.889 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T08:43:29.889 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2023-10-09T08:43:29.889 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T08:43:29.889 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T08:43:29.959 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-10-09T08:43:29.959 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T08:43:29.976 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T08:43:29.977 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi163.front.sepia.ceph.com,smithi181.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T08:49:23.888 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi163.front.sepia.ceph.com'), Remote(name='ubuntu@smithi181.front.sepia.ceph.com')] 2023-10-09T08:49:23.889 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2023-10-09T08:49:23.890 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:49:23.983 DEBUG:teuthology.orchestra.run.smithi163:> true 2023-10-09T08:49:24.104 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi163.front.sepia.ceph.com' 2023-10-09T08:49:24.104 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2023-10-09T08:49:24.105 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T08:49:24.190 DEBUG:teuthology.orchestra.run.smithi181:> true 2023-10-09T08:49:24.312 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2023-10-09T08:49:24.312 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T08:49:24.323 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T08:49:24.323 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T08:49:24.324 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T08:49:24.326 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T08:49:24.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T08:49:24.356 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-09T08:49:24.357 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: Command line: ntpd -gq 2023-10-09T08:49:24.357 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: proto: precision = 0.059 usec (-24) 2023-10-09T08:49:24.357 INFO:teuthology.orchestra.run.smithi163.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T08:49:24.358 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T08:49:24.358 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: restrict ::: KOD does nothing without LIMITED. 2023-10-09T08:49:24.358 INFO:teuthology.orchestra.run.smithi163.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-09T08:49:24.360 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: Listen and drop on 0 v6wildcard [::]:123 2023-10-09T08:49:24.360 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-09T08:49:24.360 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: Listen normally on 2 lo 127.0.0.1:123 2023-10-09T08:49:24.360 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: Listen normally on 3 enp3s0f1 172.21.15.163:123 2023-10-09T08:49:24.360 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: Listen normally on 4 lo [::1]:123 2023-10-09T08:49:24.360 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7d5b%5]:123 2023-10-09T08:49:24.361 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:24 ntpd[22723]: Listening on routing socket on fd #22 for interface updates 2023-10-09T08:49:24.385 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-09T08:49:24.386 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: Command line: ntpd -gq 2023-10-09T08:49:24.386 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: proto: precision = 0.094 usec (-23) 2023-10-09T08:49:24.387 INFO:teuthology.orchestra.run.smithi181.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T08:49:24.387 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T08:49:24.388 INFO:teuthology.orchestra.run.smithi181.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-09T08:49:24.389 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: restrict ::: KOD does nothing without LIMITED. 2023-10-09T08:49:24.389 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: Listen and drop on 0 v6wildcard [::]:123 2023-10-09T08:49:24.389 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-09T08:49:24.389 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: Listen normally on 2 lo 127.0.0.1:123 2023-10-09T08:49:24.389 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: Listen normally on 3 enp3s0f1 172.21.15.181:123 2023-10-09T08:49:24.389 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: Listen normally on 4 lo [::1]:123 2023-10-09T08:49:24.389 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7b9d%5]:123 2023-10-09T08:49:24.390 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:24 ntpd[22715]: Listening on routing socket on fd #22 for interface updates 2023-10-09T08:49:25.356 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:25 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:25.357 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:25 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:26.357 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:26.357 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:26.357 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:26.357 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:26.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:26.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:26.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:26.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:26.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:26.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:26.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:26.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:26 ntpd[22723]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:26.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:26 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:26.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:26 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:26.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:26 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:26.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:26 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:26.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:26 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:26.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:26 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:26.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:26 ntpd[22715]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:26.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:26 ntpd[22715]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:27.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:27.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:27.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:27.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:27.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:27.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:27.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:27.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:27.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:27.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:27.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:27.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:27 ntpd[22715]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:28.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:28.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:28.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:28.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:28.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:28.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:28.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:28 ntpd[22723]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:28.386 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:28 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:28.386 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:28 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:28.386 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:28 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:28.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:28 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:28.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:28 ntpd[22715]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:28.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:28 ntpd[22715]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:29.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:29.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:29.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:29.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:29.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:29.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:29.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:29.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:29.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:29.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:29.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:29.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:29 ntpd[22715]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:30.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:30.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:30.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:30.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:30.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:30.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:30.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:30.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:30 ntpd[22723]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:30.386 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:30 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:30.386 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:30 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:30.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:30 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:30.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:30 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:30.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:30 ntpd[22715]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:30.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:30 ntpd[22715]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:31.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:31.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:31.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:31.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:31.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:31.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:31.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:31.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:31.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:31.389 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:31.389 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:31.389 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:31 ntpd[22715]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:32.357 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:32.358 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:32.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:32.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:32.359 INFO:teuthology.orchestra.run.smithi163.stdout: 9 Oct 08:49:32 ntpd[22723]: ntpd: time slew -0.000322 s 2023-10-09T08:49:32.359 INFO:teuthology.orchestra.run.smithi163.stdout:ntpd: time slew -0.000322s 2023-10-09T08:49:32.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-09T08:49:32.359 INFO:teuthology.orchestra.run.smithi163.stderr: 9 Oct 08:49:32 ntpd[22723]: can't open /var/log/ntpstats/loopstats.20231009: Permission denied 2023-10-09T08:49:32.386 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:32 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:32.386 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:32 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:32.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:32 ntpd[22715]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T08:49:32.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:32 ntpd[22715]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T08:49:32.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:32 ntpd[22715]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T08:49:32.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:32 ntpd[22715]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T08:49:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: 9 Oct 08:49:32 ntpd[22715]: ntpd: time slew +0.000953 s 2023-10-09T08:49:32.387 INFO:teuthology.orchestra.run.smithi181.stdout:ntpd: time slew +0.000953s 2023-10-09T08:49:32.387 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:32 ntpd[22715]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-09T08:49:32.388 INFO:teuthology.orchestra.run.smithi181.stderr: 9 Oct 08:49:32 ntpd[22715]: can't open /var/log/ntpstats/loopstats.20231009: Permission denied 2023-10-09T08:49:32.422 INFO:teuthology.orchestra.run.smithi163.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T08:49:32.422 INFO:teuthology.orchestra.run.smithi163.stdout:============================================================================== 2023-10-09T08:49:32.422 INFO:teuthology.orchestra.run.smithi163.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:49:32.422 INFO:teuthology.orchestra.run.smithi163.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:49:32.422 INFO:teuthology.orchestra.run.smithi163.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:49:32.422 INFO:teuthology.orchestra.run.smithi163.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:49:32.454 INFO:teuthology.orchestra.run.smithi181.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T08:49:32.454 INFO:teuthology.orchestra.run.smithi181.stdout:============================================================================== 2023-10-09T08:49:32.454 INFO:teuthology.orchestra.run.smithi181.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:49:32.454 INFO:teuthology.orchestra.run.smithi181.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:49:32.454 INFO:teuthology.orchestra.run.smithi181.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:49:32.455 INFO:teuthology.orchestra.run.smithi181.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T08:49:32.455 INFO:teuthology.run_tasks:Running task install... 2023-10-09T08:49:32.464 DEBUG:teuthology.task.install:project ceph 2023-10-09T08:49:32.464 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T08:49:32.465 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T08:49:32.465 INFO:teuthology.task.install:Using flavor: default 2023-10-09T08:49:32.470 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T08:49:32.470 INFO:teuthology.task.install:extra packages: [] 2023-10-09T08:49:32.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo apt-key list | grep Ceph 2023-10-09T08:49:32.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-key list | grep Ceph 2023-10-09T08:49:32.563 INFO:teuthology.orchestra.run.smithi163.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-09T08:49:32.590 INFO:teuthology.orchestra.run.smithi181.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-09T08:49:32.619 INFO:teuthology.orchestra.run.smithi163.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-09T08:49:32.619 INFO:teuthology.orchestra.run.smithi163.stdout:uid [ unknown] Ceph.com (release key) 2023-10-09T08:49:32.620 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-09T08:49:32.620 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T08:49:32.620 INFO:teuthology.packaging:ref: None 2023-10-09T08:49:32.620 INFO:teuthology.packaging:tag: None 2023-10-09T08:49:32.621 INFO:teuthology.packaging:branch: pacific 2023-10-09T08:49:32.621 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:49:32.621 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T08:49:32.653 INFO:teuthology.orchestra.run.smithi181.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-09T08:49:32.653 INFO:teuthology.orchestra.run.smithi181.stdout:uid [ unknown] Ceph.com (release key) 2023-10-09T08:49:32.654 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-09T08:49:32.654 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T08:49:32.654 INFO:teuthology.packaging:ref: None 2023-10-09T08:49:32.654 INFO:teuthology.packaging:tag: None 2023-10-09T08:49:32.654 INFO:teuthology.packaging:branch: pacific 2023-10-09T08:49:32.654 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:49:32.654 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T08:49:32.839 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default/ 2023-10-09T08:49:32.839 INFO:teuthology.task.install.deb:Package version is 16.2.14-48-gaeb2e425-1focal 2023-10-09T08:49:32.894 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default/ 2023-10-09T08:49:32.895 INFO:teuthology.task.install.deb:Package version is 16.2.14-48-gaeb2e425-1focal 2023-10-09T08:49:33.207 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:49:33.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-09T08:49:33.223 DEBUG:teuthology.orchestra.run.smithi163:> sudo apt-get update 2023-10-09T08:49:33.278 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:49:33.278 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-09T08:49:33.294 DEBUG:teuthology.orchestra.run.smithi181:> sudo apt-get update 2023-10-09T08:49:33.421 INFO:teuthology.orchestra.run.smithi163.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:49:33.430 INFO:teuthology.orchestra.run.smithi163.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T08:49:33.456 INFO:teuthology.orchestra.run.smithi163.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T08:49:33.498 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T08:49:33.499 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T08:49:33.512 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T08:49:33.535 INFO:teuthology.orchestra.run.smithi181.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T08:49:33.594 INFO:teuthology.orchestra.run.smithi163.stdout:Ign:4 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal InRelease 2023-10-09T08:49:33.632 INFO:teuthology.orchestra.run.smithi163.stdout:Get:5 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-10-09T08:49:33.639 INFO:teuthology.orchestra.run.smithi181.stdout:Ign:5 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal InRelease 2023-10-09T08:49:33.671 INFO:teuthology.orchestra.run.smithi163.stdout:Ign:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release.gpg 2023-10-09T08:49:33.677 INFO:teuthology.orchestra.run.smithi181.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-10-09T08:49:33.708 INFO:teuthology.orchestra.run.smithi163.stdout:Get:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main i386 Packages [3,102 B] 2023-10-09T08:49:33.714 INFO:teuthology.orchestra.run.smithi181.stdout:Ign:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release.gpg 2023-10-09T08:49:33.751 INFO:teuthology.orchestra.run.smithi181.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main i386 Packages [3,102 B] 2023-10-09T08:49:33.766 INFO:teuthology.orchestra.run.smithi163.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-10-09T08:49:34.029 INFO:teuthology.orchestra.run.smithi163.stdout:Hit:9 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T08:49:34.059 INFO:teuthology.orchestra.run.smithi181.stdout:Get:9 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-10-09T08:49:34.255 INFO:teuthology.orchestra.run.smithi163.stdout:Fetched 28.0 kB in 1s (34.3 kB/s) 2023-10-09T08:49:34.340 INFO:teuthology.orchestra.run.smithi181.stdout:Fetched 28.0 kB in 1s (33.3 kB/s) 2023-10-09T08:49:35.298 INFO:teuthology.orchestra.run.smithi163.stdout:Reading package lists... 2023-10-09T08:49:35.329 DEBUG:teuthology.orchestra.run.smithi163:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-48-gaeb2e425-1focal cephadm=16.2.14-48-gaeb2e425-1focal ceph-mds=16.2.14-48-gaeb2e425-1focal ceph-mgr=16.2.14-48-gaeb2e425-1focal ceph-common=16.2.14-48-gaeb2e425-1focal ceph-fuse=16.2.14-48-gaeb2e425-1focal ceph-test=16.2.14-48-gaeb2e425-1focal radosgw=16.2.14-48-gaeb2e425-1focal python3-rados=16.2.14-48-gaeb2e425-1focal python3-rgw=16.2.14-48-gaeb2e425-1focal python3-cephfs=16.2.14-48-gaeb2e425-1focal python3-rbd=16.2.14-48-gaeb2e425-1focal libcephfs2=16.2.14-48-gaeb2e425-1focal libcephfs-dev=16.2.14-48-gaeb2e425-1focal librados2=16.2.14-48-gaeb2e425-1focal librbd1=16.2.14-48-gaeb2e425-1focal rbd-fuse=16.2.14-48-gaeb2e425-1focal 2023-10-09T08:49:35.396 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2023-10-09T08:49:35.405 INFO:teuthology.orchestra.run.smithi163.stdout:Reading package lists... 2023-10-09T08:49:35.423 DEBUG:teuthology.orchestra.run.smithi181:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-48-gaeb2e425-1focal cephadm=16.2.14-48-gaeb2e425-1focal ceph-mds=16.2.14-48-gaeb2e425-1focal ceph-mgr=16.2.14-48-gaeb2e425-1focal ceph-common=16.2.14-48-gaeb2e425-1focal ceph-fuse=16.2.14-48-gaeb2e425-1focal ceph-test=16.2.14-48-gaeb2e425-1focal radosgw=16.2.14-48-gaeb2e425-1focal python3-rados=16.2.14-48-gaeb2e425-1focal python3-rgw=16.2.14-48-gaeb2e425-1focal python3-cephfs=16.2.14-48-gaeb2e425-1focal python3-rbd=16.2.14-48-gaeb2e425-1focal libcephfs2=16.2.14-48-gaeb2e425-1focal libcephfs-dev=16.2.14-48-gaeb2e425-1focal librados2=16.2.14-48-gaeb2e425-1focal librbd1=16.2.14-48-gaeb2e425-1focal rbd-fuse=16.2.14-48-gaeb2e425-1focal 2023-10-09T08:49:35.494 INFO:teuthology.orchestra.run.smithi181.stdout:Reading package lists... 2023-10-09T08:49:35.616 INFO:teuthology.orchestra.run.smithi163.stdout:Building dependency tree... 2023-10-09T08:49:35.616 INFO:teuthology.orchestra.run.smithi163.stdout:Reading state information... 2023-10-09T08:49:35.676 INFO:teuthology.orchestra.run.smithi181.stdout:Building dependency tree... 2023-10-09T08:49:35.676 INFO:teuthology.orchestra.run.smithi181.stdout:Reading state information... 2023-10-09T08:49:35.792 INFO:teuthology.orchestra.run.smithi163.stdout:The following packages were automatically installed and are no longer required: 2023-10-09T08:49:35.793 INFO:teuthology.orchestra.run.smithi163.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-09T08:49:35.793 INFO:teuthology.orchestra.run.smithi163.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-09T08:49:35.793 INFO:teuthology.orchestra.run.smithi163.stdout:The following additional packages will be installed: 2023-10-09T08:49:35.794 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-09T08:49:35.794 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-10-09T08:49:35.794 INFO:teuthology.orchestra.run.smithi163.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-09T08:49:35.795 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-09T08:49:35.796 INFO:teuthology.orchestra.run.smithi163.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-09T08:49:35.796 INFO:teuthology.orchestra.run.smithi163.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-09T08:49:35.796 INFO:teuthology.orchestra.run.smithi163.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-09T08:49:35.796 INFO:teuthology.orchestra.run.smithi163.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-09T08:49:35.796 INFO:teuthology.orchestra.run.smithi163.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-10-09T08:49:35.796 INFO:teuthology.orchestra.run.smithi163.stdout:Suggested packages: 2023-10-09T08:49:35.796 INFO:teuthology.orchestra.run.smithi163.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-09T08:49:35.796 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-09T08:49:35.797 INFO:teuthology.orchestra.run.smithi163.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-09T08:49:35.797 INFO:teuthology.orchestra.run.smithi163.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-09T08:49:35.797 INFO:teuthology.orchestra.run.smithi163.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-09T08:49:35.797 INFO:teuthology.orchestra.run.smithi163.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-09T08:49:35.797 INFO:teuthology.orchestra.run.smithi163.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-09T08:49:35.797 INFO:teuthology.orchestra.run.smithi163.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-09T08:49:35.797 INFO:teuthology.orchestra.run.smithi163.stdout:Recommended packages: 2023-10-09T08:49:35.797 INFO:teuthology.orchestra.run.smithi163.stdout: btrfs-tools 2023-10-09T08:49:35.846 INFO:teuthology.orchestra.run.smithi181.stdout:The following packages were automatically installed and are no longer required: 2023-10-09T08:49:35.847 INFO:teuthology.orchestra.run.smithi181.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-09T08:49:35.847 INFO:teuthology.orchestra.run.smithi181.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-09T08:49:35.847 INFO:teuthology.orchestra.run.smithi181.stdout:The following additional packages will be installed: 2023-10-09T08:49:35.847 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-09T08:49:35.848 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-10-09T08:49:35.848 INFO:teuthology.orchestra.run.smithi181.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-10-09T08:49:35.848 INFO:teuthology.orchestra.run.smithi181.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-10-09T08:49:35.848 INFO:teuthology.orchestra.run.smithi181.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-10-09T08:49:35.848 INFO:teuthology.orchestra.run.smithi181.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-10-09T08:49:35.848 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-09T08:49:35.849 INFO:teuthology.orchestra.run.smithi181.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-09T08:49:35.850 INFO:teuthology.orchestra.run.smithi181.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-10-09T08:49:35.850 INFO:teuthology.orchestra.run.smithi181.stdout:Suggested packages: 2023-10-09T08:49:35.850 INFO:teuthology.orchestra.run.smithi181.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-09T08:49:35.850 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-09T08:49:35.850 INFO:teuthology.orchestra.run.smithi181.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-09T08:49:35.850 INFO:teuthology.orchestra.run.smithi181.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-09T08:49:35.851 INFO:teuthology.orchestra.run.smithi181.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-09T08:49:35.851 INFO:teuthology.orchestra.run.smithi181.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-09T08:49:35.851 INFO:teuthology.orchestra.run.smithi181.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-09T08:49:35.851 INFO:teuthology.orchestra.run.smithi181.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-09T08:49:35.851 INFO:teuthology.orchestra.run.smithi181.stdout:Recommended packages: 2023-10-09T08:49:35.851 INFO:teuthology.orchestra.run.smithi181.stdout: btrfs-tools 2023-10-09T08:49:35.913 INFO:teuthology.orchestra.run.smithi163.stdout:The following NEW packages will be installed: 2023-10-09T08:49:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-09T08:49:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-09T08:49:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-10-09T08:49:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-10-09T08:49:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-10-09T08:49:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-10-09T08:49:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-09T08:49:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-09T08:49:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-09T08:49:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-09T08:49:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-09T08:49:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-10-09T08:49:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-10-09T08:49:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-10-09T08:49:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-10-09T08:49:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-10-09T08:49:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-10-09T08:49:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-10-09T08:49:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-10-09T08:49:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-10-09T08:49:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-10-09T08:49:35.917 INFO:teuthology.orchestra.run.smithi163.stdout:The following packages will be upgraded: 2023-10-09T08:49:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: librados2 librbd1 2023-10-09T08:49:35.967 INFO:teuthology.orchestra.run.smithi181.stdout:The following NEW packages will be installed: 2023-10-09T08:49:35.967 INFO:teuthology.orchestra.run.smithi181.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-09T08:49:35.967 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-09T08:49:35.968 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-10-09T08:49:35.968 INFO:teuthology.orchestra.run.smithi181.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-10-09T08:49:35.968 INFO:teuthology.orchestra.run.smithi181.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-10-09T08:49:35.969 INFO:teuthology.orchestra.run.smithi181.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-10-09T08:49:35.969 INFO:teuthology.orchestra.run.smithi181.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-09T08:49:35.969 INFO:teuthology.orchestra.run.smithi181.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-09T08:49:35.969 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-09T08:49:35.969 INFO:teuthology.orchestra.run.smithi181.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-09T08:49:35.969 INFO:teuthology.orchestra.run.smithi181.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-09T08:49:35.969 INFO:teuthology.orchestra.run.smithi181.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-10-09T08:49:35.969 INFO:teuthology.orchestra.run.smithi181.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-10-09T08:49:35.969 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-10-09T08:49:35.969 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-10-09T08:49:35.970 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-10-09T08:49:35.970 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-10-09T08:49:35.970 INFO:teuthology.orchestra.run.smithi181.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-10-09T08:49:35.970 INFO:teuthology.orchestra.run.smithi181.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-10-09T08:49:35.970 INFO:teuthology.orchestra.run.smithi181.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-10-09T08:49:35.970 INFO:teuthology.orchestra.run.smithi181.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-10-09T08:49:35.971 INFO:teuthology.orchestra.run.smithi181.stdout:The following packages will be upgraded: 2023-10-09T08:49:35.971 INFO:teuthology.orchestra.run.smithi181.stdout: librados2 librbd1 2023-10-09T08:49:35.985 INFO:teuthology.orchestra.run.smithi163.stdout:2 upgraded, 98 newly installed, 0 to remove and 256 not upgraded. 2023-10-09T08:49:35.985 INFO:teuthology.orchestra.run.smithi163.stdout:Need to get 174 MB of archives. 2023-10-09T08:49:35.985 INFO:teuthology.orchestra.run.smithi163.stdout:After this operation, 804 MB of additional disk space will be used. 2023-10-09T08:49:35.985 INFO:teuthology.orchestra.run.smithi163.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-09T08:49:36.081 INFO:teuthology.orchestra.run.smithi163.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-09T08:49:36.083 INFO:teuthology.orchestra.run.smithi163.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-09T08:49:36.093 INFO:teuthology.orchestra.run.smithi163.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-09T08:49:36.100 INFO:teuthology.orchestra.run.smithi163.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-09T08:49:36.102 INFO:teuthology.orchestra.run.smithi163.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-48-gaeb2e425-1focal [3,960 B] 2023-10-09T08:49:36.128 INFO:teuthology.orchestra.run.smithi163.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-10-09T08:49:36.133 INFO:teuthology.orchestra.run.smithi163.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-09T08:49:36.135 INFO:teuthology.orchestra.run.smithi163.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-09T08:49:36.142 INFO:teuthology.orchestra.run.smithi163.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-09T08:49:36.143 INFO:teuthology.orchestra.run.smithi163.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-09T08:49:36.144 INFO:teuthology.orchestra.run.smithi163.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-09T08:49:36.148 INFO:teuthology.orchestra.run.smithi163.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-09T08:49:36.149 INFO:teuthology.orchestra.run.smithi163.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-09T08:49:36.149 INFO:teuthology.orchestra.run.smithi163.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-09T08:49:36.159 INFO:teuthology.orchestra.run.smithi181.stdout:2 upgraded, 98 newly installed, 0 to remove and 256 not upgraded. 2023-10-09T08:49:36.159 INFO:teuthology.orchestra.run.smithi181.stdout:Need to get 174 MB of archives. 2023-10-09T08:49:36.159 INFO:teuthology.orchestra.run.smithi181.stdout:After this operation, 804 MB of additional disk space will be used. 2023-10-09T08:49:36.159 INFO:teuthology.orchestra.run.smithi181.stdout:Get:1 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-48-gaeb2e425-1focal [3,960 B] 2023-10-09T08:49:36.162 INFO:teuthology.orchestra.run.smithi163.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-09T08:49:36.163 INFO:teuthology.orchestra.run.smithi181.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-09T08:49:36.163 INFO:teuthology.orchestra.run.smithi163.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-09T08:49:36.163 INFO:teuthology.orchestra.run.smithi163.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-09T08:49:36.176 INFO:teuthology.orchestra.run.smithi163.stdout:Get:19 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-48-gaeb2e425-1focal [3,383 kB] 2023-10-09T08:49:36.181 INFO:teuthology.orchestra.run.smithi163.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-09T08:49:36.183 INFO:teuthology.orchestra.run.smithi163.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-09T08:49:36.185 INFO:teuthology.orchestra.run.smithi163.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-09T08:49:36.187 INFO:teuthology.orchestra.run.smithi163.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-09T08:49:36.189 INFO:teuthology.orchestra.run.smithi163.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-09T08:49:36.190 INFO:teuthology.orchestra.run.smithi163.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-09T08:49:36.194 INFO:teuthology.orchestra.run.smithi163.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-09T08:49:36.201 INFO:teuthology.orchestra.run.smithi163.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-09T08:49:36.201 INFO:teuthology.orchestra.run.smithi163.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-09T08:49:36.202 INFO:teuthology.orchestra.run.smithi163.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-09T08:49:36.221 INFO:teuthology.orchestra.run.smithi163.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-09T08:49:36.225 INFO:teuthology.orchestra.run.smithi163.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-09T08:49:36.228 INFO:teuthology.orchestra.run.smithi163.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-09T08:49:36.229 INFO:teuthology.orchestra.run.smithi163.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-09T08:49:36.231 INFO:teuthology.orchestra.run.smithi163.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-09T08:49:36.232 INFO:teuthology.orchestra.run.smithi163.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-09T08:49:36.232 INFO:teuthology.orchestra.run.smithi181.stdout:Get:3 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-48-gaeb2e425-1focal [3,383 kB] 2023-10-09T08:49:36.233 INFO:teuthology.orchestra.run.smithi163.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-09T08:49:36.240 INFO:teuthology.orchestra.run.smithi163.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-09T08:49:36.241 INFO:teuthology.orchestra.run.smithi163.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-09T08:49:36.389 INFO:teuthology.orchestra.run.smithi163.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-09T08:49:36.427 INFO:teuthology.orchestra.run.smithi163.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-09T08:49:36.431 INFO:teuthology.orchestra.run.smithi163.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-09T08:49:36.446 INFO:teuthology.orchestra.run.smithi163.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-09T08:49:36.446 INFO:teuthology.orchestra.run.smithi163.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-09T08:49:36.447 INFO:teuthology.orchestra.run.smithi163.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-09T08:49:36.447 INFO:teuthology.orchestra.run.smithi163.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-09T08:49:36.447 INFO:teuthology.orchestra.run.smithi163.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-09T08:49:36.447 INFO:teuthology.orchestra.run.smithi163.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-09T08:49:36.448 INFO:teuthology.orchestra.run.smithi163.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-09T08:49:36.449 INFO:teuthology.orchestra.run.smithi163.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-09T08:49:36.468 INFO:teuthology.orchestra.run.smithi163.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-09T08:49:36.469 INFO:teuthology.orchestra.run.smithi163.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-09T08:49:36.475 INFO:teuthology.orchestra.run.smithi163.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-09T08:49:36.478 INFO:teuthology.orchestra.run.smithi163.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-09T08:49:36.480 INFO:teuthology.orchestra.run.smithi163.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-09T08:49:36.480 INFO:teuthology.orchestra.run.smithi163.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-09T08:49:36.483 INFO:teuthology.orchestra.run.smithi163.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-09T08:49:36.488 INFO:teuthology.orchestra.run.smithi163.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-09T08:49:36.491 INFO:teuthology.orchestra.run.smithi163.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-09T08:49:36.491 INFO:teuthology.orchestra.run.smithi163.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-09T08:49:36.508 INFO:teuthology.orchestra.run.smithi163.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-09T08:49:36.512 INFO:teuthology.orchestra.run.smithi163.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-09T08:49:36.543 INFO:teuthology.orchestra.run.smithi163.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-09T08:49:36.560 INFO:teuthology.orchestra.run.smithi181.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-09T08:49:36.561 INFO:teuthology.orchestra.run.smithi163.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-09T08:49:36.561 INFO:teuthology.orchestra.run.smithi163.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-09T08:49:36.561 INFO:teuthology.orchestra.run.smithi163.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-09T08:49:36.561 INFO:teuthology.orchestra.run.smithi163.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-09T08:49:36.562 INFO:teuthology.orchestra.run.smithi163.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-09T08:49:36.563 INFO:teuthology.orchestra.run.smithi163.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-09T08:49:36.564 INFO:teuthology.orchestra.run.smithi163.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-09T08:49:36.564 INFO:teuthology.orchestra.run.smithi163.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-09T08:49:36.567 INFO:teuthology.orchestra.run.smithi181.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-09T08:49:36.570 INFO:teuthology.orchestra.run.smithi163.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-09T08:49:36.584 INFO:teuthology.orchestra.run.smithi163.stdout:Get:72 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-48-gaeb2e425-1focal [3,391 kB] 2023-10-09T08:49:36.612 INFO:teuthology.orchestra.run.smithi181.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-09T08:49:36.625 INFO:teuthology.orchestra.run.smithi181.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-09T08:49:36.752 INFO:teuthology.orchestra.run.smithi181.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-10-09T08:49:36.774 INFO:teuthology.orchestra.run.smithi181.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-09T08:49:36.778 INFO:teuthology.orchestra.run.smithi163.stdout:Get:73 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-48-gaeb2e425-1focal [730 kB] 2023-10-09T08:49:36.781 INFO:teuthology.orchestra.run.smithi181.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-09T08:49:36.812 INFO:teuthology.orchestra.run.smithi181.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-09T08:49:36.814 INFO:teuthology.orchestra.run.smithi163.stdout:Get:74 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-48-gaeb2e425-1focal [413 kB] 2023-10-09T08:49:36.817 INFO:teuthology.orchestra.run.smithi181.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-09T08:49:36.820 INFO:teuthology.orchestra.run.smithi181.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-09T08:49:36.845 INFO:teuthology.orchestra.run.smithi163.stdout:Get:75 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-48-gaeb2e425-1focal [22.2 kB] 2023-10-09T08:49:36.846 INFO:teuthology.orchestra.run.smithi163.stdout:Get:76 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-48-gaeb2e425-1focal [212 kB] 2023-10-09T08:49:36.850 INFO:teuthology.orchestra.run.smithi163.stdout:Get:77 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-48-gaeb2e425-1focal [38.4 kB] 2023-10-09T08:49:36.850 INFO:teuthology.orchestra.run.smithi163.stdout:Get:78 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-48-gaeb2e425-1focal [386 kB] 2023-10-09T08:49:36.877 INFO:teuthology.orchestra.run.smithi163.stdout:Get:79 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-48-gaeb2e425-1focal [3,689 kB] 2023-10-09T08:49:36.895 INFO:teuthology.orchestra.run.smithi181.stdout:Get:14 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-48-gaeb2e425-1focal [3,391 kB] 2023-10-09T08:49:36.999 INFO:teuthology.orchestra.run.smithi181.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-09T08:49:37.038 INFO:teuthology.orchestra.run.smithi163.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-48-gaeb2e425-1focal [116 kB] 2023-10-09T08:49:37.040 INFO:teuthology.orchestra.run.smithi163.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-48-gaeb2e425-1focal [448 kB] 2023-10-09T08:49:37.064 INFO:teuthology.orchestra.run.smithi163.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-48-gaeb2e425-1focal [22.6 MB] 2023-10-09T08:49:37.167 INFO:teuthology.orchestra.run.smithi181.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-09T08:49:37.197 INFO:teuthology.orchestra.run.smithi181.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-09T08:49:37.221 INFO:teuthology.orchestra.run.smithi181.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-09T08:49:37.287 INFO:teuthology.orchestra.run.smithi181.stdout:Get:19 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-48-gaeb2e425-1focal [730 kB] 2023-10-09T08:49:37.337 INFO:teuthology.orchestra.run.smithi181.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-09T08:49:37.350 INFO:teuthology.orchestra.run.smithi181.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-09T08:49:37.355 INFO:teuthology.orchestra.run.smithi181.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-09T08:49:37.393 INFO:teuthology.orchestra.run.smithi181.stdout:Get:23 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-48-gaeb2e425-1focal [413 kB] 2023-10-09T08:49:37.421 INFO:teuthology.orchestra.run.smithi181.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-09T08:49:37.437 INFO:teuthology.orchestra.run.smithi181.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-09T08:49:37.451 INFO:teuthology.orchestra.run.smithi181.stdout:Get:26 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-48-gaeb2e425-1focal [22.2 kB] 2023-10-09T08:49:37.452 INFO:teuthology.orchestra.run.smithi181.stdout:Get:27 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-48-gaeb2e425-1focal [212 kB] 2023-10-09T08:49:37.457 INFO:teuthology.orchestra.run.smithi181.stdout:Get:28 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-48-gaeb2e425-1focal [38.4 kB] 2023-10-09T08:49:37.458 INFO:teuthology.orchestra.run.smithi181.stdout:Get:29 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-48-gaeb2e425-1focal [386 kB] 2023-10-09T08:49:37.472 INFO:teuthology.orchestra.run.smithi181.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-09T08:49:37.499 INFO:teuthology.orchestra.run.smithi181.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-09T08:49:37.517 INFO:teuthology.orchestra.run.smithi181.stdout:Get:32 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-48-gaeb2e425-1focal [3,689 kB] 2023-10-09T08:49:37.667 INFO:teuthology.orchestra.run.smithi181.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-09T08:49:37.969 INFO:teuthology.orchestra.run.smithi181.stdout:Get:34 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-48-gaeb2e425-1focal [116 kB] 2023-10-09T08:49:37.972 INFO:teuthology.orchestra.run.smithi181.stdout:Get:35 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-48-gaeb2e425-1focal [448 kB] 2023-10-09T08:49:38.052 INFO:teuthology.orchestra.run.smithi181.stdout:Get:36 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-48-gaeb2e425-1focal [22.6 MB] 2023-10-09T08:49:38.194 INFO:teuthology.orchestra.run.smithi181.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-09T08:49:38.195 INFO:teuthology.orchestra.run.smithi181.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-09T08:49:38.195 INFO:teuthology.orchestra.run.smithi163.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-48-gaeb2e425-1focal [5,916 kB] 2023-10-09T08:49:38.198 INFO:teuthology.orchestra.run.smithi181.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-09T08:49:38.201 INFO:teuthology.orchestra.run.smithi181.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-09T08:49:38.213 INFO:teuthology.orchestra.run.smithi181.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-09T08:49:38.236 INFO:teuthology.orchestra.run.smithi181.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-09T08:49:38.248 INFO:teuthology.orchestra.run.smithi181.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-09T08:49:38.250 INFO:teuthology.orchestra.run.smithi181.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-09T08:49:38.285 INFO:teuthology.orchestra.run.smithi181.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-09T08:49:38.297 INFO:teuthology.orchestra.run.smithi181.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-09T08:49:38.377 INFO:teuthology.orchestra.run.smithi163.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-48-gaeb2e425-1focal [204 kB] 2023-10-09T08:49:38.382 INFO:teuthology.orchestra.run.smithi163.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-48-gaeb2e425-1focal [126 kB] 2023-10-09T08:49:38.399 INFO:teuthology.orchestra.run.smithi163.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-48-gaeb2e425-1focal [1,536 kB] 2023-10-09T08:49:38.456 INFO:teuthology.orchestra.run.smithi163.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-48-gaeb2e425-1focal [6,993 kB] 2023-10-09T08:49:38.473 INFO:teuthology.orchestra.run.smithi181.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-09T08:49:38.557 INFO:teuthology.orchestra.run.smithi181.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-09T08:49:38.641 INFO:teuthology.orchestra.run.smithi181.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-09T08:49:38.772 INFO:teuthology.orchestra.run.smithi163.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-48-gaeb2e425-1focal [26.2 MB] 2023-10-09T08:49:39.399 INFO:teuthology.orchestra.run.smithi181.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-09T08:49:39.476 INFO:teuthology.orchestra.run.smithi181.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-09T08:49:39.478 INFO:teuthology.orchestra.run.smithi181.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-09T08:49:39.493 INFO:teuthology.orchestra.run.smithi181.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-09T08:49:39.493 INFO:teuthology.orchestra.run.smithi181.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-09T08:49:39.494 INFO:teuthology.orchestra.run.smithi181.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-09T08:49:39.494 INFO:teuthology.orchestra.run.smithi181.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-09T08:49:39.494 INFO:teuthology.orchestra.run.smithi181.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-09T08:49:39.658 INFO:teuthology.orchestra.run.smithi181.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-09T08:49:39.741 INFO:teuthology.orchestra.run.smithi181.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-09T08:49:39.920 INFO:teuthology.orchestra.run.smithi181.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-09T08:49:39.936 INFO:teuthology.orchestra.run.smithi181.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-09T08:49:39.968 INFO:teuthology.orchestra.run.smithi181.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-09T08:49:40.098 INFO:teuthology.orchestra.run.smithi181.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-09T08:49:40.134 INFO:teuthology.orchestra.run.smithi181.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-09T08:49:40.157 INFO:teuthology.orchestra.run.smithi181.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-09T08:49:40.165 INFO:teuthology.orchestra.run.smithi181.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-09T08:49:40.208 INFO:teuthology.orchestra.run.smithi181.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-09T08:49:40.235 INFO:teuthology.orchestra.run.smithi181.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-09T08:49:40.421 INFO:teuthology.orchestra.run.smithi181.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-09T08:49:40.451 INFO:teuthology.orchestra.run.smithi163.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-48-gaeb2e425-1focal [4,056 B] 2023-10-09T08:49:40.452 INFO:teuthology.orchestra.run.smithi163.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-48-gaeb2e425-1focal [834 kB] 2023-10-09T08:49:40.457 INFO:teuthology.orchestra.run.smithi163.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-48-gaeb2e425-1focal [2,364 kB] 2023-10-09T08:49:40.505 INFO:teuthology.orchestra.run.smithi181.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-09T08:49:40.606 INFO:teuthology.orchestra.run.smithi181.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-09T08:49:40.629 INFO:teuthology.orchestra.run.smithi181.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-09T08:49:40.662 INFO:teuthology.orchestra.run.smithi163.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-48-gaeb2e425-1focal [76.4 kB] 2023-10-09T08:49:40.665 INFO:teuthology.orchestra.run.smithi163.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-48-gaeb2e425-1focal [100 kB] 2023-10-09T08:49:40.669 INFO:teuthology.orchestra.run.smithi163.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-48-gaeb2e425-1focal [2,823 kB] 2023-10-09T08:49:40.745 INFO:teuthology.orchestra.run.smithi181.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-09T08:49:40.968 INFO:teuthology.orchestra.run.smithi163.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-48-gaeb2e425-1focal [7,865 kB] 2023-10-09T08:49:41.031 INFO:teuthology.orchestra.run.smithi181.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-09T08:49:41.032 INFO:teuthology.orchestra.run.smithi181.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-09T08:49:41.037 INFO:teuthology.orchestra.run.smithi181.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-09T08:49:41.038 INFO:teuthology.orchestra.run.smithi181.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-09T08:49:41.048 INFO:teuthology.orchestra.run.smithi181.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-09T08:49:41.052 INFO:teuthology.orchestra.run.smithi181.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-09T08:49:41.217 INFO:teuthology.orchestra.run.smithi181.stdout:Get:80 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-09T08:49:41.383 INFO:teuthology.orchestra.run.smithi181.stdout:Get:81 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-09T08:49:41.635 INFO:teuthology.orchestra.run.smithi181.stdout:Get:82 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-09T08:49:41.762 INFO:teuthology.orchestra.run.smithi181.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-48-gaeb2e425-1focal [5,916 kB] 2023-10-09T08:49:41.942 INFO:teuthology.orchestra.run.smithi163.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-48-gaeb2e425-1focal [4,172 B] 2023-10-09T08:49:41.943 INFO:teuthology.orchestra.run.smithi163.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-48-gaeb2e425-1focal [49.3 MB] 2023-10-09T08:49:43.223 INFO:teuthology.orchestra.run.smithi181.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-48-gaeb2e425-1focal [204 kB] 2023-10-09T08:49:43.262 INFO:teuthology.orchestra.run.smithi181.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-48-gaeb2e425-1focal [126 kB] 2023-10-09T08:49:43.294 INFO:teuthology.orchestra.run.smithi181.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-48-gaeb2e425-1focal [1,536 kB] 2023-10-09T08:49:43.618 INFO:teuthology.orchestra.run.smithi181.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-48-gaeb2e425-1focal [6,993 kB] 2023-10-09T08:49:44.975 INFO:teuthology.orchestra.run.smithi181.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-48-gaeb2e425-1focal [26.2 MB] 2023-10-09T08:49:46.777 INFO:teuthology.orchestra.run.smithi163.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-48-gaeb2e425-1focal [21.5 kB] 2023-10-09T08:49:46.778 INFO:teuthology.orchestra.run.smithi163.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-48-gaeb2e425-1focal [11.3 MB] 2023-10-09T08:49:48.057 INFO:teuthology.orchestra.run.smithi163.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-48-gaeb2e425-1focal [78.4 kB] 2023-10-09T08:49:48.703 INFO:teuthology.orchestra.run.smithi163.stdout:Fetched 174 MB in 12s (14.3 MB/s) 2023-10-09T08:49:48.792 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libjaeger. 2023-10-09T08:49:48.831 INFO:teuthology.orchestra.run.smithi163.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-10-09T08:49:48.835 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../00-libjaeger_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:48.874 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:49.193 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-09T08:49:49.202 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-09T08:49:49.285 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T08:49:49.639 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../02-librbd1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:49.764 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking librbd1 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T08:49:49.797 INFO:teuthology.orchestra.run.smithi181.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-48-gaeb2e425-1focal [4,056 B] 2023-10-09T08:49:49.797 INFO:teuthology.orchestra.run.smithi181.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-48-gaeb2e425-1focal [834 kB] 2023-10-09T08:49:49.833 INFO:teuthology.orchestra.run.smithi181.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-48-gaeb2e425-1focal [2,364 kB] 2023-10-09T08:49:50.140 INFO:teuthology.orchestra.run.smithi181.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-48-gaeb2e425-1focal [76.4 kB] 2023-10-09T08:49:50.142 INFO:teuthology.orchestra.run.smithi181.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-48-gaeb2e425-1focal [100 kB] 2023-10-09T08:49:50.147 INFO:teuthology.orchestra.run.smithi181.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-48-gaeb2e425-1focal [2,823 kB] 2023-10-09T08:49:50.424 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../03-librados2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:50.553 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking librados2 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T08:49:50.602 INFO:teuthology.orchestra.run.smithi181.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-48-gaeb2e425-1focal [7,865 kB] 2023-10-09T08:49:51.200 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libcephfs2. 2023-10-09T08:49:51.215 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:51.258 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:51.578 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-rados. 2023-10-09T08:49:51.592 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../05-python3-rados_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:51.637 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:51.923 INFO:teuthology.orchestra.run.smithi181.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-48-gaeb2e425-1focal [4,172 B] 2023-10-09T08:49:51.927 INFO:teuthology.orchestra.run.smithi181.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-48-gaeb2e425-1focal [49.3 MB] 2023-10-09T08:49:51.983 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-09T08:49:51.996 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:49:52.041 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:52.342 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-cephfs. 2023-10-09T08:49:52.355 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:52.393 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:52.828 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-09T08:49:52.843 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:49:52.887 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:53.207 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-prettytable. 2023-10-09T08:49:53.223 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-10-09T08:49:53.266 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-09T08:49:53.518 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-rbd. 2023-10-09T08:49:53.531 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:53.568 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:53.905 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-09T08:49:53.920 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-09T08:49:53.963 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T08:49:54.350 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-09T08:49:54.363 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-09T08:49:54.408 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T08:49:54.745 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-09T08:49:54.758 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-09T08:49:54.803 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T08:49:55.140 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package librgw2. 2023-10-09T08:49:55.155 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../14-librgw2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:55.198 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:55.768 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-rgw. 2023-10-09T08:49:55.783 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:55.826 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:56.179 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-10-09T08:49:56.192 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-10-09T08:49:56.222 INFO:teuthology.orchestra.run.smithi181.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-48-gaeb2e425-1focal [21.5 kB] 2023-10-09T08:49:56.222 INFO:teuthology.orchestra.run.smithi181.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-48-gaeb2e425-1focal [11.3 MB] 2023-10-09T08:49:56.237 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T08:49:56.590 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package liboath0. 2023-10-09T08:49:56.604 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-09T08:49:56.649 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-09T08:49:56.660 INFO:teuthology.orchestra.run.smithi181.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-48-gaeb2e425-1focal [78.4 kB] 2023-10-09T08:49:56.952 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libradosstriper1. 2023-10-09T08:49:56.967 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:57.010 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:57.322 INFO:teuthology.orchestra.run.smithi181.stdout:Fetched 174 MB in 21s (8,388 kB/s) 2023-10-09T08:49:57.402 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libjaeger. 2023-10-09T08:49:57.442 INFO:teuthology.orchestra.run.smithi181.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-10-09T08:49:57.446 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../00-libjaeger_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:57.447 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-common. 2023-10-09T08:49:57.460 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../19-ceph-common_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:57.485 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:57.505 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:57.804 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-09T08:49:57.812 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-09T08:49:57.896 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T08:49:58.239 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../02-librbd1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:58.341 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking librbd1 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T08:49:58.999 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../03-librados2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:59.130 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking librados2 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T08:49:59.680 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-base. 2023-10-09T08:49:59.692 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../20-ceph-base_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:59.738 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:49:59.786 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libcephfs2. 2023-10-09T08:49:59.799 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:49:59.836 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:00.152 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-rados. 2023-10-09T08:50:00.165 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../05-python3-rados_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:00.210 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:00.554 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-09T08:50:00.570 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:00.612 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:00.668 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-cryptography. 2023-10-09T08:50:00.680 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-09T08:50:00.709 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T08:50:00.908 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-cephfs. 2023-10-09T08:50:00.922 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:00.965 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:01.046 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-openssl. 2023-10-09T08:50:01.054 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-09T08:50:01.088 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-09T08:50:01.302 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-09T08:50:01.315 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:01.352 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:01.357 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-09T08:50:01.372 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:01.416 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:01.646 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-prettytable. 2023-10-09T08:50:01.660 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-10-09T08:50:01.698 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-09T08:50:01.788 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-09T08:50:01.801 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-09T08:50:01.846 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T08:50:01.916 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-rbd. 2023-10-09T08:50:01.930 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:01.965 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:02.149 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-09T08:50:02.162 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-09T08:50:02.215 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-09T08:50:02.319 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-09T08:50:02.332 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-09T08:50:02.377 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T08:50:02.545 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-jwt. 2023-10-09T08:50:02.558 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-09T08:50:02.603 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T08:50:02.730 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-09T08:50:02.743 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-09T08:50:02.780 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T08:50:02.915 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-logutils. 2023-10-09T08:50:02.928 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-10-09T08:50:02.965 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-09T08:50:03.108 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-09T08:50:03.121 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-09T08:50:03.166 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T08:50:03.293 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-09T08:50:03.306 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-09T08:50:03.343 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-09T08:50:03.519 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package librgw2. 2023-10-09T08:50:03.534 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../14-librgw2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:03.577 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:03.646 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-mako. 2023-10-09T08:50:03.659 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-09T08:50:03.704 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T08:50:04.041 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-09T08:50:04.055 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-09T08:50:04.116 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-09T08:50:04.163 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-rgw. 2023-10-09T08:50:04.178 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:04.221 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:04.419 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-09T08:50:04.432 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-09T08:50:04.477 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T08:50:04.490 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-10-09T08:50:04.506 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-10-09T08:50:04.540 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T08:50:04.764 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-webob. 2023-10-09T08:50:04.779 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-09T08:50:04.822 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-09T08:50:04.902 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package liboath0. 2023-10-09T08:50:04.916 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-09T08:50:04.960 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-09T08:50:05.150 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-09T08:50:05.164 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-09T08:50:05.208 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T08:50:05.256 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libradosstriper1. 2023-10-09T08:50:05.271 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:05.314 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:05.511 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-bs4. 2023-10-09T08:50:05.529 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-10-09T08:50:05.570 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-09T08:50:05.692 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-common. 2023-10-09T08:50:05.705 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../19-ceph-common_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:05.750 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:05.915 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-waitress. 2023-10-09T08:50:05.928 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-09T08:50:05.968 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T08:50:06.277 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-tempita. 2023-10-09T08:50:06.292 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-10-09T08:50:06.335 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-09T08:50:06.604 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-paste. 2023-10-09T08:50:06.613 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-09T08:50:06.654 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T08:50:06.975 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-09T08:50:06.983 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-09T08:50:07.025 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T08:50:07.319 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-09T08:50:07.333 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-09T08:50:07.378 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-09T08:50:07.681 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-webtest. 2023-10-09T08:50:07.694 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-10-09T08:50:07.741 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-09T08:50:08.051 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-pecan. 2023-10-09T08:50:08.064 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-09T08:50:08.109 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T08:50:08.398 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-base. 2023-10-09T08:50:08.411 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../20-ceph-base_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:08.465 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:09.356 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-09T08:50:09.370 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-09T08:50:09.411 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-cryptography. 2023-10-09T08:50:09.414 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T08:50:09.426 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-09T08:50:09.469 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T08:50:09.776 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-09T08:50:09.790 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:09.835 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:09.889 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-openssl. 2023-10-09T08:50:09.903 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-09T08:50:09.947 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-09T08:50:10.179 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-mgr. 2023-10-09T08:50:10.193 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:10.208 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-09T08:50:10.221 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:10.237 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:10.266 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:10.630 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-09T08:50:10.645 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-09T08:50:10.688 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T08:50:10.752 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-mon. 2023-10-09T08:50:10.752 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:10.791 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:10.991 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-09T08:50:11.004 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-09T08:50:11.049 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-09T08:50:11.388 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-jwt. 2023-10-09T08:50:11.403 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-09T08:50:11.447 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T08:50:11.758 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-logutils. 2023-10-09T08:50:11.771 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-10-09T08:50:11.808 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-09T08:50:11.845 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-osd. 2023-10-09T08:50:11.859 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:11.903 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:12.111 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-09T08:50:12.126 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-09T08:50:12.169 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-09T08:50:12.472 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-mako. 2023-10-09T08:50:12.485 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-09T08:50:12.522 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T08:50:12.825 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-09T08:50:12.839 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-09T08:50:12.883 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-09T08:50:13.186 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-09T08:50:13.199 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-09T08:50:13.244 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T08:50:13.555 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-webob. 2023-10-09T08:50:13.569 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-09T08:50:13.613 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-09T08:50:13.925 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-09T08:50:13.939 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-09T08:50:13.983 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T08:50:14.285 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-bs4. 2023-10-09T08:50:14.300 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-10-09T08:50:14.344 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-09T08:50:14.526 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph. 2023-10-09T08:50:14.539 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../47-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:14.584 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:14.681 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-waitress. 2023-10-09T08:50:14.695 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-09T08:50:14.742 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T08:50:14.904 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-fuse. 2023-10-09T08:50:14.919 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:14.979 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:15.050 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-tempita. 2023-10-09T08:50:15.063 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-10-09T08:50:15.100 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-09T08:50:15.390 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-mds. 2023-10-09T08:50:15.403 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-paste. 2023-10-09T08:50:15.403 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:15.418 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-09T08:50:15.440 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:15.461 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T08:50:15.781 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-09T08:50:15.794 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-09T08:50:15.839 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T08:50:15.961 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package cephadm. 2023-10-09T08:50:15.974 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../50-cephadm_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:16.019 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:16.142 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-09T08:50:16.156 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-09T08:50:16.200 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-09T08:50:16.330 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-jinja2. 2023-10-09T08:50:16.344 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-10-09T08:50:16.405 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-09T08:50:16.503 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-webtest. 2023-10-09T08:50:16.517 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-10-09T08:50:16.561 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-09T08:50:16.717 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-09T08:50:16.730 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:16.775 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:16.856 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pecan. 2023-10-09T08:50:16.871 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-09T08:50:16.906 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T08:50:17.128 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-09T08:50:17.142 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-10-09T08:50:17.186 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-09T08:50:17.269 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-09T08:50:17.283 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-09T08:50:17.327 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T08:50:17.423 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libjs-underscore. 2023-10-09T08:50:17.437 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-09T08:50:17.481 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T08:50:17.706 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-09T08:50:17.720 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:17.750 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-09T08:50:17.764 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-09T08:50:17.764 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:17.809 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T08:50:18.100 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr. 2023-10-09T08:50:18.111 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-routes. 2023-10-09T08:50:18.115 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:18.125 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-10-09T08:50:18.158 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:18.169 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-09T08:50:18.481 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-09T08:50:18.496 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:18.539 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:18.645 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mon. 2023-10-09T08:50:18.659 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:18.703 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:19.599 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-decorator. 2023-10-09T08:50:19.613 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-09T08:50:19.657 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T08:50:19.790 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-osd. 2023-10-09T08:50:19.803 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:19.848 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:19.969 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-09T08:50:19.982 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-09T08:50:20.027 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T08:50:20.346 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-scipy. 2023-10-09T08:50:20.361 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-09T08:50:20.404 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-09T08:50:21.681 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-09T08:50:21.695 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-09T08:50:21.739 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T08:50:22.260 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-joblib. 2023-10-09T08:50:22.276 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-10-09T08:50:22.318 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-09T08:50:22.395 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph. 2023-10-09T08:50:22.408 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../47-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:22.445 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:22.680 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-sklearn. 2023-10-09T08:50:22.694 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-09T08:50:22.738 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T08:50:22.764 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-fuse. 2023-10-09T08:50:22.779 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:22.822 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:23.235 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mds. 2023-10-09T08:50:23.247 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:23.447 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-09T08:50:23.461 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:23.505 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:23.580 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:24.096 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package cephadm. 2023-10-09T08:50:24.111 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../50-cephadm_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:24.146 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:24.449 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-jinja2. 2023-10-09T08:50:24.463 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-10-09T08:50:24.507 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-09T08:50:24.794 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-cachetools. 2023-10-09T08:50:24.806 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-10-09T08:50:24.818 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-09T08:50:24.834 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:24.850 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-09T08:50:24.876 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:25.154 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-09T08:50:25.168 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-09T08:50:25.212 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T08:50:25.221 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-09T08:50:25.234 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-10-09T08:50:25.271 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-09T08:50:25.524 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libjs-underscore. 2023-10-09T08:50:25.537 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-09T08:50:25.540 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-09T08:50:25.554 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-09T08:50:25.582 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T08:50:25.598 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T08:50:25.851 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-09T08:50:25.864 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-09T08:50:25.902 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-rsa. 2023-10-09T08:50:25.909 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T08:50:25.915 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-10-09T08:50:25.960 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-09T08:50:26.212 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-routes. 2023-10-09T08:50:26.225 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-10-09T08:50:26.270 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-09T08:50:26.272 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-google-auth. 2023-10-09T08:50:26.288 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-10-09T08:50:26.330 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-09T08:50:26.582 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-09T08:50:26.595 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:26.633 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-blinker. 2023-10-09T08:50:26.639 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:26.648 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-09T08:50:26.691 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T08:50:26.994 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-09T08:50:27.008 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-09T08:50:27.052 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T08:50:27.381 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-09T08:50:27.395 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-09T08:50:27.439 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T08:50:27.606 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-decorator. 2023-10-09T08:50:27.620 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-09T08:50:27.665 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T08:50:27.736 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-websocket. 2023-10-09T08:50:27.745 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-09T08:50:27.786 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T08:50:27.967 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-09T08:50:27.982 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-09T08:50:28.024 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T08:50:28.097 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-09T08:50:28.106 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-09T08:50:28.147 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T08:50:28.319 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-scipy. 2023-10-09T08:50:28.333 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-09T08:50:28.377 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-09T08:50:28.720 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-09T08:50:28.733 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:28.777 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:29.089 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-09T08:50:29.102 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-10-09T08:50:29.147 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-09T08:50:29.653 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-09T08:50:29.668 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-09T08:50:29.711 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T08:50:29.742 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-09T08:50:29.756 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T08:50:29.800 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:50:30.036 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package jq. 2023-10-09T08:50:30.050 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T08:50:30.094 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:50:30.240 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-joblib. 2023-10-09T08:50:30.253 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-10-09T08:50:30.297 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-09T08:50:30.347 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package socat. 2023-10-09T08:50:30.361 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-10-09T08:50:30.405 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-09T08:50:30.643 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-sklearn. 2023-10-09T08:50:30.658 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-09T08:50:30.700 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T08:50:30.718 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package xmlstarlet. 2023-10-09T08:50:30.732 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-09T08:50:30.776 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-09T08:50:31.200 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package ceph-test. 2023-10-09T08:50:31.216 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../81-ceph-test_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:31.258 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:31.392 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-09T08:50:31.407 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:31.450 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:32.562 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-cachetools. 2023-10-09T08:50:32.576 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-10-09T08:50:32.619 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-09T08:50:32.948 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-09T08:50:32.963 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-09T08:50:33.006 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T08:50:33.334 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-09T08:50:33.348 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-09T08:50:33.392 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T08:50:33.703 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-rsa. 2023-10-09T08:50:33.717 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-10-09T08:50:33.761 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-09T08:50:34.090 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-google-auth. 2023-10-09T08:50:34.103 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-10-09T08:50:34.148 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-09T08:50:34.467 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-blinker. 2023-10-09T08:50:34.481 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-09T08:50:34.525 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T08:50:34.828 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-09T08:50:34.843 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-09T08:50:34.886 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T08:50:35.206 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-09T08:50:35.216 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-09T08:50:35.256 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T08:50:35.471 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-09T08:50:35.485 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:35.529 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:35.567 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-websocket. 2023-10-09T08:50:35.581 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-09T08:50:35.626 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T08:50:35.916 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package nvme-cli. 2023-10-09T08:50:35.932 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-09T08:50:35.937 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-09T08:50:35.952 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-09T08:50:35.990 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T08:50:35.995 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T08:50:36.349 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-09T08:50:36.362 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-09T08:50:36.407 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T08:50:36.608 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-09T08:50:36.623 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T08:50:36.666 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:36.714 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-attr. 2023-10-09T08:50:36.728 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-10-09T08:50:36.772 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-09T08:50:36.986 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-09T08:50:36.999 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-10-09T08:50:37.044 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-09T08:50:37.109 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-webencodings. 2023-10-09T08:50:37.124 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-09T08:50:37.167 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T08:50:37.386 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-09T08:50:37.399 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T08:50:37.444 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:50:37.470 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-html5lib. 2023-10-09T08:50:37.485 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-10-09T08:50:37.536 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-09T08:50:37.722 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package jq. 2023-10-09T08:50:37.736 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T08:50:37.780 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:50:37.873 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-09T08:50:37.888 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-09T08:50:37.931 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T08:50:38.033 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package socat. 2023-10-09T08:50:38.047 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-10-09T08:50:38.091 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-09T08:50:38.376 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-packaging. 2023-10-09T08:50:38.392 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-10-09T08:50:38.402 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package xmlstarlet. 2023-10-09T08:50:38.416 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-09T08:50:38.443 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-09T08:50:38.460 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-09T08:50:38.746 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-pastescript. 2023-10-09T08:50:38.760 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-10-09T08:50:38.804 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-09T08:50:38.875 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package ceph-test. 2023-10-09T08:50:38.891 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../81-ceph-test_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:38.933 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:39.134 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-pluggy. 2023-10-09T08:50:39.149 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-10-09T08:50:39.191 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-09T08:50:39.503 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-psutil. 2023-10-09T08:50:39.517 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-09T08:50:39.562 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T08:50:39.890 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-py. 2023-10-09T08:50:39.903 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-09T08:50:39.948 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T08:50:40.251 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-09T08:50:40.265 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-09T08:50:40.309 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T08:50:40.629 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-09T08:50:40.643 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-09T08:50:40.679 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T08:50:41.007 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-pytest. 2023-10-09T08:50:41.021 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-10-09T08:50:41.065 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-09T08:50:41.385 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package python3-simplejson. 2023-10-09T08:50:41.394 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-09T08:50:41.435 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T08:50:41.865 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package radosgw. 2023-10-09T08:50:41.880 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../98-radosgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:41.922 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:43.079 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-09T08:50:43.084 INFO:teuthology.orchestra.run.smithi163.stdout:Selecting previously unselected package rbd-fuse. 2023-10-09T08:50:43.093 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:43.098 INFO:teuthology.orchestra.run.smithi163.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:43.137 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:43.142 INFO:teuthology.orchestra.run.smithi163.stdout:Unpacking rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:43.433 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-09T08:50:43.466 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package nvme-cli. 2023-10-09T08:50:43.480 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-09T08:50:43.538 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T08:50:43.722 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T08:50:43.849 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T08:50:43.918 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-09T08:50:43.932 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-09T08:50:43.975 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T08:50:43.976 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T08:50:44.184 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:44.311 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-attr. 2023-10-09T08:50:44.325 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-10-09T08:50:44.369 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-09T08:50:44.665 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-webencodings. 2023-10-09T08:50:44.679 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-09T08:50:44.722 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T08:50:44.789 INFO:teuthology.orchestra.run.smithi163.stdout:Adding system user cephadm....done 2023-10-09T08:50:44.942 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T08:50:44.999 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-html5lib. 2023-10-09T08:50:45.015 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-10-09T08:50:45.057 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-09T08:50:45.277 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-09T08:50:45.402 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-09T08:50:45.417 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-09T08:50:45.461 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T08:50:45.554 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-09T08:50:45.681 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T08:50:45.914 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-packaging. 2023-10-09T08:50:45.923 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-10-09T08:50:45.966 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-09T08:50:45.998 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-09T08:50:46.252 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pastescript. 2023-10-09T08:50:46.266 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-10-09T08:50:46.310 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-09T08:50:46.392 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T08:50:46.639 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pluggy. 2023-10-09T08:50:46.655 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-10-09T08:50:46.677 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-09T08:50:46.697 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-09T08:50:46.953 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T08:50:47.008 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-psutil. 2023-10-09T08:50:47.022 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-09T08:50:47.066 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T08:50:47.238 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T08:50:47.394 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-py. 2023-10-09T08:50:47.408 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-09T08:50:47.444 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T08:50:47.522 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:47.722 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-09T08:50:47.736 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-09T08:50:47.780 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T08:50:47.815 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T08:50:47.941 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-09T08:50:48.133 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-09T08:50:48.147 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-09T08:50:48.191 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T08:50:48.225 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T08:50:48.494 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-pytest. 2023-10-09T08:50:48.507 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-10-09T08:50:48.510 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T08:50:48.552 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-09T08:50:48.872 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package python3-simplejson. 2023-10-09T08:50:48.878 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T08:50:48.887 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-09T08:50:48.930 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T08:50:49.187 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T08:50:49.350 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package radosgw. 2023-10-09T08:50:49.366 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../98-radosgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:49.409 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:49.522 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T08:50:49.806 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:49.932 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-09T08:50:50.259 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-09T08:50:50.552 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-09T08:50:50.570 INFO:teuthology.orchestra.run.smithi181.stdout:Selecting previously unselected package rbd-fuse. 2023-10-09T08:50:50.576 INFO:teuthology.orchestra.run.smithi181.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T08:50:50.620 INFO:teuthology.orchestra.run.smithi181.stdout:Unpacking rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:50.853 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T08:50:50.922 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-09T08:50:50.988 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T08:50:51.218 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T08:50:51.280 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T08:50:51.343 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T08:50:51.469 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T08:50:51.657 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T08:50:51.662 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:52.017 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T08:50:52.274 INFO:teuthology.orchestra.run.smithi181.stdout:Adding system user cephadm....done 2023-10-09T08:50:52.377 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-09T08:50:52.427 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T08:50:52.503 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:52.770 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-09T08:50:52.813 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-09T08:50:53.054 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-09T08:50:53.165 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T08:50:53.180 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T08:50:53.316 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-09T08:50:53.434 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-09T08:50:53.498 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-09T08:50:53.718 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T08:50:53.884 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T08:50:54.021 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T08:50:54.147 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T08:50:54.169 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-09T08:50:54.473 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T08:50:54.678 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T08:50:54.750 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-09T08:50:54.954 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T08:50:55.017 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T08:50:55.144 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-09T08:50:55.238 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:55.420 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-09T08:50:55.539 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T08:50:55.665 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-09T08:50:55.713 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T08:50:55.966 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T08:50:56.015 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T08:50:56.267 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T08:50:56.451 INFO:teuthology.orchestra.run.smithi163.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-09T08:50:56.635 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T08:50:56.684 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-09T08:50:56.810 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T08:50:56.911 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T08:50:57.120 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-09T08:50:57.237 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T08:50:57.472 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T08:50:57.522 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:57.598 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T08:50:57.639 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-09T08:50:57.724 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-09T08:50:57.965 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-09T08:50:58.054 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T08:50:58.267 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-09T08:50:58.339 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up librados2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:50:58.481 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-09T08:50:58.559 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T08:50:58.685 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T08:50:58.970 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T08:50:59.363 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T08:50:59.723 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T08:50:59.963 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:00.083 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-09T08:51:00.089 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:00.218 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:00.223 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:51:00.349 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-09T08:51:00.528 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-09T08:51:00.879 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T08:51:00.901 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-09T08:51:01.005 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-09T08:51:01.131 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-09T08:51:01.186 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-09T08:51:01.424 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T08:51:01.479 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-09T08:51:01.734 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T08:51:01.814 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:01.860 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T08:51:01.940 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T08:51:02.170 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T08:51:02.259 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:02.390 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T08:51:02.454 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-09T08:51:02.725 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T08:51:02.755 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T08:51:02.890 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-09T08:51:03.174 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-09T08:51:03.484 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T08:51:03.776 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T08:51:03.783 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up librbd1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:03.918 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T08:51:04.044 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:51:04.147 INFO:teuthology.orchestra.run.smithi181.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-09T08:51:04.171 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-09T08:51:04.399 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-09T08:51:04.464 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-09T08:51:04.525 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T08:51:04.774 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:04.835 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-09T08:51:04.891 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:05.099 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T08:51:05.099 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T08:51:05.161 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T08:51:05.287 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T08:51:05.413 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-09T08:51:05.568 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:05.693 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T08:51:05.731 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T08:51:05.986 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-09T08:51:05.999 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up librados2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:06.141 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-09T08:51:06.296 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T08:51:06.665 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:06.790 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:06.916 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:07.042 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-09T08:51:07.352 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-09T08:51:07.639 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:07.730 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:07.765 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:07.856 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:07.900 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:51:08.026 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-09T08:51:08.149 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T08:51:08.394 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-09T08:51:08.699 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-09T08:51:08.992 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-09T08:51:09.119 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:09.318 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:09.444 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T08:51:09.691 INFO:teuthology.orchestra.run.smithi163.stdout:Adding group ceph....done 2023-10-09T08:51:09.771 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:09.899 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T08:51:10.066 INFO:teuthology.orchestra.run.smithi163.stdout:Adding system user ceph....done 2023-10-09T08:51:10.215 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T08:51:10.244 INFO:teuthology.orchestra.run.smithi163.stdout:Setting system user ceph properties....done 2023-10-09T08:51:10.258 INFO:teuthology.orchestra.run.smithi163.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-09T08:51:10.410 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-09T08:51:10.735 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-09T08:51:11.181 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:11.289 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up librbd1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:11.307 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:11.415 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T08:51:11.541 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T08:51:11.581 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T08:51:11.581 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T08:51:11.675 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-09T08:51:11.960 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-09T08:51:12.245 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:12.261 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:12.395 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:12.611 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T08:51:12.611 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T08:51:12.615 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-09T08:51:13.072 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:13.096 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:13.198 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T08:51:13.324 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T08:51:13.324 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T08:51:13.482 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-09T08:51:13.764 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:13.800 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T08:51:13.996 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T08:51:13.996 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T08:51:14.144 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:14.270 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:14.396 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:14.440 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:14.522 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-09T08:51:14.806 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-09T08:51:14.921 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T08:51:14.921 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T08:51:15.235 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:15.361 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:15.383 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:15.501 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:15.646 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:15.654 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T08:51:15.859 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T08:51:15.859 INFO:teuthology.orchestra.run.smithi163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T08:51:16.314 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:16.465 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:16.586 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:16.591 INFO:teuthology.orchestra.run.smithi163.stdout:Setting up ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:16.751 INFO:teuthology.orchestra.run.smithi163.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-09T08:51:17.035 INFO:teuthology.orchestra.run.smithi163.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T08:51:17.147 INFO:teuthology.orchestra.run.smithi181.stdout:Adding group ceph....done 2023-10-09T08:51:17.539 INFO:teuthology.orchestra.run.smithi181.stdout:Adding system user ceph....done 2023-10-09T08:51:17.716 INFO:teuthology.orchestra.run.smithi181.stdout:Setting system user ceph properties....done 2023-10-09T08:51:17.730 INFO:teuthology.orchestra.run.smithi181.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-09T08:51:18.083 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-09T08:51:18.197 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-09T08:51:18.657 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:18.783 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:19.087 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T08:51:19.087 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T08:51:19.726 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:20.021 INFO:teuthology.orchestra.run.smithi163.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T08:51:20.103 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-09T08:51:20.455 INFO:teuthology.orchestra.run.smithi163.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-09T08:51:20.458 DEBUG:teuthology.parallel:result is None 2023-10-09T08:51:20.580 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:20.800 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T08:51:20.800 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T08:51:21.257 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:21.483 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T08:51:21.483 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T08:51:21.950 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:22.448 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T08:51:22.448 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T08:51:22.918 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:23.044 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:23.178 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:23.390 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T08:51:23.390 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T08:51:23.846 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:23.972 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:24.098 INFO:teuthology.orchestra.run.smithi181.stdout:Setting up ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T08:51:24.242 INFO:teuthology.orchestra.run.smithi181.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-09T08:51:24.526 INFO:teuthology.orchestra.run.smithi181.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T08:51:27.505 INFO:teuthology.orchestra.run.smithi181.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T08:51:27.962 INFO:teuthology.orchestra.run.smithi181.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-09T08:51:27.964 DEBUG:teuthology.parallel:result is None 2023-10-09T08:51:27.965 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T08:51:27.965 INFO:teuthology.packaging:ref: None 2023-10-09T08:51:27.965 INFO:teuthology.packaging:tag: None 2023-10-09T08:51:27.965 INFO:teuthology.packaging:branch: pacific 2023-10-09T08:51:27.965 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:51:27.965 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T08:51:28.271 DEBUG:teuthology.orchestra.run.smithi163:> dpkg-query -W -f '${Version}' ceph 2023-10-09T08:51:28.293 INFO:teuthology.orchestra.run.smithi163.stdout:16.2.14-48-gaeb2e425-1focal 2023-10-09T08:51:28.294 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48-gaeb2e425-1focal 2023-10-09T08:51:28.294 INFO:teuthology.task.install:The correct ceph version 16.2.14-48-gaeb2e425-1focal is installed. 2023-10-09T08:51:28.294 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T08:51:28.294 INFO:teuthology.packaging:ref: None 2023-10-09T08:51:28.294 INFO:teuthology.packaging:tag: None 2023-10-09T08:51:28.294 INFO:teuthology.packaging:branch: pacific 2023-10-09T08:51:28.294 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T08:51:28.295 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T08:51:28.501 DEBUG:teuthology.orchestra.run.smithi181:> dpkg-query -W -f '${Version}' ceph 2023-10-09T08:51:28.524 INFO:teuthology.orchestra.run.smithi181.stdout:16.2.14-48-gaeb2e425-1focal 2023-10-09T08:51:28.524 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48-gaeb2e425-1focal 2023-10-09T08:51:28.524 INFO:teuthology.task.install:The correct ceph version 16.2.14-48-gaeb2e425-1focal is installed. 2023-10-09T08:51:28.524 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T08:51:28.524 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:51:28.525 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T08:51:28.542 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:51:28.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T08:51:28.581 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T08:51:28.581 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:51:28.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T08:51:28.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T08:51:28.654 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:51:28.654 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T08:51:28.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T08:51:28.723 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T08:51:28.723 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:51:28.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T08:51:28.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T08:51:28.793 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:51:28.793 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T08:51:28.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T08:51:28.869 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T08:51:28.869 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:51:28.869 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T08:51:28.886 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T08:51:28.940 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:51:28.941 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T08:51:28.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T08:51:29.011 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T08:51:29.099 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T08:51:29.099 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T08:51:29.100 INFO:tasks.cephadm:Cluster fsid is 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:51:29.100 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T08:51:29.100 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.163', 'mon.c': '[v2:172.21.15.163:3301,v1:172.21.15.163:6790]', 'mon.b': '172.21.15.181'} 2023-10-09T08:51:29.100 INFO:tasks.cephadm:First mon is mon.a on smithi163 2023-10-09T08:51:29.100 INFO:tasks.cephadm:First mgr is y 2023-10-09T08:51:29.100 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T08:51:29.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo hostname $(hostname -s) 2023-10-09T08:51:29.118 DEBUG:teuthology.orchestra.run.smithi181:> sudo hostname $(hostname -s) 2023-10-09T08:51:29.134 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-10-09T08:51:29.134 DEBUG:teuthology.orchestra.run.smithi163:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T08:51:29.322 INFO:teuthology.orchestra.run.smithi163.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Oct 9 08:51 /home/ubuntu/cephtest/cephadm 2023-10-09T08:51:29.322 DEBUG:teuthology.orchestra.run.smithi181:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T08:51:29.486 INFO:teuthology.orchestra.run.smithi181.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Oct 9 08:51 /home/ubuntu/cephtest/cephadm 2023-10-09T08:51:29.487 DEBUG:teuthology.orchestra.run.smithi163:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T08:51:29.496 DEBUG:teuthology.orchestra.run.smithi181:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T08:51:29.515 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-10-09T08:51:29.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T08:51:29.541 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T08:51:29.812 INFO:teuthology.orchestra.run.smithi163.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T08:51:29.813 INFO:teuthology.orchestra.run.smithi181.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T08:52:57.804 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T08:52:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T08:52:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T08:52:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: "repo_digests": [ 2023-10-09T08:52:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T08:52:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T08:52:57.805 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T08:52:58.987 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T08:52:58.987 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T08:52:58.987 INFO:teuthology.orchestra.run.smithi163.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T08:52:58.987 INFO:teuthology.orchestra.run.smithi163.stdout: "repo_digests": [ 2023-10-09T08:52:58.987 INFO:teuthology.orchestra.run.smithi163.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T08:52:58.987 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T08:52:58.987 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T08:52:59.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo mkdir -p /etc/ceph 2023-10-09T08:52:59.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo mkdir -p /etc/ceph 2023-10-09T08:52:59.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod 777 /etc/ceph 2023-10-09T08:52:59.086 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 777 /etc/ceph 2023-10-09T08:52:59.103 INFO:tasks.cephadm:Writing seed config... 2023-10-09T08:52:59.104 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T08:52:59.105 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T08:52:59.105 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T08:52:59.105 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T08:52:59.105 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T08:52:59.105 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-10-09T08:52:59.105 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T08:52:59.105 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T08:52:59.105 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T08:52:59.105 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T08:52:59.105 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T08:52:59.106 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:52:59.106 DEBUG:teuthology.orchestra.run.smithi163:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T08:52:59.137 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 097c0480-6681-11ee-8db5-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T08:52:59.138 DEBUG:teuthology.orchestra.run.smithi163:mon.a> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T08:52:59.180 DEBUG:teuthology.orchestra.run.smithi163:mgr.y> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T08:52:59.224 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T08:52:59.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.163 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T08:52:59.433 INFO:teuthology.orchestra.run.smithi163.stderr:-------------------------------------------------------------------------------- 2023-10-09T08:52:59.433 INFO:teuthology.orchestra.run.smithi163.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', '097c0480-6681-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.163', '--skip-admin-label'] 2023-10-09T08:52:59.433 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying podman|docker is present... 2023-10-09T08:52:59.433 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying lvm2 is present... 2023-10-09T08:52:59.433 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying time synchronization is in place... 2023-10-09T08:52:59.439 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T08:52:59.446 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2023-10-09T08:52:59.453 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-10-09T08:52:59.458 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2023-10-09T08:52:59.463 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: masked 2023-10-09T08:52:59.469 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2023-10-09T08:52:59.474 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-10-09T08:52:59.480 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2023-10-09T08:52:59.486 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: enabled 2023-10-09T08:52:59.493 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: active 2023-10-09T08:52:59.494 INFO:teuthology.orchestra.run.smithi163.stderr:Unit ntp.service is enabled and running 2023-10-09T08:52:59.494 INFO:teuthology.orchestra.run.smithi163.stderr:Repeating the final host check... 2023-10-09T08:52:59.494 INFO:teuthology.orchestra.run.smithi163.stderr:docker (/usr/bin/docker) is present 2023-10-09T08:52:59.495 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl is present 2023-10-09T08:52:59.495 INFO:teuthology.orchestra.run.smithi163.stderr:lvcreate is present 2023-10-09T08:52:59.497 INFO:journalctl@ceph.mon.a.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:52:59.500 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T08:52:59.505 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2023-10-09T08:52:59.510 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-10-09T08:52:59.511 INFO:journalctl@ceph.mgr.y.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:52:59.517 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2023-10-09T08:52:59.524 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: masked 2023-10-09T08:52:59.530 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2023-10-09T08:52:59.537 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-10-09T08:52:59.543 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2023-10-09T08:52:59.550 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: enabled 2023-10-09T08:52:59.557 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: active 2023-10-09T08:52:59.559 INFO:teuthology.orchestra.run.smithi163.stderr:Unit ntp.service is enabled and running 2023-10-09T08:52:59.559 INFO:teuthology.orchestra.run.smithi163.stderr:Host looks OK 2023-10-09T08:52:59.559 INFO:teuthology.orchestra.run.smithi163.stderr:Cluster fsid: 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:52:59.559 INFO:teuthology.orchestra.run.smithi163.stderr:Acquiring lock 139960789659600 on /run/cephadm/097c0480-6681-11ee-8db5-212e2dc638e7.lock 2023-10-09T08:52:59.559 INFO:teuthology.orchestra.run.smithi163.stderr:Lock 139960789659600 acquired on /run/cephadm/097c0480-6681-11ee-8db5-212e2dc638e7.lock 2023-10-09T08:52:59.560 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying IP 172.21.15.163 port 3300 ... 2023-10-09T08:52:59.560 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying IP 172.21.15.163 port 6789 ... 2023-10-09T08:52:59.560 INFO:teuthology.orchestra.run.smithi163.stderr:Base mon IP(s) is [172.21.15.163:3300, 172.21.15.163:6789], mon addrv is [v2:172.21.15.163:3300,v1:172.21.15.163:6789] 2023-10-09T08:52:59.564 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 2023-10-09T08:52:59.564 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.163 metric 100 2023-10-09T08:52:59.564 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2023-10-09T08:52:59.564 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.163 2023-10-09T08:52:59.565 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.163 metric 100 2023-10-09T08:52:59.568 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T08:52:59.568 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-10-09T08:52:59.569 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1792sec pref medium 2023-10-09T08:52:59.572 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T08:52:59.572 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: inet6 ::1/128 scope host 2023-10-09T08:52:59.572 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T08:52:59.572 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-09T08:52:59.573 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: inet6 fe80::ec4:7aff:fe88:7d5b/64 scope link 2023-10-09T08:52:59.573 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T08:52:59.574 INFO:teuthology.orchestra.run.smithi163.stderr:Mon IP `172.21.15.163` is in CIDR network `172.21.0.0/20` 2023-10-09T08:52:59.574 INFO:teuthology.orchestra.run.smithi163.stderr:Mon IP `172.21.15.163` is in CIDR network `172.21.0.0/20` 2023-10-09T08:52:59.574 INFO:teuthology.orchestra.run.smithi163.stderr:Mon IP `172.21.15.163` is in CIDR network `172.21.15.254/32` 2023-10-09T08:52:59.574 INFO:teuthology.orchestra.run.smithi163.stderr:Mon IP `172.21.15.163` is in CIDR network `172.21.15.254/32` 2023-10-09T08:52:59.574 INFO:teuthology.orchestra.run.smithi163.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.15.254/32', '172.21.15.254/32'] 2023-10-09T08:52:59.575 INFO:teuthology.orchestra.run.smithi163.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T08:52:59.575 INFO:teuthology.orchestra.run.smithi163.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T08:52:59.745 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/docker: pacific: Pulling from ceph-ci/ceph 2023-10-09T08:52:59.767 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/docker: Digest: sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3 2023-10-09T08:52:59.767 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/docker: Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T08:52:59.792 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/docker: quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T08:53:00.716 INFO:teuthology.orchestra.run.smithi163.stderr:ceph: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T08:53:01.273 INFO:teuthology.orchestra.run.smithi163.stderr:Ceph version: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T08:53:01.273 INFO:teuthology.orchestra.run.smithi163.stderr:Extracting ceph user uid/gid from container image... 2023-10-09T08:53:02.044 INFO:teuthology.orchestra.run.smithi163.stderr:stat: 167 167 2023-10-09T08:53:02.626 INFO:teuthology.orchestra.run.smithi163.stderr:Creating initial keys... 2023-10-09T08:53:03.372 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-authtool: AQBvvyNlYhclFhAAa8UOmMNRGPd647dNhQ95bg== 2023-10-09T08:53:04.696 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-authtool: AQBwvyNlthlyKRAAM7/th8+glN3ZwbRvDF/Gjw== 2023-10-09T08:53:05.912 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-authtool: AQBxvyNl2PdPNhAAB51sb6PbhLad9LCXIaw50w== 2023-10-09T08:53:06.449 INFO:teuthology.orchestra.run.smithi163.stderr:Creating initial monmap... 2023-10-09T08:53:07.199 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T08:53:07.199 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:07.199 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T08:53:07.720 INFO:teuthology.orchestra.run.smithi163.stderr:monmaptool for a [v2:172.21.15.163:3300,v1:172.21.15.163:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T08:53:07.720 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: set fsid to 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:07.721 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T08:53:07.721 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:53:07.721 INFO:teuthology.orchestra.run.smithi163.stderr:Creating mon... 2023-10-09T08:53:08.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.445+0000 7fe560107880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T08:53:08.452 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.445+0000 7fe560107880 1 imported monmap: 2023-10-09T08:53:08.452 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: epoch 0 2023-10-09T08:53:08.453 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: fsid 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:08.453 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: last_changed 2023-10-09T08:53:07.197738+0000 2023-10-09T08:53:08.453 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: created 2023-10-09T08:53:07.197738+0000 2023-10-09T08:53:08.453 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-10-09T08:53:08.453 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-10-09T08:53:08.453 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon.a 2023-10-09T08:53:08.453 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.453 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.445+0000 7fe560107880 0 /usr/bin/ceph-mon: set fsid to 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:08.456 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T08:53:08.456 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.456 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T08:53:08.456 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T08:53:08.456 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: DB SUMMARY 2023-10-09T08:53:08.456 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.456 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-10-09T08:53:08.456 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.456 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.create_if_missing: 1 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.env: 0x55d9427c1080 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.fs: Posix File System 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.info_log: 0x55d943749aa0 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.statistics: (nil) 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.use_fsync: 0 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T08:53:08.457 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T08:53:08.458 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T08:53:08.479 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T08:53:08.479 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T08:53:08.479 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T08:53:08.479 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T08:53:08.479 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T08:53:08.479 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.db_log_dir: 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T08:53:08.480 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.write_buffer_manager: 0x55d943751890 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T08:53:08.481 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.unordered_write: 0 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.row_cache: None 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.wal_filter: None 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T08:53:08.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T08:53:08.483 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_open_files: -1 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Compression algorithms supported: 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T08:53:08.484 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: kZSTD supported: 0 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.merge_operator: 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_filter: None 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d94365fd20) 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T08:53:08.486 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: index_type: 0 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: checksum: 1 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: block_cache: 0x55d943696d10 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-10-09T08:53:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-10-09T08:53:08.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: format_version: 2 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: block_align: 0 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compression: NoCompression 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T08:53:08.489 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.num_levels: 7 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:53:08.490 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T08:53:08.491 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T08:53:08.492 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T08:53:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T08:53:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.ttl: 2592000 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.449+0000 7fe560107880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.453+0000 7fe560107880 4 rocksdb: DB pointer 0x55d94375f800 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.453+0000 7fe548f8f700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.453+0000 7fe548f8f700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-10-09T08:53:08.495 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T08:53:08.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:53:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.498 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.499 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:53:08.500 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: 2023-10-09T08:53:08.500 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.457+0000 7fe560107880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-10-09T08:53:08.500 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.457+0000 7fe560107880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-10-09T08:53:08.500 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-mon: debug 2023-10-09T08:53:08.457+0000 7fe560107880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-10-09T08:53:08.982 INFO:teuthology.orchestra.run.smithi163.stderr:create mon.a on 2023-10-09T08:53:09.003 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T08:53:09.203 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2023-10-09T08:53:09.414 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-097c0480-6681-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-097c0480-6681-11ee-8db5-212e2dc638e7.target. 2023-10-09T08:53:09.415 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-097c0480-6681-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-097c0480-6681-11ee-8db5-212e2dc638e7.target. 2023-10-09T08:53:09.824 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to reset failed state of unit ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T08:53:09.832 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Created symlink /etc/systemd/system/ceph-097c0480-6681-11ee-8db5-212e2dc638e7.target.wants/ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.a.service -> /etc/systemd/system/ceph-097c0480-6681-11ee-8db5-212e2dc638e7@.service. 2023-10-09T08:53:10.050 INFO:teuthology.orchestra.run.smithi163.stderr:firewalld does not appear to be present 2023-10-09T08:53:10.050 INFO:teuthology.orchestra.run.smithi163.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T08:53:10.051 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mon to start... 2023-10-09T08:53:10.051 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mon... 2023-10-09T08:53:10.110 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:10 smithi163 systemd[1]: Started Ceph mon.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T08:53:11.450 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: cluster: 2023-10-09T08:53:11.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: id: 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:11.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-10-09T08:53:11.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:11.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: services: 2023-10-09T08:53:11.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.256577s) 2023-10-09T08:53:11.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mgr: no daemons active 2023-10-09T08:53:11.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-10-09T08:53:11.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:11.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: data: 2023-10-09T08:53:11.451 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-10-09T08:53:11.452 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-10-09T08:53:11.452 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-10-09T08:53:11.452 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: pgs: 2023-10-09T08:53:11.452 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:11 smithi163 bash[26962]: cluster 2023-10-09T08:53:11.193448+0000 mon.a (mon.0) 0 : [INF] mkfs 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:11 smithi163 bash[26962]: cluster 2023-10-09T08:53:11.186472+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T08:53:12.026 INFO:teuthology.orchestra.run.smithi163.stderr:mon is available 2023-10-09T08:53:12.027 INFO:teuthology.orchestra.run.smithi163.stderr:Assimilating anything we can from ceph.conf... 2023-10-09T08:53:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:12 smithi163 bash[26962]: cluster 2023-10-09T08:53:11.191831+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T08:53:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:12 smithi163 bash[26962]: cluster 2023-10-09T08:53:11.192096+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0]} removed_ranks: {} 2023-10-09T08:53:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:12 smithi163 bash[26962]: cluster 2023-10-09T08:53:11.195934+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-10-09T08:53:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:12 smithi163 bash[26962]: cluster 2023-10-09T08:53:11.199586+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T08:53:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:12 smithi163 bash[26962]: cluster 2023-10-09T08:53:11.200165+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T08:53:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:12 smithi163 bash[26962]: audit 2023-10-09T08:53:11.448321+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.163:0/1032506752' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T08:53:12.979 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:12.980 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [global] 2023-10-09T08:53:12.980 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: fsid = 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:12.981 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T08:53:12.981 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.163:3300,v1:172.21.15.163:6789] 2023-10-09T08:53:12.981 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T08:53:12.981 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T08:53:12.982 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T08:53:12.982 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T08:53:12.982 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:12.982 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [mgr] 2023-10-09T08:53:12.982 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T08:53:12.982 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:12.982 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [osd] 2023-10-09T08:53:12.982 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T08:53:12.982 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T08:53:13.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:13 smithi163 bash[26962]: audit 2023-10-09T08:53:12.972853+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.163:0/2509231785' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T08:53:13.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:13 smithi163 bash[26962]: audit 2023-10-09T08:53:12.978002+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.163:0/2509231785' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T08:53:13.588 INFO:teuthology.orchestra.run.smithi163.stderr:Generating new minimal ceph.conf... 2023-10-09T08:53:14.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:14 smithi163 bash[26962]: audit 2023-10-09T08:53:14.540072+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.163:0/1169742406' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:53:15.135 INFO:teuthology.orchestra.run.smithi163.stderr:Restarting the monitor... 2023-10-09T08:53:15.413 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:15 smithi163 systemd[1]: Stopping Ceph mon.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T08:53:15.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:15 smithi163 bash[27360]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon.a 2023-10-09T08:53:15.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:15 smithi163 bash[26962]: debug 2023-10-09T08:53:15.312+0000 7f39a9353700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T08:53:15.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:15 smithi163 bash[26962]: debug 2023-10-09T08:53:15.312+0000 7f39a9353700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T08:53:15.860 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:15 smithi163 bash[27395]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T08:53:15.883 INFO:teuthology.orchestra.run.smithi163.stderr:Setting mon public_network to 172.21.0.0/20,172.21.15.254/32 2023-10-09T08:53:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:15 smithi163 bash[27461]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T08:53:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:15 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T08:53:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:15 smithi163 systemd[1]: Stopped Ceph mon.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T08:53:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:15 smithi163 systemd[1]: Started Ceph mon.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T08:53:17.160 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.848+0000 7f8cd746d880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.848+0000 7f8cd746d880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.848+0000 7f8cd746d880 0 pidfile_write: ignore empty --pid-file 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.852+0000 7f8cd746d880 0 load: jerasure load: lrc load: isa 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: DB SUMMARY 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-10-09T08:53:17.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86739 ; 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.env: 0x55f7c2bc1080 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.fs: Posix File System 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.info_log: 0x55f7c3527a80 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.statistics: (nil) 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.use_fsync: 0 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T08:53:17.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.db_log_dir: 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T08:53:17.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.write_buffer_manager: 0x55f7c42242d0 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T08:53:17.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.unordered_write: 0 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T08:53:17.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.row_cache: None 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.wal_filter: None 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T08:53:17.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T08:53:17.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_open_files: -1 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Compression algorithms supported: 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: kZSTD supported: 0 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T08:53:17.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.merge_operator: 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_filter: None 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T08:53:17.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f7c34d5d18) 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: cache_index_and_filter_blocks: 1 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: pin_top_level_index_and_filter: 1 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: index_type: 0 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: data_block_index_type: 0 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: index_shortening: 1 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: hash_index_allow_collision: 1 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: checksum: 1 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: no_block_cache: 0 2023-10-09T08:53:17.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: block_cache: 0x55f7c350cf10 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: block_cache_name: BinnedLRUCache 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: block_cache_options: 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: capacity : 536870912 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: num_shard_bits : 4 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: strict_capacity_limit : 0 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: high_pri_pool_ratio: 0.000 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: block_cache_compressed: (nil) 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: persistent_cache: (nil) 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: block_size: 4096 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: block_size_deviation: 10 2023-10-09T08:53:17.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: block_restart_interval: 16 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: index_block_restart_interval: 1 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: metadata_block_size: 4096 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: partition_filters: 0 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: use_delta_encoding: 1 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: whole_key_filtering: 1 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: verify_compression: 0 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: read_amp_bytes_per_bit: 0 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: format_version: 2 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: enable_index_compression: 1 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: block_align: 0 2023-10-09T08:53:17.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compression: NoCompression 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.num_levels: 7 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T08:53:17.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T08:53:17.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T08:53:17.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T08:53:17.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T08:53:17.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.ttl: 2592000 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696841596860352, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-10-09T08:53:17.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696841596861408, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83339, "table_properties": {"data_size": 81612, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75933, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696841596, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696841596861855, "job": 1, "event": "recovery_finished"} 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.856+0000 7f8cd746d880 4 rocksdb: DB pointer 0x55f7c35d5800 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cbe274700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cbe274700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ** DB Stats ** 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T08:53:17.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ** Compaction Stats [default] ** 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: L0 2/0 83.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.2 0.00 0.00 1 0.001 0 0 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Sum 2/0 83.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.2 0.00 0.00 1 0.001 0 0 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.2 0.00 0.00 1 0.001 0 0 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ** Compaction Stats [default] ** 2023-10-09T08:53:17.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 123.2 0.00 0.00 1 0.001 0 0 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Cumulative compaction: 0.00 GB write, 22.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Interval compaction: 0.00 GB write, 22.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:53:17.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ** Compaction Stats [default] ** 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: L0 2/0 83.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.2 0.00 0.00 1 0.001 0 0 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Sum 2/0 83.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.2 0.00 0.00 1 0.001 0 0 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ** Compaction Stats [default] ** 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 123.2 0.00 0.00 1 0.001 0 0 2023-10-09T08:53:17.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Cumulative compaction: 0.00 GB write, 22.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cd746d880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] at bind addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cd746d880 1 mon.a@-1(???) e1 preinit fsid 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cd746d880 0 mon.a@-1(???).mds e1 new map 2023-10-09T08:53:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cd746d880 0 mon.a@-1(???).mds e1 print_map 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: e1 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: legacy client fscid: -1 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: No filesystems configured 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cd746d880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cd746d880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cd746d880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cd746d880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: audit 2023-10-09T08:53:14.540072+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.163:0/1169742406' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: debug 2023-10-09T08:53:16.860+0000 7f8cd746d880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T08:53:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: cluster 2023-10-09T08:53:16.887331+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T08:53:17.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: cluster 2023-10-09T08:53:16.887394+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0]} removed_ranks: {} 2023-10-09T08:53:17.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: cluster 2023-10-09T08:53:16.907282+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-10-09T08:53:17.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: cluster 2023-10-09T08:53:16.907324+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T08:53:17.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:16 smithi163 bash[27500]: cluster 2023-10-09T08:53:16.907760+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T08:53:17.737 INFO:teuthology.orchestra.run.smithi163.stderr:Wrote config to /etc/ceph/ceph.conf 2023-10-09T08:53:17.738 INFO:teuthology.orchestra.run.smithi163.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T08:53:17.738 INFO:teuthology.orchestra.run.smithi163.stderr:Creating mgr... 2023-10-09T08:53:17.738 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying port 9283 ... 2023-10-09T08:53:17.965 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to reset failed state of unit ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T08:53:17.972 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Created symlink /etc/systemd/system/ceph-097c0480-6681-11ee-8db5-212e2dc638e7.target.wants/ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.y.service -> /etc/systemd/system/ceph-097c0480-6681-11ee-8db5-212e2dc638e7@.service. 2023-10-09T08:53:18.177 INFO:teuthology.orchestra.run.smithi163.stderr:firewalld does not appear to be present 2023-10-09T08:53:18.177 INFO:teuthology.orchestra.run.smithi163.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T08:53:18.177 INFO:teuthology.orchestra.run.smithi163.stderr:firewalld does not appear to be present 2023-10-09T08:53:18.177 INFO:teuthology.orchestra.run.smithi163.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T08:53:18.177 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mgr to start... 2023-10-09T08:53:18.178 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mgr... 2023-10-09T08:53:18.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:18 smithi163 bash[27500]: audit 2023-10-09T08:53:17.127790+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.163:0/3648098074' entity='client.admin' 2023-10-09T08:53:19.506 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:19.506 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2023-10-09T08:53:19.506 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsid": "097c0480-6681-11ee-8db5-212e2dc638e7", 2023-10-09T08:53:19.506 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "health": { 2023-10-09T08:53:19.507 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T08:53:19.507 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T08:53:19.507 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T08:53:19.507 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:19.508 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T08:53:19.510 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T08:53:19.510 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 0 2023-10-09T08:53:19.510 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2023-10-09T08:53:19.510 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T08:53:19.510 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "a" 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T08:53:19.511 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T08:53:19.512 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T08:53:19.512 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T08:53:19.512 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:19.512 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T08:53:19.512 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T08:53:19.512 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T08:53:19.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "available": false, 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "iostat", 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "nfs", 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "restful" 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:19.516 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T08:53:19.517 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:19.517 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modified": "2023-10-09T08:53:11.194232+0000", 2023-10-09T08:53:19.517 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:53:19.517 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:19.517 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T08:53:19.517 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2023-10-09T08:53:19.748 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:19 smithi163 bash[27764]: debug 2023-10-09T08:53:19.596+0000 7fd59bb05680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:53:19.748 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:19 smithi163 bash[27500]: audit 2023-10-09T08:53:19.504570+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.163:0/67190826' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T08:53:20.144 INFO:teuthology.orchestra.run.smithi163.stderr:mgr not available, waiting (1/15)... 2023-10-09T08:53:20.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:20 smithi163 bash[27764]: debug 2023-10-09T08:53:20.040+0000 7fd59bb05680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:53:21.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:20 smithi163 bash[27764]: debug 2023-10-09T08:53:20.956+0000 7fd59bb05680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:53:21.219 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:21 smithi163 bash[27764]: debug 2023-10-09T08:53:21.056+0000 7fd59bb05680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:53:21.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:21 smithi163 bash[27764]: debug 2023-10-09T08:53:21.304+0000 7fd59bb05680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:53:22.695 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:22 smithi163 bash[27764]: debug 2023-10-09T08:53:22.428+0000 7fd59bb05680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:53:22.695 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:22 smithi163 bash[27764]: debug 2023-10-09T08:53:22.592+0000 7fd59bb05680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:53:22.695 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:22 smithi163 bash[27764]: debug 2023-10-09T08:53:22.688+0000 7fd59bb05680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:53:22.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:22 smithi163 bash[27764]: debug 2023-10-09T08:53:22.800+0000 7fd59bb05680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:53:23.204 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:23.205 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2023-10-09T08:53:23.205 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsid": "097c0480-6681-11ee-8db5-212e2dc638e7", 2023-10-09T08:53:23.205 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "health": { 2023-10-09T08:53:23.205 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T08:53:23.205 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T08:53:23.205 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T08:53:23.206 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:23.206 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T08:53:23.206 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T08:53:23.206 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 0 2023-10-09T08:53:23.206 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2023-10-09T08:53:23.206 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T08:53:23.206 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "a" 2023-10-09T08:53:23.206 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2023-10-09T08:53:23.206 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_age": 6, 2023-10-09T08:53:23.206 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T08:53:23.207 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:23.207 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T08:53:23.207 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T08:53:23.207 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:23.207 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T08:53:23.207 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:23.207 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T08:53:23.207 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T08:53:23.207 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T08:53:23.207 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T08:53:23.210 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T08:53:23.211 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "available": false, 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "iostat", 2023-10-09T08:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "nfs", 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "restful" 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modified": "2023-10-09T08:53:11.194232+0000", 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T08:53:23.213 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2023-10-09T08:53:23.407 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:23 smithi163 bash[27500]: audit 2023-10-09T08:53:23.200870+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.163:0/98820937' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T08:53:23.408 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:23 smithi163 bash[27764]: debug 2023-10-09T08:53:23.028+0000 7fd59bb05680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:53:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:23 smithi163 bash[27764]: debug 2023-10-09T08:53:23.404+0000 7fd59bb05680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:53:23.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:23 smithi163 bash[27764]: debug 2023-10-09T08:53:23.524+0000 7fd59bb05680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:53:23.764 INFO:teuthology.orchestra.run.smithi163.stderr:mgr not available, waiting (2/15)... 2023-10-09T08:53:24.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:24 smithi163 bash[27764]: debug 2023-10-09T08:53:24.392+0000 7fd59bb05680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:53:24.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:24 smithi163 bash[27764]: debug 2023-10-09T08:53:24.500+0000 7fd59bb05680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:53:24.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:24 smithi163 bash[27764]: debug 2023-10-09T08:53:24.612+0000 7fd59bb05680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:53:25.128 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:24 smithi163 bash[27764]: debug 2023-10-09T08:53:24.828+0000 7fd59bb05680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:53:25.459 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27764]: debug 2023-10-09T08:53:25.124+0000 7fd59bb05680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:53:25.459 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27764]: debug 2023-10-09T08:53:25.288+0000 7fd59bb05680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:53:25.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27764]: debug 2023-10-09T08:53:25.456+0000 7fd59bb05680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:53:25.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27764]: debug 2023-10-09T08:53:25.664+0000 7fd59bb05680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:53:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27500]: cluster 2023-10-09T08:53:25.792528+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-10-09T08:53:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27500]: cluster 2023-10-09T08:53:25.797311+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00494638s) 2023-10-09T08:53:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27500]: audit 2023-10-09T08:53:25.799020+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.163:0/3087420728' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:53:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27500]: audit 2023-10-09T08:53:25.799344+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.163:0/3087420728' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:53:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27500]: audit 2023-10-09T08:53:25.799652+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.163:0/3087420728' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:53:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27500]: audit 2023-10-09T08:53:25.799919+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.163:0/3087420728' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:53:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27500]: audit 2023-10-09T08:53:25.800142+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.163:0/3087420728' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:53:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27500]: cluster 2023-10-09T08:53:25.823587+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2023-10-09T08:53:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27500]: audit 2023-10-09T08:53:25.832084+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.163:0/3087420728' entity='mgr.y' 2023-10-09T08:53:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27500]: audit 2023-10-09T08:53:25.839880+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.163:0/3087420728' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:53:26.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:25 smithi163 bash[27764]: debug 2023-10-09T08:53:25.768+0000 7fd59bb05680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:53:26.862 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:26.863 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2023-10-09T08:53:26.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsid": "097c0480-6681-11ee-8db5-212e2dc638e7", 2023-10-09T08:53:26.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "health": { 2023-10-09T08:53:26.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T08:53:26.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T08:53:26.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T08:53:26.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:26.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T08:53:26.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T08:53:26.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 0 2023-10-09T08:53:26.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2023-10-09T08:53:26.869 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T08:53:26.869 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "a" 2023-10-09T08:53:26.869 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2023-10-09T08:53:26.869 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_age": 9, 2023-10-09T08:53:26.869 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T08:53:26.869 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:26.869 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:26.870 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T08:53:26.871 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "available": true, 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "iostat", 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "nfs", 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "restful" 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T08:53:26.872 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T08:53:26.873 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modified": "2023-10-09T08:53:11.194232+0000", 2023-10-09T08:53:26.873 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2023-10-09T08:53:26.873 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2023-10-09T08:53:26.873 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T08:53:26.873 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2023-10-09T08:53:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:26 smithi163 bash[27500]: audit 2023-10-09T08:53:25.844678+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.163:0/3087420728' entity='mgr.y' 2023-10-09T08:53:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:26 smithi163 bash[27500]: audit 2023-10-09T08:53:25.845198+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.163:0/3087420728' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:53:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:26 smithi163 bash[27500]: audit 2023-10-09T08:53:25.849135+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.163:0/3087420728' entity='mgr.y' 2023-10-09T08:53:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:26 smithi163 bash[27500]: cluster 2023-10-09T08:53:26.803773+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.01141s) 2023-10-09T08:53:27.464 INFO:teuthology.orchestra.run.smithi163.stderr:mgr is available 2023-10-09T08:53:28.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:27 smithi163 bash[27500]: audit 2023-10-09T08:53:26.861397+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.163:0/1480896431' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T08:53:28.648 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:28.648 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [global] 2023-10-09T08:53:28.648 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: fsid = 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:28.648 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T08:53:28.649 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T08:53:28.650 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T08:53:28.650 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T08:53:28.650 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T08:53:28.650 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:28.650 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [mgr] 2023-10-09T08:53:28.650 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T08:53:28.651 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2023-10-09T08:53:28.651 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [osd] 2023-10-09T08:53:28.651 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T08:53:28.651 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T08:53:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:28 smithi163 bash[27500]: cluster 2023-10-09T08:53:27.872641+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-10-09T08:53:28.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:28 smithi163 bash[27500]: audit 2023-10-09T08:53:28.645466+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.163:0/3665863028' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T08:53:29.236 INFO:teuthology.orchestra.run.smithi163.stderr:Enabling cephadm module... 2023-10-09T08:53:30.881 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:30 smithi163 bash[27500]: audit 2023-10-09T08:53:30.521074+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.163:0/2958603687' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T08:53:30.882 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:30 smithi163 bash[27764]: ignoring --setuser ceph since I am not root 2023-10-09T08:53:30.882 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:30 smithi163 bash[27764]: ignoring --setgroup ceph since I am not root 2023-10-09T08:53:30.882 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:30 smithi163 bash[27764]: debug 2023-10-09T08:53:30.772+0000 7f99274b0680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:53:30.882 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:30 smithi163 bash[27764]: debug 2023-10-09T08:53:30.876+0000 7f99274b0680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:53:31.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:31 smithi163 bash[27764]: debug 2023-10-09T08:53:31.360+0000 7f99274b0680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:53:31.857 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:31 smithi163 bash[27500]: audit 2023-10-09T08:53:30.578138+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.163:0/2958603687' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T08:53:31.857 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:31 smithi163 bash[27500]: cluster 2023-10-09T08:53:30.578306+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 4s) 2023-10-09T08:53:32.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2023-10-09T08:53:32.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 5, 2023-10-09T08:53:32.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "available": true, 2023-10-09T08:53:32.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T08:53:32.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T08:53:32.496 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2023-10-09T08:53:32.573 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:32 smithi163 bash[27764]: debug 2023-10-09T08:53:32.304+0000 7f99274b0680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:53:32.573 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:32 smithi163 bash[27764]: debug 2023-10-09T08:53:32.408+0000 7f99274b0680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:53:32.874 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:32 smithi163 bash[27764]: debug 2023-10-09T08:53:32.664+0000 7f99274b0680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:53:33.053 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for the mgr to restart... 2023-10-09T08:53:33.054 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mgr epoch 5... 2023-10-09T08:53:33.074 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:32 smithi163 bash[27500]: audit 2023-10-09T08:53:32.493682+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.163:0/1269268301' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T08:53:33.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:33 smithi163 bash[27764]: debug 2023-10-09T08:53:33.804+0000 7f99274b0680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:53:34.403 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:33 smithi163 bash[27764]: debug 2023-10-09T08:53:33.968+0000 7f99274b0680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:53:34.403 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:34 smithi163 bash[27764]: debug 2023-10-09T08:53:34.068+0000 7f99274b0680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:53:34.403 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:34 smithi163 bash[27764]: debug 2023-10-09T08:53:34.176+0000 7f99274b0680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:53:34.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:34 smithi163 bash[27764]: debug 2023-10-09T08:53:34.396+0000 7f99274b0680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:53:35.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:34 smithi163 bash[27764]: debug 2023-10-09T08:53:34.764+0000 7f99274b0680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:53:35.219 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:34 smithi163 bash[27764]: debug 2023-10-09T08:53:34.884+0000 7f99274b0680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:53:36.196 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:35 smithi163 bash[27764]: debug 2023-10-09T08:53:35.748+0000 7f99274b0680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:53:36.196 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:35 smithi163 bash[27764]: debug 2023-10-09T08:53:35.852+0000 7f99274b0680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:53:36.196 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:35 smithi163 bash[27764]: debug 2023-10-09T08:53:35.968+0000 7f99274b0680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:53:36.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:36 smithi163 bash[27764]: debug 2023-10-09T08:53:36.192+0000 7f99274b0680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:53:36.821 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:36 smithi163 bash[27764]: debug 2023-10-09T08:53:36.484+0000 7f99274b0680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:53:36.821 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:36 smithi163 bash[27764]: debug 2023-10-09T08:53:36.648+0000 7f99274b0680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:53:37.140 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:36 smithi163 bash[27764]: debug 2023-10-09T08:53:36.816+0000 7f99274b0680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:53:37.140 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27764]: debug 2023-10-09T08:53:37.028+0000 7f99274b0680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:53:37.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27764]: debug 2023-10-09T08:53:37.136+0000 7f99274b0680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:53:38.190 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2023-10-09T08:53:38.190 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-10-09T08:53:38.191 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T08:53:38.191 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2023-10-09T08:53:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: cluster 2023-10-09T08:53:37.161142+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2023-10-09T08:53:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: cluster 2023-10-09T08:53:37.161576+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2023-10-09T08:53:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: cluster 2023-10-09T08:53:37.168874+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-10-09T08:53:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: cluster 2023-10-09T08:53:37.169002+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00758218s) 2023-10-09T08:53:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.170428+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:53:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.171680+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:53:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.172828+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:53:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.173048+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:53:38.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.173247+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:53:38.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: cluster 2023-10-09T08:53:37.195001+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-10-09T08:53:38.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: cephadm 2023-10-09T08:53:37.210572+0000 mgr.y (mgr.14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-10-09T08:53:38.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.215895+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:38.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.332196+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:38.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.334339+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:53:38.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.337496+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:53:38.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.346663+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:53:38.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.361610+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:53:38.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:37 smithi163 bash[27500]: audit 2023-10-09T08:53:37.366443+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:53:38.843 INFO:teuthology.orchestra.run.smithi163.stderr:mgr epoch 5 is available 2023-10-09T08:53:38.844 INFO:teuthology.orchestra.run.smithi163.stderr:Setting orchestrator backend to cephadm... 2023-10-09T08:53:39.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:39 smithi163 bash[27500]: audit 2023-10-09T08:53:38.174610+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T08:53:39.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:39 smithi163 bash[27500]: cluster 2023-10-09T08:53:38.175713+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.01429s) 2023-10-09T08:53:39.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:39 smithi163 bash[27500]: audit 2023-10-09T08:53:38.189227+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T08:53:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:41 smithi163 bash[27500]: audit 2023-10-09T08:53:40.030851+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:53:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:41 smithi163 bash[27500]: audit 2023-10-09T08:53:40.039812+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:41 smithi163 bash[27500]: audit 2023-10-09T08:53:40.070445+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:53:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:41 smithi163 bash[27500]: cluster 2023-10-09T08:53:40.175515+0000 mon.a (mon.0) 50 : cluster [DBG] mgrmap e8: y(active, since 3s) 2023-10-09T08:53:41.771 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: value unchanged 2023-10-09T08:53:42.411 INFO:teuthology.orchestra.run.smithi163.stderr:Generating ssh key... 2023-10-09T08:53:43.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:42 smithi163 bash[27500]: audit 2023-10-09T08:53:41.769488+0000 mgr.y (mgr.14116) 5 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:53:44.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27500]: audit 2023-10-09T08:53:43.602565+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:53:44.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27500]: cephadm 2023-10-09T08:53:43.603031+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-10-09T08:53:44.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27500]: audit 2023-10-09T08:53:43.743111+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:44.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27500]: audit 2023-10-09T08:53:43.747717+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:44.219 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: Generating public/private rsa key pair. 2023-10-09T08:53:44.219 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: Your identification has been saved in /tmp/tmpvw9ezg8c/key. 2023-10-09T08:53:44.219 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: Your public key has been saved in /tmp/tmpvw9ezg8c/key.pub. 2023-10-09T08:53:44.219 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: The key fingerprint is: 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: SHA256:vs/xxqxymssD9SElmAb+DQPNlCUQpNq62hd88X4wYdU ceph-097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: The key's randomart image is: 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: +---[RSA 3072]----+ 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: | .*Bo=. . | 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: | o .B.. o E | 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: | . ..o + | 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: | o ..+= . | 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: |. .. .=S+ . | 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: | . o o.+ . | 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: |. o o.o.o | 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: | o . .++oo+ | 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: |+ .. *O+o. | 2023-10-09T08:53:44.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:43 smithi163 bash[27764]: +----[SHA256]-----+ 2023-10-09T08:53:45.558 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNff4oJiSu+tTpHmiZGt7WlYiCULAXVk3/jw+SPjcWFvC41FhV2GcSebIm1kfCsbgTMM4PADtSysm/uQzX32KX/aYTdut76j/qQhGUAo7bIbZotlb7mKfdEjTpnF4l/r4BuQRUIkPv2MCwPfVYCgKg4kSjX6vrtwGiCAcWxltgC/L1HGBjnElV8A4QBm+Vc4jYHKUc6VD6tWa1iZKZrdNup9WVJCPtfxCX+Wn5gtOD5+Y165WP/FTQ0C2bpVhAE/CcA21jGAmN/0SREskQ3LifLBay6k4zLmuvQB0Ukp8EtCkzSl6HitIXZqaB5xV1StyuhnnZQ4wAAOOYVvpNBI5rY0/epWdZCS1ZahtzCCMzlTAmisNwV485jWnPrKGah0P7SaHthBxToq/+1Fkr2HiV8jh9xoDFWY9AmMIaBAeqg8klLHTNaso96jupkkSGwt+/nlrGAWGIlajHAw6kWEWyaF+m6NlzqvRZoMhPEZXX6AffxNtGcBT0V+ZNzvQimTE= ceph-097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:53:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:45 smithi163 bash[27500]: audit 2023-10-09T08:53:45.556341+0000 mgr.y (mgr.14116) 8 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:53:46.180 INFO:teuthology.orchestra.run.smithi163.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T08:53:46.180 INFO:teuthology.orchestra.run.smithi163.stderr:Adding key to root@localhost authorized_keys... 2023-10-09T08:53:46.181 INFO:teuthology.orchestra.run.smithi163.stderr:Adding host smithi163... 2023-10-09T08:53:47.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:47 smithi163 bash[27764]: Warning: Permanently added '172.21.15.163' (ECDSA) to the list of known hosts. 2023-10-09T08:53:48.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:47 smithi163 bash[27500]: audit 2023-10-09T08:53:47.390488+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi163", "addr": "172.21.15.163", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:53:48.220 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: Added host 'smithi163' with addr '172.21.15.163' 2023-10-09T08:53:49.057 INFO:teuthology.orchestra.run.smithi163.stderr:Deploying unmanaged mon service... 2023-10-09T08:53:49.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:49 smithi163 bash[27500]: audit 2023-10-09T08:53:48.218573+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:49.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:49 smithi163 bash[27500]: cephadm 2023-10-09T08:53:48.219332+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi163 2023-10-09T08:53:49.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:49 smithi163 bash[27500]: audit 2023-10-09T08:53:48.263364+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:53:50.698 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: Scheduled mon update... 2023-10-09T08:53:51.336 INFO:teuthology.orchestra.run.smithi163.stderr:Deploying unmanaged mgr service... 2023-10-09T08:53:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:51 smithi163 bash[27500]: audit 2023-10-09T08:53:50.690954+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:53:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:51 smithi163 bash[27500]: cephadm 2023-10-09T08:53:50.692406+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-10-09T08:53:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:51 smithi163 bash[27500]: audit 2023-10-09T08:53:50.696793+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:51 smithi163 bash[27500]: audit 2023-10-09T08:53:50.966801+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:51 smithi163 bash[27500]: audit 2023-10-09T08:53:51.219906+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:52.554 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-10-09T08:53:53.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:53 smithi163 bash[27500]: audit 2023-10-09T08:53:52.545967+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:53:53.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:53 smithi163 bash[27500]: cephadm 2023-10-09T08:53:52.548327+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-10-09T08:53:53.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:53 smithi163 bash[27500]: audit 2023-10-09T08:53:52.553057+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:55.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:55 smithi163 bash[27500]: audit 2023-10-09T08:53:54.427673+0000 mon.a (mon.0) 59 : audit [INF] from='client.? 172.21.15.163:0/463830476' entity='client.admin' 2023-10-09T08:53:56.904 INFO:teuthology.orchestra.run.smithi163.stderr:Enabling the dashboard module... 2023-10-09T08:53:57.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:57 smithi163 bash[27500]: audit 2023-10-09T08:53:56.248172+0000 mon.a (mon.0) 60 : audit [INF] from='client.? 172.21.15.163:0/3980203774' entity='client.admin' 2023-10-09T08:53:57.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:57 smithi163 bash[27500]: audit 2023-10-09T08:53:56.266449+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.14116 172.21.15.163:0/1357828352' entity='mgr.y' 2023-10-09T08:53:58.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:58 smithi163 bash[27500]: cluster 2023-10-09T08:53:57.174585+0000 mgr.y (mgr.14116) 15 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:53:58.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:58 smithi163 bash[27500]: audit 2023-10-09T08:53:58.192776+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.163:0/2289050046' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T08:53:58.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:58 smithi163 bash[27764]: ignoring --setuser ceph since I am not root 2023-10-09T08:53:58.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:58 smithi163 bash[27764]: ignoring --setgroup ceph since I am not root 2023-10-09T08:53:58.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:58 smithi163 bash[27764]: debug 2023-10-09T08:53:58.451+0000 7f5c41fc3680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:53:58.936 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:58 smithi163 bash[27764]: debug 2023-10-09T08:53:58.563+0000 7f5c41fc3680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:53:59.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:59 smithi163 bash[27764]: debug 2023-10-09T08:53:59.027+0000 7f5c41fc3680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:53:59.615 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:59 smithi163 bash[27500]: audit 2023-10-09T08:53:58.254435+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.163:0/2289050046' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T08:53:59.615 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:53:59 smithi163 bash[27500]: cluster 2023-10-09T08:53:58.254568+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 21s) 2023-10-09T08:54:00.302 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2023-10-09T08:54:00.302 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 9, 2023-10-09T08:54:00.303 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "available": true, 2023-10-09T08:54:00.303 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T08:54:00.303 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T08:54:00.303 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2023-10-09T08:54:00.349 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:53:59 smithi163 bash[27764]: debug 2023-10-09T08:53:59.979+0000 7f5c41fc3680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:54:00.349 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:00 smithi163 bash[27764]: debug 2023-10-09T08:54:00.087+0000 7f5c41fc3680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:54:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:00 smithi163 bash[27500]: audit 2023-10-09T08:54:00.301073+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.163:0/45685356' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T08:54:00.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:00 smithi163 bash[27764]: debug 2023-10-09T08:54:00.359+0000 7f5c41fc3680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:54:00.928 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for the mgr to restart... 2023-10-09T08:54:00.928 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mgr epoch 9... 2023-10-09T08:54:01.736 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:01 smithi163 bash[27764]: debug 2023-10-09T08:54:01.571+0000 7f5c41fc3680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:54:02.180 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:01 smithi163 bash[27764]: debug 2023-10-09T08:54:01.735+0000 7f5c41fc3680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:54:02.180 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:01 smithi163 bash[27764]: debug 2023-10-09T08:54:01.831+0000 7f5c41fc3680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:54:02.180 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:01 smithi163 bash[27764]: debug 2023-10-09T08:54:01.943+0000 7f5c41fc3680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:54:02.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:02 smithi163 bash[27764]: debug 2023-10-09T08:54:02.175+0000 7f5c41fc3680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:54:02.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:02 smithi163 bash[27764]: debug 2023-10-09T08:54:02.563+0000 7f5c41fc3680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:54:02.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:02 smithi163 bash[27764]: debug 2023-10-09T08:54:02.691+0000 7f5c41fc3680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:54:03.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:03 smithi163 bash[27764]: debug 2023-10-09T08:54:03.579+0000 7f5c41fc3680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:54:03.969 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:03 smithi163 bash[27764]: debug 2023-10-09T08:54:03.683+0000 7f5c41fc3680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:54:03.969 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:03 smithi163 bash[27764]: debug 2023-10-09T08:54:03.799+0000 7f5c41fc3680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:54:04.324 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:04 smithi163 bash[27764]: debug 2023-10-09T08:54:04.023+0000 7f5c41fc3680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:54:04.655 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:04 smithi163 bash[27764]: debug 2023-10-09T08:54:04.319+0000 7f5c41fc3680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:54:04.655 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:04 smithi163 bash[27764]: debug 2023-10-09T08:54:04.479+0000 7f5c41fc3680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:54:04.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:04 smithi163 bash[27764]: debug 2023-10-09T08:54:04.647+0000 7f5c41fc3680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:54:04.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:04 smithi163 bash[27764]: debug 2023-10-09T08:54:04.863+0000 7f5c41fc3680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:54:05.271 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:04 smithi163 bash[27764]: debug 2023-10-09T08:54:04.975+0000 7f5c41fc3680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:54:05.272 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27764]: Warning: Permanently added '172.21.15.163' (ECDSA) to the list of known hosts. 2023-10-09T08:54:05.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27500]: cluster 2023-10-09T08:54:05.000873+0000 mon.a (mon.0) 66 : cluster [INF] Active manager daemon y restarted 2023-10-09T08:54:05.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27500]: cluster 2023-10-09T08:54:05.001307+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2023-10-09T08:54:05.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27500]: cluster 2023-10-09T08:54:05.007723+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-10-09T08:54:05.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27500]: cluster 2023-10-09T08:54:05.007985+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00684567s) 2023-10-09T08:54:05.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27500]: audit 2023-10-09T08:54:05.009828+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:54:05.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27500]: audit 2023-10-09T08:54:05.011136+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T08:54:05.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27500]: audit 2023-10-09T08:54:05.012090+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T08:54:05.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27500]: audit 2023-10-09T08:54:05.012296+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T08:54:05.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27500]: audit 2023-10-09T08:54:05.012482+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T08:54:05.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:05 smithi163 bash[27500]: cluster 2023-10-09T08:54:05.016071+0000 mon.a (mon.0) 75 : cluster [INF] Manager daemon y is now available 2023-10-09T08:54:06.024 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2023-10-09T08:54:06.025 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-10-09T08:54:06.025 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T08:54:06.025 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2023-10-09T08:54:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:06 smithi163 bash[27500]: audit 2023-10-09T08:54:05.155578+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:06 smithi163 bash[27500]: audit 2023-10-09T08:54:05.160920+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:06 smithi163 bash[27500]: audit 2023-10-09T08:54:05.197832+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T08:54:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:06 smithi163 bash[27500]: audit 2023-10-09T08:54:05.203036+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T08:54:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:06 smithi163 bash[27500]: cluster 2023-10-09T08:54:06.011468+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.01034s) 2023-10-09T08:54:06.641 INFO:teuthology.orchestra.run.smithi163.stderr:mgr epoch 9 is available 2023-10-09T08:54:06.642 INFO:teuthology.orchestra.run.smithi163.stderr:Generating a dashboard self-signed certificate... 2023-10-09T08:54:07.327 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:07 smithi163 bash[27500]: audit 2023-10-09T08:54:06.010784+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T08:54:07.328 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:07 smithi163 bash[27500]: audit 2023-10-09T08:54:06.023185+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T08:54:08.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: Self-signed certificate created 2023-10-09T08:54:08.647 INFO:teuthology.orchestra.run.smithi163.stderr:Creating initial admin user... 2023-10-09T08:54:08.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:08 smithi163 bash[27500]: audit 2023-10-09T08:54:07.875471+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:54:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:08 smithi163 bash[27500]: cluster 2023-10-09T08:54:07.883203+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-10-09T08:54:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:08 smithi163 bash[27500]: audit 2023-10-09T08:54:07.999622+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:08 smithi163 bash[27500]: audit 2023-10-09T08:54:08.002668+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:10.224 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$hEgyJ.TZPXNVV4eWmRXK1.fj55S66NUG1tAF9IAXtAmm0o1ga3AX.", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696841650, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T08:54:10.852 INFO:teuthology.orchestra.run.smithi163.stderr:Fetching dashboard port number... 2023-10-09T08:54:11.507 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:11 smithi163 bash[27500]: audit 2023-10-09T08:54:09.979976+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:54:11.507 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:11 smithi163 bash[27500]: audit 2023-10-09T08:54:10.222142+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:12.190 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 8443 2023-10-09T08:54:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:12 smithi163 bash[27500]: cluster 2023-10-09T08:54:11.228300+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-10-09T08:54:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:12 smithi163 bash[27500]: audit 2023-10-09T08:54:12.016426+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:12 smithi163 bash[27500]: audit 2023-10-09T08:54:12.189012+0000 mon.a (mon.0) 87 : audit [DBG] from='client.? 172.21.15.163:0/617567856' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T08:54:12.825 INFO:teuthology.orchestra.run.smithi163.stderr:firewalld does not appear to be present 2023-10-09T08:54:12.825 INFO:teuthology.orchestra.run.smithi163.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T08:54:12.827 INFO:teuthology.orchestra.run.smithi163.stderr:Ceph Dashboard is now available at: 2023-10-09T08:54:12.828 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:54:12.828 INFO:teuthology.orchestra.run.smithi163.stderr: URL: https://smithi163.front.sepia.ceph.com:8443/ 2023-10-09T08:54:12.828 INFO:teuthology.orchestra.run.smithi163.stderr: User: admin 2023-10-09T08:54:12.828 INFO:teuthology.orchestra.run.smithi163.stderr: Password: 4xv5iv07zw 2023-10-09T08:54:12.828 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:54:12.828 INFO:teuthology.orchestra.run.smithi163.stderr:Enabling autotune for osd_memory_target 2023-10-09T08:54:16.066 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-10-09T08:54:16.633 INFO:teuthology.orchestra.run.smithi163.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr:Or, if you are only running a single cluster on this host: 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr:Please consider enabling telemetry to help improve Ceph: 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr: ceph telemetry on 2023-10-09T08:54:16.634 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:54:16.635 INFO:teuthology.orchestra.run.smithi163.stderr:For more information see: 2023-10-09T08:54:16.635 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:54:16.635 INFO:teuthology.orchestra.run.smithi163.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-10-09T08:54:16.635 INFO:teuthology.orchestra.run.smithi163.stderr: 2023-10-09T08:54:16.635 INFO:teuthology.orchestra.run.smithi163.stderr:Bootstrap complete. 2023-10-09T08:54:16.672 INFO:tasks.cephadm:Fetching config... 2023-10-09T08:54:16.672 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:54:16.673 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T08:54:16.678 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T08:54:16.678 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:54:16.679 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T08:54:16.728 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T08:54:16.728 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:54:16.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T08:54:16.780 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T08:54:16.780 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:54:16.780 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T08:54:16.831 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T08:54:16.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNff4oJiSu+tTpHmiZGt7WlYiCULAXVk3/jw+SPjcWFvC41FhV2GcSebIm1kfCsbgTMM4PADtSysm/uQzX32KX/aYTdut76j/qQhGUAo7bIbZotlb7mKfdEjTpnF4l/r4BuQRUIkPv2MCwPfVYCgKg4kSjX6vrtwGiCAcWxltgC/L1HGBjnElV8A4QBm+Vc4jYHKUc6VD6tWa1iZKZrdNup9WVJCPtfxCX+Wn5gtOD5+Y165WP/FTQ0C2bpVhAE/CcA21jGAmN/0SREskQ3LifLBay6k4zLmuvQB0Ukp8EtCkzSl6HitIXZqaB5xV1StyuhnnZQ4wAAOOYVvpNBI5rY0/epWdZCS1ZahtzCCMzlTAmisNwV485jWnPrKGah0P7SaHthBxToq/+1Fkr2HiV8jh9xoDFWY9AmMIaBAeqg8klLHTNaso96jupkkSGwt+/nlrGAWGIlajHAw6kWEWyaF+m6NlzqvRZoMhPEZXX6AffxNtGcBT0V+ZNzvQimTE= ceph-097c0480-6681-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T08:54:16.893 INFO:teuthology.orchestra.run.smithi163.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNff4oJiSu+tTpHmiZGt7WlYiCULAXVk3/jw+SPjcWFvC41FhV2GcSebIm1kfCsbgTMM4PADtSysm/uQzX32KX/aYTdut76j/qQhGUAo7bIbZotlb7mKfdEjTpnF4l/r4BuQRUIkPv2MCwPfVYCgKg4kSjX6vrtwGiCAcWxltgC/L1HGBjnElV8A4QBm+Vc4jYHKUc6VD6tWa1iZKZrdNup9WVJCPtfxCX+Wn5gtOD5+Y165WP/FTQ0C2bpVhAE/CcA21jGAmN/0SREskQ3LifLBay6k4zLmuvQB0Ukp8EtCkzSl6HitIXZqaB5xV1StyuhnnZQ4wAAOOYVvpNBI5rY0/epWdZCS1ZahtzCCMzlTAmisNwV485jWnPrKGah0P7SaHthBxToq/+1Fkr2HiV8jh9xoDFWY9AmMIaBAeqg8klLHTNaso96jupkkSGwt+/nlrGAWGIlajHAw6kWEWyaF+m6NlzqvRZoMhPEZXX6AffxNtGcBT0V+ZNzvQimTE= ceph-097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:54:16.900 DEBUG:teuthology.orchestra.run.smithi181:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNff4oJiSu+tTpHmiZGt7WlYiCULAXVk3/jw+SPjcWFvC41FhV2GcSebIm1kfCsbgTMM4PADtSysm/uQzX32KX/aYTdut76j/qQhGUAo7bIbZotlb7mKfdEjTpnF4l/r4BuQRUIkPv2MCwPfVYCgKg4kSjX6vrtwGiCAcWxltgC/L1HGBjnElV8A4QBm+Vc4jYHKUc6VD6tWa1iZKZrdNup9WVJCPtfxCX+Wn5gtOD5+Y165WP/FTQ0C2bpVhAE/CcA21jGAmN/0SREskQ3LifLBay6k4zLmuvQB0Ukp8EtCkzSl6HitIXZqaB5xV1StyuhnnZQ4wAAOOYVvpNBI5rY0/epWdZCS1ZahtzCCMzlTAmisNwV485jWnPrKGah0P7SaHthBxToq/+1Fkr2HiV8jh9xoDFWY9AmMIaBAeqg8klLHTNaso96jupkkSGwt+/nlrGAWGIlajHAw6kWEWyaF+m6NlzqvRZoMhPEZXX6AffxNtGcBT0V+ZNzvQimTE= ceph-097c0480-6681-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T08:54:16.926 INFO:teuthology.orchestra.run.smithi181.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNff4oJiSu+tTpHmiZGt7WlYiCULAXVk3/jw+SPjcWFvC41FhV2GcSebIm1kfCsbgTMM4PADtSysm/uQzX32KX/aYTdut76j/qQhGUAo7bIbZotlb7mKfdEjTpnF4l/r4BuQRUIkPv2MCwPfVYCgKg4kSjX6vrtwGiCAcWxltgC/L1HGBjnElV8A4QBm+Vc4jYHKUc6VD6tWa1iZKZrdNup9WVJCPtfxCX+Wn5gtOD5+Y165WP/FTQ0C2bpVhAE/CcA21jGAmN/0SREskQ3LifLBay6k4zLmuvQB0Ukp8EtCkzSl6HitIXZqaB5xV1StyuhnnZQ4wAAOOYVvpNBI5rY0/epWdZCS1ZahtzCCMzlTAmisNwV485jWnPrKGah0P7SaHthBxToq/+1Fkr2HiV8jh9xoDFWY9AmMIaBAeqg8klLHTNaso96jupkkSGwt+/nlrGAWGIlajHAw6kWEWyaF+m6NlzqvRZoMhPEZXX6AffxNtGcBT0V+ZNzvQimTE= ceph-097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:54:16.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T08:54:17.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:17 smithi163 bash[27500]: audit 2023-10-09T08:54:16.059539+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.163:0/1782718224' entity='client.admin' 2023-10-09T08:54:22.618 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T08:54:22.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T08:54:22.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:22 smithi163 bash[27500]: audit 2023-10-09T08:54:21.870176+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.163:0/1816416804' entity='client.admin' 2023-10-09T08:54:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:24 smithi163 bash[27500]: audit 2023-10-09T08:54:23.457723+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:24 smithi163 bash[27500]: audit 2023-10-09T08:54:23.775163+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi163", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:54:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:24 smithi163 bash[27500]: audit 2023-10-09T08:54:23.780476+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:24 smithi163 bash[27500]: audit 2023-10-09T08:54:23.783672+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:24 smithi163 bash[27500]: audit 2023-10-09T08:54:23.788828+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:25 smithi163 bash[27500]: cluster 2023-10-09T08:54:25.013981+0000 mgr.y (mgr.14146) 5 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:26.286 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi181 2023-10-09T08:54:26.287 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:54:26.287 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/etc/ceph/ceph.conf 2023-10-09T08:54:26.294 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:54:26.294 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:54:26.344 INFO:tasks.cephadm:Adding host smithi181 to orchestrator... 2023-10-09T08:54:26.345 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi181 2023-10-09T08:54:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:26 smithi163 bash[27500]: audit 2023-10-09T08:54:25.602120+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:26 smithi163 bash[27500]: audit 2023-10-09T08:54:25.607779+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:26 smithi163 bash[27500]: audit 2023-10-09T08:54:25.610908+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:54:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:26 smithi163 bash[27500]: audit 2023-10-09T08:54:25.612173+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:54:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:26 smithi163 bash[27500]: cephadm 2023-10-09T08:54:25.614023+0000 mgr.y (mgr.14146) 7 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T08:54:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:26 smithi163 bash[27500]: cephadm 2023-10-09T08:54:25.633336+0000 mgr.y (mgr.14146) 8 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:54:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:26 smithi163 bash[27500]: audit 2023-10-09T08:54:25.664385+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:26 smithi163 bash[27500]: audit 2023-10-09T08:54:25.668221+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:27.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:27 smithi163 bash[27500]: cluster 2023-10-09T08:54:27.014474+0000 mgr.y (mgr.14146) 9 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:29.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:29 smithi163 bash[27764]: Warning: Permanently added 'smithi181,172.21.15.181' (ECDSA) to the list of known hosts. 2023-10-09T08:54:30.126 INFO:teuthology.orchestra.run.smithi163.stdout:Added host 'smithi181' with addr '172.21.15.181' 2023-10-09T08:54:30.379 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:30 smithi163 bash[27764]: Warning: Permanently added '172.21.15.181' (ECDSA) to the list of known hosts. 2023-10-09T08:54:30.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:30 smithi163 bash[27500]: cluster 2023-10-09T08:54:29.014956+0000 mgr.y (mgr.14146) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:30.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:30 smithi163 bash[27500]: audit 2023-10-09T08:54:29.190665+0000 mgr.y (mgr.14146) 11 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi181", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:54:30.743 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T08:54:31.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:31 smithi163 bash[27500]: audit 2023-10-09T08:54:30.124642+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:31.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:31 smithi163 bash[27500]: cephadm 2023-10-09T08:54:30.125385+0000 mgr.y (mgr.14146) 12 : cephadm [INF] Added host smithi181 2023-10-09T08:54:31.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:31 smithi163 bash[27500]: audit 2023-10-09T08:54:30.145783+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:31.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:31 smithi163 bash[27500]: audit 2023-10-09T08:54:30.704558+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:32 smithi163 bash[27500]: cluster 2023-10-09T08:54:31.015433+0000 mgr.y (mgr.14146) 13 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:32 smithi163 bash[27500]: audit 2023-10-09T08:54:31.964423+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:33.645 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T08:54:33.645 INFO:teuthology.orchestra.run.smithi163.stdout:[{"addr": "172.21.15.163", "hostname": "smithi163", "labels": [], "status": ""}, {"addr": "172.21.15.181", "hostname": "smithi181", "labels": [], "status": ""}] 2023-10-09T08:54:34.212 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T08:54:34.212 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T08:54:34.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:34 smithi163 bash[27500]: cluster 2023-10-09T08:54:33.015871+0000 mgr.y (mgr.14146) 14 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:35 smithi163 bash[27500]: audit 2023-10-09T08:54:33.642521+0000 mgr.y (mgr.14146) 15 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T08:54:36.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:36 smithi163 bash[27500]: cluster 2023-10-09T08:54:35.016298+0000 mgr.y (mgr.14146) 16 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:37.901 INFO:teuthology.orchestra.run.smithi163.stderr:adjusted tunables profile to default 2023-10-09T08:54:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:37 smithi163 bash[27500]: audit 2023-10-09T08:54:36.932778+0000 mon.a (mon.0) 105 : audit [INF] from='client.? 172.21.15.163:0/752582445' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T08:54:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:37 smithi163 bash[27500]: cluster 2023-10-09T08:54:37.016722+0000 mgr.y (mgr.14146) 17 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:37 smithi163 bash[27500]: audit 2023-10-09T08:54:37.043097+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:37 smithi163 bash[27500]: audit 2023-10-09T08:54:37.349266+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:37 smithi163 bash[27500]: audit 2023-10-09T08:54:37.350835+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi181", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:54:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:37 smithi163 bash[27500]: audit 2023-10-09T08:54:37.352677+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:54:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:37 smithi163 bash[27500]: audit 2023-10-09T08:54:37.353984+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:54:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:37 smithi163 bash[27500]: audit 2023-10-09T08:54:37.407458+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:37 smithi163 bash[27500]: audit 2023-10-09T08:54:37.414052+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:38.500 INFO:tasks.cephadm:Adding mon.a on smithi163 2023-10-09T08:54:38.500 INFO:tasks.cephadm:Adding mon.c on smithi163 2023-10-09T08:54:38.500 INFO:tasks.cephadm:Adding mon.b on smithi181 2023-10-09T08:54:38.500 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi181:172.21.15.181=b' 2023-10-09T08:54:39.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:38 smithi163 bash[27500]: cephadm 2023-10-09T08:54:37.356314+0000 mgr.y (mgr.14146) 18 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T08:54:39.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:38 smithi163 bash[27500]: cephadm 2023-10-09T08:54:37.377562+0000 mgr.y (mgr.14146) 19 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:54:39.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:38 smithi163 bash[27500]: audit 2023-10-09T08:54:37.895206+0000 mon.a (mon.0) 113 : audit [INF] from='client.? 172.21.15.163:0/752582445' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T08:54:39.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:38 smithi163 bash[27500]: cluster 2023-10-09T08:54:37.895275+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:54:40.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:39 smithi163 bash[27500]: cluster 2023-10-09T08:54:39.017059+0000 mgr.y (mgr.14146) 20 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:41.144 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled mon update... 2023-10-09T08:54:41.979 DEBUG:teuthology.orchestra.run.smithi163:mon.c> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T08:54:41.981 DEBUG:teuthology.orchestra.run.smithi181:mon.b> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T08:54:41.984 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T08:54:41.986 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T08:54:42.243 INFO:journalctl@ceph.mon.c.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:54:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: cluster 2023-10-09T08:54:41.017436+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: audit 2023-10-09T08:54:41.132984+0000 mgr.y (mgr.14146) 22 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi181:172.21.15.181=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:54:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: cephadm 2023-10-09T08:54:41.139133+0000 mgr.y (mgr.14146) 23 : cephadm [INF] Saving service mon spec with placement smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi181:172.21.15.181=b;count:3 2023-10-09T08:54:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: audit 2023-10-09T08:54:41.143284+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: audit 2023-10-09T08:54:41.190540+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: audit 2023-10-09T08:54:41.194164+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:54:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: audit 2023-10-09T08:54:41.195979+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:54:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: audit 2023-10-09T08:54:41.201871+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:42.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: audit 2023-10-09T08:54:41.226216+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:54:42.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: audit 2023-10-09T08:54:41.227815+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:54:42.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:42 smithi163 bash[27500]: cephadm 2023-10-09T08:54:41.229152+0000 mgr.y (mgr.14146) 24 : cephadm [INF] Deploying daemon mon.b on smithi181 2023-10-09T08:54:43.138 INFO:journalctl@ceph.mon.b.smithi181.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:54:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:44 smithi163 bash[27500]: cluster 2023-10-09T08:54:43.017924+0000 mgr.y (mgr.14146) 25 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:46 smithi163 bash[27500]: cluster 2023-10-09T08:54:45.018481+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:46.552 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T08:54:46.553 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","modified":"2023-10-09T08:53:07.197738Z","created":"2023-10-09T08:53:07.197738Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3300","nonce":0},{"type":"v1","addr":"172.21.15.163:6789","nonce":0}]},"addr":"172.21.15.163:6789/0","public_addr":"172.21.15.163:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T08:54:46.555 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2023-10-09T08:54:47.423 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:47 smithi163 bash[27500]: audit 2023-10-09T08:54:46.551930+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.181:0/3253774922' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T08:54:48.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:47 smithi181 bash[27712]: debug 2023-10-09T08:54:47.981+0000 7fde83035700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T08:54:48.149 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T08:54:48.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T08:54:51.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:54:51 smithi163 bash[33912]: debug 2023-10-09T08:54:51.726+0000 7f84df46a700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T08:54:53.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:47.993718+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:54:53.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: cluster 2023-10-09T08:54:47.994019+0000 mon.a (mon.0) 128 : cluster [INF] mon.a calling monitor election 2023-10-09T08:54:53.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:48.016470+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:48.989489+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: cluster 2023-10-09T08:54:49.019332+0000 mgr.y (mgr.14146) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:49.989099+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: cluster 2023-10-09T08:54:49.990186+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:50.989319+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: cluster 2023-10-09T08:54:51.019683+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:51.732093+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:51.989656+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:52.732631+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:52.989578+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: cluster 2023-10-09T08:54:53.036430+0000 mon.a (mon.0) 137 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T08:54:53.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: cluster 2023-10-09T08:54:53.043162+0000 mon.a (mon.0) 138 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} removed_ranks: {} 2023-10-09T08:54:53.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: cluster 2023-10-09T08:54:53.063752+0000 mon.a (mon.0) 139 : cluster [DBG] fsmap 2023-10-09T08:54:53.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: cluster 2023-10-09T08:54:53.063813+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:54:53.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: cluster 2023-10-09T08:54:53.064153+0000 mon.a (mon.0) 141 : cluster [DBG] mgrmap e13: y(active, since 48s) 2023-10-09T08:54:53.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: cluster 2023-10-09T08:54:53.070274+0000 mon.a (mon.0) 142 : cluster [INF] overall HEALTH_OK 2023-10-09T08:54:53.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:53.075265+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:53.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27500]: audit 2023-10-09T08:54:53.076968+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:47.993718+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:54:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: cluster 2023-10-09T08:54:47.994019+0000 mon.a (mon.0) 128 : cluster [INF] mon.a calling monitor election 2023-10-09T08:54:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:48.016470+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:48.989489+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: cluster 2023-10-09T08:54:49.019332+0000 mgr.y (mgr.14146) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:49.989099+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: cluster 2023-10-09T08:54:49.990186+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T08:54:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:50.989319+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: cluster 2023-10-09T08:54:51.019683+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:51.732093+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:51.989656+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:52.732631+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:53.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:52.989578+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:53.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: cluster 2023-10-09T08:54:53.036430+0000 mon.a (mon.0) 137 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T08:54:53.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: cluster 2023-10-09T08:54:53.043162+0000 mon.a (mon.0) 138 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} removed_ranks: {} 2023-10-09T08:54:53.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: cluster 2023-10-09T08:54:53.063752+0000 mon.a (mon.0) 139 : cluster [DBG] fsmap 2023-10-09T08:54:53.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: cluster 2023-10-09T08:54:53.063813+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:54:53.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: cluster 2023-10-09T08:54:53.064153+0000 mon.a (mon.0) 141 : cluster [DBG] mgrmap e13: y(active, since 48s) 2023-10-09T08:54:53.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: cluster 2023-10-09T08:54:53.070274+0000 mon.a (mon.0) 142 : cluster [INF] overall HEALTH_OK 2023-10-09T08:54:53.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:53.075265+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:53.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:53 smithi181 bash[27712]: audit 2023-10-09T08:54:53.076968+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:54:54.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 08:54:53 smithi163 bash[27764]: debug 2023-10-09T08:54:53.986+0000 7f5bf57c8700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T08:54:58.786 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T08:54:58.787 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":3,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","modified":"2023-10-09T08:54:53.733408Z","created":"2023-10-09T08:53:07.197738Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3300","nonce":0},{"type":"v1","addr":"172.21.15.163:6789","nonce":0}]},"addr":"172.21.15.163:6789/0","public_addr":"172.21.15.163:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:3300","nonce":0},{"type":"v1","addr":"172.21.15.181:6789","nonce":0}]},"addr":"172.21.15.181:6789/0","public_addr":"172.21.15.181:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3301","nonce":0},{"type":"v1","addr":"172.21.15.163:6790","nonce":0}]},"addr":"172.21.15.163:6790/0","public_addr":"172.21.15.163:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T08:54:58.792 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 3 2023-10-09T08:54:59.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:53.739544+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:54:59.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:53.739979+0000 mon.a (mon.0) 147 : cluster [INF] mon.a calling monitor election 2023-10-09T08:54:59.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:53.740000+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T08:54:59.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:53.760304+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:59.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:53.760721+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:53.932389+0000 mon.a (mon.0) 150 : audit [DBG] from='client.? 172.21.15.181:0/616459754' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T08:54:59.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:54.733277+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:55.020537+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:59.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:55.733409+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:56.733409+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:57.020931+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:59.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:57.733559+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:58.733677+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:58.760320+0000 mon.a (mon.0) 156 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T08:54:59.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:58.765941+0000 mon.a (mon.0) 157 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T08:54:59.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:58.783815+0000 mon.a (mon.0) 158 : cluster [DBG] fsmap 2023-10-09T08:54:59.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:58.783874+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:54:59.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:58.784241+0000 mon.a (mon.0) 160 : cluster [DBG] mgrmap e13: y(active, since 53s) 2023-10-09T08:54:59.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:58.784561+0000 mon.a (mon.0) 161 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T08:54:59.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:58.791969+0000 mon.a (mon.0) 162 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T08:54:59.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:58.792013+0000 mon.a (mon.0) 163 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T08:54:59.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: cluster 2023-10-09T08:54:58.792070+0000 mon.a (mon.0) 164 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] is down (out of quorum) 2023-10-09T08:54:59.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:58.794957+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:59.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:58.798454+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:59.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:58.801424+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:59.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:58.802489+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:54:59.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:58.803903+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:54:59.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:58.829986+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:59.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:58 smithi163 bash[27500]: audit 2023-10-09T08:54:58.834860+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:59.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:53.739544+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:53.739979+0000 mon.a (mon.0) 147 : cluster [INF] mon.a calling monitor election 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:53.740000+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:53.760304+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:53.760721+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:53.932389+0000 mon.a (mon.0) 150 : audit [DBG] from='client.? 172.21.15.181:0/616459754' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:54.733277+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:55.020537+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:55.733409+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:56.733409+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:57.020931+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:54:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:57.733559+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:58.733677+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:58.760320+0000 mon.a (mon.0) 156 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:58.765941+0000 mon.a (mon.0) 157 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:58.783815+0000 mon.a (mon.0) 158 : cluster [DBG] fsmap 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:58.783874+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:58.784241+0000 mon.a (mon.0) 160 : cluster [DBG] mgrmap e13: y(active, since 53s) 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:58.784561+0000 mon.a (mon.0) 161 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:58.791969+0000 mon.a (mon.0) 162 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:58.792013+0000 mon.a (mon.0) 163 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: cluster 2023-10-09T08:54:58.792070+0000 mon.a (mon.0) 164 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] is down (out of quorum) 2023-10-09T08:54:59.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:58.794957+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:59.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:58.798454+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:59.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:58.801424+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:59.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:58.802489+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:54:59.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:58.803903+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:54:59.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:58.829986+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:59.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:58 smithi181 bash[27712]: audit 2023-10-09T08:54:58.834860+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:54:59.421 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T08:54:59.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T08:54:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:59 smithi163 bash[27500]: cephadm 2023-10-09T08:54:58.805544+0000 mgr.y (mgr.14146) 34 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T08:55:00.008 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:59 smithi163 bash[27500]: cephadm 2023-10-09T08:54:58.805932+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T08:55:00.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:59 smithi163 bash[27500]: audit 2023-10-09T08:54:58.845439+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:00.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:59 smithi163 bash[27500]: cephadm 2023-10-09T08:54:58.847345+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T08:55:00.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:59 smithi163 bash[27500]: audit 2023-10-09T08:54:58.847770+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:55:00.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:59 smithi163 bash[27500]: audit 2023-10-09T08:54:58.848714+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:55:00.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:59 smithi163 bash[27500]: audit 2023-10-09T08:54:58.849638+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:00.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:59 smithi163 bash[27500]: cephadm 2023-10-09T08:54:58.850479+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring daemon mon.a on smithi163 2023-10-09T08:55:00.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:59 smithi163 bash[27500]: cluster 2023-10-09T08:54:59.021335+0000 mgr.y (mgr.14146) 38 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:00.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:54:59 smithi163 bash[27500]: audit 2023-10-09T08:54:59.733572+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:55:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:59 smithi181 bash[27712]: cephadm 2023-10-09T08:54:58.805544+0000 mgr.y (mgr.14146) 34 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T08:55:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:59 smithi181 bash[27712]: cephadm 2023-10-09T08:54:58.805932+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T08:55:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:59 smithi181 bash[27712]: audit 2023-10-09T08:54:58.845439+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:59 smithi181 bash[27712]: cephadm 2023-10-09T08:54:58.847345+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T08:55:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:59 smithi181 bash[27712]: audit 2023-10-09T08:54:58.847770+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:55:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:59 smithi181 bash[27712]: audit 2023-10-09T08:54:58.848714+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:55:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:59 smithi181 bash[27712]: audit 2023-10-09T08:54:58.849638+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:59 smithi181 bash[27712]: cephadm 2023-10-09T08:54:58.850479+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring daemon mon.a on smithi163 2023-10-09T08:55:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:59 smithi181 bash[27712]: cluster 2023-10-09T08:54:59.021335+0000 mgr.y (mgr.14146) 38 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:54:59 smithi181 bash[27712]: audit 2023-10-09T08:54:59.733572+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:55:00.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:54:55.734999+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T08:55:00.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.761959+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-10-09T08:55:00.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.762184+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-10-09T08:55:00.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.762703+0000 mon.a (mon.0) 178 : cluster [INF] mon.a calling monitor election 2023-10-09T08:55:00.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.787760+0000 mon.a (mon.0) 179 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:55:00.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.791455+0000 mon.a (mon.0) 180 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T08:55:00.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.791876+0000 mon.a (mon.0) 181 : cluster [DBG] fsmap 2023-10-09T08:55:00.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.791907+0000 mon.a (mon.0) 182 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:55:00.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.792106+0000 mon.a (mon.0) 183 : cluster [DBG] mgrmap e13: y(active, since 55s) 2023-10-09T08:55:00.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.792244+0000 mon.a (mon.0) 184 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T08:55:00.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.792263+0000 mon.a (mon.0) 185 : cluster [INF] Cluster is now healthy 2023-10-09T08:55:00.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[27500]: cluster 2023-10-09T08:55:00.795702+0000 mon.a (mon.0) 186 : cluster [INF] overall HEALTH_OK 2023-10-09T08:55:00.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cephadm 2023-10-09T08:54:58.805544+0000 mgr.y (mgr.14146) 34 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T08:55:00.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cephadm 2023-10-09T08:54:58.805932+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T08:55:00.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: audit 2023-10-09T08:54:58.845439+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:00.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cephadm 2023-10-09T08:54:58.847345+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T08:55:00.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: audit 2023-10-09T08:54:58.847770+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:55:00.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: audit 2023-10-09T08:54:58.848714+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:55:00.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: audit 2023-10-09T08:54:58.849638+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cephadm 2023-10-09T08:54:58.850479+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring daemon mon.a on smithi163 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:54:59.021335+0000 mgr.y (mgr.14146) 38 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: audit 2023-10-09T08:54:59.733572+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:54:55.734999+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.761959+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.762184+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.762703+0000 mon.a (mon.0) 178 : cluster [INF] mon.a calling monitor election 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.787760+0000 mon.a (mon.0) 179 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.791455+0000 mon.a (mon.0) 180 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.791876+0000 mon.a (mon.0) 181 : cluster [DBG] fsmap 2023-10-09T08:55:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.791907+0000 mon.a (mon.0) 182 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:55:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.792106+0000 mon.a (mon.0) 183 : cluster [DBG] mgrmap e13: y(active, since 55s) 2023-10-09T08:55:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.792244+0000 mon.a (mon.0) 184 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T08:55:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.792263+0000 mon.a (mon.0) 185 : cluster [INF] Cluster is now healthy 2023-10-09T08:55:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:00 smithi163 bash[33912]: cluster 2023-10-09T08:55:00.795702+0000 mon.a (mon.0) 186 : cluster [INF] overall HEALTH_OK 2023-10-09T08:55:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:54:55.734999+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T08:55:01.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.761959+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-10-09T08:55:01.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.762184+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-10-09T08:55:01.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.762703+0000 mon.a (mon.0) 178 : cluster [INF] mon.a calling monitor election 2023-10-09T08:55:01.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.787760+0000 mon.a (mon.0) 179 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T08:55:01.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.791455+0000 mon.a (mon.0) 180 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T08:55:01.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.791876+0000 mon.a (mon.0) 181 : cluster [DBG] fsmap 2023-10-09T08:55:01.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.791907+0000 mon.a (mon.0) 182 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T08:55:01.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.792106+0000 mon.a (mon.0) 183 : cluster [DBG] mgrmap e13: y(active, since 55s) 2023-10-09T08:55:01.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.792244+0000 mon.a (mon.0) 184 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T08:55:01.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.792263+0000 mon.a (mon.0) 185 : cluster [INF] Cluster is now healthy 2023-10-09T08:55:01.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:00 smithi181 bash[27712]: cluster 2023-10-09T08:55:00.795702+0000 mon.a (mon.0) 186 : cluster [INF] overall HEALTH_OK 2023-10-09T08:55:02.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:01 smithi181 bash[27712]: cluster 2023-10-09T08:55:01.021768+0000 mgr.y (mgr.14146) 39 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:02.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:01 smithi181 bash[27712]: audit 2023-10-09T08:55:01.712172+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:02.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:01 smithi181 bash[27712]: audit 2023-10-09T08:55:01.713719+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:55:02.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:01 smithi181 bash[27712]: audit 2023-10-09T08:55:01.714648+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:55:02.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:01 smithi181 bash[27712]: audit 2023-10-09T08:55:01.715485+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:02.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:01 smithi181 bash[27712]: audit 2023-10-09T08:55:01.733851+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:55:02.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[27500]: cluster 2023-10-09T08:55:01.021768+0000 mgr.y (mgr.14146) 39 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:02.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[27500]: audit 2023-10-09T08:55:01.712172+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:02.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[27500]: audit 2023-10-09T08:55:01.713719+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:55:02.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[27500]: audit 2023-10-09T08:55:01.714648+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:55:02.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[27500]: audit 2023-10-09T08:55:01.715485+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:02.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[27500]: audit 2023-10-09T08:55:01.733851+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:55:02.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[33912]: cluster 2023-10-09T08:55:01.021768+0000 mgr.y (mgr.14146) 39 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:02.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[33912]: audit 2023-10-09T08:55:01.712172+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:02.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[33912]: audit 2023-10-09T08:55:01.713719+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:55:02.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[33912]: audit 2023-10-09T08:55:01.714648+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:55:02.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[33912]: audit 2023-10-09T08:55:01.715485+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:02.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:01 smithi163 bash[33912]: audit 2023-10-09T08:55:01.733851+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T08:55:03.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:02 smithi163 bash[27500]: cephadm 2023-10-09T08:55:01.713136+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T08:55:03.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:02 smithi163 bash[27500]: cephadm 2023-10-09T08:55:01.716259+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.c on smithi163 2023-10-09T08:55:03.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:02 smithi163 bash[33912]: cephadm 2023-10-09T08:55:01.713136+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T08:55:03.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:02 smithi163 bash[33912]: cephadm 2023-10-09T08:55:01.716259+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.c on smithi163 2023-10-09T08:55:03.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:02 smithi181 bash[27712]: cephadm 2023-10-09T08:55:01.713136+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T08:55:03.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:02 smithi181 bash[27712]: cephadm 2023-10-09T08:55:01.716259+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.c on smithi163 2023-10-09T08:55:03.447 INFO:teuthology.orchestra.run.smithi163.stdout:# minimal ceph.conf for 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:55:03.447 INFO:teuthology.orchestra.run.smithi163.stdout:[global] 2023-10-09T08:55:03.447 INFO:teuthology.orchestra.run.smithi163.stdout: fsid = 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T08:55:03.447 INFO:teuthology.orchestra.run.smithi163.stdout: mon_host = [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] 2023-10-09T08:55:04.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:03 smithi163 bash[33912]: cluster 2023-10-09T08:55:03.022151+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:04.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:03 smithi163 bash[33912]: audit 2023-10-09T08:55:03.446821+0000 mon.a (mon.0) 192 : audit [DBG] from='client.? 172.21.15.163:0/2072208552' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:04.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:03 smithi163 bash[27500]: cluster 2023-10-09T08:55:03.022151+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:04.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:03 smithi163 bash[27500]: audit 2023-10-09T08:55:03.446821+0000 mon.a (mon.0) 192 : audit [DBG] from='client.? 172.21.15.163:0/2072208552' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:04.223 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:03 smithi181 bash[27712]: cluster 2023-10-09T08:55:03.022151+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:04.223 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:03 smithi181 bash[27712]: audit 2023-10-09T08:55:03.446821+0000 mon.a (mon.0) 192 : audit [DBG] from='client.? 172.21.15.163:0/2072208552' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:04.364 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T08:55:04.364 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:55:04.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T08:55:04.378 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:55:04.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:55:04.438 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:55:04.438 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T08:55:04.453 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:55:04.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T08:55:04.510 INFO:tasks.cephadm:Adding mgr.y on smithi163 2023-10-09T08:55:04.510 INFO:tasks.cephadm:Adding mgr.x on smithi181 2023-10-09T08:55:04.510 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi163=y;smithi181=x' 2023-10-09T08:55:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:05 smithi181 bash[27712]: audit 2023-10-09T08:55:04.013869+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:05 smithi181 bash[27712]: cephadm 2023-10-09T08:55:04.014673+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T08:55:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:05 smithi181 bash[27712]: audit 2023-10-09T08:55:04.014981+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:55:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:05 smithi181 bash[27712]: audit 2023-10-09T08:55:04.015947+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:55:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:05 smithi181 bash[27712]: audit 2023-10-09T08:55:04.016943+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:05 smithi181 bash[27712]: cephadm 2023-10-09T08:55:04.017703+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.b on smithi181 2023-10-09T08:55:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[27500]: audit 2023-10-09T08:55:04.013869+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[27500]: cephadm 2023-10-09T08:55:04.014673+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T08:55:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[27500]: audit 2023-10-09T08:55:04.014981+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:55:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[27500]: audit 2023-10-09T08:55:04.015947+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:55:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[27500]: audit 2023-10-09T08:55:04.016943+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[27500]: cephadm 2023-10-09T08:55:04.017703+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.b on smithi181 2023-10-09T08:55:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[33912]: audit 2023-10-09T08:55:04.013869+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[33912]: cephadm 2023-10-09T08:55:04.014673+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T08:55:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[33912]: audit 2023-10-09T08:55:04.014981+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T08:55:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[33912]: audit 2023-10-09T08:55:04.015947+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T08:55:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[33912]: audit 2023-10-09T08:55:04.016943+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:05 smithi163 bash[33912]: cephadm 2023-10-09T08:55:04.017703+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.b on smithi181 2023-10-09T08:55:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:06 smithi181 bash[27712]: cluster 2023-10-09T08:55:05.022465+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:06 smithi163 bash[27500]: cluster 2023-10-09T08:55:05.022465+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:06 smithi163 bash[33912]: cluster 2023-10-09T08:55:05.022465+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:07 smithi163 bash[27500]: audit 2023-10-09T08:55:06.172068+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:07 smithi163 bash[27500]: audit 2023-10-09T08:55:06.174035+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:07 smithi163 bash[27500]: audit 2023-10-09T08:55:06.177049+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:07 smithi163 bash[27500]: audit 2023-10-09T08:55:06.178478+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:07 smithi163 bash[27500]: audit 2023-10-09T08:55:06.184797+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:07 smithi163 bash[33912]: audit 2023-10-09T08:55:06.172068+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:07.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:07 smithi163 bash[33912]: audit 2023-10-09T08:55:06.174035+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:07.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:07 smithi163 bash[33912]: audit 2023-10-09T08:55:06.177049+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:07.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:07 smithi163 bash[33912]: audit 2023-10-09T08:55:06.178478+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:07.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:07 smithi163 bash[33912]: audit 2023-10-09T08:55:06.184797+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:07 smithi181 bash[27712]: audit 2023-10-09T08:55:06.172068+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:07 smithi181 bash[27712]: audit 2023-10-09T08:55:06.174035+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:07 smithi181 bash[27712]: audit 2023-10-09T08:55:06.177049+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:07 smithi181 bash[27712]: audit 2023-10-09T08:55:06.178478+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:07 smithi181 bash[27712]: audit 2023-10-09T08:55:06.184797+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:07.646 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled mgr update... 2023-10-09T08:55:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[27500]: cluster 2023-10-09T08:55:07.022937+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[27500]: audit 2023-10-09T08:55:07.644647+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[27500]: audit 2023-10-09T08:55:07.656258+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[27500]: audit 2023-10-09T08:55:07.659241+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[27500]: audit 2023-10-09T08:55:07.660752+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[27500]: audit 2023-10-09T08:55:07.668474+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[27500]: audit 2023-10-09T08:55:07.670279+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:55:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[27500]: audit 2023-10-09T08:55:07.674836+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T08:55:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[27500]: audit 2023-10-09T08:55:07.676131+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:55:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[27500]: audit 2023-10-09T08:55:07.677484+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[33912]: cluster 2023-10-09T08:55:07.022937+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[33912]: audit 2023-10-09T08:55:07.644647+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[33912]: audit 2023-10-09T08:55:07.656258+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[33912]: audit 2023-10-09T08:55:07.659241+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[33912]: audit 2023-10-09T08:55:07.660752+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[33912]: audit 2023-10-09T08:55:07.668474+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[33912]: audit 2023-10-09T08:55:07.670279+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:55:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[33912]: audit 2023-10-09T08:55:07.674836+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T08:55:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[33912]: audit 2023-10-09T08:55:07.676131+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:55:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:08 smithi163 bash[33912]: audit 2023-10-09T08:55:07.677484+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:08 smithi181 bash[27712]: cluster 2023-10-09T08:55:07.022937+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:08 smithi181 bash[27712]: audit 2023-10-09T08:55:07.644647+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:08 smithi181 bash[27712]: audit 2023-10-09T08:55:07.656258+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:08 smithi181 bash[27712]: audit 2023-10-09T08:55:07.659241+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:08 smithi181 bash[27712]: audit 2023-10-09T08:55:07.660752+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:08 smithi181 bash[27712]: audit 2023-10-09T08:55:07.668474+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:08 smithi181 bash[27712]: audit 2023-10-09T08:55:07.670279+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:55:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:08 smithi181 bash[27712]: audit 2023-10-09T08:55:07.674836+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T08:55:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:08 smithi181 bash[27712]: audit 2023-10-09T08:55:07.676131+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:55:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:08 smithi181 bash[27712]: audit 2023-10-09T08:55:07.677484+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:08.587 DEBUG:teuthology.orchestra.run.smithi181:mgr.x> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T08:55:08.589 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T08:55:08.589 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T08:55:08.589 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T08:55:08.597 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T08:55:08.597 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_1 2023-10-09T08:55:08.645 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T08:55:08.646 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:55:08.646 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 5h/5d Inode: 492 Links: 1 2023-10-09T08:55:08.646 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:55:08.646 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2023-10-09 08:47:49.452699248 +0000 2023-10-09T08:55:08.646 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2023-10-09 08:47:49.116706997 +0000 2023-10-09T08:55:08.646 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2023-10-09 08:47:49.116706997 +0000 2023-10-09T08:55:08.646 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: - 2023-10-09T08:55:08.646 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T08:55:08.701 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2023-10-09T08:55:08.701 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2023-10-09T08:55:08.701 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000417235 s, 1.2 MB/s 2023-10-09T08:55:08.702 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T08:55:08.755 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_2 2023-10-09T08:55:08.809 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T08:55:08.809 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:55:08.809 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 5h/5d Inode: 495 Links: 1 2023-10-09T08:55:08.809 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:55:08.809 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2023-10-09 08:47:49.984686980 +0000 2023-10-09T08:55:08.810 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2023-10-09 08:47:49.660694451 +0000 2023-10-09T08:55:08.810 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2023-10-09 08:47:49.660694451 +0000 2023-10-09T08:55:08.810 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: - 2023-10-09T08:55:08.810 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T08:55:08.865 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2023-10-09T08:55:08.865 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2023-10-09T08:55:08.865 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000409082 s, 1.3 MB/s 2023-10-09T08:55:08.866 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T08:55:08.915 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_3 2023-10-09T08:55:08.966 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T08:55:08.966 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:55:08.967 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 5h/5d Inode: 503 Links: 1 2023-10-09T08:55:08.967 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:55:08.967 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2023-10-09 08:47:50.512674803 +0000 2023-10-09T08:55:08.967 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2023-10-09 08:47:50.188682276 +0000 2023-10-09T08:55:08.967 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2023-10-09 08:47:50.188682276 +0000 2023-10-09T08:55:08.967 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: - 2023-10-09T08:55:08.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T08:55:09.021 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2023-10-09T08:55:09.021 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2023-10-09T08:55:09.021 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000385842 s, 1.3 MB/s 2023-10-09T08:55:09.022 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T08:55:09.072 INFO:journalctl@ceph.mgr.x.smithi181.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:55:09.075 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_4 2023-10-09T08:55:09.130 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T08:55:09.130 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:55:09.130 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 5h/5d Inode: 510 Links: 1 2023-10-09T08:55:09.130 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:55:09.131 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2023-10-09 08:47:51.072661889 +0000 2023-10-09T08:55:09.131 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2023-10-09 08:47:50.736669638 +0000 2023-10-09T08:55:09.131 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2023-10-09 08:47:50.736669638 +0000 2023-10-09T08:55:09.131 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: - 2023-10-09T08:55:09.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T08:55:09.187 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2023-10-09T08:55:09.187 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2023-10-09T08:55:09.187 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000281764 s, 1.8 MB/s 2023-10-09T08:55:09.188 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T08:55:09.241 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T08:55:09.241 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T08:55:09.249 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T08:55:09.249 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_1 2023-10-09T08:55:09.294 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T08:55:09.295 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:55:09.295 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 490 Links: 1 2023-10-09T08:55:09.295 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:55:09.295 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2023-10-09 08:47:54.717714731 +0000 2023-10-09T08:55:09.295 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2023-10-09 08:47:54.385722384 +0000 2023-10-09T08:55:09.295 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2023-10-09 08:47:54.385722384 +0000 2023-10-09T08:55:09.295 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2023-10-09T08:55:09.299 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T08:55:09.351 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2023-10-09T08:55:09.351 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2023-10-09T08:55:09.351 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000422244 s, 1.2 MB/s 2023-10-09T08:55:09.352 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T08:55:09.403 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_2 2023-10-09T08:55:09.453 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T08:55:09.453 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:55:09.453 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 498 Links: 1 2023-10-09T08:55:09.453 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:55:09.453 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2023-10-09 08:47:55.257702283 +0000 2023-10-09T08:55:09.453 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2023-10-09 08:47:54.925709938 +0000 2023-10-09T08:55:09.453 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2023-10-09 08:47:54.925709938 +0000 2023-10-09T08:55:09.453 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2023-10-09T08:55:09.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T08:55:09.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:09 smithi163 bash[27500]: audit 2023-10-09T08:55:07.634182+0000 mgr.y (mgr.14146) 47 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi163=y;smithi181=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:55:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:09 smithi163 bash[27500]: cephadm 2023-10-09T08:55:07.637838+0000 mgr.y (mgr.14146) 48 : cephadm [INF] Saving service mgr spec with placement smithi163=y;smithi181=x;count:2 2023-10-09T08:55:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:09 smithi163 bash[27500]: cephadm 2023-10-09T08:55:07.678513+0000 mgr.y (mgr.14146) 49 : cephadm [INF] Deploying daemon mgr.x on smithi181 2023-10-09T08:55:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:09 smithi163 bash[33912]: audit 2023-10-09T08:55:07.634182+0000 mgr.y (mgr.14146) 47 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi163=y;smithi181=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:55:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:09 smithi163 bash[33912]: cephadm 2023-10-09T08:55:07.637838+0000 mgr.y (mgr.14146) 48 : cephadm [INF] Saving service mgr spec with placement smithi163=y;smithi181=x;count:2 2023-10-09T08:55:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:09 smithi163 bash[33912]: cephadm 2023-10-09T08:55:07.678513+0000 mgr.y (mgr.14146) 49 : cephadm [INF] Deploying daemon mgr.x on smithi181 2023-10-09T08:55:09.501 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:09 smithi181 bash[27712]: audit 2023-10-09T08:55:07.634182+0000 mgr.y (mgr.14146) 47 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi163=y;smithi181=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:55:09.502 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:09 smithi181 bash[27712]: cephadm 2023-10-09T08:55:07.637838+0000 mgr.y (mgr.14146) 48 : cephadm [INF] Saving service mgr spec with placement smithi163=y;smithi181=x;count:2 2023-10-09T08:55:09.502 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:09 smithi181 bash[27712]: cephadm 2023-10-09T08:55:07.678513+0000 mgr.y (mgr.14146) 49 : cephadm [INF] Deploying daemon mgr.x on smithi181 2023-10-09T08:55:09.506 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2023-10-09T08:55:09.506 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2023-10-09T08:55:09.506 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000396329 s, 1.3 MB/s 2023-10-09T08:55:09.506 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T08:55:09.559 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_3 2023-10-09T08:55:09.612 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T08:55:09.613 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:55:09.613 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 503 Links: 1 2023-10-09T08:55:09.613 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:55:09.613 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2023-10-09 08:47:55.789690022 +0000 2023-10-09T08:55:09.613 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2023-10-09 08:47:55.489696937 +0000 2023-10-09T08:55:09.613 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2023-10-09 08:47:55.489696937 +0000 2023-10-09T08:55:09.613 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2023-10-09T08:55:09.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T08:55:09.672 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2023-10-09T08:55:09.673 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2023-10-09T08:55:09.673 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000398509 s, 1.3 MB/s 2023-10-09T08:55:09.674 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T08:55:09.726 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_4 2023-10-09T08:55:09.776 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T08:55:09.776 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T08:55:09.776 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 510 Links: 1 2023-10-09T08:55:09.776 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T08:55:09.776 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2023-10-09 08:47:56.353677023 +0000 2023-10-09T08:55:09.776 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2023-10-09 08:47:56.025684583 +0000 2023-10-09T08:55:09.777 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2023-10-09 08:47:56.025684583 +0000 2023-10-09T08:55:09.777 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2023-10-09T08:55:09.777 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T08:55:09.832 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2023-10-09T08:55:09.832 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2023-10-09T08:55:09.833 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000406938 s, 1.3 MB/s 2023-10-09T08:55:09.833 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T08:55:09.882 INFO:tasks.cephadm:Deploying osd.0 on smithi163 with /dev/vg_nvme/lv_4... 2023-10-09T08:55:09.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T08:55:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:10 smithi163 bash[27500]: cluster 2023-10-09T08:55:09.023301+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:10.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:10 smithi163 bash[33912]: cluster 2023-10-09T08:55:09.023301+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:10 smithi181 bash[27712]: cluster 2023-10-09T08:55:09.023301+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:11.214 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:11 smithi181 bash[28565]: debug 2023-10-09T08:55:11.061+0000 7fc06c842700 1 -- 172.21.15.181:0/3079317188 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55eea41de4e0 con 0x55eea42d2c00 2023-10-09T08:55:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:11 smithi163 bash[27500]: audit 2023-10-09T08:55:10.205235+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:11 smithi163 bash[27500]: audit 2023-10-09T08:55:10.207402+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:11 smithi163 bash[33912]: audit 2023-10-09T08:55:10.205235+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:11 smithi163 bash[33912]: audit 2023-10-09T08:55:10.207402+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:11 smithi181 bash[27712]: audit 2023-10-09T08:55:10.205235+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:11 smithi181 bash[27712]: audit 2023-10-09T08:55:10.207402+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:11.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:11 smithi181 bash[28565]: debug 2023-10-09T08:55:11.233+0000 7fc07ce28680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T08:55:11.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:11 smithi181 bash[28565]: debug 2023-10-09T08:55:11.337+0000 7fc07ce28680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T08:55:12.216 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:11 smithi181 bash[28565]: debug 2023-10-09T08:55:11.813+0000 7fc07ce28680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T08:55:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:12 smithi163 bash[27500]: cluster 2023-10-09T08:55:11.023610+0000 mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:12.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:12 smithi163 bash[33912]: cluster 2023-10-09T08:55:11.023610+0000 mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:12.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:12 smithi181 bash[27712]: cluster 2023-10-09T08:55:11.023610+0000 mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:13.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:12 smithi181 bash[28565]: debug 2023-10-09T08:55:12.729+0000 7fc07ce28680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T08:55:13.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:12 smithi181 bash[28565]: debug 2023-10-09T08:55:12.825+0000 7fc07ce28680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T08:55:13.160 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T08:55:13.160 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T08:55:13.160 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2023-10-09T08:55:13.160 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2023-10-09T08:55:13.160 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0153614 s, 683 MB/s 2023-10-09T08:55:13.160 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2023-10-09T08:55:13.566 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:13 smithi181 bash[28565]: debug 2023-10-09T08:55:13.077+0000 7fc07ce28680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T08:55:13.796 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi163:vg_nvme/lv_4 2023-10-09T08:55:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:14 smithi163 bash[27500]: cluster 2023-10-09T08:55:13.024020+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:14 smithi163 bash[27500]: audit 2023-10-09T08:55:13.802071+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:14.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:14 smithi163 bash[33912]: cluster 2023-10-09T08:55:13.024020+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:14.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:14 smithi163 bash[33912]: audit 2023-10-09T08:55:13.802071+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:14.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:14 smithi181 bash[27712]: cluster 2023-10-09T08:55:13.024020+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:14.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:14 smithi181 bash[27712]: audit 2023-10-09T08:55:13.802071+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:14.491 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:14 smithi181 bash[28565]: debug 2023-10-09T08:55:14.225+0000 7fc07ce28680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T08:55:14.491 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:14 smithi181 bash[28565]: debug 2023-10-09T08:55:14.389+0000 7fc07ce28680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:55:14.816 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:14 smithi181 bash[28565]: debug 2023-10-09T08:55:14.485+0000 7fc07ce28680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T08:55:14.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:14 smithi181 bash[28565]: debug 2023-10-09T08:55:14.597+0000 7fc07ce28680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T08:55:15.239 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:14 smithi181 bash[28565]: debug 2023-10-09T08:55:14.821+0000 7fc07ce28680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T08:55:15.566 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:15 smithi181 bash[28565]: debug 2023-10-09T08:55:15.237+0000 7fc07ce28680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T08:55:15.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:15 smithi181 bash[28565]: debug 2023-10-09T08:55:15.365+0000 7fc07ce28680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T08:55:16.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[33912]: cluster 2023-10-09T08:55:15.024505+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:16.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[33912]: audit 2023-10-09T08:55:15.247558+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:16.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[33912]: audit 2023-10-09T08:55:15.248578+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:16.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[33912]: audit 2023-10-09T08:55:15.249741+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:16.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[33912]: audit 2023-10-09T08:55:15.256740+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:16.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[33912]: cephadm 2023-10-09T08:55:15.258722+0000 mgr.y (mgr.14146) 54 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T08:55:16.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[33912]: audit 2023-10-09T08:55:15.259094+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:55:16.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[33912]: audit 2023-10-09T08:55:15.260230+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:55:16.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[33912]: audit 2023-10-09T08:55:15.261074+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:16.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[33912]: cephadm 2023-10-09T08:55:15.261854+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring daemon mgr.y on smithi163 2023-10-09T08:55:16.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[27500]: cluster 2023-10-09T08:55:15.024505+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:16.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[27500]: audit 2023-10-09T08:55:15.247558+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:16.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[27500]: audit 2023-10-09T08:55:15.248578+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:16.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[27500]: audit 2023-10-09T08:55:15.249741+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:16.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[27500]: audit 2023-10-09T08:55:15.256740+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:16.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[27500]: cephadm 2023-10-09T08:55:15.258722+0000 mgr.y (mgr.14146) 54 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T08:55:16.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[27500]: audit 2023-10-09T08:55:15.259094+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:55:16.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[27500]: audit 2023-10-09T08:55:15.260230+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:55:16.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[27500]: audit 2023-10-09T08:55:15.261074+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:16.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:16 smithi163 bash[27500]: cephadm 2023-10-09T08:55:15.261854+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring daemon mgr.y on smithi163 2023-10-09T08:55:16.566 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[28565]: debug 2023-10-09T08:55:16.253+0000 7fc07ce28680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T08:55:16.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[28565]: debug 2023-10-09T08:55:16.357+0000 7fc07ce28680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T08:55:16.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[28565]: debug 2023-10-09T08:55:16.473+0000 7fc07ce28680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T08:55:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[27712]: cluster 2023-10-09T08:55:15.024505+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[27712]: audit 2023-10-09T08:55:15.247558+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[27712]: audit 2023-10-09T08:55:15.248578+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[27712]: audit 2023-10-09T08:55:15.249741+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[27712]: audit 2023-10-09T08:55:15.256740+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[27712]: cephadm 2023-10-09T08:55:15.258722+0000 mgr.y (mgr.14146) 54 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T08:55:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[27712]: audit 2023-10-09T08:55:15.259094+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T08:55:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[27712]: audit 2023-10-09T08:55:15.260230+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T08:55:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[27712]: audit 2023-10-09T08:55:15.261074+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[27712]: cephadm 2023-10-09T08:55:15.261854+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring daemon mgr.y on smithi163 2023-10-09T08:55:16.987 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[28565]: debug 2023-10-09T08:55:16.689+0000 7fc07ce28680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T08:55:17.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:16 smithi181 bash[28565]: debug 2023-10-09T08:55:16.985+0000 7fc07ce28680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T08:55:17.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[28565]: debug 2023-10-09T08:55:17.149+0000 7fc07ce28680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T08:55:17.640 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[28565]: debug 2023-10-09T08:55:17.313+0000 7fc07ce28680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T08:55:17.640 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[28565]: debug 2023-10-09T08:55:17.529+0000 7fc07ce28680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T08:55:17.895 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[28565]: debug 2023-10-09T08:55:17.637+0000 7fc07ce28680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T08:55:18.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: cluster 2023-10-09T08:55:17.024936+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:18.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.036090+0000 mgr.y (mgr.14146) 57 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:55:18.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.039083+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:55:18.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.042355+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.043365+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.411897+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.413971+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.416411+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.417753+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.424515+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: cluster 2023-10-09T08:55:17.662559+0000 mon.a (mon.0) 229 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.663971+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.664766+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.666911+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[27500]: audit 2023-10-09T08:55:17.667404+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:55:18.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: cluster 2023-10-09T08:55:17.024936+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:18.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.036090+0000 mgr.y (mgr.14146) 57 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:55:18.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.039083+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:55:18.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.042355+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:55:18.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.043365+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:18.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.411897+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:18.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.413971+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:18.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.416411+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:18.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.417753+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:18.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.424515+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:18.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: cluster 2023-10-09T08:55:17.662559+0000 mon.a (mon.0) 229 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:55:18.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.663971+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:55:18.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.664766+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:55:18.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.666911+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:55:18.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:17 smithi163 bash[33912]: audit 2023-10-09T08:55:17.667404+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:55:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: cluster 2023-10-09T08:55:17.024936+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.036090+0000 mgr.y (mgr.14146) 57 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:55:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.039083+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:55:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.042355+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:55:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.043365+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.411897+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.413971+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.416411+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.417753+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.424515+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: cluster 2023-10-09T08:55:17.662559+0000 mon.a (mon.0) 229 : cluster [DBG] Standby manager daemon x started 2023-10-09T08:55:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.663971+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T08:55:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.664766+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T08:55:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.666911+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T08:55:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:17 smithi181 bash[27712]: audit 2023-10-09T08:55:17.667404+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.181:0/4154524291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T08:55:19.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:19 smithi163 bash[27500]: cluster 2023-10-09T08:55:18.434832+0000 mon.a (mon.0) 230 : cluster [DBG] mgrmap e14: y(active, since 73s), standbys: x 2023-10-09T08:55:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:19 smithi163 bash[27500]: audit 2023-10-09T08:55:18.435080+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:55:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:19 smithi163 bash[27500]: cluster 2023-10-09T08:55:19.025361+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:19 smithi163 bash[33912]: cluster 2023-10-09T08:55:18.434832+0000 mon.a (mon.0) 230 : cluster [DBG] mgrmap e14: y(active, since 73s), standbys: x 2023-10-09T08:55:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:19 smithi163 bash[33912]: audit 2023-10-09T08:55:18.435080+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:55:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:19 smithi163 bash[33912]: cluster 2023-10-09T08:55:19.025361+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:19 smithi181 bash[27712]: cluster 2023-10-09T08:55:18.434832+0000 mon.a (mon.0) 230 : cluster [DBG] mgrmap e14: y(active, since 73s), standbys: x 2023-10-09T08:55:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:19 smithi181 bash[27712]: audit 2023-10-09T08:55:18.435080+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T08:55:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:19 smithi181 bash[27712]: cluster 2023-10-09T08:55:19.025361+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:22.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:22 smithi163 bash[27500]: cluster 2023-10-09T08:55:21.025778+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:22 smithi163 bash[33912]: cluster 2023-10-09T08:55:21.025778+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:22.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:22 smithi181 bash[27712]: cluster 2023-10-09T08:55:21.025778+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:24 smithi163 bash[27500]: cluster 2023-10-09T08:55:23.026229+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:24 smithi163 bash[33912]: cluster 2023-10-09T08:55:23.026229+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:24.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:24 smithi181 bash[27712]: cluster 2023-10-09T08:55:23.026229+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:25.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:25 smithi163 bash[27500]: audit 2023-10-09T08:55:24.910810+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.163:0/1960201282' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09899392-dd0c-4eaf-b933-aafe194ff2b7"}]: dispatch 2023-10-09T08:55:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:25 smithi163 bash[27500]: audit 2023-10-09T08:55:24.917798+0000 mon.a (mon.0) 233 : audit [INF] from='client.? 172.21.15.163:0/1960201282' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09899392-dd0c-4eaf-b933-aafe194ff2b7"}]': finished 2023-10-09T08:55:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:25 smithi163 bash[27500]: cluster 2023-10-09T08:55:24.917941+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T08:55:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:25 smithi163 bash[27500]: audit 2023-10-09T08:55:24.918114+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:25.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:25 smithi163 bash[33912]: audit 2023-10-09T08:55:24.910810+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.163:0/1960201282' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09899392-dd0c-4eaf-b933-aafe194ff2b7"}]: dispatch 2023-10-09T08:55:25.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:25 smithi163 bash[33912]: audit 2023-10-09T08:55:24.917798+0000 mon.a (mon.0) 233 : audit [INF] from='client.? 172.21.15.163:0/1960201282' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09899392-dd0c-4eaf-b933-aafe194ff2b7"}]': finished 2023-10-09T08:55:25.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:25 smithi163 bash[33912]: cluster 2023-10-09T08:55:24.917941+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T08:55:25.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:25 smithi163 bash[33912]: audit 2023-10-09T08:55:24.918114+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:25 smithi181 bash[27712]: audit 2023-10-09T08:55:24.910810+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.163:0/1960201282' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09899392-dd0c-4eaf-b933-aafe194ff2b7"}]: dispatch 2023-10-09T08:55:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:25 smithi181 bash[27712]: audit 2023-10-09T08:55:24.917798+0000 mon.a (mon.0) 233 : audit [INF] from='client.? 172.21.15.163:0/1960201282' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09899392-dd0c-4eaf-b933-aafe194ff2b7"}]': finished 2023-10-09T08:55:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:25 smithi181 bash[27712]: cluster 2023-10-09T08:55:24.917941+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T08:55:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:25 smithi181 bash[27712]: audit 2023-10-09T08:55:24.918114+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:26.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:26 smithi163 bash[27500]: cluster 2023-10-09T08:55:25.026689+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:26 smithi163 bash[27500]: audit 2023-10-09T08:55:25.623955+0000 mon.a (mon.0) 236 : audit [DBG] from='client.? 172.21.15.163:0/1376753217' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:55:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:26 smithi163 bash[33912]: cluster 2023-10-09T08:55:25.026689+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:26 smithi163 bash[33912]: audit 2023-10-09T08:55:25.623955+0000 mon.a (mon.0) 236 : audit [DBG] from='client.? 172.21.15.163:0/1376753217' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:55:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:26 smithi181 bash[27712]: cluster 2023-10-09T08:55:25.026689+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:26 smithi181 bash[27712]: audit 2023-10-09T08:55:25.623955+0000 mon.a (mon.0) 236 : audit [DBG] from='client.? 172.21.15.163:0/1376753217' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:55:28.161 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:27 smithi163 bash[33912]: cluster 2023-10-09T08:55:27.027142+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:28.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:27 smithi163 bash[27500]: cluster 2023-10-09T08:55:27.027142+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:27 smithi181 bash[27712]: cluster 2023-10-09T08:55:27.027142+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:30 smithi163 bash[27500]: cluster 2023-10-09T08:55:29.027566+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:30.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:30 smithi163 bash[33912]: cluster 2023-10-09T08:55:29.027566+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:30.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:30 smithi181 bash[27712]: cluster 2023-10-09T08:55:29.027566+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:32 smithi163 bash[27500]: cluster 2023-10-09T08:55:31.027913+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:32.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:32 smithi163 bash[33912]: cluster 2023-10-09T08:55:31.027913+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:32.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:32 smithi181 bash[27712]: cluster 2023-10-09T08:55:31.027913+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:34.370 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:34 smithi163 bash[27500]: cluster 2023-10-09T08:55:33.028329+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:34.371 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:34 smithi163 bash[33912]: cluster 2023-10-09T08:55:33.028329+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:34.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:34 smithi181 bash[27712]: cluster 2023-10-09T08:55:33.028329+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:36.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:36 smithi163 bash[33912]: cluster 2023-10-09T08:55:35.028763+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:36.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:36 smithi163 bash[27500]: cluster 2023-10-09T08:55:35.028763+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:36 smithi181 bash[27712]: cluster 2023-10-09T08:55:35.028763+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:38.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:37 smithi163 bash[27500]: cluster 2023-10-09T08:55:37.029204+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:37 smithi163 bash[27500]: audit 2023-10-09T08:55:37.096988+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:55:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:37 smithi163 bash[27500]: audit 2023-10-09T08:55:37.098714+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:37 smithi163 bash[27500]: cephadm 2023-10-09T08:55:37.100220+0000 mgr.y (mgr.14146) 68 : cephadm [INF] Deploying daemon osd.0 on smithi163 2023-10-09T08:55:38.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:37 smithi163 bash[33912]: cluster 2023-10-09T08:55:37.029204+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:38.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:37 smithi163 bash[33912]: audit 2023-10-09T08:55:37.096988+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:55:38.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:37 smithi163 bash[33912]: audit 2023-10-09T08:55:37.098714+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:38.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:37 smithi163 bash[33912]: cephadm 2023-10-09T08:55:37.100220+0000 mgr.y (mgr.14146) 68 : cephadm [INF] Deploying daemon osd.0 on smithi163 2023-10-09T08:55:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:37 smithi181 bash[27712]: cluster 2023-10-09T08:55:37.029204+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:37 smithi181 bash[27712]: audit 2023-10-09T08:55:37.096988+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T08:55:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:37 smithi181 bash[27712]: audit 2023-10-09T08:55:37.098714+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:37 smithi181 bash[27712]: cephadm 2023-10-09T08:55:37.100220+0000 mgr.y (mgr.14146) 68 : cephadm [INF] Deploying daemon osd.0 on smithi163 2023-10-09T08:55:40.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:40 smithi163 bash[27500]: cluster 2023-10-09T08:55:39.029541+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:40.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:40 smithi163 bash[33912]: cluster 2023-10-09T08:55:39.029541+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:40 smithi181 bash[27712]: cluster 2023-10-09T08:55:39.029541+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:42.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:42 smithi163 bash[33912]: cluster 2023-10-09T08:55:41.029883+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:42.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:42 smithi163 bash[33912]: audit 2023-10-09T08:55:41.129155+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:42.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:42 smithi163 bash[33912]: audit 2023-10-09T08:55:41.157793+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:42.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:42 smithi163 bash[27500]: cluster 2023-10-09T08:55:41.029883+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:42.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:42 smithi163 bash[27500]: audit 2023-10-09T08:55:41.129155+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:42.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:42 smithi163 bash[27500]: audit 2023-10-09T08:55:41.157793+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:42 smithi181 bash[27712]: cluster 2023-10-09T08:55:41.029883+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:42 smithi181 bash[27712]: audit 2023-10-09T08:55:41.129155+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:42 smithi181 bash[27712]: audit 2023-10-09T08:55:41.157793+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:44 smithi163 bash[27500]: cluster 2023-10-09T08:55:43.030338+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:44 smithi163 bash[33912]: cluster 2023-10-09T08:55:43.030338+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:44.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:44 smithi181 bash[27712]: cluster 2023-10-09T08:55:43.030338+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:46.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:46 smithi163 bash[27500]: cluster 2023-10-09T08:55:45.030765+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:46 smithi163 bash[33912]: cluster 2023-10-09T08:55:45.030765+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:46 smithi181 bash[27712]: cluster 2023-10-09T08:55:45.030765+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:47 smithi163 bash[27500]: audit 2023-10-09T08:55:46.236554+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:47 smithi163 bash[27500]: audit 2023-10-09T08:55:46.496758+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:47 smithi163 bash[27500]: audit 2023-10-09T08:55:46.498031+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:47 smithi163 bash[27500]: audit 2023-10-09T08:55:46.505756+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:47 smithi163 bash[33912]: audit 2023-10-09T08:55:46.236554+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:47 smithi163 bash[33912]: audit 2023-10-09T08:55:46.496758+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:47 smithi163 bash[33912]: audit 2023-10-09T08:55:46.498031+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:47 smithi163 bash[33912]: audit 2023-10-09T08:55:46.505756+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:47 smithi181 bash[27712]: audit 2023-10-09T08:55:46.236554+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:47 smithi181 bash[27712]: audit 2023-10-09T08:55:46.496758+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:47 smithi181 bash[27712]: audit 2023-10-09T08:55:46.498031+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:55:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:47 smithi181 bash[27712]: audit 2023-10-09T08:55:46.505756+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:48.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:48 smithi181 bash[27712]: cluster 2023-10-09T08:55:47.031171+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:48.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:48 smithi181 bash[27712]: audit 2023-10-09T08:55:47.483819+0000 mon.a (mon.0) 245 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T08:55:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:48 smithi163 bash[33912]: cluster 2023-10-09T08:55:47.031171+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:48 smithi163 bash[33912]: audit 2023-10-09T08:55:47.483819+0000 mon.a (mon.0) 245 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T08:55:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:48 smithi163 bash[27500]: cluster 2023-10-09T08:55:47.031171+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:48 smithi163 bash[27500]: audit 2023-10-09T08:55:47.483819+0000 mon.a (mon.0) 245 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T08:55:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[27500]: audit 2023-10-09T08:55:48.243793+0000 mon.a (mon.0) 246 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T08:55:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[27500]: cluster 2023-10-09T08:55:48.243939+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T08:55:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[27500]: audit 2023-10-09T08:55:48.244628+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[27500]: audit 2023-10-09T08:55:48.245369+0000 mon.a (mon.0) 249 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:55:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[27500]: audit 2023-10-09T08:55:49.246421+0000 mon.a (mon.0) 250 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:55:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[27500]: cluster 2023-10-09T08:55:49.246626+0000 mon.a (mon.0) 251 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T08:55:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[27500]: audit 2023-10-09T08:55:49.246802+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:49.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[33912]: audit 2023-10-09T08:55:48.243793+0000 mon.a (mon.0) 246 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T08:55:49.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[33912]: cluster 2023-10-09T08:55:48.243939+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T08:55:49.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[33912]: audit 2023-10-09T08:55:48.244628+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:49.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[33912]: audit 2023-10-09T08:55:48.245369+0000 mon.a (mon.0) 249 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:55:49.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[33912]: audit 2023-10-09T08:55:49.246421+0000 mon.a (mon.0) 250 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:55:49.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[33912]: cluster 2023-10-09T08:55:49.246626+0000 mon.a (mon.0) 251 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T08:55:49.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:49 smithi163 bash[33912]: audit 2023-10-09T08:55:49.246802+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:49 smithi181 bash[27712]: audit 2023-10-09T08:55:48.243793+0000 mon.a (mon.0) 246 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T08:55:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:49 smithi181 bash[27712]: cluster 2023-10-09T08:55:48.243939+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T08:55:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:49 smithi181 bash[27712]: audit 2023-10-09T08:55:48.244628+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:49 smithi181 bash[27712]: audit 2023-10-09T08:55:48.245369+0000 mon.a (mon.0) 249 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:55:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:49 smithi181 bash[27712]: audit 2023-10-09T08:55:49.246421+0000 mon.a (mon.0) 250 : audit [INF] from='osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:55:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:49 smithi181 bash[27712]: cluster 2023-10-09T08:55:49.246626+0000 mon.a (mon.0) 251 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T08:55:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:49 smithi181 bash[27712]: audit 2023-10-09T08:55:49.246802+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:49.581 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 0 on host 'smithi163' 2023-10-09T08:55:50.202 DEBUG:teuthology.orchestra.run.smithi163:osd.0> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T08:55:50.204 INFO:tasks.cephadm:Deploying osd.1 on smithi163 with /dev/vg_nvme/lv_3... 2023-10-09T08:55:50.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T08:55:50.421 INFO:journalctl@ceph.osd.0.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:55:50.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:50 smithi163 bash[27500]: cluster 2023-10-09T08:55:49.031533+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:50 smithi163 bash[27500]: audit 2023-10-09T08:55:49.252679+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:50 smithi163 bash[27500]: audit 2023-10-09T08:55:49.579817+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:50 smithi163 bash[27500]: audit 2023-10-09T08:55:49.580471+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:50 smithi163 bash[27500]: audit 2023-10-09T08:55:50.251985+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:50 smithi163 bash[33912]: cluster 2023-10-09T08:55:49.031533+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:50 smithi163 bash[33912]: audit 2023-10-09T08:55:49.252679+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:50 smithi163 bash[33912]: audit 2023-10-09T08:55:49.579817+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:50 smithi163 bash[33912]: audit 2023-10-09T08:55:49.580471+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:50 smithi163 bash[33912]: audit 2023-10-09T08:55:50.251985+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:50 smithi181 bash[27712]: cluster 2023-10-09T08:55:49.031533+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:50 smithi181 bash[27712]: audit 2023-10-09T08:55:49.252679+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:50 smithi181 bash[27712]: audit 2023-10-09T08:55:49.579817+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:55:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:50 smithi181 bash[27712]: audit 2023-10-09T08:55:49.580471+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:55:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:50 smithi181 bash[27712]: audit 2023-10-09T08:55:50.251985+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:51.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:51 smithi163 bash[27500]: cluster 2023-10-09T08:55:48.509368+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:55:51.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:51 smithi163 bash[27500]: cluster 2023-10-09T08:55:48.509467+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:55:51.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:51 smithi163 bash[27500]: cluster 2023-10-09T08:55:50.261379+0000 mon.a (mon.0) 257 : cluster [INF] osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962] boot 2023-10-09T08:55:51.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:51 smithi163 bash[27500]: cluster 2023-10-09T08:55:50.261420+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T08:55:51.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:51 smithi163 bash[27500]: audit 2023-10-09T08:55:50.261958+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:51.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:51 smithi163 bash[33912]: cluster 2023-10-09T08:55:48.509368+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:55:51.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:51 smithi163 bash[33912]: cluster 2023-10-09T08:55:48.509467+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:55:51.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:51 smithi163 bash[33912]: cluster 2023-10-09T08:55:50.261379+0000 mon.a (mon.0) 257 : cluster [INF] osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962] boot 2023-10-09T08:55:51.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:51 smithi163 bash[33912]: cluster 2023-10-09T08:55:50.261420+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T08:55:51.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:51 smithi163 bash[33912]: audit 2023-10-09T08:55:50.261958+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:51 smithi181 bash[27712]: cluster 2023-10-09T08:55:48.509368+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:55:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:51 smithi181 bash[27712]: cluster 2023-10-09T08:55:48.509467+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:55:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:51 smithi181 bash[27712]: cluster 2023-10-09T08:55:50.261379+0000 mon.a (mon.0) 257 : cluster [INF] osd.0 [v2:172.21.15.163:6802/422484962,v1:172.21.15.163:6803/422484962] boot 2023-10-09T08:55:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:51 smithi181 bash[27712]: cluster 2023-10-09T08:55:50.261420+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T08:55:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:51 smithi181 bash[27712]: audit 2023-10-09T08:55:50.261958+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T08:55:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:52 smithi181 bash[27712]: cluster 2023-10-09T08:55:51.032005+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:52 smithi181 bash[27712]: cluster 2023-10-09T08:55:51.274836+0000 mon.a (mon.0) 260 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T08:55:52.616 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:52 smithi163 bash[27500]: cluster 2023-10-09T08:55:51.032005+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:52.616 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:52 smithi163 bash[27500]: cluster 2023-10-09T08:55:51.274836+0000 mon.a (mon.0) 260 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T08:55:52.617 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:52 smithi163 bash[33912]: cluster 2023-10-09T08:55:51.032005+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T08:55:52.617 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:52 smithi163 bash[33912]: cluster 2023-10-09T08:55:51.274836+0000 mon.a (mon.0) 260 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T08:55:53.302 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T08:55:53.303 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T08:55:53.303 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2023-10-09T08:55:53.303 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2023-10-09T08:55:53.303 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0154189 s, 680 MB/s 2023-10-09T08:55:53.303 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2023-10-09T08:55:54.044 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi163:vg_nvme/lv_3 2023-10-09T08:55:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:54 smithi163 bash[27500]: cluster 2023-10-09T08:55:53.032514+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:55:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:54 smithi163 bash[33912]: cluster 2023-10-09T08:55:53.032514+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:55:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:54 smithi181 bash[27712]: cluster 2023-10-09T08:55:53.032514+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:55:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:56 smithi163 bash[27500]: cluster 2023-10-09T08:55:55.033040+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:55:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:56 smithi163 bash[33912]: cluster 2023-10-09T08:55:55.033040+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:55:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:56 smithi181 bash[27712]: cluster 2023-10-09T08:55:55.033040+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:55:58.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:57 smithi163 bash[27500]: cluster 2023-10-09T08:55:57.033576+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:55:58.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:57 smithi163 bash[33912]: cluster 2023-10-09T08:55:57.033576+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:55:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:57 smithi181 bash[27712]: cluster 2023-10-09T08:55:57.033576+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:55:58.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:58 smithi163 bash[27500]: audit 2023-10-09T08:55:58.056044+0000 mgr.y (mgr.14146) 79 : audit [DBG] from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:55:58.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:58 smithi163 bash[27500]: audit 2023-10-09T08:55:58.062747+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:55:58.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:58 smithi163 bash[27500]: audit 2023-10-09T08:55:58.067898+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:55:58.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:58 smithi163 bash[27500]: audit 2023-10-09T08:55:58.068969+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:58.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:58 smithi163 bash[33912]: audit 2023-10-09T08:55:58.056044+0000 mgr.y (mgr.14146) 79 : audit [DBG] from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:55:58.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:58 smithi163 bash[33912]: audit 2023-10-09T08:55:58.062747+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:55:58.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:58 smithi163 bash[33912]: audit 2023-10-09T08:55:58.067898+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:55:58.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:58 smithi163 bash[33912]: audit 2023-10-09T08:55:58.068969+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:55:59.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:58 smithi181 bash[27712]: audit 2023-10-09T08:55:58.056044+0000 mgr.y (mgr.14146) 79 : audit [DBG] from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:55:59.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:58 smithi181 bash[27712]: audit 2023-10-09T08:55:58.062747+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:55:59.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:58 smithi181 bash[27712]: audit 2023-10-09T08:55:58.067898+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:55:59.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:58 smithi181 bash[27712]: audit 2023-10-09T08:55:58.068969+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:00.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[27500]: cluster 2023-10-09T08:55:59.034064+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[27500]: audit 2023-10-09T08:55:59.334353+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[27500]: audit 2023-10-09T08:55:59.335749+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:56:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[27500]: audit 2023-10-09T08:55:59.341853+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[27500]: audit 2023-10-09T08:55:59.343027+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[27500]: audit 2023-10-09T08:55:59.344010+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[27500]: audit 2023-10-09T08:55:59.352463+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[33912]: cluster 2023-10-09T08:55:59.034064+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:00.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[33912]: audit 2023-10-09T08:55:59.334353+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:00.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[33912]: audit 2023-10-09T08:55:59.335749+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:56:00.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[33912]: audit 2023-10-09T08:55:59.341853+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:00.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[33912]: audit 2023-10-09T08:55:59.343027+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:00.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[33912]: audit 2023-10-09T08:55:59.344010+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:00.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:55:59 smithi163 bash[33912]: audit 2023-10-09T08:55:59.352463+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:59 smithi181 bash[27712]: cluster 2023-10-09T08:55:59.034064+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:59 smithi181 bash[27712]: audit 2023-10-09T08:55:59.334353+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:59 smithi181 bash[27712]: audit 2023-10-09T08:55:59.335749+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:56:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:59 smithi181 bash[27712]: audit 2023-10-09T08:55:59.341853+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:59 smithi181 bash[27712]: audit 2023-10-09T08:55:59.343027+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:59 smithi181 bash[27712]: audit 2023-10-09T08:55:59.344010+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:55:59 smithi181 bash[27712]: audit 2023-10-09T08:55:59.352463+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:01.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:00 smithi163 bash[27500]: cephadm 2023-10-09T08:55:59.325780+0000 mgr.y (mgr.14146) 81 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:56:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:00 smithi163 bash[27500]: cephadm 2023-10-09T08:55:59.336735+0000 mgr.y (mgr.14146) 82 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 16241M 2023-10-09T08:56:01.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:00 smithi163 bash[33912]: cephadm 2023-10-09T08:55:59.325780+0000 mgr.y (mgr.14146) 81 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:56:01.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:00 smithi163 bash[33912]: cephadm 2023-10-09T08:55:59.336735+0000 mgr.y (mgr.14146) 82 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 16241M 2023-10-09T08:56:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:00 smithi181 bash[27712]: cephadm 2023-10-09T08:55:59.325780+0000 mgr.y (mgr.14146) 81 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:56:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:00 smithi181 bash[27712]: cephadm 2023-10-09T08:55:59.336735+0000 mgr.y (mgr.14146) 82 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 16241M 2023-10-09T08:56:02.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:01 smithi163 bash[27500]: cluster 2023-10-09T08:56:01.034542+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:02.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:01 smithi163 bash[33912]: cluster 2023-10-09T08:56:01.034542+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:01 smithi181 bash[27712]: cluster 2023-10-09T08:56:01.034542+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:04.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:04 smithi163 bash[27500]: cluster 2023-10-09T08:56:03.035115+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:04.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:04 smithi163 bash[33912]: cluster 2023-10-09T08:56:03.035115+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:04 smithi181 bash[27712]: cluster 2023-10-09T08:56:03.035115+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[27500]: cluster 2023-10-09T08:56:05.035535+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[27500]: audit 2023-10-09T08:56:05.986767+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.163:0/1751454360' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb74bcac-37d3-472b-abae-6c72df5b264b"}]: dispatch 2023-10-09T08:56:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[27500]: audit 2023-10-09T08:56:05.987677+0000 mon.a (mon.0) 270 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb74bcac-37d3-472b-abae-6c72df5b264b"}]: dispatch 2023-10-09T08:56:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[27500]: audit 2023-10-09T08:56:06.000190+0000 mon.a (mon.0) 271 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb74bcac-37d3-472b-abae-6c72df5b264b"}]': finished 2023-10-09T08:56:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[27500]: cluster 2023-10-09T08:56:06.000324+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T08:56:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[27500]: audit 2023-10-09T08:56:06.000569+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[33912]: cluster 2023-10-09T08:56:05.035535+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[33912]: audit 2023-10-09T08:56:05.986767+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.163:0/1751454360' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb74bcac-37d3-472b-abae-6c72df5b264b"}]: dispatch 2023-10-09T08:56:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[33912]: audit 2023-10-09T08:56:05.987677+0000 mon.a (mon.0) 270 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb74bcac-37d3-472b-abae-6c72df5b264b"}]: dispatch 2023-10-09T08:56:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[33912]: audit 2023-10-09T08:56:06.000190+0000 mon.a (mon.0) 271 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb74bcac-37d3-472b-abae-6c72df5b264b"}]': finished 2023-10-09T08:56:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[33912]: cluster 2023-10-09T08:56:06.000324+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T08:56:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:06 smithi163 bash[33912]: audit 2023-10-09T08:56:06.000569+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:06 smithi181 bash[27712]: cluster 2023-10-09T08:56:05.035535+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:06 smithi181 bash[27712]: audit 2023-10-09T08:56:05.986767+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.163:0/1751454360' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb74bcac-37d3-472b-abae-6c72df5b264b"}]: dispatch 2023-10-09T08:56:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:06 smithi181 bash[27712]: audit 2023-10-09T08:56:05.987677+0000 mon.a (mon.0) 270 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb74bcac-37d3-472b-abae-6c72df5b264b"}]: dispatch 2023-10-09T08:56:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:06 smithi181 bash[27712]: audit 2023-10-09T08:56:06.000190+0000 mon.a (mon.0) 271 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb74bcac-37d3-472b-abae-6c72df5b264b"}]': finished 2023-10-09T08:56:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:06 smithi181 bash[27712]: cluster 2023-10-09T08:56:06.000324+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T08:56:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:06 smithi181 bash[27712]: audit 2023-10-09T08:56:06.000569+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:07 smithi163 bash[27500]: audit 2023-10-09T08:56:06.739601+0000 mon.a (mon.0) 274 : audit [DBG] from='client.? 172.21.15.163:0/1669696343' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:56:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:07 smithi163 bash[33912]: audit 2023-10-09T08:56:06.739601+0000 mon.a (mon.0) 274 : audit [DBG] from='client.? 172.21.15.163:0/1669696343' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:56:07.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:07 smithi181 bash[27712]: audit 2023-10-09T08:56:06.739601+0000 mon.a (mon.0) 274 : audit [DBG] from='client.? 172.21.15.163:0/1669696343' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:56:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:08 smithi163 bash[27500]: cluster 2023-10-09T08:56:07.036045+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:08 smithi163 bash[33912]: cluster 2023-10-09T08:56:07.036045+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:08 smithi181 bash[27712]: cluster 2023-10-09T08:56:07.036045+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:10.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:10 smithi163 bash[27500]: cluster 2023-10-09T08:56:09.036518+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:10.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:10 smithi163 bash[33912]: cluster 2023-10-09T08:56:09.036518+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:10 smithi181 bash[27712]: cluster 2023-10-09T08:56:09.036518+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:12 smithi163 bash[27500]: cluster 2023-10-09T08:56:11.036995+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:12.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:12 smithi163 bash[33912]: cluster 2023-10-09T08:56:11.036995+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:12 smithi181 bash[27712]: cluster 2023-10-09T08:56:11.036995+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:14 smithi163 bash[27500]: cluster 2023-10-09T08:56:13.037478+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:14.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:14 smithi163 bash[33912]: cluster 2023-10-09T08:56:13.037478+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:14 smithi181 bash[27712]: cluster 2023-10-09T08:56:13.037478+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:16.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:16 smithi163 bash[33912]: cluster 2023-10-09T08:56:15.037964+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:16.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:16 smithi163 bash[27500]: cluster 2023-10-09T08:56:15.037964+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:16 smithi181 bash[27712]: cluster 2023-10-09T08:56:15.037964+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:17.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:17 smithi163 bash[27500]: cluster 2023-10-09T08:56:17.038543+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:17.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:17 smithi163 bash[33912]: cluster 2023-10-09T08:56:17.038543+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:18.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:17 smithi181 bash[27712]: cluster 2023-10-09T08:56:17.038543+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:19.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:18 smithi163 bash[27500]: audit 2023-10-09T08:56:18.146083+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:56:19.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:18 smithi163 bash[27500]: audit 2023-10-09T08:56:18.147779+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:19.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:18 smithi163 bash[27500]: cephadm 2023-10-09T08:56:18.149250+0000 mgr.y (mgr.14146) 92 : cephadm [INF] Deploying daemon osd.1 on smithi163 2023-10-09T08:56:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:18 smithi163 bash[33912]: audit 2023-10-09T08:56:18.146083+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:56:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:18 smithi163 bash[33912]: audit 2023-10-09T08:56:18.147779+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:18 smithi163 bash[33912]: cephadm 2023-10-09T08:56:18.149250+0000 mgr.y (mgr.14146) 92 : cephadm [INF] Deploying daemon osd.1 on smithi163 2023-10-09T08:56:19.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:18 smithi181 bash[27712]: audit 2023-10-09T08:56:18.146083+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T08:56:19.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:18 smithi181 bash[27712]: audit 2023-10-09T08:56:18.147779+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:19.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:18 smithi181 bash[27712]: cephadm 2023-10-09T08:56:18.149250+0000 mgr.y (mgr.14146) 92 : cephadm [INF] Deploying daemon osd.1 on smithi163 2023-10-09T08:56:20.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:19 smithi163 bash[27500]: cluster 2023-10-09T08:56:19.039125+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:20.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:19 smithi163 bash[33912]: cluster 2023-10-09T08:56:19.039125+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:19 smithi181 bash[27712]: cluster 2023-10-09T08:56:19.039125+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:22.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:22 smithi163 bash[27500]: cluster 2023-10-09T08:56:21.039559+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:22 smithi163 bash[33912]: cluster 2023-10-09T08:56:21.039559+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:22 smithi181 bash[27712]: cluster 2023-10-09T08:56:21.039559+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:23.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:23 smithi163 bash[33912]: audit 2023-10-09T08:56:22.208863+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:23.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:23 smithi163 bash[33912]: audit 2023-10-09T08:56:22.229125+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:56:23.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:23 smithi163 bash[27500]: audit 2023-10-09T08:56:22.208863+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:23.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:23 smithi163 bash[27500]: audit 2023-10-09T08:56:22.229125+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:56:23.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:23 smithi181 bash[27712]: audit 2023-10-09T08:56:22.208863+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:23 smithi181 bash[27712]: audit 2023-10-09T08:56:22.229125+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:56:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:24 smithi163 bash[27500]: cluster 2023-10-09T08:56:23.039997+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:24 smithi163 bash[33912]: cluster 2023-10-09T08:56:23.039997+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:24.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:24 smithi181 bash[27712]: cluster 2023-10-09T08:56:23.039997+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:26 smithi181 bash[27712]: cluster 2023-10-09T08:56:25.040535+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:26 smithi163 bash[27500]: cluster 2023-10-09T08:56:25.040535+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:26 smithi163 bash[33912]: cluster 2023-10-09T08:56:25.040535+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:28.158 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:27 smithi163 bash[27500]: cluster 2023-10-09T08:56:27.041038+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:28.158 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:27 smithi163 bash[27500]: audit 2023-10-09T08:56:27.385942+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:28.158 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:27 smithi163 bash[27500]: audit 2023-10-09T08:56:27.387048+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:28.158 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:27 smithi163 bash[27500]: audit 2023-10-09T08:56:27.388230+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:28.158 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:27 smithi163 bash[27500]: audit 2023-10-09T08:56:27.394309+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:28.159 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:27 smithi163 bash[33912]: cluster 2023-10-09T08:56:27.041038+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:28.159 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:27 smithi163 bash[33912]: audit 2023-10-09T08:56:27.385942+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:28.159 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:27 smithi163 bash[33912]: audit 2023-10-09T08:56:27.387048+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:28.159 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:27 smithi163 bash[33912]: audit 2023-10-09T08:56:27.388230+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:28.159 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:27 smithi163 bash[33912]: audit 2023-10-09T08:56:27.394309+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:27 smithi181 bash[27712]: cluster 2023-10-09T08:56:27.041038+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:27 smithi181 bash[27712]: audit 2023-10-09T08:56:27.385942+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:27 smithi181 bash[27712]: audit 2023-10-09T08:56:27.387048+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:27 smithi181 bash[27712]: audit 2023-10-09T08:56:27.388230+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:27 smithi181 bash[27712]: audit 2023-10-09T08:56:27.394309+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:29.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:28 smithi163 bash[27500]: audit 2023-10-09T08:56:28.758144+0000 mon.c (mon.2) 4 : audit [INF] from='osd.1 [v2:172.21.15.163:6810/1836428548,v1:172.21.15.163:6811/1836428548]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:56:29.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:28 smithi163 bash[27500]: audit 2023-10-09T08:56:28.758660+0000 mon.a (mon.0) 283 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:56:29.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:28 smithi163 bash[33912]: audit 2023-10-09T08:56:28.758144+0000 mon.c (mon.2) 4 : audit [INF] from='osd.1 [v2:172.21.15.163:6810/1836428548,v1:172.21.15.163:6811/1836428548]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:56:29.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:28 smithi163 bash[33912]: audit 2023-10-09T08:56:28.758660+0000 mon.a (mon.0) 283 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:56:29.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:28 smithi181 bash[27712]: audit 2023-10-09T08:56:28.758144+0000 mon.c (mon.2) 4 : audit [INF] from='osd.1 [v2:172.21.15.163:6810/1836428548,v1:172.21.15.163:6811/1836428548]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:56:29.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:28 smithi181 bash[27712]: audit 2023-10-09T08:56:28.758660+0000 mon.a (mon.0) 283 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T08:56:30.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[33912]: audit 2023-10-09T08:56:28.932694+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T08:56:30.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[33912]: cluster 2023-10-09T08:56:28.932784+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T08:56:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[33912]: audit 2023-10-09T08:56:28.932980+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[33912]: audit 2023-10-09T08:56:28.933741+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.163:6810/1836428548,v1:172.21.15.163:6811/1836428548]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:56:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[33912]: audit 2023-10-09T08:56:28.934222+0000 mon.a (mon.0) 287 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:56:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[33912]: cluster 2023-10-09T08:56:29.041504+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[27500]: audit 2023-10-09T08:56:28.932694+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T08:56:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[27500]: cluster 2023-10-09T08:56:28.932784+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T08:56:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[27500]: audit 2023-10-09T08:56:28.932980+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[27500]: audit 2023-10-09T08:56:28.933741+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.163:6810/1836428548,v1:172.21.15.163:6811/1836428548]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:56:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[27500]: audit 2023-10-09T08:56:28.934222+0000 mon.a (mon.0) 287 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:56:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:29 smithi163 bash[27500]: cluster 2023-10-09T08:56:29.041504+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:30.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:29 smithi181 bash[27712]: audit 2023-10-09T08:56:28.932694+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T08:56:30.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:29 smithi181 bash[27712]: cluster 2023-10-09T08:56:28.932784+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T08:56:30.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:29 smithi181 bash[27712]: audit 2023-10-09T08:56:28.932980+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:30.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:29 smithi181 bash[27712]: audit 2023-10-09T08:56:28.933741+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.163:6810/1836428548,v1:172.21.15.163:6811/1836428548]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:56:30.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:29 smithi181 bash[27712]: audit 2023-10-09T08:56:28.934222+0000 mon.a (mon.0) 287 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:56:30.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:29 smithi181 bash[27712]: cluster 2023-10-09T08:56:29.041504+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:30.610 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 1 on host 'smithi163' 2023-10-09T08:56:31.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[27500]: audit 2023-10-09T08:56:29.936276+0000 mon.a (mon.0) 288 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:56:31.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[27500]: cluster 2023-10-09T08:56:29.936402+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T08:56:31.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[27500]: audit 2023-10-09T08:56:29.937006+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:31.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[27500]: audit 2023-10-09T08:56:29.942660+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:31.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[27500]: audit 2023-10-09T08:56:30.609009+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:31.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[27500]: audit 2023-10-09T08:56:30.618907+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:56:31.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[33912]: audit 2023-10-09T08:56:29.936276+0000 mon.a (mon.0) 288 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:56:31.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[33912]: cluster 2023-10-09T08:56:29.936402+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T08:56:31.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[33912]: audit 2023-10-09T08:56:29.937006+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:31.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[33912]: audit 2023-10-09T08:56:29.942660+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:31.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[33912]: audit 2023-10-09T08:56:30.609009+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:31.199 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:30 smithi163 bash[33912]: audit 2023-10-09T08:56:30.618907+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:56:31.200 DEBUG:teuthology.orchestra.run.smithi163:osd.1> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T08:56:31.201 INFO:tasks.cephadm:Deploying osd.2 on smithi163 with /dev/vg_nvme/lv_2... 2023-10-09T08:56:31.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T08:56:31.267 INFO:journalctl@ceph.osd.1.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:56:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:30 smithi181 bash[27712]: audit 2023-10-09T08:56:29.936276+0000 mon.a (mon.0) 288 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:56:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:30 smithi181 bash[27712]: cluster 2023-10-09T08:56:29.936402+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T08:56:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:30 smithi181 bash[27712]: audit 2023-10-09T08:56:29.937006+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:30 smithi181 bash[27712]: audit 2023-10-09T08:56:29.942660+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:30 smithi181 bash[27712]: audit 2023-10-09T08:56:30.609009+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:30 smithi181 bash[27712]: audit 2023-10-09T08:56:30.618907+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:56:32.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[27500]: cluster 2023-10-09T08:56:29.770239+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:56:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[27500]: cluster 2023-10-09T08:56:29.770324+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:56:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[27500]: cluster 2023-10-09T08:56:30.945683+0000 mon.a (mon.0) 294 : cluster [INF] osd.1 [v2:172.21.15.163:6810/1836428548,v1:172.21.15.163:6811/1836428548] boot 2023-10-09T08:56:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[27500]: cluster 2023-10-09T08:56:30.945809+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T08:56:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[27500]: audit 2023-10-09T08:56:30.946548+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[27500]: cluster 2023-10-09T08:56:31.041993+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:32.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[33912]: cluster 2023-10-09T08:56:29.770239+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:56:32.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[33912]: cluster 2023-10-09T08:56:29.770324+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:56:32.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[33912]: cluster 2023-10-09T08:56:30.945683+0000 mon.a (mon.0) 294 : cluster [INF] osd.1 [v2:172.21.15.163:6810/1836428548,v1:172.21.15.163:6811/1836428548] boot 2023-10-09T08:56:32.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[33912]: cluster 2023-10-09T08:56:30.945809+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T08:56:32.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[33912]: audit 2023-10-09T08:56:30.946548+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:32.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:31 smithi163 bash[33912]: cluster 2023-10-09T08:56:31.041993+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:31 smithi181 bash[27712]: cluster 2023-10-09T08:56:29.770239+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:56:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:31 smithi181 bash[27712]: cluster 2023-10-09T08:56:29.770324+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:56:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:31 smithi181 bash[27712]: cluster 2023-10-09T08:56:30.945683+0000 mon.a (mon.0) 294 : cluster [INF] osd.1 [v2:172.21.15.163:6810/1836428548,v1:172.21.15.163:6811/1836428548] boot 2023-10-09T08:56:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:31 smithi181 bash[27712]: cluster 2023-10-09T08:56:30.945809+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T08:56:32.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:31 smithi181 bash[27712]: audit 2023-10-09T08:56:30.946548+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T08:56:32.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:31 smithi181 bash[27712]: cluster 2023-10-09T08:56:31.041993+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T08:56:33.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:32 smithi163 bash[27500]: cluster 2023-10-09T08:56:31.960588+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T08:56:33.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:32 smithi163 bash[33912]: cluster 2023-10-09T08:56:31.960588+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T08:56:33.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:32 smithi181 bash[27712]: cluster 2023-10-09T08:56:31.960588+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T08:56:34.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:33 smithi163 bash[27500]: cluster 2023-10-09T08:56:33.042351+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:34.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:33 smithi163 bash[33912]: cluster 2023-10-09T08:56:33.042351+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:34.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:33 smithi181 bash[27712]: cluster 2023-10-09T08:56:33.042351+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:34.442 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T08:56:34.442 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T08:56:34.442 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2023-10-09T08:56:34.442 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2023-10-09T08:56:34.442 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0180165 s, 582 MB/s 2023-10-09T08:56:34.442 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2023-10-09T08:56:35.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi163:vg_nvme/lv_2 2023-10-09T08:56:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:36 smithi163 bash[27500]: cluster 2023-10-09T08:56:35.042855+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:36 smithi163 bash[33912]: cluster 2023-10-09T08:56:35.042855+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:36 smithi181 bash[27712]: cluster 2023-10-09T08:56:35.042855+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:37.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:37 smithi163 bash[33912]: cluster 2023-10-09T08:56:37.043320+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:37 smithi163 bash[27500]: cluster 2023-10-09T08:56:37.043320+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:37 smithi181 bash[27712]: cluster 2023-10-09T08:56:37.043320+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:40.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:40 smithi163 bash[27500]: cluster 2023-10-09T08:56:39.043688+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:40.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:40 smithi163 bash[27500]: audit 2023-10-09T08:56:39.321361+0000 mgr.y (mgr.14146) 104 : audit [DBG] from='client.14253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:56:40.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:40 smithi163 bash[27500]: audit 2023-10-09T08:56:39.326059+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:56:40.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:40 smithi163 bash[27500]: audit 2023-10-09T08:56:39.331498+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:56:40.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:40 smithi163 bash[27500]: audit 2023-10-09T08:56:39.332859+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:40.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:40 smithi163 bash[33912]: cluster 2023-10-09T08:56:39.043688+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:40.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:40 smithi163 bash[33912]: audit 2023-10-09T08:56:39.321361+0000 mgr.y (mgr.14146) 104 : audit [DBG] from='client.14253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:56:40.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:40 smithi163 bash[33912]: audit 2023-10-09T08:56:39.326059+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:56:40.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:40 smithi163 bash[33912]: audit 2023-10-09T08:56:39.331498+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:56:40.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:40 smithi163 bash[33912]: audit 2023-10-09T08:56:39.332859+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:40 smithi181 bash[27712]: cluster 2023-10-09T08:56:39.043688+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:40 smithi181 bash[27712]: audit 2023-10-09T08:56:39.321361+0000 mgr.y (mgr.14146) 104 : audit [DBG] from='client.14253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:56:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:40 smithi181 bash[27712]: audit 2023-10-09T08:56:39.326059+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:56:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:40 smithi181 bash[27712]: audit 2023-10-09T08:56:39.331498+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:56:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:40 smithi181 bash[27712]: audit 2023-10-09T08:56:39.332859+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:41 smithi181 bash[27712]: cephadm 2023-10-09T08:56:40.534960+0000 mgr.y (mgr.14146) 105 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:56:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:41 smithi181 bash[27712]: audit 2023-10-09T08:56:40.540447+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:41 smithi181 bash[27712]: audit 2023-10-09T08:56:40.541582+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:56:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:41 smithi181 bash[27712]: audit 2023-10-09T08:56:40.542479+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:56:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:41 smithi181 bash[27712]: cephadm 2023-10-09T08:56:40.543135+0000 mgr.y (mgr.14146) 106 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 8120M 2023-10-09T08:56:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:41 smithi181 bash[27712]: audit 2023-10-09T08:56:40.548539+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:41 smithi181 bash[27712]: audit 2023-10-09T08:56:40.549980+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:41.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:41 smithi181 bash[27712]: audit 2023-10-09T08:56:40.551050+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:41.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:41 smithi181 bash[27712]: audit 2023-10-09T08:56:40.557147+0000 mon.a (mon.0) 307 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:41.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:41 smithi181 bash[27712]: cluster 2023-10-09T08:56:41.044225+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[27500]: cephadm 2023-10-09T08:56:40.534960+0000 mgr.y (mgr.14146) 105 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:56:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[27500]: audit 2023-10-09T08:56:40.540447+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[27500]: audit 2023-10-09T08:56:40.541582+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:56:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[27500]: audit 2023-10-09T08:56:40.542479+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:56:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[27500]: cephadm 2023-10-09T08:56:40.543135+0000 mgr.y (mgr.14146) 106 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 8120M 2023-10-09T08:56:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[27500]: audit 2023-10-09T08:56:40.548539+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[27500]: audit 2023-10-09T08:56:40.549980+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[27500]: audit 2023-10-09T08:56:40.551050+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[27500]: audit 2023-10-09T08:56:40.557147+0000 mon.a (mon.0) 307 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:41.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[27500]: cluster 2023-10-09T08:56:41.044225+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:41.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[33912]: cephadm 2023-10-09T08:56:40.534960+0000 mgr.y (mgr.14146) 105 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:56:41.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[33912]: audit 2023-10-09T08:56:40.540447+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:41.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[33912]: audit 2023-10-09T08:56:40.541582+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:56:41.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[33912]: audit 2023-10-09T08:56:40.542479+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:56:41.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[33912]: cephadm 2023-10-09T08:56:40.543135+0000 mgr.y (mgr.14146) 106 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 8120M 2023-10-09T08:56:41.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[33912]: audit 2023-10-09T08:56:40.548539+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:41.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[33912]: audit 2023-10-09T08:56:40.549980+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:56:41.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[33912]: audit 2023-10-09T08:56:40.551050+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:56:41.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[33912]: audit 2023-10-09T08:56:40.557147+0000 mon.a (mon.0) 307 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:56:41.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:41 smithi163 bash[33912]: cluster 2023-10-09T08:56:41.044225+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:44 smithi163 bash[33912]: cluster 2023-10-09T08:56:43.044837+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:44.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:44 smithi163 bash[27500]: cluster 2023-10-09T08:56:43.044837+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:44 smithi181 bash[27712]: cluster 2023-10-09T08:56:43.044837+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:46.457 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:46 smithi163 bash[27500]: cluster 2023-10-09T08:56:45.045414+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:46.457 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:46 smithi163 bash[33912]: cluster 2023-10-09T08:56:45.045414+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:46 smithi181 bash[27712]: cluster 2023-10-09T08:56:45.045414+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:48.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[27500]: cluster 2023-10-09T08:56:47.045962+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:48.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[27500]: audit 2023-10-09T08:56:47.395163+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.163:0/3235899045' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f368e12-b9e7-46dc-93eb-5feee9e4ea2d"}]: dispatch 2023-10-09T08:56:48.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[27500]: audit 2023-10-09T08:56:47.395950+0000 mon.a (mon.0) 308 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f368e12-b9e7-46dc-93eb-5feee9e4ea2d"}]: dispatch 2023-10-09T08:56:48.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[27500]: audit 2023-10-09T08:56:47.403615+0000 mon.a (mon.0) 309 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3f368e12-b9e7-46dc-93eb-5feee9e4ea2d"}]': finished 2023-10-09T08:56:48.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[27500]: cluster 2023-10-09T08:56:47.403756+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T08:56:48.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[27500]: audit 2023-10-09T08:56:47.403928+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:56:48.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[33912]: cluster 2023-10-09T08:56:47.045962+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:48.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[33912]: audit 2023-10-09T08:56:47.395163+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.163:0/3235899045' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f368e12-b9e7-46dc-93eb-5feee9e4ea2d"}]: dispatch 2023-10-09T08:56:48.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[33912]: audit 2023-10-09T08:56:47.395950+0000 mon.a (mon.0) 308 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f368e12-b9e7-46dc-93eb-5feee9e4ea2d"}]: dispatch 2023-10-09T08:56:48.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[33912]: audit 2023-10-09T08:56:47.403615+0000 mon.a (mon.0) 309 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3f368e12-b9e7-46dc-93eb-5feee9e4ea2d"}]': finished 2023-10-09T08:56:48.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[33912]: cluster 2023-10-09T08:56:47.403756+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T08:56:48.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:47 smithi163 bash[33912]: audit 2023-10-09T08:56:47.403928+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:56:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:47 smithi181 bash[27712]: cluster 2023-10-09T08:56:47.045962+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:47 smithi181 bash[27712]: audit 2023-10-09T08:56:47.395163+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.163:0/3235899045' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f368e12-b9e7-46dc-93eb-5feee9e4ea2d"}]: dispatch 2023-10-09T08:56:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:47 smithi181 bash[27712]: audit 2023-10-09T08:56:47.395950+0000 mon.a (mon.0) 308 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f368e12-b9e7-46dc-93eb-5feee9e4ea2d"}]: dispatch 2023-10-09T08:56:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:47 smithi181 bash[27712]: audit 2023-10-09T08:56:47.403615+0000 mon.a (mon.0) 309 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3f368e12-b9e7-46dc-93eb-5feee9e4ea2d"}]': finished 2023-10-09T08:56:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:47 smithi181 bash[27712]: cluster 2023-10-09T08:56:47.403756+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T08:56:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:47 smithi181 bash[27712]: audit 2023-10-09T08:56:47.403928+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:56:49.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:48 smithi163 bash[33912]: audit 2023-10-09T08:56:48.133316+0000 mon.a (mon.0) 312 : audit [DBG] from='client.? 172.21.15.163:0/1998647692' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:56:49.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:48 smithi163 bash[27500]: audit 2023-10-09T08:56:48.133316+0000 mon.a (mon.0) 312 : audit [DBG] from='client.? 172.21.15.163:0/1998647692' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:56:49.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:48 smithi181 bash[27712]: audit 2023-10-09T08:56:48.133316+0000 mon.a (mon.0) 312 : audit [DBG] from='client.? 172.21.15.163:0/1998647692' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:56:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:49 smithi163 bash[27500]: cluster 2023-10-09T08:56:49.046453+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:50.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:49 smithi163 bash[33912]: cluster 2023-10-09T08:56:49.046453+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:50.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:49 smithi181 bash[27712]: cluster 2023-10-09T08:56:49.046453+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:52 smithi163 bash[27500]: cluster 2023-10-09T08:56:51.046972+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:52 smithi163 bash[33912]: cluster 2023-10-09T08:56:51.046972+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:52 smithi181 bash[27712]: cluster 2023-10-09T08:56:51.046972+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:54 smithi163 bash[27500]: cluster 2023-10-09T08:56:53.047480+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:54.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:54 smithi163 bash[33912]: cluster 2023-10-09T08:56:53.047480+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:54 smithi181 bash[27712]: cluster 2023-10-09T08:56:53.047480+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:56 smithi163 bash[33912]: cluster 2023-10-09T08:56:55.047942+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:56 smithi163 bash[27500]: cluster 2023-10-09T08:56:55.047942+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:56 smithi181 bash[27712]: cluster 2023-10-09T08:56:55.047942+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:58.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:56:57 smithi163 bash[27500]: cluster 2023-10-09T08:56:57.048520+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:58.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:56:57 smithi163 bash[33912]: cluster 2023-10-09T08:56:57.048520+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:56:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:56:57 smithi181 bash[27712]: cluster 2023-10-09T08:56:57.048520+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:00.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:00 smithi163 bash[27500]: cluster 2023-10-09T08:56:59.049049+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:00.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:00 smithi163 bash[27500]: audit 2023-10-09T08:56:59.579252+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:57:00.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:00 smithi163 bash[27500]: audit 2023-10-09T08:56:59.581047+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:00 smithi163 bash[33912]: cluster 2023-10-09T08:56:59.049049+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:00 smithi163 bash[33912]: audit 2023-10-09T08:56:59.579252+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:57:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:00 smithi163 bash[33912]: audit 2023-10-09T08:56:59.581047+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:00 smithi181 bash[27712]: cluster 2023-10-09T08:56:59.049049+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:00 smithi181 bash[27712]: audit 2023-10-09T08:56:59.579252+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T08:57:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:00 smithi181 bash[27712]: audit 2023-10-09T08:56:59.581047+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:01.363 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:01 smithi163 bash[27500]: cephadm 2023-10-09T08:56:59.582310+0000 mgr.y (mgr.14146) 117 : cephadm [INF] Deploying daemon osd.2 on smithi163 2023-10-09T08:57:01.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:01 smithi163 bash[33912]: cephadm 2023-10-09T08:56:59.582310+0000 mgr.y (mgr.14146) 117 : cephadm [INF] Deploying daemon osd.2 on smithi163 2023-10-09T08:57:01.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:01 smithi181 bash[27712]: cephadm 2023-10-09T08:56:59.582310+0000 mgr.y (mgr.14146) 117 : cephadm [INF] Deploying daemon osd.2 on smithi163 2023-10-09T08:57:02.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:02 smithi163 bash[27500]: cluster 2023-10-09T08:57:01.049501+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:02.417 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:02 smithi163 bash[33912]: cluster 2023-10-09T08:57:01.049501+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:02 smithi181 bash[27712]: cluster 2023-10-09T08:57:01.049501+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:04.405 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:04 smithi163 bash[33912]: cluster 2023-10-09T08:57:03.050004+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:04.405 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:04 smithi163 bash[33912]: audit 2023-10-09T08:57:03.605964+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:04.405 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:04 smithi163 bash[33912]: audit 2023-10-09T08:57:03.632514+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:04.405 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:04 smithi163 bash[27500]: cluster 2023-10-09T08:57:03.050004+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:04.405 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:04 smithi163 bash[27500]: audit 2023-10-09T08:57:03.605964+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:04.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:04 smithi163 bash[27500]: audit 2023-10-09T08:57:03.632514+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:04 smithi181 bash[27712]: cluster 2023-10-09T08:57:03.050004+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:04 smithi181 bash[27712]: audit 2023-10-09T08:57:03.605964+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:04 smithi181 bash[27712]: audit 2023-10-09T08:57:03.632514+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:06 smithi163 bash[27500]: cluster 2023-10-09T08:57:05.050363+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:06.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:06 smithi163 bash[33912]: cluster 2023-10-09T08:57:05.050363+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:06 smithi181 bash[27712]: cluster 2023-10-09T08:57:05.050363+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:07 smithi163 bash[33912]: cluster 2023-10-09T08:57:07.050847+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:07 smithi163 bash[27500]: cluster 2023-10-09T08:57:07.050847+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:07 smithi181 bash[27712]: cluster 2023-10-09T08:57:07.050847+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:10.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:09 smithi181 bash[27712]: audit 2023-10-09T08:57:08.806442+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:10.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:09 smithi181 bash[27712]: cluster 2023-10-09T08:57:09.051237+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:10.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:09 smithi181 bash[27712]: audit 2023-10-09T08:57:09.069235+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:10.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:09 smithi181 bash[27712]: audit 2023-10-09T08:57:09.070531+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:10.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:09 smithi181 bash[27712]: audit 2023-10-09T08:57:09.077310+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:10.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:09 smithi163 bash[33912]: audit 2023-10-09T08:57:08.806442+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:10.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:09 smithi163 bash[33912]: cluster 2023-10-09T08:57:09.051237+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:10.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:09 smithi163 bash[33912]: audit 2023-10-09T08:57:09.069235+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:10.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:09 smithi163 bash[33912]: audit 2023-10-09T08:57:09.070531+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:10.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:09 smithi163 bash[33912]: audit 2023-10-09T08:57:09.077310+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:10.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:09 smithi163 bash[27500]: audit 2023-10-09T08:57:08.806442+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:10.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:09 smithi163 bash[27500]: cluster 2023-10-09T08:57:09.051237+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:10.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:09 smithi163 bash[27500]: audit 2023-10-09T08:57:09.069235+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:10.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:09 smithi163 bash[27500]: audit 2023-10-09T08:57:09.070531+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:10.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:09 smithi163 bash[27500]: audit 2023-10-09T08:57:09.077310+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:10 smithi163 bash[27500]: audit 2023-10-09T08:57:10.184558+0000 mon.c (mon.2) 7 : audit [INF] from='osd.2 [v2:172.21.15.163:6818/2248501829,v1:172.21.15.163:6819/2248501829]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:57:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:10 smithi163 bash[27500]: audit 2023-10-09T08:57:10.185012+0000 mon.a (mon.0) 321 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:57:10.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:10 smithi163 bash[33912]: audit 2023-10-09T08:57:10.184558+0000 mon.c (mon.2) 7 : audit [INF] from='osd.2 [v2:172.21.15.163:6818/2248501829,v1:172.21.15.163:6819/2248501829]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:57:10.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:10 smithi163 bash[33912]: audit 2023-10-09T08:57:10.185012+0000 mon.a (mon.0) 321 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:57:11.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:10 smithi181 bash[27712]: audit 2023-10-09T08:57:10.184558+0000 mon.c (mon.2) 7 : audit [INF] from='osd.2 [v2:172.21.15.163:6818/2248501829,v1:172.21.15.163:6819/2248501829]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:57:11.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:10 smithi181 bash[27712]: audit 2023-10-09T08:57:10.185012+0000 mon.a (mon.0) 321 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T08:57:11.976 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 2 on host 'smithi163' 2023-10-09T08:57:12.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[27500]: audit 2023-10-09T08:57:10.826965+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T08:57:12.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[27500]: cluster 2023-10-09T08:57:10.827098+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T08:57:12.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[27500]: audit 2023-10-09T08:57:10.827258+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:12.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[27500]: audit 2023-10-09T08:57:10.830110+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.163:6818/2248501829,v1:172.21.15.163:6819/2248501829]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:12.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[27500]: audit 2023-10-09T08:57:10.830534+0000 mon.a (mon.0) 325 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:12.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[27500]: cluster 2023-10-09T08:57:11.051713+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[33912]: audit 2023-10-09T08:57:10.826965+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T08:57:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[33912]: cluster 2023-10-09T08:57:10.827098+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T08:57:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[33912]: audit 2023-10-09T08:57:10.827258+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[33912]: audit 2023-10-09T08:57:10.830110+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.163:6818/2248501829,v1:172.21.15.163:6819/2248501829]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:12.178 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[33912]: audit 2023-10-09T08:57:10.830534+0000 mon.a (mon.0) 325 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:12.178 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:11 smithi163 bash[33912]: cluster 2023-10-09T08:57:11.051713+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:11 smithi181 bash[27712]: audit 2023-10-09T08:57:10.826965+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T08:57:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:11 smithi181 bash[27712]: cluster 2023-10-09T08:57:10.827098+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T08:57:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:11 smithi181 bash[27712]: audit 2023-10-09T08:57:10.827258+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:11 smithi181 bash[27712]: audit 2023-10-09T08:57:10.830110+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.163:6818/2248501829,v1:172.21.15.163:6819/2248501829]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:11 smithi181 bash[27712]: audit 2023-10-09T08:57:10.830534+0000 mon.a (mon.0) 325 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:11 smithi181 bash[27712]: cluster 2023-10-09T08:57:11.051713+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:12.627 DEBUG:teuthology.orchestra.run.smithi163:osd.2> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T08:57:12.629 INFO:tasks.cephadm:Deploying osd.3 on smithi163 with /dev/vg_nvme/lv_1... 2023-10-09T08:57:12.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T08:57:12.829 INFO:journalctl@ceph.osd.2.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:57:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:13 smithi181 bash[27712]: audit 2023-10-09T08:57:11.833206+0000 mon.a (mon.0) 326 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:57:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:13 smithi181 bash[27712]: cluster 2023-10-09T08:57:11.833348+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T08:57:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:13 smithi181 bash[27712]: audit 2023-10-09T08:57:11.834025+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:13 smithi181 bash[27712]: audit 2023-10-09T08:57:11.841951+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:13 smithi181 bash[27712]: audit 2023-10-09T08:57:11.975213+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:13 smithi181 bash[27712]: audit 2023-10-09T08:57:12.001656+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:13.608 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[27500]: audit 2023-10-09T08:57:11.833206+0000 mon.a (mon.0) 326 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:57:13.608 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[27500]: cluster 2023-10-09T08:57:11.833348+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T08:57:13.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[27500]: audit 2023-10-09T08:57:11.834025+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:13.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[27500]: audit 2023-10-09T08:57:11.841951+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:13.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[27500]: audit 2023-10-09T08:57:11.975213+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:13.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[27500]: audit 2023-10-09T08:57:12.001656+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:13.609 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[33912]: audit 2023-10-09T08:57:11.833206+0000 mon.a (mon.0) 326 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:57:13.610 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[33912]: cluster 2023-10-09T08:57:11.833348+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T08:57:13.610 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[33912]: audit 2023-10-09T08:57:11.834025+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:13.610 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[33912]: audit 2023-10-09T08:57:11.841951+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:13.610 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[33912]: audit 2023-10-09T08:57:11.975213+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:13.610 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:13 smithi163 bash[33912]: audit 2023-10-09T08:57:12.001656+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:14 smithi181 bash[27712]: cluster 2023-10-09T08:57:11.170379+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:57:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:14 smithi181 bash[27712]: cluster 2023-10-09T08:57:11.170498+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:57:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:14 smithi181 bash[27712]: cluster 2023-10-09T08:57:12.842283+0000 mon.a (mon.0) 332 : cluster [INF] osd.2 [v2:172.21.15.163:6818/2248501829,v1:172.21.15.163:6819/2248501829] boot 2023-10-09T08:57:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:14 smithi181 bash[27712]: cluster 2023-10-09T08:57:12.842395+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T08:57:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:14 smithi181 bash[27712]: audit 2023-10-09T08:57:12.842840+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:14 smithi181 bash[27712]: audit 2023-10-09T08:57:12.843611+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:57:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:14 smithi181 bash[27712]: cluster 2023-10-09T08:57:13.052186+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:14 smithi181 bash[27712]: audit 2023-10-09T08:57:13.849394+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T08:57:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:14 smithi181 bash[27712]: cluster 2023-10-09T08:57:13.849520+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T08:57:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:14 smithi181 bash[27712]: audit 2023-10-09T08:57:13.850449+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:57:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[27500]: cluster 2023-10-09T08:57:11.170379+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:57:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[27500]: cluster 2023-10-09T08:57:11.170498+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:57:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[27500]: cluster 2023-10-09T08:57:12.842283+0000 mon.a (mon.0) 332 : cluster [INF] osd.2 [v2:172.21.15.163:6818/2248501829,v1:172.21.15.163:6819/2248501829] boot 2023-10-09T08:57:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[27500]: cluster 2023-10-09T08:57:12.842395+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T08:57:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[27500]: audit 2023-10-09T08:57:12.842840+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[27500]: audit 2023-10-09T08:57:12.843611+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:57:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[27500]: cluster 2023-10-09T08:57:13.052186+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[27500]: audit 2023-10-09T08:57:13.849394+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T08:57:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[27500]: cluster 2023-10-09T08:57:13.849520+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T08:57:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[27500]: audit 2023-10-09T08:57:13.850449+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:57:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[33912]: cluster 2023-10-09T08:57:11.170379+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:57:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[33912]: cluster 2023-10-09T08:57:11.170498+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:57:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[33912]: cluster 2023-10-09T08:57:12.842283+0000 mon.a (mon.0) 332 : cluster [INF] osd.2 [v2:172.21.15.163:6818/2248501829,v1:172.21.15.163:6819/2248501829] boot 2023-10-09T08:57:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[33912]: cluster 2023-10-09T08:57:12.842395+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T08:57:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[33912]: audit 2023-10-09T08:57:12.842840+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T08:57:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[33912]: audit 2023-10-09T08:57:12.843611+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T08:57:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[33912]: cluster 2023-10-09T08:57:13.052186+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T08:57:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[33912]: audit 2023-10-09T08:57:13.849394+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T08:57:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[33912]: cluster 2023-10-09T08:57:13.849520+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T08:57:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:14 smithi163 bash[33912]: audit 2023-10-09T08:57:13.850449+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T08:57:15.864 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T08:57:15.865 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T08:57:15.865 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2023-10-09T08:57:15.865 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2023-10-09T08:57:15.865 INFO:teuthology.orchestra.run.smithi163.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.019071 s, 550 MB/s 2023-10-09T08:57:15.865 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2023-10-09T08:57:16.125 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:15 smithi163 bash[27500]: audit 2023-10-09T08:57:14.852953+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T08:57:16.125 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:15 smithi163 bash[27500]: cluster 2023-10-09T08:57:14.853068+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T08:57:16.125 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:15 smithi163 bash[27500]: cluster 2023-10-09T08:57:15.052766+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v105: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:16.126 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:15 smithi163 bash[33912]: audit 2023-10-09T08:57:14.852953+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T08:57:16.126 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:15 smithi163 bash[33912]: cluster 2023-10-09T08:57:14.853068+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T08:57:16.126 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:15 smithi163 bash[33912]: cluster 2023-10-09T08:57:15.052766+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v105: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:16.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:15 smithi181 bash[27712]: audit 2023-10-09T08:57:14.852953+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T08:57:16.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:15 smithi181 bash[27712]: cluster 2023-10-09T08:57:14.853068+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T08:57:16.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:15 smithi181 bash[27712]: cluster 2023-10-09T08:57:15.052766+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v105: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:16.644 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi163:vg_nvme/lv_1 2023-10-09T08:57:17.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:16 smithi163 bash[27500]: cluster 2023-10-09T08:57:15.860357+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T08:57:17.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:16 smithi163 bash[33912]: cluster 2023-10-09T08:57:15.860357+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T08:57:17.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:16 smithi181 bash[27712]: cluster 2023-10-09T08:57:15.860357+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T08:57:18.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:17 smithi163 bash[33912]: cluster 2023-10-09T08:57:17.053328+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:18.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:17 smithi163 bash[27500]: cluster 2023-10-09T08:57:17.053328+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:17 smithi181 bash[27712]: cluster 2023-10-09T08:57:17.053328+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:20.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:20 smithi163 bash[27500]: cluster 2023-10-09T08:57:19.053703+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:20.395 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:20 smithi163 bash[33912]: cluster 2023-10-09T08:57:19.053703+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:20 smithi181 bash[27712]: cluster 2023-10-09T08:57:19.053703+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:21.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:21 smithi163 bash[27500]: audit 2023-10-09T08:57:20.764652+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:57:21.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:21 smithi163 bash[27500]: audit 2023-10-09T08:57:20.768993+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:57:21.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:21 smithi163 bash[27500]: audit 2023-10-09T08:57:20.770243+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:21.409 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:21 smithi163 bash[33912]: audit 2023-10-09T08:57:20.764652+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:57:21.409 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:21 smithi163 bash[33912]: audit 2023-10-09T08:57:20.768993+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:57:21.409 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:21 smithi163 bash[33912]: audit 2023-10-09T08:57:20.770243+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:21 smithi181 bash[27712]: audit 2023-10-09T08:57:20.764652+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:57:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:21 smithi181 bash[27712]: audit 2023-10-09T08:57:20.768993+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:57:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:21 smithi181 bash[27712]: audit 2023-10-09T08:57:20.770243+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[33912]: audit 2023-10-09T08:57:20.758154+0000 mgr.y (mgr.14146) 128 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:57:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[33912]: cluster 2023-10-09T08:57:21.054123+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[33912]: audit 2023-10-09T08:57:22.111202+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[33912]: audit 2023-10-09T08:57:22.112776+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:57:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[33912]: audit 2023-10-09T08:57:22.113871+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:57:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[33912]: audit 2023-10-09T08:57:22.114988+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:57:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[27500]: audit 2023-10-09T08:57:20.758154+0000 mgr.y (mgr.14146) 128 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:57:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[27500]: cluster 2023-10-09T08:57:21.054123+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[27500]: audit 2023-10-09T08:57:22.111202+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[27500]: audit 2023-10-09T08:57:22.112776+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:57:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[27500]: audit 2023-10-09T08:57:22.113871+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:57:22.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:22 smithi163 bash[27500]: audit 2023-10-09T08:57:22.114988+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:57:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:22 smithi181 bash[27712]: audit 2023-10-09T08:57:20.758154+0000 mgr.y (mgr.14146) 128 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:57:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:22 smithi181 bash[27712]: cluster 2023-10-09T08:57:21.054123+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:22 smithi181 bash[27712]: audit 2023-10-09T08:57:22.111202+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:22 smithi181 bash[27712]: audit 2023-10-09T08:57:22.112776+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:57:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:22 smithi181 bash[27712]: audit 2023-10-09T08:57:22.113871+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:57:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:22 smithi181 bash[27712]: audit 2023-10-09T08:57:22.114988+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:57:23.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[27500]: cephadm 2023-10-09T08:57:22.101800+0000 mgr.y (mgr.14146) 130 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:57:23.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[27500]: cephadm 2023-10-09T08:57:22.115801+0000 mgr.y (mgr.14146) 131 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 5413M 2023-10-09T08:57:23.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[27500]: audit 2023-10-09T08:57:22.120140+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:23.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[27500]: audit 2023-10-09T08:57:22.121978+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:23.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[27500]: audit 2023-10-09T08:57:22.126405+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:23.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[27500]: audit 2023-10-09T08:57:22.131567+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:23.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[33912]: cephadm 2023-10-09T08:57:22.101800+0000 mgr.y (mgr.14146) 130 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:57:23.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[33912]: cephadm 2023-10-09T08:57:22.115801+0000 mgr.y (mgr.14146) 131 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 5413M 2023-10-09T08:57:23.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[33912]: audit 2023-10-09T08:57:22.120140+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:23.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[33912]: audit 2023-10-09T08:57:22.121978+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:23.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[33912]: audit 2023-10-09T08:57:22.126405+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:23.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:23 smithi163 bash[33912]: audit 2023-10-09T08:57:22.131567+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:23 smithi181 bash[27712]: cephadm 2023-10-09T08:57:22.101800+0000 mgr.y (mgr.14146) 130 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:57:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:23 smithi181 bash[27712]: cephadm 2023-10-09T08:57:22.115801+0000 mgr.y (mgr.14146) 131 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 5413M 2023-10-09T08:57:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:23 smithi181 bash[27712]: audit 2023-10-09T08:57:22.120140+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:23 smithi181 bash[27712]: audit 2023-10-09T08:57:22.121978+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:23.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:23 smithi181 bash[27712]: audit 2023-10-09T08:57:22.126405+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:23.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:23 smithi181 bash[27712]: audit 2023-10-09T08:57:22.131567+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:24 smithi163 bash[33912]: cluster 2023-10-09T08:57:23.054721+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:24 smithi163 bash[27500]: cluster 2023-10-09T08:57:23.054721+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:24 smithi181 bash[27712]: cluster 2023-10-09T08:57:23.054721+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:26.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:26 smithi163 bash[33912]: cluster 2023-10-09T08:57:25.055313+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:26.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:26 smithi163 bash[27500]: cluster 2023-10-09T08:57:25.055313+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:26 smithi181 bash[27712]: cluster 2023-10-09T08:57:25.055313+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:28.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:27 smithi163 bash[27500]: cluster 2023-10-09T08:57:27.055825+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:28.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:27 smithi163 bash[33912]: cluster 2023-10-09T08:57:27.055825+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:27 smithi181 bash[27712]: cluster 2023-10-09T08:57:27.055825+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:29.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:28 smithi163 bash[33912]: audit 2023-10-09T08:57:28.769683+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.163:0/2330884154' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "228a3c3f-20e9-4d74-a360-64c80ceadaf0"}]: dispatch 2023-10-09T08:57:29.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:28 smithi163 bash[33912]: audit 2023-10-09T08:57:28.770531+0000 mon.a (mon.0) 353 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "228a3c3f-20e9-4d74-a360-64c80ceadaf0"}]: dispatch 2023-10-09T08:57:29.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:28 smithi163 bash[33912]: audit 2023-10-09T08:57:28.777159+0000 mon.a (mon.0) 354 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "228a3c3f-20e9-4d74-a360-64c80ceadaf0"}]': finished 2023-10-09T08:57:29.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:28 smithi163 bash[33912]: cluster 2023-10-09T08:57:28.777307+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T08:57:29.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:28 smithi163 bash[33912]: audit 2023-10-09T08:57:28.777478+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:29.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:28 smithi163 bash[27500]: audit 2023-10-09T08:57:28.769683+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.163:0/2330884154' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "228a3c3f-20e9-4d74-a360-64c80ceadaf0"}]: dispatch 2023-10-09T08:57:29.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:28 smithi163 bash[27500]: audit 2023-10-09T08:57:28.770531+0000 mon.a (mon.0) 353 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "228a3c3f-20e9-4d74-a360-64c80ceadaf0"}]: dispatch 2023-10-09T08:57:29.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:28 smithi163 bash[27500]: audit 2023-10-09T08:57:28.777159+0000 mon.a (mon.0) 354 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "228a3c3f-20e9-4d74-a360-64c80ceadaf0"}]': finished 2023-10-09T08:57:29.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:28 smithi163 bash[27500]: cluster 2023-10-09T08:57:28.777307+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T08:57:29.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:28 smithi163 bash[27500]: audit 2023-10-09T08:57:28.777478+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:29.231 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:28 smithi181 bash[27712]: audit 2023-10-09T08:57:28.769683+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.163:0/2330884154' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "228a3c3f-20e9-4d74-a360-64c80ceadaf0"}]: dispatch 2023-10-09T08:57:29.232 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:28 smithi181 bash[27712]: audit 2023-10-09T08:57:28.770531+0000 mon.a (mon.0) 353 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "228a3c3f-20e9-4d74-a360-64c80ceadaf0"}]: dispatch 2023-10-09T08:57:29.232 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:28 smithi181 bash[27712]: audit 2023-10-09T08:57:28.777159+0000 mon.a (mon.0) 354 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "228a3c3f-20e9-4d74-a360-64c80ceadaf0"}]': finished 2023-10-09T08:57:29.232 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:28 smithi181 bash[27712]: cluster 2023-10-09T08:57:28.777307+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T08:57:29.232 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:28 smithi181 bash[27712]: audit 2023-10-09T08:57:28.777478+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:30.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:29 smithi163 bash[27500]: cluster 2023-10-09T08:57:29.056382+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:30.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:29 smithi163 bash[27500]: audit 2023-10-09T08:57:29.499718+0000 mon.a (mon.0) 357 : audit [DBG] from='client.? 172.21.15.163:0/2103167961' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:57:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:29 smithi163 bash[33912]: cluster 2023-10-09T08:57:29.056382+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:29 smithi163 bash[33912]: audit 2023-10-09T08:57:29.499718+0000 mon.a (mon.0) 357 : audit [DBG] from='client.? 172.21.15.163:0/2103167961' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:57:30.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:29 smithi181 bash[27712]: cluster 2023-10-09T08:57:29.056382+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:30.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:29 smithi181 bash[27712]: audit 2023-10-09T08:57:29.499718+0000 mon.a (mon.0) 357 : audit [DBG] from='client.? 172.21.15.163:0/2103167961' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:57:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:32 smithi163 bash[27500]: cluster 2023-10-09T08:57:31.056927+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:32 smithi163 bash[33912]: cluster 2023-10-09T08:57:31.056927+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:32 smithi181 bash[27712]: cluster 2023-10-09T08:57:31.056927+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:34.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:34 smithi163 bash[27500]: cluster 2023-10-09T08:57:33.057532+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:34.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:34 smithi163 bash[33912]: cluster 2023-10-09T08:57:33.057532+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:34 smithi181 bash[27712]: cluster 2023-10-09T08:57:33.057532+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:36 smithi163 bash[27500]: cluster 2023-10-09T08:57:35.058083+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:36 smithi163 bash[33912]: cluster 2023-10-09T08:57:35.058083+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:36 smithi181 bash[27712]: cluster 2023-10-09T08:57:35.058083+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:38.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:37 smithi163 bash[33912]: cluster 2023-10-09T08:57:37.058677+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:38.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:37 smithi163 bash[27500]: cluster 2023-10-09T08:57:37.058677+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:37 smithi181 bash[27712]: cluster 2023-10-09T08:57:37.058677+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:40.389 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:40 smithi163 bash[33912]: cluster 2023-10-09T08:57:39.059158+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:40.434 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:40 smithi163 bash[27500]: cluster 2023-10-09T08:57:39.059158+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:40.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:40 smithi181 bash[27712]: cluster 2023-10-09T08:57:39.059158+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:41.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:41 smithi163 bash[33912]: audit 2023-10-09T08:57:40.862564+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:57:41.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:41 smithi163 bash[33912]: audit 2023-10-09T08:57:40.864352+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:41 smithi163 bash[27500]: audit 2023-10-09T08:57:40.862564+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:57:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:41 smithi163 bash[27500]: audit 2023-10-09T08:57:40.864352+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:41 smithi181 bash[27712]: audit 2023-10-09T08:57:40.862564+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T08:57:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:41 smithi181 bash[27712]: audit 2023-10-09T08:57:40.864352+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:42 smithi163 bash[27500]: cephadm 2023-10-09T08:57:40.865950+0000 mgr.y (mgr.14146) 141 : cephadm [INF] Deploying daemon osd.3 on smithi163 2023-10-09T08:57:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:42 smithi163 bash[27500]: cluster 2023-10-09T08:57:41.059455+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:42 smithi163 bash[33912]: cephadm 2023-10-09T08:57:40.865950+0000 mgr.y (mgr.14146) 141 : cephadm [INF] Deploying daemon osd.3 on smithi163 2023-10-09T08:57:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:42 smithi163 bash[33912]: cluster 2023-10-09T08:57:41.059455+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:42 smithi181 bash[27712]: cephadm 2023-10-09T08:57:40.865950+0000 mgr.y (mgr.14146) 141 : cephadm [INF] Deploying daemon osd.3 on smithi163 2023-10-09T08:57:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:42 smithi181 bash[27712]: cluster 2023-10-09T08:57:41.059455+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:44.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:44 smithi163 bash[33912]: cluster 2023-10-09T08:57:43.059950+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:44.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:44 smithi163 bash[27500]: cluster 2023-10-09T08:57:43.059950+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:44.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:44 smithi181 bash[27712]: cluster 2023-10-09T08:57:43.059950+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:45 smithi163 bash[27500]: audit 2023-10-09T08:57:44.845943+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:45 smithi163 bash[27500]: audit 2023-10-09T08:57:44.854838+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:45 smithi163 bash[27500]: cluster 2023-10-09T08:57:45.060513+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:45 smithi163 bash[33912]: audit 2023-10-09T08:57:44.845943+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:45 smithi163 bash[33912]: audit 2023-10-09T08:57:44.854838+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:45 smithi163 bash[33912]: cluster 2023-10-09T08:57:45.060513+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:46.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:45 smithi181 bash[27712]: audit 2023-10-09T08:57:44.845943+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:46.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:45 smithi181 bash[27712]: audit 2023-10-09T08:57:44.854838+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:46.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:45 smithi181 bash[27712]: cluster 2023-10-09T08:57:45.060513+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:48.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:47 smithi163 bash[27500]: cluster 2023-10-09T08:57:47.061081+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:48.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:47 smithi163 bash[33912]: cluster 2023-10-09T08:57:47.061081+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:47 smithi181 bash[27712]: cluster 2023-10-09T08:57:47.061081+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:50 smithi163 bash[33912]: cluster 2023-10-09T08:57:49.061688+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:50 smithi163 bash[33912]: audit 2023-10-09T08:57:50.084173+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:50 smithi163 bash[33912]: audit 2023-10-09T08:57:50.085211+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:50 smithi163 bash[33912]: audit 2023-10-09T08:57:50.086133+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:50 smithi163 bash[33912]: audit 2023-10-09T08:57:50.091525+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:50 smithi163 bash[27500]: cluster 2023-10-09T08:57:49.061688+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:50 smithi163 bash[27500]: audit 2023-10-09T08:57:50.084173+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:50 smithi163 bash[27500]: audit 2023-10-09T08:57:50.085211+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:50 smithi163 bash[27500]: audit 2023-10-09T08:57:50.086133+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:50 smithi163 bash[27500]: audit 2023-10-09T08:57:50.091525+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:50 smithi181 bash[27712]: cluster 2023-10-09T08:57:49.061688+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:50 smithi181 bash[27712]: audit 2023-10-09T08:57:50.084173+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:50 smithi181 bash[27712]: audit 2023-10-09T08:57:50.085211+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:57:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:50 smithi181 bash[27712]: audit 2023-10-09T08:57:50.086133+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:57:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:50 smithi181 bash[27712]: audit 2023-10-09T08:57:50.091525+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:52 smithi163 bash[27500]: cluster 2023-10-09T08:57:51.062154+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:52.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:52 smithi163 bash[27500]: audit 2023-10-09T08:57:51.317540+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.163:6826/3899711337,v1:172.21.15.163:6827/3899711337]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:57:52.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:52 smithi163 bash[27500]: audit 2023-10-09T08:57:51.318161+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:57:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:52 smithi163 bash[33912]: cluster 2023-10-09T08:57:51.062154+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:52 smithi163 bash[33912]: audit 2023-10-09T08:57:51.317540+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.163:6826/3899711337,v1:172.21.15.163:6827/3899711337]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:57:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:52 smithi163 bash[33912]: audit 2023-10-09T08:57:51.318161+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:57:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:52 smithi181 bash[27712]: cluster 2023-10-09T08:57:51.062154+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:52 smithi181 bash[27712]: audit 2023-10-09T08:57:51.317540+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.163:6826/3899711337,v1:172.21.15.163:6827/3899711337]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:57:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:52 smithi181 bash[27712]: audit 2023-10-09T08:57:51.318161+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T08:57:53.442 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 3 on host 'smithi163' 2023-10-09T08:57:53.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:53 smithi163 bash[27500]: audit 2023-10-09T08:57:52.138333+0000 mon.a (mon.0) 367 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T08:57:53.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:53 smithi163 bash[27500]: cluster 2023-10-09T08:57:52.138435+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T08:57:53.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:53 smithi163 bash[27500]: audit 2023-10-09T08:57:52.138608+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:53.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:53 smithi163 bash[27500]: audit 2023-10-09T08:57:52.139390+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.163:6826/3899711337,v1:172.21.15.163:6827/3899711337]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:53.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:53 smithi163 bash[27500]: audit 2023-10-09T08:57:52.139774+0000 mon.a (mon.0) 370 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:53.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:53 smithi163 bash[33912]: audit 2023-10-09T08:57:52.138333+0000 mon.a (mon.0) 367 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T08:57:53.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:53 smithi163 bash[33912]: cluster 2023-10-09T08:57:52.138435+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T08:57:53.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:53 smithi163 bash[33912]: audit 2023-10-09T08:57:52.138608+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:53.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:53 smithi163 bash[33912]: audit 2023-10-09T08:57:52.139390+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.163:6826/3899711337,v1:172.21.15.163:6827/3899711337]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:53.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:53 smithi163 bash[33912]: audit 2023-10-09T08:57:52.139774+0000 mon.a (mon.0) 370 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:53 smithi181 bash[27712]: audit 2023-10-09T08:57:52.138333+0000 mon.a (mon.0) 367 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T08:57:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:53 smithi181 bash[27712]: cluster 2023-10-09T08:57:52.138435+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T08:57:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:53 smithi181 bash[27712]: audit 2023-10-09T08:57:52.138608+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:53 smithi181 bash[27712]: audit 2023-10-09T08:57:52.139390+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.163:6826/3899711337,v1:172.21.15.163:6827/3899711337]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:53 smithi181 bash[27712]: audit 2023-10-09T08:57:52.139774+0000 mon.a (mon.0) 370 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T08:57:54.121 DEBUG:teuthology.orchestra.run.smithi163:osd.3> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T08:57:54.124 INFO:tasks.cephadm:Deploying osd.4 on smithi181 with /dev/vg_nvme/lv_4... 2023-10-09T08:57:54.124 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T08:57:54.258 INFO:journalctl@ceph.osd.3.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:57:54.416 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:54 smithi181 bash[27712]: cluster 2023-10-09T08:57:53.062644+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:54.416 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:54 smithi181 bash[27712]: audit 2023-10-09T08:57:53.141381+0000 mon.a (mon.0) 371 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:57:54.416 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:54 smithi181 bash[27712]: cluster 2023-10-09T08:57:53.141490+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T08:57:54.416 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:54 smithi181 bash[27712]: audit 2023-10-09T08:57:53.142077+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:54.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:54 smithi181 bash[27712]: audit 2023-10-09T08:57:53.148377+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:54.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:54 smithi181 bash[27712]: audit 2023-10-09T08:57:53.440700+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:54.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:54 smithi181 bash[27712]: audit 2023-10-09T08:57:53.468316+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[27500]: cluster 2023-10-09T08:57:53.062644+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[27500]: audit 2023-10-09T08:57:53.141381+0000 mon.a (mon.0) 371 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:57:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[27500]: cluster 2023-10-09T08:57:53.141490+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T08:57:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[27500]: audit 2023-10-09T08:57:53.142077+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[27500]: audit 2023-10-09T08:57:53.148377+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[27500]: audit 2023-10-09T08:57:53.440700+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[27500]: audit 2023-10-09T08:57:53.468316+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[33912]: cluster 2023-10-09T08:57:53.062644+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[33912]: audit 2023-10-09T08:57:53.141381+0000 mon.a (mon.0) 371 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T08:57:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[33912]: cluster 2023-10-09T08:57:53.141490+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T08:57:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[33912]: audit 2023-10-09T08:57:53.142077+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[33912]: audit 2023-10-09T08:57:53.148377+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:54.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[33912]: audit 2023-10-09T08:57:53.440700+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:57:54.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:54 smithi163 bash[33912]: audit 2023-10-09T08:57:53.468316+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:57:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:55 smithi181 bash[27712]: cluster 2023-10-09T08:57:52.328605+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:57:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:55 smithi181 bash[27712]: cluster 2023-10-09T08:57:52.328655+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:57:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:55 smithi181 bash[27712]: audit 2023-10-09T08:57:54.147295+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:55 smithi181 bash[27712]: cluster 2023-10-09T08:57:54.154089+0000 mon.a (mon.0) 378 : cluster [INF] osd.3 [v2:172.21.15.163:6826/3899711337,v1:172.21.15.163:6827/3899711337] boot 2023-10-09T08:57:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:55 smithi181 bash[27712]: cluster 2023-10-09T08:57:54.154191+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T08:57:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:55 smithi181 bash[27712]: audit 2023-10-09T08:57:54.155559+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:55.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[27500]: cluster 2023-10-09T08:57:52.328605+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:57:55.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[27500]: cluster 2023-10-09T08:57:52.328655+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:57:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[27500]: audit 2023-10-09T08:57:54.147295+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[27500]: cluster 2023-10-09T08:57:54.154089+0000 mon.a (mon.0) 378 : cluster [INF] osd.3 [v2:172.21.15.163:6826/3899711337,v1:172.21.15.163:6827/3899711337] boot 2023-10-09T08:57:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[27500]: cluster 2023-10-09T08:57:54.154191+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T08:57:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[27500]: audit 2023-10-09T08:57:54.155559+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[33912]: cluster 2023-10-09T08:57:52.328605+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:57:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[33912]: cluster 2023-10-09T08:57:52.328655+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:57:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[33912]: audit 2023-10-09T08:57:54.147295+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[33912]: cluster 2023-10-09T08:57:54.154089+0000 mon.a (mon.0) 378 : cluster [INF] osd.3 [v2:172.21.15.163:6826/3899711337,v1:172.21.15.163:6827/3899711337] boot 2023-10-09T08:57:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[33912]: cluster 2023-10-09T08:57:54.154191+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T08:57:55.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:55 smithi163 bash[33912]: audit 2023-10-09T08:57:54.155559+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T08:57:56.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:56 smithi181 bash[27712]: cluster 2023-10-09T08:57:55.063208+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:56.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:56 smithi181 bash[27712]: cluster 2023-10-09T08:57:55.172824+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T08:57:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:56 smithi163 bash[27500]: cluster 2023-10-09T08:57:55.063208+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:56 smithi163 bash[27500]: cluster 2023-10-09T08:57:55.172824+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T08:57:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:56 smithi163 bash[33912]: cluster 2023-10-09T08:57:55.063208+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T08:57:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:56 smithi163 bash[33912]: cluster 2023-10-09T08:57:55.172824+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T08:57:57.119 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T08:57:57.119 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T08:57:57.119 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2023-10-09T08:57:57.120 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2023-10-09T08:57:57.120 INFO:teuthology.orchestra.run.smithi181.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0160708 s, 652 MB/s 2023-10-09T08:57:57.120 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2023-10-09T08:57:57.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi181:vg_nvme/lv_4 2023-10-09T08:57:58.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:57:57 smithi163 bash[27500]: cluster 2023-10-09T08:57:57.063756+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:57:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:57:57 smithi163 bash[33912]: cluster 2023-10-09T08:57:57.063756+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:57:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:57:57 smithi181 bash[27712]: cluster 2023-10-09T08:57:57.063756+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:00 smithi181 bash[27712]: cluster 2023-10-09T08:57:59.064235+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:00.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:00 smithi163 bash[27500]: cluster 2023-10-09T08:57:59.064235+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:00 smithi163 bash[33912]: cluster 2023-10-09T08:57:59.064235+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:01 smithi163 bash[27500]: audit 2023-10-09T08:58:00.363259+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:58:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:01 smithi163 bash[27500]: audit 2023-10-09T08:58:00.368063+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:58:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:01 smithi163 bash[27500]: audit 2023-10-09T08:58:00.369214+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:01.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:01 smithi163 bash[33912]: audit 2023-10-09T08:58:00.363259+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:58:01.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:01 smithi163 bash[33912]: audit 2023-10-09T08:58:00.368063+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:58:01.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:01 smithi163 bash[33912]: audit 2023-10-09T08:58:00.369214+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:01 smithi181 bash[27712]: audit 2023-10-09T08:58:00.363259+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:58:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:01 smithi181 bash[27712]: audit 2023-10-09T08:58:00.368063+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:58:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:01 smithi181 bash[27712]: audit 2023-10-09T08:58:00.369214+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:02 smithi163 bash[33912]: audit 2023-10-09T08:58:00.356398+0000 mgr.y (mgr.14146) 152 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:58:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:02 smithi163 bash[33912]: cluster 2023-10-09T08:58:01.064686+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:02 smithi163 bash[27500]: audit 2023-10-09T08:58:00.356398+0000 mgr.y (mgr.14146) 152 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:58:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:02 smithi163 bash[27500]: cluster 2023-10-09T08:58:01.064686+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:02 smithi181 bash[27712]: audit 2023-10-09T08:58:00.356398+0000 mgr.y (mgr.14146) 152 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:58:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:02 smithi181 bash[27712]: cluster 2023-10-09T08:58:01.064686+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: cephadm 2023-10-09T08:58:02.218139+0000 mgr.y (mgr.14146) 154 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:58:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: audit 2023-10-09T08:58:02.228987+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: audit 2023-10-09T08:58:02.230596+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: audit 2023-10-09T08:58:02.231889+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: audit 2023-10-09T08:58:02.233047+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: audit 2023-10-09T08:58:02.234196+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: cephadm 2023-10-09T08:58:02.235046+0000 mgr.y (mgr.14146) 155 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 4060M 2023-10-09T08:58:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: audit 2023-10-09T08:58:02.243873+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: audit 2023-10-09T08:58:02.245767+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: audit 2023-10-09T08:58:02.247078+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:03 smithi181 bash[27712]: audit 2023-10-09T08:58:02.257182+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: cephadm 2023-10-09T08:58:02.218139+0000 mgr.y (mgr.14146) 154 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:58:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: audit 2023-10-09T08:58:02.228987+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: audit 2023-10-09T08:58:02.230596+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: audit 2023-10-09T08:58:02.231889+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: audit 2023-10-09T08:58:02.233047+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: audit 2023-10-09T08:58:02.234196+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: cephadm 2023-10-09T08:58:02.235046+0000 mgr.y (mgr.14146) 155 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 4060M 2023-10-09T08:58:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: audit 2023-10-09T08:58:02.243873+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: audit 2023-10-09T08:58:02.245767+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: audit 2023-10-09T08:58:02.247078+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[27500]: audit 2023-10-09T08:58:02.257182+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: cephadm 2023-10-09T08:58:02.218139+0000 mgr.y (mgr.14146) 154 : cephadm [INF] Detected new or changed devices on smithi163 2023-10-09T08:58:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: audit 2023-10-09T08:58:02.228987+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: audit 2023-10-09T08:58:02.230596+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: audit 2023-10-09T08:58:02.231889+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: audit 2023-10-09T08:58:02.233047+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: audit 2023-10-09T08:58:02.234196+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: cephadm 2023-10-09T08:58:02.235046+0000 mgr.y (mgr.14146) 155 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 4060M 2023-10-09T08:58:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: audit 2023-10-09T08:58:02.243873+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: audit 2023-10-09T08:58:02.245767+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:03.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: audit 2023-10-09T08:58:02.247078+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:03.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:03 smithi163 bash[33912]: audit 2023-10-09T08:58:02.257182+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:04 smithi181 bash[27712]: cluster 2023-10-09T08:58:03.065217+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:04 smithi163 bash[27500]: cluster 2023-10-09T08:58:03.065217+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:04.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:04 smithi163 bash[33912]: cluster 2023-10-09T08:58:03.065217+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:06 smithi181 bash[27712]: cluster 2023-10-09T08:58:05.065709+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:06 smithi163 bash[27500]: cluster 2023-10-09T08:58:05.065709+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:06 smithi163 bash[33912]: cluster 2023-10-09T08:58:05.065709+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:07 smithi163 bash[27500]: cluster 2023-10-09T08:58:07.066299+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:07 smithi163 bash[33912]: cluster 2023-10-09T08:58:07.066299+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:07 smithi181 bash[27712]: cluster 2023-10-09T08:58:07.066299+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:09.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[27500]: audit 2023-10-09T08:58:08.196949+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.181:0/514152613' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70b76f58-2db5-421c-a558-db7c00067177"}]: dispatch 2023-10-09T08:58:09.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[27500]: audit 2023-10-09T08:58:08.199261+0000 mon.a (mon.0) 394 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70b76f58-2db5-421c-a558-db7c00067177"}]: dispatch 2023-10-09T08:58:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[27500]: audit 2023-10-09T08:58:08.206365+0000 mon.a (mon.0) 395 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70b76f58-2db5-421c-a558-db7c00067177"}]': finished 2023-10-09T08:58:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[27500]: cluster 2023-10-09T08:58:08.206456+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T08:58:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[27500]: audit 2023-10-09T08:58:08.206717+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[27500]: audit 2023-10-09T08:58:08.911338+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.181:0/46697068' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:58:09.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[33912]: audit 2023-10-09T08:58:08.196949+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.181:0/514152613' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70b76f58-2db5-421c-a558-db7c00067177"}]: dispatch 2023-10-09T08:58:09.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[33912]: audit 2023-10-09T08:58:08.199261+0000 mon.a (mon.0) 394 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70b76f58-2db5-421c-a558-db7c00067177"}]: dispatch 2023-10-09T08:58:09.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[33912]: audit 2023-10-09T08:58:08.206365+0000 mon.a (mon.0) 395 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70b76f58-2db5-421c-a558-db7c00067177"}]': finished 2023-10-09T08:58:09.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[33912]: cluster 2023-10-09T08:58:08.206456+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T08:58:09.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[33912]: audit 2023-10-09T08:58:08.206717+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:09.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:08 smithi163 bash[33912]: audit 2023-10-09T08:58:08.911338+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.181:0/46697068' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:58:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:08 smithi181 bash[27712]: audit 2023-10-09T08:58:08.196949+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.181:0/514152613' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70b76f58-2db5-421c-a558-db7c00067177"}]: dispatch 2023-10-09T08:58:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:08 smithi181 bash[27712]: audit 2023-10-09T08:58:08.199261+0000 mon.a (mon.0) 394 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70b76f58-2db5-421c-a558-db7c00067177"}]: dispatch 2023-10-09T08:58:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:08 smithi181 bash[27712]: audit 2023-10-09T08:58:08.206365+0000 mon.a (mon.0) 395 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70b76f58-2db5-421c-a558-db7c00067177"}]': finished 2023-10-09T08:58:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:08 smithi181 bash[27712]: cluster 2023-10-09T08:58:08.206456+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T08:58:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:08 smithi181 bash[27712]: audit 2023-10-09T08:58:08.206717+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:09.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:08 smithi181 bash[27712]: audit 2023-10-09T08:58:08.911338+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.181:0/46697068' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:58:10.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:09 smithi163 bash[27500]: cluster 2023-10-09T08:58:09.066852+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:10.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:09 smithi163 bash[33912]: cluster 2023-10-09T08:58:09.066852+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:09 smithi181 bash[27712]: cluster 2023-10-09T08:58:09.066852+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:12 smithi163 bash[27500]: cluster 2023-10-09T08:58:11.067350+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:12 smithi163 bash[33912]: cluster 2023-10-09T08:58:11.067350+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:12 smithi181 bash[27712]: cluster 2023-10-09T08:58:11.067350+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:14 smithi163 bash[27500]: cluster 2023-10-09T08:58:13.067911+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:14 smithi163 bash[33912]: cluster 2023-10-09T08:58:13.067911+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:14 smithi181 bash[27712]: cluster 2023-10-09T08:58:13.067911+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:16.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:16 smithi163 bash[27500]: cluster 2023-10-09T08:58:15.068367+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:16.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:16 smithi163 bash[33912]: cluster 2023-10-09T08:58:15.068367+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:16 smithi181 bash[27712]: cluster 2023-10-09T08:58:15.068367+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:18.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:17 smithi163 bash[27500]: cluster 2023-10-09T08:58:17.068987+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:18.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:17 smithi163 bash[33912]: cluster 2023-10-09T08:58:17.068987+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:18.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:17 smithi181 bash[27712]: cluster 2023-10-09T08:58:17.068987+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:20.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:20 smithi163 bash[27500]: cluster 2023-10-09T08:58:19.069519+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:20.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:20 smithi163 bash[27500]: audit 2023-10-09T08:58:20.067535+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T08:58:20.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:20 smithi163 bash[27500]: audit 2023-10-09T08:58:20.069385+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:20.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:20 smithi163 bash[33912]: cluster 2023-10-09T08:58:19.069519+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:20 smithi163 bash[33912]: audit 2023-10-09T08:58:20.067535+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T08:58:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:20 smithi163 bash[33912]: audit 2023-10-09T08:58:20.069385+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:20 smithi181 bash[27712]: cluster 2023-10-09T08:58:19.069519+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:20 smithi181 bash[27712]: audit 2023-10-09T08:58:20.067535+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T08:58:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:20 smithi181 bash[27712]: audit 2023-10-09T08:58:20.069385+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:21.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:21 smithi181 bash[27712]: cephadm 2023-10-09T08:58:20.070952+0000 mgr.y (mgr.14146) 165 : cephadm [INF] Deploying daemon osd.4 on smithi181 2023-10-09T08:58:21.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:21 smithi163 bash[27500]: cephadm 2023-10-09T08:58:20.070952+0000 mgr.y (mgr.14146) 165 : cephadm [INF] Deploying daemon osd.4 on smithi181 2023-10-09T08:58:21.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:21 smithi163 bash[33912]: cephadm 2023-10-09T08:58:20.070952+0000 mgr.y (mgr.14146) 165 : cephadm [INF] Deploying daemon osd.4 on smithi181 2023-10-09T08:58:22.401 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:22 smithi181 bash[27712]: cluster 2023-10-09T08:58:21.070063+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:22.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:22 smithi163 bash[27500]: cluster 2023-10-09T08:58:21.070063+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:22 smithi163 bash[33912]: cluster 2023-10-09T08:58:21.070063+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:24 smithi163 bash[27500]: cluster 2023-10-09T08:58:23.070607+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:24 smithi163 bash[27500]: audit 2023-10-09T08:58:23.821933+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:24 smithi163 bash[27500]: audit 2023-10-09T08:58:23.839910+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:58:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:24 smithi163 bash[33912]: cluster 2023-10-09T08:58:23.070607+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:24 smithi163 bash[33912]: audit 2023-10-09T08:58:23.821933+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:24 smithi163 bash[33912]: audit 2023-10-09T08:58:23.839910+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:58:24.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:24 smithi181 bash[27712]: cluster 2023-10-09T08:58:23.070607+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:24.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:24 smithi181 bash[27712]: audit 2023-10-09T08:58:23.821933+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:24.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:24 smithi181 bash[27712]: audit 2023-10-09T08:58:23.839910+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:58:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:26 smithi181 bash[27712]: cluster 2023-10-09T08:58:25.071164+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:26.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:26 smithi163 bash[27500]: cluster 2023-10-09T08:58:25.071164+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:26.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:26 smithi163 bash[33912]: cluster 2023-10-09T08:58:25.071164+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:28.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:27 smithi163 bash[27500]: cluster 2023-10-09T08:58:27.071668+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:28.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:27 smithi163 bash[33912]: cluster 2023-10-09T08:58:27.071668+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:27 smithi181 bash[27712]: cluster 2023-10-09T08:58:27.071668+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:29 smithi181 bash[27712]: audit 2023-10-09T08:58:28.896215+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:29 smithi181 bash[27712]: cluster 2023-10-09T08:58:29.072185+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:29 smithi181 bash[27712]: audit 2023-10-09T08:58:29.178927+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:30.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:29 smithi181 bash[27712]: audit 2023-10-09T08:58:29.181235+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:30.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:29 smithi181 bash[27712]: audit 2023-10-09T08:58:29.189861+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:30.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:29 smithi163 bash[27500]: audit 2023-10-09T08:58:28.896215+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:30.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:29 smithi163 bash[27500]: cluster 2023-10-09T08:58:29.072185+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:29 smithi163 bash[27500]: audit 2023-10-09T08:58:29.178927+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:29 smithi163 bash[27500]: audit 2023-10-09T08:58:29.181235+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:29 smithi163 bash[27500]: audit 2023-10-09T08:58:29.189861+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:29 smithi163 bash[33912]: audit 2023-10-09T08:58:28.896215+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:29 smithi163 bash[33912]: cluster 2023-10-09T08:58:29.072185+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:29 smithi163 bash[33912]: audit 2023-10-09T08:58:29.178927+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:29 smithi163 bash[33912]: audit 2023-10-09T08:58:29.181235+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:29 smithi163 bash[33912]: audit 2023-10-09T08:58:29.189861+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:31.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:30 smithi163 bash[27500]: audit 2023-10-09T08:58:30.156859+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T08:58:31.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:30 smithi163 bash[27500]: audit 2023-10-09T08:58:30.159258+0000 mon.a (mon.0) 406 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T08:58:31.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:30 smithi163 bash[33912]: audit 2023-10-09T08:58:30.156859+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T08:58:31.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:30 smithi163 bash[33912]: audit 2023-10-09T08:58:30.159258+0000 mon.a (mon.0) 406 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T08:58:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:30 smithi181 bash[27712]: audit 2023-10-09T08:58:30.156859+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T08:58:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:30 smithi181 bash[27712]: audit 2023-10-09T08:58:30.159258+0000 mon.a (mon.0) 406 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T08:58:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:31 smithi181 bash[27712]: audit 2023-10-09T08:58:30.915406+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:58:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:31 smithi181 bash[27712]: audit 2023-10-09T08:58:30.915672+0000 mon.a (mon.0) 407 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T08:58:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:31 smithi181 bash[27712]: cluster 2023-10-09T08:58:30.915784+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T08:58:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:31 smithi181 bash[27712]: audit 2023-10-09T08:58:30.915951+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:31 smithi181 bash[27712]: audit 2023-10-09T08:58:30.917821+0000 mon.a (mon.0) 410 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:58:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:31 smithi181 bash[27712]: cluster 2023-10-09T08:58:31.072692+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:32.208 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 4 on host 'smithi181' 2023-10-09T08:58:32.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[33912]: audit 2023-10-09T08:58:30.915406+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:58:32.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[33912]: audit 2023-10-09T08:58:30.915672+0000 mon.a (mon.0) 407 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T08:58:32.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[33912]: cluster 2023-10-09T08:58:30.915784+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T08:58:32.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[33912]: audit 2023-10-09T08:58:30.915951+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:32.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[33912]: audit 2023-10-09T08:58:30.917821+0000 mon.a (mon.0) 410 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:58:32.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[33912]: cluster 2023-10-09T08:58:31.072692+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[27500]: audit 2023-10-09T08:58:30.915406+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:58:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[27500]: audit 2023-10-09T08:58:30.915672+0000 mon.a (mon.0) 407 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T08:58:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[27500]: cluster 2023-10-09T08:58:30.915784+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T08:58:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[27500]: audit 2023-10-09T08:58:30.915951+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[27500]: audit 2023-10-09T08:58:30.917821+0000 mon.a (mon.0) 410 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:58:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:31 smithi163 bash[27500]: cluster 2023-10-09T08:58:31.072692+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:32.838 DEBUG:teuthology.orchestra.run.smithi181:osd.4> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T08:58:32.841 INFO:tasks.cephadm:Deploying osd.5 on smithi181 with /dev/vg_nvme/lv_3... 2023-10-09T08:58:32.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T08:58:33.022 INFO:journalctl@ceph.osd.4.smithi181.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:58:33.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[27500]: audit 2023-10-09T08:58:31.917703+0000 mon.a (mon.0) 411 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T08:58:33.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[27500]: cluster 2023-10-09T08:58:31.917884+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T08:58:33.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[27500]: audit 2023-10-09T08:58:31.918725+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:33.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[27500]: audit 2023-10-09T08:58:31.925245+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:33.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[27500]: cluster 2023-10-09T08:58:31.952101+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2023-10-09T08:58:33.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[27500]: audit 2023-10-09T08:58:31.952572+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:33.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[27500]: audit 2023-10-09T08:58:32.207434+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:33.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[27500]: audit 2023-10-09T08:58:32.216102+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:58:33.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[33912]: audit 2023-10-09T08:58:31.917703+0000 mon.a (mon.0) 411 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T08:58:33.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[33912]: cluster 2023-10-09T08:58:31.917884+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T08:58:33.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[33912]: audit 2023-10-09T08:58:31.918725+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:33.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[33912]: audit 2023-10-09T08:58:31.925245+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:33.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[33912]: cluster 2023-10-09T08:58:31.952101+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2023-10-09T08:58:33.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[33912]: audit 2023-10-09T08:58:31.952572+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:33.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[33912]: audit 2023-10-09T08:58:32.207434+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:33.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:32 smithi163 bash[33912]: audit 2023-10-09T08:58:32.216102+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:58:33.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:32 smithi181 bash[27712]: audit 2023-10-09T08:58:31.917703+0000 mon.a (mon.0) 411 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T08:58:33.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:32 smithi181 bash[27712]: cluster 2023-10-09T08:58:31.917884+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T08:58:33.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:32 smithi181 bash[27712]: audit 2023-10-09T08:58:31.918725+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:33.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:32 smithi181 bash[27712]: audit 2023-10-09T08:58:31.925245+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:33.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:32 smithi181 bash[27712]: cluster 2023-10-09T08:58:31.952101+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2023-10-09T08:58:33.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:32 smithi181 bash[27712]: audit 2023-10-09T08:58:31.952572+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:33.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:32 smithi181 bash[27712]: audit 2023-10-09T08:58:32.207434+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:33.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:32 smithi181 bash[27712]: audit 2023-10-09T08:58:32.216102+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:58:34.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:33 smithi181 bash[27712]: cluster 2023-10-09T08:58:31.119813+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:58:34.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:33 smithi181 bash[27712]: cluster 2023-10-09T08:58:31.119890+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:58:34.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:33 smithi181 bash[27712]: audit 2023-10-09T08:58:32.928511+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:34.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:33 smithi181 bash[27712]: cluster 2023-10-09T08:58:32.953290+0000 mon.a (mon.0) 420 : cluster [INF] osd.4 [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913] boot 2023-10-09T08:58:34.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:33 smithi181 bash[27712]: cluster 2023-10-09T08:58:32.953390+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T08:58:34.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:33 smithi181 bash[27712]: audit 2023-10-09T08:58:32.953643+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:34.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:33 smithi181 bash[27712]: cluster 2023-10-09T08:58:33.073326+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v155: 1 pgs: 1 remapped+peering; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:34.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[27500]: cluster 2023-10-09T08:58:31.119813+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:58:34.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[27500]: cluster 2023-10-09T08:58:31.119890+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:58:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[27500]: audit 2023-10-09T08:58:32.928511+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[27500]: cluster 2023-10-09T08:58:32.953290+0000 mon.a (mon.0) 420 : cluster [INF] osd.4 [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913] boot 2023-10-09T08:58:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[27500]: cluster 2023-10-09T08:58:32.953390+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T08:58:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[27500]: audit 2023-10-09T08:58:32.953643+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[27500]: cluster 2023-10-09T08:58:33.073326+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v155: 1 pgs: 1 remapped+peering; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:34.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[33912]: cluster 2023-10-09T08:58:31.119813+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:58:34.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[33912]: cluster 2023-10-09T08:58:31.119890+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:58:34.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[33912]: audit 2023-10-09T08:58:32.928511+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:34.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[33912]: cluster 2023-10-09T08:58:32.953290+0000 mon.a (mon.0) 420 : cluster [INF] osd.4 [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913] boot 2023-10-09T08:58:34.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[33912]: cluster 2023-10-09T08:58:32.953390+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T08:58:34.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[33912]: audit 2023-10-09T08:58:32.953643+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T08:58:34.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:33 smithi163 bash[33912]: cluster 2023-10-09T08:58:33.073326+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v155: 1 pgs: 1 remapped+peering; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T08:58:35.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:34 smithi163 bash[27500]: cluster 2023-10-09T08:58:33.958247+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T08:58:35.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:34 smithi163 bash[33912]: cluster 2023-10-09T08:58:33.958247+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T08:58:35.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:34 smithi181 bash[27712]: cluster 2023-10-09T08:58:33.958247+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T08:58:35.892 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T08:58:35.893 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T08:58:35.893 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2023-10-09T08:58:35.893 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2023-10-09T08:58:35.893 INFO:teuthology.orchestra.run.smithi181.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0161514 s, 649 MB/s 2023-10-09T08:58:35.893 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2023-10-09T08:58:36.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:35 smithi163 bash[27500]: cluster 2023-10-09T08:58:34.961649+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2023-10-09T08:58:36.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:35 smithi163 bash[27500]: cluster 2023-10-09T08:58:35.073910+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v158: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:36.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:35 smithi163 bash[33912]: cluster 2023-10-09T08:58:34.961649+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2023-10-09T08:58:36.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:35 smithi163 bash[33912]: cluster 2023-10-09T08:58:35.073910+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v158: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:36.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:35 smithi181 bash[27712]: cluster 2023-10-09T08:58:34.961649+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2023-10-09T08:58:36.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:35 smithi181 bash[27712]: cluster 2023-10-09T08:58:35.073910+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v158: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:36.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi181:vg_nvme/lv_3 2023-10-09T08:58:38.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:37 smithi181 bash[27712]: cluster 2023-10-09T08:58:37.074453+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:38.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:37 smithi163 bash[27500]: cluster 2023-10-09T08:58:37.074453+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:38.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:37 smithi163 bash[33912]: cluster 2023-10-09T08:58:37.074453+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:40.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:40 smithi181 bash[27712]: cluster 2023-10-09T08:58:39.074952+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:40.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:40 smithi163 bash[27500]: cluster 2023-10-09T08:58:39.074952+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:40.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:40 smithi163 bash[33912]: cluster 2023-10-09T08:58:39.074952+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:41 smithi163 bash[27500]: audit 2023-10-09T08:58:40.351104+0000 mgr.y (mgr.14146) 176 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:58:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:41 smithi163 bash[27500]: audit 2023-10-09T08:58:40.358004+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:58:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:41 smithi163 bash[27500]: audit 2023-10-09T08:58:40.363401+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:58:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:41 smithi163 bash[27500]: audit 2023-10-09T08:58:40.364830+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:41.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:41 smithi163 bash[33912]: audit 2023-10-09T08:58:40.351104+0000 mgr.y (mgr.14146) 176 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:58:41.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:41 smithi163 bash[33912]: audit 2023-10-09T08:58:40.358004+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:58:41.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:41 smithi163 bash[33912]: audit 2023-10-09T08:58:40.363401+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:58:41.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:41 smithi163 bash[33912]: audit 2023-10-09T08:58:40.364830+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:41 smithi181 bash[27712]: audit 2023-10-09T08:58:40.351104+0000 mgr.y (mgr.14146) 176 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:58:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:41 smithi181 bash[27712]: audit 2023-10-09T08:58:40.358004+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:58:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:41 smithi181 bash[27712]: audit 2023-10-09T08:58:40.363401+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:58:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:41 smithi181 bash[27712]: audit 2023-10-09T08:58:40.364830+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[27500]: cluster 2023-10-09T08:58:41.075472+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[27500]: audit 2023-10-09T08:58:41.506641+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[27500]: audit 2023-10-09T08:58:41.508203+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[27500]: audit 2023-10-09T08:58:41.517826+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[27500]: audit 2023-10-09T08:58:41.519572+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[27500]: audit 2023-10-09T08:58:41.520995+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[27500]: audit 2023-10-09T08:58:41.530045+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[33912]: cluster 2023-10-09T08:58:41.075472+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[33912]: audit 2023-10-09T08:58:41.506641+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[33912]: audit 2023-10-09T08:58:41.508203+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[33912]: audit 2023-10-09T08:58:41.517826+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[33912]: audit 2023-10-09T08:58:41.519572+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[33912]: audit 2023-10-09T08:58:41.520995+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:42 smithi163 bash[33912]: audit 2023-10-09T08:58:41.530045+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:42 smithi181 bash[27712]: cluster 2023-10-09T08:58:41.075472+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:42 smithi181 bash[27712]: audit 2023-10-09T08:58:41.506641+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:42 smithi181 bash[27712]: audit 2023-10-09T08:58:41.508203+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:58:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:42 smithi181 bash[27712]: audit 2023-10-09T08:58:41.517826+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:42 smithi181 bash[27712]: audit 2023-10-09T08:58:41.519572+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:58:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:42 smithi181 bash[27712]: audit 2023-10-09T08:58:41.520995+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:58:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:42 smithi181 bash[27712]: audit 2023-10-09T08:58:41.530045+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:58:43.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:43 smithi163 bash[27500]: cephadm 2023-10-09T08:58:41.494051+0000 mgr.y (mgr.14146) 178 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T08:58:43.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:43 smithi163 bash[27500]: cephadm 2023-10-09T08:58:41.509206+0000 mgr.y (mgr.14146) 179 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 17265M 2023-10-09T08:58:43.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:43 smithi163 bash[33912]: cephadm 2023-10-09T08:58:41.494051+0000 mgr.y (mgr.14146) 178 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T08:58:43.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:43 smithi163 bash[33912]: cephadm 2023-10-09T08:58:41.509206+0000 mgr.y (mgr.14146) 179 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 17265M 2023-10-09T08:58:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:43 smithi181 bash[27712]: cephadm 2023-10-09T08:58:41.494051+0000 mgr.y (mgr.14146) 178 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T08:58:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:43 smithi181 bash[27712]: cephadm 2023-10-09T08:58:41.509206+0000 mgr.y (mgr.14146) 179 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 17265M 2023-10-09T08:58:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:44 smithi163 bash[27500]: cluster 2023-10-09T08:58:43.076046+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:44 smithi163 bash[33912]: cluster 2023-10-09T08:58:43.076046+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:44 smithi181 bash[27712]: cluster 2023-10-09T08:58:43.076046+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:46 smithi163 bash[27500]: cluster 2023-10-09T08:58:45.076601+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:46 smithi163 bash[33912]: cluster 2023-10-09T08:58:45.076601+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:46 smithi181 bash[27712]: cluster 2023-10-09T08:58:45.076601+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:48.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:47 smithi163 bash[27500]: cluster 2023-10-09T08:58:47.077161+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:48.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:47 smithi163 bash[33912]: cluster 2023-10-09T08:58:47.077161+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:48.322 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:47 smithi181 bash[27712]: cluster 2023-10-09T08:58:47.077161+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:49.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[27500]: audit 2023-10-09T08:58:48.153659+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.181:0/1277365388' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4"}]: dispatch 2023-10-09T08:58:49.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[27500]: audit 2023-10-09T08:58:48.156365+0000 mon.a (mon.0) 434 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4"}]: dispatch 2023-10-09T08:58:49.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[27500]: audit 2023-10-09T08:58:48.163997+0000 mon.a (mon.0) 435 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4"}]': finished 2023-10-09T08:58:49.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[27500]: cluster 2023-10-09T08:58:48.164173+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T08:58:49.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[27500]: audit 2023-10-09T08:58:48.164426+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:58:49.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[27500]: audit 2023-10-09T08:58:48.874856+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.181:0/54623513' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:58:49.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[33912]: audit 2023-10-09T08:58:48.153659+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.181:0/1277365388' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4"}]: dispatch 2023-10-09T08:58:49.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[33912]: audit 2023-10-09T08:58:48.156365+0000 mon.a (mon.0) 434 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4"}]: dispatch 2023-10-09T08:58:49.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[33912]: audit 2023-10-09T08:58:48.163997+0000 mon.a (mon.0) 435 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4"}]': finished 2023-10-09T08:58:49.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[33912]: cluster 2023-10-09T08:58:48.164173+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T08:58:49.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[33912]: audit 2023-10-09T08:58:48.164426+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:58:49.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:48 smithi163 bash[33912]: audit 2023-10-09T08:58:48.874856+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.181:0/54623513' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:58:49.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:48 smithi181 bash[27712]: audit 2023-10-09T08:58:48.153659+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.181:0/1277365388' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4"}]: dispatch 2023-10-09T08:58:49.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:48 smithi181 bash[27712]: audit 2023-10-09T08:58:48.156365+0000 mon.a (mon.0) 434 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4"}]: dispatch 2023-10-09T08:58:49.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:48 smithi181 bash[27712]: audit 2023-10-09T08:58:48.163997+0000 mon.a (mon.0) 435 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4"}]': finished 2023-10-09T08:58:49.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:48 smithi181 bash[27712]: cluster 2023-10-09T08:58:48.164173+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T08:58:49.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:48 smithi181 bash[27712]: audit 2023-10-09T08:58:48.164426+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:58:49.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:48 smithi181 bash[27712]: audit 2023-10-09T08:58:48.874856+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.181:0/54623513' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:58:50.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:49 smithi181 bash[27712]: cluster 2023-10-09T08:58:49.077698+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:50.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:49 smithi163 bash[27500]: cluster 2023-10-09T08:58:49.077698+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:49 smithi163 bash[33912]: cluster 2023-10-09T08:58:49.077698+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:52 smithi163 bash[27500]: cluster 2023-10-09T08:58:51.078208+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:52 smithi163 bash[33912]: cluster 2023-10-09T08:58:51.078208+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:52 smithi181 bash[27712]: cluster 2023-10-09T08:58:51.078208+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:54 smithi163 bash[27500]: cluster 2023-10-09T08:58:53.078791+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:54.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:54 smithi163 bash[33912]: cluster 2023-10-09T08:58:53.078791+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:54 smithi181 bash[27712]: cluster 2023-10-09T08:58:53.078791+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:56 smithi163 bash[27500]: cluster 2023-10-09T08:58:55.079326+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:56 smithi163 bash[33912]: cluster 2023-10-09T08:58:55.079326+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:56 smithi181 bash[27712]: cluster 2023-10-09T08:58:55.079326+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:58.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:58:57 smithi163 bash[27500]: cluster 2023-10-09T08:58:57.079916+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:58.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:58:57 smithi163 bash[33912]: cluster 2023-10-09T08:58:57.079916+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:58:58.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:58:57 smithi181 bash[27712]: cluster 2023-10-09T08:58:57.079916+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:00.443 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:00 smithi181 bash[27712]: cluster 2023-10-09T08:58:59.080472+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:00.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:00 smithi163 bash[27500]: cluster 2023-10-09T08:58:59.080472+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:00 smithi163 bash[33912]: cluster 2023-10-09T08:58:59.080472+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:01 smithi163 bash[27500]: audit 2023-10-09T08:59:00.182385+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T08:59:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:01 smithi163 bash[27500]: audit 2023-10-09T08:59:00.184324+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:01 smithi163 bash[27500]: cephadm 2023-10-09T08:59:00.185519+0000 mgr.y (mgr.14146) 189 : cephadm [INF] Deploying daemon osd.5 on smithi181 2023-10-09T08:59:01.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:01 smithi163 bash[33912]: audit 2023-10-09T08:59:00.182385+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T08:59:01.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:01 smithi163 bash[33912]: audit 2023-10-09T08:59:00.184324+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:01.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:01 smithi163 bash[33912]: cephadm 2023-10-09T08:59:00.185519+0000 mgr.y (mgr.14146) 189 : cephadm [INF] Deploying daemon osd.5 on smithi181 2023-10-09T08:59:01.532 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:01 smithi181 bash[27712]: audit 2023-10-09T08:59:00.182385+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T08:59:01.532 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:01 smithi181 bash[27712]: audit 2023-10-09T08:59:00.184324+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:01.532 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:01 smithi181 bash[27712]: cephadm 2023-10-09T08:59:00.185519+0000 mgr.y (mgr.14146) 189 : cephadm [INF] Deploying daemon osd.5 on smithi181 2023-10-09T08:59:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:02 smithi181 bash[27712]: cluster 2023-10-09T08:59:01.080986+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:02.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:02 smithi163 bash[27500]: cluster 2023-10-09T08:59:01.080986+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:02 smithi163 bash[33912]: cluster 2023-10-09T08:59:01.080986+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:04 smithi163 bash[27500]: cluster 2023-10-09T08:59:03.081614+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:04 smithi163 bash[33912]: cluster 2023-10-09T08:59:03.081614+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:04 smithi181 bash[27712]: cluster 2023-10-09T08:59:03.081614+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:05 smithi163 bash[27500]: audit 2023-10-09T08:59:04.152716+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:05 smithi163 bash[27500]: audit 2023-10-09T08:59:04.159435+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:05 smithi163 bash[33912]: audit 2023-10-09T08:59:04.152716+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:05.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:05 smithi163 bash[33912]: audit 2023-10-09T08:59:04.159435+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:05.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:05 smithi181 bash[27712]: audit 2023-10-09T08:59:04.152716+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:05.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:05 smithi181 bash[27712]: audit 2023-10-09T08:59:04.159435+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:06 smithi163 bash[27500]: cluster 2023-10-09T08:59:05.082160+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:06.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:06 smithi163 bash[33912]: cluster 2023-10-09T08:59:05.082160+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:06 smithi181 bash[27712]: cluster 2023-10-09T08:59:05.082160+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:07 smithi163 bash[33912]: cluster 2023-10-09T08:59:07.082738+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:07 smithi163 bash[27500]: cluster 2023-10-09T08:59:07.082738+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:07 smithi181 bash[27712]: cluster 2023-10-09T08:59:07.082738+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:10.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:10 smithi181 bash[27712]: cluster 2023-10-09T08:59:09.083293+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:10.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:10 smithi181 bash[27712]: audit 2023-10-09T08:59:09.272155+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:10.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:10 smithi181 bash[27712]: audit 2023-10-09T08:59:09.273731+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:10.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:10 smithi181 bash[27712]: audit 2023-10-09T08:59:09.275350+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:59:10.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:10 smithi181 bash[27712]: audit 2023-10-09T08:59:09.282328+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:10 smithi163 bash[33912]: cluster 2023-10-09T08:59:09.083293+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:10 smithi163 bash[33912]: audit 2023-10-09T08:59:09.272155+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:10 smithi163 bash[33912]: audit 2023-10-09T08:59:09.273731+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:10 smithi163 bash[33912]: audit 2023-10-09T08:59:09.275350+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:59:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:10 smithi163 bash[33912]: audit 2023-10-09T08:59:09.282328+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:10 smithi163 bash[27500]: cluster 2023-10-09T08:59:09.083293+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:10 smithi163 bash[27500]: audit 2023-10-09T08:59:09.272155+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:10 smithi163 bash[27500]: audit 2023-10-09T08:59:09.273731+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:10 smithi163 bash[27500]: audit 2023-10-09T08:59:09.275350+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:59:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:10 smithi163 bash[27500]: audit 2023-10-09T08:59:09.282328+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:11 smithi181 bash[27712]: audit 2023-10-09T08:59:10.543738+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:59:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:11 smithi181 bash[27712]: audit 2023-10-09T08:59:10.546223+0000 mon.a (mon.0) 446 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:59:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:11 smithi163 bash[33912]: audit 2023-10-09T08:59:10.543738+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:59:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:11 smithi163 bash[33912]: audit 2023-10-09T08:59:10.546223+0000 mon.a (mon.0) 446 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:59:11.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:11 smithi163 bash[27500]: audit 2023-10-09T08:59:10.543738+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:59:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:11 smithi163 bash[27500]: audit 2023-10-09T08:59:10.546223+0000 mon.a (mon.0) 446 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T08:59:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:12 smithi181 bash[27712]: cluster 2023-10-09T08:59:11.083812+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:12 smithi181 bash[27712]: audit 2023-10-09T08:59:11.284845+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:12 smithi181 bash[27712]: audit 2023-10-09T08:59:11.284913+0000 mon.a (mon.0) 447 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T08:59:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:12 smithi181 bash[27712]: cluster 2023-10-09T08:59:11.284991+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T08:59:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:12 smithi181 bash[27712]: audit 2023-10-09T08:59:11.285186+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:12 smithi181 bash[27712]: audit 2023-10-09T08:59:11.287172+0000 mon.a (mon.0) 450 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:12.591 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 5 on host 'smithi181' 2023-10-09T08:59:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[27500]: cluster 2023-10-09T08:59:11.083812+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[27500]: audit 2023-10-09T08:59:11.284845+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[27500]: audit 2023-10-09T08:59:11.284913+0000 mon.a (mon.0) 447 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T08:59:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[27500]: cluster 2023-10-09T08:59:11.284991+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T08:59:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[27500]: audit 2023-10-09T08:59:11.285186+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[27500]: audit 2023-10-09T08:59:11.287172+0000 mon.a (mon.0) 450 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[33912]: cluster 2023-10-09T08:59:11.083812+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[33912]: audit 2023-10-09T08:59:11.284845+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[33912]: audit 2023-10-09T08:59:11.284913+0000 mon.a (mon.0) 447 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T08:59:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[33912]: cluster 2023-10-09T08:59:11.284991+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T08:59:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[33912]: audit 2023-10-09T08:59:11.285186+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:12 smithi163 bash[33912]: audit 2023-10-09T08:59:11.287172+0000 mon.a (mon.0) 450 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:13.222 DEBUG:teuthology.orchestra.run.smithi181:osd.5> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T08:59:13.225 INFO:tasks.cephadm:Deploying osd.6 on smithi181 with /dev/vg_nvme/lv_2... 2023-10-09T08:59:13.225 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T08:59:13.418 INFO:journalctl@ceph.osd.5.smithi181.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:59:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:13 smithi181 bash[27712]: audit 2023-10-09T08:59:12.289263+0000 mon.a (mon.0) 451 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T08:59:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:13 smithi181 bash[27712]: cluster 2023-10-09T08:59:12.289363+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2023-10-09T08:59:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:13 smithi181 bash[27712]: audit 2023-10-09T08:59:12.289946+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:13 smithi181 bash[27712]: audit 2023-10-09T08:59:12.295950+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:13 smithi181 bash[27712]: audit 2023-10-09T08:59:12.589804+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:13 smithi181 bash[27712]: audit 2023-10-09T08:59:12.609534+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:13 smithi181 bash[27712]: cluster 2023-10-09T08:59:13.288347+0000 mon.a (mon.0) 457 : cluster [INF] osd.5 [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733] boot 2023-10-09T08:59:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:13 smithi181 bash[27712]: cluster 2023-10-09T08:59:13.288410+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T08:59:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:13 smithi181 bash[27712]: audit 2023-10-09T08:59:13.288596+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[27500]: audit 2023-10-09T08:59:12.289263+0000 mon.a (mon.0) 451 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T08:59:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[27500]: cluster 2023-10-09T08:59:12.289363+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2023-10-09T08:59:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[27500]: audit 2023-10-09T08:59:12.289946+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[27500]: audit 2023-10-09T08:59:12.295950+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[27500]: audit 2023-10-09T08:59:12.589804+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[27500]: audit 2023-10-09T08:59:12.609534+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[27500]: cluster 2023-10-09T08:59:13.288347+0000 mon.a (mon.0) 457 : cluster [INF] osd.5 [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733] boot 2023-10-09T08:59:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[27500]: cluster 2023-10-09T08:59:13.288410+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T08:59:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[27500]: audit 2023-10-09T08:59:13.288596+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[33912]: audit 2023-10-09T08:59:12.289263+0000 mon.a (mon.0) 451 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T08:59:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[33912]: cluster 2023-10-09T08:59:12.289363+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2023-10-09T08:59:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[33912]: audit 2023-10-09T08:59:12.289946+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[33912]: audit 2023-10-09T08:59:12.295950+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[33912]: audit 2023-10-09T08:59:12.589804+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[33912]: audit 2023-10-09T08:59:12.609534+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[33912]: cluster 2023-10-09T08:59:13.288347+0000 mon.a (mon.0) 457 : cluster [INF] osd.5 [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733] boot 2023-10-09T08:59:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[33912]: cluster 2023-10-09T08:59:13.288410+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T08:59:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:13 smithi163 bash[33912]: audit 2023-10-09T08:59:13.288596+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T08:59:14.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:14 smithi181 bash[27712]: cluster 2023-10-09T08:59:11.498291+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:59:14.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:14 smithi181 bash[27712]: cluster 2023-10-09T08:59:11.498383+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:59:14.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:14 smithi181 bash[27712]: cluster 2023-10-09T08:59:13.084311+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:14.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:14 smithi181 bash[27712]: cluster 2023-10-09T08:59:14.292371+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T08:59:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:14 smithi163 bash[27500]: cluster 2023-10-09T08:59:11.498291+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:59:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:14 smithi163 bash[27500]: cluster 2023-10-09T08:59:11.498383+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:59:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:14 smithi163 bash[27500]: cluster 2023-10-09T08:59:13.084311+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:14 smithi163 bash[27500]: cluster 2023-10-09T08:59:14.292371+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T08:59:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:14 smithi163 bash[33912]: cluster 2023-10-09T08:59:11.498291+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:59:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:14 smithi163 bash[33912]: cluster 2023-10-09T08:59:11.498383+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:59:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:14 smithi163 bash[33912]: cluster 2023-10-09T08:59:13.084311+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T08:59:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:14 smithi163 bash[33912]: cluster 2023-10-09T08:59:14.292371+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T08:59:16.391 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T08:59:16.391 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T08:59:16.391 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2023-10-09T08:59:16.391 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2023-10-09T08:59:16.391 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0217208 s, 483 MB/s 2023-10-09T08:59:16.392 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2023-10-09T08:59:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:16 smithi181 bash[27712]: cluster 2023-10-09T08:59:15.084865+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:16 smithi181 bash[27712]: cluster 2023-10-09T08:59:15.294731+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2023-10-09T08:59:16.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:16 smithi163 bash[27500]: cluster 2023-10-09T08:59:15.084865+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:16.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:16 smithi163 bash[27500]: cluster 2023-10-09T08:59:15.294731+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2023-10-09T08:59:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:16 smithi163 bash[33912]: cluster 2023-10-09T08:59:15.084865+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:16 smithi163 bash[33912]: cluster 2023-10-09T08:59:15.294731+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2023-10-09T08:59:17.033 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi181:vg_nvme/lv_2 2023-10-09T08:59:18.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:17 smithi163 bash[33912]: cluster 2023-10-09T08:59:17.085414+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:18.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:17 smithi163 bash[27500]: cluster 2023-10-09T08:59:17.085414+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:17 smithi181 bash[27712]: cluster 2023-10-09T08:59:17.085414+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:20.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:20 smithi163 bash[27500]: cluster 2023-10-09T08:59:19.085986+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:20.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:20 smithi163 bash[33912]: cluster 2023-10-09T08:59:19.085986+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:20 smithi181 bash[27712]: cluster 2023-10-09T08:59:19.085986+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:22 smithi181 bash[27712]: cluster 2023-10-09T08:59:21.086516+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:22 smithi181 bash[27712]: audit 2023-10-09T08:59:21.164646+0000 mgr.y (mgr.14146) 201 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:59:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:22 smithi181 bash[27712]: audit 2023-10-09T08:59:21.171364+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:59:22.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:22 smithi181 bash[27712]: audit 2023-10-09T08:59:21.176528+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:59:22.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:22 smithi181 bash[27712]: audit 2023-10-09T08:59:21.177742+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:22 smithi163 bash[33912]: cluster 2023-10-09T08:59:21.086516+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:22 smithi163 bash[33912]: audit 2023-10-09T08:59:21.164646+0000 mgr.y (mgr.14146) 201 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:59:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:22 smithi163 bash[33912]: audit 2023-10-09T08:59:21.171364+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:59:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:22 smithi163 bash[33912]: audit 2023-10-09T08:59:21.176528+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:59:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:22 smithi163 bash[33912]: audit 2023-10-09T08:59:21.177742+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:22 smithi163 bash[27500]: cluster 2023-10-09T08:59:21.086516+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:22 smithi163 bash[27500]: audit 2023-10-09T08:59:21.164646+0000 mgr.y (mgr.14146) 201 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T08:59:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:22 smithi163 bash[27500]: audit 2023-10-09T08:59:21.171364+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T08:59:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:22 smithi163 bash[27500]: audit 2023-10-09T08:59:21.176528+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T08:59:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:22 smithi163 bash[27500]: audit 2023-10-09T08:59:21.177742+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:23.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[27500]: cephadm 2023-10-09T08:59:22.569161+0000 mgr.y (mgr.14146) 202 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T08:59:23.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[27500]: audit 2023-10-09T08:59:22.578737+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[27500]: audit 2023-10-09T08:59:22.580321+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:59:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[27500]: audit 2023-10-09T08:59:22.581601+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:59:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[27500]: cephadm 2023-10-09T08:59:22.582470+0000 mgr.y (mgr.14146) 203 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 8632M 2023-10-09T08:59:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[27500]: audit 2023-10-09T08:59:22.587837+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[27500]: audit 2023-10-09T08:59:22.589351+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[27500]: audit 2023-10-09T08:59:22.590731+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:59:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[27500]: audit 2023-10-09T08:59:22.597248+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[27500]: cluster 2023-10-09T08:59:23.087075+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:23.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[33912]: cephadm 2023-10-09T08:59:22.569161+0000 mgr.y (mgr.14146) 202 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T08:59:23.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[33912]: audit 2023-10-09T08:59:22.578737+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:23.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[33912]: audit 2023-10-09T08:59:22.580321+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:59:23.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[33912]: audit 2023-10-09T08:59:22.581601+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:59:23.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[33912]: cephadm 2023-10-09T08:59:22.582470+0000 mgr.y (mgr.14146) 203 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 8632M 2023-10-09T08:59:23.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[33912]: audit 2023-10-09T08:59:22.587837+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:23.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[33912]: audit 2023-10-09T08:59:22.589351+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:23.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[33912]: audit 2023-10-09T08:59:22.590731+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:59:23.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[33912]: audit 2023-10-09T08:59:22.597248+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:23.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:23 smithi163 bash[33912]: cluster 2023-10-09T08:59:23.087075+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:24.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:23 smithi181 bash[27712]: cephadm 2023-10-09T08:59:22.569161+0000 mgr.y (mgr.14146) 202 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T08:59:24.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:23 smithi181 bash[27712]: audit 2023-10-09T08:59:22.578737+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:24.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:23 smithi181 bash[27712]: audit 2023-10-09T08:59:22.580321+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:59:24.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:23 smithi181 bash[27712]: audit 2023-10-09T08:59:22.581601+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T08:59:24.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:23 smithi181 bash[27712]: cephadm 2023-10-09T08:59:22.582470+0000 mgr.y (mgr.14146) 203 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 8632M 2023-10-09T08:59:24.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:23 smithi181 bash[27712]: audit 2023-10-09T08:59:22.587837+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:24.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:23 smithi181 bash[27712]: audit 2023-10-09T08:59:22.589351+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:24.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:23 smithi181 bash[27712]: audit 2023-10-09T08:59:22.590731+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:59:24.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:23 smithi181 bash[27712]: audit 2023-10-09T08:59:22.597248+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:24.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:23 smithi181 bash[27712]: cluster 2023-10-09T08:59:23.087075+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:26 smithi181 bash[27712]: cluster 2023-10-09T08:59:25.087556+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:26.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:26 smithi163 bash[27500]: cluster 2023-10-09T08:59:25.087556+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:26.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:26 smithi163 bash[33912]: cluster 2023-10-09T08:59:25.087556+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:28.231 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:27 smithi181 bash[27712]: cluster 2023-10-09T08:59:27.088111+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:28.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:27 smithi163 bash[33912]: cluster 2023-10-09T08:59:27.088111+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:28.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:27 smithi163 bash[27500]: cluster 2023-10-09T08:59:27.088111+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:29.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:29 smithi163 bash[27500]: audit 2023-10-09T08:59:29.069535+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.181:0/207374099' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5de7d354-0821-4d89-ad3e-25c5aaed463a"}]: dispatch 2023-10-09T08:59:29.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:29 smithi163 bash[27500]: audit 2023-10-09T08:59:29.072307+0000 mon.a (mon.0) 472 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5de7d354-0821-4d89-ad3e-25c5aaed463a"}]: dispatch 2023-10-09T08:59:29.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:29 smithi163 bash[27500]: audit 2023-10-09T08:59:29.081053+0000 mon.a (mon.0) 473 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5de7d354-0821-4d89-ad3e-25c5aaed463a"}]': finished 2023-10-09T08:59:29.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:29 smithi163 bash[27500]: cluster 2023-10-09T08:59:29.081148+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T08:59:29.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:29 smithi163 bash[27500]: audit 2023-10-09T08:59:29.081435+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:29.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:29 smithi163 bash[33912]: audit 2023-10-09T08:59:29.069535+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.181:0/207374099' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5de7d354-0821-4d89-ad3e-25c5aaed463a"}]: dispatch 2023-10-09T08:59:29.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:29 smithi163 bash[33912]: audit 2023-10-09T08:59:29.072307+0000 mon.a (mon.0) 472 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5de7d354-0821-4d89-ad3e-25c5aaed463a"}]: dispatch 2023-10-09T08:59:29.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:29 smithi163 bash[33912]: audit 2023-10-09T08:59:29.081053+0000 mon.a (mon.0) 473 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5de7d354-0821-4d89-ad3e-25c5aaed463a"}]': finished 2023-10-09T08:59:29.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:29 smithi163 bash[33912]: cluster 2023-10-09T08:59:29.081148+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T08:59:29.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:29 smithi163 bash[33912]: audit 2023-10-09T08:59:29.081435+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:29 smithi181 bash[27712]: audit 2023-10-09T08:59:29.069535+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.181:0/207374099' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5de7d354-0821-4d89-ad3e-25c5aaed463a"}]: dispatch 2023-10-09T08:59:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:29 smithi181 bash[27712]: audit 2023-10-09T08:59:29.072307+0000 mon.a (mon.0) 472 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5de7d354-0821-4d89-ad3e-25c5aaed463a"}]: dispatch 2023-10-09T08:59:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:29 smithi181 bash[27712]: audit 2023-10-09T08:59:29.081053+0000 mon.a (mon.0) 473 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5de7d354-0821-4d89-ad3e-25c5aaed463a"}]': finished 2023-10-09T08:59:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:29 smithi181 bash[27712]: cluster 2023-10-09T08:59:29.081148+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T08:59:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:29 smithi181 bash[27712]: audit 2023-10-09T08:59:29.081435+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:30 smithi163 bash[27500]: cluster 2023-10-09T08:59:29.088693+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:30 smithi163 bash[27500]: audit 2023-10-09T08:59:29.848614+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.181:0/2998675411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:59:30.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:30 smithi163 bash[33912]: cluster 2023-10-09T08:59:29.088693+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:30.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:30 smithi163 bash[33912]: audit 2023-10-09T08:59:29.848614+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.181:0/2998675411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:59:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:30 smithi181 bash[27712]: cluster 2023-10-09T08:59:29.088693+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:30 smithi181 bash[27712]: audit 2023-10-09T08:59:29.848614+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.181:0/2998675411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T08:59:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:32 smithi163 bash[27500]: cluster 2023-10-09T08:59:31.089267+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:32 smithi163 bash[33912]: cluster 2023-10-09T08:59:31.089267+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:32.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:32 smithi181 bash[27712]: cluster 2023-10-09T08:59:31.089267+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:34 smithi163 bash[27500]: cluster 2023-10-09T08:59:33.089862+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:34 smithi163 bash[33912]: cluster 2023-10-09T08:59:33.089862+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:34 smithi181 bash[27712]: cluster 2023-10-09T08:59:33.089862+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:36 smithi163 bash[27500]: cluster 2023-10-09T08:59:35.090481+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:36 smithi163 bash[33912]: cluster 2023-10-09T08:59:35.090481+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:36 smithi181 bash[27712]: cluster 2023-10-09T08:59:35.090481+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:38.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:37 smithi181 bash[27712]: cluster 2023-10-09T08:59:37.091013+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:37 smithi163 bash[27500]: cluster 2023-10-09T08:59:37.091013+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:38.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:37 smithi163 bash[33912]: cluster 2023-10-09T08:59:37.091013+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:40.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:40 smithi181 bash[27712]: cluster 2023-10-09T08:59:39.091522+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:40.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:40 smithi163 bash[27500]: cluster 2023-10-09T08:59:39.091522+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:40.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:40 smithi163 bash[33912]: cluster 2023-10-09T08:59:39.091522+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:41 smithi163 bash[27500]: audit 2023-10-09T08:59:41.055055+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T08:59:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:41 smithi163 bash[27500]: audit 2023-10-09T08:59:41.056795+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:41.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:41 smithi163 bash[33912]: audit 2023-10-09T08:59:41.055055+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T08:59:41.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:41 smithi163 bash[33912]: audit 2023-10-09T08:59:41.056795+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:41 smithi181 bash[27712]: audit 2023-10-09T08:59:41.055055+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T08:59:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:41 smithi181 bash[27712]: audit 2023-10-09T08:59:41.056795+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:42 smithi181 bash[27712]: cephadm 2023-10-09T08:59:41.058348+0000 mgr.y (mgr.14146) 213 : cephadm [INF] Deploying daemon osd.6 on smithi181 2023-10-09T08:59:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:42 smithi181 bash[27712]: cluster 2023-10-09T08:59:41.092054+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:42 smithi163 bash[27500]: cephadm 2023-10-09T08:59:41.058348+0000 mgr.y (mgr.14146) 213 : cephadm [INF] Deploying daemon osd.6 on smithi181 2023-10-09T08:59:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:42 smithi163 bash[27500]: cluster 2023-10-09T08:59:41.092054+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:42 smithi163 bash[33912]: cephadm 2023-10-09T08:59:41.058348+0000 mgr.y (mgr.14146) 213 : cephadm [INF] Deploying daemon osd.6 on smithi181 2023-10-09T08:59:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:42 smithi163 bash[33912]: cluster 2023-10-09T08:59:41.092054+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:44.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:44 smithi181 bash[27712]: cluster 2023-10-09T08:59:43.092595+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:44 smithi163 bash[27500]: cluster 2023-10-09T08:59:43.092595+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:44 smithi163 bash[33912]: cluster 2023-10-09T08:59:43.092595+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:46.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:45 smithi181 bash[27712]: audit 2023-10-09T08:59:44.892041+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:46.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:45 smithi181 bash[27712]: audit 2023-10-09T08:59:44.927807+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:46.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:45 smithi181 bash[27712]: cluster 2023-10-09T08:59:45.093153+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:46.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:45 smithi163 bash[33912]: audit 2023-10-09T08:59:44.892041+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:46.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:45 smithi163 bash[33912]: audit 2023-10-09T08:59:44.927807+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:46.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:45 smithi163 bash[33912]: cluster 2023-10-09T08:59:45.093153+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:46.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:45 smithi163 bash[27500]: audit 2023-10-09T08:59:44.892041+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:46.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:45 smithi163 bash[27500]: audit 2023-10-09T08:59:44.927807+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:46.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:45 smithi163 bash[27500]: cluster 2023-10-09T08:59:45.093153+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:48.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:47 smithi181 bash[27712]: cluster 2023-10-09T08:59:47.093701+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:48.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:47 smithi163 bash[33912]: cluster 2023-10-09T08:59:47.093701+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:47 smithi163 bash[27500]: cluster 2023-10-09T08:59:47.093701+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:50.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:50 smithi163 bash[27500]: cluster 2023-10-09T08:59:49.094265+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:50.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:50 smithi163 bash[27500]: audit 2023-10-09T08:59:50.079786+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:50 smithi163 bash[33912]: cluster 2023-10-09T08:59:49.094265+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:50 smithi163 bash[33912]: audit 2023-10-09T08:59:50.079786+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:50.471 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:50 smithi181 bash[27712]: cluster 2023-10-09T08:59:49.094265+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:50.471 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:50 smithi181 bash[27712]: audit 2023-10-09T08:59:50.079786+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:51.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:51 smithi163 bash[27500]: audit 2023-10-09T08:59:50.330099+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:51.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:51 smithi163 bash[27500]: audit 2023-10-09T08:59:50.332109+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:59:51.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:51 smithi163 bash[27500]: audit 2023-10-09T08:59:50.342195+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:51.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:51 smithi163 bash[33912]: audit 2023-10-09T08:59:50.330099+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:51.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:51 smithi163 bash[33912]: audit 2023-10-09T08:59:50.332109+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:59:51.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:51 smithi163 bash[33912]: audit 2023-10-09T08:59:50.342195+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:51 smithi181 bash[27712]: audit 2023-10-09T08:59:50.330099+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T08:59:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:51 smithi181 bash[27712]: audit 2023-10-09T08:59:50.332109+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T08:59:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:51 smithi181 bash[27712]: audit 2023-10-09T08:59:50.342195+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:52 smithi181 bash[27712]: cluster 2023-10-09T08:59:51.094799+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:52 smithi181 bash[27712]: audit 2023-10-09T08:59:51.396408+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:59:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:52 smithi181 bash[27712]: audit 2023-10-09T08:59:51.399127+0000 mon.a (mon.0) 484 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:59:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:52 smithi163 bash[27500]: cluster 2023-10-09T08:59:51.094799+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:52 smithi163 bash[27500]: audit 2023-10-09T08:59:51.396408+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:59:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:52 smithi163 bash[27500]: audit 2023-10-09T08:59:51.399127+0000 mon.a (mon.0) 484 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:59:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:52 smithi163 bash[33912]: cluster 2023-10-09T08:59:51.094799+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:52 smithi163 bash[33912]: audit 2023-10-09T08:59:51.396408+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:59:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:52 smithi163 bash[33912]: audit 2023-10-09T08:59:51.399127+0000 mon.a (mon.0) 484 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T08:59:53.491 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 6 on host 'smithi181' 2023-10-09T08:59:53.689 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:53 smithi181 bash[27712]: audit 2023-10-09T08:59:52.347765+0000 mon.a (mon.0) 485 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T08:59:53.690 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:53 smithi181 bash[27712]: cluster 2023-10-09T08:59:52.347882+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T08:59:53.690 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:53 smithi181 bash[27712]: audit 2023-10-09T08:59:52.348512+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:53.690 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:53 smithi181 bash[27712]: audit 2023-10-09T08:59:52.349730+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:53.690 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:53 smithi181 bash[27712]: audit 2023-10-09T08:59:52.356463+0000 mon.a (mon.0) 488 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:53 smithi163 bash[27500]: audit 2023-10-09T08:59:52.347765+0000 mon.a (mon.0) 485 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T08:59:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:53 smithi163 bash[27500]: cluster 2023-10-09T08:59:52.347882+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T08:59:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:53 smithi163 bash[27500]: audit 2023-10-09T08:59:52.348512+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:53 smithi163 bash[27500]: audit 2023-10-09T08:59:52.349730+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:53 smithi163 bash[27500]: audit 2023-10-09T08:59:52.356463+0000 mon.a (mon.0) 488 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:53 smithi163 bash[33912]: audit 2023-10-09T08:59:52.347765+0000 mon.a (mon.0) 485 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T08:59:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:53 smithi163 bash[33912]: cluster 2023-10-09T08:59:52.347882+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T08:59:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:53 smithi163 bash[33912]: audit 2023-10-09T08:59:52.348512+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:53 smithi163 bash[33912]: audit 2023-10-09T08:59:52.349730+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:53 smithi163 bash[33912]: audit 2023-10-09T08:59:52.356463+0000 mon.a (mon.0) 488 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T08:59:54.098 DEBUG:teuthology.orchestra.run.smithi181:osd.6> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T08:59:54.100 INFO:tasks.cephadm:Deploying osd.7 on smithi181 with /dev/vg_nvme/lv_1... 2023-10-09T08:59:54.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T08:59:54.327 INFO:journalctl@ceph.osd.6.smithi181.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T08:59:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[27500]: cluster 2023-10-09T08:59:53.095288+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[27500]: audit 2023-10-09T08:59:53.359720+0000 mon.a (mon.0) 489 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T08:59:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[27500]: cluster 2023-10-09T08:59:53.359945+0000 mon.a (mon.0) 490 : cluster [DBG] osdmap e42: 7 total, 6 up, 7 in 2023-10-09T08:59:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[27500]: audit 2023-10-09T08:59:53.360936+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[27500]: audit 2023-10-09T08:59:53.367507+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[27500]: audit 2023-10-09T08:59:53.490280+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[27500]: audit 2023-10-09T08:59:53.519750+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[33912]: cluster 2023-10-09T08:59:53.095288+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[33912]: audit 2023-10-09T08:59:53.359720+0000 mon.a (mon.0) 489 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T08:59:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[33912]: cluster 2023-10-09T08:59:53.359945+0000 mon.a (mon.0) 490 : cluster [DBG] osdmap e42: 7 total, 6 up, 7 in 2023-10-09T08:59:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[33912]: audit 2023-10-09T08:59:53.360936+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[33912]: audit 2023-10-09T08:59:53.367507+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[33912]: audit 2023-10-09T08:59:53.490280+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:54.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:54 smithi163 bash[33912]: audit 2023-10-09T08:59:53.519750+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:54 smithi181 bash[27712]: cluster 2023-10-09T08:59:53.095288+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T08:59:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:54 smithi181 bash[27712]: audit 2023-10-09T08:59:53.359720+0000 mon.a (mon.0) 489 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T08:59:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:54 smithi181 bash[27712]: cluster 2023-10-09T08:59:53.359945+0000 mon.a (mon.0) 490 : cluster [DBG] osdmap e42: 7 total, 6 up, 7 in 2023-10-09T08:59:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:54 smithi181 bash[27712]: audit 2023-10-09T08:59:53.360936+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:54 smithi181 bash[27712]: audit 2023-10-09T08:59:53.367507+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:54 smithi181 bash[27712]: audit 2023-10-09T08:59:53.490280+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T08:59:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:54 smithi181 bash[27712]: audit 2023-10-09T08:59:53.519750+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T08:59:55.531 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:55 smithi181 bash[27712]: cluster 2023-10-09T08:59:52.424452+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:59:55.531 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:55 smithi181 bash[27712]: cluster 2023-10-09T08:59:52.424529+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:59:55.531 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:55 smithi181 bash[27712]: cluster 2023-10-09T08:59:54.362800+0000 mon.a (mon.0) 495 : cluster [INF] osd.6 [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176] boot 2023-10-09T08:59:55.531 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:55 smithi181 bash[27712]: cluster 2023-10-09T08:59:54.362978+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T08:59:55.531 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:55 smithi181 bash[27712]: audit 2023-10-09T08:59:54.363259+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:55.532 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:55 smithi181 bash[27712]: cluster 2023-10-09T08:59:55.363835+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T08:59:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[27500]: cluster 2023-10-09T08:59:52.424452+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:59:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[27500]: cluster 2023-10-09T08:59:52.424529+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:59:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[27500]: cluster 2023-10-09T08:59:54.362800+0000 mon.a (mon.0) 495 : cluster [INF] osd.6 [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176] boot 2023-10-09T08:59:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[27500]: cluster 2023-10-09T08:59:54.362978+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T08:59:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[27500]: audit 2023-10-09T08:59:54.363259+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[27500]: cluster 2023-10-09T08:59:55.363835+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T08:59:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[33912]: cluster 2023-10-09T08:59:52.424452+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T08:59:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[33912]: cluster 2023-10-09T08:59:52.424529+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T08:59:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[33912]: cluster 2023-10-09T08:59:54.362800+0000 mon.a (mon.0) 495 : cluster [INF] osd.6 [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176] boot 2023-10-09T08:59:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[33912]: cluster 2023-10-09T08:59:54.362978+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T08:59:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[33912]: audit 2023-10-09T08:59:54.363259+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T08:59:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:55 smithi163 bash[33912]: cluster 2023-10-09T08:59:55.363835+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T08:59:56.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:56 smithi181 bash[27712]: cluster 2023-10-09T08:59:55.095799+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:59:56.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:56 smithi181 bash[27712]: cluster 2023-10-09T08:59:56.365943+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e45: 7 total, 7 up, 7 in 2023-10-09T08:59:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:56 smithi163 bash[33912]: cluster 2023-10-09T08:59:55.095799+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:59:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:56 smithi163 bash[33912]: cluster 2023-10-09T08:59:56.365943+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e45: 7 total, 7 up, 7 in 2023-10-09T08:59:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:56 smithi163 bash[27500]: cluster 2023-10-09T08:59:55.095799+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:59:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:56 smithi163 bash[27500]: cluster 2023-10-09T08:59:56.365943+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e45: 7 total, 7 up, 7 in 2023-10-09T08:59:57.317 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T08:59:57.317 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T08:59:57.318 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2023-10-09T08:59:57.318 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2023-10-09T08:59:57.318 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0170859 s, 614 MB/s 2023-10-09T08:59:57.318 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2023-10-09T08:59:58.062 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi181:vg_nvme/lv_1 2023-10-09T08:59:58.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 08:59:57 smithi181 bash[27712]: cluster 2023-10-09T08:59:57.096365+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:59:58.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 08:59:57 smithi163 bash[27500]: cluster 2023-10-09T08:59:57.096365+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T08:59:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 08:59:57 smithi163 bash[33912]: cluster 2023-10-09T08:59:57.096365+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:00.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:00 smithi163 bash[27500]: cluster 2023-10-09T08:59:59.096931+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:00.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:00 smithi163 bash[27500]: cluster 2023-10-09T09:00:00.000156+0000 mon.a (mon.0) 500 : cluster [INF] overall HEALTH_OK 2023-10-09T09:00:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:00 smithi163 bash[33912]: cluster 2023-10-09T08:59:59.096931+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:00 smithi163 bash[33912]: cluster 2023-10-09T09:00:00.000156+0000 mon.a (mon.0) 500 : cluster [INF] overall HEALTH_OK 2023-10-09T09:00:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:00 smithi181 bash[27712]: cluster 2023-10-09T08:59:59.096931+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:00 smithi181 bash[27712]: cluster 2023-10-09T09:00:00.000156+0000 mon.a (mon.0) 500 : cluster [INF] overall HEALTH_OK 2023-10-09T09:00:02.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:02 smithi163 bash[27500]: cluster 2023-10-09T09:00:01.097500+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:02.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:02 smithi163 bash[27500]: audit 2023-10-09T09:00:02.144207+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:00:02.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:02 smithi163 bash[27500]: audit 2023-10-09T09:00:02.149786+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:00:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:02 smithi163 bash[27500]: audit 2023-10-09T09:00:02.151142+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:02.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:02 smithi163 bash[33912]: cluster 2023-10-09T09:00:01.097500+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:02.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:02 smithi163 bash[33912]: audit 2023-10-09T09:00:02.144207+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:00:02.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:02 smithi163 bash[33912]: audit 2023-10-09T09:00:02.149786+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:00:02.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:02 smithi163 bash[33912]: audit 2023-10-09T09:00:02.151142+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:02 smithi181 bash[27712]: cluster 2023-10-09T09:00:01.097500+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:02 smithi181 bash[27712]: audit 2023-10-09T09:00:02.144207+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:00:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:02 smithi181 bash[27712]: audit 2023-10-09T09:00:02.149786+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:00:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:02 smithi181 bash[27712]: audit 2023-10-09T09:00:02.151142+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:03 smithi163 bash[27500]: audit 2023-10-09T09:00:02.137298+0000 mgr.y (mgr.14146) 225 : audit [DBG] from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:00:03.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:03 smithi163 bash[33912]: audit 2023-10-09T09:00:02.137298+0000 mgr.y (mgr.14146) 225 : audit [DBG] from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:00:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:03 smithi181 bash[27712]: audit 2023-10-09T09:00:02.137298+0000 mgr.y (mgr.14146) 225 : audit [DBG] from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:00:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[27500]: cluster 2023-10-09T09:00:03.098228+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[27500]: audit 2023-10-09T09:00:03.553908+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[27500]: audit 2023-10-09T09:00:03.555550+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[27500]: audit 2023-10-09T09:00:03.556974+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[27500]: audit 2023-10-09T09:00:03.558329+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[27500]: audit 2023-10-09T09:00:03.565069+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[27500]: audit 2023-10-09T09:00:03.566989+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[27500]: audit 2023-10-09T09:00:03.568480+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[27500]: audit 2023-10-09T09:00:03.575420+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[33912]: cluster 2023-10-09T09:00:03.098228+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[33912]: audit 2023-10-09T09:00:03.553908+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:04.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[33912]: audit 2023-10-09T09:00:03.555550+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:04.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[33912]: audit 2023-10-09T09:00:03.556974+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:04.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[33912]: audit 2023-10-09T09:00:03.558329+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:04.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[33912]: audit 2023-10-09T09:00:03.565069+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:04.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[33912]: audit 2023-10-09T09:00:03.566989+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:04.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[33912]: audit 2023-10-09T09:00:03.568480+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:04.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:04 smithi163 bash[33912]: audit 2023-10-09T09:00:03.575420+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:04 smithi181 bash[27712]: cluster 2023-10-09T09:00:03.098228+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:04 smithi181 bash[27712]: audit 2023-10-09T09:00:03.553908+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:04 smithi181 bash[27712]: audit 2023-10-09T09:00:03.555550+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:04 smithi181 bash[27712]: audit 2023-10-09T09:00:03.556974+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:04 smithi181 bash[27712]: audit 2023-10-09T09:00:03.558329+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:04 smithi181 bash[27712]: audit 2023-10-09T09:00:03.565069+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:04 smithi181 bash[27712]: audit 2023-10-09T09:00:03.566989+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:04 smithi181 bash[27712]: audit 2023-10-09T09:00:03.568480+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:04 smithi181 bash[27712]: audit 2023-10-09T09:00:03.575420+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:05 smithi163 bash[33912]: cephadm 2023-10-09T09:00:03.544927+0000 mgr.y (mgr.14146) 227 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T09:00:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:05 smithi163 bash[33912]: cephadm 2023-10-09T09:00:03.559371+0000 mgr.y (mgr.14146) 228 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 5755M 2023-10-09T09:00:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:05 smithi163 bash[27500]: cephadm 2023-10-09T09:00:03.544927+0000 mgr.y (mgr.14146) 227 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T09:00:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:05 smithi163 bash[27500]: cephadm 2023-10-09T09:00:03.559371+0000 mgr.y (mgr.14146) 228 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 5755M 2023-10-09T09:00:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:05 smithi181 bash[27712]: cephadm 2023-10-09T09:00:03.544927+0000 mgr.y (mgr.14146) 227 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T09:00:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:05 smithi181 bash[27712]: cephadm 2023-10-09T09:00:03.559371+0000 mgr.y (mgr.14146) 228 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 5755M 2023-10-09T09:00:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:06 smithi163 bash[27500]: cluster 2023-10-09T09:00:05.098843+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:06 smithi163 bash[33912]: cluster 2023-10-09T09:00:05.098843+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:06 smithi181 bash[27712]: cluster 2023-10-09T09:00:05.098843+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:07 smithi181 bash[27712]: cluster 2023-10-09T09:00:07.099476+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:07 smithi163 bash[27500]: cluster 2023-10-09T09:00:07.099476+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:07 smithi163 bash[33912]: cluster 2023-10-09T09:00:07.099476+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:10.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:10 smithi163 bash[27500]: cluster 2023-10-09T09:00:09.100137+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:10.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:10 smithi163 bash[33912]: cluster 2023-10-09T09:00:09.100137+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:10 smithi181 bash[27712]: cluster 2023-10-09T09:00:09.100137+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[27500]: audit 2023-10-09T09:00:10.178121+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.181:0/2216897927' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fbb1831a-b34a-4d3a-b19b-507f65324930"}]: dispatch 2023-10-09T09:00:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[27500]: audit 2023-10-09T09:00:10.178839+0000 mon.a (mon.0) 512 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fbb1831a-b34a-4d3a-b19b-507f65324930"}]: dispatch 2023-10-09T09:00:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[27500]: audit 2023-10-09T09:00:10.190616+0000 mon.a (mon.0) 513 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fbb1831a-b34a-4d3a-b19b-507f65324930"}]': finished 2023-10-09T09:00:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[27500]: cluster 2023-10-09T09:00:10.190722+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T09:00:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[27500]: audit 2023-10-09T09:00:10.192704+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[27500]: audit 2023-10-09T09:00:10.916065+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.181:0/1862667358' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:00:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[33912]: audit 2023-10-09T09:00:10.178121+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.181:0/2216897927' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fbb1831a-b34a-4d3a-b19b-507f65324930"}]: dispatch 2023-10-09T09:00:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[33912]: audit 2023-10-09T09:00:10.178839+0000 mon.a (mon.0) 512 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fbb1831a-b34a-4d3a-b19b-507f65324930"}]: dispatch 2023-10-09T09:00:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[33912]: audit 2023-10-09T09:00:10.190616+0000 mon.a (mon.0) 513 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fbb1831a-b34a-4d3a-b19b-507f65324930"}]': finished 2023-10-09T09:00:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[33912]: cluster 2023-10-09T09:00:10.190722+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T09:00:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[33912]: audit 2023-10-09T09:00:10.192704+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:11 smithi163 bash[33912]: audit 2023-10-09T09:00:10.916065+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.181:0/1862667358' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:00:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:11 smithi181 bash[27712]: audit 2023-10-09T09:00:10.178121+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.181:0/2216897927' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fbb1831a-b34a-4d3a-b19b-507f65324930"}]: dispatch 2023-10-09T09:00:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:11 smithi181 bash[27712]: audit 2023-10-09T09:00:10.178839+0000 mon.a (mon.0) 512 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fbb1831a-b34a-4d3a-b19b-507f65324930"}]: dispatch 2023-10-09T09:00:11.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:11 smithi181 bash[27712]: audit 2023-10-09T09:00:10.190616+0000 mon.a (mon.0) 513 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fbb1831a-b34a-4d3a-b19b-507f65324930"}]': finished 2023-10-09T09:00:11.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:11 smithi181 bash[27712]: cluster 2023-10-09T09:00:10.190722+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T09:00:11.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:11 smithi181 bash[27712]: audit 2023-10-09T09:00:10.192704+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:11.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:11 smithi181 bash[27712]: audit 2023-10-09T09:00:10.916065+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.181:0/1862667358' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:00:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:12 smithi163 bash[33912]: cluster 2023-10-09T09:00:11.100733+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:12 smithi163 bash[27500]: cluster 2023-10-09T09:00:11.100733+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:12 smithi181 bash[27712]: cluster 2023-10-09T09:00:11.100733+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:14 smithi163 bash[27500]: cluster 2023-10-09T09:00:13.101310+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:14.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:14 smithi163 bash[33912]: cluster 2023-10-09T09:00:13.101310+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:14 smithi181 bash[27712]: cluster 2023-10-09T09:00:13.101310+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:16.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:16 smithi163 bash[27500]: cluster 2023-10-09T09:00:15.101910+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:16.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:16 smithi163 bash[33912]: cluster 2023-10-09T09:00:15.101910+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:16 smithi181 bash[27712]: cluster 2023-10-09T09:00:15.101910+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:17 smithi181 bash[27712]: cluster 2023-10-09T09:00:17.102453+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:17 smithi163 bash[27500]: cluster 2023-10-09T09:00:17.102453+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:18.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:17 smithi163 bash[33912]: cluster 2023-10-09T09:00:17.102453+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:20.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:20 smithi163 bash[27500]: cluster 2023-10-09T09:00:19.103030+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:20.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:20 smithi163 bash[33912]: cluster 2023-10-09T09:00:19.103030+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:20 smithi181 bash[27712]: cluster 2023-10-09T09:00:19.103030+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:22 smithi181 bash[27712]: cluster 2023-10-09T09:00:21.103588+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:22.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:22 smithi163 bash[27500]: cluster 2023-10-09T09:00:21.103588+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:22 smithi163 bash[33912]: cluster 2023-10-09T09:00:21.103588+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:23.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:23 smithi163 bash[27500]: audit 2023-10-09T09:00:22.567018+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:00:23.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:23 smithi163 bash[27500]: audit 2023-10-09T09:00:22.568908+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:23.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:23 smithi163 bash[33912]: audit 2023-10-09T09:00:22.567018+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:00:23.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:23 smithi163 bash[33912]: audit 2023-10-09T09:00:22.568908+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:23.546 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:23 smithi181 bash[27712]: audit 2023-10-09T09:00:22.567018+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:00:23.547 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:23 smithi181 bash[27712]: audit 2023-10-09T09:00:22.568908+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:24 smithi181 bash[27712]: cephadm 2023-10-09T09:00:22.570420+0000 mgr.y (mgr.14146) 238 : cephadm [INF] Deploying daemon osd.7 on smithi181 2023-10-09T09:00:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:24 smithi181 bash[27712]: cluster 2023-10-09T09:00:23.104116+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:24 smithi163 bash[27500]: cephadm 2023-10-09T09:00:22.570420+0000 mgr.y (mgr.14146) 238 : cephadm [INF] Deploying daemon osd.7 on smithi181 2023-10-09T09:00:24.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:24 smithi163 bash[27500]: cluster 2023-10-09T09:00:23.104116+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:24 smithi163 bash[33912]: cephadm 2023-10-09T09:00:22.570420+0000 mgr.y (mgr.14146) 238 : cephadm [INF] Deploying daemon osd.7 on smithi181 2023-10-09T09:00:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:24 smithi163 bash[33912]: cluster 2023-10-09T09:00:23.104116+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:26.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:26 smithi163 bash[27500]: cluster 2023-10-09T09:00:25.104710+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:26.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:26 smithi163 bash[33912]: cluster 2023-10-09T09:00:25.104710+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:26.516 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:26 smithi181 bash[27712]: cluster 2023-10-09T09:00:25.104710+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:27 smithi181 bash[27712]: audit 2023-10-09T09:00:26.777540+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:27 smithi181 bash[27712]: audit 2023-10-09T09:00:26.809151+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:00:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:27 smithi181 bash[27712]: cluster 2023-10-09T09:00:27.105245+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:28.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:27 smithi163 bash[27500]: audit 2023-10-09T09:00:26.777540+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:28.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:27 smithi163 bash[27500]: audit 2023-10-09T09:00:26.809151+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:00:28.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:27 smithi163 bash[27500]: cluster 2023-10-09T09:00:27.105245+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:28.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:27 smithi163 bash[33912]: audit 2023-10-09T09:00:26.777540+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:28.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:27 smithi163 bash[33912]: audit 2023-10-09T09:00:26.809151+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:00:28.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:27 smithi163 bash[33912]: cluster 2023-10-09T09:00:27.105245+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:30 smithi163 bash[27500]: cluster 2023-10-09T09:00:29.105781+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:30.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:30 smithi163 bash[33912]: cluster 2023-10-09T09:00:29.105781+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:30 smithi181 bash[27712]: cluster 2023-10-09T09:00:29.105781+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:32.170 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:32 smithi163 bash[33912]: cluster 2023-10-09T09:00:31.106345+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:32.171 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:32 smithi163 bash[33912]: audit 2023-10-09T09:00:31.992933+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:32.171 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:32 smithi163 bash[33912]: audit 2023-10-09T09:00:31.994152+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:32.171 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:32 smithi163 bash[33912]: audit 2023-10-09T09:00:31.995439+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:32.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:32 smithi163 bash[27500]: cluster 2023-10-09T09:00:31.106345+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:32.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:32 smithi163 bash[27500]: audit 2023-10-09T09:00:31.992933+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:32.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:32 smithi163 bash[27500]: audit 2023-10-09T09:00:31.994152+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:32.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:32 smithi163 bash[27500]: audit 2023-10-09T09:00:31.995439+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:32.171 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:32 smithi163 bash[27500]: audit 2023-10-09T09:00:32.001917+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:32.456 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:32 smithi181 bash[27712]: cluster 2023-10-09T09:00:31.106345+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:32.456 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:32 smithi181 bash[27712]: audit 2023-10-09T09:00:31.992933+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:32.456 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:32 smithi181 bash[27712]: audit 2023-10-09T09:00:31.994152+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:32.456 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:32 smithi181 bash[27712]: audit 2023-10-09T09:00:31.995439+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:32.456 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:32 smithi181 bash[27712]: audit 2023-10-09T09:00:32.001917+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:34 smithi163 bash[27500]: cluster 2023-10-09T09:00:33.106924+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:34 smithi163 bash[27500]: audit 2023-10-09T09:00:33.281159+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:00:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:34 smithi163 bash[27500]: audit 2023-10-09T09:00:33.284158+0000 mon.a (mon.0) 524 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:00:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:32 smithi163 bash[33912]: audit 2023-10-09T09:00:32.001917+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:34 smithi163 bash[33912]: cluster 2023-10-09T09:00:33.106924+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:34 smithi163 bash[33912]: audit 2023-10-09T09:00:33.281159+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:00:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:34 smithi163 bash[33912]: audit 2023-10-09T09:00:33.284158+0000 mon.a (mon.0) 524 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:00:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:34 smithi181 bash[27712]: cluster 2023-10-09T09:00:33.106924+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:34 smithi181 bash[27712]: audit 2023-10-09T09:00:33.281159+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:00:34.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:34 smithi181 bash[27712]: audit 2023-10-09T09:00:33.284158+0000 mon.a (mon.0) 524 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:00:35.408 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 7 on host 'smithi181' 2023-10-09T09:00:35.446 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:35 smithi181 bash[27712]: audit 2023-10-09T09:00:34.183245+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T09:00:35.446 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:35 smithi181 bash[27712]: audit 2023-10-09T09:00:34.183557+0000 mon.a (mon.0) 525 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T09:00:35.446 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:35 smithi181 bash[27712]: cluster 2023-10-09T09:00:34.183618+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T09:00:35.446 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:35 smithi181 bash[27712]: audit 2023-10-09T09:00:34.183817+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:35.446 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:35 smithi181 bash[27712]: audit 2023-10-09T09:00:34.185873+0000 mon.a (mon.0) 528 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T09:00:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:35 smithi163 bash[27500]: audit 2023-10-09T09:00:34.183245+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T09:00:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:35 smithi163 bash[27500]: audit 2023-10-09T09:00:34.183557+0000 mon.a (mon.0) 525 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T09:00:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:35 smithi163 bash[27500]: cluster 2023-10-09T09:00:34.183618+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T09:00:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:35 smithi163 bash[27500]: audit 2023-10-09T09:00:34.183817+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:35.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:35 smithi163 bash[27500]: audit 2023-10-09T09:00:34.185873+0000 mon.a (mon.0) 528 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T09:00:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:35 smithi163 bash[33912]: audit 2023-10-09T09:00:34.183245+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T09:00:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:35 smithi163 bash[33912]: audit 2023-10-09T09:00:34.183557+0000 mon.a (mon.0) 525 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T09:00:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:35 smithi163 bash[33912]: cluster 2023-10-09T09:00:34.183618+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T09:00:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:35 smithi163 bash[33912]: audit 2023-10-09T09:00:34.183817+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:35 smithi163 bash[33912]: audit 2023-10-09T09:00:34.185873+0000 mon.a (mon.0) 528 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T09:00:36.105 DEBUG:teuthology.orchestra.run.smithi181:osd.7> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T09:00:36.107 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T09:00:36.107 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T09:00:36.312 INFO:journalctl@ceph.osd.7.smithi181.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T09:00:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[27500]: cluster 2023-10-09T09:00:35.107470+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[27500]: audit 2023-10-09T09:00:35.188826+0000 mon.a (mon.0) 529 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T09:00:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[27500]: cluster 2023-10-09T09:00:35.188930+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e48: 8 total, 7 up, 8 in 2023-10-09T09:00:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[27500]: audit 2023-10-09T09:00:35.189533+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[27500]: audit 2023-10-09T09:00:35.196317+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[27500]: audit 2023-10-09T09:00:35.407219+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[27500]: audit 2023-10-09T09:00:35.430358+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:00:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[33912]: cluster 2023-10-09T09:00:35.107470+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[33912]: audit 2023-10-09T09:00:35.188826+0000 mon.a (mon.0) 529 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T09:00:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[33912]: cluster 2023-10-09T09:00:35.188930+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e48: 8 total, 7 up, 8 in 2023-10-09T09:00:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[33912]: audit 2023-10-09T09:00:35.189533+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[33912]: audit 2023-10-09T09:00:35.196317+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[33912]: audit 2023-10-09T09:00:35.407219+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:36 smithi163 bash[33912]: audit 2023-10-09T09:00:35.430358+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:00:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:36 smithi181 bash[27712]: cluster 2023-10-09T09:00:35.107470+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:00:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:36 smithi181 bash[27712]: audit 2023-10-09T09:00:35.188826+0000 mon.a (mon.0) 529 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T09:00:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:36 smithi181 bash[27712]: cluster 2023-10-09T09:00:35.188930+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e48: 8 total, 7 up, 8 in 2023-10-09T09:00:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:36 smithi181 bash[27712]: audit 2023-10-09T09:00:35.189533+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:36 smithi181 bash[27712]: audit 2023-10-09T09:00:35.196317+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:36 smithi181 bash[27712]: audit 2023-10-09T09:00:35.407219+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:36 smithi181 bash[27712]: audit 2023-10-09T09:00:35.430358+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:00:37.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[33912]: cluster 2023-10-09T09:00:34.236447+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:00:37.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[33912]: cluster 2023-10-09T09:00:34.236507+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:00:37.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[33912]: cluster 2023-10-09T09:00:36.194535+0000 mon.a (mon.0) 535 : cluster [INF] osd.7 [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963] boot 2023-10-09T09:00:37.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[33912]: cluster 2023-10-09T09:00:36.194610+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T09:00:37.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[33912]: audit 2023-10-09T09:00:36.194810+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:37.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[33912]: cluster 2023-10-09T09:00:36.985640+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T09:00:37.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[27500]: cluster 2023-10-09T09:00:34.236447+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:00:37.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[27500]: cluster 2023-10-09T09:00:34.236507+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:00:37.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[27500]: cluster 2023-10-09T09:00:36.194535+0000 mon.a (mon.0) 535 : cluster [INF] osd.7 [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963] boot 2023-10-09T09:00:37.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[27500]: cluster 2023-10-09T09:00:36.194610+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T09:00:37.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[27500]: audit 2023-10-09T09:00:36.194810+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:37.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:37 smithi163 bash[27500]: cluster 2023-10-09T09:00:36.985640+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T09:00:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:37 smithi181 bash[27712]: cluster 2023-10-09T09:00:34.236447+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:00:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:37 smithi181 bash[27712]: cluster 2023-10-09T09:00:34.236507+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:00:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:37 smithi181 bash[27712]: cluster 2023-10-09T09:00:36.194535+0000 mon.a (mon.0) 535 : cluster [INF] osd.7 [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963] boot 2023-10-09T09:00:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:37 smithi181 bash[27712]: cluster 2023-10-09T09:00:36.194610+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T09:00:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:37 smithi181 bash[27712]: audit 2023-10-09T09:00:36.194810+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:00:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:37 smithi181 bash[27712]: cluster 2023-10-09T09:00:36.985640+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T09:00:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:38 smithi163 bash[27500]: cluster 2023-10-09T09:00:37.108058+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v236: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:38 smithi163 bash[27500]: cluster 2023-10-09T09:00:37.988484+0000 mon.a (mon.0) 539 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:00:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:38 smithi163 bash[27500]: cluster 2023-10-09T09:00:37.998950+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T09:00:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:38 smithi163 bash[33912]: cluster 2023-10-09T09:00:37.108058+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v236: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:38 smithi163 bash[33912]: cluster 2023-10-09T09:00:37.988484+0000 mon.a (mon.0) 539 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:00:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:38 smithi163 bash[33912]: cluster 2023-10-09T09:00:37.998950+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T09:00:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:38 smithi181 bash[27712]: cluster 2023-10-09T09:00:37.108058+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v236: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:38 smithi181 bash[27712]: cluster 2023-10-09T09:00:37.988484+0000 mon.a (mon.0) 539 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:00:38.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:38 smithi181 bash[27712]: cluster 2023-10-09T09:00:37.998950+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T09:00:39.119 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:00:39.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:39 smithi163 bash[27500]: audit 2023-10-09T09:00:39.119675+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.163:0/1935353933' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T09:00:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:39 smithi163 bash[33912]: audit 2023-10-09T09:00:39.119675+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.163:0/1935353933' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T09:00:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:39 smithi181 bash[27712]: audit 2023-10-09T09:00:39.119675+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.163:0/1935353933' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T09:00:39.837 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":51,"num_osds":8,"num_up_osds":8,"osd_up_since":1696842036,"num_in_osds":8,"osd_in_since":1696842010,"num_remapped_pgs":0} 2023-10-09T09:00:39.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:00:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:40 smithi181 bash[27712]: cluster 2023-10-09T09:00:39.108668+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v238: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:40 smithi163 bash[27500]: cluster 2023-10-09T09:00:39.108668+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v238: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:40 smithi163 bash[33912]: cluster 2023-10-09T09:00:39.108668+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v238: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:42 smithi181 bash[27712]: cluster 2023-10-09T09:00:41.109345+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v239: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:42 smithi163 bash[27500]: cluster 2023-10-09T09:00:41.109345+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v239: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:42 smithi163 bash[33912]: cluster 2023-10-09T09:00:41.109345+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v239: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:43 smithi181 bash[27712]: cluster 2023-10-09T09:00:43.224418+0000 mon.a (mon.0) 541 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:00:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:43 smithi181 bash[27712]: cluster 2023-10-09T09:00:43.224472+0000 mon.a (mon.0) 542 : cluster [INF] Cluster is now healthy 2023-10-09T09:00:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:43 smithi163 bash[27500]: cluster 2023-10-09T09:00:43.224418+0000 mon.a (mon.0) 541 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:00:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:43 smithi163 bash[27500]: cluster 2023-10-09T09:00:43.224472+0000 mon.a (mon.0) 542 : cluster [INF] Cluster is now healthy 2023-10-09T09:00:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:43 smithi163 bash[33912]: cluster 2023-10-09T09:00:43.224418+0000 mon.a (mon.0) 541 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:00:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:43 smithi163 bash[33912]: cluster 2023-10-09T09:00:43.224472+0000 mon.a (mon.0) 542 : cluster [INF] Cluster is now healthy 2023-10-09T09:00:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:44 smithi181 bash[27712]: cluster 2023-10-09T09:00:43.109964+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:44.593 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:44.625 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:44 smithi163 bash[27500]: cluster 2023-10-09T09:00:43.109964+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:44.625 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:44 smithi163 bash[33912]: cluster 2023-10-09T09:00:43.109964+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:45.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: cephadm 2023-10-09T09:00:44.268081+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T09:00:45.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: audit 2023-10-09T09:00:44.278725+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:45.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: audit 2023-10-09T09:00:44.280724+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: audit 2023-10-09T09:00:44.284795+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: audit 2023-10-09T09:00:44.285942+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: audit 2023-10-09T09:00:44.287054+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: cephadm 2023-10-09T09:00:44.287908+0000 mgr.y (mgr.14146) 251 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 4316M 2023-10-09T09:00:45.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: audit 2023-10-09T09:00:44.293002+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:45.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: audit 2023-10-09T09:00:44.294521+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:45.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: audit 2023-10-09T09:00:44.295657+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:45.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[27500]: audit 2023-10-09T09:00:44.301925+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:45.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: cephadm 2023-10-09T09:00:44.268081+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T09:00:45.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: audit 2023-10-09T09:00:44.278725+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:45.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: audit 2023-10-09T09:00:44.280724+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: audit 2023-10-09T09:00:44.284795+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: audit 2023-10-09T09:00:44.285942+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: audit 2023-10-09T09:00:44.287054+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: cephadm 2023-10-09T09:00:44.287908+0000 mgr.y (mgr.14146) 251 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 4316M 2023-10-09T09:00:45.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: audit 2023-10-09T09:00:44.293002+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:45.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: audit 2023-10-09T09:00:44.294521+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:45.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: audit 2023-10-09T09:00:44.295657+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:45.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:45 smithi163 bash[33912]: audit 2023-10-09T09:00:44.301925+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: cephadm 2023-10-09T09:00:44.268081+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Detected new or changed devices on smithi181 2023-10-09T09:00:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: audit 2023-10-09T09:00:44.278725+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:45.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: audit 2023-10-09T09:00:44.280724+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: audit 2023-10-09T09:00:44.284795+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: audit 2023-10-09T09:00:44.285942+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: audit 2023-10-09T09:00:44.287054+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:00:45.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: cephadm 2023-10-09T09:00:44.287908+0000 mgr.y (mgr.14146) 251 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 4316M 2023-10-09T09:00:45.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: audit 2023-10-09T09:00:44.293002+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:45.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: audit 2023-10-09T09:00:44.294521+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:00:45.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: audit 2023-10-09T09:00:44.295657+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:00:45.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:45 smithi181 bash[27712]: audit 2023-10-09T09:00:44.301925+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:00:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:46 smithi181 bash[27712]: cluster 2023-10-09T09:00:45.110522+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:46 smithi163 bash[27500]: cluster 2023-10-09T09:00:45.110522+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:46 smithi163 bash[33912]: cluster 2023-10-09T09:00:45.110522+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:47.409 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:00:47.409 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":51,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:00:37.989008+0000","last_up_change":"2023-10-09T09:00:36.182187+0000","last_in_change":"2023-10-09T09:00:10.180048+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6803","nonce":422484962}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6805","nonce":422484962}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6809","nonce":422484962}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6807","nonce":422484962}]},"public_addr":"172.21.15.163:6803/422484962","cluster_addr":"172.21.15.163:6805/422484962","heartbeat_back_addr":"172.21.15.163:6809/422484962","heartbeat_front_addr":"172.21.15.163:6807/422484962","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6811","nonce":1836428548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6813","nonce":1836428548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6817","nonce":1836428548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6815","nonce":1836428548}]},"public_addr":"172.21.15.163:6811/1836428548","cluster_addr":"172.21.15.163:6813/1836428548","heartbeat_back_addr":"172.21.15.163:6817/1836428548","heartbeat_front_addr":"172.21.15.163:6815/1836428548","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6819","nonce":2248501829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6821","nonce":2248501829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6825","nonce":2248501829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6823","nonce":2248501829}]},"public_addr":"172.21.15.163:6819/2248501829","cluster_addr":"172.21.15.163:6821/2248501829","heartbeat_back_addr":"172.21.15.163:6825/2248501829","heartbeat_front_addr":"172.21.15.163:6823/2248501829","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6827","nonce":3899711337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6829","nonce":3899711337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6833","nonce":3899711337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6831","nonce":3899711337}]},"public_addr":"172.21.15.163:6827/3899711337","cluster_addr":"172.21.15.163:6829/3899711337","heartbeat_back_addr":"172.21.15.163:6833/3899711337","heartbeat_front_addr":"172.21.15.163:6831/3899711337","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:00:48.084 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-10-09T08:57:13.290916+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-10-09T09:00:48.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-10-09T09:00:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:47 smithi181 bash[27712]: cluster 2023-10-09T09:00:47.111252+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:47 smithi181 bash[27712]: audit 2023-10-09T09:00:47.407505+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.163:0/3639227868' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:00:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:47 smithi163 bash[27500]: cluster 2023-10-09T09:00:47.111252+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:47 smithi163 bash[27500]: audit 2023-10-09T09:00:47.407505+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.163:0/3639227868' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:00:48.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:47 smithi163 bash[33912]: cluster 2023-10-09T09:00:47.111252+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:48.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:47 smithi163 bash[33912]: audit 2023-10-09T09:00:47.407505+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.163:0/3639227868' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:00:50.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:50 smithi163 bash[27500]: cluster 2023-10-09T09:00:49.111860+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:50 smithi163 bash[33912]: cluster 2023-10-09T09:00:49.111860+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:50 smithi181 bash[27712]: cluster 2023-10-09T09:00:49.111860+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:52 smithi163 bash[27500]: cluster 2023-10-09T09:00:51.112415+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:52 smithi163 bash[33912]: cluster 2023-10-09T09:00:51.112415+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:52 smithi181 bash[27712]: cluster 2023-10-09T09:00:51.112415+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:52.846 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:00:54.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:54 smithi163 bash[33912]: cluster 2023-10-09T09:00:53.113025+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:54.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:54 smithi163 bash[27500]: cluster 2023-10-09T09:00:53.113025+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:54 smithi181 bash[27712]: cluster 2023-10-09T09:00:53.113025+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:55.438 INFO:teuthology.orchestra.run.smithi163.stdout:pg_num: 1 2023-10-09T09:00:56.071 INFO:tasks.cephadm:Adding prometheus.a on smithi181 2023-10-09T09:00:56.071 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi181=a' 2023-10-09T09:00:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:56 smithi163 bash[27500]: cluster 2023-10-09T09:00:55.113628+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:56 smithi163 bash[27500]: audit 2023-10-09T09:00:55.437813+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.163:0/2569629397' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T09:00:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:56 smithi163 bash[33912]: cluster 2023-10-09T09:00:55.113628+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:56 smithi163 bash[33912]: audit 2023-10-09T09:00:55.437813+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.163:0/2569629397' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T09:00:56.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:56 smithi181 bash[27712]: cluster 2023-10-09T09:00:55.113628+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:56.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:56 smithi181 bash[27712]: audit 2023-10-09T09:00:55.437813+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.163:0/2569629397' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T09:00:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:00:58 smithi181 bash[27712]: cluster 2023-10-09T09:00:57.114235+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:58.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:00:57 smithi163 bash[27500]: cluster 2023-10-09T09:00:57.114235+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:00:57 smithi163 bash[33912]: cluster 2023-10-09T09:00:57.114235+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:00:59.247 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled prometheus update... 2023-10-09T09:00:59.950 DEBUG:teuthology.orchestra.run.smithi181:prometheus.a> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T09:00:59.953 INFO:tasks.cephadm:Adding node-exporter.a on smithi163 2023-10-09T09:00:59.953 INFO:tasks.cephadm:Adding node-exporter.b on smithi181 2023-10-09T09:00:59.954 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi163=a;smithi181=b' 2023-10-09T09:01:00.012 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T09:01:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[27712]: cluster 2023-10-09T09:00:59.114790+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[27712]: audit 2023-10-09T09:00:59.235880+0000 mgr.y (mgr.14146) 260 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:01:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[27712]: cephadm 2023-10-09T09:00:59.239281+0000 mgr.y (mgr.14146) 261 : cephadm [INF] Saving service prometheus spec with placement smithi181=a;count:1 2023-10-09T09:01:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[27712]: audit 2023-10-09T09:00:59.246430+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:01:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[27712]: audit 2023-10-09T09:00:59.262263+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[27712]: audit 2023-10-09T09:00:59.529933+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[27712]: audit 2023-10-09T09:00:59.531160+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:00.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[27712]: audit 2023-10-09T09:00:59.540048+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:01:00.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[27712]: audit 2023-10-09T09:00:59.549785+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T09:01:00.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[33912]: cluster 2023-10-09T09:00:59.114790+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:00.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[33912]: audit 2023-10-09T09:00:59.235880+0000 mgr.y (mgr.14146) 260 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:01:00.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[33912]: cephadm 2023-10-09T09:00:59.239281+0000 mgr.y (mgr.14146) 261 : cephadm [INF] Saving service prometheus spec with placement smithi181=a;count:1 2023-10-09T09:01:00.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[33912]: audit 2023-10-09T09:00:59.246430+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:01:00.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[33912]: audit 2023-10-09T09:00:59.262263+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:00.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[33912]: audit 2023-10-09T09:00:59.529933+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:00.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[33912]: audit 2023-10-09T09:00:59.531160+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:00.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[33912]: audit 2023-10-09T09:00:59.540048+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:01:00.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[33912]: audit 2023-10-09T09:00:59.549785+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T09:01:00.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27500]: cluster 2023-10-09T09:00:59.114790+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:00.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27500]: audit 2023-10-09T09:00:59.235880+0000 mgr.y (mgr.14146) 260 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:01:00.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27500]: cephadm 2023-10-09T09:00:59.239281+0000 mgr.y (mgr.14146) 261 : cephadm [INF] Saving service prometheus spec with placement smithi181=a;count:1 2023-10-09T09:01:00.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27500]: audit 2023-10-09T09:00:59.246430+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:01:00.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27500]: audit 2023-10-09T09:00:59.262263+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:00.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27500]: audit 2023-10-09T09:00:59.529933+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:00.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27500]: audit 2023-10-09T09:00:59.531160+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:00.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27500]: audit 2023-10-09T09:00:59.540048+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' 2023-10-09T09:01:00.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27500]: audit 2023-10-09T09:00:59.549785+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T09:01:00.861 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27764]: ignoring --setuser ceph since I am not root 2023-10-09T09:01:00.861 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27764]: ignoring --setgroup ceph since I am not root 2023-10-09T09:01:00.861 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27764]: debug 2023-10-09T09:01:00.594+0000 7f5ff1f99700 1 -- 172.21.15.163:0/2436724867 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5600d87c24e0 con 0x5600d88b6800 2023-10-09T09:01:00.862 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27764]: debug 2023-10-09T09:01:00.750+0000 7f6002161680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:01:00.862 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:00 smithi163 bash[27764]: debug 2023-10-09T09:01:00.858+0000 7f6002161680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:01:00.863 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[28565]: ignoring --setuser ceph since I am not root 2023-10-09T09:01:00.863 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[28565]: ignoring --setgroup ceph since I am not root 2023-10-09T09:01:00.863 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[28565]: debug 2023-10-09T09:01:00.593+0000 7f90cc78a700 1 -- 172.21.15.181:0/917885364 <== mon.2 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562cffadc4e0 con 0x562cffbd0800 2023-10-09T09:01:00.863 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[28565]: debug 2023-10-09T09:01:00.753+0000 7f90dc952680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:01:00.863 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:00 smithi181 bash[28565]: debug 2023-10-09T09:01:00.857+0000 7f90dc952680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:01:01.543 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:01 smithi181 bash[28565]: debug 2023-10-09T09:01:01.345+0000 7f90dc952680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:01:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:01 smithi163 bash[27500]: audit 2023-10-09T09:01:00.550917+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T09:01:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:01 smithi163 bash[27500]: cluster 2023-10-09T09:01:00.551054+0000 mon.a (mon.0) 559 : cluster [DBG] mgrmap e15: y(active, since 6m), standbys: x 2023-10-09T09:01:01.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:01 smithi163 bash[27764]: debug 2023-10-09T09:01:01.318+0000 7f6002161680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:01:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:01 smithi163 bash[33912]: audit 2023-10-09T09:01:00.550917+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T09:01:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:01 smithi163 bash[33912]: cluster 2023-10-09T09:01:00.551054+0000 mon.a (mon.0) 559 : cluster [DBG] mgrmap e15: y(active, since 6m), standbys: x 2023-10-09T09:01:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:01 smithi181 bash[27712]: audit 2023-10-09T09:01:00.550917+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14146 172.21.15.163:0/1053789072' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T09:01:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:01 smithi181 bash[27712]: cluster 2023-10-09T09:01:00.551054+0000 mon.a (mon.0) 559 : cluster [DBG] mgrmap e15: y(active, since 6m), standbys: x 2023-10-09T09:01:02.595 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:02 smithi163 bash[27764]: debug 2023-10-09T09:01:02.246+0000 7f6002161680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:01:02.595 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:02 smithi163 bash[27764]: debug 2023-10-09T09:01:02.342+0000 7f6002161680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:01:02.680 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:02 smithi181 bash[28565]: debug 2023-10-09T09:01:02.329+0000 7f90dc952680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:01:02.680 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:02 smithi181 bash[28565]: debug 2023-10-09T09:01:02.421+0000 7f90dc952680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:01:03.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:02 smithi181 bash[28565]: debug 2023-10-09T09:01:02.677+0000 7f90dc952680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:01:04.004 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:02 smithi163 bash[27764]: debug 2023-10-09T09:01:02.594+0000 7f6002161680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:01:04.004 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:03 smithi163 bash[27764]: debug 2023-10-09T09:01:03.734+0000 7f6002161680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:01:04.004 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:03 smithi163 bash[27764]: debug 2023-10-09T09:01:03.902+0000 7f6002161680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:01:04.107 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:03 smithi181 bash[28565]: debug 2023-10-09T09:01:03.845+0000 7f90dc952680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:01:04.108 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:04 smithi181 bash[28565]: debug 2023-10-09T09:01:04.005+0000 7f90dc952680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:01:04.343 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:04 smithi163 bash[27764]: debug 2023-10-09T09:01:04.002+0000 7f6002161680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:01:04.343 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:04 smithi163 bash[27764]: debug 2023-10-09T09:01:04.118+0000 7f6002161680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:01:04.446 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:04 smithi181 bash[28565]: debug 2023-10-09T09:01:04.101+0000 7f90dc952680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:01:04.446 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:04 smithi181 bash[28565]: debug 2023-10-09T09:01:04.217+0000 7f90dc952680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:01:04.715 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:04 smithi163 bash[27764]: debug 2023-10-09T09:01:04.342+0000 7f6002161680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:01:04.813 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:04 smithi181 bash[28565]: debug 2023-10-09T09:01:04.441+0000 7f90dc952680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:01:04.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:04 smithi163 bash[27764]: debug 2023-10-09T09:01:04.714+0000 7f6002161680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:01:04.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:04 smithi163 bash[27764]: debug 2023-10-09T09:01:04.838+0000 7f6002161680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:01:05.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:04 smithi181 bash[28565]: debug 2023-10-09T09:01:04.809+0000 7f90dc952680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:01:05.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:04 smithi181 bash[28565]: debug 2023-10-09T09:01:04.933+0000 7f90dc952680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:01:05.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:05 smithi163 bash[27764]: debug 2023-10-09T09:01:05.714+0000 7f6002161680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:01:05.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:05 smithi163 bash[27764]: debug 2023-10-09T09:01:05.818+0000 7f6002161680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:01:05.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:05 smithi163 bash[27764]: debug 2023-10-09T09:01:05.934+0000 7f6002161680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:01:06.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:05 smithi181 bash[28565]: debug 2023-10-09T09:01:05.801+0000 7f90dc952680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:01:06.068 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:05 smithi181 bash[28565]: debug 2023-10-09T09:01:05.905+0000 7f90dc952680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:01:06.068 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:06 smithi181 bash[28565]: debug 2023-10-09T09:01:06.021+0000 7f90dc952680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:01:06.456 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:06 smithi163 bash[27764]: debug 2023-10-09T09:01:06.158+0000 7f6002161680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:01:06.536 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:06 smithi181 bash[28565]: debug 2023-10-09T09:01:06.241+0000 7f90dc952680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:01:06.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:06 smithi163 bash[27764]: debug 2023-10-09T09:01:06.454+0000 7f6002161680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:01:06.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:06 smithi163 bash[27764]: debug 2023-10-09T09:01:06.618+0000 7f6002161680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:01:06.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:06 smithi181 bash[28565]: debug 2023-10-09T09:01:06.533+0000 7f90dc952680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:01:06.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:06 smithi181 bash[28565]: debug 2023-10-09T09:01:06.693+0000 7f90dc952680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:01:07.122 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:06 smithi163 bash[27764]: debug 2023-10-09T09:01:06.794+0000 7f6002161680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:01:07.123 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27764]: debug 2023-10-09T09:01:07.014+0000 7f6002161680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:01:07.123 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27764]: debug 2023-10-09T09:01:07.118+0000 7f6002161680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:01:07.194 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:06 smithi181 bash[28565]: debug 2023-10-09T09:01:06.865+0000 7f90dc952680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:01:07.194 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[28565]: debug 2023-10-09T09:01:07.081+0000 7f90dc952680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:01:07.375 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27764]: [09/Oct/2023:09:01:07] ENGINE Bus STARTING 2023-10-09T09:01:07.472 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[28565]: debug 2023-10-09T09:01:07.189+0000 7f90dc952680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:01:07.472 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[28565]: [09/Oct/2023:09:01:07] ENGINE Bus STARTING 2023-10-09T09:01:07.473 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[28565]: CherryPy Checker: 2023-10-09T09:01:07.473 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[28565]: The Application mounted at '' has an empty config. 2023-10-09T09:01:07.473 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[28565]: [09/Oct/2023:09:01:07] ENGINE Serving on http://:::9283 2023-10-09T09:01:07.473 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[28565]: [09/Oct/2023:09:01:07] ENGINE Bus STARTED 2023-10-09T09:01:07.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27764]: Warning: Permanently added '172.21.15.163' (ECDSA) to the list of known hosts. 2023-10-09T09:01:07.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27764]: Warning: Permanently added '172.21.15.181' (ECDSA) to the list of known hosts. 2023-10-09T09:01:07.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27764]: CherryPy Checker: 2023-10-09T09:01:07.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27764]: The Application mounted at '' has an empty config. 2023-10-09T09:01:07.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27764]: [09/Oct/2023:09:01:07] ENGINE Serving on http://:::9283 2023-10-09T09:01:07.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27764]: [09/Oct/2023:09:01:07] ENGINE Bus STARTED 2023-10-09T09:01:08.178 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled node-exporter update... 2023-10-09T09:01:08.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: cluster 2023-10-09T09:01:07.146496+0000 mon.a (mon.0) 560 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:01:08.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: cluster 2023-10-09T09:01:07.147170+0000 mon.a (mon.0) 561 : cluster [INF] Activating manager daemon y 2023-10-09T09:01:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: cluster 2023-10-09T09:01:07.155852+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T09:01:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: cluster 2023-10-09T09:01:07.156245+0000 mon.a (mon.0) 563 : cluster [DBG] mgrmap e16: y(active, starting, since 0.00925903s), standbys: x 2023-10-09T09:01:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.161319+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:01:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.161604+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:01:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.161876+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:01:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.162896+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:01:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.163223+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:01:08.267 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.163466+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:01:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.163896+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:01:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.164376+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:01:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.164881+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:01:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.165259+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:01:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.165625+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:01:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.165895+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:01:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.166273+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:01:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.166568+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:01:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.166844+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:01:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.167436+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:01:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: cluster 2023-10-09T09:01:07.171534+0000 mon.a (mon.0) 580 : cluster [INF] Manager daemon y is now available 2023-10-09T09:01:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: cluster 2023-10-09T09:01:07.215294+0000 mon.a (mon.0) 581 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:01:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: cluster 2023-10-09T09:01:07.215574+0000 mon.a (mon.0) 582 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:01:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.217280+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:01:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.218552+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:01:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.221723+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:01:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.224018+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:01:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.300810+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.303722+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.349049+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.375836+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:01:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:07 smithi181 bash[27712]: audit 2023-10-09T09:01:07.383673+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:01:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: cluster 2023-10-09T09:01:07.146496+0000 mon.a (mon.0) 560 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:01:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: cluster 2023-10-09T09:01:07.147170+0000 mon.a (mon.0) 561 : cluster [INF] Activating manager daemon y 2023-10-09T09:01:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: cluster 2023-10-09T09:01:07.155852+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T09:01:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: cluster 2023-10-09T09:01:07.156245+0000 mon.a (mon.0) 563 : cluster [DBG] mgrmap e16: y(active, starting, since 0.00925903s), standbys: x 2023-10-09T09:01:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.161319+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:01:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.161604+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:01:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.161876+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:01:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.162896+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:01:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.163223+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:01:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.163466+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:01:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.163896+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:01:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.164376+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:01:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.164881+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:01:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.165259+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:01:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.165625+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:01:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.165895+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:01:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.166273+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:01:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.166568+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:01:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.166844+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:01:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: cluster 2023-10-09T09:01:07.146496+0000 mon.a (mon.0) 560 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:01:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: cluster 2023-10-09T09:01:07.147170+0000 mon.a (mon.0) 561 : cluster [INF] Activating manager daemon y 2023-10-09T09:01:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: cluster 2023-10-09T09:01:07.155852+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T09:01:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: cluster 2023-10-09T09:01:07.156245+0000 mon.a (mon.0) 563 : cluster [DBG] mgrmap e16: y(active, starting, since 0.00925903s), standbys: x 2023-10-09T09:01:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: audit 2023-10-09T09:01:07.161319+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:01:08.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: audit 2023-10-09T09:01:07.161604+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:01:08.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: audit 2023-10-09T09:01:07.161876+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:01:08.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: audit 2023-10-09T09:01:07.162896+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:01:08.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: audit 2023-10-09T09:01:07.163223+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:01:08.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: audit 2023-10-09T09:01:07.163466+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:01:08.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: audit 2023-10-09T09:01:07.163896+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:01:08.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[33912]: audit 2023-10-09T09:01:07.164376+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:01:08.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.164881+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:01:08.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.165259+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:01:08.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.165625+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.165895+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.166273+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.166568+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.166844+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.167436+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: cluster 2023-10-09T09:01:07.171534+0000 mon.a (mon.0) 580 : cluster [INF] Manager daemon y is now available 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: cluster 2023-10-09T09:01:07.215294+0000 mon.a (mon.0) 581 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: cluster 2023-10-09T09:01:07.215574+0000 mon.a (mon.0) 582 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.217280+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.218552+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:01:08.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.221723+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:01:08.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.224018+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:01:08.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.300810+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:08.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.303722+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:08.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.349049+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:08.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.375836+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:01:08.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[33912]: audit 2023-10-09T09:01:07.383673+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:01:08.474 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.167436+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:01:08.474 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: cluster 2023-10-09T09:01:07.171534+0000 mon.a (mon.0) 580 : cluster [INF] Manager daemon y is now available 2023-10-09T09:01:08.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: cluster 2023-10-09T09:01:07.215294+0000 mon.a (mon.0) 581 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:01:08.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: cluster 2023-10-09T09:01:07.215574+0000 mon.a (mon.0) 582 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:01:08.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:07 smithi163 bash[27500]: audit 2023-10-09T09:01:07.217280+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:01:08.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[27500]: audit 2023-10-09T09:01:07.218552+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:01:08.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[27500]: audit 2023-10-09T09:01:07.221723+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:01:08.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[27500]: audit 2023-10-09T09:01:07.224018+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.181:0/3538698301' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:01:08.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[27500]: audit 2023-10-09T09:01:07.300810+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:08.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[27500]: audit 2023-10-09T09:01:07.303722+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:08.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[27500]: audit 2023-10-09T09:01:07.349049+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:08.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[27500]: audit 2023-10-09T09:01:07.375836+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:01:08.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:08 smithi163 bash[27500]: audit 2023-10-09T09:01:07.383673+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:01:08.969 DEBUG:teuthology.orchestra.run.smithi163:node-exporter.a> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T09:01:08.972 DEBUG:teuthology.orchestra.run.smithi181:node-exporter.b> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T09:01:09.013 INFO:tasks.cephadm:Adding alertmanager.a on smithi163 2023-10-09T09:01:09.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi163=a' 2023-10-09T09:01:09.083 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T09:01:09.160 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T09:01:09.248 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:09 smithi181 bash[27712]: cluster 2023-10-09T09:01:08.163968+0000 mon.a (mon.0) 588 : cluster [DBG] mgrmap e17: y(active, since 1.01697s), standbys: x 2023-10-09T09:01:09.249 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:09 smithi181 bash[27712]: cephadm 2023-10-09T09:01:08.173211+0000 mgr.y (mgr.24355) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi163=a;smithi181=b;count:2 2023-10-09T09:01:09.249 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:09 smithi181 bash[27712]: audit 2023-10-09T09:01:08.177410+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:09 smithi163 bash[33912]: cluster 2023-10-09T09:01:08.163968+0000 mon.a (mon.0) 588 : cluster [DBG] mgrmap e17: y(active, since 1.01697s), standbys: x 2023-10-09T09:01:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:09 smithi163 bash[33912]: cephadm 2023-10-09T09:01:08.173211+0000 mgr.y (mgr.24355) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi163=a;smithi181=b;count:2 2023-10-09T09:01:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:09 smithi163 bash[33912]: audit 2023-10-09T09:01:08.177410+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:09.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:09 smithi163 bash[27500]: cluster 2023-10-09T09:01:08.163968+0000 mon.a (mon.0) 588 : cluster [DBG] mgrmap e17: y(active, since 1.01697s), standbys: x 2023-10-09T09:01:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:09 smithi163 bash[27500]: cephadm 2023-10-09T09:01:08.173211+0000 mgr.y (mgr.24355) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi163=a;smithi181=b;count:2 2023-10-09T09:01:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:09 smithi163 bash[27500]: audit 2023-10-09T09:01:08.177410+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:10 smithi181 bash[27712]: cluster 2023-10-09T09:01:09.165003+0000 mgr.y (mgr.24355) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:10.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:10 smithi163 bash[33912]: cluster 2023-10-09T09:01:09.165003+0000 mgr.y (mgr.24355) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:10.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:10 smithi163 bash[27500]: cluster 2023-10-09T09:01:09.165003+0000 mgr.y (mgr.24355) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:11 smithi163 bash[27500]: cluster 2023-10-09T09:01:10.183211+0000 mon.a (mon.0) 590 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-09T09:01:11.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:11 smithi163 bash[33912]: cluster 2023-10-09T09:01:10.183211+0000 mon.a (mon.0) 590 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-09T09:01:11.508 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:11 smithi181 bash[27712]: cluster 2023-10-09T09:01:10.183211+0000 mon.a (mon.0) 590 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-09T09:01:11.907 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled alertmanager update... 2023-10-09T09:01:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:12 smithi163 bash[33912]: cluster 2023-10-09T09:01:11.165621+0000 mgr.y (mgr.24355) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:12 smithi163 bash[33912]: audit 2023-10-09T09:01:11.907183+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:12 smithi163 bash[27500]: cluster 2023-10-09T09:01:11.165621+0000 mgr.y (mgr.24355) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:12 smithi163 bash[27500]: audit 2023-10-09T09:01:11.907183+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:12.552 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:12 smithi181 bash[27712]: cluster 2023-10-09T09:01:11.165621+0000 mgr.y (mgr.24355) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:12.552 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:12 smithi181 bash[27712]: audit 2023-10-09T09:01:11.907183+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:12.554 DEBUG:teuthology.orchestra.run.smithi163:alertmanager.a> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T09:01:12.558 INFO:tasks.cephadm:Adding grafana.a on smithi181 2023-10-09T09:01:12.558 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi181=a' 2023-10-09T09:01:12.729 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T09:01:13.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:13 smithi163 bash[27500]: audit 2023-10-09T09:01:11.897773+0000 mgr.y (mgr.24355) 6 : audit [DBG] from='client.24382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi163=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:01:13.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:13 smithi163 bash[27500]: cephadm 2023-10-09T09:01:11.900927+0000 mgr.y (mgr.24355) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi163=a;count:1 2023-10-09T09:01:13.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:13 smithi163 bash[27500]: audit 2023-10-09T09:01:12.936757+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:13 smithi163 bash[33912]: audit 2023-10-09T09:01:11.897773+0000 mgr.y (mgr.24355) 6 : audit [DBG] from='client.24382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi163=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:01:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:13 smithi163 bash[33912]: cephadm 2023-10-09T09:01:11.900927+0000 mgr.y (mgr.24355) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi163=a;count:1 2023-10-09T09:01:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:13 smithi163 bash[33912]: audit 2023-10-09T09:01:12.936757+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:13 smithi181 bash[27712]: audit 2023-10-09T09:01:11.897773+0000 mgr.y (mgr.24355) 6 : audit [DBG] from='client.24382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi163=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:01:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:13 smithi181 bash[27712]: cephadm 2023-10-09T09:01:11.900927+0000 mgr.y (mgr.24355) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi163=a;count:1 2023-10-09T09:01:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:13 smithi181 bash[27712]: audit 2023-10-09T09:01:12.936757+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:14 smithi181 bash[27712]: cluster 2023-10-09T09:01:13.166254+0000 mgr.y (mgr.24355) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:14 smithi181 bash[27712]: audit 2023-10-09T09:01:13.501910+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:14 smithi181 bash[27712]: audit 2023-10-09T09:01:13.789629+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:14 smithi181 bash[27712]: audit 2023-10-09T09:01:13.791475+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:14 smithi181 bash[27712]: audit 2023-10-09T09:01:13.848373+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:14 smithi181 bash[27712]: audit 2023-10-09T09:01:13.895639+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:14 smithi181 bash[27712]: audit 2023-10-09T09:01:13.903453+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[33912]: cluster 2023-10-09T09:01:13.166254+0000 mgr.y (mgr.24355) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:14.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[33912]: audit 2023-10-09T09:01:13.501910+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[33912]: audit 2023-10-09T09:01:13.789629+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[33912]: audit 2023-10-09T09:01:13.791475+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[33912]: audit 2023-10-09T09:01:13.848373+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[33912]: audit 2023-10-09T09:01:13.895639+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[33912]: audit 2023-10-09T09:01:13.903453+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[27500]: cluster 2023-10-09T09:01:13.166254+0000 mgr.y (mgr.24355) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[27500]: audit 2023-10-09T09:01:13.501910+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[27500]: audit 2023-10-09T09:01:13.789629+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[27500]: audit 2023-10-09T09:01:13.791475+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[27500]: audit 2023-10-09T09:01:13.848373+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[27500]: audit 2023-10-09T09:01:13.895639+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:14.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:14 smithi163 bash[27500]: audit 2023-10-09T09:01:13.903453+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:15.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:15 smithi181 bash[27712]: cephadm 2023-10-09T09:01:13.794060+0000 mgr.y (mgr.24355) 9 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:01:15.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:15 smithi181 bash[27712]: cephadm 2023-10-09T09:01:13.794565+0000 mgr.y (mgr.24355) 10 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:01:15.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:15 smithi181 bash[27712]: cephadm 2023-10-09T09:01:13.814355+0000 mgr.y (mgr.24355) 11 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:01:15.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:15 smithi181 bash[27712]: cephadm 2023-10-09T09:01:13.831009+0000 mgr.y (mgr.24355) 12 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:01:15.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:15 smithi181 bash[27712]: cephadm 2023-10-09T09:01:13.927374+0000 mgr.y (mgr.24355) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi181 2023-10-09T09:01:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:15 smithi163 bash[27500]: cephadm 2023-10-09T09:01:13.794060+0000 mgr.y (mgr.24355) 9 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:01:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:15 smithi163 bash[27500]: cephadm 2023-10-09T09:01:13.794565+0000 mgr.y (mgr.24355) 10 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:01:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:15 smithi163 bash[27500]: cephadm 2023-10-09T09:01:13.814355+0000 mgr.y (mgr.24355) 11 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:01:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:15 smithi163 bash[27500]: cephadm 2023-10-09T09:01:13.831009+0000 mgr.y (mgr.24355) 12 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:01:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:15 smithi163 bash[27500]: cephadm 2023-10-09T09:01:13.927374+0000 mgr.y (mgr.24355) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi181 2023-10-09T09:01:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:15 smithi163 bash[33912]: cephadm 2023-10-09T09:01:13.794060+0000 mgr.y (mgr.24355) 9 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:01:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:15 smithi163 bash[33912]: cephadm 2023-10-09T09:01:13.794565+0000 mgr.y (mgr.24355) 10 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:01:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:15 smithi163 bash[33912]: cephadm 2023-10-09T09:01:13.814355+0000 mgr.y (mgr.24355) 11 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:01:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:15 smithi163 bash[33912]: cephadm 2023-10-09T09:01:13.831009+0000 mgr.y (mgr.24355) 12 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:01:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:15 smithi163 bash[33912]: cephadm 2023-10-09T09:01:13.927374+0000 mgr.y (mgr.24355) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi181 2023-10-09T09:01:16.025 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled grafana update... 2023-10-09T09:01:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:16 smithi181 bash[27712]: cluster 2023-10-09T09:01:15.166965+0000 mgr.y (mgr.24355) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:16 smithi181 bash[27712]: audit 2023-10-09T09:01:16.024820+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:16 smithi163 bash[33912]: cluster 2023-10-09T09:01:15.166965+0000 mgr.y (mgr.24355) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:16 smithi163 bash[33912]: audit 2023-10-09T09:01:16.024820+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:16 smithi163 bash[27500]: cluster 2023-10-09T09:01:15.166965+0000 mgr.y (mgr.24355) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:16 smithi163 bash[27500]: audit 2023-10-09T09:01:16.024820+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:17.620 DEBUG:teuthology.orchestra.run.smithi181:grafana.a> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T09:01:17.622 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T09:01:17.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T09:01:17.752 INFO:journalctl@ceph.grafana.a.smithi181.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T09:01:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:17 smithi181 bash[27712]: audit 2023-10-09T09:01:16.015173+0000 mgr.y (mgr.24355) 15 : audit [DBG] from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:01:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:17 smithi181 bash[27712]: cephadm 2023-10-09T09:01:16.018596+0000 mgr.y (mgr.24355) 16 : cephadm [INF] Saving service grafana spec with placement smithi181=a;count:1 2023-10-09T09:01:17.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:17 smithi163 bash[33912]: audit 2023-10-09T09:01:16.015173+0000 mgr.y (mgr.24355) 15 : audit [DBG] from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:01:17.819 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:17 smithi163 bash[33912]: cephadm 2023-10-09T09:01:16.018596+0000 mgr.y (mgr.24355) 16 : cephadm [INF] Saving service grafana spec with placement smithi181=a;count:1 2023-10-09T09:01:17.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:17 smithi163 bash[27500]: audit 2023-10-09T09:01:16.015173+0000 mgr.y (mgr.24355) 15 : audit [DBG] from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:01:17.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:17 smithi163 bash[27500]: cephadm 2023-10-09T09:01:16.018596+0000 mgr.y (mgr.24355) 16 : cephadm [INF] Saving service grafana spec with placement smithi181=a;count:1 2023-10-09T09:01:18.809 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:18 smithi163 bash[33912]: cluster 2023-10-09T09:01:17.167463+0000 mgr.y (mgr.24355) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:18.809 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:18 smithi163 bash[27500]: cluster 2023-10-09T09:01:17.167463+0000 mgr.y (mgr.24355) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:18 smithi181 bash[27712]: cluster 2023-10-09T09:01:17.167463+0000 mgr.y (mgr.24355) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:20.668 INFO:teuthology.orchestra.run.smithi163.stdout:[client.0] 2023-10-09T09:01:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: key = AQBgwSNlzIlfJxAACXD/fRVhVTDkDEvU2/IbbQ== 2023-10-09T09:01:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:20 smithi181 bash[27712]: cluster 2023-10-09T09:01:19.168149+0000 mgr.y (mgr.24355) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:20.884 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:20 smithi163 bash[27500]: cluster 2023-10-09T09:01:19.168149+0000 mgr.y (mgr.24355) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:20.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:20 smithi163 bash[33912]: cluster 2023-10-09T09:01:19.168149+0000 mgr.y (mgr.24355) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:21.366 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T09:01:21.366 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T09:01:21.366 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T09:01:21.390 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T09:01:21.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:21 smithi163 bash[27500]: audit 2023-10-09T09:01:20.660143+0000 mon.a (mon.0) 600 : audit [INF] from='client.? 172.21.15.163:0/2225973429' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:01:21.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:21 smithi163 bash[27500]: audit 2023-10-09T09:01:20.668158+0000 mon.a (mon.0) 601 : audit [INF] from='client.? 172.21.15.163:0/2225973429' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:01:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:21 smithi163 bash[33912]: audit 2023-10-09T09:01:20.660143+0000 mon.a (mon.0) 600 : audit [INF] from='client.? 172.21.15.163:0/2225973429' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:01:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:21 smithi163 bash[33912]: audit 2023-10-09T09:01:20.668158+0000 mon.a (mon.0) 601 : audit [INF] from='client.? 172.21.15.163:0/2225973429' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:01:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:21 smithi181 bash[27712]: audit 2023-10-09T09:01:20.660143+0000 mon.a (mon.0) 600 : audit [INF] from='client.? 172.21.15.163:0/2225973429' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:01:21.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:21 smithi181 bash[27712]: audit 2023-10-09T09:01:20.668158+0000 mon.a (mon.0) 601 : audit [INF] from='client.? 172.21.15.163:0/2225973429' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:01:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:22 smithi181 bash[27712]: cluster 2023-10-09T09:01:21.168557+0000 mgr.y (mgr.24355) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:22.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:22 smithi163 bash[33912]: cluster 2023-10-09T09:01:21.168557+0000 mgr.y (mgr.24355) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:22.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:22 smithi163 bash[27500]: cluster 2023-10-09T09:01:21.168557+0000 mgr.y (mgr.24355) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:24.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:24 smithi181 bash[27712]: cluster 2023-10-09T09:01:23.169186+0000 mgr.y (mgr.24355) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:24 smithi163 bash[27500]: cluster 2023-10-09T09:01:23.169186+0000 mgr.y (mgr.24355) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:24.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:24 smithi163 bash[33912]: cluster 2023-10-09T09:01:23.169186+0000 mgr.y (mgr.24355) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:26.796 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:26 smithi181 bash[27712]: cluster 2023-10-09T09:01:25.169773+0000 mgr.y (mgr.24355) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:26 smithi163 bash[33912]: cluster 2023-10-09T09:01:25.169773+0000 mgr.y (mgr.24355) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:26 smithi163 bash[27500]: cluster 2023-10-09T09:01:25.169773+0000 mgr.y (mgr.24355) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:27.311 INFO:teuthology.orchestra.run.smithi181.stdout:[client.1] 2023-10-09T09:01:27.312 INFO:teuthology.orchestra.run.smithi181.stdout: key = AQBnwSNlBkQZEhAATTFTEbplN2NeAQpkqwA4Ng== 2023-10-09T09:01:28.203 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T09:01:28.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T09:01:28.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T09:01:28.222 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T09:01:28.222 DEBUG:teuthology.orchestra.run.smithi163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T09:01:28.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:28 smithi181 bash[27712]: cluster 2023-10-09T09:01:27.170333+0000 mgr.y (mgr.24355) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:28.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:28 smithi181 bash[27712]: audit 2023-10-09T09:01:27.300201+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.181:0/4280023681' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:01:28.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:28 smithi181 bash[27712]: audit 2023-10-09T09:01:27.303306+0000 mon.a (mon.0) 602 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:01:28.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:28 smithi181 bash[27712]: audit 2023-10-09T09:01:27.311539+0000 mon.a (mon.0) 603 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:01:28.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:28 smithi163 bash[27500]: cluster 2023-10-09T09:01:27.170333+0000 mgr.y (mgr.24355) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:28.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:28 smithi163 bash[27500]: audit 2023-10-09T09:01:27.300201+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.181:0/4280023681' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:01:28.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:28 smithi163 bash[27500]: audit 2023-10-09T09:01:27.303306+0000 mon.a (mon.0) 602 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:01:28.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:28 smithi163 bash[27500]: audit 2023-10-09T09:01:27.311539+0000 mon.a (mon.0) 603 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:01:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:28 smithi163 bash[33912]: cluster 2023-10-09T09:01:27.170333+0000 mgr.y (mgr.24355) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:28 smithi163 bash[33912]: audit 2023-10-09T09:01:27.300201+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.181:0/4280023681' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:01:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:28 smithi163 bash[33912]: audit 2023-10-09T09:01:27.303306+0000 mon.a (mon.0) 602 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:01:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:28 smithi163 bash[33912]: audit 2023-10-09T09:01:27.311539+0000 mon.a (mon.0) 603 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:01:28.509 INFO:teuthology.misc.health.smithi163.stdout: 2023-10-09T09:01:28.510 INFO:teuthology.misc.health.smithi163.stdout:{"epoch":52,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:01:07.146663+0000","last_up_change":"2023-10-09T09:00:36.182187+0000","last_in_change":"2023-10-09T09:00:10.180048+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6803","nonce":422484962}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6805","nonce":422484962}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6809","nonce":422484962}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6807","nonce":422484962}]},"public_addr":"172.21.15.163:6803/422484962","cluster_addr":"172.21.15.163:6805/422484962","heartbeat_back_addr":"172.21.15.163:6809/422484962","heartbeat_front_addr":"172.21.15.163:6807/422484962","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6811","nonce":1836428548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6813","nonce":1836428548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6817","nonce":1836428548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6815","nonce":1836428548}]},"public_addr":"172.21.15.163:6811/1836428548","cluster_addr":"172.21.15.163:6813/1836428548","heartbeat_back_addr":"172.21.15.163:6817/1836428548","heartbeat_front_addr":"172.21.15.163:6815/1836428548","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6819","nonce":2248501829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6821","nonce":2248501829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6825","nonce":2248501829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6823","nonce":2248501829}]},"public_addr":"172.21.15.163:6819/2248501829","cluster_addr":"172.21.15.163:6821/2248501829","heartbeat_back_addr":"172.21.15.163:6825/2248501829","heartbeat_front_addr":"172.21.15.163:6823/2248501829","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6827","nonce":3899711337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6829","nonce":3899711337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6833","nonce":3899711337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6831","nonce":3899711337}]},"public_addr":"172.21.15.163:6827/3899711337","cluster_addr":"172.21.15.163:6829/3899711337","heartbeat_back_addr":"172.21.15.163:6833/3899711337","heartbeat_front_addr":"172.21.15.163:6831/3899711337","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:6801/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6800/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/566729834":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2199287559":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:01:28.523 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T09:01:28.523 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T09:01:28.523 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T09:01:29.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:29 smithi163 bash[33912]: audit 2023-10-09T09:01:28.508341+0000 mon.a (mon.0) 604 : audit [DBG] from='client.? 172.21.15.163:0/3472562514' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:01:29.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:29 smithi163 bash[33912]: audit 2023-10-09T09:01:28.513500+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:29.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:29 smithi163 bash[33912]: cephadm 2023-10-09T09:01:28.516409+0000 mgr.y (mgr.24355) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi163 2023-10-09T09:01:29.228 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:29 smithi163 systemd[1]: Started Ceph node-exporter.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:01:29.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:29 smithi163 bash[27500]: audit 2023-10-09T09:01:28.508341+0000 mon.a (mon.0) 604 : audit [DBG] from='client.? 172.21.15.163:0/3472562514' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:01:29.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:29 smithi163 bash[27500]: audit 2023-10-09T09:01:28.513500+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:29.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:29 smithi163 bash[27500]: cephadm 2023-10-09T09:01:28.516409+0000 mgr.y (mgr.24355) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi163 2023-10-09T09:01:29.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:29 smithi181 bash[27712]: audit 2023-10-09T09:01:28.508341+0000 mon.a (mon.0) 604 : audit [DBG] from='client.? 172.21.15.163:0/3472562514' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:01:29.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:29 smithi181 bash[27712]: audit 2023-10-09T09:01:28.513500+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:29.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:29 smithi181 bash[27712]: cephadm 2023-10-09T09:01:28.516409+0000 mgr.y (mgr.24355) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi163 2023-10-09T09:01:29.678 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:29 smithi163 bash[52728]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-10-09T09:01:29.727 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:01:29 smithi181 bash[46808]: ts=2023-10-09T09:01:29.619Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T09:01:29.997 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:29 smithi181 systemd[1]: Started Ceph node-exporter.b for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:01:30.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:30 smithi181 bash[27712]: cluster 2023-10-09T09:01:29.171076+0000 mgr.y (mgr.24355) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:30.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:30 smithi181 bash[27712]: audit 2023-10-09T09:01:29.243182+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:30.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:30 smithi181 bash[27712]: cephadm 2023-10-09T09:01:29.244644+0000 mgr.y (mgr.24355) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi181 2023-10-09T09:01:30.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:30 smithi181 bash[27712]: audit 2023-10-09T09:01:29.985991+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:30.318 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:29 smithi181 bash[46973]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-10-09T09:01:30.468 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[52728]: v1.3.1: Pulling from prometheus/node-exporter 2023-10-09T09:01:30.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[33912]: cluster 2023-10-09T09:01:29.171076+0000 mgr.y (mgr.24355) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:30.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[33912]: audit 2023-10-09T09:01:29.243182+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:30.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[33912]: cephadm 2023-10-09T09:01:29.244644+0000 mgr.y (mgr.24355) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi181 2023-10-09T09:01:30.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[33912]: audit 2023-10-09T09:01:29.985991+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[27500]: cluster 2023-10-09T09:01:29.171076+0000 mgr.y (mgr.24355) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[27500]: audit 2023-10-09T09:01:29.243182+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[27500]: cephadm 2023-10-09T09:01:29.244644+0000 mgr.y (mgr.24355) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi181 2023-10-09T09:01:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[27500]: audit 2023-10-09T09:01:29.985991+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:31.027 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:30 smithi181 bash[46973]: v1.3.1: Pulling from prometheus/node-exporter 2023-10-09T09:01:31.209 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[52728]: aa2a8d90b84c: Pulling fs layer 2023-10-09T09:01:31.210 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[52728]: b45d31ee2d7f: Pulling fs layer 2023-10-09T09:01:31.210 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:30 smithi163 bash[52728]: b5db1e299295: Pulling fs layer 2023-10-09T09:01:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[27712]: cephadm 2023-10-09T09:01:29.999058+0000 mgr.y (mgr.24355) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi163 2023-10-09T09:01:31.318 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[46973]: aa2a8d90b84c: Pulling fs layer 2023-10-09T09:01:31.318 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[46973]: b45d31ee2d7f: Pulling fs layer 2023-10-09T09:01:31.318 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[46973]: b5db1e299295: Pulling fs layer 2023-10-09T09:01:31.468 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:31 smithi163 bash[52728]: b45d31ee2d7f: Verifying Checksum 2023-10-09T09:01:31.468 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:31 smithi163 bash[52728]: b45d31ee2d7f: Download complete 2023-10-09T09:01:31.468 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:31 smithi163 bash[52728]: aa2a8d90b84c: Verifying Checksum 2023-10-09T09:01:31.468 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:31 smithi163 bash[52728]: aa2a8d90b84c: Download complete 2023-10-09T09:01:31.468 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:31 smithi163 bash[52728]: b5db1e299295: Verifying Checksum 2023-10-09T09:01:31.468 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:31 smithi163 bash[52728]: b5db1e299295: Download complete 2023-10-09T09:01:31.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:31 smithi163 bash[27500]: cephadm 2023-10-09T09:01:29.999058+0000 mgr.y (mgr.24355) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi163 2023-10-09T09:01:31.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:31 smithi163 bash[33912]: cephadm 2023-10-09T09:01:29.999058+0000 mgr.y (mgr.24355) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi163 2023-10-09T09:01:31.817 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[46973]: b45d31ee2d7f: Verifying Checksum 2023-10-09T09:01:31.818 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[46973]: b45d31ee2d7f: Download complete 2023-10-09T09:01:31.818 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[46973]: aa2a8d90b84c: Verifying Checksum 2023-10-09T09:01:31.818 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[46973]: aa2a8d90b84c: Download complete 2023-10-09T09:01:31.818 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[46973]: b5db1e299295: Verifying Checksum 2023-10-09T09:01:31.818 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[46973]: b5db1e299295: Download complete 2023-10-09T09:01:31.818 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:31 smithi181 bash[46973]: aa2a8d90b84c: Pull complete 2023-10-09T09:01:32.218 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:31 smithi163 bash[52728]: aa2a8d90b84c: Pull complete 2023-10-09T09:01:32.433 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:32 smithi181 bash[27712]: cluster 2023-10-09T09:01:31.171570+0000 mgr.y (mgr.24355) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:32.433 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:32 smithi181 bash[46973]: b45d31ee2d7f: Pull complete 2023-10-09T09:01:32.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:32 smithi163 bash[33912]: cluster 2023-10-09T09:01:31.171570+0000 mgr.y (mgr.24355) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:32.514 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:32 smithi163 bash[27500]: cluster 2023-10-09T09:01:31.171570+0000 mgr.y (mgr.24355) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:32.698 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:32 smithi181 bash[46973]: b5db1e299295: Pull complete 2023-10-09T09:01:32.699 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:32 smithi181 bash[46973]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-10-09T09:01:32.699 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:32 smithi181 bash[46973]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-10-09T09:01:32.850 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:32 smithi163 bash[52728]: b45d31ee2d7f: Pull complete 2023-10-09T09:01:33.162 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:32 smithi163 bash[52728]: b5db1e299295: Pull complete 2023-10-09T09:01:33.162 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:33 smithi163 bash[52728]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-10-09T09:01:33.163 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:33 smithi163 bash[52728]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-10-09T09:01:33.367 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'rbd' created 2023-10-09T09:01:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:33 smithi163 bash[27500]: audit 2023-10-09T09:01:32.350247+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:33 smithi163 bash[27500]: audit 2023-10-09T09:01:32.845787+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.163:0/2621075964' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:01:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:33 smithi163 bash[27500]: audit 2023-10-09T09:01:32.846277+0000 mon.a (mon.0) 609 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:01:33.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:33 smithi163 bash[33912]: audit 2023-10-09T09:01:32.350247+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:33 smithi163 bash[33912]: audit 2023-10-09T09:01:32.845787+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.163:0/2621075964' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:01:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:33 smithi163 bash[33912]: audit 2023-10-09T09:01:32.846277+0000 mon.a (mon.0) 609 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:01:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:33 smithi181 bash[27712]: audit 2023-10-09T09:01:32.350247+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:33 smithi181 bash[27712]: audit 2023-10-09T09:01:32.845787+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.163:0/2621075964' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:01:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:33 smithi181 bash[27712]: audit 2023-10-09T09:01:32.846277+0000 mon.a (mon.0) 609 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:01:34.318 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.079Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T09:01:34.318 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.079Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T09:01:34.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.079Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T09:01:34.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.079Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T09:01:34.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T09:01:34.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T09:01:34.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T09:01:34.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T09:01:34.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T09:01:34.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T09:01:34.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T09:01:34.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.081Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T09:01:34.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T09:01:34.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T09:01:34.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[46973]: ts=2023-10-09T09:01:34.082Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T09:01:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:34 smithi163 bash[27500]: cluster 2023-10-09T09:01:33.171932+0000 mgr.y (mgr.24355) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:34 smithi163 bash[27500]: audit 2023-10-09T09:01:33.363497+0000 mon.a (mon.0) 610 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T09:01:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:34 smithi163 bash[27500]: cluster 2023-10-09T09:01:33.363586+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T09:01:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:34 smithi163 bash[27500]: audit 2023-10-09T09:01:33.364255+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T09:01:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:34 smithi163 bash[27500]: audit 2023-10-09T09:01:33.365621+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T09:01:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:34 smithi163 bash[33912]: cluster 2023-10-09T09:01:33.171932+0000 mgr.y (mgr.24355) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:34 smithi163 bash[33912]: audit 2023-10-09T09:01:33.363497+0000 mon.a (mon.0) 610 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T09:01:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:34 smithi163 bash[33912]: cluster 2023-10-09T09:01:33.363586+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T09:01:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:34 smithi163 bash[33912]: audit 2023-10-09T09:01:33.364255+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T09:01:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:34 smithi163 bash[33912]: audit 2023-10-09T09:01:33.365621+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T09:01:34.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[27712]: cluster 2023-10-09T09:01:33.171932+0000 mgr.y (mgr.24355) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:34.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[27712]: audit 2023-10-09T09:01:33.363497+0000 mon.a (mon.0) 610 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T09:01:34.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[27712]: cluster 2023-10-09T09:01:33.363586+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T09:01:34.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[27712]: audit 2023-10-09T09:01:33.364255+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T09:01:34.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:34 smithi181 bash[27712]: audit 2023-10-09T09:01:33.365621+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T09:01:35.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:35 smithi163 bash[27500]: audit 2023-10-09T09:01:34.364650+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T09:01:35.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:35 smithi163 bash[27500]: cluster 2023-10-09T09:01:34.364821+0000 mon.a (mon.0) 615 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T09:01:35.377 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:35 smithi163 bash[33912]: audit 2023-10-09T09:01:34.364650+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T09:01:35.377 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:35 smithi163 bash[33912]: cluster 2023-10-09T09:01:34.364821+0000 mon.a (mon.0) 615 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T09:01:35.742 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T09:01:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:35 smithi181 bash[27712]: audit 2023-10-09T09:01:34.364650+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T09:01:35.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:35 smithi181 bash[27712]: cluster 2023-10-09T09:01:34.364821+0000 mon.a (mon.0) 615 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T09:01:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[27500]: cluster 2023-10-09T09:01:35.172431+0000 mgr.y (mgr.24355) 29 : cluster [DBG] pgmap v19: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[27500]: cluster 2023-10-09T09:01:35.373973+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T09:01:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[33912]: cluster 2023-10-09T09:01:35.172431+0000 mgr.y (mgr.24355) 29 : cluster [DBG] pgmap v19: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[33912]: cluster 2023-10-09T09:01:35.373973+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T09:01:36.719 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.384Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T09:01:36.719 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.384Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T09:01:36.719 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.385Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T09:01:36.719 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.385Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T09:01:36.719 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T09:01:36.719 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T09:01:36.720 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T09:01:36.721 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T09:01:36.722 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T09:01:36.723 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T09:01:36.723 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T09:01:36.723 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T09:01:36.723 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T09:01:36.723 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T09:01:36.723 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T09:01:36.723 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.386Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T09:01:36.723 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:01:36 smithi163 bash[52728]: ts=2023-10-09T09:01:36.387Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T09:01:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:36 smithi181 bash[27712]: cluster 2023-10-09T09:01:35.172431+0000 mgr.y (mgr.24355) 29 : cluster [DBG] pgmap v19: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:36 smithi181 bash[27712]: cluster 2023-10-09T09:01:35.373973+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T09:01:38.286 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:38 smithi163 bash[27500]: cluster 2023-10-09T09:01:37.173054+0000 mgr.y (mgr.24355) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:38.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:38 smithi163 bash[33912]: cluster 2023-10-09T09:01:37.173054+0000 mgr.y (mgr.24355) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:38 smithi181 bash[27712]: cluster 2023-10-09T09:01:37.173054+0000 mgr.y (mgr.24355) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:38.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:38 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:01:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:01:40.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:39 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:01:39] "GET /metrics HTTP/1.1" 200 155657 "" "Prometheus/2.33.4" 2023-10-09T09:01:40.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:40 smithi163 bash[33912]: cluster 2023-10-09T09:01:39.173780+0000 mgr.y (mgr.24355) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:40.499 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:40 smithi163 bash[27500]: cluster 2023-10-09T09:01:39.173780+0000 mgr.y (mgr.24355) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:40 smithi181 bash[27712]: cluster 2023-10-09T09:01:39.173780+0000 mgr.y (mgr.24355) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:41.253 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T09:01:41.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:41 smithi163 bash[33912]: audit 2023-10-09T09:01:40.924700+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.163:0/2955429260' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:41.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:41 smithi163 bash[33912]: audit 2023-10-09T09:01:40.925368+0000 mon.a (mon.0) 617 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:41 smithi163 bash[27500]: audit 2023-10-09T09:01:40.924700+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.163:0/2955429260' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:41 smithi163 bash[27500]: audit 2023-10-09T09:01:40.925368+0000 mon.a (mon.0) 617 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:41 smithi181 bash[27712]: audit 2023-10-09T09:01:40.924700+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.163:0/2955429260' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:41.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:41 smithi181 bash[27712]: audit 2023-10-09T09:01:40.925368+0000 mon.a (mon.0) 617 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:01:42.251 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T09:01:42.251 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T09:01:42.251 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T09:01:42.452 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:42 smithi163 systemd[1]: Started Ceph alertmanager.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:01:42.452 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:42 smithi163 bash[33912]: cluster 2023-10-09T09:01:41.174367+0000 mgr.y (mgr.24355) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:42.452 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:42 smithi163 bash[33912]: audit 2023-10-09T09:01:41.244891+0000 mon.a (mon.0) 618 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:01:42.452 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:42 smithi163 bash[33912]: cluster 2023-10-09T09:01:41.244948+0000 mon.a (mon.0) 619 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T09:01:42.453 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:42 smithi163 bash[27500]: cluster 2023-10-09T09:01:41.174367+0000 mgr.y (mgr.24355) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:42.453 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:42 smithi163 bash[27500]: audit 2023-10-09T09:01:41.244891+0000 mon.a (mon.0) 618 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:01:42.453 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:42 smithi163 bash[27500]: cluster 2023-10-09T09:01:41.244948+0000 mon.a (mon.0) 619 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T09:01:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:42 smithi181 bash[27712]: cluster 2023-10-09T09:01:41.174367+0000 mgr.y (mgr.24355) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:42 smithi181 bash[27712]: audit 2023-10-09T09:01:41.244891+0000 mon.a (mon.0) 618 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:01:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:42 smithi181 bash[27712]: cluster 2023-10-09T09:01:41.244948+0000 mon.a (mon.0) 619 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T09:01:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[33912]: audit 2023-10-09T09:01:42.434769+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[33912]: audit 2023-10-09T09:01:42.436741+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[27500]: audit 2023-10-09T09:01:42.434769+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[27500]: audit 2023-10-09T09:01:42.436741+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:43.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[53482]: level=info ts=2023-10-09T09:01:43.486Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T09:01:43.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[53482]: level=info ts=2023-10-09T09:01:43.486Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T09:01:43.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[53482]: level=info ts=2023-10-09T09:01:43.488Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.163 port=9094 2023-10-09T09:01:43.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[53482]: level=info ts=2023-10-09T09:01:43.489Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T09:01:43.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[53482]: level=info ts=2023-10-09T09:01:43.539Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:01:43.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[53482]: level=info ts=2023-10-09T09:01:43.539Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:01:43.720 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[53482]: level=info ts=2023-10-09T09:01:43.543Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T09:01:43.720 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:43 smithi163 bash[53482]: level=info ts=2023-10-09T09:01:43.543Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T09:01:43.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:43 smithi181 bash[27712]: audit 2023-10-09T09:01:42.434769+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:43.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:43 smithi181 bash[27712]: audit 2023-10-09T09:01:42.436741+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:01:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:44 smithi163 bash[27500]: cluster 2023-10-09T09:01:43.174984+0000 mgr.y (mgr.24355) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:44 smithi163 bash[33912]: cluster 2023-10-09T09:01:43.174984+0000 mgr.y (mgr.24355) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:44 smithi181 bash[27712]: cluster 2023-10-09T09:01:43.174984+0000 mgr.y (mgr.24355) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:45.968 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:45 smithi163 bash[53482]: level=info ts=2023-10-09T09:01:45.490Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000178935s 2023-10-09T09:01:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:46 smithi163 bash[33912]: cluster 2023-10-09T09:01:45.175540+0000 mgr.y (mgr.24355) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:46 smithi163 bash[27500]: cluster 2023-10-09T09:01:45.175540+0000 mgr.y (mgr.24355) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:46 smithi181 bash[27712]: cluster 2023-10-09T09:01:45.175540+0000 mgr.y (mgr.24355) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:48.028 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:48 smithi163 bash[27500]: cluster 2023-10-09T09:01:47.176130+0000 mgr.y (mgr.24355) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:48 smithi163 bash[27500]: audit 2023-10-09T09:01:47.367554+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:48 smithi163 bash[27500]: audit 2023-10-09T09:01:48.123622+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:48 smithi163 bash[33912]: cluster 2023-10-09T09:01:47.176130+0000 mgr.y (mgr.24355) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:48 smithi163 bash[33912]: audit 2023-10-09T09:01:47.367554+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:48 smithi163 bash[33912]: audit 2023-10-09T09:01:48.123622+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:48.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:48 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:01:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:01:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:48 smithi181 bash[27712]: cluster 2023-10-09T09:01:47.176130+0000 mgr.y (mgr.24355) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:48 smithi181 bash[27712]: audit 2023-10-09T09:01:47.367554+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:48 smithi181 bash[27712]: audit 2023-10-09T09:01:48.123622+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:49.303 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:48 smithi163 bash[27764]: [09/Oct/2023:09:01:48] ENGINE Bus STOPPING 2023-10-09T09:01:49.304 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:48 smithi163 bash[27764]: [09/Oct/2023:09:01:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:01:49.304 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:48 smithi163 bash[27764]: [09/Oct/2023:09:01:48] ENGINE Bus STOPPED 2023-10-09T09:01:49.304 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:48 smithi163 bash[27764]: [09/Oct/2023:09:01:48] ENGINE Bus STARTING 2023-10-09T09:01:49.304 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27764]: [09/Oct/2023:09:01:49] ENGINE Serving on http://:::9283 2023-10-09T09:01:49.304 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27764]: [09/Oct/2023:09:01:49] ENGINE Bus STARTED 2023-10-09T09:01:50.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:01:49] "GET /metrics HTTP/1.1" 200 155657 "" "Prometheus/2.33.4" 2023-10-09T09:01:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: audit 2023-10-09T09:01:48.864523+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: audit 2023-10-09T09:01:48.865902+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: audit 2023-10-09T09:01:48.867102+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: audit 2023-10-09T09:01:48.873567+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: audit 2023-10-09T09:01:48.948751+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: audit 2023-10-09T09:01:48.952467+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: audit 2023-10-09T09:01:48.955865+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:01:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: audit 2023-10-09T09:01:48.956609+0000 mgr.y (mgr.24355) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:01:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: audit 2023-10-09T09:01:48.961686+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: cephadm 2023-10-09T09:01:48.968011+0000 mgr.y (mgr.24355) 37 : cephadm [INF] Deploying daemon grafana.a on smithi181 2023-10-09T09:01:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[33912]: cluster 2023-10-09T09:01:49.176866+0000 mgr.y (mgr.24355) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:50.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: audit 2023-10-09T09:01:48.864523+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: audit 2023-10-09T09:01:48.865902+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: audit 2023-10-09T09:01:48.867102+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: audit 2023-10-09T09:01:48.873567+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: audit 2023-10-09T09:01:48.948751+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: audit 2023-10-09T09:01:48.952467+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: audit 2023-10-09T09:01:48.955865+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:01:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: audit 2023-10-09T09:01:48.956609+0000 mgr.y (mgr.24355) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:01:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: audit 2023-10-09T09:01:48.961686+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: cephadm 2023-10-09T09:01:48.968011+0000 mgr.y (mgr.24355) 37 : cephadm [INF] Deploying daemon grafana.a on smithi181 2023-10-09T09:01:50.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:49 smithi163 bash[27500]: cluster 2023-10-09T09:01:49.176866+0000 mgr.y (mgr.24355) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: audit 2023-10-09T09:01:48.864523+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: audit 2023-10-09T09:01:48.865902+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:01:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: audit 2023-10-09T09:01:48.867102+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:01:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: audit 2023-10-09T09:01:48.873567+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: audit 2023-10-09T09:01:48.948751+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: audit 2023-10-09T09:01:48.952467+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: audit 2023-10-09T09:01:48.955865+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:01:50.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: audit 2023-10-09T09:01:48.956609+0000 mgr.y (mgr.24355) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:01:50.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: audit 2023-10-09T09:01:48.961686+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:50.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: cephadm 2023-10-09T09:01:48.968011+0000 mgr.y (mgr.24355) 37 : cephadm [INF] Deploying daemon grafana.a on smithi181 2023-10-09T09:01:50.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:49 smithi181 bash[27712]: cluster 2023-10-09T09:01:49.176866+0000 mgr.y (mgr.24355) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:51.054 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:01:51.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:51 smithi163 bash[27500]: audit 2023-10-09T09:01:51.049196+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.163:0/1744358240' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:01:51.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:51 smithi163 bash[33912]: audit 2023-10-09T09:01:51.049196+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.163:0/1744358240' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:01:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:51 smithi181 bash[27712]: audit 2023-10-09T09:01:51.049196+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.163:0/1744358240' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:01:51.764 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":18,"active_gid":24355,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6800","nonce":3919298633},{"type":"v1","addr":"172.21.15.163:6801","nonce":3919298633}]},"active_addr":"172.21.15.163:6801/3919298633","active_change":"2023-10-09T09:01:07.146973+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24358,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":52,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":536711455}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":623143106}]}]} 2023-10-09T09:01:51.768 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T09:01:51.769 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T09:01:51.769 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:01:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:52 smithi163 bash[27500]: cluster 2023-10-09T09:01:51.177483+0000 mgr.y (mgr.24355) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:52 smithi163 bash[33912]: cluster 2023-10-09T09:01:51.177483+0000 mgr.y (mgr.24355) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:52 smithi181 bash[27712]: cluster 2023-10-09T09:01:51.177483+0000 mgr.y (mgr.24355) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:53 smithi163 bash[33912]: audit 2023-10-09T09:01:52.377445+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:53.718 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:01:53 smithi163 bash[53482]: level=info ts=2023-10-09T09:01:53.492Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002567418s 2023-10-09T09:01:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:53 smithi163 bash[27500]: audit 2023-10-09T09:01:52.377445+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:53 smithi181 bash[27712]: audit 2023-10-09T09:01:52.377445+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:01:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:54 smithi163 bash[27500]: cluster 2023-10-09T09:01:53.178026+0000 mgr.y (mgr.24355) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:54.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:54 smithi163 bash[33912]: cluster 2023-10-09T09:01:53.178026+0000 mgr.y (mgr.24355) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:54 smithi181 bash[27712]: cluster 2023-10-09T09:01:53.178026+0000 mgr.y (mgr.24355) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:56.541 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:01:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:56 smithi163 bash[27500]: cluster 2023-10-09T09:01:55.178614+0000 mgr.y (mgr.24355) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:56 smithi163 bash[33912]: cluster 2023-10-09T09:01:55.178614+0000 mgr.y (mgr.24355) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:56 smithi181 bash[27712]: cluster 2023-10-09T09:01:55.178614+0000 mgr.y (mgr.24355) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:01:58 smithi181 bash[27712]: cluster 2023-10-09T09:01:57.179209+0000 mgr.y (mgr.24355) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:01:58 smithi163 bash[33912]: cluster 2023-10-09T09:01:57.179209+0000 mgr.y (mgr.24355) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:58.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:01:58 smithi163 bash[27500]: cluster 2023-10-09T09:01:57.179209+0000 mgr.y (mgr.24355) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:01:58.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:01:58 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:01:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:01:59.265 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:01:59.265 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":56,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:01:41.230835+0000","last_up_change":"2023-10-09T09:00:36.182187+0000","last_in_change":"2023-10-09T09:00:10.180048+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:01:33.304980+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6803","nonce":422484962}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6805","nonce":422484962}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6809","nonce":422484962}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6807","nonce":422484962}]},"public_addr":"172.21.15.163:6803/422484962","cluster_addr":"172.21.15.163:6805/422484962","heartbeat_back_addr":"172.21.15.163:6809/422484962","heartbeat_front_addr":"172.21.15.163:6807/422484962","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6811","nonce":1836428548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6813","nonce":1836428548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6817","nonce":1836428548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6815","nonce":1836428548}]},"public_addr":"172.21.15.163:6811/1836428548","cluster_addr":"172.21.15.163:6813/1836428548","heartbeat_back_addr":"172.21.15.163:6817/1836428548","heartbeat_front_addr":"172.21.15.163:6815/1836428548","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6819","nonce":2248501829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6821","nonce":2248501829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6825","nonce":2248501829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6823","nonce":2248501829}]},"public_addr":"172.21.15.163:6819/2248501829","cluster_addr":"172.21.15.163:6821/2248501829","heartbeat_back_addr":"172.21.15.163:6825/2248501829","heartbeat_front_addr":"172.21.15.163:6823/2248501829","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6827","nonce":3899711337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6829","nonce":3899711337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6833","nonce":3899711337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6831","nonce":3899711337}]},"public_addr":"172.21.15.163:6827/3899711337","cluster_addr":"172.21.15.163:6829/3899711337","heartbeat_back_addr":"172.21.15.163:6833/3899711337","heartbeat_front_addr":"172.21.15.163:6831/3899711337","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:6801/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6800/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/566729834":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2199287559":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:01:59.884 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T09:01:59.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:02:00.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:01:59 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:01:59] "GET /metrics HTTP/1.1" 200 155666 "" "Prometheus/2.33.4" 2023-10-09T09:02:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:00 smithi181 bash[27712]: cluster 2023-10-09T09:01:59.179813+0000 mgr.y (mgr.24355) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:00 smithi181 bash[27712]: audit 2023-10-09T09:01:59.263471+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.163:0/906896719' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:02:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:00 smithi163 bash[27500]: cluster 2023-10-09T09:01:59.179813+0000 mgr.y (mgr.24355) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:00 smithi163 bash[27500]: audit 2023-10-09T09:01:59.263471+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.163:0/906896719' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:02:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:00 smithi163 bash[33912]: cluster 2023-10-09T09:01:59.179813+0000 mgr.y (mgr.24355) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:00 smithi163 bash[33912]: audit 2023-10-09T09:01:59.263471+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.163:0/906896719' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:02:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:02 smithi181 bash[27712]: cluster 2023-10-09T09:02:01.180356+0000 mgr.y (mgr.24355) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:02.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:02 smithi163 bash[27500]: cluster 2023-10-09T09:02:01.180356+0000 mgr.y (mgr.24355) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:02.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:02 smithi163 bash[33912]: cluster 2023-10-09T09:02:01.180356+0000 mgr.y (mgr.24355) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:04 smithi181 bash[27712]: cluster 2023-10-09T09:02:03.181016+0000 mgr.y (mgr.24355) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:04.657 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:04.688 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:04 smithi163 bash[33912]: cluster 2023-10-09T09:02:03.181016+0000 mgr.y (mgr.24355) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:04.688 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:04 smithi163 bash[27500]: cluster 2023-10-09T09:02:03.181016+0000 mgr.y (mgr.24355) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:06.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:06 smithi181 bash[27712]: cluster 2023-10-09T09:02:05.181599+0000 mgr.y (mgr.24355) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:06 smithi163 bash[27500]: cluster 2023-10-09T09:02:05.181599+0000 mgr.y (mgr.24355) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:06 smithi163 bash[33912]: cluster 2023-10-09T09:02:05.181599+0000 mgr.y (mgr.24355) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:07.505 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:02:07.506 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":56,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:01:41.230835+0000","last_up_change":"2023-10-09T09:00:36.182187+0000","last_in_change":"2023-10-09T09:00:10.180048+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:01:33.304980+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6803","nonce":422484962}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6805","nonce":422484962}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6809","nonce":422484962}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6807","nonce":422484962}]},"public_addr":"172.21.15.163:6803/422484962","cluster_addr":"172.21.15.163:6805/422484962","heartbeat_back_addr":"172.21.15.163:6809/422484962","heartbeat_front_addr":"172.21.15.163:6807/422484962","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6811","nonce":1836428548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6813","nonce":1836428548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6817","nonce":1836428548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6815","nonce":1836428548}]},"public_addr":"172.21.15.163:6811/1836428548","cluster_addr":"172.21.15.163:6813/1836428548","heartbeat_back_addr":"172.21.15.163:6817/1836428548","heartbeat_front_addr":"172.21.15.163:6815/1836428548","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6819","nonce":2248501829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6821","nonce":2248501829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6825","nonce":2248501829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6823","nonce":2248501829}]},"public_addr":"172.21.15.163:6819/2248501829","cluster_addr":"172.21.15.163:6821/2248501829","heartbeat_back_addr":"172.21.15.163:6825/2248501829","heartbeat_front_addr":"172.21.15.163:6823/2248501829","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6827","nonce":3899711337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6829","nonce":3899711337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6833","nonce":3899711337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6831","nonce":3899711337}]},"public_addr":"172.21.15.163:6827/3899711337","cluster_addr":"172.21.15.163:6829/3899711337","heartbeat_back_addr":"172.21.15.163:6833/3899711337","heartbeat_front_addr":"172.21.15.163:6831/3899711337","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:6801/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6800/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/566729834":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2199287559":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:02:08.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T09:02:08.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T09:02:08.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T09:02:08.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T09:02:08.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T09:02:08.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T09:02:08.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T09:02:08.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T09:02:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:08 smithi181 bash[27712]: cluster 2023-10-09T09:02:07.182127+0000 mgr.y (mgr.24355) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:08 smithi181 bash[27712]: audit 2023-10-09T09:02:07.504032+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.163:0/4255552692' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:02:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:08 smithi163 bash[33912]: cluster 2023-10-09T09:02:07.182127+0000 mgr.y (mgr.24355) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:08 smithi163 bash[33912]: audit 2023-10-09T09:02:07.504032+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.163:0/4255552692' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:02:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:08 smithi163 bash[27500]: cluster 2023-10-09T09:02:07.182127+0000 mgr.y (mgr.24355) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:08 smithi163 bash[27500]: audit 2023-10-09T09:02:07.504032+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.163:0/4255552692' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:02:08.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:02:08 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:02:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:02:10.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:09 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:02:09] "GET /metrics HTTP/1.1" 200 155667 "" "Prometheus/2.33.4" 2023-10-09T09:02:10.220 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:10 smithi181 systemd[1]: Started Ceph grafana.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:02:10.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:10 smithi181 bash[27712]: cluster 2023-10-09T09:02:09.182969+0000 mgr.y (mgr.24355) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:10.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:10 smithi181 bash[27712]: audit 2023-10-09T09:02:10.203747+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:10.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:10 smithi181 bash[27712]: audit 2023-10-09T09:02:10.206469+0000 mon.a (mon.0) 635 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:02:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:10 smithi163 bash[27500]: cluster 2023-10-09T09:02:09.182969+0000 mgr.y (mgr.24355) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:10 smithi163 bash[27500]: audit 2023-10-09T09:02:10.203747+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:10 smithi163 bash[27500]: audit 2023-10-09T09:02:10.206469+0000 mon.a (mon.0) 635 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:02:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:10 smithi163 bash[33912]: cluster 2023-10-09T09:02:09.182969+0000 mgr.y (mgr.24355) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:10 smithi163 bash[33912]: audit 2023-10-09T09:02:10.203747+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:10 smithi163 bash[33912]: audit 2023-10-09T09:02:10.206469+0000 mon.a (mon.0) 635 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:02:11.377 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T09:02:11.377 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T09:02:11.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T09:02:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T09:02:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T09:02:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T09:02:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T09:02:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-10-09T09:02:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-10-09T09:02:11.379 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-10-09T09:02:11.629 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-10-09T09:02:11.630 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-10-09T09:02:11.630 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-10-09T09:02:11.630 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-10-09T09:02:11.630 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-10-09T09:02:11.630 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-10-09T09:02:11.630 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-10-09T09:02:11.630 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T09:02:11.630 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-10-09T09:02:11.890 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-10-09T09:02:11.890 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-10-09T09:02:11.890 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-10-09T09:02:11.890 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-10-09T09:02:11.891 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-10-09T09:02:11.891 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-10-09T09:02:11.891 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-10-09T09:02:11.891 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-10-09T09:02:12.176 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-10-09T09:02:12.176 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T09:02:12.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-10-09T09:02:12.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:11 smithi181 bash[47587]: t=2023-10-09T09:02:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-10-09T09:02:12.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-10-09T09:02:12.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-10-09T09:02:12.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-10-09T09:02:12.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-10-09T09:02:12.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-10-09T09:02:12.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T09:02:12.436 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[27712]: cluster 2023-10-09T09:02:11.183473+0000 mgr.y (mgr.24355) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:12.437 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-10-09T09:02:12.437 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-10-09T09:02:12.437 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-10-09T09:02:12.437 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-10-09T09:02:12.437 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-10-09T09:02:12.437 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-10-09T09:02:12.706 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-10-09T09:02:12.706 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T09:02:12.706 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-10-09T09:02:12.706 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-10-09T09:02:12.706 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-10-09T09:02:12.706 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-10-09T09:02:12.706 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-10-09T09:02:12.706 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-10-09T09:02:12.706 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T09:02:12.707 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-10-09T09:02:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:12 smithi163 bash[27500]: cluster 2023-10-09T09:02:11.183473+0000 mgr.y (mgr.24355) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:12 smithi163 bash[33912]: cluster 2023-10-09T09:02:11.183473+0000 mgr.y (mgr.24355) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:12.975 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-10-09T09:02:12.975 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-10-09T09:02:12.975 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-10-09T09:02:12.976 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-10-09T09:02:12.976 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-10-09T09:02:12.976 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-10-09T09:02:12.976 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-10-09T09:02:12.976 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T09:02:12.976 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T09:02:12.976 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T09:02:13.109 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:13.110 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:13.117 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:13.117 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:13.118 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:13.118 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:13.118 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:13.118 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:13.244 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:12 smithi181 bash[47587]: t=2023-10-09T09:02:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-10-09T09:02:13.244 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-10-09T09:02:13.244 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T09:02:13.244 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-10-09T09:02:13.244 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-10-09T09:02:13.244 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-10-09T09:02:13.245 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-10-09T09:02:13.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[27712]: audit 2023-10-09T09:02:12.400449+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:13.530 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-10-09T09:02:13.531 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-10-09T09:02:13.531 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-10-09T09:02:13.531 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-10-09T09:02:13.531 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-10-09T09:02:13.531 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-10-09T09:02:13.531 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-10-09T09:02:13.531 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-10-09T09:02:13.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:13 smithi163 bash[33912]: audit 2023-10-09T09:02:12.400449+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:13 smithi163 bash[27500]: audit 2023-10-09T09:02:12.400449+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:13.799 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-10-09T09:02:13.799 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-10-09T09:02:13.799 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-10-09T09:02:13.799 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-10-09T09:02:13.799 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-10-09T09:02:13.800 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-10-09T09:02:13.800 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-10-09T09:02:13.800 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-10-09T09:02:13.800 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T09:02:14.060 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-10-09T09:02:14.060 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T09:02:14.060 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-10-09T09:02:14.060 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T09:02:14.060 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T09:02:14.060 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-10-09T09:02:14.060 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T09:02:14.060 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:13 smithi181 bash[47587]: t=2023-10-09T09:02:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-10-09T09:02:14.060 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-10-09T09:02:14.061 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-10-09T09:02:14.318 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-10-09T09:02:14.318 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-10-09T09:02:14.318 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-10-09T09:02:14.318 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-10-09T09:02:14.318 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-10-09T09:02:14.318 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-10-09T09:02:14.318 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T09:02:14.318 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-10-09T09:02:14.598 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[27712]: cluster 2023-10-09T09:02:13.184065+0000 mgr.y (mgr.24355) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:14.598 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-10-09T09:02:14.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-10-09T09:02:14.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-10-09T09:02:14.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T09:02:14.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-10-09T09:02:14.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-10-09T09:02:14.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-10-09T09:02:14.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-10-09T09:02:14.599 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-10-09T09:02:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:14 smithi163 bash[27500]: cluster 2023-10-09T09:02:13.184065+0000 mgr.y (mgr.24355) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:14 smithi163 bash[33912]: cluster 2023-10-09T09:02:13.184065+0000 mgr.y (mgr.24355) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:14.868 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-10-09T09:02:14.868 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-10-09T09:02:14.868 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-10-09T09:02:14.868 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-10-09T09:02:14.868 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-10-09T09:02:14.868 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-10-09T09:02:14.868 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-10-09T09:02:14.868 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-10-09T09:02:15.128 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-10-09T09:02:15.128 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-10-09T09:02:15.128 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:14 smithi181 bash[47587]: t=2023-10-09T09:02:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-10-09T09:02:15.128 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-10-09T09:02:15.128 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-10-09T09:02:15.129 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T09:02:15.129 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-10-09T09:02:15.389 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-10-09T09:02:15.389 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-10-09T09:02:15.389 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-10-09T09:02:15.389 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-10-09T09:02:15.389 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-10-09T09:02:15.641 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-10-09T09:02:15.641 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-10-09T09:02:15.641 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-10-09T09:02:15.642 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-10-09T09:02:15.642 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-10-09T09:02:15.642 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-10-09T09:02:15.642 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-10-09T09:02:15.902 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T09:02:15.902 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T09:02:15.902 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T09:02:15.903 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T09:02:15.903 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-10-09T09:02:15.903 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T09:02:16.162 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-10-09T09:02:16.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[47587]: t=2023-10-09T09:02:15+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T09:02:16.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-10-09T09:02:16.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T09:02:16.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-10-09T09:02:16.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-10-09T09:02:16.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[27712]: audit 2023-10-09T09:02:14.921260+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:16.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:15 smithi181 bash[27712]: cluster 2023-10-09T09:02:15.184572+0000 mgr.y (mgr.24355) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:16.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:15 smithi163 bash[27500]: audit 2023-10-09T09:02:14.921260+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:16.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:15 smithi163 bash[27500]: cluster 2023-10-09T09:02:15.184572+0000 mgr.y (mgr.24355) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:16.228 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:15 smithi163 bash[33912]: audit 2023-10-09T09:02:14.921260+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:16.228 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:15 smithi163 bash[33912]: cluster 2023-10-09T09:02:15.184572+0000 mgr.y (mgr.24355) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:16.424 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T09:02:16.424 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-10-09T09:02:16.424 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-10-09T09:02:16.424 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-10-09T09:02:16.424 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-10-09T09:02:16.424 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-10-09T09:02:16.425 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-10-09T09:02:16.425 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-10-09T09:02:16.425 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-10-09T09:02:16.684 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-10-09T09:02:16.684 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-10-09T09:02:16.685 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-10-09T09:02:16.685 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-10-09T09:02:16.685 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-10-09T09:02:16.685 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-10-09T09:02:16.685 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-10-09T09:02:16.685 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-10-09T09:02:16.685 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-10-09T09:02:16.953 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-10-09T09:02:16.953 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-10-09T09:02:16.953 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-10-09T09:02:16.954 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-10-09T09:02:16.954 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T09:02:16.954 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T09:02:16.954 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T09:02:16.954 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-10-09T09:02:16.954 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T09:02:17.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[33912]: audit 2023-10-09T09:02:16.013140+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:17.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[33912]: audit 2023-10-09T09:02:16.285792+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:02:17.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[33912]: audit 2023-10-09T09:02:16.286927+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:02:17.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[33912]: audit 2023-10-09T09:02:16.294374+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:17.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[33912]: cephadm 2023-10-09T09:02:16.298492+0000 mgr.y (mgr.24355) 52 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:02:17.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[33912]: cephadm 2023-10-09T09:02:16.301939+0000 mgr.y (mgr.24355) 53 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T09:02:17.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[27500]: audit 2023-10-09T09:02:16.013140+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:17.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[27500]: audit 2023-10-09T09:02:16.285792+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:02:17.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[27500]: audit 2023-10-09T09:02:16.286927+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:02:17.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[27500]: audit 2023-10-09T09:02:16.294374+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:17.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[27500]: cephadm 2023-10-09T09:02:16.298492+0000 mgr.y (mgr.24355) 52 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:02:17.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:17 smithi163 bash[27500]: cephadm 2023-10-09T09:02:16.301939+0000 mgr.y (mgr.24355) 53 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T09:02:17.222 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-10-09T09:02:17.223 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:16 smithi181 bash[47587]: t=2023-10-09T09:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-10-09T09:02:17.223 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-10-09T09:02:17.223 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-10-09T09:02:17.223 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-10-09T09:02:17.223 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-10-09T09:02:17.223 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-10-09T09:02:17.223 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-10-09T09:02:17.223 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[27712]: audit 2023-10-09T09:02:16.013140+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:17.223 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[27712]: audit 2023-10-09T09:02:16.285792+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:02:17.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[27712]: audit 2023-10-09T09:02:16.286927+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:02:17.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[27712]: audit 2023-10-09T09:02:16.294374+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:17.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[27712]: cephadm 2023-10-09T09:02:16.298492+0000 mgr.y (mgr.24355) 52 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:02:17.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[27712]: cephadm 2023-10-09T09:02:16.301939+0000 mgr.y (mgr.24355) 53 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T09:02:17.483 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-10-09T09:02:17.484 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-10-09T09:02:17.484 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-10-09T09:02:17.484 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T09:02:17.484 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-10-09T09:02:17.484 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-10-09T09:02:17.484 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T09:02:17.484 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-10-09T09:02:17.484 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-10-09T09:02:17.753 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-10-09T09:02:17.753 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-10-09T09:02:17.753 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-10-09T09:02:17.753 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-10-09T09:02:17.753 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-10-09T09:02:17.753 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T09:02:17.753 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T09:02:17.753 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-10-09T09:02:17.753 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-10-09T09:02:17.754 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-10-09T09:02:18.013 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-10-09T09:02:18.014 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-10-09T09:02:18.014 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-10-09T09:02:18.014 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-10-09T09:02:18.014 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-10-09T09:02:18.014 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-10-09T09:02:18.014 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-10-09T09:02:18.014 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:17 smithi181 bash[47587]: t=2023-10-09T09:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-10-09T09:02:18.014 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-10-09T09:02:18.265 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[27712]: cluster 2023-10-09T09:02:17.185182+0000 mgr.y (mgr.24355) 54 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:18.266 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T09:02:18.266 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T09:02:18.266 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T09:02:18.266 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-10-09T09:02:18.266 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T09:02:18.266 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-10-09T09:02:18.267 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-10-09T09:02:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:18 smithi163 bash[27500]: cluster 2023-10-09T09:02:17.185182+0000 mgr.y (mgr.24355) 54 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:18.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:18 smithi163 bash[33912]: cluster 2023-10-09T09:02:17.185182+0000 mgr.y (mgr.24355) 54 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:18.535 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-10-09T09:02:18.535 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-10-09T09:02:18.535 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-10-09T09:02:18.535 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-10-09T09:02:18.535 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-10-09T09:02:18.535 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-10-09T09:02:18.535 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-10-09T09:02:18.536 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-10-09T09:02:18.536 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-10-09T09:02:18.536 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-10-09T09:02:18.804 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:02:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:02:18.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-10-09T09:02:18.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T09:02:18.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T09:02:18.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T09:02:18.805 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T09:02:18.805 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-10-09T09:02:18.805 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-10-09T09:02:18.805 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-10-09T09:02:19.056 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-10-09T09:02:19.056 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T09:02:19.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-10-09T09:02:19.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-10-09T09:02:19.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-10-09T09:02:19.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-10-09T09:02:19.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-10-09T09:02:19.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:18 smithi181 bash[47587]: t=2023-10-09T09:02:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-10-09T09:02:19.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-10-09T09:02:19.309 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-10-09T09:02:19.309 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T09:02:19.309 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-10-09T09:02:19.309 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-10-09T09:02:19.310 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-10-09T09:02:19.310 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-10-09T09:02:19.310 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-10-09T09:02:19.310 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-10-09T09:02:19.310 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T09:02:19.568 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T09:02:19.568 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-10-09T09:02:19.568 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-10-09T09:02:19.568 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-10-09T09:02:19.568 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-10-09T09:02:19.568 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-10-09T09:02:19.838 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-10-09T09:02:19.838 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-10-09T09:02:19.838 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-10-09T09:02:19.838 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-10-09T09:02:19.839 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-10-09T09:02:19.839 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-10-09T09:02:19.839 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T09:02:19.839 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-10-09T09:02:20.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-10-09T09:02:20.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-10-09T09:02:20.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-10-09T09:02:20.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-10-09T09:02:20.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T09:02:20.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:19 smithi181 bash[47587]: t=2023-10-09T09:02:19+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-10-09T09:02:20.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-10-09T09:02:20.100 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-10-09T09:02:20.100 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-10-09T09:02:20.100 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-10-09T09:02:20.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:19 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:02:19] "GET /metrics HTTP/1.1" 200 155667 "" "Prometheus/2.33.4" 2023-10-09T09:02:20.351 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[27712]: cluster 2023-10-09T09:02:19.185942+0000 mgr.y (mgr.24355) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:20.352 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-10-09T09:02:20.352 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-10-09T09:02:20.352 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-10-09T09:02:20.352 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-10-09T09:02:20.352 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-10-09T09:02:20.352 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-10-09T09:02:20.352 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-10-09T09:02:20.352 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-10-09T09:02:20.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:20 smithi163 bash[33912]: cluster 2023-10-09T09:02:19.185942+0000 mgr.y (mgr.24355) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:20.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:20 smithi163 bash[27500]: cluster 2023-10-09T09:02:19.185942+0000 mgr.y (mgr.24355) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:20.604 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-10-09T09:02:20.604 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-10-09T09:02:20.604 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-10-09T09:02:20.604 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-10-09T09:02:20.604 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-10-09T09:02:20.604 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-10-09T09:02:20.604 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-10-09T09:02:20.604 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-10-09T09:02:20.865 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T09:02:20.865 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-10-09T09:02:20.865 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-10-09T09:02:20.865 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-10-09T09:02:20.865 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T09:02:20.865 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-10-09T09:02:20.865 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-10-09T09:02:20.865 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-10-09T09:02:20.865 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-10-09T09:02:21.117 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T09:02:21.117 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T09:02:21.117 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T09:02:21.117 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-10-09T09:02:21.117 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:20 smithi181 bash[47587]: t=2023-10-09T09:02:20+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-10-09T09:02:21.117 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T09:02:21.118 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T09:02:21.118 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-10-09T09:02:21.377 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T09:02:21.377 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-10-09T09:02:21.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T09:02:21.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-10-09T09:02:21.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T09:02:21.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-10-09T09:02:21.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-10-09T09:02:21.378 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-10-09T09:02:21.638 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-10-09T09:02:21.638 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T09:02:21.638 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T09:02:21.638 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-10-09T09:02:21.638 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-10-09T09:02:21.639 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-10-09T09:02:21.639 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T09:02:21.639 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T09:02:21.901 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-10-09T09:02:21.901 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-10-09T09:02:21.901 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T09:02:21.901 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-10-09T09:02:21.901 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T09:02:21.902 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T09:02:21.902 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T09:02:21.902 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-10-09T09:02:21.902 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-10-09T09:02:22.162 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-10-09T09:02:22.162 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-10-09T09:02:22.162 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-10-09T09:02:22.162 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:21 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T09:02:22.162 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-10-09T09:02:22.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-10-09T09:02:22.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-10-09T09:02:22.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-10-09T09:02:22.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-10-09T09:02:22.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-10-09T09:02:22.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-10-09T09:02:22.414 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[27712]: cluster 2023-10-09T09:02:21.186496+0000 mgr.y (mgr.24355) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:22.414 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-10-09T09:02:22.414 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-10-09T09:02:22.414 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T09:02:22.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-10-09T09:02:22.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-10-09T09:02:22.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-10-09T09:02:22.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-10-09T09:02:22.675 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-10-09T09:02:22.675 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T09:02:22.675 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-10-09T09:02:22.676 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-10-09T09:02:22.676 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-10-09T09:02:22.676 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-10-09T09:02:22.676 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-10-09T09:02:22.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:22 smithi163 bash[27500]: cluster 2023-10-09T09:02:21.186496+0000 mgr.y (mgr.24355) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:22 smithi163 bash[33912]: cluster 2023-10-09T09:02:21.186496+0000 mgr.y (mgr.24355) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:22.944 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-10-09T09:02:22.944 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-10-09T09:02:22.944 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-10-09T09:02:22.944 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-10-09T09:02:22.944 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-10-09T09:02:22.944 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-10-09T09:02:22.945 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T09:02:22.945 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-10-09T09:02:22.945 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-10-09T09:02:22.945 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-10-09T09:02:23.213 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T09:02:23.214 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-10-09T09:02:23.214 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:22 smithi181 bash[47587]: t=2023-10-09T09:02:22+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-10-09T09:02:23.214 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-10-09T09:02:23.214 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-10-09T09:02:23.214 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-10-09T09:02:23.214 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-10-09T09:02:23.214 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T09:02:23.215 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-10-09T09:02:23.215 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-10-09T09:02:23.514 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-10-09T09:02:23.515 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-10-09T09:02:23.515 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=11.997983609s 2023-10-09T09:02:23.515 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T09:02:23.515 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T09:02:23.515 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T09:02:23.515 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T09:02:23.515 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T09:02:23.515 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T09:02:23.818 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T09:02:23.818 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T09:02:23.818 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T09:02:23.818 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T09:02:23.819 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:02:23 smithi181 bash[47587]: t=2023-10-09T09:02:23+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T09:02:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:24 smithi163 bash[27500]: cluster 2023-10-09T09:02:23.187133+0000 mgr.y (mgr.24355) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:24 smithi163 bash[33912]: cluster 2023-10-09T09:02:23.187133+0000 mgr.y (mgr.24355) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:24 smithi181 bash[27712]: cluster 2023-10-09T09:02:23.187133+0000 mgr.y (mgr.24355) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:26.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:26 smithi163 bash[27500]: cluster 2023-10-09T09:02:25.187529+0000 mgr.y (mgr.24355) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:26.551 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:26 smithi163 bash[33912]: cluster 2023-10-09T09:02:25.187529+0000 mgr.y (mgr.24355) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:26 smithi181 bash[27712]: cluster 2023-10-09T09:02:25.187529+0000 mgr.y (mgr.24355) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:28.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:28 smithi163 bash[33912]: cluster 2023-10-09T09:02:27.187861+0000 mgr.y (mgr.24355) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:28.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:28 smithi163 bash[27500]: cluster 2023-10-09T09:02:27.187861+0000 mgr.y (mgr.24355) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:28.108 INFO:teuthology.orchestra.run.smithi163.stdout:184683593760 2023-10-09T09:02:28.109 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T09:02:28.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:28 smithi181 bash[27712]: cluster 2023-10-09T09:02:27.187861+0000 mgr.y (mgr.24355) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:28.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:02:28 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:02:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:02:29.704 INFO:teuthology.orchestra.run.smithi163.stdout:34359738449 2023-10-09T09:02:29.704 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T09:02:30.156 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:29 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:02:29] "GET /metrics HTTP/1.1" 200 155670 "" "Prometheus/2.33.4" 2023-10-09T09:02:30.237 INFO:teuthology.orchestra.run.smithi163.stdout:158913789991 2023-10-09T09:02:30.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T09:02:30.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:30 smithi163 bash[33912]: cluster 2023-10-09T09:02:29.188258+0000 mgr.y (mgr.24355) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:30 smithi163 bash[27500]: cluster 2023-10-09T09:02:29.188258+0000 mgr.y (mgr.24355) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:30 smithi181 bash[27712]: cluster 2023-10-09T09:02:29.188258+0000 mgr.y (mgr.24355) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:31.918 INFO:teuthology.orchestra.run.smithi163.stdout:77309411393 2023-10-09T09:02:31.918 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T09:02:31.965 INFO:teuthology.orchestra.run.smithi163.stdout:210453397528 2023-10-09T09:02:31.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T09:02:32.444 INFO:teuthology.orchestra.run.smithi163.stdout:133143986226 2023-10-09T09:02:32.445 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T09:02:32.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:32 smithi163 bash[33912]: cluster 2023-10-09T09:02:31.188891+0000 mgr.y (mgr.24355) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:32.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:32 smithi163 bash[27500]: cluster 2023-10-09T09:02:31.188891+0000 mgr.y (mgr.24355) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:32 smithi181 bash[27712]: cluster 2023-10-09T09:02:31.188891+0000 mgr.y (mgr.24355) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:33.008 INFO:teuthology.orchestra.run.smithi163.stdout:107374182457 2023-10-09T09:02:33.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T09:02:33.115 INFO:teuthology.orchestra.run.smithi163.stdout:55834574922 2023-10-09T09:02:33.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T09:02:33.968 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:33 smithi163 systemd[1]: Stopping Ceph alertmanager.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:02:33.968 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:33 smithi163 bash[56870]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-alertmanager.a 2023-10-09T09:02:33.968 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:33 smithi163 bash[53482]: level=info ts=2023-10-09T09:02:33.843Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:02:34.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:34 smithi163 bash[27500]: cluster 2023-10-09T09:02:33.189384+0000 mgr.y (mgr.24355) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:34.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:34 smithi163 bash[33912]: cluster 2023-10-09T09:02:33.189384+0000 mgr.y (mgr.24355) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:34.305 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:34 smithi163 bash[56905]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T09:02:34.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:34 smithi181 bash[27712]: cluster 2023-10-09T09:02:33.189384+0000 mgr.y (mgr.24355) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:34.903 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:34 smithi163 bash[57006]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T09:02:34.903 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:34 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T09:02:34.903 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:34 smithi163 systemd[1]: Stopped Ceph alertmanager.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:02:34.903 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:34 smithi163 systemd[1]: Started Ceph alertmanager.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:02:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[33912]: audit 2023-10-09T09:02:34.560413+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[33912]: cephadm 2023-10-09T09:02:34.562608+0000 mgr.y (mgr.24355) 63 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:02:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[33912]: cephadm 2023-10-09T09:02:34.569871+0000 mgr.y (mgr.24355) 64 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:02:35.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[27500]: audit 2023-10-09T09:02:34.560413+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[27500]: cephadm 2023-10-09T09:02:34.562608+0000 mgr.y (mgr.24355) 63 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:02:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[27500]: cephadm 2023-10-09T09:02:34.569871+0000 mgr.y (mgr.24355) 64 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:02:35.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[57035]: level=info ts=2023-10-09T09:02:35.489Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T09:02:35.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[57035]: level=info ts=2023-10-09T09:02:35.489Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T09:02:35.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[57035]: level=info ts=2023-10-09T09:02:35.491Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.163 port=9094 2023-10-09T09:02:35.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[57035]: level=info ts=2023-10-09T09:02:35.492Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T09:02:35.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[57035]: level=info ts=2023-10-09T09:02:35.539Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:02:35.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[57035]: level=info ts=2023-10-09T09:02:35.539Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:02:35.719 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[57035]: level=info ts=2023-10-09T09:02:35.543Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T09:02:35.720 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:35 smithi163 bash[57035]: level=info ts=2023-10-09T09:02:35.543Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T09:02:35.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:35 smithi181 bash[27712]: audit 2023-10-09T09:02:34.560413+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:35.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:35 smithi181 bash[27712]: cephadm 2023-10-09T09:02:34.562608+0000 mgr.y (mgr.24355) 63 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:02:35.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:35 smithi181 bash[27712]: cephadm 2023-10-09T09:02:34.569871+0000 mgr.y (mgr.24355) 64 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:02:36.957 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:36 smithi181 bash[27712]: cluster 2023-10-09T09:02:35.189935+0000 mgr.y (mgr.24355) 65 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:36 smithi163 bash[27500]: cluster 2023-10-09T09:02:35.189935+0000 mgr.y (mgr.24355) 65 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:36.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:36 smithi163 bash[33912]: cluster 2023-10-09T09:02:35.189935+0000 mgr.y (mgr.24355) 65 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:37.968 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:37 smithi163 bash[57035]: level=info ts=2023-10-09T09:02:37.493Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000844565s 2023-10-09T09:02:37.992 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:37.992 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:37.993 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:38.005 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:38.011 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:38.011 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:38.015 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:38.015 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:02:38.306 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[27712]: cluster 2023-10-09T09:02:37.190497+0000 mgr.y (mgr.24355) 66 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:38.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:38 smithi163 bash[33912]: cluster 2023-10-09T09:02:37.190497+0000 mgr.y (mgr.24355) 66 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:38 smithi163 bash[27500]: cluster 2023-10-09T09:02:37.190497+0000 mgr.y (mgr.24355) 66 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:38.568 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 systemd[1]: Stopping Ceph prometheus.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:02:38.568 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[48171]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T09:02:38.568 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.465Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:02:38.568 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.465Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T09:02:38.568 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.465Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T09:02:38.568 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.465Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T09:02:38.569 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.465Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T09:02:38.569 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.465Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T09:02:38.569 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.466Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T09:02:38.569 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.466Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T09:02:38.569 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.466Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T09:02:38.569 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.466Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T09:02:38.569 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.467Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T09:02:38.569 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[46808]: ts=2023-10-09T09:02:38.467Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T09:02:39.312 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:38 smithi181 bash[48205]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T09:02:39.312 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:39 smithi181 bash[48265]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T09:02:39.313 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:39 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T09:02:39.313 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:39 smithi181 systemd[1]: Stopped Ceph prometheus.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:02:39.313 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:39 smithi181 systemd[1]: Started Ceph prometheus.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:02:39.570 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STOPPING 2023-10-09T09:02:39.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:02:39.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STOPPED 2023-10-09T09:02:39.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STARTING 2023-10-09T09:02:39.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Serving on http://:::9283 2023-10-09T09:02:39.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STARTED 2023-10-09T09:02:39.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STOPPING 2023-10-09T09:02:39.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:02:39.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STOPPED 2023-10-09T09:02:39.572 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STARTING 2023-10-09T09:02:39.572 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Serving on http://:::9283 2023-10-09T09:02:39.572 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STARTED 2023-10-09T09:02:39.572 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STOPPING 2023-10-09T09:02:39.572 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:02:39.956 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STOPPED 2023-10-09T09:02:39.956 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STARTING 2023-10-09T09:02:39.956 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Serving on http://:::9283 2023-10-09T09:02:39.956 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:39 smithi163 bash[27764]: [09/Oct/2023:09:02:39] ENGINE Bus STARTED 2023-10-09T09:02:40.325 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: cluster 2023-10-09T09:02:39.191229+0000 mgr.y (mgr.24355) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:40.326 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.285989+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.326 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.290349+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:02:40.326 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.291189+0000 mgr.y (mgr.24355) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:02:40.326 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.293047+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:02:40.326 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.293530+0000 mgr.y (mgr.24355) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:02:40.326 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.300361+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.327 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.308194+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:02:40.327 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.308784+0000 mgr.y (mgr.24355) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:02:40.327 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.315869+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:02:40.327 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.316337+0000 mgr.y (mgr.24355) 71 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:02:40.327 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.321843+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.327 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.332260+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:02:40.327 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.332746+0000 mgr.y (mgr.24355) 72 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:02:40.327 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.337313+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:02:40.328 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.337966+0000 mgr.y (mgr.24355) 73 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:02:40.328 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.344341+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.328 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[27712]: audit 2023-10-09T09:02:39.350012+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:02:40.328 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.319Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T09:02:40.328 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.319Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T09:02:40.328 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.319Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi181 (none))" 2023-10-09T09:02:40.328 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.319Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T09:02:40.328 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.319Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T09:02:40.328 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.321Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T09:02:40.329 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.322Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T09:02:40.329 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.323Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T09:02:40.453 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.327Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T09:02:40.453 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.327Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.898µs 2023-10-09T09:02:40.453 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.327Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T09:02:40.453 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.450Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T09:02:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: cluster 2023-10-09T09:02:39.191229+0000 mgr.y (mgr.24355) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.285989+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.290349+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:02:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.291189+0000 mgr.y (mgr.24355) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:02:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.293047+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:02:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.293530+0000 mgr.y (mgr.24355) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:02:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.300361+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.308194+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:02:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.308784+0000 mgr.y (mgr.24355) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:02:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.315869+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:02:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.316337+0000 mgr.y (mgr.24355) 71 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:02:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.321843+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.332260+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:02:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.332746+0000 mgr.y (mgr.24355) 72 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:02:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.337313+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:02:40.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.337966+0000 mgr.y (mgr.24355) 73 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:02:40.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.344341+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[33912]: audit 2023-10-09T09:02:39.350012+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:02:40.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: cluster 2023-10-09T09:02:39.191229+0000 mgr.y (mgr.24355) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:40.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.285989+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.290349+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:02:40.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.291189+0000 mgr.y (mgr.24355) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:02:40.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.293047+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:02:40.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.293530+0000 mgr.y (mgr.24355) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.300361+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.308194+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.308784+0000 mgr.y (mgr.24355) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.315869+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.316337+0000 mgr.y (mgr.24355) 71 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.321843+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.332260+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.332746+0000 mgr.y (mgr.24355) 72 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.337313+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.337966+0000 mgr.y (mgr.24355) 73 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:02:40.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.344341+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:40.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:40 smithi163 bash[27500]: audit 2023-10-09T09:02:39.350012+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:02:40.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.451Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T09:02:40.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.451Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=81.33µs wal_replay_duration=123.980769ms total_replay_duration=124.087053ms 2023-10-09T09:02:40.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.454Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T09:02:40.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.454Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T09:02:40.819 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.454Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T09:02:40.819 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.487Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=32.83511ms db_storage=1.228µs remote_storage=2.118µs web_handler=706ns query_engine=1.258µs scrape=444.696µs scrape_sd=50.599µs notify=39.403µs notify_sd=20.212µs rules=31.615786ms 2023-10-09T09:02:40.819 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:02:40 smithi181 bash[48295]: ts=2023-10-09T09:02:40.487Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T09:02:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:42 smithi181 bash[27712]: cluster 2023-10-09T09:02:41.191808+0000 mgr.y (mgr.24355) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:42.677 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:42 smithi163 bash[33912]: cluster 2023-10-09T09:02:41.191808+0000 mgr.y (mgr.24355) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:42.677 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:42 smithi163 bash[27500]: cluster 2023-10-09T09:02:41.191808+0000 mgr.y (mgr.24355) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:44.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:44 smithi181 bash[27712]: cluster 2023-10-09T09:02:43.192346+0000 mgr.y (mgr.24355) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:44 smithi163 bash[27500]: cluster 2023-10-09T09:02:43.192346+0000 mgr.y (mgr.24355) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:44 smithi163 bash[33912]: cluster 2023-10-09T09:02:43.192346+0000 mgr.y (mgr.24355) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:45.819 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:02:45 smithi163 bash[57035]: level=info ts=2023-10-09T09:02:45.496Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003762576s 2023-10-09T09:02:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:46 smithi163 bash[27500]: cluster 2023-10-09T09:02:45.192760+0000 mgr.y (mgr.24355) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:46.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:46 smithi163 bash[27500]: audit 2023-10-09T09:02:45.222991+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:46.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:46 smithi163 bash[33912]: cluster 2023-10-09T09:02:45.192760+0000 mgr.y (mgr.24355) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:46.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:46 smithi163 bash[33912]: audit 2023-10-09T09:02:45.222991+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:46 smithi181 bash[27712]: cluster 2023-10-09T09:02:45.192760+0000 mgr.y (mgr.24355) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:46 smithi181 bash[27712]: audit 2023-10-09T09:02:45.222991+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:02:48.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:48 smithi163 bash[33912]: cluster 2023-10-09T09:02:47.193252+0000 mgr.y (mgr.24355) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:48.230 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:48 smithi163 bash[27500]: cluster 2023-10-09T09:02:47.193252+0000 mgr.y (mgr.24355) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:48.291 INFO:teuthology.orchestra.run.smithi163.stdout:210453397531 2023-10-09T09:02:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:48 smithi181 bash[27712]: cluster 2023-10-09T09:02:47.193252+0000 mgr.y (mgr.24355) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:48.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:02:48 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:02:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:02:48.928 INFO:teuthology.orchestra.run.smithi163.stdout:34359738453 2023-10-09T09:02:49.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:49 smithi163 bash[27500]: audit 2023-10-09T09:02:48.291522+0000 mon.a (mon.0) 655 : audit [DBG] from='client.? 172.21.15.163:0/1893554646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:02:49.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:49 smithi163 bash[27500]: audit 2023-10-09T09:02:48.928213+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.163:0/3393445736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:02:49.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:49 smithi163 bash[33912]: audit 2023-10-09T09:02:48.291522+0000 mon.a (mon.0) 655 : audit [DBG] from='client.? 172.21.15.163:0/1893554646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:02:49.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:49 smithi163 bash[33912]: audit 2023-10-09T09:02:48.928213+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.163:0/3393445736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:02:49.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:49 smithi181 bash[27712]: audit 2023-10-09T09:02:48.291522+0000 mon.a (mon.0) 655 : audit [DBG] from='client.? 172.21.15.163:0/1893554646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:02:49.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:49 smithi181 bash[27712]: audit 2023-10-09T09:02:48.928213+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.163:0/3393445736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:02:49.986 INFO:teuthology.orchestra.run.smithi163.stdout:184683593764 2023-10-09T09:02:50.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:50 smithi163 bash[27500]: cluster 2023-10-09T09:02:49.193865+0000 mgr.y (mgr.24355) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:50.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:50 smithi163 bash[27500]: audit 2023-10-09T09:02:49.986913+0000 mon.a (mon.0) 656 : audit [DBG] from='client.? 172.21.15.163:0/652524612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:02:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:50 smithi163 bash[33912]: cluster 2023-10-09T09:02:49.193865+0000 mgr.y (mgr.24355) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:50 smithi163 bash[33912]: audit 2023-10-09T09:02:49.986913+0000 mon.a (mon.0) 656 : audit [DBG] from='client.? 172.21.15.163:0/652524612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:02:50.219 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:49 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:02:49] "GET /metrics HTTP/1.1" 200 155673 "" "Prometheus/2.33.4" 2023-10-09T09:02:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:50 smithi181 bash[27712]: cluster 2023-10-09T09:02:49.193865+0000 mgr.y (mgr.24355) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:50 smithi181 bash[27712]: audit 2023-10-09T09:02:49.986913+0000 mon.a (mon.0) 656 : audit [DBG] from='client.? 172.21.15.163:0/652524612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:02:51.449 INFO:teuthology.orchestra.run.smithi163.stdout:158913789996 2023-10-09T09:02:51.566 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738449 got 34359738453 for osd.0 2023-10-09T09:02:51.566 DEBUG:teuthology.parallel:result is None 2023-10-09T09:02:51.736 INFO:teuthology.orchestra.run.smithi163.stdout:133143986230 2023-10-09T09:02:52.320 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397528 got 210453397531 for osd.7 2023-10-09T09:02:52.320 DEBUG:teuthology.parallel:result is None 2023-10-09T09:02:52.426 INFO:teuthology.orchestra.run.smithi163.stdout:55834574926 2023-10-09T09:02:52.467 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593760 got 184683593764 for osd.6 2023-10-09T09:02:52.467 DEBUG:teuthology.parallel:result is None 2023-10-09T09:02:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:52 smithi181 bash[27712]: cluster 2023-10-09T09:02:51.194255+0000 mgr.y (mgr.24355) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:52 smithi181 bash[27712]: audit 2023-10-09T09:02:51.449370+0000 mon.a (mon.0) 657 : audit [DBG] from='client.? 172.21.15.163:0/1274079302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:02:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:52 smithi181 bash[27712]: audit 2023-10-09T09:02:51.736852+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.163:0/3541879902' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:02:52.609 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:52 smithi163 bash[33912]: cluster 2023-10-09T09:02:51.194255+0000 mgr.y (mgr.24355) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:52.609 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:52 smithi163 bash[33912]: audit 2023-10-09T09:02:51.449370+0000 mon.a (mon.0) 657 : audit [DBG] from='client.? 172.21.15.163:0/1274079302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:02:52.609 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:52 smithi163 bash[33912]: audit 2023-10-09T09:02:51.736852+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.163:0/3541879902' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:02:52.610 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:52 smithi163 bash[27500]: cluster 2023-10-09T09:02:51.194255+0000 mgr.y (mgr.24355) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:52.610 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:52 smithi163 bash[27500]: audit 2023-10-09T09:02:51.449370+0000 mon.a (mon.0) 657 : audit [DBG] from='client.? 172.21.15.163:0/1274079302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:02:52.610 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:52 smithi163 bash[27500]: audit 2023-10-09T09:02:51.736852+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.163:0/3541879902' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:02:53.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:53 smithi163 bash[27500]: audit 2023-10-09T09:02:52.427038+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.163:0/4135397620' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:02:53.555 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:53 smithi163 bash[33912]: audit 2023-10-09T09:02:52.427038+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.163:0/4135397620' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:02:53.556 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986226 got 133143986230 for osd.4 2023-10-09T09:02:53.556 DEBUG:teuthology.parallel:result is None 2023-10-09T09:02:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:53 smithi181 bash[27712]: audit 2023-10-09T09:02:52.427038+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.163:0/4135397620' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:02:53.688 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789991 got 158913789996 for osd.5 2023-10-09T09:02:53.688 DEBUG:teuthology.parallel:result is None 2023-10-09T09:02:54.298 INFO:teuthology.orchestra.run.smithi163.stdout:77309411398 2023-10-09T09:02:54.334 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574922 got 55834574926 for osd.1 2023-10-09T09:02:54.334 DEBUG:teuthology.parallel:result is None 2023-10-09T09:02:54.425 INFO:teuthology.orchestra.run.smithi163.stdout:107374182462 2023-10-09T09:02:54.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:54 smithi163 bash[33912]: cluster 2023-10-09T09:02:53.194923+0000 mgr.y (mgr.24355) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:54.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:54 smithi163 bash[27500]: cluster 2023-10-09T09:02:53.194923+0000 mgr.y (mgr.24355) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:54 smithi181 bash[27712]: cluster 2023-10-09T09:02:53.194923+0000 mgr.y (mgr.24355) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:55 smithi163 bash[27500]: audit 2023-10-09T09:02:54.298103+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.163:0/17132352' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:02:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:55 smithi163 bash[27500]: audit 2023-10-09T09:02:54.424803+0000 mon.a (mon.0) 658 : audit [DBG] from='client.? 172.21.15.163:0/3438827568' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:02:55.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:55 smithi163 bash[33912]: audit 2023-10-09T09:02:54.298103+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.163:0/17132352' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:02:55.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:55 smithi163 bash[33912]: audit 2023-10-09T09:02:54.424803+0000 mon.a (mon.0) 658 : audit [DBG] from='client.? 172.21.15.163:0/3438827568' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:02:55.465 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411393 got 77309411398 for osd.2 2023-10-09T09:02:55.465 DEBUG:teuthology.parallel:result is None 2023-10-09T09:02:55.528 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182457 got 107374182462 for osd.3 2023-10-09T09:02:55.529 DEBUG:teuthology.parallel:result is None 2023-10-09T09:02:55.529 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T09:02:55.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:02:55.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:55 smithi181 bash[27712]: audit 2023-10-09T09:02:54.298103+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.163:0/17132352' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:02:55.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:55 smithi181 bash[27712]: audit 2023-10-09T09:02:54.424803+0000 mon.a (mon.0) 658 : audit [DBG] from='client.? 172.21.15.163:0/3438827568' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:02:56.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:56 smithi181 bash[27712]: cluster 2023-10-09T09:02:55.195404+0000 mgr.y (mgr.24355) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:56 smithi163 bash[27500]: cluster 2023-10-09T09:02:55.195404+0000 mgr.y (mgr.24355) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:56 smithi163 bash[33912]: cluster 2023-10-09T09:02:55.195404+0000 mgr.y (mgr.24355) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:58.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:58 smithi181 bash[27712]: cluster 2023-10-09T09:02:57.196038+0000 mgr.y (mgr.24355) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:58 smithi163 bash[33912]: cluster 2023-10-09T09:02:57.196038+0000 mgr.y (mgr.24355) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:58.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:58 smithi163 bash[27500]: cluster 2023-10-09T09:02:57.196038+0000 mgr.y (mgr.24355) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:02:58.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:02:58 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:02:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:03:00.070 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:00.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:02:59] "GET /metrics HTTP/1.1" 200 155671 "" "Prometheus/2.33.4" 2023-10-09T09:03:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[27500]: audit 2023-10-09T09:02:58.965451+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[27500]: audit 2023-10-09T09:02:58.967059+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[27500]: audit 2023-10-09T09:02:58.968392+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[27500]: audit 2023-10-09T09:02:58.975088+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[27500]: audit 2023-10-09T09:02:58.981869+0000 mon.a (mon.0) 663 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:00.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[27500]: cluster 2023-10-09T09:02:59.196718+0000 mgr.y (mgr.24355) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:00.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[33912]: audit 2023-10-09T09:02:58.965451+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:00.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[33912]: audit 2023-10-09T09:02:58.967059+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:00.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[33912]: audit 2023-10-09T09:02:58.968392+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:00.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[33912]: audit 2023-10-09T09:02:58.975088+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:00.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[33912]: audit 2023-10-09T09:02:58.981869+0000 mon.a (mon.0) 663 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:00.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:02:59 smithi163 bash[33912]: cluster 2023-10-09T09:02:59.196718+0000 mgr.y (mgr.24355) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:59 smithi181 bash[27712]: audit 2023-10-09T09:02:58.965451+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:59 smithi181 bash[27712]: audit 2023-10-09T09:02:58.967059+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:59 smithi181 bash[27712]: audit 2023-10-09T09:02:58.968392+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:59 smithi181 bash[27712]: audit 2023-10-09T09:02:58.975088+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:00.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:59 smithi181 bash[27712]: audit 2023-10-09T09:02:58.981869+0000 mon.a (mon.0) 663 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:00.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:02:59 smithi181 bash[27712]: cluster 2023-10-09T09:02:59.196718+0000 mgr.y (mgr.24355) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:02 smithi181 bash[27712]: cluster 2023-10-09T09:03:01.197283+0000 mgr.y (mgr.24355) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:02.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:02 smithi163 bash[33912]: cluster 2023-10-09T09:03:01.197283+0000 mgr.y (mgr.24355) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:02.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:02 smithi163 bash[27500]: cluster 2023-10-09T09:03:01.197283+0000 mgr.y (mgr.24355) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:02.790 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:03:02.794 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:03:03.513 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":64,"stamp":"2023-10-09T09:03:01.197008+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381036,"kb_used_data":4268,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580180,"statfs":{"total":767960285184,"available":765522104320,"internally_reserved":0,"allocated":4370432,"data_stored":1793617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:38.395197+0000","last_change":"2023-10-09T09:01:34.369352+0000","last_active":"2023-10-09T09:01:38.395197+0000","last_peered":"2023-10-09T09:01:38.395197+0000","last_clean":"2023-10-09T09:01:38.395197+0000","last_became_active":"2023-10-09T09:01:34.368090+0000","last_became_peered":"2023-10-09T09:01:34.368090+0000","last_unstale":"2023-10-09T09:01:38.395197+0000","last_undegraded":"2023-10-09T09:01:38.395197+0000","last_fullsized":"2023-10-09T09:01:38.395197+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:36.512353+0000","last_change":"2023-10-09T09:01:34.370339+0000","last_active":"2023-10-09T09:01:36.512353+0000","last_peered":"2023-10-09T09:01:36.512353+0000","last_clean":"2023-10-09T09:01:36.512353+0000","last_became_active":"2023-10-09T09:01:34.369975+0000","last_became_peered":"2023-10-09T09:01:34.369975+0000","last_unstale":"2023-10-09T09:01:36.512353+0000","last_undegraded":"2023-10-09T09:01:36.512353+0000","last_fullsized":"2023-10-09T09:01:36.512353+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:37.894409+0000","last_change":"2023-10-09T09:01:34.370949+0000","last_active":"2023-10-09T09:01:37.894409+0000","last_peered":"2023-10-09T09:01:37.894409+0000","last_clean":"2023-10-09T09:01:37.894409+0000","last_became_active":"2023-10-09T09:01:34.370529+0000","last_became_peered":"2023-10-09T09:01:34.370529+0000","last_unstale":"2023-10-09T09:01:37.894409+0000","last_undegraded":"2023-10-09T09:01:37.894409+0000","last_fullsized":"2023-10-09T09:01:37.894409+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:37.894282+0000","last_change":"2023-10-09T09:01:34.369805+0000","last_active":"2023-10-09T09:01:37.894282+0000","last_peered":"2023-10-09T09:01:37.894282+0000","last_clean":"2023-10-09T09:01:37.894282+0000","last_became_active":"2023-10-09T09:01:34.369226+0000","last_became_peered":"2023-10-09T09:01:34.369226+0000","last_unstale":"2023-10-09T09:01:37.894282+0000","last_undegraded":"2023-10-09T09:01:37.894282+0000","last_fullsized":"2023-10-09T09:01:37.894282+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:37.894290+0000","last_change":"2023-10-09T09:00:37.999371+0000","last_active":"2023-10-09T09:01:37.894290+0000","last_peered":"2023-10-09T09:01:37.894290+0000","last_clean":"2023-10-09T09:01:37.894290+0000","last_became_active":"2023-10-09T09:00:37.998968+0000","last_became_peered":"2023-10-09T09:00:37.998968+0000","last_unstale":"2023-10-09T09:01:37.894290+0000","last_undegraded":"2023-10-09T09:01:37.894290+0000","last_fullsized":"2023-10-09T09:01:37.894290+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:37.270506+0000","last_change":"2023-10-09T09:01:34.371076+0000","last_active":"2023-10-09T09:01:37.270506+0000","last_peered":"2023-10-09T09:01:37.270506+0000","last_clean":"2023-10-09T09:01:37.270506+0000","last_became_active":"2023-10-09T09:01:34.370690+0000","last_became_peered":"2023-10-09T09:01:34.370690+0000","last_unstale":"2023-10-09T09:01:37.270506+0000","last_undegraded":"2023-10-09T09:01:37.270506+0000","last_fullsized":"2023-10-09T09:01:37.270506+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:35.371803+0000","last_change":"2023-10-09T09:01:34.897330+0000","last_active":"2023-10-09T09:01:35.371803+0000","last_peered":"2023-10-09T09:01:35.371803+0000","last_clean":"2023-10-09T09:01:35.371803+0000","last_became_active":"2023-10-09T09:01:34.896715+0000","last_became_peered":"2023-10-09T09:01:34.896715+0000","last_unstale":"2023-10-09T09:01:35.371803+0000","last_undegraded":"2023-10-09T09:01:35.371803+0000","last_fullsized":"2023-10-09T09:01:35.371803+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:37.270502+0000","last_change":"2023-10-09T09:01:34.372320+0000","last_active":"2023-10-09T09:01:37.270502+0000","last_peered":"2023-10-09T09:01:37.270502+0000","last_clean":"2023-10-09T09:01:37.270502+0000","last_became_active":"2023-10-09T09:01:34.371852+0000","last_became_peered":"2023-10-09T09:01:34.371852+0000","last_unstale":"2023-10-09T09:01:37.270502+0000","last_undegraded":"2023-10-09T09:01:37.270502+0000","last_fullsized":"2023-10-09T09:01:37.270502+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:36.512157+0000","last_change":"2023-10-09T09:01:34.370258+0000","last_active":"2023-10-09T09:01:36.512157+0000","last_peered":"2023-10-09T09:01:36.512157+0000","last_clean":"2023-10-09T09:01:36.512157+0000","last_became_active":"2023-10-09T09:01:34.369813+0000","last_became_peered":"2023-10-09T09:01:34.369813+0000","last_unstale":"2023-10-09T09:01:36.512157+0000","last_undegraded":"2023-10-09T09:01:36.512157+0000","last_fullsized":"2023-10-09T09:01:36.512157+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397534,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297628,"kb_used_data":532,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447524,"statfs":{"total":95995035648,"available":95690264576,"internally_reserved":0,"allocated":544768,"data_stored":224084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":0.92600000000000005,"15min":0.94499999999999995},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.1140000000000001,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.0549999999999999},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.90300000000000002,"15min":0.89700000000000002},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.86199999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.88,"15min":0.876},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.88,"5min":0.871,"15min":0.86899999999999999},"min":{"1min":0.68899999999999995,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.0840000000000001,"5min":1.155,"15min":1.155},"last":1.105}]},{"osd":2,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.91800000000000004,"15min":0.93300000000000005},"min":{"1min":0.53500000000000003,"5min":0.497,"15min":0.497},"max":{"1min":1.1399999999999999,"5min":1.484,"15min":1.484},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.92000000000000004,"15min":0.92800000000000005},"min":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"max":{"1min":1.1639999999999999,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.95799999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.94399999999999995,"15min":0.94499999999999995},"min":{"1min":0.67500000000000004,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.1779999999999999,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.98899999999999999,"15min":0.99299999999999999},"min":{"1min":0.66100000000000003,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.2629999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.1619999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.88,"15min":0.878},"min":{"1min":0.67300000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.1000000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.98699999999999999,"15min":0.98999999999999999},"min":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.288,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":1.1399999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.57199999999999995,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2909999999999999,"5min":1.403,"15min":1.403},"last":1.085},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1,"15min":1.004},"min":{"1min":0.58199999999999996,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.216,"5min":1.712,"15min":1.712},"last":0.80200000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.016,"15min":1.018},"min":{"1min":0.73899999999999999,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.244,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.01},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.0609999999999999,"15min":1.0669999999999999},"min":{"1min":0.65300000000000002,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.292,"5min":1.73,"15min":1.73},"last":0.98899999999999999}]}]},{"osd":6,"up_from":43,"seq":184683593767,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297628,"kb_used_data":532,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447524,"statfs":{"total":95995035648,"available":95690264576,"internally_reserved":0,"allocated":544768,"data_stored":224084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:01:58 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.92200000000000004,"15min":0.92600000000000005},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.83899999999999997,"15min":0.83499999999999996},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.89500000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:01:58 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.90500000000000003,"15min":0.90700000000000003},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.92300000000000004,"15min":0.93300000000000005},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.365,"5min":1.365,"15min":1.365},"last":0.79500000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:01:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.878,"15min":0.874},"min":{"1min":0.64300000000000002,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":0.97799999999999998},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.95099999999999996,"15min":0.95599999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.71999999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:01:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.89900000000000002,"15min":0.89800000000000002},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2010000000000001,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.93500000000000005,"15min":0.93700000000000006},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.054}]},{"osd":4,"last update":"Mon Oct 9 09:01:58 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.84699999999999998,"15min":0.84399999999999997},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.337,"5min":1.337,"15min":1.337},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.84199999999999997,"15min":0.82999999999999996},"min":{"1min":0.626,"5min":0.371,"15min":0.371},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.76200000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:01:58 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.92900000000000005,"15min":0.92300000000000004},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":1.0860000000000001},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.90700000000000003,"15min":0.90400000000000003},"min":{"1min":0.69099999999999995,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":1.109}]},{"osd":7,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.0349999999999999,"15min":1.0429999999999999},"min":{"1min":0.78600000000000003,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.1970000000000001,"5min":1.331,"15min":1.331},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.022,"5min":1.0549999999999999,"15min":1.0609999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.216,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.151}]}]},{"osd":1,"up_from":13,"seq":55834574928,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":223769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.67000000000000004,"15min":0.56399999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.37},"max":{"1min":1.069,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.73899999999999999,"15min":0.61299999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.33300000000000002},"max":{"1min":1.1000000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.68100000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.75600000000000001,"15min":0.66700000000000004},"min":{"1min":0.48299999999999998,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.71699999999999997,"15min":0.67100000000000004},"min":{"1min":0.439,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.022,"5min":1.125,"15min":1.125},"last":0.58599999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.75,"15min":0.71399999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.85699999999999998,"15min":0.82299999999999995},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.1419999999999999,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.96999999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.875,"15min":0.873},"min":{"1min":0.64900000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.2310000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":0.84099999999999997},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.90500000000000003,"15min":0.89500000000000002},"min":{"1min":0.65900000000000003,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.302,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.72699999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.024,"15min":1.032},"min":{"1min":0.78000000000000003,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.288,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.0109999999999999,"15min":1.0329999999999999},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.357,"5min":1.357,"15min":1.357},"last":1.0049999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:02:57 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.095,"15min":1.159},"min":{"1min":0.58599999999999997,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.262,"5min":1.585,"15min":1.585},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.1399999999999999,"15min":1.194},"min":{"1min":0.72299999999999998,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.5489999999999999,"5min":1.609,"15min":1.609},"last":1.093}]},{"osd":7,"last update":"Mon Oct 9 09:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.0760000000000001,"15min":1.0880000000000001},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.319,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0720000000000001,"15min":1.0760000000000001},"min":{"1min":0.78200000000000003,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.369,"5min":1.5289999999999999,"15min":1.5289999999999999},"last":0.93799999999999994}]}]},{"osd":0,"up_from":8,"seq":34359738456,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297632,"kb_used_data":536,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447520,"statfs":{"total":95995035648,"available":95690260480,"internally_reserved":0,"allocated":548864,"data_stored":224399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:02:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.68799999999999994,"15min":0.56899999999999995},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.69299999999999995,"15min":0.56399999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.35499999999999998},"max":{"1min":1.03,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.64700000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:02:20 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.71099999999999997,"15min":0.64100000000000001},"min":{"1min":0.435,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.1379999999999999,"5min":1.276,"15min":1.276},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.77700000000000002,"15min":0.67600000000000005},"min":{"1min":0.39600000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.75800000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.82899999999999996,"15min":0.81499999999999995},"min":{"1min":0.58499999999999996,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.365,"5min":1.365,"15min":1.365},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.81000000000000005,"15min":0.77400000000000002},"min":{"1min":0.61699999999999999,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.77000000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:02:44 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.91100000000000003,"15min":0.91900000000000004},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.0740000000000001,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.92500000000000004,"15min":0.91400000000000003},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.3049999999999999,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.78600000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:02:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.97499999999999998,"15min":0.97699999999999998},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.304,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.95199999999999996},{"interface":"front","average":{"1min":1.038,"5min":1.0369999999999999,"15min":1.0389999999999999},"min":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"max":{"1min":1.554,"5min":1.554,"15min":1.554},"last":0.85999999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:01:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.111,"15min":1.107},"min":{"1min":0.876,"5min":0.73299999999999998,"15min":0.73299999999999998},"max":{"1min":1.3140000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.097,"15min":1.097},"min":{"1min":0.90000000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"max":{"1min":1.3600000000000001,"5min":1.395,"15min":1.395},"last":0.88400000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:02:44 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.0329999999999999,"15min":1.0449999999999999},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.399,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.98299999999999998},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.115,"15min":1.125},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.716,"5min":1.716,"15min":1.716},"last":0.89600000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411399,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297632,"kb_used_data":536,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447520,"statfs":{"total":95995035648,"available":95690260480,"internally_reserved":0,"allocated":548864,"data_stored":224399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.67200000000000004,"15min":0.622},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.85399999999999998,"5min":1.04,"15min":1.04},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.71299999999999997,"15min":0.66300000000000003},"min":{"1min":0.44800000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.98699999999999999,"5min":1.29,"15min":1.29},"last":0.59599999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:02:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.68500000000000005,"15min":0.64100000000000001},"min":{"1min":0.39200000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.753,"15min":0.69299999999999995},"min":{"1min":0.40999999999999998,"5min":0.372,"15min":0.372},"max":{"1min":1.034,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.93100000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:02:05 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.80800000000000005,"15min":0.77700000000000002},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.78300000000000003,"15min":0.76200000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.1859999999999999,"5min":1.2,"15min":1.2},"last":0.68000000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.96299999999999997,"15min":0.94299999999999995},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.171,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.95399999999999996,"15min":0.94299999999999995},"min":{"1min":0.72299999999999998,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.2589999999999999,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.0489999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:02:21 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.95899999999999996,"15min":0.95799999999999996},"min":{"1min":0.63,"5min":0.63,"15min":0.63},"max":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.95699999999999996,"15min":0.95099999999999996},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.415,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.1919999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:01:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.131,"15min":1.149},"min":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"max":{"1min":1.244,"5min":1.488,"15min":1.488},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.081,"15min":1.093},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.4259999999999999,"5min":1.464,"15min":1.464},"last":1.115}]},{"osd":7,"last update":"Mon Oct 9 09:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.012,"15min":1.02},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.2569999999999999},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.1200000000000001,"15min":1.1379999999999999},"min":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.3480000000000001,"5min":1.423,"15min":1.423},"last":1.1040000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182463,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297632,"kb_used_data":536,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447520,"statfs":{"total":95995035648,"available":95690260480,"internally_reserved":0,"allocated":548864,"data_stored":224399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.67000000000000004,"15min":0.66200000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.85299999999999998,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.76000000000000001,"15min":0.75},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.0189999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.81100000000000005}]},{"osd":1,"last update":"Mon Oct 9 09:02:02 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.751,"15min":0.73599999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.024,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.75600000000000001,"15min":0.76600000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.749}]},{"osd":2,"last update":"Mon Oct 9 09:02:02 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.83899999999999997,"15min":0.84699999999999998},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.097,"5min":1.7829999999999999,"15min":1.7829999999999999},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.80900000000000005,"15min":0.80500000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.03,"5min":1.24,"15min":1.24},"last":0.88900000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.93999999999999995,"15min":0.95899999999999996},"min":{"1min":0.64800000000000002,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.1399999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.91500000000000004,"15min":0.91500000000000004},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.216,"5min":1.4099999999999999,"15min":1.4099999999999999},"last":1.052}]},{"osd":5,"last update":"Mon Oct 9 09:02:14 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.99199999999999999,"15min":0.99299999999999999},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.244,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.2889999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.01,"15min":1.004},"min":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"max":{"1min":1.3260000000000001,"5min":1.421,"15min":1.421},"last":1.024}]},{"osd":6,"last update":"Mon Oct 9 09:02:58 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.0589999999999999,"15min":1.105},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.262,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":1.2030000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.0640000000000001,"15min":1.0980000000000001},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.2569999999999999,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":1.2569999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.0720000000000001,"15min":1.099},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.1990000000000001,"5min":1.5309999999999999,"15min":1.5309999999999999},"last":1.3420000000000001},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.0980000000000001,"15min":1.1299999999999999},"min":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.3009999999999999,"5min":1.518,"15min":1.518},"last":1.2749999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986232,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297632,"kb_used_data":536,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447520,"statfs":{"total":95995035648,"available":95690260480,"internally_reserved":0,"allocated":548864,"data_stored":224399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.83499999999999996,"15min":0.82899999999999996},"min":{"1min":0.61399999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.3440000000000001,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.84399999999999997,"15min":0.84999999999999998},"min":{"1min":0.628,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.1599999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.81999999999999995,"15min":0.80900000000000005},"min":{"1min":0.59899999999999998,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.0860000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.873,"15min":0.871},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.079,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.216}]},{"osd":2,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.85599999999999998,"15min":0.83599999999999997},"min":{"1min":0.67900000000000005,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":1.0529999999999999},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.873,"15min":0.86399999999999999},"min":{"1min":0.64600000000000002,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1339999999999999,"5min":1.224,"15min":1.224},"last":0.92600000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.88,"15min":0.873},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.175,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":1.022},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.92200000000000004,"15min":0.91200000000000003},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.0549999999999999,"5min":2.028,"15min":2.028},"last":0.90300000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":0.84499999999999997,"15min":0.82099999999999995},"min":{"1min":0.504,"5min":0.441,"15min":0.441},"max":{"1min":1.1439999999999999,"5min":1.236,"15min":1.236},"last":0.999},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.93200000000000005,"15min":0.92200000000000004},"min":{"1min":0.57799999999999996,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.1879999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.137}]},{"osd":6,"last update":"Mon Oct 9 09:01:57 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.95599999999999996,"15min":0.97999999999999998},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.222,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.745},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.0740000000000001,"15min":1.0920000000000001},"min":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.4139999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.1870000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.88300000000000001,"15min":0.875},"min":{"1min":0.56299999999999994,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.343,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.002,"5min":0.93200000000000005,"15min":0.92000000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.95399999999999996}]}]},{"osd":5,"up_from":37,"seq":158913789998,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297628,"kb_used_data":532,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447524,"statfs":{"total":95995035648,"available":95690264576,"internally_reserved":0,"allocated":544768,"data_stored":224084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.84299999999999997,"15min":0.82599999999999996},"min":{"1min":0.61499999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":2.052,"5min":2.052,"15min":2.052},"last":0.748},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.84799999999999998,"15min":0.83199999999999996},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.113,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.93700000000000006}]},{"osd":1,"last update":"Mon Oct 9 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.86199999999999999,"15min":0.84799999999999998},"min":{"1min":0.59699999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":2.214,"5min":2.214,"15min":2.214},"last":1.004},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.83299999999999996,"15min":0.81799999999999995},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.2609999999999999,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":1.024}]},{"osd":2,"last update":"Mon Oct 9 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.86599999999999999,"15min":0.85199999999999998},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.2809999999999999,"5min":1.323,"15min":1.323},"last":1.1499999999999999},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.90600000000000003,"15min":0.90000000000000002},"min":{"1min":0.76000000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":2.1200000000000001,"5min":2.1200000000000001,"15min":2.1200000000000001},"last":1.103}]},{"osd":3,"last update":"Mon Oct 9 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.90800000000000003,"15min":0.90000000000000002},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":1.214},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.90700000000000003,"15min":0.89100000000000001},"min":{"1min":0.61899999999999999,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":2.1669999999999998,"5min":2.1669999999999998,"15min":2.1669999999999998},"last":1.1699999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.72799999999999998,"15min":0.70399999999999996},"min":{"1min":0.53100000000000003,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.2929999999999999,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.76100000000000001,"15min":0.72399999999999998},"min":{"1min":0.53700000000000003,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.97099999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:01:59 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.86699999999999999,"15min":0.86299999999999999},"min":{"1min":0.57599999999999996,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.88800000000000001},{"interface":"front","average":{"1min":1.014,"5min":1.0489999999999999,"15min":1.0549999999999999},"min":{"1min":0.61899999999999999,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":1.2589999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.93400000000000005,"15min":0.92500000000000004},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.2569999999999999,"5min":1.319,"15min":1.319},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.0409999999999999,"5min":0.98399999999999999,"15min":0.97399999999999998},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":2.4809999999999999,"5min":2.4809999999999999,"15min":2.4809999999999999},"last":0.91800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:03:03.516 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:03:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:04 smithi181 bash[27712]: audit 2023-10-09T09:03:02.782649+0000 mgr.y (mgr.24355) 85 : audit [DBG] from='client.24536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:03:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:04 smithi181 bash[27712]: cluster 2023-10-09T09:03:03.197967+0000 mgr.y (mgr.24355) 86 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:04 smithi163 bash[27500]: audit 2023-10-09T09:03:02.782649+0000 mgr.y (mgr.24355) 85 : audit [DBG] from='client.24536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:03:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:04 smithi163 bash[27500]: cluster 2023-10-09T09:03:03.197967+0000 mgr.y (mgr.24355) 86 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:04.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:04 smithi163 bash[33912]: audit 2023-10-09T09:03:02.782649+0000 mgr.y (mgr.24355) 85 : audit [DBG] from='client.24536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:03:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:04 smithi163 bash[33912]: cluster 2023-10-09T09:03:03.197967+0000 mgr.y (mgr.24355) 86 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:06.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:05 smithi163 bash[33912]: audit 2023-10-09T09:03:04.872135+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:06.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:05 smithi163 bash[33912]: audit 2023-10-09T09:03:04.873890+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:06.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:05 smithi163 bash[33912]: audit 2023-10-09T09:03:04.875308+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:06.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:05 smithi163 bash[33912]: audit 2023-10-09T09:03:04.883573+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:05 smithi163 bash[33912]: cluster 2023-10-09T09:03:05.198630+0000 mgr.y (mgr.24355) 87 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:05 smithi163 bash[27500]: audit 2023-10-09T09:03:04.872135+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:05 smithi163 bash[27500]: audit 2023-10-09T09:03:04.873890+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:05 smithi163 bash[27500]: audit 2023-10-09T09:03:04.875308+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:05 smithi163 bash[27500]: audit 2023-10-09T09:03:04.883573+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:05 smithi163 bash[27500]: cluster 2023-10-09T09:03:05.198630+0000 mgr.y (mgr.24355) 87 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:05 smithi181 bash[27712]: audit 2023-10-09T09:03:04.872135+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:05 smithi181 bash[27712]: audit 2023-10-09T09:03:04.873890+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:05 smithi181 bash[27712]: audit 2023-10-09T09:03:04.875308+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:05 smithi181 bash[27712]: audit 2023-10-09T09:03:04.883573+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:05 smithi181 bash[27712]: cluster 2023-10-09T09:03:05.198630+0000 mgr.y (mgr.24355) 87 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:08.287 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:08.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:08 smithi163 bash[27500]: cluster 2023-10-09T09:03:07.199195+0000 mgr.y (mgr.24355) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:08.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:08 smithi163 bash[33912]: cluster 2023-10-09T09:03:07.199195+0000 mgr.y (mgr.24355) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:08 smithi181 bash[27712]: cluster 2023-10-09T09:03:07.199195+0000 mgr.y (mgr.24355) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:08.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:03:08 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:03:10.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:03:09 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:09] "GET /metrics HTTP/1.1" 200 155665 "" "Prometheus/2.33.4" 2023-10-09T09:03:10.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:10 smithi181 bash[27712]: cluster 2023-10-09T09:03:09.199697+0000 mgr.y (mgr.24355) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:10.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:10 smithi163 bash[33912]: cluster 2023-10-09T09:03:09.199697+0000 mgr.y (mgr.24355) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:10.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:10 smithi163 bash[27500]: cluster 2023-10-09T09:03:09.199697+0000 mgr.y (mgr.24355) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:03:11.012 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:03:11.681 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":68,"stamp":"2023-10-09T09:03:09.199443+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381036,"kb_used_data":4268,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580180,"statfs":{"total":767960285184,"available":765522104320,"internally_reserved":0,"allocated":4370432,"data_stored":1793617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:38.395197+0000","last_change":"2023-10-09T09:01:34.369352+0000","last_active":"2023-10-09T09:01:38.395197+0000","last_peered":"2023-10-09T09:01:38.395197+0000","last_clean":"2023-10-09T09:01:38.395197+0000","last_became_active":"2023-10-09T09:01:34.368090+0000","last_became_peered":"2023-10-09T09:01:34.368090+0000","last_unstale":"2023-10-09T09:01:38.395197+0000","last_undegraded":"2023-10-09T09:01:38.395197+0000","last_fullsized":"2023-10-09T09:01:38.395197+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:36.512353+0000","last_change":"2023-10-09T09:01:34.370339+0000","last_active":"2023-10-09T09:01:36.512353+0000","last_peered":"2023-10-09T09:01:36.512353+0000","last_clean":"2023-10-09T09:01:36.512353+0000","last_became_active":"2023-10-09T09:01:34.369975+0000","last_became_peered":"2023-10-09T09:01:34.369975+0000","last_unstale":"2023-10-09T09:01:36.512353+0000","last_undegraded":"2023-10-09T09:01:36.512353+0000","last_fullsized":"2023-10-09T09:01:36.512353+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:37.894409+0000","last_change":"2023-10-09T09:01:34.370949+0000","last_active":"2023-10-09T09:01:37.894409+0000","last_peered":"2023-10-09T09:01:37.894409+0000","last_clean":"2023-10-09T09:01:37.894409+0000","last_became_active":"2023-10-09T09:01:34.370529+0000","last_became_peered":"2023-10-09T09:01:34.370529+0000","last_unstale":"2023-10-09T09:01:37.894409+0000","last_undegraded":"2023-10-09T09:01:37.894409+0000","last_fullsized":"2023-10-09T09:01:37.894409+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:37.894282+0000","last_change":"2023-10-09T09:01:34.369805+0000","last_active":"2023-10-09T09:01:37.894282+0000","last_peered":"2023-10-09T09:01:37.894282+0000","last_clean":"2023-10-09T09:01:37.894282+0000","last_became_active":"2023-10-09T09:01:34.369226+0000","last_became_peered":"2023-10-09T09:01:34.369226+0000","last_unstale":"2023-10-09T09:01:37.894282+0000","last_undegraded":"2023-10-09T09:01:37.894282+0000","last_fullsized":"2023-10-09T09:01:37.894282+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:37.894290+0000","last_change":"2023-10-09T09:00:37.999371+0000","last_active":"2023-10-09T09:01:37.894290+0000","last_peered":"2023-10-09T09:01:37.894290+0000","last_clean":"2023-10-09T09:01:37.894290+0000","last_became_active":"2023-10-09T09:00:37.998968+0000","last_became_peered":"2023-10-09T09:00:37.998968+0000","last_unstale":"2023-10-09T09:01:37.894290+0000","last_undegraded":"2023-10-09T09:01:37.894290+0000","last_fullsized":"2023-10-09T09:01:37.894290+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:37.270506+0000","last_change":"2023-10-09T09:01:34.371076+0000","last_active":"2023-10-09T09:01:37.270506+0000","last_peered":"2023-10-09T09:01:37.270506+0000","last_clean":"2023-10-09T09:01:37.270506+0000","last_became_active":"2023-10-09T09:01:34.370690+0000","last_became_peered":"2023-10-09T09:01:34.370690+0000","last_unstale":"2023-10-09T09:01:37.270506+0000","last_undegraded":"2023-10-09T09:01:37.270506+0000","last_fullsized":"2023-10-09T09:01:37.270506+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:35.371803+0000","last_change":"2023-10-09T09:01:34.897330+0000","last_active":"2023-10-09T09:01:35.371803+0000","last_peered":"2023-10-09T09:01:35.371803+0000","last_clean":"2023-10-09T09:01:35.371803+0000","last_became_active":"2023-10-09T09:01:34.896715+0000","last_became_peered":"2023-10-09T09:01:34.896715+0000","last_unstale":"2023-10-09T09:01:35.371803+0000","last_undegraded":"2023-10-09T09:01:35.371803+0000","last_fullsized":"2023-10-09T09:01:35.371803+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:37.270502+0000","last_change":"2023-10-09T09:01:34.372320+0000","last_active":"2023-10-09T09:01:37.270502+0000","last_peered":"2023-10-09T09:01:37.270502+0000","last_clean":"2023-10-09T09:01:37.270502+0000","last_became_active":"2023-10-09T09:01:34.371852+0000","last_became_peered":"2023-10-09T09:01:34.371852+0000","last_unstale":"2023-10-09T09:01:37.270502+0000","last_undegraded":"2023-10-09T09:01:37.270502+0000","last_fullsized":"2023-10-09T09:01:37.270502+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2023-10-09T09:01:36.512157+0000","last_change":"2023-10-09T09:01:34.370258+0000","last_active":"2023-10-09T09:01:36.512157+0000","last_peered":"2023-10-09T09:01:36.512157+0000","last_clean":"2023-10-09T09:01:36.512157+0000","last_became_active":"2023-10-09T09:01:34.369813+0000","last_became_peered":"2023-10-09T09:01:34.369813+0000","last_unstale":"2023-10-09T09:01:36.512157+0000","last_undegraded":"2023-10-09T09:01:36.512157+0000","last_fullsized":"2023-10-09T09:01:36.512157+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397536,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297628,"kb_used_data":532,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447524,"statfs":{"total":95995035648,"available":95690264576,"internally_reserved":0,"allocated":544768,"data_stored":224084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":0.92600000000000005,"15min":0.94499999999999995},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.1140000000000001,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.0760000000000001},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.90300000000000002,"15min":0.89700000000000002},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.73499999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.88,"15min":0.876},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.204,"5min":1.204,"15min":1.204},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.88,"5min":0.871,"15min":0.86899999999999999},"min":{"1min":0.68899999999999995,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.0840000000000001,"5min":1.155,"15min":1.155},"last":0.95999999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.91800000000000004,"15min":0.93300000000000005},"min":{"1min":0.53500000000000003,"5min":0.497,"15min":0.497},"max":{"1min":1.1399999999999999,"5min":1.484,"15min":1.484},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.92000000000000004,"15min":0.92800000000000005},"min":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"max":{"1min":1.1639999999999999,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":1.0109999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.94399999999999995,"15min":0.94499999999999995},"min":{"1min":0.67500000000000004,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.1779999999999999,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.98899999999999999,"15min":0.99299999999999999},"min":{"1min":0.66100000000000003,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.2629999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.0609999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.88,"15min":0.878},"min":{"1min":0.67300000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.1000000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.98699999999999999,"15min":0.98999999999999999},"min":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.288,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":0.97999999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.57199999999999995,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2909999999999999,"5min":1.403,"15min":1.403},"last":1.3740000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1,"15min":1.004},"min":{"1min":0.58199999999999996,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.216,"5min":1.712,"15min":1.712},"last":1.034}]},{"osd":6,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.016,"15min":1.018},"min":{"1min":0.73899999999999999,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.244,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.1379999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.0609999999999999,"15min":1.0669999999999999},"min":{"1min":0.65300000000000002,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.292,"5min":1.73,"15min":1.73},"last":1.165}]}]},{"osd":6,"up_from":43,"seq":184683593769,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297628,"kb_used_data":532,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447524,"statfs":{"total":95995035648,"available":95690264576,"internally_reserved":0,"allocated":544768,"data_stored":224084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:03:00 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.89300000000000002,"15min":0.91700000000000004},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":0.98599999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":1.163},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.83999999999999997,"15min":0.83499999999999996},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.93899999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:03:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.88500000000000001,"15min":0.90000000000000002},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.0649999999999999,"5min":1.371,"15min":1.371},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.90800000000000003,"15min":0.92800000000000005},"min":{"1min":0.58199999999999996,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.1599999999999999,"5min":1.365,"15min":1.365},"last":1.038}]},{"osd":2,"last update":"Mon Oct 9 09:03:00 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.871,"15min":0.872},"min":{"1min":0.60599999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.1479999999999999,"5min":1.155,"15min":1.155},"last":0.99199999999999999},{"interface":"front","average":{"1min":0.875,"5min":0.93400000000000005,"15min":0.94999999999999996},"min":{"1min":0.64700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":1.1950000000000001,"5min":1.294,"15min":1.294},"last":1.087}]},{"osd":3,"last update":"Mon Oct 9 09:03:00 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.874,"15min":0.89000000000000001},"min":{"1min":0.58299999999999996,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.044,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.97499999999999998},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.93500000000000005,"15min":0.93700000000000006},"min":{"1min":0.78100000000000003,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.165,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.115}]},{"osd":4,"last update":"Mon Oct 9 09:03:00 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.87,"15min":0.85199999999999998},"min":{"1min":0.60399999999999998,"5min":0.502,"15min":0.502},"max":{"1min":1.248,"5min":1.337,"15min":1.337},"last":0.874},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.85599999999999998,"15min":0.83499999999999996},"min":{"1min":0.629,"5min":0.371,"15min":0.371},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:03:00 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.93500000000000005,"15min":0.92500000000000004},"min":{"1min":0.61699999999999999,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.208,"5min":1.371,"15min":1.371},"last":1.01},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.91200000000000003,"15min":0.90600000000000003},"min":{"1min":0.59399999999999997,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.129,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.91300000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.0349999999999999,"15min":1.0429999999999999},"min":{"1min":0.78600000000000003,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.1970000000000001,"5min":1.331,"15min":1.331},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.022,"5min":1.0549999999999999,"15min":1.0609999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.216,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.83999999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574930,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":223769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.67000000000000004,"15min":0.56399999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.37},"max":{"1min":1.069,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.73899999999999999,"15min":0.61299999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.33300000000000002},"max":{"1min":1.1000000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":1.014}]},{"osd":2,"last update":"Mon Oct 9 09:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.75600000000000001,"15min":0.66700000000000004},"min":{"1min":0.48299999999999998,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.71699999999999997,"15min":0.67100000000000004},"min":{"1min":0.439,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.022,"5min":1.125,"15min":1.125},"last":0.63200000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:03:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.73899999999999999,"15min":0.70999999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.92400000000000004,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.84399999999999997,"15min":0.81799999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.143,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.74299999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.875,"15min":0.873},"min":{"1min":0.64900000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.2310000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.90500000000000003,"15min":0.89500000000000002},"min":{"1min":0.65900000000000003,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.302,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.89600000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.024,"15min":1.032},"min":{"1min":0.78000000000000003,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.288,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.103},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.0109999999999999,"15min":1.0329999999999999},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.357,"5min":1.357,"15min":1.357},"last":0.96799999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:02:57 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.095,"15min":1.159},"min":{"1min":0.58599999999999997,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.262,"5min":1.585,"15min":1.585},"last":1.0640000000000001},{"interface":"front","average":{"1min":1.01,"5min":1.1399999999999999,"15min":1.194},"min":{"1min":0.72299999999999998,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.5489999999999999,"5min":1.609,"15min":1.609},"last":1.1499999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.0760000000000001,"15min":1.0880000000000001},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.319,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":0.91700000000000004},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0720000000000001,"15min":1.0760000000000001},"min":{"1min":0.78200000000000003,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.369,"5min":1.5289999999999999,"15min":1.5289999999999999},"last":0.98899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738458,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297632,"kb_used_data":536,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447520,"statfs":{"total":95995035648,"available":95690260480,"internally_reserved":0,"allocated":548864,"data_stored":224399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:02:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.68799999999999994,"15min":0.56899999999999995},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.69299999999999995,"15min":0.56399999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.35499999999999998},"max":{"1min":1.03,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.85599999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:02:20 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.71099999999999997,"15min":0.64100000000000001},"min":{"1min":0.435,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.1379999999999999,"5min":1.276,"15min":1.276},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.77700000000000002,"15min":0.67600000000000005},"min":{"1min":0.39600000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.89300000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:03:05 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.80300000000000005,"15min":0.80700000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.22,"5min":1.365,"15min":1.365},"last":1.042},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.78800000000000003,"15min":0.76700000000000002},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.83499999999999996,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.82099999999999995}]},{"osd":4,"last update":"Mon Oct 9 09:02:44 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.91100000000000003,"15min":0.91900000000000004},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.0740000000000001,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.91800000000000004},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.92500000000000004,"15min":0.91400000000000003},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.3049999999999999,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.95199999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:02:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.97499999999999998,"15min":0.97699999999999998},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.304,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":1.014},{"interface":"front","average":{"1min":1.038,"5min":1.0369999999999999,"15min":1.0389999999999999},"min":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"max":{"1min":1.554,"5min":1.554,"15min":1.554},"last":0.96899999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:03:00 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.0820000000000001,"15min":1.0980000000000001},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.645,"5min":1.645,"15min":1.645},"last":1.121},{"interface":"front","average":{"1min":0.89400000000000002,"5min":1.056,"15min":1.0840000000000001},"min":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"max":{"1min":1.2669999999999999,"5min":1.395,"15min":1.395},"last":1.0700000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:02:44 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.0329999999999999,"15min":1.0449999999999999},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.399,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":1.087},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.115,"15min":1.125},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.716,"5min":1.716,"15min":1.716},"last":1.175}]}]},{"osd":2,"up_from":18,"seq":77309411401,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297632,"kb_used_data":536,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447520,"statfs":{"total":95995035648,"available":95690260480,"internally_reserved":0,"allocated":548864,"data_stored":224399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.67200000000000004,"15min":0.622},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.85399999999999998,"5min":1.04,"15min":1.04},"last":0.623},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.71299999999999997,"15min":0.66300000000000003},"min":{"1min":0.44800000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.98699999999999999,"5min":1.29,"15min":1.29},"last":0.91800000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:02:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.68500000000000005,"15min":0.64100000000000001},"min":{"1min":0.39200000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.753,"15min":0.69299999999999995},"min":{"1min":0.40999999999999998,"5min":0.372,"15min":0.372},"max":{"1min":1.034,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.75800000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.80000000000000004,"15min":0.77400000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.1359999999999999,"5min":1.278,"15min":1.278},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.77500000000000002,"15min":0.75900000000000001},"min":{"1min":0.41999999999999998,"5min":0.38,"15min":0.38},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.83699999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.96299999999999997,"15min":0.94299999999999995},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.171,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.97299999999999998},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.95399999999999996,"15min":0.94299999999999995},"min":{"1min":0.72299999999999998,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.2589999999999999,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.0089999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:02:21 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.95899999999999996,"15min":0.95799999999999996},"min":{"1min":0.63,"5min":0.63,"15min":0.63},"max":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":1.0549999999999999},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.95699999999999996,"15min":0.95099999999999996},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.415,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.0309999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:02:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.0800000000000001,"15min":1.1319999999999999},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.2949999999999999,"5min":1.488,"15min":1.488},"last":1.0720000000000001},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.0589999999999999,"15min":1.0860000000000001},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.3149999999999999,"5min":1.464,"15min":1.464},"last":1.1060000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.012,"15min":1.02},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.139},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.1200000000000001,"15min":1.1379999999999999},"min":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.3480000000000001,"5min":1.423,"15min":1.423},"last":1.121}]}]},{"osd":3,"up_from":25,"seq":107374182465,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297632,"kb_used_data":536,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447520,"statfs":{"total":95995035648,"available":95690260480,"internally_reserved":0,"allocated":548864,"data_stored":224399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.65100000000000002,"15min":0.65600000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.96799999999999997,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.83699999999999997},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.72299999999999998,"15min":0.73799999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.84099999999999997,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.61099999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.72499999999999998,"15min":0.72699999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.004,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.71499999999999997,"15min":0.752},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.78900000000000003,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.72499999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.79500000000000004,"15min":0.83299999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.052,"5min":1.7829999999999999,"15min":1.7829999999999999},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.77100000000000002,"15min":0.79200000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.93100000000000005,"5min":1.24,"15min":1.24},"last":0.75700000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.93999999999999995,"15min":0.95899999999999996},"min":{"1min":0.64800000000000002,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.1399999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.018},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.91500000000000004,"15min":0.91500000000000004},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.216,"5min":1.4099999999999999,"15min":1.4099999999999999},"last":0.90700000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:02:14 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.99199999999999999,"15min":0.99299999999999999},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.244,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.056},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.01,"15min":1.004},"min":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"max":{"1min":1.3260000000000001,"5min":1.421,"15min":1.421},"last":1.1819999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:02:58 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.0589999999999999,"15min":1.105},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.262,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":1.244},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.0640000000000001,"15min":1.0980000000000001},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.2569999999999999,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":1.0900000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:02:42 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.0720000000000001,"15min":1.099},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.1990000000000001,"5min":1.5309999999999999,"15min":1.5309999999999999},"last":1.1539999999999999},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.0980000000000001,"15min":1.1299999999999999},"min":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.3009999999999999,"5min":1.518,"15min":1.518},"last":1.2869999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986234,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297632,"kb_used_data":536,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447520,"statfs":{"total":95995035648,"available":95690260480,"internally_reserved":0,"allocated":548864,"data_stored":224399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.83499999999999996,"15min":0.82899999999999996},"min":{"1min":0.61399999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.3440000000000001,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.878},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.84399999999999997,"15min":0.84999999999999998},"min":{"1min":0.628,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.67500000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.81999999999999995,"15min":0.80900000000000005},"min":{"1min":0.59899999999999998,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.0860000000000001,"5min":1.2350000000000001,"15min":1.2350000000000001},"last":1.1639999999999999},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.873,"15min":0.871},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.079,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":0.96199999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.85599999999999998,"15min":0.83599999999999997},"min":{"1min":0.67900000000000005,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":1.117},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.873,"15min":0.86399999999999999},"min":{"1min":0.64600000000000002,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1339999999999999,"5min":1.224,"15min":1.224},"last":0.92200000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.88,"15min":0.873},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.175,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.92200000000000004,"15min":0.91200000000000003},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.0549999999999999,"5min":2.028,"15min":2.028},"last":1.0029999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":0.84499999999999997,"15min":0.82099999999999995},"min":{"1min":0.504,"5min":0.441,"15min":0.441},"max":{"1min":1.1439999999999999,"5min":1.236,"15min":1.236},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.93200000000000005,"15min":0.92200000000000004},"min":{"1min":0.57799999999999996,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.1879999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.0329999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:03:01 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.94699999999999995,"15min":0.97699999999999998},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.3049999999999999,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.034,"5min":1.0609999999999999,"15min":1.087},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.298,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":0.93999999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:02:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.88300000000000001,"15min":0.875},"min":{"1min":0.56299999999999994,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.343,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.002,"5min":0.93200000000000005,"15min":0.92000000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.749}]}]},{"osd":5,"up_from":37,"seq":158913790000,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297628,"kb_used_data":532,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447524,"statfs":{"total":95995035648,"available":95690264576,"internally_reserved":0,"allocated":544768,"data_stored":224084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.84299999999999997,"15min":0.82599999999999996},"min":{"1min":0.61499999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":2.052,"5min":2.052,"15min":2.052},"last":1.097},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.84799999999999998,"15min":0.83199999999999996},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.113,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.85199999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.86199999999999999,"15min":0.84799999999999998},"min":{"1min":0.59699999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":2.214,"5min":2.214,"15min":2.214},"last":1.1850000000000001},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.83299999999999996,"15min":0.81799999999999995},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.2609999999999999,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":0.98699999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.86599999999999999,"15min":0.85199999999999998},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.2809999999999999,"5min":1.323,"15min":1.323},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.90600000000000003,"15min":0.90000000000000002},"min":{"1min":0.76000000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":2.1200000000000001,"5min":2.1200000000000001,"15min":2.1200000000000001},"last":1.002}]},{"osd":3,"last update":"Mon Oct 9 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.90800000000000003,"15min":0.90000000000000002},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":1.127},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.90700000000000003,"15min":0.89100000000000001},"min":{"1min":0.61899999999999999,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":2.1669999999999998,"5min":2.1669999999999998,"15min":2.1669999999999998},"last":0.88900000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.72799999999999998,"15min":0.70399999999999996},"min":{"1min":0.53100000000000003,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.2929999999999999,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":1.0589999999999999},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.76100000000000001,"15min":0.72399999999999998},"min":{"1min":0.53700000000000003,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.73599999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:02:59 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.88600000000000001,"15min":0.87},"min":{"1min":0.65100000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.266,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":1.157},{"interface":"front","average":{"1min":1.054,"5min":1.048,"15min":1.054},"min":{"1min":0.71599999999999997,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.266,"5min":1.496,"15min":1.496},"last":1.0309999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.93400000000000005,"15min":0.92500000000000004},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.2569999999999999,"5min":1.319,"15min":1.319},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.0409999999999999,"5min":0.98399999999999999,"15min":0.97399999999999998},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":2.4809999999999999,"5min":2.4809999999999999,"15min":2.4809999999999999},"last":0.78900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:03:11.683 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T09:03:11.683 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T09:03:11.684 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T09:03:11.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T09:03:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:12 smithi181 bash[27712]: audit 2023-10-09T09:03:11.001531+0000 mgr.y (mgr.24355) 90 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:03:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:12 smithi181 bash[27712]: cluster 2023-10-09T09:03:11.200272+0000 mgr.y (mgr.24355) 91 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:12 smithi163 bash[33912]: audit 2023-10-09T09:03:11.001531+0000 mgr.y (mgr.24355) 90 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:03:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:12 smithi163 bash[33912]: cluster 2023-10-09T09:03:11.200272+0000 mgr.y (mgr.24355) 91 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:12 smithi163 bash[27500]: audit 2023-10-09T09:03:11.001531+0000 mgr.y (mgr.24355) 90 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:03:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:12 smithi163 bash[27500]: cluster 2023-10-09T09:03:11.200272+0000 mgr.y (mgr.24355) 91 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:14 smithi181 bash[27712]: cluster 2023-10-09T09:03:13.200925+0000 mgr.y (mgr.24355) 92 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:14 smithi163 bash[33912]: cluster 2023-10-09T09:03:13.200925+0000 mgr.y (mgr.24355) 92 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:14 smithi163 bash[27500]: cluster 2023-10-09T09:03:13.200925+0000 mgr.y (mgr.24355) 92 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:16.464 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:16 smithi181 bash[27712]: cluster 2023-10-09T09:03:15.201510+0000 mgr.y (mgr.24355) 93 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:16 smithi163 bash[33912]: cluster 2023-10-09T09:03:15.201510+0000 mgr.y (mgr.24355) 93 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:16.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:16 smithi163 bash[27500]: cluster 2023-10-09T09:03:15.201510+0000 mgr.y (mgr.24355) 93 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:18 smithi181 bash[27712]: cluster 2023-10-09T09:03:17.202086+0000 mgr.y (mgr.24355) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:18 smithi163 bash[27500]: cluster 2023-10-09T09:03:17.202086+0000 mgr.y (mgr.24355) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:18.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:18 smithi163 bash[33912]: cluster 2023-10-09T09:03:17.202086+0000 mgr.y (mgr.24355) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:18.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:03:18 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:03:19.155 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:03:19.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:19 smithi163 bash[27500]: audit 2023-10-09T09:03:19.155684+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.163:0/758306025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:03:19.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:19 smithi163 bash[33912]: audit 2023-10-09T09:03:19.155684+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.163:0/758306025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:03:19.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:19 smithi181 bash[27712]: audit 2023-10-09T09:03:19.155684+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.163:0/758306025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:03:19.845 INFO:teuthology.orchestra.run.smithi163.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T09:03:19.846 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T09:03:19.846 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T09:03:19.847 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T09:03:19.857 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi163.front.sepia.ceph.com 2023-10-09T09:03:19.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T09:03:20.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:03:19 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:19] "GET /metrics HTTP/1.1" 200 155665 "" "Prometheus/2.33.4" 2023-10-09T09:03:20.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:20 smithi163 bash[33912]: cluster 2023-10-09T09:03:19.202666+0000 mgr.y (mgr.24355) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:20.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:20 smithi163 bash[27500]: cluster 2023-10-09T09:03:19.202666+0000 mgr.y (mgr.24355) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:20.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:20 smithi181 bash[27712]: cluster 2023-10-09T09:03:19.202666+0000 mgr.y (mgr.24355) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:22 smithi181 bash[27712]: cluster 2023-10-09T09:03:21.203089+0000 mgr.y (mgr.24355) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:22 smithi163 bash[33912]: cluster 2023-10-09T09:03:21.203089+0000 mgr.y (mgr.24355) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:22.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:22 smithi163 bash[27500]: cluster 2023-10-09T09:03:21.203089+0000 mgr.y (mgr.24355) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:23 smithi181 bash[27712]: audit 2023-10-09T09:03:22.687680+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T09:03:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:23 smithi163 bash[27500]: audit 2023-10-09T09:03:22.687680+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T09:03:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:23 smithi163 bash[33912]: audit 2023-10-09T09:03:22.687680+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T09:03:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:24 smithi181 bash[27712]: audit 2023-10-09T09:03:22.686547+0000 mgr.y (mgr.24355) 97 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:03:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:24 smithi181 bash[27712]: cluster 2023-10-09T09:03:23.203664+0000 mgr.y (mgr.24355) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:24 smithi181 bash[27712]: audit 2023-10-09T09:03:23.282343+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T09:03:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:24 smithi181 bash[27712]: cluster 2023-10-09T09:03:23.282531+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T09:03:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:24 smithi181 bash[27712]: audit 2023-10-09T09:03:23.286275+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T09:03:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[33912]: audit 2023-10-09T09:03:22.686547+0000 mgr.y (mgr.24355) 97 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:03:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[33912]: cluster 2023-10-09T09:03:23.203664+0000 mgr.y (mgr.24355) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[33912]: audit 2023-10-09T09:03:23.282343+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T09:03:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[33912]: cluster 2023-10-09T09:03:23.282531+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T09:03:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[33912]: audit 2023-10-09T09:03:23.286275+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T09:03:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[27500]: audit 2023-10-09T09:03:22.686547+0000 mgr.y (mgr.24355) 97 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:03:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[27500]: cluster 2023-10-09T09:03:23.203664+0000 mgr.y (mgr.24355) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[27500]: audit 2023-10-09T09:03:23.282343+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T09:03:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[27500]: cluster 2023-10-09T09:03:23.282531+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T09:03:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[27500]: audit 2023-10-09T09:03:23.286275+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T09:03:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:24 smithi163 bash[27500]: debug 2023-10-09T09:03:24.287+0000 7f8cc0a79700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T09:03:25.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T09:03:25.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.289362+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T09:03:25.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: cluster 2023-10-09T09:03:24.289515+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T09:03:25.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.291367+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T09:03:25.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: cluster 2023-10-09T09:03:24.292402+0000 mon.a (mon.0) 675 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T09:03:25.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: cluster 2023-10-09T09:03:24.292440+0000 mon.a (mon.0) 676 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: cluster 2023-10-09T09:03:24.304540+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.304900+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: cluster 2023-10-09T09:03:24.305278+0000 mon.a (mon.0) 679 : cluster [DBG] fsmap foo:0 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: cephadm 2023-10-09T09:03:24.308635+0000 mgr.y (mgr.24355) 99 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.317257+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.329045+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.613512+0000 mon.a (mon.0) 682 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.614660+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.623243+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.633742+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.635154+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.vnodzj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:03:25.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.640298+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.vnodzj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:03:25.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:25 smithi181 bash[27712]: audit 2023-10-09T09:03:24.641223+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.289362+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T09:03:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: cluster 2023-10-09T09:03:24.289515+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T09:03:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.291367+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T09:03:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: cluster 2023-10-09T09:03:24.292402+0000 mon.a (mon.0) 675 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T09:03:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: cluster 2023-10-09T09:03:24.292440+0000 mon.a (mon.0) 676 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T09:03:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: cluster 2023-10-09T09:03:24.304540+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T09:03:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.304900+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T09:03:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: cluster 2023-10-09T09:03:24.305278+0000 mon.a (mon.0) 679 : cluster [DBG] fsmap foo:0 2023-10-09T09:03:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: cephadm 2023-10-09T09:03:24.308635+0000 mgr.y (mgr.24355) 99 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T09:03:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.317257+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.329045+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.613512+0000 mon.a (mon.0) 682 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.614660+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.623243+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:25.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.633742+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:25.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.635154+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.vnodzj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:03:25.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.640298+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.vnodzj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:03:25.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[27500]: audit 2023-10-09T09:03:24.641223+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:25.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.289362+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T09:03:25.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: cluster 2023-10-09T09:03:24.289515+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T09:03:25.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.291367+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T09:03:25.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: cluster 2023-10-09T09:03:24.292402+0000 mon.a (mon.0) 675 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T09:03:25.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: cluster 2023-10-09T09:03:24.292440+0000 mon.a (mon.0) 676 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T09:03:25.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: cluster 2023-10-09T09:03:24.304540+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.304900+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: cluster 2023-10-09T09:03:24.305278+0000 mon.a (mon.0) 679 : cluster [DBG] fsmap foo:0 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: cephadm 2023-10-09T09:03:24.308635+0000 mgr.y (mgr.24355) 99 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.317257+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.329045+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.613512+0000 mon.a (mon.0) 682 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.614660+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.623243+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.633742+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.635154+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.vnodzj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.640298+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.vnodzj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:03:25.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:25 smithi163 bash[33912]: audit 2023-10-09T09:03:24.641223+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:26.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:26 smithi163 bash[33912]: cephadm 2023-10-09T09:03:24.642171+0000 mgr.y (mgr.24355) 100 : cephadm [INF] Deploying daemon mds.foo.smithi163.vnodzj on smithi163 2023-10-09T09:03:26.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:26 smithi163 bash[33912]: cluster 2023-10-09T09:03:25.204335+0000 mgr.y (mgr.24355) 101 : cluster [DBG] pgmap v79: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:26.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:26 smithi163 bash[33912]: cluster 2023-10-09T09:03:25.306566+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T09:03:26.461 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:26 smithi163 bash[27500]: cephadm 2023-10-09T09:03:24.642171+0000 mgr.y (mgr.24355) 100 : cephadm [INF] Deploying daemon mds.foo.smithi163.vnodzj on smithi163 2023-10-09T09:03:26.461 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:26 smithi163 bash[27500]: cluster 2023-10-09T09:03:25.204335+0000 mgr.y (mgr.24355) 101 : cluster [DBG] pgmap v79: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:26.461 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:26 smithi163 bash[27500]: cluster 2023-10-09T09:03:25.306566+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T09:03:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:26 smithi181 bash[27712]: cephadm 2023-10-09T09:03:24.642171+0000 mgr.y (mgr.24355) 100 : cephadm [INF] Deploying daemon mds.foo.smithi163.vnodzj on smithi163 2023-10-09T09:03:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:26 smithi181 bash[27712]: cluster 2023-10-09T09:03:25.204335+0000 mgr.y (mgr.24355) 101 : cluster [DBG] pgmap v79: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:26 smithi181 bash[27712]: cluster 2023-10-09T09:03:25.306566+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T09:03:27.558 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:27 smithi163 bash[33912]: cluster 2023-10-09T09:03:26.324195+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T09:03:27.558 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:27 smithi163 bash[27500]: cluster 2023-10-09T09:03:26.324195+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T09:03:27.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:27 smithi181 bash[27712]: cluster 2023-10-09T09:03:26.324195+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T09:03:28.553 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:28 smithi181 bash[27712]: cluster 2023-10-09T09:03:27.204940+0000 mgr.y (mgr.24355) 102 : cluster [DBG] pgmap v82: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:28.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:28 smithi181 bash[27712]: audit 2023-10-09T09:03:27.840179+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:28.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:28 smithi181 bash[27712]: audit 2023-10-09T09:03:27.842517+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.ajqgsc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:03:28.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:28 smithi181 bash[27712]: audit 2023-10-09T09:03:27.847235+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.ajqgsc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:03:28.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:28 smithi181 bash[27712]: audit 2023-10-09T09:03:27.848411+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:28 smithi163 bash[33912]: cluster 2023-10-09T09:03:27.204940+0000 mgr.y (mgr.24355) 102 : cluster [DBG] pgmap v82: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:28 smithi163 bash[33912]: audit 2023-10-09T09:03:27.840179+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:28 smithi163 bash[33912]: audit 2023-10-09T09:03:27.842517+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.ajqgsc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:03:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:28 smithi163 bash[33912]: audit 2023-10-09T09:03:27.847235+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.ajqgsc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:03:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:28 smithi163 bash[33912]: audit 2023-10-09T09:03:27.848411+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:28 smithi163 bash[27500]: cluster 2023-10-09T09:03:27.204940+0000 mgr.y (mgr.24355) 102 : cluster [DBG] pgmap v82: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:28 smithi163 bash[27500]: audit 2023-10-09T09:03:27.840179+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:28 smithi163 bash[27500]: audit 2023-10-09T09:03:27.842517+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.ajqgsc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:03:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:28 smithi163 bash[27500]: audit 2023-10-09T09:03:27.847235+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.ajqgsc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:03:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:28 smithi163 bash[27500]: audit 2023-10-09T09:03:27.848411+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:28.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:03:28 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:03:29.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:29 smithi181 bash[27712]: cephadm 2023-10-09T09:03:27.849278+0000 mgr.y (mgr.24355) 103 : cephadm [INF] Deploying daemon mds.foo.smithi181.ajqgsc on smithi181 2023-10-09T09:03:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:29 smithi163 bash[27500]: cephadm 2023-10-09T09:03:27.849278+0000 mgr.y (mgr.24355) 103 : cephadm [INF] Deploying daemon mds.foo.smithi181.ajqgsc on smithi181 2023-10-09T09:03:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:29 smithi163 bash[33912]: cephadm 2023-10-09T09:03:27.849278+0000 mgr.y (mgr.24355) 103 : cephadm [INF] Deploying daemon mds.foo.smithi181.ajqgsc on smithi181 2023-10-09T09:03:30.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T09:03:30.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:03:29 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:29] "GET /metrics HTTP/1.1" 200 155679 "" "Prometheus/2.33.4" 2023-10-09T09:03:30.489 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:30 smithi181 bash[27712]: cluster 2023-10-09T09:03:29.205554+0000 mgr.y (mgr.24355) 104 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:30.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:30 smithi181 bash[27712]: cluster 2023-10-09T09:03:29.338658+0000 mon.a (mon.0) 695 : cluster [DBG] mds.? [v2:172.21.15.163:6834/3139884719,v1:172.21.15.163:6835/3139884719] up:boot 2023-10-09T09:03:30.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:30 smithi181 bash[27712]: cluster 2023-10-09T09:03:29.338719+0000 mon.a (mon.0) 696 : cluster [INF] daemon mds.foo.smithi163.vnodzj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T09:03:30.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:30 smithi181 bash[27712]: cluster 2023-10-09T09:03:29.338785+0000 mon.a (mon.0) 697 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T09:03:30.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:30 smithi181 bash[27712]: cluster 2023-10-09T09:03:29.338812+0000 mon.a (mon.0) 698 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T09:03:30.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:30 smithi181 bash[27712]: cluster 2023-10-09T09:03:29.338832+0000 mon.a (mon.0) 699 : cluster [INF] Cluster is now healthy 2023-10-09T09:03:30.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:30 smithi181 bash[27712]: cluster 2023-10-09T09:03:29.339080+0000 mon.a (mon.0) 700 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T09:03:30.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:30 smithi181 bash[27712]: audit 2023-10-09T09:03:29.339355+0000 mon.a (mon.0) 701 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:03:30.491 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:30 smithi181 bash[27712]: cluster 2023-10-09T09:03:29.344031+0000 mon.a (mon.0) 702 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:creating} 2023-10-09T09:03:30.491 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:30 smithi181 bash[27712]: cluster 2023-10-09T09:03:29.361578+0000 mon.a (mon.0) 703 : cluster [INF] daemon mds.foo.smithi163.vnodzj is now active in filesystem foo as rank 0 2023-10-09T09:03:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[33912]: cluster 2023-10-09T09:03:29.205554+0000 mgr.y (mgr.24355) 104 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[33912]: cluster 2023-10-09T09:03:29.338658+0000 mon.a (mon.0) 695 : cluster [DBG] mds.? [v2:172.21.15.163:6834/3139884719,v1:172.21.15.163:6835/3139884719] up:boot 2023-10-09T09:03:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[33912]: cluster 2023-10-09T09:03:29.338719+0000 mon.a (mon.0) 696 : cluster [INF] daemon mds.foo.smithi163.vnodzj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T09:03:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[33912]: cluster 2023-10-09T09:03:29.338785+0000 mon.a (mon.0) 697 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T09:03:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[33912]: cluster 2023-10-09T09:03:29.338812+0000 mon.a (mon.0) 698 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T09:03:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[33912]: cluster 2023-10-09T09:03:29.338832+0000 mon.a (mon.0) 699 : cluster [INF] Cluster is now healthy 2023-10-09T09:03:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[33912]: cluster 2023-10-09T09:03:29.339080+0000 mon.a (mon.0) 700 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T09:03:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[33912]: audit 2023-10-09T09:03:29.339355+0000 mon.a (mon.0) 701 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:03:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[33912]: cluster 2023-10-09T09:03:29.344031+0000 mon.a (mon.0) 702 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:creating} 2023-10-09T09:03:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[33912]: cluster 2023-10-09T09:03:29.361578+0000 mon.a (mon.0) 703 : cluster [INF] daemon mds.foo.smithi163.vnodzj is now active in filesystem foo as rank 0 2023-10-09T09:03:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[27500]: cluster 2023-10-09T09:03:29.205554+0000 mgr.y (mgr.24355) 104 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[27500]: cluster 2023-10-09T09:03:29.338658+0000 mon.a (mon.0) 695 : cluster [DBG] mds.? [v2:172.21.15.163:6834/3139884719,v1:172.21.15.163:6835/3139884719] up:boot 2023-10-09T09:03:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[27500]: cluster 2023-10-09T09:03:29.338719+0000 mon.a (mon.0) 696 : cluster [INF] daemon mds.foo.smithi163.vnodzj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T09:03:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[27500]: cluster 2023-10-09T09:03:29.338785+0000 mon.a (mon.0) 697 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T09:03:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[27500]: cluster 2023-10-09T09:03:29.338812+0000 mon.a (mon.0) 698 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T09:03:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[27500]: cluster 2023-10-09T09:03:29.338832+0000 mon.a (mon.0) 699 : cluster [INF] Cluster is now healthy 2023-10-09T09:03:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[27500]: cluster 2023-10-09T09:03:29.339080+0000 mon.a (mon.0) 700 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T09:03:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[27500]: audit 2023-10-09T09:03:29.339355+0000 mon.a (mon.0) 701 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:03:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[27500]: cluster 2023-10-09T09:03:29.344031+0000 mon.a (mon.0) 702 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:creating} 2023-10-09T09:03:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:30 smithi163 bash[27500]: cluster 2023-10-09T09:03:29.361578+0000 mon.a (mon.0) 703 : cluster [INF] daemon mds.foo.smithi163.vnodzj is now active in filesystem foo as rank 0 2023-10-09T09:03:31.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:31 smithi181 bash[27712]: cluster 2023-10-09T09:03:30.349781+0000 mon.a (mon.0) 704 : cluster [DBG] mds.? [v2:172.21.15.163:6834/3139884719,v1:172.21.15.163:6835/3139884719] up:active 2023-10-09T09:03:31.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:31 smithi181 bash[27712]: cluster 2023-10-09T09:03:30.349846+0000 mon.a (mon.0) 705 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 2023-10-09T09:03:31.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:31 smithi181 bash[27712]: audit 2023-10-09T09:03:30.467095+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:31.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:31 smithi181 bash[27712]: audit 2023-10-09T09:03:30.469246+0000 mon.a (mon.0) 707 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:31 smithi163 bash[33912]: cluster 2023-10-09T09:03:30.349781+0000 mon.a (mon.0) 704 : cluster [DBG] mds.? [v2:172.21.15.163:6834/3139884719,v1:172.21.15.163:6835/3139884719] up:active 2023-10-09T09:03:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:31 smithi163 bash[33912]: cluster 2023-10-09T09:03:30.349846+0000 mon.a (mon.0) 705 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 2023-10-09T09:03:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:31 smithi163 bash[33912]: audit 2023-10-09T09:03:30.467095+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:31 smithi163 bash[33912]: audit 2023-10-09T09:03:30.469246+0000 mon.a (mon.0) 707 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:31.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:31 smithi163 bash[27500]: cluster 2023-10-09T09:03:30.349781+0000 mon.a (mon.0) 704 : cluster [DBG] mds.? [v2:172.21.15.163:6834/3139884719,v1:172.21.15.163:6835/3139884719] up:active 2023-10-09T09:03:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:31 smithi163 bash[27500]: cluster 2023-10-09T09:03:30.349846+0000 mon.a (mon.0) 705 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 2023-10-09T09:03:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:31 smithi163 bash[27500]: audit 2023-10-09T09:03:30.467095+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:31 smithi163 bash[27500]: audit 2023-10-09T09:03:30.469246+0000 mon.a (mon.0) 707 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:32 smithi181 bash[27712]: cluster 2023-10-09T09:03:31.206160+0000 mgr.y (mgr.24355) 105 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:32 smithi181 bash[27712]: cluster 2023-10-09T09:03:32.042505+0000 mon.a (mon.0) 708 : cluster [DBG] mds.? [v2:172.21.15.181:6832/2837854117,v1:172.21.15.181:6833/2837854117] up:boot 2023-10-09T09:03:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:32 smithi181 bash[27712]: cluster 2023-10-09T09:03:32.042680+0000 mon.a (mon.0) 709 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 1 up:standby 2023-10-09T09:03:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:32 smithi181 bash[27712]: audit 2023-10-09T09:03:32.043300+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:03:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:32 smithi181 bash[27712]: audit 2023-10-09T09:03:32.448337+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:32 smithi163 bash[33912]: cluster 2023-10-09T09:03:31.206160+0000 mgr.y (mgr.24355) 105 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:32 smithi163 bash[33912]: cluster 2023-10-09T09:03:32.042505+0000 mon.a (mon.0) 708 : cluster [DBG] mds.? [v2:172.21.15.181:6832/2837854117,v1:172.21.15.181:6833/2837854117] up:boot 2023-10-09T09:03:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:32 smithi163 bash[33912]: cluster 2023-10-09T09:03:32.042680+0000 mon.a (mon.0) 709 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 1 up:standby 2023-10-09T09:03:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:32 smithi163 bash[33912]: audit 2023-10-09T09:03:32.043300+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:03:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:32 smithi163 bash[33912]: audit 2023-10-09T09:03:32.448337+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:32 smithi163 bash[27500]: cluster 2023-10-09T09:03:31.206160+0000 mgr.y (mgr.24355) 105 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:32 smithi163 bash[27500]: cluster 2023-10-09T09:03:32.042505+0000 mon.a (mon.0) 708 : cluster [DBG] mds.? [v2:172.21.15.181:6832/2837854117,v1:172.21.15.181:6833/2837854117] up:boot 2023-10-09T09:03:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:32 smithi163 bash[27500]: cluster 2023-10-09T09:03:32.042680+0000 mon.a (mon.0) 709 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 1 up:standby 2023-10-09T09:03:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:32 smithi163 bash[27500]: audit 2023-10-09T09:03:32.043300+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:03:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:32 smithi163 bash[27500]: audit 2023-10-09T09:03:32.448337+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:33.731 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T09:03:33.740 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T09:03:33.740 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T09:03:33.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T09:03:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:34 smithi163 bash[27500]: cluster 2023-10-09T09:03:33.206994+0000 mgr.y (mgr.24355) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:34.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:34 smithi163 bash[33912]: cluster 2023-10-09T09:03:33.206994+0000 mgr.y (mgr.24355) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:34.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:34 smithi181 bash[27712]: cluster 2023-10-09T09:03:33.206994+0000 mgr.y (mgr.24355) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:36 smithi181 bash[27712]: cluster 2023-10-09T09:03:35.207568+0000 mgr.y (mgr.24355) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:36 smithi181 bash[27712]: audit 2023-10-09T09:03:36.177712+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:36 smithi181 bash[27712]: audit 2023-10-09T09:03:36.419033+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:36 smithi181 bash[27712]: audit 2023-10-09T09:03:36.420361+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:36 smithi181 bash[27712]: audit 2023-10-09T09:03:36.421688+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:36.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:36 smithi181 bash[27712]: audit 2023-10-09T09:03:36.428511+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:36 smithi181 bash[27712]: audit 2023-10-09T09:03:36.436691+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:36.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:36 smithi181 bash[27712]: audit 2023-10-09T09:03:36.438921+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:36.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:36 smithi181 bash[27712]: audit 2023-10-09T09:03:36.440208+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:36.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:36 smithi181 bash[27712]: audit 2023-10-09T09:03:36.447167+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[27500]: cluster 2023-10-09T09:03:35.207568+0000 mgr.y (mgr.24355) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[27500]: audit 2023-10-09T09:03:36.177712+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[27500]: audit 2023-10-09T09:03:36.419033+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[27500]: audit 2023-10-09T09:03:36.420361+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:36.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[27500]: audit 2023-10-09T09:03:36.421688+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:36.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[27500]: audit 2023-10-09T09:03:36.428511+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[27500]: audit 2023-10-09T09:03:36.436691+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:36.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[27500]: audit 2023-10-09T09:03:36.438921+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:36.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[27500]: audit 2023-10-09T09:03:36.440208+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:36.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[27500]: audit 2023-10-09T09:03:36.447167+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[33912]: cluster 2023-10-09T09:03:35.207568+0000 mgr.y (mgr.24355) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:36.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[33912]: audit 2023-10-09T09:03:36.177712+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[33912]: audit 2023-10-09T09:03:36.419033+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[33912]: audit 2023-10-09T09:03:36.420361+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:36.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[33912]: audit 2023-10-09T09:03:36.421688+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:36.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[33912]: audit 2023-10-09T09:03:36.428511+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:36.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[33912]: audit 2023-10-09T09:03:36.436691+0000 mon.a (mon.0) 717 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:03:36.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[33912]: audit 2023-10-09T09:03:36.438921+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:03:36.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[33912]: audit 2023-10-09T09:03:36.440208+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:03:36.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:36 smithi163 bash[33912]: audit 2023-10-09T09:03:36.447167+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:38.285 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:38.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:38 smithi163 bash[33912]: cluster 2023-10-09T09:03:37.208216+0000 mgr.y (mgr.24355) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:38.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:38 smithi163 bash[27500]: cluster 2023-10-09T09:03:37.208216+0000 mgr.y (mgr.24355) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:38 smithi181 bash[27712]: cluster 2023-10-09T09:03:37.208216+0000 mgr.y (mgr.24355) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:38.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:03:38 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:03:40.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:03:39 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:39] "GET /metrics HTTP/1.1" 200 191367 "" "Prometheus/2.33.4" 2023-10-09T09:03:40.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:40 smithi163 bash[27500]: cluster 2023-10-09T09:03:39.209332+0000 mgr.y (mgr.24355) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:40.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:40 smithi163 bash[33912]: cluster 2023-10-09T09:03:39.209332+0000 mgr.y (mgr.24355) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:40.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:40 smithi181 bash[27712]: cluster 2023-10-09T09:03:39.209332+0000 mgr.y (mgr.24355) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:41.423 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:03:42.066 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":18,"active_gid":24355,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6800","nonce":3919298633},{"type":"v1","addr":"172.21.15.163:6801","nonce":3919298633}]},"active_addr":"172.21.15.163:6801/3919298633","active_change":"2023-10-09T09:01:07.146973+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24358,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":52,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":536711455}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":623143106}]}]} 2023-10-09T09:03:42.071 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T09:03:42.071 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T09:03:42.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:03:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:42 smithi163 bash[33912]: cluster 2023-10-09T09:03:41.209931+0000 mgr.y (mgr.24355) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:42 smithi163 bash[33912]: audit 2023-10-09T09:03:41.418363+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.163:0/2646252731' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:03:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:42 smithi163 bash[27500]: cluster 2023-10-09T09:03:41.209931+0000 mgr.y (mgr.24355) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:42 smithi163 bash[27500]: audit 2023-10-09T09:03:41.418363+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.163:0/2646252731' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:03:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:42 smithi181 bash[27712]: cluster 2023-10-09T09:03:41.209931+0000 mgr.y (mgr.24355) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:42 smithi181 bash[27712]: audit 2023-10-09T09:03:41.418363+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.163:0/2646252731' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:03:43.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:43 smithi181 bash[27712]: audit 2023-10-09T09:03:42.471406+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:43.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:43 smithi163 bash[27500]: audit 2023-10-09T09:03:42.471406+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:43.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:43 smithi163 bash[33912]: audit 2023-10-09T09:03:42.471406+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:03:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:44 smithi181 bash[27712]: cluster 2023-10-09T09:03:43.210837+0000 mgr.y (mgr.24355) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:44 smithi163 bash[27500]: cluster 2023-10-09T09:03:43.210837+0000 mgr.y (mgr.24355) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:44 smithi163 bash[33912]: cluster 2023-10-09T09:03:43.210837+0000 mgr.y (mgr.24355) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:46 smithi181 bash[27712]: cluster 2023-10-09T09:03:45.211515+0000 mgr.y (mgr.24355) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:46.846 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:46.873 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:46 smithi163 bash[33912]: cluster 2023-10-09T09:03:45.211515+0000 mgr.y (mgr.24355) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:46.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:46 smithi163 bash[27500]: cluster 2023-10-09T09:03:45.211515+0000 mgr.y (mgr.24355) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:48.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:48 smithi163 bash[27500]: cluster 2023-10-09T09:03:47.212175+0000 mgr.y (mgr.24355) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:48.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:48 smithi163 bash[33912]: cluster 2023-10-09T09:03:47.212175+0000 mgr.y (mgr.24355) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:48 smithi181 bash[27712]: cluster 2023-10-09T09:03:47.212175+0000 mgr.y (mgr.24355) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:48.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:03:48 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:03:49.614 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:03:49.614 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":61,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:03:26.316796+0000","last_up_change":"2023-10-09T09:00:36.182187+0000","last_in_change":"2023-10-09T09:00:10.180048+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:01:33.304980+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:03:23.129371+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:03:23.743990+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6803","nonce":422484962}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6805","nonce":422484962}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6809","nonce":422484962}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6807","nonce":422484962}]},"public_addr":"172.21.15.163:6803/422484962","cluster_addr":"172.21.15.163:6805/422484962","heartbeat_back_addr":"172.21.15.163:6809/422484962","heartbeat_front_addr":"172.21.15.163:6807/422484962","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6811","nonce":1836428548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6813","nonce":1836428548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6817","nonce":1836428548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6815","nonce":1836428548}]},"public_addr":"172.21.15.163:6811/1836428548","cluster_addr":"172.21.15.163:6813/1836428548","heartbeat_back_addr":"172.21.15.163:6817/1836428548","heartbeat_front_addr":"172.21.15.163:6815/1836428548","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6819","nonce":2248501829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6821","nonce":2248501829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6825","nonce":2248501829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6823","nonce":2248501829}]},"public_addr":"172.21.15.163:6819/2248501829","cluster_addr":"172.21.15.163:6821/2248501829","heartbeat_back_addr":"172.21.15.163:6825/2248501829","heartbeat_front_addr":"172.21.15.163:6823/2248501829","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6827","nonce":3899711337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6829","nonce":3899711337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6833","nonce":3899711337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6831","nonce":3899711337}]},"public_addr":"172.21.15.163:6827/3899711337","cluster_addr":"172.21.15.163:6829/3899711337","heartbeat_back_addr":"172.21.15.163:6833/3899711337","heartbeat_front_addr":"172.21.15.163:6831/3899711337","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:6801/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6800/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/566729834":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2199287559":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:03:49.967 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:03:49 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:49] "GET /metrics HTTP/1.1" 200 191367 "" "Prometheus/2.33.4" 2023-10-09T09:03:50.269 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T09:03:50.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:03:50.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:50 smithi181 bash[27712]: cluster 2023-10-09T09:03:49.213327+0000 mgr.y (mgr.24355) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:50.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:50 smithi181 bash[27712]: audit 2023-10-09T09:03:49.611989+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.163:0/2292989213' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:03:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:50 smithi163 bash[27500]: cluster 2023-10-09T09:03:49.213327+0000 mgr.y (mgr.24355) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:50 smithi163 bash[27500]: audit 2023-10-09T09:03:49.611989+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.163:0/2292989213' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:03:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:50 smithi163 bash[33912]: cluster 2023-10-09T09:03:49.213327+0000 mgr.y (mgr.24355) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:03:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:50 smithi163 bash[33912]: audit 2023-10-09T09:03:49.611989+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.163:0/2292989213' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:03:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:52 smithi181 bash[27712]: cluster 2023-10-09T09:03:51.214018+0000 mgr.y (mgr.24355) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:52.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:52 smithi163 bash[33912]: cluster 2023-10-09T09:03:51.214018+0000 mgr.y (mgr.24355) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:52 smithi163 bash[27500]: cluster 2023-10-09T09:03:51.214018+0000 mgr.y (mgr.24355) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:54 smithi181 bash[27712]: cluster 2023-10-09T09:03:53.214836+0000 mgr.y (mgr.24355) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:54 smithi163 bash[27500]: cluster 2023-10-09T09:03:53.214836+0000 mgr.y (mgr.24355) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:54.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:54 smithi163 bash[33912]: cluster 2023-10-09T09:03:53.214836+0000 mgr.y (mgr.24355) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:55.053 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:03:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:56 smithi163 bash[27500]: cluster 2023-10-09T09:03:55.215487+0000 mgr.y (mgr.24355) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:56 smithi163 bash[33912]: cluster 2023-10-09T09:03:55.215487+0000 mgr.y (mgr.24355) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:56 smithi181 bash[27712]: cluster 2023-10-09T09:03:55.215487+0000 mgr.y (mgr.24355) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:57.770 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:03:57.770 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":61,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:03:26.316796+0000","last_up_change":"2023-10-09T09:00:36.182187+0000","last_in_change":"2023-10-09T09:00:10.180048+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:01:33.304980+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:03:23.129371+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:03:23.743990+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6803","nonce":422484962}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6805","nonce":422484962}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6809","nonce":422484962}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":422484962},{"type":"v1","addr":"172.21.15.163:6807","nonce":422484962}]},"public_addr":"172.21.15.163:6803/422484962","cluster_addr":"172.21.15.163:6805/422484962","heartbeat_back_addr":"172.21.15.163:6809/422484962","heartbeat_front_addr":"172.21.15.163:6807/422484962","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6811","nonce":1836428548}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6813","nonce":1836428548}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6817","nonce":1836428548}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":1836428548},{"type":"v1","addr":"172.21.15.163:6815","nonce":1836428548}]},"public_addr":"172.21.15.163:6811/1836428548","cluster_addr":"172.21.15.163:6813/1836428548","heartbeat_back_addr":"172.21.15.163:6817/1836428548","heartbeat_front_addr":"172.21.15.163:6815/1836428548","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6819","nonce":2248501829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6821","nonce":2248501829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6825","nonce":2248501829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2248501829},{"type":"v1","addr":"172.21.15.163:6823","nonce":2248501829}]},"public_addr":"172.21.15.163:6819/2248501829","cluster_addr":"172.21.15.163:6821/2248501829","heartbeat_back_addr":"172.21.15.163:6825/2248501829","heartbeat_front_addr":"172.21.15.163:6823/2248501829","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6827","nonce":3899711337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6829","nonce":3899711337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6833","nonce":3899711337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":3899711337},{"type":"v1","addr":"172.21.15.163:6831","nonce":3899711337}]},"public_addr":"172.21.15.163:6827/3899711337","cluster_addr":"172.21.15.163:6829/3899711337","heartbeat_back_addr":"172.21.15.163:6833/3899711337","heartbeat_front_addr":"172.21.15.163:6831/3899711337","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:6801/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6800/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/566729834":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2199287559":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:03:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:58 smithi163 bash[33912]: cluster 2023-10-09T09:03:57.216160+0000 mgr.y (mgr.24355) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:03:58 smithi163 bash[33912]: audit 2023-10-09T09:03:57.768966+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.163:0/3678967998' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:03:58.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:58 smithi163 bash[27500]: cluster 2023-10-09T09:03:57.216160+0000 mgr.y (mgr.24355) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:58.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:03:58 smithi163 bash[27500]: audit 2023-10-09T09:03:57.768966+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.163:0/3678967998' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:03:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:58 smithi181 bash[27712]: cluster 2023-10-09T09:03:57.216160+0000 mgr.y (mgr.24355) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:03:58.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:03:58 smithi181 bash[27712]: audit 2023-10-09T09:03:57.768966+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.163:0/3678967998' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:03:58.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T09:03:58.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T09:03:58.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T09:03:58.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T09:03:58.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T09:03:58.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T09:03:58.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T09:03:58.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T09:03:58.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:03:58 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:04:00.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:03:59 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:03:59] "GET /metrics HTTP/1.1" 200 191363 "" "Prometheus/2.33.4" 2023-10-09T09:04:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:00 smithi181 bash[27712]: cluster 2023-10-09T09:03:59.217299+0000 mgr.y (mgr.24355) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:00 smithi163 bash[27500]: cluster 2023-10-09T09:03:59.217299+0000 mgr.y (mgr.24355) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:00 smithi163 bash[33912]: cluster 2023-10-09T09:03:59.217299+0000 mgr.y (mgr.24355) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:02 smithi181 bash[27712]: cluster 2023-10-09T09:04:01.217946+0000 mgr.y (mgr.24355) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:02.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:02 smithi163 bash[27500]: cluster 2023-10-09T09:04:01.217946+0000 mgr.y (mgr.24355) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:02.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:02 smithi163 bash[33912]: cluster 2023-10-09T09:04:01.217946+0000 mgr.y (mgr.24355) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:03.288 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:03.289 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:03.291 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:03.291 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:03.292 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:03.292 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:03.293 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:03.294 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:04 smithi181 bash[27712]: cluster 2023-10-09T09:04:03.219005+0000 mgr.y (mgr.24355) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:04.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:04 smithi163 bash[27500]: cluster 2023-10-09T09:04:03.219005+0000 mgr.y (mgr.24355) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:04.697 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:04 smithi163 bash[33912]: cluster 2023-10-09T09:04:03.219005+0000 mgr.y (mgr.24355) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:06.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:06 smithi181 bash[27712]: cluster 2023-10-09T09:04:05.219424+0000 mgr.y (mgr.24355) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:06.598 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:06 smithi163 bash[33912]: cluster 2023-10-09T09:04:05.219424+0000 mgr.y (mgr.24355) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:06.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:06 smithi163 bash[27500]: cluster 2023-10-09T09:04:05.219424+0000 mgr.y (mgr.24355) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:08 smithi163 bash[27500]: audit 2023-10-09T09:04:07.183944+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T09:04:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:08 smithi163 bash[27500]: audit 2023-10-09T09:04:07.184209+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:04:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:08 smithi163 bash[27500]: audit 2023-10-09T09:04:07.184405+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T09:04:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:08 smithi163 bash[27500]: cluster 2023-10-09T09:04:07.219904+0000 mgr.y (mgr.24355) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:08 smithi163 bash[33912]: audit 2023-10-09T09:04:07.183944+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T09:04:08.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:08 smithi163 bash[33912]: audit 2023-10-09T09:04:07.184209+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:04:08.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:08 smithi163 bash[33912]: audit 2023-10-09T09:04:07.184405+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T09:04:08.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:08 smithi163 bash[33912]: cluster 2023-10-09T09:04:07.219904+0000 mgr.y (mgr.24355) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:08 smithi181 bash[27712]: audit 2023-10-09T09:04:07.183944+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T09:04:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:08 smithi181 bash[27712]: audit 2023-10-09T09:04:07.184209+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:04:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:08 smithi181 bash[27712]: audit 2023-10-09T09:04:07.184405+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T09:04:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:08 smithi181 bash[27712]: cluster 2023-10-09T09:04:07.219904+0000 mgr.y (mgr.24355) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:08.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:04:08 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:04:09.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:09 smithi163 bash[33912]: audit 2023-10-09T09:04:08.057848+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T09:04:09.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:09 smithi163 bash[33912]: audit 2023-10-09T09:04:08.057900+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:04:09.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:09 smithi163 bash[33912]: audit 2023-10-09T09:04:08.057940+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T09:04:09.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:09 smithi163 bash[33912]: cluster 2023-10-09T09:04:08.057988+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T09:04:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:09 smithi163 bash[27500]: audit 2023-10-09T09:04:08.057848+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T09:04:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:09 smithi163 bash[27500]: audit 2023-10-09T09:04:08.057900+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:04:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:09 smithi163 bash[27500]: audit 2023-10-09T09:04:08.057940+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T09:04:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:09 smithi163 bash[27500]: cluster 2023-10-09T09:04:08.057988+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T09:04:09.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:09 smithi181 bash[27712]: audit 2023-10-09T09:04:08.057848+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T09:04:09.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:09 smithi181 bash[27712]: audit 2023-10-09T09:04:08.057900+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:04:09.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:09 smithi181 bash[27712]: audit 2023-10-09T09:04:08.057940+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T09:04:09.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:09 smithi181 bash[27712]: cluster 2023-10-09T09:04:08.057988+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T09:04:10.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:04:09 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:09] "GET /metrics HTTP/1.1" 200 191360 "" "Prometheus/2.33.4" 2023-10-09T09:04:10.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:10 smithi163 bash[33912]: cluster 2023-10-09T09:04:09.066579+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T09:04:10.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:10 smithi163 bash[33912]: cluster 2023-10-09T09:04:09.220534+0000 mgr.y (mgr.24355) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:10.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:10 smithi163 bash[27500]: cluster 2023-10-09T09:04:09.066579+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T09:04:10.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:10 smithi163 bash[27500]: cluster 2023-10-09T09:04:09.220534+0000 mgr.y (mgr.24355) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:10.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:10 smithi181 bash[27712]: cluster 2023-10-09T09:04:09.066579+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T09:04:10.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:10 smithi181 bash[27712]: cluster 2023-10-09T09:04:09.220534+0000 mgr.y (mgr.24355) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:12 smithi181 bash[27712]: cluster 2023-10-09T09:04:11.221235+0000 mgr.y (mgr.24355) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:12 smithi163 bash[27500]: cluster 2023-10-09T09:04:11.221235+0000 mgr.y (mgr.24355) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:12 smithi163 bash[33912]: cluster 2023-10-09T09:04:11.221235+0000 mgr.y (mgr.24355) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:14.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:14 smithi163 bash[33912]: cluster 2023-10-09T09:04:13.222366+0000 mgr.y (mgr.24355) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:14.380 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:14 smithi163 bash[27500]: cluster 2023-10-09T09:04:13.222366+0000 mgr.y (mgr.24355) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:14 smithi181 bash[27712]: cluster 2023-10-09T09:04:13.222366+0000 mgr.y (mgr.24355) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:15.231 INFO:teuthology.orchestra.run.smithi163.stdout:77309411414 2023-10-09T09:04:15.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T09:04:16.411 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:16 smithi163 bash[33912]: cluster 2023-10-09T09:04:15.222977+0000 mgr.y (mgr.24355) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:16.412 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:16 smithi163 bash[27500]: cluster 2023-10-09T09:04:15.222977+0000 mgr.y (mgr.24355) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:16.413 INFO:teuthology.orchestra.run.smithi163.stdout:184683593783 2023-10-09T09:04:16.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T09:04:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:16 smithi181 bash[27712]: cluster 2023-10-09T09:04:15.222977+0000 mgr.y (mgr.24355) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:17.676 INFO:teuthology.orchestra.run.smithi163.stdout:158913790014 2023-10-09T09:04:17.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T09:04:17.795 INFO:teuthology.orchestra.run.smithi163.stdout:34359738472 2023-10-09T09:04:17.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T09:04:17.962 INFO:teuthology.orchestra.run.smithi163.stdout:133143986248 2023-10-09T09:04:17.962 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T09:04:18.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:18 smithi163 bash[27500]: cluster 2023-10-09T09:04:17.223896+0000 mgr.y (mgr.24355) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:18.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:18 smithi163 bash[33912]: cluster 2023-10-09T09:04:17.223896+0000 mgr.y (mgr.24355) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:18.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:18 smithi181 bash[27712]: cluster 2023-10-09T09:04:17.223896+0000 mgr.y (mgr.24355) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:18.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:04:18 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:04:19.106 INFO:teuthology.orchestra.run.smithi163.stdout:210453397550 2023-10-09T09:04:19.107 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T09:04:19.232 INFO:teuthology.orchestra.run.smithi163.stdout:107374182479 2023-10-09T09:04:19.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T09:04:19.318 INFO:teuthology.orchestra.run.smithi163.stdout:55834574944 2023-10-09T09:04:19.318 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T09:04:20.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:04:19 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:19] "GET /metrics HTTP/1.1" 200 191360 "" "Prometheus/2.33.4" 2023-10-09T09:04:20.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:20 smithi181 bash[27712]: cluster 2023-10-09T09:04:19.224701+0000 mgr.y (mgr.24355) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:20.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:20 smithi163 bash[27500]: cluster 2023-10-09T09:04:19.224701+0000 mgr.y (mgr.24355) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:20 smithi163 bash[33912]: cluster 2023-10-09T09:04:19.224701+0000 mgr.y (mgr.24355) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:21.837 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:21.839 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:22.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:22 smithi163 bash[27500]: cluster 2023-10-09T09:04:21.225319+0000 mgr.y (mgr.24355) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:22 smithi163 bash[33912]: cluster 2023-10-09T09:04:21.225319+0000 mgr.y (mgr.24355) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:22.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:22 smithi181 bash[27712]: cluster 2023-10-09T09:04:21.225319+0000 mgr.y (mgr.24355) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:22.851 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:22.852 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:22.852 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:23.864 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:23.865 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:23.865 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:24 smithi181 bash[27712]: cluster 2023-10-09T09:04:23.225984+0000 mgr.y (mgr.24355) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:24 smithi163 bash[27500]: cluster 2023-10-09T09:04:23.225984+0000 mgr.y (mgr.24355) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:24 smithi163 bash[33912]: cluster 2023-10-09T09:04:23.225984+0000 mgr.y (mgr.24355) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:26.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:26 smithi163 bash[33912]: cluster 2023-10-09T09:04:25.226478+0000 mgr.y (mgr.24355) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:26.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:26 smithi163 bash[27500]: cluster 2023-10-09T09:04:25.226478+0000 mgr.y (mgr.24355) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:26 smithi181 bash[27712]: cluster 2023-10-09T09:04:25.226478+0000 mgr.y (mgr.24355) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:28.145 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:28 smithi163 bash[33912]: cluster 2023-10-09T09:04:27.226991+0000 mgr.y (mgr.24355) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:28.146 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:28 smithi163 bash[27500]: cluster 2023-10-09T09:04:27.226991+0000 mgr.y (mgr.24355) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:28 smithi181 bash[27712]: cluster 2023-10-09T09:04:27.226991+0000 mgr.y (mgr.24355) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:28.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:04:28 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:04:30.186 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:04:29 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:29] "GET /metrics HTTP/1.1" 200 191375 "" "Prometheus/2.33.4" 2023-10-09T09:04:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:30 smithi163 bash[27500]: cluster 2023-10-09T09:04:29.227940+0000 mgr.y (mgr.24355) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:30.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:30 smithi163 bash[33912]: cluster 2023-10-09T09:04:29.227940+0000 mgr.y (mgr.24355) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:30 smithi181 bash[27712]: cluster 2023-10-09T09:04:29.227940+0000 mgr.y (mgr.24355) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:31.630 INFO:teuthology.orchestra.run.smithi163.stdout:77309411418 2023-10-09T09:04:31.843 INFO:teuthology.orchestra.run.smithi163.stdout:184683593786 2023-10-09T09:04:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:32 smithi163 bash[27500]: cluster 2023-10-09T09:04:31.228549+0000 mgr.y (mgr.24355) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:32 smithi163 bash[27500]: audit 2023-10-09T09:04:31.630965+0000 mon.a (mon.0) 732 : audit [DBG] from='client.? 172.21.15.163:0/4245024163' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:04:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:32 smithi163 bash[27500]: audit 2023-10-09T09:04:31.843009+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.163:0/1232610359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:04:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:32 smithi163 bash[33912]: cluster 2023-10-09T09:04:31.228549+0000 mgr.y (mgr.24355) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:32 smithi163 bash[33912]: audit 2023-10-09T09:04:31.630965+0000 mon.a (mon.0) 732 : audit [DBG] from='client.? 172.21.15.163:0/4245024163' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:04:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:32 smithi163 bash[33912]: audit 2023-10-09T09:04:31.843009+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.163:0/1232610359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:04:32.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:32 smithi181 bash[27712]: cluster 2023-10-09T09:04:31.228549+0000 mgr.y (mgr.24355) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:32.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:32 smithi181 bash[27712]: audit 2023-10-09T09:04:31.630965+0000 mon.a (mon.0) 732 : audit [DBG] from='client.? 172.21.15.163:0/4245024163' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:04:32.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:32 smithi181 bash[27712]: audit 2023-10-09T09:04:31.843009+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.163:0/1232610359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:04:34.174 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411414 got 77309411418 for osd.2 2023-10-09T09:04:34.174 DEBUG:teuthology.parallel:result is None 2023-10-09T09:04:34.361 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593783 got 184683593786 for osd.6 2023-10-09T09:04:34.362 DEBUG:teuthology.parallel:result is None 2023-10-09T09:04:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:34 smithi163 bash[27500]: cluster 2023-10-09T09:04:33.229326+0000 mgr.y (mgr.24355) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:34 smithi163 bash[33912]: cluster 2023-10-09T09:04:33.229326+0000 mgr.y (mgr.24355) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:34.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:34 smithi181 bash[27712]: cluster 2023-10-09T09:04:33.229326+0000 mgr.y (mgr.24355) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:35.192 INFO:teuthology.orchestra.run.smithi163.stdout:34359738476 2023-10-09T09:04:35.192 INFO:teuthology.orchestra.run.smithi163.stdout:133143986252 2023-10-09T09:04:35.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:35 smithi163 bash[33912]: audit 2023-10-09T09:04:35.192131+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.163:0/2372442560' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:04:35.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:35 smithi163 bash[33912]: audit 2023-10-09T09:04:35.193060+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.163:0/1014687701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:04:35.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:35 smithi163 bash[27500]: audit 2023-10-09T09:04:35.192131+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.163:0/2372442560' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:04:35.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:35 smithi163 bash[27500]: audit 2023-10-09T09:04:35.193060+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.163:0/1014687701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:04:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:35 smithi181 bash[27712]: audit 2023-10-09T09:04:35.192131+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.163:0/2372442560' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:04:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:35 smithi181 bash[27712]: audit 2023-10-09T09:04:35.193060+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.163:0/1014687701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:04:35.809 INFO:teuthology.orchestra.run.smithi163.stdout:55834574948 2023-10-09T09:04:36.350 INFO:teuthology.orchestra.run.smithi163.stdout:107374182483 2023-10-09T09:04:36.364 INFO:teuthology.orchestra.run.smithi163.stdout:158913790018 2023-10-09T09:04:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:36 smithi181 bash[27712]: cluster 2023-10-09T09:04:35.229802+0000 mgr.y (mgr.24355) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:36 smithi181 bash[27712]: audit 2023-10-09T09:04:35.810173+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.163:0/629315612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:04:36.659 INFO:teuthology.orchestra.run.smithi163.stdout:210453397554 2023-10-09T09:04:36.690 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:36 smithi163 bash[27500]: cluster 2023-10-09T09:04:35.229802+0000 mgr.y (mgr.24355) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:36.690 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:36 smithi163 bash[27500]: audit 2023-10-09T09:04:35.810173+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.163:0/629315612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:04:36.690 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:36 smithi163 bash[33912]: cluster 2023-10-09T09:04:35.229802+0000 mgr.y (mgr.24355) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:36.690 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:36 smithi163 bash[33912]: audit 2023-10-09T09:04:35.810173+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.163:0/629315612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:04:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:37 smithi181 bash[27712]: audit 2023-10-09T09:04:36.351269+0000 mon.a (mon.0) 733 : audit [DBG] from='client.? 172.21.15.163:0/3789634875' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:04:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:37 smithi181 bash[27712]: audit 2023-10-09T09:04:36.364603+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.163:0/3896543132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:04:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:37 smithi181 bash[27712]: audit 2023-10-09T09:04:36.455540+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:04:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:37 smithi181 bash[27712]: audit 2023-10-09T09:04:36.659666+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.163:0/614946055' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:04:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:37 smithi181 bash[27712]: audit 2023-10-09T09:04:36.670403+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:04:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:37 smithi181 bash[27712]: audit 2023-10-09T09:04:36.723281+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:04:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:37 smithi181 bash[27712]: audit 2023-10-09T09:04:37.000456+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:04:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:37 smithi181 bash[27712]: audit 2023-10-09T09:04:37.001202+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:04:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:37 smithi181 bash[27712]: audit 2023-10-09T09:04:37.006303+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:04:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[27500]: audit 2023-10-09T09:04:36.351269+0000 mon.a (mon.0) 733 : audit [DBG] from='client.? 172.21.15.163:0/3789634875' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:04:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[27500]: audit 2023-10-09T09:04:36.364603+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.163:0/3896543132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:04:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[27500]: audit 2023-10-09T09:04:36.455540+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:04:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[27500]: audit 2023-10-09T09:04:36.659666+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.163:0/614946055' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:04:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[27500]: audit 2023-10-09T09:04:36.670403+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:04:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[27500]: audit 2023-10-09T09:04:36.723281+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:04:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[27500]: audit 2023-10-09T09:04:37.000456+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:04:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[27500]: audit 2023-10-09T09:04:37.001202+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:04:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[27500]: audit 2023-10-09T09:04:37.006303+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:04:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[33912]: audit 2023-10-09T09:04:36.351269+0000 mon.a (mon.0) 733 : audit [DBG] from='client.? 172.21.15.163:0/3789634875' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:04:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[33912]: audit 2023-10-09T09:04:36.364603+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.163:0/3896543132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:04:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[33912]: audit 2023-10-09T09:04:36.455540+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:04:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[33912]: audit 2023-10-09T09:04:36.659666+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.163:0/614946055' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:04:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[33912]: audit 2023-10-09T09:04:36.670403+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:04:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[33912]: audit 2023-10-09T09:04:36.723281+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:04:37.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[33912]: audit 2023-10-09T09:04:37.000456+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:04:37.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[33912]: audit 2023-10-09T09:04:37.001202+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:04:37.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:37 smithi163 bash[33912]: audit 2023-10-09T09:04:37.006303+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:04:37.969 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986248 got 133143986252 for osd.4 2023-10-09T09:04:37.970 DEBUG:teuthology.parallel:result is None 2023-10-09T09:04:37.993 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738472 got 34359738476 for osd.0 2023-10-09T09:04:37.993 DEBUG:teuthology.parallel:result is None 2023-10-09T09:04:38.171 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574944 got 55834574948 for osd.1 2023-10-09T09:04:38.171 DEBUG:teuthology.parallel:result is None 2023-10-09T09:04:38.385 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790014 got 158913790018 for osd.5 2023-10-09T09:04:38.385 DEBUG:teuthology.parallel:result is None 2023-10-09T09:04:38.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:38 smithi163 bash[27500]: cluster 2023-10-09T09:04:37.230331+0000 mgr.y (mgr.24355) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:38.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:38 smithi163 bash[33912]: cluster 2023-10-09T09:04:37.230331+0000 mgr.y (mgr.24355) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:38 smithi181 bash[27712]: cluster 2023-10-09T09:04:37.230331+0000 mgr.y (mgr.24355) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:38.568 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:04:38 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:04:39.088 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182479 got 107374182483 for osd.3 2023-10-09T09:04:39.088 DEBUG:teuthology.parallel:result is None 2023-10-09T09:04:39.098 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397550 got 210453397554 for osd.7 2023-10-09T09:04:39.098 DEBUG:teuthology.parallel:result is None 2023-10-09T09:04:39.098 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T09:04:39.099 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:04:40.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:04:39 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:39] "GET /metrics HTTP/1.1" 200 191380 "" "Prometheus/2.33.4" 2023-10-09T09:04:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:40 smithi163 bash[27500]: cluster 2023-10-09T09:04:39.231087+0000 mgr.y (mgr.24355) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:40 smithi163 bash[33912]: cluster 2023-10-09T09:04:39.231087+0000 mgr.y (mgr.24355) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:40 smithi181 bash[27712]: cluster 2023-10-09T09:04:39.231087+0000 mgr.y (mgr.24355) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:42 smithi163 bash[33912]: cluster 2023-10-09T09:04:41.231624+0000 mgr.y (mgr.24355) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:42 smithi163 bash[27500]: cluster 2023-10-09T09:04:41.231624+0000 mgr.y (mgr.24355) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:42 smithi181 bash[27712]: cluster 2023-10-09T09:04:41.231624+0000 mgr.y (mgr.24355) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:43.922 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:44 smithi163 bash[27500]: cluster 2023-10-09T09:04:43.232569+0000 mgr.y (mgr.24355) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:44 smithi163 bash[33912]: cluster 2023-10-09T09:04:43.232569+0000 mgr.y (mgr.24355) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:44 smithi181 bash[27712]: cluster 2023-10-09T09:04:43.232569+0000 mgr.y (mgr.24355) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:46 smithi163 bash[27500]: cluster 2023-10-09T09:04:45.233115+0000 mgr.y (mgr.24355) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:46 smithi163 bash[33912]: cluster 2023-10-09T09:04:45.233115+0000 mgr.y (mgr.24355) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:46.758 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:04:46.760 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:04:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:46 smithi181 bash[27712]: cluster 2023-10-09T09:04:45.233115+0000 mgr.y (mgr.24355) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:47.412 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":123,"stamp":"2023-10-09T09:04:45.232880+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381804,"kb_used_data":5036,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579412,"statfs":{"total":767960285184,"available":765521317888,"internally_reserved":0,"allocated":5156864,"data_stored":2195935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004105"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063994+0000","last_change":"2023-10-09T09:03:25.312556+0000","last_active":"2023-10-09T09:04:08.063994+0000","last_peered":"2023-10-09T09:04:08.063994+0000","last_clean":"2023-10-09T09:04:08.063994+0000","last_became_active":"2023-10-09T09:03:25.312265+0000","last_became_peered":"2023-10-09T09:03:25.312265+0000","last_unstale":"2023-10-09T09:04:08.063994+0000","last_undegraded":"2023-10-09T09:04:08.063994+0000","last_fullsized":"2023-10-09T09:04:08.063994+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060045+0000","last_change":"2023-10-09T09:03:24.307630+0000","last_active":"2023-10-09T09:04:08.060045+0000","last_peered":"2023-10-09T09:04:08.060045+0000","last_clean":"2023-10-09T09:04:08.060045+0000","last_became_active":"2023-10-09T09:03:24.307313+0000","last_became_peered":"2023-10-09T09:03:24.307313+0000","last_unstale":"2023-10-09T09:04:08.060045+0000","last_undegraded":"2023-10-09T09:04:08.060045+0000","last_fullsized":"2023-10-09T09:04:08.060045+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062892+0000","last_change":"2023-10-09T09:03:25.320580+0000","last_active":"2023-10-09T09:04:08.062892+0000","last_peered":"2023-10-09T09:04:08.062892+0000","last_clean":"2023-10-09T09:04:08.062892+0000","last_became_active":"2023-10-09T09:03:25.320251+0000","last_became_peered":"2023-10-09T09:03:25.320251+0000","last_unstale":"2023-10-09T09:04:08.062892+0000","last_undegraded":"2023-10-09T09:04:08.062892+0000","last_fullsized":"2023-10-09T09:04:08.062892+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063606+0000","last_change":"2023-10-09T09:03:24.312895+0000","last_active":"2023-10-09T09:04:08.063606+0000","last_peered":"2023-10-09T09:04:08.063606+0000","last_clean":"2023-10-09T09:04:08.063606+0000","last_became_active":"2023-10-09T09:03:24.312535+0000","last_became_peered":"2023-10-09T09:03:24.312535+0000","last_unstale":"2023-10-09T09:04:08.063606+0000","last_undegraded":"2023-10-09T09:04:08.063606+0000","last_fullsized":"2023-10-09T09:04:08.063606+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064297+0000","last_change":"2023-10-09T09:03:25.318238+0000","last_active":"2023-10-09T09:04:08.064297+0000","last_peered":"2023-10-09T09:04:08.064297+0000","last_clean":"2023-10-09T09:04:08.064297+0000","last_became_active":"2023-10-09T09:03:25.317878+0000","last_became_peered":"2023-10-09T09:03:25.317878+0000","last_unstale":"2023-10-09T09:04:08.064297+0000","last_undegraded":"2023-10-09T09:04:08.064297+0000","last_fullsized":"2023-10-09T09:04:08.064297+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061651+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:04:08.061651+0000","last_peered":"2023-10-09T09:04:08.061651+0000","last_clean":"2023-10-09T09:04:08.061651+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:04:08.061651+0000","last_undegraded":"2023-10-09T09:04:08.061651+0000","last_fullsized":"2023-10-09T09:04:08.061651+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064799+0000","last_change":"2023-10-09T09:03:25.321650+0000","last_active":"2023-10-09T09:04:08.064799+0000","last_peered":"2023-10-09T09:04:08.064799+0000","last_clean":"2023-10-09T09:04:08.064799+0000","last_became_active":"2023-10-09T09:03:25.321288+0000","last_became_peered":"2023-10-09T09:03:25.321288+0000","last_unstale":"2023-10-09T09:04:08.064799+0000","last_undegraded":"2023-10-09T09:04:08.064799+0000","last_fullsized":"2023-10-09T09:04:08.064799+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061301+0000","last_change":"2023-10-09T09:03:24.309727+0000","last_active":"2023-10-09T09:04:08.061301+0000","last_peered":"2023-10-09T09:04:08.061301+0000","last_clean":"2023-10-09T09:04:08.061301+0000","last_became_active":"2023-10-09T09:03:24.309132+0000","last_became_peered":"2023-10-09T09:03:24.309132+0000","last_unstale":"2023-10-09T09:04:08.061301+0000","last_undegraded":"2023-10-09T09:04:08.061301+0000","last_fullsized":"2023-10-09T09:04:08.061301+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063342+0000","last_change":"2023-10-09T09:03:25.320981+0000","last_active":"2023-10-09T09:04:08.063342+0000","last_peered":"2023-10-09T09:04:08.063342+0000","last_clean":"2023-10-09T09:04:08.063342+0000","last_became_active":"2023-10-09T09:03:25.320532+0000","last_became_peered":"2023-10-09T09:03:25.320532+0000","last_unstale":"2023-10-09T09:04:08.063342+0000","last_undegraded":"2023-10-09T09:04:08.063342+0000","last_fullsized":"2023-10-09T09:04:08.063342+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059468+0000","last_change":"2023-10-09T09:03:24.306211+0000","last_active":"2023-10-09T09:04:08.059468+0000","last_peered":"2023-10-09T09:04:08.059468+0000","last_clean":"2023-10-09T09:04:08.059468+0000","last_became_active":"2023-10-09T09:03:24.304771+0000","last_became_peered":"2023-10-09T09:03:24.304771+0000","last_unstale":"2023-10-09T09:04:08.059468+0000","last_undegraded":"2023-10-09T09:04:08.059468+0000","last_fullsized":"2023-10-09T09:04:08.059468+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064285+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:04:08.064285+0000","last_peered":"2023-10-09T09:04:08.064285+0000","last_clean":"2023-10-09T09:04:08.064285+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:04:08.064285+0000","last_undegraded":"2023-10-09T09:04:08.064285+0000","last_fullsized":"2023-10-09T09:04:08.064285+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059635+0000","last_change":"2023-10-09T09:03:24.313388+0000","last_active":"2023-10-09T09:04:08.059635+0000","last_peered":"2023-10-09T09:04:08.059635+0000","last_clean":"2023-10-09T09:04:08.059635+0000","last_became_active":"2023-10-09T09:03:24.313064+0000","last_became_peered":"2023-10-09T09:03:24.313064+0000","last_unstale":"2023-10-09T09:04:08.059635+0000","last_undegraded":"2023-10-09T09:04:08.059635+0000","last_fullsized":"2023-10-09T09:04:08.059635+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.783114+0000","last_change":"2023-10-09T09:01:34.369352+0000","last_active":"2023-10-09T09:04:08.783114+0000","last_peered":"2023-10-09T09:04:08.783114+0000","last_clean":"2023-10-09T09:04:08.783114+0000","last_became_active":"2023-10-09T09:01:34.368090+0000","last_became_peered":"2023-10-09T09:01:34.368090+0000","last_unstale":"2023-10-09T09:04:08.783114+0000","last_undegraded":"2023-10-09T09:04:08.783114+0000","last_fullsized":"2023-10-09T09:04:08.783114+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062696+0000","last_change":"2023-10-09T09:03:25.757376+0000","last_active":"2023-10-09T09:04:08.062696+0000","last_peered":"2023-10-09T09:04:08.062696+0000","last_clean":"2023-10-09T09:04:08.062696+0000","last_became_active":"2023-10-09T09:03:25.756809+0000","last_became_peered":"2023-10-09T09:03:25.756809+0000","last_unstale":"2023-10-09T09:04:08.062696+0000","last_undegraded":"2023-10-09T09:04:08.062696+0000","last_fullsized":"2023-10-09T09:04:08.062696+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060789+0000","last_change":"2023-10-09T09:01:34.370339+0000","last_active":"2023-10-09T09:04:08.060789+0000","last_peered":"2023-10-09T09:04:08.060789+0000","last_clean":"2023-10-09T09:04:08.060789+0000","last_became_active":"2023-10-09T09:01:34.369975+0000","last_became_peered":"2023-10-09T09:01:34.369975+0000","last_unstale":"2023-10-09T09:04:08.060789+0000","last_undegraded":"2023-10-09T09:04:08.060789+0000","last_fullsized":"2023-10-09T09:04:08.060789+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064161+0000","last_change":"2023-10-09T09:03:24.315589+0000","last_active":"2023-10-09T09:04:08.064161+0000","last_peered":"2023-10-09T09:04:08.064161+0000","last_clean":"2023-10-09T09:04:08.064161+0000","last_became_active":"2023-10-09T09:03:24.315228+0000","last_became_peered":"2023-10-09T09:03:24.315228+0000","last_unstale":"2023-10-09T09:04:08.064161+0000","last_undegraded":"2023-10-09T09:04:08.064161+0000","last_fullsized":"2023-10-09T09:04:08.064161+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059426+0000","last_change":"2023-10-09T09:03:25.757066+0000","last_active":"2023-10-09T09:04:08.059426+0000","last_peered":"2023-10-09T09:04:08.059426+0000","last_clean":"2023-10-09T09:04:08.059426+0000","last_became_active":"2023-10-09T09:03:25.756676+0000","last_became_peered":"2023-10-09T09:03:25.756676+0000","last_unstale":"2023-10-09T09:04:08.059426+0000","last_undegraded":"2023-10-09T09:04:08.059426+0000","last_fullsized":"2023-10-09T09:04:08.059426+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061098+0000","last_change":"2023-10-09T09:03:24.314055+0000","last_active":"2023-10-09T09:04:08.061098+0000","last_peered":"2023-10-09T09:04:08.061098+0000","last_clean":"2023-10-09T09:04:08.061098+0000","last_became_active":"2023-10-09T09:03:24.313634+0000","last_became_peered":"2023-10-09T09:03:24.313634+0000","last_unstale":"2023-10-09T09:04:08.061098+0000","last_undegraded":"2023-10-09T09:04:08.061098+0000","last_fullsized":"2023-10-09T09:04:08.061098+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058742+0000","last_change":"2023-10-09T09:01:34.370949+0000","last_active":"2023-10-09T09:04:08.058742+0000","last_peered":"2023-10-09T09:04:08.058742+0000","last_clean":"2023-10-09T09:04:08.058742+0000","last_became_active":"2023-10-09T09:01:34.370529+0000","last_became_peered":"2023-10-09T09:01:34.370529+0000","last_unstale":"2023-10-09T09:04:08.058742+0000","last_undegraded":"2023-10-09T09:04:08.058742+0000","last_fullsized":"2023-10-09T09:04:08.058742+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T09:04:09.953251+0000","last_change":"2023-10-09T09:04:09.953251+0000","last_active":"2023-10-09T09:04:09.953251+0000","last_peered":"2023-10-09T09:04:09.953251+0000","last_clean":"2023-10-09T09:04:09.953251+0000","last_became_active":"2023-10-09T09:04:09.952651+0000","last_became_peered":"2023-10-09T09:04:09.952651+0000","last_unstale":"2023-10-09T09:04:09.953251+0000","last_undegraded":"2023-10-09T09:04:09.953251+0000","last_fullsized":"2023-10-09T09:04:09.953251+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059096+0000","last_change":"2023-10-09T09:03:24.312793+0000","last_active":"2023-10-09T09:04:08.059096+0000","last_peered":"2023-10-09T09:04:08.059096+0000","last_clean":"2023-10-09T09:04:08.059096+0000","last_became_active":"2023-10-09T09:03:24.312367+0000","last_became_peered":"2023-10-09T09:03:24.312367+0000","last_unstale":"2023-10-09T09:04:08.059096+0000","last_undegraded":"2023-10-09T09:04:08.059096+0000","last_fullsized":"2023-10-09T09:04:08.059096+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058343+0000","last_change":"2023-10-09T09:01:34.369805+0000","last_active":"2023-10-09T09:04:08.058343+0000","last_peered":"2023-10-09T09:04:08.058343+0000","last_clean":"2023-10-09T09:04:08.058343+0000","last_became_active":"2023-10-09T09:01:34.369226+0000","last_became_peered":"2023-10-09T09:01:34.369226+0000","last_unstale":"2023-10-09T09:04:08.058343+0000","last_undegraded":"2023-10-09T09:04:08.058343+0000","last_fullsized":"2023-10-09T09:04:08.058343+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.783125+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:04:08.783125+0000","last_peered":"2023-10-09T09:04:08.783125+0000","last_clean":"2023-10-09T09:04:08.783125+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:04:08.783125+0000","last_undegraded":"2023-10-09T09:04:08.783125+0000","last_fullsized":"2023-10-09T09:04:08.783125+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062912+0000","last_change":"2023-10-09T09:03:24.312843+0000","last_active":"2023-10-09T09:04:08.062912+0000","last_peered":"2023-10-09T09:04:08.062912+0000","last_clean":"2023-10-09T09:04:08.062912+0000","last_became_active":"2023-10-09T09:03:24.312398+0000","last_became_peered":"2023-10-09T09:03:24.312398+0000","last_unstale":"2023-10-09T09:04:08.062912+0000","last_undegraded":"2023-10-09T09:04:08.062912+0000","last_fullsized":"2023-10-09T09:04:08.062912+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058301+0000","last_change":"2023-10-09T09:00:37.999371+0000","last_active":"2023-10-09T09:04:08.058301+0000","last_peered":"2023-10-09T09:04:08.058301+0000","last_clean":"2023-10-09T09:04:08.058301+0000","last_became_active":"2023-10-09T09:00:37.998968+0000","last_became_peered":"2023-10-09T09:00:37.998968+0000","last_unstale":"2023-10-09T09:04:08.058301+0000","last_undegraded":"2023-10-09T09:04:08.058301+0000","last_fullsized":"2023-10-09T09:04:08.058301+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061393+0000","last_change":"2023-10-09T09:01:34.371076+0000","last_active":"2023-10-09T09:04:08.061393+0000","last_peered":"2023-10-09T09:04:08.061393+0000","last_clean":"2023-10-09T09:04:08.061393+0000","last_became_active":"2023-10-09T09:01:34.370690+0000","last_became_peered":"2023-10-09T09:01:34.370690+0000","last_unstale":"2023-10-09T09:04:08.061393+0000","last_undegraded":"2023-10-09T09:04:08.061393+0000","last_fullsized":"2023-10-09T09:04:08.061393+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060860+0000","last_change":"2023-10-09T09:03:25.329498+0000","last_active":"2023-10-09T09:04:08.060860+0000","last_peered":"2023-10-09T09:04:08.060860+0000","last_clean":"2023-10-09T09:04:08.060860+0000","last_became_active":"2023-10-09T09:03:25.329219+0000","last_became_peered":"2023-10-09T09:03:25.329219+0000","last_unstale":"2023-10-09T09:04:08.060860+0000","last_undegraded":"2023-10-09T09:04:08.060860+0000","last_fullsized":"2023-10-09T09:04:08.060860+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060028+0000","last_change":"2023-10-09T09:03:24.299512+0000","last_active":"2023-10-09T09:04:08.060028+0000","last_peered":"2023-10-09T09:04:08.060028+0000","last_clean":"2023-10-09T09:04:08.060028+0000","last_became_active":"2023-10-09T09:03:24.299168+0000","last_became_peered":"2023-10-09T09:03:24.299168+0000","last_unstale":"2023-10-09T09:04:08.060028+0000","last_undegraded":"2023-10-09T09:04:08.060028+0000","last_fullsized":"2023-10-09T09:04:08.060028+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062493+0000","last_change":"2023-10-09T09:01:34.897330+0000","last_active":"2023-10-09T09:04:08.062493+0000","last_peered":"2023-10-09T09:04:08.062493+0000","last_clean":"2023-10-09T09:04:08.062493+0000","last_became_active":"2023-10-09T09:01:34.896715+0000","last_became_peered":"2023-10-09T09:01:34.896715+0000","last_unstale":"2023-10-09T09:04:08.062493+0000","last_undegraded":"2023-10-09T09:04:08.062493+0000","last_fullsized":"2023-10-09T09:04:08.062493+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061002+0000","last_change":"2023-10-09T09:03:25.327061+0000","last_active":"2023-10-09T09:04:08.061002+0000","last_peered":"2023-10-09T09:04:08.061002+0000","last_clean":"2023-10-09T09:04:08.061002+0000","last_became_active":"2023-10-09T09:03:25.326447+0000","last_became_peered":"2023-10-09T09:03:25.326447+0000","last_unstale":"2023-10-09T09:04:08.061002+0000","last_undegraded":"2023-10-09T09:04:08.061002+0000","last_fullsized":"2023-10-09T09:04:08.061002+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064444+0000","last_change":"2023-10-09T09:03:24.301505+0000","last_active":"2023-10-09T09:04:08.064444+0000","last_peered":"2023-10-09T09:04:08.064444+0000","last_clean":"2023-10-09T09:04:08.064444+0000","last_became_active":"2023-10-09T09:03:24.301139+0000","last_became_peered":"2023-10-09T09:03:24.301139+0000","last_unstale":"2023-10-09T09:04:08.064444+0000","last_undegraded":"2023-10-09T09:04:08.064444+0000","last_fullsized":"2023-10-09T09:04:08.064444+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061212+0000","last_change":"2023-10-09T09:01:34.372320+0000","last_active":"2023-10-09T09:04:08.061212+0000","last_peered":"2023-10-09T09:04:08.061212+0000","last_clean":"2023-10-09T09:04:08.061212+0000","last_became_active":"2023-10-09T09:01:34.371852+0000","last_became_peered":"2023-10-09T09:01:34.371852+0000","last_unstale":"2023-10-09T09:04:08.061212+0000","last_undegraded":"2023-10-09T09:04:08.061212+0000","last_fullsized":"2023-10-09T09:04:08.061212+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059929+0000","last_change":"2023-10-09T09:03:25.321662+0000","last_active":"2023-10-09T09:04:08.059929+0000","last_peered":"2023-10-09T09:04:08.059929+0000","last_clean":"2023-10-09T09:04:08.059929+0000","last_became_active":"2023-10-09T09:03:25.321307+0000","last_became_peered":"2023-10-09T09:03:25.321307+0000","last_unstale":"2023-10-09T09:04:08.059929+0000","last_undegraded":"2023-10-09T09:04:08.059929+0000","last_fullsized":"2023-10-09T09:04:08.059929+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061649+0000","last_change":"2023-10-09T09:03:24.309025+0000","last_active":"2023-10-09T09:04:08.061649+0000","last_peered":"2023-10-09T09:04:08.061649+0000","last_clean":"2023-10-09T09:04:08.061649+0000","last_became_active":"2023-10-09T09:03:24.308645+0000","last_became_peered":"2023-10-09T09:03:24.308645+0000","last_unstale":"2023-10-09T09:04:08.061649+0000","last_undegraded":"2023-10-09T09:04:08.061649+0000","last_fullsized":"2023-10-09T09:04:08.061649+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060910+0000","last_change":"2023-10-09T09:01:34.370258+0000","last_active":"2023-10-09T09:04:08.060910+0000","last_peered":"2023-10-09T09:04:08.060910+0000","last_clean":"2023-10-09T09:04:08.060910+0000","last_became_active":"2023-10-09T09:01:34.369813+0000","last_became_peered":"2023-10-09T09:01:34.369813+0000","last_unstale":"2023-10-09T09:04:08.060910+0000","last_undegraded":"2023-10-09T09:04:08.060910+0000","last_fullsized":"2023-10-09T09:04:08.060910+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060856+0000","last_change":"2023-10-09T09:03:25.322568+0000","last_active":"2023-10-09T09:04:08.060856+0000","last_peered":"2023-10-09T09:04:08.060856+0000","last_clean":"2023-10-09T09:04:08.060856+0000","last_became_active":"2023-10-09T09:03:25.322259+0000","last_became_peered":"2023-10-09T09:03:25.322259+0000","last_unstale":"2023-10-09T09:04:08.060856+0000","last_undegraded":"2023-10-09T09:04:08.060856+0000","last_fullsized":"2023-10-09T09:04:08.060856+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062819+0000","last_change":"2023-10-09T09:03:24.308720+0000","last_active":"2023-10-09T09:04:08.062819+0000","last_peered":"2023-10-09T09:04:08.062819+0000","last_clean":"2023-10-09T09:04:08.062819+0000","last_became_active":"2023-10-09T09:03:24.308389+0000","last_became_peered":"2023-10-09T09:03:24.308389+0000","last_unstale":"2023-10-09T09:04:08.062819+0000","last_undegraded":"2023-10-09T09:04:08.062819+0000","last_fullsized":"2023-10-09T09:04:08.062819+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064992+0000","last_change":"2023-10-09T09:03:25.323242+0000","last_active":"2023-10-09T09:04:08.064992+0000","last_peered":"2023-10-09T09:04:08.064992+0000","last_clean":"2023-10-09T09:04:08.064992+0000","last_became_active":"2023-10-09T09:03:25.321463+0000","last_became_peered":"2023-10-09T09:03:25.321463+0000","last_unstale":"2023-10-09T09:04:08.064992+0000","last_undegraded":"2023-10-09T09:04:08.064992+0000","last_fullsized":"2023-10-09T09:04:08.064992+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064278+0000","last_change":"2023-10-09T09:03:24.303003+0000","last_active":"2023-10-09T09:04:08.064278+0000","last_peered":"2023-10-09T09:04:08.064278+0000","last_clean":"2023-10-09T09:04:08.064278+0000","last_became_active":"2023-10-09T09:03:24.302583+0000","last_became_peered":"2023-10-09T09:03:24.302583+0000","last_unstale":"2023-10-09T09:04:08.064278+0000","last_undegraded":"2023-10-09T09:04:08.064278+0000","last_fullsized":"2023-10-09T09:04:08.064278+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064796+0000","last_change":"2023-10-09T09:03:25.321487+0000","last_active":"2023-10-09T09:04:08.064796+0000","last_peered":"2023-10-09T09:04:08.064796+0000","last_clean":"2023-10-09T09:04:08.064796+0000","last_became_active":"2023-10-09T09:03:25.321142+0000","last_became_peered":"2023-10-09T09:03:25.321142+0000","last_unstale":"2023-10-09T09:04:08.064796+0000","last_undegraded":"2023-10-09T09:04:08.064796+0000","last_fullsized":"2023-10-09T09:04:08.064796+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.782924+0000","last_change":"2023-10-09T09:03:24.307481+0000","last_active":"2023-10-09T09:04:08.782924+0000","last_peered":"2023-10-09T09:04:08.782924+0000","last_clean":"2023-10-09T09:04:08.782924+0000","last_became_active":"2023-10-09T09:03:24.307102+0000","last_became_peered":"2023-10-09T09:03:24.307102+0000","last_unstale":"2023-10-09T09:04:08.782924+0000","last_undegraded":"2023-10-09T09:04:08.782924+0000","last_fullsized":"2023-10-09T09:04:08.782924+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064631+0000","last_change":"2023-10-09T09:03:25.318434+0000","last_active":"2023-10-09T09:04:08.064631+0000","last_peered":"2023-10-09T09:04:08.064631+0000","last_clean":"2023-10-09T09:04:08.064631+0000","last_became_active":"2023-10-09T09:03:25.317989+0000","last_became_peered":"2023-10-09T09:03:25.317989+0000","last_unstale":"2023-10-09T09:04:08.064631+0000","last_undegraded":"2023-10-09T09:04:08.064631+0000","last_fullsized":"2023-10-09T09:04:08.064631+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063179+0000","last_change":"2023-10-09T09:03:24.317705+0000","last_active":"2023-10-09T09:04:08.063179+0000","last_peered":"2023-10-09T09:04:08.063179+0000","last_clean":"2023-10-09T09:04:08.063179+0000","last_became_active":"2023-10-09T09:03:24.317211+0000","last_became_peered":"2023-10-09T09:03:24.317211+0000","last_unstale":"2023-10-09T09:04:08.063179+0000","last_undegraded":"2023-10-09T09:04:08.063179+0000","last_fullsized":"2023-10-09T09:04:08.063179+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059281+0000","last_change":"2023-10-09T09:03:25.320402+0000","last_active":"2023-10-09T09:04:08.059281+0000","last_peered":"2023-10-09T09:04:08.059281+0000","last_clean":"2023-10-09T09:04:08.059281+0000","last_became_active":"2023-10-09T09:03:25.320032+0000","last_became_peered":"2023-10-09T09:03:25.320032+0000","last_unstale":"2023-10-09T09:04:08.059281+0000","last_undegraded":"2023-10-09T09:04:08.059281+0000","last_fullsized":"2023-10-09T09:04:08.059281+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061185+0000","last_change":"2023-10-09T09:03:24.308119+0000","last_active":"2023-10-09T09:04:08.061185+0000","last_peered":"2023-10-09T09:04:08.061185+0000","last_clean":"2023-10-09T09:04:08.061185+0000","last_became_active":"2023-10-09T09:03:24.307416+0000","last_became_peered":"2023-10-09T09:03:24.307416+0000","last_unstale":"2023-10-09T09:04:08.061185+0000","last_undegraded":"2023-10-09T09:04:08.061185+0000","last_fullsized":"2023-10-09T09:04:08.061185+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.782785+0000","last_change":"2023-10-09T09:03:25.756450+0000","last_active":"2023-10-09T09:04:08.782785+0000","last_peered":"2023-10-09T09:04:08.782785+0000","last_clean":"2023-10-09T09:04:08.782785+0000","last_became_active":"2023-10-09T09:03:25.756113+0000","last_became_peered":"2023-10-09T09:03:25.756113+0000","last_unstale":"2023-10-09T09:04:08.782785+0000","last_undegraded":"2023-10-09T09:04:08.782785+0000","last_fullsized":"2023-10-09T09:04:08.782785+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058786+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:04:08.058786+0000","last_peered":"2023-10-09T09:04:08.058786+0000","last_clean":"2023-10-09T09:04:08.058786+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:04:08.058786+0000","last_undegraded":"2023-10-09T09:04:08.058786+0000","last_fullsized":"2023-10-09T09:04:08.058786+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064116+0000","last_change":"2023-10-09T09:03:25.318210+0000","last_active":"2023-10-09T09:04:08.064116+0000","last_peered":"2023-10-09T09:04:08.064116+0000","last_clean":"2023-10-09T09:04:08.064116+0000","last_became_active":"2023-10-09T09:03:25.317818+0000","last_became_peered":"2023-10-09T09:03:25.317818+0000","last_unstale":"2023-10-09T09:04:08.064116+0000","last_undegraded":"2023-10-09T09:04:08.064116+0000","last_fullsized":"2023-10-09T09:04:08.064116+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059111+0000","last_change":"2023-10-09T09:03:24.311122+0000","last_active":"2023-10-09T09:04:08.059111+0000","last_peered":"2023-10-09T09:04:08.059111+0000","last_clean":"2023-10-09T09:04:08.059111+0000","last_became_active":"2023-10-09T09:03:24.310080+0000","last_became_peered":"2023-10-09T09:03:24.310080+0000","last_unstale":"2023-10-09T09:04:08.059111+0000","last_undegraded":"2023-10-09T09:04:08.059111+0000","last_fullsized":"2023-10-09T09:04:08.059111+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061156+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:04:08.061156+0000","last_peered":"2023-10-09T09:04:08.061156+0000","last_clean":"2023-10-09T09:04:08.061156+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:04:08.061156+0000","last_undegraded":"2023-10-09T09:04:08.061156+0000","last_fullsized":"2023-10-09T09:04:08.061156+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059390+0000","last_change":"2023-10-09T09:03:24.310831+0000","last_active":"2023-10-09T09:04:08.059390+0000","last_peered":"2023-10-09T09:04:08.059390+0000","last_clean":"2023-10-09T09:04:08.059390+0000","last_became_active":"2023-10-09T09:03:24.310045+0000","last_became_peered":"2023-10-09T09:03:24.310045+0000","last_unstale":"2023-10-09T09:04:08.059390+0000","last_undegraded":"2023-10-09T09:04:08.059390+0000","last_fullsized":"2023-10-09T09:04:08.059390+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T09:04:09.954296+0000","last_change":"2023-10-09T09:04:09.954296+0000","last_active":"2023-10-09T09:04:09.954296+0000","last_peered":"2023-10-09T09:04:09.954296+0000","last_clean":"2023-10-09T09:04:09.954296+0000","last_became_active":"2023-10-09T09:04:09.953904+0000","last_became_peered":"2023-10-09T09:04:09.953904+0000","last_unstale":"2023-10-09T09:04:09.954296+0000","last_undegraded":"2023-10-09T09:04:09.954296+0000","last_fullsized":"2023-10-09T09:04:09.954296+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.782664+0000","last_change":"2023-10-09T09:03:24.310906+0000","last_active":"2023-10-09T09:04:08.782664+0000","last_peered":"2023-10-09T09:04:08.782664+0000","last_clean":"2023-10-09T09:04:08.782664+0000","last_became_active":"2023-10-09T09:03:24.310576+0000","last_became_peered":"2023-10-09T09:03:24.310576+0000","last_unstale":"2023-10-09T09:04:08.782664+0000","last_undegraded":"2023-10-09T09:04:08.782664+0000","last_fullsized":"2023-10-09T09:04:08.782664+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059416+0000","last_change":"2023-10-09T09:03:25.756017+0000","last_active":"2023-10-09T09:04:08.059416+0000","last_peered":"2023-10-09T09:04:08.059416+0000","last_clean":"2023-10-09T09:04:08.059416+0000","last_became_active":"2023-10-09T09:03:25.755602+0000","last_became_peered":"2023-10-09T09:03:25.755602+0000","last_unstale":"2023-10-09T09:04:08.059416+0000","last_undegraded":"2023-10-09T09:04:08.059416+0000","last_fullsized":"2023-10-09T09:04:08.059416+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058339+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:04:08.058339+0000","last_peered":"2023-10-09T09:04:08.058339+0000","last_clean":"2023-10-09T09:04:08.058339+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:04:08.058339+0000","last_undegraded":"2023-10-09T09:04:08.058339+0000","last_fullsized":"2023-10-09T09:04:08.058339+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061425+0000","last_change":"2023-10-09T09:03:25.757962+0000","last_active":"2023-10-09T09:04:08.061425+0000","last_peered":"2023-10-09T09:04:08.061425+0000","last_clean":"2023-10-09T09:04:08.061425+0000","last_became_active":"2023-10-09T09:03:25.757454+0000","last_became_peered":"2023-10-09T09:03:25.757454+0000","last_unstale":"2023-10-09T09:04:08.061425+0000","last_undegraded":"2023-10-09T09:04:08.061425+0000","last_fullsized":"2023-10-09T09:04:08.061425+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059166+0000","last_change":"2023-10-09T09:03:24.311042+0000","last_active":"2023-10-09T09:04:08.059166+0000","last_peered":"2023-10-09T09:04:08.059166+0000","last_clean":"2023-10-09T09:04:08.059166+0000","last_became_active":"2023-10-09T09:03:24.310650+0000","last_became_peered":"2023-10-09T09:03:24.310650+0000","last_unstale":"2023-10-09T09:04:08.059166+0000","last_undegraded":"2023-10-09T09:04:08.059166+0000","last_fullsized":"2023-10-09T09:04:08.059166+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063527+0000","last_change":"2023-10-09T09:03:25.756248+0000","last_active":"2023-10-09T09:04:08.063527+0000","last_peered":"2023-10-09T09:04:08.063527+0000","last_clean":"2023-10-09T09:04:08.063527+0000","last_became_active":"2023-10-09T09:03:25.755903+0000","last_became_peered":"2023-10-09T09:03:25.755903+0000","last_unstale":"2023-10-09T09:04:08.063527+0000","last_undegraded":"2023-10-09T09:04:08.063527+0000","last_fullsized":"2023-10-09T09:04:08.063527+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058740+0000","last_change":"2023-10-09T09:03:24.304393+0000","last_active":"2023-10-09T09:04:08.058740+0000","last_peered":"2023-10-09T09:04:08.058740+0000","last_clean":"2023-10-09T09:04:08.058740+0000","last_became_active":"2023-10-09T09:03:24.304006+0000","last_became_peered":"2023-10-09T09:03:24.304006+0000","last_unstale":"2023-10-09T09:04:08.058740+0000","last_undegraded":"2023-10-09T09:04:08.058740+0000","last_fullsized":"2023-10-09T09:04:08.058740+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064705+0000","last_change":"2023-10-09T09:03:25.322358+0000","last_active":"2023-10-09T09:04:08.064705+0000","last_peered":"2023-10-09T09:04:08.064705+0000","last_clean":"2023-10-09T09:04:08.064705+0000","last_became_active":"2023-10-09T09:03:25.321970+0000","last_became_peered":"2023-10-09T09:03:25.321970+0000","last_unstale":"2023-10-09T09:04:08.064705+0000","last_undegraded":"2023-10-09T09:04:08.064705+0000","last_fullsized":"2023-10-09T09:04:08.064705+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"61'6","reported_seq":20,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064487+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:04:08.064487+0000","last_peered":"2023-10-09T09:04:08.064487+0000","last_clean":"2023-10-09T09:04:08.064487+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:04:08.064487+0000","last_undegraded":"2023-10-09T09:04:08.064487+0000","last_fullsized":"2023-10-09T09:04:08.064487+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059931+0000","last_change":"2023-10-09T09:03:25.324660+0000","last_active":"2023-10-09T09:04:08.059931+0000","last_peered":"2023-10-09T09:04:08.059931+0000","last_clean":"2023-10-09T09:04:08.059931+0000","last_became_active":"2023-10-09T09:03:25.324342+0000","last_became_peered":"2023-10-09T09:03:25.324342+0000","last_unstale":"2023-10-09T09:04:08.059931+0000","last_undegraded":"2023-10-09T09:04:08.059931+0000","last_fullsized":"2023-10-09T09:04:08.059931+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":17,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058651+0000","last_change":"2023-10-09T09:03:24.312250+0000","last_active":"2023-10-09T09:04:08.058651+0000","last_peered":"2023-10-09T09:04:08.058651+0000","last_clean":"2023-10-09T09:04:08.058651+0000","last_became_active":"2023-10-09T09:03:24.310944+0000","last_became_peered":"2023-10-09T09:03:24.310944+0000","last_unstale":"2023-10-09T09:04:08.058651+0000","last_undegraded":"2023-10-09T09:04:08.058651+0000","last_fullsized":"2023-10-09T09:04:08.058651+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062998+0000","last_change":"2023-10-09T09:03:25.758708+0000","last_active":"2023-10-09T09:04:08.062998+0000","last_peered":"2023-10-09T09:04:08.062998+0000","last_clean":"2023-10-09T09:04:08.062998+0000","last_became_active":"2023-10-09T09:03:25.758344+0000","last_became_peered":"2023-10-09T09:03:25.758344+0000","last_unstale":"2023-10-09T09:04:08.062998+0000","last_undegraded":"2023-10-09T09:04:08.062998+0000","last_fullsized":"2023-10-09T09:04:08.062998+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061611+0000","last_change":"2023-10-09T09:03:24.309527+0000","last_active":"2023-10-09T09:04:08.061611+0000","last_peered":"2023-10-09T09:04:08.061611+0000","last_clean":"2023-10-09T09:04:08.061611+0000","last_became_active":"2023-10-09T09:03:24.309174+0000","last_became_peered":"2023-10-09T09:03:24.309174+0000","last_unstale":"2023-10-09T09:04:08.061611+0000","last_undegraded":"2023-10-09T09:04:08.061611+0000","last_fullsized":"2023-10-09T09:04:08.061611+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063280+0000","last_change":"2023-10-09T09:03:25.332287+0000","last_active":"2023-10-09T09:04:08.063280+0000","last_peered":"2023-10-09T09:04:08.063280+0000","last_clean":"2023-10-09T09:04:08.063280+0000","last_became_active":"2023-10-09T09:03:25.331683+0000","last_became_peered":"2023-10-09T09:03:25.331683+0000","last_unstale":"2023-10-09T09:04:08.063280+0000","last_undegraded":"2023-10-09T09:04:08.063280+0000","last_fullsized":"2023-10-09T09:04:08.063280+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060135+0000","last_change":"2023-10-09T09:03:24.311666+0000","last_active":"2023-10-09T09:04:08.060135+0000","last_peered":"2023-10-09T09:04:08.060135+0000","last_clean":"2023-10-09T09:04:08.060135+0000","last_became_active":"2023-10-09T09:03:24.311324+0000","last_became_peered":"2023-10-09T09:03:24.311324+0000","last_unstale":"2023-10-09T09:04:08.060135+0000","last_undegraded":"2023-10-09T09:04:08.060135+0000","last_fullsized":"2023-10-09T09:04:08.060135+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.783067+0000","last_change":"2023-10-09T09:03:25.314654+0000","last_active":"2023-10-09T09:04:08.783067+0000","last_peered":"2023-10-09T09:04:08.783067+0000","last_clean":"2023-10-09T09:04:08.783067+0000","last_became_active":"2023-10-09T09:03:25.314288+0000","last_became_peered":"2023-10-09T09:03:25.314288+0000","last_unstale":"2023-10-09T09:04:08.783067+0000","last_undegraded":"2023-10-09T09:04:08.783067+0000","last_fullsized":"2023-10-09T09:04:08.783067+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062943+0000","last_change":"2023-10-09T09:03:24.318259+0000","last_active":"2023-10-09T09:04:08.062943+0000","last_peered":"2023-10-09T09:04:08.062943+0000","last_clean":"2023-10-09T09:04:08.062943+0000","last_became_active":"2023-10-09T09:03:24.317375+0000","last_became_peered":"2023-10-09T09:03:24.317375+0000","last_unstale":"2023-10-09T09:04:08.062943+0000","last_undegraded":"2023-10-09T09:04:08.062943+0000","last_fullsized":"2023-10-09T09:04:08.062943+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059525+0000","last_change":"2023-10-09T09:03:25.758778+0000","last_active":"2023-10-09T09:04:08.059525+0000","last_peered":"2023-10-09T09:04:08.059525+0000","last_clean":"2023-10-09T09:04:08.059525+0000","last_became_active":"2023-10-09T09:03:25.758377+0000","last_became_peered":"2023-10-09T09:03:25.758377+0000","last_unstale":"2023-10-09T09:04:08.059525+0000","last_undegraded":"2023-10-09T09:04:08.059525+0000","last_fullsized":"2023-10-09T09:04:08.059525+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061799+0000","last_change":"2023-10-09T09:03:24.307689+0000","last_active":"2023-10-09T09:04:08.061799+0000","last_peered":"2023-10-09T09:04:08.061799+0000","last_clean":"2023-10-09T09:04:08.061799+0000","last_became_active":"2023-10-09T09:03:24.307347+0000","last_became_peered":"2023-10-09T09:03:24.307347+0000","last_unstale":"2023-10-09T09:04:08.061799+0000","last_undegraded":"2023-10-09T09:04:08.061799+0000","last_fullsized":"2023-10-09T09:04:08.061799+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T09:04:09.953641+0000","last_change":"2023-10-09T09:04:09.953641+0000","last_active":"2023-10-09T09:04:09.953641+0000","last_peered":"2023-10-09T09:04:09.953641+0000","last_clean":"2023-10-09T09:04:09.953641+0000","last_became_active":"2023-10-09T09:04:09.953163+0000","last_became_peered":"2023-10-09T09:04:09.953163+0000","last_unstale":"2023-10-09T09:04:09.953641+0000","last_undegraded":"2023-10-09T09:04:09.953641+0000","last_fullsized":"2023-10-09T09:04:09.953641+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064240+0000","last_change":"2023-10-09T09:03:24.303101+0000","last_active":"2023-10-09T09:04:08.064240+0000","last_peered":"2023-10-09T09:04:08.064240+0000","last_clean":"2023-10-09T09:04:08.064240+0000","last_became_active":"2023-10-09T09:03:24.302619+0000","last_became_peered":"2023-10-09T09:03:24.302619+0000","last_unstale":"2023-10-09T09:04:08.064240+0000","last_undegraded":"2023-10-09T09:04:08.064240+0000","last_fullsized":"2023-10-09T09:04:08.064240+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397556,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297740,"kb_used_data":644,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447412,"statfs":{"total":95995035648,"available":95690149888,"internally_reserved":0,"allocated":659456,"data_stored":275261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:03:44 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.90800000000000003,"15min":0.93899999999999995},"min":{"1min":0.65500000000000003,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.2210000000000001,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.91700000000000004,"15min":0.90200000000000002},"min":{"1min":0.71799999999999997,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.2729999999999999,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.76400000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:03:44 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.89200000000000002,"15min":0.88},"min":{"1min":0.66000000000000003,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.2010000000000001,"5min":1.204,"15min":1.204},"last":1.1120000000000001},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.88100000000000001,"15min":0.873},"min":{"1min":0.55700000000000005,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.133,"5min":1.155,"15min":1.155},"last":0.96399999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:03:44 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.89900000000000002,"15min":0.92700000000000005},"min":{"1min":0.54300000000000004,"5min":0.497,"15min":0.497},"max":{"1min":1.179,"5min":1.484,"15min":1.484},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.93100000000000005,"15min":0.93200000000000005},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.244,"5min":1.244,"15min":1.244},"last":1.0509999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:03:44 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.94199999999999995,"15min":0.94399999999999995},"min":{"1min":0.64600000000000002,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.226,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.0029999999999999,"15min":0.997},"min":{"1min":0.78100000000000003,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.321,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.98899999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:03:44 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.89700000000000002,"15min":0.88400000000000001},"min":{"1min":0.69499999999999995,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.98999999999999999,"15min":0.99099999999999999},"min":{"1min":0.75700000000000001,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.234,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":0.83399999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:03:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.97099999999999997,"15min":0.95899999999999996},"min":{"1min":0.72999999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.3740000000000001,"5min":1.403,"15min":1.403},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1,"15min":1.004},"min":{"1min":0.72899999999999998,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3220000000000001,"5min":1.712,"15min":1.712},"last":1.147}]},{"osd":6,"last update":"Mon Oct 9 09:03:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.016,"15min":1.018},"min":{"1min":0.70499999999999996,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.3660000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.0580000000000001,"15min":1.0660000000000001},"min":{"1min":0.68500000000000005,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.3200000000000001,"5min":1.73,"15min":1.73},"last":0.94599999999999995}]}]},{"osd":6,"up_from":43,"seq":184683593789,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297728,"kb_used_data":632,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447424,"statfs":{"total":95995035648,"available":95690162176,"internally_reserved":0,"allocated":647168,"data_stored":275115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.88800000000000001,"15min":0.91500000000000004},"min":{"1min":0.66200000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.163,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":1.1910000000000001},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.84999999999999998,"15min":0.83799999999999997},"min":{"1min":0.54900000000000004,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.085,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.82799999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.88800000000000001,"15min":0.90100000000000002},"min":{"1min":0.66000000000000003,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.254,"5min":1.371,"15min":1.371},"last":1.093},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.91100000000000003,"15min":0.92900000000000005},"min":{"1min":0.60599999999999998,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.153,"5min":1.365,"15min":1.365},"last":0.93100000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.875,"15min":0.873},"min":{"1min":0.63300000000000001,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.137,"5min":1.155,"15min":1.155},"last":1.008},{"interface":"front","average":{"1min":0.998,"5min":0.94199999999999995,"15min":0.95299999999999996},"min":{"1min":0.67400000000000004,"5min":0.432,"15min":0.432},"max":{"1min":1.1739999999999999,"5min":1.294,"15min":1.294},"last":0.89100000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.88700000000000001,"15min":0.89400000000000002},"min":{"1min":0.65400000000000003,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.208,"5min":1.208,"15min":1.208},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.93100000000000005,"15min":0.93500000000000005},"min":{"1min":0.73899999999999999,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.119,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.141}]},{"osd":4,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.879,"15min":0.85499999999999998},"min":{"1min":0.54700000000000004,"5min":0.502,"15min":0.502},"max":{"1min":1.1990000000000001,"5min":1.337,"15min":1.337},"last":0.97599999999999998},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.871,"15min":0.83999999999999997},"min":{"1min":0.70399999999999996,"5min":0.371,"15min":0.371},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.91300000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.95999999999999996,"15min":0.93400000000000005},"min":{"1min":0.92200000000000004,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.296,"5min":1.371,"15min":1.371},"last":0.69999999999999996},{"interface":"front","average":{"1min":1.018,"5min":0.93500000000000005,"15min":0.91300000000000003},"min":{"1min":0.74099999999999999,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.78300000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:04:42 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.0369999999999999,"15min":1.0429999999999999},"min":{"1min":0.873,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.383,"5min":1.383,"15min":1.383},"last":0.873},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.052,"15min":1.0600000000000001},"min":{"1min":0.77600000000000002,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.2669999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.0649999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574950,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":273194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:03:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.66000000000000003,"15min":0.56999999999999995},"min":{"1min":0.39600000000000002,"5min":0.371,"15min":0.37},"max":{"1min":0.85399999999999998,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.748,"15min":0.625},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33300000000000002},"max":{"1min":1.046,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.63100000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.76300000000000001,"15min":0.67800000000000005},"min":{"1min":0.51500000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.95099999999999996,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.878},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.70499999999999996,"15min":0.67300000000000004},"min":{"1min":0.44500000000000001,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.93400000000000005,"5min":1.125,"15min":1.125},"last":0.93200000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.745,"15min":0.71199999999999997},"min":{"1min":0.48799999999999999,"5min":0.432,"15min":0.432},"max":{"1min":0.99299999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.84299999999999997,"15min":0.81799999999999995},"min":{"1min":0.52200000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.1200000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":1.1100000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:03:43 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.86499999999999999,"15min":0.87},"min":{"1min":0.63900000000000001,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.0149999999999999,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.004},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.90900000000000003,"15min":0.89700000000000002},"min":{"1min":0.72699999999999998,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.2609999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":1.143}]},{"osd":5,"last update":"Mon Oct 9 09:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0129999999999999,"15min":1.028},"min":{"1min":0.71199999999999997,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.2290000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":0.999,"15min":1.0289999999999999},"min":{"1min":0.78700000000000003,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":2.9180000000000001,"5min":2.9180000000000001,"15min":2.9180000000000001},"last":1.0940000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.0880000000000001,"15min":1.157},"min":{"1min":0.72599999999999998,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.5880000000000001,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":1.343},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.133,"15min":1.1910000000000001},"min":{"1min":0.83899999999999997,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":1.323}]},{"osd":7,"last update":"Mon Oct 9 09:04:39 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.1060000000000001,"15min":1.0980000000000001},"min":{"1min":0.77100000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.681,"5min":4.7469999999999999,"15min":4.7469999999999999},"last":1.395},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0620000000000001,"15min":1.0720000000000001},"min":{"1min":0.75800000000000001,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.667,"5min":1.667,"15min":1.667},"last":1.2549999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738478,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297724,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447428,"statfs":{"total":95995035648,"available":95690166272,"internally_reserved":0,"allocated":643072,"data_stored":274400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:03:50 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.69499999999999995,"15min":0.57999999999999996},"min":{"1min":0.44,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.101,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.69999999999999996,"15min":0.57499999999999996},"min":{"1min":0.36599999999999999,"5min":0.36099999999999999,"15min":0.35499999999999998},"max":{"1min":0.88600000000000001,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.65400000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:04:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.70899999999999996,"15min":0.64900000000000002},"min":{"1min":0.41299999999999998,"5min":0.313,"15min":0.313},"max":{"1min":0.83199999999999996,"5min":1.276,"15min":1.276},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.77700000000000002,"15min":0.68500000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.32200000000000001},"max":{"1min":1.038,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.90400000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.80500000000000005,"15min":0.80700000000000005},"min":{"1min":0.34999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2290000000000001,"5min":1.365,"15min":1.365},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.78300000000000003,"15min":0.76500000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.0649999999999999,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":1.071}]},{"osd":4,"last update":"Mon Oct 9 09:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.90000000000000002,"15min":0.91600000000000004},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.264,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.92600000000000005,"15min":0.91400000000000003},"min":{"1min":0.66900000000000004,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.1870000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":1.0109999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:04:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.96999999999999997},"min":{"1min":0.74399999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.1679999999999999,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":1.145},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.002,"15min":1.0269999999999999},"min":{"1min":0.64100000000000001,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.2869999999999999,"5min":1.554,"15min":1.554},"last":1.0429999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.085,"15min":1.099},"min":{"1min":0.82499999999999996,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.381,"5min":1.645,"15min":1.645},"last":1.2330000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0429999999999999,"15min":1.079},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.335,"5min":1.395,"15min":1.395},"last":1.2230000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:03:45 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0329999999999999,"15min":1.0449999999999999},"min":{"1min":0.61699999999999999,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.327,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":1.3029999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.107,"15min":1.1220000000000001},"min":{"1min":0.71599999999999997,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.3440000000000001,"5min":1.716,"15min":1.716},"last":1.173}]}]},{"osd":2,"up_from":18,"seq":77309411421,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297724,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447428,"statfs":{"total":95995035648,"available":95690166272,"internally_reserved":0,"allocated":643072,"data_stored":274412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:29 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.66200000000000003,"15min":0.623},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.97399999999999998,"5min":1.04,"15min":1.04},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.71799999999999997,"15min":0.67000000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.8140000000000001,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":0.59299999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:04:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.65500000000000003,"15min":0.63800000000000001},"min":{"1min":0.373,"5min":0.35299999999999998,"15min":0.34300000000000003},"max":{"1min":1.1859999999999999,"5min":1.1859999999999999,"15min":1.1859999999999999},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.76300000000000001,"15min":0.70299999999999996},"min":{"1min":0.437,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.6519999999999999,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.93100000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:04:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.81599999999999995,"15min":0.78000000000000003},"min":{"1min":0.52800000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.0310000000000001,"5min":2.0310000000000001,"15min":2.0310000000000001},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.81599999999999995,"5min":0.78800000000000003,"15min":0.76400000000000001},"min":{"1min":0.48599999999999999,"5min":0.38,"15min":0.38},"max":{"1min":2.1499999999999999,"5min":2.1499999999999999,"15min":2.1499999999999999},"last":0.80000000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.96699999999999997,"15min":0.94399999999999995},"min":{"1min":0.64400000000000002,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.1559999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":1.1240000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.96099999999999997,"15min":0.94499999999999995},"min":{"1min":0.70599999999999996,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.2030000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.98399999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.94999999999999996,"15min":0.95499999999999996},"min":{"1min":0.74299999999999999,"5min":0.63,"15min":0.63},"max":{"1min":1.1950000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":1.097},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.94999999999999996,"15min":0.94899999999999995},"min":{"1min":0.68100000000000005,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.294,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.1719999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0580000000000001,"15min":1.1240000000000001},"min":{"1min":0.80300000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.5960000000000001,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":1.1919999999999999},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0429999999999999,"15min":1.081},"min":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.3180000000000001,"5min":1.464,"15min":1.464},"last":1.159}]},{"osd":7,"last update":"Mon Oct 9 09:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.004,"15min":1.0169999999999999},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.3220000000000001,"5min":1.488,"15min":1.488},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.1000000000000001,"15min":1.131},"min":{"1min":0.75,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.2310000000000001,"5min":1.423,"15min":1.423},"last":1.3620000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182485,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":273914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:05 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.45700000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.94799999999999995,"5min":0.98899999999999999,"15min":1.1399999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.71999999999999997,"15min":0.73699999999999999},"min":{"1min":0.42699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.98599999999999999,"5min":1.0660000000000001,"15min":1.4239999999999999},"last":0.377}]},{"osd":1,"last update":"Mon Oct 9 09:04:05 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.71899999999999997,"15min":0.72499999999999998},"min":{"1min":0.42299999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.002,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.70399999999999996,"15min":0.748},"min":{"1min":0.441,"5min":0.309,"15min":0.309},"max":{"1min":0.95399999999999996,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.51600000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:04:05 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.78400000000000003,"15min":0.82899999999999996},"min":{"1min":0.496,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.7829999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.76500000000000001,"15min":0.79000000000000004},"min":{"1min":0.40100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.024,"5min":1.069,"15min":1.24},"last":0.65700000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.95699999999999996},"min":{"1min":0.63600000000000001,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.1919999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.627},{"interface":"front","average":{"1min":0.996,"5min":0.93100000000000005,"15min":0.92100000000000004},"min":{"1min":0.73599999999999999,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.91000000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:04:19 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.95499999999999996,"15min":0.98099999999999998},"min":{"1min":0.61599999999999999,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.331,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.98699999999999999,"15min":0.996},"min":{"1min":0.60699999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.339,"5min":1.421,"15min":1.421},"last":0.75900000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.048,"15min":1.1020000000000001},"min":{"1min":0.77300000000000002,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.5369999999999999,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":0.93400000000000005},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.056,"15min":1.095},"min":{"1min":0.77200000000000002,"5min":0.62,"15min":0.62},"max":{"1min":1.4079999999999999,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":0.79600000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:03:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.056,"15min":1.0940000000000001},"min":{"1min":0.71699999999999997,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.506,"5min":1.5309999999999999,"15min":1.5309999999999999},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.079,"15min":1.1240000000000001},"min":{"1min":0.82699999999999996,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.2869999999999999,"5min":1.518,"15min":1.518},"last":0.65300000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986254,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297740,"kb_used_data":644,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447412,"statfs":{"total":95995035648,"available":95690149888,"internally_reserved":0,"allocated":659456,"data_stored":275506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.86699999999999999,"15min":0.83999999999999997},"min":{"1min":0.56100000000000005,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.8700000000000001,"5min":2.8700000000000001,"15min":2.8700000000000001},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.86799999999999999,"15min":0.85799999999999998},"min":{"1min":0.61099999999999999,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.792,"5min":1.792,"15min":1.792},"last":0.68799999999999994}]},{"osd":1,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.83899999999999997,"15min":0.81499999999999995},"min":{"1min":0.56999999999999995,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.052,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.998,"5min":0.89700000000000002,"15min":0.879},"min":{"1min":0.61299999999999999,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":3.0550000000000002,"5min":3.0550000000000002,"15min":3.0550000000000002},"last":0.72299999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":0.93100000000000005,"15min":0.86099999999999999},"min":{"1min":0.65600000000000003,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":2.972,"5min":2.972,"15min":2.972},"last":1.087},{"interface":"front","average":{"1min":0.996,"5min":0.90500000000000003,"15min":0.875},"min":{"1min":0.69399999999999995,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.873,"5min":1.873,"15min":1.873},"last":1.0449999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.90900000000000003,"15min":0.88300000000000001},"min":{"1min":0.72099999999999997,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.9379999999999999,"5min":1.9379999999999999,"15min":1.9379999999999999},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.93899999999999995,"15min":0.91800000000000004},"min":{"1min":0.80800000000000005,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.1759999999999999,"5min":2.028,"15min":2.028},"last":0.98499999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:04:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":0.92900000000000005,"15min":0.84899999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":2.052,"5min":2.052,"15min":2.052},"last":0.81399999999999995},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.0249999999999999,"15min":0.95299999999999996},"min":{"1min":0.54400000000000004,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":3.1389999999999998,"5min":3.1389999999999998,"15min":3.1389999999999998},"last":1.137}]},{"osd":6,"last update":"Mon Oct 9 09:04:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.93500000000000005,"15min":0.97299999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.226,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.92800000000000005},{"interface":"front","average":{"1min":1.097,"5min":1.0600000000000001,"15min":1.087},"min":{"1min":0.59799999999999998,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.956,"5min":1.956,"15min":1.956},"last":1.0700000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":0.96299999999999997,"15min":0.90100000000000002},"min":{"1min":0.68100000000000005,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.9790000000000001,"5min":1.9790000000000001,"15min":1.9790000000000001},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.236,"5min":1.026,"15min":0.95199999999999996},"min":{"1min":0.83199999999999996,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":3.2309999999999999,"5min":3.2309999999999999,"15min":3.2309999999999999},"last":1.1839999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790020,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":274133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.86799999999999999,"15min":0.83399999999999996},"min":{"1min":0.68799999999999994,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.151,"5min":2.052,"15min":2.052},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.86499999999999999,"15min":0.83799999999999997},"min":{"1min":0.55500000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.042}]},{"osd":1,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.875,"15min":0.85299999999999998},"min":{"1min":0.55600000000000005,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.399,"5min":2.214,"15min":2.214},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.878,"15min":0.83299999999999996},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.24,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":0.97099999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.89100000000000001,"15min":0.86099999999999999},"min":{"1min":0.59999999999999998,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.266,"5min":1.323,"15min":1.323},"last":1},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.91600000000000004,"15min":0.90400000000000003},"min":{"1min":0.66600000000000004,"5min":0.495,"15min":0.495},"max":{"1min":1.2809999999999999,"5min":2.1200000000000001,"15min":2.1200000000000001},"last":1.0129999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.92600000000000005,"15min":0.90600000000000003},"min":{"1min":0.65300000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2150000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.93999999999999995,"15min":0.90200000000000002},"min":{"1min":0.66500000000000004,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.472,"5min":2.1669999999999998,"15min":2.1669999999999998},"last":1.153}]},{"osd":4,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.79400000000000004,"15min":0.72599999999999998},"min":{"1min":0.60699999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.85199999999999998,"15min":0.754},"min":{"1min":0.58999999999999997,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.2809999999999999,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.1850000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.91200000000000003,"15min":0.878},"min":{"1min":0.56200000000000006,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.1140000000000001},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"min":{"1min":0.78000000000000003,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.331,"5min":1.496,"15min":1.496},"last":1.135}]},{"osd":7,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":0.99099999999999999,"15min":0.94399999999999995},"min":{"1min":0.83999999999999997,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3129999999999999,"5min":1.319,"15min":1.319},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.0329999999999999,"15min":0.99099999999999999},"min":{"1min":0.95199999999999996,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.534,"5min":2.4809999999999999,"15min":2.4809999999999999},"last":1.218}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:04:47.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:04:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:48 smithi181 bash[27712]: audit 2023-10-09T09:04:46.745347+0000 mgr.y (mgr.24355) 143 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:04:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:48 smithi181 bash[27712]: cluster 2023-10-09T09:04:47.233601+0000 mgr.y (mgr.24355) 144 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:48 smithi163 bash[27500]: audit 2023-10-09T09:04:46.745347+0000 mgr.y (mgr.24355) 143 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:04:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:48 smithi163 bash[27500]: cluster 2023-10-09T09:04:47.233601+0000 mgr.y (mgr.24355) 144 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:48.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:48 smithi163 bash[33912]: audit 2023-10-09T09:04:46.745347+0000 mgr.y (mgr.24355) 143 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:04:48.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:48 smithi163 bash[33912]: cluster 2023-10-09T09:04:47.233601+0000 mgr.y (mgr.24355) 144 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:48.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:04:48 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:04:50.217 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:04:49 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:49] "GET /metrics HTTP/1.1" 200 191380 "" "Prometheus/2.33.4" 2023-10-09T09:04:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:50 smithi181 bash[27712]: cluster 2023-10-09T09:04:49.234667+0000 mgr.y (mgr.24355) 145 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:50 smithi163 bash[27500]: cluster 2023-10-09T09:04:49.234667+0000 mgr.y (mgr.24355) 145 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:50 smithi163 bash[33912]: cluster 2023-10-09T09:04:49.234667+0000 mgr.y (mgr.24355) 145 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:52.227 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:04:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:52 smithi181 bash[27712]: cluster 2023-10-09T09:04:51.235339+0000 mgr.y (mgr.24355) 146 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:52.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:52 smithi163 bash[33912]: cluster 2023-10-09T09:04:51.235339+0000 mgr.y (mgr.24355) 146 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:52 smithi163 bash[27500]: cluster 2023-10-09T09:04:51.235339+0000 mgr.y (mgr.24355) 146 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:54 smithi181 bash[27712]: cluster 2023-10-09T09:04:53.236302+0000 mgr.y (mgr.24355) 147 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:54.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:54 smithi163 bash[33912]: cluster 2023-10-09T09:04:53.236302+0000 mgr.y (mgr.24355) 147 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:54.575 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:54 smithi163 bash[27500]: cluster 2023-10-09T09:04:53.236302+0000 mgr.y (mgr.24355) 147 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:55.031 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:04:55.035 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:04:55.688 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":127,"stamp":"2023-10-09T09:04:53.235576+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381804,"kb_used_data":5036,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579412,"statfs":{"total":767960285184,"available":765521317888,"internally_reserved":0,"allocated":5156864,"data_stored":2195935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004265"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063994+0000","last_change":"2023-10-09T09:03:25.312556+0000","last_active":"2023-10-09T09:04:08.063994+0000","last_peered":"2023-10-09T09:04:08.063994+0000","last_clean":"2023-10-09T09:04:08.063994+0000","last_became_active":"2023-10-09T09:03:25.312265+0000","last_became_peered":"2023-10-09T09:03:25.312265+0000","last_unstale":"2023-10-09T09:04:08.063994+0000","last_undegraded":"2023-10-09T09:04:08.063994+0000","last_fullsized":"2023-10-09T09:04:08.063994+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060045+0000","last_change":"2023-10-09T09:03:24.307630+0000","last_active":"2023-10-09T09:04:08.060045+0000","last_peered":"2023-10-09T09:04:08.060045+0000","last_clean":"2023-10-09T09:04:08.060045+0000","last_became_active":"2023-10-09T09:03:24.307313+0000","last_became_peered":"2023-10-09T09:03:24.307313+0000","last_unstale":"2023-10-09T09:04:08.060045+0000","last_undegraded":"2023-10-09T09:04:08.060045+0000","last_fullsized":"2023-10-09T09:04:08.060045+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062892+0000","last_change":"2023-10-09T09:03:25.320580+0000","last_active":"2023-10-09T09:04:08.062892+0000","last_peered":"2023-10-09T09:04:08.062892+0000","last_clean":"2023-10-09T09:04:08.062892+0000","last_became_active":"2023-10-09T09:03:25.320251+0000","last_became_peered":"2023-10-09T09:03:25.320251+0000","last_unstale":"2023-10-09T09:04:08.062892+0000","last_undegraded":"2023-10-09T09:04:08.062892+0000","last_fullsized":"2023-10-09T09:04:08.062892+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063606+0000","last_change":"2023-10-09T09:03:24.312895+0000","last_active":"2023-10-09T09:04:08.063606+0000","last_peered":"2023-10-09T09:04:08.063606+0000","last_clean":"2023-10-09T09:04:08.063606+0000","last_became_active":"2023-10-09T09:03:24.312535+0000","last_became_peered":"2023-10-09T09:03:24.312535+0000","last_unstale":"2023-10-09T09:04:08.063606+0000","last_undegraded":"2023-10-09T09:04:08.063606+0000","last_fullsized":"2023-10-09T09:04:08.063606+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064297+0000","last_change":"2023-10-09T09:03:25.318238+0000","last_active":"2023-10-09T09:04:08.064297+0000","last_peered":"2023-10-09T09:04:08.064297+0000","last_clean":"2023-10-09T09:04:08.064297+0000","last_became_active":"2023-10-09T09:03:25.317878+0000","last_became_peered":"2023-10-09T09:03:25.317878+0000","last_unstale":"2023-10-09T09:04:08.064297+0000","last_undegraded":"2023-10-09T09:04:08.064297+0000","last_fullsized":"2023-10-09T09:04:08.064297+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061651+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:04:08.061651+0000","last_peered":"2023-10-09T09:04:08.061651+0000","last_clean":"2023-10-09T09:04:08.061651+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:04:08.061651+0000","last_undegraded":"2023-10-09T09:04:08.061651+0000","last_fullsized":"2023-10-09T09:04:08.061651+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064799+0000","last_change":"2023-10-09T09:03:25.321650+0000","last_active":"2023-10-09T09:04:08.064799+0000","last_peered":"2023-10-09T09:04:08.064799+0000","last_clean":"2023-10-09T09:04:08.064799+0000","last_became_active":"2023-10-09T09:03:25.321288+0000","last_became_peered":"2023-10-09T09:03:25.321288+0000","last_unstale":"2023-10-09T09:04:08.064799+0000","last_undegraded":"2023-10-09T09:04:08.064799+0000","last_fullsized":"2023-10-09T09:04:08.064799+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061301+0000","last_change":"2023-10-09T09:03:24.309727+0000","last_active":"2023-10-09T09:04:08.061301+0000","last_peered":"2023-10-09T09:04:08.061301+0000","last_clean":"2023-10-09T09:04:08.061301+0000","last_became_active":"2023-10-09T09:03:24.309132+0000","last_became_peered":"2023-10-09T09:03:24.309132+0000","last_unstale":"2023-10-09T09:04:08.061301+0000","last_undegraded":"2023-10-09T09:04:08.061301+0000","last_fullsized":"2023-10-09T09:04:08.061301+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063342+0000","last_change":"2023-10-09T09:03:25.320981+0000","last_active":"2023-10-09T09:04:08.063342+0000","last_peered":"2023-10-09T09:04:08.063342+0000","last_clean":"2023-10-09T09:04:08.063342+0000","last_became_active":"2023-10-09T09:03:25.320532+0000","last_became_peered":"2023-10-09T09:03:25.320532+0000","last_unstale":"2023-10-09T09:04:08.063342+0000","last_undegraded":"2023-10-09T09:04:08.063342+0000","last_fullsized":"2023-10-09T09:04:08.063342+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059468+0000","last_change":"2023-10-09T09:03:24.306211+0000","last_active":"2023-10-09T09:04:08.059468+0000","last_peered":"2023-10-09T09:04:08.059468+0000","last_clean":"2023-10-09T09:04:08.059468+0000","last_became_active":"2023-10-09T09:03:24.304771+0000","last_became_peered":"2023-10-09T09:03:24.304771+0000","last_unstale":"2023-10-09T09:04:08.059468+0000","last_undegraded":"2023-10-09T09:04:08.059468+0000","last_fullsized":"2023-10-09T09:04:08.059468+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064285+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:04:08.064285+0000","last_peered":"2023-10-09T09:04:08.064285+0000","last_clean":"2023-10-09T09:04:08.064285+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:04:08.064285+0000","last_undegraded":"2023-10-09T09:04:08.064285+0000","last_fullsized":"2023-10-09T09:04:08.064285+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059635+0000","last_change":"2023-10-09T09:03:24.313388+0000","last_active":"2023-10-09T09:04:08.059635+0000","last_peered":"2023-10-09T09:04:08.059635+0000","last_clean":"2023-10-09T09:04:08.059635+0000","last_became_active":"2023-10-09T09:03:24.313064+0000","last_became_peered":"2023-10-09T09:03:24.313064+0000","last_unstale":"2023-10-09T09:04:08.059635+0000","last_undegraded":"2023-10-09T09:04:08.059635+0000","last_fullsized":"2023-10-09T09:04:08.059635+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.783114+0000","last_change":"2023-10-09T09:01:34.369352+0000","last_active":"2023-10-09T09:04:08.783114+0000","last_peered":"2023-10-09T09:04:08.783114+0000","last_clean":"2023-10-09T09:04:08.783114+0000","last_became_active":"2023-10-09T09:01:34.368090+0000","last_became_peered":"2023-10-09T09:01:34.368090+0000","last_unstale":"2023-10-09T09:04:08.783114+0000","last_undegraded":"2023-10-09T09:04:08.783114+0000","last_fullsized":"2023-10-09T09:04:08.783114+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062696+0000","last_change":"2023-10-09T09:03:25.757376+0000","last_active":"2023-10-09T09:04:08.062696+0000","last_peered":"2023-10-09T09:04:08.062696+0000","last_clean":"2023-10-09T09:04:08.062696+0000","last_became_active":"2023-10-09T09:03:25.756809+0000","last_became_peered":"2023-10-09T09:03:25.756809+0000","last_unstale":"2023-10-09T09:04:08.062696+0000","last_undegraded":"2023-10-09T09:04:08.062696+0000","last_fullsized":"2023-10-09T09:04:08.062696+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060789+0000","last_change":"2023-10-09T09:01:34.370339+0000","last_active":"2023-10-09T09:04:08.060789+0000","last_peered":"2023-10-09T09:04:08.060789+0000","last_clean":"2023-10-09T09:04:08.060789+0000","last_became_active":"2023-10-09T09:01:34.369975+0000","last_became_peered":"2023-10-09T09:01:34.369975+0000","last_unstale":"2023-10-09T09:04:08.060789+0000","last_undegraded":"2023-10-09T09:04:08.060789+0000","last_fullsized":"2023-10-09T09:04:08.060789+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064161+0000","last_change":"2023-10-09T09:03:24.315589+0000","last_active":"2023-10-09T09:04:08.064161+0000","last_peered":"2023-10-09T09:04:08.064161+0000","last_clean":"2023-10-09T09:04:08.064161+0000","last_became_active":"2023-10-09T09:03:24.315228+0000","last_became_peered":"2023-10-09T09:03:24.315228+0000","last_unstale":"2023-10-09T09:04:08.064161+0000","last_undegraded":"2023-10-09T09:04:08.064161+0000","last_fullsized":"2023-10-09T09:04:08.064161+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059426+0000","last_change":"2023-10-09T09:03:25.757066+0000","last_active":"2023-10-09T09:04:08.059426+0000","last_peered":"2023-10-09T09:04:08.059426+0000","last_clean":"2023-10-09T09:04:08.059426+0000","last_became_active":"2023-10-09T09:03:25.756676+0000","last_became_peered":"2023-10-09T09:03:25.756676+0000","last_unstale":"2023-10-09T09:04:08.059426+0000","last_undegraded":"2023-10-09T09:04:08.059426+0000","last_fullsized":"2023-10-09T09:04:08.059426+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061098+0000","last_change":"2023-10-09T09:03:24.314055+0000","last_active":"2023-10-09T09:04:08.061098+0000","last_peered":"2023-10-09T09:04:08.061098+0000","last_clean":"2023-10-09T09:04:08.061098+0000","last_became_active":"2023-10-09T09:03:24.313634+0000","last_became_peered":"2023-10-09T09:03:24.313634+0000","last_unstale":"2023-10-09T09:04:08.061098+0000","last_undegraded":"2023-10-09T09:04:08.061098+0000","last_fullsized":"2023-10-09T09:04:08.061098+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058742+0000","last_change":"2023-10-09T09:01:34.370949+0000","last_active":"2023-10-09T09:04:08.058742+0000","last_peered":"2023-10-09T09:04:08.058742+0000","last_clean":"2023-10-09T09:04:08.058742+0000","last_became_active":"2023-10-09T09:01:34.370529+0000","last_became_peered":"2023-10-09T09:01:34.370529+0000","last_unstale":"2023-10-09T09:04:08.058742+0000","last_undegraded":"2023-10-09T09:04:08.058742+0000","last_fullsized":"2023-10-09T09:04:08.058742+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T09:04:09.953251+0000","last_change":"2023-10-09T09:04:09.953251+0000","last_active":"2023-10-09T09:04:09.953251+0000","last_peered":"2023-10-09T09:04:09.953251+0000","last_clean":"2023-10-09T09:04:09.953251+0000","last_became_active":"2023-10-09T09:04:09.952651+0000","last_became_peered":"2023-10-09T09:04:09.952651+0000","last_unstale":"2023-10-09T09:04:09.953251+0000","last_undegraded":"2023-10-09T09:04:09.953251+0000","last_fullsized":"2023-10-09T09:04:09.953251+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059096+0000","last_change":"2023-10-09T09:03:24.312793+0000","last_active":"2023-10-09T09:04:08.059096+0000","last_peered":"2023-10-09T09:04:08.059096+0000","last_clean":"2023-10-09T09:04:08.059096+0000","last_became_active":"2023-10-09T09:03:24.312367+0000","last_became_peered":"2023-10-09T09:03:24.312367+0000","last_unstale":"2023-10-09T09:04:08.059096+0000","last_undegraded":"2023-10-09T09:04:08.059096+0000","last_fullsized":"2023-10-09T09:04:08.059096+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058343+0000","last_change":"2023-10-09T09:01:34.369805+0000","last_active":"2023-10-09T09:04:08.058343+0000","last_peered":"2023-10-09T09:04:08.058343+0000","last_clean":"2023-10-09T09:04:08.058343+0000","last_became_active":"2023-10-09T09:01:34.369226+0000","last_became_peered":"2023-10-09T09:01:34.369226+0000","last_unstale":"2023-10-09T09:04:08.058343+0000","last_undegraded":"2023-10-09T09:04:08.058343+0000","last_fullsized":"2023-10-09T09:04:08.058343+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.783125+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:04:08.783125+0000","last_peered":"2023-10-09T09:04:08.783125+0000","last_clean":"2023-10-09T09:04:08.783125+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:04:08.783125+0000","last_undegraded":"2023-10-09T09:04:08.783125+0000","last_fullsized":"2023-10-09T09:04:08.783125+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062912+0000","last_change":"2023-10-09T09:03:24.312843+0000","last_active":"2023-10-09T09:04:08.062912+0000","last_peered":"2023-10-09T09:04:08.062912+0000","last_clean":"2023-10-09T09:04:08.062912+0000","last_became_active":"2023-10-09T09:03:24.312398+0000","last_became_peered":"2023-10-09T09:03:24.312398+0000","last_unstale":"2023-10-09T09:04:08.062912+0000","last_undegraded":"2023-10-09T09:04:08.062912+0000","last_fullsized":"2023-10-09T09:04:08.062912+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058301+0000","last_change":"2023-10-09T09:00:37.999371+0000","last_active":"2023-10-09T09:04:08.058301+0000","last_peered":"2023-10-09T09:04:08.058301+0000","last_clean":"2023-10-09T09:04:08.058301+0000","last_became_active":"2023-10-09T09:00:37.998968+0000","last_became_peered":"2023-10-09T09:00:37.998968+0000","last_unstale":"2023-10-09T09:04:08.058301+0000","last_undegraded":"2023-10-09T09:04:08.058301+0000","last_fullsized":"2023-10-09T09:04:08.058301+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061393+0000","last_change":"2023-10-09T09:01:34.371076+0000","last_active":"2023-10-09T09:04:08.061393+0000","last_peered":"2023-10-09T09:04:08.061393+0000","last_clean":"2023-10-09T09:04:08.061393+0000","last_became_active":"2023-10-09T09:01:34.370690+0000","last_became_peered":"2023-10-09T09:01:34.370690+0000","last_unstale":"2023-10-09T09:04:08.061393+0000","last_undegraded":"2023-10-09T09:04:08.061393+0000","last_fullsized":"2023-10-09T09:04:08.061393+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060860+0000","last_change":"2023-10-09T09:03:25.329498+0000","last_active":"2023-10-09T09:04:08.060860+0000","last_peered":"2023-10-09T09:04:08.060860+0000","last_clean":"2023-10-09T09:04:08.060860+0000","last_became_active":"2023-10-09T09:03:25.329219+0000","last_became_peered":"2023-10-09T09:03:25.329219+0000","last_unstale":"2023-10-09T09:04:08.060860+0000","last_undegraded":"2023-10-09T09:04:08.060860+0000","last_fullsized":"2023-10-09T09:04:08.060860+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060028+0000","last_change":"2023-10-09T09:03:24.299512+0000","last_active":"2023-10-09T09:04:08.060028+0000","last_peered":"2023-10-09T09:04:08.060028+0000","last_clean":"2023-10-09T09:04:08.060028+0000","last_became_active":"2023-10-09T09:03:24.299168+0000","last_became_peered":"2023-10-09T09:03:24.299168+0000","last_unstale":"2023-10-09T09:04:08.060028+0000","last_undegraded":"2023-10-09T09:04:08.060028+0000","last_fullsized":"2023-10-09T09:04:08.060028+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062493+0000","last_change":"2023-10-09T09:01:34.897330+0000","last_active":"2023-10-09T09:04:08.062493+0000","last_peered":"2023-10-09T09:04:08.062493+0000","last_clean":"2023-10-09T09:04:08.062493+0000","last_became_active":"2023-10-09T09:01:34.896715+0000","last_became_peered":"2023-10-09T09:01:34.896715+0000","last_unstale":"2023-10-09T09:04:08.062493+0000","last_undegraded":"2023-10-09T09:04:08.062493+0000","last_fullsized":"2023-10-09T09:04:08.062493+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061002+0000","last_change":"2023-10-09T09:03:25.327061+0000","last_active":"2023-10-09T09:04:08.061002+0000","last_peered":"2023-10-09T09:04:08.061002+0000","last_clean":"2023-10-09T09:04:08.061002+0000","last_became_active":"2023-10-09T09:03:25.326447+0000","last_became_peered":"2023-10-09T09:03:25.326447+0000","last_unstale":"2023-10-09T09:04:08.061002+0000","last_undegraded":"2023-10-09T09:04:08.061002+0000","last_fullsized":"2023-10-09T09:04:08.061002+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064444+0000","last_change":"2023-10-09T09:03:24.301505+0000","last_active":"2023-10-09T09:04:08.064444+0000","last_peered":"2023-10-09T09:04:08.064444+0000","last_clean":"2023-10-09T09:04:08.064444+0000","last_became_active":"2023-10-09T09:03:24.301139+0000","last_became_peered":"2023-10-09T09:03:24.301139+0000","last_unstale":"2023-10-09T09:04:08.064444+0000","last_undegraded":"2023-10-09T09:04:08.064444+0000","last_fullsized":"2023-10-09T09:04:08.064444+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061212+0000","last_change":"2023-10-09T09:01:34.372320+0000","last_active":"2023-10-09T09:04:08.061212+0000","last_peered":"2023-10-09T09:04:08.061212+0000","last_clean":"2023-10-09T09:04:08.061212+0000","last_became_active":"2023-10-09T09:01:34.371852+0000","last_became_peered":"2023-10-09T09:01:34.371852+0000","last_unstale":"2023-10-09T09:04:08.061212+0000","last_undegraded":"2023-10-09T09:04:08.061212+0000","last_fullsized":"2023-10-09T09:04:08.061212+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059929+0000","last_change":"2023-10-09T09:03:25.321662+0000","last_active":"2023-10-09T09:04:08.059929+0000","last_peered":"2023-10-09T09:04:08.059929+0000","last_clean":"2023-10-09T09:04:08.059929+0000","last_became_active":"2023-10-09T09:03:25.321307+0000","last_became_peered":"2023-10-09T09:03:25.321307+0000","last_unstale":"2023-10-09T09:04:08.059929+0000","last_undegraded":"2023-10-09T09:04:08.059929+0000","last_fullsized":"2023-10-09T09:04:08.059929+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061649+0000","last_change":"2023-10-09T09:03:24.309025+0000","last_active":"2023-10-09T09:04:08.061649+0000","last_peered":"2023-10-09T09:04:08.061649+0000","last_clean":"2023-10-09T09:04:08.061649+0000","last_became_active":"2023-10-09T09:03:24.308645+0000","last_became_peered":"2023-10-09T09:03:24.308645+0000","last_unstale":"2023-10-09T09:04:08.061649+0000","last_undegraded":"2023-10-09T09:04:08.061649+0000","last_fullsized":"2023-10-09T09:04:08.061649+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060910+0000","last_change":"2023-10-09T09:01:34.370258+0000","last_active":"2023-10-09T09:04:08.060910+0000","last_peered":"2023-10-09T09:04:08.060910+0000","last_clean":"2023-10-09T09:04:08.060910+0000","last_became_active":"2023-10-09T09:01:34.369813+0000","last_became_peered":"2023-10-09T09:01:34.369813+0000","last_unstale":"2023-10-09T09:04:08.060910+0000","last_undegraded":"2023-10-09T09:04:08.060910+0000","last_fullsized":"2023-10-09T09:04:08.060910+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060856+0000","last_change":"2023-10-09T09:03:25.322568+0000","last_active":"2023-10-09T09:04:08.060856+0000","last_peered":"2023-10-09T09:04:08.060856+0000","last_clean":"2023-10-09T09:04:08.060856+0000","last_became_active":"2023-10-09T09:03:25.322259+0000","last_became_peered":"2023-10-09T09:03:25.322259+0000","last_unstale":"2023-10-09T09:04:08.060856+0000","last_undegraded":"2023-10-09T09:04:08.060856+0000","last_fullsized":"2023-10-09T09:04:08.060856+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062819+0000","last_change":"2023-10-09T09:03:24.308720+0000","last_active":"2023-10-09T09:04:08.062819+0000","last_peered":"2023-10-09T09:04:08.062819+0000","last_clean":"2023-10-09T09:04:08.062819+0000","last_became_active":"2023-10-09T09:03:24.308389+0000","last_became_peered":"2023-10-09T09:03:24.308389+0000","last_unstale":"2023-10-09T09:04:08.062819+0000","last_undegraded":"2023-10-09T09:04:08.062819+0000","last_fullsized":"2023-10-09T09:04:08.062819+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064992+0000","last_change":"2023-10-09T09:03:25.323242+0000","last_active":"2023-10-09T09:04:08.064992+0000","last_peered":"2023-10-09T09:04:08.064992+0000","last_clean":"2023-10-09T09:04:08.064992+0000","last_became_active":"2023-10-09T09:03:25.321463+0000","last_became_peered":"2023-10-09T09:03:25.321463+0000","last_unstale":"2023-10-09T09:04:08.064992+0000","last_undegraded":"2023-10-09T09:04:08.064992+0000","last_fullsized":"2023-10-09T09:04:08.064992+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064278+0000","last_change":"2023-10-09T09:03:24.303003+0000","last_active":"2023-10-09T09:04:08.064278+0000","last_peered":"2023-10-09T09:04:08.064278+0000","last_clean":"2023-10-09T09:04:08.064278+0000","last_became_active":"2023-10-09T09:03:24.302583+0000","last_became_peered":"2023-10-09T09:03:24.302583+0000","last_unstale":"2023-10-09T09:04:08.064278+0000","last_undegraded":"2023-10-09T09:04:08.064278+0000","last_fullsized":"2023-10-09T09:04:08.064278+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064796+0000","last_change":"2023-10-09T09:03:25.321487+0000","last_active":"2023-10-09T09:04:08.064796+0000","last_peered":"2023-10-09T09:04:08.064796+0000","last_clean":"2023-10-09T09:04:08.064796+0000","last_became_active":"2023-10-09T09:03:25.321142+0000","last_became_peered":"2023-10-09T09:03:25.321142+0000","last_unstale":"2023-10-09T09:04:08.064796+0000","last_undegraded":"2023-10-09T09:04:08.064796+0000","last_fullsized":"2023-10-09T09:04:08.064796+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.782924+0000","last_change":"2023-10-09T09:03:24.307481+0000","last_active":"2023-10-09T09:04:08.782924+0000","last_peered":"2023-10-09T09:04:08.782924+0000","last_clean":"2023-10-09T09:04:08.782924+0000","last_became_active":"2023-10-09T09:03:24.307102+0000","last_became_peered":"2023-10-09T09:03:24.307102+0000","last_unstale":"2023-10-09T09:04:08.782924+0000","last_undegraded":"2023-10-09T09:04:08.782924+0000","last_fullsized":"2023-10-09T09:04:08.782924+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064631+0000","last_change":"2023-10-09T09:03:25.318434+0000","last_active":"2023-10-09T09:04:08.064631+0000","last_peered":"2023-10-09T09:04:08.064631+0000","last_clean":"2023-10-09T09:04:08.064631+0000","last_became_active":"2023-10-09T09:03:25.317989+0000","last_became_peered":"2023-10-09T09:03:25.317989+0000","last_unstale":"2023-10-09T09:04:08.064631+0000","last_undegraded":"2023-10-09T09:04:08.064631+0000","last_fullsized":"2023-10-09T09:04:08.064631+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063179+0000","last_change":"2023-10-09T09:03:24.317705+0000","last_active":"2023-10-09T09:04:08.063179+0000","last_peered":"2023-10-09T09:04:08.063179+0000","last_clean":"2023-10-09T09:04:08.063179+0000","last_became_active":"2023-10-09T09:03:24.317211+0000","last_became_peered":"2023-10-09T09:03:24.317211+0000","last_unstale":"2023-10-09T09:04:08.063179+0000","last_undegraded":"2023-10-09T09:04:08.063179+0000","last_fullsized":"2023-10-09T09:04:08.063179+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059281+0000","last_change":"2023-10-09T09:03:25.320402+0000","last_active":"2023-10-09T09:04:08.059281+0000","last_peered":"2023-10-09T09:04:08.059281+0000","last_clean":"2023-10-09T09:04:08.059281+0000","last_became_active":"2023-10-09T09:03:25.320032+0000","last_became_peered":"2023-10-09T09:03:25.320032+0000","last_unstale":"2023-10-09T09:04:08.059281+0000","last_undegraded":"2023-10-09T09:04:08.059281+0000","last_fullsized":"2023-10-09T09:04:08.059281+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061185+0000","last_change":"2023-10-09T09:03:24.308119+0000","last_active":"2023-10-09T09:04:08.061185+0000","last_peered":"2023-10-09T09:04:08.061185+0000","last_clean":"2023-10-09T09:04:08.061185+0000","last_became_active":"2023-10-09T09:03:24.307416+0000","last_became_peered":"2023-10-09T09:03:24.307416+0000","last_unstale":"2023-10-09T09:04:08.061185+0000","last_undegraded":"2023-10-09T09:04:08.061185+0000","last_fullsized":"2023-10-09T09:04:08.061185+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.782785+0000","last_change":"2023-10-09T09:03:25.756450+0000","last_active":"2023-10-09T09:04:08.782785+0000","last_peered":"2023-10-09T09:04:08.782785+0000","last_clean":"2023-10-09T09:04:08.782785+0000","last_became_active":"2023-10-09T09:03:25.756113+0000","last_became_peered":"2023-10-09T09:03:25.756113+0000","last_unstale":"2023-10-09T09:04:08.782785+0000","last_undegraded":"2023-10-09T09:04:08.782785+0000","last_fullsized":"2023-10-09T09:04:08.782785+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058786+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:04:08.058786+0000","last_peered":"2023-10-09T09:04:08.058786+0000","last_clean":"2023-10-09T09:04:08.058786+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:04:08.058786+0000","last_undegraded":"2023-10-09T09:04:08.058786+0000","last_fullsized":"2023-10-09T09:04:08.058786+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064116+0000","last_change":"2023-10-09T09:03:25.318210+0000","last_active":"2023-10-09T09:04:08.064116+0000","last_peered":"2023-10-09T09:04:08.064116+0000","last_clean":"2023-10-09T09:04:08.064116+0000","last_became_active":"2023-10-09T09:03:25.317818+0000","last_became_peered":"2023-10-09T09:03:25.317818+0000","last_unstale":"2023-10-09T09:04:08.064116+0000","last_undegraded":"2023-10-09T09:04:08.064116+0000","last_fullsized":"2023-10-09T09:04:08.064116+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059111+0000","last_change":"2023-10-09T09:03:24.311122+0000","last_active":"2023-10-09T09:04:08.059111+0000","last_peered":"2023-10-09T09:04:08.059111+0000","last_clean":"2023-10-09T09:04:08.059111+0000","last_became_active":"2023-10-09T09:03:24.310080+0000","last_became_peered":"2023-10-09T09:03:24.310080+0000","last_unstale":"2023-10-09T09:04:08.059111+0000","last_undegraded":"2023-10-09T09:04:08.059111+0000","last_fullsized":"2023-10-09T09:04:08.059111+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061156+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:04:08.061156+0000","last_peered":"2023-10-09T09:04:08.061156+0000","last_clean":"2023-10-09T09:04:08.061156+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:04:08.061156+0000","last_undegraded":"2023-10-09T09:04:08.061156+0000","last_fullsized":"2023-10-09T09:04:08.061156+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059390+0000","last_change":"2023-10-09T09:03:24.310831+0000","last_active":"2023-10-09T09:04:08.059390+0000","last_peered":"2023-10-09T09:04:08.059390+0000","last_clean":"2023-10-09T09:04:08.059390+0000","last_became_active":"2023-10-09T09:03:24.310045+0000","last_became_peered":"2023-10-09T09:03:24.310045+0000","last_unstale":"2023-10-09T09:04:08.059390+0000","last_undegraded":"2023-10-09T09:04:08.059390+0000","last_fullsized":"2023-10-09T09:04:08.059390+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T09:04:09.954296+0000","last_change":"2023-10-09T09:04:09.954296+0000","last_active":"2023-10-09T09:04:09.954296+0000","last_peered":"2023-10-09T09:04:09.954296+0000","last_clean":"2023-10-09T09:04:09.954296+0000","last_became_active":"2023-10-09T09:04:09.953904+0000","last_became_peered":"2023-10-09T09:04:09.953904+0000","last_unstale":"2023-10-09T09:04:09.954296+0000","last_undegraded":"2023-10-09T09:04:09.954296+0000","last_fullsized":"2023-10-09T09:04:09.954296+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.782664+0000","last_change":"2023-10-09T09:03:24.310906+0000","last_active":"2023-10-09T09:04:08.782664+0000","last_peered":"2023-10-09T09:04:08.782664+0000","last_clean":"2023-10-09T09:04:08.782664+0000","last_became_active":"2023-10-09T09:03:24.310576+0000","last_became_peered":"2023-10-09T09:03:24.310576+0000","last_unstale":"2023-10-09T09:04:08.782664+0000","last_undegraded":"2023-10-09T09:04:08.782664+0000","last_fullsized":"2023-10-09T09:04:08.782664+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059416+0000","last_change":"2023-10-09T09:03:25.756017+0000","last_active":"2023-10-09T09:04:08.059416+0000","last_peered":"2023-10-09T09:04:08.059416+0000","last_clean":"2023-10-09T09:04:08.059416+0000","last_became_active":"2023-10-09T09:03:25.755602+0000","last_became_peered":"2023-10-09T09:03:25.755602+0000","last_unstale":"2023-10-09T09:04:08.059416+0000","last_undegraded":"2023-10-09T09:04:08.059416+0000","last_fullsized":"2023-10-09T09:04:08.059416+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058339+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:04:08.058339+0000","last_peered":"2023-10-09T09:04:08.058339+0000","last_clean":"2023-10-09T09:04:08.058339+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:04:08.058339+0000","last_undegraded":"2023-10-09T09:04:08.058339+0000","last_fullsized":"2023-10-09T09:04:08.058339+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061425+0000","last_change":"2023-10-09T09:03:25.757962+0000","last_active":"2023-10-09T09:04:08.061425+0000","last_peered":"2023-10-09T09:04:08.061425+0000","last_clean":"2023-10-09T09:04:08.061425+0000","last_became_active":"2023-10-09T09:03:25.757454+0000","last_became_peered":"2023-10-09T09:03:25.757454+0000","last_unstale":"2023-10-09T09:04:08.061425+0000","last_undegraded":"2023-10-09T09:04:08.061425+0000","last_fullsized":"2023-10-09T09:04:08.061425+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059166+0000","last_change":"2023-10-09T09:03:24.311042+0000","last_active":"2023-10-09T09:04:08.059166+0000","last_peered":"2023-10-09T09:04:08.059166+0000","last_clean":"2023-10-09T09:04:08.059166+0000","last_became_active":"2023-10-09T09:03:24.310650+0000","last_became_peered":"2023-10-09T09:03:24.310650+0000","last_unstale":"2023-10-09T09:04:08.059166+0000","last_undegraded":"2023-10-09T09:04:08.059166+0000","last_fullsized":"2023-10-09T09:04:08.059166+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063527+0000","last_change":"2023-10-09T09:03:25.756248+0000","last_active":"2023-10-09T09:04:08.063527+0000","last_peered":"2023-10-09T09:04:08.063527+0000","last_clean":"2023-10-09T09:04:08.063527+0000","last_became_active":"2023-10-09T09:03:25.755903+0000","last_became_peered":"2023-10-09T09:03:25.755903+0000","last_unstale":"2023-10-09T09:04:08.063527+0000","last_undegraded":"2023-10-09T09:04:08.063527+0000","last_fullsized":"2023-10-09T09:04:08.063527+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":16,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058740+0000","last_change":"2023-10-09T09:03:24.304393+0000","last_active":"2023-10-09T09:04:08.058740+0000","last_peered":"2023-10-09T09:04:08.058740+0000","last_clean":"2023-10-09T09:04:08.058740+0000","last_became_active":"2023-10-09T09:03:24.304006+0000","last_became_peered":"2023-10-09T09:03:24.304006+0000","last_unstale":"2023-10-09T09:04:08.058740+0000","last_undegraded":"2023-10-09T09:04:08.058740+0000","last_fullsized":"2023-10-09T09:04:08.058740+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064705+0000","last_change":"2023-10-09T09:03:25.322358+0000","last_active":"2023-10-09T09:04:08.064705+0000","last_peered":"2023-10-09T09:04:08.064705+0000","last_clean":"2023-10-09T09:04:08.064705+0000","last_became_active":"2023-10-09T09:03:25.321970+0000","last_became_peered":"2023-10-09T09:03:25.321970+0000","last_unstale":"2023-10-09T09:04:08.064705+0000","last_undegraded":"2023-10-09T09:04:08.064705+0000","last_fullsized":"2023-10-09T09:04:08.064705+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"61'6","reported_seq":20,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064487+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:04:08.064487+0000","last_peered":"2023-10-09T09:04:08.064487+0000","last_clean":"2023-10-09T09:04:08.064487+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:04:08.064487+0000","last_undegraded":"2023-10-09T09:04:08.064487+0000","last_fullsized":"2023-10-09T09:04:08.064487+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059931+0000","last_change":"2023-10-09T09:03:25.324660+0000","last_active":"2023-10-09T09:04:08.059931+0000","last_peered":"2023-10-09T09:04:08.059931+0000","last_clean":"2023-10-09T09:04:08.059931+0000","last_became_active":"2023-10-09T09:03:25.324342+0000","last_became_peered":"2023-10-09T09:03:25.324342+0000","last_unstale":"2023-10-09T09:04:08.059931+0000","last_undegraded":"2023-10-09T09:04:08.059931+0000","last_fullsized":"2023-10-09T09:04:08.059931+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":17,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.058651+0000","last_change":"2023-10-09T09:03:24.312250+0000","last_active":"2023-10-09T09:04:08.058651+0000","last_peered":"2023-10-09T09:04:08.058651+0000","last_clean":"2023-10-09T09:04:08.058651+0000","last_became_active":"2023-10-09T09:03:24.310944+0000","last_became_peered":"2023-10-09T09:03:24.310944+0000","last_unstale":"2023-10-09T09:04:08.058651+0000","last_undegraded":"2023-10-09T09:04:08.058651+0000","last_fullsized":"2023-10-09T09:04:08.058651+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062998+0000","last_change":"2023-10-09T09:03:25.758708+0000","last_active":"2023-10-09T09:04:08.062998+0000","last_peered":"2023-10-09T09:04:08.062998+0000","last_clean":"2023-10-09T09:04:08.062998+0000","last_became_active":"2023-10-09T09:03:25.758344+0000","last_became_peered":"2023-10-09T09:03:25.758344+0000","last_unstale":"2023-10-09T09:04:08.062998+0000","last_undegraded":"2023-10-09T09:04:08.062998+0000","last_fullsized":"2023-10-09T09:04:08.062998+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061611+0000","last_change":"2023-10-09T09:03:24.309527+0000","last_active":"2023-10-09T09:04:08.061611+0000","last_peered":"2023-10-09T09:04:08.061611+0000","last_clean":"2023-10-09T09:04:08.061611+0000","last_became_active":"2023-10-09T09:03:24.309174+0000","last_became_peered":"2023-10-09T09:03:24.309174+0000","last_unstale":"2023-10-09T09:04:08.061611+0000","last_undegraded":"2023-10-09T09:04:08.061611+0000","last_fullsized":"2023-10-09T09:04:08.061611+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.063280+0000","last_change":"2023-10-09T09:03:25.332287+0000","last_active":"2023-10-09T09:04:08.063280+0000","last_peered":"2023-10-09T09:04:08.063280+0000","last_clean":"2023-10-09T09:04:08.063280+0000","last_became_active":"2023-10-09T09:03:25.331683+0000","last_became_peered":"2023-10-09T09:03:25.331683+0000","last_unstale":"2023-10-09T09:04:08.063280+0000","last_undegraded":"2023-10-09T09:04:08.063280+0000","last_fullsized":"2023-10-09T09:04:08.063280+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.060135+0000","last_change":"2023-10-09T09:03:24.311666+0000","last_active":"2023-10-09T09:04:08.060135+0000","last_peered":"2023-10-09T09:04:08.060135+0000","last_clean":"2023-10-09T09:04:08.060135+0000","last_became_active":"2023-10-09T09:03:24.311324+0000","last_became_peered":"2023-10-09T09:03:24.311324+0000","last_unstale":"2023-10-09T09:04:08.060135+0000","last_undegraded":"2023-10-09T09:04:08.060135+0000","last_fullsized":"2023-10-09T09:04:08.060135+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.783067+0000","last_change":"2023-10-09T09:03:25.314654+0000","last_active":"2023-10-09T09:04:08.783067+0000","last_peered":"2023-10-09T09:04:08.783067+0000","last_clean":"2023-10-09T09:04:08.783067+0000","last_became_active":"2023-10-09T09:03:25.314288+0000","last_became_peered":"2023-10-09T09:03:25.314288+0000","last_unstale":"2023-10-09T09:04:08.783067+0000","last_undegraded":"2023-10-09T09:04:08.783067+0000","last_fullsized":"2023-10-09T09:04:08.783067+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.062943+0000","last_change":"2023-10-09T09:03:24.318259+0000","last_active":"2023-10-09T09:04:08.062943+0000","last_peered":"2023-10-09T09:04:08.062943+0000","last_clean":"2023-10-09T09:04:08.062943+0000","last_became_active":"2023-10-09T09:03:24.317375+0000","last_became_peered":"2023-10-09T09:03:24.317375+0000","last_unstale":"2023-10-09T09:04:08.062943+0000","last_undegraded":"2023-10-09T09:04:08.062943+0000","last_fullsized":"2023-10-09T09:04:08.062943+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.059525+0000","last_change":"2023-10-09T09:03:25.758778+0000","last_active":"2023-10-09T09:04:08.059525+0000","last_peered":"2023-10-09T09:04:08.059525+0000","last_clean":"2023-10-09T09:04:08.059525+0000","last_became_active":"2023-10-09T09:03:25.758377+0000","last_became_peered":"2023-10-09T09:03:25.758377+0000","last_unstale":"2023-10-09T09:04:08.059525+0000","last_undegraded":"2023-10-09T09:04:08.059525+0000","last_fullsized":"2023-10-09T09:04:08.059525+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.061799+0000","last_change":"2023-10-09T09:03:24.307689+0000","last_active":"2023-10-09T09:04:08.061799+0000","last_peered":"2023-10-09T09:04:08.061799+0000","last_clean":"2023-10-09T09:04:08.061799+0000","last_became_active":"2023-10-09T09:03:24.307347+0000","last_became_peered":"2023-10-09T09:03:24.307347+0000","last_unstale":"2023-10-09T09:04:08.061799+0000","last_undegraded":"2023-10-09T09:04:08.061799+0000","last_fullsized":"2023-10-09T09:04:08.061799+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":22,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T09:04:09.953641+0000","last_change":"2023-10-09T09:04:09.953641+0000","last_active":"2023-10-09T09:04:09.953641+0000","last_peered":"2023-10-09T09:04:09.953641+0000","last_clean":"2023-10-09T09:04:09.953641+0000","last_became_active":"2023-10-09T09:04:09.953163+0000","last_became_peered":"2023-10-09T09:04:09.953163+0000","last_unstale":"2023-10-09T09:04:09.953641+0000","last_undegraded":"2023-10-09T09:04:09.953641+0000","last_fullsized":"2023-10-09T09:04:09.953641+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"61'1","reported_seq":15,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T09:04:08.064240+0000","last_change":"2023-10-09T09:03:24.303101+0000","last_active":"2023-10-09T09:04:08.064240+0000","last_peered":"2023-10-09T09:04:08.064240+0000","last_clean":"2023-10-09T09:04:08.064240+0000","last_became_active":"2023-10-09T09:03:24.302619+0000","last_became_peered":"2023-10-09T09:03:24.302619+0000","last_unstale":"2023-10-09T09:04:08.064240+0000","last_undegraded":"2023-10-09T09:04:08.064240+0000","last_fullsized":"2023-10-09T09:04:08.064240+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397558,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297740,"kb_used_data":644,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447412,"statfs":{"total":95995035648,"available":95690149888,"internally_reserved":0,"allocated":659456,"data_stored":275261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.91100000000000003,"15min":0.93999999999999995},"min":{"1min":0.54000000000000004,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.341,"5min":1.341,"15min":1.341},"last":1.109},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.91100000000000003,"15min":0.90000000000000002},"min":{"1min":0.59999999999999998,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.0840000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.873}]},{"osd":1,"last update":"Mon Oct 9 09:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.91000000000000003,"15min":0.88600000000000001},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":1.2270000000000001},{"interface":"front","average":{"1min":0.876,"5min":0.88200000000000001,"15min":0.873},"min":{"1min":0.59899999999999998,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.84799999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.90200000000000002,"15min":0.92800000000000005},"min":{"1min":0.64900000000000002,"5min":0.497,"15min":0.497},"max":{"1min":1.2849999999999999,"5min":1.484,"15min":1.484},"last":0.93400000000000005},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.93400000000000005,"15min":0.93300000000000005},"min":{"1min":0.65600000000000003,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.353,"5min":1.353,"15min":1.353},"last":1.133}]},{"osd":3,"last update":"Mon Oct 9 09:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.94699999999999995,"15min":0.94599999999999995},"min":{"1min":0.68600000000000005,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.3260000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.014},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.0169999999999999,"15min":1.002},"min":{"1min":0.84299999999999997,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.2509999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.044}]},{"osd":4,"last update":"Mon Oct 9 09:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.90900000000000003,"15min":0.88800000000000001},"min":{"1min":0.76800000000000002,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.135,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0109999999999999,"15min":0.998},"min":{"1min":0.83399999999999996,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.413,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":1.268}]},{"osd":5,"last update":"Mon Oct 9 09:04:49 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":0.99199999999999999,"15min":0.96599999999999997},"min":{"1min":0.67500000000000004,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.3580000000000001,"5min":1.403,"15min":1.403},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0169999999999999,"15min":1.01},"min":{"1min":0.68500000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.383,"5min":1.712,"15min":1.712},"last":1.163}]},{"osd":6,"last update":"Mon Oct 9 09:04:49 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.026,"15min":1.0209999999999999},"min":{"1min":0.63,"5min":0.63,"15min":0.63},"max":{"1min":1.4410000000000001,"5min":1.4410000000000001,"15min":1.4410000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.173,"5min":1.0780000000000001,"15min":1.0720000000000001},"min":{"1min":0.94599999999999995,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.4910000000000001,"5min":1.73,"15min":1.73},"last":1.1839999999999999}]}]},{"osd":6,"up_from":43,"seq":184683593791,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297728,"kb_used_data":632,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447424,"statfs":{"total":95995035648,"available":95690162176,"internally_reserved":0,"allocated":647168,"data_stored":275115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.88800000000000001,"15min":0.91500000000000004},"min":{"1min":0.66200000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.163,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.84999999999999998,"15min":0.83799999999999997},"min":{"1min":0.54900000000000004,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.085,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.60199999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.88800000000000001,"15min":0.90100000000000002},"min":{"1min":0.66000000000000003,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.254,"5min":1.371,"15min":1.371},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.91100000000000003,"15min":0.92900000000000005},"min":{"1min":0.60599999999999998,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.153,"5min":1.365,"15min":1.365},"last":1.107}]},{"osd":2,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.875,"15min":0.873},"min":{"1min":0.63300000000000001,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.137,"5min":1.155,"15min":1.155},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.998,"5min":0.94199999999999995,"15min":0.95299999999999996},"min":{"1min":0.67400000000000004,"5min":0.432,"15min":0.432},"max":{"1min":1.1739999999999999,"5min":1.294,"15min":1.294},"last":0.64800000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.88700000000000001,"15min":0.89400000000000002},"min":{"1min":0.65400000000000003,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.208,"5min":1.208,"15min":1.208},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.93100000000000005,"15min":0.93500000000000005},"min":{"1min":0.73899999999999999,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.119,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.018}]},{"osd":4,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.879,"15min":0.85499999999999998},"min":{"1min":0.54700000000000004,"5min":0.502,"15min":0.502},"max":{"1min":1.1990000000000001,"5min":1.337,"15min":1.337},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.871,"15min":0.83999999999999997},"min":{"1min":0.70399999999999996,"5min":0.371,"15min":0.371},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.97799999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.95999999999999996,"15min":0.93400000000000005},"min":{"1min":0.92200000000000004,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.296,"5min":1.371,"15min":1.371},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.018,"5min":0.93500000000000005,"15min":0.91300000000000003},"min":{"1min":0.74099999999999999,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":1.135}]},{"osd":7,"last update":"Mon Oct 9 09:04:42 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.0369999999999999,"15min":1.0429999999999999},"min":{"1min":0.873,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.383,"5min":1.383,"15min":1.383},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.052,"15min":1.0600000000000001},"min":{"1min":0.77600000000000002,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.2669999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.1879999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574952,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":273194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:46 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.66700000000000004,"15min":0.58299999999999996},"min":{"1min":0.42599999999999999,"5min":0.371,"15min":0.37},"max":{"1min":1.0269999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.94599999999999995},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.72699999999999998,"15min":0.63300000000000001},"min":{"1min":0.46000000000000002,"5min":0.35199999999999998,"15min":0.33300000000000002},"max":{"1min":0.995,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":1.2470000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.76300000000000001,"15min":0.67800000000000005},"min":{"1min":0.51500000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.95099999999999996,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.96399999999999997},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.70499999999999996,"15min":0.67300000000000004},"min":{"1min":0.44500000000000001,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.93400000000000005,"5min":1.125,"15min":1.125},"last":0.65200000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.745,"15min":0.71199999999999997},"min":{"1min":0.48799999999999999,"5min":0.432,"15min":0.432},"max":{"1min":0.99299999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":1.024},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.84299999999999997,"15min":0.81799999999999995},"min":{"1min":0.52200000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.1200000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":1.1479999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:04:46 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.86299999999999999,"15min":0.86899999999999999},"min":{"1min":0.66400000000000003,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.151,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.92300000000000004,"15min":0.90100000000000002},"min":{"1min":0.68899999999999995,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":1.2150000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0129999999999999,"15min":1.028},"min":{"1min":0.71199999999999997,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.82699999999999996},{"interface":"front","average":{"1min":1.0669999999999999,"5min":0.999,"15min":1.0289999999999999},"min":{"1min":0.78700000000000003,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":2.9180000000000001,"5min":2.9180000000000001,"15min":2.9180000000000001},"last":1.177}]},{"osd":6,"last update":"Mon Oct 9 09:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.0880000000000001,"15min":1.157},"min":{"1min":0.72599999999999998,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.5880000000000001,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":1.3009999999999999},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.133,"15min":1.1910000000000001},"min":{"1min":0.83899999999999997,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":1.3149999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:04:39 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.1060000000000001,"15min":1.0980000000000001},"min":{"1min":0.77100000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.681,"5min":4.7469999999999999,"15min":4.7469999999999999},"last":1.3939999999999999},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0620000000000001,"15min":1.0720000000000001},"min":{"1min":0.75800000000000001,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.667,"5min":1.667,"15min":1.667},"last":1.3700000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738480,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297724,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447428,"statfs":{"total":95995035648,"available":95690166272,"internally_reserved":0,"allocated":643072,"data_stored":274400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:04:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.66200000000000003,"15min":0.58499999999999996},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.83899999999999997,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.68300000000000005,"15min":0.58499999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35499999999999998},"max":{"1min":1.014,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.54400000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:04:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.70899999999999996,"15min":0.64900000000000002},"min":{"1min":0.41299999999999998,"5min":0.313,"15min":0.313},"max":{"1min":0.83199999999999996,"5min":1.276,"15min":1.276},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.77700000000000002,"15min":0.68500000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.32200000000000001},"max":{"1min":1.038,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.69199999999999995}]},{"osd":3,"last update":"Mon Oct 9 09:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.80500000000000005,"15min":0.80700000000000005},"min":{"1min":0.34999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2290000000000001,"5min":1.365,"15min":1.365},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.78300000000000003,"15min":0.76500000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.0649999999999999,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.65300000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:04:47 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.89400000000000002,"15min":0.91400000000000003},"min":{"1min":0.622,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.155,"5min":1.343,"15min":1.4690000000000001},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.93300000000000005,"15min":0.91700000000000004},"min":{"1min":0.59899999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.2549999999999999,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.75}]},{"osd":5,"last update":"Mon Oct 9 09:04:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.96999999999999997},"min":{"1min":0.74399999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.1679999999999999,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.002,"15min":1.0269999999999999},"min":{"1min":0.64100000000000001,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.2869999999999999,"5min":1.554,"15min":1.554},"last":0.77000000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.085,"15min":1.099},"min":{"1min":0.82499999999999996,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.381,"5min":1.645,"15min":1.645},"last":0.91100000000000003},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0429999999999999,"15min":1.079},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.335,"5min":1.395,"15min":1.395},"last":0.89700000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.028,"15min":1.044},"min":{"1min":0.67800000000000005,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.365,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.99299999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.095,"15min":1.1180000000000001},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.296,"5min":1.716,"15min":1.716},"last":0.95999999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411423,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297724,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447428,"statfs":{"total":95995035648,"available":95690166272,"internally_reserved":0,"allocated":643072,"data_stored":274412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:29 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.66200000000000003,"15min":0.623},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.97399999999999998,"5min":1.04,"15min":1.04},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.71799999999999997,"15min":0.67000000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.8140000000000001,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":0.75700000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:04:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.65500000000000003,"15min":0.63800000000000001},"min":{"1min":0.373,"5min":0.35299999999999998,"15min":0.34300000000000003},"max":{"1min":1.1859999999999999,"5min":1.1859999999999999,"15min":1.1859999999999999},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.76300000000000001,"15min":0.70299999999999996},"min":{"1min":0.437,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.6519999999999999,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.65600000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:04:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.81599999999999995,"15min":0.78000000000000003},"min":{"1min":0.52800000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.0310000000000001,"5min":2.0310000000000001,"15min":2.0310000000000001},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.81599999999999995,"5min":0.78800000000000003,"15min":0.76400000000000001},"min":{"1min":0.48599999999999999,"5min":0.38,"15min":0.38},"max":{"1min":2.1499999999999999,"5min":2.1499999999999999,"15min":2.1499999999999999},"last":1.254}]},{"osd":4,"last update":"Mon Oct 9 09:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.96799999999999997,"15min":0.94399999999999995},"min":{"1min":0.66400000000000003,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.274,"5min":1.274,"15min":1.274},"last":1.1850000000000001},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.95999999999999996,"15min":0.94499999999999995},"min":{"1min":0.72999999999999998,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.1779999999999999,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.151}]},{"osd":5,"last update":"Mon Oct 9 09:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.94999999999999996,"15min":0.95499999999999996},"min":{"1min":0.74299999999999999,"5min":0.63,"15min":0.63},"max":{"1min":1.1950000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.98499999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.94999999999999996,"15min":0.94899999999999995},"min":{"1min":0.68100000000000005,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.294,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.1399999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0580000000000001,"15min":1.1240000000000001},"min":{"1min":0.80300000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.5960000000000001,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0429999999999999,"15min":1.081},"min":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.3180000000000001,"5min":1.464,"15min":1.464},"last":1.343}]},{"osd":7,"last update":"Mon Oct 9 09:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.98899999999999999,"15min":1.012},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.337,"5min":1.488,"15min":1.488},"last":1.103},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.0640000000000001,"15min":1.119},"min":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.5389999999999999,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.288}]}]},{"osd":3,"up_from":25,"seq":107374182487,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":273914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:05 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.45700000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.94799999999999995,"5min":0.98899999999999999,"15min":1.1399999999999999},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.71999999999999997,"15min":0.73699999999999999},"min":{"1min":0.42699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.98599999999999999,"5min":1.0660000000000001,"15min":1.4239999999999999},"last":0.54100000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:04:05 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.71899999999999997,"15min":0.72499999999999998},"min":{"1min":0.42299999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.002,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.70399999999999996,"15min":0.748},"min":{"1min":0.441,"5min":0.309,"15min":0.309},"max":{"1min":0.95399999999999996,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.82999999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:04:05 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.78400000000000003,"15min":0.82899999999999996},"min":{"1min":0.496,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.7829999999999999},"last":0.752},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.76500000000000001,"15min":0.79000000000000004},"min":{"1min":0.40100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.024,"5min":1.069,"15min":1.24},"last":0.79100000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:04:48 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.91500000000000004,"15min":0.95099999999999996},"min":{"1min":0.60499999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.3340000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.878},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.93600000000000005,"15min":0.92200000000000004},"min":{"1min":0.68600000000000005,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.294,"5min":1.45,"15min":1.45},"last":0.72499999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:04:19 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.95499999999999996,"15min":0.98099999999999998},"min":{"1min":0.61599999999999999,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.331,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.98699999999999999,"15min":0.996},"min":{"1min":0.60699999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.339,"5min":1.421,"15min":1.421},"last":1.0189999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.048,"15min":1.1020000000000001},"min":{"1min":0.77300000000000002,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.5369999999999999,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.056,"15min":1.095},"min":{"1min":0.77200000000000002,"5min":0.62,"15min":0.62},"max":{"1min":1.4079999999999999,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":0.95599999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:04:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.046,"15min":1.0900000000000001},"min":{"1min":0.68100000000000005,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.532,"5min":1.532,"15min":1.532},"last":1.0389999999999999},{"interface":"front","average":{"1min":1.024,"5min":1.0549999999999999,"15min":1.1160000000000001},"min":{"1min":0.65300000000000002,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.46,"5min":1.518,"15min":1.518},"last":1.069}]}]},{"osd":4,"up_from":31,"seq":133143986256,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297740,"kb_used_data":644,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447412,"statfs":{"total":95995035648,"available":95690149888,"internally_reserved":0,"allocated":659456,"data_stored":275506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.86699999999999999,"15min":0.83999999999999997},"min":{"1min":0.56100000000000005,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.8700000000000001,"5min":2.8700000000000001,"15min":2.8700000000000001},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.86799999999999999,"15min":0.85799999999999998},"min":{"1min":0.61099999999999999,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.792,"5min":1.792,"15min":1.792},"last":0.74299999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.83899999999999997,"15min":0.81499999999999995},"min":{"1min":0.56999999999999995,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.052,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.91300000000000003},{"interface":"front","average":{"1min":0.998,"5min":0.89700000000000002,"15min":0.879},"min":{"1min":0.61299999999999999,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":3.0550000000000002,"5min":3.0550000000000002,"15min":3.0550000000000002},"last":0.96299999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":0.93100000000000005,"15min":0.86099999999999999},"min":{"1min":0.65600000000000003,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":2.972,"5min":2.972,"15min":2.972},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.996,"5min":0.90500000000000003,"15min":0.875},"min":{"1min":0.69399999999999995,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.873,"5min":1.873,"15min":1.873},"last":0.82299999999999995}]},{"osd":3,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.90900000000000003,"15min":0.88300000000000001},"min":{"1min":0.72099999999999997,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.9379999999999999,"5min":1.9379999999999999,"15min":1.9379999999999999},"last":1.01},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.93899999999999995,"15min":0.91800000000000004},"min":{"1min":0.80800000000000005,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.1759999999999999,"5min":2.028,"15min":2.028},"last":1.1080000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:04:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":0.92900000000000005,"15min":0.84899999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":2.052,"5min":2.052,"15min":2.052},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.0249999999999999,"15min":0.95299999999999996},"min":{"1min":0.54400000000000004,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":3.1389999999999998,"5min":3.1389999999999998,"15min":3.1389999999999998},"last":1.1359999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:04:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.93500000000000005,"15min":0.97299999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.226,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.157},{"interface":"front","average":{"1min":1.097,"5min":1.0600000000000001,"15min":1.087},"min":{"1min":0.59799999999999998,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.956,"5min":1.956,"15min":1.956},"last":1.244}]},{"osd":7,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":0.96299999999999997,"15min":0.90100000000000002},"min":{"1min":0.68100000000000005,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.9790000000000001,"5min":1.9790000000000001,"15min":1.9790000000000001},"last":1.032},{"interface":"front","average":{"1min":1.236,"5min":1.026,"15min":0.95199999999999996},"min":{"1min":0.83199999999999996,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":3.2309999999999999,"5min":3.2309999999999999,"15min":3.2309999999999999},"last":1.052}]}]},{"osd":5,"up_from":37,"seq":158913790022,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":274133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.86799999999999999,"15min":0.83399999999999996},"min":{"1min":0.68799999999999994,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.151,"5min":2.052,"15min":2.052},"last":1.083},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.86499999999999999,"15min":0.83799999999999997},"min":{"1min":0.55500000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":0.80600000000000005}]},{"osd":1,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.875,"15min":0.85299999999999998},"min":{"1min":0.55600000000000005,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.399,"5min":2.214,"15min":2.214},"last":0.872},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.878,"15min":0.83299999999999996},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.24,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":1.125}]},{"osd":2,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.89100000000000001,"15min":0.86099999999999999},"min":{"1min":0.59999999999999998,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.266,"5min":1.323,"15min":1.323},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.91600000000000004,"15min":0.90400000000000003},"min":{"1min":0.66600000000000004,"5min":0.495,"15min":0.495},"max":{"1min":1.2809999999999999,"5min":2.1200000000000001,"15min":2.1200000000000001},"last":1.149}]},{"osd":3,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.92600000000000005,"15min":0.90600000000000003},"min":{"1min":0.65300000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2150000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.997},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.93999999999999995,"15min":0.90200000000000002},"min":{"1min":0.66500000000000004,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.472,"5min":2.1669999999999998,"15min":2.1669999999999998},"last":0.96499999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.79400000000000004,"15min":0.72599999999999998},"min":{"1min":0.60699999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.85199999999999998,"15min":0.754},"min":{"1min":0.58999999999999997,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.2809999999999999,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.0549999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.91200000000000003,"15min":0.878},"min":{"1min":0.56200000000000006,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.0249999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"min":{"1min":0.78000000000000003,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.331,"5min":1.496,"15min":1.496},"last":1.109}]},{"osd":7,"last update":"Mon Oct 9 09:04:41 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":0.99099999999999999,"15min":0.94399999999999995},"min":{"1min":0.83999999999999997,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3129999999999999,"5min":1.319,"15min":1.319},"last":0.93200000000000005},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.0329999999999999,"15min":0.99099999999999999},"min":{"1min":0.95199999999999996,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.534,"5min":2.4809999999999999,"15min":2.4809999999999999},"last":1.194}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:04:55.693 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T09:04:55.693 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T09:04:55.694 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T09:04:55.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T09:04:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:56 smithi181 bash[27712]: audit 2023-10-09T09:04:55.014931+0000 mgr.y (mgr.24355) 148 : audit [DBG] from='client.24704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:04:56.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:56 smithi181 bash[27712]: cluster 2023-10-09T09:04:55.236696+0000 mgr.y (mgr.24355) 149 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:56 smithi163 bash[27500]: audit 2023-10-09T09:04:55.014931+0000 mgr.y (mgr.24355) 148 : audit [DBG] from='client.24704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:04:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:56 smithi163 bash[27500]: cluster 2023-10-09T09:04:55.236696+0000 mgr.y (mgr.24355) 149 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:56 smithi163 bash[33912]: audit 2023-10-09T09:04:55.014931+0000 mgr.y (mgr.24355) 148 : audit [DBG] from='client.24704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:04:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:56 smithi163 bash[33912]: cluster 2023-10-09T09:04:55.236696+0000 mgr.y (mgr.24355) 149 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:04:58 smithi163 bash[33912]: cluster 2023-10-09T09:04:57.237171+0000 mgr.y (mgr.24355) 150 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:58.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:04:58 smithi163 bash[27500]: cluster 2023-10-09T09:04:57.237171+0000 mgr.y (mgr.24355) 150 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:58.553 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:04:58 smithi181 bash[27712]: cluster 2023-10-09T09:04:57.237171+0000 mgr.y (mgr.24355) 150 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:04:58.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:04:58 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:05:00.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:04:59 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:04:59] "GET /metrics HTTP/1.1" 200 191375 "" "Prometheus/2.33.4" 2023-10-09T09:05:00.506 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:05:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:00 smithi181 bash[27712]: cluster 2023-10-09T09:04:59.238155+0000 mgr.y (mgr.24355) 151 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:00 smithi163 bash[27500]: cluster 2023-10-09T09:04:59.238155+0000 mgr.y (mgr.24355) 151 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:00 smithi163 bash[33912]: cluster 2023-10-09T09:04:59.238155+0000 mgr.y (mgr.24355) 151 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:02.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:02 smithi163 bash[27500]: cluster 2023-10-09T09:05:01.238754+0000 mgr.y (mgr.24355) 152 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:02 smithi163 bash[33912]: cluster 2023-10-09T09:05:01.238754+0000 mgr.y (mgr.24355) 152 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:02 smithi181 bash[27712]: cluster 2023-10-09T09:05:01.238754+0000 mgr.y (mgr.24355) 152 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:03.408 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:05:04.119 INFO:teuthology.orchestra.run.smithi163.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T09:05:04.120 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T09:05:04.120 INFO:teuthology.run_tasks:Running task print... 2023-10-09T09:05:04.132 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T09:05:04.132 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T09:05:04.157 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T09:05:04.158 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T09:05:04.159 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T09:05:04.160 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2023-10-09T09:05:04.162 INFO:tasks.rados:Beginning rados... 2023-10-09T09:05:04.162 INFO:tasks.rados:joining rados 2023-10-09T09:05:04.162 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T09:05:04.162 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T09:05:04.162 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi163.front.sepia.ceph.com 2023-10-09T09:05:04.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T09:05:04.163 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2023-10-09T09:05:04.163 INFO:tasks.rados:starting run 0 out of 1 2023-10-09T09:05:04.163 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T09:05:04.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T09:05:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:04 smithi163 bash[27500]: cluster 2023-10-09T09:05:03.239747+0000 mgr.y (mgr.24355) 153 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:04 smithi163 bash[27500]: audit 2023-10-09T09:05:03.408287+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.163:0/2430591986' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:05:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:04 smithi163 bash[33912]: cluster 2023-10-09T09:05:03.239747+0000 mgr.y (mgr.24355) 153 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:04 smithi163 bash[33912]: audit 2023-10-09T09:05:03.408287+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.163:0/2430591986' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:05:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:04 smithi181 bash[27712]: cluster 2023-10-09T09:05:03.239747+0000 mgr.y (mgr.24355) 153 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:04 smithi181 bash[27712]: audit 2023-10-09T09:05:03.408287+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.163:0/2430591986' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:05:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:06 smithi181 bash[27712]: cluster 2023-10-09T09:05:05.240497+0000 mgr.y (mgr.24355) 154 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:06.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:06 smithi163 bash[27500]: cluster 2023-10-09T09:05:05.240497+0000 mgr.y (mgr.24355) 154 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:06.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:06 smithi163 bash[33912]: cluster 2023-10-09T09:05:05.240497+0000 mgr.y (mgr.24355) 154 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:07.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:07 smithi163 bash[27764]: [09/Oct/2023:09:05:07] ENGINE Bus STOPPING 2023-10-09T09:05:07.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:07 smithi163 bash[27764]: [09/Oct/2023:09:05:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:05:07.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:07 smithi163 bash[27764]: [09/Oct/2023:09:05:07] ENGINE Bus STOPPED 2023-10-09T09:05:07.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:07 smithi163 bash[27764]: [09/Oct/2023:09:05:07] ENGINE Bus STARTING 2023-10-09T09:05:07.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:07 smithi163 bash[27764]: [09/Oct/2023:09:05:07] ENGINE Serving on http://:::9283 2023-10-09T09:05:07.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:07 smithi163 bash[27764]: [09/Oct/2023:09:05:07] ENGINE Bus STARTED 2023-10-09T09:05:07.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T09:05:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:08 smithi163 bash[33912]: audit 2023-10-09T09:05:07.049092+0000 mon.a (mon.0) 741 : audit [INF] from='client.? 172.21.15.163:0/2387898847' entity='client.admin' 2023-10-09T09:05:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:08 smithi163 bash[33912]: audit 2023-10-09T09:05:07.100846+0000 mon.a (mon.0) 742 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:08 smithi163 bash[33912]: cluster 2023-10-09T09:05:07.241139+0000 mgr.y (mgr.24355) 155 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:08 smithi163 bash[27500]: audit 2023-10-09T09:05:07.049092+0000 mon.a (mon.0) 741 : audit [INF] from='client.? 172.21.15.163:0/2387898847' entity='client.admin' 2023-10-09T09:05:08.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:08 smithi163 bash[27500]: audit 2023-10-09T09:05:07.100846+0000 mon.a (mon.0) 742 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:08.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:08 smithi163 bash[27500]: cluster 2023-10-09T09:05:07.241139+0000 mgr.y (mgr.24355) 155 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:08 smithi181 bash[27712]: audit 2023-10-09T09:05:07.049092+0000 mon.a (mon.0) 741 : audit [INF] from='client.? 172.21.15.163:0/2387898847' entity='client.admin' 2023-10-09T09:05:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:08 smithi181 bash[27712]: audit 2023-10-09T09:05:07.100846+0000 mon.a (mon.0) 742 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:08 smithi181 bash[27712]: cluster 2023-10-09T09:05:07.241139+0000 mgr.y (mgr.24355) 155 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:08.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:05:08 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:05:09.230 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:05:10.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:09 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:09] "GET /metrics HTTP/1.1" 200 191387 "" "Prometheus/2.33.4" 2023-10-09T09:05:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:10 smithi181 bash[27712]: cluster 2023-10-09T09:05:09.241775+0000 mgr.y (mgr.24355) 156 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:10.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:10 smithi163 bash[27500]: cluster 2023-10-09T09:05:09.241775+0000 mgr.y (mgr.24355) 156 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:10.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:10 smithi163 bash[33912]: cluster 2023-10-09T09:05:09.241775+0000 mgr.y (mgr.24355) 156 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:11.193 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:11 smithi163 bash[27764]: [09/Oct/2023:09:05:11] ENGINE Bus STOPPING 2023-10-09T09:05:11.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:11 smithi163 bash[27764]: [09/Oct/2023:09:05:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:05:11.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:11 smithi163 bash[27764]: [09/Oct/2023:09:05:11] ENGINE Bus STOPPED 2023-10-09T09:05:11.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:11 smithi163 bash[27764]: [09/Oct/2023:09:05:11] ENGINE Bus STARTING 2023-10-09T09:05:11.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:11 smithi163 bash[27764]: [09/Oct/2023:09:05:11] ENGINE Serving on http://:::9283 2023-10-09T09:05:11.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:11 smithi163 bash[27764]: [09/Oct/2023:09:05:11] ENGINE Bus STARTED 2023-10-09T09:05:12.203 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T09:05:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:12 smithi163 bash[33912]: audit 2023-10-09T09:05:11.187919+0000 mon.a (mon.0) 743 : audit [INF] from='client.? 172.21.15.163:0/2966493385' entity='client.admin' 2023-10-09T09:05:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:12 smithi163 bash[33912]: cluster 2023-10-09T09:05:11.242325+0000 mgr.y (mgr.24355) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:12 smithi163 bash[27500]: audit 2023-10-09T09:05:11.187919+0000 mon.a (mon.0) 743 : audit [INF] from='client.? 172.21.15.163:0/2966493385' entity='client.admin' 2023-10-09T09:05:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:12 smithi163 bash[27500]: cluster 2023-10-09T09:05:11.242325+0000 mgr.y (mgr.24355) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:12 smithi181 bash[27712]: audit 2023-10-09T09:05:11.187919+0000 mon.a (mon.0) 743 : audit [INF] from='client.? 172.21.15.163:0/2966493385' entity='client.admin' 2023-10-09T09:05:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:12 smithi181 bash[27712]: cluster 2023-10-09T09:05:11.242325+0000 mgr.y (mgr.24355) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:13.442 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_0' created 2023-10-09T09:05:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:12.912246+0000 mon.c (mon.2) 32 : audit [INF] from='client.? 172.21.15.163:0/1348290532' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:05:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:12.912832+0000 mon.a (mon.0) 744 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:05:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:13.390255+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:13.394918+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:13.396240+0000 mon.a (mon.0) 747 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:13.397319+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:13.402371+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:13.408420+0000 mon.a (mon.0) 750 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:13.410288+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:13.411201+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[27500]: audit 2023-10-09T09:05:13.416270+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:12.912246+0000 mon.c (mon.2) 32 : audit [INF] from='client.? 172.21.15.163:0/1348290532' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:05:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:12.912832+0000 mon.a (mon.0) 744 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:05:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:13.390255+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:13.394918+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:13.396240+0000 mon.a (mon.0) 747 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:13.397319+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:13.402371+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:13.408420+0000 mon.a (mon.0) 750 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:13.410288+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:13.411201+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:13 smithi163 bash[33912]: audit 2023-10-09T09:05:13.416270+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:12.912246+0000 mon.c (mon.2) 32 : audit [INF] from='client.? 172.21.15.163:0/1348290532' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:05:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:12.912832+0000 mon.a (mon.0) 744 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:05:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:13.390255+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:13.394918+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:13.396240+0000 mon.a (mon.0) 747 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:13.397319+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:13.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:13.402371+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:13.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:13.408420+0000 mon.a (mon.0) 750 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:13.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:13.410288+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:13.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:13.411201+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:13.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:13 smithi181 bash[27712]: audit 2023-10-09T09:05:13.416270+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:14.706 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:14 smithi163 bash[33912]: cluster 2023-10-09T09:05:13.243081+0000 mgr.y (mgr.24355) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:14.707 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:14 smithi163 bash[33912]: audit 2023-10-09T09:05:13.434475+0000 mon.a (mon.0) 754 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T09:05:14.707 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:14 smithi163 bash[33912]: cluster 2023-10-09T09:05:13.434559+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T09:05:14.707 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:14 smithi163 bash[33912]: cluster 2023-10-09T09:05:14.440011+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T09:05:14.708 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:14 smithi163 bash[27500]: cluster 2023-10-09T09:05:13.243081+0000 mgr.y (mgr.24355) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:14.708 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:14 smithi163 bash[27500]: audit 2023-10-09T09:05:13.434475+0000 mon.a (mon.0) 754 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T09:05:14.708 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:14 smithi163 bash[27500]: cluster 2023-10-09T09:05:13.434559+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T09:05:14.708 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:14 smithi163 bash[27500]: cluster 2023-10-09T09:05:14.440011+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T09:05:14.709 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T09:05:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:14 smithi181 bash[27712]: cluster 2023-10-09T09:05:13.243081+0000 mgr.y (mgr.24355) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:14 smithi181 bash[27712]: audit 2023-10-09T09:05:13.434475+0000 mon.a (mon.0) 754 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T09:05:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:14 smithi181 bash[27712]: cluster 2023-10-09T09:05:13.434559+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T09:05:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:14 smithi181 bash[27712]: cluster 2023-10-09T09:05:14.440011+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T09:05:16.033 INFO:teuthology.orchestra.run.smithi163.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:05:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[33912]: cluster 2023-10-09T09:05:15.243650+0000 mgr.y (mgr.24355) 159 : cluster [DBG] pgmap v140: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[33912]: cluster 2023-10-09T09:05:15.447302+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T09:05:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[33912]: audit 2023-10-09T09:05:16.032969+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[33912]: audit 2023-10-09T09:05:16.043049+0000 mon.a (mon.0) 759 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[33912]: audit 2023-10-09T09:05:16.045224+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[33912]: audit 2023-10-09T09:05:16.046203+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[33912]: audit 2023-10-09T09:05:16.055817+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[27500]: cluster 2023-10-09T09:05:15.243650+0000 mgr.y (mgr.24355) 159 : cluster [DBG] pgmap v140: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[27500]: cluster 2023-10-09T09:05:15.447302+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T09:05:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[27500]: audit 2023-10-09T09:05:16.032969+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[27500]: audit 2023-10-09T09:05:16.043049+0000 mon.a (mon.0) 759 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[27500]: audit 2023-10-09T09:05:16.045224+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[27500]: audit 2023-10-09T09:05:16.046203+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:16 smithi163 bash[27500]: audit 2023-10-09T09:05:16.055817+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:16 smithi181 bash[27712]: cluster 2023-10-09T09:05:15.243650+0000 mgr.y (mgr.24355) 159 : cluster [DBG] pgmap v140: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:16 smithi181 bash[27712]: cluster 2023-10-09T09:05:15.447302+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T09:05:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:16 smithi181 bash[27712]: audit 2023-10-09T09:05:16.032969+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:16 smithi181 bash[27712]: audit 2023-10-09T09:05:16.043049+0000 mon.a (mon.0) 759 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:16 smithi181 bash[27712]: audit 2023-10-09T09:05:16.045224+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:05:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:16 smithi181 bash[27712]: audit 2023-10-09T09:05:16.046203+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:05:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:16 smithi181 bash[27712]: audit 2023-10-09T09:05:16.055817+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:16.833 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:05:17.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:17 smithi163 bash[27500]: audit 2023-10-09T09:05:16.025322+0000 mgr.y (mgr.24355) 160 : audit [DBG] from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:05:17.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:17 smithi163 bash[27500]: cephadm 2023-10-09T09:05:16.026752+0000 mgr.y (mgr.24355) 161 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:05:17.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:17 smithi163 bash[27500]: cephadm 2023-10-09T09:05:16.062985+0000 mgr.y (mgr.24355) 162 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:05:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:17 smithi163 bash[33912]: audit 2023-10-09T09:05:16.025322+0000 mgr.y (mgr.24355) 160 : audit [DBG] from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:05:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:17 smithi163 bash[33912]: cephadm 2023-10-09T09:05:16.026752+0000 mgr.y (mgr.24355) 161 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:05:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:17 smithi163 bash[33912]: cephadm 2023-10-09T09:05:16.062985+0000 mgr.y (mgr.24355) 162 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:05:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:17 smithi181 bash[27712]: audit 2023-10-09T09:05:16.025322+0000 mgr.y (mgr.24355) 160 : audit [DBG] from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:05:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:17 smithi181 bash[27712]: cephadm 2023-10-09T09:05:16.026752+0000 mgr.y (mgr.24355) 161 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:05:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:17 smithi181 bash[27712]: cephadm 2023-10-09T09:05:16.062985+0000 mgr.y (mgr.24355) 162 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:05:18.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:18 smithi163 bash[27500]: cluster 2023-10-09T09:05:17.244098+0000 mgr.y (mgr.24355) 163 : cluster [DBG] pgmap v142: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:18 smithi163 bash[33912]: cluster 2023-10-09T09:05:17.244098+0000 mgr.y (mgr.24355) 163 : cluster [DBG] pgmap v142: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:18 smithi181 bash[27712]: cluster 2023-10-09T09:05:17.244098+0000 mgr.y (mgr.24355) 163 : cluster [DBG] pgmap v142: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:18.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:05:18 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:05:19.509 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:05:20.343 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:19 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:19] "GET /metrics HTTP/1.1" 200 191387 "" "Prometheus/2.33.4" 2023-10-09T09:05:20.616 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:20 smithi163 bash[33912]: cluster 2023-10-09T09:05:19.244979+0000 mgr.y (mgr.24355) 164 : cluster [DBG] pgmap v143: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:20.617 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:20 smithi163 bash[27500]: cluster 2023-10-09T09:05:19.244979+0000 mgr.y (mgr.24355) 164 : cluster [DBG] pgmap v143: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:20 smithi181 bash[27712]: cluster 2023-10-09T09:05:19.244979+0000 mgr.y (mgr.24355) 164 : cluster [DBG] pgmap v143: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:20.821 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:05:20.821 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (2m) 7s ago 3m 14.5M - 0.23.0 ba2b418f427c 3f33d6211b34 2023-10-09T09:05:20.821 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (3m) 7s ago 3m 43.0M - 8.3.5 dad864ee21e9 26db32f8d86f 2023-10-09T09:05:20.821 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.vnodzj smithi163 running (112s) 7s ago 113s 13.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 892077b22450 2023-10-09T09:05:20.821 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.ajqgsc smithi181 running (110s) 7s ago 110s 11.4M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 01d89554089b 2023-10-09T09:05:20.821 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443 running (10m) 7s ago 10m 377M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 373283eaf41f 2023-10-09T09:05:20.821 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:9283 running (12m) 7s ago 12m 427M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4d1573887dd3 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (12m) 7s ago 12m 48.9M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 429e9872cbc8 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (10m) 7s ago 10m 41.5M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 443a80433a1f 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (10m) 7s ago 10m 40.8M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d26d17ef0c6e 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (3m) 7s ago 3m 9444k - 1.3.1 1dbe0e931976 d1855daae1fa 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (3m) 7s ago 3m 9023k - 1.3.1 1dbe0e931976 54720b50cdeb 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (9m) 7s ago 9m 38.1M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d07769e94b6f 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (8m) 7s ago 8m 38.6M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7a2783c3c11c 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (8m) 7s ago 8m 38.0M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 35670f565f8c 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (7m) 7s ago 7m 38.8M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 402aa34f21f4 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (6m) 7s ago 6m 37.7M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5dd6bc4b279 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (6m) 7s ago 6m 37.6M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9f3e4fbba907 2023-10-09T09:05:20.822 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (5m) 7s ago 5m 37.7M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 30f1ac2f4524 2023-10-09T09:05:20.823 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (4m) 7s ago 4m 37.7M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 54105d827fdd 2023-10-09T09:05:20.823 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (2m) 7s ago 3m 51.2M - 2.33.4 514e6a882f6e 866421ebfe3c 2023-10-09T09:05:22.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T09:05:22.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:22 smithi163 bash[27500]: audit 2023-10-09T09:05:20.802255+0000 mgr.y (mgr.24355) 165 : audit [DBG] from='client.24740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:05:22.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:22 smithi163 bash[27500]: cluster 2023-10-09T09:05:21.245430+0000 mgr.y (mgr.24355) 166 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:22 smithi163 bash[33912]: audit 2023-10-09T09:05:20.802255+0000 mgr.y (mgr.24355) 165 : audit [DBG] from='client.24740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:05:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:22 smithi163 bash[33912]: cluster 2023-10-09T09:05:21.245430+0000 mgr.y (mgr.24355) 166 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:22 smithi181 bash[27712]: audit 2023-10-09T09:05:20.802255+0000 mgr.y (mgr.24355) 165 : audit [DBG] from='client.24740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:05:22.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:22 smithi181 bash[27712]: cluster 2023-10-09T09:05:21.245430+0000 mgr.y (mgr.24355) 166 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:24.470 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T09:05:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:24 smithi163 bash[27500]: cluster 2023-10-09T09:05:23.246507+0000 mgr.y (mgr.24355) 167 : cluster [DBG] pgmap v145: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:24 smithi163 bash[27500]: audit 2023-10-09T09:05:24.153456+0000 mon.a (mon.0) 763 : audit [INF] from='client.? 172.21.15.163:0/3623142769' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:05:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:24 smithi163 bash[33912]: cluster 2023-10-09T09:05:23.246507+0000 mgr.y (mgr.24355) 167 : cluster [DBG] pgmap v145: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:24 smithi163 bash[33912]: audit 2023-10-09T09:05:24.153456+0000 mon.a (mon.0) 763 : audit [INF] from='client.? 172.21.15.163:0/3623142769' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:05:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:24 smithi181 bash[27712]: cluster 2023-10-09T09:05:23.246507+0000 mgr.y (mgr.24355) 167 : cluster [DBG] pgmap v145: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:24.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:24 smithi181 bash[27712]: audit 2023-10-09T09:05:24.153456+0000 mon.a (mon.0) 763 : audit [INF] from='client.? 172.21.15.163:0/3623142769' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:05:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:25 smithi181 bash[27712]: audit 2023-10-09T09:05:24.465418+0000 mon.a (mon.0) 764 : audit [INF] from='client.? 172.21.15.163:0/3623142769' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:05:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:25 smithi181 bash[27712]: cluster 2023-10-09T09:05:24.465583+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T09:05:25.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:25 smithi163 bash[33912]: audit 2023-10-09T09:05:24.465418+0000 mon.a (mon.0) 764 : audit [INF] from='client.? 172.21.15.163:0/3623142769' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:05:25.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:25 smithi163 bash[33912]: cluster 2023-10-09T09:05:24.465583+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T09:05:25.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:25 smithi163 bash[27500]: audit 2023-10-09T09:05:24.465418+0000 mon.a (mon.0) 764 : audit [INF] from='client.? 172.21.15.163:0/3623142769' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:05:25.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:25 smithi163 bash[27500]: cluster 2023-10-09T09:05:24.465583+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T09:05:25.990 INFO:teuthology.orchestra.run.smithi163.stdout:wait for minority of mons to upgrade 2023-10-09T09:05:26.626 DEBUG:teuthology.orchestra.run.smithi163:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 100 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --pool unique_pool_0 2023-10-09T09:05:26.643 INFO:tasks.rados.rados.0.smithi163.stdout:adding op weight read -> 100 2023-10-09T09:05:26.643 INFO:tasks.rados.rados.0.smithi163.stdout:adding op weight write -> 100 2023-10-09T09:05:26.643 INFO:tasks.rados.rados.0.smithi163.stdout:adding op weight delete -> 50 2023-10-09T09:05:26.643 INFO:tasks.rados.rados.0.smithi163.stdout:adding op weight snap_create -> 50 2023-10-09T09:05:26.643 INFO:tasks.rados.rados.0.smithi163.stdout:adding op weight snap_remove -> 50 2023-10-09T09:05:26.643 INFO:tasks.rados.rados.0.smithi163.stdout:adding op weight rollback -> 50 2023-10-09T09:05:26.643 INFO:tasks.rados.rados.0.smithi163.stdout:ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T09:05:26.643 INFO:tasks.rados.rados.0.smithi163.stdout:Configuration: 2023-10-09T09:05:26.644 INFO:tasks.rados.rados.0.smithi163.stdout: Number of operations: 4000 2023-10-09T09:05:26.644 INFO:tasks.rados.rados.0.smithi163.stdout: Number of objects: 50 2023-10-09T09:05:26.644 INFO:tasks.rados.rados.0.smithi163.stdout: Max in flight operations: 16 2023-10-09T09:05:26.644 INFO:tasks.rados.rados.0.smithi163.stdout: Object size (in bytes): 4000000 2023-10-09T09:05:26.644 INFO:tasks.rados.rados.0.smithi163.stdout: Write stride min: 400000 2023-10-09T09:05:26.644 INFO:tasks.rados.rados.0.smithi163.stdout: Write stride max: 800000 2023-10-09T09:05:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:26 smithi181 bash[27712]: cluster 2023-10-09T09:05:25.247181+0000 mgr.y (mgr.24355) 168 : cluster [DBG] pgmap v147: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:26.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:26 smithi163 bash[33912]: cluster 2023-10-09T09:05:25.247181+0000 mgr.y (mgr.24355) 168 : cluster [DBG] pgmap v147: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:26.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:26 smithi163 bash[27500]: cluster 2023-10-09T09:05:25.247181+0000 mgr.y (mgr.24355) 168 : cluster [DBG] pgmap v147: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:26.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T09:05:27.492 INFO:tasks.rados.rados.0.smithi163.stdout:1: write initial oid 1 2023-10-09T09:05:27.492 INFO:tasks.rados.rados.0.smithi163.stdout:1: seq_num 0 ranges {416807=475249,1542129=474678} 2023-10-09T09:05:27.500 INFO:tasks.rados.rados.0.smithi163.stdout:1: writing smithi16367185-1 from 416807 to 892056 tid 1 2023-10-09T09:05:27.508 INFO:tasks.rados.rados.0.smithi163.stdout:1: writing smithi16367185-1 from 1542129 to 2016807 tid 2 2023-10-09T09:05:27.508 INFO:tasks.rados.rados.0.smithi163.stdout:2: write initial oid 2 2023-10-09T09:05:27.508 INFO:tasks.rados.rados.0.smithi163.stdout:2: seq_num 1 ranges {416807=475249,1542129=474678} 2023-10-09T09:05:27.514 INFO:tasks.rados.rados.0.smithi163.stdout:2: writing smithi16367185-2 from 416807 to 892056 tid 1 2023-10-09T09:05:27.521 INFO:tasks.rados.rados.0.smithi163.stdout:2: writing smithi16367185-2 from 1542129 to 2016807 tid 2 2023-10-09T09:05:27.522 INFO:tasks.rados.rados.0.smithi163.stdout:3: write initial oid 3 2023-10-09T09:05:27.522 INFO:tasks.rados.rados.0.smithi163.stdout:3: seq_num 2 ranges {433614=550498,1600611=433003} 2023-10-09T09:05:27.527 INFO:tasks.rados.rados.0.smithi163.stdout:3: writing smithi16367185-3 from 433614 to 984112 tid 1 2023-10-09T09:05:27.534 INFO:tasks.rados.rados.0.smithi163.stdout:3: writing smithi16367185-3 from 1600611 to 2033614 tid 2 2023-10-09T09:05:27.534 INFO:tasks.rados.rados.0.smithi163.stdout:4: write initial oid 4 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:1: finishing write tid 1 to smithi16367185-1 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:1: finishing write tid 2 to smithi16367185-1 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:1: finishing write tid 3 to smithi16367185-1 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:1: finishing write tid 4 to smithi16367185-1 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 3 (ObjNum 0 snap 0 seq_num 0) dirty exists 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:2: finishing write tid 1 to smithi16367185-2 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:2: finishing write tid 2 to smithi16367185-2 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:2: finishing write tid 3 to smithi16367185-2 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:2: finishing write tid 4 to smithi16367185-2 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 3 (ObjNum 1 snap 0 seq_num 1) dirty exists 2023-10-09T09:05:27.535 INFO:tasks.rados.rados.0.smithi163.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2023-10-09T09:05:27.536 INFO:tasks.rados.rados.0.smithi163.stdout:3: finishing write tid 1 to smithi16367185-3 2023-10-09T09:05:27.536 INFO:tasks.rados.rados.0.smithi163.stdout:4: seq_num 3 ranges {450421=625747,1659093=391328} 2023-10-09T09:05:27.541 INFO:tasks.rados.rados.0.smithi163.stdout:4: writing smithi16367185-4 from 450421 to 1076168 tid 1 2023-10-09T09:05:27.547 INFO:tasks.rados.rados.0.smithi163.stdout:4: writing smithi16367185-4 from 1659093 to 2050421 tid 2 2023-10-09T09:05:27.547 INFO:tasks.rados.rados.0.smithi163.stdout:1: done (3 left) 2023-10-09T09:05:27.548 INFO:tasks.rados.rados.0.smithi163.stdout:2: done (2 left) 2023-10-09T09:05:27.548 INFO:tasks.rados.rados.0.smithi163.stdout:5: write initial oid 5 2023-10-09T09:05:27.548 INFO:tasks.rados.rados.0.smithi163.stdout:5: seq_num 4 ranges {467228=700996,1717575=349653} 2023-10-09T09:05:27.554 INFO:tasks.rados.rados.0.smithi163.stdout:5: writing smithi16367185-5 from 467228 to 1168224 tid 1 2023-10-09T09:05:27.560 INFO:tasks.rados.rados.0.smithi163.stdout:5: writing smithi16367185-5 from 1717575 to 2067228 tid 2 2023-10-09T09:05:27.560 INFO:tasks.rados.rados.0.smithi163.stdout:6: write initial oid 6 2023-10-09T09:05:27.560 INFO:tasks.rados.rados.0.smithi163.stdout:6: seq_num 5 ranges {484035=776245,2059704=24331} 2023-10-09T09:05:27.568 INFO:tasks.rados.rados.0.smithi163.stdout:6: writing smithi16367185-6 from 484035 to 1260280 tid 1 2023-10-09T09:05:27.573 INFO:tasks.rados.rados.0.smithi163.stdout:6: writing smithi16367185-6 from 2059704 to 2084035 tid 2 2023-10-09T09:05:27.574 INFO:tasks.rados.rados.0.smithi163.stdout:7: write initial oid 7 2023-10-09T09:05:27.574 INFO:tasks.rados.rados.0.smithi163.stdout:7: seq_num 6 ranges {500842=451494,1718186=382656} 2023-10-09T09:05:27.579 INFO:tasks.rados.rados.0.smithi163.stdout:7: writing smithi16367185-7 from 500842 to 952336 tid 1 2023-10-09T09:05:27.586 INFO:tasks.rados.rados.0.smithi163.stdout:7: writing smithi16367185-7 from 1718186 to 2100842 tid 2 2023-10-09T09:05:27.586 INFO:tasks.rados.rados.0.smithi163.stdout:8: write initial oid 8 2023-10-09T09:05:27.586 INFO:tasks.rados.rados.0.smithi163.stdout:8: seq_num 7 ranges {517649=526743,1776668=340981} 2023-10-09T09:05:27.592 INFO:tasks.rados.rados.0.smithi163.stdout:8: writing smithi16367185-8 from 517649 to 1044392 tid 1 2023-10-09T09:05:27.599 INFO:tasks.rados.rados.0.smithi163.stdout:8: writing smithi16367185-8 from 1776668 to 2117649 tid 2 2023-10-09T09:05:27.599 INFO:tasks.rados.rados.0.smithi163.stdout:9: write initial oid 9 2023-10-09T09:05:27.599 INFO:tasks.rados.rados.0.smithi163.stdout:9: seq_num 8 ranges {534456=718345,1951503=182953} 2023-10-09T09:05:27.606 INFO:tasks.rados.rados.0.smithi163.stdout:9: writing smithi16367185-9 from 534456 to 1252801 tid 1 2023-10-09T09:05:27.612 INFO:tasks.rados.rados.0.smithi163.stdout:9: writing smithi16367185-9 from 1951503 to 2134456 tid 2 2023-10-09T09:05:27.612 INFO:tasks.rados.rados.0.smithi163.stdout:10: write initial oid 10 2023-10-09T09:05:27.612 INFO:tasks.rados.rados.0.smithi163.stdout:10: seq_num 9 ranges {551263=793594,1893632=257631} 2023-10-09T09:05:27.619 INFO:tasks.rados.rados.0.smithi163.stdout:10: writing smithi16367185-10 from 551263 to 1344857 tid 1 2023-10-09T09:05:27.624 INFO:tasks.rados.rados.0.smithi163.stdout:10: writing smithi16367185-10 from 1893632 to 2151263 tid 2 2023-10-09T09:05:27.625 INFO:tasks.rados.rados.0.smithi163.stdout:11: write initial oid 11 2023-10-09T09:05:27.625 INFO:tasks.rados.rados.0.smithi163.stdout:11: seq_num 10 ranges {568070=468843,1552114=615956} 2023-10-09T09:05:27.630 INFO:tasks.rados.rados.0.smithi163.stdout:11: writing smithi16367185-11 from 568070 to 1036913 tid 1 2023-10-09T09:05:27.637 INFO:tasks.rados.rados.0.smithi163.stdout:11: writing smithi16367185-11 from 1552114 to 2168070 tid 2 2023-10-09T09:05:27.637 INFO:tasks.rados.rados.0.smithi163.stdout:12: write initial oid 12 2023-10-09T09:05:27.637 INFO:tasks.rados.rados.0.smithi163.stdout:12: seq_num 11 ranges {584877=544092,1610596=562003} 2023-10-09T09:05:27.643 INFO:tasks.rados.rados.0.smithi163.stdout:12: writing smithi16367185-12 from 584877 to 1128969 tid 1 2023-10-09T09:05:27.650 INFO:tasks.rados.rados.0.smithi163.stdout:12: writing smithi16367185-12 from 1610596 to 2172599 tid 2 2023-10-09T09:05:27.650 INFO:tasks.rados.rados.0.smithi163.stdout:13: write initial oid 13 2023-10-09T09:05:27.650 INFO:tasks.rados.rados.0.smithi163.stdout:13: seq_num 12 ranges {601684=619341,1669078=532606} 2023-10-09T09:05:27.657 INFO:tasks.rados.rados.0.smithi163.stdout:13: writing smithi16367185-13 from 601684 to 1221025 tid 1 2023-10-09T09:05:27.663 INFO:tasks.rados.rados.0.smithi163.stdout:13: writing smithi16367185-13 from 1669078 to 2201684 tid 2 2023-10-09T09:05:27.663 INFO:tasks.rados.rados.0.smithi163.stdout:14: write initial oid 14 2023-10-09T09:05:27.663 INFO:tasks.rados.rados.0.smithi163.stdout:14: seq_num 13 ranges {618491=694590,2011207=207284} 2023-10-09T09:05:27.670 INFO:tasks.rados.rados.0.smithi163.stdout:14: writing smithi16367185-14 from 618491 to 1313081 tid 1 2023-10-09T09:05:27.676 INFO:tasks.rados.rados.0.smithi163.stdout:14: writing smithi16367185-14 from 2011207 to 2218491 tid 2 2023-10-09T09:05:27.676 INFO:tasks.rados.rados.0.smithi163.stdout:15: write initial oid 15 2023-10-09T09:05:27.676 INFO:tasks.rados.rados.0.smithi163.stdout:15: seq_num 14 ranges {635298=769839,2069689=165609} 2023-10-09T09:05:27.683 INFO:tasks.rados.rados.0.smithi163.stdout:15: writing smithi16367185-15 from 635298 to 1405137 tid 1 2023-10-09T09:05:27.688 INFO:tasks.rados.rados.0.smithi163.stdout:15: writing smithi16367185-15 from 2069689 to 2235298 tid 2 2023-10-09T09:05:27.688 INFO:tasks.rados.rados.0.smithi163.stdout:16: write initial oid 16 2023-10-09T09:05:27.689 INFO:tasks.rados.rados.0.smithi163.stdout:16: seq_num 15 ranges {652105=445088,1728171=452988} 2023-10-09T09:05:27.695 INFO:tasks.rados.rados.0.smithi163.stdout:16: writing smithi16367185-16 from 652105 to 1097193 tid 1 2023-10-09T09:05:27.701 INFO:tasks.rados.rados.0.smithi163.stdout:16: writing smithi16367185-16 from 1728171 to 2181159 tid 2 2023-10-09T09:05:27.701 INFO:tasks.rados.rados.0.smithi163.stdout:17: write initial oid 17 2023-10-09T09:05:27.701 INFO:tasks.rados.rados.0.smithi163.stdout:17: seq_num 16 ranges {668912=636690,1903006=365906} 2023-10-09T09:05:27.709 INFO:tasks.rados.rados.0.smithi163.stdout:17: writing smithi16367185-17 from 668912 to 1305602 tid 1 2023-10-09T09:05:27.717 INFO:tasks.rados.rados.0.smithi163.stdout:17: writing smithi16367185-17 from 1903006 to 2268912 tid 2 2023-10-09T09:05:27.717 INFO:tasks.rados.rados.0.smithi163.stdout:18: write initial oid 18 2023-10-09T09:05:27.718 INFO:tasks.rados.rados.0.smithi163.stdout:18: seq_num 17 ranges {685719=711939,1845135=440584} 2023-10-09T09:05:27.726 INFO:tasks.rados.rados.0.smithi163.stdout:18: writing smithi16367185-18 from 685719 to 1397658 tid 1 2023-10-09T09:05:27.732 INFO:tasks.rados.rados.0.smithi163.stdout:18: writing smithi16367185-18 from 1845135 to 2285719 tid 2 2023-10-09T09:05:27.732 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.732 INFO:tasks.rados.rados.0.smithi163.stdout:3: finishing write tid 2 to smithi16367185-3 2023-10-09T09:05:27.732 INFO:tasks.rados.rados.0.smithi163.stdout:3: finishing write tid 3 to smithi16367185-3 2023-10-09T09:05:27.732 INFO:tasks.rados.rados.0.smithi163.stdout:3: finishing write tid 4 to smithi16367185-3 2023-10-09T09:05:27.732 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 3 (ObjNum 2 snap 0 seq_num 2) dirty exists 2023-10-09T09:05:27.732 INFO:tasks.rados.rados.0.smithi163.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2023-10-09T09:05:27.733 INFO:tasks.rados.rados.0.smithi163.stdout:4: finishing write tid 1 to smithi16367185-4 2023-10-09T09:05:27.733 INFO:tasks.rados.rados.0.smithi163.stdout:4: finishing write tid 2 to smithi16367185-4 2023-10-09T09:05:27.733 INFO:tasks.rados.rados.0.smithi163.stdout:4: finishing write tid 3 to smithi16367185-4 2023-10-09T09:05:27.733 INFO:tasks.rados.rados.0.smithi163.stdout:4: finishing write tid 4 to smithi16367185-4 2023-10-09T09:05:27.733 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 6 (ObjNum 3 snap 0 seq_num 3) dirty exists 2023-10-09T09:05:27.733 INFO:tasks.rados.rados.0.smithi163.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2023-10-09T09:05:27.733 INFO:tasks.rados.rados.0.smithi163.stdout:3: done (15 left) 2023-10-09T09:05:27.733 INFO:tasks.rados.rados.0.smithi163.stdout:4: done (14 left) 2023-10-09T09:05:27.733 INFO:tasks.rados.rados.0.smithi163.stdout:19: write initial oid 19 2023-10-09T09:05:27.733 INFO:tasks.rados.rados.0.smithi163.stdout:19: seq_num 18 ranges {702526=787188,1903617=398909} 2023-10-09T09:05:27.740 INFO:tasks.rados.rados.0.smithi163.stdout:19: writing smithi16367185-19 from 702526 to 1489714 tid 1 2023-10-09T09:05:27.745 INFO:tasks.rados.rados.0.smithi163.stdout:19: writing smithi16367185-19 from 1903617 to 2302526 tid 2 2023-10-09T09:05:27.745 INFO:tasks.rados.rados.0.smithi163.stdout:20: write initial oid 20 2023-10-09T09:05:27.746 INFO:tasks.rados.rados.0.smithi163.stdout:20: seq_num 19 ranges {719333=462437,1962099=357234} 2023-10-09T09:05:27.752 INFO:tasks.rados.rados.0.smithi163.stdout:20: writing smithi16367185-20 from 719333 to 1181770 tid 1 2023-10-09T09:05:27.759 INFO:tasks.rados.rados.0.smithi163.stdout:20: writing smithi16367185-20 from 1962099 to 2319333 tid 2 2023-10-09T09:05:27.759 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.759 INFO:tasks.rados.rados.0.smithi163.stdout:5: finishing write tid 1 to smithi16367185-5 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:5: finishing write tid 2 to smithi16367185-5 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:5: finishing write tid 3 to smithi16367185-5 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:5: finishing write tid 4 to smithi16367185-5 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 9 (ObjNum 4 snap 0 seq_num 4) dirty exists 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:6: finishing write tid 1 to smithi16367185-6 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:6: finishing write tid 2 to smithi16367185-6 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:6: finishing write tid 3 to smithi16367185-6 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:5: done (15 left) 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:21: write initial oid 21 2023-10-09T09:05:27.760 INFO:tasks.rados.rados.0.smithi163.stdout:21: seq_num 20 ranges {736140=537686,2020581=315559} 2023-10-09T09:05:27.766 INFO:tasks.rados.rados.0.smithi163.stdout:21: writing smithi16367185-21 from 736140 to 1273826 tid 1 2023-10-09T09:05:27.775 INFO:tasks.rados.rados.0.smithi163.stdout:21: writing smithi16367185-21 from 2020581 to 2336140 tid 2 2023-10-09T09:05:27.775 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.775 INFO:tasks.rados.rados.0.smithi163.stdout:6: finishing write tid 4 to smithi16367185-6 2023-10-09T09:05:27.775 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 3 (ObjNum 5 snap 0 seq_num 5) dirty exists 2023-10-09T09:05:27.775 INFO:tasks.rados.rados.0.smithi163.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2023-10-09T09:05:27.775 INFO:tasks.rados.rados.0.smithi163.stdout:7: finishing write tid 1 to smithi16367185-7 2023-10-09T09:05:27.775 INFO:tasks.rados.rados.0.smithi163.stdout:7: finishing write tid 2 to smithi16367185-7 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:7: finishing write tid 3 to smithi16367185-7 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:7: finishing write tid 4 to smithi16367185-7 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 3 (ObjNum 6 snap 0 seq_num 6) dirty exists 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:8: finishing write tid 1 to smithi16367185-8 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:8: finishing write tid 2 to smithi16367185-8 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:8: finishing write tid 3 to smithi16367185-8 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:8: finishing write tid 4 to smithi16367185-8 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 3 (ObjNum 7 snap 0 seq_num 7) dirty exists 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:9: finishing write tid 1 to smithi16367185-9 2023-10-09T09:05:27.776 INFO:tasks.rados.rados.0.smithi163.stdout:9: finishing write tid 2 to smithi16367185-9 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:9: finishing write tid 3 to smithi16367185-9 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:9: finishing write tid 4 to smithi16367185-9 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 3 (ObjNum 8 snap 0 seq_num 8) dirty exists 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:10: finishing write tid 1 to smithi16367185-10 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:10: finishing write tid 2 to smithi16367185-10 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:10: finishing write tid 3 to smithi16367185-10 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:10: finishing write tid 4 to smithi16367185-10 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 3 (ObjNum 9 snap 0 seq_num 9) dirty exists 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:11: finishing write tid 1 to smithi16367185-11 2023-10-09T09:05:27.777 INFO:tasks.rados.rados.0.smithi163.stdout:11: finishing write tid 2 to smithi16367185-11 2023-10-09T09:05:27.778 INFO:tasks.rados.rados.0.smithi163.stdout:6: done (15 left) 2023-10-09T09:05:27.778 INFO:tasks.rados.rados.0.smithi163.stdout:7: done (14 left) 2023-10-09T09:05:27.778 INFO:tasks.rados.rados.0.smithi163.stdout:8: done (13 left) 2023-10-09T09:05:27.778 INFO:tasks.rados.rados.0.smithi163.stdout:9: done (12 left) 2023-10-09T09:05:27.778 INFO:tasks.rados.rados.0.smithi163.stdout:10: done (11 left) 2023-10-09T09:05:27.778 INFO:tasks.rados.rados.0.smithi163.stdout:22: write initial oid 22 2023-10-09T09:05:27.778 INFO:tasks.rados.rados.0.smithi163.stdout:22: seq_num 21 ranges {752947=612935,1962710=390237} 2023-10-09T09:05:27.782 INFO:tasks.rados.rados.0.smithi163.stdout:22: writing smithi16367185-22 from 752947 to 1365882 tid 1 2023-10-09T09:05:27.789 INFO:tasks.rados.rados.0.smithi163.stdout:22: writing smithi16367185-22 from 1962710 to 2352947 tid 2 2023-10-09T09:05:27.789 INFO:tasks.rados.rados.0.smithi163.stdout:23: write initial oid 23 2023-10-09T09:05:27.789 INFO:tasks.rados.rados.0.smithi163.stdout:23: seq_num 22 ranges {769754=688184,2021192=348562} 2023-10-09T09:05:27.797 INFO:tasks.rados.rados.0.smithi163.stdout:23: writing smithi16367185-23 from 769754 to 1457938 tid 1 2023-10-09T09:05:27.802 INFO:tasks.rados.rados.0.smithi163.stdout:23: writing smithi16367185-23 from 2021192 to 2369754 tid 2 2023-10-09T09:05:27.802 INFO:tasks.rados.rados.0.smithi163.stdout:24: write initial oid 24 2023-10-09T09:05:27.803 INFO:tasks.rados.rados.0.smithi163.stdout:24: seq_num 23 ranges {786561=479786,1796027=467664} 2023-10-09T09:05:27.810 INFO:tasks.rados.rados.0.smithi163.stdout:24: writing smithi16367185-24 from 786561 to 1266347 tid 1 2023-10-09T09:05:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:27 smithi181 bash[27712]: audit 2023-10-09T09:05:26.656770+0000 mon.a (mon.0) 766 : audit [INF] from='client.? 172.21.15.163:0/486094513' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2023-10-09T09:05:27.818 INFO:tasks.rados.rados.0.smithi163.stdout:24: writing smithi16367185-24 from 1796027 to 2263691 tid 2 2023-10-09T09:05:27.818 INFO:tasks.rados.rados.0.smithi163.stdout:25: write initial oid 25 2023-10-09T09:05:27.818 INFO:tasks.rados.rados.0.smithi163.stdout:25: seq_num 24 ranges {403368=555035,1454509=727675} 2023-10-09T09:05:27.823 INFO:tasks.rados.rados.0.smithi163.stdout:25: writing smithi16367185-25 from 403368 to 958403 tid 1 2023-10-09T09:05:27.831 INFO:tasks.rados.rados.0.smithi163.stdout:25: writing smithi16367185-25 from 1454509 to 2182184 tid 2 2023-10-09T09:05:27.831 INFO:tasks.rados.rados.0.smithi163.stdout:26: write initial oid 26 2023-10-09T09:05:27.831 INFO:tasks.rados.rados.0.smithi163.stdout:26: seq_num 25 ranges {420175=630284,1796638=471333} 2023-10-09T09:05:27.837 INFO:tasks.rados.rados.0.smithi163.stdout:26: writing smithi16367185-26 from 420175 to 1050459 tid 1 2023-10-09T09:05:27.844 INFO:tasks.rados.rados.0.smithi163.stdout:26: writing smithi16367185-26 from 1796638 to 2267971 tid 2 2023-10-09T09:05:27.844 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.844 INFO:tasks.rados.rados.0.smithi163.stdout:11: finishing write tid 3 to smithi16367185-11 2023-10-09T09:05:27.845 INFO:tasks.rados.rados.0.smithi163.stdout:11: finishing write tid 4 to smithi16367185-11 2023-10-09T09:05:27.845 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 6 (ObjNum 10 snap 0 seq_num 10) dirty exists 2023-10-09T09:05:27.845 INFO:tasks.rados.rados.0.smithi163.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2023-10-09T09:05:27.845 INFO:tasks.rados.rados.0.smithi163.stdout:12: finishing write tid 1 to smithi16367185-12 2023-10-09T09:05:27.845 INFO:tasks.rados.rados.0.smithi163.stdout:12: finishing write tid 2 to smithi16367185-12 2023-10-09T09:05:27.845 INFO:tasks.rados.rados.0.smithi163.stdout:11: done (15 left) 2023-10-09T09:05:27.845 INFO:tasks.rados.rados.0.smithi163.stdout:27: write initial oid 27 2023-10-09T09:05:27.845 INFO:tasks.rados.rados.0.smithi163.stdout:27: seq_num 26 ranges {436982=705533,1855120=581862} 2023-10-09T09:05:27.851 INFO:tasks.rados.rados.0.smithi163.stdout:27: writing smithi16367185-27 from 436982 to 1142515 tid 1 2023-10-09T09:05:27.859 INFO:tasks.rados.rados.0.smithi163.stdout:27: writing smithi16367185-27 from 1855120 to 2436982 tid 2 2023-10-09T09:05:27.859 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.859 INFO:tasks.rados.rados.0.smithi163.stdout:12: finishing write tid 3 to smithi16367185-12 2023-10-09T09:05:27.859 INFO:tasks.rados.rados.0.smithi163.stdout:12: finishing write tid 4 to smithi16367185-12 2023-10-09T09:05:27.859 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 3 (ObjNum 11 snap 0 seq_num 11) dirty exists 2023-10-09T09:05:27.859 INFO:tasks.rados.rados.0.smithi163.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2023-10-09T09:05:27.859 INFO:tasks.rados.rados.0.smithi163.stdout:13: finishing write tid 1 to smithi16367185-13 2023-10-09T09:05:27.860 INFO:tasks.rados.rados.0.smithi163.stdout:12: done (15 left) 2023-10-09T09:05:27.860 INFO:tasks.rados.rados.0.smithi163.stdout:28: write initial oid 28 2023-10-09T09:05:27.860 INFO:tasks.rados.rados.0.smithi163.stdout:28: seq_num 27 ranges {453789=780782,1913602=475002} 2023-10-09T09:05:27.866 INFO:tasks.rados.rados.0.smithi163.stdout:28: writing smithi16367185-28 from 453789 to 1234571 tid 1 2023-10-09T09:05:27.873 INFO:tasks.rados.rados.0.smithi163.stdout:28: writing smithi16367185-28 from 1913602 to 2388604 tid 2 2023-10-09T09:05:27.873 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.873 INFO:tasks.rados.rados.0.smithi163.stdout:13: finishing write tid 2 to smithi16367185-13 2023-10-09T09:05:27.873 INFO:tasks.rados.rados.0.smithi163.stdout:13: finishing write tid 3 to smithi16367185-13 2023-10-09T09:05:27.873 INFO:tasks.rados.rados.0.smithi163.stdout:13: finishing write tid 4 to smithi16367185-13 2023-10-09T09:05:27.873 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 3 (ObjNum 12 snap 0 seq_num 12) dirty exists 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:14: finishing write tid 1 to smithi16367185-14 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:14: finishing write tid 2 to smithi16367185-14 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:14: finishing write tid 3 to smithi16367185-14 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:14: finishing write tid 4 to smithi16367185-14 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 3 (ObjNum 13 snap 0 seq_num 13) dirty exists 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:13: done (15 left) 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:14: done (14 left) 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:29: write initial oid 29 2023-10-09T09:05:27.874 INFO:tasks.rados.rados.0.smithi163.stdout:29: seq_num 28 ranges {470596=456031,1572084=618660} 2023-10-09T09:05:27.878 INFO:tasks.rados.rados.0.smithi163.stdout:29: writing smithi16367185-29 from 470596 to 926627 tid 1 2023-10-09T09:05:27.885 INFO:tasks.rados.rados.0.smithi163.stdout:29: writing smithi16367185-29 from 1572084 to 2190744 tid 2 2023-10-09T09:05:27.885 INFO:tasks.rados.rados.0.smithi163.stdout:30: write initial oid 30 2023-10-09T09:05:27.885 INFO:tasks.rados.rados.0.smithi163.stdout:30: seq_num 29 ranges {487403=531280,1514213=478671} 2023-10-09T09:05:27.891 INFO:tasks.rados.rados.0.smithi163.stdout:30: writing smithi16367185-30 from 487403 to 1018683 tid 1 2023-10-09T09:05:27.897 INFO:tasks.rados.rados.0.smithi163.stdout:30: writing smithi16367185-30 from 1514213 to 1992884 tid 2 2023-10-09T09:05:27.898 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.898 INFO:tasks.rados.rados.0.smithi163.stdout:15: finishing write tid 1 to smithi16367185-15 2023-10-09T09:05:27.898 INFO:tasks.rados.rados.0.smithi163.stdout:15: finishing write tid 2 to smithi16367185-15 2023-10-09T09:05:27.898 INFO:tasks.rados.rados.0.smithi163.stdout:15: finishing write tid 3 to smithi16367185-15 2023-10-09T09:05:27.898 INFO:tasks.rados.rados.0.smithi163.stdout:15: finishing write tid 4 to smithi16367185-15 2023-10-09T09:05:27.898 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 6 (ObjNum 14 snap 0 seq_num 14) dirty exists 2023-10-09T09:05:27.898 INFO:tasks.rados.rados.0.smithi163.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2023-10-09T09:05:27.898 INFO:tasks.rados.rados.0.smithi163.stdout:16: finishing write tid 1 to smithi16367185-16 2023-10-09T09:05:27.898 INFO:tasks.rados.rados.0.smithi163.stdout:16: finishing write tid 2 to smithi16367185-16 2023-10-09T09:05:27.898 INFO:tasks.rados.rados.0.smithi163.stdout:16: finishing write tid 3 to smithi16367185-16 2023-10-09T09:05:27.899 INFO:tasks.rados.rados.0.smithi163.stdout:15: done (15 left) 2023-10-09T09:05:27.899 INFO:tasks.rados.rados.0.smithi163.stdout:31: write initial oid 31 2023-10-09T09:05:27.899 INFO:tasks.rados.rados.0.smithi163.stdout:31: seq_num 30 ranges {504210=606529,1572695=622329} 2023-10-09T09:05:27.906 INFO:tasks.rados.rados.0.smithi163.stdout:31: writing smithi16367185-31 from 504210 to 1110739 tid 1 2023-10-09T09:05:27.914 INFO:tasks.rados.rados.0.smithi163.stdout:31: writing smithi16367185-31 from 1572695 to 2195024 tid 2 2023-10-09T09:05:27.914 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.914 INFO:tasks.rados.rados.0.smithi163.stdout:16: finishing write tid 4 to smithi16367185-16 2023-10-09T09:05:27.914 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 12 (ObjNum 15 snap 0 seq_num 15) dirty exists 2023-10-09T09:05:27.914 INFO:tasks.rados.rados.0.smithi163.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2023-10-09T09:05:27.914 INFO:tasks.rados.rados.0.smithi163.stdout:16: done (15 left) 2023-10-09T09:05:27.914 INFO:tasks.rados.rados.0.smithi163.stdout:32: write initial oid 32 2023-10-09T09:05:27.914 INFO:tasks.rados.rados.0.smithi163.stdout:32: seq_num 31 ranges {521017=798131,1747530=482340} 2023-10-09T09:05:27.921 INFO:tasks.rados.rados.0.smithi163.stdout:32: writing smithi16367185-32 from 521017 to 1319148 tid 1 2023-10-09T09:05:27.927 INFO:tasks.rados.rados.0.smithi163.stdout:32: writing smithi16367185-32 from 1747530 to 2229870 tid 2 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout:17: finishing write tid 1 to smithi16367185-17 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout:17: finishing write tid 2 to smithi16367185-17 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout:17: finishing write tid 3 to smithi16367185-17 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout:17: finishing write tid 4 to smithi16367185-17 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 9 (ObjNum 16 snap 0 seq_num 16) dirty exists 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout:18: finishing write tid 1 to smithi16367185-18 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout:17: done (15 left) 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout:33: write initial oid 33 2023-10-09T09:05:27.928 INFO:tasks.rados.rados.0.smithi163.stdout:33: seq_num 32 ranges {537824=473380,1806012=625998} 2023-10-09T09:05:27.935 INFO:tasks.rados.rados.0.smithi163.stdout:33: writing smithi16367185-33 from 537824 to 1011204 tid 1 2023-10-09T09:05:27.941 INFO:tasks.rados.rados.0.smithi163.stdout:33: writing smithi16367185-33 from 1806012 to 2432010 tid 2 2023-10-09T09:05:27.941 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.941 INFO:tasks.rados.rados.0.smithi163.stdout:18: finishing write tid 2 to smithi16367185-18 2023-10-09T09:05:27.942 INFO:tasks.rados.rados.0.smithi163.stdout:18: finishing write tid 3 to smithi16367185-18 2023-10-09T09:05:27.942 INFO:tasks.rados.rados.0.smithi163.stdout:18: finishing write tid 4 to smithi16367185-18 2023-10-09T09:05:27.942 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 6 (ObjNum 17 snap 0 seq_num 17) dirty exists 2023-10-09T09:05:27.942 INFO:tasks.rados.rados.0.smithi163.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2023-10-09T09:05:27.942 INFO:tasks.rados.rados.0.smithi163.stdout:19: finishing write tid 1 to smithi16367185-19 2023-10-09T09:05:27.942 INFO:tasks.rados.rados.0.smithi163.stdout:18: done (15 left) 2023-10-09T09:05:27.942 INFO:tasks.rados.rados.0.smithi163.stdout:34: write initial oid 34 2023-10-09T09:05:27.942 INFO:tasks.rados.rados.0.smithi163.stdout:34: seq_num 33 ranges {554631=548629,1748141=486009} 2023-10-09T09:05:27.948 INFO:tasks.rados.rados.0.smithi163.stdout:34: writing smithi16367185-34 from 554631 to 1103260 tid 1 2023-10-09T09:05:27.955 INFO:tasks.rados.rados.0.smithi163.stdout:34: writing smithi16367185-34 from 1748141 to 2234150 tid 2 2023-10-09T09:05:27.955 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.955 INFO:tasks.rados.rados.0.smithi163.stdout:19: finishing write tid 2 to smithi16367185-19 2023-10-09T09:05:27.955 INFO:tasks.rados.rados.0.smithi163.stdout:19: finishing write tid 3 to smithi16367185-19 2023-10-09T09:05:27.955 INFO:tasks.rados.rados.0.smithi163.stdout:19: finishing write tid 4 to smithi16367185-19 2023-10-09T09:05:27.956 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 3 (ObjNum 18 snap 0 seq_num 18) dirty exists 2023-10-09T09:05:27.956 INFO:tasks.rados.rados.0.smithi163.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2023-10-09T09:05:27.956 INFO:tasks.rados.rados.0.smithi163.stdout:20: finishing write tid 1 to smithi16367185-20 2023-10-09T09:05:27.956 INFO:tasks.rados.rados.0.smithi163.stdout:20: finishing write tid 2 to smithi16367185-20 2023-10-09T09:05:27.956 INFO:tasks.rados.rados.0.smithi163.stdout:20: finishing write tid 3 to smithi16367185-20 2023-10-09T09:05:27.956 INFO:tasks.rados.rados.0.smithi163.stdout:19: done (15 left) 2023-10-09T09:05:27.956 INFO:tasks.rados.rados.0.smithi163.stdout:35: write initial oid 35 2023-10-09T09:05:27.956 INFO:tasks.rados.rados.0.smithi163.stdout:35: seq_num 34 ranges {571438=623878,1806623=629667} 2023-10-09T09:05:27.961 INFO:tasks.rados.rados.0.smithi163.stdout:35: writing smithi16367185-35 from 571438 to 1195316 tid 1 2023-10-09T09:05:27.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:27 smithi163 bash[33912]: audit 2023-10-09T09:05:26.656770+0000 mon.a (mon.0) 766 : audit [INF] from='client.? 172.21.15.163:0/486094513' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2023-10-09T09:05:27.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:27 smithi163 bash[27500]: audit 2023-10-09T09:05:26.656770+0000 mon.a (mon.0) 766 : audit [INF] from='client.? 172.21.15.163:0/486094513' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2023-10-09T09:05:27.969 INFO:tasks.rados.rados.0.smithi163.stdout:35: writing smithi16367185-35 from 1806623 to 2436290 tid 2 2023-10-09T09:05:27.969 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.969 INFO:tasks.rados.rados.0.smithi163.stdout:20: finishing write tid 4 to smithi16367185-20 2023-10-09T09:05:27.970 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 6 (ObjNum 19 snap 0 seq_num 19) dirty exists 2023-10-09T09:05:27.970 INFO:tasks.rados.rados.0.smithi163.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2023-10-09T09:05:27.970 INFO:tasks.rados.rados.0.smithi163.stdout:21: finishing write tid 1 to smithi16367185-21 2023-10-09T09:05:27.970 INFO:tasks.rados.rados.0.smithi163.stdout:20: done (15 left) 2023-10-09T09:05:27.970 INFO:tasks.rados.rados.0.smithi163.stdout:36: write initial oid 36 2023-10-09T09:05:27.970 INFO:tasks.rados.rados.0.smithi163.stdout:36: seq_num 35 ranges {588245=699127,1865105=489678} 2023-10-09T09:05:27.976 INFO:tasks.rados.rados.0.smithi163.stdout:36: writing smithi16367185-36 from 588245 to 1287372 tid 1 2023-10-09T09:05:27.984 INFO:tasks.rados.rados.0.smithi163.stdout:36: writing smithi16367185-36 from 1865105 to 2354783 tid 2 2023-10-09T09:05:27.984 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:27.984 INFO:tasks.rados.rados.0.smithi163.stdout:21: finishing write tid 2 to smithi16367185-21 2023-10-09T09:05:27.984 INFO:tasks.rados.rados.0.smithi163.stdout:21: finishing write tid 3 to smithi16367185-21 2023-10-09T09:05:27.984 INFO:tasks.rados.rados.0.smithi163.stdout:21: finishing write tid 4 to smithi16367185-21 2023-10-09T09:05:27.984 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 6 (ObjNum 20 snap 0 seq_num 20) dirty exists 2023-10-09T09:05:27.984 INFO:tasks.rados.rados.0.smithi163.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2023-10-09T09:05:27.984 INFO:tasks.rados.rados.0.smithi163.stdout:22: finishing write tid 1 to smithi16367185-22 2023-10-09T09:05:27.984 INFO:tasks.rados.rados.0.smithi163.stdout:22: finishing write tid 2 to smithi16367185-22 2023-10-09T09:05:27.984 INFO:tasks.rados.rados.0.smithi163.stdout:22: finishing write tid 3 to smithi16367185-22 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:22: finishing write tid 4 to smithi16367185-22 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 3 (ObjNum 21 snap 0 seq_num 21) dirty exists 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:23: finishing write tid 1 to smithi16367185-23 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:23: finishing write tid 2 to smithi16367185-23 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:23: finishing write tid 3 to smithi16367185-23 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:23: finishing write tid 4 to smithi16367185-23 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 3 (ObjNum 22 snap 0 seq_num 22) dirty exists 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:24: finishing write tid 1 to smithi16367185-24 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:24: finishing write tid 2 to smithi16367185-24 2023-10-09T09:05:27.985 INFO:tasks.rados.rados.0.smithi163.stdout:21: done (15 left) 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:22: done (14 left) 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:23: done (13 left) 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:37: write initial oid 37 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:24: finishing write tid 3 to smithi16367185-24 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:24: finishing write tid 4 to smithi16367185-24 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:25: finishing write tid 1 to smithi16367185-25 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:25: finishing write tid 2 to smithi16367185-25 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:25: finishing write tid 3 to smithi16367185-25 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:25: finishing write tid 4 to smithi16367185-25 2023-10-09T09:05:27.986 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 6 (ObjNum 24 snap 0 seq_num 24) dirty exists 2023-10-09T09:05:27.987 INFO:tasks.rados.rados.0.smithi163.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2023-10-09T09:05:27.987 INFO:tasks.rados.rados.0.smithi163.stdout:26: finishing write tid 1 to smithi16367185-26 2023-10-09T09:05:27.987 INFO:tasks.rados.rados.0.smithi163.stdout:37: seq_num 36 ranges {605052=774376,1923587=633336} 2023-10-09T09:05:27.995 INFO:tasks.rados.rados.0.smithi163.stdout:37: writing smithi16367185-37 from 605052 to 1379428 tid 1 2023-10-09T09:05:28.002 INFO:tasks.rados.rados.0.smithi163.stdout:37: writing smithi16367185-37 from 1923587 to 2556923 tid 2 2023-10-09T09:05:28.002 INFO:tasks.rados.rados.0.smithi163.stdout:24: done (13 left) 2023-10-09T09:05:28.002 INFO:tasks.rados.rados.0.smithi163.stdout:25: done (12 left) 2023-10-09T09:05:28.002 INFO:tasks.rados.rados.0.smithi163.stdout:38: write initial oid 38 2023-10-09T09:05:28.002 INFO:tasks.rados.rados.0.smithi163.stdout:38: seq_num 37 ranges {621859=449625,1865716=756143} 2023-10-09T09:05:28.008 INFO:tasks.rados.rados.0.smithi163.stdout:38: writing smithi16367185-38 from 621859 to 1071484 tid 1 2023-10-09T09:05:28.017 INFO:tasks.rados.rados.0.smithi163.stdout:38: writing smithi16367185-38 from 1865716 to 2621859 tid 2 2023-10-09T09:05:28.017 INFO:tasks.rados.rados.0.smithi163.stdout:39: write initial oid 39 2023-10-09T09:05:28.017 INFO:tasks.rados.rados.0.smithi163.stdout:39: seq_num 38 ranges {638666=524874,1924198=637005} 2023-10-09T09:05:28.024 INFO:tasks.rados.rados.0.smithi163.stdout:39: writing smithi16367185-39 from 638666 to 1163540 tid 1 2023-10-09T09:05:28.032 INFO:tasks.rados.rados.0.smithi163.stdout:39: writing smithi16367185-39 from 1924198 to 2561203 tid 2 2023-10-09T09:05:28.032 INFO:tasks.rados.rados.0.smithi163.stdout:40: write initial oid 40 2023-10-09T09:05:28.032 INFO:tasks.rados.rados.0.smithi163.stdout:40: seq_num 39 ranges {655473=716476,2099033=556440} 2023-10-09T09:05:28.040 INFO:tasks.rados.rados.0.smithi163.stdout:40: writing smithi16367185-40 from 655473 to 1371949 tid 1 2023-10-09T09:05:28.047 INFO:tasks.rados.rados.0.smithi163.stdout:40: writing smithi16367185-40 from 2099033 to 2655473 tid 2 2023-10-09T09:05:28.047 INFO:tasks.rados.rados.0.smithi163.stdout:41: write initial oid 41 2023-10-09T09:05:28.047 INFO:tasks.rados.rados.0.smithi163.stdout:41: seq_num 40 ranges {672280=791725,2157515=514765} 2023-10-09T09:05:28.055 INFO:tasks.rados.rados.0.smithi163.stdout:41: writing smithi16367185-41 from 672280 to 1464005 tid 1 2023-10-09T09:05:28.064 INFO:tasks.rados.rados.0.smithi163.stdout:41: writing smithi16367185-41 from 2157515 to 2672280 tid 2 2023-10-09T09:05:28.064 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:28.064 INFO:tasks.rados.rados.0.smithi163.stdout:26: finishing write tid 2 to smithi16367185-26 2023-10-09T09:05:28.064 INFO:tasks.rados.rados.0.smithi163.stdout:26: finishing write tid 3 to smithi16367185-26 2023-10-09T09:05:28.064 INFO:tasks.rados.rados.0.smithi163.stdout:26: finishing write tid 4 to smithi16367185-26 2023-10-09T09:05:28.064 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 6 (ObjNum 25 snap 0 seq_num 25) dirty exists 2023-10-09T09:05:28.065 INFO:tasks.rados.rados.0.smithi163.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:05:28.065 INFO:tasks.rados.rados.0.smithi163.stdout:27: finishing write tid 1 to smithi16367185-27 2023-10-09T09:05:28.065 INFO:tasks.rados.rados.0.smithi163.stdout:27: finishing write tid 2 to smithi16367185-27 2023-10-09T09:05:28.065 INFO:tasks.rados.rados.0.smithi163.stdout:27: finishing write tid 3 to smithi16367185-27 2023-10-09T09:05:28.065 INFO:tasks.rados.rados.0.smithi163.stdout:27: finishing write tid 4 to smithi16367185-27 2023-10-09T09:05:28.065 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 9 (ObjNum 26 snap 0 seq_num 26) dirty exists 2023-10-09T09:05:28.065 INFO:tasks.rados.rados.0.smithi163.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2023-10-09T09:05:28.065 INFO:tasks.rados.rados.0.smithi163.stdout:28: finishing write tid 1 to smithi16367185-28 2023-10-09T09:05:28.065 INFO:tasks.rados.rados.0.smithi163.stdout:28: finishing write tid 2 to smithi16367185-28 2023-10-09T09:05:28.066 INFO:tasks.rados.rados.0.smithi163.stdout:28: finishing write tid 3 to smithi16367185-28 2023-10-09T09:05:28.066 INFO:tasks.rados.rados.0.smithi163.stdout:28: finishing write tid 4 to smithi16367185-28 2023-10-09T09:05:28.066 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 15 (ObjNum 27 snap 0 seq_num 27) dirty exists 2023-10-09T09:05:28.066 INFO:tasks.rados.rados.0.smithi163.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2023-10-09T09:05:28.066 INFO:tasks.rados.rados.0.smithi163.stdout:29: finishing write tid 1 to smithi16367185-29 2023-10-09T09:05:28.066 INFO:tasks.rados.rados.0.smithi163.stdout:29: finishing write tid 2 to smithi16367185-29 2023-10-09T09:05:28.066 INFO:tasks.rados.rados.0.smithi163.stdout:29: finishing write tid 3 to smithi16367185-29 2023-10-09T09:05:28.066 INFO:tasks.rados.rados.0.smithi163.stdout:29: finishing write tid 4 to smithi16367185-29 2023-10-09T09:05:28.067 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 9 (ObjNum 28 snap 0 seq_num 28) dirty exists 2023-10-09T09:05:28.067 INFO:tasks.rados.rados.0.smithi163.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2023-10-09T09:05:28.067 INFO:tasks.rados.rados.0.smithi163.stdout:30: finishing write tid 1 to smithi16367185-30 2023-10-09T09:05:28.067 INFO:tasks.rados.rados.0.smithi163.stdout:26: done (15 left) 2023-10-09T09:05:28.067 INFO:tasks.rados.rados.0.smithi163.stdout:27: done (14 left) 2023-10-09T09:05:28.067 INFO:tasks.rados.rados.0.smithi163.stdout:28: done (13 left) 2023-10-09T09:05:28.067 INFO:tasks.rados.rados.0.smithi163.stdout:29: done (12 left) 2023-10-09T09:05:28.067 INFO:tasks.rados.rados.0.smithi163.stdout:42: write initial oid 42 2023-10-09T09:05:28.068 INFO:tasks.rados.rados.0.smithi163.stdout:42: seq_num 41 ranges {689087=466974,1699644=784332} 2023-10-09T09:05:28.073 INFO:tasks.rados.rados.0.smithi163.stdout:42: writing smithi16367185-42 from 689087 to 1156061 tid 1 2023-10-09T09:05:28.082 INFO:tasks.rados.rados.0.smithi163.stdout:42: writing smithi16367185-42 from 1699644 to 2483976 tid 2 2023-10-09T09:05:28.082 INFO:tasks.rados.rados.0.smithi163.stdout:43: write initial oid 43 2023-10-09T09:05:28.082 INFO:tasks.rados.rados.0.smithi163.stdout:43: seq_num 42 ranges {705894=542223,1758126=644343} 2023-10-09T09:05:28.089 INFO:tasks.rados.rados.0.smithi163.stdout:43: writing smithi16367185-43 from 705894 to 1248117 tid 1 2023-10-09T09:05:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:43: writing smithi16367185-43 from 1758126 to 2402469 tid 2 2023-10-09T09:05:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:44: write initial oid 44 2023-10-09T09:05:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:44: seq_num 43 ranges {722701=617472,1816608=788001} 2023-10-09T09:05:28.104 INFO:tasks.rados.rados.0.smithi163.stdout:44: writing smithi16367185-44 from 722701 to 1340173 tid 1 2023-10-09T09:05:28.111 INFO:tasks.rados.rados.0.smithi163.stdout:44: writing smithi16367185-44 from 1816608 to 2604609 tid 2 2023-10-09T09:05:28.111 INFO:tasks.rados.rados.0.smithi163.stdout:45: write initial oid 45 2023-10-09T09:05:28.111 INFO:tasks.rados.rados.0.smithi163.stdout:45: seq_num 44 ranges {739508=692721,1875090=648012} 2023-10-09T09:05:28.119 INFO:tasks.rados.rados.0.smithi163.stdout:45: writing smithi16367185-45 from 739508 to 1432229 tid 1 2023-10-09T09:05:28.126 INFO:tasks.rados.rados.0.smithi163.stdout:45: writing smithi16367185-45 from 1875090 to 2523102 tid 2 2023-10-09T09:05:28.126 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:28.126 INFO:tasks.rados.rados.0.smithi163.stdout:30: finishing write tid 2 to smithi16367185-30 2023-10-09T09:05:28.126 INFO:tasks.rados.rados.0.smithi163.stdout:30: finishing write tid 3 to smithi16367185-30 2023-10-09T09:05:28.126 INFO:tasks.rados.rados.0.smithi163.stdout:30: finishing write tid 4 to smithi16367185-30 2023-10-09T09:05:28.127 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 9 (ObjNum 29 snap 0 seq_num 29) dirty exists 2023-10-09T09:05:28.127 INFO:tasks.rados.rados.0.smithi163.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2023-10-09T09:05:28.127 INFO:tasks.rados.rados.0.smithi163.stdout:31: finishing write tid 1 to smithi16367185-31 2023-10-09T09:05:28.127 INFO:tasks.rados.rados.0.smithi163.stdout:31: finishing write tid 2 to smithi16367185-31 2023-10-09T09:05:28.127 INFO:tasks.rados.rados.0.smithi163.stdout:31: finishing write tid 3 to smithi16367185-31 2023-10-09T09:05:28.127 INFO:tasks.rados.rados.0.smithi163.stdout:31: finishing write tid 4 to smithi16367185-31 2023-10-09T09:05:28.127 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 6 (ObjNum 30 snap 0 seq_num 30) dirty exists 2023-10-09T09:05:28.127 INFO:tasks.rados.rados.0.smithi163.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2023-10-09T09:05:28.127 INFO:tasks.rados.rados.0.smithi163.stdout:30: done (15 left) 2023-10-09T09:05:28.128 INFO:tasks.rados.rados.0.smithi163.stdout:31: done (14 left) 2023-10-09T09:05:28.128 INFO:tasks.rados.rados.0.smithi163.stdout:46: write initial oid 46 2023-10-09T09:05:28.128 INFO:tasks.rados.rados.0.smithi163.stdout:46: seq_num 45 ranges {756315=767970,1933572=791670} 2023-10-09T09:05:28.135 INFO:tasks.rados.rados.0.smithi163.stdout:46: writing smithi16367185-46 from 756315 to 1524285 tid 1 2023-10-09T09:05:28.143 INFO:tasks.rados.rados.0.smithi163.stdout:46: writing smithi16367185-46 from 1933572 to 2725242 tid 2 2023-10-09T09:05:28.143 INFO:tasks.rados.rados.0.smithi163.stdout:47: write initial oid 47 2023-10-09T09:05:28.143 INFO:tasks.rados.rados.0.smithi163.stdout:47: seq_num 46 ranges {773122=559572,1992054=651681} 2023-10-09T09:05:28.150 INFO:tasks.rados.rados.0.smithi163.stdout:47: writing smithi16367185-47 from 773122 to 1332694 tid 1 2023-10-09T09:05:28.158 INFO:tasks.rados.rados.0.smithi163.stdout:47: writing smithi16367185-47 from 1992054 to 2643735 tid 2 2023-10-09T09:05:28.158 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:28.159 INFO:tasks.rados.rados.0.smithi163.stdout:32: finishing write tid 1 to smithi16367185-32 2023-10-09T09:05:28.159 INFO:tasks.rados.rados.0.smithi163.stdout:32: finishing write tid 2 to smithi16367185-32 2023-10-09T09:05:28.159 INFO:tasks.rados.rados.0.smithi163.stdout:32: finishing write tid 3 to smithi16367185-32 2023-10-09T09:05:28.159 INFO:tasks.rados.rados.0.smithi163.stdout:32: finishing write tid 4 to smithi16367185-32 2023-10-09T09:05:28.159 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 6 (ObjNum 31 snap 0 seq_num 31) dirty exists 2023-10-09T09:05:28.159 INFO:tasks.rados.rados.0.smithi163.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2023-10-09T09:05:28.159 INFO:tasks.rados.rados.0.smithi163.stdout:33: finishing write tid 1 to smithi16367185-33 2023-10-09T09:05:28.159 INFO:tasks.rados.rados.0.smithi163.stdout:33: finishing write tid 2 to smithi16367185-33 2023-10-09T09:05:28.159 INFO:tasks.rados.rados.0.smithi163.stdout:32: done (15 left) 2023-10-09T09:05:28.160 INFO:tasks.rados.rados.0.smithi163.stdout:48: write initial oid 48 2023-10-09T09:05:28.160 INFO:tasks.rados.rados.0.smithi163.stdout:48: seq_num 47 ranges {789929=634821,2050536=739393} 2023-10-09T09:05:28.166 INFO:tasks.rados.rados.0.smithi163.stdout:48: writing smithi16367185-48 from 789929 to 1424750 tid 1 2023-10-09T09:05:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:48: writing smithi16367185-48 from 2050536 to 2789929 tid 2 2023-10-09T09:05:28.175 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:33: finishing write tid 3 to smithi16367185-33 2023-10-09T09:05:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:33: finishing write tid 4 to smithi16367185-33 2023-10-09T09:05:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 12 (ObjNum 32 snap 0 seq_num 32) dirty exists 2023-10-09T09:05:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2023-10-09T09:05:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:34: finishing write tid 1 to smithi16367185-34 2023-10-09T09:05:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:34: finishing write tid 2 to smithi16367185-34 2023-10-09T09:05:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:34: finishing write tid 3 to smithi16367185-34 2023-10-09T09:05:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:34: finishing write tid 4 to smithi16367185-34 2023-10-09T09:05:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 6 (ObjNum 33 snap 0 seq_num 33) dirty exists 2023-10-09T09:05:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2023-10-09T09:05:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:35: finishing write tid 1 to smithi16367185-35 2023-10-09T09:05:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:33: done (15 left) 2023-10-09T09:05:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:34: done (14 left) 2023-10-09T09:05:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:49: write initial oid 49 2023-10-09T09:05:28.177 INFO:tasks.rados.rados.0.smithi163.stdout:49: seq_num 48 ranges {406736=710070,1709018=655350} 2023-10-09T09:05:28.181 INFO:tasks.rados.rados.0.smithi163.stdout:49: writing smithi16367185-49 from 406736 to 1116806 tid 1 2023-10-09T09:05:28.189 INFO:tasks.rados.rados.0.smithi163.stdout:49: writing smithi16367185-49 from 1709018 to 2364368 tid 2 2023-10-09T09:05:28.189 INFO:tasks.rados.rados.0.smithi163.stdout:50: write initial oid 50 2023-10-09T09:05:28.189 INFO:tasks.rados.rados.0.smithi163.stdout:50: seq_num 49 ranges {423543=785319,1767500=799008} 2023-10-09T09:05:28.196 INFO:tasks.rados.rados.0.smithi163.stdout:50: writing smithi16367185-50 from 423543 to 1208862 tid 1 2023-10-09T09:05:28.209 INFO:tasks.rados.rados.0.smithi163.stdout:50: writing smithi16367185-50 from 1767500 to 2566508 tid 2 2023-10-09T09:05:28.209 INFO:tasks.rados.rados.0.smithi163.stdout: waiting on 16 2023-10-09T09:05:28.209 INFO:tasks.rados.rados.0.smithi163.stdout:35: finishing write tid 2 to smithi16367185-35 2023-10-09T09:05:28.209 INFO:tasks.rados.rados.0.smithi163.stdout:35: finishing write tid 3 to smithi16367185-35 2023-10-09T09:05:28.210 INFO:tasks.rados.rados.0.smithi163.stdout:35: finishing write tid 4 to smithi16367185-35 2023-10-09T09:05:28.210 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 9 (ObjNum 34 snap 0 seq_num 34) dirty exists 2023-10-09T09:05:28.210 INFO:tasks.rados.rados.0.smithi163.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2023-10-09T09:05:28.210 INFO:tasks.rados.rados.0.smithi163.stdout:36: finishing write tid 1 to smithi16367185-36 2023-10-09T09:05:28.210 INFO:tasks.rados.rados.0.smithi163.stdout:36: finishing write tid 2 to smithi16367185-36 2023-10-09T09:05:28.210 INFO:tasks.rados.rados.0.smithi163.stdout:35: done (15 left) 2023-10-09T09:05:28.211 INFO:tasks.rados.rados.0.smithi163.stdout:51: write oid 2 current snap is 0 2023-10-09T09:05:28.211 INFO:tasks.rados.rados.0.smithi163.stdout:36: finishing write tid 3 to smithi16367185-36 2023-10-09T09:05:28.211 INFO:tasks.rados.rados.0.smithi163.stdout:36: finishing write tid 4 to smithi16367185-36 2023-10-09T09:05:28.211 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 9 (ObjNum 35 snap 0 seq_num 35) dirty exists 2023-10-09T09:05:28.211 INFO:tasks.rados.rados.0.smithi163.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2023-10-09T09:05:28.211 INFO:tasks.rados.rados.0.smithi163.stdout:37: finishing write tid 1 to smithi16367185-37 2023-10-09T09:05:28.211 INFO:tasks.rados.rados.0.smithi163.stdout:37: finishing write tid 2 to smithi16367185-37 2023-10-09T09:05:28.212 INFO:tasks.rados.rados.0.smithi163.stdout:37: finishing write tid 3 to smithi16367185-37 2023-10-09T09:05:28.212 INFO:tasks.rados.rados.0.smithi163.stdout:37: finishing write tid 4 to smithi16367185-37 2023-10-09T09:05:28.212 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 12 (ObjNum 36 snap 0 seq_num 36) dirty exists 2023-10-09T09:05:28.212 INFO:tasks.rados.rados.0.smithi163.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2023-10-09T09:05:28.212 INFO:tasks.rados.rados.0.smithi163.stdout:38: finishing write tid 1 to smithi16367185-38 2023-10-09T09:05:28.212 INFO:tasks.rados.rados.0.smithi163.stdout:38: finishing write tid 2 to smithi16367185-38 2023-10-09T09:05:28.212 INFO:tasks.rados.rados.0.smithi163.stdout:38: finishing write tid 3 to smithi16367185-38 2023-10-09T09:05:28.213 INFO:tasks.rados.rados.0.smithi163.stdout:38: finishing write tid 4 to smithi16367185-38 2023-10-09T09:05:28.213 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 6 (ObjNum 37 snap 0 seq_num 37) dirty exists 2023-10-09T09:05:28.213 INFO:tasks.rados.rados.0.smithi163.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2023-10-09T09:05:28.213 INFO:tasks.rados.rados.0.smithi163.stdout:39: finishing write tid 1 to smithi16367185-39 2023-10-09T09:05:28.213 INFO:tasks.rados.rados.0.smithi163.stdout:39: finishing write tid 2 to smithi16367185-39 2023-10-09T09:05:28.213 INFO:tasks.rados.rados.0.smithi163.stdout:39: finishing write tid 3 to smithi16367185-39 2023-10-09T09:05:28.214 INFO:tasks.rados.rados.0.smithi163.stdout:39: finishing write tid 4 to smithi16367185-39 2023-10-09T09:05:28.214 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 9 (ObjNum 38 snap 0 seq_num 38) dirty exists 2023-10-09T09:05:28.214 INFO:tasks.rados.rados.0.smithi163.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2023-10-09T09:05:28.214 INFO:tasks.rados.rados.0.smithi163.stdout:40: finishing write tid 1 to smithi16367185-40 2023-10-09T09:05:28.214 INFO:tasks.rados.rados.0.smithi163.stdout:40: finishing write tid 2 to smithi16367185-40 2023-10-09T09:05:28.214 INFO:tasks.rados.rados.0.smithi163.stdout:40: finishing write tid 3 to smithi16367185-40 2023-10-09T09:05:28.214 INFO:tasks.rados.rados.0.smithi163.stdout:40: finishing write tid 4 to smithi16367185-40 2023-10-09T09:05:28.215 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 15 (ObjNum 39 snap 0 seq_num 39) dirty exists 2023-10-09T09:05:28.215 INFO:tasks.rados.rados.0.smithi163.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2023-10-09T09:05:28.215 INFO:tasks.rados.rados.0.smithi163.stdout:51: seq_num 50 ranges {440350=460568,1309629=542666,2323973=516377} 2023-10-09T09:05:28.215 INFO:tasks.rados.rados.0.smithi163.stdout:51: writing smithi16367185-2 from 440350 to 900918 tid 1 2023-10-09T09:05:28.218 INFO:tasks.rados.rados.0.smithi163.stdout:51: writing smithi16367185-2 from 1309629 to 1852295 tid 2 2023-10-09T09:05:28.225 INFO:tasks.rados.rados.0.smithi163.stdout:51: writing smithi16367185-2 from 2323973 to 2840350 tid 3 2023-10-09T09:05:28.225 INFO:tasks.rados.rados.0.smithi163.stdout:36: done (15 left) 2023-10-09T09:05:28.225 INFO:tasks.rados.rados.0.smithi163.stdout:37: done (14 left) 2023-10-09T09:05:28.225 INFO:tasks.rados.rados.0.smithi163.stdout:38: done (13 left) 2023-10-09T09:05:28.225 INFO:tasks.rados.rados.0.smithi163.stdout:39: done (12 left) 2023-10-09T09:05:28.226 INFO:tasks.rados.rados.0.smithi163.stdout:40: done (11 left) 2023-10-09T09:05:28.226 INFO:tasks.rados.rados.0.smithi163.stdout:52: rollback oid 13 current snap is 0 2023-10-09T09:05:28.226 INFO:tasks.rados.rados.0.smithi163.stdout:52: done (11 left) 2023-10-09T09:05:28.226 INFO:tasks.rados.rados.0.smithi163.stdout:53: write oid 10 current snap is 0 2023-10-09T09:05:28.226 INFO:tasks.rados.rados.0.smithi163.stdout:53: seq_num 51 ranges {457157=535817,1768111=402677} 2023-10-09T09:05:28.231 INFO:tasks.rados.rados.0.smithi163.stdout:53: writing smithi16367185-10 from 457157 to 992974 tid 1 2023-10-09T09:05:28.238 INFO:tasks.rados.rados.0.smithi163.stdout:53: writing smithi16367185-10 from 1768111 to 2170788 tid 2 2023-10-09T09:05:28.238 INFO:tasks.rados.rados.0.smithi163.stdout:54: write oid 39 current snap is 0 2023-10-09T09:05:28.238 INFO:tasks.rados.rados.0.smithi163.stdout:41: finishing write tid 1 to smithi16367185-41 2023-10-09T09:05:28.238 INFO:tasks.rados.rados.0.smithi163.stdout:41: finishing write tid 2 to smithi16367185-41 2023-10-09T09:05:28.238 INFO:tasks.rados.rados.0.smithi163.stdout:41: finishing write tid 3 to smithi16367185-41 2023-10-09T09:05:28.238 INFO:tasks.rados.rados.0.smithi163.stdout:54: seq_num 52 ranges {473964=611066,1826593=546335,2778819=95145} 2023-10-09T09:05:28.244 INFO:tasks.rados.rados.0.smithi163.stdout:54: writing smithi16367185-39 from 473964 to 1085030 tid 1 2023-10-09T09:05:28.251 INFO:tasks.rados.rados.0.smithi163.stdout:54: writing smithi16367185-39 from 1826593 to 2372928 tid 2 2023-10-09T09:05:28.256 INFO:tasks.rados.rados.0.smithi163.stdout:54: writing smithi16367185-39 from 2778819 to 2873964 tid 3 2023-10-09T09:05:28.256 INFO:tasks.rados.rados.0.smithi163.stdout:55: delete oid 40 current snap is 0 2023-10-09T09:05:28.256 INFO:tasks.rados.rados.0.smithi163.stdout:41: finishing write tid 4 to smithi16367185-41 2023-10-09T09:05:28.256 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 3 (ObjNum 40 snap 0 seq_num 40) dirty exists 2023-10-09T09:05:28.256 INFO:tasks.rados.rados.0.smithi163.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2023-10-09T09:05:28.256 INFO:tasks.rados.rados.0.smithi163.stdout:42: finishing write tid 1 to smithi16367185-42 2023-10-09T09:05:28.256 INFO:tasks.rados.rados.0.smithi163.stdout:42: finishing write tid 2 to smithi16367185-42 2023-10-09T09:05:28.256 INFO:tasks.rados.rados.0.smithi163.stdout:42: finishing write tid 3 to smithi16367185-42 2023-10-09T09:05:28.256 INFO:tasks.rados.rados.0.smithi163.stdout:42: finishing write tid 4 to smithi16367185-42 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 9 (ObjNum 41 snap 0 seq_num 41) dirty exists 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:43: finishing write tid 1 to smithi16367185-43 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:43: finishing write tid 2 to smithi16367185-43 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:43: finishing write tid 3 to smithi16367185-43 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:43: finishing write tid 4 to smithi16367185-43 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 9 (ObjNum 42 snap 0 seq_num 42) dirty exists 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:44: finishing write tid 1 to smithi16367185-44 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:44: finishing write tid 2 to smithi16367185-44 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:44: finishing write tid 3 to smithi16367185-44 2023-10-09T09:05:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:44: finishing write tid 4 to smithi16367185-44 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 12 (ObjNum 43 snap 0 seq_num 43) dirty exists 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:45: finishing write tid 1 to smithi16367185-45 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:45: finishing write tid 2 to smithi16367185-45 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:45: finishing write tid 3 to smithi16367185-45 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:45: finishing write tid 4 to smithi16367185-45 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 15 (ObjNum 44 snap 0 seq_num 44) dirty exists 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:46: finishing write tid 1 to smithi16367185-46 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:46: finishing write tid 2 to smithi16367185-46 2023-10-09T09:05:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:46: finishing write tid 3 to smithi16367185-46 2023-10-09T09:05:28.259 INFO:tasks.rados.rados.0.smithi163.stdout:46: finishing write tid 4 to smithi16367185-46 2023-10-09T09:05:28.259 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 12 (ObjNum 45 snap 0 seq_num 45) dirty exists 2023-10-09T09:05:28.259 INFO:tasks.rados.rados.0.smithi163.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2023-10-09T09:05:28.259 INFO:tasks.rados.rados.0.smithi163.stdout:47: finishing write tid 1 to smithi16367185-47 2023-10-09T09:05:28.259 INFO:tasks.rados.rados.0.smithi163.stdout:47: finishing write tid 2 to smithi16367185-47 2023-10-09T09:05:28.259 INFO:tasks.rados.rados.0.smithi163.stdout:47: finishing write tid 3 to smithi16367185-47 2023-10-09T09:05:28.259 INFO:tasks.rados.rados.0.smithi163.stdout:47: finishing write tid 4 to smithi16367185-47 2023-10-09T09:05:28.259 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 18 (ObjNum 46 snap 0 seq_num 46) dirty exists 2023-10-09T09:05:28.260 INFO:tasks.rados.rados.0.smithi163.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2023-10-09T09:05:28.260 INFO:tasks.rados.rados.0.smithi163.stdout:48: finishing write tid 1 to smithi16367185-48 2023-10-09T09:05:28.260 INFO:tasks.rados.rados.0.smithi163.stdout:48: finishing write tid 2 to smithi16367185-48 2023-10-09T09:05:28.260 INFO:tasks.rados.rados.0.smithi163.stdout:48: finishing write tid 3 to smithi16367185-48 2023-10-09T09:05:28.260 INFO:tasks.rados.rados.0.smithi163.stdout:48: finishing write tid 4 to smithi16367185-48 2023-10-09T09:05:28.260 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 21 (ObjNum 47 snap 0 seq_num 47) dirty exists 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:49: finishing write tid 1 to smithi16367185-49 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:49: finishing write tid 2 to smithi16367185-49 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:49: finishing write tid 3 to smithi16367185-49 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:49: finishing write tid 4 to smithi16367185-49 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 3 (ObjNum 48 snap 0 seq_num 48) dirty exists 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:50: finishing write tid 1 to smithi16367185-50 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:50: finishing write tid 2 to smithi16367185-50 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:50: finishing write tid 3 to smithi16367185-50 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:50: finishing write tid 4 to smithi16367185-50 2023-10-09T09:05:28.261 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 12 (ObjNum 49 snap 0 seq_num 49) dirty exists 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:51: finishing write tid 1 to smithi16367185-2 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:51: finishing write tid 2 to smithi16367185-2 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:51: finishing write tid 3 to smithi16367185-2 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:51: finishing write tid 4 to smithi16367185-2 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:51: finishing write tid 5 to smithi16367185-2 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 13 (ObjNum 50 snap 0 seq_num 50) dirty exists 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:51: left oid 2 (ObjNum 50 snap 0 seq_num 50) 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:53: finishing write tid 1 to smithi16367185-10 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:53: finishing write tid 2 to smithi16367185-10 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:53: finishing write tid 3 to smithi16367185-10 2023-10-09T09:05:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:53: finishing write tid 4 to smithi16367185-10 2023-10-09T09:05:28.263 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 12 (ObjNum 51 snap 0 seq_num 51) dirty exists 2023-10-09T09:05:28.263 INFO:tasks.rados.rados.0.smithi163.stdout:53: left oid 10 (ObjNum 51 snap 0 seq_num 51) 2023-10-09T09:05:28.263 INFO:tasks.rados.rados.0.smithi163.stdout:54: finishing write tid 1 to smithi16367185-39 2023-10-09T09:05:28.264 INFO:tasks.rados.rados.0.smithi163.stdout:41: done (13 left) 2023-10-09T09:05:28.264 INFO:tasks.rados.rados.0.smithi163.stdout:42: done (12 left) 2023-10-09T09:05:28.264 INFO:tasks.rados.rados.0.smithi163.stdout:43: done (11 left) 2023-10-09T09:05:28.264 INFO:tasks.rados.rados.0.smithi163.stdout:44: done (10 left) 2023-10-09T09:05:28.264 INFO:tasks.rados.rados.0.smithi163.stdout:45: done (9 left) 2023-10-09T09:05:28.264 INFO:tasks.rados.rados.0.smithi163.stdout:46: done (8 left) 2023-10-09T09:05:28.264 INFO:tasks.rados.rados.0.smithi163.stdout:47: done (7 left) 2023-10-09T09:05:28.264 INFO:tasks.rados.rados.0.smithi163.stdout:48: done (6 left) 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:49: done (5 left) 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:50: done (4 left) 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:51: done (3 left) 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:53: done (2 left) 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:55: done (1 left) 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:56: read oid 15 snap -1 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:56: expect (ObjNum 14 snap 0 seq_num 14) 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:57: delete oid 13 current snap is 0 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:54: finishing write tid 2 to smithi16367185-39 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:57: done (2 left) 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:58: read oid 18 snap -1 2023-10-09T09:05:28.265 INFO:tasks.rados.rados.0.smithi163.stdout:58: expect (ObjNum 17 snap 0 seq_num 17) 2023-10-09T09:05:28.266 INFO:tasks.rados.rados.0.smithi163.stdout:59: read oid 27 snap -1 2023-10-09T09:05:28.266 INFO:tasks.rados.rados.0.smithi163.stdout:59: expect (ObjNum 26 snap 0 seq_num 26) 2023-10-09T09:05:28.266 INFO:tasks.rados.rados.0.smithi163.stdout:60: read oid 32 snap -1 2023-10-09T09:05:28.266 INFO:tasks.rados.rados.0.smithi163.stdout:60: expect (ObjNum 31 snap 0 seq_num 31) 2023-10-09T09:05:28.266 INFO:tasks.rados.rados.0.smithi163.stdout:61: delete oid 17 current snap is 0 2023-10-09T09:05:28.266 INFO:tasks.rados.rados.0.smithi163.stdout:54: finishing write tid 3 to smithi16367185-39 2023-10-09T09:05:28.266 INFO:tasks.rados.rados.0.smithi163.stdout:54: finishing write tid 4 to smithi16367185-39 2023-10-09T09:05:28.266 INFO:tasks.rados.rados.0.smithi163.stdout:54: finishing write tid 5 to smithi16367185-39 2023-10-09T09:05:28.266 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 16 (ObjNum 52 snap 0 seq_num 52) dirty exists 2023-10-09T09:05:28.266 INFO:tasks.rados.rados.0.smithi163.stdout:54: left oid 39 (ObjNum 52 snap 0 seq_num 52) 2023-10-09T09:05:28.267 INFO:tasks.rados.rados.0.smithi163.stdout:56: expect (ObjNum 14 snap 0 seq_num 14) 2023-10-09T09:05:28.357 INFO:tasks.rados.rados.0.smithi163.stdout:60: expect (ObjNum 31 snap 0 seq_num 31) 2023-10-09T09:05:28.449 INFO:tasks.rados.rados.0.smithi163.stdout:59: expect (ObjNum 26 snap 0 seq_num 26) 2023-10-09T09:05:28.535 INFO:tasks.rados.rados.0.smithi163.stdout:58: expect (ObjNum 17 snap 0 seq_num 17) 2023-10-09T09:05:28.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:28 smithi163 bash[27500]: cluster 2023-10-09T09:05:27.247638+0000 mgr.y (mgr.24355) 169 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:28.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:28 smithi163 bash[27500]: audit 2023-10-09T09:05:27.493190+0000 mon.a (mon.0) 767 : audit [INF] from='client.? 172.21.15.163:0/486094513' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2023-10-09T09:05:28.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:28 smithi163 bash[27500]: cluster 2023-10-09T09:05:27.493269+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T09:05:28.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:28 smithi163 bash[33912]: cluster 2023-10-09T09:05:27.247638+0000 mgr.y (mgr.24355) 169 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:28.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:28 smithi163 bash[33912]: audit 2023-10-09T09:05:27.493190+0000 mon.a (mon.0) 767 : audit [INF] from='client.? 172.21.15.163:0/486094513' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2023-10-09T09:05:28.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:28 smithi163 bash[33912]: cluster 2023-10-09T09:05:27.493269+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T09:05:28.613 INFO:tasks.rados.rados.0.smithi163.stdout:54: done (5 left) 2023-10-09T09:05:28.613 INFO:tasks.rados.rados.0.smithi163.stdout:56: done (4 left) 2023-10-09T09:05:28.613 INFO:tasks.rados.rados.0.smithi163.stdout:58: done (3 left) 2023-10-09T09:05:28.613 INFO:tasks.rados.rados.0.smithi163.stdout:59: done (2 left) 2023-10-09T09:05:28.614 INFO:tasks.rados.rados.0.smithi163.stdout:60: done (1 left) 2023-10-09T09:05:28.615 INFO:tasks.rados.rados.0.smithi163.stdout:61: done (0 left) 2023-10-09T09:05:28.615 INFO:tasks.rados.rados.0.smithi163.stdout:62: write oid 29 current snap is 0 2023-10-09T09:05:28.615 INFO:tasks.rados.rados.0.smithi163.stdout:62: seq_num 56 ranges {541192=628415,1660521=553673,2888511=52681} 2023-10-09T09:05:28.626 INFO:tasks.rados.rados.0.smithi163.stdout:62: writing smithi16367185-29 from 541192 to 1169607 tid 1 2023-10-09T09:05:28.633 INFO:tasks.rados.rados.0.smithi163.stdout:62: writing smithi16367185-29 from 1660521 to 2214194 tid 2 2023-10-09T09:05:28.637 INFO:tasks.rados.rados.0.smithi163.stdout:62: writing smithi16367185-29 from 2888511 to 2941192 tid 3 2023-10-09T09:05:28.637 INFO:tasks.rados.rados.0.smithi163.stdout:63: write oid 48 current snap is 0 2023-10-09T09:05:28.637 INFO:tasks.rados.rados.0.smithi163.stdout:62: finishing write tid 1 to smithi16367185-29 2023-10-09T09:05:28.638 INFO:tasks.rados.rados.0.smithi163.stdout:63: seq_num 57 ranges {557999=703664,1719003=413684,2632287=325712} 2023-10-09T09:05:28.644 INFO:tasks.rados.rados.0.smithi163.stdout:63: writing smithi16367185-48 from 557999 to 1261663 tid 1 2023-10-09T09:05:28.650 INFO:tasks.rados.rados.0.smithi163.stdout:63: writing smithi16367185-48 from 1719003 to 2132687 tid 2 2023-10-09T09:05:28.655 INFO:tasks.rados.rados.0.smithi163.stdout:63: writing smithi16367185-48 from 2632287 to 2957999 tid 3 2023-10-09T09:05:28.655 INFO:tasks.rados.rados.0.smithi163.stdout:64: write oid 22 current snap is 0 2023-10-09T09:05:28.655 INFO:tasks.rados.rados.0.smithi163.stdout:64: seq_num 58 ranges {574806=778913,2061132=557342} 2023-10-09T09:05:28.662 INFO:tasks.rados.rados.0.smithi163.stdout:64: writing smithi16367185-22 from 574806 to 1353719 tid 1 2023-10-09T09:05:28.670 INFO:tasks.rados.rados.0.smithi163.stdout:64: writing smithi16367185-22 from 2061132 to 2618474 tid 2 2023-10-09T09:05:28.671 INFO:tasks.rados.rados.0.smithi163.stdout:65: rollback oid 7 current snap is 0 2023-10-09T09:05:28.671 INFO:tasks.rados.rados.0.smithi163.stdout:65: done (3 left) 2023-10-09T09:05:28.671 INFO:tasks.rados.rados.0.smithi163.stdout:66: read oid 24 snap -1 2023-10-09T09:05:28.671 INFO:tasks.rados.rados.0.smithi163.stdout:66: expect (ObjNum 23 snap 0 seq_num 23) 2023-10-09T09:05:28.671 INFO:tasks.rados.rados.0.smithi163.stdout:67: write oid 12 current snap is 0 2023-10-09T09:05:28.671 INFO:tasks.rados.rados.0.smithi163.stdout:62: finishing write tid 2 to smithi16367185-29 2023-10-09T09:05:28.671 INFO:tasks.rados.rados.0.smithi163.stdout:67: seq_num 59 ranges {591613=454162,1719614=417353,2570780=420833} 2023-10-09T09:05:28.676 INFO:tasks.rados.rados.0.smithi163.stdout:67: writing smithi16367185-12 from 591613 to 1045775 tid 1 2023-10-09T09:05:28.683 INFO:tasks.rados.rados.0.smithi163.stdout:67: writing smithi16367185-12 from 1719614 to 2136967 tid 2 2023-10-09T09:05:28.688 INFO:tasks.rados.rados.0.smithi163.stdout:67: writing smithi16367185-12 from 2570780 to 2991613 tid 3 2023-10-09T09:05:28.688 INFO:tasks.rados.rados.0.smithi163.stdout:68: write oid 25 current snap is 0 2023-10-09T09:05:28.689 INFO:tasks.rados.rados.0.smithi163.stdout:62: finishing write tid 3 to smithi16367185-29 2023-10-09T09:05:28.689 INFO:tasks.rados.rados.0.smithi163.stdout:62: finishing write tid 4 to smithi16367185-29 2023-10-09T09:05:28.689 INFO:tasks.rados.rados.0.smithi163.stdout:62: finishing write tid 5 to smithi16367185-29 2023-10-09T09:05:28.689 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 16 (ObjNum 56 snap 0 seq_num 56) dirty exists 2023-10-09T09:05:28.689 INFO:tasks.rados.rados.0.smithi163.stdout:62: left oid 29 (ObjNum 56 snap 0 seq_num 56) 2023-10-09T09:05:28.689 INFO:tasks.rados.rados.0.smithi163.stdout:63: finishing write tid 1 to smithi16367185-48 2023-10-09T09:05:28.689 INFO:tasks.rados.rados.0.smithi163.stdout:63: finishing write tid 2 to smithi16367185-48 2023-10-09T09:05:28.689 INFO:tasks.rados.rados.0.smithi163.stdout:68: seq_num 60 ranges {608420=529411,1778096=561011,2881850=126570} 2023-10-09T09:05:28.699 INFO:tasks.rados.rados.0.smithi163.stdout:68: writing smithi16367185-25 from 608420 to 1137831 tid 1 2023-10-09T09:05:28.707 INFO:tasks.rados.rados.0.smithi163.stdout:68: writing smithi16367185-25 from 1778096 to 2339107 tid 2 2023-10-09T09:05:28.711 INFO:tasks.rados.rados.0.smithi163.stdout:68: writing smithi16367185-25 from 2881850 to 3008420 tid 3 2023-10-09T09:05:28.711 INFO:tasks.rados.rados.0.smithi163.stdout:62: done (5 left) 2023-10-09T09:05:28.711 INFO:tasks.rados.rados.0.smithi163.stdout:69: write oid 28 current snap is 0 2023-10-09T09:05:28.711 INFO:tasks.rados.rados.0.smithi163.stdout:63: finishing write tid 3 to smithi16367185-48 2023-10-09T09:05:28.712 INFO:tasks.rados.rados.0.smithi163.stdout:69: seq_num 61 ranges {625227=721013,1952931=704669} 2023-10-09T09:05:28.719 INFO:tasks.rados.rados.0.smithi163.stdout:69: writing smithi16367185-28 from 625227 to 1346240 tid 1 2023-10-09T09:05:28.727 INFO:tasks.rados.rados.0.smithi163.stdout:69: writing smithi16367185-28 from 1952931 to 2657600 tid 2 2023-10-09T09:05:28.727 INFO:tasks.rados.rados.0.smithi163.stdout:70: write oid 49 current snap is 0 2023-10-09T09:05:28.727 INFO:tasks.rados.rados.0.smithi163.stdout:63: finishing write tid 4 to smithi16367185-48 2023-10-09T09:05:28.728 INFO:tasks.rados.rados.0.smithi163.stdout:63: finishing write tid 5 to smithi16367185-48 2023-10-09T09:05:28.728 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 25 (ObjNum 57 snap 0 seq_num 57) dirty exists 2023-10-09T09:05:28.728 INFO:tasks.rados.rados.0.smithi163.stdout:63: left oid 48 (ObjNum 57 snap 0 seq_num 57) 2023-10-09T09:05:28.728 INFO:tasks.rados.rados.0.smithi163.stdout:64: finishing write tid 1 to smithi16367185-22 2023-10-09T09:05:28.728 INFO:tasks.rados.rados.0.smithi163.stdout:64: finishing write tid 2 to smithi16367185-22 2023-10-09T09:05:28.728 INFO:tasks.rados.rados.0.smithi163.stdout:64: finishing write tid 3 to smithi16367185-22 2023-10-09T09:05:28.728 INFO:tasks.rados.rados.0.smithi163.stdout:70: seq_num 62 ranges {642034=796262,1895060=564680} 2023-10-09T09:05:28.735 INFO:tasks.rados.rados.0.smithi163.stdout:70: writing smithi16367185-49 from 642034 to 1438296 tid 1 2023-10-09T09:05:28.741 INFO:tasks.rados.rados.0.smithi163.stdout:70: writing smithi16367185-49 from 1895060 to 2459740 tid 2 2023-10-09T09:05:28.741 INFO:tasks.rados.rados.0.smithi163.stdout:63: done (6 left) 2023-10-09T09:05:28.742 INFO:tasks.rados.rados.0.smithi163.stdout:71: snap_create 2023-10-09T09:05:28.742 INFO:tasks.rados.rados.0.smithi163.stdout:64: finishing write tid 4 to smithi16367185-22 2023-10-09T09:05:28.742 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 28 (ObjNum 58 snap 0 seq_num 58) dirty exists 2023-10-09T09:05:28.742 INFO:tasks.rados.rados.0.smithi163.stdout:64: left oid 22 (ObjNum 58 snap 0 seq_num 58) 2023-10-09T09:05:28.742 INFO:tasks.rados.rados.0.smithi163.stdout:66: expect (ObjNum 23 snap 0 seq_num 23) 2023-10-09T09:05:28.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:05:28 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:05:28.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:28 smithi181 bash[27712]: cluster 2023-10-09T09:05:27.247638+0000 mgr.y (mgr.24355) 169 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:28.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:28 smithi181 bash[27712]: audit 2023-10-09T09:05:27.493190+0000 mon.a (mon.0) 767 : audit [INF] from='client.? 172.21.15.163:0/486094513' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2023-10-09T09:05:28.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:28 smithi181 bash[27712]: cluster 2023-10-09T09:05:27.493269+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T09:05:28.827 INFO:tasks.rados.rados.0.smithi163.stdout:67: finishing write tid 1 to smithi16367185-12 2023-10-09T09:05:28.827 INFO:tasks.rados.rados.0.smithi163.stdout:67: finishing write tid 2 to smithi16367185-12 2023-10-09T09:05:28.827 INFO:tasks.rados.rados.0.smithi163.stdout:67: finishing write tid 3 to smithi16367185-12 2023-10-09T09:05:28.827 INFO:tasks.rados.rados.0.smithi163.stdout:67: finishing write tid 4 to smithi16367185-12 2023-10-09T09:05:28.827 INFO:tasks.rados.rados.0.smithi163.stdout:67: finishing write tid 5 to smithi16367185-12 2023-10-09T09:05:28.827 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 20 (ObjNum 59 snap 0 seq_num 59) dirty exists 2023-10-09T09:05:28.827 INFO:tasks.rados.rados.0.smithi163.stdout:67: left oid 12 (ObjNum 59 snap 0 seq_num 59) 2023-10-09T09:05:28.827 INFO:tasks.rados.rados.0.smithi163.stdout:68: finishing write tid 1 to smithi16367185-25 2023-10-09T09:05:28.827 INFO:tasks.rados.rados.0.smithi163.stdout:68: finishing write tid 2 to smithi16367185-25 2023-10-09T09:05:28.827 INFO:tasks.rados.rados.0.smithi163.stdout:68: finishing write tid 3 to smithi16367185-25 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:68: finishing write tid 4 to smithi16367185-25 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:68: finishing write tid 5 to smithi16367185-25 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 24 (ObjNum 60 snap 0 seq_num 60) dirty exists 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:68: left oid 25 (ObjNum 60 snap 0 seq_num 60) 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:69: finishing write tid 1 to smithi16367185-28 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:69: finishing write tid 2 to smithi16367185-28 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:69: finishing write tid 3 to smithi16367185-28 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:69: finishing write tid 4 to smithi16367185-28 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 18 (ObjNum 61 snap 0 seq_num 61) dirty exists 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:69: left oid 28 (ObjNum 61 snap 0 seq_num 61) 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:70: finishing write tid 1 to smithi16367185-49 2023-10-09T09:05:28.828 INFO:tasks.rados.rados.0.smithi163.stdout:70: finishing write tid 2 to smithi16367185-49 2023-10-09T09:05:28.829 INFO:tasks.rados.rados.0.smithi163.stdout:70: finishing write tid 3 to smithi16367185-49 2023-10-09T09:05:28.829 INFO:tasks.rados.rados.0.smithi163.stdout:70: finishing write tid 4 to smithi16367185-49 2023-10-09T09:05:28.829 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 6 (ObjNum 62 snap 0 seq_num 62) dirty exists 2023-10-09T09:05:28.829 INFO:tasks.rados.rados.0.smithi163.stdout:70: left oid 49 (ObjNum 62 snap 0 seq_num 62) 2023-10-09T09:05:29.502 INFO:tasks.rados.rados.0.smithi163.stdout:64: done (6 left) 2023-10-09T09:05:29.502 INFO:tasks.rados.rados.0.smithi163.stdout:66: done (5 left) 2023-10-09T09:05:29.502 INFO:tasks.rados.rados.0.smithi163.stdout:67: done (4 left) 2023-10-09T09:05:29.502 INFO:tasks.rados.rados.0.smithi163.stdout:68: done (3 left) 2023-10-09T09:05:29.502 INFO:tasks.rados.rados.0.smithi163.stdout:69: done (2 left) 2023-10-09T09:05:29.502 INFO:tasks.rados.rados.0.smithi163.stdout:70: done (1 left) 2023-10-09T09:05:29.502 INFO:tasks.rados.rados.0.smithi163.stdout:71: done (0 left) 2023-10-09T09:05:29.502 INFO:tasks.rados.rados.0.smithi163.stdout:72: write oid 50 current snap is 1 2023-10-09T09:05:29.502 INFO:tasks.rados.rados.0.smithi163.stdout:72: seq_num 63 ranges {658841=471511,1553542=708338,2964119=94722} 2023-10-09T09:05:29.508 INFO:tasks.rados.rados.0.smithi163.stdout:72: writing smithi16367185-50 from 658841 to 1130352 tid 1 2023-10-09T09:05:29.515 INFO:tasks.rados.rados.0.smithi163.stdout:72: writing smithi16367185-50 from 1553542 to 2261880 tid 2 2023-10-09T09:05:29.520 INFO:tasks.rados.rados.0.smithi163.stdout:72: writing smithi16367185-50 from 2964119 to 3058841 tid 3 2023-10-09T09:05:29.520 INFO:tasks.rados.rados.0.smithi163.stdout:73: rollback oid 38 current snap is 1 2023-10-09T09:05:29.520 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 0 2023-10-09T09:05:29.521 INFO:tasks.rados.rados.0.smithi163.stdout:72: finishing write tid 1 to smithi16367185-50 2023-10-09T09:05:29.521 INFO:tasks.rados.rados.0.smithi163.stdout:72: finishing write tid 2 to smithi16367185-50 2023-10-09T09:05:29.521 INFO:tasks.rados.rados.0.smithi163.stdout:74: write oid 33 current snap is 1 2023-10-09T09:05:29.521 INFO:tasks.rados.rados.0.smithi163.stdout:74: seq_num 64 ranges {675648=546760,2012024=568349} 2023-10-09T09:05:29.529 INFO:tasks.rados.rados.0.smithi163.stdout:74: writing smithi16367185-33 from 675648 to 1222408 tid 1 2023-10-09T09:05:29.538 INFO:tasks.rados.rados.0.smithi163.stdout:74: writing smithi16367185-33 from 2012024 to 2580373 tid 2 2023-10-09T09:05:29.538 INFO:tasks.rados.rados.0.smithi163.stdout:75: rollback oid 37 current snap is 1 2023-10-09T09:05:29.538 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 0 2023-10-09T09:05:29.538 INFO:tasks.rados.rados.0.smithi163.stdout:72: finishing write tid 3 to smithi16367185-50 2023-10-09T09:05:29.538 INFO:tasks.rados.rados.0.smithi163.stdout:72: finishing write tid 4 to smithi16367185-50 2023-10-09T09:05:29.538 INFO:tasks.rados.rados.0.smithi163.stdout:73: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:05:29.538 INFO:tasks.rados.rados.0.smithi163.stdout:72: finishing write tid 5 to smithi16367185-50 2023-10-09T09:05:29.538 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 21 (ObjNum 63 snap 1 seq_num 63) dirty exists 2023-10-09T09:05:29.538 INFO:tasks.rados.rados.0.smithi163.stdout:72: left oid 50 (ObjNum 63 snap 1 seq_num 63) 2023-10-09T09:05:29.539 INFO:tasks.rados.rados.0.smithi163.stdout:73: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:05:29.539 INFO:tasks.rados.rados.0.smithi163.stdout:73: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:05:29.539 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 16 (ObjNum 37 snap 0 seq_num 37) dirty exists 2023-10-09T09:05:29.539 INFO:tasks.rados.rados.0.smithi163.stdout:74: finishing write tid 1 to smithi16367185-33 2023-10-09T09:05:29.539 INFO:tasks.rados.rados.0.smithi163.stdout:72: done (3 left) 2023-10-09T09:05:29.539 INFO:tasks.rados.rados.0.smithi163.stdout:73: done (2 left) 2023-10-09T09:05:29.539 INFO:tasks.rados.rados.0.smithi163.stdout:76: snap_create 2023-10-09T09:05:29.541 INFO:tasks.rados.rados.0.smithi163.stdout:74: finishing write tid 2 to smithi16367185-33 2023-10-09T09:05:29.541 INFO:tasks.rados.rados.0.smithi163.stdout:74: finishing write tid 3 to smithi16367185-33 2023-10-09T09:05:29.541 INFO:tasks.rados.rados.0.smithi163.stdout:74: finishing write tid 4 to smithi16367185-33 2023-10-09T09:05:29.541 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 28 (ObjNum 64 snap 1 seq_num 64) dirty exists 2023-10-09T09:05:29.541 INFO:tasks.rados.rados.0.smithi163.stdout:74: left oid 33 (ObjNum 64 snap 1 seq_num 64) 2023-10-09T09:05:29.545 INFO:tasks.rados.rados.0.smithi163.stdout:75: finishing rollback tid 0 to smithi16367185-37 2023-10-09T09:05:29.545 INFO:tasks.rados.rados.0.smithi163.stdout:75: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:05:29.545 INFO:tasks.rados.rados.0.smithi163.stdout:75: finishing rollback tid 2 to smithi16367185-37 2023-10-09T09:05:29.545 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 21 (ObjNum 36 snap 0 seq_num 36) dirty exists 2023-10-09T09:05:30.185 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:29 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:29] "GET /metrics HTTP/1.1" 200 193795 "" "Prometheus/2.33.4" 2023-10-09T09:05:30.510 INFO:tasks.rados.rados.0.smithi163.stdout:74: done (2 left) 2023-10-09T09:05:30.510 INFO:tasks.rados.rados.0.smithi163.stdout:75: done (1 left) 2023-10-09T09:05:30.510 INFO:tasks.rados.rados.0.smithi163.stdout:76: done (0 left) 2023-10-09T09:05:30.510 INFO:tasks.rados.rados.0.smithi163.stdout:77: read oid 33 snap -1 2023-10-09T09:05:30.510 INFO:tasks.rados.rados.0.smithi163.stdout:77: expect (ObjNum 64 snap 1 seq_num 64) 2023-10-09T09:05:30.510 INFO:tasks.rados.rados.0.smithi163.stdout:78: snap_create 2023-10-09T09:05:30.517 INFO:tasks.rados.rados.0.smithi163.stdout:77: expect (ObjNum 64 snap 1 seq_num 64) 2023-10-09T09:05:30.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:30 smithi181 bash[27712]: cluster 2023-10-09T09:05:29.248577+0000 mgr.y (mgr.24355) 170 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 111 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 6.4 MiB/s wr, 36 op/s 2023-10-09T09:05:30.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:30 smithi181 bash[27712]: cluster 2023-10-09T09:05:29.501767+0000 mon.a (mon.0) 769 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T09:05:30.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:30 smithi163 bash[27500]: cluster 2023-10-09T09:05:29.248577+0000 mgr.y (mgr.24355) 170 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 111 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 6.4 MiB/s wr, 36 op/s 2023-10-09T09:05:30.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:30 smithi163 bash[27500]: cluster 2023-10-09T09:05:29.501767+0000 mon.a (mon.0) 769 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T09:05:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:30 smithi163 bash[33912]: cluster 2023-10-09T09:05:29.248577+0000 mgr.y (mgr.24355) 170 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 111 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 6.4 MiB/s wr, 36 op/s 2023-10-09T09:05:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:30 smithi163 bash[33912]: cluster 2023-10-09T09:05:29.501767+0000 mon.a (mon.0) 769 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T09:05:31.513 INFO:tasks.rados.rados.0.smithi163.stdout:77: done (1 left) 2023-10-09T09:05:31.514 INFO:tasks.rados.rados.0.smithi163.stdout:78: done (0 left) 2023-10-09T09:05:31.514 INFO:tasks.rados.rados.0.smithi163.stdout:79: rollback oid 9 current snap is 3 2023-10-09T09:05:31.514 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 0 2023-10-09T09:05:31.514 INFO:tasks.rados.rados.0.smithi163.stdout:80: snap_create 2023-10-09T09:05:31.519 INFO:tasks.rados.rados.0.smithi163.stdout:79: finishing rollback tid 0 to smithi16367185-9 2023-10-09T09:05:31.519 INFO:tasks.rados.rados.0.smithi163.stdout:79: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:05:31.520 INFO:tasks.rados.rados.0.smithi163.stdout:79: finishing rollback tid 2 to smithi16367185-9 2023-10-09T09:05:31.520 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 10 (ObjNum 8 snap 0 seq_num 8) dirty exists 2023-10-09T09:05:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:31 smithi181 bash[27712]: cluster 2023-10-09T09:05:30.510413+0000 mon.a (mon.0) 770 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T09:05:31.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:31 smithi181 bash[27712]: audit 2023-10-09T09:05:30.659933+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.163:0/3985235641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:31 smithi163 bash[27500]: cluster 2023-10-09T09:05:30.510413+0000 mon.a (mon.0) 770 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T09:05:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:31 smithi163 bash[27500]: audit 2023-10-09T09:05:30.659933+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.163:0/3985235641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:31.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:31 smithi163 bash[33912]: cluster 2023-10-09T09:05:30.510413+0000 mon.a (mon.0) 770 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T09:05:31.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:31 smithi163 bash[33912]: audit 2023-10-09T09:05:30.659933+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.163:0/3985235641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:32.517 INFO:tasks.rados.rados.0.smithi163.stdout:79: done (1 left) 2023-10-09T09:05:32.518 INFO:tasks.rados.rados.0.smithi163.stdout:80: done (0 left) 2023-10-09T09:05:32.518 INFO:tasks.rados.rados.0.smithi163.stdout:81: read oid 26 snap -1 2023-10-09T09:05:32.518 INFO:tasks.rados.rados.0.smithi163.stdout:81: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:05:32.518 INFO:tasks.rados.rados.0.smithi163.stdout:82: read oid 36 snap -1 2023-10-09T09:05:32.518 INFO:tasks.rados.rados.0.smithi163.stdout:82: expect (ObjNum 35 snap 0 seq_num 35) 2023-10-09T09:05:32.518 INFO:tasks.rados.rados.0.smithi163.stdout:83: snap_create 2023-10-09T09:05:32.522 INFO:tasks.rados.rados.0.smithi163.stdout:82: expect (ObjNum 35 snap 0 seq_num 35) 2023-10-09T09:05:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:81: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:05:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:32 smithi181 bash[27712]: cluster 2023-10-09T09:05:31.249169+0000 mgr.y (mgr.24355) 171 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 111 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.6 MiB/s wr, 48 op/s 2023-10-09T09:05:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:32 smithi181 bash[27712]: cluster 2023-10-09T09:05:31.512950+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T09:05:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:32 smithi163 bash[33912]: cluster 2023-10-09T09:05:31.249169+0000 mgr.y (mgr.24355) 171 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 111 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.6 MiB/s wr, 48 op/s 2023-10-09T09:05:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:32 smithi163 bash[33912]: cluster 2023-10-09T09:05:31.512950+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T09:05:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:32 smithi163 bash[27500]: cluster 2023-10-09T09:05:31.249169+0000 mgr.y (mgr.24355) 171 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 111 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.6 MiB/s wr, 48 op/s 2023-10-09T09:05:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:32 smithi163 bash[27500]: cluster 2023-10-09T09:05:31.512950+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T09:05:33.521 INFO:tasks.rados.rados.0.smithi163.stdout:81: done (2 left) 2023-10-09T09:05:33.521 INFO:tasks.rados.rados.0.smithi163.stdout:82: done (1 left) 2023-10-09T09:05:33.522 INFO:tasks.rados.rados.0.smithi163.stdout:83: done (0 left) 2023-10-09T09:05:33.522 INFO:tasks.rados.rados.0.smithi163.stdout:84: delete oid 7 current snap is 5 2023-10-09T09:05:33.526 INFO:tasks.rados.rados.0.smithi163.stdout:84: done (0 left) 2023-10-09T09:05:33.526 INFO:tasks.rados.rados.0.smithi163.stdout:85: write oid 48 current snap is 5 2023-10-09T09:05:33.526 INFO:tasks.rados.rados.0.smithi163.stdout:85: seq_num 66 ranges {709262=697258,2012635=572018,3046388=62874} 2023-10-09T09:05:33.536 INFO:tasks.rados.rados.0.smithi163.stdout:85: writing smithi16367185-48 from 709262 to 1406520 tid 1 2023-10-09T09:05:33.544 INFO:tasks.rados.rados.0.smithi163.stdout:85: writing smithi16367185-48 from 2012635 to 2584653 tid 2 2023-10-09T09:05:33.547 INFO:tasks.rados.rados.0.smithi163.stdout:85: writing smithi16367185-48 from 3046388 to 3109262 tid 3 2023-10-09T09:05:33.547 INFO:tasks.rados.rados.0.smithi163.stdout:86: snap_create 2023-10-09T09:05:33.547 INFO:tasks.rados.rados.0.smithi163.stdout:85: finishing write tid 1 to smithi16367185-48 2023-10-09T09:05:33.548 INFO:tasks.rados.rados.0.smithi163.stdout:85: finishing write tid 2 to smithi16367185-48 2023-10-09T09:05:33.549 INFO:tasks.rados.rados.0.smithi163.stdout:85: finishing write tid 3 to smithi16367185-48 2023-10-09T09:05:33.550 INFO:tasks.rados.rados.0.smithi163.stdout:85: finishing write tid 4 to smithi16367185-48 2023-10-09T09:05:33.550 INFO:tasks.rados.rados.0.smithi163.stdout:85: finishing write tid 5 to smithi16367185-48 2023-10-09T09:05:33.550 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 33 (ObjNum 66 snap 5 seq_num 66) dirty exists 2023-10-09T09:05:33.550 INFO:tasks.rados.rados.0.smithi163.stdout:85: left oid 48 (ObjNum 66 snap 5 seq_num 66) 2023-10-09T09:05:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:33 smithi181 bash[27712]: cluster 2023-10-09T09:05:32.516942+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T09:05:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:33 smithi181 bash[27712]: audit 2023-10-09T09:05:33.080638+0000 mon.a (mon.0) 774 : audit [DBG] from='client.? 172.21.15.163:0/3374590387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:33 smithi163 bash[27500]: cluster 2023-10-09T09:05:32.516942+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T09:05:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:33 smithi163 bash[27500]: audit 2023-10-09T09:05:33.080638+0000 mon.a (mon.0) 774 : audit [DBG] from='client.? 172.21.15.163:0/3374590387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:33.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:33 smithi163 bash[33912]: cluster 2023-10-09T09:05:32.516942+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T09:05:33.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:33 smithi163 bash[33912]: audit 2023-10-09T09:05:33.080638+0000 mon.a (mon.0) 774 : audit [DBG] from='client.? 172.21.15.163:0/3374590387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:34.539 INFO:tasks.rados.rados.0.smithi163.stdout:85: done (1 left) 2023-10-09T09:05:34.539 INFO:tasks.rados.rados.0.smithi163.stdout:86: done (0 left) 2023-10-09T09:05:34.539 INFO:tasks.rados.rados.0.smithi163.stdout:87: rollback oid 10 current snap is 6 2023-10-09T09:05:34.539 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 2 2023-10-09T09:05:34.539 INFO:tasks.rados.rados.0.smithi163.stdout:88: snap_create 2023-10-09T09:05:34.544 INFO:tasks.rados.rados.0.smithi163.stdout:87: finishing rollback tid 0 to smithi16367185-10 2023-10-09T09:05:34.544 INFO:tasks.rados.rados.0.smithi163.stdout:87: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:05:34.545 INFO:tasks.rados.rados.0.smithi163.stdout:87: finishing rollback tid 2 to smithi16367185-10 2023-10-09T09:05:34.545 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 16 (ObjNum 51 snap 0 seq_num 51) dirty exists 2023-10-09T09:05:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:34 smithi181 bash[27712]: cluster 2023-10-09T09:05:33.250243+0000 mgr.y (mgr.24355) 172 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 116 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.2 MiB/s wr, 37 op/s 2023-10-09T09:05:34.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:34 smithi181 bash[27712]: cluster 2023-10-09T09:05:33.520538+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T09:05:34.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:34 smithi163 bash[33912]: cluster 2023-10-09T09:05:33.250243+0000 mgr.y (mgr.24355) 172 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 116 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.2 MiB/s wr, 37 op/s 2023-10-09T09:05:34.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:34 smithi163 bash[33912]: cluster 2023-10-09T09:05:33.520538+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T09:05:34.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:34 smithi163 bash[27500]: cluster 2023-10-09T09:05:33.250243+0000 mgr.y (mgr.24355) 172 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 116 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.2 MiB/s wr, 37 op/s 2023-10-09T09:05:34.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:34 smithi163 bash[27500]: cluster 2023-10-09T09:05:33.520538+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T09:05:35.546 INFO:tasks.rados.rados.0.smithi163.stdout:87: done (1 left) 2023-10-09T09:05:35.546 INFO:tasks.rados.rados.0.smithi163.stdout:88: done (0 left) 2023-10-09T09:05:35.546 INFO:tasks.rados.rados.0.smithi163.stdout:89: delete oid 23 current snap is 7 2023-10-09T09:05:35.552 INFO:tasks.rados.rados.0.smithi163.stdout:89: done (0 left) 2023-10-09T09:05:35.552 INFO:tasks.rados.rados.0.smithi163.stdout:90: read oid 27 snap 3 2023-10-09T09:05:35.552 INFO:tasks.rados.rados.0.smithi163.stdout:90: expect (ObjNum 26 snap 0 seq_num 26) 2023-10-09T09:05:35.552 INFO:tasks.rados.rados.0.smithi163.stdout:91: write oid 12 current snap is 7 2023-10-09T09:05:35.553 INFO:tasks.rados.rados.0.smithi163.stdout:91: seq_num 68 ranges {742876=447756,1729599=575687,3101234=41642} 2023-10-09T09:05:35.562 INFO:tasks.rados.rados.0.smithi163.stdout:91: writing smithi16367185-12 from 742876 to 1190632 tid 1 2023-10-09T09:05:35.569 INFO:tasks.rados.rados.0.smithi163.stdout:91: writing smithi16367185-12 from 1729599 to 2305286 tid 2 2023-10-09T09:05:35.574 INFO:tasks.rados.rados.0.smithi163.stdout:91: writing smithi16367185-12 from 3101234 to 3142876 tid 3 2023-10-09T09:05:35.574 INFO:tasks.rados.rados.0.smithi163.stdout:92: snap_create 2023-10-09T09:05:35.574 INFO:tasks.rados.rados.0.smithi163.stdout:90: expect (ObjNum 26 snap 0 seq_num 26) 2023-10-09T09:05:35.665 INFO:tasks.rados.rados.0.smithi163.stdout:91: finishing write tid 1 to smithi16367185-12 2023-10-09T09:05:35.666 INFO:tasks.rados.rados.0.smithi163.stdout:91: finishing write tid 2 to smithi16367185-12 2023-10-09T09:05:35.666 INFO:tasks.rados.rados.0.smithi163.stdout:91: finishing write tid 3 to smithi16367185-12 2023-10-09T09:05:35.666 INFO:tasks.rados.rados.0.smithi163.stdout:91: finishing write tid 4 to smithi16367185-12 2023-10-09T09:05:35.666 INFO:tasks.rados.rados.0.smithi163.stdout:91: finishing write tid 5 to smithi16367185-12 2023-10-09T09:05:35.666 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 33 (ObjNum 68 snap 7 seq_num 68) dirty exists 2023-10-09T09:05:35.666 INFO:tasks.rados.rados.0.smithi163.stdout:91: left oid 12 (ObjNum 68 snap 7 seq_num 68) 2023-10-09T09:05:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:35 smithi181 bash[27712]: cluster 2023-10-09T09:05:34.539513+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T09:05:35.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:35 smithi163 bash[27500]: cluster 2023-10-09T09:05:34.539513+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T09:05:35.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:35 smithi163 bash[33912]: cluster 2023-10-09T09:05:34.539513+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T09:05:36.555 INFO:tasks.rados.rados.0.smithi163.stdout:90: done (2 left) 2023-10-09T09:05:36.555 INFO:tasks.rados.rados.0.smithi163.stdout:91: done (1 left) 2023-10-09T09:05:36.555 INFO:tasks.rados.rados.0.smithi163.stdout:92: done (0 left) 2023-10-09T09:05:36.555 INFO:tasks.rados.rados.0.smithi163.stdout:93: snap_remove snap 7 2023-10-09T09:05:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:36 smithi181 bash[27712]: cluster 2023-10-09T09:05:35.250747+0000 mgr.y (mgr.24355) 173 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 116 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.2 MiB/s wr, 37 op/s 2023-10-09T09:05:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:36 smithi181 bash[27712]: cluster 2023-10-09T09:05:35.546021+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:05:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:36 smithi181 bash[27712]: audit 2023-10-09T09:05:35.615861+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.163:0/4072523129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:36 smithi163 bash[27500]: cluster 2023-10-09T09:05:35.250747+0000 mgr.y (mgr.24355) 173 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 116 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.2 MiB/s wr, 37 op/s 2023-10-09T09:05:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:36 smithi163 bash[27500]: cluster 2023-10-09T09:05:35.546021+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:05:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:36 smithi163 bash[27500]: audit 2023-10-09T09:05:35.615861+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.163:0/4072523129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:36.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:36 smithi163 bash[33912]: cluster 2023-10-09T09:05:35.250747+0000 mgr.y (mgr.24355) 173 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 116 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.2 MiB/s wr, 37 op/s 2023-10-09T09:05:36.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:36 smithi163 bash[33912]: cluster 2023-10-09T09:05:35.546021+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:05:36.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:36 smithi163 bash[33912]: audit 2023-10-09T09:05:35.615861+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.163:0/4072523129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:37.568 INFO:tasks.rados.rados.0.smithi163.stdout:93: done (0 left) 2023-10-09T09:05:37.569 INFO:tasks.rados.rados.0.smithi163.stdout:94: write oid 15 current snap is 8 2023-10-09T09:05:37.569 INFO:tasks.rados.rados.0.smithi163.stdout:94: seq_num 69 ranges {759683=639358,1904434=719345,3128657=31026} 2023-10-09T09:05:37.578 INFO:tasks.rados.rados.0.smithi163.stdout:94: writing smithi16367185-15 from 759683 to 1399041 tid 1 2023-10-09T09:05:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:94: writing smithi16367185-15 from 1904434 to 2623779 tid 2 2023-10-09T09:05:37.589 INFO:tasks.rados.rados.0.smithi163.stdout:94: writing smithi16367185-15 from 3128657 to 3159683 tid 3 2023-10-09T09:05:37.589 INFO:tasks.rados.rados.0.smithi163.stdout:95: snap_create 2023-10-09T09:05:37.589 INFO:tasks.rados.rados.0.smithi163.stdout:94: finishing write tid 1 to smithi16367185-15 2023-10-09T09:05:37.593 INFO:tasks.rados.rados.0.smithi163.stdout:94: finishing write tid 2 to smithi16367185-15 2023-10-09T09:05:37.593 INFO:tasks.rados.rados.0.smithi163.stdout:94: finishing write tid 3 to smithi16367185-15 2023-10-09T09:05:37.593 INFO:tasks.rados.rados.0.smithi163.stdout:94: finishing write tid 4 to smithi16367185-15 2023-10-09T09:05:37.594 INFO:tasks.rados.rados.0.smithi163.stdout:94: finishing write tid 5 to smithi16367185-15 2023-10-09T09:05:37.594 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 26 (ObjNum 69 snap 8 seq_num 69) dirty exists 2023-10-09T09:05:37.594 INFO:tasks.rados.rados.0.smithi163.stdout:94: left oid 15 (ObjNum 69 snap 8 seq_num 69) 2023-10-09T09:05:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:37 smithi181 bash[27712]: cluster 2023-10-09T09:05:36.554160+0000 mon.a (mon.0) 778 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T09:05:37.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:37 smithi163 bash[33912]: cluster 2023-10-09T09:05:36.554160+0000 mon.a (mon.0) 778 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T09:05:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:37 smithi163 bash[27500]: cluster 2023-10-09T09:05:36.554160+0000 mon.a (mon.0) 778 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T09:05:38.573 INFO:tasks.rados.rados.0.smithi163.stdout:94: done (1 left) 2023-10-09T09:05:38.573 INFO:tasks.rados.rados.0.smithi163.stdout:95: done (0 left) 2023-10-09T09:05:38.573 INFO:tasks.rados.rados.0.smithi163.stdout:96: rollback oid 42 current snap is 9 2023-10-09T09:05:38.573 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 5 2023-10-09T09:05:38.573 INFO:tasks.rados.rados.0.smithi163.stdout:97: write oid 3 current snap is 9 2023-10-09T09:05:38.573 INFO:tasks.rados.rados.0.smithi163.stdout:97: seq_num 70 ranges {776490=714607,2246563=579356} 2023-10-09T09:05:38.582 INFO:tasks.rados.rados.0.smithi163.stdout:97: writing smithi16367185-3 from 776490 to 1491097 tid 1 2023-10-09T09:05:38.591 INFO:tasks.rados.rados.0.smithi163.stdout:97: writing smithi16367185-3 from 2246563 to 2825919 tid 2 2023-10-09T09:05:38.591 INFO:tasks.rados.rados.0.smithi163.stdout:98: delete oid 4 current snap is 9 2023-10-09T09:05:38.591 INFO:tasks.rados.rados.0.smithi163.stdout:96: finishing rollback tid 0 to smithi16367185-42 2023-10-09T09:05:38.591 INFO:tasks.rados.rados.0.smithi163.stdout:96: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:05:38.591 INFO:tasks.rados.rados.0.smithi163.stdout:96: finishing rollback tid 2 to smithi16367185-42 2023-10-09T09:05:38.591 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 37 (ObjNum 41 snap 0 seq_num 41) dirty exists 2023-10-09T09:05:38.591 INFO:tasks.rados.rados.0.smithi163.stdout:97: finishing write tid 1 to smithi16367185-3 2023-10-09T09:05:38.594 INFO:tasks.rados.rados.0.smithi163.stdout:97: finishing write tid 2 to smithi16367185-3 2023-10-09T09:05:38.594 INFO:tasks.rados.rados.0.smithi163.stdout:97: finishing write tid 3 to smithi16367185-3 2023-10-09T09:05:38.595 INFO:tasks.rados.rados.0.smithi163.stdout:97: finishing write tid 4 to smithi16367185-3 2023-10-09T09:05:38.595 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 13 (ObjNum 70 snap 9 seq_num 70) dirty exists 2023-10-09T09:05:38.595 INFO:tasks.rados.rados.0.smithi163.stdout:97: left oid 3 (ObjNum 70 snap 9 seq_num 70) 2023-10-09T09:05:38.599 INFO:tasks.rados.rados.0.smithi163.stdout:96: done (2 left) 2023-10-09T09:05:38.599 INFO:tasks.rados.rados.0.smithi163.stdout:97: done (1 left) 2023-10-09T09:05:38.599 INFO:tasks.rados.rados.0.smithi163.stdout:98: done (0 left) 2023-10-09T09:05:38.599 INFO:tasks.rados.rados.0.smithi163.stdout:99: read oid 39 snap -1 2023-10-09T09:05:38.599 INFO:tasks.rados.rados.0.smithi163.stdout:99: expect (ObjNum 52 snap 0 seq_num 52) 2023-10-09T09:05:38.599 INFO:tasks.rados.rados.0.smithi163.stdout:100: write oid 6 current snap is 9 2023-10-09T09:05:38.599 INFO:tasks.rados.rados.0.smithi163.stdout:100: seq_num 72 ranges {410104=465105,1563527=583025,2810926=399178} 2023-10-09T09:05:38.605 INFO:tasks.rados.rados.0.smithi163.stdout:100: writing smithi16367185-6 from 410104 to 875209 tid 1 2023-10-09T09:05:38.615 INFO:tasks.rados.rados.0.smithi163.stdout:100: writing smithi16367185-6 from 1563527 to 2146552 tid 2 2023-10-09T09:05:38.621 INFO:tasks.rados.rados.0.smithi163.stdout:100: writing smithi16367185-6 from 2810926 to 3210104 tid 3 2023-10-09T09:05:38.621 INFO:tasks.rados.rados.0.smithi163.stdout:101: snap_remove snap 4 2023-10-09T09:05:38.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:05:38 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:05:38.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:38 smithi181 bash[27712]: cluster 2023-10-09T09:05:37.251369+0000 mgr.y (mgr.24355) 174 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 116 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:38.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:38 smithi181 bash[27712]: audit 2023-10-09T09:05:37.556231+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:38.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:38 smithi181 bash[27712]: cluster 2023-10-09T09:05:37.566423+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T09:05:38.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:38 smithi181 bash[27712]: audit 2023-10-09T09:05:38.098308+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.163:0/1724200516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:38 smithi163 bash[33912]: cluster 2023-10-09T09:05:37.251369+0000 mgr.y (mgr.24355) 174 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 116 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:38 smithi163 bash[33912]: audit 2023-10-09T09:05:37.556231+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:38 smithi163 bash[33912]: cluster 2023-10-09T09:05:37.566423+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T09:05:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:38 smithi163 bash[33912]: audit 2023-10-09T09:05:38.098308+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.163:0/1724200516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:38 smithi163 bash[27500]: cluster 2023-10-09T09:05:37.251369+0000 mgr.y (mgr.24355) 174 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 116 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:38 smithi163 bash[27500]: audit 2023-10-09T09:05:37.556231+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:38 smithi163 bash[27500]: cluster 2023-10-09T09:05:37.566423+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T09:05:38.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:38 smithi163 bash[27500]: audit 2023-10-09T09:05:38.098308+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.163:0/1724200516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:39.585 INFO:tasks.rados.rados.0.smithi163.stdout:101: done (2 left) 2023-10-09T09:05:39.586 INFO:tasks.rados.rados.0.smithi163.stdout:102: write oid 43 current snap is 9 2023-10-09T09:05:39.586 INFO:tasks.rados.rados.0.smithi163.stdout:100: finishing write tid 1 to smithi16367185-6 2023-10-09T09:05:39.587 INFO:tasks.rados.rados.0.smithi163.stdout:99: expect (ObjNum 52 snap 0 seq_num 52) 2023-10-09T09:05:39.737 INFO:tasks.rados.rados.0.smithi163.stdout:100: finishing write tid 2 to smithi16367185-6 2023-10-09T09:05:39.737 INFO:tasks.rados.rados.0.smithi163.stdout:100: finishing write tid 3 to smithi16367185-6 2023-10-09T09:05:39.737 INFO:tasks.rados.rados.0.smithi163.stdout:102: seq_num 73 ranges {426911=540354,1622009=726683,3121996=104915} 2023-10-09T09:05:39.742 INFO:tasks.rados.rados.0.smithi163.stdout:102: writing smithi16367185-43 from 426911 to 967265 tid 1 2023-10-09T09:05:39.751 INFO:tasks.rados.rados.0.smithi163.stdout:102: writing smithi16367185-43 from 1622009 to 2348692 tid 2 2023-10-09T09:05:39.756 INFO:tasks.rados.rados.0.smithi163.stdout:102: writing smithi16367185-43 from 3121996 to 3226911 tid 3 2023-10-09T09:05:39.756 INFO:tasks.rados.rados.0.smithi163.stdout:99: done (2 left) 2023-10-09T09:05:39.756 INFO:tasks.rados.rados.0.smithi163.stdout:103: snap_remove snap 5 2023-10-09T09:05:39.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:39 smithi163 bash[33912]: cluster 2023-10-09T09:05:38.572652+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T09:05:39.902 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:39 smithi163 bash[27500]: cluster 2023-10-09T09:05:38.572652+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T09:05:40.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:39 smithi181 bash[27712]: cluster 2023-10-09T09:05:38.572652+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T09:05:40.217 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:39 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:39] "GET /metrics HTTP/1.1" 200 194648 "" "Prometheus/2.33.4" 2023-10-09T09:05:40.586 INFO:tasks.rados.rados.0.smithi163.stdout:103: done (2 left) 2023-10-09T09:05:40.586 INFO:tasks.rados.rados.0.smithi163.stdout:104: delete oid 22 current snap is 9 2023-10-09T09:05:40.586 INFO:tasks.rados.rados.0.smithi163.stdout:100: finishing write tid 4 to smithi16367185-6 2023-10-09T09:05:40.586 INFO:tasks.rados.rados.0.smithi163.stdout:100: finishing write tid 5 to smithi16367185-6 2023-10-09T09:05:40.587 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 8 (ObjNum 72 snap 9 seq_num 72) dirty exists 2023-10-09T09:05:40.587 INFO:tasks.rados.rados.0.smithi163.stdout:100: left oid 6 (ObjNum 72 snap 9 seq_num 72) 2023-10-09T09:05:40.587 INFO:tasks.rados.rados.0.smithi163.stdout:102: finishing write tid 1 to smithi16367185-43 2023-10-09T09:05:40.587 INFO:tasks.rados.rados.0.smithi163.stdout:102: finishing write tid 2 to smithi16367185-43 2023-10-09T09:05:40.587 INFO:tasks.rados.rados.0.smithi163.stdout:102: finishing write tid 3 to smithi16367185-43 2023-10-09T09:05:40.587 INFO:tasks.rados.rados.0.smithi163.stdout:102: finishing write tid 4 to smithi16367185-43 2023-10-09T09:05:40.587 INFO:tasks.rados.rados.0.smithi163.stdout:102: finishing write tid 5 to smithi16367185-43 2023-10-09T09:05:40.587 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 21 (ObjNum 73 snap 9 seq_num 73) dirty exists 2023-10-09T09:05:40.587 INFO:tasks.rados.rados.0.smithi163.stdout:102: left oid 43 (ObjNum 73 snap 9 seq_num 73) 2023-10-09T09:05:40.597 INFO:tasks.rados.rados.0.smithi163.stdout:100: done (2 left) 2023-10-09T09:05:40.597 INFO:tasks.rados.rados.0.smithi163.stdout:102: done (1 left) 2023-10-09T09:05:40.597 INFO:tasks.rados.rados.0.smithi163.stdout:104: done (0 left) 2023-10-09T09:05:40.597 INFO:tasks.rados.rados.0.smithi163.stdout:105: write oid 14 current snap is 9 2023-10-09T09:05:40.598 INFO:tasks.rados.rados.0.smithi163.stdout:105: seq_num 75 ranges {460525=690852,1622620=730352,3060489=200036} 2023-10-09T09:05:40.604 INFO:tasks.rados.rados.0.smithi163.stdout:105: writing smithi16367185-14 from 460525 to 1151377 tid 1 2023-10-09T09:05:40.613 INFO:tasks.rados.rados.0.smithi163.stdout:105: writing smithi16367185-14 from 1622620 to 2352972 tid 2 2023-10-09T09:05:40.620 INFO:tasks.rados.rados.0.smithi163.stdout:105: writing smithi16367185-14 from 3060489 to 3260525 tid 3 2023-10-09T09:05:40.620 INFO:tasks.rados.rados.0.smithi163.stdout:106: rollback oid 30 current snap is 9 2023-10-09T09:05:40.620 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 2 2023-10-09T09:05:40.620 INFO:tasks.rados.rados.0.smithi163.stdout:105: finishing write tid 1 to smithi16367185-14 2023-10-09T09:05:40.620 INFO:tasks.rados.rados.0.smithi163.stdout:105: finishing write tid 2 to smithi16367185-14 2023-10-09T09:05:40.620 INFO:tasks.rados.rados.0.smithi163.stdout:107: delete oid 10 current snap is 9 2023-10-09T09:05:40.622 INFO:tasks.rados.rados.0.smithi163.stdout:106: finishing rollback tid 0 to smithi16367185-30 2023-10-09T09:05:40.623 INFO:tasks.rados.rados.0.smithi163.stdout:105: finishing write tid 3 to smithi16367185-14 2023-10-09T09:05:40.624 INFO:tasks.rados.rados.0.smithi163.stdout:106: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:05:40.624 INFO:tasks.rados.rados.0.smithi163.stdout:106: finishing rollback tid 2 to smithi16367185-30 2023-10-09T09:05:40.624 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 24 (ObjNum 29 snap 0 seq_num 29) dirty exists 2023-10-09T09:05:40.624 INFO:tasks.rados.rados.0.smithi163.stdout:105: finishing write tid 4 to smithi16367185-14 2023-10-09T09:05:40.624 INFO:tasks.rados.rados.0.smithi163.stdout:106: done (2 left) 2023-10-09T09:05:40.624 INFO:tasks.rados.rados.0.smithi163.stdout:107: done (1 left) 2023-10-09T09:05:40.624 INFO:tasks.rados.rados.0.smithi163.stdout:108: rollback oid 16 current snap is 9 2023-10-09T09:05:40.624 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 16 to 2 2023-10-09T09:05:40.624 INFO:tasks.rados.rados.0.smithi163.stdout:109: write oid 22 current snap is 9 2023-10-09T09:05:40.624 INFO:tasks.rados.rados.0.smithi163.stdout:109: seq_num 77 ranges {494139=557703,1455937=734021,2948041=346098} 2023-10-09T09:05:40.630 INFO:tasks.rados.rados.0.smithi163.stdout:109: writing smithi16367185-22 from 494139 to 1051842 tid 1 2023-10-09T09:05:40.637 INFO:tasks.rados.rados.0.smithi163.stdout:109: writing smithi16367185-22 from 1455937 to 2189958 tid 2 2023-10-09T09:05:40.643 INFO:tasks.rados.rados.0.smithi163.stdout:109: writing smithi16367185-22 from 2948041 to 3294139 tid 3 2023-10-09T09:05:40.643 INFO:tasks.rados.rados.0.smithi163.stdout:110: write oid 40 current snap is 9 2023-10-09T09:05:40.643 INFO:tasks.rados.rados.0.smithi163.stdout:105: finishing write tid 5 to smithi16367185-14 2023-10-09T09:05:40.643 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 28 (ObjNum 75 snap 9 seq_num 75) dirty exists 2023-10-09T09:05:40.643 INFO:tasks.rados.rados.0.smithi163.stdout:105: left oid 14 (ObjNum 75 snap 9 seq_num 75) 2023-10-09T09:05:40.643 INFO:tasks.rados.rados.0.smithi163.stdout:110: seq_num 78 ranges {510946=632952,1798066=477679,2742758=457217} 2023-10-09T09:05:40.650 INFO:tasks.rados.rados.0.smithi163.stdout:110: writing smithi16367185-40 from 510946 to 1143898 tid 1 2023-10-09T09:05:40.657 INFO:tasks.rados.rados.0.smithi163.stdout:110: writing smithi16367185-40 from 1798066 to 2275745 tid 2 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:110: writing smithi16367185-40 from 2742758 to 3199975 tid 3 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:105: done (3 left) 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:111: snap_create 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:108: finishing rollback tid 0 to smithi16367185-16 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:108: finishing rollback tid 1 to smithi16367185-16 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:108: finishing rollback tid 2 to smithi16367185-16 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 28 (ObjNum 15 snap 0 seq_num 15) dirty exists 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:109: finishing write tid 1 to smithi16367185-22 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:109: finishing write tid 2 to smithi16367185-22 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:109: finishing write tid 3 to smithi16367185-22 2023-10-09T09:05:40.663 INFO:tasks.rados.rados.0.smithi163.stdout:109: finishing write tid 4 to smithi16367185-22 2023-10-09T09:05:40.664 INFO:tasks.rados.rados.0.smithi163.stdout:109: finishing write tid 5 to smithi16367185-22 2023-10-09T09:05:40.664 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 49 (ObjNum 77 snap 9 seq_num 77) dirty exists 2023-10-09T09:05:40.664 INFO:tasks.rados.rados.0.smithi163.stdout:109: left oid 22 (ObjNum 77 snap 9 seq_num 77) 2023-10-09T09:05:40.664 INFO:tasks.rados.rados.0.smithi163.stdout:110: finishing write tid 1 to smithi16367185-40 2023-10-09T09:05:40.664 INFO:tasks.rados.rados.0.smithi163.stdout:110: finishing write tid 2 to smithi16367185-40 2023-10-09T09:05:40.666 INFO:tasks.rados.rados.0.smithi163.stdout:110: finishing write tid 3 to smithi16367185-40 2023-10-09T09:05:40.666 INFO:tasks.rados.rados.0.smithi163.stdout:110: finishing write tid 4 to smithi16367185-40 2023-10-09T09:05:40.666 INFO:tasks.rados.rados.0.smithi163.stdout:110: finishing write tid 5 to smithi16367185-40 2023-10-09T09:05:40.666 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 34 (ObjNum 78 snap 9 seq_num 78) dirty exists 2023-10-09T09:05:40.666 INFO:tasks.rados.rados.0.smithi163.stdout:110: left oid 40 (ObjNum 78 snap 9 seq_num 78) 2023-10-09T09:05:40.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:40 smithi163 bash[33912]: cluster 2023-10-09T09:05:39.251942+0000 mgr.y (mgr.24355) 175 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 925 KiB/s wr, 8 op/s 2023-10-09T09:05:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:40 smithi163 bash[33912]: cluster 2023-10-09T09:05:39.583741+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T09:05:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:40 smithi163 bash[27500]: cluster 2023-10-09T09:05:39.251942+0000 mgr.y (mgr.24355) 175 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 925 KiB/s wr, 8 op/s 2023-10-09T09:05:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:40 smithi163 bash[27500]: cluster 2023-10-09T09:05:39.583741+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T09:05:41.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:40 smithi181 bash[27712]: cluster 2023-10-09T09:05:39.251942+0000 mgr.y (mgr.24355) 175 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 925 KiB/s wr, 8 op/s 2023-10-09T09:05:41.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:40 smithi181 bash[27712]: cluster 2023-10-09T09:05:39.583741+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T09:05:41.603 INFO:tasks.rados.rados.0.smithi163.stdout:108: done (3 left) 2023-10-09T09:05:41.603 INFO:tasks.rados.rados.0.smithi163.stdout:109: done (2 left) 2023-10-09T09:05:41.603 INFO:tasks.rados.rados.0.smithi163.stdout:110: done (1 left) 2023-10-09T09:05:41.603 INFO:tasks.rados.rados.0.smithi163.stdout:111: done (0 left) 2023-10-09T09:05:41.604 INFO:tasks.rados.rados.0.smithi163.stdout:112: write oid 47 current snap is 10 2023-10-09T09:05:41.604 INFO:tasks.rados.rados.0.smithi163.stdout:112: seq_num 79 ranges {527753=708201,1856548=737690,3286534=41219} 2023-10-09T09:05:41.610 INFO:tasks.rados.rados.0.smithi163.stdout:112: writing smithi16367185-47 from 527753 to 1235954 tid 1 2023-10-09T09:05:41.618 INFO:tasks.rados.rados.0.smithi163.stdout:112: writing smithi16367185-47 from 1856548 to 2594238 tid 2 2023-10-09T09:05:41.623 INFO:tasks.rados.rados.0.smithi163.stdout:112: writing smithi16367185-47 from 3286534 to 3327753 tid 3 2023-10-09T09:05:41.623 INFO:tasks.rados.rados.0.smithi163.stdout:113: delete oid 17 current snap is 10 2023-10-09T09:05:41.623 INFO:tasks.rados.rados.0.smithi163.stdout:112: finishing write tid 1 to smithi16367185-47 2023-10-09T09:05:41.624 INFO:tasks.rados.rados.0.smithi163.stdout:112: finishing write tid 2 to smithi16367185-47 2023-10-09T09:05:41.625 INFO:tasks.rados.rados.0.smithi163.stdout:112: finishing write tid 3 to smithi16367185-47 2023-10-09T09:05:41.626 INFO:tasks.rados.rados.0.smithi163.stdout:112: finishing write tid 4 to smithi16367185-47 2023-10-09T09:05:41.626 INFO:tasks.rados.rados.0.smithi163.stdout:112: finishing write tid 5 to smithi16367185-47 2023-10-09T09:05:41.627 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 54 (ObjNum 79 snap 10 seq_num 79) dirty exists 2023-10-09T09:05:41.627 INFO:tasks.rados.rados.0.smithi163.stdout:112: left oid 47 (ObjNum 79 snap 10 seq_num 79) 2023-10-09T09:05:41.628 INFO:tasks.rados.rados.0.smithi163.stdout:112: done (1 left) 2023-10-09T09:05:41.628 INFO:tasks.rados.rados.0.smithi163.stdout:113: done (0 left) 2023-10-09T09:05:41.628 INFO:tasks.rados.rados.0.smithi163.stdout:114: snap_remove snap 2 2023-10-09T09:05:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:41 smithi163 bash[27500]: cluster 2023-10-09T09:05:40.581668+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T09:05:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:41 smithi163 bash[27500]: audit 2023-10-09T09:05:40.601598+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.163:0/1513472483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:41.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:41 smithi163 bash[33912]: cluster 2023-10-09T09:05:40.581668+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T09:05:41.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:41 smithi163 bash[33912]: audit 2023-10-09T09:05:40.601598+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.163:0/1513472483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:42.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:41 smithi181 bash[27712]: cluster 2023-10-09T09:05:40.581668+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T09:05:42.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:41 smithi181 bash[27712]: audit 2023-10-09T09:05:40.601598+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.163:0/1513472483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:42.077 INFO:tasks.rados.rados.0.smithi163.stdout:114: done (0 left) 2023-10-09T09:05:42.078 INFO:tasks.rados.rados.0.smithi163.stdout:115: write oid 43 current snap is 10 2023-10-09T09:05:42.078 INFO:tasks.rados.rados.0.smithi163.stdout:115: seq_num 81 ranges {561367=458699,1573512=741359,2941380=419987} 2023-10-09T09:05:42.084 INFO:tasks.rados.rados.0.smithi163.stdout:115: writing smithi16367185-43 from 561367 to 1020066 tid 1 2023-10-09T09:05:42.092 INFO:tasks.rados.rados.0.smithi163.stdout:115: writing smithi16367185-43 from 1573512 to 2314871 tid 2 2023-10-09T09:05:42.099 INFO:tasks.rados.rados.0.smithi163.stdout:115: writing smithi16367185-43 from 2941380 to 3361367 tid 3 2023-10-09T09:05:42.099 INFO:tasks.rados.rados.0.smithi163.stdout:115: finishing write tid 1 to smithi16367185-43 2023-10-09T09:05:42.099 INFO:tasks.rados.rados.0.smithi163.stdout:115: finishing write tid 2 to smithi16367185-43 2023-10-09T09:05:42.100 INFO:tasks.rados.rados.0.smithi163.stdout:116: read oid 11 snap -1 2023-10-09T09:05:42.100 INFO:tasks.rados.rados.0.smithi163.stdout:116: expect (ObjNum 10 snap 0 seq_num 10) 2023-10-09T09:05:42.100 INFO:tasks.rados.rados.0.smithi163.stdout:117: rollback oid 37 current snap is 10 2023-10-09T09:05:42.100 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 1 2023-10-09T09:05:42.100 INFO:tasks.rados.rados.0.smithi163.stdout:118: delete oid 38 current snap is 10 2023-10-09T09:05:42.103 INFO:tasks.rados.rados.0.smithi163.stdout:115: finishing write tid 3 to smithi16367185-43 2023-10-09T09:05:42.103 INFO:tasks.rados.rados.0.smithi163.stdout:117: finishing rollback tid 0 to smithi16367185-37 2023-10-09T09:05:42.103 INFO:tasks.rados.rados.0.smithi163.stdout:117: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:05:42.104 INFO:tasks.rados.rados.0.smithi163.stdout:115: finishing write tid 4 to smithi16367185-43 2023-10-09T09:05:42.105 INFO:tasks.rados.rados.0.smithi163.stdout:115: finishing write tid 5 to smithi16367185-43 2023-10-09T09:05:42.105 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 37 (ObjNum 81 snap 10 seq_num 81) dirty exists 2023-10-09T09:05:42.105 INFO:tasks.rados.rados.0.smithi163.stdout:115: left oid 43 (ObjNum 81 snap 10 seq_num 81) 2023-10-09T09:05:42.105 INFO:tasks.rados.rados.0.smithi163.stdout:115: done (3 left) 2023-10-09T09:05:42.105 INFO:tasks.rados.rados.0.smithi163.stdout:118: done (2 left) 2023-10-09T09:05:42.105 INFO:tasks.rados.rados.0.smithi163.stdout:119: read oid 24 snap -1 2023-10-09T09:05:42.105 INFO:tasks.rados.rados.0.smithi163.stdout:119: expect (ObjNum 23 snap 0 seq_num 23) 2023-10-09T09:05:42.105 INFO:tasks.rados.rados.0.smithi163.stdout:120: read oid 2 snap -1 2023-10-09T09:05:42.105 INFO:tasks.rados.rados.0.smithi163.stdout:120: expect (ObjNum 50 snap 0 seq_num 50) 2023-10-09T09:05:42.106 INFO:tasks.rados.rados.0.smithi163.stdout:121: read oid 16 snap -1 2023-10-09T09:05:42.106 INFO:tasks.rados.rados.0.smithi163.stdout:121: expect (ObjNum 15 snap 0 seq_num 15) 2023-10-09T09:05:42.106 INFO:tasks.rados.rados.0.smithi163.stdout:122: snap_remove snap 9 2023-10-09T09:05:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:42 smithi163 bash[27500]: cluster 2023-10-09T09:05:41.252509+0000 mgr.y (mgr.24355) 176 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 925 KiB/s wr, 8 op/s 2023-10-09T09:05:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:42 smithi163 bash[27500]: cluster 2023-10-09T09:05:41.603461+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T09:05:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:42 smithi163 bash[27500]: cluster 2023-10-09T09:05:42.076919+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T09:05:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:42 smithi163 bash[33912]: cluster 2023-10-09T09:05:41.252509+0000 mgr.y (mgr.24355) 176 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 925 KiB/s wr, 8 op/s 2023-10-09T09:05:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:42 smithi163 bash[33912]: cluster 2023-10-09T09:05:41.603461+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T09:05:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:42 smithi163 bash[33912]: cluster 2023-10-09T09:05:42.076919+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T09:05:43.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:42 smithi181 bash[27712]: cluster 2023-10-09T09:05:41.252509+0000 mgr.y (mgr.24355) 176 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 925 KiB/s wr, 8 op/s 2023-10-09T09:05:43.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:42 smithi181 bash[27712]: cluster 2023-10-09T09:05:41.603461+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T09:05:43.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:42 smithi181 bash[27712]: cluster 2023-10-09T09:05:42.076919+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T09:05:43.104 INFO:tasks.rados.rados.0.smithi163.stdout:122: done (5 left) 2023-10-09T09:05:43.104 INFO:tasks.rados.rados.0.smithi163.stdout:123: write oid 40 current snap is 10 2023-10-09T09:05:43.104 INFO:tasks.rados.rados.0.smithi163.stdout:123: seq_num 83 ranges {594981=609197,1974123=745028,3279873=115108} 2023-10-09T09:05:43.112 INFO:tasks.rados.rados.0.smithi163.stdout:123: writing smithi16367185-40 from 594981 to 1204178 tid 1 2023-10-09T09:05:43.122 INFO:tasks.rados.rados.0.smithi163.stdout:123: writing smithi16367185-40 from 1974123 to 2719151 tid 2 2023-10-09T09:05:43.126 INFO:tasks.rados.rados.0.smithi163.stdout:123: writing smithi16367185-40 from 3279873 to 3394981 tid 3 2023-10-09T09:05:43.126 INFO:tasks.rados.rados.0.smithi163.stdout:124: write oid 3 current snap is 10 2023-10-09T09:05:43.126 INFO:tasks.rados.rados.0.smithi163.stdout:117: finishing rollback tid 2 to smithi16367185-37 2023-10-09T09:05:43.126 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 41 (ObjNum 36 snap 0 seq_num 36) dirty exists 2023-10-09T09:05:43.126 INFO:tasks.rados.rados.0.smithi163.stdout:116: expect (ObjNum 10 snap 0 seq_num 10) 2023-10-09T09:05:43.211 INFO:tasks.rados.rados.0.smithi163.stdout:119: expect (ObjNum 23 snap 0 seq_num 23) 2023-10-09T09:05:43.303 INFO:tasks.rados.rados.0.smithi163.stdout:121: expect (ObjNum 15 snap 0 seq_num 15) 2023-10-09T09:05:43.392 INFO:tasks.rados.rados.0.smithi163.stdout:120: expect (ObjNum 50 snap 0 seq_num 50) 2023-10-09T09:05:43.524 INFO:tasks.rados.rados.0.smithi163.stdout:123: finishing write tid 1 to smithi16367185-40 2023-10-09T09:05:43.524 INFO:tasks.rados.rados.0.smithi163.stdout:123: finishing write tid 2 to smithi16367185-40 2023-10-09T09:05:43.524 INFO:tasks.rados.rados.0.smithi163.stdout:123: finishing write tid 3 to smithi16367185-40 2023-10-09T09:05:43.525 INFO:tasks.rados.rados.0.smithi163.stdout:123: finishing write tid 4 to smithi16367185-40 2023-10-09T09:05:43.525 INFO:tasks.rados.rados.0.smithi163.stdout:123: finishing write tid 5 to smithi16367185-40 2023-10-09T09:05:43.525 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 47 (ObjNum 83 snap 10 seq_num 83) dirty exists 2023-10-09T09:05:43.525 INFO:tasks.rados.rados.0.smithi163.stdout:123: left oid 40 (ObjNum 83 snap 10 seq_num 83) 2023-10-09T09:05:43.525 INFO:tasks.rados.rados.0.smithi163.stdout:124: seq_num 84 ranges {611788=400799,1748958=488686,2907296=504492} 2023-10-09T09:05:43.533 INFO:tasks.rados.rados.0.smithi163.stdout:124: writing smithi16367185-3 from 611788 to 1012587 tid 1 2023-10-09T09:05:43.541 INFO:tasks.rados.rados.0.smithi163.stdout:124: writing smithi16367185-3 from 1748958 to 2237644 tid 2 2023-10-09T09:05:43.548 INFO:tasks.rados.rados.0.smithi163.stdout:124: writing smithi16367185-3 from 2907296 to 3411788 tid 3 2023-10-09T09:05:43.549 INFO:tasks.rados.rados.0.smithi163.stdout:116: done (6 left) 2023-10-09T09:05:43.549 INFO:tasks.rados.rados.0.smithi163.stdout:117: done (5 left) 2023-10-09T09:05:43.549 INFO:tasks.rados.rados.0.smithi163.stdout:119: done (4 left) 2023-10-09T09:05:43.549 INFO:tasks.rados.rados.0.smithi163.stdout:120: done (3 left) 2023-10-09T09:05:43.549 INFO:tasks.rados.rados.0.smithi163.stdout:121: done (2 left) 2023-10-09T09:05:43.549 INFO:tasks.rados.rados.0.smithi163.stdout:123: done (1 left) 2023-10-09T09:05:43.549 INFO:tasks.rados.rados.0.smithi163.stdout:125: write oid 1 current snap is 10 2023-10-09T09:05:43.549 INFO:tasks.rados.rados.0.smithi163.stdout:125: seq_num 85 ranges {628595=476048,1807440=632344,2934719=452474} 2023-10-09T09:05:43.555 INFO:tasks.rados.rados.0.smithi163.stdout:125: writing smithi16367185-1 from 628595 to 1104643 tid 1 2023-10-09T09:05:43.562 INFO:tasks.rados.rados.0.smithi163.stdout:125: writing smithi16367185-1 from 1807440 to 2439784 tid 2 2023-10-09T09:05:43.568 INFO:tasks.rados.rados.0.smithi163.stdout:125: writing smithi16367185-1 from 2934719 to 3387193 tid 3 2023-10-09T09:05:43.569 INFO:tasks.rados.rados.0.smithi163.stdout:126: snap_create 2023-10-09T09:05:43.569 INFO:tasks.rados.rados.0.smithi163.stdout:124: finishing write tid 1 to smithi16367185-3 2023-10-09T09:05:43.569 INFO:tasks.rados.rados.0.smithi163.stdout:124: finishing write tid 2 to smithi16367185-3 2023-10-09T09:05:43.569 INFO:tasks.rados.rados.0.smithi163.stdout:124: finishing write tid 3 to smithi16367185-3 2023-10-09T09:05:43.569 INFO:tasks.rados.rados.0.smithi163.stdout:124: finishing write tid 4 to smithi16367185-3 2023-10-09T09:05:43.569 INFO:tasks.rados.rados.0.smithi163.stdout:124: finishing write tid 5 to smithi16367185-3 2023-10-09T09:05:43.569 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 23 (ObjNum 84 snap 10 seq_num 84) dirty exists 2023-10-09T09:05:43.569 INFO:tasks.rados.rados.0.smithi163.stdout:124: left oid 3 (ObjNum 84 snap 10 seq_num 84) 2023-10-09T09:05:43.569 INFO:tasks.rados.rados.0.smithi163.stdout:125: finishing write tid 1 to smithi16367185-1 2023-10-09T09:05:43.569 INFO:tasks.rados.rados.0.smithi163.stdout:125: finishing write tid 2 to smithi16367185-1 2023-10-09T09:05:43.571 INFO:tasks.rados.rados.0.smithi163.stdout:125: finishing write tid 3 to smithi16367185-1 2023-10-09T09:05:43.571 INFO:tasks.rados.rados.0.smithi163.stdout:125: finishing write tid 4 to smithi16367185-1 2023-10-09T09:05:43.572 INFO:tasks.rados.rados.0.smithi163.stdout:125: finishing write tid 5 to smithi16367185-1 2023-10-09T09:05:43.572 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 37 (ObjNum 85 snap 10 seq_num 85) dirty exists 2023-10-09T09:05:43.572 INFO:tasks.rados.rados.0.smithi163.stdout:125: left oid 1 (ObjNum 85 snap 10 seq_num 85) 2023-10-09T09:05:43.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:43 smithi163 bash[27500]: audit 2023-10-09T09:05:43.075118+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.163:0/2635182637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:43.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:43 smithi163 bash[27500]: cluster 2023-10-09T09:05:43.102618+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T09:05:43.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:43 smithi163 bash[33912]: audit 2023-10-09T09:05:43.075118+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.163:0/2635182637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:43.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:43 smithi163 bash[33912]: cluster 2023-10-09T09:05:43.102618+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T09:05:44.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:43 smithi181 bash[27712]: audit 2023-10-09T09:05:43.075118+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.163:0/2635182637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:44.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:43 smithi181 bash[27712]: cluster 2023-10-09T09:05:43.102618+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T09:05:44.110 INFO:tasks.rados.rados.0.smithi163.stdout:124: done (2 left) 2023-10-09T09:05:44.110 INFO:tasks.rados.rados.0.smithi163.stdout:125: done (1 left) 2023-10-09T09:05:44.110 INFO:tasks.rados.rados.0.smithi163.stdout:126: done (0 left) 2023-10-09T09:05:44.110 INFO:tasks.rados.rados.0.smithi163.stdout:127: write oid 32 current snap is 11 2023-10-09T09:05:44.110 INFO:tasks.rados.rados.0.smithi163.stdout:127: seq_num 86 ranges {645402=551297,1749569=492355,2845789=599613} 2023-10-09T09:05:44.117 INFO:tasks.rados.rados.0.smithi163.stdout:127: writing smithi16367185-32 from 645402 to 1196699 tid 1 2023-10-09T09:05:44.124 INFO:tasks.rados.rados.0.smithi163.stdout:127: writing smithi16367185-32 from 1749569 to 2241924 tid 2 2023-10-09T09:05:44.133 INFO:tasks.rados.rados.0.smithi163.stdout:127: writing smithi16367185-32 from 2845789 to 3445402 tid 3 2023-10-09T09:05:44.133 INFO:tasks.rados.rados.0.smithi163.stdout:128: read oid 35 snap -1 2023-10-09T09:05:44.133 INFO:tasks.rados.rados.0.smithi163.stdout:128: expect (ObjNum 34 snap 0 seq_num 34) 2023-10-09T09:05:44.133 INFO:tasks.rados.rados.0.smithi163.stdout:129: read oid 11 snap -1 2023-10-09T09:05:44.134 INFO:tasks.rados.rados.0.smithi163.stdout:129: expect (ObjNum 10 snap 0 seq_num 10) 2023-10-09T09:05:44.134 INFO:tasks.rados.rados.0.smithi163.stdout:130: write oid 42 current snap is 11 2023-10-09T09:05:44.134 INFO:tasks.rados.rados.0.smithi163.stdout:127: finishing write tid 1 to smithi16367185-32 2023-10-09T09:05:44.134 INFO:tasks.rados.rados.0.smithi163.stdout:127: finishing write tid 2 to smithi16367185-32 2023-10-09T09:05:44.134 INFO:tasks.rados.rados.0.smithi163.stdout:130: seq_num 87 ranges {662209=626546,1808051=636013,2873212=588997} 2023-10-09T09:05:44.141 INFO:tasks.rados.rados.0.smithi163.stdout:130: writing smithi16367185-42 from 662209 to 1288755 tid 1 2023-10-09T09:05:44.148 INFO:tasks.rados.rados.0.smithi163.stdout:130: writing smithi16367185-42 from 1808051 to 2444064 tid 2 2023-10-09T09:05:44.154 INFO:tasks.rados.rados.0.smithi163.stdout:130: writing smithi16367185-42 from 2873212 to 3462209 tid 3 2023-10-09T09:05:44.154 INFO:tasks.rados.rados.0.smithi163.stdout:131: snap_remove snap 10 2023-10-09T09:05:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:44 smithi163 bash[27500]: cluster 2023-10-09T09:05:43.253444+0000 mgr.y (mgr.24355) 177 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 29 op/s 2023-10-09T09:05:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:44 smithi163 bash[27500]: cluster 2023-10-09T09:05:44.109579+0000 mon.a (mon.0) 789 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T09:05:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:44 smithi163 bash[33912]: cluster 2023-10-09T09:05:43.253444+0000 mgr.y (mgr.24355) 177 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 29 op/s 2023-10-09T09:05:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:44 smithi163 bash[33912]: cluster 2023-10-09T09:05:44.109579+0000 mon.a (mon.0) 789 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T09:05:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:44 smithi181 bash[27712]: cluster 2023-10-09T09:05:43.253444+0000 mgr.y (mgr.24355) 177 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 29 op/s 2023-10-09T09:05:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:44 smithi181 bash[27712]: cluster 2023-10-09T09:05:44.109579+0000 mon.a (mon.0) 789 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T09:05:45.117 INFO:tasks.rados.rados.0.smithi163.stdout:131: done (4 left) 2023-10-09T09:05:45.117 INFO:tasks.rados.rados.0.smithi163.stdout:132: read oid 16 snap -1 2023-10-09T09:05:45.117 INFO:tasks.rados.rados.0.smithi163.stdout:132: expect (ObjNum 15 snap 0 seq_num 15) 2023-10-09T09:05:45.117 INFO:tasks.rados.rados.0.smithi163.stdout:133: snap_remove snap 3 2023-10-09T09:05:46.130 INFO:tasks.rados.rados.0.smithi163.stdout:133: done (5 left) 2023-10-09T09:05:46.130 INFO:tasks.rados.rados.0.smithi163.stdout:134: write oid 20 current snap is 11 2023-10-09T09:05:46.130 INFO:tasks.rados.rados.0.smithi163.stdout:127: finishing write tid 3 to smithi16367185-32 2023-10-09T09:05:46.130 INFO:tasks.rados.rados.0.smithi163.stdout:127: finishing write tid 4 to smithi16367185-32 2023-10-09T09:05:46.130 INFO:tasks.rados.rados.0.smithi163.stdout:127: finishing write tid 5 to smithi16367185-32 2023-10-09T09:05:46.131 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 69 (ObjNum 86 snap 11 seq_num 86) dirty exists 2023-10-09T09:05:46.131 INFO:tasks.rados.rados.0.smithi163.stdout:127: left oid 32 (ObjNum 86 snap 11 seq_num 86) 2023-10-09T09:05:46.131 INFO:tasks.rados.rados.0.smithi163.stdout:128: expect (ObjNum 34 snap 0 seq_num 34) 2023-10-09T09:05:46.228 INFO:tasks.rados.rados.0.smithi163.stdout:130: finishing write tid 1 to smithi16367185-42 2023-10-09T09:05:46.228 INFO:tasks.rados.rados.0.smithi163.stdout:129: expect (ObjNum 10 snap 0 seq_num 10) 2023-10-09T09:05:46.308 INFO:tasks.rados.rados.0.smithi163.stdout:130: finishing write tid 2 to smithi16367185-42 2023-10-09T09:05:46.308 INFO:tasks.rados.rados.0.smithi163.stdout:130: finishing write tid 3 to smithi16367185-42 2023-10-09T09:05:46.308 INFO:tasks.rados.rados.0.smithi163.stdout:134: seq_num 88 ranges {679016=701795,1866533=496024,2900635=402643} 2023-10-09T09:05:46.316 INFO:tasks.rados.rados.0.smithi163.stdout:134: writing smithi16367185-20 from 679016 to 1380811 tid 1 2023-10-09T09:05:46.322 INFO:tasks.rados.rados.0.smithi163.stdout:134: writing smithi16367185-20 from 1866533 to 2362557 tid 2 2023-10-09T09:05:46.329 INFO:tasks.rados.rados.0.smithi163.stdout:134: writing smithi16367185-20 from 2900635 to 3303278 tid 3 2023-10-09T09:05:46.329 INFO:tasks.rados.rados.0.smithi163.stdout:127: done (5 left) 2023-10-09T09:05:46.329 INFO:tasks.rados.rados.0.smithi163.stdout:128: done (4 left) 2023-10-09T09:05:46.329 INFO:tasks.rados.rados.0.smithi163.stdout:129: done (3 left) 2023-10-09T09:05:46.329 INFO:tasks.rados.rados.0.smithi163.stdout:135: read oid 29 snap -1 2023-10-09T09:05:46.330 INFO:tasks.rados.rados.0.smithi163.stdout:135: expect (ObjNum 56 snap 0 seq_num 56) 2023-10-09T09:05:46.330 INFO:tasks.rados.rados.0.smithi163.stdout:136: write oid 34 current snap is 11 2023-10-09T09:05:46.330 INFO:tasks.rados.rados.0.smithi163.stdout:130: finishing write tid 4 to smithi16367185-42 2023-10-09T09:05:46.330 INFO:tasks.rados.rados.0.smithi163.stdout:130: finishing write tid 5 to smithi16367185-42 2023-10-09T09:05:46.330 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 74 (ObjNum 87 snap 11 seq_num 87) dirty exists 2023-10-09T09:05:46.330 INFO:tasks.rados.rados.0.smithi163.stdout:130: left oid 42 (ObjNum 87 snap 11 seq_num 87) 2023-10-09T09:05:46.331 INFO:tasks.rados.rados.0.smithi163.stdout:132: expect (ObjNum 15 snap 0 seq_num 15) 2023-10-09T09:05:46.407 INFO:tasks.rados.rados.0.smithi163.stdout:134: finishing write tid 1 to smithi16367185-20 2023-10-09T09:05:46.407 INFO:tasks.rados.rados.0.smithi163.stdout:134: finishing write tid 2 to smithi16367185-20 2023-10-09T09:05:46.407 INFO:tasks.rados.rados.0.smithi163.stdout:136: seq_num 89 ranges {695823=777044,1925015=639682,3328058=167765} 2023-10-09T09:05:46.415 INFO:tasks.rados.rados.0.smithi163.stdout:136: writing smithi16367185-34 from 695823 to 1472867 tid 1 2023-10-09T09:05:46.422 INFO:tasks.rados.rados.0.smithi163.stdout:136: writing smithi16367185-34 from 1925015 to 2564697 tid 2 2023-10-09T09:05:46.429 INFO:tasks.rados.rados.0.smithi163.stdout:136: writing smithi16367185-34 from 3328058 to 3495823 tid 3 2023-10-09T09:05:46.429 INFO:tasks.rados.rados.0.smithi163.stdout:130: done (4 left) 2023-10-09T09:05:46.429 INFO:tasks.rados.rados.0.smithi163.stdout:132: done (3 left) 2023-10-09T09:05:46.429 INFO:tasks.rados.rados.0.smithi163.stdout:137: delete oid 31 current snap is 11 2023-10-09T09:05:46.429 INFO:tasks.rados.rados.0.smithi163.stdout:134: finishing write tid 3 to smithi16367185-20 2023-10-09T09:05:46.429 INFO:tasks.rados.rados.0.smithi163.stdout:134: finishing write tid 4 to smithi16367185-20 2023-10-09T09:05:46.429 INFO:tasks.rados.rados.0.smithi163.stdout:134: finishing write tid 5 to smithi16367185-20 2023-10-09T09:05:46.429 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 12 (ObjNum 88 snap 11 seq_num 88) dirty exists 2023-10-09T09:05:46.429 INFO:tasks.rados.rados.0.smithi163.stdout:134: left oid 20 (ObjNum 88 snap 11 seq_num 88) 2023-10-09T09:05:46.429 INFO:tasks.rados.rados.0.smithi163.stdout:135: expect (ObjNum 56 snap 0 seq_num 56) 2023-10-09T09:05:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:46 smithi163 bash[33912]: cluster 2023-10-09T09:05:45.114796+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-10-09T09:05:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:46 smithi163 bash[33912]: cluster 2023-10-09T09:05:45.253917+0000 mgr.y (mgr.24355) 178 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 29 op/s 2023-10-09T09:05:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:46 smithi163 bash[33912]: audit 2023-10-09T09:05:45.523264+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.163:0/148378270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:46 smithi163 bash[27500]: cluster 2023-10-09T09:05:45.114796+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-10-09T09:05:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:46 smithi163 bash[27500]: cluster 2023-10-09T09:05:45.253917+0000 mgr.y (mgr.24355) 178 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 29 op/s 2023-10-09T09:05:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:46 smithi163 bash[27500]: audit 2023-10-09T09:05:45.523264+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.163:0/148378270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:46 smithi181 bash[27712]: cluster 2023-10-09T09:05:45.114796+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-10-09T09:05:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:46 smithi181 bash[27712]: cluster 2023-10-09T09:05:45.253917+0000 mgr.y (mgr.24355) 178 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 29 op/s 2023-10-09T09:05:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:46 smithi181 bash[27712]: audit 2023-10-09T09:05:45.523264+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.163:0/148378270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:46.579 INFO:tasks.rados.rados.0.smithi163.stdout:136: finishing write tid 1 to smithi16367185-34 2023-10-09T09:05:46.580 INFO:tasks.rados.rados.0.smithi163.stdout:136: finishing write tid 2 to smithi16367185-34 2023-10-09T09:05:46.580 INFO:tasks.rados.rados.0.smithi163.stdout:136: finishing write tid 3 to smithi16367185-34 2023-10-09T09:05:46.580 INFO:tasks.rados.rados.0.smithi163.stdout:136: finishing write tid 4 to smithi16367185-34 2023-10-09T09:05:46.580 INFO:tasks.rados.rados.0.smithi163.stdout:136: finishing write tid 5 to smithi16367185-34 2023-10-09T09:05:46.580 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 36 (ObjNum 89 snap 11 seq_num 89) dirty exists 2023-10-09T09:05:46.580 INFO:tasks.rados.rados.0.smithi163.stdout:136: left oid 34 (ObjNum 89 snap 11 seq_num 89) 2023-10-09T09:05:46.580 INFO:tasks.rados.rados.0.smithi163.stdout:134: done (3 left) 2023-10-09T09:05:46.580 INFO:tasks.rados.rados.0.smithi163.stdout:135: done (2 left) 2023-10-09T09:05:46.581 INFO:tasks.rados.rados.0.smithi163.stdout:136: done (1 left) 2023-10-09T09:05:46.581 INFO:tasks.rados.rados.0.smithi163.stdout:137: done (0 left) 2023-10-09T09:05:46.581 INFO:tasks.rados.rados.0.smithi163.stdout:138: snap_create 2023-10-09T09:05:47.081 INFO:tasks.rados.rados.0.smithi163.stdout:138: done (0 left) 2023-10-09T09:05:47.081 INFO:tasks.rados.rados.0.smithi163.stdout:139: snap_remove snap 1 2023-10-09T09:05:47.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:47 smithi163 bash[33912]: cluster 2023-10-09T09:05:46.128791+0000 mon.a (mon.0) 792 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2023-10-09T09:05:47.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:47 smithi163 bash[33912]: cluster 2023-10-09T09:05:47.081042+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2023-10-09T09:05:47.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:47 smithi163 bash[27500]: cluster 2023-10-09T09:05:46.128791+0000 mon.a (mon.0) 792 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2023-10-09T09:05:47.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:47 smithi163 bash[27500]: cluster 2023-10-09T09:05:47.081042+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2023-10-09T09:05:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:47 smithi181 bash[27712]: cluster 2023-10-09T09:05:46.128791+0000 mon.a (mon.0) 792 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2023-10-09T09:05:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:47 smithi181 bash[27712]: cluster 2023-10-09T09:05:47.081042+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2023-10-09T09:05:48.083 INFO:tasks.rados.rados.0.smithi163.stdout:139: done (0 left) 2023-10-09T09:05:48.083 INFO:tasks.rados.rados.0.smithi163.stdout:140: snap_create 2023-10-09T09:05:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:48 smithi163 bash[27500]: cluster 2023-10-09T09:05:47.254621+0000 mgr.y (mgr.24355) 179 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:48 smithi163 bash[27500]: audit 2023-10-09T09:05:47.927516+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.163:0/4194375212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:48 smithi163 bash[27500]: cluster 2023-10-09T09:05:48.083116+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2023-10-09T09:05:48.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:48 smithi163 bash[33912]: cluster 2023-10-09T09:05:47.254621+0000 mgr.y (mgr.24355) 179 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:48 smithi163 bash[33912]: audit 2023-10-09T09:05:47.927516+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.163:0/4194375212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:48 smithi163 bash[33912]: cluster 2023-10-09T09:05:48.083116+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2023-10-09T09:05:48.553 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:48 smithi181 bash[27712]: cluster 2023-10-09T09:05:47.254621+0000 mgr.y (mgr.24355) 179 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:05:48.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:48 smithi181 bash[27712]: audit 2023-10-09T09:05:47.927516+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.163:0/4194375212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:48.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:48 smithi181 bash[27712]: cluster 2023-10-09T09:05:48.083116+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2023-10-09T09:05:48.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:05:48 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:05:49.089 INFO:tasks.rados.rados.0.smithi163.stdout:140: done (0 left) 2023-10-09T09:05:49.351 INFO:tasks.rados.rados.0.smithi163.stdout:141: read oid 31 snap -1 2023-10-09T09:05:49.351 INFO:tasks.rados.rados.0.smithi163.stdout:141: expect deleted 2023-10-09T09:05:49.351 INFO:tasks.rados.rados.0.smithi163.stdout:142: read oid 13 snap -1 2023-10-09T09:05:49.351 INFO:tasks.rados.rados.0.smithi163.stdout:142: expect deleted 2023-10-09T09:05:49.351 INFO:tasks.rados.rados.0.smithi163.stdout:143: read oid 7 snap -1 2023-10-09T09:05:49.351 INFO:tasks.rados.rados.0.smithi163.stdout:143: expect deleted 2023-10-09T09:05:49.351 INFO:tasks.rados.rados.0.smithi163.stdout:144: snap_create 2023-10-09T09:05:50.097 INFO:tasks.rados.rados.0.smithi163.stdout:141: done (3 left) 2023-10-09T09:05:50.097 INFO:tasks.rados.rados.0.smithi163.stdout:142: done (2 left) 2023-10-09T09:05:50.097 INFO:tasks.rados.rados.0.smithi163.stdout:143: done (1 left) 2023-10-09T09:05:50.098 INFO:tasks.rados.rados.0.smithi163.stdout:144: done (0 left) 2023-10-09T09:05:50.098 INFO:tasks.rados.rados.0.smithi163.stdout:145: snap_create 2023-10-09T09:05:50.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:49 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:49] "GET /metrics HTTP/1.1" 200 194648 "" "Prometheus/2.33.4" 2023-10-09T09:05:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:50 smithi163 bash[33912]: cluster 2023-10-09T09:05:49.086920+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2023-10-09T09:05:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:50 smithi163 bash[33912]: cluster 2023-10-09T09:05:49.255334+0000 mgr.y (mgr.24355) 180 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2023-10-09T09:05:50.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:50 smithi163 bash[27500]: cluster 2023-10-09T09:05:49.086920+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2023-10-09T09:05:50.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:50 smithi163 bash[27500]: cluster 2023-10-09T09:05:49.255334+0000 mgr.y (mgr.24355) 180 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2023-10-09T09:05:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:50 smithi181 bash[27712]: cluster 2023-10-09T09:05:49.086920+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2023-10-09T09:05:50.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:50 smithi181 bash[27712]: cluster 2023-10-09T09:05:49.255334+0000 mgr.y (mgr.24355) 180 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2023-10-09T09:05:51.102 INFO:tasks.rados.rados.0.smithi163.stdout:145: done (0 left) 2023-10-09T09:05:51.102 INFO:tasks.rados.rados.0.smithi163.stdout:146: write oid 9 current snap is 15 2023-10-09T09:05:51.102 INFO:tasks.rados.rados.0.smithi163.stdout:146: seq_num 91 ranges {729437=527542,1925626=643351,3266551=262886} 2023-10-09T09:05:51.108 INFO:tasks.rados.rados.0.smithi163.stdout:146: writing smithi16367185-9 from 729437 to 1256979 tid 1 2023-10-09T09:05:51.116 INFO:tasks.rados.rados.0.smithi163.stdout:146: writing smithi16367185-9 from 1925626 to 2568977 tid 2 2023-10-09T09:05:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:146: writing smithi16367185-9 from 3266551 to 3529437 tid 3 2023-10-09T09:05:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:147: rollback oid 14 current snap is 15 2023-10-09T09:05:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 13 2023-10-09T09:05:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:146: finishing write tid 1 to smithi16367185-9 2023-10-09T09:05:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:146: finishing write tid 2 to smithi16367185-9 2023-10-09T09:05:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:148: write oid 6 current snap is 15 2023-10-09T09:05:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:148: seq_num 92 ranges {746244=719144,2100461=503362,3126680=419564} 2023-10-09T09:05:51.130 INFO:tasks.rados.rados.0.smithi163.stdout:148: writing smithi16367185-6 from 746244 to 1465388 tid 1 2023-10-09T09:05:51.136 INFO:tasks.rados.rados.0.smithi163.stdout:148: writing smithi16367185-6 from 2100461 to 2603823 tid 2 2023-10-09T09:05:51.142 INFO:tasks.rados.rados.0.smithi163.stdout:148: writing smithi16367185-6 from 3126680 to 3546244 tid 3 2023-10-09T09:05:51.142 INFO:tasks.rados.rados.0.smithi163.stdout:149: read oid 42 snap -1 2023-10-09T09:05:51.142 INFO:tasks.rados.rados.0.smithi163.stdout:149: expect (ObjNum 87 snap 11 seq_num 87) 2023-10-09T09:05:51.142 INFO:tasks.rados.rados.0.smithi163.stdout:150: snap_create 2023-10-09T09:05:51.142 INFO:tasks.rados.rados.0.smithi163.stdout:146: finishing write tid 3 to smithi16367185-9 2023-10-09T09:05:51.142 INFO:tasks.rados.rados.0.smithi163.stdout:146: finishing write tid 4 to smithi16367185-9 2023-10-09T09:05:51.143 INFO:tasks.rados.rados.0.smithi163.stdout:146: finishing write tid 5 to smithi16367185-9 2023-10-09T09:05:51.143 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 19 (ObjNum 91 snap 15 seq_num 91) dirty exists 2023-10-09T09:05:51.143 INFO:tasks.rados.rados.0.smithi163.stdout:146: left oid 9 (ObjNum 91 snap 15 seq_num 91) 2023-10-09T09:05:51.143 INFO:tasks.rados.rados.0.smithi163.stdout:147: finishing rollback tid 0 to smithi16367185-14 2023-10-09T09:05:51.143 INFO:tasks.rados.rados.0.smithi163.stdout:147: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:05:51.143 INFO:tasks.rados.rados.0.smithi163.stdout:147: finishing rollback tid 2 to smithi16367185-14 2023-10-09T09:05:51.143 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 59 (ObjNum 75 snap 9 seq_num 75) dirty exists 2023-10-09T09:05:51.143 INFO:tasks.rados.rados.0.smithi163.stdout:148: finishing write tid 1 to smithi16367185-6 2023-10-09T09:05:51.143 INFO:tasks.rados.rados.0.smithi163.stdout:148: finishing write tid 2 to smithi16367185-6 2023-10-09T09:05:51.145 INFO:tasks.rados.rados.0.smithi163.stdout:148: finishing write tid 3 to smithi16367185-6 2023-10-09T09:05:51.145 INFO:tasks.rados.rados.0.smithi163.stdout:148: finishing write tid 4 to smithi16367185-6 2023-10-09T09:05:51.145 INFO:tasks.rados.rados.0.smithi163.stdout:148: finishing write tid 5 to smithi16367185-6 2023-10-09T09:05:51.146 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 23 (ObjNum 92 snap 15 seq_num 92) dirty exists 2023-10-09T09:05:51.146 INFO:tasks.rados.rados.0.smithi163.stdout:148: left oid 6 (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:05:51.158 INFO:tasks.rados.rados.0.smithi163.stdout:149: expect (ObjNum 87 snap 11 seq_num 87) 2023-10-09T09:05:51.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:51 smithi163 bash[33912]: cluster 2023-10-09T09:05:50.095391+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2023-10-09T09:05:51.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:51 smithi163 bash[33912]: audit 2023-10-09T09:05:50.342401+0000 mon.a (mon.0) 797 : audit [DBG] from='client.? 172.21.15.163:0/323898956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:51.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:51 smithi163 bash[27500]: cluster 2023-10-09T09:05:50.095391+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2023-10-09T09:05:51.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:51 smithi163 bash[27500]: audit 2023-10-09T09:05:50.342401+0000 mon.a (mon.0) 797 : audit [DBG] from='client.? 172.21.15.163:0/323898956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:51 smithi181 bash[27712]: cluster 2023-10-09T09:05:50.095391+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2023-10-09T09:05:51.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:51 smithi181 bash[27712]: audit 2023-10-09T09:05:50.342401+0000 mon.a (mon.0) 797 : audit [DBG] from='client.? 172.21.15.163:0/323898956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:52.083 INFO:tasks.rados.rados.0.smithi163.stdout:146: done (4 left) 2023-10-09T09:05:52.083 INFO:tasks.rados.rados.0.smithi163.stdout:147: done (3 left) 2023-10-09T09:05:52.084 INFO:tasks.rados.rados.0.smithi163.stdout:148: done (2 left) 2023-10-09T09:05:52.084 INFO:tasks.rados.rados.0.smithi163.stdout:149: done (1 left) 2023-10-09T09:05:52.084 INFO:tasks.rados.rados.0.smithi163.stdout:150: done (0 left) 2023-10-09T09:05:52.084 INFO:tasks.rados.rados.0.smithi163.stdout:151: write oid 46 current snap is 16 2023-10-09T09:05:52.084 INFO:tasks.rados.rados.0.smithi163.stdout:151: seq_num 93 ranges {763051=794393,2158943=647020,3437750=125301} 2023-10-09T09:05:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:151: writing smithi16367185-46 from 763051 to 1557444 tid 1 2023-10-09T09:05:52.100 INFO:tasks.rados.rados.0.smithi163.stdout:151: writing smithi16367185-46 from 2158943 to 2805963 tid 2 2023-10-09T09:05:52.105 INFO:tasks.rados.rados.0.smithi163.stdout:151: writing smithi16367185-46 from 3437750 to 3563051 tid 3 2023-10-09T09:05:52.106 INFO:tasks.rados.rados.0.smithi163.stdout:152: read oid 25 snap -1 2023-10-09T09:05:52.106 INFO:tasks.rados.rados.0.smithi163.stdout:152: expect (ObjNum 60 snap 0 seq_num 60) 2023-10-09T09:05:52.106 INFO:tasks.rados.rados.0.smithi163.stdout:153: write oid 50 current snap is 16 2023-10-09T09:05:52.106 INFO:tasks.rados.rados.0.smithi163.stdout:151: finishing write tid 1 to smithi16367185-46 2023-10-09T09:05:52.106 INFO:tasks.rados.rados.0.smithi163.stdout:151: finishing write tid 2 to smithi16367185-46 2023-10-09T09:05:52.106 INFO:tasks.rados.rados.0.smithi163.stdout:153: seq_num 94 ranges {779858=469642,1817425=507031,2781526=586628} 2023-10-09T09:05:52.113 INFO:tasks.rados.rados.0.smithi163.stdout:153: writing smithi16367185-50 from 779858 to 1249500 tid 1 2023-10-09T09:05:52.119 INFO:tasks.rados.rados.0.smithi163.stdout:153: writing smithi16367185-50 from 1817425 to 2324456 tid 2 2023-10-09T09:05:52.125 INFO:tasks.rados.rados.0.smithi163.stdout:153: writing smithi16367185-50 from 2781526 to 3368154 tid 3 2023-10-09T09:05:52.125 INFO:tasks.rados.rados.0.smithi163.stdout:154: read oid 6 snap 12 2023-10-09T09:05:52.125 INFO:tasks.rados.rados.0.smithi163.stdout:154: expect (ObjNum 72 snap 9 seq_num 72) 2023-10-09T09:05:52.125 INFO:tasks.rados.rados.0.smithi163.stdout:155: snap_remove snap 8 2023-10-09T09:05:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:52 smithi163 bash[33912]: cluster 2023-10-09T09:05:51.100166+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2023-10-09T09:05:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:52 smithi163 bash[33912]: cluster 2023-10-09T09:05:51.255962+0000 mgr.y (mgr.24355) 181 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2023-10-09T09:05:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:52 smithi163 bash[33912]: cluster 2023-10-09T09:05:52.080797+0000 mon.a (mon.0) 799 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2023-10-09T09:05:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:52 smithi163 bash[27500]: cluster 2023-10-09T09:05:51.100166+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2023-10-09T09:05:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:52 smithi163 bash[27500]: cluster 2023-10-09T09:05:51.255962+0000 mgr.y (mgr.24355) 181 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2023-10-09T09:05:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:52 smithi163 bash[27500]: cluster 2023-10-09T09:05:52.080797+0000 mon.a (mon.0) 799 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2023-10-09T09:05:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:52 smithi181 bash[27712]: cluster 2023-10-09T09:05:51.100166+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2023-10-09T09:05:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:52 smithi181 bash[27712]: cluster 2023-10-09T09:05:51.255962+0000 mgr.y (mgr.24355) 181 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2023-10-09T09:05:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:52 smithi181 bash[27712]: cluster 2023-10-09T09:05:52.080797+0000 mon.a (mon.0) 799 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2023-10-09T09:05:53.129 INFO:tasks.rados.rados.0.smithi163.stdout:155: done (4 left) 2023-10-09T09:05:53.129 INFO:tasks.rados.rados.0.smithi163.stdout:156: snap_remove snap 6 2023-10-09T09:05:53.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:53 smithi163 bash[33912]: audit 2023-10-09T09:05:52.745225+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.163:0/2144573157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:53.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:53 smithi163 bash[27500]: audit 2023-10-09T09:05:52.745225+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.163:0/2144573157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:53 smithi181 bash[27712]: audit 2023-10-09T09:05:52.745225+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.163:0/2144573157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:54.142 INFO:tasks.rados.rados.0.smithi163.stdout:156: done (4 left) 2023-10-09T09:05:54.143 INFO:tasks.rados.rados.0.smithi163.stdout:157: delete oid 16 current snap is 16 2023-10-09T09:05:54.143 INFO:tasks.rados.rados.0.smithi163.stdout:151: finishing write tid 3 to smithi16367185-46 2023-10-09T09:05:54.143 INFO:tasks.rados.rados.0.smithi163.stdout:151: finishing write tid 4 to smithi16367185-46 2023-10-09T09:05:54.143 INFO:tasks.rados.rados.0.smithi163.stdout:151: finishing write tid 5 to smithi16367185-46 2023-10-09T09:05:54.143 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 64 (ObjNum 93 snap 16 seq_num 93) dirty exists 2023-10-09T09:05:54.143 INFO:tasks.rados.rados.0.smithi163.stdout:151: left oid 46 (ObjNum 93 snap 16 seq_num 93) 2023-10-09T09:05:54.143 INFO:tasks.rados.rados.0.smithi163.stdout:152: expect (ObjNum 60 snap 0 seq_num 60) 2023-10-09T09:05:54.292 INFO:tasks.rados.rados.0.smithi163.stdout:153: finishing write tid 1 to smithi16367185-50 2023-10-09T09:05:54.292 INFO:tasks.rados.rados.0.smithi163.stdout:153: finishing write tid 2 to smithi16367185-50 2023-10-09T09:05:54.292 INFO:tasks.rados.rados.0.smithi163.stdout:153: finishing write tid 3 to smithi16367185-50 2023-10-09T09:05:54.292 INFO:tasks.rados.rados.0.smithi163.stdout:153: finishing write tid 4 to smithi16367185-50 2023-10-09T09:05:54.292 INFO:tasks.rados.rados.0.smithi163.stdout:153: finishing write tid 5 to smithi16367185-50 2023-10-09T09:05:54.293 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 45 (ObjNum 94 snap 16 seq_num 94) dirty exists 2023-10-09T09:05:54.293 INFO:tasks.rados.rados.0.smithi163.stdout:153: left oid 50 (ObjNum 94 snap 16 seq_num 94) 2023-10-09T09:05:54.293 INFO:tasks.rados.rados.0.smithi163.stdout:154: expect (ObjNum 72 snap 9 seq_num 72) 2023-10-09T09:05:54.427 INFO:tasks.rados.rados.0.smithi163.stdout:151: done (4 left) 2023-10-09T09:05:54.427 INFO:tasks.rados.rados.0.smithi163.stdout:152: done (3 left) 2023-10-09T09:05:54.427 INFO:tasks.rados.rados.0.smithi163.stdout:153: done (2 left) 2023-10-09T09:05:54.427 INFO:tasks.rados.rados.0.smithi163.stdout:154: done (1 left) 2023-10-09T09:05:54.428 INFO:tasks.rados.rados.0.smithi163.stdout:157: done (0 left) 2023-10-09T09:05:54.428 INFO:tasks.rados.rados.0.smithi163.stdout:158: write oid 29 current snap is 16 2023-10-09T09:05:54.428 INFO:tasks.rados.rados.0.smithi163.stdout:158: seq_num 96 ranges {413472=620140,1818036=510700,3120019=493453} 2023-10-09T09:05:54.440 INFO:tasks.rados.rados.0.smithi163.stdout:158: writing smithi16367185-29 from 413472 to 1033612 tid 1 2023-10-09T09:05:54.447 INFO:tasks.rados.rados.0.smithi163.stdout:158: writing smithi16367185-29 from 1818036 to 2328736 tid 2 2023-10-09T09:05:54.455 INFO:tasks.rados.rados.0.smithi163.stdout:158: writing smithi16367185-29 from 3120019 to 3613472 tid 3 2023-10-09T09:05:54.455 INFO:tasks.rados.rados.0.smithi163.stdout:159: read oid 41 snap -1 2023-10-09T09:05:54.456 INFO:tasks.rados.rados.0.smithi163.stdout:159: expect (ObjNum 40 snap 0 seq_num 40) 2023-10-09T09:05:54.456 INFO:tasks.rados.rados.0.smithi163.stdout:160: delete oid 28 current snap is 16 2023-10-09T09:05:54.456 INFO:tasks.rados.rados.0.smithi163.stdout:158: finishing write tid 1 to smithi16367185-29 2023-10-09T09:05:54.456 INFO:tasks.rados.rados.0.smithi163.stdout:158: finishing write tid 2 to smithi16367185-29 2023-10-09T09:05:54.458 INFO:tasks.rados.rados.0.smithi163.stdout:160: done (2 left) 2023-10-09T09:05:54.458 INFO:tasks.rados.rados.0.smithi163.stdout:161: write oid 44 current snap is 16 2023-10-09T09:05:54.458 INFO:tasks.rados.rados.0.smithi163.stdout:161: seq_num 98 ranges {447086=770638,1935000=798016,3458512=188574} 2023-10-09T09:05:54.465 INFO:tasks.rados.rados.0.smithi163.stdout:161: writing smithi16367185-44 from 447086 to 1217724 tid 1 2023-10-09T09:05:54.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:54 smithi163 bash[33912]: cluster 2023-10-09T09:05:53.129222+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2023-10-09T09:05:54.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:54 smithi163 bash[33912]: cluster 2023-10-09T09:05:53.257119+0000 mgr.y (mgr.24355) 182 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2023-10-09T09:05:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:54 smithi163 bash[27500]: cluster 2023-10-09T09:05:53.129222+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2023-10-09T09:05:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:54 smithi163 bash[27500]: cluster 2023-10-09T09:05:53.257119+0000 mgr.y (mgr.24355) 182 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2023-10-09T09:05:54.473 INFO:tasks.rados.rados.0.smithi163.stdout:161: writing smithi16367185-44 from 1935000 to 2733016 tid 2 2023-10-09T09:05:54.479 INFO:tasks.rados.rados.0.smithi163.stdout:161: writing smithi16367185-44 from 3458512 to 3647086 tid 3 2023-10-09T09:05:54.479 INFO:tasks.rados.rados.0.smithi163.stdout:162: snap_remove snap 12 2023-10-09T09:05:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:54 smithi181 bash[27712]: cluster 2023-10-09T09:05:53.129222+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2023-10-09T09:05:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:54 smithi181 bash[27712]: cluster 2023-10-09T09:05:53.257119+0000 mgr.y (mgr.24355) 182 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2023-10-09T09:05:55.146 INFO:tasks.rados.rados.0.smithi163.stdout:162: done (3 left) 2023-10-09T09:05:55.146 INFO:tasks.rados.rados.0.smithi163.stdout:163: rollback oid 15 current snap is 16 2023-10-09T09:05:55.146 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 15 to 0 2023-10-09T09:05:55.146 INFO:tasks.rados.rados.0.smithi163.stdout:158: finishing write tid 3 to smithi16367185-29 2023-10-09T09:05:55.146 INFO:tasks.rados.rados.0.smithi163.stdout:158: finishing write tid 4 to smithi16367185-29 2023-10-09T09:05:55.146 INFO:tasks.rados.rados.0.smithi163.stdout:159: expect (ObjNum 40 snap 0 seq_num 40) 2023-10-09T09:05:55.231 INFO:tasks.rados.rados.0.smithi163.stdout:158: finishing write tid 5 to smithi16367185-29 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 50 (ObjNum 96 snap 16 seq_num 96) dirty exists 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:158: left oid 29 (ObjNum 96 snap 16 seq_num 96) 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:161: finishing write tid 1 to smithi16367185-44 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:161: finishing write tid 2 to smithi16367185-44 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:161: finishing write tid 3 to smithi16367185-44 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:161: finishing write tid 4 to smithi16367185-44 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:161: finishing write tid 5 to smithi16367185-44 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 119 (ObjNum 98 snap 16 seq_num 98) dirty exists 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:161: left oid 44 (ObjNum 98 snap 16 seq_num 98) 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:163: finishing rollback tid 0 to smithi16367185-15 2023-10-09T09:05:55.232 INFO:tasks.rados.rados.0.smithi163.stdout:163: finishing rollback tid 1 to smithi16367185-15 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:163: finishing rollback tid 2 to smithi16367185-15 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 65 (ObjNum 14 snap 0 seq_num 14) dirty exists 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:158: done (3 left) 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:159: done (2 left) 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:161: done (1 left) 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:163: done (0 left) 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:164: read oid 3 snap -1 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:164: expect (ObjNum 84 snap 10 seq_num 84) 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:165: read oid 47 snap 13 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:165: expect (ObjNum 79 snap 10 seq_num 79) 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:166: read oid 27 snap 13 2023-10-09T09:05:55.233 INFO:tasks.rados.rados.0.smithi163.stdout:166: expect (ObjNum 26 snap 0 seq_num 26) 2023-10-09T09:05:55.234 INFO:tasks.rados.rados.0.smithi163.stdout:167: write oid 19 current snap is 16 2023-10-09T09:05:55.234 INFO:tasks.rados.rados.0.smithi163.stdout:167: seq_num 99 ranges {463893=562240,1593482=658027,2685935=559341} 2023-10-09T09:05:55.241 INFO:tasks.rados.rados.0.smithi163.stdout:167: writing smithi16367185-19 from 463893 to 1026133 tid 1 2023-10-09T09:05:55.249 INFO:tasks.rados.rados.0.smithi163.stdout:167: writing smithi16367185-19 from 1593482 to 2251509 tid 2 2023-10-09T09:05:55.254 INFO:tasks.rados.rados.0.smithi163.stdout:167: writing smithi16367185-19 from 2685935 to 3245276 tid 3 2023-10-09T09:05:55.254 INFO:tasks.rados.rados.0.smithi163.stdout:168: write oid 29 current snap is 16 2023-10-09T09:05:55.255 INFO:tasks.rados.rados.0.smithi163.stdout:168: seq_num 100 ranges {480700=637489,1651964=401685,2713358=770613} 2023-10-09T09:05:55.261 INFO:tasks.rados.rados.0.smithi163.stdout:168: writing smithi16367185-29 from 480700 to 1118189 tid 1 2023-10-09T09:05:55.266 INFO:tasks.rados.rados.0.smithi163.stdout:168: writing smithi16367185-29 from 1651964 to 2053649 tid 2 2023-10-09T09:05:55.275 INFO:tasks.rados.rados.0.smithi163.stdout:168: writing smithi16367185-29 from 2713358 to 3483971 tid 3 2023-10-09T09:05:55.275 INFO:tasks.rados.rados.0.smithi163.stdout:169: write oid 18 current snap is 16 2023-10-09T09:05:55.275 INFO:tasks.rados.rados.0.smithi163.stdout:169: seq_num 101 ranges {497507=712738,1710446=661696,3140781=556726} 2023-10-09T09:05:55.282 INFO:tasks.rados.rados.0.smithi163.stdout:169: writing smithi16367185-18 from 497507 to 1210245 tid 1 2023-10-09T09:05:55.289 INFO:tasks.rados.rados.0.smithi163.stdout:169: writing smithi16367185-18 from 1710446 to 2372142 tid 2 2023-10-09T09:05:55.297 INFO:tasks.rados.rados.0.smithi163.stdout:169: writing smithi16367185-18 from 3140781 to 3697507 tid 3 2023-10-09T09:05:55.297 INFO:tasks.rados.rados.0.smithi163.stdout:170: write oid 4 current snap is 16 2023-10-09T09:05:55.297 INFO:tasks.rados.rados.0.smithi163.stdout:170: seq_num 102 ranges {514314=787987,1768928=405354,2768204=509510} 2023-10-09T09:05:55.304 INFO:tasks.rados.rados.0.smithi163.stdout:170: writing smithi16367185-4 from 514314 to 1302301 tid 1 2023-10-09T09:05:55.310 INFO:tasks.rados.rados.0.smithi163.stdout:170: writing smithi16367185-4 from 1768928 to 2174282 tid 2 2023-10-09T09:05:55.316 INFO:tasks.rados.rados.0.smithi163.stdout:170: writing smithi16367185-4 from 2768204 to 3277714 tid 3 2023-10-09T09:05:55.316 INFO:tasks.rados.rados.0.smithi163.stdout:171: rollback oid 20 current snap is 16 2023-10-09T09:05:55.317 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 15 2023-10-09T09:05:55.317 INFO:tasks.rados.rados.0.smithi163.stdout:166: expect (ObjNum 26 snap 0 seq_num 26) 2023-10-09T09:05:55.401 INFO:tasks.rados.rados.0.smithi163.stdout:164: expect (ObjNum 84 snap 10 seq_num 84) 2023-10-09T09:05:55.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:55 smithi163 bash[27500]: cluster 2023-10-09T09:05:54.139265+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2023-10-09T09:05:55.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:55 smithi163 bash[27500]: audit 2023-10-09T09:05:55.130945+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.163:0/4132674764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:55.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:55 smithi163 bash[33912]: cluster 2023-10-09T09:05:54.139265+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2023-10-09T09:05:55.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:55 smithi163 bash[33912]: audit 2023-10-09T09:05:55.130945+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.163:0/4132674764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:55 smithi181 bash[27712]: cluster 2023-10-09T09:05:54.139265+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2023-10-09T09:05:55.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:55 smithi181 bash[27712]: audit 2023-10-09T09:05:55.130945+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.163:0/4132674764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:55.592 INFO:tasks.rados.rados.0.smithi163.stdout:165: expect (ObjNum 79 snap 10 seq_num 79) 2023-10-09T09:05:55.750 INFO:tasks.rados.rados.0.smithi163.stdout:167: finishing write tid 1 to smithi16367185-19 2023-10-09T09:05:55.750 INFO:tasks.rados.rados.0.smithi163.stdout:167: finishing write tid 2 to smithi16367185-19 2023-10-09T09:05:55.750 INFO:tasks.rados.rados.0.smithi163.stdout:167: finishing write tid 3 to smithi16367185-19 2023-10-09T09:05:55.750 INFO:tasks.rados.rados.0.smithi163.stdout:167: finishing write tid 4 to smithi16367185-19 2023-10-09T09:05:55.750 INFO:tasks.rados.rados.0.smithi163.stdout:167: finishing write tid 5 to smithi16367185-19 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 14 (ObjNum 99 snap 16 seq_num 99) dirty exists 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:167: left oid 19 (ObjNum 99 snap 16 seq_num 99) 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:168: finishing write tid 1 to smithi16367185-29 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:168: finishing write tid 2 to smithi16367185-29 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:168: finishing write tid 3 to smithi16367185-29 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:168: finishing write tid 4 to smithi16367185-29 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:168: finishing write tid 5 to smithi16367185-29 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 56 (ObjNum 100 snap 16 seq_num 100) dirty exists 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:168: left oid 29 (ObjNum 100 snap 16 seq_num 100) 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:169: finishing write tid 1 to smithi16367185-18 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:169: finishing write tid 2 to smithi16367185-18 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:169: finishing write tid 3 to smithi16367185-18 2023-10-09T09:05:55.751 INFO:tasks.rados.rados.0.smithi163.stdout:169: finishing write tid 4 to smithi16367185-18 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:169: finishing write tid 5 to smithi16367185-18 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 51 (ObjNum 101 snap 16 seq_num 101) dirty exists 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:169: left oid 18 (ObjNum 101 snap 16 seq_num 101) 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:170: finishing write tid 1 to smithi16367185-4 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:170: finishing write tid 2 to smithi16367185-4 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:170: finishing write tid 3 to smithi16367185-4 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:170: finishing write tid 4 to smithi16367185-4 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:170: finishing write tid 5 to smithi16367185-4 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 73 (ObjNum 102 snap 16 seq_num 102) dirty exists 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:170: left oid 4 (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:171: finishing rollback tid 0 to smithi16367185-20 2023-10-09T09:05:55.752 INFO:tasks.rados.rados.0.smithi163.stdout:171: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:05:55.753 INFO:tasks.rados.rados.0.smithi163.stdout:171: finishing rollback tid 2 to smithi16367185-20 2023-10-09T09:05:55.753 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 22 (ObjNum 88 snap 11 seq_num 88) dirty exists 2023-10-09T09:05:55.753 INFO:tasks.rados.rados.0.smithi163.stdout:164: done (7 left) 2023-10-09T09:05:55.753 INFO:tasks.rados.rados.0.smithi163.stdout:165: done (6 left) 2023-10-09T09:05:55.754 INFO:tasks.rados.rados.0.smithi163.stdout:166: done (5 left) 2023-10-09T09:05:55.754 INFO:tasks.rados.rados.0.smithi163.stdout:167: done (4 left) 2023-10-09T09:05:55.754 INFO:tasks.rados.rados.0.smithi163.stdout:168: done (3 left) 2023-10-09T09:05:55.754 INFO:tasks.rados.rados.0.smithi163.stdout:169: done (2 left) 2023-10-09T09:05:55.754 INFO:tasks.rados.rados.0.smithi163.stdout:170: done (1 left) 2023-10-09T09:05:55.754 INFO:tasks.rados.rados.0.smithi163.stdout:171: done (0 left) 2023-10-09T09:05:55.754 INFO:tasks.rados.rados.0.smithi163.stdout:172: rollback oid 27 current snap is 16 2023-10-09T09:05:55.754 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 27 to 14 2023-10-09T09:05:55.754 INFO:tasks.rados.rados.0.smithi163.stdout:173: write oid 49 current snap is 16 2023-10-09T09:05:55.754 INFO:tasks.rados.rados.0.smithi163.stdout:173: seq_num 103 ranges {531121=463236,1711057=665365,3079274=651847} 2023-10-09T09:05:55.761 INFO:tasks.rados.rados.0.smithi163.stdout:173: writing smithi16367185-49 from 531121 to 994357 tid 1 2023-10-09T09:05:55.769 INFO:tasks.rados.rados.0.smithi163.stdout:173: writing smithi16367185-49 from 1711057 to 2376422 tid 2 2023-10-09T09:05:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:173: writing smithi16367185-49 from 3079274 to 3731121 tid 3 2023-10-09T09:05:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:172: finishing rollback tid 0 to smithi16367185-27 2023-10-09T09:05:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:172: finishing rollback tid 1 to smithi16367185-27 2023-10-09T09:05:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:172: finishing rollback tid 2 to smithi16367185-27 2023-10-09T09:05:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 18 (ObjNum 26 snap 0 seq_num 26) dirty exists 2023-10-09T09:05:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:173: finishing write tid 1 to smithi16367185-49 2023-10-09T09:05:55.779 INFO:tasks.rados.rados.0.smithi163.stdout:173: finishing write tid 2 to smithi16367185-49 2023-10-09T09:05:55.779 INFO:tasks.rados.rados.0.smithi163.stdout:174: read oid 13 snap -1 2023-10-09T09:05:55.779 INFO:tasks.rados.rados.0.smithi163.stdout:174: expect deleted 2023-10-09T09:05:55.779 INFO:tasks.rados.rados.0.smithi163.stdout:172: done (2 left) 2023-10-09T09:05:55.779 INFO:tasks.rados.rados.0.smithi163.stdout:175: read oid 15 snap -1 2023-10-09T09:05:55.779 INFO:tasks.rados.rados.0.smithi163.stdout:175: expect (ObjNum 14 snap 0 seq_num 14) 2023-10-09T09:05:55.779 INFO:tasks.rados.rados.0.smithi163.stdout:176: rollback oid 35 current snap is 16 2023-10-09T09:05:55.779 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 35 to 15 2023-10-09T09:05:55.779 INFO:tasks.rados.rados.0.smithi163.stdout:177: write oid 19 current snap is 16 2023-10-09T09:05:55.779 INFO:tasks.rados.rados.0.smithi163.stdout:177: seq_num 104 ranges {547928=538485,1769539=409023,2706697=532054} 2023-10-09T09:05:55.785 INFO:tasks.rados.rados.0.smithi163.stdout:177: writing smithi16367185-19 from 547928 to 1086413 tid 1 2023-10-09T09:05:55.792 INFO:tasks.rados.rados.0.smithi163.stdout:177: writing smithi16367185-19 from 1769539 to 2178562 tid 2 2023-10-09T09:05:55.798 INFO:tasks.rados.rados.0.smithi163.stdout:177: writing smithi16367185-19 from 2706697 to 3238751 tid 3 2023-10-09T09:05:55.798 INFO:tasks.rados.rados.0.smithi163.stdout:178: read oid 22 snap 11 2023-10-09T09:05:55.798 INFO:tasks.rados.rados.0.smithi163.stdout:178: expect (ObjNum 77 snap 9 seq_num 77) 2023-10-09T09:05:55.798 INFO:tasks.rados.rados.0.smithi163.stdout:179: rollback oid 43 current snap is 16 2023-10-09T09:05:55.798 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 14 2023-10-09T09:05:55.798 INFO:tasks.rados.rados.0.smithi163.stdout:173: finishing write tid 3 to smithi16367185-49 2023-10-09T09:05:55.799 INFO:tasks.rados.rados.0.smithi163.stdout:173: finishing write tid 4 to smithi16367185-49 2023-10-09T09:05:55.799 INFO:tasks.rados.rados.0.smithi163.stdout:173: finishing write tid 5 to smithi16367185-49 2023-10-09T09:05:55.799 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 11 (ObjNum 103 snap 16 seq_num 103) dirty exists 2023-10-09T09:05:55.799 INFO:tasks.rados.rados.0.smithi163.stdout:173: left oid 49 (ObjNum 103 snap 16 seq_num 103) 2023-10-09T09:05:55.799 INFO:tasks.rados.rados.0.smithi163.stdout:176: finishing rollback tid 0 to smithi16367185-35 2023-10-09T09:05:55.799 INFO:tasks.rados.rados.0.smithi163.stdout:176: finishing rollback tid 1 to smithi16367185-35 2023-10-09T09:05:55.799 INFO:tasks.rados.rados.0.smithi163.stdout:176: finishing rollback tid 2 to smithi16367185-35 2023-10-09T09:05:55.799 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 17 (ObjNum 34 snap 0 seq_num 34) dirty exists 2023-10-09T09:05:55.799 INFO:tasks.rados.rados.0.smithi163.stdout:175: expect (ObjNum 14 snap 0 seq_num 14) 2023-10-09T09:05:55.876 INFO:tasks.rados.rados.0.smithi163.stdout:177: finishing write tid 1 to smithi16367185-19 2023-10-09T09:05:55.876 INFO:tasks.rados.rados.0.smithi163.stdout:177: finishing write tid 2 to smithi16367185-19 2023-10-09T09:05:55.876 INFO:tasks.rados.rados.0.smithi163.stdout:177: finishing write tid 3 to smithi16367185-19 2023-10-09T09:05:55.876 INFO:tasks.rados.rados.0.smithi163.stdout:177: finishing write tid 4 to smithi16367185-19 2023-10-09T09:05:55.876 INFO:tasks.rados.rados.0.smithi163.stdout:177: finishing write tid 5 to smithi16367185-19 2023-10-09T09:05:55.876 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 22 (ObjNum 104 snap 16 seq_num 104) dirty exists 2023-10-09T09:05:55.876 INFO:tasks.rados.rados.0.smithi163.stdout:177: left oid 19 (ObjNum 104 snap 16 seq_num 104) 2023-10-09T09:05:55.876 INFO:tasks.rados.rados.0.smithi163.stdout:179: finishing rollback tid 0 to smithi16367185-43 2023-10-09T09:05:55.877 INFO:tasks.rados.rados.0.smithi163.stdout:179: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:05:55.877 INFO:tasks.rados.rados.0.smithi163.stdout:179: finishing rollback tid 2 to smithi16367185-43 2023-10-09T09:05:55.877 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 88 (ObjNum 81 snap 10 seq_num 81) dirty exists 2023-10-09T09:05:55.877 INFO:tasks.rados.rados.0.smithi163.stdout:178: expect (ObjNum 77 snap 9 seq_num 77) 2023-10-09T09:05:55.983 INFO:tasks.rados.rados.0.smithi163.stdout:173: done (6 left) 2023-10-09T09:05:55.983 INFO:tasks.rados.rados.0.smithi163.stdout:174: done (5 left) 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:175: done (4 left) 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:176: done (3 left) 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:177: done (2 left) 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:178: done (1 left) 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:179: done (0 left) 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:180: rollback oid 8 current snap is 16 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 8 to 11 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:181: read oid 50 snap 14 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:181: expect (ObjNum 63 snap 1 seq_num 63) 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:182: read oid 48 snap -1 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:182: expect (ObjNum 66 snap 5 seq_num 66) 2023-10-09T09:05:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:183: write oid 16 current snap is 16 2023-10-09T09:05:55.985 INFO:tasks.rados.rados.0.smithi163.stdout:183: seq_num 105 ranges {564735=613734,1828021=669034,3134120=630615} 2023-10-09T09:05:55.995 INFO:tasks.rados.rados.0.smithi163.stdout:183: writing smithi16367185-16 from 564735 to 1178469 tid 1 2023-10-09T09:05:56.002 INFO:tasks.rados.rados.0.smithi163.stdout:183: writing smithi16367185-16 from 1828021 to 2497055 tid 2 2023-10-09T09:05:56.009 INFO:tasks.rados.rados.0.smithi163.stdout:183: writing smithi16367185-16 from 3134120 to 3764735 tid 3 2023-10-09T09:05:56.010 INFO:tasks.rados.rados.0.smithi163.stdout:184: write oid 31 current snap is 16 2023-10-09T09:05:56.010 INFO:tasks.rados.rados.0.smithi163.stdout:180: finishing rollback tid 0 to smithi16367185-8 2023-10-09T09:05:56.010 INFO:tasks.rados.rados.0.smithi163.stdout:180: finishing rollback tid 1 to smithi16367185-8 2023-10-09T09:05:56.010 INFO:tasks.rados.rados.0.smithi163.stdout:180: finishing rollback tid 2 to smithi16367185-8 2023-10-09T09:05:56.010 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 69 (ObjNum 7 snap 0 seq_num 7) dirty exists 2023-10-09T09:05:56.010 INFO:tasks.rados.rados.0.smithi163.stdout:181: expect (ObjNum 63 snap 1 seq_num 63) 2023-10-09T09:05:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:183: finishing write tid 1 to smithi16367185-16 2023-10-09T09:05:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:182: expect (ObjNum 66 snap 5 seq_num 66) 2023-10-09T09:05:56.343 INFO:tasks.rados.rados.0.smithi163.stdout:183: finishing write tid 2 to smithi16367185-16 2023-10-09T09:05:56.343 INFO:tasks.rados.rados.0.smithi163.stdout:183: finishing write tid 3 to smithi16367185-16 2023-10-09T09:05:56.343 INFO:tasks.rados.rados.0.smithi163.stdout:183: finishing write tid 4 to smithi16367185-16 2023-10-09T09:05:56.343 INFO:tasks.rados.rados.0.smithi163.stdout:183: finishing write tid 5 to smithi16367185-16 2023-10-09T09:05:56.343 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 77 (ObjNum 105 snap 16 seq_num 105) dirty exists 2023-10-09T09:05:56.344 INFO:tasks.rados.rados.0.smithi163.stdout:183: left oid 16 (ObjNum 105 snap 16 seq_num 105) 2023-10-09T09:05:56.344 INFO:tasks.rados.rados.0.smithi163.stdout:184: seq_num 106 ranges {581542=688983,1886503=412692,2761543=670951} 2023-10-09T09:05:56.355 INFO:tasks.rados.rados.0.smithi163.stdout:184: writing smithi16367185-31 from 581542 to 1270525 tid 1 2023-10-09T09:05:56.362 INFO:tasks.rados.rados.0.smithi163.stdout:184: writing smithi16367185-31 from 1886503 to 2299195 tid 2 2023-10-09T09:05:56.369 INFO:tasks.rados.rados.0.smithi163.stdout:184: writing smithi16367185-31 from 2761543 to 3432494 tid 3 2023-10-09T09:05:56.369 INFO:tasks.rados.rados.0.smithi163.stdout:180: done (4 left) 2023-10-09T09:05:56.369 INFO:tasks.rados.rados.0.smithi163.stdout:181: done (3 left) 2023-10-09T09:05:56.369 INFO:tasks.rados.rados.0.smithi163.stdout:182: done (2 left) 2023-10-09T09:05:56.369 INFO:tasks.rados.rados.0.smithi163.stdout:183: done (1 left) 2023-10-09T09:05:56.369 INFO:tasks.rados.rados.0.smithi163.stdout:185: delete oid 2 current snap is 16 2023-10-09T09:05:56.369 INFO:tasks.rados.rados.0.smithi163.stdout:184: finishing write tid 1 to smithi16367185-31 2023-10-09T09:05:56.369 INFO:tasks.rados.rados.0.smithi163.stdout:184: finishing write tid 2 to smithi16367185-31 2023-10-09T09:05:56.372 INFO:tasks.rados.rados.0.smithi163.stdout:185: done (1 left) 2023-10-09T09:05:56.373 INFO:tasks.rados.rados.0.smithi163.stdout:186: delete oid 35 current snap is 16 2023-10-09T09:05:56.373 INFO:tasks.rados.rados.0.smithi163.stdout:184: finishing write tid 3 to smithi16367185-31 2023-10-09T09:05:56.373 INFO:tasks.rados.rados.0.smithi163.stdout:184: finishing write tid 4 to smithi16367185-31 2023-10-09T09:05:56.373 INFO:tasks.rados.rados.0.smithi163.stdout:184: finishing write tid 5 to smithi16367185-31 2023-10-09T09:05:56.374 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 43 (ObjNum 106 snap 16 seq_num 106) dirty exists 2023-10-09T09:05:56.374 INFO:tasks.rados.rados.0.smithi163.stdout:184: left oid 31 (ObjNum 106 snap 16 seq_num 106) 2023-10-09T09:05:56.375 INFO:tasks.rados.rados.0.smithi163.stdout:184: done (1 left) 2023-10-09T09:05:56.375 INFO:tasks.rados.rados.0.smithi163.stdout:186: done (0 left) 2023-10-09T09:05:56.375 INFO:tasks.rados.rados.0.smithi163.stdout:187: snap_create 2023-10-09T09:05:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:56 smithi163 bash[27500]: cluster 2023-10-09T09:05:55.143415+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2023-10-09T09:05:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:56 smithi163 bash[27500]: cluster 2023-10-09T09:05:55.257757+0000 mgr.y (mgr.24355) 183 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2023-10-09T09:05:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:56 smithi163 bash[33912]: cluster 2023-10-09T09:05:55.143415+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2023-10-09T09:05:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:56 smithi163 bash[33912]: cluster 2023-10-09T09:05:55.257757+0000 mgr.y (mgr.24355) 183 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2023-10-09T09:05:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:56 smithi181 bash[27712]: cluster 2023-10-09T09:05:55.143415+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2023-10-09T09:05:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:56 smithi181 bash[27712]: cluster 2023-10-09T09:05:55.257757+0000 mgr.y (mgr.24355) 183 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2023-10-09T09:05:57.083 INFO:tasks.rados.rados.0.smithi163.stdout:187: done (0 left) 2023-10-09T09:05:57.083 INFO:tasks.rados.rados.0.smithi163.stdout:188: read oid 42 snap -1 2023-10-09T09:05:57.083 INFO:tasks.rados.rados.0.smithi163.stdout:188: expect (ObjNum 87 snap 11 seq_num 87) 2023-10-09T09:05:57.084 INFO:tasks.rados.rados.0.smithi163.stdout:189: rollback oid 30 current snap is 17 2023-10-09T09:05:57.084 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 14 2023-10-09T09:05:57.084 INFO:tasks.rados.rados.0.smithi163.stdout:190: delete oid 20 current snap is 17 2023-10-09T09:05:57.092 INFO:tasks.rados.rados.0.smithi163.stdout:190: done (2 left) 2023-10-09T09:05:57.092 INFO:tasks.rados.rados.0.smithi163.stdout:191: snap_remove snap 0 2023-10-09T09:05:58.098 INFO:tasks.rados.rados.0.smithi163.stdout:191: done (2 left) 2023-10-09T09:05:58.098 INFO:tasks.rados.rados.0.smithi163.stdout:192: snap_remove snap 15 2023-10-09T09:05:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[33912]: cluster 2023-10-09T09:05:57.083019+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2023-10-09T09:05:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[33912]: cluster 2023-10-09T09:05:57.258418+0000 mgr.y (mgr.24355) 184 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:05:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[33912]: audit 2023-10-09T09:05:57.311538+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[33912]: cephadm 2023-10-09T09:05:57.312132+0000 mgr.y (mgr.24355) 185 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T09:05:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[33912]: cephadm 2023-10-09T09:05:57.312286+0000 mgr.y (mgr.24355) 186 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T09:05:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[33912]: audit 2023-10-09T09:05:57.314365+0000 mon.a (mon.0) 805 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[33912]: audit 2023-10-09T09:05:57.323659+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[33912]: cephadm 2023-10-09T09:05:57.324845+0000 mgr.y (mgr.24355) 187 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T09:05:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[33912]: cephadm 2023-10-09T09:05:57.543496+0000 mgr.y (mgr.24355) 188 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi181 2023-10-09T09:05:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[33912]: audit 2023-10-09T09:05:57.547048+0000 mon.a (mon.0) 807 : audit [DBG] from='client.? 172.21.15.163:0/3368942551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[27500]: cluster 2023-10-09T09:05:57.083019+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2023-10-09T09:05:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[27500]: cluster 2023-10-09T09:05:57.258418+0000 mgr.y (mgr.24355) 184 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:05:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[27500]: audit 2023-10-09T09:05:57.311538+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[27500]: cephadm 2023-10-09T09:05:57.312132+0000 mgr.y (mgr.24355) 185 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T09:05:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[27500]: cephadm 2023-10-09T09:05:57.312286+0000 mgr.y (mgr.24355) 186 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T09:05:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[27500]: audit 2023-10-09T09:05:57.314365+0000 mon.a (mon.0) 805 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[27500]: audit 2023-10-09T09:05:57.323659+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[27500]: cephadm 2023-10-09T09:05:57.324845+0000 mgr.y (mgr.24355) 187 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T09:05:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[27500]: cephadm 2023-10-09T09:05:57.543496+0000 mgr.y (mgr.24355) 188 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi181 2023-10-09T09:05:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:58 smithi163 bash[27500]: audit 2023-10-09T09:05:57.547048+0000 mon.a (mon.0) 807 : audit [DBG] from='client.? 172.21.15.163:0/3368942551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:58.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[27712]: cluster 2023-10-09T09:05:57.083019+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2023-10-09T09:05:58.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[27712]: cluster 2023-10-09T09:05:57.258418+0000 mgr.y (mgr.24355) 184 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:05:58.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[27712]: audit 2023-10-09T09:05:57.311538+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:58.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[27712]: cephadm 2023-10-09T09:05:57.312132+0000 mgr.y (mgr.24355) 185 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T09:05:58.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[27712]: cephadm 2023-10-09T09:05:57.312286+0000 mgr.y (mgr.24355) 186 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T09:05:58.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[27712]: audit 2023-10-09T09:05:57.314365+0000 mon.a (mon.0) 805 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:05:58.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[27712]: audit 2023-10-09T09:05:57.323659+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:05:58.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[27712]: cephadm 2023-10-09T09:05:57.324845+0000 mgr.y (mgr.24355) 187 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T09:05:58.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[27712]: cephadm 2023-10-09T09:05:57.543496+0000 mgr.y (mgr.24355) 188 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi181 2023-10-09T09:05:58.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[27712]: audit 2023-10-09T09:05:57.547048+0000 mon.a (mon.0) 807 : audit [DBG] from='client.? 172.21.15.163:0/3368942551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:05:58.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:05:58 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:05:59.112 INFO:tasks.rados.rados.0.smithi163.stdout:192: done (2 left) 2023-10-09T09:05:59.112 INFO:tasks.rados.rados.0.smithi163.stdout:193: read oid 11 snap 11 2023-10-09T09:05:59.113 INFO:tasks.rados.rados.0.smithi163.stdout:193: expect (ObjNum 10 snap 0 seq_num 10) 2023-10-09T09:05:59.113 INFO:tasks.rados.rados.0.smithi163.stdout:194: snap_create 2023-10-09T09:05:59.113 INFO:tasks.rados.rados.0.smithi163.stdout:189: finishing rollback tid 0 to smithi16367185-30 2023-10-09T09:05:59.113 INFO:tasks.rados.rados.0.smithi163.stdout:189: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:05:59.113 INFO:tasks.rados.rados.0.smithi163.stdout:189: finishing rollback tid 2 to smithi16367185-30 2023-10-09T09:05:59.113 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 55 (ObjNum 29 snap 0 seq_num 29) dirty exists 2023-10-09T09:05:59.114 INFO:tasks.rados.rados.0.smithi163.stdout:188: expect (ObjNum 87 snap 11 seq_num 87) 2023-10-09T09:05:59.268 INFO:tasks.rados.rados.0.smithi163.stdout:193: expect (ObjNum 10 snap 0 seq_num 10) 2023-10-09T09:05:59.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:05:59 smithi163 bash[33912]: cluster 2023-10-09T09:05:58.098486+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2023-10-09T09:05:59.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:05:59 smithi163 bash[27500]: cluster 2023-10-09T09:05:58.098486+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2023-10-09T09:05:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:05:59 smithi181 bash[27712]: cluster 2023-10-09T09:05:58.098486+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2023-10-09T09:06:00.120 INFO:tasks.rados.rados.0.smithi163.stdout:188: done (3 left) 2023-10-09T09:06:00.120 INFO:tasks.rados.rados.0.smithi163.stdout:189: done (2 left) 2023-10-09T09:06:00.120 INFO:tasks.rados.rados.0.smithi163.stdout:193: done (1 left) 2023-10-09T09:06:00.120 INFO:tasks.rados.rados.0.smithi163.stdout:194: done (0 left) 2023-10-09T09:06:00.121 INFO:tasks.rados.rados.0.smithi163.stdout:195: delete oid 45 current snap is 18 2023-10-09T09:06:00.125 INFO:tasks.rados.rados.0.smithi163.stdout:195: done (0 left) 2023-10-09T09:06:00.125 INFO:tasks.rados.rados.0.smithi163.stdout:196: read oid 12 snap 13 2023-10-09T09:06:00.125 INFO:tasks.rados.rados.0.smithi163.stdout:196: expect (ObjNum 68 snap 7 seq_num 68) 2023-10-09T09:06:00.125 INFO:tasks.rados.rados.0.smithi163.stdout:197: rollback oid 44 current snap is 18 2023-10-09T09:06:00.125 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 44 to 17 2023-10-09T09:06:00.125 INFO:tasks.rados.rados.0.smithi163.stdout:198: snap_remove snap 14 2023-10-09T09:06:00.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:05:59 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:05:59] "GET /metrics HTTP/1.1" 200 194670 "" "Prometheus/2.33.4" 2023-10-09T09:06:00.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:00 smithi163 bash[27500]: cluster 2023-10-09T09:05:59.110979+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2023-10-09T09:06:00.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:00 smithi163 bash[27500]: cluster 2023-10-09T09:05:59.258898+0000 mgr.y (mgr.24355) 189 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 145 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.1 MiB/s wr, 38 op/s 2023-10-09T09:06:00.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:00 smithi163 bash[27500]: audit 2023-10-09T09:05:59.955230+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.163:0/1886197060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:00 smithi163 bash[33912]: cluster 2023-10-09T09:05:59.110979+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2023-10-09T09:06:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:00 smithi163 bash[33912]: cluster 2023-10-09T09:05:59.258898+0000 mgr.y (mgr.24355) 189 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 145 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.1 MiB/s wr, 38 op/s 2023-10-09T09:06:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:00 smithi163 bash[33912]: audit 2023-10-09T09:05:59.955230+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.163:0/1886197060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:00 smithi181 bash[27712]: cluster 2023-10-09T09:05:59.110979+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2023-10-09T09:06:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:00 smithi181 bash[27712]: cluster 2023-10-09T09:05:59.258898+0000 mgr.y (mgr.24355) 189 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 145 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.1 MiB/s wr, 38 op/s 2023-10-09T09:06:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:00 smithi181 bash[27712]: audit 2023-10-09T09:05:59.955230+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.163:0/1886197060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:01.129 INFO:tasks.rados.rados.0.smithi163.stdout:198: done (2 left) 2023-10-09T09:06:01.129 INFO:tasks.rados.rados.0.smithi163.stdout:199: delete oid 24 current snap is 18 2023-10-09T09:06:01.129 INFO:tasks.rados.rados.0.smithi163.stdout:197: finishing rollback tid 0 to smithi16367185-44 2023-10-09T09:06:01.129 INFO:tasks.rados.rados.0.smithi163.stdout:197: finishing rollback tid 1 to smithi16367185-44 2023-10-09T09:06:01.129 INFO:tasks.rados.rados.0.smithi163.stdout:197: finishing rollback tid 2 to smithi16367185-44 2023-10-09T09:06:01.129 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 141 (ObjNum 98 snap 16 seq_num 98) dirty exists 2023-10-09T09:06:01.130 INFO:tasks.rados.rados.0.smithi163.stdout:196: expect (ObjNum 68 snap 7 seq_num 68) 2023-10-09T09:06:01.325 INFO:tasks.rados.rados.0.smithi163.stdout:196: done (2 left) 2023-10-09T09:06:01.325 INFO:tasks.rados.rados.0.smithi163.stdout:197: done (1 left) 2023-10-09T09:06:01.325 INFO:tasks.rados.rados.0.smithi163.stdout:199: done (0 left) 2023-10-09T09:06:01.325 INFO:tasks.rados.rados.0.smithi163.stdout:200: read oid 9 snap 13 2023-10-09T09:06:01.325 INFO:tasks.rados.rados.0.smithi163.stdout:200: expect (ObjNum 8 snap 0 seq_num 8) 2023-10-09T09:06:01.325 INFO:tasks.rados.rados.0.smithi163.stdout:201: rollback oid 45 current snap is 18 2023-10-09T09:06:01.326 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 45 to 16 2023-10-09T09:06:01.326 INFO:tasks.rados.rados.0.smithi163.stdout:202: rollback oid 14 current snap is 18 2023-10-09T09:06:01.326 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 13 2023-10-09T09:06:01.326 INFO:tasks.rados.rados.0.smithi163.stdout:203: write oid 34 current snap is 18 2023-10-09T09:06:01.326 INFO:tasks.rados.rados.0.smithi163.stdout:203: seq_num 112 ranges {682384=456830,1721042=423699,2809728=454936,3731357=151027} 2023-10-09T09:06:01.335 INFO:tasks.rados.rados.0.smithi163.stdout:203: writing smithi16367185-34 from 682384 to 1139214 tid 1 2023-10-09T09:06:01.342 INFO:tasks.rados.rados.0.smithi163.stdout:203: writing smithi16367185-34 from 1721042 to 2144741 tid 2 2023-10-09T09:06:01.349 INFO:tasks.rados.rados.0.smithi163.stdout:203: writing smithi16367185-34 from 2809728 to 3264664 tid 3 2023-10-09T09:06:01.352 INFO:tasks.rados.rados.0.smithi163.stdout:203: writing smithi16367185-34 from 3731357 to 3882384 tid 4 2023-10-09T09:06:01.352 INFO:tasks.rados.rados.0.smithi163.stdout:204: snap_create 2023-10-09T09:06:01.352 INFO:tasks.rados.rados.0.smithi163.stdout:201: finishing rollback tid 1 to smithi16367185-45 2023-10-09T09:06:01.353 INFO:tasks.rados.rados.0.smithi163.stdout:201: finishing rollback tid 2 to smithi16367185-45 2023-10-09T09:06:01.353 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 147 (ObjNum 44 snap 0 seq_num 44) dirty exists 2023-10-09T09:06:01.353 INFO:tasks.rados.rados.0.smithi163.stdout:202: finishing rollback tid 0 to smithi16367185-14 2023-10-09T09:06:01.353 INFO:tasks.rados.rados.0.smithi163.stdout:202: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:06:01.353 INFO:tasks.rados.rados.0.smithi163.stdout:202: finishing rollback tid 2 to smithi16367185-14 2023-10-09T09:06:01.353 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 110 (ObjNum 75 snap 9 seq_num 75) dirty exists 2023-10-09T09:06:01.353 INFO:tasks.rados.rados.0.smithi163.stdout:200: expect (ObjNum 8 snap 0 seq_num 8) 2023-10-09T09:06:01.438 INFO:tasks.rados.rados.0.smithi163.stdout:203: finishing write tid 1 to smithi16367185-34 2023-10-09T09:06:01.439 INFO:tasks.rados.rados.0.smithi163.stdout:203: finishing write tid 2 to smithi16367185-34 2023-10-09T09:06:01.439 INFO:tasks.rados.rados.0.smithi163.stdout:203: finishing write tid 3 to smithi16367185-34 2023-10-09T09:06:01.439 INFO:tasks.rados.rados.0.smithi163.stdout:203: finishing write tid 4 to smithi16367185-34 2023-10-09T09:06:01.439 INFO:tasks.rados.rados.0.smithi163.stdout:203: finishing write tid 5 to smithi16367185-34 2023-10-09T09:06:01.439 INFO:tasks.rados.rados.0.smithi163.stdout:203: finishing write tid 6 to smithi16367185-34 2023-10-09T09:06:01.439 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 73 (ObjNum 112 snap 18 seq_num 112) dirty exists 2023-10-09T09:06:01.439 INFO:tasks.rados.rados.0.smithi163.stdout:203: left oid 34 (ObjNum 112 snap 18 seq_num 112) 2023-10-09T09:06:01.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:01 smithi163 bash[33912]: cluster 2023-10-09T09:06:00.119662+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2023-10-09T09:06:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:01 smithi163 bash[27500]: cluster 2023-10-09T09:06:00.119662+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2023-10-09T09:06:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:01 smithi181 bash[27712]: cluster 2023-10-09T09:06:00.119662+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2023-10-09T09:06:02.087 INFO:tasks.rados.rados.0.smithi163.stdout:200: done (4 left) 2023-10-09T09:06:02.087 INFO:tasks.rados.rados.0.smithi163.stdout:201: done (3 left) 2023-10-09T09:06:02.088 INFO:tasks.rados.rados.0.smithi163.stdout:202: done (2 left) 2023-10-09T09:06:02.088 INFO:tasks.rados.rados.0.smithi163.stdout:203: done (1 left) 2023-10-09T09:06:02.088 INFO:tasks.rados.rados.0.smithi163.stdout:204: done (0 left) 2023-10-09T09:06:02.088 INFO:tasks.rados.rados.0.smithi163.stdout:205: write oid 49 current snap is 19 2023-10-09T09:06:02.088 INFO:tasks.rados.rados.0.smithi163.stdout:205: seq_num 113 ranges {699191=532079,1779524=567357,2837151=666208} 2023-10-09T09:06:02.096 INFO:tasks.rados.rados.0.smithi163.stdout:205: writing smithi16367185-49 from 699191 to 1231270 tid 1 2023-10-09T09:06:02.102 INFO:tasks.rados.rados.0.smithi163.stdout:205: writing smithi16367185-49 from 1779524 to 2346881 tid 2 2023-10-09T09:06:02.109 INFO:tasks.rados.rados.0.smithi163.stdout:205: writing smithi16367185-49 from 2837151 to 3503359 tid 3 2023-10-09T09:06:02.109 INFO:tasks.rados.rados.0.smithi163.stdout:206: snap_remove snap 18 2023-10-09T09:06:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:02 smithi163 bash[33912]: cluster 2023-10-09T09:06:01.129093+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2023-10-09T09:06:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:02 smithi163 bash[33912]: cluster 2023-10-09T09:06:01.259522+0000 mgr.y (mgr.24355) 190 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 145 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.2 MiB/s wr, 39 op/s 2023-10-09T09:06:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:02 smithi163 bash[33912]: cluster 2023-10-09T09:06:02.085485+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2023-10-09T09:06:02.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:02 smithi163 bash[27500]: cluster 2023-10-09T09:06:01.129093+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2023-10-09T09:06:02.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:02 smithi163 bash[27500]: cluster 2023-10-09T09:06:01.259522+0000 mgr.y (mgr.24355) 190 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 145 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.2 MiB/s wr, 39 op/s 2023-10-09T09:06:02.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:02 smithi163 bash[27500]: cluster 2023-10-09T09:06:02.085485+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2023-10-09T09:06:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:02 smithi181 bash[27712]: cluster 2023-10-09T09:06:01.129093+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2023-10-09T09:06:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:02 smithi181 bash[27712]: cluster 2023-10-09T09:06:01.259522+0000 mgr.y (mgr.24355) 190 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 145 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.2 MiB/s wr, 39 op/s 2023-10-09T09:06:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:02 smithi181 bash[27712]: cluster 2023-10-09T09:06:02.085485+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2023-10-09T09:06:03.099 INFO:tasks.rados.rados.0.smithi163.stdout:206: done (1 left) 2023-10-09T09:06:03.099 INFO:tasks.rados.rados.0.smithi163.stdout:207: rollback oid 12 current snap is 19 2023-10-09T09:06:03.099 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 12 to 11 2023-10-09T09:06:03.099 INFO:tasks.rados.rados.0.smithi163.stdout:208: snap_remove snap 13 2023-10-09T09:06:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:03 smithi163 bash[27500]: audit 2023-10-09T09:06:02.363239+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.163:0/174475251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:03 smithi163 bash[27500]: cluster 2023-10-09T09:06:03.098315+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2023-10-09T09:06:03.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:03 smithi163 bash[33912]: audit 2023-10-09T09:06:02.363239+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.163:0/174475251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:03.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:03 smithi163 bash[33912]: cluster 2023-10-09T09:06:03.098315+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2023-10-09T09:06:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:03 smithi181 bash[27712]: audit 2023-10-09T09:06:02.363239+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.163:0/174475251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:03 smithi181 bash[27712]: cluster 2023-10-09T09:06:03.098315+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2023-10-09T09:06:04.102 INFO:tasks.rados.rados.0.smithi163.stdout:208: done (2 left) 2023-10-09T09:06:04.103 INFO:tasks.rados.rados.0.smithi163.stdout:209: delete oid 18 current snap is 19 2023-10-09T09:06:04.103 INFO:tasks.rados.rados.0.smithi163.stdout:205: finishing write tid 1 to smithi16367185-49 2023-10-09T09:06:04.103 INFO:tasks.rados.rados.0.smithi163.stdout:205: finishing write tid 2 to smithi16367185-49 2023-10-09T09:06:04.103 INFO:tasks.rados.rados.0.smithi163.stdout:205: finishing write tid 3 to smithi16367185-49 2023-10-09T09:06:04.103 INFO:tasks.rados.rados.0.smithi163.stdout:205: finishing write tid 4 to smithi16367185-49 2023-10-09T09:06:04.103 INFO:tasks.rados.rados.0.smithi163.stdout:205: finishing write tid 5 to smithi16367185-49 2023-10-09T09:06:04.103 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 22 (ObjNum 113 snap 19 seq_num 113) dirty exists 2023-10-09T09:06:04.103 INFO:tasks.rados.rados.0.smithi163.stdout:205: left oid 49 (ObjNum 113 snap 19 seq_num 113) 2023-10-09T09:06:04.104 INFO:tasks.rados.rados.0.smithi163.stdout:207: finishing rollback tid 0 to smithi16367185-12 2023-10-09T09:06:04.104 INFO:tasks.rados.rados.0.smithi163.stdout:207: finishing rollback tid 1 to smithi16367185-12 2023-10-09T09:06:04.104 INFO:tasks.rados.rados.0.smithi163.stdout:207: finishing rollback tid 2 to smithi16367185-12 2023-10-09T09:06:04.104 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 70 (ObjNum 68 snap 7 seq_num 68) dirty exists 2023-10-09T09:06:04.110 INFO:tasks.rados.rados.0.smithi163.stdout:205: done (2 left) 2023-10-09T09:06:04.110 INFO:tasks.rados.rados.0.smithi163.stdout:207: done (1 left) 2023-10-09T09:06:04.110 INFO:tasks.rados.rados.0.smithi163.stdout:209: done (0 left) 2023-10-09T09:06:04.110 INFO:tasks.rados.rados.0.smithi163.stdout:210: read oid 19 snap -1 2023-10-09T09:06:04.111 INFO:tasks.rados.rados.0.smithi163.stdout:210: expect (ObjNum 104 snap 16 seq_num 104) 2023-10-09T09:06:04.111 INFO:tasks.rados.rados.0.smithi163.stdout:211: snap_remove snap 11 2023-10-09T09:06:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:04 smithi163 bash[27500]: cluster 2023-10-09T09:06:03.260723+0000 mgr.y (mgr.24355) 191 : cluster [DBG] pgmap v201: 89 pgs: 89 active+clean; 151 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 795 KiB/s wr, 12 op/s 2023-10-09T09:06:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:04 smithi163 bash[27500]: cluster 2023-10-09T09:06:04.101761+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2023-10-09T09:06:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:04 smithi163 bash[33912]: cluster 2023-10-09T09:06:03.260723+0000 mgr.y (mgr.24355) 191 : cluster [DBG] pgmap v201: 89 pgs: 89 active+clean; 151 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 795 KiB/s wr, 12 op/s 2023-10-09T09:06:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:04 smithi163 bash[33912]: cluster 2023-10-09T09:06:04.101761+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2023-10-09T09:06:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:04 smithi181 bash[27712]: cluster 2023-10-09T09:06:03.260723+0000 mgr.y (mgr.24355) 191 : cluster [DBG] pgmap v201: 89 pgs: 89 active+clean; 151 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 795 KiB/s wr, 12 op/s 2023-10-09T09:06:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:04 smithi181 bash[27712]: cluster 2023-10-09T09:06:04.101761+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2023-10-09T09:06:05.104 INFO:tasks.rados.rados.0.smithi163.stdout:211: done (1 left) 2023-10-09T09:06:05.104 INFO:tasks.rados.rados.0.smithi163.stdout:212: delete oid 2 current snap is 19 2023-10-09T09:06:05.104 INFO:tasks.rados.rados.0.smithi163.stdout:210: expect (ObjNum 104 snap 16 seq_num 104) 2023-10-09T09:06:05.351 INFO:tasks.rados.rados.0.smithi163.stdout:210: done (1 left) 2023-10-09T09:06:05.352 INFO:tasks.rados.rados.0.smithi163.stdout:212: done (0 left) 2023-10-09T09:06:05.352 INFO:tasks.rados.rados.0.smithi163.stdout:213: snap_create 2023-10-09T09:06:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:05 smithi163 bash[33912]: audit 2023-10-09T09:06:04.769120+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.163:0/313917286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:05 smithi163 bash[33912]: cluster 2023-10-09T09:06:05.103711+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2023-10-09T09:06:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:05 smithi163 bash[27500]: audit 2023-10-09T09:06:04.769120+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.163:0/313917286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:05 smithi163 bash[27500]: cluster 2023-10-09T09:06:05.103711+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2023-10-09T09:06:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:05 smithi181 bash[27712]: audit 2023-10-09T09:06:04.769120+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.163:0/313917286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:05 smithi181 bash[27712]: cluster 2023-10-09T09:06:05.103711+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2023-10-09T09:06:06.158 INFO:tasks.rados.rados.0.smithi163.stdout:213: done (0 left) 2023-10-09T09:06:06.158 INFO:tasks.rados.rados.0.smithi163.stdout:214: delete oid 46 current snap is 20 2023-10-09T09:06:06.161 INFO:tasks.rados.rados.0.smithi163.stdout:214: done (0 left) 2023-10-09T09:06:06.161 INFO:tasks.rados.rados.0.smithi163.stdout:215: write oid 16 current snap is 20 2023-10-09T09:06:06.161 INFO:tasks.rados.rados.0.smithi163.stdout:215: seq_num 117 ranges {766419=549428,2013452=574695,3346843=427649} 2023-10-09T09:06:06.170 INFO:tasks.rados.rados.0.smithi163.stdout:215: writing smithi16367185-16 from 766419 to 1315847 tid 1 2023-10-09T09:06:06.178 INFO:tasks.rados.rados.0.smithi163.stdout:215: writing smithi16367185-16 from 2013452 to 2588147 tid 2 2023-10-09T09:06:06.185 INFO:tasks.rados.rados.0.smithi163.stdout:215: writing smithi16367185-16 from 3346843 to 3774492 tid 3 2023-10-09T09:06:06.185 INFO:tasks.rados.rados.0.smithi163.stdout:216: read oid 30 snap -1 2023-10-09T09:06:06.185 INFO:tasks.rados.rados.0.smithi163.stdout:216: expect (ObjNum 29 snap 0 seq_num 29) 2023-10-09T09:06:06.185 INFO:tasks.rados.rados.0.smithi163.stdout:217: snap_remove snap 16 2023-10-09T09:06:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:06 smithi163 bash[27500]: cluster 2023-10-09T09:06:05.261398+0000 mgr.y (mgr.24355) 192 : cluster [DBG] pgmap v204: 89 pgs: 89 active+clean; 151 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 795 KiB/s wr, 12 op/s 2023-10-09T09:06:06.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:06 smithi163 bash[33912]: cluster 2023-10-09T09:06:05.261398+0000 mgr.y (mgr.24355) 192 : cluster [DBG] pgmap v204: 89 pgs: 89 active+clean; 151 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 795 KiB/s wr, 12 op/s 2023-10-09T09:06:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:06 smithi181 bash[27712]: cluster 2023-10-09T09:06:05.261398+0000 mgr.y (mgr.24355) 192 : cluster [DBG] pgmap v204: 89 pgs: 89 active+clean; 151 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 795 KiB/s wr, 12 op/s 2023-10-09T09:06:07.089 INFO:tasks.rados.rados.0.smithi163.stdout:217: done (2 left) 2023-10-09T09:06:07.090 INFO:tasks.rados.rados.0.smithi163.stdout:218: snap_remove snap 17 2023-10-09T09:06:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:07 smithi163 bash[33912]: cluster 2023-10-09T09:06:06.157674+0000 mon.a (mon.0) 817 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2023-10-09T09:06:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:07 smithi163 bash[33912]: cluster 2023-10-09T09:06:07.088639+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2023-10-09T09:06:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:07 smithi163 bash[27500]: cluster 2023-10-09T09:06:06.157674+0000 mon.a (mon.0) 817 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2023-10-09T09:06:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:07 smithi163 bash[27500]: cluster 2023-10-09T09:06:07.088639+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2023-10-09T09:06:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:07 smithi181 bash[27712]: cluster 2023-10-09T09:06:06.157674+0000 mon.a (mon.0) 817 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2023-10-09T09:06:07.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:07 smithi181 bash[27712]: cluster 2023-10-09T09:06:07.088639+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2023-10-09T09:06:08.092 INFO:tasks.rados.rados.0.smithi163.stdout:218: done (2 left) 2023-10-09T09:06:08.092 INFO:tasks.rados.rados.0.smithi163.stdout:219: write oid 18 current snap is 20 2023-10-09T09:06:08.092 INFO:tasks.rados.rados.0.smithi163.stdout:219: seq_num 118 ranges {783226=624677,2071934=434706,2974266=638921} 2023-10-09T09:06:08.103 INFO:tasks.rados.rados.0.smithi163.stdout:219: writing smithi16367185-18 from 783226 to 1407903 tid 1 2023-10-09T09:06:08.110 INFO:tasks.rados.rados.0.smithi163.stdout:219: writing smithi16367185-18 from 2071934 to 2506640 tid 2 2023-10-09T09:06:08.117 INFO:tasks.rados.rados.0.smithi163.stdout:219: writing smithi16367185-18 from 2974266 to 3613187 tid 3 2023-10-09T09:06:08.117 INFO:tasks.rados.rados.0.smithi163.stdout:220: rollback oid 7 current snap is 20 2023-10-09T09:06:08.118 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 19 2023-10-09T09:06:08.118 INFO:tasks.rados.rados.0.smithi163.stdout:215: finishing write tid 1 to smithi16367185-16 2023-10-09T09:06:08.118 INFO:tasks.rados.rados.0.smithi163.stdout:215: finishing write tid 2 to smithi16367185-16 2023-10-09T09:06:08.118 INFO:tasks.rados.rados.0.smithi163.stdout:221: snap_create 2023-10-09T09:06:08.118 INFO:tasks.rados.rados.0.smithi163.stdout:216: expect (ObjNum 29 snap 0 seq_num 29) 2023-10-09T09:06:08.206 INFO:tasks.rados.rados.0.smithi163.stdout:215: finishing write tid 3 to smithi16367185-16 2023-10-09T09:06:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:215: finishing write tid 4 to smithi16367185-16 2023-10-09T09:06:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:215: finishing write tid 5 to smithi16367185-16 2023-10-09T09:06:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 106 (ObjNum 117 snap 20 seq_num 117) dirty exists 2023-10-09T09:06:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:215: left oid 16 (ObjNum 117 snap 20 seq_num 117) 2023-10-09T09:06:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:219: finishing write tid 1 to smithi16367185-18 2023-10-09T09:06:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:219: finishing write tid 2 to smithi16367185-18 2023-10-09T09:06:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:220: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:06:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 0 (ObjNum 699112480 snap 21875 seq_num 699140928) dirty dne 2023-10-09T09:06:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:219: finishing write tid 3 to smithi16367185-18 2023-10-09T09:06:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:219: finishing write tid 4 to smithi16367185-18 2023-10-09T09:06:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:219: finishing write tid 5 to smithi16367185-18 2023-10-09T09:06:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 89 (ObjNum 118 snap 20 seq_num 118) dirty exists 2023-10-09T09:06:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:219: left oid 18 (ObjNum 118 snap 20 seq_num 118) 2023-10-09T09:06:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:08 smithi163 bash[27500]: audit 2023-10-09T09:06:07.185943+0000 mon.a (mon.0) 819 : audit [DBG] from='client.? 172.21.15.163:0/3631120158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:08 smithi163 bash[27500]: cluster 2023-10-09T09:06:07.262148+0000 mgr.y (mgr.24355) 193 : cluster [DBG] pgmap v207: 89 pgs: 89 active+clean; 151 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:08 smithi163 bash[27500]: cluster 2023-10-09T09:06:08.092270+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2023-10-09T09:06:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:08 smithi163 bash[33912]: audit 2023-10-09T09:06:07.185943+0000 mon.a (mon.0) 819 : audit [DBG] from='client.? 172.21.15.163:0/3631120158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:08 smithi163 bash[33912]: cluster 2023-10-09T09:06:07.262148+0000 mgr.y (mgr.24355) 193 : cluster [DBG] pgmap v207: 89 pgs: 89 active+clean; 151 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:08 smithi163 bash[33912]: cluster 2023-10-09T09:06:08.092270+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2023-10-09T09:06:08.553 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:08 smithi181 bash[27712]: audit 2023-10-09T09:06:07.185943+0000 mon.a (mon.0) 819 : audit [DBG] from='client.? 172.21.15.163:0/3631120158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:08.553 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:08 smithi181 bash[27712]: cluster 2023-10-09T09:06:07.262148+0000 mgr.y (mgr.24355) 193 : cluster [DBG] pgmap v207: 89 pgs: 89 active+clean; 151 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:08.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:08 smithi181 bash[27712]: cluster 2023-10-09T09:06:08.092270+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2023-10-09T09:06:08.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:08 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:06:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:06:09.095 INFO:tasks.rados.rados.0.smithi163.stdout:215: done (4 left) 2023-10-09T09:06:09.095 INFO:tasks.rados.rados.0.smithi163.stdout:216: done (3 left) 2023-10-09T09:06:09.096 INFO:tasks.rados.rados.0.smithi163.stdout:219: done (2 left) 2023-10-09T09:06:09.096 INFO:tasks.rados.rados.0.smithi163.stdout:220: done (1 left) 2023-10-09T09:06:09.096 INFO:tasks.rados.rados.0.smithi163.stdout:221: done (0 left) 2023-10-09T09:06:09.096 INFO:tasks.rados.rados.0.smithi163.stdout:222: delete oid 28 current snap is 21 2023-10-09T09:06:09.099 INFO:tasks.rados.rados.0.smithi163.stdout:222: done (0 left) 2023-10-09T09:06:09.099 INFO:tasks.rados.rados.0.smithi163.stdout:223: snap_create 2023-10-09T09:06:10.099 INFO:tasks.rados.rados.0.smithi163.stdout:223: done (0 left) 2023-10-09T09:06:10.099 INFO:tasks.rados.rados.0.smithi163.stdout:224: read oid 49 snap -1 2023-10-09T09:06:10.100 INFO:tasks.rados.rados.0.smithi163.stdout:224: expect (ObjNum 113 snap 19 seq_num 113) 2023-10-09T09:06:10.100 INFO:tasks.rados.rados.0.smithi163.stdout:225: read oid 23 snap 21 2023-10-09T09:06:10.100 INFO:tasks.rados.rados.0.smithi163.stdout:225: expect deleted 2023-10-09T09:06:10.100 INFO:tasks.rados.rados.0.smithi163.stdout:226: write oid 7 current snap is 22 2023-10-09T09:06:10.100 INFO:tasks.rados.rados.0.smithi163.stdout:226: seq_num 120 ranges {416840=775175,1672545=344295} 2023-10-09T09:06:10.107 INFO:tasks.rados.rados.0.smithi163.stdout:226: writing smithi16367185-7 from 416840 to 1192015 tid 1 2023-10-09T09:06:10.113 INFO:tasks.rados.rados.0.smithi163.stdout:226: writing smithi16367185-7 from 1672545 to 2016840 tid 2 2023-10-09T09:06:10.113 INFO:tasks.rados.rados.0.smithi163.stdout:227: snap_create 2023-10-09T09:06:10.113 INFO:tasks.rados.rados.0.smithi163.stdout:224: expect (ObjNum 113 snap 19 seq_num 113) 2023-10-09T09:06:10.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:10 smithi163 bash[27500]: cluster 2023-10-09T09:06:09.094829+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2023-10-09T09:06:10.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:10 smithi163 bash[27500]: cluster 2023-10-09T09:06:09.262848+0000 mgr.y (mgr.24355) 194 : cluster [DBG] pgmap v210: 89 pgs: 89 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 794 KiB/s wr, 7 op/s 2023-10-09T09:06:10.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:10 smithi163 bash[27500]: audit 2023-10-09T09:06:09.592106+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.163:0/3879392792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:10.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:09 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:06:09] "GET /metrics HTTP/1.1" 200 194722 "" "Prometheus/2.33.4" 2023-10-09T09:06:10.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:10 smithi163 bash[33912]: cluster 2023-10-09T09:06:09.094829+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2023-10-09T09:06:10.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:10 smithi163 bash[33912]: cluster 2023-10-09T09:06:09.262848+0000 mgr.y (mgr.24355) 194 : cluster [DBG] pgmap v210: 89 pgs: 89 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 794 KiB/s wr, 7 op/s 2023-10-09T09:06:10.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:10 smithi163 bash[33912]: audit 2023-10-09T09:06:09.592106+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.163:0/3879392792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:10.385 INFO:tasks.rados.rados.0.smithi163.stdout:226: finishing write tid 1 to smithi16367185-7 2023-10-09T09:06:10.385 INFO:tasks.rados.rados.0.smithi163.stdout:226: finishing write tid 2 to smithi16367185-7 2023-10-09T09:06:10.386 INFO:tasks.rados.rados.0.smithi163.stdout:226: finishing write tid 3 to smithi16367185-7 2023-10-09T09:06:10.386 INFO:tasks.rados.rados.0.smithi163.stdout:226: finishing write tid 4 to smithi16367185-7 2023-10-09T09:06:10.386 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 21 (ObjNum 120 snap 22 seq_num 120) dirty exists 2023-10-09T09:06:10.386 INFO:tasks.rados.rados.0.smithi163.stdout:226: left oid 7 (ObjNum 120 snap 22 seq_num 120) 2023-10-09T09:06:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:10 smithi181 bash[27712]: cluster 2023-10-09T09:06:09.094829+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2023-10-09T09:06:10.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:10 smithi181 bash[27712]: cluster 2023-10-09T09:06:09.262848+0000 mgr.y (mgr.24355) 194 : cluster [DBG] pgmap v210: 89 pgs: 89 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 794 KiB/s wr, 7 op/s 2023-10-09T09:06:10.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:10 smithi181 bash[27712]: audit 2023-10-09T09:06:09.592106+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.163:0/3879392792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:11.115 INFO:tasks.rados.rados.0.smithi163.stdout:224: done (3 left) 2023-10-09T09:06:11.115 INFO:tasks.rados.rados.0.smithi163.stdout:225: done (2 left) 2023-10-09T09:06:11.115 INFO:tasks.rados.rados.0.smithi163.stdout:226: done (1 left) 2023-10-09T09:06:11.115 INFO:tasks.rados.rados.0.smithi163.stdout:227: done (0 left) 2023-10-09T09:06:11.115 INFO:tasks.rados.rados.0.smithi163.stdout:228: rollback oid 47 current snap is 23 2023-10-09T09:06:11.115 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 47 to 19 2023-10-09T09:06:11.115 INFO:tasks.rados.rados.0.smithi163.stdout:229: delete oid 28 current snap is 23 2023-10-09T09:06:11.116 INFO:tasks.rados.rados.0.smithi163.stdout:229: done (1 left) 2023-10-09T09:06:11.116 INFO:tasks.rados.rados.0.smithi163.stdout:230: snap_remove snap 21 2023-10-09T09:06:11.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:11 smithi163 bash[33912]: cluster 2023-10-09T09:06:10.098678+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2023-10-09T09:06:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:11 smithi163 bash[27500]: cluster 2023-10-09T09:06:10.098678+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2023-10-09T09:06:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:11 smithi181 bash[27712]: cluster 2023-10-09T09:06:10.098678+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2023-10-09T09:06:12.099 INFO:tasks.rados.rados.0.smithi163.stdout:230: done (1 left) 2023-10-09T09:06:12.099 INFO:tasks.rados.rados.0.smithi163.stdout:231: write oid 28 current snap is 23 2023-10-09T09:06:12.100 INFO:tasks.rados.rados.0.smithi163.stdout:231: seq_num 122 ranges {450454=642026,1505862=544592} 2023-10-09T09:06:12.106 INFO:tasks.rados.rados.0.smithi163.stdout:231: writing smithi16367185-28 from 450454 to 1092480 tid 1 2023-10-09T09:06:12.115 INFO:tasks.rados.rados.0.smithi163.stdout:231: writing smithi16367185-28 from 1505862 to 2050454 tid 2 2023-10-09T09:06:12.115 INFO:tasks.rados.rados.0.smithi163.stdout:232: read oid 40 snap -1 2023-10-09T09:06:12.115 INFO:tasks.rados.rados.0.smithi163.stdout:232: expect (ObjNum 83 snap 10 seq_num 83) 2023-10-09T09:06:12.115 INFO:tasks.rados.rados.0.smithi163.stdout:233: read oid 46 snap -1 2023-10-09T09:06:12.115 INFO:tasks.rados.rados.0.smithi163.stdout:233: expect deleted 2023-10-09T09:06:12.115 INFO:tasks.rados.rados.0.smithi163.stdout:234: read oid 15 snap -1 2023-10-09T09:06:12.115 INFO:tasks.rados.rados.0.smithi163.stdout:234: expect (ObjNum 14 snap 0 seq_num 14) 2023-10-09T09:06:12.115 INFO:tasks.rados.rados.0.smithi163.stdout:235: delete oid 27 current snap is 23 2023-10-09T09:06:12.115 INFO:tasks.rados.rados.0.smithi163.stdout:228: finishing rollback tid 0 to smithi16367185-47 2023-10-09T09:06:12.116 INFO:tasks.rados.rados.0.smithi163.stdout:228: finishing rollback tid 1 to smithi16367185-47 2023-10-09T09:06:12.116 INFO:tasks.rados.rados.0.smithi163.stdout:228: finishing rollback tid 2 to smithi16367185-47 2023-10-09T09:06:12.116 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 167 (ObjNum 79 snap 10 seq_num 79) dirty exists 2023-10-09T09:06:12.116 INFO:tasks.rados.rados.0.smithi163.stdout:231: finishing write tid 1 to smithi16367185-28 2023-10-09T09:06:12.118 INFO:tasks.rados.rados.0.smithi163.stdout:228: done (5 left) 2023-10-09T09:06:12.118 INFO:tasks.rados.rados.0.smithi163.stdout:233: done (4 left) 2023-10-09T09:06:12.119 INFO:tasks.rados.rados.0.smithi163.stdout:235: done (3 left) 2023-10-09T09:06:12.119 INFO:tasks.rados.rados.0.smithi163.stdout:236: write oid 8 current snap is 23 2023-10-09T09:06:12.119 INFO:tasks.rados.rados.0.smithi163.stdout:236: seq_num 124 ranges {484068=792524,1906473=177595} 2023-10-09T09:06:12.128 INFO:tasks.rados.rados.0.smithi163.stdout:236: writing smithi16367185-8 from 484068 to 1276592 tid 1 2023-10-09T09:06:12.132 INFO:tasks.rados.rados.0.smithi163.stdout:236: writing smithi16367185-8 from 1906473 to 2084068 tid 2 2023-10-09T09:06:12.133 INFO:tasks.rados.rados.0.smithi163.stdout:237: write oid 23 current snap is 23 2023-10-09T09:06:12.133 INFO:tasks.rados.rados.0.smithi163.stdout:231: finishing write tid 2 to smithi16367185-28 2023-10-09T09:06:12.133 INFO:tasks.rados.rados.0.smithi163.stdout:237: seq_num 125 ranges {500875=467773,1564955=535920} 2023-10-09T09:06:12.138 INFO:tasks.rados.rados.0.smithi163.stdout:237: writing smithi16367185-23 from 500875 to 968648 tid 1 2023-10-09T09:06:12.144 INFO:tasks.rados.rados.0.smithi163.stdout:237: writing smithi16367185-23 from 1564955 to 2100875 tid 2 2023-10-09T09:06:12.145 INFO:tasks.rados.rados.0.smithi163.stdout:238: snap_create 2023-10-09T09:06:12.145 INFO:tasks.rados.rados.0.smithi163.stdout:231: finishing write tid 3 to smithi16367185-28 2023-10-09T09:06:12.145 INFO:tasks.rados.rados.0.smithi163.stdout:231: finishing write tid 4 to smithi16367185-28 2023-10-09T09:06:12.145 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 115 (ObjNum 122 snap 23 seq_num 122) dirty exists 2023-10-09T09:06:12.145 INFO:tasks.rados.rados.0.smithi163.stdout:231: left oid 28 (ObjNum 122 snap 23 seq_num 122) 2023-10-09T09:06:12.145 INFO:tasks.rados.rados.0.smithi163.stdout:232: expect (ObjNum 83 snap 10 seq_num 83) 2023-10-09T09:06:12.313 INFO:tasks.rados.rados.0.smithi163.stdout:234: expect (ObjNum 14 snap 0 seq_num 14) 2023-10-09T09:06:12.395 INFO:tasks.rados.rados.0.smithi163.stdout:236: finishing write tid 1 to smithi16367185-8 2023-10-09T09:06:12.396 INFO:tasks.rados.rados.0.smithi163.stdout:236: finishing write tid 2 to smithi16367185-8 2023-10-09T09:06:12.396 INFO:tasks.rados.rados.0.smithi163.stdout:236: finishing write tid 3 to smithi16367185-8 2023-10-09T09:06:12.396 INFO:tasks.rados.rados.0.smithi163.stdout:236: finishing write tid 4 to smithi16367185-8 2023-10-09T09:06:12.396 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 95 (ObjNum 124 snap 23 seq_num 124) dirty exists 2023-10-09T09:06:12.396 INFO:tasks.rados.rados.0.smithi163.stdout:236: left oid 8 (ObjNum 124 snap 23 seq_num 124) 2023-10-09T09:06:12.397 INFO:tasks.rados.rados.0.smithi163.stdout:237: finishing write tid 1 to smithi16367185-23 2023-10-09T09:06:12.397 INFO:tasks.rados.rados.0.smithi163.stdout:237: finishing write tid 2 to smithi16367185-23 2023-10-09T09:06:12.397 INFO:tasks.rados.rados.0.smithi163.stdout:237: finishing write tid 3 to smithi16367185-23 2023-10-09T09:06:12.397 INFO:tasks.rados.rados.0.smithi163.stdout:237: finishing write tid 4 to smithi16367185-23 2023-10-09T09:06:12.397 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 88 (ObjNum 125 snap 23 seq_num 125) dirty exists 2023-10-09T09:06:12.398 INFO:tasks.rados.rados.0.smithi163.stdout:237: left oid 23 (ObjNum 125 snap 23 seq_num 125) 2023-10-09T09:06:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:12 smithi163 bash[33912]: cluster 2023-10-09T09:06:11.115076+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2023-10-09T09:06:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:12 smithi163 bash[33912]: cluster 2023-10-09T09:06:11.263534+0000 mgr.y (mgr.24355) 195 : cluster [DBG] pgmap v213: 89 pgs: 89 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 794 KiB/s wr, 7 op/s 2023-10-09T09:06:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:12 smithi163 bash[33912]: audit 2023-10-09T09:06:12.007374+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.163:0/1126158818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:12 smithi163 bash[33912]: cluster 2023-10-09T09:06:12.097377+0000 mon.a (mon.0) 824 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2023-10-09T09:06:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:12 smithi163 bash[27500]: cluster 2023-10-09T09:06:11.115076+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2023-10-09T09:06:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:12 smithi163 bash[27500]: cluster 2023-10-09T09:06:11.263534+0000 mgr.y (mgr.24355) 195 : cluster [DBG] pgmap v213: 89 pgs: 89 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 794 KiB/s wr, 7 op/s 2023-10-09T09:06:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:12 smithi163 bash[27500]: audit 2023-10-09T09:06:12.007374+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.163:0/1126158818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:12 smithi163 bash[27500]: cluster 2023-10-09T09:06:12.097377+0000 mon.a (mon.0) 824 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2023-10-09T09:06:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:12 smithi181 bash[27712]: cluster 2023-10-09T09:06:11.115076+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2023-10-09T09:06:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:12 smithi181 bash[27712]: cluster 2023-10-09T09:06:11.263534+0000 mgr.y (mgr.24355) 195 : cluster [DBG] pgmap v213: 89 pgs: 89 active+clean; 119 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 794 KiB/s wr, 7 op/s 2023-10-09T09:06:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:12 smithi181 bash[27712]: audit 2023-10-09T09:06:12.007374+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.163:0/1126158818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:12 smithi181 bash[27712]: cluster 2023-10-09T09:06:12.097377+0000 mon.a (mon.0) 824 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2023-10-09T09:06:13.135 INFO:tasks.rados.rados.0.smithi163.stdout:231: done (5 left) 2023-10-09T09:06:13.135 INFO:tasks.rados.rados.0.smithi163.stdout:232: done (4 left) 2023-10-09T09:06:13.135 INFO:tasks.rados.rados.0.smithi163.stdout:234: done (3 left) 2023-10-09T09:06:13.135 INFO:tasks.rados.rados.0.smithi163.stdout:236: done (2 left) 2023-10-09T09:06:13.135 INFO:tasks.rados.rados.0.smithi163.stdout:237: done (1 left) 2023-10-09T09:06:13.135 INFO:tasks.rados.rados.0.smithi163.stdout:238: done (0 left) 2023-10-09T09:06:13.135 INFO:tasks.rados.rados.0.smithi163.stdout:239: read oid 44 snap -1 2023-10-09T09:06:13.136 INFO:tasks.rados.rados.0.smithi163.stdout:239: expect (ObjNum 98 snap 16 seq_num 98) 2023-10-09T09:06:13.136 INFO:tasks.rados.rados.0.smithi163.stdout:240: write oid 22 current snap is 24 2023-10-09T09:06:13.136 INFO:tasks.rados.rados.0.smithi163.stdout:240: seq_num 126 ranges {517682=543022,1623437=494245} 2023-10-09T09:06:13.143 INFO:tasks.rados.rados.0.smithi163.stdout:240: writing smithi16367185-22 from 517682 to 1060704 tid 1 2023-10-09T09:06:13.152 INFO:tasks.rados.rados.0.smithi163.stdout:240: writing smithi16367185-22 from 1623437 to 2117682 tid 2 2023-10-09T09:06:13.152 INFO:tasks.rados.rados.0.smithi163.stdout:241: snap_create 2023-10-09T09:06:13.152 INFO:tasks.rados.rados.0.smithi163.stdout:239: expect (ObjNum 98 snap 16 seq_num 98) 2023-10-09T09:06:13.315 INFO:tasks.rados.rados.0.smithi163.stdout:240: finishing write tid 1 to smithi16367185-22 2023-10-09T09:06:13.315 INFO:tasks.rados.rados.0.smithi163.stdout:240: finishing write tid 2 to smithi16367185-22 2023-10-09T09:06:13.315 INFO:tasks.rados.rados.0.smithi163.stdout:240: finishing write tid 3 to smithi16367185-22 2023-10-09T09:06:13.315 INFO:tasks.rados.rados.0.smithi163.stdout:240: finishing write tid 4 to smithi16367185-22 2023-10-09T09:06:13.315 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 173 (ObjNum 126 snap 24 seq_num 126) dirty exists 2023-10-09T09:06:13.316 INFO:tasks.rados.rados.0.smithi163.stdout:240: left oid 22 (ObjNum 126 snap 24 seq_num 126) 2023-10-09T09:06:14.144 INFO:tasks.rados.rados.0.smithi163.stdout:239: done (2 left) 2023-10-09T09:06:14.144 INFO:tasks.rados.rados.0.smithi163.stdout:240: done (1 left) 2023-10-09T09:06:14.144 INFO:tasks.rados.rados.0.smithi163.stdout:241: done (0 left) 2023-10-09T09:06:14.144 INFO:tasks.rados.rados.0.smithi163.stdout:242: rollback oid 28 current snap is 25 2023-10-09T09:06:14.144 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 24 2023-10-09T09:06:14.144 INFO:tasks.rados.rados.0.smithi163.stdout:243: delete oid 29 current snap is 25 2023-10-09T09:06:14.149 INFO:tasks.rados.rados.0.smithi163.stdout:243: done (1 left) 2023-10-09T09:06:14.149 INFO:tasks.rados.rados.0.smithi163.stdout:244: rollback oid 26 current snap is 25 2023-10-09T09:06:14.149 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 26 to 24 2023-10-09T09:06:14.150 INFO:tasks.rados.rados.0.smithi163.stdout:245: rollback oid 16 current snap is 25 2023-10-09T09:06:14.150 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 16 to 20 2023-10-09T09:06:14.150 INFO:tasks.rados.rados.0.smithi163.stdout:246: snap_create 2023-10-09T09:06:14.150 INFO:tasks.rados.rados.0.smithi163.stdout:242: finishing rollback tid 0 to smithi16367185-28 2023-10-09T09:06:14.150 INFO:tasks.rados.rados.0.smithi163.stdout:242: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:06:14.151 INFO:tasks.rados.rados.0.smithi163.stdout:242: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:06:14.151 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 119 (ObjNum 122 snap 23 seq_num 122) dirty exists 2023-10-09T09:06:14.153 INFO:tasks.rados.rados.0.smithi163.stdout:245: finishing rollback tid 0 to smithi16367185-16 2023-10-09T09:06:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:245: finishing rollback tid 1 to smithi16367185-16 2023-10-09T09:06:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:245: finishing rollback tid 2 to smithi16367185-16 2023-10-09T09:06:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 123 (ObjNum 117 snap 20 seq_num 117) dirty exists 2023-10-09T09:06:14.155 INFO:tasks.rados.rados.0.smithi163.stdout:244: finishing rollback tid 0 to smithi16367185-26 2023-10-09T09:06:14.155 INFO:tasks.rados.rados.0.smithi163.stdout:244: finishing rollback tid 1 to smithi16367185-26 2023-10-09T09:06:14.155 INFO:tasks.rados.rados.0.smithi163.stdout:244: finishing rollback tid 2 to smithi16367185-26 2023-10-09T09:06:14.155 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 48 (ObjNum 25 snap 0 seq_num 25) dirty exists 2023-10-09T09:06:14.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:14 smithi163 bash[33912]: cluster 2023-10-09T09:06:13.134837+0000 mon.a (mon.0) 825 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2023-10-09T09:06:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:14 smithi163 bash[33912]: cluster 2023-10-09T09:06:13.264611+0000 mgr.y (mgr.24355) 196 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2023-10-09T09:06:14.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:14 smithi163 bash[27500]: cluster 2023-10-09T09:06:13.134837+0000 mon.a (mon.0) 825 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2023-10-09T09:06:14.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:14 smithi163 bash[27500]: cluster 2023-10-09T09:06:13.264611+0000 mgr.y (mgr.24355) 196 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2023-10-09T09:06:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:14 smithi181 bash[27712]: cluster 2023-10-09T09:06:13.134837+0000 mon.a (mon.0) 825 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2023-10-09T09:06:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:14 smithi181 bash[27712]: cluster 2023-10-09T09:06:13.264611+0000 mgr.y (mgr.24355) 196 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2023-10-09T09:06:15.152 INFO:tasks.rados.rados.0.smithi163.stdout:242: done (3 left) 2023-10-09T09:06:15.152 INFO:tasks.rados.rados.0.smithi163.stdout:244: done (2 left) 2023-10-09T09:06:15.152 INFO:tasks.rados.rados.0.smithi163.stdout:245: done (1 left) 2023-10-09T09:06:15.152 INFO:tasks.rados.rados.0.smithi163.stdout:246: done (0 left) 2023-10-09T09:06:15.152 INFO:tasks.rados.rados.0.smithi163.stdout:247: snap_create 2023-10-09T09:06:15.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:15 smithi163 bash[33912]: cluster 2023-10-09T09:06:14.142316+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2023-10-09T09:06:15.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:15 smithi163 bash[33912]: audit 2023-10-09T09:06:14.414892+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.163:0/1849199949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:15.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:15 smithi163 bash[27500]: cluster 2023-10-09T09:06:14.142316+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2023-10-09T09:06:15.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:15 smithi163 bash[27500]: audit 2023-10-09T09:06:14.414892+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.163:0/1849199949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:15 smithi181 bash[27712]: cluster 2023-10-09T09:06:14.142316+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2023-10-09T09:06:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:15 smithi181 bash[27712]: audit 2023-10-09T09:06:14.414892+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.163:0/1849199949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:16.151 INFO:tasks.rados.rados.0.smithi163.stdout:247: done (0 left) 2023-10-09T09:06:16.151 INFO:tasks.rados.rados.0.smithi163.stdout:248: write oid 48 current snap is 27 2023-10-09T09:06:16.151 INFO:tasks.rados.rados.0.smithi163.stdout:248: seq_num 128 ranges {551296=693520,2024048=127248} 2023-10-09T09:06:16.161 INFO:tasks.rados.rados.0.smithi163.stdout:248: writing smithi16367185-48 from 551296 to 1244816 tid 1 2023-10-09T09:06:16.166 INFO:tasks.rados.rados.0.smithi163.stdout:248: writing smithi16367185-48 from 2024048 to 2151296 tid 2 2023-10-09T09:06:16.167 INFO:tasks.rados.rados.0.smithi163.stdout:249: write oid 44 current snap is 27 2023-10-09T09:06:16.167 INFO:tasks.rados.rados.0.smithi163.stdout:249: seq_num 129 ranges {568103=768769,2082530=85573} 2023-10-09T09:06:16.174 INFO:tasks.rados.rados.0.smithi163.stdout:249: writing smithi16367185-44 from 568103 to 1336872 tid 1 2023-10-09T09:06:16.179 INFO:tasks.rados.rados.0.smithi163.stdout:249: writing smithi16367185-44 from 2082530 to 2168103 tid 2 2023-10-09T09:06:16.180 INFO:tasks.rados.rados.0.smithi163.stdout:250: snap_remove snap 26 2023-10-09T09:06:16.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:16 smithi163 bash[33912]: cluster 2023-10-09T09:06:15.151465+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2023-10-09T09:06:16.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:16 smithi163 bash[33912]: cluster 2023-10-09T09:06:15.265283+0000 mgr.y (mgr.24355) 197 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2023-10-09T09:06:16.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:16 smithi163 bash[33912]: cluster 2023-10-09T09:06:16.151132+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2023-10-09T09:06:16.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:16 smithi163 bash[27500]: cluster 2023-10-09T09:06:15.151465+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2023-10-09T09:06:16.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:16 smithi163 bash[27500]: cluster 2023-10-09T09:06:15.265283+0000 mgr.y (mgr.24355) 197 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2023-10-09T09:06:16.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:16 smithi163 bash[27500]: cluster 2023-10-09T09:06:16.151132+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2023-10-09T09:06:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:16 smithi181 bash[27712]: cluster 2023-10-09T09:06:15.151465+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2023-10-09T09:06:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:16 smithi181 bash[27712]: cluster 2023-10-09T09:06:15.265283+0000 mgr.y (mgr.24355) 197 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2023-10-09T09:06:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:16 smithi181 bash[27712]: cluster 2023-10-09T09:06:16.151132+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2023-10-09T09:06:17.097 INFO:tasks.rados.rados.0.smithi163.stdout:250: done (2 left) 2023-10-09T09:06:17.097 INFO:tasks.rados.rados.0.smithi163.stdout:251: write oid 8 current snap is 27 2023-10-09T09:06:17.097 INFO:tasks.rados.rados.0.smithi163.stdout:248: finishing write tid 1 to smithi16367185-48 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:248: finishing write tid 2 to smithi16367185-48 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:248: finishing write tid 3 to smithi16367185-48 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:248: finishing write tid 4 to smithi16367185-48 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 177 (ObjNum 128 snap 27 seq_num 128) dirty exists 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:248: left oid 48 (ObjNum 128 snap 27 seq_num 128) 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:249: finishing write tid 1 to smithi16367185-44 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:249: finishing write tid 2 to smithi16367185-44 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:249: finishing write tid 3 to smithi16367185-44 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:249: finishing write tid 4 to smithi16367185-44 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 181 (ObjNum 129 snap 27 seq_num 129) dirty exists 2023-10-09T09:06:17.098 INFO:tasks.rados.rados.0.smithi163.stdout:249: left oid 44 (ObjNum 129 snap 27 seq_num 129) 2023-10-09T09:06:17.099 INFO:tasks.rados.rados.0.smithi163.stdout:251: seq_num 130 ranges {584910=560371,1857365=327545} 2023-10-09T09:06:17.103 INFO:tasks.rados.rados.0.smithi163.stdout:251: writing smithi16367185-8 from 584910 to 1145281 tid 1 2023-10-09T09:06:17.109 INFO:tasks.rados.rados.0.smithi163.stdout:251: writing smithi16367185-8 from 1857365 to 2184910 tid 2 2023-10-09T09:06:17.110 INFO:tasks.rados.rados.0.smithi163.stdout:248: done (2 left) 2023-10-09T09:06:17.110 INFO:tasks.rados.rados.0.smithi163.stdout:249: done (1 left) 2023-10-09T09:06:17.110 INFO:tasks.rados.rados.0.smithi163.stdout:252: snap_create 2023-10-09T09:06:17.110 INFO:tasks.rados.rados.0.smithi163.stdout:251: finishing write tid 1 to smithi16367185-8 2023-10-09T09:06:17.113 INFO:tasks.rados.rados.0.smithi163.stdout:251: finishing write tid 2 to smithi16367185-8 2023-10-09T09:06:17.114 INFO:tasks.rados.rados.0.smithi163.stdout:251: finishing write tid 3 to smithi16367185-8 2023-10-09T09:06:17.114 INFO:tasks.rados.rados.0.smithi163.stdout:251: finishing write tid 4 to smithi16367185-8 2023-10-09T09:06:17.114 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 99 (ObjNum 130 snap 27 seq_num 130) dirty exists 2023-10-09T09:06:17.114 INFO:tasks.rados.rados.0.smithi163.stdout:251: left oid 8 (ObjNum 130 snap 27 seq_num 130) 2023-10-09T09:06:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:17 smithi163 bash[27500]: audit 2023-10-09T09:06:16.825671+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.163:0/1136373648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:17 smithi163 bash[27500]: cluster 2023-10-09T09:06:17.095995+0000 mon.a (mon.0) 830 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2023-10-09T09:06:17.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:17 smithi163 bash[33912]: audit 2023-10-09T09:06:16.825671+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.163:0/1136373648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:17.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:17 smithi163 bash[33912]: cluster 2023-10-09T09:06:17.095995+0000 mon.a (mon.0) 830 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2023-10-09T09:06:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:17 smithi181 bash[27712]: audit 2023-10-09T09:06:16.825671+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.163:0/1136373648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:17 smithi181 bash[27712]: cluster 2023-10-09T09:06:17.095995+0000 mon.a (mon.0) 830 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2023-10-09T09:06:18.111 INFO:tasks.rados.rados.0.smithi163.stdout:251: done (1 left) 2023-10-09T09:06:18.111 INFO:tasks.rados.rados.0.smithi163.stdout:252: done (0 left) 2023-10-09T09:06:18.111 INFO:tasks.rados.rados.0.smithi163.stdout:253: snap_remove snap 27 2023-10-09T09:06:18.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:18 smithi163 bash[33912]: cluster 2023-10-09T09:06:17.265973+0000 mgr.y (mgr.24355) 198 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:18.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:18 smithi163 bash[33912]: cluster 2023-10-09T09:06:18.111236+0000 mon.a (mon.0) 831 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2023-10-09T09:06:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:18 smithi163 bash[27500]: cluster 2023-10-09T09:06:17.265973+0000 mgr.y (mgr.24355) 198 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:18 smithi163 bash[27500]: cluster 2023-10-09T09:06:18.111236+0000 mon.a (mon.0) 831 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2023-10-09T09:06:18.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:18 smithi181 bash[27712]: cluster 2023-10-09T09:06:17.265973+0000 mgr.y (mgr.24355) 198 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:18.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:18 smithi181 bash[27712]: cluster 2023-10-09T09:06:18.111236+0000 mon.a (mon.0) 831 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2023-10-09T09:06:18.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:18 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:06:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:06:19.113 INFO:tasks.rados.rados.0.smithi163.stdout:253: done (0 left) 2023-10-09T09:06:19.113 INFO:tasks.rados.rados.0.smithi163.stdout:254: read oid 29 snap -1 2023-10-09T09:06:19.113 INFO:tasks.rados.rados.0.smithi163.stdout:254: expect deleted 2023-10-09T09:06:19.113 INFO:tasks.rados.rados.0.smithi163.stdout:255: read oid 15 snap -1 2023-10-09T09:06:19.113 INFO:tasks.rados.rados.0.smithi163.stdout:255: expect (ObjNum 14 snap 0 seq_num 14) 2023-10-09T09:06:19.113 INFO:tasks.rados.rados.0.smithi163.stdout:256: delete oid 36 current snap is 28 2023-10-09T09:06:19.117 INFO:tasks.rados.rados.0.smithi163.stdout:254: done (2 left) 2023-10-09T09:06:19.117 INFO:tasks.rados.rados.0.smithi163.stdout:256: done (1 left) 2023-10-09T09:06:19.117 INFO:tasks.rados.rados.0.smithi163.stdout:257: delete oid 47 current snap is 28 2023-10-09T09:06:19.120 INFO:tasks.rados.rados.0.smithi163.stdout:255: expect (ObjNum 14 snap 0 seq_num 14) 2023-10-09T09:06:19.199 INFO:tasks.rados.rados.0.smithi163.stdout:255: done (1 left) 2023-10-09T09:06:19.199 INFO:tasks.rados.rados.0.smithi163.stdout:257: done (0 left) 2023-10-09T09:06:19.199 INFO:tasks.rados.rados.0.smithi163.stdout:258: read oid 29 snap -1 2023-10-09T09:06:19.199 INFO:tasks.rados.rados.0.smithi163.stdout:258: expect deleted 2023-10-09T09:06:19.199 INFO:tasks.rados.rados.0.smithi163.stdout:259: write oid 19 current snap is 28 2023-10-09T09:06:19.199 INFO:tasks.rados.rados.0.smithi163.stdout:259: seq_num 133 ranges {635331=786118,1916458=318873} 2023-10-09T09:06:19.214 INFO:tasks.rados.rados.0.smithi163.stdout:259: writing smithi16367185-19 from 635331 to 1421449 tid 1 2023-10-09T09:06:19.219 INFO:tasks.rados.rados.0.smithi163.stdout:259: writing smithi16367185-19 from 1916458 to 2235331 tid 2 2023-10-09T09:06:19.219 INFO:tasks.rados.rados.0.smithi163.stdout:260: snap_remove snap 19 2023-10-09T09:06:20.121 INFO:tasks.rados.rados.0.smithi163.stdout:260: done (2 left) 2023-10-09T09:06:20.121 INFO:tasks.rados.rados.0.smithi163.stdout:261: write oid 36 current snap is 28 2023-10-09T09:06:20.121 INFO:tasks.rados.rados.0.smithi163.stdout:259: finishing write tid 1 to smithi16367185-19 2023-10-09T09:06:20.121 INFO:tasks.rados.rados.0.smithi163.stdout:259: finishing write tid 2 to smithi16367185-19 2023-10-09T09:06:20.121 INFO:tasks.rados.rados.0.smithi163.stdout:261: seq_num 134 ranges {652138=461367,1574940=677198} 2023-10-09T09:06:20.127 INFO:tasks.rados.rados.0.smithi163.stdout:261: writing smithi16367185-36 from 652138 to 1113505 tid 1 2023-10-09T09:06:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:261: writing smithi16367185-36 from 1574940 to 2252138 tid 2 2023-10-09T09:06:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:258: done (2 left) 2023-10-09T09:06:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:262: read oid 4 snap -1 2023-10-09T09:06:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:262: expect (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:06:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:263: write oid 24 current snap is 28 2023-10-09T09:06:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:259: finishing write tid 3 to smithi16367185-19 2023-10-09T09:06:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:259: finishing write tid 4 to smithi16367185-19 2023-10-09T09:06:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 52 (ObjNum 133 snap 28 seq_num 133) dirty exists 2023-10-09T09:06:20.135 INFO:tasks.rados.rados.0.smithi163.stdout:259: left oid 19 (ObjNum 133 snap 28 seq_num 133) 2023-10-09T09:06:20.135 INFO:tasks.rados.rados.0.smithi163.stdout:263: seq_num 135 ranges {668945=536616,1633422=491363} 2023-10-09T09:06:20.141 INFO:tasks.rados.rados.0.smithi163.stdout:263: writing smithi16367185-24 from 668945 to 1205561 tid 1 2023-10-09T09:06:20.147 INFO:tasks.rados.rados.0.smithi163.stdout:263: writing smithi16367185-24 from 1633422 to 2124785 tid 2 2023-10-09T09:06:20.147 INFO:tasks.rados.rados.0.smithi163.stdout:259: done (3 left) 2023-10-09T09:06:20.148 INFO:tasks.rados.rados.0.smithi163.stdout:264: read oid 14 snap -1 2023-10-09T09:06:20.148 INFO:tasks.rados.rados.0.smithi163.stdout:264: expect (ObjNum 75 snap 9 seq_num 75) 2023-10-09T09:06:20.148 INFO:tasks.rados.rados.0.smithi163.stdout:265: read oid 20 snap -1 2023-10-09T09:06:20.148 INFO:tasks.rados.rados.0.smithi163.stdout:265: expect deleted 2023-10-09T09:06:20.148 INFO:tasks.rados.rados.0.smithi163.stdout:266: write oid 19 current snap is 28 2023-10-09T09:06:20.148 INFO:tasks.rados.rados.0.smithi163.stdout:266: seq_num 136 ranges {685752=611865,1975551=310201} 2023-10-09T09:06:20.155 INFO:tasks.rados.rados.0.smithi163.stdout:266: writing smithi16367185-19 from 685752 to 1297617 tid 1 2023-10-09T09:06:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:266: writing smithi16367185-19 from 1975551 to 2285752 tid 2 2023-10-09T09:06:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:267: rollback oid 18 current snap is 28 2023-10-09T09:06:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 18 to 24 2023-10-09T09:06:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:261: finishing write tid 1 to smithi16367185-36 2023-10-09T09:06:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:261: finishing write tid 2 to smithi16367185-36 2023-10-09T09:06:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:268: write oid 17 current snap is 28 2023-10-09T09:06:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:261: finishing write tid 3 to smithi16367185-36 2023-10-09T09:06:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:268: seq_num 137 ranges {702559=403467,1750386=495032} 2023-10-09T09:06:20.168 INFO:tasks.rados.rados.0.smithi163.stdout:268: writing smithi16367185-17 from 702559 to 1106026 tid 1 2023-10-09T09:06:20.175 INFO:tasks.rados.rados.0.smithi163.stdout:268: writing smithi16367185-17 from 1750386 to 2245418 tid 2 2023-10-09T09:06:20.175 INFO:tasks.rados.rados.0.smithi163.stdout:269: read oid 6 snap -1 2023-10-09T09:06:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:269: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:06:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:270: rollback oid 10 current snap is 28 2023-10-09T09:06:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 24 2023-10-09T09:06:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:261: finishing write tid 4 to smithi16367185-36 2023-10-09T09:06:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 54 (ObjNum 134 snap 28 seq_num 134) dirty exists 2023-10-09T09:06:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:261: left oid 36 (ObjNum 134 snap 28 seq_num 134) 2023-10-09T09:06:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:261: done (9 left) 2023-10-09T09:06:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:271: snap_create 2023-10-09T09:06:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:263: finishing write tid 1 to smithi16367185-24 2023-10-09T09:06:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:262: expect (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:06:20.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:20 smithi163 bash[33912]: cluster 2023-10-09T09:06:19.112624+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2023-10-09T09:06:20.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:20 smithi163 bash[33912]: audit 2023-10-09T09:06:19.240067+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.163:0/914695494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:20.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:20 smithi163 bash[33912]: cluster 2023-10-09T09:06:19.266608+0000 mgr.y (mgr.24355) 199 : cluster [DBG] pgmap v225: 89 pgs: 89 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 626 KiB/s wr, 5 op/s 2023-10-09T09:06:20.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:19 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:06:19] "GET /metrics HTTP/1.1" 200 194722 "" "Prometheus/2.33.4" 2023-10-09T09:06:20.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:20 smithi163 bash[27500]: cluster 2023-10-09T09:06:19.112624+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2023-10-09T09:06:20.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:20 smithi163 bash[27500]: audit 2023-10-09T09:06:19.240067+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.163:0/914695494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:20.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:20 smithi163 bash[27500]: cluster 2023-10-09T09:06:19.266608+0000 mgr.y (mgr.24355) 199 : cluster [DBG] pgmap v225: 89 pgs: 89 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 626 KiB/s wr, 5 op/s 2023-10-09T09:06:20.305 INFO:tasks.rados.rados.0.smithi163.stdout:263: finishing write tid 2 to smithi16367185-24 2023-10-09T09:06:20.305 INFO:tasks.rados.rados.0.smithi163.stdout:263: finishing write tid 3 to smithi16367185-24 2023-10-09T09:06:20.305 INFO:tasks.rados.rados.0.smithi163.stdout:263: finishing write tid 4 to smithi16367185-24 2023-10-09T09:06:20.305 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 46 (ObjNum 135 snap 28 seq_num 135) dirty exists 2023-10-09T09:06:20.306 INFO:tasks.rados.rados.0.smithi163.stdout:263: left oid 24 (ObjNum 135 snap 28 seq_num 135) 2023-10-09T09:06:20.306 INFO:tasks.rados.rados.0.smithi163.stdout:266: finishing write tid 1 to smithi16367185-19 2023-10-09T09:06:20.306 INFO:tasks.rados.rados.0.smithi163.stdout:264: expect (ObjNum 75 snap 9 seq_num 75) 2023-10-09T09:06:20.454 INFO:tasks.rados.rados.0.smithi163.stdout:266: finishing write tid 2 to smithi16367185-19 2023-10-09T09:06:20.454 INFO:tasks.rados.rados.0.smithi163.stdout:266: finishing write tid 3 to smithi16367185-19 2023-10-09T09:06:20.454 INFO:tasks.rados.rados.0.smithi163.stdout:266: finishing write tid 4 to smithi16367185-19 2023-10-09T09:06:20.454 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 61 (ObjNum 136 snap 28 seq_num 136) dirty exists 2023-10-09T09:06:20.454 INFO:tasks.rados.rados.0.smithi163.stdout:266: left oid 19 (ObjNum 136 snap 28 seq_num 136) 2023-10-09T09:06:20.454 INFO:tasks.rados.rados.0.smithi163.stdout:267: finishing rollback tid 0 to smithi16367185-18 2023-10-09T09:06:20.454 INFO:tasks.rados.rados.0.smithi163.stdout:267: finishing rollback tid 1 to smithi16367185-18 2023-10-09T09:06:20.454 INFO:tasks.rados.rados.0.smithi163.stdout:267: finishing rollback tid 2 to smithi16367185-18 2023-10-09T09:06:20.455 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 93 (ObjNum 118 snap 20 seq_num 118) dirty exists 2023-10-09T09:06:20.455 INFO:tasks.rados.rados.0.smithi163.stdout:268: finishing write tid 1 to smithi16367185-17 2023-10-09T09:06:20.455 INFO:tasks.rados.rados.0.smithi163.stdout:270: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:06:20.455 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:06:20.455 INFO:tasks.rados.rados.0.smithi163.stdout:268: finishing write tid 2 to smithi16367185-17 2023-10-09T09:06:20.455 INFO:tasks.rados.rados.0.smithi163.stdout:268: finishing write tid 3 to smithi16367185-17 2023-10-09T09:06:20.455 INFO:tasks.rados.rados.0.smithi163.stdout:268: finishing write tid 4 to smithi16367185-17 2023-10-09T09:06:20.455 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 104 (ObjNum 137 snap 28 seq_num 137) dirty exists 2023-10-09T09:06:20.455 INFO:tasks.rados.rados.0.smithi163.stdout:268: left oid 17 (ObjNum 137 snap 28 seq_num 137) 2023-10-09T09:06:20.455 INFO:tasks.rados.rados.0.smithi163.stdout:269: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:06:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:20 smithi181 bash[27712]: cluster 2023-10-09T09:06:19.112624+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2023-10-09T09:06:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:20 smithi181 bash[27712]: audit 2023-10-09T09:06:19.240067+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.163:0/914695494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:20 smithi181 bash[27712]: cluster 2023-10-09T09:06:19.266608+0000 mgr.y (mgr.24355) 199 : cluster [DBG] pgmap v225: 89 pgs: 89 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 626 KiB/s wr, 5 op/s 2023-10-09T09:06:21.120 INFO:tasks.rados.rados.0.smithi163.stdout:262: done (9 left) 2023-10-09T09:06:21.120 INFO:tasks.rados.rados.0.smithi163.stdout:263: done (8 left) 2023-10-09T09:06:21.120 INFO:tasks.rados.rados.0.smithi163.stdout:264: done (7 left) 2023-10-09T09:06:21.120 INFO:tasks.rados.rados.0.smithi163.stdout:265: done (6 left) 2023-10-09T09:06:21.121 INFO:tasks.rados.rados.0.smithi163.stdout:266: done (5 left) 2023-10-09T09:06:21.121 INFO:tasks.rados.rados.0.smithi163.stdout:267: done (4 left) 2023-10-09T09:06:21.121 INFO:tasks.rados.rados.0.smithi163.stdout:268: done (3 left) 2023-10-09T09:06:21.121 INFO:tasks.rados.rados.0.smithi163.stdout:269: done (2 left) 2023-10-09T09:06:21.121 INFO:tasks.rados.rados.0.smithi163.stdout:270: done (1 left) 2023-10-09T09:06:21.121 INFO:tasks.rados.rados.0.smithi163.stdout:271: done (0 left) 2023-10-09T09:06:21.121 INFO:tasks.rados.rados.0.smithi163.stdout:272: snap_create 2023-10-09T09:06:21.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:21 smithi163 bash[27500]: cluster 2023-10-09T09:06:20.115717+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2023-10-09T09:06:21.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:21 smithi163 bash[33912]: cluster 2023-10-09T09:06:20.115717+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2023-10-09T09:06:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:21 smithi181 bash[27712]: cluster 2023-10-09T09:06:20.115717+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2023-10-09T09:06:22.098 INFO:tasks.rados.rados.0.smithi163.stdout:272: done (0 left) 2023-10-09T09:06:22.098 INFO:tasks.rados.rados.0.smithi163.stdout:273: write oid 31 current snap is 30 2023-10-09T09:06:22.099 INFO:tasks.rados.rados.0.smithi163.stdout:273: seq_num 138 ranges {719366=478716,1808868=510498} 2023-10-09T09:06:22.105 INFO:tasks.rados.rados.0.smithi163.stdout:273: writing smithi16367185-31 from 719366 to 1198082 tid 1 2023-10-09T09:06:22.112 INFO:tasks.rados.rados.0.smithi163.stdout:273: writing smithi16367185-31 from 1808868 to 2319366 tid 2 2023-10-09T09:06:22.112 INFO:tasks.rados.rados.0.smithi163.stdout:274: delete oid 38 current snap is 30 2023-10-09T09:06:22.114 INFO:tasks.rados.rados.0.smithi163.stdout:274: done (1 left) 2023-10-09T09:06:22.115 INFO:tasks.rados.rados.0.smithi163.stdout:275: rollback oid 6 current snap is 30 2023-10-09T09:06:22.115 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 6 to 25 2023-10-09T09:06:22.115 INFO:tasks.rados.rados.0.smithi163.stdout:276: snap_create 2023-10-09T09:06:22.117 INFO:tasks.rados.rados.0.smithi163.stdout:273: finishing write tid 1 to smithi16367185-31 2023-10-09T09:06:22.117 INFO:tasks.rados.rados.0.smithi163.stdout:273: finishing write tid 2 to smithi16367185-31 2023-10-09T09:06:22.117 INFO:tasks.rados.rados.0.smithi163.stdout:273: finishing write tid 3 to smithi16367185-31 2023-10-09T09:06:22.118 INFO:tasks.rados.rados.0.smithi163.stdout:273: finishing write tid 4 to smithi16367185-31 2023-10-09T09:06:22.118 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 58 (ObjNum 138 snap 30 seq_num 138) dirty exists 2023-10-09T09:06:22.118 INFO:tasks.rados.rados.0.smithi163.stdout:273: left oid 31 (ObjNum 138 snap 30 seq_num 138) 2023-10-09T09:06:22.119 INFO:tasks.rados.rados.0.smithi163.stdout:275: finishing rollback tid 0 to smithi16367185-6 2023-10-09T09:06:22.120 INFO:tasks.rados.rados.0.smithi163.stdout:275: finishing rollback tid 1 to smithi16367185-6 2023-10-09T09:06:22.120 INFO:tasks.rados.rados.0.smithi163.stdout:275: finishing rollback tid 2 to smithi16367185-6 2023-10-09T09:06:22.120 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 41 (ObjNum 92 snap 15 seq_num 92) dirty exists 2023-10-09T09:06:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:22 smithi163 bash[33912]: cluster 2023-10-09T09:06:21.120052+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2023-10-09T09:06:22.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:22 smithi163 bash[33912]: cluster 2023-10-09T09:06:21.267312+0000 mgr.y (mgr.24355) 200 : cluster [DBG] pgmap v228: 89 pgs: 89 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 626 KiB/s wr, 5 op/s 2023-10-09T09:06:22.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:22 smithi163 bash[33912]: audit 2023-10-09T09:06:21.655612+0000 mon.a (mon.0) 835 : audit [DBG] from='client.? 172.21.15.163:0/2402823169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:22.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:22 smithi163 bash[33912]: cluster 2023-10-09T09:06:22.098045+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2023-10-09T09:06:22.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:22 smithi163 bash[27500]: cluster 2023-10-09T09:06:21.120052+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2023-10-09T09:06:22.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:22 smithi163 bash[27500]: cluster 2023-10-09T09:06:21.267312+0000 mgr.y (mgr.24355) 200 : cluster [DBG] pgmap v228: 89 pgs: 89 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 626 KiB/s wr, 5 op/s 2023-10-09T09:06:22.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:22 smithi163 bash[27500]: audit 2023-10-09T09:06:21.655612+0000 mon.a (mon.0) 835 : audit [DBG] from='client.? 172.21.15.163:0/2402823169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:22.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:22 smithi163 bash[27500]: cluster 2023-10-09T09:06:22.098045+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2023-10-09T09:06:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:22 smithi181 bash[27712]: cluster 2023-10-09T09:06:21.120052+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2023-10-09T09:06:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:22 smithi181 bash[27712]: cluster 2023-10-09T09:06:21.267312+0000 mgr.y (mgr.24355) 200 : cluster [DBG] pgmap v228: 89 pgs: 89 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 626 KiB/s wr, 5 op/s 2023-10-09T09:06:22.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:22 smithi181 bash[27712]: audit 2023-10-09T09:06:21.655612+0000 mon.a (mon.0) 835 : audit [DBG] from='client.? 172.21.15.163:0/2402823169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:22.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:22 smithi181 bash[27712]: cluster 2023-10-09T09:06:22.098045+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2023-10-09T09:06:23.108 INFO:tasks.rados.rados.0.smithi163.stdout:273: done (2 left) 2023-10-09T09:06:23.108 INFO:tasks.rados.rados.0.smithi163.stdout:275: done (1 left) 2023-10-09T09:06:23.109 INFO:tasks.rados.rados.0.smithi163.stdout:276: done (0 left) 2023-10-09T09:06:23.109 INFO:tasks.rados.rados.0.smithi163.stdout:277: write oid 19 current snap is 31 2023-10-09T09:06:23.109 INFO:tasks.rados.rados.0.smithi163.stdout:277: seq_num 140 ranges {752980=629214,1809479=543501} 2023-10-09T09:06:23.119 INFO:tasks.rados.rados.0.smithi163.stdout:277: writing smithi16367185-19 from 752980 to 1382194 tid 1 2023-10-09T09:06:23.125 INFO:tasks.rados.rados.0.smithi163.stdout:277: writing smithi16367185-19 from 1809479 to 2352980 tid 2 2023-10-09T09:06:23.125 INFO:tasks.rados.rados.0.smithi163.stdout:278: snap_remove snap 29 2023-10-09T09:06:24.118 INFO:tasks.rados.rados.0.smithi163.stdout:278: done (1 left) 2023-10-09T09:06:24.118 INFO:tasks.rados.rados.0.smithi163.stdout:279: delete oid 33 current snap is 31 2023-10-09T09:06:24.118 INFO:tasks.rados.rados.0.smithi163.stdout:277: finishing write tid 1 to smithi16367185-19 2023-10-09T09:06:24.118 INFO:tasks.rados.rados.0.smithi163.stdout:277: finishing write tid 2 to smithi16367185-19 2023-10-09T09:06:24.118 INFO:tasks.rados.rados.0.smithi163.stdout:277: finishing write tid 3 to smithi16367185-19 2023-10-09T09:06:24.118 INFO:tasks.rados.rados.0.smithi163.stdout:277: finishing write tid 4 to smithi16367185-19 2023-10-09T09:06:24.118 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 65 (ObjNum 140 snap 31 seq_num 140) dirty exists 2023-10-09T09:06:24.118 INFO:tasks.rados.rados.0.smithi163.stdout:277: left oid 19 (ObjNum 140 snap 31 seq_num 140) 2023-10-09T09:06:24.119 INFO:tasks.rados.rados.0.smithi163.stdout:277: done (1 left) 2023-10-09T09:06:24.119 INFO:tasks.rados.rados.0.smithi163.stdout:279: done (0 left) 2023-10-09T09:06:24.119 INFO:tasks.rados.rados.0.smithi163.stdout:280: snap_remove snap 24 2023-10-09T09:06:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:24 smithi163 bash[27500]: cluster 2023-10-09T09:06:23.107896+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2023-10-09T09:06:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:24 smithi163 bash[27500]: cluster 2023-10-09T09:06:23.268503+0000 mgr.y (mgr.24355) 201 : cluster [DBG] pgmap v231: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2023-10-09T09:06:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:24 smithi163 bash[27500]: audit 2023-10-09T09:06:24.068637+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.163:0/579730248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:24 smithi163 bash[33912]: cluster 2023-10-09T09:06:23.107896+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2023-10-09T09:06:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:24 smithi163 bash[33912]: cluster 2023-10-09T09:06:23.268503+0000 mgr.y (mgr.24355) 201 : cluster [DBG] pgmap v231: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2023-10-09T09:06:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:24 smithi163 bash[33912]: audit 2023-10-09T09:06:24.068637+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.163:0/579730248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:24 smithi181 bash[27712]: cluster 2023-10-09T09:06:23.107896+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2023-10-09T09:06:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:24 smithi181 bash[27712]: cluster 2023-10-09T09:06:23.268503+0000 mgr.y (mgr.24355) 201 : cluster [DBG] pgmap v231: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2023-10-09T09:06:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:24 smithi181 bash[27712]: audit 2023-10-09T09:06:24.068637+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.163:0/579730248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:25.120 INFO:tasks.rados.rados.0.smithi163.stdout:280: done (0 left) 2023-10-09T09:06:25.120 INFO:tasks.rados.rados.0.smithi163.stdout:281: rollback oid 49 current snap is 31 2023-10-09T09:06:25.120 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 49 to 22 2023-10-09T09:06:25.120 INFO:tasks.rados.rados.0.smithi163.stdout:282: snap_remove snap 25 2023-10-09T09:06:25.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:25 smithi163 bash[27500]: cluster 2023-10-09T09:06:24.113263+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2023-10-09T09:06:25.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:25 smithi163 bash[33912]: cluster 2023-10-09T09:06:24.113263+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2023-10-09T09:06:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:25 smithi181 bash[27712]: cluster 2023-10-09T09:06:24.113263+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2023-10-09T09:06:26.123 INFO:tasks.rados.rados.0.smithi163.stdout:282: done (1 left) 2023-10-09T09:06:26.124 INFO:tasks.rados.rados.0.smithi163.stdout:283: read oid 9 snap -1 2023-10-09T09:06:26.124 INFO:tasks.rados.rados.0.smithi163.stdout:283: expect (ObjNum 91 snap 15 seq_num 91) 2023-10-09T09:06:26.124 INFO:tasks.rados.rados.0.smithi163.stdout:284: snap_create 2023-10-09T09:06:26.124 INFO:tasks.rados.rados.0.smithi163.stdout:281: finishing rollback tid 0 to smithi16367185-49 2023-10-09T09:06:26.124 INFO:tasks.rados.rados.0.smithi163.stdout:281: finishing rollback tid 1 to smithi16367185-49 2023-10-09T09:06:26.124 INFO:tasks.rados.rados.0.smithi163.stdout:281: finishing rollback tid 2 to smithi16367185-49 2023-10-09T09:06:26.124 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 36 (ObjNum 113 snap 19 seq_num 113) dirty exists 2023-10-09T09:06:26.138 INFO:tasks.rados.rados.0.smithi163.stdout:283: expect (ObjNum 91 snap 15 seq_num 91) 2023-10-09T09:06:26.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:26 smithi163 bash[27500]: cluster 2023-10-09T09:06:25.119606+0000 mon.a (mon.0) 840 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2023-10-09T09:06:26.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:26 smithi163 bash[27500]: cluster 2023-10-09T09:06:25.269203+0000 mgr.y (mgr.24355) 202 : cluster [DBG] pgmap v234: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2023-10-09T09:06:26.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:26 smithi163 bash[33912]: cluster 2023-10-09T09:06:25.119606+0000 mon.a (mon.0) 840 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2023-10-09T09:06:26.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:26 smithi163 bash[33912]: cluster 2023-10-09T09:06:25.269203+0000 mgr.y (mgr.24355) 202 : cluster [DBG] pgmap v234: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2023-10-09T09:06:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:26 smithi181 bash[27712]: cluster 2023-10-09T09:06:25.119606+0000 mon.a (mon.0) 840 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2023-10-09T09:06:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:26 smithi181 bash[27712]: cluster 2023-10-09T09:06:25.269203+0000 mgr.y (mgr.24355) 202 : cluster [DBG] pgmap v234: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2023-10-09T09:06:27.099 INFO:tasks.rados.rados.0.smithi163.stdout:281: done (2 left) 2023-10-09T09:06:27.100 INFO:tasks.rados.rados.0.smithi163.stdout:283: done (1 left) 2023-10-09T09:06:27.100 INFO:tasks.rados.rados.0.smithi163.stdout:284: done (0 left) 2023-10-09T09:06:27.100 INFO:tasks.rados.rados.0.smithi163.stdout:285: snap_create 2023-10-09T09:06:27.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:27 smithi163 bash[27500]: cluster 2023-10-09T09:06:26.122632+0000 mon.a (mon.0) 841 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2023-10-09T09:06:27.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:27 smithi163 bash[27500]: audit 2023-10-09T09:06:26.486272+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.163:0/1201394182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:27.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:27 smithi163 bash[27500]: cluster 2023-10-09T09:06:27.099425+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2023-10-09T09:06:27.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:27 smithi163 bash[33912]: cluster 2023-10-09T09:06:26.122632+0000 mon.a (mon.0) 841 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2023-10-09T09:06:27.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:27 smithi163 bash[33912]: audit 2023-10-09T09:06:26.486272+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.163:0/1201394182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:27.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:27 smithi163 bash[33912]: cluster 2023-10-09T09:06:27.099425+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2023-10-09T09:06:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:27 smithi181 bash[27712]: cluster 2023-10-09T09:06:26.122632+0000 mon.a (mon.0) 841 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2023-10-09T09:06:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:27 smithi181 bash[27712]: audit 2023-10-09T09:06:26.486272+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.163:0/1201394182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:27 smithi181 bash[27712]: cluster 2023-10-09T09:06:27.099425+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2023-10-09T09:06:28.103 INFO:tasks.rados.rados.0.smithi163.stdout:285: done (0 left) 2023-10-09T09:06:28.103 INFO:tasks.rados.rados.0.smithi163.stdout:286: snap_create 2023-10-09T09:06:28.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:28 smithi163 bash[33912]: cluster 2023-10-09T09:06:27.269886+0000 mgr.y (mgr.24355) 203 : cluster [DBG] pgmap v237: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:28.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:28 smithi163 bash[33912]: cluster 2023-10-09T09:06:28.103527+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2023-10-09T09:06:28.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:28 smithi163 bash[27500]: cluster 2023-10-09T09:06:27.269886+0000 mgr.y (mgr.24355) 203 : cluster [DBG] pgmap v237: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:28.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:28 smithi163 bash[27500]: cluster 2023-10-09T09:06:28.103527+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2023-10-09T09:06:28.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:28 smithi181 bash[27712]: cluster 2023-10-09T09:06:27.269886+0000 mgr.y (mgr.24355) 203 : cluster [DBG] pgmap v237: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:28.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:28 smithi181 bash[27712]: cluster 2023-10-09T09:06:28.103527+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2023-10-09T09:06:28.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:28 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:06:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:06:29.104 INFO:tasks.rados.rados.0.smithi163.stdout:286: done (0 left) 2023-10-09T09:06:29.104 INFO:tasks.rados.rados.0.smithi163.stdout:287: write oid 13 current snap is 34 2023-10-09T09:06:29.105 INFO:tasks.rados.rados.0.smithi163.stdout:287: seq_num 142 ranges {786594=779712,2326443=60151} 2023-10-09T09:06:29.115 INFO:tasks.rados.rados.0.smithi163.stdout:287: writing smithi16367185-13 from 786594 to 1566306 tid 1 2023-10-09T09:06:29.120 INFO:tasks.rados.rados.0.smithi163.stdout:287: writing smithi16367185-13 from 2326443 to 2386594 tid 2 2023-10-09T09:06:29.120 INFO:tasks.rados.rados.0.smithi163.stdout:288: write oid 29 current snap is 34 2023-10-09T09:06:29.120 INFO:tasks.rados.rados.0.smithi163.stdout:288: seq_num 143 ranges {403401=454961,1584925=506039} 2023-10-09T09:06:29.125 INFO:tasks.rados.rados.0.smithi163.stdout:288: writing smithi16367185-29 from 403401 to 858362 tid 1 2023-10-09T09:06:29.132 INFO:tasks.rados.rados.0.smithi163.stdout:288: writing smithi16367185-29 from 1584925 to 2090964 tid 2 2023-10-09T09:06:29.133 INFO:tasks.rados.rados.0.smithi163.stdout:289: read oid 30 snap -1 2023-10-09T09:06:29.133 INFO:tasks.rados.rados.0.smithi163.stdout:289: expect (ObjNum 29 snap 0 seq_num 29) 2023-10-09T09:06:29.133 INFO:tasks.rados.rados.0.smithi163.stdout:290: delete oid 22 current snap is 34 2023-10-09T09:06:29.133 INFO:tasks.rados.rados.0.smithi163.stdout:287: finishing write tid 1 to smithi16367185-13 2023-10-09T09:06:29.133 INFO:tasks.rados.rados.0.smithi163.stdout:288: finishing write tid 1 to smithi16367185-29 2023-10-09T09:06:29.135 INFO:tasks.rados.rados.0.smithi163.stdout:290: done (3 left) 2023-10-09T09:06:29.136 INFO:tasks.rados.rados.0.smithi163.stdout:291: write oid 28 current snap is 34 2023-10-09T09:06:29.136 INFO:tasks.rados.rados.0.smithi163.stdout:291: seq_num 145 ranges {437015=721812,1701889=509708} 2023-10-09T09:06:29.142 INFO:tasks.rados.rados.0.smithi163.stdout:291: writing smithi16367185-28 from 437015 to 1158827 tid 1 2023-10-09T09:06:29.149 INFO:tasks.rados.rados.0.smithi163.stdout:291: writing smithi16367185-28 from 1701889 to 2211597 tid 2 2023-10-09T09:06:29.149 INFO:tasks.rados.rados.0.smithi163.stdout:292: delete oid 22 current snap is 34 2023-10-09T09:06:29.149 INFO:tasks.rados.rados.0.smithi163.stdout:289: expect (ObjNum 29 snap 0 seq_num 29) 2023-10-09T09:06:29.235 INFO:tasks.rados.rados.0.smithi163.stdout:288: finishing write tid 2 to smithi16367185-29 2023-10-09T09:06:29.235 INFO:tasks.rados.rados.0.smithi163.stdout:288: finishing write tid 3 to smithi16367185-29 2023-10-09T09:06:29.235 INFO:tasks.rados.rados.0.smithi163.stdout:288: finishing write tid 4 to smithi16367185-29 2023-10-09T09:06:29.235 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 97 (ObjNum 143 snap 34 seq_num 143) dirty exists 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:288: left oid 29 (ObjNum 143 snap 34 seq_num 143) 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:291: finishing write tid 1 to smithi16367185-28 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:288: done (4 left) 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:289: done (3 left) 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:292: done (2 left) 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:293: write oid 40 current snap is 34 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:291: finishing write tid 2 to smithi16367185-28 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:291: finishing write tid 3 to smithi16367185-28 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:291: finishing write tid 4 to smithi16367185-28 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 133 (ObjNum 145 snap 34 seq_num 145) dirty exists 2023-10-09T09:06:29.236 INFO:tasks.rados.rados.0.smithi163.stdout:291: left oid 28 (ObjNum 145 snap 34 seq_num 145) 2023-10-09T09:06:29.237 INFO:tasks.rados.rados.0.smithi163.stdout:287: finishing write tid 2 to smithi16367185-13 2023-10-09T09:06:29.237 INFO:tasks.rados.rados.0.smithi163.stdout:293: seq_num 147 ranges {470629=472310,1418853=513377} 2023-10-09T09:06:29.240 INFO:tasks.rados.rados.0.smithi163.stdout:293: writing smithi16367185-40 from 470629 to 942939 tid 1 2023-10-09T09:06:29.247 INFO:tasks.rados.rados.0.smithi163.stdout:293: writing smithi16367185-40 from 1418853 to 1932230 tid 2 2023-10-09T09:06:29.247 INFO:tasks.rados.rados.0.smithi163.stdout:291: done (2 left) 2023-10-09T09:06:29.247 INFO:tasks.rados.rados.0.smithi163.stdout:294: write oid 11 current snap is 34 2023-10-09T09:06:29.248 INFO:tasks.rados.rados.0.smithi163.stdout:287: finishing write tid 3 to smithi16367185-13 2023-10-09T09:06:29.248 INFO:tasks.rados.rados.0.smithi163.stdout:287: finishing write tid 4 to smithi16367185-13 2023-10-09T09:06:29.248 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 39 (ObjNum 142 snap 34 seq_num 142) dirty exists 2023-10-09T09:06:29.248 INFO:tasks.rados.rados.0.smithi163.stdout:287: left oid 13 (ObjNum 142 snap 34 seq_num 142) 2023-10-09T09:06:29.248 INFO:tasks.rados.rados.0.smithi163.stdout:294: seq_num 148 ranges {487436=547559,1760982=657035} 2023-10-09T09:06:29.253 INFO:tasks.rados.rados.0.smithi163.stdout:294: writing smithi16367185-11 from 487436 to 1034995 tid 1 2023-10-09T09:06:29.261 INFO:tasks.rados.rados.0.smithi163.stdout:294: writing smithi16367185-11 from 1760982 to 2418017 tid 2 2023-10-09T09:06:29.262 INFO:tasks.rados.rados.0.smithi163.stdout:287: done (2 left) 2023-10-09T09:06:29.262 INFO:tasks.rados.rados.0.smithi163.stdout:295: rollback oid 7 current snap is 34 2023-10-09T09:06:29.262 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 30 2023-10-09T09:06:29.262 INFO:tasks.rados.rados.0.smithi163.stdout:293: finishing write tid 1 to smithi16367185-40 2023-10-09T09:06:29.262 INFO:tasks.rados.rados.0.smithi163.stdout:293: finishing write tid 2 to smithi16367185-40 2023-10-09T09:06:29.262 INFO:tasks.rados.rados.0.smithi163.stdout:293: finishing write tid 3 to smithi16367185-40 2023-10-09T09:06:29.262 INFO:tasks.rados.rados.0.smithi163.stdout:293: finishing write tid 4 to smithi16367185-40 2023-10-09T09:06:29.262 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 112 (ObjNum 147 snap 34 seq_num 147) dirty exists 2023-10-09T09:06:29.262 INFO:tasks.rados.rados.0.smithi163.stdout:293: left oid 40 (ObjNum 147 snap 34 seq_num 147) 2023-10-09T09:06:29.262 INFO:tasks.rados.rados.0.smithi163.stdout:293: done (2 left) 2023-10-09T09:06:29.263 INFO:tasks.rados.rados.0.smithi163.stdout:296: rollback oid 10 current snap is 34 2023-10-09T09:06:29.263 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 20 2023-10-09T09:06:29.263 INFO:tasks.rados.rados.0.smithi163.stdout:294: finishing write tid 1 to smithi16367185-11 2023-10-09T09:06:29.263 INFO:tasks.rados.rados.0.smithi163.stdout:297: delete oid 8 current snap is 34 2023-10-09T09:06:29.263 INFO:tasks.rados.rados.0.smithi163.stdout:296: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:06:29.263 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 0 (ObjNum 2 snap 0 seq_num 1) dirty dne 2023-10-09T09:06:29.264 INFO:tasks.rados.rados.0.smithi163.stdout:296: done (3 left) 2023-10-09T09:06:29.265 INFO:tasks.rados.rados.0.smithi163.stdout:297: done (2 left) 2023-10-09T09:06:29.265 INFO:tasks.rados.rados.0.smithi163.stdout:298: snap_remove snap 28 2023-10-09T09:06:29.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:29 smithi163 bash[33912]: audit 2023-10-09T09:06:28.894876+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.163:0/4015859518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:29.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:29 smithi163 bash[33912]: cluster 2023-10-09T09:06:29.104079+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2023-10-09T09:06:29.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:29 smithi163 bash[27500]: audit 2023-10-09T09:06:28.894876+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.163:0/4015859518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:29.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:29 smithi163 bash[27500]: cluster 2023-10-09T09:06:29.104079+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2023-10-09T09:06:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:29 smithi181 bash[27712]: audit 2023-10-09T09:06:28.894876+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.163:0/4015859518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:29 smithi181 bash[27712]: cluster 2023-10-09T09:06:29.104079+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2023-10-09T09:06:30.163 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:29 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:06:29] "GET /metrics HTTP/1.1" 200 194750 "" "Prometheus/2.33.4" 2023-10-09T09:06:30.166 INFO:tasks.rados.rados.0.smithi163.stdout:295: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:06:30.166 INFO:tasks.rados.rados.0.smithi163.stdout:295: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:06:30.166 INFO:tasks.rados.rados.0.smithi163.stdout:294: finishing write tid 2 to smithi16367185-11 2023-10-09T09:06:30.166 INFO:tasks.rados.rados.0.smithi163.stdout:295: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:06:30.166 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 25 (ObjNum 120 snap 22 seq_num 120) dirty exists 2023-10-09T09:06:30.167 INFO:tasks.rados.rados.0.smithi163.stdout:294: finishing write tid 3 to smithi16367185-11 2023-10-09T09:06:30.167 INFO:tasks.rados.rados.0.smithi163.stdout:294: finishing write tid 4 to smithi16367185-11 2023-10-09T09:06:30.167 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 45 (ObjNum 148 snap 34 seq_num 148) dirty exists 2023-10-09T09:06:30.167 INFO:tasks.rados.rados.0.smithi163.stdout:294: left oid 11 (ObjNum 148 snap 34 seq_num 148) 2023-10-09T09:06:30.167 INFO:tasks.rados.rados.0.smithi163.stdout:294: done (2 left) 2023-10-09T09:06:30.167 INFO:tasks.rados.rados.0.smithi163.stdout:295: done (1 left) 2023-10-09T09:06:30.167 INFO:tasks.rados.rados.0.smithi163.stdout:298: done (0 left) 2023-10-09T09:06:30.167 INFO:tasks.rados.rados.0.smithi163.stdout:299: snap_remove snap 32 2023-10-09T09:06:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:30 smithi163 bash[27500]: cluster 2023-10-09T09:06:29.270351+0000 mgr.y (mgr.24355) 204 : cluster [DBG] pgmap v240: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:06:30.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:30 smithi163 bash[33912]: cluster 2023-10-09T09:06:29.270351+0000 mgr.y (mgr.24355) 204 : cluster [DBG] pgmap v240: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:06:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:30 smithi181 bash[27712]: cluster 2023-10-09T09:06:29.270351+0000 mgr.y (mgr.24355) 204 : cluster [DBG] pgmap v240: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:06:31.174 INFO:tasks.rados.rados.0.smithi163.stdout:299: done (0 left) 2023-10-09T09:06:31.174 INFO:tasks.rados.rados.0.smithi163.stdout:300: rollback oid 36 current snap is 34 2023-10-09T09:06:31.175 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 36 to 30 2023-10-09T09:06:31.175 INFO:tasks.rados.rados.0.smithi163.stdout:301: snap_create 2023-10-09T09:06:31.178 INFO:tasks.rados.rados.0.smithi163.stdout:300: finishing rollback tid 0 to smithi16367185-36 2023-10-09T09:06:31.178 INFO:tasks.rados.rados.0.smithi163.stdout:300: finishing rollback tid 1 to smithi16367185-36 2023-10-09T09:06:31.178 INFO:tasks.rados.rados.0.smithi163.stdout:300: finishing rollback tid 2 to smithi16367185-36 2023-10-09T09:06:31.178 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 74 (ObjNum 134 snap 28 seq_num 134) dirty exists 2023-10-09T09:06:31.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:31 smithi163 bash[27500]: cluster 2023-10-09T09:06:30.165962+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2023-10-09T09:06:31.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:31 smithi163 bash[33912]: cluster 2023-10-09T09:06:30.165962+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2023-10-09T09:06:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:31 smithi181 bash[27712]: cluster 2023-10-09T09:06:30.165962+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2023-10-09T09:06:32.103 INFO:tasks.rados.rados.0.smithi163.stdout:300: done (1 left) 2023-10-09T09:06:32.103 INFO:tasks.rados.rados.0.smithi163.stdout:301: done (0 left) 2023-10-09T09:06:32.103 INFO:tasks.rados.rados.0.smithi163.stdout:302: write oid 3 current snap is 35 2023-10-09T09:06:32.103 INFO:tasks.rados.rados.0.smithi163.stdout:302: seq_num 150 ranges {521050=698057,1877946=643104} 2023-10-09T09:06:32.110 INFO:tasks.rados.rados.0.smithi163.stdout:302: writing smithi16367185-3 from 521050 to 1219107 tid 1 2023-10-09T09:06:32.119 INFO:tasks.rados.rados.0.smithi163.stdout:302: writing smithi16367185-3 from 1877946 to 2521050 tid 2 2023-10-09T09:06:32.119 INFO:tasks.rados.rados.0.smithi163.stdout:303: read oid 25 snap 30 2023-10-09T09:06:32.119 INFO:tasks.rados.rados.0.smithi163.stdout:303: expect (ObjNum 60 snap 0 seq_num 60) 2023-10-09T09:06:32.119 INFO:tasks.rados.rados.0.smithi163.stdout:304: snap_create 2023-10-09T09:06:32.119 INFO:tasks.rados.rados.0.smithi163.stdout:302: finishing write tid 1 to smithi16367185-3 2023-10-09T09:06:32.124 INFO:tasks.rados.rados.0.smithi163.stdout:303: expect (ObjNum 60 snap 0 seq_num 60) 2023-10-09T09:06:32.264 INFO:tasks.rados.rados.0.smithi163.stdout:302: finishing write tid 2 to smithi16367185-3 2023-10-09T09:06:32.264 INFO:tasks.rados.rados.0.smithi163.stdout:302: finishing write tid 3 to smithi16367185-3 2023-10-09T09:06:32.264 INFO:tasks.rados.rados.0.smithi163.stdout:302: finishing write tid 4 to smithi16367185-3 2023-10-09T09:06:32.264 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 78 (ObjNum 150 snap 35 seq_num 150) dirty exists 2023-10-09T09:06:32.264 INFO:tasks.rados.rados.0.smithi163.stdout:302: left oid 3 (ObjNum 150 snap 35 seq_num 150) 2023-10-09T09:06:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:32 smithi163 bash[33912]: cluster 2023-10-09T09:06:31.172656+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2023-10-09T09:06:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:32 smithi163 bash[33912]: cluster 2023-10-09T09:06:31.270786+0000 mgr.y (mgr.24355) 205 : cluster [DBG] pgmap v243: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:06:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:32 smithi163 bash[33912]: audit 2023-10-09T09:06:31.306776+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.163:0/3746550627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:32 smithi163 bash[33912]: cluster 2023-10-09T09:06:32.102455+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2023-10-09T09:06:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:32 smithi163 bash[27500]: cluster 2023-10-09T09:06:31.172656+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2023-10-09T09:06:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:32 smithi163 bash[27500]: cluster 2023-10-09T09:06:31.270786+0000 mgr.y (mgr.24355) 205 : cluster [DBG] pgmap v243: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:06:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:32 smithi163 bash[27500]: audit 2023-10-09T09:06:31.306776+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.163:0/3746550627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:32.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:32 smithi163 bash[27500]: cluster 2023-10-09T09:06:32.102455+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2023-10-09T09:06:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:32 smithi181 bash[27712]: cluster 2023-10-09T09:06:31.172656+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2023-10-09T09:06:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:32 smithi181 bash[27712]: cluster 2023-10-09T09:06:31.270786+0000 mgr.y (mgr.24355) 205 : cluster [DBG] pgmap v243: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:06:32.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:32 smithi181 bash[27712]: audit 2023-10-09T09:06:31.306776+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.163:0/3746550627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:32.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:32 smithi181 bash[27712]: cluster 2023-10-09T09:06:32.102455+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2023-10-09T09:06:33.116 INFO:tasks.rados.rados.0.smithi163.stdout:302: done (2 left) 2023-10-09T09:06:33.116 INFO:tasks.rados.rados.0.smithi163.stdout:303: done (1 left) 2023-10-09T09:06:33.116 INFO:tasks.rados.rados.0.smithi163.stdout:304: done (0 left) 2023-10-09T09:06:33.116 INFO:tasks.rados.rados.0.smithi163.stdout:305: snap_remove snap 34 2023-10-09T09:06:34.117 INFO:tasks.rados.rados.0.smithi163.stdout:305: done (0 left) 2023-10-09T09:06:34.117 INFO:tasks.rados.rados.0.smithi163.stdout:306: read oid 30 snap -1 2023-10-09T09:06:34.117 INFO:tasks.rados.rados.0.smithi163.stdout:306: expect (ObjNum 29 snap 0 seq_num 29) 2023-10-09T09:06:34.117 INFO:tasks.rados.rados.0.smithi163.stdout:307: write oid 9 current snap is 36 2023-10-09T09:06:34.117 INFO:tasks.rados.rados.0.smithi163.stdout:307: seq_num 151 ranges {537857=773306,1936428=520715} 2023-10-09T09:06:34.125 INFO:tasks.rados.rados.0.smithi163.stdout:307: writing smithi16367185-9 from 537857 to 1311163 tid 1 2023-10-09T09:06:34.132 INFO:tasks.rados.rados.0.smithi163.stdout:307: writing smithi16367185-9 from 1936428 to 2457143 tid 2 2023-10-09T09:06:34.132 INFO:tasks.rados.rados.0.smithi163.stdout:308: delete oid 1 current snap is 36 2023-10-09T09:06:34.133 INFO:tasks.rados.rados.0.smithi163.stdout:306: expect (ObjNum 29 snap 0 seq_num 29) 2023-10-09T09:06:34.220 INFO:tasks.rados.rados.0.smithi163.stdout:307: finishing write tid 1 to smithi16367185-9 2023-10-09T09:06:34.220 INFO:tasks.rados.rados.0.smithi163.stdout:307: finishing write tid 2 to smithi16367185-9 2023-10-09T09:06:34.220 INFO:tasks.rados.rados.0.smithi163.stdout:307: finishing write tid 3 to smithi16367185-9 2023-10-09T09:06:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:307: finishing write tid 4 to smithi16367185-9 2023-10-09T09:06:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 50 (ObjNum 151 snap 36 seq_num 151) dirty exists 2023-10-09T09:06:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:307: left oid 9 (ObjNum 151 snap 36 seq_num 151) 2023-10-09T09:06:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:306: done (2 left) 2023-10-09T09:06:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:307: done (1 left) 2023-10-09T09:06:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:308: done (0 left) 2023-10-09T09:06:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:309: write oid 49 current snap is 36 2023-10-09T09:06:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:309: seq_num 153 ranges {571471=640157,1653392=524384} 2023-10-09T09:06:34.233 INFO:tasks.rados.rados.0.smithi163.stdout:309: writing smithi16367185-49 from 571471 to 1211628 tid 1 2023-10-09T09:06:34.239 INFO:tasks.rados.rados.0.smithi163.stdout:309: writing smithi16367185-49 from 1653392 to 2177776 tid 2 2023-10-09T09:06:34.239 INFO:tasks.rados.rados.0.smithi163.stdout:310: snap_create 2023-10-09T09:06:34.242 INFO:tasks.rados.rados.0.smithi163.stdout:309: finishing write tid 1 to smithi16367185-49 2023-10-09T09:06:34.243 INFO:tasks.rados.rados.0.smithi163.stdout:309: finishing write tid 2 to smithi16367185-49 2023-10-09T09:06:34.243 INFO:tasks.rados.rados.0.smithi163.stdout:309: finishing write tid 3 to smithi16367185-49 2023-10-09T09:06:34.244 INFO:tasks.rados.rados.0.smithi163.stdout:309: finishing write tid 4 to smithi16367185-49 2023-10-09T09:06:34.244 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 44 (ObjNum 153 snap 36 seq_num 153) dirty exists 2023-10-09T09:06:34.244 INFO:tasks.rados.rados.0.smithi163.stdout:309: left oid 49 (ObjNum 153 snap 36 seq_num 153) 2023-10-09T09:06:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:34 smithi163 bash[27500]: cluster 2023-10-09T09:06:33.115800+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2023-10-09T09:06:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:34 smithi163 bash[27500]: cluster 2023-10-09T09:06:33.271931+0000 mgr.y (mgr.24355) 206 : cluster [DBG] pgmap v246: 89 pgs: 89 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2023-10-09T09:06:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:34 smithi163 bash[27500]: audit 2023-10-09T09:06:33.715561+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.163:0/819274949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:34 smithi163 bash[33912]: cluster 2023-10-09T09:06:33.115800+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2023-10-09T09:06:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:34 smithi163 bash[33912]: cluster 2023-10-09T09:06:33.271931+0000 mgr.y (mgr.24355) 206 : cluster [DBG] pgmap v246: 89 pgs: 89 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2023-10-09T09:06:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:34 smithi163 bash[33912]: audit 2023-10-09T09:06:33.715561+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.163:0/819274949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:34 smithi181 bash[27712]: cluster 2023-10-09T09:06:33.115800+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2023-10-09T09:06:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:34 smithi181 bash[27712]: cluster 2023-10-09T09:06:33.271931+0000 mgr.y (mgr.24355) 206 : cluster [DBG] pgmap v246: 89 pgs: 89 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2023-10-09T09:06:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:34 smithi181 bash[27712]: audit 2023-10-09T09:06:33.715561+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.163:0/819274949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:35.137 INFO:tasks.rados.rados.0.smithi163.stdout:309: done (1 left) 2023-10-09T09:06:35.137 INFO:tasks.rados.rados.0.smithi163.stdout:310: done (0 left) 2023-10-09T09:06:35.137 INFO:tasks.rados.rados.0.smithi163.stdout:311: snap_create 2023-10-09T09:06:35.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:35 smithi163 bash[33912]: cluster 2023-10-09T09:06:34.116526+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2023-10-09T09:06:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:35 smithi163 bash[27500]: cluster 2023-10-09T09:06:34.116526+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2023-10-09T09:06:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:35 smithi181 bash[27712]: cluster 2023-10-09T09:06:34.116526+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2023-10-09T09:06:36.142 INFO:tasks.rados.rados.0.smithi163.stdout:311: done (0 left) 2023-10-09T09:06:36.142 INFO:tasks.rados.rados.0.smithi163.stdout:312: read oid 35 snap -1 2023-10-09T09:06:36.142 INFO:tasks.rados.rados.0.smithi163.stdout:312: expect deleted 2023-10-09T09:06:36.142 INFO:tasks.rados.rados.0.smithi163.stdout:313: delete oid 29 current snap is 38 2023-10-09T09:06:36.148 INFO:tasks.rados.rados.0.smithi163.stdout:312: done (1 left) 2023-10-09T09:06:36.149 INFO:tasks.rados.rados.0.smithi163.stdout:313: done (0 left) 2023-10-09T09:06:36.149 INFO:tasks.rados.rados.0.smithi163.stdout:314: snap_remove snap 22 2023-10-09T09:06:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:36 smithi163 bash[27500]: cluster 2023-10-09T09:06:35.137464+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2023-10-09T09:06:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:36 smithi163 bash[27500]: cluster 2023-10-09T09:06:35.272580+0000 mgr.y (mgr.24355) 207 : cluster [DBG] pgmap v249: 89 pgs: 89 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2023-10-09T09:06:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:36 smithi163 bash[27500]: audit 2023-10-09T09:06:36.128951+0000 mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.163:0/3769273701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:36 smithi163 bash[33912]: cluster 2023-10-09T09:06:35.137464+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2023-10-09T09:06:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:36 smithi163 bash[33912]: cluster 2023-10-09T09:06:35.272580+0000 mgr.y (mgr.24355) 207 : cluster [DBG] pgmap v249: 89 pgs: 89 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2023-10-09T09:06:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:36 smithi163 bash[33912]: audit 2023-10-09T09:06:36.128951+0000 mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.163:0/3769273701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:36.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:36 smithi181 bash[27712]: cluster 2023-10-09T09:06:35.137464+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2023-10-09T09:06:36.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:36 smithi181 bash[27712]: cluster 2023-10-09T09:06:35.272580+0000 mgr.y (mgr.24355) 207 : cluster [DBG] pgmap v249: 89 pgs: 89 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2023-10-09T09:06:36.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:36 smithi181 bash[27712]: audit 2023-10-09T09:06:36.128951+0000 mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.163:0/3769273701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:37.103 INFO:tasks.rados.rados.0.smithi163.stdout:314: done (0 left) 2023-10-09T09:06:37.103 INFO:tasks.rados.rados.0.smithi163.stdout:315: snap_remove snap 30 2023-10-09T09:06:37.396 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:37 smithi181 bash[27712]: cluster 2023-10-09T09:06:36.140393+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2023-10-09T09:06:37.396 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:37 smithi181 bash[27712]: cluster 2023-10-09T09:06:37.102922+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2023-10-09T09:06:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:37 smithi163 bash[33912]: cluster 2023-10-09T09:06:36.140393+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2023-10-09T09:06:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:37 smithi163 bash[33912]: cluster 2023-10-09T09:06:37.102922+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2023-10-09T09:06:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:37 smithi163 bash[27500]: cluster 2023-10-09T09:06:36.140393+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2023-10-09T09:06:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:37 smithi163 bash[27500]: cluster 2023-10-09T09:06:37.102922+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2023-10-09T09:06:38.107 INFO:tasks.rados.rados.0.smithi163.stdout:315: done (0 left) 2023-10-09T09:06:38.107 INFO:tasks.rados.rados.0.smithi163.stdout:316: write oid 29 current snap is 38 2023-10-09T09:06:38.107 INFO:tasks.rados.rados.0.smithi163.stdout:316: seq_num 155 ranges {605085=790655,2170356=434729} 2023-10-09T09:06:38.116 INFO:tasks.rados.rados.0.smithi163.stdout:316: writing smithi16367185-29 from 605085 to 1395740 tid 1 2023-10-09T09:06:38.124 INFO:tasks.rados.rados.0.smithi163.stdout:316: writing smithi16367185-29 from 2170356 to 2605085 tid 2 2023-10-09T09:06:38.124 INFO:tasks.rados.rados.0.smithi163.stdout:317: snap_create 2023-10-09T09:06:38.124 INFO:tasks.rados.rados.0.smithi163.stdout:316: finishing write tid 1 to smithi16367185-29 2023-10-09T09:06:38.126 INFO:tasks.rados.rados.0.smithi163.stdout:316: finishing write tid 2 to smithi16367185-29 2023-10-09T09:06:38.127 INFO:tasks.rados.rados.0.smithi163.stdout:316: finishing write tid 3 to smithi16367185-29 2023-10-09T09:06:38.127 INFO:tasks.rados.rados.0.smithi163.stdout:316: finishing write tid 4 to smithi16367185-29 2023-10-09T09:06:38.127 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 110 (ObjNum 155 snap 38 seq_num 155) dirty exists 2023-10-09T09:06:38.127 INFO:tasks.rados.rados.0.smithi163.stdout:316: left oid 29 (ObjNum 155 snap 38 seq_num 155) 2023-10-09T09:06:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[27500]: cephadm 2023-10-09T09:06:37.170631+0000 mgr.y (mgr.24355) 208 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:06:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[27500]: audit 2023-10-09T09:06:37.179012+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[27500]: audit 2023-10-09T09:06:37.180611+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:06:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[27500]: audit 2023-10-09T09:06:37.182087+0000 mon.a (mon.0) 857 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:06:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[27500]: audit 2023-10-09T09:06:37.183493+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:06:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[27500]: cephadm 2023-10-09T09:06:37.184460+0000 mgr.y (mgr.24355) 209 : cephadm [INF] Deploying daemon mgr.x on smithi181 2023-10-09T09:06:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[27500]: cluster 2023-10-09T09:06:37.273197+0000 mgr.y (mgr.24355) 210 : cluster [DBG] pgmap v252: 89 pgs: 89 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:38.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[27500]: cluster 2023-10-09T09:06:38.106673+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2023-10-09T09:06:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[33912]: cephadm 2023-10-09T09:06:37.170631+0000 mgr.y (mgr.24355) 208 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:06:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[33912]: audit 2023-10-09T09:06:37.179012+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[33912]: audit 2023-10-09T09:06:37.180611+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:06:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[33912]: audit 2023-10-09T09:06:37.182087+0000 mon.a (mon.0) 857 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:06:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[33912]: audit 2023-10-09T09:06:37.183493+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:06:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[33912]: cephadm 2023-10-09T09:06:37.184460+0000 mgr.y (mgr.24355) 209 : cephadm [INF] Deploying daemon mgr.x on smithi181 2023-10-09T09:06:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[33912]: cluster 2023-10-09T09:06:37.273197+0000 mgr.y (mgr.24355) 210 : cluster [DBG] pgmap v252: 89 pgs: 89 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:38 smithi163 bash[33912]: cluster 2023-10-09T09:06:38.106673+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2023-10-09T09:06:38.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:38 smithi181 bash[27712]: cephadm 2023-10-09T09:06:37.170631+0000 mgr.y (mgr.24355) 208 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:06:38.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:38 smithi181 bash[27712]: audit 2023-10-09T09:06:37.179012+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:38.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:38 smithi181 bash[27712]: audit 2023-10-09T09:06:37.180611+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:06:38.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:38 smithi181 bash[27712]: audit 2023-10-09T09:06:37.182087+0000 mon.a (mon.0) 857 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:06:38.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:38 smithi181 bash[27712]: audit 2023-10-09T09:06:37.183493+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:06:38.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:38 smithi181 bash[27712]: cephadm 2023-10-09T09:06:37.184460+0000 mgr.y (mgr.24355) 209 : cephadm [INF] Deploying daemon mgr.x on smithi181 2023-10-09T09:06:38.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:38 smithi181 bash[27712]: cluster 2023-10-09T09:06:37.273197+0000 mgr.y (mgr.24355) 210 : cluster [DBG] pgmap v252: 89 pgs: 89 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:38.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:38 smithi181 bash[27712]: cluster 2023-10-09T09:06:38.106673+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2023-10-09T09:06:38.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:38 smithi181 bash[28565]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:06:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:06:39.112 INFO:tasks.rados.rados.0.smithi163.stdout:316: done (1 left) 2023-10-09T09:06:39.112 INFO:tasks.rados.rados.0.smithi163.stdout:317: done (0 left) 2023-10-09T09:06:39.112 INFO:tasks.rados.rados.0.smithi163.stdout:318: write oid 7 current snap is 39 2023-10-09T09:06:39.112 INFO:tasks.rados.rados.0.smithi163.stdout:318: seq_num 156 ranges {621892=465904,1712485=671711} 2023-10-09T09:06:39.119 INFO:tasks.rados.rados.0.smithi163.stdout:318: writing smithi16367185-7 from 621892 to 1087796 tid 1 2023-10-09T09:06:39.127 INFO:tasks.rados.rados.0.smithi163.stdout:318: writing smithi16367185-7 from 1712485 to 2384196 tid 2 2023-10-09T09:06:39.127 INFO:tasks.rados.rados.0.smithi163.stdout:319: rollback oid 50 current snap is 39 2023-10-09T09:06:39.127 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 50 to 36 2023-10-09T09:06:39.127 INFO:tasks.rados.rados.0.smithi163.stdout:318: finishing write tid 1 to smithi16367185-7 2023-10-09T09:06:39.127 INFO:tasks.rados.rados.0.smithi163.stdout:320: read oid 36 snap -1 2023-10-09T09:06:39.127 INFO:tasks.rados.rados.0.smithi163.stdout:320: expect (ObjNum 134 snap 28 seq_num 134) 2023-10-09T09:06:39.127 INFO:tasks.rados.rados.0.smithi163.stdout:321: rollback oid 44 current snap is 39 2023-10-09T09:06:39.127 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 44 to 37 2023-10-09T09:06:39.127 INFO:tasks.rados.rados.0.smithi163.stdout:322: write oid 17 current snap is 39 2023-10-09T09:06:39.128 INFO:tasks.rados.rados.0.smithi163.stdout:322: seq_num 157 ranges {638699=541153,1770967=531722} 2023-10-09T09:06:39.134 INFO:tasks.rados.rados.0.smithi163.stdout:322: writing smithi16367185-17 from 638699 to 1179852 tid 1 2023-10-09T09:06:39.140 INFO:tasks.rados.rados.0.smithi163.stdout:322: writing smithi16367185-17 from 1770967 to 2302689 tid 2 2023-10-09T09:06:39.141 INFO:tasks.rados.rados.0.smithi163.stdout:323: snap_create 2023-10-09T09:06:39.141 INFO:tasks.rados.rados.0.smithi163.stdout:318: finishing write tid 2 to smithi16367185-7 2023-10-09T09:06:39.141 INFO:tasks.rados.rados.0.smithi163.stdout:318: finishing write tid 3 to smithi16367185-7 2023-10-09T09:06:39.141 INFO:tasks.rados.rados.0.smithi163.stdout:318: finishing write tid 4 to smithi16367185-7 2023-10-09T09:06:39.141 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 37 (ObjNum 156 snap 39 seq_num 156) dirty exists 2023-10-09T09:06:39.141 INFO:tasks.rados.rados.0.smithi163.stdout:318: left oid 7 (ObjNum 156 snap 39 seq_num 156) 2023-10-09T09:06:39.141 INFO:tasks.rados.rados.0.smithi163.stdout:320: expect (ObjNum 134 snap 28 seq_num 134) 2023-10-09T09:06:39.221 INFO:tasks.rados.rados.0.smithi163.stdout:321: finishing rollback tid 0 to smithi16367185-44 2023-10-09T09:06:39.221 INFO:tasks.rados.rados.0.smithi163.stdout:321: finishing rollback tid 1 to smithi16367185-44 2023-10-09T09:06:39.221 INFO:tasks.rados.rados.0.smithi163.stdout:319: finishing rollback tid 0 to smithi16367185-50 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:319: finishing rollback tid 1 to smithi16367185-50 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:319: finishing rollback tid 2 to smithi16367185-50 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 92 (ObjNum 94 snap 16 seq_num 94) dirty exists 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:321: finishing rollback tid 2 to smithi16367185-44 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 228 (ObjNum 129 snap 27 seq_num 129) dirty exists 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:322: finishing write tid 1 to smithi16367185-17 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:322: finishing write tid 2 to smithi16367185-17 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:322: finishing write tid 3 to smithi16367185-17 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:322: finishing write tid 4 to smithi16367185-17 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 134 (ObjNum 157 snap 39 seq_num 157) dirty exists 2023-10-09T09:06:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:322: left oid 17 (ObjNum 157 snap 39 seq_num 157) 2023-10-09T09:06:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:39 smithi181 bash[27712]: audit 2023-10-09T09:06:38.550152+0000 mon.a (mon.0) 860 : audit [DBG] from='client.? 172.21.15.163:0/4030056250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:39 smithi181 bash[27712]: cluster 2023-10-09T09:06:39.110414+0000 mon.a (mon.0) 861 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2023-10-09T09:06:39.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:39 smithi163 bash[27500]: audit 2023-10-09T09:06:38.550152+0000 mon.a (mon.0) 860 : audit [DBG] from='client.? 172.21.15.163:0/4030056250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:39.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:39 smithi163 bash[27500]: cluster 2023-10-09T09:06:39.110414+0000 mon.a (mon.0) 861 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2023-10-09T09:06:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:39 smithi163 bash[33912]: audit 2023-10-09T09:06:38.550152+0000 mon.a (mon.0) 860 : audit [DBG] from='client.? 172.21.15.163:0/4030056250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:39 smithi163 bash[33912]: cluster 2023-10-09T09:06:39.110414+0000 mon.a (mon.0) 861 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2023-10-09T09:06:39.651 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:39 smithi181 systemd[1]: Stopping Ceph mgr.x for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:06:39.652 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:39 smithi181 bash[50319]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mgr.x 2023-10-09T09:06:40.112 INFO:tasks.rados.rados.0.smithi163.stdout:318: done (5 left) 2023-10-09T09:06:40.112 INFO:tasks.rados.rados.0.smithi163.stdout:319: done (4 left) 2023-10-09T09:06:40.112 INFO:tasks.rados.rados.0.smithi163.stdout:320: done (3 left) 2023-10-09T09:06:40.112 INFO:tasks.rados.rados.0.smithi163.stdout:321: done (2 left) 2023-10-09T09:06:40.113 INFO:tasks.rados.rados.0.smithi163.stdout:322: done (1 left) 2023-10-09T09:06:40.113 INFO:tasks.rados.rados.0.smithi163.stdout:323: done (0 left) 2023-10-09T09:06:40.113 INFO:tasks.rados.rados.0.smithi163.stdout:324: write oid 40 current snap is 40 2023-10-09T09:06:40.113 INFO:tasks.rados.rados.0.smithi163.stdout:324: seq_num 158 ranges {655506=616402,1829449=675380} 2023-10-09T09:06:40.121 INFO:tasks.rados.rados.0.smithi163.stdout:324: writing smithi16367185-40 from 655506 to 1271908 tid 1 2023-10-09T09:06:40.129 INFO:tasks.rados.rados.0.smithi163.stdout:324: writing smithi16367185-40 from 1829449 to 2504829 tid 2 2023-10-09T09:06:40.129 INFO:tasks.rados.rados.0.smithi163.stdout:325: snap_create 2023-10-09T09:06:40.130 INFO:tasks.rados.rados.0.smithi163.stdout:324: finishing write tid 1 to smithi16367185-40 2023-10-09T09:06:40.134 INFO:tasks.rados.rados.0.smithi163.stdout:324: finishing write tid 2 to smithi16367185-40 2023-10-09T09:06:40.134 INFO:tasks.rados.rados.0.smithi163.stdout:324: finishing write tid 3 to smithi16367185-40 2023-10-09T09:06:40.134 INFO:tasks.rados.rados.0.smithi163.stdout:324: finishing write tid 4 to smithi16367185-40 2023-10-09T09:06:40.134 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 138 (ObjNum 158 snap 40 seq_num 158) dirty exists 2023-10-09T09:06:40.134 INFO:tasks.rados.rados.0.smithi163.stdout:324: left oid 40 (ObjNum 158 snap 40 seq_num 158) 2023-10-09T09:06:40.191 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:39 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:06:39] "GET /metrics HTTP/1.1" 200 194766 "" "Prometheus/2.33.4" 2023-10-09T09:06:40.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:39 smithi181 bash[50384]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T09:06:40.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:40 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:06:40.318 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:40 smithi181 bash[50482]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T09:06:40.318 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:40 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T09:06:40.318 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:40 smithi181 systemd[1]: Stopped Ceph mgr.x for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:06:40.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:40 smithi181 bash[27712]: cluster 2023-10-09T09:06:39.273813+0000 mgr.y (mgr.24355) 211 : cluster [DBG] pgmap v255: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 900 KiB/s wr, 6 op/s 2023-10-09T09:06:40.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:40 smithi181 bash[27712]: cluster 2023-10-09T09:06:40.112457+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2023-10-09T09:06:40.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:40 smithi163 bash[33912]: cluster 2023-10-09T09:06:39.273813+0000 mgr.y (mgr.24355) 211 : cluster [DBG] pgmap v255: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 900 KiB/s wr, 6 op/s 2023-10-09T09:06:40.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:40 smithi163 bash[33912]: cluster 2023-10-09T09:06:40.112457+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2023-10-09T09:06:40.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:40 smithi163 bash[27500]: cluster 2023-10-09T09:06:39.273813+0000 mgr.y (mgr.24355) 211 : cluster [DBG] pgmap v255: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 900 KiB/s wr, 6 op/s 2023-10-09T09:06:40.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:40 smithi163 bash[27500]: cluster 2023-10-09T09:06:40.112457+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2023-10-09T09:06:40.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:40 smithi181 systemd[1]: Started Ceph mgr.x for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:06:41.113 INFO:tasks.rados.rados.0.smithi163.stdout:324: done (1 left) 2023-10-09T09:06:41.113 INFO:tasks.rados.rados.0.smithi163.stdout:325: done (0 left) 2023-10-09T09:06:41.113 INFO:tasks.rados.rados.0.smithi163.stdout:326: write oid 33 current snap is 41 2023-10-09T09:06:41.113 INFO:tasks.rados.rados.0.smithi163.stdout:326: seq_num 159 ranges {672313=691651,1887931=419038} 2023-10-09T09:06:41.123 INFO:tasks.rados.rados.0.smithi163.stdout:326: writing smithi16367185-33 from 672313 to 1363964 tid 1 2023-10-09T09:06:41.129 INFO:tasks.rados.rados.0.smithi163.stdout:326: writing smithi16367185-33 from 1887931 to 2306969 tid 2 2023-10-09T09:06:41.129 INFO:tasks.rados.rados.0.smithi163.stdout:327: read oid 18 snap 23 2023-10-09T09:06:41.129 INFO:tasks.rados.rados.0.smithi163.stdout:327: expect (ObjNum 118 snap 20 seq_num 118) 2023-10-09T09:06:41.130 INFO:tasks.rados.rados.0.smithi163.stdout:328: read oid 48 snap 40 2023-10-09T09:06:41.130 INFO:tasks.rados.rados.0.smithi163.stdout:328: expect (ObjNum 128 snap 27 seq_num 128) 2023-10-09T09:06:41.130 INFO:tasks.rados.rados.0.smithi163.stdout:329: write oid 49 current snap is 41 2023-10-09T09:06:41.130 INFO:tasks.rados.rados.0.smithi163.stdout:329: seq_num 160 ranges {689120=483253,1946413=679049} 2023-10-09T09:06:41.136 INFO:tasks.rados.rados.0.smithi163.stdout:329: writing smithi16367185-49 from 689120 to 1172373 tid 1 2023-10-09T09:06:41.145 INFO:tasks.rados.rados.0.smithi163.stdout:329: writing smithi16367185-49 from 1946413 to 2625462 tid 2 2023-10-09T09:06:41.145 INFO:tasks.rados.rados.0.smithi163.stdout:330: snap_create 2023-10-09T09:06:41.145 INFO:tasks.rados.rados.0.smithi163.stdout:326: finishing write tid 1 to smithi16367185-33 2023-10-09T09:06:41.145 INFO:tasks.rados.rados.0.smithi163.stdout:326: finishing write tid 2 to smithi16367185-33 2023-10-09T09:06:41.145 INFO:tasks.rados.rados.0.smithi163.stdout:326: finishing write tid 3 to smithi16367185-33 2023-10-09T09:06:41.145 INFO:tasks.rados.rados.0.smithi163.stdout:326: finishing write tid 4 to smithi16367185-33 2023-10-09T09:06:41.145 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 113 (ObjNum 159 snap 41 seq_num 159) dirty exists 2023-10-09T09:06:41.145 INFO:tasks.rados.rados.0.smithi163.stdout:326: left oid 33 (ObjNum 159 snap 41 seq_num 159) 2023-10-09T09:06:41.145 INFO:tasks.rados.rados.0.smithi163.stdout:328: expect (ObjNum 128 snap 27 seq_num 128) 2023-10-09T09:06:41.305 INFO:tasks.rados.rados.0.smithi163.stdout:329: finishing write tid 1 to smithi16367185-49 2023-10-09T09:06:41.305 INFO:tasks.rados.rados.0.smithi163.stdout:327: expect (ObjNum 118 snap 20 seq_num 118) 2023-10-09T09:06:41.445 INFO:tasks.rados.rados.0.smithi163.stdout:329: finishing write tid 2 to smithi16367185-49 2023-10-09T09:06:41.445 INFO:tasks.rados.rados.0.smithi163.stdout:329: finishing write tid 3 to smithi16367185-49 2023-10-09T09:06:41.445 INFO:tasks.rados.rados.0.smithi163.stdout:329: finishing write tid 4 to smithi16367185-49 2023-10-09T09:06:41.445 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 52 (ObjNum 160 snap 41 seq_num 160) dirty exists 2023-10-09T09:06:41.445 INFO:tasks.rados.rados.0.smithi163.stdout:329: left oid 49 (ObjNum 160 snap 41 seq_num 160) 2023-10-09T09:06:41.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:41 smithi181 bash[50540]: debug 2023-10-09T09:06:41.467+0000 7f335a5ca700 1 -- 172.21.15.181:0/1222903126 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560d9fae71e0 con 0x560da0866800 2023-10-09T09:06:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:41 smithi181 bash[27712]: audit 2023-10-09T09:06:40.462268+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:41.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:41 smithi181 bash[27712]: audit 2023-10-09T09:06:40.463707+0000 mon.a (mon.0) 864 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:41.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:41 smithi181 bash[27712]: audit 2023-10-09T09:06:40.950382+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.163:0/3985016425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:41.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:41 smithi181 bash[27712]: cluster 2023-10-09T09:06:41.113391+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2023-10-09T09:06:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:41 smithi163 bash[33912]: audit 2023-10-09T09:06:40.462268+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:41 smithi163 bash[33912]: audit 2023-10-09T09:06:40.463707+0000 mon.a (mon.0) 864 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:41 smithi163 bash[33912]: audit 2023-10-09T09:06:40.950382+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.163:0/3985016425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:41 smithi163 bash[33912]: cluster 2023-10-09T09:06:41.113391+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2023-10-09T09:06:41.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:41 smithi163 bash[27500]: audit 2023-10-09T09:06:40.462268+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:41 smithi163 bash[27500]: audit 2023-10-09T09:06:40.463707+0000 mon.a (mon.0) 864 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:41 smithi163 bash[27500]: audit 2023-10-09T09:06:40.950382+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.163:0/3985016425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:41 smithi163 bash[27500]: cluster 2023-10-09T09:06:41.113391+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2023-10-09T09:06:42.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:41 smithi181 bash[50540]: debug 2023-10-09T09:06:41.631+0000 7f3364718200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:06:42.107 INFO:tasks.rados.rados.0.smithi163.stdout:326: done (4 left) 2023-10-09T09:06:42.107 INFO:tasks.rados.rados.0.smithi163.stdout:327: done (3 left) 2023-10-09T09:06:42.107 INFO:tasks.rados.rados.0.smithi163.stdout:328: done (2 left) 2023-10-09T09:06:42.107 INFO:tasks.rados.rados.0.smithi163.stdout:329: done (1 left) 2023-10-09T09:06:42.107 INFO:tasks.rados.rados.0.smithi163.stdout:330: done (0 left) 2023-10-09T09:06:42.107 INFO:tasks.rados.rados.0.smithi163.stdout:331: snap_remove snap 39 2023-10-09T09:06:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:42 smithi163 bash[33912]: cluster 2023-10-09T09:06:41.274290+0000 mgr.y (mgr.24355) 212 : cluster [DBG] pgmap v258: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 900 KiB/s wr, 6 op/s 2023-10-09T09:06:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:42 smithi163 bash[33912]: cluster 2023-10-09T09:06:42.105986+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2023-10-09T09:06:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:42 smithi163 bash[27500]: cluster 2023-10-09T09:06:41.274290+0000 mgr.y (mgr.24355) 212 : cluster [DBG] pgmap v258: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 900 KiB/s wr, 6 op/s 2023-10-09T09:06:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:42 smithi163 bash[27500]: cluster 2023-10-09T09:06:42.105986+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2023-10-09T09:06:42.777 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:42 smithi181 bash[50540]: debug 2023-10-09T09:06:42.559+0000 7f3364718200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:06:42.777 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:42 smithi181 bash[50540]: debug 2023-10-09T09:06:42.663+0000 7f3364718200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:06:42.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:42 smithi181 bash[27712]: cluster 2023-10-09T09:06:41.274290+0000 mgr.y (mgr.24355) 212 : cluster [DBG] pgmap v258: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 900 KiB/s wr, 6 op/s 2023-10-09T09:06:42.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:42 smithi181 bash[27712]: cluster 2023-10-09T09:06:42.105986+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2023-10-09T09:06:43.045 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:42 smithi181 bash[50540]: debug 2023-10-09T09:06:42.771+0000 7f3364718200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:06:43.118 INFO:tasks.rados.rados.0.smithi163.stdout:331: done (0 left) 2023-10-09T09:06:43.118 INFO:tasks.rados.rados.0.smithi163.stdout:332: rollback oid 33 current snap is 42 2023-10-09T09:06:43.118 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 38 2023-10-09T09:06:43.119 INFO:tasks.rados.rados.0.smithi163.stdout:333: snap_remove snap 33 2023-10-09T09:06:43.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:43 smithi181 bash[50540]: debug 2023-10-09T09:06:43.039+0000 7f3364718200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:06:43.789 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:43 smithi181 bash[50540]: debug 2023-10-09T09:06:43.403+0000 7f3364718200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:06:43.789 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:43 smithi181 bash[50540]: debug 2023-10-09T09:06:43.567+0000 7f3364718200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:06:44.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:43 smithi181 bash[50540]: debug 2023-10-09T09:06:43.783+0000 7f3364718200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:06:44.129 INFO:tasks.rados.rados.0.smithi163.stdout:333: done (1 left) 2023-10-09T09:06:44.129 INFO:tasks.rados.rados.0.smithi163.stdout:334: rollback oid 2 current snap is 42 2023-10-09T09:06:44.129 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 2 to 37 2023-10-09T09:06:44.129 INFO:tasks.rados.rados.0.smithi163.stdout:335: write oid 10 current snap is 42 2023-10-09T09:06:44.129 INFO:tasks.rados.rados.0.smithi163.stdout:335: seq_num 161 ranges {705927=558502,2004895=422707} 2023-10-09T09:06:44.139 INFO:tasks.rados.rados.0.smithi163.stdout:335: writing smithi16367185-10 from 705927 to 1264429 tid 1 2023-10-09T09:06:44.146 INFO:tasks.rados.rados.0.smithi163.stdout:335: writing smithi16367185-10 from 2004895 to 2427602 tid 2 2023-10-09T09:06:44.146 INFO:tasks.rados.rados.0.smithi163.stdout:336: snap_create 2023-10-09T09:06:44.146 INFO:tasks.rados.rados.0.smithi163.stdout:332: finishing rollback tid 0 to smithi16367185-33 2023-10-09T09:06:44.146 INFO:tasks.rados.rados.0.smithi163.stdout:332: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:06:44.147 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 116 (ObjNum 699018832 snap 21875 seq_num 0) dirty dne 2023-10-09T09:06:44.147 INFO:tasks.rados.rados.0.smithi163.stdout:334: finishing rollback tid 1 to smithi16367185-2 2023-10-09T09:06:44.147 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 0 (ObjNum 699103664 snap 21875 seq_num 697217040) dirty dne 2023-10-09T09:06:44.147 INFO:tasks.rados.rados.0.smithi163.stdout:335: finishing write tid 1 to smithi16367185-10 2023-10-09T09:06:44.148 INFO:tasks.rados.rados.0.smithi163.stdout:335: finishing write tid 2 to smithi16367185-10 2023-10-09T09:06:44.148 INFO:tasks.rados.rados.0.smithi163.stdout:335: finishing write tid 3 to smithi16367185-10 2023-10-09T09:06:44.149 INFO:tasks.rados.rados.0.smithi163.stdout:335: finishing write tid 4 to smithi16367185-10 2023-10-09T09:06:44.149 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 103 (ObjNum 161 snap 42 seq_num 161) dirty exists 2023-10-09T09:06:44.149 INFO:tasks.rados.rados.0.smithi163.stdout:335: left oid 10 (ObjNum 161 snap 42 seq_num 161) 2023-10-09T09:06:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:44 smithi163 bash[33912]: cluster 2023-10-09T09:06:43.117650+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2023-10-09T09:06:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:44 smithi163 bash[33912]: cluster 2023-10-09T09:06:43.275415+0000 mgr.y (mgr.24355) 213 : cluster [DBG] pgmap v261: 89 pgs: 89 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2023-10-09T09:06:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:44 smithi163 bash[33912]: audit 2023-10-09T09:06:43.365425+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.163:0/610843418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:44 smithi163 bash[27500]: cluster 2023-10-09T09:06:43.117650+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2023-10-09T09:06:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:44 smithi163 bash[27500]: cluster 2023-10-09T09:06:43.275415+0000 mgr.y (mgr.24355) 213 : cluster [DBG] pgmap v261: 89 pgs: 89 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2023-10-09T09:06:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:44 smithi163 bash[27500]: audit 2023-10-09T09:06:43.365425+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.163:0/610843418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:44 smithi181 bash[27712]: cluster 2023-10-09T09:06:43.117650+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2023-10-09T09:06:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:44 smithi181 bash[27712]: cluster 2023-10-09T09:06:43.275415+0000 mgr.y (mgr.24355) 213 : cluster [DBG] pgmap v261: 89 pgs: 89 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2023-10-09T09:06:44.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:44 smithi181 bash[27712]: audit 2023-10-09T09:06:43.365425+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.163:0/610843418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:45.131 INFO:tasks.rados.rados.0.smithi163.stdout:332: done (3 left) 2023-10-09T09:06:45.131 INFO:tasks.rados.rados.0.smithi163.stdout:334: done (2 left) 2023-10-09T09:06:45.131 INFO:tasks.rados.rados.0.smithi163.stdout:335: done (1 left) 2023-10-09T09:06:45.131 INFO:tasks.rados.rados.0.smithi163.stdout:336: done (0 left) 2023-10-09T09:06:45.131 INFO:tasks.rados.rados.0.smithi163.stdout:337: read oid 36 snap -1 2023-10-09T09:06:45.131 INFO:tasks.rados.rados.0.smithi163.stdout:337: expect (ObjNum 134 snap 28 seq_num 134) 2023-10-09T09:06:45.131 INFO:tasks.rados.rados.0.smithi163.stdout:338: delete oid 35 current snap is 43 2023-10-09T09:06:45.134 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:44 smithi181 bash[50540]: debug 2023-10-09T09:06:44.855+0000 7f3364718200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:06:45.134 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:44 smithi181 bash[50540]: debug 2023-10-09T09:06:44.971+0000 7f3364718200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:06:45.135 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:45 smithi181 bash[50540]: debug 2023-10-09T09:06:45.087+0000 7f3364718200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:06:45.136 INFO:tasks.rados.rados.0.smithi163.stdout:337: expect (ObjNum 134 snap 28 seq_num 134) 2023-10-09T09:06:45.211 INFO:tasks.rados.rados.0.smithi163.stdout:337: done (1 left) 2023-10-09T09:06:45.211 INFO:tasks.rados.rados.0.smithi163.stdout:338: done (0 left) 2023-10-09T09:06:45.211 INFO:tasks.rados.rados.0.smithi163.stdout:339: rollback oid 38 current snap is 43 2023-10-09T09:06:45.211 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 35 2023-10-09T09:06:45.211 INFO:tasks.rados.rados.0.smithi163.stdout:340: write oid 36 current snap is 43 2023-10-09T09:06:45.211 INFO:tasks.rados.rados.0.smithi163.stdout:340: seq_num 163 ranges {739541=709000,2121859=426376} 2023-10-09T09:06:45.220 INFO:tasks.rados.rados.0.smithi163.stdout:340: writing smithi16367185-36 from 739541 to 1448541 tid 1 2023-10-09T09:06:45.227 INFO:tasks.rados.rados.0.smithi163.stdout:340: writing smithi16367185-36 from 2121859 to 2548235 tid 2 2023-10-09T09:06:45.228 INFO:tasks.rados.rados.0.smithi163.stdout:341: rollback oid 14 current snap is 43 2023-10-09T09:06:45.228 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 40 2023-10-09T09:06:45.228 INFO:tasks.rados.rados.0.smithi163.stdout:342: delete oid 45 current snap is 43 2023-10-09T09:06:45.228 INFO:tasks.rados.rados.0.smithi163.stdout:339: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:06:45.228 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 0 (ObjNum 699125008 snap 21875 seq_num 698776816) dirty dne 2023-10-09T09:06:45.228 INFO:tasks.rados.rados.0.smithi163.stdout:340: finishing write tid 1 to smithi16367185-36 2023-10-09T09:06:45.233 INFO:tasks.rados.rados.0.smithi163.stdout:341: finishing rollback tid 0 to smithi16367185-14 2023-10-09T09:06:45.233 INFO:tasks.rados.rados.0.smithi163.stdout:341: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:06:45.233 INFO:tasks.rados.rados.0.smithi163.stdout:341: finishing rollback tid 2 to smithi16367185-14 2023-10-09T09:06:45.233 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 139 (ObjNum 75 snap 9 seq_num 75) dirty exists 2023-10-09T09:06:45.234 INFO:tasks.rados.rados.0.smithi163.stdout:340: finishing write tid 2 to smithi16367185-36 2023-10-09T09:06:45.234 INFO:tasks.rados.rados.0.smithi163.stdout:340: finishing write tid 3 to smithi16367185-36 2023-10-09T09:06:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:340: finishing write tid 4 to smithi16367185-36 2023-10-09T09:06:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 98 (ObjNum 163 snap 43 seq_num 163) dirty exists 2023-10-09T09:06:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:340: left oid 36 (ObjNum 163 snap 43 seq_num 163) 2023-10-09T09:06:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:339: done (3 left) 2023-10-09T09:06:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:340: done (2 left) 2023-10-09T09:06:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:341: done (1 left) 2023-10-09T09:06:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:342: done (0 left) 2023-10-09T09:06:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:343: delete oid 3 current snap is 43 2023-10-09T09:06:45.236 INFO:tasks.rados.rados.0.smithi163.stdout:343: done (0 left) 2023-10-09T09:06:45.236 INFO:tasks.rados.rados.0.smithi163.stdout:344: write oid 7 current snap is 43 2023-10-09T09:06:45.236 INFO:tasks.rados.rados.0.smithi163.stdout:344: seq_num 166 ranges {789962=534747,1780952=690056} 2023-10-09T09:06:45.245 INFO:tasks.rados.rados.0.smithi163.stdout:344: writing smithi16367185-7 from 789962 to 1324709 tid 1 2023-10-09T09:06:45.254 INFO:tasks.rados.rados.0.smithi163.stdout:344: writing smithi16367185-7 from 1780952 to 2471008 tid 2 2023-10-09T09:06:45.254 INFO:tasks.rados.rados.0.smithi163.stdout:345: read oid 31 snap -1 2023-10-09T09:06:45.254 INFO:tasks.rados.rados.0.smithi163.stdout:345: expect (ObjNum 138 snap 30 seq_num 138) 2023-10-09T09:06:45.254 INFO:tasks.rados.rados.0.smithi163.stdout:346: write oid 12 current snap is 43 2023-10-09T09:06:45.255 INFO:tasks.rados.rados.0.smithi163.stdout:344: finishing write tid 1 to smithi16367185-7 2023-10-09T09:06:45.255 INFO:tasks.rados.rados.0.smithi163.stdout:346: seq_num 167 ranges {406769=609996,1439434=433714,2303522=503247} 2023-10-09T09:06:45.261 INFO:tasks.rados.rados.0.smithi163.stdout:346: writing smithi16367185-12 from 406769 to 1016765 tid 1 2023-10-09T09:06:45.265 INFO:tasks.rados.rados.0.smithi163.stdout:346: writing smithi16367185-12 from 1439434 to 1873148 tid 2 2023-10-09T09:06:45.271 INFO:tasks.rados.rados.0.smithi163.stdout:346: writing smithi16367185-12 from 2303522 to 2806769 tid 3 2023-10-09T09:06:45.271 INFO:tasks.rados.rados.0.smithi163.stdout:347: rollback oid 36 current snap is 43 2023-10-09T09:06:45.272 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 36 to 40 2023-10-09T09:06:45.272 INFO:tasks.rados.rados.0.smithi163.stdout:344: finishing write tid 2 to smithi16367185-7 2023-10-09T09:06:45.272 INFO:tasks.rados.rados.0.smithi163.stdout:344: finishing write tid 3 to smithi16367185-7 2023-10-09T09:06:45.272 INFO:tasks.rados.rados.0.smithi163.stdout:344: finishing write tid 4 to smithi16367185-7 2023-10-09T09:06:45.272 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 43 (ObjNum 166 snap 43 seq_num 166) dirty exists 2023-10-09T09:06:45.272 INFO:tasks.rados.rados.0.smithi163.stdout:344: left oid 7 (ObjNum 166 snap 43 seq_num 166) 2023-10-09T09:06:45.272 INFO:tasks.rados.rados.0.smithi163.stdout:345: expect (ObjNum 138 snap 30 seq_num 138) 2023-10-09T09:06:45.379 INFO:tasks.rados.rados.0.smithi163.stdout:346: finishing write tid 1 to smithi16367185-12 2023-10-09T09:06:45.379 INFO:tasks.rados.rados.0.smithi163.stdout:346: finishing write tid 2 to smithi16367185-12 2023-10-09T09:06:45.379 INFO:tasks.rados.rados.0.smithi163.stdout:347: finishing rollback tid 0 to smithi16367185-36 2023-10-09T09:06:45.379 INFO:tasks.rados.rados.0.smithi163.stdout:347: finishing rollback tid 1 to smithi16367185-36 2023-10-09T09:06:45.379 INFO:tasks.rados.rados.0.smithi163.stdout:347: finishing rollback tid 2 to smithi16367185-36 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 103 (ObjNum 134 snap 28 seq_num 134) dirty exists 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:346: finishing write tid 3 to smithi16367185-12 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:346: finishing write tid 4 to smithi16367185-12 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:346: finishing write tid 5 to smithi16367185-12 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 121 (ObjNum 167 snap 43 seq_num 167) dirty exists 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:346: left oid 12 (ObjNum 167 snap 43 seq_num 167) 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:344: done (3 left) 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:345: done (2 left) 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:346: done (1 left) 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:347: done (0 left) 2023-10-09T09:06:45.380 INFO:tasks.rados.rados.0.smithi163.stdout:348: snap_create 2023-10-09T09:06:45.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:45 smithi163 bash[33912]: cluster 2023-10-09T09:06:44.128365+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2023-10-09T09:06:45.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:45 smithi163 bash[27500]: cluster 2023-10-09T09:06:44.128365+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2023-10-09T09:06:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:45 smithi181 bash[27712]: cluster 2023-10-09T09:06:44.128365+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2023-10-09T09:06:45.878 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:45 smithi181 bash[50540]: debug 2023-10-09T09:06:45.619+0000 7f3364718200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:06:46.142 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:45 smithi181 bash[50540]: debug 2023-10-09T09:06:45.871+0000 7f3364718200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:06:46.149 INFO:tasks.rados.rados.0.smithi163.stdout:348: done (0 left) 2023-10-09T09:06:46.149 INFO:tasks.rados.rados.0.smithi163.stdout:349: write oid 13 current snap is 44 2023-10-09T09:06:46.149 INFO:tasks.rados.rados.0.smithi163.stdout:349: seq_num 168 ranges {423576=401598,1497916=693725} 2023-10-09T09:06:46.154 INFO:tasks.rados.rados.0.smithi163.stdout:349: writing smithi16367185-13 from 423576 to 825174 tid 1 2023-10-09T09:06:46.163 INFO:tasks.rados.rados.0.smithi163.stdout:349: writing smithi16367185-13 from 1497916 to 2191641 tid 2 2023-10-09T09:06:46.163 INFO:tasks.rados.rados.0.smithi163.stdout:350: write oid 44 current snap is 44 2023-10-09T09:06:46.163 INFO:tasks.rados.rados.0.smithi163.stdout:350: seq_num 169 ranges {440383=476847,1556398=437383,2474721=365662} 2023-10-09T09:06:46.169 INFO:tasks.rados.rados.0.smithi163.stdout:350: writing smithi16367185-44 from 440383 to 917230 tid 1 2023-10-09T09:06:46.176 INFO:tasks.rados.rados.0.smithi163.stdout:350: writing smithi16367185-44 from 1556398 to 1993781 tid 2 2023-10-09T09:06:46.181 INFO:tasks.rados.rados.0.smithi163.stdout:350: writing smithi16367185-44 from 2474721 to 2840383 tid 3 2023-10-09T09:06:46.181 INFO:tasks.rados.rados.0.smithi163.stdout:351: write oid 21 current snap is 44 2023-10-09T09:06:46.181 INFO:tasks.rados.rados.0.smithi163.stdout:351: seq_num 170 ranges {457190=552096,1614880=581041,2785791=71399} 2023-10-09T09:06:46.187 INFO:tasks.rados.rados.0.smithi163.stdout:351: writing smithi16367185-21 from 457190 to 1009286 tid 1 2023-10-09T09:06:46.194 INFO:tasks.rados.rados.0.smithi163.stdout:351: writing smithi16367185-21 from 1614880 to 2195921 tid 2 2023-10-09T09:06:46.198 INFO:tasks.rados.rados.0.smithi163.stdout:351: writing smithi16367185-21 from 2785791 to 2857190 tid 3 2023-10-09T09:06:46.198 INFO:tasks.rados.rados.0.smithi163.stdout:352: write oid 14 current snap is 44 2023-10-09T09:06:46.198 INFO:tasks.rados.rados.0.smithi163.stdout:352: seq_num 171 ranges {473997=627345,1673362=441052,2529567=344430} 2023-10-09T09:06:46.205 INFO:tasks.rados.rados.0.smithi163.stdout:352: writing smithi16367185-14 from 473997 to 1101342 tid 1 2023-10-09T09:06:46.213 INFO:tasks.rados.rados.0.smithi163.stdout:352: writing smithi16367185-14 from 1673362 to 2114414 tid 2 2023-10-09T09:06:46.218 INFO:tasks.rados.rados.0.smithi163.stdout:352: writing smithi16367185-14 from 2529567 to 2873997 tid 3 2023-10-09T09:06:46.218 INFO:tasks.rados.rados.0.smithi163.stdout:353: snap_create 2023-10-09T09:06:46.218 INFO:tasks.rados.rados.0.smithi163.stdout:349: finishing write tid 1 to smithi16367185-13 2023-10-09T09:06:46.218 INFO:tasks.rados.rados.0.smithi163.stdout:349: finishing write tid 2 to smithi16367185-13 2023-10-09T09:06:46.218 INFO:tasks.rados.rados.0.smithi163.stdout:349: finishing write tid 3 to smithi16367185-13 2023-10-09T09:06:46.218 INFO:tasks.rados.rados.0.smithi163.stdout:349: finishing write tid 4 to smithi16367185-13 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 43 (ObjNum 168 snap 44 seq_num 168) dirty exists 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:349: left oid 13 (ObjNum 168 snap 44 seq_num 168) 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:350: finishing write tid 1 to smithi16367185-44 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:350: finishing write tid 2 to smithi16367185-44 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:350: finishing write tid 3 to smithi16367185-44 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:350: finishing write tid 4 to smithi16367185-44 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:350: finishing write tid 5 to smithi16367185-44 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 239 (ObjNum 169 snap 44 seq_num 169) dirty exists 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:350: left oid 44 (ObjNum 169 snap 44 seq_num 169) 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:351: finishing write tid 1 to smithi16367185-21 2023-10-09T09:06:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:351: finishing write tid 2 to smithi16367185-21 2023-10-09T09:06:46.220 INFO:tasks.rados.rados.0.smithi163.stdout:351: finishing write tid 3 to smithi16367185-21 2023-10-09T09:06:46.220 INFO:tasks.rados.rados.0.smithi163.stdout:351: finishing write tid 4 to smithi16367185-21 2023-10-09T09:06:46.220 INFO:tasks.rados.rados.0.smithi163.stdout:351: finishing write tid 5 to smithi16367185-21 2023-10-09T09:06:46.220 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 48 (ObjNum 170 snap 44 seq_num 170) dirty exists 2023-10-09T09:06:46.220 INFO:tasks.rados.rados.0.smithi163.stdout:351: left oid 21 (ObjNum 170 snap 44 seq_num 170) 2023-10-09T09:06:46.220 INFO:tasks.rados.rados.0.smithi163.stdout:352: finishing write tid 1 to smithi16367185-14 2023-10-09T09:06:46.220 INFO:tasks.rados.rados.0.smithi163.stdout:352: finishing write tid 2 to smithi16367185-14 2023-10-09T09:06:46.220 INFO:tasks.rados.rados.0.smithi163.stdout:352: finishing write tid 3 to smithi16367185-14 2023-10-09T09:06:46.221 INFO:tasks.rados.rados.0.smithi163.stdout:352: finishing write tid 4 to smithi16367185-14 2023-10-09T09:06:46.221 INFO:tasks.rados.rados.0.smithi163.stdout:352: finishing write tid 5 to smithi16367185-14 2023-10-09T09:06:46.221 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 144 (ObjNum 171 snap 44 seq_num 171) dirty exists 2023-10-09T09:06:46.221 INFO:tasks.rados.rados.0.smithi163.stdout:352: left oid 14 (ObjNum 171 snap 44 seq_num 171) 2023-10-09T09:06:46.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:46 smithi181 bash[27712]: cluster 2023-10-09T09:06:45.130344+0000 mon.a (mon.0) 870 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2023-10-09T09:06:46.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:46 smithi181 bash[27712]: cluster 2023-10-09T09:06:45.275949+0000 mgr.y (mgr.24355) 214 : cluster [DBG] pgmap v264: 89 pgs: 89 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2023-10-09T09:06:46.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:46 smithi181 bash[27712]: audit 2023-10-09T09:06:45.792023+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.163:0/2344549281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:46 smithi163 bash[27500]: cluster 2023-10-09T09:06:45.130344+0000 mon.a (mon.0) 870 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2023-10-09T09:06:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:46 smithi163 bash[27500]: cluster 2023-10-09T09:06:45.275949+0000 mgr.y (mgr.24355) 214 : cluster [DBG] pgmap v264: 89 pgs: 89 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2023-10-09T09:06:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:46 smithi163 bash[27500]: audit 2023-10-09T09:06:45.792023+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.163:0/2344549281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:46 smithi163 bash[33912]: cluster 2023-10-09T09:06:45.130344+0000 mon.a (mon.0) 870 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2023-10-09T09:06:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:46 smithi163 bash[33912]: cluster 2023-10-09T09:06:45.275949+0000 mgr.y (mgr.24355) 214 : cluster [DBG] pgmap v264: 89 pgs: 89 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2023-10-09T09:06:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:46 smithi163 bash[33912]: audit 2023-10-09T09:06:45.792023+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.163:0/2344549281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:46.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:46 smithi181 bash[50540]: debug 2023-10-09T09:06:46.403+0000 7f3364718200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:06:46.888 INFO:tasks.rados.rados.0.smithi163.stdout:349: done (4 left) 2023-10-09T09:06:46.888 INFO:tasks.rados.rados.0.smithi163.stdout:350: done (3 left) 2023-10-09T09:06:46.888 INFO:tasks.rados.rados.0.smithi163.stdout:351: done (2 left) 2023-10-09T09:06:46.888 INFO:tasks.rados.rados.0.smithi163.stdout:352: done (1 left) 2023-10-09T09:06:46.888 INFO:tasks.rados.rados.0.smithi163.stdout:353: done (0 left) 2023-10-09T09:06:46.889 INFO:tasks.rados.rados.0.smithi163.stdout:354: rollback oid 19 current snap is 45 2023-10-09T09:06:46.889 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 19 to 38 2023-10-09T09:06:46.889 INFO:tasks.rados.rados.0.smithi163.stdout:355: read oid 36 snap -1 2023-10-09T09:06:46.889 INFO:tasks.rados.rados.0.smithi163.stdout:355: expect (ObjNum 134 snap 28 seq_num 134) 2023-10-09T09:06:46.889 INFO:tasks.rados.rados.0.smithi163.stdout:356: write oid 27 current snap is 45 2023-10-09T09:06:46.889 INFO:tasks.rados.rados.0.smithi163.stdout:356: seq_num 172 ranges {490804=702594,1615491=584710,2724284=166520} 2023-10-09T09:06:46.896 INFO:tasks.rados.rados.0.smithi163.stdout:356: writing smithi16367185-27 from 490804 to 1193398 tid 1 2023-10-09T09:06:46.903 INFO:tasks.rados.rados.0.smithi163.stdout:356: writing smithi16367185-27 from 1615491 to 2200201 tid 2 2023-10-09T09:06:46.907 INFO:tasks.rados.rados.0.smithi163.stdout:356: writing smithi16367185-27 from 2724284 to 2890804 tid 3 2023-10-09T09:06:46.908 INFO:tasks.rados.rados.0.smithi163.stdout:357: snap_create 2023-10-09T09:06:46.908 INFO:tasks.rados.rados.0.smithi163.stdout:355: expect (ObjNum 134 snap 28 seq_num 134) 2023-10-09T09:06:46.991 INFO:tasks.rados.rados.0.smithi163.stdout:354: finishing rollback tid 0 to smithi16367185-19 2023-10-09T09:06:46.991 INFO:tasks.rados.rados.0.smithi163.stdout:354: finishing rollback tid 1 to smithi16367185-19 2023-10-09T09:06:46.991 INFO:tasks.rados.rados.0.smithi163.stdout:354: finishing rollback tid 2 to smithi16367185-19 2023-10-09T09:06:46.991 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 87 (ObjNum 140 snap 31 seq_num 140) dirty exists 2023-10-09T09:06:46.991 INFO:tasks.rados.rados.0.smithi163.stdout:356: finishing write tid 1 to smithi16367185-27 2023-10-09T09:06:46.991 INFO:tasks.rados.rados.0.smithi163.stdout:356: finishing write tid 2 to smithi16367185-27 2023-10-09T09:06:46.992 INFO:tasks.rados.rados.0.smithi163.stdout:356: finishing write tid 3 to smithi16367185-27 2023-10-09T09:06:46.992 INFO:tasks.rados.rados.0.smithi163.stdout:356: finishing write tid 4 to smithi16367185-27 2023-10-09T09:06:46.992 INFO:tasks.rados.rados.0.smithi163.stdout:356: finishing write tid 5 to smithi16367185-27 2023-10-09T09:06:46.992 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 91 (ObjNum 172 snap 45 seq_num 172) dirty exists 2023-10-09T09:06:46.992 INFO:tasks.rados.rados.0.smithi163.stdout:356: left oid 27 (ObjNum 172 snap 45 seq_num 172) 2023-10-09T09:06:47.144 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:46 smithi163 bash[27764]: debug 2023-10-09T09:06:46.878+0000 7f5fef794700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T09:06:47.144 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:46 smithi163 bash[27764]: ignoring --setuser ceph since I am not root 2023-10-09T09:06:47.144 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:46 smithi163 bash[27764]: ignoring --setgroup ceph since I am not root 2023-10-09T09:06:47.144 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:46 smithi163 bash[27764]: debug 2023-10-09T09:06:46.938+0000 7f92076c2700 1 -- 172.21.15.163:0/2841813954 <== mon.2 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5653a554a4e0 con 0x5653a563ec00 2023-10-09T09:06:47.144 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27764]: debug 2023-10-09T09:06:47.098+0000 7f921788a680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:06:47.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:46 smithi181 bash[50540]: debug 2023-10-09T09:06:46.967+0000 7f3364718200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:06:47.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: cluster 2023-10-09T09:06:46.146727+0000 mon.a (mon.0) 871 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2023-10-09T09:06:47.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: audit 2023-10-09T09:06:46.147116+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:47.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: audit 2023-10-09T09:06:46.561117+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:47.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: audit 2023-10-09T09:06:46.845747+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:06:47.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: audit 2023-10-09T09:06:46.847729+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:06:47.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: audit 2023-10-09T09:06:46.857560+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:47.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: audit 2023-10-09T09:06:46.865716+0000 mon.a (mon.0) 877 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:47.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: audit 2023-10-09T09:06:46.871791+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T09:06:47.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: cluster 2023-10-09T09:06:46.883991+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2023-10-09T09:06:47.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: audit 2023-10-09T09:06:46.884301+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T09:06:47.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[27712]: cluster 2023-10-09T09:06:46.884430+0000 mon.a (mon.0) 881 : cluster [DBG] mgrmap e19: x(active, starting, since 0.0121033s) 2023-10-09T09:06:47.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27764]: debug 2023-10-09T09:06:47.206+0000 7f921788a680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:06:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: cluster 2023-10-09T09:06:46.146727+0000 mon.a (mon.0) 871 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2023-10-09T09:06:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: audit 2023-10-09T09:06:46.147116+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: audit 2023-10-09T09:06:46.561117+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: audit 2023-10-09T09:06:46.845747+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:06:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: audit 2023-10-09T09:06:46.847729+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:06:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: audit 2023-10-09T09:06:46.857560+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: audit 2023-10-09T09:06:46.865716+0000 mon.a (mon.0) 877 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: audit 2023-10-09T09:06:46.871791+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T09:06:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: cluster 2023-10-09T09:06:46.883991+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2023-10-09T09:06:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: audit 2023-10-09T09:06:46.884301+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T09:06:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27500]: cluster 2023-10-09T09:06:46.884430+0000 mon.a (mon.0) 881 : cluster [DBG] mgrmap e19: x(active, starting, since 0.0121033s) 2023-10-09T09:06:47.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: cluster 2023-10-09T09:06:46.146727+0000 mon.a (mon.0) 871 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2023-10-09T09:06:47.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: audit 2023-10-09T09:06:46.147116+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:47.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: audit 2023-10-09T09:06:46.561117+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:47.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: audit 2023-10-09T09:06:46.845747+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:06:47.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: audit 2023-10-09T09:06:46.847729+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:06:47.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: audit 2023-10-09T09:06:46.857560+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' 2023-10-09T09:06:47.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: audit 2023-10-09T09:06:46.865716+0000 mon.a (mon.0) 877 : audit [DBG] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:47.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: audit 2023-10-09T09:06:46.871791+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T09:06:47.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: cluster 2023-10-09T09:06:46.883991+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2023-10-09T09:06:47.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: audit 2023-10-09T09:06:46.884301+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.24355 172.21.15.163:0/272351944' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T09:06:47.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[33912]: cluster 2023-10-09T09:06:46.884430+0000 mon.a (mon.0) 881 : cluster [DBG] mgrmap e19: x(active, starting, since 0.0121033s) 2023-10-09T09:06:47.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[50540]: debug 2023-10-09T09:06:47.559+0000 7f3364718200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:06:47.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:47 smithi181 bash[50540]: debug 2023-10-09T09:06:47.671+0000 7f3364718200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:06:47.888 INFO:tasks.rados.rados.0.smithi163.stdout:354: done (3 left) 2023-10-09T09:06:47.888 INFO:tasks.rados.rados.0.smithi163.stdout:355: done (2 left) 2023-10-09T09:06:47.888 INFO:tasks.rados.rados.0.smithi163.stdout:356: done (1 left) 2023-10-09T09:06:47.889 INFO:tasks.rados.rados.0.smithi163.stdout:357: done (0 left) 2023-10-09T09:06:47.889 INFO:tasks.rados.rados.0.smithi163.stdout:358: write oid 28 current snap is 46 2023-10-09T09:06:47.889 INFO:tasks.rados.rados.0.smithi163.stdout:358: seq_num 173 ranges {507611=777843,2073973=444721} 2023-10-09T09:06:47.896 INFO:tasks.rados.rados.0.smithi163.stdout:358: writing smithi16367185-28 from 507611 to 1285454 tid 1 2023-10-09T09:06:47.903 INFO:tasks.rados.rados.0.smithi163.stdout:358: writing smithi16367185-28 from 2073973 to 2518694 tid 2 2023-10-09T09:06:47.903 INFO:tasks.rados.rados.0.smithi163.stdout:359: snap_create 2023-10-09T09:06:47.904 INFO:tasks.rados.rados.0.smithi163.stdout:358: finishing write tid 1 to smithi16367185-28 2023-10-09T09:06:47.906 INFO:tasks.rados.rados.0.smithi163.stdout:358: finishing write tid 2 to smithi16367185-28 2023-10-09T09:06:47.906 INFO:tasks.rados.rados.0.smithi163.stdout:358: finishing write tid 3 to smithi16367185-28 2023-10-09T09:06:47.907 INFO:tasks.rados.rados.0.smithi163.stdout:358: finishing write tid 4 to smithi16367185-28 2023-10-09T09:06:47.907 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 155 (ObjNum 173 snap 46 seq_num 173) dirty exists 2023-10-09T09:06:47.907 INFO:tasks.rados.rados.0.smithi163.stdout:358: left oid 28 (ObjNum 173 snap 46 seq_num 173) 2023-10-09T09:06:47.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:47 smithi163 bash[27764]: debug 2023-10-09T09:06:47.686+0000 7f921788a680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:06:48.286 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:48 smithi181 bash[50540]: debug 2023-10-09T09:06:48.023+0000 7f3364718200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:06:48.287 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:48 smithi181 bash[50540]: debug 2023-10-09T09:06:48.135+0000 7f3364718200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:06:48.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:48 smithi181 bash[50540]: debug 2023-10-09T09:06:48.279+0000 7f3364718200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:06:48.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:48 smithi181 bash[50540]: debug 2023-10-09T09:06:48.395+0000 7f3364718200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:06:48.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:48 smithi181 bash[50540]: debug 2023-10-09T09:06:48.519+0000 7f3364718200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:06:48.891 INFO:tasks.rados.rados.0.smithi163.stdout:358: done (1 left) 2023-10-09T09:06:48.892 INFO:tasks.rados.rados.0.smithi163.stdout:359: done (0 left) 2023-10-09T09:06:48.892 INFO:tasks.rados.rados.0.smithi163.stdout:360: snap_remove snap 45 2023-10-09T09:06:48.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:48 smithi163 bash[27764]: debug 2023-10-09T09:06:48.706+0000 7f921788a680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:06:48.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:48 smithi163 bash[27764]: debug 2023-10-09T09:06:48.806+0000 7f921788a680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:06:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:48 smithi163 bash[33912]: cluster 2023-10-09T09:06:47.886930+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2023-10-09T09:06:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:48 smithi163 bash[33912]: audit 2023-10-09T09:06:48.204300+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.163:0/46756663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:48.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:48 smithi163 bash[27500]: cluster 2023-10-09T09:06:47.886930+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2023-10-09T09:06:48.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:48 smithi163 bash[27500]: audit 2023-10-09T09:06:48.204300+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.163:0/46756663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:49.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:48 smithi181 bash[27712]: cluster 2023-10-09T09:06:47.886930+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2023-10-09T09:06:49.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:48 smithi181 bash[27712]: audit 2023-10-09T09:06:48.204300+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.163:0/46756663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:49.068 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:48 smithi181 bash[50540]: debug 2023-10-09T09:06:48.719+0000 7f3364718200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:06:49.068 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:48 smithi181 bash[50540]: debug 2023-10-09T09:06:48.847+0000 7f3364718200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:06:49.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:49 smithi163 bash[27764]: debug 2023-10-09T09:06:49.058+0000 7f921788a680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:06:49.900 INFO:tasks.rados.rados.0.smithi163.stdout:360: done (0 left) 2023-10-09T09:06:49.900 INFO:tasks.rados.rados.0.smithi163.stdout:361: read oid 32 snap -1 2023-10-09T09:06:49.900 INFO:tasks.rados.rados.0.smithi163.stdout:361: expect (ObjNum 86 snap 11 seq_num 86) 2023-10-09T09:06:49.900 INFO:tasks.rados.rados.0.smithi163.stdout:362: read oid 10 snap 43 2023-10-09T09:06:49.900 INFO:tasks.rados.rados.0.smithi163.stdout:362: expect (ObjNum 161 snap 42 seq_num 161) 2023-10-09T09:06:49.900 INFO:tasks.rados.rados.0.smithi163.stdout:363: read oid 26 snap -1 2023-10-09T09:06:49.900 INFO:tasks.rados.rados.0.smithi163.stdout:363: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:06:49.900 INFO:tasks.rados.rados.0.smithi163.stdout:364: read oid 2 snap -1 2023-10-09T09:06:49.900 INFO:tasks.rados.rados.0.smithi163.stdout:364: expect deleted 2023-10-09T09:06:49.900 INFO:tasks.rados.rados.0.smithi163.stdout:365: delete oid 40 current snap is 47 2023-10-09T09:06:49.906 INFO:tasks.rados.rados.0.smithi163.stdout:364: done (4 left) 2023-10-09T09:06:49.906 INFO:tasks.rados.rados.0.smithi163.stdout:365: done (3 left) 2023-10-09T09:06:49.906 INFO:tasks.rados.rados.0.smithi163.stdout:366: read oid 28 snap -1 2023-10-09T09:06:49.906 INFO:tasks.rados.rados.0.smithi163.stdout:366: expect (ObjNum 173 snap 46 seq_num 173) 2023-10-09T09:06:49.906 INFO:tasks.rados.rados.0.smithi163.stdout:367: rollback oid 30 current snap is 47 2023-10-09T09:06:49.906 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 38 2023-10-09T09:06:49.906 INFO:tasks.rados.rados.0.smithi163.stdout:368: rollback oid 7 current snap is 47 2023-10-09T09:06:49.907 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 43 2023-10-09T09:06:49.907 INFO:tasks.rados.rados.0.smithi163.stdout:369: write oid 46 current snap is 47 2023-10-09T09:06:49.907 INFO:tasks.rados.rados.0.smithi163.stdout:369: seq_num 175 ranges {541225=644694,1907290=448390} 2023-10-09T09:06:49.914 INFO:tasks.rados.rados.0.smithi163.stdout:369: writing smithi16367185-46 from 541225 to 1185919 tid 1 2023-10-09T09:06:49.921 INFO:tasks.rados.rados.0.smithi163.stdout:369: writing smithi16367185-46 from 1907290 to 2355680 tid 2 2023-10-09T09:06:49.922 INFO:tasks.rados.rados.0.smithi163.stdout:370: write oid 42 current snap is 47 2023-10-09T09:06:49.922 INFO:tasks.rados.rados.0.smithi163.stdout:370: seq_num 176 ranges {558032=719943,1849419=592048} 2023-10-09T09:06:49.928 INFO:tasks.rados.rados.0.smithi163.stdout:370: writing smithi16367185-42 from 558032 to 1277975 tid 1 2023-10-09T09:06:49.935 INFO:tasks.rados.rados.0.smithi163.stdout:370: writing smithi16367185-42 from 1849419 to 2441467 tid 2 2023-10-09T09:06:49.936 INFO:tasks.rados.rados.0.smithi163.stdout:371: read oid 6 snap -1 2023-10-09T09:06:49.936 INFO:tasks.rados.rados.0.smithi163.stdout:371: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:06:49.936 INFO:tasks.rados.rados.0.smithi163.stdout:372: delete oid 18 current snap is 47 2023-10-09T09:06:49.936 INFO:tasks.rados.rados.0.smithi163.stdout:362: expect (ObjNum 161 snap 42 seq_num 161) 2023-10-09T09:06:50.035 INFO:tasks.rados.rados.0.smithi163.stdout:363: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:06:50.116 INFO:tasks.rados.rados.0.smithi163.stdout:361: expect (ObjNum 86 snap 11 seq_num 86) 2023-10-09T09:06:50.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:49 smithi163 bash[27500]: cluster 2023-10-09T09:06:48.892148+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2023-10-09T09:06:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:49 smithi163 bash[33912]: cluster 2023-10-09T09:06:48.892148+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2023-10-09T09:06:50.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:49 smithi181 bash[27712]: cluster 2023-10-09T09:06:48.892148+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2023-10-09T09:06:50.228 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.087+0000 7f3364718200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:06:50.228 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: [09/Oct/2023:09:06:50] ENGINE Bus STARTING 2023-10-09T09:06:50.268 INFO:tasks.rados.rados.0.smithi163.stdout:368: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:06:50.268 INFO:tasks.rados.rados.0.smithi163.stdout:368: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:06:50.268 INFO:tasks.rados.rados.0.smithi163.stdout:368: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:06:50.268 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 52 (ObjNum 166 snap 43 seq_num 166) dirty exists 2023-10-09T09:06:50.268 INFO:tasks.rados.rados.0.smithi163.stdout:366: expect (ObjNum 173 snap 46 seq_num 173) 2023-10-09T09:06:50.430 INFO:tasks.rados.rados.0.smithi163.stdout:367: finishing rollback tid 0 to smithi16367185-30 2023-10-09T09:06:50.430 INFO:tasks.rados.rados.0.smithi163.stdout:367: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:06:50.430 INFO:tasks.rados.rados.0.smithi163.stdout:367: finishing rollback tid 2 to smithi16367185-30 2023-10-09T09:06:50.430 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 107 (ObjNum 29 snap 0 seq_num 29) dirty exists 2023-10-09T09:06:50.430 INFO:tasks.rados.rados.0.smithi163.stdout:369: finishing write tid 1 to smithi16367185-46 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:369: finishing write tid 2 to smithi16367185-46 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:361: done (9 left) 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:362: done (8 left) 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:363: done (7 left) 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:366: done (6 left) 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:367: done (5 left) 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:368: done (4 left) 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:372: done (3 left) 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:373: write oid 25 current snap is 47 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:369: finishing write tid 3 to smithi16367185-46 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:369: finishing write tid 4 to smithi16367185-46 2023-10-09T09:06:50.431 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 147 (ObjNum 175 snap 47 seq_num 175) dirty exists 2023-10-09T09:06:50.432 INFO:tasks.rados.rados.0.smithi163.stdout:369: left oid 46 (ObjNum 175 snap 47 seq_num 175) 2023-10-09T09:06:50.432 INFO:tasks.rados.rados.0.smithi163.stdout:370: finishing write tid 1 to smithi16367185-42 2023-10-09T09:06:50.432 INFO:tasks.rados.rados.0.smithi163.stdout:370: finishing write tid 2 to smithi16367185-42 2023-10-09T09:06:50.432 INFO:tasks.rados.rados.0.smithi163.stdout:370: finishing write tid 3 to smithi16367185-42 2023-10-09T09:06:50.432 INFO:tasks.rados.rados.0.smithi163.stdout:370: finishing write tid 4 to smithi16367185-42 2023-10-09T09:06:50.432 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 243 (ObjNum 176 snap 47 seq_num 176) dirty exists 2023-10-09T09:06:50.432 INFO:tasks.rados.rados.0.smithi163.stdout:370: left oid 42 (ObjNum 176 snap 47 seq_num 176) 2023-10-09T09:06:50.432 INFO:tasks.rados.rados.0.smithi163.stdout:371: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:06:50.485 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: CherryPy Checker: 2023-10-09T09:06:50.485 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: The Application mounted at '' has an empty config. 2023-10-09T09:06:50.485 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: [09/Oct/2023:09:06:50] ENGINE Serving on http://:::9283 2023-10-09T09:06:50.485 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: [09/Oct/2023:09:06:50] ENGINE Bus STARTED 2023-10-09T09:06:50.485 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: [09/Oct/2023:09:06:50] ENGINE Bus STOPPING 2023-10-09T09:06:50.485 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: [09/Oct/2023:09:06:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:06:50.485 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: [09/Oct/2023:09:06:50] ENGINE Bus STOPPED 2023-10-09T09:06:50.485 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: [09/Oct/2023:09:06:50] ENGINE Bus STARTING 2023-10-09T09:06:50.529 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27764]: debug 2023-10-09T09:06:50.238+0000 7f921788a680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:06:50.530 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27764]: debug 2023-10-09T09:06:50.418+0000 7f921788a680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:06:50.633 INFO:tasks.rados.rados.0.smithi163.stdout:373: seq_num 178 ranges {591646=470441,1566383=595717,2888822=102824} 2023-10-09T09:06:50.640 INFO:tasks.rados.rados.0.smithi163.stdout:373: writing smithi16367185-25 from 591646 to 1062087 tid 1 2023-10-09T09:06:50.652 INFO:tasks.rados.rados.0.smithi163.stdout:373: writing smithi16367185-25 from 1566383 to 2162100 tid 2 2023-10-09T09:06:50.658 INFO:tasks.rados.rados.0.smithi163.stdout:373: writing smithi16367185-25 from 2888822 to 2991646 tid 3 2023-10-09T09:06:50.659 INFO:tasks.rados.rados.0.smithi163.stdout:369: done (3 left) 2023-10-09T09:06:50.659 INFO:tasks.rados.rados.0.smithi163.stdout:370: done (2 left) 2023-10-09T09:06:50.659 INFO:tasks.rados.rados.0.smithi163.stdout:371: done (1 left) 2023-10-09T09:06:50.659 INFO:tasks.rados.rados.0.smithi163.stdout:374: write oid 49 current snap is 47 2023-10-09T09:06:50.659 INFO:tasks.rados.rados.0.smithi163.stdout:373: finishing write tid 1 to smithi16367185-25 2023-10-09T09:06:50.659 INFO:tasks.rados.rados.0.smithi163.stdout:373: finishing write tid 2 to smithi16367185-25 2023-10-09T09:06:50.659 INFO:tasks.rados.rados.0.smithi163.stdout:374: seq_num 179 ranges {608453=545690,1624865=455728,2632598=375855} 2023-10-09T09:06:50.669 INFO:tasks.rados.rados.0.smithi163.stdout:374: writing smithi16367185-49 from 608453 to 1154143 tid 1 2023-10-09T09:06:50.679 INFO:tasks.rados.rados.0.smithi163.stdout:374: writing smithi16367185-49 from 1624865 to 2080593 tid 2 2023-10-09T09:06:50.687 INFO:tasks.rados.rados.0.smithi163.stdout:374: writing smithi16367185-49 from 2632598 to 3008453 tid 3 2023-10-09T09:06:50.687 INFO:tasks.rados.rados.0.smithi163.stdout:375: rollback oid 37 current snap is 47 2023-10-09T09:06:50.687 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 46 2023-10-09T09:06:50.687 INFO:tasks.rados.rados.0.smithi163.stdout:373: finishing write tid 3 to smithi16367185-25 2023-10-09T09:06:50.687 INFO:tasks.rados.rados.0.smithi163.stdout:373: finishing write tid 4 to smithi16367185-25 2023-10-09T09:06:50.687 INFO:tasks.rados.rados.0.smithi163.stdout:373: finishing write tid 5 to smithi16367185-25 2023-10-09T09:06:50.688 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 126 (ObjNum 178 snap 47 seq_num 178) dirty exists 2023-10-09T09:06:50.688 INFO:tasks.rados.rados.0.smithi163.stdout:373: left oid 25 (ObjNum 178 snap 47 seq_num 178) 2023-10-09T09:06:50.688 INFO:tasks.rados.rados.0.smithi163.stdout:374: finishing write tid 1 to smithi16367185-49 2023-10-09T09:06:50.688 INFO:tasks.rados.rados.0.smithi163.stdout:374: finishing write tid 2 to smithi16367185-49 2023-10-09T09:06:50.688 INFO:tasks.rados.rados.0.smithi163.stdout:373: done (2 left) 2023-10-09T09:06:50.688 INFO:tasks.rados.rados.0.smithi163.stdout:376: delete oid 33 current snap is 47 2023-10-09T09:06:50.689 INFO:tasks.rados.rados.0.smithi163.stdout:376: done (2 left) 2023-10-09T09:06:50.690 INFO:tasks.rados.rados.0.smithi163.stdout:377: snap_remove snap 37 2023-10-09T09:06:50.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.491+0000 7f32c43ba700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:06:50.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.491+0000 7f32c43ba700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:06:50.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.491+0000 7f32c43ba700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:06:50.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.491+0000 7f32c43ba700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:06:50.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.491+0000 7f32c43ba700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:06:50.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.491+0000 7f32c2bb7700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:06:50.819 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.491+0000 7f32c2bb7700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:06:50.819 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.491+0000 7f32c2bb7700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:06:50.819 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.491+0000 7f32c2bb7700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:06:50.819 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: debug 2023-10-09T09:06:50.491+0000 7f32c2bb7700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:06:50.819 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: [09/Oct/2023:09:06:50] ENGINE Serving on http://:::9283 2023-10-09T09:06:50.819 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[50540]: [09/Oct/2023:09:06:50] ENGINE Bus STARTED 2023-10-09T09:06:50.905 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27764]: debug 2023-10-09T09:06:50.526+0000 7f921788a680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:06:50.905 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27764]: debug 2023-10-09T09:06:50.654+0000 7f921788a680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:06:51.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27764]: debug 2023-10-09T09:06:50.914+0000 7f921788a680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:06:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: cluster 2023-10-09T09:06:49.898104+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2023-10-09T09:06:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: cluster 2023-10-09T09:06:50.116573+0000 mon.a (mon.0) 885 : cluster [INF] Active manager daemon x restarted 2023-10-09T09:06:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: cluster 2023-10-09T09:06:50.117015+0000 mon.a (mon.0) 886 : cluster [INF] Activating manager daemon x 2023-10-09T09:06:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: cluster 2023-10-09T09:06:50.125450+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2023-10-09T09:06:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.126614+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:06:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.126859+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:06:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.127068+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:06:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.127251+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:06:51.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.127443+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:06:51.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.127698+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:06:51.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.127939+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:06:51.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: cluster 2023-10-09T09:06:50.127958+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0111261s) 2023-10-09T09:06:51.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.128270+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:06:51.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.128494+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:06:51.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.128754+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:06:51.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.129004+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:06:51.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.129239+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:06:51.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.129490+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:06:51.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.129743+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:06:51.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.130009+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:06:51.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.130723+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:06:51.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.131118+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:06:51.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.131616+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:06:51.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.131877+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:06:51.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.133206+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:06:51.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.133533+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:06:51.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: cluster 2023-10-09T09:06:50.318184+0000 mon.a (mon.0) 889 : cluster [INF] Manager daemon x is now available 2023-10-09T09:06:51.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.354792+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.365354+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:51.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.372085+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:51.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.381599+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:51.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.384475+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:51.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.407395+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:51.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.418397+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:06:51.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.421150+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:06:51.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.458993+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.464357+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:06:51.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.467023+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:06:51.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.602395+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.163:0/3662890617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:51.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.845831+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[33912]: audit 2023-10-09T09:06:50.854566+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: cluster 2023-10-09T09:06:49.898104+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2023-10-09T09:06:51.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: cluster 2023-10-09T09:06:50.116573+0000 mon.a (mon.0) 885 : cluster [INF] Active manager daemon x restarted 2023-10-09T09:06:51.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: cluster 2023-10-09T09:06:50.117015+0000 mon.a (mon.0) 886 : cluster [INF] Activating manager daemon x 2023-10-09T09:06:51.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: cluster 2023-10-09T09:06:50.125450+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2023-10-09T09:06:51.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.126614+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:06:51.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.126859+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:06:51.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.127068+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:06:51.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.127251+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:06:51.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.127443+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:06:51.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.127698+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:06:51.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.127939+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:06:51.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: cluster 2023-10-09T09:06:50.127958+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0111261s) 2023-10-09T09:06:51.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.128270+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:06:51.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.128494+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:06:51.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.128754+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:06:51.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.129004+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:06:51.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.129239+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:06:51.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.129490+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:06:51.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.129743+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:06:51.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.130009+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:06:51.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.130723+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:06:51.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.131118+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:06:51.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.131616+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:06:51.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.131877+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:06:51.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.133206+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:06:51.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.133533+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:06:51.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: cluster 2023-10-09T09:06:50.318184+0000 mon.a (mon.0) 889 : cluster [INF] Manager daemon x is now available 2023-10-09T09:06:51.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.354792+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.365354+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:51.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.372085+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:51.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.381599+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:51.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.384475+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:51.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.407395+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:51.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.418397+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:06:51.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.421150+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:06:51.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.458993+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.464357+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:06:51.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.467023+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:06:51.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.602395+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.163:0/3662890617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:51.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.845831+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:50 smithi163 bash[27500]: audit 2023-10-09T09:06:50.854566+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: cluster 2023-10-09T09:06:49.898104+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2023-10-09T09:06:51.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: cluster 2023-10-09T09:06:50.116573+0000 mon.a (mon.0) 885 : cluster [INF] Active manager daemon x restarted 2023-10-09T09:06:51.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: cluster 2023-10-09T09:06:50.117015+0000 mon.a (mon.0) 886 : cluster [INF] Activating manager daemon x 2023-10-09T09:06:51.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: cluster 2023-10-09T09:06:50.125450+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2023-10-09T09:06:51.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.126614+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:06:51.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.126859+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.127068+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.127251+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.127443+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.127698+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.127939+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: cluster 2023-10-09T09:06:50.127958+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0111261s) 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.128270+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.128494+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.128754+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.129004+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.129239+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:06:51.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.129490+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.129743+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.130009+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.130723+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.131118+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.131616+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.131877+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.133206+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.133533+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: cluster 2023-10-09T09:06:50.318184+0000 mon.a (mon.0) 889 : cluster [INF] Manager daemon x is now available 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.354792+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.365354+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:51.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.372085+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.381599+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.384475+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.407395+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.418397+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.421150+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.458993+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.464357+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.467023+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.602395+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.163:0/3662890617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.845831+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.321 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:50 smithi181 bash[27712]: audit 2023-10-09T09:06:50.854566+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:51.360 INFO:tasks.rados.rados.0.smithi163.stdout:377: done (2 left) 2023-10-09T09:06:51.361 INFO:tasks.rados.rados.0.smithi163.stdout:378: read oid 29 snap -1 2023-10-09T09:06:51.361 INFO:tasks.rados.rados.0.smithi163.stdout:378: expect (ObjNum 155 snap 38 seq_num 155) 2023-10-09T09:06:51.361 INFO:tasks.rados.rados.0.smithi163.stdout:379: delete oid 33 current snap is 47 2023-10-09T09:06:51.361 INFO:tasks.rados.rados.0.smithi163.stdout:374: finishing write tid 3 to smithi16367185-49 2023-10-09T09:06:51.361 INFO:tasks.rados.rados.0.smithi163.stdout:374: finishing write tid 4 to smithi16367185-49 2023-10-09T09:06:51.361 INFO:tasks.rados.rados.0.smithi163.stdout:375: finishing rollback tid 0 to smithi16367185-37 2023-10-09T09:06:51.361 INFO:tasks.rados.rados.0.smithi163.stdout:374: finishing write tid 5 to smithi16367185-49 2023-10-09T09:06:51.361 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 61 (ObjNum 179 snap 47 seq_num 179) dirty exists 2023-10-09T09:06:51.362 INFO:tasks.rados.rados.0.smithi163.stdout:374: left oid 49 (ObjNum 179 snap 47 seq_num 179) 2023-10-09T09:06:51.362 INFO:tasks.rados.rados.0.smithi163.stdout:375: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:06:51.362 INFO:tasks.rados.rados.0.smithi163.stdout:375: finishing rollback tid 2 to smithi16367185-37 2023-10-09T09:06:51.362 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 152 (ObjNum 36 snap 0 seq_num 36) dirty exists 2023-10-09T09:06:51.373 INFO:tasks.rados.rados.0.smithi163.stdout:378: expect (ObjNum 155 snap 38 seq_num 155) 2023-10-09T09:06:51.468 INFO:tasks.rados.rados.0.smithi163.stdout:374: done (3 left) 2023-10-09T09:06:51.468 INFO:tasks.rados.rados.0.smithi163.stdout:375: done (2 left) 2023-10-09T09:06:51.468 INFO:tasks.rados.rados.0.smithi163.stdout:378: done (1 left) 2023-10-09T09:06:51.469 INFO:tasks.rados.rados.0.smithi163.stdout:379: done (0 left) 2023-10-09T09:06:51.469 INFO:tasks.rados.rados.0.smithi163.stdout:380: snap_create 2023-10-09T09:06:51.717 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:51 smithi163 bash[27764]: debug 2023-10-09T09:06:51.326+0000 7f921788a680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:06:51.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:51 smithi163 bash[27764]: debug 2023-10-09T09:06:51.466+0000 7f921788a680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:06:52.363 INFO:tasks.rados.rados.0.smithi163.stdout:380: done (0 left) 2023-10-09T09:06:52.364 INFO:tasks.rados.rados.0.smithi163.stdout:381: read oid 34 snap -1 2023-10-09T09:06:52.364 INFO:tasks.rados.rados.0.smithi163.stdout:381: expect (ObjNum 112 snap 18 seq_num 112) 2023-10-09T09:06:52.364 INFO:tasks.rados.rados.0.smithi163.stdout:382: read oid 47 snap -1 2023-10-09T09:06:52.364 INFO:tasks.rados.rados.0.smithi163.stdout:382: expect deleted 2023-10-09T09:06:52.364 INFO:tasks.rados.rados.0.smithi163.stdout:383: write oid 25 current snap is 48 2023-10-09T09:06:52.364 INFO:tasks.rados.rados.0.smithi163.stdout:383: seq_num 182 ranges {658874=771437,2083958=603055} 2023-10-09T09:06:52.372 INFO:tasks.rados.rados.0.smithi163.stdout:383: writing smithi16367185-25 from 658874 to 1430311 tid 1 2023-10-09T09:06:52.381 INFO:tasks.rados.rados.0.smithi163.stdout:383: writing smithi16367185-25 from 2083958 to 2687013 tid 2 2023-10-09T09:06:52.382 INFO:tasks.rados.rados.0.smithi163.stdout:384: rollback oid 1 current snap is 48 2023-10-09T09:06:52.382 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 1 to 35 2023-10-09T09:06:52.382 INFO:tasks.rados.rados.0.smithi163.stdout:385: write oid 18 current snap is 48 2023-10-09T09:06:52.382 INFO:tasks.rados.rados.0.smithi163.stdout:385: seq_num 183 ranges {675681=563039,1858793=746713,3025937=49744} 2023-10-09T09:06:52.389 INFO:tasks.rados.rados.0.smithi163.stdout:385: writing smithi16367185-18 from 675681 to 1238720 tid 1 2023-10-09T09:06:52.397 INFO:tasks.rados.rados.0.smithi163.stdout:385: writing smithi16367185-18 from 1858793 to 2605506 tid 2 2023-10-09T09:06:52.400 INFO:tasks.rados.rados.0.smithi163.stdout:385: writing smithi16367185-18 from 3025937 to 3075681 tid 3 2023-10-09T09:06:52.400 INFO:tasks.rados.rados.0.smithi163.stdout:386: snap_create 2023-10-09T09:06:52.400 INFO:tasks.rados.rados.0.smithi163.stdout:383: finishing write tid 1 to smithi16367185-25 2023-10-09T09:06:52.400 INFO:tasks.rados.rados.0.smithi163.stdout:381: expect (ObjNum 112 snap 18 seq_num 112) 2023-10-09T09:06:52.437 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[33912]: cephadm 2023-10-09T09:06:50.931851+0000 mgr.x (mgr.24751) 1 : cephadm [INF] Deploying cephadm binary to smithi163 2023-10-09T09:06:52.437 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[33912]: cephadm 2023-10-09T09:06:50.935123+0000 mgr.x (mgr.24751) 2 : cephadm [INF] Deploying cephadm binary to smithi181 2023-10-09T09:06:52.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[33912]: cluster 2023-10-09T09:06:51.132326+0000 mon.a (mon.0) 898 : cluster [DBG] mgrmap e21: x(active, since 1.0155s) 2023-10-09T09:06:52.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[33912]: cluster 2023-10-09T09:06:51.158303+0000 mgr.x (mgr.24751) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 178 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:52.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[33912]: audit 2023-10-09T09:06:51.359792+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:06:52.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[33912]: cluster 2023-10-09T09:06:51.359917+0000 mon.a (mon.0) 900 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2023-10-09T09:06:52.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[33912]: audit 2023-10-09T09:06:51.368995+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:52.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[33912]: audit 2023-10-09T09:06:51.372004+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:52.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[33912]: cephadm 2023-10-09T09:06:51.473961+0000 mgr.x (mgr.24751) 4 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Bus STARTING 2023-10-09T09:06:52.439 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27764]: debug 2023-10-09T09:06:52.434+0000 7f921788a680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:06:52.439 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27500]: cephadm 2023-10-09T09:06:50.931851+0000 mgr.x (mgr.24751) 1 : cephadm [INF] Deploying cephadm binary to smithi163 2023-10-09T09:06:52.439 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27500]: cephadm 2023-10-09T09:06:50.935123+0000 mgr.x (mgr.24751) 2 : cephadm [INF] Deploying cephadm binary to smithi181 2023-10-09T09:06:52.439 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27500]: cluster 2023-10-09T09:06:51.132326+0000 mon.a (mon.0) 898 : cluster [DBG] mgrmap e21: x(active, since 1.0155s) 2023-10-09T09:06:52.439 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27500]: cluster 2023-10-09T09:06:51.158303+0000 mgr.x (mgr.24751) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 178 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:52.439 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27500]: audit 2023-10-09T09:06:51.359792+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:06:52.439 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27500]: cluster 2023-10-09T09:06:51.359917+0000 mon.a (mon.0) 900 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2023-10-09T09:06:52.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27500]: audit 2023-10-09T09:06:51.368995+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:52.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27500]: audit 2023-10-09T09:06:51.372004+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:52.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27500]: cephadm 2023-10-09T09:06:51.473961+0000 mgr.x (mgr.24751) 4 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Bus STARTING 2023-10-09T09:06:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:52 smithi181 bash[27712]: cephadm 2023-10-09T09:06:50.931851+0000 mgr.x (mgr.24751) 1 : cephadm [INF] Deploying cephadm binary to smithi163 2023-10-09T09:06:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:52 smithi181 bash[27712]: cephadm 2023-10-09T09:06:50.935123+0000 mgr.x (mgr.24751) 2 : cephadm [INF] Deploying cephadm binary to smithi181 2023-10-09T09:06:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:52 smithi181 bash[27712]: cluster 2023-10-09T09:06:51.132326+0000 mon.a (mon.0) 898 : cluster [DBG] mgrmap e21: x(active, since 1.0155s) 2023-10-09T09:06:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:52 smithi181 bash[27712]: cluster 2023-10-09T09:06:51.158303+0000 mgr.x (mgr.24751) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 178 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:52 smithi181 bash[27712]: audit 2023-10-09T09:06:51.359792+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:06:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:52 smithi181 bash[27712]: cluster 2023-10-09T09:06:51.359917+0000 mon.a (mon.0) 900 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2023-10-09T09:06:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:52 smithi181 bash[27712]: audit 2023-10-09T09:06:51.368995+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:52 smithi181 bash[27712]: audit 2023-10-09T09:06:51.372004+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:06:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:52 smithi181 bash[27712]: cephadm 2023-10-09T09:06:51.473961+0000 mgr.x (mgr.24751) 4 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Bus STARTING 2023-10-09T09:06:52.643 INFO:tasks.rados.rados.0.smithi163.stdout:384: finishing rollback tid 1 to smithi16367185-1 2023-10-09T09:06:52.644 INFO:tasks.rados.rados.0.smithi163.stdout:383: finishing write tid 2 to smithi16367185-25 2023-10-09T09:06:52.644 INFO:tasks.rados.rados.0.smithi163.stdout:384: finishing rollback tid 2 to smithi16367185-1 2023-10-09T09:06:52.644 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 161 (ObjNum 85 snap 10 seq_num 85) dirty exists 2023-10-09T09:06:52.644 INFO:tasks.rados.rados.0.smithi163.stdout:383: finishing write tid 3 to smithi16367185-25 2023-10-09T09:06:52.644 INFO:tasks.rados.rados.0.smithi163.stdout:383: finishing write tid 4 to smithi16367185-25 2023-10-09T09:06:52.644 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 138 (ObjNum 182 snap 48 seq_num 182) dirty exists 2023-10-09T09:06:52.644 INFO:tasks.rados.rados.0.smithi163.stdout:383: left oid 25 (ObjNum 182 snap 48 seq_num 182) 2023-10-09T09:06:52.644 INFO:tasks.rados.rados.0.smithi163.stdout:385: finishing write tid 1 to smithi16367185-18 2023-10-09T09:06:52.644 INFO:tasks.rados.rados.0.smithi163.stdout:385: finishing write tid 2 to smithi16367185-18 2023-10-09T09:06:52.644 INFO:tasks.rados.rados.0.smithi163.stdout:385: finishing write tid 3 to smithi16367185-18 2023-10-09T09:06:52.645 INFO:tasks.rados.rados.0.smithi163.stdout:385: finishing write tid 4 to smithi16367185-18 2023-10-09T09:06:52.645 INFO:tasks.rados.rados.0.smithi163.stdout:385: finishing write tid 5 to smithi16367185-18 2023-10-09T09:06:52.645 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 117 (ObjNum 183 snap 48 seq_num 183) dirty exists 2023-10-09T09:06:52.645 INFO:tasks.rados.rados.0.smithi163.stdout:385: left oid 18 (ObjNum 183 snap 48 seq_num 183) 2023-10-09T09:06:52.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27764]: debug 2023-10-09T09:06:52.550+0000 7f921788a680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:06:52.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27764]: debug 2023-10-09T09:06:52.682+0000 7f921788a680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:06:53.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:52 smithi163 bash[27764]: debug 2023-10-09T09:06:52.934+0000 7f921788a680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:06:53.376 INFO:tasks.rados.rados.0.smithi163.stdout:381: done (5 left) 2023-10-09T09:06:53.377 INFO:tasks.rados.rados.0.smithi163.stdout:382: done (4 left) 2023-10-09T09:06:53.377 INFO:tasks.rados.rados.0.smithi163.stdout:383: done (3 left) 2023-10-09T09:06:53.377 INFO:tasks.rados.rados.0.smithi163.stdout:384: done (2 left) 2023-10-09T09:06:53.377 INFO:tasks.rados.rados.0.smithi163.stdout:385: done (1 left) 2023-10-09T09:06:53.377 INFO:tasks.rados.rados.0.smithi163.stdout:386: done (0 left) 2023-10-09T09:06:53.378 INFO:tasks.rados.rados.0.smithi163.stdout:387: read oid 39 snap -1 2023-10-09T09:06:53.378 INFO:tasks.rados.rados.0.smithi163.stdout:387: expect (ObjNum 52 snap 0 seq_num 52) 2023-10-09T09:06:53.378 INFO:tasks.rados.rados.0.smithi163.stdout:388: delete oid 22 current snap is 49 2023-10-09T09:06:53.385 INFO:tasks.rados.rados.0.smithi163.stdout:387: expect (ObjNum 52 snap 0 seq_num 52) 2023-10-09T09:06:53.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[33912]: cephadm 2023-10-09T09:06:51.695034+0000 mgr.x (mgr.24751) 5 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Serving on https://172.21.15.181:7150 2023-10-09T09:06:53.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[33912]: cephadm 2023-10-09T09:06:51.808461+0000 mgr.x (mgr.24751) 6 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Serving on http://172.21.15.181:8765 2023-10-09T09:06:53.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[33912]: cephadm 2023-10-09T09:06:51.808692+0000 mgr.x (mgr.24751) 7 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Bus STARTED 2023-10-09T09:06:53.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[33912]: cluster 2023-10-09T09:06:52.131105+0000 mgr.x (mgr.24751) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 178 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:53.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[33912]: audit 2023-10-09T09:06:52.362900+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:06:53.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[33912]: cluster 2023-10-09T09:06:52.363006+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2023-10-09T09:06:53.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[33912]: audit 2023-10-09T09:06:53.039211+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.163:0/2769091180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:53.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[33912]: cluster 2023-10-09T09:06:53.145029+0000 mon.a (mon.0) 905 : cluster [DBG] mgrmap e22: x(active, since 3s) 2023-10-09T09:06:53.518 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[57035]: level=warn ts=2023-10-09T09:06:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2023-10-09T09:06:53.518 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27500]: cephadm 2023-10-09T09:06:51.695034+0000 mgr.x (mgr.24751) 5 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Serving on https://172.21.15.181:7150 2023-10-09T09:06:53.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27500]: cephadm 2023-10-09T09:06:51.808461+0000 mgr.x (mgr.24751) 6 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Serving on http://172.21.15.181:8765 2023-10-09T09:06:53.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27500]: cephadm 2023-10-09T09:06:51.808692+0000 mgr.x (mgr.24751) 7 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Bus STARTED 2023-10-09T09:06:53.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27500]: cluster 2023-10-09T09:06:52.131105+0000 mgr.x (mgr.24751) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 178 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:53.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27500]: audit 2023-10-09T09:06:52.362900+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:06:53.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27500]: cluster 2023-10-09T09:06:52.363006+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2023-10-09T09:06:53.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27500]: audit 2023-10-09T09:06:53.039211+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.163:0/2769091180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:53.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27500]: cluster 2023-10-09T09:06:53.145029+0000 mon.a (mon.0) 905 : cluster [DBG] mgrmap e22: x(active, since 3s) 2023-10-09T09:06:53.519 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27764]: debug 2023-10-09T09:06:53.250+0000 7f921788a680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:06:53.519 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27764]: debug 2023-10-09T09:06:53.442+0000 7f921788a680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:06:53.521 INFO:tasks.rados.rados.0.smithi163.stdout:387: done (1 left) 2023-10-09T09:06:53.521 INFO:tasks.rados.rados.0.smithi163.stdout:388: done (0 left) 2023-10-09T09:06:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:389: rollback oid 46 current snap is 49 2023-10-09T09:06:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 46 to 38 2023-10-09T09:06:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:390: read oid 7 snap -1 2023-10-09T09:06:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:390: expect (ObjNum 166 snap 43 seq_num 166) 2023-10-09T09:06:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:391: read oid 39 snap -1 2023-10-09T09:06:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:391: expect (ObjNum 52 snap 0 seq_num 52) 2023-10-09T09:06:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:392: rollback oid 45 current snap is 49 2023-10-09T09:06:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 45 to 48 2023-10-09T09:06:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:393: snap_remove snap 23 2023-10-09T09:06:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:53 smithi181 bash[27712]: cephadm 2023-10-09T09:06:51.695034+0000 mgr.x (mgr.24751) 5 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Serving on https://172.21.15.181:7150 2023-10-09T09:06:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:53 smithi181 bash[27712]: cephadm 2023-10-09T09:06:51.808461+0000 mgr.x (mgr.24751) 6 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Serving on http://172.21.15.181:8765 2023-10-09T09:06:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:53 smithi181 bash[27712]: cephadm 2023-10-09T09:06:51.808692+0000 mgr.x (mgr.24751) 7 : cephadm [INF] [09/Oct/2023:09:06:51] ENGINE Bus STARTED 2023-10-09T09:06:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:53 smithi181 bash[27712]: cluster 2023-10-09T09:06:52.131105+0000 mgr.x (mgr.24751) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 178 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:53 smithi181 bash[27712]: audit 2023-10-09T09:06:52.362900+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:06:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:53 smithi181 bash[27712]: cluster 2023-10-09T09:06:52.363006+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2023-10-09T09:06:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:53 smithi181 bash[27712]: audit 2023-10-09T09:06:53.039211+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.163:0/2769091180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:53 smithi181 bash[27712]: cluster 2023-10-09T09:06:53.145029+0000 mon.a (mon.0) 905 : cluster [DBG] mgrmap e22: x(active, since 3s) 2023-10-09T09:06:53.878 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27764]: debug 2023-10-09T09:06:53.626+0000 7f921788a680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:06:54.132 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[57035]: level=info ts=2023-10-09T09:06:54.108Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=2 2023-10-09T09:06:54.133 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:53 smithi163 bash[27764]: debug 2023-10-09T09:06:53.874+0000 7f921788a680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:06:54.133 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27764]: debug 2023-10-09T09:06:53.998+0000 7f921788a680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:06:54.133 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27764]: [09/Oct/2023:09:06:54] ENGINE Bus STARTING 2023-10-09T09:06:54.133 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27764]: CherryPy Checker: 2023-10-09T09:06:54.386 INFO:tasks.rados.rados.0.smithi163.stdout:393: done (4 left) 2023-10-09T09:06:54.386 INFO:tasks.rados.rados.0.smithi163.stdout:394: write oid 14 current snap is 49 2023-10-09T09:06:54.386 INFO:tasks.rados.rados.0.smithi163.stdout:394: seq_num 185 ranges {709295=713537,1859404=750382} 2023-10-09T09:06:54.389 INFO:tasks.rados.rados.0.smithi163.stdout:394: writing smithi16367185-14 from 709295 to 1422832 tid 1 2023-10-09T09:06:54.397 INFO:tasks.rados.rados.0.smithi163.stdout:394: writing smithi16367185-14 from 1859404 to 2609786 tid 2 2023-10-09T09:06:54.397 INFO:tasks.rados.rados.0.smithi163.stdout:389: finishing rollback tid 0 to smithi16367185-46 2023-10-09T09:06:54.397 INFO:tasks.rados.rados.0.smithi163.stdout:389: finishing rollback tid 1 to smithi16367185-46 2023-10-09T09:06:54.397 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 152 (ObjNum 699163136 snap 21875 seq_num 697217040) dirty dne 2023-10-09T09:06:54.397 INFO:tasks.rados.rados.0.smithi163.stdout:391: expect (ObjNum 52 snap 0 seq_num 52) 2023-10-09T09:06:54.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27764]: The Application mounted at '' has an empty config. 2023-10-09T09:06:54.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27764]: [09/Oct/2023:09:06:54] ENGINE Serving on http://:::9283 2023-10-09T09:06:54.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27764]: [09/Oct/2023:09:06:54] ENGINE Bus STARTED 2023-10-09T09:06:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27500]: cluster 2023-10-09T09:06:53.374835+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2023-10-09T09:06:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27500]: audit 2023-10-09T09:06:53.654859+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27500]: cluster 2023-10-09T09:06:54.021030+0000 mon.a (mon.0) 908 : cluster [DBG] Standby manager daemon y started 2023-10-09T09:06:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27500]: audit 2023-10-09T09:06:54.025003+0000 mon.a (mon.0) 909 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:06:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27500]: audit 2023-10-09T09:06:54.030376+0000 mon.a (mon.0) 910 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:06:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27500]: audit 2023-10-09T09:06:54.033655+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:06:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[27500]: audit 2023-10-09T09:06:54.034176+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:06:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[33912]: cluster 2023-10-09T09:06:53.374835+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2023-10-09T09:06:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[33912]: audit 2023-10-09T09:06:53.654859+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[33912]: cluster 2023-10-09T09:06:54.021030+0000 mon.a (mon.0) 908 : cluster [DBG] Standby manager daemon y started 2023-10-09T09:06:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[33912]: audit 2023-10-09T09:06:54.025003+0000 mon.a (mon.0) 909 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:06:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[33912]: audit 2023-10-09T09:06:54.030376+0000 mon.a (mon.0) 910 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:06:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[33912]: audit 2023-10-09T09:06:54.033655+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:06:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:54 smithi163 bash[33912]: audit 2023-10-09T09:06:54.034176+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:06:54.535 INFO:tasks.rados.rados.0.smithi163.stdout:392: finishing rollback tid 1 to smithi16367185-45 2023-10-09T09:06:54.535 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 234 (ObjNum 0 snap 0 seq_num 84) dirty dne 2023-10-09T09:06:54.536 INFO:tasks.rados.rados.0.smithi163.stdout:390: expect (ObjNum 166 snap 43 seq_num 166) 2023-10-09T09:06:54.701 INFO:tasks.rados.rados.0.smithi163.stdout:394: finishing write tid 1 to smithi16367185-14 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:394: finishing write tid 2 to smithi16367185-14 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:394: finishing write tid 3 to smithi16367185-14 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:394: finishing write tid 4 to smithi16367185-14 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 158 (ObjNum 185 snap 49 seq_num 185) dirty exists 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:394: left oid 14 (ObjNum 185 snap 49 seq_num 185) 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:389: done (4 left) 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:390: done (3 left) 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:391: done (2 left) 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:392: done (1 left) 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:394: done (0 left) 2023-10-09T09:06:54.702 INFO:tasks.rados.rados.0.smithi163.stdout:395: write oid 20 current snap is 49 2023-10-09T09:06:54.703 INFO:tasks.rados.rados.0.smithi163.stdout:395: seq_num 186 ranges {726102=788786,1917886=610393,3108206=17896} 2023-10-09T09:06:54.715 INFO:tasks.rados.rados.0.smithi163.stdout:395: writing smithi16367185-20 from 726102 to 1514888 tid 1 2023-10-09T09:06:54.723 INFO:tasks.rados.rados.0.smithi163.stdout:395: writing smithi16367185-20 from 1917886 to 2528279 tid 2 2023-10-09T09:06:54.727 INFO:tasks.rados.rados.0.smithi163.stdout:395: writing smithi16367185-20 from 3108206 to 3126102 tid 3 2023-10-09T09:06:54.727 INFO:tasks.rados.rados.0.smithi163.stdout:396: write oid 44 current snap is 49 2023-10-09T09:06:54.727 INFO:tasks.rados.rados.0.smithi163.stdout:396: seq_num 187 ranges {742909=464035,1976368=754051} 2023-10-09T09:06:54.734 INFO:tasks.rados.rados.0.smithi163.stdout:396: writing smithi16367185-44 from 742909 to 1206944 tid 1 2023-10-09T09:06:54.742 INFO:tasks.rados.rados.0.smithi163.stdout:396: writing smithi16367185-44 from 1976368 to 2730419 tid 2 2023-10-09T09:06:54.742 INFO:tasks.rados.rados.0.smithi163.stdout:397: read oid 35 snap 47 2023-10-09T09:06:54.743 INFO:tasks.rados.rados.0.smithi163.stdout:397: expect deleted 2023-10-09T09:06:54.743 INFO:tasks.rados.rados.0.smithi163.stdout:398: snap_create 2023-10-09T09:06:54.743 INFO:tasks.rados.rados.0.smithi163.stdout:395: finishing write tid 1 to smithi16367185-20 2023-10-09T09:06:54.743 INFO:tasks.rados.rados.0.smithi163.stdout:395: finishing write tid 2 to smithi16367185-20 2023-10-09T09:06:54.743 INFO:tasks.rados.rados.0.smithi163.stdout:395: finishing write tid 3 to smithi16367185-20 2023-10-09T09:06:54.743 INFO:tasks.rados.rados.0.smithi163.stdout:395: finishing write tid 4 to smithi16367185-20 2023-10-09T09:06:54.743 INFO:tasks.rados.rados.0.smithi163.stdout:395: finishing write tid 5 to smithi16367185-20 2023-10-09T09:06:54.743 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 49 (ObjNum 186 snap 49 seq_num 186) dirty exists 2023-10-09T09:06:54.743 INFO:tasks.rados.rados.0.smithi163.stdout:395: left oid 20 (ObjNum 186 snap 49 seq_num 186) 2023-10-09T09:06:54.743 INFO:tasks.rados.rados.0.smithi163.stdout:396: finishing write tid 1 to smithi16367185-44 2023-10-09T09:06:54.748 INFO:tasks.rados.rados.0.smithi163.stdout:396: finishing write tid 2 to smithi16367185-44 2023-10-09T09:06:54.748 INFO:tasks.rados.rados.0.smithi163.stdout:396: finishing write tid 3 to smithi16367185-44 2023-10-09T09:06:54.748 INFO:tasks.rados.rados.0.smithi163.stdout:396: finishing write tid 4 to smithi16367185-44 2023-10-09T09:06:54.748 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 267 (ObjNum 187 snap 49 seq_num 187) dirty exists 2023-10-09T09:06:54.749 INFO:tasks.rados.rados.0.smithi163.stdout:396: left oid 44 (ObjNum 187 snap 49 seq_num 187) 2023-10-09T09:06:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:54 smithi181 bash[27712]: cluster 2023-10-09T09:06:53.374835+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2023-10-09T09:06:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:54 smithi181 bash[27712]: audit 2023-10-09T09:06:53.654859+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:54 smithi181 bash[27712]: cluster 2023-10-09T09:06:54.021030+0000 mon.a (mon.0) 908 : cluster [DBG] Standby manager daemon y started 2023-10-09T09:06:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:54 smithi181 bash[27712]: audit 2023-10-09T09:06:54.025003+0000 mon.a (mon.0) 909 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:06:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:54 smithi181 bash[27712]: audit 2023-10-09T09:06:54.030376+0000 mon.a (mon.0) 910 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:06:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:54 smithi181 bash[27712]: audit 2023-10-09T09:06:54.033655+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:06:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:54 smithi181 bash[27712]: audit 2023-10-09T09:06:54.034176+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.? 172.21.15.163:0/1711285490' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:06:55.671 INFO:tasks.rados.rados.0.smithi163.stdout:395: done (3 left) 2023-10-09T09:06:55.672 INFO:tasks.rados.rados.0.smithi163.stdout:396: done (2 left) 2023-10-09T09:06:55.672 INFO:tasks.rados.rados.0.smithi163.stdout:397: done (1 left) 2023-10-09T09:06:55.672 INFO:tasks.rados.rados.0.smithi163.stdout:398: done (0 left) 2023-10-09T09:06:55.672 INFO:tasks.rados.rados.0.smithi163.stdout:399: read oid 47 snap 46 2023-10-09T09:06:55.672 INFO:tasks.rados.rados.0.smithi163.stdout:399: expect deleted 2023-10-09T09:06:55.672 INFO:tasks.rados.rados.0.smithi163.stdout:400: rollback oid 24 current snap is 50 2023-10-09T09:06:55.673 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 24 to 20 2023-10-09T09:06:55.673 INFO:tasks.rados.rados.0.smithi163.stdout:401: delete oid 8 current snap is 50 2023-10-09T09:06:55.677 INFO:tasks.rados.rados.0.smithi163.stdout:399: done (2 left) 2023-10-09T09:06:55.677 INFO:tasks.rados.rados.0.smithi163.stdout:401: done (1 left) 2023-10-09T09:06:55.677 INFO:tasks.rados.rados.0.smithi163.stdout:402: snap_remove snap 35 2023-10-09T09:06:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:55 smithi163 bash[27500]: cluster 2023-10-09T09:06:54.131787+0000 mgr.x (mgr.24751) 9 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 178 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:55 smithi163 bash[27500]: cluster 2023-10-09T09:06:54.378755+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2023-10-09T09:06:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:55 smithi163 bash[27500]: cluster 2023-10-09T09:06:54.668120+0000 mon.a (mon.0) 914 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-09T09:06:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:55 smithi163 bash[27500]: audit 2023-10-09T09:06:54.669063+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:06:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:55 smithi163 bash[33912]: cluster 2023-10-09T09:06:54.131787+0000 mgr.x (mgr.24751) 9 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 178 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:55 smithi163 bash[33912]: cluster 2023-10-09T09:06:54.378755+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2023-10-09T09:06:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:55 smithi163 bash[33912]: cluster 2023-10-09T09:06:54.668120+0000 mon.a (mon.0) 914 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-09T09:06:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:55 smithi163 bash[33912]: audit 2023-10-09T09:06:54.669063+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:06:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:55 smithi181 bash[27712]: cluster 2023-10-09T09:06:54.131787+0000 mgr.x (mgr.24751) 9 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 178 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:06:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:55 smithi181 bash[27712]: cluster 2023-10-09T09:06:54.378755+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2023-10-09T09:06:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:55 smithi181 bash[27712]: cluster 2023-10-09T09:06:54.668120+0000 mon.a (mon.0) 914 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-09T09:06:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:55 smithi181 bash[27712]: audit 2023-10-09T09:06:54.669063+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:06:56.680 INFO:tasks.rados.rados.0.smithi163.stdout:402: done (1 left) 2023-10-09T09:06:56.680 INFO:tasks.rados.rados.0.smithi163.stdout:403: write oid 8 current snap is 50 2023-10-09T09:06:56.680 INFO:tasks.rados.rados.0.smithi163.stdout:400: finishing rollback tid 0 to smithi16367185-24 2023-10-09T09:06:56.680 INFO:tasks.rados.rados.0.smithi163.stdout:400: finishing rollback tid 1 to smithi16367185-24 2023-10-09T09:06:56.680 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 61 (ObjNum 1 snap 32603 seq_num 1275498272) dirty dne 2023-10-09T09:06:56.680 INFO:tasks.rados.rados.0.smithi163.stdout:403: seq_num 189 ranges {776523=614533,1976979=757720} 2023-10-09T09:06:56.692 INFO:tasks.rados.rados.0.smithi163.stdout:403: writing smithi16367185-8 from 776523 to 1391056 tid 1 2023-10-09T09:06:56.701 INFO:tasks.rados.rados.0.smithi163.stdout:403: writing smithi16367185-8 from 1976979 to 2734699 tid 2 2023-10-09T09:06:56.701 INFO:tasks.rados.rados.0.smithi163.stdout:400: done (1 left) 2023-10-09T09:06:56.701 INFO:tasks.rados.rados.0.smithi163.stdout:404: write oid 9 current snap is 50 2023-10-09T09:06:56.701 INFO:tasks.rados.rados.0.smithi163.stdout:403: finishing write tid 1 to smithi16367185-8 2023-10-09T09:06:56.702 INFO:tasks.rados.rados.0.smithi163.stdout:404: seq_num 190 ranges {793330=689782,2035461=617731,3101545=91785} 2023-10-09T09:06:56.709 INFO:tasks.rados.rados.0.smithi163.stdout:404: writing smithi16367185-9 from 793330 to 1483112 tid 1 2023-10-09T09:06:56.716 INFO:tasks.rados.rados.0.smithi163.stdout:404: writing smithi16367185-9 from 2035461 to 2653192 tid 2 2023-10-09T09:06:56.717 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:56 smithi163 bash[33912]: audit 2023-10-09T09:06:55.476798+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.163:0/3757346711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:56 smithi163 bash[33912]: cluster 2023-10-09T09:06:55.670940+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2023-10-09T09:06:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:56 smithi163 bash[27500]: audit 2023-10-09T09:06:55.476798+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.163:0/3757346711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:56 smithi163 bash[27500]: cluster 2023-10-09T09:06:55.670940+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2023-10-09T09:06:56.719 INFO:tasks.rados.rados.0.smithi163.stdout:404: writing smithi16367185-9 from 3101545 to 3193330 tid 3 2023-10-09T09:06:56.719 INFO:tasks.rados.rados.0.smithi163.stdout:405: write oid 21 current snap is 50 2023-10-09T09:06:56.719 INFO:tasks.rados.rados.0.smithi163.stdout:403: finishing write tid 2 to smithi16367185-8 2023-10-09T09:06:56.719 INFO:tasks.rados.rados.0.smithi163.stdout:403: finishing write tid 3 to smithi16367185-8 2023-10-09T09:06:56.720 INFO:tasks.rados.rados.0.smithi163.stdout:403: finishing write tid 4 to smithi16367185-8 2023-10-09T09:06:56.720 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 174 (ObjNum 189 snap 50 seq_num 189) dirty exists 2023-10-09T09:06:56.720 INFO:tasks.rados.rados.0.smithi163.stdout:403: left oid 8 (ObjNum 189 snap 50 seq_num 189) 2023-10-09T09:06:56.720 INFO:tasks.rados.rados.0.smithi163.stdout:404: finishing write tid 1 to smithi16367185-9 2023-10-09T09:06:56.720 INFO:tasks.rados.rados.0.smithi163.stdout:405: seq_num 191 ranges {410137=481384,1410296=761389,2728968=481169} 2023-10-09T09:06:56.724 INFO:tasks.rados.rados.0.smithi163.stdout:405: writing smithi16367185-21 from 410137 to 891521 tid 1 2023-10-09T09:06:56.732 INFO:tasks.rados.rados.0.smithi163.stdout:405: writing smithi16367185-21 from 1410296 to 2171685 tid 2 2023-10-09T09:06:56.738 INFO:tasks.rados.rados.0.smithi163.stdout:405: writing smithi16367185-21 from 2728968 to 3210137 tid 3 2023-10-09T09:06:56.738 INFO:tasks.rados.rados.0.smithi163.stdout:403: done (2 left) 2023-10-09T09:06:56.738 INFO:tasks.rados.rados.0.smithi163.stdout:406: rollback oid 38 current snap is 50 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 38 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:404: finishing write tid 2 to smithi16367185-9 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:407: write oid 43 current snap is 50 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:404: finishing write tid 3 to smithi16367185-9 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:404: finishing write tid 4 to smithi16367185-9 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:404: finishing write tid 5 to smithi16367185-9 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 70 (ObjNum 190 snap 50 seq_num 190) dirty exists 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:404: left oid 9 (ObjNum 190 snap 50 seq_num 190) 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:405: finishing write tid 1 to smithi16367185-21 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:405: finishing write tid 2 to smithi16367185-21 2023-10-09T09:06:56.739 INFO:tasks.rados.rados.0.smithi163.stdout:407: seq_num 192 ranges {426944=556633,1468778=621400,2872744=354200} 2023-10-09T09:06:56.743 INFO:tasks.rados.rados.0.smithi163.stdout:407: writing smithi16367185-43 from 426944 to 983577 tid 1 2023-10-09T09:06:56.750 INFO:tasks.rados.rados.0.smithi163.stdout:407: writing smithi16367185-43 from 1468778 to 2090178 tid 2 2023-10-09T09:06:56.757 INFO:tasks.rados.rados.0.smithi163.stdout:407: writing smithi16367185-43 from 2872744 to 3226944 tid 3 2023-10-09T09:06:56.757 INFO:tasks.rados.rados.0.smithi163.stdout:404: done (3 left) 2023-10-09T09:06:56.757 INFO:tasks.rados.rados.0.smithi163.stdout:408: snap_create 2023-10-09T09:06:56.757 INFO:tasks.rados.rados.0.smithi163.stdout:406: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:06:56.757 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 0 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:06:56.757 INFO:tasks.rados.rados.0.smithi163.stdout:405: finishing write tid 3 to smithi16367185-21 2023-10-09T09:06:56.757 INFO:tasks.rados.rados.0.smithi163.stdout:405: finishing write tid 4 to smithi16367185-21 2023-10-09T09:06:56.757 INFO:tasks.rados.rados.0.smithi163.stdout:405: finishing write tid 5 to smithi16367185-21 2023-10-09T09:06:56.757 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 69 (ObjNum 191 snap 50 seq_num 191) dirty exists 2023-10-09T09:06:56.757 INFO:tasks.rados.rados.0.smithi163.stdout:405: left oid 21 (ObjNum 191 snap 50 seq_num 191) 2023-10-09T09:06:56.758 INFO:tasks.rados.rados.0.smithi163.stdout:407: finishing write tid 1 to smithi16367185-43 2023-10-09T09:06:56.758 INFO:tasks.rados.rados.0.smithi163.stdout:407: finishing write tid 2 to smithi16367185-43 2023-10-09T09:06:56.760 INFO:tasks.rados.rados.0.smithi163.stdout:407: finishing write tid 3 to smithi16367185-43 2023-10-09T09:06:56.760 INFO:tasks.rados.rados.0.smithi163.stdout:407: finishing write tid 4 to smithi16367185-43 2023-10-09T09:06:56.761 INFO:tasks.rados.rados.0.smithi163.stdout:407: finishing write tid 5 to smithi16367185-43 2023-10-09T09:06:56.761 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 166 (ObjNum 192 snap 50 seq_num 192) dirty exists 2023-10-09T09:06:56.761 INFO:tasks.rados.rados.0.smithi163.stdout:407: left oid 43 (ObjNum 192 snap 50 seq_num 192) 2023-10-09T09:06:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:56 smithi181 bash[27712]: audit 2023-10-09T09:06:55.476798+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.163:0/3757346711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:56 smithi181 bash[27712]: cluster 2023-10-09T09:06:55.670940+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2023-10-09T09:06:57.113 INFO:tasks.rados.rados.0.smithi163.stdout:405: done (3 left) 2023-10-09T09:06:57.113 INFO:tasks.rados.rados.0.smithi163.stdout:406: done (2 left) 2023-10-09T09:06:57.113 INFO:tasks.rados.rados.0.smithi163.stdout:407: done (1 left) 2023-10-09T09:06:57.113 INFO:tasks.rados.rados.0.smithi163.stdout:408: done (0 left) 2023-10-09T09:06:57.113 INFO:tasks.rados.rados.0.smithi163.stdout:409: snap_create 2023-10-09T09:06:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:57 smithi163 bash[27500]: cluster 2023-10-09T09:06:56.132525+0000 mgr.x (mgr.24751) 10 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 179 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 298 KiB/s wr, 7 op/s 2023-10-09T09:06:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:57 smithi163 bash[27500]: cluster 2023-10-09T09:06:56.679743+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2023-10-09T09:06:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:57 smithi163 bash[27500]: cluster 2023-10-09T09:06:57.111421+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2023-10-09T09:06:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:57 smithi163 bash[33912]: cluster 2023-10-09T09:06:56.132525+0000 mgr.x (mgr.24751) 10 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 179 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 298 KiB/s wr, 7 op/s 2023-10-09T09:06:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:57 smithi163 bash[33912]: cluster 2023-10-09T09:06:56.679743+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2023-10-09T09:06:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:57 smithi163 bash[33912]: cluster 2023-10-09T09:06:57.111421+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2023-10-09T09:06:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:57 smithi181 bash[27712]: cluster 2023-10-09T09:06:56.132525+0000 mgr.x (mgr.24751) 10 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 179 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 298 KiB/s wr, 7 op/s 2023-10-09T09:06:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:57 smithi181 bash[27712]: cluster 2023-10-09T09:06:56.679743+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2023-10-09T09:06:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:57 smithi181 bash[27712]: cluster 2023-10-09T09:06:57.111421+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2023-10-09T09:06:58.115 INFO:tasks.rados.rados.0.smithi163.stdout:409: done (0 left) 2023-10-09T09:06:58.115 INFO:tasks.rados.rados.0.smithi163.stdout:410: write oid 8 current snap is 52 2023-10-09T09:06:58.115 INFO:tasks.rados.rados.0.smithi163.stdout:410: seq_num 193 ranges {443751=631882,1810907=765058,3067461=176290} 2023-10-09T09:06:58.122 INFO:tasks.rados.rados.0.smithi163.stdout:410: writing smithi16367185-8 from 443751 to 1075633 tid 1 2023-10-09T09:06:58.131 INFO:tasks.rados.rados.0.smithi163.stdout:410: writing smithi16367185-8 from 1810907 to 2575965 tid 2 2023-10-09T09:06:58.135 INFO:tasks.rados.rados.0.smithi163.stdout:410: writing smithi16367185-8 from 3067461 to 3243751 tid 3 2023-10-09T09:06:58.135 INFO:tasks.rados.rados.0.smithi163.stdout:411: snap_create 2023-10-09T09:06:58.136 INFO:tasks.rados.rados.0.smithi163.stdout:410: finishing write tid 1 to smithi16367185-8 2023-10-09T09:06:58.138 INFO:tasks.rados.rados.0.smithi163.stdout:410: finishing write tid 2 to smithi16367185-8 2023-10-09T09:06:58.139 INFO:tasks.rados.rados.0.smithi163.stdout:410: finishing write tid 3 to smithi16367185-8 2023-10-09T09:06:58.139 INFO:tasks.rados.rados.0.smithi163.stdout:410: finishing write tid 4 to smithi16367185-8 2023-10-09T09:06:58.139 INFO:tasks.rados.rados.0.smithi163.stdout:410: finishing write tid 5 to smithi16367185-8 2023-10-09T09:06:58.139 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 179 (ObjNum 193 snap 52 seq_num 193) dirty exists 2023-10-09T09:06:58.139 INFO:tasks.rados.rados.0.smithi163.stdout:410: left oid 8 (ObjNum 193 snap 52 seq_num 193) 2023-10-09T09:06:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:58 smithi163 bash[33912]: audit 2023-10-09T09:06:57.405769+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:58 smithi163 bash[33912]: audit 2023-10-09T09:06:57.908906+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.163:0/869325189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:58 smithi163 bash[33912]: cluster 2023-10-09T09:06:58.114661+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2023-10-09T09:06:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:58 smithi163 bash[27500]: audit 2023-10-09T09:06:57.405769+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:58 smithi163 bash[27500]: audit 2023-10-09T09:06:57.908906+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.163:0/869325189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:58 smithi163 bash[27500]: cluster 2023-10-09T09:06:58.114661+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2023-10-09T09:06:58.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:06:58 smithi181 bash[50540]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:06:58] "GET /metrics HTTP/1.1" 200 33174 "" "Prometheus/2.33.4" 2023-10-09T09:06:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:58 smithi181 bash[27712]: audit 2023-10-09T09:06:57.405769+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:06:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:58 smithi181 bash[27712]: audit 2023-10-09T09:06:57.908906+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.163:0/869325189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:06:58.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:58 smithi181 bash[27712]: cluster 2023-10-09T09:06:58.114661+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2023-10-09T09:06:59.120 INFO:tasks.rados.rados.0.smithi163.stdout:410: done (1 left) 2023-10-09T09:06:59.120 INFO:tasks.rados.rados.0.smithi163.stdout:411: done (0 left) 2023-10-09T09:06:59.120 INFO:tasks.rados.rados.0.smithi163.stdout:412: snap_create 2023-10-09T09:06:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:59 smithi163 bash[33912]: cluster 2023-10-09T09:06:58.133079+0000 mgr.x (mgr.24751) 11 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.2 MiB/s wr, 78 op/s 2023-10-09T09:06:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:06:59 smithi163 bash[33912]: cluster 2023-10-09T09:06:59.119825+0000 mon.a (mon.0) 920 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2023-10-09T09:06:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:59 smithi163 bash[27500]: cluster 2023-10-09T09:06:58.133079+0000 mgr.x (mgr.24751) 11 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.2 MiB/s wr, 78 op/s 2023-10-09T09:06:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:06:59 smithi163 bash[27500]: cluster 2023-10-09T09:06:59.119825+0000 mon.a (mon.0) 920 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2023-10-09T09:06:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:59 smithi181 bash[27712]: cluster 2023-10-09T09:06:58.133079+0000 mgr.x (mgr.24751) 11 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.2 MiB/s wr, 78 op/s 2023-10-09T09:06:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:06:59 smithi181 bash[27712]: cluster 2023-10-09T09:06:59.119825+0000 mon.a (mon.0) 920 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2023-10-09T09:07:00.122 INFO:tasks.rados.rados.0.smithi163.stdout:412: done (0 left) 2023-10-09T09:07:00.122 INFO:tasks.rados.rados.0.smithi163.stdout:413: snap_remove snap 46 2023-10-09T09:07:00.217 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:06:59 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:06:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:07:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[33912]: audit 2023-10-09T09:06:59.559631+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[33912]: audit 2023-10-09T09:06:59.568863+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[33912]: audit 2023-10-09T09:06:59.587864+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[33912]: audit 2023-10-09T09:06:59.597230+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[33912]: cluster 2023-10-09T09:07:00.122405+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2023-10-09T09:07:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[33912]: audit 2023-10-09T09:07:00.237712+0000 mon.a (mon.0) 926 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:07:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[27500]: audit 2023-10-09T09:06:59.559631+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[27500]: audit 2023-10-09T09:06:59.568863+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[27500]: audit 2023-10-09T09:06:59.587864+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[27500]: audit 2023-10-09T09:06:59.597230+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[27500]: cluster 2023-10-09T09:07:00.122405+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2023-10-09T09:07:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:01 smithi163 bash[27500]: audit 2023-10-09T09:07:00.237712+0000 mon.a (mon.0) 926 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:07:02.098 INFO:tasks.rados.rados.0.smithi163.stdout:413: done (0 left) 2023-10-09T09:07:02.098 INFO:tasks.rados.rados.0.smithi163.stdout:414: rollback oid 9 current snap is 54 2023-10-09T09:07:02.098 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 49 2023-10-09T09:07:02.099 INFO:tasks.rados.rados.0.smithi163.stdout:415: rollback oid 11 current snap is 54 2023-10-09T09:07:02.099 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 11 to 40 2023-10-09T09:07:02.099 INFO:tasks.rados.rados.0.smithi163.stdout:416: write oid 25 current snap is 54 2023-10-09T09:07:02.099 INFO:tasks.rados.rados.0.smithi163.stdout:416: seq_num 194 ranges {460558=707131,1869389=508716,3094884=165674} 2023-10-09T09:07:02.105 INFO:tasks.rados.rados.0.smithi163.stdout:416: writing smithi16367185-25 from 460558 to 1167689 tid 1 2023-10-09T09:07:02.113 INFO:tasks.rados.rados.0.smithi163.stdout:416: writing smithi16367185-25 from 1869389 to 2378105 tid 2 2023-10-09T09:07:02.118 INFO:tasks.rados.rados.0.smithi163.stdout:416: writing smithi16367185-25 from 3094884 to 3260558 tid 3 2023-10-09T09:07:02.118 INFO:tasks.rados.rados.0.smithi163.stdout:417: write oid 5 current snap is 54 2023-10-09T09:07:02.118 INFO:tasks.rados.rados.0.smithi163.stdout:415: finishing rollback tid 0 to smithi16367185-11 2023-10-09T09:07:02.118 INFO:tasks.rados.rados.0.smithi163.stdout:417: seq_num 195 ranges {477365=782380,1927871=768727,3122307=155058} 2023-10-09T09:07:02.125 INFO:tasks.rados.rados.0.smithi163.stdout:417: writing smithi16367185-5 from 477365 to 1259745 tid 1 2023-10-09T09:07:02.133 INFO:tasks.rados.rados.0.smithi163.stdout:417: writing smithi16367185-5 from 1927871 to 2696598 tid 2 2023-10-09T09:07:02.137 INFO:tasks.rados.rados.0.smithi163.stdout:417: writing smithi16367185-5 from 3122307 to 3277365 tid 3 2023-10-09T09:07:02.137 INFO:tasks.rados.rados.0.smithi163.stdout:418: read oid 6 snap -1 2023-10-09T09:07:02.137 INFO:tasks.rados.rados.0.smithi163.stdout:418: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:07:02.137 INFO:tasks.rados.rados.0.smithi163.stdout:419: write oid 35 current snap is 54 2023-10-09T09:07:02.137 INFO:tasks.rados.rados.0.smithi163.stdout:415: finishing rollback tid 1 to smithi16367185-11 2023-10-09T09:07:02.137 INFO:tasks.rados.rados.0.smithi163.stdout:415: finishing rollback tid 2 to smithi16367185-11 2023-10-09T09:07:02.137 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 62 (ObjNum 148 snap 34 seq_num 148) dirty exists 2023-10-09T09:07:02.137 INFO:tasks.rados.rados.0.smithi163.stdout:414: finishing rollback tid 0 to smithi16367185-9 2023-10-09T09:07:02.138 INFO:tasks.rados.rados.0.smithi163.stdout:414: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:07:02.138 INFO:tasks.rados.rados.0.smithi163.stdout:414: finishing rollback tid 2 to smithi16367185-9 2023-10-09T09:07:02.138 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 74 (ObjNum 151 snap 36 seq_num 151) dirty exists 2023-10-09T09:07:02.138 INFO:tasks.rados.rados.0.smithi163.stdout:416: finishing write tid 1 to smithi16367185-25 2023-10-09T09:07:02.138 INFO:tasks.rados.rados.0.smithi163.stdout:419: seq_num 196 ranges {494172=457629,1586353=512385,2749730=544442} 2023-10-09T09:07:02.142 INFO:tasks.rados.rados.0.smithi163.stdout:419: writing smithi16367185-35 from 494172 to 951801 tid 1 2023-10-09T09:07:02.149 INFO:tasks.rados.rados.0.smithi163.stdout:419: writing smithi16367185-35 from 1586353 to 2098738 tid 2 2023-10-09T09:07:02.156 INFO:tasks.rados.rados.0.smithi163.stdout:419: writing smithi16367185-35 from 2749730 to 3294172 tid 3 2023-10-09T09:07:02.156 INFO:tasks.rados.rados.0.smithi163.stdout:414: done (5 left) 2023-10-09T09:07:02.156 INFO:tasks.rados.rados.0.smithi163.stdout:415: done (4 left) 2023-10-09T09:07:02.156 INFO:tasks.rados.rados.0.smithi163.stdout:420: rollback oid 28 current snap is 54 2023-10-09T09:07:02.156 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 36 2023-10-09T09:07:02.156 INFO:tasks.rados.rados.0.smithi163.stdout:416: finishing write tid 2 to smithi16367185-25 2023-10-09T09:07:02.156 INFO:tasks.rados.rados.0.smithi163.stdout:416: finishing write tid 3 to smithi16367185-25 2023-10-09T09:07:02.157 INFO:tasks.rados.rados.0.smithi163.stdout:421: write oid 9 current snap is 54 2023-10-09T09:07:02.157 INFO:tasks.rados.rados.0.smithi163.stdout:416: finishing write tid 4 to smithi16367185-25 2023-10-09T09:07:02.157 INFO:tasks.rados.rados.0.smithi163.stdout:416: finishing write tid 5 to smithi16367185-25 2023-10-09T09:07:02.157 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 159 (ObjNum 194 snap 54 seq_num 194) dirty exists 2023-10-09T09:07:02.157 INFO:tasks.rados.rados.0.smithi163.stdout:416: left oid 25 (ObjNum 194 snap 54 seq_num 194) 2023-10-09T09:07:02.157 INFO:tasks.rados.rados.0.smithi163.stdout:417: finishing write tid 1 to smithi16367185-5 2023-10-09T09:07:02.157 INFO:tasks.rados.rados.0.smithi163.stdout:417: finishing write tid 2 to smithi16367185-5 2023-10-09T09:07:02.157 INFO:tasks.rados.rados.0.smithi163.stdout:417: finishing write tid 3 to smithi16367185-5 2023-10-09T09:07:02.157 INFO:tasks.rados.rados.0.smithi163.stdout:421: seq_num 197 ranges {510979=532878,1528482=772396,3060800=250179} 2023-10-09T09:07:02.163 INFO:tasks.rados.rados.0.smithi163.stdout:421: writing smithi16367185-9 from 510979 to 1043857 tid 1 2023-10-09T09:07:02.170 INFO:tasks.rados.rados.0.smithi163.stdout:421: writing smithi16367185-9 from 1528482 to 2300878 tid 2 2023-10-09T09:07:02.176 INFO:tasks.rados.rados.0.smithi163.stdout:421: writing smithi16367185-9 from 3060800 to 3310979 tid 3 2023-10-09T09:07:02.176 INFO:tasks.rados.rados.0.smithi163.stdout:416: done (5 left) 2023-10-09T09:07:02.176 INFO:tasks.rados.rados.0.smithi163.stdout:422: snap_create 2023-10-09T09:07:02.176 INFO:tasks.rados.rados.0.smithi163.stdout:417: finishing write tid 4 to smithi16367185-5 2023-10-09T09:07:02.177 INFO:tasks.rados.rados.0.smithi163.stdout:417: finishing write tid 5 to smithi16367185-5 2023-10-09T09:07:02.177 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 176 (ObjNum 195 snap 54 seq_num 195) dirty exists 2023-10-09T09:07:02.177 INFO:tasks.rados.rados.0.smithi163.stdout:417: left oid 5 (ObjNum 195 snap 54 seq_num 195) 2023-10-09T09:07:02.177 INFO:tasks.rados.rados.0.smithi163.stdout:419: finishing write tid 1 to smithi16367185-35 2023-10-09T09:07:02.177 INFO:tasks.rados.rados.0.smithi163.stdout:418: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:07:02.386 INFO:tasks.rados.rados.0.smithi163.stdout:419: finishing write tid 2 to smithi16367185-35 2023-10-09T09:07:02.386 INFO:tasks.rados.rados.0.smithi163.stdout:419: finishing write tid 3 to smithi16367185-35 2023-10-09T09:07:02.386 INFO:tasks.rados.rados.0.smithi163.stdout:419: finishing write tid 4 to smithi16367185-35 2023-10-09T09:07:02.386 INFO:tasks.rados.rados.0.smithi163.stdout:419: finishing write tid 5 to smithi16367185-35 2023-10-09T09:07:02.386 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 66 (ObjNum 196 snap 54 seq_num 196) dirty exists 2023-10-09T09:07:02.386 INFO:tasks.rados.rados.0.smithi163.stdout:419: left oid 35 (ObjNum 196 snap 54 seq_num 196) 2023-10-09T09:07:02.386 INFO:tasks.rados.rados.0.smithi163.stdout:420: finishing rollback tid 0 to smithi16367185-28 2023-10-09T09:07:02.386 INFO:tasks.rados.rados.0.smithi163.stdout:420: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:07:02.386 INFO:tasks.rados.rados.0.smithi163.stdout:420: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:07:02.386 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 180 (ObjNum 145 snap 34 seq_num 145) dirty exists 2023-10-09T09:07:02.387 INFO:tasks.rados.rados.0.smithi163.stdout:421: finishing write tid 1 to smithi16367185-9 2023-10-09T09:07:02.387 INFO:tasks.rados.rados.0.smithi163.stdout:421: finishing write tid 2 to smithi16367185-9 2023-10-09T09:07:02.387 INFO:tasks.rados.rados.0.smithi163.stdout:421: finishing write tid 3 to smithi16367185-9 2023-10-09T09:07:02.387 INFO:tasks.rados.rados.0.smithi163.stdout:421: finishing write tid 4 to smithi16367185-9 2023-10-09T09:07:02.387 INFO:tasks.rados.rados.0.smithi163.stdout:421: finishing write tid 5 to smithi16367185-9 2023-10-09T09:07:02.387 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 82 (ObjNum 197 snap 54 seq_num 197) dirty exists 2023-10-09T09:07:02.387 INFO:tasks.rados.rados.0.smithi163.stdout:421: left oid 9 (ObjNum 197 snap 54 seq_num 197) 2023-10-09T09:07:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:02 smithi181 bash[27712]: audit 2023-10-09T09:06:59.559631+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:02 smithi181 bash[27712]: audit 2023-10-09T09:06:59.568863+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:02 smithi181 bash[27712]: audit 2023-10-09T09:06:59.587864+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:02 smithi181 bash[27712]: audit 2023-10-09T09:06:59.597230+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:02 smithi181 bash[27712]: cluster 2023-10-09T09:07:00.122405+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2023-10-09T09:07:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:02 smithi181 bash[27712]: audit 2023-10-09T09:07:00.237712+0000 mon.a (mon.0) 926 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:07:03.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: cluster 2023-10-09T09:07:00.133717+0000 mgr.x (mgr.24751) 12 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:07:03.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:00.235974+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:07:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:00.236306+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:07:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:00.236597+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:07:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:00.370297+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.163:0/4288509398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:01.160882+0000 mon.a (mon.0) 927 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:07:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:01.162367+0000 mon.b (mon.1) 61 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:07:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:02.080989+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:07:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:02.087367+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:07:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:02.087567+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:07:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:02.087734+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:07:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: cluster 2023-10-09T09:07:02.096933+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2023-10-09T09:07:03.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:02.097150+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:03.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:03 smithi181 bash[27712]: audit 2023-10-09T09:07:02.103526+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: cluster 2023-10-09T09:07:00.133717+0000 mgr.x (mgr.24751) 12 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:07:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:00.235974+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:07:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:00.236306+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:07:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:00.236597+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:00.370297+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.163:0/4288509398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:01.160882+0000 mon.a (mon.0) 927 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:01.162367+0000 mon.b (mon.1) 61 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:02.080989+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:02.087367+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:02.087567+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:02.087734+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: cluster 2023-10-09T09:07:02.096933+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:02.097150+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[27500]: audit 2023-10-09T09:07:02.103526+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:03.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: cluster 2023-10-09T09:07:00.133717+0000 mgr.x (mgr.24751) 12 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 191 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:07:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:00.235974+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:07:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:00.236306+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:07:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:00.236597+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:07:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:00.370297+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.163:0/4288509398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:01.160882+0000 mon.a (mon.0) 927 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:07:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:01.162367+0000 mon.b (mon.1) 61 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:07:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:02.080989+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:07:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:02.087367+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:07:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:02.087567+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:07:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:02.087734+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:07:03.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: cluster 2023-10-09T09:07:02.096933+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2023-10-09T09:07:03.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:02.097150+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:03.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:03 smithi163 bash[33912]: audit 2023-10-09T09:07:02.103526+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:04.062 INFO:tasks.rados.rados.0.smithi163.stdout:417: done (5 left) 2023-10-09T09:07:04.062 INFO:tasks.rados.rados.0.smithi163.stdout:418: done (4 left) 2023-10-09T09:07:04.062 INFO:tasks.rados.rados.0.smithi163.stdout:419: done (3 left) 2023-10-09T09:07:04.062 INFO:tasks.rados.rados.0.smithi163.stdout:420: done (2 left) 2023-10-09T09:07:04.062 INFO:tasks.rados.rados.0.smithi163.stdout:421: done (1 left) 2023-10-09T09:07:04.062 INFO:tasks.rados.rados.0.smithi163.stdout:422: done (0 left) 2023-10-09T09:07:04.062 INFO:tasks.rados.rados.0.smithi163.stdout:423: rollback oid 48 current snap is 55 2023-10-09T09:07:04.062 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 48 to 38 2023-10-09T09:07:04.063 INFO:tasks.rados.rados.0.smithi163.stdout:424: snap_create 2023-10-09T09:07:04.073 INFO:tasks.rados.rados.0.smithi163.stdout:423: finishing rollback tid 0 to smithi16367185-48 2023-10-09T09:07:04.073 INFO:tasks.rados.rados.0.smithi163.stdout:423: finishing rollback tid 1 to smithi16367185-48 2023-10-09T09:07:04.073 INFO:tasks.rados.rados.0.smithi163.stdout:423: finishing rollback tid 2 to smithi16367185-48 2023-10-09T09:07:04.073 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 281 (ObjNum 128 snap 27 seq_num 128) dirty exists 2023-10-09T09:07:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:04 smithi181 bash[27712]: audit 2023-10-09T09:07:02.104652+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:07:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:04 smithi181 bash[27712]: audit 2023-10-09T09:07:02.104854+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:07:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:04 smithi181 bash[27712]: audit 2023-10-09T09:07:02.105034+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:07:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:04 smithi181 bash[27712]: audit 2023-10-09T09:07:02.105541+0000 mon.c (mon.2) 52 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:07:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:04 smithi181 bash[27712]: cluster 2023-10-09T09:07:02.134364+0000 mgr.x (mgr.24751) 13 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 MiB/s wr, 15 op/s 2023-10-09T09:07:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:04 smithi181 bash[27712]: audit 2023-10-09T09:07:02.799239+0000 mon.a (mon.0) 931 : audit [DBG] from='client.? 172.21.15.163:0/2781557075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:04 smithi181 bash[27712]: audit 2023-10-09T09:07:03.045851+0000 mon.c (mon.2) 53 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:07:04.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:04 smithi181 bash[27712]: cluster 2023-10-09T09:07:03.053458+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2023-10-09T09:07:04.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:04 smithi181 bash[27712]: audit 2023-10-09T09:07:03.060246+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:04.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:04 smithi181 bash[27712]: audit 2023-10-09T09:07:03.065890+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[33912]: audit 2023-10-09T09:07:02.104652+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:07:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[33912]: audit 2023-10-09T09:07:02.104854+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:07:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[33912]: audit 2023-10-09T09:07:02.105034+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:07:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[33912]: audit 2023-10-09T09:07:02.105541+0000 mon.c (mon.2) 52 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:07:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[33912]: cluster 2023-10-09T09:07:02.134364+0000 mgr.x (mgr.24751) 13 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 MiB/s wr, 15 op/s 2023-10-09T09:07:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[33912]: audit 2023-10-09T09:07:02.799239+0000 mon.a (mon.0) 931 : audit [DBG] from='client.? 172.21.15.163:0/2781557075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[33912]: audit 2023-10-09T09:07:03.045851+0000 mon.c (mon.2) 53 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:07:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[33912]: cluster 2023-10-09T09:07:03.053458+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2023-10-09T09:07:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[33912]: audit 2023-10-09T09:07:03.060246+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[33912]: audit 2023-10-09T09:07:03.065890+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[27500]: audit 2023-10-09T09:07:02.104652+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:07:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[27500]: audit 2023-10-09T09:07:02.104854+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:07:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[27500]: audit 2023-10-09T09:07:02.105034+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:07:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[27500]: audit 2023-10-09T09:07:02.105541+0000 mon.c (mon.2) 52 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:07:04.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[27500]: cluster 2023-10-09T09:07:02.134364+0000 mgr.x (mgr.24751) 13 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 MiB/s wr, 15 op/s 2023-10-09T09:07:04.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[27500]: audit 2023-10-09T09:07:02.799239+0000 mon.a (mon.0) 931 : audit [DBG] from='client.? 172.21.15.163:0/2781557075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:04.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[27500]: audit 2023-10-09T09:07:03.045851+0000 mon.c (mon.2) 53 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:07:04.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[27500]: cluster 2023-10-09T09:07:03.053458+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2023-10-09T09:07:04.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[27500]: audit 2023-10-09T09:07:03.060246+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:04.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:04 smithi163 bash[27500]: audit 2023-10-09T09:07:03.065890+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:05.077 INFO:tasks.rados.rados.0.smithi163.stdout:423: done (1 left) 2023-10-09T09:07:05.077 INFO:tasks.rados.rados.0.smithi163.stdout:424: done (0 left) 2023-10-09T09:07:05.077 INFO:tasks.rados.rados.0.smithi163.stdout:425: snap_remove snap 41 2023-10-09T09:07:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:05 smithi181 bash[27712]: cluster 2023-10-09T09:07:04.059947+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2023-10-09T09:07:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:05 smithi181 bash[27712]: cluster 2023-10-09T09:07:04.135088+0000 mgr.x (mgr.24751) 14 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 MiB/s wr, 15 op/s 2023-10-09T09:07:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:05 smithi163 bash[33912]: cluster 2023-10-09T09:07:04.059947+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2023-10-09T09:07:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:05 smithi163 bash[33912]: cluster 2023-10-09T09:07:04.135088+0000 mgr.x (mgr.24751) 14 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 MiB/s wr, 15 op/s 2023-10-09T09:07:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:05 smithi163 bash[27500]: cluster 2023-10-09T09:07:04.059947+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2023-10-09T09:07:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:05 smithi163 bash[27500]: cluster 2023-10-09T09:07:04.135088+0000 mgr.x (mgr.24751) 14 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 MiB/s wr, 15 op/s 2023-10-09T09:07:06.082 INFO:tasks.rados.rados.0.smithi163.stdout:425: done (0 left) 2023-10-09T09:07:06.082 INFO:tasks.rados.rados.0.smithi163.stdout:426: delete oid 27 current snap is 56 2023-10-09T09:07:06.091 INFO:tasks.rados.rados.0.smithi163.stdout:426: done (0 left) 2023-10-09T09:07:06.092 INFO:tasks.rados.rados.0.smithi163.stdout:427: rollback oid 33 current snap is 56 2023-10-09T09:07:06.092 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 31 2023-10-09T09:07:06.092 INFO:tasks.rados.rados.0.smithi163.stdout:428: read oid 45 snap -1 2023-10-09T09:07:06.092 INFO:tasks.rados.rados.0.smithi163.stdout:428: expect deleted 2023-10-09T09:07:06.092 INFO:tasks.rados.rados.0.smithi163.stdout:429: read oid 26 snap -1 2023-10-09T09:07:06.092 INFO:tasks.rados.rados.0.smithi163.stdout:429: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:07:06.092 INFO:tasks.rados.rados.0.smithi163.stdout:430: write oid 43 current snap is 56 2023-10-09T09:07:06.092 INFO:tasks.rados.rados.0.smithi163.stdout:430: seq_num 199 ranges {544593=799729,1761799=776065,2948352=396241} 2023-10-09T09:07:06.100 INFO:tasks.rados.rados.0.smithi163.stdout:430: writing smithi16367185-43 from 544593 to 1344322 tid 1 2023-10-09T09:07:06.107 INFO:tasks.rados.rados.0.smithi163.stdout:430: writing smithi16367185-43 from 1761799 to 2537864 tid 2 2023-10-09T09:07:06.112 INFO:tasks.rados.rados.0.smithi163.stdout:430: writing smithi16367185-43 from 2948352 to 3344593 tid 3 2023-10-09T09:07:06.112 INFO:tasks.rados.rados.0.smithi163.stdout:431: read oid 47 snap -1 2023-10-09T09:07:06.112 INFO:tasks.rados.rados.0.smithi163.stdout:431: expect deleted 2023-10-09T09:07:06.112 INFO:tasks.rados.rados.0.smithi163.stdout:429: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:07:06.199 INFO:tasks.rados.rados.0.smithi163.stdout:427: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:07:06.199 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 116 (ObjNum 699247040 snap 21875 seq_num 699161712) dirty dne 2023-10-09T09:07:06.199 INFO:tasks.rados.rados.0.smithi163.stdout:430: finishing write tid 1 to smithi16367185-43 2023-10-09T09:07:06.199 INFO:tasks.rados.rados.0.smithi163.stdout:430: finishing write tid 2 to smithi16367185-43 2023-10-09T09:07:06.199 INFO:tasks.rados.rados.0.smithi163.stdout:430: finishing write tid 3 to smithi16367185-43 2023-10-09T09:07:06.199 INFO:tasks.rados.rados.0.smithi163.stdout:430: finishing write tid 4 to smithi16367185-43 2023-10-09T09:07:06.199 INFO:tasks.rados.rados.0.smithi163.stdout:430: finishing write tid 5 to smithi16367185-43 2023-10-09T09:07:06.199 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 179 (ObjNum 199 snap 56 seq_num 199) dirty exists 2023-10-09T09:07:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:430: left oid 43 (ObjNum 199 snap 56 seq_num 199) 2023-10-09T09:07:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:432: read oid 31 snap -1 2023-10-09T09:07:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:432: expect (ObjNum 138 snap 30 seq_num 138) 2023-10-09T09:07:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:427: done (5 left) 2023-10-09T09:07:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:428: done (4 left) 2023-10-09T09:07:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:429: done (3 left) 2023-10-09T09:07:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:430: done (2 left) 2023-10-09T09:07:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:431: done (1 left) 2023-10-09T09:07:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:433: write oid 50 current snap is 56 2023-10-09T09:07:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:433: seq_num 200 ranges {561400=474978,1820281=519723,2859422=457579} 2023-10-09T09:07:06.211 INFO:tasks.rados.rados.0.smithi163.stdout:433: writing smithi16367185-50 from 561400 to 1036378 tid 1 2023-10-09T09:07:06.219 INFO:tasks.rados.rados.0.smithi163.stdout:433: writing smithi16367185-50 from 1820281 to 2340004 tid 2 2023-10-09T09:07:06.224 INFO:tasks.rados.rados.0.smithi163.stdout:433: writing smithi16367185-50 from 2859422 to 3317001 tid 3 2023-10-09T09:07:06.224 INFO:tasks.rados.rados.0.smithi163.stdout:434: write oid 37 current snap is 56 2023-10-09T09:07:06.225 INFO:tasks.rados.rados.0.smithi163.stdout:434: seq_num 201 ranges {578207=550227,1762410=779734,3286845=91362} 2023-10-09T09:07:06.231 INFO:tasks.rados.rados.0.smithi163.stdout:434: writing smithi16367185-37 from 578207 to 1128434 tid 1 2023-10-09T09:07:06.239 INFO:tasks.rados.rados.0.smithi163.stdout:434: writing smithi16367185-37 from 1762410 to 2542144 tid 2 2023-10-09T09:07:06.244 INFO:tasks.rados.rados.0.smithi163.stdout:434: writing smithi16367185-37 from 3286845 to 3378207 tid 3 2023-10-09T09:07:06.244 INFO:tasks.rados.rados.0.smithi163.stdout:435: rollback oid 40 current snap is 56 2023-10-09T09:07:06.244 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 40 to 47 2023-10-09T09:07:06.244 INFO:tasks.rados.rados.0.smithi163.stdout:436: snap_remove snap 53 2023-10-09T09:07:06.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:06 smithi163 bash[33912]: cluster 2023-10-09T09:07:05.077784+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2023-10-09T09:07:06.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:06 smithi163 bash[33912]: audit 2023-10-09T09:07:05.244006+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.163:0/2630599063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:06.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:06 smithi163 bash[33912]: audit 2023-10-09T09:07:05.407097+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:06 smithi163 bash[27500]: cluster 2023-10-09T09:07:05.077784+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2023-10-09T09:07:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:06 smithi163 bash[27500]: audit 2023-10-09T09:07:05.244006+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.163:0/2630599063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:06 smithi163 bash[27500]: audit 2023-10-09T09:07:05.407097+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:06 smithi181 bash[27712]: cluster 2023-10-09T09:07:05.077784+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2023-10-09T09:07:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:06 smithi181 bash[27712]: audit 2023-10-09T09:07:05.244006+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.163:0/2630599063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:06 smithi181 bash[27712]: audit 2023-10-09T09:07:05.407097+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:07.102 INFO:tasks.rados.rados.0.smithi163.stdout:432: expect (ObjNum 138 snap 30 seq_num 138) 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:433: finishing write tid 1 to smithi16367185-50 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:433: finishing write tid 2 to smithi16367185-50 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:433: finishing write tid 3 to smithi16367185-50 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:433: finishing write tid 4 to smithi16367185-50 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:433: finishing write tid 5 to smithi16367185-50 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 105 (ObjNum 200 snap 56 seq_num 200) dirty exists 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:433: left oid 50 (ObjNum 200 snap 56 seq_num 200) 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:434: finishing write tid 1 to smithi16367185-37 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:434: finishing write tid 2 to smithi16367185-37 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:434: finishing write tid 3 to smithi16367185-37 2023-10-09T09:07:07.203 INFO:tasks.rados.rados.0.smithi163.stdout:434: finishing write tid 4 to smithi16367185-37 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:434: finishing write tid 5 to smithi16367185-37 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 192 (ObjNum 201 snap 56 seq_num 201) dirty exists 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:434: left oid 37 (ObjNum 201 snap 56 seq_num 201) 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:435: finishing rollback tid 1 to smithi16367185-40 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 148 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:432: done (4 left) 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:433: done (3 left) 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:434: done (2 left) 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:435: done (1 left) 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:436: done (0 left) 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:437: write oid 31 current snap is 56 2023-10-09T09:07:07.204 INFO:tasks.rados.rados.0.smithi163.stdout:437: seq_num 202 ranges {595014=625476,1820892=523392,2797915=596476} 2023-10-09T09:07:07.215 INFO:tasks.rados.rados.0.smithi163.stdout:437: writing smithi16367185-31 from 595014 to 1220490 tid 1 2023-10-09T09:07:07.222 INFO:tasks.rados.rados.0.smithi163.stdout:437: writing smithi16367185-31 from 1820892 to 2344284 tid 2 2023-10-09T09:07:07.229 INFO:tasks.rados.rados.0.smithi163.stdout:437: writing smithi16367185-31 from 2797915 to 3394391 tid 3 2023-10-09T09:07:07.229 INFO:tasks.rados.rados.0.smithi163.stdout:438: write oid 49 current snap is 56 2023-10-09T09:07:07.229 INFO:tasks.rados.rados.0.smithi163.stdout:438: seq_num 203 ranges {611821=700725,1879374=783403,3341691=70130} 2023-10-09T09:07:07.237 INFO:tasks.rados.rados.0.smithi163.stdout:438: writing smithi16367185-49 from 611821 to 1312546 tid 1 2023-10-09T09:07:07.245 INFO:tasks.rados.rados.0.smithi163.stdout:438: writing smithi16367185-49 from 1879374 to 2662777 tid 2 2023-10-09T09:07:07.250 INFO:tasks.rados.rados.0.smithi163.stdout:438: writing smithi16367185-49 from 3341691 to 3411821 tid 3 2023-10-09T09:07:07.250 INFO:tasks.rados.rados.0.smithi163.stdout:439: snap_create 2023-10-09T09:07:07.250 INFO:tasks.rados.rados.0.smithi163.stdout:437: finishing write tid 1 to smithi16367185-31 2023-10-09T09:07:07.250 INFO:tasks.rados.rados.0.smithi163.stdout:437: finishing write tid 2 to smithi16367185-31 2023-10-09T09:07:07.250 INFO:tasks.rados.rados.0.smithi163.stdout:437: finishing write tid 3 to smithi16367185-31 2023-10-09T09:07:07.250 INFO:tasks.rados.rados.0.smithi163.stdout:437: finishing write tid 4 to smithi16367185-31 2023-10-09T09:07:07.251 INFO:tasks.rados.rados.0.smithi163.stdout:437: finishing write tid 5 to smithi16367185-31 2023-10-09T09:07:07.251 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 126 (ObjNum 202 snap 56 seq_num 202) dirty exists 2023-10-09T09:07:07.251 INFO:tasks.rados.rados.0.smithi163.stdout:437: left oid 31 (ObjNum 202 snap 56 seq_num 202) 2023-10-09T09:07:07.251 INFO:tasks.rados.rados.0.smithi163.stdout:438: finishing write tid 1 to smithi16367185-49 2023-10-09T09:07:07.251 INFO:tasks.rados.rados.0.smithi163.stdout:438: finishing write tid 2 to smithi16367185-49 2023-10-09T09:07:07.252 INFO:tasks.rados.rados.0.smithi163.stdout:438: finishing write tid 3 to smithi16367185-49 2023-10-09T09:07:07.252 INFO:tasks.rados.rados.0.smithi163.stdout:438: finishing write tid 4 to smithi16367185-49 2023-10-09T09:07:07.252 INFO:tasks.rados.rados.0.smithi163.stdout:438: finishing write tid 5 to smithi16367185-49 2023-10-09T09:07:07.252 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 76 (ObjNum 203 snap 56 seq_num 203) dirty exists 2023-10-09T09:07:07.253 INFO:tasks.rados.rados.0.smithi163.stdout:438: left oid 49 (ObjNum 203 snap 56 seq_num 203) 2023-10-09T09:07:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:07 smithi163 bash[33912]: cluster 2023-10-09T09:07:06.080372+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2023-10-09T09:07:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:07 smithi163 bash[33912]: cluster 2023-10-09T09:07:06.135887+0000 mgr.x (mgr.24751) 15 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:07:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:07 smithi163 bash[27500]: cluster 2023-10-09T09:07:06.080372+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2023-10-09T09:07:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:07 smithi163 bash[27500]: cluster 2023-10-09T09:07:06.135887+0000 mgr.x (mgr.24751) 15 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:07:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:07 smithi181 bash[27712]: cluster 2023-10-09T09:07:06.080372+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2023-10-09T09:07:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:07 smithi181 bash[27712]: cluster 2023-10-09T09:07:06.135887+0000 mgr.x (mgr.24751) 15 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:07:08.123 INFO:tasks.rados.rados.0.smithi163.stdout:437: done (2 left) 2023-10-09T09:07:08.123 INFO:tasks.rados.rados.0.smithi163.stdout:438: done (1 left) 2023-10-09T09:07:08.123 INFO:tasks.rados.rados.0.smithi163.stdout:439: done (0 left) 2023-10-09T09:07:08.123 INFO:tasks.rados.rados.0.smithi163.stdout:440: snap_remove snap 44 2023-10-09T09:07:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:08 smithi163 bash[33912]: cluster 2023-10-09T09:07:07.101049+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2023-10-09T09:07:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:08 smithi163 bash[33912]: cluster 2023-10-09T09:07:07.113840+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2023-10-09T09:07:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:08 smithi163 bash[33912]: audit 2023-10-09T09:07:07.691458+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.163:0/2778257042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:08 smithi163 bash[27500]: cluster 2023-10-09T09:07:07.101049+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2023-10-09T09:07:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:08 smithi163 bash[27500]: cluster 2023-10-09T09:07:07.113840+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2023-10-09T09:07:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:08 smithi163 bash[27500]: audit 2023-10-09T09:07:07.691458+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.163:0/2778257042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:08 smithi181 bash[27712]: cluster 2023-10-09T09:07:07.101049+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2023-10-09T09:07:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:08 smithi181 bash[27712]: cluster 2023-10-09T09:07:07.113840+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2023-10-09T09:07:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:08 smithi181 bash[27712]: audit 2023-10-09T09:07:07.691458+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.163:0/2778257042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:08.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:07:08 smithi181 bash[50540]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:07:08] "GET /metrics HTTP/1.1" 200 36090 "" "Prometheus/2.33.4" 2023-10-09T09:07:09.137 INFO:tasks.rados.rados.0.smithi163.stdout:440: done (0 left) 2023-10-09T09:07:09.137 INFO:tasks.rados.rados.0.smithi163.stdout:441: read oid 36 snap -1 2023-10-09T09:07:09.137 INFO:tasks.rados.rados.0.smithi163.stdout:441: expect (ObjNum 134 snap 28 seq_num 134) 2023-10-09T09:07:09.137 INFO:tasks.rados.rados.0.smithi163.stdout:442: read oid 35 snap 49 2023-10-09T09:07:09.137 INFO:tasks.rados.rados.0.smithi163.stdout:442: expect deleted 2023-10-09T09:07:09.138 INFO:tasks.rados.rados.0.smithi163.stdout:443: snap_remove snap 56 2023-10-09T09:07:09.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:09 smithi163 bash[27500]: cluster 2023-10-09T09:07:08.123577+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2023-10-09T09:07:09.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:09 smithi163 bash[27500]: cluster 2023-10-09T09:07:08.136509+0000 mgr.x (mgr.24751) 16 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 210 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 26 op/s 2023-10-09T09:07:09.392 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:09 smithi163 bash[33912]: cluster 2023-10-09T09:07:08.123577+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2023-10-09T09:07:09.392 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:09 smithi163 bash[33912]: cluster 2023-10-09T09:07:08.136509+0000 mgr.x (mgr.24751) 16 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 210 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 26 op/s 2023-10-09T09:07:09.475 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:09 smithi181 bash[27712]: cluster 2023-10-09T09:07:08.123577+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2023-10-09T09:07:09.475 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:09 smithi181 bash[27712]: cluster 2023-10-09T09:07:08.136509+0000 mgr.x (mgr.24751) 16 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 210 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 26 op/s 2023-10-09T09:07:10.139 INFO:tasks.rados.rados.0.smithi163.stdout:443: done (2 left) 2023-10-09T09:07:10.139 INFO:tasks.rados.rados.0.smithi163.stdout:444: rollback oid 37 current snap is 57 2023-10-09T09:07:10.139 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 48 2023-10-09T09:07:10.139 INFO:tasks.rados.rados.0.smithi163.stdout:441: expect (ObjNum 134 snap 28 seq_num 134) 2023-10-09T09:07:10.213 INFO:tasks.rados.rados.0.smithi163.stdout:444: finishing rollback tid 0 to smithi16367185-37 2023-10-09T09:07:10.213 INFO:tasks.rados.rados.0.smithi163.stdout:444: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:07:10.213 INFO:tasks.rados.rados.0.smithi163.stdout:444: finishing rollback tid 2 to smithi16367185-37 2023-10-09T09:07:10.214 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 202 (ObjNum 36 snap 0 seq_num 36) dirty exists 2023-10-09T09:07:10.214 INFO:tasks.rados.rados.0.smithi163.stdout:441: done (2 left) 2023-10-09T09:07:10.214 INFO:tasks.rados.rados.0.smithi163.stdout:442: done (1 left) 2023-10-09T09:07:10.214 INFO:tasks.rados.rados.0.smithi163.stdout:444: done (0 left) 2023-10-09T09:07:10.214 INFO:tasks.rados.rados.0.smithi163.stdout:445: snap_remove snap 48 2023-10-09T09:07:10.217 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:07:09 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:07:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:07:10.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:10 smithi163 bash[33912]: cluster 2023-10-09T09:07:09.136310+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2023-10-09T09:07:10.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:10 smithi163 bash[27500]: cluster 2023-10-09T09:07:09.136310+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2023-10-09T09:07:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:10 smithi181 bash[27712]: cluster 2023-10-09T09:07:09.136310+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2023-10-09T09:07:11.155 INFO:tasks.rados.rados.0.smithi163.stdout:445: done (0 left) 2023-10-09T09:07:11.155 INFO:tasks.rados.rados.0.smithi163.stdout:446: write oid 49 current snap is 57 2023-10-09T09:07:11.156 INFO:tasks.rados.rados.0.smithi163.stdout:446: seq_num 204 ranges {628628=775974,1937856=527061,3252761=175867} 2023-10-09T09:07:11.164 INFO:tasks.rados.rados.0.smithi163.stdout:446: writing smithi16367185-49 from 628628 to 1404602 tid 1 2023-10-09T09:07:11.172 INFO:tasks.rados.rados.0.smithi163.stdout:446: writing smithi16367185-49 from 1937856 to 2464917 tid 2 2023-10-09T09:07:11.178 INFO:tasks.rados.rados.0.smithi163.stdout:446: writing smithi16367185-49 from 3252761 to 3428628 tid 3 2023-10-09T09:07:11.178 INFO:tasks.rados.rados.0.smithi163.stdout:447: read oid 34 snap 55 2023-10-09T09:07:11.178 INFO:tasks.rados.rados.0.smithi163.stdout:447: expect (ObjNum 112 snap 18 seq_num 112) 2023-10-09T09:07:11.178 INFO:tasks.rados.rados.0.smithi163.stdout:448: read oid 8 snap -1 2023-10-09T09:07:11.178 INFO:tasks.rados.rados.0.smithi163.stdout:448: expect (ObjNum 193 snap 52 seq_num 193) 2023-10-09T09:07:11.178 INFO:tasks.rados.rados.0.smithi163.stdout:449: write oid 42 current snap is 57 2023-10-09T09:07:11.178 INFO:tasks.rados.rados.0.smithi163.stdout:446: finishing write tid 1 to smithi16367185-49 2023-10-09T09:07:11.178 INFO:tasks.rados.rados.0.smithi163.stdout:446: finishing write tid 2 to smithi16367185-49 2023-10-09T09:07:11.178 INFO:tasks.rados.rados.0.smithi163.stdout:449: seq_num 205 ranges {645435=451223,1879985=787072,3280184=165251} 2023-10-09T09:07:11.184 INFO:tasks.rados.rados.0.smithi163.stdout:449: writing smithi16367185-42 from 645435 to 1096658 tid 1 2023-10-09T09:07:11.194 INFO:tasks.rados.rados.0.smithi163.stdout:449: writing smithi16367185-42 from 1879985 to 2667057 tid 2 2023-10-09T09:07:11.200 INFO:tasks.rados.rados.0.smithi163.stdout:449: writing smithi16367185-42 from 3280184 to 3445435 tid 3 2023-10-09T09:07:11.200 INFO:tasks.rados.rados.0.smithi163.stdout:450: snap_create 2023-10-09T09:07:11.200 INFO:tasks.rados.rados.0.smithi163.stdout:446: finishing write tid 3 to smithi16367185-49 2023-10-09T09:07:11.200 INFO:tasks.rados.rados.0.smithi163.stdout:446: finishing write tid 4 to smithi16367185-49 2023-10-09T09:07:11.201 INFO:tasks.rados.rados.0.smithi163.stdout:446: finishing write tid 5 to smithi16367185-49 2023-10-09T09:07:11.201 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 84 (ObjNum 204 snap 57 seq_num 204) dirty exists 2023-10-09T09:07:11.201 INFO:tasks.rados.rados.0.smithi163.stdout:446: left oid 49 (ObjNum 204 snap 57 seq_num 204) 2023-10-09T09:07:11.201 INFO:tasks.rados.rados.0.smithi163.stdout:447: expect (ObjNum 112 snap 18 seq_num 112) 2023-10-09T09:07:11.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:11 smithi181 bash[27712]: cluster 2023-10-09T09:07:10.137039+0000 mgr.x (mgr.24751) 17 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 210 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:07:11.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:11 smithi181 bash[27712]: cluster 2023-10-09T09:07:10.137392+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2023-10-09T09:07:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:11 smithi181 bash[27712]: audit 2023-10-09T09:07:10.142959+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.163:0/2529932660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:11.417 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:11 smithi163 bash[33912]: cluster 2023-10-09T09:07:10.137039+0000 mgr.x (mgr.24751) 17 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 210 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:07:11.417 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:11 smithi163 bash[33912]: cluster 2023-10-09T09:07:10.137392+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2023-10-09T09:07:11.417 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:11 smithi163 bash[33912]: audit 2023-10-09T09:07:10.142959+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.163:0/2529932660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:11.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:11 smithi163 bash[27500]: cluster 2023-10-09T09:07:10.137039+0000 mgr.x (mgr.24751) 17 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 210 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:07:11.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:11 smithi163 bash[27500]: cluster 2023-10-09T09:07:10.137392+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2023-10-09T09:07:11.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:11 smithi163 bash[27500]: audit 2023-10-09T09:07:10.142959+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.163:0/2529932660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:11.448 INFO:tasks.rados.rados.0.smithi163.stdout:449: finishing write tid 1 to smithi16367185-42 2023-10-09T09:07:11.448 INFO:tasks.rados.rados.0.smithi163.stdout:448: expect (ObjNum 193 snap 52 seq_num 193) 2023-10-09T09:07:11.597 INFO:tasks.rados.rados.0.smithi163.stdout:449: finishing write tid 2 to smithi16367185-42 2023-10-09T09:07:11.597 INFO:tasks.rados.rados.0.smithi163.stdout:449: finishing write tid 3 to smithi16367185-42 2023-10-09T09:07:11.598 INFO:tasks.rados.rados.0.smithi163.stdout:449: finishing write tid 4 to smithi16367185-42 2023-10-09T09:07:11.598 INFO:tasks.rados.rados.0.smithi163.stdout:449: finishing write tid 5 to smithi16367185-42 2023-10-09T09:07:11.598 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 306 (ObjNum 205 snap 57 seq_num 205) dirty exists 2023-10-09T09:07:11.598 INFO:tasks.rados.rados.0.smithi163.stdout:449: left oid 42 (ObjNum 205 snap 57 seq_num 205) 2023-10-09T09:07:12.168 INFO:tasks.rados.rados.0.smithi163.stdout:446: done (4 left) 2023-10-09T09:07:12.168 INFO:tasks.rados.rados.0.smithi163.stdout:447: done (3 left) 2023-10-09T09:07:12.169 INFO:tasks.rados.rados.0.smithi163.stdout:448: done (2 left) 2023-10-09T09:07:12.169 INFO:tasks.rados.rados.0.smithi163.stdout:449: done (1 left) 2023-10-09T09:07:12.169 INFO:tasks.rados.rados.0.smithi163.stdout:450: done (0 left) 2023-10-09T09:07:12.169 INFO:tasks.rados.rados.0.smithi163.stdout:451: snap_create 2023-10-09T09:07:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: cluster 2023-10-09T09:07:11.153422+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2023-10-09T09:07:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.065303+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.073751+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.074412+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.077076+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.077391+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.079040+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.079775+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.081323+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.081646+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.084078+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[27712]: audit 2023-10-09T09:07:12.093606+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: cluster 2023-10-09T09:07:11.153422+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2023-10-09T09:07:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.065303+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.073751+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.074412+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.077076+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.077391+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.079040+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.079775+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.081323+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.081646+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.084078+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[27500]: audit 2023-10-09T09:07:12.093606+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:12.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: cluster 2023-10-09T09:07:11.153422+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2023-10-09T09:07:12.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.065303+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:12.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.073751+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:12.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.074412+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.077076+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.077391+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.079040+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.079775+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.081323+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.081646+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.084078+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:12.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:12 smithi163 bash[33912]: audit 2023-10-09T09:07:12.093606+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:12.567 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:12 smithi181 bash[48295]: ts=2023-10-09T09:07:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.163:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:07:13.172 INFO:tasks.rados.rados.0.smithi163.stdout:451: done (0 left) 2023-10-09T09:07:13.172 INFO:tasks.rados.rados.0.smithi163.stdout:452: delete oid 44 current snap is 59 2023-10-09T09:07:13.178 INFO:tasks.rados.rados.0.smithi163.stdout:452: done (0 left) 2023-10-09T09:07:13.178 INFO:tasks.rados.rados.0.smithi163.stdout:453: rollback oid 27 current snap is 59 2023-10-09T09:07:13.178 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 27 to 49 2023-10-09T09:07:13.178 INFO:tasks.rados.rados.0.smithi163.stdout:454: rollback oid 7 current snap is 59 2023-10-09T09:07:13.178 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 42 2023-10-09T09:07:13.178 INFO:tasks.rados.rados.0.smithi163.stdout:455: read oid 25 snap -1 2023-10-09T09:07:13.179 INFO:tasks.rados.rados.0.smithi163.stdout:455: expect (ObjNum 194 snap 54 seq_num 194) 2023-10-09T09:07:13.179 INFO:tasks.rados.rados.0.smithi163.stdout:456: read oid 48 snap -1 2023-10-09T09:07:13.179 INFO:tasks.rados.rados.0.smithi163.stdout:456: expect (ObjNum 128 snap 27 seq_num 128) 2023-10-09T09:07:13.179 INFO:tasks.rados.rados.0.smithi163.stdout:457: write oid 8 current snap is 59 2023-10-09T09:07:13.179 INFO:tasks.rados.rados.0.smithi163.stdout:457: seq_num 207 ranges {679049=718074,2113302=674388,3335030=144019} 2023-10-09T09:07:13.187 INFO:tasks.rados.rados.0.smithi163.stdout:457: writing smithi16367185-8 from 679049 to 1397123 tid 1 2023-10-09T09:07:13.196 INFO:tasks.rados.rados.0.smithi163.stdout:457: writing smithi16367185-8 from 2113302 to 2787690 tid 2 2023-10-09T09:07:13.199 INFO:tasks.rados.rados.0.smithi163.stdout:457: writing smithi16367185-8 from 3335030 to 3479049 tid 3 2023-10-09T09:07:13.200 INFO:tasks.rados.rados.0.smithi163.stdout:458: snap_remove snap 52 2023-10-09T09:07:13.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: cephadm 2023-10-09T09:07:12.083477+0000 mgr.x (mgr.24751) 18 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 2524M 2023-10-09T09:07:13.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: cluster 2023-10-09T09:07:12.137768+0000 mgr.x (mgr.24751) 19 : cluster [DBG] pgmap v35: 89 pgs: 6 active+clean+snaptrim, 82 active+clean, 1 active+clean+snaptrim_wait; 217 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2023-10-09T09:07:13.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: cluster 2023-10-09T09:07:12.167460+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2023-10-09T09:07:13.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.567294+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.577388+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.578146+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.580657+0000 mon.b (mon.1) 75 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.580958+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.581951+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.163:0/1751695771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.582856+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.583380+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.585015+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.585522+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.587617+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.595903+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.595990+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[33912]: audit 2023-10-09T09:07:12.597243+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:07:13.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: cephadm 2023-10-09T09:07:12.083477+0000 mgr.x (mgr.24751) 18 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 2524M 2023-10-09T09:07:13.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: cluster 2023-10-09T09:07:12.137768+0000 mgr.x (mgr.24751) 19 : cluster [DBG] pgmap v35: 89 pgs: 6 active+clean+snaptrim, 82 active+clean, 1 active+clean+snaptrim_wait; 217 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2023-10-09T09:07:13.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: cluster 2023-10-09T09:07:12.167460+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2023-10-09T09:07:13.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.567294+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:13.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.577388+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:13.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.578146+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.580657+0000 mon.b (mon.1) 75 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.580958+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.581951+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.163:0/1751695771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:13.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.582856+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.583380+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.585015+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.585522+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.587617+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.595903+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:07:13.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.595990+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:13.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[27500]: audit 2023-10-09T09:07:12.597243+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:07:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: cephadm 2023-10-09T09:07:12.083477+0000 mgr.x (mgr.24751) 18 : cephadm [INF] Adjusting osd_memory_target on smithi163 to 2524M 2023-10-09T09:07:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: cluster 2023-10-09T09:07:12.137768+0000 mgr.x (mgr.24751) 19 : cluster [DBG] pgmap v35: 89 pgs: 6 active+clean+snaptrim, 82 active+clean, 1 active+clean+snaptrim_wait; 217 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2023-10-09T09:07:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: cluster 2023-10-09T09:07:12.167460+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2023-10-09T09:07:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.567294+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.577388+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.578146+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.580657+0000 mon.b (mon.1) 75 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.580958+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.581951+0000 mon.a (mon.0) 957 : audit [DBG] from='client.? 172.21.15.163:0/1751695771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.582856+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.583380+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.585015+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.585522+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.587617+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:07:13.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.595903+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:07:13.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.595990+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:13.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:13 smithi181 bash[27712]: audit 2023-10-09T09:07:12.597243+0000 mon.b (mon.1) 79 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:07:14.166 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:13 smithi163 systemd[1]: Stopping Ceph node-exporter.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:07:14.166 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:13 smithi163 bash[70109]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-node-exporter.a 2023-10-09T09:07:14.198 INFO:tasks.rados.rados.0.smithi163.stdout:458: done (5 left) 2023-10-09T09:07:14.198 INFO:tasks.rados.rados.0.smithi163.stdout:459: write oid 44 current snap is 59 2023-10-09T09:07:14.198 INFO:tasks.rados.rados.0.smithi163.stdout:459: seq_num 208 ranges {695856=793323,2171784=534399,3362453=133403} 2023-10-09T09:07:14.199 INFO:tasks.rados.rados.0.smithi163.stdout:459: writing smithi16367185-44 from 695856 to 1489179 tid 1 2023-10-09T09:07:14.206 INFO:tasks.rados.rados.0.smithi163.stdout:459: writing smithi16367185-44 from 2171784 to 2706183 tid 2 2023-10-09T09:07:14.212 INFO:tasks.rados.rados.0.smithi163.stdout:459: writing smithi16367185-44 from 3362453 to 3495856 tid 3 2023-10-09T09:07:14.212 INFO:tasks.rados.rados.0.smithi163.stdout:460: delete oid 31 current snap is 59 2023-10-09T09:07:14.212 INFO:tasks.rados.rados.0.smithi163.stdout:453: finishing rollback tid 1 to smithi16367185-27 2023-10-09T09:07:14.212 INFO:tasks.rados.rados.0.smithi163.stdout:456: expect (ObjNum 128 snap 27 seq_num 128) 2023-10-09T09:07:14.373 INFO:tasks.rados.rados.0.smithi163.stdout:454: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:07:14.373 INFO:tasks.rados.rados.0.smithi163.stdout:454: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:07:14.373 INFO:tasks.rados.rados.0.smithi163.stdout:454: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:07:14.373 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 87 (ObjNum 156 snap 39 seq_num 156) dirty exists 2023-10-09T09:07:14.373 INFO:tasks.rados.rados.0.smithi163.stdout:453: finishing rollback tid 2 to smithi16367185-27 2023-10-09T09:07:14.373 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 111 (ObjNum 172 snap 45 seq_num 172) dirty exists 2023-10-09T09:07:14.374 INFO:tasks.rados.rados.0.smithi163.stdout:455: expect (ObjNum 194 snap 54 seq_num 194) 2023-10-09T09:07:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: cephadm 2023-10-09T09:07:12.586746+0000 mgr.x (mgr.24751) 20 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 2524M 2023-10-09T09:07:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: cephadm 2023-10-09T09:07:12.599187+0000 mgr.x (mgr.24751) 21 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:07:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: cephadm 2023-10-09T09:07:12.599533+0000 mgr.x (mgr.24751) 22 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:07:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: cephadm 2023-10-09T09:07:12.765469+0000 mgr.x (mgr.24751) 23 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:07:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: cephadm 2023-10-09T09:07:12.816063+0000 mgr.x (mgr.24751) 24 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:07:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: cephadm 2023-10-09T09:07:12.930819+0000 mgr.x (mgr.24751) 25 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:07:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: cephadm 2023-10-09T09:07:13.031223+0000 mgr.x (mgr.24751) 26 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:07:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: cephadm 2023-10-09T09:07:13.095665+0000 mgr.x (mgr.24751) 27 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:07:14.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: cluster 2023-10-09T09:07:13.170982+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: cephadm 2023-10-09T09:07:13.196653+0000 mgr.x (mgr.24751) 28 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: audit 2023-10-09T09:07:13.271792+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: audit 2023-10-09T09:07:13.283257+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: audit 2023-10-09T09:07:13.375240+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: audit 2023-10-09T09:07:13.384794+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: audit 2023-10-09T09:07:13.393205+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: audit 2023-10-09T09:07:13.472965+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: audit 2023-10-09T09:07:13.478232+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: audit 2023-10-09T09:07:13.483117+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[27500]: audit 2023-10-09T09:07:13.488230+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: cephadm 2023-10-09T09:07:12.586746+0000 mgr.x (mgr.24751) 20 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 2524M 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: cephadm 2023-10-09T09:07:12.599187+0000 mgr.x (mgr.24751) 21 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: cephadm 2023-10-09T09:07:12.599533+0000 mgr.x (mgr.24751) 22 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: cephadm 2023-10-09T09:07:12.765469+0000 mgr.x (mgr.24751) 23 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: cephadm 2023-10-09T09:07:12.816063+0000 mgr.x (mgr.24751) 24 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: cephadm 2023-10-09T09:07:12.930819+0000 mgr.x (mgr.24751) 25 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: cephadm 2023-10-09T09:07:13.031223+0000 mgr.x (mgr.24751) 26 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: cephadm 2023-10-09T09:07:13.095665+0000 mgr.x (mgr.24751) 27 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: cluster 2023-10-09T09:07:13.170982+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: cephadm 2023-10-09T09:07:13.196653+0000 mgr.x (mgr.24751) 28 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: audit 2023-10-09T09:07:13.271792+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: audit 2023-10-09T09:07:13.283257+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: audit 2023-10-09T09:07:13.375240+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: audit 2023-10-09T09:07:13.384794+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: audit 2023-10-09T09:07:13.393205+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: audit 2023-10-09T09:07:13.472965+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: audit 2023-10-09T09:07:13.478232+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: audit 2023-10-09T09:07:13.483117+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[33912]: audit 2023-10-09T09:07:13.488230+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: cephadm 2023-10-09T09:07:12.586746+0000 mgr.x (mgr.24751) 20 : cephadm [INF] Adjusting osd_memory_target on smithi181 to 2524M 2023-10-09T09:07:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: cephadm 2023-10-09T09:07:12.599187+0000 mgr.x (mgr.24751) 21 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:07:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: cephadm 2023-10-09T09:07:12.599533+0000 mgr.x (mgr.24751) 22 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:07:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: cephadm 2023-10-09T09:07:12.765469+0000 mgr.x (mgr.24751) 23 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:07:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: cephadm 2023-10-09T09:07:12.816063+0000 mgr.x (mgr.24751) 24 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:07:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: cephadm 2023-10-09T09:07:12.930819+0000 mgr.x (mgr.24751) 25 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:07:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: cephadm 2023-10-09T09:07:13.031223+0000 mgr.x (mgr.24751) 26 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:07:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: cephadm 2023-10-09T09:07:13.095665+0000 mgr.x (mgr.24751) 27 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:07:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: cluster 2023-10-09T09:07:13.170982+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2023-10-09T09:07:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: cephadm 2023-10-09T09:07:13.196653+0000 mgr.x (mgr.24751) 28 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:07:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: audit 2023-10-09T09:07:13.271792+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: audit 2023-10-09T09:07:13.283257+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: audit 2023-10-09T09:07:13.375240+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: audit 2023-10-09T09:07:13.384794+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: audit 2023-10-09T09:07:13.393205+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: audit 2023-10-09T09:07:13.472965+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: audit 2023-10-09T09:07:13.478232+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: audit 2023-10-09T09:07:13.483117+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:14 smithi181 bash[27712]: audit 2023-10-09T09:07:13.488230+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:14.663 INFO:tasks.rados.rados.0.smithi163.stdout:457: finishing write tid 1 to smithi16367185-8 2023-10-09T09:07:14.663 INFO:tasks.rados.rados.0.smithi163.stdout:457: finishing write tid 2 to smithi16367185-8 2023-10-09T09:07:14.663 INFO:tasks.rados.rados.0.smithi163.stdout:457: finishing write tid 3 to smithi16367185-8 2023-10-09T09:07:14.663 INFO:tasks.rados.rados.0.smithi163.stdout:457: finishing write tid 4 to smithi16367185-8 2023-10-09T09:07:14.663 INFO:tasks.rados.rados.0.smithi163.stdout:457: finishing write tid 5 to smithi16367185-8 2023-10-09T09:07:14.663 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 209 (ObjNum 207 snap 59 seq_num 207) dirty exists 2023-10-09T09:07:14.663 INFO:tasks.rados.rados.0.smithi163.stdout:457: left oid 8 (ObjNum 207 snap 59 seq_num 207) 2023-10-09T09:07:14.663 INFO:tasks.rados.rados.0.smithi163.stdout:459: finishing write tid 1 to smithi16367185-44 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:459: finishing write tid 2 to smithi16367185-44 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:459: finishing write tid 3 to smithi16367185-44 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:459: finishing write tid 4 to smithi16367185-44 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:459: finishing write tid 5 to smithi16367185-44 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 318 (ObjNum 208 snap 59 seq_num 208) dirty exists 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:459: left oid 44 (ObjNum 208 snap 59 seq_num 208) 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:453: done (6 left) 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:454: done (5 left) 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:455: done (4 left) 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:456: done (3 left) 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:457: done (2 left) 2023-10-09T09:07:14.664 INFO:tasks.rados.rados.0.smithi163.stdout:459: done (1 left) 2023-10-09T09:07:14.665 INFO:tasks.rados.rados.0.smithi163.stdout:460: done (0 left) 2023-10-09T09:07:14.665 INFO:tasks.rados.rados.0.smithi163.stdout:461: rollback oid 3 current snap is 59 2023-10-09T09:07:14.665 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 55 2023-10-09T09:07:14.665 INFO:tasks.rados.rados.0.smithi163.stdout:462: rollback oid 19 current snap is 59 2023-10-09T09:07:14.665 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 19 to 50 2023-10-09T09:07:14.665 INFO:tasks.rados.rados.0.smithi163.stdout:463: snap_remove snap 40 2023-10-09T09:07:14.860 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[70145]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T09:07:14.861 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:14 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:07:14.861 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:14 smithi163 bash[70231]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T09:07:14.861 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:14 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T09:07:14.861 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:14 smithi163 systemd[1]: Stopped Ceph node-exporter.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:07:14.861 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:14 smithi163 systemd[1]: Started Ceph node-exporter.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:07:15.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[33912]: cephadm 2023-10-09T09:07:13.487882+0000 mgr.x (mgr.24751) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T09:07:15.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[33912]: cephadm 2023-10-09T09:07:13.488655+0000 mgr.x (mgr.24751) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi163 2023-10-09T09:07:15.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[33912]: cluster 2023-10-09T09:07:14.138618+0000 mgr.x (mgr.24751) 31 : cluster [DBG] pgmap v38: 89 pgs: 6 active+clean+snaptrim, 82 active+clean, 1 active+clean+snaptrim_wait; 217 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2023-10-09T09:07:15.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[33912]: cluster 2023-10-09T09:07:14.188282+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2023-10-09T09:07:15.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[33912]: audit 2023-10-09T09:07:14.843998+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:15.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[33912]: audit 2023-10-09T09:07:14.851649+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:15.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[33912]: audit 2023-10-09T09:07:15.019962+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.163:0/2314226813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:15.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[27500]: cephadm 2023-10-09T09:07:13.487882+0000 mgr.x (mgr.24751) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T09:07:15.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[27500]: cephadm 2023-10-09T09:07:13.488655+0000 mgr.x (mgr.24751) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi163 2023-10-09T09:07:15.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[27500]: cluster 2023-10-09T09:07:14.138618+0000 mgr.x (mgr.24751) 31 : cluster [DBG] pgmap v38: 89 pgs: 6 active+clean+snaptrim, 82 active+clean, 1 active+clean+snaptrim_wait; 217 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2023-10-09T09:07:15.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[27500]: cluster 2023-10-09T09:07:14.188282+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2023-10-09T09:07:15.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[27500]: audit 2023-10-09T09:07:14.843998+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:15.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[27500]: audit 2023-10-09T09:07:14.851649+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:15.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[27500]: audit 2023-10-09T09:07:15.019962+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.163:0/2314226813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:15.496 INFO:tasks.rados.rados.0.smithi163.stdout:463: done (2 left) 2023-10-09T09:07:15.496 INFO:tasks.rados.rados.0.smithi163.stdout:464: delete oid 29 current snap is 59 2023-10-09T09:07:15.496 INFO:tasks.rados.rados.0.smithi163.stdout:461: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:07:15.496 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 100 (ObjNum 669367496 snap 21875 seq_num 5) dirty dne 2023-10-09T09:07:15.496 INFO:tasks.rados.rados.0.smithi163.stdout:462: finishing rollback tid 0 to smithi16367185-19 2023-10-09T09:07:15.496 INFO:tasks.rados.rados.0.smithi163.stdout:462: finishing rollback tid 1 to smithi16367185-19 2023-10-09T09:07:15.496 INFO:tasks.rados.rados.0.smithi163.stdout:462: finishing rollback tid 2 to smithi16367185-19 2023-10-09T09:07:15.496 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 117 (ObjNum 140 snap 31 seq_num 140) dirty exists 2023-10-09T09:07:15.503 INFO:tasks.rados.rados.0.smithi163.stdout:461: done (2 left) 2023-10-09T09:07:15.503 INFO:tasks.rados.rados.0.smithi163.stdout:462: done (1 left) 2023-10-09T09:07:15.503 INFO:tasks.rados.rados.0.smithi163.stdout:464: done (0 left) 2023-10-09T09:07:15.503 INFO:tasks.rados.rados.0.smithi163.stdout:465: snap_create 2023-10-09T09:07:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:15 smithi181 bash[27712]: cephadm 2023-10-09T09:07:13.487882+0000 mgr.x (mgr.24751) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T09:07:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:15 smithi181 bash[27712]: cephadm 2023-10-09T09:07:13.488655+0000 mgr.x (mgr.24751) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi163 2023-10-09T09:07:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:15 smithi181 bash[27712]: cluster 2023-10-09T09:07:14.138618+0000 mgr.x (mgr.24751) 31 : cluster [DBG] pgmap v38: 89 pgs: 6 active+clean+snaptrim, 82 active+clean, 1 active+clean+snaptrim_wait; 217 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2023-10-09T09:07:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:15 smithi181 bash[27712]: cluster 2023-10-09T09:07:14.188282+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2023-10-09T09:07:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:15 smithi181 bash[27712]: audit 2023-10-09T09:07:14.843998+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:15 smithi181 bash[27712]: audit 2023-10-09T09:07:14.851649+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:15 smithi181 bash[27712]: audit 2023-10-09T09:07:15.019962+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.163:0/2314226813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:15.968 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.842Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.842Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.843Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.843Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T09:07:15.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T09:07:15.970 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T09:07:15.971 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T09:07:15.973 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T09:07:15.973 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T09:07:15.973 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T09:07:15.973 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T09:07:15.974 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T09:07:15.974 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T09:07:15.974 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T09:07:15.974 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T09:07:15.974 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T09:07:15.974 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T09:07:15.974 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T09:07:15.974 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T09:07:15.974 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.844Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T09:07:15.974 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 09:07:15 smithi163 bash[70258]: ts=2023-10-09T09:07:15.845Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T09:07:16.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:16 smithi163 bash[27500]: cephadm 2023-10-09T09:07:14.857808+0000 mgr.x (mgr.24751) 32 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:07:16.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:16 smithi163 bash[27500]: cephadm 2023-10-09T09:07:14.878653+0000 mgr.x (mgr.24751) 33 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T09:07:16.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:16 smithi163 bash[27500]: cluster 2023-10-09T09:07:15.495827+0000 mon.a (mon.0) 975 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2023-10-09T09:07:16.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:16 smithi163 bash[33912]: cephadm 2023-10-09T09:07:14.857808+0000 mgr.x (mgr.24751) 32 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:07:16.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:16 smithi163 bash[33912]: cephadm 2023-10-09T09:07:14.878653+0000 mgr.x (mgr.24751) 33 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T09:07:16.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:16 smithi163 bash[33912]: cluster 2023-10-09T09:07:15.495827+0000 mon.a (mon.0) 975 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2023-10-09T09:07:16.504 INFO:tasks.rados.rados.0.smithi163.stdout:465: done (0 left) 2023-10-09T09:07:16.504 INFO:tasks.rados.rados.0.smithi163.stdout:466: read oid 26 snap -1 2023-10-09T09:07:16.504 INFO:tasks.rados.rados.0.smithi163.stdout:466: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:07:16.504 INFO:tasks.rados.rados.0.smithi163.stdout:467: delete oid 16 current snap is 60 2023-10-09T09:07:16.512 INFO:tasks.rados.rados.0.smithi163.stdout:466: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:07:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:16 smithi181 bash[27712]: cephadm 2023-10-09T09:07:14.857808+0000 mgr.x (mgr.24751) 32 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:07:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:16 smithi181 bash[27712]: cephadm 2023-10-09T09:07:14.878653+0000 mgr.x (mgr.24751) 33 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T09:07:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:16 smithi181 bash[27712]: cluster 2023-10-09T09:07:15.495827+0000 mon.a (mon.0) 975 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2023-10-09T09:07:16.610 INFO:tasks.rados.rados.0.smithi163.stdout:466: done (1 left) 2023-10-09T09:07:16.610 INFO:tasks.rados.rados.0.smithi163.stdout:467: done (0 left) 2023-10-09T09:07:16.611 INFO:tasks.rados.rados.0.smithi163.stdout:468: snap_remove snap 42 2023-10-09T09:07:17.119 INFO:tasks.rados.rados.0.smithi163.stdout:468: done (0 left) 2023-10-09T09:07:17.119 INFO:tasks.rados.rados.0.smithi163.stdout:469: snap_remove snap 47 2023-10-09T09:07:17.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:17 smithi163 bash[33912]: cluster 2023-10-09T09:07:16.139441+0000 mgr.x (mgr.24751) 34 : cluster [DBG] pgmap v41: 89 pgs: 6 active+clean+snaptrim, 82 active+clean, 1 active+clean+snaptrim_wait; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 700 KiB/s wr, 7 op/s 2023-10-09T09:07:17.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:17 smithi163 bash[33912]: cluster 2023-10-09T09:07:16.503048+0000 mon.a (mon.0) 976 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2023-10-09T09:07:17.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:17 smithi163 bash[33912]: cluster 2023-10-09T09:07:17.118180+0000 mon.a (mon.0) 977 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2023-10-09T09:07:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:17 smithi163 bash[27500]: cluster 2023-10-09T09:07:16.139441+0000 mgr.x (mgr.24751) 34 : cluster [DBG] pgmap v41: 89 pgs: 6 active+clean+snaptrim, 82 active+clean, 1 active+clean+snaptrim_wait; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 700 KiB/s wr, 7 op/s 2023-10-09T09:07:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:17 smithi163 bash[27500]: cluster 2023-10-09T09:07:16.503048+0000 mon.a (mon.0) 976 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2023-10-09T09:07:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:17 smithi163 bash[27500]: cluster 2023-10-09T09:07:17.118180+0000 mon.a (mon.0) 977 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2023-10-09T09:07:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:17 smithi181 bash[27712]: cluster 2023-10-09T09:07:16.139441+0000 mgr.x (mgr.24751) 34 : cluster [DBG] pgmap v41: 89 pgs: 6 active+clean+snaptrim, 82 active+clean, 1 active+clean+snaptrim_wait; 222 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 700 KiB/s wr, 7 op/s 2023-10-09T09:07:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:17 smithi181 bash[27712]: cluster 2023-10-09T09:07:16.503048+0000 mon.a (mon.0) 976 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2023-10-09T09:07:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:17 smithi181 bash[27712]: cluster 2023-10-09T09:07:17.118180+0000 mon.a (mon.0) 977 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2023-10-09T09:07:18.121 INFO:tasks.rados.rados.0.smithi163.stdout:469: done (0 left) 2023-10-09T09:07:18.121 INFO:tasks.rados.rados.0.smithi163.stdout:470: snap_create 2023-10-09T09:07:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:18 smithi163 bash[27500]: audit 2023-10-09T09:07:17.522463+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.163:0/2361373080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:18 smithi163 bash[27500]: cluster 2023-10-09T09:07:18.119987+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2023-10-09T09:07:18.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:18 smithi163 bash[33912]: audit 2023-10-09T09:07:17.522463+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.163:0/2361373080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:18.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:18 smithi163 bash[33912]: cluster 2023-10-09T09:07:18.119987+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2023-10-09T09:07:18.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:18 smithi181 bash[27712]: audit 2023-10-09T09:07:17.522463+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.163:0/2361373080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:18.554 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:18 smithi181 bash[27712]: cluster 2023-10-09T09:07:18.119987+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2023-10-09T09:07:18.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:07:18 smithi181 bash[50540]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:07:18] "GET /metrics HTTP/1.1" 200 36090 "" "Prometheus/2.33.4" 2023-10-09T09:07:19.128 INFO:tasks.rados.rados.0.smithi163.stdout:470: done (0 left) 2023-10-09T09:07:19.128 INFO:tasks.rados.rados.0.smithi163.stdout:471: read oid 27 snap -1 2023-10-09T09:07:19.128 INFO:tasks.rados.rados.0.smithi163.stdout:471: expect (ObjNum 172 snap 45 seq_num 172) 2023-10-09T09:07:19.128 INFO:tasks.rados.rados.0.smithi163.stdout:472: rollback oid 48 current snap is 61 2023-10-09T09:07:19.128 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 48 to 54 2023-10-09T09:07:19.129 INFO:tasks.rados.rados.0.smithi163.stdout:473: read oid 17 snap 55 2023-10-09T09:07:19.130 INFO:tasks.rados.rados.0.smithi163.stdout:473: expect (ObjNum 157 snap 39 seq_num 157) 2023-10-09T09:07:19.130 INFO:tasks.rados.rados.0.smithi163.stdout:474: read oid 14 snap -1 2023-10-09T09:07:19.130 INFO:tasks.rados.rados.0.smithi163.stdout:474: expect (ObjNum 185 snap 49 seq_num 185) 2023-10-09T09:07:19.130 INFO:tasks.rados.rados.0.smithi163.stdout:475: write oid 46 current snap is 61 2023-10-09T09:07:19.130 INFO:tasks.rados.rados.0.smithi163.stdout:475: seq_num 212 ranges {763084=694319,1889359=541737,2955792=541902} 2023-10-09T09:07:19.141 INFO:tasks.rados.rados.0.smithi163.stdout:475: writing smithi16367185-46 from 763084 to 1457403 tid 1 2023-10-09T09:07:19.148 INFO:tasks.rados.rados.0.smithi163.stdout:475: writing smithi16367185-46 from 1889359 to 2431096 tid 2 2023-10-09T09:07:19.154 INFO:tasks.rados.rados.0.smithi163.stdout:475: writing smithi16367185-46 from 2955792 to 3497694 tid 3 2023-10-09T09:07:19.154 INFO:tasks.rados.rados.0.smithi163.stdout:476: snap_remove snap 57 2023-10-09T09:07:19.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:19 smithi163 bash[27500]: cluster 2023-10-09T09:07:18.140083+0000 mgr.x (mgr.24751) 35 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2023-10-09T09:07:19.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:19 smithi163 bash[27500]: cluster 2023-10-09T09:07:19.128039+0000 mon.a (mon.0) 979 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2023-10-09T09:07:19.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:19 smithi163 bash[33912]: cluster 2023-10-09T09:07:18.140083+0000 mgr.x (mgr.24751) 35 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2023-10-09T09:07:19.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:19 smithi163 bash[33912]: cluster 2023-10-09T09:07:19.128039+0000 mon.a (mon.0) 979 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2023-10-09T09:07:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:19 smithi181 bash[27712]: cluster 2023-10-09T09:07:18.140083+0000 mgr.x (mgr.24751) 35 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2023-10-09T09:07:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:19 smithi181 bash[27712]: cluster 2023-10-09T09:07:19.128039+0000 mon.a (mon.0) 979 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2023-10-09T09:07:20.133 INFO:tasks.rados.rados.0.smithi163.stdout:476: done (5 left) 2023-10-09T09:07:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:477: read oid 4 snap -1 2023-10-09T09:07:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:477: expect (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:07:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:478: rollback oid 7 current snap is 61 2023-10-09T09:07:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 49 2023-10-09T09:07:20.134 INFO:tasks.rados.rados.0.smithi163.stdout:472: finishing rollback tid 0 to smithi16367185-48 2023-10-09T09:07:20.135 INFO:tasks.rados.rados.0.smithi163.stdout:472: finishing rollback tid 1 to smithi16367185-48 2023-10-09T09:07:20.135 INFO:tasks.rados.rados.0.smithi163.stdout:472: finishing rollback tid 2 to smithi16367185-48 2023-10-09T09:07:20.135 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 344 (ObjNum 128 snap 27 seq_num 128) dirty exists 2023-10-09T09:07:20.135 INFO:tasks.rados.rados.0.smithi163.stdout:471: expect (ObjNum 172 snap 45 seq_num 172) 2023-10-09T09:07:20.203 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:07:19 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:07:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:07:20.204 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:20 smithi163 bash[27500]: audit 2023-10-09T09:07:19.958406+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.163:0/3280438009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:20.204 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:20 smithi163 bash[27500]: cluster 2023-10-09T09:07:20.132412+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2023-10-09T09:07:20.229 INFO:tasks.rados.rados.0.smithi163.stdout:474: expect (ObjNum 185 snap 49 seq_num 185) 2023-10-09T09:07:20.390 INFO:tasks.rados.rados.0.smithi163.stdout:473: expect (ObjNum 157 snap 39 seq_num 157) 2023-10-09T09:07:20.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:20 smithi163 bash[33912]: audit 2023-10-09T09:07:19.958406+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.163:0/3280438009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:20.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:20 smithi163 bash[33912]: cluster 2023-10-09T09:07:20.132412+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2023-10-09T09:07:20.509 INFO:tasks.rados.rados.0.smithi163.stdout:475: finishing write tid 1 to smithi16367185-46 2023-10-09T09:07:20.510 INFO:tasks.rados.rados.0.smithi163.stdout:475: finishing write tid 2 to smithi16367185-46 2023-10-09T09:07:20.510 INFO:tasks.rados.rados.0.smithi163.stdout:475: finishing write tid 3 to smithi16367185-46 2023-10-09T09:07:20.510 INFO:tasks.rados.rados.0.smithi163.stdout:475: finishing write tid 4 to smithi16367185-46 2023-10-09T09:07:20.510 INFO:tasks.rados.rados.0.smithi163.stdout:475: finishing write tid 5 to smithi16367185-46 2023-10-09T09:07:20.510 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 211 (ObjNum 212 snap 61 seq_num 212) dirty exists 2023-10-09T09:07:20.510 INFO:tasks.rados.rados.0.smithi163.stdout:475: left oid 46 (ObjNum 212 snap 61 seq_num 212) 2023-10-09T09:07:20.510 INFO:tasks.rados.rados.0.smithi163.stdout:478: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:07:20.510 INFO:tasks.rados.rados.0.smithi163.stdout:477: expect (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:07:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:20 smithi181 bash[27712]: audit 2023-10-09T09:07:19.958406+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.163:0/3280438009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:20 smithi181 bash[27712]: cluster 2023-10-09T09:07:20.132412+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2023-10-09T09:07:20.626 INFO:tasks.rados.rados.0.smithi163.stdout:478: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:07:20.626 INFO:tasks.rados.rados.0.smithi163.stdout:478: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:07:20.627 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 105 (ObjNum 166 snap 43 seq_num 166) dirty exists 2023-10-09T09:07:20.627 INFO:tasks.rados.rados.0.smithi163.stdout:471: done (6 left) 2023-10-09T09:07:20.627 INFO:tasks.rados.rados.0.smithi163.stdout:472: done (5 left) 2023-10-09T09:07:20.627 INFO:tasks.rados.rados.0.smithi163.stdout:473: done (4 left) 2023-10-09T09:07:20.627 INFO:tasks.rados.rados.0.smithi163.stdout:474: done (3 left) 2023-10-09T09:07:20.627 INFO:tasks.rados.rados.0.smithi163.stdout:475: done (2 left) 2023-10-09T09:07:20.627 INFO:tasks.rados.rados.0.smithi163.stdout:477: done (1 left) 2023-10-09T09:07:20.627 INFO:tasks.rados.rados.0.smithi163.stdout:478: done (0 left) 2023-10-09T09:07:20.627 INFO:tasks.rados.rados.0.smithi163.stdout:479: rollback oid 33 current snap is 61 2023-10-09T09:07:20.627 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 51 2023-10-09T09:07:20.628 INFO:tasks.rados.rados.0.smithi163.stdout:480: rollback oid 29 current snap is 61 2023-10-09T09:07:20.628 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 29 to 43 2023-10-09T09:07:20.628 INFO:tasks.rados.rados.0.smithi163.stdout:481: snap_remove snap 50 2023-10-09T09:07:21.217 INFO:tasks.rados.rados.0.smithi163.stdout:481: done (2 left) 2023-10-09T09:07:21.218 INFO:tasks.rados.rados.0.smithi163.stdout:482: delete oid 40 current snap is 61 2023-10-09T09:07:21.218 INFO:tasks.rados.rados.0.smithi163.stdout:479: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:07:21.218 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 116 (ObjNum 1 snap 32603 seq_num 698818432) dirty dne 2023-10-09T09:07:21.218 INFO:tasks.rados.rados.0.smithi163.stdout:480: finishing rollback tid 1 to smithi16367185-29 2023-10-09T09:07:21.218 INFO:tasks.rados.rados.0.smithi163.stdout:480: finishing rollback tid 2 to smithi16367185-29 2023-10-09T09:07:21.218 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 195 (ObjNum 155 snap 38 seq_num 155) dirty exists 2023-10-09T09:07:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:479: done (2 left) 2023-10-09T09:07:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:480: done (1 left) 2023-10-09T09:07:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:482: done (0 left) 2023-10-09T09:07:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:483: read oid 37 snap -1 2023-10-09T09:07:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:483: expect (ObjNum 36 snap 0 seq_num 36) 2023-10-09T09:07:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:484: read oid 47 snap 58 2023-10-09T09:07:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:484: expect deleted 2023-10-09T09:07:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:485: snap_remove snap 49 2023-10-09T09:07:21.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:21 smithi163 bash[27500]: cluster 2023-10-09T09:07:20.140662+0000 mgr.x (mgr.24751) 36 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:07:21.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:21 smithi163 bash[27500]: audit 2023-10-09T09:07:20.407515+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:21.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:21 smithi163 bash[33912]: cluster 2023-10-09T09:07:20.140662+0000 mgr.x (mgr.24751) 36 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:07:21.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:21 smithi163 bash[33912]: audit 2023-10-09T09:07:20.407515+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:21 smithi181 bash[27712]: cluster 2023-10-09T09:07:20.140662+0000 mgr.x (mgr.24751) 36 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:07:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:21 smithi181 bash[27712]: audit 2023-10-09T09:07:20.407515+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:22.124 INFO:tasks.rados.rados.0.smithi163.stdout:485: done (2 left) 2023-10-09T09:07:22.124 INFO:tasks.rados.rados.0.smithi163.stdout:486: rollback oid 17 current snap is 61 2023-10-09T09:07:22.124 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 38 2023-10-09T09:07:22.124 INFO:tasks.rados.rados.0.smithi163.stdout:487: write oid 19 current snap is 61 2023-10-09T09:07:22.124 INFO:tasks.rados.rados.0.smithi163.stdout:487: seq_num 214 ranges {796698=561170,2006323=545406,3126991=469707} 2023-10-09T09:07:22.133 INFO:tasks.rados.rados.0.smithi163.stdout:487: writing smithi16367185-19 from 796698 to 1357868 tid 1 2023-10-09T09:07:22.141 INFO:tasks.rados.rados.0.smithi163.stdout:487: writing smithi16367185-19 from 2006323 to 2551729 tid 2 2023-10-09T09:07:22.148 INFO:tasks.rados.rados.0.smithi163.stdout:487: writing smithi16367185-19 from 3126991 to 3596698 tid 3 2023-10-09T09:07:22.148 INFO:tasks.rados.rados.0.smithi163.stdout:488: delete oid 39 current snap is 61 2023-10-09T09:07:22.148 INFO:tasks.rados.rados.0.smithi163.stdout:483: expect (ObjNum 36 snap 0 seq_num 36) 2023-10-09T09:07:22.235 INFO:tasks.rados.rados.0.smithi163.stdout:486: finishing rollback tid 0 to smithi16367185-17 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:486: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:486: finishing rollback tid 2 to smithi16367185-17 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 234 (ObjNum 137 snap 28 seq_num 137) dirty exists 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:487: finishing write tid 1 to smithi16367185-19 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:487: finishing write tid 2 to smithi16367185-19 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:487: finishing write tid 3 to smithi16367185-19 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:487: finishing write tid 4 to smithi16367185-19 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:487: finishing write tid 5 to smithi16367185-19 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 140 (ObjNum 214 snap 61 seq_num 214) dirty exists 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:487: left oid 19 (ObjNum 214 snap 61 seq_num 214) 2023-10-09T09:07:22.236 INFO:tasks.rados.rados.0.smithi163.stdout:483: done (4 left) 2023-10-09T09:07:22.237 INFO:tasks.rados.rados.0.smithi163.stdout:484: done (3 left) 2023-10-09T09:07:22.237 INFO:tasks.rados.rados.0.smithi163.stdout:486: done (2 left) 2023-10-09T09:07:22.237 INFO:tasks.rados.rados.0.smithi163.stdout:487: done (1 left) 2023-10-09T09:07:22.237 INFO:tasks.rados.rados.0.smithi163.stdout:488: done (0 left) 2023-10-09T09:07:22.237 INFO:tasks.rados.rados.0.smithi163.stdout:489: write oid 18 current snap is 61 2023-10-09T09:07:22.237 INFO:tasks.rados.rados.0.smithi163.stdout:489: seq_num 216 ranges {430312=711668,1723287=549075,2781837=703343} 2023-10-09T09:07:22.248 INFO:tasks.rados.rados.0.smithi163.stdout:489: writing smithi16367185-18 from 430312 to 1141980 tid 1 2023-10-09T09:07:22.255 INFO:tasks.rados.rados.0.smithi163.stdout:489: writing smithi16367185-18 from 1723287 to 2272362 tid 2 2023-10-09T09:07:22.262 INFO:tasks.rados.rados.0.smithi163.stdout:489: writing smithi16367185-18 from 2781837 to 3485180 tid 3 2023-10-09T09:07:22.262 INFO:tasks.rados.rados.0.smithi163.stdout:490: read oid 21 snap 36 2023-10-09T09:07:22.262 INFO:tasks.rados.rados.0.smithi163.stdout:490: expect (ObjNum 20 snap 0 seq_num 20) 2023-10-09T09:07:22.262 INFO:tasks.rados.rados.0.smithi163.stdout:491: write oid 29 current snap is 61 2023-10-09T09:07:22.262 INFO:tasks.rados.rados.0.smithi163.stdout:491: seq_num 217 ranges {447119=786917,1665416=692733,2976554=514615} 2023-10-09T09:07:22.269 INFO:tasks.rados.rados.0.smithi163.stdout:491: writing smithi16367185-29 from 447119 to 1234036 tid 1 2023-10-09T09:07:22.276 INFO:tasks.rados.rados.0.smithi163.stdout:491: writing smithi16367185-29 from 1665416 to 2358149 tid 2 2023-10-09T09:07:22.284 INFO:tasks.rados.rados.0.smithi163.stdout:491: writing smithi16367185-29 from 2976554 to 3491169 tid 3 2023-10-09T09:07:22.284 INFO:tasks.rados.rados.0.smithi163.stdout:492: read oid 28 snap -1 2023-10-09T09:07:22.284 INFO:tasks.rados.rados.0.smithi163.stdout:492: expect (ObjNum 145 snap 34 seq_num 145) 2023-10-09T09:07:22.284 INFO:tasks.rados.rados.0.smithi163.stdout:493: rollback oid 42 current snap is 61 2023-10-09T09:07:22.284 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 60 2023-10-09T09:07:22.284 INFO:tasks.rados.rados.0.smithi163.stdout:489: finishing write tid 1 to smithi16367185-18 2023-10-09T09:07:22.284 INFO:tasks.rados.rados.0.smithi163.stdout:489: finishing write tid 2 to smithi16367185-18 2023-10-09T09:07:22.284 INFO:tasks.rados.rados.0.smithi163.stdout:489: finishing write tid 3 to smithi16367185-18 2023-10-09T09:07:22.284 INFO:tasks.rados.rados.0.smithi163.stdout:489: finishing write tid 4 to smithi16367185-18 2023-10-09T09:07:22.284 INFO:tasks.rados.rados.0.smithi163.stdout:489: finishing write tid 5 to smithi16367185-18 2023-10-09T09:07:22.285 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 150 (ObjNum 216 snap 61 seq_num 216) dirty exists 2023-10-09T09:07:22.285 INFO:tasks.rados.rados.0.smithi163.stdout:489: left oid 18 (ObjNum 216 snap 61 seq_num 216) 2023-10-09T09:07:22.285 INFO:tasks.rados.rados.0.smithi163.stdout:490: expect (ObjNum 20 snap 0 seq_num 20) 2023-10-09T09:07:22.377 INFO:tasks.rados.rados.0.smithi163.stdout:491: finishing write tid 1 to smithi16367185-29 2023-10-09T09:07:22.377 INFO:tasks.rados.rados.0.smithi163.stdout:491: finishing write tid 2 to smithi16367185-29 2023-10-09T09:07:22.377 INFO:tasks.rados.rados.0.smithi163.stdout:493: finishing rollback tid 0 to smithi16367185-42 2023-10-09T09:07:22.378 INFO:tasks.rados.rados.0.smithi163.stdout:493: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:07:22.378 INFO:tasks.rados.rados.0.smithi163.stdout:493: finishing rollback tid 2 to smithi16367185-42 2023-10-09T09:07:22.378 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 364 (ObjNum 205 snap 57 seq_num 205) dirty exists 2023-10-09T09:07:22.378 INFO:tasks.rados.rados.0.smithi163.stdout:492: expect (ObjNum 145 snap 34 seq_num 145) 2023-10-09T09:07:22.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:22 smithi163 bash[27500]: cluster 2023-10-09T09:07:21.216217+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2023-10-09T09:07:22.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:22 smithi163 bash[27500]: cluster 2023-10-09T09:07:22.123191+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2023-10-09T09:07:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:22 smithi163 bash[33912]: cluster 2023-10-09T09:07:21.216217+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2023-10-09T09:07:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:22 smithi163 bash[33912]: cluster 2023-10-09T09:07:22.123191+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2023-10-09T09:07:22.468 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:22 smithi163 systemd[1]: Stopping Ceph alertmanager.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:07:22.469 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:22 smithi163 bash[70595]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-alertmanager.a 2023-10-09T09:07:22.469 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:22 smithi163 bash[57035]: level=info ts=2023-10-09T09:07:22.408Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:07:22.493 INFO:tasks.rados.rados.0.smithi163.stdout:491: finishing write tid 3 to smithi16367185-29 2023-10-09T09:07:22.493 INFO:tasks.rados.rados.0.smithi163.stdout:491: finishing write tid 4 to smithi16367185-29 2023-10-09T09:07:22.493 INFO:tasks.rados.rados.0.smithi163.stdout:491: finishing write tid 5 to smithi16367185-29 2023-10-09T09:07:22.493 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 207 (ObjNum 217 snap 61 seq_num 217) dirty exists 2023-10-09T09:07:22.493 INFO:tasks.rados.rados.0.smithi163.stdout:491: left oid 29 (ObjNum 217 snap 61 seq_num 217) 2023-10-09T09:07:22.493 INFO:tasks.rados.rados.0.smithi163.stdout:489: done (4 left) 2023-10-09T09:07:22.494 INFO:tasks.rados.rados.0.smithi163.stdout:490: done (3 left) 2023-10-09T09:07:22.494 INFO:tasks.rados.rados.0.smithi163.stdout:491: done (2 left) 2023-10-09T09:07:22.494 INFO:tasks.rados.rados.0.smithi163.stdout:492: done (1 left) 2023-10-09T09:07:22.494 INFO:tasks.rados.rados.0.smithi163.stdout:493: done (0 left) 2023-10-09T09:07:22.494 INFO:tasks.rados.rados.0.smithi163.stdout:494: delete oid 42 current snap is 61 2023-10-09T09:07:22.496 INFO:tasks.rados.rados.0.smithi163.stdout:494: done (0 left) 2023-10-09T09:07:22.496 INFO:tasks.rados.rados.0.smithi163.stdout:495: write oid 5 current snap is 61 2023-10-09T09:07:22.496 INFO:tasks.rados.rados.0.smithi163.stdout:495: seq_num 219 ranges {480733=537415,1782380=696402,3031400=537159} 2023-10-09T09:07:22.506 INFO:tasks.rados.rados.0.smithi163.stdout:495: writing smithi16367185-5 from 480733 to 1018148 tid 1 2023-10-09T09:07:22.515 INFO:tasks.rados.rados.0.smithi163.stdout:495: writing smithi16367185-5 from 1782380 to 2478782 tid 2 2023-10-09T09:07:22.521 INFO:tasks.rados.rados.0.smithi163.stdout:495: writing smithi16367185-5 from 3031400 to 3568559 tid 3 2023-10-09T09:07:22.522 INFO:tasks.rados.rados.0.smithi163.stdout:496: rollback oid 41 current snap is 61 2023-10-09T09:07:22.522 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 41 to 51 2023-10-09T09:07:22.522 INFO:tasks.rados.rados.0.smithi163.stdout:495: finishing write tid 1 to smithi16367185-5 2023-10-09T09:07:22.522 INFO:tasks.rados.rados.0.smithi163.stdout:495: finishing write tid 2 to smithi16367185-5 2023-10-09T09:07:22.522 INFO:tasks.rados.rados.0.smithi163.stdout:497: write oid 9 current snap is 61 2023-10-09T09:07:22.522 INFO:tasks.rados.rados.0.smithi163.stdout:497: seq_num 220 ranges {497540=612664,1840862=440060,3058823=464784} 2023-10-09T09:07:22.528 INFO:tasks.rados.rados.0.smithi163.stdout:497: writing smithi16367185-9 from 497540 to 1110204 tid 1 2023-10-09T09:07:22.534 INFO:tasks.rados.rados.0.smithi163.stdout:497: writing smithi16367185-9 from 1840862 to 2280922 tid 2 2023-10-09T09:07:22.542 INFO:tasks.rados.rados.0.smithi163.stdout:497: writing smithi16367185-9 from 3058823 to 3523607 tid 3 2023-10-09T09:07:22.542 INFO:tasks.rados.rados.0.smithi163.stdout:498: read oid 43 snap -1 2023-10-09T09:07:22.542 INFO:tasks.rados.rados.0.smithi163.stdout:498: expect (ObjNum 199 snap 56 seq_num 199) 2023-10-09T09:07:22.542 INFO:tasks.rados.rados.0.smithi163.stdout:499: write oid 38 current snap is 61 2023-10-09T09:07:22.542 INFO:tasks.rados.rados.0.smithi163.stdout:495: finishing write tid 3 to smithi16367185-5 2023-10-09T09:07:22.542 INFO:tasks.rados.rados.0.smithi163.stdout:499: seq_num 221 ranges {514347=404266,1499344=700071,2686246=676056} 2023-10-09T09:07:22.549 INFO:tasks.rados.rados.0.smithi163.stdout:499: writing smithi16367185-38 from 514347 to 918613 tid 1 2023-10-09T09:07:22.559 INFO:tasks.rados.rados.0.smithi163.stdout:499: writing smithi16367185-38 from 1499344 to 2199415 tid 2 2023-10-09T09:07:22.567 INFO:tasks.rados.rados.0.smithi163.stdout:499: writing smithi16367185-38 from 2686246 to 3362302 tid 3 2023-10-09T09:07:22.567 INFO:tasks.rados.rados.0.smithi163.stdout:500: snap_remove snap 54 2023-10-09T09:07:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:22 smithi181 bash[27712]: cluster 2023-10-09T09:07:21.216217+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2023-10-09T09:07:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:22 smithi181 bash[27712]: cluster 2023-10-09T09:07:22.123191+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2023-10-09T09:07:22.568 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:22 smithi181 bash[48295]: ts=2023-10-09T09:07:22.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.163:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:07:23.139 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:22 smithi163 bash[70630]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T09:07:23.139 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:23 smithi163 bash[70695]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T09:07:23.139 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:23 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T09:07:23.139 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:23 smithi163 systemd[1]: Stopped Ceph alertmanager.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:07:23.139 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:23 smithi163 systemd[1]: Started Ceph alertmanager.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:07:23.222 INFO:tasks.rados.rados.0.smithi163.stdout:500: done (5 left) 2023-10-09T09:07:23.222 INFO:tasks.rados.rados.0.smithi163.stdout:501: delete oid 2 current snap is 61 2023-10-09T09:07:23.222 INFO:tasks.rados.rados.0.smithi163.stdout:495: finishing write tid 4 to smithi16367185-5 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:495: finishing write tid 5 to smithi16367185-5 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 237 (ObjNum 219 snap 61 seq_num 219) dirty exists 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:495: left oid 5 (ObjNum 219 snap 61 seq_num 219) 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:496: finishing rollback tid 0 to smithi16367185-41 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:496: finishing rollback tid 1 to smithi16367185-41 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:496: finishing rollback tid 2 to smithi16367185-41 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 7 (ObjNum 40 snap 0 seq_num 40) dirty exists 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:497: finishing write tid 1 to smithi16367185-9 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:497: finishing write tid 2 to smithi16367185-9 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:497: finishing write tid 3 to smithi16367185-9 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:497: finishing write tid 4 to smithi16367185-9 2023-10-09T09:07:23.223 INFO:tasks.rados.rados.0.smithi163.stdout:497: finishing write tid 5 to smithi16367185-9 2023-10-09T09:07:23.224 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 121 (ObjNum 220 snap 61 seq_num 220) dirty exists 2023-10-09T09:07:23.224 INFO:tasks.rados.rados.0.smithi163.stdout:497: left oid 9 (ObjNum 220 snap 61 seq_num 220) 2023-10-09T09:07:23.224 INFO:tasks.rados.rados.0.smithi163.stdout:498: expect (ObjNum 199 snap 56 seq_num 199) 2023-10-09T09:07:23.426 INFO:tasks.rados.rados.0.smithi163.stdout:499: finishing write tid 1 to smithi16367185-38 2023-10-09T09:07:23.426 INFO:tasks.rados.rados.0.smithi163.stdout:499: finishing write tid 2 to smithi16367185-38 2023-10-09T09:07:23.426 INFO:tasks.rados.rados.0.smithi163.stdout:495: done (5 left) 2023-10-09T09:07:23.426 INFO:tasks.rados.rados.0.smithi163.stdout:496: done (4 left) 2023-10-09T09:07:23.426 INFO:tasks.rados.rados.0.smithi163.stdout:497: done (3 left) 2023-10-09T09:07:23.427 INFO:tasks.rados.rados.0.smithi163.stdout:498: done (2 left) 2023-10-09T09:07:23.427 INFO:tasks.rados.rados.0.smithi163.stdout:501: done (1 left) 2023-10-09T09:07:23.427 INFO:tasks.rados.rados.0.smithi163.stdout:502: snap_create 2023-10-09T09:07:23.427 INFO:tasks.rados.rados.0.smithi163.stdout:499: finishing write tid 3 to smithi16367185-38 2023-10-09T09:07:23.427 INFO:tasks.rados.rados.0.smithi163.stdout:499: finishing write tid 4 to smithi16367185-38 2023-10-09T09:07:23.427 INFO:tasks.rados.rados.0.smithi163.stdout:499: finishing write tid 5 to smithi16367185-38 2023-10-09T09:07:23.427 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 219 (ObjNum 221 snap 61 seq_num 221) dirty exists 2023-10-09T09:07:23.427 INFO:tasks.rados.rados.0.smithi163.stdout:499: left oid 38 (ObjNum 221 snap 61 seq_num 221) 2023-10-09T09:07:23.452 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:23 smithi163 bash[33912]: cluster 2023-10-09T09:07:22.141210+0000 mgr.x (mgr.24751) 37 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 435 KiB/s wr, 14 op/s 2023-10-09T09:07:23.452 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:23 smithi163 bash[33912]: audit 2023-10-09T09:07:22.435290+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.163:0/4115247051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:23.452 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:23 smithi163 bash[33912]: audit 2023-10-09T09:07:23.116275+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:23.452 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:23 smithi163 bash[33912]: audit 2023-10-09T09:07:23.127944+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:23.452 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:23 smithi163 bash[27500]: cluster 2023-10-09T09:07:22.141210+0000 mgr.x (mgr.24751) 37 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 435 KiB/s wr, 14 op/s 2023-10-09T09:07:23.453 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:23 smithi163 bash[27500]: audit 2023-10-09T09:07:22.435290+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.163:0/4115247051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:23.453 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:23 smithi163 bash[27500]: audit 2023-10-09T09:07:23.116275+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:23.453 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:23 smithi163 bash[27500]: audit 2023-10-09T09:07:23.127944+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:23 smithi181 bash[27712]: cluster 2023-10-09T09:07:22.141210+0000 mgr.x (mgr.24751) 37 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 435 KiB/s wr, 14 op/s 2023-10-09T09:07:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:23 smithi181 bash[27712]: audit 2023-10-09T09:07:22.435290+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.163:0/4115247051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:23 smithi181 bash[27712]: audit 2023-10-09T09:07:23.116275+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:23.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:23 smithi181 bash[27712]: audit 2023-10-09T09:07:23.127944+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:24.241 INFO:tasks.rados.rados.0.smithi163.stdout:499: done (1 left) 2023-10-09T09:07:24.241 INFO:tasks.rados.rados.0.smithi163.stdout:502: done (0 left) 2023-10-09T09:07:24.241 INFO:tasks.rados.rados.0.smithi163.stdout:503: write oid 32 current snap is 62 2023-10-09T09:07:24.241 INFO:tasks.rados.rados.0.smithi163.stdout:503: seq_num 223 ranges {547961=554764,1616308=703740,2741092=698600} 2023-10-09T09:07:24.246 INFO:tasks.rados.rados.0.smithi163.stdout:503: writing smithi16367185-32 from 547961 to 1102725 tid 1 2023-10-09T09:07:24.257 INFO:tasks.rados.rados.0.smithi163.stdout:503: writing smithi16367185-32 from 1616308 to 2320048 tid 2 2023-10-09T09:07:24.264 INFO:tasks.rados.rados.0.smithi163.stdout:503: writing smithi16367185-32 from 2741092 to 3439692 tid 3 2023-10-09T09:07:24.265 INFO:tasks.rados.rados.0.smithi163.stdout:504: read oid 47 snap -1 2023-10-09T09:07:24.265 INFO:tasks.rados.rados.0.smithi163.stdout:504: expect deleted 2023-10-09T09:07:24.265 INFO:tasks.rados.rados.0.smithi163.stdout:505: write oid 21 current snap is 62 2023-10-09T09:07:24.265 INFO:tasks.rados.rados.0.smithi163.stdout:505: seq_num 224 ranges {564768=630013,1674790=447398,2768515=626225} 2023-10-09T09:07:24.272 INFO:tasks.rados.rados.0.smithi163.stdout:505: writing smithi16367185-21 from 564768 to 1194781 tid 1 2023-10-09T09:07:24.278 INFO:tasks.rados.rados.0.smithi163.stdout:505: writing smithi16367185-21 from 1674790 to 2122188 tid 2 2023-10-09T09:07:24.286 INFO:tasks.rados.rados.0.smithi163.stdout:505: writing smithi16367185-21 from 2768515 to 3394740 tid 3 2023-10-09T09:07:24.286 INFO:tasks.rados.rados.0.smithi163.stdout:503: finishing write tid 1 to smithi16367185-32 2023-10-09T09:07:24.286 INFO:tasks.rados.rados.0.smithi163.stdout:506: rollback oid 11 current snap is 62 2023-10-09T09:07:24.286 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 11 to 38 2023-10-09T09:07:24.286 INFO:tasks.rados.rados.0.smithi163.stdout:503: finishing write tid 2 to smithi16367185-32 2023-10-09T09:07:24.286 INFO:tasks.rados.rados.0.smithi163.stdout:503: finishing write tid 3 to smithi16367185-32 2023-10-09T09:07:24.287 INFO:tasks.rados.rados.0.smithi163.stdout:503: finishing write tid 4 to smithi16367185-32 2023-10-09T09:07:24.287 INFO:tasks.rados.rados.0.smithi163.stdout:503: finishing write tid 5 to smithi16367185-32 2023-10-09T09:07:24.287 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 376 (ObjNum 223 snap 62 seq_num 223) dirty exists 2023-10-09T09:07:24.287 INFO:tasks.rados.rados.0.smithi163.stdout:503: left oid 32 (ObjNum 223 snap 62 seq_num 223) 2023-10-09T09:07:24.287 INFO:tasks.rados.rados.0.smithi163.stdout:505: finishing write tid 1 to smithi16367185-21 2023-10-09T09:07:24.287 INFO:tasks.rados.rados.0.smithi163.stdout:505: finishing write tid 2 to smithi16367185-21 2023-10-09T09:07:24.287 INFO:tasks.rados.rados.0.smithi163.stdout:503: done (3 left) 2023-10-09T09:07:24.287 INFO:tasks.rados.rados.0.smithi163.stdout:504: done (2 left) 2023-10-09T09:07:24.287 INFO:tasks.rados.rados.0.smithi163.stdout:507: snap_create 2023-10-09T09:07:24.289 INFO:tasks.rados.rados.0.smithi163.stdout:505: finishing write tid 3 to smithi16367185-21 2023-10-09T09:07:24.290 INFO:tasks.rados.rados.0.smithi163.stdout:505: finishing write tid 4 to smithi16367185-21 2023-10-09T09:07:24.290 INFO:tasks.rados.rados.0.smithi163.stdout:505: finishing write tid 5 to smithi16367185-21 2023-10-09T09:07:24.290 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 120 (ObjNum 224 snap 62 seq_num 224) dirty exists 2023-10-09T09:07:24.290 INFO:tasks.rados.rados.0.smithi163.stdout:505: left oid 21 (ObjNum 224 snap 62 seq_num 224) 2023-10-09T09:07:24.293 INFO:tasks.rados.rados.0.smithi163.stdout:506: finishing rollback tid 0 to smithi16367185-11 2023-10-09T09:07:24.293 INFO:tasks.rados.rados.0.smithi163.stdout:506: finishing rollback tid 1 to smithi16367185-11 2023-10-09T09:07:24.293 INFO:tasks.rados.rados.0.smithi163.stdout:506: finishing rollback tid 2 to smithi16367185-11 2023-10-09T09:07:24.294 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 91 (ObjNum 148 snap 34 seq_num 148) dirty exists 2023-10-09T09:07:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[27500]: cephadm 2023-10-09T09:07:23.167433+0000 mgr.x (mgr.24751) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:07:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[27500]: cluster 2023-10-09T09:07:23.220471+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2023-10-09T09:07:24.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[27500]: cephadm 2023-10-09T09:07:23.411702+0000 mgr.x (mgr.24751) 39 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:07:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[33912]: cephadm 2023-10-09T09:07:23.167433+0000 mgr.x (mgr.24751) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:07:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[33912]: cluster 2023-10-09T09:07:23.220471+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2023-10-09T09:07:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[33912]: cephadm 2023-10-09T09:07:23.411702+0000 mgr.x (mgr.24751) 39 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:07:24.469 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[70722]: level=info ts=2023-10-09T09:07:24.219Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T09:07:24.469 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[70722]: level=info ts=2023-10-09T09:07:24.219Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T09:07:24.469 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[70722]: level=info ts=2023-10-09T09:07:24.223Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.163 port=9094 2023-10-09T09:07:24.469 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[70722]: level=info ts=2023-10-09T09:07:24.224Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T09:07:24.469 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[70722]: level=info ts=2023-10-09T09:07:24.273Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:07:24.469 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[70722]: level=info ts=2023-10-09T09:07:24.273Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:07:24.469 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[70722]: level=info ts=2023-10-09T09:07:24.276Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T09:07:24.469 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:24 smithi163 bash[70722]: level=info ts=2023-10-09T09:07:24.277Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T09:07:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:24 smithi181 bash[27712]: cephadm 2023-10-09T09:07:23.167433+0000 mgr.x (mgr.24751) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:07:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:24 smithi181 bash[27712]: cluster 2023-10-09T09:07:23.220471+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2023-10-09T09:07:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:24 smithi181 bash[27712]: cephadm 2023-10-09T09:07:23.411702+0000 mgr.x (mgr.24751) 39 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:07:25.243 INFO:tasks.rados.rados.0.smithi163.stdout:505: done (2 left) 2023-10-09T09:07:25.243 INFO:tasks.rados.rados.0.smithi163.stdout:506: done (1 left) 2023-10-09T09:07:25.243 INFO:tasks.rados.rados.0.smithi163.stdout:507: done (0 left) 2023-10-09T09:07:25.243 INFO:tasks.rados.rados.0.smithi163.stdout:508: delete oid 45 current snap is 63 2023-10-09T09:07:25.249 INFO:tasks.rados.rados.0.smithi163.stdout:508: done (0 left) 2023-10-09T09:07:25.249 INFO:tasks.rados.rados.0.smithi163.stdout:509: snap_create 2023-10-09T09:07:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:25 smithi181 bash[27712]: cluster 2023-10-09T09:07:24.141963+0000 mgr.x (mgr.24751) 40 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 433 KiB/s wr, 14 op/s 2023-10-09T09:07:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:25 smithi181 bash[27712]: cluster 2023-10-09T09:07:24.239715+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2023-10-09T09:07:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:25 smithi181 bash[27712]: audit 2023-10-09T09:07:24.875550+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.163:0/1758251446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:25 smithi163 bash[33912]: cluster 2023-10-09T09:07:24.141963+0000 mgr.x (mgr.24751) 40 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 433 KiB/s wr, 14 op/s 2023-10-09T09:07:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:25 smithi163 bash[33912]: cluster 2023-10-09T09:07:24.239715+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2023-10-09T09:07:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:25 smithi163 bash[33912]: audit 2023-10-09T09:07:24.875550+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.163:0/1758251446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:25 smithi163 bash[27500]: cluster 2023-10-09T09:07:24.141963+0000 mgr.x (mgr.24751) 40 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 229 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 433 KiB/s wr, 14 op/s 2023-10-09T09:07:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:25 smithi163 bash[27500]: cluster 2023-10-09T09:07:24.239715+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2023-10-09T09:07:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:25 smithi163 bash[27500]: audit 2023-10-09T09:07:24.875550+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.163:0/1758251446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:26.258 INFO:tasks.rados.rados.0.smithi163.stdout:509: done (0 left) 2023-10-09T09:07:26.263 INFO:tasks.rados.rados.0.smithi163.stdout:510: read oid 2 snap -1 2023-10-09T09:07:26.263 INFO:tasks.rados.rados.0.smithi163.stdout:510: expect deleted 2023-10-09T09:07:26.263 INFO:tasks.rados.rados.0.smithi163.stdout:511: rollback oid 25 current snap is 64 2023-10-09T09:07:26.263 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 25 to 36 2023-10-09T09:07:26.263 INFO:tasks.rados.rados.0.smithi163.stdout:512: write oid 50 current snap is 64 2023-10-09T09:07:26.263 INFO:tasks.rados.rados.0.smithi163.stdout:512: seq_num 226 ranges {598382=780511,2075401=451067,3107008=648769} 2023-10-09T09:07:26.267 INFO:tasks.rados.rados.0.smithi163.stdout:512: writing smithi16367185-50 from 598382 to 1378893 tid 1 2023-10-09T09:07:26.274 INFO:tasks.rados.rados.0.smithi163.stdout:512: writing smithi16367185-50 from 2075401 to 2526468 tid 2 2023-10-09T09:07:26.282 INFO:tasks.rados.rados.0.smithi163.stdout:512: writing smithi16367185-50 from 3107008 to 3755777 tid 3 2023-10-09T09:07:26.292 INFO:tasks.rados.rados.0.smithi163.stdout:513: read oid 34 snap -1 2023-10-09T09:07:26.292 INFO:tasks.rados.rados.0.smithi163.stdout:513: expect (ObjNum 112 snap 18 seq_num 112) 2023-10-09T09:07:26.292 INFO:tasks.rados.rados.0.smithi163.stdout:514: read oid 1 snap 60 2023-10-09T09:07:26.293 INFO:tasks.rados.rados.0.smithi163.stdout:514: expect (ObjNum 85 snap 10 seq_num 85) 2023-10-09T09:07:26.293 INFO:tasks.rados.rados.0.smithi163.stdout:515: read oid 39 snap -1 2023-10-09T09:07:26.293 INFO:tasks.rados.rados.0.smithi163.stdout:515: expect deleted 2023-10-09T09:07:26.293 INFO:tasks.rados.rados.0.smithi163.stdout:516: write oid 27 current snap is 64 2023-10-09T09:07:26.293 INFO:tasks.rados.rados.0.smithi163.stdout:516: seq_num 227 ranges {615189=455760,1733883=711078,3134431=460041} 2023-10-09T09:07:26.293 INFO:tasks.rados.rados.0.smithi163.stdout:516: writing smithi16367185-27 from 615189 to 1070949 tid 1 2023-10-09T09:07:26.295 INFO:tasks.rados.rados.0.smithi163.stdout:516: writing smithi16367185-27 from 1733883 to 2444961 tid 2 2023-10-09T09:07:26.302 INFO:tasks.rados.rados.0.smithi163.stdout:516: writing smithi16367185-27 from 3134431 to 3594472 tid 3 2023-10-09T09:07:26.302 INFO:tasks.rados.rados.0.smithi163.stdout:517: write oid 44 current snap is 64 2023-10-09T09:07:26.303 INFO:tasks.rados.rados.0.smithi163.stdout:511: finishing rollback tid 0 to smithi16367185-25 2023-10-09T09:07:26.303 INFO:tasks.rados.rados.0.smithi163.stdout:511: finishing rollback tid 1 to smithi16367185-25 2023-10-09T09:07:26.303 INFO:tasks.rados.rados.0.smithi163.stdout:511: finishing rollback tid 2 to smithi16367185-25 2023-10-09T09:07:26.303 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 213 (ObjNum 60 snap 0 seq_num 60) dirty exists 2023-10-09T09:07:26.303 INFO:tasks.rados.rados.0.smithi163.stdout:512: finishing write tid 1 to smithi16367185-50 2023-10-09T09:07:26.303 INFO:tasks.rados.rados.0.smithi163.stdout:517: seq_num 228 ranges {631996=531009,1792365=454736,2761854=671313} 2023-10-09T09:07:26.309 INFO:tasks.rados.rados.0.smithi163.stdout:517: writing smithi16367185-44 from 631996 to 1163005 tid 1 2023-10-09T09:07:26.315 INFO:tasks.rados.rados.0.smithi163.stdout:517: writing smithi16367185-44 from 1792365 to 2247101 tid 2 2023-10-09T09:07:26.322 INFO:tasks.rados.rados.0.smithi163.stdout:517: writing smithi16367185-44 from 2761854 to 3433167 tid 3 2023-10-09T09:07:26.322 INFO:tasks.rados.rados.0.smithi163.stdout:510: done (7 left) 2023-10-09T09:07:26.322 INFO:tasks.rados.rados.0.smithi163.stdout:511: done (6 left) 2023-10-09T09:07:26.322 INFO:tasks.rados.rados.0.smithi163.stdout:518: read oid 18 snap 43 2023-10-09T09:07:26.322 INFO:tasks.rados.rados.0.smithi163.stdout:518: expect (ObjNum 118 snap 20 seq_num 118) 2023-10-09T09:07:26.323 INFO:tasks.rados.rados.0.smithi163.stdout:519: rollback oid 3 current snap is 64 2023-10-09T09:07:26.323 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 60 2023-10-09T09:07:26.323 INFO:tasks.rados.rados.0.smithi163.stdout:512: finishing write tid 2 to smithi16367185-50 2023-10-09T09:07:26.323 INFO:tasks.rados.rados.0.smithi163.stdout:512: finishing write tid 3 to smithi16367185-50 2023-10-09T09:07:26.323 INFO:tasks.rados.rados.0.smithi163.stdout:512: finishing write tid 4 to smithi16367185-50 2023-10-09T09:07:26.323 INFO:tasks.rados.rados.0.smithi163.stdout:520: snap_create 2023-10-09T09:07:26.323 INFO:tasks.rados.rados.0.smithi163.stdout:514: expect (ObjNum 85 snap 10 seq_num 85) 2023-10-09T09:07:26.492 INFO:tasks.rados.rados.0.smithi163.stdout:512: finishing write tid 5 to smithi16367185-50 2023-10-09T09:07:26.492 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 134 (ObjNum 226 snap 64 seq_num 226) dirty exists 2023-10-09T09:07:26.493 INFO:tasks.rados.rados.0.smithi163.stdout:512: left oid 50 (ObjNum 226 snap 64 seq_num 226) 2023-10-09T09:07:26.493 INFO:tasks.rados.rados.0.smithi163.stdout:513: expect (ObjNum 112 snap 18 seq_num 112) 2023-10-09T09:07:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:26 smithi181 bash[27712]: cluster 2023-10-09T09:07:25.243114+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2023-10-09T09:07:26.718 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:26 smithi163 bash[70722]: level=info ts=2023-10-09T09:07:26.225Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000815496s 2023-10-09T09:07:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:26 smithi163 bash[27500]: cluster 2023-10-09T09:07:25.243114+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2023-10-09T09:07:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:26 smithi163 bash[33912]: cluster 2023-10-09T09:07:25.243114+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2023-10-09T09:07:26.722 INFO:tasks.rados.rados.0.smithi163.stdout:516: finishing write tid 1 to smithi16367185-27 2023-10-09T09:07:26.722 INFO:tasks.rados.rados.0.smithi163.stdout:516: finishing write tid 2 to smithi16367185-27 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:516: finishing write tid 3 to smithi16367185-27 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:516: finishing write tid 4 to smithi16367185-27 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:516: finishing write tid 5 to smithi16367185-27 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 149 (ObjNum 227 snap 64 seq_num 227) dirty exists 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:516: left oid 27 (ObjNum 227 snap 64 seq_num 227) 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:517: finishing write tid 1 to smithi16367185-44 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:517: finishing write tid 2 to smithi16367185-44 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:519: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 100 (ObjNum 669362160 snap 21875 seq_num 1) dirty dne 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:517: finishing write tid 3 to smithi16367185-44 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:517: finishing write tid 4 to smithi16367185-44 2023-10-09T09:07:26.723 INFO:tasks.rados.rados.0.smithi163.stdout:517: finishing write tid 5 to smithi16367185-44 2023-10-09T09:07:26.724 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 382 (ObjNum 228 snap 64 seq_num 228) dirty exists 2023-10-09T09:07:26.724 INFO:tasks.rados.rados.0.smithi163.stdout:517: left oid 44 (ObjNum 228 snap 64 seq_num 228) 2023-10-09T09:07:26.724 INFO:tasks.rados.rados.0.smithi163.stdout:518: expect (ObjNum 118 snap 20 seq_num 118) 2023-10-09T09:07:27.123 INFO:tasks.rados.rados.0.smithi163.stdout:512: done (8 left) 2023-10-09T09:07:27.123 INFO:tasks.rados.rados.0.smithi163.stdout:513: done (7 left) 2023-10-09T09:07:27.123 INFO:tasks.rados.rados.0.smithi163.stdout:514: done (6 left) 2023-10-09T09:07:27.123 INFO:tasks.rados.rados.0.smithi163.stdout:515: done (5 left) 2023-10-09T09:07:27.124 INFO:tasks.rados.rados.0.smithi163.stdout:516: done (4 left) 2023-10-09T09:07:27.124 INFO:tasks.rados.rados.0.smithi163.stdout:517: done (3 left) 2023-10-09T09:07:27.124 INFO:tasks.rados.rados.0.smithi163.stdout:518: done (2 left) 2023-10-09T09:07:27.124 INFO:tasks.rados.rados.0.smithi163.stdout:519: done (1 left) 2023-10-09T09:07:27.124 INFO:tasks.rados.rados.0.smithi163.stdout:520: done (0 left) 2023-10-09T09:07:27.124 INFO:tasks.rados.rados.0.smithi163.stdout:521: delete oid 22 current snap is 65 2023-10-09T09:07:27.129 INFO:tasks.rados.rados.0.smithi163.stdout:521: done (0 left) 2023-10-09T09:07:27.129 INFO:tasks.rados.rados.0.smithi163.stdout:522: snap_create 2023-10-09T09:07:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:27 smithi181 bash[27712]: cluster 2023-10-09T09:07:26.142635+0000 mgr.x (mgr.24751) 41 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 254 B/s rd, 475 KiB/s wr, 2 op/s 2023-10-09T09:07:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:27 smithi181 bash[27712]: cluster 2023-10-09T09:07:26.257617+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2023-10-09T09:07:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:27 smithi181 bash[27712]: cluster 2023-10-09T09:07:27.123447+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2023-10-09T09:07:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:27 smithi163 bash[27500]: cluster 2023-10-09T09:07:26.142635+0000 mgr.x (mgr.24751) 41 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 254 B/s rd, 475 KiB/s wr, 2 op/s 2023-10-09T09:07:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:27 smithi163 bash[27500]: cluster 2023-10-09T09:07:26.257617+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2023-10-09T09:07:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:27 smithi163 bash[27500]: cluster 2023-10-09T09:07:27.123447+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2023-10-09T09:07:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:27 smithi163 bash[33912]: cluster 2023-10-09T09:07:26.142635+0000 mgr.x (mgr.24751) 41 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 231 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 254 B/s rd, 475 KiB/s wr, 2 op/s 2023-10-09T09:07:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:27 smithi163 bash[33912]: cluster 2023-10-09T09:07:26.257617+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2023-10-09T09:07:27.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:27 smithi163 bash[33912]: cluster 2023-10-09T09:07:27.123447+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2023-10-09T09:07:28.134 INFO:tasks.rados.rados.0.smithi163.stdout:522: done (0 left) 2023-10-09T09:07:28.134 INFO:tasks.rados.rados.0.smithi163.stdout:523: snap_remove snap 55 2023-10-09T09:07:28.553 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:28 smithi181 bash[27712]: audit 2023-10-09T09:07:27.317188+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.163:0/2909897896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:28.553 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:28 smithi181 bash[27712]: cluster 2023-10-09T09:07:28.134042+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2023-10-09T09:07:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:28 smithi163 bash[33912]: audit 2023-10-09T09:07:27.317188+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.163:0/2909897896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:28 smithi163 bash[33912]: cluster 2023-10-09T09:07:28.134042+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2023-10-09T09:07:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:28 smithi163 bash[27500]: audit 2023-10-09T09:07:27.317188+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.163:0/2909897896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:28 smithi163 bash[27500]: cluster 2023-10-09T09:07:28.134042+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2023-10-09T09:07:28.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:07:28 smithi181 bash[50540]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:07:28] "GET /metrics HTTP/1.1" 200 36077 "" "Prometheus/2.33.4" 2023-10-09T09:07:29.141 INFO:tasks.rados.rados.0.smithi163.stdout:523: done (0 left) 2023-10-09T09:07:29.142 INFO:tasks.rados.rados.0.smithi163.stdout:524: write oid 45 current snap is 66 2023-10-09T09:07:29.142 INFO:tasks.rados.rados.0.smithi163.stdout:524: seq_num 230 ranges {665610=797860,1909329=458405,2816700=410210} 2023-10-09T09:07:29.150 INFO:tasks.rados.rados.0.smithi163.stdout:524: writing smithi16367185-45 from 665610 to 1463470 tid 1 2023-10-09T09:07:29.156 INFO:tasks.rados.rados.0.smithi163.stdout:524: writing smithi16367185-45 from 1909329 to 2367734 tid 2 2023-10-09T09:07:29.161 INFO:tasks.rados.rados.0.smithi163.stdout:524: writing smithi16367185-45 from 2816700 to 3226910 tid 3 2023-10-09T09:07:29.161 INFO:tasks.rados.rados.0.smithi163.stdout:525: snap_remove snap 38 2023-10-09T09:07:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:29 smithi181 bash[27712]: cluster 2023-10-09T09:07:28.143290+0000 mgr.x (mgr.24751) 42 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 245 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.5 MiB/s wr, 23 op/s 2023-10-09T09:07:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:29 smithi181 bash[27712]: cluster 2023-10-09T09:07:29.141824+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2023-10-09T09:07:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:29 smithi163 bash[27500]: cluster 2023-10-09T09:07:28.143290+0000 mgr.x (mgr.24751) 42 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 245 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.5 MiB/s wr, 23 op/s 2023-10-09T09:07:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:29 smithi163 bash[27500]: cluster 2023-10-09T09:07:29.141824+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2023-10-09T09:07:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:29 smithi163 bash[33912]: cluster 2023-10-09T09:07:28.143290+0000 mgr.x (mgr.24751) 42 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 245 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.5 MiB/s wr, 23 op/s 2023-10-09T09:07:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:29 smithi163 bash[33912]: cluster 2023-10-09T09:07:29.141824+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2023-10-09T09:07:30.146 INFO:tasks.rados.rados.0.smithi163.stdout:525: done (1 left) 2023-10-09T09:07:30.146 INFO:tasks.rados.rados.0.smithi163.stdout:526: rollback oid 12 current snap is 66 2023-10-09T09:07:30.146 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 12 to 64 2023-10-09T09:07:30.146 INFO:tasks.rados.rados.0.smithi163.stdout:527: snap_create 2023-10-09T09:07:30.146 INFO:tasks.rados.rados.0.smithi163.stdout:524: finishing write tid 1 to smithi16367185-45 2023-10-09T09:07:30.146 INFO:tasks.rados.rados.0.smithi163.stdout:524: finishing write tid 2 to smithi16367185-45 2023-10-09T09:07:30.146 INFO:tasks.rados.rados.0.smithi163.stdout:524: finishing write tid 3 to smithi16367185-45 2023-10-09T09:07:30.147 INFO:tasks.rados.rados.0.smithi163.stdout:524: finishing write tid 4 to smithi16367185-45 2023-10-09T09:07:30.147 INFO:tasks.rados.rados.0.smithi163.stdout:524: finishing write tid 5 to smithi16367185-45 2023-10-09T09:07:30.147 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 395 (ObjNum 230 snap 66 seq_num 230) dirty exists 2023-10-09T09:07:30.147 INFO:tasks.rados.rados.0.smithi163.stdout:524: left oid 45 (ObjNum 230 snap 66 seq_num 230) 2023-10-09T09:07:30.155 INFO:tasks.rados.rados.0.smithi163.stdout:526: finishing rollback tid 0 to smithi16367185-12 2023-10-09T09:07:30.156 INFO:tasks.rados.rados.0.smithi163.stdout:526: finishing rollback tid 1 to smithi16367185-12 2023-10-09T09:07:30.156 INFO:tasks.rados.rados.0.smithi163.stdout:526: finishing rollback tid 2 to smithi16367185-12 2023-10-09T09:07:30.156 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 221 (ObjNum 167 snap 43 seq_num 167) dirty exists 2023-10-09T09:07:30.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:07:29 smithi163 bash[27764]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:07:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:07:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:30 smithi181 bash[27712]: audit 2023-10-09T09:07:29.759086+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.163:0/3783880959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:30 smithi181 bash[27712]: cluster 2023-10-09T09:07:30.144420+0000 mon.a (mon.0) 993 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2023-10-09T09:07:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:30 smithi163 bash[33912]: audit 2023-10-09T09:07:29.759086+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.163:0/3783880959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:30 smithi163 bash[33912]: cluster 2023-10-09T09:07:30.144420+0000 mon.a (mon.0) 993 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2023-10-09T09:07:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:30 smithi163 bash[27500]: audit 2023-10-09T09:07:29.759086+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.163:0/3783880959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:30 smithi163 bash[27500]: cluster 2023-10-09T09:07:30.144420+0000 mon.a (mon.0) 993 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2023-10-09T09:07:31.154 INFO:tasks.rados.rados.0.smithi163.stdout:524: done (2 left) 2023-10-09T09:07:31.155 INFO:tasks.rados.rados.0.smithi163.stdout:526: done (1 left) 2023-10-09T09:07:31.155 INFO:tasks.rados.rados.0.smithi163.stdout:527: done (0 left) 2023-10-09T09:07:31.155 INFO:tasks.rados.rados.0.smithi163.stdout:528: rollback oid 7 current snap is 67 2023-10-09T09:07:31.155 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 43 2023-10-09T09:07:31.155 INFO:tasks.rados.rados.0.smithi163.stdout:529: read oid 4 snap -1 2023-10-09T09:07:31.155 INFO:tasks.rados.rados.0.smithi163.stdout:529: expect (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:07:31.155 INFO:tasks.rados.rados.0.smithi163.stdout:530: write oid 34 current snap is 67 2023-10-09T09:07:31.155 INFO:tasks.rados.rados.0.smithi163.stdout:530: seq_num 231 ranges {682417=473109,1567811=602063,2844123=621482} 2023-10-09T09:07:31.163 INFO:tasks.rados.rados.0.smithi163.stdout:530: writing smithi16367185-34 from 682417 to 1155526 tid 1 2023-10-09T09:07:31.169 INFO:tasks.rados.rados.0.smithi163.stdout:530: writing smithi16367185-34 from 1567811 to 2169874 tid 2 2023-10-09T09:07:31.177 INFO:tasks.rados.rados.0.smithi163.stdout:530: writing smithi16367185-34 from 2844123 to 3465605 tid 3 2023-10-09T09:07:31.177 INFO:tasks.rados.rados.0.smithi163.stdout:531: snap_create 2023-10-09T09:07:31.177 INFO:tasks.rados.rados.0.smithi163.stdout:528: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:07:31.177 INFO:tasks.rados.rados.0.smithi163.stdout:528: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:07:31.177 INFO:tasks.rados.rados.0.smithi163.stdout:528: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:07:31.178 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 132 (ObjNum 166 snap 43 seq_num 166) dirty exists 2023-10-09T09:07:31.178 INFO:tasks.rados.rados.0.smithi163.stdout:529: expect (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:07:31.309 INFO:tasks.rados.rados.0.smithi163.stdout:530: finishing write tid 1 to smithi16367185-34 2023-10-09T09:07:31.309 INFO:tasks.rados.rados.0.smithi163.stdout:530: finishing write tid 2 to smithi16367185-34 2023-10-09T09:07:31.309 INFO:tasks.rados.rados.0.smithi163.stdout:530: finishing write tid 3 to smithi16367185-34 2023-10-09T09:07:31.309 INFO:tasks.rados.rados.0.smithi163.stdout:530: finishing write tid 4 to smithi16367185-34 2023-10-09T09:07:31.309 INFO:tasks.rados.rados.0.smithi163.stdout:530: finishing write tid 5 to smithi16367185-34 2023-10-09T09:07:31.310 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 147 (ObjNum 231 snap 67 seq_num 231) dirty exists 2023-10-09T09:07:31.310 INFO:tasks.rados.rados.0.smithi163.stdout:530: left oid 34 (ObjNum 231 snap 67 seq_num 231) 2023-10-09T09:07:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:31 smithi181 bash[27712]: cluster 2023-10-09T09:07:30.145603+0000 mgr.x (mgr.24751) 43 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 245 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:07:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:31 smithi181 bash[27712]: cluster 2023-10-09T09:07:31.151986+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2023-10-09T09:07:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:31 smithi163 bash[33912]: cluster 2023-10-09T09:07:30.145603+0000 mgr.x (mgr.24751) 43 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 245 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:07:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:31 smithi163 bash[33912]: cluster 2023-10-09T09:07:31.151986+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2023-10-09T09:07:31.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:31 smithi163 bash[27500]: cluster 2023-10-09T09:07:30.145603+0000 mgr.x (mgr.24751) 43 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 245 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:07:31.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:31 smithi163 bash[27500]: cluster 2023-10-09T09:07:31.151986+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2023-10-09T09:07:32.154 INFO:tasks.rados.rados.0.smithi163.stdout:528: done (3 left) 2023-10-09T09:07:32.155 INFO:tasks.rados.rados.0.smithi163.stdout:529: done (2 left) 2023-10-09T09:07:32.155 INFO:tasks.rados.rados.0.smithi163.stdout:530: done (1 left) 2023-10-09T09:07:32.155 INFO:tasks.rados.rados.0.smithi163.stdout:531: done (0 left) 2023-10-09T09:07:32.155 INFO:tasks.rados.rados.0.smithi163.stdout:532: delete oid 49 current snap is 68 2023-10-09T09:07:32.158 INFO:tasks.rados.rados.0.smithi163.stdout:532: done (0 left) 2023-10-09T09:07:32.158 INFO:tasks.rados.rados.0.smithi163.stdout:533: snap_remove snap 61 2023-10-09T09:07:32.594 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:32 smithi181 bash[48295]: ts=2023-10-09T09:07:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.163:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:07:33.160 INFO:tasks.rados.rados.0.smithi163.stdout:533: done (0 left) 2023-10-09T09:07:33.160 INFO:tasks.rados.rados.0.smithi163.stdout:534: write oid 45 current snap is 68 2023-10-09T09:07:33.160 INFO:tasks.rados.rados.0.smithi163.stdout:534: seq_num 233 ranges {716031=623607,2084775=605732,3298969=617062} 2023-10-09T09:07:33.169 INFO:tasks.rados.rados.0.smithi163.stdout:534: writing smithi16367185-45 from 716031 to 1339638 tid 1 2023-10-09T09:07:33.177 INFO:tasks.rados.rados.0.smithi163.stdout:534: writing smithi16367185-45 from 2084775 to 2690507 tid 2 2023-10-09T09:07:33.185 INFO:tasks.rados.rados.0.smithi163.stdout:534: writing smithi16367185-45 from 3298969 to 3916031 tid 3 2023-10-09T09:07:33.185 INFO:tasks.rados.rados.0.smithi163.stdout:535: delete oid 10 current snap is 68 2023-10-09T09:07:33.185 INFO:tasks.rados.rados.0.smithi163.stdout:534: finishing write tid 1 to smithi16367185-45 2023-10-09T09:07:33.185 INFO:tasks.rados.rados.0.smithi163.stdout:534: finishing write tid 2 to smithi16367185-45 2023-10-09T09:07:33.192 INFO:tasks.rados.rados.0.smithi163.stdout:534: finishing write tid 3 to smithi16367185-45 2023-10-09T09:07:33.192 INFO:tasks.rados.rados.0.smithi163.stdout:534: finishing write tid 4 to smithi16367185-45 2023-10-09T09:07:33.192 INFO:tasks.rados.rados.0.smithi163.stdout:534: finishing write tid 5 to smithi16367185-45 2023-10-09T09:07:33.192 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 414 (ObjNum 233 snap 68 seq_num 233) dirty exists 2023-10-09T09:07:33.192 INFO:tasks.rados.rados.0.smithi163.stdout:534: left oid 45 (ObjNum 233 snap 68 seq_num 233) 2023-10-09T09:07:33.193 INFO:tasks.rados.rados.0.smithi163.stdout:534: done (1 left) 2023-10-09T09:07:33.193 INFO:tasks.rados.rados.0.smithi163.stdout:535: done (0 left) 2023-10-09T09:07:33.193 INFO:tasks.rados.rados.0.smithi163.stdout:536: write oid 28 current snap is 68 2023-10-09T09:07:33.193 INFO:tasks.rados.rados.0.smithi163.stdout:536: seq_num 235 ranges {749645=774105,2085386=609401,3237462=712183} 2023-10-09T09:07:33.202 INFO:tasks.rados.rados.0.smithi163.stdout:536: writing smithi16367185-28 from 749645 to 1523750 tid 1 2023-10-09T09:07:33.209 INFO:tasks.rados.rados.0.smithi163.stdout:536: writing smithi16367185-28 from 2085386 to 2694787 tid 2 2023-10-09T09:07:33.217 INFO:tasks.rados.rados.0.smithi163.stdout:536: writing smithi16367185-28 from 3237462 to 3949645 tid 3 2023-10-09T09:07:33.217 INFO:tasks.rados.rados.0.smithi163.stdout:537: snap_create 2023-10-09T09:07:33.217 INFO:tasks.rados.rados.0.smithi163.stdout:536: finishing write tid 1 to smithi16367185-28 2023-10-09T09:07:33.217 INFO:tasks.rados.rados.0.smithi163.stdout:536: finishing write tid 2 to smithi16367185-28 2023-10-09T09:07:33.220 INFO:tasks.rados.rados.0.smithi163.stdout:536: finishing write tid 3 to smithi16367185-28 2023-10-09T09:07:33.220 INFO:tasks.rados.rados.0.smithi163.stdout:536: finishing write tid 4 to smithi16367185-28 2023-10-09T09:07:33.220 INFO:tasks.rados.rados.0.smithi163.stdout:536: finishing write tid 5 to smithi16367185-28 2023-10-09T09:07:33.220 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 256 (ObjNum 235 snap 68 seq_num 235) dirty exists 2023-10-09T09:07:33.220 INFO:tasks.rados.rados.0.smithi163.stdout:536: left oid 28 (ObjNum 235 snap 68 seq_num 235) 2023-10-09T09:07:33.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:33 smithi163 bash[33912]: cluster 2023-10-09T09:07:32.146884+0000 mgr.x (mgr.24751) 44 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 258 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2023-10-09T09:07:33.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:33 smithi163 bash[33912]: cluster 2023-10-09T09:07:32.152876+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2023-10-09T09:07:33.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:33 smithi163 bash[33912]: audit 2023-10-09T09:07:32.215251+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.163:0/889641219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:33 smithi163 bash[27500]: cluster 2023-10-09T09:07:32.146884+0000 mgr.x (mgr.24751) 44 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 258 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2023-10-09T09:07:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:33 smithi163 bash[27500]: cluster 2023-10-09T09:07:32.152876+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2023-10-09T09:07:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:33 smithi163 bash[27500]: audit 2023-10-09T09:07:32.215251+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.163:0/889641219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:33 smithi181 bash[27712]: cluster 2023-10-09T09:07:32.146884+0000 mgr.x (mgr.24751) 44 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 258 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2023-10-09T09:07:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:33 smithi181 bash[27712]: cluster 2023-10-09T09:07:32.152876+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2023-10-09T09:07:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:33 smithi181 bash[27712]: audit 2023-10-09T09:07:32.215251+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.163:0/889641219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:34.174 INFO:tasks.rados.rados.0.smithi163.stdout:536: done (1 left) 2023-10-09T09:07:34.174 INFO:tasks.rados.rados.0.smithi163.stdout:537: done (0 left) 2023-10-09T09:07:34.174 INFO:tasks.rados.rados.0.smithi163.stdout:538: read oid 42 snap -1 2023-10-09T09:07:34.174 INFO:tasks.rados.rados.0.smithi163.stdout:538: expect deleted 2023-10-09T09:07:34.174 INFO:tasks.rados.rados.0.smithi163.stdout:539: snap_create 2023-10-09T09:07:34.425 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:34 smithi181 bash[27712]: cluster 2023-10-09T09:07:33.159900+0000 mon.a (mon.0) 996 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2023-10-09T09:07:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:34 smithi163 bash[27500]: cluster 2023-10-09T09:07:33.159900+0000 mon.a (mon.0) 996 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2023-10-09T09:07:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:34 smithi163 bash[33912]: cluster 2023-10-09T09:07:33.159900+0000 mon.a (mon.0) 996 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2023-10-09T09:07:34.468 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 09:07:34 smithi163 bash[70722]: level=info ts=2023-10-09T09:07:34.228Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004296922s 2023-10-09T09:07:35.180 INFO:tasks.rados.rados.0.smithi163.stdout:538: done (1 left) 2023-10-09T09:07:35.180 INFO:tasks.rados.rados.0.smithi163.stdout:539: done (0 left) 2023-10-09T09:07:35.180 INFO:tasks.rados.rados.0.smithi163.stdout:540: delete oid 14 current snap is 70 2023-10-09T09:07:35.183 INFO:tasks.rados.rados.0.smithi163.stdout:540: done (0 left) 2023-10-09T09:07:35.183 INFO:tasks.rados.rados.0.smithi163.stdout:541: write oid 24 current snap is 70 2023-10-09T09:07:35.183 INFO:tasks.rados.rados.0.smithi163.stdout:541: seq_num 237 ranges {783259=640956,1918703=613070,3008661=405467,3821939=161320} 2023-10-09T09:07:35.192 INFO:tasks.rados.rados.0.smithi163.stdout:541: writing smithi16367185-24 from 783259 to 1424215 tid 1 2023-10-09T09:07:35.199 INFO:tasks.rados.rados.0.smithi163.stdout:541: writing smithi16367185-24 from 1918703 to 2531773 tid 2 2023-10-09T09:07:35.204 INFO:tasks.rados.rados.0.smithi163.stdout:541: writing smithi16367185-24 from 3008661 to 3414128 tid 3 2023-10-09T09:07:35.208 INFO:tasks.rados.rados.0.smithi163.stdout:541: writing smithi16367185-24 from 3821939 to 3983259 tid 4 2023-10-09T09:07:35.208 INFO:tasks.rados.rados.0.smithi163.stdout:542: read oid 16 snap 36 2023-10-09T09:07:35.208 INFO:tasks.rados.rados.0.smithi163.stdout:542: expect (ObjNum 117 snap 20 seq_num 117) 2023-10-09T09:07:35.208 INFO:tasks.rados.rados.0.smithi163.stdout:543: snap_remove snap 43 2023-10-09T09:07:35.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[27712]: cluster 2023-10-09T09:07:34.147621+0000 mgr.x (mgr.24751) 45 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 258 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2023-10-09T09:07:35.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[27712]: cluster 2023-10-09T09:07:34.174368+0000 mon.a (mon.0) 997 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2023-10-09T09:07:35.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[27712]: audit 2023-10-09T09:07:34.661480+0000 mon.a (mon.0) 998 : audit [DBG] from='client.? 172.21.15.163:0/2341976780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:35.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 systemd[1]: Stopping Ceph prometheus.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:07:35.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[52822]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T09:07:35.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.214Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:07:35.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.214Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T09:07:35.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.214Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T09:07:35.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.214Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T09:07:35.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.214Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T09:07:35.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.214Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T09:07:35.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.214Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T09:07:35.319 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.214Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T09:07:35.319 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.214Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T09:07:35.319 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.215Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T09:07:35.319 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.215Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T09:07:35.319 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[48295]: ts=2023-10-09T09:07:35.215Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T09:07:35.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:35 smithi163 bash[33912]: cluster 2023-10-09T09:07:34.147621+0000 mgr.x (mgr.24751) 45 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 258 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2023-10-09T09:07:35.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:35 smithi163 bash[33912]: cluster 2023-10-09T09:07:34.174368+0000 mon.a (mon.0) 997 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2023-10-09T09:07:35.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:35 smithi163 bash[33912]: audit 2023-10-09T09:07:34.661480+0000 mon.a (mon.0) 998 : audit [DBG] from='client.? 172.21.15.163:0/2341976780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:35 smithi163 bash[27500]: cluster 2023-10-09T09:07:34.147621+0000 mgr.x (mgr.24751) 45 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 258 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2023-10-09T09:07:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:35 smithi163 bash[27500]: cluster 2023-10-09T09:07:34.174368+0000 mon.a (mon.0) 997 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2023-10-09T09:07:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:35 smithi163 bash[27500]: audit 2023-10-09T09:07:34.661480+0000 mon.a (mon.0) 998 : audit [DBG] from='client.? 172.21.15.163:0/2341976780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:35.980 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[52857]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T09:07:35.980 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 bash[52920]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T09:07:35.981 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T09:07:35.981 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 systemd[1]: Stopped Ceph prometheus.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:07:35.981 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:35 smithi181 systemd[1]: Started Ceph prometheus.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:07:36.204 INFO:tasks.rados.rados.0.smithi163.stdout:543: done (2 left) 2023-10-09T09:07:36.204 INFO:tasks.rados.rados.0.smithi163.stdout:544: snap_create 2023-10-09T09:07:36.204 INFO:tasks.rados.rados.0.smithi163.stdout:541: finishing write tid 1 to smithi16367185-24 2023-10-09T09:07:36.204 INFO:tasks.rados.rados.0.smithi163.stdout:541: finishing write tid 2 to smithi16367185-24 2023-10-09T09:07:36.204 INFO:tasks.rados.rados.0.smithi163.stdout:541: finishing write tid 3 to smithi16367185-24 2023-10-09T09:07:36.205 INFO:tasks.rados.rados.0.smithi163.stdout:541: finishing write tid 4 to smithi16367185-24 2023-10-09T09:07:36.205 INFO:tasks.rados.rados.0.smithi163.stdout:541: finishing write tid 5 to smithi16367185-24 2023-10-09T09:07:36.205 INFO:tasks.rados.rados.0.smithi163.stdout:541: finishing write tid 6 to smithi16367185-24 2023-10-09T09:07:36.205 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 134 (ObjNum 237 snap 70 seq_num 237) dirty exists 2023-10-09T09:07:36.205 INFO:tasks.rados.rados.0.smithi163.stdout:541: left oid 24 (ObjNum 237 snap 70 seq_num 237) 2023-10-09T09:07:36.205 INFO:tasks.rados.rados.0.smithi163.stdout:542: expect (ObjNum 117 snap 20 seq_num 117) 2023-10-09T09:07:36.296 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:36 smithi181 bash[27712]: cluster 2023-10-09T09:07:35.179104+0000 mon.a (mon.0) 999 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2023-10-09T09:07:36.296 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:36 smithi181 bash[27712]: audit 2023-10-09T09:07:35.407168+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:36.296 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:36 smithi181 bash[27712]: audit 2023-10-09T09:07:35.959051+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:36.296 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:36 smithi181 bash[27712]: audit 2023-10-09T09:07:35.967503+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:36 smithi163 bash[27500]: cluster 2023-10-09T09:07:35.179104+0000 mon.a (mon.0) 999 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2023-10-09T09:07:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:36 smithi163 bash[27500]: audit 2023-10-09T09:07:35.407168+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:36 smithi163 bash[27500]: audit 2023-10-09T09:07:35.959051+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:36 smithi163 bash[27500]: audit 2023-10-09T09:07:35.967503+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:36 smithi163 bash[33912]: cluster 2023-10-09T09:07:35.179104+0000 mon.a (mon.0) 999 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2023-10-09T09:07:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:36 smithi163 bash[33912]: audit 2023-10-09T09:07:35.407168+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:36 smithi163 bash[33912]: audit 2023-10-09T09:07:35.959051+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:36 smithi163 bash[33912]: audit 2023-10-09T09:07:35.967503+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:36.567 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:36 smithi181 systemd[1]: Stopping Ceph node-exporter.b for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:07:36.567 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:36 smithi181 bash[52969]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-node-exporter.b 2023-10-09T09:07:37.126 INFO:tasks.rados.rados.0.smithi163.stdout:541: done (2 left) 2023-10-09T09:07:37.126 INFO:tasks.rados.rados.0.smithi163.stdout:542: done (1 left) 2023-10-09T09:07:37.126 INFO:tasks.rados.rados.0.smithi163.stdout:544: done (0 left) 2023-10-09T09:07:37.126 INFO:tasks.rados.rados.0.smithi163.stdout:545: write oid 42 current snap is 71 2023-10-09T09:07:37.126 INFO:tasks.rados.rados.0.smithi163.stdout:545: seq_num 238 ranges {400066=716205,1860832=139234} 2023-10-09T09:07:37.133 INFO:tasks.rados.rados.0.smithi163.stdout:545: writing smithi16367185-42 from 400066 to 1116271 tid 1 2023-10-09T09:07:37.139 INFO:tasks.rados.rados.0.smithi163.stdout:545: writing smithi16367185-42 from 1860832 to 2000066 tid 2 2023-10-09T09:07:37.139 INFO:tasks.rados.rados.0.smithi163.stdout:546: snap_remove snap 62 2023-10-09T09:07:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[33912]: cephadm 2023-10-09T09:07:36.006746+0000 mgr.x (mgr.24751) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:07:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[33912]: cephadm 2023-10-09T09:07:36.007332+0000 mgr.x (mgr.24751) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi181 2023-10-09T09:07:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[33912]: cluster 2023-10-09T09:07:36.148369+0000 mgr.x (mgr.24751) 48 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 261 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 865 KiB/s wr, 2 op/s 2023-10-09T09:07:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[33912]: cluster 2023-10-09T09:07:36.203667+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2023-10-09T09:07:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[33912]: audit 2023-10-09T09:07:37.110465+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.163:0/979814022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[33912]: cluster 2023-10-09T09:07:37.125777+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2023-10-09T09:07:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[27500]: cephadm 2023-10-09T09:07:36.006746+0000 mgr.x (mgr.24751) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:07:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[27500]: cephadm 2023-10-09T09:07:36.007332+0000 mgr.x (mgr.24751) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi181 2023-10-09T09:07:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[27500]: cluster 2023-10-09T09:07:36.148369+0000 mgr.x (mgr.24751) 48 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 261 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 865 KiB/s wr, 2 op/s 2023-10-09T09:07:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[27500]: cluster 2023-10-09T09:07:36.203667+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2023-10-09T09:07:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[27500]: audit 2023-10-09T09:07:37.110465+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.163:0/979814022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:37 smithi163 bash[27500]: cluster 2023-10-09T09:07:37.125777+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2023-10-09T09:07:37.484 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[27712]: cephadm 2023-10-09T09:07:36.006746+0000 mgr.x (mgr.24751) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:07:37.484 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[27712]: cephadm 2023-10-09T09:07:36.007332+0000 mgr.x (mgr.24751) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi181 2023-10-09T09:07:37.484 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[27712]: cluster 2023-10-09T09:07:36.148369+0000 mgr.x (mgr.24751) 48 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 261 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 865 KiB/s wr, 2 op/s 2023-10-09T09:07:37.484 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[27712]: cluster 2023-10-09T09:07:36.203667+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2023-10-09T09:07:37.485 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[27712]: audit 2023-10-09T09:07:37.110465+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.163:0/979814022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:37.485 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[27712]: cluster 2023-10-09T09:07:37.125777+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2023-10-09T09:07:37.485 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[53007]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T09:07:37.817 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[53099]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T09:07:37.817 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:37 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:07:37.817 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:37 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T09:07:37.817 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:37 smithi181 systemd[1]: Stopped Ceph node-exporter.b for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:07:37.817 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:37 smithi181 systemd[1]: Started Ceph node-exporter.b for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:07:37.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.608Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T09:07:37.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.608Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T09:07:37.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.608Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi181 (none))" 2023-10-09T09:07:37.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.608Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T09:07:37.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.608Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T09:07:37.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.610Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T09:07:37.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.610Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T09:07:37.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.613Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T09:07:37.818 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.615Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T09:07:37.819 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.615Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.464µs 2023-10-09T09:07:37.819 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.615Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T09:07:38.109 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:37 smithi181 bash[52947]: ts=2023-10-09T09:07:37.826Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T09:07:38.137 INFO:tasks.rados.rados.0.smithi163.stdout:546: done (1 left) 2023-10-09T09:07:38.138 INFO:tasks.rados.rados.0.smithi163.stdout:547: snap_create 2023-10-09T09:07:38.138 INFO:tasks.rados.rados.0.smithi163.stdout:545: finishing write tid 1 to smithi16367185-42 2023-10-09T09:07:38.138 INFO:tasks.rados.rados.0.smithi163.stdout:545: finishing write tid 2 to smithi16367185-42 2023-10-09T09:07:38.138 INFO:tasks.rados.rados.0.smithi163.stdout:545: finishing write tid 3 to smithi16367185-42 2023-10-09T09:07:38.138 INFO:tasks.rados.rados.0.smithi163.stdout:545: finishing write tid 4 to smithi16367185-42 2023-10-09T09:07:38.138 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 425 (ObjNum 238 snap 71 seq_num 238) dirty exists 2023-10-09T09:07:38.138 INFO:tasks.rados.rados.0.smithi163.stdout:545: left oid 42 (ObjNum 238 snap 71 seq_num 238) 2023-10-09T09:07:38.560 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[52947]: ts=2023-10-09T09:07:38.107Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T09:07:38.560 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[52947]: ts=2023-10-09T09:07:38.107Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T09:07:38.560 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[52947]: ts=2023-10-09T09:07:38.107Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=71.016µs wal_replay_duration=491.729524ms total_replay_duration=491.841645ms 2023-10-09T09:07:38.561 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[52947]: ts=2023-10-09T09:07:38.113Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T09:07:38.561 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[52947]: ts=2023-10-09T09:07:38.113Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T09:07:38.561 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[52947]: ts=2023-10-09T09:07:38.113Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T09:07:38.561 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[52947]: ts=2023-10-09T09:07:38.145Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=32.755976ms db_storage=1.28µs remote_storage=2.449µs web_handler=671ns query_engine=1.226µs scrape=491.003µs scrape_sd=247.013µs notify=27.391µs notify_sd=22.885µs rules=31.366314ms 2023-10-09T09:07:38.561 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[52947]: ts=2023-10-09T09:07:38.146Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T09:07:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[27712]: audit 2023-10-09T09:07:37.561351+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[27712]: audit 2023-10-09T09:07:37.570446+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[27712]: cephadm 2023-10-09T09:07:37.575462+0000 mgr.x (mgr.24751) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:07:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[27712]: cephadm 2023-10-09T09:07:37.595698+0000 mgr.x (mgr.24751) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi181 2023-10-09T09:07:38.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[27712]: cluster 2023-10-09T09:07:38.137404+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2023-10-09T09:07:38.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:38 smithi181 bash[27712]: cluster 2023-10-09T09:07:38.148809+0000 mgr.x (mgr.24751) 51 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.8 MiB/s wr, 10 op/s 2023-10-09T09:07:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[27500]: audit 2023-10-09T09:07:37.561351+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[27500]: audit 2023-10-09T09:07:37.570446+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[27500]: cephadm 2023-10-09T09:07:37.575462+0000 mgr.x (mgr.24751) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:07:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[27500]: cephadm 2023-10-09T09:07:37.595698+0000 mgr.x (mgr.24751) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi181 2023-10-09T09:07:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[27500]: cluster 2023-10-09T09:07:38.137404+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2023-10-09T09:07:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[27500]: cluster 2023-10-09T09:07:38.148809+0000 mgr.x (mgr.24751) 51 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.8 MiB/s wr, 10 op/s 2023-10-09T09:07:38.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[33912]: audit 2023-10-09T09:07:37.561351+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:38.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[33912]: audit 2023-10-09T09:07:37.570446+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:07:38.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[33912]: cephadm 2023-10-09T09:07:37.575462+0000 mgr.x (mgr.24751) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:07:38.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[33912]: cephadm 2023-10-09T09:07:37.595698+0000 mgr.x (mgr.24751) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi181 2023-10-09T09:07:38.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[33912]: cluster 2023-10-09T09:07:38.137404+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2023-10-09T09:07:38.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:38 smithi163 bash[33912]: cluster 2023-10-09T09:07:38.148809+0000 mgr.x (mgr.24751) 51 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.8 MiB/s wr, 10 op/s 2023-10-09T09:07:39.145 INFO:tasks.rados.rados.0.smithi163.stdout:545: done (1 left) 2023-10-09T09:07:39.145 INFO:tasks.rados.rados.0.smithi163.stdout:547: done (0 left) 2023-10-09T09:07:39.145 INFO:tasks.rados.rados.0.smithi163.stdout:548: snap_remove snap 65 2023-10-09T09:07:39.318 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.007Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T09:07:39.318 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.007Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T09:07:39.318 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.007Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.007Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T09:07:39.319 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T09:07:39.320 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T09:07:39.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T09:07:39.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T09:07:39.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T09:07:39.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T09:07:39.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T09:07:39.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T09:07:39.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T09:07:39.321 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T09:07:39.322 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T09:07:39.323 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T09:07:39.323 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.009Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T09:07:39.323 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 09:07:39 smithi181 bash[53144]: ts=2023-10-09T09:07:39.010Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T09:07:40.150 INFO:tasks.rados.rados.0.smithi163.stdout:548: done (0 left) 2023-10-09T09:07:40.150 INFO:tasks.rados.rados.0.smithi163.stdout:549: rollback oid 39 current snap is 72 2023-10-09T09:07:40.150 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 64 2023-10-09T09:07:40.150 INFO:tasks.rados.rados.0.smithi163.stdout:550: read oid 9 snap -1 2023-10-09T09:07:40.151 INFO:tasks.rados.rados.0.smithi163.stdout:550: expect (ObjNum 220 snap 61 seq_num 220) 2023-10-09T09:07:40.151 INFO:tasks.rados.rados.0.smithi163.stdout:551: snap_create 2023-10-09T09:07:40.157 INFO:tasks.rados.rados.0.smithi163.stdout:549: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:07:40.157 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 125 (ObjNum 698773056 snap 21875 seq_num 699191552) dirty dne 2023-10-09T09:07:40.161 INFO:tasks.rados.rados.0.smithi163.stdout:550: expect (ObjNum 220 snap 61 seq_num 220) 2023-10-09T09:07:40.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:40 smithi163 bash[27500]: cluster 2023-10-09T09:07:39.143452+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2023-10-09T09:07:40.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:40 smithi163 bash[27500]: audit 2023-10-09T09:07:39.560996+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.163:0/1841284653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:40.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:40 smithi163 bash[33912]: cluster 2023-10-09T09:07:39.143452+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2023-10-09T09:07:40.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:40 smithi163 bash[33912]: audit 2023-10-09T09:07:39.560996+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.163:0/1841284653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:40 smithi181 bash[27712]: cluster 2023-10-09T09:07:39.143452+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2023-10-09T09:07:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:40 smithi181 bash[27712]: audit 2023-10-09T09:07:39.560996+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.163:0/1841284653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:41.153 INFO:tasks.rados.rados.0.smithi163.stdout:549: done (2 left) 2023-10-09T09:07:41.153 INFO:tasks.rados.rados.0.smithi163.stdout:550: done (1 left) 2023-10-09T09:07:41.154 INFO:tasks.rados.rados.0.smithi163.stdout:551: done (0 left) 2023-10-09T09:07:41.154 INFO:tasks.rados.rados.0.smithi163.stdout:552: write oid 40 current snap is 73 2023-10-09T09:07:41.154 INFO:tasks.rados.rados.0.smithi163.stdout:552: seq_num 239 ranges {416873=791454,1919314=97559} 2023-10-09T09:07:41.162 INFO:tasks.rados.rados.0.smithi163.stdout:552: writing smithi16367185-40 from 416873 to 1208327 tid 1 2023-10-09T09:07:41.170 INFO:tasks.rados.rados.0.smithi163.stdout:552: writing smithi16367185-40 from 1919314 to 2016873 tid 2 2023-10-09T09:07:41.170 INFO:tasks.rados.rados.0.smithi163.stdout:553: read oid 4 snap 69 2023-10-09T09:07:41.170 INFO:tasks.rados.rados.0.smithi163.stdout:553: expect (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:07:41.170 INFO:tasks.rados.rados.0.smithi163.stdout:554: write oid 24 current snap is 73 2023-10-09T09:07:41.171 INFO:tasks.rados.rados.0.smithi163.stdout:552: finishing write tid 1 to smithi16367185-40 2023-10-09T09:07:41.171 INFO:tasks.rados.rados.0.smithi163.stdout:554: seq_num 240 ranges {433680=466703,1577796=455884} 2023-10-09T09:07:41.175 INFO:tasks.rados.rados.0.smithi163.stdout:554: writing smithi16367185-24 from 433680 to 900383 tid 1 2023-10-09T09:07:41.183 INFO:tasks.rados.rados.0.smithi163.stdout:554: writing smithi16367185-24 from 1577796 to 2033680 tid 2 2023-10-09T09:07:41.183 INFO:tasks.rados.rados.0.smithi163.stdout:555: rollback oid 23 current snap is 73 2023-10-09T09:07:41.183 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 23 to 31 2023-10-09T09:07:41.183 INFO:tasks.rados.rados.0.smithi163.stdout:552: finishing write tid 2 to smithi16367185-40 2023-10-09T09:07:41.183 INFO:tasks.rados.rados.0.smithi163.stdout:552: finishing write tid 3 to smithi16367185-40 2023-10-09T09:07:41.184 INFO:tasks.rados.rados.0.smithi163.stdout:552: finishing write tid 4 to smithi16367185-40 2023-10-09T09:07:41.184 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 263 (ObjNum 239 snap 73 seq_num 239) dirty exists 2023-10-09T09:07:41.184 INFO:tasks.rados.rados.0.smithi163.stdout:552: left oid 40 (ObjNum 239 snap 73 seq_num 239) 2023-10-09T09:07:41.184 INFO:tasks.rados.rados.0.smithi163.stdout:554: finishing write tid 1 to smithi16367185-24 2023-10-09T09:07:41.184 INFO:tasks.rados.rados.0.smithi163.stdout:553: expect (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:07:41.311 INFO:tasks.rados.rados.0.smithi163.stdout:555: finishing rollback tid 0 to smithi16367185-23 2023-10-09T09:07:41.311 INFO:tasks.rados.rados.0.smithi163.stdout:554: finishing write tid 2 to smithi16367185-24 2023-10-09T09:07:41.311 INFO:tasks.rados.rados.0.smithi163.stdout:554: finishing write tid 3 to smithi16367185-24 2023-10-09T09:07:41.311 INFO:tasks.rados.rados.0.smithi163.stdout:554: finishing write tid 4 to smithi16367185-24 2023-10-09T09:07:41.311 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 142 (ObjNum 240 snap 73 seq_num 240) dirty exists 2023-10-09T09:07:41.311 INFO:tasks.rados.rados.0.smithi163.stdout:554: left oid 24 (ObjNum 240 snap 73 seq_num 240) 2023-10-09T09:07:41.311 INFO:tasks.rados.rados.0.smithi163.stdout:555: finishing rollback tid 1 to smithi16367185-23 2023-10-09T09:07:41.311 INFO:tasks.rados.rados.0.smithi163.stdout:555: finishing rollback tid 2 to smithi16367185-23 2023-10-09T09:07:41.312 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 168 (ObjNum 125 snap 23 seq_num 125) dirty exists 2023-10-09T09:07:41.312 INFO:tasks.rados.rados.0.smithi163.stdout:552: done (3 left) 2023-10-09T09:07:41.312 INFO:tasks.rados.rados.0.smithi163.stdout:553: done (2 left) 2023-10-09T09:07:41.312 INFO:tasks.rados.rados.0.smithi163.stdout:554: done (1 left) 2023-10-09T09:07:41.312 INFO:tasks.rados.rados.0.smithi163.stdout:555: done (0 left) 2023-10-09T09:07:41.312 INFO:tasks.rados.rados.0.smithi163.stdout:556: read oid 1 snap -1 2023-10-09T09:07:41.312 INFO:tasks.rados.rados.0.smithi163.stdout:556: expect (ObjNum 85 snap 10 seq_num 85) 2023-10-09T09:07:41.312 INFO:tasks.rados.rados.0.smithi163.stdout:557: write oid 12 current snap is 73 2023-10-09T09:07:41.312 INFO:tasks.rados.rados.0.smithi163.stdout:557: seq_num 241 ranges {450487=541952,1636278=414209} 2023-10-09T09:07:41.324 INFO:tasks.rados.rados.0.smithi163.stdout:557: writing smithi16367185-12 from 450487 to 992439 tid 1 2023-10-09T09:07:41.332 INFO:tasks.rados.rados.0.smithi163.stdout:557: writing smithi16367185-12 from 1636278 to 2050487 tid 2 2023-10-09T09:07:41.332 INFO:tasks.rados.rados.0.smithi163.stdout:558: delete oid 37 current snap is 73 2023-10-09T09:07:41.332 INFO:tasks.rados.rados.0.smithi163.stdout:556: expect (ObjNum 85 snap 10 seq_num 85) 2023-10-09T09:07:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:41 smithi163 bash[27500]: cluster 2023-10-09T09:07:40.149250+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2023-10-09T09:07:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:41 smithi163 bash[27500]: cluster 2023-10-09T09:07:40.149364+0000 mgr.x (mgr.24751) 52 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:07:41.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:41 smithi163 bash[27500]: cluster 2023-10-09T09:07:41.151862+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2023-10-09T09:07:41.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:41 smithi163 bash[33912]: cluster 2023-10-09T09:07:40.149250+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2023-10-09T09:07:41.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:41 smithi163 bash[33912]: cluster 2023-10-09T09:07:40.149364+0000 mgr.x (mgr.24751) 52 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:07:41.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:41 smithi163 bash[33912]: cluster 2023-10-09T09:07:41.151862+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2023-10-09T09:07:41.502 INFO:tasks.rados.rados.0.smithi163.stdout:557: finishing write tid 1 to smithi16367185-12 2023-10-09T09:07:41.502 INFO:tasks.rados.rados.0.smithi163.stdout:557: finishing write tid 2 to smithi16367185-12 2023-10-09T09:07:41.502 INFO:tasks.rados.rados.0.smithi163.stdout:557: finishing write tid 3 to smithi16367185-12 2023-10-09T09:07:41.502 INFO:tasks.rados.rados.0.smithi163.stdout:557: finishing write tid 4 to smithi16367185-12 2023-10-09T09:07:41.502 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 247 (ObjNum 241 snap 73 seq_num 241) dirty exists 2023-10-09T09:07:41.502 INFO:tasks.rados.rados.0.smithi163.stdout:557: left oid 12 (ObjNum 241 snap 73 seq_num 241) 2023-10-09T09:07:41.502 INFO:tasks.rados.rados.0.smithi163.stdout:556: done (2 left) 2023-10-09T09:07:41.502 INFO:tasks.rados.rados.0.smithi163.stdout:557: done (1 left) 2023-10-09T09:07:41.503 INFO:tasks.rados.rados.0.smithi163.stdout:558: done (0 left) 2023-10-09T09:07:41.503 INFO:tasks.rados.rados.0.smithi163.stdout:559: write oid 32 current snap is 73 2023-10-09T09:07:41.503 INFO:tasks.rados.rados.0.smithi163.stdout:559: seq_num 243 ranges {484101=692450,1636889=447212} 2023-10-09T09:07:41.513 INFO:tasks.rados.rados.0.smithi163.stdout:559: writing smithi16367185-32 from 484101 to 1176551 tid 1 2023-10-09T09:07:41.520 INFO:tasks.rados.rados.0.smithi163.stdout:559: writing smithi16367185-32 from 1636889 to 2084101 tid 2 2023-10-09T09:07:41.521 INFO:tasks.rados.rados.0.smithi163.stdout:560: rollback oid 21 current snap is 73 2023-10-09T09:07:41.521 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 67 2023-10-09T09:07:41.521 INFO:tasks.rados.rados.0.smithi163.stdout:559: finishing write tid 1 to smithi16367185-32 2023-10-09T09:07:41.521 INFO:tasks.rados.rados.0.smithi163.stdout:561: delete oid 44 current snap is 73 2023-10-09T09:07:41.527 INFO:tasks.rados.rados.0.smithi163.stdout:559: finishing write tid 2 to smithi16367185-32 2023-10-09T09:07:41.527 INFO:tasks.rados.rados.0.smithi163.stdout:560: finishing rollback tid 0 to smithi16367185-21 2023-10-09T09:07:41.527 INFO:tasks.rados.rados.0.smithi163.stdout:559: finishing write tid 3 to smithi16367185-32 2023-10-09T09:07:41.527 INFO:tasks.rados.rados.0.smithi163.stdout:560: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:07:41.527 INFO:tasks.rados.rados.0.smithi163.stdout:560: finishing rollback tid 2 to smithi16367185-21 2023-10-09T09:07:41.527 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 148 (ObjNum 224 snap 62 seq_num 224) dirty exists 2023-10-09T09:07:41.527 INFO:tasks.rados.rados.0.smithi163.stdout:559: finishing write tid 4 to smithi16367185-32 2023-10-09T09:07:41.527 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 431 (ObjNum 243 snap 73 seq_num 243) dirty exists 2023-10-09T09:07:41.528 INFO:tasks.rados.rados.0.smithi163.stdout:559: left oid 32 (ObjNum 243 snap 73 seq_num 243) 2023-10-09T09:07:41.528 INFO:tasks.rados.rados.0.smithi163.stdout:559: done (2 left) 2023-10-09T09:07:41.528 INFO:tasks.rados.rados.0.smithi163.stdout:560: done (1 left) 2023-10-09T09:07:41.528 INFO:tasks.rados.rados.0.smithi163.stdout:561: done (0 left) 2023-10-09T09:07:41.528 INFO:tasks.rados.rados.0.smithi163.stdout:562: delete oid 27 current snap is 73 2023-10-09T09:07:41.529 INFO:tasks.rados.rados.0.smithi163.stdout:562: done (0 left) 2023-10-09T09:07:41.529 INFO:tasks.rados.rados.0.smithi163.stdout:563: delete oid 24 current snap is 73 2023-10-09T09:07:41.531 INFO:tasks.rados.rados.0.smithi163.stdout:563: done (0 left) 2023-10-09T09:07:41.531 INFO:tasks.rados.rados.0.smithi163.stdout:564: write oid 4 current snap is 73 2023-10-09T09:07:41.531 INFO:tasks.rados.rados.0.smithi163.stdout:564: seq_num 247 ranges {551329=709799,1870817=280512} 2023-10-09T09:07:41.540 INFO:tasks.rados.rados.0.smithi163.stdout:564: writing smithi16367185-4 from 551329 to 1261128 tid 1 2023-10-09T09:07:41.545 INFO:tasks.rados.rados.0.smithi163.stdout:564: writing smithi16367185-4 from 1870817 to 2151329 tid 2 2023-10-09T09:07:41.546 INFO:tasks.rados.rados.0.smithi163.stdout:565: read oid 31 snap -1 2023-10-09T09:07:41.546 INFO:tasks.rados.rados.0.smithi163.stdout:565: expect deleted 2023-10-09T09:07:41.546 INFO:tasks.rados.rados.0.smithi163.stdout:566: snap_create 2023-10-09T09:07:41.546 INFO:tasks.rados.rados.0.smithi163.stdout:564: finishing write tid 1 to smithi16367185-4 2023-10-09T09:07:41.549 INFO:tasks.rados.rados.0.smithi163.stdout:564: finishing write tid 2 to smithi16367185-4 2023-10-09T09:07:41.549 INFO:tasks.rados.rados.0.smithi163.stdout:564: finishing write tid 3 to smithi16367185-4 2023-10-09T09:07:41.549 INFO:tasks.rados.rados.0.smithi163.stdout:564: finishing write tid 4 to smithi16367185-4 2023-10-09T09:07:41.549 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 270 (ObjNum 247 snap 73 seq_num 247) dirty exists 2023-10-09T09:07:41.549 INFO:tasks.rados.rados.0.smithi163.stdout:564: left oid 4 (ObjNum 247 snap 73 seq_num 247) 2023-10-09T09:07:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:41 smithi181 bash[27712]: cluster 2023-10-09T09:07:40.149250+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2023-10-09T09:07:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:41 smithi181 bash[27712]: cluster 2023-10-09T09:07:40.149364+0000 mgr.x (mgr.24751) 52 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:07:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:41 smithi181 bash[27712]: cluster 2023-10-09T09:07:41.151862+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2023-10-09T09:07:42.172 INFO:tasks.rados.rados.0.smithi163.stdout:564: done (2 left) 2023-10-09T09:07:42.172 INFO:tasks.rados.rados.0.smithi163.stdout:565: done (1 left) 2023-10-09T09:07:42.173 INFO:tasks.rados.rados.0.smithi163.stdout:566: done (0 left) 2023-10-09T09:07:42.173 INFO:tasks.rados.rados.0.smithi163.stdout:567: write oid 36 current snap is 74 2023-10-09T09:07:42.173 INFO:tasks.rados.rados.0.smithi163.stdout:567: seq_num 248 ranges {568136=785048,1929299=238837} 2023-10-09T09:07:42.182 INFO:tasks.rados.rados.0.smithi163.stdout:567: writing smithi16367185-36 from 568136 to 1353184 tid 1 2023-10-09T09:07:42.187 INFO:tasks.rados.rados.0.smithi163.stdout:567: writing smithi16367185-36 from 1929299 to 2168136 tid 2 2023-10-09T09:07:42.187 INFO:tasks.rados.rados.0.smithi163.stdout:568: rollback oid 12 current snap is 74 2023-10-09T09:07:42.187 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 12 to 73 2023-10-09T09:07:42.187 INFO:tasks.rados.rados.0.smithi163.stdout:569: write oid 18 current snap is 74 2023-10-09T09:07:42.187 INFO:tasks.rados.rados.0.smithi163.stdout:569: seq_num 249 ranges {584943=460297,1587781=597162} 2023-10-09T09:07:42.193 INFO:tasks.rados.rados.0.smithi163.stdout:569: writing smithi16367185-18 from 584943 to 1045240 tid 1 2023-10-09T09:07:42.200 INFO:tasks.rados.rados.0.smithi163.stdout:569: writing smithi16367185-18 from 1587781 to 2184943 tid 2 2023-10-09T09:07:42.200 INFO:tasks.rados.rados.0.smithi163.stdout:570: rollback oid 21 current snap is 74 2023-10-09T09:07:42.200 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 68 2023-10-09T09:07:42.200 INFO:tasks.rados.rados.0.smithi163.stdout:567: finishing write tid 1 to smithi16367185-36 2023-10-09T09:07:42.201 INFO:tasks.rados.rados.0.smithi163.stdout:567: finishing write tid 2 to smithi16367185-36 2023-10-09T09:07:42.201 INFO:tasks.rados.rados.0.smithi163.stdout:568: finishing rollback tid 0 to smithi16367185-12 2023-10-09T09:07:42.201 INFO:tasks.rados.rados.0.smithi163.stdout:567: finishing write tid 3 to smithi16367185-36 2023-10-09T09:07:42.201 INFO:tasks.rados.rados.0.smithi163.stdout:571: snap_remove snap 64 2023-10-09T09:07:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:42 smithi163 bash[33912]: audit 2023-10-09T09:07:41.997755+0000 mon.b (mon.1) 82 : audit [DBG] from='client.? 172.21.15.163:0/2606595131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:42 smithi163 bash[27500]: audit 2023-10-09T09:07:41.997755+0000 mon.b (mon.1) 82 : audit [DBG] from='client.? 172.21.15.163:0/2606595131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:42 smithi181 bash[27712]: audit 2023-10-09T09:07:41.997755+0000 mon.b (mon.1) 82 : audit [DBG] from='client.? 172.21.15.163:0/2606595131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:43.193 INFO:tasks.rados.rados.0.smithi163.stdout:568: finishing rollback tid 1 to smithi16367185-12 2023-10-09T09:07:43.194 INFO:tasks.rados.rados.0.smithi163.stdout:567: finishing write tid 4 to smithi16367185-36 2023-10-09T09:07:43.194 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 172 (ObjNum 248 snap 74 seq_num 248) dirty exists 2023-10-09T09:07:43.194 INFO:tasks.rados.rados.0.smithi163.stdout:567: left oid 36 (ObjNum 248 snap 74 seq_num 248) 2023-10-09T09:07:43.194 INFO:tasks.rados.rados.0.smithi163.stdout:568: finishing rollback tid 2 to smithi16367185-12 2023-10-09T09:07:43.194 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 251 (ObjNum 241 snap 73 seq_num 241) dirty exists 2023-10-09T09:07:43.194 INFO:tasks.rados.rados.0.smithi163.stdout:569: finishing write tid 1 to smithi16367185-18 2023-10-09T09:07:43.194 INFO:tasks.rados.rados.0.smithi163.stdout:570: finishing rollback tid 0 to smithi16367185-21 2023-10-09T09:07:43.194 INFO:tasks.rados.rados.0.smithi163.stdout:570: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:07:43.194 INFO:tasks.rados.rados.0.smithi163.stdout:569: finishing write tid 2 to smithi16367185-18 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:570: finishing rollback tid 2 to smithi16367185-21 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 152 (ObjNum 224 snap 62 seq_num 224) dirty exists 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:569: finishing write tid 3 to smithi16367185-18 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:569: finishing write tid 4 to smithi16367185-18 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 174 (ObjNum 249 snap 74 seq_num 249) dirty exists 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:569: left oid 18 (ObjNum 249 snap 74 seq_num 249) 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:567: done (4 left) 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:568: done (3 left) 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:569: done (2 left) 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:570: done (1 left) 2023-10-09T09:07:43.195 INFO:tasks.rados.rados.0.smithi163.stdout:571: done (0 left) 2023-10-09T09:07:43.196 INFO:tasks.rados.rados.0.smithi163.stdout:572: read oid 26 snap -1 2023-10-09T09:07:43.196 INFO:tasks.rados.rados.0.smithi163.stdout:572: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:07:43.196 INFO:tasks.rados.rados.0.smithi163.stdout:573: snap_remove snap 70 2023-10-09T09:07:43.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:43 smithi163 bash[33912]: cluster 2023-10-09T09:07:42.150288+0000 mgr.x (mgr.24751) 53 : cluster [DBG] pgmap v80: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 208 KiB/s wr, 3 op/s 2023-10-09T09:07:43.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:43 smithi163 bash[33912]: cluster 2023-10-09T09:07:42.171126+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2023-10-09T09:07:43.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:43 smithi163 bash[27500]: cluster 2023-10-09T09:07:42.150288+0000 mgr.x (mgr.24751) 53 : cluster [DBG] pgmap v80: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 208 KiB/s wr, 3 op/s 2023-10-09T09:07:43.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:43 smithi163 bash[27500]: cluster 2023-10-09T09:07:42.171126+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2023-10-09T09:07:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:43 smithi181 bash[27712]: cluster 2023-10-09T09:07:42.150288+0000 mgr.x (mgr.24751) 53 : cluster [DBG] pgmap v80: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 208 KiB/s wr, 3 op/s 2023-10-09T09:07:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:43 smithi181 bash[27712]: cluster 2023-10-09T09:07:42.171126+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2023-10-09T09:07:44.198 INFO:tasks.rados.rados.0.smithi163.stdout:573: done (1 left) 2023-10-09T09:07:44.198 INFO:tasks.rados.rados.0.smithi163.stdout:574: snap_create 2023-10-09T09:07:44.198 INFO:tasks.rados.rados.0.smithi163.stdout:572: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:07:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:44 smithi163 bash[27500]: cluster 2023-10-09T09:07:43.192850+0000 mon.a (mon.0) 1011 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2023-10-09T09:07:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:44 smithi163 bash[33912]: cluster 2023-10-09T09:07:43.192850+0000 mon.a (mon.0) 1011 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2023-10-09T09:07:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:44 smithi181 bash[27712]: cluster 2023-10-09T09:07:43.192850+0000 mon.a (mon.0) 1011 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2023-10-09T09:07:45.209 INFO:tasks.rados.rados.0.smithi163.stdout:572: done (1 left) 2023-10-09T09:07:45.209 INFO:tasks.rados.rados.0.smithi163.stdout:574: done (0 left) 2023-10-09T09:07:45.209 INFO:tasks.rados.rados.0.smithi163.stdout:575: snap_create 2023-10-09T09:07:45.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:45 smithi163 bash[27500]: cluster 2023-10-09T09:07:44.150831+0000 mgr.x (mgr.24751) 54 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 209 KiB/s wr, 3 op/s 2023-10-09T09:07:45.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:45 smithi163 bash[27500]: cluster 2023-10-09T09:07:44.197793+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2023-10-09T09:07:45.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:45 smithi163 bash[27500]: audit 2023-10-09T09:07:44.457275+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.163:0/1757719910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:45 smithi163 bash[33912]: cluster 2023-10-09T09:07:44.150831+0000 mgr.x (mgr.24751) 54 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 209 KiB/s wr, 3 op/s 2023-10-09T09:07:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:45 smithi163 bash[33912]: cluster 2023-10-09T09:07:44.197793+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2023-10-09T09:07:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:45 smithi163 bash[33912]: audit 2023-10-09T09:07:44.457275+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.163:0/1757719910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:45 smithi181 bash[27712]: cluster 2023-10-09T09:07:44.150831+0000 mgr.x (mgr.24751) 54 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 267 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 209 KiB/s wr, 3 op/s 2023-10-09T09:07:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:45 smithi181 bash[27712]: cluster 2023-10-09T09:07:44.197793+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2023-10-09T09:07:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:45 smithi181 bash[27712]: audit 2023-10-09T09:07:44.457275+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.163:0/1757719910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:46.212 INFO:tasks.rados.rados.0.smithi163.stdout:575: done (0 left) 2023-10-09T09:07:46.212 INFO:tasks.rados.rados.0.smithi163.stdout:576: snap_remove snap 59 2023-10-09T09:07:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:46 smithi163 bash[27500]: cluster 2023-10-09T09:07:45.208045+0000 mon.a (mon.0) 1013 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2023-10-09T09:07:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:46 smithi163 bash[33912]: cluster 2023-10-09T09:07:45.208045+0000 mon.a (mon.0) 1013 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2023-10-09T09:07:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:46 smithi181 bash[27712]: cluster 2023-10-09T09:07:45.208045+0000 mon.a (mon.0) 1013 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2023-10-09T09:07:47.132 INFO:tasks.rados.rados.0.smithi163.stdout:576: done (0 left) 2023-10-09T09:07:47.132 INFO:tasks.rados.rados.0.smithi163.stdout:577: delete oid 39 current snap is 76 2023-10-09T09:07:47.140 INFO:tasks.rados.rados.0.smithi163.stdout:577: done (0 left) 2023-10-09T09:07:47.140 INFO:tasks.rados.rados.0.smithi163.stdout:578: delete oid 19 current snap is 76 2023-10-09T09:07:47.147 INFO:tasks.rados.rados.0.smithi163.stdout:578: done (0 left) 2023-10-09T09:07:47.147 INFO:tasks.rados.rados.0.smithi163.stdout:579: read oid 6 snap -1 2023-10-09T09:07:47.147 INFO:tasks.rados.rados.0.smithi163.stdout:579: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:07:47.147 INFO:tasks.rados.rados.0.smithi163.stdout:580: write oid 45 current snap is 76 2023-10-09T09:07:47.147 INFO:tasks.rados.rados.0.smithi163.stdout:580: seq_num 252 ranges {635364=402397,1763227=472137} 2023-10-09T09:07:47.155 INFO:tasks.rados.rados.0.smithi163.stdout:580: writing smithi16367185-45 from 635364 to 1037761 tid 1 2023-10-09T09:07:47.163 INFO:tasks.rados.rados.0.smithi163.stdout:580: writing smithi16367185-45 from 1763227 to 2235364 tid 2 2023-10-09T09:07:47.163 INFO:tasks.rados.rados.0.smithi163.stdout:581: snap_create 2023-10-09T09:07:47.163 INFO:tasks.rados.rados.0.smithi163.stdout:579: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:07:47.359 INFO:tasks.rados.rados.0.smithi163.stdout:580: finishing write tid 1 to smithi16367185-45 2023-10-09T09:07:47.360 INFO:tasks.rados.rados.0.smithi163.stdout:580: finishing write tid 2 to smithi16367185-45 2023-10-09T09:07:47.360 INFO:tasks.rados.rados.0.smithi163.stdout:580: finishing write tid 3 to smithi16367185-45 2023-10-09T09:07:47.360 INFO:tasks.rados.rados.0.smithi163.stdout:580: finishing write tid 4 to smithi16367185-45 2023-10-09T09:07:47.360 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 453 (ObjNum 252 snap 76 seq_num 252) dirty exists 2023-10-09T09:07:47.360 INFO:tasks.rados.rados.0.smithi163.stdout:580: left oid 45 (ObjNum 252 snap 76 seq_num 252) 2023-10-09T09:07:47.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:47 smithi163 bash[33912]: cluster 2023-10-09T09:07:46.151467+0000 mgr.x (mgr.24751) 55 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 268 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 278 KiB/s wr, 1 op/s 2023-10-09T09:07:47.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:47 smithi163 bash[33912]: cluster 2023-10-09T09:07:46.211242+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2023-10-09T09:07:47.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:47 smithi163 bash[33912]: audit 2023-10-09T09:07:46.896930+0000 mon.a (mon.0) 1015 : audit [DBG] from='client.? 172.21.15.163:0/2994356453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:47.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:47 smithi163 bash[33912]: cluster 2023-10-09T09:07:47.132189+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2023-10-09T09:07:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:47 smithi163 bash[27500]: cluster 2023-10-09T09:07:46.151467+0000 mgr.x (mgr.24751) 55 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 268 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 278 KiB/s wr, 1 op/s 2023-10-09T09:07:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:47 smithi163 bash[27500]: cluster 2023-10-09T09:07:46.211242+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2023-10-09T09:07:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:47 smithi163 bash[27500]: audit 2023-10-09T09:07:46.896930+0000 mon.a (mon.0) 1015 : audit [DBG] from='client.? 172.21.15.163:0/2994356453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:47.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:47 smithi163 bash[27500]: cluster 2023-10-09T09:07:47.132189+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2023-10-09T09:07:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:47 smithi181 bash[27712]: cluster 2023-10-09T09:07:46.151467+0000 mgr.x (mgr.24751) 55 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 268 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 278 KiB/s wr, 1 op/s 2023-10-09T09:07:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:47 smithi181 bash[27712]: cluster 2023-10-09T09:07:46.211242+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2023-10-09T09:07:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:47 smithi181 bash[27712]: audit 2023-10-09T09:07:46.896930+0000 mon.a (mon.0) 1015 : audit [DBG] from='client.? 172.21.15.163:0/2994356453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:47 smithi181 bash[27712]: cluster 2023-10-09T09:07:47.132189+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2023-10-09T09:07:48.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:07:47 smithi181 bash[50540]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:07:47] "GET /metrics HTTP/1.1" 200 36086 "" "Prometheus/2.33.4" 2023-10-09T09:07:48.145 INFO:tasks.rados.rados.0.smithi163.stdout:579: done (2 left) 2023-10-09T09:07:48.146 INFO:tasks.rados.rados.0.smithi163.stdout:580: done (1 left) 2023-10-09T09:07:48.146 INFO:tasks.rados.rados.0.smithi163.stdout:581: done (0 left) 2023-10-09T09:07:48.146 INFO:tasks.rados.rados.0.smithi163.stdout:582: write oid 2 current snap is 77 2023-10-09T09:07:48.146 INFO:tasks.rados.rados.0.smithi163.stdout:582: seq_num 253 ranges {652171=477646,1821709=430462} 2023-10-09T09:07:48.155 INFO:tasks.rados.rados.0.smithi163.stdout:582: writing smithi16367185-2 from 652171 to 1129817 tid 1 2023-10-09T09:07:48.161 INFO:tasks.rados.rados.0.smithi163.stdout:582: writing smithi16367185-2 from 1821709 to 2252171 tid 2 2023-10-09T09:07:48.161 INFO:tasks.rados.rados.0.smithi163.stdout:583: write oid 17 current snap is 77 2023-10-09T09:07:48.161 INFO:tasks.rados.rados.0.smithi163.stdout:583: seq_num 254 ranges {668978=552895,1763838=505140} 2023-10-09T09:07:48.168 INFO:tasks.rados.rados.0.smithi163.stdout:583: writing smithi16367185-17 from 668978 to 1221873 tid 1 2023-10-09T09:07:48.174 INFO:tasks.rados.rados.0.smithi163.stdout:583: writing smithi16367185-17 from 1763838 to 2268978 tid 2 2023-10-09T09:07:48.174 INFO:tasks.rados.rados.0.smithi163.stdout:584: write oid 30 current snap is 77 2023-10-09T09:07:48.175 INFO:tasks.rados.rados.0.smithi163.stdout:582: finishing write tid 1 to smithi16367185-2 2023-10-09T09:07:48.175 INFO:tasks.rados.rados.0.smithi163.stdout:584: seq_num 255 ranges {685785=628144,1822320=463465} 2023-10-09T09:07:48.182 INFO:tasks.rados.rados.0.smithi163.stdout:584: writing smithi16367185-30 from 685785 to 1313929 tid 1 2023-10-09T09:07:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:584: writing smithi16367185-30 from 1822320 to 2285785 tid 2 2023-10-09T09:07:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:585: delete oid 23 current snap is 77 2023-10-09T09:07:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:582: finishing write tid 2 to smithi16367185-2 2023-10-09T09:07:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:582: finishing write tid 3 to smithi16367185-2 2023-10-09T09:07:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:582: finishing write tid 4 to smithi16367185-2 2023-10-09T09:07:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 104 (ObjNum 253 snap 77 seq_num 253) dirty exists 2023-10-09T09:07:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:582: left oid 2 (ObjNum 253 snap 77 seq_num 253) 2023-10-09T09:07:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:583: finishing write tid 1 to smithi16367185-17 2023-10-09T09:07:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:583: finishing write tid 2 to smithi16367185-17 2023-10-09T09:07:48.189 INFO:tasks.rados.rados.0.smithi163.stdout:583: finishing write tid 3 to smithi16367185-17 2023-10-09T09:07:48.189 INFO:tasks.rados.rados.0.smithi163.stdout:583: finishing write tid 4 to smithi16367185-17 2023-10-09T09:07:48.189 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 277 (ObjNum 254 snap 77 seq_num 254) dirty exists 2023-10-09T09:07:48.189 INFO:tasks.rados.rados.0.smithi163.stdout:583: left oid 17 (ObjNum 254 snap 77 seq_num 254) 2023-10-09T09:07:48.189 INFO:tasks.rados.rados.0.smithi163.stdout:584: finishing write tid 1 to smithi16367185-30 2023-10-09T09:07:48.189 INFO:tasks.rados.rados.0.smithi163.stdout:582: done (3 left) 2023-10-09T09:07:48.190 INFO:tasks.rados.rados.0.smithi163.stdout:583: done (2 left) 2023-10-09T09:07:48.190 INFO:tasks.rados.rados.0.smithi163.stdout:585: done (1 left) 2023-10-09T09:07:48.190 INFO:tasks.rados.rados.0.smithi163.stdout:586: read oid 39 snap 74 2023-10-09T09:07:48.190 INFO:tasks.rados.rados.0.smithi163.stdout:586: expect deleted 2023-10-09T09:07:48.190 INFO:tasks.rados.rados.0.smithi163.stdout:587: snap_remove snap 58 2023-10-09T09:07:49.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:48 smithi181 bash[52947]: ts=2023-10-09T09:07:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:07:49.150 INFO:tasks.rados.rados.0.smithi163.stdout:584: finishing write tid 2 to smithi16367185-30 2023-10-09T09:07:49.150 INFO:tasks.rados.rados.0.smithi163.stdout:584: finishing write tid 3 to smithi16367185-30 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:584: finishing write tid 4 to smithi16367185-30 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 188 (ObjNum 255 snap 77 seq_num 255) dirty exists 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:584: left oid 30 (ObjNum 255 snap 77 seq_num 255) 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:584: done (2 left) 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:586: done (1 left) 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:587: done (0 left) 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:588: rollback oid 48 current snap is 77 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 48 to 75 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:589: rollback oid 17 current snap is 77 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 71 2023-10-09T09:07:49.151 INFO:tasks.rados.rados.0.smithi163.stdout:590: delete oid 29 current snap is 77 2023-10-09T09:07:49.157 INFO:tasks.rados.rados.0.smithi163.stdout:590: done (2 left) 2023-10-09T09:07:49.157 INFO:tasks.rados.rados.0.smithi163.stdout:591: read oid 5 snap 36 2023-10-09T09:07:49.157 INFO:tasks.rados.rados.0.smithi163.stdout:591: expect (ObjNum 4 snap 0 seq_num 4) 2023-10-09T09:07:49.157 INFO:tasks.rados.rados.0.smithi163.stdout:592: write oid 49 current snap is 77 2023-10-09T09:07:49.157 INFO:tasks.rados.rados.0.smithi163.stdout:592: seq_num 258 ranges {736206=453891,1881413=454793} 2023-10-09T09:07:49.164 INFO:tasks.rados.rados.0.smithi163.stdout:592: writing smithi16367185-49 from 736206 to 1190097 tid 1 2023-10-09T09:07:49.171 INFO:tasks.rados.rados.0.smithi163.stdout:592: writing smithi16367185-49 from 1881413 to 2336206 tid 2 2023-10-09T09:07:49.171 INFO:tasks.rados.rados.0.smithi163.stdout:593: snap_create 2023-10-09T09:07:49.171 INFO:tasks.rados.rados.0.smithi163.stdout:589: finishing rollback tid 0 to smithi16367185-17 2023-10-09T09:07:49.171 INFO:tasks.rados.rados.0.smithi163.stdout:589: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:07:49.172 INFO:tasks.rados.rados.0.smithi163.stdout:589: finishing rollback tid 2 to smithi16367185-17 2023-10-09T09:07:49.172 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 280 (ObjNum 137 snap 28 seq_num 137) dirty exists 2023-10-09T09:07:49.172 INFO:tasks.rados.rados.0.smithi163.stdout:591: expect (ObjNum 4 snap 0 seq_num 4) 2023-10-09T09:07:49.245 INFO:tasks.rados.rados.0.smithi163.stdout:592: finishing write tid 1 to smithi16367185-49 2023-10-09T09:07:49.245 INFO:tasks.rados.rados.0.smithi163.stdout:588: finishing rollback tid 0 to smithi16367185-48 2023-10-09T09:07:49.245 INFO:tasks.rados.rados.0.smithi163.stdout:588: finishing rollback tid 1 to smithi16367185-48 2023-10-09T09:07:49.245 INFO:tasks.rados.rados.0.smithi163.stdout:588: finishing rollback tid 2 to smithi16367185-48 2023-10-09T09:07:49.245 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 461 (ObjNum 128 snap 27 seq_num 128) dirty exists 2023-10-09T09:07:49.246 INFO:tasks.rados.rados.0.smithi163.stdout:592: finishing write tid 2 to smithi16367185-49 2023-10-09T09:07:49.246 INFO:tasks.rados.rados.0.smithi163.stdout:592: finishing write tid 3 to smithi16367185-49 2023-10-09T09:07:49.246 INFO:tasks.rados.rados.0.smithi163.stdout:592: finishing write tid 4 to smithi16367185-49 2023-10-09T09:07:49.246 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 121 (ObjNum 258 snap 77 seq_num 258) dirty exists 2023-10-09T09:07:49.246 INFO:tasks.rados.rados.0.smithi163.stdout:592: left oid 49 (ObjNum 258 snap 77 seq_num 258) 2023-10-09T09:07:49.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:49 smithi163 bash[27500]: cluster 2023-10-09T09:07:48.145669+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2023-10-09T09:07:49.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:49 smithi163 bash[27500]: cluster 2023-10-09T09:07:48.152011+0000 mgr.x (mgr.24751) 56 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 274 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2023-10-09T09:07:49.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:49 smithi163 bash[33912]: cluster 2023-10-09T09:07:48.145669+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2023-10-09T09:07:49.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:49 smithi163 bash[33912]: cluster 2023-10-09T09:07:48.152011+0000 mgr.x (mgr.24751) 56 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 274 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2023-10-09T09:07:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:49 smithi181 bash[27712]: cluster 2023-10-09T09:07:48.145669+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2023-10-09T09:07:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:49 smithi181 bash[27712]: cluster 2023-10-09T09:07:48.152011+0000 mgr.x (mgr.24751) 56 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 274 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2023-10-09T09:07:50.160 INFO:tasks.rados.rados.0.smithi163.stdout:588: done (4 left) 2023-10-09T09:07:50.160 INFO:tasks.rados.rados.0.smithi163.stdout:589: done (3 left) 2023-10-09T09:07:50.160 INFO:tasks.rados.rados.0.smithi163.stdout:591: done (2 left) 2023-10-09T09:07:50.160 INFO:tasks.rados.rados.0.smithi163.stdout:592: done (1 left) 2023-10-09T09:07:50.160 INFO:tasks.rados.rados.0.smithi163.stdout:593: done (0 left) 2023-10-09T09:07:50.160 INFO:tasks.rados.rados.0.smithi163.stdout:594: snap_create 2023-10-09T09:07:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:50 smithi163 bash[33912]: cluster 2023-10-09T09:07:49.149248+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2023-10-09T09:07:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:50 smithi163 bash[33912]: audit 2023-10-09T09:07:49.350641+0000 mon.a (mon.0) 1019 : audit [DBG] from='client.? 172.21.15.163:0/3308576311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:50.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:50 smithi163 bash[27500]: cluster 2023-10-09T09:07:49.149248+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2023-10-09T09:07:50.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:50 smithi163 bash[27500]: audit 2023-10-09T09:07:49.350641+0000 mon.a (mon.0) 1019 : audit [DBG] from='client.? 172.21.15.163:0/3308576311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:50 smithi181 bash[27712]: cluster 2023-10-09T09:07:49.149248+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2023-10-09T09:07:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:50 smithi181 bash[27712]: audit 2023-10-09T09:07:49.350641+0000 mon.a (mon.0) 1019 : audit [DBG] from='client.? 172.21.15.163:0/3308576311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:51.175 INFO:tasks.rados.rados.0.smithi163.stdout:594: done (0 left) 2023-10-09T09:07:51.175 INFO:tasks.rados.rados.0.smithi163.stdout:595: read oid 13 snap -1 2023-10-09T09:07:51.176 INFO:tasks.rados.rados.0.smithi163.stdout:595: expect (ObjNum 168 snap 44 seq_num 168) 2023-10-09T09:07:51.176 INFO:tasks.rados.rados.0.smithi163.stdout:596: snap_remove snap 69 2023-10-09T09:07:51.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:51 smithi163 bash[27500]: cluster 2023-10-09T09:07:50.152548+0000 mgr.x (mgr.24751) 57 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 274 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2023-10-09T09:07:51.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:51 smithi163 bash[27500]: cluster 2023-10-09T09:07:50.160547+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2023-10-09T09:07:51.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:51 smithi163 bash[27500]: audit 2023-10-09T09:07:50.409024+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:51.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:51 smithi163 bash[33912]: cluster 2023-10-09T09:07:50.152548+0000 mgr.x (mgr.24751) 57 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 274 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2023-10-09T09:07:51.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:51 smithi163 bash[33912]: cluster 2023-10-09T09:07:50.160547+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2023-10-09T09:07:51.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:51 smithi163 bash[33912]: audit 2023-10-09T09:07:50.409024+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:51 smithi181 bash[27712]: cluster 2023-10-09T09:07:50.152548+0000 mgr.x (mgr.24751) 57 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 274 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2023-10-09T09:07:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:51 smithi181 bash[27712]: cluster 2023-10-09T09:07:50.160547+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2023-10-09T09:07:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:51 smithi181 bash[27712]: audit 2023-10-09T09:07:50.409024+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:07:52.176 INFO:tasks.rados.rados.0.smithi163.stdout:595: expect (ObjNum 168 snap 44 seq_num 168) 2023-10-09T09:07:52.297 INFO:tasks.rados.rados.0.smithi163.stdout:595: done (1 left) 2023-10-09T09:07:52.297 INFO:tasks.rados.rados.0.smithi163.stdout:596: done (0 left) 2023-10-09T09:07:52.297 INFO:tasks.rados.rados.0.smithi163.stdout:597: write oid 30 current snap is 79 2023-10-09T09:07:52.298 INFO:tasks.rados.rados.0.smithi163.stdout:597: seq_num 259 ranges {753013=645493,2056248=296765} 2023-10-09T09:07:52.310 INFO:tasks.rados.rados.0.smithi163.stdout:597: writing smithi16367185-30 from 753013 to 1398506 tid 1 2023-10-09T09:07:52.316 INFO:tasks.rados.rados.0.smithi163.stdout:597: writing smithi16367185-30 from 2056248 to 2353013 tid 2 2023-10-09T09:07:52.316 INFO:tasks.rados.rados.0.smithi163.stdout:598: read oid 23 snap 72 2023-10-09T09:07:52.316 INFO:tasks.rados.rados.0.smithi163.stdout:598: expect (ObjNum 125 snap 23 seq_num 125) 2023-10-09T09:07:52.316 INFO:tasks.rados.rados.0.smithi163.stdout:599: write oid 16 current snap is 79 2023-10-09T09:07:52.316 INFO:tasks.rados.rados.0.smithi163.stdout:599: seq_num 260 ranges {769820=720742,2114730=255090} 2023-10-09T09:07:52.324 INFO:tasks.rados.rados.0.smithi163.stdout:599: writing smithi16367185-16 from 769820 to 1490562 tid 1 2023-10-09T09:07:52.330 INFO:tasks.rados.rados.0.smithi163.stdout:599: writing smithi16367185-16 from 2114730 to 2369820 tid 2 2023-10-09T09:07:52.330 INFO:tasks.rados.rados.0.smithi163.stdout:600: delete oid 41 current snap is 79 2023-10-09T09:07:52.330 INFO:tasks.rados.rados.0.smithi163.stdout:597: finishing write tid 1 to smithi16367185-30 2023-10-09T09:07:52.330 INFO:tasks.rados.rados.0.smithi163.stdout:597: finishing write tid 2 to smithi16367185-30 2023-10-09T09:07:52.330 INFO:tasks.rados.rados.0.smithi163.stdout:597: finishing write tid 3 to smithi16367185-30 2023-10-09T09:07:52.330 INFO:tasks.rados.rados.0.smithi163.stdout:597: finishing write tid 4 to smithi16367185-30 2023-10-09T09:07:52.330 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 202 (ObjNum 259 snap 79 seq_num 259) dirty exists 2023-10-09T09:07:52.330 INFO:tasks.rados.rados.0.smithi163.stdout:597: left oid 30 (ObjNum 259 snap 79 seq_num 259) 2023-10-09T09:07:52.330 INFO:tasks.rados.rados.0.smithi163.stdout:598: expect (ObjNum 125 snap 23 seq_num 125) 2023-10-09T09:07:52.404 INFO:tasks.rados.rados.0.smithi163.stdout:599: finishing write tid 1 to smithi16367185-16 2023-10-09T09:07:52.404 INFO:tasks.rados.rados.0.smithi163.stdout:599: finishing write tid 2 to smithi16367185-16 2023-10-09T09:07:52.404 INFO:tasks.rados.rados.0.smithi163.stdout:599: finishing write tid 3 to smithi16367185-16 2023-10-09T09:07:52.404 INFO:tasks.rados.rados.0.smithi163.stdout:599: finishing write tid 4 to smithi16367185-16 2023-10-09T09:07:52.404 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 297 (ObjNum 260 snap 79 seq_num 260) dirty exists 2023-10-09T09:07:52.404 INFO:tasks.rados.rados.0.smithi163.stdout:599: left oid 16 (ObjNum 260 snap 79 seq_num 260) 2023-10-09T09:07:52.404 INFO:tasks.rados.rados.0.smithi163.stdout:597: done (3 left) 2023-10-09T09:07:52.405 INFO:tasks.rados.rados.0.smithi163.stdout:598: done (2 left) 2023-10-09T09:07:52.405 INFO:tasks.rados.rados.0.smithi163.stdout:599: done (1 left) 2023-10-09T09:07:52.405 INFO:tasks.rados.rados.0.smithi163.stdout:600: done (0 left) 2023-10-09T09:07:52.405 INFO:tasks.rados.rados.0.smithi163.stdout:601: snap_remove snap 76 2023-10-09T09:07:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:52 smithi163 bash[33912]: cluster 2023-10-09T09:07:51.174582+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2023-10-09T09:07:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:52 smithi163 bash[33912]: audit 2023-10-09T09:07:51.793651+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.163:0/641590211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:52 smithi163 bash[33912]: cluster 2023-10-09T09:07:52.170091+0000 mon.a (mon.0) 1022 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2023-10-09T09:07:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:52 smithi163 bash[27500]: cluster 2023-10-09T09:07:51.174582+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2023-10-09T09:07:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:52 smithi163 bash[27500]: audit 2023-10-09T09:07:51.793651+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.163:0/641590211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:52 smithi163 bash[27500]: cluster 2023-10-09T09:07:52.170091+0000 mon.a (mon.0) 1022 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2023-10-09T09:07:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:52 smithi181 bash[27712]: cluster 2023-10-09T09:07:51.174582+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2023-10-09T09:07:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:52 smithi181 bash[27712]: audit 2023-10-09T09:07:51.793651+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.163:0/641590211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:52 smithi181 bash[27712]: cluster 2023-10-09T09:07:52.170091+0000 mon.a (mon.0) 1022 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2023-10-09T09:07:53.195 INFO:tasks.rados.rados.0.smithi163.stdout:601: done (0 left) 2023-10-09T09:07:53.195 INFO:tasks.rados.rados.0.smithi163.stdout:602: snap_create 2023-10-09T09:07:53.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:53 smithi163 bash[27500]: cluster 2023-10-09T09:07:52.153868+0000 mgr.x (mgr.24751) 58 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 279 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2023-10-09T09:07:53.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:53 smithi163 bash[33912]: cluster 2023-10-09T09:07:52.153868+0000 mgr.x (mgr.24751) 58 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 279 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2023-10-09T09:07:53.584 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:53 smithi181 bash[27712]: cluster 2023-10-09T09:07:52.153868+0000 mgr.x (mgr.24751) 58 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 279 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2023-10-09T09:07:54.202 INFO:tasks.rados.rados.0.smithi163.stdout:602: done (0 left) 2023-10-09T09:07:54.202 INFO:tasks.rados.rados.0.smithi163.stdout:603: write oid 46 current snap is 80 2023-10-09T09:07:54.202 INFO:tasks.rados.rados.0.smithi163.stdout:603: seq_num 262 ranges {403434=471240,1315341=400756} 2023-10-09T09:07:54.208 INFO:tasks.rados.rados.0.smithi163.stdout:603: writing smithi16367185-46 from 403434 to 874674 tid 1 2023-10-09T09:07:54.213 INFO:tasks.rados.rados.0.smithi163.stdout:603: writing smithi16367185-46 from 1315341 to 1716097 tid 2 2023-10-09T09:07:54.213 INFO:tasks.rados.rados.0.smithi163.stdout:604: read oid 50 snap -1 2023-10-09T09:07:54.214 INFO:tasks.rados.rados.0.smithi163.stdout:604: expect (ObjNum 226 snap 64 seq_num 226) 2023-10-09T09:07:54.214 INFO:tasks.rados.rados.0.smithi163.stdout:605: delete oid 4 current snap is 80 2023-10-09T09:07:54.214 INFO:tasks.rados.rados.0.smithi163.stdout:603: finishing write tid 1 to smithi16367185-46 2023-10-09T09:07:54.217 INFO:tasks.rados.rados.0.smithi163.stdout:603: finishing write tid 2 to smithi16367185-46 2023-10-09T09:07:54.218 INFO:tasks.rados.rados.0.smithi163.stdout:603: finishing write tid 3 to smithi16367185-46 2023-10-09T09:07:54.218 INFO:tasks.rados.rados.0.smithi163.stdout:605: done (2 left) 2023-10-09T09:07:54.218 INFO:tasks.rados.rados.0.smithi163.stdout:606: write oid 27 current snap is 80 2023-10-09T09:07:54.218 INFO:tasks.rados.rados.0.smithi163.stdout:606: seq_num 264 ranges {437048=621738,1832305=404425} 2023-10-09T09:07:54.230 INFO:tasks.rados.rados.0.smithi163.stdout:606: writing smithi16367185-27 from 437048 to 1058786 tid 1 2023-10-09T09:07:54.241 INFO:tasks.rados.rados.0.smithi163.stdout:606: writing smithi16367185-27 from 1832305 to 2236730 tid 2 2023-10-09T09:07:54.241 INFO:tasks.rados.rados.0.smithi163.stdout:607: write oid 18 current snap is 80 2023-10-09T09:07:54.241 INFO:tasks.rados.rados.0.smithi163.stdout:607: seq_num 265 ranges {453855=696987,1890787=548083} 2023-10-09T09:07:54.247 INFO:tasks.rados.rados.0.smithi163.stdout:607: writing smithi16367185-18 from 453855 to 1150842 tid 1 2023-10-09T09:07:54.255 INFO:tasks.rados.rados.0.smithi163.stdout:607: writing smithi16367185-18 from 1890787 to 2438870 tid 2 2023-10-09T09:07:54.255 INFO:tasks.rados.rados.0.smithi163.stdout:608: write oid 44 current snap is 80 2023-10-09T09:07:54.255 INFO:tasks.rados.rados.0.smithi163.stdout:603: finishing write tid 4 to smithi16367185-46 2023-10-09T09:07:54.255 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 249 (ObjNum 262 snap 80 seq_num 262) dirty exists 2023-10-09T09:07:54.256 INFO:tasks.rados.rados.0.smithi163.stdout:603: left oid 46 (ObjNum 262 snap 80 seq_num 262) 2023-10-09T09:07:54.256 INFO:tasks.rados.rados.0.smithi163.stdout:608: seq_num 266 ranges {470662=772236,1832916=408094} 2023-10-09T09:07:54.262 INFO:tasks.rados.rados.0.smithi163.stdout:608: writing smithi16367185-44 from 470662 to 1242898 tid 1 2023-10-09T09:07:54.268 INFO:tasks.rados.rados.0.smithi163.stdout:608: writing smithi16367185-44 from 1832916 to 2241010 tid 2 2023-10-09T09:07:54.268 INFO:tasks.rados.rados.0.smithi163.stdout:603: done (4 left) 2023-10-09T09:07:54.268 INFO:tasks.rados.rados.0.smithi163.stdout:609: snap_create 2023-10-09T09:07:54.269 INFO:tasks.rados.rados.0.smithi163.stdout:604: expect (ObjNum 226 snap 64 seq_num 226) 2023-10-09T09:07:54.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:54 smithi163 bash[33912]: cluster 2023-10-09T09:07:53.195565+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2023-10-09T09:07:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:54 smithi163 bash[27500]: cluster 2023-10-09T09:07:53.195565+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2023-10-09T09:07:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:54 smithi181 bash[27712]: cluster 2023-10-09T09:07:53.195565+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2023-10-09T09:07:54.567 INFO:tasks.rados.rados.0.smithi163.stdout:606: finishing write tid 1 to smithi16367185-27 2023-10-09T09:07:54.568 INFO:tasks.rados.rados.0.smithi163.stdout:606: finishing write tid 2 to smithi16367185-27 2023-10-09T09:07:54.568 INFO:tasks.rados.rados.0.smithi163.stdout:606: finishing write tid 3 to smithi16367185-27 2023-10-09T09:07:54.568 INFO:tasks.rados.rados.0.smithi163.stdout:606: finishing write tid 4 to smithi16367185-27 2023-10-09T09:07:54.568 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 182 (ObjNum 264 snap 80 seq_num 264) dirty exists 2023-10-09T09:07:54.568 INFO:tasks.rados.rados.0.smithi163.stdout:606: left oid 27 (ObjNum 264 snap 80 seq_num 264) 2023-10-09T09:07:54.568 INFO:tasks.rados.rados.0.smithi163.stdout:607: finishing write tid 1 to smithi16367185-18 2023-10-09T09:07:54.568 INFO:tasks.rados.rados.0.smithi163.stdout:607: finishing write tid 2 to smithi16367185-18 2023-10-09T09:07:54.568 INFO:tasks.rados.rados.0.smithi163.stdout:607: finishing write tid 3 to smithi16367185-18 2023-10-09T09:07:54.568 INFO:tasks.rados.rados.0.smithi163.stdout:607: finishing write tid 4 to smithi16367185-18 2023-10-09T09:07:54.568 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 208 (ObjNum 265 snap 80 seq_num 265) dirty exists 2023-10-09T09:07:54.569 INFO:tasks.rados.rados.0.smithi163.stdout:607: left oid 18 (ObjNum 265 snap 80 seq_num 265) 2023-10-09T09:07:54.569 INFO:tasks.rados.rados.0.smithi163.stdout:608: finishing write tid 1 to smithi16367185-44 2023-10-09T09:07:54.569 INFO:tasks.rados.rados.0.smithi163.stdout:608: finishing write tid 2 to smithi16367185-44 2023-10-09T09:07:54.569 INFO:tasks.rados.rados.0.smithi163.stdout:608: finishing write tid 3 to smithi16367185-44 2023-10-09T09:07:54.569 INFO:tasks.rados.rados.0.smithi163.stdout:608: finishing write tid 4 to smithi16367185-44 2023-10-09T09:07:54.569 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 478 (ObjNum 266 snap 80 seq_num 266) dirty exists 2023-10-09T09:07:54.569 INFO:tasks.rados.rados.0.smithi163.stdout:608: left oid 44 (ObjNum 266 snap 80 seq_num 266) 2023-10-09T09:07:55.207 INFO:tasks.rados.rados.0.smithi163.stdout:604: done (4 left) 2023-10-09T09:07:55.207 INFO:tasks.rados.rados.0.smithi163.stdout:606: done (3 left) 2023-10-09T09:07:55.208 INFO:tasks.rados.rados.0.smithi163.stdout:607: done (2 left) 2023-10-09T09:07:55.208 INFO:tasks.rados.rados.0.smithi163.stdout:608: done (1 left) 2023-10-09T09:07:55.208 INFO:tasks.rados.rados.0.smithi163.stdout:609: done (0 left) 2023-10-09T09:07:55.208 INFO:tasks.rados.rados.0.smithi163.stdout:610: write oid 25 current snap is 81 2023-10-09T09:07:55.208 INFO:tasks.rados.rados.0.smithi163.stdout:610: seq_num 267 ranges {487469=563838,1607751=551752} 2023-10-09T09:07:55.213 INFO:tasks.rados.rados.0.smithi163.stdout:610: writing smithi16367185-25 from 487469 to 1051307 tid 1 2023-10-09T09:07:55.220 INFO:tasks.rados.rados.0.smithi163.stdout:610: writing smithi16367185-25 from 1607751 to 2159503 tid 2 2023-10-09T09:07:55.220 INFO:tasks.rados.rados.0.smithi163.stdout:611: snap_create 2023-10-09T09:07:55.221 INFO:tasks.rados.rados.0.smithi163.stdout:610: finishing write tid 1 to smithi16367185-25 2023-10-09T09:07:55.224 INFO:tasks.rados.rados.0.smithi163.stdout:610: finishing write tid 2 to smithi16367185-25 2023-10-09T09:07:55.224 INFO:tasks.rados.rados.0.smithi163.stdout:610: finishing write tid 3 to smithi16367185-25 2023-10-09T09:07:55.224 INFO:tasks.rados.rados.0.smithi163.stdout:610: finishing write tid 4 to smithi16367185-25 2023-10-09T09:07:55.224 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 277 (ObjNum 267 snap 81 seq_num 267) dirty exists 2023-10-09T09:07:55.224 INFO:tasks.rados.rados.0.smithi163.stdout:610: left oid 25 (ObjNum 267 snap 81 seq_num 267) 2023-10-09T09:07:55.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:55 smithi163 bash[33912]: cluster 2023-10-09T09:07:54.154395+0000 mgr.x (mgr.24751) 59 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 279 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2023-10-09T09:07:55.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:55 smithi163 bash[33912]: cluster 2023-10-09T09:07:54.202322+0000 mon.a (mon.0) 1024 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2023-10-09T09:07:55.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:55 smithi163 bash[33912]: audit 2023-10-09T09:07:54.253553+0000 mon.a (mon.0) 1025 : audit [DBG] from='client.? 172.21.15.163:0/3446324530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:55.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:55 smithi163 bash[27500]: cluster 2023-10-09T09:07:54.154395+0000 mgr.x (mgr.24751) 59 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 279 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2023-10-09T09:07:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:55 smithi163 bash[27500]: cluster 2023-10-09T09:07:54.202322+0000 mon.a (mon.0) 1024 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2023-10-09T09:07:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:55 smithi163 bash[27500]: audit 2023-10-09T09:07:54.253553+0000 mon.a (mon.0) 1025 : audit [DBG] from='client.? 172.21.15.163:0/3446324530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:55 smithi181 bash[27712]: cluster 2023-10-09T09:07:54.154395+0000 mgr.x (mgr.24751) 59 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 279 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2023-10-09T09:07:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:55 smithi181 bash[27712]: cluster 2023-10-09T09:07:54.202322+0000 mon.a (mon.0) 1024 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2023-10-09T09:07:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:55 smithi181 bash[27712]: audit 2023-10-09T09:07:54.253553+0000 mon.a (mon.0) 1025 : audit [DBG] from='client.? 172.21.15.163:0/3446324530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:56.231 INFO:tasks.rados.rados.0.smithi163.stdout:610: done (1 left) 2023-10-09T09:07:56.231 INFO:tasks.rados.rados.0.smithi163.stdout:611: done (0 left) 2023-10-09T09:07:56.231 INFO:tasks.rados.rados.0.smithi163.stdout:612: delete oid 25 current snap is 82 2023-10-09T09:07:56.237 INFO:tasks.rados.rados.0.smithi163.stdout:612: done (0 left) 2023-10-09T09:07:56.237 INFO:tasks.rados.rados.0.smithi163.stdout:613: snap_remove snap 79 2023-10-09T09:07:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:56 smithi163 bash[27500]: cluster 2023-10-09T09:07:55.207399+0000 mon.a (mon.0) 1026 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2023-10-09T09:07:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:56 smithi163 bash[33912]: cluster 2023-10-09T09:07:55.207399+0000 mon.a (mon.0) 1026 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2023-10-09T09:07:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:56 smithi181 bash[27712]: cluster 2023-10-09T09:07:55.207399+0000 mon.a (mon.0) 1026 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2023-10-09T09:07:57.139 INFO:tasks.rados.rados.0.smithi163.stdout:613: done (0 left) 2023-10-09T09:07:57.139 INFO:tasks.rados.rados.0.smithi163.stdout:614: snap_create 2023-10-09T09:07:57.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:57 smithi181 bash[27712]: cluster 2023-10-09T09:07:56.155024+0000 mgr.x (mgr.24751) 60 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 281 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 288 KiB/s wr, 5 op/s 2023-10-09T09:07:57.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:57 smithi181 bash[27712]: cluster 2023-10-09T09:07:56.231154+0000 mon.a (mon.0) 1027 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2023-10-09T09:07:57.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:57 smithi181 bash[27712]: audit 2023-10-09T09:07:56.708832+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.163:0/3018950438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:57.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:57 smithi181 bash[27712]: cluster 2023-10-09T09:07:57.138956+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2023-10-09T09:07:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:57 smithi163 bash[33912]: cluster 2023-10-09T09:07:56.155024+0000 mgr.x (mgr.24751) 60 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 281 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 288 KiB/s wr, 5 op/s 2023-10-09T09:07:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:57 smithi163 bash[33912]: cluster 2023-10-09T09:07:56.231154+0000 mon.a (mon.0) 1027 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2023-10-09T09:07:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:57 smithi163 bash[33912]: audit 2023-10-09T09:07:56.708832+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.163:0/3018950438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:57 smithi163 bash[33912]: cluster 2023-10-09T09:07:57.138956+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2023-10-09T09:07:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:57 smithi163 bash[27500]: cluster 2023-10-09T09:07:56.155024+0000 mgr.x (mgr.24751) 60 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 281 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 288 KiB/s wr, 5 op/s 2023-10-09T09:07:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:57 smithi163 bash[27500]: cluster 2023-10-09T09:07:56.231154+0000 mon.a (mon.0) 1027 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2023-10-09T09:07:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:57 smithi163 bash[27500]: audit 2023-10-09T09:07:56.708832+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.163:0/3018950438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:07:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:57 smithi163 bash[27500]: cluster 2023-10-09T09:07:57.138956+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2023-10-09T09:07:57.860 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:07:57 smithi181 bash[50540]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:07:57] "GET /metrics HTTP/1.1" 200 36084 "" "Prometheus/2.33.4" 2023-10-09T09:07:58.138 INFO:tasks.rados.rados.0.smithi163.stdout:614: done (0 left) 2023-10-09T09:07:58.138 INFO:tasks.rados.rados.0.smithi163.stdout:615: snap_create 2023-10-09T09:07:59.068 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:07:58 smithi181 bash[52947]: ts=2023-10-09T09:07:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:07:59.145 INFO:tasks.rados.rados.0.smithi163.stdout:615: done (0 left) 2023-10-09T09:07:59.145 INFO:tasks.rados.rados.0.smithi163.stdout:616: write oid 9 current snap is 84 2023-10-09T09:07:59.145 INFO:tasks.rados.rados.0.smithi163.stdout:616: seq_num 269 ranges {521083=714336,1724715=555421} 2023-10-09T09:07:59.154 INFO:tasks.rados.rados.0.smithi163.stdout:616: writing smithi16367185-9 from 521083 to 1235419 tid 1 2023-10-09T09:07:59.160 INFO:tasks.rados.rados.0.smithi163.stdout:616: writing smithi16367185-9 from 1724715 to 2280136 tid 2 2023-10-09T09:07:59.161 INFO:tasks.rados.rados.0.smithi163.stdout:617: write oid 15 current snap is 84 2023-10-09T09:07:59.161 INFO:tasks.rados.rados.0.smithi163.stdout:617: seq_num 270 ranges {537890=789585,1783197=699079} 2023-10-09T09:07:59.168 INFO:tasks.rados.rados.0.smithi163.stdout:617: writing smithi16367185-15 from 537890 to 1327475 tid 1 2023-10-09T09:07:59.175 INFO:tasks.rados.rados.0.smithi163.stdout:617: writing smithi16367185-15 from 1783197 to 2482276 tid 2 2023-10-09T09:07:59.175 INFO:tasks.rados.rados.0.smithi163.stdout:616: finishing write tid 1 to smithi16367185-9 2023-10-09T09:07:59.175 INFO:tasks.rados.rados.0.smithi163.stdout:616: finishing write tid 2 to smithi16367185-9 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:616: finishing write tid 3 to smithi16367185-9 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:616: finishing write tid 4 to smithi16367185-9 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 153 (ObjNum 269 snap 84 seq_num 269) dirty exists 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:616: left oid 9 (ObjNum 269 snap 84 seq_num 269) 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:616: done (1 left) 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:618: read oid 9 snap -1 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:618: expect (ObjNum 269 snap 84 seq_num 269) 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:619: read oid 50 snap 51 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:619: expect (ObjNum 94 snap 16 seq_num 94) 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:620: read oid 48 snap -1 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:620: expect (ObjNum 128 snap 27 seq_num 128) 2023-10-09T09:07:59.176 INFO:tasks.rados.rados.0.smithi163.stdout:621: write oid 43 current snap is 84 2023-10-09T09:07:59.177 INFO:tasks.rados.rados.0.smithi163.stdout:621: seq_num 271 ranges {554697=464834,1725326=559090} 2023-10-09T09:07:59.181 INFO:tasks.rados.rados.0.smithi163.stdout:621: writing smithi16367185-43 from 554697 to 1019531 tid 1 2023-10-09T09:07:59.189 INFO:tasks.rados.rados.0.smithi163.stdout:621: writing smithi16367185-43 from 1725326 to 2284416 tid 2 2023-10-09T09:07:59.189 INFO:tasks.rados.rados.0.smithi163.stdout:622: read oid 12 snap 31 2023-10-09T09:07:59.189 INFO:tasks.rados.rados.0.smithi163.stdout:622: expect (ObjNum 68 snap 7 seq_num 68) 2023-10-09T09:07:59.189 INFO:tasks.rados.rados.0.smithi163.stdout:623: snap_remove snap 82 2023-10-09T09:07:59.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:59 smithi163 bash[27500]: cluster 2023-10-09T09:07:58.137617+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2023-10-09T09:07:59.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:07:59 smithi163 bash[27500]: cluster 2023-10-09T09:07:58.155629+0000 mgr.x (mgr.24751) 61 : cluster [DBG] pgmap v105: 89 pgs: 89 active+clean; 290 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2023-10-09T09:07:59.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:59 smithi163 bash[33912]: cluster 2023-10-09T09:07:58.137617+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2023-10-09T09:07:59.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:07:59 smithi163 bash[33912]: cluster 2023-10-09T09:07:58.155629+0000 mgr.x (mgr.24751) 61 : cluster [DBG] pgmap v105: 89 pgs: 89 active+clean; 290 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2023-10-09T09:07:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:59 smithi181 bash[27712]: cluster 2023-10-09T09:07:58.137617+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2023-10-09T09:07:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:07:59 smithi181 bash[27712]: cluster 2023-10-09T09:07:58.155629+0000 mgr.x (mgr.24751) 61 : cluster [DBG] pgmap v105: 89 pgs: 89 active+clean; 290 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2023-10-09T09:08:00.155 INFO:tasks.rados.rados.0.smithi163.stdout:623: done (6 left) 2023-10-09T09:08:00.155 INFO:tasks.rados.rados.0.smithi163.stdout:624: snap_remove snap 36 2023-10-09T09:08:00.155 INFO:tasks.rados.rados.0.smithi163.stdout:617: finishing write tid 1 to smithi16367185-15 2023-10-09T09:08:00.155 INFO:tasks.rados.rados.0.smithi163.stdout:617: finishing write tid 2 to smithi16367185-15 2023-10-09T09:08:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:617: finishing write tid 3 to smithi16367185-15 2023-10-09T09:08:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:617: finishing write tid 4 to smithi16367185-15 2023-10-09T09:08:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 296 (ObjNum 270 snap 84 seq_num 270) dirty exists 2023-10-09T09:08:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:617: left oid 15 (ObjNum 270 snap 84 seq_num 270) 2023-10-09T09:08:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:618: expect (ObjNum 269 snap 84 seq_num 269) 2023-10-09T09:08:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:00 smithi181 bash[27712]: cluster 2023-10-09T09:07:59.143608+0000 mon.a (mon.0) 1030 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2023-10-09T09:08:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:00 smithi181 bash[27712]: audit 2023-10-09T09:07:59.157012+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.163:0/721512918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:00.400 INFO:tasks.rados.rados.0.smithi163.stdout:621: finishing write tid 1 to smithi16367185-43 2023-10-09T09:08:00.400 INFO:tasks.rados.rados.0.smithi163.stdout:621: finishing write tid 2 to smithi16367185-43 2023-10-09T09:08:00.400 INFO:tasks.rados.rados.0.smithi163.stdout:621: finishing write tid 3 to smithi16367185-43 2023-10-09T09:08:00.400 INFO:tasks.rados.rados.0.smithi163.stdout:621: finishing write tid 4 to smithi16367185-43 2023-10-09T09:08:00.400 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 255 (ObjNum 271 snap 84 seq_num 271) dirty exists 2023-10-09T09:08:00.401 INFO:tasks.rados.rados.0.smithi163.stdout:621: left oid 43 (ObjNum 271 snap 84 seq_num 271) 2023-10-09T09:08:00.401 INFO:tasks.rados.rados.0.smithi163.stdout:619: expect (ObjNum 94 snap 16 seq_num 94) 2023-10-09T09:08:00.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:00 smithi163 bash[27500]: cluster 2023-10-09T09:07:59.143608+0000 mon.a (mon.0) 1030 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2023-10-09T09:08:00.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:00 smithi163 bash[27500]: audit 2023-10-09T09:07:59.157012+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.163:0/721512918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:00 smithi163 bash[33912]: cluster 2023-10-09T09:07:59.143608+0000 mon.a (mon.0) 1030 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2023-10-09T09:08:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:00 smithi163 bash[33912]: audit 2023-10-09T09:07:59.157012+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.163:0/721512918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:00.608 INFO:tasks.rados.rados.0.smithi163.stdout:620: expect (ObjNum 128 snap 27 seq_num 128) 2023-10-09T09:08:00.727 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:00 smithi181 systemd[1]: Stopping Ceph grafana.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:08:00.727 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:00 smithi181 bash[53405]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-grafana.a 2023-10-09T09:08:00.727 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:00 smithi181 bash[47587]: t=2023-10-09T09:08:00+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T09:08:00.761 INFO:tasks.rados.rados.0.smithi163.stdout:622: expect (ObjNum 68 snap 7 seq_num 68) 2023-10-09T09:08:01.164 INFO:tasks.rados.rados.0.smithi163.stdout:617: done (6 left) 2023-10-09T09:08:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:618: done (5 left) 2023-10-09T09:08:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:619: done (4 left) 2023-10-09T09:08:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:620: done (3 left) 2023-10-09T09:08:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:621: done (2 left) 2023-10-09T09:08:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:622: done (1 left) 2023-10-09T09:08:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:624: done (0 left) 2023-10-09T09:08:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:625: rollback oid 34 current snap is 84 2023-10-09T09:08:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 34 to 75 2023-10-09T09:08:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:626: snap_remove snap 68 2023-10-09T09:08:01.453 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:01 smithi181 bash[27712]: cluster 2023-10-09T09:08:00.155200+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2023-10-09T09:08:01.453 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:01 smithi181 bash[27712]: cluster 2023-10-09T09:08:00.156229+0000 mgr.x (mgr.24751) 62 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 290 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:01.453 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:01 smithi181 bash[53441]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T09:08:01.454 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:01 smithi181 bash[53505]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T09:08:01.454 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:01 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T09:08:01.454 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:01 smithi181 systemd[1]: Stopped Ceph grafana.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:08:01.454 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:01 smithi181 systemd[1]: Started Ceph grafana.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:08:01.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:01 smithi163 bash[33912]: cluster 2023-10-09T09:08:00.155200+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2023-10-09T09:08:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:01 smithi163 bash[33912]: cluster 2023-10-09T09:08:00.156229+0000 mgr.x (mgr.24751) 62 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 290 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:01 smithi163 bash[27500]: cluster 2023-10-09T09:08:00.155200+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2023-10-09T09:08:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:01 smithi163 bash[27500]: cluster 2023-10-09T09:08:00.156229+0000 mgr.x (mgr.24751) 62 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 290 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:02.171 INFO:tasks.rados.rados.0.smithi163.stdout:626: done (1 left) 2023-10-09T09:08:02.171 INFO:tasks.rados.rados.0.smithi163.stdout:627: read oid 3 snap -1 2023-10-09T09:08:02.171 INFO:tasks.rados.rados.0.smithi163.stdout:627: expect deleted 2023-10-09T09:08:02.171 INFO:tasks.rados.rados.0.smithi163.stdout:628: write oid 13 current snap is 84 2023-10-09T09:08:02.171 INFO:tasks.rados.rados.0.smithi163.stdout:625: finishing rollback tid 0 to smithi16367185-34 2023-10-09T09:08:02.172 INFO:tasks.rados.rados.0.smithi163.stdout:628: seq_num 272 ranges {571504=540083,1783808=702748} 2023-10-09T09:08:02.177 INFO:tasks.rados.rados.0.smithi163.stdout:628: writing smithi16367185-13 from 571504 to 1111587 tid 1 2023-10-09T09:08:02.186 INFO:tasks.rados.rados.0.smithi163.stdout:628: writing smithi16367185-13 from 1783808 to 2486556 tid 2 2023-10-09T09:08:02.186 INFO:tasks.rados.rados.0.smithi163.stdout:629: snap_remove snap 78 2023-10-09T09:08:02.429 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: cluster 2023-10-09T09:08:01.164171+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2023-10-09T09:08:02.429 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: audit 2023-10-09T09:08:01.426226+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:02.429 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: audit 2023-10-09T09:08:01.434357+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:02.429 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: audit 2023-10-09T09:08:01.465460+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:08:02.429 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: audit 2023-10-09T09:08:01.466409+0000 mgr.x (mgr.24751) 63 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:08:02.429 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: audit 2023-10-09T09:08:01.472359+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:02.430 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: audit 2023-10-09T09:08:01.472894+0000 mgr.x (mgr.24751) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:02.430 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: audit 2023-10-09T09:08:01.477426+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:08:02.430 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: audit 2023-10-09T09:08:01.477945+0000 mgr.x (mgr.24751) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:08:02.430 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: audit 2023-10-09T09:08:01.485357+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:02.430 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[27712]: audit 2023-10-09T09:08:01.584951+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.163:0/2012270699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: cluster 2023-10-09T09:08:01.164171+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2023-10-09T09:08:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: audit 2023-10-09T09:08:01.426226+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: audit 2023-10-09T09:08:01.434357+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: audit 2023-10-09T09:08:01.465460+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:08:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: audit 2023-10-09T09:08:01.466409+0000 mgr.x (mgr.24751) 63 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:08:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: audit 2023-10-09T09:08:01.472359+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: audit 2023-10-09T09:08:01.472894+0000 mgr.x (mgr.24751) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: audit 2023-10-09T09:08:01.477426+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:08:02.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: audit 2023-10-09T09:08:01.477945+0000 mgr.x (mgr.24751) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:08:02.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: audit 2023-10-09T09:08:01.485357+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:02.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[33912]: audit 2023-10-09T09:08:01.584951+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.163:0/2012270699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: cluster 2023-10-09T09:08:01.164171+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2023-10-09T09:08:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: audit 2023-10-09T09:08:01.426226+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: audit 2023-10-09T09:08:01.434357+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: audit 2023-10-09T09:08:01.465460+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:08:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: audit 2023-10-09T09:08:01.466409+0000 mgr.x (mgr.24751) 63 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:08:02.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: audit 2023-10-09T09:08:01.472359+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:02.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: audit 2023-10-09T09:08:01.472894+0000 mgr.x (mgr.24751) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:02.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: audit 2023-10-09T09:08:01.477426+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:08:02.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: audit 2023-10-09T09:08:01.477945+0000 mgr.x (mgr.24751) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:08:02.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: audit 2023-10-09T09:08:01.485357+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:02.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:02 smithi163 bash[27500]: audit 2023-10-09T09:08:01.584951+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.163:0/2012270699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:02.697 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T09:08:02.697 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T09:08:02.697 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T09:08:02.698 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T09:08:02.698 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T09:08:02.698 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T09:08:02.698 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T09:08:02.698 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T09:08:02.698 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T09:08:02.698 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T09:08:02.698 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T09:08:02.698 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T09:08:02.698 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T09:08:02.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T09:08:02.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T09:08:02.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T09:08:02.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T09:08:02.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=593.675µs 2023-10-09T09:08:02.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T09:08:02.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T09:08:02.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T09:08:02.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T09:08:02.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T09:08:02.700 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T09:08:02.700 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-10-09T09:08:02.700 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T09:08:02.700 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-10-09T09:08:02.700 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T09:08:03.067 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T09:08:03.067 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 09:08:02 smithi181 bash[53532]: t=2023-10-09T09:08:02+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T09:08:03.193 INFO:tasks.rados.rados.0.smithi163.stdout:629: done (3 left) 2023-10-09T09:08:03.193 INFO:tasks.rados.rados.0.smithi163.stdout:630: write oid 35 current snap is 84 2023-10-09T09:08:03.193 INFO:tasks.rados.rados.0.smithi163.stdout:630: seq_num 273 ranges {588311=615332,1842290=562759} 2023-10-09T09:08:03.202 INFO:tasks.rados.rados.0.smithi163.stdout:630: writing smithi16367185-35 from 588311 to 1203643 tid 1 2023-10-09T09:08:03.210 INFO:tasks.rados.rados.0.smithi163.stdout:630: writing smithi16367185-35 from 1842290 to 2405049 tid 2 2023-10-09T09:08:03.210 INFO:tasks.rados.rados.0.smithi163.stdout:631: delete oid 1 current snap is 84 2023-10-09T09:08:03.210 INFO:tasks.rados.rados.0.smithi163.stdout:625: finishing rollback tid 1 to smithi16367185-34 2023-10-09T09:08:03.210 INFO:tasks.rados.rados.0.smithi163.stdout:625: finishing rollback tid 2 to smithi16367185-34 2023-10-09T09:08:03.210 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 201 (ObjNum 231 snap 67 seq_num 231) dirty exists 2023-10-09T09:08:03.210 INFO:tasks.rados.rados.0.smithi163.stdout:628: finishing write tid 1 to smithi16367185-13 2023-10-09T09:08:03.211 INFO:tasks.rados.rados.0.smithi163.stdout:628: finishing write tid 2 to smithi16367185-13 2023-10-09T09:08:03.211 INFO:tasks.rados.rados.0.smithi163.stdout:628: finishing write tid 3 to smithi16367185-13 2023-10-09T09:08:03.211 INFO:tasks.rados.rados.0.smithi163.stdout:628: finishing write tid 4 to smithi16367185-13 2023-10-09T09:08:03.211 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 67 (ObjNum 272 snap 84 seq_num 272) dirty exists 2023-10-09T09:08:03.211 INFO:tasks.rados.rados.0.smithi163.stdout:628: left oid 13 (ObjNum 272 snap 84 seq_num 272) 2023-10-09T09:08:03.211 INFO:tasks.rados.rados.0.smithi163.stdout:630: finishing write tid 1 to smithi16367185-35 2023-10-09T09:08:03.212 INFO:tasks.rados.rados.0.smithi163.stdout:625: done (4 left) 2023-10-09T09:08:03.212 INFO:tasks.rados.rados.0.smithi163.stdout:627: done (3 left) 2023-10-09T09:08:03.212 INFO:tasks.rados.rados.0.smithi163.stdout:628: done (2 left) 2023-10-09T09:08:03.212 INFO:tasks.rados.rados.0.smithi163.stdout:631: done (1 left) 2023-10-09T09:08:03.212 INFO:tasks.rados.rados.0.smithi163.stdout:632: snap_create 2023-10-09T09:08:03.213 INFO:tasks.rados.rados.0.smithi163.stdout:630: finishing write tid 2 to smithi16367185-35 2023-10-09T09:08:03.213 INFO:tasks.rados.rados.0.smithi163.stdout:630: finishing write tid 3 to smithi16367185-35 2023-10-09T09:08:03.214 INFO:tasks.rados.rados.0.smithi163.stdout:630: finishing write tid 4 to smithi16367185-35 2023-10-09T09:08:03.214 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 112 (ObjNum 273 snap 84 seq_num 273) dirty exists 2023-10-09T09:08:03.214 INFO:tasks.rados.rados.0.smithi163.stdout:630: left oid 35 (ObjNum 273 snap 84 seq_num 273) 2023-10-09T09:08:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:03 smithi163 bash[27500]: cephadm 2023-10-09T09:08:01.488589+0000 mgr.x (mgr.24751) 66 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:08:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:03 smithi163 bash[27500]: cephadm 2023-10-09T09:08:01.489177+0000 mgr.x (mgr.24751) 67 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:08:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:03 smithi163 bash[27500]: cluster 2023-10-09T09:08:02.157494+0000 mgr.x (mgr.24751) 68 : cluster [DBG] pgmap v110: 89 pgs: 89 active+clean; 293 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 918 KiB/s wr, 12 op/s 2023-10-09T09:08:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:03 smithi163 bash[27500]: cluster 2023-10-09T09:08:02.169713+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2023-10-09T09:08:03.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:03 smithi163 bash[33912]: cephadm 2023-10-09T09:08:01.488589+0000 mgr.x (mgr.24751) 66 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:08:03.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:03 smithi163 bash[33912]: cephadm 2023-10-09T09:08:01.489177+0000 mgr.x (mgr.24751) 67 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:08:03.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:03 smithi163 bash[33912]: cluster 2023-10-09T09:08:02.157494+0000 mgr.x (mgr.24751) 68 : cluster [DBG] pgmap v110: 89 pgs: 89 active+clean; 293 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 918 KiB/s wr, 12 op/s 2023-10-09T09:08:03.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:03 smithi163 bash[33912]: cluster 2023-10-09T09:08:02.169713+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2023-10-09T09:08:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:03 smithi181 bash[27712]: cephadm 2023-10-09T09:08:01.488589+0000 mgr.x (mgr.24751) 66 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:08:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:03 smithi181 bash[27712]: cephadm 2023-10-09T09:08:01.489177+0000 mgr.x (mgr.24751) 67 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:08:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:03 smithi181 bash[27712]: cluster 2023-10-09T09:08:02.157494+0000 mgr.x (mgr.24751) 68 : cluster [DBG] pgmap v110: 89 pgs: 89 active+clean; 293 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 918 KiB/s wr, 12 op/s 2023-10-09T09:08:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:03 smithi181 bash[27712]: cluster 2023-10-09T09:08:02.169713+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2023-10-09T09:08:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:630: done (1 left) 2023-10-09T09:08:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:632: done (0 left) 2023-10-09T09:08:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:633: rollback oid 19 current snap is 85 2023-10-09T09:08:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 19 to 60 2023-10-09T09:08:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:634: rollback oid 6 current snap is 85 2023-10-09T09:08:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 6 to 81 2023-10-09T09:08:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:635: snap_remove snap 66 2023-10-09T09:08:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[33912]: cluster 2023-10-09T09:08:03.192643+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2023-10-09T09:08:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[33912]: audit 2023-10-09T09:08:03.769786+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[33912]: audit 2023-10-09T09:08:03.771079+0000 mon.b (mon.1) 88 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:08:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[33912]: audit 2023-10-09T09:08:03.773527+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:08:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[33912]: audit 2023-10-09T09:08:03.773541+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:08:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[33912]: audit 2023-10-09T09:08:03.775109+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[33912]: audit 2023-10-09T09:08:04.037455+0000 mon.a (mon.0) 1039 : audit [DBG] from='client.? 172.21.15.163:0/2878327231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[27500]: cluster 2023-10-09T09:08:03.192643+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2023-10-09T09:08:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[27500]: audit 2023-10-09T09:08:03.769786+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[27500]: audit 2023-10-09T09:08:03.771079+0000 mon.b (mon.1) 88 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:08:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[27500]: audit 2023-10-09T09:08:03.773527+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:08:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[27500]: audit 2023-10-09T09:08:03.773541+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:08:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[27500]: audit 2023-10-09T09:08:03.775109+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:04 smithi163 bash[27500]: audit 2023-10-09T09:08:04.037455+0000 mon.a (mon.0) 1039 : audit [DBG] from='client.? 172.21.15.163:0/2878327231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:04 smithi181 bash[27712]: cluster 2023-10-09T09:08:03.192643+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2023-10-09T09:08:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:04 smithi181 bash[27712]: audit 2023-10-09T09:08:03.769786+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:04 smithi181 bash[27712]: audit 2023-10-09T09:08:03.771079+0000 mon.b (mon.1) 88 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:08:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:04 smithi181 bash[27712]: audit 2023-10-09T09:08:03.773527+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:08:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:04 smithi181 bash[27712]: audit 2023-10-09T09:08:03.773541+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:08:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:04 smithi181 bash[27712]: audit 2023-10-09T09:08:03.775109+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:04 smithi181 bash[27712]: audit 2023-10-09T09:08:04.037455+0000 mon.a (mon.0) 1039 : audit [DBG] from='client.? 172.21.15.163:0/2878327231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:05.211 INFO:tasks.rados.rados.0.smithi163.stdout:635: done (2 left) 2023-10-09T09:08:05.211 INFO:tasks.rados.rados.0.smithi163.stdout:636: read oid 26 snap -1 2023-10-09T09:08:05.211 INFO:tasks.rados.rados.0.smithi163.stdout:636: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:08:05.211 INFO:tasks.rados.rados.0.smithi163.stdout:637: rollback oid 14 current snap is 85 2023-10-09T09:08:05.211 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 71 2023-10-09T09:08:05.211 INFO:tasks.rados.rados.0.smithi163.stdout:634: finishing rollback tid 0 to smithi16367185-6 2023-10-09T09:08:05.211 INFO:tasks.rados.rados.0.smithi163.stdout:634: finishing rollback tid 1 to smithi16367185-6 2023-10-09T09:08:05.211 INFO:tasks.rados.rados.0.smithi163.stdout:634: finishing rollback tid 2 to smithi16367185-6 2023-10-09T09:08:05.211 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 57 (ObjNum 92 snap 15 seq_num 92) dirty exists 2023-10-09T09:08:05.212 INFO:tasks.rados.rados.0.smithi163.stdout:633: finishing rollback tid 1 to smithi16367185-19 2023-10-09T09:08:05.212 INFO:tasks.rados.rados.0.smithi163.stdout:634: done (3 left) 2023-10-09T09:08:05.212 INFO:tasks.rados.rados.0.smithi163.stdout:638: snap_create 2023-10-09T09:08:05.212 INFO:tasks.rados.rados.0.smithi163.stdout:633: finishing rollback tid 2 to smithi16367185-19 2023-10-09T09:08:05.212 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 190 (ObjNum 140 snap 31 seq_num 140) dirty exists 2023-10-09T09:08:05.218 INFO:tasks.rados.rados.0.smithi163.stdout:637: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:08:05.218 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 229 (ObjNum 699226048 snap 21875 seq_num 699226048) dirty dne 2023-10-09T09:08:05.221 INFO:tasks.rados.rados.0.smithi163.stdout:636: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:08:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:05 smithi163 bash[33912]: cephadm 2023-10-09T09:08:03.758942+0000 mgr.x (mgr.24751) 69 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T09:08:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:05 smithi163 bash[33912]: cephadm 2023-10-09T09:08:03.776612+0000 mgr.x (mgr.24751) 70 : cephadm [INF] Deploying daemon mgr.y on smithi163 2023-10-09T09:08:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:05 smithi163 bash[33912]: cluster 2023-10-09T09:08:04.158240+0000 mgr.x (mgr.24751) 71 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 293 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 922 KiB/s wr, 12 op/s 2023-10-09T09:08:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:05 smithi163 bash[33912]: cluster 2023-10-09T09:08:04.201321+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2023-10-09T09:08:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:05 smithi163 bash[27500]: cephadm 2023-10-09T09:08:03.758942+0000 mgr.x (mgr.24751) 69 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T09:08:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:05 smithi163 bash[27500]: cephadm 2023-10-09T09:08:03.776612+0000 mgr.x (mgr.24751) 70 : cephadm [INF] Deploying daemon mgr.y on smithi163 2023-10-09T09:08:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:05 smithi163 bash[27500]: cluster 2023-10-09T09:08:04.158240+0000 mgr.x (mgr.24751) 71 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 293 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 922 KiB/s wr, 12 op/s 2023-10-09T09:08:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:05 smithi163 bash[27500]: cluster 2023-10-09T09:08:04.201321+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2023-10-09T09:08:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:05 smithi181 bash[27712]: cephadm 2023-10-09T09:08:03.758942+0000 mgr.x (mgr.24751) 69 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T09:08:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:05 smithi181 bash[27712]: cephadm 2023-10-09T09:08:03.776612+0000 mgr.x (mgr.24751) 70 : cephadm [INF] Deploying daemon mgr.y on smithi163 2023-10-09T09:08:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:05 smithi181 bash[27712]: cluster 2023-10-09T09:08:04.158240+0000 mgr.x (mgr.24751) 71 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 293 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 922 KiB/s wr, 12 op/s 2023-10-09T09:08:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:05 smithi181 bash[27712]: cluster 2023-10-09T09:08:04.201321+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2023-10-09T09:08:06.172 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:05 smithi163 systemd[1]: Stopping Ceph mgr.y for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:08:06.215 INFO:tasks.rados.rados.0.smithi163.stdout:633: done (3 left) 2023-10-09T09:08:06.215 INFO:tasks.rados.rados.0.smithi163.stdout:636: done (2 left) 2023-10-09T09:08:06.216 INFO:tasks.rados.rados.0.smithi163.stdout:637: done (1 left) 2023-10-09T09:08:06.216 INFO:tasks.rados.rados.0.smithi163.stdout:638: done (0 left) 2023-10-09T09:08:06.216 INFO:tasks.rados.rados.0.smithi163.stdout:639: read oid 18 snap -1 2023-10-09T09:08:06.216 INFO:tasks.rados.rados.0.smithi163.stdout:639: expect (ObjNum 265 snap 80 seq_num 265) 2023-10-09T09:08:06.216 INFO:tasks.rados.rados.0.smithi163.stdout:640: snap_remove snap 83 2023-10-09T09:08:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:06 smithi163 bash[27500]: cluster 2023-10-09T09:08:05.209328+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2023-10-09T09:08:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:06 smithi163 bash[27500]: audit 2023-10-09T09:08:05.409765+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:06.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:06 smithi163 bash[33912]: cluster 2023-10-09T09:08:05.209328+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2023-10-09T09:08:06.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:06 smithi163 bash[33912]: audit 2023-10-09T09:08:05.409765+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:06 smithi181 bash[27712]: cluster 2023-10-09T09:08:05.209328+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2023-10-09T09:08:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:06 smithi181 bash[27712]: audit 2023-10-09T09:08:05.409765+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:06.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:06 smithi163 bash[71454]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T09:08:06.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:06 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:08:06.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:06 smithi163 bash[71542]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T09:08:06.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:06 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T09:08:06.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:06 smithi163 systemd[1]: Stopped Ceph mgr.y for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:08:07.143 INFO:tasks.rados.rados.0.smithi163.stdout:640: done (1 left) 2023-10-09T09:08:07.143 INFO:tasks.rados.rados.0.smithi163.stdout:641: read oid 13 snap -1 2023-10-09T09:08:07.143 INFO:tasks.rados.rados.0.smithi163.stdout:641: expect (ObjNum 272 snap 84 seq_num 272) 2023-10-09T09:08:07.143 INFO:tasks.rados.rados.0.smithi163.stdout:642: delete oid 39 current snap is 86 2023-10-09T09:08:07.143 INFO:tasks.rados.rados.0.smithi163.stdout:639: expect (ObjNum 265 snap 80 seq_num 265) 2023-10-09T09:08:07.291 INFO:tasks.rados.rados.0.smithi163.stdout:641: expect (ObjNum 272 snap 84 seq_num 272) 2023-10-09T09:08:07.438 INFO:tasks.rados.rados.0.smithi163.stdout:639: done (2 left) 2023-10-09T09:08:07.438 INFO:tasks.rados.rados.0.smithi163.stdout:641: done (1 left) 2023-10-09T09:08:07.439 INFO:tasks.rados.rados.0.smithi163.stdout:642: done (0 left) 2023-10-09T09:08:07.439 INFO:tasks.rados.rados.0.smithi163.stdout:643: read oid 40 snap -1 2023-10-09T09:08:07.439 INFO:tasks.rados.rados.0.smithi163.stdout:643: expect (ObjNum 239 snap 73 seq_num 239) 2023-10-09T09:08:07.439 INFO:tasks.rados.rados.0.smithi163.stdout:644: read oid 11 snap -1 2023-10-09T09:08:07.439 INFO:tasks.rados.rados.0.smithi163.stdout:644: expect (ObjNum 148 snap 34 seq_num 148) 2023-10-09T09:08:07.439 INFO:tasks.rados.rados.0.smithi163.stdout:645: snap_remove snap 73 2023-10-09T09:08:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[27500]: cluster 2023-10-09T09:08:06.159094+0000 mgr.x (mgr.24751) 72 : cluster [DBG] pgmap v116: 89 pgs: 89 active+clean; 293 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:08:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[27500]: cluster 2023-10-09T09:08:06.215315+0000 mon.a (mon.0) 1042 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2023-10-09T09:08:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[27500]: audit 2023-10-09T09:08:06.495249+0000 mon.a (mon.0) 1043 : audit [DBG] from='client.? 172.21.15.163:0/59235988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[27500]: audit 2023-10-09T09:08:07.064133+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[27500]: audit 2023-10-09T09:08:07.076218+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[27500]: audit 2023-10-09T09:08:07.100209+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:07.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[27500]: cluster 2023-10-09T09:08:07.141047+0000 mon.a (mon.0) 1046 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2023-10-09T09:08:07.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:07 smithi163 systemd[1]: Started Ceph mgr.y for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:08:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[33912]: cluster 2023-10-09T09:08:06.159094+0000 mgr.x (mgr.24751) 72 : cluster [DBG] pgmap v116: 89 pgs: 89 active+clean; 293 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:08:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[33912]: cluster 2023-10-09T09:08:06.215315+0000 mon.a (mon.0) 1042 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2023-10-09T09:08:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[33912]: audit 2023-10-09T09:08:06.495249+0000 mon.a (mon.0) 1043 : audit [DBG] from='client.? 172.21.15.163:0/59235988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[33912]: audit 2023-10-09T09:08:07.064133+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[33912]: audit 2023-10-09T09:08:07.076218+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[33912]: audit 2023-10-09T09:08:07.100209+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:07 smithi163 bash[33912]: cluster 2023-10-09T09:08:07.141047+0000 mon.a (mon.0) 1046 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2023-10-09T09:08:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:07 smithi181 bash[27712]: cluster 2023-10-09T09:08:06.159094+0000 mgr.x (mgr.24751) 72 : cluster [DBG] pgmap v116: 89 pgs: 89 active+clean; 293 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:08:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:07 smithi181 bash[27712]: cluster 2023-10-09T09:08:06.215315+0000 mon.a (mon.0) 1042 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2023-10-09T09:08:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:07 smithi181 bash[27712]: audit 2023-10-09T09:08:06.495249+0000 mon.a (mon.0) 1043 : audit [DBG] from='client.? 172.21.15.163:0/59235988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:07 smithi181 bash[27712]: audit 2023-10-09T09:08:07.064133+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:07 smithi181 bash[27712]: audit 2023-10-09T09:08:07.076218+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:07 smithi181 bash[27712]: audit 2023-10-09T09:08:07.100209+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:07.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:07 smithi181 bash[27712]: cluster 2023-10-09T09:08:07.141047+0000 mon.a (mon.0) 1046 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2023-10-09T09:08:08.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:07 smithi181 bash[50540]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:08:07] "GET /metrics HTTP/1.1" 200 36088 "" "Prometheus/2.33.4" 2023-10-09T09:08:08.199 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:08 smithi163 bash[71599]: debug 2023-10-09T09:08:08.036+0000 7f9710339700 1 -- 172.21.15.163:0/1367562313 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5572695eb1e0 con 0x55726a36a400 2023-10-09T09:08:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:645: done (2 left) 2023-10-09T09:08:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:646: read oid 18 snap -1 2023-10-09T09:08:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:646: expect (ObjNum 265 snap 80 seq_num 265) 2023-10-09T09:08:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:647: read oid 25 snap -1 2023-10-09T09:08:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:647: expect deleted 2023-10-09T09:08:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:648: delete oid 14 current snap is 86 2023-10-09T09:08:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:644: expect (ObjNum 148 snap 34 seq_num 148) 2023-10-09T09:08:08.353 INFO:tasks.rados.rados.0.smithi163.stdout:643: expect (ObjNum 239 snap 73 seq_num 239) 2023-10-09T09:08:08.421 INFO:tasks.rados.rados.0.smithi163.stdout:646: expect (ObjNum 265 snap 80 seq_num 265) 2023-10-09T09:08:08.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:08 smithi163 bash[71599]: debug 2023-10-09T09:08:08.196+0000 7f971a487200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:08:08.565 INFO:tasks.rados.rados.0.smithi163.stdout:643: done (4 left) 2023-10-09T09:08:08.566 INFO:tasks.rados.rados.0.smithi163.stdout:644: done (3 left) 2023-10-09T09:08:08.566 INFO:tasks.rados.rados.0.smithi163.stdout:646: done (2 left) 2023-10-09T09:08:08.566 INFO:tasks.rados.rados.0.smithi163.stdout:647: done (1 left) 2023-10-09T09:08:08.566 INFO:tasks.rados.rados.0.smithi163.stdout:648: done (0 left) 2023-10-09T09:08:08.566 INFO:tasks.rados.rados.0.smithi163.stdout:649: snap_remove snap 67 2023-10-09T09:08:09.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:08 smithi181 bash[52947]: ts=2023-10-09T09:08:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:08:09.250 INFO:tasks.rados.rados.0.smithi163.stdout:649: done (0 left) 2023-10-09T09:08:09.250 INFO:tasks.rados.rados.0.smithi163.stdout:650: write oid 29 current snap is 86 2023-10-09T09:08:09.250 INFO:tasks.rados.rados.0.smithi163.stdout:650: seq_num 277 ranges {655539=632681,2076218=570097} 2023-10-09T09:08:09.250 INFO:tasks.rados.rados.0.smithi163.stdout:650: writing smithi16367185-29 from 655539 to 1288220 tid 1 2023-10-09T09:08:09.258 INFO:tasks.rados.rados.0.smithi163.stdout:650: writing smithi16367185-29 from 2076218 to 2646315 tid 2 2023-10-09T09:08:09.258 INFO:tasks.rados.rados.0.smithi163.stdout:651: delete oid 28 current snap is 86 2023-10-09T09:08:09.258 INFO:tasks.rados.rados.0.smithi163.stdout:650: finishing write tid 1 to smithi16367185-29 2023-10-09T09:08:09.261 INFO:tasks.rados.rados.0.smithi163.stdout:650: finishing write tid 2 to smithi16367185-29 2023-10-09T09:08:09.261 INFO:tasks.rados.rados.0.smithi163.stdout:651: done (1 left) 2023-10-09T09:08:09.261 INFO:tasks.rados.rados.0.smithi163.stdout:652: delete oid 7 current snap is 86 2023-10-09T09:08:09.261 INFO:tasks.rados.rados.0.smithi163.stdout:650: finishing write tid 3 to smithi16367185-29 2023-10-09T09:08:09.262 INFO:tasks.rados.rados.0.smithi163.stdout:650: finishing write tid 4 to smithi16367185-29 2023-10-09T09:08:09.262 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 316 (ObjNum 277 snap 86 seq_num 277) dirty exists 2023-10-09T09:08:09.262 INFO:tasks.rados.rados.0.smithi163.stdout:650: left oid 29 (ObjNum 277 snap 86 seq_num 277) 2023-10-09T09:08:09.264 INFO:tasks.rados.rados.0.smithi163.stdout:650: done (1 left) 2023-10-09T09:08:09.265 INFO:tasks.rados.rados.0.smithi163.stdout:652: done (0 left) 2023-10-09T09:08:09.265 INFO:tasks.rados.rados.0.smithi163.stdout:653: read oid 4 snap -1 2023-10-09T09:08:09.265 INFO:tasks.rados.rados.0.smithi163.stdout:653: expect deleted 2023-10-09T09:08:09.265 INFO:tasks.rados.rados.0.smithi163.stdout:654: rollback oid 16 current snap is 86 2023-10-09T09:08:09.265 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 16 to 31 2023-10-09T09:08:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:655: write oid 45 current snap is 86 2023-10-09T09:08:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:655: seq_num 280 ranges {705960=458428,1735311=717424} 2023-10-09T09:08:09.272 INFO:tasks.rados.rados.0.smithi163.stdout:655: writing smithi16367185-45 from 705960 to 1164388 tid 1 2023-10-09T09:08:09.281 INFO:tasks.rados.rados.0.smithi163.stdout:655: writing smithi16367185-45 from 1735311 to 2452735 tid 2 2023-10-09T09:08:09.282 INFO:tasks.rados.rados.0.smithi163.stdout:656: delete oid 11 current snap is 86 2023-10-09T09:08:09.282 INFO:tasks.rados.rados.0.smithi163.stdout:654: finishing rollback tid 0 to smithi16367185-16 2023-10-09T09:08:09.282 INFO:tasks.rados.rados.0.smithi163.stdout:654: finishing rollback tid 1 to smithi16367185-16 2023-10-09T09:08:09.282 INFO:tasks.rados.rados.0.smithi163.stdout:654: finishing rollback tid 2 to smithi16367185-16 2023-10-09T09:08:09.282 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 339 (ObjNum 117 snap 20 seq_num 117) dirty exists 2023-10-09T09:08:09.282 INFO:tasks.rados.rados.0.smithi163.stdout:655: finishing write tid 1 to smithi16367185-45 2023-10-09T09:08:09.286 INFO:tasks.rados.rados.0.smithi163.stdout:653: done (3 left) 2023-10-09T09:08:09.286 INFO:tasks.rados.rados.0.smithi163.stdout:654: done (2 left) 2023-10-09T09:08:09.286 INFO:tasks.rados.rados.0.smithi163.stdout:656: done (1 left) 2023-10-09T09:08:09.286 INFO:tasks.rados.rados.0.smithi163.stdout:657: read oid 41 snap -1 2023-10-09T09:08:09.286 INFO:tasks.rados.rados.0.smithi163.stdout:657: expect deleted 2023-10-09T09:08:09.286 INFO:tasks.rados.rados.0.smithi163.stdout:658: read oid 8 snap 63 2023-10-09T09:08:09.287 INFO:tasks.rados.rados.0.smithi163.stdout:658: expect (ObjNum 207 snap 59 seq_num 207) 2023-10-09T09:08:09.287 INFO:tasks.rados.rados.0.smithi163.stdout:659: rollback oid 10 current snap is 86 2023-10-09T09:08:09.287 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 72 2023-10-09T09:08:09.287 INFO:tasks.rados.rados.0.smithi163.stdout:660: snap_remove snap 31 2023-10-09T09:08:09.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:09 smithi163 bash[71599]: debug 2023-10-09T09:08:09.148+0000 7f971a487200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:08:09.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:09 smithi163 bash[71599]: debug 2023-10-09T09:08:09.260+0000 7f971a487200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:08:09.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:09 smithi163 bash[71599]: debug 2023-10-09T09:08:09.368+0000 7f971a487200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:08:09.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:09 smithi163 bash[27500]: cluster 2023-10-09T09:08:08.159794+0000 mgr.x (mgr.24751) 73 : cluster [DBG] pgmap v119: 89 pgs: 89 active+clean; 289 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.5 MiB/s rd, 591 KiB/s wr, 6 op/s 2023-10-09T09:08:09.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:09 smithi163 bash[27500]: cluster 2023-10-09T09:08:08.230123+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2023-10-09T09:08:09.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:09 smithi163 bash[27500]: audit 2023-10-09T09:08:08.934080+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.163:0/3419938818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:09 smithi163 bash[33912]: cluster 2023-10-09T09:08:08.159794+0000 mgr.x (mgr.24751) 73 : cluster [DBG] pgmap v119: 89 pgs: 89 active+clean; 289 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.5 MiB/s rd, 591 KiB/s wr, 6 op/s 2023-10-09T09:08:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:09 smithi163 bash[33912]: cluster 2023-10-09T09:08:08.230123+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2023-10-09T09:08:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:09 smithi163 bash[33912]: audit 2023-10-09T09:08:08.934080+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.163:0/3419938818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:09 smithi181 bash[27712]: cluster 2023-10-09T09:08:08.159794+0000 mgr.x (mgr.24751) 73 : cluster [DBG] pgmap v119: 89 pgs: 89 active+clean; 289 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.5 MiB/s rd, 591 KiB/s wr, 6 op/s 2023-10-09T09:08:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:09 smithi181 bash[27712]: cluster 2023-10-09T09:08:08.230123+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2023-10-09T09:08:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:09 smithi181 bash[27712]: audit 2023-10-09T09:08:08.934080+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.163:0/3419938818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:09.967 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:09 smithi163 bash[71599]: debug 2023-10-09T09:08:09.636+0000 7f971a487200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:08:10.243 INFO:tasks.rados.rados.0.smithi163.stdout:660: done (4 left) 2023-10-09T09:08:10.243 INFO:tasks.rados.rados.0.smithi163.stdout:661: read oid 11 snap -1 2023-10-09T09:08:10.243 INFO:tasks.rados.rados.0.smithi163.stdout:661: expect deleted 2023-10-09T09:08:10.243 INFO:tasks.rados.rados.0.smithi163.stdout:662: snap_remove snap 77 2023-10-09T09:08:10.417 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:10 smithi163 bash[33912]: cluster 2023-10-09T09:08:09.236506+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2023-10-09T09:08:10.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:10 smithi163 bash[27500]: cluster 2023-10-09T09:08:09.236506+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2023-10-09T09:08:10.418 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:10 smithi163 bash[71599]: debug 2023-10-09T09:08:10.000+0000 7f971a487200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:08:10.418 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:10 smithi163 bash[71599]: debug 2023-10-09T09:08:10.176+0000 7f971a487200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:08:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:10 smithi181 bash[27712]: cluster 2023-10-09T09:08:09.236506+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2023-10-09T09:08:10.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:10 smithi163 bash[71599]: debug 2023-10-09T09:08:10.412+0000 7f971a487200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:08:11.248 INFO:tasks.rados.rados.0.smithi163.stdout:662: done (5 left) 2023-10-09T09:08:11.248 INFO:tasks.rados.rados.0.smithi163.stdout:663: delete oid 16 current snap is 86 2023-10-09T09:08:11.248 INFO:tasks.rados.rados.0.smithi163.stdout:655: finishing write tid 2 to smithi16367185-45 2023-10-09T09:08:11.248 INFO:tasks.rados.rados.0.smithi163.stdout:655: finishing write tid 3 to smithi16367185-45 2023-10-09T09:08:11.249 INFO:tasks.rados.rados.0.smithi163.stdout:655: finishing write tid 4 to smithi16367185-45 2023-10-09T09:08:11.249 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 520 (ObjNum 280 snap 86 seq_num 280) dirty exists 2023-10-09T09:08:11.249 INFO:tasks.rados.rados.0.smithi163.stdout:655: left oid 45 (ObjNum 280 snap 86 seq_num 280) 2023-10-09T09:08:11.249 INFO:tasks.rados.rados.0.smithi163.stdout:659: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:08:11.249 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 160 (ObjNum 1 snap 0 seq_num 1945163136) dirty dne 2023-10-09T09:08:11.249 INFO:tasks.rados.rados.0.smithi163.stdout:658: expect (ObjNum 207 snap 59 seq_num 207) 2023-10-09T09:08:11.455 INFO:tasks.rados.rados.0.smithi163.stdout:655: done (5 left) 2023-10-09T09:08:11.455 INFO:tasks.rados.rados.0.smithi163.stdout:657: done (4 left) 2023-10-09T09:08:11.455 INFO:tasks.rados.rados.0.smithi163.stdout:658: done (3 left) 2023-10-09T09:08:11.456 INFO:tasks.rados.rados.0.smithi163.stdout:659: done (2 left) 2023-10-09T09:08:11.456 INFO:tasks.rados.rados.0.smithi163.stdout:661: done (1 left) 2023-10-09T09:08:11.456 INFO:tasks.rados.rados.0.smithi163.stdout:663: done (0 left) 2023-10-09T09:08:11.456 INFO:tasks.rados.rados.0.smithi163.stdout:664: rollback oid 37 current snap is 86 2023-10-09T09:08:11.456 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 74 2023-10-09T09:08:11.456 INFO:tasks.rados.rados.0.smithi163.stdout:665: rollback oid 21 current snap is 86 2023-10-09T09:08:11.456 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 63 2023-10-09T09:08:11.456 INFO:tasks.rados.rados.0.smithi163.stdout:666: write oid 40 current snap is 86 2023-10-09T09:08:11.456 INFO:tasks.rados.rados.0.smithi163.stdout:666: seq_num 283 ranges {756381=400528,1910757=464751} 2023-10-09T09:08:11.461 INFO:tasks.rados.rados.0.smithi163.stdout:666: writing smithi16367185-40 from 756381 to 1156909 tid 1 2023-10-09T09:08:11.470 INFO:tasks.rados.rados.0.smithi163.stdout:666: writing smithi16367185-40 from 1910757 to 2375508 tid 2 2023-10-09T09:08:11.470 INFO:tasks.rados.rados.0.smithi163.stdout:667: write oid 42 current snap is 86 2023-10-09T09:08:11.470 INFO:tasks.rados.rados.0.smithi163.stdout:664: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:08:11.470 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 265 (ObjNum 669364008 snap 21875 seq_num 1) dirty dne 2023-10-09T09:08:11.470 INFO:tasks.rados.rados.0.smithi163.stdout:667: seq_num 284 ranges {773188=475777,1969239=724762} 2023-10-09T09:08:11.477 INFO:tasks.rados.rados.0.smithi163.stdout:667: writing smithi16367185-42 from 773188 to 1248965 tid 1 2023-10-09T09:08:11.486 INFO:tasks.rados.rados.0.smithi163.stdout:667: writing smithi16367185-42 from 1969239 to 2694001 tid 2 2023-10-09T09:08:11.486 INFO:tasks.rados.rados.0.smithi163.stdout:664: done (3 left) 2023-10-09T09:08:11.486 INFO:tasks.rados.rados.0.smithi163.stdout:668: snap_create 2023-10-09T09:08:11.486 INFO:tasks.rados.rados.0.smithi163.stdout:665: finishing rollback tid 0 to smithi16367185-21 2023-10-09T09:08:11.486 INFO:tasks.rados.rados.0.smithi163.stdout:665: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:08:11.487 INFO:tasks.rados.rados.0.smithi163.stdout:665: finishing rollback tid 2 to smithi16367185-21 2023-10-09T09:08:11.487 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 194 (ObjNum 224 snap 62 seq_num 224) dirty exists 2023-10-09T09:08:11.487 INFO:tasks.rados.rados.0.smithi163.stdout:666: finishing write tid 1 to smithi16367185-40 2023-10-09T09:08:11.487 INFO:tasks.rados.rados.0.smithi163.stdout:666: finishing write tid 2 to smithi16367185-40 2023-10-09T09:08:11.487 INFO:tasks.rados.rados.0.smithi163.stdout:666: finishing write tid 3 to smithi16367185-40 2023-10-09T09:08:11.487 INFO:tasks.rados.rados.0.smithi163.stdout:666: finishing write tid 4 to smithi16367185-40 2023-10-09T09:08:11.487 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 349 (ObjNum 283 snap 86 seq_num 283) dirty exists 2023-10-09T09:08:11.487 INFO:tasks.rados.rados.0.smithi163.stdout:666: left oid 40 (ObjNum 283 snap 86 seq_num 283) 2023-10-09T09:08:11.487 INFO:tasks.rados.rados.0.smithi163.stdout:667: finishing write tid 1 to smithi16367185-42 2023-10-09T09:08:11.491 INFO:tasks.rados.rados.0.smithi163.stdout:667: finishing write tid 2 to smithi16367185-42 2023-10-09T09:08:11.491 INFO:tasks.rados.rados.0.smithi163.stdout:667: finishing write tid 3 to smithi16367185-42 2023-10-09T09:08:11.491 INFO:tasks.rados.rados.0.smithi163.stdout:667: finishing write tid 4 to smithi16367185-42 2023-10-09T09:08:11.491 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 538 (ObjNum 284 snap 86 seq_num 284) dirty exists 2023-10-09T09:08:11.491 INFO:tasks.rados.rados.0.smithi163.stdout:667: left oid 42 (ObjNum 284 snap 86 seq_num 284) 2023-10-09T09:08:11.518 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:11 smithi163 bash[27500]: cluster 2023-10-09T09:08:10.160583+0000 mgr.x (mgr.24751) 74 : cluster [DBG] pgmap v122: 89 pgs: 89 active+clean; 289 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.5 MiB/s rd, 591 KiB/s wr, 5 op/s 2023-10-09T09:08:11.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:11 smithi163 bash[27500]: cluster 2023-10-09T09:08:10.242847+0000 mon.a (mon.0) 1049 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2023-10-09T09:08:11.519 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:11 smithi163 bash[33912]: cluster 2023-10-09T09:08:10.160583+0000 mgr.x (mgr.24751) 74 : cluster [DBG] pgmap v122: 89 pgs: 89 active+clean; 289 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.5 MiB/s rd, 591 KiB/s wr, 5 op/s 2023-10-09T09:08:11.519 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:11 smithi163 bash[33912]: cluster 2023-10-09T09:08:10.242847+0000 mon.a (mon.0) 1049 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2023-10-09T09:08:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:11 smithi181 bash[27712]: cluster 2023-10-09T09:08:10.160583+0000 mgr.x (mgr.24751) 74 : cluster [DBG] pgmap v122: 89 pgs: 89 active+clean; 289 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.5 MiB/s rd, 591 KiB/s wr, 5 op/s 2023-10-09T09:08:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:11 smithi181 bash[27712]: cluster 2023-10-09T09:08:10.242847+0000 mon.a (mon.0) 1049 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2023-10-09T09:08:11.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:11 smithi163 bash[71599]: debug 2023-10-09T09:08:11.516+0000 7f971a487200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:08:11.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:11 smithi163 bash[71599]: debug 2023-10-09T09:08:11.636+0000 7f971a487200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:08:11.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:11 smithi163 bash[71599]: debug 2023-10-09T09:08:11.764+0000 7f971a487200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:08:12.144 INFO:tasks.rados.rados.0.smithi163.stdout:665: done (3 left) 2023-10-09T09:08:12.144 INFO:tasks.rados.rados.0.smithi163.stdout:666: done (2 left) 2023-10-09T09:08:12.144 INFO:tasks.rados.rados.0.smithi163.stdout:667: done (1 left) 2023-10-09T09:08:12.145 INFO:tasks.rados.rados.0.smithi163.stdout:668: done (0 left) 2023-10-09T09:08:12.145 INFO:tasks.rados.rados.0.smithi163.stdout:669: rollback oid 28 current snap is 87 2023-10-09T09:08:12.145 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 20 2023-10-09T09:08:12.145 INFO:tasks.rados.rados.0.smithi163.stdout:670: snap_remove snap 72 2023-10-09T09:08:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:12 smithi181 bash[27712]: cluster 2023-10-09T09:08:11.245678+0000 mon.a (mon.0) 1050 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2023-10-09T09:08:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:12 smithi181 bash[27712]: audit 2023-10-09T09:08:11.414440+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.163:0/120544756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:12 smithi181 bash[27712]: cluster 2023-10-09T09:08:12.143814+0000 mon.a (mon.0) 1051 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2023-10-09T09:08:12.574 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:12 smithi163 bash[71599]: debug 2023-10-09T09:08:12.292+0000 7f971a487200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:08:12.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:12 smithi163 bash[33912]: cluster 2023-10-09T09:08:11.245678+0000 mon.a (mon.0) 1050 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2023-10-09T09:08:12.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:12 smithi163 bash[33912]: audit 2023-10-09T09:08:11.414440+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.163:0/120544756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:12.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:12 smithi163 bash[33912]: cluster 2023-10-09T09:08:12.143814+0000 mon.a (mon.0) 1051 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2023-10-09T09:08:12.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:12 smithi163 bash[27500]: cluster 2023-10-09T09:08:11.245678+0000 mon.a (mon.0) 1050 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2023-10-09T09:08:12.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:12 smithi163 bash[27500]: audit 2023-10-09T09:08:11.414440+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.163:0/120544756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:12.575 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:12 smithi163 bash[27500]: cluster 2023-10-09T09:08:12.143814+0000 mon.a (mon.0) 1051 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2023-10-09T09:08:12.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:12 smithi163 bash[71599]: debug 2023-10-09T09:08:12.568+0000 7f971a487200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:08:13.156 INFO:tasks.rados.rados.0.smithi163.stdout:670: done (1 left) 2023-10-09T09:08:13.156 INFO:tasks.rados.rados.0.smithi163.stdout:671: delete oid 29 current snap is 87 2023-10-09T09:08:13.156 INFO:tasks.rados.rados.0.smithi163.stdout:669: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:08:13.157 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 335 (ObjNum 698864768 snap 21875 seq_num 699180656) dirty dne 2023-10-09T09:08:13.168 INFO:tasks.rados.rados.0.smithi163.stdout:669: done (1 left) 2023-10-09T09:08:13.168 INFO:tasks.rados.rados.0.smithi163.stdout:671: done (0 left) 2023-10-09T09:08:13.168 INFO:tasks.rados.rados.0.smithi163.stdout:672: read oid 5 snap -1 2023-10-09T09:08:13.168 INFO:tasks.rados.rados.0.smithi163.stdout:672: expect (ObjNum 219 snap 61 seq_num 219) 2023-10-09T09:08:13.168 INFO:tasks.rados.rados.0.smithi163.stdout:673: snap_remove snap 71 2023-10-09T09:08:13.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:13 smithi163 bash[71599]: debug 2023-10-09T09:08:13.088+0000 7f971a487200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:08:13.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:13 smithi163 bash[27500]: cluster 2023-10-09T09:08:12.161168+0000 mgr.x (mgr.24751) 75 : cluster [DBG] pgmap v126: 89 pgs: 89 active+clean; 286 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 590 KiB/s wr, 16 op/s 2023-10-09T09:08:13.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:13 smithi163 bash[27500]: cluster 2023-10-09T09:08:13.156587+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2023-10-09T09:08:13.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:13 smithi163 bash[27500]: audit 2023-10-09T09:08:13.191349+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:13.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:13 smithi163 bash[27500]: audit 2023-10-09T09:08:13.201632+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:13 smithi163 bash[33912]: cluster 2023-10-09T09:08:12.161168+0000 mgr.x (mgr.24751) 75 : cluster [DBG] pgmap v126: 89 pgs: 89 active+clean; 286 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 590 KiB/s wr, 16 op/s 2023-10-09T09:08:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:13 smithi163 bash[33912]: cluster 2023-10-09T09:08:13.156587+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2023-10-09T09:08:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:13 smithi163 bash[33912]: audit 2023-10-09T09:08:13.191349+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:13.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:13 smithi163 bash[33912]: audit 2023-10-09T09:08:13.201632+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:13 smithi181 bash[27712]: cluster 2023-10-09T09:08:12.161168+0000 mgr.x (mgr.24751) 75 : cluster [DBG] pgmap v126: 89 pgs: 89 active+clean; 286 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 590 KiB/s wr, 16 op/s 2023-10-09T09:08:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:13 smithi181 bash[27712]: cluster 2023-10-09T09:08:13.156587+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2023-10-09T09:08:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:13 smithi181 bash[27712]: audit 2023-10-09T09:08:13.191349+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:13 smithi181 bash[27712]: audit 2023-10-09T09:08:13.201632+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:13.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:13 smithi163 bash[71599]: debug 2023-10-09T09:08:13.696+0000 7f971a487200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:08:14.163 INFO:tasks.rados.rados.0.smithi163.stdout:673: done (1 left) 2023-10-09T09:08:14.163 INFO:tasks.rados.rados.0.smithi163.stdout:674: write oid 24 current snap is 87 2023-10-09T09:08:14.163 INFO:tasks.rados.rados.0.smithi163.stdout:674: seq_num 286 ranges {406802=626275,1686203=728431} 2023-10-09T09:08:14.171 INFO:tasks.rados.rados.0.smithi163.stdout:674: writing smithi16367185-24 from 406802 to 1033077 tid 1 2023-10-09T09:08:14.181 INFO:tasks.rados.rados.0.smithi163.stdout:674: writing smithi16367185-24 from 1686203 to 2414634 tid 2 2023-10-09T09:08:14.181 INFO:tasks.rados.rados.0.smithi163.stdout:675: snap_remove snap 80 2023-10-09T09:08:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[27500]: audit 2023-10-09T09:08:13.347926+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[27500]: audit 2023-10-09T09:08:13.360456+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[27500]: audit 2023-10-09T09:08:13.937561+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.163:0/3252051079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[27500]: audit 2023-10-09T09:08:14.106363+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[27500]: audit 2023-10-09T09:08:14.115327+0000 mon.a (mon.0) 1058 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[27500]: cluster 2023-10-09T09:08:14.163866+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2023-10-09T09:08:14.720 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[71599]: debug 2023-10-09T09:08:14.336+0000 7f971a487200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:08:14.720 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[71599]: debug 2023-10-09T09:08:14.464+0000 7f971a487200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:08:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[33912]: audit 2023-10-09T09:08:13.347926+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[33912]: audit 2023-10-09T09:08:13.360456+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[33912]: audit 2023-10-09T09:08:13.937561+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.163:0/3252051079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:14.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[33912]: audit 2023-10-09T09:08:14.106363+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[33912]: audit 2023-10-09T09:08:14.115327+0000 mon.a (mon.0) 1058 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[33912]: cluster 2023-10-09T09:08:14.163866+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2023-10-09T09:08:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:14 smithi181 bash[27712]: audit 2023-10-09T09:08:13.347926+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:14 smithi181 bash[27712]: audit 2023-10-09T09:08:13.360456+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:14 smithi181 bash[27712]: audit 2023-10-09T09:08:13.937561+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.163:0/3252051079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:14 smithi181 bash[27712]: audit 2023-10-09T09:08:14.106363+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:14 smithi181 bash[27712]: audit 2023-10-09T09:08:14.115327+0000 mon.a (mon.0) 1058 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:14 smithi181 bash[27712]: cluster 2023-10-09T09:08:14.163866+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2023-10-09T09:08:15.107 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[71599]: debug 2023-10-09T09:08:14.844+0000 7f971a487200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:08:15.107 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:14 smithi163 bash[71599]: debug 2023-10-09T09:08:14.960+0000 7f971a487200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:08:15.170 INFO:tasks.rados.rados.0.smithi163.stdout:675: done (2 left) 2023-10-09T09:08:15.170 INFO:tasks.rados.rados.0.smithi163.stdout:676: write oid 30 current snap is 87 2023-10-09T09:08:15.170 INFO:tasks.rados.rados.0.smithi163.stdout:676: seq_num 287 ranges {423609=701524,1628332=472089,2648987=174622} 2023-10-09T09:08:15.179 INFO:tasks.rados.rados.0.smithi163.stdout:676: writing smithi16367185-30 from 423609 to 1125133 tid 1 2023-10-09T09:08:15.188 INFO:tasks.rados.rados.0.smithi163.stdout:676: writing smithi16367185-30 from 1628332 to 2100421 tid 2 2023-10-09T09:08:15.193 INFO:tasks.rados.rados.0.smithi163.stdout:676: writing smithi16367185-30 from 2648987 to 2823609 tid 3 2023-10-09T09:08:15.193 INFO:tasks.rados.rados.0.smithi163.stdout:672: expect (ObjNum 219 snap 61 seq_num 219) 2023-10-09T09:08:15.381 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:15 smithi163 bash[71599]: debug 2023-10-09T09:08:15.104+0000 7f971a487200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:08:15.381 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:15 smithi163 bash[71599]: debug 2023-10-09T09:08:15.236+0000 7f971a487200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:08:15.382 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:15 smithi163 bash[33912]: cluster 2023-10-09T09:08:14.161731+0000 mgr.x (mgr.24751) 76 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 286 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 593 KiB/s wr, 16 op/s 2023-10-09T09:08:15.382 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:15 smithi163 bash[33912]: cluster 2023-10-09T09:08:15.166551+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2023-10-09T09:08:15.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:15 smithi163 bash[27500]: cluster 2023-10-09T09:08:14.161731+0000 mgr.x (mgr.24751) 76 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 286 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 593 KiB/s wr, 16 op/s 2023-10-09T09:08:15.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:15 smithi163 bash[27500]: cluster 2023-10-09T09:08:15.166551+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2023-10-09T09:08:15.418 INFO:tasks.rados.rados.0.smithi163.stdout:674: finishing write tid 1 to smithi16367185-24 2023-10-09T09:08:15.418 INFO:tasks.rados.rados.0.smithi163.stdout:674: finishing write tid 2 to smithi16367185-24 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:674: finishing write tid 3 to smithi16367185-24 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:674: finishing write tid 4 to smithi16367185-24 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 188 (ObjNum 286 snap 87 seq_num 286) dirty exists 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:674: left oid 24 (ObjNum 286 snap 87 seq_num 286) 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:676: finishing write tid 1 to smithi16367185-30 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:676: finishing write tid 2 to smithi16367185-30 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:676: finishing write tid 3 to smithi16367185-30 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:676: finishing write tid 4 to smithi16367185-30 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:676: finishing write tid 5 to smithi16367185-30 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 260 (ObjNum 287 snap 87 seq_num 287) dirty exists 2023-10-09T09:08:15.419 INFO:tasks.rados.rados.0.smithi163.stdout:676: left oid 30 (ObjNum 287 snap 87 seq_num 287) 2023-10-09T09:08:15.420 INFO:tasks.rados.rados.0.smithi163.stdout:677: read oid 13 snap 74 2023-10-09T09:08:15.420 INFO:tasks.rados.rados.0.smithi163.stdout:677: expect (ObjNum 168 snap 44 seq_num 168) 2023-10-09T09:08:15.420 INFO:tasks.rados.rados.0.smithi163.stdout:672: done (3 left) 2023-10-09T09:08:15.420 INFO:tasks.rados.rados.0.smithi163.stdout:674: done (2 left) 2023-10-09T09:08:15.420 INFO:tasks.rados.rados.0.smithi163.stdout:676: done (1 left) 2023-10-09T09:08:15.420 INFO:tasks.rados.rados.0.smithi163.stdout:678: read oid 11 snap 60 2023-10-09T09:08:15.420 INFO:tasks.rados.rados.0.smithi163.stdout:678: expect (ObjNum 148 snap 34 seq_num 148) 2023-10-09T09:08:15.420 INFO:tasks.rados.rados.0.smithi163.stdout:679: write oid 5 current snap is 87 2023-10-09T09:08:15.420 INFO:tasks.rados.rados.0.smithi163.stdout:679: seq_num 288 ranges {440416=776773,1686814=732100} 2023-10-09T09:08:15.431 INFO:tasks.rados.rados.0.smithi163.stdout:679: writing smithi16367185-5 from 440416 to 1217189 tid 1 2023-10-09T09:08:15.442 INFO:tasks.rados.rados.0.smithi163.stdout:679: writing smithi16367185-5 from 1686814 to 2418914 tid 2 2023-10-09T09:08:15.442 INFO:tasks.rados.rados.0.smithi163.stdout:680: read oid 4 snap 81 2023-10-09T09:08:15.442 INFO:tasks.rados.rados.0.smithi163.stdout:680: expect deleted 2023-10-09T09:08:15.442 INFO:tasks.rados.rados.0.smithi163.stdout:681: snap_create 2023-10-09T09:08:15.442 INFO:tasks.rados.rados.0.smithi163.stdout:677: expect (ObjNum 168 snap 44 seq_num 168) 2023-10-09T09:08:15.580 INFO:tasks.rados.rados.0.smithi163.stdout:678: expect (ObjNum 148 snap 34 seq_num 148) 2023-10-09T09:08:15.701 INFO:tasks.rados.rados.0.smithi163.stdout:679: finishing write tid 1 to smithi16367185-5 2023-10-09T09:08:15.701 INFO:tasks.rados.rados.0.smithi163.stdout:679: finishing write tid 2 to smithi16367185-5 2023-10-09T09:08:15.701 INFO:tasks.rados.rados.0.smithi163.stdout:679: finishing write tid 3 to smithi16367185-5 2023-10-09T09:08:15.701 INFO:tasks.rados.rados.0.smithi163.stdout:679: finishing write tid 4 to smithi16367185-5 2023-10-09T09:08:15.701 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 379 (ObjNum 288 snap 87 seq_num 288) dirty exists 2023-10-09T09:08:15.701 INFO:tasks.rados.rados.0.smithi163.stdout:679: left oid 5 (ObjNum 288 snap 87 seq_num 288) 2023-10-09T09:08:15.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:15 smithi163 bash[71599]: debug 2023-10-09T09:08:15.376+0000 7f971a487200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:08:15.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:15 smithi163 bash[71599]: debug 2023-10-09T09:08:15.596+0000 7f971a487200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:08:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:15 smithi181 bash[27712]: cluster 2023-10-09T09:08:14.161731+0000 mgr.x (mgr.24751) 76 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 286 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 593 KiB/s wr, 16 op/s 2023-10-09T09:08:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:15 smithi181 bash[27712]: cluster 2023-10-09T09:08:15.166551+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2023-10-09T09:08:16.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:15 smithi163 bash[71599]: debug 2023-10-09T09:08:15.736+0000 7f971a487200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:08:16.366 INFO:tasks.rados.rados.0.smithi163.stdout:677: done (4 left) 2023-10-09T09:08:16.366 INFO:tasks.rados.rados.0.smithi163.stdout:678: done (3 left) 2023-10-09T09:08:16.366 INFO:tasks.rados.rados.0.smithi163.stdout:679: done (2 left) 2023-10-09T09:08:16.366 INFO:tasks.rados.rados.0.smithi163.stdout:680: done (1 left) 2023-10-09T09:08:16.366 INFO:tasks.rados.rados.0.smithi163.stdout:681: done (0 left) 2023-10-09T09:08:16.366 INFO:tasks.rados.rados.0.smithi163.stdout:682: write oid 15 current snap is 88 2023-10-09T09:08:16.366 INFO:tasks.rados.rados.0.smithi163.stdout:682: seq_num 289 ranges {457223=568375,1461649=475758,2420186=437037} 2023-10-09T09:08:16.376 INFO:tasks.rados.rados.0.smithi163.stdout:682: writing smithi16367185-15 from 457223 to 1025598 tid 1 2023-10-09T09:08:16.386 INFO:tasks.rados.rados.0.smithi163.stdout:682: writing smithi16367185-15 from 1461649 to 1937407 tid 2 2023-10-09T09:08:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:682: writing smithi16367185-15 from 2420186 to 2857223 tid 3 2023-10-09T09:08:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:683: rollback oid 5 current snap is 88 2023-10-09T09:08:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 85 2023-10-09T09:08:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:682: finishing write tid 1 to smithi16367185-15 2023-10-09T09:08:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:682: finishing write tid 2 to smithi16367185-15 2023-10-09T09:08:16.397 INFO:tasks.rados.rados.0.smithi163.stdout:684: read oid 36 snap -1 2023-10-09T09:08:16.397 INFO:tasks.rados.rados.0.smithi163.stdout:684: expect (ObjNum 248 snap 74 seq_num 248) 2023-10-09T09:08:16.397 INFO:tasks.rados.rados.0.smithi163.stdout:685: snap_remove snap 74 2023-10-09T09:08:17.145 INFO:tasks.rados.rados.0.smithi163.stdout:685: done (3 left) 2023-10-09T09:08:17.145 INFO:tasks.rados.rados.0.smithi163.stdout:686: read oid 2 snap -1 2023-10-09T09:08:17.145 INFO:tasks.rados.rados.0.smithi163.stdout:686: expect (ObjNum 253 snap 77 seq_num 253) 2023-10-09T09:08:17.145 INFO:tasks.rados.rados.0.smithi163.stdout:687: snap_create 2023-10-09T09:08:17.145 INFO:tasks.rados.rados.0.smithi163.stdout:683: finishing rollback tid 0 to smithi16367185-5 2023-10-09T09:08:17.145 INFO:tasks.rados.rados.0.smithi163.stdout:684: expect (ObjNum 248 snap 74 seq_num 248) 2023-10-09T09:08:17.249 INFO:tasks.rados.rados.0.smithi163.stdout:683: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:08:17.249 INFO:tasks.rados.rados.0.smithi163.stdout:682: finishing write tid 3 to smithi16367185-15 2023-10-09T09:08:17.249 INFO:tasks.rados.rados.0.smithi163.stdout:682: finishing write tid 4 to smithi16367185-15 2023-10-09T09:08:17.249 INFO:tasks.rados.rados.0.smithi163.stdout:683: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:08:17.249 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 383 (ObjNum 219 snap 61 seq_num 219) dirty exists 2023-10-09T09:08:17.249 INFO:tasks.rados.rados.0.smithi163.stdout:682: finishing write tid 5 to smithi16367185-15 2023-10-09T09:08:17.249 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 370 (ObjNum 289 snap 88 seq_num 289) dirty exists 2023-10-09T09:08:17.249 INFO:tasks.rados.rados.0.smithi163.stdout:682: left oid 15 (ObjNum 289 snap 88 seq_num 289) 2023-10-09T09:08:17.249 INFO:tasks.rados.rados.0.smithi163.stdout:686: expect (ObjNum 253 snap 77 seq_num 253) 2023-10-09T09:08:17.364 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[71599]: debug 2023-10-09T09:08:17.060+0000 7f971a487200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:08:17.364 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[71599]: [09/Oct/2023:09:08:17] ENGINE Bus STARTING 2023-10-09T09:08:17.364 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[71599]: CherryPy Checker: 2023-10-09T09:08:17.364 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[71599]: The Application mounted at '' has an empty config. 2023-10-09T09:08:17.364 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[71599]: [09/Oct/2023:09:08:17] ENGINE Serving on http://:::9283 2023-10-09T09:08:17.364 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[71599]: [09/Oct/2023:09:08:17] ENGINE Bus STARTED 2023-10-09T09:08:17.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[27500]: cluster 2023-10-09T09:08:16.162584+0000 mgr.x (mgr.24751) 77 : cluster [DBG] pgmap v131: 89 pgs: 89 active+clean; 286 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 254 B/s rd, 291 KiB/s wr, 1 op/s 2023-10-09T09:08:17.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[27500]: cluster 2023-10-09T09:08:16.363751+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2023-10-09T09:08:17.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[27500]: audit 2023-10-09T09:08:16.374139+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.163:0/3812159675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[27500]: cluster 2023-10-09T09:08:17.089389+0000 mon.a (mon.0) 1063 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T09:08:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[27500]: cluster 2023-10-09T09:08:17.089726+0000 mon.a (mon.0) 1064 : cluster [DBG] Standby manager daemon y started 2023-10-09T09:08:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[27500]: audit 2023-10-09T09:08:17.094285+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:08:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[27500]: audit 2023-10-09T09:08:17.095195+0000 mon.a (mon.0) 1066 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:08:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[27500]: audit 2023-10-09T09:08:17.099556+0000 mon.a (mon.0) 1067 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:08:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[27500]: audit 2023-10-09T09:08:17.101027+0000 mon.a (mon.0) 1068 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:08:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[27500]: cluster 2023-10-09T09:08:17.143921+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2023-10-09T09:08:17.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[33912]: cluster 2023-10-09T09:08:16.162584+0000 mgr.x (mgr.24751) 77 : cluster [DBG] pgmap v131: 89 pgs: 89 active+clean; 286 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 254 B/s rd, 291 KiB/s wr, 1 op/s 2023-10-09T09:08:17.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[33912]: cluster 2023-10-09T09:08:16.363751+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2023-10-09T09:08:17.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[33912]: audit 2023-10-09T09:08:16.374139+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.163:0/3812159675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:17.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[33912]: cluster 2023-10-09T09:08:17.089389+0000 mon.a (mon.0) 1063 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T09:08:17.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[33912]: cluster 2023-10-09T09:08:17.089726+0000 mon.a (mon.0) 1064 : cluster [DBG] Standby manager daemon y started 2023-10-09T09:08:17.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[33912]: audit 2023-10-09T09:08:17.094285+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:08:17.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[33912]: audit 2023-10-09T09:08:17.095195+0000 mon.a (mon.0) 1066 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:08:17.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[33912]: audit 2023-10-09T09:08:17.099556+0000 mon.a (mon.0) 1067 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:08:17.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[33912]: audit 2023-10-09T09:08:17.101027+0000 mon.a (mon.0) 1068 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:08:17.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:17 smithi163 bash[33912]: cluster 2023-10-09T09:08:17.143921+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2023-10-09T09:08:17.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[50540]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:08:17] "GET /metrics HTTP/1.1" 200 36088 "" "Prometheus/2.33.4" 2023-10-09T09:08:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[27712]: cluster 2023-10-09T09:08:16.162584+0000 mgr.x (mgr.24751) 77 : cluster [DBG] pgmap v131: 89 pgs: 89 active+clean; 286 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 254 B/s rd, 291 KiB/s wr, 1 op/s 2023-10-09T09:08:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[27712]: cluster 2023-10-09T09:08:16.363751+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2023-10-09T09:08:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[27712]: audit 2023-10-09T09:08:16.374139+0000 mon.a (mon.0) 1062 : audit [DBG] from='client.? 172.21.15.163:0/3812159675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[27712]: cluster 2023-10-09T09:08:17.089389+0000 mon.a (mon.0) 1063 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T09:08:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[27712]: cluster 2023-10-09T09:08:17.089726+0000 mon.a (mon.0) 1064 : cluster [DBG] Standby manager daemon y started 2023-10-09T09:08:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[27712]: audit 2023-10-09T09:08:17.094285+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:08:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[27712]: audit 2023-10-09T09:08:17.095195+0000 mon.a (mon.0) 1066 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:08:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[27712]: audit 2023-10-09T09:08:17.099556+0000 mon.a (mon.0) 1067 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:08:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[27712]: audit 2023-10-09T09:08:17.101027+0000 mon.a (mon.0) 1068 : audit [DBG] from='mgr.? 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:08:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:17 smithi181 bash[27712]: cluster 2023-10-09T09:08:17.143921+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2023-10-09T09:08:18.146 INFO:tasks.rados.rados.0.smithi163.stdout:682: done (4 left) 2023-10-09T09:08:18.147 INFO:tasks.rados.rados.0.smithi163.stdout:683: done (3 left) 2023-10-09T09:08:18.147 INFO:tasks.rados.rados.0.smithi163.stdout:684: done (2 left) 2023-10-09T09:08:18.147 INFO:tasks.rados.rados.0.smithi163.stdout:686: done (1 left) 2023-10-09T09:08:18.147 INFO:tasks.rados.rados.0.smithi163.stdout:687: done (0 left) 2023-10-09T09:08:18.147 INFO:tasks.rados.rados.0.smithi163.stdout:688: write oid 27 current snap is 89 2023-10-09T09:08:18.147 INFO:tasks.rados.rados.0.smithi163.stdout:688: seq_num 290 ranges {474030=643624,1520131=735769} 2023-10-09T09:08:18.153 INFO:tasks.rados.rados.0.smithi163.stdout:688: writing smithi16367185-27 from 474030 to 1117654 tid 1 2023-10-09T09:08:18.160 INFO:tasks.rados.rados.0.smithi163.stdout:688: writing smithi16367185-27 from 1520131 to 2255900 tid 2 2023-10-09T09:08:18.160 INFO:tasks.rados.rados.0.smithi163.stdout:689: read oid 16 snap -1 2023-10-09T09:08:18.160 INFO:tasks.rados.rados.0.smithi163.stdout:689: expect deleted 2023-10-09T09:08:18.160 INFO:tasks.rados.rados.0.smithi163.stdout:690: snap_remove snap 81 2023-10-09T09:08:18.665 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:18 smithi181 bash[27712]: cluster 2023-10-09T09:08:17.377622+0000 mon.a (mon.0) 1070 : cluster [DBG] mgrmap e24: x(active, since 87s), standbys: y 2023-10-09T09:08:18.665 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:18 smithi181 bash[27712]: cluster 2023-10-09T09:08:18.146662+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2023-10-09T09:08:18.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:18 smithi163 bash[27500]: cluster 2023-10-09T09:08:17.377622+0000 mon.a (mon.0) 1070 : cluster [DBG] mgrmap e24: x(active, since 87s), standbys: y 2023-10-09T09:08:18.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:18 smithi163 bash[27500]: cluster 2023-10-09T09:08:18.146662+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2023-10-09T09:08:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:18 smithi163 bash[33912]: cluster 2023-10-09T09:08:17.377622+0000 mon.a (mon.0) 1070 : cluster [DBG] mgrmap e24: x(active, since 87s), standbys: y 2023-10-09T09:08:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:18 smithi163 bash[33912]: cluster 2023-10-09T09:08:18.146662+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2023-10-09T09:08:19.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:18 smithi181 bash[52947]: ts=2023-10-09T09:08:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:08:19.154 INFO:tasks.rados.rados.0.smithi163.stdout:690: done (2 left) 2023-10-09T09:08:19.155 INFO:tasks.rados.rados.0.smithi163.stdout:691: rollback oid 3 current snap is 89 2023-10-09T09:08:19.155 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 20 2023-10-09T09:08:19.155 INFO:tasks.rados.rados.0.smithi163.stdout:692: snap_create 2023-10-09T09:08:19.155 INFO:tasks.rados.rados.0.smithi163.stdout:688: finishing write tid 1 to smithi16367185-27 2023-10-09T09:08:19.155 INFO:tasks.rados.rados.0.smithi163.stdout:688: finishing write tid 2 to smithi16367185-27 2023-10-09T09:08:19.155 INFO:tasks.rados.rados.0.smithi163.stdout:688: finishing write tid 3 to smithi16367185-27 2023-10-09T09:08:19.156 INFO:tasks.rados.rados.0.smithi163.stdout:688: finishing write tid 4 to smithi16367185-27 2023-10-09T09:08:19.156 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 216 (ObjNum 290 snap 89 seq_num 290) dirty exists 2023-10-09T09:08:19.156 INFO:tasks.rados.rados.0.smithi163.stdout:688: left oid 27 (ObjNum 290 snap 89 seq_num 290) 2023-10-09T09:08:19.163 INFO:tasks.rados.rados.0.smithi163.stdout:691: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:08:19.164 INFO:tasks.rados.rados.0.smithi163.stdout:691: finishing rollback tid 2 to smithi16367185-3 2023-10-09T09:08:19.164 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 210 (ObjNum 84 snap 10 seq_num 84) dirty exists 2023-10-09T09:08:19.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:19 smithi163 bash[27500]: cluster 2023-10-09T09:08:18.163103+0000 mgr.x (mgr.24751) 78 : cluster [DBG] pgmap v135: 89 pgs: 89 active+clean; 282 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2023-10-09T09:08:19.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:19 smithi163 bash[27500]: audit 2023-10-09T09:08:18.820702+0000 mon.a (mon.0) 1072 : audit [DBG] from='client.? 172.21.15.163:0/1255425356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:19.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:19 smithi163 bash[27500]: cluster 2023-10-09T09:08:19.151635+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2023-10-09T09:08:19.774 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:19 smithi163 bash[33912]: cluster 2023-10-09T09:08:18.163103+0000 mgr.x (mgr.24751) 78 : cluster [DBG] pgmap v135: 89 pgs: 89 active+clean; 282 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2023-10-09T09:08:19.775 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:19 smithi163 bash[33912]: audit 2023-10-09T09:08:18.820702+0000 mon.a (mon.0) 1072 : audit [DBG] from='client.? 172.21.15.163:0/1255425356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:19.775 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:19 smithi163 bash[33912]: cluster 2023-10-09T09:08:19.151635+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2023-10-09T09:08:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:19 smithi181 bash[27712]: cluster 2023-10-09T09:08:18.163103+0000 mgr.x (mgr.24751) 78 : cluster [DBG] pgmap v135: 89 pgs: 89 active+clean; 282 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2023-10-09T09:08:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:19 smithi181 bash[27712]: audit 2023-10-09T09:08:18.820702+0000 mon.a (mon.0) 1072 : audit [DBG] from='client.? 172.21.15.163:0/1255425356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:19 smithi181 bash[27712]: cluster 2023-10-09T09:08:19.151635+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2023-10-09T09:08:20.158 INFO:tasks.rados.rados.0.smithi163.stdout:688: done (3 left) 2023-10-09T09:08:20.158 INFO:tasks.rados.rados.0.smithi163.stdout:689: done (2 left) 2023-10-09T09:08:20.158 INFO:tasks.rados.rados.0.smithi163.stdout:691: done (1 left) 2023-10-09T09:08:20.158 INFO:tasks.rados.rados.0.smithi163.stdout:692: done (0 left) 2023-10-09T09:08:20.158 INFO:tasks.rados.rados.0.smithi163.stdout:693: read oid 17 snap 75 2023-10-09T09:08:20.159 INFO:tasks.rados.rados.0.smithi163.stdout:693: expect (ObjNum 137 snap 28 seq_num 137) 2023-10-09T09:08:20.159 INFO:tasks.rados.rados.0.smithi163.stdout:694: rollback oid 3 current snap is 90 2023-10-09T09:08:20.159 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 85 2023-10-09T09:08:20.159 INFO:tasks.rados.rados.0.smithi163.stdout:695: rollback oid 39 current snap is 90 2023-10-09T09:08:20.159 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 75 2023-10-09T09:08:20.159 INFO:tasks.rados.rados.0.smithi163.stdout:696: write oid 37 current snap is 90 2023-10-09T09:08:20.159 INFO:tasks.rados.rados.0.smithi163.stdout:696: seq_num 291 ranges {490837=718873,1862260=479427,2875032=15805} 2023-10-09T09:08:20.164 INFO:tasks.rados.rados.0.smithi163.stdout:696: writing smithi16367185-37 from 490837 to 1209710 tid 1 2023-10-09T09:08:20.171 INFO:tasks.rados.rados.0.smithi163.stdout:696: writing smithi16367185-37 from 1862260 to 2341687 tid 2 2023-10-09T09:08:20.175 INFO:tasks.rados.rados.0.smithi163.stdout:696: writing smithi16367185-37 from 2875032 to 2890837 tid 3 2023-10-09T09:08:20.175 INFO:tasks.rados.rados.0.smithi163.stdout:697: snap_create 2023-10-09T09:08:20.175 INFO:tasks.rados.rados.0.smithi163.stdout:694: finishing rollback tid 0 to smithi16367185-3 2023-10-09T09:08:20.175 INFO:tasks.rados.rados.0.smithi163.stdout:694: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:08:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 213 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:08:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:695: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:08:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 125 (ObjNum 698891824 snap 21875 seq_num 0) dirty dne 2023-10-09T09:08:20.176 INFO:tasks.rados.rados.0.smithi163.stdout:693: expect (ObjNum 137 snap 28 seq_num 137) 2023-10-09T09:08:20.261 INFO:tasks.rados.rados.0.smithi163.stdout:696: finishing write tid 1 to smithi16367185-37 2023-10-09T09:08:20.261 INFO:tasks.rados.rados.0.smithi163.stdout:696: finishing write tid 2 to smithi16367185-37 2023-10-09T09:08:20.261 INFO:tasks.rados.rados.0.smithi163.stdout:696: finishing write tid 3 to smithi16367185-37 2023-10-09T09:08:20.261 INFO:tasks.rados.rados.0.smithi163.stdout:696: finishing write tid 4 to smithi16367185-37 2023-10-09T09:08:20.261 INFO:tasks.rados.rados.0.smithi163.stdout:696: finishing write tid 5 to smithi16367185-37 2023-10-09T09:08:20.262 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 384 (ObjNum 291 snap 90 seq_num 291) dirty exists 2023-10-09T09:08:20.262 INFO:tasks.rados.rados.0.smithi163.stdout:696: left oid 37 (ObjNum 291 snap 90 seq_num 291) 2023-10-09T09:08:21.170 INFO:tasks.rados.rados.0.smithi163.stdout:693: done (4 left) 2023-10-09T09:08:21.170 INFO:tasks.rados.rados.0.smithi163.stdout:694: done (3 left) 2023-10-09T09:08:21.170 INFO:tasks.rados.rados.0.smithi163.stdout:695: done (2 left) 2023-10-09T09:08:21.170 INFO:tasks.rados.rados.0.smithi163.stdout:696: done (1 left) 2023-10-09T09:08:21.170 INFO:tasks.rados.rados.0.smithi163.stdout:697: done (0 left) 2023-10-09T09:08:21.170 INFO:tasks.rados.rados.0.smithi163.stdout:698: rollback oid 45 current snap is 91 2023-10-09T09:08:21.171 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 45 to 85 2023-10-09T09:08:21.171 INFO:tasks.rados.rados.0.smithi163.stdout:699: read oid 9 snap -1 2023-10-09T09:08:21.171 INFO:tasks.rados.rados.0.smithi163.stdout:699: expect (ObjNum 269 snap 84 seq_num 269) 2023-10-09T09:08:21.171 INFO:tasks.rados.rados.0.smithi163.stdout:700: write oid 10 current snap is 91 2023-10-09T09:08:21.171 INFO:tasks.rados.rados.0.smithi163.stdout:700: seq_num 292 ranges {507644=794122,1920742=623085} 2023-10-09T09:08:21.179 INFO:tasks.rados.rados.0.smithi163.stdout:700: writing smithi16367185-10 from 507644 to 1301766 tid 1 2023-10-09T09:08:21.186 INFO:tasks.rados.rados.0.smithi163.stdout:700: writing smithi16367185-10 from 1920742 to 2543827 tid 2 2023-10-09T09:08:21.186 INFO:tasks.rados.rados.0.smithi163.stdout:701: write oid 46 current snap is 91 2023-10-09T09:08:21.186 INFO:tasks.rados.rados.0.smithi163.stdout:701: seq_num 293 ranges {524451=469371,1579224=483096,2529878=394573} 2023-10-09T09:08:21.192 INFO:tasks.rados.rados.0.smithi163.stdout:701: writing smithi16367185-46 from 524451 to 993822 tid 1 2023-10-09T09:08:21.199 INFO:tasks.rados.rados.0.smithi163.stdout:701: writing smithi16367185-46 from 1579224 to 2062320 tid 2 2023-10-09T09:08:21.205 INFO:tasks.rados.rados.0.smithi163.stdout:701: writing smithi16367185-46 from 2529878 to 2924451 tid 3 2023-10-09T09:08:21.205 INFO:tasks.rados.rados.0.smithi163.stdout:702: write oid 42 current snap is 91 2023-10-09T09:08:21.205 INFO:tasks.rados.rados.0.smithi163.stdout:702: seq_num 294 ranges {541258=544620,1637706=626754,2840948=100310} 2023-10-09T09:08:21.211 INFO:tasks.rados.rados.0.smithi163.stdout:702: writing smithi16367185-42 from 541258 to 1085878 tid 1 2023-10-09T09:08:21.217 INFO:tasks.rados.rados.0.smithi163.stdout:702: writing smithi16367185-42 from 1637706 to 2264460 tid 2 2023-10-09T09:08:21.222 INFO:tasks.rados.rados.0.smithi163.stdout:702: writing smithi16367185-42 from 2840948 to 2941258 tid 3 2023-10-09T09:08:21.222 INFO:tasks.rados.rados.0.smithi163.stdout:703: write oid 28 current snap is 91 2023-10-09T09:08:21.222 INFO:tasks.rados.rados.0.smithi163.stdout:703: seq_num 295 ranges {558065=619869,1579835=486765,2468371=489694} 2023-10-09T09:08:21.228 INFO:tasks.rados.rados.0.smithi163.stdout:703: writing smithi16367185-28 from 558065 to 1177934 tid 1 2023-10-09T09:08:21.233 INFO:tasks.rados.rados.0.smithi163.stdout:703: writing smithi16367185-28 from 1579835 to 2066600 tid 2 2023-10-09T09:08:21.239 INFO:tasks.rados.rados.0.smithi163.stdout:703: writing smithi16367185-28 from 2468371 to 2958065 tid 3 2023-10-09T09:08:21.239 INFO:tasks.rados.rados.0.smithi163.stdout:704: read oid 47 snap -1 2023-10-09T09:08:21.239 INFO:tasks.rados.rados.0.smithi163.stdout:704: expect deleted 2023-10-09T09:08:21.239 INFO:tasks.rados.rados.0.smithi163.stdout:705: write oid 49 current snap is 91 2023-10-09T09:08:21.239 INFO:tasks.rados.rados.0.smithi163.stdout:698: finishing rollback tid 0 to smithi16367185-45 2023-10-09T09:08:21.239 INFO:tasks.rados.rados.0.smithi163.stdout:698: finishing rollback tid 1 to smithi16367185-45 2023-10-09T09:08:21.239 INFO:tasks.rados.rados.0.smithi163.stdout:698: finishing rollback tid 2 to smithi16367185-45 2023-10-09T09:08:21.240 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 576 (ObjNum 252 snap 76 seq_num 252) dirty exists 2023-10-09T09:08:21.240 INFO:tasks.rados.rados.0.smithi163.stdout:699: expect (ObjNum 269 snap 84 seq_num 269) 2023-10-09T09:08:21.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:21 smithi163 bash[27500]: cluster 2023-10-09T09:08:20.156834+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2023-10-09T09:08:21.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:21 smithi163 bash[27500]: cluster 2023-10-09T09:08:20.163584+0000 mgr.x (mgr.24751) 79 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 282 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:21.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:21 smithi163 bash[27500]: audit 2023-10-09T09:08:20.410094+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:21.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:21 smithi163 bash[33912]: cluster 2023-10-09T09:08:20.156834+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2023-10-09T09:08:21.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:21 smithi163 bash[33912]: cluster 2023-10-09T09:08:20.163584+0000 mgr.x (mgr.24751) 79 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 282 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:21.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:21 smithi163 bash[33912]: audit 2023-10-09T09:08:20.410094+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:21.490 INFO:tasks.rados.rados.0.smithi163.stdout:700: finishing write tid 1 to smithi16367185-10 2023-10-09T09:08:21.490 INFO:tasks.rados.rados.0.smithi163.stdout:700: finishing write tid 2 to smithi16367185-10 2023-10-09T09:08:21.490 INFO:tasks.rados.rados.0.smithi163.stdout:700: finishing write tid 3 to smithi16367185-10 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:700: finishing write tid 4 to smithi16367185-10 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 269 (ObjNum 292 snap 91 seq_num 292) dirty exists 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:700: left oid 10 (ObjNum 292 snap 91 seq_num 292) 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:701: finishing write tid 1 to smithi16367185-46 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:701: finishing write tid 2 to smithi16367185-46 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:701: finishing write tid 3 to smithi16367185-46 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:701: finishing write tid 4 to smithi16367185-46 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:701: finishing write tid 5 to smithi16367185-46 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 320 (ObjNum 293 snap 91 seq_num 293) dirty exists 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:701: left oid 46 (ObjNum 293 snap 91 seq_num 293) 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:702: finishing write tid 1 to smithi16367185-42 2023-10-09T09:08:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:702: finishing write tid 2 to smithi16367185-42 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:702: finishing write tid 3 to smithi16367185-42 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:702: finishing write tid 4 to smithi16367185-42 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:702: finishing write tid 5 to smithi16367185-42 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 581 (ObjNum 294 snap 91 seq_num 294) dirty exists 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:702: left oid 42 (ObjNum 294 snap 91 seq_num 294) 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:703: finishing write tid 1 to smithi16367185-28 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:703: finishing write tid 2 to smithi16367185-28 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:703: finishing write tid 3 to smithi16367185-28 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:703: finishing write tid 4 to smithi16367185-28 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:703: finishing write tid 5 to smithi16367185-28 2023-10-09T09:08:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 403 (ObjNum 295 snap 91 seq_num 295) dirty exists 2023-10-09T09:08:21.493 INFO:tasks.rados.rados.0.smithi163.stdout:703: left oid 28 (ObjNum 295 snap 91 seq_num 295) 2023-10-09T09:08:21.493 INFO:tasks.rados.rados.0.smithi163.stdout:705: seq_num 296 ranges {574872=695118,2038317=630423} 2023-10-09T09:08:21.503 INFO:tasks.rados.rados.0.smithi163.stdout:705: writing smithi16367185-49 from 574872 to 1269990 tid 1 2023-10-09T09:08:21.512 INFO:tasks.rados.rados.0.smithi163.stdout:705: writing smithi16367185-49 from 2038317 to 2668740 tid 2 2023-10-09T09:08:21.512 INFO:tasks.rados.rados.0.smithi163.stdout:698: done (7 left) 2023-10-09T09:08:21.512 INFO:tasks.rados.rados.0.smithi163.stdout:699: done (6 left) 2023-10-09T09:08:21.512 INFO:tasks.rados.rados.0.smithi163.stdout:700: done (5 left) 2023-10-09T09:08:21.512 INFO:tasks.rados.rados.0.smithi163.stdout:701: done (4 left) 2023-10-09T09:08:21.512 INFO:tasks.rados.rados.0.smithi163.stdout:702: done (3 left) 2023-10-09T09:08:21.512 INFO:tasks.rados.rados.0.smithi163.stdout:703: done (2 left) 2023-10-09T09:08:21.512 INFO:tasks.rados.rados.0.smithi163.stdout:704: done (1 left) 2023-10-09T09:08:21.512 INFO:tasks.rados.rados.0.smithi163.stdout:706: read oid 41 snap -1 2023-10-09T09:08:21.512 INFO:tasks.rados.rados.0.smithi163.stdout:706: expect deleted 2023-10-09T09:08:21.513 INFO:tasks.rados.rados.0.smithi163.stdout:707: write oid 29 current snap is 91 2023-10-09T09:08:21.513 INFO:tasks.rados.rados.0.smithi163.stdout:705: finishing write tid 1 to smithi16367185-49 2023-10-09T09:08:21.513 INFO:tasks.rados.rados.0.smithi163.stdout:707: seq_num 297 ranges {591679=486720,1813152=490434} 2023-10-09T09:08:21.517 INFO:tasks.rados.rados.0.smithi163.stdout:707: writing smithi16367185-29 from 591679 to 1078399 tid 1 2023-10-09T09:08:21.524 INFO:tasks.rados.rados.0.smithi163.stdout:707: writing smithi16367185-29 from 1813152 to 2303586 tid 2 2023-10-09T09:08:21.525 INFO:tasks.rados.rados.0.smithi163.stdout:708: write oid 40 current snap is 91 2023-10-09T09:08:21.525 INFO:tasks.rados.rados.0.smithi163.stdout:708: seq_num 298 ranges {608486=561969,1871634=634092,2950640=57846} 2023-10-09T09:08:21.531 INFO:tasks.rados.rados.0.smithi163.stdout:708: writing smithi16367185-40 from 608486 to 1170455 tid 1 2023-10-09T09:08:21.539 INFO:tasks.rados.rados.0.smithi163.stdout:708: writing smithi16367185-40 from 1871634 to 2505726 tid 2 2023-10-09T09:08:21.542 INFO:tasks.rados.rados.0.smithi163.stdout:708: writing smithi16367185-40 from 2950640 to 3008486 tid 3 2023-10-09T09:08:21.542 INFO:tasks.rados.rados.0.smithi163.stdout:709: read oid 46 snap -1 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:709: expect (ObjNum 293 snap 91 seq_num 293) 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:710: delete oid 2 current snap is 91 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:705: finishing write tid 2 to smithi16367185-49 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:705: finishing write tid 3 to smithi16367185-49 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:705: finishing write tid 4 to smithi16367185-49 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 133 (ObjNum 296 snap 91 seq_num 296) dirty exists 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:705: left oid 49 (ObjNum 296 snap 91 seq_num 296) 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:707: finishing write tid 1 to smithi16367185-29 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:707: finishing write tid 2 to smithi16367185-29 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:707: finishing write tid 3 to smithi16367185-29 2023-10-09T09:08:21.543 INFO:tasks.rados.rados.0.smithi163.stdout:707: finishing write tid 4 to smithi16367185-29 2023-10-09T09:08:21.544 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 347 (ObjNum 297 snap 91 seq_num 297) dirty exists 2023-10-09T09:08:21.544 INFO:tasks.rados.rados.0.smithi163.stdout:707: left oid 29 (ObjNum 297 snap 91 seq_num 297) 2023-10-09T09:08:21.544 INFO:tasks.rados.rados.0.smithi163.stdout:708: finishing write tid 1 to smithi16367185-40 2023-10-09T09:08:21.544 INFO:tasks.rados.rados.0.smithi163.stdout:708: finishing write tid 2 to smithi16367185-40 2023-10-09T09:08:21.546 INFO:tasks.rados.rados.0.smithi163.stdout:705: done (5 left) 2023-10-09T09:08:21.546 INFO:tasks.rados.rados.0.smithi163.stdout:706: done (4 left) 2023-10-09T09:08:21.546 INFO:tasks.rados.rados.0.smithi163.stdout:707: done (3 left) 2023-10-09T09:08:21.546 INFO:tasks.rados.rados.0.smithi163.stdout:710: done (2 left) 2023-10-09T09:08:21.546 INFO:tasks.rados.rados.0.smithi163.stdout:711: snap_remove snap 51 2023-10-09T09:08:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:21 smithi181 bash[27712]: cluster 2023-10-09T09:08:20.156834+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2023-10-09T09:08:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:21 smithi181 bash[27712]: cluster 2023-10-09T09:08:20.163584+0000 mgr.x (mgr.24751) 79 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 282 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:21 smithi181 bash[27712]: audit 2023-10-09T09:08:20.410094+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:22.178 INFO:tasks.rados.rados.0.smithi163.stdout:711: done (2 left) 2023-10-09T09:08:22.178 INFO:tasks.rados.rados.0.smithi163.stdout:712: read oid 26 snap -1 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:712: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:713: read oid 49 snap -1 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:713: expect (ObjNum 296 snap 91 seq_num 296) 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:714: rollback oid 11 current snap is 91 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 11 to 85 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:708: finishing write tid 3 to smithi16367185-40 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:708: finishing write tid 4 to smithi16367185-40 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:708: finishing write tid 5 to smithi16367185-40 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 389 (ObjNum 298 snap 91 seq_num 298) dirty exists 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:708: left oid 40 (ObjNum 298 snap 91 seq_num 298) 2023-10-09T09:08:22.179 INFO:tasks.rados.rados.0.smithi163.stdout:709: expect (ObjNum 293 snap 91 seq_num 293) 2023-10-09T09:08:22.350 INFO:tasks.rados.rados.0.smithi163.stdout:714: finishing rollback tid 1 to smithi16367185-11 2023-10-09T09:08:22.350 INFO:tasks.rados.rados.0.smithi163.stdout:712: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:08:22.431 INFO:tasks.rados.rados.0.smithi163.stdout:714: finishing rollback tid 2 to smithi16367185-11 2023-10-09T09:08:22.431 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 152 (ObjNum 148 snap 34 seq_num 148) dirty exists 2023-10-09T09:08:22.432 INFO:tasks.rados.rados.0.smithi163.stdout:713: expect (ObjNum 296 snap 91 seq_num 296) 2023-10-09T09:08:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:22 smithi163 bash[33912]: cluster 2023-10-09T09:08:21.168017+0000 mon.a (mon.0) 1075 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-10-09T09:08:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:22 smithi163 bash[33912]: cluster 2023-10-09T09:08:21.168080+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2023-10-09T09:08:22.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:22 smithi163 bash[33912]: audit 2023-10-09T09:08:21.299110+0000 mon.a (mon.0) 1077 : audit [DBG] from='client.? 172.21.15.163:0/1275051112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:22.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:22 smithi163 bash[27500]: cluster 2023-10-09T09:08:21.168017+0000 mon.a (mon.0) 1075 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-10-09T09:08:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:22 smithi163 bash[27500]: cluster 2023-10-09T09:08:21.168080+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2023-10-09T09:08:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:22 smithi163 bash[27500]: audit 2023-10-09T09:08:21.299110+0000 mon.a (mon.0) 1077 : audit [DBG] from='client.? 172.21.15.163:0/1275051112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:22 smithi181 bash[27712]: cluster 2023-10-09T09:08:21.168017+0000 mon.a (mon.0) 1075 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-10-09T09:08:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:22 smithi181 bash[27712]: cluster 2023-10-09T09:08:21.168080+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2023-10-09T09:08:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:22 smithi181 bash[27712]: audit 2023-10-09T09:08:21.299110+0000 mon.a (mon.0) 1077 : audit [DBG] from='client.? 172.21.15.163:0/1275051112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:22.573 INFO:tasks.rados.rados.0.smithi163.stdout:708: done (4 left) 2023-10-09T09:08:22.573 INFO:tasks.rados.rados.0.smithi163.stdout:709: done (3 left) 2023-10-09T09:08:22.573 INFO:tasks.rados.rados.0.smithi163.stdout:712: done (2 left) 2023-10-09T09:08:22.573 INFO:tasks.rados.rados.0.smithi163.stdout:713: done (1 left) 2023-10-09T09:08:22.574 INFO:tasks.rados.rados.0.smithi163.stdout:714: done (0 left) 2023-10-09T09:08:22.574 INFO:tasks.rados.rados.0.smithi163.stdout:715: write oid 50 current snap is 91 2023-10-09T09:08:22.574 INFO:tasks.rados.rados.0.smithi163.stdout:715: seq_num 300 ranges {642100=712467,1872245=637761} 2023-10-09T09:08:22.584 INFO:tasks.rados.rados.0.smithi163.stdout:715: writing smithi16367185-50 from 642100 to 1354567 tid 1 2023-10-09T09:08:22.591 INFO:tasks.rados.rados.0.smithi163.stdout:715: writing smithi16367185-50 from 1872245 to 2510006 tid 2 2023-10-09T09:08:22.591 INFO:tasks.rados.rados.0.smithi163.stdout:716: write oid 29 current snap is 91 2023-10-09T09:08:22.591 INFO:tasks.rados.rados.0.smithi163.stdout:716: seq_num 301 ranges {658907=787716,1930727=497772,3032909=25998} 2023-10-09T09:08:22.599 INFO:tasks.rados.rados.0.smithi163.stdout:716: writing smithi16367185-29 from 658907 to 1446623 tid 1 2023-10-09T09:08:22.605 INFO:tasks.rados.rados.0.smithi163.stdout:716: writing smithi16367185-29 from 1930727 to 2428499 tid 2 2023-10-09T09:08:22.610 INFO:tasks.rados.rados.0.smithi163.stdout:716: writing smithi16367185-29 from 3032909 to 3058907 tid 3 2023-10-09T09:08:22.610 INFO:tasks.rados.rados.0.smithi163.stdout:717: write oid 31 current snap is 91 2023-10-09T09:08:22.610 INFO:tasks.rados.rados.0.smithi163.stdout:715: finishing write tid 1 to smithi16367185-50 2023-10-09T09:08:22.611 INFO:tasks.rados.rados.0.smithi163.stdout:717: seq_num 302 ranges {675714=462965,1589209=641430,2943979=131735} 2023-10-09T09:08:22.617 INFO:tasks.rados.rados.0.smithi163.stdout:717: writing smithi16367185-31 from 675714 to 1138679 tid 1 2023-10-09T09:08:22.623 INFO:tasks.rados.rados.0.smithi163.stdout:717: writing smithi16367185-31 from 1589209 to 2230639 tid 2 2023-10-09T09:08:22.629 INFO:tasks.rados.rados.0.smithi163.stdout:717: writing smithi16367185-31 from 2943979 to 3075714 tid 3 2023-10-09T09:08:22.629 INFO:tasks.rados.rados.0.smithi163.stdout:718: write oid 15 current snap is 91 2023-10-09T09:08:22.629 INFO:tasks.rados.rados.0.smithi163.stdout:715: finishing write tid 2 to smithi16367185-50 2023-10-09T09:08:22.630 INFO:tasks.rados.rados.0.smithi163.stdout:718: seq_num 303 ranges {692521=538214,1931338=785088} 2023-10-09T09:08:22.635 INFO:tasks.rados.rados.0.smithi163.stdout:718: writing smithi16367185-15 from 692521 to 1230735 tid 1 2023-10-09T09:08:22.644 INFO:tasks.rados.rados.0.smithi163.stdout:718: writing smithi16367185-15 from 1931338 to 2716426 tid 2 2023-10-09T09:08:22.644 INFO:tasks.rados.rados.0.smithi163.stdout:719: snap_create 2023-10-09T09:08:22.644 INFO:tasks.rados.rados.0.smithi163.stdout:715: finishing write tid 3 to smithi16367185-50 2023-10-09T09:08:22.644 INFO:tasks.rados.rados.0.smithi163.stdout:715: finishing write tid 4 to smithi16367185-50 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 269 (ObjNum 300 snap 91 seq_num 300) dirty exists 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:715: left oid 50 (ObjNum 300 snap 91 seq_num 300) 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:716: finishing write tid 1 to smithi16367185-29 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:716: finishing write tid 2 to smithi16367185-29 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:716: finishing write tid 3 to smithi16367185-29 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:716: finishing write tid 4 to smithi16367185-29 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:716: finishing write tid 5 to smithi16367185-29 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 357 (ObjNum 301 snap 91 seq_num 301) dirty exists 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:716: left oid 29 (ObjNum 301 snap 91 seq_num 301) 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:717: finishing write tid 1 to smithi16367185-31 2023-10-09T09:08:22.645 INFO:tasks.rados.rados.0.smithi163.stdout:717: finishing write tid 2 to smithi16367185-31 2023-10-09T09:08:22.646 INFO:tasks.rados.rados.0.smithi163.stdout:717: finishing write tid 3 to smithi16367185-31 2023-10-09T09:08:22.646 INFO:tasks.rados.rados.0.smithi163.stdout:717: finishing write tid 4 to smithi16367185-31 2023-10-09T09:08:22.646 INFO:tasks.rados.rados.0.smithi163.stdout:717: finishing write tid 5 to smithi16367185-31 2023-10-09T09:08:22.646 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 221 (ObjNum 302 snap 91 seq_num 302) dirty exists 2023-10-09T09:08:22.646 INFO:tasks.rados.rados.0.smithi163.stdout:717: left oid 31 (ObjNum 302 snap 91 seq_num 302) 2023-10-09T09:08:22.646 INFO:tasks.rados.rados.0.smithi163.stdout:718: finishing write tid 1 to smithi16367185-15 2023-10-09T09:08:22.649 INFO:tasks.rados.rados.0.smithi163.stdout:718: finishing write tid 2 to smithi16367185-15 2023-10-09T09:08:22.649 INFO:tasks.rados.rados.0.smithi163.stdout:718: finishing write tid 3 to smithi16367185-15 2023-10-09T09:08:22.650 INFO:tasks.rados.rados.0.smithi163.stdout:718: finishing write tid 4 to smithi16367185-15 2023-10-09T09:08:22.650 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 401 (ObjNum 303 snap 91 seq_num 303) dirty exists 2023-10-09T09:08:22.650 INFO:tasks.rados.rados.0.smithi163.stdout:718: left oid 15 (ObjNum 303 snap 91 seq_num 303) 2023-10-09T09:08:23.186 INFO:tasks.rados.rados.0.smithi163.stdout:715: done (4 left) 2023-10-09T09:08:23.187 INFO:tasks.rados.rados.0.smithi163.stdout:716: done (3 left) 2023-10-09T09:08:23.187 INFO:tasks.rados.rados.0.smithi163.stdout:717: done (2 left) 2023-10-09T09:08:23.187 INFO:tasks.rados.rados.0.smithi163.stdout:718: done (1 left) 2023-10-09T09:08:23.187 INFO:tasks.rados.rados.0.smithi163.stdout:719: done (0 left) 2023-10-09T09:08:23.187 INFO:tasks.rados.rados.0.smithi163.stdout:720: snap_remove snap 84 2023-10-09T09:08:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:720: done (0 left) 2023-10-09T09:08:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:721: read oid 6 snap -1 2023-10-09T09:08:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:721: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:08:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:722: delete oid 29 current snap is 92 2023-10-09T09:08:23.467 INFO:tasks.rados.rados.0.smithi163.stdout:722: done (1 left) 2023-10-09T09:08:23.467 INFO:tasks.rados.rados.0.smithi163.stdout:723: write oid 46 current snap is 92 2023-10-09T09:08:23.467 INFO:tasks.rados.rados.0.smithi163.stdout:723: seq_num 305 ranges {726135=405065,1764655=788757,3026248=99887} 2023-10-09T09:08:23.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[27500]: cluster 2023-10-09T09:08:22.164789+0000 mgr.x (mgr.24751) 80 : cluster [DBG] pgmap v140: 89 pgs: 89 active+clean; 293 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 991 KiB/s wr, 12 op/s 2023-10-09T09:08:23.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[27500]: cluster 2023-10-09T09:08:22.177464+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2023-10-09T09:08:23.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[33912]: cluster 2023-10-09T09:08:22.164789+0000 mgr.x (mgr.24751) 80 : cluster [DBG] pgmap v140: 89 pgs: 89 active+clean; 293 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 991 KiB/s wr, 12 op/s 2023-10-09T09:08:23.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[33912]: cluster 2023-10-09T09:08:22.177464+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2023-10-09T09:08:23.475 INFO:tasks.rados.rados.0.smithi163.stdout:723: writing smithi16367185-46 from 726135 to 1131200 tid 1 2023-10-09T09:08:23.484 INFO:tasks.rados.rados.0.smithi163.stdout:723: writing smithi16367185-46 from 1764655 to 2553412 tid 2 2023-10-09T09:08:23.490 INFO:tasks.rados.rados.0.smithi163.stdout:723: writing smithi16367185-46 from 3026248 to 3126135 tid 3 2023-10-09T09:08:23.491 INFO:tasks.rados.rados.0.smithi163.stdout:724: read oid 18 snap -1 2023-10-09T09:08:23.491 INFO:tasks.rados.rados.0.smithi163.stdout:724: expect (ObjNum 265 snap 80 seq_num 265) 2023-10-09T09:08:23.491 INFO:tasks.rados.rados.0.smithi163.stdout:725: rollback oid 35 current snap is 92 2023-10-09T09:08:23.491 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 35 to 60 2023-10-09T09:08:23.491 INFO:tasks.rados.rados.0.smithi163.stdout:721: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:08:23.495 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:23 smithi181 bash[27712]: cluster 2023-10-09T09:08:22.164789+0000 mgr.x (mgr.24751) 80 : cluster [DBG] pgmap v140: 89 pgs: 89 active+clean; 293 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 991 KiB/s wr, 12 op/s 2023-10-09T09:08:23.496 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:23 smithi181 bash[27712]: cluster 2023-10-09T09:08:22.177464+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2023-10-09T09:08:23.747 INFO:tasks.rados.rados.0.smithi163.stdout:723: finishing write tid 1 to smithi16367185-46 2023-10-09T09:08:23.748 INFO:tasks.rados.rados.0.smithi163.stdout:723: finishing write tid 2 to smithi16367185-46 2023-10-09T09:08:23.748 INFO:tasks.rados.rados.0.smithi163.stdout:721: done (3 left) 2023-10-09T09:08:23.748 INFO:tasks.rados.rados.0.smithi163.stdout:726: write oid 31 current snap is 92 2023-10-09T09:08:23.748 INFO:tasks.rados.rados.0.smithi163.stdout:726: seq_num 306 ranges {742942=480314,1823137=648768} 2023-10-09T09:08:23.748 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: [09/Oct/2023:09:08:23] ENGINE Bus STOPPING 2023-10-09T09:08:23.749 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: [09/Oct/2023:09:08:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:08:23.749 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: [09/Oct/2023:09:08:23] ENGINE Bus STOPPED 2023-10-09T09:08:23.749 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: [09/Oct/2023:09:08:23] ENGINE Bus STARTING 2023-10-09T09:08:23.749 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: [09/Oct/2023:09:08:23] ENGINE Serving on http://:::9283 2023-10-09T09:08:23.749 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: [09/Oct/2023:09:08:23] ENGINE Bus STARTED 2023-10-09T09:08:23.749 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: debug 2023-10-09T09:08:23.712+0000 7f9677921700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:08:23.749 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: debug 2023-10-09T09:08:23.712+0000 7f9677921700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:08:23.749 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: debug 2023-10-09T09:08:23.712+0000 7f9677921700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:08:23.749 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: debug 2023-10-09T09:08:23.712+0000 7f9677921700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:08:23.750 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: debug 2023-10-09T09:08:23.712+0000 7f9677921700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:08:23.750 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: debug 2023-10-09T09:08:23.712+0000 7f967491b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:08:23.750 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: debug 2023-10-09T09:08:23.712+0000 7f967491b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:08:23.750 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: debug 2023-10-09T09:08:23.712+0000 7f967491b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:08:23.750 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: debug 2023-10-09T09:08:23.712+0000 7f967491b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:08:23.750 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:23 smithi163 bash[71599]: debug 2023-10-09T09:08:23.712+0000 7f967491b700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:08:23.760 INFO:tasks.rados.rados.0.smithi163.stdout:726: writing smithi16367185-31 from 742942 to 1223256 tid 1 2023-10-09T09:08:23.765 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:23 smithi181 bash[50540]: ignoring --setuser ceph since I am not root 2023-10-09T09:08:23.765 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:23 smithi181 bash[50540]: ignoring --setgroup ceph since I am not root 2023-10-09T09:08:23.766 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:23 smithi181 bash[50540]: debug 2023-10-09T09:08:23.517+0000 7fed35518700 1 -- 172.21.15.181:0/3880322548 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5611594f31e0 con 0x56115a272800 2023-10-09T09:08:23.766 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:23 smithi181 bash[50540]: debug 2023-10-09T09:08:23.685+0000 7fed3f248200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:08:23.774 INFO:tasks.rados.rados.0.smithi163.stdout:726: writing smithi16367185-31 from 1823137 to 2471905 tid 2 2023-10-09T09:08:23.774 INFO:tasks.rados.rados.0.smithi163.stdout:727: write oid 32 current snap is 92 2023-10-09T09:08:23.774 INFO:tasks.rados.rados.0.smithi163.stdout:727: seq_num 307 ranges {759749=555563,1765266=792426,2964741=195008} 2023-10-09T09:08:23.786 INFO:tasks.rados.rados.0.smithi163.stdout:727: writing smithi16367185-32 from 759749 to 1315312 tid 1 2023-10-09T09:08:23.800 INFO:tasks.rados.rados.0.smithi163.stdout:727: writing smithi16367185-32 from 1765266 to 2557692 tid 2 2023-10-09T09:08:23.805 INFO:tasks.rados.rados.0.smithi163.stdout:727: writing smithi16367185-32 from 2964741 to 3159749 tid 3 2023-10-09T09:08:23.806 INFO:tasks.rados.rados.0.smithi163.stdout:728: delete oid 7 current snap is 92 2023-10-09T09:08:23.807 INFO:tasks.rados.rados.0.smithi163.stdout:725: finishing rollback tid 0 to smithi16367185-35 2023-10-09T09:08:23.807 INFO:tasks.rados.rados.0.smithi163.stdout:723: finishing write tid 3 to smithi16367185-46 2023-10-09T09:08:23.807 INFO:tasks.rados.rados.0.smithi163.stdout:723: finishing write tid 4 to smithi16367185-46 2023-10-09T09:08:23.807 INFO:tasks.rados.rados.0.smithi163.stdout:723: finishing write tid 5 to smithi16367185-46 2023-10-09T09:08:23.807 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 331 (ObjNum 305 snap 92 seq_num 305) dirty exists 2023-10-09T09:08:23.808 INFO:tasks.rados.rados.0.smithi163.stdout:723: left oid 46 (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:08:23.808 INFO:tasks.rados.rados.0.smithi163.stdout:725: finishing rollback tid 1 to smithi16367185-35 2023-10-09T09:08:23.808 INFO:tasks.rados.rados.0.smithi163.stdout:725: finishing rollback tid 2 to smithi16367185-35 2023-10-09T09:08:23.808 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 162 (ObjNum 196 snap 54 seq_num 196) dirty exists 2023-10-09T09:08:23.808 INFO:tasks.rados.rados.0.smithi163.stdout:724: expect (ObjNum 265 snap 80 seq_num 265) 2023-10-09T09:08:23.964 INFO:tasks.rados.rados.0.smithi163.stdout:726: finishing write tid 1 to smithi16367185-31 2023-10-09T09:08:23.964 INFO:tasks.rados.rados.0.smithi163.stdout:726: finishing write tid 2 to smithi16367185-31 2023-10-09T09:08:23.964 INFO:tasks.rados.rados.0.smithi163.stdout:726: finishing write tid 3 to smithi16367185-31 2023-10-09T09:08:23.965 INFO:tasks.rados.rados.0.smithi163.stdout:726: finishing write tid 4 to smithi16367185-31 2023-10-09T09:08:23.965 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 225 (ObjNum 306 snap 92 seq_num 306) dirty exists 2023-10-09T09:08:23.965 INFO:tasks.rados.rados.0.smithi163.stdout:726: left oid 31 (ObjNum 306 snap 92 seq_num 306) 2023-10-09T09:08:23.965 INFO:tasks.rados.rados.0.smithi163.stdout:727: finishing write tid 1 to smithi16367185-32 2023-10-09T09:08:23.965 INFO:tasks.rados.rados.0.smithi163.stdout:727: finishing write tid 2 to smithi16367185-32 2023-10-09T09:08:23.965 INFO:tasks.rados.rados.0.smithi163.stdout:727: finishing write tid 3 to smithi16367185-32 2023-10-09T09:08:23.965 INFO:tasks.rados.rados.0.smithi163.stdout:727: finishing write tid 4 to smithi16367185-32 2023-10-09T09:08:23.965 INFO:tasks.rados.rados.0.smithi163.stdout:727: finishing write tid 5 to smithi16367185-32 2023-10-09T09:08:23.965 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 598 (ObjNum 307 snap 92 seq_num 307) dirty exists 2023-10-09T09:08:23.965 INFO:tasks.rados.rados.0.smithi163.stdout:727: left oid 32 (ObjNum 307 snap 92 seq_num 307) 2023-10-09T09:08:23.967 INFO:tasks.rados.rados.0.smithi163.stdout:723: done (5 left) 2023-10-09T09:08:23.967 INFO:tasks.rados.rados.0.smithi163.stdout:724: done (4 left) 2023-10-09T09:08:23.967 INFO:tasks.rados.rados.0.smithi163.stdout:725: done (3 left) 2023-10-09T09:08:23.967 INFO:tasks.rados.rados.0.smithi163.stdout:726: done (2 left) 2023-10-09T09:08:23.967 INFO:tasks.rados.rados.0.smithi163.stdout:727: done (1 left) 2023-10-09T09:08:23.967 INFO:tasks.rados.rados.0.smithi163.stdout:728: done (0 left) 2023-10-09T09:08:23.967 INFO:tasks.rados.rados.0.smithi163.stdout:729: snap_remove snap 20 2023-10-09T09:08:24.190 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: cluster 2023-10-09T09:08:23.186203+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2023-10-09T09:08:24.190 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.316988+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:24.190 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.327020+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:24.191 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.327708+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:24.191 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.329265+0000 mon.b (mon.1) 95 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:08:24.191 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.339607+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:24.191 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.422724+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:24.191 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.432486+0000 mon.b (mon.1) 97 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:08:24.191 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.434855+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:08:24.191 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: cluster 2023-10-09T09:08:23.446736+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2023-10-09T09:08:24.191 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.449055+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T09:08:24.191 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: cluster 2023-10-09T09:08:23.449148+0000 mon.a (mon.0) 1086 : cluster [DBG] mgrmap e26: y(active, starting, since 0.0135074s) 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.459756+0000 mon.a (mon.0) 1087 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.460089+0000 mon.a (mon.0) 1088 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.460372+0000 mon.a (mon.0) 1089 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.460565+0000 mon.a (mon.0) 1090 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.460978+0000 mon.a (mon.0) 1091 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.461432+0000 mon.a (mon.0) 1092 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.462312+0000 mon.a (mon.0) 1093 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.462668+0000 mon.a (mon.0) 1094 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.463170+0000 mon.a (mon.0) 1095 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.463515+0000 mon.a (mon.0) 1096 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:08:24.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.463771+0000 mon.a (mon.0) 1097 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.464047+0000 mon.a (mon.0) 1098 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.464338+0000 mon.a (mon.0) 1099 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.464673+0000 mon.a (mon.0) 1100 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.467645+0000 mon.a (mon.0) 1101 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.468164+0000 mon.a (mon.0) 1102 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.468901+0000 mon.a (mon.0) 1103 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: cluster 2023-10-09T09:08:23.514544+0000 mon.a (mon.0) 1104 : cluster [INF] Manager daemon y is now available 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.550826+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.567625+0000 mon.a (mon.0) 1106 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.600741+0000 mon.a (mon.0) 1107 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.606001+0000 mon.a (mon.0) 1108 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:08:24.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.652765+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:08:24.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[27500]: audit 2023-10-09T09:08:23.921397+0000 mon.a (mon.0) 1110 : audit [DBG] from='client.? 172.21.15.163:0/1703747826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:24.462 INFO:tasks.rados.rados.0.smithi163.stdout:729: done (0 left) 2023-10-09T09:08:24.462 INFO:tasks.rados.rados.0.smithi163.stdout:730: write oid 45 current snap is 92 2023-10-09T09:08:24.462 INFO:tasks.rados.rados.0.smithi163.stdout:730: seq_num 309 ranges {793363=706061,2282230=796095} 2023-10-09T09:08:24.467 INFO:tasks.rados.rados.0.smithi163.stdout:730: writing smithi16367185-45 from 793363 to 1499424 tid 1 2023-10-09T09:08:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: cluster 2023-10-09T09:08:23.186203+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2023-10-09T09:08:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.316988+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.327020+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.327708+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.329265+0000 mon.b (mon.1) 95 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:08:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.339607+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.422724+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.432486+0000 mon.b (mon.1) 97 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:08:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.434855+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: cluster 2023-10-09T09:08:23.446736+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.449055+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: cluster 2023-10-09T09:08:23.449148+0000 mon.a (mon.0) 1086 : cluster [DBG] mgrmap e26: y(active, starting, since 0.0135074s) 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.459756+0000 mon.a (mon.0) 1087 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.460089+0000 mon.a (mon.0) 1088 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.460372+0000 mon.a (mon.0) 1089 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.460565+0000 mon.a (mon.0) 1090 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.460978+0000 mon.a (mon.0) 1091 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.461432+0000 mon.a (mon.0) 1092 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.462312+0000 mon.a (mon.0) 1093 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.462668+0000 mon.a (mon.0) 1094 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:08:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.463170+0000 mon.a (mon.0) 1095 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.463515+0000 mon.a (mon.0) 1096 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.463771+0000 mon.a (mon.0) 1097 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.464047+0000 mon.a (mon.0) 1098 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.464338+0000 mon.a (mon.0) 1099 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.464673+0000 mon.a (mon.0) 1100 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.467645+0000 mon.a (mon.0) 1101 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.468164+0000 mon.a (mon.0) 1102 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.468901+0000 mon.a (mon.0) 1103 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: cluster 2023-10-09T09:08:23.514544+0000 mon.a (mon.0) 1104 : cluster [INF] Manager daemon y is now available 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.550826+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.567625+0000 mon.a (mon.0) 1106 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.600741+0000 mon.a (mon.0) 1107 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:24.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.606001+0000 mon.a (mon.0) 1108 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:08:24.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.652765+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:08:24.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:24 smithi163 bash[33912]: audit 2023-10-09T09:08:23.921397+0000 mon.a (mon.0) 1110 : audit [DBG] from='client.? 172.21.15.163:0/1703747826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:24.477 INFO:tasks.rados.rados.0.smithi163.stdout:730: writing smithi16367185-45 from 2282230 to 3078325 tid 2 2023-10-09T09:08:24.478 INFO:tasks.rados.rados.0.smithi163.stdout:731: delete oid 19 current snap is 92 2023-10-09T09:08:24.478 INFO:tasks.rados.rados.0.smithi163.stdout:730: finishing write tid 1 to smithi16367185-45 2023-10-09T09:08:24.484 INFO:tasks.rados.rados.0.smithi163.stdout:731: done (1 left) 2023-10-09T09:08:24.484 INFO:tasks.rados.rados.0.smithi163.stdout:732: write oid 34 current snap is 92 2023-10-09T09:08:24.485 INFO:tasks.rados.rados.0.smithi163.stdout:732: seq_num 311 ranges {426977=456559,1482841=799764,2958080=268897} 2023-10-09T09:08:24.489 INFO:tasks.rados.rados.0.smithi163.stdout:732: writing smithi16367185-34 from 426977 to 883536 tid 1 2023-10-09T09:08:24.498 INFO:tasks.rados.rados.0.smithi163.stdout:732: writing smithi16367185-34 from 1482841 to 2282605 tid 2 2023-10-09T09:08:24.503 INFO:tasks.rados.rados.0.smithi163.stdout:732: writing smithi16367185-34 from 2958080 to 3226977 tid 3 2023-10-09T09:08:24.503 INFO:tasks.rados.rados.0.smithi163.stdout:733: write oid 33 current snap is 92 2023-10-09T09:08:24.503 INFO:tasks.rados.rados.0.smithi163.stdout:733: seq_num 312 ranges {443784=648161,1657676=659775,2818209=425575} 2023-10-09T09:08:24.509 INFO:tasks.rados.rados.0.smithi163.stdout:733: writing smithi16367185-33 from 443784 to 1091945 tid 1 2023-10-09T09:08:24.516 INFO:tasks.rados.rados.0.smithi163.stdout:733: writing smithi16367185-33 from 1657676 to 2317451 tid 2 2023-10-09T09:08:24.522 INFO:tasks.rados.rados.0.smithi163.stdout:733: writing smithi16367185-33 from 2818209 to 3243784 tid 3 2023-10-09T09:08:24.522 INFO:tasks.rados.rados.0.smithi163.stdout:734: rollback oid 37 current snap is 92 2023-10-09T09:08:24.523 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 88 2023-10-09T09:08:24.523 INFO:tasks.rados.rados.0.smithi163.stdout:730: finishing write tid 2 to smithi16367185-45 2023-10-09T09:08:24.523 INFO:tasks.rados.rados.0.smithi163.stdout:730: finishing write tid 3 to smithi16367185-45 2023-10-09T09:08:24.523 INFO:tasks.rados.rados.0.smithi163.stdout:730: finishing write tid 4 to smithi16367185-45 2023-10-09T09:08:24.523 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 614 (ObjNum 309 snap 92 seq_num 309) dirty exists 2023-10-09T09:08:24.523 INFO:tasks.rados.rados.0.smithi163.stdout:730: left oid 45 (ObjNum 309 snap 92 seq_num 309) 2023-10-09T09:08:24.523 INFO:tasks.rados.rados.0.smithi163.stdout:732: finishing write tid 1 to smithi16367185-34 2023-10-09T09:08:24.523 INFO:tasks.rados.rados.0.smithi163.stdout:732: finishing write tid 2 to smithi16367185-34 2023-10-09T09:08:24.523 INFO:tasks.rados.rados.0.smithi163.stdout:732: finishing write tid 3 to smithi16367185-34 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:732: finishing write tid 4 to smithi16367185-34 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:732: finishing write tid 5 to smithi16367185-34 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 282 (ObjNum 311 snap 92 seq_num 311) dirty exists 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:732: left oid 34 (ObjNum 311 snap 92 seq_num 311) 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:733: finishing write tid 1 to smithi16367185-33 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:733: finishing write tid 2 to smithi16367185-33 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:730: done (3 left) 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:732: done (2 left) 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:735: rollback oid 23 current snap is 92 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 23 to 91 2023-10-09T09:08:24.524 INFO:tasks.rados.rados.0.smithi163.stdout:736: read oid 17 snap 91 2023-10-09T09:08:24.525 INFO:tasks.rados.rados.0.smithi163.stdout:736: expect (ObjNum 137 snap 28 seq_num 137) 2023-10-09T09:08:24.525 INFO:tasks.rados.rados.0.smithi163.stdout:737: read oid 22 snap -1 2023-10-09T09:08:24.525 INFO:tasks.rados.rados.0.smithi163.stdout:737: expect deleted 2023-10-09T09:08:24.525 INFO:tasks.rados.rados.0.smithi163.stdout:738: snap_create 2023-10-09T09:08:24.526 INFO:tasks.rados.rados.0.smithi163.stdout:733: finishing write tid 3 to smithi16367185-33 2023-10-09T09:08:24.526 INFO:tasks.rados.rados.0.smithi163.stdout:733: finishing write tid 4 to smithi16367185-33 2023-10-09T09:08:24.526 INFO:tasks.rados.rados.0.smithi163.stdout:735: finishing rollback tid 1 to smithi16367185-23 2023-10-09T09:08:24.526 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 185 (ObjNum 2 snap 0 seq_num 1) dirty dne 2023-10-09T09:08:24.526 INFO:tasks.rados.rados.0.smithi163.stdout:733: finishing write tid 5 to smithi16367185-33 2023-10-09T09:08:24.526 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 371 (ObjNum 312 snap 92 seq_num 312) dirty exists 2023-10-09T09:08:24.526 INFO:tasks.rados.rados.0.smithi163.stdout:733: left oid 33 (ObjNum 312 snap 92 seq_num 312) 2023-10-09T09:08:24.526 INFO:tasks.rados.rados.0.smithi163.stdout:734: finishing rollback tid 0 to smithi16367185-37 2023-10-09T09:08:24.532 INFO:tasks.rados.rados.0.smithi163.stdout:734: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:08:24.532 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 416 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:08:24.533 INFO:tasks.rados.rados.0.smithi163.stdout:736: expect (ObjNum 137 snap 28 seq_num 137) 2023-10-09T09:08:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: cluster 2023-10-09T09:08:23.186203+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2023-10-09T09:08:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.316988+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.327020+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.327708+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.329265+0000 mon.b (mon.1) 95 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:08:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.339607+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.24751 ' entity='mgr.x' 2023-10-09T09:08:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.422724+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.432486+0000 mon.b (mon.1) 97 : audit [INF] from='mgr.24751 172.21.15.181:0/50106539' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.434855+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: cluster 2023-10-09T09:08:23.446736+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.449055+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24751 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: cluster 2023-10-09T09:08:23.449148+0000 mon.a (mon.0) 1086 : cluster [DBG] mgrmap e26: y(active, starting, since 0.0135074s) 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.459756+0000 mon.a (mon.0) 1087 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.460089+0000 mon.a (mon.0) 1088 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.460372+0000 mon.a (mon.0) 1089 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.460565+0000 mon.a (mon.0) 1090 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.460978+0000 mon.a (mon.0) 1091 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.461432+0000 mon.a (mon.0) 1092 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.462312+0000 mon.a (mon.0) 1093 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:08:24.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.462668+0000 mon.a (mon.0) 1094 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.463170+0000 mon.a (mon.0) 1095 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.463515+0000 mon.a (mon.0) 1096 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.463771+0000 mon.a (mon.0) 1097 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.464047+0000 mon.a (mon.0) 1098 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.464338+0000 mon.a (mon.0) 1099 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.464673+0000 mon.a (mon.0) 1100 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.467645+0000 mon.a (mon.0) 1101 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.468164+0000 mon.a (mon.0) 1102 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.468901+0000 mon.a (mon.0) 1103 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: cluster 2023-10-09T09:08:23.514544+0000 mon.a (mon.0) 1104 : cluster [INF] Manager daemon y is now available 2023-10-09T09:08:24.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.550826+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:24.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.567625+0000 mon.a (mon.0) 1106 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:24.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.600741+0000 mon.a (mon.0) 1107 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:24.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.606001+0000 mon.a (mon.0) 1108 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:08:24.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.652765+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:08:24.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[27712]: audit 2023-10-09T09:08:23.921397+0000 mon.a (mon.0) 1110 : audit [DBG] from='client.? 172.21.15.163:0/1703747826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:25.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[50540]: debug 2023-10-09T09:08:24.689+0000 7fed3f248200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:08:25.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[50540]: debug 2023-10-09T09:08:24.801+0000 7fed3f248200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:08:25.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:24 smithi181 bash[50540]: debug 2023-10-09T09:08:24.913+0000 7fed3f248200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:08:25.448 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[50540]: debug 2023-10-09T09:08:25.193+0000 7fed3f248200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:08:25.469 INFO:tasks.rados.rados.0.smithi163.stdout:733: done (5 left) 2023-10-09T09:08:25.469 INFO:tasks.rados.rados.0.smithi163.stdout:734: done (4 left) 2023-10-09T09:08:25.469 INFO:tasks.rados.rados.0.smithi163.stdout:735: done (3 left) 2023-10-09T09:08:25.469 INFO:tasks.rados.rados.0.smithi163.stdout:736: done (2 left) 2023-10-09T09:08:25.469 INFO:tasks.rados.rados.0.smithi163.stdout:737: done (1 left) 2023-10-09T09:08:25.469 INFO:tasks.rados.rados.0.smithi163.stdout:738: done (0 left) 2023-10-09T09:08:25.469 INFO:tasks.rados.rados.0.smithi163.stdout:739: snap_create 2023-10-09T09:08:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[27500]: cephadm 2023-10-09T09:08:24.247589+0000 mgr.y (mgr.15273) 1 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Bus STARTING 2023-10-09T09:08:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[27500]: cephadm 2023-10-09T09:08:24.349971+0000 mgr.y (mgr.15273) 2 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:08:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[27500]: cluster 2023-10-09T09:08:24.452394+0000 mon.a (mon.0) 1111 : cluster [DBG] mgrmap e27: y(active, since 1.01675s) 2023-10-09T09:08:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[27500]: cluster 2023-10-09T09:08:24.457771+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2023-10-09T09:08:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[27500]: cephadm 2023-10-09T09:08:24.484563+0000 mgr.y (mgr.15273) 3 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:08:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[27500]: cephadm 2023-10-09T09:08:24.485051+0000 mgr.y (mgr.15273) 4 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Bus STARTED 2023-10-09T09:08:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[27500]: cluster 2023-10-09T09:08:24.491851+0000 mgr.y (mgr.15273) 5 : cluster [DBG] pgmap v4: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[33912]: cephadm 2023-10-09T09:08:24.247589+0000 mgr.y (mgr.15273) 1 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Bus STARTING 2023-10-09T09:08:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[33912]: cephadm 2023-10-09T09:08:24.349971+0000 mgr.y (mgr.15273) 2 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:08:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[33912]: cluster 2023-10-09T09:08:24.452394+0000 mon.a (mon.0) 1111 : cluster [DBG] mgrmap e27: y(active, since 1.01675s) 2023-10-09T09:08:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[33912]: cluster 2023-10-09T09:08:24.457771+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2023-10-09T09:08:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[33912]: cephadm 2023-10-09T09:08:24.484563+0000 mgr.y (mgr.15273) 3 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:08:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[33912]: cephadm 2023-10-09T09:08:24.485051+0000 mgr.y (mgr.15273) 4 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Bus STARTED 2023-10-09T09:08:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:25 smithi163 bash[33912]: cluster 2023-10-09T09:08:24.491851+0000 mgr.y (mgr.15273) 5 : cluster [DBG] pgmap v4: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:25.724 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[50540]: debug 2023-10-09T09:08:25.561+0000 7fed3f248200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:08:25.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[27712]: cephadm 2023-10-09T09:08:24.247589+0000 mgr.y (mgr.15273) 1 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Bus STARTING 2023-10-09T09:08:25.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[27712]: cephadm 2023-10-09T09:08:24.349971+0000 mgr.y (mgr.15273) 2 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:08:25.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[27712]: cluster 2023-10-09T09:08:24.452394+0000 mon.a (mon.0) 1111 : cluster [DBG] mgrmap e27: y(active, since 1.01675s) 2023-10-09T09:08:25.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[27712]: cluster 2023-10-09T09:08:24.457771+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2023-10-09T09:08:25.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[27712]: cephadm 2023-10-09T09:08:24.484563+0000 mgr.y (mgr.15273) 3 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:08:25.725 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[27712]: cephadm 2023-10-09T09:08:24.485051+0000 mgr.y (mgr.15273) 4 : cephadm [INF] [09/Oct/2023:09:08:24] ENGINE Bus STARTED 2023-10-09T09:08:25.725 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[27712]: cluster 2023-10-09T09:08:24.491851+0000 mgr.y (mgr.15273) 5 : cluster [DBG] pgmap v4: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:26.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[50540]: debug 2023-10-09T09:08:25.721+0000 7fed3f248200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:08:26.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:25 smithi181 bash[50540]: debug 2023-10-09T09:08:25.933+0000 7fed3f248200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:08:26.474 INFO:tasks.rados.rados.0.smithi163.stdout:739: done (0 left) 2023-10-09T09:08:26.474 INFO:tasks.rados.rados.0.smithi163.stdout:740: rollback oid 17 current snap is 94 2023-10-09T09:08:26.474 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 91 2023-10-09T09:08:26.474 INFO:tasks.rados.rados.0.smithi163.stdout:741: write oid 42 current snap is 94 2023-10-09T09:08:26.474 INFO:tasks.rados.rados.0.smithi163.stdout:741: seq_num 313 ranges {460591=723410,1716158=403433,2729279=440140} 2023-10-09T09:08:26.477 INFO:tasks.rados.rados.0.smithi163.stdout:741: writing smithi16367185-42 from 460591 to 1184001 tid 1 2023-10-09T09:08:26.484 INFO:tasks.rados.rados.0.smithi163.stdout:741: writing smithi16367185-42 from 1716158 to 2119591 tid 2 2023-10-09T09:08:26.490 INFO:tasks.rados.rados.0.smithi163.stdout:741: writing smithi16367185-42 from 2729279 to 3169419 tid 3 2023-10-09T09:08:26.491 INFO:tasks.rados.rados.0.smithi163.stdout:742: rollback oid 47 current snap is 94 2023-10-09T09:08:26.491 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 47 to 63 2023-10-09T09:08:26.491 INFO:tasks.rados.rados.0.smithi163.stdout:740: finishing rollback tid 0 to smithi16367185-17 2023-10-09T09:08:26.491 INFO:tasks.rados.rados.0.smithi163.stdout:743: write oid 50 current snap is 94 2023-10-09T09:08:26.491 INFO:tasks.rados.rados.0.smithi163.stdout:743: seq_num 314 ranges {477398=798659,1774640=663444,2873055=404343} 2023-10-09T09:08:26.498 INFO:tasks.rados.rados.0.smithi163.stdout:743: writing smithi16367185-50 from 477398 to 1276057 tid 1 2023-10-09T09:08:26.505 INFO:tasks.rados.rados.0.smithi163.stdout:743: writing smithi16367185-50 from 1774640 to 2438084 tid 2 2023-10-09T09:08:26.510 INFO:tasks.rados.rados.0.smithi163.stdout:743: writing smithi16367185-50 from 2873055 to 3277398 tid 3 2023-10-09T09:08:26.510 INFO:tasks.rados.rados.0.smithi163.stdout:744: write oid 1 current snap is 94 2023-10-09T09:08:26.510 INFO:tasks.rados.rados.0.smithi163.stdout:740: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:08:26.510 INFO:tasks.rados.rados.0.smithi163.stdout:740: finishing rollback tid 2 to smithi16367185-17 2023-10-09T09:08:26.510 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 420 (ObjNum 137 snap 28 seq_num 137) dirty exists 2023-10-09T09:08:26.510 INFO:tasks.rados.rados.0.smithi163.stdout:741: finishing write tid 1 to smithi16367185-42 2023-10-09T09:08:26.510 INFO:tasks.rados.rados.0.smithi163.stdout:741: finishing write tid 2 to smithi16367185-42 2023-10-09T09:08:26.511 INFO:tasks.rados.rados.0.smithi163.stdout:744: seq_num 315 ranges {494205=473908,1433122=407102,2384125=462684} 2023-10-09T09:08:26.516 INFO:tasks.rados.rados.0.smithi163.stdout:744: writing smithi16367185-1 from 494205 to 968113 tid 1 2023-10-09T09:08:26.523 INFO:tasks.rados.rados.0.smithi163.stdout:744: writing smithi16367185-1 from 1433122 to 1840224 tid 2 2023-10-09T09:08:26.529 INFO:tasks.rados.rados.0.smithi163.stdout:744: writing smithi16367185-1 from 2384125 to 2846809 tid 3 2023-10-09T09:08:26.530 INFO:tasks.rados.rados.0.smithi163.stdout:740: done (4 left) 2023-10-09T09:08:26.530 INFO:tasks.rados.rados.0.smithi163.stdout:745: read oid 38 snap -1 2023-10-09T09:08:26.530 INFO:tasks.rados.rados.0.smithi163.stdout:745: expect (ObjNum 221 snap 61 seq_num 221) 2023-10-09T09:08:26.530 INFO:tasks.rados.rados.0.smithi163.stdout:746: delete oid 32 current snap is 94 2023-10-09T09:08:26.530 INFO:tasks.rados.rados.0.smithi163.stdout:742: finishing rollback tid 1 to smithi16367185-47 2023-10-09T09:08:26.530 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 0 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:08:26.531 INFO:tasks.rados.rados.0.smithi163.stdout:741: finishing write tid 3 to smithi16367185-42 2023-10-09T09:08:26.531 INFO:tasks.rados.rados.0.smithi163.stdout:741: finishing write tid 4 to smithi16367185-42 2023-10-09T09:08:26.531 INFO:tasks.rados.rados.0.smithi163.stdout:741: finishing write tid 5 to smithi16367185-42 2023-10-09T09:08:26.531 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 621 (ObjNum 313 snap 94 seq_num 313) dirty exists 2023-10-09T09:08:26.531 INFO:tasks.rados.rados.0.smithi163.stdout:741: left oid 42 (ObjNum 313 snap 94 seq_num 313) 2023-10-09T09:08:26.531 INFO:tasks.rados.rados.0.smithi163.stdout:743: finishing write tid 1 to smithi16367185-50 2023-10-09T09:08:26.531 INFO:tasks.rados.rados.0.smithi163.stdout:743: finishing write tid 2 to smithi16367185-50 2023-10-09T09:08:26.532 INFO:tasks.rados.rados.0.smithi163.stdout:743: finishing write tid 3 to smithi16367185-50 2023-10-09T09:08:26.532 INFO:tasks.rados.rados.0.smithi163.stdout:743: finishing write tid 4 to smithi16367185-50 2023-10-09T09:08:26.532 INFO:tasks.rados.rados.0.smithi163.stdout:743: finishing write tid 5 to smithi16367185-50 2023-10-09T09:08:26.532 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 288 (ObjNum 314 snap 94 seq_num 314) dirty exists 2023-10-09T09:08:26.532 INFO:tasks.rados.rados.0.smithi163.stdout:743: left oid 50 (ObjNum 314 snap 94 seq_num 314) 2023-10-09T09:08:26.532 INFO:tasks.rados.rados.0.smithi163.stdout:744: finishing write tid 1 to smithi16367185-1 2023-10-09T09:08:26.532 INFO:tasks.rados.rados.0.smithi163.stdout:744: finishing write tid 2 to smithi16367185-1 2023-10-09T09:08:26.533 INFO:tasks.rados.rados.0.smithi163.stdout:744: finishing write tid 3 to smithi16367185-1 2023-10-09T09:08:26.533 INFO:tasks.rados.rados.0.smithi163.stdout:741: done (5 left) 2023-10-09T09:08:26.534 INFO:tasks.rados.rados.0.smithi163.stdout:742: done (4 left) 2023-10-09T09:08:26.534 INFO:tasks.rados.rados.0.smithi163.stdout:743: done (3 left) 2023-10-09T09:08:26.534 INFO:tasks.rados.rados.0.smithi163.stdout:746: done (2 left) 2023-10-09T09:08:26.534 INFO:tasks.rados.rados.0.smithi163.stdout:747: snap_create 2023-10-09T09:08:26.534 INFO:tasks.rados.rados.0.smithi163.stdout:744: finishing write tid 4 to smithi16367185-1 2023-10-09T09:08:26.534 INFO:tasks.rados.rados.0.smithi163.stdout:744: finishing write tid 5 to smithi16367185-1 2023-10-09T09:08:26.535 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 431 (ObjNum 315 snap 94 seq_num 315) dirty exists 2023-10-09T09:08:26.535 INFO:tasks.rados.rados.0.smithi163.stdout:744: left oid 1 (ObjNum 315 snap 94 seq_num 315) 2023-10-09T09:08:26.544 INFO:tasks.rados.rados.0.smithi163.stdout:745: expect (ObjNum 221 snap 61 seq_num 221) 2023-10-09T09:08:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:26 smithi181 bash[27712]: cluster 2023-10-09T09:08:25.464376+0000 mgr.y (mgr.15273) 6 : cluster [DBG] pgmap v6: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:26 smithi181 bash[27712]: cluster 2023-10-09T09:08:25.467219+0000 mon.a (mon.0) 1113 : cluster [DBG] mgrmap e28: y(active, since 2s) 2023-10-09T09:08:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:26 smithi181 bash[27712]: cluster 2023-10-09T09:08:25.467310+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2023-10-09T09:08:26.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:26 smithi181 bash[27712]: audit 2023-10-09T09:08:26.366063+0000 mon.a (mon.0) 1115 : audit [DBG] from='client.? 172.21.15.163:0/1136684195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:26 smithi163 bash[27500]: cluster 2023-10-09T09:08:25.464376+0000 mgr.y (mgr.15273) 6 : cluster [DBG] pgmap v6: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:26 smithi163 bash[27500]: cluster 2023-10-09T09:08:25.467219+0000 mon.a (mon.0) 1113 : cluster [DBG] mgrmap e28: y(active, since 2s) 2023-10-09T09:08:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:26 smithi163 bash[27500]: cluster 2023-10-09T09:08:25.467310+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2023-10-09T09:08:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:26 smithi163 bash[27500]: audit 2023-10-09T09:08:26.366063+0000 mon.a (mon.0) 1115 : audit [DBG] from='client.? 172.21.15.163:0/1136684195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:26 smithi163 bash[33912]: cluster 2023-10-09T09:08:25.464376+0000 mgr.y (mgr.15273) 6 : cluster [DBG] pgmap v6: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:26 smithi163 bash[33912]: cluster 2023-10-09T09:08:25.467219+0000 mon.a (mon.0) 1113 : cluster [DBG] mgrmap e28: y(active, since 2s) 2023-10-09T09:08:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:26 smithi163 bash[33912]: cluster 2023-10-09T09:08:25.467310+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2023-10-09T09:08:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:26 smithi163 bash[33912]: audit 2023-10-09T09:08:26.366063+0000 mon.a (mon.0) 1115 : audit [DBG] from='client.? 172.21.15.163:0/1136684195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:27.149 INFO:tasks.rados.rados.0.smithi163.stdout:744: done (2 left) 2023-10-09T09:08:27.149 INFO:tasks.rados.rados.0.smithi163.stdout:745: done (1 left) 2023-10-09T09:08:27.150 INFO:tasks.rados.rados.0.smithi163.stdout:747: done (0 left) 2023-10-09T09:08:27.150 INFO:tasks.rados.rados.0.smithi163.stdout:748: rollback oid 17 current snap is 95 2023-10-09T09:08:27.150 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 90 2023-10-09T09:08:27.150 INFO:tasks.rados.rados.0.smithi163.stdout:749: snap_remove snap 89 2023-10-09T09:08:27.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:26 smithi181 bash[50540]: debug 2023-10-09T09:08:26.977+0000 7fed3f248200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:08:27.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:27 smithi181 bash[50540]: debug 2023-10-09T09:08:27.093+0000 7fed3f248200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:08:27.318 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:27 smithi181 bash[50540]: debug 2023-10-09T09:08:27.213+0000 7fed3f248200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:08:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:27 smithi181 bash[27712]: cluster 2023-10-09T09:08:26.467373+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2023-10-09T09:08:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:27 smithi181 bash[27712]: cluster 2023-10-09T09:08:27.146976+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2023-10-09T09:08:27.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:27 smithi181 bash[50540]: debug 2023-10-09T09:08:27.697+0000 7fed3f248200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:08:27.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:27 smithi163 bash[27500]: cluster 2023-10-09T09:08:26.467373+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2023-10-09T09:08:27.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:27 smithi163 bash[27500]: cluster 2023-10-09T09:08:27.146976+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2023-10-09T09:08:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:27 smithi163 bash[33912]: cluster 2023-10-09T09:08:26.467373+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2023-10-09T09:08:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:27 smithi163 bash[33912]: cluster 2023-10-09T09:08:27.146976+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2023-10-09T09:08:28.152 INFO:tasks.rados.rados.0.smithi163.stdout:749: done (1 left) 2023-10-09T09:08:28.152 INFO:tasks.rados.rados.0.smithi163.stdout:750: rollback oid 3 current snap is 95 2023-10-09T09:08:28.152 INFO:tasks.rados.rados.0.smithi163.stdout:748: finishing rollback tid 0 to smithi16367185-17 2023-10-09T09:08:28.152 INFO:tasks.rados.rados.0.smithi163.stdout:748: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:08:28.153 INFO:tasks.rados.rados.0.smithi163.stdout:748: finishing rollback tid 2 to smithi16367185-17 2023-10-09T09:08:28.153 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 424 (ObjNum 137 snap 28 seq_num 137) dirty exists 2023-10-09T09:08:28.153 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 75 2023-10-09T09:08:28.153 INFO:tasks.rados.rados.0.smithi163.stdout:748: done (1 left) 2023-10-09T09:08:28.153 INFO:tasks.rados.rados.0.smithi163.stdout:751: delete oid 13 current snap is 95 2023-10-09T09:08:28.160 INFO:tasks.rados.rados.0.smithi163.stdout:750: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:08:28.160 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 0 (ObjNum 1 snap 32603 seq_num 698687456) dirty dne 2023-10-09T09:08:28.160 INFO:tasks.rados.rados.0.smithi163.stdout:750: done (1 left) 2023-10-09T09:08:28.160 INFO:tasks.rados.rados.0.smithi163.stdout:751: done (0 left) 2023-10-09T09:08:28.160 INFO:tasks.rados.rados.0.smithi163.stdout:752: delete oid 17 current snap is 95 2023-10-09T09:08:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:752: done (0 left) 2023-10-09T09:08:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:753: write oid 27 current snap is 95 2023-10-09T09:08:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:753: seq_num 319 ranges {561433=774904,1950697=414440,2777464=583969} 2023-10-09T09:08:28.171 INFO:tasks.rados.rados.0.smithi163.stdout:753: writing smithi16367185-27 from 561433 to 1336337 tid 1 2023-10-09T09:08:28.178 INFO:tasks.rados.rados.0.smithi163.stdout:753: writing smithi16367185-27 from 1950697 to 2365137 tid 2 2023-10-09T09:08:28.183 INFO:tasks.rados.rados.0.smithi163.stdout:753: writing smithi16367185-27 from 2777464 to 3361433 tid 3 2023-10-09T09:08:28.184 INFO:tasks.rados.rados.0.smithi163.stdout:754: delete oid 39 current snap is 95 2023-10-09T09:08:28.184 INFO:tasks.rados.rados.0.smithi163.stdout:753: finishing write tid 1 to smithi16367185-27 2023-10-09T09:08:28.184 INFO:tasks.rados.rados.0.smithi163.stdout:753: finishing write tid 2 to smithi16367185-27 2023-10-09T09:08:28.186 INFO:tasks.rados.rados.0.smithi163.stdout:754: done (1 left) 2023-10-09T09:08:28.186 INFO:tasks.rados.rados.0.smithi163.stdout:755: snap_create 2023-10-09T09:08:28.188 INFO:tasks.rados.rados.0.smithi163.stdout:753: finishing write tid 3 to smithi16367185-27 2023-10-09T09:08:28.188 INFO:tasks.rados.rados.0.smithi163.stdout:753: finishing write tid 4 to smithi16367185-27 2023-10-09T09:08:28.188 INFO:tasks.rados.rados.0.smithi163.stdout:753: finishing write tid 5 to smithi16367185-27 2023-10-09T09:08:28.188 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 239 (ObjNum 319 snap 95 seq_num 319) dirty exists 2023-10-09T09:08:28.189 INFO:tasks.rados.rados.0.smithi163.stdout:753: left oid 27 (ObjNum 319 snap 95 seq_num 319) 2023-10-09T09:08:28.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:27 smithi181 bash[50540]: debug 2023-10-09T09:08:27.925+0000 7fed3f248200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:08:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:28 smithi181 bash[27712]: cluster 2023-10-09T09:08:27.465160+0000 mgr.y (mgr.15273) 7 : cluster [DBG] pgmap v9: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:28 smithi181 bash[27712]: cluster 2023-10-09T09:08:28.150325+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2023-10-09T09:08:28.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:28 smithi181 bash[50540]: debug 2023-10-09T09:08:28.393+0000 7fed3f248200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:08:28.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:28 smithi163 bash[33912]: cluster 2023-10-09T09:08:27.465160+0000 mgr.y (mgr.15273) 7 : cluster [DBG] pgmap v9: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:28.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:28 smithi163 bash[33912]: cluster 2023-10-09T09:08:28.150325+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2023-10-09T09:08:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:28 smithi163 bash[27500]: cluster 2023-10-09T09:08:27.465160+0000 mgr.y (mgr.15273) 7 : cluster [DBG] pgmap v9: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:28 smithi163 bash[27500]: cluster 2023-10-09T09:08:28.150325+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2023-10-09T09:08:29.163 INFO:tasks.rados.rados.0.smithi163.stdout:753: done (1 left) 2023-10-09T09:08:29.163 INFO:tasks.rados.rados.0.smithi163.stdout:755: done (0 left) 2023-10-09T09:08:29.163 INFO:tasks.rados.rados.0.smithi163.stdout:756: delete oid 21 current snap is 96 2023-10-09T09:08:29.167 INFO:tasks.rados.rados.0.smithi163.stdout:756: done (0 left) 2023-10-09T09:08:29.167 INFO:tasks.rados.rados.0.smithi163.stdout:757: snap_remove snap 86 2023-10-09T09:08:29.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:28 smithi181 bash[50540]: debug 2023-10-09T09:08:28.901+0000 7fed3f248200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:08:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:29 smithi181 bash[27712]: audit 2023-10-09T09:08:28.813670+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.163:0/4049118564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:29 smithi181 bash[27712]: cluster 2023-10-09T09:08:29.162550+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2023-10-09T09:08:29.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:29 smithi181 bash[50540]: debug 2023-10-09T09:08:29.505+0000 7fed3f248200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:08:29.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:29 smithi181 bash[50540]: debug 2023-10-09T09:08:29.629+0000 7fed3f248200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:08:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:29 smithi163 bash[33912]: audit 2023-10-09T09:08:28.813670+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.163:0/4049118564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:29 smithi163 bash[33912]: cluster 2023-10-09T09:08:29.162550+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2023-10-09T09:08:29.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:29 smithi163 bash[27500]: audit 2023-10-09T09:08:28.813670+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.163:0/4049118564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:29.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:29 smithi163 bash[27500]: cluster 2023-10-09T09:08:29.162550+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2023-10-09T09:08:30.168 INFO:tasks.rados.rados.0.smithi163.stdout:757: done (0 left) 2023-10-09T09:08:30.169 INFO:tasks.rados.rados.0.smithi163.stdout:758: read oid 41 snap 85 2023-10-09T09:08:30.169 INFO:tasks.rados.rados.0.smithi163.stdout:758: expect deleted 2023-10-09T09:08:30.169 INFO:tasks.rados.rados.0.smithi163.stdout:759: write oid 14 current snap is 96 2023-10-09T09:08:30.169 INFO:tasks.rados.rados.0.smithi163.stdout:759: seq_num 322 ranges {611854=717004,2126143=561767,3259733=152121} 2023-10-09T09:08:30.178 INFO:tasks.rados.rados.0.smithi163.stdout:759: writing smithi16367185-14 from 611854 to 1328858 tid 1 2023-10-09T09:08:30.187 INFO:tasks.rados.rados.0.smithi163.stdout:759: writing smithi16367185-14 from 2126143 to 2687910 tid 2 2023-10-09T09:08:30.194 INFO:tasks.rados.rados.0.smithi163.stdout:759: writing smithi16367185-14 from 3259733 to 3411854 tid 3 2023-10-09T09:08:30.194 INFO:tasks.rados.rados.0.smithi163.stdout:760: read oid 10 snap 87 2023-10-09T09:08:30.194 INFO:tasks.rados.rados.0.smithi163.stdout:760: expect deleted 2023-10-09T09:08:30.194 INFO:tasks.rados.rados.0.smithi163.stdout:761: read oid 6 snap -1 2023-10-09T09:08:30.194 INFO:tasks.rados.rados.0.smithi163.stdout:761: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:08:30.194 INFO:tasks.rados.rados.0.smithi163.stdout:762: snap_create 2023-10-09T09:08:30.194 INFO:tasks.rados.rados.0.smithi163.stdout:759: finishing write tid 1 to smithi16367185-14 2023-10-09T09:08:30.195 INFO:tasks.rados.rados.0.smithi163.stdout:759: finishing write tid 2 to smithi16367185-14 2023-10-09T09:08:30.204 INFO:tasks.rados.rados.0.smithi163.stdout:761: expect (ObjNum 92 snap 15 seq_num 92) 2023-10-09T09:08:30.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[50540]: debug 2023-10-09T09:08:30.025+0000 7fed3f248200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:08:30.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[50540]: debug 2023-10-09T09:08:30.153+0000 7fed3f248200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:08:30.409 INFO:tasks.rados.rados.0.smithi163.stdout:759: finishing write tid 3 to smithi16367185-14 2023-10-09T09:08:30.409 INFO:tasks.rados.rados.0.smithi163.stdout:759: finishing write tid 4 to smithi16367185-14 2023-10-09T09:08:30.409 INFO:tasks.rados.rados.0.smithi163.stdout:759: finishing write tid 5 to smithi16367185-14 2023-10-09T09:08:30.409 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 343 (ObjNum 322 snap 96 seq_num 322) dirty exists 2023-10-09T09:08:30.410 INFO:tasks.rados.rados.0.smithi163.stdout:759: left oid 14 (ObjNum 322 snap 96 seq_num 322) 2023-10-09T09:08:30.629 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[50540]: debug 2023-10-09T09:08:30.337+0000 7fed3f248200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:08:30.629 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[50540]: debug 2023-10-09T09:08:30.477+0000 7fed3f248200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:08:30.629 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[27712]: cluster 2023-10-09T09:08:29.465889+0000 mgr.y (mgr.15273) 8 : cluster [DBG] pgmap v12: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:30.629 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[27712]: audit 2023-10-09T09:08:30.160790+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[27712]: cluster 2023-10-09T09:08:30.167352+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2023-10-09T09:08:30.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[27712]: audit 2023-10-09T09:08:30.182990+0000 mon.a (mon.0) 1122 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[27712]: audit 2023-10-09T09:08:30.229545+0000 mon.a (mon.0) 1123 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[27712]: audit 2023-10-09T09:08:30.236546+0000 mon.a (mon.0) 1124 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[33912]: cluster 2023-10-09T09:08:29.465889+0000 mgr.y (mgr.15273) 8 : cluster [DBG] pgmap v12: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[33912]: audit 2023-10-09T09:08:30.160790+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[33912]: cluster 2023-10-09T09:08:30.167352+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2023-10-09T09:08:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[33912]: audit 2023-10-09T09:08:30.182990+0000 mon.a (mon.0) 1122 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[33912]: audit 2023-10-09T09:08:30.229545+0000 mon.a (mon.0) 1123 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[33912]: audit 2023-10-09T09:08:30.236546+0000 mon.a (mon.0) 1124 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[27500]: cluster 2023-10-09T09:08:29.465889+0000 mgr.y (mgr.15273) 8 : cluster [DBG] pgmap v12: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 233 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[27500]: audit 2023-10-09T09:08:30.160790+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[27500]: cluster 2023-10-09T09:08:30.167352+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2023-10-09T09:08:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[27500]: audit 2023-10-09T09:08:30.182990+0000 mon.a (mon.0) 1122 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[27500]: audit 2023-10-09T09:08:30.229545+0000 mon.a (mon.0) 1123 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:30 smithi163 bash[27500]: audit 2023-10-09T09:08:30.236546+0000 mon.a (mon.0) 1124 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:30.991 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[50540]: debug 2023-10-09T09:08:30.625+0000 7fed3f248200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:08:30.991 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[50540]: debug 2023-10-09T09:08:30.845+0000 7fed3f248200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:08:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:758: done (4 left) 2023-10-09T09:08:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:759: done (3 left) 2023-10-09T09:08:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:760: done (2 left) 2023-10-09T09:08:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:761: done (1 left) 2023-10-09T09:08:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:762: done (0 left) 2023-10-09T09:08:31.189 INFO:tasks.rados.rados.0.smithi163.stdout:763: snap_create 2023-10-09T09:08:31.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:30 smithi181 bash[50540]: debug 2023-10-09T09:08:30.989+0000 7fed3f248200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:08:32.190 INFO:tasks.rados.rados.0.smithi163.stdout:763: done (0 left) 2023-10-09T09:08:32.191 INFO:tasks.rados.rados.0.smithi163.stdout:764: write oid 42 current snap is 98 2023-10-09T09:08:32.191 INFO:tasks.rados.rados.0.smithi163.stdout:764: seq_num 323 ranges {628661=792253,2184625=421778,3403509=25152} 2023-10-09T09:08:32.199 INFO:tasks.rados.rados.0.smithi163.stdout:764: writing smithi16367185-42 from 628661 to 1420914 tid 1 2023-10-09T09:08:32.207 INFO:tasks.rados.rados.0.smithi163.stdout:764: writing smithi16367185-42 from 2184625 to 2606403 tid 2 2023-10-09T09:08:32.212 INFO:tasks.rados.rados.0.smithi163.stdout:764: writing smithi16367185-42 from 3403509 to 3428661 tid 3 2023-10-09T09:08:32.212 INFO:tasks.rados.rados.0.smithi163.stdout:764: finishing write tid 1 to smithi16367185-42 2023-10-09T09:08:32.212 INFO:tasks.rados.rados.0.smithi163.stdout:764: finishing write tid 2 to smithi16367185-42 2023-10-09T09:08:32.212 INFO:tasks.rados.rados.0.smithi163.stdout:765: read oid 4 snap 60 2023-10-09T09:08:32.212 INFO:tasks.rados.rados.0.smithi163.stdout:765: expect (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:08:32.212 INFO:tasks.rados.rados.0.smithi163.stdout:766: rollback oid 35 current snap is 98 2023-10-09T09:08:32.212 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 35 to 63 2023-10-09T09:08:32.212 INFO:tasks.rados.rados.0.smithi163.stdout:767: snap_create 2023-10-09T09:08:32.214 INFO:tasks.rados.rados.0.smithi163.stdout:764: finishing write tid 3 to smithi16367185-42 2023-10-09T09:08:32.216 INFO:tasks.rados.rados.0.smithi163.stdout:764: finishing write tid 4 to smithi16367185-42 2023-10-09T09:08:32.216 INFO:tasks.rados.rados.0.smithi163.stdout:764: finishing write tid 5 to smithi16367185-42 2023-10-09T09:08:32.216 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 640 (ObjNum 323 snap 98 seq_num 323) dirty exists 2023-10-09T09:08:32.216 INFO:tasks.rados.rados.0.smithi163.stdout:764: left oid 42 (ObjNum 323 snap 98 seq_num 323) 2023-10-09T09:08:32.218 INFO:tasks.rados.rados.0.smithi163.stdout:766: finishing rollback tid 0 to smithi16367185-35 2023-10-09T09:08:32.219 INFO:tasks.rados.rados.0.smithi163.stdout:766: finishing rollback tid 1 to smithi16367185-35 2023-10-09T09:08:32.219 INFO:tasks.rados.rados.0.smithi163.stdout:766: finishing rollback tid 2 to smithi16367185-35 2023-10-09T09:08:32.219 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 176 (ObjNum 196 snap 54 seq_num 196) dirty exists 2023-10-09T09:08:32.222 INFO:tasks.rados.rados.0.smithi163.stdout:765: expect (ObjNum 102 snap 16 seq_num 102) 2023-10-09T09:08:32.437 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[50540]: debug 2023-10-09T09:08:32.301+0000 7fed3f248200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:08:32.437 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[50540]: [09/Oct/2023:09:08:32] ENGINE Bus STARTING 2023-10-09T09:08:32.437 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[50540]: CherryPy Checker: 2023-10-09T09:08:32.437 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[27712]: audit 2023-10-09T09:08:31.082103+0000 mon.a (mon.0) 1125 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.437 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[27712]: audit 2023-10-09T09:08:31.089427+0000 mon.a (mon.0) 1126 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.437 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[27712]: audit 2023-10-09T09:08:31.093465+0000 mon.a (mon.0) 1127 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[27712]: audit 2023-10-09T09:08:31.099161+0000 mon.a (mon.0) 1128 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[27712]: cluster 2023-10-09T09:08:31.187085+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2023-10-09T09:08:32.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[27712]: audit 2023-10-09T09:08:31.266581+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.163:0/1528081395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:32.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[27712]: cluster 2023-10-09T09:08:31.466616+0000 mgr.y (mgr.15273) 9 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2023-10-09T09:08:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[33912]: audit 2023-10-09T09:08:31.082103+0000 mon.a (mon.0) 1125 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[33912]: audit 2023-10-09T09:08:31.089427+0000 mon.a (mon.0) 1126 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[33912]: audit 2023-10-09T09:08:31.093465+0000 mon.a (mon.0) 1127 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[33912]: audit 2023-10-09T09:08:31.099161+0000 mon.a (mon.0) 1128 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[33912]: cluster 2023-10-09T09:08:31.187085+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2023-10-09T09:08:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[33912]: audit 2023-10-09T09:08:31.266581+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.163:0/1528081395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[33912]: cluster 2023-10-09T09:08:31.466616+0000 mgr.y (mgr.15273) 9 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2023-10-09T09:08:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[27500]: audit 2023-10-09T09:08:31.082103+0000 mon.a (mon.0) 1125 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[27500]: audit 2023-10-09T09:08:31.089427+0000 mon.a (mon.0) 1126 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[27500]: audit 2023-10-09T09:08:31.093465+0000 mon.a (mon.0) 1127 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[27500]: audit 2023-10-09T09:08:31.099161+0000 mon.a (mon.0) 1128 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[27500]: cluster 2023-10-09T09:08:31.187085+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2023-10-09T09:08:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[27500]: audit 2023-10-09T09:08:31.266581+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.163:0/1528081395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:32 smithi163 bash[27500]: cluster 2023-10-09T09:08:31.466616+0000 mgr.y (mgr.15273) 9 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2023-10-09T09:08:32.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[50540]: The Application mounted at '' has an empty config. 2023-10-09T09:08:32.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[50540]: [09/Oct/2023:09:08:32] ENGINE Serving on http://:::9283 2023-10-09T09:08:32.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:32 smithi181 bash[50540]: [09/Oct/2023:09:08:32] ENGINE Bus STARTED 2023-10-09T09:08:33.201 INFO:tasks.rados.rados.0.smithi163.stdout:764: done (3 left) 2023-10-09T09:08:33.201 INFO:tasks.rados.rados.0.smithi163.stdout:765: done (2 left) 2023-10-09T09:08:33.202 INFO:tasks.rados.rados.0.smithi163.stdout:766: done (1 left) 2023-10-09T09:08:33.202 INFO:tasks.rados.rados.0.smithi163.stdout:767: done (0 left) 2023-10-09T09:08:33.202 INFO:tasks.rados.rados.0.smithi163.stdout:768: write oid 19 current snap is 99 2023-10-09T09:08:33.202 INFO:tasks.rados.rados.0.smithi163.stdout:768: seq_num 324 ranges {645468=467502,1726754=565436,2798226=596838} 2023-10-09T09:08:33.209 INFO:tasks.rados.rados.0.smithi163.stdout:768: writing smithi16367185-19 from 645468 to 1112970 tid 1 2023-10-09T09:08:33.216 INFO:tasks.rados.rados.0.smithi163.stdout:768: writing smithi16367185-19 from 1726754 to 2292190 tid 2 2023-10-09T09:08:33.223 INFO:tasks.rados.rados.0.smithi163.stdout:768: writing smithi16367185-19 from 2798226 to 3395064 tid 3 2023-10-09T09:08:33.223 INFO:tasks.rados.rados.0.smithi163.stdout:769: write oid 44 current snap is 99 2023-10-09T09:08:33.223 INFO:tasks.rados.rados.0.smithi163.stdout:768: finishing write tid 1 to smithi16367185-19 2023-10-09T09:08:33.223 INFO:tasks.rados.rados.0.smithi163.stdout:769: seq_num 325 ranges {662275=542751,1785236=425447,2942002=520273} 2023-10-09T09:08:33.230 INFO:tasks.rados.rados.0.smithi163.stdout:769: writing smithi16367185-44 from 662275 to 1205026 tid 1 2023-10-09T09:08:33.237 INFO:tasks.rados.rados.0.smithi163.stdout:769: writing smithi16367185-44 from 1785236 to 2210683 tid 2 2023-10-09T09:08:33.245 INFO:tasks.rados.rados.0.smithi163.stdout:769: writing smithi16367185-44 from 2942002 to 3462275 tid 3 2023-10-09T09:08:33.245 INFO:tasks.rados.rados.0.smithi163.stdout:770: snap_remove snap 97 2023-10-09T09:08:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[27500]: cluster 2023-10-09T09:08:32.189808+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2023-10-09T09:08:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[27500]: cluster 2023-10-09T09:08:32.327083+0000 mon.a (mon.0) 1131 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:08:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[27500]: audit 2023-10-09T09:08:32.329852+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:08:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[27500]: audit 2023-10-09T09:08:32.330746+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:08:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[27500]: audit 2023-10-09T09:08:32.337463+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:08:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[27500]: audit 2023-10-09T09:08:32.338085+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:08:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[33912]: cluster 2023-10-09T09:08:32.189808+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2023-10-09T09:08:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[33912]: cluster 2023-10-09T09:08:32.327083+0000 mon.a (mon.0) 1131 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:08:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[33912]: audit 2023-10-09T09:08:32.329852+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:08:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[33912]: audit 2023-10-09T09:08:32.330746+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:08:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[33912]: audit 2023-10-09T09:08:32.337463+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:08:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:33 smithi163 bash[33912]: audit 2023-10-09T09:08:32.338085+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:08:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:33 smithi181 bash[27712]: cluster 2023-10-09T09:08:32.189808+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2023-10-09T09:08:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:33 smithi181 bash[27712]: cluster 2023-10-09T09:08:32.327083+0000 mon.a (mon.0) 1131 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:08:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:33 smithi181 bash[27712]: audit 2023-10-09T09:08:32.329852+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:08:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:33 smithi181 bash[27712]: audit 2023-10-09T09:08:32.330746+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:08:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:33 smithi181 bash[27712]: audit 2023-10-09T09:08:32.337463+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:08:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:33 smithi181 bash[27712]: audit 2023-10-09T09:08:32.338085+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.? 172.21.15.181:0/1269123423' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:08:34.220 INFO:tasks.rados.rados.0.smithi163.stdout:770: done (2 left) 2023-10-09T09:08:34.220 INFO:tasks.rados.rados.0.smithi163.stdout:771: delete oid 41 current snap is 99 2023-10-09T09:08:34.220 INFO:tasks.rados.rados.0.smithi163.stdout:768: finishing write tid 2 to smithi16367185-19 2023-10-09T09:08:34.220 INFO:tasks.rados.rados.0.smithi163.stdout:768: finishing write tid 3 to smithi16367185-19 2023-10-09T09:08:34.220 INFO:tasks.rados.rados.0.smithi163.stdout:768: finishing write tid 4 to smithi16367185-19 2023-10-09T09:08:34.220 INFO:tasks.rados.rados.0.smithi163.stdout:768: finishing write tid 5 to smithi16367185-19 2023-10-09T09:08:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 247 (ObjNum 324 snap 99 seq_num 324) dirty exists 2023-10-09T09:08:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:768: left oid 19 (ObjNum 324 snap 99 seq_num 324) 2023-10-09T09:08:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:769: finishing write tid 1 to smithi16367185-44 2023-10-09T09:08:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:769: finishing write tid 2 to smithi16367185-44 2023-10-09T09:08:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:769: finishing write tid 3 to smithi16367185-44 2023-10-09T09:08:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:769: finishing write tid 4 to smithi16367185-44 2023-10-09T09:08:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:769: finishing write tid 5 to smithi16367185-44 2023-10-09T09:08:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 645 (ObjNum 325 snap 99 seq_num 325) dirty exists 2023-10-09T09:08:34.221 INFO:tasks.rados.rados.0.smithi163.stdout:769: left oid 44 (ObjNum 325 snap 99 seq_num 325) 2023-10-09T09:08:34.225 INFO:tasks.rados.rados.0.smithi163.stdout:768: done (2 left) 2023-10-09T09:08:34.225 INFO:tasks.rados.rados.0.smithi163.stdout:769: done (1 left) 2023-10-09T09:08:34.225 INFO:tasks.rados.rados.0.smithi163.stdout:771: done (0 left) 2023-10-09T09:08:34.225 INFO:tasks.rados.rados.0.smithi163.stdout:772: snap_remove snap 91 2023-10-09T09:08:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:34 smithi163 bash[33912]: cluster 2023-10-09T09:08:33.200462+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2023-10-09T09:08:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:34 smithi163 bash[33912]: cluster 2023-10-09T09:08:33.212249+0000 mon.a (mon.0) 1133 : cluster [DBG] mgrmap e29: y(active, since 9s), standbys: x 2023-10-09T09:08:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:34 smithi163 bash[33912]: audit 2023-10-09T09:08:33.212783+0000 mon.a (mon.0) 1134 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:08:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:34 smithi163 bash[33912]: cluster 2023-10-09T09:08:33.467243+0000 mgr.y (mgr.15273) 10 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2023-10-09T09:08:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:34 smithi163 bash[33912]: audit 2023-10-09T09:08:33.723465+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.163:0/668568859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:34 smithi163 bash[27500]: cluster 2023-10-09T09:08:33.200462+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2023-10-09T09:08:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:34 smithi163 bash[27500]: cluster 2023-10-09T09:08:33.212249+0000 mon.a (mon.0) 1133 : cluster [DBG] mgrmap e29: y(active, since 9s), standbys: x 2023-10-09T09:08:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:34 smithi163 bash[27500]: audit 2023-10-09T09:08:33.212783+0000 mon.a (mon.0) 1134 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:08:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:34 smithi163 bash[27500]: cluster 2023-10-09T09:08:33.467243+0000 mgr.y (mgr.15273) 10 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2023-10-09T09:08:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:34 smithi163 bash[27500]: audit 2023-10-09T09:08:33.723465+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.163:0/668568859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:34 smithi181 bash[27712]: cluster 2023-10-09T09:08:33.200462+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2023-10-09T09:08:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:34 smithi181 bash[27712]: cluster 2023-10-09T09:08:33.212249+0000 mon.a (mon.0) 1133 : cluster [DBG] mgrmap e29: y(active, since 9s), standbys: x 2023-10-09T09:08:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:34 smithi181 bash[27712]: audit 2023-10-09T09:08:33.212783+0000 mon.a (mon.0) 1134 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:08:34.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:34 smithi181 bash[27712]: cluster 2023-10-09T09:08:33.467243+0000 mgr.y (mgr.15273) 10 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2023-10-09T09:08:34.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:34 smithi181 bash[27712]: audit 2023-10-09T09:08:33.723465+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.163:0/668568859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:35.226 INFO:tasks.rados.rados.0.smithi163.stdout:772: done (0 left) 2023-10-09T09:08:35.226 INFO:tasks.rados.rados.0.smithi163.stdout:773: write oid 24 current snap is 99 2023-10-09T09:08:35.226 INFO:tasks.rados.rados.0.smithi163.stdout:773: seq_num 327 ranges {695889=409602,1618553=712763,2996848=499041} 2023-10-09T09:08:35.230 INFO:tasks.rados.rados.0.smithi163.stdout:773: writing smithi16367185-24 from 695889 to 1105491 tid 1 2023-10-09T09:08:35.238 INFO:tasks.rados.rados.0.smithi163.stdout:773: writing smithi16367185-24 from 1618553 to 2331316 tid 2 2023-10-09T09:08:35.245 INFO:tasks.rados.rados.0.smithi163.stdout:773: writing smithi16367185-24 from 2996848 to 3495889 tid 3 2023-10-09T09:08:35.245 INFO:tasks.rados.rados.0.smithi163.stdout:774: read oid 19 snap 88 2023-10-09T09:08:35.245 INFO:tasks.rados.rados.0.smithi163.stdout:774: expect (ObjNum 140 snap 31 seq_num 140) 2023-10-09T09:08:35.245 INFO:tasks.rados.rados.0.smithi163.stdout:773: finishing write tid 1 to smithi16367185-24 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:773: finishing write tid 2 to smithi16367185-24 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:775: read oid 3 snap 95 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:775: expect deleted 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:776: read oid 43 snap -1 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:776: expect (ObjNum 271 snap 84 seq_num 271) 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:777: read oid 46 snap -1 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:777: expect (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:778: rollback oid 4 current snap is 99 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 94 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:779: write oid 18 current snap is 99 2023-10-09T09:08:35.246 INFO:tasks.rados.rados.0.smithi163.stdout:779: seq_num 328 ranges {712696=484851,1960682=572774,3307918=204778} 2023-10-09T09:08:35.252 INFO:tasks.rados.rados.0.smithi163.stdout:779: writing smithi16367185-18 from 712696 to 1197547 tid 1 2023-10-09T09:08:35.261 INFO:tasks.rados.rados.0.smithi163.stdout:779: writing smithi16367185-18 from 1960682 to 2533456 tid 2 2023-10-09T09:08:35.267 INFO:tasks.rados.rados.0.smithi163.stdout:779: writing smithi16367185-18 from 3307918 to 3512696 tid 3 2023-10-09T09:08:35.267 INFO:tasks.rados.rados.0.smithi163.stdout:780: snap_create 2023-10-09T09:08:35.267 INFO:tasks.rados.rados.0.smithi163.stdout:778: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:08:35.267 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 299 (ObjNum 184 snap 0 seq_num 182) dirty dne 2023-10-09T09:08:35.267 INFO:tasks.rados.rados.0.smithi163.stdout:773: finishing write tid 3 to smithi16367185-24 2023-10-09T09:08:35.267 INFO:tasks.rados.rados.0.smithi163.stdout:774: expect (ObjNum 140 snap 31 seq_num 140) 2023-10-09T09:08:35.467 INFO:tasks.rados.rados.0.smithi163.stdout:773: finishing write tid 4 to smithi16367185-24 2023-10-09T09:08:35.467 INFO:tasks.rados.rados.0.smithi163.stdout:773: finishing write tid 5 to smithi16367185-24 2023-10-09T09:08:35.467 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 203 (ObjNum 327 snap 99 seq_num 327) dirty exists 2023-10-09T09:08:35.467 INFO:tasks.rados.rados.0.smithi163.stdout:773: left oid 24 (ObjNum 327 snap 99 seq_num 327) 2023-10-09T09:08:35.467 INFO:tasks.rados.rados.0.smithi163.stdout:776: expect (ObjNum 271 snap 84 seq_num 271) 2023-10-09T09:08:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:35 smithi181 bash[27712]: cluster 2023-10-09T09:08:34.219763+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2023-10-09T09:08:35.618 INFO:tasks.rados.rados.0.smithi163.stdout:779: finishing write tid 1 to smithi16367185-18 2023-10-09T09:08:35.618 INFO:tasks.rados.rados.0.smithi163.stdout:777: expect (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:08:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:35 smithi163 bash[33912]: cluster 2023-10-09T09:08:34.219763+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2023-10-09T09:08:35.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:35 smithi163 bash[27500]: cluster 2023-10-09T09:08:34.219763+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2023-10-09T09:08:35.802 INFO:tasks.rados.rados.0.smithi163.stdout:779: finishing write tid 2 to smithi16367185-18 2023-10-09T09:08:35.802 INFO:tasks.rados.rados.0.smithi163.stdout:779: finishing write tid 3 to smithi16367185-18 2023-10-09T09:08:35.802 INFO:tasks.rados.rados.0.smithi163.stdout:779: finishing write tid 4 to smithi16367185-18 2023-10-09T09:08:35.802 INFO:tasks.rados.rados.0.smithi163.stdout:779: finishing write tid 5 to smithi16367185-18 2023-10-09T09:08:35.802 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 288 (ObjNum 328 snap 99 seq_num 328) dirty exists 2023-10-09T09:08:35.802 INFO:tasks.rados.rados.0.smithi163.stdout:779: left oid 18 (ObjNum 328 snap 99 seq_num 328) 2023-10-09T09:08:36.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:36 smithi163 bash[27500]: cluster 2023-10-09T09:08:35.222326+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2023-10-09T09:08:36.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:36 smithi163 bash[27500]: cluster 2023-10-09T09:08:35.467847+0000 mgr.y (mgr.15273) 11 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2023-10-09T09:08:36.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:36 smithi163 bash[27500]: audit 2023-10-09T09:08:36.161879+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.163:0/3882337387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:36.240 INFO:tasks.rados.rados.0.smithi163.stdout:773: done (7 left) 2023-10-09T09:08:36.240 INFO:tasks.rados.rados.0.smithi163.stdout:774: done (6 left) 2023-10-09T09:08:36.240 INFO:tasks.rados.rados.0.smithi163.stdout:775: done (5 left) 2023-10-09T09:08:36.240 INFO:tasks.rados.rados.0.smithi163.stdout:776: done (4 left) 2023-10-09T09:08:36.240 INFO:tasks.rados.rados.0.smithi163.stdout:777: done (3 left) 2023-10-09T09:08:36.240 INFO:tasks.rados.rados.0.smithi163.stdout:778: done (2 left) 2023-10-09T09:08:36.240 INFO:tasks.rados.rados.0.smithi163.stdout:779: done (1 left) 2023-10-09T09:08:36.240 INFO:tasks.rados.rados.0.smithi163.stdout:780: done (0 left) 2023-10-09T09:08:36.240 INFO:tasks.rados.rados.0.smithi163.stdout:781: snap_remove snap 60 2023-10-09T09:08:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:36 smithi181 bash[27712]: cluster 2023-10-09T09:08:35.222326+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2023-10-09T09:08:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:36 smithi181 bash[27712]: cluster 2023-10-09T09:08:35.467847+0000 mgr.y (mgr.15273) 11 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2023-10-09T09:08:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:36 smithi181 bash[27712]: audit 2023-10-09T09:08:36.161879+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.163:0/3882337387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:36 smithi163 bash[33912]: cluster 2023-10-09T09:08:35.222326+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2023-10-09T09:08:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:36 smithi163 bash[33912]: cluster 2023-10-09T09:08:35.467847+0000 mgr.y (mgr.15273) 11 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2023-10-09T09:08:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:36 smithi163 bash[33912]: audit 2023-10-09T09:08:36.161879+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.163:0/3882337387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:37.153 INFO:tasks.rados.rados.0.smithi163.stdout:781: done (0 left) 2023-10-09T09:08:37.154 INFO:tasks.rados.rados.0.smithi163.stdout:782: read oid 47 snap 98 2023-10-09T09:08:37.154 INFO:tasks.rados.rados.0.smithi163.stdout:782: expect deleted 2023-10-09T09:08:37.154 INFO:tasks.rados.rados.0.smithi163.stdout:783: read oid 28 snap 94 2023-10-09T09:08:37.154 INFO:tasks.rados.rados.0.smithi163.stdout:783: expect (ObjNum 295 snap 91 seq_num 295) 2023-10-09T09:08:37.154 INFO:tasks.rados.rados.0.smithi163.stdout:784: snap_remove snap 85 2023-10-09T09:08:37.409 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:37 smithi163 bash[27500]: cluster 2023-10-09T09:08:36.240242+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2023-10-09T09:08:37.409 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:37 smithi163 bash[27500]: cluster 2023-10-09T09:08:37.151890+0000 mon.a (mon.0) 1138 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2023-10-09T09:08:37.409 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:37 smithi163 bash[33912]: cluster 2023-10-09T09:08:36.240242+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2023-10-09T09:08:37.410 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:37 smithi163 bash[33912]: cluster 2023-10-09T09:08:37.151890+0000 mon.a (mon.0) 1138 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2023-10-09T09:08:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:37 smithi181 bash[27712]: cluster 2023-10-09T09:08:36.240242+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2023-10-09T09:08:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:37 smithi181 bash[27712]: cluster 2023-10-09T09:08:37.151890+0000 mon.a (mon.0) 1138 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2023-10-09T09:08:38.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:37 smithi181 bash[50540]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:08:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:08:38.167 INFO:tasks.rados.rados.0.smithi163.stdout:784: done (2 left) 2023-10-09T09:08:38.167 INFO:tasks.rados.rados.0.smithi163.stdout:785: write oid 36 current snap is 100 2023-10-09T09:08:38.167 INFO:tasks.rados.rados.0.smithi163.stdout:785: seq_num 329 ranges {729503=560100,2019164=716432,3335341=194162} 2023-10-09T09:08:38.176 INFO:tasks.rados.rados.0.smithi163.stdout:785: writing smithi16367185-36 from 729503 to 1289603 tid 1 2023-10-09T09:08:38.184 INFO:tasks.rados.rados.0.smithi163.stdout:785: writing smithi16367185-36 from 2019164 to 2735596 tid 2 2023-10-09T09:08:38.189 INFO:tasks.rados.rados.0.smithi163.stdout:785: writing smithi16367185-36 from 3335341 to 3529503 tid 3 2023-10-09T09:08:38.189 INFO:tasks.rados.rados.0.smithi163.stdout:786: snap_create 2023-10-09T09:08:38.190 INFO:tasks.rados.rados.0.smithi163.stdout:783: expect (ObjNum 295 snap 91 seq_num 295) 2023-10-09T09:08:38.286 INFO:tasks.rados.rados.0.smithi163.stdout:785: finishing write tid 1 to smithi16367185-36 2023-10-09T09:08:38.287 INFO:tasks.rados.rados.0.smithi163.stdout:785: finishing write tid 2 to smithi16367185-36 2023-10-09T09:08:38.287 INFO:tasks.rados.rados.0.smithi163.stdout:785: finishing write tid 3 to smithi16367185-36 2023-10-09T09:08:38.287 INFO:tasks.rados.rados.0.smithi163.stdout:785: finishing write tid 4 to smithi16367185-36 2023-10-09T09:08:38.287 INFO:tasks.rados.rados.0.smithi163.stdout:785: finishing write tid 5 to smithi16367185-36 2023-10-09T09:08:38.287 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 243 (ObjNum 329 snap 100 seq_num 329) dirty exists 2023-10-09T09:08:38.288 INFO:tasks.rados.rados.0.smithi163.stdout:785: left oid 36 (ObjNum 329 snap 100 seq_num 329) 2023-10-09T09:08:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:38 smithi181 bash[27712]: cluster 2023-10-09T09:08:37.468361+0000 mgr.y (mgr.15273) 12 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2023-10-09T09:08:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:38 smithi181 bash[27712]: cluster 2023-10-09T09:08:38.166569+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2023-10-09T09:08:38.567 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:38 smithi181 bash[52947]: ts=2023-10-09T09:08:38.119Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.181:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.181:8765: connect: connection refused" 2023-10-09T09:08:38.568 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:38 smithi181 bash[52947]: ts=2023-10-09T09:08:38.121Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.181:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.181:8765: connect: connection refused" 2023-10-09T09:08:38.568 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:38 smithi181 bash[52947]: ts=2023-10-09T09:08:38.125Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.181:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.181:8765: connect: connection refused" 2023-10-09T09:08:38.568 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:38 smithi181 bash[52947]: ts=2023-10-09T09:08:38.125Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.181:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.181:8765: connect: connection refused" 2023-10-09T09:08:38.717 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:38 smithi163 bash[27500]: cluster 2023-10-09T09:08:37.468361+0000 mgr.y (mgr.15273) 12 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2023-10-09T09:08:38.717 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:38 smithi163 bash[27500]: cluster 2023-10-09T09:08:38.166569+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2023-10-09T09:08:38.717 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:38 smithi163 bash[33912]: cluster 2023-10-09T09:08:37.468361+0000 mgr.y (mgr.15273) 12 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2023-10-09T09:08:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:38 smithi163 bash[33912]: cluster 2023-10-09T09:08:38.166569+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2023-10-09T09:08:39.169 INFO:tasks.rados.rados.0.smithi163.stdout:782: done (3 left) 2023-10-09T09:08:39.169 INFO:tasks.rados.rados.0.smithi163.stdout:783: done (2 left) 2023-10-09T09:08:39.169 INFO:tasks.rados.rados.0.smithi163.stdout:785: done (1 left) 2023-10-09T09:08:39.169 INFO:tasks.rados.rados.0.smithi163.stdout:786: done (0 left) 2023-10-09T09:08:39.169 INFO:tasks.rados.rados.0.smithi163.stdout:787: delete oid 3 current snap is 101 2023-10-09T09:08:39.171 INFO:tasks.rados.rados.0.smithi163.stdout:787: done (0 left) 2023-10-09T09:08:39.171 INFO:tasks.rados.rados.0.smithi163.stdout:788: delete oid 4 current snap is 101 2023-10-09T09:08:39.175 INFO:tasks.rados.rados.0.smithi163.stdout:788: done (0 left) 2023-10-09T09:08:39.175 INFO:tasks.rados.rados.0.smithi163.stdout:789: read oid 46 snap 100 2023-10-09T09:08:39.175 INFO:tasks.rados.rados.0.smithi163.stdout:789: expect (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:08:39.175 INFO:tasks.rados.rados.0.smithi163.stdout:790: delete oid 9 current snap is 101 2023-10-09T09:08:39.178 INFO:tasks.rados.rados.0.smithi163.stdout:790: done (1 left) 2023-10-09T09:08:39.178 INFO:tasks.rados.rados.0.smithi163.stdout:791: read oid 3 snap 87 2023-10-09T09:08:39.178 INFO:tasks.rados.rados.0.smithi163.stdout:791: expect deleted 2023-10-09T09:08:39.178 INFO:tasks.rados.rados.0.smithi163.stdout:792: read oid 39 snap -1 2023-10-09T09:08:39.179 INFO:tasks.rados.rados.0.smithi163.stdout:792: expect deleted 2023-10-09T09:08:39.179 INFO:tasks.rados.rados.0.smithi163.stdout:793: snap_remove snap 93 2023-10-09T09:08:39.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:39 smithi163 bash[27500]: audit 2023-10-09T09:08:38.596419+0000 mon.a (mon.0) 1140 : audit [DBG] from='client.? 172.21.15.163:0/1033977651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:39.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:39 smithi163 bash[27500]: audit 2023-10-09T09:08:38.596785+0000 mon.a (mon.0) 1141 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:39.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:39 smithi163 bash[27500]: cluster 2023-10-09T09:08:39.169236+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2023-10-09T09:08:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:39 smithi163 bash[33912]: audit 2023-10-09T09:08:38.596419+0000 mon.a (mon.0) 1140 : audit [DBG] from='client.? 172.21.15.163:0/1033977651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:39.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:39 smithi163 bash[33912]: audit 2023-10-09T09:08:38.596785+0000 mon.a (mon.0) 1141 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:39.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:39 smithi163 bash[33912]: cluster 2023-10-09T09:08:39.169236+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2023-10-09T09:08:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:39 smithi181 bash[27712]: audit 2023-10-09T09:08:38.596419+0000 mon.a (mon.0) 1140 : audit [DBG] from='client.? 172.21.15.163:0/1033977651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:39 smithi181 bash[27712]: audit 2023-10-09T09:08:38.596785+0000 mon.a (mon.0) 1141 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:39 smithi181 bash[27712]: cluster 2023-10-09T09:08:39.169236+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2023-10-09T09:08:40.171 INFO:tasks.rados.rados.0.smithi163.stdout:793: done (3 left) 2023-10-09T09:08:40.171 INFO:tasks.rados.rados.0.smithi163.stdout:794: rollback oid 26 current snap is 101 2023-10-09T09:08:40.172 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 26 to 88 2023-10-09T09:08:40.172 INFO:tasks.rados.rados.0.smithi163.stdout:789: expect (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:08:40.362 INFO:tasks.rados.rados.0.smithi163.stdout:794: finishing rollback tid 0 to smithi16367185-26 2023-10-09T09:08:40.362 INFO:tasks.rados.rados.0.smithi163.stdout:794: finishing rollback tid 1 to smithi16367185-26 2023-10-09T09:08:40.362 INFO:tasks.rados.rados.0.smithi163.stdout:794: finishing rollback tid 2 to smithi16367185-26 2023-10-09T09:08:40.362 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 263 (ObjNum 25 snap 0 seq_num 25) dirty exists 2023-10-09T09:08:40.362 INFO:tasks.rados.rados.0.smithi163.stdout:789: done (3 left) 2023-10-09T09:08:40.362 INFO:tasks.rados.rados.0.smithi163.stdout:791: done (2 left) 2023-10-09T09:08:40.363 INFO:tasks.rados.rados.0.smithi163.stdout:792: done (1 left) 2023-10-09T09:08:40.363 INFO:tasks.rados.rados.0.smithi163.stdout:794: done (0 left) 2023-10-09T09:08:40.363 INFO:tasks.rados.rados.0.smithi163.stdout:795: write oid 36 current snap is 101 2023-10-09T09:08:40.363 INFO:tasks.rados.rados.0.smithi163.stdout:795: seq_num 333 ranges {796731=461096,1736739=723770,2928680=668051} 2023-10-09T09:08:40.374 INFO:tasks.rados.rados.0.smithi163.stdout:795: writing smithi16367185-36 from 796731 to 1257827 tid 1 2023-10-09T09:08:40.385 INFO:tasks.rados.rados.0.smithi163.stdout:795: writing smithi16367185-36 from 1736739 to 2460509 tid 2 2023-10-09T09:08:40.393 INFO:tasks.rados.rados.0.smithi163.stdout:795: writing smithi16367185-36 from 2928680 to 3596731 tid 3 2023-10-09T09:08:40.393 INFO:tasks.rados.rados.0.smithi163.stdout:796: snap_remove snap 94 2023-10-09T09:08:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:40 smithi181 bash[27712]: cluster 2023-10-09T09:08:39.468832+0000 mgr.y (mgr.15273) 13 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:40 smithi181 bash[27712]: cluster 2023-10-09T09:08:40.171634+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2023-10-09T09:08:40.717 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:40 smithi163 bash[33912]: cluster 2023-10-09T09:08:39.468832+0000 mgr.y (mgr.15273) 13 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:40 smithi163 bash[33912]: cluster 2023-10-09T09:08:40.171634+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2023-10-09T09:08:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:40 smithi163 bash[27500]: cluster 2023-10-09T09:08:39.468832+0000 mgr.y (mgr.15273) 13 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:40 smithi163 bash[27500]: cluster 2023-10-09T09:08:40.171634+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2023-10-09T09:08:41.358 INFO:tasks.rados.rados.0.smithi163.stdout:796: done (1 left) 2023-10-09T09:08:41.358 INFO:tasks.rados.rados.0.smithi163.stdout:797: write oid 47 current snap is 101 2023-10-09T09:08:41.358 INFO:tasks.rados.rados.0.smithi163.stdout:795: finishing write tid 1 to smithi16367185-36 2023-10-09T09:08:41.358 INFO:tasks.rados.rados.0.smithi163.stdout:795: finishing write tid 2 to smithi16367185-36 2023-10-09T09:08:41.358 INFO:tasks.rados.rados.0.smithi163.stdout:797: seq_num 334 ranges {413538=536345,1395221=583781,2556103=658617} 2023-10-09T09:08:41.363 INFO:tasks.rados.rados.0.smithi163.stdout:797: writing smithi16367185-47 from 413538 to 949883 tid 1 2023-10-09T09:08:41.370 INFO:tasks.rados.rados.0.smithi163.stdout:797: writing smithi16367185-47 from 1395221 to 1979002 tid 2 2023-10-09T09:08:41.379 INFO:tasks.rados.rados.0.smithi163.stdout:797: writing smithi16367185-47 from 2556103 to 3214720 tid 3 2023-10-09T09:08:41.379 INFO:tasks.rados.rados.0.smithi163.stdout:798: read oid 19 snap 99 2023-10-09T09:08:41.379 INFO:tasks.rados.rados.0.smithi163.stdout:798: expect (ObjNum 324 snap 99 seq_num 324) 2023-10-09T09:08:41.379 INFO:tasks.rados.rados.0.smithi163.stdout:799: snap_remove snap 90 2023-10-09T09:08:41.659 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: audit 2023-10-09T09:08:40.344148+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.659 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: audit 2023-10-09T09:08:40.350054+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: audit 2023-10-09T09:08:40.351892+0000 mon.a (mon.0) 1146 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: audit 2023-10-09T09:08:40.352783+0000 mon.a (mon.0) 1147 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: cephadm 2023-10-09T09:08:40.354122+0000 mgr.y (mgr.15273) 14 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: cephadm 2023-10-09T09:08:40.354386+0000 mgr.y (mgr.15273) 15 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: cephadm 2023-10-09T09:08:40.518790+0000 mgr.y (mgr.15273) 16 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: cephadm 2023-10-09T09:08:40.519664+0000 mgr.y (mgr.15273) 17 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: cephadm 2023-10-09T09:08:40.683583+0000 mgr.y (mgr.15273) 18 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: cephadm 2023-10-09T09:08:40.684511+0000 mgr.y (mgr.15273) 19 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: cephadm 2023-10-09T09:08:40.847975+0000 mgr.y (mgr.15273) 20 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: cephadm 2023-10-09T09:08:40.850184+0000 mgr.y (mgr.15273) 21 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: audit 2023-10-09T09:08:41.021053+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: audit 2023-10-09T09:08:41.025862+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: audit 2023-10-09T09:08:41.030141+0000 mon.a (mon.0) 1150 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: audit 2023-10-09T09:08:41.034148+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: audit 2023-10-09T09:08:41.039219+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:41 smithi181 bash[27712]: audit 2023-10-09T09:08:41.040092+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.163:0/45783494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: audit 2023-10-09T09:08:40.344148+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: audit 2023-10-09T09:08:40.350054+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: audit 2023-10-09T09:08:40.351892+0000 mon.a (mon.0) 1146 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: audit 2023-10-09T09:08:40.352783+0000 mon.a (mon.0) 1147 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:08:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: cephadm 2023-10-09T09:08:40.354122+0000 mgr.y (mgr.15273) 14 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:08:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: cephadm 2023-10-09T09:08:40.354386+0000 mgr.y (mgr.15273) 15 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:08:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: cephadm 2023-10-09T09:08:40.518790+0000 mgr.y (mgr.15273) 16 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:08:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: cephadm 2023-10-09T09:08:40.519664+0000 mgr.y (mgr.15273) 17 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:08:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: cephadm 2023-10-09T09:08:40.683583+0000 mgr.y (mgr.15273) 18 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:08:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: cephadm 2023-10-09T09:08:40.684511+0000 mgr.y (mgr.15273) 19 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:08:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: cephadm 2023-10-09T09:08:40.847975+0000 mgr.y (mgr.15273) 20 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:08:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: cephadm 2023-10-09T09:08:40.850184+0000 mgr.y (mgr.15273) 21 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:08:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: audit 2023-10-09T09:08:41.021053+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: audit 2023-10-09T09:08:41.025862+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: audit 2023-10-09T09:08:41.030141+0000 mon.a (mon.0) 1150 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: audit 2023-10-09T09:08:41.034148+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: audit 2023-10-09T09:08:41.039219+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[27500]: audit 2023-10-09T09:08:41.040092+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.163:0/45783494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:41.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: audit 2023-10-09T09:08:40.344148+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: audit 2023-10-09T09:08:40.350054+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: audit 2023-10-09T09:08:40.351892+0000 mon.a (mon.0) 1146 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:41.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: audit 2023-10-09T09:08:40.352783+0000 mon.a (mon.0) 1147 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:08:41.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: cephadm 2023-10-09T09:08:40.354122+0000 mgr.y (mgr.15273) 14 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:08:41.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: cephadm 2023-10-09T09:08:40.354386+0000 mgr.y (mgr.15273) 15 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:08:41.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: cephadm 2023-10-09T09:08:40.518790+0000 mgr.y (mgr.15273) 16 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:08:41.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: cephadm 2023-10-09T09:08:40.519664+0000 mgr.y (mgr.15273) 17 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:08:41.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: cephadm 2023-10-09T09:08:40.683583+0000 mgr.y (mgr.15273) 18 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:08:41.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: cephadm 2023-10-09T09:08:40.684511+0000 mgr.y (mgr.15273) 19 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:08:41.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: cephadm 2023-10-09T09:08:40.847975+0000 mgr.y (mgr.15273) 20 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:08:41.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: cephadm 2023-10-09T09:08:40.850184+0000 mgr.y (mgr.15273) 21 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:08:41.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: audit 2023-10-09T09:08:41.021053+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: audit 2023-10-09T09:08:41.025862+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: audit 2023-10-09T09:08:41.030141+0000 mon.a (mon.0) 1150 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: audit 2023-10-09T09:08:41.034148+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: audit 2023-10-09T09:08:41.039219+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:41.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:41 smithi163 bash[33912]: audit 2023-10-09T09:08:41.040092+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.163:0/45783494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:42.366 INFO:tasks.rados.rados.0.smithi163.stdout:795: finishing write tid 3 to smithi16367185-36 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:795: finishing write tid 4 to smithi16367185-36 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:795: finishing write tid 5 to smithi16367185-36 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 251 (ObjNum 333 snap 101 seq_num 333) dirty exists 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:795: left oid 36 (ObjNum 333 snap 101 seq_num 333) 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:797: finishing write tid 1 to smithi16367185-47 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:797: finishing write tid 2 to smithi16367185-47 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:797: finishing write tid 3 to smithi16367185-47 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:797: finishing write tid 4 to smithi16367185-47 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:797: finishing write tid 5 to smithi16367185-47 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 687 (ObjNum 334 snap 101 seq_num 334) dirty exists 2023-10-09T09:08:42.367 INFO:tasks.rados.rados.0.smithi163.stdout:797: left oid 47 (ObjNum 334 snap 101 seq_num 334) 2023-10-09T09:08:42.368 INFO:tasks.rados.rados.0.smithi163.stdout:798: expect (ObjNum 324 snap 99 seq_num 324) 2023-10-09T09:08:42.487 INFO:tasks.rados.rados.0.smithi163.stdout:795: done (3 left) 2023-10-09T09:08:42.487 INFO:tasks.rados.rados.0.smithi163.stdout:797: done (2 left) 2023-10-09T09:08:42.487 INFO:tasks.rados.rados.0.smithi163.stdout:798: done (1 left) 2023-10-09T09:08:42.487 INFO:tasks.rados.rados.0.smithi163.stdout:799: done (0 left) 2023-10-09T09:08:42.487 INFO:tasks.rados.rados.0.smithi163.stdout:800: write oid 29 current snap is 101 2023-10-09T09:08:42.487 INFO:tasks.rados.rados.0.smithi163.stdout:800: seq_num 335 ranges {430345=727947,1570056=727439,2699879=469889} 2023-10-09T09:08:42.499 INFO:tasks.rados.rados.0.smithi163.stdout:800: writing smithi16367185-29 from 430345 to 1158292 tid 1 2023-10-09T09:08:42.506 INFO:tasks.rados.rados.0.smithi163.stdout:800: writing smithi16367185-29 from 1570056 to 2297495 tid 2 2023-10-09T09:08:42.511 INFO:tasks.rados.rados.0.smithi163.stdout:800: writing smithi16367185-29 from 2699879 to 3169768 tid 3 2023-10-09T09:08:42.511 INFO:tasks.rados.rados.0.smithi163.stdout:801: write oid 30 current snap is 101 2023-10-09T09:08:42.511 INFO:tasks.rados.rados.0.smithi163.stdout:800: finishing write tid 1 to smithi16367185-29 2023-10-09T09:08:42.511 INFO:tasks.rados.rados.0.smithi163.stdout:801: seq_num 336 ranges {447152=403196,1512185=587450,2727302=681161} 2023-10-09T09:08:42.518 INFO:tasks.rados.rados.0.smithi163.stdout:801: writing smithi16367185-30 from 447152 to 850348 tid 1 2023-10-09T09:08:42.526 INFO:tasks.rados.rados.0.smithi163.stdout:801: writing smithi16367185-30 from 1512185 to 2099635 tid 2 2023-10-09T09:08:42.533 INFO:tasks.rados.rados.0.smithi163.stdout:801: writing smithi16367185-30 from 2727302 to 3408463 tid 3 2023-10-09T09:08:42.534 INFO:tasks.rados.rados.0.smithi163.stdout:802: snap_create 2023-10-09T09:08:42.534 INFO:tasks.rados.rados.0.smithi163.stdout:800: finishing write tid 2 to smithi16367185-29 2023-10-09T09:08:42.534 INFO:tasks.rados.rados.0.smithi163.stdout:800: finishing write tid 3 to smithi16367185-29 2023-10-09T09:08:42.534 INFO:tasks.rados.rados.0.smithi163.stdout:800: finishing write tid 4 to smithi16367185-29 2023-10-09T09:08:42.534 INFO:tasks.rados.rados.0.smithi163.stdout:800: finishing write tid 5 to smithi16367185-29 2023-10-09T09:08:42.534 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 383 (ObjNum 335 snap 101 seq_num 335) dirty exists 2023-10-09T09:08:42.534 INFO:tasks.rados.rados.0.smithi163.stdout:800: left oid 29 (ObjNum 335 snap 101 seq_num 335) 2023-10-09T09:08:42.534 INFO:tasks.rados.rados.0.smithi163.stdout:801: finishing write tid 1 to smithi16367185-30 2023-10-09T09:08:42.534 INFO:tasks.rados.rados.0.smithi163.stdout:801: finishing write tid 2 to smithi16367185-30 2023-10-09T09:08:42.537 INFO:tasks.rados.rados.0.smithi163.stdout:801: finishing write tid 3 to smithi16367185-30 2023-10-09T09:08:42.538 INFO:tasks.rados.rados.0.smithi163.stdout:801: finishing write tid 4 to smithi16367185-30 2023-10-09T09:08:42.538 INFO:tasks.rados.rados.0.smithi163.stdout:801: finishing write tid 5 to smithi16367185-30 2023-10-09T09:08:42.538 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 309 (ObjNum 336 snap 101 seq_num 336) dirty exists 2023-10-09T09:08:42.538 INFO:tasks.rados.rados.0.smithi163.stdout:801: left oid 30 (ObjNum 336 snap 101 seq_num 336) 2023-10-09T09:08:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:42 smithi181 bash[27712]: cephadm 2023-10-09T09:08:41.112637+0000 mgr.y (mgr.15273) 22 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:08:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:42 smithi181 bash[27712]: cluster 2023-10-09T09:08:41.355775+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2023-10-09T09:08:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:42 smithi181 bash[27712]: cephadm 2023-10-09T09:08:41.367646+0000 mgr.y (mgr.15273) 23 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:08:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:42 smithi181 bash[27712]: cluster 2023-10-09T09:08:41.469536+0000 mgr.y (mgr.15273) 24 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2023-10-09T09:08:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:42 smithi163 bash[27500]: cephadm 2023-10-09T09:08:41.112637+0000 mgr.y (mgr.15273) 22 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:08:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:42 smithi163 bash[27500]: cluster 2023-10-09T09:08:41.355775+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2023-10-09T09:08:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:42 smithi163 bash[27500]: cephadm 2023-10-09T09:08:41.367646+0000 mgr.y (mgr.15273) 23 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:08:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:42 smithi163 bash[27500]: cluster 2023-10-09T09:08:41.469536+0000 mgr.y (mgr.15273) 24 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2023-10-09T09:08:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:42 smithi163 bash[33912]: cephadm 2023-10-09T09:08:41.112637+0000 mgr.y (mgr.15273) 22 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:08:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:42 smithi163 bash[33912]: cluster 2023-10-09T09:08:41.355775+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2023-10-09T09:08:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:42 smithi163 bash[33912]: cephadm 2023-10-09T09:08:41.367646+0000 mgr.y (mgr.15273) 23 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:08:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:42 smithi163 bash[33912]: cluster 2023-10-09T09:08:41.469536+0000 mgr.y (mgr.15273) 24 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2023-10-09T09:08:43.371 INFO:tasks.rados.rados.0.smithi163.stdout:800: done (2 left) 2023-10-09T09:08:43.372 INFO:tasks.rados.rados.0.smithi163.stdout:801: done (1 left) 2023-10-09T09:08:43.372 INFO:tasks.rados.rados.0.smithi163.stdout:802: done (0 left) 2023-10-09T09:08:43.372 INFO:tasks.rados.rados.0.smithi163.stdout:803: delete oid 35 current snap is 102 2023-10-09T09:08:43.377 INFO:tasks.rados.rados.0.smithi163.stdout:803: done (0 left) 2023-10-09T09:08:43.377 INFO:tasks.rados.rados.0.smithi163.stdout:804: write oid 17 current snap is 102 2023-10-09T09:08:43.377 INFO:tasks.rados.rados.0.smithi163.stdout:804: seq_num 338 ranges {480766=553694,1629149=591119,2782148=420058} 2023-10-09T09:08:43.383 INFO:tasks.rados.rados.0.smithi163.stdout:804: writing smithi16367185-17 from 480766 to 1034460 tid 1 2023-10-09T09:08:43.391 INFO:tasks.rados.rados.0.smithi163.stdout:804: writing smithi16367185-17 from 1629149 to 2220268 tid 2 2023-10-09T09:08:43.397 INFO:tasks.rados.rados.0.smithi163.stdout:804: writing smithi16367185-17 from 2782148 to 3202206 tid 3 2023-10-09T09:08:43.397 INFO:tasks.rados.rados.0.smithi163.stdout:805: read oid 50 snap -1 2023-10-09T09:08:43.397 INFO:tasks.rados.rados.0.smithi163.stdout:805: expect (ObjNum 314 snap 94 seq_num 314) 2023-10-09T09:08:43.397 INFO:tasks.rados.rados.0.smithi163.stdout:806: write oid 1 current snap is 102 2023-10-09T09:08:43.397 INFO:tasks.rados.rados.0.smithi163.stdout:804: finishing write tid 1 to smithi16367185-17 2023-10-09T09:08:43.397 INFO:tasks.rados.rados.0.smithi163.stdout:806: seq_num 339 ranges {497573=628943,1687631=734777,3093218=604355} 2023-10-09T09:08:43.403 INFO:tasks.rados.rados.0.smithi163.stdout:806: writing smithi16367185-1 from 497573 to 1126516 tid 1 2023-10-09T09:08:43.412 INFO:tasks.rados.rados.0.smithi163.stdout:806: writing smithi16367185-1 from 1687631 to 2422408 tid 2 2023-10-09T09:08:43.419 INFO:tasks.rados.rados.0.smithi163.stdout:806: writing smithi16367185-1 from 3093218 to 3697573 tid 3 2023-10-09T09:08:43.420 INFO:tasks.rados.rados.0.smithi163.stdout:807: snap_remove snap 100 2023-10-09T09:08:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:43 smithi163 bash[33912]: cluster 2023-10-09T09:08:42.365198+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2023-10-09T09:08:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:43 smithi163 bash[27500]: cluster 2023-10-09T09:08:42.365198+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2023-10-09T09:08:43.808 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:43 smithi181 bash[27712]: cluster 2023-10-09T09:08:42.365198+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2023-10-09T09:08:44.387 INFO:tasks.rados.rados.0.smithi163.stdout:807: done (3 left) 2023-10-09T09:08:44.388 INFO:tasks.rados.rados.0.smithi163.stdout:808: read oid 14 snap -1 2023-10-09T09:08:44.388 INFO:tasks.rados.rados.0.smithi163.stdout:808: expect (ObjNum 322 snap 96 seq_num 322) 2023-10-09T09:08:44.388 INFO:tasks.rados.rados.0.smithi163.stdout:809: snap_remove snap 92 2023-10-09T09:08:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:44 smithi181 bash[27712]: cluster 2023-10-09T09:08:43.371282+0000 mon.a (mon.0) 1155 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2023-10-09T09:08:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:44 smithi181 bash[27712]: cluster 2023-10-09T09:08:43.470021+0000 mgr.y (mgr.15273) 25 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2023-10-09T09:08:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:44 smithi181 bash[27712]: audit 2023-10-09T09:08:43.506215+0000 mon.a (mon.0) 1156 : audit [DBG] from='client.? 172.21.15.163:0/1860900066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:44 smithi163 bash[27500]: cluster 2023-10-09T09:08:43.371282+0000 mon.a (mon.0) 1155 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2023-10-09T09:08:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:44 smithi163 bash[27500]: cluster 2023-10-09T09:08:43.470021+0000 mgr.y (mgr.15273) 25 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2023-10-09T09:08:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:44 smithi163 bash[27500]: audit 2023-10-09T09:08:43.506215+0000 mon.a (mon.0) 1156 : audit [DBG] from='client.? 172.21.15.163:0/1860900066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:44 smithi163 bash[33912]: cluster 2023-10-09T09:08:43.371282+0000 mon.a (mon.0) 1155 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2023-10-09T09:08:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:44 smithi163 bash[33912]: cluster 2023-10-09T09:08:43.470021+0000 mgr.y (mgr.15273) 25 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2023-10-09T09:08:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:44 smithi163 bash[33912]: audit 2023-10-09T09:08:43.506215+0000 mon.a (mon.0) 1156 : audit [DBG] from='client.? 172.21.15.163:0/1860900066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:45.396 INFO:tasks.rados.rados.0.smithi163.stdout:809: done (4 left) 2023-10-09T09:08:45.396 INFO:tasks.rados.rados.0.smithi163.stdout:810: rollback oid 7 current snap is 102 2023-10-09T09:08:45.396 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 88 2023-10-09T09:08:45.396 INFO:tasks.rados.rados.0.smithi163.stdout:804: finishing write tid 2 to smithi16367185-17 2023-10-09T09:08:45.396 INFO:tasks.rados.rados.0.smithi163.stdout:804: finishing write tid 3 to smithi16367185-17 2023-10-09T09:08:45.397 INFO:tasks.rados.rados.0.smithi163.stdout:804: finishing write tid 4 to smithi16367185-17 2023-10-09T09:08:45.397 INFO:tasks.rados.rados.0.smithi163.stdout:804: finishing write tid 5 to smithi16367185-17 2023-10-09T09:08:45.397 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 469 (ObjNum 338 snap 102 seq_num 338) dirty exists 2023-10-09T09:08:45.397 INFO:tasks.rados.rados.0.smithi163.stdout:804: left oid 17 (ObjNum 338 snap 102 seq_num 338) 2023-10-09T09:08:45.397 INFO:tasks.rados.rados.0.smithi163.stdout:805: expect (ObjNum 314 snap 94 seq_num 314) 2023-10-09T09:08:45.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[27712]: cluster 2023-10-09T09:08:44.387278+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2023-10-09T09:08:45.490 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 systemd[1]: Stopping Ceph prometheus.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[54779]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.315Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.315Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.315Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.315Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.315Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.315Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.316Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.316Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.316Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T09:08:45.491 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.317Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T09:08:45.492 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.317Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T09:08:45.492 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[52947]: ts=2023-10-09T09:08:45.317Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T09:08:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:45 smithi163 bash[33912]: cluster 2023-10-09T09:08:44.387278+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2023-10-09T09:08:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:45 smithi163 bash[27500]: cluster 2023-10-09T09:08:44.387278+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2023-10-09T09:08:45.750 INFO:tasks.rados.rados.0.smithi163.stdout:806: finishing write tid 1 to smithi16367185-1 2023-10-09T09:08:45.750 INFO:tasks.rados.rados.0.smithi163.stdout:806: finishing write tid 2 to smithi16367185-1 2023-10-09T09:08:45.750 INFO:tasks.rados.rados.0.smithi163.stdout:806: finishing write tid 3 to smithi16367185-1 2023-10-09T09:08:45.751 INFO:tasks.rados.rados.0.smithi163.stdout:806: finishing write tid 4 to smithi16367185-1 2023-10-09T09:08:45.751 INFO:tasks.rados.rados.0.smithi163.stdout:806: finishing write tid 5 to smithi16367185-1 2023-10-09T09:08:45.751 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 454 (ObjNum 339 snap 102 seq_num 339) dirty exists 2023-10-09T09:08:45.751 INFO:tasks.rados.rados.0.smithi163.stdout:806: left oid 1 (ObjNum 339 snap 102 seq_num 339) 2023-10-09T09:08:45.751 INFO:tasks.rados.rados.0.smithi163.stdout:808: expect (ObjNum 322 snap 96 seq_num 322) 2023-10-09T09:08:45.817 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[54816]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T09:08:45.867 INFO:tasks.rados.rados.0.smithi163.stdout:810: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:08:45.867 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 184 (ObjNum 669367424 snap 21875 seq_num 4000000) dirty dne 2023-10-09T09:08:45.868 INFO:tasks.rados.rados.0.smithi163.stdout:804: done (4 left) 2023-10-09T09:08:45.868 INFO:tasks.rados.rados.0.smithi163.stdout:805: done (3 left) 2023-10-09T09:08:45.868 INFO:tasks.rados.rados.0.smithi163.stdout:806: done (2 left) 2023-10-09T09:08:45.868 INFO:tasks.rados.rados.0.smithi163.stdout:808: done (1 left) 2023-10-09T09:08:45.868 INFO:tasks.rados.rados.0.smithi163.stdout:810: done (0 left) 2023-10-09T09:08:45.868 INFO:tasks.rados.rados.0.smithi163.stdout:811: read oid 19 snap 98 2023-10-09T09:08:45.868 INFO:tasks.rados.rados.0.smithi163.stdout:811: expect deleted 2023-10-09T09:08:45.868 INFO:tasks.rados.rados.0.smithi163.stdout:812: rollback oid 47 current snap is 102 2023-10-09T09:08:45.869 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 47 to 75 2023-10-09T09:08:45.869 INFO:tasks.rados.rados.0.smithi163.stdout:813: rollback oid 9 current snap is 102 2023-10-09T09:08:45.869 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 88 2023-10-09T09:08:45.869 INFO:tasks.rados.rados.0.smithi163.stdout:814: snap_remove snap 95 2023-10-09T09:08:46.317 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 bash[54877]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T09:08:46.317 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T09:08:46.317 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 systemd[1]: Stopped Ceph prometheus.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:08:46.317 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:45 smithi181 systemd[1]: Started Ceph prometheus.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:08:46.411 INFO:tasks.rados.rados.0.smithi163.stdout:814: done (3 left) 2023-10-09T09:08:46.412 INFO:tasks.rados.rados.0.smithi163.stdout:815: write oid 11 current snap is 102 2023-10-09T09:08:46.412 INFO:tasks.rados.rados.0.smithi163.stdout:815: seq_num 340 ranges {514380=704192,1629760=478435,2604288=442602} 2023-10-09T09:08:46.419 INFO:tasks.rados.rados.0.smithi163.stdout:815: writing smithi16367185-11 from 514380 to 1218572 tid 1 2023-10-09T09:08:46.425 INFO:tasks.rados.rados.0.smithi163.stdout:815: writing smithi16367185-11 from 1629760 to 2108195 tid 2 2023-10-09T09:08:46.430 INFO:tasks.rados.rados.0.smithi163.stdout:815: writing smithi16367185-11 from 2604288 to 3046890 tid 3 2023-10-09T09:08:46.430 INFO:tasks.rados.rados.0.smithi163.stdout:816: snap_remove snap 101 2023-10-09T09:08:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[27500]: cluster 2023-10-09T09:08:45.395062+0000 mon.a (mon.0) 1158 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2023-10-09T09:08:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[27500]: cluster 2023-10-09T09:08:45.470473+0000 mgr.y (mgr.15273) 26 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2023-10-09T09:08:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[27500]: audit 2023-10-09T09:08:45.941900+0000 mon.a (mon.0) 1159 : audit [DBG] from='client.? 172.21.15.163:0/2753690473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[27500]: audit 2023-10-09T09:08:45.969448+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[27500]: audit 2023-10-09T09:08:45.977931+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[27500]: audit 2023-10-09T09:08:46.009031+0000 mon.a (mon.0) 1162 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[27500]: audit 2023-10-09T09:08:46.010096+0000 mgr.y (mgr.15273) 27 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[27500]: audit 2023-10-09T09:08:46.019644+0000 mon.a (mon.0) 1163 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[33912]: cluster 2023-10-09T09:08:45.395062+0000 mon.a (mon.0) 1158 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2023-10-09T09:08:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[33912]: cluster 2023-10-09T09:08:45.470473+0000 mgr.y (mgr.15273) 26 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2023-10-09T09:08:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[33912]: audit 2023-10-09T09:08:45.941900+0000 mon.a (mon.0) 1159 : audit [DBG] from='client.? 172.21.15.163:0/2753690473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[33912]: audit 2023-10-09T09:08:45.969448+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[33912]: audit 2023-10-09T09:08:45.977931+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[33912]: audit 2023-10-09T09:08:46.009031+0000 mon.a (mon.0) 1162 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[33912]: audit 2023-10-09T09:08:46.010096+0000 mgr.y (mgr.15273) 27 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:46 smithi163 bash[33912]: audit 2023-10-09T09:08:46.019644+0000 mon.a (mon.0) 1163 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:46 smithi181 bash[27712]: cluster 2023-10-09T09:08:45.395062+0000 mon.a (mon.0) 1158 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2023-10-09T09:08:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:46 smithi181 bash[27712]: cluster 2023-10-09T09:08:45.470473+0000 mgr.y (mgr.15273) 26 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2023-10-09T09:08:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:46 smithi181 bash[27712]: audit 2023-10-09T09:08:45.941900+0000 mon.a (mon.0) 1159 : audit [DBG] from='client.? 172.21.15.163:0/2753690473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:46 smithi181 bash[27712]: audit 2023-10-09T09:08:45.969448+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:46 smithi181 bash[27712]: audit 2023-10-09T09:08:45.977931+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:46 smithi181 bash[27712]: audit 2023-10-09T09:08:46.009031+0000 mon.a (mon.0) 1162 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:46 smithi181 bash[27712]: audit 2023-10-09T09:08:46.010096+0000 mgr.y (mgr.15273) 27 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:08:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:46 smithi181 bash[27712]: audit 2023-10-09T09:08:46.019644+0000 mon.a (mon.0) 1163 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:47.157 INFO:tasks.rados.rados.0.smithi163.stdout:816: done (4 left) 2023-10-09T09:08:47.158 INFO:tasks.rados.rados.0.smithi163.stdout:817: snap_remove snap 96 2023-10-09T09:08:47.694 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[27712]: cluster 2023-10-09T09:08:46.410778+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2023-10-09T09:08:47.694 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[27712]: cluster 2023-10-09T09:08:47.156524+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2023-10-09T09:08:47.694 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.531Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T09:08:47.694 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.531Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T09:08:47.695 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.531Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi181 (none))" 2023-10-09T09:08:47.695 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.531Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T09:08:47.695 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.531Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T09:08:47.695 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.533Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T09:08:47.695 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.534Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T09:08:47.695 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.536Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T09:08:47.695 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.540Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T09:08:47.695 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.540Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.97µs 2023-10-09T09:08:47.695 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.540Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T09:08:47.695 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.692Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T09:08:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:47 smithi163 bash[33912]: cluster 2023-10-09T09:08:46.410778+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2023-10-09T09:08:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:47 smithi163 bash[33912]: cluster 2023-10-09T09:08:47.156524+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2023-10-09T09:08:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:47 smithi163 bash[27500]: cluster 2023-10-09T09:08:46.410778+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2023-10-09T09:08:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:47 smithi163 bash[27500]: cluster 2023-10-09T09:08:47.156524+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2023-10-09T09:08:48.175 INFO:tasks.rados.rados.0.smithi163.stdout:817: done (4 left) 2023-10-09T09:08:48.175 INFO:tasks.rados.rados.0.smithi163.stdout:818: read oid 39 snap 99 2023-10-09T09:08:48.175 INFO:tasks.rados.rados.0.smithi163.stdout:818: expect deleted 2023-10-09T09:08:48.175 INFO:tasks.rados.rados.0.smithi163.stdout:813: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:813: finishing rollback tid 2 to smithi16367185-9 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 227 (ObjNum 269 snap 84 seq_num 269) dirty exists 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:812: finishing rollback tid 0 to smithi16367185-47 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:812: finishing rollback tid 1 to smithi16367185-47 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 704 (ObjNum 0 snap 32603 seq_num 698877936) dirty dne 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:815: finishing write tid 1 to smithi16367185-11 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:815: finishing write tid 2 to smithi16367185-11 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:815: finishing write tid 3 to smithi16367185-11 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:815: finishing write tid 4 to smithi16367185-11 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:815: finishing write tid 5 to smithi16367185-11 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 211 (ObjNum 340 snap 102 seq_num 340) dirty exists 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:815: left oid 11 (ObjNum 340 snap 102 seq_num 340) 2023-10-09T09:08:48.176 INFO:tasks.rados.rados.0.smithi163.stdout:811: done (4 left) 2023-10-09T09:08:48.177 INFO:tasks.rados.rados.0.smithi163.stdout:812: done (3 left) 2023-10-09T09:08:48.177 INFO:tasks.rados.rados.0.smithi163.stdout:813: done (2 left) 2023-10-09T09:08:48.177 INFO:tasks.rados.rados.0.smithi163.stdout:815: done (1 left) 2023-10-09T09:08:48.177 INFO:tasks.rados.rados.0.smithi163.stdout:819: read oid 43 snap 88 2023-10-09T09:08:48.177 INFO:tasks.rados.rados.0.smithi163.stdout:819: expect (ObjNum 271 snap 84 seq_num 271) 2023-10-09T09:08:48.177 INFO:tasks.rados.rados.0.smithi163.stdout:820: snap_create 2023-10-09T09:08:48.185 INFO:tasks.rados.rados.0.smithi163.stdout:819: expect (ObjNum 271 snap 84 seq_num 271) 2023-10-09T09:08:48.317 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.944Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T09:08:48.317 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.988Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T09:08:48.317 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.989Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T09:08:48.317 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.989Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=89.047µs wal_replay_duration=448.888862ms total_replay_duration=449.005507ms 2023-10-09T09:08:48.317 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.995Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T09:08:48.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.995Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T09:08:48.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:47 smithi181 bash[54907]: ts=2023-10-09T09:08:47.995Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T09:08:48.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:48 smithi181 bash[54907]: ts=2023-10-09T09:08:48.020Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.33528ms db_storage=1.456µs remote_storage=1.788µs web_handler=586ns query_engine=1.067µs scrape=330.875µs scrape_sd=106.784µs notify=21.56µs notify_sd=22.354µs rules=24.336988ms 2023-10-09T09:08:48.318 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:48 smithi181 bash[54907]: ts=2023-10-09T09:08:48.020Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T09:08:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:48 smithi163 bash[27500]: cluster 2023-10-09T09:08:47.471158+0000 mgr.y (mgr.15273) 28 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2023-10-09T09:08:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:48 smithi163 bash[27500]: cluster 2023-10-09T09:08:48.175326+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2023-10-09T09:08:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:48 smithi163 bash[27500]: audit 2023-10-09T09:08:48.389993+0000 mon.a (mon.0) 1167 : audit [DBG] from='client.? 172.21.15.163:0/3224637085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:48 smithi163 bash[33912]: cluster 2023-10-09T09:08:47.471158+0000 mgr.y (mgr.15273) 28 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2023-10-09T09:08:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:48 smithi163 bash[33912]: cluster 2023-10-09T09:08:48.175326+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2023-10-09T09:08:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:48 smithi163 bash[33912]: audit 2023-10-09T09:08:48.389993+0000 mon.a (mon.0) 1167 : audit [DBG] from='client.? 172.21.15.163:0/3224637085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:48.747 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:48 smithi181 bash[27712]: cluster 2023-10-09T09:08:47.471158+0000 mgr.y (mgr.15273) 28 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2023-10-09T09:08:48.747 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:48 smithi181 bash[27712]: cluster 2023-10-09T09:08:48.175326+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2023-10-09T09:08:48.747 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:48 smithi181 bash[27712]: audit 2023-10-09T09:08:48.389993+0000 mon.a (mon.0) 1167 : audit [DBG] from='client.? 172.21.15.163:0/3224637085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:49.170 INFO:tasks.rados.rados.0.smithi163.stdout:818: done (2 left) 2023-10-09T09:08:49.171 INFO:tasks.rados.rados.0.smithi163.stdout:819: done (1 left) 2023-10-09T09:08:49.171 INFO:tasks.rados.rados.0.smithi163.stdout:820: done (0 left) 2023-10-09T09:08:49.171 INFO:tasks.rados.rados.0.smithi163.stdout:821: write oid 35 current snap is 103 2023-10-09T09:08:49.171 INFO:tasks.rados.rados.0.smithi163.stdout:821: seq_num 341 ranges {531187=779441,2088242=738446,3431711=299476} 2023-10-09T09:08:49.177 INFO:tasks.rados.rados.0.smithi163.stdout:821: writing smithi16367185-35 from 531187 to 1310628 tid 1 2023-10-09T09:08:49.186 INFO:tasks.rados.rados.0.smithi163.stdout:821: writing smithi16367185-35 from 2088242 to 2826688 tid 2 2023-10-09T09:08:49.191 INFO:tasks.rados.rados.0.smithi163.stdout:821: writing smithi16367185-35 from 3431711 to 3731187 tid 3 2023-10-09T09:08:49.191 INFO:tasks.rados.rados.0.smithi163.stdout:822: write oid 48 current snap is 103 2023-10-09T09:08:49.191 INFO:tasks.rados.rados.0.smithi163.stdout:821: finishing write tid 1 to smithi16367185-35 2023-10-09T09:08:49.191 INFO:tasks.rados.rados.0.smithi163.stdout:821: finishing write tid 2 to smithi16367185-35 2023-10-09T09:08:49.191 INFO:tasks.rados.rados.0.smithi163.stdout:822: seq_num 342 ranges {547994=454690,1746724=482104,2659134=465146,3605976=142018} 2023-10-09T09:08:49.196 INFO:tasks.rados.rados.0.smithi163.stdout:822: writing smithi16367185-48 from 547994 to 1002684 tid 1 2023-10-09T09:08:49.204 INFO:tasks.rados.rados.0.smithi163.stdout:822: writing smithi16367185-48 from 1746724 to 2228828 tid 2 2023-10-09T09:08:49.209 INFO:tasks.rados.rados.0.smithi163.stdout:822: writing smithi16367185-48 from 2659134 to 3124280 tid 3 2023-10-09T09:08:49.213 INFO:tasks.rados.rados.0.smithi163.stdout:822: writing smithi16367185-48 from 3605976 to 3747994 tid 4 2023-10-09T09:08:49.214 INFO:tasks.rados.rados.0.smithi163.stdout:823: write oid 49 current snap is 103 2023-10-09T09:08:49.214 INFO:tasks.rados.rados.0.smithi163.stdout:821: finishing write tid 3 to smithi16367185-35 2023-10-09T09:08:49.214 INFO:tasks.rados.rados.0.smithi163.stdout:823: seq_num 343 ranges {564801=646292,1921559=742115,3202910=561891} 2023-10-09T09:08:49.220 INFO:tasks.rados.rados.0.smithi163.stdout:823: writing smithi16367185-49 from 564801 to 1211093 tid 1 2023-10-09T09:08:49.228 INFO:tasks.rados.rados.0.smithi163.stdout:823: writing smithi16367185-49 from 1921559 to 2663674 tid 2 2023-10-09T09:08:49.235 INFO:tasks.rados.rados.0.smithi163.stdout:823: writing smithi16367185-49 from 3202910 to 3764801 tid 3 2023-10-09T09:08:49.235 INFO:tasks.rados.rados.0.smithi163.stdout:824: read oid 32 snap -1 2023-10-09T09:08:49.235 INFO:tasks.rados.rados.0.smithi163.stdout:824: expect deleted 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:825: delete oid 27 current snap is 103 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:821: finishing write tid 4 to smithi16367185-35 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:821: finishing write tid 5 to smithi16367185-35 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 223 (ObjNum 341 snap 103 seq_num 341) dirty exists 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:821: left oid 35 (ObjNum 341 snap 103 seq_num 341) 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:822: finishing write tid 1 to smithi16367185-48 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:822: finishing write tid 2 to smithi16367185-48 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:822: finishing write tid 3 to smithi16367185-48 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:822: finishing write tid 4 to smithi16367185-48 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:822: finishing write tid 5 to smithi16367185-48 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:822: finishing write tid 6 to smithi16367185-48 2023-10-09T09:08:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 720 (ObjNum 342 snap 103 seq_num 342) dirty exists 2023-10-09T09:08:49.237 INFO:tasks.rados.rados.0.smithi163.stdout:822: left oid 48 (ObjNum 342 snap 103 seq_num 342) 2023-10-09T09:08:49.237 INFO:tasks.rados.rados.0.smithi163.stdout:823: finishing write tid 1 to smithi16367185-49 2023-10-09T09:08:49.237 INFO:tasks.rados.rados.0.smithi163.stdout:823: finishing write tid 2 to smithi16367185-49 2023-10-09T09:08:49.238 INFO:tasks.rados.rados.0.smithi163.stdout:823: finishing write tid 3 to smithi16367185-49 2023-10-09T09:08:49.238 INFO:tasks.rados.rados.0.smithi163.stdout:823: finishing write tid 4 to smithi16367185-49 2023-10-09T09:08:49.239 INFO:tasks.rados.rados.0.smithi163.stdout:823: finishing write tid 5 to smithi16367185-49 2023-10-09T09:08:49.239 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 154 (ObjNum 343 snap 103 seq_num 343) dirty exists 2023-10-09T09:08:49.239 INFO:tasks.rados.rados.0.smithi163.stdout:823: left oid 49 (ObjNum 343 snap 103 seq_num 343) 2023-10-09T09:08:49.240 INFO:tasks.rados.rados.0.smithi163.stdout:821: done (4 left) 2023-10-09T09:08:49.240 INFO:tasks.rados.rados.0.smithi163.stdout:822: done (3 left) 2023-10-09T09:08:49.240 INFO:tasks.rados.rados.0.smithi163.stdout:823: done (2 left) 2023-10-09T09:08:49.240 INFO:tasks.rados.rados.0.smithi163.stdout:824: done (1 left) 2023-10-09T09:08:49.240 INFO:tasks.rados.rados.0.smithi163.stdout:825: done (0 left) 2023-10-09T09:08:49.240 INFO:tasks.rados.rados.0.smithi163.stdout:826: write oid 41 current snap is 103 2023-10-09T09:08:49.240 INFO:tasks.rados.rados.0.smithi163.stdout:826: seq_num 345 ranges {598415=796790,1922170=745784,3141403=415315} 2023-10-09T09:08:49.248 INFO:tasks.rados.rados.0.smithi163.stdout:826: writing smithi16367185-41 from 598415 to 1395205 tid 1 2023-10-09T09:08:49.256 INFO:tasks.rados.rados.0.smithi163.stdout:826: writing smithi16367185-41 from 1922170 to 2667954 tid 2 2023-10-09T09:08:49.261 INFO:tasks.rados.rados.0.smithi163.stdout:826: writing smithi16367185-41 from 3141403 to 3556718 tid 3 2023-10-09T09:08:49.261 INFO:tasks.rados.rados.0.smithi163.stdout:827: write oid 49 current snap is 103 2023-10-09T09:08:49.261 INFO:tasks.rados.rados.0.smithi163.stdout:826: finishing write tid 1 to smithi16367185-41 2023-10-09T09:08:49.261 INFO:tasks.rados.rados.0.smithi163.stdout:826: finishing write tid 2 to smithi16367185-41 2023-10-09T09:08:49.261 INFO:tasks.rados.rados.0.smithi163.stdout:827: seq_num 346 ranges {615222=472039,1580652=489442,2768826=626587} 2023-10-09T09:08:49.267 INFO:tasks.rados.rados.0.smithi163.stdout:827: writing smithi16367185-49 from 615222 to 1087261 tid 1 2023-10-09T09:08:49.272 INFO:tasks.rados.rados.0.smithi163.stdout:827: writing smithi16367185-49 from 1580652 to 2070094 tid 2 2023-10-09T09:08:49.281 INFO:tasks.rados.rados.0.smithi163.stdout:827: writing smithi16367185-49 from 2768826 to 3395413 tid 3 2023-10-09T09:08:49.281 INFO:tasks.rados.rados.0.smithi163.stdout:828: read oid 37 snap 99 2023-10-09T09:08:49.281 INFO:tasks.rados.rados.0.smithi163.stdout:828: expect deleted 2023-10-09T09:08:49.281 INFO:tasks.rados.rados.0.smithi163.stdout:829: rollback oid 17 current snap is 103 2023-10-09T09:08:49.281 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 98 2023-10-09T09:08:49.281 INFO:tasks.rados.rados.0.smithi163.stdout:826: finishing write tid 3 to smithi16367185-41 2023-10-09T09:08:49.281 INFO:tasks.rados.rados.0.smithi163.stdout:826: finishing write tid 4 to smithi16367185-41 2023-10-09T09:08:49.281 INFO:tasks.rados.rados.0.smithi163.stdout:830: delete oid 23 current snap is 103 2023-10-09T09:08:49.281 INFO:tasks.rados.rados.0.smithi163.stdout:826: finishing write tid 5 to smithi16367185-41 2023-10-09T09:08:49.281 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 56 (ObjNum 345 snap 103 seq_num 345) dirty exists 2023-10-09T09:08:49.282 INFO:tasks.rados.rados.0.smithi163.stdout:826: left oid 41 (ObjNum 345 snap 103 seq_num 345) 2023-10-09T09:08:49.282 INFO:tasks.rados.rados.0.smithi163.stdout:827: finishing write tid 1 to smithi16367185-49 2023-10-09T09:08:49.282 INFO:tasks.rados.rados.0.smithi163.stdout:827: finishing write tid 2 to smithi16367185-49 2023-10-09T09:08:49.283 INFO:tasks.rados.rados.0.smithi163.stdout:826: done (4 left) 2023-10-09T09:08:49.283 INFO:tasks.rados.rados.0.smithi163.stdout:828: done (3 left) 2023-10-09T09:08:49.283 INFO:tasks.rados.rados.0.smithi163.stdout:830: done (2 left) 2023-10-09T09:08:49.284 INFO:tasks.rados.rados.0.smithi163.stdout:831: read oid 26 snap -1 2023-10-09T09:08:49.284 INFO:tasks.rados.rados.0.smithi163.stdout:831: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:08:49.284 INFO:tasks.rados.rados.0.smithi163.stdout:832: snap_create 2023-10-09T09:08:49.285 INFO:tasks.rados.rados.0.smithi163.stdout:827: finishing write tid 3 to smithi16367185-49 2023-10-09T09:08:49.285 INFO:tasks.rados.rados.0.smithi163.stdout:827: finishing write tid 4 to smithi16367185-49 2023-10-09T09:08:49.286 INFO:tasks.rados.rados.0.smithi163.stdout:827: finishing write tid 5 to smithi16367185-49 2023-10-09T09:08:49.286 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 158 (ObjNum 346 snap 103 seq_num 346) dirty exists 2023-10-09T09:08:49.286 INFO:tasks.rados.rados.0.smithi163.stdout:827: left oid 49 (ObjNum 346 snap 103 seq_num 346) 2023-10-09T09:08:49.286 INFO:tasks.rados.rados.0.smithi163.stdout:829: finishing rollback tid 0 to smithi16367185-17 2023-10-09T09:08:49.286 INFO:tasks.rados.rados.0.smithi163.stdout:829: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:08:49.286 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 474 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:08:49.290 INFO:tasks.rados.rados.0.smithi163.stdout:831: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:08:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[27500]: cephadm 2023-10-09T09:08:48.444054+0000 mgr.y (mgr.15273) 29 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:08:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[27500]: audit 2023-10-09T09:08:48.444796+0000 mon.a (mon.0) 1168 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:08:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[27500]: audit 2023-10-09T09:08:48.446822+0000 mon.a (mon.0) 1169 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:08:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[27500]: audit 2023-10-09T09:08:48.448720+0000 mon.a (mon.0) 1170 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[27500]: cephadm 2023-10-09T09:08:48.450490+0000 mgr.y (mgr.15273) 30 : cephadm [INF] Deploying daemon mgr.x on smithi181 2023-10-09T09:08:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[27500]: cluster 2023-10-09T09:08:49.170311+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2023-10-09T09:08:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[33912]: cephadm 2023-10-09T09:08:48.444054+0000 mgr.y (mgr.15273) 29 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:08:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[33912]: audit 2023-10-09T09:08:48.444796+0000 mon.a (mon.0) 1168 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:08:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[33912]: audit 2023-10-09T09:08:48.446822+0000 mon.a (mon.0) 1169 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:08:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[33912]: audit 2023-10-09T09:08:48.448720+0000 mon.a (mon.0) 1170 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[33912]: cephadm 2023-10-09T09:08:48.450490+0000 mgr.y (mgr.15273) 30 : cephadm [INF] Deploying daemon mgr.x on smithi181 2023-10-09T09:08:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:49 smithi163 bash[33912]: cluster 2023-10-09T09:08:49.170311+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2023-10-09T09:08:49.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:49 smithi181 bash[27712]: cephadm 2023-10-09T09:08:48.444054+0000 mgr.y (mgr.15273) 29 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:08:49.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:49 smithi181 bash[27712]: audit 2023-10-09T09:08:48.444796+0000 mon.a (mon.0) 1168 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:08:49.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:49 smithi181 bash[27712]: audit 2023-10-09T09:08:48.446822+0000 mon.a (mon.0) 1169 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:08:49.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:49 smithi181 bash[27712]: audit 2023-10-09T09:08:48.448720+0000 mon.a (mon.0) 1170 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:08:49.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:49 smithi181 bash[27712]: cephadm 2023-10-09T09:08:48.450490+0000 mgr.y (mgr.15273) 30 : cephadm [INF] Deploying daemon mgr.x on smithi181 2023-10-09T09:08:49.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:49 smithi181 bash[27712]: cluster 2023-10-09T09:08:49.170311+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2023-10-09T09:08:50.174 INFO:tasks.rados.rados.0.smithi163.stdout:827: done (3 left) 2023-10-09T09:08:50.175 INFO:tasks.rados.rados.0.smithi163.stdout:829: done (2 left) 2023-10-09T09:08:50.175 INFO:tasks.rados.rados.0.smithi163.stdout:831: done (1 left) 2023-10-09T09:08:50.175 INFO:tasks.rados.rados.0.smithi163.stdout:832: done (0 left) 2023-10-09T09:08:50.175 INFO:tasks.rados.rados.0.smithi163.stdout:833: rollback oid 18 current snap is 104 2023-10-09T09:08:50.175 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 18 to 87 2023-10-09T09:08:50.175 INFO:tasks.rados.rados.0.smithi163.stdout:834: snap_create 2023-10-09T09:08:50.179 INFO:tasks.rados.rados.0.smithi163.stdout:833: finishing rollback tid 0 to smithi16367185-18 2023-10-09T09:08:50.179 INFO:tasks.rados.rados.0.smithi163.stdout:833: finishing rollback tid 1 to smithi16367185-18 2023-10-09T09:08:50.179 INFO:tasks.rados.rados.0.smithi163.stdout:833: finishing rollback tid 2 to smithi16367185-18 2023-10-09T09:08:50.179 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 327 (ObjNum 265 snap 80 seq_num 265) dirty exists 2023-10-09T09:08:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:50 smithi163 bash[33912]: cluster 2023-10-09T09:08:49.471763+0000 mgr.y (mgr.15273) 31 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:50 smithi163 bash[33912]: cluster 2023-10-09T09:08:50.174317+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2023-10-09T09:08:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:50 smithi163 bash[27500]: cluster 2023-10-09T09:08:49.471763+0000 mgr.y (mgr.15273) 31 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:50 smithi163 bash[27500]: cluster 2023-10-09T09:08:50.174317+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2023-10-09T09:08:50.730 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:50 smithi181 bash[27712]: cluster 2023-10-09T09:08:49.471763+0000 mgr.y (mgr.15273) 31 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 241 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:08:50.730 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:50 smithi181 bash[27712]: cluster 2023-10-09T09:08:50.174317+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2023-10-09T09:08:51.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:50 smithi181 systemd[1]: Stopping Ceph mgr.x for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:08:51.178 INFO:tasks.rados.rados.0.smithi163.stdout:833: done (1 left) 2023-10-09T09:08:51.179 INFO:tasks.rados.rados.0.smithi163.stdout:834: done (0 left) 2023-10-09T09:08:51.179 INFO:tasks.rados.rados.0.smithi163.stdout:835: rollback oid 38 current snap is 105 2023-10-09T09:08:51.179 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 75 2023-10-09T09:08:51.179 INFO:tasks.rados.rados.0.smithi163.stdout:836: read oid 40 snap 103 2023-10-09T09:08:51.179 INFO:tasks.rados.rados.0.smithi163.stdout:836: expect (ObjNum 298 snap 91 seq_num 298) 2023-10-09T09:08:51.179 INFO:tasks.rados.rados.0.smithi163.stdout:837: write oid 39 current snap is 105 2023-10-09T09:08:51.179 INFO:tasks.rados.rados.0.smithi163.stdout:837: seq_num 348 ranges {648836=622537,1981263=493111,3107319=741517} 2023-10-09T09:08:51.187 INFO:tasks.rados.rados.0.smithi163.stdout:837: writing smithi16367185-39 from 648836 to 1271373 tid 1 2023-10-09T09:08:51.196 INFO:tasks.rados.rados.0.smithi163.stdout:837: writing smithi16367185-39 from 1981263 to 2474374 tid 2 2023-10-09T09:08:51.203 INFO:tasks.rados.rados.0.smithi163.stdout:837: writing smithi16367185-39 from 3107319 to 3848836 tid 3 2023-10-09T09:08:51.203 INFO:tasks.rados.rados.0.smithi163.stdout:838: write oid 36 current snap is 105 2023-10-09T09:08:51.203 INFO:tasks.rados.rados.0.smithi163.stdout:838: seq_num 349 ranges {665643=697786,2039745=753122,3534742=330901} 2023-10-09T09:08:51.211 INFO:tasks.rados.rados.0.smithi163.stdout:838: writing smithi16367185-36 from 665643 to 1363429 tid 1 2023-10-09T09:08:51.219 INFO:tasks.rados.rados.0.smithi163.stdout:838: writing smithi16367185-36 from 2039745 to 2792867 tid 2 2023-10-09T09:08:51.225 INFO:tasks.rados.rados.0.smithi163.stdout:838: writing smithi16367185-36 from 3534742 to 3865643 tid 3 2023-10-09T09:08:51.225 INFO:tasks.rados.rados.0.smithi163.stdout:839: snap_remove snap 99 2023-10-09T09:08:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:51 smithi163 bash[33912]: audit 2023-10-09T09:08:50.844177+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.163:0/1186686368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:51 smithi163 bash[33912]: cluster 2023-10-09T09:08:51.178237+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2023-10-09T09:08:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:51 smithi163 bash[27500]: audit 2023-10-09T09:08:50.844177+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.163:0/1186686368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:51 smithi163 bash[27500]: cluster 2023-10-09T09:08:51.178237+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2023-10-09T09:08:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:51 smithi181 bash[27712]: audit 2023-10-09T09:08:50.844177+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.163:0/1186686368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:51 smithi181 bash[27712]: cluster 2023-10-09T09:08:51.178237+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2023-10-09T09:08:51.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:51 smithi181 bash[55209]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T09:08:51.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:51 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:08:51.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:51 smithi181 bash[55274]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T09:08:51.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:51 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T09:08:51.818 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:51 smithi181 systemd[1]: Stopped Ceph mgr.x for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:08:52.182 INFO:tasks.rados.rados.0.smithi163.stdout:839: done (4 left) 2023-10-09T09:08:52.182 INFO:tasks.rados.rados.0.smithi163.stdout:840: write oid 5 current snap is 105 2023-10-09T09:08:52.182 INFO:tasks.rados.rados.0.smithi163.stdout:835: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:08:52.182 INFO:tasks.rados.rados.0.smithi163.stdout:835: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:08:52.183 INFO:tasks.rados.rados.0.smithi163.stdout:835: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:08:52.183 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 357 (ObjNum 221 snap 61 seq_num 221) dirty exists 2023-10-09T09:08:52.183 INFO:tasks.rados.rados.0.smithi163.stdout:837: finishing write tid 1 to smithi16367185-39 2023-10-09T09:08:52.183 INFO:tasks.rados.rados.0.smithi163.stdout:836: expect (ObjNum 298 snap 91 seq_num 298) 2023-10-09T09:08:52.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:51 smithi181 systemd[1]: Started Ceph mgr.x for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:08:52.374 INFO:tasks.rados.rados.0.smithi163.stdout:837: finishing write tid 2 to smithi16367185-39 2023-10-09T09:08:52.374 INFO:tasks.rados.rados.0.smithi163.stdout:837: finishing write tid 3 to smithi16367185-39 2023-10-09T09:08:52.374 INFO:tasks.rados.rados.0.smithi163.stdout:837: finishing write tid 4 to smithi16367185-39 2023-10-09T09:08:52.374 INFO:tasks.rados.rados.0.smithi163.stdout:837: finishing write tid 5 to smithi16367185-39 2023-10-09T09:08:52.374 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 326 (ObjNum 348 snap 105 seq_num 348) dirty exists 2023-10-09T09:08:52.375 INFO:tasks.rados.rados.0.smithi163.stdout:837: left oid 39 (ObjNum 348 snap 105 seq_num 348) 2023-10-09T09:08:52.375 INFO:tasks.rados.rados.0.smithi163.stdout:838: finishing write tid 1 to smithi16367185-36 2023-10-09T09:08:52.375 INFO:tasks.rados.rados.0.smithi163.stdout:838: finishing write tid 2 to smithi16367185-36 2023-10-09T09:08:52.375 INFO:tasks.rados.rados.0.smithi163.stdout:838: finishing write tid 3 to smithi16367185-36 2023-10-09T09:08:52.375 INFO:tasks.rados.rados.0.smithi163.stdout:838: finishing write tid 4 to smithi16367185-36 2023-10-09T09:08:52.375 INFO:tasks.rados.rados.0.smithi163.stdout:838: finishing write tid 5 to smithi16367185-36 2023-10-09T09:08:52.375 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 268 (ObjNum 349 snap 105 seq_num 349) dirty exists 2023-10-09T09:08:52.375 INFO:tasks.rados.rados.0.smithi163.stdout:838: left oid 36 (ObjNum 349 snap 105 seq_num 349) 2023-10-09T09:08:52.375 INFO:tasks.rados.rados.0.smithi163.stdout:840: seq_num 350 ranges {682450=489388,1814580=496780,2878518=788028} 2023-10-09T09:08:52.388 INFO:tasks.rados.rados.0.smithi163.stdout:840: writing smithi16367185-5 from 682450 to 1171838 tid 1 2023-10-09T09:08:52.395 INFO:tasks.rados.rados.0.smithi163.stdout:840: writing smithi16367185-5 from 1814580 to 2311360 tid 2 2023-10-09T09:08:52.403 INFO:tasks.rados.rados.0.smithi163.stdout:840: writing smithi16367185-5 from 2878518 to 3666546 tid 3 2023-10-09T09:08:52.403 INFO:tasks.rados.rados.0.smithi163.stdout:835: done (4 left) 2023-10-09T09:08:52.403 INFO:tasks.rados.rados.0.smithi163.stdout:836: done (3 left) 2023-10-09T09:08:52.403 INFO:tasks.rados.rados.0.smithi163.stdout:837: done (2 left) 2023-10-09T09:08:52.403 INFO:tasks.rados.rados.0.smithi163.stdout:838: done (1 left) 2023-10-09T09:08:52.403 INFO:tasks.rados.rados.0.smithi163.stdout:841: write oid 45 current snap is 105 2023-10-09T09:08:52.403 INFO:tasks.rados.rados.0.smithi163.stdout:840: finishing write tid 1 to smithi16367185-5 2023-10-09T09:08:52.403 INFO:tasks.rados.rados.0.smithi163.stdout:841: seq_num 351 ranges {699257=564637,1873062=640438,3305941=593316} 2023-10-09T09:08:52.410 INFO:tasks.rados.rados.0.smithi163.stdout:841: writing smithi16367185-45 from 699257 to 1263894 tid 1 2023-10-09T09:08:52.417 INFO:tasks.rados.rados.0.smithi163.stdout:841: writing smithi16367185-45 from 1873062 to 2513500 tid 2 2023-10-09T09:08:52.425 INFO:tasks.rados.rados.0.smithi163.stdout:841: writing smithi16367185-45 from 3305941 to 3899257 tid 3 2023-10-09T09:08:52.425 INFO:tasks.rados.rados.0.smithi163.stdout:842: snap_create 2023-10-09T09:08:52.425 INFO:tasks.rados.rados.0.smithi163.stdout:840: finishing write tid 2 to smithi16367185-5 2023-10-09T09:08:52.425 INFO:tasks.rados.rados.0.smithi163.stdout:840: finishing write tid 3 to smithi16367185-5 2023-10-09T09:08:52.425 INFO:tasks.rados.rados.0.smithi163.stdout:840: finishing write tid 4 to smithi16367185-5 2023-10-09T09:08:52.425 INFO:tasks.rados.rados.0.smithi163.stdout:840: finishing write tid 5 to smithi16367185-5 2023-10-09T09:08:52.425 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 469 (ObjNum 350 snap 105 seq_num 350) dirty exists 2023-10-09T09:08:52.426 INFO:tasks.rados.rados.0.smithi163.stdout:840: left oid 5 (ObjNum 350 snap 105 seq_num 350) 2023-10-09T09:08:52.426 INFO:tasks.rados.rados.0.smithi163.stdout:841: finishing write tid 1 to smithi16367185-45 2023-10-09T09:08:52.426 INFO:tasks.rados.rados.0.smithi163.stdout:841: finishing write tid 2 to smithi16367185-45 2023-10-09T09:08:52.430 INFO:tasks.rados.rados.0.smithi163.stdout:841: finishing write tid 3 to smithi16367185-45 2023-10-09T09:08:52.430 INFO:tasks.rados.rados.0.smithi163.stdout:841: finishing write tid 4 to smithi16367185-45 2023-10-09T09:08:52.430 INFO:tasks.rados.rados.0.smithi163.stdout:841: finishing write tid 5 to smithi16367185-45 2023-10-09T09:08:52.430 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 727 (ObjNum 351 snap 105 seq_num 351) dirty exists 2023-10-09T09:08:52.430 INFO:tasks.rados.rados.0.smithi163.stdout:841: left oid 45 (ObjNum 351 snap 105 seq_num 351) 2023-10-09T09:08:53.158 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:52 smithi181 bash[27712]: cluster 2023-10-09T09:08:51.472303+0000 mgr.y (mgr.15273) 32 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 247 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2023-10-09T09:08:53.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:52 smithi181 bash[27712]: audit 2023-10-09T09:08:51.934409+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:53.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:52 smithi181 bash[27712]: audit 2023-10-09T09:08:51.944408+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:53.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:52 smithi181 bash[27712]: audit 2023-10-09T09:08:51.973025+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:53.159 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:52 smithi181 bash[27712]: cluster 2023-10-09T09:08:52.180153+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2023-10-09T09:08:53.159 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:52 smithi181 bash[55332]: debug 2023-10-09T09:08:52.988+0000 7f8a1625e700 1 -- 172.21.15.181:0/2540472823 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560ee1fab1e0 con 0x560ee2d2a400 2023-10-09T09:08:53.183 INFO:tasks.rados.rados.0.smithi163.stdout:840: done (2 left) 2023-10-09T09:08:53.183 INFO:tasks.rados.rados.0.smithi163.stdout:841: done (1 left) 2023-10-09T09:08:53.183 INFO:tasks.rados.rados.0.smithi163.stdout:842: done (0 left) 2023-10-09T09:08:53.183 INFO:tasks.rados.rados.0.smithi163.stdout:843: rollback oid 4 current snap is 106 2023-10-09T09:08:53.183 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 75 2023-10-09T09:08:53.183 INFO:tasks.rados.rados.0.smithi163.stdout:844: write oid 5 current snap is 106 2023-10-09T09:08:53.183 INFO:tasks.rados.rados.0.smithi163.stdout:844: seq_num 352 ranges {716064=639886,1815191=500449,2817011=526925} 2023-10-09T09:08:53.192 INFO:tasks.rados.rados.0.smithi163.stdout:844: writing smithi16367185-5 from 716064 to 1355950 tid 1 2023-10-09T09:08:53.198 INFO:tasks.rados.rados.0.smithi163.stdout:844: writing smithi16367185-5 from 1815191 to 2315640 tid 2 2023-10-09T09:08:53.205 INFO:tasks.rados.rados.0.smithi163.stdout:844: writing smithi16367185-5 from 2817011 to 3343936 tid 3 2023-10-09T09:08:53.205 INFO:tasks.rados.rados.0.smithi163.stdout:845: snap_create 2023-10-09T09:08:53.205 INFO:tasks.rados.rados.0.smithi163.stdout:843: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:08:53.206 INFO:tasks.rados.rados.0.smithi163.stdout:843: finishing rollback tid 2 to smithi16367185-4 2023-10-09T09:08:53.206 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 471 (ObjNum 247 snap 73 seq_num 247) dirty exists 2023-10-09T09:08:53.206 INFO:tasks.rados.rados.0.smithi163.stdout:844: finishing write tid 1 to smithi16367185-5 2023-10-09T09:08:53.206 INFO:tasks.rados.rados.0.smithi163.stdout:844: finishing write tid 2 to smithi16367185-5 2023-10-09T09:08:53.209 INFO:tasks.rados.rados.0.smithi163.stdout:844: finishing write tid 3 to smithi16367185-5 2023-10-09T09:08:53.209 INFO:tasks.rados.rados.0.smithi163.stdout:844: finishing write tid 4 to smithi16367185-5 2023-10-09T09:08:53.210 INFO:tasks.rados.rados.0.smithi163.stdout:844: finishing write tid 5 to smithi16367185-5 2023-10-09T09:08:53.210 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 476 (ObjNum 352 snap 106 seq_num 352) dirty exists 2023-10-09T09:08:53.210 INFO:tasks.rados.rados.0.smithi163.stdout:844: left oid 5 (ObjNum 352 snap 106 seq_num 352) 2023-10-09T09:08:53.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:52 smithi163 bash[27500]: cluster 2023-10-09T09:08:51.472303+0000 mgr.y (mgr.15273) 32 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 247 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2023-10-09T09:08:53.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:52 smithi163 bash[27500]: audit 2023-10-09T09:08:51.934409+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:53.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:52 smithi163 bash[27500]: audit 2023-10-09T09:08:51.944408+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:53.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:52 smithi163 bash[27500]: audit 2023-10-09T09:08:51.973025+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:53.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:52 smithi163 bash[27500]: cluster 2023-10-09T09:08:52.180153+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2023-10-09T09:08:53.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:52 smithi163 bash[33912]: cluster 2023-10-09T09:08:51.472303+0000 mgr.y (mgr.15273) 32 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 247 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2023-10-09T09:08:53.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:52 smithi163 bash[33912]: audit 2023-10-09T09:08:51.934409+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:53.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:52 smithi163 bash[33912]: audit 2023-10-09T09:08:51.944408+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:53.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:52 smithi163 bash[33912]: audit 2023-10-09T09:08:51.973025+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:08:53.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:52 smithi163 bash[33912]: cluster 2023-10-09T09:08:52.180153+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2023-10-09T09:08:53.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:53 smithi181 bash[55332]: debug 2023-10-09T09:08:53.152+0000 7f8a203ac200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:08:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:08:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:08:53] "GET /metrics HTTP/1.1" 200 36084 "" "Prometheus/2.33.4" 2023-10-09T09:08:54.195 INFO:tasks.rados.rados.0.smithi163.stdout:843: done (2 left) 2023-10-09T09:08:54.195 INFO:tasks.rados.rados.0.smithi163.stdout:844: done (1 left) 2023-10-09T09:08:54.195 INFO:tasks.rados.rados.0.smithi163.stdout:845: done (0 left) 2023-10-09T09:08:54.195 INFO:tasks.rados.rados.0.smithi163.stdout:846: read oid 12 snap -1 2023-10-09T09:08:54.195 INFO:tasks.rados.rados.0.smithi163.stdout:846: expect (ObjNum 241 snap 73 seq_num 241) 2023-10-09T09:08:54.195 INFO:tasks.rados.rados.0.smithi163.stdout:847: rollback oid 9 current snap is 107 2023-10-09T09:08:54.196 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 75 2023-10-09T09:08:54.196 INFO:tasks.rados.rados.0.smithi163.stdout:848: snap_remove snap 88 2023-10-09T09:08:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:54 smithi163 bash[27500]: cluster 2023-10-09T09:08:53.182637+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2023-10-09T09:08:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:54 smithi163 bash[27500]: audit 2023-10-09T09:08:53.302857+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.163:0/3461791055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:54 smithi163 bash[27500]: cluster 2023-10-09T09:08:53.472926+0000 mgr.y (mgr.15273) 33 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 247 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2023-10-09T09:08:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:54 smithi163 bash[27500]: audit 2023-10-09T09:08:53.597628+0000 mon.a (mon.0) 1179 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:54 smithi163 bash[33912]: cluster 2023-10-09T09:08:53.182637+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2023-10-09T09:08:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:54 smithi163 bash[33912]: audit 2023-10-09T09:08:53.302857+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.163:0/3461791055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:54 smithi163 bash[33912]: cluster 2023-10-09T09:08:53.472926+0000 mgr.y (mgr.15273) 33 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 247 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2023-10-09T09:08:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:54 smithi163 bash[33912]: audit 2023-10-09T09:08:53.597628+0000 mon.a (mon.0) 1179 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:54 smithi181 bash[27712]: cluster 2023-10-09T09:08:53.182637+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2023-10-09T09:08:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:54 smithi181 bash[27712]: audit 2023-10-09T09:08:53.302857+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.163:0/3461791055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:54 smithi181 bash[27712]: cluster 2023-10-09T09:08:53.472926+0000 mgr.y (mgr.15273) 33 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 247 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2023-10-09T09:08:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:54 smithi181 bash[27712]: audit 2023-10-09T09:08:53.597628+0000 mon.a (mon.0) 1179 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:08:54.568 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:54 smithi181 bash[55332]: debug 2023-10-09T09:08:54.088+0000 7f8a203ac200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:08:54.568 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:54 smithi181 bash[55332]: debug 2023-10-09T09:08:54.192+0000 7f8a203ac200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:08:54.568 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:54 smithi181 bash[55332]: debug 2023-10-09T09:08:54.300+0000 7f8a203ac200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:08:54.945 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:54 smithi181 bash[55332]: debug 2023-10-09T09:08:54.568+0000 7f8a203ac200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:08:55.200 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:55 smithi181 bash[27712]: cluster 2023-10-09T09:08:54.194040+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2023-10-09T09:08:55.201 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:54 smithi181 bash[55332]: debug 2023-10-09T09:08:54.940+0000 7f8a203ac200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:08:55.201 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:55 smithi181 bash[55332]: debug 2023-10-09T09:08:55.100+0000 7f8a203ac200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:08:55.202 INFO:tasks.rados.rados.0.smithi163.stdout:847: finishing rollback tid 0 to smithi16367185-9 2023-10-09T09:08:55.202 INFO:tasks.rados.rados.0.smithi163.stdout:847: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:08:55.202 INFO:tasks.rados.rados.0.smithi163.stdout:847: finishing rollback tid 2 to smithi16367185-9 2023-10-09T09:08:55.202 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 241 (ObjNum 220 snap 61 seq_num 220) dirty exists 2023-10-09T09:08:55.202 INFO:tasks.rados.rados.0.smithi163.stdout:846: expect (ObjNum 241 snap 73 seq_num 241) 2023-10-09T09:08:55.309 INFO:tasks.rados.rados.0.smithi163.stdout:846: done (2 left) 2023-10-09T09:08:55.309 INFO:tasks.rados.rados.0.smithi163.stdout:847: done (1 left) 2023-10-09T09:08:55.309 INFO:tasks.rados.rados.0.smithi163.stdout:848: done (0 left) 2023-10-09T09:08:55.309 INFO:tasks.rados.rados.0.smithi163.stdout:849: write oid 40 current snap is 107 2023-10-09T09:08:55.309 INFO:tasks.rados.rados.0.smithi163.stdout:849: seq_num 353 ranges {732871=715135,1873673=644107,3244434=688437} 2023-10-09T09:08:55.318 INFO:tasks.rados.rados.0.smithi163.stdout:849: writing smithi16367185-40 from 732871 to 1448006 tid 1 2023-10-09T09:08:55.324 INFO:tasks.rados.rados.0.smithi163.stdout:849: writing smithi16367185-40 from 1873673 to 2517780 tid 2 2023-10-09T09:08:55.333 INFO:tasks.rados.rados.0.smithi163.stdout:849: writing smithi16367185-40 from 3244434 to 3932871 tid 3 2023-10-09T09:08:55.333 INFO:tasks.rados.rados.0.smithi163.stdout:850: delete oid 23 current snap is 107 2023-10-09T09:08:55.333 INFO:tasks.rados.rados.0.smithi163.stdout:849: finishing write tid 1 to smithi16367185-40 2023-10-09T09:08:55.333 INFO:tasks.rados.rados.0.smithi163.stdout:849: finishing write tid 2 to smithi16367185-40 2023-10-09T09:08:55.335 INFO:tasks.rados.rados.0.smithi163.stdout:850: done (1 left) 2023-10-09T09:08:55.335 INFO:tasks.rados.rados.0.smithi163.stdout:851: read oid 5 snap -1 2023-10-09T09:08:55.335 INFO:tasks.rados.rados.0.smithi163.stdout:851: expect (ObjNum 352 snap 106 seq_num 352) 2023-10-09T09:08:55.336 INFO:tasks.rados.rados.0.smithi163.stdout:852: rollback oid 26 current snap is 107 2023-10-09T09:08:55.336 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 26 to 106 2023-10-09T09:08:55.336 INFO:tasks.rados.rados.0.smithi163.stdout:853: snap_create 2023-10-09T09:08:55.336 INFO:tasks.rados.rados.0.smithi163.stdout:849: finishing write tid 3 to smithi16367185-40 2023-10-09T09:08:55.337 INFO:tasks.rados.rados.0.smithi163.stdout:849: finishing write tid 4 to smithi16367185-40 2023-10-09T09:08:55.337 INFO:tasks.rados.rados.0.smithi163.stdout:849: finishing write tid 5 to smithi16367185-40 2023-10-09T09:08:55.337 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 485 (ObjNum 353 snap 107 seq_num 353) dirty exists 2023-10-09T09:08:55.337 INFO:tasks.rados.rados.0.smithi163.stdout:849: left oid 40 (ObjNum 353 snap 107 seq_num 353) 2023-10-09T09:08:55.339 INFO:tasks.rados.rados.0.smithi163.stdout:852: finishing rollback tid 0 to smithi16367185-26 2023-10-09T09:08:55.339 INFO:tasks.rados.rados.0.smithi163.stdout:852: finishing rollback tid 1 to smithi16367185-26 2023-10-09T09:08:55.340 INFO:tasks.rados.rados.0.smithi163.stdout:852: finishing rollback tid 2 to smithi16367185-26 2023-10-09T09:08:55.340 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 301 (ObjNum 25 snap 0 seq_num 25) dirty exists 2023-10-09T09:08:55.349 INFO:tasks.rados.rados.0.smithi163.stdout:851: expect (ObjNum 352 snap 106 seq_num 352) 2023-10-09T09:08:55.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:55 smithi163 bash[33912]: cluster 2023-10-09T09:08:54.194040+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2023-10-09T09:08:55.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:55 smithi163 bash[27500]: cluster 2023-10-09T09:08:54.194040+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2023-10-09T09:08:55.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:55 smithi181 bash[55332]: debug 2023-10-09T09:08:55.328+0000 7f8a203ac200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:08:56.214 INFO:tasks.rados.rados.0.smithi163.stdout:849: done (3 left) 2023-10-09T09:08:56.214 INFO:tasks.rados.rados.0.smithi163.stdout:851: done (2 left) 2023-10-09T09:08:56.215 INFO:tasks.rados.rados.0.smithi163.stdout:852: done (1 left) 2023-10-09T09:08:56.215 INFO:tasks.rados.rados.0.smithi163.stdout:853: done (0 left) 2023-10-09T09:08:56.215 INFO:tasks.rados.rados.0.smithi163.stdout:854: read oid 25 snap -1 2023-10-09T09:08:56.215 INFO:tasks.rados.rados.0.smithi163.stdout:854: expect deleted 2023-10-09T09:08:56.215 INFO:tasks.rados.rados.0.smithi163.stdout:855: snap_remove snap 98 2023-10-09T09:08:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:56 smithi163 bash[27500]: cluster 2023-10-09T09:08:55.195905+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2023-10-09T09:08:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:56 smithi163 bash[27500]: cluster 2023-10-09T09:08:55.473600+0000 mgr.y (mgr.15273) 34 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:08:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:56 smithi163 bash[27500]: audit 2023-10-09T09:08:55.742523+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.163:0/3642191584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:56 smithi163 bash[33912]: cluster 2023-10-09T09:08:55.195905+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2023-10-09T09:08:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:56 smithi163 bash[33912]: cluster 2023-10-09T09:08:55.473600+0000 mgr.y (mgr.15273) 34 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:08:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:56 smithi163 bash[33912]: audit 2023-10-09T09:08:55.742523+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.163:0/3642191584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:56.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:56 smithi181 bash[27712]: cluster 2023-10-09T09:08:55.195905+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2023-10-09T09:08:56.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:56 smithi181 bash[27712]: cluster 2023-10-09T09:08:55.473600+0000 mgr.y (mgr.15273) 34 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:08:56.483 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:56 smithi181 bash[27712]: audit 2023-10-09T09:08:55.742523+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.163:0/3642191584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:56.483 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:56 smithi181 bash[55332]: debug 2023-10-09T09:08:56.356+0000 7f8a203ac200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:08:56.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:56 smithi181 bash[55332]: debug 2023-10-09T09:08:56.476+0000 7f8a203ac200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:08:56.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:56 smithi181 bash[55332]: debug 2023-10-09T09:08:56.608+0000 7f8a203ac200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:08:57.165 INFO:tasks.rados.rados.0.smithi163.stdout:855: done (1 left) 2023-10-09T09:08:57.165 INFO:tasks.rados.rados.0.smithi163.stdout:856: write oid 12 current snap is 108 2023-10-09T09:08:57.165 INFO:tasks.rados.rados.0.smithi163.stdout:856: seq_num 355 ranges {766485=465633,1990637=647776,3299280=667205} 2023-10-09T09:08:57.172 INFO:tasks.rados.rados.0.smithi163.stdout:856: writing smithi16367185-12 from 766485 to 1232118 tid 1 2023-10-09T09:08:57.181 INFO:tasks.rados.rados.0.smithi163.stdout:856: writing smithi16367185-12 from 1990637 to 2638413 tid 2 2023-10-09T09:08:57.189 INFO:tasks.rados.rados.0.smithi163.stdout:856: writing smithi16367185-12 from 3299280 to 3966485 tid 3 2023-10-09T09:08:57.189 INFO:tasks.rados.rados.0.smithi163.stdout:857: delete oid 11 current snap is 108 2023-10-09T09:08:57.189 INFO:tasks.rados.rados.0.smithi163.stdout:856: finishing write tid 1 to smithi16367185-12 2023-10-09T09:08:57.189 INFO:tasks.rados.rados.0.smithi163.stdout:856: finishing write tid 2 to smithi16367185-12 2023-10-09T09:08:57.193 INFO:tasks.rados.rados.0.smithi163.stdout:856: finishing write tid 3 to smithi16367185-12 2023-10-09T09:08:57.193 INFO:tasks.rados.rados.0.smithi163.stdout:854: done (2 left) 2023-10-09T09:08:57.193 INFO:tasks.rados.rados.0.smithi163.stdout:857: done (1 left) 2023-10-09T09:08:57.193 INFO:tasks.rados.rados.0.smithi163.stdout:858: read oid 26 snap -1 2023-10-09T09:08:57.193 INFO:tasks.rados.rados.0.smithi163.stdout:858: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:08:57.193 INFO:tasks.rados.rados.0.smithi163.stdout:859: read oid 9 snap -1 2023-10-09T09:08:57.193 INFO:tasks.rados.rados.0.smithi163.stdout:859: expect (ObjNum 220 snap 61 seq_num 220) 2023-10-09T09:08:57.193 INFO:tasks.rados.rados.0.smithi163.stdout:860: snap_remove snap 87 2023-10-09T09:08:57.411 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:57 smithi181 bash[27712]: cluster 2023-10-09T09:08:56.213764+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2023-10-09T09:08:57.411 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:57 smithi181 bash[27712]: cluster 2023-10-09T09:08:57.162225+0000 mon.a (mon.0) 1183 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2023-10-09T09:08:57.412 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:57 smithi181 bash[55332]: debug 2023-10-09T09:08:57.140+0000 7f8a203ac200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:08:57.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:57 smithi163 bash[27500]: cluster 2023-10-09T09:08:56.213764+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2023-10-09T09:08:57.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:57 smithi163 bash[27500]: cluster 2023-10-09T09:08:57.162225+0000 mon.a (mon.0) 1183 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2023-10-09T09:08:57.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:57 smithi163 bash[33912]: cluster 2023-10-09T09:08:56.213764+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2023-10-09T09:08:57.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:57 smithi163 bash[33912]: cluster 2023-10-09T09:08:57.162225+0000 mon.a (mon.0) 1183 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2023-10-09T09:08:57.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:57 smithi181 bash[55332]: debug 2023-10-09T09:08:57.408+0000 7f8a203ac200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:08:58.174 INFO:tasks.rados.rados.0.smithi163.stdout:860: done (3 left) 2023-10-09T09:08:58.174 INFO:tasks.rados.rados.0.smithi163.stdout:861: delete oid 40 current snap is 108 2023-10-09T09:08:58.174 INFO:tasks.rados.rados.0.smithi163.stdout:856: finishing write tid 4 to smithi16367185-12 2023-10-09T09:08:58.174 INFO:tasks.rados.rados.0.smithi163.stdout:856: finishing write tid 5 to smithi16367185-12 2023-10-09T09:08:58.174 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 388 (ObjNum 355 snap 108 seq_num 355) dirty exists 2023-10-09T09:08:58.174 INFO:tasks.rados.rados.0.smithi163.stdout:856: left oid 12 (ObjNum 355 snap 108 seq_num 355) 2023-10-09T09:08:58.174 INFO:tasks.rados.rados.0.smithi163.stdout:858: expect (ObjNum 25 snap 0 seq_num 25) 2023-10-09T09:08:58.217 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:57 smithi181 bash[55332]: debug 2023-10-09T09:08:57.960+0000 7f8a203ac200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:08:58.260 INFO:tasks.rados.rados.0.smithi163.stdout:859: expect (ObjNum 220 snap 61 seq_num 220) 2023-10-09T09:08:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:58 smithi163 bash[33912]: cluster 2023-10-09T09:08:57.474319+0000 mgr.y (mgr.15273) 35 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:08:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:58 smithi163 bash[33912]: audit 2023-10-09T09:08:58.116072+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:58 smithi163 bash[33912]: audit 2023-10-09T09:08:58.122564+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:58 smithi163 bash[33912]: cluster 2023-10-09T09:08:58.173812+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2023-10-09T09:08:58.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:58 smithi163 bash[33912]: audit 2023-10-09T09:08:58.196140+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.163:0/1080418942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:58.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:58 smithi163 bash[27500]: cluster 2023-10-09T09:08:57.474319+0000 mgr.y (mgr.15273) 35 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:08:58.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:58 smithi163 bash[27500]: audit 2023-10-09T09:08:58.116072+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:58 smithi163 bash[27500]: audit 2023-10-09T09:08:58.122564+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:58 smithi163 bash[27500]: cluster 2023-10-09T09:08:58.173812+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2023-10-09T09:08:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:58 smithi163 bash[27500]: audit 2023-10-09T09:08:58.196140+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.163:0/1080418942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:58.523 INFO:tasks.rados.rados.0.smithi163.stdout:856: done (3 left) 2023-10-09T09:08:58.523 INFO:tasks.rados.rados.0.smithi163.stdout:858: done (2 left) 2023-10-09T09:08:58.523 INFO:tasks.rados.rados.0.smithi163.stdout:859: done (1 left) 2023-10-09T09:08:58.525 INFO:tasks.rados.rados.0.smithi163.stdout:861: done (0 left) 2023-10-09T09:08:58.525 INFO:tasks.rados.rados.0.smithi163.stdout:862: read oid 11 snap 102 2023-10-09T09:08:58.525 INFO:tasks.rados.rados.0.smithi163.stdout:862: expect (ObjNum 340 snap 102 seq_num 340) 2023-10-09T09:08:58.525 INFO:tasks.rados.rados.0.smithi163.stdout:863: rollback oid 3 current snap is 108 2023-10-09T09:08:58.525 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 63 2023-10-09T09:08:58.525 INFO:tasks.rados.rados.0.smithi163.stdout:864: rollback oid 41 current snap is 108 2023-10-09T09:08:58.525 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 41 to 102 2023-10-09T09:08:58.525 INFO:tasks.rados.rados.0.smithi163.stdout:865: snap_remove snap 75 2023-10-09T09:08:58.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:58 smithi181 bash[27712]: cluster 2023-10-09T09:08:57.474319+0000 mgr.y (mgr.15273) 35 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:08:58.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:58 smithi181 bash[27712]: audit 2023-10-09T09:08:58.116072+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:58.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:58 smithi181 bash[27712]: audit 2023-10-09T09:08:58.122564+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:08:58.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:58 smithi181 bash[27712]: cluster 2023-10-09T09:08:58.173812+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2023-10-09T09:08:58.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:58 smithi181 bash[27712]: audit 2023-10-09T09:08:58.196140+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.163:0/1080418942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:08:58.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:58 smithi181 bash[55332]: debug 2023-10-09T09:08:58.528+0000 7f8a203ac200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:08:58.817 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:08:58 smithi181 bash[54907]: ts=2023-10-09T09:08:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:08:59.228 INFO:tasks.rados.rados.0.smithi163.stdout:865: done (3 left) 2023-10-09T09:08:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:866: snap_remove snap 105 2023-10-09T09:08:59.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:59 smithi181 bash[55332]: debug 2023-10-09T09:08:59.164+0000 7f8a203ac200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:08:59.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:59 smithi181 bash[55332]: debug 2023-10-09T09:08:59.296+0000 7f8a203ac200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:08:59.944 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:59 smithi181 bash[55332]: debug 2023-10-09T09:08:59.676+0000 7f8a203ac200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:08:59.944 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:59 smithi181 bash[55332]: debug 2023-10-09T09:08:59.796+0000 7f8a203ac200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:09:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:59 smithi163 bash[33912]: audit 2023-10-09T09:08:58.970454+0000 mon.a (mon.0) 1187 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:59 smithi163 bash[33912]: audit 2023-10-09T09:08:58.978954+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:08:59 smithi163 bash[33912]: cluster 2023-10-09T09:08:59.228416+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2023-10-09T09:09:00.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:59 smithi163 bash[27500]: audit 2023-10-09T09:08:58.970454+0000 mon.a (mon.0) 1187 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:00.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:59 smithi163 bash[27500]: audit 2023-10-09T09:08:58.978954+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:00.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:08:59 smithi163 bash[27500]: cluster 2023-10-09T09:08:59.228416+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2023-10-09T09:09:00.236 INFO:tasks.rados.rados.0.smithi163.stdout:866: done (3 left) 2023-10-09T09:09:00.236 INFO:tasks.rados.rados.0.smithi163.stdout:867: read oid 22 snap -1 2023-10-09T09:09:00.236 INFO:tasks.rados.rados.0.smithi163.stdout:867: expect deleted 2023-10-09T09:09:00.236 INFO:tasks.rados.rados.0.smithi163.stdout:868: write oid 34 current snap is 108 2023-10-09T09:09:00.236 INFO:tasks.rados.rados.0.smithi163.stdout:863: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:09:00.236 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 0 (ObjNum 698905544 snap 21875 seq_num 1) dirty dne 2023-10-09T09:09:00.236 INFO:tasks.rados.rados.0.smithi163.stdout:864: finishing rollback tid 0 to smithi16367185-41 2023-10-09T09:09:00.236 INFO:tasks.rados.rados.0.smithi163.stdout:864: finishing rollback tid 1 to smithi16367185-41 2023-10-09T09:09:00.236 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 59 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:09:00.236 INFO:tasks.rados.rados.0.smithi163.stdout:862: expect (ObjNum 340 snap 102 seq_num 340) 2023-10-09T09:09:00.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:08:59 smithi181 bash[55332]: debug 2023-10-09T09:08:59.940+0000 7f8a203ac200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:09:00.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:00 smithi181 bash[55332]: debug 2023-10-09T09:09:00.060+0000 7f8a203ac200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:09:00.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:00 smithi181 bash[55332]: debug 2023-10-09T09:09:00.184+0000 7f8a203ac200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:09:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:59 smithi181 bash[27712]: audit 2023-10-09T09:08:58.970454+0000 mon.a (mon.0) 1187 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:59 smithi181 bash[27712]: audit 2023-10-09T09:08:58.978954+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:08:59 smithi181 bash[27712]: cluster 2023-10-09T09:08:59.228416+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2023-10-09T09:09:00.464 INFO:tasks.rados.rados.0.smithi163.stdout:868: seq_num 358 ranges {416906=407733,1366083=511456} 2023-10-09T09:09:00.469 INFO:tasks.rados.rados.0.smithi163.stdout:868: writing smithi16367185-34 from 416906 to 824639 tid 1 2023-10-09T09:09:00.477 INFO:tasks.rados.rados.0.smithi163.stdout:868: writing smithi16367185-34 from 1366083 to 1877539 tid 2 2023-10-09T09:09:00.477 INFO:tasks.rados.rados.0.smithi163.stdout:862: done (4 left) 2023-10-09T09:09:00.477 INFO:tasks.rados.rados.0.smithi163.stdout:863: done (3 left) 2023-10-09T09:09:00.477 INFO:tasks.rados.rados.0.smithi163.stdout:864: done (2 left) 2023-10-09T09:09:00.477 INFO:tasks.rados.rados.0.smithi163.stdout:867: done (1 left) 2023-10-09T09:09:00.477 INFO:tasks.rados.rados.0.smithi163.stdout:869: write oid 16 current snap is 108 2023-10-09T09:09:00.477 INFO:tasks.rados.rados.0.smithi163.stdout:869: seq_num 359 ranges {433713=482982,1424565=609148} 2023-10-09T09:09:00.483 INFO:tasks.rados.rados.0.smithi163.stdout:869: writing smithi16367185-16 from 433713 to 916695 tid 1 2023-10-09T09:09:00.491 INFO:tasks.rados.rados.0.smithi163.stdout:869: writing smithi16367185-16 from 1424565 to 2033713 tid 2 2023-10-09T09:09:00.491 INFO:tasks.rados.rados.0.smithi163.stdout:870: read oid 24 snap 63 2023-10-09T09:09:00.491 INFO:tasks.rados.rados.0.smithi163.stdout:870: expect deleted 2023-10-09T09:09:00.493 INFO:tasks.rados.rados.0.smithi163.stdout:871: write oid 11 current snap is 108 2023-10-09T09:09:00.493 INFO:tasks.rados.rados.0.smithi163.stdout:871: seq_num 360 ranges {450520=558231,1483047=515125} 2023-10-09T09:09:00.498 INFO:tasks.rados.rados.0.smithi163.stdout:871: writing smithi16367185-11 from 450520 to 1008751 tid 1 2023-10-09T09:09:00.505 INFO:tasks.rados.rados.0.smithi163.stdout:871: writing smithi16367185-11 from 1483047 to 1998172 tid 2 2023-10-09T09:09:00.505 INFO:tasks.rados.rados.0.smithi163.stdout:872: read oid 35 snap -1 2023-10-09T09:09:00.505 INFO:tasks.rados.rados.0.smithi163.stdout:872: expect (ObjNum 341 snap 103 seq_num 341) 2023-10-09T09:09:00.505 INFO:tasks.rados.rados.0.smithi163.stdout:873: read oid 12 snap 107 2023-10-09T09:09:00.505 INFO:tasks.rados.rados.0.smithi163.stdout:873: expect (ObjNum 241 snap 73 seq_num 241) 2023-10-09T09:09:00.506 INFO:tasks.rados.rados.0.smithi163.stdout:874: write oid 45 current snap is 108 2023-10-09T09:09:00.506 INFO:tasks.rados.rados.0.smithi163.stdout:868: finishing write tid 1 to smithi16367185-34 2023-10-09T09:09:00.506 INFO:tasks.rados.rados.0.smithi163.stdout:874: seq_num 361 ranges {467327=633480,1825176=242151} 2023-10-09T09:09:00.513 INFO:tasks.rados.rados.0.smithi163.stdout:874: writing smithi16367185-45 from 467327 to 1100807 tid 1 2023-10-09T09:09:00.522 INFO:tasks.rados.rados.0.smithi163.stdout:874: writing smithi16367185-45 from 1825176 to 2067327 tid 2 2023-10-09T09:09:00.522 INFO:tasks.rados.rados.0.smithi163.stdout:875: snap_create 2023-10-09T09:09:00.522 INFO:tasks.rados.rados.0.smithi163.stdout:868: finishing write tid 2 to smithi16367185-34 2023-10-09T09:09:00.522 INFO:tasks.rados.rados.0.smithi163.stdout:868: finishing write tid 3 to smithi16367185-34 2023-10-09T09:09:00.522 INFO:tasks.rados.rados.0.smithi163.stdout:868: finishing write tid 4 to smithi16367185-34 2023-10-09T09:09:00.522 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 345 (ObjNum 358 snap 108 seq_num 358) dirty exists 2023-10-09T09:09:00.522 INFO:tasks.rados.rados.0.smithi163.stdout:868: left oid 34 (ObjNum 358 snap 108 seq_num 358) 2023-10-09T09:09:00.522 INFO:tasks.rados.rados.0.smithi163.stdout:869: finishing write tid 1 to smithi16367185-16 2023-10-09T09:09:00.522 INFO:tasks.rados.rados.0.smithi163.stdout:869: finishing write tid 2 to smithi16367185-16 2023-10-09T09:09:00.522 INFO:tasks.rados.rados.0.smithi163.stdout:869: finishing write tid 3 to smithi16367185-16 2023-10-09T09:09:00.523 INFO:tasks.rados.rados.0.smithi163.stdout:869: finishing write tid 4 to smithi16367185-16 2023-10-09T09:09:00.523 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 497 (ObjNum 359 snap 108 seq_num 359) dirty exists 2023-10-09T09:09:00.523 INFO:tasks.rados.rados.0.smithi163.stdout:869: left oid 16 (ObjNum 359 snap 108 seq_num 359) 2023-10-09T09:09:00.523 INFO:tasks.rados.rados.0.smithi163.stdout:871: finishing write tid 1 to smithi16367185-11 2023-10-09T09:09:00.523 INFO:tasks.rados.rados.0.smithi163.stdout:873: expect (ObjNum 241 snap 73 seq_num 241) 2023-10-09T09:09:00.631 INFO:tasks.rados.rados.0.smithi163.stdout:871: finishing write tid 2 to smithi16367185-11 2023-10-09T09:09:00.632 INFO:tasks.rados.rados.0.smithi163.stdout:871: finishing write tid 3 to smithi16367185-11 2023-10-09T09:09:00.632 INFO:tasks.rados.rados.0.smithi163.stdout:871: finishing write tid 4 to smithi16367185-11 2023-10-09T09:09:00.632 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 250 (ObjNum 360 snap 108 seq_num 360) dirty exists 2023-10-09T09:09:00.632 INFO:tasks.rados.rados.0.smithi163.stdout:871: left oid 11 (ObjNum 360 snap 108 seq_num 360) 2023-10-09T09:09:00.632 INFO:tasks.rados.rados.0.smithi163.stdout:872: expect (ObjNum 341 snap 103 seq_num 341) 2023-10-09T09:09:00.754 INFO:tasks.rados.rados.0.smithi163.stdout:874: finishing write tid 1 to smithi16367185-45 2023-10-09T09:09:00.754 INFO:tasks.rados.rados.0.smithi163.stdout:874: finishing write tid 2 to smithi16367185-45 2023-10-09T09:09:00.754 INFO:tasks.rados.rados.0.smithi163.stdout:874: finishing write tid 3 to smithi16367185-45 2023-10-09T09:09:00.754 INFO:tasks.rados.rados.0.smithi163.stdout:874: finishing write tid 4 to smithi16367185-45 2023-10-09T09:09:00.754 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 765 (ObjNum 361 snap 108 seq_num 361) dirty exists 2023-10-09T09:09:00.755 INFO:tasks.rados.rados.0.smithi163.stdout:874: left oid 45 (ObjNum 361 snap 108 seq_num 361) 2023-10-09T09:09:00.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:00 smithi181 bash[55332]: debug 2023-10-09T09:09:00.392+0000 7f8a203ac200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:09:00.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:00 smithi181 bash[55332]: debug 2023-10-09T09:09:00.532+0000 7f8a203ac200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:09:01.238 INFO:tasks.rados.rados.0.smithi163.stdout:868: done (7 left) 2023-10-09T09:09:01.238 INFO:tasks.rados.rados.0.smithi163.stdout:869: done (6 left) 2023-10-09T09:09:01.238 INFO:tasks.rados.rados.0.smithi163.stdout:870: done (5 left) 2023-10-09T09:09:01.238 INFO:tasks.rados.rados.0.smithi163.stdout:871: done (4 left) 2023-10-09T09:09:01.238 INFO:tasks.rados.rados.0.smithi163.stdout:872: done (3 left) 2023-10-09T09:09:01.239 INFO:tasks.rados.rados.0.smithi163.stdout:873: done (2 left) 2023-10-09T09:09:01.239 INFO:tasks.rados.rados.0.smithi163.stdout:874: done (1 left) 2023-10-09T09:09:01.239 INFO:tasks.rados.rados.0.smithi163.stdout:875: done (0 left) 2023-10-09T09:09:01.239 INFO:tasks.rados.rados.0.smithi163.stdout:876: write oid 45 current snap is 109 2023-10-09T09:09:01.239 INFO:tasks.rados.rados.0.smithi163.stdout:876: seq_num 362 ranges {484134=708729,1883658=200476} 2023-10-09T09:09:01.246 INFO:tasks.rados.rados.0.smithi163.stdout:876: writing smithi16367185-45 from 484134 to 1192863 tid 1 2023-10-09T09:09:01.251 INFO:tasks.rados.rados.0.smithi163.stdout:876: writing smithi16367185-45 from 1883658 to 2084134 tid 2 2023-10-09T09:09:01.251 INFO:tasks.rados.rados.0.smithi163.stdout:877: rollback oid 30 current snap is 109 2023-10-09T09:09:01.251 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 106 2023-10-09T09:09:01.251 INFO:tasks.rados.rados.0.smithi163.stdout:878: read oid 50 snap -1 2023-10-09T09:09:01.251 INFO:tasks.rados.rados.0.smithi163.stdout:878: expect (ObjNum 314 snap 94 seq_num 314) 2023-10-09T09:09:01.252 INFO:tasks.rados.rados.0.smithi163.stdout:879: delete oid 22 current snap is 109 2023-10-09T09:09:01.260 INFO:tasks.rados.rados.0.smithi163.stdout:877: finishing rollback tid 0 to smithi16367185-30 2023-10-09T09:09:01.260 INFO:tasks.rados.rados.0.smithi163.stdout:877: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:09:01.261 INFO:tasks.rados.rados.0.smithi163.stdout:877: finishing rollback tid 2 to smithi16367185-30 2023-10-09T09:09:01.261 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 351 (ObjNum 336 snap 101 seq_num 336) dirty exists 2023-10-09T09:09:01.262 INFO:tasks.rados.rados.0.smithi163.stdout:876: finishing write tid 1 to smithi16367185-45 2023-10-09T09:09:01.262 INFO:tasks.rados.rados.0.smithi163.stdout:876: finishing write tid 2 to smithi16367185-45 2023-10-09T09:09:01.264 INFO:tasks.rados.rados.0.smithi163.stdout:878: expect (ObjNum 314 snap 94 seq_num 314) 2023-10-09T09:09:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:00 smithi181 bash[27712]: cluster 2023-10-09T09:08:59.474969+0000 mgr.y (mgr.15273) 36 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:09:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:00 smithi181 bash[27712]: cluster 2023-10-09T09:09:00.233313+0000 mon.a (mon.0) 1190 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2023-10-09T09:09:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:00 smithi181 bash[27712]: audit 2023-10-09T09:09:00.660095+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.163:0/1661133363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:00 smithi163 bash[27500]: cluster 2023-10-09T09:08:59.474969+0000 mgr.y (mgr.15273) 36 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:09:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:00 smithi163 bash[27500]: cluster 2023-10-09T09:09:00.233313+0000 mon.a (mon.0) 1190 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2023-10-09T09:09:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:00 smithi163 bash[27500]: audit 2023-10-09T09:09:00.660095+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.163:0/1661133363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:00 smithi163 bash[33912]: cluster 2023-10-09T09:08:59.474969+0000 mgr.y (mgr.15273) 36 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:09:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:00 smithi163 bash[33912]: cluster 2023-10-09T09:09:00.233313+0000 mon.a (mon.0) 1190 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2023-10-09T09:09:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:00 smithi163 bash[33912]: audit 2023-10-09T09:09:00.660095+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.163:0/1661133363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:01.617 INFO:tasks.rados.rados.0.smithi163.stdout:876: finishing write tid 3 to smithi16367185-45 2023-10-09T09:09:01.617 INFO:tasks.rados.rados.0.smithi163.stdout:876: finishing write tid 4 to smithi16367185-45 2023-10-09T09:09:01.617 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 769 (ObjNum 362 snap 109 seq_num 362) dirty exists 2023-10-09T09:09:01.617 INFO:tasks.rados.rados.0.smithi163.stdout:876: left oid 45 (ObjNum 362 snap 109 seq_num 362) 2023-10-09T09:09:01.617 INFO:tasks.rados.rados.0.smithi163.stdout:876: done (3 left) 2023-10-09T09:09:01.617 INFO:tasks.rados.rados.0.smithi163.stdout:877: done (2 left) 2023-10-09T09:09:01.617 INFO:tasks.rados.rados.0.smithi163.stdout:878: done (1 left) 2023-10-09T09:09:01.618 INFO:tasks.rados.rados.0.smithi163.stdout:879: done (0 left) 2023-10-09T09:09:01.618 INFO:tasks.rados.rados.0.smithi163.stdout:880: snap_create 2023-10-09T09:09:02.247 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:01 smithi181 bash[55332]: debug 2023-10-09T09:09:01.864+0000 7f8a203ac200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:09:02.247 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:01 smithi181 bash[55332]: [09/Oct/2023:09:09:01] ENGINE Bus STARTING 2023-10-09T09:09:02.247 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[55332]: CherryPy Checker: 2023-10-09T09:09:02.247 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[55332]: The Application mounted at '' has an empty config. 2023-10-09T09:09:02.247 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[55332]: [09/Oct/2023:09:09:02] ENGINE Serving on http://:::9283 2023-10-09T09:09:02.247 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[55332]: [09/Oct/2023:09:09:02] ENGINE Bus STARTED 2023-10-09T09:09:02.255 INFO:tasks.rados.rados.0.smithi163.stdout:880: done (0 left) 2023-10-09T09:09:02.255 INFO:tasks.rados.rados.0.smithi163.stdout:881: snap_create 2023-10-09T09:09:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[27712]: cluster 2023-10-09T09:09:01.237999+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2023-10-09T09:09:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[27712]: cluster 2023-10-09T09:09:01.475701+0000 mgr.y (mgr.15273) 37 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 224 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 935 KiB/s wr, 13 op/s 2023-10-09T09:09:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[27712]: cluster 2023-10-09T09:09:01.892162+0000 mon.a (mon.0) 1192 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:09:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[27712]: cluster 2023-10-09T09:09:01.892370+0000 mon.a (mon.0) 1193 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:09:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[27712]: audit 2023-10-09T09:09:01.895886+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:09:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[27712]: audit 2023-10-09T09:09:01.897076+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:09:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[27712]: audit 2023-10-09T09:09:01.901636+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:09:02.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:02 smithi181 bash[27712]: audit 2023-10-09T09:09:01.903506+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:09:02.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[27500]: cluster 2023-10-09T09:09:01.237999+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2023-10-09T09:09:02.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[27500]: cluster 2023-10-09T09:09:01.475701+0000 mgr.y (mgr.15273) 37 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 224 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 935 KiB/s wr, 13 op/s 2023-10-09T09:09:02.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[27500]: cluster 2023-10-09T09:09:01.892162+0000 mon.a (mon.0) 1192 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:09:02.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[27500]: cluster 2023-10-09T09:09:01.892370+0000 mon.a (mon.0) 1193 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:09:02.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[27500]: audit 2023-10-09T09:09:01.895886+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:09:02.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[27500]: audit 2023-10-09T09:09:01.897076+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:09:02.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[27500]: audit 2023-10-09T09:09:01.901636+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:09:02.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[27500]: audit 2023-10-09T09:09:01.903506+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:09:02.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[33912]: cluster 2023-10-09T09:09:01.237999+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2023-10-09T09:09:02.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[33912]: cluster 2023-10-09T09:09:01.475701+0000 mgr.y (mgr.15273) 37 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 224 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 935 KiB/s wr, 13 op/s 2023-10-09T09:09:02.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[33912]: cluster 2023-10-09T09:09:01.892162+0000 mon.a (mon.0) 1192 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:09:02.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[33912]: cluster 2023-10-09T09:09:01.892370+0000 mon.a (mon.0) 1193 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:09:02.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[33912]: audit 2023-10-09T09:09:01.895886+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:09:02.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[33912]: audit 2023-10-09T09:09:01.897076+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:09:02.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[33912]: audit 2023-10-09T09:09:01.901636+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:09:02.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:02 smithi163 bash[33912]: audit 2023-10-09T09:09:01.903506+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.? 172.21.15.181:0/895519340' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:09:03.251 INFO:tasks.rados.rados.0.smithi163.stdout:881: done (0 left) 2023-10-09T09:09:03.251 INFO:tasks.rados.rados.0.smithi163.stdout:882: snap_create 2023-10-09T09:09:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:03 smithi181 bash[27712]: cluster 2023-10-09T09:09:02.254567+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2023-10-09T09:09:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:03 smithi181 bash[27712]: cluster 2023-10-09T09:09:02.255326+0000 mon.a (mon.0) 1195 : cluster [DBG] mgrmap e30: y(active, since 38s), standbys: x 2023-10-09T09:09:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:03 smithi181 bash[27712]: audit 2023-10-09T09:09:03.107832+0000 mon.a (mon.0) 1196 : audit [DBG] from='client.? 172.21.15.163:0/2941015044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:09:03] "GET /metrics HTTP/1.1" 200 36082 "" "Prometheus/2.33.4" 2023-10-09T09:09:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:03 smithi163 bash[27500]: cluster 2023-10-09T09:09:02.254567+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2023-10-09T09:09:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:03 smithi163 bash[27500]: cluster 2023-10-09T09:09:02.255326+0000 mon.a (mon.0) 1195 : cluster [DBG] mgrmap e30: y(active, since 38s), standbys: x 2023-10-09T09:09:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:03 smithi163 bash[27500]: audit 2023-10-09T09:09:03.107832+0000 mon.a (mon.0) 1196 : audit [DBG] from='client.? 172.21.15.163:0/2941015044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:03 smithi163 bash[33912]: cluster 2023-10-09T09:09:02.254567+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2023-10-09T09:09:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:03 smithi163 bash[33912]: cluster 2023-10-09T09:09:02.255326+0000 mon.a (mon.0) 1195 : cluster [DBG] mgrmap e30: y(active, since 38s), standbys: x 2023-10-09T09:09:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:03 smithi163 bash[33912]: audit 2023-10-09T09:09:03.107832+0000 mon.a (mon.0) 1196 : audit [DBG] from='client.? 172.21.15.163:0/2941015044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:04.252 INFO:tasks.rados.rados.0.smithi163.stdout:882: done (0 left) 2023-10-09T09:09:04.253 INFO:tasks.rados.rados.0.smithi163.stdout:883: delete oid 27 current snap is 112 2023-10-09T09:09:04.258 INFO:tasks.rados.rados.0.smithi163.stdout:883: done (0 left) 2023-10-09T09:09:04.258 INFO:tasks.rados.rados.0.smithi163.stdout:884: read oid 34 snap -1 2023-10-09T09:09:04.258 INFO:tasks.rados.rados.0.smithi163.stdout:884: expect (ObjNum 358 snap 108 seq_num 358) 2023-10-09T09:09:04.258 INFO:tasks.rados.rados.0.smithi163.stdout:885: delete oid 17 current snap is 112 2023-10-09T09:09:04.261 INFO:tasks.rados.rados.0.smithi163.stdout:885: done (1 left) 2023-10-09T09:09:04.261 INFO:tasks.rados.rados.0.smithi163.stdout:886: snap_remove snap 63 2023-10-09T09:09:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:04 smithi181 bash[27712]: cluster 2023-10-09T09:09:03.250888+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2023-10-09T09:09:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:04 smithi181 bash[27712]: cluster 2023-10-09T09:09:03.476428+0000 mgr.y (mgr.15273) 38 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 224 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 935 KiB/s wr, 13 op/s 2023-10-09T09:09:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:04 smithi181 bash[27712]: cluster 2023-10-09T09:09:04.252547+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2023-10-09T09:09:04.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:04 smithi163 bash[33912]: cluster 2023-10-09T09:09:03.250888+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2023-10-09T09:09:04.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:04 smithi163 bash[33912]: cluster 2023-10-09T09:09:03.476428+0000 mgr.y (mgr.15273) 38 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 224 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 935 KiB/s wr, 13 op/s 2023-10-09T09:09:04.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:04 smithi163 bash[33912]: cluster 2023-10-09T09:09:04.252547+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2023-10-09T09:09:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:04 smithi163 bash[27500]: cluster 2023-10-09T09:09:03.250888+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2023-10-09T09:09:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:04 smithi163 bash[27500]: cluster 2023-10-09T09:09:03.476428+0000 mgr.y (mgr.15273) 38 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 224 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 935 KiB/s wr, 13 op/s 2023-10-09T09:09:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:04 smithi163 bash[27500]: cluster 2023-10-09T09:09:04.252547+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2023-10-09T09:09:05.270 INFO:tasks.rados.rados.0.smithi163.stdout:886: done (1 left) 2023-10-09T09:09:05.270 INFO:tasks.rados.rados.0.smithi163.stdout:887: read oid 47 snap 108 2023-10-09T09:09:05.271 INFO:tasks.rados.rados.0.smithi163.stdout:887: expect deleted 2023-10-09T09:09:05.271 INFO:tasks.rados.rados.0.smithi163.stdout:888: snap_create 2023-10-09T09:09:05.271 INFO:tasks.rados.rados.0.smithi163.stdout:884: expect (ObjNum 358 snap 108 seq_num 358) 2023-10-09T09:09:06.278 INFO:tasks.rados.rados.0.smithi163.stdout:884: done (2 left) 2023-10-09T09:09:06.278 INFO:tasks.rados.rados.0.smithi163.stdout:887: done (1 left) 2023-10-09T09:09:06.278 INFO:tasks.rados.rados.0.smithi163.stdout:888: done (0 left) 2023-10-09T09:09:06.278 INFO:tasks.rados.rados.0.smithi163.stdout:889: read oid 5 snap -1 2023-10-09T09:09:06.278 INFO:tasks.rados.rados.0.smithi163.stdout:889: expect (ObjNum 352 snap 106 seq_num 352) 2023-10-09T09:09:06.278 INFO:tasks.rados.rados.0.smithi163.stdout:890: read oid 36 snap -1 2023-10-09T09:09:06.278 INFO:tasks.rados.rados.0.smithi163.stdout:890: expect (ObjNum 349 snap 105 seq_num 349) 2023-10-09T09:09:06.278 INFO:tasks.rados.rados.0.smithi163.stdout:891: snap_remove snap 111 2023-10-09T09:09:06.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:06 smithi181 bash[27712]: cluster 2023-10-09T09:09:05.268050+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2023-10-09T09:09:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:06 smithi181 bash[27712]: cluster 2023-10-09T09:09:05.477078+0000 mgr.y (mgr.15273) 39 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:09:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:06 smithi181 bash[27712]: audit 2023-10-09T09:09:05.579096+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.163:0/1846118058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:06 smithi163 bash[33912]: cluster 2023-10-09T09:09:05.268050+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2023-10-09T09:09:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:06 smithi163 bash[33912]: cluster 2023-10-09T09:09:05.477078+0000 mgr.y (mgr.15273) 39 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:09:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:06 smithi163 bash[33912]: audit 2023-10-09T09:09:05.579096+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.163:0/1846118058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:06 smithi163 bash[27500]: cluster 2023-10-09T09:09:05.268050+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2023-10-09T09:09:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:06 smithi163 bash[27500]: cluster 2023-10-09T09:09:05.477078+0000 mgr.y (mgr.15273) 39 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:09:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:06 smithi163 bash[27500]: audit 2023-10-09T09:09:05.579096+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.163:0/1846118058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:07.168 INFO:tasks.rados.rados.0.smithi163.stdout:891: done (2 left) 2023-10-09T09:09:07.168 INFO:tasks.rados.rados.0.smithi163.stdout:892: snap_create 2023-10-09T09:09:07.168 INFO:tasks.rados.rados.0.smithi163.stdout:889: expect (ObjNum 352 snap 106 seq_num 352) 2023-10-09T09:09:07.518 INFO:tasks.rados.rados.0.smithi163.stdout:890: expect (ObjNum 349 snap 105 seq_num 349) 2023-10-09T09:09:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:07 smithi181 bash[27712]: cluster 2023-10-09T09:09:06.277552+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2023-10-09T09:09:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:07 smithi181 bash[27712]: cluster 2023-10-09T09:09:07.167803+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2023-10-09T09:09:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:07 smithi163 bash[27500]: cluster 2023-10-09T09:09:06.277552+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2023-10-09T09:09:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:07 smithi163 bash[27500]: cluster 2023-10-09T09:09:07.167803+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2023-10-09T09:09:07.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:07 smithi163 bash[33912]: cluster 2023-10-09T09:09:06.277552+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2023-10-09T09:09:07.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:07 smithi163 bash[33912]: cluster 2023-10-09T09:09:07.167803+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2023-10-09T09:09:08.179 INFO:tasks.rados.rados.0.smithi163.stdout:889: done (2 left) 2023-10-09T09:09:08.179 INFO:tasks.rados.rados.0.smithi163.stdout:890: done (1 left) 2023-10-09T09:09:08.179 INFO:tasks.rados.rados.0.smithi163.stdout:892: done (0 left) 2023-10-09T09:09:08.179 INFO:tasks.rados.rados.0.smithi163.stdout:893: write oid 35 current snap is 114 2023-10-09T09:09:08.179 INFO:tasks.rados.rados.0.smithi163.stdout:893: seq_num 366 ranges {551362=726078,1717586=409779} 2023-10-09T09:09:08.189 INFO:tasks.rados.rados.0.smithi163.stdout:893: writing smithi16367185-35 from 551362 to 1277440 tid 1 2023-10-09T09:09:08.195 INFO:tasks.rados.rados.0.smithi163.stdout:893: writing smithi16367185-35 from 1717586 to 2127365 tid 2 2023-10-09T09:09:08.195 INFO:tasks.rados.rados.0.smithi163.stdout:894: read oid 45 snap -1 2023-10-09T09:09:08.195 INFO:tasks.rados.rados.0.smithi163.stdout:894: expect (ObjNum 362 snap 109 seq_num 362) 2023-10-09T09:09:08.196 INFO:tasks.rados.rados.0.smithi163.stdout:895: snap_remove snap 104 2023-10-09T09:09:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:08 smithi181 bash[27712]: cluster 2023-10-09T09:09:07.477714+0000 mgr.y (mgr.15273) 40 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:09:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:08 smithi181 bash[27712]: audit 2023-10-09T09:09:08.017375+0000 mon.a (mon.0) 1202 : audit [DBG] from='client.? 172.21.15.163:0/2759728276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:08 smithi181 bash[27712]: cluster 2023-10-09T09:09:08.178372+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2023-10-09T09:09:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:08 smithi181 bash[27712]: audit 2023-10-09T09:09:08.227049+0000 mon.a (mon.0) 1204 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:08 smithi181 bash[27712]: audit 2023-10-09T09:09:08.233895+0000 mon.a (mon.0) 1205 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:08 smithi181 bash[27712]: audit 2023-10-09T09:09:08.235310+0000 mon.a (mon.0) 1206 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:08 smithi181 bash[27712]: audit 2023-10-09T09:09:08.236516+0000 mon.a (mon.0) 1207 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:09:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:08 smithi181 bash[27712]: audit 2023-10-09T09:09:08.243308+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:08.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[27500]: cluster 2023-10-09T09:09:07.477714+0000 mgr.y (mgr.15273) 40 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:09:08.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[27500]: audit 2023-10-09T09:09:08.017375+0000 mon.a (mon.0) 1202 : audit [DBG] from='client.? 172.21.15.163:0/2759728276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:08.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[27500]: cluster 2023-10-09T09:09:08.178372+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2023-10-09T09:09:08.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[27500]: audit 2023-10-09T09:09:08.227049+0000 mon.a (mon.0) 1204 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:08.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[27500]: audit 2023-10-09T09:09:08.233895+0000 mon.a (mon.0) 1205 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:08.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[27500]: audit 2023-10-09T09:09:08.235310+0000 mon.a (mon.0) 1206 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:08.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[27500]: audit 2023-10-09T09:09:08.236516+0000 mon.a (mon.0) 1207 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:09:08.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[27500]: audit 2023-10-09T09:09:08.243308+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:08.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[33912]: cluster 2023-10-09T09:09:07.477714+0000 mgr.y (mgr.15273) 40 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:09:08.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[33912]: audit 2023-10-09T09:09:08.017375+0000 mon.a (mon.0) 1202 : audit [DBG] from='client.? 172.21.15.163:0/2759728276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:08.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[33912]: cluster 2023-10-09T09:09:08.178372+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2023-10-09T09:09:08.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[33912]: audit 2023-10-09T09:09:08.227049+0000 mon.a (mon.0) 1204 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:08.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[33912]: audit 2023-10-09T09:09:08.233895+0000 mon.a (mon.0) 1205 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:08.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[33912]: audit 2023-10-09T09:09:08.235310+0000 mon.a (mon.0) 1206 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:08.650 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[33912]: audit 2023-10-09T09:09:08.236516+0000 mon.a (mon.0) 1207 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:09:08.650 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:08 smithi163 bash[33912]: audit 2023-10-09T09:09:08.243308+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:09.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:09:08 smithi181 bash[54907]: ts=2023-10-09T09:09:08.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:09:09.183 INFO:tasks.rados.rados.0.smithi163.stdout:895: done (2 left) 2023-10-09T09:09:09.183 INFO:tasks.rados.rados.0.smithi163.stdout:896: write oid 18 current snap is 114 2023-10-09T09:09:09.184 INFO:tasks.rados.rados.0.smithi163.stdout:896: seq_num 367 ranges {568169=401327,1376068=669790} 2023-10-09T09:09:09.188 INFO:tasks.rados.rados.0.smithi163.stdout:896: writing smithi16367185-18 from 568169 to 969496 tid 1 2023-10-09T09:09:09.195 INFO:tasks.rados.rados.0.smithi163.stdout:896: writing smithi16367185-18 from 1376068 to 2045858 tid 2 2023-10-09T09:09:09.195 INFO:tasks.rados.rados.0.smithi163.stdout:897: snap_remove snap 109 2023-10-09T09:09:09.563 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: audit 2023-10-09T09:09:08.327654+0000 mon.a (mon.0) 1209 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:09.563 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: audit 2023-10-09T09:09:08.330825+0000 mon.a (mon.0) 1210 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:09:09.563 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: cephadm 2023-10-09T09:09:08.331692+0000 mgr.y (mgr.15273) 41 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T09:09:09.563 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: audit 2023-10-09T09:09:08.338651+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:09.564 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: audit 2023-10-09T09:09:08.340070+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T09:09:09.564 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: audit 2023-10-09T09:09:08.344898+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T09:09:09.564 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: audit 2023-10-09T09:09:08.346364+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T09:09:09.564 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: audit 2023-10-09T09:09:08.350363+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T09:09:09.564 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: audit 2023-10-09T09:09:08.352579+0000 mon.a (mon.0) 1216 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:09:09.564 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: audit 2023-10-09T09:09:08.353689+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T09:09:09.564 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: cephadm 2023-10-09T09:09:08.354461+0000 mgr.y (mgr.15273) 42 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T09:09:09.564 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: audit 2023-10-09T09:09:08.597739+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:09.564 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[33912]: cluster 2023-10-09T09:09:09.181581+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2023-10-09T09:09:09.564 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: audit 2023-10-09T09:09:08.327654+0000 mon.a (mon.0) 1209 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: audit 2023-10-09T09:09:08.330825+0000 mon.a (mon.0) 1210 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: cephadm 2023-10-09T09:09:08.331692+0000 mgr.y (mgr.15273) 41 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: audit 2023-10-09T09:09:08.338651+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: audit 2023-10-09T09:09:08.340070+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: audit 2023-10-09T09:09:08.344898+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: audit 2023-10-09T09:09:08.346364+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: audit 2023-10-09T09:09:08.350363+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: audit 2023-10-09T09:09:08.352579+0000 mon.a (mon.0) 1216 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: audit 2023-10-09T09:09:08.353689+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: cephadm 2023-10-09T09:09:08.354461+0000 mgr.y (mgr.15273) 42 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T09:09:09.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: audit 2023-10-09T09:09:08.597739+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:09.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:09 smithi163 bash[27500]: cluster 2023-10-09T09:09:09.181581+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2023-10-09T09:09:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: audit 2023-10-09T09:09:08.327654+0000 mon.a (mon.0) 1209 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: audit 2023-10-09T09:09:08.330825+0000 mon.a (mon.0) 1210 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:09:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: cephadm 2023-10-09T09:09:08.331692+0000 mgr.y (mgr.15273) 41 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T09:09:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: audit 2023-10-09T09:09:08.338651+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: audit 2023-10-09T09:09:08.340070+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T09:09:09.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: audit 2023-10-09T09:09:08.344898+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T09:09:09.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: audit 2023-10-09T09:09:08.346364+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T09:09:09.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: audit 2023-10-09T09:09:08.350363+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T09:09:09.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: audit 2023-10-09T09:09:08.352579+0000 mon.a (mon.0) 1216 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:09:09.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: audit 2023-10-09T09:09:08.353689+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T09:09:09.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: cephadm 2023-10-09T09:09:08.354461+0000 mgr.y (mgr.15273) 42 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T09:09:09.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: audit 2023-10-09T09:09:08.597739+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:09.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:09 smithi181 bash[27712]: cluster 2023-10-09T09:09:09.181581+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2023-10-09T09:09:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:897: done (3 left) 2023-10-09T09:09:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:898: rollback oid 15 current snap is 114 2023-10-09T09:09:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 15 to 103 2023-10-09T09:09:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:893: finishing write tid 1 to smithi16367185-35 2023-10-09T09:09:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:893: finishing write tid 2 to smithi16367185-35 2023-10-09T09:09:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:893: finishing write tid 3 to smithi16367185-35 2023-10-09T09:09:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:893: finishing write tid 4 to smithi16367185-35 2023-10-09T09:09:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 258 (ObjNum 366 snap 114 seq_num 366) dirty exists 2023-10-09T09:09:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:893: left oid 35 (ObjNum 366 snap 114 seq_num 366) 2023-10-09T09:09:10.182 INFO:tasks.rados.rados.0.smithi163.stdout:894: expect (ObjNum 362 snap 109 seq_num 362) 2023-10-09T09:09:10.362 INFO:tasks.rados.rados.0.smithi163.stdout:896: finishing write tid 1 to smithi16367185-18 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:896: finishing write tid 2 to smithi16367185-18 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:896: finishing write tid 3 to smithi16367185-18 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:896: finishing write tid 4 to smithi16367185-18 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 363 (ObjNum 367 snap 114 seq_num 367) dirty exists 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:896: left oid 18 (ObjNum 367 snap 114 seq_num 367) 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:898: finishing rollback tid 0 to smithi16367185-15 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:898: finishing rollback tid 1 to smithi16367185-15 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:898: finishing rollback tid 2 to smithi16367185-15 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 516 (ObjNum 303 snap 91 seq_num 303) dirty exists 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:893: done (3 left) 2023-10-09T09:09:10.363 INFO:tasks.rados.rados.0.smithi163.stdout:894: done (2 left) 2023-10-09T09:09:10.364 INFO:tasks.rados.rados.0.smithi163.stdout:896: done (1 left) 2023-10-09T09:09:10.364 INFO:tasks.rados.rados.0.smithi163.stdout:898: done (0 left) 2023-10-09T09:09:10.364 INFO:tasks.rados.rados.0.smithi163.stdout:899: read oid 12 snap -1 2023-10-09T09:09:10.364 INFO:tasks.rados.rados.0.smithi163.stdout:899: expect (ObjNum 355 snap 108 seq_num 355) 2023-10-09T09:09:10.364 INFO:tasks.rados.rados.0.smithi163.stdout:900: snap_create 2023-10-09T09:09:10.372 INFO:tasks.rados.rados.0.smithi163.stdout:899: expect (ObjNum 355 snap 108 seq_num 355) 2023-10-09T09:09:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:10 smithi181 bash[27712]: cluster 2023-10-09T09:09:09.478178+0000 mgr.y (mgr.15273) 43 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:09:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:10 smithi181 bash[27712]: cluster 2023-10-09T09:09:10.180413+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2023-10-09T09:09:10.672 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:10 smithi163 bash[33912]: cluster 2023-10-09T09:09:09.478178+0000 mgr.y (mgr.15273) 43 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:09:10.672 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:10 smithi163 bash[33912]: cluster 2023-10-09T09:09:10.180413+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2023-10-09T09:09:10.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:10 smithi163 bash[27500]: cluster 2023-10-09T09:09:09.478178+0000 mgr.y (mgr.15273) 43 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:09:10.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:10 smithi163 bash[27500]: cluster 2023-10-09T09:09:10.180413+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2023-10-09T09:09:11.301 INFO:tasks.rados.rados.0.smithi163.stdout:899: done (1 left) 2023-10-09T09:09:11.302 INFO:tasks.rados.rados.0.smithi163.stdout:900: done (0 left) 2023-10-09T09:09:11.302 INFO:tasks.rados.rados.0.smithi163.stdout:901: write oid 12 current snap is 115 2023-10-09T09:09:11.302 INFO:tasks.rados.rados.0.smithi163.stdout:901: seq_num 368 ranges {584976=476576,1834550=350426} 2023-10-09T09:09:11.307 INFO:tasks.rados.rados.0.smithi163.stdout:901: writing smithi16367185-12 from 584976 to 1061552 tid 1 2023-10-09T09:09:11.314 INFO:tasks.rados.rados.0.smithi163.stdout:901: writing smithi16367185-12 from 1834550 to 2184976 tid 2 2023-10-09T09:09:11.314 INFO:tasks.rados.rados.0.smithi163.stdout:902: delete oid 36 current snap is 115 2023-10-09T09:09:11.314 INFO:tasks.rados.rados.0.smithi163.stdout:901: finishing write tid 1 to smithi16367185-12 2023-10-09T09:09:11.318 INFO:tasks.rados.rados.0.smithi163.stdout:902: done (1 left) 2023-10-09T09:09:11.318 INFO:tasks.rados.rados.0.smithi163.stdout:903: read oid 15 snap -1 2023-10-09T09:09:11.318 INFO:tasks.rados.rados.0.smithi163.stdout:903: expect (ObjNum 303 snap 91 seq_num 303) 2023-10-09T09:09:11.318 INFO:tasks.rados.rados.0.smithi163.stdout:901: finishing write tid 2 to smithi16367185-12 2023-10-09T09:09:11.319 INFO:tasks.rados.rados.0.smithi163.stdout:904: delete oid 17 current snap is 115 2023-10-09T09:09:11.319 INFO:tasks.rados.rados.0.smithi163.stdout:901: finishing write tid 3 to smithi16367185-12 2023-10-09T09:09:11.319 INFO:tasks.rados.rados.0.smithi163.stdout:901: finishing write tid 4 to smithi16367185-12 2023-10-09T09:09:11.319 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 410 (ObjNum 368 snap 115 seq_num 368) dirty exists 2023-10-09T09:09:11.319 INFO:tasks.rados.rados.0.smithi163.stdout:901: left oid 12 (ObjNum 368 snap 115 seq_num 368) 2023-10-09T09:09:11.333 INFO:tasks.rados.rados.0.smithi163.stdout:903: expect (ObjNum 303 snap 91 seq_num 303) 2023-10-09T09:09:11.522 INFO:tasks.rados.rados.0.smithi163.stdout:901: done (2 left) 2023-10-09T09:09:11.522 INFO:tasks.rados.rados.0.smithi163.stdout:903: done (1 left) 2023-10-09T09:09:11.522 INFO:tasks.rados.rados.0.smithi163.stdout:904: done (0 left) 2023-10-09T09:09:11.522 INFO:tasks.rados.rados.0.smithi163.stdout:905: snap_create 2023-10-09T09:09:11.711 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[27500]: cephadm 2023-10-09T09:09:10.374175+0000 mgr.y (mgr.15273) 44 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T09:09:11.712 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[27500]: audit 2023-10-09T09:09:10.379756+0000 mon.a (mon.0) 1221 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:11.712 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[27500]: audit 2023-10-09T09:09:10.381165+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:09:11.712 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[27500]: audit 2023-10-09T09:09:10.382064+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:09:11.712 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[27500]: audit 2023-10-09T09:09:10.383051+0000 mon.a (mon.0) 1224 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:11.712 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[27500]: cephadm 2023-10-09T09:09:10.384145+0000 mgr.y (mgr.15273) 45 : cephadm [INF] Deploying daemon mon.a on smithi163 2023-10-09T09:09:11.712 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[27500]: audit 2023-10-09T09:09:10.473468+0000 mon.a (mon.0) 1225 : audit [DBG] from='client.? 172.21.15.163:0/2483253341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:11.712 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[27500]: cluster 2023-10-09T09:09:11.301537+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2023-10-09T09:09:11.712 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[33912]: cephadm 2023-10-09T09:09:10.374175+0000 mgr.y (mgr.15273) 44 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T09:09:11.712 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[33912]: audit 2023-10-09T09:09:10.379756+0000 mon.a (mon.0) 1221 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:11.713 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[33912]: audit 2023-10-09T09:09:10.381165+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:09:11.713 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[33912]: audit 2023-10-09T09:09:10.382064+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:09:11.713 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[33912]: audit 2023-10-09T09:09:10.383051+0000 mon.a (mon.0) 1224 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:11.713 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[33912]: cephadm 2023-10-09T09:09:10.384145+0000 mgr.y (mgr.15273) 45 : cephadm [INF] Deploying daemon mon.a on smithi163 2023-10-09T09:09:11.713 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[33912]: audit 2023-10-09T09:09:10.473468+0000 mon.a (mon.0) 1225 : audit [DBG] from='client.? 172.21.15.163:0/2483253341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:11.713 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:11 smithi163 bash[33912]: cluster 2023-10-09T09:09:11.301537+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2023-10-09T09:09:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:11 smithi181 bash[27712]: cephadm 2023-10-09T09:09:10.374175+0000 mgr.y (mgr.15273) 44 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T09:09:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:11 smithi181 bash[27712]: audit 2023-10-09T09:09:10.379756+0000 mon.a (mon.0) 1221 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' 2023-10-09T09:09:11.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:11 smithi181 bash[27712]: audit 2023-10-09T09:09:10.381165+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:09:11.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:11 smithi181 bash[27712]: audit 2023-10-09T09:09:10.382064+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:09:11.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:11 smithi181 bash[27712]: audit 2023-10-09T09:09:10.383051+0000 mon.a (mon.0) 1224 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:11.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:11 smithi181 bash[27712]: cephadm 2023-10-09T09:09:10.384145+0000 mgr.y (mgr.15273) 45 : cephadm [INF] Deploying daemon mon.a on smithi163 2023-10-09T09:09:11.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:11 smithi181 bash[27712]: audit 2023-10-09T09:09:10.473468+0000 mon.a (mon.0) 1225 : audit [DBG] from='client.? 172.21.15.163:0/2483253341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:11.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:11 smithi181 bash[27712]: cluster 2023-10-09T09:09:11.301537+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2023-10-09T09:09:12.394 INFO:tasks.rados.rados.0.smithi163.stdout:905: done (0 left) 2023-10-09T09:09:12.394 INFO:tasks.rados.rados.0.smithi163.stdout:906: delete oid 23 current snap is 116 2023-10-09T09:09:12.399 INFO:tasks.rados.rados.0.smithi163.stdout:906: done (0 left) 2023-10-09T09:09:12.400 INFO:tasks.rados.rados.0.smithi163.stdout:907: delete oid 30 current snap is 116 2023-10-09T09:09:12.404 INFO:tasks.rados.rados.0.smithi163.stdout:907: done (0 left) 2023-10-09T09:09:12.404 INFO:tasks.rados.rados.0.smithi163.stdout:908: write oid 39 current snap is 116 2023-10-09T09:09:12.404 INFO:tasks.rados.rados.0.smithi163.stdout:908: seq_num 373 ranges {669011=569174,2010607=258404} 2023-10-09T09:09:12.412 INFO:tasks.rados.rados.0.smithi163.stdout:908: writing smithi16367185-39 from 669011 to 1238185 tid 1 2023-10-09T09:09:12.421 INFO:tasks.rados.rados.0.smithi163.stdout:908: writing smithi16367185-39 from 2010607 to 2269011 tid 2 2023-10-09T09:09:12.421 INFO:tasks.rados.rados.0.smithi163.stdout:909: snap_create 2023-10-09T09:09:12.421 INFO:tasks.rados.rados.0.smithi163.stdout:908: finishing write tid 1 to smithi16367185-39 2023-10-09T09:09:12.423 INFO:tasks.rados.rados.0.smithi163.stdout:908: finishing write tid 2 to smithi16367185-39 2023-10-09T09:09:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:908: finishing write tid 3 to smithi16367185-39 2023-10-09T09:09:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:908: finishing write tid 4 to smithi16367185-39 2023-10-09T09:09:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 358 (ObjNum 373 snap 116 seq_num 373) dirty exists 2023-10-09T09:09:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:908: left oid 39 (ObjNum 373 snap 116 seq_num 373) 2023-10-09T09:09:12.717 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:12 smithi163 bash[33912]: cluster 2023-10-09T09:09:11.478580+0000 mgr.y (mgr.15273) 46 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 146 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 539 KiB/s wr, 8 op/s 2023-10-09T09:09:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:12 smithi163 bash[27500]: cluster 2023-10-09T09:09:11.478580+0000 mgr.y (mgr.15273) 46 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 146 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 539 KiB/s wr, 8 op/s 2023-10-09T09:09:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:12 smithi181 bash[27712]: cluster 2023-10-09T09:09:11.478580+0000 mgr.y (mgr.15273) 46 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 146 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 539 KiB/s wr, 8 op/s 2023-10-09T09:09:13.012 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:12 smithi163 bash[71599]: [09/Oct/2023:09:09:12] ENGINE Bus STOPPING 2023-10-09T09:09:13.012 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:12 smithi163 bash[71599]: [09/Oct/2023:09:09:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:09:13.012 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:12 smithi163 bash[71599]: [09/Oct/2023:09:09:12] ENGINE Bus STOPPED 2023-10-09T09:09:13.012 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:12 smithi163 bash[71599]: [09/Oct/2023:09:09:12] ENGINE Bus STARTING 2023-10-09T09:09:13.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:12 smithi163 systemd[1]: Stopping Ceph mon.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:09:13.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:12 smithi163 bash[27500]: debug 2023-10-09T09:09:12.895+0000 7f8cc7286700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T09:09:13.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:12 smithi163 bash[27500]: debug 2023-10-09T09:09:12.895+0000 7f8cc7286700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T09:09:13.362 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:13 smithi163 bash[71599]: [09/Oct/2023:09:09:13] ENGINE Serving on http://:::9283 2023-10-09T09:09:13.362 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:13 smithi163 bash[71599]: [09/Oct/2023:09:09:13] ENGINE Bus STARTED 2023-10-09T09:09:13.467 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:09:13] "GET /metrics HTTP/1.1" 200 36083 "" "Prometheus/2.33.4" 2023-10-09T09:09:13.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:13 smithi163 bash[75292]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T09:09:14.024 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:13 smithi163 bash[75369]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T09:09:14.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:13 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T09:09:14.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:13 smithi163 systemd[1]: Stopped Ceph mon.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:09:14.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:13 smithi163 systemd[1]: Started Ceph mon.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:09:15.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.883+0000 7f2301a04c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T09:09:15.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.883+0000 7f2301a04c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T09:09:15.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.883+0000 7f2301a04c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T09:09:15.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 0 load: jerasure load: lrc 2023-10-09T09:09:15.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T09:09:15.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Git sha 0 2023-10-09T09:09:15.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T09:09:15.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: DB SUMMARY 2023-10-09T09:09:15.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: DB Session ID: NVQUWTJGJHQQ1CUE35DZ 2023-10-09T09:09:15.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T09:09:15.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T09:09:15.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: MANIFEST file: MANIFEST-000008 size: 1156 Bytes 2023-10-09T09:09:15.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000026.sst 2023-10-09T09:09:15.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000024.log size: 1984337 ; 2023-10-09T09:09:15.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T09:09:15.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T09:09:15.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T09:09:15.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T09:09:15.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T09:09:15.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T09:09:15.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.env: 0x55df3bf98720 2023-10-09T09:09:15.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T09:09:15.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.info_log: 0x55df3e749480 2023-10-09T09:09:15.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T09:09:15.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T09:09:15.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T09:09:15.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T09:09:15.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T09:09:15.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T09:09:15.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T09:09:15.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T09:09:15.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T09:09:15.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T09:09:15.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T09:09:15.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T09:09:15.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T09:09:15.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T09:09:15.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.db_log_dir: 2023-10-09T09:09:15.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.wal_dir: 2023-10-09T09:09:15.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T09:09:15.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T09:09:15.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T09:09:15.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T09:09:15.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T09:09:15.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T09:09:15.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T09:09:15.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T09:09:15.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.write_buffer_manager: 0x55df3d9d6c80 2023-10-09T09:09:15.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T09:09:15.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T09:09:15.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T09:09:15.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T09:09:15.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T09:09:15.226 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T09:09:15.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T09:09:15.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T09:09:15.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T09:09:15.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T09:09:15.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T09:09:15.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T09:09:15.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T09:09:15.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.row_cache: None 2023-10-09T09:09:15.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.wal_filter: None 2023-10-09T09:09:15.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T09:09:15.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T09:09:15.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T09:09:15.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T09:09:15.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T09:09:15.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T09:09:15.230 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T09:09:15.230 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T09:09:15.230 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T09:09:15.230 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T09:09:15.230 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T09:09:15.230 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T09:09:15.231 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T09:09:15.231 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T09:09:15.231 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T09:09:15.231 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T09:09:15.231 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T09:09:15.231 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T09:09:15.231 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T09:09:15.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T09:09:15.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T09:09:15.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T09:09:15.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T09:09:15.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T09:09:15.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T09:09:15.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T09:09:15.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T09:09:15.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T09:09:15.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T09:09:15.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T09:09:15.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T09:09:15.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T09:09:15.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T09:09:15.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T09:09:15.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Compression algorithms supported: 2023-10-09T09:09:15.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T09:09:15.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: kZSTD supported: 0 2023-10-09T09:09:15.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T09:09:15.235 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T09:09:15.235 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T09:09:15.235 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T09:09:15.235 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T09:09:15.235 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T09:09:15.235 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T09:09:15.235 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T09:09:15.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-10-09T09:09:15.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T09:09:15.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T09:09:15.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.merge_operator: 2023-10-09T09:09:15.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T09:09:15.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T09:09:15.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T09:09:15.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T09:09:15.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T09:09:15.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55df3e749580) 2023-10-09T09:09:15.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: cache_index_and_filter_blocks: 1 2023-10-09T09:09:15.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T09:09:15.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T09:09:15.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: pin_top_level_index_and_filter: 1 2023-10-09T09:09:15.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: index_type: 0 2023-10-09T09:09:15.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: data_block_index_type: 0 2023-10-09T09:09:15.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: index_shortening: 1 2023-10-09T09:09:15.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T09:09:15.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: checksum: 4 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: no_block_cache: 0 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: block_cache: 0x55df3da591f0 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: block_cache_name: BinnedLRUCache 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: block_cache_options: 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: capacity : 536870912 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: num_shard_bits : 4 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: strict_capacity_limit : 0 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: high_pri_pool_ratio: 0.000 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: block_cache_compressed: (nil) 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: persistent_cache: (nil) 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: block_size: 4096 2023-10-09T09:09:15.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: block_size_deviation: 10 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: block_restart_interval: 16 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: index_block_restart_interval: 1 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: metadata_block_size: 4096 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: partition_filters: 0 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: use_delta_encoding: 1 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: filter_policy: bloomfilter 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: whole_key_filtering: 1 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: verify_compression: 0 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: read_amp_bytes_per_bit: 0 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: format_version: 5 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: enable_index_compression: 1 2023-10-09T09:09:15.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: block_align: 0 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: max_auto_readahead_size: 262144 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: prepopulate_block_cache: 0 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: initial_auto_readahead_size: 8192 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: num_file_reads_for_auto_readahead: 2 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.num_levels: 7 2023-10-09T09:09:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:09:15.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T09:09:15.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T09:09:15.244 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T09:09:15.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T09:09:15.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T09:09:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T09:09:15.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T09:09:15.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T09:09:15.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T09:09:15.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T09:09:15.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T09:09:15.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T09:09:15.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T09:09:15.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.887+0000 7f2301a04c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T09:09:15.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.891+0000 7f22f6220700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 26.sst 2023-10-09T09:09:15.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.891+0000 7f2301a04c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 28, last_sequence is 16738, log_number is 24,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T09:09:15.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.891+0000 7f2301a04c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 24 2023-10-09T09:09:15.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.891+0000 7f2301a04c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 50c9ec51-a9bc-4676-bce4-6c3a6df806e0 2023-10-09T09:09:15.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.891+0000 7f2301a04c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696842554896093, "job": 1, "event": "recovery_started", "wal_files": [24]} 2023-10-09T09:09:15.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.891+0000 7f2301a04c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #24 mode 2 2023-10-09T09:09:15.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.899+0000 7f2301a04c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696842554904388, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 29, "file_size": 1940345, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 16739, "largest_seqno": 17353, "table_properties": {"data_size": 1936010, "index_size": 2681, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 645, "raw_key_size": 5782, "raw_average_key_size": 24, "raw_value_size": 1929552, "raw_average_value_size": 8176, "num_data_blocks": 118, "num_entries": 236, "num_filter_entries": 236, "num_deletions": 17, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696842554, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "50c9ec51-a9bc-4676-bce4-6c3a6df806e0", "db_session_id": "NVQUWTJGJHQQ1CUE35DZ", "orig_file_number": 29, "seqno_to_time_mapping": "N/A"}} 2023-10-09T09:09:15.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.899+0000 7f2301a04c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696842554904544, "job": 1, "event": "recovery_finished"} 2023-10-09T09:09:15.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.899+0000 7f2301a04c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 31 2023-10-09T09:09:15.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.899+0000 7f2301a04c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000024.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.899+0000 7f2301a04c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55df3dafa000 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.899+0000 7f2301a04c80 4 rocksdb: DB pointer 0x55df3dae8000 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.899+0000 7f22f0214700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.899+0000 7f22f0214700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: ** DB Stats ** 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:09:15.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: ** Compaction Stats [default] ** 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: L0 1/0 1.85 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 419.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: L6 1/0 30.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Sum 2/0 32.48 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 419.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 419.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: ** Compaction Stats [default] ** 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 419.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:09:15.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Flush(GB): cumulative 0.002, interval 0.002 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Cumulative compaction: 0.00 GB write, 155.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Interval compaction: 0.00 GB write, 155.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Block cache BinnedLRUCache@0x55df3da591f0#7 capacity: 512.00 MB usage: 3.55 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.1e-05 secs_since: 0 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Block cache entry stats(count,size,portion): FilterBlock(1,0.72 KB,0.000137091%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2023-10-09T09:09:15.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:09:15.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.903+0000 7f2301a04c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] at bind addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T09:09:15.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.903+0000 7f2301a04c80 1 mon.a@-1(???) e3 preinit fsid 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T09:09:15.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.903+0000 7f2301a04c80 0 mon.a@-1(???).mds e6 new map 2023-10-09T09:09:15.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.903+0000 7f2301a04c80 0 mon.a@-1(???).mds e6 print_map 2023-10-09T09:09:15.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: e6 2023-10-09T09:09:15.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T09:09:15.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:09:15.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: legacy client fscid: 1 2023-10-09T09:09:15.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Filesystem 'foo' (1) 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: fs_name foo 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: epoch 5 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: created 2023-10-09T09:03:24.292218+0000 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: modified 2023-10-09T09:03:30.346509+0000 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: tableserver 0 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: root 0 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: session_timeout 60 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: session_autoclose 300 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: max_file_size 1099511627776 2023-10-09T09:09:15.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: required_client_features {} 2023-10-09T09:09:15.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: last_failure 0 2023-10-09T09:09:15.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: last_failure_osd_epoch 0 2023-10-09T09:09:15.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:09:15.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: max_mds 1 2023-10-09T09:09:15.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: in 0 2023-10-09T09:09:15.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: up {0=14640} 2023-10-09T09:09:15.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: failed 2023-10-09T09:09:15.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: damaged 2023-10-09T09:09:15.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: stopped 2023-10-09T09:09:15.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: data_pools [4] 2023-10-09T09:09:15.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: metadata_pool 3 2023-10-09T09:09:15.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: inline_data disabled 2023-10-09T09:09:15.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: balancer 2023-10-09T09:09:15.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: bal_rank_mask -1 2023-10-09T09:09:15.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: standby_count_wanted 1 2023-10-09T09:09:15.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: [mds.foo.smithi163.vnodzj{0:14640} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.163:6834/3139884719,v1:172.21.15.163:6835/3139884719] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:09:15.257 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: 2023-10-09T09:09:15.257 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: 2023-10-09T09:09:15.257 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: Standby daemons: 2023-10-09T09:09:15.257 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: 2023-10-09T09:09:15.257 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: [mds.foo.smithi181.ajqgsc{-1:24511} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.181:6832/2837854117,v1:172.21.15.181:6833/2837854117] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:09:15.257 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.903+0000 7f2301a04c80 0 mon.a@-1(???).osd e293 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T09:09:15.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.903+0000 7f2301a04c80 0 mon.a@-1(???).osd e293 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:09:15.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.903+0000 7f2301a04c80 0 mon.a@-1(???).osd e293 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:09:15.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.903+0000 7f2301a04c80 0 mon.a@-1(???).osd e293 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:09:15.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.903+0000 7f2301a04c80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-09T09:09:15.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta expand map: {default=false} 2023-10-09T09:09:15.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta from 'false' to 'false' 2023-10-09T09:09:15.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta expanded map: {default=false} 2023-10-09T09:09:15.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta expand map: {default=info} 2023-10-09T09:09:15.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta from 'info' to 'info' 2023-10-09T09:09:15.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta expanded map: {default=info} 2023-10-09T09:09:15.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta expand map: {default=daemon} 2023-10-09T09:09:15.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T09:09:15.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta expanded map: {default=daemon} 2023-10-09T09:09:15.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta expand map: {default=debug} 2023-10-09T09:09:15.260 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta from 'debug' to 'debug' 2023-10-09T09:09:15.260 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:09:14 smithi163 bash[75425]: debug 2023-10-09T09:09:14.907+0000 7f2301a04c80 20 expand_channel_meta expanded map: {default=debug} 2023-10-09T09:09:16.432 INFO:tasks.rados.rados.0.smithi163.stdout:908: done (1 left) 2023-10-09T09:09:16.433 INFO:tasks.rados.rados.0.smithi163.stdout:909: done (0 left) 2023-10-09T09:09:16.433 INFO:tasks.rados.rados.0.smithi163.stdout:910: delete oid 43 current snap is 117 2023-10-09T09:09:16.439 INFO:tasks.rados.rados.0.smithi163.stdout:910: done (0 left) 2023-10-09T09:09:16.439 INFO:tasks.rados.rados.0.smithi163.stdout:911: delete oid 5 current snap is 117 2023-10-09T09:09:16.448 INFO:tasks.rados.rados.0.smithi163.stdout:911: done (0 left) 2023-10-09T09:09:16.448 INFO:tasks.rados.rados.0.smithi163.stdout:912: read oid 3 snap -1 2023-10-09T09:09:16.448 INFO:tasks.rados.rados.0.smithi163.stdout:912: expect deleted 2023-10-09T09:09:16.449 INFO:tasks.rados.rados.0.smithi163.stdout:913: rollback oid 37 current snap is 117 2023-10-09T09:09:16.449 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 103 2023-10-09T09:09:16.449 INFO:tasks.rados.rados.0.smithi163.stdout:914: snap_remove snap 112 2023-10-09T09:09:16.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[71599]: ignoring --setuser ceph since I am not root 2023-10-09T09:09:16.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[71599]: ignoring --setgroup ceph since I am not root 2023-10-09T09:09:16.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[71599]: debug 2023-10-09T09:09:16.467+0000 7f5ca6c15700 1 -- 172.21.15.163:0/1277082262 <== mon.2 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e90cc9b1e0 con 0x55e90da1a400 2023-10-09T09:09:16.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[71599]: debug 2023-10-09T09:09:16.627+0000 7f5cb0945200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: audit 2023-10-09T09:09:12.909658+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: audit 2023-10-09T09:09:12.910040+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: audit 2023-10-09T09:09:12.910272+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: audit 2023-10-09T09:09:12.947602+0000 mon.b (mon.1) 98 : audit [DBG] from='client.15477 172.21.15.163:0/2502736380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: cluster 2023-10-09T09:09:13.479181+0000 mgr.y (mgr.15273) 47 : cluster [DBG] pgmap v77: 89 pgs: 89 active+clean; 146 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 501 KiB/s wr, 8 op/s 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: audit 2023-10-09T09:09:15.390284+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.163:0/2670722764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: cluster 2023-10-09T09:09:15.480380+0000 mgr.y (mgr.15273) 48 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 629 KiB/s wr, 11 op/s 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.328062+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.352868+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.362771+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T09:09:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.363542+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 1 up:standby 2023-10-09T09:09:16.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.363577+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2023-10-09T09:09:16.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.364832+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e30: y(active, since 52s), standbys: x 2023-10-09T09:09:16.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.397179+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e31: y(active, since 52s), standbys: x 2023-10-09T09:09:16.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.397221+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T09:09:16.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: audit 2023-10-09T09:09:16.397329+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15273 ' entity='' 2023-10-09T09:09:16.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:16 smithi163 bash[33912]: debug 2023-10-09T09:09:16.419+0000 7f84df46a700 -1 mon.c@2(peon).osd e294 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:16.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[55332]: ignoring --setuser ceph since I am not root 2023-10-09T09:09:16.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[55332]: ignoring --setgroup ceph since I am not root 2023-10-09T09:09:16.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[55332]: debug 2023-10-09T09:09:16.464+0000 7ff742c09700 1 -- 172.21.15.181:0/3068855636 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c568b171e0 con 0x55c569896800 2023-10-09T09:09:16.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[55332]: debug 2023-10-09T09:09:16.620+0000 7ff74c939200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:09:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: audit 2023-10-09T09:09:12.909658+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:09:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: audit 2023-10-09T09:09:12.910040+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:09:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: audit 2023-10-09T09:09:12.910272+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.15273 172.21.15.163:0/1818731017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:09:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: audit 2023-10-09T09:09:12.947602+0000 mon.b (mon.1) 98 : audit [DBG] from='client.15477 172.21.15.163:0/2502736380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: cluster 2023-10-09T09:09:13.479181+0000 mgr.y (mgr.15273) 47 : cluster [DBG] pgmap v77: 89 pgs: 89 active+clean; 146 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 501 KiB/s wr, 8 op/s 2023-10-09T09:09:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: audit 2023-10-09T09:09:15.390284+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.163:0/2670722764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: cluster 2023-10-09T09:09:15.480380+0000 mgr.y (mgr.15273) 48 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 629 KiB/s wr, 11 op/s 2023-10-09T09:09:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.328062+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T09:09:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.352868+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:09:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.362771+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T09:09:16.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.363542+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 1 up:standby 2023-10-09T09:09:16.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.363577+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2023-10-09T09:09:16.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.364832+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e30: y(active, since 52s), standbys: x 2023-10-09T09:09:16.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.397179+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e31: y(active, since 52s), standbys: x 2023-10-09T09:09:16.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.397221+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T09:09:16.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: audit 2023-10-09T09:09:16.397329+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15273 ' entity='' 2023-10-09T09:09:16.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:16 smithi181 bash[27712]: debug 2023-10-09T09:09:16.420+0000 7fde83035700 -1 mon.b@1(peon).osd e294 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:17.450 INFO:tasks.rados.rados.0.smithi163.stdout:914: done (2 left) 2023-10-09T09:09:17.450 INFO:tasks.rados.rados.0.smithi163.stdout:915: rollback oid 40 current snap is 117 2023-10-09T09:09:17.450 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 40 to 113 2023-10-09T09:09:17.450 INFO:tasks.rados.rados.0.smithi163.stdout:916: write oid 2 current snap is 117 2023-10-09T09:09:17.450 INFO:tasks.rados.rados.0.smithi163.stdout:916: seq_num 376 ranges {719432=794921,2186053=133379} 2023-10-09T09:09:17.457 INFO:tasks.rados.rados.0.smithi163.stdout:916: writing smithi16367185-2 from 719432 to 1514353 tid 1 2023-10-09T09:09:17.463 INFO:tasks.rados.rados.0.smithi163.stdout:916: writing smithi16367185-2 from 2186053 to 2319432 tid 2 2023-10-09T09:09:17.463 INFO:tasks.rados.rados.0.smithi163.stdout:917: delete oid 25 current snap is 117 2023-10-09T09:09:17.463 INFO:tasks.rados.rados.0.smithi163.stdout:913: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:09:17.463 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 0 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:09:17.463 INFO:tasks.rados.rados.0.smithi163.stdout:915: finishing rollback tid 1 to smithi16367185-40 2023-10-09T09:09:17.463 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 489 (ObjNum 0 snap 32603 seq_num 698828704) dirty dne 2023-10-09T09:09:17.466 INFO:tasks.rados.rados.0.smithi163.stdout:912: done (4 left) 2023-10-09T09:09:17.466 INFO:tasks.rados.rados.0.smithi163.stdout:913: done (3 left) 2023-10-09T09:09:17.466 INFO:tasks.rados.rados.0.smithi163.stdout:915: done (2 left) 2023-10-09T09:09:17.467 INFO:tasks.rados.rados.0.smithi163.stdout:917: done (1 left) 2023-10-09T09:09:17.467 INFO:tasks.rados.rados.0.smithi163.stdout:918: rollback oid 27 current snap is 117 2023-10-09T09:09:17.467 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 27 to 115 2023-10-09T09:09:17.467 INFO:tasks.rados.rados.0.smithi163.stdout:919: read oid 38 snap -1 2023-10-09T09:09:17.467 INFO:tasks.rados.rados.0.smithi163.stdout:919: expect (ObjNum 221 snap 61 seq_num 221) 2023-10-09T09:09:17.467 INFO:tasks.rados.rados.0.smithi163.stdout:920: read oid 33 snap -1 2023-10-09T09:09:17.467 INFO:tasks.rados.rados.0.smithi163.stdout:920: expect (ObjNum 312 snap 92 seq_num 312) 2023-10-09T09:09:17.467 INFO:tasks.rados.rados.0.smithi163.stdout:921: rollback oid 7 current snap is 117 2023-10-09T09:09:17.467 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 110 2023-10-09T09:09:17.467 INFO:tasks.rados.rados.0.smithi163.stdout:922: snap_remove snap 114 2023-10-09T09:09:17.701 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:17 smithi163 bash[71599]: debug 2023-10-09T09:09:17.591+0000 7f5cb0945200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:09:17.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:17 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.427779+0000 mon.a (mon.0) 10 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2023-10-09T09:09:17.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:17 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.429055+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e294 with expected crc 2023-10-09T09:09:17.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:17 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.429144+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e294 with expected crc 2023-10-09T09:09:17.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:17 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.432534+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e294 with expected crc 2023-10-09T09:09:17.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:17 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.441418+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e294 with expected crc 2023-10-09T09:09:17.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:17 smithi163 bash[33912]: cluster 2023-10-09T09:09:16.448391+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e294 with expected crc 2023-10-09T09:09:17.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:17 smithi163 bash[33912]: debug 2023-10-09T09:09:17.427+0000 7f84df46a700 -1 mon.c@2(peon).osd e295 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:17.776 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:17 smithi181 bash[55332]: debug 2023-10-09T09:09:17.564+0000 7ff74c939200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:09:17.776 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:17 smithi181 bash[55332]: debug 2023-10-09T09:09:17.668+0000 7ff74c939200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:09:17.776 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:17 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.427779+0000 mon.a (mon.0) 10 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2023-10-09T09:09:17.776 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:17 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.429055+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e294 with expected crc 2023-10-09T09:09:17.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:17 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.429144+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e294 with expected crc 2023-10-09T09:09:17.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:17 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.432534+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e294 with expected crc 2023-10-09T09:09:17.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:17 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.441418+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e294 with expected crc 2023-10-09T09:09:17.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:17 smithi181 bash[27712]: cluster 2023-10-09T09:09:16.448391+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e294 with expected crc 2023-10-09T09:09:17.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:17 smithi181 bash[27712]: debug 2023-10-09T09:09:17.428+0000 7fde83035700 -1 mon.b@1(peon).osd e295 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:17.861 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T09:09:17.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:17 smithi163 bash[71599]: debug 2023-10-09T09:09:17.695+0000 7f5cb0945200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:09:17.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:17 smithi163 bash[71599]: debug 2023-10-09T09:09:17.803+0000 7f5cb0945200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:09:18.053 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:17 smithi181 bash[55332]: debug 2023-10-09T09:09:17.772+0000 7ff74c939200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:09:18.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[55332]: debug 2023-10-09T09:09:18.048+0000 7ff74c939200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:09:18.436 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:18 smithi163 bash[71599]: debug 2023-10-09T09:09:18.083+0000 7f5cb0945200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:922: done (5 left) 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:923: rollback oid 5 current snap is 117 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 116 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:916: finishing write tid 1 to smithi16367185-2 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:916: finishing write tid 2 to smithi16367185-2 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:916: finishing write tid 3 to smithi16367185-2 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:916: finishing write tid 4 to smithi16367185-2 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 269 (ObjNum 376 snap 117 seq_num 376) dirty exists 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:916: left oid 2 (ObjNum 376 snap 117 seq_num 376) 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:921: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:09:18.447 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 0 (ObjNum 669367424 snap 21875 seq_num 4000000) dirty dne 2023-10-09T09:09:18.448 INFO:tasks.rados.rados.0.smithi163.stdout:918: finishing rollback tid 1 to smithi16367185-27 2023-10-09T09:09:18.448 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 287 (ObjNum 0 snap 0 seq_num 1476397264) dirty dne 2023-10-09T09:09:18.448 INFO:tasks.rados.rados.0.smithi163.stdout:920: expect (ObjNum 312 snap 92 seq_num 312) 2023-10-09T09:09:18.555 INFO:tasks.rados.rados.0.smithi163.stdout:919: expect (ObjNum 221 snap 61 seq_num 221) 2023-10-09T09:09:18.635 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:09:18.677 INFO:tasks.rados.rados.0.smithi163.stdout:923: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:09:18.677 INFO:tasks.rados.rados.0.smithi163.stdout:923: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:09:18.678 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 513 (ObjNum 352 snap 106 seq_num 352) dirty exists 2023-10-09T09:09:18.678 INFO:tasks.rados.rados.0.smithi163.stdout:916: done (5 left) 2023-10-09T09:09:18.678 INFO:tasks.rados.rados.0.smithi163.stdout:918: done (4 left) 2023-10-09T09:09:18.678 INFO:tasks.rados.rados.0.smithi163.stdout:919: done (3 left) 2023-10-09T09:09:18.678 INFO:tasks.rados.rados.0.smithi163.stdout:920: done (2 left) 2023-10-09T09:09:18.678 INFO:tasks.rados.rados.0.smithi163.stdout:921: done (1 left) 2023-10-09T09:09:18.678 INFO:tasks.rados.rados.0.smithi163.stdout:923: done (0 left) 2023-10-09T09:09:18.678 INFO:tasks.rados.rados.0.smithi163.stdout:924: snap_remove snap 103 2023-10-09T09:09:18.690 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:18 smithi163 bash[71599]: debug 2023-10-09T09:09:18.443+0000 7f5cb0945200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:09:18.690 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:18 smithi163 bash[71599]: debug 2023-10-09T09:09:18.607+0000 7f5cb0945200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:09:18.690 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:18 smithi163 bash[33912]: debug 2023-10-09T09:09:18.431+0000 7f84df46a700 -1 mon.c@2(peon).osd e296 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:18.690 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:18 smithi163 bash[33912]: cluster 2023-10-09T09:09:17.436619+0000 mon.a (mon.0) 11 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2023-10-09T09:09:18.690 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:18 smithi163 bash[33912]: cluster 2023-10-09T09:09:17.440121+0000 osd.6 (osd.6) 3 : cluster [WRN] failed to encode map e295 with expected crc 2023-10-09T09:09:18.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:18 smithi163 bash[33912]: cluster 2023-10-09T09:09:17.441364+0000 osd.3 (osd.3) 4 : cluster [WRN] failed to encode map e295 with expected crc 2023-10-09T09:09:18.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:18 smithi163 bash[33912]: cluster 2023-10-09T09:09:17.450012+0000 osd.7 (osd.7) 4 : cluster [WRN] failed to encode map e295 with expected crc 2023-10-09T09:09:18.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:18 smithi163 bash[33912]: audit 2023-10-09T09:09:17.846619+0000 mon.a (mon.0) 12 : audit [DBG] from='client.? 172.21.15.163:0/3863008571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:18.797 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[55332]: debug 2023-10-09T09:09:18.408+0000 7ff74c939200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:09:18.797 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[55332]: debug 2023-10-09T09:09:18.576+0000 7ff74c939200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:09:18.798 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[54907]: ts=2023-10-09T09:09:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:09:18.798 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[27712]: debug 2023-10-09T09:09:18.432+0000 7fde83035700 -1 mon.b@1(peon).osd e296 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:18.798 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[27712]: cluster 2023-10-09T09:09:17.436619+0000 mon.a (mon.0) 11 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2023-10-09T09:09:18.798 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[27712]: cluster 2023-10-09T09:09:17.440121+0000 osd.6 (osd.6) 3 : cluster [WRN] failed to encode map e295 with expected crc 2023-10-09T09:09:18.798 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[27712]: cluster 2023-10-09T09:09:17.441364+0000 osd.3 (osd.3) 4 : cluster [WRN] failed to encode map e295 with expected crc 2023-10-09T09:09:18.798 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[27712]: cluster 2023-10-09T09:09:17.450012+0000 osd.7 (osd.7) 4 : cluster [WRN] failed to encode map e295 with expected crc 2023-10-09T09:09:18.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[27712]: audit 2023-10-09T09:09:17.846619+0000 mon.a (mon.0) 12 : audit [DBG] from='client.? 172.21.15.163:0/3863008571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:09:18.967 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:18 smithi163 bash[71599]: debug 2023-10-09T09:09:18.839+0000 7f5cb0945200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:09:19.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:18 smithi181 bash[55332]: debug 2023-10-09T09:09:18.792+0000 7ff74c939200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:09:19.464 INFO:tasks.rados.rados.0.smithi163.stdout:924: done (0 left) 2023-10-09T09:09:19.464 INFO:tasks.rados.rados.0.smithi163.stdout:925: read oid 14 snap -1 2023-10-09T09:09:19.464 INFO:tasks.rados.rados.0.smithi163.stdout:925: expect (ObjNum 322 snap 96 seq_num 322) 2023-10-09T09:09:19.464 INFO:tasks.rados.rados.0.smithi163.stdout:926: write oid 35 current snap is 117 2023-10-09T09:09:19.464 INFO:tasks.rados.rados.0.smithi163.stdout:926: seq_num 378 ranges {753046=545419,1786664=431793} 2023-10-09T09:09:19.471 INFO:tasks.rados.rados.0.smithi163.stdout:926: writing smithi16367185-35 from 753046 to 1298465 tid 1 2023-10-09T09:09:19.477 INFO:tasks.rados.rados.0.smithi163.stdout:926: writing smithi16367185-35 from 1786664 to 2218457 tid 2 2023-10-09T09:09:19.477 INFO:tasks.rados.rados.0.smithi163.stdout:927: write oid 18 current snap is 117 2023-10-09T09:09:19.477 INFO:tasks.rados.rados.0.smithi163.stdout:927: seq_num 379 ranges {769853=620668,1845146=524707} 2023-10-09T09:09:19.485 INFO:tasks.rados.rados.0.smithi163.stdout:927: writing smithi16367185-18 from 769853 to 1390521 tid 1 2023-10-09T09:09:19.491 INFO:tasks.rados.rados.0.smithi163.stdout:927: writing smithi16367185-18 from 1845146 to 2369853 tid 2 2023-10-09T09:09:19.491 INFO:tasks.rados.rados.0.smithi163.stdout:928: read oid 31 snap -1 2023-10-09T09:09:19.491 INFO:tasks.rados.rados.0.smithi163.stdout:928: expect (ObjNum 306 snap 92 seq_num 306) 2023-10-09T09:09:19.491 INFO:tasks.rados.rados.0.smithi163.stdout:929: snap_remove snap 108 2023-10-09T09:09:19.717 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:19 smithi163 bash[33912]: cluster 2023-10-09T09:09:18.441803+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e296 with expected crc 2023-10-09T09:09:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:19 smithi163 bash[33912]: cluster 2023-10-09T09:09:18.442309+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2023-10-09T09:09:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:19 smithi163 bash[33912]: cluster 2023-10-09T09:09:18.443778+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e296 with expected crc 2023-10-09T09:09:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:19 smithi163 bash[33912]: cluster 2023-10-09T09:09:18.448564+0000 osd.2 (osd.2) 4 : cluster [WRN] failed to encode map e296 with expected crc 2023-10-09T09:09:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:19 smithi163 bash[33912]: debug 2023-10-09T09:09:19.451+0000 7f84df46a700 -1 mon.c@2(peon).osd e297 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:19.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:19 smithi181 bash[27712]: cluster 2023-10-09T09:09:18.441803+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e296 with expected crc 2023-10-09T09:09:19.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:19 smithi181 bash[27712]: cluster 2023-10-09T09:09:18.442309+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2023-10-09T09:09:19.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:19 smithi181 bash[27712]: cluster 2023-10-09T09:09:18.443778+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e296 with expected crc 2023-10-09T09:09:19.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:19 smithi181 bash[27712]: cluster 2023-10-09T09:09:18.448564+0000 osd.2 (osd.2) 4 : cluster [WRN] failed to encode map e296 with expected crc 2023-10-09T09:09:19.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:19 smithi181 bash[27712]: debug 2023-10-09T09:09:19.452+0000 7fde83035700 -1 mon.b@1(peon).osd e297 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:19.927 INFO:tasks.rados.rados.0.smithi163.stdout:929: done (4 left) 2023-10-09T09:09:19.927 INFO:tasks.rados.rados.0.smithi163.stdout:930: read oid 34 snap -1 2023-10-09T09:09:19.927 INFO:tasks.rados.rados.0.smithi163.stdout:930: expect (ObjNum 358 snap 108 seq_num 358) 2023-10-09T09:09:19.927 INFO:tasks.rados.rados.0.smithi163.stdout:931: snap_create 2023-10-09T09:09:19.927 INFO:tasks.rados.rados.0.smithi163.stdout:925: expect (ObjNum 322 snap 96 seq_num 322) 2023-10-09T09:09:20.012 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:19 smithi163 bash[33912]: debug 2023-10-09T09:09:19.911+0000 7f84df46a700 -1 mon.c@2(peon).osd e298 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:20.012 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:19 smithi163 bash[71599]: debug 2023-10-09T09:09:19.879+0000 7f5cb0945200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:09:20.056 INFO:tasks.rados.rados.0.smithi163.stdout:926: finishing write tid 1 to smithi16367185-35 2023-10-09T09:09:20.056 INFO:tasks.rados.rados.0.smithi163.stdout:926: finishing write tid 2 to smithi16367185-35 2023-10-09T09:09:20.056 INFO:tasks.rados.rados.0.smithi163.stdout:926: finishing write tid 3 to smithi16367185-35 2023-10-09T09:09:20.056 INFO:tasks.rados.rados.0.smithi163.stdout:926: finishing write tid 4 to smithi16367185-35 2023-10-09T09:09:20.056 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 273 (ObjNum 378 snap 117 seq_num 378) dirty exists 2023-10-09T09:09:20.056 INFO:tasks.rados.rados.0.smithi163.stdout:926: left oid 35 (ObjNum 378 snap 117 seq_num 378) 2023-10-09T09:09:20.057 INFO:tasks.rados.rados.0.smithi163.stdout:927: finishing write tid 1 to smithi16367185-18 2023-10-09T09:09:20.057 INFO:tasks.rados.rados.0.smithi163.stdout:927: finishing write tid 2 to smithi16367185-18 2023-10-09T09:09:20.057 INFO:tasks.rados.rados.0.smithi163.stdout:927: finishing write tid 3 to smithi16367185-18 2023-10-09T09:09:20.057 INFO:tasks.rados.rados.0.smithi163.stdout:927: finishing write tid 4 to smithi16367185-18 2023-10-09T09:09:20.057 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 381 (ObjNum 379 snap 117 seq_num 379) dirty exists 2023-10-09T09:09:20.057 INFO:tasks.rados.rados.0.smithi163.stdout:927: left oid 18 (ObjNum 379 snap 117 seq_num 379) 2023-10-09T09:09:20.057 INFO:tasks.rados.rados.0.smithi163.stdout:928: expect (ObjNum 306 snap 92 seq_num 306) 2023-10-09T09:09:20.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:19 smithi181 bash[27712]: debug 2023-10-09T09:09:19.912+0000 7fde83035700 -1 mon.b@1(peon).osd e298 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:20.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:19 smithi181 bash[55332]: debug 2023-10-09T09:09:19.808+0000 7ff74c939200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:09:20.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:19 smithi181 bash[55332]: debug 2023-10-09T09:09:19.924+0000 7ff74c939200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:09:20.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[55332]: debug 2023-10-09T09:09:20.048+0000 7ff74c939200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:09:20.214 INFO:tasks.rados.rados.0.smithi163.stdout:930: expect (ObjNum 358 snap 108 seq_num 358) 2023-10-09T09:09:20.462 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[71599]: debug 2023-10-09T09:09:20.007+0000 7f5cb0945200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:09:20.463 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[71599]: debug 2023-10-09T09:09:20.131+0000 7f5cb0945200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:09:20.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[71599]: debug 2023-10-09T09:09:20.659+0000 7f5cb0945200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:09:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[33912]: cluster 2023-10-09T09:09:19.460693+0000 osd.4 (osd.4) 5 : cluster [WRN] failed to encode map e297 with expected crc 2023-10-09T09:09:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[33912]: cluster 2023-10-09T09:09:19.461967+0000 osd.3 (osd.3) 5 : cluster [WRN] failed to encode map e297 with expected crc 2023-10-09T09:09:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[33912]: cluster 2023-10-09T09:09:19.462671+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2023-10-09T09:09:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[33912]: cluster 2023-10-09T09:09:19.472379+0000 osd.7 (osd.7) 5 : cluster [WRN] failed to encode map e297 with expected crc 2023-10-09T09:09:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[33912]: cluster 2023-10-09T09:09:19.919716+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2023-10-09T09:09:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[33912]: cluster 2023-10-09T09:09:19.923789+0000 osd.4 (osd.4) 6 : cluster [WRN] failed to encode map e298 with expected crc 2023-10-09T09:09:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[33912]: cluster 2023-10-09T09:09:20.261371+0000 osd.7 (osd.7) 6 : cluster [WRN] failed to encode map e298 with expected crc 2023-10-09T09:09:20.771 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[55332]: debug 2023-10-09T09:09:20.532+0000 7ff74c939200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:09:20.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[27712]: cluster 2023-10-09T09:09:19.460693+0000 osd.4 (osd.4) 5 : cluster [WRN] failed to encode map e297 with expected crc 2023-10-09T09:09:20.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[27712]: cluster 2023-10-09T09:09:19.461967+0000 osd.3 (osd.3) 5 : cluster [WRN] failed to encode map e297 with expected crc 2023-10-09T09:09:20.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[27712]: cluster 2023-10-09T09:09:19.462671+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2023-10-09T09:09:20.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[27712]: cluster 2023-10-09T09:09:19.472379+0000 osd.7 (osd.7) 5 : cluster [WRN] failed to encode map e297 with expected crc 2023-10-09T09:09:20.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[27712]: cluster 2023-10-09T09:09:19.919716+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2023-10-09T09:09:20.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[27712]: cluster 2023-10-09T09:09:19.923789+0000 osd.4 (osd.4) 6 : cluster [WRN] failed to encode map e298 with expected crc 2023-10-09T09:09:20.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[27712]: cluster 2023-10-09T09:09:20.261371+0000 osd.7 (osd.7) 6 : cluster [WRN] failed to encode map e298 with expected crc 2023-10-09T09:09:20.932 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[71599]: debug 2023-10-09T09:09:20.891+0000 7f5cb0945200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:09:20.941 INFO:tasks.rados.rados.0.smithi163.stdout:925: done (5 left) 2023-10-09T09:09:20.941 INFO:tasks.rados.rados.0.smithi163.stdout:926: done (4 left) 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:927: done (3 left) 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:928: done (2 left) 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:930: done (1 left) 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:931: done (0 left) 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:932: rollback oid 28 current snap is 118 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 106 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:933: read oid 29 snap -1 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:933: expect (ObjNum 335 snap 101 seq_num 335) 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:934: rollback oid 42 current snap is 118 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 117 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:935: read oid 46 snap -1 2023-10-09T09:09:20.942 INFO:tasks.rados.rados.0.smithi163.stdout:935: expect (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:09:20.943 INFO:tasks.rados.rados.0.smithi163.stdout:936: write oid 1 current snap is 118 2023-10-09T09:09:20.943 INFO:tasks.rados.rados.0.smithi163.stdout:936: seq_num 380 ranges {786660=695917,1903628=435462} 2023-10-09T09:09:20.950 INFO:tasks.rados.rados.0.smithi163.stdout:936: writing smithi16367185-1 from 786660 to 1482577 tid 1 2023-10-09T09:09:20.956 INFO:tasks.rados.rados.0.smithi163.stdout:936: writing smithi16367185-1 from 1903628 to 2339090 tid 2 2023-10-09T09:09:20.956 INFO:tasks.rados.rados.0.smithi163.stdout:937: read oid 30 snap -1 2023-10-09T09:09:20.957 INFO:tasks.rados.rados.0.smithi163.stdout:937: expect deleted 2023-10-09T09:09:20.957 INFO:tasks.rados.rados.0.smithi163.stdout:938: delete oid 48 current snap is 118 2023-10-09T09:09:20.957 INFO:tasks.rados.rados.0.smithi163.stdout:934: finishing rollback tid 0 to smithi16367185-42 2023-10-09T09:09:20.959 INFO:tasks.rados.rados.0.smithi163.stdout:934: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:09:20.959 INFO:tasks.rados.rados.0.smithi163.stdout:934: finishing rollback tid 2 to smithi16367185-42 2023-10-09T09:09:20.959 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 786 (ObjNum 323 snap 98 seq_num 323) dirty exists 2023-10-09T09:09:20.960 INFO:tasks.rados.rados.0.smithi163.stdout:933: expect (ObjNum 335 snap 101 seq_num 335) 2023-10-09T09:09:21.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[27712]: debug 2023-10-09T09:09:20.924+0000 7fde83035700 -1 mon.b@1(peon).osd e299 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:21.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:20 smithi181 bash[55332]: debug 2023-10-09T09:09:20.764+0000 7ff74c939200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:09:21.075 INFO:tasks.rados.rados.0.smithi163.stdout:932: finishing rollback tid 0 to smithi16367185-28 2023-10-09T09:09:21.075 INFO:tasks.rados.rados.0.smithi163.stdout:932: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:09:21.075 INFO:tasks.rados.rados.0.smithi163.stdout:932: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:09:21.076 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 523 (ObjNum 295 snap 91 seq_num 295) dirty exists 2023-10-09T09:09:21.076 INFO:tasks.rados.rados.0.smithi163.stdout:935: expect (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:09:21.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:20 smithi163 bash[33912]: debug 2023-10-09T09:09:20.927+0000 7f84df46a700 -1 mon.c@2(peon).osd e299 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:21.265 INFO:tasks.rados.rados.0.smithi163.stdout:936: finishing write tid 1 to smithi16367185-1 2023-10-09T09:09:21.265 INFO:tasks.rados.rados.0.smithi163.stdout:936: finishing write tid 2 to smithi16367185-1 2023-10-09T09:09:21.265 INFO:tasks.rados.rados.0.smithi163.stdout:936: finishing write tid 3 to smithi16367185-1 2023-10-09T09:09:21.265 INFO:tasks.rados.rados.0.smithi163.stdout:936: finishing write tid 4 to smithi16367185-1 2023-10-09T09:09:21.265 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 527 (ObjNum 380 snap 118 seq_num 380) dirty exists 2023-10-09T09:09:21.266 INFO:tasks.rados.rados.0.smithi163.stdout:936: left oid 1 (ObjNum 380 snap 118 seq_num 380) 2023-10-09T09:09:21.266 INFO:tasks.rados.rados.0.smithi163.stdout:932: done (6 left) 2023-10-09T09:09:21.266 INFO:tasks.rados.rados.0.smithi163.stdout:933: done (5 left) 2023-10-09T09:09:21.266 INFO:tasks.rados.rados.0.smithi163.stdout:934: done (4 left) 2023-10-09T09:09:21.266 INFO:tasks.rados.rados.0.smithi163.stdout:935: done (3 left) 2023-10-09T09:09:21.266 INFO:tasks.rados.rados.0.smithi163.stdout:936: done (2 left) 2023-10-09T09:09:21.266 INFO:tasks.rados.rados.0.smithi163.stdout:937: done (1 left) 2023-10-09T09:09:21.266 INFO:tasks.rados.rados.0.smithi163.stdout:938: done (0 left) 2023-10-09T09:09:21.266 INFO:tasks.rados.rados.0.smithi163.stdout:939: snap_remove snap 107 2023-10-09T09:09:21.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:21 smithi181 bash[55332]: debug 2023-10-09T09:09:21.232+0000 7ff74c939200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:09:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:21 smithi181 bash[27712]: cluster 2023-10-09T09:09:19.923775+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e298 with expected crc 2023-10-09T09:09:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:21 smithi181 bash[27712]: cluster 2023-10-09T09:09:19.930112+0000 osd.5 (osd.5) 4 : cluster [WRN] failed to encode map e298 with expected crc 2023-10-09T09:09:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:21 smithi181 bash[27712]: cluster 2023-10-09T09:09:20.934521+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2023-10-09T09:09:21.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:21 smithi181 bash[27712]: cluster 2023-10-09T09:09:20.938180+0000 osd.4 (osd.4) 7 : cluster [WRN] failed to encode map e299 with expected crc 2023-10-09T09:09:21.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:21 smithi181 bash[27712]: cluster 2023-10-09T09:09:20.941968+0000 osd.3 (osd.3) 6 : cluster [WRN] failed to encode map e299 with expected crc 2023-10-09T09:09:21.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:21 smithi181 bash[27712]: cluster 2023-10-09T09:09:20.944191+0000 osd.2 (osd.2) 5 : cluster [WRN] failed to encode map e299 with expected crc 2023-10-09T09:09:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:21 smithi163 bash[33912]: cluster 2023-10-09T09:09:19.923775+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e298 with expected crc 2023-10-09T09:09:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:21 smithi163 bash[33912]: cluster 2023-10-09T09:09:19.930112+0000 osd.5 (osd.5) 4 : cluster [WRN] failed to encode map e298 with expected crc 2023-10-09T09:09:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:21 smithi163 bash[33912]: cluster 2023-10-09T09:09:20.934521+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2023-10-09T09:09:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:21 smithi163 bash[33912]: cluster 2023-10-09T09:09:20.938180+0000 osd.4 (osd.4) 7 : cluster [WRN] failed to encode map e299 with expected crc 2023-10-09T09:09:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:21 smithi163 bash[33912]: cluster 2023-10-09T09:09:20.941968+0000 osd.3 (osd.3) 6 : cluster [WRN] failed to encode map e299 with expected crc 2023-10-09T09:09:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:21 smithi163 bash[33912]: cluster 2023-10-09T09:09:20.944191+0000 osd.2 (osd.2) 5 : cluster [WRN] failed to encode map e299 with expected crc 2023-10-09T09:09:21.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:21 smithi163 bash[71599]: debug 2023-10-09T09:09:21.367+0000 7f5cb0945200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:09:21.942 INFO:tasks.rados.rados.0.smithi163.stdout:939: done (0 left) 2023-10-09T09:09:21.943 INFO:tasks.rados.rados.0.smithi163.stdout:940: read oid 5 snap -1 2023-10-09T09:09:21.943 INFO:tasks.rados.rados.0.smithi163.stdout:940: expect (ObjNum 352 snap 106 seq_num 352) 2023-10-09T09:09:21.943 INFO:tasks.rados.rados.0.smithi163.stdout:941: delete oid 37 current snap is 118 2023-10-09T09:09:21.953 INFO:tasks.rados.rados.0.smithi163.stdout:941: done (1 left) 2023-10-09T09:09:21.953 INFO:tasks.rados.rados.0.smithi163.stdout:942: write oid 45 current snap is 118 2023-10-09T09:09:21.953 INFO:tasks.rados.rados.0.smithi163.stdout:942: seq_num 383 ranges {437081=638017,1679074=582789} 2023-10-09T09:09:21.960 INFO:tasks.rados.rados.0.smithi163.stdout:942: writing smithi16367185-45 from 437081 to 1075098 tid 1 2023-10-09T09:09:21.968 INFO:tasks.rados.rados.0.smithi163.stdout:942: writing smithi16367185-45 from 1679074 to 2261863 tid 2 2023-10-09T09:09:21.968 INFO:tasks.rados.rados.0.smithi163.stdout:943: read oid 33 snap -1 2023-10-09T09:09:21.968 INFO:tasks.rados.rados.0.smithi163.stdout:943: expect (ObjNum 312 snap 92 seq_num 312) 2023-10-09T09:09:21.968 INFO:tasks.rados.rados.0.smithi163.stdout:944: snap_remove snap 115 2023-10-09T09:09:22.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:21 smithi181 bash[27712]: debug 2023-10-09T09:09:21.928+0000 7fde83035700 -1 mon.b@1(peon).osd e300 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:22.067 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:21 smithi181 bash[55332]: debug 2023-10-09T09:09:21.744+0000 7ff74c939200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:09:22.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:21 smithi163 bash[71599]: debug 2023-10-09T09:09:21.883+0000 7f5cb0945200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:09:22.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:21 smithi163 bash[33912]: debug 2023-10-09T09:09:21.927+0000 7f84df46a700 -1 mon.c@2(peon).osd e300 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:22.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:22 smithi181 bash[55332]: debug 2023-10-09T09:09:22.308+0000 7ff74c939200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:09:22.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:22 smithi181 bash[55332]: debug 2023-10-09T09:09:22.420+0000 7ff74c939200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:09:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:22 smithi181 bash[27712]: cluster 2023-10-09T09:09:20.941267+0000 osd.5 (osd.5) 5 : cluster [WRN] failed to encode map e299 with expected crc 2023-10-09T09:09:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:22 smithi181 bash[27712]: cluster 2023-10-09T09:09:21.936241+0000 mon.a (mon.0) 17 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2023-10-09T09:09:22.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:22 smithi181 bash[27712]: cluster 2023-10-09T09:09:21.939516+0000 osd.4 (osd.4) 8 : cluster [WRN] failed to encode map e300 with expected crc 2023-10-09T09:09:22.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:22 smithi181 bash[27712]: cluster 2023-10-09T09:09:21.942321+0000 osd.7 (osd.7) 7 : cluster [WRN] failed to encode map e300 with expected crc 2023-10-09T09:09:22.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:22 smithi181 bash[27712]: cluster 2023-10-09T09:09:21.944230+0000 osd.2 (osd.2) 6 : cluster [WRN] failed to encode map e300 with expected crc 2023-10-09T09:09:22.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:22 smithi163 bash[71599]: debug 2023-10-09T09:09:22.447+0000 7f5cb0945200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:09:22.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:22 smithi163 bash[71599]: debug 2023-10-09T09:09:22.571+0000 7f5cb0945200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:09:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:22 smithi163 bash[33912]: cluster 2023-10-09T09:09:20.941267+0000 osd.5 (osd.5) 5 : cluster [WRN] failed to encode map e299 with expected crc 2023-10-09T09:09:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:22 smithi163 bash[33912]: cluster 2023-10-09T09:09:21.936241+0000 mon.a (mon.0) 17 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2023-10-09T09:09:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:22 smithi163 bash[33912]: cluster 2023-10-09T09:09:21.939516+0000 osd.4 (osd.4) 8 : cluster [WRN] failed to encode map e300 with expected crc 2023-10-09T09:09:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:22 smithi163 bash[33912]: cluster 2023-10-09T09:09:21.942321+0000 osd.7 (osd.7) 7 : cluster [WRN] failed to encode map e300 with expected crc 2023-10-09T09:09:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:22 smithi163 bash[33912]: cluster 2023-10-09T09:09:21.944230+0000 osd.2 (osd.2) 6 : cluster [WRN] failed to encode map e300 with expected crc 2023-10-09T09:09:22.947 INFO:tasks.rados.rados.0.smithi163.stdout:944: done (3 left) 2023-10-09T09:09:22.947 INFO:tasks.rados.rados.0.smithi163.stdout:945: write oid 42 current snap is 118 2023-10-09T09:09:22.947 INFO:tasks.rados.rados.0.smithi163.stdout:940: expect (ObjNum 352 snap 106 seq_num 352) 2023-10-09T09:09:23.052 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:22 smithi181 bash[55332]: debug 2023-10-09T09:09:22.776+0000 7ff74c939200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:09:23.052 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:22 smithi181 bash[55332]: debug 2023-10-09T09:09:22.896+0000 7ff74c939200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:09:23.053 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:22 smithi181 bash[27712]: debug 2023-10-09T09:09:22.932+0000 7fde83035700 -1 mon.b@1(peon).osd e301 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:23.196 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:22 smithi163 bash[71599]: debug 2023-10-09T09:09:22.919+0000 7f5cb0945200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:09:23.196 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[71599]: debug 2023-10-09T09:09:23.043+0000 7f5cb0945200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:09:23.196 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[71599]: debug 2023-10-09T09:09:23.191+0000 7f5cb0945200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:09:23.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:22 smithi163 bash[33912]: debug 2023-10-09T09:09:22.931+0000 7f84df46a700 -1 mon.c@2(peon).osd e301 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:23.296 INFO:tasks.rados.rados.0.smithi163.stdout:942: finishing write tid 1 to smithi16367185-45 2023-10-09T09:09:23.297 INFO:tasks.rados.rados.0.smithi163.stdout:942: finishing write tid 2 to smithi16367185-45 2023-10-09T09:09:23.297 INFO:tasks.rados.rados.0.smithi163.stdout:943: expect (ObjNum 312 snap 92 seq_num 312) 2023-10-09T09:09:23.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:23 smithi181 bash[55332]: debug 2023-10-09T09:09:23.048+0000 7ff74c939200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:09:23.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:23 smithi181 bash[55332]: debug 2023-10-09T09:09:23.168+0000 7ff74c939200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:09:23.317 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:23 smithi181 bash[55332]: debug 2023-10-09T09:09:23.296+0000 7ff74c939200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:09:23.398 INFO:tasks.rados.rados.0.smithi163.stdout:942: finishing write tid 3 to smithi16367185-45 2023-10-09T09:09:23.398 INFO:tasks.rados.rados.0.smithi163.stdout:942: finishing write tid 4 to smithi16367185-45 2023-10-09T09:09:23.398 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 798 (ObjNum 383 snap 118 seq_num 383) dirty exists 2023-10-09T09:09:23.398 INFO:tasks.rados.rados.0.smithi163.stdout:942: left oid 45 (ObjNum 383 snap 118 seq_num 383) 2023-10-09T09:09:23.398 INFO:tasks.rados.rados.0.smithi163.stdout:945: seq_num 384 ranges {453888=713266,1737556=442800} 2023-10-09T09:09:23.414 INFO:tasks.rados.rados.0.smithi163.stdout:945: writing smithi16367185-42 from 453888 to 1167154 tid 1 2023-10-09T09:09:23.424 INFO:tasks.rados.rados.0.smithi163.stdout:945: writing smithi16367185-42 from 1737556 to 2180356 tid 2 2023-10-09T09:09:23.424 INFO:tasks.rados.rados.0.smithi163.stdout:940: done (3 left) 2023-10-09T09:09:23.424 INFO:tasks.rados.rados.0.smithi163.stdout:942: done (2 left) 2023-10-09T09:09:23.425 INFO:tasks.rados.rados.0.smithi163.stdout:943: done (1 left) 2023-10-09T09:09:23.425 INFO:tasks.rados.rados.0.smithi163.stdout:946: write oid 38 current snap is 118 2023-10-09T09:09:23.425 INFO:tasks.rados.rados.0.smithi163.stdout:946: seq_num 385 ranges {470695=788515,1679685=586458} 2023-10-09T09:09:23.438 INFO:tasks.rados.rados.0.smithi163.stdout:946: writing smithi16367185-38 from 470695 to 1259210 tid 1 2023-10-09T09:09:23.447 INFO:tasks.rados.rados.0.smithi163.stdout:946: writing smithi16367185-38 from 1679685 to 2266143 tid 2 2023-10-09T09:09:23.447 INFO:tasks.rados.rados.0.smithi163.stdout:947: rollback oid 24 current snap is 118 2023-10-09T09:09:23.447 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 24 to 106 2023-10-09T09:09:23.447 INFO:tasks.rados.rados.0.smithi163.stdout:945: finishing write tid 1 to smithi16367185-42 2023-10-09T09:09:23.447 INFO:tasks.rados.rados.0.smithi163.stdout:945: finishing write tid 2 to smithi16367185-42 2023-10-09T09:09:23.447 INFO:tasks.rados.rados.0.smithi163.stdout:948: write oid 45 current snap is 118 2023-10-09T09:09:23.447 INFO:tasks.rados.rados.0.smithi163.stdout:948: seq_num 386 ranges {487502=463764,1738167=446469} 2023-10-09T09:09:23.452 INFO:tasks.rados.rados.0.smithi163.stdout:948: writing smithi16367185-45 from 487502 to 951266 tid 1 2023-10-09T09:09:23.459 INFO:tasks.rados.rados.0.smithi163.stdout:948: writing smithi16367185-45 from 1738167 to 2184636 tid 2 2023-10-09T09:09:23.459 INFO:tasks.rados.rados.0.smithi163.stdout:949: snap_remove snap 116 2023-10-09T09:09:23.464 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[71599]: debug 2023-10-09T09:09:23.311+0000 7f5cb0945200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:09:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[71599]: debug 2023-10-09T09:09:23.459+0000 7f5cb0945200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:09:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[71599]: debug 2023-10-09T09:09:23.667+0000 7f5cb0945200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:09:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[33912]: cluster 2023-10-09T09:09:21.941223+0000 osd.1 (osd.1) 4 : cluster [WRN] failed to encode map e300 with expected crc 2023-10-09T09:09:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[33912]: cluster 2023-10-09T09:09:22.943277+0000 mon.a (mon.0) 18 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2023-10-09T09:09:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[33912]: cluster 2023-10-09T09:09:22.943761+0000 osd.4 (osd.4) 9 : cluster [WRN] failed to encode map e301 with expected crc 2023-10-09T09:09:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[33912]: cluster 2023-10-09T09:09:23.306395+0000 osd.7 (osd.7) 8 : cluster [WRN] failed to encode map e301 with expected crc 2023-10-09T09:09:23.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:23 smithi181 bash[55332]: debug 2023-10-09T09:09:23.496+0000 7ff74c939200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:09:23.817 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:23 smithi181 bash[55332]: debug 2023-10-09T09:09:23.620+0000 7ff74c939200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:09:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:23 smithi181 bash[27712]: cluster 2023-10-09T09:09:21.941223+0000 osd.1 (osd.1) 4 : cluster [WRN] failed to encode map e300 with expected crc 2023-10-09T09:09:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:23 smithi181 bash[27712]: cluster 2023-10-09T09:09:22.943277+0000 mon.a (mon.0) 18 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2023-10-09T09:09:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:23 smithi181 bash[27712]: cluster 2023-10-09T09:09:22.943761+0000 osd.4 (osd.4) 9 : cluster [WRN] failed to encode map e301 with expected crc 2023-10-09T09:09:23.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:23 smithi181 bash[27712]: cluster 2023-10-09T09:09:23.306395+0000 osd.7 (osd.7) 8 : cluster [WRN] failed to encode map e301 with expected crc 2023-10-09T09:09:23.955 INFO:tasks.rados.rados.0.smithi163.stdout:949: done (4 left) 2023-10-09T09:09:23.955 INFO:tasks.rados.rados.0.smithi163.stdout:950: snap_create 2023-10-09T09:09:23.955 INFO:tasks.rados.rados.0.smithi163.stdout:945: finishing write tid 3 to smithi16367185-42 2023-10-09T09:09:23.955 INFO:tasks.rados.rados.0.smithi163.stdout:945: finishing write tid 4 to smithi16367185-42 2023-10-09T09:09:23.955 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 807 (ObjNum 384 snap 118 seq_num 384) dirty exists 2023-10-09T09:09:23.955 INFO:tasks.rados.rados.0.smithi163.stdout:945: left oid 42 (ObjNum 384 snap 118 seq_num 384) 2023-10-09T09:09:23.955 INFO:tasks.rados.rados.0.smithi163.stdout:946: finishing write tid 1 to smithi16367185-38 2023-10-09T09:09:23.955 INFO:tasks.rados.rados.0.smithi163.stdout:946: finishing write tid 2 to smithi16367185-38 2023-10-09T09:09:23.955 INFO:tasks.rados.rados.0.smithi163.stdout:946: finishing write tid 3 to smithi16367185-38 2023-10-09T09:09:23.955 INFO:tasks.rados.rados.0.smithi163.stdout:946: finishing write tid 4 to smithi16367185-38 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 405 (ObjNum 385 snap 118 seq_num 385) dirty exists 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:946: left oid 38 (ObjNum 385 snap 118 seq_num 385) 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:947: finishing rollback tid 0 to smithi16367185-24 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:947: finishing rollback tid 1 to smithi16367185-24 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:947: finishing rollback tid 2 to smithi16367185-24 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 267 (ObjNum 327 snap 99 seq_num 327) dirty exists 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:948: finishing write tid 1 to smithi16367185-45 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:948: finishing write tid 2 to smithi16367185-45 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:948: finishing write tid 3 to smithi16367185-45 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:948: finishing write tid 4 to smithi16367185-45 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 810 (ObjNum 386 snap 118 seq_num 386) dirty exists 2023-10-09T09:09:23.956 INFO:tasks.rados.rados.0.smithi163.stdout:948: left oid 45 (ObjNum 386 snap 118 seq_num 386) 2023-10-09T09:09:24.218 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[71599]: debug 2023-10-09T09:09:23.807+0000 7f5cb0945200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:09:24.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:23 smithi163 bash[33912]: debug 2023-10-09T09:09:23.939+0000 7f84df46a700 -1 mon.c@2(peon).osd e302 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:23 smithi181 bash[27712]: debug 2023-10-09T09:09:23.936+0000 7fde83035700 -1 mon.b@1(peon).osd e302 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:24 smithi181 bash[27712]: cluster 2023-10-09T09:09:22.945316+0000 osd.1 (osd.1) 5 : cluster [WRN] failed to encode map e301 with expected crc 2023-10-09T09:09:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:24 smithi181 bash[27712]: cluster 2023-10-09T09:09:23.352374+0000 osd.2 (osd.2) 7 : cluster [WRN] failed to encode map e301 with expected crc 2023-10-09T09:09:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:24 smithi181 bash[27712]: cluster 2023-10-09T09:09:23.944992+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2023-10-09T09:09:24.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:24 smithi181 bash[27712]: cluster 2023-10-09T09:09:24.222195+0000 osd.4 (osd.4) 10 : cluster [WRN] failed to encode map e302 with expected crc 2023-10-09T09:09:24.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:24 smithi181 bash[27712]: cluster 2023-10-09T09:09:24.353457+0000 osd.7 (osd.7) 9 : cluster [WRN] failed to encode map e302 with expected crc 2023-10-09T09:09:24.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:24 smithi163 bash[33912]: cluster 2023-10-09T09:09:22.945316+0000 osd.1 (osd.1) 5 : cluster [WRN] failed to encode map e301 with expected crc 2023-10-09T09:09:24.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:24 smithi163 bash[33912]: cluster 2023-10-09T09:09:23.352374+0000 osd.2 (osd.2) 7 : cluster [WRN] failed to encode map e301 with expected crc 2023-10-09T09:09:24.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:24 smithi163 bash[33912]: cluster 2023-10-09T09:09:23.944992+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2023-10-09T09:09:24.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:24 smithi163 bash[33912]: cluster 2023-10-09T09:09:24.222195+0000 osd.4 (osd.4) 10 : cluster [WRN] failed to encode map e302 with expected crc 2023-10-09T09:09:24.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:24 smithi163 bash[33912]: cluster 2023-10-09T09:09:24.353457+0000 osd.7 (osd.7) 9 : cluster [WRN] failed to encode map e302 with expected crc 2023-10-09T09:09:24.943 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:24 smithi163 bash[33912]: debug 2023-10-09T09:09:24.935+0000 7f84df46a700 -1 mon.c@2(peon).osd e303 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:24.953 INFO:tasks.rados.rados.0.smithi163.stdout:945: done (4 left) 2023-10-09T09:09:24.953 INFO:tasks.rados.rados.0.smithi163.stdout:946: done (3 left) 2023-10-09T09:09:24.953 INFO:tasks.rados.rados.0.smithi163.stdout:947: done (2 left) 2023-10-09T09:09:24.954 INFO:tasks.rados.rados.0.smithi163.stdout:948: done (1 left) 2023-10-09T09:09:24.954 INFO:tasks.rados.rados.0.smithi163.stdout:950: done (0 left) 2023-10-09T09:09:24.954 INFO:tasks.rados.rados.0.smithi163.stdout:951: snap_remove snap 110 2023-10-09T09:09:25.194 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.159+0000 7f5cb0945200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:09:25.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:24 smithi181 bash[27712]: debug 2023-10-09T09:09:24.936+0000 7fde83035700 -1 mon.b@1(peon).osd e303 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:25.195 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:24 smithi181 bash[55332]: debug 2023-10-09T09:09:24.924+0000 7ff74c939200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:09:25.195 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:24 smithi181 bash[55332]: [09/Oct/2023:09:09:24] ENGINE Bus STARTING 2023-10-09T09:09:25.195 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[55332]: CherryPy Checker: 2023-10-09T09:09:25.195 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[55332]: The Application mounted at '' has an empty config. 2023-10-09T09:09:25.195 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[55332]: [09/Oct/2023:09:09:25] ENGINE Serving on http://:::9283 2023-10-09T09:09:25.195 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[55332]: [09/Oct/2023:09:09:25] ENGINE Bus STARTED 2023-10-09T09:09:25.209 INFO:tasks.rados.rados.0.smithi163.stdout:951: done (0 left) 2023-10-09T09:09:25.209 INFO:tasks.rados.rados.0.smithi163.stdout:952: delete oid 50 current snap is 119 2023-10-09T09:09:25.222 INFO:tasks.rados.rados.0.smithi163.stdout:952: done (0 left) 2023-10-09T09:09:25.222 INFO:tasks.rados.rados.0.smithi163.stdout:953: delete oid 37 current snap is 119 2023-10-09T09:09:25.225 INFO:tasks.rados.rados.0.smithi163.stdout:953: done (0 left) 2023-10-09T09:09:25.225 INFO:tasks.rados.rados.0.smithi163.stdout:954: write oid 39 current snap is 119 2023-10-09T09:09:25.225 INFO:tasks.rados.rados.0.smithi163.stdout:954: seq_num 389 ranges {537923=405864,1513613=593796} 2023-10-09T09:09:25.230 INFO:tasks.rados.rados.0.smithi163.stdout:954: writing smithi16367185-39 from 537923 to 943787 tid 1 2023-10-09T09:09:25.238 INFO:tasks.rados.rados.0.smithi163.stdout:954: writing smithi16367185-39 from 1513613 to 2107409 tid 2 2023-10-09T09:09:25.238 INFO:tasks.rados.rados.0.smithi163.stdout:955: rollback oid 32 current snap is 119 2023-10-09T09:09:25.239 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 117 2023-10-09T09:09:25.239 INFO:tasks.rados.rados.0.smithi163.stdout:954: finishing write tid 1 to smithi16367185-39 2023-10-09T09:09:25.239 INFO:tasks.rados.rados.0.smithi163.stdout:956: snap_remove snap 106 2023-10-09T09:09:25.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: debug 2023-10-09T09:09:25.187+0000 7f84df46a700 -1 mon.c@2(peon).osd e304 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:25.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: [09/Oct/2023:09:09:25] ENGINE Bus STARTING 2023-10-09T09:09:25.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.407+0000 7f5c124a8700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:09:25.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.407+0000 7f5c124a8700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:09:25.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.407+0000 7f5c124a8700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:09:25.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.407+0000 7f5c124a8700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:09:25.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.407+0000 7f5c124a8700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:09:25.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.407+0000 7f5c09c97700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:09:25.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.407+0000 7f5c09c97700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:09:25.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.407+0000 7f5c09c97700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:09:25.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.407+0000 7f5c09c97700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:09:25.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: debug 2023-10-09T09:09:25.407+0000 7f5c09c97700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:09:25.472 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: debug 2023-10-09T09:09:25.188+0000 7fde83035700 -1 mon.b@1(peon).osd e304 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:23.950888+0000 osd.6 (osd.6) 4 : cluster [WRN] failed to encode map e302 with expected crc 2023-10-09T09:09:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:23.950920+0000 osd.0 (osd.0) 4 : cluster [WRN] failed to encode map e302 with expected crc 2023-10-09T09:09:25.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:24.321043+0000 osd.2 (osd.2) 8 : cluster [WRN] failed to encode map e302 with expected crc 2023-10-09T09:09:25.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:24.947943+0000 osd.2 (osd.2) 9 : cluster [WRN] failed to encode map e303 with expected crc 2023-10-09T09:09:25.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:24.948486+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2023-10-09T09:09:25.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:24.950291+0000 osd.4 (osd.4) 11 : cluster [WRN] failed to encode map e303 with expected crc 2023-10-09T09:09:25.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:24.955093+0000 mon.a (mon.0) 21 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:09:25.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:24.955178+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:09:25.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:24.956912+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.? 172.21.15.181:0/2884086963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:09:25.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:24.957599+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.181:0/2884086963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:09:25.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:24.961264+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.181:0/2884086963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:09:25.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:24.961907+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.? 172.21.15.181:0/2884086963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:25.185332+0000 mon.a (mon.0) 23 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:25.186289+0000 mon.a (mon.0) 24 : cluster [INF] Activating manager daemon y 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:25.198405+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:25.198925+0000 mon.a (mon.0) 26 : cluster [DBG] mgrmap e32: y(active, starting, since 0.0128096s), standbys: x 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:25.200804+0000 osd.7 (osd.7) 10 : cluster [WRN] failed to encode map e303 with expected crc 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.202378+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:25.202379+0000 osd.3 (osd.3) 7 : cluster [WRN] failed to encode map e303 with expected crc 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.202630+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.202913+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.203147+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:09:25.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.203548+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.204444+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.204687+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.206691+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.207070+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.207445+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.207773+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.207973+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:25.208356+0000 osd.4 (osd.4) 12 : cluster [WRN] failed to encode map e304 with expected crc 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.208388+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.208787+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.209131+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.210018+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:09:25.821 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.210427+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:09:25.822 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.211102+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:09:25.822 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: cluster 2023-10-09T09:09:25.246687+0000 mon.a (mon.0) 45 : cluster [INF] Manager daemon y is now available 2023-10-09T09:09:25.822 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.279018+0000 mon.a (mon.0) 46 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:25.822 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.284102+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:25.822 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.325744+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:25.822 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.329291+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:09:25.822 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:25 smithi181 bash[27712]: audit 2023-10-09T09:09:25.358761+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:09:25.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: CherryPy Checker: 2023-10-09T09:09:25.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: The Application mounted at '' has an empty config. 2023-10-09T09:09:25.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: [09/Oct/2023:09:09:25] ENGINE Serving on http://:::9283 2023-10-09T09:09:25.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[71599]: [09/Oct/2023:09:09:25] ENGINE Bus STARTED 2023-10-09T09:09:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:23.950888+0000 osd.6 (osd.6) 4 : cluster [WRN] failed to encode map e302 with expected crc 2023-10-09T09:09:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:23.950920+0000 osd.0 (osd.0) 4 : cluster [WRN] failed to encode map e302 with expected crc 2023-10-09T09:09:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:24.321043+0000 osd.2 (osd.2) 8 : cluster [WRN] failed to encode map e302 with expected crc 2023-10-09T09:09:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:24.947943+0000 osd.2 (osd.2) 9 : cluster [WRN] failed to encode map e303 with expected crc 2023-10-09T09:09:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:24.948486+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2023-10-09T09:09:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:24.950291+0000 osd.4 (osd.4) 11 : cluster [WRN] failed to encode map e303 with expected crc 2023-10-09T09:09:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:24.955093+0000 mon.a (mon.0) 21 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:09:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:24.955178+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:09:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:24.956912+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.? 172.21.15.181:0/2884086963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:24.957599+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.181:0/2884086963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:24.961264+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.181:0/2884086963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:24.961907+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.? 172.21.15.181:0/2884086963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:25.185332+0000 mon.a (mon.0) 23 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:25.186289+0000 mon.a (mon.0) 24 : cluster [INF] Activating manager daemon y 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:25.198405+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:25.198925+0000 mon.a (mon.0) 26 : cluster [DBG] mgrmap e32: y(active, starting, since 0.0128096s), standbys: x 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:25.200804+0000 osd.7 (osd.7) 10 : cluster [WRN] failed to encode map e303 with expected crc 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.202378+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:25.202379+0000 osd.3 (osd.3) 7 : cluster [WRN] failed to encode map e303 with expected crc 2023-10-09T09:09:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.202630+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.202913+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.203147+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.vnodzj"}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.203548+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.ajqgsc"}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.204444+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.204687+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.206691+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.207070+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.207445+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.207773+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.207973+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:25.208356+0000 osd.4 (osd.4) 12 : cluster [WRN] failed to encode map e304 with expected crc 2023-10-09T09:09:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.208388+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.208787+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.209131+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.210018+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.210427+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.211102+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: cluster 2023-10-09T09:09:25.246687+0000 mon.a (mon.0) 45 : cluster [INF] Manager daemon y is now available 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.279018+0000 mon.a (mon.0) 46 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.284102+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.325744+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.329291+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:09:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:25 smithi163 bash[33912]: audit 2023-10-09T09:09:25.358761+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:09:26.224 INFO:tasks.rados.rados.0.smithi163.stdout:956: done (2 left) 2023-10-09T09:09:26.224 INFO:tasks.rados.rados.0.smithi163.stdout:957: write oid 19 current snap is 119 2023-10-09T09:09:26.224 INFO:tasks.rados.rados.0.smithi163.stdout:955: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:09:26.224 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 0 (ObjNum 698858128 snap 21875 seq_num 697217040) dirty dne 2023-10-09T09:09:26.225 INFO:tasks.rados.rados.0.smithi163.stdout:954: finishing write tid 2 to smithi16367185-39 2023-10-09T09:09:26.225 INFO:tasks.rados.rados.0.smithi163.stdout:954: finishing write tid 3 to smithi16367185-39 2023-10-09T09:09:26.225 INFO:tasks.rados.rados.0.smithi163.stdout:957: seq_num 390 ranges {554730=481113,1572095=737454} 2023-10-09T09:09:26.230 INFO:tasks.rados.rados.0.smithi163.stdout:957: writing smithi16367185-19 from 554730 to 1035843 tid 1 2023-10-09T09:09:26.238 INFO:tasks.rados.rados.0.smithi163.stdout:957: writing smithi16367185-19 from 1572095 to 2309549 tid 2 2023-10-09T09:09:26.239 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:09:26.239 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (2m) - 7m 13.5M - 0.23.0 ba2b418f427c 82a8abc84166 2023-10-09T09:09:26.239 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (84s) - 7m 40.7M - 8.3.5 dad864ee21e9 9d80a71ddd1d 2023-10-09T09:09:26.239 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.vnodzj smithi163 running (5m) - 5m 15.7M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 892077b22450 2023-10-09T09:09:26.240 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.ajqgsc smithi181 running (5m) - 5m 12.9M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 01d89554089b 2023-10-09T09:09:26.240 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443,9283,8765 running (34s) - 14m 77.9M - 18.2.0-598-gb4fa663f 3060897fdf97 ca7a91dd3dab 2023-10-09T09:09:26.240 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (79s) - 16m 457M - 18.2.0-598-gb4fa663f 3060897fdf97 fc427389eb0c 2023-10-09T09:09:26.241 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (16m) - 16m 66.0M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 429e9872cbc8 2023-10-09T09:09:26.241 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (14m) - 14m 50.7M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 443a80433a1f 2023-10-09T09:09:26.241 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (14m) - 14m 60.6M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d26d17ef0c6e 2023-10-09T09:09:26.241 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (2m) - 7m 10.7M - 1.3.1 1dbe0e931976 8a6fec29baf3 2023-10-09T09:09:26.241 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (108s) - 7m 8487k - 1.3.1 1dbe0e931976 3f766d093db8 2023-10-09T09:09:26.241 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (13m) - 13m 77.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d07769e94b6f 2023-10-09T09:09:26.242 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (13m) - 13m 64.7M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7a2783c3c11c 2023-10-09T09:09:26.242 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (12m) - 12m 85.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 35670f565f8c 2023-10-09T09:09:26.242 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (11m) - 11m 56.8M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 402aa34f21f4 2023-10-09T09:09:26.242 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (11m) - 11m 80.8M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5dd6bc4b279 2023-10-09T09:09:26.242 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (10m) - 10m 81.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9f3e4fbba907 2023-10-09T09:09:26.242 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (9m) - 9m 60.2M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 30f1ac2f4524 2023-10-09T09:09:26.243 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (8m) - 8m 68.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 54105d827fdd 2023-10-09T09:09:26.243 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (40s) - 7m 40.8M - 2.33.4 514e6a882f6e cc81501e414f 2023-10-09T09:09:26.243 INFO:tasks.rados.rados.0.smithi163.stdout:955: done (2 left) 2023-10-09T09:09:26.243 INFO:tasks.rados.rados.0.smithi163.stdout:958: snap_remove snap 102 2023-10-09T09:09:26.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:26 smithi163 bash[71599]: debug 2023-10-09T09:09:26.207+0000 7f5c688fd700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T09:09:26.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:26 smithi163 bash[33912]: debug 2023-10-09T09:09:26.215+0000 7f84df46a700 -1 mon.c@2(peon).osd e305 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:26.475 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:26 smithi181 bash[27712]: debug 2023-10-09T09:09:26.212+0000 7fde83035700 -1 mon.b@1(peon).osd e305 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:26 smithi163 bash[33912]: cluster 2023-10-09T09:09:25.210078+0000 osd.5 (osd.5) 6 : cluster [WRN] failed to encode map e303 with expected crc 2023-10-09T09:09:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:26 smithi163 bash[33912]: cluster 2023-10-09T09:09:26.210251+0000 mon.a (mon.0) 51 : cluster [DBG] mgrmap e33: y(active, since 1.02413s), standbys: x 2023-10-09T09:09:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:26 smithi163 bash[33912]: cluster 2023-10-09T09:09:26.221020+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2023-10-09T09:09:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:26 smithi163 bash[33912]: cluster 2023-10-09T09:09:26.223906+0000 osd.2 (osd.2) 10 : cluster [WRN] failed to encode map e305 with expected crc 2023-10-09T09:09:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:26 smithi181 bash[27712]: cluster 2023-10-09T09:09:25.210078+0000 osd.5 (osd.5) 6 : cluster [WRN] failed to encode map e303 with expected crc 2023-10-09T09:09:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:26 smithi181 bash[27712]: cluster 2023-10-09T09:09:26.210251+0000 mon.a (mon.0) 51 : cluster [DBG] mgrmap e33: y(active, since 1.02413s), standbys: x 2023-10-09T09:09:26.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:26 smithi181 bash[27712]: cluster 2023-10-09T09:09:26.221020+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2023-10-09T09:09:26.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:26 smithi181 bash[27712]: cluster 2023-10-09T09:09:26.223906+0000 osd.2 (osd.2) 10 : cluster [WRN] failed to encode map e305 with expected crc 2023-10-09T09:09:27.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T09:09:27.235 INFO:tasks.rados.rados.0.smithi163.stdout:958: done (2 left) 2023-10-09T09:09:27.235 INFO:tasks.rados.rados.0.smithi163.stdout:959: rollback oid 9 current snap is 119 2023-10-09T09:09:27.235 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 117 2023-10-09T09:09:27.235 INFO:tasks.rados.rados.0.smithi163.stdout:954: finishing write tid 4 to smithi16367185-39 2023-10-09T09:09:27.235 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 380 (ObjNum 389 snap 119 seq_num 389) dirty exists 2023-10-09T09:09:27.235 INFO:tasks.rados.rados.0.smithi163.stdout:954: left oid 39 (ObjNum 389 snap 119 seq_num 389) 2023-10-09T09:09:27.235 INFO:tasks.rados.rados.0.smithi163.stdout:957: finishing write tid 1 to smithi16367185-19 2023-10-09T09:09:27.236 INFO:tasks.rados.rados.0.smithi163.stdout:957: finishing write tid 2 to smithi16367185-19 2023-10-09T09:09:27.236 INFO:tasks.rados.rados.0.smithi163.stdout:957: finishing write tid 3 to smithi16367185-19 2023-10-09T09:09:27.236 INFO:tasks.rados.rados.0.smithi163.stdout:957: finishing write tid 4 to smithi16367185-19 2023-10-09T09:09:27.236 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 335 (ObjNum 390 snap 119 seq_num 390) dirty exists 2023-10-09T09:09:27.236 INFO:tasks.rados.rados.0.smithi163.stdout:957: left oid 19 (ObjNum 390 snap 119 seq_num 390) 2023-10-09T09:09:27.236 INFO:tasks.rados.rados.0.smithi163.stdout:954: done (2 left) 2023-10-09T09:09:27.236 INFO:tasks.rados.rados.0.smithi163.stdout:957: done (1 left) 2023-10-09T09:09:27.236 INFO:tasks.rados.rados.0.smithi163.stdout:960: write oid 17 current snap is 119 2023-10-09T09:09:27.236 INFO:tasks.rados.rados.0.smithi163.stdout:960: seq_num 391 ranges {571537=556362,1630577=597465} 2023-10-09T09:09:27.245 INFO:tasks.rados.rados.0.smithi163.stdout:960: writing smithi16367185-17 from 571537 to 1127899 tid 1 2023-10-09T09:09:27.254 INFO:tasks.rados.rados.0.smithi163.stdout:960: writing smithi16367185-17 from 1630577 to 2228042 tid 2 2023-10-09T09:09:27.255 INFO:tasks.rados.rados.0.smithi163.stdout:959: finishing rollback tid 0 to smithi16367185-9 2023-10-09T09:09:27.255 INFO:tasks.rados.rados.0.smithi163.stdout:959: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:09:27.255 INFO:tasks.rados.rados.0.smithi163.stdout:959: finishing rollback tid 2 to smithi16367185-9 2023-10-09T09:09:27.255 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 279 (ObjNum 220 snap 61 seq_num 220) dirty exists 2023-10-09T09:09:27.255 INFO:tasks.rados.rados.0.smithi163.stdout:959: done (1 left) 2023-10-09T09:09:27.256 INFO:tasks.rados.rados.0.smithi163.stdout:961: delete oid 27 current snap is 119 2023-10-09T09:09:27.260 INFO:tasks.rados.rados.0.smithi163.stdout:960: finishing write tid 1 to smithi16367185-17 2023-10-09T09:09:27.261 INFO:tasks.rados.rados.0.smithi163.stdout:960: finishing write tid 2 to smithi16367185-17 2023-10-09T09:09:27.261 INFO:tasks.rados.rados.0.smithi163.stdout:960: finishing write tid 3 to smithi16367185-17 2023-10-09T09:09:27.261 INFO:tasks.rados.rados.0.smithi163.stdout:961: done (1 left) 2023-10-09T09:09:27.262 INFO:tasks.rados.rados.0.smithi163.stdout:962: write oid 11 current snap is 119 2023-10-09T09:09:27.262 INFO:tasks.rados.rados.0.smithi163.stdout:962: seq_num 393 ranges {605151=706860,2031188=573963} 2023-10-09T09:09:27.269 INFO:tasks.rados.rados.0.smithi163.stdout:962: writing smithi16367185-11 from 605151 to 1312011 tid 1 2023-10-09T09:09:27.276 INFO:tasks.rados.rados.0.smithi163.stdout:962: writing smithi16367185-11 from 2031188 to 2605151 tid 2 2023-10-09T09:09:27.276 INFO:tasks.rados.rados.0.smithi163.stdout:963: snap_remove snap 113 2023-10-09T09:09:27.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: debug 2023-10-09T09:09:27.223+0000 7f84df46a700 -1 mon.c@2(peon).osd e306 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:27.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: debug 2023-10-09T09:09:27.224+0000 7fde83035700 -1 mon.b@1(peon).osd e306 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:27.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cephadm 2023-10-09T09:09:25.760954+0000 mgr.y (mgr.34107) 1 : cephadm [INF] [09/Oct/2023:09:09:25] ENGINE Bus STARTING 2023-10-09T09:09:27.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cephadm 2023-10-09T09:09:25.906276+0000 mgr.y (mgr.34107) 2 : cephadm [INF] [09/Oct/2023:09:09:25] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:09:27.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cephadm 2023-10-09T09:09:26.008553+0000 mgr.y (mgr.34107) 3 : cephadm [INF] [09/Oct/2023:09:09:26] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:09:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cephadm 2023-10-09T09:09:26.009590+0000 mgr.y (mgr.34107) 4 : cephadm [INF] [09/Oct/2023:09:09:26] ENGINE Bus STARTED 2023-10-09T09:09:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: audit 2023-10-09T09:09:26.213302+0000 mgr.y (mgr.34107) 5 : audit [DBG] from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:09:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cluster 2023-10-09T09:09:26.220554+0000 osd.5 (osd.5) 7 : cluster [WRN] failed to encode map e305 with expected crc 2023-10-09T09:09:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cluster 2023-10-09T09:09:26.220994+0000 mgr.y (mgr.34107) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 158 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 383 GiB/s rd, 88 GiB/s wr, 983.43k op/s 2023-10-09T09:09:27.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cluster 2023-10-09T09:09:27.230981+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2023-10-09T09:09:27.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cluster 2023-10-09T09:09:27.231615+0000 osd.4 (osd.4) 13 : cluster [WRN] failed to encode map e306 with expected crc 2023-10-09T09:09:27.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cluster 2023-10-09T09:09:27.234298+0000 osd.2 (osd.2) 11 : cluster [WRN] failed to encode map e306 with expected crc 2023-10-09T09:09:27.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cluster 2023-10-09T09:09:27.237410+0000 osd.3 (osd.3) 8 : cluster [WRN] failed to encode map e306 with expected crc 2023-10-09T09:09:27.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:27 smithi181 bash[27712]: cluster 2023-10-09T09:09:27.247271+0000 osd.7 (osd.7) 11 : cluster [WRN] failed to encode map e306 with expected crc 2023-10-09T09:09:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cephadm 2023-10-09T09:09:25.760954+0000 mgr.y (mgr.34107) 1 : cephadm [INF] [09/Oct/2023:09:09:25] ENGINE Bus STARTING 2023-10-09T09:09:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cephadm 2023-10-09T09:09:25.906276+0000 mgr.y (mgr.34107) 2 : cephadm [INF] [09/Oct/2023:09:09:25] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:09:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cephadm 2023-10-09T09:09:26.008553+0000 mgr.y (mgr.34107) 3 : cephadm [INF] [09/Oct/2023:09:09:26] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:09:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cephadm 2023-10-09T09:09:26.009590+0000 mgr.y (mgr.34107) 4 : cephadm [INF] [09/Oct/2023:09:09:26] ENGINE Bus STARTED 2023-10-09T09:09:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: audit 2023-10-09T09:09:26.213302+0000 mgr.y (mgr.34107) 5 : audit [DBG] from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:09:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cluster 2023-10-09T09:09:26.220554+0000 osd.5 (osd.5) 7 : cluster [WRN] failed to encode map e305 with expected crc 2023-10-09T09:09:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cluster 2023-10-09T09:09:26.220994+0000 mgr.y (mgr.34107) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 158 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 383 GiB/s rd, 88 GiB/s wr, 983.43k op/s 2023-10-09T09:09:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cluster 2023-10-09T09:09:27.230981+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2023-10-09T09:09:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cluster 2023-10-09T09:09:27.231615+0000 osd.4 (osd.4) 13 : cluster [WRN] failed to encode map e306 with expected crc 2023-10-09T09:09:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cluster 2023-10-09T09:09:27.234298+0000 osd.2 (osd.2) 11 : cluster [WRN] failed to encode map e306 with expected crc 2023-10-09T09:09:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cluster 2023-10-09T09:09:27.237410+0000 osd.3 (osd.3) 8 : cluster [WRN] failed to encode map e306 with expected crc 2023-10-09T09:09:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:27 smithi163 bash[33912]: cluster 2023-10-09T09:09:27.247271+0000 osd.7 (osd.7) 11 : cluster [WRN] failed to encode map e306 with expected crc 2023-10-09T09:09:28.241 INFO:tasks.rados.rados.0.smithi163.stdout:963: done (2 left) 2023-10-09T09:09:28.241 INFO:tasks.rados.rados.0.smithi163.stdout:964: snap_create 2023-10-09T09:09:28.241 INFO:tasks.rados.rados.0.smithi163.stdout:960: finishing write tid 4 to smithi16367185-17 2023-10-09T09:09:28.241 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 545 (ObjNum 391 snap 119 seq_num 391) dirty exists 2023-10-09T09:09:28.242 INFO:tasks.rados.rados.0.smithi163.stdout:960: left oid 17 (ObjNum 391 snap 119 seq_num 391) 2023-10-09T09:09:28.242 INFO:tasks.rados.rados.0.smithi163.stdout:962: finishing write tid 1 to smithi16367185-11 2023-10-09T09:09:28.242 INFO:tasks.rados.rados.0.smithi163.stdout:962: finishing write tid 2 to smithi16367185-11 2023-10-09T09:09:28.242 INFO:tasks.rados.rados.0.smithi163.stdout:962: finishing write tid 3 to smithi16367185-11 2023-10-09T09:09:28.242 INFO:tasks.rados.rados.0.smithi163.stdout:962: finishing write tid 4 to smithi16367185-11 2023-10-09T09:09:28.242 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 299 (ObjNum 393 snap 119 seq_num 393) dirty exists 2023-10-09T09:09:28.242 INFO:tasks.rados.rados.0.smithi163.stdout:962: left oid 11 (ObjNum 393 snap 119 seq_num 393) 2023-10-09T09:09:28.481 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:28 smithi181 bash[27712]: debug 2023-10-09T09:09:28.224+0000 7fde83035700 -1 mon.b@1(peon).osd e307 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:28.481 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:28 smithi181 bash[27712]: cluster 2023-10-09T09:09:27.207656+0000 mgr.y (mgr.34107) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 158 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.0 GiB/s rd, 237 MiB/s wr, 2.59k op/s 2023-10-09T09:09:28.481 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:28 smithi181 bash[27712]: cluster 2023-10-09T09:09:27.490249+0000 mon.a (mon.0) 54 : cluster [DBG] mgrmap e34: y(active, since 2s), standbys: x 2023-10-09T09:09:28.481 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:28 smithi181 bash[27712]: cluster 2023-10-09T09:09:28.234697+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2023-10-09T09:09:28.482 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:28 smithi163 bash[33912]: debug 2023-10-09T09:09:28.222+0000 7f84df46a700 -1 mon.c@2(peon).osd e307 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:28 smithi181 bash[27712]: cluster 2023-10-09T09:09:28.237609+0000 osd.4 (osd.4) 14 : cluster [WRN] failed to encode map e307 with expected crc 2023-10-09T09:09:28.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:28 smithi163 bash[33912]: cluster 2023-10-09T09:09:27.207656+0000 mgr.y (mgr.34107) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 158 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.0 GiB/s rd, 237 MiB/s wr, 2.59k op/s 2023-10-09T09:09:28.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:28 smithi163 bash[33912]: cluster 2023-10-09T09:09:27.490249+0000 mon.a (mon.0) 54 : cluster [DBG] mgrmap e34: y(active, since 2s), standbys: x 2023-10-09T09:09:28.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:28 smithi163 bash[33912]: cluster 2023-10-09T09:09:28.234697+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2023-10-09T09:09:28.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:28 smithi163 bash[33912]: cluster 2023-10-09T09:09:28.237609+0000 osd.4 (osd.4) 14 : cluster [WRN] failed to encode map e307 with expected crc 2023-10-09T09:09:29.243 INFO:tasks.rados.rados.0.smithi163.stdout:960: done (2 left) 2023-10-09T09:09:29.243 INFO:tasks.rados.rados.0.smithi163.stdout:962: done (1 left) 2023-10-09T09:09:29.243 INFO:tasks.rados.rados.0.smithi163.stdout:964: done (0 left) 2023-10-09T09:09:29.243 INFO:tasks.rados.rados.0.smithi163.stdout:965: snap_create 2023-10-09T09:09:29.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:29 smithi163 bash[33912]: debug 2023-10-09T09:09:29.226+0000 7f84df46a700 -1 mon.c@2(peon).osd e308 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:29.488 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:29 smithi181 bash[27712]: debug 2023-10-09T09:09:29.228+0000 7fde83035700 -1 mon.b@1(peon).osd e308 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:29 smithi181 bash[27712]: cluster 2023-10-09T09:09:28.238613+0000 osd.0 (osd.0) 5 : cluster [WRN] failed to encode map e307 with expected crc 2023-10-09T09:09:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:29 smithi181 bash[27712]: cluster 2023-10-09T09:09:28.416114+0000 osd.7 (osd.7) 12 : cluster [WRN] failed to encode map e307 with expected crc 2023-10-09T09:09:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:29 smithi181 bash[27712]: cluster 2023-10-09T09:09:29.208234+0000 mgr.y (mgr.34107) 8 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 158 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 342 MiB/s rd, 78 MiB/s wr, 857 op/s 2023-10-09T09:09:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:29 smithi181 bash[27712]: cluster 2023-10-09T09:09:29.236551+0000 mon.a (mon.0) 56 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2023-10-09T09:09:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:29 smithi181 bash[27712]: cluster 2023-10-09T09:09:29.239567+0000 osd.4 (osd.4) 15 : cluster [WRN] failed to encode map e308 with expected crc 2023-10-09T09:09:29.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:29 smithi163 bash[33912]: cluster 2023-10-09T09:09:28.238613+0000 osd.0 (osd.0) 5 : cluster [WRN] failed to encode map e307 with expected crc 2023-10-09T09:09:29.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:29 smithi163 bash[33912]: cluster 2023-10-09T09:09:28.416114+0000 osd.7 (osd.7) 12 : cluster [WRN] failed to encode map e307 with expected crc 2023-10-09T09:09:29.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:29 smithi163 bash[33912]: cluster 2023-10-09T09:09:29.208234+0000 mgr.y (mgr.34107) 8 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 158 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 342 MiB/s rd, 78 MiB/s wr, 857 op/s 2023-10-09T09:09:29.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:29 smithi163 bash[33912]: cluster 2023-10-09T09:09:29.236551+0000 mon.a (mon.0) 56 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2023-10-09T09:09:29.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:29 smithi163 bash[33912]: cluster 2023-10-09T09:09:29.239567+0000 osd.4 (osd.4) 15 : cluster [WRN] failed to encode map e308 with expected crc 2023-10-09T09:09:29.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:29 smithi163 bash[33912]: debug 2023-10-09T09:09:29.914+0000 7f84df46a700 -1 mon.c@2(peon).osd e309 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:29.932 INFO:tasks.rados.rados.0.smithi163.stdout:965: done (0 left) 2023-10-09T09:09:29.932 INFO:tasks.rados.rados.0.smithi163.stdout:966: write oid 31 current snap is 121 2023-10-09T09:09:29.932 INFO:tasks.rados.rados.0.smithi163.stdout:966: seq_num 394 ranges {621958=782109,2089670=532288} 2023-10-09T09:09:29.940 INFO:tasks.rados.rados.0.smithi163.stdout:966: writing smithi16367185-31 from 621958 to 1404067 tid 1 2023-10-09T09:09:29.948 INFO:tasks.rados.rados.0.smithi163.stdout:966: writing smithi16367185-31 from 2089670 to 2621958 tid 2 2023-10-09T09:09:29.948 INFO:tasks.rados.rados.0.smithi163.stdout:967: write oid 7 current snap is 121 2023-10-09T09:09:29.949 INFO:tasks.rados.rados.0.smithi163.stdout:967: seq_num 395 ranges {638765=457358,1748152=604803} 2023-10-09T09:09:29.955 INFO:tasks.rados.rados.0.smithi163.stdout:967: writing smithi16367185-7 from 638765 to 1096123 tid 1 2023-10-09T09:09:29.963 INFO:tasks.rados.rados.0.smithi163.stdout:967: writing smithi16367185-7 from 1748152 to 2352955 tid 2 2023-10-09T09:09:29.963 INFO:tasks.rados.rados.0.smithi163.stdout:968: write oid 45 current snap is 121 2023-10-09T09:09:29.963 INFO:tasks.rados.rados.0.smithi163.stdout:966: finishing write tid 1 to smithi16367185-31 2023-10-09T09:09:29.963 INFO:tasks.rados.rados.0.smithi163.stdout:966: finishing write tid 2 to smithi16367185-31 2023-10-09T09:09:29.963 INFO:tasks.rados.rados.0.smithi163.stdout:966: finishing write tid 3 to smithi16367185-31 2023-10-09T09:09:29.963 INFO:tasks.rados.rados.0.smithi163.stdout:966: finishing write tid 4 to smithi16367185-31 2023-10-09T09:09:29.963 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 307 (ObjNum 394 snap 121 seq_num 394) dirty exists 2023-10-09T09:09:29.963 INFO:tasks.rados.rados.0.smithi163.stdout:966: left oid 31 (ObjNum 394 snap 121 seq_num 394) 2023-10-09T09:09:29.963 INFO:tasks.rados.rados.0.smithi163.stdout:967: finishing write tid 1 to smithi16367185-7 2023-10-09T09:09:29.964 INFO:tasks.rados.rados.0.smithi163.stdout:968: seq_num 396 ranges {655572=648960,1922987=732585} 2023-10-09T09:09:29.970 INFO:tasks.rados.rados.0.smithi163.stdout:968: writing smithi16367185-45 from 655572 to 1304532 tid 1 2023-10-09T09:09:29.979 INFO:tasks.rados.rados.0.smithi163.stdout:968: writing smithi16367185-45 from 1922987 to 2655572 tid 2 2023-10-09T09:09:29.979 INFO:tasks.rados.rados.0.smithi163.stdout:966: done (2 left) 2023-10-09T09:09:29.979 INFO:tasks.rados.rados.0.smithi163.stdout:969: read oid 34 snap -1 2023-10-09T09:09:29.979 INFO:tasks.rados.rados.0.smithi163.stdout:969: expect (ObjNum 358 snap 108 seq_num 358) 2023-10-09T09:09:29.979 INFO:tasks.rados.rados.0.smithi163.stdout:970: write oid 36 current snap is 121 2023-10-09T09:09:29.979 INFO:tasks.rados.rados.0.smithi163.stdout:967: finishing write tid 2 to smithi16367185-7 2023-10-09T09:09:29.979 INFO:tasks.rados.rados.0.smithi163.stdout:967: finishing write tid 3 to smithi16367185-7 2023-10-09T09:09:29.979 INFO:tasks.rados.rados.0.smithi163.stdout:970: seq_num 397 ranges {672379=724209,1865116=608472} 2023-10-09T09:09:29.986 INFO:tasks.rados.rados.0.smithi163.stdout:970: writing smithi16367185-36 from 672379 to 1396588 tid 1 2023-10-09T09:09:29.993 INFO:tasks.rados.rados.0.smithi163.stdout:970: writing smithi16367185-36 from 1865116 to 2473588 tid 2 2023-10-09T09:09:29.994 INFO:tasks.rados.rados.0.smithi163.stdout:971: read oid 24 snap -1 2023-10-09T09:09:29.994 INFO:tasks.rados.rados.0.smithi163.stdout:971: expect (ObjNum 327 snap 99 seq_num 327) 2023-10-09T09:09:29.994 INFO:tasks.rados.rados.0.smithi163.stdout:972: read oid 23 snap -1 2023-10-09T09:09:29.994 INFO:tasks.rados.rados.0.smithi163.stdout:972: expect deleted 2023-10-09T09:09:29.994 INFO:tasks.rados.rados.0.smithi163.stdout:973: rollback oid 3 current snap is 121 2023-10-09T09:09:29.994 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 117 2023-10-09T09:09:29.994 INFO:tasks.rados.rados.0.smithi163.stdout:967: finishing write tid 4 to smithi16367185-7 2023-10-09T09:09:29.994 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 265 (ObjNum 395 snap 121 seq_num 395) dirty exists 2023-10-09T09:09:29.994 INFO:tasks.rados.rados.0.smithi163.stdout:967: left oid 7 (ObjNum 395 snap 121 seq_num 395) 2023-10-09T09:09:29.994 INFO:tasks.rados.rados.0.smithi163.stdout:968: finishing write tid 1 to smithi16367185-45 2023-10-09T09:09:29.995 INFO:tasks.rados.rados.0.smithi163.stdout:967: done (6 left) 2023-10-09T09:09:29.995 INFO:tasks.rados.rados.0.smithi163.stdout:974: delete oid 40 current snap is 121 2023-10-09T09:09:29.995 INFO:tasks.rados.rados.0.smithi163.stdout:968: finishing write tid 2 to smithi16367185-45 2023-10-09T09:09:29.995 INFO:tasks.rados.rados.0.smithi163.stdout:969: expect (ObjNum 358 snap 108 seq_num 358) 2023-10-09T09:09:30.135 INFO:tasks.rados.rados.0.smithi163.stdout:968: finishing write tid 3 to smithi16367185-45 2023-10-09T09:09:30.135 INFO:tasks.rados.rados.0.smithi163.stdout:968: finishing write tid 4 to smithi16367185-45 2023-10-09T09:09:30.135 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 844 (ObjNum 396 snap 121 seq_num 396) dirty exists 2023-10-09T09:09:30.135 INFO:tasks.rados.rados.0.smithi163.stdout:968: left oid 45 (ObjNum 396 snap 121 seq_num 396) 2023-10-09T09:09:30.135 INFO:tasks.rados.rados.0.smithi163.stdout:968: done (6 left) 2023-10-09T09:09:30.135 INFO:tasks.rados.rados.0.smithi163.stdout:969: done (5 left) 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:974: done (4 left) 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:975: read oid 27 snap -1 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:975: expect deleted 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:976: rollback oid 11 current snap is 121 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 11 to 117 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:970: finishing write tid 1 to smithi16367185-36 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:973: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 0 (ObjNum 669367496 snap 21875 seq_num 3) dirty dne 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:970: finishing write tid 2 to smithi16367185-36 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:970: finishing write tid 3 to smithi16367185-36 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:970: finishing write tid 4 to smithi16367185-36 2023-10-09T09:09:30.136 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 310 (ObjNum 397 snap 121 seq_num 397) dirty exists 2023-10-09T09:09:30.137 INFO:tasks.rados.rados.0.smithi163.stdout:970: left oid 36 (ObjNum 397 snap 121 seq_num 397) 2023-10-09T09:09:30.137 INFO:tasks.rados.rados.0.smithi163.stdout:971: expect (ObjNum 327 snap 99 seq_num 327) 2023-10-09T09:09:30.298 INFO:tasks.rados.rados.0.smithi163.stdout:976: finishing rollback tid 0 to smithi16367185-11 2023-10-09T09:09:30.298 INFO:tasks.rados.rados.0.smithi163.stdout:976: finishing rollback tid 1 to smithi16367185-11 2023-10-09T09:09:30.298 INFO:tasks.rados.rados.0.smithi163.stdout:976: finishing rollback tid 2 to smithi16367185-11 2023-10-09T09:09:30.298 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 307 (ObjNum 360 snap 108 seq_num 360) dirty exists 2023-10-09T09:09:30.298 INFO:tasks.rados.rados.0.smithi163.stdout:970: done (5 left) 2023-10-09T09:09:30.298 INFO:tasks.rados.rados.0.smithi163.stdout:971: done (4 left) 2023-10-09T09:09:30.298 INFO:tasks.rados.rados.0.smithi163.stdout:972: done (3 left) 2023-10-09T09:09:30.299 INFO:tasks.rados.rados.0.smithi163.stdout:973: done (2 left) 2023-10-09T09:09:30.299 INFO:tasks.rados.rados.0.smithi163.stdout:975: done (1 left) 2023-10-09T09:09:30.299 INFO:tasks.rados.rados.0.smithi163.stdout:976: done (0 left) 2023-10-09T09:09:30.299 INFO:tasks.rados.rados.0.smithi163.stdout:977: snap_remove snap 120 2023-10-09T09:09:30.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:29 smithi181 bash[27712]: debug 2023-10-09T09:09:29.916+0000 7fde83035700 -1 mon.b@1(peon).osd e309 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:30.461 INFO:teuthology.orchestra.run.smithi163.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:09:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:30 smithi181 bash[27712]: cluster 2023-10-09T09:09:29.237343+0000 osd.0 (osd.0) 6 : cluster [WRN] failed to encode map e308 with expected crc 2023-10-09T09:09:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:30 smithi181 bash[27712]: cluster 2023-10-09T09:09:29.926477+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2023-10-09T09:09:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:30 smithi181 bash[27712]: cluster 2023-10-09T09:09:29.926855+0000 osd.2 (osd.2) 12 : cluster [WRN] failed to encode map e309 with expected crc 2023-10-09T09:09:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:30 smithi181 bash[27712]: cluster 2023-10-09T09:09:29.949581+0000 osd.3 (osd.3) 9 : cluster [WRN] failed to encode map e309 with expected crc 2023-10-09T09:09:30.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:30 smithi181 bash[27712]: cluster 2023-10-09T09:09:29.994122+0000 osd.7 (osd.7) 13 : cluster [WRN] failed to encode map e309 with expected crc 2023-10-09T09:09:30.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:30 smithi181 bash[27712]: audit 2023-10-09T09:09:30.452158+0000 mgr.y (mgr.34107) 9 : audit [DBG] from='client.25472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:09:30.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:30 smithi181 bash[27712]: cephadm 2023-10-09T09:09:30.452875+0000 mgr.y (mgr.34107) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:09:30.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:30 smithi181 bash[27712]: audit 2023-10-09T09:09:30.459771+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:30.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:30 smithi163 bash[33912]: cluster 2023-10-09T09:09:29.237343+0000 osd.0 (osd.0) 6 : cluster [WRN] failed to encode map e308 with expected crc 2023-10-09T09:09:30.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:30 smithi163 bash[33912]: cluster 2023-10-09T09:09:29.926477+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2023-10-09T09:09:30.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:30 smithi163 bash[33912]: cluster 2023-10-09T09:09:29.926855+0000 osd.2 (osd.2) 12 : cluster [WRN] failed to encode map e309 with expected crc 2023-10-09T09:09:30.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:30 smithi163 bash[33912]: cluster 2023-10-09T09:09:29.949581+0000 osd.3 (osd.3) 9 : cluster [WRN] failed to encode map e309 with expected crc 2023-10-09T09:09:30.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:30 smithi163 bash[33912]: cluster 2023-10-09T09:09:29.994122+0000 osd.7 (osd.7) 13 : cluster [WRN] failed to encode map e309 with expected crc 2023-10-09T09:09:30.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:30 smithi163 bash[33912]: audit 2023-10-09T09:09:30.452158+0000 mgr.y (mgr.34107) 9 : audit [DBG] from='client.25472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:09:30.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:30 smithi163 bash[33912]: cephadm 2023-10-09T09:09:30.452875+0000 mgr.y (mgr.34107) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:09:30.940 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:30 smithi163 bash[33912]: audit 2023-10-09T09:09:30.459771+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:30.940 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:30 smithi163 bash[33912]: debug 2023-10-09T09:09:30.934+0000 7f84df46a700 -1 mon.c@2(peon).osd e310 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:30.947 INFO:tasks.rados.rados.0.smithi163.stdout:977: done (0 left) 2023-10-09T09:09:30.947 INFO:tasks.rados.rados.0.smithi163.stdout:978: snap_remove snap 117 2023-10-09T09:09:31.139 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T09:09:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:30 smithi181 bash[27712]: debug 2023-10-09T09:09:30.932+0000 7fde83035700 -1 mon.b@1(peon).osd e310 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:31 smithi181 bash[27712]: cluster 2023-10-09T09:09:29.928760+0000 osd.5 (osd.5) 8 : cluster [WRN] failed to encode map e309 with expected crc 2023-10-09T09:09:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:31 smithi181 bash[27712]: cluster 2023-10-09T09:09:29.943363+0000 osd.0 (osd.0) 7 : cluster [WRN] failed to encode map e309 with expected crc 2023-10-09T09:09:31.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:31 smithi181 bash[27712]: cluster 2023-10-09T09:09:30.943136+0000 mon.a (mon.0) 59 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2023-10-09T09:09:31.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:31 smithi181 bash[27712]: cluster 2023-10-09T09:09:30.943623+0000 osd.6 (osd.6) 5 : cluster [WRN] failed to encode map e310 with expected crc 2023-10-09T09:09:31.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:31 smithi181 bash[27712]: cluster 2023-10-09T09:09:31.208759+0000 mgr.y (mgr.34107) 11 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 158 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:09:31.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:31 smithi163 bash[33912]: cluster 2023-10-09T09:09:29.928760+0000 osd.5 (osd.5) 8 : cluster [WRN] failed to encode map e309 with expected crc 2023-10-09T09:09:31.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:31 smithi163 bash[33912]: cluster 2023-10-09T09:09:29.943363+0000 osd.0 (osd.0) 7 : cluster [WRN] failed to encode map e309 with expected crc 2023-10-09T09:09:31.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:31 smithi163 bash[33912]: cluster 2023-10-09T09:09:30.943136+0000 mon.a (mon.0) 59 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2023-10-09T09:09:31.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:31 smithi163 bash[33912]: cluster 2023-10-09T09:09:30.943623+0000 osd.6 (osd.6) 5 : cluster [WRN] failed to encode map e310 with expected crc 2023-10-09T09:09:31.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:31 smithi163 bash[33912]: cluster 2023-10-09T09:09:31.208759+0000 mgr.y (mgr.34107) 11 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 158 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:09:31.954 INFO:tasks.rados.rados.0.smithi163.stdout:978: done (0 left) 2023-10-09T09:09:31.954 INFO:tasks.rados.rados.0.smithi163.stdout:979: write oid 15 current snap is 121 2023-10-09T09:09:31.954 INFO:tasks.rados.rados.0.smithi163.stdout:979: seq_num 399 ranges {705993=474707,1582080=612141} 2023-10-09T09:09:31.961 INFO:tasks.rados.rados.0.smithi163.stdout:979: writing smithi16367185-15 from 705993 to 1180700 tid 1 2023-10-09T09:09:31.968 INFO:tasks.rados.rados.0.smithi163.stdout:979: writing smithi16367185-15 from 1582080 to 2194221 tid 2 2023-10-09T09:09:31.968 INFO:tasks.rados.rados.0.smithi163.stdout:980: snap_remove snap 119 2023-10-09T09:09:32.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:31 smithi163 bash[33912]: debug 2023-10-09T09:09:31.938+0000 7f84df46a700 -1 mon.c@2(peon).osd e311 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:31 smithi181 bash[27712]: debug 2023-10-09T09:09:31.940+0000 7fde83035700 -1 mon.b@1(peon).osd e311 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:32.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:32 smithi163 bash[33912]: cluster 2023-10-09T09:09:30.946205+0000 osd.0 (osd.0) 8 : cluster [WRN] failed to encode map e310 with expected crc 2023-10-09T09:09:32.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:32 smithi163 bash[33912]: cluster 2023-10-09T09:09:31.382878+0000 osd.7 (osd.7) 14 : cluster [WRN] failed to encode map e310 with expected crc 2023-10-09T09:09:32.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:32 smithi163 bash[33912]: cluster 2023-10-09T09:09:31.429732+0000 osd.2 (osd.2) 13 : cluster [WRN] failed to encode map e310 with expected crc 2023-10-09T09:09:32.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:32 smithi163 bash[33912]: cluster 2023-10-09T09:09:31.899351+0000 osd.3 (osd.3) 10 : cluster [WRN] failed to encode map e310 with expected crc 2023-10-09T09:09:32.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:32 smithi163 bash[33912]: cluster 2023-10-09T09:09:31.949723+0000 osd.4 (osd.4) 16 : cluster [WRN] failed to encode map e311 with expected crc 2023-10-09T09:09:32.650 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:32 smithi163 bash[33912]: cluster 2023-10-09T09:09:31.951768+0000 mon.a (mon.0) 60 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2023-10-09T09:09:32.650 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:32 smithi163 bash[33912]: cluster 2023-10-09T09:09:31.963139+0000 osd.7 (osd.7) 15 : cluster [WRN] failed to encode map e311 with expected crc 2023-10-09T09:09:32.650 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:32 smithi163 bash[33912]: audit 2023-10-09T09:09:32.018227+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:32.650 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:32 smithi163 bash[33912]: audit 2023-10-09T09:09:32.025030+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:32 smithi181 bash[27712]: cluster 2023-10-09T09:09:30.946205+0000 osd.0 (osd.0) 8 : cluster [WRN] failed to encode map e310 with expected crc 2023-10-09T09:09:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:32 smithi181 bash[27712]: cluster 2023-10-09T09:09:31.382878+0000 osd.7 (osd.7) 14 : cluster [WRN] failed to encode map e310 with expected crc 2023-10-09T09:09:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:32 smithi181 bash[27712]: cluster 2023-10-09T09:09:31.429732+0000 osd.2 (osd.2) 13 : cluster [WRN] failed to encode map e310 with expected crc 2023-10-09T09:09:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:32 smithi181 bash[27712]: cluster 2023-10-09T09:09:31.899351+0000 osd.3 (osd.3) 10 : cluster [WRN] failed to encode map e310 with expected crc 2023-10-09T09:09:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:32 smithi181 bash[27712]: cluster 2023-10-09T09:09:31.949723+0000 osd.4 (osd.4) 16 : cluster [WRN] failed to encode map e311 with expected crc 2023-10-09T09:09:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:32 smithi181 bash[27712]: cluster 2023-10-09T09:09:31.951768+0000 mon.a (mon.0) 60 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2023-10-09T09:09:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:32 smithi181 bash[27712]: cluster 2023-10-09T09:09:31.963139+0000 osd.7 (osd.7) 15 : cluster [WRN] failed to encode map e311 with expected crc 2023-10-09T09:09:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:32 smithi181 bash[27712]: audit 2023-10-09T09:09:32.018227+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:32 smithi181 bash[27712]: audit 2023-10-09T09:09:32.025030+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:32.959 INFO:tasks.rados.rados.0.smithi163.stdout:980: done (1 left) 2023-10-09T09:09:32.959 INFO:tasks.rados.rados.0.smithi163.stdout:981: snap_remove snap 118 2023-10-09T09:09:33.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:32 smithi163 bash[33912]: debug 2023-10-09T09:09:32.942+0000 7f84df46a700 -1 mon.c@2(peon).osd e312 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:33.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:32 smithi181 bash[27712]: debug 2023-10-09T09:09:32.944+0000 7fde83035700 -1 mon.b@1(peon).osd e312 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:33.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:09:33] "GET /metrics HTTP/1.1" 200 33359 "" "Prometheus/2.33.4" 2023-10-09T09:09:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: cluster 2023-10-09T09:09:31.950345+0000 osd.0 (osd.0) 9 : cluster [WRN] failed to encode map e311 with expected crc 2023-10-09T09:09:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: audit 2023-10-09T09:09:32.632247+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: audit 2023-10-09T09:09:32.640207+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: audit 2023-10-09T09:09:32.812706+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: audit 2023-10-09T09:09:32.818924+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: cluster 2023-10-09T09:09:32.951234+0000 mon.a (mon.0) 67 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2023-10-09T09:09:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: cluster 2023-10-09T09:09:32.956021+0000 osd.4 (osd.4) 17 : cluster [WRN] failed to encode map e312 with expected crc 2023-10-09T09:09:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: cluster 2023-10-09T09:09:32.957342+0000 osd.2 (osd.2) 14 : cluster [WRN] failed to encode map e312 with expected crc 2023-10-09T09:09:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: cluster 2023-10-09T09:09:33.209418+0000 mgr.y (mgr.34107) 12 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 115 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:09:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: audit 2023-10-09T09:09:33.438210+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: audit 2023-10-09T09:09:33.445344+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: cluster 2023-10-09T09:09:31.950345+0000 osd.0 (osd.0) 9 : cluster [WRN] failed to encode map e311 with expected crc 2023-10-09T09:09:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: audit 2023-10-09T09:09:32.632247+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: audit 2023-10-09T09:09:32.640207+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: audit 2023-10-09T09:09:32.812706+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: audit 2023-10-09T09:09:32.818924+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: cluster 2023-10-09T09:09:32.951234+0000 mon.a (mon.0) 67 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2023-10-09T09:09:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: cluster 2023-10-09T09:09:32.956021+0000 osd.4 (osd.4) 17 : cluster [WRN] failed to encode map e312 with expected crc 2023-10-09T09:09:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: cluster 2023-10-09T09:09:32.957342+0000 osd.2 (osd.2) 14 : cluster [WRN] failed to encode map e312 with expected crc 2023-10-09T09:09:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: cluster 2023-10-09T09:09:33.209418+0000 mgr.y (mgr.34107) 12 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 115 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:09:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: audit 2023-10-09T09:09:33.438210+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: audit 2023-10-09T09:09:33.445344+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:33.964 INFO:tasks.rados.rados.0.smithi163.stdout:981: done (1 left) 2023-10-09T09:09:33.964 INFO:tasks.rados.rados.0.smithi163.stdout:982: delete oid 35 current snap is 121 2023-10-09T09:09:33.964 INFO:tasks.rados.rados.0.smithi163.stdout:979: finishing write tid 1 to smithi16367185-15 2023-10-09T09:09:33.964 INFO:tasks.rados.rados.0.smithi163.stdout:979: finishing write tid 2 to smithi16367185-15 2023-10-09T09:09:33.964 INFO:tasks.rados.rados.0.smithi163.stdout:979: finishing write tid 3 to smithi16367185-15 2023-10-09T09:09:33.964 INFO:tasks.rados.rados.0.smithi163.stdout:979: finishing write tid 4 to smithi16367185-15 2023-10-09T09:09:33.964 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 554 (ObjNum 399 snap 121 seq_num 399) dirty exists 2023-10-09T09:09:33.964 INFO:tasks.rados.rados.0.smithi163.stdout:979: left oid 15 (ObjNum 399 snap 121 seq_num 399) 2023-10-09T09:09:33.976 INFO:tasks.rados.rados.0.smithi163.stdout:979: done (1 left) 2023-10-09T09:09:33.976 INFO:tasks.rados.rados.0.smithi163.stdout:982: done (0 left) 2023-10-09T09:09:33.976 INFO:tasks.rados.rados.0.smithi163.stdout:983: read oid 32 snap -1 2023-10-09T09:09:33.977 INFO:tasks.rados.rados.0.smithi163.stdout:983: expect deleted 2023-10-09T09:09:33.977 INFO:tasks.rados.rados.0.smithi163.stdout:984: read oid 37 snap -1 2023-10-09T09:09:33.977 INFO:tasks.rados.rados.0.smithi163.stdout:984: expect deleted 2023-10-09T09:09:33.977 INFO:tasks.rados.rados.0.smithi163.stdout:985: rollback oid 44 current snap is 121 2023-10-09T09:09:33.977 INFO:tasks.rados.rados.0.smithi163.stdout:985: done (2 left) 2023-10-09T09:09:33.977 INFO:tasks.rados.rados.0.smithi163.stdout:986: rollback oid 1 current snap is 121 2023-10-09T09:09:33.977 INFO:tasks.rados.rados.0.smithi163.stdout:986: done (2 left) 2023-10-09T09:09:33.977 INFO:tasks.rados.rados.0.smithi163.stdout:987: delete oid 9 current snap is 121 2023-10-09T09:09:33.982 INFO:tasks.rados.rados.0.smithi163.stdout:983: done (2 left) 2023-10-09T09:09:33.982 INFO:tasks.rados.rados.0.smithi163.stdout:984: done (1 left) 2023-10-09T09:09:33.982 INFO:tasks.rados.rados.0.smithi163.stdout:987: done (0 left) 2023-10-09T09:09:33.983 INFO:tasks.rados.rados.0.smithi163.stdout:988: write oid 24 current snap is 121 2023-10-09T09:09:33.983 INFO:tasks.rados.rados.0.smithi163.stdout:988: seq_num 402 ranges {756414=700454,2041173=715241} 2023-10-09T09:09:33.993 INFO:tasks.rados.rados.0.smithi163.stdout:988: writing smithi16367185-24 from 756414 to 1456868 tid 1 2023-10-09T09:09:34.002 INFO:tasks.rados.rados.0.smithi163.stdout:988: writing smithi16367185-24 from 2041173 to 2756414 tid 2 2023-10-09T09:09:34.002 INFO:tasks.rados.rados.0.smithi163.stdout:989: read oid 7 snap -1 2023-10-09T09:09:34.002 INFO:tasks.rados.rados.0.smithi163.stdout:989: expect (ObjNum 395 snap 121 seq_num 395) 2023-10-09T09:09:34.002 INFO:tasks.rados.rados.0.smithi163.stdout:990: read oid 28 snap -1 2023-10-09T09:09:34.002 INFO:tasks.rados.rados.0.smithi163.stdout:990: expect (ObjNum 295 snap 91 seq_num 295) 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:991: rollback oid 9 current snap is 121 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:991: done (3 left) 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:992: rollback oid 29 current snap is 121 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:992: done (3 left) 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:993: read oid 10 snap -1 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:993: expect (ObjNum 292 snap 91 seq_num 292) 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:994: rollback oid 31 current snap is 121 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:994: done (4 left) 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:995: read oid 23 snap -1 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:995: expect deleted 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:996: read oid 13 snap -1 2023-10-09T09:09:34.003 INFO:tasks.rados.rados.0.smithi163.stdout:996: expect deleted 2023-10-09T09:09:34.004 INFO:tasks.rados.rados.0.smithi163.stdout:997: rollback oid 42 current snap is 121 2023-10-09T09:09:34.004 INFO:tasks.rados.rados.0.smithi163.stdout:997: done (6 left) 2023-10-09T09:09:34.004 INFO:tasks.rados.rados.0.smithi163.stdout:998: snap_create 2023-10-09T09:09:34.004 INFO:tasks.rados.rados.0.smithi163.stdout:988: finishing write tid 1 to smithi16367185-24 2023-10-09T09:09:34.009 INFO:tasks.rados.rados.0.smithi163.stdout:989: expect (ObjNum 395 snap 121 seq_num 395) 2023-10-09T09:09:34.103 INFO:tasks.rados.rados.0.smithi163.stdout:988: finishing write tid 2 to smithi16367185-24 2023-10-09T09:09:34.103 INFO:tasks.rados.rados.0.smithi163.stdout:990: expect (ObjNum 295 snap 91 seq_num 295) 2023-10-09T09:09:34.199 INFO:tasks.rados.rados.0.smithi163.stdout:988: finishing write tid 3 to smithi16367185-24 2023-10-09T09:09:34.199 INFO:tasks.rados.rados.0.smithi163.stdout:988: finishing write tid 4 to smithi16367185-24 2023-10-09T09:09:34.199 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 297 (ObjNum 402 snap 121 seq_num 402) dirty exists 2023-10-09T09:09:34.199 INFO:tasks.rados.rados.0.smithi163.stdout:988: left oid 24 (ObjNum 402 snap 121 seq_num 402) 2023-10-09T09:09:34.199 INFO:tasks.rados.rados.0.smithi163.stdout:993: expect (ObjNum 292 snap 91 seq_num 292) 2023-10-09T09:09:34.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:33 smithi163 bash[33912]: debug 2023-10-09T09:09:33.954+0000 7f84df46a700 -1 mon.c@2(peon).osd e313 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:34.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:33 smithi181 bash[27712]: debug 2023-10-09T09:09:33.952+0000 7fde83035700 -1 mon.b@1(peon).osd e313 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:34.934 INFO:tasks.rados.rados.0.smithi163.stdout:988: done (6 left) 2023-10-09T09:09:34.935 INFO:tasks.rados.rados.0.smithi163.stdout:989: done (5 left) 2023-10-09T09:09:34.935 INFO:tasks.rados.rados.0.smithi163.stdout:990: done (4 left) 2023-10-09T09:09:34.935 INFO:tasks.rados.rados.0.smithi163.stdout:993: done (3 left) 2023-10-09T09:09:34.935 INFO:tasks.rados.rados.0.smithi163.stdout:995: done (2 left) 2023-10-09T09:09:34.935 INFO:tasks.rados.rados.0.smithi163.stdout:996: done (1 left) 2023-10-09T09:09:34.935 INFO:tasks.rados.rados.0.smithi163.stdout:998: done (0 left) 2023-10-09T09:09:34.935 INFO:tasks.rados.rados.0.smithi163.stdout:999: rollback oid 43 current snap is 122 2023-10-09T09:09:34.935 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 121 2023-10-09T09:09:34.935 INFO:tasks.rados.rados.0.smithi163.stdout:1000: snap_create 2023-10-09T09:09:34.944 INFO:tasks.rados.rados.0.smithi163.stdout:999: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:09:34.944 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 0 (ObjNum 0 snap 32603 seq_num 699196064) dirty dne 2023-10-09T09:09:35.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:34 smithi163 bash[33912]: debug 2023-10-09T09:09:34.922+0000 7f84df46a700 -1 mon.c@2(peon).osd e314 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:35.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:34 smithi163 bash[33912]: cluster 2023-10-09T09:09:33.957982+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2023-10-09T09:09:35.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:34 smithi163 bash[33912]: cluster 2023-10-09T09:09:33.960728+0000 osd.7 (osd.7) 16 : cluster [WRN] failed to encode map e313 with expected crc 2023-10-09T09:09:35.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:34 smithi163 bash[33912]: cluster 2023-10-09T09:09:33.962102+0000 osd.0 (osd.0) 10 : cluster [WRN] failed to encode map e313 with expected crc 2023-10-09T09:09:35.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:34 smithi163 bash[33912]: cluster 2023-10-09T09:09:33.978247+0000 osd.3 (osd.3) 11 : cluster [WRN] failed to encode map e313 with expected crc 2023-10-09T09:09:35.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:34 smithi163 bash[33912]: cluster 2023-10-09T09:09:34.927677+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2023-10-09T09:09:35.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:34 smithi181 bash[27712]: debug 2023-10-09T09:09:34.920+0000 7fde83035700 -1 mon.b@1(peon).osd e314 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:35.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:34 smithi181 bash[27712]: cluster 2023-10-09T09:09:33.957982+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2023-10-09T09:09:35.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:34 smithi181 bash[27712]: cluster 2023-10-09T09:09:33.960728+0000 osd.7 (osd.7) 16 : cluster [WRN] failed to encode map e313 with expected crc 2023-10-09T09:09:35.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:34 smithi181 bash[27712]: cluster 2023-10-09T09:09:33.962102+0000 osd.0 (osd.0) 10 : cluster [WRN] failed to encode map e313 with expected crc 2023-10-09T09:09:35.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:34 smithi181 bash[27712]: cluster 2023-10-09T09:09:33.978247+0000 osd.3 (osd.3) 11 : cluster [WRN] failed to encode map e313 with expected crc 2023-10-09T09:09:35.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:34 smithi181 bash[27712]: cluster 2023-10-09T09:09:34.927677+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2023-10-09T09:09:35.951 INFO:tasks.rados.rados.0.smithi163.stdout:999: done (1 left) 2023-10-09T09:09:35.951 INFO:tasks.rados.rados.0.smithi163.stdout:1000: done (0 left) 2023-10-09T09:09:35.952 INFO:tasks.rados.rados.0.smithi163.stdout:1001: delete oid 21 current snap is 123 2023-10-09T09:09:35.955 INFO:tasks.rados.rados.0.smithi163.stdout:1001: done (0 left) 2023-10-09T09:09:35.955 INFO:tasks.rados.rados.0.smithi163.stdout:1002: snap_remove snap 122 2023-10-09T09:09:36.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:35 smithi163 bash[33912]: debug 2023-10-09T09:09:35.938+0000 7f84df46a700 -1 mon.c@2(peon).osd e315 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:36.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:35 smithi163 bash[33912]: cluster 2023-10-09T09:09:34.930351+0000 osd.5 (osd.5) 9 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:36.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:35 smithi163 bash[33912]: cluster 2023-10-09T09:09:34.931722+0000 osd.3 (osd.3) 12 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:36.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:35 smithi163 bash[33912]: cluster 2023-10-09T09:09:34.934498+0000 osd.4 (osd.4) 18 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:36.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:35 smithi163 bash[33912]: cluster 2023-10-09T09:09:35.210254+0000 mgr.y (mgr.34107) 13 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 115 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:09:36.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:35 smithi163 bash[33912]: cluster 2023-10-09T09:09:35.381769+0000 osd.0 (osd.0) 11 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:36.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:35 smithi163 bash[33912]: cluster 2023-10-09T09:09:35.946420+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2023-10-09T09:09:36.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:35 smithi181 bash[27712]: debug 2023-10-09T09:09:35.936+0000 7fde83035700 -1 mon.b@1(peon).osd e315 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:36.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:35 smithi181 bash[27712]: cluster 2023-10-09T09:09:34.930351+0000 osd.5 (osd.5) 9 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:36.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:35 smithi181 bash[27712]: cluster 2023-10-09T09:09:34.931722+0000 osd.3 (osd.3) 12 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:36.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:35 smithi181 bash[27712]: cluster 2023-10-09T09:09:34.934498+0000 osd.4 (osd.4) 18 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:36.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:35 smithi181 bash[27712]: cluster 2023-10-09T09:09:35.210254+0000 mgr.y (mgr.34107) 13 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 115 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:09:36.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:35 smithi181 bash[27712]: cluster 2023-10-09T09:09:35.381769+0000 osd.0 (osd.0) 11 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:36.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:35 smithi181 bash[27712]: cluster 2023-10-09T09:09:35.946420+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2023-10-09T09:09:36.955 INFO:tasks.rados.rados.0.smithi163.stdout:1002: done (0 left) 2023-10-09T09:09:36.955 INFO:tasks.rados.rados.0.smithi163.stdout:1003: read oid 31 snap -1 2023-10-09T09:09:36.955 INFO:tasks.rados.rados.0.smithi163.stdout:1003: expect (ObjNum 394 snap 121 seq_num 394) 2023-10-09T09:09:36.955 INFO:tasks.rados.rados.0.smithi163.stdout:1004: snap_create 2023-10-09T09:09:36.964 INFO:tasks.rados.rados.0.smithi163.stdout:1003: expect (ObjNum 394 snap 121 seq_num 394) 2023-10-09T09:09:37.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:36 smithi163 bash[33912]: debug 2023-10-09T09:09:36.942+0000 7f84df46a700 -1 mon.c@2(peon).osd e316 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:37.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:36 smithi163 bash[33912]: cluster 2023-10-09T09:09:35.523695+0000 osd.2 (osd.2) 15 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:37.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:36 smithi163 bash[33912]: cluster 2023-10-09T09:09:35.618435+0000 osd.6 (osd.6) 6 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:37.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:36 smithi163 bash[33912]: cluster 2023-10-09T09:09:35.947578+0000 osd.4 (osd.4) 19 : cluster [WRN] failed to encode map e315 with expected crc 2023-10-09T09:09:37.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:36 smithi163 bash[33912]: cluster 2023-10-09T09:09:35.949185+0000 osd.1 (osd.1) 6 : cluster [WRN] failed to encode map e315 with expected crc 2023-10-09T09:09:37.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:36 smithi163 bash[33912]: cluster 2023-10-09T09:09:36.948623+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2023-10-09T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:36 smithi181 bash[27712]: debug 2023-10-09T09:09:36.940+0000 7fde83035700 -1 mon.b@1(peon).osd e316 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:36 smithi181 bash[27712]: cluster 2023-10-09T09:09:35.523695+0000 osd.2 (osd.2) 15 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:36 smithi181 bash[27712]: cluster 2023-10-09T09:09:35.618435+0000 osd.6 (osd.6) 6 : cluster [WRN] failed to encode map e314 with expected crc 2023-10-09T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:36 smithi181 bash[27712]: cluster 2023-10-09T09:09:35.947578+0000 osd.4 (osd.4) 19 : cluster [WRN] failed to encode map e315 with expected crc 2023-10-09T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:36 smithi181 bash[27712]: cluster 2023-10-09T09:09:35.949185+0000 osd.1 (osd.1) 6 : cluster [WRN] failed to encode map e315 with expected crc 2023-10-09T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:36 smithi181 bash[27712]: cluster 2023-10-09T09:09:36.948623+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2023-10-09T09:09:37.952 INFO:tasks.rados.rados.0.smithi163.stdout:1003: done (1 left) 2023-10-09T09:09:37.952 INFO:tasks.rados.rados.0.smithi163.stdout:1004: done (0 left) 2023-10-09T09:09:37.953 INFO:tasks.rados.rados.0.smithi163.stdout:1005: read oid 29 snap 123 2023-10-09T09:09:37.953 INFO:tasks.rados.rados.0.smithi163.stdout:1005: expect (ObjNum 335 snap 101 seq_num 335) 2023-10-09T09:09:37.953 INFO:tasks.rados.rados.0.smithi163.stdout:1006: delete oid 9 current snap is 124 2023-10-09T09:09:37.963 INFO:tasks.rados.rados.0.smithi163.stdout:1006: done (1 left) 2023-10-09T09:09:37.963 INFO:tasks.rados.rados.0.smithi163.stdout:1007: read oid 16 snap 123 2023-10-09T09:09:37.963 INFO:tasks.rados.rados.0.smithi163.stdout:1007: expect (ObjNum 359 snap 108 seq_num 359) 2023-10-09T09:09:37.963 INFO:tasks.rados.rados.0.smithi163.stdout:1008: read oid 21 snap -1 2023-10-09T09:09:37.963 INFO:tasks.rados.rados.0.smithi163.stdout:1008: expect deleted 2023-10-09T09:09:37.963 INFO:tasks.rados.rados.0.smithi163.stdout:1009: rollback oid 10 current snap is 124 2023-10-09T09:09:37.963 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 121 2023-10-09T09:09:37.963 INFO:tasks.rados.rados.0.smithi163.stdout:1010: delete oid 35 current snap is 124 2023-10-09T09:09:37.966 INFO:tasks.rados.rados.0.smithi163.stdout:1008: done (4 left) 2023-10-09T09:09:37.966 INFO:tasks.rados.rados.0.smithi163.stdout:1010: done (3 left) 2023-10-09T09:09:37.966 INFO:tasks.rados.rados.0.smithi163.stdout:1011: read oid 19 snap -1 2023-10-09T09:09:37.967 INFO:tasks.rados.rados.0.smithi163.stdout:1011: expect (ObjNum 390 snap 119 seq_num 390) 2023-10-09T09:09:37.967 INFO:tasks.rados.rados.0.smithi163.stdout:1012: write oid 36 current snap is 124 2023-10-09T09:09:37.967 INFO:tasks.rados.rados.0.smithi163.stdout:1012: seq_num 406 ranges {423642=717803,1875101=766806} 2023-10-09T09:09:37.974 INFO:tasks.rados.rados.0.smithi163.stdout:1012: writing smithi16367185-36 from 423642 to 1141445 tid 1 2023-10-09T09:09:37.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:37 smithi181 bash[27712]: debug 2023-10-09T09:09:37.940+0000 7fde83035700 -1 mon.b@1(peon).osd e317 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:37.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:37 smithi181 bash[27712]: cluster 2023-10-09T09:09:36.950137+0000 osd.4 (osd.4) 20 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:37.984 INFO:tasks.rados.rados.0.smithi163.stdout:1012: writing smithi16367185-36 from 1875101 to 2641907 tid 2 2023-10-09T09:09:37.984 INFO:tasks.rados.rados.0.smithi163.stdout:1013: rollback oid 28 current snap is 124 2023-10-09T09:09:37.984 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 121 2023-10-09T09:09:37.984 INFO:tasks.rados.rados.0.smithi163.stdout:1009: finishing rollback tid 0 to smithi16367185-10 2023-10-09T09:09:37.984 INFO:tasks.rados.rados.0.smithi163.stdout:1009: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:09:37.985 INFO:tasks.rados.rados.0.smithi163.stdout:1005: expect (ObjNum 335 snap 101 seq_num 335) 2023-10-09T09:09:38.107 INFO:tasks.rados.rados.0.smithi163.stdout:1009: finishing rollback tid 2 to smithi16367185-10 2023-10-09T09:09:38.107 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 415 (ObjNum 292 snap 91 seq_num 292) dirty exists 2023-10-09T09:09:38.107 INFO:tasks.rados.rados.0.smithi163.stdout:1011: expect (ObjNum 390 snap 119 seq_num 390) 2023-10-09T09:09:38.208 INFO:tasks.rados.rados.0.smithi163.stdout:1007: expect (ObjNum 359 snap 108 seq_num 359) 2023-10-09T09:09:38.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:37 smithi163 bash[33912]: debug 2023-10-09T09:09:37.942+0000 7f84df46a700 -1 mon.c@2(peon).osd e317 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:38.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:37 smithi163 bash[33912]: cluster 2023-10-09T09:09:36.950137+0000 osd.4 (osd.4) 20 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:38.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:37 smithi163 bash[33912]: cluster 2023-10-09T09:09:36.951298+0000 osd.0 (osd.0) 12 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:38.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:37 smithi163 bash[33912]: cluster 2023-10-09T09:09:37.211034+0000 mgr.y (mgr.34107) 14 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 611 KiB/s wr, 8 op/s 2023-10-09T09:09:38.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:37 smithi163 bash[33912]: cluster 2023-10-09T09:09:37.806289+0000 osd.1 (osd.1) 7 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:38.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:37 smithi163 bash[33912]: cluster 2023-10-09T09:09:37.950702+0000 mon.a (mon.0) 74 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2023-10-09T09:09:38.273 INFO:tasks.rados.rados.0.smithi163.stdout:1012: finishing write tid 1 to smithi16367185-36 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:1012: finishing write tid 2 to smithi16367185-36 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:1012: finishing write tid 3 to smithi16367185-36 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:1012: finishing write tid 4 to smithi16367185-36 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 322 (ObjNum 406 snap 124 seq_num 406) dirty exists 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:1012: left oid 36 (ObjNum 406 snap 124 seq_num 406) 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:1013: finishing rollback tid 0 to smithi16367185-28 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:1013: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:1013: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 577 (ObjNum 295 snap 91 seq_num 295) dirty exists 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:1005: done (5 left) 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:1007: done (4 left) 2023-10-09T09:09:38.274 INFO:tasks.rados.rados.0.smithi163.stdout:1009: done (3 left) 2023-10-09T09:09:38.275 INFO:tasks.rados.rados.0.smithi163.stdout:1011: done (2 left) 2023-10-09T09:09:38.275 INFO:tasks.rados.rados.0.smithi163.stdout:1012: done (1 left) 2023-10-09T09:09:38.275 INFO:tasks.rados.rados.0.smithi163.stdout:1013: done (0 left) 2023-10-09T09:09:38.275 INFO:tasks.rados.rados.0.smithi163.stdout:1014: read oid 8 snap 121 2023-10-09T09:09:38.275 INFO:tasks.rados.rados.0.smithi163.stdout:1014: expect (ObjNum 207 snap 59 seq_num 207) 2023-10-09T09:09:38.275 INFO:tasks.rados.rados.0.smithi163.stdout:1015: delete oid 45 current snap is 124 2023-10-09T09:09:38.278 INFO:tasks.rados.rados.0.smithi163.stdout:1015: done (1 left) 2023-10-09T09:09:38.278 INFO:tasks.rados.rados.0.smithi163.stdout:1016: rollback oid 25 current snap is 124 2023-10-09T09:09:38.278 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 25 to 123 2023-10-09T09:09:38.278 INFO:tasks.rados.rados.0.smithi163.stdout:1017: rollback oid 27 current snap is 124 2023-10-09T09:09:38.278 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 27 to 121 2023-10-09T09:09:38.278 INFO:tasks.rados.rados.0.smithi163.stdout:1018: write oid 44 current snap is 124 2023-10-09T09:09:38.278 INFO:tasks.rados.rados.0.smithi163.stdout:1018: seq_num 408 ranges {457256=468301,1592065=770475,2854581=2675} 2023-10-09T09:09:38.287 INFO:tasks.rados.rados.0.smithi163.stdout:1018: writing smithi16367185-44 from 457256 to 925557 tid 1 2023-10-09T09:09:38.297 INFO:tasks.rados.rados.0.smithi163.stdout:1018: writing smithi16367185-44 from 1592065 to 2362540 tid 2 2023-10-09T09:09:38.300 INFO:tasks.rados.rados.0.smithi163.stdout:1018: writing smithi16367185-44 from 2854581 to 2857256 tid 3 2023-10-09T09:09:38.300 INFO:tasks.rados.rados.0.smithi163.stdout:1019: read oid 14 snap -1 2023-10-09T09:09:38.301 INFO:tasks.rados.rados.0.smithi163.stdout:1019: expect (ObjNum 322 snap 96 seq_num 322) 2023-10-09T09:09:38.301 INFO:tasks.rados.rados.0.smithi163.stdout:1020: write oid 50 current snap is 124 2023-10-09T09:09:38.301 INFO:tasks.rados.rados.0.smithi163.stdout:1016: finishing rollback tid 1 to smithi16367185-25 2023-10-09T09:09:38.301 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 0 (ObjNum 699187824 snap 21875 seq_num 0) dirty dne 2023-10-09T09:09:38.301 INFO:tasks.rados.rados.0.smithi163.stdout:1017: finishing rollback tid 1 to smithi16367185-27 2023-10-09T09:09:38.301 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 0 (ObjNum 699154624 snap 21875 seq_num 0) dirty dne 2023-10-09T09:09:38.301 INFO:tasks.rados.rados.0.smithi163.stdout:1020: seq_num 409 ranges {474063=543550,1650547=514133,2765651=108412} 2023-10-09T09:09:38.306 INFO:tasks.rados.rados.0.smithi163.stdout:1020: writing smithi16367185-50 from 474063 to 1017613 tid 1 2023-10-09T09:09:38.312 INFO:tasks.rados.rados.0.smithi163.stdout:1020: writing smithi16367185-50 from 1650547 to 2164680 tid 2 2023-10-09T09:09:38.317 INFO:tasks.rados.rados.0.smithi163.stdout:1020: writing smithi16367185-50 from 2765651 to 2874063 tid 3 2023-10-09T09:09:38.317 INFO:tasks.rados.rados.0.smithi163.stdout:1016: done (5 left) 2023-10-09T09:09:38.317 INFO:tasks.rados.rados.0.smithi163.stdout:1017: done (4 left) 2023-10-09T09:09:38.317 INFO:tasks.rados.rados.0.smithi163.stdout:1021: snap_create 2023-10-09T09:09:38.317 INFO:tasks.rados.rados.0.smithi163.stdout:1014: expect (ObjNum 207 snap 59 seq_num 207) 2023-10-09T09:09:38.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:37 smithi181 bash[27712]: cluster 2023-10-09T09:09:36.951298+0000 osd.0 (osd.0) 12 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:38.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:37 smithi181 bash[27712]: cluster 2023-10-09T09:09:37.211034+0000 mgr.y (mgr.34107) 14 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 611 KiB/s wr, 8 op/s 2023-10-09T09:09:38.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:37 smithi181 bash[27712]: cluster 2023-10-09T09:09:37.806289+0000 osd.1 (osd.1) 7 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:38.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:37 smithi181 bash[27712]: cluster 2023-10-09T09:09:37.950702+0000 mon.a (mon.0) 74 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2023-10-09T09:09:38.485 INFO:tasks.rados.rados.0.smithi163.stdout:1018: finishing write tid 1 to smithi16367185-44 2023-10-09T09:09:38.486 INFO:tasks.rados.rados.0.smithi163.stdout:1018: finishing write tid 2 to smithi16367185-44 2023-10-09T09:09:38.486 INFO:tasks.rados.rados.0.smithi163.stdout:1018: finishing write tid 3 to smithi16367185-44 2023-10-09T09:09:38.486 INFO:tasks.rados.rados.0.smithi163.stdout:1018: finishing write tid 4 to smithi16367185-44 2023-10-09T09:09:38.486 INFO:tasks.rados.rados.0.smithi163.stdout:1018: finishing write tid 5 to smithi16367185-44 2023-10-09T09:09:38.486 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 863 (ObjNum 408 snap 124 seq_num 408) dirty exists 2023-10-09T09:09:38.486 INFO:tasks.rados.rados.0.smithi163.stdout:1018: left oid 44 (ObjNum 408 snap 124 seq_num 408) 2023-10-09T09:09:38.486 INFO:tasks.rados.rados.0.smithi163.stdout:1019: expect (ObjNum 322 snap 96 seq_num 322) 2023-10-09T09:09:38.600 INFO:tasks.rados.rados.0.smithi163.stdout:1020: finishing write tid 1 to smithi16367185-50 2023-10-09T09:09:38.600 INFO:tasks.rados.rados.0.smithi163.stdout:1020: finishing write tid 2 to smithi16367185-50 2023-10-09T09:09:38.600 INFO:tasks.rados.rados.0.smithi163.stdout:1020: finishing write tid 3 to smithi16367185-50 2023-10-09T09:09:38.600 INFO:tasks.rados.rados.0.smithi163.stdout:1020: finishing write tid 4 to smithi16367185-50 2023-10-09T09:09:38.600 INFO:tasks.rados.rados.0.smithi163.stdout:1020: finishing write tid 5 to smithi16367185-50 2023-10-09T09:09:38.600 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 406 (ObjNum 409 snap 124 seq_num 409) dirty exists 2023-10-09T09:09:38.601 INFO:tasks.rados.rados.0.smithi163.stdout:1020: left oid 50 (ObjNum 409 snap 124 seq_num 409) 2023-10-09T09:09:38.968 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:09:38 smithi181 bash[54907]: ts=2023-10-09T09:09:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:09:38.994 INFO:tasks.rados.rados.0.smithi163.stdout:1014: done (4 left) 2023-10-09T09:09:38.994 INFO:tasks.rados.rados.0.smithi163.stdout:1018: done (3 left) 2023-10-09T09:09:38.994 INFO:tasks.rados.rados.0.smithi163.stdout:1019: done (2 left) 2023-10-09T09:09:38.994 INFO:tasks.rados.rados.0.smithi163.stdout:1020: done (1 left) 2023-10-09T09:09:38.994 INFO:tasks.rados.rados.0.smithi163.stdout:1021: done (0 left) 2023-10-09T09:09:38.994 INFO:tasks.rados.rados.0.smithi163.stdout:1022: read oid 15 snap -1 2023-10-09T09:09:38.994 INFO:tasks.rados.rados.0.smithi163.stdout:1022: expect (ObjNum 399 snap 121 seq_num 399) 2023-10-09T09:09:38.994 INFO:tasks.rados.rados.0.smithi163.stdout:1023: delete oid 48 current snap is 125 2023-10-09T09:09:38.998 INFO:tasks.rados.rados.0.smithi163.stdout:1023: done (1 left) 2023-10-09T09:09:38.999 INFO:tasks.rados.rados.0.smithi163.stdout:1024: read oid 39 snap 123 2023-10-09T09:09:38.999 INFO:tasks.rados.rados.0.smithi163.stdout:1024: expect (ObjNum 389 snap 119 seq_num 389) 2023-10-09T09:09:38.999 INFO:tasks.rados.rados.0.smithi163.stdout:1025: write oid 31 current snap is 125 2023-10-09T09:09:38.999 INFO:tasks.rados.rados.0.smithi163.stdout:1025: seq_num 411 ranges {507677=410401,1367511=517802,2420497=487180} 2023-10-09T09:09:39.007 INFO:tasks.rados.rados.0.smithi163.stdout:1025: writing smithi16367185-31 from 507677 to 918078 tid 1 2023-10-09T09:09:39.013 INFO:tasks.rados.rados.0.smithi163.stdout:1025: writing smithi16367185-31 from 1367511 to 1885313 tid 2 2023-10-09T09:09:39.019 INFO:tasks.rados.rados.0.smithi163.stdout:1025: writing smithi16367185-31 from 2420497 to 2907677 tid 3 2023-10-09T09:09:39.019 INFO:tasks.rados.rados.0.smithi163.stdout:1026: read oid 20 snap -1 2023-10-09T09:09:39.019 INFO:tasks.rados.rados.0.smithi163.stdout:1026: expect (ObjNum 186 snap 49 seq_num 186) 2023-10-09T09:09:39.019 INFO:tasks.rados.rados.0.smithi163.stdout:1027: rollback oid 28 current snap is 125 2023-10-09T09:09:39.020 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 124 2023-10-09T09:09:39.020 INFO:tasks.rados.rados.0.smithi163.stdout:1025: finishing write tid 1 to smithi16367185-31 2023-10-09T09:09:39.020 INFO:tasks.rados.rados.0.smithi163.stdout:1024: expect (ObjNum 389 snap 119 seq_num 389) 2023-10-09T09:09:39.168 INFO:tasks.rados.rados.0.smithi163.stdout:1022: expect (ObjNum 399 snap 121 seq_num 399) 2023-10-09T09:09:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:38 smithi181 bash[27712]: cluster 2023-10-09T09:09:37.401513+0000 osd.7 (osd.7) 17 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:38 smithi181 bash[27712]: cluster 2023-10-09T09:09:37.506009+0000 osd.2 (osd.2) 16 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:38 smithi181 bash[27712]: cluster 2023-10-09T09:09:37.606373+0000 osd.6 (osd.6) 7 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:38 smithi181 bash[27712]: cluster 2023-10-09T09:09:37.948568+0000 osd.7 (osd.7) 18 : cluster [WRN] failed to encode map e317 with expected crc 2023-10-09T09:09:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:38 smithi181 bash[27712]: cluster 2023-10-09T09:09:37.953897+0000 osd.3 (osd.3) 13 : cluster [WRN] failed to encode map e317 with expected crc 2023-10-09T09:09:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:38 smithi181 bash[27712]: cluster 2023-10-09T09:09:37.957011+0000 osd.3 (osd.3) 14 : cluster [WRN] failed to encode map e317 with expected crc 2023-10-09T09:09:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:38 smithi181 bash[27712]: cluster 2023-10-09T09:09:37.957194+0000 osd.2 (osd.2) 17 : cluster [WRN] failed to encode map e317 with expected crc 2023-10-09T09:09:39.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:38 smithi181 bash[27712]: debug 2023-10-09T09:09:38.984+0000 7fde83035700 -1 mon.b@1(peon).osd e318 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:39.361 INFO:tasks.rados.rados.0.smithi163.stdout:1025: finishing write tid 2 to smithi16367185-31 2023-10-09T09:09:39.361 INFO:tasks.rados.rados.0.smithi163.stdout:1025: finishing write tid 3 to smithi16367185-31 2023-10-09T09:09:39.361 INFO:tasks.rados.rados.0.smithi163.stdout:1027: finishing rollback tid 0 to smithi16367185-28 2023-10-09T09:09:39.361 INFO:tasks.rados.rados.0.smithi163.stdout:1027: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:09:39.361 INFO:tasks.rados.rados.0.smithi163.stdout:1027: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:09:39.362 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 581 (ObjNum 295 snap 91 seq_num 295) dirty exists 2023-10-09T09:09:39.362 INFO:tasks.rados.rados.0.smithi163.stdout:1025: finishing write tid 4 to smithi16367185-31 2023-10-09T09:09:39.362 INFO:tasks.rados.rados.0.smithi163.stdout:1025: finishing write tid 5 to smithi16367185-31 2023-10-09T09:09:39.362 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 327 (ObjNum 411 snap 125 seq_num 411) dirty exists 2023-10-09T09:09:39.362 INFO:tasks.rados.rados.0.smithi163.stdout:1025: left oid 31 (ObjNum 411 snap 125 seq_num 411) 2023-10-09T09:09:39.362 INFO:tasks.rados.rados.0.smithi163.stdout:1026: expect (ObjNum 186 snap 49 seq_num 186) 2023-10-09T09:09:39.466 INFO:tasks.rados.rados.0.smithi163.stdout:1022: done (4 left) 2023-10-09T09:09:39.466 INFO:tasks.rados.rados.0.smithi163.stdout:1024: done (3 left) 2023-10-09T09:09:39.466 INFO:tasks.rados.rados.0.smithi163.stdout:1025: done (2 left) 2023-10-09T09:09:39.466 INFO:tasks.rados.rados.0.smithi163.stdout:1026: done (1 left) 2023-10-09T09:09:39.466 INFO:tasks.rados.rados.0.smithi163.stdout:1027: done (0 left) 2023-10-09T09:09:39.466 INFO:tasks.rados.rados.0.smithi163.stdout:1028: snap_create 2023-10-09T09:09:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:38 smithi163 bash[33912]: cluster 2023-10-09T09:09:37.401513+0000 osd.7 (osd.7) 17 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:38 smithi163 bash[33912]: cluster 2023-10-09T09:09:37.506009+0000 osd.2 (osd.2) 16 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:38 smithi163 bash[33912]: cluster 2023-10-09T09:09:37.606373+0000 osd.6 (osd.6) 7 : cluster [WRN] failed to encode map e316 with expected crc 2023-10-09T09:09:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:38 smithi163 bash[33912]: cluster 2023-10-09T09:09:37.948568+0000 osd.7 (osd.7) 18 : cluster [WRN] failed to encode map e317 with expected crc 2023-10-09T09:09:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:38 smithi163 bash[33912]: cluster 2023-10-09T09:09:37.953897+0000 osd.3 (osd.3) 13 : cluster [WRN] failed to encode map e317 with expected crc 2023-10-09T09:09:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:38 smithi163 bash[33912]: cluster 2023-10-09T09:09:37.957011+0000 osd.3 (osd.3) 14 : cluster [WRN] failed to encode map e317 with expected crc 2023-10-09T09:09:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:38 smithi163 bash[33912]: cluster 2023-10-09T09:09:37.957194+0000 osd.2 (osd.2) 17 : cluster [WRN] failed to encode map e317 with expected crc 2023-10-09T09:09:39.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:38 smithi163 bash[33912]: debug 2023-10-09T09:09:38.986+0000 7f84df46a700 -1 mon.c@2(peon).osd e318 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:39.933 INFO:tasks.rados.rados.0.smithi163.stdout:1028: done (0 left) 2023-10-09T09:09:39.933 INFO:tasks.rados.rados.0.smithi163.stdout:1029: read oid 1 snap 124 2023-10-09T09:09:39.933 INFO:tasks.rados.rados.0.smithi163.stdout:1029: expect (ObjNum 380 snap 118 seq_num 380) 2023-10-09T09:09:39.933 INFO:tasks.rados.rados.0.smithi163.stdout:1030: write oid 28 current snap is 126 2023-10-09T09:09:39.933 INFO:tasks.rados.rados.0.smithi163.stdout:1030: seq_num 412 ranges {524484=485650,1425993=661460,2847920=76564} 2023-10-09T09:09:39.938 INFO:tasks.rados.rados.0.smithi163.stdout:1030: writing smithi16367185-28 from 524484 to 1010134 tid 1 2023-10-09T09:09:39.945 INFO:tasks.rados.rados.0.smithi163.stdout:1030: writing smithi16367185-28 from 1425993 to 2087453 tid 2 2023-10-09T09:09:39.950 INFO:tasks.rados.rados.0.smithi163.stdout:1030: writing smithi16367185-28 from 2847920 to 2924484 tid 3 2023-10-09T09:09:39.950 INFO:tasks.rados.rados.0.smithi163.stdout:1031: write oid 26 current snap is 126 2023-10-09T09:09:39.950 INFO:tasks.rados.rados.0.smithi163.stdout:1031: seq_num 413 ranges {541291=560899,1884475=521471} 2023-10-09T09:09:39.957 INFO:tasks.rados.rados.0.smithi163.stdout:1031: writing smithi16367185-26 from 541291 to 1102190 tid 1 2023-10-09T09:09:39.965 INFO:tasks.rados.rados.0.smithi163.stdout:1031: writing smithi16367185-26 from 1884475 to 2405946 tid 2 2023-10-09T09:09:39.965 INFO:tasks.rados.rados.0.smithi163.stdout:1032: snap_create 2023-10-09T09:09:39.965 INFO:tasks.rados.rados.0.smithi163.stdout:1029: expect (ObjNum 380 snap 118 seq_num 380) 2023-10-09T09:09:39.995 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:39 smithi181 bash[27712]: debug 2023-10-09T09:09:39.920+0000 7fde83035700 -1 mon.b@1(peon).osd e319 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:40.105 INFO:tasks.rados.rados.0.smithi163.stdout:1030: finishing write tid 1 to smithi16367185-28 2023-10-09T09:09:40.105 INFO:tasks.rados.rados.0.smithi163.stdout:1030: finishing write tid 2 to smithi16367185-28 2023-10-09T09:09:40.105 INFO:tasks.rados.rados.0.smithi163.stdout:1030: finishing write tid 3 to smithi16367185-28 2023-10-09T09:09:40.105 INFO:tasks.rados.rados.0.smithi163.stdout:1030: finishing write tid 4 to smithi16367185-28 2023-10-09T09:09:40.105 INFO:tasks.rados.rados.0.smithi163.stdout:1030: finishing write tid 5 to smithi16367185-28 2023-10-09T09:09:40.105 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 586 (ObjNum 412 snap 126 seq_num 412) dirty exists 2023-10-09T09:09:40.106 INFO:tasks.rados.rados.0.smithi163.stdout:1030: left oid 28 (ObjNum 412 snap 126 seq_num 412) 2023-10-09T09:09:40.106 INFO:tasks.rados.rados.0.smithi163.stdout:1031: finishing write tid 1 to smithi16367185-26 2023-10-09T09:09:40.106 INFO:tasks.rados.rados.0.smithi163.stdout:1031: finishing write tid 2 to smithi16367185-26 2023-10-09T09:09:40.106 INFO:tasks.rados.rados.0.smithi163.stdout:1031: finishing write tid 3 to smithi16367185-26 2023-10-09T09:09:40.106 INFO:tasks.rados.rados.0.smithi163.stdout:1031: finishing write tid 4 to smithi16367185-26 2023-10-09T09:09:40.106 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 352 (ObjNum 413 snap 126 seq_num 413) dirty exists 2023-10-09T09:09:40.106 INFO:tasks.rados.rados.0.smithi163.stdout:1031: left oid 26 (ObjNum 413 snap 126 seq_num 413) 2023-10-09T09:09:40.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:39 smithi163 bash[33912]: debug 2023-10-09T09:09:39.926+0000 7f84df46a700 -1 mon.c@2(peon).osd e319 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:40.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:39 smithi163 bash[33912]: cluster 2023-10-09T09:09:38.990508+0000 osd.5 (osd.5) 10 : cluster [WRN] failed to encode map e318 with expected crc 2023-10-09T09:09:40.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:39 smithi163 bash[33912]: cluster 2023-10-09T09:09:38.991275+0000 mon.a (mon.0) 75 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2023-10-09T09:09:40.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:39 smithi163 bash[33912]: cluster 2023-10-09T09:09:38.993756+0000 osd.2 (osd.2) 18 : cluster [WRN] failed to encode map e318 with expected crc 2023-10-09T09:09:40.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:39 smithi163 bash[33912]: cluster 2023-10-09T09:09:38.994477+0000 osd.7 (osd.7) 19 : cluster [WRN] failed to encode map e318 with expected crc 2023-10-09T09:09:40.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:39 smithi163 bash[33912]: cluster 2023-10-09T09:09:39.019611+0000 osd.6 (osd.6) 8 : cluster [WRN] failed to encode map e318 with expected crc 2023-10-09T09:09:40.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:39 smithi163 bash[33912]: cluster 2023-10-09T09:09:39.211694+0000 mgr.y (mgr.34107) 15 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 611 KiB/s wr, 8 op/s 2023-10-09T09:09:40.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:39 smithi163 bash[33912]: cluster 2023-10-09T09:09:39.932644+0000 mon.a (mon.0) 76 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2023-10-09T09:09:40.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:39 smithi181 bash[27712]: cluster 2023-10-09T09:09:38.990508+0000 osd.5 (osd.5) 10 : cluster [WRN] failed to encode map e318 with expected crc 2023-10-09T09:09:40.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:39 smithi181 bash[27712]: cluster 2023-10-09T09:09:38.991275+0000 mon.a (mon.0) 75 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2023-10-09T09:09:40.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:39 smithi181 bash[27712]: cluster 2023-10-09T09:09:38.993756+0000 osd.2 (osd.2) 18 : cluster [WRN] failed to encode map e318 with expected crc 2023-10-09T09:09:40.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:39 smithi181 bash[27712]: cluster 2023-10-09T09:09:38.994477+0000 osd.7 (osd.7) 19 : cluster [WRN] failed to encode map e318 with expected crc 2023-10-09T09:09:40.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:39 smithi181 bash[27712]: cluster 2023-10-09T09:09:39.019611+0000 osd.6 (osd.6) 8 : cluster [WRN] failed to encode map e318 with expected crc 2023-10-09T09:09:40.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:39 smithi181 bash[27712]: cluster 2023-10-09T09:09:39.211694+0000 mgr.y (mgr.34107) 15 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 611 KiB/s wr, 8 op/s 2023-10-09T09:09:40.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:39 smithi181 bash[27712]: cluster 2023-10-09T09:09:39.932644+0000 mon.a (mon.0) 76 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2023-10-09T09:09:40.957 INFO:tasks.rados.rados.0.smithi163.stdout:1029: done (3 left) 2023-10-09T09:09:40.957 INFO:tasks.rados.rados.0.smithi163.stdout:1030: done (2 left) 2023-10-09T09:09:40.957 INFO:tasks.rados.rados.0.smithi163.stdout:1031: done (1 left) 2023-10-09T09:09:40.957 INFO:tasks.rados.rados.0.smithi163.stdout:1032: done (0 left) 2023-10-09T09:09:40.957 INFO:tasks.rados.rados.0.smithi163.stdout:1033: delete oid 26 current snap is 127 2023-10-09T09:09:40.966 INFO:tasks.rados.rados.0.smithi163.stdout:1033: done (0 left) 2023-10-09T09:09:40.966 INFO:tasks.rados.rados.0.smithi163.stdout:1034: write oid 47 current snap is 127 2023-10-09T09:09:40.966 INFO:tasks.rados.rados.0.smithi163.stdout:1034: seq_num 415 ranges {574905=711397,1885086=525140,2930189=44716} 2023-10-09T09:09:40.976 INFO:tasks.rados.rados.0.smithi163.stdout:1034: writing smithi16367185-47 from 574905 to 1286302 tid 1 2023-10-09T09:09:40.985 INFO:tasks.rados.rados.0.smithi163.stdout:1034: writing smithi16367185-47 from 1885086 to 2410226 tid 2 2023-10-09T09:09:40.988 INFO:tasks.rados.rados.0.smithi163.stdout:1034: writing smithi16367185-47 from 2930189 to 2974905 tid 3 2023-10-09T09:09:40.988 INFO:tasks.rados.rados.0.smithi163.stdout:1035: write oid 20 current snap is 127 2023-10-09T09:09:40.989 INFO:tasks.rados.rados.0.smithi163.stdout:1034: finishing write tid 1 to smithi16367185-47 2023-10-09T09:09:40.989 INFO:tasks.rados.rados.0.smithi163.stdout:1035: seq_num 416 ranges {591712=786646,1943568=668798} 2023-10-09T09:09:40.996 INFO:tasks.rados.rados.0.smithi163.stdout:1035: writing smithi16367185-20 from 591712 to 1378358 tid 1 2023-10-09T09:09:41.003 INFO:tasks.rados.rados.0.smithi163.stdout:1035: writing smithi16367185-20 from 1943568 to 2612366 tid 2 2023-10-09T09:09:41.004 INFO:tasks.rados.rados.0.smithi163.stdout:1036: snap_create 2023-10-09T09:09:41.004 INFO:tasks.rados.rados.0.smithi163.stdout:1034: finishing write tid 2 to smithi16367185-47 2023-10-09T09:09:41.004 INFO:tasks.rados.rados.0.smithi163.stdout:1034: finishing write tid 3 to smithi16367185-47 2023-10-09T09:09:41.004 INFO:tasks.rados.rados.0.smithi163.stdout:1034: finishing write tid 4 to smithi16367185-47 2023-10-09T09:09:41.004 INFO:tasks.rados.rados.0.smithi163.stdout:1034: finishing write tid 5 to smithi16367185-47 2023-10-09T09:09:41.004 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 868 (ObjNum 415 snap 127 seq_num 415) dirty exists 2023-10-09T09:09:41.005 INFO:tasks.rados.rados.0.smithi163.stdout:1034: left oid 47 (ObjNum 415 snap 127 seq_num 415) 2023-10-09T09:09:41.010 INFO:tasks.rados.rados.0.smithi163.stdout:1035: finishing write tid 1 to smithi16367185-20 2023-10-09T09:09:41.011 INFO:tasks.rados.rados.0.smithi163.stdout:1035: finishing write tid 2 to smithi16367185-20 2023-10-09T09:09:41.011 INFO:tasks.rados.rados.0.smithi163.stdout:1035: finishing write tid 3 to smithi16367185-20 2023-10-09T09:09:41.011 INFO:tasks.rados.rados.0.smithi163.stdout:1035: finishing write tid 4 to smithi16367185-20 2023-10-09T09:09:41.011 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 121 (ObjNum 416 snap 127 seq_num 416) dirty exists 2023-10-09T09:09:41.011 INFO:tasks.rados.rados.0.smithi163.stdout:1035: left oid 20 (ObjNum 416 snap 127 seq_num 416) 2023-10-09T09:09:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:40 smithi163 bash[33912]: debug 2023-10-09T09:09:40.946+0000 7f84df46a700 -1 mon.c@2(peon).osd e320 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:40 smithi163 bash[33912]: cluster 2023-10-09T09:09:39.929387+0000 osd.5 (osd.5) 11 : cluster [WRN] failed to encode map e319 with expected crc 2023-10-09T09:09:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:40 smithi163 bash[33912]: cluster 2023-10-09T09:09:39.937186+0000 osd.3 (osd.3) 15 : cluster [WRN] failed to encode map e319 with expected crc 2023-10-09T09:09:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:40 smithi163 bash[33912]: cluster 2023-10-09T09:09:39.940114+0000 osd.2 (osd.2) 19 : cluster [WRN] failed to encode map e319 with expected crc 2023-10-09T09:09:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:40 smithi163 bash[33912]: cluster 2023-10-09T09:09:40.323659+0000 osd.0 (osd.0) 13 : cluster [WRN] failed to encode map e319 with expected crc 2023-10-09T09:09:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:40 smithi163 bash[33912]: audit 2023-10-09T09:09:40.325196+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:40 smithi163 bash[33912]: cluster 2023-10-09T09:09:40.951456+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2023-10-09T09:09:41.290 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:40 smithi181 bash[27712]: debug 2023-10-09T09:09:40.940+0000 7fde83035700 -1 mon.b@1(peon).osd e320 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:41.290 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:40 smithi181 bash[27712]: cluster 2023-10-09T09:09:39.929387+0000 osd.5 (osd.5) 11 : cluster [WRN] failed to encode map e319 with expected crc 2023-10-09T09:09:41.290 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:40 smithi181 bash[27712]: cluster 2023-10-09T09:09:39.937186+0000 osd.3 (osd.3) 15 : cluster [WRN] failed to encode map e319 with expected crc 2023-10-09T09:09:41.290 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:40 smithi181 bash[27712]: cluster 2023-10-09T09:09:39.940114+0000 osd.2 (osd.2) 19 : cluster [WRN] failed to encode map e319 with expected crc 2023-10-09T09:09:41.290 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:40 smithi181 bash[27712]: cluster 2023-10-09T09:09:40.323659+0000 osd.0 (osd.0) 13 : cluster [WRN] failed to encode map e319 with expected crc 2023-10-09T09:09:41.290 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:40 smithi181 bash[27712]: audit 2023-10-09T09:09:40.325196+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:41.290 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:40 smithi181 bash[27712]: cluster 2023-10-09T09:09:40.951456+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2023-10-09T09:09:42.014 INFO:tasks.rados.rados.0.smithi163.stdout:1034: done (2 left) 2023-10-09T09:09:42.015 INFO:tasks.rados.rados.0.smithi163.stdout:1035: done (1 left) 2023-10-09T09:09:42.015 INFO:tasks.rados.rados.0.smithi163.stdout:1036: done (0 left) 2023-10-09T09:09:42.015 INFO:tasks.rados.rados.0.smithi163.stdout:1037: read oid 36 snap 124 2023-10-09T09:09:42.015 INFO:tasks.rados.rados.0.smithi163.stdout:1037: expect (ObjNum 406 snap 124 seq_num 406) 2023-10-09T09:09:42.015 INFO:tasks.rados.rados.0.smithi163.stdout:1038: write oid 8 current snap is 128 2023-10-09T09:09:42.015 INFO:tasks.rados.rados.0.smithi163.stdout:1038: seq_num 417 ranges {608519=461895,1602050=528809,2585035=423484} 2023-10-09T09:09:42.026 INFO:tasks.rados.rados.0.smithi163.stdout:1038: writing smithi16367185-8 from 608519 to 1070414 tid 1 2023-10-09T09:09:42.034 INFO:tasks.rados.rados.0.smithi163.stdout:1038: writing smithi16367185-8 from 1602050 to 2130859 tid 2 2023-10-09T09:09:42.040 INFO:tasks.rados.rados.0.smithi163.stdout:1038: writing smithi16367185-8 from 2585035 to 3008519 tid 3 2023-10-09T09:09:42.040 INFO:tasks.rados.rados.0.smithi163.stdout:1039: write oid 16 current snap is 128 2023-10-09T09:09:42.040 INFO:tasks.rados.rados.0.smithi163.stdout:1039: seq_num 418 ranges {625326=537144,1944179=672467} 2023-10-09T09:09:42.046 INFO:tasks.rados.rados.0.smithi163.stdout:1039: writing smithi16367185-16 from 625326 to 1162470 tid 1 2023-10-09T09:09:42.055 INFO:tasks.rados.rados.0.smithi163.stdout:1039: writing smithi16367185-16 from 1944179 to 2616646 tid 2 2023-10-09T09:09:42.055 INFO:tasks.rados.rados.0.smithi163.stdout:1040: rollback oid 9 current snap is 128 2023-10-09T09:09:42.055 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 126 2023-10-09T09:09:42.055 INFO:tasks.rados.rados.0.smithi163.stdout:1037: expect (ObjNum 406 snap 124 seq_num 406) 2023-10-09T09:09:42.188 INFO:tasks.rados.rados.0.smithi163.stdout:1038: finishing write tid 1 to smithi16367185-8 2023-10-09T09:09:42.189 INFO:tasks.rados.rados.0.smithi163.stdout:1038: finishing write tid 2 to smithi16367185-8 2023-10-09T09:09:42.189 INFO:tasks.rados.rados.0.smithi163.stdout:1038: finishing write tid 3 to smithi16367185-8 2023-10-09T09:09:42.189 INFO:tasks.rados.rados.0.smithi163.stdout:1038: finishing write tid 4 to smithi16367185-8 2023-10-09T09:09:42.189 INFO:tasks.rados.rados.0.smithi163.stdout:1038: finishing write tid 5 to smithi16367185-8 2023-10-09T09:09:42.189 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 563 (ObjNum 417 snap 128 seq_num 417) dirty exists 2023-10-09T09:09:42.189 INFO:tasks.rados.rados.0.smithi163.stdout:1038: left oid 8 (ObjNum 417 snap 128 seq_num 417) 2023-10-09T09:09:42.189 INFO:tasks.rados.rados.0.smithi163.stdout:1039: finishing write tid 1 to smithi16367185-16 2023-10-09T09:09:42.189 INFO:tasks.rados.rados.0.smithi163.stdout:1037: done (3 left) 2023-10-09T09:09:42.189 INFO:tasks.rados.rados.0.smithi163.stdout:1038: done (2 left) 2023-10-09T09:09:42.189 INFO:tasks.rados.rados.0.smithi163.stdout:1041: read oid 21 snap -1 2023-10-09T09:09:42.190 INFO:tasks.rados.rados.0.smithi163.stdout:1041: expect deleted 2023-10-09T09:09:42.190 INFO:tasks.rados.rados.0.smithi163.stdout:1042: write oid 2 current snap is 128 2023-10-09T09:09:42.190 INFO:tasks.rados.rados.0.smithi163.stdout:1042: seq_num 419 ranges {642133=728746,2119014=532478} 2023-10-09T09:09:42.199 INFO:tasks.rados.rados.0.smithi163.stdout:1042: writing smithi16367185-2 from 642133 to 1370879 tid 1 2023-10-09T09:09:42.208 INFO:tasks.rados.rados.0.smithi163.stdout:1042: writing smithi16367185-2 from 2119014 to 2651492 tid 2 2023-10-09T09:09:42.208 INFO:tasks.rados.rados.0.smithi163.stdout:1043: snap_create 2023-10-09T09:09:42.208 INFO:tasks.rados.rados.0.smithi163.stdout:1040: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:09:42.208 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 0 (ObjNum 699062544 snap 21875 seq_num 1618308224) dirty dne 2023-10-09T09:09:42.209 INFO:tasks.rados.rados.0.smithi163.stdout:1039: finishing write tid 2 to smithi16367185-16 2023-10-09T09:09:42.209 INFO:tasks.rados.rados.0.smithi163.stdout:1039: finishing write tid 3 to smithi16367185-16 2023-10-09T09:09:42.209 INFO:tasks.rados.rados.0.smithi163.stdout:1039: finishing write tid 4 to smithi16367185-16 2023-10-09T09:09:42.209 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 590 (ObjNum 418 snap 128 seq_num 418) dirty exists 2023-10-09T09:09:42.209 INFO:tasks.rados.rados.0.smithi163.stdout:1039: left oid 16 (ObjNum 418 snap 128 seq_num 418) 2023-10-09T09:09:42.209 INFO:tasks.rados.rados.0.smithi163.stdout:1042: finishing write tid 1 to smithi16367185-2 2023-10-09T09:09:42.212 INFO:tasks.rados.rados.0.smithi163.stdout:1042: finishing write tid 2 to smithi16367185-2 2023-10-09T09:09:42.212 INFO:tasks.rados.rados.0.smithi163.stdout:1042: finishing write tid 3 to smithi16367185-2 2023-10-09T09:09:42.213 INFO:tasks.rados.rados.0.smithi163.stdout:1042: finishing write tid 4 to smithi16367185-2 2023-10-09T09:09:42.213 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 321 (ObjNum 419 snap 128 seq_num 419) dirty exists 2023-10-09T09:09:42.213 INFO:tasks.rados.rados.0.smithi163.stdout:1042: left oid 2 (ObjNum 419 snap 128 seq_num 419) 2023-10-09T09:09:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:42 smithi181 bash[27712]: debug 2023-10-09T09:09:42.000+0000 7fde83035700 -1 mon.b@1(peon).osd e321 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:42 smithi181 bash[27712]: cluster 2023-10-09T09:09:40.601136+0000 osd.6 (osd.6) 9 : cluster [WRN] failed to encode map e319 with expected crc 2023-10-09T09:09:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:42 smithi181 bash[27712]: cluster 2023-10-09T09:09:40.951824+0000 osd.4 (osd.4) 21 : cluster [WRN] failed to encode map e320 with expected crc 2023-10-09T09:09:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:42 smithi181 bash[27712]: cluster 2023-10-09T09:09:40.955342+0000 osd.0 (osd.0) 14 : cluster [WRN] failed to encode map e320 with expected crc 2023-10-09T09:09:42.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:42 smithi181 bash[27712]: cluster 2023-10-09T09:09:40.959156+0000 osd.2 (osd.2) 20 : cluster [WRN] failed to encode map e320 with expected crc 2023-10-09T09:09:42.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:42 smithi181 bash[27712]: cluster 2023-10-09T09:09:41.001260+0000 osd.6 (osd.6) 10 : cluster [WRN] failed to encode map e320 with expected crc 2023-10-09T09:09:42.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:42 smithi181 bash[27712]: cluster 2023-10-09T09:09:41.212288+0000 mgr.y (mgr.34107) 16 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:09:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:42 smithi163 bash[33912]: debug 2023-10-09T09:09:42.006+0000 7f84df46a700 -1 mon.c@2(peon).osd e321 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:42 smithi163 bash[33912]: cluster 2023-10-09T09:09:40.601136+0000 osd.6 (osd.6) 9 : cluster [WRN] failed to encode map e319 with expected crc 2023-10-09T09:09:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:42 smithi163 bash[33912]: cluster 2023-10-09T09:09:40.951824+0000 osd.4 (osd.4) 21 : cluster [WRN] failed to encode map e320 with expected crc 2023-10-09T09:09:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:42 smithi163 bash[33912]: cluster 2023-10-09T09:09:40.955342+0000 osd.0 (osd.0) 14 : cluster [WRN] failed to encode map e320 with expected crc 2023-10-09T09:09:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:42 smithi163 bash[33912]: cluster 2023-10-09T09:09:40.959156+0000 osd.2 (osd.2) 20 : cluster [WRN] failed to encode map e320 with expected crc 2023-10-09T09:09:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:42 smithi163 bash[33912]: cluster 2023-10-09T09:09:41.001260+0000 osd.6 (osd.6) 10 : cluster [WRN] failed to encode map e320 with expected crc 2023-10-09T09:09:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:42 smithi163 bash[33912]: cluster 2023-10-09T09:09:41.212288+0000 mgr.y (mgr.34107) 16 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:09:43.038 INFO:tasks.rados.rados.0.smithi163.stdout:1039: done (4 left) 2023-10-09T09:09:43.039 INFO:tasks.rados.rados.0.smithi163.stdout:1040: done (3 left) 2023-10-09T09:09:43.039 INFO:tasks.rados.rados.0.smithi163.stdout:1041: done (2 left) 2023-10-09T09:09:43.039 INFO:tasks.rados.rados.0.smithi163.stdout:1042: done (1 left) 2023-10-09T09:09:43.039 INFO:tasks.rados.rados.0.smithi163.stdout:1043: done (0 left) 2023-10-09T09:09:43.039 INFO:tasks.rados.rados.0.smithi163.stdout:1044: rollback oid 27 current snap is 129 2023-10-09T09:09:43.039 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 27 to 128 2023-10-09T09:09:43.039 INFO:tasks.rados.rados.0.smithi163.stdout:1045: write oid 39 current snap is 129 2023-10-09T09:09:43.039 INFO:tasks.rados.rados.0.smithi163.stdout:1045: seq_num 420 ranges {658940=403995,1777496=676136,2950951=107989} 2023-10-09T09:09:43.048 INFO:tasks.rados.rados.0.smithi163.stdout:1045: writing smithi16367185-39 from 658940 to 1062935 tid 1 2023-10-09T09:09:43.057 INFO:tasks.rados.rados.0.smithi163.stdout:1045: writing smithi16367185-39 from 1777496 to 2453632 tid 2 2023-10-09T09:09:43.060 INFO:tasks.rados.rados.0.smithi163.stdout:1045: writing smithi16367185-39 from 2950951 to 3058940 tid 3 2023-10-09T09:09:43.060 INFO:tasks.rados.rados.0.smithi163.stdout:1046: write oid 38 current snap is 129 2023-10-09T09:09:43.060 INFO:tasks.rados.rados.0.smithi163.stdout:1046: seq_num 421 ranges {675747=479244,1835978=536147} 2023-10-09T09:09:43.069 INFO:tasks.rados.rados.0.smithi163.stdout:1046: writing smithi16367185-38 from 675747 to 1154991 tid 1 2023-10-09T09:09:43.074 INFO:tasks.rados.rados.0.smithi163.stdout:1046: writing smithi16367185-38 from 1835978 to 2372125 tid 2 2023-10-09T09:09:43.074 INFO:tasks.rados.rados.0.smithi163.stdout:1047: write oid 32 current snap is 129 2023-10-09T09:09:43.074 INFO:tasks.rados.rados.0.smithi163.stdout:1047: seq_num 422 ranges {692554=554493,1778107=679805,2889444=203110} 2023-10-09T09:09:43.081 INFO:tasks.rados.rados.0.smithi163.stdout:1047: writing smithi16367185-32 from 692554 to 1247047 tid 1 2023-10-09T09:09:43.088 INFO:tasks.rados.rados.0.smithi163.stdout:1047: writing smithi16367185-32 from 1778107 to 2457912 tid 2 2023-10-09T09:09:43.093 INFO:tasks.rados.rados.0.smithi163.stdout:1047: writing smithi16367185-32 from 2889444 to 3092554 tid 3 2023-10-09T09:09:43.093 INFO:tasks.rados.rados.0.smithi163.stdout:1048: read oid 24 snap 123 2023-10-09T09:09:43.093 INFO:tasks.rados.rados.0.smithi163.stdout:1048: expect (ObjNum 402 snap 121 seq_num 402) 2023-10-09T09:09:43.093 INFO:tasks.rados.rados.0.smithi163.stdout:1049: rollback oid 47 current snap is 129 2023-10-09T09:09:43.093 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 47 to 127 2023-10-09T09:09:43.093 INFO:tasks.rados.rados.0.smithi163.stdout:1044: finishing rollback tid 1 to smithi16367185-27 2023-10-09T09:09:43.093 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:09:43.093 INFO:tasks.rados.rados.0.smithi163.stdout:1045: finishing write tid 1 to smithi16367185-39 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:1045: finishing write tid 2 to smithi16367185-39 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:1045: finishing write tid 3 to smithi16367185-39 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:1045: finishing write tid 4 to smithi16367185-39 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:1045: finishing write tid 5 to smithi16367185-39 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 411 (ObjNum 420 snap 129 seq_num 420) dirty exists 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:1045: left oid 39 (ObjNum 420 snap 129 seq_num 420) 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:1046: finishing write tid 1 to smithi16367185-38 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:1046: finishing write tid 2 to smithi16367185-38 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:1046: finishing write tid 3 to smithi16367185-38 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:1046: finishing write tid 4 to smithi16367185-38 2023-10-09T09:09:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 431 (ObjNum 421 snap 129 seq_num 421) dirty exists 2023-10-09T09:09:43.095 INFO:tasks.rados.rados.0.smithi163.stdout:1046: left oid 38 (ObjNum 421 snap 129 seq_num 421) 2023-10-09T09:09:43.095 INFO:tasks.rados.rados.0.smithi163.stdout:1047: finishing write tid 1 to smithi16367185-32 2023-10-09T09:09:43.095 INFO:tasks.rados.rados.0.smithi163.stdout:1044: done (5 left) 2023-10-09T09:09:43.095 INFO:tasks.rados.rados.0.smithi163.stdout:1045: done (4 left) 2023-10-09T09:09:43.095 INFO:tasks.rados.rados.0.smithi163.stdout:1046: done (3 left) 2023-10-09T09:09:43.095 INFO:tasks.rados.rados.0.smithi163.stdout:1050: snap_create 2023-10-09T09:09:43.095 INFO:tasks.rados.rados.0.smithi163.stdout:1047: finishing write tid 2 to smithi16367185-32 2023-10-09T09:09:43.096 INFO:tasks.rados.rados.0.smithi163.stdout:1047: finishing write tid 3 to smithi16367185-32 2023-10-09T09:09:43.097 INFO:tasks.rados.rados.0.smithi163.stdout:1047: finishing write tid 4 to smithi16367185-32 2023-10-09T09:09:43.098 INFO:tasks.rados.rados.0.smithi163.stdout:1047: finishing write tid 5 to smithi16367185-32 2023-10-09T09:09:43.098 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 872 (ObjNum 422 snap 129 seq_num 422) dirty exists 2023-10-09T09:09:43.098 INFO:tasks.rados.rados.0.smithi163.stdout:1047: left oid 32 (ObjNum 422 snap 129 seq_num 422) 2023-10-09T09:09:43.099 INFO:tasks.rados.rados.0.smithi163.stdout:1049: finishing rollback tid 0 to smithi16367185-47 2023-10-09T09:09:43.100 INFO:tasks.rados.rados.0.smithi163.stdout:1049: finishing rollback tid 1 to smithi16367185-47 2023-10-09T09:09:43.100 INFO:tasks.rados.rados.0.smithi163.stdout:1049: finishing rollback tid 2 to smithi16367185-47 2023-10-09T09:09:43.100 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 876 (ObjNum 415 snap 127 seq_num 415) dirty exists 2023-10-09T09:09:43.103 INFO:tasks.rados.rados.0.smithi163.stdout:1048: expect (ObjNum 402 snap 121 seq_num 402) 2023-10-09T09:09:43.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cluster 2023-10-09T09:09:42.010892+0000 osd.6 (osd.6) 11 : cluster [WRN] failed to encode map e321 with expected crc 2023-10-09T09:09:43.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cluster 2023-10-09T09:09:42.011471+0000 mon.a (mon.0) 79 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2023-10-09T09:09:43.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cluster 2023-10-09T09:09:42.013527+0000 osd.0 (osd.0) 15 : cluster [WRN] failed to encode map e321 with expected crc 2023-10-09T09:09:43.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cluster 2023-10-09T09:09:42.017138+0000 osd.0 (osd.0) 16 : cluster [WRN] failed to encode map e321 with expected crc 2023-10-09T09:09:43.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.020367+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cluster 2023-10-09T09:09:42.026932+0000 osd.7 (osd.7) 20 : cluster [WRN] failed to encode map e321 with expected crc 2023-10-09T09:09:43.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.028971+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.031246+0000 mon.a (mon.0) 82 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.032379+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cephadm 2023-10-09T09:09:42.033797+0000 mgr.y (mgr.34107) 17 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cephadm 2023-10-09T09:09:42.034174+0000 mgr.y (mgr.34107) 18 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cluster 2023-10-09T09:09:42.190386+0000 osd.1 (osd.1) 8 : cluster [WRN] failed to encode map e321 with expected crc 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cephadm 2023-10-09T09:09:42.199742+0000 mgr.y (mgr.34107) 19 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cephadm 2023-10-09T09:09:42.200193+0000 mgr.y (mgr.34107) 20 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cephadm 2023-10-09T09:09:42.364578+0000 mgr.y (mgr.34107) 21 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: cephadm 2023-10-09T09:09:42.366375+0000 mgr.y (mgr.34107) 22 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.705185+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.712443+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.720620+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.727200+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.736447+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.820106+0000 mon.a (mon.0) 89 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:43.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.822934+0000 mon.a (mon.0) 90 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:43.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.824412+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:09:43.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: audit 2023-10-09T09:09:42.833298+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:43 smithi181 bash[27712]: debug 2023-10-09T09:09:43.024+0000 7fde83035700 -1 mon.b@1(peon).osd e322 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:43.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cluster 2023-10-09T09:09:42.010892+0000 osd.6 (osd.6) 11 : cluster [WRN] failed to encode map e321 with expected crc 2023-10-09T09:09:43.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cluster 2023-10-09T09:09:42.011471+0000 mon.a (mon.0) 79 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2023-10-09T09:09:43.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cluster 2023-10-09T09:09:42.013527+0000 osd.0 (osd.0) 15 : cluster [WRN] failed to encode map e321 with expected crc 2023-10-09T09:09:43.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cluster 2023-10-09T09:09:42.017138+0000 osd.0 (osd.0) 16 : cluster [WRN] failed to encode map e321 with expected crc 2023-10-09T09:09:43.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.020367+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cluster 2023-10-09T09:09:42.026932+0000 osd.7 (osd.7) 20 : cluster [WRN] failed to encode map e321 with expected crc 2023-10-09T09:09:43.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.028971+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.031246+0000 mon.a (mon.0) 82 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:43.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.032379+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:09:43.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cephadm 2023-10-09T09:09:42.033797+0000 mgr.y (mgr.34107) 17 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:09:43.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cephadm 2023-10-09T09:09:42.034174+0000 mgr.y (mgr.34107) 18 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:09:43.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cluster 2023-10-09T09:09:42.190386+0000 osd.1 (osd.1) 8 : cluster [WRN] failed to encode map e321 with expected crc 2023-10-09T09:09:43.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cephadm 2023-10-09T09:09:42.199742+0000 mgr.y (mgr.34107) 19 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:09:43.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cephadm 2023-10-09T09:09:42.200193+0000 mgr.y (mgr.34107) 20 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:09:43.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cephadm 2023-10-09T09:09:42.364578+0000 mgr.y (mgr.34107) 21 : cephadm [INF] Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:09:43.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: cephadm 2023-10-09T09:09:42.366375+0000 mgr.y (mgr.34107) 22 : cephadm [INF] Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:09:43.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.705185+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.712443+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.720620+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.727200+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.736447+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.820106+0000 mon.a (mon.0) 89 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:09:43.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.822934+0000 mon.a (mon.0) 90 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:09:43.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.824412+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:09:43.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: audit 2023-10-09T09:09:42.833298+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:09:43.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[33912]: debug 2023-10-09T09:09:43.026+0000 7f84df46a700 -1 mon.c@2(peon).osd e322 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:09:43] "GET /metrics HTTP/1.1" 200 36086 "" "Prometheus/2.33.4" 2023-10-09T09:09:44.043 INFO:tasks.rados.rados.0.smithi163.stdout:1047: done (3 left) 2023-10-09T09:09:44.043 INFO:tasks.rados.rados.0.smithi163.stdout:1048: done (2 left) 2023-10-09T09:09:44.043 INFO:tasks.rados.rados.0.smithi163.stdout:1049: done (1 left) 2023-10-09T09:09:44.044 INFO:tasks.rados.rados.0.smithi163.stdout:1050: done (0 left) 2023-10-09T09:09:44.044 INFO:tasks.rados.rados.0.smithi163.stdout:1051: snap_remove snap 123 2023-10-09T09:09:44.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:44 smithi181 bash[27712]: debug 2023-10-09T09:09:44.032+0000 7fde83035700 -1 mon.b@1(peon).osd e323 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:44.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:44 smithi181 bash[27712]: cephadm 2023-10-09T09:09:42.529221+0000 mgr.y (mgr.34107) 23 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:09:44.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:44 smithi181 bash[27712]: cephadm 2023-10-09T09:09:42.531686+0000 mgr.y (mgr.34107) 24 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:09:44.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:44 smithi181 bash[27712]: cluster 2023-10-09T09:09:43.030993+0000 mon.a (mon.0) 93 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2023-10-09T09:09:44.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:44 smithi181 bash[27712]: cluster 2023-10-09T09:09:43.035016+0000 osd.7 (osd.7) 21 : cluster [WRN] failed to encode map e322 with expected crc 2023-10-09T09:09:44.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:44 smithi181 bash[27712]: cluster 2023-10-09T09:09:43.036803+0000 osd.0 (osd.0) 17 : cluster [WRN] failed to encode map e322 with expected crc 2023-10-09T09:09:44.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:44 smithi181 bash[27712]: cluster 2023-10-09T09:09:43.040381+0000 osd.2 (osd.2) 21 : cluster [WRN] failed to encode map e322 with expected crc 2023-10-09T09:09:44.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:44 smithi181 bash[27712]: cluster 2023-10-09T09:09:43.094238+0000 osd.3 (osd.3) 16 : cluster [WRN] failed to encode map e322 with expected crc 2023-10-09T09:09:44.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:44 smithi181 bash[27712]: cluster 2023-10-09T09:09:43.212944+0000 mgr.y (mgr.34107) 25 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 99 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 2.5 MiB/s wr, 34 op/s 2023-10-09T09:09:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:44 smithi163 bash[33912]: debug 2023-10-09T09:09:44.038+0000 7f84df46a700 -1 mon.c@2(peon).osd e323 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:44 smithi163 bash[33912]: cephadm 2023-10-09T09:09:42.529221+0000 mgr.y (mgr.34107) 23 : cephadm [INF] Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:09:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:44 smithi163 bash[33912]: cephadm 2023-10-09T09:09:42.531686+0000 mgr.y (mgr.34107) 24 : cephadm [INF] Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:09:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:44 smithi163 bash[33912]: cluster 2023-10-09T09:09:43.030993+0000 mon.a (mon.0) 93 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2023-10-09T09:09:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:44 smithi163 bash[33912]: cluster 2023-10-09T09:09:43.035016+0000 osd.7 (osd.7) 21 : cluster [WRN] failed to encode map e322 with expected crc 2023-10-09T09:09:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:44 smithi163 bash[33912]: cluster 2023-10-09T09:09:43.036803+0000 osd.0 (osd.0) 17 : cluster [WRN] failed to encode map e322 with expected crc 2023-10-09T09:09:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:44 smithi163 bash[33912]: cluster 2023-10-09T09:09:43.040381+0000 osd.2 (osd.2) 21 : cluster [WRN] failed to encode map e322 with expected crc 2023-10-09T09:09:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:44 smithi163 bash[33912]: cluster 2023-10-09T09:09:43.094238+0000 osd.3 (osd.3) 16 : cluster [WRN] failed to encode map e322 with expected crc 2023-10-09T09:09:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:44 smithi163 bash[33912]: cluster 2023-10-09T09:09:43.212944+0000 mgr.y (mgr.34107) 25 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 99 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 2.5 MiB/s wr, 34 op/s 2023-10-09T09:09:44.940 INFO:tasks.rados.rados.0.smithi163.stdout:1051: done (0 left) 2023-10-09T09:09:44.940 INFO:tasks.rados.rados.0.smithi163.stdout:1052: snap_create 2023-10-09T09:09:45.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:44 smithi163 bash[33912]: debug 2023-10-09T09:09:44.930+0000 7f84df46a700 -1 mon.c@2(peon).osd e324 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:45.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:45 smithi163 bash[33912]: cluster 2023-10-09T09:09:44.039811+0000 osd.6 (osd.6) 12 : cluster [WRN] failed to encode map e323 with expected crc 2023-10-09T09:09:45.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:45 smithi163 bash[33912]: cluster 2023-10-09T09:09:44.041552+0000 mon.a (mon.0) 94 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2023-10-09T09:09:45.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:45 smithi163 bash[33912]: cluster 2023-10-09T09:09:44.044039+0000 osd.3 (osd.3) 17 : cluster [WRN] failed to encode map e323 with expected crc 2023-10-09T09:09:45.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:45 smithi163 bash[33912]: cluster 2023-10-09T09:09:44.936953+0000 mon.a (mon.0) 95 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2023-10-09T09:09:45.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:44 smithi181 bash[27712]: debug 2023-10-09T09:09:44.924+0000 7fde83035700 -1 mon.b@1(peon).osd e324 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:45.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:45 smithi181 bash[27712]: cluster 2023-10-09T09:09:44.039811+0000 osd.6 (osd.6) 12 : cluster [WRN] failed to encode map e323 with expected crc 2023-10-09T09:09:45.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:45 smithi181 bash[27712]: cluster 2023-10-09T09:09:44.041552+0000 mon.a (mon.0) 94 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2023-10-09T09:09:45.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:45 smithi181 bash[27712]: cluster 2023-10-09T09:09:44.044039+0000 osd.3 (osd.3) 17 : cluster [WRN] failed to encode map e323 with expected crc 2023-10-09T09:09:45.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:45 smithi181 bash[27712]: cluster 2023-10-09T09:09:44.936953+0000 mon.a (mon.0) 95 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2023-10-09T09:09:45.961 INFO:tasks.rados.rados.0.smithi163.stdout:1052: done (0 left) 2023-10-09T09:09:45.961 INFO:tasks.rados.rados.0.smithi163.stdout:1053: snap_remove snap 130 2023-10-09T09:09:46.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:45 smithi163 bash[33912]: debug 2023-10-09T09:09:45.946+0000 7f84df46a700 -1 mon.c@2(peon).osd e325 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:46.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:46 smithi163 bash[33912]: cluster 2023-10-09T09:09:44.935886+0000 osd.4 (osd.4) 22 : cluster [WRN] failed to encode map e324 with expected crc 2023-10-09T09:09:46.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:46 smithi163 bash[33912]: cluster 2023-10-09T09:09:44.940789+0000 osd.1 (osd.1) 9 : cluster [WRN] failed to encode map e324 with expected crc 2023-10-09T09:09:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:46 smithi163 bash[33912]: cluster 2023-10-09T09:09:45.213674+0000 mgr.y (mgr.34107) 26 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 99 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 2.5 MiB/s wr, 34 op/s 2023-10-09T09:09:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:46 smithi163 bash[33912]: cluster 2023-10-09T09:09:45.952422+0000 mon.a (mon.0) 96 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2023-10-09T09:09:46.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:45 smithi181 bash[27712]: debug 2023-10-09T09:09:45.943+0000 7fde83035700 -1 mon.b@1(peon).osd e325 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:46.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:46 smithi181 bash[27712]: cluster 2023-10-09T09:09:44.935886+0000 osd.4 (osd.4) 22 : cluster [WRN] failed to encode map e324 with expected crc 2023-10-09T09:09:46.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:46 smithi181 bash[27712]: cluster 2023-10-09T09:09:44.940789+0000 osd.1 (osd.1) 9 : cluster [WRN] failed to encode map e324 with expected crc 2023-10-09T09:09:46.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:46 smithi181 bash[27712]: cluster 2023-10-09T09:09:45.213674+0000 mgr.y (mgr.34107) 26 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 99 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 2.5 MiB/s wr, 34 op/s 2023-10-09T09:09:46.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:46 smithi181 bash[27712]: cluster 2023-10-09T09:09:45.952422+0000 mon.a (mon.0) 96 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2023-10-09T09:09:46.961 INFO:tasks.rados.rados.0.smithi163.stdout:1053: done (0 left) 2023-10-09T09:09:46.962 INFO:tasks.rados.rados.0.smithi163.stdout:1054: delete oid 17 current snap is 131 2023-10-09T09:09:46.969 INFO:tasks.rados.rados.0.smithi163.stdout:1054: done (0 left) 2023-10-09T09:09:46.970 INFO:tasks.rados.rados.0.smithi163.stdout:1055: read oid 16 snap -1 2023-10-09T09:09:46.970 INFO:tasks.rados.rados.0.smithi163.stdout:1055: expect (ObjNum 418 snap 128 seq_num 418) 2023-10-09T09:09:46.970 INFO:tasks.rados.rados.0.smithi163.stdout:1056: read oid 42 snap -1 2023-10-09T09:09:46.970 INFO:tasks.rados.rados.0.smithi163.stdout:1056: expect (ObjNum 384 snap 118 seq_num 384) 2023-10-09T09:09:46.970 INFO:tasks.rados.rados.0.smithi163.stdout:1057: snap_remove snap 127 2023-10-09T09:09:47.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:46 smithi163 bash[33912]: debug 2023-10-09T09:09:46.946+0000 7f84df46a700 -1 mon.c@2(peon).osd e326 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:47.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:47 smithi163 bash[33912]: cluster 2023-10-09T09:09:45.428079+0000 osd.7 (osd.7) 22 : cluster [WRN] failed to encode map e324 with expected crc 2023-10-09T09:09:47.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:47 smithi163 bash[33912]: cluster 2023-10-09T09:09:45.955112+0000 osd.4 (osd.4) 23 : cluster [WRN] failed to encode map e325 with expected crc 2023-10-09T09:09:47.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:47 smithi163 bash[33912]: cluster 2023-10-09T09:09:45.956873+0000 osd.0 (osd.0) 18 : cluster [WRN] failed to encode map e325 with expected crc 2023-10-09T09:09:47.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:47 smithi163 bash[33912]: cluster 2023-10-09T09:09:46.953844+0000 mon.a (mon.0) 97 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2023-10-09T09:09:47.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:46 smithi181 bash[27712]: debug 2023-10-09T09:09:46.943+0000 7fde83035700 -1 mon.b@1(peon).osd e326 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:47.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:47 smithi181 bash[27712]: cluster 2023-10-09T09:09:45.428079+0000 osd.7 (osd.7) 22 : cluster [WRN] failed to encode map e324 with expected crc 2023-10-09T09:09:47.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:47 smithi181 bash[27712]: cluster 2023-10-09T09:09:45.955112+0000 osd.4 (osd.4) 23 : cluster [WRN] failed to encode map e325 with expected crc 2023-10-09T09:09:47.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:47 smithi181 bash[27712]: cluster 2023-10-09T09:09:45.956873+0000 osd.0 (osd.0) 18 : cluster [WRN] failed to encode map e325 with expected crc 2023-10-09T09:09:47.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:47 smithi181 bash[27712]: cluster 2023-10-09T09:09:46.953844+0000 mon.a (mon.0) 97 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2023-10-09T09:09:47.963 INFO:tasks.rados.rados.0.smithi163.stdout:1057: done (2 left) 2023-10-09T09:09:47.964 INFO:tasks.rados.rados.0.smithi163.stdout:1058: snap_create 2023-10-09T09:09:47.964 INFO:tasks.rados.rados.0.smithi163.stdout:1056: expect (ObjNum 384 snap 118 seq_num 384) 2023-10-09T09:09:48.155 INFO:tasks.rados.rados.0.smithi163.stdout:1055: expect (ObjNum 418 snap 128 seq_num 418) 2023-10-09T09:09:48.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:47 smithi163 bash[33912]: debug 2023-10-09T09:09:47.950+0000 7f84df46a700 -1 mon.c@2(peon).osd e327 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:48.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:48 smithi163 bash[33912]: cluster 2023-10-09T09:09:46.463578+0000 osd.2 (osd.2) 22 : cluster [WRN] failed to encode map e325 with expected crc 2023-10-09T09:09:48.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:48 smithi163 bash[33912]: cluster 2023-10-09T09:09:46.495277+0000 osd.6 (osd.6) 13 : cluster [WRN] failed to encode map e325 with expected crc 2023-10-09T09:09:48.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:48 smithi163 bash[33912]: cluster 2023-10-09T09:09:46.957953+0000 osd.4 (osd.4) 24 : cluster [WRN] failed to encode map e326 with expected crc 2023-10-09T09:09:48.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:48 smithi163 bash[33912]: cluster 2023-10-09T09:09:46.958135+0000 osd.2 (osd.2) 23 : cluster [WRN] failed to encode map e326 with expected crc 2023-10-09T09:09:48.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:48 smithi163 bash[33912]: cluster 2023-10-09T09:09:46.961547+0000 osd.7 (osd.7) 23 : cluster [WRN] failed to encode map e326 with expected crc 2023-10-09T09:09:48.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:48 smithi163 bash[33912]: cluster 2023-10-09T09:09:47.214454+0000 mgr.y (mgr.34107) 27 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 110 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2023-10-09T09:09:48.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:48 smithi163 bash[33912]: cluster 2023-10-09T09:09:47.956940+0000 mon.a (mon.0) 98 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2023-10-09T09:09:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:47 smithi181 bash[27712]: debug 2023-10-09T09:09:47.947+0000 7fde83035700 -1 mon.b@1(peon).osd e327 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:48 smithi181 bash[27712]: cluster 2023-10-09T09:09:46.463578+0000 osd.2 (osd.2) 22 : cluster [WRN] failed to encode map e325 with expected crc 2023-10-09T09:09:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:48 smithi181 bash[27712]: cluster 2023-10-09T09:09:46.495277+0000 osd.6 (osd.6) 13 : cluster [WRN] failed to encode map e325 with expected crc 2023-10-09T09:09:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:48 smithi181 bash[27712]: cluster 2023-10-09T09:09:46.957953+0000 osd.4 (osd.4) 24 : cluster [WRN] failed to encode map e326 with expected crc 2023-10-09T09:09:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:48 smithi181 bash[27712]: cluster 2023-10-09T09:09:46.958135+0000 osd.2 (osd.2) 23 : cluster [WRN] failed to encode map e326 with expected crc 2023-10-09T09:09:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:48 smithi181 bash[27712]: cluster 2023-10-09T09:09:46.961547+0000 osd.7 (osd.7) 23 : cluster [WRN] failed to encode map e326 with expected crc 2023-10-09T09:09:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:48 smithi181 bash[27712]: cluster 2023-10-09T09:09:47.214454+0000 mgr.y (mgr.34107) 27 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 110 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2023-10-09T09:09:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:48 smithi181 bash[27712]: cluster 2023-10-09T09:09:47.956940+0000 mon.a (mon.0) 98 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2023-10-09T09:09:48.956 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:09:48 smithi181 bash[54907]: ts=2023-10-09T09:09:48.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:09:48.967 INFO:tasks.rados.rados.0.smithi163.stdout:1055: done (2 left) 2023-10-09T09:09:48.967 INFO:tasks.rados.rados.0.smithi163.stdout:1056: done (1 left) 2023-10-09T09:09:48.967 INFO:tasks.rados.rados.0.smithi163.stdout:1058: done (0 left) 2023-10-09T09:09:48.967 INFO:tasks.rados.rados.0.smithi163.stdout:1059: read oid 27 snap -1 2023-10-09T09:09:48.968 INFO:tasks.rados.rados.0.smithi163.stdout:1059: expect deleted 2023-10-09T09:09:48.968 INFO:tasks.rados.rados.0.smithi163.stdout:1060: read oid 41 snap -1 2023-10-09T09:09:48.968 INFO:tasks.rados.rados.0.smithi163.stdout:1060: expect deleted 2023-10-09T09:09:48.968 INFO:tasks.rados.rados.0.smithi163.stdout:1061: write oid 6 current snap is 132 2023-10-09T09:09:48.968 INFO:tasks.rados.rados.0.smithi163.stdout:1061: seq_num 424 ranges {726168=704991,1895071=683474} 2023-10-09T09:09:48.979 INFO:tasks.rados.rados.0.smithi163.stdout:1061: writing smithi16367185-6 from 726168 to 1431159 tid 1 2023-10-09T09:09:48.986 INFO:tasks.rados.rados.0.smithi163.stdout:1061: writing smithi16367185-6 from 1895071 to 2578545 tid 2 2023-10-09T09:09:48.986 INFO:tasks.rados.rados.0.smithi163.stdout:1062: delete oid 23 current snap is 132 2023-10-09T09:09:48.986 INFO:tasks.rados.rados.0.smithi163.stdout:1061: finishing write tid 1 to smithi16367185-6 2023-10-09T09:09:48.990 INFO:tasks.rados.rados.0.smithi163.stdout:1061: finishing write tid 2 to smithi16367185-6 2023-10-09T09:09:48.991 INFO:tasks.rados.rados.0.smithi163.stdout:1061: finishing write tid 3 to smithi16367185-6 2023-10-09T09:09:48.991 INFO:tasks.rados.rados.0.smithi163.stdout:1061: finishing write tid 4 to smithi16367185-6 2023-10-09T09:09:48.991 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 95 (ObjNum 424 snap 132 seq_num 424) dirty exists 2023-10-09T09:09:48.991 INFO:tasks.rados.rados.0.smithi163.stdout:1061: left oid 6 (ObjNum 424 snap 132 seq_num 424) 2023-10-09T09:09:48.994 INFO:tasks.rados.rados.0.smithi163.stdout:1059: done (3 left) 2023-10-09T09:09:48.994 INFO:tasks.rados.rados.0.smithi163.stdout:1060: done (2 left) 2023-10-09T09:09:48.994 INFO:tasks.rados.rados.0.smithi163.stdout:1061: done (1 left) 2023-10-09T09:09:48.994 INFO:tasks.rados.rados.0.smithi163.stdout:1062: done (0 left) 2023-10-09T09:09:48.995 INFO:tasks.rados.rados.0.smithi163.stdout:1063: read oid 37 snap 131 2023-10-09T09:09:48.995 INFO:tasks.rados.rados.0.smithi163.stdout:1063: expect deleted 2023-10-09T09:09:48.995 INFO:tasks.rados.rados.0.smithi163.stdout:1064: read oid 32 snap -1 2023-10-09T09:09:48.995 INFO:tasks.rados.rados.0.smithi163.stdout:1064: expect (ObjNum 422 snap 129 seq_num 422) 2023-10-09T09:09:48.995 INFO:tasks.rados.rados.0.smithi163.stdout:1065: read oid 45 snap -1 2023-10-09T09:09:48.995 INFO:tasks.rados.rados.0.smithi163.stdout:1065: expect deleted 2023-10-09T09:09:48.995 INFO:tasks.rados.rados.0.smithi163.stdout:1066: write oid 24 current snap is 132 2023-10-09T09:09:48.995 INFO:tasks.rados.rados.0.smithi163.stdout:1066: seq_num 426 ranges {759782=571842,2012035=687143} 2023-10-09T09:09:49.010 INFO:tasks.rados.rados.0.smithi163.stdout:1066: writing smithi16367185-24 from 759782 to 1331624 tid 1 2023-10-09T09:09:49.019 INFO:tasks.rados.rados.0.smithi163.stdout:1066: writing smithi16367185-24 from 2012035 to 2699178 tid 2 2023-10-09T09:09:49.019 INFO:tasks.rados.rados.0.smithi163.stdout:1067: read oid 48 snap -1 2023-10-09T09:09:49.019 INFO:tasks.rados.rados.0.smithi163.stdout:1067: expect deleted 2023-10-09T09:09:49.019 INFO:tasks.rados.rados.0.smithi163.stdout:1068: rollback oid 22 current snap is 132 2023-10-09T09:09:49.019 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 22 to 124 2023-10-09T09:09:49.019 INFO:tasks.rados.rados.0.smithi163.stdout:1069: read oid 11 snap -1 2023-10-09T09:09:49.019 INFO:tasks.rados.rados.0.smithi163.stdout:1069: expect (ObjNum 360 snap 108 seq_num 360) 2023-10-09T09:09:49.019 INFO:tasks.rados.rados.0.smithi163.stdout:1070: rollback oid 30 current snap is 132 2023-10-09T09:09:49.020 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 128 2023-10-09T09:09:49.020 INFO:tasks.rados.rados.0.smithi163.stdout:1064: expect (ObjNum 422 snap 129 seq_num 422) 2023-10-09T09:09:49.125 INFO:tasks.rados.rados.0.smithi163.stdout:1066: finishing write tid 1 to smithi16367185-24 2023-10-09T09:09:49.126 INFO:tasks.rados.rados.0.smithi163.stdout:1070: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:09:49.126 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 0 (ObjNum 699172128 snap 21875 seq_num 698869696) dirty dne 2023-10-09T09:09:49.126 INFO:tasks.rados.rados.0.smithi163.stdout:1068: finishing rollback tid 1 to smithi16367185-22 2023-10-09T09:09:49.126 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 0 (ObjNum 669362160 snap 21875 seq_num 1) dirty dne 2023-10-09T09:09:49.126 INFO:tasks.rados.rados.0.smithi163.stdout:1066: finishing write tid 2 to smithi16367185-24 2023-10-09T09:09:49.126 INFO:tasks.rados.rados.0.smithi163.stdout:1066: finishing write tid 3 to smithi16367185-24 2023-10-09T09:09:49.126 INFO:tasks.rados.rados.0.smithi163.stdout:1066: finishing write tid 4 to smithi16367185-24 2023-10-09T09:09:49.126 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 302 (ObjNum 426 snap 132 seq_num 426) dirty exists 2023-10-09T09:09:49.126 INFO:tasks.rados.rados.0.smithi163.stdout:1066: left oid 24 (ObjNum 426 snap 132 seq_num 426) 2023-10-09T09:09:49.126 INFO:tasks.rados.rados.0.smithi163.stdout:1069: expect (ObjNum 360 snap 108 seq_num 360) 2023-10-09T09:09:49.192 INFO:tasks.rados.rados.0.smithi163.stdout:1063: done (7 left) 2023-10-09T09:09:49.192 INFO:tasks.rados.rados.0.smithi163.stdout:1064: done (6 left) 2023-10-09T09:09:49.192 INFO:tasks.rados.rados.0.smithi163.stdout:1065: done (5 left) 2023-10-09T09:09:49.192 INFO:tasks.rados.rados.0.smithi163.stdout:1066: done (4 left) 2023-10-09T09:09:49.192 INFO:tasks.rados.rados.0.smithi163.stdout:1067: done (3 left) 2023-10-09T09:09:49.192 INFO:tasks.rados.rados.0.smithi163.stdout:1068: done (2 left) 2023-10-09T09:09:49.192 INFO:tasks.rados.rados.0.smithi163.stdout:1069: done (1 left) 2023-10-09T09:09:49.192 INFO:tasks.rados.rados.0.smithi163.stdout:1070: done (0 left) 2023-10-09T09:09:49.192 INFO:tasks.rados.rados.0.smithi163.stdout:1071: write oid 36 current snap is 132 2023-10-09T09:09:49.193 INFO:tasks.rados.rados.0.smithi163.stdout:1071: seq_num 427 ranges {776589=647091,2070517=430801,3026559=150030} 2023-10-09T09:09:49.206 INFO:tasks.rados.rados.0.smithi163.stdout:1071: writing smithi16367185-36 from 776589 to 1423680 tid 1 2023-10-09T09:09:49.212 INFO:tasks.rados.rados.0.smithi163.stdout:1071: writing smithi16367185-36 from 2070517 to 2501318 tid 2 2023-10-09T09:09:49.217 INFO:tasks.rados.rados.0.smithi163.stdout:1071: writing smithi16367185-36 from 3026559 to 3176589 tid 3 2023-10-09T09:09:49.217 INFO:tasks.rados.rados.0.smithi163.stdout:1072: snap_create 2023-10-09T09:09:49.217 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:48 smithi163 bash[33912]: debug 2023-10-09T09:09:48.954+0000 7f84df46a700 -1 mon.c@2(peon).osd e328 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:49.217 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:49 smithi163 bash[33912]: cluster 2023-10-09T09:09:47.959741+0000 osd.4 (osd.4) 25 : cluster [WRN] failed to encode map e327 with expected crc 2023-10-09T09:09:49.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:49 smithi163 bash[33912]: cluster 2023-10-09T09:09:47.960959+0000 osd.2 (osd.2) 24 : cluster [WRN] failed to encode map e327 with expected crc 2023-10-09T09:09:49.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:49 smithi163 bash[33912]: cluster 2023-10-09T09:09:48.959858+0000 mon.a (mon.0) 99 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2023-10-09T09:09:49.220 INFO:tasks.rados.rados.0.smithi163.stdout:1071: finishing write tid 1 to smithi16367185-36 2023-10-09T09:09:49.221 INFO:tasks.rados.rados.0.smithi163.stdout:1071: finishing write tid 2 to smithi16367185-36 2023-10-09T09:09:49.222 INFO:tasks.rados.rados.0.smithi163.stdout:1071: finishing write tid 3 to smithi16367185-36 2023-10-09T09:09:49.222 INFO:tasks.rados.rados.0.smithi163.stdout:1071: finishing write tid 4 to smithi16367185-36 2023-10-09T09:09:49.222 INFO:tasks.rados.rados.0.smithi163.stdout:1071: finishing write tid 5 to smithi16367185-36 2023-10-09T09:09:49.223 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 336 (ObjNum 427 snap 132 seq_num 427) dirty exists 2023-10-09T09:09:49.223 INFO:tasks.rados.rados.0.smithi163.stdout:1071: left oid 36 (ObjNum 427 snap 132 seq_num 427) 2023-10-09T09:09:49.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:48 smithi181 bash[27712]: debug 2023-10-09T09:09:48.951+0000 7fde83035700 -1 mon.b@1(peon).osd e328 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:49.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:49 smithi181 bash[27712]: cluster 2023-10-09T09:09:47.959741+0000 osd.4 (osd.4) 25 : cluster [WRN] failed to encode map e327 with expected crc 2023-10-09T09:09:49.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:49 smithi181 bash[27712]: cluster 2023-10-09T09:09:47.960959+0000 osd.2 (osd.2) 24 : cluster [WRN] failed to encode map e327 with expected crc 2023-10-09T09:09:49.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:49 smithi181 bash[27712]: cluster 2023-10-09T09:09:48.959858+0000 mon.a (mon.0) 99 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2023-10-09T09:09:49.943 INFO:tasks.rados.rados.0.smithi163.stdout:1071: done (1 left) 2023-10-09T09:09:49.943 INFO:tasks.rados.rados.0.smithi163.stdout:1072: done (0 left) 2023-10-09T09:09:49.943 INFO:tasks.rados.rados.0.smithi163.stdout:1073: snap_create 2023-10-09T09:09:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:49 smithi163 bash[33912]: debug 2023-10-09T09:09:49.930+0000 7f84df46a700 -1 mon.c@2(peon).osd e329 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:50 smithi163 bash[33912]: cluster 2023-10-09T09:09:48.963468+0000 osd.4 (osd.4) 26 : cluster [WRN] failed to encode map e328 with expected crc 2023-10-09T09:09:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:50 smithi163 bash[33912]: cluster 2023-10-09T09:09:48.963470+0000 osd.3 (osd.3) 18 : cluster [WRN] failed to encode map e328 with expected crc 2023-10-09T09:09:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:50 smithi163 bash[33912]: cluster 2023-10-09T09:09:48.969668+0000 osd.2 (osd.2) 25 : cluster [WRN] failed to encode map e328 with expected crc 2023-10-09T09:09:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:50 smithi163 bash[33912]: cluster 2023-10-09T09:09:48.986443+0000 osd.5 (osd.5) 12 : cluster [WRN] failed to encode map e328 with expected crc 2023-10-09T09:09:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:50 smithi163 bash[33912]: cluster 2023-10-09T09:09:49.209609+0000 osd.0 (osd.0) 19 : cluster [WRN] failed to encode map e328 with expected crc 2023-10-09T09:09:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:50 smithi163 bash[33912]: cluster 2023-10-09T09:09:49.215566+0000 mgr.y (mgr.34107) 28 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 110 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2023-10-09T09:09:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:50 smithi163 bash[33912]: cluster 2023-10-09T09:09:49.936161+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2023-10-09T09:09:50.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:49 smithi181 bash[27712]: debug 2023-10-09T09:09:49.927+0000 7fde83035700 -1 mon.b@1(peon).osd e329 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:50.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:50 smithi181 bash[27712]: cluster 2023-10-09T09:09:48.963468+0000 osd.4 (osd.4) 26 : cluster [WRN] failed to encode map e328 with expected crc 2023-10-09T09:09:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:50 smithi181 bash[27712]: cluster 2023-10-09T09:09:48.963470+0000 osd.3 (osd.3) 18 : cluster [WRN] failed to encode map e328 with expected crc 2023-10-09T09:09:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:50 smithi181 bash[27712]: cluster 2023-10-09T09:09:48.969668+0000 osd.2 (osd.2) 25 : cluster [WRN] failed to encode map e328 with expected crc 2023-10-09T09:09:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:50 smithi181 bash[27712]: cluster 2023-10-09T09:09:48.986443+0000 osd.5 (osd.5) 12 : cluster [WRN] failed to encode map e328 with expected crc 2023-10-09T09:09:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:50 smithi181 bash[27712]: cluster 2023-10-09T09:09:49.209609+0000 osd.0 (osd.0) 19 : cluster [WRN] failed to encode map e328 with expected crc 2023-10-09T09:09:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:50 smithi181 bash[27712]: cluster 2023-10-09T09:09:49.215566+0000 mgr.y (mgr.34107) 28 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 110 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2023-10-09T09:09:50.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:50 smithi181 bash[27712]: cluster 2023-10-09T09:09:49.936161+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2023-10-09T09:09:50.959 INFO:tasks.rados.rados.0.smithi163.stdout:1073: done (0 left) 2023-10-09T09:09:50.959 INFO:tasks.rados.rados.0.smithi163.stdout:1074: read oid 47 snap -1 2023-10-09T09:09:50.959 INFO:tasks.rados.rados.0.smithi163.stdout:1074: expect (ObjNum 415 snap 127 seq_num 415) 2023-10-09T09:09:50.959 INFO:tasks.rados.rados.0.smithi163.stdout:1075: write oid 26 current snap is 134 2023-10-09T09:09:50.959 INFO:tasks.rados.rados.0.smithi163.stdout:1075: seq_num 428 ranges {793396=722340,2128999=690812} 2023-10-09T09:09:50.967 INFO:tasks.rados.rados.0.smithi163.stdout:1075: writing smithi16367185-26 from 793396 to 1515736 tid 1 2023-10-09T09:09:50.975 INFO:tasks.rados.rados.0.smithi163.stdout:1075: writing smithi16367185-26 from 2128999 to 2819811 tid 2 2023-10-09T09:09:50.975 INFO:tasks.rados.rados.0.smithi163.stdout:1076: write oid 49 current snap is 134 2023-10-09T09:09:50.975 INFO:tasks.rados.rados.0.smithi163.stdout:1076: seq_num 429 ranges {410203=797589,1787481=434470,2681405=528798} 2023-10-09T09:09:50.982 INFO:tasks.rados.rados.0.smithi163.stdout:1076: writing smithi16367185-49 from 410203 to 1207792 tid 1 2023-10-09T09:09:50.988 INFO:tasks.rados.rados.0.smithi163.stdout:1076: writing smithi16367185-49 from 1787481 to 2221951 tid 2 2023-10-09T09:09:50.994 INFO:tasks.rados.rados.0.smithi163.stdout:1076: writing smithi16367185-49 from 2681405 to 3210203 tid 3 2023-10-09T09:09:50.994 INFO:tasks.rados.rados.0.smithi163.stdout:1077: write oid 2 current snap is 134 2023-10-09T09:09:50.994 INFO:tasks.rados.rados.0.smithi163.stdout:1077: seq_num 430 ranges {427010=472838,1329610=694481,2708828=518182} 2023-10-09T09:09:50.999 INFO:tasks.rados.rados.0.smithi163.stdout:1077: writing smithi16367185-2 from 427010 to 899848 tid 1 2023-10-09T09:09:51.005 INFO:tasks.rados.rados.0.smithi163.stdout:1077: writing smithi16367185-2 from 1329610 to 2024091 tid 2 2023-10-09T09:09:51.012 INFO:tasks.rados.rados.0.smithi163.stdout:1077: writing smithi16367185-2 from 2708828 to 3227010 tid 3 2023-10-09T09:09:51.012 INFO:tasks.rados.rados.0.smithi163.stdout:1078: read oid 36 snap -1 2023-10-09T09:09:51.012 INFO:tasks.rados.rados.0.smithi163.stdout:1078: expect (ObjNum 427 snap 132 seq_num 427) 2023-10-09T09:09:51.013 INFO:tasks.rados.rados.0.smithi163.stdout:1079: read oid 16 snap -1 2023-10-09T09:09:51.013 INFO:tasks.rados.rados.0.smithi163.stdout:1079: expect (ObjNum 418 snap 128 seq_num 418) 2023-10-09T09:09:51.013 INFO:tasks.rados.rados.0.smithi163.stdout:1080: rollback oid 4 current snap is 134 2023-10-09T09:09:51.013 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 121 2023-10-09T09:09:51.013 INFO:tasks.rados.rados.0.smithi163.stdout:1074: expect (ObjNum 415 snap 127 seq_num 415) 2023-10-09T09:09:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:1075: finishing write tid 1 to smithi16367185-26 2023-10-09T09:09:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:1075: finishing write tid 2 to smithi16367185-26 2023-10-09T09:09:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:1075: finishing write tid 3 to smithi16367185-26 2023-10-09T09:09:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:1075: finishing write tid 4 to smithi16367185-26 2023-10-09T09:09:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 360 (ObjNum 428 snap 134 seq_num 428) dirty exists 2023-10-09T09:09:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:1075: left oid 26 (ObjNum 428 snap 134 seq_num 428) 2023-10-09T09:09:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:1076: finishing write tid 1 to smithi16367185-49 2023-10-09T09:09:51.122 INFO:tasks.rados.rados.0.smithi163.stdout:1076: finishing write tid 2 to smithi16367185-49 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:1076: finishing write tid 3 to smithi16367185-49 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:1076: finishing write tid 4 to smithi16367185-49 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:1076: finishing write tid 5 to smithi16367185-49 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 175 (ObjNum 429 snap 134 seq_num 429) dirty exists 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:1076: left oid 49 (ObjNum 429 snap 134 seq_num 429) 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:1077: finishing write tid 1 to smithi16367185-2 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:1077: finishing write tid 2 to smithi16367185-2 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:1077: finishing write tid 3 to smithi16367185-2 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:1077: finishing write tid 4 to smithi16367185-2 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:1077: finishing write tid 5 to smithi16367185-2 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 330 (ObjNum 430 snap 134 seq_num 430) dirty exists 2023-10-09T09:09:51.123 INFO:tasks.rados.rados.0.smithi163.stdout:1077: left oid 2 (ObjNum 430 snap 134 seq_num 430) 2023-10-09T09:09:51.124 INFO:tasks.rados.rados.0.smithi163.stdout:1079: expect (ObjNum 418 snap 128 seq_num 418) 2023-10-09T09:09:51.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:50 smithi163 bash[33912]: debug 2023-10-09T09:09:50.946+0000 7f84df46a700 -1 mon.c@2(peon).osd e330 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:51.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:51 smithi163 bash[33912]: cluster 2023-10-09T09:09:49.939371+0000 osd.4 (osd.4) 27 : cluster [WRN] failed to encode map e329 with expected crc 2023-10-09T09:09:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:51 smithi163 bash[33912]: cluster 2023-10-09T09:09:49.940263+0000 osd.2 (osd.2) 26 : cluster [WRN] failed to encode map e329 with expected crc 2023-10-09T09:09:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:51 smithi163 bash[33912]: cluster 2023-10-09T09:09:50.254173+0000 osd.0 (osd.0) 20 : cluster [WRN] failed to encode map e329 with expected crc 2023-10-09T09:09:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:51 smithi163 bash[33912]: cluster 2023-10-09T09:09:50.285842+0000 osd.7 (osd.7) 24 : cluster [WRN] failed to encode map e329 with expected crc 2023-10-09T09:09:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:51 smithi163 bash[33912]: cluster 2023-10-09T09:09:50.951331+0000 mon.a (mon.0) 101 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2023-10-09T09:09:51.259 INFO:tasks.rados.rados.0.smithi163.stdout:1080: finishing rollback tid 0 to smithi16367185-4 2023-10-09T09:09:51.259 INFO:tasks.rados.rados.0.smithi163.stdout:1080: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:09:51.259 INFO:tasks.rados.rados.0.smithi163.stdout:1078: expect (ObjNum 427 snap 132 seq_num 427) 2023-10-09T09:09:51.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:50 smithi181 bash[27712]: debug 2023-10-09T09:09:50.943+0000 7fde83035700 -1 mon.b@1(peon).osd e330 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:51.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:51 smithi181 bash[27712]: cluster 2023-10-09T09:09:49.939371+0000 osd.4 (osd.4) 27 : cluster [WRN] failed to encode map e329 with expected crc 2023-10-09T09:09:51.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:51 smithi181 bash[27712]: cluster 2023-10-09T09:09:49.940263+0000 osd.2 (osd.2) 26 : cluster [WRN] failed to encode map e329 with expected crc 2023-10-09T09:09:51.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:51 smithi181 bash[27712]: cluster 2023-10-09T09:09:50.254173+0000 osd.0 (osd.0) 20 : cluster [WRN] failed to encode map e329 with expected crc 2023-10-09T09:09:51.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:51 smithi181 bash[27712]: cluster 2023-10-09T09:09:50.285842+0000 osd.7 (osd.7) 24 : cluster [WRN] failed to encode map e329 with expected crc 2023-10-09T09:09:51.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:51 smithi181 bash[27712]: cluster 2023-10-09T09:09:50.951331+0000 mon.a (mon.0) 101 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2023-10-09T09:09:51.451 INFO:tasks.rados.rados.0.smithi163.stdout:1080: finishing rollback tid 2 to smithi16367185-4 2023-10-09T09:09:51.451 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 600 (ObjNum 247 snap 73 seq_num 247) dirty exists 2023-10-09T09:09:51.451 INFO:tasks.rados.rados.0.smithi163.stdout:1074: done (6 left) 2023-10-09T09:09:51.452 INFO:tasks.rados.rados.0.smithi163.stdout:1075: done (5 left) 2023-10-09T09:09:51.452 INFO:tasks.rados.rados.0.smithi163.stdout:1076: done (4 left) 2023-10-09T09:09:51.452 INFO:tasks.rados.rados.0.smithi163.stdout:1077: done (3 left) 2023-10-09T09:09:51.452 INFO:tasks.rados.rados.0.smithi163.stdout:1078: done (2 left) 2023-10-09T09:09:51.452 INFO:tasks.rados.rados.0.smithi163.stdout:1079: done (1 left) 2023-10-09T09:09:51.452 INFO:tasks.rados.rados.0.smithi163.stdout:1080: done (0 left) 2023-10-09T09:09:51.452 INFO:tasks.rados.rados.0.smithi163.stdout:1081: write oid 6 current snap is 134 2023-10-09T09:09:51.452 INFO:tasks.rados.rados.0.smithi163.stdout:1081: seq_num 431 ranges {443817=548087,1788092=438139,3019898=223919} 2023-10-09T09:09:51.461 INFO:tasks.rados.rados.0.smithi163.stdout:1081: writing smithi16367185-6 from 443817 to 991904 tid 1 2023-10-09T09:09:51.470 INFO:tasks.rados.rados.0.smithi163.stdout:1081: writing smithi16367185-6 from 1788092 to 2226231 tid 2 2023-10-09T09:09:51.476 INFO:tasks.rados.rados.0.smithi163.stdout:1081: writing smithi16367185-6 from 3019898 to 3243817 tid 3 2023-10-09T09:09:51.476 INFO:tasks.rados.rados.0.smithi163.stdout:1082: snap_create 2023-10-09T09:09:51.476 INFO:tasks.rados.rados.0.smithi163.stdout:1081: finishing write tid 1 to smithi16367185-6 2023-10-09T09:09:51.476 INFO:tasks.rados.rados.0.smithi163.stdout:1081: finishing write tid 2 to smithi16367185-6 2023-10-09T09:09:51.479 INFO:tasks.rados.rados.0.smithi163.stdout:1081: finishing write tid 3 to smithi16367185-6 2023-10-09T09:09:51.480 INFO:tasks.rados.rados.0.smithi163.stdout:1081: finishing write tid 4 to smithi16367185-6 2023-10-09T09:09:51.480 INFO:tasks.rados.rados.0.smithi163.stdout:1081: finishing write tid 5 to smithi16367185-6 2023-10-09T09:09:51.480 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 100 (ObjNum 431 snap 134 seq_num 431) dirty exists 2023-10-09T09:09:51.480 INFO:tasks.rados.rados.0.smithi163.stdout:1081: left oid 6 (ObjNum 431 snap 134 seq_num 431) 2023-10-09T09:09:52.092 INFO:tasks.rados.rados.0.smithi163.stdout:1081: done (1 left) 2023-10-09T09:09:52.092 INFO:tasks.rados.rados.0.smithi163.stdout:1082: done (0 left) 2023-10-09T09:09:52.092 INFO:tasks.rados.rados.0.smithi163.stdout:1083: rollback oid 17 current snap is 135 2023-10-09T09:09:52.092 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 134 2023-10-09T09:09:52.092 INFO:tasks.rados.rados.0.smithi163.stdout:1084: rollback oid 23 current snap is 135 2023-10-09T09:09:52.092 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 23 to 125 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1085: read oid 24 snap 128 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1085: expect (ObjNum 402 snap 121 seq_num 402) 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1086: read oid 20 snap -1 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1086: expect (ObjNum 416 snap 127 seq_num 416) 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1087: read oid 21 snap -1 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1087: expect deleted 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1088: read oid 34 snap -1 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1088: expect (ObjNum 358 snap 108 seq_num 358) 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1089: read oid 1 snap -1 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1089: expect (ObjNum 380 snap 118 seq_num 380) 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1090: write oid 48 current snap is 135 2023-10-09T09:09:52.093 INFO:tasks.rados.rados.0.smithi163.stdout:1090: seq_num 432 ranges {460624=623336,1846574=698150,3163674=96950} 2023-10-09T09:09:52.100 INFO:tasks.rados.rados.0.smithi163.stdout:1090: writing smithi16367185-48 from 460624 to 1083960 tid 1 2023-10-09T09:09:52.109 INFO:tasks.rados.rados.0.smithi163.stdout:1090: writing smithi16367185-48 from 1846574 to 2544724 tid 2 2023-10-09T09:09:52.114 INFO:tasks.rados.rados.0.smithi163.stdout:1090: writing smithi16367185-48 from 3163674 to 3260624 tid 3 2023-10-09T09:09:52.114 INFO:tasks.rados.rados.0.smithi163.stdout:1091: rollback oid 2 current snap is 135 2023-10-09T09:09:52.114 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 2 to 124 2023-10-09T09:09:52.114 INFO:tasks.rados.rados.0.smithi163.stdout:1084: finishing rollback tid 1 to smithi16367185-23 2023-10-09T09:09:52.115 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:09:52.115 INFO:tasks.rados.rados.0.smithi163.stdout:1084: done (8 left) 2023-10-09T09:09:52.115 INFO:tasks.rados.rados.0.smithi163.stdout:1087: done (7 left) 2023-10-09T09:09:52.115 INFO:tasks.rados.rados.0.smithi163.stdout:1092: delete oid 47 current snap is 135 2023-10-09T09:09:52.115 INFO:tasks.rados.rados.0.smithi163.stdout:1083: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:09:52.115 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 567 (ObjNum 698927136 snap 21875 seq_num 699028944) dirty dne 2023-10-09T09:09:52.115 INFO:tasks.rados.rados.0.smithi163.stdout:1085: expect (ObjNum 402 snap 121 seq_num 402) 2023-10-09T09:09:52.241 INFO:tasks.rados.rados.0.smithi163.stdout:1089: expect (ObjNum 380 snap 118 seq_num 380) 2023-10-09T09:09:52.376 INFO:tasks.rados.rados.0.smithi163.stdout:1088: expect (ObjNum 358 snap 108 seq_num 358) 2023-10-09T09:09:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:52 smithi163 bash[33912]: debug 2023-10-09T09:09:52.082+0000 7f84df46a700 -1 mon.c@2(peon).osd e331 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:52 smithi163 bash[33912]: cluster 2023-10-09T09:09:50.853990+0000 osd.5 (osd.5) 13 : cluster [WRN] failed to encode map e329 with expected crc 2023-10-09T09:09:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:52 smithi163 bash[33912]: cluster 2023-10-09T09:09:50.954892+0000 osd.6 (osd.6) 14 : cluster [WRN] failed to encode map e330 with expected crc 2023-10-09T09:09:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:52 smithi163 bash[33912]: cluster 2023-10-09T09:09:50.956003+0000 osd.1 (osd.1) 10 : cluster [WRN] failed to encode map e330 with expected crc 2023-10-09T09:09:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:52 smithi163 bash[33912]: cluster 2023-10-09T09:09:50.959027+0000 osd.5 (osd.5) 14 : cluster [WRN] failed to encode map e330 with expected crc 2023-10-09T09:09:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:52 smithi163 bash[33912]: cluster 2023-10-09T09:09:50.969930+0000 osd.2 (osd.2) 27 : cluster [WRN] failed to encode map e330 with expected crc 2023-10-09T09:09:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:52 smithi163 bash[33912]: cluster 2023-10-09T09:09:50.984685+0000 osd.0 (osd.0) 21 : cluster [WRN] failed to encode map e330 with expected crc 2023-10-09T09:09:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:52 smithi163 bash[33912]: cluster 2023-10-09T09:09:51.216143+0000 mgr.y (mgr.34107) 29 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 110 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:09:52.509 INFO:tasks.rados.rados.0.smithi163.stdout:1090: finishing write tid 1 to smithi16367185-48 2023-10-09T09:09:52.509 INFO:tasks.rados.rados.0.smithi163.stdout:1086: expect (ObjNum 416 snap 127 seq_num 416) 2023-10-09T09:09:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:52 smithi181 bash[27712]: debug 2023-10-09T09:09:52.079+0000 7fde83035700 -1 mon.b@1(peon).osd e331 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:52 smithi181 bash[27712]: cluster 2023-10-09T09:09:50.853990+0000 osd.5 (osd.5) 13 : cluster [WRN] failed to encode map e329 with expected crc 2023-10-09T09:09:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:52 smithi181 bash[27712]: cluster 2023-10-09T09:09:50.954892+0000 osd.6 (osd.6) 14 : cluster [WRN] failed to encode map e330 with expected crc 2023-10-09T09:09:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:52 smithi181 bash[27712]: cluster 2023-10-09T09:09:50.956003+0000 osd.1 (osd.1) 10 : cluster [WRN] failed to encode map e330 with expected crc 2023-10-09T09:09:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:52 smithi181 bash[27712]: cluster 2023-10-09T09:09:50.959027+0000 osd.5 (osd.5) 14 : cluster [WRN] failed to encode map e330 with expected crc 2023-10-09T09:09:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:52 smithi181 bash[27712]: cluster 2023-10-09T09:09:50.969930+0000 osd.2 (osd.2) 27 : cluster [WRN] failed to encode map e330 with expected crc 2023-10-09T09:09:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:52 smithi181 bash[27712]: cluster 2023-10-09T09:09:50.984685+0000 osd.0 (osd.0) 21 : cluster [WRN] failed to encode map e330 with expected crc 2023-10-09T09:09:52.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:52 smithi181 bash[27712]: cluster 2023-10-09T09:09:51.216143+0000 mgr.y (mgr.34107) 29 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 110 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:09:52.628 INFO:tasks.rados.rados.0.smithi163.stdout:1090: finishing write tid 2 to smithi16367185-48 2023-10-09T09:09:52.629 INFO:tasks.rados.rados.0.smithi163.stdout:1090: finishing write tid 3 to smithi16367185-48 2023-10-09T09:09:52.629 INFO:tasks.rados.rados.0.smithi163.stdout:1090: finishing write tid 4 to smithi16367185-48 2023-10-09T09:09:52.629 INFO:tasks.rados.rados.0.smithi163.stdout:1091: finishing rollback tid 0 to smithi16367185-2 2023-10-09T09:09:52.629 INFO:tasks.rados.rados.0.smithi163.stdout:1090: finishing write tid 5 to smithi16367185-48 2023-10-09T09:09:52.629 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 886 (ObjNum 432 snap 135 seq_num 432) dirty exists 2023-10-09T09:09:52.629 INFO:tasks.rados.rados.0.smithi163.stdout:1090: left oid 48 (ObjNum 432 snap 135 seq_num 432) 2023-10-09T09:09:52.629 INFO:tasks.rados.rados.0.smithi163.stdout:1091: finishing rollback tid 1 to smithi16367185-2 2023-10-09T09:09:52.629 INFO:tasks.rados.rados.0.smithi163.stdout:1091: finishing rollback tid 2 to smithi16367185-2 2023-10-09T09:09:52.629 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 334 (ObjNum 376 snap 117 seq_num 376) dirty exists 2023-10-09T09:09:52.629 INFO:tasks.rados.rados.0.smithi163.stdout:1083: done (7 left) 2023-10-09T09:09:52.630 INFO:tasks.rados.rados.0.smithi163.stdout:1085: done (6 left) 2023-10-09T09:09:52.630 INFO:tasks.rados.rados.0.smithi163.stdout:1086: done (5 left) 2023-10-09T09:09:52.630 INFO:tasks.rados.rados.0.smithi163.stdout:1088: done (4 left) 2023-10-09T09:09:52.630 INFO:tasks.rados.rados.0.smithi163.stdout:1089: done (3 left) 2023-10-09T09:09:52.630 INFO:tasks.rados.rados.0.smithi163.stdout:1090: done (2 left) 2023-10-09T09:09:52.630 INFO:tasks.rados.rados.0.smithi163.stdout:1091: done (1 left) 2023-10-09T09:09:52.630 INFO:tasks.rados.rados.0.smithi163.stdout:1092: done (0 left) 2023-10-09T09:09:52.630 INFO:tasks.rados.rados.0.smithi163.stdout:1093: rollback oid 7 current snap is 135 2023-10-09T09:09:52.630 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 126 2023-10-09T09:09:52.631 INFO:tasks.rados.rados.0.smithi163.stdout:1094: snap_create 2023-10-09T09:09:52.631 INFO:tasks.rados.rados.0.smithi163.stdout:1093: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:09:52.632 INFO:tasks.rados.rados.0.smithi163.stdout:1093: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:09:52.632 INFO:tasks.rados.rados.0.smithi163.stdout:1093: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:09:52.632 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 270 (ObjNum 395 snap 121 seq_num 395) dirty exists 2023-10-09T09:09:53.100 INFO:tasks.rados.rados.0.smithi163.stdout:1093: done (1 left) 2023-10-09T09:09:53.100 INFO:tasks.rados.rados.0.smithi163.stdout:1094: done (0 left) 2023-10-09T09:09:53.100 INFO:tasks.rados.rados.0.smithi163.stdout:1095: rollback oid 11 current snap is 136 2023-10-09T09:09:53.100 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 11 to 125 2023-10-09T09:09:53.100 INFO:tasks.rados.rados.0.smithi163.stdout:1096: snap_remove snap 134 2023-10-09T09:09:53.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[33912]: cluster 2023-10-09T09:09:52.086205+0000 osd.4 (osd.4) 28 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[33912]: cluster 2023-10-09T09:09:52.090658+0000 mon.a (mon.0) 102 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2023-10-09T09:09:53.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[33912]: cluster 2023-10-09T09:09:52.091601+0000 osd.7 (osd.7) 25 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[33912]: cluster 2023-10-09T09:09:52.091710+0000 osd.6 (osd.6) 15 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[33912]: cluster 2023-10-09T09:09:52.091924+0000 osd.5 (osd.5) 15 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[33912]: cluster 2023-10-09T09:09:52.092096+0000 osd.1 (osd.1) 11 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[33912]: cluster 2023-10-09T09:09:52.097382+0000 osd.1 (osd.1) 12 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[33912]: cluster 2023-10-09T09:09:52.097486+0000 osd.2 (osd.2) 28 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[33912]: cluster 2023-10-09T09:09:52.097527+0000 osd.3 (osd.3) 19 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[33912]: debug 2023-10-09T09:09:53.090+0000 7f84df46a700 -1 mon.c@2(peon).osd e332 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:53 smithi181 bash[27712]: cluster 2023-10-09T09:09:52.086205+0000 osd.4 (osd.4) 28 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:53 smithi181 bash[27712]: cluster 2023-10-09T09:09:52.090658+0000 mon.a (mon.0) 102 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2023-10-09T09:09:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:53 smithi181 bash[27712]: cluster 2023-10-09T09:09:52.091601+0000 osd.7 (osd.7) 25 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:53 smithi181 bash[27712]: cluster 2023-10-09T09:09:52.091710+0000 osd.6 (osd.6) 15 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:53 smithi181 bash[27712]: cluster 2023-10-09T09:09:52.091924+0000 osd.5 (osd.5) 15 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:53 smithi181 bash[27712]: cluster 2023-10-09T09:09:52.092096+0000 osd.1 (osd.1) 11 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:53 smithi181 bash[27712]: cluster 2023-10-09T09:09:52.097382+0000 osd.1 (osd.1) 12 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:53 smithi181 bash[27712]: cluster 2023-10-09T09:09:52.097486+0000 osd.2 (osd.2) 28 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:53 smithi181 bash[27712]: cluster 2023-10-09T09:09:52.097527+0000 osd.3 (osd.3) 19 : cluster [WRN] failed to encode map e331 with expected crc 2023-10-09T09:09:53.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:53 smithi181 bash[27712]: debug 2023-10-09T09:09:53.087+0000 7fde83035700 -1 mon.b@1(peon).osd e332 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:09:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:09:53] "GET /metrics HTTP/1.1" 200 36086 "" "Prometheus/2.33.4" 2023-10-09T09:09:54.109 INFO:tasks.rados.rados.0.smithi163.stdout:1096: done (1 left) 2023-10-09T09:09:54.109 INFO:tasks.rados.rados.0.smithi163.stdout:1097: write oid 26 current snap is 136 2023-10-09T09:09:54.109 INFO:tasks.rados.rados.0.smithi163.stdout:1095: finishing rollback tid 0 to smithi16367185-11 2023-10-09T09:09:54.109 INFO:tasks.rados.rados.0.smithi163.stdout:1095: finishing rollback tid 1 to smithi16367185-11 2023-10-09T09:09:54.110 INFO:tasks.rados.rados.0.smithi163.stdout:1095: finishing rollback tid 2 to smithi16367185-11 2023-10-09T09:09:54.110 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 338 (ObjNum 360 snap 108 seq_num 360) dirty exists 2023-10-09T09:09:54.110 INFO:tasks.rados.rados.0.smithi163.stdout:1097: seq_num 434 ranges {494238=490187,1563538=701819,2818520=475718} 2023-10-09T09:09:54.120 INFO:tasks.rados.rados.0.smithi163.stdout:1097: writing smithi16367185-26 from 494238 to 984425 tid 1 2023-10-09T09:09:54.129 INFO:tasks.rados.rados.0.smithi163.stdout:1097: writing smithi16367185-26 from 1563538 to 2265357 tid 2 2023-10-09T09:09:54.135 INFO:tasks.rados.rados.0.smithi163.stdout:1097: writing smithi16367185-26 from 2818520 to 3294238 tid 3 2023-10-09T09:09:54.135 INFO:tasks.rados.rados.0.smithi163.stdout:1095: done (1 left) 2023-10-09T09:09:54.135 INFO:tasks.rados.rados.0.smithi163.stdout:1098: read oid 30 snap -1 2023-10-09T09:09:54.136 INFO:tasks.rados.rados.0.smithi163.stdout:1098: expect deleted 2023-10-09T09:09:54.136 INFO:tasks.rados.rados.0.smithi163.stdout:1099: write oid 10 current snap is 136 2023-10-09T09:09:54.136 INFO:tasks.rados.rados.0.smithi163.stdout:1097: finishing write tid 1 to smithi16367185-26 2023-10-09T09:09:54.136 INFO:tasks.rados.rados.0.smithi163.stdout:1097: finishing write tid 2 to smithi16367185-26 2023-10-09T09:09:54.136 INFO:tasks.rados.rados.0.smithi163.stdout:1099: seq_num 435 ranges {511045=565436,1622020=445477,2729590=556855} 2023-10-09T09:09:54.141 INFO:tasks.rados.rados.0.smithi163.stdout:1099: writing smithi16367185-10 from 511045 to 1076481 tid 1 2023-10-09T09:09:54.147 INFO:tasks.rados.rados.0.smithi163.stdout:1099: writing smithi16367185-10 from 1622020 to 2067497 tid 2 2023-10-09T09:09:54.154 INFO:tasks.rados.rados.0.smithi163.stdout:1099: writing smithi16367185-10 from 2729590 to 3286445 tid 3 2023-10-09T09:09:54.154 INFO:tasks.rados.rados.0.smithi163.stdout:1100: write oid 5 current snap is 136 2023-10-09T09:09:54.154 INFO:tasks.rados.rados.0.smithi163.stdout:1100: seq_num 436 ranges {527852=640685,1680502=589135,2757013=570839} 2023-10-09T09:09:54.161 INFO:tasks.rados.rados.0.smithi163.stdout:1100: writing smithi16367185-5 from 527852 to 1168537 tid 1 2023-10-09T09:09:54.167 INFO:tasks.rados.rados.0.smithi163.stdout:1100: writing smithi16367185-5 from 1680502 to 2269637 tid 2 2023-10-09T09:09:54.175 INFO:tasks.rados.rados.0.smithi163.stdout:1100: writing smithi16367185-5 from 2757013 to 3327852 tid 3 2023-10-09T09:09:54.175 INFO:tasks.rados.rados.0.smithi163.stdout:1101: read oid 7 snap -1 2023-10-09T09:09:54.175 INFO:tasks.rados.rados.0.smithi163.stdout:1101: expect (ObjNum 395 snap 121 seq_num 395) 2023-10-09T09:09:54.175 INFO:tasks.rados.rados.0.smithi163.stdout:1102: rollback oid 24 current snap is 136 2023-10-09T09:09:54.176 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 24 to 132 2023-10-09T09:09:54.176 INFO:tasks.rados.rados.0.smithi163.stdout:1097: finishing write tid 3 to smithi16367185-26 2023-10-09T09:09:54.176 INFO:tasks.rados.rados.0.smithi163.stdout:1098: done (5 left) 2023-10-09T09:09:54.176 INFO:tasks.rados.rados.0.smithi163.stdout:1103: delete oid 36 current snap is 136 2023-10-09T09:09:54.176 INFO:tasks.rados.rados.0.smithi163.stdout:1097: finishing write tid 4 to smithi16367185-26 2023-10-09T09:09:54.177 INFO:tasks.rados.rados.0.smithi163.stdout:1097: finishing write tid 5 to smithi16367185-26 2023-10-09T09:09:54.177 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 365 (ObjNum 434 snap 136 seq_num 434) dirty exists 2023-10-09T09:09:54.177 INFO:tasks.rados.rados.0.smithi163.stdout:1097: left oid 26 (ObjNum 434 snap 136 seq_num 434) 2023-10-09T09:09:54.177 INFO:tasks.rados.rados.0.smithi163.stdout:1099: finishing write tid 1 to smithi16367185-10 2023-10-09T09:09:54.177 INFO:tasks.rados.rados.0.smithi163.stdout:1099: finishing write tid 2 to smithi16367185-10 2023-10-09T09:09:54.177 INFO:tasks.rados.rados.0.smithi163.stdout:1099: finishing write tid 3 to smithi16367185-10 2023-10-09T09:09:54.178 INFO:tasks.rados.rados.0.smithi163.stdout:1099: finishing write tid 4 to smithi16367185-10 2023-10-09T09:09:54.178 INFO:tasks.rados.rados.0.smithi163.stdout:1099: finishing write tid 5 to smithi16367185-10 2023-10-09T09:09:54.178 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 422 (ObjNum 435 snap 136 seq_num 435) dirty exists 2023-10-09T09:09:54.178 INFO:tasks.rados.rados.0.smithi163.stdout:1099: left oid 10 (ObjNum 435 snap 136 seq_num 435) 2023-10-09T09:09:54.178 INFO:tasks.rados.rados.0.smithi163.stdout:1100: finishing write tid 1 to smithi16367185-5 2023-10-09T09:09:54.179 INFO:tasks.rados.rados.0.smithi163.stdout:1100: finishing write tid 2 to smithi16367185-5 2023-10-09T09:09:54.180 INFO:tasks.rados.rados.0.smithi163.stdout:1100: finishing write tid 3 to smithi16367185-5 2023-10-09T09:09:54.180 INFO:tasks.rados.rados.0.smithi163.stdout:1101: expect (ObjNum 395 snap 121 seq_num 395) 2023-10-09T09:09:54.271 INFO:tasks.rados.rados.0.smithi163.stdout:1100: finishing write tid 4 to smithi16367185-5 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:1102: finishing rollback tid 0 to smithi16367185-24 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:1102: finishing rollback tid 1 to smithi16367185-24 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:1102: finishing rollback tid 2 to smithi16367185-24 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 306 (ObjNum 426 snap 132 seq_num 426) dirty exists 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:1100: finishing write tid 5 to smithi16367185-5 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 605 (ObjNum 436 snap 136 seq_num 436) dirty exists 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:1100: left oid 5 (ObjNum 436 snap 136 seq_num 436) 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:1097: done (5 left) 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:1099: done (4 left) 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:1100: done (3 left) 2023-10-09T09:09:54.272 INFO:tasks.rados.rados.0.smithi163.stdout:1101: done (2 left) 2023-10-09T09:09:54.273 INFO:tasks.rados.rados.0.smithi163.stdout:1102: done (1 left) 2023-10-09T09:09:54.273 INFO:tasks.rados.rados.0.smithi163.stdout:1103: done (0 left) 2023-10-09T09:09:54.273 INFO:tasks.rados.rados.0.smithi163.stdout:1104: write oid 14 current snap is 136 2023-10-09T09:09:54.273 INFO:tasks.rados.rados.0.smithi163.stdout:1104: seq_num 438 ranges {561466=791183,2081113=592804,3095506=265960} 2023-10-09T09:09:54.284 INFO:tasks.rados.rados.0.smithi163.stdout:1104: writing smithi16367185-14 from 561466 to 1352649 tid 1 2023-10-09T09:09:54.293 INFO:tasks.rados.rados.0.smithi163.stdout:1104: writing smithi16367185-14 from 2081113 to 2673917 tid 2 2023-10-09T09:09:54.297 INFO:tasks.rados.rados.0.smithi163.stdout:1104: writing smithi16367185-14 from 3095506 to 3361466 tid 3 2023-10-09T09:09:54.297 INFO:tasks.rados.rados.0.smithi163.stdout:1105: write oid 31 current snap is 136 2023-10-09T09:09:54.297 INFO:tasks.rados.rados.0.smithi163.stdout:1104: finishing write tid 1 to smithi16367185-14 2023-10-09T09:09:54.297 INFO:tasks.rados.rados.0.smithi163.stdout:1104: finishing write tid 2 to smithi16367185-14 2023-10-09T09:09:54.298 INFO:tasks.rados.rados.0.smithi163.stdout:1105: seq_num 439 ranges {578273=466432,1739595=452815,2722929=655344} 2023-10-09T09:09:54.302 INFO:tasks.rados.rados.0.smithi163.stdout:1105: writing smithi16367185-31 from 578273 to 1044705 tid 1 2023-10-09T09:09:54.309 INFO:tasks.rados.rados.0.smithi163.stdout:1105: writing smithi16367185-31 from 1739595 to 2192410 tid 2 2023-10-09T09:09:54.316 INFO:tasks.rados.rados.0.smithi163.stdout:1105: writing smithi16367185-31 from 2722929 to 3378273 tid 3 2023-10-09T09:09:54.316 INFO:tasks.rados.rados.0.smithi163.stdout:1106: snap_create 2023-10-09T09:09:54.316 INFO:tasks.rados.rados.0.smithi163.stdout:1104: finishing write tid 3 to smithi16367185-14 2023-10-09T09:09:54.316 INFO:tasks.rados.rados.0.smithi163.stdout:1104: finishing write tid 4 to smithi16367185-14 2023-10-09T09:09:54.316 INFO:tasks.rados.rados.0.smithi163.stdout:1104: finishing write tid 5 to smithi16367185-14 2023-10-09T09:09:54.316 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 440 (ObjNum 438 snap 136 seq_num 438) dirty exists 2023-10-09T09:09:54.316 INFO:tasks.rados.rados.0.smithi163.stdout:1104: left oid 14 (ObjNum 438 snap 136 seq_num 438) 2023-10-09T09:09:54.317 INFO:tasks.rados.rados.0.smithi163.stdout:1105: finishing write tid 1 to smithi16367185-31 2023-10-09T09:09:54.318 INFO:tasks.rados.rados.0.smithi163.stdout:1105: finishing write tid 2 to smithi16367185-31 2023-10-09T09:09:54.320 INFO:tasks.rados.rados.0.smithi163.stdout:1105: finishing write tid 3 to smithi16367185-31 2023-10-09T09:09:54.320 INFO:tasks.rados.rados.0.smithi163.stdout:1105: finishing write tid 4 to smithi16367185-31 2023-10-09T09:09:54.321 INFO:tasks.rados.rados.0.smithi163.stdout:1105: finishing write tid 5 to smithi16367185-31 2023-10-09T09:09:54.321 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 343 (ObjNum 439 snap 136 seq_num 439) dirty exists 2023-10-09T09:09:54.321 INFO:tasks.rados.rados.0.smithi163.stdout:1105: left oid 31 (ObjNum 439 snap 136 seq_num 439) 2023-10-09T09:09:54.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:54 smithi163 bash[33912]: cluster 2023-10-09T09:09:53.093695+0000 mon.a (mon.0) 103 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2023-10-09T09:09:54.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:54 smithi163 bash[33912]: cluster 2023-10-09T09:09:53.096443+0000 osd.4 (osd.4) 29 : cluster [WRN] failed to encode map e332 with expected crc 2023-10-09T09:09:54.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:54 smithi163 bash[33912]: cluster 2023-10-09T09:09:53.098761+0000 osd.0 (osd.0) 22 : cluster [WRN] failed to encode map e332 with expected crc 2023-10-09T09:09:54.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:54 smithi163 bash[33912]: cluster 2023-10-09T09:09:53.099661+0000 osd.7 (osd.7) 26 : cluster [WRN] failed to encode map e332 with expected crc 2023-10-09T09:09:54.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:54 smithi163 bash[33912]: cluster 2023-10-09T09:09:53.216875+0000 mgr.y (mgr.34107) 30 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 120 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2023-10-09T09:09:54.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:54 smithi163 bash[33912]: debug 2023-10-09T09:09:54.106+0000 7f84df46a700 -1 mon.c@2(peon).osd e333 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:54 smithi181 bash[27712]: cluster 2023-10-09T09:09:53.093695+0000 mon.a (mon.0) 103 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2023-10-09T09:09:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:54 smithi181 bash[27712]: cluster 2023-10-09T09:09:53.096443+0000 osd.4 (osd.4) 29 : cluster [WRN] failed to encode map e332 with expected crc 2023-10-09T09:09:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:54 smithi181 bash[27712]: cluster 2023-10-09T09:09:53.098761+0000 osd.0 (osd.0) 22 : cluster [WRN] failed to encode map e332 with expected crc 2023-10-09T09:09:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:54 smithi181 bash[27712]: cluster 2023-10-09T09:09:53.099661+0000 osd.7 (osd.7) 26 : cluster [WRN] failed to encode map e332 with expected crc 2023-10-09T09:09:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:54 smithi181 bash[27712]: cluster 2023-10-09T09:09:53.216875+0000 mgr.y (mgr.34107) 30 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 120 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2023-10-09T09:09:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:54 smithi181 bash[27712]: debug 2023-10-09T09:09:54.099+0000 7fde83035700 -1 mon.b@1(peon).osd e333 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:54.946 INFO:tasks.rados.rados.0.smithi163.stdout:1104: done (2 left) 2023-10-09T09:09:54.947 INFO:tasks.rados.rados.0.smithi163.stdout:1105: done (1 left) 2023-10-09T09:09:54.947 INFO:tasks.rados.rados.0.smithi163.stdout:1106: done (0 left) 2023-10-09T09:09:54.947 INFO:tasks.rados.rados.0.smithi163.stdout:1107: read oid 45 snap -1 2023-10-09T09:09:54.947 INFO:tasks.rados.rados.0.smithi163.stdout:1107: expect deleted 2023-10-09T09:09:54.947 INFO:tasks.rados.rados.0.smithi163.stdout:1108: snap_create 2023-10-09T09:09:55.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:54 smithi163 bash[33912]: debug 2023-10-09T09:09:54.934+0000 7f84df46a700 -1 mon.c@2(peon).osd e334 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:55.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:55 smithi163 bash[33912]: cluster 2023-10-09T09:09:54.105979+0000 osd.5 (osd.5) 16 : cluster [WRN] failed to encode map e333 with expected crc 2023-10-09T09:09:55.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:55 smithi163 bash[33912]: cluster 2023-10-09T09:09:54.107686+0000 mon.a (mon.0) 104 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2023-10-09T09:09:55.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:55 smithi163 bash[33912]: cluster 2023-10-09T09:09:54.122793+0000 osd.2 (osd.2) 29 : cluster [WRN] failed to encode map e333 with expected crc 2023-10-09T09:09:55.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:55 smithi163 bash[33912]: cluster 2023-10-09T09:09:54.940564+0000 mon.a (mon.0) 105 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2023-10-09T09:09:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:54 smithi181 bash[27712]: debug 2023-10-09T09:09:54.927+0000 7fde83035700 -1 mon.b@1(peon).osd e334 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:55 smithi181 bash[27712]: cluster 2023-10-09T09:09:54.105979+0000 osd.5 (osd.5) 16 : cluster [WRN] failed to encode map e333 with expected crc 2023-10-09T09:09:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:55 smithi181 bash[27712]: cluster 2023-10-09T09:09:54.107686+0000 mon.a (mon.0) 104 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2023-10-09T09:09:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:55 smithi181 bash[27712]: cluster 2023-10-09T09:09:54.122793+0000 osd.2 (osd.2) 29 : cluster [WRN] failed to encode map e333 with expected crc 2023-10-09T09:09:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:55 smithi181 bash[27712]: cluster 2023-10-09T09:09:54.940564+0000 mon.a (mon.0) 105 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2023-10-09T09:09:55.963 INFO:tasks.rados.rados.0.smithi163.stdout:1107: done (1 left) 2023-10-09T09:09:55.964 INFO:tasks.rados.rados.0.smithi163.stdout:1108: done (0 left) 2023-10-09T09:09:55.964 INFO:tasks.rados.rados.0.smithi163.stdout:1109: read oid 37 snap -1 2023-10-09T09:09:55.964 INFO:tasks.rados.rados.0.smithi163.stdout:1109: expect deleted 2023-10-09T09:09:55.964 INFO:tasks.rados.rados.0.smithi163.stdout:1110: rollback oid 38 current snap is 138 2023-10-09T09:09:55.964 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 135 2023-10-09T09:09:55.964 INFO:tasks.rados.rados.0.smithi163.stdout:1111: write oid 1 current snap is 138 2023-10-09T09:09:55.964 INFO:tasks.rados.rados.0.smithi163.stdout:1111: seq_num 440 ranges {595080=541681,1798077=596473,3150352=244728} 2023-10-09T09:09:55.970 INFO:tasks.rados.rados.0.smithi163.stdout:1111: writing smithi16367185-1 from 595080 to 1136761 tid 1 2023-10-09T09:09:55.977 INFO:tasks.rados.rados.0.smithi163.stdout:1111: writing smithi16367185-1 from 1798077 to 2394550 tid 2 2023-10-09T09:09:55.983 INFO:tasks.rados.rados.0.smithi163.stdout:1111: writing smithi16367185-1 from 3150352 to 3395080 tid 3 2023-10-09T09:09:55.983 INFO:tasks.rados.rados.0.smithi163.stdout:1112: delete oid 7 current snap is 138 2023-10-09T09:09:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:1110: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:09:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:1110: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:09:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:1110: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:09:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 444 (ObjNum 421 snap 129 seq_num 421) dirty exists 2023-10-09T09:09:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:1111: finishing write tid 1 to smithi16367185-1 2023-10-09T09:09:55.984 INFO:tasks.rados.rados.0.smithi163.stdout:1111: finishing write tid 2 to smithi16367185-1 2023-10-09T09:09:55.986 INFO:tasks.rados.rados.0.smithi163.stdout:1111: finishing write tid 3 to smithi16367185-1 2023-10-09T09:09:55.987 INFO:tasks.rados.rados.0.smithi163.stdout:1111: finishing write tid 4 to smithi16367185-1 2023-10-09T09:09:55.987 INFO:tasks.rados.rados.0.smithi163.stdout:1111: finishing write tid 5 to smithi16367185-1 2023-10-09T09:09:55.987 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 610 (ObjNum 440 snap 138 seq_num 440) dirty exists 2023-10-09T09:09:55.987 INFO:tasks.rados.rados.0.smithi163.stdout:1111: left oid 1 (ObjNum 440 snap 138 seq_num 440) 2023-10-09T09:09:55.990 INFO:tasks.rados.rados.0.smithi163.stdout:1109: done (3 left) 2023-10-09T09:09:55.990 INFO:tasks.rados.rados.0.smithi163.stdout:1110: done (2 left) 2023-10-09T09:09:55.990 INFO:tasks.rados.rados.0.smithi163.stdout:1111: done (1 left) 2023-10-09T09:09:55.990 INFO:tasks.rados.rados.0.smithi163.stdout:1112: done (0 left) 2023-10-09T09:09:55.991 INFO:tasks.rados.rados.0.smithi163.stdout:1113: rollback oid 45 current snap is 138 2023-10-09T09:09:55.991 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 45 to 124 2023-10-09T09:09:55.991 INFO:tasks.rados.rados.0.smithi163.stdout:1114: write oid 16 current snap is 138 2023-10-09T09:09:55.991 INFO:tasks.rados.rados.0.smithi163.stdout:1114: seq_num 442 ranges {628694=408532,1515041=600142,2805198=552112} 2023-10-09T09:09:55.999 INFO:tasks.rados.rados.0.smithi163.stdout:1114: writing smithi16367185-16 from 628694 to 1037226 tid 1 2023-10-09T09:09:56.008 INFO:tasks.rados.rados.0.smithi163.stdout:1114: writing smithi16367185-16 from 1515041 to 2115183 tid 2 2023-10-09T09:09:56.017 INFO:tasks.rados.rados.0.smithi163.stdout:1114: writing smithi16367185-16 from 2805198 to 3357310 tid 3 2023-10-09T09:09:56.017 INFO:tasks.rados.rados.0.smithi163.stdout:1115: write oid 25 current snap is 138 2023-10-09T09:09:56.017 INFO:tasks.rados.rados.0.smithi163.stdout:1115: seq_num 443 ranges {645501=483781,1573523=460153,2548974=763384} 2023-10-09T09:09:56.023 INFO:tasks.rados.rados.0.smithi163.stdout:1115: writing smithi16367185-25 from 645501 to 1129282 tid 1 2023-10-09T09:09:56.029 INFO:tasks.rados.rados.0.smithi163.stdout:1115: writing smithi16367185-25 from 1573523 to 2033676 tid 2 2023-10-09T09:09:56.037 INFO:tasks.rados.rados.0.smithi163.stdout:1115: writing smithi16367185-25 from 2548974 to 3312358 tid 3 2023-10-09T09:09:56.037 INFO:tasks.rados.rados.0.smithi163.stdout:1116: read oid 18 snap -1 2023-10-09T09:09:56.037 INFO:tasks.rados.rados.0.smithi163.stdout:1116: expect (ObjNum 379 snap 117 seq_num 379) 2023-10-09T09:09:56.037 INFO:tasks.rados.rados.0.smithi163.stdout:1113: finishing rollback tid 1 to smithi16367185-45 2023-10-09T09:09:56.037 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 858 (ObjNum 698906464 snap 21875 seq_num 0) dirty dne 2023-10-09T09:09:56.037 INFO:tasks.rados.rados.0.smithi163.stdout:1114: finishing write tid 1 to smithi16367185-16 2023-10-09T09:09:56.037 INFO:tasks.rados.rados.0.smithi163.stdout:1114: finishing write tid 2 to smithi16367185-16 2023-10-09T09:09:56.038 INFO:tasks.rados.rados.0.smithi163.stdout:1114: finishing write tid 3 to smithi16367185-16 2023-10-09T09:09:56.038 INFO:tasks.rados.rados.0.smithi163.stdout:1113: done (3 left) 2023-10-09T09:09:56.038 INFO:tasks.rados.rados.0.smithi163.stdout:1117: read oid 50 snap -1 2023-10-09T09:09:56.038 INFO:tasks.rados.rados.0.smithi163.stdout:1117: expect (ObjNum 409 snap 124 seq_num 409) 2023-10-09T09:09:56.038 INFO:tasks.rados.rados.0.smithi163.stdout:1118: snap_create 2023-10-09T09:09:56.038 INFO:tasks.rados.rados.0.smithi163.stdout:1114: finishing write tid 4 to smithi16367185-16 2023-10-09T09:09:56.038 INFO:tasks.rados.rados.0.smithi163.stdout:1114: finishing write tid 5 to smithi16367185-16 2023-10-09T09:09:56.038 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 615 (ObjNum 442 snap 138 seq_num 442) dirty exists 2023-10-09T09:09:56.038 INFO:tasks.rados.rados.0.smithi163.stdout:1114: left oid 16 (ObjNum 442 snap 138 seq_num 442) 2023-10-09T09:09:56.039 INFO:tasks.rados.rados.0.smithi163.stdout:1115: finishing write tid 1 to smithi16367185-25 2023-10-09T09:09:56.039 INFO:tasks.rados.rados.0.smithi163.stdout:1115: finishing write tid 2 to smithi16367185-25 2023-10-09T09:09:56.042 INFO:tasks.rados.rados.0.smithi163.stdout:1115: finishing write tid 3 to smithi16367185-25 2023-10-09T09:09:56.043 INFO:tasks.rados.rados.0.smithi163.stdout:1116: expect (ObjNum 379 snap 117 seq_num 379) 2023-10-09T09:09:56.162 INFO:tasks.rados.rados.0.smithi163.stdout:1115: finishing write tid 4 to smithi16367185-25 2023-10-09T09:09:56.162 INFO:tasks.rados.rados.0.smithi163.stdout:1115: finishing write tid 5 to smithi16367185-25 2023-10-09T09:09:56.162 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 433 (ObjNum 443 snap 138 seq_num 443) dirty exists 2023-10-09T09:09:56.163 INFO:tasks.rados.rados.0.smithi163.stdout:1115: left oid 25 (ObjNum 443 snap 138 seq_num 443) 2023-10-09T09:09:56.163 INFO:tasks.rados.rados.0.smithi163.stdout:1117: expect (ObjNum 409 snap 124 seq_num 409) 2023-10-09T09:09:56.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:55 smithi163 bash[33912]: debug 2023-10-09T09:09:55.950+0000 7f84df46a700 -1 mon.c@2(peon).osd e335 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:56.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:56 smithi163 bash[33912]: cluster 2023-10-09T09:09:54.942646+0000 osd.4 (osd.4) 30 : cluster [WRN] failed to encode map e334 with expected crc 2023-10-09T09:09:56.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:56 smithi163 bash[33912]: cluster 2023-10-09T09:09:54.944214+0000 osd.3 (osd.3) 20 : cluster [WRN] failed to encode map e334 with expected crc 2023-10-09T09:09:56.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:56 smithi163 bash[33912]: cluster 2023-10-09T09:09:54.946716+0000 osd.5 (osd.5) 17 : cluster [WRN] failed to encode map e334 with expected crc 2023-10-09T09:09:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:56 smithi163 bash[33912]: cluster 2023-10-09T09:09:55.217598+0000 mgr.y (mgr.34107) 31 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 120 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2023-10-09T09:09:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:56 smithi163 bash[33912]: cluster 2023-10-09T09:09:55.321351+0000 osd.0 (osd.0) 23 : cluster [WRN] failed to encode map e334 with expected crc 2023-10-09T09:09:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:56 smithi163 bash[33912]: audit 2023-10-09T09:09:55.325846+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:56 smithi163 bash[33912]: cluster 2023-10-09T09:09:55.348012+0000 osd.7 (osd.7) 27 : cluster [WRN] failed to encode map e334 with expected crc 2023-10-09T09:09:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:56 smithi163 bash[33912]: cluster 2023-10-09T09:09:55.956250+0000 mon.a (mon.0) 107 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2023-10-09T09:09:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:55 smithi181 bash[27712]: debug 2023-10-09T09:09:55.947+0000 7fde83035700 -1 mon.b@1(peon).osd e335 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:56 smithi181 bash[27712]: cluster 2023-10-09T09:09:54.942646+0000 osd.4 (osd.4) 30 : cluster [WRN] failed to encode map e334 with expected crc 2023-10-09T09:09:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:56 smithi181 bash[27712]: cluster 2023-10-09T09:09:54.944214+0000 osd.3 (osd.3) 20 : cluster [WRN] failed to encode map e334 with expected crc 2023-10-09T09:09:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:56 smithi181 bash[27712]: cluster 2023-10-09T09:09:54.946716+0000 osd.5 (osd.5) 17 : cluster [WRN] failed to encode map e334 with expected crc 2023-10-09T09:09:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:56 smithi181 bash[27712]: cluster 2023-10-09T09:09:55.217598+0000 mgr.y (mgr.34107) 31 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 120 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2023-10-09T09:09:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:56 smithi181 bash[27712]: cluster 2023-10-09T09:09:55.321351+0000 osd.0 (osd.0) 23 : cluster [WRN] failed to encode map e334 with expected crc 2023-10-09T09:09:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:56 smithi181 bash[27712]: audit 2023-10-09T09:09:55.325846+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:09:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:56 smithi181 bash[27712]: cluster 2023-10-09T09:09:55.348012+0000 osd.7 (osd.7) 27 : cluster [WRN] failed to encode map e334 with expected crc 2023-10-09T09:09:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:56 smithi181 bash[27712]: cluster 2023-10-09T09:09:55.956250+0000 mon.a (mon.0) 107 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2023-10-09T09:09:56.967 INFO:tasks.rados.rados.0.smithi163.stdout:1114: done (4 left) 2023-10-09T09:09:56.968 INFO:tasks.rados.rados.0.smithi163.stdout:1115: done (3 left) 2023-10-09T09:09:56.968 INFO:tasks.rados.rados.0.smithi163.stdout:1116: done (2 left) 2023-10-09T09:09:56.968 INFO:tasks.rados.rados.0.smithi163.stdout:1117: done (1 left) 2023-10-09T09:09:56.968 INFO:tasks.rados.rados.0.smithi163.stdout:1118: done (0 left) 2023-10-09T09:09:56.968 INFO:tasks.rados.rados.0.smithi163.stdout:1119: rollback oid 34 current snap is 139 2023-10-09T09:09:56.968 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 34 to 126 2023-10-09T09:09:56.968 INFO:tasks.rados.rados.0.smithi163.stdout:1120: snap_create 2023-10-09T09:09:56.979 INFO:tasks.rados.rados.0.smithi163.stdout:1119: finishing rollback tid 0 to smithi16367185-34 2023-10-09T09:09:56.979 INFO:tasks.rados.rados.0.smithi163.stdout:1119: finishing rollback tid 1 to smithi16367185-34 2023-10-09T09:09:56.980 INFO:tasks.rados.rados.0.smithi163.stdout:1119: finishing rollback tid 2 to smithi16367185-34 2023-10-09T09:09:56.980 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 420 (ObjNum 358 snap 108 seq_num 358) dirty exists 2023-10-09T09:09:57.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:56 smithi163 bash[33912]: debug 2023-10-09T09:09:56.954+0000 7f84df46a700 -1 mon.c@2(peon).osd e336 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:57.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:57 smithi163 bash[33912]: cluster 2023-10-09T09:09:55.959622+0000 osd.4 (osd.4) 31 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:57.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:57 smithi163 bash[33912]: cluster 2023-10-09T09:09:55.960988+0000 osd.3 (osd.3) 21 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:57.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:57 smithi163 bash[33912]: cluster 2023-10-09T09:09:55.963686+0000 osd.7 (osd.7) 28 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:57.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:57 smithi163 bash[33912]: cluster 2023-10-09T09:09:55.972601+0000 osd.2 (osd.2) 30 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:57.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:57 smithi163 bash[33912]: cluster 2023-10-09T09:09:55.985282+0000 osd.1 (osd.1) 13 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:57.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:57 smithi163 bash[33912]: cluster 2023-10-09T09:09:56.960030+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2023-10-09T09:09:57.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:56 smithi181 bash[27712]: debug 2023-10-09T09:09:56.947+0000 7fde83035700 -1 mon.b@1(peon).osd e336 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:57.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:57 smithi181 bash[27712]: cluster 2023-10-09T09:09:55.959622+0000 osd.4 (osd.4) 31 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:57.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:57 smithi181 bash[27712]: cluster 2023-10-09T09:09:55.960988+0000 osd.3 (osd.3) 21 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:57.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:57 smithi181 bash[27712]: cluster 2023-10-09T09:09:55.963686+0000 osd.7 (osd.7) 28 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:57.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:57 smithi181 bash[27712]: cluster 2023-10-09T09:09:55.972601+0000 osd.2 (osd.2) 30 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:57.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:57 smithi181 bash[27712]: cluster 2023-10-09T09:09:55.985282+0000 osd.1 (osd.1) 13 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:57.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:57 smithi181 bash[27712]: cluster 2023-10-09T09:09:56.960030+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2023-10-09T09:09:57.969 INFO:tasks.rados.rados.0.smithi163.stdout:1119: done (1 left) 2023-10-09T09:09:57.970 INFO:tasks.rados.rados.0.smithi163.stdout:1120: done (0 left) 2023-10-09T09:09:57.970 INFO:tasks.rados.rados.0.smithi163.stdout:1121: snap_create 2023-10-09T09:09:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:57 smithi163 bash[33912]: debug 2023-10-09T09:09:57.954+0000 7f84df46a700 -1 mon.c@2(peon).osd e337 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:58 smithi163 bash[33912]: cluster 2023-10-09T09:09:56.498088+0000 osd.6 (osd.6) 16 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:58 smithi163 bash[33912]: cluster 2023-10-09T09:09:56.962801+0000 osd.4 (osd.4) 32 : cluster [WRN] failed to encode map e336 with expected crc 2023-10-09T09:09:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:58 smithi163 bash[33912]: cluster 2023-10-09T09:09:56.964565+0000 osd.3 (osd.3) 22 : cluster [WRN] failed to encode map e336 with expected crc 2023-10-09T09:09:58.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:58 smithi163 bash[33912]: cluster 2023-10-09T09:09:56.968497+0000 osd.5 (osd.5) 18 : cluster [WRN] failed to encode map e336 with expected crc 2023-10-09T09:09:58.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:58 smithi163 bash[33912]: cluster 2023-10-09T09:09:57.218386+0000 mgr.y (mgr.34107) 32 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.7 MiB/s wr, 33 op/s 2023-10-09T09:09:58.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:58 smithi163 bash[33912]: cluster 2023-10-09T09:09:57.962528+0000 mon.a (mon.0) 109 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2023-10-09T09:09:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:57 smithi181 bash[27712]: debug 2023-10-09T09:09:57.951+0000 7fde83035700 -1 mon.b@1(peon).osd e337 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:58.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:58 smithi181 bash[27712]: cluster 2023-10-09T09:09:56.498088+0000 osd.6 (osd.6) 16 : cluster [WRN] failed to encode map e335 with expected crc 2023-10-09T09:09:58.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:58 smithi181 bash[27712]: cluster 2023-10-09T09:09:56.962801+0000 osd.4 (osd.4) 32 : cluster [WRN] failed to encode map e336 with expected crc 2023-10-09T09:09:58.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:58 smithi181 bash[27712]: cluster 2023-10-09T09:09:56.964565+0000 osd.3 (osd.3) 22 : cluster [WRN] failed to encode map e336 with expected crc 2023-10-09T09:09:58.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:58 smithi181 bash[27712]: cluster 2023-10-09T09:09:56.968497+0000 osd.5 (osd.5) 18 : cluster [WRN] failed to encode map e336 with expected crc 2023-10-09T09:09:58.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:58 smithi181 bash[27712]: cluster 2023-10-09T09:09:57.218386+0000 mgr.y (mgr.34107) 32 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.7 MiB/s wr, 33 op/s 2023-10-09T09:09:58.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:58 smithi181 bash[27712]: cluster 2023-10-09T09:09:57.962528+0000 mon.a (mon.0) 109 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2023-10-09T09:09:58.961 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:09:58 smithi181 bash[54907]: ts=2023-10-09T09:09:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:09:58.970 INFO:tasks.rados.rados.0.smithi163.stdout:1121: done (0 left) 2023-10-09T09:09:58.971 INFO:tasks.rados.rados.0.smithi163.stdout:1122: write oid 16 current snap is 141 2023-10-09T09:09:58.971 INFO:tasks.rados.rados.0.smithi163.stdout:1122: seq_num 444 ranges {662308=559030,1632005=603811,2860044=602264} 2023-10-09T09:09:58.978 INFO:tasks.rados.rados.0.smithi163.stdout:1122: writing smithi16367185-16 from 662308 to 1221338 tid 1 2023-10-09T09:09:58.986 INFO:tasks.rados.rados.0.smithi163.stdout:1122: writing smithi16367185-16 from 1632005 to 2235816 tid 2 2023-10-09T09:09:58.993 INFO:tasks.rados.rados.0.smithi163.stdout:1122: writing smithi16367185-16 from 2860044 to 3462308 tid 3 2023-10-09T09:09:58.993 INFO:tasks.rados.rados.0.smithi163.stdout:1123: read oid 3 snap -1 2023-10-09T09:09:58.993 INFO:tasks.rados.rados.0.smithi163.stdout:1123: expect deleted 2023-10-09T09:09:58.993 INFO:tasks.rados.rados.0.smithi163.stdout:1124: snap_remove snap 128 2023-10-09T09:09:59.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:58 smithi163 bash[33912]: debug 2023-10-09T09:09:58.958+0000 7f84df46a700 -1 mon.c@2(peon).osd e338 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:59.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:59 smithi163 bash[33912]: cluster 2023-10-09T09:09:57.965935+0000 osd.1 (osd.1) 14 : cluster [WRN] failed to encode map e337 with expected crc 2023-10-09T09:09:59.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:59 smithi163 bash[33912]: cluster 2023-10-09T09:09:57.966132+0000 osd.7 (osd.7) 29 : cluster [WRN] failed to encode map e337 with expected crc 2023-10-09T09:09:59.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:59 smithi163 bash[33912]: cluster 2023-10-09T09:09:58.239154+0000 osd.0 (osd.0) 24 : cluster [WRN] failed to encode map e337 with expected crc 2023-10-09T09:09:59.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:59 smithi163 bash[33912]: cluster 2023-10-09T09:09:58.477254+0000 osd.6 (osd.6) 17 : cluster [WRN] failed to encode map e337 with expected crc 2023-10-09T09:09:59.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:59 smithi163 bash[33912]: cluster 2023-10-09T09:09:58.964288+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2023-10-09T09:09:59.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:58 smithi181 bash[27712]: debug 2023-10-09T09:09:58.955+0000 7fde83035700 -1 mon.b@1(peon).osd e338 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:09:59.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:59 smithi181 bash[27712]: cluster 2023-10-09T09:09:57.965935+0000 osd.1 (osd.1) 14 : cluster [WRN] failed to encode map e337 with expected crc 2023-10-09T09:09:59.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:59 smithi181 bash[27712]: cluster 2023-10-09T09:09:57.966132+0000 osd.7 (osd.7) 29 : cluster [WRN] failed to encode map e337 with expected crc 2023-10-09T09:09:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:59 smithi181 bash[27712]: cluster 2023-10-09T09:09:58.239154+0000 osd.0 (osd.0) 24 : cluster [WRN] failed to encode map e337 with expected crc 2023-10-09T09:09:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:59 smithi181 bash[27712]: cluster 2023-10-09T09:09:58.477254+0000 osd.6 (osd.6) 17 : cluster [WRN] failed to encode map e337 with expected crc 2023-10-09T09:09:59.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:59 smithi181 bash[27712]: cluster 2023-10-09T09:09:58.964288+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2023-10-09T09:09:59.947 INFO:tasks.rados.rados.0.smithi163.stdout:1124: done (2 left) 2023-10-09T09:09:59.948 INFO:tasks.rados.rados.0.smithi163.stdout:1125: read oid 48 snap -1 2023-10-09T09:09:59.948 INFO:tasks.rados.rados.0.smithi163.stdout:1125: expect (ObjNum 432 snap 135 seq_num 432) 2023-10-09T09:09:59.948 INFO:tasks.rados.rados.0.smithi163.stdout:1126: delete oid 45 current snap is 141 2023-10-09T09:09:59.948 INFO:tasks.rados.rados.0.smithi163.stdout:1122: finishing write tid 1 to smithi16367185-16 2023-10-09T09:09:59.948 INFO:tasks.rados.rados.0.smithi163.stdout:1122: finishing write tid 2 to smithi16367185-16 2023-10-09T09:09:59.948 INFO:tasks.rados.rados.0.smithi163.stdout:1122: finishing write tid 3 to smithi16367185-16 2023-10-09T09:09:59.948 INFO:tasks.rados.rados.0.smithi163.stdout:1122: finishing write tid 4 to smithi16367185-16 2023-10-09T09:09:59.948 INFO:tasks.rados.rados.0.smithi163.stdout:1122: finishing write tid 5 to smithi16367185-16 2023-10-09T09:09:59.948 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 620 (ObjNum 444 snap 141 seq_num 444) dirty exists 2023-10-09T09:09:59.948 INFO:tasks.rados.rados.0.smithi163.stdout:1122: left oid 16 (ObjNum 444 snap 141 seq_num 444) 2023-10-09T09:09:59.972 INFO:tasks.rados.rados.0.smithi163.stdout:1125: expect (ObjNum 432 snap 135 seq_num 432) 2023-10-09T09:10:00.086 INFO:tasks.rados.rados.0.smithi163.stdout:1122: done (3 left) 2023-10-09T09:10:00.087 INFO:tasks.rados.rados.0.smithi163.stdout:1123: done (2 left) 2023-10-09T09:10:00.087 INFO:tasks.rados.rados.0.smithi163.stdout:1125: done (1 left) 2023-10-09T09:10:00.087 INFO:tasks.rados.rados.0.smithi163.stdout:1126: done (0 left) 2023-10-09T09:10:00.087 INFO:tasks.rados.rados.0.smithi163.stdout:1127: snap_create 2023-10-09T09:10:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:09:59 smithi163 bash[33912]: debug 2023-10-09T09:09:59.934+0000 7f84df46a700 -1 mon.c@2(peon).osd e339 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:00 smithi163 bash[33912]: cluster 2023-10-09T09:09:58.967085+0000 osd.4 (osd.4) 33 : cluster [WRN] failed to encode map e338 with expected crc 2023-10-09T09:10:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:00 smithi163 bash[33912]: cluster 2023-10-09T09:09:58.969217+0000 osd.0 (osd.0) 25 : cluster [WRN] failed to encode map e338 with expected crc 2023-10-09T09:10:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:00 smithi163 bash[33912]: cluster 2023-10-09T09:09:58.980683+0000 osd.2 (osd.2) 31 : cluster [WRN] failed to encode map e338 with expected crc 2023-10-09T09:10:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:00 smithi163 bash[33912]: cluster 2023-10-09T09:09:59.219246+0000 mgr.y (mgr.34107) 33 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.7 MiB/s wr, 33 op/s 2023-10-09T09:10:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:00 smithi163 bash[33912]: cluster 2023-10-09T09:09:59.941116+0000 mon.a (mon.0) 111 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2023-10-09T09:10:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:00 smithi163 bash[33912]: cluster 2023-10-09T09:10:00.000119+0000 mon.a (mon.0) 112 : cluster [INF] overall HEALTH_OK 2023-10-09T09:10:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:09:59 smithi181 bash[27712]: debug 2023-10-09T09:09:59.931+0000 7fde83035700 -1 mon.b@1(peon).osd e339 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:00 smithi181 bash[27712]: cluster 2023-10-09T09:09:58.967085+0000 osd.4 (osd.4) 33 : cluster [WRN] failed to encode map e338 with expected crc 2023-10-09T09:10:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:00 smithi181 bash[27712]: cluster 2023-10-09T09:09:58.969217+0000 osd.0 (osd.0) 25 : cluster [WRN] failed to encode map e338 with expected crc 2023-10-09T09:10:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:00 smithi181 bash[27712]: cluster 2023-10-09T09:09:58.980683+0000 osd.2 (osd.2) 31 : cluster [WRN] failed to encode map e338 with expected crc 2023-10-09T09:10:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:00 smithi181 bash[27712]: cluster 2023-10-09T09:09:59.219246+0000 mgr.y (mgr.34107) 33 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.7 MiB/s wr, 33 op/s 2023-10-09T09:10:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:00 smithi181 bash[27712]: cluster 2023-10-09T09:09:59.941116+0000 mon.a (mon.0) 111 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2023-10-09T09:10:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:00 smithi181 bash[27712]: cluster 2023-10-09T09:10:00.000119+0000 mon.a (mon.0) 112 : cluster [INF] overall HEALTH_OK 2023-10-09T09:10:00.962 INFO:tasks.rados.rados.0.smithi163.stdout:1127: done (0 left) 2023-10-09T09:10:00.962 INFO:tasks.rados.rados.0.smithi163.stdout:1128: delete oid 24 current snap is 142 2023-10-09T09:10:00.966 INFO:tasks.rados.rados.0.smithi163.stdout:1128: done (0 left) 2023-10-09T09:10:00.966 INFO:tasks.rados.rados.0.smithi163.stdout:1129: read oid 43 snap 124 2023-10-09T09:10:00.967 INFO:tasks.rados.rados.0.smithi163.stdout:1129: expect deleted 2023-10-09T09:10:00.967 INFO:tasks.rados.rados.0.smithi163.stdout:1130: delete oid 39 current snap is 142 2023-10-09T09:10:00.974 INFO:tasks.rados.rados.0.smithi163.stdout:1129: done (1 left) 2023-10-09T09:10:00.975 INFO:tasks.rados.rados.0.smithi163.stdout:1130: done (0 left) 2023-10-09T09:10:00.975 INFO:tasks.rados.rados.0.smithi163.stdout:1131: snap_create 2023-10-09T09:10:01.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:00 smithi163 bash[33912]: debug 2023-10-09T09:10:00.950+0000 7f84df46a700 -1 mon.c@2(peon).osd e340 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:01.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:01 smithi163 bash[33912]: cluster 2023-10-09T09:09:59.943822+0000 osd.4 (osd.4) 34 : cluster [WRN] failed to encode map e339 with expected crc 2023-10-09T09:10:01.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:01 smithi163 bash[33912]: cluster 2023-10-09T09:09:59.945107+0000 osd.3 (osd.3) 23 : cluster [WRN] failed to encode map e339 with expected crc 2023-10-09T09:10:01.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:01 smithi163 bash[33912]: cluster 2023-10-09T09:09:59.950947+0000 osd.5 (osd.5) 19 : cluster [WRN] failed to encode map e339 with expected crc 2023-10-09T09:10:01.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:01 smithi163 bash[33912]: cluster 2023-10-09T09:10:00.955415+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2023-10-09T09:10:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:00 smithi181 bash[27712]: debug 2023-10-09T09:10:00.947+0000 7fde83035700 -1 mon.b@1(peon).osd e340 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:01 smithi181 bash[27712]: cluster 2023-10-09T09:09:59.943822+0000 osd.4 (osd.4) 34 : cluster [WRN] failed to encode map e339 with expected crc 2023-10-09T09:10:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:01 smithi181 bash[27712]: cluster 2023-10-09T09:09:59.945107+0000 osd.3 (osd.3) 23 : cluster [WRN] failed to encode map e339 with expected crc 2023-10-09T09:10:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:01 smithi181 bash[27712]: cluster 2023-10-09T09:09:59.950947+0000 osd.5 (osd.5) 19 : cluster [WRN] failed to encode map e339 with expected crc 2023-10-09T09:10:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:01 smithi181 bash[27712]: cluster 2023-10-09T09:10:00.955415+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2023-10-09T09:10:01.960 INFO:tasks.rados.rados.0.smithi163.stdout:1131: done (0 left) 2023-10-09T09:10:01.961 INFO:tasks.rados.rados.0.smithi163.stdout:1132: read oid 6 snap -1 2023-10-09T09:10:01.961 INFO:tasks.rados.rados.0.smithi163.stdout:1132: expect (ObjNum 431 snap 134 seq_num 431) 2023-10-09T09:10:01.961 INFO:tasks.rados.rados.0.smithi163.stdout:1133: read oid 21 snap -1 2023-10-09T09:10:01.961 INFO:tasks.rados.rados.0.smithi163.stdout:1133: expect deleted 2023-10-09T09:10:01.961 INFO:tasks.rados.rados.0.smithi163.stdout:1134: snap_create 2023-10-09T09:10:01.977 INFO:tasks.rados.rados.0.smithi163.stdout:1132: expect (ObjNum 431 snap 134 seq_num 431) 2023-10-09T09:10:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:01 smithi163 bash[33912]: debug 2023-10-09T09:10:01.954+0000 7f84df46a700 -1 mon.c@2(peon).osd e341 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:02 smithi163 bash[33912]: cluster 2023-10-09T09:10:00.957162+0000 osd.4 (osd.4) 35 : cluster [WRN] failed to encode map e340 with expected crc 2023-10-09T09:10:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:02 smithi163 bash[33912]: cluster 2023-10-09T09:10:00.960040+0000 osd.3 (osd.3) 24 : cluster [WRN] failed to encode map e340 with expected crc 2023-10-09T09:10:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:02 smithi163 bash[33912]: cluster 2023-10-09T09:10:00.966896+0000 osd.5 (osd.5) 20 : cluster [WRN] failed to encode map e340 with expected crc 2023-10-09T09:10:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:02 smithi163 bash[33912]: cluster 2023-10-09T09:10:01.219908+0000 mgr.y (mgr.34107) 34 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:10:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:02 smithi163 bash[33912]: cluster 2023-10-09T09:10:01.960199+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2023-10-09T09:10:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:01 smithi181 bash[27712]: debug 2023-10-09T09:10:01.947+0000 7fde83035700 -1 mon.b@1(peon).osd e341 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:02 smithi181 bash[27712]: cluster 2023-10-09T09:10:00.957162+0000 osd.4 (osd.4) 35 : cluster [WRN] failed to encode map e340 with expected crc 2023-10-09T09:10:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:02 smithi181 bash[27712]: cluster 2023-10-09T09:10:00.960040+0000 osd.3 (osd.3) 24 : cluster [WRN] failed to encode map e340 with expected crc 2023-10-09T09:10:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:02 smithi181 bash[27712]: cluster 2023-10-09T09:10:00.966896+0000 osd.5 (osd.5) 20 : cluster [WRN] failed to encode map e340 with expected crc 2023-10-09T09:10:02.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:02 smithi181 bash[27712]: cluster 2023-10-09T09:10:01.219908+0000 mgr.y (mgr.34107) 34 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:10:02.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:02 smithi181 bash[27712]: cluster 2023-10-09T09:10:01.960199+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2023-10-09T09:10:02.965 INFO:tasks.rados.rados.0.smithi163.stdout:1132: done (2 left) 2023-10-09T09:10:02.966 INFO:tasks.rados.rados.0.smithi163.stdout:1133: done (1 left) 2023-10-09T09:10:02.966 INFO:tasks.rados.rados.0.smithi163.stdout:1134: done (0 left) 2023-10-09T09:10:02.966 INFO:tasks.rados.rados.0.smithi163.stdout:1135: write oid 49 current snap is 144 2023-10-09T09:10:02.966 INFO:tasks.rados.rados.0.smithi163.stdout:1135: seq_num 448 ranges {729536=460026,1749580=611149,2853383=452450} 2023-10-09T09:10:02.975 INFO:tasks.rados.rados.0.smithi163.stdout:1135: writing smithi16367185-49 from 729536 to 1189562 tid 1 2023-10-09T09:10:02.984 INFO:tasks.rados.rados.0.smithi163.stdout:1135: writing smithi16367185-49 from 1749580 to 2360729 tid 2 2023-10-09T09:10:02.990 INFO:tasks.rados.rados.0.smithi163.stdout:1135: writing smithi16367185-49 from 2853383 to 3305833 tid 3 2023-10-09T09:10:02.990 INFO:tasks.rados.rados.0.smithi163.stdout:1136: snap_remove snap 141 2023-10-09T09:10:03.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:02 smithi163 bash[33912]: debug 2023-10-09T09:10:02.954+0000 7f84df46a700 -1 mon.c@2(peon).osd e342 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:03.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:03 smithi163 bash[33912]: cluster 2023-10-09T09:10:01.956687+0000 osd.4 (osd.4) 36 : cluster [WRN] failed to encode map e341 with expected crc 2023-10-09T09:10:03.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:03 smithi163 bash[33912]: cluster 2023-10-09T09:10:01.964196+0000 osd.0 (osd.0) 26 : cluster [WRN] failed to encode map e341 with expected crc 2023-10-09T09:10:03.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:03 smithi163 bash[33912]: cluster 2023-10-09T09:10:01.966745+0000 osd.1 (osd.1) 15 : cluster [WRN] failed to encode map e341 with expected crc 2023-10-09T09:10:03.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:03 smithi163 bash[33912]: cluster 2023-10-09T09:10:02.961056+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2023-10-09T09:10:03.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:02 smithi181 bash[27712]: debug 2023-10-09T09:10:02.951+0000 7fde83035700 -1 mon.b@1(peon).osd e342 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:03.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:03 smithi181 bash[27712]: cluster 2023-10-09T09:10:01.956687+0000 osd.4 (osd.4) 36 : cluster [WRN] failed to encode map e341 with expected crc 2023-10-09T09:10:03.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:03 smithi181 bash[27712]: cluster 2023-10-09T09:10:01.964196+0000 osd.0 (osd.0) 26 : cluster [WRN] failed to encode map e341 with expected crc 2023-10-09T09:10:03.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:03 smithi181 bash[27712]: cluster 2023-10-09T09:10:01.966745+0000 osd.1 (osd.1) 15 : cluster [WRN] failed to encode map e341 with expected crc 2023-10-09T09:10:03.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:03 smithi181 bash[27712]: cluster 2023-10-09T09:10:02.961056+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2023-10-09T09:10:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:10:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:10:03] "GET /metrics HTTP/1.1" 200 36081 "" "Prometheus/2.33.4" 2023-10-09T09:10:03.970 INFO:tasks.rados.rados.0.smithi163.stdout:1136: done (1 left) 2023-10-09T09:10:03.971 INFO:tasks.rados.rados.0.smithi163.stdout:1137: read oid 1 snap -1 2023-10-09T09:10:03.971 INFO:tasks.rados.rados.0.smithi163.stdout:1137: expect (ObjNum 440 snap 138 seq_num 440) 2023-10-09T09:10:03.971 INFO:tasks.rados.rados.0.smithi163.stdout:1138: read oid 40 snap 142 2023-10-09T09:10:03.971 INFO:tasks.rados.rados.0.smithi163.stdout:1138: expect deleted 2023-10-09T09:10:03.971 INFO:tasks.rados.rados.0.smithi163.stdout:1139: snap_remove snap 143 2023-10-09T09:10:04.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:03 smithi163 bash[33912]: debug 2023-10-09T09:10:03.958+0000 7f84df46a700 -1 mon.c@2(peon).osd e343 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:04.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:04 smithi163 bash[33912]: cluster 2023-10-09T09:10:02.961633+0000 osd.4 (osd.4) 37 : cluster [WRN] failed to encode map e342 with expected crc 2023-10-09T09:10:04.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:04 smithi163 bash[33912]: cluster 2023-10-09T09:10:02.965325+0000 osd.3 (osd.3) 25 : cluster [WRN] failed to encode map e342 with expected crc 2023-10-09T09:10:04.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:04 smithi163 bash[33912]: cluster 2023-10-09T09:10:02.978403+0000 osd.0 (osd.0) 27 : cluster [WRN] failed to encode map e342 with expected crc 2023-10-09T09:10:04.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:04 smithi163 bash[33912]: cluster 2023-10-09T09:10:03.220731+0000 mgr.y (mgr.34107) 35 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 141 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 431 KiB/s wr, 4 op/s 2023-10-09T09:10:04.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:04 smithi163 bash[33912]: cluster 2023-10-09T09:10:03.963191+0000 mon.a (mon.0) 116 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2023-10-09T09:10:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:03 smithi181 bash[27712]: debug 2023-10-09T09:10:03.951+0000 7fde83035700 -1 mon.b@1(peon).osd e343 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:04 smithi181 bash[27712]: cluster 2023-10-09T09:10:02.961633+0000 osd.4 (osd.4) 37 : cluster [WRN] failed to encode map e342 with expected crc 2023-10-09T09:10:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:04 smithi181 bash[27712]: cluster 2023-10-09T09:10:02.965325+0000 osd.3 (osd.3) 25 : cluster [WRN] failed to encode map e342 with expected crc 2023-10-09T09:10:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:04 smithi181 bash[27712]: cluster 2023-10-09T09:10:02.978403+0000 osd.0 (osd.0) 27 : cluster [WRN] failed to encode map e342 with expected crc 2023-10-09T09:10:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:04 smithi181 bash[27712]: cluster 2023-10-09T09:10:03.220731+0000 mgr.y (mgr.34107) 35 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 141 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 431 KiB/s wr, 4 op/s 2023-10-09T09:10:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:04 smithi181 bash[27712]: cluster 2023-10-09T09:10:03.963191+0000 mon.a (mon.0) 116 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2023-10-09T09:10:04.951 INFO:tasks.rados.rados.0.smithi163.stdout:1139: done (3 left) 2023-10-09T09:10:04.951 INFO:tasks.rados.rados.0.smithi163.stdout:1140: read oid 37 snap 133 2023-10-09T09:10:04.951 INFO:tasks.rados.rados.0.smithi163.stdout:1140: expect deleted 2023-10-09T09:10:04.952 INFO:tasks.rados.rados.0.smithi163.stdout:1141: read oid 4 snap -1 2023-10-09T09:10:04.952 INFO:tasks.rados.rados.0.smithi163.stdout:1141: expect (ObjNum 247 snap 73 seq_num 247) 2023-10-09T09:10:04.952 INFO:tasks.rados.rados.0.smithi163.stdout:1142: read oid 20 snap -1 2023-10-09T09:10:04.952 INFO:tasks.rados.rados.0.smithi163.stdout:1142: expect (ObjNum 416 snap 127 seq_num 416) 2023-10-09T09:10:04.952 INFO:tasks.rados.rados.0.smithi163.stdout:1143: write oid 35 current snap is 144 2023-10-09T09:10:04.952 INFO:tasks.rados.rados.0.smithi163.stdout:1135: finishing write tid 1 to smithi16367185-49 2023-10-09T09:10:04.953 INFO:tasks.rados.rados.0.smithi163.stdout:1135: finishing write tid 2 to smithi16367185-49 2023-10-09T09:10:04.953 INFO:tasks.rados.rados.0.smithi163.stdout:1135: finishing write tid 3 to smithi16367185-49 2023-10-09T09:10:04.953 INFO:tasks.rados.rados.0.smithi163.stdout:1143: seq_num 449 ranges {746343=651628,1924415=754807,3397159=149184} 2023-10-09T09:10:04.965 INFO:tasks.rados.rados.0.smithi163.stdout:1143: writing smithi16367185-35 from 746343 to 1397971 tid 1 2023-10-09T09:10:04.972 INFO:tasks.rados.rados.0.smithi163.stdout:1143: writing smithi16367185-35 from 1924415 to 2679222 tid 2 2023-10-09T09:10:04.977 INFO:tasks.rados.rados.0.smithi163.stdout:1143: writing smithi16367185-35 from 3397159 to 3546343 tid 3 2023-10-09T09:10:04.977 INFO:tasks.rados.rados.0.smithi163.stdout:1144: read oid 18 snap -1 2023-10-09T09:10:04.978 INFO:tasks.rados.rados.0.smithi163.stdout:1144: expect (ObjNum 379 snap 117 seq_num 379) 2023-10-09T09:10:04.978 INFO:tasks.rados.rados.0.smithi163.stdout:1145: snap_remove snap 138 2023-10-09T09:10:05.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:04 smithi163 bash[33912]: debug 2023-10-09T09:10:04.938+0000 7f84df46a700 -1 mon.c@2(peon).osd e344 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:05.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:05 smithi163 bash[33912]: cluster 2023-10-09T09:10:03.966921+0000 osd.4 (osd.4) 38 : cluster [WRN] failed to encode map e343 with expected crc 2023-10-09T09:10:05.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:05 smithi163 bash[33912]: cluster 2023-10-09T09:10:03.967572+0000 osd.1 (osd.1) 16 : cluster [WRN] failed to encode map e343 with expected crc 2023-10-09T09:10:05.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:05 smithi163 bash[33912]: cluster 2023-10-09T09:10:03.971258+0000 osd.7 (osd.7) 30 : cluster [WRN] failed to encode map e343 with expected crc 2023-10-09T09:10:05.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:05 smithi163 bash[33912]: cluster 2023-10-09T09:10:03.972947+0000 osd.2 (osd.2) 32 : cluster [WRN] failed to encode map e343 with expected crc 2023-10-09T09:10:05.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:05 smithi163 bash[33912]: cluster 2023-10-09T09:10:04.242713+0000 osd.0 (osd.0) 28 : cluster [WRN] failed to encode map e343 with expected crc 2023-10-09T09:10:05.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:05 smithi163 bash[33912]: cluster 2023-10-09T09:10:04.946345+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2023-10-09T09:10:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:04 smithi181 bash[27712]: debug 2023-10-09T09:10:04.935+0000 7fde83035700 -1 mon.b@1(peon).osd e344 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:05 smithi181 bash[27712]: cluster 2023-10-09T09:10:03.966921+0000 osd.4 (osd.4) 38 : cluster [WRN] failed to encode map e343 with expected crc 2023-10-09T09:10:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:05 smithi181 bash[27712]: cluster 2023-10-09T09:10:03.967572+0000 osd.1 (osd.1) 16 : cluster [WRN] failed to encode map e343 with expected crc 2023-10-09T09:10:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:05 smithi181 bash[27712]: cluster 2023-10-09T09:10:03.971258+0000 osd.7 (osd.7) 30 : cluster [WRN] failed to encode map e343 with expected crc 2023-10-09T09:10:05.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:05 smithi181 bash[27712]: cluster 2023-10-09T09:10:03.972947+0000 osd.2 (osd.2) 32 : cluster [WRN] failed to encode map e343 with expected crc 2023-10-09T09:10:05.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:05 smithi181 bash[27712]: cluster 2023-10-09T09:10:04.242713+0000 osd.0 (osd.0) 28 : cluster [WRN] failed to encode map e343 with expected crc 2023-10-09T09:10:05.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:05 smithi181 bash[27712]: cluster 2023-10-09T09:10:04.946345+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2023-10-09T09:10:05.967 INFO:tasks.rados.rados.0.smithi163.stdout:1145: done (8 left) 2023-10-09T09:10:05.967 INFO:tasks.rados.rados.0.smithi163.stdout:1146: read oid 43 snap -1 2023-10-09T09:10:05.967 INFO:tasks.rados.rados.0.smithi163.stdout:1146: expect deleted 2023-10-09T09:10:05.968 INFO:tasks.rados.rados.0.smithi163.stdout:1147: read oid 25 snap -1 2023-10-09T09:10:05.968 INFO:tasks.rados.rados.0.smithi163.stdout:1147: expect (ObjNum 443 snap 138 seq_num 443) 2023-10-09T09:10:05.968 INFO:tasks.rados.rados.0.smithi163.stdout:1148: delete oid 17 current snap is 144 2023-10-09T09:10:05.968 INFO:tasks.rados.rados.0.smithi163.stdout:1135: finishing write tid 4 to smithi16367185-49 2023-10-09T09:10:05.968 INFO:tasks.rados.rados.0.smithi163.stdout:1135: finishing write tid 5 to smithi16367185-49 2023-10-09T09:10:05.968 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 182 (ObjNum 448 snap 144 seq_num 448) dirty exists 2023-10-09T09:10:05.968 INFO:tasks.rados.rados.0.smithi163.stdout:1135: left oid 49 (ObjNum 448 snap 144 seq_num 448) 2023-10-09T09:10:05.968 INFO:tasks.rados.rados.0.smithi163.stdout:1137: expect (ObjNum 440 snap 138 seq_num 440) 2023-10-09T09:10:06.178 INFO:tasks.rados.rados.0.smithi163.stdout:1141: expect (ObjNum 247 snap 73 seq_num 247) 2023-10-09T09:10:06.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:05 smithi163 bash[33912]: debug 2023-10-09T09:10:05.954+0000 7f84df46a700 -1 mon.c@2(peon).osd e345 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:06.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:06 smithi163 bash[33912]: cluster 2023-10-09T09:10:04.947354+0000 osd.4 (osd.4) 39 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:06 smithi163 bash[33912]: cluster 2023-10-09T09:10:04.948499+0000 osd.0 (osd.0) 29 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:06 smithi163 bash[33912]: cluster 2023-10-09T09:10:04.952588+0000 osd.7 (osd.7) 31 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:06 smithi163 bash[33912]: cluster 2023-10-09T09:10:04.952663+0000 osd.6 (osd.6) 18 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:06 smithi163 bash[33912]: cluster 2023-10-09T09:10:04.954575+0000 osd.2 (osd.2) 33 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:06 smithi163 bash[33912]: cluster 2023-10-09T09:10:05.221448+0000 mgr.y (mgr.34107) 36 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 141 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 431 KiB/s wr, 4 op/s 2023-10-09T09:10:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:06 smithi163 bash[33912]: cluster 2023-10-09T09:10:05.590444+0000 osd.1 (osd.1) 17 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:06 smithi163 bash[33912]: cluster 2023-10-09T09:10:05.961595+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2023-10-09T09:10:06.279 INFO:tasks.rados.rados.0.smithi163.stdout:1143: finishing write tid 1 to smithi16367185-35 2023-10-09T09:10:06.279 INFO:tasks.rados.rados.0.smithi163.stdout:1142: expect (ObjNum 416 snap 127 seq_num 416) 2023-10-09T09:10:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:05 smithi181 bash[27712]: debug 2023-10-09T09:10:05.951+0000 7fde83035700 -1 mon.b@1(peon).osd e345 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:06 smithi181 bash[27712]: cluster 2023-10-09T09:10:04.947354+0000 osd.4 (osd.4) 39 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:06 smithi181 bash[27712]: cluster 2023-10-09T09:10:04.948499+0000 osd.0 (osd.0) 29 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:06 smithi181 bash[27712]: cluster 2023-10-09T09:10:04.952588+0000 osd.7 (osd.7) 31 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:06 smithi181 bash[27712]: cluster 2023-10-09T09:10:04.952663+0000 osd.6 (osd.6) 18 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:06 smithi181 bash[27712]: cluster 2023-10-09T09:10:04.954575+0000 osd.2 (osd.2) 33 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:06 smithi181 bash[27712]: cluster 2023-10-09T09:10:05.221448+0000 mgr.y (mgr.34107) 36 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 141 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 431 KiB/s wr, 4 op/s 2023-10-09T09:10:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:06 smithi181 bash[27712]: cluster 2023-10-09T09:10:05.590444+0000 osd.1 (osd.1) 17 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:06 smithi181 bash[27712]: cluster 2023-10-09T09:10:05.961595+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2023-10-09T09:10:06.406 INFO:tasks.rados.rados.0.smithi163.stdout:1143: finishing write tid 2 to smithi16367185-35 2023-10-09T09:10:06.406 INFO:tasks.rados.rados.0.smithi163.stdout:1143: finishing write tid 3 to smithi16367185-35 2023-10-09T09:10:06.407 INFO:tasks.rados.rados.0.smithi163.stdout:1143: finishing write tid 4 to smithi16367185-35 2023-10-09T09:10:06.407 INFO:tasks.rados.rados.0.smithi163.stdout:1143: finishing write tid 5 to smithi16367185-35 2023-10-09T09:10:06.407 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 350 (ObjNum 449 snap 144 seq_num 449) dirty exists 2023-10-09T09:10:06.407 INFO:tasks.rados.rados.0.smithi163.stdout:1143: left oid 35 (ObjNum 449 snap 144 seq_num 449) 2023-10-09T09:10:06.407 INFO:tasks.rados.rados.0.smithi163.stdout:1144: expect (ObjNum 379 snap 117 seq_num 379) 2023-10-09T09:10:06.516 INFO:tasks.rados.rados.0.smithi163.stdout:1147: expect (ObjNum 443 snap 138 seq_num 443) 2023-10-09T09:10:06.620 INFO:tasks.rados.rados.0.smithi163.stdout:1135: done (10 left) 2023-10-09T09:10:06.620 INFO:tasks.rados.rados.0.smithi163.stdout:1137: done (9 left) 2023-10-09T09:10:06.620 INFO:tasks.rados.rados.0.smithi163.stdout:1138: done (8 left) 2023-10-09T09:10:06.620 INFO:tasks.rados.rados.0.smithi163.stdout:1140: done (7 left) 2023-10-09T09:10:06.620 INFO:tasks.rados.rados.0.smithi163.stdout:1141: done (6 left) 2023-10-09T09:10:06.621 INFO:tasks.rados.rados.0.smithi163.stdout:1142: done (5 left) 2023-10-09T09:10:06.621 INFO:tasks.rados.rados.0.smithi163.stdout:1143: done (4 left) 2023-10-09T09:10:06.621 INFO:tasks.rados.rados.0.smithi163.stdout:1144: done (3 left) 2023-10-09T09:10:06.621 INFO:tasks.rados.rados.0.smithi163.stdout:1146: done (2 left) 2023-10-09T09:10:06.621 INFO:tasks.rados.rados.0.smithi163.stdout:1147: done (1 left) 2023-10-09T09:10:06.621 INFO:tasks.rados.rados.0.smithi163.stdout:1148: done (0 left) 2023-10-09T09:10:06.621 INFO:tasks.rados.rados.0.smithi163.stdout:1149: snap_create 2023-10-09T09:10:07.191 INFO:tasks.rados.rados.0.smithi163.stdout:1149: done (0 left) 2023-10-09T09:10:07.191 INFO:tasks.rados.rados.0.smithi163.stdout:1150: snap_remove snap 142 2023-10-09T09:10:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:07 smithi163 bash[33912]: debug 2023-10-09T09:10:07.178+0000 7f84df46a700 -1 mon.c@2(peon).osd e346 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:07 smithi163 bash[33912]: cluster 2023-10-09T09:10:05.644050+0000 osd.3 (osd.3) 26 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:07 smithi163 bash[33912]: cluster 2023-10-09T09:10:05.963206+0000 osd.6 (osd.6) 19 : cluster [WRN] failed to encode map e345 with expected crc 2023-10-09T09:10:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:07 smithi163 bash[33912]: cluster 2023-10-09T09:10:05.965423+0000 osd.3 (osd.3) 27 : cluster [WRN] failed to encode map e345 with expected crc 2023-10-09T09:10:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:07 smithi163 bash[33912]: cluster 2023-10-09T09:10:05.967269+0000 osd.7 (osd.7) 32 : cluster [WRN] failed to encode map e345 with expected crc 2023-10-09T09:10:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:07 smithi163 bash[33912]: cluster 2023-10-09T09:10:05.967664+0000 osd.4 (osd.4) 40 : cluster [WRN] failed to encode map e345 with expected crc 2023-10-09T09:10:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:07 smithi163 bash[33912]: cluster 2023-10-09T09:10:05.968832+0000 osd.2 (osd.2) 34 : cluster [WRN] failed to encode map e345 with expected crc 2023-10-09T09:10:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:07 smithi181 bash[27712]: debug 2023-10-09T09:10:07.175+0000 7fde83035700 -1 mon.b@1(peon).osd e346 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:07 smithi181 bash[27712]: cluster 2023-10-09T09:10:05.644050+0000 osd.3 (osd.3) 26 : cluster [WRN] failed to encode map e344 with expected crc 2023-10-09T09:10:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:07 smithi181 bash[27712]: cluster 2023-10-09T09:10:05.963206+0000 osd.6 (osd.6) 19 : cluster [WRN] failed to encode map e345 with expected crc 2023-10-09T09:10:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:07 smithi181 bash[27712]: cluster 2023-10-09T09:10:05.965423+0000 osd.3 (osd.3) 27 : cluster [WRN] failed to encode map e345 with expected crc 2023-10-09T09:10:07.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:07 smithi181 bash[27712]: cluster 2023-10-09T09:10:05.967269+0000 osd.7 (osd.7) 32 : cluster [WRN] failed to encode map e345 with expected crc 2023-10-09T09:10:07.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:07 smithi181 bash[27712]: cluster 2023-10-09T09:10:05.967664+0000 osd.4 (osd.4) 40 : cluster [WRN] failed to encode map e345 with expected crc 2023-10-09T09:10:07.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:07 smithi181 bash[27712]: cluster 2023-10-09T09:10:05.968832+0000 osd.2 (osd.2) 34 : cluster [WRN] failed to encode map e345 with expected crc 2023-10-09T09:10:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:1150: done (0 left) 2023-10-09T09:10:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:1151: rollback oid 42 current snap is 145 2023-10-09T09:10:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 135 2023-10-09T09:10:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:1152: read oid 44 snap -1 2023-10-09T09:10:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:1152: expect (ObjNum 408 snap 124 seq_num 408) 2023-10-09T09:10:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:1153: read oid 43 snap -1 2023-10-09T09:10:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:1153: expect deleted 2023-10-09T09:10:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:1154: write oid 22 current snap is 145 2023-10-09T09:10:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:1154: seq_num 451 ranges {779957=402126,1925026=758476,3335652=244305} 2023-10-09T09:10:08.215 INFO:tasks.rados.rados.0.smithi163.stdout:1154: writing smithi16367185-22 from 779957 to 1182083 tid 1 2023-10-09T09:10:08.224 INFO:tasks.rados.rados.0.smithi163.stdout:1154: writing smithi16367185-22 from 1925026 to 2683502 tid 2 2023-10-09T09:10:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:1154: writing smithi16367185-22 from 3335652 to 3579957 tid 3 2023-10-09T09:10:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:1155: read oid 21 snap -1 2023-10-09T09:10:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:1155: expect deleted 2023-10-09T09:10:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:1156: write oid 48 current snap is 145 2023-10-09T09:10:08.230 INFO:tasks.rados.rados.0.smithi163.stdout:1156: seq_num 452 ranges {796764=477375,1983508=618487,3363075=233689} 2023-10-09T09:10:08.241 INFO:tasks.rados.rados.0.smithi163.stdout:1156: writing smithi16367185-48 from 796764 to 1274139 tid 1 2023-10-09T09:10:08.250 INFO:tasks.rados.rados.0.smithi163.stdout:1156: writing smithi16367185-48 from 1983508 to 2601995 tid 2 2023-10-09T09:10:08.257 INFO:tasks.rados.rados.0.smithi163.stdout:1156: writing smithi16367185-48 from 3363075 to 3596764 tid 3 2023-10-09T09:10:08.257 INFO:tasks.rados.rados.0.smithi163.stdout:1157: snap_create 2023-10-09T09:10:08.257 INFO:tasks.rados.rados.0.smithi163.stdout:1152: expect (ObjNum 408 snap 124 seq_num 408) 2023-10-09T09:10:08.416 INFO:tasks.rados.rados.0.smithi163.stdout:1151: finishing rollback tid 0 to smithi16367185-42 2023-10-09T09:10:08.416 INFO:tasks.rados.rados.0.smithi163.stdout:1151: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:10:08.416 INFO:tasks.rados.rados.0.smithi163.stdout:1151: finishing rollback tid 2 to smithi16367185-42 2023-10-09T09:10:08.416 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 898 (ObjNum 384 snap 118 seq_num 384) dirty exists 2023-10-09T09:10:08.416 INFO:tasks.rados.rados.0.smithi163.stdout:1154: finishing write tid 1 to smithi16367185-22 2023-10-09T09:10:08.416 INFO:tasks.rados.rados.0.smithi163.stdout:1154: finishing write tid 2 to smithi16367185-22 2023-10-09T09:10:08.416 INFO:tasks.rados.rados.0.smithi163.stdout:1154: finishing write tid 3 to smithi16367185-22 2023-10-09T09:10:08.416 INFO:tasks.rados.rados.0.smithi163.stdout:1154: finishing write tid 4 to smithi16367185-22 2023-10-09T09:10:08.416 INFO:tasks.rados.rados.0.smithi163.stdout:1154: finishing write tid 5 to smithi16367185-22 2023-10-09T09:10:08.416 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 902 (ObjNum 451 snap 145 seq_num 451) dirty exists 2023-10-09T09:10:08.417 INFO:tasks.rados.rados.0.smithi163.stdout:1154: left oid 22 (ObjNum 451 snap 145 seq_num 451) 2023-10-09T09:10:08.417 INFO:tasks.rados.rados.0.smithi163.stdout:1156: finishing write tid 1 to smithi16367185-48 2023-10-09T09:10:08.417 INFO:tasks.rados.rados.0.smithi163.stdout:1156: finishing write tid 2 to smithi16367185-48 2023-10-09T09:10:08.417 INFO:tasks.rados.rados.0.smithi163.stdout:1156: finishing write tid 3 to smithi16367185-48 2023-10-09T09:10:08.417 INFO:tasks.rados.rados.0.smithi163.stdout:1156: finishing write tid 4 to smithi16367185-48 2023-10-09T09:10:08.417 INFO:tasks.rados.rados.0.smithi163.stdout:1156: finishing write tid 5 to smithi16367185-48 2023-10-09T09:10:08.417 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 907 (ObjNum 452 snap 145 seq_num 452) dirty exists 2023-10-09T09:10:08.417 INFO:tasks.rados.rados.0.smithi163.stdout:1156: left oid 48 (ObjNum 452 snap 145 seq_num 452) 2023-10-09T09:10:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:08 smithi163 bash[33912]: debug 2023-10-09T09:10:08.194+0000 7f84df46a700 -1 mon.c@2(peon).osd e347 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:08 smithi163 bash[33912]: cluster 2023-10-09T09:10:07.185120+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2023-10-09T09:10:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:08 smithi163 bash[33912]: cluster 2023-10-09T09:10:07.187527+0000 osd.4 (osd.4) 41 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:08 smithi163 bash[33912]: cluster 2023-10-09T09:10:07.188518+0000 osd.2 (osd.2) 35 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:08.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:08 smithi163 bash[33912]: cluster 2023-10-09T09:10:07.222035+0000 mgr.y (mgr.34107) 37 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 146 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 752 KiB/s wr, 15 op/s 2023-10-09T09:10:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:08 smithi181 bash[27712]: debug 2023-10-09T09:10:08.191+0000 7fde83035700 -1 mon.b@1(peon).osd e347 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:08 smithi181 bash[27712]: cluster 2023-10-09T09:10:07.185120+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2023-10-09T09:10:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:08 smithi181 bash[27712]: cluster 2023-10-09T09:10:07.187527+0000 osd.4 (osd.4) 41 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:08 smithi181 bash[27712]: cluster 2023-10-09T09:10:07.188518+0000 osd.2 (osd.2) 35 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:08 smithi181 bash[27712]: cluster 2023-10-09T09:10:07.222035+0000 mgr.y (mgr.34107) 37 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 146 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 752 KiB/s wr, 15 op/s 2023-10-09T09:10:09.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:10:08 smithi181 bash[54907]: ts=2023-10-09T09:10:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:10:09.230 INFO:tasks.rados.rados.0.smithi163.stdout:1151: done (6 left) 2023-10-09T09:10:09.230 INFO:tasks.rados.rados.0.smithi163.stdout:1152: done (5 left) 2023-10-09T09:10:09.230 INFO:tasks.rados.rados.0.smithi163.stdout:1153: done (4 left) 2023-10-09T09:10:09.230 INFO:tasks.rados.rados.0.smithi163.stdout:1154: done (3 left) 2023-10-09T09:10:09.230 INFO:tasks.rados.rados.0.smithi163.stdout:1155: done (2 left) 2023-10-09T09:10:09.230 INFO:tasks.rados.rados.0.smithi163.stdout:1156: done (1 left) 2023-10-09T09:10:09.230 INFO:tasks.rados.rados.0.smithi163.stdout:1157: done (0 left) 2023-10-09T09:10:09.230 INFO:tasks.rados.rados.0.smithi163.stdout:1158: write oid 7 current snap is 146 2023-10-09T09:10:09.230 INFO:tasks.rados.rados.0.smithi163.stdout:1158: seq_num 453 ranges {413571=552624,1641990=762145,2990498=425163} 2023-10-09T09:10:09.235 INFO:tasks.rados.rados.0.smithi163.stdout:1158: writing smithi16367185-7 from 413571 to 966195 tid 1 2023-10-09T09:10:09.244 INFO:tasks.rados.rados.0.smithi163.stdout:1158: writing smithi16367185-7 from 1641990 to 2404135 tid 2 2023-10-09T09:10:09.250 INFO:tasks.rados.rados.0.smithi163.stdout:1158: writing smithi16367185-7 from 2990498 to 3415661 tid 3 2023-10-09T09:10:09.250 INFO:tasks.rados.rados.0.smithi163.stdout:1159: write oid 19 current snap is 146 2023-10-09T09:10:09.250 INFO:tasks.rados.rados.0.smithi163.stdout:1158: finishing write tid 1 to smithi16367185-7 2023-10-09T09:10:09.250 INFO:tasks.rados.rados.0.smithi163.stdout:1158: finishing write tid 2 to smithi16367185-7 2023-10-09T09:10:09.250 INFO:tasks.rados.rados.0.smithi163.stdout:1159: seq_num 454 ranges {430378=627873,1700472=622156,3017921=612457} 2023-10-09T09:10:09.256 INFO:tasks.rados.rados.0.smithi163.stdout:1159: writing smithi16367185-19 from 430378 to 1058251 tid 1 2023-10-09T09:10:09.263 INFO:tasks.rados.rados.0.smithi163.stdout:1159: writing smithi16367185-19 from 1700472 to 2322628 tid 2 2023-10-09T09:10:09.271 INFO:tasks.rados.rados.0.smithi163.stdout:1159: writing smithi16367185-19 from 3017921 to 3630378 tid 3 2023-10-09T09:10:09.271 INFO:tasks.rados.rados.0.smithi163.stdout:1160: snap_create 2023-10-09T09:10:09.271 INFO:tasks.rados.rados.0.smithi163.stdout:1158: finishing write tid 3 to smithi16367185-7 2023-10-09T09:10:09.271 INFO:tasks.rados.rados.0.smithi163.stdout:1158: finishing write tid 4 to smithi16367185-7 2023-10-09T09:10:09.271 INFO:tasks.rados.rados.0.smithi163.stdout:1158: finishing write tid 5 to smithi16367185-7 2023-10-09T09:10:09.271 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 280 (ObjNum 453 snap 146 seq_num 453) dirty exists 2023-10-09T09:10:09.271 INFO:tasks.rados.rados.0.smithi163.stdout:1158: left oid 7 (ObjNum 453 snap 146 seq_num 453) 2023-10-09T09:10:09.271 INFO:tasks.rados.rados.0.smithi163.stdout:1159: finishing write tid 1 to smithi16367185-19 2023-10-09T09:10:09.272 INFO:tasks.rados.rados.0.smithi163.stdout:1159: finishing write tid 2 to smithi16367185-19 2023-10-09T09:10:09.278 INFO:tasks.rados.rados.0.smithi163.stdout:1159: finishing write tid 3 to smithi16367185-19 2023-10-09T09:10:09.278 INFO:tasks.rados.rados.0.smithi163.stdout:1159: finishing write tid 4 to smithi16367185-19 2023-10-09T09:10:09.278 INFO:tasks.rados.rados.0.smithi163.stdout:1159: finishing write tid 5 to smithi16367185-19 2023-10-09T09:10:09.278 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 370 (ObjNum 454 snap 146 seq_num 454) dirty exists 2023-10-09T09:10:09.278 INFO:tasks.rados.rados.0.smithi163.stdout:1159: left oid 19 (ObjNum 454 snap 146 seq_num 454) 2023-10-09T09:10:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:09 smithi163 bash[33912]: cluster 2023-10-09T09:10:08.199268+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2023-10-09T09:10:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:09 smithi163 bash[33912]: cluster 2023-10-09T09:10:08.203143+0000 osd.6 (osd.6) 20 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:09 smithi163 bash[33912]: cluster 2023-10-09T09:10:08.204079+0000 osd.0 (osd.0) 30 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:09 smithi163 bash[33912]: cluster 2023-10-09T09:10:08.212973+0000 osd.4 (osd.4) 42 : cluster [WRN] failed to encode map e347 with expected crc 2023-10-09T09:10:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:09 smithi163 bash[33912]: cluster 2023-10-09T09:10:08.213338+0000 osd.5 (osd.5) 21 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:09 smithi163 bash[33912]: cluster 2023-10-09T09:10:08.321161+0000 osd.7 (osd.7) 33 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:09 smithi163 bash[33912]: debug 2023-10-09T09:10:09.218+0000 7f84df46a700 -1 mon.c@2(peon).osd e348 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:09.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:09 smithi181 bash[27712]: cluster 2023-10-09T09:10:08.199268+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2023-10-09T09:10:09.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:09 smithi181 bash[27712]: cluster 2023-10-09T09:10:08.203143+0000 osd.6 (osd.6) 20 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:09.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:09 smithi181 bash[27712]: cluster 2023-10-09T09:10:08.204079+0000 osd.0 (osd.0) 30 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:09.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:09 smithi181 bash[27712]: cluster 2023-10-09T09:10:08.212973+0000 osd.4 (osd.4) 42 : cluster [WRN] failed to encode map e347 with expected crc 2023-10-09T09:10:09.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:09 smithi181 bash[27712]: cluster 2023-10-09T09:10:08.213338+0000 osd.5 (osd.5) 21 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:09.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:09 smithi181 bash[27712]: cluster 2023-10-09T09:10:08.321161+0000 osd.7 (osd.7) 33 : cluster [WRN] failed to encode map e346 with expected crc 2023-10-09T09:10:09.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:09 smithi181 bash[27712]: debug 2023-10-09T09:10:09.215+0000 7fde83035700 -1 mon.b@1(peon).osd e348 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:09.952 INFO:tasks.rados.rados.0.smithi163.stdout:1158: done (2 left) 2023-10-09T09:10:09.952 INFO:tasks.rados.rados.0.smithi163.stdout:1159: done (1 left) 2023-10-09T09:10:09.952 INFO:tasks.rados.rados.0.smithi163.stdout:1160: done (0 left) 2023-10-09T09:10:09.952 INFO:tasks.rados.rados.0.smithi163.stdout:1161: read oid 47 snap 131 2023-10-09T09:10:09.953 INFO:tasks.rados.rados.0.smithi163.stdout:1161: expect (ObjNum 415 snap 127 seq_num 415) 2023-10-09T09:10:09.953 INFO:tasks.rados.rados.0.smithi163.stdout:1162: snap_remove snap 132 2023-10-09T09:10:10.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:09 smithi163 bash[33912]: debug 2023-10-09T09:10:09.937+0000 7f84df46a700 -1 mon.c@2(peon).osd e349 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:10.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:09 smithi181 bash[27712]: debug 2023-10-09T09:10:09.935+0000 7fde83035700 -1 mon.b@1(peon).osd e349 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:10 smithi181 bash[27712]: cluster 2023-10-09T09:10:08.397079+0000 osd.2 (osd.2) 36 : cluster [WRN] failed to encode map e347 with expected crc 2023-10-09T09:10:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:10 smithi181 bash[27712]: cluster 2023-10-09T09:10:09.222514+0000 mgr.y (mgr.34107) 38 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 146 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 752 KiB/s wr, 15 op/s 2023-10-09T09:10:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:10 smithi181 bash[27712]: cluster 2023-10-09T09:10:09.222908+0000 mon.a (mon.0) 121 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2023-10-09T09:10:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:10 smithi181 bash[27712]: cluster 2023-10-09T09:10:09.225667+0000 osd.5 (osd.5) 22 : cluster [WRN] failed to encode map e348 with expected crc 2023-10-09T09:10:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:10 smithi181 bash[27712]: cluster 2023-10-09T09:10:09.225784+0000 osd.2 (osd.2) 37 : cluster [WRN] failed to encode map e348 with expected crc 2023-10-09T09:10:10.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:10 smithi181 bash[27712]: cluster 2023-10-09T09:10:09.237720+0000 osd.1 (osd.1) 18 : cluster [WRN] failed to encode map e348 with expected crc 2023-10-09T09:10:10.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:10 smithi181 bash[27712]: cluster 2023-10-09T09:10:09.945971+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2023-10-09T09:10:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:10 smithi163 bash[33912]: cluster 2023-10-09T09:10:08.397079+0000 osd.2 (osd.2) 36 : cluster [WRN] failed to encode map e347 with expected crc 2023-10-09T09:10:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:10 smithi163 bash[33912]: cluster 2023-10-09T09:10:09.222514+0000 mgr.y (mgr.34107) 38 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 146 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 752 KiB/s wr, 15 op/s 2023-10-09T09:10:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:10 smithi163 bash[33912]: cluster 2023-10-09T09:10:09.222908+0000 mon.a (mon.0) 121 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2023-10-09T09:10:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:10 smithi163 bash[33912]: cluster 2023-10-09T09:10:09.225667+0000 osd.5 (osd.5) 22 : cluster [WRN] failed to encode map e348 with expected crc 2023-10-09T09:10:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:10 smithi163 bash[33912]: cluster 2023-10-09T09:10:09.225784+0000 osd.2 (osd.2) 37 : cluster [WRN] failed to encode map e348 with expected crc 2023-10-09T09:10:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:10 smithi163 bash[33912]: cluster 2023-10-09T09:10:09.237720+0000 osd.1 (osd.1) 18 : cluster [WRN] failed to encode map e348 with expected crc 2023-10-09T09:10:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:10 smithi163 bash[33912]: cluster 2023-10-09T09:10:09.945971+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2023-10-09T09:10:10.967 INFO:tasks.rados.rados.0.smithi163.stdout:1162: done (1 left) 2023-10-09T09:10:10.967 INFO:tasks.rados.rados.0.smithi163.stdout:1163: snap_remove snap 146 2023-10-09T09:10:11.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:10 smithi163 bash[33912]: debug 2023-10-09T09:10:10.953+0000 7f84df46a700 -1 mon.c@2(peon).osd e350 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:11.232 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:10 smithi181 bash[27712]: debug 2023-10-09T09:10:10.951+0000 7fde83035700 -1 mon.b@1(peon).osd e350 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:11 smithi181 bash[27712]: cluster 2023-10-09T09:10:09.947644+0000 osd.4 (osd.4) 43 : cluster [WRN] failed to encode map e349 with expected crc 2023-10-09T09:10:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:11 smithi181 bash[27712]: cluster 2023-10-09T09:10:09.950383+0000 osd.0 (osd.0) 31 : cluster [WRN] failed to encode map e349 with expected crc 2023-10-09T09:10:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:11 smithi181 bash[27712]: cluster 2023-10-09T09:10:09.952309+0000 osd.5 (osd.5) 23 : cluster [WRN] failed to encode map e349 with expected crc 2023-10-09T09:10:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:11 smithi181 bash[27712]: audit 2023-10-09T09:10:10.325930+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:11 smithi181 bash[27712]: cluster 2023-10-09T09:10:10.961400+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2023-10-09T09:10:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:11 smithi163 bash[33912]: cluster 2023-10-09T09:10:09.947644+0000 osd.4 (osd.4) 43 : cluster [WRN] failed to encode map e349 with expected crc 2023-10-09T09:10:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:11 smithi163 bash[33912]: cluster 2023-10-09T09:10:09.950383+0000 osd.0 (osd.0) 31 : cluster [WRN] failed to encode map e349 with expected crc 2023-10-09T09:10:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:11 smithi163 bash[33912]: cluster 2023-10-09T09:10:09.952309+0000 osd.5 (osd.5) 23 : cluster [WRN] failed to encode map e349 with expected crc 2023-10-09T09:10:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:11 smithi163 bash[33912]: audit 2023-10-09T09:10:10.325930+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:11 smithi163 bash[33912]: cluster 2023-10-09T09:10:10.961400+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2023-10-09T09:10:11.972 INFO:tasks.rados.rados.0.smithi163.stdout:1163: done (1 left) 2023-10-09T09:10:11.972 INFO:tasks.rados.rados.0.smithi163.stdout:1164: delete oid 21 current snap is 147 2023-10-09T09:10:11.972 INFO:tasks.rados.rados.0.smithi163.stdout:1161: expect (ObjNum 415 snap 127 seq_num 415) 2023-10-09T09:10:12.081 INFO:tasks.rados.rados.0.smithi163.stdout:1161: done (1 left) 2023-10-09T09:10:12.081 INFO:tasks.rados.rados.0.smithi163.stdout:1164: done (0 left) 2023-10-09T09:10:12.081 INFO:tasks.rados.rados.0.smithi163.stdout:1165: rollback oid 19 current snap is 147 2023-10-09T09:10:12.081 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 19 to 140 2023-10-09T09:10:12.081 INFO:tasks.rados.rados.0.smithi163.stdout:1166: read oid 5 snap -1 2023-10-09T09:10:12.081 INFO:tasks.rados.rados.0.smithi163.stdout:1166: expect (ObjNum 436 snap 136 seq_num 436) 2023-10-09T09:10:12.081 INFO:tasks.rados.rados.0.smithi163.stdout:1167: write oid 1 current snap is 147 2023-10-09T09:10:12.081 INFO:tasks.rados.rados.0.smithi163.stdout:1167: seq_num 456 ranges {463992=778371,1701083=625825,2956414=658979} 2023-10-09T09:10:12.093 INFO:tasks.rados.rados.0.smithi163.stdout:1167: writing smithi16367185-1 from 463992 to 1242363 tid 1 2023-10-09T09:10:12.101 INFO:tasks.rados.rados.0.smithi163.stdout:1167: writing smithi16367185-1 from 1701083 to 2326908 tid 2 2023-10-09T09:10:12.108 INFO:tasks.rados.rados.0.smithi163.stdout:1167: writing smithi16367185-1 from 2956414 to 3615393 tid 3 2023-10-09T09:10:12.108 INFO:tasks.rados.rados.0.smithi163.stdout:1168: rollback oid 10 current snap is 147 2023-10-09T09:10:12.108 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 145 2023-10-09T09:10:12.108 INFO:tasks.rados.rados.0.smithi163.stdout:1166: expect (ObjNum 436 snap 136 seq_num 436) 2023-10-09T09:10:12.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:11 smithi163 bash[33912]: debug 2023-10-09T09:10:11.961+0000 7f84df46a700 -1 mon.c@2(peon).osd e351 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:12.235 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:11 smithi181 bash[27712]: debug 2023-10-09T09:10:11.959+0000 7fde83035700 -1 mon.b@1(peon).osd e351 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:12.403 INFO:tasks.rados.rados.0.smithi163.stdout:1165: finishing rollback tid 0 to smithi16367185-19 2023-10-09T09:10:12.423 INFO:tasks.rados.rados.0.smithi163.stdout:1165: finishing rollback tid 1 to smithi16367185-19 2023-10-09T09:10:12.423 INFO:tasks.rados.rados.0.smithi163.stdout:1165: finishing rollback tid 2 to smithi16367185-19 2023-10-09T09:10:12.423 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 375 (ObjNum 390 snap 119 seq_num 390) dirty exists 2023-10-09T09:10:12.423 INFO:tasks.rados.rados.0.smithi163.stdout:1167: finishing write tid 1 to smithi16367185-1 2023-10-09T09:10:12.423 INFO:tasks.rados.rados.0.smithi163.stdout:1167: finishing write tid 2 to smithi16367185-1 2023-10-09T09:10:12.423 INFO:tasks.rados.rados.0.smithi163.stdout:1167: finishing write tid 3 to smithi16367185-1 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:1168: finishing rollback tid 0 to smithi16367185-10 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:1167: finishing write tid 4 to smithi16367185-1 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:1167: finishing write tid 5 to smithi16367185-1 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 643 (ObjNum 456 snap 147 seq_num 456) dirty exists 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:1167: left oid 1 (ObjNum 456 snap 147 seq_num 456) 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:1168: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:1168: finishing rollback tid 2 to smithi16367185-10 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 430 (ObjNum 435 snap 136 seq_num 435) dirty exists 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:1165: done (3 left) 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:1166: done (2 left) 2023-10-09T09:10:12.424 INFO:tasks.rados.rados.0.smithi163.stdout:1167: done (1 left) 2023-10-09T09:10:12.425 INFO:tasks.rados.rados.0.smithi163.stdout:1168: done (0 left) 2023-10-09T09:10:12.425 INFO:tasks.rados.rados.0.smithi163.stdout:1169: read oid 2 snap -1 2023-10-09T09:10:12.425 INFO:tasks.rados.rados.0.smithi163.stdout:1169: expect (ObjNum 376 snap 117 seq_num 376) 2023-10-09T09:10:12.425 INFO:tasks.rados.rados.0.smithi163.stdout:1170: write oid 12 current snap is 147 2023-10-09T09:10:12.425 INFO:tasks.rados.rados.0.smithi163.stdout:1170: seq_num 457 ranges {480799=569973,1475918=769483,2700190=586604} 2023-10-09T09:10:12.425 INFO:tasks.rados.rados.0.smithi163.stdout:1170: writing smithi16367185-12 from 480799 to 1050772 tid 1 2023-10-09T09:10:12.426 INFO:tasks.rados.rados.0.smithi163.stdout:1170: writing smithi16367185-12 from 1475918 to 2245401 tid 2 2023-10-09T09:10:12.429 INFO:tasks.rados.rados.0.smithi163.stdout:1170: writing smithi16367185-12 from 2700190 to 3286794 tid 3 2023-10-09T09:10:12.429 INFO:tasks.rados.rados.0.smithi163.stdout:1171: write oid 47 current snap is 147 2023-10-09T09:10:12.429 INFO:tasks.rados.rados.0.smithi163.stdout:1171: seq_num 458 ranges {497606=645222,1934400=629494,3243966=453640} 2023-10-09T09:10:12.435 INFO:tasks.rados.rados.0.smithi163.stdout:1171: writing smithi16367185-47 from 497606 to 1142828 tid 1 2023-10-09T09:10:12.443 INFO:tasks.rados.rados.0.smithi163.stdout:1171: writing smithi16367185-47 from 1934400 to 2563894 tid 2 2023-10-09T09:10:12.450 INFO:tasks.rados.rados.0.smithi163.stdout:1171: writing smithi16367185-47 from 3243966 to 3697606 tid 3 2023-10-09T09:10:12.450 INFO:tasks.rados.rados.0.smithi163.stdout:1172: write oid 1 current snap is 147 2023-10-09T09:10:12.450 INFO:tasks.rados.rados.0.smithi163.stdout:1172: seq_num 459 ranges {514413=720471,1876529=773152,3438683=275730} 2023-10-09T09:10:12.457 INFO:tasks.rados.rados.0.smithi163.stdout:1172: writing smithi16367185-1 from 514413 to 1234884 tid 1 2023-10-09T09:10:12.465 INFO:tasks.rados.rados.0.smithi163.stdout:1172: writing smithi16367185-1 from 1876529 to 2649681 tid 2 2023-10-09T09:10:12.471 INFO:tasks.rados.rados.0.smithi163.stdout:1172: writing smithi16367185-1 from 3438683 to 3714413 tid 3 2023-10-09T09:10:12.472 INFO:tasks.rados.rados.0.smithi163.stdout:1173: rollback oid 25 current snap is 147 2023-10-09T09:10:12.472 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 25 to 144 2023-10-09T09:10:12.472 INFO:tasks.rados.rados.0.smithi163.stdout:1169: expect (ObjNum 376 snap 117 seq_num 376) 2023-10-09T09:10:12.555 INFO:tasks.rados.rados.0.smithi163.stdout:1170: finishing write tid 1 to smithi16367185-12 2023-10-09T09:10:12.556 INFO:tasks.rados.rados.0.smithi163.stdout:1170: finishing write tid 2 to smithi16367185-12 2023-10-09T09:10:12.556 INFO:tasks.rados.rados.0.smithi163.stdout:1170: finishing write tid 3 to smithi16367185-12 2023-10-09T09:10:12.556 INFO:tasks.rados.rados.0.smithi163.stdout:1170: finishing write tid 4 to smithi16367185-12 2023-10-09T09:10:12.556 INFO:tasks.rados.rados.0.smithi163.stdout:1170: finishing write tid 5 to smithi16367185-12 2023-10-09T09:10:12.556 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 438 (ObjNum 457 snap 147 seq_num 457) dirty exists 2023-10-09T09:10:12.556 INFO:tasks.rados.rados.0.smithi163.stdout:1170: left oid 12 (ObjNum 457 snap 147 seq_num 457) 2023-10-09T09:10:12.556 INFO:tasks.rados.rados.0.smithi163.stdout:1171: finishing write tid 1 to smithi16367185-47 2023-10-09T09:10:12.556 INFO:tasks.rados.rados.0.smithi163.stdout:1171: finishing write tid 2 to smithi16367185-47 2023-10-09T09:10:12.557 INFO:tasks.rados.rados.0.smithi163.stdout:1171: finishing write tid 3 to smithi16367185-47 2023-10-09T09:10:12.557 INFO:tasks.rados.rados.0.smithi163.stdout:1171: finishing write tid 4 to smithi16367185-47 2023-10-09T09:10:12.557 INFO:tasks.rados.rados.0.smithi163.stdout:1171: finishing write tid 5 to smithi16367185-47 2023-10-09T09:10:12.557 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 915 (ObjNum 458 snap 147 seq_num 458) dirty exists 2023-10-09T09:10:12.557 INFO:tasks.rados.rados.0.smithi163.stdout:1171: left oid 47 (ObjNum 458 snap 147 seq_num 458) 2023-10-09T09:10:12.557 INFO:tasks.rados.rados.0.smithi163.stdout:1172: finishing write tid 1 to smithi16367185-1 2023-10-09T09:10:12.557 INFO:tasks.rados.rados.0.smithi163.stdout:1172: finishing write tid 2 to smithi16367185-1 2023-10-09T09:10:12.558 INFO:tasks.rados.rados.0.smithi163.stdout:1173: finishing rollback tid 0 to smithi16367185-25 2023-10-09T09:10:12.558 INFO:tasks.rados.rados.0.smithi163.stdout:1173: finishing rollback tid 1 to smithi16367185-25 2023-10-09T09:10:12.558 INFO:tasks.rados.rados.0.smithi163.stdout:1173: finishing rollback tid 2 to smithi16367185-25 2023-10-09T09:10:12.558 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 442 (ObjNum 443 snap 138 seq_num 443) dirty exists 2023-10-09T09:10:12.558 INFO:tasks.rados.rados.0.smithi163.stdout:1172: finishing write tid 3 to smithi16367185-1 2023-10-09T09:10:12.558 INFO:tasks.rados.rados.0.smithi163.stdout:1172: finishing write tid 4 to smithi16367185-1 2023-10-09T09:10:12.558 INFO:tasks.rados.rados.0.smithi163.stdout:1172: finishing write tid 5 to smithi16367185-1 2023-10-09T09:10:12.558 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 647 (ObjNum 459 snap 147 seq_num 459) dirty exists 2023-10-09T09:10:12.558 INFO:tasks.rados.rados.0.smithi163.stdout:1172: left oid 1 (ObjNum 459 snap 147 seq_num 459) 2023-10-09T09:10:12.559 INFO:tasks.rados.rados.0.smithi163.stdout:1169: done (4 left) 2023-10-09T09:10:12.559 INFO:tasks.rados.rados.0.smithi163.stdout:1170: done (3 left) 2023-10-09T09:10:12.559 INFO:tasks.rados.rados.0.smithi163.stdout:1171: done (2 left) 2023-10-09T09:10:12.559 INFO:tasks.rados.rados.0.smithi163.stdout:1172: done (1 left) 2023-10-09T09:10:12.559 INFO:tasks.rados.rados.0.smithi163.stdout:1173: done (0 left) 2023-10-09T09:10:12.559 INFO:tasks.rados.rados.0.smithi163.stdout:1174: write oid 12 current snap is 147 2023-10-09T09:10:12.559 INFO:tasks.rados.rados.0.smithi163.stdout:1174: seq_num 460 ranges {531220=795720,1935011=516810,3066106=420420} 2023-10-09T09:10:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:12 smithi181 bash[27712]: cluster 2023-10-09T09:10:10.963508+0000 osd.4 (osd.4) 44 : cluster [WRN] failed to encode map e350 with expected crc 2023-10-09T09:10:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:12 smithi181 bash[27712]: cluster 2023-10-09T09:10:10.965873+0000 osd.0 (osd.0) 32 : cluster [WRN] failed to encode map e350 with expected crc 2023-10-09T09:10:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:12 smithi181 bash[27712]: cluster 2023-10-09T09:10:11.223639+0000 mgr.y (mgr.34107) 39 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 152 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 880 KiB/s wr, 2 op/s 2023-10-09T09:10:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:12 smithi181 bash[27712]: cluster 2023-10-09T09:10:11.968162+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2023-10-09T09:10:12.570 INFO:tasks.rados.rados.0.smithi163.stdout:1174: writing smithi16367185-12 from 531220 to 1326940 tid 1 2023-10-09T09:10:12.576 INFO:tasks.rados.rados.0.smithi163.stdout:1174: writing smithi16367185-12 from 1935011 to 2451821 tid 2 2023-10-09T09:10:12.582 INFO:tasks.rados.rados.0.smithi163.stdout:1174: writing smithi16367185-12 from 3066106 to 3486526 tid 3 2023-10-09T09:10:12.582 INFO:tasks.rados.rados.0.smithi163.stdout:1175: snap_create 2023-10-09T09:10:12.582 INFO:tasks.rados.rados.0.smithi163.stdout:1174: finishing write tid 1 to smithi16367185-12 2023-10-09T09:10:12.583 INFO:tasks.rados.rados.0.smithi163.stdout:1174: finishing write tid 2 to smithi16367185-12 2023-10-09T09:10:12.586 INFO:tasks.rados.rados.0.smithi163.stdout:1174: finishing write tid 3 to smithi16367185-12 2023-10-09T09:10:12.586 INFO:tasks.rados.rados.0.smithi163.stdout:1174: finishing write tid 4 to smithi16367185-12 2023-10-09T09:10:12.586 INFO:tasks.rados.rados.0.smithi163.stdout:1174: finishing write tid 5 to smithi16367185-12 2023-10-09T09:10:12.586 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 446 (ObjNum 460 snap 147 seq_num 460) dirty exists 2023-10-09T09:10:12.586 INFO:tasks.rados.rados.0.smithi163.stdout:1174: left oid 12 (ObjNum 460 snap 147 seq_num 460) 2023-10-09T09:10:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:12 smithi163 bash[33912]: cluster 2023-10-09T09:10:10.963508+0000 osd.4 (osd.4) 44 : cluster [WRN] failed to encode map e350 with expected crc 2023-10-09T09:10:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:12 smithi163 bash[33912]: cluster 2023-10-09T09:10:10.965873+0000 osd.0 (osd.0) 32 : cluster [WRN] failed to encode map e350 with expected crc 2023-10-09T09:10:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:12 smithi163 bash[33912]: cluster 2023-10-09T09:10:11.223639+0000 mgr.y (mgr.34107) 39 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 152 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 880 KiB/s wr, 2 op/s 2023-10-09T09:10:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:12 smithi163 bash[33912]: cluster 2023-10-09T09:10:11.968162+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2023-10-09T09:10:13.258 INFO:tasks.rados.rados.0.smithi163.stdout:1174: done (1 left) 2023-10-09T09:10:13.258 INFO:tasks.rados.rados.0.smithi163.stdout:1175: done (0 left) 2023-10-09T09:10:13.258 INFO:tasks.rados.rados.0.smithi163.stdout:1176: rollback oid 13 current snap is 148 2023-10-09T09:10:13.258 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 13 to 124 2023-10-09T09:10:13.258 INFO:tasks.rados.rados.0.smithi163.stdout:1177: read oid 18 snap 135 2023-10-09T09:10:13.258 INFO:tasks.rados.rados.0.smithi163.stdout:1177: expect (ObjNum 379 snap 117 seq_num 379) 2023-10-09T09:10:13.258 INFO:tasks.rados.rados.0.smithi163.stdout:1178: delete oid 24 current snap is 148 2023-10-09T09:10:13.259 INFO:tasks.rados.rados.0.smithi163.stdout:1176: finishing rollback tid 1 to smithi16367185-13 2023-10-09T09:10:13.259 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 0 (ObjNum 698991616 snap 21875 seq_num 1342179536) dirty dne 2023-10-09T09:10:13.263 INFO:tasks.rados.rados.0.smithi163.stdout:1176: done (2 left) 2023-10-09T09:10:13.263 INFO:tasks.rados.rados.0.smithi163.stdout:1178: done (1 left) 2023-10-09T09:10:13.263 INFO:tasks.rados.rados.0.smithi163.stdout:1179: snap_create 2023-10-09T09:10:13.267 INFO:tasks.rados.rados.0.smithi163.stdout:1177: expect (ObjNum 379 snap 117 seq_num 379) 2023-10-09T09:10:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:13 smithi181 bash[27712]: debug 2023-10-09T09:10:13.239+0000 7fde83035700 -1 mon.b@1(peon).osd e352 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:13 smithi181 bash[27712]: cluster 2023-10-09T09:10:11.431882+0000 osd.2 (osd.2) 38 : cluster [WRN] failed to encode map e350 with expected crc 2023-10-09T09:10:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:13 smithi181 bash[27712]: cluster 2023-10-09T09:10:11.968785+0000 osd.4 (osd.4) 45 : cluster [WRN] failed to encode map e351 with expected crc 2023-10-09T09:10:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:13 smithi181 bash[27712]: cluster 2023-10-09T09:10:11.971835+0000 osd.2 (osd.2) 39 : cluster [WRN] failed to encode map e351 with expected crc 2023-10-09T09:10:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:13 smithi181 bash[27712]: cluster 2023-10-09T09:10:11.974518+0000 osd.1 (osd.1) 19 : cluster [WRN] failed to encode map e351 with expected crc 2023-10-09T09:10:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:10:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:10:13] "GET /metrics HTTP/1.1" 200 36080 "" "Prometheus/2.33.4" 2023-10-09T09:10:13.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:13 smithi163 bash[33912]: debug 2023-10-09T09:10:13.245+0000 7f84df46a700 -1 mon.c@2(peon).osd e352 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:13.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:13 smithi163 bash[33912]: cluster 2023-10-09T09:10:11.431882+0000 osd.2 (osd.2) 38 : cluster [WRN] failed to encode map e350 with expected crc 2023-10-09T09:10:13.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:13 smithi163 bash[33912]: cluster 2023-10-09T09:10:11.968785+0000 osd.4 (osd.4) 45 : cluster [WRN] failed to encode map e351 with expected crc 2023-10-09T09:10:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:13 smithi163 bash[33912]: cluster 2023-10-09T09:10:11.971835+0000 osd.2 (osd.2) 39 : cluster [WRN] failed to encode map e351 with expected crc 2023-10-09T09:10:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:13 smithi163 bash[33912]: cluster 2023-10-09T09:10:11.974518+0000 osd.1 (osd.1) 19 : cluster [WRN] failed to encode map e351 with expected crc 2023-10-09T09:10:14.265 INFO:tasks.rados.rados.0.smithi163.stdout:1177: done (1 left) 2023-10-09T09:10:14.265 INFO:tasks.rados.rados.0.smithi163.stdout:1179: done (0 left) 2023-10-09T09:10:14.265 INFO:tasks.rados.rados.0.smithi163.stdout:1180: snap_remove snap 125 2023-10-09T09:10:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:14 smithi181 bash[27712]: cluster 2023-10-09T09:10:13.224475+0000 mgr.y (mgr.34107) 40 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 157 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2023-10-09T09:10:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:14 smithi181 bash[27712]: cluster 2023-10-09T09:10:13.250924+0000 osd.7 (osd.7) 34 : cluster [WRN] failed to encode map e352 with expected crc 2023-10-09T09:10:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:14 smithi181 bash[27712]: cluster 2023-10-09T09:10:13.251546+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2023-10-09T09:10:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:14 smithi181 bash[27712]: cluster 2023-10-09T09:10:13.254736+0000 osd.1 (osd.1) 20 : cluster [WRN] failed to encode map e352 with expected crc 2023-10-09T09:10:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:14 smithi181 bash[27712]: cluster 2023-10-09T09:10:13.254760+0000 osd.6 (osd.6) 21 : cluster [WRN] failed to encode map e352 with expected crc 2023-10-09T09:10:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:14 smithi181 bash[27712]: cluster 2023-10-09T09:10:13.257945+0000 osd.2 (osd.2) 40 : cluster [WRN] failed to encode map e352 with expected crc 2023-10-09T09:10:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:14 smithi181 bash[27712]: cluster 2023-10-09T09:10:13.258067+0000 osd.3 (osd.3) 28 : cluster [WRN] failed to encode map e352 with expected crc 2023-10-09T09:10:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:14 smithi181 bash[27712]: debug 2023-10-09T09:10:14.251+0000 7fde83035700 -1 mon.b@1(peon).osd e353 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:14 smithi163 bash[33912]: cluster 2023-10-09T09:10:13.224475+0000 mgr.y (mgr.34107) 40 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 157 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2023-10-09T09:10:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:14 smithi163 bash[33912]: cluster 2023-10-09T09:10:13.250924+0000 osd.7 (osd.7) 34 : cluster [WRN] failed to encode map e352 with expected crc 2023-10-09T09:10:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:14 smithi163 bash[33912]: cluster 2023-10-09T09:10:13.251546+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2023-10-09T09:10:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:14 smithi163 bash[33912]: cluster 2023-10-09T09:10:13.254736+0000 osd.1 (osd.1) 20 : cluster [WRN] failed to encode map e352 with expected crc 2023-10-09T09:10:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:14 smithi163 bash[33912]: cluster 2023-10-09T09:10:13.254760+0000 osd.6 (osd.6) 21 : cluster [WRN] failed to encode map e352 with expected crc 2023-10-09T09:10:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:14 smithi163 bash[33912]: cluster 2023-10-09T09:10:13.257945+0000 osd.2 (osd.2) 40 : cluster [WRN] failed to encode map e352 with expected crc 2023-10-09T09:10:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:14 smithi163 bash[33912]: cluster 2023-10-09T09:10:13.258067+0000 osd.3 (osd.3) 28 : cluster [WRN] failed to encode map e352 with expected crc 2023-10-09T09:10:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:14 smithi163 bash[33912]: debug 2023-10-09T09:10:14.253+0000 7f84df46a700 -1 mon.c@2(peon).osd e353 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:14.956 INFO:tasks.rados.rados.0.smithi163.stdout:1180: done (0 left) 2023-10-09T09:10:14.956 INFO:tasks.rados.rados.0.smithi163.stdout:1181: snap_remove snap 131 2023-10-09T09:10:15.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:14 smithi163 bash[33912]: debug 2023-10-09T09:10:14.941+0000 7f84df46a700 -1 mon.c@2(peon).osd e354 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:15.260 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:14 smithi181 bash[27712]: debug 2023-10-09T09:10:14.939+0000 7fde83035700 -1 mon.b@1(peon).osd e354 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:15 smithi181 bash[27712]: cluster 2023-10-09T09:10:14.259238+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2023-10-09T09:10:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:15 smithi181 bash[27712]: cluster 2023-10-09T09:10:14.261560+0000 osd.4 (osd.4) 46 : cluster [WRN] failed to encode map e353 with expected crc 2023-10-09T09:10:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:15 smithi181 bash[27712]: cluster 2023-10-09T09:10:14.262151+0000 osd.0 (osd.0) 33 : cluster [WRN] failed to encode map e353 with expected crc 2023-10-09T09:10:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:15 smithi181 bash[27712]: cluster 2023-10-09T09:10:14.948455+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2023-10-09T09:10:15.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:15 smithi163 bash[33912]: cluster 2023-10-09T09:10:14.259238+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2023-10-09T09:10:15.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:15 smithi163 bash[33912]: cluster 2023-10-09T09:10:14.261560+0000 osd.4 (osd.4) 46 : cluster [WRN] failed to encode map e353 with expected crc 2023-10-09T09:10:15.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:15 smithi163 bash[33912]: cluster 2023-10-09T09:10:14.262151+0000 osd.0 (osd.0) 33 : cluster [WRN] failed to encode map e353 with expected crc 2023-10-09T09:10:15.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:15 smithi163 bash[33912]: cluster 2023-10-09T09:10:14.948455+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2023-10-09T09:10:15.974 INFO:tasks.rados.rados.0.smithi163.stdout:1181: done (0 left) 2023-10-09T09:10:15.974 INFO:tasks.rados.rados.0.smithi163.stdout:1182: rollback oid 2 current snap is 149 2023-10-09T09:10:15.974 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 2 to 137 2023-10-09T09:10:15.974 INFO:tasks.rados.rados.0.smithi163.stdout:1183: write oid 26 current snap is 149 2023-10-09T09:10:15.974 INFO:tasks.rados.rados.0.smithi163.stdout:1183: seq_num 462 ranges {564834=546218,1651975=520479,2720952=559317} 2023-10-09T09:10:15.985 INFO:tasks.rados.rados.0.smithi163.stdout:1183: writing smithi16367185-26 from 564834 to 1111052 tid 1 2023-10-09T09:10:15.992 INFO:tasks.rados.rados.0.smithi163.stdout:1183: writing smithi16367185-26 from 1651975 to 2172454 tid 2 2023-10-09T09:10:15.999 INFO:tasks.rados.rados.0.smithi163.stdout:1183: writing smithi16367185-26 from 2720952 to 3280269 tid 3 2023-10-09T09:10:15.999 INFO:tasks.rados.rados.0.smithi163.stdout:1184: rollback oid 44 current snap is 149 2023-10-09T09:10:15.999 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 44 to 147 2023-10-09T09:10:15.999 INFO:tasks.rados.rados.0.smithi163.stdout:1182: finishing rollback tid 0 to smithi16367185-2 2023-10-09T09:10:15.999 INFO:tasks.rados.rados.0.smithi163.stdout:1182: finishing rollback tid 1 to smithi16367185-2 2023-10-09T09:10:15.999 INFO:tasks.rados.rados.0.smithi163.stdout:1182: finishing rollback tid 2 to smithi16367185-2 2023-10-09T09:10:15.999 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 362 (ObjNum 376 snap 117 seq_num 376) dirty exists 2023-10-09T09:10:16.000 INFO:tasks.rados.rados.0.smithi163.stdout:1183: finishing write tid 1 to smithi16367185-26 2023-10-09T09:10:16.000 INFO:tasks.rados.rados.0.smithi163.stdout:1183: finishing write tid 2 to smithi16367185-26 2023-10-09T09:10:16.000 INFO:tasks.rados.rados.0.smithi163.stdout:1182: done (2 left) 2023-10-09T09:10:16.000 INFO:tasks.rados.rados.0.smithi163.stdout:1185: rollback oid 14 current snap is 149 2023-10-09T09:10:16.000 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 147 2023-10-09T09:10:16.000 INFO:tasks.rados.rados.0.smithi163.stdout:1186: snap_remove snap 139 2023-10-09T09:10:16.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:15 smithi163 bash[33912]: debug 2023-10-09T09:10:15.961+0000 7f84df46a700 -1 mon.c@2(peon).osd e355 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:16.262 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:15 smithi181 bash[27712]: debug 2023-10-09T09:10:15.963+0000 7fde83035700 -1 mon.b@1(peon).osd e355 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:16 smithi181 bash[27712]: cluster 2023-10-09T09:10:14.951643+0000 osd.7 (osd.7) 35 : cluster [WRN] failed to encode map e354 with expected crc 2023-10-09T09:10:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:16 smithi181 bash[27712]: cluster 2023-10-09T09:10:14.952765+0000 osd.1 (osd.1) 21 : cluster [WRN] failed to encode map e354 with expected crc 2023-10-09T09:10:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:16 smithi181 bash[27712]: cluster 2023-10-09T09:10:15.225172+0000 mgr.y (mgr.34107) 41 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 157 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 668 KiB/s wr, 7 op/s 2023-10-09T09:10:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:16 smithi181 bash[27712]: cluster 2023-10-09T09:10:15.968188+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2023-10-09T09:10:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:16 smithi163 bash[33912]: cluster 2023-10-09T09:10:14.951643+0000 osd.7 (osd.7) 35 : cluster [WRN] failed to encode map e354 with expected crc 2023-10-09T09:10:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:16 smithi163 bash[33912]: cluster 2023-10-09T09:10:14.952765+0000 osd.1 (osd.1) 21 : cluster [WRN] failed to encode map e354 with expected crc 2023-10-09T09:10:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:16 smithi163 bash[33912]: cluster 2023-10-09T09:10:15.225172+0000 mgr.y (mgr.34107) 41 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 157 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 668 KiB/s wr, 7 op/s 2023-10-09T09:10:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:16 smithi163 bash[33912]: cluster 2023-10-09T09:10:15.968188+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2023-10-09T09:10:16.978 INFO:tasks.rados.rados.0.smithi163.stdout:1186: done (3 left) 2023-10-09T09:10:16.978 INFO:tasks.rados.rados.0.smithi163.stdout:1187: write oid 29 current snap is 149 2023-10-09T09:10:16.978 INFO:tasks.rados.rados.0.smithi163.stdout:1183: finishing write tid 3 to smithi16367185-26 2023-10-09T09:10:16.978 INFO:tasks.rados.rados.0.smithi163.stdout:1187: seq_num 463 ranges {581641=621467,1994104=780490,3432022=349619} 2023-10-09T09:10:16.989 INFO:tasks.rados.rados.0.smithi163.stdout:1187: writing smithi16367185-29 from 581641 to 1203108 tid 1 2023-10-09T09:10:17.000 INFO:tasks.rados.rados.0.smithi163.stdout:1187: writing smithi16367185-29 from 1994104 to 2774594 tid 2 2023-10-09T09:10:17.007 INFO:tasks.rados.rados.0.smithi163.stdout:1187: writing smithi16367185-29 from 3432022 to 3781641 tid 3 2023-10-09T09:10:17.007 INFO:tasks.rados.rados.0.smithi163.stdout:1188: snap_remove snap 126 2023-10-09T09:10:17.265 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:16 smithi181 bash[27712]: debug 2023-10-09T09:10:16.963+0000 7fde83035700 -1 mon.b@1(peon).osd e356 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:17.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:16 smithi163 bash[33912]: debug 2023-10-09T09:10:16.969+0000 7f84df46a700 -1 mon.c@2(peon).osd e356 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:17 smithi181 bash[27712]: cluster 2023-10-09T09:10:15.970596+0000 osd.4 (osd.4) 47 : cluster [WRN] failed to encode map e355 with expected crc 2023-10-09T09:10:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:17 smithi181 bash[27712]: cluster 2023-10-09T09:10:15.970827+0000 osd.2 (osd.2) 41 : cluster [WRN] failed to encode map e355 with expected crc 2023-10-09T09:10:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:17 smithi181 bash[27712]: cluster 2023-10-09T09:10:15.974150+0000 osd.7 (osd.7) 36 : cluster [WRN] failed to encode map e355 with expected crc 2023-10-09T09:10:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:17 smithi181 bash[27712]: cluster 2023-10-09T09:10:16.974746+0000 mon.a (mon.0) 130 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2023-10-09T09:10:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:17 smithi163 bash[33912]: cluster 2023-10-09T09:10:15.970596+0000 osd.4 (osd.4) 47 : cluster [WRN] failed to encode map e355 with expected crc 2023-10-09T09:10:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:17 smithi163 bash[33912]: cluster 2023-10-09T09:10:15.970827+0000 osd.2 (osd.2) 41 : cluster [WRN] failed to encode map e355 with expected crc 2023-10-09T09:10:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:17 smithi163 bash[33912]: cluster 2023-10-09T09:10:15.974150+0000 osd.7 (osd.7) 36 : cluster [WRN] failed to encode map e355 with expected crc 2023-10-09T09:10:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:17 smithi163 bash[33912]: cluster 2023-10-09T09:10:16.974746+0000 mon.a (mon.0) 130 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2023-10-09T09:10:17.987 INFO:tasks.rados.rados.0.smithi163.stdout:1188: done (4 left) 2023-10-09T09:10:17.987 INFO:tasks.rados.rados.0.smithi163.stdout:1189: rollback oid 41 current snap is 149 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 41 to 147 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:1183: finishing write tid 4 to smithi16367185-26 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:1183: finishing write tid 5 to smithi16367185-26 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 386 (ObjNum 462 snap 149 seq_num 462) dirty exists 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:1183: left oid 26 (ObjNum 462 snap 149 seq_num 462) 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:1185: finishing rollback tid 0 to smithi16367185-14 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:1184: finishing rollback tid 0 to smithi16367185-44 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:1184: finishing rollback tid 1 to smithi16367185-44 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:1185: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:1185: finishing rollback tid 2 to smithi16367185-14 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 464 (ObjNum 438 snap 136 seq_num 438) dirty exists 2023-10-09T09:10:17.988 INFO:tasks.rados.rados.0.smithi163.stdout:1184: finishing rollback tid 2 to smithi16367185-44 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 925 (ObjNum 408 snap 124 seq_num 408) dirty exists 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:1187: finishing write tid 1 to smithi16367185-29 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:1187: finishing write tid 2 to smithi16367185-29 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:1187: finishing write tid 3 to smithi16367185-29 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:1187: finishing write tid 4 to smithi16367185-29 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:1187: finishing write tid 5 to smithi16367185-29 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 459 (ObjNum 463 snap 149 seq_num 463) dirty exists 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:1187: left oid 29 (ObjNum 463 snap 149 seq_num 463) 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:1183: done (4 left) 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:1184: done (3 left) 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:1185: done (2 left) 2023-10-09T09:10:17.989 INFO:tasks.rados.rados.0.smithi163.stdout:1187: done (1 left) 2023-10-09T09:10:17.990 INFO:tasks.rados.rados.0.smithi163.stdout:1190: delete oid 2 current snap is 149 2023-10-09T09:10:17.992 INFO:tasks.rados.rados.0.smithi163.stdout:1189: finishing rollback tid 1 to smithi16367185-41 2023-10-09T09:10:17.992 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 0 (ObjNum 698711344 snap 21875 seq_num 699136944) dirty dne 2023-10-09T09:10:17.998 INFO:tasks.rados.rados.0.smithi163.stdout:1189: done (1 left) 2023-10-09T09:10:17.998 INFO:tasks.rados.rados.0.smithi163.stdout:1190: done (0 left) 2023-10-09T09:10:17.998 INFO:tasks.rados.rados.0.smithi163.stdout:1191: rollback oid 48 current snap is 149 2023-10-09T09:10:17.998 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 48 to 137 2023-10-09T09:10:17.999 INFO:tasks.rados.rados.0.smithi163.stdout:1192: snap_create 2023-10-09T09:10:18.003 INFO:tasks.rados.rados.0.smithi163.stdout:1191: finishing rollback tid 0 to smithi16367185-48 2023-10-09T09:10:18.004 INFO:tasks.rados.rados.0.smithi163.stdout:1191: finishing rollback tid 1 to smithi16367185-48 2023-10-09T09:10:18.004 INFO:tasks.rados.rados.0.smithi163.stdout:1191: finishing rollback tid 2 to smithi16367185-48 2023-10-09T09:10:18.004 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 939 (ObjNum 432 snap 135 seq_num 432) dirty exists 2023-10-09T09:10:18.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:17 smithi181 bash[27712]: debug 2023-10-09T09:10:17.971+0000 7fde83035700 -1 mon.b@1(peon).osd e357 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:18.268 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:17 smithi163 bash[33912]: debug 2023-10-09T09:10:17.977+0000 7f84df46a700 -1 mon.c@2(peon).osd e357 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:18 smithi181 bash[27712]: cluster 2023-10-09T09:10:16.973083+0000 osd.4 (osd.4) 48 : cluster [WRN] failed to encode map e356 with expected crc 2023-10-09T09:10:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:18 smithi181 bash[27712]: cluster 2023-10-09T09:10:16.979657+0000 osd.0 (osd.0) 34 : cluster [WRN] failed to encode map e356 with expected crc 2023-10-09T09:10:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:18 smithi181 bash[27712]: cluster 2023-10-09T09:10:16.992172+0000 osd.2 (osd.2) 42 : cluster [WRN] failed to encode map e356 with expected crc 2023-10-09T09:10:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:18 smithi181 bash[27712]: cluster 2023-10-09T09:10:17.225884+0000 mgr.y (mgr.34107) 42 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 167 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.6 MiB/s wr, 18 op/s 2023-10-09T09:10:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:18 smithi181 bash[27712]: cluster 2023-10-09T09:10:17.986305+0000 mon.a (mon.0) 131 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2023-10-09T09:10:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:18 smithi163 bash[33912]: cluster 2023-10-09T09:10:16.973083+0000 osd.4 (osd.4) 48 : cluster [WRN] failed to encode map e356 with expected crc 2023-10-09T09:10:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:18 smithi163 bash[33912]: cluster 2023-10-09T09:10:16.979657+0000 osd.0 (osd.0) 34 : cluster [WRN] failed to encode map e356 with expected crc 2023-10-09T09:10:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:18 smithi163 bash[33912]: cluster 2023-10-09T09:10:16.992172+0000 osd.2 (osd.2) 42 : cluster [WRN] failed to encode map e356 with expected crc 2023-10-09T09:10:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:18 smithi163 bash[33912]: cluster 2023-10-09T09:10:17.225884+0000 mgr.y (mgr.34107) 42 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 167 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.6 MiB/s wr, 18 op/s 2023-10-09T09:10:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:18 smithi163 bash[33912]: cluster 2023-10-09T09:10:17.986305+0000 mon.a (mon.0) 131 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2023-10-09T09:10:18.982 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:10:18 smithi181 bash[54907]: ts=2023-10-09T09:10:18.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:10:18.994 INFO:tasks.rados.rados.0.smithi163.stdout:1191: done (1 left) 2023-10-09T09:10:18.994 INFO:tasks.rados.rados.0.smithi163.stdout:1192: done (0 left) 2023-10-09T09:10:18.994 INFO:tasks.rados.rados.0.smithi163.stdout:1193: delete oid 11 current snap is 150 2023-10-09T09:10:19.004 INFO:tasks.rados.rados.0.smithi163.stdout:1193: done (0 left) 2023-10-09T09:10:19.004 INFO:tasks.rados.rados.0.smithi163.stdout:1194: delete oid 37 current snap is 150 2023-10-09T09:10:19.006 INFO:tasks.rados.rados.0.smithi163.stdout:1194: done (0 left) 2023-10-09T09:10:19.007 INFO:tasks.rados.rados.0.smithi163.stdout:1195: read oid 46 snap -1 2023-10-09T09:10:19.007 INFO:tasks.rados.rados.0.smithi163.stdout:1195: expect (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:10:19.007 INFO:tasks.rados.rados.0.smithi163.stdout:1196: rollback oid 6 current snap is 150 2023-10-09T09:10:19.007 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 6 to 144 2023-10-09T09:10:19.007 INFO:tasks.rados.rados.0.smithi163.stdout:1197: snap_create 2023-10-09T09:10:19.016 INFO:tasks.rados.rados.0.smithi163.stdout:1195: expect (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:10:19.204 INFO:tasks.rados.rados.0.smithi163.stdout:1196: finishing rollback tid 0 to smithi16367185-6 2023-10-09T09:10:19.204 INFO:tasks.rados.rados.0.smithi163.stdout:1196: finishing rollback tid 1 to smithi16367185-6 2023-10-09T09:10:19.204 INFO:tasks.rados.rados.0.smithi163.stdout:1196: finishing rollback tid 2 to smithi16367185-6 2023-10-09T09:10:19.205 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 114 (ObjNum 431 snap 134 seq_num 431) dirty exists 2023-10-09T09:10:19.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:18 smithi181 bash[27712]: debug 2023-10-09T09:10:18.975+0000 7fde83035700 -1 mon.b@1(peon).osd e358 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:19.271 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:18 smithi163 bash[33912]: debug 2023-10-09T09:10:18.977+0000 7f84df46a700 -1 mon.c@2(peon).osd e358 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:19 smithi181 bash[27712]: cluster 2023-10-09T09:10:17.982089+0000 osd.4 (osd.4) 49 : cluster [WRN] failed to encode map e357 with expected crc 2023-10-09T09:10:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:19 smithi181 bash[27712]: cluster 2023-10-09T09:10:17.986010+0000 osd.0 (osd.0) 35 : cluster [WRN] failed to encode map e357 with expected crc 2023-10-09T09:10:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:19 smithi181 bash[27712]: cluster 2023-10-09T09:10:17.988912+0000 osd.2 (osd.2) 43 : cluster [WRN] failed to encode map e357 with expected crc 2023-10-09T09:10:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:19 smithi181 bash[27712]: cluster 2023-10-09T09:10:18.986249+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2023-10-09T09:10:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:19 smithi181 bash[27712]: cluster 2023-10-09T09:10:18.990833+0000 osd.4 (osd.4) 50 : cluster [WRN] failed to encode map e358 with expected crc 2023-10-09T09:10:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:19 smithi163 bash[33912]: cluster 2023-10-09T09:10:17.982089+0000 osd.4 (osd.4) 49 : cluster [WRN] failed to encode map e357 with expected crc 2023-10-09T09:10:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:19 smithi163 bash[33912]: cluster 2023-10-09T09:10:17.986010+0000 osd.0 (osd.0) 35 : cluster [WRN] failed to encode map e357 with expected crc 2023-10-09T09:10:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:19 smithi163 bash[33912]: cluster 2023-10-09T09:10:17.988912+0000 osd.2 (osd.2) 43 : cluster [WRN] failed to encode map e357 with expected crc 2023-10-09T09:10:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:19 smithi163 bash[33912]: cluster 2023-10-09T09:10:18.986249+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2023-10-09T09:10:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:19 smithi163 bash[33912]: cluster 2023-10-09T09:10:18.990833+0000 osd.4 (osd.4) 50 : cluster [WRN] failed to encode map e358 with expected crc 2023-10-09T09:10:19.959 INFO:tasks.rados.rados.0.smithi163.stdout:1195: done (2 left) 2023-10-09T09:10:19.959 INFO:tasks.rados.rados.0.smithi163.stdout:1196: done (1 left) 2023-10-09T09:10:19.959 INFO:tasks.rados.rados.0.smithi163.stdout:1197: done (0 left) 2023-10-09T09:10:19.959 INFO:tasks.rados.rados.0.smithi163.stdout:1198: write oid 29 current snap is 151 2023-10-09T09:10:19.959 INFO:tasks.rados.rados.0.smithi163.stdout:1198: seq_num 467 ranges {648869=638816,1828032=787828,3141714=532030} 2023-10-09T09:10:19.972 INFO:tasks.rados.rados.0.smithi163.stdout:1198: writing smithi16367185-29 from 648869 to 1287685 tid 1 2023-10-09T09:10:19.982 INFO:tasks.rados.rados.0.smithi163.stdout:1198: writing smithi16367185-29 from 1828032 to 2615860 tid 2 2023-10-09T09:10:19.989 INFO:tasks.rados.rados.0.smithi163.stdout:1198: writing smithi16367185-29 from 3141714 to 3673744 tid 3 2023-10-09T09:10:19.989 INFO:tasks.rados.rados.0.smithi163.stdout:1199: write oid 41 current snap is 151 2023-10-09T09:10:19.989 INFO:tasks.rados.rados.0.smithi163.stdout:1198: finishing write tid 1 to smithi16367185-29 2023-10-09T09:10:19.989 INFO:tasks.rados.rados.0.smithi163.stdout:1198: finishing write tid 2 to smithi16367185-29 2023-10-09T09:10:19.989 INFO:tasks.rados.rados.0.smithi163.stdout:1199: seq_num 468 ranges {665676=714065,1886514=531486,3169137=696539} 2023-10-09T09:10:19.996 INFO:tasks.rados.rados.0.smithi163.stdout:1199: writing smithi16367185-41 from 665676 to 1379741 tid 1 2023-10-09T09:10:20.002 INFO:tasks.rados.rados.0.smithi163.stdout:1199: writing smithi16367185-41 from 1886514 to 2418000 tid 2 2023-10-09T09:10:20.011 INFO:tasks.rados.rados.0.smithi163.stdout:1199: writing smithi16367185-41 from 3169137 to 3865676 tid 3 2023-10-09T09:10:20.011 INFO:tasks.rados.rados.0.smithi163.stdout:1200: read oid 24 snap -1 2023-10-09T09:10:20.011 INFO:tasks.rados.rados.0.smithi163.stdout:1200: expect deleted 2023-10-09T09:10:20.011 INFO:tasks.rados.rados.0.smithi163.stdout:1201: snap_remove snap 136 2023-10-09T09:10:20.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:19 smithi163 bash[33912]: debug 2023-10-09T09:10:19.945+0000 7f84df46a700 -1 mon.c@2(peon).osd e359 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:20.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:19 smithi181 bash[27712]: debug 2023-10-09T09:10:19.943+0000 7fde83035700 -1 mon.b@1(peon).osd e359 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:20 smithi181 bash[27712]: cluster 2023-10-09T09:10:18.991863+0000 osd.0 (osd.0) 36 : cluster [WRN] failed to encode map e358 with expected crc 2023-10-09T09:10:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:20 smithi181 bash[27712]: cluster 2023-10-09T09:10:18.994782+0000 osd.7 (osd.7) 37 : cluster [WRN] failed to encode map e358 with expected crc 2023-10-09T09:10:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:20 smithi181 bash[27712]: cluster 2023-10-09T09:10:19.226383+0000 mgr.y (mgr.34107) 43 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 167 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.6 MiB/s wr, 18 op/s 2023-10-09T09:10:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:20 smithi181 bash[27712]: cluster 2023-10-09T09:10:19.951197+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2023-10-09T09:10:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:20 smithi181 bash[27712]: cluster 2023-10-09T09:10:19.955185+0000 osd.4 (osd.4) 51 : cluster [WRN] failed to encode map e359 with expected crc 2023-10-09T09:10:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:20 smithi163 bash[33912]: cluster 2023-10-09T09:10:18.991863+0000 osd.0 (osd.0) 36 : cluster [WRN] failed to encode map e358 with expected crc 2023-10-09T09:10:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:20 smithi163 bash[33912]: cluster 2023-10-09T09:10:18.994782+0000 osd.7 (osd.7) 37 : cluster [WRN] failed to encode map e358 with expected crc 2023-10-09T09:10:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:20 smithi163 bash[33912]: cluster 2023-10-09T09:10:19.226383+0000 mgr.y (mgr.34107) 43 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 167 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.6 MiB/s wr, 18 op/s 2023-10-09T09:10:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:20 smithi163 bash[33912]: cluster 2023-10-09T09:10:19.951197+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2023-10-09T09:10:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:20 smithi163 bash[33912]: cluster 2023-10-09T09:10:19.955185+0000 osd.4 (osd.4) 51 : cluster [WRN] failed to encode map e359 with expected crc 2023-10-09T09:10:20.998 INFO:tasks.rados.rados.0.smithi163.stdout:1201: done (3 left) 2023-10-09T09:10:20.998 INFO:tasks.rados.rados.0.smithi163.stdout:1202: snap_create 2023-10-09T09:10:20.998 INFO:tasks.rados.rados.0.smithi163.stdout:1198: finishing write tid 3 to smithi16367185-29 2023-10-09T09:10:20.998 INFO:tasks.rados.rados.0.smithi163.stdout:1198: finishing write tid 4 to smithi16367185-29 2023-10-09T09:10:20.998 INFO:tasks.rados.rados.0.smithi163.stdout:1198: finishing write tid 5 to smithi16367185-29 2023-10-09T09:10:20.998 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 468 (ObjNum 467 snap 151 seq_num 467) dirty exists 2023-10-09T09:10:20.998 INFO:tasks.rados.rados.0.smithi163.stdout:1198: left oid 29 (ObjNum 467 snap 151 seq_num 467) 2023-10-09T09:10:20.998 INFO:tasks.rados.rados.0.smithi163.stdout:1199: finishing write tid 1 to smithi16367185-41 2023-10-09T09:10:20.999 INFO:tasks.rados.rados.0.smithi163.stdout:1199: finishing write tid 2 to smithi16367185-41 2023-10-09T09:10:20.999 INFO:tasks.rados.rados.0.smithi163.stdout:1199: finishing write tid 3 to smithi16367185-41 2023-10-09T09:10:20.999 INFO:tasks.rados.rados.0.smithi163.stdout:1199: finishing write tid 4 to smithi16367185-41 2023-10-09T09:10:20.999 INFO:tasks.rados.rados.0.smithi163.stdout:1199: finishing write tid 5 to smithi16367185-41 2023-10-09T09:10:20.999 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 77 (ObjNum 468 snap 151 seq_num 468) dirty exists 2023-10-09T09:10:20.999 INFO:tasks.rados.rados.0.smithi163.stdout:1199: left oid 41 (ObjNum 468 snap 151 seq_num 468) 2023-10-09T09:10:21.276 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:20 smithi181 bash[27712]: debug 2023-10-09T09:10:20.979+0000 7fde83035700 -1 mon.b@1(peon).osd e360 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:21.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:20 smithi163 bash[33912]: debug 2023-10-09T09:10:20.985+0000 7f84df46a700 -1 mon.c@2(peon).osd e360 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:21 smithi181 bash[27712]: cluster 2023-10-09T09:10:19.954733+0000 osd.0 (osd.0) 37 : cluster [WRN] failed to encode map e359 with expected crc 2023-10-09T09:10:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:21 smithi181 bash[27712]: cluster 2023-10-09T09:10:19.974587+0000 osd.2 (osd.2) 44 : cluster [WRN] failed to encode map e359 with expected crc 2023-10-09T09:10:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:21 smithi181 bash[27712]: cluster 2023-10-09T09:10:20.013064+0000 osd.3 (osd.3) 29 : cluster [WRN] failed to encode map e359 with expected crc 2023-10-09T09:10:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:21 smithi181 bash[27712]: cluster 2023-10-09T09:10:20.991253+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2023-10-09T09:10:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:21 smithi181 bash[27712]: cluster 2023-10-09T09:10:20.994247+0000 osd.4 (osd.4) 52 : cluster [WRN] failed to encode map e360 with expected crc 2023-10-09T09:10:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:21 smithi163 bash[33912]: cluster 2023-10-09T09:10:19.954733+0000 osd.0 (osd.0) 37 : cluster [WRN] failed to encode map e359 with expected crc 2023-10-09T09:10:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:21 smithi163 bash[33912]: cluster 2023-10-09T09:10:19.974587+0000 osd.2 (osd.2) 44 : cluster [WRN] failed to encode map e359 with expected crc 2023-10-09T09:10:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:21 smithi163 bash[33912]: cluster 2023-10-09T09:10:20.013064+0000 osd.3 (osd.3) 29 : cluster [WRN] failed to encode map e359 with expected crc 2023-10-09T09:10:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:21 smithi163 bash[33912]: cluster 2023-10-09T09:10:20.991253+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2023-10-09T09:10:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:21 smithi163 bash[33912]: cluster 2023-10-09T09:10:20.994247+0000 osd.4 (osd.4) 52 : cluster [WRN] failed to encode map e360 with expected crc 2023-10-09T09:10:22.018 INFO:tasks.rados.rados.0.smithi163.stdout:1198: done (3 left) 2023-10-09T09:10:22.018 INFO:tasks.rados.rados.0.smithi163.stdout:1199: done (2 left) 2023-10-09T09:10:22.018 INFO:tasks.rados.rados.0.smithi163.stdout:1200: done (1 left) 2023-10-09T09:10:22.018 INFO:tasks.rados.rados.0.smithi163.stdout:1202: done (0 left) 2023-10-09T09:10:22.018 INFO:tasks.rados.rados.0.smithi163.stdout:1203: read oid 37 snap -1 2023-10-09T09:10:22.018 INFO:tasks.rados.rados.0.smithi163.stdout:1203: expect deleted 2023-10-09T09:10:22.018 INFO:tasks.rados.rados.0.smithi163.stdout:1204: snap_remove snap 135 2023-10-09T09:10:22.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:22 smithi163 bash[33912]: debug 2023-10-09T09:10:22.009+0000 7f84df46a700 -1 mon.c@2(peon).osd e361 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:22.279 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:22 smithi181 bash[27712]: debug 2023-10-09T09:10:22.003+0000 7fde83035700 -1 mon.b@1(peon).osd e361 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:22 smithi181 bash[27712]: cluster 2023-10-09T09:10:20.996578+0000 osd.2 (osd.2) 45 : cluster [WRN] failed to encode map e360 with expected crc 2023-10-09T09:10:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:22 smithi181 bash[27712]: cluster 2023-10-09T09:10:21.227484+0000 mgr.y (mgr.34107) 44 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 173 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2023-10-09T09:10:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:22 smithi181 bash[27712]: cluster 2023-10-09T09:10:22.013545+0000 osd.4 (osd.4) 53 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:22.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:22 smithi181 bash[27712]: cluster 2023-10-09T09:10:22.017453+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2023-10-09T09:10:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:22 smithi163 bash[33912]: cluster 2023-10-09T09:10:20.996578+0000 osd.2 (osd.2) 45 : cluster [WRN] failed to encode map e360 with expected crc 2023-10-09T09:10:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:22 smithi163 bash[33912]: cluster 2023-10-09T09:10:21.227484+0000 mgr.y (mgr.34107) 44 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 173 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2023-10-09T09:10:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:22 smithi163 bash[33912]: cluster 2023-10-09T09:10:22.013545+0000 osd.4 (osd.4) 53 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:22 smithi163 bash[33912]: cluster 2023-10-09T09:10:22.017453+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2023-10-09T09:10:23.025 INFO:tasks.rados.rados.0.smithi163.stdout:1204: done (1 left) 2023-10-09T09:10:23.026 INFO:tasks.rados.rados.0.smithi163.stdout:1205: rollback oid 41 current snap is 152 2023-10-09T09:10:23.026 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 41 to 144 2023-10-09T09:10:23.026 INFO:tasks.rados.rados.0.smithi163.stdout:1203: done (1 left) 2023-10-09T09:10:23.026 INFO:tasks.rados.rados.0.smithi163.stdout:1206: rollback oid 10 current snap is 152 2023-10-09T09:10:23.026 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 145 2023-10-09T09:10:23.026 INFO:tasks.rados.rados.0.smithi163.stdout:1207: snap_remove snap 137 2023-10-09T09:10:23.280 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:23 smithi181 bash[27712]: debug 2023-10-09T09:10:23.007+0000 7fde83035700 -1 mon.b@1(peon).osd e362 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:23.281 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:23 smithi163 bash[33912]: debug 2023-10-09T09:10:23.009+0000 7f84df46a700 -1 mon.c@2(peon).osd e362 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:23 smithi181 bash[27712]: cluster 2023-10-09T09:10:22.017711+0000 osd.1 (osd.1) 22 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:23 smithi181 bash[27712]: cluster 2023-10-09T09:10:22.018118+0000 osd.7 (osd.7) 38 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:23 smithi181 bash[27712]: cluster 2023-10-09T09:10:22.428845+0000 osd.0 (osd.0) 38 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:23 smithi181 bash[27712]: cluster 2023-10-09T09:10:23.017603+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2023-10-09T09:10:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:23 smithi181 bash[27712]: cluster 2023-10-09T09:10:23.021359+0000 osd.4 (osd.4) 54 : cluster [WRN] failed to encode map e362 with expected crc 2023-10-09T09:10:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:10:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:10:23] "GET /metrics HTTP/1.1" 200 36080 "" "Prometheus/2.33.4" 2023-10-09T09:10:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:23 smithi163 bash[33912]: cluster 2023-10-09T09:10:22.017711+0000 osd.1 (osd.1) 22 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:23 smithi163 bash[33912]: cluster 2023-10-09T09:10:22.018118+0000 osd.7 (osd.7) 38 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:23 smithi163 bash[33912]: cluster 2023-10-09T09:10:22.428845+0000 osd.0 (osd.0) 38 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:23 smithi163 bash[33912]: cluster 2023-10-09T09:10:23.017603+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2023-10-09T09:10:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:23 smithi163 bash[33912]: cluster 2023-10-09T09:10:23.021359+0000 osd.4 (osd.4) 54 : cluster [WRN] failed to encode map e362 with expected crc 2023-10-09T09:10:24.029 INFO:tasks.rados.rados.0.smithi163.stdout:1207: done (2 left) 2023-10-09T09:10:24.029 INFO:tasks.rados.rados.0.smithi163.stdout:1208: write oid 20 current snap is 152 2023-10-09T09:10:24.029 INFO:tasks.rados.rados.0.smithi163.stdout:1206: finishing rollback tid 0 to smithi16367185-10 2023-10-09T09:10:24.029 INFO:tasks.rados.rados.0.smithi163.stdout:1205: finishing rollback tid 0 to smithi16367185-41 2023-10-09T09:10:24.029 INFO:tasks.rados.rados.0.smithi163.stdout:1206: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:10:24.029 INFO:tasks.rados.rados.0.smithi163.stdout:1205: finishing rollback tid 1 to smithi16367185-41 2023-10-09T09:10:24.029 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 80 (ObjNum 698991504 snap 21875 seq_num 698748640) dirty dne 2023-10-09T09:10:24.030 INFO:tasks.rados.rados.0.smithi163.stdout:1206: finishing rollback tid 2 to smithi16367185-10 2023-10-09T09:10:24.030 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 444 (ObjNum 435 snap 136 seq_num 435) dirty exists 2023-10-09T09:10:24.030 INFO:tasks.rados.rados.0.smithi163.stdout:1208: seq_num 469 ranges {682483=789314,1944996=791497,3196560=554574} 2023-10-09T09:10:24.041 INFO:tasks.rados.rados.0.smithi163.stdout:1208: writing smithi16367185-20 from 682483 to 1471797 tid 1 2023-10-09T09:10:24.049 INFO:tasks.rados.rados.0.smithi163.stdout:1208: writing smithi16367185-20 from 1944996 to 2736493 tid 2 2023-10-09T09:10:24.055 INFO:tasks.rados.rados.0.smithi163.stdout:1208: writing smithi16367185-20 from 3196560 to 3751134 tid 3 2023-10-09T09:10:24.056 INFO:tasks.rados.rados.0.smithi163.stdout:1205: done (2 left) 2023-10-09T09:10:24.056 INFO:tasks.rados.rados.0.smithi163.stdout:1206: done (1 left) 2023-10-09T09:10:24.056 INFO:tasks.rados.rados.0.smithi163.stdout:1209: rollback oid 5 current snap is 152 2023-10-09T09:10:24.056 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 151 2023-10-09T09:10:24.056 INFO:tasks.rados.rados.0.smithi163.stdout:1208: finishing write tid 1 to smithi16367185-20 2023-10-09T09:10:24.056 INFO:tasks.rados.rados.0.smithi163.stdout:1208: finishing write tid 2 to smithi16367185-20 2023-10-09T09:10:24.056 INFO:tasks.rados.rados.0.smithi163.stdout:1210: rollback oid 28 current snap is 152 2023-10-09T09:10:24.056 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 124 2023-10-09T09:10:24.056 INFO:tasks.rados.rados.0.smithi163.stdout:1211: delete oid 2 current snap is 152 2023-10-09T09:10:24.059 INFO:tasks.rados.rados.0.smithi163.stdout:1211: done (3 left) 2023-10-09T09:10:24.059 INFO:tasks.rados.rados.0.smithi163.stdout:1212: write oid 8 current snap is 152 2023-10-09T09:10:24.059 INFO:tasks.rados.rados.0.smithi163.stdout:1212: seq_num 471 ranges {716097=539812,1945607=795166,3535053=381044} 2023-10-09T09:10:24.066 INFO:tasks.rados.rados.0.smithi163.stdout:1212: writing smithi16367185-8 from 716097 to 1255909 tid 1 2023-10-09T09:10:24.075 INFO:tasks.rados.rados.0.smithi163.stdout:1212: writing smithi16367185-8 from 1945607 to 2740773 tid 2 2023-10-09T09:10:24.082 INFO:tasks.rados.rados.0.smithi163.stdout:1212: writing smithi16367185-8 from 3535053 to 3916097 tid 3 2023-10-09T09:10:24.082 INFO:tasks.rados.rados.0.smithi163.stdout:1213: snap_remove snap 121 2023-10-09T09:10:24.284 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:24 smithi163 bash[33912]: debug 2023-10-09T09:10:24.017+0000 7f84df46a700 -1 mon.c@2(peon).osd e363 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:24.284 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:24 smithi181 bash[27712]: debug 2023-10-09T09:10:24.015+0000 7fde83035700 -1 mon.b@1(peon).osd e363 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:24 smithi181 bash[27712]: cluster 2023-10-09T09:10:22.528159+0000 osd.6 (osd.6) 22 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:24 smithi181 bash[27712]: cluster 2023-10-09T09:10:22.993760+0000 osd.5 (osd.5) 24 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:24 smithi181 bash[27712]: cluster 2023-10-09T09:10:23.021761+0000 osd.1 (osd.1) 23 : cluster [WRN] failed to encode map e362 with expected crc 2023-10-09T09:10:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:24 smithi181 bash[27712]: cluster 2023-10-09T09:10:23.027621+0000 osd.2 (osd.2) 46 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:24 smithi181 bash[27712]: cluster 2023-10-09T09:10:23.228199+0000 mgr.y (mgr.34107) 45 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 173 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2023-10-09T09:10:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:24 smithi181 bash[27712]: cluster 2023-10-09T09:10:24.023431+0000 mon.a (mon.0) 137 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2023-10-09T09:10:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:24 smithi163 bash[33912]: cluster 2023-10-09T09:10:22.528159+0000 osd.6 (osd.6) 22 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:24 smithi163 bash[33912]: cluster 2023-10-09T09:10:22.993760+0000 osd.5 (osd.5) 24 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:24 smithi163 bash[33912]: cluster 2023-10-09T09:10:23.021761+0000 osd.1 (osd.1) 23 : cluster [WRN] failed to encode map e362 with expected crc 2023-10-09T09:10:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:24 smithi163 bash[33912]: cluster 2023-10-09T09:10:23.027621+0000 osd.2 (osd.2) 46 : cluster [WRN] failed to encode map e361 with expected crc 2023-10-09T09:10:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:24 smithi163 bash[33912]: cluster 2023-10-09T09:10:23.228199+0000 mgr.y (mgr.34107) 45 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 173 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2023-10-09T09:10:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:24 smithi163 bash[33912]: cluster 2023-10-09T09:10:24.023431+0000 mon.a (mon.0) 137 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2023-10-09T09:10:24.962 INFO:tasks.rados.rados.0.smithi163.stdout:1213: done (4 left) 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:1214: write oid 31 current snap is 152 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:1208: finishing write tid 3 to smithi16367185-20 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:1208: finishing write tid 4 to smithi16367185-20 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:1208: finishing write tid 5 to smithi16367185-20 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 132 (ObjNum 469 snap 152 seq_num 469) dirty exists 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:1208: left oid 20 (ObjNum 469 snap 152 seq_num 469) 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:1209: finishing rollback tid 0 to smithi16367185-5 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:1209: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:1209: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 695 (ObjNum 436 snap 136 seq_num 436) dirty exists 2023-10-09T09:10:24.963 INFO:tasks.rados.rados.0.smithi163.stdout:1210: finishing rollback tid 0 to smithi16367185-28 2023-10-09T09:10:24.964 INFO:tasks.rados.rados.0.smithi163.stdout:1210: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:10:24.964 INFO:tasks.rados.rados.0.smithi163.stdout:1214: seq_num 472 ranges {732904=731414,2120442=538824,3278829=504743} 2023-10-09T09:10:24.971 INFO:tasks.rados.rados.0.smithi163.stdout:1214: writing smithi16367185-31 from 732904 to 1464318 tid 1 2023-10-09T09:10:24.979 INFO:tasks.rados.rados.0.smithi163.stdout:1214: writing smithi16367185-31 from 2120442 to 2659266 tid 2 2023-10-09T09:10:24.987 INFO:tasks.rados.rados.0.smithi163.stdout:1214: writing smithi16367185-31 from 3278829 to 3783572 tid 3 2023-10-09T09:10:24.987 INFO:tasks.rados.rados.0.smithi163.stdout:1208: done (4 left) 2023-10-09T09:10:24.987 INFO:tasks.rados.rados.0.smithi163.stdout:1209: done (3 left) 2023-10-09T09:10:24.987 INFO:tasks.rados.rados.0.smithi163.stdout:1215: rollback oid 36 current snap is 152 2023-10-09T09:10:24.987 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 36 to 145 2023-10-09T09:10:24.987 INFO:tasks.rados.rados.0.smithi163.stdout:1210: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:10:24.987 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 699 (ObjNum 295 snap 91 seq_num 295) dirty exists 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1212: finishing write tid 1 to smithi16367185-8 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1212: finishing write tid 2 to smithi16367185-8 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1212: finishing write tid 3 to smithi16367185-8 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1212: finishing write tid 4 to smithi16367185-8 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1212: finishing write tid 5 to smithi16367185-8 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 589 (ObjNum 471 snap 152 seq_num 471) dirty exists 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1212: left oid 8 (ObjNum 471 snap 152 seq_num 471) 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1214: finishing write tid 1 to smithi16367185-31 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1214: finishing write tid 2 to smithi16367185-31 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1210: done (3 left) 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1212: done (2 left) 2023-10-09T09:10:24.988 INFO:tasks.rados.rados.0.smithi163.stdout:1216: snap_remove snap 150 2023-10-09T09:10:25.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:24 smithi163 bash[33912]: debug 2023-10-09T09:10:24.945+0000 7f84df46a700 -1 mon.c@2(peon).osd e364 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:25.286 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:24 smithi181 bash[27712]: debug 2023-10-09T09:10:24.943+0000 7fde83035700 -1 mon.b@1(peon).osd e364 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:25 smithi181 bash[27712]: cluster 2023-10-09T09:10:24.025722+0000 osd.7 (osd.7) 39 : cluster [WRN] failed to encode map e363 with expected crc 2023-10-09T09:10:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:25 smithi181 bash[27712]: cluster 2023-10-09T09:10:24.027117+0000 osd.2 (osd.2) 47 : cluster [WRN] failed to encode map e363 with expected crc 2023-10-09T09:10:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:25 smithi181 bash[27712]: cluster 2023-10-09T09:10:24.956656+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2023-10-09T09:10:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:25 smithi163 bash[33912]: cluster 2023-10-09T09:10:24.025722+0000 osd.7 (osd.7) 39 : cluster [WRN] failed to encode map e363 with expected crc 2023-10-09T09:10:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:25 smithi163 bash[33912]: cluster 2023-10-09T09:10:24.027117+0000 osd.2 (osd.2) 47 : cluster [WRN] failed to encode map e363 with expected crc 2023-10-09T09:10:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:25 smithi163 bash[33912]: cluster 2023-10-09T09:10:24.956656+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2023-10-09T09:10:25.980 INFO:tasks.rados.rados.0.smithi163.stdout:1216: done (2 left) 2023-10-09T09:10:25.980 INFO:tasks.rados.rados.0.smithi163.stdout:1217: delete oid 45 current snap is 152 2023-10-09T09:10:25.980 INFO:tasks.rados.rados.0.smithi163.stdout:1214: finishing write tid 3 to smithi16367185-31 2023-10-09T09:10:25.980 INFO:tasks.rados.rados.0.smithi163.stdout:1214: finishing write tid 4 to smithi16367185-31 2023-10-09T09:10:25.980 INFO:tasks.rados.rados.0.smithi163.stdout:1214: finishing write tid 5 to smithi16367185-31 2023-10-09T09:10:25.980 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 376 (ObjNum 472 snap 152 seq_num 472) dirty exists 2023-10-09T09:10:25.980 INFO:tasks.rados.rados.0.smithi163.stdout:1214: left oid 31 (ObjNum 472 snap 152 seq_num 472) 2023-10-09T09:10:25.981 INFO:tasks.rados.rados.0.smithi163.stdout:1215: finishing rollback tid 1 to smithi16367185-36 2023-10-09T09:10:25.981 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 338 (ObjNum 699059776 snap 21875 seq_num 1207961808) dirty dne 2023-10-09T09:10:25.989 INFO:tasks.rados.rados.0.smithi163.stdout:1214: done (2 left) 2023-10-09T09:10:25.989 INFO:tasks.rados.rados.0.smithi163.stdout:1215: done (1 left) 2023-10-09T09:10:25.990 INFO:tasks.rados.rados.0.smithi163.stdout:1217: done (0 left) 2023-10-09T09:10:25.990 INFO:tasks.rados.rados.0.smithi163.stdout:1218: write oid 50 current snap is 152 2023-10-09T09:10:25.990 INFO:tasks.rados.rados.0.smithi163.stdout:1218: seq_num 474 ranges {766518=481912,1837406=542493,2933675=527287,3876584=89934} 2023-10-09T09:10:25.998 INFO:tasks.rados.rados.0.smithi163.stdout:1218: writing smithi16367185-50 from 766518 to 1248430 tid 1 2023-10-09T09:10:26.005 INFO:tasks.rados.rados.0.smithi163.stdout:1218: writing smithi16367185-50 from 1837406 to 2379899 tid 2 2023-10-09T09:10:26.012 INFO:tasks.rados.rados.0.smithi163.stdout:1218: writing smithi16367185-50 from 2933675 to 3460962 tid 3 2023-10-09T09:10:26.015 INFO:tasks.rados.rados.0.smithi163.stdout:1218: writing smithi16367185-50 from 3876584 to 3966518 tid 4 2023-10-09T09:10:26.015 INFO:tasks.rados.rados.0.smithi163.stdout:1219: read oid 31 snap 133 2023-10-09T09:10:26.015 INFO:tasks.rados.rados.0.smithi163.stdout:1219: expect (ObjNum 411 snap 125 seq_num 411) 2023-10-09T09:10:26.015 INFO:tasks.rados.rados.0.smithi163.stdout:1220: write oid 34 current snap is 152 2023-10-09T09:10:26.015 INFO:tasks.rados.rados.0.smithi163.stdout:1218: finishing write tid 1 to smithi16367185-50 2023-10-09T09:10:26.015 INFO:tasks.rados.rados.0.smithi163.stdout:1218: finishing write tid 2 to smithi16367185-50 2023-10-09T09:10:26.015 INFO:tasks.rados.rados.0.smithi163.stdout:1220: seq_num 475 ranges {783325=557161,1779535=686151,3244745=454912} 2023-10-09T09:10:26.022 INFO:tasks.rados.rados.0.smithi163.stdout:1220: writing smithi16367185-34 from 783325 to 1340486 tid 1 2023-10-09T09:10:26.031 INFO:tasks.rados.rados.0.smithi163.stdout:1220: writing smithi16367185-34 from 1779535 to 2465686 tid 2 2023-10-09T09:10:26.038 INFO:tasks.rados.rados.0.smithi163.stdout:1220: writing smithi16367185-34 from 3244745 to 3699657 tid 3 2023-10-09T09:10:26.038 INFO:tasks.rados.rados.0.smithi163.stdout:1221: write oid 10 current snap is 152 2023-10-09T09:10:26.038 INFO:tasks.rados.rados.0.smithi163.stdout:1218: finishing write tid 3 to smithi16367185-50 2023-10-09T09:10:26.038 INFO:tasks.rados.rados.0.smithi163.stdout:1218: finishing write tid 4 to smithi16367185-50 2023-10-09T09:10:26.038 INFO:tasks.rados.rados.0.smithi163.stdout:1218: finishing write tid 5 to smithi16367185-50 2023-10-09T09:10:26.038 INFO:tasks.rados.rados.0.smithi163.stdout:1218: finishing write tid 6 to smithi16367185-50 2023-10-09T09:10:26.038 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 472 (ObjNum 474 snap 152 seq_num 474) dirty exists 2023-10-09T09:10:26.038 INFO:tasks.rados.rados.0.smithi163.stdout:1218: left oid 50 (ObjNum 474 snap 152 seq_num 474) 2023-10-09T09:10:26.038 INFO:tasks.rados.rados.0.smithi163.stdout:1219: expect (ObjNum 411 snap 125 seq_num 411) 2023-10-09T09:10:26.193 INFO:tasks.rados.rados.0.smithi163.stdout:1220: finishing write tid 1 to smithi16367185-34 2023-10-09T09:10:26.193 INFO:tasks.rados.rados.0.smithi163.stdout:1220: finishing write tid 2 to smithi16367185-34 2023-10-09T09:10:26.193 INFO:tasks.rados.rados.0.smithi163.stdout:1220: finishing write tid 3 to smithi16367185-34 2023-10-09T09:10:26.194 INFO:tasks.rados.rados.0.smithi163.stdout:1220: finishing write tid 4 to smithi16367185-34 2023-10-09T09:10:26.194 INFO:tasks.rados.rados.0.smithi163.stdout:1220: finishing write tid 5 to smithi16367185-34 2023-10-09T09:10:26.194 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 477 (ObjNum 475 snap 152 seq_num 475) dirty exists 2023-10-09T09:10:26.194 INFO:tasks.rados.rados.0.smithi163.stdout:1220: left oid 34 (ObjNum 475 snap 152 seq_num 475) 2023-10-09T09:10:26.194 INFO:tasks.rados.rados.0.smithi163.stdout:1221: seq_num 476 ranges {400132=632410,1438017=546162} 2023-10-09T09:10:26.199 INFO:tasks.rados.rados.0.smithi163.stdout:1221: writing smithi16367185-10 from 400132 to 1032542 tid 1 2023-10-09T09:10:26.204 INFO:tasks.rados.rados.0.smithi163.stdout:1221: writing smithi16367185-10 from 1438017 to 1984179 tid 2 2023-10-09T09:10:26.205 INFO:tasks.rados.rados.0.smithi163.stdout:1218: done (3 left) 2023-10-09T09:10:26.205 INFO:tasks.rados.rados.0.smithi163.stdout:1219: done (2 left) 2023-10-09T09:10:26.205 INFO:tasks.rados.rados.0.smithi163.stdout:1220: done (1 left) 2023-10-09T09:10:26.205 INFO:tasks.rados.rados.0.smithi163.stdout:1222: write oid 33 current snap is 152 2023-10-09T09:10:26.205 INFO:tasks.rados.rados.0.smithi163.stdout:1222: seq_num 477 ranges {416939=707659,1896499=120440} 2023-10-09T09:10:26.211 INFO:tasks.rados.rados.0.smithi163.stdout:1222: writing smithi16367185-33 from 416939 to 1124598 tid 1 2023-10-09T09:10:26.216 INFO:tasks.rados.rados.0.smithi163.stdout:1222: writing smithi16367185-33 from 1896499 to 2016939 tid 2 2023-10-09T09:10:26.216 INFO:tasks.rados.rados.0.smithi163.stdout:1223: snap_remove snap 133 2023-10-09T09:10:26.290 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:25 smithi181 bash[27712]: debug 2023-10-09T09:10:25.967+0000 7fde83035700 -1 mon.b@1(peon).osd e365 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:26.291 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:25 smithi163 bash[33912]: debug 2023-10-09T09:10:25.969+0000 7f84df46a700 -1 mon.c@2(peon).osd e365 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:26 smithi181 bash[27712]: cluster 2023-10-09T09:10:24.958894+0000 osd.6 (osd.6) 23 : cluster [WRN] failed to encode map e364 with expected crc 2023-10-09T09:10:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:26 smithi181 bash[27712]: cluster 2023-10-09T09:10:24.959861+0000 osd.3 (osd.3) 30 : cluster [WRN] failed to encode map e364 with expected crc 2023-10-09T09:10:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:26 smithi181 bash[27712]: cluster 2023-10-09T09:10:25.228974+0000 mgr.y (mgr.34107) 46 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 173 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:10:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:26 smithi181 bash[27712]: audit 2023-10-09T09:10:25.331432+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:26 smithi181 bash[27712]: cluster 2023-10-09T09:10:25.974886+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2023-10-09T09:10:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:26 smithi181 bash[27712]: cluster 2023-10-09T09:10:25.977012+0000 osd.4 (osd.4) 55 : cluster [WRN] failed to encode map e365 with expected crc 2023-10-09T09:10:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:26 smithi163 bash[33912]: cluster 2023-10-09T09:10:24.958894+0000 osd.6 (osd.6) 23 : cluster [WRN] failed to encode map e364 with expected crc 2023-10-09T09:10:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:26 smithi163 bash[33912]: cluster 2023-10-09T09:10:24.959861+0000 osd.3 (osd.3) 30 : cluster [WRN] failed to encode map e364 with expected crc 2023-10-09T09:10:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:26 smithi163 bash[33912]: cluster 2023-10-09T09:10:25.228974+0000 mgr.y (mgr.34107) 46 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 173 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:10:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:26 smithi163 bash[33912]: audit 2023-10-09T09:10:25.331432+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:26 smithi163 bash[33912]: cluster 2023-10-09T09:10:25.974886+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2023-10-09T09:10:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:26 smithi163 bash[33912]: cluster 2023-10-09T09:10:25.977012+0000 osd.4 (osd.4) 55 : cluster [WRN] failed to encode map e365 with expected crc 2023-10-09T09:10:26.987 INFO:tasks.rados.rados.0.smithi163.stdout:1223: done (2 left) 2023-10-09T09:10:26.987 INFO:tasks.rados.rados.0.smithi163.stdout:1224: read oid 32 snap -1 2023-10-09T09:10:26.987 INFO:tasks.rados.rados.0.smithi163.stdout:1224: expect (ObjNum 422 snap 129 seq_num 422) 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:1225: delete oid 43 current snap is 152 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:1221: finishing write tid 1 to smithi16367185-10 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:1221: finishing write tid 2 to smithi16367185-10 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:1221: finishing write tid 3 to smithi16367185-10 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:1221: finishing write tid 4 to smithi16367185-10 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 455 (ObjNum 476 snap 152 seq_num 476) dirty exists 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:1221: left oid 10 (ObjNum 476 snap 152 seq_num 476) 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:1222: finishing write tid 1 to smithi16367185-33 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:1222: finishing write tid 2 to smithi16367185-33 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:1222: finishing write tid 3 to smithi16367185-33 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:1222: finishing write tid 4 to smithi16367185-33 2023-10-09T09:10:26.988 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 490 (ObjNum 477 snap 152 seq_num 477) dirty exists 2023-10-09T09:10:26.989 INFO:tasks.rados.rados.0.smithi163.stdout:1222: left oid 33 (ObjNum 477 snap 152 seq_num 477) 2023-10-09T09:10:26.997 INFO:tasks.rados.rados.0.smithi163.stdout:1221: done (3 left) 2023-10-09T09:10:26.997 INFO:tasks.rados.rados.0.smithi163.stdout:1222: done (2 left) 2023-10-09T09:10:26.997 INFO:tasks.rados.rados.0.smithi163.stdout:1225: done (1 left) 2023-10-09T09:10:26.997 INFO:tasks.rados.rados.0.smithi163.stdout:1226: write oid 38 current snap is 152 2023-10-09T09:10:26.997 INFO:tasks.rados.rados.0.smithi163.stdout:1226: seq_num 479 ranges {450553=574510,1613463=437090} 2023-10-09T09:10:27.003 INFO:tasks.rados.rados.0.smithi163.stdout:1226: writing smithi16367185-38 from 450553 to 1025063 tid 1 2023-10-09T09:10:27.011 INFO:tasks.rados.rados.0.smithi163.stdout:1226: writing smithi16367185-38 from 1613463 to 2050553 tid 2 2023-10-09T09:10:27.012 INFO:tasks.rados.rados.0.smithi163.stdout:1227: snap_remove snap 147 2023-10-09T09:10:27.293 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:26 smithi181 bash[27712]: debug 2023-10-09T09:10:26.967+0000 7fde83035700 -1 mon.b@1(peon).osd e366 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:27.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:26 smithi163 bash[33912]: debug 2023-10-09T09:10:26.973+0000 7f84df46a700 -1 mon.c@2(peon).osd e366 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:27 smithi181 bash[27712]: cluster 2023-10-09T09:10:25.977679+0000 osd.0 (osd.0) 39 : cluster [WRN] failed to encode map e365 with expected crc 2023-10-09T09:10:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:27 smithi181 bash[27712]: cluster 2023-10-09T09:10:25.979847+0000 osd.5 (osd.5) 25 : cluster [WRN] failed to encode map e365 with expected crc 2023-10-09T09:10:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:27 smithi181 bash[27712]: cluster 2023-10-09T09:10:26.980534+0000 mon.a (mon.0) 141 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2023-10-09T09:10:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:27 smithi181 bash[27712]: cluster 2023-10-09T09:10:26.983537+0000 osd.4 (osd.4) 56 : cluster [WRN] failed to encode map e366 with expected crc 2023-10-09T09:10:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:27 smithi163 bash[33912]: cluster 2023-10-09T09:10:25.977679+0000 osd.0 (osd.0) 39 : cluster [WRN] failed to encode map e365 with expected crc 2023-10-09T09:10:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:27 smithi163 bash[33912]: cluster 2023-10-09T09:10:25.979847+0000 osd.5 (osd.5) 25 : cluster [WRN] failed to encode map e365 with expected crc 2023-10-09T09:10:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:27 smithi163 bash[33912]: cluster 2023-10-09T09:10:26.980534+0000 mon.a (mon.0) 141 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2023-10-09T09:10:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:27 smithi163 bash[33912]: cluster 2023-10-09T09:10:26.983537+0000 osd.4 (osd.4) 56 : cluster [WRN] failed to encode map e366 with expected crc 2023-10-09T09:10:27.989 INFO:tasks.rados.rados.0.smithi163.stdout:1227: done (2 left) 2023-10-09T09:10:27.990 INFO:tasks.rados.rados.0.smithi163.stdout:1228: rollback oid 17 current snap is 152 2023-10-09T09:10:27.990 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 124 2023-10-09T09:10:27.990 INFO:tasks.rados.rados.0.smithi163.stdout:1226: finishing write tid 1 to smithi16367185-38 2023-10-09T09:10:27.990 INFO:tasks.rados.rados.0.smithi163.stdout:1224: expect (ObjNum 422 snap 129 seq_num 422) 2023-10-09T09:10:28.108 INFO:tasks.rados.rados.0.smithi163.stdout:1226: finishing write tid 2 to smithi16367185-38 2023-10-09T09:10:28.108 INFO:tasks.rados.rados.0.smithi163.stdout:1226: finishing write tid 3 to smithi16367185-38 2023-10-09T09:10:28.108 INFO:tasks.rados.rados.0.smithi163.stdout:1226: finishing write tid 4 to smithi16367185-38 2023-10-09T09:10:28.108 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 495 (ObjNum 479 snap 152 seq_num 479) dirty exists 2023-10-09T09:10:28.108 INFO:tasks.rados.rados.0.smithi163.stdout:1226: left oid 38 (ObjNum 479 snap 152 seq_num 479) 2023-10-09T09:10:28.108 INFO:tasks.rados.rados.0.smithi163.stdout:1228: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:10:28.108 INFO:tasks.rados.rados.0.smithi163.stdout:1228: finishing rollback tid 2 to smithi16367185-17 2023-10-09T09:10:28.108 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 599 (ObjNum 391 snap 119 seq_num 391) dirty exists 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:1224: done (2 left) 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:1226: done (1 left) 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:1228: done (0 left) 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:1229: read oid 39 snap -1 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:1229: expect deleted 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:1230: rollback oid 19 current snap is 152 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 19 to 151 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:1231: read oid 38 snap -1 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:1231: expect (ObjNum 479 snap 152 seq_num 479) 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:1232: rollback oid 32 current snap is 152 2023-10-09T09:10:28.109 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 149 2023-10-09T09:10:28.110 INFO:tasks.rados.rados.0.smithi163.stdout:1233: write oid 41 current snap is 152 2023-10-09T09:10:28.110 INFO:tasks.rados.rados.0.smithi163.stdout:1233: seq_num 480 ranges {467360=649759,1671945=395415} 2023-10-09T09:10:28.119 INFO:tasks.rados.rados.0.smithi163.stdout:1233: writing smithi16367185-41 from 467360 to 1117119 tid 1 2023-10-09T09:10:28.125 INFO:tasks.rados.rados.0.smithi163.stdout:1233: writing smithi16367185-41 from 1671945 to 2067360 tid 2 2023-10-09T09:10:28.125 INFO:tasks.rados.rados.0.smithi163.stdout:1234: read oid 29 snap -1 2023-10-09T09:10:28.125 INFO:tasks.rados.rados.0.smithi163.stdout:1234: expect (ObjNum 467 snap 151 seq_num 467) 2023-10-09T09:10:28.125 INFO:tasks.rados.rados.0.smithi163.stdout:1235: write oid 15 current snap is 152 2023-10-09T09:10:28.125 INFO:tasks.rados.rados.0.smithi163.stdout:1235: seq_num 481 ranges {484167=725008,1730427=353740} 2023-10-09T09:10:28.132 INFO:tasks.rados.rados.0.smithi163.stdout:1235: writing smithi16367185-15 from 484167 to 1209175 tid 1 2023-10-09T09:10:28.137 INFO:tasks.rados.rados.0.smithi163.stdout:1235: writing smithi16367185-15 from 1730427 to 2084167 tid 2 2023-10-09T09:10:28.137 INFO:tasks.rados.rados.0.smithi163.stdout:1236: write oid 43 current snap is 152 2023-10-09T09:10:28.137 INFO:tasks.rados.rados.0.smithi163.stdout:1236: seq_num 482 ranges {500974=400257,1388909=557169} 2023-10-09T09:10:28.142 INFO:tasks.rados.rados.0.smithi163.stdout:1236: writing smithi16367185-43 from 500974 to 901231 tid 1 2023-10-09T09:10:28.149 INFO:tasks.rados.rados.0.smithi163.stdout:1236: writing smithi16367185-43 from 1388909 to 1946078 tid 2 2023-10-09T09:10:28.149 INFO:tasks.rados.rados.0.smithi163.stdout:1237: write oid 4 current snap is 152 2023-10-09T09:10:28.149 INFO:tasks.rados.rados.0.smithi163.stdout:1237: seq_num 483 ranges {517781=475506,1731038=386743} 2023-10-09T09:10:28.155 INFO:tasks.rados.rados.0.smithi163.stdout:1237: writing smithi16367185-4 from 517781 to 993287 tid 1 2023-10-09T09:10:28.162 INFO:tasks.rados.rados.0.smithi163.stdout:1237: writing smithi16367185-4 from 1731038 to 2117781 tid 2 2023-10-09T09:10:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:1238: snap_create 2023-10-09T09:10:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:1230: finishing rollback tid 0 to smithi16367185-19 2023-10-09T09:10:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:1230: finishing rollback tid 1 to smithi16367185-19 2023-10-09T09:10:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:1232: finishing rollback tid 0 to smithi16367185-32 2023-10-09T09:10:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:1232: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:10:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:1230: finishing rollback tid 2 to smithi16367185-19 2023-10-09T09:10:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 418 (ObjNum 390 snap 119 seq_num 390) dirty exists 2023-10-09T09:10:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:1232: finishing rollback tid 2 to smithi16367185-32 2023-10-09T09:10:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 978 (ObjNum 422 snap 129 seq_num 422) dirty exists 2023-10-09T09:10:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:1231: expect (ObjNum 479 snap 152 seq_num 479) 2023-10-09T09:10:28.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:27 smithi163 bash[33912]: debug 2023-10-09T09:10:27.981+0000 7f84df46a700 -1 mon.c@2(peon).osd e367 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:28.296 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:27 smithi181 bash[27712]: debug 2023-10-09T09:10:27.975+0000 7fde83035700 -1 mon.b@1(peon).osd e367 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:28.300 INFO:tasks.rados.rados.0.smithi163.stdout:1233: finishing write tid 1 to smithi16367185-41 2023-10-09T09:10:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:1233: finishing write tid 2 to smithi16367185-41 2023-10-09T09:10:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:1233: finishing write tid 3 to smithi16367185-41 2023-10-09T09:10:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:1233: finishing write tid 4 to smithi16367185-41 2023-10-09T09:10:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 83 (ObjNum 480 snap 152 seq_num 480) dirty exists 2023-10-09T09:10:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:1233: left oid 41 (ObjNum 480 snap 152 seq_num 480) 2023-10-09T09:10:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:1234: expect (ObjNum 467 snap 151 seq_num 467) 2023-10-09T09:10:28.525 INFO:tasks.rados.rados.0.smithi163.stdout:1235: finishing write tid 1 to smithi16367185-15 2023-10-09T09:10:28.525 INFO:tasks.rados.rados.0.smithi163.stdout:1235: finishing write tid 2 to smithi16367185-15 2023-10-09T09:10:28.525 INFO:tasks.rados.rados.0.smithi163.stdout:1235: finishing write tid 3 to smithi16367185-15 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:1235: finishing write tid 4 to smithi16367185-15 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 605 (ObjNum 481 snap 152 seq_num 481) dirty exists 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:1235: left oid 15 (ObjNum 481 snap 152 seq_num 481) 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:1236: finishing write tid 1 to smithi16367185-43 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:1236: finishing write tid 2 to smithi16367185-43 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:1236: finishing write tid 3 to smithi16367185-43 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:1236: finishing write tid 4 to smithi16367185-43 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 502 (ObjNum 482 snap 152 seq_num 482) dirty exists 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:1236: left oid 43 (ObjNum 482 snap 152 seq_num 482) 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:1237: finishing write tid 1 to smithi16367185-4 2023-10-09T09:10:28.526 INFO:tasks.rados.rados.0.smithi163.stdout:1237: finishing write tid 2 to smithi16367185-4 2023-10-09T09:10:28.527 INFO:tasks.rados.rados.0.smithi163.stdout:1237: finishing write tid 3 to smithi16367185-4 2023-10-09T09:10:28.527 INFO:tasks.rados.rados.0.smithi163.stdout:1237: finishing write tid 4 to smithi16367185-4 2023-10-09T09:10:28.527 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 731 (ObjNum 483 snap 152 seq_num 483) dirty exists 2023-10-09T09:10:28.527 INFO:tasks.rados.rados.0.smithi163.stdout:1237: left oid 4 (ObjNum 483 snap 152 seq_num 483) 2023-10-09T09:10:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:28 smithi181 bash[27712]: cluster 2023-10-09T09:10:26.983939+0000 osd.2 (osd.2) 48 : cluster [WRN] failed to encode map e366 with expected crc 2023-10-09T09:10:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:28 smithi181 bash[27712]: cluster 2023-10-09T09:10:26.987192+0000 osd.5 (osd.5) 26 : cluster [WRN] failed to encode map e366 with expected crc 2023-10-09T09:10:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:28 smithi181 bash[27712]: cluster 2023-10-09T09:10:27.229764+0000 mgr.y (mgr.34107) 47 : cluster [DBG] pgmap v96: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.6 MiB/s wr, 14 op/s 2023-10-09T09:10:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:28 smithi181 bash[27712]: cluster 2023-10-09T09:10:27.985661+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2023-10-09T09:10:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:28 smithi181 bash[27712]: cluster 2023-10-09T09:10:27.986418+0000 osd.4 (osd.4) 57 : cluster [WRN] failed to encode map e367 with expected crc 2023-10-09T09:10:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:28 smithi163 bash[33912]: cluster 2023-10-09T09:10:26.983939+0000 osd.2 (osd.2) 48 : cluster [WRN] failed to encode map e366 with expected crc 2023-10-09T09:10:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:28 smithi163 bash[33912]: cluster 2023-10-09T09:10:26.987192+0000 osd.5 (osd.5) 26 : cluster [WRN] failed to encode map e366 with expected crc 2023-10-09T09:10:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:28 smithi163 bash[33912]: cluster 2023-10-09T09:10:27.229764+0000 mgr.y (mgr.34107) 47 : cluster [DBG] pgmap v96: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.6 MiB/s wr, 14 op/s 2023-10-09T09:10:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:28 smithi163 bash[33912]: cluster 2023-10-09T09:10:27.985661+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2023-10-09T09:10:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:28 smithi163 bash[33912]: cluster 2023-10-09T09:10:27.986418+0000 osd.4 (osd.4) 57 : cluster [WRN] failed to encode map e367 with expected crc 2023-10-09T09:10:28.986 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:10:28 smithi181 bash[54907]: ts=2023-10-09T09:10:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:10:28.999 INFO:tasks.rados.rados.0.smithi163.stdout:1229: done (9 left) 2023-10-09T09:10:28.999 INFO:tasks.rados.rados.0.smithi163.stdout:1230: done (8 left) 2023-10-09T09:10:28.999 INFO:tasks.rados.rados.0.smithi163.stdout:1231: done (7 left) 2023-10-09T09:10:28.999 INFO:tasks.rados.rados.0.smithi163.stdout:1232: done (6 left) 2023-10-09T09:10:28.999 INFO:tasks.rados.rados.0.smithi163.stdout:1233: done (5 left) 2023-10-09T09:10:28.999 INFO:tasks.rados.rados.0.smithi163.stdout:1234: done (4 left) 2023-10-09T09:10:28.999 INFO:tasks.rados.rados.0.smithi163.stdout:1235: done (3 left) 2023-10-09T09:10:28.999 INFO:tasks.rados.rados.0.smithi163.stdout:1236: done (2 left) 2023-10-09T09:10:28.999 INFO:tasks.rados.rados.0.smithi163.stdout:1237: done (1 left) 2023-10-09T09:10:29.000 INFO:tasks.rados.rados.0.smithi163.stdout:1238: done (0 left) 2023-10-09T09:10:29.000 INFO:tasks.rados.rados.0.smithi163.stdout:1239: delete oid 32 current snap is 153 2023-10-09T09:10:29.009 INFO:tasks.rados.rados.0.smithi163.stdout:1239: done (0 left) 2023-10-09T09:10:29.009 INFO:tasks.rados.rados.0.smithi163.stdout:1240: delete oid 41 current snap is 153 2023-10-09T09:10:29.016 INFO:tasks.rados.rados.0.smithi163.stdout:1240: done (0 left) 2023-10-09T09:10:29.016 INFO:tasks.rados.rados.0.smithi163.stdout:1241: snap_create 2023-10-09T09:10:29.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:28 smithi163 bash[33912]: debug 2023-10-09T09:10:28.981+0000 7f84df46a700 -1 mon.c@2(peon).osd e368 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:29.298 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:28 smithi181 bash[27712]: debug 2023-10-09T09:10:28.979+0000 7fde83035700 -1 mon.b@1(peon).osd e368 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:29 smithi181 bash[27712]: cluster 2023-10-09T09:10:27.989625+0000 osd.2 (osd.2) 49 : cluster [WRN] failed to encode map e367 with expected crc 2023-10-09T09:10:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:29 smithi181 bash[27712]: cluster 2023-10-09T09:10:27.989917+0000 osd.7 (osd.7) 40 : cluster [WRN] failed to encode map e367 with expected crc 2023-10-09T09:10:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:29 smithi181 bash[27712]: cluster 2023-10-09T09:10:28.989907+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2023-10-09T09:10:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:29 smithi181 bash[27712]: cluster 2023-10-09T09:10:28.994605+0000 osd.4 (osd.4) 58 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:29 smithi163 bash[33912]: cluster 2023-10-09T09:10:27.989625+0000 osd.2 (osd.2) 49 : cluster [WRN] failed to encode map e367 with expected crc 2023-10-09T09:10:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:29 smithi163 bash[33912]: cluster 2023-10-09T09:10:27.989917+0000 osd.7 (osd.7) 40 : cluster [WRN] failed to encode map e367 with expected crc 2023-10-09T09:10:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:29 smithi163 bash[33912]: cluster 2023-10-09T09:10:28.989907+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2023-10-09T09:10:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:29 smithi163 bash[33912]: cluster 2023-10-09T09:10:28.994605+0000 osd.4 (osd.4) 58 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:29.964 INFO:tasks.rados.rados.0.smithi163.stdout:1241: done (0 left) 2023-10-09T09:10:29.964 INFO:tasks.rados.rados.0.smithi163.stdout:1242: delete oid 34 current snap is 154 2023-10-09T09:10:29.977 INFO:tasks.rados.rados.0.smithi163.stdout:1242: done (0 left) 2023-10-09T09:10:29.977 INFO:tasks.rados.rados.0.smithi163.stdout:1243: write oid 46 current snap is 154 2023-10-09T09:10:29.977 INFO:tasks.rados.rados.0.smithi163.stdout:1243: seq_num 487 ranges {585009=492855,1564966=620043} 2023-10-09T09:10:29.987 INFO:tasks.rados.rados.0.smithi163.stdout:1243: writing smithi16367185-46 from 585009 to 1077864 tid 1 2023-10-09T09:10:29.995 INFO:tasks.rados.rados.0.smithi163.stdout:1243: writing smithi16367185-46 from 1564966 to 2185009 tid 2 2023-10-09T09:10:29.995 INFO:tasks.rados.rados.0.smithi163.stdout:1244: write oid 48 current snap is 154 2023-10-09T09:10:29.995 INFO:tasks.rados.rados.0.smithi163.stdout:1243: finishing write tid 1 to smithi16367185-46 2023-10-09T09:10:29.995 INFO:tasks.rados.rados.0.smithi163.stdout:1244: seq_num 488 ranges {601816=568104,1623448=451823} 2023-10-09T09:10:30.001 INFO:tasks.rados.rados.0.smithi163.stdout:1244: writing smithi16367185-48 from 601816 to 1169920 tid 1 2023-10-09T09:10:30.007 INFO:tasks.rados.rados.0.smithi163.stdout:1244: writing smithi16367185-48 from 1623448 to 2075271 tid 2 2023-10-09T09:10:30.008 INFO:tasks.rados.rados.0.smithi163.stdout:1245: write oid 18 current snap is 154 2023-10-09T09:10:30.008 INFO:tasks.rados.rados.0.smithi163.stdout:1243: finishing write tid 2 to smithi16367185-46 2023-10-09T09:10:30.008 INFO:tasks.rados.rados.0.smithi163.stdout:1243: finishing write tid 3 to smithi16367185-46 2023-10-09T09:10:30.008 INFO:tasks.rados.rados.0.smithi163.stdout:1243: finishing write tid 4 to smithi16367185-46 2023-10-09T09:10:30.008 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 506 (ObjNum 487 snap 154 seq_num 487) dirty exists 2023-10-09T09:10:30.008 INFO:tasks.rados.rados.0.smithi163.stdout:1243: left oid 46 (ObjNum 487 snap 154 seq_num 487) 2023-10-09T09:10:30.008 INFO:tasks.rados.rados.0.smithi163.stdout:1245: seq_num 489 ranges {618623=643353,1681930=536693} 2023-10-09T09:10:30.014 INFO:tasks.rados.rados.0.smithi163.stdout:1245: writing smithi16367185-18 from 618623 to 1261976 tid 1 2023-10-09T09:10:30.020 INFO:tasks.rados.rados.0.smithi163.stdout:1245: writing smithi16367185-18 from 1681930 to 2218623 tid 2 2023-10-09T09:10:30.020 INFO:tasks.rados.rados.0.smithi163.stdout:1243: done (2 left) 2023-10-09T09:10:30.020 INFO:tasks.rados.rados.0.smithi163.stdout:1246: snap_remove snap 151 2023-10-09T09:10:30.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:29 smithi163 bash[33912]: debug 2023-10-09T09:10:29.949+0000 7f84df46a700 -1 mon.c@2(peon).osd e369 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:30.300 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:29 smithi181 bash[27712]: debug 2023-10-09T09:10:29.947+0000 7fde83035700 -1 mon.b@1(peon).osd e369 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:30 smithi181 bash[27712]: cluster 2023-10-09T09:10:28.994047+0000 osd.1 (osd.1) 24 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:30 smithi181 bash[27712]: cluster 2023-10-09T09:10:28.999790+0000 osd.5 (osd.5) 27 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:30 smithi181 bash[27712]: cluster 2023-10-09T09:10:29.011557+0000 osd.2 (osd.2) 50 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:30 smithi181 bash[27712]: cluster 2023-10-09T09:10:29.230437+0000 mgr.y (mgr.34107) 48 : cluster [DBG] pgmap v99: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.6 MiB/s wr, 14 op/s 2023-10-09T09:10:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:30 smithi181 bash[27712]: cluster 2023-10-09T09:10:29.420945+0000 osd.6 (osd.6) 24 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:30 smithi181 bash[27712]: cluster 2023-10-09T09:10:29.957389+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2023-10-09T09:10:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:30 smithi163 bash[33912]: cluster 2023-10-09T09:10:28.994047+0000 osd.1 (osd.1) 24 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:30 smithi163 bash[33912]: cluster 2023-10-09T09:10:28.999790+0000 osd.5 (osd.5) 27 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:30 smithi163 bash[33912]: cluster 2023-10-09T09:10:29.011557+0000 osd.2 (osd.2) 50 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:30 smithi163 bash[33912]: cluster 2023-10-09T09:10:29.230437+0000 mgr.y (mgr.34107) 48 : cluster [DBG] pgmap v99: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.6 MiB/s wr, 14 op/s 2023-10-09T09:10:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:30 smithi163 bash[33912]: cluster 2023-10-09T09:10:29.420945+0000 osd.6 (osd.6) 24 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:30 smithi163 bash[33912]: cluster 2023-10-09T09:10:29.957389+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2023-10-09T09:10:31.000 INFO:tasks.rados.rados.0.smithi163.stdout:1246: done (2 left) 2023-10-09T09:10:31.000 INFO:tasks.rados.rados.0.smithi163.stdout:1247: write oid 20 current snap is 154 2023-10-09T09:10:31.002 INFO:tasks.rados.rados.0.smithi163.stdout:1244: finishing write tid 1 to smithi16367185-48 2023-10-09T09:10:31.002 INFO:tasks.rados.rados.0.smithi163.stdout:1247: seq_num 490 ranges {635430=718602,2140412=95018} 2023-10-09T09:10:31.014 INFO:tasks.rados.rados.0.smithi163.stdout:1247: writing smithi16367185-20 from 635430 to 1354032 tid 1 2023-10-09T09:10:31.020 INFO:tasks.rados.rados.0.smithi163.stdout:1247: writing smithi16367185-20 from 2140412 to 2235430 tid 2 2023-10-09T09:10:31.020 INFO:tasks.rados.rados.0.smithi163.stdout:1248: snap_create 2023-10-09T09:10:31.020 INFO:tasks.rados.rados.0.smithi163.stdout:1244: finishing write tid 2 to smithi16367185-48 2023-10-09T09:10:31.020 INFO:tasks.rados.rados.0.smithi163.stdout:1244: finishing write tid 3 to smithi16367185-48 2023-10-09T09:10:31.020 INFO:tasks.rados.rados.0.smithi163.stdout:1244: finishing write tid 4 to smithi16367185-48 2023-10-09T09:10:31.020 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 984 (ObjNum 488 snap 154 seq_num 488) dirty exists 2023-10-09T09:10:31.020 INFO:tasks.rados.rados.0.smithi163.stdout:1244: left oid 48 (ObjNum 488 snap 154 seq_num 488) 2023-10-09T09:10:31.021 INFO:tasks.rados.rados.0.smithi163.stdout:1245: finishing write tid 1 to smithi16367185-18 2023-10-09T09:10:31.021 INFO:tasks.rados.rados.0.smithi163.stdout:1245: finishing write tid 2 to smithi16367185-18 2023-10-09T09:10:31.021 INFO:tasks.rados.rados.0.smithi163.stdout:1245: finishing write tid 3 to smithi16367185-18 2023-10-09T09:10:31.021 INFO:tasks.rados.rados.0.smithi163.stdout:1245: finishing write tid 4 to smithi16367185-18 2023-10-09T09:10:31.021 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 463 (ObjNum 489 snap 154 seq_num 489) dirty exists 2023-10-09T09:10:31.021 INFO:tasks.rados.rados.0.smithi163.stdout:1245: left oid 18 (ObjNum 489 snap 154 seq_num 489) 2023-10-09T09:10:31.021 INFO:tasks.rados.rados.0.smithi163.stdout:1247: finishing write tid 1 to smithi16367185-20 2023-10-09T09:10:31.023 INFO:tasks.rados.rados.0.smithi163.stdout:1247: finishing write tid 2 to smithi16367185-20 2023-10-09T09:10:31.023 INFO:tasks.rados.rados.0.smithi163.stdout:1247: finishing write tid 3 to smithi16367185-20 2023-10-09T09:10:31.023 INFO:tasks.rados.rados.0.smithi163.stdout:1247: finishing write tid 4 to smithi16367185-20 2023-10-09T09:10:31.023 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 146 (ObjNum 490 snap 154 seq_num 490) dirty exists 2023-10-09T09:10:31.024 INFO:tasks.rados.rados.0.smithi163.stdout:1247: left oid 20 (ObjNum 490 snap 154 seq_num 490) 2023-10-09T09:10:31.303 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:30 smithi181 bash[27712]: debug 2023-10-09T09:10:30.983+0000 7fde83035700 -1 mon.b@1(peon).osd e370 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:31.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:30 smithi163 bash[33912]: debug 2023-10-09T09:10:30.985+0000 7f84df46a700 -1 mon.c@2(peon).osd e370 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:31 smithi181 bash[27712]: cluster 2023-10-09T09:10:29.534295+0000 osd.3 (osd.3) 31 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:31 smithi181 bash[27712]: cluster 2023-10-09T09:10:29.960349+0000 osd.6 (osd.6) 25 : cluster [WRN] failed to encode map e369 with expected crc 2023-10-09T09:10:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:31 smithi181 bash[27712]: cluster 2023-10-09T09:10:29.961115+0000 osd.2 (osd.2) 51 : cluster [WRN] failed to encode map e369 with expected crc 2023-10-09T09:10:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:31 smithi181 bash[27712]: cluster 2023-10-09T09:10:29.965726+0000 osd.5 (osd.5) 28 : cluster [WRN] failed to encode map e369 with expected crc 2023-10-09T09:10:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:31 smithi181 bash[27712]: cluster 2023-10-09T09:10:30.995212+0000 mon.a (mon.0) 145 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2023-10-09T09:10:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:31 smithi163 bash[33912]: cluster 2023-10-09T09:10:29.534295+0000 osd.3 (osd.3) 31 : cluster [WRN] failed to encode map e368 with expected crc 2023-10-09T09:10:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:31 smithi163 bash[33912]: cluster 2023-10-09T09:10:29.960349+0000 osd.6 (osd.6) 25 : cluster [WRN] failed to encode map e369 with expected crc 2023-10-09T09:10:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:31 smithi163 bash[33912]: cluster 2023-10-09T09:10:29.961115+0000 osd.2 (osd.2) 51 : cluster [WRN] failed to encode map e369 with expected crc 2023-10-09T09:10:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:31 smithi163 bash[33912]: cluster 2023-10-09T09:10:29.965726+0000 osd.5 (osd.5) 28 : cluster [WRN] failed to encode map e369 with expected crc 2023-10-09T09:10:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:31 smithi163 bash[33912]: cluster 2023-10-09T09:10:30.995212+0000 mon.a (mon.0) 145 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2023-10-09T09:10:32.012 INFO:tasks.rados.rados.0.smithi163.stdout:1244: done (3 left) 2023-10-09T09:10:32.012 INFO:tasks.rados.rados.0.smithi163.stdout:1245: done (2 left) 2023-10-09T09:10:32.012 INFO:tasks.rados.rados.0.smithi163.stdout:1247: done (1 left) 2023-10-09T09:10:32.012 INFO:tasks.rados.rados.0.smithi163.stdout:1248: done (0 left) 2023-10-09T09:10:32.012 INFO:tasks.rados.rados.0.smithi163.stdout:1249: snap_remove snap 149 2023-10-09T09:10:32.306 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:31 smithi181 bash[27712]: debug 2023-10-09T09:10:31.995+0000 7fde83035700 -1 mon.b@1(peon).osd e371 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:32.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:32 smithi163 bash[33912]: debug 2023-10-09T09:10:31.997+0000 7f84df46a700 -1 mon.c@2(peon).osd e371 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:32 smithi181 bash[27712]: cluster 2023-10-09T09:10:30.996312+0000 osd.5 (osd.5) 29 : cluster [WRN] failed to encode map e370 with expected crc 2023-10-09T09:10:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:32 smithi181 bash[27712]: cluster 2023-10-09T09:10:30.999583+0000 osd.2 (osd.2) 52 : cluster [WRN] failed to encode map e370 with expected crc 2023-10-09T09:10:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:32 smithi181 bash[27712]: cluster 2023-10-09T09:10:31.231635+0000 mgr.y (mgr.34107) 49 : cluster [DBG] pgmap v102: 89 pgs: 89 active+clean; 188 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.2 MiB/s wr, 20 op/s 2023-10-09T09:10:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:32 smithi181 bash[27712]: cluster 2023-10-09T09:10:32.001933+0000 mon.a (mon.0) 146 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2023-10-09T09:10:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:32 smithi181 bash[27712]: cluster 2023-10-09T09:10:32.008250+0000 osd.4 (osd.4) 59 : cluster [WRN] failed to encode map e371 with expected crc 2023-10-09T09:10:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:32 smithi163 bash[33912]: cluster 2023-10-09T09:10:30.996312+0000 osd.5 (osd.5) 29 : cluster [WRN] failed to encode map e370 with expected crc 2023-10-09T09:10:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:32 smithi163 bash[33912]: cluster 2023-10-09T09:10:30.999583+0000 osd.2 (osd.2) 52 : cluster [WRN] failed to encode map e370 with expected crc 2023-10-09T09:10:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:32 smithi163 bash[33912]: cluster 2023-10-09T09:10:31.231635+0000 mgr.y (mgr.34107) 49 : cluster [DBG] pgmap v102: 89 pgs: 89 active+clean; 188 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.2 MiB/s wr, 20 op/s 2023-10-09T09:10:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:32 smithi163 bash[33912]: cluster 2023-10-09T09:10:32.001933+0000 mon.a (mon.0) 146 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2023-10-09T09:10:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:32 smithi163 bash[33912]: cluster 2023-10-09T09:10:32.008250+0000 osd.4 (osd.4) 59 : cluster [WRN] failed to encode map e371 with expected crc 2023-10-09T09:10:33.015 INFO:tasks.rados.rados.0.smithi163.stdout:1249: done (0 left) 2023-10-09T09:10:33.015 INFO:tasks.rados.rados.0.smithi163.stdout:1250: read oid 18 snap -1 2023-10-09T09:10:33.015 INFO:tasks.rados.rados.0.smithi163.stdout:1250: expect (ObjNum 489 snap 154 seq_num 489) 2023-10-09T09:10:33.015 INFO:tasks.rados.rados.0.smithi163.stdout:1251: snap_remove snap 124 2023-10-09T09:10:33.308 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:33 smithi181 bash[27712]: debug 2023-10-09T09:10:32.995+0000 7fde83035700 -1 mon.b@1(peon).osd e372 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:33.309 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:33 smithi163 bash[33912]: debug 2023-10-09T09:10:33.001+0000 7f84df46a700 -1 mon.c@2(peon).osd e372 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:33 smithi181 bash[27712]: cluster 2023-10-09T09:10:32.007846+0000 osd.0 (osd.0) 40 : cluster [WRN] failed to encode map e371 with expected crc 2023-10-09T09:10:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:33 smithi181 bash[27712]: cluster 2023-10-09T09:10:33.008757+0000 mon.a (mon.0) 147 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2023-10-09T09:10:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:33 smithi181 bash[27712]: cluster 2023-10-09T09:10:33.011329+0000 osd.4 (osd.4) 60 : cluster [WRN] failed to encode map e372 with expected crc 2023-10-09T09:10:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:10:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:10:33] "GET /metrics HTTP/1.1" 200 36080 "" "Prometheus/2.33.4" 2023-10-09T09:10:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:33 smithi163 bash[33912]: cluster 2023-10-09T09:10:32.007846+0000 osd.0 (osd.0) 40 : cluster [WRN] failed to encode map e371 with expected crc 2023-10-09T09:10:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:33 smithi163 bash[33912]: cluster 2023-10-09T09:10:33.008757+0000 mon.a (mon.0) 147 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2023-10-09T09:10:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:33 smithi163 bash[33912]: cluster 2023-10-09T09:10:33.011329+0000 osd.4 (osd.4) 60 : cluster [WRN] failed to encode map e372 with expected crc 2023-10-09T09:10:34.020 INFO:tasks.rados.rados.0.smithi163.stdout:1251: done (1 left) 2023-10-09T09:10:34.020 INFO:tasks.rados.rados.0.smithi163.stdout:1252: rollback oid 32 current snap is 155 2023-10-09T09:10:34.020 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 145 2023-10-09T09:10:34.020 INFO:tasks.rados.rados.0.smithi163.stdout:1250: expect (ObjNum 489 snap 154 seq_num 489) 2023-10-09T09:10:34.168 INFO:tasks.rados.rados.0.smithi163.stdout:1252: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:10:34.168 INFO:tasks.rados.rados.0.smithi163.stdout:1252: finishing rollback tid 2 to smithi16367185-32 2023-10-09T09:10:34.168 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 994 (ObjNum 422 snap 129 seq_num 422) dirty exists 2023-10-09T09:10:34.168 INFO:tasks.rados.rados.0.smithi163.stdout:1250: done (1 left) 2023-10-09T09:10:34.168 INFO:tasks.rados.rados.0.smithi163.stdout:1252: done (0 left) 2023-10-09T09:10:34.168 INFO:tasks.rados.rados.0.smithi163.stdout:1253: write oid 37 current snap is 155 2023-10-09T09:10:34.168 INFO:tasks.rados.rados.0.smithi163.stdout:1253: seq_num 491 ranges {652237=793851,2198894=53343} 2023-10-09T09:10:34.184 INFO:tasks.rados.rados.0.smithi163.stdout:1253: writing smithi16367185-37 from 652237 to 1446088 tid 1 2023-10-09T09:10:34.189 INFO:tasks.rados.rados.0.smithi163.stdout:1253: writing smithi16367185-37 from 2198894 to 2252237 tid 2 2023-10-09T09:10:34.189 INFO:tasks.rados.rados.0.smithi163.stdout:1254: write oid 39 current snap is 155 2023-10-09T09:10:34.190 INFO:tasks.rados.rados.0.smithi163.stdout:1254: seq_num 492 ranges {669044=469100,1741023=459161} 2023-10-09T09:10:34.196 INFO:tasks.rados.rados.0.smithi163.stdout:1254: writing smithi16367185-39 from 669044 to 1138144 tid 1 2023-10-09T09:10:34.202 INFO:tasks.rados.rados.0.smithi163.stdout:1254: writing smithi16367185-39 from 1741023 to 2200184 tid 2 2023-10-09T09:10:34.202 INFO:tasks.rados.rados.0.smithi163.stdout:1255: delete oid 38 current snap is 155 2023-10-09T09:10:34.202 INFO:tasks.rados.rados.0.smithi163.stdout:1253: finishing write tid 1 to smithi16367185-37 2023-10-09T09:10:34.202 INFO:tasks.rados.rados.0.smithi163.stdout:1253: finishing write tid 2 to smithi16367185-37 2023-10-09T09:10:34.202 INFO:tasks.rados.rados.0.smithi163.stdout:1253: finishing write tid 3 to smithi16367185-37 2023-10-09T09:10:34.203 INFO:tasks.rados.rados.0.smithi163.stdout:1253: finishing write tid 4 to smithi16367185-37 2023-10-09T09:10:34.203 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 622 (ObjNum 491 snap 155 seq_num 491) dirty exists 2023-10-09T09:10:34.203 INFO:tasks.rados.rados.0.smithi163.stdout:1253: left oid 37 (ObjNum 491 snap 155 seq_num 491) 2023-10-09T09:10:34.203 INFO:tasks.rados.rados.0.smithi163.stdout:1254: finishing write tid 1 to smithi16367185-39 2023-10-09T09:10:34.205 INFO:tasks.rados.rados.0.smithi163.stdout:1253: done (2 left) 2023-10-09T09:10:34.205 INFO:tasks.rados.rados.0.smithi163.stdout:1255: done (1 left) 2023-10-09T09:10:34.205 INFO:tasks.rados.rados.0.smithi163.stdout:1256: rollback oid 25 current snap is 155 2023-10-09T09:10:34.205 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 25 to 154 2023-10-09T09:10:34.205 INFO:tasks.rados.rados.0.smithi163.stdout:1257: read oid 7 snap -1 2023-10-09T09:10:34.206 INFO:tasks.rados.rados.0.smithi163.stdout:1257: expect (ObjNum 453 snap 146 seq_num 453) 2023-10-09T09:10:34.206 INFO:tasks.rados.rados.0.smithi163.stdout:1258: delete oid 33 current snap is 155 2023-10-09T09:10:34.206 INFO:tasks.rados.rados.0.smithi163.stdout:1254: finishing write tid 2 to smithi16367185-39 2023-10-09T09:10:34.206 INFO:tasks.rados.rados.0.smithi163.stdout:1254: finishing write tid 3 to smithi16367185-39 2023-10-09T09:10:34.207 INFO:tasks.rados.rados.0.smithi163.stdout:1254: finishing write tid 4 to smithi16367185-39 2023-10-09T09:10:34.207 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 506 (ObjNum 492 snap 155 seq_num 492) dirty exists 2023-10-09T09:10:34.207 INFO:tasks.rados.rados.0.smithi163.stdout:1254: left oid 39 (ObjNum 492 snap 155 seq_num 492) 2023-10-09T09:10:34.209 INFO:tasks.rados.rados.0.smithi163.stdout:1256: finishing rollback tid 0 to smithi16367185-25 2023-10-09T09:10:34.210 INFO:tasks.rados.rados.0.smithi163.stdout:1256: finishing rollback tid 1 to smithi16367185-25 2023-10-09T09:10:34.212 INFO:tasks.rados.rados.0.smithi163.stdout:1256: finishing rollback tid 2 to smithi16367185-25 2023-10-09T09:10:34.212 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 516 (ObjNum 443 snap 138 seq_num 443) dirty exists 2023-10-09T09:10:34.212 INFO:tasks.rados.rados.0.smithi163.stdout:1257: expect (ObjNum 453 snap 146 seq_num 453) 2023-10-09T09:10:34.309 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:34 smithi163 bash[33912]: debug 2023-10-09T09:10:34.009+0000 7f84df46a700 -1 mon.c@2(peon).osd e373 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:34.310 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:34 smithi181 bash[27712]: debug 2023-10-09T09:10:34.007+0000 7fde83035700 -1 mon.b@1(peon).osd e373 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:34.340 INFO:tasks.rados.rados.0.smithi163.stdout:1254: done (3 left) 2023-10-09T09:10:34.340 INFO:tasks.rados.rados.0.smithi163.stdout:1256: done (2 left) 2023-10-09T09:10:34.340 INFO:tasks.rados.rados.0.smithi163.stdout:1257: done (1 left) 2023-10-09T09:10:34.340 INFO:tasks.rados.rados.0.smithi163.stdout:1258: done (0 left) 2023-10-09T09:10:34.340 INFO:tasks.rados.rados.0.smithi163.stdout:1259: write oid 7 current snap is 155 2023-10-09T09:10:34.340 INFO:tasks.rados.rados.0.smithi163.stdout:1259: seq_num 495 ranges {719465=411200,1632822=686643} 2023-10-09T09:10:34.346 INFO:tasks.rados.rados.0.smithi163.stdout:1259: writing smithi16367185-7 from 719465 to 1130665 tid 1 2023-10-09T09:10:34.353 INFO:tasks.rados.rados.0.smithi163.stdout:1259: writing smithi16367185-7 from 1632822 to 2319465 tid 2 2023-10-09T09:10:34.353 INFO:tasks.rados.rados.0.smithi163.stdout:1260: snap_create 2023-10-09T09:10:34.353 INFO:tasks.rados.rados.0.smithi163.stdout:1259: finishing write tid 1 to smithi16367185-7 2023-10-09T09:10:34.357 INFO:tasks.rados.rados.0.smithi163.stdout:1259: finishing write tid 2 to smithi16367185-7 2023-10-09T09:10:34.357 INFO:tasks.rados.rados.0.smithi163.stdout:1259: finishing write tid 3 to smithi16367185-7 2023-10-09T09:10:34.357 INFO:tasks.rados.rados.0.smithi163.stdout:1259: finishing write tid 4 to smithi16367185-7 2023-10-09T09:10:34.357 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 305 (ObjNum 495 snap 155 seq_num 495) dirty exists 2023-10-09T09:10:34.357 INFO:tasks.rados.rados.0.smithi163.stdout:1259: left oid 7 (ObjNum 495 snap 155 seq_num 495) 2023-10-09T09:10:34.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:34 smithi163 bash[33912]: cluster 2023-10-09T09:10:33.012714+0000 osd.2 (osd.2) 53 : cluster [WRN] failed to encode map e372 with expected crc 2023-10-09T09:10:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:34 smithi163 bash[33912]: cluster 2023-10-09T09:10:33.232324+0000 mgr.y (mgr.34107) 50 : cluster [DBG] pgmap v105: 89 pgs: 89 active+clean; 188 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.2 MiB/s wr, 20 op/s 2023-10-09T09:10:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:34 smithi163 bash[33912]: cluster 2023-10-09T09:10:34.016385+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2023-10-09T09:10:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:34 smithi181 bash[27712]: cluster 2023-10-09T09:10:33.012714+0000 osd.2 (osd.2) 53 : cluster [WRN] failed to encode map e372 with expected crc 2023-10-09T09:10:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:34 smithi181 bash[27712]: cluster 2023-10-09T09:10:33.232324+0000 mgr.y (mgr.34107) 50 : cluster [DBG] pgmap v105: 89 pgs: 89 active+clean; 188 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.2 MiB/s wr, 20 op/s 2023-10-09T09:10:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:34 smithi181 bash[27712]: cluster 2023-10-09T09:10:34.016385+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2023-10-09T09:10:34.803 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T09:10:34.966 INFO:tasks.rados.rados.0.smithi163.stdout:1259: done (1 left) 2023-10-09T09:10:34.967 INFO:tasks.rados.rados.0.smithi163.stdout:1260: done (0 left) 2023-10-09T09:10:34.967 INFO:tasks.rados.rados.0.smithi163.stdout:1261: write oid 44 current snap is 156 2023-10-09T09:10:34.967 INFO:tasks.rados.rados.0.smithi163.stdout:1261: seq_num 496 ranges {736272=486449,1974951=361321} 2023-10-09T09:10:34.977 INFO:tasks.rados.rados.0.smithi163.stdout:1261: writing smithi16367185-44 from 736272 to 1222721 tid 1 2023-10-09T09:10:34.985 INFO:tasks.rados.rados.0.smithi163.stdout:1261: writing smithi16367185-44 from 1974951 to 2336272 tid 2 2023-10-09T09:10:34.985 INFO:tasks.rados.rados.0.smithi163.stdout:1262: read oid 49 snap 153 2023-10-09T09:10:34.985 INFO:tasks.rados.rados.0.smithi163.stdout:1262: expect (ObjNum 448 snap 144 seq_num 448) 2023-10-09T09:10:34.985 INFO:tasks.rados.rados.0.smithi163.stdout:1263: rollback oid 32 current snap is 156 2023-10-09T09:10:34.986 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 152 2023-10-09T09:10:34.986 INFO:tasks.rados.rados.0.smithi163.stdout:1264: read oid 5 snap -1 2023-10-09T09:10:34.986 INFO:tasks.rados.rados.0.smithi163.stdout:1264: expect (ObjNum 436 snap 136 seq_num 436) 2023-10-09T09:10:34.986 INFO:tasks.rados.rados.0.smithi163.stdout:1265: snap_remove snap 145 2023-10-09T09:10:35.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:34 smithi163 bash[33912]: debug 2023-10-09T09:10:34.953+0000 7f84df46a700 -1 mon.c@2(peon).osd e374 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:35.312 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:34 smithi181 bash[27712]: debug 2023-10-09T09:10:34.951+0000 7fde83035700 -1 mon.b@1(peon).osd e374 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:35 smithi181 bash[27712]: cluster 2023-10-09T09:10:34.016692+0000 osd.6 (osd.6) 26 : cluster [WRN] failed to encode map e373 with expected crc 2023-10-09T09:10:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:35 smithi181 bash[27712]: cluster 2023-10-09T09:10:34.019973+0000 osd.5 (osd.5) 30 : cluster [WRN] failed to encode map e373 with expected crc 2023-10-09T09:10:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:35 smithi181 bash[27712]: cluster 2023-10-09T09:10:34.024810+0000 osd.1 (osd.1) 25 : cluster [WRN] failed to encode map e373 with expected crc 2023-10-09T09:10:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:35 smithi181 bash[27712]: cluster 2023-10-09T09:10:34.959113+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2023-10-09T09:10:35.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:35 smithi181 bash[27712]: cluster 2023-10-09T09:10:34.963095+0000 osd.4 (osd.4) 61 : cluster [WRN] failed to encode map e374 with expected crc 2023-10-09T09:10:35.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:35 smithi181 bash[27712]: cluster 2023-10-09T09:10:34.989178+0000 osd.2 (osd.2) 54 : cluster [WRN] failed to encode map e374 with expected crc 2023-10-09T09:10:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:35 smithi163 bash[33912]: cluster 2023-10-09T09:10:34.016692+0000 osd.6 (osd.6) 26 : cluster [WRN] failed to encode map e373 with expected crc 2023-10-09T09:10:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:35 smithi163 bash[33912]: cluster 2023-10-09T09:10:34.019973+0000 osd.5 (osd.5) 30 : cluster [WRN] failed to encode map e373 with expected crc 2023-10-09T09:10:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:35 smithi163 bash[33912]: cluster 2023-10-09T09:10:34.024810+0000 osd.1 (osd.1) 25 : cluster [WRN] failed to encode map e373 with expected crc 2023-10-09T09:10:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:35 smithi163 bash[33912]: cluster 2023-10-09T09:10:34.959113+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2023-10-09T09:10:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:35 smithi163 bash[33912]: cluster 2023-10-09T09:10:34.963095+0000 osd.4 (osd.4) 61 : cluster [WRN] failed to encode map e374 with expected crc 2023-10-09T09:10:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:35 smithi163 bash[33912]: cluster 2023-10-09T09:10:34.989178+0000 osd.2 (osd.2) 54 : cluster [WRN] failed to encode map e374 with expected crc 2023-10-09T09:10:35.984 INFO:tasks.rados.rados.0.smithi163.stdout:1261: finishing write tid 1 to smithi16367185-44 2023-10-09T09:10:35.985 INFO:tasks.rados.rados.0.smithi163.stdout:1261: finishing write tid 2 to smithi16367185-44 2023-10-09T09:10:35.985 INFO:tasks.rados.rados.0.smithi163.stdout:1261: finishing write tid 3 to smithi16367185-44 2023-10-09T09:10:35.985 INFO:tasks.rados.rados.0.smithi163.stdout:1261: finishing write tid 4 to smithi16367185-44 2023-10-09T09:10:35.985 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1002 (ObjNum 496 snap 156 seq_num 496) dirty exists 2023-10-09T09:10:35.985 INFO:tasks.rados.rados.0.smithi163.stdout:1261: left oid 44 (ObjNum 496 snap 156 seq_num 496) 2023-10-09T09:10:35.985 INFO:tasks.rados.rados.0.smithi163.stdout:1263: finishing rollback tid 0 to smithi16367185-32 2023-10-09T09:10:35.985 INFO:tasks.rados.rados.0.smithi163.stdout:1262: expect (ObjNum 448 snap 144 seq_num 448) 2023-10-09T09:10:36.155 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:35 smithi163 bash[33912]: debug 2023-10-09T09:10:35.973+0000 7f84df46a700 -1 mon.c@2(peon).osd e375 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:36.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:35 smithi181 bash[27712]: debug 2023-10-09T09:10:35.971+0000 7fde83035700 -1 mon.b@1(peon).osd e375 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:36.316 INFO:tasks.rados.rados.0.smithi163.stdout:1263: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:10:36.316 INFO:tasks.rados.rados.0.smithi163.stdout:1264: expect (ObjNum 436 snap 136 seq_num 436) 2023-10-09T09:10:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:36 smithi163 bash[33912]: cluster 2023-10-09T09:10:34.963206+0000 osd.1 (osd.1) 26 : cluster [WRN] failed to encode map e374 with expected crc 2023-10-09T09:10:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:36 smithi163 bash[33912]: cluster 2023-10-09T09:10:34.977127+0000 osd.5 (osd.5) 31 : cluster [WRN] failed to encode map e374 with expected crc 2023-10-09T09:10:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:36 smithi163 bash[33912]: cluster 2023-10-09T09:10:35.220812+0000 osd.7 (osd.7) 41 : cluster [WRN] failed to encode map e374 with expected crc 2023-10-09T09:10:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:36 smithi163 bash[33912]: cluster 2023-10-09T09:10:35.232956+0000 mgr.y (mgr.34107) 51 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 188 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:10:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:36 smithi163 bash[33912]: cluster 2023-10-09T09:10:35.977961+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2023-10-09T09:10:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:36 smithi163 bash[33912]: cluster 2023-10-09T09:10:35.980854+0000 osd.4 (osd.4) 62 : cluster [WRN] failed to encode map e375 with expected crc 2023-10-09T09:10:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:36 smithi181 bash[27712]: cluster 2023-10-09T09:10:34.963206+0000 osd.1 (osd.1) 26 : cluster [WRN] failed to encode map e374 with expected crc 2023-10-09T09:10:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:36 smithi181 bash[27712]: cluster 2023-10-09T09:10:34.977127+0000 osd.5 (osd.5) 31 : cluster [WRN] failed to encode map e374 with expected crc 2023-10-09T09:10:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:36 smithi181 bash[27712]: cluster 2023-10-09T09:10:35.220812+0000 osd.7 (osd.7) 41 : cluster [WRN] failed to encode map e374 with expected crc 2023-10-09T09:10:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:36 smithi181 bash[27712]: cluster 2023-10-09T09:10:35.232956+0000 mgr.y (mgr.34107) 51 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 188 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:10:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:36 smithi181 bash[27712]: cluster 2023-10-09T09:10:35.977961+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2023-10-09T09:10:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:36 smithi181 bash[27712]: cluster 2023-10-09T09:10:35.980854+0000 osd.4 (osd.4) 62 : cluster [WRN] failed to encode map e375 with expected crc 2023-10-09T09:10:36.584 INFO:tasks.rados.rados.0.smithi163.stdout:1263: finishing rollback tid 2 to smithi16367185-32 2023-10-09T09:10:36.584 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1006 (ObjNum 422 snap 129 seq_num 422) dirty exists 2023-10-09T09:10:36.584 INFO:tasks.rados.rados.0.smithi163.stdout:1261: done (4 left) 2023-10-09T09:10:36.584 INFO:tasks.rados.rados.0.smithi163.stdout:1262: done (3 left) 2023-10-09T09:10:36.584 INFO:tasks.rados.rados.0.smithi163.stdout:1263: done (2 left) 2023-10-09T09:10:36.584 INFO:tasks.rados.rados.0.smithi163.stdout:1264: done (1 left) 2023-10-09T09:10:36.584 INFO:tasks.rados.rados.0.smithi163.stdout:1265: done (0 left) 2023-10-09T09:10:36.585 INFO:tasks.rados.rados.0.smithi163.stdout:1266: write oid 7 current snap is 156 2023-10-09T09:10:36.585 INFO:tasks.rados.rados.0.smithi163.stdout:1266: seq_num 497 ranges {753079=561698,2033433=319646} 2023-10-09T09:10:36.597 INFO:tasks.rados.rados.0.smithi163.stdout:1266: writing smithi16367185-7 from 753079 to 1314777 tid 1 2023-10-09T09:10:36.604 INFO:tasks.rados.rados.0.smithi163.stdout:1266: writing smithi16367185-7 from 2033433 to 2353079 tid 2 2023-10-09T09:10:36.604 INFO:tasks.rados.rados.0.smithi163.stdout:1267: rollback oid 36 current snap is 156 2023-10-09T09:10:36.604 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 36 to 155 2023-10-09T09:10:36.604 INFO:tasks.rados.rados.0.smithi163.stdout:1268: write oid 44 current snap is 156 2023-10-09T09:10:36.604 INFO:tasks.rados.rados.0.smithi163.stdout:1266: finishing write tid 1 to smithi16367185-7 2023-10-09T09:10:36.604 INFO:tasks.rados.rados.0.smithi163.stdout:1268: seq_num 498 ranges {769886=636947,2091915=277971} 2023-10-09T09:10:36.611 INFO:tasks.rados.rados.0.smithi163.stdout:1268: writing smithi16367185-44 from 769886 to 1406833 tid 1 2023-10-09T09:10:36.618 INFO:tasks.rados.rados.0.smithi163.stdout:1268: writing smithi16367185-44 from 2091915 to 2369886 tid 2 2023-10-09T09:10:36.619 INFO:tasks.rados.rados.0.smithi163.stdout:1269: delete oid 39 current snap is 156 2023-10-09T09:10:36.619 INFO:tasks.rados.rados.0.smithi163.stdout:1267: finishing rollback tid 1 to smithi16367185-36 2023-10-09T09:10:36.619 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 338 (ObjNum 698872336 snap 21875 seq_num 699074896) dirty dne 2023-10-09T09:10:36.619 INFO:tasks.rados.rados.0.smithi163.stdout:1266: finishing write tid 2 to smithi16367185-7 2023-10-09T09:10:36.619 INFO:tasks.rados.rados.0.smithi163.stdout:1266: finishing write tid 3 to smithi16367185-7 2023-10-09T09:10:36.619 INFO:tasks.rados.rados.0.smithi163.stdout:1266: finishing write tid 4 to smithi16367185-7 2023-10-09T09:10:36.619 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 309 (ObjNum 497 snap 156 seq_num 497) dirty exists 2023-10-09T09:10:36.619 INFO:tasks.rados.rados.0.smithi163.stdout:1266: left oid 7 (ObjNum 497 snap 156 seq_num 497) 2023-10-09T09:10:36.619 INFO:tasks.rados.rados.0.smithi163.stdout:1268: finishing write tid 1 to smithi16367185-44 2023-10-09T09:10:36.621 INFO:tasks.rados.rados.0.smithi163.stdout:1268: finishing write tid 2 to smithi16367185-44 2023-10-09T09:10:36.622 INFO:tasks.rados.rados.0.smithi163.stdout:1268: finishing write tid 3 to smithi16367185-44 2023-10-09T09:10:36.622 INFO:tasks.rados.rados.0.smithi163.stdout:1268: finishing write tid 4 to smithi16367185-44 2023-10-09T09:10:36.622 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1015 (ObjNum 498 snap 156 seq_num 498) dirty exists 2023-10-09T09:10:36.622 INFO:tasks.rados.rados.0.smithi163.stdout:1268: left oid 44 (ObjNum 498 snap 156 seq_num 498) 2023-10-09T09:10:36.624 INFO:tasks.rados.rados.0.smithi163.stdout:1266: done (3 left) 2023-10-09T09:10:36.624 INFO:tasks.rados.rados.0.smithi163.stdout:1267: done (2 left) 2023-10-09T09:10:36.624 INFO:tasks.rados.rados.0.smithi163.stdout:1268: done (1 left) 2023-10-09T09:10:36.624 INFO:tasks.rados.rados.0.smithi163.stdout:1269: done (0 left) 2023-10-09T09:10:36.625 INFO:tasks.rados.rados.0.smithi163.stdout:1270: read oid 6 snap 140 2023-10-09T09:10:36.625 INFO:tasks.rados.rados.0.smithi163.stdout:1270: expect (ObjNum 431 snap 134 seq_num 431) 2023-10-09T09:10:36.625 INFO:tasks.rados.rados.0.smithi163.stdout:1271: write oid 24 current snap is 156 2023-10-09T09:10:36.625 INFO:tasks.rados.rados.0.smithi163.stdout:1271: seq_num 500 ranges {403500=787445,1692526=473837} 2023-10-09T09:10:36.634 INFO:tasks.rados.rados.0.smithi163.stdout:1271: writing smithi16367185-24 from 403500 to 1190945 tid 1 2023-10-09T09:10:36.640 INFO:tasks.rados.rados.0.smithi163.stdout:1271: writing smithi16367185-24 from 1692526 to 2166363 tid 2 2023-10-09T09:10:36.640 INFO:tasks.rados.rados.0.smithi163.stdout:1272: write oid 7 current snap is 156 2023-10-09T09:10:36.640 INFO:tasks.rados.rados.0.smithi163.stdout:1272: seq_num 501 ranges {420307=462694,1351008=617495} 2023-10-09T09:10:36.647 INFO:tasks.rados.rados.0.smithi163.stdout:1272: writing smithi16367185-7 from 420307 to 883001 tid 1 2023-10-09T09:10:36.653 INFO:tasks.rados.rados.0.smithi163.stdout:1272: writing smithi16367185-7 from 1351008 to 1968503 tid 2 2023-10-09T09:10:36.653 INFO:tasks.rados.rados.0.smithi163.stdout:1273: snap_create 2023-10-09T09:10:36.653 INFO:tasks.rados.rados.0.smithi163.stdout:1270: expect (ObjNum 431 snap 134 seq_num 431) 2023-10-09T09:10:36.927 INFO:tasks.rados.rados.0.smithi163.stdout:1271: finishing write tid 1 to smithi16367185-24 2023-10-09T09:10:36.927 INFO:tasks.rados.rados.0.smithi163.stdout:1271: finishing write tid 2 to smithi16367185-24 2023-10-09T09:10:36.927 INFO:tasks.rados.rados.0.smithi163.stdout:1271: finishing write tid 3 to smithi16367185-24 2023-10-09T09:10:36.927 INFO:tasks.rados.rados.0.smithi163.stdout:1271: finishing write tid 4 to smithi16367185-24 2023-10-09T09:10:36.927 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 340 (ObjNum 500 snap 156 seq_num 500) dirty exists 2023-10-09T09:10:36.928 INFO:tasks.rados.rados.0.smithi163.stdout:1271: left oid 24 (ObjNum 500 snap 156 seq_num 500) 2023-10-09T09:10:36.928 INFO:tasks.rados.rados.0.smithi163.stdout:1272: finishing write tid 1 to smithi16367185-7 2023-10-09T09:10:36.928 INFO:tasks.rados.rados.0.smithi163.stdout:1272: finishing write tid 2 to smithi16367185-7 2023-10-09T09:10:36.928 INFO:tasks.rados.rados.0.smithi163.stdout:1272: finishing write tid 3 to smithi16367185-7 2023-10-09T09:10:36.928 INFO:tasks.rados.rados.0.smithi163.stdout:1272: finishing write tid 4 to smithi16367185-7 2023-10-09T09:10:36.928 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 312 (ObjNum 501 snap 156 seq_num 501) dirty exists 2023-10-09T09:10:36.928 INFO:tasks.rados.rados.0.smithi163.stdout:1272: left oid 7 (ObjNum 501 snap 156 seq_num 501) 2023-10-09T09:10:37.331 INFO:tasks.rados.rados.0.smithi163.stdout:1270: done (3 left) 2023-10-09T09:10:37.331 INFO:tasks.rados.rados.0.smithi163.stdout:1271: done (2 left) 2023-10-09T09:10:37.331 INFO:tasks.rados.rados.0.smithi163.stdout:1272: done (1 left) 2023-10-09T09:10:37.331 INFO:tasks.rados.rados.0.smithi163.stdout:1273: done (0 left) 2023-10-09T09:10:37.331 INFO:tasks.rados.rados.0.smithi163.stdout:1274: rollback oid 42 current snap is 157 2023-10-09T09:10:37.331 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 153 2023-10-09T09:10:37.331 INFO:tasks.rados.rados.0.smithi163.stdout:1275: rollback oid 39 current snap is 157 2023-10-09T09:10:37.331 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 155 2023-10-09T09:10:37.332 INFO:tasks.rados.rados.0.smithi163.stdout:1276: write oid 21 current snap is 157 2023-10-09T09:10:37.332 INFO:tasks.rados.rados.0.smithi163.stdout:1276: seq_num 502 ranges {437114=654296,1525843=477506} 2023-10-09T09:10:37.338 INFO:tasks.rados.rados.0.smithi163.stdout:1276: writing smithi16367185-21 from 437114 to 1091410 tid 1 2023-10-09T09:10:37.344 INFO:tasks.rados.rados.0.smithi163.stdout:1276: writing smithi16367185-21 from 1525843 to 2003349 tid 2 2023-10-09T09:10:37.344 INFO:tasks.rados.rados.0.smithi163.stdout:1277: read oid 50 snap -1 2023-10-09T09:10:37.344 INFO:tasks.rados.rados.0.smithi163.stdout:1277: expect (ObjNum 474 snap 152 seq_num 474) 2023-10-09T09:10:37.345 INFO:tasks.rados.rados.0.smithi163.stdout:1278: write oid 38 current snap is 157 2023-10-09T09:10:37.345 INFO:tasks.rados.rados.0.smithi163.stdout:1275: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:10:37.345 INFO:tasks.rados.rados.0.smithi163.stdout:1278: seq_num 503 ranges {453921=729545,1584325=621164} 2023-10-09T09:10:37.351 INFO:tasks.rados.rados.0.smithi163.stdout:1278: writing smithi16367185-38 from 453921 to 1183466 tid 1 2023-10-09T09:10:37.358 INFO:tasks.rados.rados.0.smithi163.stdout:1278: writing smithi16367185-38 from 1584325 to 2205489 tid 2 2023-10-09T09:10:37.358 INFO:tasks.rados.rados.0.smithi163.stdout:1279: read oid 29 snap -1 2023-10-09T09:10:37.358 INFO:tasks.rados.rados.0.smithi163.stdout:1279: expect (ObjNum 467 snap 151 seq_num 467) 2023-10-09T09:10:37.358 INFO:tasks.rados.rados.0.smithi163.stdout:1280: write oid 17 current snap is 157 2023-10-09T09:10:37.358 INFO:tasks.rados.rados.0.smithi163.stdout:1275: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:10:37.359 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 514 (ObjNum 492 snap 155 seq_num 492) dirty exists 2023-10-09T09:10:37.359 INFO:tasks.rados.rados.0.smithi163.stdout:1274: finishing rollback tid 0 to smithi16367185-42 2023-10-09T09:10:37.359 INFO:tasks.rados.rados.0.smithi163.stdout:1280: seq_num 504 ranges {470728=404794,1526454=481175} 2023-10-09T09:10:37.363 INFO:tasks.rados.rados.0.smithi163.stdout:1280: writing smithi16367185-17 from 470728 to 875522 tid 1 2023-10-09T09:10:37.370 INFO:tasks.rados.rados.0.smithi163.stdout:1280: writing smithi16367185-17 from 1526454 to 2007629 tid 2 2023-10-09T09:10:37.370 INFO:tasks.rados.rados.0.smithi163.stdout:1275: done (6 left) 2023-10-09T09:10:37.370 INFO:tasks.rados.rados.0.smithi163.stdout:1281: write oid 16 current snap is 157 2023-10-09T09:10:37.370 INFO:tasks.rados.rados.0.smithi163.stdout:1274: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:10:37.371 INFO:tasks.rados.rados.0.smithi163.stdout:1276: finishing write tid 1 to smithi16367185-21 2023-10-09T09:10:37.371 INFO:tasks.rados.rados.0.smithi163.stdout:1274: finishing rollback tid 2 to smithi16367185-42 2023-10-09T09:10:37.371 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 1019 (ObjNum 384 snap 118 seq_num 384) dirty exists 2023-10-09T09:10:37.371 INFO:tasks.rados.rados.0.smithi163.stdout:1281: seq_num 505 ranges {487535=480043,1584936=624833} 2023-10-09T09:10:37.376 INFO:tasks.rados.rados.0.smithi163.stdout:1281: writing smithi16367185-16 from 487535 to 967578 tid 1 2023-10-09T09:10:37.383 INFO:tasks.rados.rados.0.smithi163.stdout:1281: writing smithi16367185-16 from 1584936 to 2209769 tid 2 2023-10-09T09:10:37.383 INFO:tasks.rados.rados.0.smithi163.stdout:1274: done (6 left) 2023-10-09T09:10:37.383 INFO:tasks.rados.rados.0.smithi163.stdout:1282: read oid 24 snap 144 2023-10-09T09:10:37.383 INFO:tasks.rados.rados.0.smithi163.stdout:1282: expect deleted 2023-10-09T09:10:37.383 INFO:tasks.rados.rados.0.smithi163.stdout:1283: snap_create 2023-10-09T09:10:37.383 INFO:tasks.rados.rados.0.smithi163.stdout:1276: finishing write tid 2 to smithi16367185-21 2023-10-09T09:10:37.384 INFO:tasks.rados.rados.0.smithi163.stdout:1276: finishing write tid 3 to smithi16367185-21 2023-10-09T09:10:37.384 INFO:tasks.rados.rados.0.smithi163.stdout:1276: finishing write tid 4 to smithi16367185-21 2023-10-09T09:10:37.384 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 315 (ObjNum 502 snap 157 seq_num 502) dirty exists 2023-10-09T09:10:37.384 INFO:tasks.rados.rados.0.smithi163.stdout:1276: left oid 21 (ObjNum 502 snap 157 seq_num 502) 2023-10-09T09:10:37.384 INFO:tasks.rados.rados.0.smithi163.stdout:1278: finishing write tid 1 to smithi16367185-38 2023-10-09T09:10:37.384 INFO:tasks.rados.rados.0.smithi163.stdout:1277: expect (ObjNum 474 snap 152 seq_num 474) 2023-10-09T09:10:37.579 INFO:tasks.rados.rados.0.smithi163.stdout:1278: finishing write tid 2 to smithi16367185-38 2023-10-09T09:10:37.579 INFO:tasks.rados.rados.0.smithi163.stdout:1278: finishing write tid 3 to smithi16367185-38 2023-10-09T09:10:37.580 INFO:tasks.rados.rados.0.smithi163.stdout:1278: finishing write tid 4 to smithi16367185-38 2023-10-09T09:10:37.580 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 531 (ObjNum 503 snap 157 seq_num 503) dirty exists 2023-10-09T09:10:37.580 INFO:tasks.rados.rados.0.smithi163.stdout:1278: left oid 38 (ObjNum 503 snap 157 seq_num 503) 2023-10-09T09:10:37.580 INFO:tasks.rados.rados.0.smithi163.stdout:1279: expect (ObjNum 467 snap 151 seq_num 467) 2023-10-09T09:10:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:37 smithi163 bash[33912]: debug 2023-10-09T09:10:37.321+0000 7f84df46a700 -1 mon.c@2(peon).osd e376 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:37 smithi163 bash[33912]: cluster 2023-10-09T09:10:35.981384+0000 osd.3 (osd.3) 32 : cluster [WRN] failed to encode map e375 with expected crc 2023-10-09T09:10:37.808 INFO:tasks.rados.rados.0.smithi163.stdout:1280: finishing write tid 1 to smithi16367185-17 2023-10-09T09:10:37.808 INFO:tasks.rados.rados.0.smithi163.stdout:1280: finishing write tid 2 to smithi16367185-17 2023-10-09T09:10:37.808 INFO:tasks.rados.rados.0.smithi163.stdout:1280: finishing write tid 3 to smithi16367185-17 2023-10-09T09:10:37.808 INFO:tasks.rados.rados.0.smithi163.stdout:1280: finishing write tid 4 to smithi16367185-17 2023-10-09T09:10:37.808 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 630 (ObjNum 504 snap 157 seq_num 504) dirty exists 2023-10-09T09:10:37.808 INFO:tasks.rados.rados.0.smithi163.stdout:1280: left oid 17 (ObjNum 504 snap 157 seq_num 504) 2023-10-09T09:10:37.808 INFO:tasks.rados.rados.0.smithi163.stdout:1281: finishing write tid 1 to smithi16367185-16 2023-10-09T09:10:37.808 INFO:tasks.rados.rados.0.smithi163.stdout:1281: finishing write tid 2 to smithi16367185-16 2023-10-09T09:10:37.808 INFO:tasks.rados.rados.0.smithi163.stdout:1281: finishing write tid 3 to smithi16367185-16 2023-10-09T09:10:37.808 INFO:tasks.rados.rados.0.smithi163.stdout:1281: finishing write tid 4 to smithi16367185-16 2023-10-09T09:10:37.809 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 765 (ObjNum 505 snap 157 seq_num 505) dirty exists 2023-10-09T09:10:37.809 INFO:tasks.rados.rados.0.smithi163.stdout:1281: left oid 16 (ObjNum 505 snap 157 seq_num 505) 2023-10-09T09:10:37.815 INFO:teuthology.orchestra.run.smithi163.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:10:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:37 smithi181 bash[27712]: debug 2023-10-09T09:10:37.319+0000 7fde83035700 -1 mon.b@1(peon).osd e376 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:37 smithi181 bash[27712]: cluster 2023-10-09T09:10:35.981384+0000 osd.3 (osd.3) 32 : cluster [WRN] failed to encode map e375 with expected crc 2023-10-09T09:10:38.339 INFO:tasks.rados.rados.0.smithi163.stdout:1276: done (7 left) 2023-10-09T09:10:38.339 INFO:tasks.rados.rados.0.smithi163.stdout:1277: done (6 left) 2023-10-09T09:10:38.339 INFO:tasks.rados.rados.0.smithi163.stdout:1278: done (5 left) 2023-10-09T09:10:38.339 INFO:tasks.rados.rados.0.smithi163.stdout:1279: done (4 left) 2023-10-09T09:10:38.339 INFO:tasks.rados.rados.0.smithi163.stdout:1280: done (3 left) 2023-10-09T09:10:38.339 INFO:tasks.rados.rados.0.smithi163.stdout:1281: done (2 left) 2023-10-09T09:10:38.340 INFO:tasks.rados.rados.0.smithi163.stdout:1282: done (1 left) 2023-10-09T09:10:38.340 INFO:tasks.rados.rados.0.smithi163.stdout:1283: done (0 left) 2023-10-09T09:10:38.340 INFO:tasks.rados.rados.0.smithi163.stdout:1284: snap_create 2023-10-09T09:10:38.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T09:10:38.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:38 smithi181 bash[27712]: cluster 2023-10-09T09:10:37.234165+0000 mgr.y (mgr.34107) 52 : cluster [DBG] pgmap v110: 89 pgs: 89 active+clean; 187 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 861 KiB/s wr, 15 op/s 2023-10-09T09:10:38.665 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:38 smithi181 bash[27712]: cluster 2023-10-09T09:10:37.326794+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2023-10-09T09:10:38.665 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:38 smithi181 bash[27712]: cluster 2023-10-09T09:10:37.327375+0000 osd.4 (osd.4) 63 : cluster [WRN] failed to encode map e376 with expected crc 2023-10-09T09:10:38.665 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:38 smithi181 bash[27712]: cluster 2023-10-09T09:10:37.329482+0000 osd.0 (osd.0) 41 : cluster [WRN] failed to encode map e376 with expected crc 2023-10-09T09:10:38.665 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:38 smithi181 bash[27712]: cluster 2023-10-09T09:10:37.331127+0000 osd.5 (osd.5) 32 : cluster [WRN] failed to encode map e376 with expected crc 2023-10-09T09:10:38.665 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:38 smithi181 bash[27712]: audit 2023-10-09T09:10:37.801904+0000 mon.a (mon.0) 152 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:38.665 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:38 smithi181 bash[27712]: audit 2023-10-09T09:10:37.836000+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:10:38.666 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:38 smithi181 bash[27712]: debug 2023-10-09T09:10:38.326+0000 7fde83035700 -1 mon.b@1(peon).osd e377 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:38 smithi163 bash[33912]: cluster 2023-10-09T09:10:37.234165+0000 mgr.y (mgr.34107) 52 : cluster [DBG] pgmap v110: 89 pgs: 89 active+clean; 187 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 861 KiB/s wr, 15 op/s 2023-10-09T09:10:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:38 smithi163 bash[33912]: cluster 2023-10-09T09:10:37.326794+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2023-10-09T09:10:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:38 smithi163 bash[33912]: cluster 2023-10-09T09:10:37.327375+0000 osd.4 (osd.4) 63 : cluster [WRN] failed to encode map e376 with expected crc 2023-10-09T09:10:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:38 smithi163 bash[33912]: cluster 2023-10-09T09:10:37.329482+0000 osd.0 (osd.0) 41 : cluster [WRN] failed to encode map e376 with expected crc 2023-10-09T09:10:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:38 smithi163 bash[33912]: cluster 2023-10-09T09:10:37.331127+0000 osd.5 (osd.5) 32 : cluster [WRN] failed to encode map e376 with expected crc 2023-10-09T09:10:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:38 smithi163 bash[33912]: audit 2023-10-09T09:10:37.801904+0000 mon.a (mon.0) 152 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:38 smithi163 bash[33912]: audit 2023-10-09T09:10:37.836000+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:10:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:38 smithi163 bash[33912]: debug 2023-10-09T09:10:38.329+0000 7f84df46a700 -1 mon.c@2(peon).osd e377 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:39.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:10:38 smithi181 bash[54907]: ts=2023-10-09T09:10:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:10:39.350 INFO:tasks.rados.rados.0.smithi163.stdout:1284: done (0 left) 2023-10-09T09:10:39.350 INFO:tasks.rados.rados.0.smithi163.stdout:1285: read oid 13 snap -1 2023-10-09T09:10:39.350 INFO:tasks.rados.rados.0.smithi163.stdout:1285: expect deleted 2023-10-09T09:10:39.350 INFO:tasks.rados.rados.0.smithi163.stdout:1286: read oid 44 snap -1 2023-10-09T09:10:39.350 INFO:tasks.rados.rados.0.smithi163.stdout:1286: expect (ObjNum 498 snap 156 seq_num 498) 2023-10-09T09:10:39.350 INFO:tasks.rados.rados.0.smithi163.stdout:1287: snap_remove snap 144 2023-10-09T09:10:39.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:39 smithi163 bash[33912]: debug 2023-10-09T09:10:39.337+0000 7f84df46a700 -1 mon.c@2(peon).osd e378 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:39.655 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:39 smithi163 bash[33912]: audit 2023-10-09T09:10:37.794499+0000 mgr.y (mgr.34107) 53 : audit [DBG] from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:10:39.655 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:39 smithi163 bash[33912]: cephadm 2023-10-09T09:10:37.795190+0000 mgr.y (mgr.34107) 54 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:10:39.655 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:39 smithi163 bash[33912]: cluster 2023-10-09T09:10:38.334352+0000 mon.a (mon.0) 154 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2023-10-09T09:10:39.655 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:39 smithi163 bash[33912]: cluster 2023-10-09T09:10:38.335775+0000 osd.4 (osd.4) 64 : cluster [WRN] failed to encode map e377 with expected crc 2023-10-09T09:10:39.655 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:39 smithi163 bash[33912]: cluster 2023-10-09T09:10:38.339764+0000 osd.0 (osd.0) 42 : cluster [WRN] failed to encode map e377 with expected crc 2023-10-09T09:10:39.655 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:39 smithi163 bash[33912]: cluster 2023-10-09T09:10:38.586519+0000 osd.1 (osd.1) 27 : cluster [WRN] failed to encode map e377 with expected crc 2023-10-09T09:10:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:39 smithi181 bash[27712]: debug 2023-10-09T09:10:39.334+0000 7fde83035700 -1 mon.b@1(peon).osd e378 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:39 smithi181 bash[27712]: audit 2023-10-09T09:10:37.794499+0000 mgr.y (mgr.34107) 53 : audit [DBG] from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:10:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:39 smithi181 bash[27712]: cephadm 2023-10-09T09:10:37.795190+0000 mgr.y (mgr.34107) 54 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:10:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:39 smithi181 bash[27712]: cluster 2023-10-09T09:10:38.334352+0000 mon.a (mon.0) 154 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2023-10-09T09:10:39.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:39 smithi181 bash[27712]: cluster 2023-10-09T09:10:38.335775+0000 osd.4 (osd.4) 64 : cluster [WRN] failed to encode map e377 with expected crc 2023-10-09T09:10:39.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:39 smithi181 bash[27712]: cluster 2023-10-09T09:10:38.339764+0000 osd.0 (osd.0) 42 : cluster [WRN] failed to encode map e377 with expected crc 2023-10-09T09:10:39.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:39 smithi181 bash[27712]: cluster 2023-10-09T09:10:38.586519+0000 osd.1 (osd.1) 27 : cluster [WRN] failed to encode map e377 with expected crc 2023-10-09T09:10:39.970 INFO:tasks.rados.rados.0.smithi163.stdout:1287: done (2 left) 2023-10-09T09:10:39.971 INFO:tasks.rados.rados.0.smithi163.stdout:1288: read oid 35 snap -1 2023-10-09T09:10:39.971 INFO:tasks.rados.rados.0.smithi163.stdout:1288: expect (ObjNum 449 snap 144 seq_num 449) 2023-10-09T09:10:39.971 INFO:tasks.rados.rados.0.smithi163.stdout:1289: rollback oid 29 current snap is 159 2023-10-09T09:10:39.971 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 29 to 129 2023-10-09T09:10:39.971 INFO:tasks.rados.rados.0.smithi163.stdout:1286: expect (ObjNum 498 snap 156 seq_num 498) 2023-10-09T09:10:40.119 INFO:tasks.rados.rados.0.smithi163.stdout:1289: finishing rollback tid 0 to smithi16367185-29 2023-10-09T09:10:40.120 INFO:tasks.rados.rados.0.smithi163.stdout:1288: expect (ObjNum 449 snap 144 seq_num 449) 2023-10-09T09:10:40.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:39 smithi163 bash[33912]: debug 2023-10-09T09:10:39.961+0000 7f84df46a700 -1 mon.c@2(peon).osd e379 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:40.240 INFO:tasks.rados.rados.0.smithi163.stdout:1289: finishing rollback tid 1 to smithi16367185-29 2023-10-09T09:10:40.240 INFO:tasks.rados.rados.0.smithi163.stdout:1289: finishing rollback tid 2 to smithi16367185-29 2023-10-09T09:10:40.240 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 530 (ObjNum 335 snap 101 seq_num 335) dirty exists 2023-10-09T09:10:40.240 INFO:tasks.rados.rados.0.smithi163.stdout:1285: done (3 left) 2023-10-09T09:10:40.241 INFO:tasks.rados.rados.0.smithi163.stdout:1286: done (2 left) 2023-10-09T09:10:40.241 INFO:tasks.rados.rados.0.smithi163.stdout:1288: done (1 left) 2023-10-09T09:10:40.241 INFO:tasks.rados.rados.0.smithi163.stdout:1289: done (0 left) 2023-10-09T09:10:40.241 INFO:tasks.rados.rados.0.smithi163.stdout:1290: write oid 30 current snap is 159 2023-10-09T09:10:40.241 INFO:tasks.rados.rados.0.smithi163.stdout:1290: seq_num 506 ranges {504342=555292,1643418=484844} 2023-10-09T09:10:40.252 INFO:tasks.rados.rados.0.smithi163.stdout:1290: writing smithi16367185-30 from 504342 to 1059634 tid 1 2023-10-09T09:10:40.259 INFO:tasks.rados.rados.0.smithi163.stdout:1290: writing smithi16367185-30 from 1643418 to 2128262 tid 2 2023-10-09T09:10:40.259 INFO:tasks.rados.rados.0.smithi163.stdout:1291: write oid 33 current snap is 159 2023-10-09T09:10:40.259 INFO:tasks.rados.rados.0.smithi163.stdout:1290: finishing write tid 1 to smithi16367185-30 2023-10-09T09:10:40.259 INFO:tasks.rados.rados.0.smithi163.stdout:1291: seq_num 507 ranges {521149=630541,1701900=628502} 2023-10-09T09:10:40.265 INFO:tasks.rados.rados.0.smithi163.stdout:1291: writing smithi16367185-33 from 521149 to 1151690 tid 1 2023-10-09T09:10:40.273 INFO:tasks.rados.rados.0.smithi163.stdout:1291: writing smithi16367185-33 from 1701900 to 2330402 tid 2 2023-10-09T09:10:40.273 INFO:tasks.rados.rados.0.smithi163.stdout:1292: write oid 10 current snap is 159 2023-10-09T09:10:40.273 INFO:tasks.rados.rados.0.smithi163.stdout:1292: seq_num 508 ranges {537956=705790,1644029=772160} 2023-10-09T09:10:40.280 INFO:tasks.rados.rados.0.smithi163.stdout:1292: writing smithi16367185-10 from 537956 to 1243746 tid 1 2023-10-09T09:10:40.287 INFO:tasks.rados.rados.0.smithi163.stdout:1292: writing smithi16367185-10 from 1644029 to 2416189 tid 2 2023-10-09T09:10:40.287 INFO:tasks.rados.rados.0.smithi163.stdout:1293: read oid 18 snap -1 2023-10-09T09:10:40.288 INFO:tasks.rados.rados.0.smithi163.stdout:1293: expect (ObjNum 489 snap 154 seq_num 489) 2023-10-09T09:10:40.288 INFO:tasks.rados.rados.0.smithi163.stdout:1294: rollback oid 48 current snap is 159 2023-10-09T09:10:40.288 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 48 to 129 2023-10-09T09:10:40.288 INFO:tasks.rados.rados.0.smithi163.stdout:1290: finishing write tid 2 to smithi16367185-30 2023-10-09T09:10:40.288 INFO:tasks.rados.rados.0.smithi163.stdout:1290: finishing write tid 3 to smithi16367185-30 2023-10-09T09:10:40.288 INFO:tasks.rados.rados.0.smithi163.stdout:1290: finishing write tid 4 to smithi16367185-30 2023-10-09T09:10:40.288 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 486 (ObjNum 506 snap 159 seq_num 506) dirty exists 2023-10-09T09:10:40.288 INFO:tasks.rados.rados.0.smithi163.stdout:1290: left oid 30 (ObjNum 506 snap 159 seq_num 506) 2023-10-09T09:10:40.288 INFO:tasks.rados.rados.0.smithi163.stdout:1291: finishing write tid 1 to smithi16367185-33 2023-10-09T09:10:40.288 INFO:tasks.rados.rados.0.smithi163.stdout:1291: finishing write tid 2 to smithi16367185-33 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:1291: finishing write tid 3 to smithi16367185-33 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:1291: finishing write tid 4 to smithi16367185-33 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 541 (ObjNum 507 snap 159 seq_num 507) dirty exists 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:1291: left oid 33 (ObjNum 507 snap 159 seq_num 507) 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:1292: finishing write tid 1 to smithi16367185-10 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:1290: done (4 left) 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:1291: done (3 left) 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:1295: rollback oid 9 current snap is 159 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 153 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:1296: read oid 36 snap 154 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:1296: expect deleted 2023-10-09T09:10:40.289 INFO:tasks.rados.rados.0.smithi163.stdout:1297: read oid 14 snap -1 2023-10-09T09:10:40.290 INFO:tasks.rados.rados.0.smithi163.stdout:1297: expect (ObjNum 438 snap 136 seq_num 438) 2023-10-09T09:10:40.290 INFO:tasks.rados.rados.0.smithi163.stdout:1298: write oid 45 current snap is 159 2023-10-09T09:10:40.290 INFO:tasks.rados.rados.0.smithi163.stdout:1298: seq_num 509 ranges {554763=781039,2102511=452252} 2023-10-09T09:10:40.295 INFO:tasks.rados.rados.0.smithi163.stdout:1298: writing smithi16367185-45 from 554763 to 1335802 tid 1 2023-10-09T09:10:40.302 INFO:tasks.rados.rados.0.smithi163.stdout:1298: writing smithi16367185-45 from 2102511 to 2554763 tid 2 2023-10-09T09:10:40.302 INFO:tasks.rados.rados.0.smithi163.stdout:1299: read oid 20 snap -1 2023-10-09T09:10:40.303 INFO:tasks.rados.rados.0.smithi163.stdout:1299: expect (ObjNum 490 snap 154 seq_num 490) 2023-10-09T09:10:40.303 INFO:tasks.rados.rados.0.smithi163.stdout:1300: read oid 12 snap 152 2023-10-09T09:10:40.303 INFO:tasks.rados.rados.0.smithi163.stdout:1300: expect (ObjNum 460 snap 147 seq_num 460) 2023-10-09T09:10:40.303 INFO:tasks.rados.rados.0.smithi163.stdout:1301: snap_remove snap 148 2023-10-09T09:10:40.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:39 smithi181 bash[27712]: debug 2023-10-09T09:10:39.950+0000 7fde83035700 -1 mon.b@1(peon).osd e379 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.234915+0000 mgr.y (mgr.34107) 55 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 187 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 850 KiB/s wr, 14 op/s 2023-10-09T09:10:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.343539+0000 osd.4 (osd.4) 65 : cluster [WRN] failed to encode map e378 with expected crc 2023-10-09T09:10:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.346562+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2023-10-09T09:10:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.347961+0000 osd.3 (osd.3) 33 : cluster [WRN] failed to encode map e378 with expected crc 2023-10-09T09:10:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.350492+0000 osd.6 (osd.6) 27 : cluster [WRN] failed to encode map e378 with expected crc 2023-10-09T09:10:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.350982+0000 osd.5 (osd.5) 33 : cluster [WRN] failed to encode map e378 with expected crc 2023-10-09T09:10:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.962360+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2023-10-09T09:10:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.966964+0000 osd.4 (osd.4) 66 : cluster [WRN] failed to encode map e379 with expected crc 2023-10-09T09:10:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.967334+0000 osd.2 (osd.2) 55 : cluster [WRN] failed to encode map e379 with expected crc 2023-10-09T09:10:40.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.972111+0000 osd.2 (osd.2) 56 : cluster [WRN] failed to encode map e379 with expected crc 2023-10-09T09:10:40.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: audit 2023-10-09T09:10:40.331668+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.234915+0000 mgr.y (mgr.34107) 55 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 187 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 850 KiB/s wr, 14 op/s 2023-10-09T09:10:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.343539+0000 osd.4 (osd.4) 65 : cluster [WRN] failed to encode map e378 with expected crc 2023-10-09T09:10:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.346562+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2023-10-09T09:10:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.347961+0000 osd.3 (osd.3) 33 : cluster [WRN] failed to encode map e378 with expected crc 2023-10-09T09:10:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.350492+0000 osd.6 (osd.6) 27 : cluster [WRN] failed to encode map e378 with expected crc 2023-10-09T09:10:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.350982+0000 osd.5 (osd.5) 33 : cluster [WRN] failed to encode map e378 with expected crc 2023-10-09T09:10:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.962360+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2023-10-09T09:10:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.966964+0000 osd.4 (osd.4) 66 : cluster [WRN] failed to encode map e379 with expected crc 2023-10-09T09:10:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.967334+0000 osd.2 (osd.2) 55 : cluster [WRN] failed to encode map e379 with expected crc 2023-10-09T09:10:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.972111+0000 osd.2 (osd.2) 56 : cluster [WRN] failed to encode map e379 with expected crc 2023-10-09T09:10:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: audit 2023-10-09T09:10:40.331668+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:10:40.985 INFO:tasks.rados.rados.0.smithi163.stdout:1301: done (9 left) 2023-10-09T09:10:40.985 INFO:tasks.rados.rados.0.smithi163.stdout:1302: write oid 1 current snap is 159 2023-10-09T09:10:40.985 INFO:tasks.rados.rados.0.smithi163.stdout:1302: seq_num 510 ranges {571570=572641,1877346=694224} 2023-10-09T09:10:41.000 INFO:tasks.rados.rados.0.smithi163.stdout:1302: writing smithi16367185-1 from 571570 to 1144211 tid 1 2023-10-09T09:10:41.010 INFO:tasks.rados.rados.0.smithi163.stdout:1302: writing smithi16367185-1 from 1877346 to 2571570 tid 2 2023-10-09T09:10:41.010 INFO:tasks.rados.rados.0.smithi163.stdout:1303: snap_create 2023-10-09T09:10:41.010 INFO:tasks.rados.rados.0.smithi163.stdout:1295: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:10:41.010 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 0 (ObjNum 699237824 snap 21875 seq_num 1342179536) dirty dne 2023-10-09T09:10:41.010 INFO:tasks.rados.rados.0.smithi163.stdout:1294: finishing rollback tid 0 to smithi16367185-48 2023-10-09T09:10:41.011 INFO:tasks.rados.rados.0.smithi163.stdout:1294: finishing rollback tid 1 to smithi16367185-48 2023-10-09T09:10:41.011 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 1030 (ObjNum 0 snap 21875 seq_num 698955680) dirty dne 2023-10-09T09:10:41.011 INFO:tasks.rados.rados.0.smithi163.stdout:1292: finishing write tid 2 to smithi16367185-10 2023-10-09T09:10:41.011 INFO:tasks.rados.rados.0.smithi163.stdout:1293: expect (ObjNum 489 snap 154 seq_num 489) 2023-10-09T09:10:41.146 INFO:tasks.rados.rados.0.smithi163.stdout:1292: finishing write tid 3 to smithi16367185-10 2023-10-09T09:10:41.146 INFO:tasks.rados.rados.0.smithi163.stdout:1292: finishing write tid 4 to smithi16367185-10 2023-10-09T09:10:41.146 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 490 (ObjNum 508 snap 159 seq_num 508) dirty exists 2023-10-09T09:10:41.146 INFO:tasks.rados.rados.0.smithi163.stdout:1292: left oid 10 (ObjNum 508 snap 159 seq_num 508) 2023-10-09T09:10:41.146 INFO:tasks.rados.rados.0.smithi163.stdout:1297: expect (ObjNum 438 snap 136 seq_num 438) 2023-10-09T09:10:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:40 smithi163 bash[33912]: debug 2023-10-09T09:10:40.973+0000 7f84df46a700 -1 mon.c@2(peon).osd e380 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:41.290 INFO:teuthology.orchestra.run.smithi163.stdout:wait for majority of mons to upgrade 2023-10-09T09:10:41.305 INFO:tasks.rados.rados.0.smithi163.stdout:1298: finishing write tid 1 to smithi16367185-45 2023-10-09T09:10:41.306 INFO:tasks.rados.rados.0.smithi163.stdout:1298: finishing write tid 2 to smithi16367185-45 2023-10-09T09:10:41.306 INFO:tasks.rados.rados.0.smithi163.stdout:1298: finishing write tid 3 to smithi16367185-45 2023-10-09T09:10:41.306 INFO:tasks.rados.rados.0.smithi163.stdout:1298: finishing write tid 4 to smithi16367185-45 2023-10-09T09:10:41.306 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 1033 (ObjNum 509 snap 159 seq_num 509) dirty exists 2023-10-09T09:10:41.306 INFO:tasks.rados.rados.0.smithi163.stdout:1298: left oid 45 (ObjNum 509 snap 159 seq_num 509) 2023-10-09T09:10:41.306 INFO:tasks.rados.rados.0.smithi163.stdout:1300: expect (ObjNum 460 snap 147 seq_num 460) 2023-10-09T09:10:41.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:40 smithi181 bash[27712]: debug 2023-10-09T09:10:40.970+0000 7fde83035700 -1 mon.b@1(peon).osd e380 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:41.576 INFO:tasks.rados.rados.0.smithi163.stdout:1299: expect (ObjNum 490 snap 154 seq_num 490) 2023-10-09T09:10:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:41 smithi163 bash[33912]: cluster 2023-10-09T09:10:39.970587+0000 osd.7 (osd.7) 42 : cluster [WRN] failed to encode map e379 with expected crc 2023-10-09T09:10:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:41 smithi163 bash[33912]: cluster 2023-10-09T09:10:40.981114+0000 osd.4 (osd.4) 67 : cluster [WRN] failed to encode map e380 with expected crc 2023-10-09T09:10:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:41 smithi163 bash[33912]: cluster 2023-10-09T09:10:40.981475+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2023-10-09T09:10:41.749 INFO:tasks.rados.rados.0.smithi163.stdout:1302: finishing write tid 1 to smithi16367185-1 2023-10-09T09:10:41.749 INFO:tasks.rados.rados.0.smithi163.stdout:1302: finishing write tid 2 to smithi16367185-1 2023-10-09T09:10:41.749 INFO:tasks.rados.rados.0.smithi163.stdout:1302: finishing write tid 3 to smithi16367185-1 2023-10-09T09:10:41.749 INFO:tasks.rados.rados.0.smithi163.stdout:1302: finishing write tid 4 to smithi16367185-1 2023-10-09T09:10:41.749 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 787 (ObjNum 510 snap 159 seq_num 510) dirty exists 2023-10-09T09:10:41.749 INFO:tasks.rados.rados.0.smithi163.stdout:1302: left oid 1 (ObjNum 510 snap 159 seq_num 510) 2023-10-09T09:10:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:41 smithi181 bash[27712]: cluster 2023-10-09T09:10:39.970587+0000 osd.7 (osd.7) 42 : cluster [WRN] failed to encode map e379 with expected crc 2023-10-09T09:10:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:41 smithi181 bash[27712]: cluster 2023-10-09T09:10:40.981114+0000 osd.4 (osd.4) 67 : cluster [WRN] failed to encode map e380 with expected crc 2023-10-09T09:10:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:41 smithi181 bash[27712]: cluster 2023-10-09T09:10:40.981475+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2023-10-09T09:10:41.991 INFO:tasks.rados.rados.0.smithi163.stdout:1292: done (10 left) 2023-10-09T09:10:41.991 INFO:tasks.rados.rados.0.smithi163.stdout:1293: done (9 left) 2023-10-09T09:10:41.991 INFO:tasks.rados.rados.0.smithi163.stdout:1294: done (8 left) 2023-10-09T09:10:41.991 INFO:tasks.rados.rados.0.smithi163.stdout:1295: done (7 left) 2023-10-09T09:10:41.991 INFO:tasks.rados.rados.0.smithi163.stdout:1296: done (6 left) 2023-10-09T09:10:41.991 INFO:tasks.rados.rados.0.smithi163.stdout:1297: done (5 left) 2023-10-09T09:10:41.991 INFO:tasks.rados.rados.0.smithi163.stdout:1298: done (4 left) 2023-10-09T09:10:41.991 INFO:tasks.rados.rados.0.smithi163.stdout:1299: done (3 left) 2023-10-09T09:10:41.991 INFO:tasks.rados.rados.0.smithi163.stdout:1300: done (2 left) 2023-10-09T09:10:41.991 INFO:tasks.rados.rados.0.smithi163.stdout:1302: done (1 left) 2023-10-09T09:10:41.992 INFO:tasks.rados.rados.0.smithi163.stdout:1303: done (0 left) 2023-10-09T09:10:41.992 INFO:tasks.rados.rados.0.smithi163.stdout:1304: write oid 25 current snap is 160 2023-10-09T09:10:41.992 INFO:tasks.rados.rados.0.smithi163.stdout:1304: seq_num 511 ranges {588377=647890,1935828=635840} 2023-10-09T09:10:42.001 INFO:tasks.rados.rados.0.smithi163.stdout:1304: writing smithi16367185-25 from 588377 to 1236267 tid 1 2023-10-09T09:10:42.010 INFO:tasks.rados.rados.0.smithi163.stdout:1304: writing smithi16367185-25 from 1935828 to 2571668 tid 2 2023-10-09T09:10:42.010 INFO:tasks.rados.rados.0.smithi163.stdout:1305: read oid 21 snap -1 2023-10-09T09:10:42.010 INFO:tasks.rados.rados.0.smithi163.stdout:1305: expect (ObjNum 502 snap 157 seq_num 502) 2023-10-09T09:10:42.010 INFO:tasks.rados.rados.0.smithi163.stdout:1306: rollback oid 5 current snap is 160 2023-10-09T09:10:42.010 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 140 2023-10-09T09:10:42.010 INFO:tasks.rados.rados.0.smithi163.stdout:1307: write oid 26 current snap is 160 2023-10-09T09:10:42.011 INFO:tasks.rados.rados.0.smithi163.stdout:1307: seq_num 512 ranges {605184=723139,1877957=727227} 2023-10-09T09:10:42.018 INFO:tasks.rados.rados.0.smithi163.stdout:1307: writing smithi16367185-26 from 605184 to 1328323 tid 1 2023-10-09T09:10:42.026 INFO:tasks.rados.rados.0.smithi163.stdout:1307: writing smithi16367185-26 from 1877957 to 2605184 tid 2 2023-10-09T09:10:42.026 INFO:tasks.rados.rados.0.smithi163.stdout:1308: write oid 42 current snap is 160 2023-10-09T09:10:42.026 INFO:tasks.rados.rados.0.smithi163.stdout:1308: seq_num 513 ranges {621991=798388,1936439=639509} 2023-10-09T09:10:42.034 INFO:tasks.rados.rados.0.smithi163.stdout:1308: writing smithi16367185-42 from 621991 to 1420379 tid 1 2023-10-09T09:10:42.041 INFO:tasks.rados.rados.0.smithi163.stdout:1308: writing smithi16367185-42 from 1936439 to 2575948 tid 2 2023-10-09T09:10:42.041 INFO:tasks.rados.rados.0.smithi163.stdout:1309: snap_create 2023-10-09T09:10:42.041 INFO:tasks.rados.rados.0.smithi163.stdout:1304: finishing write tid 1 to smithi16367185-25 2023-10-09T09:10:42.041 INFO:tasks.rados.rados.0.smithi163.stdout:1305: expect (ObjNum 502 snap 157 seq_num 502) 2023-10-09T09:10:42.044 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T09:10:42.125 INFO:tasks.rados.rados.0.smithi163.stdout:1306: finishing rollback tid 0 to smithi16367185-5 2023-10-09T09:10:42.125 INFO:tasks.rados.rados.0.smithi163.stdout:1304: finishing write tid 2 to smithi16367185-25 2023-10-09T09:10:42.125 INFO:tasks.rados.rados.0.smithi163.stdout:1304: finishing write tid 3 to smithi16367185-25 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:1306: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:1304: finishing write tid 4 to smithi16367185-25 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 551 (ObjNum 511 snap 160 seq_num 511) dirty exists 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:1304: left oid 25 (ObjNum 511 snap 160 seq_num 511) 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:1306: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 791 (ObjNum 436 snap 136 seq_num 436) dirty exists 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:1307: finishing write tid 1 to smithi16367185-26 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:1307: finishing write tid 2 to smithi16367185-26 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:1307: finishing write tid 3 to smithi16367185-26 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:1307: finishing write tid 4 to smithi16367185-26 2023-10-09T09:10:42.126 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 442 (ObjNum 512 snap 160 seq_num 512) dirty exists 2023-10-09T09:10:42.127 INFO:tasks.rados.rados.0.smithi163.stdout:1307: left oid 26 (ObjNum 512 snap 160 seq_num 512) 2023-10-09T09:10:42.127 INFO:tasks.rados.rados.0.smithi163.stdout:1308: finishing write tid 1 to smithi16367185-42 2023-10-09T09:10:42.127 INFO:tasks.rados.rados.0.smithi163.stdout:1308: finishing write tid 2 to smithi16367185-42 2023-10-09T09:10:42.127 INFO:tasks.rados.rados.0.smithi163.stdout:1308: finishing write tid 3 to smithi16367185-42 2023-10-09T09:10:42.127 INFO:tasks.rados.rados.0.smithi163.stdout:1308: finishing write tid 4 to smithi16367185-42 2023-10-09T09:10:42.127 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 1045 (ObjNum 513 snap 160 seq_num 513) dirty exists 2023-10-09T09:10:42.127 INFO:tasks.rados.rados.0.smithi163.stdout:1308: left oid 42 (ObjNum 513 snap 160 seq_num 513) 2023-10-09T09:10:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:41 smithi181 bash[27712]: debug 2023-10-09T09:10:41.974+0000 7fde83035700 -1 mon.b@1(peon).osd e381 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:42.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:41 smithi163 bash[33912]: debug 2023-10-09T09:10:41.981+0000 7f84df46a700 -1 mon.c@2(peon).osd e381 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:42 smithi163 bash[33912]: cluster 2023-10-09T09:10:40.980860+0000 osd.0 (osd.0) 43 : cluster [WRN] failed to encode map e380 with expected crc 2023-10-09T09:10:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:42 smithi163 bash[33912]: cluster 2023-10-09T09:10:40.985940+0000 osd.5 (osd.5) 34 : cluster [WRN] failed to encode map e380 with expected crc 2023-10-09T09:10:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:42 smithi163 bash[33912]: cluster 2023-10-09T09:10:41.235653+0000 mgr.y (mgr.34107) 56 : cluster [DBG] pgmap v117: 89 pgs: 89 active+clean; 211 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 3.6 MiB/s wr, 38 op/s 2023-10-09T09:10:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:42 smithi163 bash[33912]: cluster 2023-10-09T09:10:41.987442+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2023-10-09T09:10:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:42 smithi163 bash[33912]: cluster 2023-10-09T09:10:42.003357+0000 osd.2 (osd.2) 57 : cluster [WRN] failed to encode map e381 with expected crc 2023-10-09T09:10:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:42 smithi181 bash[27712]: cluster 2023-10-09T09:10:40.980860+0000 osd.0 (osd.0) 43 : cluster [WRN] failed to encode map e380 with expected crc 2023-10-09T09:10:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:42 smithi181 bash[27712]: cluster 2023-10-09T09:10:40.985940+0000 osd.5 (osd.5) 34 : cluster [WRN] failed to encode map e380 with expected crc 2023-10-09T09:10:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:42 smithi181 bash[27712]: cluster 2023-10-09T09:10:41.235653+0000 mgr.y (mgr.34107) 56 : cluster [DBG] pgmap v117: 89 pgs: 89 active+clean; 211 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 3.6 MiB/s wr, 38 op/s 2023-10-09T09:10:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:42 smithi181 bash[27712]: cluster 2023-10-09T09:10:41.987442+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2023-10-09T09:10:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:42 smithi181 bash[27712]: cluster 2023-10-09T09:10:42.003357+0000 osd.2 (osd.2) 57 : cluster [WRN] failed to encode map e381 with expected crc 2023-10-09T09:10:42.996 INFO:tasks.rados.rados.0.smithi163.stdout:1304: done (5 left) 2023-10-09T09:10:42.996 INFO:tasks.rados.rados.0.smithi163.stdout:1305: done (4 left) 2023-10-09T09:10:42.996 INFO:tasks.rados.rados.0.smithi163.stdout:1306: done (3 left) 2023-10-09T09:10:42.996 INFO:tasks.rados.rados.0.smithi163.stdout:1307: done (2 left) 2023-10-09T09:10:42.997 INFO:tasks.rados.rados.0.smithi163.stdout:1308: done (1 left) 2023-10-09T09:10:42.997 INFO:tasks.rados.rados.0.smithi163.stdout:1309: done (0 left) 2023-10-09T09:10:42.997 INFO:tasks.rados.rados.0.smithi163.stdout:1310: read oid 24 snap -1 2023-10-09T09:10:42.997 INFO:tasks.rados.rados.0.smithi163.stdout:1310: expect (ObjNum 500 snap 156 seq_num 500) 2023-10-09T09:10:42.997 INFO:tasks.rados.rados.0.smithi163.stdout:1311: delete oid 31 current snap is 161 2023-10-09T09:10:43.003 INFO:tasks.rados.rados.0.smithi163.stdout:1311: done (1 left) 2023-10-09T09:10:43.003 INFO:tasks.rados.rados.0.smithi163.stdout:1312: delete oid 19 current snap is 161 2023-10-09T09:10:43.005 INFO:tasks.rados.rados.0.smithi163.stdout:1310: expect (ObjNum 500 snap 156 seq_num 500) 2023-10-09T09:10:43.091 INFO:tasks.rados.rados.0.smithi163.stdout:1310: done (1 left) 2023-10-09T09:10:43.092 INFO:tasks.rados.rados.0.smithi163.stdout:1312: done (0 left) 2023-10-09T09:10:43.092 INFO:tasks.rados.rados.0.smithi163.stdout:1313: snap_remove snap 157 2023-10-09T09:10:43.246 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:42 smithi163 bash[33912]: debug 2023-10-09T09:10:42.981+0000 7f84df46a700 -1 mon.c@2(peon).osd e382 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:43.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:42 smithi181 bash[27712]: debug 2023-10-09T09:10:42.978+0000 7fde83035700 -1 mon.b@1(peon).osd e382 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:10:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:10:43] "GET /metrics HTTP/1.1" 200 36081 "" "Prometheus/2.33.4" 2023-10-09T09:10:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:43 smithi163 bash[33912]: cluster 2023-10-09T09:10:41.986865+0000 osd.6 (osd.6) 28 : cluster [WRN] failed to encode map e381 with expected crc 2023-10-09T09:10:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:43 smithi163 bash[33912]: cluster 2023-10-09T09:10:41.989502+0000 osd.3 (osd.3) 34 : cluster [WRN] failed to encode map e381 with expected crc 2023-10-09T09:10:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:43 smithi163 bash[33912]: cluster 2023-10-09T09:10:42.988480+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2023-10-09T09:10:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:43 smithi163 bash[33912]: cluster 2023-10-09T09:10:42.991595+0000 osd.4 (osd.4) 68 : cluster [WRN] failed to encode map e382 with expected crc 2023-10-09T09:10:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:43 smithi163 bash[33912]: cluster 2023-10-09T09:10:43.005443+0000 osd.2 (osd.2) 58 : cluster [WRN] failed to encode map e382 with expected crc 2023-10-09T09:10:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:43 smithi181 bash[27712]: cluster 2023-10-09T09:10:41.986865+0000 osd.6 (osd.6) 28 : cluster [WRN] failed to encode map e381 with expected crc 2023-10-09T09:10:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:43 smithi181 bash[27712]: cluster 2023-10-09T09:10:41.989502+0000 osd.3 (osd.3) 34 : cluster [WRN] failed to encode map e381 with expected crc 2023-10-09T09:10:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:43 smithi181 bash[27712]: cluster 2023-10-09T09:10:42.988480+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2023-10-09T09:10:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:43 smithi181 bash[27712]: cluster 2023-10-09T09:10:42.991595+0000 osd.4 (osd.4) 68 : cluster [WRN] failed to encode map e382 with expected crc 2023-10-09T09:10:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:43 smithi181 bash[27712]: cluster 2023-10-09T09:10:43.005443+0000 osd.2 (osd.2) 58 : cluster [WRN] failed to encode map e382 with expected crc 2023-10-09T09:10:44.009 INFO:tasks.rados.rados.0.smithi163.stdout:1313: done (0 left) 2023-10-09T09:10:44.009 INFO:tasks.rados.rados.0.smithi163.stdout:1314: snap_create 2023-10-09T09:10:44.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:43 smithi163 bash[33912]: debug 2023-10-09T09:10:43.993+0000 7f84df46a700 -1 mon.c@2(peon).osd e383 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:44.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:43 smithi181 bash[27712]: debug 2023-10-09T09:10:43.986+0000 7fde83035700 -1 mon.b@1(peon).osd e383 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:44.688 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: cluster 2023-10-09T09:10:42.989847+0000 osd.1 (osd.1) 28 : cluster [WRN] failed to encode map e382 with expected crc 2023-10-09T09:10:44.688 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: cluster 2023-10-09T09:10:42.997797+0000 osd.0 (osd.0) 44 : cluster [WRN] failed to encode map e382 with expected crc 2023-10-09T09:10:44.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: cluster 2023-10-09T09:10:42.997797+0000 osd.3 (osd.3) 35 : cluster [WRN] failed to encode map e382 with expected crc 2023-10-09T09:10:44.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: cluster 2023-10-09T09:10:43.236045+0000 mgr.y (mgr.34107) 57 : cluster [DBG] pgmap v120: 89 pgs: 89 active+clean; 211 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 3.7 MiB/s wr, 38 op/s 2023-10-09T09:10:44.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: audit 2023-10-09T09:10:43.789501+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:44.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: audit 2023-10-09T09:10:43.796613+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:44.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.003051+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2023-10-09T09:10:44.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.003950+0000 osd.4 (osd.4) 69 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:44.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: audit 2023-10-09T09:10:44.144046+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:44.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: audit 2023-10-09T09:10:44.152717+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: cluster 2023-10-09T09:10:42.989847+0000 osd.1 (osd.1) 28 : cluster [WRN] failed to encode map e382 with expected crc 2023-10-09T09:10:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: cluster 2023-10-09T09:10:42.997797+0000 osd.0 (osd.0) 44 : cluster [WRN] failed to encode map e382 with expected crc 2023-10-09T09:10:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: cluster 2023-10-09T09:10:42.997797+0000 osd.3 (osd.3) 35 : cluster [WRN] failed to encode map e382 with expected crc 2023-10-09T09:10:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: cluster 2023-10-09T09:10:43.236045+0000 mgr.y (mgr.34107) 57 : cluster [DBG] pgmap v120: 89 pgs: 89 active+clean; 211 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 3.7 MiB/s wr, 38 op/s 2023-10-09T09:10:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: audit 2023-10-09T09:10:43.789501+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: audit 2023-10-09T09:10:43.796613+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.003051+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2023-10-09T09:10:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.003950+0000 osd.4 (osd.4) 69 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: audit 2023-10-09T09:10:44.144046+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: audit 2023-10-09T09:10:44.152717+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:44.973 INFO:tasks.rados.rados.0.smithi163.stdout:1314: done (0 left) 2023-10-09T09:10:44.973 INFO:tasks.rados.rados.0.smithi163.stdout:1315: read oid 32 snap -1 2023-10-09T09:10:44.973 INFO:tasks.rados.rados.0.smithi163.stdout:1315: expect (ObjNum 422 snap 129 seq_num 422) 2023-10-09T09:10:44.973 INFO:tasks.rados.rados.0.smithi163.stdout:1316: rollback oid 49 current snap is 162 2023-10-09T09:10:44.973 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 49 to 159 2023-10-09T09:10:44.973 INFO:tasks.rados.rados.0.smithi163.stdout:1317: delete oid 50 current snap is 162 2023-10-09T09:10:44.986 INFO:tasks.rados.rados.0.smithi163.stdout:1316: finishing rollback tid 0 to smithi16367185-49 2023-10-09T09:10:44.988 INFO:tasks.rados.rados.0.smithi163.stdout:1316: finishing rollback tid 1 to smithi16367185-49 2023-10-09T09:10:44.989 INFO:tasks.rados.rados.0.smithi163.stdout:1316: finishing rollback tid 2 to smithi16367185-49 2023-10-09T09:10:44.989 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 216 (ObjNum 448 snap 144 seq_num 448) dirty exists 2023-10-09T09:10:44.989 INFO:tasks.rados.rados.0.smithi163.stdout:1316: done (2 left) 2023-10-09T09:10:44.989 INFO:tasks.rados.rados.0.smithi163.stdout:1317: done (1 left) 2023-10-09T09:10:44.990 INFO:tasks.rados.rados.0.smithi163.stdout:1318: read oid 46 snap 152 2023-10-09T09:10:44.990 INFO:tasks.rados.rados.0.smithi163.stdout:1318: expect (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:10:44.990 INFO:tasks.rados.rados.0.smithi163.stdout:1319: delete oid 28 current snap is 162 2023-10-09T09:10:44.992 INFO:tasks.rados.rados.0.smithi163.stdout:1315: expect (ObjNum 422 snap 129 seq_num 422) 2023-10-09T09:10:45.099 INFO:tasks.rados.rados.0.smithi163.stdout:1318: expect (ObjNum 305 snap 92 seq_num 305) 2023-10-09T09:10:45.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:44 smithi163 bash[33912]: debug 2023-10-09T09:10:44.957+0000 7f84df46a700 -1 mon.c@2(peon).osd e384 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:45.285 INFO:tasks.rados.rados.0.smithi163.stdout:1315: done (2 left) 2023-10-09T09:10:45.285 INFO:tasks.rados.rados.0.smithi163.stdout:1318: done (1 left) 2023-10-09T09:10:45.285 INFO:tasks.rados.rados.0.smithi163.stdout:1319: done (0 left) 2023-10-09T09:10:45.285 INFO:tasks.rados.rados.0.smithi163.stdout:1320: read oid 3 snap 161 2023-10-09T09:10:45.285 INFO:tasks.rados.rados.0.smithi163.stdout:1320: expect deleted 2023-10-09T09:10:45.286 INFO:tasks.rados.rados.0.smithi163.stdout:1321: rollback oid 19 current snap is 162 2023-10-09T09:10:45.286 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 19 to 161 2023-10-09T09:10:45.286 INFO:tasks.rados.rados.0.smithi163.stdout:1322: write oid 23 current snap is 162 2023-10-09T09:10:45.286 INFO:tasks.rados.rados.0.smithi163.stdout:1322: seq_num 518 ranges {706026=490986,1828849=790505} 2023-10-09T09:10:45.291 INFO:tasks.rados.rados.0.smithi163.stdout:1322: writing smithi16367185-23 from 706026 to 1197012 tid 1 2023-10-09T09:10:45.300 INFO:tasks.rados.rados.0.smithi163.stdout:1322: writing smithi16367185-23 from 1828849 to 2619354 tid 2 2023-10-09T09:10:45.300 INFO:tasks.rados.rados.0.smithi163.stdout:1323: snap_create 2023-10-09T09:10:45.300 INFO:tasks.rados.rados.0.smithi163.stdout:1321: finishing rollback tid 1 to smithi16367185-19 2023-10-09T09:10:45.300 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 444 (ObjNum 698963952 snap 21875 seq_num 1618308288) dirty dne 2023-10-09T09:10:45.300 INFO:tasks.rados.rados.0.smithi163.stdout:1322: finishing write tid 1 to smithi16367185-23 2023-10-09T09:10:45.305 INFO:tasks.rados.rados.0.smithi163.stdout:1322: finishing write tid 2 to smithi16367185-23 2023-10-09T09:10:45.305 INFO:tasks.rados.rados.0.smithi163.stdout:1322: finishing write tid 3 to smithi16367185-23 2023-10-09T09:10:45.305 INFO:tasks.rados.rados.0.smithi163.stdout:1322: finishing write tid 4 to smithi16367185-23 2023-10-09T09:10:45.305 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 527 (ObjNum 518 snap 162 seq_num 518) dirty exists 2023-10-09T09:10:45.306 INFO:tasks.rados.rados.0.smithi163.stdout:1322: left oid 23 (ObjNum 518 snap 162 seq_num 518) 2023-10-09T09:10:45.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:44 smithi181 bash[27712]: debug 2023-10-09T09:10:44.954+0000 7fde83035700 -1 mon.b@1(peon).osd e384 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.003232+0000 osd.1 (osd.1) 29 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.385581+0000 osd.3 (osd.3) 36 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.477905+0000 osd.0 (osd.0) 45 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.635585+0000 osd.2 (osd.2) 59 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: audit 2023-10-09T09:10:44.741405+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:10:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: audit 2023-10-09T09:10:44.743142+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:10:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: audit 2023-10-09T09:10:44.752771+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: audit 2023-10-09T09:10:44.835436+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:10:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: audit 2023-10-09T09:10:44.839690+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:10:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: audit 2023-10-09T09:10:44.841917+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:10:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: audit 2023-10-09T09:10:44.843043+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T09:10:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.969028+0000 osd.4 (osd.4) 70 : cluster [WRN] failed to encode map e384 with expected crc 2023-10-09T09:10:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.969311+0000 mon.a (mon.0) 173 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2023-10-09T09:10:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.991153+0000 osd.2 (osd.2) 60 : cluster [WRN] failed to encode map e384 with expected crc 2023-10-09T09:10:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: audit 2023-10-09T09:10:45.263947+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.163:0/3809846710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:10:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.003232+0000 osd.1 (osd.1) 29 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.385581+0000 osd.3 (osd.3) 36 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.477905+0000 osd.0 (osd.0) 45 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.635585+0000 osd.2 (osd.2) 59 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: audit 2023-10-09T09:10:44.741405+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: audit 2023-10-09T09:10:44.743142+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: audit 2023-10-09T09:10:44.752771+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: audit 2023-10-09T09:10:44.835436+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: audit 2023-10-09T09:10:44.839690+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: audit 2023-10-09T09:10:44.841917+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: audit 2023-10-09T09:10:44.843043+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.969028+0000 osd.4 (osd.4) 70 : cluster [WRN] failed to encode map e384 with expected crc 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.969311+0000 mon.a (mon.0) 173 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2023-10-09T09:10:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.991153+0000 osd.2 (osd.2) 60 : cluster [WRN] failed to encode map e384 with expected crc 2023-10-09T09:10:45.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: audit 2023-10-09T09:10:45.263947+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.163:0/3809846710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:10:45.989 INFO:tasks.rados.rados.0.smithi163.stdout:1320: done (3 left) 2023-10-09T09:10:45.989 INFO:tasks.rados.rados.0.smithi163.stdout:1321: done (2 left) 2023-10-09T09:10:45.989 INFO:tasks.rados.rados.0.smithi163.stdout:1322: done (1 left) 2023-10-09T09:10:45.989 INFO:tasks.rados.rados.0.smithi163.stdout:1323: done (0 left) 2023-10-09T09:10:45.990 INFO:tasks.rados.rados.0.smithi163.stdout:1324: delete oid 41 current snap is 163 2023-10-09T09:10:45.997 INFO:tasks.rados.rados.0.smithi163.stdout:1324: done (0 left) 2023-10-09T09:10:45.997 INFO:tasks.rados.rados.0.smithi163.stdout:1325: delete oid 45 current snap is 163 2023-10-09T09:10:46.004 INFO:tasks.rados.rados.0.smithi163.stdout:1325: done (0 left) 2023-10-09T09:10:46.004 INFO:tasks.rados.rados.0.smithi163.stdout:1326: snap_create 2023-10-09T09:10:46.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:45 smithi163 bash[33912]: debug 2023-10-09T09:10:45.981+0000 7f84df46a700 -1 mon.c@2(peon).osd e385 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:46.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:45 smithi181 bash[27712]: debug 2023-10-09T09:10:45.974+0000 7fde83035700 -1 mon.b@1(peon).osd e385 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:46.704 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.392516+0000 osd.7 (osd.7) 43 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:46.704 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: cephadm 2023-10-09T09:10:44.843839+0000 mgr.y (mgr.34107) 58 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T09:10:46.704 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.971510+0000 osd.0 (osd.0) 46 : cluster [WRN] failed to encode map e384 with expected crc 2023-10-09T09:10:46.704 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.976559+0000 osd.5 (osd.5) 35 : cluster [WRN] failed to encode map e384 with expected crc 2023-10-09T09:10:46.704 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: cluster 2023-10-09T09:10:44.977490+0000 osd.0 (osd.0) 47 : cluster [WRN] failed to encode map e384 with expected crc 2023-10-09T09:10:46.705 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: cluster 2023-10-09T09:10:45.236615+0000 mgr.y (mgr.34107) 59 : cluster [DBG] pgmap v123: 89 pgs: 89 active+clean; 211 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:10:46.705 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: cluster 2023-10-09T09:10:45.986295+0000 osd.4 (osd.4) 71 : cluster [WRN] failed to encode map e385 with expected crc 2023-10-09T09:10:46.705 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: cluster 2023-10-09T09:10:45.988633+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2023-10-09T09:10:46.705 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: cluster 2023-10-09T09:10:45.991770+0000 osd.2 (osd.2) 61 : cluster [WRN] failed to encode map e385 with expected crc 2023-10-09T09:10:46.705 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: cluster 2023-10-09T09:10:45.994396+0000 osd.2 (osd.2) 62 : cluster [WRN] failed to encode map e385 with expected crc 2023-10-09T09:10:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.392516+0000 osd.7 (osd.7) 43 : cluster [WRN] failed to encode map e383 with expected crc 2023-10-09T09:10:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: cephadm 2023-10-09T09:10:44.843839+0000 mgr.y (mgr.34107) 58 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T09:10:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.971510+0000 osd.0 (osd.0) 46 : cluster [WRN] failed to encode map e384 with expected crc 2023-10-09T09:10:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.976559+0000 osd.5 (osd.5) 35 : cluster [WRN] failed to encode map e384 with expected crc 2023-10-09T09:10:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: cluster 2023-10-09T09:10:44.977490+0000 osd.0 (osd.0) 47 : cluster [WRN] failed to encode map e384 with expected crc 2023-10-09T09:10:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: cluster 2023-10-09T09:10:45.236615+0000 mgr.y (mgr.34107) 59 : cluster [DBG] pgmap v123: 89 pgs: 89 active+clean; 211 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:10:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: cluster 2023-10-09T09:10:45.986295+0000 osd.4 (osd.4) 71 : cluster [WRN] failed to encode map e385 with expected crc 2023-10-09T09:10:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: cluster 2023-10-09T09:10:45.988633+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2023-10-09T09:10:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: cluster 2023-10-09T09:10:45.991770+0000 osd.2 (osd.2) 61 : cluster [WRN] failed to encode map e385 with expected crc 2023-10-09T09:10:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: cluster 2023-10-09T09:10:45.994396+0000 osd.2 (osd.2) 62 : cluster [WRN] failed to encode map e385 with expected crc 2023-10-09T09:10:46.997 INFO:tasks.rados.rados.0.smithi163.stdout:1326: done (0 left) 2023-10-09T09:10:46.997 INFO:tasks.rados.rados.0.smithi163.stdout:1327: write oid 34 current snap is 164 2023-10-09T09:10:46.997 INFO:tasks.rados.rados.0.smithi163.stdout:1327: seq_num 521 ranges {756447=716733,1887942=537832} 2023-10-09T09:10:47.005 INFO:tasks.rados.rados.0.smithi163.stdout:1327: writing smithi16367185-34 from 756447 to 1473180 tid 1 2023-10-09T09:10:47.010 INFO:tasks.rados.rados.0.smithi163.stdout:1327: writing smithi16367185-34 from 1887942 to 2425774 tid 2 2023-10-09T09:10:47.011 INFO:tasks.rados.rados.0.smithi163.stdout:1328: read oid 50 snap -1 2023-10-09T09:10:47.011 INFO:tasks.rados.rados.0.smithi163.stdout:1328: expect deleted 2023-10-09T09:10:47.011 INFO:tasks.rados.rados.0.smithi163.stdout:1329: read oid 19 snap -1 2023-10-09T09:10:47.011 INFO:tasks.rados.rados.0.smithi163.stdout:1329: expect deleted 2023-10-09T09:10:47.011 INFO:tasks.rados.rados.0.smithi163.stdout:1330: snap_remove snap 159 2023-10-09T09:10:47.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:46 smithi181 bash[27712]: debug 2023-10-09T09:10:46.978+0000 7fde83035700 -1 mon.b@1(peon).osd e386 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:47.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:46 smithi163 bash[33912]: debug 2023-10-09T09:10:46.981+0000 7f84df46a700 -1 mon.c@2(peon).osd e386 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:48.003 INFO:tasks.rados.rados.0.smithi163.stdout:1330: done (3 left) 2023-10-09T09:10:48.003 INFO:tasks.rados.rados.0.smithi163.stdout:1331: write oid 8 current snap is 164 2023-10-09T09:10:48.003 INFO:tasks.rados.rados.0.smithi163.stdout:1331: seq_num 522 ranges {773254=791982,2346424=426830} 2023-10-09T09:10:48.013 INFO:tasks.rados.rados.0.smithi163.stdout:1331: writing smithi16367185-8 from 773254 to 1565236 tid 1 2023-10-09T09:10:48.021 INFO:tasks.rados.rados.0.smithi163.stdout:1331: writing smithi16367185-8 from 2346424 to 2773254 tid 2 2023-10-09T09:10:48.021 INFO:tasks.rados.rados.0.smithi163.stdout:1332: read oid 1 snap -1 2023-10-09T09:10:48.021 INFO:tasks.rados.rados.0.smithi163.stdout:1332: expect (ObjNum 510 snap 159 seq_num 510) 2023-10-09T09:10:48.021 INFO:tasks.rados.rados.0.smithi163.stdout:1333: read oid 43 snap -1 2023-10-09T09:10:48.021 INFO:tasks.rados.rados.0.smithi163.stdout:1333: expect (ObjNum 482 snap 152 seq_num 482) 2023-10-09T09:10:48.022 INFO:tasks.rados.rados.0.smithi163.stdout:1327: finishing write tid 1 to smithi16367185-34 2023-10-09T09:10:48.022 INFO:tasks.rados.rados.0.smithi163.stdout:1327: finishing write tid 2 to smithi16367185-34 2023-10-09T09:10:48.022 INFO:tasks.rados.rados.0.smithi163.stdout:1327: finishing write tid 3 to smithi16367185-34 2023-10-09T09:10:48.022 INFO:tasks.rados.rados.0.smithi163.stdout:1327: finishing write tid 4 to smithi16367185-34 2023-10-09T09:10:48.022 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 530 (ObjNum 521 snap 164 seq_num 521) dirty exists 2023-10-09T09:10:48.022 INFO:tasks.rados.rados.0.smithi163.stdout:1327: left oid 34 (ObjNum 521 snap 164 seq_num 521) 2023-10-09T09:10:48.022 INFO:tasks.rados.rados.0.smithi163.stdout:1327: done (5 left) 2023-10-09T09:10:48.022 INFO:tasks.rados.rados.0.smithi163.stdout:1328: done (4 left) 2023-10-09T09:10:48.022 INFO:tasks.rados.rados.0.smithi163.stdout:1329: done (3 left) 2023-10-09T09:10:48.022 INFO:tasks.rados.rados.0.smithi163.stdout:1334: read oid 20 snap -1 2023-10-09T09:10:48.023 INFO:tasks.rados.rados.0.smithi163.stdout:1334: expect (ObjNum 490 snap 154 seq_num 490) 2023-10-09T09:10:48.023 INFO:tasks.rados.rados.0.smithi163.stdout:1335: snap_create 2023-10-09T09:10:48.024 INFO:tasks.rados.rados.0.smithi163.stdout:1331: finishing write tid 1 to smithi16367185-8 2023-10-09T09:10:48.025 INFO:tasks.rados.rados.0.smithi163.stdout:1331: finishing write tid 2 to smithi16367185-8 2023-10-09T09:10:48.028 INFO:tasks.rados.rados.0.smithi163.stdout:1331: finishing write tid 3 to smithi16367185-8 2023-10-09T09:10:48.028 INFO:tasks.rados.rados.0.smithi163.stdout:1331: finishing write tid 4 to smithi16367185-8 2023-10-09T09:10:48.028 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 642 (ObjNum 522 snap 164 seq_num 522) dirty exists 2023-10-09T09:10:48.028 INFO:tasks.rados.rados.0.smithi163.stdout:1331: left oid 8 (ObjNum 522 snap 164 seq_num 522) 2023-10-09T09:10:48.033 INFO:tasks.rados.rados.0.smithi163.stdout:1332: expect (ObjNum 510 snap 159 seq_num 510) 2023-10-09T09:10:48.263 INFO:tasks.rados.rados.0.smithi163.stdout:1334: expect (ObjNum 490 snap 154 seq_num 490) 2023-10-09T09:10:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: cluster 2023-10-09T09:10:46.989123+0000 mon.a (mon.0) 176 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2023-10-09T09:10:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: cluster 2023-10-09T09:10:46.991099+0000 osd.3 (osd.3) 37 : cluster [WRN] failed to encode map e386 with expected crc 2023-10-09T09:10:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: cluster 2023-10-09T09:10:46.993659+0000 osd.4 (osd.4) 72 : cluster [WRN] failed to encode map e386 with expected crc 2023-10-09T09:10:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: cluster 2023-10-09T09:10:47.006573+0000 osd.5 (osd.5) 36 : cluster [WRN] failed to encode map e386 with expected crc 2023-10-09T09:10:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: cluster 2023-10-09T09:10:47.013022+0000 osd.2 (osd.2) 63 : cluster [WRN] failed to encode map e386 with expected crc 2023-10-09T09:10:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: cephadm 2023-10-09T09:10:47.213712+0000 mgr.y (mgr.34107) 60 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T09:10:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: audit 2023-10-09T09:10:47.222923+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: audit 2023-10-09T09:10:47.224846+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:10:48.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: audit 2023-10-09T09:10:47.226004+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:10:48.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: audit 2023-10-09T09:10:47.227129+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:10:48.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: cephadm 2023-10-09T09:10:47.228598+0000 mgr.y (mgr.34107) 61 : cephadm [INF] Deploying daemon mon.c on smithi163 2023-10-09T09:10:48.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: cluster 2023-10-09T09:10:47.237258+0000 mgr.y (mgr.34107) 62 : cluster [DBG] pgmap v126: 89 pgs: 89 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2023-10-09T09:10:48.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: audit 2023-10-09T09:10:47.721811+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.163:0/2236837363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:10:48.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:47 smithi163 bash[33912]: debug 2023-10-09T09:10:47.993+0000 7f84df46a700 -1 mon.c@2(peon).osd e387 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: cluster 2023-10-09T09:10:46.989123+0000 mon.a (mon.0) 176 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2023-10-09T09:10:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: cluster 2023-10-09T09:10:46.991099+0000 osd.3 (osd.3) 37 : cluster [WRN] failed to encode map e386 with expected crc 2023-10-09T09:10:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: cluster 2023-10-09T09:10:46.993659+0000 osd.4 (osd.4) 72 : cluster [WRN] failed to encode map e386 with expected crc 2023-10-09T09:10:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: cluster 2023-10-09T09:10:47.006573+0000 osd.5 (osd.5) 36 : cluster [WRN] failed to encode map e386 with expected crc 2023-10-09T09:10:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: cluster 2023-10-09T09:10:47.013022+0000 osd.2 (osd.2) 63 : cluster [WRN] failed to encode map e386 with expected crc 2023-10-09T09:10:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: cephadm 2023-10-09T09:10:47.213712+0000 mgr.y (mgr.34107) 60 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T09:10:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: audit 2023-10-09T09:10:47.222923+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:10:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: audit 2023-10-09T09:10:47.224846+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:10:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: audit 2023-10-09T09:10:47.226004+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:10:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: audit 2023-10-09T09:10:47.227129+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:10:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: cephadm 2023-10-09T09:10:47.228598+0000 mgr.y (mgr.34107) 61 : cephadm [INF] Deploying daemon mon.c on smithi163 2023-10-09T09:10:48.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: cluster 2023-10-09T09:10:47.237258+0000 mgr.y (mgr.34107) 62 : cluster [DBG] pgmap v126: 89 pgs: 89 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2023-10-09T09:10:48.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: audit 2023-10-09T09:10:47.721811+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.163:0/2236837363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:10:48.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:47 smithi181 bash[27712]: debug 2023-10-09T09:10:47.990+0000 7fde83035700 -1 mon.b@1(peon).osd e387 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:48.443 INFO:tasks.rados.rados.0.smithi163.stdout:1333: expect (ObjNum 482 snap 152 seq_num 482) 2023-10-09T09:10:48.999 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:10:48 smithi181 bash[54907]: ts=2023-10-09T09:10:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:10:49.016 INFO:tasks.rados.rados.0.smithi163.stdout:1331: done (4 left) 2023-10-09T09:10:49.017 INFO:tasks.rados.rados.0.smithi163.stdout:1332: done (3 left) 2023-10-09T09:10:49.017 INFO:tasks.rados.rados.0.smithi163.stdout:1333: done (2 left) 2023-10-09T09:10:49.017 INFO:tasks.rados.rados.0.smithi163.stdout:1334: done (1 left) 2023-10-09T09:10:49.017 INFO:tasks.rados.rados.0.smithi163.stdout:1335: done (0 left) 2023-10-09T09:10:49.017 INFO:tasks.rados.rados.0.smithi163.stdout:1336: write oid 19 current snap is 165 2023-10-09T09:10:49.017 INFO:tasks.rados.rados.0.smithi163.stdout:1336: seq_num 523 ranges {790061=467231,2004906=541501} 2023-10-09T09:10:49.026 INFO:tasks.rados.rados.0.smithi163.stdout:1336: writing smithi16367185-19 from 790061 to 1257292 tid 1 2023-10-09T09:10:49.035 INFO:tasks.rados.rados.0.smithi163.stdout:1336: writing smithi16367185-19 from 2004906 to 2546407 tid 2 2023-10-09T09:10:49.035 INFO:tasks.rados.rados.0.smithi163.stdout:1337: rollback oid 15 current snap is 165 2023-10-09T09:10:49.035 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 15 to 140 2023-10-09T09:10:49.035 INFO:tasks.rados.rados.0.smithi163.stdout:1336: finishing write tid 1 to smithi16367185-19 2023-10-09T09:10:49.035 INFO:tasks.rados.rados.0.smithi163.stdout:1338: read oid 38 snap -1 2023-10-09T09:10:49.035 INFO:tasks.rados.rados.0.smithi163.stdout:1338: expect (ObjNum 503 snap 157 seq_num 503) 2023-10-09T09:10:49.035 INFO:tasks.rados.rados.0.smithi163.stdout:1339: read oid 2 snap -1 2023-10-09T09:10:49.035 INFO:tasks.rados.rados.0.smithi163.stdout:1339: expect deleted 2023-10-09T09:10:49.035 INFO:tasks.rados.rados.0.smithi163.stdout:1340: delete oid 5 current snap is 165 2023-10-09T09:10:49.041 INFO:tasks.rados.rados.0.smithi163.stdout:1336: finishing write tid 2 to smithi16367185-19 2023-10-09T09:10:49.041 INFO:tasks.rados.rados.0.smithi163.stdout:1339: done (4 left) 2023-10-09T09:10:49.041 INFO:tasks.rados.rados.0.smithi163.stdout:1340: done (3 left) 2023-10-09T09:10:49.041 INFO:tasks.rados.rados.0.smithi163.stdout:1341: read oid 8 snap 158 2023-10-09T09:10:49.041 INFO:tasks.rados.rados.0.smithi163.stdout:1341: expect (ObjNum 471 snap 152 seq_num 471) 2023-10-09T09:10:49.041 INFO:tasks.rados.rados.0.smithi163.stdout:1342: read oid 29 snap -1 2023-10-09T09:10:49.041 INFO:tasks.rados.rados.0.smithi163.stdout:1342: expect (ObjNum 335 snap 101 seq_num 335) 2023-10-09T09:10:49.041 INFO:tasks.rados.rados.0.smithi163.stdout:1343: rollback oid 18 current snap is 165 2023-10-09T09:10:49.041 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 18 to 156 2023-10-09T09:10:49.042 INFO:tasks.rados.rados.0.smithi163.stdout:1344: rollback oid 33 current snap is 165 2023-10-09T09:10:49.042 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 161 2023-10-09T09:10:49.042 INFO:tasks.rados.rados.0.smithi163.stdout:1345: snap_remove snap 155 2023-10-09T09:10:49.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:48 smithi181 bash[27712]: cluster 2023-10-09T09:10:47.998630+0000 mon.a (mon.0) 181 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2023-10-09T09:10:49.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:48 smithi181 bash[27712]: cluster 2023-10-09T09:10:48.000083+0000 osd.4 (osd.4) 73 : cluster [WRN] failed to encode map e387 with expected crc 2023-10-09T09:10:49.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:48 smithi181 bash[27712]: cluster 2023-10-09T09:10:48.000662+0000 osd.0 (osd.0) 48 : cluster [WRN] failed to encode map e387 with expected crc 2023-10-09T09:10:49.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:48 smithi181 bash[27712]: cluster 2023-10-09T09:10:48.014648+0000 osd.7 (osd.7) 44 : cluster [WRN] failed to encode map e387 with expected crc 2023-10-09T09:10:49.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:49 smithi181 bash[27712]: debug 2023-10-09T09:10:49.002+0000 7fde83035700 -1 mon.b@1(peon).osd e388 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:49.368 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:49 smithi163 bash[33912]: cluster 2023-10-09T09:10:47.998630+0000 mon.a (mon.0) 181 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2023-10-09T09:10:49.368 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:49 smithi163 bash[33912]: cluster 2023-10-09T09:10:48.000083+0000 osd.4 (osd.4) 73 : cluster [WRN] failed to encode map e387 with expected crc 2023-10-09T09:10:49.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:49 smithi163 bash[33912]: cluster 2023-10-09T09:10:48.000662+0000 osd.0 (osd.0) 48 : cluster [WRN] failed to encode map e387 with expected crc 2023-10-09T09:10:49.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:49 smithi163 bash[33912]: cluster 2023-10-09T09:10:48.014648+0000 osd.7 (osd.7) 44 : cluster [WRN] failed to encode map e387 with expected crc 2023-10-09T09:10:49.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:49 smithi163 bash[33912]: debug 2023-10-09T09:10:49.005+0000 7f84df46a700 -1 mon.c@2(peon).osd e388 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:49.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:49 smithi163 systemd[1]: Stopping Ceph mon.c for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:10:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:49 smithi163 bash[33912]: debug 2023-10-09T09:10:49.509+0000 7f84e5c77700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T09:10:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:49 smithi163 bash[33912]: debug 2023-10-09T09:10:49.509+0000 7f84e5c77700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T09:10:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:50 smithi163 bash[77962]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T09:10:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:50 smithi163 bash[78053]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T09:10:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:50 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T09:10:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:50 smithi163 systemd[1]: Stopped Ceph mon.c for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:10:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:50 smithi163 systemd[1]: Started Ceph mon.c for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:10:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.405+0000 7fe5da2a0c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T09:10:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.405+0000 7fe5da2a0c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T09:10:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.405+0000 7fe5da2a0c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T09:10:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 0 load: jerasure load: lrc 2023-10-09T09:10:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T09:10:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Git sha 0 2023-10-09T09:10:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T09:10:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: DB SUMMARY 2023-10-09T09:10:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: DB Session ID: E2RYKMBDC36T6QT8X0GQ 2023-10-09T09:10:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T09:10:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1271 Bytes 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000026.sst 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000024.log size: 3763857 ; 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.env: 0x55c2b557b720 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T09:10:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.info_log: 0x55c2b7fc5480 2023-10-09T09:10:51.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T09:10:51.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T09:10:51.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T09:10:51.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T09:10:51.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T09:10:51.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T09:10:51.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T09:10:51.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T09:10:51.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T09:10:51.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T09:10:51.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T09:10:51.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.db_log_dir: 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.wal_dir: 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T09:10:51.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.write_buffer_manager: 0x55c2b7252c80 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T09:10:51.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.row_cache: None 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.wal_filter: None 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T09:10:51.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T09:10:51.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T09:10:51.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Compression algorithms supported: 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: kZSTD supported: 0 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T09:10:51.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.merge_operator: 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T09:10:51.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c2b7fc5580) 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: cache_index_and_filter_blocks: 1 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: pin_top_level_index_and_filter: 1 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: index_type: 0 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: data_block_index_type: 0 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: index_shortening: 1 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: checksum: 4 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: no_block_cache: 0 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: block_cache: 0x55c2b72d51f0 2023-10-09T09:10:51.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: block_cache_name: BinnedLRUCache 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: block_cache_options: 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: capacity : 536870912 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: num_shard_bits : 4 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: strict_capacity_limit : 0 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: high_pri_pool_ratio: 0.000 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: block_cache_compressed: (nil) 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: persistent_cache: (nil) 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: block_size: 4096 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: block_size_deviation: 10 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: block_restart_interval: 16 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: index_block_restart_interval: 1 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: metadata_block_size: 4096 2023-10-09T09:10:51.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: partition_filters: 0 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: use_delta_encoding: 1 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: filter_policy: bloomfilter 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: whole_key_filtering: 1 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: verify_compression: 0 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: read_amp_bytes_per_bit: 0 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: format_version: 5 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: enable_index_compression: 1 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: block_align: 0 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: max_auto_readahead_size: 262144 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: prepopulate_block_cache: 0 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: initial_auto_readahead_size: 8192 2023-10-09T09:10:51.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: num_file_reads_for_auto_readahead: 2 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.num_levels: 7 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T09:10:51.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T09:10:51.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T09:10:51.735 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T09:10:51.736 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T09:10:51.737 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T09:10:51.737 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T09:10:51.737 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T09:10:51.737 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T09:10:51.737 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T09:10:51.737 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T09:10:51.738 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T09:10:51.738 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T09:10:51.738 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T09:10:51.738 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T09:10:51.738 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T09:10:51.738 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T09:10:51.738 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T09:10:51.738 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T09:10:51.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5cfabe700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 26.sst 2023-10-09T09:10:51.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 28, last_sequence is 20723, log_number is 24,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 24 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d048ad77-725b-45e5-9413-a7c89e16e6dc 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696842651422156, "job": 1, "event": "recovery_started", "wal_files": [24]} 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.417+0000 7fe5da2a0c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #24 mode 2 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.429+0000 7fe5da2a0c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696842651432383, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 29, "file_size": 2496690, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 20728, "largest_seqno": 21555, "table_properties": {"data_size": 2491287, "index_size": 3680, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 6592, "raw_average_key_size": 23, "raw_value_size": 2483396, "raw_average_value_size": 8965, "num_data_blocks": 159, "num_entries": 277, "num_filter_entries": 277, "num_deletions": 21, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696842651, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d048ad77-725b-45e5-9413-a7c89e16e6dc", "db_session_id": "E2RYKMBDC36T6QT8X0GQ", "orig_file_number": 29, "seqno_to_time_mapping": "N/A"}} 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.429+0000 7fe5da2a0c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696842651432513, "job": 1, "event": "recovery_finished"} 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.429+0000 7fe5da2a0c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 31 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.433+0000 7fe5da2a0c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000024.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.433+0000 7fe5da2a0c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c2b7376000 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.433+0000 7fe5da2a0c80 4 rocksdb: DB pointer 0x55c2b7364000 2023-10-09T09:10:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.433+0000 7fe5c8ab0700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.433+0000 7fe5c8ab0700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: ** DB Stats ** 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: ** Compaction Stats [default] ** 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:10:51.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: L0 1/0 2.38 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 548.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: L6 1/0 34.86 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Sum 2/0 37.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 548.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 548.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: ** Compaction Stats [default] ** 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 548.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Flush(GB): cumulative 0.002, interval 0.002 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:10:51.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Cumulative compaction: 0.00 GB write, 164.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Interval compaction: 0.00 GB write, 164.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Block cache BinnedLRUCache@0x55c2b72d51f0#7 capacity: 512.00 MB usage: 4.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Block cache entry stats(count,size,portion): FilterBlock(1,0.78 KB,0.000149012%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.433+0000 7fe5da2a0c80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] at bind addrs [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.433+0000 7fe5da2a0c80 1 mon.c@-1(???) e3 preinit fsid 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.437+0000 7fe5da2a0c80 0 mon.c@-1(???).mds e6 new map 2023-10-09T09:10:51.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.437+0000 7fe5da2a0c80 0 mon.c@-1(???).mds e6 print_map 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: e6 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: legacy client fscid: 1 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Filesystem 'foo' (1) 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: fs_name foo 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: epoch 5 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: created 2023-10-09T09:03:24.292218+0000 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: modified 2023-10-09T09:03:30.346509+0000 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: tableserver 0 2023-10-09T09:10:51.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: root 0 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: session_timeout 60 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: session_autoclose 300 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: max_file_size 1099511627776 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: required_client_features {} 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: last_failure 0 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: last_failure_osd_epoch 0 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: max_mds 1 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: in 0 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: up {0=14640} 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: failed 2023-10-09T09:10:51.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: damaged 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: stopped 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: data_pools [4] 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: metadata_pool 3 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: inline_data disabled 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: balancer 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: bal_rank_mask -1 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: standby_count_wanted 1 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: [mds.foo.smithi163.vnodzj{0:14640} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.163:6834/3139884719,v1:172.21.15.163:6835/3139884719] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: Standby daemons: 2023-10-09T09:10:51.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: 2023-10-09T09:10:51.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: [mds.foo.smithi181.ajqgsc{-1:24511} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.181:6832/2837854117,v1:172.21.15.181:6833/2837854117] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:10:51.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.437+0000 7fe5da2a0c80 0 mon.c@-1(???).osd e388 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T09:10:51.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.437+0000 7fe5da2a0c80 0 mon.c@-1(???).osd e388 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:10:51.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.437+0000 7fe5da2a0c80 0 mon.c@-1(???).osd e388 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:10:51.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.437+0000 7fe5da2a0c80 0 mon.c@-1(???).osd e388 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:10:51.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:10:51 smithi163 bash[78107]: debug 2023-10-09T09:10:51.437+0000 7fe5da2a0c80 1 mon.c@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-10-09T09:10:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:10:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:10:53] "GET /metrics HTTP/1.1" 200 36081 "" "Prometheus/2.33.4" 2023-10-09T09:10:56.618 INFO:tasks.rados.rados.0.smithi163.stdout:1345: done (7 left) 2023-10-09T09:10:56.618 INFO:tasks.rados.rados.0.smithi163.stdout:1346: delete oid 27 current snap is 165 2023-10-09T09:10:56.618 INFO:tasks.rados.rados.0.smithi163.stdout:1337: finishing rollback tid 0 to smithi16367185-15 2023-10-09T09:10:56.618 INFO:tasks.rados.rados.0.smithi163.stdout:1337: finishing rollback tid 1 to smithi16367185-15 2023-10-09T09:10:56.618 INFO:tasks.rados.rados.0.smithi163.stdout:1337: finishing rollback tid 2 to smithi16367185-15 2023-10-09T09:10:56.618 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 646 (ObjNum 399 snap 121 seq_num 399) dirty exists 2023-10-09T09:10:56.618 INFO:tasks.rados.rados.0.smithi163.stdout:1336: finishing write tid 3 to smithi16367185-19 2023-10-09T09:10:56.618 INFO:tasks.rados.rados.0.smithi163.stdout:1336: finishing write tid 4 to smithi16367185-19 2023-10-09T09:10:56.619 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 456 (ObjNum 523 snap 165 seq_num 523) dirty exists 2023-10-09T09:10:56.619 INFO:tasks.rados.rados.0.smithi163.stdout:1336: left oid 19 (ObjNum 523 snap 165 seq_num 523) 2023-10-09T09:10:56.619 INFO:tasks.rados.rados.0.smithi163.stdout:1343: finishing rollback tid 0 to smithi16367185-18 2023-10-09T09:10:56.619 INFO:tasks.rados.rados.0.smithi163.stdout:1342: expect (ObjNum 335 snap 101 seq_num 335) 2023-10-09T09:10:56.750 INFO:tasks.rados.rados.0.smithi163.stdout:1343: finishing rollback tid 1 to smithi16367185-18 2023-10-09T09:10:56.750 INFO:tasks.rados.rados.0.smithi163.stdout:1343: finishing rollback tid 2 to smithi16367185-18 2023-10-09T09:10:56.751 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 500 (ObjNum 489 snap 154 seq_num 489) dirty exists 2023-10-09T09:10:56.751 INFO:tasks.rados.rados.0.smithi163.stdout:1344: finishing rollback tid 0 to smithi16367185-33 2023-10-09T09:10:56.751 INFO:tasks.rados.rados.0.smithi163.stdout:1344: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:10:56.751 INFO:tasks.rados.rados.0.smithi163.stdout:1344: finishing rollback tid 2 to smithi16367185-33 2023-10-09T09:10:56.751 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 561 (ObjNum 507 snap 159 seq_num 507) dirty exists 2023-10-09T09:10:56.751 INFO:tasks.rados.rados.0.smithi163.stdout:1338: expect (ObjNum 503 snap 157 seq_num 503) 2023-10-09T09:10:56.830 INFO:tasks.rados.rados.0.smithi163.stdout:1341: expect (ObjNum 471 snap 152 seq_num 471) 2023-10-09T09:10:57.054 INFO:tasks.rados.rados.0.smithi163.stdout:1336: done (7 left) 2023-10-09T09:10:57.054 INFO:tasks.rados.rados.0.smithi163.stdout:1337: done (6 left) 2023-10-09T09:10:57.054 INFO:tasks.rados.rados.0.smithi163.stdout:1338: done (5 left) 2023-10-09T09:10:57.055 INFO:tasks.rados.rados.0.smithi163.stdout:1341: done (4 left) 2023-10-09T09:10:57.055 INFO:tasks.rados.rados.0.smithi163.stdout:1342: done (3 left) 2023-10-09T09:10:57.055 INFO:tasks.rados.rados.0.smithi163.stdout:1343: done (2 left) 2023-10-09T09:10:57.055 INFO:tasks.rados.rados.0.smithi163.stdout:1344: done (1 left) 2023-10-09T09:10:57.055 INFO:tasks.rados.rados.0.smithi163.stdout:1346: done (0 left) 2023-10-09T09:10:57.055 INFO:tasks.rados.rados.0.smithi163.stdout:1347: snap_remove snap 154 2023-10-09T09:10:57.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:56 smithi181 bash[27712]: debug 2023-10-09T09:10:56.582+0000 7fde83035700 -1 mon.b@1(peon).osd e389 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:57.725 INFO:tasks.rados.rados.0.smithi163.stdout:1347: done (0 left) 2023-10-09T09:10:57.725 INFO:tasks.rados.rados.0.smithi163.stdout:1348: read oid 19 snap 129 2023-10-09T09:10:57.725 INFO:tasks.rados.rados.0.smithi163.stdout:1348: expect (ObjNum 390 snap 119 seq_num 390) 2023-10-09T09:10:57.726 INFO:tasks.rados.rados.0.smithi163.stdout:1349: write oid 21 current snap is 165 2023-10-09T09:10:57.726 INFO:tasks.rados.rados.0.smithi163.stdout:1349: seq_num 526 ranges {440482=409331,1380352=405181,2461553=378929} 2023-10-09T09:10:57.731 INFO:tasks.rados.rados.0.smithi163.stdout:1349: writing smithi16367185-21 from 440482 to 849813 tid 1 2023-10-09T09:10:57.739 INFO:tasks.rados.rados.0.smithi163.stdout:1349: writing smithi16367185-21 from 1380352 to 1785533 tid 2 2023-10-09T09:10:57.746 INFO:tasks.rados.rados.0.smithi163.stdout:1349: writing smithi16367185-21 from 2461553 to 2840482 tid 3 2023-10-09T09:10:57.746 INFO:tasks.rados.rados.0.smithi163.stdout:1350: write oid 44 current snap is 165 2023-10-09T09:10:57.746 INFO:tasks.rados.rados.0.smithi163.stdout:1350: seq_num 527 ranges {457289=484580,1438834=548839,2772623=84666} 2023-10-09T09:10:57.752 INFO:tasks.rados.rados.0.smithi163.stdout:1350: writing smithi16367185-44 from 457289 to 941869 tid 1 2023-10-09T09:10:57.758 INFO:tasks.rados.rados.0.smithi163.stdout:1350: writing smithi16367185-44 from 1438834 to 1987673 tid 2 2023-10-09T09:10:57.764 INFO:tasks.rados.rados.0.smithi163.stdout:1350: writing smithi16367185-44 from 2772623 to 2857289 tid 3 2023-10-09T09:10:57.764 INFO:tasks.rados.rados.0.smithi163.stdout:1351: read oid 35 snap -1 2023-10-09T09:10:57.764 INFO:tasks.rados.rados.0.smithi163.stdout:1351: expect (ObjNum 449 snap 144 seq_num 449) 2023-10-09T09:10:57.764 INFO:tasks.rados.rados.0.smithi163.stdout:1352: read oid 50 snap -1 2023-10-09T09:10:57.764 INFO:tasks.rados.rados.0.smithi163.stdout:1352: expect deleted 2023-10-09T09:10:57.764 INFO:tasks.rados.rados.0.smithi163.stdout:1353: write oid 26 current snap is 165 2023-10-09T09:10:57.764 INFO:tasks.rados.rados.0.smithi163.stdout:1353: seq_num 528 ranges {474096=559829,1780963=408850,2800046=74050} 2023-10-09T09:10:57.770 INFO:tasks.rados.rados.0.smithi163.stdout:1353: writing smithi16367185-26 from 474096 to 1033925 tid 1 2023-10-09T09:10:57.777 INFO:tasks.rados.rados.0.smithi163.stdout:1353: writing smithi16367185-26 from 1780963 to 2189813 tid 2 2023-10-09T09:10:57.781 INFO:tasks.rados.rados.0.smithi163.stdout:1353: writing smithi16367185-26 from 2800046 to 2874096 tid 3 2023-10-09T09:10:57.781 INFO:tasks.rados.rados.0.smithi163.stdout:1354: snap_remove snap 152 2023-10-09T09:10:58.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: debug 2023-10-09T09:10:57.710+0000 7fde83035700 -1 mon.b@1(peon).osd e390 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:58.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:51.549440+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T09:10:58.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.572786+0000 mon.b (mon.1) 104 : cluster [INF] mon.b calling monitor election 2023-10-09T09:10:58.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.575211+0000 mon.a (mon.0) 199 : cluster [INF] mon.a calling monitor election 2023-10-09T09:10:58.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.613528+0000 mon.a (mon.0) 200 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:10:58.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.624188+0000 mon.a (mon.0) 201 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T09:10:58.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.643800+0000 mon.a (mon.0) 202 : cluster [DBG] fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 1 up:standby 2023-10-09T09:10:58.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.643859+0000 mon.a (mon.0) 203 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2023-10-09T09:10:58.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.644404+0000 mon.a (mon.0) 204 : cluster [DBG] mgrmap e34: y(active, since 92s), standbys: x 2023-10-09T09:10:58.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.644684+0000 mon.a (mon.0) 205 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T09:10:58.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.644723+0000 mon.a (mon.0) 206 : cluster [INF] Cluster is now healthy 2023-10-09T09:10:58.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:57 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.660015+0000 mon.a (mon.0) 207 : cluster [INF] overall HEALTH_OK 2023-10-09T09:10:58.735 INFO:tasks.rados.rados.0.smithi163.stdout:1354: done (6 left) 2023-10-09T09:10:58.735 INFO:tasks.rados.rados.0.smithi163.stdout:1355: rollback oid 29 current snap is 165 2023-10-09T09:10:58.735 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 29 to 164 2023-10-09T09:10:58.735 INFO:tasks.rados.rados.0.smithi163.stdout:1348: expect (ObjNum 390 snap 119 seq_num 390) 2023-10-09T09:10:58.844 INFO:tasks.rados.rados.0.smithi163.stdout:1349: finishing write tid 1 to smithi16367185-21 2023-10-09T09:10:58.844 INFO:tasks.rados.rados.0.smithi163.stdout:1349: finishing write tid 2 to smithi16367185-21 2023-10-09T09:10:58.844 INFO:tasks.rados.rados.0.smithi163.stdout:1349: finishing write tid 3 to smithi16367185-21 2023-10-09T09:10:58.844 INFO:tasks.rados.rados.0.smithi163.stdout:1349: finishing write tid 4 to smithi16367185-21 2023-10-09T09:10:58.844 INFO:tasks.rados.rados.0.smithi163.stdout:1349: finishing write tid 5 to smithi16367185-21 2023-10-09T09:10:58.845 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 326 (ObjNum 526 snap 165 seq_num 526) dirty exists 2023-10-09T09:10:58.845 INFO:tasks.rados.rados.0.smithi163.stdout:1349: left oid 21 (ObjNum 526 snap 165 seq_num 526) 2023-10-09T09:10:58.845 INFO:tasks.rados.rados.0.smithi163.stdout:1350: finishing write tid 1 to smithi16367185-44 2023-10-09T09:10:58.845 INFO:tasks.rados.rados.0.smithi163.stdout:1350: finishing write tid 2 to smithi16367185-44 2023-10-09T09:10:58.845 INFO:tasks.rados.rados.0.smithi163.stdout:1350: finishing write tid 3 to smithi16367185-44 2023-10-09T09:10:58.845 INFO:tasks.rados.rados.0.smithi163.stdout:1350: finishing write tid 4 to smithi16367185-44 2023-10-09T09:10:58.845 INFO:tasks.rados.rados.0.smithi163.stdout:1350: finishing write tid 5 to smithi16367185-44 2023-10-09T09:10:58.845 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1074 (ObjNum 527 snap 165 seq_num 527) dirty exists 2023-10-09T09:10:58.845 INFO:tasks.rados.rados.0.smithi163.stdout:1350: left oid 44 (ObjNum 527 snap 165 seq_num 527) 2023-10-09T09:10:58.845 INFO:tasks.rados.rados.0.smithi163.stdout:1351: expect (ObjNum 449 snap 144 seq_num 449) 2023-10-09T09:10:58.965 INFO:tasks.rados.rados.0.smithi163.stdout:1353: finishing write tid 1 to smithi16367185-26 2023-10-09T09:10:58.965 INFO:tasks.rados.rados.0.smithi163.stdout:1353: finishing write tid 2 to smithi16367185-26 2023-10-09T09:10:58.965 INFO:tasks.rados.rados.0.smithi163.stdout:1353: finishing write tid 3 to smithi16367185-26 2023-10-09T09:10:58.965 INFO:tasks.rados.rados.0.smithi163.stdout:1353: finishing write tid 4 to smithi16367185-26 2023-10-09T09:10:58.965 INFO:tasks.rados.rados.0.smithi163.stdout:1353: finishing write tid 5 to smithi16367185-26 2023-10-09T09:10:58.965 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 470 (ObjNum 528 snap 165 seq_num 528) dirty exists 2023-10-09T09:10:58.965 INFO:tasks.rados.rados.0.smithi163.stdout:1353: left oid 26 (ObjNum 528 snap 165 seq_num 528) 2023-10-09T09:10:58.965 INFO:tasks.rados.rados.0.smithi163.stdout:1355: finishing rollback tid 0 to smithi16367185-29 2023-10-09T09:10:58.965 INFO:tasks.rados.rados.0.smithi163.stdout:1355: finishing rollback tid 1 to smithi16367185-29 2023-10-09T09:10:58.965 INFO:tasks.rados.rados.0.smithi163.stdout:1355: finishing rollback tid 2 to smithi16367185-29 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 575 (ObjNum 335 snap 101 seq_num 335) dirty exists 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:1348: done (6 left) 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:1349: done (5 left) 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:1350: done (4 left) 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:1351: done (3 left) 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:1352: done (2 left) 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:1353: done (1 left) 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:1355: done (0 left) 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:1356: read oid 24 snap -1 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:1356: expect (ObjNum 500 snap 156 seq_num 500) 2023-10-09T09:10:58.966 INFO:tasks.rados.rados.0.smithi163.stdout:1357: write oid 39 current snap is 165 2023-10-09T09:10:58.967 INFO:tasks.rados.rados.0.smithi163.stdout:1357: seq_num 529 ranges {490903=635078,1839445=552508} 2023-10-09T09:10:58.981 INFO:tasks.rados.rados.0.smithi163.stdout:1357: writing smithi16367185-39 from 490903 to 1125981 tid 1 2023-10-09T09:10:58.988 INFO:tasks.rados.rados.0.smithi163.stdout:1357: writing smithi16367185-39 from 1839445 to 2391953 tid 2 2023-10-09T09:10:58.988 INFO:tasks.rados.rados.0.smithi163.stdout:1358: write oid 7 current snap is 165 2023-10-09T09:10:58.988 INFO:tasks.rados.rados.0.smithi163.stdout:1358: seq_num 530 ranges {507710=710327,1897927=412519,2854892=52818} 2023-10-09T09:10:58.995 INFO:tasks.rados.rados.0.smithi163.stdout:1358: writing smithi16367185-7 from 507710 to 1218037 tid 1 2023-10-09T09:10:59.001 INFO:tasks.rados.rados.0.smithi163.stdout:1358: writing smithi16367185-7 from 1897927 to 2310446 tid 2 2023-10-09T09:10:59.005 INFO:tasks.rados.rados.0.smithi163.stdout:1358: writing smithi16367185-7 from 2854892 to 2907710 tid 3 2023-10-09T09:10:59.005 INFO:tasks.rados.rados.0.smithi163.stdout:1359: delete oid 18 current snap is 165 2023-10-09T09:10:59.005 INFO:tasks.rados.rados.0.smithi163.stdout:1356: expect (ObjNum 500 snap 156 seq_num 500) 2023-10-09T09:10:59.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:10:58 smithi181 bash[54907]: ts=2023-10-09T09:10:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:10:59.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:58 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.711243+0000 mon.a (mon.0) 208 : cluster [DBG] mgrmap e35: y(active, since 92s), standbys: x 2023-10-09T09:10:59.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:58 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.718315+0000 osd.6 (osd.6) 30 : cluster [WRN] failed to encode map e390 with expected crc 2023-10-09T09:10:59.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:58 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.718664+0000 osd.3 (osd.3) 39 : cluster [WRN] failed to encode map e390 with expected crc 2023-10-09T09:10:59.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:58 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.724972+0000 mon.a (mon.0) 209 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2023-10-09T09:10:59.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:58 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.727321+0000 osd.2 (osd.2) 66 : cluster [WRN] failed to encode map e390 with expected crc 2023-10-09T09:10:59.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:58 smithi181 bash[27712]: cluster 2023-10-09T09:10:57.733834+0000 osd.1 (osd.1) 30 : cluster [WRN] failed to encode map e390 with expected crc 2023-10-09T09:10:59.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:58 smithi181 bash[27712]: debug 2023-10-09T09:10:58.722+0000 7fde83035700 -1 mon.b@1(peon).osd e391 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:10:59.089 INFO:tasks.rados.rados.0.smithi163.stdout:1357: finishing write tid 1 to smithi16367185-39 2023-10-09T09:10:59.089 INFO:tasks.rados.rados.0.smithi163.stdout:1357: finishing write tid 2 to smithi16367185-39 2023-10-09T09:10:59.089 INFO:tasks.rados.rados.0.smithi163.stdout:1357: finishing write tid 3 to smithi16367185-39 2023-10-09T09:10:59.089 INFO:tasks.rados.rados.0.smithi163.stdout:1357: finishing write tid 4 to smithi16367185-39 2023-10-09T09:10:59.089 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 546 (ObjNum 529 snap 165 seq_num 529) dirty exists 2023-10-09T09:10:59.090 INFO:tasks.rados.rados.0.smithi163.stdout:1357: left oid 39 (ObjNum 529 snap 165 seq_num 529) 2023-10-09T09:10:59.090 INFO:tasks.rados.rados.0.smithi163.stdout:1358: finishing write tid 1 to smithi16367185-7 2023-10-09T09:10:59.090 INFO:tasks.rados.rados.0.smithi163.stdout:1358: finishing write tid 2 to smithi16367185-7 2023-10-09T09:10:59.090 INFO:tasks.rados.rados.0.smithi163.stdout:1358: finishing write tid 3 to smithi16367185-7 2023-10-09T09:10:59.090 INFO:tasks.rados.rados.0.smithi163.stdout:1358: finishing write tid 4 to smithi16367185-7 2023-10-09T09:10:59.091 INFO:tasks.rados.rados.0.smithi163.stdout:1358: finishing write tid 5 to smithi16367185-7 2023-10-09T09:10:59.091 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 333 (ObjNum 530 snap 165 seq_num 530) dirty exists 2023-10-09T09:10:59.091 INFO:tasks.rados.rados.0.smithi163.stdout:1358: left oid 7 (ObjNum 530 snap 165 seq_num 530) 2023-10-09T09:10:59.091 INFO:tasks.rados.rados.0.smithi163.stdout:1356: done (3 left) 2023-10-09T09:10:59.091 INFO:tasks.rados.rados.0.smithi163.stdout:1357: done (2 left) 2023-10-09T09:10:59.091 INFO:tasks.rados.rados.0.smithi163.stdout:1358: done (1 left) 2023-10-09T09:10:59.092 INFO:tasks.rados.rados.0.smithi163.stdout:1359: done (0 left) 2023-10-09T09:10:59.092 INFO:tasks.rados.rados.0.smithi163.stdout:1360: read oid 14 snap -1 2023-10-09T09:10:59.092 INFO:tasks.rados.rados.0.smithi163.stdout:1360: expect (ObjNum 438 snap 136 seq_num 438) 2023-10-09T09:10:59.092 INFO:tasks.rados.rados.0.smithi163.stdout:1361: read oid 22 snap 164 2023-10-09T09:10:59.092 INFO:tasks.rados.rados.0.smithi163.stdout:1361: expect (ObjNum 451 snap 145 seq_num 451) 2023-10-09T09:10:59.092 INFO:tasks.rados.rados.0.smithi163.stdout:1362: snap_create 2023-10-09T09:10:59.108 INFO:tasks.rados.rados.0.smithi163.stdout:1360: expect (ObjNum 438 snap 136 seq_num 438) 2023-10-09T09:10:59.270 INFO:tasks.rados.rados.0.smithi163.stdout:1361: expect (ObjNum 451 snap 145 seq_num 451) 2023-10-09T09:10:59.562 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T09:10:59.747 INFO:tasks.rados.rados.0.smithi163.stdout:1360: done (2 left) 2023-10-09T09:10:59.747 INFO:tasks.rados.rados.0.smithi163.stdout:1361: done (1 left) 2023-10-09T09:10:59.747 INFO:tasks.rados.rados.0.smithi163.stdout:1362: done (0 left) 2023-10-09T09:10:59.747 INFO:tasks.rados.rados.0.smithi163.stdout:1363: read oid 17 snap -1 2023-10-09T09:10:59.747 INFO:tasks.rados.rados.0.smithi163.stdout:1363: expect (ObjNum 504 snap 157 seq_num 504) 2023-10-09T09:10:59.747 INFO:tasks.rados.rados.0.smithi163.stdout:1364: write oid 41 current snap is 166 2023-10-09T09:10:59.747 INFO:tasks.rados.rados.0.smithi163.stdout:1364: seq_num 532 ranges {541324=460825,1498538=416188,2393385=547939} 2023-10-09T09:10:59.755 INFO:tasks.rados.rados.0.smithi163.stdout:1364: writing smithi16367185-41 from 541324 to 1002149 tid 1 2023-10-09T09:10:59.761 INFO:tasks.rados.rados.0.smithi163.stdout:1364: writing smithi16367185-41 from 1498538 to 1914726 tid 2 2023-10-09T09:10:59.767 INFO:tasks.rados.rados.0.smithi163.stdout:1364: writing smithi16367185-41 from 2393385 to 2941324 tid 3 2023-10-09T09:10:59.767 INFO:tasks.rados.rados.0.smithi163.stdout:1365: snap_create 2023-10-09T09:10:59.767 INFO:tasks.rados.rados.0.smithi163.stdout:1363: expect (ObjNum 504 snap 157 seq_num 504) 2023-10-09T09:10:59.886 INFO:tasks.rados.rados.0.smithi163.stdout:1364: finishing write tid 1 to smithi16367185-41 2023-10-09T09:10:59.886 INFO:tasks.rados.rados.0.smithi163.stdout:1364: finishing write tid 2 to smithi16367185-41 2023-10-09T09:10:59.886 INFO:tasks.rados.rados.0.smithi163.stdout:1364: finishing write tid 3 to smithi16367185-41 2023-10-09T09:10:59.886 INFO:tasks.rados.rados.0.smithi163.stdout:1364: finishing write tid 4 to smithi16367185-41 2023-10-09T09:10:59.886 INFO:tasks.rados.rados.0.smithi163.stdout:1364: finishing write tid 5 to smithi16367185-41 2023-10-09T09:10:59.886 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 94 (ObjNum 532 snap 166 seq_num 532) dirty exists 2023-10-09T09:10:59.886 INFO:tasks.rados.rados.0.smithi163.stdout:1364: left oid 41 (ObjNum 532 snap 166 seq_num 532) 2023-10-09T09:11:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:59 smithi181 bash[27712]: debug 2023-10-09T09:10:59.730+0000 7fde83035700 -1 mon.b@1(peon).osd e392 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:59 smithi181 bash[27712]: cluster 2023-10-09T09:10:58.727958+0000 osd.3 (osd.3) 40 : cluster [WRN] failed to encode map e391 with expected crc 2023-10-09T09:11:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:59 smithi181 bash[27712]: cluster 2023-10-09T09:10:58.730191+0000 mon.a (mon.0) 210 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2023-10-09T09:11:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:59 smithi181 bash[27712]: cluster 2023-10-09T09:10:58.733182+0000 osd.1 (osd.1) 31 : cluster [WRN] failed to encode map e391 with expected crc 2023-10-09T09:11:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:59 smithi181 bash[27712]: cluster 2023-10-09T09:10:58.737814+0000 osd.2 (osd.2) 67 : cluster [WRN] failed to encode map e391 with expected crc 2023-10-09T09:11:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:59 smithi181 bash[27712]: cluster 2023-10-09T09:10:59.241519+0000 mgr.y (mgr.34107) 68 : cluster [DBG] pgmap v137: 89 pgs: 89 active+clean; 226 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s wr, 0 op/s 2023-10-09T09:11:00.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:10:59 smithi181 bash[27712]: audit 2023-10-09T09:10:59.548027+0000 mon.a (mon.0) 211 : audit [DBG] from='client.? 172.21.15.163:0/2481174109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:11:00.337 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:11:00.753 INFO:tasks.rados.rados.0.smithi163.stdout:1363: done (2 left) 2023-10-09T09:11:00.753 INFO:tasks.rados.rados.0.smithi163.stdout:1364: done (1 left) 2023-10-09T09:11:00.753 INFO:tasks.rados.rados.0.smithi163.stdout:1365: done (0 left) 2023-10-09T09:11:00.753 INFO:tasks.rados.rados.0.smithi163.stdout:1366: write oid 10 current snap is 167 2023-10-09T09:11:00.754 INFO:tasks.rados.rados.0.smithi163.stdout:1366: seq_num 533 ranges {558131=652427,1673373=559846,2820808=137323} 2023-10-09T09:11:00.762 INFO:tasks.rados.rados.0.smithi163.stdout:1366: writing smithi16367185-10 from 558131 to 1210558 tid 1 2023-10-09T09:11:00.770 INFO:tasks.rados.rados.0.smithi163.stdout:1366: writing smithi16367185-10 from 1673373 to 2233219 tid 2 2023-10-09T09:11:00.775 INFO:tasks.rados.rados.0.smithi163.stdout:1366: writing smithi16367185-10 from 2820808 to 2958131 tid 3 2023-10-09T09:11:00.775 INFO:tasks.rados.rados.0.smithi163.stdout:1367: delete oid 29 current snap is 167 2023-10-09T09:11:00.775 INFO:tasks.rados.rados.0.smithi163.stdout:1366: finishing write tid 1 to smithi16367185-10 2023-10-09T09:11:00.778 INFO:tasks.rados.rados.0.smithi163.stdout:1366: finishing write tid 2 to smithi16367185-10 2023-10-09T09:11:00.779 INFO:tasks.rados.rados.0.smithi163.stdout:1367: done (1 left) 2023-10-09T09:11:00.779 INFO:tasks.rados.rados.0.smithi163.stdout:1368: read oid 32 snap -1 2023-10-09T09:11:00.779 INFO:tasks.rados.rados.0.smithi163.stdout:1368: expect (ObjNum 422 snap 129 seq_num 422) 2023-10-09T09:11:00.779 INFO:tasks.rados.rados.0.smithi163.stdout:1369: snap_create 2023-10-09T09:11:00.779 INFO:tasks.rados.rados.0.smithi163.stdout:1366: finishing write tid 3 to smithi16367185-10 2023-10-09T09:11:00.779 INFO:tasks.rados.rados.0.smithi163.stdout:1366: finishing write tid 4 to smithi16367185-10 2023-10-09T09:11:00.779 INFO:tasks.rados.rados.0.smithi163.stdout:1366: finishing write tid 5 to smithi16367185-10 2023-10-09T09:11:00.779 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 518 (ObjNum 533 snap 167 seq_num 533) dirty exists 2023-10-09T09:11:00.780 INFO:tasks.rados.rados.0.smithi163.stdout:1366: left oid 10 (ObjNum 533 snap 167 seq_num 533) 2023-10-09T09:11:00.784 INFO:tasks.rados.rados.0.smithi163.stdout:1368: expect (ObjNum 422 snap 129 seq_num 422) 2023-10-09T09:11:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:00 smithi181 bash[27712]: cluster 2023-10-09T09:10:59.740730+0000 osd.3 (osd.3) 41 : cluster [WRN] failed to encode map e392 with expected crc 2023-10-09T09:11:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:00 smithi181 bash[27712]: cluster 2023-10-09T09:10:59.742203+0000 osd.5 (osd.5) 37 : cluster [WRN] failed to encode map e392 with expected crc 2023-10-09T09:11:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:00 smithi181 bash[27712]: cluster 2023-10-09T09:10:59.746667+0000 mon.a (mon.0) 212 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2023-10-09T09:11:01.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:00 smithi181 bash[27712]: cluster 2023-10-09T09:10:59.747022+0000 osd.7 (osd.7) 45 : cluster [WRN] failed to encode map e392 with expected crc 2023-10-09T09:11:01.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:00 smithi181 bash[27712]: cluster 2023-10-09T09:10:59.757732+0000 osd.2 (osd.2) 68 : cluster [WRN] failed to encode map e392 with expected crc 2023-10-09T09:11:01.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:00 smithi181 bash[27712]: debug 2023-10-09T09:11:00.742+0000 7fde83035700 -1 mon.b@1(peon).osd e393 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:01.537 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:11:01 smithi163 bash[71599]: debug 2023-10-09T09:11:01.444+0000 7f5c688fd700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T09:11:01.764 INFO:tasks.rados.rados.0.smithi163.stdout:1366: done (2 left) 2023-10-09T09:11:01.764 INFO:tasks.rados.rados.0.smithi163.stdout:1368: done (1 left) 2023-10-09T09:11:01.764 INFO:tasks.rados.rados.0.smithi163.stdout:1369: done (0 left) 2023-10-09T09:11:01.764 INFO:tasks.rados.rados.0.smithi163.stdout:1370: read oid 41 snap -1 2023-10-09T09:11:01.765 INFO:tasks.rados.rados.0.smithi163.stdout:1370: expect (ObjNum 532 snap 166 seq_num 532) 2023-10-09T09:11:01.765 INFO:tasks.rados.rados.0.smithi163.stdout:1371: delete oid 20 current snap is 168 2023-10-09T09:11:01.773 INFO:tasks.rados.rados.0.smithi163.stdout:1370: expect (ObjNum 532 snap 166 seq_num 532) 2023-10-09T09:11:01.873 INFO:tasks.rados.rados.0.smithi163.stdout:1370: done (1 left) 2023-10-09T09:11:01.873 INFO:tasks.rados.rados.0.smithi163.stdout:1371: done (0 left) 2023-10-09T09:11:01.873 INFO:tasks.rados.rados.0.smithi163.stdout:1372: write oid 24 current snap is 168 2023-10-09T09:11:01.873 INFO:tasks.rados.rados.0.smithi163.stdout:1372: seq_num 536 ranges {608552=478174,1848819=707173} 2023-10-09T09:11:01.883 INFO:tasks.rados.rados.0.smithi163.stdout:1372: writing smithi16367185-24 from 608552 to 1086726 tid 1 2023-10-09T09:11:01.893 INFO:tasks.rados.rados.0.smithi163.stdout:1372: writing smithi16367185-24 from 1848819 to 2555992 tid 2 2023-10-09T09:11:01.893 INFO:tasks.rados.rados.0.smithi163.stdout:1373: write oid 40 current snap is 168 2023-10-09T09:11:01.893 INFO:tasks.rados.rados.0.smithi163.stdout:1372: finishing write tid 1 to smithi16367185-24 2023-10-09T09:11:01.893 INFO:tasks.rados.rados.0.smithi163.stdout:1373: seq_num 537 ranges {625359=553423,1790948=567184,2930500=94859} 2023-10-09T09:11:01.899 INFO:tasks.rados.rados.0.smithi163.stdout:1373: writing smithi16367185-40 from 625359 to 1178782 tid 1 2023-10-09T09:11:01.906 INFO:tasks.rados.rados.0.smithi163.stdout:1373: writing smithi16367185-40 from 1790948 to 2358132 tid 2 2023-10-09T09:11:01.910 INFO:tasks.rados.rados.0.smithi163.stdout:1373: writing smithi16367185-40 from 2930500 to 3025359 tid 3 2023-10-09T09:11:01.911 INFO:tasks.rados.rados.0.smithi163.stdout:1374: rollback oid 11 current snap is 168 2023-10-09T09:11:01.911 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 11 to 160 2023-10-09T09:11:01.911 INFO:tasks.rados.rados.0.smithi163.stdout:1375: write oid 46 current snap is 168 2023-10-09T09:11:01.911 INFO:tasks.rados.rados.0.smithi163.stdout:1375: seq_num 538 ranges {642166=628672,1849430=710842} 2023-10-09T09:11:01.918 INFO:tasks.rados.rados.0.smithi163.stdout:1375: writing smithi16367185-46 from 642166 to 1270838 tid 1 2023-10-09T09:11:01.927 INFO:tasks.rados.rados.0.smithi163.stdout:1375: writing smithi16367185-46 from 1849430 to 2560272 tid 2 2023-10-09T09:11:01.927 INFO:tasks.rados.rados.0.smithi163.stdout:1376: snap_create 2023-10-09T09:11:01.927 INFO:tasks.rados.rados.0.smithi163.stdout:1372: finishing write tid 2 to smithi16367185-24 2023-10-09T09:11:01.927 INFO:tasks.rados.rados.0.smithi163.stdout:1372: finishing write tid 3 to smithi16367185-24 2023-10-09T09:11:01.927 INFO:tasks.rados.rados.0.smithi163.stdout:1372: finishing write tid 4 to smithi16367185-24 2023-10-09T09:11:01.927 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 345 (ObjNum 536 snap 168 seq_num 536) dirty exists 2023-10-09T09:11:01.927 INFO:tasks.rados.rados.0.smithi163.stdout:1372: left oid 24 (ObjNum 536 snap 168 seq_num 536) 2023-10-09T09:11:01.928 INFO:tasks.rados.rados.0.smithi163.stdout:1373: finishing write tid 1 to smithi16367185-40 2023-10-09T09:11:01.928 INFO:tasks.rados.rados.0.smithi163.stdout:1373: finishing write tid 2 to smithi16367185-40 2023-10-09T09:11:01.928 INFO:tasks.rados.rados.0.smithi163.stdout:1373: finishing write tid 3 to smithi16367185-40 2023-10-09T09:11:01.928 INFO:tasks.rados.rados.0.smithi163.stdout:1374: finishing rollback tid 1 to smithi16367185-11 2023-10-09T09:11:01.928 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 375 (ObjNum 699059776 snap 21875 seq_num 1207961808) dirty dne 2023-10-09T09:11:01.928 INFO:tasks.rados.rados.0.smithi163.stdout:1373: finishing write tid 4 to smithi16367185-40 2023-10-09T09:11:01.928 INFO:tasks.rados.rados.0.smithi163.stdout:1373: finishing write tid 5 to smithi16367185-40 2023-10-09T09:11:01.928 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 668 (ObjNum 537 snap 168 seq_num 537) dirty exists 2023-10-09T09:11:01.928 INFO:tasks.rados.rados.0.smithi163.stdout:1373: left oid 40 (ObjNum 537 snap 168 seq_num 537) 2023-10-09T09:11:01.928 INFO:tasks.rados.rados.0.smithi163.stdout:1375: finishing write tid 1 to smithi16367185-46 2023-10-09T09:11:01.931 INFO:tasks.rados.rados.0.smithi163.stdout:1375: finishing write tid 2 to smithi16367185-46 2023-10-09T09:11:01.931 INFO:tasks.rados.rados.0.smithi163.stdout:1375: finishing write tid 3 to smithi16367185-46 2023-10-09T09:11:01.932 INFO:tasks.rados.rados.0.smithi163.stdout:1375: finishing write tid 4 to smithi16367185-46 2023-10-09T09:11:01.932 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 553 (ObjNum 538 snap 168 seq_num 538) dirty exists 2023-10-09T09:11:01.932 INFO:tasks.rados.rados.0.smithi163.stdout:1375: left oid 46 (ObjNum 538 snap 168 seq_num 538) 2023-10-09T09:11:02.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:01 smithi181 bash[27712]: cluster 2023-10-09T09:11:00.747702+0000 osd.3 (osd.3) 42 : cluster [WRN] failed to encode map e393 with expected crc 2023-10-09T09:11:02.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:01 smithi181 bash[27712]: cluster 2023-10-09T09:11:00.749297+0000 mon.a (mon.0) 213 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2023-10-09T09:11:02.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:01 smithi181 bash[27712]: cluster 2023-10-09T09:11:00.750242+0000 osd.4 (osd.4) 76 : cluster [WRN] failed to encode map e393 with expected crc 2023-10-09T09:11:02.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:01 smithi181 bash[27712]: cluster 2023-10-09T09:11:00.765383+0000 osd.2 (osd.2) 69 : cluster [WRN] failed to encode map e393 with expected crc 2023-10-09T09:11:02.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:01 smithi181 bash[27712]: cluster 2023-10-09T09:11:01.242324+0000 mgr.y (mgr.34107) 69 : cluster [DBG] pgmap v140: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 2.0 MiB/s wr, 25 op/s 2023-10-09T09:11:02.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:01 smithi181 bash[27712]: debug 2023-10-09T09:11:01.750+0000 7fde83035700 -1 mon.b@1(peon).osd e394 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:02.769 INFO:tasks.rados.rados.0.smithi163.stdout:1372: done (4 left) 2023-10-09T09:11:02.770 INFO:tasks.rados.rados.0.smithi163.stdout:1373: done (3 left) 2023-10-09T09:11:02.770 INFO:tasks.rados.rados.0.smithi163.stdout:1374: done (2 left) 2023-10-09T09:11:02.770 INFO:tasks.rados.rados.0.smithi163.stdout:1375: done (1 left) 2023-10-09T09:11:02.770 INFO:tasks.rados.rados.0.smithi163.stdout:1376: done (0 left) 2023-10-09T09:11:02.770 INFO:tasks.rados.rados.0.smithi163.stdout:1377: delete oid 36 current snap is 169 2023-10-09T09:11:02.780 INFO:tasks.rados.rados.0.smithi163.stdout:1377: done (0 left) 2023-10-09T09:11:02.780 INFO:tasks.rados.rados.0.smithi163.stdout:1378: read oid 1 snap 165 2023-10-09T09:11:02.780 INFO:tasks.rados.rados.0.smithi163.stdout:1378: expect (ObjNum 510 snap 159 seq_num 510) 2023-10-09T09:11:02.780 INFO:tasks.rados.rados.0.smithi163.stdout:1379: write oid 35 current snap is 169 2023-10-09T09:11:02.780 INFO:tasks.rados.rados.0.smithi163.stdout:1379: seq_num 540 ranges {675780=495523,1682747=714511,3012769=63011} 2023-10-09T09:11:02.788 INFO:tasks.rados.rados.0.smithi163.stdout:1379: writing smithi16367185-35 from 675780 to 1171303 tid 1 2023-10-09T09:11:02.796 INFO:tasks.rados.rados.0.smithi163.stdout:1379: writing smithi16367185-35 from 1682747 to 2397258 tid 2 2023-10-09T09:11:02.800 INFO:tasks.rados.rados.0.smithi163.stdout:1379: writing smithi16367185-35 from 3012769 to 3075780 tid 3 2023-10-09T09:11:02.800 INFO:tasks.rados.rados.0.smithi163.stdout:1380: read oid 45 snap -1 2023-10-09T09:11:02.800 INFO:tasks.rados.rados.0.smithi163.stdout:1380: expect deleted 2023-10-09T09:11:02.800 INFO:tasks.rados.rados.0.smithi163.stdout:1381: write oid 41 current snap is 169 2023-10-09T09:11:02.801 INFO:tasks.rados.rados.0.smithi163.stdout:1381: seq_num 541 ranges {692587=570772,2024876=574522,3040192=52395} 2023-10-09T09:11:02.808 INFO:tasks.rados.rados.0.smithi163.stdout:1381: writing smithi16367185-41 from 692587 to 1263359 tid 1 2023-10-09T09:11:02.817 INFO:tasks.rados.rados.0.smithi163.stdout:1381: writing smithi16367185-41 from 2024876 to 2599398 tid 2 2023-10-09T09:11:02.820 INFO:tasks.rados.rados.0.smithi163.stdout:1381: writing smithi16367185-41 from 3040192 to 3092587 tid 3 2023-10-09T09:11:02.820 INFO:tasks.rados.rados.0.smithi163.stdout:1382: delete oid 24 current snap is 169 2023-10-09T09:11:02.820 INFO:tasks.rados.rados.0.smithi163.stdout:1378: expect (ObjNum 510 snap 159 seq_num 510) 2023-10-09T09:11:03.048 INFO:tasks.rados.rados.0.smithi163.stdout:1379: finishing write tid 1 to smithi16367185-35 2023-10-09T09:11:03.048 INFO:tasks.rados.rados.0.smithi163.stdout:1379: finishing write tid 2 to smithi16367185-35 2023-10-09T09:11:03.049 INFO:tasks.rados.rados.0.smithi163.stdout:1379: finishing write tid 3 to smithi16367185-35 2023-10-09T09:11:03.049 INFO:tasks.rados.rados.0.smithi163.stdout:1379: finishing write tid 4 to smithi16367185-35 2023-10-09T09:11:03.049 INFO:tasks.rados.rados.0.smithi163.stdout:1379: finishing write tid 5 to smithi16367185-35 2023-10-09T09:11:03.049 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 424 (ObjNum 540 snap 169 seq_num 540) dirty exists 2023-10-09T09:11:03.049 INFO:tasks.rados.rados.0.smithi163.stdout:1379: left oid 35 (ObjNum 540 snap 169 seq_num 540) 2023-10-09T09:11:03.049 INFO:tasks.rados.rados.0.smithi163.stdout:1381: finishing write tid 1 to smithi16367185-41 2023-10-09T09:11:03.049 INFO:tasks.rados.rados.0.smithi163.stdout:1381: finishing write tid 2 to smithi16367185-41 2023-10-09T09:11:03.049 INFO:tasks.rados.rados.0.smithi163.stdout:1378: done (4 left) 2023-10-09T09:11:03.049 INFO:tasks.rados.rados.0.smithi163.stdout:1379: done (3 left) 2023-10-09T09:11:03.049 INFO:tasks.rados.rados.0.smithi163.stdout:1380: done (2 left) 2023-10-09T09:11:03.050 INFO:tasks.rados.rados.0.smithi163.stdout:1382: done (1 left) 2023-10-09T09:11:03.050 INFO:tasks.rados.rados.0.smithi163.stdout:1383: snap_remove snap 165 2023-10-09T09:11:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:02 smithi181 bash[27712]: debug 2023-10-09T09:11:02.754+0000 7fde83035700 -1 mon.b@1(peon).osd e395 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:02 smithi181 bash[27712]: cluster 2023-10-09T09:11:01.759217+0000 osd.3 (osd.3) 43 : cluster [WRN] failed to encode map e394 with expected crc 2023-10-09T09:11:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:02 smithi181 bash[27712]: cluster 2023-10-09T09:11:01.761213+0000 mon.a (mon.0) 214 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2023-10-09T09:11:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:02 smithi181 bash[27712]: cluster 2023-10-09T09:11:01.762654+0000 osd.0 (osd.0) 49 : cluster [WRN] failed to encode map e394 with expected crc 2023-10-09T09:11:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:02 smithi181 bash[27712]: cluster 2023-10-09T09:11:01.764040+0000 osd.6 (osd.6) 31 : cluster [WRN] failed to encode map e394 with expected crc 2023-10-09T09:11:03.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:02 smithi181 bash[27712]: cluster 2023-10-09T09:11:01.765775+0000 osd.2 (osd.2) 70 : cluster [WRN] failed to encode map e394 with expected crc 2023-10-09T09:11:03.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:02 smithi181 bash[27712]: cluster 2023-10-09T09:11:01.900480+0000 osd.7 (osd.7) 46 : cluster [WRN] failed to encode map e394 with expected crc 2023-10-09T09:11:03.434 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:11:03.434 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (3m) - 9m 14.3M - 0.23.0 ba2b418f427c 82a8abc84166 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (3m) - 8m 40.2M - 8.3.5 dad864ee21e9 9d80a71ddd1d 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.vnodzj smithi163 running (7m) - 7m 17.6M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 892077b22450 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.ajqgsc smithi181 running (7m) - 7m 13.4M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 01d89554089b 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443,9283,8765 running (2m) - 15m 426M - 18.2.0-598-gb4fa663f 3060897fdf97 ca7a91dd3dab 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (2m) - 17m 475M - 18.2.0-598-gb4fa663f 3060897fdf97 fc427389eb0c 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (109s) - 17m 54.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 097aa5987723 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (16m) - 16m 72.4M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 443a80433a1f 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (12s) - 16m 20.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 adc323592e84 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (3m) - 9m 9728k - 1.3.1 1dbe0e931976 8a6fec29baf3 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (3m) - 9m 9276k - 1.3.1 1dbe0e931976 3f766d093db8 2023-10-09T09:11:03.435 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (15m) - 15m 94.7M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d07769e94b6f 2023-10-09T09:11:03.436 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (14m) - 14m 77.7M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7a2783c3c11c 2023-10-09T09:11:03.436 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (13m) - 13m 101M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 35670f565f8c 2023-10-09T09:11:03.436 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (13m) - 13m 68.5M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 402aa34f21f4 2023-10-09T09:11:03.436 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (12m) - 12m 94.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5dd6bc4b279 2023-10-09T09:11:03.436 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (11m) - 11m 96.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9f3e4fbba907 2023-10-09T09:11:03.436 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (11m) - 11m 67.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 30f1ac2f4524 2023-10-09T09:11:03.436 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (10m) - 10m 79.8M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 54105d827fdd 2023-10-09T09:11:03.436 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (2m) - 9m 54.3M - 2.33.4 514e6a882f6e cc81501e414f 2023-10-09T09:11:03.650 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:11:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:11:03] "GET /metrics HTTP/1.1" 200 36077 "" "Prometheus/2.33.4" 2023-10-09T09:11:03.928 INFO:tasks.rados.rados.0.smithi163.stdout:1383: done (1 left) 2023-10-09T09:11:03.928 INFO:tasks.rados.rados.0.smithi163.stdout:1384: snap_create 2023-10-09T09:11:03.929 INFO:tasks.rados.rados.0.smithi163.stdout:1381: finishing write tid 3 to smithi16367185-41 2023-10-09T09:11:03.929 INFO:tasks.rados.rados.0.smithi163.stdout:1381: finishing write tid 4 to smithi16367185-41 2023-10-09T09:11:03.929 INFO:tasks.rados.rados.0.smithi163.stdout:1381: finishing write tid 5 to smithi16367185-41 2023-10-09T09:11:03.929 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 99 (ObjNum 541 snap 169 seq_num 541) dirty exists 2023-10-09T09:11:03.929 INFO:tasks.rados.rados.0.smithi163.stdout:1381: left oid 41 (ObjNum 541 snap 169 seq_num 541) 2023-10-09T09:11:04.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: cluster 2023-10-09T09:11:02.759290+0000 osd.3 (osd.3) 44 : cluster [WRN] failed to encode map e395 with expected crc 2023-10-09T09:11:04.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: cluster 2023-10-09T09:11:02.761828+0000 mon.a (mon.0) 215 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2023-10-09T09:11:04.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: cluster 2023-10-09T09:11:02.765409+0000 osd.4 (osd.4) 77 : cluster [WRN] failed to encode map e395 with expected crc 2023-10-09T09:11:04.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: cluster 2023-10-09T09:11:02.775429+0000 osd.0 (osd.0) 50 : cluster [WRN] failed to encode map e395 with expected crc 2023-10-09T09:11:04.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: cluster 2023-10-09T09:11:02.782178+0000 osd.2 (osd.2) 71 : cluster [WRN] failed to encode map e395 with expected crc 2023-10-09T09:11:04.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: cluster 2023-10-09T09:11:02.789295+0000 osd.7 (osd.7) 47 : cluster [WRN] failed to encode map e395 with expected crc 2023-10-09T09:11:04.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: cluster 2023-10-09T09:11:02.800984+0000 osd.5 (osd.5) 38 : cluster [WRN] failed to encode map e395 with expected crc 2023-10-09T09:11:04.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: audit 2023-10-09T09:11:02.908311+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:11:04.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: audit 2023-10-09T09:11:02.915742+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:11:04.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: cluster 2023-10-09T09:11:03.242838+0000 mgr.y (mgr.34107) 70 : cluster [DBG] pgmap v143: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 2.0 MiB/s wr, 25 op/s 2023-10-09T09:11:04.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: audit 2023-10-09T09:11:03.413966+0000 mgr.y (mgr.34107) 71 : audit [DBG] from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:11:04.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: audit 2023-10-09T09:11:03.707993+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:11:04.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: audit 2023-10-09T09:11:03.713860+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:11:04.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:03 smithi181 bash[27712]: debug 2023-10-09T09:11:03.914+0000 7fde83035700 -1 mon.b@1(peon).osd e396 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:04.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T09:11:04.935 INFO:tasks.rados.rados.0.smithi163.stdout:1381: done (1 left) 2023-10-09T09:11:04.935 INFO:tasks.rados.rados.0.smithi163.stdout:1384: done (0 left) 2023-10-09T09:11:04.935 INFO:tasks.rados.rados.0.smithi163.stdout:1385: delete oid 44 current snap is 170 2023-10-09T09:11:04.947 INFO:tasks.rados.rados.0.smithi163.stdout:1385: done (0 left) 2023-10-09T09:11:04.947 INFO:tasks.rados.rados.0.smithi163.stdout:1386: write oid 12 current snap is 170 2023-10-09T09:11:04.947 INFO:tasks.rados.rados.0.smithi163.stdout:1386: seq_num 544 ranges {743008=796519,2200322=721849} 2023-10-09T09:11:04.960 INFO:tasks.rados.rados.0.smithi163.stdout:1386: writing smithi16367185-12 from 743008 to 1539527 tid 1 2023-10-09T09:11:04.969 INFO:tasks.rados.rados.0.smithi163.stdout:1386: writing smithi16367185-12 from 2200322 to 2922171 tid 2 2023-10-09T09:11:04.969 INFO:tasks.rados.rados.0.smithi163.stdout:1387: read oid 40 snap -1 2023-10-09T09:11:04.969 INFO:tasks.rados.rados.0.smithi163.stdout:1387: expect (ObjNum 537 snap 168 seq_num 537) 2023-10-09T09:11:04.969 INFO:tasks.rados.rados.0.smithi163.stdout:1388: rollback oid 2 current snap is 170 2023-10-09T09:11:04.969 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 2 to 158 2023-10-09T09:11:04.969 INFO:tasks.rados.rados.0.smithi163.stdout:1386: finishing write tid 1 to smithi16367185-12 2023-10-09T09:11:04.969 INFO:tasks.rados.rados.0.smithi163.stdout:1389: rollback oid 42 current snap is 170 2023-10-09T09:11:04.969 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 166 2023-10-09T09:11:04.970 INFO:tasks.rados.rados.0.smithi163.stdout:1390: read oid 45 snap -1 2023-10-09T09:11:04.970 INFO:tasks.rados.rados.0.smithi163.stdout:1390: expect deleted 2023-10-09T09:11:04.970 INFO:tasks.rados.rados.0.smithi163.stdout:1391: write oid 23 current snap is 170 2023-10-09T09:11:04.970 INFO:tasks.rados.rados.0.smithi163.stdout:1391: seq_num 545 ranges {759815=471768,1742451=465507,2917178=242637} 2023-10-09T09:11:04.976 INFO:tasks.rados.rados.0.smithi163.stdout:1391: writing smithi16367185-23 from 759815 to 1231583 tid 1 2023-10-09T09:11:04.982 INFO:tasks.rados.rados.0.smithi163.stdout:1391: writing smithi16367185-23 from 1742451 to 2207958 tid 2 2023-10-09T09:11:04.987 INFO:tasks.rados.rados.0.smithi163.stdout:1391: writing smithi16367185-23 from 2917178 to 3159815 tid 3 2023-10-09T09:11:04.987 INFO:tasks.rados.rados.0.smithi163.stdout:1392: snap_remove snap 168 2023-10-09T09:11:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:04 smithi181 bash[27712]: debug 2023-10-09T09:11:04.918+0000 7fde83035700 -1 mon.b@1(peon).osd e397 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:04 smithi181 bash[27712]: cluster 2023-10-09T09:11:03.924458+0000 osd.3 (osd.3) 45 : cluster [WRN] failed to encode map e396 with expected crc 2023-10-09T09:11:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:04 smithi181 bash[27712]: cluster 2023-10-09T09:11:03.925338+0000 osd.4 (osd.4) 78 : cluster [WRN] failed to encode map e396 with expected crc 2023-10-09T09:11:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:04 smithi181 bash[27712]: cluster 2023-10-09T09:11:03.926186+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2023-10-09T09:11:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:04 smithi181 bash[27712]: cluster 2023-10-09T09:11:04.243825+0000 osd.7 (osd.7) 48 : cluster [WRN] failed to encode map e396 with expected crc 2023-10-09T09:11:05.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:04 smithi181 bash[27712]: debug 2023-10-09T09:11:04.974+0000 7fde83035700 -1 mon.b@1(peon).osd e398 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:06.010 INFO:tasks.rados.rados.0.smithi163.stdout:1392: done (6 left) 2023-10-09T09:11:06.010 INFO:tasks.rados.rados.0.smithi163.stdout:1393: delete oid 19 current snap is 170 2023-10-09T09:11:06.010 INFO:tasks.rados.rados.0.smithi163.stdout:1386: finishing write tid 2 to smithi16367185-12 2023-10-09T09:11:06.010 INFO:tasks.rados.rados.0.smithi163.stdout:1386: finishing write tid 3 to smithi16367185-12 2023-10-09T09:11:06.010 INFO:tasks.rados.rados.0.smithi163.stdout:1386: finishing write tid 4 to smithi16367185-12 2023-10-09T09:11:06.010 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 589 (ObjNum 544 snap 170 seq_num 544) dirty exists 2023-10-09T09:11:06.010 INFO:tasks.rados.rados.0.smithi163.stdout:1386: left oid 12 (ObjNum 544 snap 170 seq_num 544) 2023-10-09T09:11:06.010 INFO:tasks.rados.rados.0.smithi163.stdout:1389: finishing rollback tid 0 to smithi16367185-42 2023-10-09T09:11:06.011 INFO:tasks.rados.rados.0.smithi163.stdout:1389: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:11:06.011 INFO:tasks.rados.rados.0.smithi163.stdout:1389: finishing rollback tid 2 to smithi16367185-42 2023-10-09T09:11:06.011 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 1088 (ObjNum 513 snap 160 seq_num 513) dirty exists 2023-10-09T09:11:06.011 INFO:tasks.rados.rados.0.smithi163.stdout:1388: finishing rollback tid 1 to smithi16367185-2 2023-10-09T09:11:06.011 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 369 (ObjNum 1 snap 21875 seq_num 698786080) dirty dne 2023-10-09T09:11:06.011 INFO:tasks.rados.rados.0.smithi163.stdout:1387: expect (ObjNum 537 snap 168 seq_num 537) 2023-10-09T09:11:06.118 INFO:tasks.rados.rados.0.smithi163.stdout:1391: finishing write tid 1 to smithi16367185-23 2023-10-09T09:11:06.118 INFO:tasks.rados.rados.0.smithi163.stdout:1391: finishing write tid 2 to smithi16367185-23 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:1391: finishing write tid 3 to smithi16367185-23 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:1391: finishing write tid 4 to smithi16367185-23 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:1391: finishing write tid 5 to smithi16367185-23 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 551 (ObjNum 545 snap 170 seq_num 545) dirty exists 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:1391: left oid 23 (ObjNum 545 snap 170 seq_num 545) 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:1386: done (6 left) 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:1387: done (5 left) 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:1388: done (4 left) 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:1389: done (3 left) 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:1390: done (2 left) 2023-10-09T09:11:06.119 INFO:tasks.rados.rados.0.smithi163.stdout:1391: done (1 left) 2023-10-09T09:11:06.120 INFO:tasks.rados.rados.0.smithi163.stdout:1393: done (0 left) 2023-10-09T09:11:06.120 INFO:tasks.rados.rados.0.smithi163.stdout:1394: read oid 9 snap -1 2023-10-09T09:11:06.120 INFO:tasks.rados.rados.0.smithi163.stdout:1394: expect deleted 2023-10-09T09:11:06.120 INFO:tasks.rados.rados.0.smithi163.stdout:1395: read oid 16 snap -1 2023-10-09T09:11:06.120 INFO:tasks.rados.rados.0.smithi163.stdout:1395: expect (ObjNum 505 snap 157 seq_num 505) 2023-10-09T09:11:06.120 INFO:tasks.rados.rados.0.smithi163.stdout:1396: write oid 24 current snap is 170 2023-10-09T09:11:06.120 INFO:tasks.rados.rados.0.smithi163.stdout:1396: seq_num 547 ranges {793429=622266,1859415=469176,2972024=221405} 2023-10-09T09:11:06.126 INFO:tasks.rados.rados.0.smithi163.stdout:1396: writing smithi16367185-24 from 793429 to 1415695 tid 1 2023-10-09T09:11:06.131 INFO:tasks.rados.rados.0.smithi163.stdout:1396: writing smithi16367185-24 from 1859415 to 2328591 tid 2 2023-10-09T09:11:06.136 INFO:tasks.rados.rados.0.smithi163.stdout:1396: writing smithi16367185-24 from 2972024 to 3193429 tid 3 2023-10-09T09:11:06.136 INFO:tasks.rados.rados.0.smithi163.stdout:1397: snap_remove snap 163 2023-10-09T09:11:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:04.534302+0000 osd.2 (osd.2) 72 : cluster [WRN] failed to encode map e396 with expected crc 2023-10-09T09:11:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:04.925474+0000 osd.3 (osd.3) 46 : cluster [WRN] failed to encode map e397 with expected crc 2023-10-09T09:11:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:04.928413+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:04.933211+0000 osd.2 (osd.2) 73 : cluster [WRN] failed to encode map e397 with expected crc 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:04.938916+0000 osd.5 (osd.5) 39 : cluster [WRN] failed to encode map e397 with expected crc 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:04.969285+0000 osd.7 (osd.7) 49 : cluster [WRN] failed to encode map e397 with expected crc 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:04.979296+0000 osd.3 (osd.3) 47 : cluster [WRN] failed to encode map e398 with expected crc 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:04.981338+0000 mon.a (mon.0) 222 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:04.987213+0000 osd.4 (osd.4) 79 : cluster [WRN] failed to encode map e398 with expected crc 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:05.243478+0000 mgr.y (mgr.34107) 72 : cluster [DBG] pgmap v147: 89 pgs: 89 active+clean; 229 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:05.278125+0000 osd.7 (osd.7) 50 : cluster [WRN] failed to encode map e398 with expected crc 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:05.448527+0000 osd.1 (osd.1) 32 : cluster [WRN] failed to encode map e398 with expected crc 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:05.465911+0000 osd.6 (osd.6) 32 : cluster [WRN] failed to encode map e398 with expected crc 2023-10-09T09:11:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:05 smithi181 bash[27712]: cluster 2023-10-09T09:11:05.544050+0000 osd.0 (osd.0) 51 : cluster [WRN] failed to encode map e398 with expected crc 2023-10-09T09:11:06.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:06 smithi181 bash[27712]: debug 2023-10-09T09:11:05.994+0000 7fde83035700 -1 mon.b@1(peon).osd e399 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:07.023 INFO:tasks.rados.rados.0.smithi163.stdout:1397: done (3 left) 2023-10-09T09:11:07.024 INFO:tasks.rados.rados.0.smithi163.stdout:1398: snap_remove snap 169 2023-10-09T09:11:07.217 INFO:teuthology.orchestra.run.smithi163.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:11:07.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:07 smithi181 bash[27712]: cluster 2023-10-09T09:11:05.525856+0000 osd.2 (osd.2) 74 : cluster [WRN] failed to encode map e398 with expected crc 2023-10-09T09:11:07.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:07 smithi181 bash[27712]: cluster 2023-10-09T09:11:06.002622+0000 osd.3 (osd.3) 48 : cluster [WRN] failed to encode map e399 with expected crc 2023-10-09T09:11:07.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:07 smithi181 bash[27712]: cluster 2023-10-09T09:11:06.004772+0000 mon.a (mon.0) 223 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2023-10-09T09:11:07.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:07 smithi181 bash[27712]: cluster 2023-10-09T09:11:06.005853+0000 osd.4 (osd.4) 80 : cluster [WRN] failed to encode map e399 with expected crc 2023-10-09T09:11:07.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:07 smithi181 bash[27712]: cluster 2023-10-09T09:11:06.015951+0000 osd.2 (osd.2) 75 : cluster [WRN] failed to encode map e399 with expected crc 2023-10-09T09:11:07.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:07 smithi181 bash[27712]: debug 2023-10-09T09:11:07.006+0000 7fde83035700 -1 mon.b@1(peon).osd e400 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:07.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T09:11:08.026 INFO:tasks.rados.rados.0.smithi163.stdout:1398: done (3 left) 2023-10-09T09:11:08.027 INFO:tasks.rados.rados.0.smithi163.stdout:1399: write oid 36 current snap is 170 2023-10-09T09:11:08.027 INFO:tasks.rados.rados.0.smithi163.stdout:1399: seq_num 548 ranges {410236=413868,1234250=729187,2715800=423063} 2023-10-09T09:11:08.032 INFO:tasks.rados.rados.0.smithi163.stdout:1399: writing smithi16367185-36 from 410236 to 824104 tid 1 2023-10-09T09:11:08.040 INFO:tasks.rados.rados.0.smithi163.stdout:1399: writing smithi16367185-36 from 1234250 to 1963437 tid 2 2023-10-09T09:11:08.048 INFO:tasks.rados.rados.0.smithi163.stdout:1399: writing smithi16367185-36 from 2715800 to 3138863 tid 3 2023-10-09T09:11:08.048 INFO:tasks.rados.rados.0.smithi163.stdout:1400: snap_create 2023-10-09T09:11:08.048 INFO:tasks.rados.rados.0.smithi163.stdout:1395: expect (ObjNum 505 snap 157 seq_num 505) 2023-10-09T09:11:08.255 INFO:tasks.rados.rados.0.smithi163.stdout:1396: finishing write tid 1 to smithi16367185-24 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:1396: finishing write tid 2 to smithi16367185-24 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:1396: finishing write tid 3 to smithi16367185-24 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:1396: finishing write tid 4 to smithi16367185-24 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:1396: finishing write tid 5 to smithi16367185-24 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 355 (ObjNum 547 snap 170 seq_num 547) dirty exists 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:1396: left oid 24 (ObjNum 547 snap 170 seq_num 547) 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:1399: finishing write tid 1 to smithi16367185-36 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:1399: finishing write tid 2 to smithi16367185-36 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:1399: finishing write tid 3 to smithi16367185-36 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:1399: finishing write tid 4 to smithi16367185-36 2023-10-09T09:11:08.256 INFO:tasks.rados.rados.0.smithi163.stdout:1399: finishing write tid 5 to smithi16367185-36 2023-10-09T09:11:08.257 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 417 (ObjNum 548 snap 170 seq_num 548) dirty exists 2023-10-09T09:11:08.257 INFO:tasks.rados.rados.0.smithi163.stdout:1399: left oid 36 (ObjNum 548 snap 170 seq_num 548) 2023-10-09T09:11:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:08 smithi181 bash[27712]: debug 2023-10-09T09:11:08.010+0000 7fde83035700 -1 mon.b@1(peon).osd e401 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:08 smithi181 bash[27712]: cluster 2023-10-09T09:11:07.013231+0000 osd.3 (osd.3) 49 : cluster [WRN] failed to encode map e400 with expected crc 2023-10-09T09:11:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:08 smithi181 bash[27712]: cluster 2023-10-09T09:11:07.016460+0000 mon.a (mon.0) 224 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2023-10-09T09:11:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:08 smithi181 bash[27712]: audit 2023-10-09T09:11:07.209925+0000 mgr.y (mgr.34107) 73 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:11:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:08 smithi181 bash[27712]: cephadm 2023-10-09T09:11:07.210552+0000 mgr.y (mgr.34107) 74 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:11:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:08 smithi181 bash[27712]: audit 2023-10-09T09:11:07.216879+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:11:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:08 smithi181 bash[27712]: cluster 2023-10-09T09:11:07.243934+0000 mgr.y (mgr.34107) 75 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 242 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.9 MiB/s wr, 25 op/s 2023-10-09T09:11:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:08 smithi181 bash[27712]: cluster 2023-10-09T09:11:07.277263+0000 osd.7 (osd.7) 51 : cluster [WRN] failed to encode map e400 with expected crc 2023-10-09T09:11:09.031 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:11:08 smithi181 bash[54907]: ts=2023-10-09T09:11:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:11:09.045 INFO:tasks.rados.rados.0.smithi163.stdout:1394: done (4 left) 2023-10-09T09:11:09.046 INFO:tasks.rados.rados.0.smithi163.stdout:1395: done (3 left) 2023-10-09T09:11:09.046 INFO:tasks.rados.rados.0.smithi163.stdout:1396: done (2 left) 2023-10-09T09:11:09.046 INFO:tasks.rados.rados.0.smithi163.stdout:1399: done (1 left) 2023-10-09T09:11:09.046 INFO:tasks.rados.rados.0.smithi163.stdout:1400: done (0 left) 2023-10-09T09:11:09.046 INFO:tasks.rados.rados.0.smithi163.stdout:1401: write oid 24 current snap is 171 2023-10-09T09:11:09.046 INFO:tasks.rados.rados.0.smithi163.stdout:1401: seq_num 549 ranges {427043=489117,1576379=472845,2626870=600173} 2023-10-09T09:11:09.054 INFO:tasks.rados.rados.0.smithi163.stdout:1401: writing smithi16367185-24 from 427043 to 916160 tid 1 2023-10-09T09:11:09.061 INFO:tasks.rados.rados.0.smithi163.stdout:1401: writing smithi16367185-24 from 1576379 to 2049224 tid 2 2023-10-09T09:11:09.069 INFO:tasks.rados.rados.0.smithi163.stdout:1401: writing smithi16367185-24 from 2626870 to 3227043 tid 3 2023-10-09T09:11:09.069 INFO:tasks.rados.rados.0.smithi163.stdout:1402: read oid 49 snap -1 2023-10-09T09:11:09.069 INFO:tasks.rados.rados.0.smithi163.stdout:1402: expect (ObjNum 448 snap 144 seq_num 448) 2023-10-09T09:11:09.069 INFO:tasks.rados.rados.0.smithi163.stdout:1403: write oid 9 current snap is 171 2023-10-09T09:11:09.069 INFO:tasks.rados.rados.0.smithi163.stdout:1401: finishing write tid 1 to smithi16367185-24 2023-10-09T09:11:09.070 INFO:tasks.rados.rados.0.smithi163.stdout:1403: seq_num 550 ranges {443850=564366,1634861=732856,2770646=473204} 2023-10-09T09:11:09.075 INFO:tasks.rados.rados.0.smithi163.stdout:1403: writing smithi16367185-9 from 443850 to 1008216 tid 1 2023-10-09T09:11:09.083 INFO:tasks.rados.rados.0.smithi163.stdout:1403: writing smithi16367185-9 from 1634861 to 2367717 tid 2 2023-10-09T09:11:09.088 INFO:tasks.rados.rados.0.smithi163.stdout:1403: writing smithi16367185-9 from 2770646 to 3243850 tid 3 2023-10-09T09:11:09.089 INFO:tasks.rados.rados.0.smithi163.stdout:1404: read oid 12 snap 160 2023-10-09T09:11:09.089 INFO:tasks.rados.rados.0.smithi163.stdout:1404: expect (ObjNum 460 snap 147 seq_num 460) 2023-10-09T09:11:09.089 INFO:tasks.rados.rados.0.smithi163.stdout:1405: delete oid 10 current snap is 171 2023-10-09T09:11:09.089 INFO:tasks.rados.rados.0.smithi163.stdout:1401: finishing write tid 2 to smithi16367185-24 2023-10-09T09:11:09.089 INFO:tasks.rados.rados.0.smithi163.stdout:1401: finishing write tid 3 to smithi16367185-24 2023-10-09T09:11:09.089 INFO:tasks.rados.rados.0.smithi163.stdout:1401: finishing write tid 4 to smithi16367185-24 2023-10-09T09:11:09.089 INFO:tasks.rados.rados.0.smithi163.stdout:1401: finishing write tid 5 to smithi16367185-24 2023-10-09T09:11:09.089 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 362 (ObjNum 549 snap 171 seq_num 549) dirty exists 2023-10-09T09:11:09.089 INFO:tasks.rados.rados.0.smithi163.stdout:1401: left oid 24 (ObjNum 549 snap 171 seq_num 549) 2023-10-09T09:11:09.089 INFO:tasks.rados.rados.0.smithi163.stdout:1402: expect (ObjNum 448 snap 144 seq_num 448) 2023-10-09T09:11:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:09 smithi181 bash[27712]: cluster 2023-10-09T09:11:08.018029+0000 osd.3 (osd.3) 50 : cluster [WRN] failed to encode map e401 with expected crc 2023-10-09T09:11:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:09 smithi181 bash[27712]: cluster 2023-10-09T09:11:08.021407+0000 mon.a (mon.0) 226 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2023-10-09T09:11:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:09 smithi181 bash[27712]: cluster 2023-10-09T09:11:08.024822+0000 osd.1 (osd.1) 33 : cluster [WRN] failed to encode map e401 with expected crc 2023-10-09T09:11:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:09 smithi181 bash[27712]: cluster 2023-10-09T09:11:08.034835+0000 osd.0 (osd.0) 52 : cluster [WRN] failed to encode map e401 with expected crc 2023-10-09T09:11:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:09 smithi181 bash[27712]: cluster 2023-10-09T09:11:08.053854+0000 osd.5 (osd.5) 40 : cluster [WRN] failed to encode map e401 with expected crc 2023-10-09T09:11:09.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:09 smithi181 bash[27712]: cluster 2023-10-09T09:11:08.311687+0000 osd.7 (osd.7) 52 : cluster [WRN] failed to encode map e401 with expected crc 2023-10-09T09:11:09.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:09 smithi181 bash[27712]: debug 2023-10-09T09:11:09.030+0000 7fde83035700 -1 mon.b@1(peon).osd e402 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:09.416 INFO:tasks.rados.rados.0.smithi163.stdout:1403: finishing write tid 1 to smithi16367185-9 2023-10-09T09:11:09.416 INFO:tasks.rados.rados.0.smithi163.stdout:1401: done (4 left) 2023-10-09T09:11:09.416 INFO:tasks.rados.rados.0.smithi163.stdout:1402: done (3 left) 2023-10-09T09:11:09.416 INFO:tasks.rados.rados.0.smithi163.stdout:1405: done (2 left) 2023-10-09T09:11:09.416 INFO:tasks.rados.rados.0.smithi163.stdout:1406: snap_remove snap 170 2023-10-09T09:11:09.995 INFO:tasks.rados.rados.0.smithi163.stdout:1406: done (2 left) 2023-10-09T09:11:09.995 INFO:tasks.rados.rados.0.smithi163.stdout:1407: read oid 50 snap -1 2023-10-09T09:11:09.995 INFO:tasks.rados.rados.0.smithi163.stdout:1407: expect deleted 2023-10-09T09:11:09.995 INFO:tasks.rados.rados.0.smithi163.stdout:1408: delete oid 8 current snap is 171 2023-10-09T09:11:09.996 INFO:tasks.rados.rados.0.smithi163.stdout:1403: finishing write tid 2 to smithi16367185-9 2023-10-09T09:11:09.996 INFO:tasks.rados.rados.0.smithi163.stdout:1403: finishing write tid 3 to smithi16367185-9 2023-10-09T09:11:09.996 INFO:tasks.rados.rados.0.smithi163.stdout:1403: finishing write tid 4 to smithi16367185-9 2023-10-09T09:11:09.996 INFO:tasks.rados.rados.0.smithi163.stdout:1403: finishing write tid 5 to smithi16367185-9 2023-10-09T09:11:09.996 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 366 (ObjNum 550 snap 171 seq_num 550) dirty exists 2023-10-09T09:11:09.996 INFO:tasks.rados.rados.0.smithi163.stdout:1403: left oid 9 (ObjNum 550 snap 171 seq_num 550) 2023-10-09T09:11:09.996 INFO:tasks.rados.rados.0.smithi163.stdout:1404: expect (ObjNum 460 snap 147 seq_num 460) 2023-10-09T09:11:10.265 INFO:tasks.rados.rados.0.smithi163.stdout:1403: done (3 left) 2023-10-09T09:11:10.265 INFO:tasks.rados.rados.0.smithi163.stdout:1404: done (2 left) 2023-10-09T09:11:10.265 INFO:tasks.rados.rados.0.smithi163.stdout:1407: done (1 left) 2023-10-09T09:11:10.265 INFO:tasks.rados.rados.0.smithi163.stdout:1408: done (0 left) 2023-10-09T09:11:10.265 INFO:tasks.rados.rados.0.smithi163.stdout:1409: delete oid 24 current snap is 171 2023-10-09T09:11:10.268 INFO:tasks.rados.rados.0.smithi163.stdout:1409: done (0 left) 2023-10-09T09:11:10.268 INFO:tasks.rados.rados.0.smithi163.stdout:1410: write oid 34 current snap is 171 2023-10-09T09:11:10.268 INFO:tasks.rados.rados.0.smithi163.stdout:1410: seq_num 554 ranges {511078=465362,1752436=740194,3163985=147093} 2023-10-09T09:11:10.279 INFO:tasks.rados.rados.0.smithi163.stdout:1410: writing smithi16367185-34 from 511078 to 976440 tid 1 2023-10-09T09:11:10.287 INFO:tasks.rados.rados.0.smithi163.stdout:1410: writing smithi16367185-34 from 1752436 to 2492630 tid 2 2023-10-09T09:11:10.292 INFO:tasks.rados.rados.0.smithi163.stdout:1410: writing smithi16367185-34 from 3163985 to 3311078 tid 3 2023-10-09T09:11:10.292 INFO:tasks.rados.rados.0.smithi163.stdout:1411: write oid 8 current snap is 171 2023-10-09T09:11:10.292 INFO:tasks.rados.rados.0.smithi163.stdout:1411: seq_num 555 ranges {527885=656964,1927271=483852,3191408=136477} 2023-10-09T09:11:10.299 INFO:tasks.rados.rados.0.smithi163.stdout:1411: writing smithi16367185-8 from 527885 to 1184849 tid 1 2023-10-09T09:11:10.306 INFO:tasks.rados.rados.0.smithi163.stdout:1411: writing smithi16367185-8 from 1927271 to 2411123 tid 2 2023-10-09T09:11:10.311 INFO:tasks.rados.rados.0.smithi163.stdout:1411: writing smithi16367185-8 from 3191408 to 3327885 tid 3 2023-10-09T09:11:10.312 INFO:tasks.rados.rados.0.smithi163.stdout:1412: read oid 28 snap -1 2023-10-09T09:11:10.312 INFO:tasks.rados.rados.0.smithi163.stdout:1412: expect deleted 2023-10-09T09:11:10.312 INFO:tasks.rados.rados.0.smithi163.stdout:1413: snap_remove snap 161 2023-10-09T09:11:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:09 smithi181 bash[27712]: debug 2023-10-09T09:11:09.974+0000 7fde83035700 -1 mon.b@1(peon).osd e403 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:10 smithi181 bash[27712]: cluster 2023-10-09T09:11:08.478203+0000 osd.2 (osd.2) 76 : cluster [WRN] failed to encode map e401 with expected crc 2023-10-09T09:11:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:10 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.037742+0000 osd.3 (osd.3) 51 : cluster [WRN] failed to encode map e402 with expected crc 2023-10-09T09:11:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:10 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.040187+0000 mon.a (mon.0) 227 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2023-10-09T09:11:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:10 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.043923+0000 osd.0 (osd.0) 53 : cluster [WRN] failed to encode map e402 with expected crc 2023-10-09T09:11:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:10 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.090540+0000 osd.2 (osd.2) 77 : cluster [WRN] failed to encode map e402 with expected crc 2023-10-09T09:11:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:10 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.244310+0000 mgr.y (mgr.34107) 76 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 242 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 2.4 MiB/s wr, 20 op/s 2023-10-09T09:11:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:10 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.987489+0000 mon.a (mon.0) 228 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2023-10-09T09:11:11.064 INFO:tasks.rados.rados.0.smithi163.stdout:1413: done (3 left) 2023-10-09T09:11:11.065 INFO:tasks.rados.rados.0.smithi163.stdout:1414: read oid 17 snap -1 2023-10-09T09:11:11.065 INFO:tasks.rados.rados.0.smithi163.stdout:1414: expect (ObjNum 504 snap 157 seq_num 504) 2023-10-09T09:11:11.065 INFO:tasks.rados.rados.0.smithi163.stdout:1415: snap_remove snap 162 2023-10-09T09:11:11.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:11 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.532744+0000 osd.6 (osd.6) 33 : cluster [WRN] failed to encode map e402 with expected crc 2023-10-09T09:11:11.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:11 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.982000+0000 osd.3 (osd.3) 52 : cluster [WRN] failed to encode map e403 with expected crc 2023-10-09T09:11:11.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:11 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.990285+0000 osd.2 (osd.2) 78 : cluster [WRN] failed to encode map e403 with expected crc 2023-10-09T09:11:11.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:11 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.999301+0000 osd.5 (osd.5) 41 : cluster [WRN] failed to encode map e403 with expected crc 2023-10-09T09:11:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:11 smithi181 bash[27712]: cluster 2023-10-09T09:11:09.999304+0000 osd.7 (osd.7) 53 : cluster [WRN] failed to encode map e403 with expected crc 2023-10-09T09:11:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:11 smithi181 bash[27712]: audit 2023-10-09T09:11:10.332325+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:11 smithi181 bash[27712]: debug 2023-10-09T09:11:11.046+0000 7fde83035700 -1 mon.b@1(peon).osd e404 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:12.069 INFO:tasks.rados.rados.0.smithi163.stdout:1415: done (4 left) 2023-10-09T09:11:12.070 INFO:tasks.rados.rados.0.smithi163.stdout:1416: snap_create 2023-10-09T09:11:12.070 INFO:tasks.rados.rados.0.smithi163.stdout:1410: finishing write tid 1 to smithi16367185-34 2023-10-09T09:11:12.070 INFO:tasks.rados.rados.0.smithi163.stdout:1410: finishing write tid 2 to smithi16367185-34 2023-10-09T09:11:12.070 INFO:tasks.rados.rados.0.smithi163.stdout:1410: finishing write tid 3 to smithi16367185-34 2023-10-09T09:11:12.070 INFO:tasks.rados.rados.0.smithi163.stdout:1410: finishing write tid 4 to smithi16367185-34 2023-10-09T09:11:12.070 INFO:tasks.rados.rados.0.smithi163.stdout:1410: finishing write tid 5 to smithi16367185-34 2023-10-09T09:11:12.070 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 564 (ObjNum 554 snap 171 seq_num 554) dirty exists 2023-10-09T09:11:12.070 INFO:tasks.rados.rados.0.smithi163.stdout:1410: left oid 34 (ObjNum 554 snap 171 seq_num 554) 2023-10-09T09:11:12.070 INFO:tasks.rados.rados.0.smithi163.stdout:1411: finishing write tid 1 to smithi16367185-8 2023-10-09T09:11:12.070 INFO:tasks.rados.rados.0.smithi163.stdout:1411: finishing write tid 2 to smithi16367185-8 2023-10-09T09:11:12.071 INFO:tasks.rados.rados.0.smithi163.stdout:1411: finishing write tid 3 to smithi16367185-8 2023-10-09T09:11:12.071 INFO:tasks.rados.rados.0.smithi163.stdout:1411: finishing write tid 4 to smithi16367185-8 2023-10-09T09:11:12.071 INFO:tasks.rados.rados.0.smithi163.stdout:1411: finishing write tid 5 to smithi16367185-8 2023-10-09T09:11:12.071 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 678 (ObjNum 555 snap 171 seq_num 555) dirty exists 2023-10-09T09:11:12.071 INFO:tasks.rados.rados.0.smithi163.stdout:1411: left oid 8 (ObjNum 555 snap 171 seq_num 555) 2023-10-09T09:11:12.071 INFO:tasks.rados.rados.0.smithi163.stdout:1414: expect (ObjNum 504 snap 157 seq_num 504) 2023-10-09T09:11:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:12 smithi181 bash[27712]: debug 2023-10-09T09:11:12.054+0000 7fde83035700 -1 mon.b@1(peon).osd e405 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:12 smithi181 bash[27712]: cluster 2023-10-09T09:11:11.053806+0000 osd.3 (osd.3) 53 : cluster [WRN] failed to encode map e404 with expected crc 2023-10-09T09:11:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:12 smithi181 bash[27712]: cluster 2023-10-09T09:11:11.055544+0000 mon.a (mon.0) 230 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2023-10-09T09:11:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:12 smithi181 bash[27712]: cluster 2023-10-09T09:11:11.058513+0000 osd.7 (osd.7) 54 : cluster [WRN] failed to encode map e404 with expected crc 2023-10-09T09:11:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:12 smithi181 bash[27712]: cluster 2023-10-09T09:11:11.245481+0000 mgr.y (mgr.34107) 77 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 250 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2023-10-09T09:11:13.075 INFO:tasks.rados.rados.0.smithi163.stdout:1410: done (4 left) 2023-10-09T09:11:13.075 INFO:tasks.rados.rados.0.smithi163.stdout:1411: done (3 left) 2023-10-09T09:11:13.075 INFO:tasks.rados.rados.0.smithi163.stdout:1412: done (2 left) 2023-10-09T09:11:13.075 INFO:tasks.rados.rados.0.smithi163.stdout:1414: done (1 left) 2023-10-09T09:11:13.075 INFO:tasks.rados.rados.0.smithi163.stdout:1416: done (0 left) 2023-10-09T09:11:13.075 INFO:tasks.rados.rados.0.smithi163.stdout:1417: delete oid 36 current snap is 172 2023-10-09T09:11:13.086 INFO:tasks.rados.rados.0.smithi163.stdout:1417: done (0 left) 2023-10-09T09:11:13.086 INFO:tasks.rados.rados.0.smithi163.stdout:1418: read oid 29 snap 153 2023-10-09T09:11:13.086 INFO:tasks.rados.rados.0.smithi163.stdout:1418: expect (ObjNum 467 snap 151 seq_num 467) 2023-10-09T09:11:13.086 INFO:tasks.rados.rados.0.smithi163.stdout:1419: write oid 27 current snap is 172 2023-10-09T09:11:13.086 INFO:tasks.rados.rados.0.smithi163.stdout:1419: seq_num 557 ranges {561499=407462,1527882=487521,2729901=557217} 2023-10-09T09:11:13.092 INFO:tasks.rados.rados.0.smithi163.stdout:1419: writing smithi16367185-27 from 561499 to 968961 tid 1 2023-10-09T09:11:13.099 INFO:tasks.rados.rados.0.smithi163.stdout:1419: writing smithi16367185-27 from 1527882 to 2015403 tid 2 2023-10-09T09:11:13.107 INFO:tasks.rados.rados.0.smithi163.stdout:1419: writing smithi16367185-27 from 2729901 to 3287118 tid 3 2023-10-09T09:11:13.107 INFO:tasks.rados.rados.0.smithi163.stdout:1420: read oid 43 snap -1 2023-10-09T09:11:13.107 INFO:tasks.rados.rados.0.smithi163.stdout:1420: expect (ObjNum 482 snap 152 seq_num 482) 2023-10-09T09:11:13.107 INFO:tasks.rados.rados.0.smithi163.stdout:1421: delete oid 44 current snap is 172 2023-10-09T09:11:13.107 INFO:tasks.rados.rados.0.smithi163.stdout:1418: expect (ObjNum 467 snap 151 seq_num 467) 2023-10-09T09:11:13.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:13 smithi181 bash[27712]: debug 2023-10-09T09:11:13.058+0000 7fde83035700 -1 mon.b@1(peon).osd e406 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:13.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:13 smithi181 bash[27712]: cluster 2023-10-09T09:11:12.064646+0000 osd.3 (osd.3) 54 : cluster [WRN] failed to encode map e405 with expected crc 2023-10-09T09:11:13.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:13 smithi181 bash[27712]: cluster 2023-10-09T09:11:12.065570+0000 osd.4 (osd.4) 81 : cluster [WRN] failed to encode map e405 with expected crc 2023-10-09T09:11:13.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:13 smithi181 bash[27712]: cluster 2023-10-09T09:11:12.069606+0000 mon.a (mon.0) 231 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2023-10-09T09:11:13.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:13 smithi181 bash[27712]: cluster 2023-10-09T09:11:12.321625+0000 osd.7 (osd.7) 55 : cluster [WRN] failed to encode map e405 with expected crc 2023-10-09T09:11:13.338 INFO:tasks.rados.rados.0.smithi163.stdout:1419: finishing write tid 1 to smithi16367185-27 2023-10-09T09:11:13.338 INFO:tasks.rados.rados.0.smithi163.stdout:1419: finishing write tid 2 to smithi16367185-27 2023-10-09T09:11:13.338 INFO:tasks.rados.rados.0.smithi163.stdout:1419: finishing write tid 3 to smithi16367185-27 2023-10-09T09:11:13.338 INFO:tasks.rados.rados.0.smithi163.stdout:1419: finishing write tid 4 to smithi16367185-27 2023-10-09T09:11:13.338 INFO:tasks.rados.rados.0.smithi163.stdout:1419: finishing write tid 5 to smithi16367185-27 2023-10-09T09:11:13.338 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 488 (ObjNum 557 snap 172 seq_num 557) dirty exists 2023-10-09T09:11:13.338 INFO:tasks.rados.rados.0.smithi163.stdout:1419: left oid 27 (ObjNum 557 snap 172 seq_num 557) 2023-10-09T09:11:13.338 INFO:tasks.rados.rados.0.smithi163.stdout:1420: expect (ObjNum 482 snap 152 seq_num 482) 2023-10-09T09:11:13.410 INFO:tasks.rados.rados.0.smithi163.stdout:1418: done (3 left) 2023-10-09T09:11:13.411 INFO:tasks.rados.rados.0.smithi163.stdout:1419: done (2 left) 2023-10-09T09:11:13.411 INFO:tasks.rados.rados.0.smithi163.stdout:1420: done (1 left) 2023-10-09T09:11:13.411 INFO:tasks.rados.rados.0.smithi163.stdout:1421: done (0 left) 2023-10-09T09:11:13.411 INFO:tasks.rados.rados.0.smithi163.stdout:1422: write oid 9 current snap is 172 2023-10-09T09:11:13.411 INFO:tasks.rados.rados.0.smithi163.stdout:1422: seq_num 559 ranges {595113=557960,1644846=491190,2784747=610366} 2023-10-09T09:11:13.421 INFO:tasks.rados.rados.0.smithi163.stdout:1422: writing smithi16367185-9 from 595113 to 1153073 tid 1 2023-10-09T09:11:13.427 INFO:tasks.rados.rados.0.smithi163.stdout:1422: writing smithi16367185-9 from 1644846 to 2136036 tid 2 2023-10-09T09:11:13.435 INFO:tasks.rados.rados.0.smithi163.stdout:1422: writing smithi16367185-9 from 2784747 to 3395113 tid 3 2023-10-09T09:11:13.436 INFO:tasks.rados.rados.0.smithi163.stdout:1423: snap_remove snap 158 2023-10-09T09:11:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:11:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:11:13] "GET /metrics HTTP/1.1" 200 36071 "" "Prometheus/2.33.4" 2023-10-09T09:11:14.094 INFO:tasks.rados.rados.0.smithi163.stdout:1423: done (1 left) 2023-10-09T09:11:14.094 INFO:tasks.rados.rados.0.smithi163.stdout:1424: read oid 15 snap -1 2023-10-09T09:11:14.094 INFO:tasks.rados.rados.0.smithi163.stdout:1424: expect (ObjNum 399 snap 121 seq_num 399) 2023-10-09T09:11:14.094 INFO:tasks.rados.rados.0.smithi163.stdout:1425: write oid 19 current snap is 172 2023-10-09T09:11:14.094 INFO:tasks.rados.rados.0.smithi163.stdout:1422: finishing write tid 1 to smithi16367185-9 2023-10-09T09:11:14.095 INFO:tasks.rados.rados.0.smithi163.stdout:1422: finishing write tid 2 to smithi16367185-9 2023-10-09T09:11:14.095 INFO:tasks.rados.rados.0.smithi163.stdout:1422: finishing write tid 3 to smithi16367185-9 2023-10-09T09:11:14.095 INFO:tasks.rados.rados.0.smithi163.stdout:1422: finishing write tid 4 to smithi16367185-9 2023-10-09T09:11:14.095 INFO:tasks.rados.rados.0.smithi163.stdout:1425: seq_num 560 ranges {611920=633209,1703328=634848,2812170=507386} 2023-10-09T09:11:14.102 INFO:tasks.rados.rados.0.smithi163.stdout:1425: writing smithi16367185-19 from 611920 to 1245129 tid 1 2023-10-09T09:11:14.109 INFO:tasks.rados.rados.0.smithi163.stdout:1425: writing smithi16367185-19 from 1703328 to 2338176 tid 2 2023-10-09T09:11:14.115 INFO:tasks.rados.rados.0.smithi163.stdout:1425: writing smithi16367185-19 from 2812170 to 3319556 tid 3 2023-10-09T09:11:14.115 INFO:tasks.rados.rados.0.smithi163.stdout:1426: write oid 47 current snap is 172 2023-10-09T09:11:14.116 INFO:tasks.rados.rados.0.smithi163.stdout:1422: finishing write tid 5 to smithi16367185-9 2023-10-09T09:11:14.116 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 378 (ObjNum 559 snap 172 seq_num 559) dirty exists 2023-10-09T09:11:14.116 INFO:tasks.rados.rados.0.smithi163.stdout:1422: left oid 9 (ObjNum 559 snap 172 seq_num 559) 2023-10-09T09:11:14.116 INFO:tasks.rados.rados.0.smithi163.stdout:1426: seq_num 561 ranges {628727=708458,2045457=494859,3123240=305487} 2023-10-09T09:11:14.123 INFO:tasks.rados.rados.0.smithi163.stdout:1426: writing smithi16367185-47 from 628727 to 1337185 tid 1 2023-10-09T09:11:14.130 INFO:tasks.rados.rados.0.smithi163.stdout:1426: writing smithi16367185-47 from 2045457 to 2540316 tid 2 2023-10-09T09:11:14.136 INFO:tasks.rados.rados.0.smithi163.stdout:1426: writing smithi16367185-47 from 3123240 to 3428727 tid 3 2023-10-09T09:11:14.136 INFO:tasks.rados.rados.0.smithi163.stdout:1422: done (3 left) 2023-10-09T09:11:14.136 INFO:tasks.rados.rados.0.smithi163.stdout:1427: read oid 28 snap -1 2023-10-09T09:11:14.136 INFO:tasks.rados.rados.0.smithi163.stdout:1427: expect deleted 2023-10-09T09:11:14.136 INFO:tasks.rados.rados.0.smithi163.stdout:1428: write oid 50 current snap is 172 2023-10-09T09:11:14.136 INFO:tasks.rados.rados.0.smithi163.stdout:1425: finishing write tid 1 to smithi16367185-19 2023-10-09T09:11:14.136 INFO:tasks.rados.rados.0.smithi163.stdout:1425: finishing write tid 2 to smithi16367185-19 2023-10-09T09:11:14.137 INFO:tasks.rados.rados.0.smithi163.stdout:1424: expect (ObjNum 399 snap 121 seq_num 399) 2023-10-09T09:11:14.335 INFO:tasks.rados.rados.0.smithi163.stdout:1425: finishing write tid 3 to smithi16367185-19 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1425: finishing write tid 4 to smithi16367185-19 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1425: finishing write tid 5 to smithi16367185-19 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 492 (ObjNum 560 snap 172 seq_num 560) dirty exists 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1425: left oid 19 (ObjNum 560 snap 172 seq_num 560) 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1426: finishing write tid 1 to smithi16367185-47 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1426: finishing write tid 2 to smithi16367185-47 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1426: finishing write tid 3 to smithi16367185-47 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1426: finishing write tid 4 to smithi16367185-47 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1426: finishing write tid 5 to smithi16367185-47 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 1124 (ObjNum 561 snap 172 seq_num 561) dirty exists 2023-10-09T09:11:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1426: left oid 47 (ObjNum 561 snap 172 seq_num 561) 2023-10-09T09:11:14.337 INFO:tasks.rados.rados.0.smithi163.stdout:1428: seq_num 562 ranges {645534=783707,2103939=638517,3150663=294871} 2023-10-09T09:11:14.349 INFO:tasks.rados.rados.0.smithi163.stdout:1428: writing smithi16367185-50 from 645534 to 1429241 tid 1 2023-10-09T09:11:14.358 INFO:tasks.rados.rados.0.smithi163.stdout:1428: writing smithi16367185-50 from 2103939 to 2742456 tid 2 2023-10-09T09:11:14.362 INFO:tasks.rados.rados.0.smithi163.stdout:1428: writing smithi16367185-50 from 3150663 to 3445534 tid 3 2023-10-09T09:11:14.362 INFO:tasks.rados.rados.0.smithi163.stdout:1424: done (4 left) 2023-10-09T09:11:14.362 INFO:tasks.rados.rados.0.smithi163.stdout:1425: done (3 left) 2023-10-09T09:11:14.362 INFO:tasks.rados.rados.0.smithi163.stdout:1426: done (2 left) 2023-10-09T09:11:14.362 INFO:tasks.rados.rados.0.smithi163.stdout:1427: done (1 left) 2023-10-09T09:11:14.363 INFO:tasks.rados.rados.0.smithi163.stdout:1429: rollback oid 1 current snap is 172 2023-10-09T09:11:14.363 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 1 to 167 2023-10-09T09:11:14.363 INFO:tasks.rados.rados.0.smithi163.stdout:1428: finishing write tid 1 to smithi16367185-50 2023-10-09T09:11:14.363 INFO:tasks.rados.rados.0.smithi163.stdout:1430: write oid 30 current snap is 172 2023-10-09T09:11:14.363 INFO:tasks.rados.rados.0.smithi163.stdout:1430: seq_num 563 ranges {662341=575309,1878774=498528,2894439=457555} 2023-10-09T09:11:14.370 INFO:tasks.rados.rados.0.smithi163.stdout:1430: writing smithi16367185-30 from 662341 to 1237650 tid 1 2023-10-09T09:11:14.377 INFO:tasks.rados.rados.0.smithi163.stdout:1430: writing smithi16367185-30 from 1878774 to 2377302 tid 2 2023-10-09T09:11:14.382 INFO:tasks.rados.rados.0.smithi163.stdout:1430: writing smithi16367185-30 from 2894439 to 3351994 tid 3 2023-10-09T09:11:14.383 INFO:tasks.rados.rados.0.smithi163.stdout:1431: write oid 45 current snap is 172 2023-10-09T09:11:14.383 INFO:tasks.rados.rados.0.smithi163.stdout:1428: finishing write tid 2 to smithi16367185-50 2023-10-09T09:11:14.383 INFO:tasks.rados.rados.0.smithi163.stdout:1428: finishing write tid 3 to smithi16367185-50 2023-10-09T09:11:14.383 INFO:tasks.rados.rados.0.smithi163.stdout:1428: finishing write tid 4 to smithi16367185-50 2023-10-09T09:11:14.383 INFO:tasks.rados.rados.0.smithi163.stdout:1431: seq_num 564 ranges {679148=650558,1937256=642186,3321862=157286} 2023-10-09T09:11:14.390 INFO:tasks.rados.rados.0.smithi163.stdout:1431: writing smithi16367185-45 from 679148 to 1329706 tid 1 2023-10-09T09:11:14.397 INFO:tasks.rados.rados.0.smithi163.stdout:1431: writing smithi16367185-45 from 1937256 to 2579442 tid 2 2023-10-09T09:11:14.404 INFO:tasks.rados.rados.0.smithi163.stdout:1431: writing smithi16367185-45 from 3321862 to 3479148 tid 3 2023-10-09T09:11:14.404 INFO:tasks.rados.rados.0.smithi163.stdout:1432: snap_remove snap 164 2023-10-09T09:11:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: debug 2023-10-09T09:11:14.078+0000 7fde83035700 -1 mon.b@1(peon).osd e407 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: cluster 2023-10-09T09:11:13.065502+0000 osd.3 (osd.3) 55 : cluster [WRN] failed to encode map e406 with expected crc 2023-10-09T09:11:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: cluster 2023-10-09T09:11:13.069531+0000 mon.a (mon.0) 232 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2023-10-09T09:11:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: cluster 2023-10-09T09:11:13.071506+0000 osd.6 (osd.6) 34 : cluster [WRN] failed to encode map e406 with expected crc 2023-10-09T09:11:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: cluster 2023-10-09T09:11:13.079393+0000 osd.0 (osd.0) 54 : cluster [WRN] failed to encode map e406 with expected crc 2023-10-09T09:11:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: cluster 2023-10-09T09:11:13.087701+0000 osd.2 (osd.2) 79 : cluster [WRN] failed to encode map e406 with expected crc 2023-10-09T09:11:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: cluster 2023-10-09T09:11:13.245918+0000 mgr.y (mgr.34107) 78 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 250 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2023-10-09T09:11:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: audit 2023-10-09T09:11:13.462448+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:11:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: audit 2023-10-09T09:11:13.470066+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:11:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: audit 2023-10-09T09:11:13.471565+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:11:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: audit 2023-10-09T09:11:13.472735+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:11:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: audit 2023-10-09T09:11:13.479390+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:11:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: audit 2023-10-09T09:11:13.564968+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:11:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: audit 2023-10-09T09:11:13.567529+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:11:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: audit 2023-10-09T09:11:13.568562+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:11:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: audit 2023-10-09T09:11:13.575538+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:11:15.000 INFO:tasks.rados.rados.0.smithi163.stdout:1432: done (4 left) 2023-10-09T09:11:15.000 INFO:tasks.rados.rados.0.smithi163.stdout:1433: read oid 34 snap -1 2023-10-09T09:11:15.000 INFO:tasks.rados.rados.0.smithi163.stdout:1433: expect (ObjNum 554 snap 171 seq_num 554) 2023-10-09T09:11:15.000 INFO:tasks.rados.rados.0.smithi163.stdout:1434: rollback oid 25 current snap is 172 2023-10-09T09:11:15.000 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 25 to 156 2023-10-09T09:11:15.001 INFO:tasks.rados.rados.0.smithi163.stdout:1428: finishing write tid 5 to smithi16367185-50 2023-10-09T09:11:15.001 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 580 (ObjNum 562 snap 172 seq_num 562) dirty exists 2023-10-09T09:11:15.001 INFO:tasks.rados.rados.0.smithi163.stdout:1428: left oid 50 (ObjNum 562 snap 172 seq_num 562) 2023-10-09T09:11:15.001 INFO:tasks.rados.rados.0.smithi163.stdout:1429: finishing rollback tid 0 to smithi16367185-1 2023-10-09T09:11:15.001 INFO:tasks.rados.rados.0.smithi163.stdout:1429: finishing rollback tid 1 to smithi16367185-1 2023-10-09T09:11:15.001 INFO:tasks.rados.rados.0.smithi163.stdout:1429: finishing rollback tid 2 to smithi16367185-1 2023-10-09T09:11:15.001 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 845 (ObjNum 510 snap 159 seq_num 510) dirty exists 2023-10-09T09:11:15.002 INFO:tasks.rados.rados.0.smithi163.stdout:1430: finishing write tid 1 to smithi16367185-30 2023-10-09T09:11:15.002 INFO:tasks.rados.rados.0.smithi163.stdout:1430: finishing write tid 2 to smithi16367185-30 2023-10-09T09:11:15.002 INFO:tasks.rados.rados.0.smithi163.stdout:1430: finishing write tid 3 to smithi16367185-30 2023-10-09T09:11:15.002 INFO:tasks.rados.rados.0.smithi163.stdout:1430: finishing write tid 4 to smithi16367185-30 2023-10-09T09:11:15.002 INFO:tasks.rados.rados.0.smithi163.stdout:1430: finishing write tid 5 to smithi16367185-30 2023-10-09T09:11:15.002 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 545 (ObjNum 563 snap 172 seq_num 563) dirty exists 2023-10-09T09:11:15.002 INFO:tasks.rados.rados.0.smithi163.stdout:1430: left oid 30 (ObjNum 563 snap 172 seq_num 563) 2023-10-09T09:11:15.003 INFO:tasks.rados.rados.0.smithi163.stdout:1431: finishing write tid 1 to smithi16367185-45 2023-10-09T09:11:15.003 INFO:tasks.rados.rados.0.smithi163.stdout:1431: finishing write tid 2 to smithi16367185-45 2023-10-09T09:11:15.003 INFO:tasks.rados.rados.0.smithi163.stdout:1431: finishing write tid 3 to smithi16367185-45 2023-10-09T09:11:15.003 INFO:tasks.rados.rados.0.smithi163.stdout:1431: finishing write tid 4 to smithi16367185-45 2023-10-09T09:11:15.003 INFO:tasks.rados.rados.0.smithi163.stdout:1431: finishing write tid 5 to smithi16367185-45 2023-10-09T09:11:15.003 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 1128 (ObjNum 564 snap 172 seq_num 564) dirty exists 2023-10-09T09:11:15.003 INFO:tasks.rados.rados.0.smithi163.stdout:1431: left oid 45 (ObjNum 564 snap 172 seq_num 564) 2023-10-09T09:11:15.003 INFO:tasks.rados.rados.0.smithi163.stdout:1428: done (5 left) 2023-10-09T09:11:15.004 INFO:tasks.rados.rados.0.smithi163.stdout:1429: done (4 left) 2023-10-09T09:11:15.004 INFO:tasks.rados.rados.0.smithi163.stdout:1430: done (3 left) 2023-10-09T09:11:15.004 INFO:tasks.rados.rados.0.smithi163.stdout:1431: done (2 left) 2023-10-09T09:11:15.004 INFO:tasks.rados.rados.0.smithi163.stdout:1435: rollback oid 47 current snap is 172 2023-10-09T09:11:15.004 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 47 to 160 2023-10-09T09:11:15.004 INFO:tasks.rados.rados.0.smithi163.stdout:1436: write oid 10 current snap is 172 2023-10-09T09:11:15.004 INFO:tasks.rados.rados.0.smithi163.stdout:1436: seq_num 565 ranges {695955=725807,1879385=502197,2949285=480099} 2023-10-09T09:11:15.011 INFO:tasks.rados.rados.0.smithi163.stdout:1436: writing smithi16367185-10 from 695955 to 1421762 tid 1 2023-10-09T09:11:15.017 INFO:tasks.rados.rados.0.smithi163.stdout:1436: writing smithi16367185-10 from 1879385 to 2381582 tid 2 2023-10-09T09:11:15.026 INFO:tasks.rados.rados.0.smithi163.stdout:1436: writing smithi16367185-10 from 2949285 to 3429384 tid 3 2023-10-09T09:11:15.026 INFO:tasks.rados.rados.0.smithi163.stdout:1437: rollback oid 24 current snap is 172 2023-10-09T09:11:15.026 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 24 to 156 2023-10-09T09:11:15.026 INFO:tasks.rados.rados.0.smithi163.stdout:1433: expect (ObjNum 554 snap 171 seq_num 554) 2023-10-09T09:11:15.195 INFO:tasks.rados.rados.0.smithi163.stdout:1435: finishing rollback tid 0 to smithi16367185-47 2023-10-09T09:11:15.195 INFO:tasks.rados.rados.0.smithi163.stdout:1435: finishing rollback tid 1 to smithi16367185-47 2023-10-09T09:11:15.195 INFO:tasks.rados.rados.0.smithi163.stdout:1435: finishing rollback tid 2 to smithi16367185-47 2023-10-09T09:11:15.195 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 1131 (ObjNum 458 snap 147 seq_num 458) dirty exists 2023-10-09T09:11:15.195 INFO:tasks.rados.rados.0.smithi163.stdout:1437: finishing rollback tid 1 to smithi16367185-24 2023-10-09T09:11:15.195 INFO:tasks.rados.rados.0.smithi163.stdout:1437: finishing rollback tid 2 to smithi16367185-24 2023-10-09T09:11:15.195 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 384 (ObjNum 500 snap 156 seq_num 500) dirty exists 2023-10-09T09:11:15.195 INFO:tasks.rados.rados.0.smithi163.stdout:1436: finishing write tid 1 to smithi16367185-10 2023-10-09T09:11:15.195 INFO:tasks.rados.rados.0.smithi163.stdout:1436: finishing write tid 2 to smithi16367185-10 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1436: finishing write tid 3 to smithi16367185-10 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1434: finishing rollback tid 0 to smithi16367185-25 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1436: finishing write tid 4 to smithi16367185-10 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1434: finishing rollback tid 1 to smithi16367185-25 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1434: finishing rollback tid 2 to smithi16367185-25 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 621 (ObjNum 443 snap 138 seq_num 443) dirty exists 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1436: finishing write tid 5 to smithi16367185-10 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 549 (ObjNum 565 snap 172 seq_num 565) dirty exists 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1436: left oid 10 (ObjNum 565 snap 172 seq_num 565) 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1433: done (4 left) 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1434: done (3 left) 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1435: done (2 left) 2023-10-09T09:11:15.196 INFO:tasks.rados.rados.0.smithi163.stdout:1436: done (1 left) 2023-10-09T09:11:15.197 INFO:tasks.rados.rados.0.smithi163.stdout:1437: done (0 left) 2023-10-09T09:11:15.197 INFO:tasks.rados.rados.0.smithi163.stdout:1438: rollback oid 36 current snap is 172 2023-10-09T09:11:15.197 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 36 to 140 2023-10-09T09:11:15.197 INFO:tasks.rados.rados.0.smithi163.stdout:1439: read oid 38 snap -1 2023-10-09T09:11:15.197 INFO:tasks.rados.rados.0.smithi163.stdout:1439: expect (ObjNum 503 snap 157 seq_num 503) 2023-10-09T09:11:15.197 INFO:tasks.rados.rados.0.smithi163.stdout:1440: snap_remove snap 153 2023-10-09T09:11:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:14 smithi181 bash[27712]: debug 2023-10-09T09:11:14.982+0000 7fde83035700 -1 mon.b@1(peon).osd e408 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:15 smithi181 bash[27712]: cluster 2023-10-09T09:11:14.085753+0000 osd.3 (osd.3) 56 : cluster [WRN] failed to encode map e407 with expected crc 2023-10-09T09:11:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:15 smithi181 bash[27712]: cluster 2023-10-09T09:11:14.089738+0000 osd.4 (osd.4) 82 : cluster [WRN] failed to encode map e407 with expected crc 2023-10-09T09:11:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:15 smithi181 bash[27712]: cluster 2023-10-09T09:11:14.090660+0000 mon.a (mon.0) 242 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2023-10-09T09:11:15.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:15 smithi181 bash[27712]: cluster 2023-10-09T09:11:14.094166+0000 osd.7 (osd.7) 56 : cluster [WRN] failed to encode map e407 with expected crc 2023-10-09T09:11:15.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:15 smithi181 bash[27712]: cluster 2023-10-09T09:11:14.104695+0000 osd.2 (osd.2) 80 : cluster [WRN] failed to encode map e407 with expected crc 2023-10-09T09:11:15.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:15 smithi181 bash[27712]: cluster 2023-10-09T09:11:14.993029+0000 mon.a (mon.0) 243 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2023-10-09T09:11:16.141 INFO:tasks.rados.rados.0.smithi163.stdout:1440: done (2 left) 2023-10-09T09:11:16.141 INFO:tasks.rados.rados.0.smithi163.stdout:1441: read oid 20 snap -1 2023-10-09T09:11:16.142 INFO:tasks.rados.rados.0.smithi163.stdout:1441: expect deleted 2023-10-09T09:11:16.142 INFO:tasks.rados.rados.0.smithi163.stdout:1438: finishing rollback tid 1 to smithi16367185-36 2023-10-09T09:11:16.142 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 419 (ObjNum 698963968 snap 21875 seq_num 698742048) dirty dne 2023-10-09T09:11:16.142 INFO:tasks.rados.rados.0.smithi163.stdout:1439: expect (ObjNum 503 snap 157 seq_num 503) 2023-10-09T09:11:16.225 INFO:tasks.rados.rados.0.smithi163.stdout:1438: done (2 left) 2023-10-09T09:11:16.226 INFO:tasks.rados.rados.0.smithi163.stdout:1439: done (1 left) 2023-10-09T09:11:16.226 INFO:tasks.rados.rados.0.smithi163.stdout:1441: done (0 left) 2023-10-09T09:11:16.226 INFO:tasks.rados.rados.0.smithi163.stdout:1442: snap_create 2023-10-09T09:11:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:16 smithi181 bash[27712]: debug 2023-10-09T09:11:16.126+0000 7fde83035700 -1 mon.b@1(peon).osd e409 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:16 smithi181 bash[27712]: cluster 2023-10-09T09:11:14.989888+0000 osd.3 (osd.3) 57 : cluster [WRN] failed to encode map e408 with expected crc 2023-10-09T09:11:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:16 smithi181 bash[27712]: cluster 2023-10-09T09:11:14.995187+0000 osd.4 (osd.4) 83 : cluster [WRN] failed to encode map e408 with expected crc 2023-10-09T09:11:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:16 smithi181 bash[27712]: cluster 2023-10-09T09:11:15.020895+0000 osd.2 (osd.2) 81 : cluster [WRN] failed to encode map e408 with expected crc 2023-10-09T09:11:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:16 smithi181 bash[27712]: cluster 2023-10-09T09:11:15.246550+0000 mgr.y (mgr.34107) 79 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 250 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:11:17.148 INFO:tasks.rados.rados.0.smithi163.stdout:1442: done (0 left) 2023-10-09T09:11:17.148 INFO:tasks.rados.rados.0.smithi163.stdout:1443: snap_create 2023-10-09T09:11:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:17 smithi181 bash[27712]: cluster 2023-10-09T09:11:16.136284+0000 osd.3 (osd.3) 58 : cluster [WRN] failed to encode map e409 with expected crc 2023-10-09T09:11:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:17 smithi181 bash[27712]: cluster 2023-10-09T09:11:16.139333+0000 osd.1 (osd.1) 34 : cluster [WRN] failed to encode map e409 with expected crc 2023-10-09T09:11:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:17 smithi181 bash[27712]: cluster 2023-10-09T09:11:16.140218+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2023-10-09T09:11:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:17 smithi181 bash[27712]: cluster 2023-10-09T09:11:16.141840+0000 osd.6 (osd.6) 35 : cluster [WRN] failed to encode map e409 with expected crc 2023-10-09T09:11:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:17 smithi181 bash[27712]: cluster 2023-10-09T09:11:16.301986+0000 osd.7 (osd.7) 57 : cluster [WRN] failed to encode map e409 with expected crc 2023-10-09T09:11:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:17 smithi181 bash[27712]: debug 2023-10-09T09:11:17.134+0000 7fde83035700 -1 mon.b@1(peon).osd e410 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:18.160 INFO:tasks.rados.rados.0.smithi163.stdout:1443: done (0 left) 2023-10-09T09:11:18.160 INFO:tasks.rados.rados.0.smithi163.stdout:1444: write oid 17 current snap is 174 2023-10-09T09:11:18.160 INFO:tasks.rados.rados.0.smithi163.stdout:1444: seq_num 566 ranges {712762=401056,1537867=645855,2860355=652407} 2023-10-09T09:11:18.172 INFO:tasks.rados.rados.0.smithi163.stdout:1444: writing smithi16367185-17 from 712762 to 1113818 tid 1 2023-10-09T09:11:18.179 INFO:tasks.rados.rados.0.smithi163.stdout:1444: writing smithi16367185-17 from 1537867 to 2183722 tid 2 2023-10-09T09:11:18.188 INFO:tasks.rados.rados.0.smithi163.stdout:1444: writing smithi16367185-17 from 2860355 to 3512762 tid 3 2023-10-09T09:11:18.188 INFO:tasks.rados.rados.0.smithi163.stdout:1445: delete oid 4 current snap is 174 2023-10-09T09:11:18.188 INFO:tasks.rados.rados.0.smithi163.stdout:1444: finishing write tid 1 to smithi16367185-17 2023-10-09T09:11:18.188 INFO:tasks.rados.rados.0.smithi163.stdout:1444: finishing write tid 2 to smithi16367185-17 2023-10-09T09:11:18.192 INFO:tasks.rados.rados.0.smithi163.stdout:1445: done (1 left) 2023-10-09T09:11:18.192 INFO:tasks.rados.rados.0.smithi163.stdout:1446: snap_create 2023-10-09T09:11:18.193 INFO:tasks.rados.rados.0.smithi163.stdout:1444: finishing write tid 3 to smithi16367185-17 2023-10-09T09:11:18.193 INFO:tasks.rados.rados.0.smithi163.stdout:1444: finishing write tid 4 to smithi16367185-17 2023-10-09T09:11:18.193 INFO:tasks.rados.rados.0.smithi163.stdout:1444: finishing write tid 5 to smithi16367185-17 2023-10-09T09:11:18.193 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 705 (ObjNum 566 snap 174 seq_num 566) dirty exists 2023-10-09T09:11:18.193 INFO:tasks.rados.rados.0.smithi163.stdout:1444: left oid 17 (ObjNum 566 snap 174 seq_num 566) 2023-10-09T09:11:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:18 smithi181 bash[27712]: debug 2023-10-09T09:11:18.146+0000 7fde83035700 -1 mon.b@1(peon).osd e411 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:18 smithi181 bash[27712]: cluster 2023-10-09T09:11:17.143287+0000 osd.3 (osd.3) 59 : cluster [WRN] failed to encode map e410 with expected crc 2023-10-09T09:11:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:18 smithi181 bash[27712]: cluster 2023-10-09T09:11:17.145329+0000 osd.5 (osd.5) 42 : cluster [WRN] failed to encode map e410 with expected crc 2023-10-09T09:11:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:18 smithi181 bash[27712]: cluster 2023-10-09T09:11:17.145557+0000 mon.a (mon.0) 245 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2023-10-09T09:11:18.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:18 smithi181 bash[27712]: cluster 2023-10-09T09:11:17.247290+0000 mgr.y (mgr.34107) 80 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 266 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2023-10-09T09:11:18.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:18 smithi181 bash[27712]: cluster 2023-10-09T09:11:17.347087+0000 osd.1 (osd.1) 35 : cluster [WRN] failed to encode map e410 with expected crc 2023-10-09T09:11:19.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:11:18 smithi181 bash[54907]: ts=2023-10-09T09:11:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:11:19.165 INFO:tasks.rados.rados.0.smithi163.stdout:1444: done (1 left) 2023-10-09T09:11:19.166 INFO:tasks.rados.rados.0.smithi163.stdout:1446: done (0 left) 2023-10-09T09:11:19.166 INFO:tasks.rados.rados.0.smithi163.stdout:1447: snap_create 2023-10-09T09:11:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:19 smithi181 bash[27712]: debug 2023-10-09T09:11:19.150+0000 7fde83035700 -1 mon.b@1(peon).osd e412 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:19 smithi181 bash[27712]: cluster 2023-10-09T09:11:17.598199+0000 osd.6 (osd.6) 36 : cluster [WRN] failed to encode map e410 with expected crc 2023-10-09T09:11:19.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:19 smithi181 bash[27712]: cluster 2023-10-09T09:11:18.152619+0000 osd.3 (osd.3) 60 : cluster [WRN] failed to encode map e411 with expected crc 2023-10-09T09:11:19.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:19 smithi181 bash[27712]: cluster 2023-10-09T09:11:18.154817+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2023-10-09T09:11:19.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:19 smithi181 bash[27712]: cluster 2023-10-09T09:11:18.156573+0000 osd.5 (osd.5) 43 : cluster [WRN] failed to encode map e411 with expected crc 2023-10-09T09:11:19.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:19 smithi181 bash[27712]: cluster 2023-10-09T09:11:18.173368+0000 osd.7 (osd.7) 58 : cluster [WRN] failed to encode map e410 with expected crc 2023-10-09T09:11:20.003 INFO:tasks.rados.rados.0.smithi163.stdout:1447: done (0 left) 2023-10-09T09:11:20.003 INFO:tasks.rados.rados.0.smithi163.stdout:1448: rollback oid 17 current snap is 176 2023-10-09T09:11:20.003 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 172 2023-10-09T09:11:20.003 INFO:tasks.rados.rados.0.smithi163.stdout:1449: read oid 29 snap -1 2023-10-09T09:11:20.003 INFO:tasks.rados.rados.0.smithi163.stdout:1449: expect deleted 2023-10-09T09:11:20.003 INFO:tasks.rados.rados.0.smithi163.stdout:1450: snap_create 2023-10-09T09:11:20.016 INFO:tasks.rados.rados.0.smithi163.stdout:1448: finishing rollback tid 0 to smithi16367185-17 2023-10-09T09:11:20.017 INFO:tasks.rados.rados.0.smithi163.stdout:1448: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:11:20.017 INFO:tasks.rados.rados.0.smithi163.stdout:1448: finishing rollback tid 2 to smithi16367185-17 2023-10-09T09:11:20.017 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 709 (ObjNum 504 snap 157 seq_num 504) dirty exists 2023-10-09T09:11:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:19 smithi181 bash[27712]: debug 2023-10-09T09:11:19.986+0000 7fde83035700 -1 mon.b@1(peon).osd e413 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:20 smithi181 bash[27712]: cluster 2023-10-09T09:11:19.160005+0000 osd.3 (osd.3) 61 : cluster [WRN] failed to encode map e412 with expected crc 2023-10-09T09:11:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:20 smithi181 bash[27712]: cluster 2023-10-09T09:11:19.162067+0000 osd.7 (osd.7) 59 : cluster [WRN] failed to encode map e412 with expected crc 2023-10-09T09:11:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:20 smithi181 bash[27712]: cluster 2023-10-09T09:11:19.163275+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2023-10-09T09:11:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:20 smithi181 bash[27712]: cluster 2023-10-09T09:11:19.248013+0000 mgr.y (mgr.34107) 81 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 266 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2023-10-09T09:11:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:20 smithi181 bash[27712]: cluster 2023-10-09T09:11:19.996349+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2023-10-09T09:11:21.017 INFO:tasks.rados.rados.0.smithi163.stdout:1448: done (2 left) 2023-10-09T09:11:21.017 INFO:tasks.rados.rados.0.smithi163.stdout:1449: done (1 left) 2023-10-09T09:11:21.017 INFO:tasks.rados.rados.0.smithi163.stdout:1450: done (0 left) 2023-10-09T09:11:21.017 INFO:tasks.rados.rados.0.smithi163.stdout:1451: delete oid 6 current snap is 177 2023-10-09T09:11:21.025 INFO:tasks.rados.rados.0.smithi163.stdout:1451: done (0 left) 2023-10-09T09:11:21.025 INFO:tasks.rados.rados.0.smithi163.stdout:1452: rollback oid 25 current snap is 177 2023-10-09T09:11:21.025 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 25 to 167 2023-10-09T09:11:21.025 INFO:tasks.rados.rados.0.smithi163.stdout:1453: read oid 45 snap -1 2023-10-09T09:11:21.025 INFO:tasks.rados.rados.0.smithi163.stdout:1453: expect (ObjNum 564 snap 172 seq_num 564) 2023-10-09T09:11:21.025 INFO:tasks.rados.rados.0.smithi163.stdout:1454: read oid 21 snap -1 2023-10-09T09:11:21.025 INFO:tasks.rados.rados.0.smithi163.stdout:1454: expect (ObjNum 526 snap 165 seq_num 526) 2023-10-09T09:11:21.025 INFO:tasks.rados.rados.0.smithi163.stdout:1455: read oid 46 snap -1 2023-10-09T09:11:21.025 INFO:tasks.rados.rados.0.smithi163.stdout:1455: expect (ObjNum 538 snap 168 seq_num 538) 2023-10-09T09:11:21.025 INFO:tasks.rados.rados.0.smithi163.stdout:1456: rollback oid 40 current snap is 177 2023-10-09T09:11:21.026 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 40 to 174 2023-10-09T09:11:21.026 INFO:tasks.rados.rados.0.smithi163.stdout:1457: snap_remove snap 173 2023-10-09T09:11:21.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:21 smithi181 bash[27712]: debug 2023-10-09T09:11:20.998+0000 7fde83035700 -1 mon.b@1(peon).osd e414 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:21.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:21 smithi181 bash[27712]: cluster 2023-10-09T09:11:19.992954+0000 osd.3 (osd.3) 62 : cluster [WRN] failed to encode map e413 with expected crc 2023-10-09T09:11:21.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:21 smithi181 bash[27712]: cluster 2023-10-09T09:11:20.000630+0000 osd.0 (osd.0) 55 : cluster [WRN] failed to encode map e413 with expected crc 2023-10-09T09:11:21.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:21 smithi181 bash[27712]: cluster 2023-10-09T09:11:20.006790+0000 osd.7 (osd.7) 60 : cluster [WRN] failed to encode map e413 with expected crc 2023-10-09T09:11:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:21 smithi181 bash[27712]: cluster 2023-10-09T09:11:20.008541+0000 osd.2 (osd.2) 82 : cluster [WRN] failed to encode map e413 with expected crc 2023-10-09T09:11:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:21 smithi181 bash[27712]: cluster 2023-10-09T09:11:20.281543+0000 osd.1 (osd.1) 36 : cluster [WRN] failed to encode map e413 with expected crc 2023-10-09T09:11:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:21 smithi181 bash[27712]: cluster 2023-10-09T09:11:21.013348+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2023-10-09T09:11:22.019 INFO:tasks.rados.rados.0.smithi163.stdout:1457: done (5 left) 2023-10-09T09:11:22.019 INFO:tasks.rados.rados.0.smithi163.stdout:1458: read oid 49 snap -1 2023-10-09T09:11:22.020 INFO:tasks.rados.rados.0.smithi163.stdout:1458: expect (ObjNum 448 snap 144 seq_num 448) 2023-10-09T09:11:22.020 INFO:tasks.rados.rados.0.smithi163.stdout:1459: snap_remove snap 129 2023-10-09T09:11:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:22 smithi181 bash[27712]: debug 2023-10-09T09:11:22.002+0000 7fde83035700 -1 mon.b@1(peon).osd e415 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:22 smithi181 bash[27712]: cluster 2023-10-09T09:11:20.654559+0000 osd.4 (osd.4) 84 : cluster [WRN] failed to encode map e413 with expected crc 2023-10-09T09:11:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:22 smithi181 bash[27712]: cluster 2023-10-09T09:11:20.675102+0000 osd.6 (osd.6) 37 : cluster [WRN] failed to encode map e413 with expected crc 2023-10-09T09:11:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:22 smithi181 bash[27712]: cluster 2023-10-09T09:11:21.009258+0000 osd.3 (osd.3) 63 : cluster [WRN] failed to encode map e414 with expected crc 2023-10-09T09:11:22.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:22 smithi181 bash[27712]: cluster 2023-10-09T09:11:21.012998+0000 osd.5 (osd.5) 44 : cluster [WRN] failed to encode map e414 with expected crc 2023-10-09T09:11:22.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:22 smithi181 bash[27712]: cluster 2023-10-09T09:11:21.017023+0000 osd.4 (osd.4) 85 : cluster [WRN] failed to encode map e414 with expected crc 2023-10-09T09:11:22.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:22 smithi181 bash[27712]: cluster 2023-10-09T09:11:21.026846+0000 osd.2 (osd.2) 83 : cluster [WRN] failed to encode map e414 with expected crc 2023-10-09T09:11:22.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:22 smithi181 bash[27712]: cluster 2023-10-09T09:11:21.026974+0000 osd.1 (osd.1) 37 : cluster [WRN] failed to encode map e414 with expected crc 2023-10-09T09:11:22.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:22 smithi181 bash[27712]: cluster 2023-10-09T09:11:21.249196+0000 mgr.y (mgr.34107) 82 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 270 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 MiB/s rd, 415 KiB/s wr, 9 op/s 2023-10-09T09:11:22.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:22 smithi181 bash[27712]: cluster 2023-10-09T09:11:22.013385+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2023-10-09T09:11:23.024 INFO:tasks.rados.rados.0.smithi163.stdout:1459: done (6 left) 2023-10-09T09:11:23.024 INFO:tasks.rados.rados.0.smithi163.stdout:1460: snap_remove snap 171 2023-10-09T09:11:23.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:23 smithi181 bash[27712]: debug 2023-10-09T09:11:23.006+0000 7fde83035700 -1 mon.b@1(peon).osd e416 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:23.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:23 smithi181 bash[27712]: cluster 2023-10-09T09:11:22.009641+0000 osd.3 (osd.3) 64 : cluster [WRN] failed to encode map e415 with expected crc 2023-10-09T09:11:23.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:23 smithi181 bash[27712]: cluster 2023-10-09T09:11:22.017267+0000 osd.1 (osd.1) 38 : cluster [WRN] failed to encode map e415 with expected crc 2023-10-09T09:11:23.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:23 smithi181 bash[27712]: cluster 2023-10-09T09:11:22.021418+0000 osd.0 (osd.0) 56 : cluster [WRN] failed to encode map e415 with expected crc 2023-10-09T09:11:23.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:23 smithi181 bash[27712]: cluster 2023-10-09T09:11:22.248356+0000 osd.5 (osd.5) 45 : cluster [WRN] failed to encode map e415 with expected crc 2023-10-09T09:11:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:23 smithi181 bash[27712]: cluster 2023-10-09T09:11:23.015957+0000 mon.a (mon.0) 251 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2023-10-09T09:11:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:11:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:11:23] "GET /metrics HTTP/1.1" 200 36071 "" "Prometheus/2.33.4" 2023-10-09T09:11:24.027 INFO:tasks.rados.rados.0.smithi163.stdout:1460: done (6 left) 2023-10-09T09:11:24.028 INFO:tasks.rados.rados.0.smithi163.stdout:1461: read oid 7 snap -1 2023-10-09T09:11:24.028 INFO:tasks.rados.rados.0.smithi163.stdout:1461: expect (ObjNum 530 snap 165 seq_num 530) 2023-10-09T09:11:24.028 INFO:tasks.rados.rados.0.smithi163.stdout:1462: snap_remove snap 160 2023-10-09T09:11:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:24 smithi181 bash[27712]: debug 2023-10-09T09:11:24.006+0000 7fde83035700 -1 mon.b@1(peon).osd e417 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:24 smithi181 bash[27712]: cluster 2023-10-09T09:11:22.654955+0000 osd.4 (osd.4) 86 : cluster [WRN] failed to encode map e415 with expected crc 2023-10-09T09:11:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:24 smithi181 bash[27712]: cluster 2023-10-09T09:11:23.012019+0000 osd.3 (osd.3) 65 : cluster [WRN] failed to encode map e416 with expected crc 2023-10-09T09:11:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:24 smithi181 bash[27712]: cluster 2023-10-09T09:11:23.021027+0000 osd.2 (osd.2) 84 : cluster [WRN] failed to encode map e416 with expected crc 2023-10-09T09:11:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:24 smithi181 bash[27712]: cluster 2023-10-09T09:11:23.249907+0000 mgr.y (mgr.34107) 83 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 270 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 MiB/s rd, 415 KiB/s wr, 9 op/s 2023-10-09T09:11:24.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:24 smithi181 bash[27712]: cluster 2023-10-09T09:11:24.018454+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2023-10-09T09:11:25.003 INFO:tasks.rados.rados.0.smithi163.stdout:1462: done (7 left) 2023-10-09T09:11:25.003 INFO:tasks.rados.rados.0.smithi163.stdout:1463: delete oid 19 current snap is 177 2023-10-09T09:11:25.003 INFO:tasks.rados.rados.0.smithi163.stdout:1456: finishing rollback tid 0 to smithi16367185-40 2023-10-09T09:11:25.003 INFO:tasks.rados.rados.0.smithi163.stdout:1456: finishing rollback tid 1 to smithi16367185-40 2023-10-09T09:11:25.003 INFO:tasks.rados.rados.0.smithi163.stdout:1456: finishing rollback tid 2 to smithi16367185-40 2023-10-09T09:11:25.004 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 713 (ObjNum 537 snap 168 seq_num 537) dirty exists 2023-10-09T09:11:25.004 INFO:tasks.rados.rados.0.smithi163.stdout:1452: finishing rollback tid 0 to smithi16367185-25 2023-10-09T09:11:25.004 INFO:tasks.rados.rados.0.smithi163.stdout:1452: finishing rollback tid 1 to smithi16367185-25 2023-10-09T09:11:25.004 INFO:tasks.rados.rados.0.smithi163.stdout:1452: finishing rollback tid 2 to smithi16367185-25 2023-10-09T09:11:25.004 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 639 (ObjNum 511 snap 160 seq_num 511) dirty exists 2023-10-09T09:11:25.004 INFO:tasks.rados.rados.0.smithi163.stdout:1455: expect (ObjNum 538 snap 168 seq_num 538) 2023-10-09T09:11:25.200 INFO:tasks.rados.rados.0.smithi163.stdout:1454: expect (ObjNum 526 snap 165 seq_num 526) 2023-10-09T09:11:25.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:24 smithi181 bash[27712]: debug 2023-10-09T09:11:24.986+0000 7fde83035700 -1 mon.b@1(peon).osd e418 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:25.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:25 smithi181 bash[27712]: cluster 2023-10-09T09:11:24.014806+0000 osd.3 (osd.3) 66 : cluster [WRN] failed to encode map e417 with expected crc 2023-10-09T09:11:25.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:25 smithi181 bash[27712]: cluster 2023-10-09T09:11:24.023699+0000 osd.6 (osd.6) 38 : cluster [WRN] failed to encode map e417 with expected crc 2023-10-09T09:11:25.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:25 smithi181 bash[27712]: cluster 2023-10-09T09:11:24.997166+0000 mon.a (mon.0) 253 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2023-10-09T09:11:25.335 INFO:tasks.rados.rados.0.smithi163.stdout:1453: expect (ObjNum 564 snap 172 seq_num 564) 2023-10-09T09:11:25.458 INFO:tasks.rados.rados.0.smithi163.stdout:1458: expect (ObjNum 448 snap 144 seq_num 448) 2023-10-09T09:11:25.760 INFO:tasks.rados.rados.0.smithi163.stdout:1461: expect (ObjNum 530 snap 165 seq_num 530) 2023-10-09T09:11:25.923 INFO:tasks.rados.rados.0.smithi163.stdout:1452: done (7 left) 2023-10-09T09:11:25.923 INFO:tasks.rados.rados.0.smithi163.stdout:1453: done (6 left) 2023-10-09T09:11:25.923 INFO:tasks.rados.rados.0.smithi163.stdout:1454: done (5 left) 2023-10-09T09:11:25.923 INFO:tasks.rados.rados.0.smithi163.stdout:1455: done (4 left) 2023-10-09T09:11:25.923 INFO:tasks.rados.rados.0.smithi163.stdout:1456: done (3 left) 2023-10-09T09:11:25.923 INFO:tasks.rados.rados.0.smithi163.stdout:1458: done (2 left) 2023-10-09T09:11:25.924 INFO:tasks.rados.rados.0.smithi163.stdout:1461: done (1 left) 2023-10-09T09:11:25.924 INFO:tasks.rados.rados.0.smithi163.stdout:1463: done (0 left) 2023-10-09T09:11:25.924 INFO:tasks.rados.rados.0.smithi163.stdout:1464: write oid 14 current snap is 177 2023-10-09T09:11:25.924 INFO:tasks.rados.rados.0.smithi163.stdout:1464: seq_num 570 ranges {779990=702052,2055442=653193,3253694=326296} 2023-10-09T09:11:25.937 INFO:tasks.rados.rados.0.smithi163.stdout:1464: writing smithi16367185-14 from 779990 to 1482042 tid 1 2023-10-09T09:11:25.944 INFO:tasks.rados.rados.0.smithi163.stdout:1464: writing smithi16367185-14 from 2055442 to 2708635 tid 2 2023-10-09T09:11:25.949 INFO:tasks.rados.rados.0.smithi163.stdout:1464: writing smithi16367185-14 from 3253694 to 3579990 tid 3 2023-10-09T09:11:25.949 INFO:tasks.rados.rados.0.smithi163.stdout:1465: read oid 30 snap -1 2023-10-09T09:11:25.950 INFO:tasks.rados.rados.0.smithi163.stdout:1465: expect (ObjNum 563 snap 172 seq_num 563) 2023-10-09T09:11:25.950 INFO:tasks.rados.rados.0.smithi163.stdout:1466: snap_create 2023-10-09T09:11:25.950 INFO:tasks.rados.rados.0.smithi163.stdout:1464: finishing write tid 1 to smithi16367185-14 2023-10-09T09:11:25.950 INFO:tasks.rados.rados.0.smithi163.stdout:1464: finishing write tid 2 to smithi16367185-14 2023-10-09T09:11:25.952 INFO:tasks.rados.rados.0.smithi163.stdout:1464: finishing write tid 3 to smithi16367185-14 2023-10-09T09:11:25.953 INFO:tasks.rados.rados.0.smithi163.stdout:1464: finishing write tid 4 to smithi16367185-14 2023-10-09T09:11:25.953 INFO:tasks.rados.rados.0.smithi163.stdout:1464: finishing write tid 5 to smithi16367185-14 2023-10-09T09:11:25.953 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 582 (ObjNum 570 snap 177 seq_num 570) dirty exists 2023-10-09T09:11:25.953 INFO:tasks.rados.rados.0.smithi163.stdout:1464: left oid 14 (ObjNum 570 snap 177 seq_num 570) 2023-10-09T09:11:25.956 INFO:tasks.rados.rados.0.smithi163.stdout:1465: expect (ObjNum 563 snap 172 seq_num 563) 2023-10-09T09:11:26.220 INFO:tasks.rados.rados.0.smithi163.stdout:1464: done (2 left) 2023-10-09T09:11:26.221 INFO:tasks.rados.rados.0.smithi163.stdout:1465: done (1 left) 2023-10-09T09:11:26.221 INFO:tasks.rados.rados.0.smithi163.stdout:1466: done (0 left) 2023-10-09T09:11:26.221 INFO:tasks.rados.rados.0.smithi163.stdout:1467: write oid 32 current snap is 178 2023-10-09T09:11:26.221 INFO:tasks.rados.rados.0.smithi163.stdout:1467: seq_num 571 ranges {796797=493654,1830277=796851,3397470=199327} 2023-10-09T09:11:26.228 INFO:tasks.rados.rados.0.smithi163.stdout:1467: writing smithi16367185-32 from 796797 to 1290451 tid 1 2023-10-09T09:11:26.236 INFO:tasks.rados.rados.0.smithi163.stdout:1467: writing smithi16367185-32 from 1830277 to 2627128 tid 2 2023-10-09T09:11:26.243 INFO:tasks.rados.rados.0.smithi163.stdout:1467: writing smithi16367185-32 from 3397470 to 3596797 tid 3 2023-10-09T09:11:26.243 INFO:tasks.rados.rados.0.smithi163.stdout:1468: snap_create 2023-10-09T09:11:26.243 INFO:tasks.rados.rados.0.smithi163.stdout:1467: finishing write tid 1 to smithi16367185-32 2023-10-09T09:11:26.244 INFO:tasks.rados.rados.0.smithi163.stdout:1467: finishing write tid 2 to smithi16367185-32 2023-10-09T09:11:26.246 INFO:tasks.rados.rados.0.smithi163.stdout:1467: finishing write tid 3 to smithi16367185-32 2023-10-09T09:11:26.246 INFO:tasks.rados.rados.0.smithi163.stdout:1467: finishing write tid 4 to smithi16367185-32 2023-10-09T09:11:26.246 INFO:tasks.rados.rados.0.smithi163.stdout:1467: finishing write tid 5 to smithi16367185-32 2023-10-09T09:11:26.246 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1168 (ObjNum 571 snap 178 seq_num 571) dirty exists 2023-10-09T09:11:26.247 INFO:tasks.rados.rados.0.smithi163.stdout:1467: left oid 32 (ObjNum 571 snap 178 seq_num 571) 2023-10-09T09:11:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:26 smithi181 bash[27712]: debug 2023-10-09T09:11:26.206+0000 7fde83035700 -1 mon.b@1(peon).osd e419 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:26 smithi181 bash[27712]: cluster 2023-10-09T09:11:24.993364+0000 osd.3 (osd.3) 67 : cluster [WRN] failed to encode map e418 with expected crc 2023-10-09T09:11:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:26 smithi181 bash[27712]: cluster 2023-10-09T09:11:24.998399+0000 osd.4 (osd.4) 87 : cluster [WRN] failed to encode map e418 with expected crc 2023-10-09T09:11:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:26 smithi181 bash[27712]: cluster 2023-10-09T09:11:25.250546+0000 mgr.y (mgr.34107) 84 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 270 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:11:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:26 smithi181 bash[27712]: audit 2023-10-09T09:11:25.338747+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:27.228 INFO:tasks.rados.rados.0.smithi163.stdout:1467: done (1 left) 2023-10-09T09:11:27.229 INFO:tasks.rados.rados.0.smithi163.stdout:1468: done (0 left) 2023-10-09T09:11:27.229 INFO:tasks.rados.rados.0.smithi163.stdout:1469: write oid 12 current snap is 179 2023-10-09T09:11:27.229 INFO:tasks.rados.rados.0.smithi163.stdout:1469: seq_num 572 ranges {413604=568903,1488759=656862,2624893=591709} 2023-10-09T09:11:27.235 INFO:tasks.rados.rados.0.smithi163.stdout:1469: writing smithi16367185-12 from 413604 to 982507 tid 1 2023-10-09T09:11:27.244 INFO:tasks.rados.rados.0.smithi163.stdout:1469: writing smithi16367185-12 from 1488759 to 2145621 tid 2 2023-10-09T09:11:27.251 INFO:tasks.rados.rados.0.smithi163.stdout:1469: writing smithi16367185-12 from 2624893 to 3216602 tid 3 2023-10-09T09:11:27.251 INFO:tasks.rados.rados.0.smithi163.stdout:1470: snap_create 2023-10-09T09:11:27.251 INFO:tasks.rados.rados.0.smithi163.stdout:1469: finishing write tid 1 to smithi16367185-12 2023-10-09T09:11:27.251 INFO:tasks.rados.rados.0.smithi163.stdout:1469: finishing write tid 2 to smithi16367185-12 2023-10-09T09:11:27.254 INFO:tasks.rados.rados.0.smithi163.stdout:1469: finishing write tid 3 to smithi16367185-12 2023-10-09T09:11:27.255 INFO:tasks.rados.rados.0.smithi163.stdout:1469: finishing write tid 4 to smithi16367185-12 2023-10-09T09:11:27.255 INFO:tasks.rados.rados.0.smithi163.stdout:1469: finishing write tid 5 to smithi16367185-12 2023-10-09T09:11:27.255 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 662 (ObjNum 572 snap 179 seq_num 572) dirty exists 2023-10-09T09:11:27.255 INFO:tasks.rados.rados.0.smithi163.stdout:1469: left oid 12 (ObjNum 572 snap 179 seq_num 572) 2023-10-09T09:11:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:27 smithi181 bash[27712]: cluster 2023-10-09T09:11:26.215734+0000 osd.3 (osd.3) 68 : cluster [WRN] failed to encode map e419 with expected crc 2023-10-09T09:11:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:27 smithi181 bash[27712]: cluster 2023-10-09T09:11:26.218439+0000 mon.a (mon.0) 255 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2023-10-09T09:11:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:27 smithi181 bash[27712]: cluster 2023-10-09T09:11:26.219187+0000 osd.0 (osd.0) 57 : cluster [WRN] failed to encode map e419 with expected crc 2023-10-09T09:11:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:27 smithi181 bash[27712]: cluster 2023-10-09T09:11:26.230223+0000 osd.5 (osd.5) 46 : cluster [WRN] failed to encode map e419 with expected crc 2023-10-09T09:11:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:27 smithi181 bash[27712]: debug 2023-10-09T09:11:27.214+0000 7fde83035700 -1 mon.b@1(peon).osd e420 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:28.236 INFO:tasks.rados.rados.0.smithi163.stdout:1469: done (1 left) 2023-10-09T09:11:28.236 INFO:tasks.rados.rados.0.smithi163.stdout:1470: done (0 left) 2023-10-09T09:11:28.236 INFO:tasks.rados.rados.0.smithi163.stdout:1471: read oid 36 snap -1 2023-10-09T09:11:28.237 INFO:tasks.rados.rados.0.smithi163.stdout:1471: expect deleted 2023-10-09T09:11:28.237 INFO:tasks.rados.rados.0.smithi163.stdout:1472: write oid 10 current snap is 180 2023-10-09T09:11:28.237 INFO:tasks.rados.rados.0.smithi163.stdout:1472: seq_num 573 ranges {430411=644152,1830888=400520,2935963=402981} 2023-10-09T09:11:28.244 INFO:tasks.rados.rados.0.smithi163.stdout:1472: writing smithi16367185-10 from 430411 to 1074563 tid 1 2023-10-09T09:11:28.251 INFO:tasks.rados.rados.0.smithi163.stdout:1472: writing smithi16367185-10 from 1830888 to 2231408 tid 2 2023-10-09T09:11:28.257 INFO:tasks.rados.rados.0.smithi163.stdout:1472: writing smithi16367185-10 from 2935963 to 3338944 tid 3 2023-10-09T09:11:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:1473: rollback oid 48 current snap is 180 2023-10-09T09:11:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 48 to 177 2023-10-09T09:11:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:1472: finishing write tid 1 to smithi16367185-10 2023-10-09T09:11:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:1472: finishing write tid 2 to smithi16367185-10 2023-10-09T09:11:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:1471: done (2 left) 2023-10-09T09:11:28.258 INFO:tasks.rados.rados.0.smithi163.stdout:1474: snap_remove snap 166 2023-10-09T09:11:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:28 smithi181 bash[27712]: debug 2023-10-09T09:11:28.218+0000 7fde83035700 -1 mon.b@1(peon).osd e421 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:28 smithi181 bash[27712]: cluster 2023-10-09T09:11:26.398002+0000 osd.7 (osd.7) 61 : cluster [WRN] failed to encode map e419 with expected crc 2023-10-09T09:11:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:28 smithi181 bash[27712]: cluster 2023-10-09T09:11:26.409807+0000 osd.2 (osd.2) 85 : cluster [WRN] failed to encode map e419 with expected crc 2023-10-09T09:11:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:28 smithi181 bash[27712]: cluster 2023-10-09T09:11:26.688642+0000 osd.4 (osd.4) 88 : cluster [WRN] failed to encode map e419 with expected crc 2023-10-09T09:11:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:28 smithi181 bash[27712]: cluster 2023-10-09T09:11:27.223941+0000 osd.3 (osd.3) 69 : cluster [WRN] failed to encode map e420 with expected crc 2023-10-09T09:11:28.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:28 smithi181 bash[27712]: cluster 2023-10-09T09:11:27.225875+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2023-10-09T09:11:28.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:28 smithi181 bash[27712]: cluster 2023-10-09T09:11:27.227426+0000 osd.2 (osd.2) 86 : cluster [WRN] failed to encode map e420 with expected crc 2023-10-09T09:11:28.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:28 smithi181 bash[27712]: cluster 2023-10-09T09:11:27.251091+0000 mgr.y (mgr.34107) 85 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 235 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 MiB/s rd, 411 KiB/s wr, 7 op/s 2023-10-09T09:11:29.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:11:28 smithi181 bash[54907]: ts=2023-10-09T09:11:28.667Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:11:29.255 INFO:tasks.rados.rados.0.smithi163.stdout:1472: finishing write tid 3 to smithi16367185-10 2023-10-09T09:11:29.255 INFO:tasks.rados.rados.0.smithi163.stdout:1473: finishing rollback tid 1 to smithi16367185-48 2023-10-09T09:11:29.255 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 1030 (ObjNum 698971168 snap 21875 seq_num 0) dirty dne 2023-10-09T09:11:29.255 INFO:tasks.rados.rados.0.smithi163.stdout:1472: finishing write tid 4 to smithi16367185-10 2023-10-09T09:11:29.255 INFO:tasks.rados.rados.0.smithi163.stdout:1472: finishing write tid 5 to smithi16367185-10 2023-10-09T09:11:29.255 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 576 (ObjNum 573 snap 180 seq_num 573) dirty exists 2023-10-09T09:11:29.256 INFO:tasks.rados.rados.0.smithi163.stdout:1472: left oid 10 (ObjNum 573 snap 180 seq_num 573) 2023-10-09T09:11:29.256 INFO:tasks.rados.rados.0.smithi163.stdout:1472: done (2 left) 2023-10-09T09:11:29.256 INFO:tasks.rados.rados.0.smithi163.stdout:1473: done (1 left) 2023-10-09T09:11:29.256 INFO:tasks.rados.rados.0.smithi163.stdout:1474: done (0 left) 2023-10-09T09:11:29.256 INFO:tasks.rados.rados.0.smithi163.stdout:1475: snap_remove snap 178 2023-10-09T09:11:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:29 smithi181 bash[27712]: cluster 2023-10-09T09:11:28.228211+0000 osd.3 (osd.3) 70 : cluster [WRN] failed to encode map e421 with expected crc 2023-10-09T09:11:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:29 smithi181 bash[27712]: cluster 2023-10-09T09:11:28.231940+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2023-10-09T09:11:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:29 smithi181 bash[27712]: cluster 2023-10-09T09:11:28.233865+0000 osd.2 (osd.2) 87 : cluster [WRN] failed to encode map e421 with expected crc 2023-10-09T09:11:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:29 smithi181 bash[27712]: cluster 2023-10-09T09:11:28.240608+0000 osd.0 (osd.0) 58 : cluster [WRN] failed to encode map e421 with expected crc 2023-10-09T09:11:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:29 smithi181 bash[27712]: debug 2023-10-09T09:11:29.241+0000 7fde83035700 -1 mon.b@1(peon).osd e422 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:30.008 INFO:tasks.rados.rados.0.smithi163.stdout:1475: done (0 left) 2023-10-09T09:11:30.009 INFO:tasks.rados.rados.0.smithi163.stdout:1476: read oid 48 snap -1 2023-10-09T09:11:30.009 INFO:tasks.rados.rados.0.smithi163.stdout:1476: expect deleted 2023-10-09T09:11:30.009 INFO:tasks.rados.rados.0.smithi163.stdout:1477: write oid 25 current snap is 180 2023-10-09T09:11:30.009 INFO:tasks.rados.rados.0.smithi163.stdout:1477: seq_num 574 ranges {447218=719401,1889370=660531,2963386=614253} 2023-10-09T09:11:30.016 INFO:tasks.rados.rados.0.smithi163.stdout:1477: writing smithi16367185-25 from 447218 to 1166619 tid 1 2023-10-09T09:11:30.024 INFO:tasks.rados.rados.0.smithi163.stdout:1477: writing smithi16367185-25 from 1889370 to 2549901 tid 2 2023-10-09T09:11:30.031 INFO:tasks.rados.rados.0.smithi163.stdout:1477: writing smithi16367185-25 from 2963386 to 3577639 tid 3 2023-10-09T09:11:30.031 INFO:tasks.rados.rados.0.smithi163.stdout:1478: read oid 33 snap -1 2023-10-09T09:11:30.031 INFO:tasks.rados.rados.0.smithi163.stdout:1478: expect (ObjNum 507 snap 159 seq_num 507) 2023-10-09T09:11:30.031 INFO:tasks.rados.rados.0.smithi163.stdout:1479: snap_create 2023-10-09T09:11:30.031 INFO:tasks.rados.rados.0.smithi163.stdout:1477: finishing write tid 1 to smithi16367185-25 2023-10-09T09:11:30.032 INFO:tasks.rados.rados.0.smithi163.stdout:1477: finishing write tid 2 to smithi16367185-25 2023-10-09T09:11:30.037 INFO:tasks.rados.rados.0.smithi163.stdout:1477: finishing write tid 3 to smithi16367185-25 2023-10-09T09:11:30.037 INFO:tasks.rados.rados.0.smithi163.stdout:1477: finishing write tid 4 to smithi16367185-25 2023-10-09T09:11:30.037 INFO:tasks.rados.rados.0.smithi163.stdout:1477: finishing write tid 5 to smithi16367185-25 2023-10-09T09:11:30.038 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 675 (ObjNum 574 snap 180 seq_num 574) dirty exists 2023-10-09T09:11:30.038 INFO:tasks.rados.rados.0.smithi163.stdout:1477: left oid 25 (ObjNum 574 snap 180 seq_num 574) 2023-10-09T09:11:30.038 INFO:tasks.rados.rados.0.smithi163.stdout:1478: expect (ObjNum 507 snap 159 seq_num 507) 2023-10-09T09:11:30.258 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:29 smithi181 bash[27712]: debug 2023-10-09T09:11:29.989+0000 7fde83035700 -1 mon.b@1(peon).osd e423 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:30 smithi181 bash[27712]: cluster 2023-10-09T09:11:29.248252+0000 osd.3 (osd.3) 71 : cluster [WRN] failed to encode map e422 with expected crc 2023-10-09T09:11:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:30 smithi181 bash[27712]: cluster 2023-10-09T09:11:29.251198+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2023-10-09T09:11:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:30 smithi181 bash[27712]: cluster 2023-10-09T09:11:29.251538+0000 mgr.y (mgr.34107) 86 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 235 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 MiB/s rd, 411 KiB/s wr, 7 op/s 2023-10-09T09:11:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:30 smithi181 bash[27712]: cluster 2023-10-09T09:11:29.253729+0000 osd.2 (osd.2) 88 : cluster [WRN] failed to encode map e422 with expected crc 2023-10-09T09:11:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:30 smithi181 bash[27712]: cluster 2023-10-09T09:11:30.000635+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2023-10-09T09:11:31.022 INFO:tasks.rados.rados.0.smithi163.stdout:1476: done (3 left) 2023-10-09T09:11:31.022 INFO:tasks.rados.rados.0.smithi163.stdout:1477: done (2 left) 2023-10-09T09:11:31.023 INFO:tasks.rados.rados.0.smithi163.stdout:1478: done (1 left) 2023-10-09T09:11:31.023 INFO:tasks.rados.rados.0.smithi163.stdout:1479: done (0 left) 2023-10-09T09:11:31.023 INFO:tasks.rados.rados.0.smithi163.stdout:1480: write oid 7 current snap is 181 2023-10-09T09:11:31.023 INFO:tasks.rados.rados.0.smithi163.stdout:1480: seq_num 575 ranges {464025=794650,1947852=404189,2990809=425525} 2023-10-09T09:11:31.030 INFO:tasks.rados.rados.0.smithi163.stdout:1480: writing smithi16367185-7 from 464025 to 1258675 tid 1 2023-10-09T09:11:31.037 INFO:tasks.rados.rados.0.smithi163.stdout:1480: writing smithi16367185-7 from 1947852 to 2352041 tid 2 2023-10-09T09:11:31.044 INFO:tasks.rados.rados.0.smithi163.stdout:1480: writing smithi16367185-7 from 2990809 to 3416334 tid 3 2023-10-09T09:11:31.044 INFO:tasks.rados.rados.0.smithi163.stdout:1481: write oid 21 current snap is 181 2023-10-09T09:11:31.044 INFO:tasks.rados.rados.0.smithi163.stdout:1481: seq_num 576 ranges {480832=469899,1606334=664200,3018232=662600} 2023-10-09T09:11:31.049 INFO:tasks.rados.rados.0.smithi163.stdout:1481: writing smithi16367185-21 from 480832 to 950731 tid 1 2023-10-09T09:11:31.057 INFO:tasks.rados.rados.0.smithi163.stdout:1481: writing smithi16367185-21 from 1606334 to 2270534 tid 2 2023-10-09T09:11:31.065 INFO:tasks.rados.rados.0.smithi163.stdout:1481: writing smithi16367185-21 from 3018232 to 3680832 tid 3 2023-10-09T09:11:31.065 INFO:tasks.rados.rados.0.smithi163.stdout:1482: write oid 44 current snap is 181 2023-10-09T09:11:31.065 INFO:tasks.rados.rados.0.smithi163.stdout:1480: finishing write tid 1 to smithi16367185-7 2023-10-09T09:11:31.065 INFO:tasks.rados.rados.0.smithi163.stdout:1482: seq_num 577 ranges {497639=545148,1548463=407858,2529302=564422,3528143=169496} 2023-10-09T09:11:31.071 INFO:tasks.rados.rados.0.smithi163.stdout:1482: writing smithi16367185-44 from 497639 to 1042787 tid 1 2023-10-09T09:11:31.076 INFO:tasks.rados.rados.0.smithi163.stdout:1482: writing smithi16367185-44 from 1548463 to 1956321 tid 2 2023-10-09T09:11:31.083 INFO:tasks.rados.rados.0.smithi163.stdout:1482: writing smithi16367185-44 from 2529302 to 3093724 tid 3 2023-10-09T09:11:31.087 INFO:tasks.rados.rados.0.smithi163.stdout:1482: writing smithi16367185-44 from 3528143 to 3697639 tid 4 2023-10-09T09:11:31.087 INFO:tasks.rados.rados.0.smithi163.stdout:1483: delete oid 12 current snap is 181 2023-10-09T09:11:31.087 INFO:tasks.rados.rados.0.smithi163.stdout:1480: finishing write tid 2 to smithi16367185-7 2023-10-09T09:11:31.087 INFO:tasks.rados.rados.0.smithi163.stdout:1480: finishing write tid 3 to smithi16367185-7 2023-10-09T09:11:31.087 INFO:tasks.rados.rados.0.smithi163.stdout:1480: finishing write tid 4 to smithi16367185-7 2023-10-09T09:11:31.087 INFO:tasks.rados.rados.0.smithi163.stdout:1480: finishing write tid 5 to smithi16367185-7 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 366 (ObjNum 575 snap 181 seq_num 575) dirty exists 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:1480: left oid 7 (ObjNum 575 snap 181 seq_num 575) 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:1481: finishing write tid 1 to smithi16367185-21 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:1481: finishing write tid 2 to smithi16367185-21 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:1481: finishing write tid 3 to smithi16367185-21 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:1481: finishing write tid 4 to smithi16367185-21 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:1481: finishing write tid 5 to smithi16367185-21 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 371 (ObjNum 576 snap 181 seq_num 576) dirty exists 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:1481: left oid 21 (ObjNum 576 snap 181 seq_num 576) 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:1482: finishing write tid 1 to smithi16367185-44 2023-10-09T09:11:31.088 INFO:tasks.rados.rados.0.smithi163.stdout:1482: finishing write tid 2 to smithi16367185-44 2023-10-09T09:11:31.089 INFO:tasks.rados.rados.0.smithi163.stdout:1482: finishing write tid 3 to smithi16367185-44 2023-10-09T09:11:31.091 INFO:tasks.rados.rados.0.smithi163.stdout:1482: finishing write tid 4 to smithi16367185-44 2023-10-09T09:11:31.091 INFO:tasks.rados.rados.0.smithi163.stdout:1482: finishing write tid 5 to smithi16367185-44 2023-10-09T09:11:31.092 INFO:tasks.rados.rados.0.smithi163.stdout:1482: finishing write tid 6 to smithi16367185-44 2023-10-09T09:11:31.092 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1182 (ObjNum 577 snap 181 seq_num 577) dirty exists 2023-10-09T09:11:31.092 INFO:tasks.rados.rados.0.smithi163.stdout:1482: left oid 44 (ObjNum 577 snap 181 seq_num 577) 2023-10-09T09:11:31.095 INFO:tasks.rados.rados.0.smithi163.stdout:1480: done (3 left) 2023-10-09T09:11:31.095 INFO:tasks.rados.rados.0.smithi163.stdout:1481: done (2 left) 2023-10-09T09:11:31.095 INFO:tasks.rados.rados.0.smithi163.stdout:1482: done (1 left) 2023-10-09T09:11:31.095 INFO:tasks.rados.rados.0.smithi163.stdout:1483: done (0 left) 2023-10-09T09:11:31.095 INFO:tasks.rados.rados.0.smithi163.stdout:1484: write oid 3 current snap is 181 2023-10-09T09:11:31.095 INFO:tasks.rados.rados.0.smithi163.stdout:1484: seq_num 579 ranges {531253=411999,1381780=411527,2300501=586966,3376974=354279} 2023-10-09T09:11:31.101 INFO:tasks.rados.rados.0.smithi163.stdout:1484: writing smithi16367185-3 from 531253 to 943252 tid 1 2023-10-09T09:11:31.107 INFO:tasks.rados.rados.0.smithi163.stdout:1484: writing smithi16367185-3 from 1381780 to 1793307 tid 2 2023-10-09T09:11:31.113 INFO:tasks.rados.rados.0.smithi163.stdout:1484: writing smithi16367185-3 from 2300501 to 2887467 tid 3 2023-10-09T09:11:31.118 INFO:tasks.rados.rados.0.smithi163.stdout:1484: writing smithi16367185-3 from 3376974 to 3731253 tid 4 2023-10-09T09:11:31.118 INFO:tasks.rados.rados.0.smithi163.stdout:1485: snap_remove snap 177 2023-10-09T09:11:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:31 smithi181 bash[27712]: debug 2023-10-09T09:11:31.005+0000 7fde83035700 -1 mon.b@1(peon).osd e424 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:31 smithi181 bash[27712]: cluster 2023-10-09T09:11:29.996287+0000 osd.3 (osd.3) 72 : cluster [WRN] failed to encode map e423 with expected crc 2023-10-09T09:11:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:31 smithi181 bash[27712]: cluster 2023-10-09T09:11:30.005878+0000 osd.0 (osd.0) 59 : cluster [WRN] failed to encode map e423 with expected crc 2023-10-09T09:11:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:31 smithi181 bash[27712]: cluster 2023-10-09T09:11:30.009966+0000 osd.5 (osd.5) 47 : cluster [WRN] failed to encode map e423 with expected crc 2023-10-09T09:11:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:31 smithi181 bash[27712]: cluster 2023-10-09T09:11:30.018518+0000 osd.2 (osd.2) 89 : cluster [WRN] failed to encode map e423 with expected crc 2023-10-09T09:11:31.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:31 smithi181 bash[27712]: cluster 2023-10-09T09:11:31.017599+0000 mon.a (mon.0) 260 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2023-10-09T09:11:32.029 INFO:tasks.rados.rados.0.smithi163.stdout:1485: done (1 left) 2023-10-09T09:11:32.029 INFO:tasks.rados.rados.0.smithi163.stdout:1486: read oid 36 snap -1 2023-10-09T09:11:32.029 INFO:tasks.rados.rados.0.smithi163.stdout:1486: expect deleted 2023-10-09T09:11:32.029 INFO:tasks.rados.rados.0.smithi163.stdout:1487: write oid 2 current snap is 181 2023-10-09T09:11:32.030 INFO:tasks.rados.rados.0.smithi163.stdout:1484: finishing write tid 1 to smithi16367185-3 2023-10-09T09:11:32.030 INFO:tasks.rados.rados.0.smithi163.stdout:1487: seq_num 580 ranges {548060=487248,1440262=671538,2844277=798238} 2023-10-09T09:11:32.035 INFO:tasks.rados.rados.0.smithi163.stdout:1487: writing smithi16367185-2 from 548060 to 1035308 tid 1 2023-10-09T09:11:32.042 INFO:tasks.rados.rados.0.smithi163.stdout:1487: writing smithi16367185-2 from 1440262 to 2111800 tid 2 2023-10-09T09:11:32.051 INFO:tasks.rados.rados.0.smithi163.stdout:1487: writing smithi16367185-2 from 2844277 to 3642515 tid 3 2023-10-09T09:11:32.051 INFO:tasks.rados.rados.0.smithi163.stdout:1488: snap_create 2023-10-09T09:11:32.051 INFO:tasks.rados.rados.0.smithi163.stdout:1484: finishing write tid 2 to smithi16367185-3 2023-10-09T09:11:32.051 INFO:tasks.rados.rados.0.smithi163.stdout:1484: finishing write tid 3 to smithi16367185-3 2023-10-09T09:11:32.052 INFO:tasks.rados.rados.0.smithi163.stdout:1484: finishing write tid 4 to smithi16367185-3 2023-10-09T09:11:32.052 INFO:tasks.rados.rados.0.smithi163.stdout:1484: finishing write tid 5 to smithi16367185-3 2023-10-09T09:11:32.052 INFO:tasks.rados.rados.0.smithi163.stdout:1484: finishing write tid 6 to smithi16367185-3 2023-10-09T09:11:32.052 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 437 (ObjNum 579 snap 181 seq_num 579) dirty exists 2023-10-09T09:11:32.052 INFO:tasks.rados.rados.0.smithi163.stdout:1484: left oid 3 (ObjNum 579 snap 181 seq_num 579) 2023-10-09T09:11:32.052 INFO:tasks.rados.rados.0.smithi163.stdout:1487: finishing write tid 1 to smithi16367185-2 2023-10-09T09:11:32.053 INFO:tasks.rados.rados.0.smithi163.stdout:1487: finishing write tid 2 to smithi16367185-2 2023-10-09T09:11:32.055 INFO:tasks.rados.rados.0.smithi163.stdout:1487: finishing write tid 3 to smithi16367185-2 2023-10-09T09:11:32.055 INFO:tasks.rados.rados.0.smithi163.stdout:1487: finishing write tid 4 to smithi16367185-2 2023-10-09T09:11:32.055 INFO:tasks.rados.rados.0.smithi163.stdout:1487: finishing write tid 5 to smithi16367185-2 2023-10-09T09:11:32.056 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 453 (ObjNum 580 snap 181 seq_num 580) dirty exists 2023-10-09T09:11:32.056 INFO:tasks.rados.rados.0.smithi163.stdout:1487: left oid 2 (ObjNum 580 snap 181 seq_num 580) 2023-10-09T09:11:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:32 smithi181 bash[27712]: debug 2023-10-09T09:11:32.013+0000 7fde83035700 -1 mon.b@1(peon).osd e425 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:32 smithi181 bash[27712]: cluster 2023-10-09T09:11:30.675031+0000 osd.6 (osd.6) 39 : cluster [WRN] failed to encode map e423 with expected crc 2023-10-09T09:11:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:32 smithi181 bash[27712]: cluster 2023-10-09T09:11:30.710151+0000 osd.4 (osd.4) 89 : cluster [WRN] failed to encode map e423 with expected crc 2023-10-09T09:11:32.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:32 smithi181 bash[27712]: cluster 2023-10-09T09:11:31.014155+0000 osd.3 (osd.3) 73 : cluster [WRN] failed to encode map e424 with expected crc 2023-10-09T09:11:32.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:32 smithi181 bash[27712]: cluster 2023-10-09T09:11:31.018310+0000 osd.5 (osd.5) 48 : cluster [WRN] failed to encode map e424 with expected crc 2023-10-09T09:11:32.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:32 smithi181 bash[27712]: cluster 2023-10-09T09:11:31.033523+0000 osd.1 (osd.1) 39 : cluster [WRN] failed to encode map e424 with expected crc 2023-10-09T09:11:32.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:32 smithi181 bash[27712]: cluster 2023-10-09T09:11:31.089140+0000 osd.2 (osd.2) 90 : cluster [WRN] failed to encode map e424 with expected crc 2023-10-09T09:11:32.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:32 smithi181 bash[27712]: cluster 2023-10-09T09:11:31.252689+0000 mgr.y (mgr.34107) 87 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 250 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.3 MiB/s wr, 15 op/s 2023-10-09T09:11:32.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:32 smithi181 bash[27712]: cluster 2023-10-09T09:11:32.024126+0000 mon.a (mon.0) 261 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2023-10-09T09:11:33.035 INFO:tasks.rados.rados.0.smithi163.stdout:1484: done (3 left) 2023-10-09T09:11:33.035 INFO:tasks.rados.rados.0.smithi163.stdout:1486: done (2 left) 2023-10-09T09:11:33.035 INFO:tasks.rados.rados.0.smithi163.stdout:1487: done (1 left) 2023-10-09T09:11:33.035 INFO:tasks.rados.rados.0.smithi163.stdout:1488: done (0 left) 2023-10-09T09:11:33.035 INFO:tasks.rados.rados.0.smithi163.stdout:1489: snap_create 2023-10-09T09:11:33.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:33 smithi181 bash[27712]: debug 2023-10-09T09:11:33.017+0000 7fde83035700 -1 mon.b@1(peon).osd e426 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:33.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:33 smithi181 bash[27712]: cluster 2023-10-09T09:11:32.021901+0000 osd.3 (osd.3) 74 : cluster [WRN] failed to encode map e425 with expected crc 2023-10-09T09:11:33.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:33 smithi181 bash[27712]: cluster 2023-10-09T09:11:32.026813+0000 osd.2 (osd.2) 91 : cluster [WRN] failed to encode map e425 with expected crc 2023-10-09T09:11:33.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:33 smithi181 bash[27712]: cluster 2023-10-09T09:11:32.033012+0000 osd.0 (osd.0) 60 : cluster [WRN] failed to encode map e425 with expected crc 2023-10-09T09:11:33.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:33 smithi181 bash[27712]: cluster 2023-10-09T09:11:32.037432+0000 osd.7 (osd.7) 62 : cluster [WRN] failed to encode map e425 with expected crc 2023-10-09T09:11:33.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:33 smithi181 bash[27712]: cluster 2023-10-09T09:11:33.029077+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2023-10-09T09:11:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:11:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:11:33] "GET /metrics HTTP/1.1" 200 36073 "" "Prometheus/2.33.4" 2023-10-09T09:11:34.042 INFO:tasks.rados.rados.0.smithi163.stdout:1489: done (0 left) 2023-10-09T09:11:34.042 INFO:tasks.rados.rados.0.smithi163.stdout:1490: snap_create 2023-10-09T09:11:34.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:34 smithi181 bash[27712]: debug 2023-10-09T09:11:34.025+0000 7fde83035700 -1 mon.b@1(peon).osd e427 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:34.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:34 smithi181 bash[27712]: cluster 2023-10-09T09:11:33.025203+0000 osd.3 (osd.3) 75 : cluster [WRN] failed to encode map e426 with expected crc 2023-10-09T09:11:34.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:34 smithi181 bash[27712]: cluster 2023-10-09T09:11:33.031342+0000 osd.4 (osd.4) 90 : cluster [WRN] failed to encode map e426 with expected crc 2023-10-09T09:11:34.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:34 smithi181 bash[27712]: cluster 2023-10-09T09:11:33.253347+0000 mgr.y (mgr.34107) 88 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 250 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2023-10-09T09:11:34.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:34 smithi181 bash[27712]: cluster 2023-10-09T09:11:34.035487+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2023-10-09T09:11:35.010 INFO:tasks.rados.rados.0.smithi163.stdout:1490: done (0 left) 2023-10-09T09:11:35.010 INFO:tasks.rados.rados.0.smithi163.stdout:1491: snap_remove snap 174 2023-10-09T09:11:35.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:34 smithi181 bash[27712]: debug 2023-10-09T09:11:34.989+0000 7fde83035700 -1 mon.b@1(peon).osd e428 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:35 smithi181 bash[27712]: cluster 2023-10-09T09:11:34.033075+0000 osd.3 (osd.3) 76 : cluster [WRN] failed to encode map e427 with expected crc 2023-10-09T09:11:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:35 smithi181 bash[27712]: cluster 2023-10-09T09:11:34.039989+0000 osd.0 (osd.0) 61 : cluster [WRN] failed to encode map e427 with expected crc 2023-10-09T09:11:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:35 smithi181 bash[27712]: cluster 2023-10-09T09:11:35.001051+0000 mon.a (mon.0) 264 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2023-10-09T09:11:36.030 INFO:tasks.rados.rados.0.smithi163.stdout:1491: done (0 left) 2023-10-09T09:11:36.030 INFO:tasks.rados.rados.0.smithi163.stdout:1492: snap_create 2023-10-09T09:11:36.276 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:36 smithi181 bash[27712]: debug 2023-10-09T09:11:36.013+0000 7fde83035700 -1 mon.b@1(peon).osd e429 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:36 smithi181 bash[27712]: cluster 2023-10-09T09:11:34.405282+0000 osd.7 (osd.7) 63 : cluster [WRN] failed to encode map e427 with expected crc 2023-10-09T09:11:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:36 smithi181 bash[27712]: cluster 2023-10-09T09:11:34.998369+0000 osd.3 (osd.3) 77 : cluster [WRN] failed to encode map e428 with expected crc 2023-10-09T09:11:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:36 smithi181 bash[27712]: cluster 2023-10-09T09:11:35.006224+0000 osd.5 (osd.5) 49 : cluster [WRN] failed to encode map e427 with expected crc 2023-10-09T09:11:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:36 smithi181 bash[27712]: cluster 2023-10-09T09:11:35.110015+0000 osd.1 (osd.1) 40 : cluster [WRN] failed to encode map e427 with expected crc 2023-10-09T09:11:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:36 smithi181 bash[27712]: cluster 2023-10-09T09:11:35.254109+0000 mgr.y (mgr.34107) 89 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 250 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:11:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:36 smithi181 bash[27712]: cluster 2023-10-09T09:11:35.284679+0000 osd.2 (osd.2) 92 : cluster [WRN] failed to encode map e427 with expected crc 2023-10-09T09:11:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:36 smithi181 bash[27712]: cluster 2023-10-09T09:11:35.483778+0000 osd.0 (osd.0) 62 : cluster [WRN] failed to encode map e428 with expected crc 2023-10-09T09:11:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:36 smithi181 bash[27712]: cluster 2023-10-09T09:11:36.029721+0000 mon.a (mon.0) 265 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2023-10-09T09:11:37.035 INFO:tasks.rados.rados.0.smithi163.stdout:1492: done (0 left) 2023-10-09T09:11:37.036 INFO:tasks.rados.rados.0.smithi163.stdout:1493: snap_remove snap 180 2023-10-09T09:11:37.280 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:37 smithi181 bash[27712]: debug 2023-10-09T09:11:37.017+0000 7fde83035700 -1 mon.b@1(peon).osd e430 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:37 smithi181 bash[27712]: cluster 2023-10-09T09:11:35.433930+0000 osd.7 (osd.7) 64 : cluster [WRN] failed to encode map e428 with expected crc 2023-10-09T09:11:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:37 smithi181 bash[27712]: cluster 2023-10-09T09:11:36.027480+0000 osd.0 (osd.0) 63 : cluster [WRN] failed to encode map e429 with expected crc 2023-10-09T09:11:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:37 smithi181 bash[27712]: cluster 2023-10-09T09:11:36.027496+0000 osd.3 (osd.3) 78 : cluster [WRN] failed to encode map e429 with expected crc 2023-10-09T09:11:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:37 smithi181 bash[27712]: cluster 2023-10-09T09:11:36.092181+0000 osd.1 (osd.1) 41 : cluster [WRN] failed to encode map e429 with expected crc 2023-10-09T09:11:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:37 smithi181 bash[27712]: cluster 2023-10-09T09:11:37.033642+0000 mon.a (mon.0) 266 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2023-10-09T09:11:38.041 INFO:tasks.rados.rados.0.smithi163.stdout:1493: done (0 left) 2023-10-09T09:11:38.041 INFO:tasks.rados.rados.0.smithi163.stdout:1494: rollback oid 50 current snap is 185 2023-10-09T09:11:38.041 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 50 to 175 2023-10-09T09:11:38.041 INFO:tasks.rados.rados.0.smithi163.stdout:1495: write oid 24 current snap is 185 2023-10-09T09:11:38.041 INFO:tasks.rados.rados.0.smithi163.stdout:1495: seq_num 581 ranges {564867=562497,1898744=415196,2755347=725863} 2023-10-09T09:11:38.048 INFO:tasks.rados.rados.0.smithi163.stdout:1495: writing smithi16367185-24 from 564867 to 1127364 tid 1 2023-10-09T09:11:38.055 INFO:tasks.rados.rados.0.smithi163.stdout:1495: writing smithi16367185-24 from 1898744 to 2313940 tid 2 2023-10-09T09:11:38.063 INFO:tasks.rados.rados.0.smithi163.stdout:1495: writing smithi16367185-24 from 2755347 to 3481210 tid 3 2023-10-09T09:11:38.063 INFO:tasks.rados.rados.0.smithi163.stdout:1496: write oid 49 current snap is 185 2023-10-09T09:11:38.063 INFO:tasks.rados.rados.0.smithi163.stdout:1494: finishing rollback tid 0 to smithi16367185-50 2023-10-09T09:11:38.063 INFO:tasks.rados.rados.0.smithi163.stdout:1496: seq_num 582 ranges {581674=637746,1840873=558854,3066417=537135} 2023-10-09T09:11:38.069 INFO:tasks.rados.rados.0.smithi163.stdout:1496: writing smithi16367185-49 from 581674 to 1219420 tid 1 2023-10-09T09:11:38.076 INFO:tasks.rados.rados.0.smithi163.stdout:1496: writing smithi16367185-49 from 1840873 to 2399727 tid 2 2023-10-09T09:11:38.083 INFO:tasks.rados.rados.0.smithi163.stdout:1496: writing smithi16367185-49 from 3066417 to 3603552 tid 3 2023-10-09T09:11:38.083 INFO:tasks.rados.rados.0.smithi163.stdout:1497: snap_remove snap 140 2023-10-09T09:11:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:38 smithi181 bash[27712]: debug 2023-10-09T09:11:38.025+0000 7fde83035700 -1 mon.b@1(peon).osd e431 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:38 smithi181 bash[27712]: cluster 2023-10-09T09:11:37.030119+0000 osd.3 (osd.3) 79 : cluster [WRN] failed to encode map e430 with expected crc 2023-10-09T09:11:38.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:38 smithi181 bash[27712]: cluster 2023-10-09T09:11:37.031768+0000 osd.4 (osd.4) 91 : cluster [WRN] failed to encode map e430 with expected crc 2023-10-09T09:11:38.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:38 smithi181 bash[27712]: cluster 2023-10-09T09:11:37.254820+0000 mgr.y (mgr.34107) 90 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 254 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 478 KiB/s wr, 1 op/s 2023-10-09T09:11:38.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:38 smithi181 bash[27712]: cluster 2023-10-09T09:11:38.039032+0000 mon.a (mon.0) 267 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2023-10-09T09:11:39.037 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:11:38 smithi181 bash[54907]: ts=2023-10-09T09:11:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:11:39.046 INFO:tasks.rados.rados.0.smithi163.stdout:1497: done (3 left) 2023-10-09T09:11:39.047 INFO:tasks.rados.rados.0.smithi163.stdout:1498: delete oid 4 current snap is 185 2023-10-09T09:11:39.047 INFO:tasks.rados.rados.0.smithi163.stdout:1495: finishing write tid 1 to smithi16367185-24 2023-10-09T09:11:39.047 INFO:tasks.rados.rados.0.smithi163.stdout:1494: finishing rollback tid 1 to smithi16367185-50 2023-10-09T09:11:39.047 INFO:tasks.rados.rados.0.smithi163.stdout:1494: finishing rollback tid 2 to smithi16367185-50 2023-10-09T09:11:39.047 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 608 (ObjNum 562 snap 172 seq_num 562) dirty exists 2023-10-09T09:11:39.047 INFO:tasks.rados.rados.0.smithi163.stdout:1495: finishing write tid 2 to smithi16367185-24 2023-10-09T09:11:39.047 INFO:tasks.rados.rados.0.smithi163.stdout:1495: finishing write tid 3 to smithi16367185-24 2023-10-09T09:11:39.047 INFO:tasks.rados.rados.0.smithi163.stdout:1495: finishing write tid 4 to smithi16367185-24 2023-10-09T09:11:39.047 INFO:tasks.rados.rados.0.smithi163.stdout:1495: finishing write tid 5 to smithi16367185-24 2023-10-09T09:11:39.047 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 397 (ObjNum 581 snap 185 seq_num 581) dirty exists 2023-10-09T09:11:39.048 INFO:tasks.rados.rados.0.smithi163.stdout:1495: left oid 24 (ObjNum 581 snap 185 seq_num 581) 2023-10-09T09:11:39.048 INFO:tasks.rados.rados.0.smithi163.stdout:1496: finishing write tid 1 to smithi16367185-49 2023-10-09T09:11:39.048 INFO:tasks.rados.rados.0.smithi163.stdout:1496: finishing write tid 2 to smithi16367185-49 2023-10-09T09:11:39.048 INFO:tasks.rados.rados.0.smithi163.stdout:1496: finishing write tid 3 to smithi16367185-49 2023-10-09T09:11:39.048 INFO:tasks.rados.rados.0.smithi163.stdout:1496: finishing write tid 4 to smithi16367185-49 2023-10-09T09:11:39.048 INFO:tasks.rados.rados.0.smithi163.stdout:1496: finishing write tid 5 to smithi16367185-49 2023-10-09T09:11:39.048 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 239 (ObjNum 582 snap 185 seq_num 582) dirty exists 2023-10-09T09:11:39.048 INFO:tasks.rados.rados.0.smithi163.stdout:1496: left oid 49 (ObjNum 582 snap 185 seq_num 582) 2023-10-09T09:11:39.059 INFO:tasks.rados.rados.0.smithi163.stdout:1494: done (3 left) 2023-10-09T09:11:39.060 INFO:tasks.rados.rados.0.smithi163.stdout:1495: done (2 left) 2023-10-09T09:11:39.060 INFO:tasks.rados.rados.0.smithi163.stdout:1496: done (1 left) 2023-10-09T09:11:39.060 INFO:tasks.rados.rados.0.smithi163.stdout:1498: done (0 left) 2023-10-09T09:11:39.060 INFO:tasks.rados.rados.0.smithi163.stdout:1499: read oid 1 snap -1 2023-10-09T09:11:39.060 INFO:tasks.rados.rados.0.smithi163.stdout:1499: expect (ObjNum 510 snap 159 seq_num 510) 2023-10-09T09:11:39.060 INFO:tasks.rados.rados.0.smithi163.stdout:1500: read oid 47 snap -1 2023-10-09T09:11:39.060 INFO:tasks.rados.rados.0.smithi163.stdout:1500: expect (ObjNum 458 snap 147 seq_num 458) 2023-10-09T09:11:39.060 INFO:tasks.rados.rados.0.smithi163.stdout:1501: read oid 16 snap -1 2023-10-09T09:11:39.061 INFO:tasks.rados.rados.0.smithi163.stdout:1501: expect (ObjNum 505 snap 157 seq_num 505) 2023-10-09T09:11:39.061 INFO:tasks.rados.rados.0.smithi163.stdout:1502: snap_remove snap 179 2023-10-09T09:11:39.287 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:39 smithi181 bash[27712]: debug 2023-10-09T09:11:39.029+0000 7fde83035700 -1 mon.b@1(peon).osd e432 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:39.287 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:39 smithi181 bash[27712]: cluster 2023-10-09T09:11:38.036893+0000 osd.3 (osd.3) 80 : cluster [WRN] failed to encode map e431 with expected crc 2023-10-09T09:11:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:39 smithi181 bash[27712]: cluster 2023-10-09T09:11:38.037593+0000 osd.6 (osd.6) 40 : cluster [WRN] failed to encode map e430 with expected crc 2023-10-09T09:11:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:39 smithi181 bash[27712]: cluster 2023-10-09T09:11:38.042459+0000 osd.5 (osd.5) 50 : cluster [WRN] failed to encode map e431 with expected crc 2023-10-09T09:11:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:39 smithi181 bash[27712]: cluster 2023-10-09T09:11:39.043020+0000 mon.a (mon.0) 268 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2023-10-09T09:11:40.015 INFO:tasks.rados.rados.0.smithi163.stdout:1502: done (3 left) 2023-10-09T09:11:40.015 INFO:tasks.rados.rados.0.smithi163.stdout:1503: write oid 50 current snap is 185 2023-10-09T09:11:40.015 INFO:tasks.rados.rados.0.smithi163.stdout:1503: seq_num 584 ranges {615288=788244,1957837=562523,3121263=559679} 2023-10-09T09:11:40.025 INFO:tasks.rados.rados.0.smithi163.stdout:1503: writing smithi16367185-50 from 615288 to 1403532 tid 1 2023-10-09T09:11:40.030 INFO:tasks.rados.rados.0.smithi163.stdout:1503: writing smithi16367185-50 from 1957837 to 2520360 tid 2 2023-10-09T09:11:40.037 INFO:tasks.rados.rados.0.smithi163.stdout:1503: writing smithi16367185-50 from 3121263 to 3680942 tid 3 2023-10-09T09:11:40.037 INFO:tasks.rados.rados.0.smithi163.stdout:1504: delete oid 42 current snap is 185 2023-10-09T09:11:40.038 INFO:tasks.rados.rados.0.smithi163.stdout:1499: expect (ObjNum 510 snap 159 seq_num 510) 2023-10-09T09:11:40.255 INFO:tasks.rados.rados.0.smithi163.stdout:1501: expect (ObjNum 505 snap 157 seq_num 505) 2023-10-09T09:11:40.289 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:39 smithi181 bash[27712]: debug 2023-10-09T09:11:39.993+0000 7fde83035700 -1 mon.b@1(peon).osd e433 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:40.460 INFO:tasks.rados.rados.0.smithi163.stdout:1500: expect (ObjNum 458 snap 147 seq_num 458) 2023-10-09T09:11:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:40 smithi181 bash[27712]: cluster 2023-10-09T09:11:39.037656+0000 osd.3 (osd.3) 81 : cluster [WRN] failed to encode map e432 with expected crc 2023-10-09T09:11:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:40 smithi181 bash[27712]: cluster 2023-10-09T09:11:39.044204+0000 osd.2 (osd.2) 93 : cluster [WRN] failed to encode map e432 with expected crc 2023-10-09T09:11:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:40 smithi181 bash[27712]: cluster 2023-10-09T09:11:39.255524+0000 mgr.y (mgr.34107) 91 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 254 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 478 KiB/s wr, 1 op/s 2023-10-09T09:11:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:40 smithi181 bash[27712]: cluster 2023-10-09T09:11:40.004287+0000 mon.a (mon.0) 269 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2023-10-09T09:11:40.583 INFO:tasks.rados.rados.0.smithi163.stdout:1503: finishing write tid 1 to smithi16367185-50 2023-10-09T09:11:40.583 INFO:tasks.rados.rados.0.smithi163.stdout:1503: finishing write tid 2 to smithi16367185-50 2023-10-09T09:11:40.583 INFO:tasks.rados.rados.0.smithi163.stdout:1499: done (4 left) 2023-10-09T09:11:40.583 INFO:tasks.rados.rados.0.smithi163.stdout:1500: done (3 left) 2023-10-09T09:11:40.583 INFO:tasks.rados.rados.0.smithi163.stdout:1501: done (2 left) 2023-10-09T09:11:40.584 INFO:tasks.rados.rados.0.smithi163.stdout:1504: done (1 left) 2023-10-09T09:11:40.584 INFO:tasks.rados.rados.0.smithi163.stdout:1505: read oid 46 snap -1 2023-10-09T09:11:40.584 INFO:tasks.rados.rados.0.smithi163.stdout:1505: expect (ObjNum 538 snap 168 seq_num 538) 2023-10-09T09:11:40.584 INFO:tasks.rados.rados.0.smithi163.stdout:1506: snap_create 2023-10-09T09:11:40.584 INFO:tasks.rados.rados.0.smithi163.stdout:1503: finishing write tid 3 to smithi16367185-50 2023-10-09T09:11:40.584 INFO:tasks.rados.rados.0.smithi163.stdout:1503: finishing write tid 4 to smithi16367185-50 2023-10-09T09:11:40.584 INFO:tasks.rados.rados.0.smithi163.stdout:1503: finishing write tid 5 to smithi16367185-50 2023-10-09T09:11:40.584 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 620 (ObjNum 584 snap 185 seq_num 584) dirty exists 2023-10-09T09:11:40.584 INFO:tasks.rados.rados.0.smithi163.stdout:1503: left oid 50 (ObjNum 584 snap 185 seq_num 584) 2023-10-09T09:11:40.596 INFO:tasks.rados.rados.0.smithi163.stdout:1505: expect (ObjNum 538 snap 168 seq_num 538) 2023-10-09T09:11:41.305 INFO:tasks.rados.rados.0.smithi163.stdout:1503: done (2 left) 2023-10-09T09:11:41.306 INFO:tasks.rados.rados.0.smithi163.stdout:1505: done (1 left) 2023-10-09T09:11:41.306 INFO:tasks.rados.rados.0.smithi163.stdout:1506: done (0 left) 2023-10-09T09:11:41.306 INFO:tasks.rados.rados.0.smithi163.stdout:1507: delete oid 15 current snap is 186 2023-10-09T09:11:41.316 INFO:tasks.rados.rados.0.smithi163.stdout:1507: done (0 left) 2023-10-09T09:11:41.316 INFO:tasks.rados.rados.0.smithi163.stdout:1508: read oid 24 snap -1 2023-10-09T09:11:41.316 INFO:tasks.rados.rados.0.smithi163.stdout:1508: expect (ObjNum 581 snap 185 seq_num 581) 2023-10-09T09:11:41.316 INFO:tasks.rados.rados.0.smithi163.stdout:1509: read oid 34 snap 167 2023-10-09T09:11:41.316 INFO:tasks.rados.rados.0.smithi163.stdout:1509: expect (ObjNum 521 snap 164 seq_num 521) 2023-10-09T09:11:41.317 INFO:tasks.rados.rados.0.smithi163.stdout:1510: snap_remove snap 182 2023-10-09T09:11:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:41 smithi181 bash[27712]: cluster 2023-10-09T09:11:40.000930+0000 osd.3 (osd.3) 82 : cluster [WRN] failed to encode map e433 with expected crc 2023-10-09T09:11:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:41 smithi181 bash[27712]: audit 2023-10-09T09:11:40.337481+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:41 smithi181 bash[27712]: debug 2023-10-09T09:11:41.289+0000 7fde83035700 -1 mon.b@1(peon).osd e434 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:42.317 INFO:tasks.rados.rados.0.smithi163.stdout:1510: done (2 left) 2023-10-09T09:11:42.317 INFO:tasks.rados.rados.0.smithi163.stdout:1511: write oid 29 current snap is 186 2023-10-09T09:11:42.317 INFO:tasks.rados.rados.0.smithi163.stdout:1511: seq_num 587 ranges {665709=730344,2133283=426203,3203532=509848} 2023-10-09T09:11:42.326 INFO:tasks.rados.rados.0.smithi163.stdout:1511: writing smithi16367185-29 from 665709 to 1396053 tid 1 2023-10-09T09:11:42.334 INFO:tasks.rados.rados.0.smithi163.stdout:1511: writing smithi16367185-29 from 2133283 to 2559486 tid 2 2023-10-09T09:11:42.342 INFO:tasks.rados.rados.0.smithi163.stdout:1511: writing smithi16367185-29 from 3203532 to 3713380 tid 3 2023-10-09T09:11:42.342 INFO:tasks.rados.rados.0.smithi163.stdout:1512: read oid 31 snap -1 2023-10-09T09:11:42.342 INFO:tasks.rados.rados.0.smithi163.stdout:1512: expect deleted 2023-10-09T09:11:42.342 INFO:tasks.rados.rados.0.smithi163.stdout:1513: snap_remove snap 185 2023-10-09T09:11:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:42 smithi181 bash[27712]: cluster 2023-10-09T09:11:41.256737+0000 mgr.y (mgr.34107) 92 : cluster [DBG] pgmap v200: 89 pgs: 89 active+clean; 206 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2023-10-09T09:11:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:42 smithi181 bash[27712]: cluster 2023-10-09T09:11:41.298137+0000 osd.3 (osd.3) 83 : cluster [WRN] failed to encode map e434 with expected crc 2023-10-09T09:11:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:42 smithi181 bash[27712]: cluster 2023-10-09T09:11:41.299887+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2023-10-09T09:11:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:42 smithi181 bash[27712]: cluster 2023-10-09T09:11:41.303953+0000 osd.0 (osd.0) 64 : cluster [WRN] failed to encode map e434 with expected crc 2023-10-09T09:11:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:42 smithi181 bash[27712]: cluster 2023-10-09T09:11:41.306176+0000 osd.7 (osd.7) 65 : cluster [WRN] failed to encode map e434 with expected crc 2023-10-09T09:11:42.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:42 smithi181 bash[27712]: debug 2023-10-09T09:11:42.301+0000 7fde83035700 -1 mon.b@1(peon).osd e435 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:43.323 INFO:tasks.rados.rados.0.smithi163.stdout:1513: done (4 left) 2023-10-09T09:11:43.324 INFO:tasks.rados.rados.0.smithi163.stdout:1514: write oid 4 current snap is 186 2023-10-09T09:11:43.324 INFO:tasks.rados.rados.0.smithi163.stdout:1514: seq_num 588 ranges {682516=405593,1791765=569861,2830955=721120} 2023-10-09T09:11:43.331 INFO:tasks.rados.rados.0.smithi163.stdout:1514: writing smithi16367185-4 from 682516 to 1088109 tid 1 2023-10-09T09:11:43.339 INFO:tasks.rados.rados.0.smithi163.stdout:1514: writing smithi16367185-4 from 1791765 to 2361626 tid 2 2023-10-09T09:11:43.347 INFO:tasks.rados.rados.0.smithi163.stdout:1514: writing smithi16367185-4 from 2830955 to 3552075 tid 3 2023-10-09T09:11:43.347 INFO:tasks.rados.rados.0.smithi163.stdout:1515: read oid 7 snap -1 2023-10-09T09:11:43.347 INFO:tasks.rados.rados.0.smithi163.stdout:1515: expect (ObjNum 575 snap 181 seq_num 575) 2023-10-09T09:11:43.347 INFO:tasks.rados.rados.0.smithi163.stdout:1516: snap_remove snap 181 2023-10-09T09:11:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:43 smithi181 bash[27712]: debug 2023-10-09T09:11:43.305+0000 7fde83035700 -1 mon.b@1(peon).osd e436 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:43 smithi181 bash[27712]: cluster 2023-10-09T09:11:41.519587+0000 osd.6 (osd.6) 41 : cluster [WRN] failed to encode map e434 with expected crc 2023-10-09T09:11:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:43 smithi181 bash[27712]: cluster 2023-10-09T09:11:42.310757+0000 osd.3 (osd.3) 84 : cluster [WRN] failed to encode map e435 with expected crc 2023-10-09T09:11:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:43 smithi181 bash[27712]: cluster 2023-10-09T09:11:42.312688+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2023-10-09T09:11:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:43 smithi181 bash[27712]: cluster 2023-10-09T09:11:42.315302+0000 osd.1 (osd.1) 42 : cluster [WRN] failed to encode map e435 with expected crc 2023-10-09T09:11:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:43 smithi181 bash[27712]: cluster 2023-10-09T09:11:42.366205+0000 osd.7 (osd.7) 66 : cluster [WRN] failed to encode map e435 with expected crc 2023-10-09T09:11:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:11:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:11:43] "GET /metrics HTTP/1.1" 200 36081 "" "Prometheus/2.33.4" 2023-10-09T09:11:44.343 INFO:tasks.rados.rados.0.smithi163.stdout:1516: done (6 left) 2023-10-09T09:11:44.343 INFO:tasks.rados.rados.0.smithi163.stdout:1517: rollback oid 42 current snap is 186 2023-10-09T09:11:44.343 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 176 2023-10-09T09:11:44.343 INFO:tasks.rados.rados.0.smithi163.stdout:1509: expect (ObjNum 521 snap 164 seq_num 521) 2023-10-09T09:11:44.440 INFO:tasks.rados.rados.0.smithi163.stdout:1508: expect (ObjNum 581 snap 185 seq_num 581) 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:1511: finishing write tid 1 to smithi16367185-29 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:1511: finishing write tid 2 to smithi16367185-29 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:1511: finishing write tid 3 to smithi16367185-29 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:1511: finishing write tid 4 to smithi16367185-29 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:1511: finishing write tid 5 to smithi16367185-29 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 703 (ObjNum 587 snap 186 seq_num 587) dirty exists 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:1511: left oid 29 (ObjNum 587 snap 186 seq_num 587) 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:1514: finishing write tid 1 to smithi16367185-4 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:1514: finishing write tid 2 to smithi16367185-4 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:1514: finishing write tid 3 to smithi16367185-4 2023-10-09T09:11:44.619 INFO:tasks.rados.rados.0.smithi163.stdout:1514: finishing write tid 4 to smithi16367185-4 2023-10-09T09:11:44.620 INFO:tasks.rados.rados.0.smithi163.stdout:1514: finishing write tid 5 to smithi16367185-4 2023-10-09T09:11:44.620 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 902 (ObjNum 588 snap 186 seq_num 588) dirty exists 2023-10-09T09:11:44.620 INFO:tasks.rados.rados.0.smithi163.stdout:1514: left oid 4 (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:11:44.620 INFO:tasks.rados.rados.0.smithi163.stdout:1515: expect (ObjNum 575 snap 181 seq_num 575) 2023-10-09T09:11:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:44 smithi181 bash[27712]: cluster 2023-10-09T09:11:42.329384+0000 osd.2 (osd.2) 94 : cluster [WRN] failed to encode map e435 with expected crc 2023-10-09T09:11:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:44 smithi181 bash[27712]: cluster 2023-10-09T09:11:43.257392+0000 mgr.y (mgr.34107) 93 : cluster [DBG] pgmap v203: 89 pgs: 89 active+clean; 206 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2023-10-09T09:11:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:44 smithi181 bash[27712]: cluster 2023-10-09T09:11:43.314116+0000 osd.3 (osd.3) 85 : cluster [WRN] failed to encode map e436 with expected crc 2023-10-09T09:11:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:44 smithi181 bash[27712]: cluster 2023-10-09T09:11:43.318251+0000 mon.a (mon.0) 273 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2023-10-09T09:11:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:44 smithi181 bash[27712]: cluster 2023-10-09T09:11:43.321769+0000 osd.0 (osd.0) 65 : cluster [WRN] failed to encode map e436 with expected crc 2023-10-09T09:11:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:44 smithi181 bash[27712]: cluster 2023-10-09T09:11:43.333492+0000 osd.2 (osd.2) 95 : cluster [WRN] failed to encode map e436 with expected crc 2023-10-09T09:11:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:44 smithi181 bash[27712]: cluster 2023-10-09T09:11:43.348571+0000 osd.1 (osd.1) 43 : cluster [WRN] failed to encode map e436 with expected crc 2023-10-09T09:11:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:44 smithi181 bash[27712]: cluster 2023-10-09T09:11:43.504209+0000 osd.6 (osd.6) 42 : cluster [WRN] failed to encode map e436 with expected crc 2023-10-09T09:11:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:44 smithi181 bash[27712]: debug 2023-10-09T09:11:44.329+0000 7fde83035700 -1 mon.b@1(peon).osd e437 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:44.869 INFO:tasks.rados.rados.0.smithi163.stdout:1517: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:11:44.869 INFO:tasks.rados.rados.0.smithi163.stdout:1517: finishing rollback tid 2 to smithi16367185-42 2023-10-09T09:11:44.869 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 1200 (ObjNum 513 snap 160 seq_num 513) dirty exists 2023-10-09T09:11:44.869 INFO:tasks.rados.rados.0.smithi163.stdout:1508: done (6 left) 2023-10-09T09:11:44.870 INFO:tasks.rados.rados.0.smithi163.stdout:1509: done (5 left) 2023-10-09T09:11:44.870 INFO:tasks.rados.rados.0.smithi163.stdout:1511: done (4 left) 2023-10-09T09:11:44.870 INFO:tasks.rados.rados.0.smithi163.stdout:1512: done (3 left) 2023-10-09T09:11:44.870 INFO:tasks.rados.rados.0.smithi163.stdout:1514: done (2 left) 2023-10-09T09:11:44.870 INFO:tasks.rados.rados.0.smithi163.stdout:1515: done (1 left) 2023-10-09T09:11:44.870 INFO:tasks.rados.rados.0.smithi163.stdout:1517: done (0 left) 2023-10-09T09:11:44.870 INFO:tasks.rados.rados.0.smithi163.stdout:1518: write oid 9 current snap is 186 2023-10-09T09:11:44.870 INFO:tasks.rados.rados.0.smithi163.stdout:1518: seq_num 589 ranges {699323=480842,1850247=429872,2858378=532392} 2023-10-09T09:11:44.881 INFO:tasks.rados.rados.0.smithi163.stdout:1518: writing smithi16367185-9 from 699323 to 1180165 tid 1 2023-10-09T09:11:44.888 INFO:tasks.rados.rados.0.smithi163.stdout:1518: writing smithi16367185-9 from 1850247 to 2280119 tid 2 2023-10-09T09:11:44.894 INFO:tasks.rados.rados.0.smithi163.stdout:1518: writing smithi16367185-9 from 2858378 to 3390770 tid 3 2023-10-09T09:11:44.894 INFO:tasks.rados.rados.0.smithi163.stdout:1519: write oid 37 current snap is 186 2023-10-09T09:11:44.894 INFO:tasks.rados.rados.0.smithi163.stdout:1519: seq_num 590 ranges {716130=556091,1792376=573530,2769448=460017,3821956=94174} 2023-10-09T09:11:44.901 INFO:tasks.rados.rados.0.smithi163.stdout:1519: writing smithi16367185-37 from 716130 to 1272221 tid 1 2023-10-09T09:11:44.907 INFO:tasks.rados.rados.0.smithi163.stdout:1519: writing smithi16367185-37 from 1792376 to 2365906 tid 2 2023-10-09T09:11:44.912 INFO:tasks.rados.rados.0.smithi163.stdout:1519: writing smithi16367185-37 from 2769448 to 3229465 tid 3 2023-10-09T09:11:44.916 INFO:tasks.rados.rados.0.smithi163.stdout:1519: writing smithi16367185-37 from 3821956 to 3916130 tid 4 2023-10-09T09:11:44.916 INFO:tasks.rados.rados.0.smithi163.stdout:1520: write oid 30 current snap is 186 2023-10-09T09:11:44.916 INFO:tasks.rados.rados.0.smithi163.stdout:1520: seq_num 591 ranges {732937=631340,1850858=433541,2796871=671289,3888195=44742} 2023-10-09T09:11:44.924 INFO:tasks.rados.rados.0.smithi163.stdout:1520: writing smithi16367185-30 from 732937 to 1364277 tid 1 2023-10-09T09:11:44.929 INFO:tasks.rados.rados.0.smithi163.stdout:1520: writing smithi16367185-30 from 1850858 to 2284399 tid 2 2023-10-09T09:11:44.936 INFO:tasks.rados.rados.0.smithi163.stdout:1520: writing smithi16367185-30 from 2796871 to 3468160 tid 3 2023-10-09T09:11:44.939 INFO:tasks.rados.rados.0.smithi163.stdout:1520: writing smithi16367185-30 from 3888195 to 3932937 tid 4 2023-10-09T09:11:44.939 INFO:tasks.rados.rados.0.smithi163.stdout:1521: snap_remove snap 167 2023-10-09T09:11:45.016 INFO:tasks.rados.rados.0.smithi163.stdout:1521: done (3 left) 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:1522: delete oid 10 current snap is 186 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:1518: finishing write tid 1 to smithi16367185-9 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:1518: finishing write tid 2 to smithi16367185-9 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:1518: finishing write tid 3 to smithi16367185-9 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:1518: finishing write tid 4 to smithi16367185-9 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:1518: finishing write tid 5 to smithi16367185-9 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 410 (ObjNum 589 snap 186 seq_num 589) dirty exists 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:1518: left oid 9 (ObjNum 589 snap 186 seq_num 589) 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:1519: finishing write tid 1 to smithi16367185-37 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:1519: finishing write tid 2 to smithi16367185-37 2023-10-09T09:11:45.017 INFO:tasks.rados.rados.0.smithi163.stdout:1519: finishing write tid 3 to smithi16367185-37 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:1519: finishing write tid 4 to smithi16367185-37 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:1519: finishing write tid 5 to smithi16367185-37 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:1519: finishing write tid 6 to smithi16367185-37 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 759 (ObjNum 590 snap 186 seq_num 590) dirty exists 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:1519: left oid 37 (ObjNum 590 snap 186 seq_num 590) 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:1520: finishing write tid 1 to smithi16367185-30 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:1520: finishing write tid 2 to smithi16367185-30 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:1520: finishing write tid 3 to smithi16367185-30 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:1520: finishing write tid 4 to smithi16367185-30 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:1520: finishing write tid 5 to smithi16367185-30 2023-10-09T09:11:45.018 INFO:tasks.rados.rados.0.smithi163.stdout:1520: finishing write tid 6 to smithi16367185-30 2023-10-09T09:11:45.019 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 598 (ObjNum 591 snap 186 seq_num 591) dirty exists 2023-10-09T09:11:45.019 INFO:tasks.rados.rados.0.smithi163.stdout:1520: left oid 30 (ObjNum 591 snap 186 seq_num 591) 2023-10-09T09:11:45.030 INFO:tasks.rados.rados.0.smithi163.stdout:1518: done (3 left) 2023-10-09T09:11:45.030 INFO:tasks.rados.rados.0.smithi163.stdout:1519: done (2 left) 2023-10-09T09:11:45.031 INFO:tasks.rados.rados.0.smithi163.stdout:1520: done (1 left) 2023-10-09T09:11:45.031 INFO:tasks.rados.rados.0.smithi163.stdout:1522: done (0 left) 2023-10-09T09:11:45.031 INFO:tasks.rados.rados.0.smithi163.stdout:1523: read oid 40 snap -1 2023-10-09T09:11:45.031 INFO:tasks.rados.rados.0.smithi163.stdout:1523: expect (ObjNum 537 snap 168 seq_num 537) 2023-10-09T09:11:45.031 INFO:tasks.rados.rados.0.smithi163.stdout:1524: write oid 14 current snap is 186 2023-10-09T09:11:45.031 INFO:tasks.rados.rados.0.smithi163.stdout:1524: seq_num 593 ranges {766551=498191,1684175=720857,2851717=693833} 2023-10-09T09:11:45.038 INFO:tasks.rados.rados.0.smithi163.stdout:1524: writing smithi16367185-14 from 766551 to 1264742 tid 1 2023-10-09T09:11:45.044 INFO:tasks.rados.rados.0.smithi163.stdout:1524: writing smithi16367185-14 from 1684175 to 2405032 tid 2 2023-10-09T09:11:45.051 INFO:tasks.rados.rados.0.smithi163.stdout:1524: writing smithi16367185-14 from 2851717 to 3545550 tid 3 2023-10-09T09:11:45.051 INFO:tasks.rados.rados.0.smithi163.stdout:1525: snap_create 2023-10-09T09:11:45.051 INFO:tasks.rados.rados.0.smithi163.stdout:1523: expect (ObjNum 537 snap 168 seq_num 537) 2023-10-09T09:11:45.149 INFO:tasks.rados.rados.0.smithi163.stdout:1524: finishing write tid 1 to smithi16367185-14 2023-10-09T09:11:45.149 INFO:tasks.rados.rados.0.smithi163.stdout:1524: finishing write tid 2 to smithi16367185-14 2023-10-09T09:11:45.149 INFO:tasks.rados.rados.0.smithi163.stdout:1524: finishing write tid 3 to smithi16367185-14 2023-10-09T09:11:45.149 INFO:tasks.rados.rados.0.smithi163.stdout:1524: finishing write tid 4 to smithi16367185-14 2023-10-09T09:11:45.149 INFO:tasks.rados.rados.0.smithi163.stdout:1524: finishing write tid 5 to smithi16367185-14 2023-10-09T09:11:45.150 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 603 (ObjNum 593 snap 186 seq_num 593) dirty exists 2023-10-09T09:11:45.150 INFO:tasks.rados.rados.0.smithi163.stdout:1524: left oid 14 (ObjNum 593 snap 186 seq_num 593) 2023-10-09T09:11:45.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:45 smithi181 bash[27712]: debug 2023-10-09T09:11:44.997+0000 7fde83035700 -1 mon.b@1(peon).osd e438 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:45 smithi181 bash[27712]: cluster 2023-10-09T09:11:43.388841+0000 osd.7 (osd.7) 67 : cluster [WRN] failed to encode map e436 with expected crc 2023-10-09T09:11:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:45 smithi181 bash[27712]: cluster 2023-10-09T09:11:44.335887+0000 osd.3 (osd.3) 86 : cluster [WRN] failed to encode map e437 with expected crc 2023-10-09T09:11:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:45 smithi181 bash[27712]: cluster 2023-10-09T09:11:44.337880+0000 mon.a (mon.0) 274 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2023-10-09T09:11:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:45 smithi181 bash[27712]: cluster 2023-10-09T09:11:44.339481+0000 osd.7 (osd.7) 68 : cluster [WRN] failed to encode map e437 with expected crc 2023-10-09T09:11:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:45 smithi181 bash[27712]: cluster 2023-10-09T09:11:44.344006+0000 osd.5 (osd.5) 51 : cluster [WRN] failed to encode map e437 with expected crc 2023-10-09T09:11:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:45 smithi181 bash[27712]: cluster 2023-10-09T09:11:45.006884+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2023-10-09T09:11:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:45 smithi181 bash[27712]: cluster 2023-10-09T09:11:45.021368+0000 osd.2 (osd.2) 96 : cluster [WRN] failed to encode map e438 with expected crc 2023-10-09T09:11:46.034 INFO:tasks.rados.rados.0.smithi163.stdout:1523: done (2 left) 2023-10-09T09:11:46.034 INFO:tasks.rados.rados.0.smithi163.stdout:1524: done (1 left) 2023-10-09T09:11:46.034 INFO:tasks.rados.rados.0.smithi163.stdout:1525: done (0 left) 2023-10-09T09:11:46.034 INFO:tasks.rados.rados.0.smithi163.stdout:1526: write oid 26 current snap is 187 2023-10-09T09:11:46.034 INFO:tasks.rados.rados.0.smithi163.stdout:1526: seq_num 594 ranges {783358=573440,2026304=580868,3279140=621458} 2023-10-09T09:11:46.042 INFO:tasks.rados.rados.0.smithi163.stdout:1526: writing smithi16367185-26 from 783358 to 1356798 tid 1 2023-10-09T09:11:46.052 INFO:tasks.rados.rados.0.smithi163.stdout:1526: writing smithi16367185-26 from 2026304 to 2607172 tid 2 2023-10-09T09:11:46.060 INFO:tasks.rados.rados.0.smithi163.stdout:1526: writing smithi16367185-26 from 3279140 to 3900598 tid 3 2023-10-09T09:11:46.060 INFO:tasks.rados.rados.0.smithi163.stdout:1527: snap_remove snap 156 2023-10-09T09:11:46.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:46 smithi181 bash[27712]: debug 2023-10-09T09:11:46.021+0000 7fde83035700 -1 mon.b@1(peon).osd e439 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:46 smithi181 bash[27712]: cluster 2023-10-09T09:11:45.003550+0000 osd.3 (osd.3) 87 : cluster [WRN] failed to encode map e438 with expected crc 2023-10-09T09:11:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:46 smithi181 bash[27712]: cluster 2023-10-09T09:11:45.011902+0000 osd.5 (osd.5) 52 : cluster [WRN] failed to encode map e438 with expected crc 2023-10-09T09:11:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:46 smithi181 bash[27712]: cluster 2023-10-09T09:11:45.258028+0000 mgr.y (mgr.34107) 94 : cluster [DBG] pgmap v207: 89 pgs: 89 active+clean; 206 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:11:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:46 smithi181 bash[27712]: cluster 2023-10-09T09:11:46.031146+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2023-10-09T09:11:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:46 smithi181 bash[27712]: cluster 2023-10-09T09:11:46.045640+0000 osd.2 (osd.2) 97 : cluster [WRN] failed to encode map e439 with expected crc 2023-10-09T09:11:47.048 INFO:tasks.rados.rados.0.smithi163.stdout:1527: done (1 left) 2023-10-09T09:11:47.048 INFO:tasks.rados.rados.0.smithi163.stdout:1528: delete oid 22 current snap is 187 2023-10-09T09:11:47.048 INFO:tasks.rados.rados.0.smithi163.stdout:1526: finishing write tid 1 to smithi16367185-26 2023-10-09T09:11:47.048 INFO:tasks.rados.rados.0.smithi163.stdout:1526: finishing write tid 2 to smithi16367185-26 2023-10-09T09:11:47.048 INFO:tasks.rados.rados.0.smithi163.stdout:1526: finishing write tid 3 to smithi16367185-26 2023-10-09T09:11:47.049 INFO:tasks.rados.rados.0.smithi163.stdout:1526: finishing write tid 4 to smithi16367185-26 2023-10-09T09:11:47.049 INFO:tasks.rados.rados.0.smithi163.stdout:1526: finishing write tid 5 to smithi16367185-26 2023-10-09T09:11:47.049 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 525 (ObjNum 594 snap 187 seq_num 594) dirty exists 2023-10-09T09:11:47.049 INFO:tasks.rados.rados.0.smithi163.stdout:1526: left oid 26 (ObjNum 594 snap 187 seq_num 594) 2023-10-09T09:11:47.056 INFO:tasks.rados.rados.0.smithi163.stdout:1526: done (1 left) 2023-10-09T09:11:47.056 INFO:tasks.rados.rados.0.smithi163.stdout:1528: done (0 left) 2023-10-09T09:11:47.057 INFO:tasks.rados.rados.0.smithi163.stdout:1529: snap_create 2023-10-09T09:11:47.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:47 smithi181 bash[27712]: debug 2023-10-09T09:11:47.025+0000 7fde83035700 -1 mon.b@1(peon).osd e440 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:47 smithi181 bash[27712]: cluster 2023-10-09T09:11:46.027630+0000 osd.3 (osd.3) 88 : cluster [WRN] failed to encode map e439 with expected crc 2023-10-09T09:11:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:47 smithi181 bash[27712]: cluster 2023-10-09T09:11:46.030662+0000 osd.5 (osd.5) 53 : cluster [WRN] failed to encode map e439 with expected crc 2023-10-09T09:11:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:47 smithi181 bash[27712]: cluster 2023-10-09T09:11:46.511194+0000 osd.0 (osd.0) 66 : cluster [WRN] failed to encode map e439 with expected crc 2023-10-09T09:11:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:47 smithi181 bash[27712]: cluster 2023-10-09T09:11:47.034376+0000 mon.a (mon.0) 277 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2023-10-09T09:11:48.044 INFO:tasks.rados.rados.0.smithi163.stdout:1529: done (0 left) 2023-10-09T09:11:48.045 INFO:tasks.rados.rados.0.smithi163.stdout:1530: snap_remove snap 183 2023-10-09T09:11:48.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:48 smithi181 bash[27712]: debug 2023-10-09T09:11:48.033+0000 7fde83035700 -1 mon.b@1(peon).osd e441 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:48.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:48 smithi181 bash[27712]: cluster 2023-10-09T09:11:47.030658+0000 osd.3 (osd.3) 89 : cluster [WRN] failed to encode map e440 with expected crc 2023-10-09T09:11:48.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:48 smithi181 bash[27712]: cluster 2023-10-09T09:11:47.040936+0000 osd.0 (osd.0) 67 : cluster [WRN] failed to encode map e440 with expected crc 2023-10-09T09:11:48.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:48 smithi181 bash[27712]: cluster 2023-10-09T09:11:47.258672+0000 mgr.y (mgr.34107) 95 : cluster [DBG] pgmap v210: 89 pgs: 89 active+clean; 214 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2023-10-09T09:11:48.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:48 smithi181 bash[27712]: cluster 2023-10-09T09:11:48.040240+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2023-10-09T09:11:48.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:48 smithi181 bash[27712]: cluster 2023-10-09T09:11:48.043644+0000 osd.2 (osd.2) 98 : cluster [WRN] failed to encode map e441 with expected crc 2023-10-09T09:11:49.041 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:11:48 smithi181 bash[54907]: ts=2023-10-09T09:11:48.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:11:49.051 INFO:tasks.rados.rados.0.smithi163.stdout:1530: done (0 left) 2023-10-09T09:11:49.051 INFO:tasks.rados.rados.0.smithi163.stdout:1531: read oid 45 snap -1 2023-10-09T09:11:49.051 INFO:tasks.rados.rados.0.smithi163.stdout:1531: expect (ObjNum 564 snap 172 seq_num 564) 2023-10-09T09:11:49.051 INFO:tasks.rados.rados.0.smithi163.stdout:1532: snap_remove snap 184 2023-10-09T09:11:49.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:49 smithi181 bash[27712]: debug 2023-10-09T09:11:49.037+0000 7fde83035700 -1 mon.b@1(peon).osd e442 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:49 smithi181 bash[27712]: cluster 2023-10-09T09:11:48.038433+0000 osd.3 (osd.3) 90 : cluster [WRN] failed to encode map e441 with expected crc 2023-10-09T09:11:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:49 smithi181 bash[27712]: cluster 2023-10-09T09:11:48.349255+0000 osd.7 (osd.7) 69 : cluster [WRN] failed to encode map e441 with expected crc 2023-10-09T09:11:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:49 smithi181 bash[27712]: cluster 2023-10-09T09:11:48.527333+0000 osd.0 (osd.0) 68 : cluster [WRN] failed to encode map e441 with expected crc 2023-10-09T09:11:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:49 smithi181 bash[27712]: cluster 2023-10-09T09:11:48.793554+0000 osd.4 (osd.4) 92 : cluster [WRN] failed to encode map e441 with expected crc 2023-10-09T09:11:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:49 smithi181 bash[27712]: cluster 2023-10-09T09:11:49.044708+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2023-10-09T09:11:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:49 smithi181 bash[27712]: cluster 2023-10-09T09:11:49.048204+0000 osd.2 (osd.2) 99 : cluster [WRN] failed to encode map e442 with expected crc 2023-10-09T09:11:50.017 INFO:tasks.rados.rados.0.smithi163.stdout:1532: done (1 left) 2023-10-09T09:11:50.018 INFO:tasks.rados.rados.0.smithi163.stdout:1533: read oid 38 snap -1 2023-10-09T09:11:50.018 INFO:tasks.rados.rados.0.smithi163.stdout:1533: expect (ObjNum 503 snap 157 seq_num 503) 2023-10-09T09:11:50.018 INFO:tasks.rados.rados.0.smithi163.stdout:1534: snap_remove snap 175 2023-10-09T09:11:50.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:50 smithi181 bash[27712]: debug 2023-10-09T09:11:50.001+0000 7fde83035700 -1 mon.b@1(peon).osd e443 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:50 smithi181 bash[27712]: cluster 2023-10-09T09:11:48.523384+0000 osd.6 (osd.6) 43 : cluster [WRN] failed to encode map e441 with expected crc 2023-10-09T09:11:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:50 smithi181 bash[27712]: cluster 2023-10-09T09:11:49.041329+0000 osd.3 (osd.3) 91 : cluster [WRN] failed to encode map e442 with expected crc 2023-10-09T09:11:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:50 smithi181 bash[27712]: cluster 2023-10-09T09:11:49.259375+0000 mgr.y (mgr.34107) 96 : cluster [DBG] pgmap v213: 89 pgs: 89 active+clean; 214 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2023-10-09T09:11:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:50 smithi181 bash[27712]: cluster 2023-10-09T09:11:50.009743+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2023-10-09T09:11:51.038 INFO:tasks.rados.rados.0.smithi163.stdout:1531: expect (ObjNum 564 snap 172 seq_num 564) 2023-10-09T09:11:51.154 INFO:tasks.rados.rados.0.smithi163.stdout:1533: expect (ObjNum 503 snap 157 seq_num 503) 2023-10-09T09:11:51.233 INFO:tasks.rados.rados.0.smithi163.stdout:1531: done (2 left) 2023-10-09T09:11:51.233 INFO:tasks.rados.rados.0.smithi163.stdout:1533: done (1 left) 2023-10-09T09:11:51.233 INFO:tasks.rados.rados.0.smithi163.stdout:1534: done (0 left) 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1535: read oid 26 snap -1 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1535: expect (ObjNum 594 snap 187 seq_num 594) 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1536: read oid 27 snap 187 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1536: expect (ObjNum 557 snap 172 seq_num 557) 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1537: read oid 36 snap 187 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1537: expect deleted 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1538: read oid 18 snap 176 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1538: expect deleted 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1539: read oid 21 snap -1 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1539: expect (ObjNum 576 snap 181 seq_num 576) 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1540: read oid 14 snap 187 2023-10-09T09:11:51.234 INFO:tasks.rados.rados.0.smithi163.stdout:1540: expect (ObjNum 593 snap 186 seq_num 593) 2023-10-09T09:11:51.235 INFO:tasks.rados.rados.0.smithi163.stdout:1541: read oid 22 snap -1 2023-10-09T09:11:51.235 INFO:tasks.rados.rados.0.smithi163.stdout:1541: expect deleted 2023-10-09T09:11:51.235 INFO:tasks.rados.rados.0.smithi163.stdout:1542: delete oid 30 current snap is 188 2023-10-09T09:11:51.239 INFO:tasks.rados.rados.0.smithi163.stdout:1537: done (7 left) 2023-10-09T09:11:51.239 INFO:tasks.rados.rados.0.smithi163.stdout:1538: done (6 left) 2023-10-09T09:11:51.239 INFO:tasks.rados.rados.0.smithi163.stdout:1541: done (5 left) 2023-10-09T09:11:51.240 INFO:tasks.rados.rados.0.smithi163.stdout:1542: done (4 left) 2023-10-09T09:11:51.240 INFO:tasks.rados.rados.0.smithi163.stdout:1543: read oid 41 snap -1 2023-10-09T09:11:51.240 INFO:tasks.rados.rados.0.smithi163.stdout:1543: expect (ObjNum 541 snap 169 seq_num 541) 2023-10-09T09:11:51.240 INFO:tasks.rados.rados.0.smithi163.stdout:1544: write oid 6 current snap is 188 2023-10-09T09:11:51.240 INFO:tasks.rados.rados.0.smithi163.stdout:1544: seq_num 597 ranges {433779=799187,1801750=232029} 2023-10-09T09:11:51.248 INFO:tasks.rados.rados.0.smithi163.stdout:1544: writing smithi16367185-6 from 433779 to 1232966 tid 1 2023-10-09T09:11:51.252 INFO:tasks.rados.rados.0.smithi163.stdout:1544: writing smithi16367185-6 from 1801750 to 2033779 tid 2 2023-10-09T09:11:51.253 INFO:tasks.rados.rados.0.smithi163.stdout:1545: read oid 10 snap -1 2023-10-09T09:11:51.253 INFO:tasks.rados.rados.0.smithi163.stdout:1545: expect deleted 2023-10-09T09:11:51.253 INFO:tasks.rados.rados.0.smithi163.stdout:1546: write oid 25 current snap is 188 2023-10-09T09:11:51.253 INFO:tasks.rados.rados.0.smithi163.stdout:1546: seq_num 598 ranges {450586=474436,1343879=588206} 2023-10-09T09:11:51.258 INFO:tasks.rados.rados.0.smithi163.stdout:1546: writing smithi16367185-25 from 450586 to 925022 tid 1 2023-10-09T09:11:51.264 INFO:tasks.rados.rados.0.smithi163.stdout:1546: writing smithi16367185-25 from 1343879 to 1932085 tid 2 2023-10-09T09:11:51.264 INFO:tasks.rados.rados.0.smithi163.stdout:1547: delete oid 44 current snap is 188 2023-10-09T09:11:51.264 INFO:tasks.rados.rados.0.smithi163.stdout:1539: expect (ObjNum 576 snap 181 seq_num 576) 2023-10-09T09:11:51.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:51 smithi181 bash[27712]: debug 2023-10-09T09:11:51.017+0000 7fde83035700 -1 mon.b@1(peon).osd e444 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:51.466 INFO:tasks.rados.rados.0.smithi163.stdout:1535: expect (ObjNum 594 snap 187 seq_num 594) 2023-10-09T09:11:51.774 INFO:tasks.rados.rados.0.smithi163.stdout:1543: expect (ObjNum 541 snap 169 seq_num 541) 2023-10-09T09:11:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:51 smithi181 bash[27712]: cluster 2023-10-09T09:11:50.005679+0000 osd.3 (osd.3) 92 : cluster [WRN] failed to encode map e443 with expected crc 2023-10-09T09:11:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:51 smithi181 bash[27712]: cluster 2023-10-09T09:11:50.011897+0000 osd.7 (osd.7) 70 : cluster [WRN] failed to encode map e443 with expected crc 2023-10-09T09:11:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:51 smithi181 bash[27712]: cluster 2023-10-09T09:11:51.026641+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2023-10-09T09:11:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:51 smithi181 bash[27712]: cluster 2023-10-09T09:11:51.029367+0000 osd.2 (osd.2) 100 : cluster [WRN] failed to encode map e444 with expected crc 2023-10-09T09:11:51.907 INFO:tasks.rados.rados.0.smithi163.stdout:1536: expect (ObjNum 557 snap 172 seq_num 557) 2023-10-09T09:11:52.013 INFO:tasks.rados.rados.0.smithi163.stdout:1540: expect (ObjNum 593 snap 186 seq_num 593) 2023-10-09T09:11:52.194 INFO:tasks.rados.rados.0.smithi163.stdout:1544: finishing write tid 1 to smithi16367185-6 2023-10-09T09:11:52.195 INFO:tasks.rados.rados.0.smithi163.stdout:1544: finishing write tid 2 to smithi16367185-6 2023-10-09T09:11:52.195 INFO:tasks.rados.rados.0.smithi163.stdout:1544: finishing write tid 3 to smithi16367185-6 2023-10-09T09:11:52.195 INFO:tasks.rados.rados.0.smithi163.stdout:1546: finishing write tid 1 to smithi16367185-25 2023-10-09T09:11:52.195 INFO:tasks.rados.rados.0.smithi163.stdout:1544: finishing write tid 4 to smithi16367185-6 2023-10-09T09:11:52.195 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 161 (ObjNum 597 snap 188 seq_num 597) dirty exists 2023-10-09T09:11:52.195 INFO:tasks.rados.rados.0.smithi163.stdout:1544: left oid 6 (ObjNum 597 snap 188 seq_num 597) 2023-10-09T09:11:52.195 INFO:tasks.rados.rados.0.smithi163.stdout:1546: finishing write tid 2 to smithi16367185-25 2023-10-09T09:11:52.196 INFO:tasks.rados.rados.0.smithi163.stdout:1546: finishing write tid 3 to smithi16367185-25 2023-10-09T09:11:52.196 INFO:tasks.rados.rados.0.smithi163.stdout:1546: finishing write tid 4 to smithi16367185-25 2023-10-09T09:11:52.196 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 721 (ObjNum 598 snap 188 seq_num 598) dirty exists 2023-10-09T09:11:52.196 INFO:tasks.rados.rados.0.smithi163.stdout:1546: left oid 25 (ObjNum 598 snap 188 seq_num 598) 2023-10-09T09:11:52.196 INFO:tasks.rados.rados.0.smithi163.stdout:1535: done (8 left) 2023-10-09T09:11:52.197 INFO:tasks.rados.rados.0.smithi163.stdout:1536: done (7 left) 2023-10-09T09:11:52.197 INFO:tasks.rados.rados.0.smithi163.stdout:1539: done (6 left) 2023-10-09T09:11:52.198 INFO:tasks.rados.rados.0.smithi163.stdout:1540: done (5 left) 2023-10-09T09:11:52.198 INFO:tasks.rados.rados.0.smithi163.stdout:1543: done (4 left) 2023-10-09T09:11:52.198 INFO:tasks.rados.rados.0.smithi163.stdout:1544: done (3 left) 2023-10-09T09:11:52.198 INFO:tasks.rados.rados.0.smithi163.stdout:1545: done (2 left) 2023-10-09T09:11:52.198 INFO:tasks.rados.rados.0.smithi163.stdout:1546: done (1 left) 2023-10-09T09:11:52.198 INFO:tasks.rados.rados.0.smithi163.stdout:1547: done (0 left) 2023-10-09T09:11:52.198 INFO:tasks.rados.rados.0.smithi163.stdout:1548: read oid 34 snap -1 2023-10-09T09:11:52.198 INFO:tasks.rados.rados.0.smithi163.stdout:1548: expect (ObjNum 554 snap 171 seq_num 554) 2023-10-09T09:11:52.198 INFO:tasks.rados.rados.0.smithi163.stdout:1549: read oid 20 snap -1 2023-10-09T09:11:52.199 INFO:tasks.rados.rados.0.smithi163.stdout:1549: expect deleted 2023-10-09T09:11:52.199 INFO:tasks.rados.rados.0.smithi163.stdout:1550: snap_remove snap 186 2023-10-09T09:11:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:52 smithi181 bash[27712]: cluster 2023-10-09T09:11:51.021922+0000 osd.3 (osd.3) 93 : cluster [WRN] failed to encode map e444 with expected crc 2023-10-09T09:11:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:52 smithi181 bash[27712]: cluster 2023-10-09T09:11:51.260592+0000 mgr.y (mgr.34107) 97 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:11:53.038 INFO:tasks.rados.rados.0.smithi163.stdout:1550: done (2 left) 2023-10-09T09:11:53.038 INFO:tasks.rados.rados.0.smithi163.stdout:1551: delete oid 11 current snap is 188 2023-10-09T09:11:53.038 INFO:tasks.rados.rados.0.smithi163.stdout:1548: expect (ObjNum 554 snap 171 seq_num 554) 2023-10-09T09:11:53.188 INFO:tasks.rados.rados.0.smithi163.stdout:1548: done (2 left) 2023-10-09T09:11:53.188 INFO:tasks.rados.rados.0.smithi163.stdout:1549: done (1 left) 2023-10-09T09:11:53.188 INFO:tasks.rados.rados.0.smithi163.stdout:1551: done (0 left) 2023-10-09T09:11:53.188 INFO:tasks.rados.rados.0.smithi163.stdout:1552: write oid 29 current snap is 188 2023-10-09T09:11:53.188 INFO:tasks.rados.rados.0.smithi163.stdout:1552: seq_num 601 ranges {501007=416536,1635678=465329} 2023-10-09T09:11:53.196 INFO:tasks.rados.rados.0.smithi163.stdout:1552: writing smithi16367185-29 from 501007 to 917543 tid 1 2023-10-09T09:11:53.203 INFO:tasks.rados.rados.0.smithi163.stdout:1552: writing smithi16367185-29 from 1635678 to 2101007 tid 2 2023-10-09T09:11:53.204 INFO:tasks.rados.rados.0.smithi163.stdout:1553: snap_remove snap 187 2023-10-09T09:11:53.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:53 smithi181 bash[27712]: debug 2023-10-09T09:11:53.021+0000 7fde83035700 -1 mon.b@1(peon).osd e445 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:11:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:11:53] "GET /metrics HTTP/1.1" 200 36081 "" "Prometheus/2.33.4" 2023-10-09T09:11:54.043 INFO:tasks.rados.rados.0.smithi163.stdout:1553: done (1 left) 2023-10-09T09:11:54.043 INFO:tasks.rados.rados.0.smithi163.stdout:1554: read oid 46 snap -1 2023-10-09T09:11:54.043 INFO:tasks.rados.rados.0.smithi163.stdout:1554: expect (ObjNum 538 snap 168 seq_num 538) 2023-10-09T09:11:54.043 INFO:tasks.rados.rados.0.smithi163.stdout:1555: rollback oid 22 current snap is 188 2023-10-09T09:11:54.043 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 22 to 172 2023-10-09T09:11:54.043 INFO:tasks.rados.rados.0.smithi163.stdout:1552: finishing write tid 1 to smithi16367185-29 2023-10-09T09:11:54.043 INFO:tasks.rados.rados.0.smithi163.stdout:1552: finishing write tid 2 to smithi16367185-29 2023-10-09T09:11:54.044 INFO:tasks.rados.rados.0.smithi163.stdout:1552: finishing write tid 3 to smithi16367185-29 2023-10-09T09:11:54.044 INFO:tasks.rados.rados.0.smithi163.stdout:1552: finishing write tid 4 to smithi16367185-29 2023-10-09T09:11:54.044 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 727 (ObjNum 601 snap 188 seq_num 601) dirty exists 2023-10-09T09:11:54.044 INFO:tasks.rados.rados.0.smithi163.stdout:1552: left oid 29 (ObjNum 601 snap 188 seq_num 601) 2023-10-09T09:11:54.044 INFO:tasks.rados.rados.0.smithi163.stdout:1552: done (2 left) 2023-10-09T09:11:54.044 INFO:tasks.rados.rados.0.smithi163.stdout:1556: read oid 29 snap -1 2023-10-09T09:11:54.044 INFO:tasks.rados.rados.0.smithi163.stdout:1556: expect (ObjNum 601 snap 188 seq_num 601) 2023-10-09T09:11:54.044 INFO:tasks.rados.rados.0.smithi163.stdout:1557: read oid 12 snap -1 2023-10-09T09:11:54.044 INFO:tasks.rados.rados.0.smithi163.stdout:1557: expect deleted 2023-10-09T09:11:54.044 INFO:tasks.rados.rados.0.smithi163.stdout:1558: snap_create 2023-10-09T09:11:54.052 INFO:tasks.rados.rados.0.smithi163.stdout:1555: finishing rollback tid 1 to smithi16367185-22 2023-10-09T09:11:54.059 INFO:tasks.rados.rados.0.smithi163.stdout:1554: expect (ObjNum 538 snap 168 seq_num 538) 2023-10-09T09:11:54.251 INFO:tasks.rados.rados.0.smithi163.stdout:1555: finishing rollback tid 2 to smithi16367185-22 2023-10-09T09:11:54.251 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 1244 (ObjNum 451 snap 145 seq_num 451) dirty exists 2023-10-09T09:11:54.251 INFO:tasks.rados.rados.0.smithi163.stdout:1556: expect (ObjNum 601 snap 188 seq_num 601) 2023-10-09T09:11:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:54 smithi181 bash[27712]: debug 2023-10-09T09:11:54.029+0000 7fde83035700 -1 mon.b@1(peon).osd e446 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:54 smithi181 bash[27712]: cluster 2023-10-09T09:11:53.026342+0000 osd.3 (osd.3) 94 : cluster [WRN] failed to encode map e445 with expected crc 2023-10-09T09:11:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:54 smithi181 bash[27712]: cluster 2023-10-09T09:11:53.029927+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2023-10-09T09:11:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:54 smithi181 bash[27712]: cluster 2023-10-09T09:11:53.036200+0000 osd.0 (osd.0) 69 : cluster [WRN] failed to encode map e445 with expected crc 2023-10-09T09:11:54.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:54 smithi181 bash[27712]: cluster 2023-10-09T09:11:53.038474+0000 osd.7 (osd.7) 71 : cluster [WRN] failed to encode map e445 with expected crc 2023-10-09T09:11:54.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:54 smithi181 bash[27712]: cluster 2023-10-09T09:11:53.158764+0000 osd.5 (osd.5) 54 : cluster [WRN] failed to encode map e445 with expected crc 2023-10-09T09:11:54.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:54 smithi181 bash[27712]: cluster 2023-10-09T09:11:53.261280+0000 mgr.y (mgr.34107) 98 : cluster [DBG] pgmap v218: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:11:55.025 INFO:tasks.rados.rados.0.smithi163.stdout:1554: done (4 left) 2023-10-09T09:11:55.025 INFO:tasks.rados.rados.0.smithi163.stdout:1555: done (3 left) 2023-10-09T09:11:55.025 INFO:tasks.rados.rados.0.smithi163.stdout:1556: done (2 left) 2023-10-09T09:11:55.025 INFO:tasks.rados.rados.0.smithi163.stdout:1557: done (1 left) 2023-10-09T09:11:55.026 INFO:tasks.rados.rados.0.smithi163.stdout:1558: done (0 left) 2023-10-09T09:11:55.026 INFO:tasks.rados.rados.0.smithi163.stdout:1559: snap_create 2023-10-09T09:11:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:55 smithi181 bash[27712]: debug 2023-10-09T09:11:55.009+0000 7fde83035700 -1 mon.b@1(peon).osd e447 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:55.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:55 smithi181 bash[27712]: cluster 2023-10-09T09:11:54.034429+0000 osd.3 (osd.3) 95 : cluster [WRN] failed to encode map e446 with expected crc 2023-10-09T09:11:55.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:55 smithi181 bash[27712]: cluster 2023-10-09T09:11:54.038941+0000 osd.1 (osd.1) 44 : cluster [WRN] failed to encode map e446 with expected crc 2023-10-09T09:11:55.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:55 smithi181 bash[27712]: cluster 2023-10-09T09:11:54.042015+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2023-10-09T09:11:55.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:55 smithi181 bash[27712]: cluster 2023-10-09T09:11:54.042583+0000 osd.4 (osd.4) 93 : cluster [WRN] failed to encode map e446 with expected crc 2023-10-09T09:11:55.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:55 smithi181 bash[27712]: cluster 2023-10-09T09:11:54.042674+0000 osd.5 (osd.5) 55 : cluster [WRN] failed to encode map e446 with expected crc 2023-10-09T09:11:55.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:55 smithi181 bash[27712]: cluster 2023-10-09T09:11:54.049302+0000 osd.2 (osd.2) 101 : cluster [WRN] failed to encode map e446 with expected crc 2023-10-09T09:11:55.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:55 smithi181 bash[27712]: cluster 2023-10-09T09:11:54.330570+0000 osd.7 (osd.7) 72 : cluster [WRN] failed to encode map e446 with expected crc 2023-10-09T09:11:55.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:55 smithi181 bash[27712]: cluster 2023-10-09T09:11:55.018486+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2023-10-09T09:11:56.040 INFO:tasks.rados.rados.0.smithi163.stdout:1559: done (0 left) 2023-10-09T09:11:56.040 INFO:tasks.rados.rados.0.smithi163.stdout:1560: rollback oid 6 current snap is 190 2023-10-09T09:11:56.040 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 6 to 172 2023-10-09T09:11:56.041 INFO:tasks.rados.rados.0.smithi163.stdout:1561: write oid 32 current snap is 190 2023-10-09T09:11:56.041 INFO:tasks.rados.rados.0.smithi163.stdout:1561: seq_num 602 ranges {517814=491785,1577807=540007} 2023-10-09T09:11:56.047 INFO:tasks.rados.rados.0.smithi163.stdout:1561: writing smithi16367185-32 from 517814 to 1009599 tid 1 2023-10-09T09:11:56.055 INFO:tasks.rados.rados.0.smithi163.stdout:1561: writing smithi16367185-32 from 1577807 to 2117814 tid 2 2023-10-09T09:11:56.055 INFO:tasks.rados.rados.0.smithi163.stdout:1562: rollback oid 46 current snap is 190 2023-10-09T09:11:56.055 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 46 to 176 2023-10-09T09:11:56.055 INFO:tasks.rados.rados.0.smithi163.stdout:1563: read oid 14 snap 189 2023-10-09T09:11:56.055 INFO:tasks.rados.rados.0.smithi163.stdout:1563: expect (ObjNum 593 snap 186 seq_num 593) 2023-10-09T09:11:56.055 INFO:tasks.rados.rados.0.smithi163.stdout:1564: snap_create 2023-10-09T09:11:56.056 INFO:tasks.rados.rados.0.smithi163.stdout:1560: finishing rollback tid 0 to smithi16367185-6 2023-10-09T09:11:56.059 INFO:tasks.rados.rados.0.smithi163.stdout:1560: finishing rollback tid 1 to smithi16367185-6 2023-10-09T09:11:56.059 INFO:tasks.rados.rados.0.smithi163.stdout:1560: finishing rollback tid 2 to smithi16367185-6 2023-10-09T09:11:56.059 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 165 (ObjNum 431 snap 134 seq_num 431) dirty exists 2023-10-09T09:11:56.065 INFO:tasks.rados.rados.0.smithi163.stdout:1561: finishing write tid 1 to smithi16367185-32 2023-10-09T09:11:56.065 INFO:tasks.rados.rados.0.smithi163.stdout:1561: finishing write tid 2 to smithi16367185-32 2023-10-09T09:11:56.065 INFO:tasks.rados.rados.0.smithi163.stdout:1561: finishing write tid 3 to smithi16367185-32 2023-10-09T09:11:56.066 INFO:tasks.rados.rados.0.smithi163.stdout:1561: finishing write tid 4 to smithi16367185-32 2023-10-09T09:11:56.066 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1250 (ObjNum 602 snap 190 seq_num 602) dirty exists 2023-10-09T09:11:56.066 INFO:tasks.rados.rados.0.smithi163.stdout:1561: left oid 32 (ObjNum 602 snap 190 seq_num 602) 2023-10-09T09:11:56.067 INFO:tasks.rados.rados.0.smithi163.stdout:1562: finishing rollback tid 0 to smithi16367185-46 2023-10-09T09:11:56.068 INFO:tasks.rados.rados.0.smithi163.stdout:1562: finishing rollback tid 1 to smithi16367185-46 2023-10-09T09:11:56.068 INFO:tasks.rados.rados.0.smithi163.stdout:1562: finishing rollback tid 2 to smithi16367185-46 2023-10-09T09:11:56.068 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 617 (ObjNum 538 snap 168 seq_num 538) dirty exists 2023-10-09T09:11:56.071 INFO:tasks.rados.rados.0.smithi163.stdout:1563: expect (ObjNum 593 snap 186 seq_num 593) 2023-10-09T09:11:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: debug 2023-10-09T09:11:56.025+0000 7fde83035700 -1 mon.b@1(peon).osd e448 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: cluster 2023-10-09T09:11:54.559899+0000 osd.6 (osd.6) 44 : cluster [WRN] failed to encode map e446 with expected crc 2023-10-09T09:11:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: cluster 2023-10-09T09:11:55.016666+0000 osd.3 (osd.3) 96 : cluster [WRN] failed to encode map e447 with expected crc 2023-10-09T09:11:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: cluster 2023-10-09T09:11:55.021471+0000 osd.4 (osd.4) 94 : cluster [WRN] failed to encode map e447 with expected crc 2023-10-09T09:11:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: cluster 2023-10-09T09:11:55.160853+0000 osd.5 (osd.5) 56 : cluster [WRN] failed to encode map e447 with expected crc 2023-10-09T09:11:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: cluster 2023-10-09T09:11:55.261948+0000 mgr.y (mgr.34107) 99 : cluster [DBG] pgmap v221: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:11:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: audit 2023-10-09T09:11:55.338785+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:11:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: cluster 2023-10-09T09:11:55.346139+0000 osd.7 (osd.7) 73 : cluster [WRN] failed to encode map e447 with expected crc 2023-10-09T09:11:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: cluster 2023-10-09T09:11:55.509785+0000 osd.0 (osd.0) 70 : cluster [WRN] failed to encode map e447 with expected crc 2023-10-09T09:11:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: cluster 2023-10-09T09:11:55.873472+0000 osd.1 (osd.1) 45 : cluster [WRN] failed to encode map e447 with expected crc 2023-10-09T09:11:56.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:56 smithi181 bash[27712]: cluster 2023-10-09T09:11:56.035177+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2023-10-09T09:11:57.059 INFO:tasks.rados.rados.0.smithi163.stdout:1560: done (4 left) 2023-10-09T09:11:57.059 INFO:tasks.rados.rados.0.smithi163.stdout:1561: done (3 left) 2023-10-09T09:11:57.059 INFO:tasks.rados.rados.0.smithi163.stdout:1562: done (2 left) 2023-10-09T09:11:57.059 INFO:tasks.rados.rados.0.smithi163.stdout:1563: done (1 left) 2023-10-09T09:11:57.059 INFO:tasks.rados.rados.0.smithi163.stdout:1564: done (0 left) 2023-10-09T09:11:57.059 INFO:tasks.rados.rados.0.smithi163.stdout:1565: read oid 17 snap 172 2023-10-09T09:11:57.059 INFO:tasks.rados.rados.0.smithi163.stdout:1565: expect (ObjNum 504 snap 157 seq_num 504) 2023-10-09T09:11:57.059 INFO:tasks.rados.rados.0.smithi163.stdout:1566: read oid 36 snap -1 2023-10-09T09:11:57.059 INFO:tasks.rados.rados.0.smithi163.stdout:1566: expect deleted 2023-10-09T09:11:57.059 INFO:tasks.rados.rados.0.smithi163.stdout:1567: snap_remove snap 189 2023-10-09T09:11:57.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:57 smithi181 bash[27712]: debug 2023-10-09T09:11:57.045+0000 7fde83035700 -1 mon.b@1(peon).osd e449 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:57.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:57 smithi181 bash[27712]: cluster 2023-10-09T09:11:56.029947+0000 osd.3 (osd.3) 97 : cluster [WRN] failed to encode map e448 with expected crc 2023-10-09T09:11:57.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:57 smithi181 bash[27712]: cluster 2023-10-09T09:11:56.036406+0000 osd.6 (osd.6) 45 : cluster [WRN] failed to encode map e448 with expected crc 2023-10-09T09:11:57.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:57 smithi181 bash[27712]: cluster 2023-10-09T09:11:56.042069+0000 osd.4 (osd.4) 95 : cluster [WRN] failed to encode map e448 with expected crc 2023-10-09T09:11:57.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:57 smithi181 bash[27712]: cluster 2023-10-09T09:11:56.049663+0000 osd.5 (osd.5) 57 : cluster [WRN] failed to encode map e448 with expected crc 2023-10-09T09:11:57.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:57 smithi181 bash[27712]: cluster 2023-10-09T09:11:56.263223+0000 osd.2 (osd.2) 102 : cluster [WRN] failed to encode map e447 with expected crc 2023-10-09T09:11:58.080 INFO:tasks.rados.rados.0.smithi163.stdout:1567: done (2 left) 2023-10-09T09:11:58.081 INFO:tasks.rados.rados.0.smithi163.stdout:1568: snap_remove snap 176 2023-10-09T09:11:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:58 smithi181 bash[27712]: debug 2023-10-09T09:11:58.065+0000 7fde83035700 -1 mon.b@1(peon).osd e450 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:11:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:58 smithi181 bash[27712]: cluster 2023-10-09T09:11:57.049463+0000 osd.3 (osd.3) 98 : cluster [WRN] failed to encode map e449 with expected crc 2023-10-09T09:11:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:58 smithi181 bash[27712]: cluster 2023-10-09T09:11:57.053845+0000 mon.a (mon.0) 287 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2023-10-09T09:11:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:58 smithi181 bash[27712]: cluster 2023-10-09T09:11:57.054307+0000 osd.4 (osd.4) 96 : cluster [WRN] failed to encode map e449 with expected crc 2023-10-09T09:11:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:58 smithi181 bash[27712]: cluster 2023-10-09T09:11:57.061554+0000 osd.7 (osd.7) 74 : cluster [WRN] failed to encode map e449 with expected crc 2023-10-09T09:11:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:58 smithi181 bash[27712]: cluster 2023-10-09T09:11:57.063326+0000 osd.0 (osd.0) 71 : cluster [WRN] failed to encode map e449 with expected crc 2023-10-09T09:11:58.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:58 smithi181 bash[27712]: cluster 2023-10-09T09:11:57.262699+0000 mgr.y (mgr.34107) 100 : cluster [DBG] pgmap v224: 89 pgs: 89 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 979 KiB/s wr, 27 op/s 2023-10-09T09:11:59.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:11:58 smithi181 bash[54907]: ts=2023-10-09T09:11:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:11:59.088 INFO:tasks.rados.rados.0.smithi163.stdout:1568: done (2 left) 2023-10-09T09:11:59.088 INFO:tasks.rados.rados.0.smithi163.stdout:1569: write oid 7 current snap is 191 2023-10-09T09:11:59.088 INFO:tasks.rados.rados.0.smithi163.stdout:1569: seq_num 603 ranges {534621=567034,1636289=498332} 2023-10-09T09:11:59.095 INFO:tasks.rados.rados.0.smithi163.stdout:1569: writing smithi16367185-7 from 534621 to 1101655 tid 1 2023-10-09T09:11:59.102 INFO:tasks.rados.rados.0.smithi163.stdout:1569: writing smithi16367185-7 from 1636289 to 2134621 tid 2 2023-10-09T09:11:59.102 INFO:tasks.rados.rados.0.smithi163.stdout:1570: write oid 37 current snap is 191 2023-10-09T09:11:59.102 INFO:tasks.rados.rados.0.smithi163.stdout:1570: seq_num 604 ranges {551428=642283,1694771=456657} 2023-10-09T09:11:59.109 INFO:tasks.rados.rados.0.smithi163.stdout:1570: writing smithi16367185-37 from 551428 to 1193711 tid 1 2023-10-09T09:11:59.114 INFO:tasks.rados.rados.0.smithi163.stdout:1570: writing smithi16367185-37 from 1694771 to 2151428 tid 2 2023-10-09T09:11:59.115 INFO:tasks.rados.rados.0.smithi163.stdout:1571: write oid 14 current snap is 191 2023-10-09T09:11:59.115 INFO:tasks.rados.rados.0.smithi163.stdout:1571: seq_num 605 ranges {568235=717532,1753253=414982} 2023-10-09T09:11:59.121 INFO:tasks.rados.rados.0.smithi163.stdout:1571: writing smithi16367185-14 from 568235 to 1285767 tid 1 2023-10-09T09:11:59.127 INFO:tasks.rados.rados.0.smithi163.stdout:1571: writing smithi16367185-14 from 1753253 to 2168235 tid 2 2023-10-09T09:11:59.127 INFO:tasks.rados.rados.0.smithi163.stdout:1572: read oid 24 snap -1 2023-10-09T09:11:59.127 INFO:tasks.rados.rados.0.smithi163.stdout:1572: expect (ObjNum 581 snap 185 seq_num 581) 2023-10-09T09:11:59.127 INFO:tasks.rados.rados.0.smithi163.stdout:1573: rollback oid 50 current snap is 191 2023-10-09T09:11:59.127 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 50 to 172 2023-10-09T09:11:59.128 INFO:tasks.rados.rados.0.smithi163.stdout:1565: expect (ObjNum 504 snap 157 seq_num 504) 2023-10-09T09:11:59.250 INFO:tasks.rados.rados.0.smithi163.stdout:1569: finishing write tid 1 to smithi16367185-7 2023-10-09T09:11:59.250 INFO:tasks.rados.rados.0.smithi163.stdout:1569: finishing write tid 2 to smithi16367185-7 2023-10-09T09:11:59.250 INFO:tasks.rados.rados.0.smithi163.stdout:1569: finishing write tid 3 to smithi16367185-7 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1569: finishing write tid 4 to smithi16367185-7 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 405 (ObjNum 603 snap 191 seq_num 603) dirty exists 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1569: left oid 7 (ObjNum 603 snap 191 seq_num 603) 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1570: finishing write tid 1 to smithi16367185-37 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1570: finishing write tid 2 to smithi16367185-37 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1570: finishing write tid 3 to smithi16367185-37 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1570: finishing write tid 4 to smithi16367185-37 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 801 (ObjNum 604 snap 191 seq_num 604) dirty exists 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1570: left oid 37 (ObjNum 604 snap 191 seq_num 604) 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1571: finishing write tid 1 to smithi16367185-14 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1571: finishing write tid 2 to smithi16367185-14 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1573: finishing rollback tid 0 to smithi16367185-50 2023-10-09T09:11:59.251 INFO:tasks.rados.rados.0.smithi163.stdout:1571: finishing write tid 3 to smithi16367185-14 2023-10-09T09:11:59.252 INFO:tasks.rados.rados.0.smithi163.stdout:1573: finishing rollback tid 1 to smithi16367185-50 2023-10-09T09:11:59.252 INFO:tasks.rados.rados.0.smithi163.stdout:1571: finishing write tid 4 to smithi16367185-14 2023-10-09T09:11:59.252 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 627 (ObjNum 605 snap 191 seq_num 605) dirty exists 2023-10-09T09:11:59.252 INFO:tasks.rados.rados.0.smithi163.stdout:1571: left oid 14 (ObjNum 605 snap 191 seq_num 605) 2023-10-09T09:11:59.252 INFO:tasks.rados.rados.0.smithi163.stdout:1573: finishing rollback tid 2 to smithi16367185-50 2023-10-09T09:11:59.252 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 642 (ObjNum 562 snap 172 seq_num 562) dirty exists 2023-10-09T09:11:59.252 INFO:tasks.rados.rados.0.smithi163.stdout:1572: expect (ObjNum 581 snap 185 seq_num 581) 2023-10-09T09:11:59.426 INFO:tasks.rados.rados.0.smithi163.stdout:1565: done (6 left) 2023-10-09T09:11:59.427 INFO:tasks.rados.rados.0.smithi163.stdout:1566: done (5 left) 2023-10-09T09:11:59.427 INFO:tasks.rados.rados.0.smithi163.stdout:1569: done (4 left) 2023-10-09T09:11:59.427 INFO:tasks.rados.rados.0.smithi163.stdout:1570: done (3 left) 2023-10-09T09:11:59.427 INFO:tasks.rados.rados.0.smithi163.stdout:1571: done (2 left) 2023-10-09T09:11:59.427 INFO:tasks.rados.rados.0.smithi163.stdout:1572: done (1 left) 2023-10-09T09:11:59.427 INFO:tasks.rados.rados.0.smithi163.stdout:1573: done (0 left) 2023-10-09T09:11:59.427 INFO:tasks.rados.rados.0.smithi163.stdout:1574: snap_remove snap 190 2023-10-09T09:11:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:59 smithi181 bash[27712]: cluster 2023-10-09T09:11:58.069916+0000 osd.3 (osd.3) 99 : cluster [WRN] failed to encode map e450 with expected crc 2023-10-09T09:11:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:59 smithi181 bash[27712]: cluster 2023-10-09T09:11:58.073424+0000 mon.a (mon.0) 288 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2023-10-09T09:11:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:59 smithi181 bash[27712]: cluster 2023-10-09T09:11:58.076110+0000 osd.5 (osd.5) 58 : cluster [WRN] failed to encode map e450 with expected crc 2023-10-09T09:11:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:11:59 smithi181 bash[27712]: debug 2023-10-09T09:11:59.077+0000 7fde83035700 -1 mon.b@1(peon).osd e451 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:00.027 INFO:tasks.rados.rados.0.smithi163.stdout:1574: done (0 left) 2023-10-09T09:12:00.027 INFO:tasks.rados.rados.0.smithi163.stdout:1575: snap_create 2023-10-09T09:12:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:00 smithi181 bash[27712]: debug 2023-10-09T09:12:00.013+0000 7fde83035700 -1 mon.b@1(peon).osd e452 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:00 smithi181 bash[27712]: cluster 2023-10-09T09:11:59.082364+0000 osd.3 (osd.3) 100 : cluster [WRN] failed to encode map e451 with expected crc 2023-10-09T09:12:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:00 smithi181 bash[27712]: cluster 2023-10-09T09:11:59.084241+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2023-10-09T09:12:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:00 smithi181 bash[27712]: cluster 2023-10-09T09:11:59.085871+0000 osd.1 (osd.1) 46 : cluster [WRN] failed to encode map e451 with expected crc 2023-10-09T09:12:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:00 smithi181 bash[27712]: cluster 2023-10-09T09:11:59.263339+0000 mgr.y (mgr.34107) 101 : cluster [DBG] pgmap v227: 89 pgs: 89 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 979 KiB/s wr, 27 op/s 2023-10-09T09:12:00.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:00 smithi181 bash[27712]: cluster 2023-10-09T09:12:00.021274+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2023-10-09T09:12:01.043 INFO:tasks.rados.rados.0.smithi163.stdout:1575: done (0 left) 2023-10-09T09:12:01.043 INFO:tasks.rados.rados.0.smithi163.stdout:1576: snap_remove snap 191 2023-10-09T09:12:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:01 smithi181 bash[27712]: debug 2023-10-09T09:12:01.029+0000 7fde83035700 -1 mon.b@1(peon).osd e453 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:01 smithi181 bash[27712]: cluster 2023-10-09T09:12:00.017630+0000 osd.3 (osd.3) 101 : cluster [WRN] failed to encode map e452 with expected crc 2023-10-09T09:12:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:01 smithi181 bash[27712]: cluster 2023-10-09T09:12:00.024014+0000 osd.6 (osd.6) 46 : cluster [WRN] failed to encode map e452 with expected crc 2023-10-09T09:12:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:01 smithi181 bash[27712]: cluster 2023-10-09T09:12:01.037556+0000 mon.a (mon.0) 291 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2023-10-09T09:12:02.045 INFO:tasks.rados.rados.0.smithi163.stdout:1576: done (0 left) 2023-10-09T09:12:02.045 INFO:tasks.rados.rados.0.smithi163.stdout:1577: read oid 14 snap -1 2023-10-09T09:12:02.045 INFO:tasks.rados.rados.0.smithi163.stdout:1577: expect (ObjNum 605 snap 191 seq_num 605) 2023-10-09T09:12:02.045 INFO:tasks.rados.rados.0.smithi163.stdout:1578: read oid 5 snap -1 2023-10-09T09:12:02.045 INFO:tasks.rados.rados.0.smithi163.stdout:1578: expect deleted 2023-10-09T09:12:02.045 INFO:tasks.rados.rados.0.smithi163.stdout:1579: rollback oid 23 current snap is 192 2023-10-09T09:12:02.046 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 23 to 172 2023-10-09T09:12:02.046 INFO:tasks.rados.rados.0.smithi163.stdout:1580: read oid 1 snap 188 2023-10-09T09:12:02.046 INFO:tasks.rados.rados.0.smithi163.stdout:1580: expect (ObjNum 510 snap 159 seq_num 510) 2023-10-09T09:12:02.046 INFO:tasks.rados.rados.0.smithi163.stdout:1581: write oid 22 current snap is 192 2023-10-09T09:12:02.046 INFO:tasks.rados.rados.0.smithi163.stdout:1581: seq_num 606 ranges {585042=792781,2095382=89660} 2023-10-09T09:12:02.057 INFO:tasks.rados.rados.0.smithi163.stdout:1581: writing smithi16367185-22 from 585042 to 1377823 tid 1 2023-10-09T09:12:02.062 INFO:tasks.rados.rados.0.smithi163.stdout:1581: writing smithi16367185-22 from 2095382 to 2185042 tid 2 2023-10-09T09:12:02.062 INFO:tasks.rados.rados.0.smithi163.stdout:1582: write oid 11 current snap is 192 2023-10-09T09:12:02.062 INFO:tasks.rados.rados.0.smithi163.stdout:1582: seq_num 607 ranges {601849=468030,1753864=447985} 2023-10-09T09:12:02.068 INFO:tasks.rados.rados.0.smithi163.stdout:1582: writing smithi16367185-11 from 601849 to 1069879 tid 1 2023-10-09T09:12:02.075 INFO:tasks.rados.rados.0.smithi163.stdout:1582: writing smithi16367185-11 from 1753864 to 2201849 tid 2 2023-10-09T09:12:02.075 INFO:tasks.rados.rados.0.smithi163.stdout:1583: write oid 27 current snap is 192 2023-10-09T09:12:02.075 INFO:tasks.rados.rados.0.smithi163.stdout:1583: seq_num 608 ranges {618656=543279,1812346=406310} 2023-10-09T09:12:02.081 INFO:tasks.rados.rados.0.smithi163.stdout:1583: writing smithi16367185-27 from 618656 to 1161935 tid 1 2023-10-09T09:12:02.087 INFO:tasks.rados.rados.0.smithi163.stdout:1583: writing smithi16367185-27 from 1812346 to 2218656 tid 2 2023-10-09T09:12:02.088 INFO:tasks.rados.rados.0.smithi163.stdout:1584: delete oid 8 current snap is 192 2023-10-09T09:12:02.088 INFO:tasks.rados.rados.0.smithi163.stdout:1580: expect (ObjNum 510 snap 159 seq_num 510) 2023-10-09T09:12:02.311 INFO:tasks.rados.rados.0.smithi163.stdout:1579: finishing rollback tid 0 to smithi16367185-23 2023-10-09T09:12:02.312 INFO:tasks.rados.rados.0.smithi163.stdout:1579: finishing rollback tid 1 to smithi16367185-23 2023-10-09T09:12:02.312 INFO:tasks.rados.rados.0.smithi163.stdout:1579: finishing rollback tid 2 to smithi16367185-23 2023-10-09T09:12:02.312 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 648 (ObjNum 545 snap 170 seq_num 545) dirty exists 2023-10-09T09:12:02.312 INFO:tasks.rados.rados.0.smithi163.stdout:1577: expect (ObjNum 605 snap 191 seq_num 605) 2023-10-09T09:12:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:02 smithi181 bash[27712]: debug 2023-10-09T09:12:02.029+0000 7fde83035700 -1 mon.b@1(peon).osd e454 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:02 smithi181 bash[27712]: cluster 2023-10-09T09:12:01.032517+0000 osd.3 (osd.3) 102 : cluster [WRN] failed to encode map e453 with expected crc 2023-10-09T09:12:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:02 smithi181 bash[27712]: cluster 2023-10-09T09:12:01.038394+0000 osd.7 (osd.7) 75 : cluster [WRN] failed to encode map e453 with expected crc 2023-10-09T09:12:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:02 smithi181 bash[27712]: cluster 2023-10-09T09:12:01.264430+0000 mgr.y (mgr.34107) 102 : cluster [DBG] pgmap v230: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 805 KiB/s wr, 8 op/s 2023-10-09T09:12:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:02 smithi181 bash[27712]: cluster 2023-10-09T09:12:02.039032+0000 mon.a (mon.0) 292 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2023-10-09T09:12:02.441 INFO:tasks.rados.rados.0.smithi163.stdout:1581: finishing write tid 1 to smithi16367185-22 2023-10-09T09:12:02.441 INFO:tasks.rados.rados.0.smithi163.stdout:1581: finishing write tid 2 to smithi16367185-22 2023-10-09T09:12:02.441 INFO:tasks.rados.rados.0.smithi163.stdout:1581: finishing write tid 3 to smithi16367185-22 2023-10-09T09:12:02.441 INFO:tasks.rados.rados.0.smithi163.stdout:1581: finishing write tid 4 to smithi16367185-22 2023-10-09T09:12:02.441 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 1262 (ObjNum 606 snap 192 seq_num 606) dirty exists 2023-10-09T09:12:02.441 INFO:tasks.rados.rados.0.smithi163.stdout:1581: left oid 22 (ObjNum 606 snap 192 seq_num 606) 2023-10-09T09:12:02.441 INFO:tasks.rados.rados.0.smithi163.stdout:1582: finishing write tid 1 to smithi16367185-11 2023-10-09T09:12:02.441 INFO:tasks.rados.rados.0.smithi163.stdout:1582: finishing write tid 2 to smithi16367185-11 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:1582: finishing write tid 3 to smithi16367185-11 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:1582: finishing write tid 4 to smithi16367185-11 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 465 (ObjNum 607 snap 192 seq_num 607) dirty exists 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:1582: left oid 11 (ObjNum 607 snap 192 seq_num 607) 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:1583: finishing write tid 1 to smithi16367185-27 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:1583: finishing write tid 2 to smithi16367185-27 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:1583: finishing write tid 3 to smithi16367185-27 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:1583: finishing write tid 4 to smithi16367185-27 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 547 (ObjNum 608 snap 192 seq_num 608) dirty exists 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:1583: left oid 27 (ObjNum 608 snap 192 seq_num 608) 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:1577: done (7 left) 2023-10-09T09:12:02.442 INFO:tasks.rados.rados.0.smithi163.stdout:1578: done (6 left) 2023-10-09T09:12:02.443 INFO:tasks.rados.rados.0.smithi163.stdout:1579: done (5 left) 2023-10-09T09:12:02.443 INFO:tasks.rados.rados.0.smithi163.stdout:1580: done (4 left) 2023-10-09T09:12:02.443 INFO:tasks.rados.rados.0.smithi163.stdout:1581: done (3 left) 2023-10-09T09:12:02.443 INFO:tasks.rados.rados.0.smithi163.stdout:1582: done (2 left) 2023-10-09T09:12:02.443 INFO:tasks.rados.rados.0.smithi163.stdout:1583: done (1 left) 2023-10-09T09:12:02.443 INFO:tasks.rados.rados.0.smithi163.stdout:1584: done (0 left) 2023-10-09T09:12:02.443 INFO:tasks.rados.rados.0.smithi163.stdout:1585: write oid 43 current snap is 192 2023-10-09T09:12:02.443 INFO:tasks.rados.rados.0.smithi163.stdout:1585: seq_num 610 ranges {652270=410130,1529310=493867} 2023-10-09T09:12:02.451 INFO:tasks.rados.rados.0.smithi163.stdout:1585: writing smithi16367185-43 from 652270 to 1062400 tid 1 2023-10-09T09:12:02.457 INFO:tasks.rados.rados.0.smithi163.stdout:1585: writing smithi16367185-43 from 1529310 to 2023177 tid 2 2023-10-09T09:12:02.457 INFO:tasks.rados.rados.0.smithi163.stdout:1586: read oid 48 snap 188 2023-10-09T09:12:02.457 INFO:tasks.rados.rados.0.smithi163.stdout:1586: expect deleted 2023-10-09T09:12:02.457 INFO:tasks.rados.rados.0.smithi163.stdout:1587: delete oid 24 current snap is 192 2023-10-09T09:12:02.458 INFO:tasks.rados.rados.0.smithi163.stdout:1585: finishing write tid 1 to smithi16367185-43 2023-10-09T09:12:02.461 INFO:tasks.rados.rados.0.smithi163.stdout:1586: done (2 left) 2023-10-09T09:12:02.461 INFO:tasks.rados.rados.0.smithi163.stdout:1587: done (1 left) 2023-10-09T09:12:02.462 INFO:tasks.rados.rados.0.smithi163.stdout:1588: write oid 14 current snap is 192 2023-10-09T09:12:02.462 INFO:tasks.rados.rados.0.smithi163.stdout:1588: seq_num 612 ranges {685884=560628,2046274=239610} 2023-10-09T09:12:02.469 INFO:tasks.rados.rados.0.smithi163.stdout:1588: writing smithi16367185-14 from 685884 to 1246512 tid 1 2023-10-09T09:12:02.475 INFO:tasks.rados.rados.0.smithi163.stdout:1588: writing smithi16367185-14 from 2046274 to 2285884 tid 2 2023-10-09T09:12:02.475 INFO:tasks.rados.rados.0.smithi163.stdout:1589: snap_remove snap 188 2023-10-09T09:12:03.172 INFO:tasks.rados.rados.0.smithi163.stdout:1589: done (2 left) 2023-10-09T09:12:03.173 INFO:tasks.rados.rados.0.smithi163.stdout:1590: snap_remove snap 172 2023-10-09T09:12:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:03 smithi181 bash[27712]: cluster 2023-10-09T09:12:02.035283+0000 osd.3 (osd.3) 103 : cluster [WRN] failed to encode map e454 with expected crc 2023-10-09T09:12:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:03 smithi181 bash[27712]: cluster 2023-10-09T09:12:02.042075+0000 osd.1 (osd.1) 47 : cluster [WRN] failed to encode map e454 with expected crc 2023-10-09T09:12:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:03 smithi181 bash[27712]: cluster 2023-10-09T09:12:02.044644+0000 osd.4 (osd.4) 97 : cluster [WRN] failed to encode map e454 with expected crc 2023-10-09T09:12:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:03 smithi181 bash[27712]: cluster 2023-10-09T09:12:02.044894+0000 osd.5 (osd.5) 59 : cluster [WRN] failed to encode map e454 with expected crc 2023-10-09T09:12:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:03 smithi181 bash[27712]: cluster 2023-10-09T09:12:02.047797+0000 osd.2 (osd.2) 103 : cluster [WRN] failed to encode map e454 with expected crc 2023-10-09T09:12:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:03 smithi181 bash[27712]: cluster 2023-10-09T09:12:02.069078+0000 osd.7 (osd.7) 76 : cluster [WRN] failed to encode map e454 with expected crc 2023-10-09T09:12:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:03 smithi181 bash[27712]: debug 2023-10-09T09:12:03.157+0000 7fde83035700 -1 mon.b@1(peon).osd e455 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:12:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:12:03] "GET /metrics HTTP/1.1" 200 36086 "" "Prometheus/2.33.4" 2023-10-09T09:12:04.180 INFO:tasks.rados.rados.0.smithi163.stdout:1590: done (2 left) 2023-10-09T09:12:04.180 INFO:tasks.rados.rados.0.smithi163.stdout:1591: delete oid 7 current snap is 192 2023-10-09T09:12:04.180 INFO:tasks.rados.rados.0.smithi163.stdout:1585: finishing write tid 2 to smithi16367185-43 2023-10-09T09:12:04.180 INFO:tasks.rados.rados.0.smithi163.stdout:1585: finishing write tid 3 to smithi16367185-43 2023-10-09T09:12:04.180 INFO:tasks.rados.rados.0.smithi163.stdout:1585: finishing write tid 4 to smithi16367185-43 2023-10-09T09:12:04.180 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 633 (ObjNum 610 snap 192 seq_num 610) dirty exists 2023-10-09T09:12:04.180 INFO:tasks.rados.rados.0.smithi163.stdout:1585: left oid 43 (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:12:04.180 INFO:tasks.rados.rados.0.smithi163.stdout:1588: finishing write tid 1 to smithi16367185-14 2023-10-09T09:12:04.181 INFO:tasks.rados.rados.0.smithi163.stdout:1588: finishing write tid 2 to smithi16367185-14 2023-10-09T09:12:04.181 INFO:tasks.rados.rados.0.smithi163.stdout:1588: finishing write tid 3 to smithi16367185-14 2023-10-09T09:12:04.181 INFO:tasks.rados.rados.0.smithi163.stdout:1588: finishing write tid 4 to smithi16367185-14 2023-10-09T09:12:04.181 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 636 (ObjNum 612 snap 192 seq_num 612) dirty exists 2023-10-09T09:12:04.181 INFO:tasks.rados.rados.0.smithi163.stdout:1588: left oid 14 (ObjNum 612 snap 192 seq_num 612) 2023-10-09T09:12:04.186 INFO:tasks.rados.rados.0.smithi163.stdout:1585: done (2 left) 2023-10-09T09:12:04.186 INFO:tasks.rados.rados.0.smithi163.stdout:1588: done (1 left) 2023-10-09T09:12:04.186 INFO:tasks.rados.rados.0.smithi163.stdout:1591: done (0 left) 2023-10-09T09:12:04.186 INFO:tasks.rados.rados.0.smithi163.stdout:1592: snap_create 2023-10-09T09:12:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:04 smithi181 bash[27712]: cluster 2023-10-09T09:12:03.163138+0000 osd.3 (osd.3) 104 : cluster [WRN] failed to encode map e455 with expected crc 2023-10-09T09:12:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:04 smithi181 bash[27712]: cluster 2023-10-09T09:12:03.164697+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2023-10-09T09:12:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:04 smithi181 bash[27712]: cluster 2023-10-09T09:12:03.265148+0000 mgr.y (mgr.34107) 103 : cluster [DBG] pgmap v233: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 805 KiB/s wr, 8 op/s 2023-10-09T09:12:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:04 smithi181 bash[27712]: debug 2023-10-09T09:12:04.169+0000 7fde83035700 -1 mon.b@1(peon).osd e456 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:05.030 INFO:tasks.rados.rados.0.smithi163.stdout:1592: done (0 left) 2023-10-09T09:12:05.030 INFO:tasks.rados.rados.0.smithi163.stdout:1593: rollback oid 25 current snap is 193 2023-10-09T09:12:05.030 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 25 to 192 2023-10-09T09:12:05.030 INFO:tasks.rados.rados.0.smithi163.stdout:1594: rollback oid 38 current snap is 193 2023-10-09T09:12:05.030 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 192 2023-10-09T09:12:05.030 INFO:tasks.rados.rados.0.smithi163.stdout:1595: rollback oid 43 current snap is 193 2023-10-09T09:12:05.030 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 192 2023-10-09T09:12:05.031 INFO:tasks.rados.rados.0.smithi163.stdout:1596: rollback oid 6 current snap is 193 2023-10-09T09:12:05.031 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 6 to 192 2023-10-09T09:12:05.031 INFO:tasks.rados.rados.0.smithi163.stdout:1597: read oid 2 snap -1 2023-10-09T09:12:05.031 INFO:tasks.rados.rados.0.smithi163.stdout:1597: expect (ObjNum 580 snap 181 seq_num 580) 2023-10-09T09:12:05.031 INFO:tasks.rados.rados.0.smithi163.stdout:1598: delete oid 9 current snap is 193 2023-10-09T09:12:05.034 INFO:tasks.rados.rados.0.smithi163.stdout:1598: done (5 left) 2023-10-09T09:12:05.034 INFO:tasks.rados.rados.0.smithi163.stdout:1599: write oid 31 current snap is 193 2023-10-09T09:12:05.035 INFO:tasks.rados.rados.0.smithi163.stdout:1599: seq_num 615 ranges {736305=786375,2105367=230938} 2023-10-09T09:12:05.046 INFO:tasks.rados.rados.0.smithi163.stdout:1599: writing smithi16367185-31 from 736305 to 1522680 tid 1 2023-10-09T09:12:05.051 INFO:tasks.rados.rados.0.smithi163.stdout:1599: writing smithi16367185-31 from 2105367 to 2336305 tid 2 2023-10-09T09:12:05.052 INFO:tasks.rados.rados.0.smithi163.stdout:1600: rollback oid 18 current snap is 193 2023-10-09T09:12:05.052 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 18 to 192 2023-10-09T09:12:05.052 INFO:tasks.rados.rados.0.smithi163.stdout:1596: finishing rollback tid 0 to smithi16367185-6 2023-10-09T09:12:05.052 INFO:tasks.rados.rados.0.smithi163.stdout:1593: finishing rollback tid 0 to smithi16367185-25 2023-10-09T09:12:05.052 INFO:tasks.rados.rados.0.smithi163.stdout:1593: finishing rollback tid 1 to smithi16367185-25 2023-10-09T09:12:05.052 INFO:tasks.rados.rados.0.smithi163.stdout:1601: snap_create 2023-10-09T09:12:05.052 INFO:tasks.rados.rados.0.smithi163.stdout:1593: finishing rollback tid 2 to smithi16367185-25 2023-10-09T09:12:05.052 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 743 (ObjNum 598 snap 188 seq_num 598) dirty exists 2023-10-09T09:12:05.052 INFO:tasks.rados.rados.0.smithi163.stdout:1594: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:12:05.052 INFO:tasks.rados.rados.0.smithi163.stdout:1594: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:1594: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 652 (ObjNum 503 snap 157 seq_num 503) dirty exists 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:1595: finishing rollback tid 0 to smithi16367185-43 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:1596: finishing rollback tid 1 to smithi16367185-6 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:1595: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:1595: finishing rollback tid 2 to smithi16367185-43 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 656 (ObjNum 610 snap 192 seq_num 610) dirty exists 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:1596: finishing rollback tid 2 to smithi16367185-6 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 177 (ObjNum 431 snap 134 seq_num 431) dirty exists 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:1599: finishing write tid 1 to smithi16367185-31 2023-10-09T09:12:05.053 INFO:tasks.rados.rados.0.smithi163.stdout:1600: finishing rollback tid 1 to smithi16367185-18 2023-10-09T09:12:05.054 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 0 (ObjNum 5 snap 1 seq_num 0) dirty dne 2023-10-09T09:12:05.054 INFO:tasks.rados.rados.0.smithi163.stdout:1599: finishing write tid 2 to smithi16367185-31 2023-10-09T09:12:05.054 INFO:tasks.rados.rados.0.smithi163.stdout:1599: finishing write tid 3 to smithi16367185-31 2023-10-09T09:12:05.054 INFO:tasks.rados.rados.0.smithi163.stdout:1599: finishing write tid 4 to smithi16367185-31 2023-10-09T09:12:05.055 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 444 (ObjNum 615 snap 193 seq_num 615) dirty exists 2023-10-09T09:12:05.055 INFO:tasks.rados.rados.0.smithi163.stdout:1599: left oid 31 (ObjNum 615 snap 193 seq_num 615) 2023-10-09T09:12:05.055 INFO:tasks.rados.rados.0.smithi163.stdout:1597: expect (ObjNum 580 snap 181 seq_num 580) 2023-10-09T09:12:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:05 smithi181 bash[27712]: debug 2023-10-09T09:12:05.017+0000 7fde83035700 -1 mon.b@1(peon).osd e457 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:05 smithi181 bash[27712]: cluster 2023-10-09T09:12:04.173592+0000 osd.3 (osd.3) 105 : cluster [WRN] failed to encode map e456 with expected crc 2023-10-09T09:12:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:05 smithi181 bash[27712]: cluster 2023-10-09T09:12:04.175187+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2023-10-09T09:12:05.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:05 smithi181 bash[27712]: cluster 2023-10-09T09:12:04.177768+0000 osd.1 (osd.1) 48 : cluster [WRN] failed to encode map e456 with expected crc 2023-10-09T09:12:05.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:05 smithi181 bash[27712]: cluster 2023-10-09T09:12:05.025398+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2023-10-09T09:12:06.046 INFO:tasks.rados.rados.0.smithi163.stdout:1593: done (7 left) 2023-10-09T09:12:06.047 INFO:tasks.rados.rados.0.smithi163.stdout:1594: done (6 left) 2023-10-09T09:12:06.047 INFO:tasks.rados.rados.0.smithi163.stdout:1595: done (5 left) 2023-10-09T09:12:06.047 INFO:tasks.rados.rados.0.smithi163.stdout:1596: done (4 left) 2023-10-09T09:12:06.047 INFO:tasks.rados.rados.0.smithi163.stdout:1597: done (3 left) 2023-10-09T09:12:06.047 INFO:tasks.rados.rados.0.smithi163.stdout:1599: done (2 left) 2023-10-09T09:12:06.047 INFO:tasks.rados.rados.0.smithi163.stdout:1600: done (1 left) 2023-10-09T09:12:06.047 INFO:tasks.rados.rados.0.smithi163.stdout:1601: done (0 left) 2023-10-09T09:12:06.047 INFO:tasks.rados.rados.0.smithi163.stdout:1602: read oid 25 snap -1 2023-10-09T09:12:06.047 INFO:tasks.rados.rados.0.smithi163.stdout:1602: expect (ObjNum 598 snap 188 seq_num 598) 2023-10-09T09:12:06.047 INFO:tasks.rados.rados.0.smithi163.stdout:1603: snap_create 2023-10-09T09:12:06.054 INFO:tasks.rados.rados.0.smithi163.stdout:1602: expect (ObjNum 598 snap 188 seq_num 598) 2023-10-09T09:12:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:06 smithi181 bash[27712]: debug 2023-10-09T09:12:06.029+0000 7fde83035700 -1 mon.b@1(peon).osd e458 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:06 smithi181 bash[27712]: cluster 2023-10-09T09:12:05.022013+0000 osd.3 (osd.3) 106 : cluster [WRN] failed to encode map e457 with expected crc 2023-10-09T09:12:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:06 smithi181 bash[27712]: cluster 2023-10-09T09:12:05.025557+0000 osd.5 (osd.5) 60 : cluster [WRN] failed to encode map e457 with expected crc 2023-10-09T09:12:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:06 smithi181 bash[27712]: cluster 2023-10-09T09:12:05.031577+0000 osd.2 (osd.2) 104 : cluster [WRN] failed to encode map e457 with expected crc 2023-10-09T09:12:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:06 smithi181 bash[27712]: cluster 2023-10-09T09:12:05.265806+0000 mgr.y (mgr.34107) 104 : cluster [DBG] pgmap v236: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:06.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:06 smithi181 bash[27712]: cluster 2023-10-09T09:12:06.038439+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2023-10-09T09:12:07.047 INFO:tasks.rados.rados.0.smithi163.stdout:1602: done (1 left) 2023-10-09T09:12:07.047 INFO:tasks.rados.rados.0.smithi163.stdout:1603: done (0 left) 2023-10-09T09:12:07.047 INFO:tasks.rados.rados.0.smithi163.stdout:1604: delete oid 41 current snap is 195 2023-10-09T09:12:07.056 INFO:tasks.rados.rados.0.smithi163.stdout:1604: done (0 left) 2023-10-09T09:12:07.056 INFO:tasks.rados.rados.0.smithi163.stdout:1605: write oid 41 current snap is 195 2023-10-09T09:12:07.056 INFO:tasks.rados.rados.0.smithi163.stdout:1605: seq_num 617 ranges {769919=653226,1938684=431235} 2023-10-09T09:12:07.070 INFO:tasks.rados.rados.0.smithi163.stdout:1605: writing smithi16367185-41 from 769919 to 1423145 tid 1 2023-10-09T09:12:07.076 INFO:tasks.rados.rados.0.smithi163.stdout:1605: writing smithi16367185-41 from 1938684 to 2369919 tid 2 2023-10-09T09:12:07.076 INFO:tasks.rados.rados.0.smithi163.stdout:1606: rollback oid 43 current snap is 195 2023-10-09T09:12:07.076 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 194 2023-10-09T09:12:07.076 INFO:tasks.rados.rados.0.smithi163.stdout:1605: finishing write tid 1 to smithi16367185-41 2023-10-09T09:12:07.076 INFO:tasks.rados.rados.0.smithi163.stdout:1607: snap_create 2023-10-09T09:12:07.080 INFO:tasks.rados.rados.0.smithi163.stdout:1605: finishing write tid 2 to smithi16367185-41 2023-10-09T09:12:07.080 INFO:tasks.rados.rados.0.smithi163.stdout:1605: finishing write tid 3 to smithi16367185-41 2023-10-09T09:12:07.080 INFO:tasks.rados.rados.0.smithi163.stdout:1605: finishing write tid 4 to smithi16367185-41 2023-10-09T09:12:07.080 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 110 (ObjNum 617 snap 195 seq_num 617) dirty exists 2023-10-09T09:12:07.080 INFO:tasks.rados.rados.0.smithi163.stdout:1605: left oid 41 (ObjNum 617 snap 195 seq_num 617) 2023-10-09T09:12:07.086 INFO:tasks.rados.rados.0.smithi163.stdout:1606: finishing rollback tid 0 to smithi16367185-43 2023-10-09T09:12:07.086 INFO:tasks.rados.rados.0.smithi163.stdout:1606: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:12:07.086 INFO:tasks.rados.rados.0.smithi163.stdout:1606: finishing rollback tid 2 to smithi16367185-43 2023-10-09T09:12:07.086 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 660 (ObjNum 610 snap 192 seq_num 610) dirty exists 2023-10-09T09:12:07.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:07 smithi181 bash[27712]: debug 2023-10-09T09:12:07.033+0000 7fde83035700 -1 mon.b@1(peon).osd e459 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:07.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:07 smithi181 bash[27712]: cluster 2023-10-09T09:12:06.034977+0000 osd.3 (osd.3) 107 : cluster [WRN] failed to encode map e458 with expected crc 2023-10-09T09:12:07.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:07 smithi181 bash[27712]: cluster 2023-10-09T09:12:06.041989+0000 osd.4 (osd.4) 98 : cluster [WRN] failed to encode map e458 with expected crc 2023-10-09T09:12:07.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:07 smithi181 bash[27712]: cluster 2023-10-09T09:12:06.048066+0000 osd.2 (osd.2) 105 : cluster [WRN] failed to encode map e458 with expected crc 2023-10-09T09:12:07.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:07 smithi181 bash[27712]: cluster 2023-10-09T09:12:06.403089+0000 osd.5 (osd.5) 61 : cluster [WRN] failed to encode map e458 with expected crc 2023-10-09T09:12:07.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:07 smithi181 bash[27712]: cluster 2023-10-09T09:12:06.830163+0000 osd.1 (osd.1) 49 : cluster [WRN] failed to encode map e458 with expected crc 2023-10-09T09:12:07.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:07 smithi181 bash[27712]: cluster 2023-10-09T09:12:07.042985+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2023-10-09T09:12:08.051 INFO:tasks.rados.rados.0.smithi163.stdout:1605: done (2 left) 2023-10-09T09:12:08.051 INFO:tasks.rados.rados.0.smithi163.stdout:1606: done (1 left) 2023-10-09T09:12:08.051 INFO:tasks.rados.rados.0.smithi163.stdout:1607: done (0 left) 2023-10-09T09:12:08.051 INFO:tasks.rados.rados.0.smithi163.stdout:1608: write oid 37 current snap is 196 2023-10-09T09:12:08.051 INFO:tasks.rados.rados.0.smithi163.stdout:1608: seq_num 618 ranges {786726=728475,2280813=105913} 2023-10-09T09:12:08.066 INFO:tasks.rados.rados.0.smithi163.stdout:1608: writing smithi16367185-37 from 786726 to 1515201 tid 1 2023-10-09T09:12:08.071 INFO:tasks.rados.rados.0.smithi163.stdout:1608: writing smithi16367185-37 from 2280813 to 2386726 tid 2 2023-10-09T09:12:08.071 INFO:tasks.rados.rados.0.smithi163.stdout:1609: snap_remove snap 193 2023-10-09T09:12:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:08 smithi181 bash[27712]: debug 2023-10-09T09:12:08.033+0000 7fde83035700 -1 mon.b@1(peon).osd e460 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:08 smithi181 bash[27712]: cluster 2023-10-09T09:12:07.037678+0000 osd.3 (osd.3) 108 : cluster [WRN] failed to encode map e459 with expected crc 2023-10-09T09:12:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:08 smithi181 bash[27712]: cluster 2023-10-09T09:12:07.043597+0000 osd.6 (osd.6) 47 : cluster [WRN] failed to encode map e459 with expected crc 2023-10-09T09:12:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:08 smithi181 bash[27712]: cluster 2023-10-09T09:12:07.049305+0000 osd.2 (osd.2) 106 : cluster [WRN] failed to encode map e459 with expected crc 2023-10-09T09:12:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:08 smithi181 bash[27712]: cluster 2023-10-09T09:12:07.076311+0000 osd.4 (osd.4) 99 : cluster [WRN] failed to encode map e459 with expected crc 2023-10-09T09:12:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:08 smithi181 bash[27712]: cluster 2023-10-09T09:12:07.266499+0000 mgr.y (mgr.34107) 105 : cluster [DBG] pgmap v239: 89 pgs: 89 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.3 MiB/s wr, 19 op/s 2023-10-09T09:12:08.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:08 smithi181 bash[27712]: cluster 2023-10-09T09:12:08.043496+0000 mon.a (mon.0) 298 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2023-10-09T09:12:09.041 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:12:08 smithi181 bash[54907]: ts=2023-10-09T09:12:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:12:09.051 INFO:tasks.rados.rados.0.smithi163.stdout:1609: done (1 left) 2023-10-09T09:12:09.051 INFO:tasks.rados.rados.0.smithi163.stdout:1610: read oid 48 snap -1 2023-10-09T09:12:09.051 INFO:tasks.rados.rados.0.smithi163.stdout:1610: expect deleted 2023-10-09T09:12:09.051 INFO:tasks.rados.rados.0.smithi163.stdout:1611: read oid 19 snap -1 2023-10-09T09:12:09.051 INFO:tasks.rados.rados.0.smithi163.stdout:1611: expect deleted 2023-10-09T09:12:09.051 INFO:tasks.rados.rados.0.smithi163.stdout:1612: snap_remove snap 192 2023-10-09T09:12:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:09 smithi181 bash[27712]: debug 2023-10-09T09:12:09.037+0000 7fde83035700 -1 mon.b@1(peon).osd e461 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:09 smithi181 bash[27712]: cluster 2023-10-09T09:12:08.039704+0000 osd.3 (osd.3) 109 : cluster [WRN] failed to encode map e460 with expected crc 2023-10-09T09:12:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:09 smithi181 bash[27712]: cluster 2023-10-09T09:12:08.047300+0000 osd.5 (osd.5) 62 : cluster [WRN] failed to encode map e460 with expected crc 2023-10-09T09:12:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:09 smithi181 bash[27712]: cluster 2023-10-09T09:12:08.067721+0000 osd.7 (osd.7) 77 : cluster [WRN] failed to encode map e460 with expected crc 2023-10-09T09:12:09.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:09 smithi181 bash[27712]: cluster 2023-10-09T09:12:09.049606+0000 mon.a (mon.0) 299 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2023-10-09T09:12:10.032 INFO:tasks.rados.rados.0.smithi163.stdout:1612: done (3 left) 2023-10-09T09:12:10.032 INFO:tasks.rados.rados.0.smithi163.stdout:1613: write oid 41 current snap is 196 2023-10-09T09:12:10.033 INFO:tasks.rados.rados.0.smithi163.stdout:1613: seq_num 619 ranges {403533=403724,1539295=652201} 2023-10-09T09:12:10.040 INFO:tasks.rados.rados.0.smithi163.stdout:1613: writing smithi16367185-41 from 403533 to 807257 tid 1 2023-10-09T09:12:10.049 INFO:tasks.rados.rados.0.smithi163.stdout:1613: writing smithi16367185-41 from 1539295 to 2191496 tid 2 2023-10-09T09:12:10.050 INFO:tasks.rados.rados.0.smithi163.stdout:1608: finishing write tid 1 to smithi16367185-37 2023-10-09T09:12:10.050 INFO:tasks.rados.rados.0.smithi163.stdout:1608: finishing write tid 2 to smithi16367185-37 2023-10-09T09:12:10.050 INFO:tasks.rados.rados.0.smithi163.stdout:1608: finishing write tid 3 to smithi16367185-37 2023-10-09T09:12:10.050 INFO:tasks.rados.rados.0.smithi163.stdout:1608: finishing write tid 4 to smithi16367185-37 2023-10-09T09:12:10.050 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 823 (ObjNum 618 snap 196 seq_num 618) dirty exists 2023-10-09T09:12:10.051 INFO:tasks.rados.rados.0.smithi163.stdout:1608: left oid 37 (ObjNum 618 snap 196 seq_num 618) 2023-10-09T09:12:10.051 INFO:tasks.rados.rados.0.smithi163.stdout:1608: done (3 left) 2023-10-09T09:12:10.051 INFO:tasks.rados.rados.0.smithi163.stdout:1610: done (2 left) 2023-10-09T09:12:10.051 INFO:tasks.rados.rados.0.smithi163.stdout:1611: done (1 left) 2023-10-09T09:12:10.051 INFO:tasks.rados.rados.0.smithi163.stdout:1614: snap_remove snap 195 2023-10-09T09:12:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:10 smithi181 bash[27712]: debug 2023-10-09T09:12:10.017+0000 7fde83035700 -1 mon.b@1(peon).osd e462 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:10 smithi181 bash[27712]: cluster 2023-10-09T09:12:09.047128+0000 osd.3 (osd.3) 110 : cluster [WRN] failed to encode map e461 with expected crc 2023-10-09T09:12:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:10 smithi181 bash[27712]: cluster 2023-10-09T09:12:09.048200+0000 osd.0 (osd.0) 72 : cluster [WRN] failed to encode map e461 with expected crc 2023-10-09T09:12:10.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:10 smithi181 bash[27712]: cluster 2023-10-09T09:12:09.051542+0000 osd.5 (osd.5) 63 : cluster [WRN] failed to encode map e461 with expected crc 2023-10-09T09:12:10.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:10 smithi181 bash[27712]: cluster 2023-10-09T09:12:09.053031+0000 osd.2 (osd.2) 107 : cluster [WRN] failed to encode map e461 with expected crc 2023-10-09T09:12:10.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:10 smithi181 bash[27712]: cluster 2023-10-09T09:12:09.267164+0000 mgr.y (mgr.34107) 106 : cluster [DBG] pgmap v242: 89 pgs: 89 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.3 MiB/s wr, 19 op/s 2023-10-09T09:12:10.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:10 smithi181 bash[27712]: cluster 2023-10-09T09:12:10.028345+0000 mon.a (mon.0) 300 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2023-10-09T09:12:11.048 INFO:tasks.rados.rados.0.smithi163.stdout:1614: done (1 left) 2023-10-09T09:12:11.048 INFO:tasks.rados.rados.0.smithi163.stdout:1615: read oid 3 snap -1 2023-10-09T09:12:11.048 INFO:tasks.rados.rados.0.smithi163.stdout:1615: expect (ObjNum 579 snap 181 seq_num 579) 2023-10-09T09:12:11.048 INFO:tasks.rados.rados.0.smithi163.stdout:1616: snap_create 2023-10-09T09:12:11.048 INFO:tasks.rados.rados.0.smithi163.stdout:1613: finishing write tid 1 to smithi16367185-41 2023-10-09T09:12:11.048 INFO:tasks.rados.rados.0.smithi163.stdout:1613: finishing write tid 2 to smithi16367185-41 2023-10-09T09:12:11.048 INFO:tasks.rados.rados.0.smithi163.stdout:1613: finishing write tid 3 to smithi16367185-41 2023-10-09T09:12:11.048 INFO:tasks.rados.rados.0.smithi163.stdout:1613: finishing write tid 4 to smithi16367185-41 2023-10-09T09:12:11.048 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 118 (ObjNum 619 snap 196 seq_num 619) dirty exists 2023-10-09T09:12:11.048 INFO:tasks.rados.rados.0.smithi163.stdout:1613: left oid 41 (ObjNum 619 snap 196 seq_num 619) 2023-10-09T09:12:11.065 INFO:tasks.rados.rados.0.smithi163.stdout:1615: expect (ObjNum 579 snap 181 seq_num 579) 2023-10-09T09:12:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:11 smithi181 bash[27712]: debug 2023-10-09T09:12:11.033+0000 7fde83035700 -1 mon.b@1(peon).osd e463 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:11 smithi181 bash[27712]: cluster 2023-10-09T09:12:09.585008+0000 osd.6 (osd.6) 48 : cluster [WRN] failed to encode map e461 with expected crc 2023-10-09T09:12:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:11 smithi181 bash[27712]: cluster 2023-10-09T09:12:09.739095+0000 osd.4 (osd.4) 100 : cluster [WRN] failed to encode map e461 with expected crc 2023-10-09T09:12:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:11 smithi181 bash[27712]: cluster 2023-10-09T09:12:10.025281+0000 osd.3 (osd.3) 111 : cluster [WRN] failed to encode map e462 with expected crc 2023-10-09T09:12:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:11 smithi181 bash[27712]: cluster 2023-10-09T09:12:10.027551+0000 osd.4 (osd.4) 101 : cluster [WRN] failed to encode map e462 with expected crc 2023-10-09T09:12:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:11 smithi181 bash[27712]: cluster 2023-10-09T09:12:10.042542+0000 osd.2 (osd.2) 108 : cluster [WRN] failed to encode map e462 with expected crc 2023-10-09T09:12:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:11 smithi181 bash[27712]: audit 2023-10-09T09:12:10.339101+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:11.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:11 smithi181 bash[27712]: cluster 2023-10-09T09:12:11.041203+0000 mon.a (mon.0) 302 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2023-10-09T09:12:12.053 INFO:tasks.rados.rados.0.smithi163.stdout:1613: done (2 left) 2023-10-09T09:12:12.053 INFO:tasks.rados.rados.0.smithi163.stdout:1615: done (1 left) 2023-10-09T09:12:12.053 INFO:tasks.rados.rados.0.smithi163.stdout:1616: done (0 left) 2023-10-09T09:12:12.053 INFO:tasks.rados.rados.0.smithi163.stdout:1617: snap_remove snap 194 2023-10-09T09:12:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:12 smithi181 bash[27712]: debug 2023-10-09T09:12:12.033+0000 7fde83035700 -1 mon.b@1(peon).osd e464 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:12 smithi181 bash[27712]: cluster 2023-10-09T09:12:10.629353+0000 osd.6 (osd.6) 49 : cluster [WRN] failed to encode map e462 with expected crc 2023-10-09T09:12:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:12 smithi181 bash[27712]: cluster 2023-10-09T09:12:11.037016+0000 osd.3 (osd.3) 112 : cluster [WRN] failed to encode map e463 with expected crc 2023-10-09T09:12:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:12 smithi181 bash[27712]: cluster 2023-10-09T09:12:11.043333+0000 osd.7 (osd.7) 78 : cluster [WRN] failed to encode map e463 with expected crc 2023-10-09T09:12:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:12 smithi181 bash[27712]: cluster 2023-10-09T09:12:11.049538+0000 osd.0 (osd.0) 73 : cluster [WRN] failed to encode map e463 with expected crc 2023-10-09T09:12:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:12 smithi181 bash[27712]: cluster 2023-10-09T09:12:11.268303+0000 mgr.y (mgr.34107) 107 : cluster [DBG] pgmap v245: 89 pgs: 89 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 727 KiB/s wr, 6 op/s 2023-10-09T09:12:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:12 smithi181 bash[27712]: cluster 2023-10-09T09:12:12.045459+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2023-10-09T09:12:12.759 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T09:12:13.052 INFO:tasks.rados.rados.0.smithi163.stdout:1617: done (0 left) 2023-10-09T09:12:13.052 INFO:tasks.rados.rados.0.smithi163.stdout:1618: read oid 34 snap -1 2023-10-09T09:12:13.052 INFO:tasks.rados.rados.0.smithi163.stdout:1618: expect (ObjNum 554 snap 171 seq_num 554) 2023-10-09T09:12:13.052 INFO:tasks.rados.rados.0.smithi163.stdout:1619: read oid 35 snap -1 2023-10-09T09:12:13.052 INFO:tasks.rados.rados.0.smithi163.stdout:1619: expect (ObjNum 540 snap 169 seq_num 540) 2023-10-09T09:12:13.052 INFO:tasks.rados.rados.0.smithi163.stdout:1620: read oid 30 snap -1 2023-10-09T09:12:13.052 INFO:tasks.rados.rados.0.smithi163.stdout:1620: expect deleted 2023-10-09T09:12:13.052 INFO:tasks.rados.rados.0.smithi163.stdout:1621: delete oid 14 current snap is 197 2023-10-09T09:12:13.066 INFO:tasks.rados.rados.0.smithi163.stdout:1620: done (3 left) 2023-10-09T09:12:13.066 INFO:tasks.rados.rados.0.smithi163.stdout:1621: done (2 left) 2023-10-09T09:12:13.066 INFO:tasks.rados.rados.0.smithi163.stdout:1622: delete oid 20 current snap is 197 2023-10-09T09:12:13.071 INFO:tasks.rados.rados.0.smithi163.stdout:1622: done (2 left) 2023-10-09T09:12:13.071 INFO:tasks.rados.rados.0.smithi163.stdout:1623: read oid 44 snap -1 2023-10-09T09:12:13.071 INFO:tasks.rados.rados.0.smithi163.stdout:1623: expect deleted 2023-10-09T09:12:13.071 INFO:tasks.rados.rados.0.smithi163.stdout:1624: read oid 40 snap -1 2023-10-09T09:12:13.072 INFO:tasks.rados.rados.0.smithi163.stdout:1624: expect (ObjNum 537 snap 168 seq_num 537) 2023-10-09T09:12:13.072 INFO:tasks.rados.rados.0.smithi163.stdout:1625: write oid 14 current snap is 197 2023-10-09T09:12:13.072 INFO:tasks.rados.rados.0.smithi163.stdout:1625: seq_num 622 ranges {453954=629471,1598388=515881} 2023-10-09T09:12:13.078 INFO:tasks.rados.rados.0.smithi163.stdout:1625: writing smithi16367185-14 from 453954 to 1083425 tid 1 2023-10-09T09:12:13.085 INFO:tasks.rados.rados.0.smithi163.stdout:1625: writing smithi16367185-14 from 1598388 to 2114269 tid 2 2023-10-09T09:12:13.085 INFO:tasks.rados.rados.0.smithi163.stdout:1626: delete oid 6 current snap is 197 2023-10-09T09:12:13.085 INFO:tasks.rados.rados.0.smithi163.stdout:1618: expect (ObjNum 554 snap 171 seq_num 554) 2023-10-09T09:12:13.249 INFO:tasks.rados.rados.0.smithi163.stdout:1619: expect (ObjNum 540 snap 169 seq_num 540) 2023-10-09T09:12:13.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:13 smithi181 bash[27712]: debug 2023-10-09T09:12:13.037+0000 7fde83035700 -1 mon.b@1(peon).osd e465 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:13.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:13 smithi181 bash[27712]: cluster 2023-10-09T09:12:12.041400+0000 osd.3 (osd.3) 113 : cluster [WRN] failed to encode map e464 with expected crc 2023-10-09T09:12:13.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:13 smithi181 bash[27712]: cluster 2023-10-09T09:12:12.047366+0000 osd.4 (osd.4) 102 : cluster [WRN] failed to encode map e464 with expected crc 2023-10-09T09:12:13.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:13 smithi181 bash[27712]: cluster 2023-10-09T09:12:13.044306+0000 mon.a (mon.0) 304 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2023-10-09T09:12:13.394 INFO:tasks.rados.rados.0.smithi163.stdout:1624: expect (ObjNum 537 snap 168 seq_num 537) 2023-10-09T09:12:13.500 INFO:tasks.rados.rados.0.smithi163.stdout:1625: finishing write tid 1 to smithi16367185-14 2023-10-09T09:12:13.500 INFO:tasks.rados.rados.0.smithi163.stdout:1625: finishing write tid 2 to smithi16367185-14 2023-10-09T09:12:13.500 INFO:tasks.rados.rados.0.smithi163.stdout:1625: finishing write tid 3 to smithi16367185-14 2023-10-09T09:12:13.500 INFO:tasks.rados.rados.0.smithi163.stdout:1625: finishing write tid 4 to smithi16367185-14 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 673 (ObjNum 622 snap 197 seq_num 622) dirty exists 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:1625: left oid 14 (ObjNum 622 snap 197 seq_num 622) 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:1618: done (5 left) 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:1619: done (4 left) 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:1623: done (3 left) 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:1624: done (2 left) 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:1625: done (1 left) 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:1626: done (0 left) 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:1627: read oid 36 snap 196 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:1627: expect deleted 2023-10-09T09:12:13.501 INFO:tasks.rados.rados.0.smithi163.stdout:1628: read oid 50 snap -1 2023-10-09T09:12:13.502 INFO:tasks.rados.rados.0.smithi163.stdout:1628: expect (ObjNum 562 snap 172 seq_num 562) 2023-10-09T09:12:13.502 INFO:tasks.rados.rados.0.smithi163.stdout:1629: snap_create 2023-10-09T09:12:13.514 INFO:tasks.rados.rados.0.smithi163.stdout:1628: expect (ObjNum 562 snap 172 seq_num 562) 2023-10-09T09:12:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:12:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:12:13] "GET /metrics HTTP/1.1" 200 36083 "" "Prometheus/2.33.4" 2023-10-09T09:12:14.240 INFO:tasks.rados.rados.0.smithi163.stdout:1627: done (2 left) 2023-10-09T09:12:14.240 INFO:tasks.rados.rados.0.smithi163.stdout:1628: done (1 left) 2023-10-09T09:12:14.240 INFO:tasks.rados.rados.0.smithi163.stdout:1629: done (0 left) 2023-10-09T09:12:14.240 INFO:tasks.rados.rados.0.smithi163.stdout:1630: read oid 48 snap -1 2023-10-09T09:12:14.241 INFO:tasks.rados.rados.0.smithi163.stdout:1630: expect deleted 2023-10-09T09:12:14.241 INFO:tasks.rados.rados.0.smithi163.stdout:1631: write oid 22 current snap is 198 2023-10-09T09:12:14.241 INFO:tasks.rados.rados.0.smithi163.stdout:1631: seq_num 624 ranges {487568=496322,1431705=519550} 2023-10-09T09:12:14.245 INFO:tasks.rados.rados.0.smithi163.stdout:1631: writing smithi16367185-22 from 487568 to 983890 tid 1 2023-10-09T09:12:14.252 INFO:tasks.rados.rados.0.smithi163.stdout:1631: writing smithi16367185-22 from 1431705 to 1951255 tid 2 2023-10-09T09:12:14.252 INFO:tasks.rados.rados.0.smithi163.stdout:1632: snap_remove snap 196 2023-10-09T09:12:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:14 smithi181 bash[27712]: debug 2023-10-09T09:12:14.229+0000 7fde83035700 -1 mon.b@1(peon).osd e466 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:14.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:14 smithi181 bash[27712]: cluster 2023-10-09T09:12:13.042238+0000 osd.3 (osd.3) 114 : cluster [WRN] failed to encode map e465 with expected crc 2023-10-09T09:12:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:14 smithi181 bash[27712]: cluster 2023-10-09T09:12:13.047938+0000 osd.5 (osd.5) 64 : cluster [WRN] failed to encode map e465 with expected crc 2023-10-09T09:12:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:14 smithi181 bash[27712]: cluster 2023-10-09T09:12:13.052257+0000 osd.4 (osd.4) 103 : cluster [WRN] failed to encode map e465 with expected crc 2023-10-09T09:12:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:14 smithi181 bash[27712]: cluster 2023-10-09T09:12:13.052280+0000 osd.7 (osd.7) 79 : cluster [WRN] failed to encode map e465 with expected crc 2023-10-09T09:12:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:14 smithi181 bash[27712]: cluster 2023-10-09T09:12:13.053635+0000 osd.2 (osd.2) 109 : cluster [WRN] failed to encode map e465 with expected crc 2023-10-09T09:12:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:14 smithi181 bash[27712]: cluster 2023-10-09T09:12:13.065599+0000 osd.6 (osd.6) 50 : cluster [WRN] failed to encode map e465 with expected crc 2023-10-09T09:12:14.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:14 smithi181 bash[27712]: cluster 2023-10-09T09:12:13.268753+0000 mgr.y (mgr.34107) 108 : cluster [DBG] pgmap v248: 89 pgs: 89 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 727 KiB/s wr, 6 op/s 2023-10-09T09:12:14.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:14 smithi181 bash[27712]: audit 2023-10-09T09:12:13.660596+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:15.035 INFO:tasks.rados.rados.0.smithi163.stdout:1632: done (2 left) 2023-10-09T09:12:15.035 INFO:tasks.rados.rados.0.smithi163.stdout:1633: read oid 28 snap -1 2023-10-09T09:12:15.035 INFO:tasks.rados.rados.0.smithi163.stdout:1633: expect deleted 2023-10-09T09:12:15.035 INFO:tasks.rados.rados.0.smithi163.stdout:1634: snap_remove snap 197 2023-10-09T09:12:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:15 smithi181 bash[27712]: debug 2023-10-09T09:12:15.021+0000 7fde83035700 -1 mon.b@1(peon).osd e467 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:15 smithi181 bash[27712]: cluster 2023-10-09T09:12:14.233651+0000 osd.3 (osd.3) 115 : cluster [WRN] failed to encode map e466 with expected crc 2023-10-09T09:12:15.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:15 smithi181 bash[27712]: cluster 2023-10-09T09:12:14.236322+0000 osd.1 (osd.1) 50 : cluster [WRN] failed to encode map e466 with expected crc 2023-10-09T09:12:15.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:15 smithi181 bash[27712]: cluster 2023-10-09T09:12:14.239927+0000 mon.a (mon.0) 306 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2023-10-09T09:12:15.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:15 smithi181 bash[27712]: cluster 2023-10-09T09:12:14.240050+0000 osd.5 (osd.5) 65 : cluster [WRN] failed to encode map e466 with expected crc 2023-10-09T09:12:15.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:15 smithi181 bash[27712]: cluster 2023-10-09T09:12:15.027211+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2023-10-09T09:12:15.761 INFO:teuthology.orchestra.run.smithi163.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:16.052 INFO:tasks.rados.rados.0.smithi163.stdout:1634: done (3 left) 2023-10-09T09:12:16.052 INFO:tasks.rados.rados.0.smithi163.stdout:1635: delete oid 41 current snap is 198 2023-10-09T09:12:16.052 INFO:tasks.rados.rados.0.smithi163.stdout:1631: finishing write tid 1 to smithi16367185-22 2023-10-09T09:12:16.052 INFO:tasks.rados.rados.0.smithi163.stdout:1631: finishing write tid 2 to smithi16367185-22 2023-10-09T09:12:16.052 INFO:tasks.rados.rados.0.smithi163.stdout:1631: finishing write tid 3 to smithi16367185-22 2023-10-09T09:12:16.052 INFO:tasks.rados.rados.0.smithi163.stdout:1631: finishing write tid 4 to smithi16367185-22 2023-10-09T09:12:16.052 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 1276 (ObjNum 624 snap 198 seq_num 624) dirty exists 2023-10-09T09:12:16.053 INFO:tasks.rados.rados.0.smithi163.stdout:1631: left oid 22 (ObjNum 624 snap 198 seq_num 624) 2023-10-09T09:12:16.058 INFO:tasks.rados.rados.0.smithi163.stdout:1630: done (3 left) 2023-10-09T09:12:16.058 INFO:tasks.rados.rados.0.smithi163.stdout:1631: done (2 left) 2023-10-09T09:12:16.058 INFO:tasks.rados.rados.0.smithi163.stdout:1633: done (1 left) 2023-10-09T09:12:16.058 INFO:tasks.rados.rados.0.smithi163.stdout:1635: done (0 left) 2023-10-09T09:12:16.059 INFO:tasks.rados.rados.0.smithi163.stdout:1636: read oid 37 snap -1 2023-10-09T09:12:16.059 INFO:tasks.rados.rados.0.smithi163.stdout:1636: expect (ObjNum 618 snap 196 seq_num 618) 2023-10-09T09:12:16.059 INFO:tasks.rados.rados.0.smithi163.stdout:1637: snap_create 2023-10-09T09:12:16.074 INFO:tasks.rados.rados.0.smithi163.stdout:1636: expect (ObjNum 618 snap 196 seq_num 618) 2023-10-09T09:12:16.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:16 smithi181 bash[27712]: debug 2023-10-09T09:12:16.033+0000 7fde83035700 -1 mon.b@1(peon).osd e468 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:16.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:16 smithi181 bash[27712]: cluster 2023-10-09T09:12:15.024726+0000 osd.3 (osd.3) 116 : cluster [WRN] failed to encode map e467 with expected crc 2023-10-09T09:12:16.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:16 smithi181 bash[27712]: cluster 2023-10-09T09:12:15.031389+0000 osd.4 (osd.4) 104 : cluster [WRN] failed to encode map e467 with expected crc 2023-10-09T09:12:16.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:16 smithi181 bash[27712]: cluster 2023-10-09T09:12:15.040420+0000 osd.2 (osd.2) 110 : cluster [WRN] failed to encode map e467 with expected crc 2023-10-09T09:12:16.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:16 smithi181 bash[27712]: cluster 2023-10-09T09:12:15.269463+0000 mgr.y (mgr.34107) 109 : cluster [DBG] pgmap v251: 89 pgs: 89 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:16.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:16 smithi181 bash[27712]: audit 2023-10-09T09:12:15.760518+0000 mon.a (mon.0) 308 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:16.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:16 smithi181 bash[27712]: cluster 2023-10-09T09:12:16.042584+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2023-10-09T09:12:16.534 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T09:12:17.053 INFO:tasks.rados.rados.0.smithi163.stdout:1636: done (1 left) 2023-10-09T09:12:17.053 INFO:tasks.rados.rados.0.smithi163.stdout:1637: done (0 left) 2023-10-09T09:12:17.053 INFO:tasks.rados.rados.0.smithi163.stdout:1638: rollback oid 21 current snap is 199 2023-10-09T09:12:17.054 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 198 2023-10-09T09:12:17.054 INFO:tasks.rados.rados.0.smithi163.stdout:1639: rollback oid 48 current snap is 199 2023-10-09T09:12:17.054 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 48 to 198 2023-10-09T09:12:17.054 INFO:tasks.rados.rados.0.smithi163.stdout:1640: snap_create 2023-10-09T09:12:17.059 INFO:tasks.rados.rados.0.smithi163.stdout:1639: finishing rollback tid 1 to smithi16367185-48 2023-10-09T09:12:17.059 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 0 (ObjNum 698865760 snap 21875 seq_num 698997136) dirty dne 2023-10-09T09:12:17.061 INFO:tasks.rados.rados.0.smithi163.stdout:1638: finishing rollback tid 0 to smithi16367185-21 2023-10-09T09:12:17.062 INFO:tasks.rados.rados.0.smithi163.stdout:1638: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:12:17.062 INFO:tasks.rados.rados.0.smithi163.stdout:1638: finishing rollback tid 2 to smithi16367185-21 2023-10-09T09:12:17.062 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 418 (ObjNum 576 snap 181 seq_num 576) dirty exists 2023-10-09T09:12:17.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:17 smithi181 bash[27712]: debug 2023-10-09T09:12:17.037+0000 7fde83035700 -1 mon.b@1(peon).osd e469 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:17.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:17 smithi181 bash[27712]: audit 2023-10-09T09:12:15.753027+0000 mgr.y (mgr.34107) 110 : audit [DBG] from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:17.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:17 smithi181 bash[27712]: cephadm 2023-10-09T09:12:15.753766+0000 mgr.y (mgr.34107) 111 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:17.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:17 smithi181 bash[27712]: cluster 2023-10-09T09:12:16.039432+0000 osd.3 (osd.3) 117 : cluster [WRN] failed to encode map e468 with expected crc 2023-10-09T09:12:17.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:17 smithi181 bash[27712]: cluster 2023-10-09T09:12:16.047260+0000 osd.4 (osd.4) 105 : cluster [WRN] failed to encode map e468 with expected crc 2023-10-09T09:12:17.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:17 smithi181 bash[27712]: cluster 2023-10-09T09:12:16.053196+0000 osd.2 (osd.2) 111 : cluster [WRN] failed to encode map e468 with expected crc 2023-10-09T09:12:17.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:17 smithi181 bash[27712]: cluster 2023-10-09T09:12:17.044778+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2023-10-09T09:12:18.053 INFO:tasks.rados.rados.0.smithi163.stdout:1638: done (2 left) 2023-10-09T09:12:18.054 INFO:tasks.rados.rados.0.smithi163.stdout:1639: done (1 left) 2023-10-09T09:12:18.054 INFO:tasks.rados.rados.0.smithi163.stdout:1640: done (0 left) 2023-10-09T09:12:18.054 INFO:tasks.rados.rados.0.smithi163.stdout:1641: write oid 40 current snap is 200 2023-10-09T09:12:18.054 INFO:tasks.rados.rados.0.smithi163.stdout:1641: seq_num 626 ranges {521182=646820,1948669=523219} 2023-10-09T09:12:18.060 INFO:tasks.rados.rados.0.smithi163.stdout:1641: writing smithi16367185-40 from 521182 to 1168002 tid 1 2023-10-09T09:12:18.067 INFO:tasks.rados.rados.0.smithi163.stdout:1641: writing smithi16367185-40 from 1948669 to 2471888 tid 2 2023-10-09T09:12:18.067 INFO:tasks.rados.rados.0.smithi163.stdout:1642: rollback oid 8 current snap is 200 2023-10-09T09:12:18.067 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 8 to 198 2023-10-09T09:12:18.068 INFO:tasks.rados.rados.0.smithi163.stdout:1643: write oid 47 current snap is 200 2023-10-09T09:12:18.068 INFO:tasks.rados.rados.0.smithi163.stdout:1643: seq_num 627 ranges {537989=722069,1890798=647191} 2023-10-09T09:12:18.075 INFO:tasks.rados.rados.0.smithi163.stdout:1643: writing smithi16367185-47 from 537989 to 1260058 tid 1 2023-10-09T09:12:18.083 INFO:tasks.rados.rados.0.smithi163.stdout:1643: writing smithi16367185-47 from 1890798 to 2537989 tid 2 2023-10-09T09:12:18.083 INFO:tasks.rados.rados.0.smithi163.stdout:1642: finishing rollback tid 1 to smithi16367185-8 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 0 (ObjNum 698859824 snap 21875 seq_num 699228656) dirty dne 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:1644: read oid 7 snap -1 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:1644: expect deleted 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:1642: done (3 left) 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:1645: write oid 21 current snap is 200 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:1641: finishing write tid 1 to smithi16367185-40 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:1641: finishing write tid 2 to smithi16367185-40 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:1641: finishing write tid 3 to smithi16367185-40 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:1641: finishing write tid 4 to smithi16367185-40 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 835 (ObjNum 626 snap 200 seq_num 626) dirty exists 2023-10-09T09:12:18.084 INFO:tasks.rados.rados.0.smithi163.stdout:1641: left oid 40 (ObjNum 626 snap 200 seq_num 626) 2023-10-09T09:12:18.085 INFO:tasks.rados.rados.0.smithi163.stdout:1643: finishing write tid 1 to smithi16367185-47 2023-10-09T09:12:18.085 INFO:tasks.rados.rados.0.smithi163.stdout:1645: seq_num 628 ranges {554796=797318,1949280=526888} 2023-10-09T09:12:18.091 INFO:tasks.rados.rados.0.smithi163.stdout:1645: writing smithi16367185-21 from 554796 to 1352114 tid 1 2023-10-09T09:12:18.098 INFO:tasks.rados.rados.0.smithi163.stdout:1645: writing smithi16367185-21 from 1949280 to 2476168 tid 2 2023-10-09T09:12:18.098 INFO:tasks.rados.rados.0.smithi163.stdout:1641: done (3 left) 2023-10-09T09:12:18.098 INFO:tasks.rados.rados.0.smithi163.stdout:1646: write oid 42 current snap is 200 2023-10-09T09:12:18.098 INFO:tasks.rados.rados.0.smithi163.stdout:1646: seq_num 629 ranges {571603=472567,1607762=670546} 2023-10-09T09:12:18.104 INFO:tasks.rados.rados.0.smithi163.stdout:1646: writing smithi16367185-42 from 571603 to 1044170 tid 1 2023-10-09T09:12:18.111 INFO:tasks.rados.rados.0.smithi163.stdout:1646: writing smithi16367185-42 from 1607762 to 2278308 tid 2 2023-10-09T09:12:18.111 INFO:tasks.rados.rados.0.smithi163.stdout:1647: read oid 35 snap -1 2023-10-09T09:12:18.111 INFO:tasks.rados.rados.0.smithi163.stdout:1647: expect (ObjNum 540 snap 169 seq_num 540) 2023-10-09T09:12:18.112 INFO:tasks.rados.rados.0.smithi163.stdout:1648: snap_remove snap 199 2023-10-09T09:12:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:18 smithi181 bash[27712]: debug 2023-10-09T09:12:18.041+0000 7fde83035700 -1 mon.b@1(peon).osd e470 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:18 smithi181 bash[27712]: cluster 2023-10-09T09:12:16.710741+0000 osd.6 (osd.6) 51 : cluster [WRN] failed to encode map e468 with expected crc 2023-10-09T09:12:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:18 smithi181 bash[27712]: cluster 2023-10-09T09:12:17.042797+0000 osd.3 (osd.3) 118 : cluster [WRN] failed to encode map e469 with expected crc 2023-10-09T09:12:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:18 smithi181 bash[27712]: cluster 2023-10-09T09:12:17.049271+0000 osd.6 (osd.6) 52 : cluster [WRN] failed to encode map e469 with expected crc 2023-10-09T09:12:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:18 smithi181 bash[27712]: cluster 2023-10-09T09:12:17.053274+0000 osd.5 (osd.5) 66 : cluster [WRN] failed to encode map e469 with expected crc 2023-10-09T09:12:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:18 smithi181 bash[27712]: cluster 2023-10-09T09:12:17.055014+0000 osd.1 (osd.1) 51 : cluster [WRN] failed to encode map e469 with expected crc 2023-10-09T09:12:18.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:18 smithi181 bash[27712]: cluster 2023-10-09T09:12:17.270107+0000 mgr.y (mgr.34107) 112 : cluster [DBG] pgmap v254: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 528 KiB/s wr, 14 op/s 2023-10-09T09:12:18.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:18 smithi181 bash[27712]: cluster 2023-10-09T09:12:18.048331+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2023-10-09T09:12:19.045 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 09:12:18 smithi181 bash[54907]: ts=2023-10-09T09:12:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:12:19.052 INFO:tasks.rados.rados.0.smithi163.stdout:1648: done (5 left) 2023-10-09T09:12:19.052 INFO:tasks.rados.rados.0.smithi163.stdout:1649: rollback oid 9 current snap is 200 2023-10-09T09:12:19.052 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 198 2023-10-09T09:12:19.052 INFO:tasks.rados.rados.0.smithi163.stdout:1643: finishing write tid 2 to smithi16367185-47 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:1643: finishing write tid 3 to smithi16367185-47 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:1643: finishing write tid 4 to smithi16367185-47 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 1284 (ObjNum 627 snap 200 seq_num 627) dirty exists 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:1643: left oid 47 (ObjNum 627 snap 200 seq_num 627) 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:1645: finishing write tid 1 to smithi16367185-21 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:1645: finishing write tid 2 to smithi16367185-21 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:1645: finishing write tid 3 to smithi16367185-21 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:1645: finishing write tid 4 to smithi16367185-21 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 422 (ObjNum 628 snap 200 seq_num 628) dirty exists 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:1645: left oid 21 (ObjNum 628 snap 200 seq_num 628) 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:1646: finishing write tid 1 to smithi16367185-42 2023-10-09T09:12:19.053 INFO:tasks.rados.rados.0.smithi163.stdout:1646: finishing write tid 2 to smithi16367185-42 2023-10-09T09:12:19.054 INFO:tasks.rados.rados.0.smithi163.stdout:1646: finishing write tid 3 to smithi16367185-42 2023-10-09T09:12:19.054 INFO:tasks.rados.rados.0.smithi163.stdout:1646: finishing write tid 4 to smithi16367185-42 2023-10-09T09:12:19.054 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 1288 (ObjNum 629 snap 200 seq_num 629) dirty exists 2023-10-09T09:12:19.054 INFO:tasks.rados.rados.0.smithi163.stdout:1646: left oid 42 (ObjNum 629 snap 200 seq_num 629) 2023-10-09T09:12:19.054 INFO:tasks.rados.rados.0.smithi163.stdout:1647: expect (ObjNum 540 snap 169 seq_num 540) 2023-10-09T09:12:19.202 INFO:tasks.rados.rados.0.smithi163.stdout:1649: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:12:19.202 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 0 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:12:19.202 INFO:tasks.rados.rados.0.smithi163.stdout:1643: done (5 left) 2023-10-09T09:12:19.202 INFO:tasks.rados.rados.0.smithi163.stdout:1644: done (4 left) 2023-10-09T09:12:19.202 INFO:tasks.rados.rados.0.smithi163.stdout:1645: done (3 left) 2023-10-09T09:12:19.203 INFO:tasks.rados.rados.0.smithi163.stdout:1646: done (2 left) 2023-10-09T09:12:19.203 INFO:tasks.rados.rados.0.smithi163.stdout:1647: done (1 left) 2023-10-09T09:12:19.203 INFO:tasks.rados.rados.0.smithi163.stdout:1649: done (0 left) 2023-10-09T09:12:19.203 INFO:tasks.rados.rados.0.smithi163.stdout:1650: read oid 26 snap -1 2023-10-09T09:12:19.203 INFO:tasks.rados.rados.0.smithi163.stdout:1650: expect (ObjNum 594 snap 187 seq_num 594) 2023-10-09T09:12:19.203 INFO:tasks.rados.rados.0.smithi163.stdout:1651: snap_create 2023-10-09T09:12:19.211 INFO:tasks.rados.rados.0.smithi163.stdout:1650: expect (ObjNum 594 snap 187 seq_num 594) 2023-10-09T09:12:19.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:19 smithi181 bash[27712]: debug 2023-10-09T09:12:19.040+0000 7fde83035700 -1 mon.b@1(peon).osd e471 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:19.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:19 smithi181 bash[27712]: cluster 2023-10-09T09:12:18.046316+0000 osd.3 (osd.3) 119 : cluster [WRN] failed to encode map e470 with expected crc 2023-10-09T09:12:19.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:19 smithi181 bash[27712]: cluster 2023-10-09T09:12:18.051122+0000 osd.1 (osd.1) 52 : cluster [WRN] failed to encode map e470 with expected crc 2023-10-09T09:12:19.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:19 smithi181 bash[27712]: cluster 2023-10-09T09:12:18.061698+0000 osd.7 (osd.7) 80 : cluster [WRN] failed to encode map e470 with expected crc 2023-10-09T09:12:19.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:19 smithi181 bash[27712]: cluster 2023-10-09T09:12:19.051932+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2023-10-09T09:12:19.319 INFO:teuthology.orchestra.run.smithi163.stdout:wait for all mons to upgrade 2023-10-09T09:12:20.042 INFO:tasks.rados.rados.0.smithi163.stdout:1650: done (1 left) 2023-10-09T09:12:20.043 INFO:tasks.rados.rados.0.smithi163.stdout:1651: done (0 left) 2023-10-09T09:12:20.043 INFO:tasks.rados.rados.0.smithi163.stdout:1652: delete oid 27 current snap is 201 2023-10-09T09:12:20.051 INFO:tasks.rados.rados.0.smithi163.stdout:1652: done (0 left) 2023-10-09T09:12:20.051 INFO:tasks.rados.rados.0.smithi163.stdout:1653: delete oid 15 current snap is 201 2023-10-09T09:12:20.055 INFO:tasks.rados.rados.0.smithi163.stdout:1653: done (0 left) 2023-10-09T09:12:20.055 INFO:tasks.rados.rados.0.smithi163.stdout:1654: snap_remove snap 200 2023-10-09T09:12:20.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T09:12:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: debug 2023-10-09T09:12:20.028+0000 7fde83035700 -1 mon.b@1(peon).osd e472 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: cluster 2023-10-09T09:12:19.046683+0000 osd.3 (osd.3) 120 : cluster [WRN] failed to encode map e471 with expected crc 2023-10-09T09:12:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: cluster 2023-10-09T09:12:19.048140+0000 osd.4 (osd.4) 106 : cluster [WRN] failed to encode map e471 with expected crc 2023-10-09T09:12:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: cluster 2023-10-09T09:12:19.203874+0000 osd.2 (osd.2) 112 : cluster [WRN] failed to encode map e471 with expected crc 2023-10-09T09:12:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: cluster 2023-10-09T09:12:19.270650+0000 mgr.y (mgr.34107) 113 : cluster [DBG] pgmap v257: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 528 KiB/s wr, 14 op/s 2023-10-09T09:12:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: cluster 2023-10-09T09:12:19.428518+0000 osd.5 (osd.5) 67 : cluster [WRN] failed to encode map e471 with expected crc 2023-10-09T09:12:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: audit 2023-10-09T09:12:19.953388+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: audit 2023-10-09T09:12:19.961237+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: cluster 2023-10-09T09:12:20.034096+0000 mon.a (mon.0) 315 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2023-10-09T09:12:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: audit 2023-10-09T09:12:20.055277+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:20.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:20 smithi181 bash[27712]: audit 2023-10-09T09:12:20.061149+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:21.067 INFO:tasks.rados.rados.0.smithi163.stdout:1654: done (0 left) 2023-10-09T09:12:21.067 INFO:tasks.rados.rados.0.smithi163.stdout:1655: read oid 37 snap 198 2023-10-09T09:12:21.068 INFO:tasks.rados.rados.0.smithi163.stdout:1655: expect (ObjNum 618 snap 196 seq_num 618) 2023-10-09T09:12:21.068 INFO:tasks.rados.rados.0.smithi163.stdout:1656: rollback oid 14 current snap is 201 2023-10-09T09:12:21.068 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 198 2023-10-09T09:12:21.068 INFO:tasks.rados.rados.0.smithi163.stdout:1657: write oid 30 current snap is 201 2023-10-09T09:12:21.068 INFO:tasks.rados.rados.0.smithi163.stdout:1657: seq_num 632 ranges {622024=414667,1783208=417873} 2023-10-09T09:12:21.076 INFO:tasks.rados.rados.0.smithi163.stdout:1657: writing smithi16367185-30 from 622024 to 1036691 tid 1 2023-10-09T09:12:21.085 INFO:tasks.rados.rados.0.smithi163.stdout:1657: writing smithi16367185-30 from 1783208 to 2201081 tid 2 2023-10-09T09:12:21.085 INFO:tasks.rados.rados.0.smithi163.stdout:1658: delete oid 9 current snap is 201 2023-10-09T09:12:21.085 INFO:tasks.rados.rados.0.smithi163.stdout:1656: finishing rollback tid 0 to smithi16367185-14 2023-10-09T09:12:21.086 INFO:tasks.rados.rados.0.smithi163.stdout:1656: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:12:21.086 INFO:tasks.rados.rados.0.smithi163.stdout:1656: finishing rollback tid 2 to smithi16367185-14 2023-10-09T09:12:21.086 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 679 (ObjNum 622 snap 197 seq_num 622) dirty exists 2023-10-09T09:12:21.086 INFO:tasks.rados.rados.0.smithi163.stdout:1655: expect (ObjNum 618 snap 196 seq_num 618) 2023-10-09T09:12:21.200 INFO:tasks.rados.rados.0.smithi163.stdout:1657: finishing write tid 1 to smithi16367185-30 2023-10-09T09:12:21.201 INFO:tasks.rados.rados.0.smithi163.stdout:1657: finishing write tid 2 to smithi16367185-30 2023-10-09T09:12:21.201 INFO:tasks.rados.rados.0.smithi163.stdout:1655: done (3 left) 2023-10-09T09:12:21.201 INFO:tasks.rados.rados.0.smithi163.stdout:1656: done (2 left) 2023-10-09T09:12:21.201 INFO:tasks.rados.rados.0.smithi163.stdout:1658: done (1 left) 2023-10-09T09:12:21.201 INFO:tasks.rados.rados.0.smithi163.stdout:1659: snap_create 2023-10-09T09:12:21.201 INFO:tasks.rados.rados.0.smithi163.stdout:1657: finishing write tid 3 to smithi16367185-30 2023-10-09T09:12:21.201 INFO:tasks.rados.rados.0.smithi163.stdout:1657: finishing write tid 4 to smithi16367185-30 2023-10-09T09:12:21.201 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 637 (ObjNum 632 snap 201 seq_num 632) dirty exists 2023-10-09T09:12:21.201 INFO:tasks.rados.rados.0.smithi163.stdout:1657: left oid 30 (ObjNum 632 snap 201 seq_num 632) 2023-10-09T09:12:21.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: debug 2023-10-09T09:12:21.052+0000 7fde83035700 -1 mon.b@1(peon).osd e473 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:21.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: cluster 2023-10-09T09:12:19.477244+0000 osd.7 (osd.7) 81 : cluster [WRN] failed to encode map e471 with expected crc 2023-10-09T09:12:21.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: cluster 2023-10-09T09:12:20.032171+0000 osd.3 (osd.3) 121 : cluster [WRN] failed to encode map e472 with expected crc 2023-10-09T09:12:21.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: cluster 2023-10-09T09:12:20.041037+0000 osd.0 (osd.0) 74 : cluster [WRN] failed to encode map e472 with expected crc 2023-10-09T09:12:21.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: cluster 2023-10-09T09:12:20.045862+0000 osd.2 (osd.2) 113 : cluster [WRN] failed to encode map e472 with expected crc 2023-10-09T09:12:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: audit 2023-10-09T09:12:20.542570+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: audit 2023-10-09T09:12:20.543809+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: audit 2023-10-09T09:12:20.555463+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: audit 2023-10-09T09:12:20.636922+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: audit 2023-10-09T09:12:20.640693+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:12:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: audit 2023-10-09T09:12:20.642597+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:12:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: audit 2023-10-09T09:12:20.643990+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T09:12:21.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:21 smithi181 bash[27712]: cluster 2023-10-09T09:12:21.059986+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2023-10-09T09:12:22.069 INFO:tasks.rados.rados.0.smithi163.stdout:1657: done (1 left) 2023-10-09T09:12:22.069 INFO:tasks.rados.rados.0.smithi163.stdout:1659: done (0 left) 2023-10-09T09:12:22.070 INFO:tasks.rados.rados.0.smithi163.stdout:1660: write oid 33 current snap is 202 2023-10-09T09:12:22.070 INFO:tasks.rados.rados.0.smithi163.stdout:1660: seq_num 634 ranges {655638=565165,1900172=421542} 2023-10-09T09:12:22.077 INFO:tasks.rados.rados.0.smithi163.stdout:1660: writing smithi16367185-33 from 655638 to 1220803 tid 1 2023-10-09T09:12:22.085 INFO:tasks.rados.rados.0.smithi163.stdout:1660: writing smithi16367185-33 from 1900172 to 2321714 tid 2 2023-10-09T09:12:22.085 INFO:tasks.rados.rados.0.smithi163.stdout:1661: delete oid 15 current snap is 202 2023-10-09T09:12:22.092 INFO:tasks.rados.rados.0.smithi163.stdout:1660: finishing write tid 1 to smithi16367185-33 2023-10-09T09:12:22.092 INFO:tasks.rados.rados.0.smithi163.stdout:1661: done (1 left) 2023-10-09T09:12:22.093 INFO:tasks.rados.rados.0.smithi163.stdout:1662: write oid 49 current snap is 202 2023-10-09T09:12:22.093 INFO:tasks.rados.rados.0.smithi163.stdout:1662: seq_num 636 ranges {689252=715663,1900783=425211} 2023-10-09T09:12:22.102 INFO:tasks.rados.rados.0.smithi163.stdout:1662: writing smithi16367185-49 from 689252 to 1404915 tid 1 2023-10-09T09:12:22.108 INFO:tasks.rados.rados.0.smithi163.stdout:1662: writing smithi16367185-49 from 1900783 to 2325994 tid 2 2023-10-09T09:12:22.109 INFO:tasks.rados.rados.0.smithi163.stdout:1663: read oid 37 snap -1 2023-10-09T09:12:22.109 INFO:tasks.rados.rados.0.smithi163.stdout:1663: expect (ObjNum 618 snap 196 seq_num 618) 2023-10-09T09:12:22.109 INFO:tasks.rados.rados.0.smithi163.stdout:1664: write oid 50 current snap is 202 2023-10-09T09:12:22.109 INFO:tasks.rados.rados.0.smithi163.stdout:1660: finishing write tid 2 to smithi16367185-33 2023-10-09T09:12:22.109 INFO:tasks.rados.rados.0.smithi163.stdout:1660: finishing write tid 3 to smithi16367185-33 2023-10-09T09:12:22.109 INFO:tasks.rados.rados.0.smithi163.stdout:1660: finishing write tid 4 to smithi16367185-33 2023-10-09T09:12:22.109 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 749 (ObjNum 634 snap 202 seq_num 634) dirty exists 2023-10-09T09:12:22.109 INFO:tasks.rados.rados.0.smithi163.stdout:1660: left oid 33 (ObjNum 634 snap 202 seq_num 634) 2023-10-09T09:12:22.109 INFO:tasks.rados.rados.0.smithi163.stdout:1664: seq_num 637 ranges {706059=790912,1959265=685222} 2023-10-09T09:12:22.117 INFO:tasks.rados.rados.0.smithi163.stdout:1664: writing smithi16367185-50 from 706059 to 1496971 tid 1 2023-10-09T09:12:22.125 INFO:tasks.rados.rados.0.smithi163.stdout:1664: writing smithi16367185-50 from 1959265 to 2644487 tid 2 2023-10-09T09:12:22.125 INFO:tasks.rados.rados.0.smithi163.stdout:1660: done (3 left) 2023-10-09T09:12:22.125 INFO:tasks.rados.rados.0.smithi163.stdout:1665: delete oid 7 current snap is 202 2023-10-09T09:12:22.125 INFO:tasks.rados.rados.0.smithi163.stdout:1662: finishing write tid 1 to smithi16367185-49 2023-10-09T09:12:22.125 INFO:tasks.rados.rados.0.smithi163.stdout:1662: finishing write tid 2 to smithi16367185-49 2023-10-09T09:12:22.125 INFO:tasks.rados.rados.0.smithi163.stdout:1662: finishing write tid 3 to smithi16367185-49 2023-10-09T09:12:22.125 INFO:tasks.rados.rados.0.smithi163.stdout:1662: finishing write tid 4 to smithi16367185-49 2023-10-09T09:12:22.125 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 265 (ObjNum 636 snap 202 seq_num 636) dirty exists 2023-10-09T09:12:22.125 INFO:tasks.rados.rados.0.smithi163.stdout:1662: left oid 49 (ObjNum 636 snap 202 seq_num 636) 2023-10-09T09:12:22.126 INFO:tasks.rados.rados.0.smithi163.stdout:1663: expect (ObjNum 618 snap 196 seq_num 618) 2023-10-09T09:12:22.239 INFO:tasks.rados.rados.0.smithi163.stdout:1664: finishing write tid 1 to smithi16367185-50 2023-10-09T09:12:22.239 INFO:tasks.rados.rados.0.smithi163.stdout:1664: finishing write tid 2 to smithi16367185-50 2023-10-09T09:12:22.240 INFO:tasks.rados.rados.0.smithi163.stdout:1664: finishing write tid 3 to smithi16367185-50 2023-10-09T09:12:22.240 INFO:tasks.rados.rados.0.smithi163.stdout:1664: finishing write tid 4 to smithi16367185-50 2023-10-09T09:12:22.240 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 660 (ObjNum 637 snap 202 seq_num 637) dirty exists 2023-10-09T09:12:22.240 INFO:tasks.rados.rados.0.smithi163.stdout:1664: left oid 50 (ObjNum 637 snap 202 seq_num 637) 2023-10-09T09:12:22.240 INFO:tasks.rados.rados.0.smithi163.stdout:1662: done (3 left) 2023-10-09T09:12:22.240 INFO:tasks.rados.rados.0.smithi163.stdout:1663: done (2 left) 2023-10-09T09:12:22.240 INFO:tasks.rados.rados.0.smithi163.stdout:1664: done (1 left) 2023-10-09T09:12:22.240 INFO:tasks.rados.rados.0.smithi163.stdout:1665: done (0 left) 2023-10-09T09:12:22.240 INFO:tasks.rados.rados.0.smithi163.stdout:1666: snap_remove snap 201 2023-10-09T09:12:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: debug 2023-10-09T09:12:22.052+0000 7fde83035700 -1 mon.b@1(peon).osd e474 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:22.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: cluster 2023-10-09T09:12:20.641813+0000 osd.4 (osd.4) 107 : cluster [WRN] failed to encode map e472 with expected crc 2023-10-09T09:12:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: cephadm 2023-10-09T09:12:20.644847+0000 mgr.y (mgr.34107) 114 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-10-09T09:12:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: cluster 2023-10-09T09:12:20.708362+0000 osd.6 (osd.6) 53 : cluster [WRN] failed to encode map e472 with expected crc 2023-10-09T09:12:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: cluster 2023-10-09T09:12:20.949356+0000 osd.1 (osd.1) 53 : cluster [WRN] failed to encode map e472 with expected crc 2023-10-09T09:12:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: cluster 2023-10-09T09:12:21.056437+0000 osd.3 (osd.3) 122 : cluster [WRN] failed to encode map e473 with expected crc 2023-10-09T09:12:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: cluster 2023-10-09T09:12:21.063056+0000 osd.4 (osd.4) 108 : cluster [WRN] failed to encode map e473 with expected crc 2023-10-09T09:12:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: cluster 2023-10-09T09:12:21.067174+0000 osd.7 (osd.7) 82 : cluster [WRN] failed to encode map e473 with expected crc 2023-10-09T09:12:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: cluster 2023-10-09T09:12:21.078644+0000 osd.2 (osd.2) 114 : cluster [WRN] failed to encode map e473 with expected crc 2023-10-09T09:12:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: cluster 2023-10-09T09:12:21.271764+0000 mgr.y (mgr.34107) 115 : cluster [DBG] pgmap v260: 89 pgs: 89 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2023-10-09T09:12:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:22 smithi181 bash[27712]: cluster 2023-10-09T09:12:22.063781+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2023-10-09T09:12:23.072 INFO:tasks.rados.rados.0.smithi163.stdout:1666: done (0 left) 2023-10-09T09:12:23.073 INFO:tasks.rados.rados.0.smithi163.stdout:1667: rollback oid 23 current snap is 202 2023-10-09T09:12:23.073 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 23 to 198 2023-10-09T09:12:23.073 INFO:tasks.rados.rados.0.smithi163.stdout:1668: read oid 11 snap -1 2023-10-09T09:12:23.073 INFO:tasks.rados.rados.0.smithi163.stdout:1668: expect (ObjNum 607 snap 192 seq_num 607) 2023-10-09T09:12:23.073 INFO:tasks.rados.rados.0.smithi163.stdout:1669: delete oid 44 current snap is 202 2023-10-09T09:12:23.086 INFO:tasks.rados.rados.0.smithi163.stdout:1667: finishing rollback tid 0 to smithi16367185-23 2023-10-09T09:12:23.087 INFO:tasks.rados.rados.0.smithi163.stdout:1669: done (2 left) 2023-10-09T09:12:23.087 INFO:tasks.rados.rados.0.smithi163.stdout:1670: snap_create 2023-10-09T09:12:23.087 INFO:tasks.rados.rados.0.smithi163.stdout:1667: finishing rollback tid 1 to smithi16367185-23 2023-10-09T09:12:23.087 INFO:tasks.rados.rados.0.smithi163.stdout:1667: finishing rollback tid 2 to smithi16367185-23 2023-10-09T09:12:23.087 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 664 (ObjNum 545 snap 170 seq_num 545) dirty exists 2023-10-09T09:12:23.088 INFO:tasks.rados.rados.0.smithi163.stdout:1668: expect (ObjNum 607 snap 192 seq_num 607) 2023-10-09T09:12:23.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: debug 2023-10-09T09:12:23.056+0000 7fde83035700 -1 mon.b@1(peon).osd e475 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: cluster 2023-10-09T09:12:22.059044+0000 osd.3 (osd.3) 123 : cluster [WRN] failed to encode map e474 with expected crc 2023-10-09T09:12:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: cluster 2023-10-09T09:12:22.065238+0000 osd.4 (osd.4) 109 : cluster [WRN] failed to encode map e474 with expected crc 2023-10-09T09:12:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: cluster 2023-10-09T09:12:22.080092+0000 osd.2 (osd.2) 115 : cluster [WRN] failed to encode map e474 with expected crc 2023-10-09T09:12:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: cluster 2023-10-09T09:12:22.085430+0000 osd.7 (osd.7) 83 : cluster [WRN] failed to encode map e474 with expected crc 2023-10-09T09:12:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: cluster 2023-10-09T09:12:23.065898+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2023-10-09T09:12:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: audit 2023-10-09T09:12:23.092510+0000 mon.a (mon.0) 328 : audit [DBG] from='client.? 172.21.15.163:0/3016853494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: audit 2023-10-09T09:12:23.222393+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: audit 2023-10-09T09:12:23.223906+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: audit 2023-10-09T09:12:23.225052+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:23.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:23 smithi181 bash[27712]: audit 2023-10-09T09:12:23.226056+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:12:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:12:23] "GET /metrics HTTP/1.1" 200 36083 "" "Prometheus/2.33.4" 2023-10-09T09:12:24.075 INFO:tasks.rados.rados.0.smithi163.stdout:1667: done (2 left) 2023-10-09T09:12:24.075 INFO:tasks.rados.rados.0.smithi163.stdout:1668: done (1 left) 2023-10-09T09:12:24.075 INFO:tasks.rados.rados.0.smithi163.stdout:1670: done (0 left) 2023-10-09T09:12:24.075 INFO:tasks.rados.rados.0.smithi163.stdout:1671: snap_create 2023-10-09T09:12:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:24 smithi181 bash[27712]: debug 2023-10-09T09:12:24.060+0000 7fde83035700 -1 mon.b@1(peon).osd e476 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:24 smithi181 bash[27712]: cluster 2023-10-09T09:12:23.061456+0000 osd.3 (osd.3) 124 : cluster [WRN] failed to encode map e475 with expected crc 2023-10-09T09:12:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:24 smithi181 bash[27712]: cluster 2023-10-09T09:12:23.070317+0000 osd.2 (osd.2) 116 : cluster [WRN] failed to encode map e475 with expected crc 2023-10-09T09:12:24.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:24 smithi181 bash[27712]: cluster 2023-10-09T09:12:23.072585+0000 osd.7 (osd.7) 84 : cluster [WRN] failed to encode map e475 with expected crc 2023-10-09T09:12:24.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:24 smithi181 bash[27712]: cluster 2023-10-09T09:12:23.072693+0000 osd.5 (osd.5) 68 : cluster [WRN] failed to encode map e475 with expected crc 2023-10-09T09:12:24.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:24 smithi181 bash[27712]: cephadm 2023-10-09T09:12:23.214488+0000 mgr.y (mgr.34107) 116 : cephadm [INF] Upgrade: Updating mon.b 2023-10-09T09:12:24.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:24 smithi181 bash[27712]: cephadm 2023-10-09T09:12:23.227200+0000 mgr.y (mgr.34107) 117 : cephadm [INF] Deploying daemon mon.b on smithi181 2023-10-09T09:12:24.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:24 smithi181 bash[27712]: cluster 2023-10-09T09:12:23.272402+0000 mgr.y (mgr.34107) 118 : cluster [DBG] pgmap v263: 89 pgs: 89 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2023-10-09T09:12:24.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:24 smithi181 bash[27712]: cluster 2023-10-09T09:12:24.070515+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2023-10-09T09:12:25.048 INFO:tasks.rados.rados.0.smithi163.stdout:1671: done (0 left) 2023-10-09T09:12:25.048 INFO:tasks.rados.rados.0.smithi163.stdout:1672: snap_remove snap 198 2023-10-09T09:12:25.285 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:25 smithi181 bash[27712]: debug 2023-10-09T09:12:25.028+0000 7fde83035700 -1 mon.b@1(peon).osd e477 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:12:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:25 smithi181 bash[27712]: cluster 2023-10-09T09:12:24.066178+0000 osd.3 (osd.3) 125 : cluster [WRN] failed to encode map e476 with expected crc 2023-10-09T09:12:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:25 smithi181 bash[27712]: cluster 2023-10-09T09:12:24.070893+0000 osd.4 (osd.4) 110 : cluster [WRN] failed to encode map e476 with expected crc 2023-10-09T09:12:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:25 smithi181 bash[27712]: cluster 2023-10-09T09:12:25.039190+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2023-10-09T09:12:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:25 smithi181 systemd[1]: Stopping Ceph mon.b for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:12:25.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:25 smithi181 bash[27712]: debug 2023-10-09T09:12:25.664+0000 7fde89842700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T09:12:25.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:25 smithi181 bash[27712]: debug 2023-10-09T09:12:25.664+0000 7fde89842700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T09:12:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:26 smithi181 bash[59224]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T09:12:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:26 smithi181 bash[59289]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T09:12:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:26 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T09:12:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:26 smithi181 systemd[1]: Stopped Ceph mon.b for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:12:27.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:26 smithi181 systemd[1]: Started Ceph mon.b for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:12:28.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.792+0000 7f565cf8bc80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T09:12:28.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.792+0000 7f565cf8bc80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T09:12:28.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.792+0000 7f565cf8bc80 0 pidfile_write: ignore empty --pid-file 2023-10-09T09:12:28.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 0 load: jerasure load: lrc 2023-10-09T09:12:28.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T09:12:28.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Git sha 0 2023-10-09T09:12:28.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T09:12:28.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: DB SUMMARY 2023-10-09T09:12:28.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: DB Session ID: BYWHYV0CVPKKT18BCCFE 2023-10-09T09:12:28.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 2059 Bytes 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000038.sst 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000036.log size: 3476634 ; 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.env: 0x55be2c730720 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T09:12:28.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.info_log: 0x55be2df2f480 2023-10-09T09:12:28.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T09:12:28.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.statistics: (nil) 2023-10-09T09:12:28.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.use_fsync: 0 2023-10-09T09:12:28.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T09:12:28.071 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T09:12:28.071 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T09:12:28.071 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T09:12:28.071 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T09:12:28.071 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T09:12:28.071 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T09:12:28.071 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.db_log_dir: 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.wal_dir: 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T09:12:28.072 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.write_buffer_manager: 0x55be2d1bcc80 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T09:12:28.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.unordered_write: 0 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.row_cache: None 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.wal_filter: None 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T09:12:28.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.wal_compression: 0 2023-10-09T09:12:28.075 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T09:12:28.075 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T09:12:28.075 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T09:12:28.075 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T09:12:28.075 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T09:12:28.075 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T09:12:28.075 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T09:12:28.075 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T09:12:28.075 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T09:12:28.075 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T09:12:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_open_files: -1 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Compression algorithms supported: 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: kZSTD supported: 0 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T09:12:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.merge_operator: 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_filter: None 2023-10-09T09:12:28.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55be2df2f580) 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: cache_index_and_filter_blocks: 1 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: pin_top_level_index_and_filter: 1 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: index_type: 0 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: data_block_index_type: 0 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: index_shortening: 1 2023-10-09T09:12:28.079 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: checksum: 4 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: no_block_cache: 0 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: block_cache: 0x55be2d23f1f0 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: block_cache_name: BinnedLRUCache 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: block_cache_options: 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: capacity : 536870912 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: num_shard_bits : 4 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: strict_capacity_limit : 0 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: high_pri_pool_ratio: 0.000 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: block_cache_compressed: (nil) 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: persistent_cache: (nil) 2023-10-09T09:12:28.080 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: block_size: 4096 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: block_size_deviation: 10 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: block_restart_interval: 16 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: index_block_restart_interval: 1 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: metadata_block_size: 4096 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: partition_filters: 0 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: use_delta_encoding: 1 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: filter_policy: bloomfilter 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: whole_key_filtering: 1 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: verify_compression: 0 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: read_amp_bytes_per_bit: 0 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: format_version: 5 2023-10-09T09:12:28.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: enable_index_compression: 1 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: block_align: 0 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: max_auto_readahead_size: 262144 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: prepopulate_block_cache: 0 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: initial_auto_readahead_size: 8192 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: num_file_reads_for_auto_readahead: 2 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compression: NoCompression 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T09:12:28.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.num_levels: 7 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:12:28.083 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T09:12:28.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T09:12:28.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T09:12:28.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T09:12:28.087 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T09:12:28.087 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T09:12:28.088 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.ttl: 2592000 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T09:12:28.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.796+0000 7f565cf8bc80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.800+0000 7f56527a9700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 38.sst 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.800+0000 7f565cf8bc80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 40, last_sequence is 24755, log_number is 36,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.800+0000 7f565cf8bc80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 36 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.800+0000 7f565cf8bc80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 26b2b7de-ad42-4154-9c15-fc06a01d779b 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.800+0000 7f565cf8bc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696842747802682, "job": 1, "event": "recovery_started", "wal_files": [36]} 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.800+0000 7f565cf8bc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #36 mode 2 2023-10-09T09:12:28.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.808+0000 7f565cf8bc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696842747812188, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 41, "file_size": 2306830, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 24760, "largest_seqno": 25512, "table_properties": {"data_size": 2301781, "index_size": 3390, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 645, "raw_key_size": 5657, "raw_average_key_size": 22, "raw_value_size": 2294603, "raw_average_value_size": 9289, "num_data_blocks": 147, "num_entries": 247, "num_filter_entries": 247, "num_deletions": 13, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696842747, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "26b2b7de-ad42-4154-9c15-fc06a01d779b", "db_session_id": "BYWHYV0CVPKKT18BCCFE", "orig_file_number": 41, "seqno_to_time_mapping": "N/A"}} 2023-10-09T09:12:28.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.808+0000 7f565cf8bc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696842747812313, "job": 1, "event": "recovery_finished"} 2023-10-09T09:12:28.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.808+0000 7f565cf8bc80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 43 2023-10-09T09:12:28.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000036.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T09:12:28.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55be2d2e0000 2023-10-09T09:12:28.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 4 rocksdb: DB pointer 0x55be2d2ce000 2023-10-09T09:12:28.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f564b79b700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T09:12:28.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f564b79b700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T09:12:28.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: ** DB Stats ** 2023-10-09T09:12:28.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: ** Compaction Stats [default] ** 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: L0 1/0 2.20 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 542.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: L6 1/0 30.01 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T09:12:28.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Sum 2/0 32.21 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 542.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 542.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: ** Compaction Stats [default] ** 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 542.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Flush(GB): cumulative 0.002, interval 0.002 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:12:28.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Cumulative compaction: 0.00 GB write, 171.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Interval compaction: 0.00 GB write, 171.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Block cache BinnedLRUCache@0x55be2d23f1f0#7 capacity: 512.00 MB usage: 4.80 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.9e-05 secs_since: 0 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Block cache entry stats(count,size,portion): FilterBlock(1,0.72 KB,0.000137091%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 1 mon.b@-1(???) e3 preinit fsid 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 0 mon.b@-1(???).mds e6 new map 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 0 mon.b@-1(???).mds e6 print_map 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: e6 2023-10-09T09:12:28.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: legacy client fscid: 1 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Filesystem 'foo' (1) 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: fs_name foo 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: epoch 5 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: created 2023-10-09T09:03:24.292218+0000 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: modified 2023-10-09T09:03:30.346509+0000 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: tableserver 0 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: root 0 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: session_timeout 60 2023-10-09T09:12:28.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: session_autoclose 300 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: max_file_size 1099511627776 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: required_client_features {} 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: last_failure 0 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: last_failure_osd_epoch 0 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: max_mds 1 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: in 0 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: up {0=14640} 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: failed 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: damaged 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: stopped 2023-10-09T09:12:28.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: data_pools [4] 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: metadata_pool 3 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: inline_data disabled 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: balancer 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: bal_rank_mask -1 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: standby_count_wanted 1 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: [mds.foo.smithi163.vnodzj{0:14640} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.163:6834/3139884719,v1:172.21.15.163:6835/3139884719] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: Standby daemons: 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: [mds.foo.smithi181.ajqgsc{-1:24511} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.181:6832/2837854117,v1:172.21.15.181:6833/2837854117] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 0 mon.b@-1(???).osd e477 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T09:12:28.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 0 mon.b@-1(???).osd e477 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 0 mon.b@-1(???).osd e477 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 0 mon.b@-1(???).osd e477 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.812+0000 7f565cf8bc80 1 mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta expand map: {default=false} 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta from 'false' to 'false' 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta expanded map: {default=false} 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta expand map: {default=info} 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta from 'info' to 'info' 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta expanded map: {default=info} 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta expand map: {default=daemon} 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T09:12:28.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta expanded map: {default=daemon} 2023-10-09T09:12:28.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta expand map: {default=debug} 2023-10-09T09:12:28.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta from 'debug' to 'debug' 2023-10-09T09:12:28.099 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:27 smithi181 bash[59345]: debug 2023-10-09T09:12:27.816+0000 7f565cf8bc80 20 expand_channel_meta expanded map: {default=debug} 2023-10-09T09:12:30.148 INFO:tasks.rados.rados.0.smithi163.stdout:1672: done (0 left) 2023-10-09T09:12:30.148 INFO:tasks.rados.rados.0.smithi163.stdout:1673: snap_remove snap 202 2023-10-09T09:12:30.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T09:12:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T08:53:11.193448+0000 mon.a (mon.0) 0 : 1 mkfs 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T09:12:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:07:36.006746+0000 mgr.x (mgr.24751) 46 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:12:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:07:36.007332+0000 mgr.x (mgr.24751) 47 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi181 2023-10-09T09:12:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:07:37.575462+0000 mgr.x (mgr.24751) 49 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:12:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:07:37.595698+0000 mgr.x (mgr.24751) 50 : cephadm 1 Reconfiguring daemon grafana.a on smithi181 2023-10-09T09:12:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:01.488589+0000 mgr.x (mgr.24751) 66 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:12:30.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:01.489177+0000 mgr.x (mgr.24751) 67 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:03.758942+0000 mgr.x (mgr.24751) 69 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:03.776612+0000 mgr.x (mgr.24751) 70 : cephadm 1 Deploying daemon mgr.y on smithi163 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:24.247589+0000 mgr.y (mgr.15273) 1 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Bus STARTING 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:24.349971+0000 mgr.y (mgr.15273) 2 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:24.484563+0000 mgr.y (mgr.15273) 3 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:24.485051+0000 mgr.y (mgr.15273) 4 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Bus STARTED 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:40.354122+0000 mgr.y (mgr.15273) 14 : cephadm 1 Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:40.354386+0000 mgr.y (mgr.15273) 15 : cephadm 1 Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:40.518790+0000 mgr.y (mgr.15273) 16 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:40.519664+0000 mgr.y (mgr.15273) 17 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:40.683583+0000 mgr.y (mgr.15273) 18 : cephadm 1 Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:40.684511+0000 mgr.y (mgr.15273) 19 : cephadm 1 Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:40.847975+0000 mgr.y (mgr.15273) 20 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:40.850184+0000 mgr.y (mgr.15273) 21 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:41.112637+0000 mgr.y (mgr.15273) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:41.367646+0000 mgr.y (mgr.15273) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:48.444054+0000 mgr.y (mgr.15273) 29 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:08:48.450490+0000 mgr.y (mgr.15273) 30 : cephadm 1 Deploying daemon mgr.x on smithi181 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:08.331692+0000 mgr.y (mgr.15273) 41 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:08.354461+0000 mgr.y (mgr.15273) 42 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:10.374175+0000 mgr.y (mgr.15273) 44 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:10.384145+0000 mgr.y (mgr.15273) 45 : cephadm 1 Deploying daemon mon.a on smithi163 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:25.760954+0000 mgr.y (mgr.34107) 1 : cephadm 1 [09/Oct/2023:09:09:25] ENGINE Bus STARTING 2023-10-09T09:12:30.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:25.906276+0000 mgr.y (mgr.34107) 2 : cephadm 1 [09/Oct/2023:09:09:25] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:26.008553+0000 mgr.y (mgr.34107) 3 : cephadm 1 [09/Oct/2023:09:09:26] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:26.009590+0000 mgr.y (mgr.34107) 4 : cephadm 1 [09/Oct/2023:09:09:26] ENGINE Bus STARTED 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:30.452875+0000 mgr.y (mgr.34107) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:42.033797+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:42.034174+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:42.199742+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:42.200193+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:42.364578+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:42.366375+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:42.529221+0000 mgr.y (mgr.34107) 23 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:09:42.531686+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:10:37.795190+0000 mgr.y (mgr.34107) 54 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.472 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:10:44.839690+0000 mon.a (mon.0) 170 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:12:30.472 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:10:44.841917+0000 mon.a (mon.0) 171 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:12:30.472 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:10:44.843043+0000 mon.a (mon.0) 172 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T09:12:30.472 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:10:44.843839+0000 mgr.y (mgr.34107) 58 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T09:12:30.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T08:53:11.193448+0000 mon.a (mon.0) 0 : 1 mkfs 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T09:12:30.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:07:36.006746+0000 mgr.x (mgr.24751) 46 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:12:30.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:07:36.007332+0000 mgr.x (mgr.24751) 47 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi181 2023-10-09T09:12:30.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:07:37.575462+0000 mgr.x (mgr.24751) 49 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:12:30.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:07:37.595698+0000 mgr.x (mgr.24751) 50 : cephadm 1 Reconfiguring daemon grafana.a on smithi181 2023-10-09T09:12:30.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:01.488589+0000 mgr.x (mgr.24751) 66 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:01.489177+0000 mgr.x (mgr.24751) 67 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:03.758942+0000 mgr.x (mgr.24751) 69 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:03.776612+0000 mgr.x (mgr.24751) 70 : cephadm 1 Deploying daemon mgr.y on smithi163 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:24.247589+0000 mgr.y (mgr.15273) 1 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Bus STARTING 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:24.349971+0000 mgr.y (mgr.15273) 2 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:24.484563+0000 mgr.y (mgr.15273) 3 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:24.485051+0000 mgr.y (mgr.15273) 4 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Bus STARTED 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:40.354122+0000 mgr.y (mgr.15273) 14 : cephadm 1 Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:40.354386+0000 mgr.y (mgr.15273) 15 : cephadm 1 Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:40.518790+0000 mgr.y (mgr.15273) 16 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:40.519664+0000 mgr.y (mgr.15273) 17 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:40.683583+0000 mgr.y (mgr.15273) 18 : cephadm 1 Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:40.684511+0000 mgr.y (mgr.15273) 19 : cephadm 1 Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:40.847975+0000 mgr.y (mgr.15273) 20 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:40.850184+0000 mgr.y (mgr.15273) 21 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:41.112637+0000 mgr.y (mgr.15273) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:12:30.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:41.367646+0000 mgr.y (mgr.15273) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:12:30.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:48.444054+0000 mgr.y (mgr.15273) 29 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T09:12:30.474 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:10:45.263947+0000 mon.a (mon.0) 174 : audit 0 from='client.? 172.21.15.163:0/3809846710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.474 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:10:47.213712+0000 mgr.y (mgr.34107) 60 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:10:47.222923+0000 mon.a (mon.0) 177 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:10:47.224846+0000 mon.a (mon.0) 178 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:10:47.226004+0000 mon.a (mon.0) 179 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:10:47.227129+0000 mon.a (mon.0) 180 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:10:47.228598+0000 mgr.y (mgr.34107) 61 : cephadm 1 Deploying daemon mon.c on smithi163 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:10:47.721811+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.163:0/2236837363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:10:59.548027+0000 mon.a (mon.0) 211 : audit 0 from='client.? 172.21.15.163:0/2481174109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:02.908311+0000 mon.a (mon.0) 216 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:02.915742+0000 mon.a (mon.0) 217 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:03.413966+0000 mgr.y (mgr.34107) 71 : audit 0 from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:30.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:03.707993+0000 mon.a (mon.0) 218 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:03.713860+0000 mon.a (mon.0) 219 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:07.209925+0000 mgr.y (mgr.34107) 73 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:11:07.210552+0000 mgr.y (mgr.34107) 74 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:07.216879+0000 mon.a (mon.0) 225 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:10.332325+0000 mon.a (mon.0) 229 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:13.462448+0000 mon.a (mon.0) 233 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:13.470066+0000 mon.a (mon.0) 234 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:13.471565+0000 mon.a (mon.0) 235 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:13.472735+0000 mon.a (mon.0) 236 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:13.479390+0000 mon.a (mon.0) 237 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:13.564968+0000 mon.a (mon.0) 238 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:13.567529+0000 mon.a (mon.0) 239 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:13.568562+0000 mon.a (mon.0) 240 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:13.575538+0000 mon.a (mon.0) 241 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:25.338747+0000 mon.a (mon.0) 254 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:40.337481+0000 mon.a (mon.0) 270 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:11:55.338785+0000 mon.a (mon.0) 285 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:10.339101+0000 mon.a (mon.0) 301 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:13.660596+0000 mon.a (mon.0) 305 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:15.753027+0000 mgr.y (mgr.34107) 110 : audit 0 from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:12:15.753766+0000 mgr.y (mgr.34107) 111 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:15.760518+0000 mon.a (mon.0) 308 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:16.039432+0000 osd.3 (osd.3) 117 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:16.042584+0000 mon.a (mon.0) 309 : cluster 0 osdmap e468: 8 total, 8 up, 8 in 2023-10-09T09:12:30.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:16.047260+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:16.053196+0000 osd.2 (osd.2) 111 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:16.710741+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:17.042797+0000 osd.3 (osd.3) 118 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:17.044778+0000 mon.a (mon.0) 310 : cluster 0 osdmap e469: 8 total, 8 up, 8 in 2023-10-09T09:12:30.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:17.049271+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:08:48.450490+0000 mgr.y (mgr.15273) 30 : cephadm 1 Deploying daemon mgr.x on smithi181 2023-10-09T09:12:30.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:08.331692+0000 mgr.y (mgr.15273) 41 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T09:12:30.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:08.354461+0000 mgr.y (mgr.15273) 42 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T09:12:30.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:10.374175+0000 mgr.y (mgr.15273) 44 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:10.384145+0000 mgr.y (mgr.15273) 45 : cephadm 1 Deploying daemon mon.a on smithi163 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:25.760954+0000 mgr.y (mgr.34107) 1 : cephadm 1 [09/Oct/2023:09:09:25] ENGINE Bus STARTING 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:25.906276+0000 mgr.y (mgr.34107) 2 : cephadm 1 [09/Oct/2023:09:09:25] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:26.008553+0000 mgr.y (mgr.34107) 3 : cephadm 1 [09/Oct/2023:09:09:26] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:26.009590+0000 mgr.y (mgr.34107) 4 : cephadm 1 [09/Oct/2023:09:09:26] ENGINE Bus STARTED 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:30.452875+0000 mgr.y (mgr.34107) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:42.033797+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:42.034174+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:42.199742+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:42.200193+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:42.364578+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:42.366375+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:42.529221+0000 mgr.y (mgr.34107) 23 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:09:42.531686+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:10:37.795190+0000 mgr.y (mgr.34107) 54 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:10:44.839690+0000 mon.a (mon.0) 170 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:12:30.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:10:44.841917+0000 mon.a (mon.0) 171 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:12:30.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:10:44.843043+0000 mon.a (mon.0) 172 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T09:12:30.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:10:44.843839+0000 mgr.y (mgr.34107) 58 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:17.053274+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:17.055014+0000 osd.1 (osd.1) 51 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:17.270107+0000 mgr.y (mgr.34107) 112 : cluster 0 pgmap v254: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 528 KiB/s wr, 14 op/s 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:18.046316+0000 osd.3 (osd.3) 119 : cluster 3 failed to encode map e470 with expected crc 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:18.048331+0000 mon.a (mon.0) 311 : cluster 0 osdmap e470: 8 total, 8 up, 8 in 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:18.051122+0000 osd.1 (osd.1) 52 : cluster 3 failed to encode map e470 with expected crc 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:18.061698+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e470 with expected crc 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:19.046683+0000 osd.3 (osd.3) 120 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:19.048140+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:19.051932+0000 mon.a (mon.0) 312 : cluster 0 osdmap e471: 8 total, 8 up, 8 in 2023-10-09T09:12:30.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:19.203874+0000 osd.2 (osd.2) 112 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:19.270650+0000 mgr.y (mgr.34107) 113 : cluster 0 pgmap v257: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 528 KiB/s wr, 14 op/s 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:19.428518+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:19.477244+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:19.953388+0000 mon.a (mon.0) 313 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:19.961237+0000 mon.a (mon.0) 314 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:20.032171+0000 osd.3 (osd.3) 121 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:20.034096+0000 mon.a (mon.0) 315 : cluster 0 osdmap e472: 8 total, 8 up, 8 in 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:20.041037+0000 osd.0 (osd.0) 74 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:20.045862+0000 osd.2 (osd.2) 113 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:20.055277+0000 mon.a (mon.0) 316 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:20.061149+0000 mon.a (mon.0) 317 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:20.542570+0000 mon.a (mon.0) 318 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:20.543809+0000 mon.a (mon.0) 319 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:20.555463+0000 mon.a (mon.0) 320 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:20.636922+0000 mon.a (mon.0) 321 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:20.640693+0000 mon.a (mon.0) 322 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:20.641813+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:20.642597+0000 mon.a (mon.0) 323 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:20.643990+0000 mon.a (mon.0) 324 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:12:20.644847+0000 mgr.y (mgr.34107) 114 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:20.708362+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:20.949356+0000 osd.1 (osd.1) 53 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:21.056437+0000 osd.3 (osd.3) 122 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:21.059986+0000 mon.a (mon.0) 325 : cluster 0 osdmap e473: 8 total, 8 up, 8 in 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:21.063056+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:21.067174+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:21.078644+0000 osd.2 (osd.2) 114 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:21.271764+0000 mgr.y (mgr.34107) 115 : cluster 0 pgmap v260: 89 pgs: 89 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:22.059044+0000 osd.3 (osd.3) 123 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:22.063781+0000 mon.a (mon.0) 326 : cluster 0 osdmap e474: 8 total, 8 up, 8 in 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:22.065238+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:22.080092+0000 osd.2 (osd.2) 115 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:22.085430+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:23.061456+0000 osd.3 (osd.3) 124 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:23.065898+0000 mon.a (mon.0) 327 : cluster 0 osdmap e475: 8 total, 8 up, 8 in 2023-10-09T09:12:30.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:23.070317+0000 osd.2 (osd.2) 116 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.485 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:23.072585+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.485 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:23.072693+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:10:45.263947+0000 mon.a (mon.0) 174 : audit 0 from='client.? 172.21.15.163:0/3809846710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:10:47.213712+0000 mgr.y (mgr.34107) 60 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T09:12:30.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:10:47.222923+0000 mon.a (mon.0) 177 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:10:47.224846+0000 mon.a (mon.0) 178 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:30.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:10:47.226004+0000 mon.a (mon.0) 179 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:30.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:10:47.227129+0000 mon.a (mon.0) 180 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:10:47.228598+0000 mgr.y (mgr.34107) 61 : cephadm 1 Deploying daemon mon.c on smithi163 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:10:47.721811+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.163:0/2236837363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:10:59.548027+0000 mon.a (mon.0) 211 : audit 0 from='client.? 172.21.15.163:0/2481174109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:02.908311+0000 mon.a (mon.0) 216 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:02.915742+0000 mon.a (mon.0) 217 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:03.413966+0000 mgr.y (mgr.34107) 71 : audit 0 from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:03.707993+0000 mon.a (mon.0) 218 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:03.713860+0000 mon.a (mon.0) 219 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:07.209925+0000 mgr.y (mgr.34107) 73 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:11:07.210552+0000 mgr.y (mgr.34107) 74 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:07.216879+0000 mon.a (mon.0) 225 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:10.332325+0000 mon.a (mon.0) 229 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:13.462448+0000 mon.a (mon.0) 233 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.487 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:13.470066+0000 mon.a (mon.0) 234 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.487 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:13.471565+0000 mon.a (mon.0) 235 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:23.092510+0000 mon.a (mon.0) 328 : audit 0 from='client.? 172.21.15.163:0/3016853494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:12:23.214488+0000 mgr.y (mgr.34107) 116 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T09:12:30.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:23.222393+0000 mon.a (mon.0) 329 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:23.223906+0000 mon.a (mon.0) 330 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:30.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:23.225052+0000 mon.a (mon.0) 331 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:23.226056+0000 mon.a (mon.0) 332 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cephadm 2023-10-09T09:12:23.227200+0000 mgr.y (mgr.34107) 117 : cephadm 1 Deploying daemon mon.b on smithi181 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:23.272402+0000 mgr.y (mgr.34107) 118 : cluster 0 pgmap v263: 89 pgs: 89 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:24.066178+0000 osd.3 (osd.3) 125 : cluster 3 failed to encode map e476 with expected crc 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:24.070515+0000 mon.a (mon.0) 333 : cluster 0 osdmap e476: 8 total, 8 up, 8 in 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:24.070893+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e476 with expected crc 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:25.039190+0000 mon.a (mon.0) 334 : cluster 0 osdmap e477: 8 total, 8 up, 8 in 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.130503+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.131301+0000 mon.a (mon.0) 343 : cluster 1 overall HEALTH_OK 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:29.131818+0000 mon.a (mon.0) 344 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:12:30.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.132218+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.132228+0000 mon.a (mon.0) 345 : cluster 1 mon.a calling monitor election 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:29.133867+0000 mon.a (mon.0) 346 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:29.134138+0000 mon.a (mon.0) 347 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.157511+0000 mon.a (mon.0) 348 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.163863+0000 mon.a (mon.0) 349 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.164735+0000 mon.a (mon.0) 350 : cluster 0 fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 1 up:standby 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.164785+0000 mon.a (mon.0) 351 : cluster 0 osdmap e477: 8 total, 8 up, 8 in 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.165381+0000 mon.a (mon.0) 352 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.171393+0000 mon.a (mon.0) 353 : cluster 1 overall HEALTH_OK 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:29.176512+0000 mon.a (mon.0) 354 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:29.185610+0000 mon.a (mon.0) 355 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: audit 2023-10-09T09:12:29.210678+0000 mon.a (mon.0) 356 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.490 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[75425]: cluster 2023-10-09T09:12:29.275023+0000 mgr.y (mgr.34107) 121 : cluster 0 pgmap v268: 89 pgs: 89 active+clean; 97 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 567 KiB/s wr, 5 op/s 2023-10-09T09:12:30.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:13.472735+0000 mon.a (mon.0) 236 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:30.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:13.479390+0000 mon.a (mon.0) 237 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:13.564968+0000 mon.a (mon.0) 238 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:13.567529+0000 mon.a (mon.0) 239 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:13.568562+0000 mon.a (mon.0) 240 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:13.575538+0000 mon.a (mon.0) 241 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:25.338747+0000 mon.a (mon.0) 254 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:40.337481+0000 mon.a (mon.0) 270 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:11:55.338785+0000 mon.a (mon.0) 285 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:10.339101+0000 mon.a (mon.0) 301 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:13.660596+0000 mon.a (mon.0) 305 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:15.753027+0000 mgr.y (mgr.34107) 110 : audit 0 from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:12:15.753766+0000 mgr.y (mgr.34107) 111 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:15.760518+0000 mon.a (mon.0) 308 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:16.039432+0000 osd.3 (osd.3) 117 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:16.042584+0000 mon.a (mon.0) 309 : cluster 0 osdmap e468: 8 total, 8 up, 8 in 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:16.047260+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:16.053196+0000 osd.2 (osd.2) 111 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:16.710741+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:17.042797+0000 osd.3 (osd.3) 118 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:17.044778+0000 mon.a (mon.0) 310 : cluster 0 osdmap e469: 8 total, 8 up, 8 in 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:17.049271+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:17.053274+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:17.055014+0000 osd.1 (osd.1) 51 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:17.270107+0000 mgr.y (mgr.34107) 112 : cluster 0 pgmap v254: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 528 KiB/s wr, 14 op/s 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:18.046316+0000 osd.3 (osd.3) 119 : cluster 3 failed to encode map e470 with expected crc 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:18.048331+0000 mon.a (mon.0) 311 : cluster 0 osdmap e470: 8 total, 8 up, 8 in 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:18.051122+0000 osd.1 (osd.1) 52 : cluster 3 failed to encode map e470 with expected crc 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:18.061698+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e470 with expected crc 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:19.046683+0000 osd.3 (osd.3) 120 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:19.048140+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:19.051932+0000 mon.a (mon.0) 312 : cluster 0 osdmap e471: 8 total, 8 up, 8 in 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:19.203874+0000 osd.2 (osd.2) 112 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:19.270650+0000 mgr.y (mgr.34107) 113 : cluster 0 pgmap v257: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 528 KiB/s wr, 14 op/s 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:19.428518+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:19.477244+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:19.953388+0000 mon.a (mon.0) 313 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:19.961237+0000 mon.a (mon.0) 314 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:20.032171+0000 osd.3 (osd.3) 121 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:20.034096+0000 mon.a (mon.0) 315 : cluster 0 osdmap e472: 8 total, 8 up, 8 in 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:20.041037+0000 osd.0 (osd.0) 74 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:20.045862+0000 osd.2 (osd.2) 113 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:20.055277+0000 mon.a (mon.0) 316 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:20.061149+0000 mon.a (mon.0) 317 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:20.542570+0000 mon.a (mon.0) 318 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:20.543809+0000 mon.a (mon.0) 319 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:20.555463+0000 mon.a (mon.0) 320 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:20.636922+0000 mon.a (mon.0) 321 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:20.640693+0000 mon.a (mon.0) 322 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:20.641813+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:20.642597+0000 mon.a (mon.0) 323 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:20.643990+0000 mon.a (mon.0) 324 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:12:20.644847+0000 mgr.y (mgr.34107) 114 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:20.708362+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:20.949356+0000 osd.1 (osd.1) 53 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:21.056437+0000 osd.3 (osd.3) 122 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:21.059986+0000 mon.a (mon.0) 325 : cluster 0 osdmap e473: 8 total, 8 up, 8 in 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:21.063056+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:21.067174+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:21.078644+0000 osd.2 (osd.2) 114 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:21.271764+0000 mgr.y (mgr.34107) 115 : cluster 0 pgmap v260: 89 pgs: 89 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:22.059044+0000 osd.3 (osd.3) 123 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:22.063781+0000 mon.a (mon.0) 326 : cluster 0 osdmap e474: 8 total, 8 up, 8 in 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:22.065238+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:22.080092+0000 osd.2 (osd.2) 115 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:22.085430+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:23.061456+0000 osd.3 (osd.3) 124 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:23.065898+0000 mon.a (mon.0) 327 : cluster 0 osdmap e475: 8 total, 8 up, 8 in 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:23.070317+0000 osd.2 (osd.2) 116 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:23.072585+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:23.072693+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:23.092510+0000 mon.a (mon.0) 328 : audit 0 from='client.? 172.21.15.163:0/3016853494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:12:23.214488+0000 mgr.y (mgr.34107) 116 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T09:12:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:23.222393+0000 mon.a (mon.0) 329 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:23.223906+0000 mon.a (mon.0) 330 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:23.225052+0000 mon.a (mon.0) 331 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:23.226056+0000 mon.a (mon.0) 332 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cephadm 2023-10-09T09:12:23.227200+0000 mgr.y (mgr.34107) 117 : cephadm 1 Deploying daemon mon.b on smithi181 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:23.272402+0000 mgr.y (mgr.34107) 118 : cluster 0 pgmap v263: 89 pgs: 89 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:24.066178+0000 osd.3 (osd.3) 125 : cluster 3 failed to encode map e476 with expected crc 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:24.070515+0000 mon.a (mon.0) 333 : cluster 0 osdmap e476: 8 total, 8 up, 8 in 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:24.070893+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e476 with expected crc 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:25.039190+0000 mon.a (mon.0) 334 : cluster 0 osdmap e477: 8 total, 8 up, 8 in 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.130503+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.131301+0000 mon.a (mon.0) 343 : cluster 1 overall HEALTH_OK 2023-10-09T09:12:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:29.131818+0000 mon.a (mon.0) 344 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.132218+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.132228+0000 mon.a (mon.0) 345 : cluster 1 mon.a calling monitor election 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:29.133867+0000 mon.a (mon.0) 346 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:29.134138+0000 mon.a (mon.0) 347 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.157511+0000 mon.a (mon.0) 348 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.163863+0000 mon.a (mon.0) 349 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.164735+0000 mon.a (mon.0) 350 : cluster 0 fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 1 up:standby 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.164785+0000 mon.a (mon.0) 351 : cluster 0 osdmap e477: 8 total, 8 up, 8 in 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.165381+0000 mon.a (mon.0) 352 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.171393+0000 mon.a (mon.0) 353 : cluster 1 overall HEALTH_OK 2023-10-09T09:12:30.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:29.176512+0000 mon.a (mon.0) 354 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.499 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:29.185610+0000 mon.a (mon.0) 355 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.499 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: audit 2023-10-09T09:12:29.210678+0000 mon.a (mon.0) 356 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.499 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:30 smithi163 bash[78107]: cluster 2023-10-09T09:12:29.275023+0000 mgr.y (mgr.34107) 121 : cluster 0 pgmap v268: 89 pgs: 89 active+clean; 97 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 567 KiB/s wr, 5 op/s 2023-10-09T09:12:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T08:53:11.193448+0000 mon.a (mon.0) 0 : 1 mkfs 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T09:12:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:07:36.006746+0000 mgr.x (mgr.24751) 46 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:12:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:07:36.007332+0000 mgr.x (mgr.24751) 47 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi181 2023-10-09T09:12:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:07:37.575462+0000 mgr.x (mgr.24751) 49 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:12:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:07:37.595698+0000 mgr.x (mgr.24751) 50 : cephadm 1 Reconfiguring daemon grafana.a on smithi181 2023-10-09T09:12:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:01.488589+0000 mgr.x (mgr.24751) 66 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:12:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:01.489177+0000 mgr.x (mgr.24751) 67 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:12:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:03.758942+0000 mgr.x (mgr.24751) 69 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T09:12:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:03.776612+0000 mgr.x (mgr.24751) 70 : cephadm 1 Deploying daemon mgr.y on smithi163 2023-10-09T09:12:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:24.247589+0000 mgr.y (mgr.15273) 1 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Bus STARTING 2023-10-09T09:12:30.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:24.349971+0000 mgr.y (mgr.15273) 2 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:24.484563+0000 mgr.y (mgr.15273) 3 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:24.485051+0000 mgr.y (mgr.15273) 4 : cephadm 1 [09/Oct/2023:09:08:24] ENGINE Bus STARTED 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:40.354122+0000 mgr.y (mgr.15273) 14 : cephadm 1 Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:40.354386+0000 mgr.y (mgr.15273) 15 : cephadm 1 Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:40.518790+0000 mgr.y (mgr.15273) 16 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:40.519664+0000 mgr.y (mgr.15273) 17 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:40.683583+0000 mgr.y (mgr.15273) 18 : cephadm 1 Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:40.684511+0000 mgr.y (mgr.15273) 19 : cephadm 1 Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:40.847975+0000 mgr.y (mgr.15273) 20 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:40.850184+0000 mgr.y (mgr.15273) 21 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:41.112637+0000 mgr.y (mgr.15273) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:12:30.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:41.367646+0000 mgr.y (mgr.15273) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi181 2023-10-09T09:12:30.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:48.444054+0000 mgr.y (mgr.15273) 29 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T09:12:30.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:08:48.450490+0000 mgr.y (mgr.15273) 30 : cephadm 1 Deploying daemon mgr.x on smithi181 2023-10-09T09:12:30.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:08.331692+0000 mgr.y (mgr.15273) 41 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T09:12:30.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:08.354461+0000 mgr.y (mgr.15273) 42 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T09:12:30.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:10.374175+0000 mgr.y (mgr.15273) 44 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T09:12:30.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:10.384145+0000 mgr.y (mgr.15273) 45 : cephadm 1 Deploying daemon mon.a on smithi163 2023-10-09T09:12:30.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:25.760954+0000 mgr.y (mgr.34107) 1 : cephadm 1 [09/Oct/2023:09:09:25] ENGINE Bus STARTING 2023-10-09T09:12:30.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:25.906276+0000 mgr.y (mgr.34107) 2 : cephadm 1 [09/Oct/2023:09:09:25] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T09:12:30.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:26.008553+0000 mgr.y (mgr.34107) 3 : cephadm 1 [09/Oct/2023:09:09:26] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T09:12:30.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:26.009590+0000 mgr.y (mgr.34107) 4 : cephadm 1 [09/Oct/2023:09:09:26] ENGINE Bus STARTED 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:30.452875+0000 mgr.y (mgr.34107) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:42.033797+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:42.034174+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:42.199742+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:42.200193+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:42.364578+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:42.366375+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:42.529221+0000 mgr.y (mgr.34107) 23 : cephadm 1 Updating smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:09:42.531686+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:10:37.795190+0000 mgr.y (mgr.34107) 54 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:10:44.839690+0000 mon.a (mon.0) 170 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:12:30.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:10:44.841917+0000 mon.a (mon.0) 171 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:10:44.843043+0000 mon.a (mon.0) 172 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:10:44.843839+0000 mgr.y (mgr.34107) 58 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:10:45.263947+0000 mon.a (mon.0) 174 : audit 0 from='client.? 172.21.15.163:0/3809846710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:10:47.213712+0000 mgr.y (mgr.34107) 60 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:10:47.222923+0000 mon.a (mon.0) 177 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:10:47.224846+0000 mon.a (mon.0) 178 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:10:47.226004+0000 mon.a (mon.0) 179 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:10:47.227129+0000 mon.a (mon.0) 180 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:10:47.228598+0000 mgr.y (mgr.34107) 61 : cephadm 1 Deploying daemon mon.c on smithi163 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:10:47.721811+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.163:0/2236837363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:10:59.548027+0000 mon.a (mon.0) 211 : audit 0 from='client.? 172.21.15.163:0/2481174109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:02.908311+0000 mon.a (mon.0) 216 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:02.915742+0000 mon.a (mon.0) 217 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:03.413966+0000 mgr.y (mgr.34107) 71 : audit 0 from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:03.707993+0000 mon.a (mon.0) 218 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:03.713860+0000 mon.a (mon.0) 219 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:07.209925+0000 mgr.y (mgr.34107) 73 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:11:07.210552+0000 mgr.y (mgr.34107) 74 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:07.216879+0000 mon.a (mon.0) 225 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:10.332325+0000 mon.a (mon.0) 229 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:13.462448+0000 mon.a (mon.0) 233 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:13.470066+0000 mon.a (mon.0) 234 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:13.471565+0000 mon.a (mon.0) 235 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:13.472735+0000 mon.a (mon.0) 236 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:13.479390+0000 mon.a (mon.0) 237 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:13.564968+0000 mon.a (mon.0) 238 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:13.567529+0000 mon.a (mon.0) 239 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:13.568562+0000 mon.a (mon.0) 240 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:13.575538+0000 mon.a (mon.0) 241 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:25.338747+0000 mon.a (mon.0) 254 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:40.337481+0000 mon.a (mon.0) 270 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:11:55.338785+0000 mon.a (mon.0) 285 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:10.339101+0000 mon.a (mon.0) 301 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:13.660596+0000 mon.a (mon.0) 305 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:15.753027+0000 mgr.y (mgr.34107) 110 : audit 0 from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:30.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:12:15.753766+0000 mgr.y (mgr.34107) 111 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:15.760518+0000 mon.a (mon.0) 308 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:16.039432+0000 osd.3 (osd.3) 117 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:16.042584+0000 mon.a (mon.0) 309 : cluster 0 osdmap e468: 8 total, 8 up, 8 in 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:16.047260+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:16.053196+0000 osd.2 (osd.2) 111 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:16.710741+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e468 with expected crc 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:17.042797+0000 osd.3 (osd.3) 118 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:17.044778+0000 mon.a (mon.0) 310 : cluster 0 osdmap e469: 8 total, 8 up, 8 in 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:17.049271+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:17.053274+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:17.055014+0000 osd.1 (osd.1) 51 : cluster 3 failed to encode map e469 with expected crc 2023-10-09T09:12:30.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:17.270107+0000 mgr.y (mgr.34107) 112 : cluster 0 pgmap v254: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 528 KiB/s wr, 14 op/s 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:18.046316+0000 osd.3 (osd.3) 119 : cluster 3 failed to encode map e470 with expected crc 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:18.048331+0000 mon.a (mon.0) 311 : cluster 0 osdmap e470: 8 total, 8 up, 8 in 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:18.051122+0000 osd.1 (osd.1) 52 : cluster 3 failed to encode map e470 with expected crc 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:18.061698+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e470 with expected crc 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:19.046683+0000 osd.3 (osd.3) 120 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:19.048140+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:19.051932+0000 mon.a (mon.0) 312 : cluster 0 osdmap e471: 8 total, 8 up, 8 in 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:19.203874+0000 osd.2 (osd.2) 112 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:19.270650+0000 mgr.y (mgr.34107) 113 : cluster 0 pgmap v257: 89 pgs: 89 active+clean; 86 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 528 KiB/s wr, 14 op/s 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:19.428518+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:19.477244+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e471 with expected crc 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:19.953388+0000 mon.a (mon.0) 313 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:19.961237+0000 mon.a (mon.0) 314 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:20.032171+0000 osd.3 (osd.3) 121 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:20.034096+0000 mon.a (mon.0) 315 : cluster 0 osdmap e472: 8 total, 8 up, 8 in 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:20.041037+0000 osd.0 (osd.0) 74 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:20.045862+0000 osd.2 (osd.2) 113 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:20.055277+0000 mon.a (mon.0) 316 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:20.061149+0000 mon.a (mon.0) 317 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:20.542570+0000 mon.a (mon.0) 318 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:20.543809+0000 mon.a (mon.0) 319 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:20.555463+0000 mon.a (mon.0) 320 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:20.636922+0000 mon.a (mon.0) 321 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:20.640693+0000 mon.a (mon.0) 322 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:12:30.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:20.641813+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:20.642597+0000 mon.a (mon.0) 323 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:20.643990+0000 mon.a (mon.0) 324 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:12:20.644847+0000 mgr.y (mgr.34107) 114 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:20.708362+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:20.949356+0000 osd.1 (osd.1) 53 : cluster 3 failed to encode map e472 with expected crc 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:21.056437+0000 osd.3 (osd.3) 122 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:21.059986+0000 mon.a (mon.0) 325 : cluster 0 osdmap e473: 8 total, 8 up, 8 in 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:21.063056+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:21.067174+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:21.078644+0000 osd.2 (osd.2) 114 : cluster 3 failed to encode map e473 with expected crc 2023-10-09T09:12:30.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:21.271764+0000 mgr.y (mgr.34107) 115 : cluster 0 pgmap v260: 89 pgs: 89 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:22.059044+0000 osd.3 (osd.3) 123 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:22.063781+0000 mon.a (mon.0) 326 : cluster 0 osdmap e474: 8 total, 8 up, 8 in 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:22.065238+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:22.080092+0000 osd.2 (osd.2) 115 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:22.085430+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e474 with expected crc 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:23.061456+0000 osd.3 (osd.3) 124 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:23.065898+0000 mon.a (mon.0) 327 : cluster 0 osdmap e475: 8 total, 8 up, 8 in 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:23.070317+0000 osd.2 (osd.2) 116 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:23.072585+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:23.072693+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e475 with expected crc 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:23.092510+0000 mon.a (mon.0) 328 : audit 0 from='client.? 172.21.15.163:0/3016853494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:12:23.214488+0000 mgr.y (mgr.34107) 116 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T09:12:30.580 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:23.222393+0000 mon.a (mon.0) 329 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:23.223906+0000 mon.a (mon.0) 330 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:23.225052+0000 mon.a (mon.0) 331 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:23.226056+0000 mon.a (mon.0) 332 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cephadm 2023-10-09T09:12:23.227200+0000 mgr.y (mgr.34107) 117 : cephadm 1 Deploying daemon mon.b on smithi181 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:23.272402+0000 mgr.y (mgr.34107) 118 : cluster 0 pgmap v263: 89 pgs: 89 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:24.066178+0000 osd.3 (osd.3) 125 : cluster 3 failed to encode map e476 with expected crc 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:24.070515+0000 mon.a (mon.0) 333 : cluster 0 osdmap e476: 8 total, 8 up, 8 in 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:24.070893+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e476 with expected crc 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:25.039190+0000 mon.a (mon.0) 334 : cluster 0 osdmap e477: 8 total, 8 up, 8 in 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.130503+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.131301+0000 mon.a (mon.0) 343 : cluster 1 overall HEALTH_OK 2023-10-09T09:12:30.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:29.131818+0000 mon.a (mon.0) 344 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.132218+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.132228+0000 mon.a (mon.0) 345 : cluster 1 mon.a calling monitor election 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:29.133867+0000 mon.a (mon.0) 346 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:29.134138+0000 mon.a (mon.0) 347 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.157511+0000 mon.a (mon.0) 348 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.163863+0000 mon.a (mon.0) 349 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.164735+0000 mon.a (mon.0) 350 : cluster 0 fsmap foo:1 {0=foo.smithi163.vnodzj=up:active} 1 up:standby 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.164785+0000 mon.a (mon.0) 351 : cluster 0 osdmap e477: 8 total, 8 up, 8 in 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.165381+0000 mon.a (mon.0) 352 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.171393+0000 mon.a (mon.0) 353 : cluster 1 overall HEALTH_OK 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:29.176512+0000 mon.a (mon.0) 354 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:29.185610+0000 mon.a (mon.0) 355 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:30.583 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: audit 2023-10-09T09:12:29.210678+0000 mon.a (mon.0) 356 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:12:30.583 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:30 smithi181 bash[59345]: cluster 2023-10-09T09:12:29.275023+0000 mgr.y (mgr.34107) 121 : cluster 0 pgmap v268: 89 pgs: 89 active+clean; 97 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 567 KiB/s wr, 5 op/s 2023-10-09T09:12:31.153 INFO:tasks.rados.rados.0.smithi163.stdout:1673: done (0 left) 2023-10-09T09:12:31.154 INFO:tasks.rados.rados.0.smithi163.stdout:1674: write oid 25 current snap is 204 2023-10-09T09:12:31.154 INFO:tasks.rados.rados.0.smithi163.stdout:1674: seq_num 640 ranges {756480=733012,2134711=432549} 2023-10-09T09:12:31.165 INFO:tasks.rados.rados.0.smithi163.stdout:1674: writing smithi16367185-25 from 756480 to 1489492 tid 1 2023-10-09T09:12:31.172 INFO:tasks.rados.rados.0.smithi163.stdout:1674: writing smithi16367185-25 from 2134711 to 2567260 tid 2 2023-10-09T09:12:31.174 INFO:tasks.rados.rados.0.smithi163.stdout:1675: write oid 31 current snap is 204 2023-10-09T09:12:31.174 INFO:tasks.rados.rados.0.smithi163.stdout:1675: seq_num 641 ranges {773287=408261,1793193=576207} 2023-10-09T09:12:31.181 INFO:tasks.rados.rados.0.smithi163.stdout:1675: writing smithi16367185-31 from 773287 to 1181548 tid 1 2023-10-09T09:12:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:1675: writing smithi16367185-31 from 1793193 to 2369400 tid 2 2023-10-09T09:12:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:1676: snap_create 2023-10-09T09:12:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:1674: finishing write tid 1 to smithi16367185-25 2023-10-09T09:12:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:1674: finishing write tid 2 to smithi16367185-25 2023-10-09T09:12:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:1674: finishing write tid 3 to smithi16367185-25 2023-10-09T09:12:31.188 INFO:tasks.rados.rados.0.smithi163.stdout:1674: finishing write tid 4 to smithi16367185-25 2023-10-09T09:12:31.189 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 757 (ObjNum 640 snap 204 seq_num 640) dirty exists 2023-10-09T09:12:31.189 INFO:tasks.rados.rados.0.smithi163.stdout:1674: left oid 25 (ObjNum 640 snap 204 seq_num 640) 2023-10-09T09:12:31.189 INFO:tasks.rados.rados.0.smithi163.stdout:1675: finishing write tid 1 to smithi16367185-31 2023-10-09T09:12:31.191 INFO:tasks.rados.rados.0.smithi163.stdout:1675: finishing write tid 2 to smithi16367185-31 2023-10-09T09:12:31.192 INFO:tasks.rados.rados.0.smithi163.stdout:1675: finishing write tid 3 to smithi16367185-31 2023-10-09T09:12:31.192 INFO:tasks.rados.rados.0.smithi163.stdout:1675: finishing write tid 4 to smithi16367185-31 2023-10-09T09:12:31.192 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 448 (ObjNum 641 snap 204 seq_num 641) dirty exists 2023-10-09T09:12:31.193 INFO:tasks.rados.rados.0.smithi163.stdout:1675: left oid 31 (ObjNum 641 snap 204 seq_num 641) 2023-10-09T09:12:31.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:12:31.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:31 smithi163 bash[75425]: cluster 2023-10-09T09:12:30.144964+0000 osd.1 (osd.1) 54 : cluster 3 failed to encode map e478 with expected crc 2023-10-09T09:12:31.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:31 smithi163 bash[75425]: cluster 2023-10-09T09:12:30.145859+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e478 with expected crc 2023-10-09T09:12:31.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:31 smithi163 bash[75425]: cluster 2023-10-09T09:12:30.148210+0000 mon.a (mon.0) 357 : cluster 0 osdmap e478: 8 total, 8 up, 8 in 2023-10-09T09:12:31.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:31 smithi163 bash[75425]: audit 2023-10-09T09:12:30.450226+0000 mon.a (mon.0) 358 : audit 0 from='client.? 172.21.15.163:0/3725678689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:31.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:31 smithi163 bash[78107]: cluster 2023-10-09T09:12:30.144964+0000 osd.1 (osd.1) 54 : cluster 3 failed to encode map e478 with expected crc 2023-10-09T09:12:31.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:31 smithi163 bash[78107]: cluster 2023-10-09T09:12:30.145859+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e478 with expected crc 2023-10-09T09:12:31.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:31 smithi163 bash[78107]: cluster 2023-10-09T09:12:30.148210+0000 mon.a (mon.0) 357 : cluster 0 osdmap e478: 8 total, 8 up, 8 in 2023-10-09T09:12:31.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:31 smithi163 bash[78107]: audit 2023-10-09T09:12:30.450226+0000 mon.a (mon.0) 358 : audit 0 from='client.? 172.21.15.163:0/3725678689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:31 smithi181 bash[59345]: cluster 2023-10-09T09:12:30.144964+0000 osd.1 (osd.1) 54 : cluster 3 failed to encode map e478 with expected crc 2023-10-09T09:12:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:31 smithi181 bash[59345]: cluster 2023-10-09T09:12:30.145859+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e478 with expected crc 2023-10-09T09:12:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:31 smithi181 bash[59345]: cluster 2023-10-09T09:12:30.148210+0000 mon.a (mon.0) 357 : cluster 0 osdmap e478: 8 total, 8 up, 8 in 2023-10-09T09:12:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:31 smithi181 bash[59345]: audit 2023-10-09T09:12:30.450226+0000 mon.a (mon.0) 358 : audit 0 from='client.? 172.21.15.163:0/3725678689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:12:32.176 INFO:tasks.rados.rados.0.smithi163.stdout:1674: done (2 left) 2023-10-09T09:12:32.177 INFO:tasks.rados.rados.0.smithi163.stdout:1675: done (1 left) 2023-10-09T09:12:32.177 INFO:tasks.rados.rados.0.smithi163.stdout:1676: done (0 left) 2023-10-09T09:12:32.177 INFO:tasks.rados.rados.0.smithi163.stdout:1677: snap_remove snap 203 2023-10-09T09:12:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[78107]: cluster 2023-10-09T09:12:31.150126+0000 osd.1 (osd.1) 55 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[78107]: cluster 2023-10-09T09:12:31.150874+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[78107]: cluster 2023-10-09T09:12:31.152315+0000 mon.a (mon.0) 359 : cluster 0 osdmap e479: 8 total, 8 up, 8 in 2023-10-09T09:12:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[78107]: cluster 2023-10-09T09:12:31.167565+0000 osd.2 (osd.2) 117 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:32.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[78107]: cluster 2023-10-09T09:12:31.276067+0000 mgr.y (mgr.34107) 122 : cluster 0 pgmap v271: 89 pgs: 89 active+clean; 85 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 901 KiB/s wr, 6 op/s 2023-10-09T09:12:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[75425]: cluster 2023-10-09T09:12:31.150126+0000 osd.1 (osd.1) 55 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[75425]: cluster 2023-10-09T09:12:31.150874+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[75425]: cluster 2023-10-09T09:12:31.152315+0000 mon.a (mon.0) 359 : cluster 0 osdmap e479: 8 total, 8 up, 8 in 2023-10-09T09:12:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[75425]: cluster 2023-10-09T09:12:31.167565+0000 osd.2 (osd.2) 117 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[75425]: cluster 2023-10-09T09:12:31.276067+0000 mgr.y (mgr.34107) 122 : cluster 0 pgmap v271: 89 pgs: 89 active+clean; 85 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 901 KiB/s wr, 6 op/s 2023-10-09T09:12:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:32 smithi181 bash[59345]: cluster 2023-10-09T09:12:31.150126+0000 osd.1 (osd.1) 55 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:32 smithi181 bash[59345]: cluster 2023-10-09T09:12:31.150874+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:32 smithi181 bash[59345]: cluster 2023-10-09T09:12:31.152315+0000 mon.a (mon.0) 359 : cluster 0 osdmap e479: 8 total, 8 up, 8 in 2023-10-09T09:12:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:32 smithi181 bash[59345]: cluster 2023-10-09T09:12:31.167565+0000 osd.2 (osd.2) 117 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:32.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:32 smithi181 bash[59345]: cluster 2023-10-09T09:12:31.276067+0000 mgr.y (mgr.34107) 122 : cluster 0 pgmap v271: 89 pgs: 89 active+clean; 85 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 901 KiB/s wr, 6 op/s 2023-10-09T09:12:32.968 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:12:32 smithi163 bash[71599]: debug 2023-10-09T09:12:32.822+0000 7f5c688fd700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T09:12:33.187 INFO:tasks.rados.rados.0.smithi163.stdout:1677: done (0 left) 2023-10-09T09:12:33.187 INFO:tasks.rados.rados.0.smithi163.stdout:1678: delete oid 40 current snap is 205 2023-10-09T09:12:33.202 INFO:tasks.rados.rados.0.smithi163.stdout:1678: done (0 left) 2023-10-09T09:12:33.202 INFO:tasks.rados.rados.0.smithi163.stdout:1679: read oid 22 snap -1 2023-10-09T09:12:33.202 INFO:tasks.rados.rados.0.smithi163.stdout:1679: expect (ObjNum 624 snap 198 seq_num 624) 2023-10-09T09:12:33.203 INFO:tasks.rados.rados.0.smithi163.stdout:1680: write oid 35 current snap is 205 2023-10-09T09:12:33.203 INFO:tasks.rados.rados.0.smithi163.stdout:1680: seq_num 643 ranges {406901=558759,1393804=579876,2608396=198505} 2023-10-09T09:12:33.209 INFO:tasks.rados.rados.0.smithi163.stdout:1680: writing smithi16367185-35 from 406901 to 965660 tid 1 2023-10-09T09:12:33.214 INFO:tasks.rados.rados.0.smithi163.stdout:1680: writing smithi16367185-35 from 1393804 to 1973680 tid 2 2023-10-09T09:12:33.219 INFO:tasks.rados.rados.0.smithi163.stdout:1680: writing smithi16367185-35 from 2608396 to 2806901 tid 3 2023-10-09T09:12:33.219 INFO:tasks.rados.rados.0.smithi163.stdout:1681: delete oid 20 current snap is 205 2023-10-09T09:12:33.219 INFO:tasks.rados.rados.0.smithi163.stdout:1679: expect (ObjNum 624 snap 198 seq_num 624) 2023-10-09T09:12:33.380 INFO:tasks.rados.rados.0.smithi163.stdout:1680: finishing write tid 1 to smithi16367185-35 2023-10-09T09:12:33.380 INFO:tasks.rados.rados.0.smithi163.stdout:1680: finishing write tid 2 to smithi16367185-35 2023-10-09T09:12:33.380 INFO:tasks.rados.rados.0.smithi163.stdout:1680: finishing write tid 3 to smithi16367185-35 2023-10-09T09:12:33.380 INFO:tasks.rados.rados.0.smithi163.stdout:1680: finishing write tid 4 to smithi16367185-35 2023-10-09T09:12:33.380 INFO:tasks.rados.rados.0.smithi163.stdout:1680: finishing write tid 5 to smithi16367185-35 2023-10-09T09:12:33.380 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 470 (ObjNum 643 snap 205 seq_num 643) dirty exists 2023-10-09T09:12:33.380 INFO:tasks.rados.rados.0.smithi163.stdout:1680: left oid 35 (ObjNum 643 snap 205 seq_num 643) 2023-10-09T09:12:33.380 INFO:tasks.rados.rados.0.smithi163.stdout:1679: done (2 left) 2023-10-09T09:12:33.380 INFO:tasks.rados.rados.0.smithi163.stdout:1680: done (1 left) 2023-10-09T09:12:33.380 INFO:tasks.rados.rados.0.smithi163.stdout:1681: done (0 left) 2023-10-09T09:12:33.381 INFO:tasks.rados.rados.0.smithi163.stdout:1682: write oid 12 current snap is 205 2023-10-09T09:12:33.381 INFO:tasks.rados.rados.0.smithi163.stdout:1682: seq_num 645 ranges {440515=709257,1910768=583545} 2023-10-09T09:12:33.386 INFO:tasks.rados.rados.0.smithi163.stdout:1682: writing smithi16367185-12 from 440515 to 1149772 tid 1 2023-10-09T09:12:33.394 INFO:tasks.rados.rados.0.smithi163.stdout:1682: writing smithi16367185-12 from 1910768 to 2494313 tid 2 2023-10-09T09:12:33.394 INFO:tasks.rados.rados.0.smithi163.stdout:1683: read oid 9 snap -1 2023-10-09T09:12:33.395 INFO:tasks.rados.rados.0.smithi163.stdout:1683: expect deleted 2023-10-09T09:12:33.395 INFO:tasks.rados.rados.0.smithi163.stdout:1684: write oid 23 current snap is 205 2023-10-09T09:12:33.395 INFO:tasks.rados.rados.0.smithi163.stdout:1682: finishing write tid 1 to smithi16367185-12 2023-10-09T09:12:33.395 INFO:tasks.rados.rados.0.smithi163.stdout:1684: seq_num 646 ranges {457322=784506,1969250=443556} 2023-10-09T09:12:33.402 INFO:tasks.rados.rados.0.smithi163.stdout:1684: writing smithi16367185-23 from 457322 to 1241828 tid 1 2023-10-09T09:12:33.409 INFO:tasks.rados.rados.0.smithi163.stdout:1684: writing smithi16367185-23 from 1969250 to 2412806 tid 2 2023-10-09T09:12:33.409 INFO:tasks.rados.rados.0.smithi163.stdout:1685: read oid 19 snap -1 2023-10-09T09:12:33.409 INFO:tasks.rados.rados.0.smithi163.stdout:1685: expect deleted 2023-10-09T09:12:33.409 INFO:tasks.rados.rados.0.smithi163.stdout:1686: delete oid 21 current snap is 205 2023-10-09T09:12:33.410 INFO:tasks.rados.rados.0.smithi163.stdout:1682: finishing write tid 2 to smithi16367185-12 2023-10-09T09:12:33.410 INFO:tasks.rados.rados.0.smithi163.stdout:1682: finishing write tid 3 to smithi16367185-12 2023-10-09T09:12:33.410 INFO:tasks.rados.rados.0.smithi163.stdout:1682: finishing write tid 4 to smithi16367185-12 2023-10-09T09:12:33.410 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 762 (ObjNum 645 snap 205 seq_num 645) dirty exists 2023-10-09T09:12:33.410 INFO:tasks.rados.rados.0.smithi163.stdout:1682: left oid 12 (ObjNum 645 snap 205 seq_num 645) 2023-10-09T09:12:33.412 INFO:tasks.rados.rados.0.smithi163.stdout:1682: done (4 left) 2023-10-09T09:12:33.412 INFO:tasks.rados.rados.0.smithi163.stdout:1683: done (3 left) 2023-10-09T09:12:33.412 INFO:tasks.rados.rados.0.smithi163.stdout:1685: done (2 left) 2023-10-09T09:12:33.412 INFO:tasks.rados.rados.0.smithi163.stdout:1686: done (1 left) 2023-10-09T09:12:33.412 INFO:tasks.rados.rados.0.smithi163.stdout:1687: write oid 1 current snap is 205 2023-10-09T09:12:33.412 INFO:tasks.rados.rados.0.smithi163.stdout:1687: seq_num 648 ranges {490936=651357,1686214=447225,2861864=29072} 2023-10-09T09:12:33.418 INFO:tasks.rados.rados.0.smithi163.stdout:1687: writing smithi16367185-1 from 490936 to 1142293 tid 1 2023-10-09T09:12:33.424 INFO:tasks.rados.rados.0.smithi163.stdout:1687: writing smithi16367185-1 from 1686214 to 2133439 tid 2 2023-10-09T09:12:33.429 INFO:tasks.rados.rados.0.smithi163.stdout:1687: writing smithi16367185-1 from 2861864 to 2890936 tid 3 2023-10-09T09:12:33.429 INFO:tasks.rados.rados.0.smithi163.stdout:1688: delete oid 37 current snap is 205 2023-10-09T09:12:33.429 INFO:tasks.rados.rados.0.smithi163.stdout:1684: finishing write tid 1 to smithi16367185-23 2023-10-09T09:12:33.429 INFO:tasks.rados.rados.0.smithi163.stdout:1684: finishing write tid 2 to smithi16367185-23 2023-10-09T09:12:33.430 INFO:tasks.rados.rados.0.smithi163.stdout:1684: finishing write tid 3 to smithi16367185-23 2023-10-09T09:12:33.430 INFO:tasks.rados.rados.0.smithi163.stdout:1684: finishing write tid 4 to smithi16367185-23 2023-10-09T09:12:33.430 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 674 (ObjNum 646 snap 205 seq_num 646) dirty exists 2023-10-09T09:12:33.430 INFO:tasks.rados.rados.0.smithi163.stdout:1684: left oid 23 (ObjNum 646 snap 205 seq_num 646) 2023-10-09T09:12:33.430 INFO:tasks.rados.rados.0.smithi163.stdout:1687: finishing write tid 1 to smithi16367185-1 2023-10-09T09:12:33.430 INFO:tasks.rados.rados.0.smithi163.stdout:1687: finishing write tid 2 to smithi16367185-1 2023-10-09T09:12:33.431 INFO:tasks.rados.rados.0.smithi163.stdout:1687: finishing write tid 3 to smithi16367185-1 2023-10-09T09:12:33.431 INFO:tasks.rados.rados.0.smithi163.stdout:1687: finishing write tid 4 to smithi16367185-1 2023-10-09T09:12:33.431 INFO:tasks.rados.rados.0.smithi163.stdout:1687: finishing write tid 5 to smithi16367185-1 2023-10-09T09:12:33.431 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 923 (ObjNum 648 snap 205 seq_num 648) dirty exists 2023-10-09T09:12:33.431 INFO:tasks.rados.rados.0.smithi163.stdout:1687: left oid 1 (ObjNum 648 snap 205 seq_num 648) 2023-10-09T09:12:33.432 INFO:tasks.rados.rados.0.smithi163.stdout:1684: done (2 left) 2023-10-09T09:12:33.432 INFO:tasks.rados.rados.0.smithi163.stdout:1687: done (1 left) 2023-10-09T09:12:33.432 INFO:tasks.rados.rados.0.smithi163.stdout:1688: done (0 left) 2023-10-09T09:12:33.432 INFO:tasks.rados.rados.0.smithi163.stdout:1689: read oid 28 snap -1 2023-10-09T09:12:33.432 INFO:tasks.rados.rados.0.smithi163.stdout:1689: expect deleted 2023-10-09T09:12:33.432 INFO:tasks.rados.rados.0.smithi163.stdout:1690: snap_create 2023-10-09T09:12:33.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:12:33] "GET /metrics HTTP/1.1" 200 36085 "" "Prometheus/2.33.4" 2023-10-09T09:12:33.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[78107]: cluster 2023-10-09T09:12:31.465046+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:33.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[78107]: cluster 2023-10-09T09:12:31.468050+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:33.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[78107]: cluster 2023-10-09T09:12:32.171251+0000 osd.1 (osd.1) 56 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[78107]: cluster 2023-10-09T09:12:32.172039+0000 osd.2 (osd.2) 118 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[78107]: cluster 2023-10-09T09:12:32.176209+0000 mon.a (mon.0) 360 : cluster 0 osdmap e480: 8 total, 8 up, 8 in 2023-10-09T09:12:33.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[75425]: cluster 2023-10-09T09:12:31.465046+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:33.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[75425]: cluster 2023-10-09T09:12:31.468050+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:33.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[75425]: cluster 2023-10-09T09:12:32.171251+0000 osd.1 (osd.1) 56 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:33.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[75425]: cluster 2023-10-09T09:12:32.172039+0000 osd.2 (osd.2) 118 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:33.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:33 smithi163 bash[75425]: cluster 2023-10-09T09:12:32.176209+0000 mon.a (mon.0) 360 : cluster 0 osdmap e480: 8 total, 8 up, 8 in 2023-10-09T09:12:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:33 smithi181 bash[59345]: cluster 2023-10-09T09:12:31.465046+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:33 smithi181 bash[59345]: cluster 2023-10-09T09:12:31.468050+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e479 with expected crc 2023-10-09T09:12:33.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:33 smithi181 bash[59345]: cluster 2023-10-09T09:12:32.171251+0000 osd.1 (osd.1) 56 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:33.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:33 smithi181 bash[59345]: cluster 2023-10-09T09:12:32.172039+0000 osd.2 (osd.2) 118 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:33.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:33 smithi181 bash[59345]: cluster 2023-10-09T09:12:32.176209+0000 mon.a (mon.0) 360 : cluster 0 osdmap e480: 8 total, 8 up, 8 in 2023-10-09T09:12:34.200 INFO:tasks.rados.rados.0.smithi163.stdout:1689: done (1 left) 2023-10-09T09:12:34.201 INFO:tasks.rados.rados.0.smithi163.stdout:1690: done (0 left) 2023-10-09T09:12:34.201 INFO:tasks.rados.rados.0.smithi163.stdout:1691: read oid 48 snap 205 2023-10-09T09:12:34.201 INFO:tasks.rados.rados.0.smithi163.stdout:1691: expect deleted 2023-10-09T09:12:34.201 INFO:tasks.rados.rados.0.smithi163.stdout:1692: rollback oid 3 current snap is 206 2023-10-09T09:12:34.201 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 205 2023-10-09T09:12:34.201 INFO:tasks.rados.rados.0.smithi163.stdout:1693: read oid 15 snap -1 2023-10-09T09:12:34.201 INFO:tasks.rados.rados.0.smithi163.stdout:1693: expect deleted 2023-10-09T09:12:34.201 INFO:tasks.rados.rados.0.smithi163.stdout:1694: delete oid 42 current snap is 206 2023-10-09T09:12:34.211 INFO:tasks.rados.rados.0.smithi163.stdout:1692: finishing rollback tid 0 to smithi16367185-3 2023-10-09T09:12:34.212 INFO:tasks.rados.rados.0.smithi163.stdout:1692: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:12:34.212 INFO:tasks.rados.rados.0.smithi163.stdout:1692: finishing rollback tid 2 to smithi16367185-3 2023-10-09T09:12:34.212 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 454 (ObjNum 579 snap 181 seq_num 579) dirty exists 2023-10-09T09:12:34.215 INFO:tasks.rados.rados.0.smithi163.stdout:1691: done (3 left) 2023-10-09T09:12:34.215 INFO:tasks.rados.rados.0.smithi163.stdout:1692: done (2 left) 2023-10-09T09:12:34.215 INFO:tasks.rados.rados.0.smithi163.stdout:1693: done (1 left) 2023-10-09T09:12:34.215 INFO:tasks.rados.rados.0.smithi163.stdout:1694: done (0 left) 2023-10-09T09:12:34.215 INFO:tasks.rados.rados.0.smithi163.stdout:1695: snap_remove snap 204 2023-10-09T09:12:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:34 smithi163 bash[75425]: cluster 2023-10-09T09:12:33.185203+0000 osd.3 (osd.3) 127 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:34 smithi163 bash[75425]: cluster 2023-10-09T09:12:33.185967+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e481 with expected crc 2023-10-09T09:12:34.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:34 smithi163 bash[75425]: cluster 2023-10-09T09:12:33.187223+0000 mon.a (mon.0) 361 : cluster 0 osdmap e481: 8 total, 8 up, 8 in 2023-10-09T09:12:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:34 smithi163 bash[75425]: cluster 2023-10-09T09:12:33.189402+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:34 smithi163 bash[75425]: cluster 2023-10-09T09:12:33.276665+0000 mgr.y (mgr.34107) 123 : cluster 0 pgmap v274: 89 pgs: 89 active+clean; 85 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 525 KiB/s wr, 2 op/s 2023-10-09T09:12:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:34 smithi163 bash[78107]: cluster 2023-10-09T09:12:33.185203+0000 osd.3 (osd.3) 127 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:34 smithi163 bash[78107]: cluster 2023-10-09T09:12:33.185967+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e481 with expected crc 2023-10-09T09:12:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:34 smithi163 bash[78107]: cluster 2023-10-09T09:12:33.187223+0000 mon.a (mon.0) 361 : cluster 0 osdmap e481: 8 total, 8 up, 8 in 2023-10-09T09:12:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:34 smithi163 bash[78107]: cluster 2023-10-09T09:12:33.189402+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:34 smithi163 bash[78107]: cluster 2023-10-09T09:12:33.276665+0000 mgr.y (mgr.34107) 123 : cluster 0 pgmap v274: 89 pgs: 89 active+clean; 85 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 525 KiB/s wr, 2 op/s 2023-10-09T09:12:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:34 smithi181 bash[59345]: cluster 2023-10-09T09:12:33.185203+0000 osd.3 (osd.3) 127 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:34 smithi181 bash[59345]: cluster 2023-10-09T09:12:33.185967+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e481 with expected crc 2023-10-09T09:12:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:34 smithi181 bash[59345]: cluster 2023-10-09T09:12:33.187223+0000 mon.a (mon.0) 361 : cluster 0 osdmap e481: 8 total, 8 up, 8 in 2023-10-09T09:12:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:34 smithi181 bash[59345]: cluster 2023-10-09T09:12:33.189402+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e480 with expected crc 2023-10-09T09:12:34.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:34 smithi181 bash[59345]: cluster 2023-10-09T09:12:33.276665+0000 mgr.y (mgr.34107) 123 : cluster 0 pgmap v274: 89 pgs: 89 active+clean; 85 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 525 KiB/s wr, 2 op/s 2023-10-09T09:12:34.590 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (5m) - 10m 13.5M - 0.23.0 ba2b418f427c 82a8abc84166 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (4m) - 10m 40.6M - 8.3.5 dad864ee21e9 9d80a71ddd1d 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.vnodzj smithi163 running (9m) - 9m 18.1M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 892077b22450 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.ajqgsc smithi181 running (9m) - 9m 13.6M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 01d89554089b 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443,9283,8765 running (3m) - 17m 427M - 18.2.0-598-gb4fa663f 3060897fdf97 ca7a91dd3dab 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (4m) - 19m 477M - 18.2.0-598-gb4fa663f 3060897fdf97 fc427389eb0c 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (3m) - 19m 63.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 097aa5987723 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 starting - - - 2048M 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (103s) - 17m 40.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 adc323592e84 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (5m) - 11m 8899k - 1.3.1 1dbe0e931976 8a6fec29baf3 2023-10-09T09:12:34.591 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (4m) - 11m 9455k - 1.3.1 1dbe0e931976 3f766d093db8 2023-10-09T09:12:34.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (16m) - 16m 96.8M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d07769e94b6f 2023-10-09T09:12:34.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (16m) - 16m 82.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7a2783c3c11c 2023-10-09T09:12:34.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (15m) - 15m 106M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 35670f565f8c 2023-10-09T09:12:34.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (14m) - 14m 69.5M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 402aa34f21f4 2023-10-09T09:12:34.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (14m) - 14m 96.7M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5dd6bc4b279 2023-10-09T09:12:34.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (13m) - 13m 101M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9f3e4fbba907 2023-10-09T09:12:34.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (12m) - 12m 70.5M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 30f1ac2f4524 2023-10-09T09:12:34.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (12m) - 12m 83.7M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 54105d827fdd 2023-10-09T09:12:34.592 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (3m) - 11m 56.4M - 2.33.4 514e6a882f6e cc81501e414f 2023-10-09T09:12:35.040 INFO:tasks.rados.rados.0.smithi163.stdout:1695: done (0 left) 2023-10-09T09:12:35.041 INFO:tasks.rados.rados.0.smithi163.stdout:1696: read oid 48 snap 205 2023-10-09T09:12:35.041 INFO:tasks.rados.rados.0.smithi163.stdout:1696: expect deleted 2023-10-09T09:12:35.041 INFO:tasks.rados.rados.0.smithi163.stdout:1697: read oid 31 snap -1 2023-10-09T09:12:35.041 INFO:tasks.rados.rados.0.smithi163.stdout:1697: expect (ObjNum 641 snap 204 seq_num 641) 2023-10-09T09:12:35.041 INFO:tasks.rados.rados.0.smithi163.stdout:1698: rollback oid 7 current snap is 206 2023-10-09T09:12:35.041 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 205 2023-10-09T09:12:35.041 INFO:tasks.rados.rados.0.smithi163.stdout:1699: snap_create 2023-10-09T09:12:35.048 INFO:tasks.rados.rados.0.smithi163.stdout:1698: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:12:35.048 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 0 (ObjNum 0 snap 32603 seq_num 698877760) dirty dne 2023-10-09T09:12:35.059 INFO:tasks.rados.rados.0.smithi163.stdout:1697: expect (ObjNum 641 snap 204 seq_num 641) 2023-10-09T09:12:35.366 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T09:12:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[75425]: cluster 2023-10-09T09:12:34.194129+0000 osd.1 (osd.1) 57 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[75425]: cluster 2023-10-09T09:12:34.195056+0000 osd.2 (osd.2) 119 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[75425]: cluster 2023-10-09T09:12:34.200152+0000 mon.a (mon.0) 362 : cluster 0 osdmap e482: 8 total, 8 up, 8 in 2023-10-09T09:12:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[75425]: cluster 2023-10-09T09:12:34.202542+0000 osd.0 (osd.0) 75 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[75425]: cluster 2023-10-09T09:12:34.203458+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[75425]: cluster 2023-10-09T09:12:34.204011+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[75425]: cluster 2023-10-09T09:12:35.040601+0000 mon.a (mon.0) 363 : cluster 0 osdmap e483: 8 total, 8 up, 8 in 2023-10-09T09:12:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[78107]: cluster 2023-10-09T09:12:34.194129+0000 osd.1 (osd.1) 57 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[78107]: cluster 2023-10-09T09:12:34.195056+0000 osd.2 (osd.2) 119 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[78107]: cluster 2023-10-09T09:12:34.200152+0000 mon.a (mon.0) 362 : cluster 0 osdmap e482: 8 total, 8 up, 8 in 2023-10-09T09:12:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[78107]: cluster 2023-10-09T09:12:34.202542+0000 osd.0 (osd.0) 75 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[78107]: cluster 2023-10-09T09:12:34.203458+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[78107]: cluster 2023-10-09T09:12:34.204011+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:35 smithi163 bash[78107]: cluster 2023-10-09T09:12:35.040601+0000 mon.a (mon.0) 363 : cluster 0 osdmap e483: 8 total, 8 up, 8 in 2023-10-09T09:12:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:35 smithi181 bash[59345]: cluster 2023-10-09T09:12:34.194129+0000 osd.1 (osd.1) 57 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:35 smithi181 bash[59345]: cluster 2023-10-09T09:12:34.195056+0000 osd.2 (osd.2) 119 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:35 smithi181 bash[59345]: cluster 2023-10-09T09:12:34.200152+0000 mon.a (mon.0) 362 : cluster 0 osdmap e482: 8 total, 8 up, 8 in 2023-10-09T09:12:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:35 smithi181 bash[59345]: cluster 2023-10-09T09:12:34.202542+0000 osd.0 (osd.0) 75 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:35 smithi181 bash[59345]: cluster 2023-10-09T09:12:34.203458+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:35 smithi181 bash[59345]: cluster 2023-10-09T09:12:34.204011+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e482 with expected crc 2023-10-09T09:12:35.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:35 smithi181 bash[59345]: cluster 2023-10-09T09:12:35.040601+0000 mon.a (mon.0) 363 : cluster 0 osdmap e483: 8 total, 8 up, 8 in 2023-10-09T09:12:36.044 INFO:tasks.rados.rados.0.smithi163.stdout:1696: done (3 left) 2023-10-09T09:12:36.044 INFO:tasks.rados.rados.0.smithi163.stdout:1697: done (2 left) 2023-10-09T09:12:36.044 INFO:tasks.rados.rados.0.smithi163.stdout:1698: done (1 left) 2023-10-09T09:12:36.044 INFO:tasks.rados.rados.0.smithi163.stdout:1699: done (0 left) 2023-10-09T09:12:36.044 INFO:tasks.rados.rados.0.smithi163.stdout:1700: write oid 47 current snap is 207 2023-10-09T09:12:36.045 INFO:tasks.rados.rados.0.smithi163.stdout:1700: seq_num 651 ranges {541357=477104,1745307=594552} 2023-10-09T09:12:36.053 INFO:tasks.rados.rados.0.smithi163.stdout:1700: writing smithi16367185-47 from 541357 to 1018461 tid 1 2023-10-09T09:12:36.061 INFO:tasks.rados.rados.0.smithi163.stdout:1700: writing smithi16367185-47 from 1745307 to 2339859 tid 2 2023-10-09T09:12:36.062 INFO:tasks.rados.rados.0.smithi163.stdout:1701: snap_create 2023-10-09T09:12:36.062 INFO:tasks.rados.rados.0.smithi163.stdout:1700: finishing write tid 1 to smithi16367185-47 2023-10-09T09:12:36.066 INFO:tasks.rados.rados.0.smithi163.stdout:1700: finishing write tid 2 to smithi16367185-47 2023-10-09T09:12:36.067 INFO:tasks.rados.rados.0.smithi163.stdout:1700: finishing write tid 3 to smithi16367185-47 2023-10-09T09:12:36.067 INFO:tasks.rados.rados.0.smithi163.stdout:1700: finishing write tid 4 to smithi16367185-47 2023-10-09T09:12:36.067 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 1305 (ObjNum 651 snap 207 seq_num 651) dirty exists 2023-10-09T09:12:36.067 INFO:tasks.rados.rados.0.smithi163.stdout:1700: left oid 47 (ObjNum 651 snap 207 seq_num 651) 2023-10-09T09:12:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[78107]: audit 2023-10-09T09:12:34.572030+0000 mgr.y (mgr.34107) 124 : audit 0 from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[78107]: cluster 2023-10-09T09:12:35.038915+0000 osd.1 (osd.1) 58 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[78107]: cluster 2023-10-09T09:12:35.040016+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[78107]: cluster 2023-10-09T09:12:35.041015+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[78107]: cluster 2023-10-09T09:12:35.042599+0000 osd.1 (osd.1) 59 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[78107]: cluster 2023-10-09T09:12:35.046318+0000 osd.0 (osd.0) 76 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[78107]: cluster 2023-10-09T09:12:35.277341+0000 mgr.y (mgr.34107) 125 : cluster 0 pgmap v277: 89 pgs: 89 active+clean; 85 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[78107]: audit 2023-10-09T09:12:35.454830+0000 mon.a (mon.0) 364 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[78107]: audit 2023-10-09T09:12:35.462687+0000 mon.a (mon.0) 365 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[78107]: cluster 2023-10-09T09:12:36.043779+0000 mon.a (mon.0) 366 : cluster 0 osdmap e484: 8 total, 8 up, 8 in 2023-10-09T09:12:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[75425]: audit 2023-10-09T09:12:34.572030+0000 mgr.y (mgr.34107) 124 : audit 0 from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[75425]: cluster 2023-10-09T09:12:35.038915+0000 osd.1 (osd.1) 58 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[75425]: cluster 2023-10-09T09:12:35.040016+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[75425]: cluster 2023-10-09T09:12:35.041015+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[75425]: cluster 2023-10-09T09:12:35.042599+0000 osd.1 (osd.1) 59 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[75425]: cluster 2023-10-09T09:12:35.046318+0000 osd.0 (osd.0) 76 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[75425]: cluster 2023-10-09T09:12:35.277341+0000 mgr.y (mgr.34107) 125 : cluster 0 pgmap v277: 89 pgs: 89 active+clean; 85 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:36.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[75425]: audit 2023-10-09T09:12:35.454830+0000 mon.a (mon.0) 364 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:36.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[75425]: audit 2023-10-09T09:12:35.462687+0000 mon.a (mon.0) 365 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:36.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:36 smithi163 bash[75425]: cluster 2023-10-09T09:12:36.043779+0000 mon.a (mon.0) 366 : cluster 0 osdmap e484: 8 total, 8 up, 8 in 2023-10-09T09:12:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:36 smithi181 bash[59345]: audit 2023-10-09T09:12:34.572030+0000 mgr.y (mgr.34107) 124 : audit 0 from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:36 smithi181 bash[59345]: cluster 2023-10-09T09:12:35.038915+0000 osd.1 (osd.1) 58 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:36 smithi181 bash[59345]: cluster 2023-10-09T09:12:35.040016+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:36 smithi181 bash[59345]: cluster 2023-10-09T09:12:35.041015+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:36 smithi181 bash[59345]: cluster 2023-10-09T09:12:35.042599+0000 osd.1 (osd.1) 59 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:36 smithi181 bash[59345]: cluster 2023-10-09T09:12:35.046318+0000 osd.0 (osd.0) 76 : cluster 3 failed to encode map e483 with expected crc 2023-10-09T09:12:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:36 smithi181 bash[59345]: cluster 2023-10-09T09:12:35.277341+0000 mgr.y (mgr.34107) 125 : cluster 0 pgmap v277: 89 pgs: 89 active+clean; 85 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:36 smithi181 bash[59345]: audit 2023-10-09T09:12:35.454830+0000 mon.a (mon.0) 364 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:36 smithi181 bash[59345]: audit 2023-10-09T09:12:35.462687+0000 mon.a (mon.0) 365 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:36.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:36 smithi181 bash[59345]: cluster 2023-10-09T09:12:36.043779+0000 mon.a (mon.0) 366 : cluster 0 osdmap e484: 8 total, 8 up, 8 in 2023-10-09T09:12:37.047 INFO:tasks.rados.rados.0.smithi163.stdout:1700: done (1 left) 2023-10-09T09:12:37.047 INFO:tasks.rados.rados.0.smithi163.stdout:1701: done (0 left) 2023-10-09T09:12:37.047 INFO:tasks.rados.rados.0.smithi163.stdout:1702: write oid 37 current snap is 208 2023-10-09T09:12:37.047 INFO:tasks.rados.rados.0.smithi163.stdout:1702: seq_num 652 ranges {558164=552353,1803789=454563,2855203=102961} 2023-10-09T09:12:37.053 INFO:tasks.rados.rados.0.smithi163.stdout:1702: writing smithi16367185-37 from 558164 to 1110517 tid 1 2023-10-09T09:12:37.060 INFO:tasks.rados.rados.0.smithi163.stdout:1702: writing smithi16367185-37 from 1803789 to 2258352 tid 2 2023-10-09T09:12:37.064 INFO:tasks.rados.rados.0.smithi163.stdout:1702: writing smithi16367185-37 from 2855203 to 2958164 tid 3 2023-10-09T09:12:37.064 INFO:tasks.rados.rados.0.smithi163.stdout:1703: snap_remove snap 205 2023-10-09T09:12:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:37 smithi181 bash[59345]: cluster 2023-10-09T09:12:36.039564+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:37 smithi181 bash[59345]: cluster 2023-10-09T09:12:36.042129+0000 osd.2 (osd.2) 120 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:37 smithi181 bash[59345]: audit 2023-10-09T09:12:36.258215+0000 mon.a (mon.0) 367 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:37 smithi181 bash[59345]: audit 2023-10-09T09:12:36.268156+0000 mon.a (mon.0) 368 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:37 smithi181 bash[59345]: cluster 2023-10-09T09:12:36.374558+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:37.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:37 smithi181 bash[59345]: cluster 2023-10-09T09:12:37.047384+0000 mon.a (mon.0) 369 : cluster 0 osdmap e485: 8 total, 8 up, 8 in 2023-10-09T09:12:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[78107]: cluster 2023-10-09T09:12:36.039564+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[78107]: cluster 2023-10-09T09:12:36.042129+0000 osd.2 (osd.2) 120 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[78107]: audit 2023-10-09T09:12:36.258215+0000 mon.a (mon.0) 367 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[78107]: audit 2023-10-09T09:12:36.268156+0000 mon.a (mon.0) 368 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[78107]: cluster 2023-10-09T09:12:36.374558+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[78107]: cluster 2023-10-09T09:12:37.047384+0000 mon.a (mon.0) 369 : cluster 0 osdmap e485: 8 total, 8 up, 8 in 2023-10-09T09:12:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[75425]: cluster 2023-10-09T09:12:36.039564+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[75425]: cluster 2023-10-09T09:12:36.042129+0000 osd.2 (osd.2) 120 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[75425]: audit 2023-10-09T09:12:36.258215+0000 mon.a (mon.0) 367 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[75425]: audit 2023-10-09T09:12:36.268156+0000 mon.a (mon.0) 368 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[75425]: cluster 2023-10-09T09:12:36.374558+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:37 smithi163 bash[75425]: cluster 2023-10-09T09:12:37.047384+0000 mon.a (mon.0) 369 : cluster 0 osdmap e485: 8 total, 8 up, 8 in 2023-10-09T09:12:38.049 INFO:tasks.rados.rados.0.smithi163.stdout:1703: done (1 left) 2023-10-09T09:12:38.050 INFO:tasks.rados.rados.0.smithi163.stdout:1704: write oid 38 current snap is 208 2023-10-09T09:12:38.050 INFO:tasks.rados.rados.0.smithi163.stdout:1702: finishing write tid 1 to smithi16367185-37 2023-10-09T09:12:38.050 INFO:tasks.rados.rados.0.smithi163.stdout:1702: finishing write tid 2 to smithi16367185-37 2023-10-09T09:12:38.050 INFO:tasks.rados.rados.0.smithi163.stdout:1702: finishing write tid 3 to smithi16367185-37 2023-10-09T09:12:38.050 INFO:tasks.rados.rados.0.smithi163.stdout:1702: finishing write tid 4 to smithi16367185-37 2023-10-09T09:12:38.050 INFO:tasks.rados.rados.0.smithi163.stdout:1704: seq_num 653 ranges {574971=627602,1862271=598221} 2023-10-09T09:12:38.060 INFO:tasks.rados.rados.0.smithi163.stdout:1704: writing smithi16367185-38 from 574971 to 1202573 tid 1 2023-10-09T09:12:38.068 INFO:tasks.rados.rados.0.smithi163.stdout:1704: writing smithi16367185-38 from 1862271 to 2460492 tid 2 2023-10-09T09:12:38.068 INFO:tasks.rados.rados.0.smithi163.stdout:1705: read oid 9 snap -1 2023-10-09T09:12:38.069 INFO:tasks.rados.rados.0.smithi163.stdout:1705: expect deleted 2023-10-09T09:12:38.069 INFO:tasks.rados.rados.0.smithi163.stdout:1706: read oid 4 snap -1 2023-10-09T09:12:38.069 INFO:tasks.rados.rados.0.smithi163.stdout:1706: expect (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:12:38.069 INFO:tasks.rados.rados.0.smithi163.stdout:1707: delete oid 47 current snap is 208 2023-10-09T09:12:38.069 INFO:tasks.rados.rados.0.smithi163.stdout:1702: finishing write tid 5 to smithi16367185-37 2023-10-09T09:12:38.069 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 853 (ObjNum 652 snap 208 seq_num 652) dirty exists 2023-10-09T09:12:38.069 INFO:tasks.rados.rados.0.smithi163.stdout:1702: left oid 37 (ObjNum 652 snap 208 seq_num 652) 2023-10-09T09:12:38.078 INFO:tasks.rados.rados.0.smithi163.stdout:1702: done (4 left) 2023-10-09T09:12:38.078 INFO:tasks.rados.rados.0.smithi163.stdout:1705: done (3 left) 2023-10-09T09:12:38.078 INFO:tasks.rados.rados.0.smithi163.stdout:1707: done (2 left) 2023-10-09T09:12:38.078 INFO:tasks.rados.rados.0.smithi163.stdout:1708: write oid 12 current snap is 208 2023-10-09T09:12:38.078 INFO:tasks.rados.rados.0.smithi163.stdout:1708: seq_num 655 ranges {608585=494453,1579235=601890,2821119=187466} 2023-10-09T09:12:38.086 INFO:tasks.rados.rados.0.smithi163.stdout:1708: writing smithi16367185-12 from 608585 to 1103038 tid 1 2023-10-09T09:12:38.093 INFO:tasks.rados.rados.0.smithi163.stdout:1708: writing smithi16367185-12 from 1579235 to 2181125 tid 2 2023-10-09T09:12:38.098 INFO:tasks.rados.rados.0.smithi163.stdout:1708: writing smithi16367185-12 from 2821119 to 3008585 tid 3 2023-10-09T09:12:38.098 INFO:tasks.rados.rados.0.smithi163.stdout:1709: delete oid 45 current snap is 208 2023-10-09T09:12:38.098 INFO:tasks.rados.rados.0.smithi163.stdout:1706: expect (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:12:38.234 INFO:tasks.rados.rados.0.smithi163.stdout:1704: finishing write tid 1 to smithi16367185-38 2023-10-09T09:12:38.234 INFO:tasks.rados.rados.0.smithi163.stdout:1706: done (3 left) 2023-10-09T09:12:38.234 INFO:tasks.rados.rados.0.smithi163.stdout:1709: done (2 left) 2023-10-09T09:12:38.235 INFO:tasks.rados.rados.0.smithi163.stdout:1710: snap_create 2023-10-09T09:12:38.235 INFO:tasks.rados.rados.0.smithi163.stdout:1704: finishing write tid 2 to smithi16367185-38 2023-10-09T09:12:38.235 INFO:tasks.rados.rados.0.smithi163.stdout:1704: finishing write tid 3 to smithi16367185-38 2023-10-09T09:12:38.235 INFO:tasks.rados.rados.0.smithi163.stdout:1704: finishing write tid 4 to smithi16367185-38 2023-10-09T09:12:38.235 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 685 (ObjNum 653 snap 208 seq_num 653) dirty exists 2023-10-09T09:12:38.235 INFO:tasks.rados.rados.0.smithi163.stdout:1704: left oid 38 (ObjNum 653 snap 208 seq_num 653) 2023-10-09T09:12:38.235 INFO:tasks.rados.rados.0.smithi163.stdout:1708: finishing write tid 1 to smithi16367185-12 2023-10-09T09:12:38.235 INFO:tasks.rados.rados.0.smithi163.stdout:1708: finishing write tid 2 to smithi16367185-12 2023-10-09T09:12:38.236 INFO:tasks.rados.rados.0.smithi163.stdout:1708: finishing write tid 3 to smithi16367185-12 2023-10-09T09:12:38.236 INFO:tasks.rados.rados.0.smithi163.stdout:1708: finishing write tid 4 to smithi16367185-12 2023-10-09T09:12:38.236 INFO:tasks.rados.rados.0.smithi163.stdout:1708: finishing write tid 5 to smithi16367185-12 2023-10-09T09:12:38.236 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 767 (ObjNum 655 snap 208 seq_num 655) dirty exists 2023-10-09T09:12:38.236 INFO:tasks.rados.rados.0.smithi163.stdout:1708: left oid 12 (ObjNum 655 snap 208 seq_num 655) 2023-10-09T09:12:38.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[78107]: cluster 2023-10-09T09:12:36.628915+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:38.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[78107]: cluster 2023-10-09T09:12:37.042003+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[78107]: cluster 2023-10-09T09:12:37.042646+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[78107]: cluster 2023-10-09T09:12:37.054344+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[78107]: cluster 2023-10-09T09:12:37.278036+0000 mgr.y (mgr.34107) 126 : cluster 0 pgmap v280: 89 pgs: 89 active+clean; 80 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:12:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[78107]: cluster 2023-10-09T09:12:38.049587+0000 mon.a (mon.0) 370 : cluster 0 osdmap e486: 8 total, 8 up, 8 in 2023-10-09T09:12:38.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[75425]: cluster 2023-10-09T09:12:36.628915+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:38.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[75425]: cluster 2023-10-09T09:12:37.042003+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:38.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[75425]: cluster 2023-10-09T09:12:37.042646+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:38.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[75425]: cluster 2023-10-09T09:12:37.054344+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:38.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[75425]: cluster 2023-10-09T09:12:37.278036+0000 mgr.y (mgr.34107) 126 : cluster 0 pgmap v280: 89 pgs: 89 active+clean; 80 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:12:38.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:38 smithi163 bash[75425]: cluster 2023-10-09T09:12:38.049587+0000 mon.a (mon.0) 370 : cluster 0 osdmap e486: 8 total, 8 up, 8 in 2023-10-09T09:12:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:38 smithi181 bash[59345]: cluster 2023-10-09T09:12:36.628915+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e484 with expected crc 2023-10-09T09:12:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:38 smithi181 bash[59345]: cluster 2023-10-09T09:12:37.042003+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:38 smithi181 bash[59345]: cluster 2023-10-09T09:12:37.042646+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:38 smithi181 bash[59345]: cluster 2023-10-09T09:12:37.054344+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:38 smithi181 bash[59345]: cluster 2023-10-09T09:12:37.278036+0000 mgr.y (mgr.34107) 126 : cluster 0 pgmap v280: 89 pgs: 89 active+clean; 80 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:12:38.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:38 smithi181 bash[59345]: cluster 2023-10-09T09:12:38.049587+0000 mon.a (mon.0) 370 : cluster 0 osdmap e486: 8 total, 8 up, 8 in 2023-10-09T09:12:38.622 INFO:teuthology.orchestra.run.smithi163.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:39.056 INFO:tasks.rados.rados.0.smithi163.stdout:1704: done (2 left) 2023-10-09T09:12:39.056 INFO:tasks.rados.rados.0.smithi163.stdout:1708: done (1 left) 2023-10-09T09:12:39.056 INFO:tasks.rados.rados.0.smithi163.stdout:1710: done (0 left) 2023-10-09T09:12:39.056 INFO:tasks.rados.rados.0.smithi163.stdout:1711: snap_remove snap 207 2023-10-09T09:12:39.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T09:12:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:39 smithi181 bash[59345]: cluster 2023-10-09T09:12:37.640907+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:39 smithi181 bash[59345]: cluster 2023-10-09T09:12:37.817655+0000 osd.0 (osd.0) 77 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:39 smithi181 bash[59345]: cluster 2023-10-09T09:12:38.046254+0000 osd.1 (osd.1) 60 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:39 smithi181 bash[59345]: cluster 2023-10-09T09:12:38.048455+0000 osd.2 (osd.2) 121 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:39 smithi181 bash[59345]: cluster 2023-10-09T09:12:38.061659+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:39 smithi181 bash[59345]: cluster 2023-10-09T09:12:38.070220+0000 osd.3 (osd.3) 128 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:39 smithi181 bash[59345]: cluster 2023-10-09T09:12:38.374857+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:39 smithi181 bash[59345]: audit 2023-10-09T09:12:38.621997+0000 mon.a (mon.0) 371 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:39.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:39 smithi181 bash[59345]: cluster 2023-10-09T09:12:39.056211+0000 mon.a (mon.0) 372 : cluster 0 osdmap e487: 8 total, 8 up, 8 in 2023-10-09T09:12:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[78107]: cluster 2023-10-09T09:12:37.640907+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[78107]: cluster 2023-10-09T09:12:37.817655+0000 osd.0 (osd.0) 77 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[78107]: cluster 2023-10-09T09:12:38.046254+0000 osd.1 (osd.1) 60 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[78107]: cluster 2023-10-09T09:12:38.048455+0000 osd.2 (osd.2) 121 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[78107]: cluster 2023-10-09T09:12:38.061659+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[78107]: cluster 2023-10-09T09:12:38.070220+0000 osd.3 (osd.3) 128 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[78107]: cluster 2023-10-09T09:12:38.374857+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[78107]: audit 2023-10-09T09:12:38.621997+0000 mon.a (mon.0) 371 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[78107]: cluster 2023-10-09T09:12:39.056211+0000 mon.a (mon.0) 372 : cluster 0 osdmap e487: 8 total, 8 up, 8 in 2023-10-09T09:12:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[75425]: cluster 2023-10-09T09:12:37.640907+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[75425]: cluster 2023-10-09T09:12:37.817655+0000 osd.0 (osd.0) 77 : cluster 3 failed to encode map e485 with expected crc 2023-10-09T09:12:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[75425]: cluster 2023-10-09T09:12:38.046254+0000 osd.1 (osd.1) 60 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[75425]: cluster 2023-10-09T09:12:38.048455+0000 osd.2 (osd.2) 121 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[75425]: cluster 2023-10-09T09:12:38.061659+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[75425]: cluster 2023-10-09T09:12:38.070220+0000 osd.3 (osd.3) 128 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[75425]: cluster 2023-10-09T09:12:38.374857+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[75425]: audit 2023-10-09T09:12:38.621997+0000 mon.a (mon.0) 371 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:39 smithi163 bash[75425]: cluster 2023-10-09T09:12:39.056211+0000 mon.a (mon.0) 372 : cluster 0 osdmap e487: 8 total, 8 up, 8 in 2023-10-09T09:12:40.045 INFO:tasks.rados.rados.0.smithi163.stdout:1711: done (0 left) 2023-10-09T09:12:40.045 INFO:tasks.rados.rados.0.smithi163.stdout:1712: delete oid 29 current snap is 209 2023-10-09T09:12:40.060 INFO:tasks.rados.rados.0.smithi163.stdout:1712: done (0 left) 2023-10-09T09:12:40.060 INFO:tasks.rados.rados.0.smithi163.stdout:1713: rollback oid 42 current snap is 209 2023-10-09T09:12:40.060 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 206 2023-10-09T09:12:40.060 INFO:tasks.rados.rados.0.smithi163.stdout:1714: write oid 17 current snap is 209 2023-10-09T09:12:40.060 INFO:tasks.rados.rados.0.smithi163.stdout:1714: seq_num 658 ranges {659006=720200,2154681=749217} 2023-10-09T09:12:40.067 INFO:tasks.rados.rados.0.smithi163.stdout:1714: writing smithi16367185-17 from 659006 to 1379206 tid 1 2023-10-09T09:12:40.076 INFO:tasks.rados.rados.0.smithi163.stdout:1714: writing smithi16367185-17 from 2154681 to 2903898 tid 2 2023-10-09T09:12:40.076 INFO:tasks.rados.rados.0.smithi163.stdout:1715: write oid 23 current snap is 209 2023-10-09T09:12:40.076 INFO:tasks.rados.rados.0.smithi163.stdout:1713: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:12:40.077 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 0 (ObjNum 1 snap 32603 seq_num 699157232) dirty dne 2023-10-09T09:12:40.077 INFO:tasks.rados.rados.0.smithi163.stdout:1715: seq_num 659 ranges {675813=795449,2096810=609228} 2023-10-09T09:12:40.085 INFO:tasks.rados.rados.0.smithi163.stdout:1715: writing smithi16367185-23 from 675813 to 1471262 tid 1 2023-10-09T09:12:40.092 INFO:tasks.rados.rados.0.smithi163.stdout:1715: writing smithi16367185-23 from 2096810 to 2706038 tid 2 2023-10-09T09:12:40.092 INFO:tasks.rados.rados.0.smithi163.stdout:1713: done (2 left) 2023-10-09T09:12:40.092 INFO:tasks.rados.rados.0.smithi163.stdout:1716: write oid 47 current snap is 209 2023-10-09T09:12:40.093 INFO:tasks.rados.rados.0.smithi163.stdout:1714: finishing write tid 1 to smithi16367185-17 2023-10-09T09:12:40.093 INFO:tasks.rados.rados.0.smithi163.stdout:1714: finishing write tid 2 to smithi16367185-17 2023-10-09T09:12:40.093 INFO:tasks.rados.rados.0.smithi163.stdout:1716: seq_num 660 ranges {692620=470698,1755292=752886} 2023-10-09T09:12:40.099 INFO:tasks.rados.rados.0.smithi163.stdout:1716: writing smithi16367185-47 from 692620 to 1163318 tid 1 2023-10-09T09:12:40.107 INFO:tasks.rados.rados.0.smithi163.stdout:1716: writing smithi16367185-47 from 1755292 to 2508178 tid 2 2023-10-09T09:12:40.107 INFO:tasks.rados.rados.0.smithi163.stdout:1717: read oid 43 snap -1 2023-10-09T09:12:40.107 INFO:tasks.rados.rados.0.smithi163.stdout:1717: expect (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:12:40.107 INFO:tasks.rados.rados.0.smithi163.stdout:1718: read oid 12 snap -1 2023-10-09T09:12:40.108 INFO:tasks.rados.rados.0.smithi163.stdout:1718: expect (ObjNum 655 snap 208 seq_num 655) 2023-10-09T09:12:40.108 INFO:tasks.rados.rados.0.smithi163.stdout:1719: write oid 3 current snap is 209 2023-10-09T09:12:40.108 INFO:tasks.rados.rados.0.smithi163.stdout:1719: seq_num 661 ranges {709427=545947,1813774=612897,2985657=123770} 2023-10-09T09:12:40.114 INFO:tasks.rados.rados.0.smithi163.stdout:1719: writing smithi16367185-3 from 709427 to 1255374 tid 1 2023-10-09T09:12:40.121 INFO:tasks.rados.rados.0.smithi163.stdout:1719: writing smithi16367185-3 from 1813774 to 2426671 tid 2 2023-10-09T09:12:40.125 INFO:tasks.rados.rados.0.smithi163.stdout:1719: writing smithi16367185-3 from 2985657 to 3109427 tid 3 2023-10-09T09:12:40.126 INFO:tasks.rados.rados.0.smithi163.stdout:1720: snap_remove snap 208 2023-10-09T09:12:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:40 smithi181 bash[59345]: audit 2023-10-09T09:12:38.613161+0000 mgr.y (mgr.34107) 127 : audit 0 from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:40 smithi181 bash[59345]: cephadm 2023-10-09T09:12:38.613816+0000 mgr.y (mgr.34107) 128 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:40.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:40 smithi181 bash[59345]: cluster 2023-10-09T09:12:38.635946+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:40.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:40 smithi181 bash[59345]: cluster 2023-10-09T09:12:39.052697+0000 osd.1 (osd.1) 61 : cluster 3 failed to encode map e487 with expected crc 2023-10-09T09:12:40.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:40 smithi181 bash[59345]: cluster 2023-10-09T09:12:39.053607+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e487 with expected crc 2023-10-09T09:12:40.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:40 smithi181 bash[59345]: cluster 2023-10-09T09:12:39.278639+0000 mgr.y (mgr.34107) 129 : cluster 0 pgmap v283: 89 pgs: 89 active+clean; 80 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:12:40.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:40 smithi181 bash[59345]: cluster 2023-10-09T09:12:40.043276+0000 mon.a (mon.0) 373 : cluster 0 osdmap e488: 8 total, 8 up, 8 in 2023-10-09T09:12:40.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[78107]: audit 2023-10-09T09:12:38.613161+0000 mgr.y (mgr.34107) 127 : audit 0 from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:40.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[78107]: cephadm 2023-10-09T09:12:38.613816+0000 mgr.y (mgr.34107) 128 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:40.668 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[78107]: cluster 2023-10-09T09:12:38.635946+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:40.668 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[78107]: cluster 2023-10-09T09:12:39.052697+0000 osd.1 (osd.1) 61 : cluster 3 failed to encode map e487 with expected crc 2023-10-09T09:12:40.668 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[78107]: cluster 2023-10-09T09:12:39.053607+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e487 with expected crc 2023-10-09T09:12:40.668 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[78107]: cluster 2023-10-09T09:12:39.278639+0000 mgr.y (mgr.34107) 129 : cluster 0 pgmap v283: 89 pgs: 89 active+clean; 80 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:12:40.668 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[78107]: cluster 2023-10-09T09:12:40.043276+0000 mon.a (mon.0) 373 : cluster 0 osdmap e488: 8 total, 8 up, 8 in 2023-10-09T09:12:40.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[75425]: audit 2023-10-09T09:12:38.613161+0000 mgr.y (mgr.34107) 127 : audit 0 from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:12:40.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[75425]: cephadm 2023-10-09T09:12:38.613816+0000 mgr.y (mgr.34107) 128 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:12:40.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[75425]: cluster 2023-10-09T09:12:38.635946+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e486 with expected crc 2023-10-09T09:12:40.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[75425]: cluster 2023-10-09T09:12:39.052697+0000 osd.1 (osd.1) 61 : cluster 3 failed to encode map e487 with expected crc 2023-10-09T09:12:40.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[75425]: cluster 2023-10-09T09:12:39.053607+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e487 with expected crc 2023-10-09T09:12:40.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[75425]: cluster 2023-10-09T09:12:39.278639+0000 mgr.y (mgr.34107) 129 : cluster 0 pgmap v283: 89 pgs: 89 active+clean; 80 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:12:40.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:40 smithi163 bash[75425]: cluster 2023-10-09T09:12:40.043276+0000 mon.a (mon.0) 373 : cluster 0 osdmap e488: 8 total, 8 up, 8 in 2023-10-09T09:12:41.054 INFO:tasks.rados.rados.0.smithi163.stdout:1720: done (6 left) 2023-10-09T09:12:41.055 INFO:tasks.rados.rados.0.smithi163.stdout:1721: snap_remove snap 206 2023-10-09T09:12:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:41 smithi181 bash[59345]: cluster 2023-10-09T09:12:40.039569+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:41 smithi181 bash[59345]: cluster 2023-10-09T09:12:40.043521+0000 osd.2 (osd.2) 122 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:41 smithi181 bash[59345]: cluster 2023-10-09T09:12:40.048268+0000 osd.2 (osd.2) 123 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:41 smithi181 bash[59345]: cluster 2023-10-09T09:12:40.069769+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:41 smithi181 bash[59345]: cluster 2023-10-09T09:12:40.079095+0000 osd.3 (osd.3) 129 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:41 smithi181 bash[59345]: audit 2023-10-09T09:12:40.339812+0000 mon.a (mon.0) 374 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:41.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:41 smithi181 bash[59345]: cluster 2023-10-09T09:12:41.054509+0000 mon.a (mon.0) 375 : cluster 0 osdmap e489: 8 total, 8 up, 8 in 2023-10-09T09:12:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[78107]: cluster 2023-10-09T09:12:40.039569+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[78107]: cluster 2023-10-09T09:12:40.043521+0000 osd.2 (osd.2) 122 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[78107]: cluster 2023-10-09T09:12:40.048268+0000 osd.2 (osd.2) 123 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[78107]: cluster 2023-10-09T09:12:40.069769+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[78107]: cluster 2023-10-09T09:12:40.079095+0000 osd.3 (osd.3) 129 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[78107]: audit 2023-10-09T09:12:40.339812+0000 mon.a (mon.0) 374 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[78107]: cluster 2023-10-09T09:12:41.054509+0000 mon.a (mon.0) 375 : cluster 0 osdmap e489: 8 total, 8 up, 8 in 2023-10-09T09:12:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[75425]: cluster 2023-10-09T09:12:40.039569+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[75425]: cluster 2023-10-09T09:12:40.043521+0000 osd.2 (osd.2) 122 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[75425]: cluster 2023-10-09T09:12:40.048268+0000 osd.2 (osd.2) 123 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[75425]: cluster 2023-10-09T09:12:40.069769+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[75425]: cluster 2023-10-09T09:12:40.079095+0000 osd.3 (osd.3) 129 : cluster 3 failed to encode map e488 with expected crc 2023-10-09T09:12:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[75425]: audit 2023-10-09T09:12:40.339812+0000 mon.a (mon.0) 374 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:41.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:41 smithi163 bash[75425]: cluster 2023-10-09T09:12:41.054509+0000 mon.a (mon.0) 375 : cluster 0 osdmap e489: 8 total, 8 up, 8 in 2023-10-09T09:12:42.058 INFO:tasks.rados.rados.0.smithi163.stdout:1721: done (6 left) 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:1722: rollback oid 19 current snap is 209 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:1722: done (6 left) 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:1723: read oid 19 snap -1 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:1723: expect deleted 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:1724: delete oid 30 current snap is 209 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:1714: finishing write tid 3 to smithi16367185-17 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:1714: finishing write tid 4 to smithi16367185-17 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 857 (ObjNum 658 snap 209 seq_num 658) dirty exists 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:1714: left oid 17 (ObjNum 658 snap 209 seq_num 658) 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:1715: finishing write tid 1 to smithi16367185-23 2023-10-09T09:12:42.059 INFO:tasks.rados.rados.0.smithi163.stdout:1715: finishing write tid 2 to smithi16367185-23 2023-10-09T09:12:42.060 INFO:tasks.rados.rados.0.smithi163.stdout:1715: finishing write tid 3 to smithi16367185-23 2023-10-09T09:12:42.060 INFO:tasks.rados.rados.0.smithi163.stdout:1715: finishing write tid 4 to smithi16367185-23 2023-10-09T09:12:42.060 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 680 (ObjNum 659 snap 209 seq_num 659) dirty exists 2023-10-09T09:12:42.060 INFO:tasks.rados.rados.0.smithi163.stdout:1715: left oid 23 (ObjNum 659 snap 209 seq_num 659) 2023-10-09T09:12:42.060 INFO:tasks.rados.rados.0.smithi163.stdout:1716: finishing write tid 1 to smithi16367185-47 2023-10-09T09:12:42.060 INFO:tasks.rados.rados.0.smithi163.stdout:1718: expect (ObjNum 655 snap 208 seq_num 655) 2023-10-09T09:12:42.203 INFO:tasks.rados.rados.0.smithi163.stdout:1716: finishing write tid 2 to smithi16367185-47 2023-10-09T09:12:42.203 INFO:tasks.rados.rados.0.smithi163.stdout:1716: finishing write tid 3 to smithi16367185-47 2023-10-09T09:12:42.203 INFO:tasks.rados.rados.0.smithi163.stdout:1716: finishing write tid 4 to smithi16367185-47 2023-10-09T09:12:42.204 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 1320 (ObjNum 660 snap 209 seq_num 660) dirty exists 2023-10-09T09:12:42.204 INFO:tasks.rados.rados.0.smithi163.stdout:1716: left oid 47 (ObjNum 660 snap 209 seq_num 660) 2023-10-09T09:12:42.204 INFO:tasks.rados.rados.0.smithi163.stdout:1717: expect (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:1719: finishing write tid 1 to smithi16367185-3 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:1719: finishing write tid 2 to smithi16367185-3 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:1719: finishing write tid 3 to smithi16367185-3 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:1719: finishing write tid 4 to smithi16367185-3 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:1719: finishing write tid 5 to smithi16367185-3 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 463 (ObjNum 661 snap 209 seq_num 661) dirty exists 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:1719: left oid 3 (ObjNum 661 snap 209 seq_num 661) 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:1714: done (7 left) 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:1715: done (6 left) 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:1716: done (5 left) 2023-10-09T09:12:42.312 INFO:tasks.rados.rados.0.smithi163.stdout:1717: done (4 left) 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1718: done (3 left) 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1719: done (2 left) 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1723: done (1 left) 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1724: done (0 left) 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1725: read oid 19 snap -1 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1725: expect deleted 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1726: read oid 31 snap -1 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1726: expect (ObjNum 641 snap 204 seq_num 641) 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1727: read oid 21 snap -1 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1727: expect deleted 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1728: write oid 25 current snap is 209 2023-10-09T09:12:42.313 INFO:tasks.rados.rados.0.smithi163.stdout:1728: seq_num 663 ranges {743041=412798,1930738=616566,3040503=102538} 2023-10-09T09:12:42.324 INFO:tasks.rados.rados.0.smithi163.stdout:1728: writing smithi16367185-25 from 743041 to 1155839 tid 1 2023-10-09T09:12:42.332 INFO:tasks.rados.rados.0.smithi163.stdout:1728: writing smithi16367185-25 from 1930738 to 2547304 tid 2 2023-10-09T09:12:42.336 INFO:tasks.rados.rados.0.smithi163.stdout:1728: writing smithi16367185-25 from 3040503 to 3143041 tid 3 2023-10-09T09:12:42.336 INFO:tasks.rados.rados.0.smithi163.stdout:1729: read oid 42 snap -1 2023-10-09T09:12:42.336 INFO:tasks.rados.rados.0.smithi163.stdout:1729: expect deleted 2023-10-09T09:12:42.336 INFO:tasks.rados.rados.0.smithi163.stdout:1726: expect (ObjNum 641 snap 204 seq_num 641) 2023-10-09T09:12:42.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:42 smithi181 bash[59345]: cluster 2023-10-09T09:12:41.052731+0000 osd.3 (osd.3) 130 : cluster 3 failed to encode map e489 with expected crc 2023-10-09T09:12:42.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:42 smithi181 bash[59345]: cluster 2023-10-09T09:12:41.053838+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e489 with expected crc 2023-10-09T09:12:42.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:42 smithi181 bash[59345]: cluster 2023-10-09T09:12:41.279927+0000 mgr.y (mgr.34107) 130 : cluster 0 pgmap v286: 89 pgs: 89 active+clean; 89 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2023-10-09T09:12:42.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:42 smithi181 bash[59345]: cluster 2023-10-09T09:12:42.057174+0000 mon.a (mon.0) 376 : cluster 0 osdmap e490: 8 total, 8 up, 8 in 2023-10-09T09:12:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:42 smithi163 bash[75425]: cluster 2023-10-09T09:12:41.052731+0000 osd.3 (osd.3) 130 : cluster 3 failed to encode map e489 with expected crc 2023-10-09T09:12:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:42 smithi163 bash[75425]: cluster 2023-10-09T09:12:41.053838+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e489 with expected crc 2023-10-09T09:12:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:42 smithi163 bash[75425]: cluster 2023-10-09T09:12:41.279927+0000 mgr.y (mgr.34107) 130 : cluster 0 pgmap v286: 89 pgs: 89 active+clean; 89 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2023-10-09T09:12:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:42 smithi163 bash[75425]: cluster 2023-10-09T09:12:42.057174+0000 mon.a (mon.0) 376 : cluster 0 osdmap e490: 8 total, 8 up, 8 in 2023-10-09T09:12:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:42 smithi163 bash[78107]: cluster 2023-10-09T09:12:41.052731+0000 osd.3 (osd.3) 130 : cluster 3 failed to encode map e489 with expected crc 2023-10-09T09:12:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:42 smithi163 bash[78107]: cluster 2023-10-09T09:12:41.053838+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e489 with expected crc 2023-10-09T09:12:42.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:42 smithi163 bash[78107]: cluster 2023-10-09T09:12:41.279927+0000 mgr.y (mgr.34107) 130 : cluster 0 pgmap v286: 89 pgs: 89 active+clean; 89 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2023-10-09T09:12:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:42 smithi163 bash[78107]: cluster 2023-10-09T09:12:42.057174+0000 mon.a (mon.0) 376 : cluster 0 osdmap e490: 8 total, 8 up, 8 in 2023-10-09T09:12:42.471 INFO:tasks.rados.rados.0.smithi163.stdout:1728: finishing write tid 1 to smithi16367185-25 2023-10-09T09:12:42.471 INFO:tasks.rados.rados.0.smithi163.stdout:1728: finishing write tid 2 to smithi16367185-25 2023-10-09T09:12:42.471 INFO:tasks.rados.rados.0.smithi163.stdout:1728: finishing write tid 3 to smithi16367185-25 2023-10-09T09:12:42.471 INFO:tasks.rados.rados.0.smithi163.stdout:1725: done (4 left) 2023-10-09T09:12:42.471 INFO:tasks.rados.rados.0.smithi163.stdout:1726: done (3 left) 2023-10-09T09:12:42.472 INFO:tasks.rados.rados.0.smithi163.stdout:1727: done (2 left) 2023-10-09T09:12:42.472 INFO:tasks.rados.rados.0.smithi163.stdout:1730: delete oid 16 current snap is 209 2023-10-09T09:12:42.472 INFO:tasks.rados.rados.0.smithi163.stdout:1728: finishing write tid 4 to smithi16367185-25 2023-10-09T09:12:42.472 INFO:tasks.rados.rados.0.smithi163.stdout:1728: finishing write tid 5 to smithi16367185-25 2023-10-09T09:12:42.472 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 781 (ObjNum 663 snap 209 seq_num 663) dirty exists 2023-10-09T09:12:42.472 INFO:tasks.rados.rados.0.smithi163.stdout:1728: left oid 25 (ObjNum 663 snap 209 seq_num 663) 2023-10-09T09:12:42.475 INFO:tasks.rados.rados.0.smithi163.stdout:1728: done (2 left) 2023-10-09T09:12:42.475 INFO:tasks.rados.rados.0.smithi163.stdout:1729: done (1 left) 2023-10-09T09:12:42.475 INFO:tasks.rados.rados.0.smithi163.stdout:1730: done (0 left) 2023-10-09T09:12:42.475 INFO:tasks.rados.rados.0.smithi163.stdout:1731: delete oid 36 current snap is 209 2023-10-09T09:12:42.477 INFO:tasks.rados.rados.0.smithi163.stdout:1731: done (0 left) 2023-10-09T09:12:42.477 INFO:tasks.rados.rados.0.smithi163.stdout:1732: delete oid 27 current snap is 209 2023-10-09T09:12:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:1732: done (0 left) 2023-10-09T09:12:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:1733: read oid 13 snap -1 2023-10-09T09:12:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:1733: expect deleted 2023-10-09T09:12:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:1734: snap_create 2023-10-09T09:12:43.298 INFO:tasks.rados.rados.0.smithi163.stdout:1733: done (1 left) 2023-10-09T09:12:43.298 INFO:tasks.rados.rados.0.smithi163.stdout:1734: done (0 left) 2023-10-09T09:12:43.298 INFO:tasks.rados.rados.0.smithi163.stdout:1735: snap_create 2023-10-09T09:12:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:43 smithi181 bash[59345]: cluster 2023-10-09T09:12:42.052854+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e490 with expected crc 2023-10-09T09:12:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:43 smithi181 bash[59345]: cluster 2023-10-09T09:12:42.055525+0000 osd.2 (osd.2) 124 : cluster 3 failed to encode map e490 with expected crc 2023-10-09T09:12:43.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:43 smithi181 bash[59345]: cluster 2023-10-09T09:12:42.376950+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e490 with expected crc 2023-10-09T09:12:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:12:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:12:43] "GET /metrics HTTP/1.1" 200 36078 "" "Prometheus/2.33.4" 2023-10-09T09:12:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:43 smithi163 bash[75425]: cluster 2023-10-09T09:12:42.052854+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e490 with expected crc 2023-10-09T09:12:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:43 smithi163 bash[75425]: cluster 2023-10-09T09:12:42.055525+0000 osd.2 (osd.2) 124 : cluster 3 failed to encode map e490 with expected crc 2023-10-09T09:12:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:43 smithi163 bash[75425]: cluster 2023-10-09T09:12:42.376950+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e490 with expected crc 2023-10-09T09:12:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:43 smithi163 bash[78107]: cluster 2023-10-09T09:12:42.052854+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e490 with expected crc 2023-10-09T09:12:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:43 smithi163 bash[78107]: cluster 2023-10-09T09:12:42.055525+0000 osd.2 (osd.2) 124 : cluster 3 failed to encode map e490 with expected crc 2023-10-09T09:12:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:43 smithi163 bash[78107]: cluster 2023-10-09T09:12:42.376950+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e490 with expected crc 2023-10-09T09:12:44.301 INFO:tasks.rados.rados.0.smithi163.stdout:1735: done (0 left) 2023-10-09T09:12:44.301 INFO:tasks.rados.rados.0.smithi163.stdout:1736: write oid 50 current snap is 211 2023-10-09T09:12:44.301 INFO:tasks.rados.rados.0.smithi163.stdout:1736: seq_num 667 ranges {410269=713794,1648313=507551,2917489=292780} 2023-10-09T09:12:44.309 INFO:tasks.rados.rados.0.smithi163.stdout:1736: writing smithi16367185-50 from 410269 to 1124063 tid 1 2023-10-09T09:12:44.315 INFO:tasks.rados.rados.0.smithi163.stdout:1736: writing smithi16367185-50 from 1648313 to 2155864 tid 2 2023-10-09T09:12:44.321 INFO:tasks.rados.rados.0.smithi163.stdout:1736: writing smithi16367185-50 from 2917489 to 3210269 tid 3 2023-10-09T09:12:44.321 INFO:tasks.rados.rados.0.smithi163.stdout:1737: read oid 40 snap -1 2023-10-09T09:12:44.322 INFO:tasks.rados.rados.0.smithi163.stdout:1737: expect deleted 2023-10-09T09:12:44.322 INFO:tasks.rados.rados.0.smithi163.stdout:1738: snap_create 2023-10-09T09:12:44.322 INFO:tasks.rados.rados.0.smithi163.stdout:1736: finishing write tid 1 to smithi16367185-50 2023-10-09T09:12:44.322 INFO:tasks.rados.rados.0.smithi163.stdout:1736: finishing write tid 2 to smithi16367185-50 2023-10-09T09:12:44.324 INFO:tasks.rados.rados.0.smithi163.stdout:1736: finishing write tid 3 to smithi16367185-50 2023-10-09T09:12:44.324 INFO:tasks.rados.rados.0.smithi163.stdout:1736: finishing write tid 4 to smithi16367185-50 2023-10-09T09:12:44.324 INFO:tasks.rados.rados.0.smithi163.stdout:1736: finishing write tid 5 to smithi16367185-50 2023-10-09T09:12:44.324 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 689 (ObjNum 667 snap 211 seq_num 667) dirty exists 2023-10-09T09:12:44.324 INFO:tasks.rados.rados.0.smithi163.stdout:1736: left oid 50 (ObjNum 667 snap 211 seq_num 667) 2023-10-09T09:12:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:44 smithi181 bash[59345]: cluster 2023-10-09T09:12:43.280714+0000 mgr.y (mgr.34107) 131 : cluster 0 pgmap v288: 89 pgs: 89 active+clean; 89 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2023-10-09T09:12:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:44 smithi181 bash[59345]: cluster 2023-10-09T09:12:43.294289+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:44 smithi181 bash[59345]: cluster 2023-10-09T09:12:43.295103+0000 osd.3 (osd.3) 131 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:44 smithi181 bash[59345]: cluster 2023-10-09T09:12:43.298237+0000 mon.a (mon.0) 377 : cluster 0 osdmap e491: 8 total, 8 up, 8 in 2023-10-09T09:12:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:44 smithi163 bash[75425]: cluster 2023-10-09T09:12:43.280714+0000 mgr.y (mgr.34107) 131 : cluster 0 pgmap v288: 89 pgs: 89 active+clean; 89 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2023-10-09T09:12:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:44 smithi163 bash[75425]: cluster 2023-10-09T09:12:43.294289+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:44 smithi163 bash[75425]: cluster 2023-10-09T09:12:43.295103+0000 osd.3 (osd.3) 131 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:44 smithi163 bash[75425]: cluster 2023-10-09T09:12:43.298237+0000 mon.a (mon.0) 377 : cluster 0 osdmap e491: 8 total, 8 up, 8 in 2023-10-09T09:12:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:44 smithi163 bash[78107]: cluster 2023-10-09T09:12:43.280714+0000 mgr.y (mgr.34107) 131 : cluster 0 pgmap v288: 89 pgs: 89 active+clean; 89 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2023-10-09T09:12:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:44 smithi163 bash[78107]: cluster 2023-10-09T09:12:43.294289+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:44 smithi163 bash[78107]: cluster 2023-10-09T09:12:43.295103+0000 osd.3 (osd.3) 131 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:44 smithi163 bash[78107]: cluster 2023-10-09T09:12:43.298237+0000 mon.a (mon.0) 377 : cluster 0 osdmap e491: 8 total, 8 up, 8 in 2023-10-09T09:12:45.048 INFO:tasks.rados.rados.0.smithi163.stdout:1736: done (2 left) 2023-10-09T09:12:45.048 INFO:tasks.rados.rados.0.smithi163.stdout:1737: done (1 left) 2023-10-09T09:12:45.049 INFO:tasks.rados.rados.0.smithi163.stdout:1738: done (0 left) 2023-10-09T09:12:45.049 INFO:tasks.rados.rados.0.smithi163.stdout:1739: read oid 25 snap -1 2023-10-09T09:12:45.049 INFO:tasks.rados.rados.0.smithi163.stdout:1739: expect (ObjNum 663 snap 209 seq_num 663) 2023-10-09T09:12:45.049 INFO:tasks.rados.rados.0.smithi163.stdout:1740: read oid 1 snap -1 2023-10-09T09:12:45.049 INFO:tasks.rados.rados.0.smithi163.stdout:1740: expect (ObjNum 648 snap 205 seq_num 648) 2023-10-09T09:12:45.049 INFO:tasks.rados.rados.0.smithi163.stdout:1741: write oid 29 current snap is 212 2023-10-09T09:12:45.049 INFO:tasks.rados.rados.0.smithi163.stdout:1741: seq_num 668 ranges {427076=789043,1706795=767562,2944912=282164} 2023-10-09T09:12:45.059 INFO:tasks.rados.rados.0.smithi163.stdout:1741: writing smithi16367185-29 from 427076 to 1216119 tid 1 2023-10-09T09:12:45.067 INFO:tasks.rados.rados.0.smithi163.stdout:1741: writing smithi16367185-29 from 1706795 to 2474357 tid 2 2023-10-09T09:12:45.072 INFO:tasks.rados.rados.0.smithi163.stdout:1741: writing smithi16367185-29 from 2944912 to 3227076 tid 3 2023-10-09T09:12:45.072 INFO:tasks.rados.rados.0.smithi163.stdout:1742: delete oid 3 current snap is 212 2023-10-09T09:12:45.072 INFO:tasks.rados.rados.0.smithi163.stdout:1740: expect (ObjNum 648 snap 205 seq_num 648) 2023-10-09T09:12:45.301 INFO:tasks.rados.rados.0.smithi163.stdout:1739: expect (ObjNum 663 snap 209 seq_num 663) 2023-10-09T09:12:45.495 INFO:tasks.rados.rados.0.smithi163.stdout:1741: finishing write tid 1 to smithi16367185-29 2023-10-09T09:12:45.495 INFO:tasks.rados.rados.0.smithi163.stdout:1739: done (3 left) 2023-10-09T09:12:45.495 INFO:tasks.rados.rados.0.smithi163.stdout:1740: done (2 left) 2023-10-09T09:12:45.496 INFO:tasks.rados.rados.0.smithi163.stdout:1742: done (1 left) 2023-10-09T09:12:45.496 INFO:tasks.rados.rados.0.smithi163.stdout:1743: delete oid 41 current snap is 212 2023-10-09T09:12:45.496 INFO:tasks.rados.rados.0.smithi163.stdout:1741: finishing write tid 2 to smithi16367185-29 2023-10-09T09:12:45.496 INFO:tasks.rados.rados.0.smithi163.stdout:1741: finishing write tid 3 to smithi16367185-29 2023-10-09T09:12:45.496 INFO:tasks.rados.rados.0.smithi163.stdout:1741: finishing write tid 4 to smithi16367185-29 2023-10-09T09:12:45.496 INFO:tasks.rados.rados.0.smithi163.stdout:1741: finishing write tid 5 to smithi16367185-29 2023-10-09T09:12:45.497 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 785 (ObjNum 668 snap 212 seq_num 668) dirty exists 2023-10-09T09:12:45.497 INFO:tasks.rados.rados.0.smithi163.stdout:1741: left oid 29 (ObjNum 668 snap 212 seq_num 668) 2023-10-09T09:12:45.497 INFO:tasks.rados.rados.0.smithi163.stdout:1741: done (1 left) 2023-10-09T09:12:45.497 INFO:tasks.rados.rados.0.smithi163.stdout:1743: done (0 left) 2023-10-09T09:12:45.498 INFO:tasks.rados.rados.0.smithi163.stdout:1744: write oid 25 current snap is 212 2023-10-09T09:12:45.498 INFO:tasks.rados.rados.0.smithi163.stdout:1744: seq_num 671 ranges {477497=731143,1998594=514889,3143534=133963} 2023-10-09T09:12:45.505 INFO:tasks.rados.rados.0.smithi163.stdout:1744: writing smithi16367185-25 from 477497 to 1208640 tid 1 2023-10-09T09:12:45.513 INFO:tasks.rados.rados.0.smithi163.stdout:1744: writing smithi16367185-25 from 1998594 to 2513483 tid 2 2023-10-09T09:12:45.518 INFO:tasks.rados.rados.0.smithi163.stdout:1744: writing smithi16367185-25 from 3143534 to 3277497 tid 3 2023-10-09T09:12:45.518 INFO:tasks.rados.rados.0.smithi163.stdout:1745: snap_remove snap 211 2023-10-09T09:12:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:45 smithi181 bash[59345]: cluster 2023-10-09T09:12:44.295823+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:45 smithi181 bash[59345]: cluster 2023-10-09T09:12:44.297645+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e492 with expected crc 2023-10-09T09:12:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:45 smithi181 bash[59345]: cluster 2023-10-09T09:12:44.300923+0000 mon.a (mon.0) 378 : cluster 0 osdmap e492: 8 total, 8 up, 8 in 2023-10-09T09:12:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:45 smithi181 bash[59345]: cluster 2023-10-09T09:12:44.321734+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:45 smithi181 bash[59345]: cluster 2023-10-09T09:12:45.048005+0000 mon.a (mon.0) 379 : cluster 0 osdmap e493: 8 total, 8 up, 8 in 2023-10-09T09:12:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:45 smithi163 bash[78107]: cluster 2023-10-09T09:12:44.295823+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:45 smithi163 bash[78107]: cluster 2023-10-09T09:12:44.297645+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e492 with expected crc 2023-10-09T09:12:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:45 smithi163 bash[78107]: cluster 2023-10-09T09:12:44.300923+0000 mon.a (mon.0) 378 : cluster 0 osdmap e492: 8 total, 8 up, 8 in 2023-10-09T09:12:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:45 smithi163 bash[78107]: cluster 2023-10-09T09:12:44.321734+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:45 smithi163 bash[78107]: cluster 2023-10-09T09:12:45.048005+0000 mon.a (mon.0) 379 : cluster 0 osdmap e493: 8 total, 8 up, 8 in 2023-10-09T09:12:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:45 smithi163 bash[75425]: cluster 2023-10-09T09:12:44.295823+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:45 smithi163 bash[75425]: cluster 2023-10-09T09:12:44.297645+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e492 with expected crc 2023-10-09T09:12:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:45 smithi163 bash[75425]: cluster 2023-10-09T09:12:44.300923+0000 mon.a (mon.0) 378 : cluster 0 osdmap e492: 8 total, 8 up, 8 in 2023-10-09T09:12:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:45 smithi163 bash[75425]: cluster 2023-10-09T09:12:44.321734+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e491 with expected crc 2023-10-09T09:12:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:45 smithi163 bash[75425]: cluster 2023-10-09T09:12:45.048005+0000 mon.a (mon.0) 379 : cluster 0 osdmap e493: 8 total, 8 up, 8 in 2023-10-09T09:12:46.483 INFO:tasks.rados.rados.0.smithi163.stdout:1745: done (1 left) 2023-10-09T09:12:46.483 INFO:tasks.rados.rados.0.smithi163.stdout:1746: snap_remove snap 210 2023-10-09T09:12:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: cluster 2023-10-09T09:12:45.042549+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: cluster 2023-10-09T09:12:45.043045+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: cluster 2023-10-09T09:12:45.055429+0000 osd.2 (osd.2) 125 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: cluster 2023-10-09T09:12:45.281185+0000 mgr.y (mgr.34107) 132 : cluster 0 pgmap v292: 89 pgs: 89 active+clean; 89 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: cluster 2023-10-09T09:12:45.351398+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: audit 2023-10-09T09:12:45.463306+0000 mon.a (mon.0) 380 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: audit 2023-10-09T09:12:45.469467+0000 mon.a (mon.0) 381 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: audit 2023-10-09T09:12:45.470641+0000 mon.a (mon.0) 382 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: audit 2023-10-09T09:12:45.471586+0000 mon.a (mon.0) 383 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: audit 2023-10-09T09:12:45.476927+0000 mon.a (mon.0) 384 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: audit 2023-10-09T09:12:45.546304+0000 mon.a (mon.0) 385 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: audit 2023-10-09T09:12:45.547657+0000 mon.a (mon.0) 386 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:46.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:46 smithi181 bash[59345]: audit 2023-10-09T09:12:45.548727+0000 mon.a (mon.0) 387 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:46.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: cluster 2023-10-09T09:12:45.042549+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: cluster 2023-10-09T09:12:45.043045+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: cluster 2023-10-09T09:12:45.055429+0000 osd.2 (osd.2) 125 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: cluster 2023-10-09T09:12:45.281185+0000 mgr.y (mgr.34107) 132 : cluster 0 pgmap v292: 89 pgs: 89 active+clean; 89 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:46.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: cluster 2023-10-09T09:12:45.351398+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: audit 2023-10-09T09:12:45.463306+0000 mon.a (mon.0) 380 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:46.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: audit 2023-10-09T09:12:45.469467+0000 mon.a (mon.0) 381 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:46.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: audit 2023-10-09T09:12:45.470641+0000 mon.a (mon.0) 382 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:46.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: audit 2023-10-09T09:12:45.471586+0000 mon.a (mon.0) 383 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:46.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: audit 2023-10-09T09:12:45.476927+0000 mon.a (mon.0) 384 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:46.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: audit 2023-10-09T09:12:45.546304+0000 mon.a (mon.0) 385 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:46.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: audit 2023-10-09T09:12:45.547657+0000 mon.a (mon.0) 386 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:46.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[75425]: audit 2023-10-09T09:12:45.548727+0000 mon.a (mon.0) 387 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:46.669 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: cluster 2023-10-09T09:12:45.042549+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.669 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: cluster 2023-10-09T09:12:45.043045+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.669 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: cluster 2023-10-09T09:12:45.055429+0000 osd.2 (osd.2) 125 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.669 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: cluster 2023-10-09T09:12:45.281185+0000 mgr.y (mgr.34107) 132 : cluster 0 pgmap v292: 89 pgs: 89 active+clean; 89 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:46.669 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: cluster 2023-10-09T09:12:45.351398+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e493 with expected crc 2023-10-09T09:12:46.669 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: audit 2023-10-09T09:12:45.463306+0000 mon.a (mon.0) 380 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:46.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: audit 2023-10-09T09:12:45.469467+0000 mon.a (mon.0) 381 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:46.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: audit 2023-10-09T09:12:45.470641+0000 mon.a (mon.0) 382 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:46.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: audit 2023-10-09T09:12:45.471586+0000 mon.a (mon.0) 383 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:12:46.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: audit 2023-10-09T09:12:45.476927+0000 mon.a (mon.0) 384 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:46.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: audit 2023-10-09T09:12:45.546304+0000 mon.a (mon.0) 385 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:46.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: audit 2023-10-09T09:12:45.547657+0000 mon.a (mon.0) 386 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:46.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:46 smithi163 bash[78107]: audit 2023-10-09T09:12:45.548727+0000 mon.a (mon.0) 387 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:47.495 INFO:tasks.rados.rados.0.smithi163.stdout:1746: done (1 left) 2023-10-09T09:12:47.496 INFO:tasks.rados.rados.0.smithi163.stdout:1747: snap_remove snap 209 2023-10-09T09:12:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:47 smithi181 bash[59345]: cephadm 2023-10-09T09:12:45.545719+0000 mgr.y (mgr.34107) 133 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T09:12:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:47 smithi181 bash[59345]: cephadm 2023-10-09T09:12:45.550018+0000 mgr.y (mgr.34107) 134 : cephadm 1 Reconfiguring daemon mon.a on smithi163 2023-10-09T09:12:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:47 smithi181 bash[59345]: cluster 2023-10-09T09:12:46.477433+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:47 smithi181 bash[59345]: cluster 2023-10-09T09:12:46.479214+0000 osd.3 (osd.3) 132 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:47 smithi181 bash[59345]: cluster 2023-10-09T09:12:46.483228+0000 mon.a (mon.0) 388 : cluster 0 osdmap e494: 8 total, 8 up, 8 in 2023-10-09T09:12:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:47 smithi181 bash[59345]: cluster 2023-10-09T09:12:46.549636+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:47 smithi181 bash[59345]: cluster 2023-10-09T09:12:46.742187+0000 osd.1 (osd.1) 62 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:47 smithi181 bash[59345]: cluster 2023-10-09T09:12:46.955081+0000 osd.2 (osd.2) 126 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.924 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[78107]: cephadm 2023-10-09T09:12:45.545719+0000 mgr.y (mgr.34107) 133 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T09:12:47.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[78107]: cephadm 2023-10-09T09:12:45.550018+0000 mgr.y (mgr.34107) 134 : cephadm 1 Reconfiguring daemon mon.a on smithi163 2023-10-09T09:12:47.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[78107]: cluster 2023-10-09T09:12:46.477433+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[78107]: cluster 2023-10-09T09:12:46.479214+0000 osd.3 (osd.3) 132 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[78107]: cluster 2023-10-09T09:12:46.483228+0000 mon.a (mon.0) 388 : cluster 0 osdmap e494: 8 total, 8 up, 8 in 2023-10-09T09:12:47.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[78107]: cluster 2023-10-09T09:12:46.549636+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[78107]: cluster 2023-10-09T09:12:46.742187+0000 osd.1 (osd.1) 62 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[78107]: cluster 2023-10-09T09:12:46.955081+0000 osd.2 (osd.2) 126 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.925 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[75425]: cephadm 2023-10-09T09:12:45.545719+0000 mgr.y (mgr.34107) 133 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T09:12:47.925 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[75425]: cephadm 2023-10-09T09:12:45.550018+0000 mgr.y (mgr.34107) 134 : cephadm 1 Reconfiguring daemon mon.a on smithi163 2023-10-09T09:12:47.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[75425]: cluster 2023-10-09T09:12:46.477433+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[75425]: cluster 2023-10-09T09:12:46.479214+0000 osd.3 (osd.3) 132 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[75425]: cluster 2023-10-09T09:12:46.483228+0000 mon.a (mon.0) 388 : cluster 0 osdmap e494: 8 total, 8 up, 8 in 2023-10-09T09:12:47.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[75425]: cluster 2023-10-09T09:12:46.549636+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[75425]: cluster 2023-10-09T09:12:46.742187+0000 osd.1 (osd.1) 62 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:47.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:47 smithi163 bash[75425]: cluster 2023-10-09T09:12:46.955081+0000 osd.2 (osd.2) 126 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:48.508 INFO:tasks.rados.rados.0.smithi163.stdout:1747: done (1 left) 2023-10-09T09:12:48.508 INFO:tasks.rados.rados.0.smithi163.stdout:1748: write oid 38 current snap is 212 2023-10-09T09:12:48.508 INFO:tasks.rados.rados.0.smithi163.stdout:1744: finishing write tid 1 to smithi16367185-25 2023-10-09T09:12:48.508 INFO:tasks.rados.rados.0.smithi163.stdout:1748: seq_num 672 ranges {494304=406392,1540723=774900,2770957=523347} 2023-10-09T09:12:48.512 INFO:tasks.rados.rados.0.smithi163.stdout:1748: writing smithi16367185-38 from 494304 to 900696 tid 1 2023-10-09T09:12:48.521 INFO:tasks.rados.rados.0.smithi163.stdout:1748: writing smithi16367185-38 from 1540723 to 2315623 tid 2 2023-10-09T09:12:48.527 INFO:tasks.rados.rados.0.smithi163.stdout:1748: writing smithi16367185-38 from 2770957 to 3294304 tid 3 2023-10-09T09:12:48.527 INFO:tasks.rados.rados.0.smithi163.stdout:1749: rollback oid 29 current snap is 212 2023-10-09T09:12:48.527 INFO:tasks.rados.rados.0.smithi163.stdout:1749: done (2 left) 2023-10-09T09:12:48.528 INFO:tasks.rados.rados.0.smithi163.stdout:1750: write oid 34 current snap is 212 2023-10-09T09:12:48.528 INFO:tasks.rados.rados.0.smithi163.stdout:1750: seq_num 673 ranges {511111=481641,1599205=518558,2682027=489947} 2023-10-09T09:12:48.532 INFO:tasks.rados.rados.0.smithi163.stdout:1750: writing smithi16367185-34 from 511111 to 992752 tid 1 2023-10-09T09:12:48.539 INFO:tasks.rados.rados.0.smithi163.stdout:1750: writing smithi16367185-34 from 1599205 to 2117763 tid 2 2023-10-09T09:12:48.545 INFO:tasks.rados.rados.0.smithi163.stdout:1750: writing smithi16367185-34 from 2682027 to 3171974 tid 3 2023-10-09T09:12:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:1751: read oid 35 snap -1 2023-10-09T09:12:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:1751: expect (ObjNum 643 snap 205 seq_num 643) 2023-10-09T09:12:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:1752: write oid 15 current snap is 212 2023-10-09T09:12:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:1744: finishing write tid 2 to smithi16367185-25 2023-10-09T09:12:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:1744: finishing write tid 3 to smithi16367185-25 2023-10-09T09:12:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:1744: finishing write tid 4 to smithi16367185-25 2023-10-09T09:12:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:1744: finishing write tid 5 to smithi16367185-25 2023-10-09T09:12:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 790 (ObjNum 671 snap 212 seq_num 671) dirty exists 2023-10-09T09:12:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:1744: left oid 25 (ObjNum 671 snap 212 seq_num 671) 2023-10-09T09:12:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:1748: finishing write tid 1 to smithi16367185-38 2023-10-09T09:12:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:1748: finishing write tid 2 to smithi16367185-38 2023-10-09T09:12:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:1748: finishing write tid 3 to smithi16367185-38 2023-10-09T09:12:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:1748: finishing write tid 4 to smithi16367185-38 2023-10-09T09:12:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:1748: finishing write tid 5 to smithi16367185-38 2023-10-09T09:12:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 693 (ObjNum 672 snap 212 seq_num 672) dirty exists 2023-10-09T09:12:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:1748: left oid 38 (ObjNum 672 snap 212 seq_num 672) 2023-10-09T09:12:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:1750: finishing write tid 1 to smithi16367185-34 2023-10-09T09:12:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:1750: finishing write tid 2 to smithi16367185-34 2023-10-09T09:12:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:1752: seq_num 674 ranges {527918=556890,1657687=778569,3225803=102115} 2023-10-09T09:12:48.551 INFO:tasks.rados.rados.0.smithi163.stdout:1752: writing smithi16367185-15 from 527918 to 1084808 tid 1 2023-10-09T09:12:48.560 INFO:tasks.rados.rados.0.smithi163.stdout:1752: writing smithi16367185-15 from 1657687 to 2436256 tid 2 2023-10-09T09:12:48.566 INFO:tasks.rados.rados.0.smithi163.stdout:1752: writing smithi16367185-15 from 3225803 to 3327918 tid 3 2023-10-09T09:12:48.566 INFO:tasks.rados.rados.0.smithi163.stdout:1744: done (4 left) 2023-10-09T09:12:48.566 INFO:tasks.rados.rados.0.smithi163.stdout:1748: done (3 left) 2023-10-09T09:12:48.566 INFO:tasks.rados.rados.0.smithi163.stdout:1753: write oid 28 current snap is 212 2023-10-09T09:12:48.566 INFO:tasks.rados.rados.0.smithi163.stdout:1750: finishing write tid 3 to smithi16367185-34 2023-10-09T09:12:48.566 INFO:tasks.rados.rados.0.smithi163.stdout:1750: finishing write tid 4 to smithi16367185-34 2023-10-09T09:12:48.567 INFO:tasks.rados.rados.0.smithi163.stdout:1750: finishing write tid 5 to smithi16367185-34 2023-10-09T09:12:48.567 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 697 (ObjNum 673 snap 212 seq_num 673) dirty exists 2023-10-09T09:12:48.567 INFO:tasks.rados.rados.0.smithi163.stdout:1750: left oid 34 (ObjNum 673 snap 212 seq_num 673) 2023-10-09T09:12:48.567 INFO:tasks.rados.rados.0.smithi163.stdout:1753: seq_num 675 ranges {544725=632139,1716169=522227,2736873=512491} 2023-10-09T09:12:48.572 INFO:tasks.rados.rados.0.smithi163.stdout:1753: writing smithi16367185-28 from 544725 to 1176864 tid 1 2023-10-09T09:12:48.578 INFO:tasks.rados.rados.0.smithi163.stdout:1753: writing smithi16367185-28 from 1716169 to 2238396 tid 2 2023-10-09T09:12:48.584 INFO:tasks.rados.rados.0.smithi163.stdout:1753: writing smithi16367185-28 from 2736873 to 3249364 tid 3 2023-10-09T09:12:48.584 INFO:tasks.rados.rados.0.smithi163.stdout:1750: done (3 left) 2023-10-09T09:12:48.584 INFO:tasks.rados.rados.0.smithi163.stdout:1754: snap_create 2023-10-09T09:12:48.584 INFO:tasks.rados.rados.0.smithi163.stdout:1751: expect (ObjNum 643 snap 205 seq_num 643) 2023-10-09T09:12:48.754 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: cluster 2023-10-09T09:12:46.599126+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:48.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: cluster 2023-10-09T09:12:46.934341+0000 osd.0 (osd.0) 78 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:48.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: cluster 2023-10-09T09:12:47.281607+0000 mgr.y (mgr.34107) 135 : cluster 0 pgmap v294: 89 pgs: 89 active+clean; 77 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:12:48.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: cluster 2023-10-09T09:12:47.488991+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e495 with expected crc 2023-10-09T09:12:48.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: cluster 2023-10-09T09:12:47.493095+0000 osd.2 (osd.2) 127 : cluster 3 failed to encode map e495 with expected crc 2023-10-09T09:12:48.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: cluster 2023-10-09T09:12:47.495736+0000 mon.a (mon.0) 389 : cluster 0 osdmap e495: 8 total, 8 up, 8 in 2023-10-09T09:12:48.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: audit 2023-10-09T09:12:47.577860+0000 mon.a (mon.0) 390 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:48.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: audit 2023-10-09T09:12:47.587478+0000 mon.a (mon.0) 391 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:48.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: audit 2023-10-09T09:12:47.620430+0000 mon.a (mon.0) 392 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:12:48.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: audit 2023-10-09T09:12:47.622438+0000 mon.a (mon.0) 393 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:12:48.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[78107]: audit 2023-10-09T09:12:47.624267+0000 mon.a (mon.0) 394 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:48.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: cluster 2023-10-09T09:12:46.599126+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:48.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: cluster 2023-10-09T09:12:46.934341+0000 osd.0 (osd.0) 78 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:48.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: cluster 2023-10-09T09:12:47.281607+0000 mgr.y (mgr.34107) 135 : cluster 0 pgmap v294: 89 pgs: 89 active+clean; 77 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:12:48.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: cluster 2023-10-09T09:12:47.488991+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e495 with expected crc 2023-10-09T09:12:48.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: cluster 2023-10-09T09:12:47.493095+0000 osd.2 (osd.2) 127 : cluster 3 failed to encode map e495 with expected crc 2023-10-09T09:12:48.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: cluster 2023-10-09T09:12:47.495736+0000 mon.a (mon.0) 389 : cluster 0 osdmap e495: 8 total, 8 up, 8 in 2023-10-09T09:12:48.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: audit 2023-10-09T09:12:47.577860+0000 mon.a (mon.0) 390 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:48.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: audit 2023-10-09T09:12:47.587478+0000 mon.a (mon.0) 391 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:48.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: audit 2023-10-09T09:12:47.620430+0000 mon.a (mon.0) 392 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:12:48.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: audit 2023-10-09T09:12:47.622438+0000 mon.a (mon.0) 393 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:12:48.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:48 smithi163 bash[75425]: audit 2023-10-09T09:12:47.624267+0000 mon.a (mon.0) 394 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:48.758 INFO:tasks.rados.rados.0.smithi163.stdout:1752: finishing write tid 1 to smithi16367185-15 2023-10-09T09:12:48.758 INFO:tasks.rados.rados.0.smithi163.stdout:1752: finishing write tid 2 to smithi16367185-15 2023-10-09T09:12:48.758 INFO:tasks.rados.rados.0.smithi163.stdout:1752: finishing write tid 3 to smithi16367185-15 2023-10-09T09:12:48.758 INFO:tasks.rados.rados.0.smithi163.stdout:1752: finishing write tid 4 to smithi16367185-15 2023-10-09T09:12:48.758 INFO:tasks.rados.rados.0.smithi163.stdout:1752: finishing write tid 5 to smithi16367185-15 2023-10-09T09:12:48.758 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 865 (ObjNum 674 snap 212 seq_num 674) dirty exists 2023-10-09T09:12:48.758 INFO:tasks.rados.rados.0.smithi163.stdout:1752: left oid 15 (ObjNum 674 snap 212 seq_num 674) 2023-10-09T09:12:48.758 INFO:tasks.rados.rados.0.smithi163.stdout:1753: finishing write tid 1 to smithi16367185-28 2023-10-09T09:12:48.758 INFO:tasks.rados.rados.0.smithi163.stdout:1753: finishing write tid 2 to smithi16367185-28 2023-10-09T09:12:48.758 INFO:tasks.rados.rados.0.smithi163.stdout:1753: finishing write tid 3 to smithi16367185-28 2023-10-09T09:12:48.759 INFO:tasks.rados.rados.0.smithi163.stdout:1753: finishing write tid 4 to smithi16367185-28 2023-10-09T09:12:48.759 INFO:tasks.rados.rados.0.smithi163.stdout:1753: finishing write tid 5 to smithi16367185-28 2023-10-09T09:12:48.759 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 930 (ObjNum 675 snap 212 seq_num 675) dirty exists 2023-10-09T09:12:48.759 INFO:tasks.rados.rados.0.smithi163.stdout:1753: left oid 28 (ObjNum 675 snap 212 seq_num 675) 2023-10-09T09:12:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: cluster 2023-10-09T09:12:46.599126+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: cluster 2023-10-09T09:12:46.934341+0000 osd.0 (osd.0) 78 : cluster 3 failed to encode map e494 with expected crc 2023-10-09T09:12:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: cluster 2023-10-09T09:12:47.281607+0000 mgr.y (mgr.34107) 135 : cluster 0 pgmap v294: 89 pgs: 89 active+clean; 77 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2023-10-09T09:12:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: cluster 2023-10-09T09:12:47.488991+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e495 with expected crc 2023-10-09T09:12:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: cluster 2023-10-09T09:12:47.493095+0000 osd.2 (osd.2) 127 : cluster 3 failed to encode map e495 with expected crc 2023-10-09T09:12:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: cluster 2023-10-09T09:12:47.495736+0000 mon.a (mon.0) 389 : cluster 0 osdmap e495: 8 total, 8 up, 8 in 2023-10-09T09:12:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: audit 2023-10-09T09:12:47.577860+0000 mon.a (mon.0) 390 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: audit 2023-10-09T09:12:47.587478+0000 mon.a (mon.0) 391 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: audit 2023-10-09T09:12:47.620430+0000 mon.a (mon.0) 392 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:12:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: audit 2023-10-09T09:12:47.622438+0000 mon.a (mon.0) 393 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:12:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:48 smithi181 bash[59345]: audit 2023-10-09T09:12:47.624267+0000 mon.a (mon.0) 394 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:49.518 INFO:tasks.rados.rados.0.smithi163.stdout:1751: done (3 left) 2023-10-09T09:12:49.518 INFO:tasks.rados.rados.0.smithi163.stdout:1752: done (2 left) 2023-10-09T09:12:49.518 INFO:tasks.rados.rados.0.smithi163.stdout:1753: done (1 left) 2023-10-09T09:12:49.519 INFO:tasks.rados.rados.0.smithi163.stdout:1754: done (0 left) 2023-10-09T09:12:49.519 INFO:tasks.rados.rados.0.smithi163.stdout:1755: snap_remove snap 212 2023-10-09T09:12:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:49 smithi181 bash[59345]: cephadm 2023-10-09T09:12:47.619716+0000 mgr.y (mgr.34107) 136 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T09:12:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:49 smithi181 bash[59345]: cephadm 2023-10-09T09:12:47.626093+0000 mgr.y (mgr.34107) 137 : cephadm 1 Reconfiguring daemon mgr.y on smithi163 2023-10-09T09:12:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:49 smithi181 bash[59345]: cluster 2023-10-09T09:12:48.502722+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e496 with expected crc 2023-10-09T09:12:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:49 smithi181 bash[59345]: cluster 2023-10-09T09:12:48.504555+0000 osd.3 (osd.3) 133 : cluster 3 failed to encode map e496 with expected crc 2023-10-09T09:12:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:49 smithi181 bash[59345]: cluster 2023-10-09T09:12:48.507672+0000 mon.a (mon.0) 395 : cluster 0 osdmap e496: 8 total, 8 up, 8 in 2023-10-09T09:12:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:49 smithi181 bash[59345]: cluster 2023-10-09T09:12:48.546307+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e496 with expected crc 2023-10-09T09:12:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[75425]: cephadm 2023-10-09T09:12:47.619716+0000 mgr.y (mgr.34107) 136 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T09:12:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[75425]: cephadm 2023-10-09T09:12:47.626093+0000 mgr.y (mgr.34107) 137 : cephadm 1 Reconfiguring daemon mgr.y on smithi163 2023-10-09T09:12:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[75425]: cluster 2023-10-09T09:12:48.502722+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e496 with expected crc 2023-10-09T09:12:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[75425]: cluster 2023-10-09T09:12:48.504555+0000 osd.3 (osd.3) 133 : cluster 3 failed to encode map e496 with expected crc 2023-10-09T09:12:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[75425]: cluster 2023-10-09T09:12:48.507672+0000 mon.a (mon.0) 395 : cluster 0 osdmap e496: 8 total, 8 up, 8 in 2023-10-09T09:12:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[75425]: cluster 2023-10-09T09:12:48.546307+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e496 with expected crc 2023-10-09T09:12:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[78107]: cephadm 2023-10-09T09:12:47.619716+0000 mgr.y (mgr.34107) 136 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T09:12:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[78107]: cephadm 2023-10-09T09:12:47.626093+0000 mgr.y (mgr.34107) 137 : cephadm 1 Reconfiguring daemon mgr.y on smithi163 2023-10-09T09:12:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[78107]: cluster 2023-10-09T09:12:48.502722+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e496 with expected crc 2023-10-09T09:12:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[78107]: cluster 2023-10-09T09:12:48.504555+0000 osd.3 (osd.3) 133 : cluster 3 failed to encode map e496 with expected crc 2023-10-09T09:12:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[78107]: cluster 2023-10-09T09:12:48.507672+0000 mon.a (mon.0) 395 : cluster 0 osdmap e496: 8 total, 8 up, 8 in 2023-10-09T09:12:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:49 smithi163 bash[78107]: cluster 2023-10-09T09:12:48.546307+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e496 with expected crc 2023-10-09T09:12:50.053 INFO:tasks.rados.rados.0.smithi163.stdout:1755: done (0 left) 2023-10-09T09:12:50.053 INFO:tasks.rados.rados.0.smithi163.stdout:1756: snap_create 2023-10-09T09:12:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: cluster 2023-10-09T09:12:49.282168+0000 mgr.y (mgr.34107) 138 : cluster 0 pgmap v297: 89 pgs: 89 active+clean; 77 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: cluster 2023-10-09T09:12:49.513006+0000 osd.3 (osd.3) 134 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: cluster 2023-10-09T09:12:49.516249+0000 osd.0 (osd.0) 79 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: cluster 2023-10-09T09:12:49.518206+0000 mon.a (mon.0) 396 : cluster 0 osdmap e497: 8 total, 8 up, 8 in 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: audit 2023-10-09T09:12:49.643361+0000 mon.a (mon.0) 397 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: audit 2023-10-09T09:12:49.651545+0000 mon.a (mon.0) 398 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: audit 2023-10-09T09:12:49.694901+0000 mon.a (mon.0) 399 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: audit 2023-10-09T09:12:49.696896+0000 mon.a (mon.0) 400 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: audit 2023-10-09T09:12:49.698582+0000 mon.a (mon.0) 401 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: cluster 2023-10-09T09:12:50.020072+0000 osd.2 (osd.2) 128 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: cluster 2023-10-09T09:12:50.045719+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: cluster 2023-10-09T09:12:50.052742+0000 mon.a (mon.0) 402 : cluster 0 osdmap e498: 8 total, 8 up, 8 in 2023-10-09T09:12:50.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:50 smithi181 bash[59345]: cluster 2023-10-09T09:12:50.128960+0000 osd.3 (osd.3) 135 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:50.866 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: cluster 2023-10-09T09:12:49.282168+0000 mgr.y (mgr.34107) 138 : cluster 0 pgmap v297: 89 pgs: 89 active+clean; 77 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2023-10-09T09:12:50.866 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: cluster 2023-10-09T09:12:49.513006+0000 osd.3 (osd.3) 134 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: cluster 2023-10-09T09:12:49.516249+0000 osd.0 (osd.0) 79 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: cluster 2023-10-09T09:12:49.518206+0000 mon.a (mon.0) 396 : cluster 0 osdmap e497: 8 total, 8 up, 8 in 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: audit 2023-10-09T09:12:49.643361+0000 mon.a (mon.0) 397 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: audit 2023-10-09T09:12:49.651545+0000 mon.a (mon.0) 398 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: audit 2023-10-09T09:12:49.694901+0000 mon.a (mon.0) 399 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: audit 2023-10-09T09:12:49.696896+0000 mon.a (mon.0) 400 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: audit 2023-10-09T09:12:49.698582+0000 mon.a (mon.0) 401 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: cluster 2023-10-09T09:12:50.020072+0000 osd.2 (osd.2) 128 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: cluster 2023-10-09T09:12:50.045719+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: cluster 2023-10-09T09:12:50.052742+0000 mon.a (mon.0) 402 : cluster 0 osdmap e498: 8 total, 8 up, 8 in 2023-10-09T09:12:50.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[78107]: cluster 2023-10-09T09:12:50.128960+0000 osd.3 (osd.3) 135 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: cluster 2023-10-09T09:12:49.282168+0000 mgr.y (mgr.34107) 138 : cluster 0 pgmap v297: 89 pgs: 89 active+clean; 77 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: cluster 2023-10-09T09:12:49.513006+0000 osd.3 (osd.3) 134 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: cluster 2023-10-09T09:12:49.516249+0000 osd.0 (osd.0) 79 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: cluster 2023-10-09T09:12:49.518206+0000 mon.a (mon.0) 396 : cluster 0 osdmap e497: 8 total, 8 up, 8 in 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: audit 2023-10-09T09:12:49.643361+0000 mon.a (mon.0) 397 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: audit 2023-10-09T09:12:49.651545+0000 mon.a (mon.0) 398 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: audit 2023-10-09T09:12:49.694901+0000 mon.a (mon.0) 399 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: audit 2023-10-09T09:12:49.696896+0000 mon.a (mon.0) 400 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: audit 2023-10-09T09:12:49.698582+0000 mon.a (mon.0) 401 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: cluster 2023-10-09T09:12:50.020072+0000 osd.2 (osd.2) 128 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: cluster 2023-10-09T09:12:50.045719+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:50.869 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: cluster 2023-10-09T09:12:50.052742+0000 mon.a (mon.0) 402 : cluster 0 osdmap e498: 8 total, 8 up, 8 in 2023-10-09T09:12:50.869 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:50 smithi163 bash[75425]: cluster 2023-10-09T09:12:50.128960+0000 osd.3 (osd.3) 135 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:51.054 INFO:tasks.rados.rados.0.smithi163.stdout:1756: done (0 left) 2023-10-09T09:12:51.054 INFO:tasks.rados.rados.0.smithi163.stdout:1757: read oid 19 snap -1 2023-10-09T09:12:51.054 INFO:tasks.rados.rados.0.smithi163.stdout:1757: expect deleted 2023-10-09T09:12:51.054 INFO:tasks.rados.rados.0.smithi163.stdout:1758: snap_create 2023-10-09T09:12:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:51 smithi181 bash[59345]: cephadm 2023-10-09T09:12:49.694210+0000 mgr.y (mgr.34107) 139 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T09:12:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:51 smithi181 bash[59345]: cephadm 2023-10-09T09:12:49.700506+0000 mgr.y (mgr.34107) 140 : cephadm 1 Reconfiguring daemon mon.c on smithi163 2023-10-09T09:12:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:51 smithi181 bash[59345]: cluster 2023-10-09T09:12:50.050044+0000 osd.1 (osd.1) 63 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:51 smithi181 bash[59345]: cluster 2023-10-09T09:12:50.427624+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:51 smithi181 bash[59345]: cluster 2023-10-09T09:12:50.506341+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:51 smithi181 bash[59345]: cluster 2023-10-09T09:12:51.047418+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:51 smithi181 bash[59345]: cluster 2023-10-09T09:12:51.053341+0000 mon.a (mon.0) 403 : cluster 0 osdmap e499: 8 total, 8 up, 8 in 2023-10-09T09:12:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:51 smithi181 bash[59345]: cluster 2023-10-09T09:12:51.055128+0000 osd.2 (osd.2) 129 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:51.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[75425]: cephadm 2023-10-09T09:12:49.694210+0000 mgr.y (mgr.34107) 139 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T09:12:51.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[75425]: cephadm 2023-10-09T09:12:49.700506+0000 mgr.y (mgr.34107) 140 : cephadm 1 Reconfiguring daemon mon.c on smithi163 2023-10-09T09:12:51.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[75425]: cluster 2023-10-09T09:12:50.050044+0000 osd.1 (osd.1) 63 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:51.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[75425]: cluster 2023-10-09T09:12:50.427624+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:51.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[75425]: cluster 2023-10-09T09:12:50.506341+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:51.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[75425]: cluster 2023-10-09T09:12:51.047418+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[75425]: cluster 2023-10-09T09:12:51.053341+0000 mon.a (mon.0) 403 : cluster 0 osdmap e499: 8 total, 8 up, 8 in 2023-10-09T09:12:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[75425]: cluster 2023-10-09T09:12:51.055128+0000 osd.2 (osd.2) 129 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[78107]: cephadm 2023-10-09T09:12:49.694210+0000 mgr.y (mgr.34107) 139 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T09:12:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[78107]: cephadm 2023-10-09T09:12:49.700506+0000 mgr.y (mgr.34107) 140 : cephadm 1 Reconfiguring daemon mon.c on smithi163 2023-10-09T09:12:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[78107]: cluster 2023-10-09T09:12:50.050044+0000 osd.1 (osd.1) 63 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[78107]: cluster 2023-10-09T09:12:50.427624+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[78107]: cluster 2023-10-09T09:12:50.506341+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e497 with expected crc 2023-10-09T09:12:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[78107]: cluster 2023-10-09T09:12:51.047418+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[78107]: cluster 2023-10-09T09:12:51.053341+0000 mon.a (mon.0) 403 : cluster 0 osdmap e499: 8 total, 8 up, 8 in 2023-10-09T09:12:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:51 smithi163 bash[78107]: cluster 2023-10-09T09:12:51.055128+0000 osd.2 (osd.2) 129 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:52.064 INFO:tasks.rados.rados.0.smithi163.stdout:1757: done (1 left) 2023-10-09T09:12:52.065 INFO:tasks.rados.rados.0.smithi163.stdout:1758: done (0 left) 2023-10-09T09:12:52.065 INFO:tasks.rados.rados.0.smithi163.stdout:1759: delete oid 42 current snap is 215 2023-10-09T09:12:52.075 INFO:tasks.rados.rados.0.smithi163.stdout:1759: done (0 left) 2023-10-09T09:12:52.075 INFO:tasks.rados.rados.0.smithi163.stdout:1760: rollback oid 48 current snap is 215 2023-10-09T09:12:52.076 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 48 to 214 2023-10-09T09:12:52.076 INFO:tasks.rados.rados.0.smithi163.stdout:1761: write oid 24 current snap is 215 2023-10-09T09:12:52.076 INFO:tasks.rados.rados.0.smithi163.stdout:1761: seq_num 677 ranges {578339=498990,1833133=525896,2791719=586620} 2023-10-09T09:12:52.084 INFO:tasks.rados.rados.0.smithi163.stdout:1761: writing smithi16367185-24 from 578339 to 1077329 tid 1 2023-10-09T09:12:52.091 INFO:tasks.rados.rados.0.smithi163.stdout:1761: writing smithi16367185-24 from 1833133 to 2359029 tid 2 2023-10-09T09:12:52.097 INFO:tasks.rados.rados.0.smithi163.stdout:1761: writing smithi16367185-24 from 2791719 to 3378339 tid 3 2023-10-09T09:12:52.098 INFO:tasks.rados.rados.0.smithi163.stdout:1762: delete oid 11 current snap is 215 2023-10-09T09:12:52.098 INFO:tasks.rados.rados.0.smithi163.stdout:1760: finishing rollback tid 1 to smithi16367185-48 2023-10-09T09:12:52.098 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:12:52.098 INFO:tasks.rados.rados.0.smithi163.stdout:1761: finishing write tid 1 to smithi16367185-24 2023-10-09T09:12:52.098 INFO:tasks.rados.rados.0.smithi163.stdout:1761: finishing write tid 2 to smithi16367185-24 2023-10-09T09:12:52.101 INFO:tasks.rados.rados.0.smithi163.stdout:1761: finishing write tid 3 to smithi16367185-24 2023-10-09T09:12:52.101 INFO:tasks.rados.rados.0.smithi163.stdout:1761: finishing write tid 4 to smithi16367185-24 2023-10-09T09:12:52.101 INFO:tasks.rados.rados.0.smithi163.stdout:1760: done (2 left) 2023-10-09T09:12:52.102 INFO:tasks.rados.rados.0.smithi163.stdout:1762: done (1 left) 2023-10-09T09:12:52.102 INFO:tasks.rados.rados.0.smithi163.stdout:1763: read oid 33 snap 214 2023-10-09T09:12:52.102 INFO:tasks.rados.rados.0.smithi163.stdout:1763: expect (ObjNum 634 snap 202 seq_num 634) 2023-10-09T09:12:52.102 INFO:tasks.rados.rados.0.smithi163.stdout:1764: write oid 31 current snap is 215 2023-10-09T09:12:52.102 INFO:tasks.rados.rados.0.smithi163.stdout:1764: seq_num 679 ranges {611953=649488,1950097=529565,3246565=165388} 2023-10-09T09:12:52.112 INFO:tasks.rados.rados.0.smithi163.stdout:1764: writing smithi16367185-31 from 611953 to 1261441 tid 1 2023-10-09T09:12:52.119 INFO:tasks.rados.rados.0.smithi163.stdout:1764: writing smithi16367185-31 from 1950097 to 2479662 tid 2 2023-10-09T09:12:52.125 INFO:tasks.rados.rados.0.smithi163.stdout:1764: writing smithi16367185-31 from 3246565 to 3411953 tid 3 2023-10-09T09:12:52.125 INFO:tasks.rados.rados.0.smithi163.stdout:1765: snap_create 2023-10-09T09:12:52.125 INFO:tasks.rados.rados.0.smithi163.stdout:1761: finishing write tid 5 to smithi16367185-24 2023-10-09T09:12:52.125 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 447 (ObjNum 677 snap 215 seq_num 677) dirty exists 2023-10-09T09:12:52.125 INFO:tasks.rados.rados.0.smithi163.stdout:1761: left oid 24 (ObjNum 677 snap 215 seq_num 677) 2023-10-09T09:12:52.125 INFO:tasks.rados.rados.0.smithi163.stdout:1763: expect (ObjNum 634 snap 202 seq_num 634) 2023-10-09T09:12:52.260 INFO:tasks.rados.rados.0.smithi163.stdout:1764: finishing write tid 1 to smithi16367185-31 2023-10-09T09:12:52.261 INFO:tasks.rados.rados.0.smithi163.stdout:1764: finishing write tid 2 to smithi16367185-31 2023-10-09T09:12:52.261 INFO:tasks.rados.rados.0.smithi163.stdout:1764: finishing write tid 3 to smithi16367185-31 2023-10-09T09:12:52.261 INFO:tasks.rados.rados.0.smithi163.stdout:1764: finishing write tid 4 to smithi16367185-31 2023-10-09T09:12:52.261 INFO:tasks.rados.rados.0.smithi163.stdout:1764: finishing write tid 5 to smithi16367185-31 2023-10-09T09:12:52.261 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 481 (ObjNum 679 snap 215 seq_num 679) dirty exists 2023-10-09T09:12:52.261 INFO:tasks.rados.rados.0.smithi163.stdout:1764: left oid 31 (ObjNum 679 snap 215 seq_num 679) 2023-10-09T09:12:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cluster 2023-10-09T09:12:50.656438+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cluster 2023-10-09T09:12:50.968935+0000 osd.0 (osd.0) 80 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cluster 2023-10-09T09:12:51.049392+0000 osd.0 (osd.0) 81 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cluster 2023-10-09T09:12:51.283282+0000 mgr.y (mgr.34107) 141 : cluster 0 pgmap v301: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2023-10-09T09:12:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: audit 2023-10-09T09:12:51.791513+0000 mon.a (mon.0) 404 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: audit 2023-10-09T09:12:51.800147+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cephadm 2023-10-09T09:12:51.819744+0000 mgr.y (mgr.34107) 142 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T09:12:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: audit 2023-10-09T09:12:51.820357+0000 mon.a (mon.0) 406 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:12:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: audit 2023-10-09T09:12:51.821991+0000 mon.a (mon.0) 407 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cephadm 2023-10-09T09:12:51.826750+0000 mgr.y (mgr.34107) 143 : cephadm 1 Reconfiguring daemon osd.0 on smithi163 2023-10-09T09:12:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cluster 2023-10-09T09:12:52.060244+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cluster 2023-10-09T09:12:52.062086+0000 osd.3 (osd.3) 136 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:52.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cluster 2023-10-09T09:12:52.064671+0000 mon.a (mon.0) 408 : cluster 0 osdmap e500: 8 total, 8 up, 8 in 2023-10-09T09:12:52.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cluster 2023-10-09T09:12:52.064924+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e500 with expected crc 2023-10-09T09:12:52.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:52 smithi181 bash[59345]: cluster 2023-10-09T09:12:52.102999+0000 osd.2 (osd.2) 130 : cluster 3 failed to encode map e500 with expected crc 2023-10-09T09:12:52.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cluster 2023-10-09T09:12:50.656438+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:52.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cluster 2023-10-09T09:12:50.968935+0000 osd.0 (osd.0) 80 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:52.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cluster 2023-10-09T09:12:51.049392+0000 osd.0 (osd.0) 81 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:52.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cluster 2023-10-09T09:12:51.283282+0000 mgr.y (mgr.34107) 141 : cluster 0 pgmap v301: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2023-10-09T09:12:52.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: audit 2023-10-09T09:12:51.791513+0000 mon.a (mon.0) 404 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:52.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: audit 2023-10-09T09:12:51.800147+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:52.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cephadm 2023-10-09T09:12:51.819744+0000 mgr.y (mgr.34107) 142 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T09:12:52.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: audit 2023-10-09T09:12:51.820357+0000 mon.a (mon.0) 406 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:12:52.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: audit 2023-10-09T09:12:51.821991+0000 mon.a (mon.0) 407 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:52.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cephadm 2023-10-09T09:12:51.826750+0000 mgr.y (mgr.34107) 143 : cephadm 1 Reconfiguring daemon osd.0 on smithi163 2023-10-09T09:12:52.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cluster 2023-10-09T09:12:52.060244+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:52.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cluster 2023-10-09T09:12:52.062086+0000 osd.3 (osd.3) 136 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:52.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cluster 2023-10-09T09:12:52.064671+0000 mon.a (mon.0) 408 : cluster 0 osdmap e500: 8 total, 8 up, 8 in 2023-10-09T09:12:52.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cluster 2023-10-09T09:12:52.064924+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e500 with expected crc 2023-10-09T09:12:52.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[75425]: cluster 2023-10-09T09:12:52.102999+0000 osd.2 (osd.2) 130 : cluster 3 failed to encode map e500 with expected crc 2023-10-09T09:12:52.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cluster 2023-10-09T09:12:50.656438+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:52.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cluster 2023-10-09T09:12:50.968935+0000 osd.0 (osd.0) 80 : cluster 3 failed to encode map e498 with expected crc 2023-10-09T09:12:52.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cluster 2023-10-09T09:12:51.049392+0000 osd.0 (osd.0) 81 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cluster 2023-10-09T09:12:51.283282+0000 mgr.y (mgr.34107) 141 : cluster 0 pgmap v301: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: audit 2023-10-09T09:12:51.791513+0000 mon.a (mon.0) 404 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: audit 2023-10-09T09:12:51.800147+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cephadm 2023-10-09T09:12:51.819744+0000 mgr.y (mgr.34107) 142 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: audit 2023-10-09T09:12:51.820357+0000 mon.a (mon.0) 406 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: audit 2023-10-09T09:12:51.821991+0000 mon.a (mon.0) 407 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cephadm 2023-10-09T09:12:51.826750+0000 mgr.y (mgr.34107) 143 : cephadm 1 Reconfiguring daemon osd.0 on smithi163 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cluster 2023-10-09T09:12:52.060244+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cluster 2023-10-09T09:12:52.062086+0000 osd.3 (osd.3) 136 : cluster 3 failed to encode map e499 with expected crc 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cluster 2023-10-09T09:12:52.064671+0000 mon.a (mon.0) 408 : cluster 0 osdmap e500: 8 total, 8 up, 8 in 2023-10-09T09:12:52.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cluster 2023-10-09T09:12:52.064924+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e500 with expected crc 2023-10-09T09:12:52.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:52 smithi163 bash[78107]: cluster 2023-10-09T09:12:52.102999+0000 osd.2 (osd.2) 130 : cluster 3 failed to encode map e500 with expected crc 2023-10-09T09:12:53.065 INFO:tasks.rados.rados.0.smithi163.stdout:1761: done (3 left) 2023-10-09T09:12:53.065 INFO:tasks.rados.rados.0.smithi163.stdout:1763: done (2 left) 2023-10-09T09:12:53.065 INFO:tasks.rados.rados.0.smithi163.stdout:1764: done (1 left) 2023-10-09T09:12:53.066 INFO:tasks.rados.rados.0.smithi163.stdout:1765: done (0 left) 2023-10-09T09:12:53.066 INFO:tasks.rados.rados.0.smithi163.stdout:1766: rollback oid 26 current snap is 216 2023-10-09T09:12:53.066 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 26 to 213 2023-10-09T09:12:53.066 INFO:tasks.rados.rados.0.smithi163.stdout:1767: read oid 24 snap -1 2023-10-09T09:12:53.066 INFO:tasks.rados.rados.0.smithi163.stdout:1767: expect (ObjNum 677 snap 215 seq_num 677) 2023-10-09T09:12:53.066 INFO:tasks.rados.rados.0.smithi163.stdout:1768: delete oid 31 current snap is 216 2023-10-09T09:12:53.076 INFO:tasks.rados.rados.0.smithi163.stdout:1767: expect (ObjNum 677 snap 215 seq_num 677) 2023-10-09T09:12:53.193 INFO:tasks.rados.rados.0.smithi163.stdout:1766: finishing rollback tid 0 to smithi16367185-26 2023-10-09T09:12:53.194 INFO:tasks.rados.rados.0.smithi163.stdout:1766: finishing rollback tid 1 to smithi16367185-26 2023-10-09T09:12:53.194 INFO:tasks.rados.rados.0.smithi163.stdout:1766: finishing rollback tid 2 to smithi16367185-26 2023-10-09T09:12:53.194 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 566 (ObjNum 594 snap 187 seq_num 594) dirty exists 2023-10-09T09:12:53.194 INFO:tasks.rados.rados.0.smithi163.stdout:1766: done (2 left) 2023-10-09T09:12:53.194 INFO:tasks.rados.rados.0.smithi163.stdout:1767: done (1 left) 2023-10-09T09:12:53.194 INFO:tasks.rados.rados.0.smithi163.stdout:1768: done (0 left) 2023-10-09T09:12:53.194 INFO:tasks.rados.rados.0.smithi163.stdout:1769: rollback oid 40 current snap is 216 2023-10-09T09:12:53.194 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 40 to 213 2023-10-09T09:12:53.194 INFO:tasks.rados.rados.0.smithi163.stdout:1770: read oid 30 snap -1 2023-10-09T09:12:53.194 INFO:tasks.rados.rados.0.smithi163.stdout:1770: expect deleted 2023-10-09T09:12:53.195 INFO:tasks.rados.rados.0.smithi163.stdout:1771: snap_create 2023-10-09T09:12:53.195 INFO:tasks.rados.rados.0.smithi163.stdout:1769: finishing rollback tid 1 to smithi16367185-40 2023-10-09T09:12:53.195 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 0 (ObjNum 669367496 snap 21875 seq_num 2) dirty dne 2023-10-09T09:12:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:12:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:12:53] "GET /metrics HTTP/1.1" 200 36078 "" "Prometheus/2.33.4" 2023-10-09T09:12:54.075 INFO:tasks.rados.rados.0.smithi163.stdout:1769: done (2 left) 2023-10-09T09:12:54.076 INFO:tasks.rados.rados.0.smithi163.stdout:1770: done (1 left) 2023-10-09T09:12:54.076 INFO:tasks.rados.rados.0.smithi163.stdout:1771: done (0 left) 2023-10-09T09:12:54.076 INFO:tasks.rados.rados.0.smithi163.stdout:1772: write oid 35 current snap is 217 2023-10-09T09:12:54.076 INFO:tasks.rados.rados.0.smithi163.stdout:1772: seq_num 681 ranges {645567=799986,1950708=533234,3185058=260509} 2023-10-09T09:12:54.085 INFO:tasks.rados.rados.0.smithi163.stdout:1772: writing smithi16367185-35 from 645567 to 1445553 tid 1 2023-10-09T09:12:54.091 INFO:tasks.rados.rados.0.smithi163.stdout:1772: writing smithi16367185-35 from 1950708 to 2483942 tid 2 2023-10-09T09:12:54.097 INFO:tasks.rados.rados.0.smithi163.stdout:1772: writing smithi16367185-35 from 3185058 to 3445567 tid 3 2023-10-09T09:12:54.097 INFO:tasks.rados.rados.0.smithi163.stdout:1773: write oid 48 current snap is 217 2023-10-09T09:12:54.097 INFO:tasks.rados.rados.0.smithi163.stdout:1773: seq_num 682 ranges {662374=475235,1609190=676892,2812481=624101} 2023-10-09T09:12:54.103 INFO:tasks.rados.rados.0.smithi163.stdout:1773: writing smithi16367185-48 from 662374 to 1137609 tid 1 2023-10-09T09:12:54.109 INFO:tasks.rados.rados.0.smithi163.stdout:1773: writing smithi16367185-48 from 1609190 to 2286082 tid 2 2023-10-09T09:12:54.117 INFO:tasks.rados.rados.0.smithi163.stdout:1773: writing smithi16367185-48 from 2812481 to 3436582 tid 3 2023-10-09T09:12:54.117 INFO:tasks.rados.rados.0.smithi163.stdout:1774: rollback oid 20 current snap is 217 2023-10-09T09:12:54.117 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 213 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1772: finishing write tid 1 to smithi16367185-35 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1772: finishing write tid 2 to smithi16367185-35 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1772: finishing write tid 3 to smithi16367185-35 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1772: finishing write tid 4 to smithi16367185-35 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1772: finishing write tid 5 to smithi16367185-35 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 479 (ObjNum 681 snap 217 seq_num 681) dirty exists 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1772: left oid 35 (ObjNum 681 snap 217 seq_num 681) 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1773: finishing write tid 1 to smithi16367185-48 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1773: finishing write tid 2 to smithi16367185-48 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1772: done (2 left) 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1775: read oid 35 snap -1 2023-10-09T09:12:54.118 INFO:tasks.rados.rados.0.smithi163.stdout:1775: expect (ObjNum 681 snap 217 seq_num 681) 2023-10-09T09:12:54.119 INFO:tasks.rados.rados.0.smithi163.stdout:1776: snap_create 2023-10-09T09:12:54.119 INFO:tasks.rados.rados.0.smithi163.stdout:1774: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:12:54.119 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 0 (ObjNum 288 snap 0 seq_num 33) dirty dne 2023-10-09T09:12:54.122 INFO:tasks.rados.rados.0.smithi163.stdout:1773: finishing write tid 3 to smithi16367185-48 2023-10-09T09:12:54.122 INFO:tasks.rados.rados.0.smithi163.stdout:1773: finishing write tid 4 to smithi16367185-48 2023-10-09T09:12:54.122 INFO:tasks.rados.rados.0.smithi163.stdout:1773: finishing write tid 5 to smithi16367185-48 2023-10-09T09:12:54.122 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 1330 (ObjNum 682 snap 217 seq_num 682) dirty exists 2023-10-09T09:12:54.123 INFO:tasks.rados.rados.0.smithi163.stdout:1773: left oid 48 (ObjNum 682 snap 217 seq_num 682) 2023-10-09T09:12:54.131 INFO:tasks.rados.rados.0.smithi163.stdout:1775: expect (ObjNum 681 snap 217 seq_num 681) 2023-10-09T09:12:54.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: cluster 2023-10-09T09:12:53.061692+0000 osd.1 (osd.1) 64 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: cluster 2023-10-09T09:12:53.062632+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: cluster 2023-10-09T09:12:53.065020+0000 mon.a (mon.0) 409 : cluster 0 osdmap e501: 8 total, 8 up, 8 in 2023-10-09T09:12:54.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: cluster 2023-10-09T09:12:53.067552+0000 osd.3 (osd.3) 137 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: cluster 2023-10-09T09:12:53.068840+0000 osd.2 (osd.2) 131 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: cluster 2023-10-09T09:12:53.070130+0000 osd.0 (osd.0) 82 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: cluster 2023-10-09T09:12:53.283771+0000 mgr.y (mgr.34107) 144 : cluster 0 pgmap v304: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2023-10-09T09:12:54.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: audit 2023-10-09T09:12:54.018111+0000 mon.a (mon.0) 410 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:54.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: audit 2023-10-09T09:12:54.027950+0000 mon.a (mon.0) 411 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:54.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: audit 2023-10-09T09:12:54.045680+0000 mon.a (mon.0) 412 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:12:54.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[78107]: audit 2023-10-09T09:12:54.047464+0000 mon.a (mon.0) 413 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:54.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: cluster 2023-10-09T09:12:53.061692+0000 osd.1 (osd.1) 64 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: cluster 2023-10-09T09:12:53.062632+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: cluster 2023-10-09T09:12:53.065020+0000 mon.a (mon.0) 409 : cluster 0 osdmap e501: 8 total, 8 up, 8 in 2023-10-09T09:12:54.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: cluster 2023-10-09T09:12:53.067552+0000 osd.3 (osd.3) 137 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: cluster 2023-10-09T09:12:53.068840+0000 osd.2 (osd.2) 131 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: cluster 2023-10-09T09:12:53.070130+0000 osd.0 (osd.0) 82 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: cluster 2023-10-09T09:12:53.283771+0000 mgr.y (mgr.34107) 144 : cluster 0 pgmap v304: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2023-10-09T09:12:54.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: audit 2023-10-09T09:12:54.018111+0000 mon.a (mon.0) 410 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:54.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: audit 2023-10-09T09:12:54.027950+0000 mon.a (mon.0) 411 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:54.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: audit 2023-10-09T09:12:54.045680+0000 mon.a (mon.0) 412 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:12:54.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:54 smithi163 bash[75425]: audit 2023-10-09T09:12:54.047464+0000 mon.a (mon.0) 413 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: cluster 2023-10-09T09:12:53.061692+0000 osd.1 (osd.1) 64 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: cluster 2023-10-09T09:12:53.062632+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: cluster 2023-10-09T09:12:53.065020+0000 mon.a (mon.0) 409 : cluster 0 osdmap e501: 8 total, 8 up, 8 in 2023-10-09T09:12:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: cluster 2023-10-09T09:12:53.067552+0000 osd.3 (osd.3) 137 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: cluster 2023-10-09T09:12:53.068840+0000 osd.2 (osd.2) 131 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: cluster 2023-10-09T09:12:53.070130+0000 osd.0 (osd.0) 82 : cluster 3 failed to encode map e501 with expected crc 2023-10-09T09:12:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: cluster 2023-10-09T09:12:53.283771+0000 mgr.y (mgr.34107) 144 : cluster 0 pgmap v304: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2023-10-09T09:12:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: audit 2023-10-09T09:12:54.018111+0000 mon.a (mon.0) 410 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: audit 2023-10-09T09:12:54.027950+0000 mon.a (mon.0) 411 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: audit 2023-10-09T09:12:54.045680+0000 mon.a (mon.0) 412 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:12:54.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:54 smithi181 bash[59345]: audit 2023-10-09T09:12:54.047464+0000 mon.a (mon.0) 413 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:55.054 INFO:tasks.rados.rados.0.smithi163.stdout:1773: done (3 left) 2023-10-09T09:12:55.054 INFO:tasks.rados.rados.0.smithi163.stdout:1774: done (2 left) 2023-10-09T09:12:55.054 INFO:tasks.rados.rados.0.smithi163.stdout:1775: done (1 left) 2023-10-09T09:12:55.054 INFO:tasks.rados.rados.0.smithi163.stdout:1776: done (0 left) 2023-10-09T09:12:55.054 INFO:tasks.rados.rados.0.smithi163.stdout:1777: delete oid 48 current snap is 218 2023-10-09T09:12:55.068 INFO:tasks.rados.rados.0.smithi163.stdout:1777: done (0 left) 2023-10-09T09:12:55.069 INFO:tasks.rados.rados.0.smithi163.stdout:1778: write oid 37 current snap is 218 2023-10-09T09:12:55.069 INFO:tasks.rados.rados.0.smithi163.stdout:1778: seq_num 684 ranges {695988=625733,2009801=680561,3150974=345014} 2023-10-09T09:12:55.078 INFO:tasks.rados.rados.0.smithi163.stdout:1778: writing smithi16367185-37 from 695988 to 1321721 tid 1 2023-10-09T09:12:55.086 INFO:tasks.rados.rados.0.smithi163.stdout:1778: writing smithi16367185-37 from 2009801 to 2690362 tid 2 2023-10-09T09:12:55.091 INFO:tasks.rados.rados.0.smithi163.stdout:1778: writing smithi16367185-37 from 3150974 to 3495988 tid 3 2023-10-09T09:12:55.091 INFO:tasks.rados.rados.0.smithi163.stdout:1779: snap_create 2023-10-09T09:12:55.091 INFO:tasks.rados.rados.0.smithi163.stdout:1778: finishing write tid 1 to smithi16367185-37 2023-10-09T09:12:55.091 INFO:tasks.rados.rados.0.smithi163.stdout:1778: finishing write tid 2 to smithi16367185-37 2023-10-09T09:12:55.094 INFO:tasks.rados.rados.0.smithi163.stdout:1778: finishing write tid 3 to smithi16367185-37 2023-10-09T09:12:55.095 INFO:tasks.rados.rados.0.smithi163.stdout:1778: finishing write tid 4 to smithi16367185-37 2023-10-09T09:12:55.095 INFO:tasks.rados.rados.0.smithi163.stdout:1778: finishing write tid 5 to smithi16367185-37 2023-10-09T09:12:55.095 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 870 (ObjNum 684 snap 218 seq_num 684) dirty exists 2023-10-09T09:12:55.095 INFO:tasks.rados.rados.0.smithi163.stdout:1778: left oid 37 (ObjNum 684 snap 218 seq_num 684) 2023-10-09T09:12:55.397 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[78107]: cephadm 2023-10-09T09:12:54.045102+0000 mgr.y (mgr.34107) 145 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T09:12:55.397 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[78107]: cephadm 2023-10-09T09:12:54.051868+0000 mgr.y (mgr.34107) 146 : cephadm 1 Reconfiguring daemon osd.1 on smithi163 2023-10-09T09:12:55.397 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[78107]: cluster 2023-10-09T09:12:54.066979+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.397 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[78107]: cluster 2023-10-09T09:12:54.068210+0000 osd.3 (osd.3) 138 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.397 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[78107]: cluster 2023-10-09T09:12:54.075157+0000 mon.a (mon.0) 414 : cluster 0 osdmap e502: 8 total, 8 up, 8 in 2023-10-09T09:12:55.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[78107]: cluster 2023-10-09T09:12:54.086956+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[78107]: cluster 2023-10-09T09:12:54.104009+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[78107]: cluster 2023-10-09T09:12:55.053025+0000 mon.a (mon.0) 415 : cluster 0 osdmap e503: 8 total, 8 up, 8 in 2023-10-09T09:12:55.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[75425]: cephadm 2023-10-09T09:12:54.045102+0000 mgr.y (mgr.34107) 145 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T09:12:55.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[75425]: cephadm 2023-10-09T09:12:54.051868+0000 mgr.y (mgr.34107) 146 : cephadm 1 Reconfiguring daemon osd.1 on smithi163 2023-10-09T09:12:55.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[75425]: cluster 2023-10-09T09:12:54.066979+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[75425]: cluster 2023-10-09T09:12:54.068210+0000 osd.3 (osd.3) 138 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[75425]: cluster 2023-10-09T09:12:54.075157+0000 mon.a (mon.0) 414 : cluster 0 osdmap e502: 8 total, 8 up, 8 in 2023-10-09T09:12:55.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[75425]: cluster 2023-10-09T09:12:54.086956+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[75425]: cluster 2023-10-09T09:12:54.104009+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:55 smithi163 bash[75425]: cluster 2023-10-09T09:12:55.053025+0000 mon.a (mon.0) 415 : cluster 0 osdmap e503: 8 total, 8 up, 8 in 2023-10-09T09:12:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:55 smithi181 bash[59345]: cephadm 2023-10-09T09:12:54.045102+0000 mgr.y (mgr.34107) 145 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T09:12:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:55 smithi181 bash[59345]: cephadm 2023-10-09T09:12:54.051868+0000 mgr.y (mgr.34107) 146 : cephadm 1 Reconfiguring daemon osd.1 on smithi163 2023-10-09T09:12:55.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:55 smithi181 bash[59345]: cluster 2023-10-09T09:12:54.066979+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:55 smithi181 bash[59345]: cluster 2023-10-09T09:12:54.068210+0000 osd.3 (osd.3) 138 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:55 smithi181 bash[59345]: cluster 2023-10-09T09:12:54.075157+0000 mon.a (mon.0) 414 : cluster 0 osdmap e502: 8 total, 8 up, 8 in 2023-10-09T09:12:55.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:55 smithi181 bash[59345]: cluster 2023-10-09T09:12:54.086956+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:55 smithi181 bash[59345]: cluster 2023-10-09T09:12:54.104009+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:55.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:55 smithi181 bash[59345]: cluster 2023-10-09T09:12:55.053025+0000 mon.a (mon.0) 415 : cluster 0 osdmap e503: 8 total, 8 up, 8 in 2023-10-09T09:12:56.083 INFO:tasks.rados.rados.0.smithi163.stdout:1778: done (1 left) 2023-10-09T09:12:56.083 INFO:tasks.rados.rados.0.smithi163.stdout:1779: done (0 left) 2023-10-09T09:12:56.083 INFO:tasks.rados.rados.0.smithi163.stdout:1780: delete oid 20 current snap is 219 2023-10-09T09:12:56.087 INFO:tasks.rados.rados.0.smithi163.stdout:1780: done (0 left) 2023-10-09T09:12:56.087 INFO:tasks.rados.rados.0.smithi163.stdout:1781: write oid 16 current snap is 219 2023-10-09T09:12:56.094 INFO:tasks.rados.rados.0.smithi163.stdout:1781: seq_num 686 ranges {729602=492584,1843118=684230,3322173=207429} 2023-10-09T09:12:56.098 INFO:tasks.rados.rados.0.smithi163.stdout:1781: writing smithi16367185-16 from 729602 to 1222186 tid 1 2023-10-09T09:12:56.106 INFO:tasks.rados.rados.0.smithi163.stdout:1781: writing smithi16367185-16 from 1843118 to 2527348 tid 2 2023-10-09T09:12:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1781: writing smithi16367185-16 from 3322173 to 3529602 tid 3 2023-10-09T09:12:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1782: read oid 11 snap -1 2023-10-09T09:12:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1782: expect deleted 2023-10-09T09:12:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1783: read oid 3 snap -1 2023-10-09T09:12:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1783: expect deleted 2023-10-09T09:12:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1784: rollback oid 28 current snap is 219 2023-10-09T09:12:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 217 2023-10-09T09:12:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1781: finishing write tid 1 to smithi16367185-16 2023-10-09T09:12:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1781: finishing write tid 2 to smithi16367185-16 2023-10-09T09:12:56.114 INFO:tasks.rados.rados.0.smithi163.stdout:1785: write oid 26 current snap is 219 2023-10-09T09:12:56.114 INFO:tasks.rados.rados.0.smithi163.stdout:1785: seq_num 687 ranges {746409=567833,1901600=544241,3065949=480460} 2023-10-09T09:12:56.120 INFO:tasks.rados.rados.0.smithi163.stdout:1785: writing smithi16367185-26 from 746409 to 1314242 tid 1 2023-10-09T09:12:56.127 INFO:tasks.rados.rados.0.smithi163.stdout:1785: writing smithi16367185-26 from 1901600 to 2445841 tid 2 2023-10-09T09:12:56.134 INFO:tasks.rados.rados.0.smithi163.stdout:1785: writing smithi16367185-26 from 3065949 to 3546409 tid 3 2023-10-09T09:12:56.134 INFO:tasks.rados.rados.0.smithi163.stdout:1786: rollback oid 45 current snap is 219 2023-10-09T09:12:56.134 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 45 to 215 2023-10-09T09:12:56.134 INFO:tasks.rados.rados.0.smithi163.stdout:1781: finishing write tid 3 to smithi16367185-16 2023-10-09T09:12:56.134 INFO:tasks.rados.rados.0.smithi163.stdout:1783: done (5 left) 2023-10-09T09:12:56.134 INFO:tasks.rados.rados.0.smithi163.stdout:1787: write oid 13 current snap is 219 2023-10-09T09:12:56.135 INFO:tasks.rados.rados.0.smithi163.stdout:1781: finishing write tid 4 to smithi16367185-16 2023-10-09T09:12:56.135 INFO:tasks.rados.rados.0.smithi163.stdout:1781: finishing write tid 5 to smithi16367185-16 2023-10-09T09:12:56.135 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 934 (ObjNum 686 snap 219 seq_num 686) dirty exists 2023-10-09T09:12:56.135 INFO:tasks.rados.rados.0.smithi163.stdout:1781: left oid 16 (ObjNum 686 snap 219 seq_num 686) 2023-10-09T09:12:56.135 INFO:tasks.rados.rados.0.smithi163.stdout:1784: finishing rollback tid 0 to smithi16367185-28 2023-10-09T09:12:56.135 INFO:tasks.rados.rados.0.smithi163.stdout:1784: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:12:56.135 INFO:tasks.rados.rados.0.smithi163.stdout:1784: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:12:56.135 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 938 (ObjNum 675 snap 212 seq_num 675) dirty exists 2023-10-09T09:12:56.135 INFO:tasks.rados.rados.0.smithi163.stdout:1785: finishing write tid 1 to smithi16367185-26 2023-10-09T09:12:56.135 INFO:tasks.rados.rados.0.smithi163.stdout:1787: seq_num 688 ranges {763216=643082,1843729=687899,3260666=302550} 2023-10-09T09:12:56.142 INFO:tasks.rados.rados.0.smithi163.stdout:1787: writing smithi16367185-13 from 763216 to 1406298 tid 1 2023-10-09T09:12:56.149 INFO:tasks.rados.rados.0.smithi163.stdout:1787: writing smithi16367185-13 from 1843729 to 2531628 tid 2 2023-10-09T09:12:56.155 INFO:tasks.rados.rados.0.smithi163.stdout:1787: writing smithi16367185-13 from 3260666 to 3563216 tid 3 2023-10-09T09:12:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:1781: done (5 left) 2023-10-09T09:12:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:1782: done (4 left) 2023-10-09T09:12:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:1784: done (3 left) 2023-10-09T09:12:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:1788: read oid 38 snap -1 2023-10-09T09:12:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:1788: expect (ObjNum 672 snap 212 seq_num 672) 2023-10-09T09:12:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:1789: rollback oid 12 current snap is 219 2023-10-09T09:12:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 12 to 215 2023-10-09T09:12:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:1785: finishing write tid 2 to smithi16367185-26 2023-10-09T09:12:56.156 INFO:tasks.rados.rados.0.smithi163.stdout:1786: finishing rollback tid 1 to smithi16367185-45 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 0 (ObjNum 698837312 snap 21875 seq_num 697217040) dirty dne 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:1785: finishing write tid 3 to smithi16367185-26 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:1785: finishing write tid 4 to smithi16367185-26 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:1785: finishing write tid 5 to smithi16367185-26 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 571 (ObjNum 687 snap 219 seq_num 687) dirty exists 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:1785: left oid 26 (ObjNum 687 snap 219 seq_num 687) 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:1787: finishing write tid 1 to smithi16367185-13 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:1787: finishing write tid 2 to smithi16367185-13 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:1785: done (4 left) 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:1786: done (3 left) 2023-10-09T09:12:56.157 INFO:tasks.rados.rados.0.smithi163.stdout:1790: read oid 7 snap -1 2023-10-09T09:12:56.158 INFO:tasks.rados.rados.0.smithi163.stdout:1790: expect deleted 2023-10-09T09:12:56.158 INFO:tasks.rados.rados.0.smithi163.stdout:1791: snap_remove snap 213 2023-10-09T09:12:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[75425]: cluster 2023-10-09T09:12:55.048722+0000 osd.3 (osd.3) 139 : cluster 3 failed to encode map e503 with expected crc 2023-10-09T09:12:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[75425]: cluster 2023-10-09T09:12:55.049033+0000 osd.2 (osd.2) 132 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[75425]: cluster 2023-10-09T09:12:55.057781+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e503 with expected crc 2023-10-09T09:12:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[75425]: cluster 2023-10-09T09:12:55.079438+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e503 with expected crc 2023-10-09T09:12:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[75425]: cluster 2023-10-09T09:12:55.284463+0000 mgr.y (mgr.34107) 147 : cluster 0 pgmap v307: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[75425]: audit 2023-10-09T09:12:55.340290+0000 mon.a (mon.0) 416 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[78107]: cluster 2023-10-09T09:12:55.048722+0000 osd.3 (osd.3) 139 : cluster 3 failed to encode map e503 with expected crc 2023-10-09T09:12:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[78107]: cluster 2023-10-09T09:12:55.049033+0000 osd.2 (osd.2) 132 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[78107]: cluster 2023-10-09T09:12:55.057781+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e503 with expected crc 2023-10-09T09:12:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[78107]: cluster 2023-10-09T09:12:55.079438+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e503 with expected crc 2023-10-09T09:12:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[78107]: cluster 2023-10-09T09:12:55.284463+0000 mgr.y (mgr.34107) 147 : cluster 0 pgmap v307: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:56 smithi163 bash[78107]: audit 2023-10-09T09:12:55.340290+0000 mon.a (mon.0) 416 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:56 smithi181 bash[59345]: cluster 2023-10-09T09:12:55.048722+0000 osd.3 (osd.3) 139 : cluster 3 failed to encode map e503 with expected crc 2023-10-09T09:12:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:56 smithi181 bash[59345]: cluster 2023-10-09T09:12:55.049033+0000 osd.2 (osd.2) 132 : cluster 3 failed to encode map e502 with expected crc 2023-10-09T09:12:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:56 smithi181 bash[59345]: cluster 2023-10-09T09:12:55.057781+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e503 with expected crc 2023-10-09T09:12:56.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:56 smithi181 bash[59345]: cluster 2023-10-09T09:12:55.079438+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e503 with expected crc 2023-10-09T09:12:56.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:56 smithi181 bash[59345]: cluster 2023-10-09T09:12:55.284463+0000 mgr.y (mgr.34107) 147 : cluster 0 pgmap v307: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:12:56.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:56 smithi181 bash[59345]: audit 2023-10-09T09:12:55.340290+0000 mon.a (mon.0) 416 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:12:57.119 INFO:tasks.rados.rados.0.smithi163.stdout:1791: done (4 left) 2023-10-09T09:12:57.119 INFO:tasks.rados.rados.0.smithi163.stdout:1792: snap_create 2023-10-09T09:12:57.119 INFO:tasks.rados.rados.0.smithi163.stdout:1787: finishing write tid 3 to smithi16367185-13 2023-10-09T09:12:57.119 INFO:tasks.rados.rados.0.smithi163.stdout:1787: finishing write tid 4 to smithi16367185-13 2023-10-09T09:12:57.119 INFO:tasks.rados.rados.0.smithi163.stdout:1787: finishing write tid 5 to smithi16367185-13 2023-10-09T09:12:57.119 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 193 (ObjNum 688 snap 219 seq_num 688) dirty exists 2023-10-09T09:12:57.119 INFO:tasks.rados.rados.0.smithi163.stdout:1787: left oid 13 (ObjNum 688 snap 219 seq_num 688) 2023-10-09T09:12:57.119 INFO:tasks.rados.rados.0.smithi163.stdout:1789: finishing rollback tid 0 to smithi16367185-12 2023-10-09T09:12:57.120 INFO:tasks.rados.rados.0.smithi163.stdout:1789: finishing rollback tid 1 to smithi16367185-12 2023-10-09T09:12:57.120 INFO:tasks.rados.rados.0.smithi163.stdout:1789: finishing rollback tid 2 to smithi16367185-12 2023-10-09T09:12:57.120 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 800 (ObjNum 655 snap 208 seq_num 655) dirty exists 2023-10-09T09:12:57.120 INFO:tasks.rados.rados.0.smithi163.stdout:1788: expect (ObjNum 672 snap 212 seq_num 672) 2023-10-09T09:12:57.397 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: cluster 2023-10-09T09:12:56.078373+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.397 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: cluster 2023-10-09T09:12:56.079240+0000 osd.1 (osd.1) 65 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.397 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: cluster 2023-10-09T09:12:56.082989+0000 mon.a (mon.0) 417 : cluster 0 osdmap e504: 8 total, 8 up, 8 in 2023-10-09T09:12:57.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: cluster 2023-10-09T09:12:56.102461+0000 osd.2 (osd.2) 133 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: audit 2023-10-09T09:12:56.103040+0000 mon.a (mon.0) 418 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:57.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: cluster 2023-10-09T09:12:56.112873+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: audit 2023-10-09T09:12:56.112903+0000 mon.a (mon.0) 419 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:57.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: cephadm 2023-10-09T09:12:56.120425+0000 mgr.y (mgr.34107) 148 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T09:12:57.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: audit 2023-10-09T09:12:56.120789+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:12:57.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: audit 2023-10-09T09:12:56.121643+0000 mon.a (mon.0) 421 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:57.398 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[78107]: cephadm 2023-10-09T09:12:56.124804+0000 mgr.y (mgr.34107) 149 : cephadm 1 Reconfiguring daemon osd.2 on smithi163 2023-10-09T09:12:57.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: cluster 2023-10-09T09:12:56.078373+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: cluster 2023-10-09T09:12:56.079240+0000 osd.1 (osd.1) 65 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: cluster 2023-10-09T09:12:56.082989+0000 mon.a (mon.0) 417 : cluster 0 osdmap e504: 8 total, 8 up, 8 in 2023-10-09T09:12:57.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: cluster 2023-10-09T09:12:56.102461+0000 osd.2 (osd.2) 133 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: audit 2023-10-09T09:12:56.103040+0000 mon.a (mon.0) 418 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:57.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: cluster 2023-10-09T09:12:56.112873+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: audit 2023-10-09T09:12:56.112903+0000 mon.a (mon.0) 419 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:57.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: cephadm 2023-10-09T09:12:56.120425+0000 mgr.y (mgr.34107) 148 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T09:12:57.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: audit 2023-10-09T09:12:56.120789+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:12:57.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: audit 2023-10-09T09:12:56.121643+0000 mon.a (mon.0) 421 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:57.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:57 smithi163 bash[75425]: cephadm 2023-10-09T09:12:56.124804+0000 mgr.y (mgr.34107) 149 : cephadm 1 Reconfiguring daemon osd.2 on smithi163 2023-10-09T09:12:57.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: cluster 2023-10-09T09:12:56.078373+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: cluster 2023-10-09T09:12:56.079240+0000 osd.1 (osd.1) 65 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: cluster 2023-10-09T09:12:56.082989+0000 mon.a (mon.0) 417 : cluster 0 osdmap e504: 8 total, 8 up, 8 in 2023-10-09T09:12:57.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: cluster 2023-10-09T09:12:56.102461+0000 osd.2 (osd.2) 133 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: audit 2023-10-09T09:12:56.103040+0000 mon.a (mon.0) 418 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:57.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: cluster 2023-10-09T09:12:56.112873+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e504 with expected crc 2023-10-09T09:12:57.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: audit 2023-10-09T09:12:56.112903+0000 mon.a (mon.0) 419 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:57.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: cephadm 2023-10-09T09:12:56.120425+0000 mgr.y (mgr.34107) 148 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T09:12:57.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: audit 2023-10-09T09:12:56.120789+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:12:57.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: audit 2023-10-09T09:12:56.121643+0000 mon.a (mon.0) 421 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:57.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:57 smithi181 bash[59345]: cephadm 2023-10-09T09:12:56.124804+0000 mgr.y (mgr.34107) 149 : cephadm 1 Reconfiguring daemon osd.2 on smithi163 2023-10-09T09:12:58.124 INFO:tasks.rados.rados.0.smithi163.stdout:1787: done (4 left) 2023-10-09T09:12:58.125 INFO:tasks.rados.rados.0.smithi163.stdout:1788: done (3 left) 2023-10-09T09:12:58.125 INFO:tasks.rados.rados.0.smithi163.stdout:1789: done (2 left) 2023-10-09T09:12:58.125 INFO:tasks.rados.rados.0.smithi163.stdout:1790: done (1 left) 2023-10-09T09:12:58.125 INFO:tasks.rados.rados.0.smithi163.stdout:1792: done (0 left) 2023-10-09T09:12:58.125 INFO:tasks.rados.rados.0.smithi163.stdout:1793: read oid 43 snap 219 2023-10-09T09:12:58.125 INFO:tasks.rados.rados.0.smithi163.stdout:1793: expect (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:12:58.125 INFO:tasks.rados.rados.0.smithi163.stdout:1794: write oid 48 current snap is 220 2023-10-09T09:12:58.125 INFO:tasks.rados.rados.0.smithi163.stdout:1794: seq_num 689 ranges {780023=718331,1902211=547910,3004442=575581} 2023-10-09T09:12:58.137 INFO:tasks.rados.rados.0.smithi163.stdout:1794: writing smithi16367185-48 from 780023 to 1498354 tid 1 2023-10-09T09:12:58.144 INFO:tasks.rados.rados.0.smithi163.stdout:1794: writing smithi16367185-48 from 1902211 to 2450121 tid 2 2023-10-09T09:12:58.151 INFO:tasks.rados.rados.0.smithi163.stdout:1794: writing smithi16367185-48 from 3004442 to 3580023 tid 3 2023-10-09T09:12:58.151 INFO:tasks.rados.rados.0.smithi163.stdout:1795: write oid 7 current snap is 220 2023-10-09T09:12:58.151 INFO:tasks.rados.rados.0.smithi163.stdout:1795: seq_num 690 ranges {796830=793580,2360693=691568} 2023-10-09T09:12:58.160 INFO:tasks.rados.rados.0.smithi163.stdout:1795: writing smithi16367185-7 from 796830 to 1590410 tid 1 2023-10-09T09:12:58.168 INFO:tasks.rados.rados.0.smithi163.stdout:1795: writing smithi16367185-7 from 2360693 to 3052261 tid 2 2023-10-09T09:12:58.168 INFO:tasks.rados.rados.0.smithi163.stdout:1796: snap_create 2023-10-09T09:12:58.169 INFO:tasks.rados.rados.0.smithi163.stdout:1793: expect (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:12:58.282 INFO:tasks.rados.rados.0.smithi163.stdout:1794: finishing write tid 1 to smithi16367185-48 2023-10-09T09:12:58.282 INFO:tasks.rados.rados.0.smithi163.stdout:1794: finishing write tid 2 to smithi16367185-48 2023-10-09T09:12:58.282 INFO:tasks.rados.rados.0.smithi163.stdout:1794: finishing write tid 3 to smithi16367185-48 2023-10-09T09:12:58.282 INFO:tasks.rados.rados.0.smithi163.stdout:1794: finishing write tid 4 to smithi16367185-48 2023-10-09T09:12:58.282 INFO:tasks.rados.rados.0.smithi163.stdout:1794: finishing write tid 5 to smithi16367185-48 2023-10-09T09:12:58.282 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 1336 (ObjNum 689 snap 220 seq_num 689) dirty exists 2023-10-09T09:12:58.282 INFO:tasks.rados.rados.0.smithi163.stdout:1794: left oid 48 (ObjNum 689 snap 220 seq_num 689) 2023-10-09T09:12:58.282 INFO:tasks.rados.rados.0.smithi163.stdout:1795: finishing write tid 1 to smithi16367185-7 2023-10-09T09:12:58.283 INFO:tasks.rados.rados.0.smithi163.stdout:1795: finishing write tid 2 to smithi16367185-7 2023-10-09T09:12:58.283 INFO:tasks.rados.rados.0.smithi163.stdout:1795: finishing write tid 3 to smithi16367185-7 2023-10-09T09:12:58.283 INFO:tasks.rados.rados.0.smithi163.stdout:1795: finishing write tid 4 to smithi16367185-7 2023-10-09T09:12:58.283 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 434 (ObjNum 690 snap 220 seq_num 690) dirty exists 2023-10-09T09:12:58.283 INFO:tasks.rados.rados.0.smithi163.stdout:1795: left oid 7 (ObjNum 690 snap 220 seq_num 690) 2023-10-09T09:12:58.435 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:58 smithi163 bash[78107]: cluster 2023-10-09T09:12:57.112256+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e505 with expected crc 2023-10-09T09:12:58.435 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:58 smithi163 bash[78107]: cluster 2023-10-09T09:12:57.113305+0000 osd.3 (osd.3) 140 : cluster 3 failed to encode map e505 with expected crc 2023-10-09T09:12:58.436 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:58 smithi163 bash[78107]: cluster 2023-10-09T09:12:57.118779+0000 mon.a (mon.0) 422 : cluster 0 osdmap e505: 8 total, 8 up, 8 in 2023-10-09T09:12:58.436 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:58 smithi163 bash[78107]: cluster 2023-10-09T09:12:57.284935+0000 mgr.y (mgr.34107) 150 : cluster 0 pgmap v310: 89 pgs: 89 active+clean; 98 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2023-10-09T09:12:58.436 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:58 smithi163 bash[75425]: cluster 2023-10-09T09:12:57.112256+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e505 with expected crc 2023-10-09T09:12:58.436 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:58 smithi163 bash[75425]: cluster 2023-10-09T09:12:57.113305+0000 osd.3 (osd.3) 140 : cluster 3 failed to encode map e505 with expected crc 2023-10-09T09:12:58.436 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:58 smithi163 bash[75425]: cluster 2023-10-09T09:12:57.118779+0000 mon.a (mon.0) 422 : cluster 0 osdmap e505: 8 total, 8 up, 8 in 2023-10-09T09:12:58.436 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:58 smithi163 bash[75425]: cluster 2023-10-09T09:12:57.284935+0000 mgr.y (mgr.34107) 150 : cluster 0 pgmap v310: 89 pgs: 89 active+clean; 98 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2023-10-09T09:12:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:58 smithi181 bash[59345]: cluster 2023-10-09T09:12:57.112256+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e505 with expected crc 2023-10-09T09:12:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:58 smithi181 bash[59345]: cluster 2023-10-09T09:12:57.113305+0000 osd.3 (osd.3) 140 : cluster 3 failed to encode map e505 with expected crc 2023-10-09T09:12:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:58 smithi181 bash[59345]: cluster 2023-10-09T09:12:57.118779+0000 mon.a (mon.0) 422 : cluster 0 osdmap e505: 8 total, 8 up, 8 in 2023-10-09T09:12:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:58 smithi181 bash[59345]: cluster 2023-10-09T09:12:57.284935+0000 mgr.y (mgr.34107) 150 : cluster 0 pgmap v310: 89 pgs: 89 active+clean; 98 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2023-10-09T09:12:59.141 INFO:tasks.rados.rados.0.smithi163.stdout:1793: done (3 left) 2023-10-09T09:12:59.141 INFO:tasks.rados.rados.0.smithi163.stdout:1794: done (2 left) 2023-10-09T09:12:59.141 INFO:tasks.rados.rados.0.smithi163.stdout:1795: done (1 left) 2023-10-09T09:12:59.141 INFO:tasks.rados.rados.0.smithi163.stdout:1796: done (0 left) 2023-10-09T09:12:59.141 INFO:tasks.rados.rados.0.smithi163.stdout:1797: rollback oid 29 current snap is 221 2023-10-09T09:12:59.141 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 29 to 220 2023-10-09T09:12:59.142 INFO:tasks.rados.rados.0.smithi163.stdout:1798: write oid 18 current snap is 221 2023-10-09T09:12:59.142 INFO:tasks.rados.rados.0.smithi163.stdout:1798: seq_num 691 ranges {413637=468829,1619175=435226,2542935=758255} 2023-10-09T09:12:59.150 INFO:tasks.rados.rados.0.smithi163.stdout:1798: writing smithi16367185-18 from 413637 to 882466 tid 1 2023-10-09T09:12:59.157 INFO:tasks.rados.rados.0.smithi163.stdout:1798: writing smithi16367185-18 from 1619175 to 2054401 tid 2 2023-10-09T09:12:59.164 INFO:tasks.rados.rados.0.smithi163.stdout:1798: writing smithi16367185-18 from 2542935 to 3301190 tid 3 2023-10-09T09:12:59.164 INFO:tasks.rados.rados.0.smithi163.stdout:1799: write oid 34 current snap is 221 2023-10-09T09:12:59.165 INFO:tasks.rados.rados.0.smithi163.stdout:1799: seq_num 692 ranges {430444=660431,1677657=695237,2970358=569527} 2023-10-09T09:12:59.170 INFO:tasks.rados.rados.0.smithi163.stdout:1799: writing smithi16367185-34 from 430444 to 1090875 tid 1 2023-10-09T09:12:59.177 INFO:tasks.rados.rados.0.smithi163.stdout:1799: writing smithi16367185-34 from 1677657 to 2372894 tid 2 2023-10-09T09:12:59.184 INFO:tasks.rados.rados.0.smithi163.stdout:1799: writing smithi16367185-34 from 2970358 to 3539885 tid 3 2023-10-09T09:12:59.185 INFO:tasks.rados.rados.0.smithi163.stdout:1800: write oid 38 current snap is 221 2023-10-09T09:12:59.185 INFO:tasks.rados.rados.0.smithi163.stdout:1800: seq_num 693 ranges {447251=735680,1736139=438895,2597781=780799} 2023-10-09T09:12:59.191 INFO:tasks.rados.rados.0.smithi163.stdout:1800: writing smithi16367185-38 from 447251 to 1182931 tid 1 2023-10-09T09:12:59.197 INFO:tasks.rados.rados.0.smithi163.stdout:1800: writing smithi16367185-38 from 1736139 to 2175034 tid 2 2023-10-09T09:12:59.204 INFO:tasks.rados.rados.0.smithi163.stdout:1800: writing smithi16367185-38 from 2597781 to 3378580 tid 3 2023-10-09T09:12:59.204 INFO:tasks.rados.rados.0.smithi163.stdout:1801: write oid 20 current snap is 221 2023-10-09T09:12:59.205 INFO:tasks.rados.rados.0.smithi163.stdout:1797: finishing rollback tid 0 to smithi16367185-29 2023-10-09T09:12:59.205 INFO:tasks.rados.rados.0.smithi163.stdout:1801: seq_num 694 ranges {464058=410929,1394621=698906,2625204=592071,3656107=7951} 2023-10-09T09:12:59.209 INFO:tasks.rados.rados.0.smithi163.stdout:1801: writing smithi16367185-20 from 464058 to 874987 tid 1 2023-10-09T09:12:59.218 INFO:tasks.rados.rados.0.smithi163.stdout:1801: writing smithi16367185-20 from 1394621 to 2093527 tid 2 2023-10-09T09:12:59.225 INFO:tasks.rados.rados.0.smithi163.stdout:1801: writing smithi16367185-20 from 2625204 to 3217275 tid 3 2023-10-09T09:12:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:1801: writing smithi16367185-20 from 3656107 to 3664058 tid 4 2023-10-09T09:12:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:1802: snap_create 2023-10-09T09:12:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:1798: finishing write tid 1 to smithi16367185-18 2023-10-09T09:12:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:1797: finishing rollback tid 1 to smithi16367185-29 2023-10-09T09:12:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:1797: finishing rollback tid 2 to smithi16367185-29 2023-10-09T09:12:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 806 (ObjNum 668 snap 212 seq_num 668) dirty exists 2023-10-09T09:12:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:1798: finishing write tid 2 to smithi16367185-18 2023-10-09T09:12:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:1798: finishing write tid 3 to smithi16367185-18 2023-10-09T09:12:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:1798: finishing write tid 4 to smithi16367185-18 2023-10-09T09:12:59.229 INFO:tasks.rados.rados.0.smithi163.stdout:1798: finishing write tid 5 to smithi16367185-18 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 642 (ObjNum 691 snap 221 seq_num 691) dirty exists 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:1798: left oid 18 (ObjNum 691 snap 221 seq_num 691) 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:1799: finishing write tid 1 to smithi16367185-34 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:1799: finishing write tid 2 to smithi16367185-34 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:1799: finishing write tid 3 to smithi16367185-34 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:1799: finishing write tid 4 to smithi16367185-34 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:1799: finishing write tid 5 to smithi16367185-34 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 702 (ObjNum 692 snap 221 seq_num 692) dirty exists 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:1799: left oid 34 (ObjNum 692 snap 221 seq_num 692) 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:1800: finishing write tid 1 to smithi16367185-38 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:1800: finishing write tid 2 to smithi16367185-38 2023-10-09T09:12:59.230 INFO:tasks.rados.rados.0.smithi163.stdout:1800: finishing write tid 3 to smithi16367185-38 2023-10-09T09:12:59.231 INFO:tasks.rados.rados.0.smithi163.stdout:1800: finishing write tid 4 to smithi16367185-38 2023-10-09T09:12:59.231 INFO:tasks.rados.rados.0.smithi163.stdout:1800: finishing write tid 5 to smithi16367185-38 2023-10-09T09:12:59.231 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 698 (ObjNum 693 snap 221 seq_num 693) dirty exists 2023-10-09T09:12:59.231 INFO:tasks.rados.rados.0.smithi163.stdout:1800: left oid 38 (ObjNum 693 snap 221 seq_num 693) 2023-10-09T09:12:59.231 INFO:tasks.rados.rados.0.smithi163.stdout:1801: finishing write tid 1 to smithi16367185-20 2023-10-09T09:12:59.231 INFO:tasks.rados.rados.0.smithi163.stdout:1801: finishing write tid 2 to smithi16367185-20 2023-10-09T09:12:59.232 INFO:tasks.rados.rados.0.smithi163.stdout:1801: finishing write tid 3 to smithi16367185-20 2023-10-09T09:12:59.232 INFO:tasks.rados.rados.0.smithi163.stdout:1801: finishing write tid 4 to smithi16367185-20 2023-10-09T09:12:59.232 INFO:tasks.rados.rados.0.smithi163.stdout:1801: finishing write tid 5 to smithi16367185-20 2023-10-09T09:12:59.232 INFO:tasks.rados.rados.0.smithi163.stdout:1801: finishing write tid 6 to smithi16367185-20 2023-10-09T09:12:59.232 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 198 (ObjNum 694 snap 221 seq_num 694) dirty exists 2023-10-09T09:12:59.232 INFO:tasks.rados.rados.0.smithi163.stdout:1801: left oid 20 (ObjNum 694 snap 221 seq_num 694) 2023-10-09T09:12:59.415 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[78107]: cluster 2023-10-09T09:12:58.119112+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:12:59.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[78107]: cluster 2023-10-09T09:12:58.120937+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:12:59.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[78107]: cluster 2023-10-09T09:12:58.124235+0000 mon.a (mon.0) 423 : cluster 0 osdmap e506: 8 total, 8 up, 8 in 2023-10-09T09:12:59.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[78107]: cluster 2023-10-09T09:12:58.124830+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:12:59.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[78107]: audit 2023-10-09T09:12:58.130101+0000 mon.a (mon.0) 424 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:59.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[78107]: audit 2023-10-09T09:12:58.137809+0000 mon.a (mon.0) 425 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:59.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[78107]: cephadm 2023-10-09T09:12:58.142064+0000 mgr.y (mgr.34107) 151 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T09:12:59.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[78107]: audit 2023-10-09T09:12:58.145064+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:12:59.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[78107]: audit 2023-10-09T09:12:58.146337+0000 mon.a (mon.0) 427 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:59.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[78107]: cephadm 2023-10-09T09:12:58.149680+0000 mgr.y (mgr.34107) 152 : cephadm 1 Reconfiguring daemon osd.3 on smithi163 2023-10-09T09:12:59.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[75425]: cluster 2023-10-09T09:12:58.119112+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:12:59.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[75425]: cluster 2023-10-09T09:12:58.120937+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:12:59.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[75425]: cluster 2023-10-09T09:12:58.124235+0000 mon.a (mon.0) 423 : cluster 0 osdmap e506: 8 total, 8 up, 8 in 2023-10-09T09:12:59.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[75425]: cluster 2023-10-09T09:12:58.124830+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:12:59.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[75425]: audit 2023-10-09T09:12:58.130101+0000 mon.a (mon.0) 424 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:59.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[75425]: audit 2023-10-09T09:12:58.137809+0000 mon.a (mon.0) 425 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:59.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[75425]: cephadm 2023-10-09T09:12:58.142064+0000 mgr.y (mgr.34107) 151 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T09:12:59.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[75425]: audit 2023-10-09T09:12:58.145064+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:12:59.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[75425]: audit 2023-10-09T09:12:58.146337+0000 mon.a (mon.0) 427 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:59.418 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:12:59 smithi163 bash[75425]: cephadm 2023-10-09T09:12:58.149680+0000 mgr.y (mgr.34107) 152 : cephadm 1 Reconfiguring daemon osd.3 on smithi163 2023-10-09T09:12:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:59 smithi181 bash[59345]: cluster 2023-10-09T09:12:58.119112+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:12:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:59 smithi181 bash[59345]: cluster 2023-10-09T09:12:58.120937+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:12:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:59 smithi181 bash[59345]: cluster 2023-10-09T09:12:58.124235+0000 mon.a (mon.0) 423 : cluster 0 osdmap e506: 8 total, 8 up, 8 in 2023-10-09T09:12:59.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:59 smithi181 bash[59345]: cluster 2023-10-09T09:12:58.124830+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:12:59.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:59 smithi181 bash[59345]: audit 2023-10-09T09:12:58.130101+0000 mon.a (mon.0) 424 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:59.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:59 smithi181 bash[59345]: audit 2023-10-09T09:12:58.137809+0000 mon.a (mon.0) 425 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:12:59.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:59 smithi181 bash[59345]: cephadm 2023-10-09T09:12:58.142064+0000 mgr.y (mgr.34107) 151 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T09:12:59.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:59 smithi181 bash[59345]: audit 2023-10-09T09:12:58.145064+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:12:59.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:59 smithi181 bash[59345]: audit 2023-10-09T09:12:58.146337+0000 mon.a (mon.0) 427 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:12:59.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:12:59 smithi181 bash[59345]: cephadm 2023-10-09T09:12:58.149680+0000 mgr.y (mgr.34107) 152 : cephadm 1 Reconfiguring daemon osd.3 on smithi163 2023-10-09T09:13:00.155 INFO:tasks.rados.rados.0.smithi163.stdout:1797: done (5 left) 2023-10-09T09:13:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:1798: done (4 left) 2023-10-09T09:13:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:1799: done (3 left) 2023-10-09T09:13:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:1800: done (2 left) 2023-10-09T09:13:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:1801: done (1 left) 2023-10-09T09:13:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:1802: done (0 left) 2023-10-09T09:13:00.156 INFO:tasks.rados.rados.0.smithi163.stdout:1803: snap_create 2023-10-09T09:13:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[78107]: cluster 2023-10-09T09:12:58.469949+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:13:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[78107]: cluster 2023-10-09T09:12:59.136423+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e507 with expected crc 2023-10-09T09:13:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[78107]: cluster 2023-10-09T09:12:59.136964+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e507 with expected crc 2023-10-09T09:13:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[78107]: cluster 2023-10-09T09:12:59.141012+0000 mon.a (mon.0) 428 : cluster 0 osdmap e507: 8 total, 8 up, 8 in 2023-10-09T09:13:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[78107]: cluster 2023-10-09T09:12:59.142924+0000 osd.2 (osd.2) 134 : cluster 3 failed to encode map e507 with expected crc 2023-10-09T09:13:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[78107]: cluster 2023-10-09T09:12:59.285629+0000 mgr.y (mgr.34107) 153 : cluster 0 pgmap v313: 89 pgs: 89 active+clean; 98 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2023-10-09T09:13:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[75425]: cluster 2023-10-09T09:12:58.469949+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:13:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[75425]: cluster 2023-10-09T09:12:59.136423+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e507 with expected crc 2023-10-09T09:13:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[75425]: cluster 2023-10-09T09:12:59.136964+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e507 with expected crc 2023-10-09T09:13:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[75425]: cluster 2023-10-09T09:12:59.141012+0000 mon.a (mon.0) 428 : cluster 0 osdmap e507: 8 total, 8 up, 8 in 2023-10-09T09:13:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[75425]: cluster 2023-10-09T09:12:59.142924+0000 osd.2 (osd.2) 134 : cluster 3 failed to encode map e507 with expected crc 2023-10-09T09:13:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:00 smithi163 bash[75425]: cluster 2023-10-09T09:12:59.285629+0000 mgr.y (mgr.34107) 153 : cluster 0 pgmap v313: 89 pgs: 89 active+clean; 98 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2023-10-09T09:13:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:00 smithi181 bash[59345]: cluster 2023-10-09T09:12:58.469949+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e506 with expected crc 2023-10-09T09:13:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:00 smithi181 bash[59345]: cluster 2023-10-09T09:12:59.136423+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e507 with expected crc 2023-10-09T09:13:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:00 smithi181 bash[59345]: cluster 2023-10-09T09:12:59.136964+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e507 with expected crc 2023-10-09T09:13:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:00 smithi181 bash[59345]: cluster 2023-10-09T09:12:59.141012+0000 mon.a (mon.0) 428 : cluster 0 osdmap e507: 8 total, 8 up, 8 in 2023-10-09T09:13:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:00 smithi181 bash[59345]: cluster 2023-10-09T09:12:59.142924+0000 osd.2 (osd.2) 134 : cluster 3 failed to encode map e507 with expected crc 2023-10-09T09:13:00.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:00 smithi181 bash[59345]: cluster 2023-10-09T09:12:59.285629+0000 mgr.y (mgr.34107) 153 : cluster 0 pgmap v313: 89 pgs: 89 active+clean; 98 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2023-10-09T09:13:01.155 INFO:tasks.rados.rados.0.smithi163.stdout:1803: done (0 left) 2023-10-09T09:13:01.155 INFO:tasks.rados.rados.0.smithi163.stdout:1804: read oid 48 snap 221 2023-10-09T09:13:01.155 INFO:tasks.rados.rados.0.smithi163.stdout:1804: expect (ObjNum 689 snap 220 seq_num 689) 2023-10-09T09:13:01.155 INFO:tasks.rados.rados.0.smithi163.stdout:1805: read oid 5 snap 222 2023-10-09T09:13:01.156 INFO:tasks.rados.rados.0.smithi163.stdout:1805: expect deleted 2023-10-09T09:13:01.156 INFO:tasks.rados.rados.0.smithi163.stdout:1806: snap_create 2023-10-09T09:13:01.173 INFO:tasks.rados.rados.0.smithi163.stdout:1804: expect (ObjNum 689 snap 220 seq_num 689) 2023-10-09T09:13:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[75425]: cluster 2023-10-09T09:13:00.148423+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[75425]: cluster 2023-10-09T09:13:00.152697+0000 osd.2 (osd.2) 135 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[75425]: cluster 2023-10-09T09:13:00.155100+0000 mon.a (mon.0) 429 : cluster 0 osdmap e508: 8 total, 8 up, 8 in 2023-10-09T09:13:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[75425]: audit 2023-10-09T09:13:00.162341+0000 mon.a (mon.0) 430 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[75425]: audit 2023-10-09T09:13:00.172394+0000 mon.a (mon.0) 431 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[75425]: cephadm 2023-10-09T09:13:00.218670+0000 mgr.y (mgr.34107) 154 : cephadm 1 Reconfiguring mds.foo.smithi163.vnodzj (monmap changed)... 2023-10-09T09:13:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[75425]: audit 2023-10-09T09:13:00.219400+0000 mon.a (mon.0) 432 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.vnodzj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:13:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[75425]: audit 2023-10-09T09:13:00.221302+0000 mon.a (mon.0) 433 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:01.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[75425]: cephadm 2023-10-09T09:13:00.223289+0000 mgr.y (mgr.34107) 155 : cephadm 1 Reconfiguring daemon mds.foo.smithi163.vnodzj on smithi163 2023-10-09T09:13:01.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[78107]: cluster 2023-10-09T09:13:00.148423+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:01.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[78107]: cluster 2023-10-09T09:13:00.152697+0000 osd.2 (osd.2) 135 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:01.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[78107]: cluster 2023-10-09T09:13:00.155100+0000 mon.a (mon.0) 429 : cluster 0 osdmap e508: 8 total, 8 up, 8 in 2023-10-09T09:13:01.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[78107]: audit 2023-10-09T09:13:00.162341+0000 mon.a (mon.0) 430 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:01.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[78107]: audit 2023-10-09T09:13:00.172394+0000 mon.a (mon.0) 431 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:01.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[78107]: cephadm 2023-10-09T09:13:00.218670+0000 mgr.y (mgr.34107) 154 : cephadm 1 Reconfiguring mds.foo.smithi163.vnodzj (monmap changed)... 2023-10-09T09:13:01.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[78107]: audit 2023-10-09T09:13:00.219400+0000 mon.a (mon.0) 432 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.vnodzj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:13:01.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[78107]: audit 2023-10-09T09:13:00.221302+0000 mon.a (mon.0) 433 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:01.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:01 smithi163 bash[78107]: cephadm 2023-10-09T09:13:00.223289+0000 mgr.y (mgr.34107) 155 : cephadm 1 Reconfiguring daemon mds.foo.smithi163.vnodzj on smithi163 2023-10-09T09:13:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:01 smithi181 bash[59345]: cluster 2023-10-09T09:13:00.148423+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:01 smithi181 bash[59345]: cluster 2023-10-09T09:13:00.152697+0000 osd.2 (osd.2) 135 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:01 smithi181 bash[59345]: cluster 2023-10-09T09:13:00.155100+0000 mon.a (mon.0) 429 : cluster 0 osdmap e508: 8 total, 8 up, 8 in 2023-10-09T09:13:01.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:01 smithi181 bash[59345]: audit 2023-10-09T09:13:00.162341+0000 mon.a (mon.0) 430 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:01.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:01 smithi181 bash[59345]: audit 2023-10-09T09:13:00.172394+0000 mon.a (mon.0) 431 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:01.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:01 smithi181 bash[59345]: cephadm 2023-10-09T09:13:00.218670+0000 mgr.y (mgr.34107) 154 : cephadm 1 Reconfiguring mds.foo.smithi163.vnodzj (monmap changed)... 2023-10-09T09:13:01.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:01 smithi181 bash[59345]: audit 2023-10-09T09:13:00.219400+0000 mon.a (mon.0) 432 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.vnodzj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:13:01.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:01 smithi181 bash[59345]: audit 2023-10-09T09:13:00.221302+0000 mon.a (mon.0) 433 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:01.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:01 smithi181 bash[59345]: cephadm 2023-10-09T09:13:00.223289+0000 mgr.y (mgr.34107) 155 : cephadm 1 Reconfiguring daemon mds.foo.smithi163.vnodzj on smithi163 2023-10-09T09:13:02.160 INFO:tasks.rados.rados.0.smithi163.stdout:1804: done (2 left) 2023-10-09T09:13:02.160 INFO:tasks.rados.rados.0.smithi163.stdout:1805: done (1 left) 2023-10-09T09:13:02.160 INFO:tasks.rados.rados.0.smithi163.stdout:1806: done (0 left) 2023-10-09T09:13:02.160 INFO:tasks.rados.rados.0.smithi163.stdout:1807: snap_create 2023-10-09T09:13:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[78107]: cluster 2023-10-09T09:13:00.446348+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[78107]: cluster 2023-10-09T09:13:00.526366+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[78107]: cluster 2023-10-09T09:13:01.150187+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[78107]: cluster 2023-10-09T09:13:01.152388+0000 osd.2 (osd.2) 136 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[78107]: cluster 2023-10-09T09:13:01.154728+0000 mon.a (mon.0) 434 : cluster 0 osdmap e509: 8 total, 8 up, 8 in 2023-10-09T09:13:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[78107]: cluster 2023-10-09T09:13:01.286583+0000 mgr.y (mgr.34107) 156 : cluster 0 pgmap v316: 89 pgs: 89 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2023-10-09T09:13:02.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[78107]: cluster 2023-10-09T09:13:01.835475+0000 osd.1 (osd.1) 66 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[75425]: cluster 2023-10-09T09:13:00.446348+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[75425]: cluster 2023-10-09T09:13:00.526366+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[75425]: cluster 2023-10-09T09:13:01.150187+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[75425]: cluster 2023-10-09T09:13:01.152388+0000 osd.2 (osd.2) 136 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[75425]: cluster 2023-10-09T09:13:01.154728+0000 mon.a (mon.0) 434 : cluster 0 osdmap e509: 8 total, 8 up, 8 in 2023-10-09T09:13:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[75425]: cluster 2023-10-09T09:13:01.286583+0000 mgr.y (mgr.34107) 156 : cluster 0 pgmap v316: 89 pgs: 89 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2023-10-09T09:13:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:02 smithi163 bash[75425]: cluster 2023-10-09T09:13:01.835475+0000 osd.1 (osd.1) 66 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:02.540 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:02 smithi181 bash[59345]: cluster 2023-10-09T09:13:00.446348+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:02.540 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:02 smithi181 bash[59345]: cluster 2023-10-09T09:13:00.526366+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e508 with expected crc 2023-10-09T09:13:02.540 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:02 smithi181 bash[59345]: cluster 2023-10-09T09:13:01.150187+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:02.540 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:02 smithi181 bash[59345]: cluster 2023-10-09T09:13:01.152388+0000 osd.2 (osd.2) 136 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:02.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:02 smithi181 bash[59345]: cluster 2023-10-09T09:13:01.154728+0000 mon.a (mon.0) 434 : cluster 0 osdmap e509: 8 total, 8 up, 8 in 2023-10-09T09:13:02.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:02 smithi181 bash[59345]: cluster 2023-10-09T09:13:01.286583+0000 mgr.y (mgr.34107) 156 : cluster 0 pgmap v316: 89 pgs: 89 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2023-10-09T09:13:02.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:02 smithi181 bash[59345]: cluster 2023-10-09T09:13:01.835475+0000 osd.1 (osd.1) 66 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:03.163 INFO:tasks.rados.rados.0.smithi163.stdout:1807: done (0 left) 2023-10-09T09:13:03.163 INFO:tasks.rados.rados.0.smithi163.stdout:1808: read oid 31 snap 219 2023-10-09T09:13:03.164 INFO:tasks.rados.rados.0.smithi163.stdout:1808: expect deleted 2023-10-09T09:13:03.164 INFO:tasks.rados.rados.0.smithi163.stdout:1809: rollback oid 3 current snap is 225 2023-10-09T09:13:03.164 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 215 2023-10-09T09:13:03.164 INFO:tasks.rados.rados.0.smithi163.stdout:1810: rollback oid 37 current snap is 225 2023-10-09T09:13:03.164 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 223 2023-10-09T09:13:03.164 INFO:tasks.rados.rados.0.smithi163.stdout:1811: write oid 34 current snap is 225 2023-10-09T09:13:03.164 INFO:tasks.rados.rados.0.smithi163.stdout:1811: seq_num 695 ranges {480865=486178,1453103=442564,2652627=519696} 2023-10-09T09:13:03.173 INFO:tasks.rados.rados.0.smithi163.stdout:1811: writing smithi16367185-34 from 480865 to 967043 tid 1 2023-10-09T09:13:03.179 INFO:tasks.rados.rados.0.smithi163.stdout:1811: writing smithi16367185-34 from 1453103 to 1895667 tid 2 2023-10-09T09:13:03.186 INFO:tasks.rados.rados.0.smithi163.stdout:1811: writing smithi16367185-34 from 2652627 to 3172323 tid 3 2023-10-09T09:13:03.186 INFO:tasks.rados.rados.0.smithi163.stdout:1812: snap_create 2023-10-09T09:13:03.186 INFO:tasks.rados.rados.0.smithi163.stdout:1809: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:13:03.187 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 0 (ObjNum 669362840 snap 21875 seq_num 1810) dirty dne 2023-10-09T09:13:03.187 INFO:tasks.rados.rados.0.smithi163.stdout:1810: finishing rollback tid 0 to smithi16367185-37 2023-10-09T09:13:03.187 INFO:tasks.rados.rados.0.smithi163.stdout:1810: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:13:03.187 INFO:tasks.rados.rados.0.smithi163.stdout:1810: finishing rollback tid 2 to smithi16367185-37 2023-10-09T09:13:03.188 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 876 (ObjNum 684 snap 218 seq_num 684) dirty exists 2023-10-09T09:13:03.189 INFO:tasks.rados.rados.0.smithi163.stdout:1811: finishing write tid 1 to smithi16367185-34 2023-10-09T09:13:03.189 INFO:tasks.rados.rados.0.smithi163.stdout:1811: finishing write tid 2 to smithi16367185-34 2023-10-09T09:13:03.189 INFO:tasks.rados.rados.0.smithi163.stdout:1811: finishing write tid 3 to smithi16367185-34 2023-10-09T09:13:03.189 INFO:tasks.rados.rados.0.smithi163.stdout:1811: finishing write tid 4 to smithi16367185-34 2023-10-09T09:13:03.190 INFO:tasks.rados.rados.0.smithi163.stdout:1811: finishing write tid 5 to smithi16367185-34 2023-10-09T09:13:03.190 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 707 (ObjNum 695 snap 225 seq_num 695) dirty exists 2023-10-09T09:13:03.190 INFO:tasks.rados.rados.0.smithi163.stdout:1811: left oid 34 (ObjNum 695 snap 225 seq_num 695) 2023-10-09T09:13:03.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:13:03] "GET /metrics HTTP/1.1" 200 36079 "" "Prometheus/2.33.4" 2023-10-09T09:13:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: cluster 2023-10-09T09:13:01.494010+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: cluster 2023-10-09T09:13:01.747377+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: cluster 2023-10-09T09:13:02.155206+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e510 with expected crc 2023-10-09T09:13:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: cluster 2023-10-09T09:13:02.156445+0000 osd.3 (osd.3) 141 : cluster 3 failed to encode map e510 with expected crc 2023-10-09T09:13:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: cluster 2023-10-09T09:13:02.159963+0000 mon.a (mon.0) 435 : cluster 0 osdmap e510: 8 total, 8 up, 8 in 2023-10-09T09:13:03.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: audit 2023-10-09T09:13:02.195373+0000 mon.a (mon.0) 436 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:03.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: audit 2023-10-09T09:13:02.201811+0000 mon.a (mon.0) 437 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:03.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: cephadm 2023-10-09T09:13:02.241775+0000 mgr.y (mgr.34107) 157 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T09:13:03.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: audit 2023-10-09T09:13:02.242539+0000 mon.a (mon.0) 438 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:13:03.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: audit 2023-10-09T09:13:02.244316+0000 mon.a (mon.0) 439 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:13:03.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: audit 2023-10-09T09:13:02.245979+0000 mon.a (mon.0) 440 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:03.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: cephadm 2023-10-09T09:13:02.247683+0000 mgr.y (mgr.34107) 158 : cephadm 1 Reconfiguring daemon mon.b on smithi181 2023-10-09T09:13:03.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[75425]: cluster 2023-10-09T09:13:03.163102+0000 mon.a (mon.0) 441 : cluster 0 osdmap e511: 8 total, 8 up, 8 in 2023-10-09T09:13:03.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: cluster 2023-10-09T09:13:01.494010+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:03.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: cluster 2023-10-09T09:13:01.747377+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:03.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: cluster 2023-10-09T09:13:02.155206+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e510 with expected crc 2023-10-09T09:13:03.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: cluster 2023-10-09T09:13:02.156445+0000 osd.3 (osd.3) 141 : cluster 3 failed to encode map e510 with expected crc 2023-10-09T09:13:03.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: cluster 2023-10-09T09:13:02.159963+0000 mon.a (mon.0) 435 : cluster 0 osdmap e510: 8 total, 8 up, 8 in 2023-10-09T09:13:03.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: audit 2023-10-09T09:13:02.195373+0000 mon.a (mon.0) 436 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:03.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: audit 2023-10-09T09:13:02.201811+0000 mon.a (mon.0) 437 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:03.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: cephadm 2023-10-09T09:13:02.241775+0000 mgr.y (mgr.34107) 157 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T09:13:03.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: audit 2023-10-09T09:13:02.242539+0000 mon.a (mon.0) 438 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:13:03.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: audit 2023-10-09T09:13:02.244316+0000 mon.a (mon.0) 439 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:13:03.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: audit 2023-10-09T09:13:02.245979+0000 mon.a (mon.0) 440 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:03.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: cephadm 2023-10-09T09:13:02.247683+0000 mgr.y (mgr.34107) 158 : cephadm 1 Reconfiguring daemon mon.b on smithi181 2023-10-09T09:13:03.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:03 smithi163 bash[78107]: cluster 2023-10-09T09:13:03.163102+0000 mon.a (mon.0) 441 : cluster 0 osdmap e511: 8 total, 8 up, 8 in 2023-10-09T09:13:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: cluster 2023-10-09T09:13:01.494010+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: cluster 2023-10-09T09:13:01.747377+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e509 with expected crc 2023-10-09T09:13:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: cluster 2023-10-09T09:13:02.155206+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e510 with expected crc 2023-10-09T09:13:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: cluster 2023-10-09T09:13:02.156445+0000 osd.3 (osd.3) 141 : cluster 3 failed to encode map e510 with expected crc 2023-10-09T09:13:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: cluster 2023-10-09T09:13:02.159963+0000 mon.a (mon.0) 435 : cluster 0 osdmap e510: 8 total, 8 up, 8 in 2023-10-09T09:13:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: audit 2023-10-09T09:13:02.195373+0000 mon.a (mon.0) 436 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:03.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: audit 2023-10-09T09:13:02.201811+0000 mon.a (mon.0) 437 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:03.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: cephadm 2023-10-09T09:13:02.241775+0000 mgr.y (mgr.34107) 157 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T09:13:03.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: audit 2023-10-09T09:13:02.242539+0000 mon.a (mon.0) 438 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:13:03.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: audit 2023-10-09T09:13:02.244316+0000 mon.a (mon.0) 439 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:13:03.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: audit 2023-10-09T09:13:02.245979+0000 mon.a (mon.0) 440 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:03.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: cephadm 2023-10-09T09:13:02.247683+0000 mgr.y (mgr.34107) 158 : cephadm 1 Reconfiguring daemon mon.b on smithi181 2023-10-09T09:13:03.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:03 smithi181 bash[59345]: cluster 2023-10-09T09:13:03.163102+0000 mon.a (mon.0) 441 : cluster 0 osdmap e511: 8 total, 8 up, 8 in 2023-10-09T09:13:04.186 INFO:tasks.rados.rados.0.smithi163.stdout:1808: done (4 left) 2023-10-09T09:13:04.186 INFO:tasks.rados.rados.0.smithi163.stdout:1809: done (3 left) 2023-10-09T09:13:04.186 INFO:tasks.rados.rados.0.smithi163.stdout:1810: done (2 left) 2023-10-09T09:13:04.187 INFO:tasks.rados.rados.0.smithi163.stdout:1811: done (1 left) 2023-10-09T09:13:04.187 INFO:tasks.rados.rados.0.smithi163.stdout:1812: done (0 left) 2023-10-09T09:13:04.187 INFO:tasks.rados.rados.0.smithi163.stdout:1813: rollback oid 49 current snap is 226 2023-10-09T09:13:04.187 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 49 to 219 2023-10-09T09:13:04.187 INFO:tasks.rados.rados.0.smithi163.stdout:1814: snap_create 2023-10-09T09:13:04.194 INFO:tasks.rados.rados.0.smithi163.stdout:1813: finishing rollback tid 0 to smithi16367185-49 2023-10-09T09:13:04.194 INFO:tasks.rados.rados.0.smithi163.stdout:1813: finishing rollback tid 1 to smithi16367185-49 2023-10-09T09:13:04.194 INFO:tasks.rados.rados.0.smithi163.stdout:1813: finishing rollback tid 2 to smithi16367185-49 2023-10-09T09:13:04.194 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 273 (ObjNum 636 snap 202 seq_num 636) dirty exists 2023-10-09T09:13:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:04 smithi181 bash[59345]: cluster 2023-10-09T09:13:03.157444+0000 osd.3 (osd.3) 142 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:04 smithi181 bash[59345]: cluster 2023-10-09T09:13:03.158194+0000 osd.2 (osd.2) 137 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:04 smithi181 bash[59345]: cluster 2023-10-09T09:13:03.163620+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:04 smithi181 bash[59345]: cluster 2023-10-09T09:13:03.166588+0000 osd.0 (osd.0) 83 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:04 smithi181 bash[59345]: cluster 2023-10-09T09:13:03.287276+0000 mgr.y (mgr.34107) 159 : cluster 0 pgmap v319: 89 pgs: 89 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2023-10-09T09:13:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:04 smithi163 bash[75425]: cluster 2023-10-09T09:13:03.157444+0000 osd.3 (osd.3) 142 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:04 smithi163 bash[75425]: cluster 2023-10-09T09:13:03.158194+0000 osd.2 (osd.2) 137 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:04 smithi163 bash[75425]: cluster 2023-10-09T09:13:03.163620+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:04 smithi163 bash[75425]: cluster 2023-10-09T09:13:03.166588+0000 osd.0 (osd.0) 83 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:04 smithi163 bash[75425]: cluster 2023-10-09T09:13:03.287276+0000 mgr.y (mgr.34107) 159 : cluster 0 pgmap v319: 89 pgs: 89 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2023-10-09T09:13:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:04 smithi163 bash[78107]: cluster 2023-10-09T09:13:03.157444+0000 osd.3 (osd.3) 142 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:04 smithi163 bash[78107]: cluster 2023-10-09T09:13:03.158194+0000 osd.2 (osd.2) 137 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:04 smithi163 bash[78107]: cluster 2023-10-09T09:13:03.163620+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:04 smithi163 bash[78107]: cluster 2023-10-09T09:13:03.166588+0000 osd.0 (osd.0) 83 : cluster 3 failed to encode map e511 with expected crc 2023-10-09T09:13:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:04 smithi163 bash[78107]: cluster 2023-10-09T09:13:03.287276+0000 mgr.y (mgr.34107) 159 : cluster 0 pgmap v319: 89 pgs: 89 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2023-10-09T09:13:05.057 INFO:tasks.rados.rados.0.smithi163.stdout:1813: done (1 left) 2023-10-09T09:13:05.057 INFO:tasks.rados.rados.0.smithi163.stdout:1814: done (0 left) 2023-10-09T09:13:05.057 INFO:tasks.rados.rados.0.smithi163.stdout:1815: delete oid 18 current snap is 227 2023-10-09T09:13:05.070 INFO:tasks.rados.rados.0.smithi163.stdout:1815: done (0 left) 2023-10-09T09:13:05.071 INFO:tasks.rados.rados.0.smithi163.stdout:1816: write oid 38 current snap is 227 2023-10-09T09:13:05.071 INFO:tasks.rados.rados.0.smithi163.stdout:1816: seq_num 697 ranges {514479=636676,1853714=446233,2991120=542240} 2023-10-09T09:13:05.080 INFO:tasks.rados.rados.0.smithi163.stdout:1816: writing smithi16367185-38 from 514479 to 1151155 tid 1 2023-10-09T09:13:05.089 INFO:tasks.rados.rados.0.smithi163.stdout:1816: writing smithi16367185-38 from 1853714 to 2299947 tid 2 2023-10-09T09:13:05.096 INFO:tasks.rados.rados.0.smithi163.stdout:1816: writing smithi16367185-38 from 2991120 to 3533360 tid 3 2023-10-09T09:13:05.096 INFO:tasks.rados.rados.0.smithi163.stdout:1817: snap_create 2023-10-09T09:13:05.096 INFO:tasks.rados.rados.0.smithi163.stdout:1816: finishing write tid 1 to smithi16367185-38 2023-10-09T09:13:05.096 INFO:tasks.rados.rados.0.smithi163.stdout:1816: finishing write tid 2 to smithi16367185-38 2023-10-09T09:13:05.100 INFO:tasks.rados.rados.0.smithi163.stdout:1816: finishing write tid 3 to smithi16367185-38 2023-10-09T09:13:05.100 INFO:tasks.rados.rados.0.smithi163.stdout:1816: finishing write tid 4 to smithi16367185-38 2023-10-09T09:13:05.100 INFO:tasks.rados.rados.0.smithi163.stdout:1816: finishing write tid 5 to smithi16367185-38 2023-10-09T09:13:05.100 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 703 (ObjNum 697 snap 227 seq_num 697) dirty exists 2023-10-09T09:13:05.100 INFO:tasks.rados.rados.0.smithi163.stdout:1816: left oid 38 (ObjNum 697 snap 227 seq_num 697) 2023-10-09T09:13:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[78107]: cluster 2023-10-09T09:13:04.180464+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e512 with expected crc 2023-10-09T09:13:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[78107]: cluster 2023-10-09T09:13:04.181286+0000 osd.1 (osd.1) 67 : cluster 3 failed to encode map e512 with expected crc 2023-10-09T09:13:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[78107]: cluster 2023-10-09T09:13:04.186051+0000 mon.a (mon.0) 442 : cluster 0 osdmap e512: 8 total, 8 up, 8 in 2023-10-09T09:13:05.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[78107]: cluster 2023-10-09T09:13:04.188191+0000 osd.0 (osd.0) 84 : cluster 3 failed to encode map e512 with expected crc 2023-10-09T09:13:05.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[78107]: audit 2023-10-09T09:13:04.488293+0000 mon.a (mon.0) 443 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:05.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[78107]: audit 2023-10-09T09:13:04.499158+0000 mon.a (mon.0) 444 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:05.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[78107]: audit 2023-10-09T09:13:04.518760+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:13:05.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[78107]: audit 2023-10-09T09:13:04.520852+0000 mon.a (mon.0) 446 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:13:05.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[78107]: audit 2023-10-09T09:13:04.522622+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:05.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[78107]: cluster 2023-10-09T09:13:05.057127+0000 mon.a (mon.0) 448 : cluster 0 osdmap e513: 8 total, 8 up, 8 in 2023-10-09T09:13:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[75425]: cluster 2023-10-09T09:13:04.180464+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e512 with expected crc 2023-10-09T09:13:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[75425]: cluster 2023-10-09T09:13:04.181286+0000 osd.1 (osd.1) 67 : cluster 3 failed to encode map e512 with expected crc 2023-10-09T09:13:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[75425]: cluster 2023-10-09T09:13:04.186051+0000 mon.a (mon.0) 442 : cluster 0 osdmap e512: 8 total, 8 up, 8 in 2023-10-09T09:13:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[75425]: cluster 2023-10-09T09:13:04.188191+0000 osd.0 (osd.0) 84 : cluster 3 failed to encode map e512 with expected crc 2023-10-09T09:13:05.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[75425]: audit 2023-10-09T09:13:04.488293+0000 mon.a (mon.0) 443 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:05.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[75425]: audit 2023-10-09T09:13:04.499158+0000 mon.a (mon.0) 444 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:05.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[75425]: audit 2023-10-09T09:13:04.518760+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:13:05.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[75425]: audit 2023-10-09T09:13:04.520852+0000 mon.a (mon.0) 446 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:13:05.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[75425]: audit 2023-10-09T09:13:04.522622+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:05.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:05 smithi163 bash[75425]: cluster 2023-10-09T09:13:05.057127+0000 mon.a (mon.0) 448 : cluster 0 osdmap e513: 8 total, 8 up, 8 in 2023-10-09T09:13:05.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:05 smithi181 bash[59345]: cluster 2023-10-09T09:13:04.180464+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e512 with expected crc 2023-10-09T09:13:05.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:05 smithi181 bash[59345]: cluster 2023-10-09T09:13:04.181286+0000 osd.1 (osd.1) 67 : cluster 3 failed to encode map e512 with expected crc 2023-10-09T09:13:05.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:05 smithi181 bash[59345]: cluster 2023-10-09T09:13:04.186051+0000 mon.a (mon.0) 442 : cluster 0 osdmap e512: 8 total, 8 up, 8 in 2023-10-09T09:13:05.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:05 smithi181 bash[59345]: cluster 2023-10-09T09:13:04.188191+0000 osd.0 (osd.0) 84 : cluster 3 failed to encode map e512 with expected crc 2023-10-09T09:13:05.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:05 smithi181 bash[59345]: audit 2023-10-09T09:13:04.488293+0000 mon.a (mon.0) 443 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:05.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:05 smithi181 bash[59345]: audit 2023-10-09T09:13:04.499158+0000 mon.a (mon.0) 444 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:05.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:05 smithi181 bash[59345]: audit 2023-10-09T09:13:04.518760+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:13:05.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:05 smithi181 bash[59345]: audit 2023-10-09T09:13:04.520852+0000 mon.a (mon.0) 446 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:13:05.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:05 smithi181 bash[59345]: audit 2023-10-09T09:13:04.522622+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:05.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:05 smithi181 bash[59345]: cluster 2023-10-09T09:13:05.057127+0000 mon.a (mon.0) 448 : cluster 0 osdmap e513: 8 total, 8 up, 8 in 2023-10-09T09:13:06.071 INFO:tasks.rados.rados.0.smithi163.stdout:1816: done (1 left) 2023-10-09T09:13:06.072 INFO:tasks.rados.rados.0.smithi163.stdout:1817: done (0 left) 2023-10-09T09:13:06.072 INFO:tasks.rados.rados.0.smithi163.stdout:1818: snap_remove snap 222 2023-10-09T09:13:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[75425]: cephadm 2023-10-09T09:13:04.518094+0000 mgr.y (mgr.34107) 160 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T09:13:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[75425]: cephadm 2023-10-09T09:13:04.524550+0000 mgr.y (mgr.34107) 161 : cephadm 1 Reconfiguring daemon mgr.x on smithi181 2023-10-09T09:13:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[75425]: cluster 2023-10-09T09:13:05.054548+0000 osd.3 (osd.3) 143 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[75425]: cluster 2023-10-09T09:13:05.055438+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[75425]: cluster 2023-10-09T09:13:05.061996+0000 osd.2 (osd.2) 138 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[75425]: cluster 2023-10-09T09:13:05.081283+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[75425]: cluster 2023-10-09T09:13:05.287959+0000 mgr.y (mgr.34107) 162 : cluster 0 pgmap v322: 89 pgs: 89 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[75425]: cluster 2023-10-09T09:13:06.071686+0000 mon.a (mon.0) 449 : cluster 0 osdmap e514: 8 total, 8 up, 8 in 2023-10-09T09:13:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[78107]: cephadm 2023-10-09T09:13:04.518094+0000 mgr.y (mgr.34107) 160 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T09:13:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[78107]: cephadm 2023-10-09T09:13:04.524550+0000 mgr.y (mgr.34107) 161 : cephadm 1 Reconfiguring daemon mgr.x on smithi181 2023-10-09T09:13:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[78107]: cluster 2023-10-09T09:13:05.054548+0000 osd.3 (osd.3) 143 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[78107]: cluster 2023-10-09T09:13:05.055438+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[78107]: cluster 2023-10-09T09:13:05.061996+0000 osd.2 (osd.2) 138 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[78107]: cluster 2023-10-09T09:13:05.081283+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[78107]: cluster 2023-10-09T09:13:05.287959+0000 mgr.y (mgr.34107) 162 : cluster 0 pgmap v322: 89 pgs: 89 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:06 smithi163 bash[78107]: cluster 2023-10-09T09:13:06.071686+0000 mon.a (mon.0) 449 : cluster 0 osdmap e514: 8 total, 8 up, 8 in 2023-10-09T09:13:06.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:06 smithi181 bash[59345]: cephadm 2023-10-09T09:13:04.518094+0000 mgr.y (mgr.34107) 160 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T09:13:06.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:06 smithi181 bash[59345]: cephadm 2023-10-09T09:13:04.524550+0000 mgr.y (mgr.34107) 161 : cephadm 1 Reconfiguring daemon mgr.x on smithi181 2023-10-09T09:13:06.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:06 smithi181 bash[59345]: cluster 2023-10-09T09:13:05.054548+0000 osd.3 (osd.3) 143 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:06 smithi181 bash[59345]: cluster 2023-10-09T09:13:05.055438+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:06 smithi181 bash[59345]: cluster 2023-10-09T09:13:05.061996+0000 osd.2 (osd.2) 138 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:06 smithi181 bash[59345]: cluster 2023-10-09T09:13:05.081283+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:06.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:06 smithi181 bash[59345]: cluster 2023-10-09T09:13:05.287959+0000 mgr.y (mgr.34107) 162 : cluster 0 pgmap v322: 89 pgs: 89 active+clean; 116 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:06.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:06 smithi181 bash[59345]: cluster 2023-10-09T09:13:06.071686+0000 mon.a (mon.0) 449 : cluster 0 osdmap e514: 8 total, 8 up, 8 in 2023-10-09T09:13:07.074 INFO:tasks.rados.rados.0.smithi163.stdout:1818: done (0 left) 2023-10-09T09:13:07.074 INFO:tasks.rados.rados.0.smithi163.stdout:1819: read oid 4 snap -1 2023-10-09T09:13:07.074 INFO:tasks.rados.rados.0.smithi163.stdout:1819: expect (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:13:07.074 INFO:tasks.rados.rados.0.smithi163.stdout:1820: snap_remove snap 218 2023-10-09T09:13:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[78107]: cluster 2023-10-09T09:13:05.791372+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[78107]: cluster 2023-10-09T09:13:06.064901+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e514 with expected crc 2023-10-09T09:13:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[78107]: cluster 2023-10-09T09:13:06.067600+0000 osd.2 (osd.2) 139 : cluster 3 failed to encode map e514 with expected crc 2023-10-09T09:13:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[78107]: audit 2023-10-09T09:13:06.435421+0000 mon.a (mon.0) 450 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[78107]: audit 2023-10-09T09:13:06.441067+0000 mon.a (mon.0) 451 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[78107]: cephadm 2023-10-09T09:13:06.443492+0000 mgr.y (mgr.34107) 163 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[78107]: audit 2023-10-09T09:13:06.443887+0000 mon.a (mon.0) 452 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[78107]: audit 2023-10-09T09:13:06.444966+0000 mon.a (mon.0) 453 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[78107]: cephadm 2023-10-09T09:13:06.448622+0000 mgr.y (mgr.34107) 164 : cephadm 1 Reconfiguring daemon osd.4 on smithi181 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[78107]: cluster 2023-10-09T09:13:07.073568+0000 mon.a (mon.0) 454 : cluster 0 osdmap e515: 8 total, 8 up, 8 in 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[75425]: cluster 2023-10-09T09:13:05.791372+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[75425]: cluster 2023-10-09T09:13:06.064901+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e514 with expected crc 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[75425]: cluster 2023-10-09T09:13:06.067600+0000 osd.2 (osd.2) 139 : cluster 3 failed to encode map e514 with expected crc 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[75425]: audit 2023-10-09T09:13:06.435421+0000 mon.a (mon.0) 450 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[75425]: audit 2023-10-09T09:13:06.441067+0000 mon.a (mon.0) 451 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:07.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[75425]: cephadm 2023-10-09T09:13:06.443492+0000 mgr.y (mgr.34107) 163 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T09:13:07.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[75425]: audit 2023-10-09T09:13:06.443887+0000 mon.a (mon.0) 452 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T09:13:07.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[75425]: audit 2023-10-09T09:13:06.444966+0000 mon.a (mon.0) 453 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:07.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[75425]: cephadm 2023-10-09T09:13:06.448622+0000 mgr.y (mgr.34107) 164 : cephadm 1 Reconfiguring daemon osd.4 on smithi181 2023-10-09T09:13:07.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:07 smithi163 bash[75425]: cluster 2023-10-09T09:13:07.073568+0000 mon.a (mon.0) 454 : cluster 0 osdmap e515: 8 total, 8 up, 8 in 2023-10-09T09:13:07.543 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:07 smithi181 bash[59345]: cluster 2023-10-09T09:13:05.791372+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e513 with expected crc 2023-10-09T09:13:07.543 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:07 smithi181 bash[59345]: cluster 2023-10-09T09:13:06.064901+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e514 with expected crc 2023-10-09T09:13:07.543 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:07 smithi181 bash[59345]: cluster 2023-10-09T09:13:06.067600+0000 osd.2 (osd.2) 139 : cluster 3 failed to encode map e514 with expected crc 2023-10-09T09:13:07.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:07 smithi181 bash[59345]: audit 2023-10-09T09:13:06.435421+0000 mon.a (mon.0) 450 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:07.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:07 smithi181 bash[59345]: audit 2023-10-09T09:13:06.441067+0000 mon.a (mon.0) 451 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:07.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:07 smithi181 bash[59345]: cephadm 2023-10-09T09:13:06.443492+0000 mgr.y (mgr.34107) 163 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T09:13:07.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:07 smithi181 bash[59345]: audit 2023-10-09T09:13:06.443887+0000 mon.a (mon.0) 452 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T09:13:07.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:07 smithi181 bash[59345]: audit 2023-10-09T09:13:06.444966+0000 mon.a (mon.0) 453 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:07.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:07 smithi181 bash[59345]: cephadm 2023-10-09T09:13:06.448622+0000 mgr.y (mgr.34107) 164 : cephadm 1 Reconfiguring daemon osd.4 on smithi181 2023-10-09T09:13:07.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:07 smithi181 bash[59345]: cluster 2023-10-09T09:13:07.073568+0000 mon.a (mon.0) 454 : cluster 0 osdmap e515: 8 total, 8 up, 8 in 2023-10-09T09:13:08.076 INFO:tasks.rados.rados.0.smithi163.stdout:1820: done (1 left) 2023-10-09T09:13:08.076 INFO:tasks.rados.rados.0.smithi163.stdout:1821: delete oid 17 current snap is 228 2023-10-09T09:13:08.076 INFO:tasks.rados.rados.0.smithi163.stdout:1819: expect (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:13:08.215 INFO:tasks.rados.rados.0.smithi163.stdout:1819: done (1 left) 2023-10-09T09:13:08.215 INFO:tasks.rados.rados.0.smithi163.stdout:1821: done (0 left) 2023-10-09T09:13:08.215 INFO:tasks.rados.rados.0.smithi163.stdout:1822: read oid 11 snap -1 2023-10-09T09:13:08.215 INFO:tasks.rados.rados.0.smithi163.stdout:1822: expect deleted 2023-10-09T09:13:08.215 INFO:tasks.rados.rados.0.smithi163.stdout:1823: rollback oid 39 current snap is 228 2023-10-09T09:13:08.215 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 214 2023-10-09T09:13:08.216 INFO:tasks.rados.rados.0.smithi163.stdout:1824: write oid 36 current snap is 228 2023-10-09T09:13:08.216 INFO:tasks.rados.rados.0.smithi163.stdout:1824: seq_num 699 ranges {548093=787174,1970678=449902,3045966=681137} 2023-10-09T09:13:08.227 INFO:tasks.rados.rados.0.smithi163.stdout:1824: writing smithi16367185-36 from 548093 to 1335267 tid 1 2023-10-09T09:13:08.234 INFO:tasks.rados.rados.0.smithi163.stdout:1824: writing smithi16367185-36 from 1970678 to 2420580 tid 2 2023-10-09T09:13:08.242 INFO:tasks.rados.rados.0.smithi163.stdout:1824: writing smithi16367185-36 from 3045966 to 3727103 tid 3 2023-10-09T09:13:08.242 INFO:tasks.rados.rados.0.smithi163.stdout:1825: write oid 9 current snap is 228 2023-10-09T09:13:08.242 INFO:tasks.rados.rados.0.smithi163.stdout:1825: seq_num 700 ranges {564900=578776,1629160=709913,3073389=492409} 2023-10-09T09:13:08.248 INFO:tasks.rados.rados.0.smithi163.stdout:1825: writing smithi16367185-9 from 564900 to 1143676 tid 1 2023-10-09T09:13:08.255 INFO:tasks.rados.rados.0.smithi163.stdout:1825: writing smithi16367185-9 from 1629160 to 2339073 tid 2 2023-10-09T09:13:08.262 INFO:tasks.rados.rados.0.smithi163.stdout:1825: writing smithi16367185-9 from 3073389 to 3565798 tid 3 2023-10-09T09:13:08.263 INFO:tasks.rados.rados.0.smithi163.stdout:1823: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:13:08.263 INFO:tasks.rados.rados.0.smithi163.stdout:1823: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:13:08.263 INFO:tasks.rados.rados.0.smithi163.stdout:1826: read oid 20 snap -1 2023-10-09T09:13:08.263 INFO:tasks.rados.rados.0.smithi163.stdout:1826: expect (ObjNum 694 snap 221 seq_num 694) 2023-10-09T09:13:08.263 INFO:tasks.rados.rados.0.smithi163.stdout:1822: done (4 left) 2023-10-09T09:13:08.263 INFO:tasks.rados.rados.0.smithi163.stdout:1827: snap_remove snap 224 2023-10-09T09:13:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[75425]: cluster 2023-10-09T09:13:07.069519+0000 osd.3 (osd.3) 144 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[75425]: cluster 2023-10-09T09:13:07.070207+0000 osd.0 (osd.0) 85 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[75425]: cluster 2023-10-09T09:13:07.075735+0000 osd.2 (osd.2) 140 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[75425]: cluster 2023-10-09T09:13:07.288647+0000 mgr.y (mgr.34107) 165 : cluster 0 pgmap v325: 89 pgs: 89 active+clean; 119 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 751 KiB/s wr, 4 op/s 2023-10-09T09:13:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[75425]: cluster 2023-10-09T09:13:07.304570+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[75425]: cluster 2023-10-09T09:13:08.076000+0000 mon.a (mon.0) 455 : cluster 0 osdmap e516: 8 total, 8 up, 8 in 2023-10-09T09:13:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[78107]: cluster 2023-10-09T09:13:07.069519+0000 osd.3 (osd.3) 144 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[78107]: cluster 2023-10-09T09:13:07.070207+0000 osd.0 (osd.0) 85 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[78107]: cluster 2023-10-09T09:13:07.075735+0000 osd.2 (osd.2) 140 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[78107]: cluster 2023-10-09T09:13:07.288647+0000 mgr.y (mgr.34107) 165 : cluster 0 pgmap v325: 89 pgs: 89 active+clean; 119 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 751 KiB/s wr, 4 op/s 2023-10-09T09:13:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[78107]: cluster 2023-10-09T09:13:07.304570+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:08 smithi163 bash[78107]: cluster 2023-10-09T09:13:08.076000+0000 mon.a (mon.0) 455 : cluster 0 osdmap e516: 8 total, 8 up, 8 in 2023-10-09T09:13:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:08 smithi181 bash[59345]: cluster 2023-10-09T09:13:07.069519+0000 osd.3 (osd.3) 144 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:08 smithi181 bash[59345]: cluster 2023-10-09T09:13:07.070207+0000 osd.0 (osd.0) 85 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:08 smithi181 bash[59345]: cluster 2023-10-09T09:13:07.075735+0000 osd.2 (osd.2) 140 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:08 smithi181 bash[59345]: cluster 2023-10-09T09:13:07.288647+0000 mgr.y (mgr.34107) 165 : cluster 0 pgmap v325: 89 pgs: 89 active+clean; 119 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 751 KiB/s wr, 4 op/s 2023-10-09T09:13:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:08 smithi181 bash[59345]: cluster 2023-10-09T09:13:07.304570+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e515 with expected crc 2023-10-09T09:13:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:08 smithi181 bash[59345]: cluster 2023-10-09T09:13:08.076000+0000 mon.a (mon.0) 455 : cluster 0 osdmap e516: 8 total, 8 up, 8 in 2023-10-09T09:13:09.222 INFO:tasks.rados.rados.0.smithi163.stdout:1827: done (4 left) 2023-10-09T09:13:09.222 INFO:tasks.rados.rados.0.smithi163.stdout:1828: write oid 12 current snap is 228 2023-10-09T09:13:09.223 INFO:tasks.rados.rados.0.smithi163.stdout:1823: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:13:09.223 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 715 (ObjNum 529 snap 165 seq_num 529) dirty exists 2023-10-09T09:13:09.223 INFO:tasks.rados.rados.0.smithi163.stdout:1824: finishing write tid 1 to smithi16367185-36 2023-10-09T09:13:09.223 INFO:tasks.rados.rados.0.smithi163.stdout:1824: finishing write tid 2 to smithi16367185-36 2023-10-09T09:13:09.223 INFO:tasks.rados.rados.0.smithi163.stdout:1828: seq_num 701 ranges {581707=654025,1687642=453571,2700812=703681} 2023-10-09T09:13:09.233 INFO:tasks.rados.rados.0.smithi163.stdout:1828: writing smithi16367185-12 from 581707 to 1235732 tid 1 2023-10-09T09:13:09.238 INFO:tasks.rados.rados.0.smithi163.stdout:1828: writing smithi16367185-12 from 1687642 to 2141213 tid 2 2023-10-09T09:13:09.247 INFO:tasks.rados.rados.0.smithi163.stdout:1828: writing smithi16367185-12 from 2700812 to 3404493 tid 3 2023-10-09T09:13:09.247 INFO:tasks.rados.rados.0.smithi163.stdout:1823: done (4 left) 2023-10-09T09:13:09.247 INFO:tasks.rados.rados.0.smithi163.stdout:1829: rollback oid 47 current snap is 228 2023-10-09T09:13:09.247 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 47 to 215 2023-10-09T09:13:09.247 INFO:tasks.rados.rados.0.smithi163.stdout:1824: finishing write tid 3 to smithi16367185-36 2023-10-09T09:13:09.247 INFO:tasks.rados.rados.0.smithi163.stdout:1824: finishing write tid 4 to smithi16367185-36 2023-10-09T09:13:09.247 INFO:tasks.rados.rados.0.smithi163.stdout:1824: finishing write tid 5 to smithi16367185-36 2023-10-09T09:13:09.247 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 489 (ObjNum 699 snap 228 seq_num 699) dirty exists 2023-10-09T09:13:09.247 INFO:tasks.rados.rados.0.smithi163.stdout:1824: left oid 36 (ObjNum 699 snap 228 seq_num 699) 2023-10-09T09:13:09.247 INFO:tasks.rados.rados.0.smithi163.stdout:1825: finishing write tid 1 to smithi16367185-9 2023-10-09T09:13:09.248 INFO:tasks.rados.rados.0.smithi163.stdout:1825: finishing write tid 2 to smithi16367185-9 2023-10-09T09:13:09.248 INFO:tasks.rados.rados.0.smithi163.stdout:1825: finishing write tid 3 to smithi16367185-9 2023-10-09T09:13:09.248 INFO:tasks.rados.rados.0.smithi163.stdout:1825: finishing write tid 4 to smithi16367185-9 2023-10-09T09:13:09.248 INFO:tasks.rados.rados.0.smithi163.stdout:1825: finishing write tid 5 to smithi16367185-9 2023-10-09T09:13:09.248 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 451 (ObjNum 700 snap 228 seq_num 700) dirty exists 2023-10-09T09:13:09.248 INFO:tasks.rados.rados.0.smithi163.stdout:1825: left oid 9 (ObjNum 700 snap 228 seq_num 700) 2023-10-09T09:13:09.248 INFO:tasks.rados.rados.0.smithi163.stdout:1824: done (4 left) 2023-10-09T09:13:09.248 INFO:tasks.rados.rados.0.smithi163.stdout:1825: done (3 left) 2023-10-09T09:13:09.248 INFO:tasks.rados.rados.0.smithi163.stdout:1830: delete oid 42 current snap is 228 2023-10-09T09:13:09.248 INFO:tasks.rados.rados.0.smithi163.stdout:1826: expect (ObjNum 694 snap 221 seq_num 694) 2023-10-09T09:13:09.373 INFO:tasks.rados.rados.0.smithi163.stdout:1828: finishing write tid 1 to smithi16367185-12 2023-10-09T09:13:09.373 INFO:tasks.rados.rados.0.smithi163.stdout:1828: finishing write tid 2 to smithi16367185-12 2023-10-09T09:13:09.373 INFO:tasks.rados.rados.0.smithi163.stdout:1826: done (3 left) 2023-10-09T09:13:09.374 INFO:tasks.rados.rados.0.smithi163.stdout:1830: done (2 left) 2023-10-09T09:13:09.374 INFO:tasks.rados.rados.0.smithi163.stdout:1831: write oid 40 current snap is 228 2023-10-09T09:13:09.374 INFO:tasks.rados.rados.0.smithi163.stdout:1829: finishing rollback tid 0 to smithi16367185-47 2023-10-09T09:13:09.374 INFO:tasks.rados.rados.0.smithi163.stdout:1831: seq_num 703 ranges {615321=404523,1804606=457240,2755658=726225} 2023-10-09T09:13:09.380 INFO:tasks.rados.rados.0.smithi163.stdout:1831: writing smithi16367185-40 from 615321 to 1019844 tid 1 2023-10-09T09:13:09.389 INFO:tasks.rados.rados.0.smithi163.stdout:1831: writing smithi16367185-40 from 1804606 to 2261846 tid 2 2023-10-09T09:13:09.396 INFO:tasks.rados.rados.0.smithi163.stdout:1831: writing smithi16367185-40 from 2755658 to 3481883 tid 3 2023-10-09T09:13:09.396 INFO:tasks.rados.rados.0.smithi163.stdout:1832: write oid 17 current snap is 228 2023-10-09T09:13:09.397 INFO:tasks.rados.rados.0.smithi163.stdout:1832: seq_num 704 ranges {632128=479772,1746735=600898,3066728=653850} 2023-10-09T09:13:09.402 INFO:tasks.rados.rados.0.smithi163.stdout:1832: writing smithi16367185-17 from 632128 to 1111900 tid 1 2023-10-09T09:13:09.409 INFO:tasks.rados.rados.0.smithi163.stdout:1832: writing smithi16367185-17 from 1746735 to 2347633 tid 2 2023-10-09T09:13:09.417 INFO:tasks.rados.rados.0.smithi163.stdout:1832: writing smithi16367185-17 from 3066728 to 3720578 tid 3 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:1833: rollback oid 22 current snap is 228 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 22 to 217 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:1828: finishing write tid 3 to smithi16367185-12 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:1828: finishing write tid 4 to smithi16367185-12 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:1828: finishing write tid 5 to smithi16367185-12 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 815 (ObjNum 701 snap 228 seq_num 701) dirty exists 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:1828: left oid 12 (ObjNum 701 snap 228 seq_num 701) 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:1829: finishing rollback tid 1 to smithi16367185-47 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:1829: finishing rollback tid 2 to smithi16367185-47 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 1340 (ObjNum 660 snap 209 seq_num 660) dirty exists 2023-10-09T09:13:09.418 INFO:tasks.rados.rados.0.smithi163.stdout:1831: finishing write tid 1 to smithi16367185-40 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:1831: finishing write tid 2 to smithi16367185-40 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:1828: done (4 left) 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:1829: done (3 left) 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:1834: rollback oid 21 current snap is 228 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 220 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:1831: finishing write tid 3 to smithi16367185-40 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:1831: finishing write tid 4 to smithi16367185-40 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:1831: finishing write tid 5 to smithi16367185-40 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 890 (ObjNum 703 snap 228 seq_num 703) dirty exists 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:1831: left oid 40 (ObjNum 703 snap 228 seq_num 703) 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:1832: finishing write tid 1 to smithi16367185-17 2023-10-09T09:13:09.419 INFO:tasks.rados.rados.0.smithi163.stdout:1832: finishing write tid 2 to smithi16367185-17 2023-10-09T09:13:09.420 INFO:tasks.rados.rados.0.smithi163.stdout:1831: done (3 left) 2023-10-09T09:13:09.420 INFO:tasks.rados.rados.0.smithi163.stdout:1835: snap_create 2023-10-09T09:13:09.420 INFO:tasks.rados.rados.0.smithi163.stdout:1834: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:13:09.420 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:13:09.423 INFO:tasks.rados.rados.0.smithi163.stdout:1832: finishing write tid 3 to smithi16367185-17 2023-10-09T09:13:09.423 INFO:tasks.rados.rados.0.smithi163.stdout:1832: finishing write tid 4 to smithi16367185-17 2023-10-09T09:13:09.423 INFO:tasks.rados.rados.0.smithi163.stdout:1832: finishing write tid 5 to smithi16367185-17 2023-10-09T09:13:09.424 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 894 (ObjNum 704 snap 228 seq_num 704) dirty exists 2023-10-09T09:13:09.424 INFO:tasks.rados.rados.0.smithi163.stdout:1832: left oid 17 (ObjNum 704 snap 228 seq_num 704) 2023-10-09T09:13:09.424 INFO:tasks.rados.rados.0.smithi163.stdout:1833: finishing rollback tid 0 to smithi16367185-22 2023-10-09T09:13:09.424 INFO:tasks.rados.rados.0.smithi163.stdout:1833: finishing rollback tid 1 to smithi16367185-22 2023-10-09T09:13:09.424 INFO:tasks.rados.rados.0.smithi163.stdout:1833: finishing rollback tid 2 to smithi16367185-22 2023-10-09T09:13:09.425 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 1345 (ObjNum 624 snap 198 seq_num 624) dirty exists 2023-10-09T09:13:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[78107]: cluster 2023-10-09T09:13:08.070149+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e516 with expected crc 2023-10-09T09:13:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[78107]: cluster 2023-10-09T09:13:08.071669+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e516 with expected crc 2023-10-09T09:13:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[78107]: cluster 2023-10-09T09:13:08.076339+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e516 with expected crc 2023-10-09T09:13:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[78107]: audit 2023-10-09T09:13:08.451157+0000 mon.a (mon.0) 456 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[78107]: audit 2023-10-09T09:13:08.458600+0000 mon.a (mon.0) 457 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[78107]: cephadm 2023-10-09T09:13:08.467072+0000 mgr.y (mgr.34107) 166 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T09:13:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[78107]: audit 2023-10-09T09:13:08.467679+0000 mon.a (mon.0) 458 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T09:13:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[78107]: audit 2023-10-09T09:13:08.469229+0000 mon.a (mon.0) 459 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:09.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[78107]: cephadm 2023-10-09T09:13:08.473367+0000 mgr.y (mgr.34107) 167 : cephadm 1 Reconfiguring daemon osd.5 on smithi181 2023-10-09T09:13:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[75425]: cluster 2023-10-09T09:13:08.070149+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e516 with expected crc 2023-10-09T09:13:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[75425]: cluster 2023-10-09T09:13:08.071669+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e516 with expected crc 2023-10-09T09:13:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[75425]: cluster 2023-10-09T09:13:08.076339+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e516 with expected crc 2023-10-09T09:13:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[75425]: audit 2023-10-09T09:13:08.451157+0000 mon.a (mon.0) 456 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:09.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[75425]: audit 2023-10-09T09:13:08.458600+0000 mon.a (mon.0) 457 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:09.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[75425]: cephadm 2023-10-09T09:13:08.467072+0000 mgr.y (mgr.34107) 166 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T09:13:09.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[75425]: audit 2023-10-09T09:13:08.467679+0000 mon.a (mon.0) 458 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T09:13:09.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[75425]: audit 2023-10-09T09:13:08.469229+0000 mon.a (mon.0) 459 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:09.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:09 smithi163 bash[75425]: cephadm 2023-10-09T09:13:08.473367+0000 mgr.y (mgr.34107) 167 : cephadm 1 Reconfiguring daemon osd.5 on smithi181 2023-10-09T09:13:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:09 smithi181 bash[59345]: cluster 2023-10-09T09:13:08.070149+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e516 with expected crc 2023-10-09T09:13:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:09 smithi181 bash[59345]: cluster 2023-10-09T09:13:08.071669+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e516 with expected crc 2023-10-09T09:13:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:09 smithi181 bash[59345]: cluster 2023-10-09T09:13:08.076339+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e516 with expected crc 2023-10-09T09:13:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:09 smithi181 bash[59345]: audit 2023-10-09T09:13:08.451157+0000 mon.a (mon.0) 456 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:09 smithi181 bash[59345]: audit 2023-10-09T09:13:08.458600+0000 mon.a (mon.0) 457 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:09 smithi181 bash[59345]: cephadm 2023-10-09T09:13:08.467072+0000 mgr.y (mgr.34107) 166 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T09:13:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:09 smithi181 bash[59345]: audit 2023-10-09T09:13:08.467679+0000 mon.a (mon.0) 458 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T09:13:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:09 smithi181 bash[59345]: audit 2023-10-09T09:13:08.469229+0000 mon.a (mon.0) 459 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:09.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:09 smithi181 bash[59345]: cephadm 2023-10-09T09:13:08.473367+0000 mgr.y (mgr.34107) 167 : cephadm 1 Reconfiguring daemon osd.5 on smithi181 2023-10-09T09:13:10.235 INFO:tasks.rados.rados.0.smithi163.stdout:1832: done (3 left) 2023-10-09T09:13:10.235 INFO:tasks.rados.rados.0.smithi163.stdout:1833: done (2 left) 2023-10-09T09:13:10.235 INFO:tasks.rados.rados.0.smithi163.stdout:1834: done (1 left) 2023-10-09T09:13:10.235 INFO:tasks.rados.rados.0.smithi163.stdout:1835: done (0 left) 2023-10-09T09:13:10.236 INFO:tasks.rados.rados.0.smithi163.stdout:1836: snap_create 2023-10-09T09:13:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:10 smithi181 bash[59345]: cluster 2023-10-09T09:13:09.219352+0000 osd.3 (osd.3) 145 : cluster 3 failed to encode map e517 with expected crc 2023-10-09T09:13:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:10 smithi181 bash[59345]: cluster 2023-10-09T09:13:09.221819+0000 osd.2 (osd.2) 141 : cluster 3 failed to encode map e517 with expected crc 2023-10-09T09:13:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:10 smithi181 bash[59345]: cluster 2023-10-09T09:13:09.222351+0000 mon.a (mon.0) 460 : cluster 0 osdmap e517: 8 total, 8 up, 8 in 2023-10-09T09:13:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:10 smithi181 bash[59345]: cluster 2023-10-09T09:13:09.258949+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e517 with expected crc 2023-10-09T09:13:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:10 smithi181 bash[59345]: cluster 2023-10-09T09:13:09.289263+0000 mgr.y (mgr.34107) 168 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 119 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 751 KiB/s wr, 4 op/s 2023-10-09T09:13:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:10 smithi163 bash[75425]: cluster 2023-10-09T09:13:09.219352+0000 osd.3 (osd.3) 145 : cluster 3 failed to encode map e517 with expected crc 2023-10-09T09:13:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:10 smithi163 bash[75425]: cluster 2023-10-09T09:13:09.221819+0000 osd.2 (osd.2) 141 : cluster 3 failed to encode map e517 with expected crc 2023-10-09T09:13:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:10 smithi163 bash[75425]: cluster 2023-10-09T09:13:09.222351+0000 mon.a (mon.0) 460 : cluster 0 osdmap e517: 8 total, 8 up, 8 in 2023-10-09T09:13:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:10 smithi163 bash[75425]: cluster 2023-10-09T09:13:09.258949+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e517 with expected crc 2023-10-09T09:13:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:10 smithi163 bash[75425]: cluster 2023-10-09T09:13:09.289263+0000 mgr.y (mgr.34107) 168 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 119 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 751 KiB/s wr, 4 op/s 2023-10-09T09:13:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:10 smithi163 bash[78107]: cluster 2023-10-09T09:13:09.219352+0000 osd.3 (osd.3) 145 : cluster 3 failed to encode map e517 with expected crc 2023-10-09T09:13:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:10 smithi163 bash[78107]: cluster 2023-10-09T09:13:09.221819+0000 osd.2 (osd.2) 141 : cluster 3 failed to encode map e517 with expected crc 2023-10-09T09:13:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:10 smithi163 bash[78107]: cluster 2023-10-09T09:13:09.222351+0000 mon.a (mon.0) 460 : cluster 0 osdmap e517: 8 total, 8 up, 8 in 2023-10-09T09:13:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:10 smithi163 bash[78107]: cluster 2023-10-09T09:13:09.258949+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e517 with expected crc 2023-10-09T09:13:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:10 smithi163 bash[78107]: cluster 2023-10-09T09:13:09.289263+0000 mgr.y (mgr.34107) 168 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 119 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 751 KiB/s wr, 4 op/s 2023-10-09T09:13:11.240 INFO:tasks.rados.rados.0.smithi163.stdout:1836: done (0 left) 2023-10-09T09:13:11.240 INFO:tasks.rados.rados.0.smithi163.stdout:1837: snap_create 2023-10-09T09:13:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:11 smithi181 bash[59345]: cluster 2023-10-09T09:13:10.230487+0000 osd.1 (osd.1) 68 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:11.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:11 smithi181 bash[59345]: cluster 2023-10-09T09:13:10.231274+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:11.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:11 smithi181 bash[59345]: cluster 2023-10-09T09:13:10.235439+0000 mon.a (mon.0) 461 : cluster 0 osdmap e518: 8 total, 8 up, 8 in 2023-10-09T09:13:11.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:11 smithi181 bash[59345]: audit 2023-10-09T09:13:10.340634+0000 mon.a (mon.0) 462 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:11.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:11 smithi181 bash[59345]: audit 2023-10-09T09:13:10.519391+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:11.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:11 smithi181 bash[59345]: audit 2023-10-09T09:13:10.527252+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:11.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:11 smithi181 bash[59345]: audit 2023-10-09T09:13:10.543019+0000 mon.a (mon.0) 465 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T09:13:11.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:11 smithi181 bash[59345]: audit 2023-10-09T09:13:10.544771+0000 mon.a (mon.0) 466 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[78107]: cluster 2023-10-09T09:13:10.230487+0000 osd.1 (osd.1) 68 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[78107]: cluster 2023-10-09T09:13:10.231274+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[78107]: cluster 2023-10-09T09:13:10.235439+0000 mon.a (mon.0) 461 : cluster 0 osdmap e518: 8 total, 8 up, 8 in 2023-10-09T09:13:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[78107]: audit 2023-10-09T09:13:10.340634+0000 mon.a (mon.0) 462 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[78107]: audit 2023-10-09T09:13:10.519391+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[78107]: audit 2023-10-09T09:13:10.527252+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[78107]: audit 2023-10-09T09:13:10.543019+0000 mon.a (mon.0) 465 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T09:13:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[78107]: audit 2023-10-09T09:13:10.544771+0000 mon.a (mon.0) 466 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[75425]: cluster 2023-10-09T09:13:10.230487+0000 osd.1 (osd.1) 68 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[75425]: cluster 2023-10-09T09:13:10.231274+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:11.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[75425]: cluster 2023-10-09T09:13:10.235439+0000 mon.a (mon.0) 461 : cluster 0 osdmap e518: 8 total, 8 up, 8 in 2023-10-09T09:13:11.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[75425]: audit 2023-10-09T09:13:10.340634+0000 mon.a (mon.0) 462 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:11.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[75425]: audit 2023-10-09T09:13:10.519391+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:11.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[75425]: audit 2023-10-09T09:13:10.527252+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:11.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[75425]: audit 2023-10-09T09:13:10.543019+0000 mon.a (mon.0) 465 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T09:13:11.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:11 smithi163 bash[75425]: audit 2023-10-09T09:13:10.544771+0000 mon.a (mon.0) 466 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:12.238 INFO:tasks.rados.rados.0.smithi163.stdout:1837: done (0 left) 2023-10-09T09:13:12.238 INFO:tasks.rados.rados.0.smithi163.stdout:1838: read oid 26 snap 229 2023-10-09T09:13:12.238 INFO:tasks.rados.rados.0.smithi163.stdout:1838: expect (ObjNum 687 snap 219 seq_num 687) 2023-10-09T09:13:12.238 INFO:tasks.rados.rados.0.smithi163.stdout:1839: write oid 30 current snap is 231 2023-10-09T09:13:12.238 INFO:tasks.rados.rados.0.smithi163.stdout:1839: seq_num 705 ranges {648935=555021,1805217=460909,2694151=465122,3817442=31493} 2023-10-09T09:13:12.250 INFO:tasks.rados.rados.0.smithi163.stdout:1839: writing smithi16367185-30 from 648935 to 1203956 tid 1 2023-10-09T09:13:12.257 INFO:tasks.rados.rados.0.smithi163.stdout:1839: writing smithi16367185-30 from 1805217 to 2266126 tid 2 2023-10-09T09:13:12.262 INFO:tasks.rados.rados.0.smithi163.stdout:1839: writing smithi16367185-30 from 2694151 to 3159273 tid 3 2023-10-09T09:13:12.266 INFO:tasks.rados.rados.0.smithi163.stdout:1839: writing smithi16367185-30 from 3817442 to 3848935 tid 4 2023-10-09T09:13:12.266 INFO:tasks.rados.rados.0.smithi163.stdout:1840: snap_create 2023-10-09T09:13:12.266 INFO:tasks.rados.rados.0.smithi163.stdout:1838: expect (ObjNum 687 snap 219 seq_num 687) 2023-10-09T09:13:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:12 smithi181 bash[59345]: cephadm 2023-10-09T09:13:10.542469+0000 mgr.y (mgr.34107) 169 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T09:13:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:12 smithi181 bash[59345]: cluster 2023-10-09T09:13:10.543367+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:12.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:12 smithi181 bash[59345]: cephadm 2023-10-09T09:13:10.549607+0000 mgr.y (mgr.34107) 170 : cephadm 1 Reconfiguring daemon osd.6 on smithi181 2023-10-09T09:13:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:12 smithi181 bash[59345]: cluster 2023-10-09T09:13:11.102939+0000 osd.0 (osd.0) 86 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:12 smithi181 bash[59345]: cluster 2023-10-09T09:13:11.234504+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:12 smithi181 bash[59345]: cluster 2023-10-09T09:13:11.236464+0000 osd.3 (osd.3) 146 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:12 smithi181 bash[59345]: cluster 2023-10-09T09:13:11.239917+0000 mon.a (mon.0) 467 : cluster 0 osdmap e519: 8 total, 8 up, 8 in 2023-10-09T09:13:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:12 smithi181 bash[59345]: cluster 2023-10-09T09:13:11.290486+0000 mgr.y (mgr.34107) 171 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 135 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:13:12.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:12 smithi181 bash[59345]: cluster 2023-10-09T09:13:12.237761+0000 mon.a (mon.0) 468 : cluster 0 osdmap e520: 8 total, 8 up, 8 in 2023-10-09T09:13:12.595 INFO:tasks.rados.rados.0.smithi163.stdout:1839: finishing write tid 1 to smithi16367185-30 2023-10-09T09:13:12.595 INFO:tasks.rados.rados.0.smithi163.stdout:1839: finishing write tid 2 to smithi16367185-30 2023-10-09T09:13:12.595 INFO:tasks.rados.rados.0.smithi163.stdout:1839: finishing write tid 3 to smithi16367185-30 2023-10-09T09:13:12.595 INFO:tasks.rados.rados.0.smithi163.stdout:1839: finishing write tid 4 to smithi16367185-30 2023-10-09T09:13:12.596 INFO:tasks.rados.rados.0.smithi163.stdout:1839: finishing write tid 5 to smithi16367185-30 2023-10-09T09:13:12.596 INFO:tasks.rados.rados.0.smithi163.stdout:1839: finishing write tid 6 to smithi16367185-30 2023-10-09T09:13:12.596 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 653 (ObjNum 705 snap 231 seq_num 705) dirty exists 2023-10-09T09:13:12.596 INFO:tasks.rados.rados.0.smithi163.stdout:1839: left oid 30 (ObjNum 705 snap 231 seq_num 705) 2023-10-09T09:13:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[78107]: cephadm 2023-10-09T09:13:10.542469+0000 mgr.y (mgr.34107) 169 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T09:13:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[78107]: cluster 2023-10-09T09:13:10.543367+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[78107]: cephadm 2023-10-09T09:13:10.549607+0000 mgr.y (mgr.34107) 170 : cephadm 1 Reconfiguring daemon osd.6 on smithi181 2023-10-09T09:13:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[78107]: cluster 2023-10-09T09:13:11.102939+0000 osd.0 (osd.0) 86 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:12.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[78107]: cluster 2023-10-09T09:13:11.234504+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:12.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[78107]: cluster 2023-10-09T09:13:11.236464+0000 osd.3 (osd.3) 146 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:12.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[78107]: cluster 2023-10-09T09:13:11.239917+0000 mon.a (mon.0) 467 : cluster 0 osdmap e519: 8 total, 8 up, 8 in 2023-10-09T09:13:12.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[78107]: cluster 2023-10-09T09:13:11.290486+0000 mgr.y (mgr.34107) 171 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 135 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:13:12.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[78107]: cluster 2023-10-09T09:13:12.237761+0000 mon.a (mon.0) 468 : cluster 0 osdmap e520: 8 total, 8 up, 8 in 2023-10-09T09:13:12.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[75425]: cephadm 2023-10-09T09:13:10.542469+0000 mgr.y (mgr.34107) 169 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T09:13:12.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[75425]: cluster 2023-10-09T09:13:10.543367+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:12.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[75425]: cephadm 2023-10-09T09:13:10.549607+0000 mgr.y (mgr.34107) 170 : cephadm 1 Reconfiguring daemon osd.6 on smithi181 2023-10-09T09:13:12.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[75425]: cluster 2023-10-09T09:13:11.102939+0000 osd.0 (osd.0) 86 : cluster 3 failed to encode map e518 with expected crc 2023-10-09T09:13:12.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[75425]: cluster 2023-10-09T09:13:11.234504+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:12.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[75425]: cluster 2023-10-09T09:13:11.236464+0000 osd.3 (osd.3) 146 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:12.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[75425]: cluster 2023-10-09T09:13:11.239917+0000 mon.a (mon.0) 467 : cluster 0 osdmap e519: 8 total, 8 up, 8 in 2023-10-09T09:13:12.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[75425]: cluster 2023-10-09T09:13:11.290486+0000 mgr.y (mgr.34107) 171 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 135 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:13:12.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:12 smithi163 bash[75425]: cluster 2023-10-09T09:13:12.237761+0000 mon.a (mon.0) 468 : cluster 0 osdmap e520: 8 total, 8 up, 8 in 2023-10-09T09:13:13.260 INFO:tasks.rados.rados.0.smithi163.stdout:1838: done (2 left) 2023-10-09T09:13:13.260 INFO:tasks.rados.rados.0.smithi163.stdout:1839: done (1 left) 2023-10-09T09:13:13.260 INFO:tasks.rados.rados.0.smithi163.stdout:1840: done (0 left) 2023-10-09T09:13:13.260 INFO:tasks.rados.rados.0.smithi163.stdout:1841: delete oid 30 current snap is 232 2023-10-09T09:13:13.274 INFO:tasks.rados.rados.0.smithi163.stdout:1841: done (0 left) 2023-10-09T09:13:13.274 INFO:tasks.rados.rados.0.smithi163.stdout:1842: read oid 24 snap -1 2023-10-09T09:13:13.274 INFO:tasks.rados.rados.0.smithi163.stdout:1842: expect (ObjNum 677 snap 215 seq_num 677) 2023-10-09T09:13:13.274 INFO:tasks.rados.rados.0.smithi163.stdout:1843: read oid 19 snap 225 2023-10-09T09:13:13.274 INFO:tasks.rados.rados.0.smithi163.stdout:1843: expect deleted 2023-10-09T09:13:13.275 INFO:tasks.rados.rados.0.smithi163.stdout:1844: snap_create 2023-10-09T09:13:13.287 INFO:tasks.rados.rados.0.smithi163.stdout:1842: expect (ObjNum 677 snap 215 seq_num 677) 2023-10-09T09:13:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:13 smithi181 bash[59345]: cluster 2023-10-09T09:13:12.232251+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:13 smithi181 bash[59345]: cluster 2023-10-09T09:13:12.232669+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e520 with expected crc 2023-10-09T09:13:13.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:13 smithi181 bash[59345]: cluster 2023-10-09T09:13:12.240006+0000 osd.2 (osd.2) 142 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:13 smithi181 bash[59345]: audit 2023-10-09T09:13:12.551145+0000 mon.a (mon.0) 469 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:13 smithi181 bash[59345]: audit 2023-10-09T09:13:12.558666+0000 mon.a (mon.0) 470 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:13 smithi181 bash[59345]: audit 2023-10-09T09:13:12.568377+0000 mon.a (mon.0) 471 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:13:13.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:13 smithi181 bash[59345]: audit 2023-10-09T09:13:12.569594+0000 mon.a (mon.0) 472 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:13:13] "GET /metrics HTTP/1.1" 200 36010 "" "Prometheus/2.33.4" 2023-10-09T09:13:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[75425]: cluster 2023-10-09T09:13:12.232251+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[75425]: cluster 2023-10-09T09:13:12.232669+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e520 with expected crc 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[75425]: cluster 2023-10-09T09:13:12.240006+0000 osd.2 (osd.2) 142 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[75425]: audit 2023-10-09T09:13:12.551145+0000 mon.a (mon.0) 469 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[75425]: audit 2023-10-09T09:13:12.558666+0000 mon.a (mon.0) 470 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[75425]: audit 2023-10-09T09:13:12.568377+0000 mon.a (mon.0) 471 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[75425]: audit 2023-10-09T09:13:12.569594+0000 mon.a (mon.0) 472 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[78107]: cluster 2023-10-09T09:13:12.232251+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[78107]: cluster 2023-10-09T09:13:12.232669+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e520 with expected crc 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[78107]: cluster 2023-10-09T09:13:12.240006+0000 osd.2 (osd.2) 142 : cluster 3 failed to encode map e519 with expected crc 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[78107]: audit 2023-10-09T09:13:12.551145+0000 mon.a (mon.0) 469 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[78107]: audit 2023-10-09T09:13:12.558666+0000 mon.a (mon.0) 470 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[78107]: audit 2023-10-09T09:13:12.568377+0000 mon.a (mon.0) 471 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:13:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:13 smithi163 bash[78107]: audit 2023-10-09T09:13:12.569594+0000 mon.a (mon.0) 472 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:14.283 INFO:tasks.rados.rados.0.smithi163.stdout:1842: done (2 left) 2023-10-09T09:13:14.283 INFO:tasks.rados.rados.0.smithi163.stdout:1843: done (1 left) 2023-10-09T09:13:14.283 INFO:tasks.rados.rados.0.smithi163.stdout:1844: done (0 left) 2023-10-09T09:13:14.284 INFO:tasks.rados.rados.0.smithi163.stdout:1845: write oid 21 current snap is 233 2023-10-09T09:13:14.284 INFO:tasks.rados.rados.0.smithi163.stdout:1845: seq_num 707 ranges {682549=705519,1922181=464578,3148997=487666} 2023-10-09T09:13:14.293 INFO:tasks.rados.rados.0.smithi163.stdout:1845: writing smithi16367185-21 from 682549 to 1388068 tid 1 2023-10-09T09:13:14.306 INFO:tasks.rados.rados.0.smithi163.stdout:1845: writing smithi16367185-21 from 1922181 to 2386759 tid 2 2023-10-09T09:13:14.314 INFO:tasks.rados.rados.0.smithi163.stdout:1845: writing smithi16367185-21 from 3148997 to 3636663 tid 3 2023-10-09T09:13:14.314 INFO:tasks.rados.rados.0.smithi163.stdout:1846: write oid 30 current snap is 233 2023-10-09T09:13:14.314 INFO:tasks.rados.rados.0.smithi163.stdout:1846: seq_num 708 ranges {699356=497121,1980663=608236,3176420=415291} 2023-10-09T09:13:14.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:14 smithi181 bash[59345]: cephadm 2023-10-09T09:13:12.567883+0000 mgr.y (mgr.34107) 172 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T09:13:14.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:14 smithi181 bash[59345]: cephadm 2023-10-09T09:13:12.573465+0000 mgr.y (mgr.34107) 173 : cephadm 1 Reconfiguring daemon osd.7 on smithi181 2023-10-09T09:13:14.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:14 smithi181 bash[59345]: cluster 2023-10-09T09:13:13.254640+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:14 smithi181 bash[59345]: cluster 2023-10-09T09:13:13.255387+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:14 smithi181 bash[59345]: cluster 2023-10-09T09:13:13.259540+0000 mon.a (mon.0) 473 : cluster 0 osdmap e521: 8 total, 8 up, 8 in 2023-10-09T09:13:14.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:14 smithi181 bash[59345]: cluster 2023-10-09T09:13:13.261683+0000 osd.2 (osd.2) 143 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:14 smithi181 bash[59345]: cluster 2023-10-09T09:13:13.276124+0000 osd.3 (osd.3) 147 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:14 smithi181 bash[59345]: cluster 2023-10-09T09:13:13.290989+0000 mgr.y (mgr.34107) 174 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 135 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:13:14.321 INFO:tasks.rados.rados.0.smithi163.stdout:1846: writing smithi16367185-30 from 699356 to 1196477 tid 1 2023-10-09T09:13:14.330 INFO:tasks.rados.rados.0.smithi163.stdout:1846: writing smithi16367185-30 from 1980663 to 2588899 tid 2 2023-10-09T09:13:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1846: writing smithi16367185-30 from 3176420 to 3591711 tid 3 2023-10-09T09:13:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1847: delete oid 4 current snap is 233 2023-10-09T09:13:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1845: finishing write tid 1 to smithi16367185-21 2023-10-09T09:13:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1845: finishing write tid 2 to smithi16367185-21 2023-10-09T09:13:14.336 INFO:tasks.rados.rados.0.smithi163.stdout:1845: finishing write tid 3 to smithi16367185-21 2023-10-09T09:13:14.337 INFO:tasks.rados.rados.0.smithi163.stdout:1845: finishing write tid 4 to smithi16367185-21 2023-10-09T09:13:14.337 INFO:tasks.rados.rados.0.smithi163.stdout:1845: finishing write tid 5 to smithi16367185-21 2023-10-09T09:13:14.337 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 438 (ObjNum 707 snap 233 seq_num 707) dirty exists 2023-10-09T09:13:14.337 INFO:tasks.rados.rados.0.smithi163.stdout:1845: left oid 21 (ObjNum 707 snap 233 seq_num 707) 2023-10-09T09:13:14.337 INFO:tasks.rados.rados.0.smithi163.stdout:1846: finishing write tid 1 to smithi16367185-30 2023-10-09T09:13:14.337 INFO:tasks.rados.rados.0.smithi163.stdout:1846: finishing write tid 2 to smithi16367185-30 2023-10-09T09:13:14.339 INFO:tasks.rados.rados.0.smithi163.stdout:1845: done (2 left) 2023-10-09T09:13:14.339 INFO:tasks.rados.rados.0.smithi163.stdout:1847: done (1 left) 2023-10-09T09:13:14.339 INFO:tasks.rados.rados.0.smithi163.stdout:1848: snap_create 2023-10-09T09:13:14.340 INFO:tasks.rados.rados.0.smithi163.stdout:1846: finishing write tid 3 to smithi16367185-30 2023-10-09T09:13:14.340 INFO:tasks.rados.rados.0.smithi163.stdout:1846: finishing write tid 4 to smithi16367185-30 2023-10-09T09:13:14.340 INFO:tasks.rados.rados.0.smithi163.stdout:1846: finishing write tid 5 to smithi16367185-30 2023-10-09T09:13:14.340 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 659 (ObjNum 708 snap 233 seq_num 708) dirty exists 2023-10-09T09:13:14.340 INFO:tasks.rados.rados.0.smithi163.stdout:1846: left oid 30 (ObjNum 708 snap 233 seq_num 708) 2023-10-09T09:13:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[75425]: cephadm 2023-10-09T09:13:12.567883+0000 mgr.y (mgr.34107) 172 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T09:13:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[75425]: cephadm 2023-10-09T09:13:12.573465+0000 mgr.y (mgr.34107) 173 : cephadm 1 Reconfiguring daemon osd.7 on smithi181 2023-10-09T09:13:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[75425]: cluster 2023-10-09T09:13:13.254640+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[75425]: cluster 2023-10-09T09:13:13.255387+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[75425]: cluster 2023-10-09T09:13:13.259540+0000 mon.a (mon.0) 473 : cluster 0 osdmap e521: 8 total, 8 up, 8 in 2023-10-09T09:13:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[75425]: cluster 2023-10-09T09:13:13.261683+0000 osd.2 (osd.2) 143 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[75425]: cluster 2023-10-09T09:13:13.276124+0000 osd.3 (osd.3) 147 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[75425]: cluster 2023-10-09T09:13:13.290989+0000 mgr.y (mgr.34107) 174 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 135 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:13:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[78107]: cephadm 2023-10-09T09:13:12.567883+0000 mgr.y (mgr.34107) 172 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T09:13:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[78107]: cephadm 2023-10-09T09:13:12.573465+0000 mgr.y (mgr.34107) 173 : cephadm 1 Reconfiguring daemon osd.7 on smithi181 2023-10-09T09:13:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[78107]: cluster 2023-10-09T09:13:13.254640+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[78107]: cluster 2023-10-09T09:13:13.255387+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[78107]: cluster 2023-10-09T09:13:13.259540+0000 mon.a (mon.0) 473 : cluster 0 osdmap e521: 8 total, 8 up, 8 in 2023-10-09T09:13:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[78107]: cluster 2023-10-09T09:13:13.261683+0000 osd.2 (osd.2) 143 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[78107]: cluster 2023-10-09T09:13:13.276124+0000 osd.3 (osd.3) 147 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:14.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:14 smithi163 bash[78107]: cluster 2023-10-09T09:13:13.290989+0000 mgr.y (mgr.34107) 174 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 135 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2023-10-09T09:13:15.061 INFO:tasks.rados.rados.0.smithi163.stdout:1846: done (1 left) 2023-10-09T09:13:15.062 INFO:tasks.rados.rados.0.smithi163.stdout:1848: done (0 left) 2023-10-09T09:13:15.062 INFO:tasks.rados.rados.0.smithi163.stdout:1849: read oid 7 snap -1 2023-10-09T09:13:15.062 INFO:tasks.rados.rados.0.smithi163.stdout:1849: expect (ObjNum 690 snap 220 seq_num 690) 2023-10-09T09:13:15.062 INFO:tasks.rados.rados.0.smithi163.stdout:1850: snap_remove snap 216 2023-10-09T09:13:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:15 smithi181 bash[59345]: cluster 2023-10-09T09:13:14.276394+0000 osd.3 (osd.3) 148 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:15.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:15 smithi181 bash[59345]: cluster 2023-10-09T09:13:14.279239+0000 osd.0 (osd.0) 87 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:15 smithi181 bash[59345]: cluster 2023-10-09T09:13:14.282597+0000 mon.a (mon.0) 474 : cluster 0 osdmap e522: 8 total, 8 up, 8 in 2023-10-09T09:13:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:15 smithi181 bash[59345]: cluster 2023-10-09T09:13:14.296138+0000 osd.1 (osd.1) 69 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:15 smithi181 bash[59345]: cluster 2023-10-09T09:13:14.324416+0000 osd.2 (osd.2) 144 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:15 smithi181 bash[59345]: audit 2023-10-09T09:13:14.630545+0000 mon.a (mon.0) 475 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:15 smithi181 bash[59345]: audit 2023-10-09T09:13:14.641191+0000 mon.a (mon.0) 476 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:15 smithi181 bash[59345]: audit 2023-10-09T09:13:14.693941+0000 mon.a (mon.0) 477 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.ajqgsc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:13:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:15 smithi181 bash[59345]: audit 2023-10-09T09:13:14.695868+0000 mon.a (mon.0) 478 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:15.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:15 smithi181 bash[59345]: cluster 2023-10-09T09:13:15.060996+0000 mon.a (mon.0) 479 : cluster 0 osdmap e523: 8 total, 8 up, 8 in 2023-10-09T09:13:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[75425]: cluster 2023-10-09T09:13:14.276394+0000 osd.3 (osd.3) 148 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[75425]: cluster 2023-10-09T09:13:14.279239+0000 osd.0 (osd.0) 87 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[75425]: cluster 2023-10-09T09:13:14.282597+0000 mon.a (mon.0) 474 : cluster 0 osdmap e522: 8 total, 8 up, 8 in 2023-10-09T09:13:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[75425]: cluster 2023-10-09T09:13:14.296138+0000 osd.1 (osd.1) 69 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[75425]: cluster 2023-10-09T09:13:14.324416+0000 osd.2 (osd.2) 144 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[75425]: audit 2023-10-09T09:13:14.630545+0000 mon.a (mon.0) 475 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[75425]: audit 2023-10-09T09:13:14.641191+0000 mon.a (mon.0) 476 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[75425]: audit 2023-10-09T09:13:14.693941+0000 mon.a (mon.0) 477 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.ajqgsc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:13:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[75425]: audit 2023-10-09T09:13:14.695868+0000 mon.a (mon.0) 478 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[75425]: cluster 2023-10-09T09:13:15.060996+0000 mon.a (mon.0) 479 : cluster 0 osdmap e523: 8 total, 8 up, 8 in 2023-10-09T09:13:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[78107]: cluster 2023-10-09T09:13:14.276394+0000 osd.3 (osd.3) 148 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:15.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[78107]: cluster 2023-10-09T09:13:14.279239+0000 osd.0 (osd.0) 87 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:15.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[78107]: cluster 2023-10-09T09:13:14.282597+0000 mon.a (mon.0) 474 : cluster 0 osdmap e522: 8 total, 8 up, 8 in 2023-10-09T09:13:15.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[78107]: cluster 2023-10-09T09:13:14.296138+0000 osd.1 (osd.1) 69 : cluster 3 failed to encode map e521 with expected crc 2023-10-09T09:13:15.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[78107]: cluster 2023-10-09T09:13:14.324416+0000 osd.2 (osd.2) 144 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:15.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[78107]: audit 2023-10-09T09:13:14.630545+0000 mon.a (mon.0) 475 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:15.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[78107]: audit 2023-10-09T09:13:14.641191+0000 mon.a (mon.0) 476 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:15.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[78107]: audit 2023-10-09T09:13:14.693941+0000 mon.a (mon.0) 477 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.ajqgsc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:13:15.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[78107]: audit 2023-10-09T09:13:14.695868+0000 mon.a (mon.0) 478 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:15.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:15 smithi163 bash[78107]: cluster 2023-10-09T09:13:15.060996+0000 mon.a (mon.0) 479 : cluster 0 osdmap e523: 8 total, 8 up, 8 in 2023-10-09T09:13:16.065 INFO:tasks.rados.rados.0.smithi163.stdout:1850: done (1 left) 2023-10-09T09:13:16.065 INFO:tasks.rados.rados.0.smithi163.stdout:1851: snap_create 2023-10-09T09:13:16.065 INFO:tasks.rados.rados.0.smithi163.stdout:1849: expect (ObjNum 690 snap 220 seq_num 690) 2023-10-09T09:13:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:16 smithi181 bash[59345]: cephadm 2023-10-09T09:13:14.693209+0000 mgr.y (mgr.34107) 175 : cephadm 1 Reconfiguring mds.foo.smithi181.ajqgsc (monmap changed)... 2023-10-09T09:13:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:16 smithi181 bash[59345]: cephadm 2023-10-09T09:13:14.697769+0000 mgr.y (mgr.34107) 176 : cephadm 1 Reconfiguring daemon mds.foo.smithi181.ajqgsc on smithi181 2023-10-09T09:13:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:16 smithi181 bash[59345]: cluster 2023-10-09T09:13:14.716418+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:16 smithi181 bash[59345]: cluster 2023-10-09T09:13:15.058252+0000 osd.1 (osd.1) 70 : cluster 3 failed to encode map e523 with expected crc 2023-10-09T09:13:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:16 smithi181 bash[59345]: cluster 2023-10-09T09:13:15.059281+0000 osd.2 (osd.2) 145 : cluster 3 failed to encode map e523 with expected crc 2023-10-09T09:13:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:16 smithi181 bash[59345]: cluster 2023-10-09T09:13:15.146738+0000 osd.0 (osd.0) 88 : cluster 3 failed to encode map e523 with expected crc 2023-10-09T09:13:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:16 smithi181 bash[59345]: cluster 2023-10-09T09:13:15.291456+0000 mgr.y (mgr.34107) 177 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 135 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:16.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:16 smithi181 bash[59345]: cluster 2023-10-09T09:13:16.064746+0000 mon.a (mon.0) 480 : cluster 0 osdmap e524: 8 total, 8 up, 8 in 2023-10-09T09:13:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[78107]: cephadm 2023-10-09T09:13:14.693209+0000 mgr.y (mgr.34107) 175 : cephadm 1 Reconfiguring mds.foo.smithi181.ajqgsc (monmap changed)... 2023-10-09T09:13:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[78107]: cephadm 2023-10-09T09:13:14.697769+0000 mgr.y (mgr.34107) 176 : cephadm 1 Reconfiguring daemon mds.foo.smithi181.ajqgsc on smithi181 2023-10-09T09:13:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[78107]: cluster 2023-10-09T09:13:14.716418+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[78107]: cluster 2023-10-09T09:13:15.058252+0000 osd.1 (osd.1) 70 : cluster 3 failed to encode map e523 with expected crc 2023-10-09T09:13:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[78107]: cluster 2023-10-09T09:13:15.059281+0000 osd.2 (osd.2) 145 : cluster 3 failed to encode map e523 with expected crc 2023-10-09T09:13:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[78107]: cluster 2023-10-09T09:13:15.146738+0000 osd.0 (osd.0) 88 : cluster 3 failed to encode map e523 with expected crc 2023-10-09T09:13:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[78107]: cluster 2023-10-09T09:13:15.291456+0000 mgr.y (mgr.34107) 177 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 135 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[78107]: cluster 2023-10-09T09:13:16.064746+0000 mon.a (mon.0) 480 : cluster 0 osdmap e524: 8 total, 8 up, 8 in 2023-10-09T09:13:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[75425]: cephadm 2023-10-09T09:13:14.693209+0000 mgr.y (mgr.34107) 175 : cephadm 1 Reconfiguring mds.foo.smithi181.ajqgsc (monmap changed)... 2023-10-09T09:13:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[75425]: cephadm 2023-10-09T09:13:14.697769+0000 mgr.y (mgr.34107) 176 : cephadm 1 Reconfiguring daemon mds.foo.smithi181.ajqgsc on smithi181 2023-10-09T09:13:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[75425]: cluster 2023-10-09T09:13:14.716418+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e522 with expected crc 2023-10-09T09:13:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[75425]: cluster 2023-10-09T09:13:15.058252+0000 osd.1 (osd.1) 70 : cluster 3 failed to encode map e523 with expected crc 2023-10-09T09:13:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[75425]: cluster 2023-10-09T09:13:15.059281+0000 osd.2 (osd.2) 145 : cluster 3 failed to encode map e523 with expected crc 2023-10-09T09:13:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[75425]: cluster 2023-10-09T09:13:15.146738+0000 osd.0 (osd.0) 88 : cluster 3 failed to encode map e523 with expected crc 2023-10-09T09:13:16.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[75425]: cluster 2023-10-09T09:13:15.291456+0000 mgr.y (mgr.34107) 177 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 135 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:16.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:16 smithi163 bash[75425]: cluster 2023-10-09T09:13:16.064746+0000 mon.a (mon.0) 480 : cluster 0 osdmap e524: 8 total, 8 up, 8 in 2023-10-09T09:13:17.067 INFO:tasks.rados.rados.0.smithi163.stdout:1849: done (1 left) 2023-10-09T09:13:17.067 INFO:tasks.rados.rados.0.smithi163.stdout:1851: done (0 left) 2023-10-09T09:13:17.067 INFO:tasks.rados.rados.0.smithi163.stdout:1852: read oid 38 snap -1 2023-10-09T09:13:17.068 INFO:tasks.rados.rados.0.smithi163.stdout:1852: expect (ObjNum 697 snap 227 seq_num 697) 2023-10-09T09:13:17.068 INFO:tasks.rados.rados.0.smithi163.stdout:1853: snap_create 2023-10-09T09:13:17.084 INFO:tasks.rados.rados.0.smithi163.stdout:1852: expect (ObjNum 697 snap 227 seq_num 697) 2023-10-09T09:13:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:17 smithi181 bash[59345]: cluster 2023-10-09T09:13:16.059196+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e524 with expected crc 2023-10-09T09:13:17.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:17 smithi181 bash[59345]: cluster 2023-10-09T09:13:16.059739+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e524 with expected crc 2023-10-09T09:13:17.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:17 smithi181 bash[59345]: cluster 2023-10-09T09:13:16.288125+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e524 with expected crc 2023-10-09T09:13:17.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:17 smithi181 bash[59345]: audit 2023-10-09T09:13:16.687090+0000 mon.a (mon.0) 481 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:17.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:17 smithi181 bash[59345]: audit 2023-10-09T09:13:16.694763+0000 mon.a (mon.0) 482 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:17.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:17 smithi181 bash[59345]: audit 2023-10-09T09:13:16.726150+0000 mon.a (mon.0) 483 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:17.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:17 smithi181 bash[59345]: audit 2023-10-09T09:13:16.729825+0000 mon.a (mon.0) 484 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:17.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:17 smithi181 bash[59345]: audit 2023-10-09T09:13:16.731182+0000 mon.a (mon.0) 485 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:13:17.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:17 smithi181 bash[59345]: audit 2023-10-09T09:13:16.739298+0000 mon.a (mon.0) 486 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:17.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:17 smithi181 bash[59345]: cluster 2023-10-09T09:13:17.067087+0000 mon.a (mon.0) 487 : cluster 0 osdmap e525: 8 total, 8 up, 8 in 2023-10-09T09:13:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[78107]: cluster 2023-10-09T09:13:16.059196+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e524 with expected crc 2023-10-09T09:13:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[78107]: cluster 2023-10-09T09:13:16.059739+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e524 with expected crc 2023-10-09T09:13:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[78107]: cluster 2023-10-09T09:13:16.288125+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e524 with expected crc 2023-10-09T09:13:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[78107]: audit 2023-10-09T09:13:16.687090+0000 mon.a (mon.0) 481 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:17.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[78107]: audit 2023-10-09T09:13:16.694763+0000 mon.a (mon.0) 482 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:17.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[78107]: audit 2023-10-09T09:13:16.726150+0000 mon.a (mon.0) 483 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:17.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[78107]: audit 2023-10-09T09:13:16.729825+0000 mon.a (mon.0) 484 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:17.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[78107]: audit 2023-10-09T09:13:16.731182+0000 mon.a (mon.0) 485 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:13:17.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[78107]: audit 2023-10-09T09:13:16.739298+0000 mon.a (mon.0) 486 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:17.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[78107]: cluster 2023-10-09T09:13:17.067087+0000 mon.a (mon.0) 487 : cluster 0 osdmap e525: 8 total, 8 up, 8 in 2023-10-09T09:13:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[75425]: cluster 2023-10-09T09:13:16.059196+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e524 with expected crc 2023-10-09T09:13:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[75425]: cluster 2023-10-09T09:13:16.059739+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e524 with expected crc 2023-10-09T09:13:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[75425]: cluster 2023-10-09T09:13:16.288125+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e524 with expected crc 2023-10-09T09:13:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[75425]: audit 2023-10-09T09:13:16.687090+0000 mon.a (mon.0) 481 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:17.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[75425]: audit 2023-10-09T09:13:16.694763+0000 mon.a (mon.0) 482 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:17.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[75425]: audit 2023-10-09T09:13:16.726150+0000 mon.a (mon.0) 483 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:17.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[75425]: audit 2023-10-09T09:13:16.729825+0000 mon.a (mon.0) 484 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:17.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[75425]: audit 2023-10-09T09:13:16.731182+0000 mon.a (mon.0) 485 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:13:17.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[75425]: audit 2023-10-09T09:13:16.739298+0000 mon.a (mon.0) 486 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:17.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:17 smithi163 bash[75425]: cluster 2023-10-09T09:13:17.067087+0000 mon.a (mon.0) 487 : cluster 0 osdmap e525: 8 total, 8 up, 8 in 2023-10-09T09:13:18.070 INFO:tasks.rados.rados.0.smithi163.stdout:1852: done (1 left) 2023-10-09T09:13:18.070 INFO:tasks.rados.rados.0.smithi163.stdout:1853: done (0 left) 2023-10-09T09:13:18.070 INFO:tasks.rados.rados.0.smithi163.stdout:1854: snap_create 2023-10-09T09:13:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:18 smithi181 bash[59345]: cluster 2023-10-09T09:13:17.061804+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:18.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:18 smithi181 bash[59345]: cluster 2023-10-09T09:13:17.063148+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:18.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:18 smithi181 bash[59345]: cluster 2023-10-09T09:13:17.067403+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:18.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:18 smithi181 bash[59345]: cluster 2023-10-09T09:13:17.291964+0000 mgr.y (mgr.34107) 178 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2023-10-09T09:13:18.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:18 smithi181 bash[59345]: cluster 2023-10-09T09:13:18.069843+0000 mon.a (mon.0) 488 : cluster 0 osdmap e526: 8 total, 8 up, 8 in 2023-10-09T09:13:18.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:18 smithi163 bash[75425]: cluster 2023-10-09T09:13:17.061804+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:18.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:18 smithi163 bash[75425]: cluster 2023-10-09T09:13:17.063148+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:18.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:18 smithi163 bash[75425]: cluster 2023-10-09T09:13:17.067403+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:18.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:18 smithi163 bash[75425]: cluster 2023-10-09T09:13:17.291964+0000 mgr.y (mgr.34107) 178 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2023-10-09T09:13:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:18 smithi163 bash[75425]: cluster 2023-10-09T09:13:18.069843+0000 mon.a (mon.0) 488 : cluster 0 osdmap e526: 8 total, 8 up, 8 in 2023-10-09T09:13:18.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:18 smithi163 bash[78107]: cluster 2023-10-09T09:13:17.061804+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:18.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:18 smithi163 bash[78107]: cluster 2023-10-09T09:13:17.063148+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:18.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:18 smithi163 bash[78107]: cluster 2023-10-09T09:13:17.067403+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:18.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:18 smithi163 bash[78107]: cluster 2023-10-09T09:13:17.291964+0000 mgr.y (mgr.34107) 178 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2023-10-09T09:13:18.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:18 smithi163 bash[78107]: cluster 2023-10-09T09:13:18.069843+0000 mon.a (mon.0) 488 : cluster 0 osdmap e526: 8 total, 8 up, 8 in 2023-10-09T09:13:19.072 INFO:tasks.rados.rados.0.smithi163.stdout:1854: done (0 left) 2023-10-09T09:13:19.072 INFO:tasks.rados.rados.0.smithi163.stdout:1855: write oid 5 current snap is 237 2023-10-09T09:13:19.072 INFO:tasks.rados.rados.0.smithi163.stdout:1855: seq_num 710 ranges {732970=647619,2097627=611905,3231266=437835} 2023-10-09T09:13:19.081 INFO:tasks.rados.rados.0.smithi163.stdout:1855: writing smithi16367185-5 from 732970 to 1380589 tid 1 2023-10-09T09:13:19.089 INFO:tasks.rados.rados.0.smithi163.stdout:1855: writing smithi16367185-5 from 2097627 to 2709532 tid 2 2023-10-09T09:13:19.095 INFO:tasks.rados.rados.0.smithi163.stdout:1855: writing smithi16367185-5 from 3231266 to 3669101 tid 3 2023-10-09T09:13:19.095 INFO:tasks.rados.rados.0.smithi163.stdout:1856: rollback oid 4 current snap is 237 2023-10-09T09:13:19.095 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 215 2023-10-09T09:13:19.095 INFO:tasks.rados.rados.0.smithi163.stdout:1855: finishing write tid 1 to smithi16367185-5 2023-10-09T09:13:19.096 INFO:tasks.rados.rados.0.smithi163.stdout:1855: finishing write tid 2 to smithi16367185-5 2023-10-09T09:13:19.096 INFO:tasks.rados.rados.0.smithi163.stdout:1857: snap_create 2023-10-09T09:13:19.101 INFO:tasks.rados.rados.0.smithi163.stdout:1855: finishing write tid 3 to smithi16367185-5 2023-10-09T09:13:19.101 INFO:tasks.rados.rados.0.smithi163.stdout:1855: finishing write tid 4 to smithi16367185-5 2023-10-09T09:13:19.101 INFO:tasks.rados.rados.0.smithi163.stdout:1855: finishing write tid 5 to smithi16367185-5 2023-10-09T09:13:19.101 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 952 (ObjNum 710 snap 237 seq_num 710) dirty exists 2023-10-09T09:13:19.101 INFO:tasks.rados.rados.0.smithi163.stdout:1855: left oid 5 (ObjNum 710 snap 237 seq_num 710) 2023-10-09T09:13:19.101 INFO:tasks.rados.rados.0.smithi163.stdout:1856: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:13:19.101 INFO:tasks.rados.rados.0.smithi163.stdout:1856: finishing rollback tid 2 to smithi16367185-4 2023-10-09T09:13:19.101 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 954 (ObjNum 588 snap 186 seq_num 588) dirty exists 2023-10-09T09:13:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:19 smithi181 bash[59345]: cluster 2023-10-09T09:13:17.706023+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:19 smithi181 bash[59345]: cluster 2023-10-09T09:13:18.064219+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:19 smithi181 bash[59345]: cluster 2023-10-09T09:13:18.065327+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:19 smithi181 bash[59345]: cluster 2023-10-09T09:13:19.072067+0000 mon.a (mon.0) 489 : cluster 0 osdmap e527: 8 total, 8 up, 8 in 2023-10-09T09:13:19.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:19 smithi163 bash[75425]: cluster 2023-10-09T09:13:17.706023+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:19.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:19 smithi163 bash[75425]: cluster 2023-10-09T09:13:18.064219+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:19 smithi163 bash[75425]: cluster 2023-10-09T09:13:18.065327+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:19 smithi163 bash[75425]: cluster 2023-10-09T09:13:19.072067+0000 mon.a (mon.0) 489 : cluster 0 osdmap e527: 8 total, 8 up, 8 in 2023-10-09T09:13:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:19 smithi163 bash[78107]: cluster 2023-10-09T09:13:17.706023+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e525 with expected crc 2023-10-09T09:13:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:19 smithi163 bash[78107]: cluster 2023-10-09T09:13:18.064219+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:19 smithi163 bash[78107]: cluster 2023-10-09T09:13:18.065327+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:19 smithi163 bash[78107]: cluster 2023-10-09T09:13:19.072067+0000 mon.a (mon.0) 489 : cluster 0 osdmap e527: 8 total, 8 up, 8 in 2023-10-09T09:13:20.081 INFO:tasks.rados.rados.0.smithi163.stdout:1855: done (2 left) 2023-10-09T09:13:20.081 INFO:tasks.rados.rados.0.smithi163.stdout:1856: done (1 left) 2023-10-09T09:13:20.081 INFO:tasks.rados.rados.0.smithi163.stdout:1857: done (0 left) 2023-10-09T09:13:20.081 INFO:tasks.rados.rados.0.smithi163.stdout:1858: read oid 32 snap -1 2023-10-09T09:13:20.081 INFO:tasks.rados.rados.0.smithi163.stdout:1858: expect (ObjNum 602 snap 190 seq_num 602) 2023-10-09T09:13:20.082 INFO:tasks.rados.rados.0.smithi163.stdout:1859: snap_create 2023-10-09T09:13:20.094 INFO:tasks.rados.rados.0.smithi163.stdout:1858: expect (ObjNum 602 snap 190 seq_num 602) 2023-10-09T09:13:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:20 smithi181 bash[59345]: cluster 2023-10-09T09:13:18.674528+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:20 smithi181 bash[59345]: cluster 2023-10-09T09:13:18.906494+0000 osd.2 (osd.2) 146 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:20 smithi181 bash[59345]: cluster 2023-10-09T09:13:18.982369+0000 osd.3 (osd.3) 149 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:20 smithi181 bash[59345]: cluster 2023-10-09T09:13:19.066869+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e527 with expected crc 2023-10-09T09:13:20.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:20 smithi181 bash[59345]: cluster 2023-10-09T09:13:19.069273+0000 osd.2 (osd.2) 147 : cluster 3 failed to encode map e527 with expected crc 2023-10-09T09:13:20.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:20 smithi181 bash[59345]: cluster 2023-10-09T09:13:19.102776+0000 osd.0 (osd.0) 89 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:20 smithi181 bash[59345]: cluster 2023-10-09T09:13:19.292630+0000 mgr.y (mgr.34107) 179 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2023-10-09T09:13:20.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:20 smithi181 bash[59345]: cluster 2023-10-09T09:13:20.081098+0000 mon.a (mon.0) 490 : cluster 0 osdmap e528: 8 total, 8 up, 8 in 2023-10-09T09:13:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[78107]: cluster 2023-10-09T09:13:18.674528+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[78107]: cluster 2023-10-09T09:13:18.906494+0000 osd.2 (osd.2) 146 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[78107]: cluster 2023-10-09T09:13:18.982369+0000 osd.3 (osd.3) 149 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[78107]: cluster 2023-10-09T09:13:19.066869+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e527 with expected crc 2023-10-09T09:13:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[78107]: cluster 2023-10-09T09:13:19.069273+0000 osd.2 (osd.2) 147 : cluster 3 failed to encode map e527 with expected crc 2023-10-09T09:13:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[78107]: cluster 2023-10-09T09:13:19.102776+0000 osd.0 (osd.0) 89 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[78107]: cluster 2023-10-09T09:13:19.292630+0000 mgr.y (mgr.34107) 179 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2023-10-09T09:13:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[78107]: cluster 2023-10-09T09:13:20.081098+0000 mon.a (mon.0) 490 : cluster 0 osdmap e528: 8 total, 8 up, 8 in 2023-10-09T09:13:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[75425]: cluster 2023-10-09T09:13:18.674528+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[75425]: cluster 2023-10-09T09:13:18.906494+0000 osd.2 (osd.2) 146 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[75425]: cluster 2023-10-09T09:13:18.982369+0000 osd.3 (osd.3) 149 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[75425]: cluster 2023-10-09T09:13:19.066869+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e527 with expected crc 2023-10-09T09:13:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[75425]: cluster 2023-10-09T09:13:19.069273+0000 osd.2 (osd.2) 147 : cluster 3 failed to encode map e527 with expected crc 2023-10-09T09:13:20.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[75425]: cluster 2023-10-09T09:13:19.102776+0000 osd.0 (osd.0) 89 : cluster 3 failed to encode map e526 with expected crc 2023-10-09T09:13:20.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[75425]: cluster 2023-10-09T09:13:19.292630+0000 mgr.y (mgr.34107) 179 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2023-10-09T09:13:20.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:20 smithi163 bash[75425]: cluster 2023-10-09T09:13:20.081098+0000 mon.a (mon.0) 490 : cluster 0 osdmap e528: 8 total, 8 up, 8 in 2023-10-09T09:13:21.084 INFO:tasks.rados.rados.0.smithi163.stdout:1858: done (1 left) 2023-10-09T09:13:21.085 INFO:tasks.rados.rados.0.smithi163.stdout:1859: done (0 left) 2023-10-09T09:13:21.085 INFO:tasks.rados.rados.0.smithi163.stdout:1860: snap_create 2023-10-09T09:13:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:21 smithi181 bash[59345]: cluster 2023-10-09T09:13:20.076507+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e528 with expected crc 2023-10-09T09:13:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:21 smithi181 bash[59345]: cluster 2023-10-09T09:13:20.078040+0000 osd.3 (osd.3) 150 : cluster 3 failed to encode map e528 with expected crc 2023-10-09T09:13:21.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:21 smithi181 bash[59345]: cluster 2023-10-09T09:13:20.081655+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e528 with expected crc 2023-10-09T09:13:21.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:21 smithi181 bash[59345]: cluster 2023-10-09T09:13:21.084291+0000 mon.a (mon.0) 491 : cluster 0 osdmap e529: 8 total, 8 up, 8 in 2023-10-09T09:13:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:21 smithi163 bash[78107]: cluster 2023-10-09T09:13:20.076507+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e528 with expected crc 2023-10-09T09:13:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:21 smithi163 bash[78107]: cluster 2023-10-09T09:13:20.078040+0000 osd.3 (osd.3) 150 : cluster 3 failed to encode map e528 with expected crc 2023-10-09T09:13:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:21 smithi163 bash[78107]: cluster 2023-10-09T09:13:20.081655+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e528 with expected crc 2023-10-09T09:13:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:21 smithi163 bash[78107]: cluster 2023-10-09T09:13:21.084291+0000 mon.a (mon.0) 491 : cluster 0 osdmap e529: 8 total, 8 up, 8 in 2023-10-09T09:13:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:21 smithi163 bash[75425]: cluster 2023-10-09T09:13:20.076507+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e528 with expected crc 2023-10-09T09:13:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:21 smithi163 bash[75425]: cluster 2023-10-09T09:13:20.078040+0000 osd.3 (osd.3) 150 : cluster 3 failed to encode map e528 with expected crc 2023-10-09T09:13:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:21 smithi163 bash[75425]: cluster 2023-10-09T09:13:20.081655+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e528 with expected crc 2023-10-09T09:13:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:21 smithi163 bash[75425]: cluster 2023-10-09T09:13:21.084291+0000 mon.a (mon.0) 491 : cluster 0 osdmap e529: 8 total, 8 up, 8 in 2023-10-09T09:13:22.090 INFO:tasks.rados.rados.0.smithi163.stdout:1860: done (0 left) 2023-10-09T09:13:22.091 INFO:tasks.rados.rados.0.smithi163.stdout:1861: write oid 39 current snap is 240 2023-10-09T09:13:22.091 INFO:tasks.rados.rados.0.smithi163.stdout:1861: seq_num 711 ranges {749777=722868,2156109=471916,3258689=649107} 2023-10-09T09:13:22.100 INFO:tasks.rados.rados.0.smithi163.stdout:1861: writing smithi16367185-39 from 749777 to 1472645 tid 1 2023-10-09T09:13:22.107 INFO:tasks.rados.rados.0.smithi163.stdout:1861: writing smithi16367185-39 from 2156109 to 2628025 tid 2 2023-10-09T09:13:22.115 INFO:tasks.rados.rados.0.smithi163.stdout:1861: writing smithi16367185-39 from 3258689 to 3907796 tid 3 2023-10-09T09:13:22.115 INFO:tasks.rados.rados.0.smithi163.stdout:1862: read oid 32 snap -1 2023-10-09T09:13:22.115 INFO:tasks.rados.rados.0.smithi163.stdout:1862: expect (ObjNum 602 snap 190 seq_num 602) 2023-10-09T09:13:22.115 INFO:tasks.rados.rados.0.smithi163.stdout:1863: snap_create 2023-10-09T09:13:22.115 INFO:tasks.rados.rados.0.smithi163.stdout:1861: finishing write tid 1 to smithi16367185-39 2023-10-09T09:13:22.115 INFO:tasks.rados.rados.0.smithi163.stdout:1861: finishing write tid 2 to smithi16367185-39 2023-10-09T09:13:22.122 INFO:tasks.rados.rados.0.smithi163.stdout:1861: finishing write tid 3 to smithi16367185-39 2023-10-09T09:13:22.122 INFO:tasks.rados.rados.0.smithi163.stdout:1861: finishing write tid 4 to smithi16367185-39 2023-10-09T09:13:22.123 INFO:tasks.rados.rados.0.smithi163.stdout:1861: finishing write tid 5 to smithi16367185-39 2023-10-09T09:13:22.123 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 728 (ObjNum 711 snap 240 seq_num 711) dirty exists 2023-10-09T09:13:22.123 INFO:tasks.rados.rados.0.smithi163.stdout:1861: left oid 39 (ObjNum 711 snap 240 seq_num 711) 2023-10-09T09:13:22.125 INFO:tasks.rados.rados.0.smithi163.stdout:1862: expect (ObjNum 602 snap 190 seq_num 602) 2023-10-09T09:13:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:22 smithi181 bash[59345]: cluster 2023-10-09T09:13:21.078835+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e529 with expected crc 2023-10-09T09:13:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:22 smithi181 bash[59345]: cluster 2023-10-09T09:13:21.079553+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e529 with expected crc 2023-10-09T09:13:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:22 smithi181 bash[59345]: cluster 2023-10-09T09:13:21.293878+0000 mgr.y (mgr.34107) 180 : cluster 0 pgmap v346: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 415 KiB/s wr, 5 op/s 2023-10-09T09:13:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:22 smithi181 bash[59345]: cluster 2023-10-09T09:13:22.090744+0000 mon.a (mon.0) 492 : cluster 0 osdmap e530: 8 total, 8 up, 8 in 2023-10-09T09:13:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:22 smithi163 bash[78107]: cluster 2023-10-09T09:13:21.078835+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e529 with expected crc 2023-10-09T09:13:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:22 smithi163 bash[78107]: cluster 2023-10-09T09:13:21.079553+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e529 with expected crc 2023-10-09T09:13:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:22 smithi163 bash[78107]: cluster 2023-10-09T09:13:21.293878+0000 mgr.y (mgr.34107) 180 : cluster 0 pgmap v346: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 415 KiB/s wr, 5 op/s 2023-10-09T09:13:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:22 smithi163 bash[78107]: cluster 2023-10-09T09:13:22.090744+0000 mon.a (mon.0) 492 : cluster 0 osdmap e530: 8 total, 8 up, 8 in 2023-10-09T09:13:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:22 smithi163 bash[75425]: cluster 2023-10-09T09:13:21.078835+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e529 with expected crc 2023-10-09T09:13:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:22 smithi163 bash[75425]: cluster 2023-10-09T09:13:21.079553+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e529 with expected crc 2023-10-09T09:13:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:22 smithi163 bash[75425]: cluster 2023-10-09T09:13:21.293878+0000 mgr.y (mgr.34107) 180 : cluster 0 pgmap v346: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 415 KiB/s wr, 5 op/s 2023-10-09T09:13:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:22 smithi163 bash[75425]: cluster 2023-10-09T09:13:22.090744+0000 mon.a (mon.0) 492 : cluster 0 osdmap e530: 8 total, 8 up, 8 in 2023-10-09T09:13:23.096 INFO:tasks.rados.rados.0.smithi163.stdout:1861: done (2 left) 2023-10-09T09:13:23.096 INFO:tasks.rados.rados.0.smithi163.stdout:1862: done (1 left) 2023-10-09T09:13:23.096 INFO:tasks.rados.rados.0.smithi163.stdout:1863: done (0 left) 2023-10-09T09:13:23.096 INFO:tasks.rados.rados.0.smithi163.stdout:1864: snap_create 2023-10-09T09:13:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:23 smithi181 bash[59345]: cluster 2023-10-09T09:13:22.087734+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:23 smithi181 bash[59345]: cluster 2023-10-09T09:13:22.088667+0000 osd.3 (osd.3) 151 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:23 smithi181 bash[59345]: cluster 2023-10-09T09:13:22.102138+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:23 smithi181 bash[59345]: cluster 2023-10-09T09:13:22.822225+0000 osd.1 (osd.1) 71 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:23 smithi181 bash[59345]: cluster 2023-10-09T09:13:23.090362+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:23.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:23 smithi181 bash[59345]: cluster 2023-10-09T09:13:23.095678+0000 mon.a (mon.0) 493 : cluster 0 osdmap e531: 8 total, 8 up, 8 in 2023-10-09T09:13:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:13:23] "GET /metrics HTTP/1.1" 200 36010 "" "Prometheus/2.33.4" 2023-10-09T09:13:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[78107]: cluster 2023-10-09T09:13:22.087734+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[78107]: cluster 2023-10-09T09:13:22.088667+0000 osd.3 (osd.3) 151 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[78107]: cluster 2023-10-09T09:13:22.102138+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[78107]: cluster 2023-10-09T09:13:22.822225+0000 osd.1 (osd.1) 71 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[78107]: cluster 2023-10-09T09:13:23.090362+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[78107]: cluster 2023-10-09T09:13:23.095678+0000 mon.a (mon.0) 493 : cluster 0 osdmap e531: 8 total, 8 up, 8 in 2023-10-09T09:13:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[75425]: cluster 2023-10-09T09:13:22.087734+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[75425]: cluster 2023-10-09T09:13:22.088667+0000 osd.3 (osd.3) 151 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[75425]: cluster 2023-10-09T09:13:22.102138+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[75425]: cluster 2023-10-09T09:13:22.822225+0000 osd.1 (osd.1) 71 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:23.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[75425]: cluster 2023-10-09T09:13:23.090362+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:23.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:23 smithi163 bash[75425]: cluster 2023-10-09T09:13:23.095678+0000 mon.a (mon.0) 493 : cluster 0 osdmap e531: 8 total, 8 up, 8 in 2023-10-09T09:13:24.100 INFO:tasks.rados.rados.0.smithi163.stdout:1864: done (0 left) 2023-10-09T09:13:24.100 INFO:tasks.rados.rados.0.smithi163.stdout:1865: read oid 35 snap -1 2023-10-09T09:13:24.101 INFO:tasks.rados.rados.0.smithi163.stdout:1865: expect (ObjNum 681 snap 217 seq_num 681) 2023-10-09T09:13:24.101 INFO:tasks.rados.rados.0.smithi163.stdout:1866: snap_remove snap 241 2023-10-09T09:13:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:24 smithi181 bash[59345]: cluster 2023-10-09T09:13:22.987816+0000 osd.2 (osd.2) 148 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:24 smithi181 bash[59345]: cluster 2023-10-09T09:13:23.091651+0000 osd.3 (osd.3) 152 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:24 smithi181 bash[59345]: cluster 2023-10-09T09:13:23.137961+0000 osd.0 (osd.0) 90 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:24 smithi181 bash[59345]: cluster 2023-10-09T09:13:23.294597+0000 mgr.y (mgr.34107) 181 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 415 KiB/s wr, 5 op/s 2023-10-09T09:13:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:24 smithi181 bash[59345]: cluster 2023-10-09T09:13:24.099764+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e532 with expected crc 2023-10-09T09:13:24.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:24 smithi181 bash[59345]: cluster 2023-10-09T09:13:24.100397+0000 mon.a (mon.0) 494 : cluster 0 osdmap e532: 8 total, 8 up, 8 in 2023-10-09T09:13:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[75425]: cluster 2023-10-09T09:13:22.987816+0000 osd.2 (osd.2) 148 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[75425]: cluster 2023-10-09T09:13:23.091651+0000 osd.3 (osd.3) 152 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[75425]: cluster 2023-10-09T09:13:23.137961+0000 osd.0 (osd.0) 90 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[75425]: cluster 2023-10-09T09:13:23.294597+0000 mgr.y (mgr.34107) 181 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 415 KiB/s wr, 5 op/s 2023-10-09T09:13:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[75425]: cluster 2023-10-09T09:13:24.099764+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e532 with expected crc 2023-10-09T09:13:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[75425]: cluster 2023-10-09T09:13:24.100397+0000 mon.a (mon.0) 494 : cluster 0 osdmap e532: 8 total, 8 up, 8 in 2023-10-09T09:13:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[78107]: cluster 2023-10-09T09:13:22.987816+0000 osd.2 (osd.2) 148 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[78107]: cluster 2023-10-09T09:13:23.091651+0000 osd.3 (osd.3) 152 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[78107]: cluster 2023-10-09T09:13:23.137961+0000 osd.0 (osd.0) 90 : cluster 3 failed to encode map e530 with expected crc 2023-10-09T09:13:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[78107]: cluster 2023-10-09T09:13:23.294597+0000 mgr.y (mgr.34107) 181 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 415 KiB/s wr, 5 op/s 2023-10-09T09:13:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[78107]: cluster 2023-10-09T09:13:24.099764+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e532 with expected crc 2023-10-09T09:13:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:24 smithi163 bash[78107]: cluster 2023-10-09T09:13:24.100397+0000 mon.a (mon.0) 494 : cluster 0 osdmap e532: 8 total, 8 up, 8 in 2023-10-09T09:13:25.068 INFO:tasks.rados.rados.0.smithi163.stdout:1866: done (1 left) 2023-10-09T09:13:25.068 INFO:tasks.rados.rados.0.smithi163.stdout:1867: write oid 25 current snap is 242 2023-10-09T09:13:25.068 INFO:tasks.rados.rados.0.smithi163.stdout:1867: seq_num 712 ranges {766584=798117,2098238=615574,3169759=460379} 2023-10-09T09:13:25.077 INFO:tasks.rados.rados.0.smithi163.stdout:1867: writing smithi16367185-25 from 766584 to 1564701 tid 1 2023-10-09T09:13:25.085 INFO:tasks.rados.rados.0.smithi163.stdout:1867: writing smithi16367185-25 from 2098238 to 2713812 tid 2 2023-10-09T09:13:25.090 INFO:tasks.rados.rados.0.smithi163.stdout:1867: writing smithi16367185-25 from 3169759 to 3630138 tid 3 2023-10-09T09:13:25.090 INFO:tasks.rados.rados.0.smithi163.stdout:1868: snap_remove snap 214 2023-10-09T09:13:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:25 smithi181 bash[59345]: cluster 2023-10-09T09:13:23.570200+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:25 smithi181 bash[59345]: cluster 2023-10-09T09:13:23.576663+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:25 smithi181 bash[59345]: cluster 2023-10-09T09:13:24.096764+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e532 with expected crc 2023-10-09T09:13:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:25 smithi181 bash[59345]: cluster 2023-10-09T09:13:24.102396+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e532 with expected crc 2023-10-09T09:13:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:25 smithi181 bash[59345]: cluster 2023-10-09T09:13:25.067587+0000 mon.a (mon.0) 495 : cluster 0 osdmap e533: 8 total, 8 up, 8 in 2023-10-09T09:13:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:25 smithi163 bash[75425]: cluster 2023-10-09T09:13:23.570200+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:25 smithi163 bash[75425]: cluster 2023-10-09T09:13:23.576663+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:25 smithi163 bash[75425]: cluster 2023-10-09T09:13:24.096764+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e532 with expected crc 2023-10-09T09:13:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:25 smithi163 bash[75425]: cluster 2023-10-09T09:13:24.102396+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e532 with expected crc 2023-10-09T09:13:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:25 smithi163 bash[75425]: cluster 2023-10-09T09:13:25.067587+0000 mon.a (mon.0) 495 : cluster 0 osdmap e533: 8 total, 8 up, 8 in 2023-10-09T09:13:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:25 smithi163 bash[78107]: cluster 2023-10-09T09:13:23.570200+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:25 smithi163 bash[78107]: cluster 2023-10-09T09:13:23.576663+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e531 with expected crc 2023-10-09T09:13:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:25 smithi163 bash[78107]: cluster 2023-10-09T09:13:24.096764+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e532 with expected crc 2023-10-09T09:13:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:25 smithi163 bash[78107]: cluster 2023-10-09T09:13:24.102396+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e532 with expected crc 2023-10-09T09:13:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:25 smithi163 bash[78107]: cluster 2023-10-09T09:13:25.067587+0000 mon.a (mon.0) 495 : cluster 0 osdmap e533: 8 total, 8 up, 8 in 2023-10-09T09:13:26.087 INFO:tasks.rados.rados.0.smithi163.stdout:1868: done (2 left) 2023-10-09T09:13:26.087 INFO:tasks.rados.rados.0.smithi163.stdout:1869: write oid 2 current snap is 242 2023-10-09T09:13:26.087 INFO:tasks.rados.rados.0.smithi163.stdout:1869: seq_num 713 ranges {783391=473366,1756720=475585,2797182=788004} 2023-10-09T09:13:26.096 INFO:tasks.rados.rados.0.smithi163.stdout:1869: writing smithi16367185-2 from 783391 to 1256757 tid 1 2023-10-09T09:13:26.103 INFO:tasks.rados.rados.0.smithi163.stdout:1869: writing smithi16367185-2 from 1756720 to 2232305 tid 2 2023-10-09T09:13:26.112 INFO:tasks.rados.rados.0.smithi163.stdout:1869: writing smithi16367185-2 from 2797182 to 3585186 tid 3 2023-10-09T09:13:26.112 INFO:tasks.rados.rados.0.smithi163.stdout:1870: read oid 6 snap -1 2023-10-09T09:13:26.113 INFO:tasks.rados.rados.0.smithi163.stdout:1870: expect deleted 2023-10-09T09:13:26.113 INFO:tasks.rados.rados.0.smithi163.stdout:1871: snap_remove snap 223 2023-10-09T09:13:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:26 smithi181 bash[59345]: cluster 2023-10-09T09:13:25.062746+0000 osd.3 (osd.3) 153 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:26 smithi181 bash[59345]: cluster 2023-10-09T09:13:25.065477+0000 osd.2 (osd.2) 149 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:26 smithi181 bash[59345]: cluster 2023-10-09T09:13:25.183079+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:26 smithi181 bash[59345]: cluster 2023-10-09T09:13:25.295360+0000 mgr.y (mgr.34107) 182 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:26 smithi181 bash[59345]: audit 2023-10-09T09:13:25.340931+0000 mon.a (mon.0) 496 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:26 smithi181 bash[59345]: cluster 2023-10-09T09:13:25.597708+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:26 smithi181 bash[59345]: cluster 2023-10-09T09:13:26.087253+0000 mon.a (mon.0) 497 : cluster 0 osdmap e534: 8 total, 8 up, 8 in 2023-10-09T09:13:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[75425]: cluster 2023-10-09T09:13:25.062746+0000 osd.3 (osd.3) 153 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[75425]: cluster 2023-10-09T09:13:25.065477+0000 osd.2 (osd.2) 149 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[75425]: cluster 2023-10-09T09:13:25.183079+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[75425]: cluster 2023-10-09T09:13:25.295360+0000 mgr.y (mgr.34107) 182 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[75425]: audit 2023-10-09T09:13:25.340931+0000 mon.a (mon.0) 496 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[75425]: cluster 2023-10-09T09:13:25.597708+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[75425]: cluster 2023-10-09T09:13:26.087253+0000 mon.a (mon.0) 497 : cluster 0 osdmap e534: 8 total, 8 up, 8 in 2023-10-09T09:13:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[78107]: cluster 2023-10-09T09:13:25.062746+0000 osd.3 (osd.3) 153 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[78107]: cluster 2023-10-09T09:13:25.065477+0000 osd.2 (osd.2) 149 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[78107]: cluster 2023-10-09T09:13:25.183079+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[78107]: cluster 2023-10-09T09:13:25.295360+0000 mgr.y (mgr.34107) 182 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[78107]: audit 2023-10-09T09:13:25.340931+0000 mon.a (mon.0) 496 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:26.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[78107]: cluster 2023-10-09T09:13:25.597708+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:26.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:26 smithi163 bash[78107]: cluster 2023-10-09T09:13:26.087253+0000 mon.a (mon.0) 497 : cluster 0 osdmap e534: 8 total, 8 up, 8 in 2023-10-09T09:13:27.094 INFO:tasks.rados.rados.0.smithi163.stdout:1871: done (4 left) 2023-10-09T09:13:27.094 INFO:tasks.rados.rados.0.smithi163.stdout:1872: snap_create 2023-10-09T09:13:27.094 INFO:tasks.rados.rados.0.smithi163.stdout:1865: expect (ObjNum 681 snap 217 seq_num 681) 2023-10-09T09:13:27.290 INFO:tasks.rados.rados.0.smithi163.stdout:1867: finishing write tid 1 to smithi16367185-25 2023-10-09T09:13:27.290 INFO:tasks.rados.rados.0.smithi163.stdout:1867: finishing write tid 2 to smithi16367185-25 2023-10-09T09:13:27.290 INFO:tasks.rados.rados.0.smithi163.stdout:1867: finishing write tid 3 to smithi16367185-25 2023-10-09T09:13:27.290 INFO:tasks.rados.rados.0.smithi163.stdout:1867: finishing write tid 4 to smithi16367185-25 2023-10-09T09:13:27.290 INFO:tasks.rados.rados.0.smithi163.stdout:1867: finishing write tid 5 to smithi16367185-25 2023-10-09T09:13:27.290 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 824 (ObjNum 712 snap 242 seq_num 712) dirty exists 2023-10-09T09:13:27.290 INFO:tasks.rados.rados.0.smithi163.stdout:1867: left oid 25 (ObjNum 712 snap 242 seq_num 712) 2023-10-09T09:13:27.290 INFO:tasks.rados.rados.0.smithi163.stdout:1869: finishing write tid 1 to smithi16367185-2 2023-10-09T09:13:27.291 INFO:tasks.rados.rados.0.smithi163.stdout:1869: finishing write tid 2 to smithi16367185-2 2023-10-09T09:13:27.291 INFO:tasks.rados.rados.0.smithi163.stdout:1869: finishing write tid 3 to smithi16367185-2 2023-10-09T09:13:27.291 INFO:tasks.rados.rados.0.smithi163.stdout:1869: finishing write tid 4 to smithi16367185-2 2023-10-09T09:13:27.291 INFO:tasks.rados.rados.0.smithi163.stdout:1869: finishing write tid 5 to smithi16367185-2 2023-10-09T09:13:27.291 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 494 (ObjNum 713 snap 242 seq_num 713) dirty exists 2023-10-09T09:13:27.291 INFO:tasks.rados.rados.0.smithi163.stdout:1869: left oid 2 (ObjNum 713 snap 242 seq_num 713) 2023-10-09T09:13:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:27 smithi181 bash[59345]: cluster 2023-10-09T09:13:25.522250+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:27 smithi181 bash[59345]: cluster 2023-10-09T09:13:26.083976+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e534 with expected crc 2023-10-09T09:13:27.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:27 smithi181 bash[59345]: cluster 2023-10-09T09:13:26.084727+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e534 with expected crc 2023-10-09T09:13:27.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:27 smithi181 bash[59345]: cluster 2023-10-09T09:13:26.098302+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e534 with expected crc 2023-10-09T09:13:27.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:27 smithi181 bash[59345]: cluster 2023-10-09T09:13:27.093094+0000 mon.a (mon.0) 498 : cluster 0 osdmap e535: 8 total, 8 up, 8 in 2023-10-09T09:13:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:27 smithi163 bash[78107]: cluster 2023-10-09T09:13:25.522250+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:27 smithi163 bash[78107]: cluster 2023-10-09T09:13:26.083976+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e534 with expected crc 2023-10-09T09:13:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:27 smithi163 bash[78107]: cluster 2023-10-09T09:13:26.084727+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e534 with expected crc 2023-10-09T09:13:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:27 smithi163 bash[78107]: cluster 2023-10-09T09:13:26.098302+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e534 with expected crc 2023-10-09T09:13:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:27 smithi163 bash[78107]: cluster 2023-10-09T09:13:27.093094+0000 mon.a (mon.0) 498 : cluster 0 osdmap e535: 8 total, 8 up, 8 in 2023-10-09T09:13:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:27 smithi163 bash[75425]: cluster 2023-10-09T09:13:25.522250+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e533 with expected crc 2023-10-09T09:13:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:27 smithi163 bash[75425]: cluster 2023-10-09T09:13:26.083976+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e534 with expected crc 2023-10-09T09:13:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:27 smithi163 bash[75425]: cluster 2023-10-09T09:13:26.084727+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e534 with expected crc 2023-10-09T09:13:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:27 smithi163 bash[75425]: cluster 2023-10-09T09:13:26.098302+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e534 with expected crc 2023-10-09T09:13:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:27 smithi163 bash[75425]: cluster 2023-10-09T09:13:27.093094+0000 mon.a (mon.0) 498 : cluster 0 osdmap e535: 8 total, 8 up, 8 in 2023-10-09T09:13:28.095 INFO:tasks.rados.rados.0.smithi163.stdout:1865: done (4 left) 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:1867: done (3 left) 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:1869: done (2 left) 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:1870: done (1 left) 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:1872: done (0 left) 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:1873: rollback oid 36 current snap is 243 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 36 to 235 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:1874: read oid 34 snap -1 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:1874: expect (ObjNum 695 snap 225 seq_num 695) 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:1875: rollback oid 46 current snap is 243 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 46 to 217 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:1876: rollback oid 8 current snap is 243 2023-10-09T09:13:28.096 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 8 to 215 2023-10-09T09:13:28.097 INFO:tasks.rados.rados.0.smithi163.stdout:1877: read oid 14 snap -1 2023-10-09T09:13:28.097 INFO:tasks.rados.rados.0.smithi163.stdout:1877: expect (ObjNum 622 snap 197 seq_num 622) 2023-10-09T09:13:28.097 INFO:tasks.rados.rados.0.smithi163.stdout:1878: snap_remove snap 225 2023-10-09T09:13:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:28 smithi181 bash[59345]: cluster 2023-10-09T09:13:27.088609+0000 osd.1 (osd.1) 72 : cluster 3 failed to encode map e535 with expected crc 2023-10-09T09:13:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:28 smithi181 bash[59345]: cluster 2023-10-09T09:13:27.091835+0000 osd.2 (osd.2) 150 : cluster 3 failed to encode map e535 with expected crc 2023-10-09T09:13:28.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:28 smithi181 bash[59345]: cluster 2023-10-09T09:13:27.116502+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e535 with expected crc 2023-10-09T09:13:28.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:28 smithi181 bash[59345]: cluster 2023-10-09T09:13:27.295995+0000 mgr.y (mgr.34107) 183 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 157 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2023-10-09T09:13:28.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:28 smithi181 bash[59345]: cluster 2023-10-09T09:13:28.090217+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:28.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:28 smithi181 bash[59345]: cluster 2023-10-09T09:13:28.094726+0000 mon.a (mon.0) 499 : cluster 0 osdmap e536: 8 total, 8 up, 8 in 2023-10-09T09:13:28.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:28 smithi181 bash[59345]: cluster 2023-10-09T09:13:28.095379+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[75425]: cluster 2023-10-09T09:13:27.088609+0000 osd.1 (osd.1) 72 : cluster 3 failed to encode map e535 with expected crc 2023-10-09T09:13:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[75425]: cluster 2023-10-09T09:13:27.091835+0000 osd.2 (osd.2) 150 : cluster 3 failed to encode map e535 with expected crc 2023-10-09T09:13:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[75425]: cluster 2023-10-09T09:13:27.116502+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e535 with expected crc 2023-10-09T09:13:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[75425]: cluster 2023-10-09T09:13:27.295995+0000 mgr.y (mgr.34107) 183 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 157 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2023-10-09T09:13:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[75425]: cluster 2023-10-09T09:13:28.090217+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[75425]: cluster 2023-10-09T09:13:28.094726+0000 mon.a (mon.0) 499 : cluster 0 osdmap e536: 8 total, 8 up, 8 in 2023-10-09T09:13:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[75425]: cluster 2023-10-09T09:13:28.095379+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[78107]: cluster 2023-10-09T09:13:27.088609+0000 osd.1 (osd.1) 72 : cluster 3 failed to encode map e535 with expected crc 2023-10-09T09:13:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[78107]: cluster 2023-10-09T09:13:27.091835+0000 osd.2 (osd.2) 150 : cluster 3 failed to encode map e535 with expected crc 2023-10-09T09:13:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[78107]: cluster 2023-10-09T09:13:27.116502+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e535 with expected crc 2023-10-09T09:13:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[78107]: cluster 2023-10-09T09:13:27.295995+0000 mgr.y (mgr.34107) 183 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 157 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2023-10-09T09:13:28.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[78107]: cluster 2023-10-09T09:13:28.090217+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:28.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[78107]: cluster 2023-10-09T09:13:28.094726+0000 mon.a (mon.0) 499 : cluster 0 osdmap e536: 8 total, 8 up, 8 in 2023-10-09T09:13:28.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:28 smithi163 bash[78107]: cluster 2023-10-09T09:13:28.095379+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.097 INFO:tasks.rados.rados.0.smithi163.stdout:1878: done (5 left) 2023-10-09T09:13:29.098 INFO:tasks.rados.rados.0.smithi163.stdout:1879: write oid 6 current snap is 243 2023-10-09T09:13:29.098 INFO:tasks.rados.rados.0.smithi163.stdout:1876: finishing rollback tid 1 to smithi16367185-8 2023-10-09T09:13:29.098 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 0 (ObjNum 698915088 snap 21875 seq_num 698975376) dirty dne 2023-10-09T09:13:29.098 INFO:tasks.rados.rados.0.smithi163.stdout:1873: finishing rollback tid 0 to smithi16367185-36 2023-10-09T09:13:29.098 INFO:tasks.rados.rados.0.smithi163.stdout:1873: finishing rollback tid 1 to smithi16367185-36 2023-10-09T09:13:29.098 INFO:tasks.rados.rados.0.smithi163.stdout:1879: seq_num 714 ranges {400198=548615,1415202=584996} 2023-10-09T09:13:29.104 INFO:tasks.rados.rados.0.smithi163.stdout:1879: writing smithi16367185-6 from 400198 to 948813 tid 1 2023-10-09T09:13:29.112 INFO:tasks.rados.rados.0.smithi163.stdout:1879: writing smithi16367185-6 from 1415202 to 2000198 tid 2 2023-10-09T09:13:29.112 INFO:tasks.rados.rados.0.smithi163.stdout:1876: done (5 left) 2023-10-09T09:13:29.112 INFO:tasks.rados.rados.0.smithi163.stdout:1880: write oid 9 current snap is 243 2023-10-09T09:13:29.112 INFO:tasks.rados.rados.0.smithi163.stdout:1873: finishing rollback tid 2 to smithi16367185-36 2023-10-09T09:13:29.113 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 495 (ObjNum 699 snap 228 seq_num 699) dirty exists 2023-10-09T09:13:29.113 INFO:tasks.rados.rados.0.smithi163.stdout:1874: expect (ObjNum 695 snap 225 seq_num 695) 2023-10-09T09:13:29.407 INFO:tasks.rados.rados.0.smithi163.stdout:1877: expect (ObjNum 622 snap 197 seq_num 622) 2023-10-09T09:13:29.488 INFO:tasks.rados.rados.0.smithi163.stdout:1875: finishing rollback tid 0 to smithi16367185-46 2023-10-09T09:13:29.488 INFO:tasks.rados.rados.0.smithi163.stdout:1875: finishing rollback tid 1 to smithi16367185-46 2023-10-09T09:13:29.488 INFO:tasks.rados.rados.0.smithi163.stdout:1875: finishing rollback tid 2 to smithi16367185-46 2023-10-09T09:13:29.488 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 719 (ObjNum 538 snap 168 seq_num 538) dirty exists 2023-10-09T09:13:29.488 INFO:tasks.rados.rados.0.smithi163.stdout:1879: finishing write tid 1 to smithi16367185-6 2023-10-09T09:13:29.488 INFO:tasks.rados.rados.0.smithi163.stdout:1879: finishing write tid 2 to smithi16367185-6 2023-10-09T09:13:29.488 INFO:tasks.rados.rados.0.smithi163.stdout:1879: finishing write tid 3 to smithi16367185-6 2023-10-09T09:13:29.488 INFO:tasks.rados.rados.0.smithi163.stdout:1879: finishing write tid 4 to smithi16367185-6 2023-10-09T09:13:29.488 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 186 (ObjNum 714 snap 243 seq_num 714) dirty exists 2023-10-09T09:13:29.488 INFO:tasks.rados.rados.0.smithi163.stdout:1879: left oid 6 (ObjNum 714 snap 243 seq_num 714) 2023-10-09T09:13:29.489 INFO:tasks.rados.rados.0.smithi163.stdout:1880: seq_num 715 ranges {417005=740217,1590037=426968} 2023-10-09T09:13:29.498 INFO:tasks.rados.rados.0.smithi163.stdout:1880: writing smithi16367185-9 from 417005 to 1157222 tid 1 2023-10-09T09:13:29.504 INFO:tasks.rados.rados.0.smithi163.stdout:1880: writing smithi16367185-9 from 1590037 to 2017005 tid 2 2023-10-09T09:13:29.504 INFO:tasks.rados.rados.0.smithi163.stdout:1873: done (5 left) 2023-10-09T09:13:29.504 INFO:tasks.rados.rados.0.smithi163.stdout:1874: done (4 left) 2023-10-09T09:13:29.504 INFO:tasks.rados.rados.0.smithi163.stdout:1875: done (3 left) 2023-10-09T09:13:29.504 INFO:tasks.rados.rados.0.smithi163.stdout:1877: done (2 left) 2023-10-09T09:13:29.504 INFO:tasks.rados.rados.0.smithi163.stdout:1879: done (1 left) 2023-10-09T09:13:29.504 INFO:tasks.rados.rados.0.smithi163.stdout:1881: read oid 43 snap -1 2023-10-09T09:13:29.504 INFO:tasks.rados.rados.0.smithi163.stdout:1881: expect (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:13:29.505 INFO:tasks.rados.rados.0.smithi163.stdout:1882: write oid 32 current snap is 243 2023-10-09T09:13:29.505 INFO:tasks.rados.rados.0.smithi163.stdout:1882: seq_num 716 ranges {433812=415466,1648519=385293} 2023-10-09T09:13:29.509 INFO:tasks.rados.rados.0.smithi163.stdout:1882: writing smithi16367185-32 from 433812 to 849278 tid 1 2023-10-09T09:13:29.517 INFO:tasks.rados.rados.0.smithi163.stdout:1882: writing smithi16367185-32 from 1648519 to 2033812 tid 2 2023-10-09T09:13:29.517 INFO:tasks.rados.rados.0.smithi163.stdout:1883: read oid 46 snap -1 2023-10-09T09:13:29.517 INFO:tasks.rados.rados.0.smithi163.stdout:1883: expect (ObjNum 538 snap 168 seq_num 538) 2023-10-09T09:13:29.517 INFO:tasks.rados.rados.0.smithi163.stdout:1884: snap_remove snap 221 2023-10-09T09:13:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:29 smithi163 bash[75425]: cluster 2023-10-09T09:13:28.090741+0000 osd.1 (osd.1) 73 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:29 smithi163 bash[75425]: cluster 2023-10-09T09:13:28.097114+0000 osd.0 (osd.0) 91 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:29 smithi163 bash[75425]: cluster 2023-10-09T09:13:28.097459+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:29 smithi163 bash[75425]: cluster 2023-10-09T09:13:28.100168+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:29 smithi163 bash[75425]: cluster 2023-10-09T09:13:29.097203+0000 mon.a (mon.0) 500 : cluster 0 osdmap e537: 8 total, 8 up, 8 in 2023-10-09T09:13:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:29 smithi163 bash[78107]: cluster 2023-10-09T09:13:28.090741+0000 osd.1 (osd.1) 73 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:29 smithi163 bash[78107]: cluster 2023-10-09T09:13:28.097114+0000 osd.0 (osd.0) 91 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:29 smithi163 bash[78107]: cluster 2023-10-09T09:13:28.097459+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:29 smithi163 bash[78107]: cluster 2023-10-09T09:13:28.100168+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:29 smithi163 bash[78107]: cluster 2023-10-09T09:13:29.097203+0000 mon.a (mon.0) 500 : cluster 0 osdmap e537: 8 total, 8 up, 8 in 2023-10-09T09:13:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:29 smithi181 bash[59345]: cluster 2023-10-09T09:13:28.090741+0000 osd.1 (osd.1) 73 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:29 smithi181 bash[59345]: cluster 2023-10-09T09:13:28.097114+0000 osd.0 (osd.0) 91 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:29 smithi181 bash[59345]: cluster 2023-10-09T09:13:28.097459+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:29 smithi181 bash[59345]: cluster 2023-10-09T09:13:28.100168+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e536 with expected crc 2023-10-09T09:13:29.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:29 smithi181 bash[59345]: cluster 2023-10-09T09:13:29.097203+0000 mon.a (mon.0) 500 : cluster 0 osdmap e537: 8 total, 8 up, 8 in 2023-10-09T09:13:30.070 INFO:tasks.rados.rados.0.smithi163.stdout:1884: done (4 left) 2023-10-09T09:13:30.070 INFO:tasks.rados.rados.0.smithi163.stdout:1885: snap_remove snap 240 2023-10-09T09:13:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:30 smithi163 bash[75425]: cluster 2023-10-09T09:13:29.091681+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e537 with expected crc 2023-10-09T09:13:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:30 smithi163 bash[75425]: cluster 2023-10-09T09:13:29.093971+0000 osd.2 (osd.2) 151 : cluster 3 failed to encode map e537 with expected crc 2023-10-09T09:13:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:30 smithi163 bash[75425]: cluster 2023-10-09T09:13:29.296635+0000 mgr.y (mgr.34107) 184 : cluster 0 pgmap v358: 89 pgs: 89 active+clean; 157 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2023-10-09T09:13:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:30 smithi163 bash[75425]: cluster 2023-10-09T09:13:30.069741+0000 mon.a (mon.0) 501 : cluster 0 osdmap e538: 8 total, 8 up, 8 in 2023-10-09T09:13:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:30 smithi163 bash[78107]: cluster 2023-10-09T09:13:29.091681+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e537 with expected crc 2023-10-09T09:13:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:30 smithi163 bash[78107]: cluster 2023-10-09T09:13:29.093971+0000 osd.2 (osd.2) 151 : cluster 3 failed to encode map e537 with expected crc 2023-10-09T09:13:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:30 smithi163 bash[78107]: cluster 2023-10-09T09:13:29.296635+0000 mgr.y (mgr.34107) 184 : cluster 0 pgmap v358: 89 pgs: 89 active+clean; 157 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2023-10-09T09:13:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:30 smithi163 bash[78107]: cluster 2023-10-09T09:13:30.069741+0000 mon.a (mon.0) 501 : cluster 0 osdmap e538: 8 total, 8 up, 8 in 2023-10-09T09:13:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:30 smithi181 bash[59345]: cluster 2023-10-09T09:13:29.091681+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e537 with expected crc 2023-10-09T09:13:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:30 smithi181 bash[59345]: cluster 2023-10-09T09:13:29.093971+0000 osd.2 (osd.2) 151 : cluster 3 failed to encode map e537 with expected crc 2023-10-09T09:13:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:30 smithi181 bash[59345]: cluster 2023-10-09T09:13:29.296635+0000 mgr.y (mgr.34107) 184 : cluster 0 pgmap v358: 89 pgs: 89 active+clean; 157 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2023-10-09T09:13:30.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:30 smithi181 bash[59345]: cluster 2023-10-09T09:13:30.069741+0000 mon.a (mon.0) 501 : cluster 0 osdmap e538: 8 total, 8 up, 8 in 2023-10-09T09:13:31.073 INFO:tasks.rados.rados.0.smithi163.stdout:1885: done (4 left) 2023-10-09T09:13:31.073 INFO:tasks.rados.rados.0.smithi163.stdout:1886: snap_remove snap 227 2023-10-09T09:13:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:31 smithi163 bash[78107]: cluster 2023-10-09T09:13:30.064394+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e538 with expected crc 2023-10-09T09:13:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:31 smithi163 bash[78107]: cluster 2023-10-09T09:13:30.065925+0000 osd.2 (osd.2) 152 : cluster 3 failed to encode map e538 with expected crc 2023-10-09T09:13:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:31 smithi163 bash[78107]: cluster 2023-10-09T09:13:31.072960+0000 mon.a (mon.0) 502 : cluster 0 osdmap e539: 8 total, 8 up, 8 in 2023-10-09T09:13:31.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:31 smithi163 bash[75425]: cluster 2023-10-09T09:13:30.064394+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e538 with expected crc 2023-10-09T09:13:31.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:31 smithi163 bash[75425]: cluster 2023-10-09T09:13:30.065925+0000 osd.2 (osd.2) 152 : cluster 3 failed to encode map e538 with expected crc 2023-10-09T09:13:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:31 smithi163 bash[75425]: cluster 2023-10-09T09:13:31.072960+0000 mon.a (mon.0) 502 : cluster 0 osdmap e539: 8 total, 8 up, 8 in 2023-10-09T09:13:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:31 smithi181 bash[59345]: cluster 2023-10-09T09:13:30.064394+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e538 with expected crc 2023-10-09T09:13:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:31 smithi181 bash[59345]: cluster 2023-10-09T09:13:30.065925+0000 osd.2 (osd.2) 152 : cluster 3 failed to encode map e538 with expected crc 2023-10-09T09:13:31.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:31 smithi181 bash[59345]: cluster 2023-10-09T09:13:31.072960+0000 mon.a (mon.0) 502 : cluster 0 osdmap e539: 8 total, 8 up, 8 in 2023-10-09T09:13:32.079 INFO:tasks.rados.rados.0.smithi163.stdout:1886: done (4 left) 2023-10-09T09:13:32.079 INFO:tasks.rados.rados.0.smithi163.stdout:1887: rollback oid 37 current snap is 243 2023-10-09T09:13:32.079 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 231 2023-10-09T09:13:32.079 INFO:tasks.rados.rados.0.smithi163.stdout:1880: finishing write tid 1 to smithi16367185-9 2023-10-09T09:13:32.079 INFO:tasks.rados.rados.0.smithi163.stdout:1880: finishing write tid 2 to smithi16367185-9 2023-10-09T09:13:32.079 INFO:tasks.rados.rados.0.smithi163.stdout:1880: finishing write tid 3 to smithi16367185-9 2023-10-09T09:13:32.079 INFO:tasks.rados.rados.0.smithi163.stdout:1880: finishing write tid 4 to smithi16367185-9 2023-10-09T09:13:32.079 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 455 (ObjNum 715 snap 243 seq_num 715) dirty exists 2023-10-09T09:13:32.080 INFO:tasks.rados.rados.0.smithi163.stdout:1880: left oid 9 (ObjNum 715 snap 243 seq_num 715) 2023-10-09T09:13:32.080 INFO:tasks.rados.rados.0.smithi163.stdout:1882: finishing write tid 1 to smithi16367185-32 2023-10-09T09:13:32.080 INFO:tasks.rados.rados.0.smithi163.stdout:1881: expect (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:13:32.194 INFO:tasks.rados.rados.0.smithi163.stdout:1882: finishing write tid 2 to smithi16367185-32 2023-10-09T09:13:32.194 INFO:tasks.rados.rados.0.smithi163.stdout:1882: finishing write tid 3 to smithi16367185-32 2023-10-09T09:13:32.194 INFO:tasks.rados.rados.0.smithi163.stdout:1882: finishing write tid 4 to smithi16367185-32 2023-10-09T09:13:32.194 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1365 (ObjNum 716 snap 243 seq_num 716) dirty exists 2023-10-09T09:13:32.195 INFO:tasks.rados.rados.0.smithi163.stdout:1882: left oid 32 (ObjNum 716 snap 243 seq_num 716) 2023-10-09T09:13:32.195 INFO:tasks.rados.rados.0.smithi163.stdout:1883: expect (ObjNum 538 snap 168 seq_num 538) 2023-10-09T09:13:32.389 INFO:tasks.rados.rados.0.smithi163.stdout:1887: finishing rollback tid 0 to smithi16367185-37 2023-10-09T09:13:32.389 INFO:tasks.rados.rados.0.smithi163.stdout:1887: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:1887: finishing rollback tid 2 to smithi16367185-37 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 917 (ObjNum 684 snap 218 seq_num 684) dirty exists 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:1880: done (4 left) 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:1881: done (3 left) 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:1882: done (2 left) 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:1883: done (1 left) 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:1887: done (0 left) 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:1888: rollback oid 41 current snap is 243 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 41 to 236 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:1889: rollback oid 4 current snap is 243 2023-10-09T09:13:32.390 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 217 2023-10-09T09:13:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:1890: read oid 2 snap -1 2023-10-09T09:13:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:1890: expect (ObjNum 713 snap 242 seq_num 713) 2023-10-09T09:13:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:1891: read oid 3 snap -1 2023-10-09T09:13:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:1891: expect deleted 2023-10-09T09:13:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:1892: read oid 26 snap -1 2023-10-09T09:13:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:1892: expect (ObjNum 687 snap 219 seq_num 687) 2023-10-09T09:13:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:1893: snap_create 2023-10-09T09:13:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:1888: finishing rollback tid 1 to smithi16367185-41 2023-10-09T09:13:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 0 (ObjNum 698922896 snap 21875 seq_num 0) dirty dne 2023-10-09T09:13:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:1892: expect (ObjNum 687 snap 219 seq_num 687) 2023-10-09T09:13:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[78107]: cluster 2023-10-09T09:13:31.068018+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e539 with expected crc 2023-10-09T09:13:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[78107]: cluster 2023-10-09T09:13:31.068349+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e539 with expected crc 2023-10-09T09:13:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[78107]: cluster 2023-10-09T09:13:31.162961+0000 osd.3 (osd.3) 154 : cluster 3 failed to encode map e539 with expected crc 2023-10-09T09:13:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[78107]: cluster 2023-10-09T09:13:31.297894+0000 mgr.y (mgr.34107) 185 : cluster 0 pgmap v361: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 757 KiB/s wr, 13 op/s 2023-10-09T09:13:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[78107]: cluster 2023-10-09T09:13:32.073811+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e540 with expected crc 2023-10-09T09:13:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[78107]: cluster 2023-10-09T09:13:32.078553+0000 mon.a (mon.0) 503 : cluster 0 osdmap e540: 8 total, 8 up, 8 in 2023-10-09T09:13:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[75425]: cluster 2023-10-09T09:13:31.068018+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e539 with expected crc 2023-10-09T09:13:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[75425]: cluster 2023-10-09T09:13:31.068349+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e539 with expected crc 2023-10-09T09:13:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[75425]: cluster 2023-10-09T09:13:31.162961+0000 osd.3 (osd.3) 154 : cluster 3 failed to encode map e539 with expected crc 2023-10-09T09:13:32.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[75425]: cluster 2023-10-09T09:13:31.297894+0000 mgr.y (mgr.34107) 185 : cluster 0 pgmap v361: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 757 KiB/s wr, 13 op/s 2023-10-09T09:13:32.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[75425]: cluster 2023-10-09T09:13:32.073811+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e540 with expected crc 2023-10-09T09:13:32.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:32 smithi163 bash[75425]: cluster 2023-10-09T09:13:32.078553+0000 mon.a (mon.0) 503 : cluster 0 osdmap e540: 8 total, 8 up, 8 in 2023-10-09T09:13:32.720 INFO:tasks.rados.rados.0.smithi163.stdout:1889: finishing rollback tid 0 to smithi16367185-4 2023-10-09T09:13:32.720 INFO:tasks.rados.rados.0.smithi163.stdout:1889: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:13:32.721 INFO:tasks.rados.rados.0.smithi163.stdout:1889: finishing rollback tid 2 to smithi16367185-4 2023-10-09T09:13:32.721 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 970 (ObjNum 588 snap 186 seq_num 588) dirty exists 2023-10-09T09:13:32.721 INFO:tasks.rados.rados.0.smithi163.stdout:1890: expect (ObjNum 713 snap 242 seq_num 713) 2023-10-09T09:13:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:32 smithi181 bash[59345]: cluster 2023-10-09T09:13:31.068018+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e539 with expected crc 2023-10-09T09:13:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:32 smithi181 bash[59345]: cluster 2023-10-09T09:13:31.068349+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e539 with expected crc 2023-10-09T09:13:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:32 smithi181 bash[59345]: cluster 2023-10-09T09:13:31.162961+0000 osd.3 (osd.3) 154 : cluster 3 failed to encode map e539 with expected crc 2023-10-09T09:13:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:32 smithi181 bash[59345]: cluster 2023-10-09T09:13:31.297894+0000 mgr.y (mgr.34107) 185 : cluster 0 pgmap v361: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 757 KiB/s wr, 13 op/s 2023-10-09T09:13:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:32 smithi181 bash[59345]: cluster 2023-10-09T09:13:32.073811+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e540 with expected crc 2023-10-09T09:13:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:32 smithi181 bash[59345]: cluster 2023-10-09T09:13:32.078553+0000 mon.a (mon.0) 503 : cluster 0 osdmap e540: 8 total, 8 up, 8 in 2023-10-09T09:13:33.335 INFO:tasks.rados.rados.0.smithi163.stdout:1888: done (5 left) 2023-10-09T09:13:33.335 INFO:tasks.rados.rados.0.smithi163.stdout:1889: done (4 left) 2023-10-09T09:13:33.336 INFO:tasks.rados.rados.0.smithi163.stdout:1890: done (3 left) 2023-10-09T09:13:33.336 INFO:tasks.rados.rados.0.smithi163.stdout:1891: done (2 left) 2023-10-09T09:13:33.336 INFO:tasks.rados.rados.0.smithi163.stdout:1892: done (1 left) 2023-10-09T09:13:33.336 INFO:tasks.rados.rados.0.smithi163.stdout:1893: done (0 left) 2023-10-09T09:13:33.336 INFO:tasks.rados.rados.0.smithi163.stdout:1894: rollback oid 11 current snap is 244 2023-10-09T09:13:33.336 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 11 to 243 2023-10-09T09:13:33.336 INFO:tasks.rados.rados.0.smithi163.stdout:1895: snap_remove snap 233 2023-10-09T09:13:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:13:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:13:33] "GET /metrics HTTP/1.1" 200 36079 "" "Prometheus/2.33.4" 2023-10-09T09:13:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:33 smithi163 bash[78107]: cluster 2023-10-09T09:13:32.075749+0000 osd.3 (osd.3) 155 : cluster 3 failed to encode map e540 with expected crc 2023-10-09T09:13:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:33 smithi163 bash[78107]: cluster 2023-10-09T09:13:32.078662+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e540 with expected crc 2023-10-09T09:13:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:33 smithi163 bash[75425]: cluster 2023-10-09T09:13:32.075749+0000 osd.3 (osd.3) 155 : cluster 3 failed to encode map e540 with expected crc 2023-10-09T09:13:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:33 smithi163 bash[75425]: cluster 2023-10-09T09:13:32.078662+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e540 with expected crc 2023-10-09T09:13:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:33 smithi181 bash[59345]: cluster 2023-10-09T09:13:32.075749+0000 osd.3 (osd.3) 155 : cluster 3 failed to encode map e540 with expected crc 2023-10-09T09:13:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:33 smithi181 bash[59345]: cluster 2023-10-09T09:13:32.078662+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e540 with expected crc 2023-10-09T09:13:34.343 INFO:tasks.rados.rados.0.smithi163.stdout:1895: done (1 left) 2023-10-09T09:13:34.343 INFO:tasks.rados.rados.0.smithi163.stdout:1896: write oid 41 current snap is 244 2023-10-09T09:13:34.343 INFO:tasks.rados.rados.0.smithi163.stdout:1894: finishing rollback tid 1 to smithi16367185-11 2023-10-09T09:13:34.344 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 0 (ObjNum 0 snap 0 seq_num 1769327376) dirty dne 2023-10-09T09:13:34.344 INFO:tasks.rados.rados.0.smithi163.stdout:1896: seq_num 717 ranges {450619=490715,1590648=459971} 2023-10-09T09:13:34.351 INFO:tasks.rados.rados.0.smithi163.stdout:1896: writing smithi16367185-41 from 450619 to 941334 tid 1 2023-10-09T09:13:34.359 INFO:tasks.rados.rados.0.smithi163.stdout:1896: writing smithi16367185-41 from 1590648 to 2050619 tid 2 2023-10-09T09:13:34.359 INFO:tasks.rados.rados.0.smithi163.stdout:1894: done (1 left) 2023-10-09T09:13:34.359 INFO:tasks.rados.rados.0.smithi163.stdout:1897: snap_remove snap 243 2023-10-09T09:13:34.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:34 smithi163 bash[78107]: cluster 2023-10-09T09:13:33.298566+0000 mgr.y (mgr.34107) 186 : cluster 0 pgmap v363: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 721 KiB/s wr, 12 op/s 2023-10-09T09:13:34.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:34 smithi163 bash[78107]: cluster 2023-10-09T09:13:33.330310+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e541 with expected crc 2023-10-09T09:13:34.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:34 smithi163 bash[78107]: cluster 2023-10-09T09:13:33.332299+0000 osd.1 (osd.1) 74 : cluster 3 failed to encode map e541 with expected crc 2023-10-09T09:13:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:34 smithi163 bash[78107]: cluster 2023-10-09T09:13:33.335552+0000 mon.a (mon.0) 504 : cluster 0 osdmap e541: 8 total, 8 up, 8 in 2023-10-09T09:13:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:34 smithi163 bash[75425]: cluster 2023-10-09T09:13:33.298566+0000 mgr.y (mgr.34107) 186 : cluster 0 pgmap v363: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 721 KiB/s wr, 12 op/s 2023-10-09T09:13:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:34 smithi163 bash[75425]: cluster 2023-10-09T09:13:33.330310+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e541 with expected crc 2023-10-09T09:13:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:34 smithi163 bash[75425]: cluster 2023-10-09T09:13:33.332299+0000 osd.1 (osd.1) 74 : cluster 3 failed to encode map e541 with expected crc 2023-10-09T09:13:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:34 smithi163 bash[75425]: cluster 2023-10-09T09:13:33.335552+0000 mon.a (mon.0) 504 : cluster 0 osdmap e541: 8 total, 8 up, 8 in 2023-10-09T09:13:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:34 smithi181 bash[59345]: cluster 2023-10-09T09:13:33.298566+0000 mgr.y (mgr.34107) 186 : cluster 0 pgmap v363: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 721 KiB/s wr, 12 op/s 2023-10-09T09:13:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:34 smithi181 bash[59345]: cluster 2023-10-09T09:13:33.330310+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e541 with expected crc 2023-10-09T09:13:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:34 smithi181 bash[59345]: cluster 2023-10-09T09:13:33.332299+0000 osd.1 (osd.1) 74 : cluster 3 failed to encode map e541 with expected crc 2023-10-09T09:13:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:34 smithi181 bash[59345]: cluster 2023-10-09T09:13:33.335552+0000 mon.a (mon.0) 504 : cluster 0 osdmap e541: 8 total, 8 up, 8 in 2023-10-09T09:13:35.074 INFO:tasks.rados.rados.0.smithi163.stdout:1897: done (1 left) 2023-10-09T09:13:35.074 INFO:tasks.rados.rados.0.smithi163.stdout:1898: delete oid 40 current snap is 244 2023-10-09T09:13:35.074 INFO:tasks.rados.rados.0.smithi163.stdout:1896: finishing write tid 1 to smithi16367185-41 2023-10-09T09:13:35.074 INFO:tasks.rados.rados.0.smithi163.stdout:1896: finishing write tid 2 to smithi16367185-41 2023-10-09T09:13:35.075 INFO:tasks.rados.rados.0.smithi163.stdout:1896: finishing write tid 3 to smithi16367185-41 2023-10-09T09:13:35.075 INFO:tasks.rados.rados.0.smithi163.stdout:1896: finishing write tid 4 to smithi16367185-41 2023-10-09T09:13:35.075 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 128 (ObjNum 717 snap 244 seq_num 717) dirty exists 2023-10-09T09:13:35.075 INFO:tasks.rados.rados.0.smithi163.stdout:1896: left oid 41 (ObjNum 717 snap 244 seq_num 717) 2023-10-09T09:13:35.090 INFO:tasks.rados.rados.0.smithi163.stdout:1896: done (1 left) 2023-10-09T09:13:35.091 INFO:tasks.rados.rados.0.smithi163.stdout:1898: done (0 left) 2023-10-09T09:13:35.091 INFO:tasks.rados.rados.0.smithi163.stdout:1899: snap_create 2023-10-09T09:13:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:35 smithi163 bash[78107]: cluster 2023-10-09T09:13:34.339305+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e542 with expected crc 2023-10-09T09:13:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:35 smithi163 bash[78107]: cluster 2023-10-09T09:13:34.341234+0000 osd.0 (osd.0) 92 : cluster 3 failed to encode map e542 with expected crc 2023-10-09T09:13:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:35 smithi163 bash[78107]: cluster 2023-10-09T09:13:34.343186+0000 mon.a (mon.0) 505 : cluster 0 osdmap e542: 8 total, 8 up, 8 in 2023-10-09T09:13:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:35 smithi163 bash[78107]: cluster 2023-10-09T09:13:35.073832+0000 mon.a (mon.0) 506 : cluster 0 osdmap e543: 8 total, 8 up, 8 in 2023-10-09T09:13:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:35 smithi163 bash[75425]: cluster 2023-10-09T09:13:34.339305+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e542 with expected crc 2023-10-09T09:13:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:35 smithi163 bash[75425]: cluster 2023-10-09T09:13:34.341234+0000 osd.0 (osd.0) 92 : cluster 3 failed to encode map e542 with expected crc 2023-10-09T09:13:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:35 smithi163 bash[75425]: cluster 2023-10-09T09:13:34.343186+0000 mon.a (mon.0) 505 : cluster 0 osdmap e542: 8 total, 8 up, 8 in 2023-10-09T09:13:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:35 smithi163 bash[75425]: cluster 2023-10-09T09:13:35.073832+0000 mon.a (mon.0) 506 : cluster 0 osdmap e543: 8 total, 8 up, 8 in 2023-10-09T09:13:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:35 smithi181 bash[59345]: cluster 2023-10-09T09:13:34.339305+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e542 with expected crc 2023-10-09T09:13:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:35 smithi181 bash[59345]: cluster 2023-10-09T09:13:34.341234+0000 osd.0 (osd.0) 92 : cluster 3 failed to encode map e542 with expected crc 2023-10-09T09:13:35.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:35 smithi181 bash[59345]: cluster 2023-10-09T09:13:34.343186+0000 mon.a (mon.0) 505 : cluster 0 osdmap e542: 8 total, 8 up, 8 in 2023-10-09T09:13:35.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:35 smithi181 bash[59345]: cluster 2023-10-09T09:13:35.073832+0000 mon.a (mon.0) 506 : cluster 0 osdmap e543: 8 total, 8 up, 8 in 2023-10-09T09:13:36.096 INFO:tasks.rados.rados.0.smithi163.stdout:1899: done (0 left) 2023-10-09T09:13:36.096 INFO:tasks.rados.rados.0.smithi163.stdout:1900: write oid 7 current snap is 245 2023-10-09T09:13:36.096 INFO:tasks.rados.rados.0.smithi163.stdout:1900: seq_num 719 ranges {484233=641213,1707612=376621} 2023-10-09T09:13:36.103 INFO:tasks.rados.rados.0.smithi163.stdout:1900: writing smithi16367185-7 from 484233 to 1125446 tid 1 2023-10-09T09:13:36.109 INFO:tasks.rados.rados.0.smithi163.stdout:1900: writing smithi16367185-7 from 1707612 to 2084233 tid 2 2023-10-09T09:13:36.110 INFO:tasks.rados.rados.0.smithi163.stdout:1901: rollback oid 45 current snap is 245 2023-10-09T09:13:36.110 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 45 to 220 2023-10-09T09:13:36.110 INFO:tasks.rados.rados.0.smithi163.stdout:1900: finishing write tid 1 to smithi16367185-7 2023-10-09T09:13:36.110 INFO:tasks.rados.rados.0.smithi163.stdout:1902: read oid 44 snap -1 2023-10-09T09:13:36.110 INFO:tasks.rados.rados.0.smithi163.stdout:1902: expect deleted 2023-10-09T09:13:36.110 INFO:tasks.rados.rados.0.smithi163.stdout:1903: write oid 9 current snap is 245 2023-10-09T09:13:36.110 INFO:tasks.rados.rados.0.smithi163.stdout:1903: seq_num 720 ranges {501040=716462,1766094=334946} 2023-10-09T09:13:36.116 INFO:tasks.rados.rados.0.smithi163.stdout:1903: writing smithi16367185-9 from 501040 to 1217502 tid 1 2023-10-09T09:13:36.122 INFO:tasks.rados.rados.0.smithi163.stdout:1903: writing smithi16367185-9 from 1766094 to 2101040 tid 2 2023-10-09T09:13:36.122 INFO:tasks.rados.rados.0.smithi163.stdout:1900: finishing write tid 2 to smithi16367185-7 2023-10-09T09:13:36.122 INFO:tasks.rados.rados.0.smithi163.stdout:1900: finishing write tid 3 to smithi16367185-7 2023-10-09T09:13:36.123 INFO:tasks.rados.rados.0.smithi163.stdout:1900: finishing write tid 4 to smithi16367185-7 2023-10-09T09:13:36.123 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 442 (ObjNum 719 snap 245 seq_num 719) dirty exists 2023-10-09T09:13:36.123 INFO:tasks.rados.rados.0.smithi163.stdout:1900: left oid 7 (ObjNum 719 snap 245 seq_num 719) 2023-10-09T09:13:36.123 INFO:tasks.rados.rados.0.smithi163.stdout:1901: finishing rollback tid 1 to smithi16367185-45 2023-10-09T09:13:36.123 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 0 (ObjNum 1 snap 32603 seq_num 699140240) dirty dne 2023-10-09T09:13:36.123 INFO:tasks.rados.rados.0.smithi163.stdout:1900: done (3 left) 2023-10-09T09:13:36.123 INFO:tasks.rados.rados.0.smithi163.stdout:1901: done (2 left) 2023-10-09T09:13:36.123 INFO:tasks.rados.rados.0.smithi163.stdout:1902: done (1 left) 2023-10-09T09:13:36.123 INFO:tasks.rados.rados.0.smithi163.stdout:1904: write oid 29 current snap is 245 2023-10-09T09:13:36.123 INFO:tasks.rados.rados.0.smithi163.stdout:1904: seq_num 721 ranges {517847=791711,2108223=9624} 2023-10-09T09:13:36.130 INFO:tasks.rados.rados.0.smithi163.stdout:1904: writing smithi16367185-29 from 517847 to 1309558 tid 1 2023-10-09T09:13:36.135 INFO:tasks.rados.rados.0.smithi163.stdout:1904: writing smithi16367185-29 from 2108223 to 2117847 tid 2 2023-10-09T09:13:36.135 INFO:tasks.rados.rados.0.smithi163.stdout:1905: read oid 43 snap -1 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:1905: expect (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:1906: rollback oid 32 current snap is 245 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 239 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:1903: finishing write tid 1 to smithi16367185-9 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:1903: finishing write tid 2 to smithi16367185-9 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:1903: finishing write tid 3 to smithi16367185-9 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:1903: finishing write tid 4 to smithi16367185-9 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 463 (ObjNum 720 snap 245 seq_num 720) dirty exists 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:1903: left oid 9 (ObjNum 720 snap 245 seq_num 720) 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:1903: done (3 left) 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:1907: write oid 34 current snap is 245 2023-10-09T09:13:36.136 INFO:tasks.rados.rados.0.smithi163.stdout:1907: seq_num 722 ranges {534654=466960,1766705=367949} 2023-10-09T09:13:36.141 INFO:tasks.rados.rados.0.smithi163.stdout:1907: writing smithi16367185-34 from 534654 to 1001614 tid 1 2023-10-09T09:13:36.148 INFO:tasks.rados.rados.0.smithi163.stdout:1907: writing smithi16367185-34 from 1766705 to 2134654 tid 2 2023-10-09T09:13:36.148 INFO:tasks.rados.rados.0.smithi163.stdout:1908: write oid 41 current snap is 245 2023-10-09T09:13:36.148 INFO:tasks.rados.rados.0.smithi163.stdout:1908: seq_num 723 ranges {551461=658562,1941540=209921} 2023-10-09T09:13:36.154 INFO:tasks.rados.rados.0.smithi163.stdout:1908: writing smithi16367185-41 from 551461 to 1210023 tid 1 2023-10-09T09:13:36.160 INFO:tasks.rados.rados.0.smithi163.stdout:1908: writing smithi16367185-41 from 1941540 to 2151461 tid 2 2023-10-09T09:13:36.160 INFO:tasks.rados.rados.0.smithi163.stdout:1909: delete oid 1 current snap is 245 2023-10-09T09:13:36.160 INFO:tasks.rados.rados.0.smithi163.stdout:1904: finishing write tid 1 to smithi16367185-29 2023-10-09T09:13:36.160 INFO:tasks.rados.rados.0.smithi163.stdout:1904: finishing write tid 2 to smithi16367185-29 2023-10-09T09:13:36.160 INFO:tasks.rados.rados.0.smithi163.stdout:1904: finishing write tid 3 to smithi16367185-29 2023-10-09T09:13:36.161 INFO:tasks.rados.rados.0.smithi163.stdout:1904: finishing write tid 4 to smithi16367185-29 2023-10-09T09:13:36.161 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 854 (ObjNum 721 snap 245 seq_num 721) dirty exists 2023-10-09T09:13:36.161 INFO:tasks.rados.rados.0.smithi163.stdout:1904: left oid 29 (ObjNum 721 snap 245 seq_num 721) 2023-10-09T09:13:36.161 INFO:tasks.rados.rados.0.smithi163.stdout:1906: finishing rollback tid 0 to smithi16367185-32 2023-10-09T09:13:36.161 INFO:tasks.rados.rados.0.smithi163.stdout:1905: expect (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:13:36.274 INFO:tasks.rados.rados.0.smithi163.stdout:1906: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:13:36.274 INFO:tasks.rados.rados.0.smithi163.stdout:1906: finishing rollback tid 2 to smithi16367185-32 2023-10-09T09:13:36.274 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1386 (ObjNum 602 snap 190 seq_num 602) dirty exists 2023-10-09T09:13:36.274 INFO:tasks.rados.rados.0.smithi163.stdout:1907: finishing write tid 1 to smithi16367185-34 2023-10-09T09:13:36.274 INFO:tasks.rados.rados.0.smithi163.stdout:1907: finishing write tid 2 to smithi16367185-34 2023-10-09T09:13:36.274 INFO:tasks.rados.rados.0.smithi163.stdout:1907: finishing write tid 3 to smithi16367185-34 2023-10-09T09:13:36.274 INFO:tasks.rados.rados.0.smithi163.stdout:1907: finishing write tid 4 to smithi16367185-34 2023-10-09T09:13:36.274 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 750 (ObjNum 722 snap 245 seq_num 722) dirty exists 2023-10-09T09:13:36.274 INFO:tasks.rados.rados.0.smithi163.stdout:1907: left oid 34 (ObjNum 722 snap 245 seq_num 722) 2023-10-09T09:13:36.274 INFO:tasks.rados.rados.0.smithi163.stdout:1908: finishing write tid 1 to smithi16367185-41 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1908: finishing write tid 2 to smithi16367185-41 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1908: finishing write tid 3 to smithi16367185-41 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1908: finishing write tid 4 to smithi16367185-41 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 132 (ObjNum 723 snap 245 seq_num 723) dirty exists 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1908: left oid 41 (ObjNum 723 snap 245 seq_num 723) 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1904: done (5 left) 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1905: done (4 left) 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1906: done (3 left) 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1907: done (2 left) 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1908: done (1 left) 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1909: done (0 left) 2023-10-09T09:13:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:1910: rollback oid 20 current snap is 245 2023-10-09T09:13:36.276 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 229 2023-10-09T09:13:36.276 INFO:tasks.rados.rados.0.smithi163.stdout:1911: snap_remove snap 242 2023-10-09T09:13:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[75425]: cluster 2023-10-09T09:13:34.353649+0000 osd.2 (osd.2) 153 : cluster 3 failed to encode map e542 with expected crc 2023-10-09T09:13:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[75425]: cluster 2023-10-09T09:13:35.069547+0000 osd.3 (osd.3) 156 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[75425]: cluster 2023-10-09T09:13:35.072485+0000 osd.2 (osd.2) 154 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[75425]: cluster 2023-10-09T09:13:35.076078+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[75425]: cluster 2023-10-09T09:13:35.174347+0000 osd.0 (osd.0) 93 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[75425]: cluster 2023-10-09T09:13:35.299190+0000 mgr.y (mgr.34107) 187 : cluster 0 pgmap v367: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[75425]: cluster 2023-10-09T09:13:35.420537+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[75425]: cluster 2023-10-09T09:13:36.095835+0000 mon.a (mon.0) 507 : cluster 0 osdmap e544: 8 total, 8 up, 8 in 2023-10-09T09:13:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[78107]: cluster 2023-10-09T09:13:34.353649+0000 osd.2 (osd.2) 153 : cluster 3 failed to encode map e542 with expected crc 2023-10-09T09:13:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[78107]: cluster 2023-10-09T09:13:35.069547+0000 osd.3 (osd.3) 156 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[78107]: cluster 2023-10-09T09:13:35.072485+0000 osd.2 (osd.2) 154 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[78107]: cluster 2023-10-09T09:13:35.076078+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[78107]: cluster 2023-10-09T09:13:35.174347+0000 osd.0 (osd.0) 93 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[78107]: cluster 2023-10-09T09:13:35.299190+0000 mgr.y (mgr.34107) 187 : cluster 0 pgmap v367: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:36.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[78107]: cluster 2023-10-09T09:13:35.420537+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:36 smithi163 bash[78107]: cluster 2023-10-09T09:13:36.095835+0000 mon.a (mon.0) 507 : cluster 0 osdmap e544: 8 total, 8 up, 8 in 2023-10-09T09:13:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:36 smithi181 bash[59345]: cluster 2023-10-09T09:13:34.353649+0000 osd.2 (osd.2) 153 : cluster 3 failed to encode map e542 with expected crc 2023-10-09T09:13:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:36 smithi181 bash[59345]: cluster 2023-10-09T09:13:35.069547+0000 osd.3 (osd.3) 156 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:36 smithi181 bash[59345]: cluster 2023-10-09T09:13:35.072485+0000 osd.2 (osd.2) 154 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:36 smithi181 bash[59345]: cluster 2023-10-09T09:13:35.076078+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:36 smithi181 bash[59345]: cluster 2023-10-09T09:13:35.174347+0000 osd.0 (osd.0) 93 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:36 smithi181 bash[59345]: cluster 2023-10-09T09:13:35.299190+0000 mgr.y (mgr.34107) 187 : cluster 0 pgmap v367: 89 pgs: 89 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:36 smithi181 bash[59345]: cluster 2023-10-09T09:13:35.420537+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e543 with expected crc 2023-10-09T09:13:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:36 smithi181 bash[59345]: cluster 2023-10-09T09:13:36.095835+0000 mon.a (mon.0) 507 : cluster 0 osdmap e544: 8 total, 8 up, 8 in 2023-10-09T09:13:37.098 INFO:tasks.rados.rados.0.smithi163.stdout:1911: done (1 left) 2023-10-09T09:13:37.098 INFO:tasks.rados.rados.0.smithi163.stdout:1912: delete oid 10 current snap is 245 2023-10-09T09:13:37.098 INFO:tasks.rados.rados.0.smithi163.stdout:1910: finishing rollback tid 0 to smithi16367185-20 2023-10-09T09:13:37.098 INFO:tasks.rados.rados.0.smithi163.stdout:1910: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:13:37.098 INFO:tasks.rados.rados.0.smithi163.stdout:1910: finishing rollback tid 2 to smithi16367185-20 2023-10-09T09:13:37.098 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 202 (ObjNum 694 snap 221 seq_num 694) dirty exists 2023-10-09T09:13:37.104 INFO:tasks.rados.rados.0.smithi163.stdout:1910: done (1 left) 2023-10-09T09:13:37.104 INFO:tasks.rados.rados.0.smithi163.stdout:1912: done (0 left) 2023-10-09T09:13:37.104 INFO:tasks.rados.rados.0.smithi163.stdout:1913: snap_remove snap 228 2023-10-09T09:13:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[75425]: cluster 2023-10-09T09:13:36.093750+0000 osd.1 (osd.1) 75 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[75425]: cluster 2023-10-09T09:13:36.094580+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[75425]: cluster 2023-10-09T09:13:36.109848+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[75425]: cluster 2023-10-09T09:13:36.119326+0000 osd.3 (osd.3) 157 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[75425]: cluster 2023-10-09T09:13:36.132478+0000 osd.2 (osd.2) 155 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[75425]: cluster 2023-10-09T09:13:37.095390+0000 osd.2 (osd.2) 156 : cluster 3 failed to encode map e545 with expected crc 2023-10-09T09:13:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[75425]: cluster 2023-10-09T09:13:37.097853+0000 mon.a (mon.0) 508 : cluster 0 osdmap e545: 8 total, 8 up, 8 in 2023-10-09T09:13:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[78107]: cluster 2023-10-09T09:13:36.093750+0000 osd.1 (osd.1) 75 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[78107]: cluster 2023-10-09T09:13:36.094580+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[78107]: cluster 2023-10-09T09:13:36.109848+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[78107]: cluster 2023-10-09T09:13:36.119326+0000 osd.3 (osd.3) 157 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[78107]: cluster 2023-10-09T09:13:36.132478+0000 osd.2 (osd.2) 155 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[78107]: cluster 2023-10-09T09:13:37.095390+0000 osd.2 (osd.2) 156 : cluster 3 failed to encode map e545 with expected crc 2023-10-09T09:13:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:37 smithi163 bash[78107]: cluster 2023-10-09T09:13:37.097853+0000 mon.a (mon.0) 508 : cluster 0 osdmap e545: 8 total, 8 up, 8 in 2023-10-09T09:13:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:37 smithi181 bash[59345]: cluster 2023-10-09T09:13:36.093750+0000 osd.1 (osd.1) 75 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:37 smithi181 bash[59345]: cluster 2023-10-09T09:13:36.094580+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:37 smithi181 bash[59345]: cluster 2023-10-09T09:13:36.109848+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:37 smithi181 bash[59345]: cluster 2023-10-09T09:13:36.119326+0000 osd.3 (osd.3) 157 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:37 smithi181 bash[59345]: cluster 2023-10-09T09:13:36.132478+0000 osd.2 (osd.2) 155 : cluster 3 failed to encode map e544 with expected crc 2023-10-09T09:13:37.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:37 smithi181 bash[59345]: cluster 2023-10-09T09:13:37.095390+0000 osd.2 (osd.2) 156 : cluster 3 failed to encode map e545 with expected crc 2023-10-09T09:13:37.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:37 smithi181 bash[59345]: cluster 2023-10-09T09:13:37.097853+0000 mon.a (mon.0) 508 : cluster 0 osdmap e545: 8 total, 8 up, 8 in 2023-10-09T09:13:38.106 INFO:tasks.rados.rados.0.smithi163.stdout:1913: done (0 left) 2023-10-09T09:13:38.106 INFO:tasks.rados.rados.0.smithi163.stdout:1914: rollback oid 38 current snap is 245 2023-10-09T09:13:38.106 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 238 2023-10-09T09:13:38.106 INFO:tasks.rados.rados.0.smithi163.stdout:1915: read oid 36 snap -1 2023-10-09T09:13:38.107 INFO:tasks.rados.rados.0.smithi163.stdout:1915: expect (ObjNum 699 snap 228 seq_num 699) 2023-10-09T09:13:38.110 INFO:tasks.rados.rados.0.smithi163.stdout:1916: delete oid 20 current snap is 245 2023-10-09T09:13:38.119 INFO:tasks.rados.rados.0.smithi163.stdout:1914: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:13:38.119 INFO:tasks.rados.rados.0.smithi163.stdout:1916: done (2 left) 2023-10-09T09:13:38.119 INFO:tasks.rados.rados.0.smithi163.stdout:1917: read oid 28 snap -1 2023-10-09T09:13:38.119 INFO:tasks.rados.rados.0.smithi163.stdout:1917: expect (ObjNum 675 snap 212 seq_num 675) 2023-10-09T09:13:38.119 INFO:tasks.rados.rados.0.smithi163.stdout:1918: snap_remove snap 229 2023-10-09T09:13:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[78107]: cluster 2023-10-09T09:13:37.094453+0000 osd.3 (osd.3) 158 : cluster 3 failed to encode map e545 with expected crc 2023-10-09T09:13:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[78107]: cluster 2023-10-09T09:13:37.136425+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e545 with expected crc 2023-10-09T09:13:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[78107]: cluster 2023-10-09T09:13:37.299938+0000 mgr.y (mgr.34107) 188 : cluster 0 pgmap v370: 89 pgs: 89 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2023-10-09T09:13:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[78107]: cluster 2023-10-09T09:13:38.102940+0000 osd.2 (osd.2) 157 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[78107]: cluster 2023-10-09T09:13:38.105321+0000 mon.a (mon.0) 509 : cluster 0 osdmap e546: 8 total, 8 up, 8 in 2023-10-09T09:13:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[78107]: cluster 2023-10-09T09:13:38.105849+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[75425]: cluster 2023-10-09T09:13:37.094453+0000 osd.3 (osd.3) 158 : cluster 3 failed to encode map e545 with expected crc 2023-10-09T09:13:38.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[75425]: cluster 2023-10-09T09:13:37.136425+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e545 with expected crc 2023-10-09T09:13:38.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[75425]: cluster 2023-10-09T09:13:37.299938+0000 mgr.y (mgr.34107) 188 : cluster 0 pgmap v370: 89 pgs: 89 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2023-10-09T09:13:38.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[75425]: cluster 2023-10-09T09:13:38.102940+0000 osd.2 (osd.2) 157 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:38.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[75425]: cluster 2023-10-09T09:13:38.105321+0000 mon.a (mon.0) 509 : cluster 0 osdmap e546: 8 total, 8 up, 8 in 2023-10-09T09:13:38.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:38 smithi163 bash[75425]: cluster 2023-10-09T09:13:38.105849+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:38 smithi181 bash[59345]: cluster 2023-10-09T09:13:37.094453+0000 osd.3 (osd.3) 158 : cluster 3 failed to encode map e545 with expected crc 2023-10-09T09:13:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:38 smithi181 bash[59345]: cluster 2023-10-09T09:13:37.136425+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e545 with expected crc 2023-10-09T09:13:38.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:38 smithi181 bash[59345]: cluster 2023-10-09T09:13:37.299938+0000 mgr.y (mgr.34107) 188 : cluster 0 pgmap v370: 89 pgs: 89 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2023-10-09T09:13:38.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:38 smithi181 bash[59345]: cluster 2023-10-09T09:13:38.102940+0000 osd.2 (osd.2) 157 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:38.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:38 smithi181 bash[59345]: cluster 2023-10-09T09:13:38.105321+0000 mon.a (mon.0) 509 : cluster 0 osdmap e546: 8 total, 8 up, 8 in 2023-10-09T09:13:38.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:38 smithi181 bash[59345]: cluster 2023-10-09T09:13:38.105849+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.112 INFO:tasks.rados.rados.0.smithi163.stdout:1918: done (3 left) 2023-10-09T09:13:39.113 INFO:tasks.rados.rados.0.smithi163.stdout:1919: delete oid 17 current snap is 245 2023-10-09T09:13:39.113 INFO:tasks.rados.rados.0.smithi163.stdout:1914: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:13:39.113 INFO:tasks.rados.rados.0.smithi163.stdout:1914: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:13:39.113 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 739 (ObjNum 697 snap 227 seq_num 697) dirty exists 2023-10-09T09:13:39.113 INFO:tasks.rados.rados.0.smithi163.stdout:1915: expect (ObjNum 699 snap 228 seq_num 699) 2023-10-09T09:13:39.242 INFO:tasks.rados.rados.0.smithi163.stdout:1917: expect (ObjNum 675 snap 212 seq_num 675) 2023-10-09T09:13:39.352 INFO:tasks.rados.rados.0.smithi163.stdout:1914: done (3 left) 2023-10-09T09:13:39.352 INFO:tasks.rados.rados.0.smithi163.stdout:1915: done (2 left) 2023-10-09T09:13:39.352 INFO:tasks.rados.rados.0.smithi163.stdout:1917: done (1 left) 2023-10-09T09:13:39.352 INFO:tasks.rados.rados.0.smithi163.stdout:1919: done (0 left) 2023-10-09T09:13:39.352 INFO:tasks.rados.rados.0.smithi163.stdout:1920: rollback oid 47 current snap is 245 2023-10-09T09:13:39.352 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 47 to 232 2023-10-09T09:13:39.352 INFO:tasks.rados.rados.0.smithi163.stdout:1921: write oid 48 current snap is 245 2023-10-09T09:13:39.353 INFO:tasks.rados.rados.0.smithi163.stdout:1921: seq_num 728 ranges {635496=634807,1717597=517899} 2023-10-09T09:13:39.363 INFO:tasks.rados.rados.0.smithi163.stdout:1921: writing smithi16367185-48 from 635496 to 1270303 tid 1 2023-10-09T09:13:39.369 INFO:tasks.rados.rados.0.smithi163.stdout:1921: writing smithi16367185-48 from 1717597 to 2235496 tid 2 2023-10-09T09:13:39.369 INFO:tasks.rados.rados.0.smithi163.stdout:1922: delete oid 35 current snap is 245 2023-10-09T09:13:39.370 INFO:tasks.rados.rados.0.smithi163.stdout:1920: finishing rollback tid 0 to smithi16367185-47 2023-10-09T09:13:39.370 INFO:tasks.rados.rados.0.smithi163.stdout:1920: finishing rollback tid 1 to smithi16367185-47 2023-10-09T09:13:39.370 INFO:tasks.rados.rados.0.smithi163.stdout:1920: finishing rollback tid 2 to smithi16367185-47 2023-10-09T09:13:39.370 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 1396 (ObjNum 660 snap 209 seq_num 660) dirty exists 2023-10-09T09:13:39.374 INFO:tasks.rados.rados.0.smithi163.stdout:1921: finishing write tid 1 to smithi16367185-48 2023-10-09T09:13:39.374 INFO:tasks.rados.rados.0.smithi163.stdout:1920: done (2 left) 2023-10-09T09:13:39.374 INFO:tasks.rados.rados.0.smithi163.stdout:1922: done (1 left) 2023-10-09T09:13:39.374 INFO:tasks.rados.rados.0.smithi163.stdout:1923: delete oid 1 current snap is 245 2023-10-09T09:13:39.375 INFO:tasks.rados.rados.0.smithi163.stdout:1921: finishing write tid 2 to smithi16367185-48 2023-10-09T09:13:39.376 INFO:tasks.rados.rados.0.smithi163.stdout:1921: finishing write tid 3 to smithi16367185-48 2023-10-09T09:13:39.376 INFO:tasks.rados.rados.0.smithi163.stdout:1921: finishing write tid 4 to smithi16367185-48 2023-10-09T09:13:39.376 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 1400 (ObjNum 728 snap 245 seq_num 728) dirty exists 2023-10-09T09:13:39.376 INFO:tasks.rados.rados.0.smithi163.stdout:1921: left oid 48 (ObjNum 728 snap 245 seq_num 728) 2023-10-09T09:13:39.377 INFO:tasks.rados.rados.0.smithi163.stdout:1921: done (1 left) 2023-10-09T09:13:39.377 INFO:tasks.rados.rados.0.smithi163.stdout:1923: done (0 left) 2023-10-09T09:13:39.377 INFO:tasks.rados.rados.0.smithi163.stdout:1924: snap_create 2023-10-09T09:13:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[78107]: cluster 2023-10-09T09:13:38.102135+0000 osd.3 (osd.3) 159 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[78107]: cluster 2023-10-09T09:13:38.105834+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[78107]: cluster 2023-10-09T09:13:38.111976+0000 osd.0 (osd.0) 94 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[78107]: cluster 2023-10-09T09:13:38.172548+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[78107]: cluster 2023-10-09T09:13:39.109305+0000 osd.2 (osd.2) 158 : cluster 3 failed to encode map e547 with expected crc 2023-10-09T09:13:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[78107]: cluster 2023-10-09T09:13:39.112445+0000 mon.a (mon.0) 510 : cluster 0 osdmap e547: 8 total, 8 up, 8 in 2023-10-09T09:13:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[75425]: cluster 2023-10-09T09:13:38.102135+0000 osd.3 (osd.3) 159 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[75425]: cluster 2023-10-09T09:13:38.105834+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[75425]: cluster 2023-10-09T09:13:38.111976+0000 osd.0 (osd.0) 94 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[75425]: cluster 2023-10-09T09:13:38.172548+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[75425]: cluster 2023-10-09T09:13:39.109305+0000 osd.2 (osd.2) 158 : cluster 3 failed to encode map e547 with expected crc 2023-10-09T09:13:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:39 smithi163 bash[75425]: cluster 2023-10-09T09:13:39.112445+0000 mon.a (mon.0) 510 : cluster 0 osdmap e547: 8 total, 8 up, 8 in 2023-10-09T09:13:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:39 smithi181 bash[59345]: cluster 2023-10-09T09:13:38.102135+0000 osd.3 (osd.3) 159 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:39 smithi181 bash[59345]: cluster 2023-10-09T09:13:38.105834+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:39 smithi181 bash[59345]: cluster 2023-10-09T09:13:38.111976+0000 osd.0 (osd.0) 94 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:39 smithi181 bash[59345]: cluster 2023-10-09T09:13:38.172548+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e546 with expected crc 2023-10-09T09:13:39.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:39 smithi181 bash[59345]: cluster 2023-10-09T09:13:39.109305+0000 osd.2 (osd.2) 158 : cluster 3 failed to encode map e547 with expected crc 2023-10-09T09:13:39.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:39 smithi181 bash[59345]: cluster 2023-10-09T09:13:39.112445+0000 mon.a (mon.0) 510 : cluster 0 osdmap e547: 8 total, 8 up, 8 in 2023-10-09T09:13:40.075 INFO:tasks.rados.rados.0.smithi163.stdout:1924: done (0 left) 2023-10-09T09:13:40.075 INFO:tasks.rados.rados.0.smithi163.stdout:1925: read oid 34 snap -1 2023-10-09T09:13:40.075 INFO:tasks.rados.rados.0.smithi163.stdout:1925: expect (ObjNum 722 snap 245 seq_num 722) 2023-10-09T09:13:40.075 INFO:tasks.rados.rados.0.smithi163.stdout:1926: read oid 33 snap -1 2023-10-09T09:13:40.075 INFO:tasks.rados.rados.0.smithi163.stdout:1926: expect (ObjNum 634 snap 202 seq_num 634) 2023-10-09T09:13:40.075 INFO:tasks.rados.rados.0.smithi163.stdout:1927: rollback oid 12 current snap is 246 2023-10-09T09:13:40.075 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 12 to 220 2023-10-09T09:13:40.075 INFO:tasks.rados.rados.0.smithi163.stdout:1928: write oid 26 current snap is 246 2023-10-09T09:13:40.075 INFO:tasks.rados.rados.0.smithi163.stdout:1928: seq_num 731 ranges {685917=576907,1893043=392874} 2023-10-09T09:13:40.086 INFO:tasks.rados.rados.0.smithi163.stdout:1928: writing smithi16367185-26 from 685917 to 1262824 tid 1 2023-10-09T09:13:40.093 INFO:tasks.rados.rados.0.smithi163.stdout:1928: writing smithi16367185-26 from 1893043 to 2285917 tid 2 2023-10-09T09:13:40.093 INFO:tasks.rados.rados.0.smithi163.stdout:1929: read oid 48 snap -1 2023-10-09T09:13:40.093 INFO:tasks.rados.rados.0.smithi163.stdout:1929: expect (ObjNum 728 snap 245 seq_num 728) 2023-10-09T09:13:40.093 INFO:tasks.rados.rados.0.smithi163.stdout:1930: read oid 9 snap 237 2023-10-09T09:13:40.093 INFO:tasks.rados.rados.0.smithi163.stdout:1930: expect (ObjNum 700 snap 228 seq_num 700) 2023-10-09T09:13:40.093 INFO:tasks.rados.rados.0.smithi163.stdout:1931: read oid 20 snap 236 2023-10-09T09:13:40.094 INFO:tasks.rados.rados.0.smithi163.stdout:1931: expect (ObjNum 694 snap 221 seq_num 694) 2023-10-09T09:13:40.094 INFO:tasks.rados.rados.0.smithi163.stdout:1932: snap_remove snap 226 2023-10-09T09:13:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[75425]: cluster 2023-10-09T09:13:39.108770+0000 osd.1 (osd.1) 76 : cluster 3 failed to encode map e547 with expected crc 2023-10-09T09:13:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[75425]: cluster 2023-10-09T09:13:39.114127+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e547 with expected crc 2023-10-09T09:13:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[75425]: cluster 2023-10-09T09:13:39.300566+0000 mgr.y (mgr.34107) 189 : cluster 0 pgmap v373: 89 pgs: 89 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2023-10-09T09:13:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[75425]: cluster 2023-10-09T09:13:40.068986+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[75425]: cluster 2023-10-09T09:13:40.074023+0000 mon.a (mon.0) 511 : cluster 0 osdmap e548: 8 total, 8 up, 8 in 2023-10-09T09:13:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[75425]: cluster 2023-10-09T09:13:40.079938+0000 osd.2 (osd.2) 159 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[75425]: audit 2023-10-09T09:13:40.340596+0000 mon.a (mon.0) 512 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[78107]: cluster 2023-10-09T09:13:39.108770+0000 osd.1 (osd.1) 76 : cluster 3 failed to encode map e547 with expected crc 2023-10-09T09:13:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[78107]: cluster 2023-10-09T09:13:39.114127+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e547 with expected crc 2023-10-09T09:13:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[78107]: cluster 2023-10-09T09:13:39.300566+0000 mgr.y (mgr.34107) 189 : cluster 0 pgmap v373: 89 pgs: 89 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2023-10-09T09:13:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[78107]: cluster 2023-10-09T09:13:40.068986+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[78107]: cluster 2023-10-09T09:13:40.074023+0000 mon.a (mon.0) 511 : cluster 0 osdmap e548: 8 total, 8 up, 8 in 2023-10-09T09:13:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[78107]: cluster 2023-10-09T09:13:40.079938+0000 osd.2 (osd.2) 159 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:40.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:40 smithi163 bash[78107]: audit 2023-10-09T09:13:40.340596+0000 mon.a (mon.0) 512 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:40 smithi181 bash[59345]: cluster 2023-10-09T09:13:39.108770+0000 osd.1 (osd.1) 76 : cluster 3 failed to encode map e547 with expected crc 2023-10-09T09:13:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:40 smithi181 bash[59345]: cluster 2023-10-09T09:13:39.114127+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e547 with expected crc 2023-10-09T09:13:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:40 smithi181 bash[59345]: cluster 2023-10-09T09:13:39.300566+0000 mgr.y (mgr.34107) 189 : cluster 0 pgmap v373: 89 pgs: 89 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2023-10-09T09:13:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:40 smithi181 bash[59345]: cluster 2023-10-09T09:13:40.068986+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:40 smithi181 bash[59345]: cluster 2023-10-09T09:13:40.074023+0000 mon.a (mon.0) 511 : cluster 0 osdmap e548: 8 total, 8 up, 8 in 2023-10-09T09:13:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:40 smithi181 bash[59345]: cluster 2023-10-09T09:13:40.079938+0000 osd.2 (osd.2) 159 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:40 smithi181 bash[59345]: audit 2023-10-09T09:13:40.340596+0000 mon.a (mon.0) 512 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:41.086 INFO:tasks.rados.rados.0.smithi163.stdout:1932: done (7 left) 2023-10-09T09:13:41.086 INFO:tasks.rados.rados.0.smithi163.stdout:1933: delete oid 44 current snap is 246 2023-10-09T09:13:41.086 INFO:tasks.rados.rados.0.smithi163.stdout:1926: expect (ObjNum 634 snap 202 seq_num 634) 2023-10-09T09:13:41.221 INFO:tasks.rados.rados.0.smithi163.stdout:1925: expect (ObjNum 722 snap 245 seq_num 722) 2023-10-09T09:13:41.355 INFO:tasks.rados.rados.0.smithi163.stdout:1927: finishing rollback tid 0 to smithi16367185-12 2023-10-09T09:13:41.356 INFO:tasks.rados.rados.0.smithi163.stdout:1927: finishing rollback tid 1 to smithi16367185-12 2023-10-09T09:13:41.356 INFO:tasks.rados.rados.0.smithi163.stdout:1927: finishing rollback tid 2 to smithi16367185-12 2023-10-09T09:13:41.356 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 868 (ObjNum 655 snap 208 seq_num 655) dirty exists 2023-10-09T09:13:41.356 INFO:tasks.rados.rados.0.smithi163.stdout:1928: finishing write tid 1 to smithi16367185-26 2023-10-09T09:13:41.356 INFO:tasks.rados.rados.0.smithi163.stdout:1925: done (7 left) 2023-10-09T09:13:41.356 INFO:tasks.rados.rados.0.smithi163.stdout:1926: done (6 left) 2023-10-09T09:13:41.356 INFO:tasks.rados.rados.0.smithi163.stdout:1927: done (5 left) 2023-10-09T09:13:41.357 INFO:tasks.rados.rados.0.smithi163.stdout:1933: done (4 left) 2023-10-09T09:13:41.357 INFO:tasks.rados.rados.0.smithi163.stdout:1934: write oid 43 current snap is 246 2023-10-09T09:13:41.357 INFO:tasks.rados.rados.0.smithi163.stdout:1928: finishing write tid 2 to smithi16367185-26 2023-10-09T09:13:41.357 INFO:tasks.rados.rados.0.smithi163.stdout:1928: finishing write tid 3 to smithi16367185-26 2023-10-09T09:13:41.357 INFO:tasks.rados.rados.0.smithi163.stdout:1928: finishing write tid 4 to smithi16367185-26 2023-10-09T09:13:41.357 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 583 (ObjNum 731 snap 246 seq_num 731) dirty exists 2023-10-09T09:13:41.357 INFO:tasks.rados.rados.0.smithi163.stdout:1928: left oid 26 (ObjNum 731 snap 246 seq_num 731) 2023-10-09T09:13:41.357 INFO:tasks.rados.rados.0.smithi163.stdout:1929: expect (ObjNum 728 snap 245 seq_num 728) 2023-10-09T09:13:41.451 INFO:tasks.rados.rados.0.smithi163.stdout:1930: expect (ObjNum 700 snap 228 seq_num 700) 2023-10-09T09:13:41.575 INFO:tasks.rados.rados.0.smithi163.stdout:1931: expect (ObjNum 694 snap 221 seq_num 694) 2023-10-09T09:13:41.692 INFO:tasks.rados.rados.0.smithi163.stdout:1934: seq_num 733 ranges {719531=727405,1893654=425877} 2023-10-09T09:13:41.705 INFO:tasks.rados.rados.0.smithi163.stdout:1934: writing smithi16367185-43 from 719531 to 1446936 tid 1 2023-10-09T09:13:41.711 INFO:tasks.rados.rados.0.smithi163.stdout:1934: writing smithi16367185-43 from 1893654 to 2319531 tid 2 2023-10-09T09:13:41.711 INFO:tasks.rados.rados.0.smithi163.stdout:1928: done (4 left) 2023-10-09T09:13:41.711 INFO:tasks.rados.rados.0.smithi163.stdout:1929: done (3 left) 2023-10-09T09:13:41.711 INFO:tasks.rados.rados.0.smithi163.stdout:1930: done (2 left) 2023-10-09T09:13:41.711 INFO:tasks.rados.rados.0.smithi163.stdout:1931: done (1 left) 2023-10-09T09:13:41.711 INFO:tasks.rados.rados.0.smithi163.stdout:1935: read oid 39 snap -1 2023-10-09T09:13:41.711 INFO:tasks.rados.rados.0.smithi163.stdout:1935: expect (ObjNum 711 snap 240 seq_num 711) 2023-10-09T09:13:41.711 INFO:tasks.rados.rados.0.smithi163.stdout:1936: snap_create 2023-10-09T09:13:41.716 INFO:tasks.rados.rados.0.smithi163.stdout:1934: finishing write tid 1 to smithi16367185-43 2023-10-09T09:13:41.717 INFO:tasks.rados.rados.0.smithi163.stdout:1934: finishing write tid 2 to smithi16367185-43 2023-10-09T09:13:41.717 INFO:tasks.rados.rados.0.smithi163.stdout:1934: finishing write tid 3 to smithi16367185-43 2023-10-09T09:13:41.718 INFO:tasks.rados.rados.0.smithi163.stdout:1934: finishing write tid 4 to smithi16367185-43 2023-10-09T09:13:41.718 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 753 (ObjNum 733 snap 246 seq_num 733) dirty exists 2023-10-09T09:13:41.718 INFO:tasks.rados.rados.0.smithi163.stdout:1934: left oid 43 (ObjNum 733 snap 246 seq_num 733) 2023-10-09T09:13:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:41 smithi163 bash[75425]: cluster 2023-10-09T09:13:40.068988+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:41 smithi163 bash[75425]: cluster 2023-10-09T09:13:40.094064+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:41 smithi163 bash[75425]: cluster 2023-10-09T09:13:40.095843+0000 osd.3 (osd.3) 160 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:41 smithi163 bash[75425]: cluster 2023-10-09T09:13:41.085781+0000 mon.a (mon.0) 513 : cluster 0 osdmap e549: 8 total, 8 up, 8 in 2023-10-09T09:13:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:41 smithi163 bash[78107]: cluster 2023-10-09T09:13:40.068988+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:41 smithi163 bash[78107]: cluster 2023-10-09T09:13:40.094064+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:41 smithi163 bash[78107]: cluster 2023-10-09T09:13:40.095843+0000 osd.3 (osd.3) 160 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:41 smithi163 bash[78107]: cluster 2023-10-09T09:13:41.085781+0000 mon.a (mon.0) 513 : cluster 0 osdmap e549: 8 total, 8 up, 8 in 2023-10-09T09:13:41.730 INFO:tasks.rados.rados.0.smithi163.stdout:1935: expect (ObjNum 711 snap 240 seq_num 711) 2023-10-09T09:13:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:41 smithi181 bash[59345]: cluster 2023-10-09T09:13:40.068988+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:41 smithi181 bash[59345]: cluster 2023-10-09T09:13:40.094064+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:41 smithi181 bash[59345]: cluster 2023-10-09T09:13:40.095843+0000 osd.3 (osd.3) 160 : cluster 3 failed to encode map e548 with expected crc 2023-10-09T09:13:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:41 smithi181 bash[59345]: cluster 2023-10-09T09:13:41.085781+0000 mon.a (mon.0) 513 : cluster 0 osdmap e549: 8 total, 8 up, 8 in 2023-10-09T09:13:42.361 INFO:tasks.rados.rados.0.smithi163.stdout:1934: done (2 left) 2023-10-09T09:13:42.361 INFO:tasks.rados.rados.0.smithi163.stdout:1935: done (1 left) 2023-10-09T09:13:42.361 INFO:tasks.rados.rados.0.smithi163.stdout:1936: done (0 left) 2023-10-09T09:13:42.361 INFO:tasks.rados.rados.0.smithi163.stdout:1937: read oid 49 snap -1 2023-10-09T09:13:42.361 INFO:tasks.rados.rados.0.smithi163.stdout:1937: expect (ObjNum 636 snap 202 seq_num 636) 2023-10-09T09:13:42.361 INFO:tasks.rados.rados.0.smithi163.stdout:1938: snap_create 2023-10-09T09:13:42.380 INFO:tasks.rados.rados.0.smithi163.stdout:1937: expect (ObjNum 636 snap 202 seq_num 636) 2023-10-09T09:13:42.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:42 smithi163 bash[78107]: cluster 2023-10-09T09:13:41.080093+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e549 with expected crc 2023-10-09T09:13:42.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:42 smithi163 bash[78107]: cluster 2023-10-09T09:13:41.082440+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e549 with expected crc 2023-10-09T09:13:42.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:42 smithi163 bash[78107]: cluster 2023-10-09T09:13:41.301257+0000 mgr.y (mgr.34107) 190 : cluster 0 pgmap v376: 89 pgs: 89 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 518 KiB/s wr, 19 op/s 2023-10-09T09:13:42.507 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:42 smithi163 bash[75425]: cluster 2023-10-09T09:13:41.080093+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e549 with expected crc 2023-10-09T09:13:42.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:42 smithi163 bash[75425]: cluster 2023-10-09T09:13:41.082440+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e549 with expected crc 2023-10-09T09:13:42.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:42 smithi163 bash[75425]: cluster 2023-10-09T09:13:41.301257+0000 mgr.y (mgr.34107) 190 : cluster 0 pgmap v376: 89 pgs: 89 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 518 KiB/s wr, 19 op/s 2023-10-09T09:13:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:42 smithi181 bash[59345]: cluster 2023-10-09T09:13:41.080093+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e549 with expected crc 2023-10-09T09:13:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:42 smithi181 bash[59345]: cluster 2023-10-09T09:13:41.082440+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e549 with expected crc 2023-10-09T09:13:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:42 smithi181 bash[59345]: cluster 2023-10-09T09:13:41.301257+0000 mgr.y (mgr.34107) 190 : cluster 0 pgmap v376: 89 pgs: 89 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 518 KiB/s wr, 19 op/s 2023-10-09T09:13:43.126 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T09:13:43.363 INFO:tasks.rados.rados.0.smithi163.stdout:1937: done (1 left) 2023-10-09T09:13:43.363 INFO:tasks.rados.rados.0.smithi163.stdout:1938: done (0 left) 2023-10-09T09:13:43.364 INFO:tasks.rados.rados.0.smithi163.stdout:1939: rollback oid 34 current snap is 248 2023-10-09T09:13:43.364 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 34 to 220 2023-10-09T09:13:43.364 INFO:tasks.rados.rados.0.smithi163.stdout:1940: snap_create 2023-10-09T09:13:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:1939: finishing rollback tid 0 to smithi16367185-34 2023-10-09T09:13:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:1939: finishing rollback tid 1 to smithi16367185-34 2023-10-09T09:13:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:1939: finishing rollback tid 2 to smithi16367185-34 2023-10-09T09:13:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 768 (ObjNum 673 snap 212 seq_num 673) dirty exists 2023-10-09T09:13:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:13:43] "GET /metrics HTTP/1.1" 200 36077 "" "Prometheus/2.33.4" 2023-10-09T09:13:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[75425]: cluster 2023-10-09T09:13:42.356717+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e550 with expected crc 2023-10-09T09:13:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[75425]: cluster 2023-10-09T09:13:42.360167+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e550 with expected crc 2023-10-09T09:13:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[75425]: cluster 2023-10-09T09:13:42.360396+0000 mon.a (mon.0) 514 : cluster 0 osdmap e550: 8 total, 8 up, 8 in 2023-10-09T09:13:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[75425]: cluster 2023-10-09T09:13:42.365572+0000 osd.0 (osd.0) 95 : cluster 3 failed to encode map e550 with expected crc 2023-10-09T09:13:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[75425]: cluster 2023-10-09T09:13:43.363204+0000 mon.a (mon.0) 515 : cluster 0 osdmap e551: 8 total, 8 up, 8 in 2023-10-09T09:13:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[78107]: cluster 2023-10-09T09:13:42.356717+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e550 with expected crc 2023-10-09T09:13:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[78107]: cluster 2023-10-09T09:13:42.360167+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e550 with expected crc 2023-10-09T09:13:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[78107]: cluster 2023-10-09T09:13:42.360396+0000 mon.a (mon.0) 514 : cluster 0 osdmap e550: 8 total, 8 up, 8 in 2023-10-09T09:13:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[78107]: cluster 2023-10-09T09:13:42.365572+0000 osd.0 (osd.0) 95 : cluster 3 failed to encode map e550 with expected crc 2023-10-09T09:13:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:43 smithi163 bash[78107]: cluster 2023-10-09T09:13:43.363204+0000 mon.a (mon.0) 515 : cluster 0 osdmap e551: 8 total, 8 up, 8 in 2023-10-09T09:13:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:43 smithi181 bash[59345]: cluster 2023-10-09T09:13:42.356717+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e550 with expected crc 2023-10-09T09:13:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:43 smithi181 bash[59345]: cluster 2023-10-09T09:13:42.360167+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e550 with expected crc 2023-10-09T09:13:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:43 smithi181 bash[59345]: cluster 2023-10-09T09:13:42.360396+0000 mon.a (mon.0) 514 : cluster 0 osdmap e550: 8 total, 8 up, 8 in 2023-10-09T09:13:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:43 smithi181 bash[59345]: cluster 2023-10-09T09:13:42.365572+0000 osd.0 (osd.0) 95 : cluster 3 failed to encode map e550 with expected crc 2023-10-09T09:13:43.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:43 smithi181 bash[59345]: cluster 2023-10-09T09:13:43.363204+0000 mon.a (mon.0) 515 : cluster 0 osdmap e551: 8 total, 8 up, 8 in 2023-10-09T09:13:44.369 INFO:tasks.rados.rados.0.smithi163.stdout:1939: done (1 left) 2023-10-09T09:13:44.369 INFO:tasks.rados.rados.0.smithi163.stdout:1940: done (0 left) 2023-10-09T09:13:44.369 INFO:tasks.rados.rados.0.smithi163.stdout:1941: read oid 37 snap -1 2023-10-09T09:13:44.370 INFO:tasks.rados.rados.0.smithi163.stdout:1941: expect (ObjNum 684 snap 218 seq_num 684) 2023-10-09T09:13:44.370 INFO:tasks.rados.rados.0.smithi163.stdout:1942: write oid 18 current snap is 249 2023-10-09T09:13:44.370 INFO:tasks.rados.rados.0.smithi163.stdout:1942: seq_num 734 ranges {736338=402654,1552136=539580} 2023-10-09T09:13:44.378 INFO:tasks.rados.rados.0.smithi163.stdout:1942: writing smithi16367185-18 from 736338 to 1138992 tid 1 2023-10-09T09:13:44.384 INFO:tasks.rados.rados.0.smithi163.stdout:1942: writing smithi16367185-18 from 1552136 to 2091716 tid 2 2023-10-09T09:13:44.384 INFO:tasks.rados.rados.0.smithi163.stdout:1943: delete oid 29 current snap is 249 2023-10-09T09:13:44.393 INFO:tasks.rados.rados.0.smithi163.stdout:1941: expect (ObjNum 684 snap 218 seq_num 684) 2023-10-09T09:13:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:44 smithi163 bash[75425]: cluster 2023-10-09T09:13:43.301670+0000 mgr.y (mgr.34107) 191 : cluster 0 pgmap v378: 89 pgs: 89 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 495 KiB/s wr, 18 op/s 2023-10-09T09:13:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:44 smithi163 bash[75425]: cluster 2023-10-09T09:13:43.361246+0000 osd.3 (osd.3) 161 : cluster 3 failed to encode map e551 with expected crc 2023-10-09T09:13:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:44 smithi163 bash[75425]: cluster 2023-10-09T09:13:43.363822+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e551 with expected crc 2023-10-09T09:13:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:44 smithi163 bash[75425]: cluster 2023-10-09T09:13:43.364590+0000 osd.2 (osd.2) 160 : cluster 3 failed to encode map e551 with expected crc 2023-10-09T09:13:44.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:44 smithi163 bash[75425]: cluster 2023-10-09T09:13:44.369167+0000 mon.a (mon.0) 516 : cluster 0 osdmap e552: 8 total, 8 up, 8 in 2023-10-09T09:13:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:44 smithi163 bash[78107]: cluster 2023-10-09T09:13:43.301670+0000 mgr.y (mgr.34107) 191 : cluster 0 pgmap v378: 89 pgs: 89 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 495 KiB/s wr, 18 op/s 2023-10-09T09:13:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:44 smithi163 bash[78107]: cluster 2023-10-09T09:13:43.361246+0000 osd.3 (osd.3) 161 : cluster 3 failed to encode map e551 with expected crc 2023-10-09T09:13:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:44 smithi163 bash[78107]: cluster 2023-10-09T09:13:43.363822+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e551 with expected crc 2023-10-09T09:13:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:44 smithi163 bash[78107]: cluster 2023-10-09T09:13:43.364590+0000 osd.2 (osd.2) 160 : cluster 3 failed to encode map e551 with expected crc 2023-10-09T09:13:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:44 smithi163 bash[78107]: cluster 2023-10-09T09:13:44.369167+0000 mon.a (mon.0) 516 : cluster 0 osdmap e552: 8 total, 8 up, 8 in 2023-10-09T09:13:44.563 INFO:tasks.rados.rados.0.smithi163.stdout:1942: finishing write tid 1 to smithi16367185-18 2023-10-09T09:13:44.563 INFO:tasks.rados.rados.0.smithi163.stdout:1942: finishing write tid 2 to smithi16367185-18 2023-10-09T09:13:44.563 INFO:tasks.rados.rados.0.smithi163.stdout:1942: finishing write tid 3 to smithi16367185-18 2023-10-09T09:13:44.563 INFO:tasks.rados.rados.0.smithi163.stdout:1942: finishing write tid 4 to smithi16367185-18 2023-10-09T09:13:44.563 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 671 (ObjNum 734 snap 249 seq_num 734) dirty exists 2023-10-09T09:13:44.563 INFO:tasks.rados.rados.0.smithi163.stdout:1942: left oid 18 (ObjNum 734 snap 249 seq_num 734) 2023-10-09T09:13:44.563 INFO:tasks.rados.rados.0.smithi163.stdout:1941: done (2 left) 2023-10-09T09:13:44.564 INFO:tasks.rados.rados.0.smithi163.stdout:1942: done (1 left) 2023-10-09T09:13:44.564 INFO:tasks.rados.rados.0.smithi163.stdout:1943: done (0 left) 2023-10-09T09:13:44.564 INFO:tasks.rados.rados.0.smithi163.stdout:1944: read oid 12 snap -1 2023-10-09T09:13:44.564 INFO:tasks.rados.rados.0.smithi163.stdout:1944: expect (ObjNum 655 snap 208 seq_num 655) 2023-10-09T09:13:44.564 INFO:tasks.rados.rados.0.smithi163.stdout:1945: read oid 17 snap 237 2023-10-09T09:13:44.564 INFO:tasks.rados.rados.0.smithi163.stdout:1945: expect (ObjNum 704 snap 228 seq_num 704) 2023-10-09T09:13:44.564 INFO:tasks.rados.rados.0.smithi163.stdout:1946: read oid 23 snap -1 2023-10-09T09:13:44.564 INFO:tasks.rados.rados.0.smithi163.stdout:1946: expect (ObjNum 659 snap 209 seq_num 659) 2023-10-09T09:13:44.564 INFO:tasks.rados.rados.0.smithi163.stdout:1947: snap_create 2023-10-09T09:13:44.570 INFO:tasks.rados.rados.0.smithi163.stdout:1944: expect (ObjNum 655 snap 208 seq_num 655) 2023-10-09T09:13:44.712 INFO:tasks.rados.rados.0.smithi163.stdout:1946: expect (ObjNum 659 snap 209 seq_num 659) 2023-10-09T09:13:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:44 smithi181 bash[59345]: cluster 2023-10-09T09:13:43.301670+0000 mgr.y (mgr.34107) 191 : cluster 0 pgmap v378: 89 pgs: 89 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 495 KiB/s wr, 18 op/s 2023-10-09T09:13:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:44 smithi181 bash[59345]: cluster 2023-10-09T09:13:43.361246+0000 osd.3 (osd.3) 161 : cluster 3 failed to encode map e551 with expected crc 2023-10-09T09:13:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:44 smithi181 bash[59345]: cluster 2023-10-09T09:13:43.363822+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e551 with expected crc 2023-10-09T09:13:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:44 smithi181 bash[59345]: cluster 2023-10-09T09:13:43.364590+0000 osd.2 (osd.2) 160 : cluster 3 failed to encode map e551 with expected crc 2023-10-09T09:13:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:44 smithi181 bash[59345]: cluster 2023-10-09T09:13:44.369167+0000 mon.a (mon.0) 516 : cluster 0 osdmap e552: 8 total, 8 up, 8 in 2023-10-09T09:13:44.930 INFO:tasks.rados.rados.0.smithi163.stdout:1945: expect (ObjNum 704 snap 228 seq_num 704) 2023-10-09T09:13:45.075 INFO:tasks.rados.rados.0.smithi163.stdout:1944: done (3 left) 2023-10-09T09:13:45.076 INFO:tasks.rados.rados.0.smithi163.stdout:1945: done (2 left) 2023-10-09T09:13:45.076 INFO:tasks.rados.rados.0.smithi163.stdout:1946: done (1 left) 2023-10-09T09:13:45.076 INFO:tasks.rados.rados.0.smithi163.stdout:1947: done (0 left) 2023-10-09T09:13:45.076 INFO:tasks.rados.rados.0.smithi163.stdout:1948: snap_remove snap 230 2023-10-09T09:13:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:45 smithi163 bash[75425]: cluster 2023-10-09T09:13:44.364387+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:45 smithi163 bash[75425]: cluster 2023-10-09T09:13:44.366034+0000 osd.3 (osd.3) 162 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:45 smithi163 bash[75425]: cluster 2023-10-09T09:13:44.369955+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:45 smithi163 bash[75425]: cluster 2023-10-09T09:13:44.380200+0000 osd.2 (osd.2) 161 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:45 smithi163 bash[75425]: cluster 2023-10-09T09:13:45.075480+0000 mon.a (mon.0) 517 : cluster 0 osdmap e553: 8 total, 8 up, 8 in 2023-10-09T09:13:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:45 smithi163 bash[78107]: cluster 2023-10-09T09:13:44.364387+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:45 smithi163 bash[78107]: cluster 2023-10-09T09:13:44.366034+0000 osd.3 (osd.3) 162 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:45 smithi163 bash[78107]: cluster 2023-10-09T09:13:44.369955+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:45 smithi163 bash[78107]: cluster 2023-10-09T09:13:44.380200+0000 osd.2 (osd.2) 161 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:45 smithi163 bash[78107]: cluster 2023-10-09T09:13:45.075480+0000 mon.a (mon.0) 517 : cluster 0 osdmap e553: 8 total, 8 up, 8 in 2023-10-09T09:13:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:45 smithi181 bash[59345]: cluster 2023-10-09T09:13:44.364387+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:45 smithi181 bash[59345]: cluster 2023-10-09T09:13:44.366034+0000 osd.3 (osd.3) 162 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:45 smithi181 bash[59345]: cluster 2023-10-09T09:13:44.369955+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:45 smithi181 bash[59345]: cluster 2023-10-09T09:13:44.380200+0000 osd.2 (osd.2) 161 : cluster 3 failed to encode map e552 with expected crc 2023-10-09T09:13:45.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:45 smithi181 bash[59345]: cluster 2023-10-09T09:13:45.075480+0000 mon.a (mon.0) 517 : cluster 0 osdmap e553: 8 total, 8 up, 8 in 2023-10-09T09:13:46.077 INFO:tasks.rados.rados.0.smithi163.stdout:1948: done (0 left) 2023-10-09T09:13:46.077 INFO:tasks.rados.rados.0.smithi163.stdout:1949: read oid 43 snap 245 2023-10-09T09:13:46.077 INFO:tasks.rados.rados.0.smithi163.stdout:1949: expect (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:13:46.077 INFO:tasks.rados.rados.0.smithi163.stdout:1950: delete oid 11 current snap is 250 2023-10-09T09:13:46.092 INFO:tasks.rados.rados.0.smithi163.stdout:1950: done (1 left) 2023-10-09T09:13:46.092 INFO:tasks.rados.rados.0.smithi163.stdout:1951: read oid 21 snap 235 2023-10-09T09:13:46.092 INFO:tasks.rados.rados.0.smithi163.stdout:1951: expect (ObjNum 707 snap 233 seq_num 707) 2023-10-09T09:13:46.092 INFO:tasks.rados.rados.0.smithi163.stdout:1952: snap_remove snap 247 2023-10-09T09:13:46.321 INFO:teuthology.orchestra.run.smithi163.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:13:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[75425]: cluster 2023-10-09T09:13:45.070491+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[75425]: cluster 2023-10-09T09:13:45.071328+0000 osd.1 (osd.1) 77 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[75425]: cluster 2023-10-09T09:13:45.275327+0000 osd.0 (osd.0) 96 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[75425]: cluster 2023-10-09T09:13:45.302234+0000 mgr.y (mgr.34107) 192 : cluster 0 pgmap v382: 89 pgs: 89 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[75425]: cluster 2023-10-09T09:13:45.373115+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[75425]: cluster 2023-10-09T09:13:46.074571+0000 osd.2 (osd.2) 162 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[75425]: cluster 2023-10-09T09:13:46.076729+0000 mon.a (mon.0) 518 : cluster 0 osdmap e554: 8 total, 8 up, 8 in 2023-10-09T09:13:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[75425]: cluster 2023-10-09T09:13:46.077480+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[75425]: audit 2023-10-09T09:13:46.321376+0000 mon.a (mon.0) 519 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[75425]: audit 2023-10-09T09:13:46.368793+0000 mon.a (mon.0) 520 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[78107]: cluster 2023-10-09T09:13:45.070491+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[78107]: cluster 2023-10-09T09:13:45.071328+0000 osd.1 (osd.1) 77 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[78107]: cluster 2023-10-09T09:13:45.275327+0000 osd.0 (osd.0) 96 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[78107]: cluster 2023-10-09T09:13:45.302234+0000 mgr.y (mgr.34107) 192 : cluster 0 pgmap v382: 89 pgs: 89 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[78107]: cluster 2023-10-09T09:13:45.373115+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[78107]: cluster 2023-10-09T09:13:46.074571+0000 osd.2 (osd.2) 162 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[78107]: cluster 2023-10-09T09:13:46.076729+0000 mon.a (mon.0) 518 : cluster 0 osdmap e554: 8 total, 8 up, 8 in 2023-10-09T09:13:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[78107]: cluster 2023-10-09T09:13:46.077480+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[78107]: audit 2023-10-09T09:13:46.321376+0000 mon.a (mon.0) 519 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:46 smithi163 bash[78107]: audit 2023-10-09T09:13:46.368793+0000 mon.a (mon.0) 520 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:46 smithi181 bash[59345]: cluster 2023-10-09T09:13:45.070491+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:46 smithi181 bash[59345]: cluster 2023-10-09T09:13:45.071328+0000 osd.1 (osd.1) 77 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:46 smithi181 bash[59345]: cluster 2023-10-09T09:13:45.275327+0000 osd.0 (osd.0) 96 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:46 smithi181 bash[59345]: cluster 2023-10-09T09:13:45.302234+0000 mgr.y (mgr.34107) 192 : cluster 0 pgmap v382: 89 pgs: 89 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:46 smithi181 bash[59345]: cluster 2023-10-09T09:13:45.373115+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e553 with expected crc 2023-10-09T09:13:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:46 smithi181 bash[59345]: cluster 2023-10-09T09:13:46.074571+0000 osd.2 (osd.2) 162 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:46 smithi181 bash[59345]: cluster 2023-10-09T09:13:46.076729+0000 mon.a (mon.0) 518 : cluster 0 osdmap e554: 8 total, 8 up, 8 in 2023-10-09T09:13:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:46 smithi181 bash[59345]: cluster 2023-10-09T09:13:46.077480+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:46 smithi181 bash[59345]: audit 2023-10-09T09:13:46.321376+0000 mon.a (mon.0) 519 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:46 smithi181 bash[59345]: audit 2023-10-09T09:13:46.368793+0000 mon.a (mon.0) 520 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:47.065 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T09:13:47.090 INFO:tasks.rados.rados.0.smithi163.stdout:1952: done (2 left) 2023-10-09T09:13:47.090 INFO:tasks.rados.rados.0.smithi163.stdout:1953: snap_remove snap 238 2023-10-09T09:13:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[75425]: cluster 2023-10-09T09:13:46.074260+0000 osd.3 (osd.3) 163 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[75425]: cluster 2023-10-09T09:13:46.077494+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[75425]: audit 2023-10-09T09:13:46.313979+0000 mgr.y (mgr.34107) 193 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:13:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[75425]: cephadm 2023-10-09T09:13:46.314377+0000 mgr.y (mgr.34107) 194 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:13:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[75425]: cluster 2023-10-09T09:13:47.085909+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[75425]: cluster 2023-10-09T09:13:47.090087+0000 mon.a (mon.0) 521 : cluster 0 osdmap e555: 8 total, 8 up, 8 in 2023-10-09T09:13:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[78107]: cluster 2023-10-09T09:13:46.074260+0000 osd.3 (osd.3) 163 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[78107]: cluster 2023-10-09T09:13:46.077494+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[78107]: audit 2023-10-09T09:13:46.313979+0000 mgr.y (mgr.34107) 193 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:13:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[78107]: cephadm 2023-10-09T09:13:46.314377+0000 mgr.y (mgr.34107) 194 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:13:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[78107]: cluster 2023-10-09T09:13:47.085909+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:47 smithi163 bash[78107]: cluster 2023-10-09T09:13:47.090087+0000 mon.a (mon.0) 521 : cluster 0 osdmap e555: 8 total, 8 up, 8 in 2023-10-09T09:13:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:47 smithi181 bash[59345]: cluster 2023-10-09T09:13:46.074260+0000 osd.3 (osd.3) 163 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:47 smithi181 bash[59345]: cluster 2023-10-09T09:13:46.077494+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e554 with expected crc 2023-10-09T09:13:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:47 smithi181 bash[59345]: audit 2023-10-09T09:13:46.313979+0000 mgr.y (mgr.34107) 193 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:13:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:47 smithi181 bash[59345]: cephadm 2023-10-09T09:13:46.314377+0000 mgr.y (mgr.34107) 194 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:13:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:47 smithi181 bash[59345]: cluster 2023-10-09T09:13:47.085909+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:47 smithi181 bash[59345]: cluster 2023-10-09T09:13:47.090087+0000 mon.a (mon.0) 521 : cluster 0 osdmap e555: 8 total, 8 up, 8 in 2023-10-09T09:13:48.092 INFO:tasks.rados.rados.0.smithi163.stdout:1953: done (2 left) 2023-10-09T09:13:48.093 INFO:tasks.rados.rados.0.smithi163.stdout:1954: write oid 24 current snap is 250 2023-10-09T09:13:48.093 INFO:tasks.rados.rados.0.smithi163.stdout:1954: seq_num 737 ranges {786759=628401,2011229=375530} 2023-10-09T09:13:48.100 INFO:tasks.rados.rados.0.smithi163.stdout:1954: writing smithi16367185-24 from 786759 to 1415160 tid 1 2023-10-09T09:13:48.106 INFO:tasks.rados.rados.0.smithi163.stdout:1954: writing smithi16367185-24 from 2011229 to 2386759 tid 2 2023-10-09T09:13:48.107 INFO:tasks.rados.rados.0.smithi163.stdout:1955: write oid 29 current snap is 250 2023-10-09T09:13:48.107 INFO:tasks.rados.rados.0.smithi163.stdout:1955: seq_num 738 ranges {403566=420003,1386064=546918} 2023-10-09T09:13:48.112 INFO:tasks.rados.rados.0.smithi163.stdout:1955: writing smithi16367185-29 from 403566 to 823569 tid 1 2023-10-09T09:13:48.119 INFO:tasks.rados.rados.0.smithi163.stdout:1955: writing smithi16367185-29 from 1386064 to 1932982 tid 2 2023-10-09T09:13:48.119 INFO:tasks.rados.rados.0.smithi163.stdout:1956: snap_remove snap 231 2023-10-09T09:13:48.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[75425]: cluster 2023-10-09T09:13:47.086544+0000 osd.1 (osd.1) 78 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[75425]: cluster 2023-10-09T09:13:47.148330+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[75425]: cluster 2023-10-09T09:13:47.272589+0000 osd.0 (osd.0) 97 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[75425]: cluster 2023-10-09T09:13:47.283660+0000 osd.3 (osd.3) 164 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[75425]: cluster 2023-10-09T09:13:47.302697+0000 mgr.y (mgr.34107) 195 : cluster 0 pgmap v385: 89 pgs: 89 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 520 KiB/s wr, 21 op/s 2023-10-09T09:13:48.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[75425]: cluster 2023-10-09T09:13:48.087779+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:48.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[75425]: cluster 2023-10-09T09:13:48.092256+0000 mon.a (mon.0) 522 : cluster 0 osdmap e556: 8 total, 8 up, 8 in 2023-10-09T09:13:48.418 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[75425]: cluster 2023-10-09T09:13:48.114849+0000 osd.2 (osd.2) 163 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[78107]: cluster 2023-10-09T09:13:47.086544+0000 osd.1 (osd.1) 78 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[78107]: cluster 2023-10-09T09:13:47.148330+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[78107]: cluster 2023-10-09T09:13:47.272589+0000 osd.0 (osd.0) 97 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[78107]: cluster 2023-10-09T09:13:47.283660+0000 osd.3 (osd.3) 164 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[78107]: cluster 2023-10-09T09:13:47.302697+0000 mgr.y (mgr.34107) 195 : cluster 0 pgmap v385: 89 pgs: 89 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 520 KiB/s wr, 21 op/s 2023-10-09T09:13:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[78107]: cluster 2023-10-09T09:13:48.087779+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[78107]: cluster 2023-10-09T09:13:48.092256+0000 mon.a (mon.0) 522 : cluster 0 osdmap e556: 8 total, 8 up, 8 in 2023-10-09T09:13:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:48 smithi163 bash[78107]: cluster 2023-10-09T09:13:48.114849+0000 osd.2 (osd.2) 163 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:48 smithi181 bash[59345]: cluster 2023-10-09T09:13:47.086544+0000 osd.1 (osd.1) 78 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:48 smithi181 bash[59345]: cluster 2023-10-09T09:13:47.148330+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:48 smithi181 bash[59345]: cluster 2023-10-09T09:13:47.272589+0000 osd.0 (osd.0) 97 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:48 smithi181 bash[59345]: cluster 2023-10-09T09:13:47.283660+0000 osd.3 (osd.3) 164 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:48 smithi181 bash[59345]: cluster 2023-10-09T09:13:47.302697+0000 mgr.y (mgr.34107) 195 : cluster 0 pgmap v385: 89 pgs: 89 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 520 KiB/s wr, 21 op/s 2023-10-09T09:13:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:48 smithi181 bash[59345]: cluster 2023-10-09T09:13:48.087779+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:48 smithi181 bash[59345]: cluster 2023-10-09T09:13:48.092256+0000 mon.a (mon.0) 522 : cluster 0 osdmap e556: 8 total, 8 up, 8 in 2023-10-09T09:13:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:48 smithi181 bash[59345]: cluster 2023-10-09T09:13:48.114849+0000 osd.2 (osd.2) 163 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:49.103 INFO:tasks.rados.rados.0.smithi163.stdout:1956: done (4 left) 2023-10-09T09:13:49.103 INFO:tasks.rados.rados.0.smithi163.stdout:1957: read oid 40 snap 219 2023-10-09T09:13:49.103 INFO:tasks.rados.rados.0.smithi163.stdout:1957: expect deleted 2023-10-09T09:13:49.103 INFO:tasks.rados.rados.0.smithi163.stdout:1958: write oid 2 current snap is 250 2023-10-09T09:13:49.103 INFO:tasks.rados.rados.0.smithi163.stdout:1958: seq_num 739 ranges {420373=495252,1444546=690576} 2023-10-09T09:13:49.109 INFO:tasks.rados.rados.0.smithi163.stdout:1958: writing smithi16367185-2 from 420373 to 915625 tid 1 2023-10-09T09:13:49.118 INFO:tasks.rados.rados.0.smithi163.stdout:1958: writing smithi16367185-2 from 1444546 to 2135122 tid 2 2023-10-09T09:13:49.119 INFO:tasks.rados.rados.0.smithi163.stdout:1959: write oid 23 current snap is 250 2023-10-09T09:13:49.119 INFO:tasks.rados.rados.0.smithi163.stdout:1949: expect (ObjNum 610 snap 192 seq_num 610) 2023-10-09T09:13:49.230 INFO:tasks.rados.rados.0.smithi163.stdout:1951: expect (ObjNum 707 snap 233 seq_num 707) 2023-10-09T09:13:49.364 INFO:tasks.rados.rados.0.smithi163.stdout:1954: finishing write tid 1 to smithi16367185-24 2023-10-09T09:13:49.364 INFO:tasks.rados.rados.0.smithi163.stdout:1954: finishing write tid 2 to smithi16367185-24 2023-10-09T09:13:49.364 INFO:tasks.rados.rados.0.smithi163.stdout:1954: finishing write tid 3 to smithi16367185-24 2023-10-09T09:13:49.364 INFO:tasks.rados.rados.0.smithi163.stdout:1954: finishing write tid 4 to smithi16367185-24 2023-10-09T09:13:49.364 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 475 (ObjNum 737 snap 250 seq_num 737) dirty exists 2023-10-09T09:13:49.365 INFO:tasks.rados.rados.0.smithi163.stdout:1954: left oid 24 (ObjNum 737 snap 250 seq_num 737) 2023-10-09T09:13:49.365 INFO:tasks.rados.rados.0.smithi163.stdout:1955: finishing write tid 1 to smithi16367185-29 2023-10-09T09:13:49.365 INFO:tasks.rados.rados.0.smithi163.stdout:1955: finishing write tid 2 to smithi16367185-29 2023-10-09T09:13:49.365 INFO:tasks.rados.rados.0.smithi163.stdout:1955: finishing write tid 3 to smithi16367185-29 2023-10-09T09:13:49.365 INFO:tasks.rados.rados.0.smithi163.stdout:1955: finishing write tid 4 to smithi16367185-29 2023-10-09T09:13:49.365 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 891 (ObjNum 738 snap 250 seq_num 738) dirty exists 2023-10-09T09:13:49.366 INFO:tasks.rados.rados.0.smithi163.stdout:1955: left oid 29 (ObjNum 738 snap 250 seq_num 738) 2023-10-09T09:13:49.366 INFO:tasks.rados.rados.0.smithi163.stdout:1958: finishing write tid 1 to smithi16367185-2 2023-10-09T09:13:49.366 INFO:tasks.rados.rados.0.smithi163.stdout:1958: finishing write tid 2 to smithi16367185-2 2023-10-09T09:13:49.366 INFO:tasks.rados.rados.0.smithi163.stdout:1958: finishing write tid 3 to smithi16367185-2 2023-10-09T09:13:49.366 INFO:tasks.rados.rados.0.smithi163.stdout:1958: finishing write tid 4 to smithi16367185-2 2023-10-09T09:13:49.366 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 533 (ObjNum 739 snap 250 seq_num 739) dirty exists 2023-10-09T09:13:49.366 INFO:tasks.rados.rados.0.smithi163.stdout:1958: left oid 2 (ObjNum 739 snap 250 seq_num 739) 2023-10-09T09:13:49.367 INFO:tasks.rados.rados.0.smithi163.stdout:1959: seq_num 740 ranges {437180=570501,1503028=550587} 2023-10-09T09:13:49.376 INFO:tasks.rados.rados.0.smithi163.stdout:1959: writing smithi16367185-23 from 437180 to 1007681 tid 1 2023-10-09T09:13:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:1959: writing smithi16367185-23 from 1503028 to 2053615 tid 2 2023-10-09T09:13:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:1949: done (6 left) 2023-10-09T09:13:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:1951: done (5 left) 2023-10-09T09:13:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:1954: done (4 left) 2023-10-09T09:13:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:1955: done (3 left) 2023-10-09T09:13:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:1957: done (2 left) 2023-10-09T09:13:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:1958: done (1 left) 2023-10-09T09:13:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:1960: snap_create 2023-10-09T09:13:49.383 INFO:tasks.rados.rados.0.smithi163.stdout:1959: finishing write tid 1 to smithi16367185-23 2023-10-09T09:13:49.387 INFO:tasks.rados.rados.0.smithi163.stdout:1959: finishing write tid 2 to smithi16367185-23 2023-10-09T09:13:49.387 INFO:tasks.rados.rados.0.smithi163.stdout:1959: finishing write tid 3 to smithi16367185-23 2023-10-09T09:13:49.387 INFO:tasks.rados.rados.0.smithi163.stdout:1959: finishing write tid 4 to smithi16367185-23 2023-10-09T09:13:49.388 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 786 (ObjNum 740 snap 250 seq_num 740) dirty exists 2023-10-09T09:13:49.388 INFO:tasks.rados.rados.0.smithi163.stdout:1959: left oid 23 (ObjNum 740 snap 250 seq_num 740) 2023-10-09T09:13:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:49 smithi163 bash[75425]: cluster 2023-10-09T09:13:47.563180+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:49 smithi163 bash[75425]: cluster 2023-10-09T09:13:48.087121+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:49 smithi163 bash[75425]: cluster 2023-10-09T09:13:48.106883+0000 osd.3 (osd.3) 165 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:49 smithi163 bash[75425]: cluster 2023-10-09T09:13:49.102787+0000 mon.a (mon.0) 523 : cluster 0 osdmap e557: 8 total, 8 up, 8 in 2023-10-09T09:13:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:49 smithi163 bash[78107]: cluster 2023-10-09T09:13:47.563180+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:49 smithi163 bash[78107]: cluster 2023-10-09T09:13:48.087121+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:49 smithi163 bash[78107]: cluster 2023-10-09T09:13:48.106883+0000 osd.3 (osd.3) 165 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:49 smithi163 bash[78107]: cluster 2023-10-09T09:13:49.102787+0000 mon.a (mon.0) 523 : cluster 0 osdmap e557: 8 total, 8 up, 8 in 2023-10-09T09:13:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:49 smithi181 bash[59345]: cluster 2023-10-09T09:13:47.563180+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e555 with expected crc 2023-10-09T09:13:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:49 smithi181 bash[59345]: cluster 2023-10-09T09:13:48.087121+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:49 smithi181 bash[59345]: cluster 2023-10-09T09:13:48.106883+0000 osd.3 (osd.3) 165 : cluster 3 failed to encode map e556 with expected crc 2023-10-09T09:13:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:49 smithi181 bash[59345]: cluster 2023-10-09T09:13:49.102787+0000 mon.a (mon.0) 523 : cluster 0 osdmap e557: 8 total, 8 up, 8 in 2023-10-09T09:13:49.906 INFO:teuthology.orchestra.run.smithi163.stdout:wait for half of osds to upgrade 2023-10-09T09:13:50.078 INFO:tasks.rados.rados.0.smithi163.stdout:1959: done (1 left) 2023-10-09T09:13:50.078 INFO:tasks.rados.rados.0.smithi163.stdout:1960: done (0 left) 2023-10-09T09:13:50.078 INFO:tasks.rados.rados.0.smithi163.stdout:1961: snap_remove snap 220 2023-10-09T09:13:50.646 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T09:13:50.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:50 smithi163 bash[78107]: cluster 2023-10-09T09:13:49.097146+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e557 with expected crc 2023-10-09T09:13:50.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:50 smithi163 bash[78107]: cluster 2023-10-09T09:13:49.099048+0000 osd.3 (osd.3) 166 : cluster 3 failed to encode map e557 with expected crc 2023-10-09T09:13:50.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:50 smithi163 bash[78107]: cluster 2023-10-09T09:13:49.103166+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e557 with expected crc 2023-10-09T09:13:50.703 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:50 smithi163 bash[78107]: cluster 2023-10-09T09:13:49.303306+0000 mgr.y (mgr.34107) 196 : cluster 0 pgmap v388: 89 pgs: 89 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 512 KiB/s wr, 21 op/s 2023-10-09T09:13:50.703 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:50 smithi163 bash[78107]: cluster 2023-10-09T09:13:50.078244+0000 mon.a (mon.0) 524 : cluster 0 osdmap e558: 8 total, 8 up, 8 in 2023-10-09T09:13:50.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:50 smithi163 bash[75425]: cluster 2023-10-09T09:13:49.097146+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e557 with expected crc 2023-10-09T09:13:50.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:50 smithi163 bash[75425]: cluster 2023-10-09T09:13:49.099048+0000 osd.3 (osd.3) 166 : cluster 3 failed to encode map e557 with expected crc 2023-10-09T09:13:50.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:50 smithi163 bash[75425]: cluster 2023-10-09T09:13:49.103166+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e557 with expected crc 2023-10-09T09:13:50.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:50 smithi163 bash[75425]: cluster 2023-10-09T09:13:49.303306+0000 mgr.y (mgr.34107) 196 : cluster 0 pgmap v388: 89 pgs: 89 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 512 KiB/s wr, 21 op/s 2023-10-09T09:13:50.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:50 smithi163 bash[75425]: cluster 2023-10-09T09:13:50.078244+0000 mon.a (mon.0) 524 : cluster 0 osdmap e558: 8 total, 8 up, 8 in 2023-10-09T09:13:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:50 smithi181 bash[59345]: cluster 2023-10-09T09:13:49.097146+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e557 with expected crc 2023-10-09T09:13:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:50 smithi181 bash[59345]: cluster 2023-10-09T09:13:49.099048+0000 osd.3 (osd.3) 166 : cluster 3 failed to encode map e557 with expected crc 2023-10-09T09:13:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:50 smithi181 bash[59345]: cluster 2023-10-09T09:13:49.103166+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e557 with expected crc 2023-10-09T09:13:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:50 smithi181 bash[59345]: cluster 2023-10-09T09:13:49.303306+0000 mgr.y (mgr.34107) 196 : cluster 0 pgmap v388: 89 pgs: 89 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 512 KiB/s wr, 21 op/s 2023-10-09T09:13:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:50 smithi181 bash[59345]: cluster 2023-10-09T09:13:50.078244+0000 mon.a (mon.0) 524 : cluster 0 osdmap e558: 8 total, 8 up, 8 in 2023-10-09T09:13:51.079 INFO:tasks.rados.rados.0.smithi163.stdout:1961: done (0 left) 2023-10-09T09:13:51.079 INFO:tasks.rados.rados.0.smithi163.stdout:1962: snap_create 2023-10-09T09:13:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:51 smithi163 bash[78107]: cluster 2023-10-09T09:13:50.071211+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e558 with expected crc 2023-10-09T09:13:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:51 smithi163 bash[78107]: cluster 2023-10-09T09:13:50.071996+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e558 with expected crc 2023-10-09T09:13:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:51 smithi163 bash[78107]: cluster 2023-10-09T09:13:50.238405+0000 osd.0 (osd.0) 98 : cluster 3 failed to encode map e558 with expected crc 2023-10-09T09:13:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:51 smithi163 bash[78107]: cluster 2023-10-09T09:13:51.079616+0000 mon.a (mon.0) 525 : cluster 0 osdmap e559: 8 total, 8 up, 8 in 2023-10-09T09:13:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:51 smithi163 bash[75425]: cluster 2023-10-09T09:13:50.071211+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e558 with expected crc 2023-10-09T09:13:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:51 smithi163 bash[75425]: cluster 2023-10-09T09:13:50.071996+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e558 with expected crc 2023-10-09T09:13:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:51 smithi163 bash[75425]: cluster 2023-10-09T09:13:50.238405+0000 osd.0 (osd.0) 98 : cluster 3 failed to encode map e558 with expected crc 2023-10-09T09:13:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:51 smithi163 bash[75425]: cluster 2023-10-09T09:13:51.079616+0000 mon.a (mon.0) 525 : cluster 0 osdmap e559: 8 total, 8 up, 8 in 2023-10-09T09:13:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:51 smithi181 bash[59345]: cluster 2023-10-09T09:13:50.071211+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e558 with expected crc 2023-10-09T09:13:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:51 smithi181 bash[59345]: cluster 2023-10-09T09:13:50.071996+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e558 with expected crc 2023-10-09T09:13:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:51 smithi181 bash[59345]: cluster 2023-10-09T09:13:50.238405+0000 osd.0 (osd.0) 98 : cluster 3 failed to encode map e558 with expected crc 2023-10-09T09:13:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:51 smithi181 bash[59345]: cluster 2023-10-09T09:13:51.079616+0000 mon.a (mon.0) 525 : cluster 0 osdmap e559: 8 total, 8 up, 8 in 2023-10-09T09:13:52.086 INFO:tasks.rados.rados.0.smithi163.stdout:1962: done (0 left) 2023-10-09T09:13:52.086 INFO:tasks.rados.rados.0.smithi163.stdout:1963: read oid 49 snap -1 2023-10-09T09:13:52.087 INFO:tasks.rados.rados.0.smithi163.stdout:1963: expect (ObjNum 636 snap 202 seq_num 636) 2023-10-09T09:13:52.087 INFO:tasks.rados.rados.0.smithi163.stdout:1964: read oid 35 snap 234 2023-10-09T09:13:52.087 INFO:tasks.rados.rados.0.smithi163.stdout:1964: expect (ObjNum 681 snap 217 seq_num 681) 2023-10-09T09:13:52.087 INFO:tasks.rados.rados.0.smithi163.stdout:1965: snap_remove snap 250 2023-10-09T09:13:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[75425]: cluster 2023-10-09T09:13:51.075831+0000 osd.0 (osd.0) 99 : cluster 3 failed to encode map e559 with expected crc 2023-10-09T09:13:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[75425]: cluster 2023-10-09T09:13:51.075862+0000 osd.1 (osd.1) 79 : cluster 3 failed to encode map e559 with expected crc 2023-10-09T09:13:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[75425]: cluster 2023-10-09T09:13:51.304322+0000 mgr.y (mgr.34107) 197 : cluster 0 pgmap v391: 89 pgs: 89 active+clean; 177 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.0 MiB/s wr, 4 op/s 2023-10-09T09:13:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[75425]: cluster 2023-10-09T09:13:52.086053+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[75425]: cluster 2023-10-09T09:13:52.086303+0000 mon.a (mon.0) 526 : cluster 0 osdmap e560: 8 total, 8 up, 8 in 2023-10-09T09:13:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[75425]: audit 2023-10-09T09:13:52.309505+0000 mon.a (mon.0) 527 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:52.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[75425]: audit 2023-10-09T09:13:52.316869+0000 mon.a (mon.0) 528 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[78107]: cluster 2023-10-09T09:13:51.075831+0000 osd.0 (osd.0) 99 : cluster 3 failed to encode map e559 with expected crc 2023-10-09T09:13:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[78107]: cluster 2023-10-09T09:13:51.075862+0000 osd.1 (osd.1) 79 : cluster 3 failed to encode map e559 with expected crc 2023-10-09T09:13:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[78107]: cluster 2023-10-09T09:13:51.304322+0000 mgr.y (mgr.34107) 197 : cluster 0 pgmap v391: 89 pgs: 89 active+clean; 177 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.0 MiB/s wr, 4 op/s 2023-10-09T09:13:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[78107]: cluster 2023-10-09T09:13:52.086053+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[78107]: cluster 2023-10-09T09:13:52.086303+0000 mon.a (mon.0) 526 : cluster 0 osdmap e560: 8 total, 8 up, 8 in 2023-10-09T09:13:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[78107]: audit 2023-10-09T09:13:52.309505+0000 mon.a (mon.0) 527 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:52 smithi163 bash[78107]: audit 2023-10-09T09:13:52.316869+0000 mon.a (mon.0) 528 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:52 smithi181 bash[59345]: cluster 2023-10-09T09:13:51.075831+0000 osd.0 (osd.0) 99 : cluster 3 failed to encode map e559 with expected crc 2023-10-09T09:13:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:52 smithi181 bash[59345]: cluster 2023-10-09T09:13:51.075862+0000 osd.1 (osd.1) 79 : cluster 3 failed to encode map e559 with expected crc 2023-10-09T09:13:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:52 smithi181 bash[59345]: cluster 2023-10-09T09:13:51.304322+0000 mgr.y (mgr.34107) 197 : cluster 0 pgmap v391: 89 pgs: 89 active+clean; 177 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.0 MiB/s wr, 4 op/s 2023-10-09T09:13:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:52 smithi181 bash[59345]: cluster 2023-10-09T09:13:52.086053+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:52 smithi181 bash[59345]: cluster 2023-10-09T09:13:52.086303+0000 mon.a (mon.0) 526 : cluster 0 osdmap e560: 8 total, 8 up, 8 in 2023-10-09T09:13:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:52 smithi181 bash[59345]: audit 2023-10-09T09:13:52.309505+0000 mon.a (mon.0) 527 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:52 smithi181 bash[59345]: audit 2023-10-09T09:13:52.316869+0000 mon.a (mon.0) 528 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.090 INFO:tasks.rados.rados.0.smithi163.stdout:1965: done (2 left) 2023-10-09T09:13:53.090 INFO:tasks.rados.rados.0.smithi163.stdout:1966: read oid 28 snap -1 2023-10-09T09:13:53.090 INFO:tasks.rados.rados.0.smithi163.stdout:1966: expect (ObjNum 675 snap 212 seq_num 675) 2023-10-09T09:13:53.090 INFO:tasks.rados.rados.0.smithi163.stdout:1967: write oid 45 current snap is 252 2023-10-09T09:13:53.090 INFO:tasks.rados.rados.0.smithi163.stdout:1967: seq_num 741 ranges {453987=645750,1845157=608830} 2023-10-09T09:13:53.099 INFO:tasks.rados.rados.0.smithi163.stdout:1967: writing smithi16367185-45 from 453987 to 1099737 tid 1 2023-10-09T09:13:53.108 INFO:tasks.rados.rados.0.smithi163.stdout:1967: writing smithi16367185-45 from 1845157 to 2453987 tid 2 2023-10-09T09:13:53.108 INFO:tasks.rados.rados.0.smithi163.stdout:1968: snap_remove snap 236 2023-10-09T09:13:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:13:53] "GET /metrics HTTP/1.1" 200 36077 "" "Prometheus/2.33.4" 2023-10-09T09:13:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: cluster 2023-10-09T09:13:52.082556+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: cluster 2023-10-09T09:13:52.089647+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: cluster 2023-10-09T09:13:52.091336+0000 osd.0 (osd.0) 100 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:52.617081+0000 mon.a (mon.0) 529 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:52.627291+0000 mon.a (mon.0) 530 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: cluster 2023-10-09T09:13:53.086743+0000 osd.2 (osd.2) 164 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: cluster 2023-10-09T09:13:53.089766+0000 mon.a (mon.0) 531 : cluster 0 osdmap e561: 8 total, 8 up, 8 in 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.108447+0000 mon.a (mon.0) 532 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.109639+0000 mon.a (mon.0) 533 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.116879+0000 mon.a (mon.0) 534 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.198103+0000 mon.a (mon.0) 535 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.202264+0000 mon.a (mon.0) 536 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:13:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.205040+0000 mon.a (mon.0) 537 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:13:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.212733+0000 mon.a (mon.0) 538 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.214112+0000 mon.a (mon.0) 539 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T09:13:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.220230+0000 mon.a (mon.0) 540 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T09:13:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.221670+0000 mon.a (mon.0) 541 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T09:13:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.227434+0000 mon.a (mon.0) 542 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T09:13:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.228763+0000 mon.a (mon.0) 543 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T09:13:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.232747+0000 mon.a (mon.0) 544 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T09:13:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.234987+0000 mon.a (mon.0) 545 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:13:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.240416+0000 mon.a (mon.0) 546 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[75425]: audit 2023-10-09T09:13:53.242975+0000 mon.a (mon.0) 547 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: cluster 2023-10-09T09:13:52.082556+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: cluster 2023-10-09T09:13:52.089647+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: cluster 2023-10-09T09:13:52.091336+0000 osd.0 (osd.0) 100 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:52.617081+0000 mon.a (mon.0) 529 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:52.627291+0000 mon.a (mon.0) 530 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: cluster 2023-10-09T09:13:53.086743+0000 osd.2 (osd.2) 164 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: cluster 2023-10-09T09:13:53.089766+0000 mon.a (mon.0) 531 : cluster 0 osdmap e561: 8 total, 8 up, 8 in 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.108447+0000 mon.a (mon.0) 532 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.109639+0000 mon.a (mon.0) 533 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.116879+0000 mon.a (mon.0) 534 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.198103+0000 mon.a (mon.0) 535 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.202264+0000 mon.a (mon.0) 536 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.205040+0000 mon.a (mon.0) 537 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.212733+0000 mon.a (mon.0) 538 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.214112+0000 mon.a (mon.0) 539 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.220230+0000 mon.a (mon.0) 540 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.221670+0000 mon.a (mon.0) 541 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.227434+0000 mon.a (mon.0) 542 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.228763+0000 mon.a (mon.0) 543 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.232747+0000 mon.a (mon.0) 544 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.234987+0000 mon.a (mon.0) 545 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:13:53.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.240416+0000 mon.a (mon.0) 546 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:53 smithi163 bash[78107]: audit 2023-10-09T09:13:53.242975+0000 mon.a (mon.0) 547 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:13:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: cluster 2023-10-09T09:13:52.082556+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: cluster 2023-10-09T09:13:52.089647+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: cluster 2023-10-09T09:13:52.091336+0000 osd.0 (osd.0) 100 : cluster 3 failed to encode map e560 with expected crc 2023-10-09T09:13:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:52.617081+0000 mon.a (mon.0) 529 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:52.627291+0000 mon.a (mon.0) 530 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: cluster 2023-10-09T09:13:53.086743+0000 osd.2 (osd.2) 164 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: cluster 2023-10-09T09:13:53.089766+0000 mon.a (mon.0) 531 : cluster 0 osdmap e561: 8 total, 8 up, 8 in 2023-10-09T09:13:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.108447+0000 mon.a (mon.0) 532 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.109639+0000 mon.a (mon.0) 533 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:13:53.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.116879+0000 mon.a (mon.0) 534 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.198103+0000 mon.a (mon.0) 535 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.202264+0000 mon.a (mon.0) 536 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.205040+0000 mon.a (mon.0) 537 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.212733+0000 mon.a (mon.0) 538 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.214112+0000 mon.a (mon.0) 539 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.220230+0000 mon.a (mon.0) 540 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.221670+0000 mon.a (mon.0) 541 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.227434+0000 mon.a (mon.0) 542 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.228763+0000 mon.a (mon.0) 543 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.232747+0000 mon.a (mon.0) 544 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.234987+0000 mon.a (mon.0) 545 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:13:53.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.240416+0000 mon.a (mon.0) 546 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:53.820 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:53 smithi181 bash[59345]: audit 2023-10-09T09:13:53.242975+0000 mon.a (mon.0) 547 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:13:54.092 INFO:tasks.rados.rados.0.smithi163.stdout:1968: done (4 left) 2023-10-09T09:13:54.092 INFO:tasks.rados.rados.0.smithi163.stdout:1969: write oid 40 current snap is 252 2023-10-09T09:13:54.092 INFO:tasks.rados.rados.0.smithi163.stdout:1969: seq_num 742 ranges {470794=720999,1903639=554256} 2023-10-09T09:13:54.101 INFO:tasks.rados.rados.0.smithi163.stdout:1969: writing smithi16367185-40 from 470794 to 1191793 tid 1 2023-10-09T09:13:54.109 INFO:tasks.rados.rados.0.smithi163.stdout:1969: writing smithi16367185-40 from 1903639 to 2457895 tid 2 2023-10-09T09:13:54.109 INFO:tasks.rados.rados.0.smithi163.stdout:1970: rollback oid 1 current snap is 252 2023-10-09T09:13:54.110 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 1 to 235 2023-10-09T09:13:54.110 INFO:tasks.rados.rados.0.smithi163.stdout:1963: expect (ObjNum 636 snap 202 seq_num 636) 2023-10-09T09:13:54.472 INFO:tasks.rados.rados.0.smithi163.stdout:1964: expect (ObjNum 681 snap 217 seq_num 681) 2023-10-09T09:13:54.665 INFO:tasks.rados.rados.0.smithi163.stdout:1966: expect (ObjNum 675 snap 212 seq_num 675) 2023-10-09T09:13:54.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: cluster 2023-10-09T09:13:53.084749+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: cluster 2023-10-09T09:13:53.188290+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: cephadm 2023-10-09T09:13:53.206247+0000 mgr.y (mgr.34107) 198 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T09:13:54.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: cephadm 2023-10-09T09:13:53.236113+0000 mgr.y (mgr.34107) 199 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T09:13:54.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: audit 2023-10-09T09:13:53.243259+0000 mgr.y (mgr.34107) 200 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:13:54.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: cephadm 2023-10-09T09:13:53.244660+0000 mgr.y (mgr.34107) 201 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T09:13:54.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: cluster 2023-10-09T09:13:53.272423+0000 osd.3 (osd.3) 167 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: cluster 2023-10-09T09:13:53.304843+0000 mgr.y (mgr.34107) 202 : cluster 0 pgmap v394: 89 pgs: 89 active+clean; 177 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.0 MiB/s wr, 4 op/s 2023-10-09T09:13:54.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: cluster 2023-10-09T09:13:53.410693+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.670 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: audit 2023-10-09T09:13:53.839950+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.163:0/1593608591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:13:54.670 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: cluster 2023-10-09T09:13:54.088365+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e562 with expected crc 2023-10-09T09:13:54.670 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[75425]: cluster 2023-10-09T09:13:54.092092+0000 mon.a (mon.0) 548 : cluster 0 osdmap e562: 8 total, 8 up, 8 in 2023-10-09T09:13:54.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: cluster 2023-10-09T09:13:53.084749+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: cluster 2023-10-09T09:13:53.188290+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: cephadm 2023-10-09T09:13:53.206247+0000 mgr.y (mgr.34107) 198 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T09:13:54.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: cephadm 2023-10-09T09:13:53.236113+0000 mgr.y (mgr.34107) 199 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T09:13:54.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: audit 2023-10-09T09:13:53.243259+0000 mgr.y (mgr.34107) 200 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:13:54.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: cephadm 2023-10-09T09:13:53.244660+0000 mgr.y (mgr.34107) 201 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T09:13:54.670 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: cluster 2023-10-09T09:13:53.272423+0000 osd.3 (osd.3) 167 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.671 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: cluster 2023-10-09T09:13:53.304843+0000 mgr.y (mgr.34107) 202 : cluster 0 pgmap v394: 89 pgs: 89 active+clean; 177 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.0 MiB/s wr, 4 op/s 2023-10-09T09:13:54.671 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: cluster 2023-10-09T09:13:53.410693+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.671 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: audit 2023-10-09T09:13:53.839950+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.163:0/1593608591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:13:54.671 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: cluster 2023-10-09T09:13:54.088365+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e562 with expected crc 2023-10-09T09:13:54.671 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:54 smithi163 bash[78107]: cluster 2023-10-09T09:13:54.092092+0000 mon.a (mon.0) 548 : cluster 0 osdmap e562: 8 total, 8 up, 8 in 2023-10-09T09:13:54.779 INFO:tasks.rados.rados.0.smithi163.stdout:1967: finishing write tid 1 to smithi16367185-45 2023-10-09T09:13:54.779 INFO:tasks.rados.rados.0.smithi163.stdout:1967: finishing write tid 2 to smithi16367185-45 2023-10-09T09:13:54.779 INFO:tasks.rados.rados.0.smithi163.stdout:1963: done (5 left) 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1964: done (4 left) 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1966: done (3 left) 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1971: snap_create 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1967: finishing write tid 3 to smithi16367185-45 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1967: finishing write tid 4 to smithi16367185-45 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 1438 (ObjNum 741 snap 252 seq_num 741) dirty exists 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1967: left oid 45 (ObjNum 741 snap 252 seq_num 741) 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1969: finishing write tid 1 to smithi16367185-40 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1969: finishing write tid 2 to smithi16367185-40 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1969: finishing write tid 3 to smithi16367185-40 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1969: finishing write tid 4 to smithi16367185-40 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 966 (ObjNum 742 snap 252 seq_num 742) dirty exists 2023-10-09T09:13:54.780 INFO:tasks.rados.rados.0.smithi163.stdout:1969: left oid 40 (ObjNum 742 snap 252 seq_num 742) 2023-10-09T09:13:54.781 INFO:tasks.rados.rados.0.smithi163.stdout:1970: finishing rollback tid 1 to smithi16367185-1 2023-10-09T09:13:54.781 INFO:tasks.rados.rados.0.smithi163.stdout:1970: finishing rollback tid 2 to smithi16367185-1 2023-10-09T09:13:54.781 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1007 (ObjNum 648 snap 205 seq_num 648) dirty exists 2023-10-09T09:13:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: cluster 2023-10-09T09:13:53.084749+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: cluster 2023-10-09T09:13:53.188290+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: cephadm 2023-10-09T09:13:53.206247+0000 mgr.y (mgr.34107) 198 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T09:13:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: cephadm 2023-10-09T09:13:53.236113+0000 mgr.y (mgr.34107) 199 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T09:13:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: audit 2023-10-09T09:13:53.243259+0000 mgr.y (mgr.34107) 200 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:13:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: cephadm 2023-10-09T09:13:53.244660+0000 mgr.y (mgr.34107) 201 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T09:13:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: cluster 2023-10-09T09:13:53.272423+0000 osd.3 (osd.3) 167 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: cluster 2023-10-09T09:13:53.304843+0000 mgr.y (mgr.34107) 202 : cluster 0 pgmap v394: 89 pgs: 89 active+clean; 177 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.0 MiB/s wr, 4 op/s 2023-10-09T09:13:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: cluster 2023-10-09T09:13:53.410693+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: audit 2023-10-09T09:13:53.839950+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.163:0/1593608591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:13:54.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: cluster 2023-10-09T09:13:54.088365+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e562 with expected crc 2023-10-09T09:13:54.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:54 smithi181 bash[59345]: cluster 2023-10-09T09:13:54.092092+0000 mon.a (mon.0) 548 : cluster 0 osdmap e562: 8 total, 8 up, 8 in 2023-10-09T09:13:55.079 INFO:tasks.rados.rados.0.smithi163.stdout:1967: done (3 left) 2023-10-09T09:13:55.079 INFO:tasks.rados.rados.0.smithi163.stdout:1969: done (2 left) 2023-10-09T09:13:55.079 INFO:tasks.rados.rados.0.smithi163.stdout:1970: done (1 left) 2023-10-09T09:13:55.079 INFO:tasks.rados.rados.0.smithi163.stdout:1971: done (0 left) 2023-10-09T09:13:55.080 INFO:tasks.rados.rados.0.smithi163.stdout:1972: read oid 22 snap -1 2023-10-09T09:13:55.080 INFO:tasks.rados.rados.0.smithi163.stdout:1972: expect (ObjNum 624 snap 198 seq_num 624) 2023-10-09T09:13:55.080 INFO:tasks.rados.rados.0.smithi163.stdout:1973: delete oid 21 current snap is 253 2023-10-09T09:13:55.091 INFO:tasks.rados.rados.0.smithi163.stdout:1973: done (1 left) 2023-10-09T09:13:55.091 INFO:tasks.rados.rados.0.smithi163.stdout:1974: delete oid 28 current snap is 253 2023-10-09T09:13:55.098 INFO:tasks.rados.rados.0.smithi163.stdout:1972: expect (ObjNum 624 snap 198 seq_num 624) 2023-10-09T09:13:55.254 INFO:tasks.rados.rados.0.smithi163.stdout:1972: done (1 left) 2023-10-09T09:13:55.254 INFO:tasks.rados.rados.0.smithi163.stdout:1974: done (0 left) 2023-10-09T09:13:55.254 INFO:tasks.rados.rados.0.smithi163.stdout:1975: snap_remove snap 215 2023-10-09T09:13:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[75425]: cluster 2023-10-09T09:13:53.739041+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[75425]: cluster 2023-10-09T09:13:54.089280+0000 osd.3 (osd.3) 168 : cluster 3 failed to encode map e562 with expected crc 2023-10-09T09:13:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[75425]: cluster 2023-10-09T09:13:54.104276+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e562 with expected crc 2023-10-09T09:13:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[75425]: cluster 2023-10-09T09:13:55.078795+0000 mon.a (mon.0) 549 : cluster 0 osdmap e563: 8 total, 8 up, 8 in 2023-10-09T09:13:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[75425]: cluster 2023-10-09T09:13:55.092991+0000 osd.2 (osd.2) 165 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[75425]: audit 2023-10-09T09:13:55.323503+0000 mon.a (mon.0) 550 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[75425]: audit 2023-10-09T09:13:55.324896+0000 mon.a (mon.0) 551 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:13:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[75425]: audit 2023-10-09T09:13:55.325896+0000 mon.a (mon.0) 552 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[75425]: audit 2023-10-09T09:13:55.340729+0000 mon.a (mon.0) 553 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[78107]: cluster 2023-10-09T09:13:53.739041+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[78107]: cluster 2023-10-09T09:13:54.089280+0000 osd.3 (osd.3) 168 : cluster 3 failed to encode map e562 with expected crc 2023-10-09T09:13:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[78107]: cluster 2023-10-09T09:13:54.104276+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e562 with expected crc 2023-10-09T09:13:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[78107]: cluster 2023-10-09T09:13:55.078795+0000 mon.a (mon.0) 549 : cluster 0 osdmap e563: 8 total, 8 up, 8 in 2023-10-09T09:13:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[78107]: cluster 2023-10-09T09:13:55.092991+0000 osd.2 (osd.2) 165 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[78107]: audit 2023-10-09T09:13:55.323503+0000 mon.a (mon.0) 550 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[78107]: audit 2023-10-09T09:13:55.324896+0000 mon.a (mon.0) 551 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:13:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[78107]: audit 2023-10-09T09:13:55.325896+0000 mon.a (mon.0) 552 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:55 smithi163 bash[78107]: audit 2023-10-09T09:13:55.340729+0000 mon.a (mon.0) 553 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:55 smithi181 bash[59345]: cluster 2023-10-09T09:13:53.739041+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e561 with expected crc 2023-10-09T09:13:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:55 smithi181 bash[59345]: cluster 2023-10-09T09:13:54.089280+0000 osd.3 (osd.3) 168 : cluster 3 failed to encode map e562 with expected crc 2023-10-09T09:13:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:55 smithi181 bash[59345]: cluster 2023-10-09T09:13:54.104276+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e562 with expected crc 2023-10-09T09:13:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:55 smithi181 bash[59345]: cluster 2023-10-09T09:13:55.078795+0000 mon.a (mon.0) 549 : cluster 0 osdmap e563: 8 total, 8 up, 8 in 2023-10-09T09:13:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:55 smithi181 bash[59345]: cluster 2023-10-09T09:13:55.092991+0000 osd.2 (osd.2) 165 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:55 smithi181 bash[59345]: audit 2023-10-09T09:13:55.323503+0000 mon.a (mon.0) 550 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:13:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:55 smithi181 bash[59345]: audit 2023-10-09T09:13:55.324896+0000 mon.a (mon.0) 551 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:13:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:55 smithi181 bash[59345]: audit 2023-10-09T09:13:55.325896+0000 mon.a (mon.0) 552 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:13:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:55 smithi181 bash[59345]: audit 2023-10-09T09:13:55.340729+0000 mon.a (mon.0) 553 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:13:56.092 INFO:tasks.rados.rados.0.smithi163.stdout:1975: done (0 left) 2023-10-09T09:13:56.092 INFO:tasks.rados.rados.0.smithi163.stdout:1976: write oid 6 current snap is 253 2023-10-09T09:13:56.093 INFO:tasks.rados.rados.0.smithi163.stdout:1976: seq_num 745 ranges {521215=546746,1562732=701583} 2023-10-09T09:13:56.099 INFO:tasks.rados.rados.0.smithi163.stdout:1976: writing smithi16367185-6 from 521215 to 1067961 tid 1 2023-10-09T09:13:56.107 INFO:tasks.rados.rados.0.smithi163.stdout:1976: writing smithi16367185-6 from 1562732 to 2264315 tid 2 2023-10-09T09:13:56.107 INFO:tasks.rados.rados.0.smithi163.stdout:1977: read oid 41 snap -1 2023-10-09T09:13:56.107 INFO:tasks.rados.rados.0.smithi163.stdout:1977: expect (ObjNum 723 snap 245 seq_num 723) 2023-10-09T09:13:56.107 INFO:tasks.rados.rados.0.smithi163.stdout:1978: read oid 27 snap 245 2023-10-09T09:13:56.107 INFO:tasks.rados.rados.0.smithi163.stdout:1978: expect deleted 2023-10-09T09:13:56.108 INFO:tasks.rados.rados.0.smithi163.stdout:1979: read oid 8 snap -1 2023-10-09T09:13:56.108 INFO:tasks.rados.rados.0.smithi163.stdout:1979: expect deleted 2023-10-09T09:13:56.108 INFO:tasks.rados.rados.0.smithi163.stdout:1980: delete oid 15 current snap is 253 2023-10-09T09:13:56.108 INFO:tasks.rados.rados.0.smithi163.stdout:1976: finishing write tid 1 to smithi16367185-6 2023-10-09T09:13:56.112 INFO:tasks.rados.rados.0.smithi163.stdout:1976: finishing write tid 2 to smithi16367185-6 2023-10-09T09:13:56.112 INFO:tasks.rados.rados.0.smithi163.stdout:1976: finishing write tid 3 to smithi16367185-6 2023-10-09T09:13:56.112 INFO:tasks.rados.rados.0.smithi163.stdout:1976: finishing write tid 4 to smithi16367185-6 2023-10-09T09:13:56.112 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 190 (ObjNum 745 snap 253 seq_num 745) dirty exists 2023-10-09T09:13:56.112 INFO:tasks.rados.rados.0.smithi163.stdout:1976: left oid 6 (ObjNum 745 snap 253 seq_num 745) 2023-10-09T09:13:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1976: done (4 left) 2023-10-09T09:13:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1978: done (3 left) 2023-10-09T09:13:56.113 INFO:tasks.rados.rados.0.smithi163.stdout:1979: done (2 left) 2023-10-09T09:13:56.114 INFO:tasks.rados.rados.0.smithi163.stdout:1980: done (1 left) 2023-10-09T09:13:56.114 INFO:tasks.rados.rados.0.smithi163.stdout:1981: read oid 31 snap -1 2023-10-09T09:13:56.114 INFO:tasks.rados.rados.0.smithi163.stdout:1981: expect deleted 2023-10-09T09:13:56.114 INFO:tasks.rados.rados.0.smithi163.stdout:1982: read oid 20 snap 245 2023-10-09T09:13:56.114 INFO:tasks.rados.rados.0.smithi163.stdout:1982: expect deleted 2023-10-09T09:13:56.114 INFO:tasks.rados.rados.0.smithi163.stdout:1983: snap_remove snap 249 2023-10-09T09:13:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cluster 2023-10-09T09:13:55.075280+0000 osd.3 (osd.3) 169 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cluster 2023-10-09T09:13:55.075738+0000 osd.0 (osd.0) 101 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cluster 2023-10-09T09:13:55.082786+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cluster 2023-10-09T09:13:55.084319+0000 osd.1 (osd.1) 80 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cluster 2023-10-09T09:13:55.155788+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cluster 2023-10-09T09:13:55.305489+0000 mgr.y (mgr.34107) 203 : cluster 0 pgmap v397: 89 pgs: 89 active+clean; 177 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cephadm 2023-10-09T09:13:55.314229+0000 mgr.y (mgr.34107) 204 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T09:13:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cephadm 2023-10-09T09:13:55.329471+0000 mgr.y (mgr.34107) 205 : cephadm 1 Deploying daemon osd.0 on smithi163 2023-10-09T09:13:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cluster 2023-10-09T09:13:56.088364+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e564 with expected crc 2023-10-09T09:13:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cluster 2023-10-09T09:13:56.092540+0000 mon.a (mon.0) 554 : cluster 0 osdmap e564: 8 total, 8 up, 8 in 2023-10-09T09:13:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: cluster 2023-10-09T09:13:56.108780+0000 osd.2 (osd.2) 166 : cluster 3 failed to encode map e564 with expected crc 2023-10-09T09:13:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[78107]: audit 2023-10-09T09:13:56.214035+0000 mon.a (mon.0) 555 : audit 0 from='client.? 172.21.15.163:0/2312096548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:13:56.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cluster 2023-10-09T09:13:55.075280+0000 osd.3 (osd.3) 169 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cluster 2023-10-09T09:13:55.075738+0000 osd.0 (osd.0) 101 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cluster 2023-10-09T09:13:55.082786+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cluster 2023-10-09T09:13:55.084319+0000 osd.1 (osd.1) 80 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cluster 2023-10-09T09:13:55.155788+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cluster 2023-10-09T09:13:55.305489+0000 mgr.y (mgr.34107) 203 : cluster 0 pgmap v397: 89 pgs: 89 active+clean; 177 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:56.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cephadm 2023-10-09T09:13:55.314229+0000 mgr.y (mgr.34107) 204 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T09:13:56.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cephadm 2023-10-09T09:13:55.329471+0000 mgr.y (mgr.34107) 205 : cephadm 1 Deploying daemon osd.0 on smithi163 2023-10-09T09:13:56.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cluster 2023-10-09T09:13:56.088364+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e564 with expected crc 2023-10-09T09:13:56.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cluster 2023-10-09T09:13:56.092540+0000 mon.a (mon.0) 554 : cluster 0 osdmap e564: 8 total, 8 up, 8 in 2023-10-09T09:13:56.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: cluster 2023-10-09T09:13:56.108780+0000 osd.2 (osd.2) 166 : cluster 3 failed to encode map e564 with expected crc 2023-10-09T09:13:56.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:56 smithi163 bash[75425]: audit 2023-10-09T09:13:56.214035+0000 mon.a (mon.0) 555 : audit 0 from='client.? 172.21.15.163:0/2312096548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:13:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cluster 2023-10-09T09:13:55.075280+0000 osd.3 (osd.3) 169 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cluster 2023-10-09T09:13:55.075738+0000 osd.0 (osd.0) 101 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cluster 2023-10-09T09:13:55.082786+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cluster 2023-10-09T09:13:55.084319+0000 osd.1 (osd.1) 80 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cluster 2023-10-09T09:13:55.155788+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e563 with expected crc 2023-10-09T09:13:56.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cluster 2023-10-09T09:13:55.305489+0000 mgr.y (mgr.34107) 203 : cluster 0 pgmap v397: 89 pgs: 89 active+clean; 177 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:13:56.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cephadm 2023-10-09T09:13:55.314229+0000 mgr.y (mgr.34107) 204 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T09:13:56.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cephadm 2023-10-09T09:13:55.329471+0000 mgr.y (mgr.34107) 205 : cephadm 1 Deploying daemon osd.0 on smithi163 2023-10-09T09:13:56.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cluster 2023-10-09T09:13:56.088364+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e564 with expected crc 2023-10-09T09:13:56.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cluster 2023-10-09T09:13:56.092540+0000 mon.a (mon.0) 554 : cluster 0 osdmap e564: 8 total, 8 up, 8 in 2023-10-09T09:13:56.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: cluster 2023-10-09T09:13:56.108780+0000 osd.2 (osd.2) 166 : cluster 3 failed to encode map e564 with expected crc 2023-10-09T09:13:56.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:56 smithi181 bash[59345]: audit 2023-10-09T09:13:56.214035+0000 mon.a (mon.0) 555 : audit 0 from='client.? 172.21.15.163:0/2312096548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:13:57.096 INFO:tasks.rados.rados.0.smithi163.stdout:1983: done (3 left) 2023-10-09T09:13:57.096 INFO:tasks.rados.rados.0.smithi163.stdout:1984: rollback oid 50 current snap is 253 2023-10-09T09:13:57.096 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 50 to 245 2023-10-09T09:13:57.096 INFO:tasks.rados.rados.0.smithi163.stdout:1977: expect (ObjNum 723 snap 245 seq_num 723) 2023-10-09T09:13:57.202 INFO:tasks.rados.rados.0.smithi163.stdout:1984: finishing rollback tid 0 to smithi16367185-50 2023-10-09T09:13:57.202 INFO:tasks.rados.rados.0.smithi163.stdout:1984: finishing rollback tid 1 to smithi16367185-50 2023-10-09T09:13:57.202 INFO:tasks.rados.rados.0.smithi163.stdout:1984: finishing rollback tid 2 to smithi16367185-50 2023-10-09T09:13:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 808 (ObjNum 667 snap 211 seq_num 667) dirty exists 2023-10-09T09:13:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:1977: done (3 left) 2023-10-09T09:13:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:1981: done (2 left) 2023-10-09T09:13:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:1982: done (1 left) 2023-10-09T09:13:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:1984: done (0 left) 2023-10-09T09:13:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:1985: read oid 14 snap -1 2023-10-09T09:13:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:1985: expect (ObjNum 622 snap 197 seq_num 622) 2023-10-09T09:13:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:1986: write oid 33 current snap is 253 2023-10-09T09:13:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:1986: seq_num 747 ranges {554829=413597,1396049=705252} 2023-10-09T09:13:57.208 INFO:tasks.rados.rados.0.smithi163.stdout:1986: writing smithi16367185-33 from 554829 to 968426 tid 1 2023-10-09T09:13:57.215 INFO:tasks.rados.rados.0.smithi163.stdout:1986: writing smithi16367185-33 from 1396049 to 2101301 tid 2 2023-10-09T09:13:57.215 INFO:tasks.rados.rados.0.smithi163.stdout:1987: snap_create 2023-10-09T09:13:57.218 INFO:tasks.rados.rados.0.smithi163.stdout:1985: expect (ObjNum 622 snap 197 seq_num 622) 2023-10-09T09:13:57.311 INFO:tasks.rados.rados.0.smithi163.stdout:1986: finishing write tid 1 to smithi16367185-33 2023-10-09T09:13:57.312 INFO:tasks.rados.rados.0.smithi163.stdout:1986: finishing write tid 2 to smithi16367185-33 2023-10-09T09:13:57.312 INFO:tasks.rados.rados.0.smithi163.stdout:1986: finishing write tid 3 to smithi16367185-33 2023-10-09T09:13:57.312 INFO:tasks.rados.rados.0.smithi163.stdout:1986: finishing write tid 4 to smithi16367185-33 2023-10-09T09:13:57.312 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 921 (ObjNum 747 snap 253 seq_num 747) dirty exists 2023-10-09T09:13:57.312 INFO:tasks.rados.rados.0.smithi163.stdout:1986: left oid 33 (ObjNum 747 snap 253 seq_num 747) 2023-10-09T09:13:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:57 smithi163 bash[75425]: cluster 2023-10-09T09:13:56.089720+0000 osd.3 (osd.3) 170 : cluster 3 failed to encode map e564 with expected crc 2023-10-09T09:13:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:57 smithi163 bash[75425]: cluster 2023-10-09T09:13:57.091031+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e565 with expected crc 2023-10-09T09:13:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:57 smithi163 bash[75425]: cluster 2023-10-09T09:13:57.096091+0000 mon.a (mon.0) 556 : cluster 0 osdmap e565: 8 total, 8 up, 8 in 2023-10-09T09:13:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:57 smithi163 bash[78107]: cluster 2023-10-09T09:13:56.089720+0000 osd.3 (osd.3) 170 : cluster 3 failed to encode map e564 with expected crc 2023-10-09T09:13:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:57 smithi163 bash[78107]: cluster 2023-10-09T09:13:57.091031+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e565 with expected crc 2023-10-09T09:13:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:57 smithi163 bash[78107]: cluster 2023-10-09T09:13:57.096091+0000 mon.a (mon.0) 556 : cluster 0 osdmap e565: 8 total, 8 up, 8 in 2023-10-09T09:13:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:57 smithi181 bash[59345]: cluster 2023-10-09T09:13:56.089720+0000 osd.3 (osd.3) 170 : cluster 3 failed to encode map e564 with expected crc 2023-10-09T09:13:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:57 smithi181 bash[59345]: cluster 2023-10-09T09:13:57.091031+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e565 with expected crc 2023-10-09T09:13:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:57 smithi181 bash[59345]: cluster 2023-10-09T09:13:57.096091+0000 mon.a (mon.0) 556 : cluster 0 osdmap e565: 8 total, 8 up, 8 in 2023-10-09T09:13:58.102 INFO:tasks.rados.rados.0.smithi163.stdout:1985: done (2 left) 2023-10-09T09:13:58.102 INFO:tasks.rados.rados.0.smithi163.stdout:1986: done (1 left) 2023-10-09T09:13:58.103 INFO:tasks.rados.rados.0.smithi163.stdout:1987: done (0 left) 2023-10-09T09:13:58.103 INFO:tasks.rados.rados.0.smithi163.stdout:1988: snap_remove snap 217 2023-10-09T09:13:58.413 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:58 smithi163 bash[75425]: cluster 2023-10-09T09:13:57.090948+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e565 with expected crc 2023-10-09T09:13:58.413 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:58 smithi163 bash[75425]: cluster 2023-10-09T09:13:57.306139+0000 mgr.y (mgr.34107) 206 : cluster 0 pgmap v400: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 923 KiB/s wr, 15 op/s 2023-10-09T09:13:58.413 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:58 smithi163 bash[75425]: cluster 2023-10-09T09:13:58.101137+0000 osd.2 (osd.2) 167 : cluster 3 failed to encode map e566 with expected crc 2023-10-09T09:13:58.413 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:58 smithi163 bash[75425]: cluster 2023-10-09T09:13:58.102373+0000 mon.a (mon.0) 557 : cluster 0 osdmap e566: 8 total, 8 up, 8 in 2023-10-09T09:13:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:58 smithi163 bash[78107]: cluster 2023-10-09T09:13:57.090948+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e565 with expected crc 2023-10-09T09:13:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:58 smithi163 bash[78107]: cluster 2023-10-09T09:13:57.306139+0000 mgr.y (mgr.34107) 206 : cluster 0 pgmap v400: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 923 KiB/s wr, 15 op/s 2023-10-09T09:13:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:58 smithi163 bash[78107]: cluster 2023-10-09T09:13:58.101137+0000 osd.2 (osd.2) 167 : cluster 3 failed to encode map e566 with expected crc 2023-10-09T09:13:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:58 smithi163 bash[78107]: cluster 2023-10-09T09:13:58.102373+0000 mon.a (mon.0) 557 : cluster 0 osdmap e566: 8 total, 8 up, 8 in 2023-10-09T09:13:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:58 smithi181 bash[59345]: cluster 2023-10-09T09:13:57.090948+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e565 with expected crc 2023-10-09T09:13:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:58 smithi181 bash[59345]: cluster 2023-10-09T09:13:57.306139+0000 mgr.y (mgr.34107) 206 : cluster 0 pgmap v400: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 923 KiB/s wr, 15 op/s 2023-10-09T09:13:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:58 smithi181 bash[59345]: cluster 2023-10-09T09:13:58.101137+0000 osd.2 (osd.2) 167 : cluster 3 failed to encode map e566 with expected crc 2023-10-09T09:13:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:58 smithi181 bash[59345]: cluster 2023-10-09T09:13:58.102373+0000 mon.a (mon.0) 557 : cluster 0 osdmap e566: 8 total, 8 up, 8 in 2023-10-09T09:13:59.108 INFO:tasks.rados.rados.0.smithi163.stdout:1988: done (0 left) 2023-10-09T09:13:59.108 INFO:tasks.rados.rados.0.smithi163.stdout:1989: read oid 40 snap 219 2023-10-09T09:13:59.108 INFO:tasks.rados.rados.0.smithi163.stdout:1989: expect deleted 2023-10-09T09:13:59.108 INFO:tasks.rados.rados.0.smithi163.stdout:1990: delete oid 42 current snap is 254 2023-10-09T09:13:59.124 INFO:tasks.rados.rados.0.smithi163.stdout:1989: done (1 left) 2023-10-09T09:13:59.124 INFO:tasks.rados.rados.0.smithi163.stdout:1990: done (0 left) 2023-10-09T09:13:59.124 INFO:tasks.rados.rados.0.smithi163.stdout:1991: snap_create 2023-10-09T09:13:59.645 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:59 smithi163 bash[78107]: cluster 2023-10-09T09:13:58.100344+0000 osd.1 (osd.1) 81 : cluster 3 failed to encode map e566 with expected crc 2023-10-09T09:13:59.645 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:59 smithi163 bash[78107]: audit 2023-10-09T09:13:58.578682+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.163:0/988052836' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:13:59.645 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:13:59 smithi163 bash[78107]: cluster 2023-10-09T09:13:59.107166+0000 mon.a (mon.0) 558 : cluster 0 osdmap e567: 8 total, 8 up, 8 in 2023-10-09T09:13:59.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:59 smithi163 bash[75425]: cluster 2023-10-09T09:13:58.100344+0000 osd.1 (osd.1) 81 : cluster 3 failed to encode map e566 with expected crc 2023-10-09T09:13:59.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:59 smithi163 bash[75425]: audit 2023-10-09T09:13:58.578682+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.163:0/988052836' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:13:59.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:13:59 smithi163 bash[75425]: cluster 2023-10-09T09:13:59.107166+0000 mon.a (mon.0) 558 : cluster 0 osdmap e567: 8 total, 8 up, 8 in 2023-10-09T09:13:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:59 smithi181 bash[59345]: cluster 2023-10-09T09:13:58.100344+0000 osd.1 (osd.1) 81 : cluster 3 failed to encode map e566 with expected crc 2023-10-09T09:13:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:59 smithi181 bash[59345]: audit 2023-10-09T09:13:58.578682+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.163:0/988052836' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:13:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:13:59 smithi181 bash[59345]: cluster 2023-10-09T09:13:59.107166+0000 mon.a (mon.0) 558 : cluster 0 osdmap e567: 8 total, 8 up, 8 in 2023-10-09T09:13:59.968 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:13:59 smithi163 systemd[1]: Stopping Ceph osd.0 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:13:59.968 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:13:59 smithi163 bash[37081]: debug 2023-10-09T09:13:59.788+0000 7f5befdf5700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:13:59.968 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:13:59 smithi163 bash[37081]: debug 2023-10-09T09:13:59.788+0000 7f5befdf5700 -1 osd.0 567 *** Got signal Terminated *** 2023-10-09T09:13:59.968 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:13:59 smithi163 bash[37081]: debug 2023-10-09T09:13:59.788+0000 7f5befdf5700 -1 osd.0 567 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:14:00.084 INFO:tasks.rados.rados.0.smithi163.stdout:1991: done (0 left) 2023-10-09T09:14:00.084 INFO:tasks.rados.rados.0.smithi163.stdout:1992: rollback oid 31 current snap is 255 2023-10-09T09:14:00.084 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 31 to 237 2023-10-09T09:14:00.085 INFO:tasks.rados.rados.0.smithi163.stdout:1993: delete oid 14 current snap is 255 2023-10-09T09:14:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[78107]: cluster 2023-10-09T09:13:58.667288+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e566 with expected crc 2023-10-09T09:14:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[78107]: cluster 2023-10-09T09:13:59.102356+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e567 with expected crc 2023-10-09T09:14:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[78107]: cluster 2023-10-09T09:13:59.103127+0000 osd.3 (osd.3) 171 : cluster 3 failed to encode map e567 with expected crc 2023-10-09T09:14:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[78107]: cluster 2023-10-09T09:13:59.108446+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e567 with expected crc 2023-10-09T09:14:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[78107]: cluster 2023-10-09T09:13:59.306898+0000 mgr.y (mgr.34107) 207 : cluster 0 pgmap v403: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 923 KiB/s wr, 15 op/s 2023-10-09T09:14:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[78107]: cluster 2023-10-09T09:13:59.795147+0000 mon.a (mon.0) 559 : cluster 1 osd.0 marked itself down and dead 2023-10-09T09:14:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[78107]: cluster 2023-10-09T09:14:00.075389+0000 mon.a (mon.0) 560 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:14:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[78107]: cluster 2023-10-09T09:14:00.078576+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[78107]: cluster 2023-10-09T09:14:00.083516+0000 mon.a (mon.0) 561 : cluster 0 osdmap e568: 8 total, 7 up, 8 in 2023-10-09T09:14:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[78107]: cluster 2023-10-09T09:14:00.086535+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[75425]: cluster 2023-10-09T09:13:58.667288+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e566 with expected crc 2023-10-09T09:14:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[75425]: cluster 2023-10-09T09:13:59.102356+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e567 with expected crc 2023-10-09T09:14:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[75425]: cluster 2023-10-09T09:13:59.103127+0000 osd.3 (osd.3) 171 : cluster 3 failed to encode map e567 with expected crc 2023-10-09T09:14:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[75425]: cluster 2023-10-09T09:13:59.108446+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e567 with expected crc 2023-10-09T09:14:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[75425]: cluster 2023-10-09T09:13:59.306898+0000 mgr.y (mgr.34107) 207 : cluster 0 pgmap v403: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 923 KiB/s wr, 15 op/s 2023-10-09T09:14:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[75425]: cluster 2023-10-09T09:13:59.795147+0000 mon.a (mon.0) 559 : cluster 1 osd.0 marked itself down and dead 2023-10-09T09:14:00.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[75425]: cluster 2023-10-09T09:14:00.075389+0000 mon.a (mon.0) 560 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:14:00.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[75425]: cluster 2023-10-09T09:14:00.078576+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:00.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[75425]: cluster 2023-10-09T09:14:00.083516+0000 mon.a (mon.0) 561 : cluster 0 osdmap e568: 8 total, 7 up, 8 in 2023-10-09T09:14:00.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[75425]: cluster 2023-10-09T09:14:00.086535+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:00 smithi181 bash[59345]: cluster 2023-10-09T09:13:58.667288+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e566 with expected crc 2023-10-09T09:14:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:00 smithi181 bash[59345]: cluster 2023-10-09T09:13:59.102356+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e567 with expected crc 2023-10-09T09:14:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:00 smithi181 bash[59345]: cluster 2023-10-09T09:13:59.103127+0000 osd.3 (osd.3) 171 : cluster 3 failed to encode map e567 with expected crc 2023-10-09T09:14:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:00 smithi181 bash[59345]: cluster 2023-10-09T09:13:59.108446+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e567 with expected crc 2023-10-09T09:14:00.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:00 smithi181 bash[59345]: cluster 2023-10-09T09:13:59.306898+0000 mgr.y (mgr.34107) 207 : cluster 0 pgmap v403: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 923 KiB/s wr, 15 op/s 2023-10-09T09:14:00.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:00 smithi181 bash[59345]: cluster 2023-10-09T09:13:59.795147+0000 mon.a (mon.0) 559 : cluster 1 osd.0 marked itself down and dead 2023-10-09T09:14:00.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:00 smithi181 bash[59345]: cluster 2023-10-09T09:14:00.075389+0000 mon.a (mon.0) 560 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:14:00.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:00 smithi181 bash[59345]: cluster 2023-10-09T09:14:00.078576+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:00.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:00 smithi181 bash[59345]: cluster 2023-10-09T09:14:00.083516+0000 mon.a (mon.0) 561 : cluster 0 osdmap e568: 8 total, 7 up, 8 in 2023-10-09T09:14:00.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:00 smithi181 bash[59345]: cluster 2023-10-09T09:14:00.086535+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:00.968 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[83496]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T09:14:00.968 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:00 smithi163 bash[83585]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T09:14:01.111 INFO:tasks.rados.rados.0.smithi163.stdout:1993: done (1 left) 2023-10-09T09:14:01.111 INFO:tasks.rados.rados.0.smithi163.stdout:1994: write oid 32 current snap is 255 2023-10-09T09:14:01.111 INFO:tasks.rados.rados.0.smithi163.stdout:1992: finishing rollback tid 1 to smithi16367185-31 2023-10-09T09:14:01.111 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:14:01.111 INFO:tasks.rados.rados.0.smithi163.stdout:1994: seq_num 750 ranges {605250=639344,1855142=452579} 2023-10-09T09:14:01.117 INFO:tasks.rados.rados.0.smithi163.stdout:1994: writing smithi16367185-32 from 605250 to 1244594 tid 1 2023-10-09T09:14:01.124 INFO:tasks.rados.rados.0.smithi163.stdout:1994: writing smithi16367185-32 from 1855142 to 2307721 tid 2 2023-10-09T09:14:01.124 INFO:tasks.rados.rados.0.smithi163.stdout:1992: done (1 left) 2023-10-09T09:14:01.124 INFO:tasks.rados.rados.0.smithi163.stdout:1995: write oid 48 current snap is 255 2023-10-09T09:14:01.124 INFO:tasks.rados.rados.0.smithi163.stdout:1994: finishing write tid 1 to smithi16367185-32 2023-10-09T09:14:01.124 INFO:tasks.rados.rados.0.smithi163.stdout:1995: seq_num 751 ranges {622057=714593,1913624=708433} 2023-10-09T09:14:01.131 INFO:tasks.rados.rados.0.smithi163.stdout:1995: writing smithi16367185-48 from 622057 to 1336650 tid 1 2023-10-09T09:14:01.138 INFO:tasks.rados.rados.0.smithi163.stdout:1995: writing smithi16367185-48 from 1913624 to 2622057 tid 2 2023-10-09T09:14:01.138 INFO:tasks.rados.rados.0.smithi163.stdout:1996: snap_remove snap 253 2023-10-09T09:14:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[75425]: cluster 2023-10-09T09:14:00.077707+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[75425]: cluster 2023-10-09T09:14:00.089398+0000 osd.1 (osd.1) 82 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[75425]: audit 2023-10-09T09:14:00.929719+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.163:0/2651668701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[75425]: cluster 2023-10-09T09:14:01.090023+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[75425]: cluster 2023-10-09T09:14:01.094312+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[75425]: cluster 2023-10-09T09:14:01.095410+0000 mon.a (mon.0) 562 : cluster 0 osdmap e569: 8 total, 7 up, 8 in 2023-10-09T09:14:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[75425]: cluster 2023-10-09T09:14:01.095985+0000 osd.2 (osd.2) 168 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[78107]: cluster 2023-10-09T09:14:00.077707+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[78107]: cluster 2023-10-09T09:14:00.089398+0000 osd.1 (osd.1) 82 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[78107]: audit 2023-10-09T09:14:00.929719+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.163:0/2651668701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[78107]: cluster 2023-10-09T09:14:01.090023+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[78107]: cluster 2023-10-09T09:14:01.094312+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[78107]: cluster 2023-10-09T09:14:01.095410+0000 mon.a (mon.0) 562 : cluster 0 osdmap e569: 8 total, 7 up, 8 in 2023-10-09T09:14:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:01 smithi163 bash[78107]: cluster 2023-10-09T09:14:01.095985+0000 osd.2 (osd.2) 168 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:01 smithi181 bash[59345]: cluster 2023-10-09T09:14:00.077707+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:01 smithi181 bash[59345]: cluster 2023-10-09T09:14:00.089398+0000 osd.1 (osd.1) 82 : cluster 3 failed to encode map e568 with expected crc 2023-10-09T09:14:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:01 smithi181 bash[59345]: audit 2023-10-09T09:14:00.929719+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.163:0/2651668701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:01 smithi181 bash[59345]: cluster 2023-10-09T09:14:01.090023+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:01.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:01 smithi181 bash[59345]: cluster 2023-10-09T09:14:01.094312+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:01.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:01 smithi181 bash[59345]: cluster 2023-10-09T09:14:01.095410+0000 mon.a (mon.0) 562 : cluster 0 osdmap e569: 8 total, 7 up, 8 in 2023-10-09T09:14:01.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:01 smithi181 bash[59345]: cluster 2023-10-09T09:14:01.095985+0000 osd.2 (osd.2) 168 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.099 INFO:tasks.rados.rados.0.smithi163.stdout:1996: done (2 left) 2023-10-09T09:14:02.099 INFO:tasks.rados.rados.0.smithi163.stdout:1997: delete oid 49 current snap is 255 2023-10-09T09:14:02.099 INFO:tasks.rados.rados.0.smithi163.stdout:1994: finishing write tid 2 to smithi16367185-32 2023-10-09T09:14:02.099 INFO:tasks.rados.rados.0.smithi163.stdout:1994: finishing write tid 3 to smithi16367185-32 2023-10-09T09:14:02.099 INFO:tasks.rados.rados.0.smithi163.stdout:1994: finishing write tid 4 to smithi16367185-32 2023-10-09T09:14:02.099 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1463 (ObjNum 750 snap 255 seq_num 750) dirty exists 2023-10-09T09:14:02.100 INFO:tasks.rados.rados.0.smithi163.stdout:1994: left oid 32 (ObjNum 750 snap 255 seq_num 750) 2023-10-09T09:14:02.100 INFO:tasks.rados.rados.0.smithi163.stdout:1995: finishing write tid 1 to smithi16367185-48 2023-10-09T09:14:02.100 INFO:tasks.rados.rados.0.smithi163.stdout:1995: finishing write tid 2 to smithi16367185-48 2023-10-09T09:14:02.100 INFO:tasks.rados.rados.0.smithi163.stdout:1995: finishing write tid 3 to smithi16367185-48 2023-10-09T09:14:02.100 INFO:tasks.rados.rados.0.smithi163.stdout:1995: finishing write tid 4 to smithi16367185-48 2023-10-09T09:14:02.100 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 1467 (ObjNum 751 snap 255 seq_num 751) dirty exists 2023-10-09T09:14:02.100 INFO:tasks.rados.rados.0.smithi163.stdout:1995: left oid 48 (ObjNum 751 snap 255 seq_num 751) 2023-10-09T09:14:02.108 INFO:tasks.rados.rados.0.smithi163.stdout:1994: done (2 left) 2023-10-09T09:14:02.108 INFO:tasks.rados.rados.0.smithi163.stdout:1995: done (1 left) 2023-10-09T09:14:02.108 INFO:tasks.rados.rados.0.smithi163.stdout:1997: done (0 left) 2023-10-09T09:14:02.109 INFO:tasks.rados.rados.0.smithi163.stdout:1998: rollback oid 25 current snap is 255 2023-10-09T09:14:02.109 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 25 to 245 2023-10-09T09:14:02.109 INFO:tasks.rados.rados.0.smithi163.stdout:1999: snap_create 2023-10-09T09:14:02.118 INFO:tasks.rados.rados.0.smithi163.stdout:1998: finishing rollback tid 0 to smithi16367185-25 2023-10-09T09:14:02.119 INFO:tasks.rados.rados.0.smithi163.stdout:1998: finishing rollback tid 1 to smithi16367185-25 2023-10-09T09:14:02.119 INFO:tasks.rados.rados.0.smithi163.stdout:1998: finishing rollback tid 2 to smithi16367185-25 2023-10-09T09:14:02.119 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 933 (ObjNum 712 snap 242 seq_num 712) dirty exists 2023-10-09T09:14:02.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[75425]: cluster 2023-10-09T09:14:01.090028+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[75425]: cluster 2023-10-09T09:14:01.094170+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[75425]: cluster 2023-10-09T09:14:01.094339+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[75425]: cluster 2023-10-09T09:14:01.096067+0000 osd.3 (osd.3) 172 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[75425]: cluster 2023-10-09T09:14:01.096118+0000 osd.1 (osd.1) 83 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[75425]: cluster 2023-10-09T09:14:01.307890+0000 mgr.y (mgr.34107) 208 : cluster 0 pgmap v406: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 178 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 887 KiB/s wr, 7 op/s; 58/348 objects degraded (16.667%) 2023-10-09T09:14:02.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[75425]: cluster 2023-10-09T09:14:02.087715+0000 mon.a (mon.0) 563 : cluster 3 Health check failed: Degraded data redundancy: 58/348 objects degraded (16.667%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:02.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[75425]: cluster 2023-10-09T09:14:02.095877+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:02.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[75425]: cluster 2023-10-09T09:14:02.099045+0000 mon.a (mon.0) 564 : cluster 0 osdmap e570: 8 total, 7 up, 8 in 2023-10-09T09:14:02.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[75425]: cluster 2023-10-09T09:14:02.110106+0000 osd.2 (osd.2) 169 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:02.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[78107]: cluster 2023-10-09T09:14:01.090028+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[78107]: cluster 2023-10-09T09:14:01.094170+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[78107]: cluster 2023-10-09T09:14:01.094339+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[78107]: cluster 2023-10-09T09:14:01.096067+0000 osd.3 (osd.3) 172 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[78107]: cluster 2023-10-09T09:14:01.096118+0000 osd.1 (osd.1) 83 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[78107]: cluster 2023-10-09T09:14:01.307890+0000 mgr.y (mgr.34107) 208 : cluster 0 pgmap v406: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 178 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 887 KiB/s wr, 7 op/s; 58/348 objects degraded (16.667%) 2023-10-09T09:14:02.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[78107]: cluster 2023-10-09T09:14:02.087715+0000 mon.a (mon.0) 563 : cluster 3 Health check failed: Degraded data redundancy: 58/348 objects degraded (16.667%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:02.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[78107]: cluster 2023-10-09T09:14:02.095877+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:02.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[78107]: cluster 2023-10-09T09:14:02.099045+0000 mon.a (mon.0) 564 : cluster 0 osdmap e570: 8 total, 7 up, 8 in 2023-10-09T09:14:02.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:02 smithi163 bash[78107]: cluster 2023-10-09T09:14:02.110106+0000 osd.2 (osd.2) 169 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:02 smithi181 bash[59345]: cluster 2023-10-09T09:14:01.090028+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:02 smithi181 bash[59345]: cluster 2023-10-09T09:14:01.094170+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:02 smithi181 bash[59345]: cluster 2023-10-09T09:14:01.094339+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:02 smithi181 bash[59345]: cluster 2023-10-09T09:14:01.096067+0000 osd.3 (osd.3) 172 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:02 smithi181 bash[59345]: cluster 2023-10-09T09:14:01.096118+0000 osd.1 (osd.1) 83 : cluster 3 failed to encode map e569 with expected crc 2023-10-09T09:14:02.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:02 smithi181 bash[59345]: cluster 2023-10-09T09:14:01.307890+0000 mgr.y (mgr.34107) 208 : cluster 0 pgmap v406: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 178 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 887 KiB/s wr, 7 op/s; 58/348 objects degraded (16.667%) 2023-10-09T09:14:02.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:02 smithi181 bash[59345]: cluster 2023-10-09T09:14:02.087715+0000 mon.a (mon.0) 563 : cluster 3 Health check failed: Degraded data redundancy: 58/348 objects degraded (16.667%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:02.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:02 smithi181 bash[59345]: cluster 2023-10-09T09:14:02.095877+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:02.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:02 smithi181 bash[59345]: cluster 2023-10-09T09:14:02.099045+0000 mon.a (mon.0) 564 : cluster 0 osdmap e570: 8 total, 7 up, 8 in 2023-10-09T09:14:02.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:02 smithi181 bash[59345]: cluster 2023-10-09T09:14:02.110106+0000 osd.2 (osd.2) 169 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:03.104 INFO:tasks.rados.rados.0.smithi163.stdout:1998: done (1 left) 2023-10-09T09:14:03.104 INFO:tasks.rados.rados.0.smithi163.stdout:1999: done (0 left) 2023-10-09T09:14:03.104 INFO:tasks.rados.rados.0.smithi163.stdout:2000: write oid 42 current snap is 256 2023-10-09T09:14:03.104 INFO:tasks.rados.rados.0.smithi163.stdout:2000: seq_num 753 ranges {655671=581444,2030588=625083} 2023-10-09T09:14:03.111 INFO:tasks.rados.rados.0.smithi163.stdout:2000: writing smithi16367185-42 from 655671 to 1237115 tid 1 2023-10-09T09:14:03.120 INFO:tasks.rados.rados.0.smithi163.stdout:2000: writing smithi16367185-42 from 2030588 to 2655671 tid 2 2023-10-09T09:14:03.120 INFO:tasks.rados.rados.0.smithi163.stdout:2001: delete oid 14 current snap is 256 2023-10-09T09:14:03.120 INFO:tasks.rados.rados.0.smithi163.stdout:2000: finishing write tid 1 to smithi16367185-42 2023-10-09T09:14:03.124 INFO:tasks.rados.rados.0.smithi163.stdout:2000: finishing write tid 2 to smithi16367185-42 2023-10-09T09:14:03.124 INFO:tasks.rados.rados.0.smithi163.stdout:2000: finishing write tid 3 to smithi16367185-42 2023-10-09T09:14:03.124 INFO:tasks.rados.rados.0.smithi163.stdout:2000: finishing write tid 4 to smithi16367185-42 2023-10-09T09:14:03.124 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 1470 (ObjNum 753 snap 256 seq_num 753) dirty exists 2023-10-09T09:14:03.124 INFO:tasks.rados.rados.0.smithi163.stdout:2000: left oid 42 (ObjNum 753 snap 256 seq_num 753) 2023-10-09T09:14:03.129 INFO:tasks.rados.rados.0.smithi163.stdout:2000: done (1 left) 2023-10-09T09:14:03.129 INFO:tasks.rados.rados.0.smithi163.stdout:2001: done (0 left) 2023-10-09T09:14:03.130 INFO:tasks.rados.rados.0.smithi163.stdout:2002: read oid 11 snap -1 2023-10-09T09:14:03.130 INFO:tasks.rados.rados.0.smithi163.stdout:2002: expect deleted 2023-10-09T09:14:03.130 INFO:tasks.rados.rados.0.smithi163.stdout:2003: write oid 24 current snap is 256 2023-10-09T09:14:03.130 INFO:tasks.rados.rados.0.smithi163.stdout:2003: seq_num 755 ranges {689285=731942,2147552=541733} 2023-10-09T09:14:03.141 INFO:tasks.rados.rados.0.smithi163.stdout:2003: writing smithi16367185-24 from 689285 to 1421227 tid 1 2023-10-09T09:14:03.151 INFO:tasks.rados.rados.0.smithi163.stdout:2003: writing smithi16367185-24 from 2147552 to 2689285 tid 2 2023-10-09T09:14:03.151 INFO:tasks.rados.rados.0.smithi163.stdout:2004: write oid 3 current snap is 256 2023-10-09T09:14:03.151 INFO:tasks.rados.rados.0.smithi163.stdout:2004: seq_num 756 ranges {706092=407191,1806034=463586} 2023-10-09T09:14:03.158 INFO:tasks.rados.rados.0.smithi163.stdout:2004: writing smithi16367185-3 from 706092 to 1113283 tid 1 2023-10-09T09:14:03.165 INFO:tasks.rados.rados.0.smithi163.stdout:2004: writing smithi16367185-3 from 1806034 to 2269620 tid 2 2023-10-09T09:14:03.166 INFO:tasks.rados.rados.0.smithi163.stdout:2005: write oid 42 current snap is 256 2023-10-09T09:14:03.166 INFO:tasks.rados.rados.0.smithi163.stdout:2005: seq_num 757 ranges {722899=482440,1748163=723597} 2023-10-09T09:14:03.172 INFO:tasks.rados.rados.0.smithi163.stdout:2005: writing smithi16367185-42 from 722899 to 1205339 tid 1 2023-10-09T09:14:03.180 INFO:tasks.rados.rados.0.smithi163.stdout:2005: writing smithi16367185-42 from 1748163 to 2471760 tid 2 2023-10-09T09:14:03.180 INFO:tasks.rados.rados.0.smithi163.stdout:2006: read oid 26 snap -1 2023-10-09T09:14:03.180 INFO:tasks.rados.rados.0.smithi163.stdout:2006: expect (ObjNum 731 snap 246 seq_num 731) 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:2007: snap_create 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:2003: finishing write tid 1 to smithi16367185-24 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:2003: finishing write tid 2 to smithi16367185-24 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:2003: finishing write tid 3 to smithi16367185-24 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:2004: finishing write tid 1 to smithi16367185-3 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:2003: finishing write tid 4 to smithi16367185-24 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 497 (ObjNum 755 snap 256 seq_num 755) dirty exists 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:2003: left oid 24 (ObjNum 755 snap 256 seq_num 755) 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:2004: finishing write tid 2 to smithi16367185-3 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:2004: finishing write tid 3 to smithi16367185-3 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:2004: finishing write tid 4 to smithi16367185-3 2023-10-09T09:14:03.181 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 519 (ObjNum 756 snap 256 seq_num 756) dirty exists 2023-10-09T09:14:03.182 INFO:tasks.rados.rados.0.smithi163.stdout:2004: left oid 3 (ObjNum 756 snap 256 seq_num 756) 2023-10-09T09:14:03.182 INFO:tasks.rados.rados.0.smithi163.stdout:2005: finishing write tid 1 to smithi16367185-42 2023-10-09T09:14:03.184 INFO:tasks.rados.rados.0.smithi163.stdout:2006: expect (ObjNum 731 snap 246 seq_num 731) 2023-10-09T09:14:03.324 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:03 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T09:14:03.324 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:03 smithi163 systemd[1]: Stopped Ceph osd.0 for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:14:03.324 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:03 smithi163 systemd[1]: Started Ceph osd.0 for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:14:03.420 INFO:tasks.rados.rados.0.smithi163.stdout:2005: finishing write tid 2 to smithi16367185-42 2023-10-09T09:14:03.420 INFO:tasks.rados.rados.0.smithi163.stdout:2005: finishing write tid 3 to smithi16367185-42 2023-10-09T09:14:03.421 INFO:tasks.rados.rados.0.smithi163.stdout:2005: finishing write tid 4 to smithi16367185-42 2023-10-09T09:14:03.421 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 1473 (ObjNum 757 snap 256 seq_num 757) dirty exists 2023-10-09T09:14:03.421 INFO:tasks.rados.rados.0.smithi163.stdout:2005: left oid 42 (ObjNum 757 snap 256 seq_num 757) 2023-10-09T09:14:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:14:03] "GET /metrics HTTP/1.1" 200 36076 "" "Prometheus/2.33.4" 2023-10-09T09:14:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: cluster 2023-10-09T09:14:02.095211+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: cluster 2023-10-09T09:14:02.099214+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: cluster 2023-10-09T09:14:02.127992+0000 osd.3 (osd.3) 173 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: cluster 2023-10-09T09:14:03.100169+0000 osd.2 (osd.2) 170 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: cluster 2023-10-09T09:14:03.104167+0000 mon.a (mon.0) 565 : cluster 0 osdmap e571: 8 total, 7 up, 8 in 2023-10-09T09:14:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: cluster 2023-10-09T09:14:03.120396+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: cluster 2023-10-09T09:14:03.129908+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: audit 2023-10-09T09:14:03.324574+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.163:0/3593498171' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: audit 2023-10-09T09:14:03.361471+0000 mon.a (mon.0) 566 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: audit 2023-10-09T09:14:03.368417+0000 mon.a (mon.0) 567 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[75425]: audit 2023-10-09T09:14:03.414937+0000 mon.a (mon.0) 568 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: cluster 2023-10-09T09:14:02.095211+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: cluster 2023-10-09T09:14:02.099214+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: cluster 2023-10-09T09:14:02.127992+0000 osd.3 (osd.3) 173 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: cluster 2023-10-09T09:14:03.100169+0000 osd.2 (osd.2) 170 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: cluster 2023-10-09T09:14:03.104167+0000 mon.a (mon.0) 565 : cluster 0 osdmap e571: 8 total, 7 up, 8 in 2023-10-09T09:14:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: cluster 2023-10-09T09:14:03.120396+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: cluster 2023-10-09T09:14:03.129908+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: audit 2023-10-09T09:14:03.324574+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.163:0/3593498171' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: audit 2023-10-09T09:14:03.361471+0000 mon.a (mon.0) 566 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: audit 2023-10-09T09:14:03.368417+0000 mon.a (mon.0) 567 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:03.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:03 smithi163 bash[78107]: audit 2023-10-09T09:14:03.414937+0000 mon.a (mon.0) 568 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: cluster 2023-10-09T09:14:02.095211+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: cluster 2023-10-09T09:14:02.099214+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: cluster 2023-10-09T09:14:02.127992+0000 osd.3 (osd.3) 173 : cluster 3 failed to encode map e570 with expected crc 2023-10-09T09:14:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: cluster 2023-10-09T09:14:03.100169+0000 osd.2 (osd.2) 170 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:03.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: cluster 2023-10-09T09:14:03.104167+0000 mon.a (mon.0) 565 : cluster 0 osdmap e571: 8 total, 7 up, 8 in 2023-10-09T09:14:03.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: cluster 2023-10-09T09:14:03.120396+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:03.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: cluster 2023-10-09T09:14:03.129908+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:03.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: audit 2023-10-09T09:14:03.324574+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.163:0/3593498171' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:03.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: audit 2023-10-09T09:14:03.361471+0000 mon.a (mon.0) 566 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:03.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: audit 2023-10-09T09:14:03.368417+0000 mon.a (mon.0) 567 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:03.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:03 smithi181 bash[59345]: audit 2023-10-09T09:14:03.414937+0000 mon.a (mon.0) 568 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:04.109 INFO:tasks.rados.rados.0.smithi163.stdout:2002: done (5 left) 2023-10-09T09:14:04.109 INFO:tasks.rados.rados.0.smithi163.stdout:2003: done (4 left) 2023-10-09T09:14:04.109 INFO:tasks.rados.rados.0.smithi163.stdout:2004: done (3 left) 2023-10-09T09:14:04.109 INFO:tasks.rados.rados.0.smithi163.stdout:2005: done (2 left) 2023-10-09T09:14:04.109 INFO:tasks.rados.rados.0.smithi163.stdout:2006: done (1 left) 2023-10-09T09:14:04.110 INFO:tasks.rados.rados.0.smithi163.stdout:2007: done (0 left) 2023-10-09T09:14:04.110 INFO:tasks.rados.rados.0.smithi163.stdout:2008: snap_remove snap 244 2023-10-09T09:14:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[75425]: cluster 2023-10-09T09:14:03.097751+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[75425]: cluster 2023-10-09T09:14:03.144175+0000 osd.3 (osd.3) 174 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[75425]: cluster 2023-10-09T09:14:03.308600+0000 mgr.y (mgr.34107) 209 : cluster 0 pgmap v409: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 178 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 887 KiB/s wr, 7 op/s; 58/348 objects degraded (16.667%) 2023-10-09T09:14:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[75425]: cluster 2023-10-09T09:14:04.106317+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[75425]: cluster 2023-10-09T09:14:04.109265+0000 mon.a (mon.0) 569 : cluster 0 osdmap e572: 8 total, 7 up, 8 in 2023-10-09T09:14:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[78107]: cluster 2023-10-09T09:14:03.097751+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:04.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[78107]: cluster 2023-10-09T09:14:03.144175+0000 osd.3 (osd.3) 174 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:04.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[78107]: cluster 2023-10-09T09:14:03.308600+0000 mgr.y (mgr.34107) 209 : cluster 0 pgmap v409: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 178 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 887 KiB/s wr, 7 op/s; 58/348 objects degraded (16.667%) 2023-10-09T09:14:04.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[78107]: cluster 2023-10-09T09:14:04.106317+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:04.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[78107]: cluster 2023-10-09T09:14:04.109265+0000 mon.a (mon.0) 569 : cluster 0 osdmap e572: 8 total, 7 up, 8 in 2023-10-09T09:14:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:04 smithi181 bash[59345]: cluster 2023-10-09T09:14:03.097751+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:04 smithi181 bash[59345]: cluster 2023-10-09T09:14:03.144175+0000 osd.3 (osd.3) 174 : cluster 3 failed to encode map e571 with expected crc 2023-10-09T09:14:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:04 smithi181 bash[59345]: cluster 2023-10-09T09:14:03.308600+0000 mgr.y (mgr.34107) 209 : cluster 0 pgmap v409: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 178 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 887 KiB/s wr, 7 op/s; 58/348 objects degraded (16.667%) 2023-10-09T09:14:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:04 smithi181 bash[59345]: cluster 2023-10-09T09:14:04.106317+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:04 smithi181 bash[59345]: cluster 2023-10-09T09:14:04.109265+0000 mon.a (mon.0) 569 : cluster 0 osdmap e572: 8 total, 7 up, 8 in 2023-10-09T09:14:05.085 INFO:tasks.rados.rados.0.smithi163.stdout:2008: done (0 left) 2023-10-09T09:14:05.085 INFO:tasks.rados.rados.0.smithi163.stdout:2009: delete oid 10 current snap is 257 2023-10-09T09:14:05.100 INFO:tasks.rados.rados.0.smithi163.stdout:2009: done (0 left) 2023-10-09T09:14:05.100 INFO:tasks.rados.rados.0.smithi163.stdout:2010: read oid 18 snap -1 2023-10-09T09:14:05.100 INFO:tasks.rados.rados.0.smithi163.stdout:2010: expect (ObjNum 734 snap 249 seq_num 734) 2023-10-09T09:14:05.101 INFO:tasks.rados.rados.0.smithi163.stdout:2011: read oid 28 snap -1 2023-10-09T09:14:05.101 INFO:tasks.rados.rados.0.smithi163.stdout:2011: expect deleted 2023-10-09T09:14:05.101 INFO:tasks.rados.rados.0.smithi163.stdout:2012: snap_remove snap 232 2023-10-09T09:14:05.218 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[83771]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T09:14:05.218 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[83771]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T09:14:05.218 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[83771]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T09:14:05.218 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[83771]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T09:14:05.219 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[83771]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T09:14:05.219 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[83771]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T09:14:05.219 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[83771]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T09:14:05.219 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:04 smithi163 bash[83771]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T09:14:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:05 smithi163 bash[78107]: cluster 2023-10-09T09:14:04.106127+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:05 smithi163 bash[78107]: cluster 2023-10-09T09:14:04.709353+0000 osd.2 (osd.2) 171 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:05 smithi163 bash[78107]: cluster 2023-10-09T09:14:05.080071+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e573 with expected crc 2023-10-09T09:14:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:05 smithi163 bash[78107]: cluster 2023-10-09T09:14:05.085022+0000 mon.a (mon.0) 570 : cluster 0 osdmap e573: 8 total, 7 up, 8 in 2023-10-09T09:14:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:05 smithi163 bash[78107]: cluster 2023-10-09T09:14:05.087804+0000 osd.2 (osd.2) 172 : cluster 3 failed to encode map e573 with expected crc 2023-10-09T09:14:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:05 smithi163 bash[75425]: cluster 2023-10-09T09:14:04.106127+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:05 smithi163 bash[75425]: cluster 2023-10-09T09:14:04.709353+0000 osd.2 (osd.2) 171 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:05.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:05 smithi163 bash[75425]: cluster 2023-10-09T09:14:05.080071+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e573 with expected crc 2023-10-09T09:14:05.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:05 smithi163 bash[75425]: cluster 2023-10-09T09:14:05.085022+0000 mon.a (mon.0) 570 : cluster 0 osdmap e573: 8 total, 7 up, 8 in 2023-10-09T09:14:05.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:05 smithi163 bash[75425]: cluster 2023-10-09T09:14:05.087804+0000 osd.2 (osd.2) 172 : cluster 3 failed to encode map e573 with expected crc 2023-10-09T09:14:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:05 smithi181 bash[59345]: cluster 2023-10-09T09:14:04.106127+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:05 smithi181 bash[59345]: cluster 2023-10-09T09:14:04.709353+0000 osd.2 (osd.2) 171 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:05 smithi181 bash[59345]: cluster 2023-10-09T09:14:05.080071+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e573 with expected crc 2023-10-09T09:14:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:05 smithi181 bash[59345]: cluster 2023-10-09T09:14:05.085022+0000 mon.a (mon.0) 570 : cluster 0 osdmap e573: 8 total, 7 up, 8 in 2023-10-09T09:14:05.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:05 smithi181 bash[59345]: cluster 2023-10-09T09:14:05.087804+0000 osd.2 (osd.2) 172 : cluster 3 failed to encode map e573 with expected crc 2023-10-09T09:14:06.101 INFO:tasks.rados.rados.0.smithi163.stdout:2012: done (2 left) 2023-10-09T09:14:06.102 INFO:tasks.rados.rados.0.smithi163.stdout:2013: read oid 10 snap -1 2023-10-09T09:14:06.102 INFO:tasks.rados.rados.0.smithi163.stdout:2013: expect deleted 2023-10-09T09:14:06.102 INFO:tasks.rados.rados.0.smithi163.stdout:2014: write oid 6 current snap is 257 2023-10-09T09:14:06.102 INFO:tasks.rados.rados.0.smithi163.stdout:2014: seq_num 759 ranges {756513=632938,1865127=727266} 2023-10-09T09:14:06.109 INFO:tasks.rados.rados.0.smithi163.stdout:2014: writing smithi16367185-6 from 756513 to 1389451 tid 1 2023-10-09T09:14:06.117 INFO:tasks.rados.rados.0.smithi163.stdout:2014: writing smithi16367185-6 from 1865127 to 2592393 tid 2 2023-10-09T09:14:06.117 INFO:tasks.rados.rados.0.smithi163.stdout:2015: read oid 45 snap -1 2023-10-09T09:14:06.118 INFO:tasks.rados.rados.0.smithi163.stdout:2015: expect (ObjNum 741 snap 252 seq_num 741) 2023-10-09T09:14:06.118 INFO:tasks.rados.rados.0.smithi163.stdout:2016: read oid 49 snap -1 2023-10-09T09:14:06.118 INFO:tasks.rados.rados.0.smithi163.stdout:2016: expect deleted 2023-10-09T09:14:06.118 INFO:tasks.rados.rados.0.smithi163.stdout:2017: snap_create 2023-10-09T09:14:06.118 INFO:tasks.rados.rados.0.smithi163.stdout:2010: expect (ObjNum 734 snap 249 seq_num 734) 2023-10-09T09:14:06.200 INFO:tasks.rados.rados.0.smithi163.stdout:2014: finishing write tid 1 to smithi16367185-6 2023-10-09T09:14:06.201 INFO:tasks.rados.rados.0.smithi163.stdout:2014: finishing write tid 2 to smithi16367185-6 2023-10-09T09:14:06.201 INFO:tasks.rados.rados.0.smithi163.stdout:2014: finishing write tid 3 to smithi16367185-6 2023-10-09T09:14:06.201 INFO:tasks.rados.rados.0.smithi163.stdout:2014: finishing write tid 4 to smithi16367185-6 2023-10-09T09:14:06.201 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 198 (ObjNum 759 snap 257 seq_num 759) dirty exists 2023-10-09T09:14:06.201 INFO:tasks.rados.rados.0.smithi163.stdout:2014: left oid 6 (ObjNum 759 snap 257 seq_num 759) 2023-10-09T09:14:06.201 INFO:tasks.rados.rados.0.smithi163.stdout:2015: expect (ObjNum 741 snap 252 seq_num 741) 2023-10-09T09:14:06.718 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[84060]: debug 2023-10-09T09:14:06.520+0000 7f802445d700 1 -- 172.21.15.163:0/1346021024 <== mon.2 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562d68e944e0 con 0x562d68eeec00 2023-10-09T09:14:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[78107]: cluster 2023-10-09T09:14:04.981670+0000 osd.1 (osd.1) 84 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[78107]: cluster 2023-10-09T09:14:05.079967+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e573 with expected crc 2023-10-09T09:14:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[78107]: cluster 2023-10-09T09:14:05.309085+0000 mgr.y (mgr.34107) 210 : cluster 0 pgmap v412: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 178 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 58/348 objects degraded (16.667%) 2023-10-09T09:14:06.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[78107]: audit 2023-10-09T09:14:05.682748+0000 mon.a (mon.0) 571 : audit 0 from='client.? 172.21.15.163:0/2600207729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:06.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[78107]: cluster 2023-10-09T09:14:06.100877+0000 mon.a (mon.0) 572 : cluster 0 osdmap e574: 8 total, 7 up, 8 in 2023-10-09T09:14:06.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[78107]: cluster 2023-10-09T09:14:06.101840+0000 osd.2 (osd.2) 173 : cluster 3 failed to encode map e574 with expected crc 2023-10-09T09:14:06.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[78107]: cluster 2023-10-09T09:14:06.104508+0000 osd.2 (osd.2) 174 : cluster 3 failed to encode map e574 with expected crc 2023-10-09T09:14:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[75425]: cluster 2023-10-09T09:14:04.981670+0000 osd.1 (osd.1) 84 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[75425]: cluster 2023-10-09T09:14:05.079967+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e573 with expected crc 2023-10-09T09:14:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[75425]: cluster 2023-10-09T09:14:05.309085+0000 mgr.y (mgr.34107) 210 : cluster 0 pgmap v412: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 178 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 58/348 objects degraded (16.667%) 2023-10-09T09:14:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[75425]: audit 2023-10-09T09:14:05.682748+0000 mon.a (mon.0) 571 : audit 0 from='client.? 172.21.15.163:0/2600207729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:06.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[75425]: cluster 2023-10-09T09:14:06.100877+0000 mon.a (mon.0) 572 : cluster 0 osdmap e574: 8 total, 7 up, 8 in 2023-10-09T09:14:06.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[75425]: cluster 2023-10-09T09:14:06.101840+0000 osd.2 (osd.2) 173 : cluster 3 failed to encode map e574 with expected crc 2023-10-09T09:14:06.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:06 smithi163 bash[75425]: cluster 2023-10-09T09:14:06.104508+0000 osd.2 (osd.2) 174 : cluster 3 failed to encode map e574 with expected crc 2023-10-09T09:14:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:06 smithi181 bash[59345]: cluster 2023-10-09T09:14:04.981670+0000 osd.1 (osd.1) 84 : cluster 3 failed to encode map e572 with expected crc 2023-10-09T09:14:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:06 smithi181 bash[59345]: cluster 2023-10-09T09:14:05.079967+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e573 with expected crc 2023-10-09T09:14:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:06 smithi181 bash[59345]: cluster 2023-10-09T09:14:05.309085+0000 mgr.y (mgr.34107) 210 : cluster 0 pgmap v412: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 178 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 58/348 objects degraded (16.667%) 2023-10-09T09:14:06.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:06 smithi181 bash[59345]: audit 2023-10-09T09:14:05.682748+0000 mon.a (mon.0) 571 : audit 0 from='client.? 172.21.15.163:0/2600207729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:06.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:06 smithi181 bash[59345]: cluster 2023-10-09T09:14:06.100877+0000 mon.a (mon.0) 572 : cluster 0 osdmap e574: 8 total, 7 up, 8 in 2023-10-09T09:14:06.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:06 smithi181 bash[59345]: cluster 2023-10-09T09:14:06.101840+0000 osd.2 (osd.2) 173 : cluster 3 failed to encode map e574 with expected crc 2023-10-09T09:14:06.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:06 smithi181 bash[59345]: cluster 2023-10-09T09:14:06.104508+0000 osd.2 (osd.2) 174 : cluster 3 failed to encode map e574 with expected crc 2023-10-09T09:14:07.106 INFO:tasks.rados.rados.0.smithi163.stdout:2010: done (6 left) 2023-10-09T09:14:07.106 INFO:tasks.rados.rados.0.smithi163.stdout:2011: done (5 left) 2023-10-09T09:14:07.106 INFO:tasks.rados.rados.0.smithi163.stdout:2013: done (4 left) 2023-10-09T09:14:07.106 INFO:tasks.rados.rados.0.smithi163.stdout:2014: done (3 left) 2023-10-09T09:14:07.106 INFO:tasks.rados.rados.0.smithi163.stdout:2015: done (2 left) 2023-10-09T09:14:07.106 INFO:tasks.rados.rados.0.smithi163.stdout:2016: done (1 left) 2023-10-09T09:14:07.107 INFO:tasks.rados.rados.0.smithi163.stdout:2017: done (0 left) 2023-10-09T09:14:07.107 INFO:tasks.rados.rados.0.smithi163.stdout:2018: rollback oid 21 current snap is 258 2023-10-09T09:14:07.107 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 237 2023-10-09T09:14:07.107 INFO:tasks.rados.rados.0.smithi163.stdout:2019: snap_create 2023-10-09T09:14:07.111 INFO:tasks.rados.rados.0.smithi163.stdout:2018: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:14:07.111 INFO:tasks.rados.rados.0.smithi163.stdout:2018: finishing rollback tid 2 to smithi16367185-21 2023-10-09T09:14:07.111 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 466 (ObjNum 707 snap 233 seq_num 707) dirty exists 2023-10-09T09:14:07.429 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:07 smithi163 bash[84060]: debug 2023-10-09T09:14:07.068+0000 7f802d5e3640 -1 Falling back to public interface 2023-10-09T09:14:07.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:07 smithi163 bash[78107]: cluster 2023-10-09T09:14:06.097854+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e574 with expected crc 2023-10-09T09:14:07.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:07 smithi163 bash[78107]: cluster 2023-10-09T09:14:07.100889+0000 osd.2 (osd.2) 175 : cluster 3 failed to encode map e575 with expected crc 2023-10-09T09:14:07.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:07 smithi163 bash[78107]: cluster 2023-10-09T09:14:07.105668+0000 mon.a (mon.0) 573 : cluster 0 osdmap e575: 8 total, 7 up, 8 in 2023-10-09T09:14:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:07 smithi163 bash[75425]: cluster 2023-10-09T09:14:06.097854+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e574 with expected crc 2023-10-09T09:14:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:07 smithi163 bash[75425]: cluster 2023-10-09T09:14:07.100889+0000 osd.2 (osd.2) 175 : cluster 3 failed to encode map e575 with expected crc 2023-10-09T09:14:07.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:07 smithi163 bash[75425]: cluster 2023-10-09T09:14:07.105668+0000 mon.a (mon.0) 573 : cluster 0 osdmap e575: 8 total, 7 up, 8 in 2023-10-09T09:14:07.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:07 smithi181 bash[59345]: cluster 2023-10-09T09:14:06.097854+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e574 with expected crc 2023-10-09T09:14:07.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:07 smithi181 bash[59345]: cluster 2023-10-09T09:14:07.100889+0000 osd.2 (osd.2) 175 : cluster 3 failed to encode map e575 with expected crc 2023-10-09T09:14:07.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:07 smithi181 bash[59345]: cluster 2023-10-09T09:14:07.105668+0000 mon.a (mon.0) 573 : cluster 0 osdmap e575: 8 total, 7 up, 8 in 2023-10-09T09:14:08.109 INFO:tasks.rados.rados.0.smithi163.stdout:2018: done (1 left) 2023-10-09T09:14:08.109 INFO:tasks.rados.rados.0.smithi163.stdout:2019: done (0 left) 2023-10-09T09:14:08.109 INFO:tasks.rados.rados.0.smithi163.stdout:2020: read oid 43 snap -1 2023-10-09T09:14:08.110 INFO:tasks.rados.rados.0.smithi163.stdout:2020: expect (ObjNum 733 snap 246 seq_num 733) 2023-10-09T09:14:08.110 INFO:tasks.rados.rados.0.smithi163.stdout:2021: write oid 8 current snap is 259 2023-10-09T09:14:08.110 INFO:tasks.rados.rados.0.smithi163.stdout:2021: seq_num 760 ranges {773320=708187,1923609=470924} 2023-10-09T09:14:08.121 INFO:tasks.rados.rados.0.smithi163.stdout:2021: writing smithi16367185-8 from 773320 to 1481507 tid 1 2023-10-09T09:14:08.127 INFO:tasks.rados.rados.0.smithi163.stdout:2021: writing smithi16367185-8 from 1923609 to 2394533 tid 2 2023-10-09T09:14:08.127 INFO:tasks.rados.rados.0.smithi163.stdout:2022: snap_remove snap 237 2023-10-09T09:14:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[78107]: cluster 2023-10-09T09:14:07.100724+0000 osd.1 (osd.1) 85 : cluster 3 failed to encode map e575 with expected crc 2023-10-09T09:14:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[78107]: cluster 2023-10-09T09:14:07.309943+0000 mgr.y (mgr.34107) 211 : cluster 0 pgmap v415: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 62/360 objects degraded (17.222%) 2023-10-09T09:14:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[78107]: audit 2023-10-09T09:14:08.048844+0000 mon.a (mon.0) 574 : audit 0 from='client.? 172.21.15.163:0/3200104180' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[78107]: cluster 2023-10-09T09:14:08.098624+0000 mon.a (mon.0) 575 : cluster 3 Health check update: Degraded data redundancy: 62/360 objects degraded (17.222%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:08.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[78107]: cluster 2023-10-09T09:14:08.109478+0000 mon.a (mon.0) 576 : cluster 0 osdmap e576: 8 total, 7 up, 8 in 2023-10-09T09:14:08.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[78107]: cluster 2023-10-09T09:14:08.110431+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:08.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[78107]: cluster 2023-10-09T09:14:08.122639+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[75425]: cluster 2023-10-09T09:14:07.100724+0000 osd.1 (osd.1) 85 : cluster 3 failed to encode map e575 with expected crc 2023-10-09T09:14:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[75425]: cluster 2023-10-09T09:14:07.309943+0000 mgr.y (mgr.34107) 211 : cluster 0 pgmap v415: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 62/360 objects degraded (17.222%) 2023-10-09T09:14:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[75425]: audit 2023-10-09T09:14:08.048844+0000 mon.a (mon.0) 574 : audit 0 from='client.? 172.21.15.163:0/3200104180' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[75425]: cluster 2023-10-09T09:14:08.098624+0000 mon.a (mon.0) 575 : cluster 3 Health check update: Degraded data redundancy: 62/360 objects degraded (17.222%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[75425]: cluster 2023-10-09T09:14:08.109478+0000 mon.a (mon.0) 576 : cluster 0 osdmap e576: 8 total, 7 up, 8 in 2023-10-09T09:14:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[75425]: cluster 2023-10-09T09:14:08.110431+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:08.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:08 smithi163 bash[75425]: cluster 2023-10-09T09:14:08.122639+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:08 smithi181 bash[59345]: cluster 2023-10-09T09:14:07.100724+0000 osd.1 (osd.1) 85 : cluster 3 failed to encode map e575 with expected crc 2023-10-09T09:14:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:08 smithi181 bash[59345]: cluster 2023-10-09T09:14:07.309943+0000 mgr.y (mgr.34107) 211 : cluster 0 pgmap v415: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 62/360 objects degraded (17.222%) 2023-10-09T09:14:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:08 smithi181 bash[59345]: audit 2023-10-09T09:14:08.048844+0000 mon.a (mon.0) 574 : audit 0 from='client.? 172.21.15.163:0/3200104180' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:08 smithi181 bash[59345]: cluster 2023-10-09T09:14:08.098624+0000 mon.a (mon.0) 575 : cluster 3 Health check update: Degraded data redundancy: 62/360 objects degraded (17.222%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:08.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:08 smithi181 bash[59345]: cluster 2023-10-09T09:14:08.109478+0000 mon.a (mon.0) 576 : cluster 0 osdmap e576: 8 total, 7 up, 8 in 2023-10-09T09:14:08.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:08 smithi181 bash[59345]: cluster 2023-10-09T09:14:08.110431+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:08.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:08 smithi181 bash[59345]: cluster 2023-10-09T09:14:08.122639+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:09.114 INFO:tasks.rados.rados.0.smithi163.stdout:2022: done (2 left) 2023-10-09T09:14:09.115 INFO:tasks.rados.rados.0.smithi163.stdout:2023: write oid 25 current snap is 259 2023-10-09T09:14:09.115 INFO:tasks.rados.rados.0.smithi163.stdout:2023: seq_num 761 ranges {790127=499789,1698444=730935} 2023-10-09T09:14:09.123 INFO:tasks.rados.rados.0.smithi163.stdout:2023: writing smithi16367185-25 from 790127 to 1289916 tid 1 2023-10-09T09:14:09.131 INFO:tasks.rados.rados.0.smithi163.stdout:2023: writing smithi16367185-25 from 1698444 to 2429379 tid 2 2023-10-09T09:14:09.131 INFO:tasks.rados.rados.0.smithi163.stdout:2024: rollback oid 14 current snap is 259 2023-10-09T09:14:09.131 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 252 2023-10-09T09:14:09.131 INFO:tasks.rados.rados.0.smithi163.stdout:2020: expect (ObjNum 733 snap 246 seq_num 733) 2023-10-09T09:14:09.265 INFO:tasks.rados.rados.0.smithi163.stdout:2021: finishing write tid 1 to smithi16367185-8 2023-10-09T09:14:09.265 INFO:tasks.rados.rados.0.smithi163.stdout:2021: finishing write tid 2 to smithi16367185-8 2023-10-09T09:14:09.265 INFO:tasks.rados.rados.0.smithi163.stdout:2021: finishing write tid 3 to smithi16367185-8 2023-10-09T09:14:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:2021: finishing write tid 4 to smithi16367185-8 2023-10-09T09:14:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1001 (ObjNum 760 snap 259 seq_num 760) dirty exists 2023-10-09T09:14:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:2021: left oid 8 (ObjNum 760 snap 259 seq_num 760) 2023-10-09T09:14:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:2023: finishing write tid 1 to smithi16367185-25 2023-10-09T09:14:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:2020: done (3 left) 2023-10-09T09:14:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:2021: done (2 left) 2023-10-09T09:14:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:2025: delete oid 29 current snap is 259 2023-10-09T09:14:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:2023: finishing write tid 2 to smithi16367185-25 2023-10-09T09:14:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:2023: finishing write tid 3 to smithi16367185-25 2023-10-09T09:14:09.266 INFO:tasks.rados.rados.0.smithi163.stdout:2023: finishing write tid 4 to smithi16367185-25 2023-10-09T09:14:09.267 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 937 (ObjNum 761 snap 259 seq_num 761) dirty exists 2023-10-09T09:14:09.267 INFO:tasks.rados.rados.0.smithi163.stdout:2023: left oid 25 (ObjNum 761 snap 259 seq_num 761) 2023-10-09T09:14:09.267 INFO:tasks.rados.rados.0.smithi163.stdout:2024: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:14:09.267 INFO:tasks.rados.rados.0.smithi163.stdout:2024: finishing rollback tid 2 to smithi16367185-14 2023-10-09T09:14:09.267 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 800 (ObjNum 622 snap 197 seq_num 622) dirty exists 2023-10-09T09:14:09.267 INFO:tasks.rados.rados.0.smithi163.stdout:2023: done (2 left) 2023-10-09T09:14:09.267 INFO:tasks.rados.rados.0.smithi163.stdout:2024: done (1 left) 2023-10-09T09:14:09.268 INFO:tasks.rados.rados.0.smithi163.stdout:2025: done (0 left) 2023-10-09T09:14:09.268 INFO:tasks.rados.rados.0.smithi163.stdout:2026: snap_remove snap 257 2023-10-09T09:14:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:09 smithi163 bash[78107]: cluster 2023-10-09T09:14:08.106704+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:09 smithi163 bash[78107]: cluster 2023-10-09T09:14:08.107740+0000 osd.1 (osd.1) 86 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:09 smithi163 bash[78107]: cluster 2023-10-09T09:14:09.110027+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e577 with expected crc 2023-10-09T09:14:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:09 smithi163 bash[78107]: cluster 2023-10-09T09:14:09.114187+0000 mon.a (mon.0) 577 : cluster 0 osdmap e577: 8 total, 7 up, 8 in 2023-10-09T09:14:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:09 smithi163 bash[78107]: cluster 2023-10-09T09:14:09.125000+0000 osd.2 (osd.2) 176 : cluster 3 failed to encode map e577 with expected crc 2023-10-09T09:14:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:09 smithi163 bash[75425]: cluster 2023-10-09T09:14:08.106704+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:09 smithi163 bash[75425]: cluster 2023-10-09T09:14:08.107740+0000 osd.1 (osd.1) 86 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:09 smithi163 bash[75425]: cluster 2023-10-09T09:14:09.110027+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e577 with expected crc 2023-10-09T09:14:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:09 smithi163 bash[75425]: cluster 2023-10-09T09:14:09.114187+0000 mon.a (mon.0) 577 : cluster 0 osdmap e577: 8 total, 7 up, 8 in 2023-10-09T09:14:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:09 smithi163 bash[75425]: cluster 2023-10-09T09:14:09.125000+0000 osd.2 (osd.2) 176 : cluster 3 failed to encode map e577 with expected crc 2023-10-09T09:14:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:09 smithi181 bash[59345]: cluster 2023-10-09T09:14:08.106704+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:09 smithi181 bash[59345]: cluster 2023-10-09T09:14:08.107740+0000 osd.1 (osd.1) 86 : cluster 3 failed to encode map e576 with expected crc 2023-10-09T09:14:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:09 smithi181 bash[59345]: cluster 2023-10-09T09:14:09.110027+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e577 with expected crc 2023-10-09T09:14:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:09 smithi181 bash[59345]: cluster 2023-10-09T09:14:09.114187+0000 mon.a (mon.0) 577 : cluster 0 osdmap e577: 8 total, 7 up, 8 in 2023-10-09T09:14:09.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:09 smithi181 bash[59345]: cluster 2023-10-09T09:14:09.125000+0000 osd.2 (osd.2) 176 : cluster 3 failed to encode map e577 with expected crc 2023-10-09T09:14:10.129 INFO:tasks.rados.rados.0.smithi163.stdout:2026: done (0 left) 2023-10-09T09:14:10.129 INFO:tasks.rados.rados.0.smithi163.stdout:2027: write oid 7 current snap is 259 2023-10-09T09:14:10.129 INFO:tasks.rados.rados.0.smithi163.stdout:2027: seq_num 763 ranges {423741=650287,1699055=618251} 2023-10-09T09:14:10.135 INFO:tasks.rados.rados.0.smithi163.stdout:2027: writing smithi16367185-7 from 423741 to 1074028 tid 1 2023-10-09T09:14:10.144 INFO:tasks.rados.rados.0.smithi163.stdout:2027: writing smithi16367185-7 from 1699055 to 2317306 tid 2 2023-10-09T09:14:10.144 INFO:tasks.rados.rados.0.smithi163.stdout:2028: rollback oid 45 current snap is 259 2023-10-09T09:14:10.144 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 45 to 239 2023-10-09T09:14:10.144 INFO:tasks.rados.rados.0.smithi163.stdout:2027: finishing write tid 1 to smithi16367185-7 2023-10-09T09:14:10.144 INFO:tasks.rados.rados.0.smithi163.stdout:2029: read oid 17 snap -1 2023-10-09T09:14:10.144 INFO:tasks.rados.rados.0.smithi163.stdout:2029: expect deleted 2023-10-09T09:14:10.144 INFO:tasks.rados.rados.0.smithi163.stdout:2030: read oid 32 snap -1 2023-10-09T09:14:10.144 INFO:tasks.rados.rados.0.smithi163.stdout:2030: expect (ObjNum 750 snap 255 seq_num 750) 2023-10-09T09:14:10.145 INFO:tasks.rados.rados.0.smithi163.stdout:2031: snap_create 2023-10-09T09:14:10.147 INFO:tasks.rados.rados.0.smithi163.stdout:2027: finishing write tid 2 to smithi16367185-7 2023-10-09T09:14:10.147 INFO:tasks.rados.rados.0.smithi163.stdout:2027: finishing write tid 3 to smithi16367185-7 2023-10-09T09:14:10.148 INFO:tasks.rados.rados.0.smithi163.stdout:2027: finishing write tid 4 to smithi16367185-7 2023-10-09T09:14:10.148 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 470 (ObjNum 763 snap 259 seq_num 763) dirty exists 2023-10-09T09:14:10.148 INFO:tasks.rados.rados.0.smithi163.stdout:2027: left oid 7 (ObjNum 763 snap 259 seq_num 763) 2023-10-09T09:14:10.161 INFO:tasks.rados.rados.0.smithi163.stdout:2030: expect (ObjNum 750 snap 255 seq_num 750) 2023-10-09T09:14:10.346 INFO:tasks.rados.rados.0.smithi163.stdout:2028: finishing rollback tid 0 to smithi16367185-45 2023-10-09T09:14:10.347 INFO:tasks.rados.rados.0.smithi163.stdout:2028: finishing rollback tid 1 to smithi16367185-45 2023-10-09T09:14:10.347 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 1476 (ObjNum 699113640 snap 21875 seq_num 0) dirty dne 2023-10-09T09:14:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: cluster 2023-10-09T09:14:09.110726+0000 osd.3 (osd.3) 175 : cluster 3 failed to encode map e577 with expected crc 2023-10-09T09:14:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: cluster 2023-10-09T09:14:09.310732+0000 mgr.y (mgr.34107) 212 : cluster 0 pgmap v418: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 62/360 objects degraded (17.222%) 2023-10-09T09:14:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: audit 2023-10-09T09:14:09.460489+0000 mon.a (mon.0) 578 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: audit 2023-10-09T09:14:09.468029+0000 mon.a (mon.0) 579 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: cluster 2023-10-09T09:14:10.124653+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: cluster 2023-10-09T09:14:10.128815+0000 mon.a (mon.0) 580 : cluster 0 osdmap e578: 8 total, 7 up, 8 in 2023-10-09T09:14:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: cluster 2023-10-09T09:14:10.144026+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: cluster 2023-10-09T09:14:10.144118+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: audit 2023-10-09T09:14:10.259515+0000 mon.a (mon.0) 581 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: audit 2023-10-09T09:14:10.266860+0000 mon.a (mon.0) 582 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: audit 2023-10-09T09:14:10.346226+0000 mon.a (mon.0) 583 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[78107]: audit 2023-10-09T09:14:10.348068+0000 mon.a (mon.0) 584 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: cluster 2023-10-09T09:14:09.110726+0000 osd.3 (osd.3) 175 : cluster 3 failed to encode map e577 with expected crc 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: cluster 2023-10-09T09:14:09.310732+0000 mgr.y (mgr.34107) 212 : cluster 0 pgmap v418: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 62/360 objects degraded (17.222%) 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: audit 2023-10-09T09:14:09.460489+0000 mon.a (mon.0) 578 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: audit 2023-10-09T09:14:09.468029+0000 mon.a (mon.0) 579 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: cluster 2023-10-09T09:14:10.124653+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: cluster 2023-10-09T09:14:10.128815+0000 mon.a (mon.0) 580 : cluster 0 osdmap e578: 8 total, 7 up, 8 in 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: cluster 2023-10-09T09:14:10.144026+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: cluster 2023-10-09T09:14:10.144118+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: audit 2023-10-09T09:14:10.259515+0000 mon.a (mon.0) 581 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: audit 2023-10-09T09:14:10.266860+0000 mon.a (mon.0) 582 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: audit 2023-10-09T09:14:10.346226+0000 mon.a (mon.0) 583 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:10 smithi163 bash[75425]: audit 2023-10-09T09:14:10.348068+0000 mon.a (mon.0) 584 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: cluster 2023-10-09T09:14:09.110726+0000 osd.3 (osd.3) 175 : cluster 3 failed to encode map e577 with expected crc 2023-10-09T09:14:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: cluster 2023-10-09T09:14:09.310732+0000 mgr.y (mgr.34107) 212 : cluster 0 pgmap v418: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 62/360 objects degraded (17.222%) 2023-10-09T09:14:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: audit 2023-10-09T09:14:09.460489+0000 mon.a (mon.0) 578 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: audit 2023-10-09T09:14:09.468029+0000 mon.a (mon.0) 579 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: cluster 2023-10-09T09:14:10.124653+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:10.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: cluster 2023-10-09T09:14:10.128815+0000 mon.a (mon.0) 580 : cluster 0 osdmap e578: 8 total, 7 up, 8 in 2023-10-09T09:14:10.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: cluster 2023-10-09T09:14:10.144026+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:10.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: cluster 2023-10-09T09:14:10.144118+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:10.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: audit 2023-10-09T09:14:10.259515+0000 mon.a (mon.0) 581 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: audit 2023-10-09T09:14:10.266860+0000 mon.a (mon.0) 582 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: audit 2023-10-09T09:14:10.346226+0000 mon.a (mon.0) 583 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:10.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:10 smithi181 bash[59345]: audit 2023-10-09T09:14:10.348068+0000 mon.a (mon.0) 584 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:11.126 INFO:tasks.rados.rados.0.smithi163.stdout:2027: done (4 left) 2023-10-09T09:14:11.126 INFO:tasks.rados.rados.0.smithi163.stdout:2028: done (3 left) 2023-10-09T09:14:11.126 INFO:tasks.rados.rados.0.smithi163.stdout:2029: done (2 left) 2023-10-09T09:14:11.126 INFO:tasks.rados.rados.0.smithi163.stdout:2030: done (1 left) 2023-10-09T09:14:11.126 INFO:tasks.rados.rados.0.smithi163.stdout:2031: done (0 left) 2023-10-09T09:14:11.127 INFO:tasks.rados.rados.0.smithi163.stdout:2032: write oid 20 current snap is 260 2023-10-09T09:14:11.127 INFO:tasks.rados.rados.0.smithi163.stdout:2032: seq_num 764 ranges {440548=725536,1757537=478262,2813990=26558} 2023-10-09T09:14:11.133 INFO:tasks.rados.rados.0.smithi163.stdout:2032: writing smithi16367185-20 from 440548 to 1166084 tid 1 2023-10-09T09:14:11.140 INFO:tasks.rados.rados.0.smithi163.stdout:2032: writing smithi16367185-20 from 1757537 to 2235799 tid 2 2023-10-09T09:14:11.143 INFO:tasks.rados.rados.0.smithi163.stdout:2032: writing smithi16367185-20 from 2813990 to 2840548 tid 3 2023-10-09T09:14:11.144 INFO:tasks.rados.rados.0.smithi163.stdout:2033: write oid 36 current snap is 260 2023-10-09T09:14:11.144 INFO:tasks.rados.rados.0.smithi163.stdout:2033: seq_num 765 ranges {457355=400785,1416019=621920,2725060=132295} 2023-10-09T09:14:11.149 INFO:tasks.rados.rados.0.smithi163.stdout:2033: writing smithi16367185-36 from 457355 to 858140 tid 1 2023-10-09T09:14:11.156 INFO:tasks.rados.rados.0.smithi163.stdout:2033: writing smithi16367185-36 from 1416019 to 2037939 tid 2 2023-10-09T09:14:11.162 INFO:tasks.rados.rados.0.smithi163.stdout:2033: writing smithi16367185-36 from 2725060 to 2857355 tid 3 2023-10-09T09:14:11.162 INFO:tasks.rados.rados.0.smithi163.stdout:2034: delete oid 2 current snap is 260 2023-10-09T09:14:11.162 INFO:tasks.rados.rados.0.smithi163.stdout:2032: finishing write tid 1 to smithi16367185-20 2023-10-09T09:14:11.162 INFO:tasks.rados.rados.0.smithi163.stdout:2032: finishing write tid 2 to smithi16367185-20 2023-10-09T09:14:11.162 INFO:tasks.rados.rados.0.smithi163.stdout:2032: finishing write tid 3 to smithi16367185-20 2023-10-09T09:14:11.162 INFO:tasks.rados.rados.0.smithi163.stdout:2032: finishing write tid 4 to smithi16367185-20 2023-10-09T09:14:11.162 INFO:tasks.rados.rados.0.smithi163.stdout:2032: finishing write tid 5 to smithi16367185-20 2023-10-09T09:14:11.162 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 223 (ObjNum 764 snap 260 seq_num 764) dirty exists 2023-10-09T09:14:11.162 INFO:tasks.rados.rados.0.smithi163.stdout:2032: left oid 20 (ObjNum 764 snap 260 seq_num 764) 2023-10-09T09:14:11.162 INFO:tasks.rados.rados.0.smithi163.stdout:2033: finishing write tid 1 to smithi16367185-36 2023-10-09T09:14:11.163 INFO:tasks.rados.rados.0.smithi163.stdout:2033: finishing write tid 2 to smithi16367185-36 2023-10-09T09:14:11.164 INFO:tasks.rados.rados.0.smithi163.stdout:2033: finishing write tid 3 to smithi16367185-36 2023-10-09T09:14:11.164 INFO:tasks.rados.rados.0.smithi163.stdout:2033: finishing write tid 4 to smithi16367185-36 2023-10-09T09:14:11.164 INFO:tasks.rados.rados.0.smithi163.stdout:2033: finishing write tid 5 to smithi16367185-36 2023-10-09T09:14:11.164 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 524 (ObjNum 765 snap 260 seq_num 765) dirty exists 2023-10-09T09:14:11.165 INFO:tasks.rados.rados.0.smithi163.stdout:2033: left oid 36 (ObjNum 765 snap 260 seq_num 765) 2023-10-09T09:14:11.168 INFO:tasks.rados.rados.0.smithi163.stdout:2032: done (2 left) 2023-10-09T09:14:11.168 INFO:tasks.rados.rados.0.smithi163.stdout:2033: done (1 left) 2023-10-09T09:14:11.168 INFO:tasks.rados.rados.0.smithi163.stdout:2034: done (0 left) 2023-10-09T09:14:11.168 INFO:tasks.rados.rados.0.smithi163.stdout:2035: read oid 11 snap -1 2023-10-09T09:14:11.169 INFO:tasks.rados.rados.0.smithi163.stdout:2035: expect deleted 2023-10-09T09:14:11.169 INFO:tasks.rados.rados.0.smithi163.stdout:2036: snap_create 2023-10-09T09:14:11.718 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[84060]: debug 2023-10-09T09:14:11.600+0000 7f802d5e3640 -1 osd.0 567 log_to_monitors true 2023-10-09T09:14:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[78107]: cluster 2023-10-09T09:14:10.126085+0000 osd.1 (osd.1) 87 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[78107]: cluster 2023-10-09T09:14:10.150764+0000 osd.3 (osd.3) 176 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[78107]: audit 2023-10-09T09:14:10.463731+0000 mon.a (mon.0) 585 : audit 0 from='client.? 172.21.15.163:0/4043398860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[78107]: cluster 2023-10-09T09:14:11.122761+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[78107]: cluster 2023-10-09T09:14:11.125081+0000 osd.2 (osd.2) 177 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[78107]: cluster 2023-10-09T09:14:11.126177+0000 mon.a (mon.0) 586 : cluster 0 osdmap e579: 8 total, 7 up, 8 in 2023-10-09T09:14:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[78107]: cluster 2023-10-09T09:14:11.161982+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[75425]: cluster 2023-10-09T09:14:10.126085+0000 osd.1 (osd.1) 87 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[75425]: cluster 2023-10-09T09:14:10.150764+0000 osd.3 (osd.3) 176 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[75425]: audit 2023-10-09T09:14:10.463731+0000 mon.a (mon.0) 585 : audit 0 from='client.? 172.21.15.163:0/4043398860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[75425]: cluster 2023-10-09T09:14:11.122761+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[75425]: cluster 2023-10-09T09:14:11.125081+0000 osd.2 (osd.2) 177 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:11.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[75425]: cluster 2023-10-09T09:14:11.126177+0000 mon.a (mon.0) 586 : cluster 0 osdmap e579: 8 total, 7 up, 8 in 2023-10-09T09:14:11.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:11 smithi163 bash[75425]: cluster 2023-10-09T09:14:11.161982+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:11 smithi181 bash[59345]: cluster 2023-10-09T09:14:10.126085+0000 osd.1 (osd.1) 87 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:11 smithi181 bash[59345]: cluster 2023-10-09T09:14:10.150764+0000 osd.3 (osd.3) 176 : cluster 3 failed to encode map e578 with expected crc 2023-10-09T09:14:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:11 smithi181 bash[59345]: audit 2023-10-09T09:14:10.463731+0000 mon.a (mon.0) 585 : audit 0 from='client.? 172.21.15.163:0/4043398860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:11 smithi181 bash[59345]: cluster 2023-10-09T09:14:11.122761+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:11.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:11 smithi181 bash[59345]: cluster 2023-10-09T09:14:11.125081+0000 osd.2 (osd.2) 177 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:11.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:11 smithi181 bash[59345]: cluster 2023-10-09T09:14:11.126177+0000 mon.a (mon.0) 586 : cluster 0 osdmap e579: 8 total, 7 up, 8 in 2023-10-09T09:14:11.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:11 smithi181 bash[59345]: cluster 2023-10-09T09:14:11.161982+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:12.136 INFO:tasks.rados.rados.0.smithi163.stdout:2035: done (1 left) 2023-10-09T09:14:12.136 INFO:tasks.rados.rados.0.smithi163.stdout:2036: done (0 left) 2023-10-09T09:14:12.136 INFO:tasks.rados.rados.0.smithi163.stdout:2037: read oid 22 snap -1 2023-10-09T09:14:12.136 INFO:tasks.rados.rados.0.smithi163.stdout:2037: expect (ObjNum 624 snap 198 seq_num 624) 2023-10-09T09:14:12.137 INFO:tasks.rados.rados.0.smithi163.stdout:2038: delete oid 41 current snap is 261 2023-10-09T09:14:12.141 INFO:tasks.rados.rados.0.smithi163.stdout:2038: done (1 left) 2023-10-09T09:14:12.141 INFO:tasks.rados.rados.0.smithi163.stdout:2039: snap_create 2023-10-09T09:14:12.150 INFO:tasks.rados.rados.0.smithi163.stdout:2037: expect (ObjNum 624 snap 198 seq_num 624) 2023-10-09T09:14:12.463 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[84060]: debug 2023-10-09T09:14:12.184+0000 7f801d24c700 -1 osd.0 567 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:14:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: cluster 2023-10-09T09:14:11.134844+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: cluster 2023-10-09T09:14:11.151830+0000 osd.1 (osd.1) 88 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: cluster 2023-10-09T09:14:11.312152+0000 mgr.y (mgr.34107) 213 : cluster 0 pgmap v421: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s; 67/381 objects degraded (17.585%) 2023-10-09T09:14:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: audit 2023-10-09T09:14:11.610219+0000 mon.c (mon.2) 8 : audit 1 from='osd.0 [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:14:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: audit 2023-10-09T09:14:11.610599+0000 mon.a (mon.0) 587 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:14:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: cluster 2023-10-09T09:14:12.133423+0000 osd.2 (osd.2) 178 : cluster 3 failed to encode map e580 with expected crc 2023-10-09T09:14:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: audit 2023-10-09T09:14:12.136010+0000 mon.a (mon.0) 588 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:14:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: cluster 2023-10-09T09:14:12.136120+0000 mon.a (mon.0) 589 : cluster 0 osdmap e580: 8 total, 7 up, 8 in 2023-10-09T09:14:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: cluster 2023-10-09T09:14:12.136833+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e580 with expected crc 2023-10-09T09:14:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: audit 2023-10-09T09:14:12.137293+0000 mon.c (mon.2) 9 : audit 1 from='osd.0 [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[75425]: audit 2023-10-09T09:14:12.139229+0000 mon.a (mon.0) 590 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: cluster 2023-10-09T09:14:11.134844+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: cluster 2023-10-09T09:14:11.151830+0000 osd.1 (osd.1) 88 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: cluster 2023-10-09T09:14:11.312152+0000 mgr.y (mgr.34107) 213 : cluster 0 pgmap v421: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s; 67/381 objects degraded (17.585%) 2023-10-09T09:14:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: audit 2023-10-09T09:14:11.610219+0000 mon.c (mon.2) 8 : audit 1 from='osd.0 [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:14:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: audit 2023-10-09T09:14:11.610599+0000 mon.a (mon.0) 587 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:14:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: cluster 2023-10-09T09:14:12.133423+0000 osd.2 (osd.2) 178 : cluster 3 failed to encode map e580 with expected crc 2023-10-09T09:14:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: audit 2023-10-09T09:14:12.136010+0000 mon.a (mon.0) 588 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:14:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: cluster 2023-10-09T09:14:12.136120+0000 mon.a (mon.0) 589 : cluster 0 osdmap e580: 8 total, 7 up, 8 in 2023-10-09T09:14:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: cluster 2023-10-09T09:14:12.136833+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e580 with expected crc 2023-10-09T09:14:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: audit 2023-10-09T09:14:12.137293+0000 mon.c (mon.2) 9 : audit 1 from='osd.0 [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:12 smithi163 bash[78107]: audit 2023-10-09T09:14:12.139229+0000 mon.a (mon.0) 590 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: cluster 2023-10-09T09:14:11.134844+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: cluster 2023-10-09T09:14:11.151830+0000 osd.1 (osd.1) 88 : cluster 3 failed to encode map e579 with expected crc 2023-10-09T09:14:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: cluster 2023-10-09T09:14:11.312152+0000 mgr.y (mgr.34107) 213 : cluster 0 pgmap v421: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s; 67/381 objects degraded (17.585%) 2023-10-09T09:14:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: audit 2023-10-09T09:14:11.610219+0000 mon.c (mon.2) 8 : audit 1 from='osd.0 [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:14:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: audit 2023-10-09T09:14:11.610599+0000 mon.a (mon.0) 587 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:14:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: cluster 2023-10-09T09:14:12.133423+0000 osd.2 (osd.2) 178 : cluster 3 failed to encode map e580 with expected crc 2023-10-09T09:14:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: audit 2023-10-09T09:14:12.136010+0000 mon.a (mon.0) 588 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:14:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: cluster 2023-10-09T09:14:12.136120+0000 mon.a (mon.0) 589 : cluster 0 osdmap e580: 8 total, 7 up, 8 in 2023-10-09T09:14:12.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: cluster 2023-10-09T09:14:12.136833+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e580 with expected crc 2023-10-09T09:14:12.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: audit 2023-10-09T09:14:12.137293+0000 mon.c (mon.2) 9 : audit 1 from='osd.0 [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:12.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:12 smithi181 bash[59345]: audit 2023-10-09T09:14:12.139229+0000 mon.a (mon.0) 590 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:13.141 INFO:tasks.rados.rados.0.smithi163.stdout:2037: done (1 left) 2023-10-09T09:14:13.295 INFO:tasks.rados.rados.0.smithi163.stdout:2039: done (0 left) 2023-10-09T09:14:13.296 INFO:tasks.rados.rados.0.smithi163.stdout:2040: read oid 25 snap -1 2023-10-09T09:14:13.296 INFO:tasks.rados.rados.0.smithi163.stdout:2040: expect (ObjNum 761 snap 259 seq_num 761) 2023-10-09T09:14:13.296 INFO:tasks.rados.rados.0.smithi163.stdout:2041: read oid 11 snap -1 2023-10-09T09:14:13.296 INFO:tasks.rados.rados.0.smithi163.stdout:2041: expect deleted 2023-10-09T09:14:13.296 INFO:tasks.rados.rados.0.smithi163.stdout:2042: snap_remove snap 255 2023-10-09T09:14:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:14:13] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.33.4" 2023-10-09T09:14:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[75425]: cluster 2023-10-09T09:14:12.133242+0000 osd.1 (osd.1) 89 : cluster 3 failed to encode map e580 with expected crc 2023-10-09T09:14:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[75425]: audit 2023-10-09T09:14:12.817404+0000 mon.a (mon.0) 591 : audit 0 from='client.? 172.21.15.163:0/3250038697' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[75425]: cluster 2023-10-09T09:14:13.131913+0000 mon.a (mon.0) 592 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:14:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[75425]: cluster 2023-10-09T09:14:13.139467+0000 osd.2 (osd.2) 179 : cluster 3 failed to encode map e581 with expected crc 2023-10-09T09:14:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[75425]: cluster 2023-10-09T09:14:13.140316+0000 mon.a (mon.0) 593 : cluster 1 osd.0 [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309] boot 2023-10-09T09:14:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[75425]: cluster 2023-10-09T09:14:13.140371+0000 mon.a (mon.0) 594 : cluster 0 osdmap e581: 8 total, 8 up, 8 in 2023-10-09T09:14:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[75425]: audit 2023-10-09T09:14:13.140758+0000 mon.a (mon.0) 595 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:14:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[75425]: cluster 2023-10-09T09:14:13.142691+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e581 with expected crc 2023-10-09T09:14:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[78107]: cluster 2023-10-09T09:14:12.133242+0000 osd.1 (osd.1) 89 : cluster 3 failed to encode map e580 with expected crc 2023-10-09T09:14:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[78107]: audit 2023-10-09T09:14:12.817404+0000 mon.a (mon.0) 591 : audit 0 from='client.? 172.21.15.163:0/3250038697' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[78107]: cluster 2023-10-09T09:14:13.131913+0000 mon.a (mon.0) 592 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:14:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[78107]: cluster 2023-10-09T09:14:13.139467+0000 osd.2 (osd.2) 179 : cluster 3 failed to encode map e581 with expected crc 2023-10-09T09:14:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[78107]: cluster 2023-10-09T09:14:13.140316+0000 mon.a (mon.0) 593 : cluster 1 osd.0 [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309] boot 2023-10-09T09:14:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[78107]: cluster 2023-10-09T09:14:13.140371+0000 mon.a (mon.0) 594 : cluster 0 osdmap e581: 8 total, 8 up, 8 in 2023-10-09T09:14:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[78107]: audit 2023-10-09T09:14:13.140758+0000 mon.a (mon.0) 595 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:14:13.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:13 smithi163 bash[78107]: cluster 2023-10-09T09:14:13.142691+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e581 with expected crc 2023-10-09T09:14:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:13 smithi181 bash[59345]: cluster 2023-10-09T09:14:12.133242+0000 osd.1 (osd.1) 89 : cluster 3 failed to encode map e580 with expected crc 2023-10-09T09:14:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:13 smithi181 bash[59345]: audit 2023-10-09T09:14:12.817404+0000 mon.a (mon.0) 591 : audit 0 from='client.? 172.21.15.163:0/3250038697' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:13 smithi181 bash[59345]: cluster 2023-10-09T09:14:13.131913+0000 mon.a (mon.0) 592 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:14:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:13 smithi181 bash[59345]: cluster 2023-10-09T09:14:13.139467+0000 osd.2 (osd.2) 179 : cluster 3 failed to encode map e581 with expected crc 2023-10-09T09:14:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:13 smithi181 bash[59345]: cluster 2023-10-09T09:14:13.140316+0000 mon.a (mon.0) 593 : cluster 1 osd.0 [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309] boot 2023-10-09T09:14:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:13 smithi181 bash[59345]: cluster 2023-10-09T09:14:13.140371+0000 mon.a (mon.0) 594 : cluster 0 osdmap e581: 8 total, 8 up, 8 in 2023-10-09T09:14:13.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:13 smithi181 bash[59345]: audit 2023-10-09T09:14:13.140758+0000 mon.a (mon.0) 595 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:14:13.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:13 smithi181 bash[59345]: cluster 2023-10-09T09:14:13.142691+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e581 with expected crc 2023-10-09T09:14:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:2042: done (2 left) 2023-10-09T09:14:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:2043: read oid 31 snap -1 2023-10-09T09:14:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:2043: expect deleted 2023-10-09T09:14:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:2044: read oid 21 snap -1 2023-10-09T09:14:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:2044: expect (ObjNum 707 snap 233 seq_num 707) 2023-10-09T09:14:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:2045: read oid 47 snap -1 2023-10-09T09:14:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:2045: expect (ObjNum 660 snap 209 seq_num 660) 2023-10-09T09:14:14.154 INFO:tasks.rados.rados.0.smithi163.stdout:2046: delete oid 42 current snap is 262 2023-10-09T09:14:14.192 INFO:tasks.rados.rados.0.smithi163.stdout:2040: expect (ObjNum 761 snap 259 seq_num 761) 2023-10-09T09:14:14.434 INFO:tasks.rados.rados.0.smithi163.stdout:2045: expect (ObjNum 660 snap 209 seq_num 660) 2023-10-09T09:14:14.544 INFO:tasks.rados.rados.0.smithi163.stdout:2044: expect (ObjNum 707 snap 233 seq_num 707) 2023-10-09T09:14:14.680 INFO:tasks.rados.rados.0.smithi163.stdout:2040: done (5 left) 2023-10-09T09:14:14.680 INFO:tasks.rados.rados.0.smithi163.stdout:2041: done (4 left) 2023-10-09T09:14:14.681 INFO:tasks.rados.rados.0.smithi163.stdout:2043: done (3 left) 2023-10-09T09:14:14.681 INFO:tasks.rados.rados.0.smithi163.stdout:2044: done (2 left) 2023-10-09T09:14:14.681 INFO:tasks.rados.rados.0.smithi163.stdout:2045: done (1 left) 2023-10-09T09:14:14.681 INFO:tasks.rados.rados.0.smithi163.stdout:2046: done (0 left) 2023-10-09T09:14:14.681 INFO:tasks.rados.rados.0.smithi163.stdout:2047: write oid 46 current snap is 262 2023-10-09T09:14:14.682 INFO:tasks.rados.rados.0.smithi163.stdout:2047: seq_num 769 ranges {524583=418134,1649947=629258,2834752=89831} 2023-10-09T09:14:14.689 INFO:tasks.rados.rados.0.smithi163.stdout:2047: writing smithi16367185-46 from 524583 to 942717 tid 1 2023-10-09T09:14:14.697 INFO:tasks.rados.rados.0.smithi163.stdout:2047: writing smithi16367185-46 from 1649947 to 2279205 tid 2 2023-10-09T09:14:14.701 INFO:tasks.rados.rados.0.smithi163.stdout:2047: writing smithi16367185-46 from 2834752 to 2924583 tid 3 2023-10-09T09:14:14.701 INFO:tasks.rados.rados.0.smithi163.stdout:2048: read oid 2 snap 246 2023-10-09T09:14:14.701 INFO:tasks.rados.rados.0.smithi163.stdout:2048: expect (ObjNum 713 snap 242 seq_num 713) 2023-10-09T09:14:14.701 INFO:tasks.rados.rados.0.smithi163.stdout:2049: snap_create 2023-10-09T09:14:14.702 INFO:tasks.rados.rados.0.smithi163.stdout:2047: finishing write tid 1 to smithi16367185-46 2023-10-09T09:14:14.702 INFO:tasks.rados.rados.0.smithi163.stdout:2047: finishing write tid 2 to smithi16367185-46 2023-10-09T09:14:14.704 INFO:tasks.rados.rados.0.smithi163.stdout:2047: finishing write tid 3 to smithi16367185-46 2023-10-09T09:14:14.704 INFO:tasks.rados.rados.0.smithi163.stdout:2047: finishing write tid 4 to smithi16367185-46 2023-10-09T09:14:14.704 INFO:tasks.rados.rados.0.smithi163.stdout:2047: finishing write tid 5 to smithi16367185-46 2023-10-09T09:14:14.704 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 813 (ObjNum 769 snap 262 seq_num 769) dirty exists 2023-10-09T09:14:14.705 INFO:tasks.rados.rados.0.smithi163.stdout:2047: left oid 46 (ObjNum 769 snap 262 seq_num 769) 2023-10-09T09:14:14.716 INFO:tasks.rados.rados.0.smithi163.stdout:2048: expect (ObjNum 713 snap 242 seq_num 713) 2023-10-09T09:14:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[75425]: cluster 2023-10-09T09:14:13.137429+0000 osd.3 (osd.3) 177 : cluster 3 failed to encode map e581 with expected crc 2023-10-09T09:14:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[75425]: cluster 2023-10-09T09:14:13.313105+0000 mgr.y (mgr.34107) 214 : cluster 0 pgmap v424: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 200 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s; 67/381 objects degraded (17.585%) 2023-10-09T09:14:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.148079+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.152290+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.152322+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.153113+0000 mon.a (mon.0) 596 : cluster 0 osdmap e582: 8 total, 8 up, 8 in 2023-10-09T09:14:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.153977+0000 osd.2 (osd.2) 180 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[78107]: cluster 2023-10-09T09:14:13.137429+0000 osd.3 (osd.3) 177 : cluster 3 failed to encode map e581 with expected crc 2023-10-09T09:14:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[78107]: cluster 2023-10-09T09:14:13.313105+0000 mgr.y (mgr.34107) 214 : cluster 0 pgmap v424: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 200 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s; 67/381 objects degraded (17.585%) 2023-10-09T09:14:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.148079+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.152290+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.152322+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.153113+0000 mon.a (mon.0) 596 : cluster 0 osdmap e582: 8 total, 8 up, 8 in 2023-10-09T09:14:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:14 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.153977+0000 osd.2 (osd.2) 180 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:14 smithi181 bash[59345]: cluster 2023-10-09T09:14:13.137429+0000 osd.3 (osd.3) 177 : cluster 3 failed to encode map e581 with expected crc 2023-10-09T09:14:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:14 smithi181 bash[59345]: cluster 2023-10-09T09:14:13.313105+0000 mgr.y (mgr.34107) 214 : cluster 0 pgmap v424: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 200 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 12 op/s; 67/381 objects degraded (17.585%) 2023-10-09T09:14:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:14 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.148079+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:14 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.152290+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:14 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.152322+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:14 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.153113+0000 mon.a (mon.0) 596 : cluster 0 osdmap e582: 8 total, 8 up, 8 in 2023-10-09T09:14:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:14 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.153977+0000 osd.2 (osd.2) 180 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.480 INFO:tasks.rados.rados.0.smithi163.stdout:2047: done (2 left) 2023-10-09T09:14:15.480 INFO:tasks.rados.rados.0.smithi163.stdout:2048: done (1 left) 2023-10-09T09:14:15.480 INFO:tasks.rados.rados.0.smithi163.stdout:2049: done (0 left) 2023-10-09T09:14:15.480 INFO:tasks.rados.rados.0.smithi163.stdout:2050: snap_remove snap 259 2023-10-09T09:14:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:15 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.149645+0000 osd.1 (osd.1) 90 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:15 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.152466+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:15 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.153528+0000 osd.3 (osd.3) 178 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:15 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.153754+0000 osd.1 (osd.1) 91 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:15 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.157523+0000 osd.3 (osd.3) 179 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:15 smithi181 bash[59345]: cluster 2023-10-09T09:14:14.160862+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:15 smithi181 bash[59345]: cluster 2023-10-09T09:14:15.080829+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 67/381 objects degraded (17.585%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:15.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:15 smithi181 bash[59345]: audit 2023-10-09T09:14:15.172063+0000 mon.a (mon.0) 598 : audit 0 from='client.? 172.21.15.163:0/1643623797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.149645+0000 osd.1 (osd.1) 90 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.152466+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.153528+0000 osd.3 (osd.3) 178 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.153754+0000 osd.1 (osd.1) 91 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.157523+0000 osd.3 (osd.3) 179 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[78107]: cluster 2023-10-09T09:14:14.160862+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[78107]: cluster 2023-10-09T09:14:15.080829+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 67/381 objects degraded (17.585%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[78107]: audit 2023-10-09T09:14:15.172063+0000 mon.a (mon.0) 598 : audit 0 from='client.? 172.21.15.163:0/1643623797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.149645+0000 osd.1 (osd.1) 90 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.152466+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.153528+0000 osd.3 (osd.3) 178 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.153754+0000 osd.1 (osd.1) 91 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.157523+0000 osd.3 (osd.3) 179 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[75425]: cluster 2023-10-09T09:14:14.160862+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e582 with expected crc 2023-10-09T09:14:15.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[75425]: cluster 2023-10-09T09:14:15.080829+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 67/381 objects degraded (17.585%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:15.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:15 smithi163 bash[75425]: audit 2023-10-09T09:14:15.172063+0000 mon.a (mon.0) 598 : audit 0 from='client.? 172.21.15.163:0/1643623797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:16.485 INFO:tasks.rados.rados.0.smithi163.stdout:2050: done (0 left) 2023-10-09T09:14:16.485 INFO:tasks.rados.rados.0.smithi163.stdout:2051: rollback oid 10 current snap is 263 2023-10-09T09:14:16.485 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 256 2023-10-09T09:14:16.485 INFO:tasks.rados.rados.0.smithi163.stdout:2052: read oid 44 snap 219 2023-10-09T09:14:16.485 INFO:tasks.rados.rados.0.smithi163.stdout:2052: expect deleted 2023-10-09T09:14:16.485 INFO:tasks.rados.rados.0.smithi163.stdout:2053: rollback oid 43 current snap is 263 2023-10-09T09:14:16.486 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 251 2023-10-09T09:14:16.486 INFO:tasks.rados.rados.0.smithi163.stdout:2054: snap_create 2023-10-09T09:14:16.497 INFO:tasks.rados.rados.0.smithi163.stdout:2051: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:14:16.497 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 0 (ObjNum 699113640 snap 21875 seq_num 0) dirty dne 2023-10-09T09:14:16.502 INFO:tasks.rados.rados.0.smithi163.stdout:2053: finishing rollback tid 0 to smithi16367185-43 2023-10-09T09:14:16.502 INFO:tasks.rados.rados.0.smithi163.stdout:2053: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:14:16.503 INFO:tasks.rados.rados.0.smithi163.stdout:2053: finishing rollback tid 2 to smithi16367185-43 2023-10-09T09:14:16.503 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 817 (ObjNum 733 snap 246 seq_num 733) dirty exists 2023-10-09T09:14:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:16 smithi181 bash[59345]: cluster 2023-10-09T09:14:15.313975+0000 mgr.y (mgr.34107) 215 : cluster 0 pgmap v426: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 200 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.4 MiB/s wr, 11 op/s; 67/381 objects degraded (17.585%) 2023-10-09T09:14:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:16 smithi181 bash[59345]: cluster 2023-10-09T09:14:15.477603+0000 osd.3 (osd.3) 180 : cluster 3 failed to encode map e583 with expected crc 2023-10-09T09:14:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:16 smithi181 bash[59345]: cluster 2023-10-09T09:14:15.478373+0000 osd.2 (osd.2) 181 : cluster 3 failed to encode map e583 with expected crc 2023-10-09T09:14:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:16 smithi181 bash[59345]: cluster 2023-10-09T09:14:15.480080+0000 mon.a (mon.0) 599 : cluster 0 osdmap e583: 8 total, 8 up, 8 in 2023-10-09T09:14:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:16 smithi163 bash[78107]: cluster 2023-10-09T09:14:15.313975+0000 mgr.y (mgr.34107) 215 : cluster 0 pgmap v426: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 200 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.4 MiB/s wr, 11 op/s; 67/381 objects degraded (17.585%) 2023-10-09T09:14:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:16 smithi163 bash[78107]: cluster 2023-10-09T09:14:15.477603+0000 osd.3 (osd.3) 180 : cluster 3 failed to encode map e583 with expected crc 2023-10-09T09:14:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:16 smithi163 bash[78107]: cluster 2023-10-09T09:14:15.478373+0000 osd.2 (osd.2) 181 : cluster 3 failed to encode map e583 with expected crc 2023-10-09T09:14:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:16 smithi163 bash[78107]: cluster 2023-10-09T09:14:15.480080+0000 mon.a (mon.0) 599 : cluster 0 osdmap e583: 8 total, 8 up, 8 in 2023-10-09T09:14:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:16 smithi163 bash[75425]: cluster 2023-10-09T09:14:15.313975+0000 mgr.y (mgr.34107) 215 : cluster 0 pgmap v426: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 200 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.4 MiB/s wr, 11 op/s; 67/381 objects degraded (17.585%) 2023-10-09T09:14:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:16 smithi163 bash[75425]: cluster 2023-10-09T09:14:15.477603+0000 osd.3 (osd.3) 180 : cluster 3 failed to encode map e583 with expected crc 2023-10-09T09:14:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:16 smithi163 bash[75425]: cluster 2023-10-09T09:14:15.478373+0000 osd.2 (osd.2) 181 : cluster 3 failed to encode map e583 with expected crc 2023-10-09T09:14:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:16 smithi163 bash[75425]: cluster 2023-10-09T09:14:15.480080+0000 mon.a (mon.0) 599 : cluster 0 osdmap e583: 8 total, 8 up, 8 in 2023-10-09T09:14:17.484 INFO:tasks.rados.rados.0.smithi163.stdout:2051: done (3 left) 2023-10-09T09:14:17.484 INFO:tasks.rados.rados.0.smithi163.stdout:2052: done (2 left) 2023-10-09T09:14:17.484 INFO:tasks.rados.rados.0.smithi163.stdout:2053: done (1 left) 2023-10-09T09:14:17.484 INFO:tasks.rados.rados.0.smithi163.stdout:2054: done (0 left) 2023-10-09T09:14:17.484 INFO:tasks.rados.rados.0.smithi163.stdout:2055: snap_remove snap 262 2023-10-09T09:14:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:17 smithi181 bash[59345]: cluster 2023-10-09T09:14:16.480791+0000 osd.3 (osd.3) 181 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:17 smithi181 bash[59345]: cluster 2023-10-09T09:14:16.481543+0000 osd.2 (osd.2) 182 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:17 smithi181 bash[59345]: cluster 2023-10-09T09:14:16.484630+0000 mon.a (mon.0) 600 : cluster 0 osdmap e584: 8 total, 8 up, 8 in 2023-10-09T09:14:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:17 smithi181 bash[59345]: cluster 2023-10-09T09:14:16.485572+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:17 smithi181 bash[59345]: cluster 2023-10-09T09:14:16.487311+0000 osd.2 (osd.2) 183 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:17 smithi181 bash[59345]: cluster 2023-10-09T09:14:16.488175+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e583 with expected crc 2023-10-09T09:14:17.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:17 smithi181 bash[59345]: cluster 2023-10-09T09:14:17.484116+0000 mon.a (mon.0) 601 : cluster 0 osdmap e585: 8 total, 8 up, 8 in 2023-10-09T09:14:17.953 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[75425]: cluster 2023-10-09T09:14:16.480791+0000 osd.3 (osd.3) 181 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.953 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[75425]: cluster 2023-10-09T09:14:16.481543+0000 osd.2 (osd.2) 182 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.953 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[75425]: cluster 2023-10-09T09:14:16.484630+0000 mon.a (mon.0) 600 : cluster 0 osdmap e584: 8 total, 8 up, 8 in 2023-10-09T09:14:17.953 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[75425]: cluster 2023-10-09T09:14:16.485572+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.953 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[75425]: cluster 2023-10-09T09:14:16.487311+0000 osd.2 (osd.2) 183 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.953 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[75425]: cluster 2023-10-09T09:14:16.488175+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e583 with expected crc 2023-10-09T09:14:17.954 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[75425]: cluster 2023-10-09T09:14:17.484116+0000 mon.a (mon.0) 601 : cluster 0 osdmap e585: 8 total, 8 up, 8 in 2023-10-09T09:14:17.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[78107]: cluster 2023-10-09T09:14:16.480791+0000 osd.3 (osd.3) 181 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[78107]: cluster 2023-10-09T09:14:16.481543+0000 osd.2 (osd.2) 182 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[78107]: cluster 2023-10-09T09:14:16.484630+0000 mon.a (mon.0) 600 : cluster 0 osdmap e584: 8 total, 8 up, 8 in 2023-10-09T09:14:17.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[78107]: cluster 2023-10-09T09:14:16.485572+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[78107]: cluster 2023-10-09T09:14:16.487311+0000 osd.2 (osd.2) 183 : cluster 3 failed to encode map e584 with expected crc 2023-10-09T09:14:17.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[78107]: cluster 2023-10-09T09:14:16.488175+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e583 with expected crc 2023-10-09T09:14:17.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:17 smithi163 bash[78107]: cluster 2023-10-09T09:14:17.484116+0000 mon.a (mon.0) 601 : cluster 0 osdmap e585: 8 total, 8 up, 8 in 2023-10-09T09:14:18.487 INFO:tasks.rados.rados.0.smithi163.stdout:2055: done (0 left) 2023-10-09T09:14:18.487 INFO:tasks.rados.rados.0.smithi163.stdout:2056: write oid 3 current snap is 264 2023-10-09T09:14:18.487 INFO:tasks.rados.rados.0.smithi163.stdout:2056: seq_num 770 ranges {541390=493383,1592076=489269,2862175=79215} 2023-10-09T09:14:18.495 INFO:tasks.rados.rados.0.smithi163.stdout:2056: writing smithi16367185-3 from 541390 to 1034773 tid 1 2023-10-09T09:14:18.502 INFO:tasks.rados.rados.0.smithi163.stdout:2056: writing smithi16367185-3 from 1592076 to 2081345 tid 2 2023-10-09T09:14:18.507 INFO:tasks.rados.rados.0.smithi163.stdout:2056: writing smithi16367185-3 from 2862175 to 2941390 tid 3 2023-10-09T09:14:18.508 INFO:tasks.rados.rados.0.smithi163.stdout:2057: snap_create 2023-10-09T09:14:18.508 INFO:tasks.rados.rados.0.smithi163.stdout:2056: finishing write tid 1 to smithi16367185-3 2023-10-09T09:14:18.510 INFO:tasks.rados.rados.0.smithi163.stdout:2056: finishing write tid 2 to smithi16367185-3 2023-10-09T09:14:18.511 INFO:tasks.rados.rados.0.smithi163.stdout:2056: finishing write tid 3 to smithi16367185-3 2023-10-09T09:14:18.511 INFO:tasks.rados.rados.0.smithi163.stdout:2056: finishing write tid 4 to smithi16367185-3 2023-10-09T09:14:18.511 INFO:tasks.rados.rados.0.smithi163.stdout:2056: finishing write tid 5 to smithi16367185-3 2023-10-09T09:14:18.511 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 535 (ObjNum 770 snap 264 seq_num 770) dirty exists 2023-10-09T09:14:18.512 INFO:tasks.rados.rados.0.smithi163.stdout:2056: left oid 3 (ObjNum 770 snap 264 seq_num 770) 2023-10-09T09:14:18.761 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[75425]: cluster 2023-10-09T09:14:17.314606+0000 mgr.y (mgr.34107) 216 : cluster 0 pgmap v429: 89 pgs: 89 active+clean; 200 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 266 KiB/s wr, 10 op/s; 4.7 MiB/s, 5 objects/s recovering 2023-10-09T09:14:18.762 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[75425]: cluster 2023-10-09T09:14:17.481470+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:18.762 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[75425]: cluster 2023-10-09T09:14:17.482520+0000 osd.3 (osd.3) 182 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:18.762 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[75425]: cluster 2023-10-09T09:14:17.487430+0000 mon.a (mon.0) 602 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 67/381 objects degraded (17.585%), 15 pgs degraded) 2023-10-09T09:14:18.762 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[75425]: cluster 2023-10-09T09:14:17.487459+0000 mon.a (mon.0) 603 : cluster 1 Cluster is now healthy 2023-10-09T09:14:18.762 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[75425]: audit 2023-10-09T09:14:17.528088+0000 mon.a (mon.0) 604 : audit 0 from='client.? 172.21.15.163:0/482255936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:18.762 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[75425]: cluster 2023-10-09T09:14:18.047015+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:18.762 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[75425]: cluster 2023-10-09T09:14:18.486516+0000 mon.a (mon.0) 605 : cluster 0 osdmap e586: 8 total, 8 up, 8 in 2023-10-09T09:14:18.763 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[78107]: cluster 2023-10-09T09:14:17.314606+0000 mgr.y (mgr.34107) 216 : cluster 0 pgmap v429: 89 pgs: 89 active+clean; 200 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 266 KiB/s wr, 10 op/s; 4.7 MiB/s, 5 objects/s recovering 2023-10-09T09:14:18.763 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[78107]: cluster 2023-10-09T09:14:17.481470+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:18.763 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[78107]: cluster 2023-10-09T09:14:17.482520+0000 osd.3 (osd.3) 182 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:18.763 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[78107]: cluster 2023-10-09T09:14:17.487430+0000 mon.a (mon.0) 602 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 67/381 objects degraded (17.585%), 15 pgs degraded) 2023-10-09T09:14:18.763 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[78107]: cluster 2023-10-09T09:14:17.487459+0000 mon.a (mon.0) 603 : cluster 1 Cluster is now healthy 2023-10-09T09:14:18.763 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[78107]: audit 2023-10-09T09:14:17.528088+0000 mon.a (mon.0) 604 : audit 0 from='client.? 172.21.15.163:0/482255936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:18.763 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[78107]: cluster 2023-10-09T09:14:18.047015+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:18.763 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:18 smithi163 bash[78107]: cluster 2023-10-09T09:14:18.486516+0000 mon.a (mon.0) 605 : cluster 0 osdmap e586: 8 total, 8 up, 8 in 2023-10-09T09:14:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:18 smithi181 bash[59345]: cluster 2023-10-09T09:14:17.314606+0000 mgr.y (mgr.34107) 216 : cluster 0 pgmap v429: 89 pgs: 89 active+clean; 200 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 266 KiB/s wr, 10 op/s; 4.7 MiB/s, 5 objects/s recovering 2023-10-09T09:14:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:18 smithi181 bash[59345]: cluster 2023-10-09T09:14:17.481470+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:18 smithi181 bash[59345]: cluster 2023-10-09T09:14:17.482520+0000 osd.3 (osd.3) 182 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:18 smithi181 bash[59345]: cluster 2023-10-09T09:14:17.487430+0000 mon.a (mon.0) 602 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 67/381 objects degraded (17.585%), 15 pgs degraded) 2023-10-09T09:14:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:18 smithi181 bash[59345]: cluster 2023-10-09T09:14:17.487459+0000 mon.a (mon.0) 603 : cluster 1 Cluster is now healthy 2023-10-09T09:14:18.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:18 smithi181 bash[59345]: audit 2023-10-09T09:14:17.528088+0000 mon.a (mon.0) 604 : audit 0 from='client.? 172.21.15.163:0/482255936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:18.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:18 smithi181 bash[59345]: cluster 2023-10-09T09:14:18.047015+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:18.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:18 smithi181 bash[59345]: cluster 2023-10-09T09:14:18.486516+0000 mon.a (mon.0) 605 : cluster 0 osdmap e586: 8 total, 8 up, 8 in 2023-10-09T09:14:19.513 INFO:tasks.rados.rados.0.smithi163.stdout:2056: done (1 left) 2023-10-09T09:14:19.513 INFO:tasks.rados.rados.0.smithi163.stdout:2057: done (0 left) 2023-10-09T09:14:19.513 INFO:tasks.rados.rados.0.smithi163.stdout:2058: delete oid 20 current snap is 265 2023-10-09T09:14:19.526 INFO:tasks.rados.rados.0.smithi163.stdout:2058: done (0 left) 2023-10-09T09:14:19.526 INFO:tasks.rados.rados.0.smithi163.stdout:2059: rollback oid 21 current snap is 265 2023-10-09T09:14:19.526 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 219 2023-10-09T09:14:19.526 INFO:tasks.rados.rados.0.smithi163.stdout:2060: snap_remove snap 245 2023-10-09T09:14:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:19 smithi181 bash[59345]: cluster 2023-10-09T09:14:18.483949+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:19 smithi181 bash[59345]: cluster 2023-10-09T09:14:18.484447+0000 osd.3 (osd.3) 183 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:19 smithi181 bash[59345]: cluster 2023-10-09T09:14:18.503032+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:19 smithi181 bash[59345]: cluster 2023-10-09T09:14:18.504398+0000 osd.1 (osd.1) 92 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:19 smithi181 bash[59345]: cluster 2023-10-09T09:14:18.608730+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:19 smithi181 bash[59345]: cluster 2023-10-09T09:14:18.609796+0000 osd.2 (osd.2) 184 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:19.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[78107]: cluster 2023-10-09T09:14:18.483949+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[78107]: cluster 2023-10-09T09:14:18.484447+0000 osd.3 (osd.3) 183 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[78107]: cluster 2023-10-09T09:14:18.503032+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[78107]: cluster 2023-10-09T09:14:18.504398+0000 osd.1 (osd.1) 92 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:19.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[78107]: cluster 2023-10-09T09:14:18.608730+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[78107]: cluster 2023-10-09T09:14:18.609796+0000 osd.2 (osd.2) 184 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:19.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[75425]: cluster 2023-10-09T09:14:18.483949+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[75425]: cluster 2023-10-09T09:14:18.484447+0000 osd.3 (osd.3) 183 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[75425]: cluster 2023-10-09T09:14:18.503032+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[75425]: cluster 2023-10-09T09:14:18.504398+0000 osd.1 (osd.1) 92 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:19.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[75425]: cluster 2023-10-09T09:14:18.608730+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e586 with expected crc 2023-10-09T09:14:19.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:19 smithi163 bash[75425]: cluster 2023-10-09T09:14:18.609796+0000 osd.2 (osd.2) 184 : cluster 3 failed to encode map e585 with expected crc 2023-10-09T09:14:20.091 INFO:tasks.rados.rados.0.smithi163.stdout:2060: done (1 left) 2023-10-09T09:14:20.091 INFO:tasks.rados.rados.0.smithi163.stdout:2061: snap_create 2023-10-09T09:14:20.091 INFO:tasks.rados.rados.0.smithi163.stdout:2059: finishing rollback tid 0 to smithi16367185-21 2023-10-09T09:14:20.091 INFO:tasks.rados.rados.0.smithi163.stdout:2059: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:14:20.091 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 479 (ObjNum 1 snap 32603 seq_num 699024736) dirty dne 2023-10-09T09:14:20.783 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: cluster 2023-10-09T09:14:19.315330+0000 mgr.y (mgr.34107) 217 : cluster 0 pgmap v432: 89 pgs: 89 active+clean; 200 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 278 KiB/s wr, 10 op/s; 5.0 MiB/s, 5 objects/s recovering 2023-10-09T09:14:20.783 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: cluster 2023-10-09T09:14:19.508671+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.783 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: cluster 2023-10-09T09:14:19.509207+0000 osd.1 (osd.1) 93 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.783 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: cluster 2023-10-09T09:14:19.512535+0000 mon.a (mon.0) 606 : cluster 0 osdmap e587: 8 total, 8 up, 8 in 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: cluster 2023-10-09T09:14:19.514160+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.519152+0000 mon.a (mon.0) 607 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.527646+0000 mon.a (mon.0) 608 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.529461+0000 mon.a (mon.0) 609 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.530964+0000 mon.a (mon.0) 610 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.536827+0000 mon.a (mon.0) 611 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.619438+0000 mon.a (mon.0) 612 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.624281+0000 mon.a (mon.0) 613 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.627308+0000 mon.a (mon.0) 614 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.628782+0000 mon.a (mon.0) 615 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:20.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.630533+0000 mon.a (mon.0) 616 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:14:20.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: audit 2023-10-09T09:14:19.876991+0000 mon.a (mon.0) 617 : audit 0 from='client.? 172.21.15.163:0/1006184121' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:20.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: cluster 2023-10-09T09:14:20.085591+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e588 with expected crc 2023-10-09T09:14:20.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: cluster 2023-10-09T09:14:20.087083+0000 osd.3 (osd.3) 184 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[78107]: cluster 2023-10-09T09:14:20.090376+0000 mon.a (mon.0) 618 : cluster 0 osdmap e588: 8 total, 8 up, 8 in 2023-10-09T09:14:20.785 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: cluster 2023-10-09T09:14:19.315330+0000 mgr.y (mgr.34107) 217 : cluster 0 pgmap v432: 89 pgs: 89 active+clean; 200 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 278 KiB/s wr, 10 op/s; 5.0 MiB/s, 5 objects/s recovering 2023-10-09T09:14:20.785 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: cluster 2023-10-09T09:14:19.508671+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.785 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: cluster 2023-10-09T09:14:19.509207+0000 osd.1 (osd.1) 93 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.785 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: cluster 2023-10-09T09:14:19.512535+0000 mon.a (mon.0) 606 : cluster 0 osdmap e587: 8 total, 8 up, 8 in 2023-10-09T09:14:20.785 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: cluster 2023-10-09T09:14:19.514160+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.785 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.519152+0000 mon.a (mon.0) 607 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.527646+0000 mon.a (mon.0) 608 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.529461+0000 mon.a (mon.0) 609 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.530964+0000 mon.a (mon.0) 610 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.536827+0000 mon.a (mon.0) 611 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.619438+0000 mon.a (mon.0) 612 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.624281+0000 mon.a (mon.0) 613 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.627308+0000 mon.a (mon.0) 614 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.628782+0000 mon.a (mon.0) 615 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.630533+0000 mon.a (mon.0) 616 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: audit 2023-10-09T09:14:19.876991+0000 mon.a (mon.0) 617 : audit 0 from='client.? 172.21.15.163:0/1006184121' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: cluster 2023-10-09T09:14:20.085591+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e588 with expected crc 2023-10-09T09:14:20.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: cluster 2023-10-09T09:14:20.087083+0000 osd.3 (osd.3) 184 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:20 smithi163 bash[75425]: cluster 2023-10-09T09:14:20.090376+0000 mon.a (mon.0) 618 : cluster 0 osdmap e588: 8 total, 8 up, 8 in 2023-10-09T09:14:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: cluster 2023-10-09T09:14:19.315330+0000 mgr.y (mgr.34107) 217 : cluster 0 pgmap v432: 89 pgs: 89 active+clean; 200 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 278 KiB/s wr, 10 op/s; 5.0 MiB/s, 5 objects/s recovering 2023-10-09T09:14:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: cluster 2023-10-09T09:14:19.508671+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: cluster 2023-10-09T09:14:19.509207+0000 osd.1 (osd.1) 93 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: cluster 2023-10-09T09:14:19.512535+0000 mon.a (mon.0) 606 : cluster 0 osdmap e587: 8 total, 8 up, 8 in 2023-10-09T09:14:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: cluster 2023-10-09T09:14:19.514160+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.519152+0000 mon.a (mon.0) 607 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.527646+0000 mon.a (mon.0) 608 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.529461+0000 mon.a (mon.0) 609 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.530964+0000 mon.a (mon.0) 610 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:14:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.536827+0000 mon.a (mon.0) 611 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.619438+0000 mon.a (mon.0) 612 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.624281+0000 mon.a (mon.0) 613 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.627308+0000 mon.a (mon.0) 614 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.628782+0000 mon.a (mon.0) 615 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.630533+0000 mon.a (mon.0) 616 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:14:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: audit 2023-10-09T09:14:19.876991+0000 mon.a (mon.0) 617 : audit 0 from='client.? 172.21.15.163:0/1006184121' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: cluster 2023-10-09T09:14:20.085591+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e588 with expected crc 2023-10-09T09:14:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: cluster 2023-10-09T09:14:20.087083+0000 osd.3 (osd.3) 184 : cluster 3 failed to encode map e587 with expected crc 2023-10-09T09:14:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:20 smithi181 bash[59345]: cluster 2023-10-09T09:14:20.090376+0000 mon.a (mon.0) 618 : cluster 0 osdmap e588: 8 total, 8 up, 8 in 2023-10-09T09:14:21.092 INFO:tasks.rados.rados.0.smithi163.stdout:2059: done (1 left) 2023-10-09T09:14:21.092 INFO:tasks.rados.rados.0.smithi163.stdout:2061: done (0 left) 2023-10-09T09:14:21.092 INFO:tasks.rados.rados.0.smithi163.stdout:2062: write oid 8 current snap is 266 2023-10-09T09:14:21.093 INFO:tasks.rados.rados.0.smithi163.stdout:2062: seq_num 772 ranges {575004=643881,1709040=492938,2917021=57983} 2023-10-09T09:14:21.104 INFO:tasks.rados.rados.0.smithi163.stdout:2062: writing smithi16367185-8 from 575004 to 1218885 tid 1 2023-10-09T09:14:21.110 INFO:tasks.rados.rados.0.smithi163.stdout:2062: writing smithi16367185-8 from 1709040 to 2201978 tid 2 2023-10-09T09:14:21.115 INFO:tasks.rados.rados.0.smithi163.stdout:2062: writing smithi16367185-8 from 2917021 to 2975004 tid 3 2023-10-09T09:14:21.115 INFO:tasks.rados.rados.0.smithi163.stdout:2063: snap_create 2023-10-09T09:14:21.120 INFO:tasks.rados.rados.0.smithi163.stdout:2062: finishing write tid 1 to smithi16367185-8 2023-10-09T09:14:21.120 INFO:tasks.rados.rados.0.smithi163.stdout:2062: finishing write tid 2 to smithi16367185-8 2023-10-09T09:14:21.121 INFO:tasks.rados.rados.0.smithi163.stdout:2062: finishing write tid 3 to smithi16367185-8 2023-10-09T09:14:21.122 INFO:tasks.rados.rados.0.smithi163.stdout:2062: finishing write tid 4 to smithi16367185-8 2023-10-09T09:14:21.123 INFO:tasks.rados.rados.0.smithi163.stdout:2062: finishing write tid 5 to smithi16367185-8 2023-10-09T09:14:21.123 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1016 (ObjNum 772 snap 266 seq_num 772) dirty exists 2023-10-09T09:14:21.123 INFO:tasks.rados.rados.0.smithi163.stdout:2062: left oid 8 (ObjNum 772 snap 266 seq_num 772) 2023-10-09T09:14:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:21 smithi181 bash[59345]: audit 2023-10-09T09:14:19.630753+0000 mgr.y (mgr.34107) 218 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:14:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:21 smithi181 bash[59345]: cephadm 2023-10-09T09:14:19.631815+0000 mgr.y (mgr.34107) 219 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T09:14:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:21 smithi181 bash[59345]: cluster 2023-10-09T09:14:21.088243+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e589 with expected crc 2023-10-09T09:14:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:21 smithi181 bash[59345]: cluster 2023-10-09T09:14:21.088781+0000 osd.3 (osd.3) 185 : cluster 3 failed to encode map e589 with expected crc 2023-10-09T09:14:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:21 smithi181 bash[59345]: cluster 2023-10-09T09:14:21.092161+0000 mon.a (mon.0) 619 : cluster 0 osdmap e589: 8 total, 8 up, 8 in 2023-10-09T09:14:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:21 smithi181 bash[59345]: cluster 2023-10-09T09:14:21.105870+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e589 with expected crc 2023-10-09T09:14:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[75425]: audit 2023-10-09T09:14:19.630753+0000 mgr.y (mgr.34107) 218 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:14:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[75425]: cephadm 2023-10-09T09:14:19.631815+0000 mgr.y (mgr.34107) 219 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T09:14:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[75425]: cluster 2023-10-09T09:14:21.088243+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e589 with expected crc 2023-10-09T09:14:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[75425]: cluster 2023-10-09T09:14:21.088781+0000 osd.3 (osd.3) 185 : cluster 3 failed to encode map e589 with expected crc 2023-10-09T09:14:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[75425]: cluster 2023-10-09T09:14:21.092161+0000 mon.a (mon.0) 619 : cluster 0 osdmap e589: 8 total, 8 up, 8 in 2023-10-09T09:14:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[75425]: cluster 2023-10-09T09:14:21.105870+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e589 with expected crc 2023-10-09T09:14:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[78107]: audit 2023-10-09T09:14:19.630753+0000 mgr.y (mgr.34107) 218 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:14:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[78107]: cephadm 2023-10-09T09:14:19.631815+0000 mgr.y (mgr.34107) 219 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T09:14:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[78107]: cluster 2023-10-09T09:14:21.088243+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e589 with expected crc 2023-10-09T09:14:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[78107]: cluster 2023-10-09T09:14:21.088781+0000 osd.3 (osd.3) 185 : cluster 3 failed to encode map e589 with expected crc 2023-10-09T09:14:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[78107]: cluster 2023-10-09T09:14:21.092161+0000 mon.a (mon.0) 619 : cluster 0 osdmap e589: 8 total, 8 up, 8 in 2023-10-09T09:14:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:21 smithi163 bash[78107]: cluster 2023-10-09T09:14:21.105870+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e589 with expected crc 2023-10-09T09:14:22.102 INFO:tasks.rados.rados.0.smithi163.stdout:2062: done (1 left) 2023-10-09T09:14:22.102 INFO:tasks.rados.rados.0.smithi163.stdout:2063: done (0 left) 2023-10-09T09:14:22.102 INFO:tasks.rados.rados.0.smithi163.stdout:2064: write oid 10 current snap is 267 2023-10-09T09:14:22.102 INFO:tasks.rados.rados.0.smithi163.stdout:2064: seq_num 773 ranges {591811=719130,1767522=636596,2828091=163720} 2023-10-09T09:14:22.105 INFO:tasks.rados.rados.0.smithi163.stdout:2064: writing smithi16367185-10 from 591811 to 1310941 tid 1 2023-10-09T09:14:22.113 INFO:tasks.rados.rados.0.smithi163.stdout:2064: writing smithi16367185-10 from 1767522 to 2404118 tid 2 2023-10-09T09:14:22.116 INFO:tasks.rados.rados.0.smithi163.stdout:2064: writing smithi16367185-10 from 2828091 to 2991811 tid 3 2023-10-09T09:14:22.116 INFO:tasks.rados.rados.0.smithi163.stdout:2065: delete oid 47 current snap is 267 2023-10-09T09:14:22.121 INFO:tasks.rados.rados.0.smithi163.stdout:2064: finishing write tid 1 to smithi16367185-10 2023-10-09T09:14:22.121 INFO:tasks.rados.rados.0.smithi163.stdout:2064: finishing write tid 2 to smithi16367185-10 2023-10-09T09:14:22.121 INFO:tasks.rados.rados.0.smithi163.stdout:2065: done (1 left) 2023-10-09T09:14:22.122 INFO:tasks.rados.rados.0.smithi163.stdout:2066: snap_create 2023-10-09T09:14:22.123 INFO:tasks.rados.rados.0.smithi163.stdout:2064: finishing write tid 3 to smithi16367185-10 2023-10-09T09:14:22.124 INFO:tasks.rados.rados.0.smithi163.stdout:2064: finishing write tid 4 to smithi16367185-10 2023-10-09T09:14:22.124 INFO:tasks.rados.rados.0.smithi163.stdout:2064: finishing write tid 5 to smithi16367185-10 2023-10-09T09:14:22.124 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 679 (ObjNum 773 snap 267 seq_num 773) dirty exists 2023-10-09T09:14:22.124 INFO:tasks.rados.rados.0.smithi163.stdout:2064: left oid 10 (ObjNum 773 snap 267 seq_num 773) 2023-10-09T09:14:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:22 smithi181 bash[59345]: cluster 2023-10-09T09:14:21.316506+0000 mgr.y (mgr.34107) 220 : cluster 0 pgmap v436: 89 pgs: 89 active+clean; 202 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 267 B/s rd, 305 KiB/s wr, 3 op/s 2023-10-09T09:14:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:22 smithi181 bash[59345]: audit 2023-10-09T09:14:21.708112+0000 mon.a (mon.0) 620 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:22 smithi181 bash[59345]: audit 2023-10-09T09:14:21.709603+0000 mon.a (mon.0) 621 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:14:22.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:22 smithi181 bash[59345]: audit 2023-10-09T09:14:21.710689+0000 mon.a (mon.0) 622 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:22.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:22 smithi181 bash[59345]: cluster 2023-10-09T09:14:22.094189+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e590 with expected crc 2023-10-09T09:14:22.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:22 smithi181 bash[59345]: cluster 2023-10-09T09:14:22.098613+0000 mon.a (mon.0) 623 : cluster 0 osdmap e590: 8 total, 8 up, 8 in 2023-10-09T09:14:22.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:22 smithi181 bash[59345]: cluster 2023-10-09T09:14:22.108217+0000 osd.2 (osd.2) 185 : cluster 3 failed to encode map e590 with expected crc 2023-10-09T09:14:22.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:22 smithi181 bash[59345]: audit 2023-10-09T09:14:22.235841+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.163:0/2155760626' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:22.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[75425]: cluster 2023-10-09T09:14:21.316506+0000 mgr.y (mgr.34107) 220 : cluster 0 pgmap v436: 89 pgs: 89 active+clean; 202 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 267 B/s rd, 305 KiB/s wr, 3 op/s 2023-10-09T09:14:22.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[75425]: audit 2023-10-09T09:14:21.708112+0000 mon.a (mon.0) 620 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:22.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[75425]: audit 2023-10-09T09:14:21.709603+0000 mon.a (mon.0) 621 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:14:22.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[75425]: audit 2023-10-09T09:14:21.710689+0000 mon.a (mon.0) 622 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:22.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[75425]: cluster 2023-10-09T09:14:22.094189+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e590 with expected crc 2023-10-09T09:14:22.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[75425]: cluster 2023-10-09T09:14:22.098613+0000 mon.a (mon.0) 623 : cluster 0 osdmap e590: 8 total, 8 up, 8 in 2023-10-09T09:14:22.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[75425]: cluster 2023-10-09T09:14:22.108217+0000 osd.2 (osd.2) 185 : cluster 3 failed to encode map e590 with expected crc 2023-10-09T09:14:22.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[75425]: audit 2023-10-09T09:14:22.235841+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.163:0/2155760626' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:22.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[78107]: cluster 2023-10-09T09:14:21.316506+0000 mgr.y (mgr.34107) 220 : cluster 0 pgmap v436: 89 pgs: 89 active+clean; 202 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 267 B/s rd, 305 KiB/s wr, 3 op/s 2023-10-09T09:14:22.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[78107]: audit 2023-10-09T09:14:21.708112+0000 mon.a (mon.0) 620 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:22.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[78107]: audit 2023-10-09T09:14:21.709603+0000 mon.a (mon.0) 621 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:14:22.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[78107]: audit 2023-10-09T09:14:21.710689+0000 mon.a (mon.0) 622 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:22.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[78107]: cluster 2023-10-09T09:14:22.094189+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e590 with expected crc 2023-10-09T09:14:22.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[78107]: cluster 2023-10-09T09:14:22.098613+0000 mon.a (mon.0) 623 : cluster 0 osdmap e590: 8 total, 8 up, 8 in 2023-10-09T09:14:22.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[78107]: cluster 2023-10-09T09:14:22.108217+0000 osd.2 (osd.2) 185 : cluster 3 failed to encode map e590 with expected crc 2023-10-09T09:14:22.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:22 smithi163 bash[78107]: audit 2023-10-09T09:14:22.235841+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.163:0/2155760626' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:23.101 INFO:tasks.rados.rados.0.smithi163.stdout:2064: done (1 left) 2023-10-09T09:14:23.101 INFO:tasks.rados.rados.0.smithi163.stdout:2066: done (0 left) 2023-10-09T09:14:23.101 INFO:tasks.rados.rados.0.smithi163.stdout:2067: rollback oid 33 current snap is 268 2023-10-09T09:14:23.102 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 267 2023-10-09T09:14:23.102 INFO:tasks.rados.rados.0.smithi163.stdout:2068: read oid 44 snap -1 2023-10-09T09:14:23.102 INFO:tasks.rados.rados.0.smithi163.stdout:2068: expect deleted 2023-10-09T09:14:23.102 INFO:tasks.rados.rados.0.smithi163.stdout:2069: write oid 23 current snap is 268 2023-10-09T09:14:23.102 INFO:tasks.rados.rados.0.smithi163.stdout:2069: seq_num 775 ranges {625425=469628,1768133=640265} 2023-10-09T09:14:23.109 INFO:tasks.rados.rados.0.smithi163.stdout:2069: writing smithi16367185-23 from 625425 to 1095053 tid 1 2023-10-09T09:14:23.117 INFO:tasks.rados.rados.0.smithi163.stdout:2069: writing smithi16367185-23 from 1768133 to 2408398 tid 2 2023-10-09T09:14:23.117 INFO:tasks.rados.rados.0.smithi163.stdout:2070: snap_remove snap 234 2023-10-09T09:14:23.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:14:23] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.33.4" 2023-10-09T09:14:23.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[75425]: cephadm 2023-10-09T09:14:21.699317+0000 mgr.y (mgr.34107) 221 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T09:14:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[75425]: cephadm 2023-10-09T09:14:21.713371+0000 mgr.y (mgr.34107) 222 : cephadm 1 Deploying daemon osd.1 on smithi163 2023-10-09T09:14:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[75425]: cluster 2023-10-09T09:14:22.094431+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e590 with expected crc 2023-10-09T09:14:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[75425]: cluster 2023-10-09T09:14:23.100188+0000 mon.a (mon.0) 624 : cluster 0 osdmap e591: 8 total, 8 up, 8 in 2023-10-09T09:14:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[75425]: cluster 2023-10-09T09:14:23.102106+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[75425]: cluster 2023-10-09T09:14:23.103716+0000 osd.2 (osd.2) 186 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:23.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[78107]: cephadm 2023-10-09T09:14:21.699317+0000 mgr.y (mgr.34107) 221 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T09:14:23.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[78107]: cephadm 2023-10-09T09:14:21.713371+0000 mgr.y (mgr.34107) 222 : cephadm 1 Deploying daemon osd.1 on smithi163 2023-10-09T09:14:23.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[78107]: cluster 2023-10-09T09:14:22.094431+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e590 with expected crc 2023-10-09T09:14:23.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[78107]: cluster 2023-10-09T09:14:23.100188+0000 mon.a (mon.0) 624 : cluster 0 osdmap e591: 8 total, 8 up, 8 in 2023-10-09T09:14:23.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[78107]: cluster 2023-10-09T09:14:23.102106+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:23.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:23 smithi163 bash[78107]: cluster 2023-10-09T09:14:23.103716+0000 osd.2 (osd.2) 186 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:23.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:23 smithi181 bash[59345]: cephadm 2023-10-09T09:14:21.699317+0000 mgr.y (mgr.34107) 221 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T09:14:23.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:23 smithi181 bash[59345]: cephadm 2023-10-09T09:14:21.713371+0000 mgr.y (mgr.34107) 222 : cephadm 1 Deploying daemon osd.1 on smithi163 2023-10-09T09:14:23.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:23 smithi181 bash[59345]: cluster 2023-10-09T09:14:22.094431+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e590 with expected crc 2023-10-09T09:14:23.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:23 smithi181 bash[59345]: cluster 2023-10-09T09:14:23.100188+0000 mon.a (mon.0) 624 : cluster 0 osdmap e591: 8 total, 8 up, 8 in 2023-10-09T09:14:23.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:23 smithi181 bash[59345]: cluster 2023-10-09T09:14:23.102106+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:23.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:23 smithi181 bash[59345]: cluster 2023-10-09T09:14:23.103716+0000 osd.2 (osd.2) 186 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:24.108 INFO:tasks.rados.rados.0.smithi163.stdout:2070: done (3 left) 2023-10-09T09:14:24.109 INFO:tasks.rados.rados.0.smithi163.stdout:2071: read oid 7 snap 251 2023-10-09T09:14:24.109 INFO:tasks.rados.rados.0.smithi163.stdout:2071: expect (ObjNum 719 snap 245 seq_num 719) 2023-10-09T09:14:24.109 INFO:tasks.rados.rados.0.smithi163.stdout:2072: read oid 20 snap -1 2023-10-09T09:14:24.109 INFO:tasks.rados.rados.0.smithi163.stdout:2072: expect deleted 2023-10-09T09:14:24.109 INFO:tasks.rados.rados.0.smithi163.stdout:2073: delete oid 11 current snap is 268 2023-10-09T09:14:24.109 INFO:tasks.rados.rados.0.smithi163.stdout:2067: finishing rollback tid 0 to smithi16367185-33 2023-10-09T09:14:24.109 INFO:tasks.rados.rados.0.smithi163.stdout:2067: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:14:24.109 INFO:tasks.rados.rados.0.smithi163.stdout:2067: finishing rollback tid 2 to smithi16367185-33 2023-10-09T09:14:24.109 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 965 (ObjNum 747 snap 253 seq_num 747) dirty exists 2023-10-09T09:14:24.110 INFO:tasks.rados.rados.0.smithi163.stdout:2069: finishing write tid 1 to smithi16367185-23 2023-10-09T09:14:24.110 INFO:tasks.rados.rados.0.smithi163.stdout:2069: finishing write tid 2 to smithi16367185-23 2023-10-09T09:14:24.110 INFO:tasks.rados.rados.0.smithi163.stdout:2069: finishing write tid 3 to smithi16367185-23 2023-10-09T09:14:24.110 INFO:tasks.rados.rados.0.smithi163.stdout:2069: finishing write tid 4 to smithi16367185-23 2023-10-09T09:14:24.110 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 850 (ObjNum 775 snap 268 seq_num 775) dirty exists 2023-10-09T09:14:24.110 INFO:tasks.rados.rados.0.smithi163.stdout:2069: left oid 23 (ObjNum 775 snap 268 seq_num 775) 2023-10-09T09:14:24.113 INFO:tasks.rados.rados.0.smithi163.stdout:2071: expect (ObjNum 719 snap 245 seq_num 719) 2023-10-09T09:14:24.207 INFO:tasks.rados.rados.0.smithi163.stdout:2067: done (5 left) 2023-10-09T09:14:24.207 INFO:tasks.rados.rados.0.smithi163.stdout:2068: done (4 left) 2023-10-09T09:14:24.207 INFO:tasks.rados.rados.0.smithi163.stdout:2069: done (3 left) 2023-10-09T09:14:24.207 INFO:tasks.rados.rados.0.smithi163.stdout:2071: done (2 left) 2023-10-09T09:14:24.207 INFO:tasks.rados.rados.0.smithi163.stdout:2072: done (1 left) 2023-10-09T09:14:24.207 INFO:tasks.rados.rados.0.smithi163.stdout:2073: done (0 left) 2023-10-09T09:14:24.207 INFO:tasks.rados.rados.0.smithi163.stdout:2074: snap_remove snap 263 2023-10-09T09:14:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:24 smithi181 bash[59345]: cluster 2023-10-09T09:14:23.097428+0000 osd.1 (osd.1) 94 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:24 smithi181 bash[59345]: cluster 2023-10-09T09:14:23.317217+0000 mgr.y (mgr.34107) 223 : cluster 0 pgmap v439: 89 pgs: 89 active+clean; 203 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 537 B/s rd, 579 KiB/s wr, 4 op/s 2023-10-09T09:14:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:24 smithi181 bash[59345]: cluster 2023-10-09T09:14:24.108107+0000 mon.a (mon.0) 625 : cluster 0 osdmap e592: 8 total, 8 up, 8 in 2023-10-09T09:14:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:24 smithi181 bash[59345]: cluster 2023-10-09T09:14:24.109299+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:24 smithi181 bash[59345]: cluster 2023-10-09T09:14:24.122529+0000 osd.3 (osd.3) 186 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:24.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:24 smithi163 bash[78107]: cluster 2023-10-09T09:14:23.097428+0000 osd.1 (osd.1) 94 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:24.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:24 smithi163 bash[78107]: cluster 2023-10-09T09:14:23.317217+0000 mgr.y (mgr.34107) 223 : cluster 0 pgmap v439: 89 pgs: 89 active+clean; 203 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 537 B/s rd, 579 KiB/s wr, 4 op/s 2023-10-09T09:14:24.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:24 smithi163 bash[78107]: cluster 2023-10-09T09:14:24.108107+0000 mon.a (mon.0) 625 : cluster 0 osdmap e592: 8 total, 8 up, 8 in 2023-10-09T09:14:24.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:24 smithi163 bash[78107]: cluster 2023-10-09T09:14:24.109299+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:24.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:24 smithi163 bash[78107]: cluster 2023-10-09T09:14:24.122529+0000 osd.3 (osd.3) 186 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:24.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:24 smithi163 bash[75425]: cluster 2023-10-09T09:14:23.097428+0000 osd.1 (osd.1) 94 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:24.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:24 smithi163 bash[75425]: cluster 2023-10-09T09:14:23.317217+0000 mgr.y (mgr.34107) 223 : cluster 0 pgmap v439: 89 pgs: 89 active+clean; 203 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 537 B/s rd, 579 KiB/s wr, 4 op/s 2023-10-09T09:14:24.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:24 smithi163 bash[75425]: cluster 2023-10-09T09:14:24.108107+0000 mon.a (mon.0) 625 : cluster 0 osdmap e592: 8 total, 8 up, 8 in 2023-10-09T09:14:24.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:24 smithi163 bash[75425]: cluster 2023-10-09T09:14:24.109299+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:24.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:24 smithi163 bash[75425]: cluster 2023-10-09T09:14:24.122529+0000 osd.3 (osd.3) 186 : cluster 3 failed to encode map e591 with expected crc 2023-10-09T09:14:25.095 INFO:tasks.rados.rados.0.smithi163.stdout:2074: done (0 left) 2023-10-09T09:14:25.095 INFO:tasks.rados.rados.0.smithi163.stdout:2075: snap_remove snap 260 2023-10-09T09:14:25.771 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: cluster 2023-10-09T09:14:24.104965+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e592 with expected crc 2023-10-09T09:14:25.771 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: cluster 2023-10-09T09:14:24.106190+0000 osd.1 (osd.1) 95 : cluster 3 failed to encode map e592 with expected crc 2023-10-09T09:14:25.771 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: audit 2023-10-09T09:14:24.581992+0000 mon.a (mon.0) 626 : audit 0 from='client.? 172.21.15.163:0/155866134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:25.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: cluster 2023-10-09T09:14:25.089980+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: cluster 2023-10-09T09:14:25.091436+0000 osd.2 (osd.2) 187 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: cluster 2023-10-09T09:14:25.094789+0000 mon.a (mon.0) 627 : cluster 0 osdmap e593: 8 total, 8 up, 8 in 2023-10-09T09:14:25.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: cluster 2023-10-09T09:14:25.145766+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: cluster 2023-10-09T09:14:25.161849+0000 osd.3 (osd.3) 187 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: audit 2023-10-09T09:14:25.346600+0000 mon.a (mon.0) 628 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:25.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: audit 2023-10-09T09:14:25.348360+0000 mon.a (mon.0) 629 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:25.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[78107]: audit 2023-10-09T09:14:25.402121+0000 mon.a (mon.0) 630 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:25.772 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: cluster 2023-10-09T09:14:24.104965+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e592 with expected crc 2023-10-09T09:14:25.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: cluster 2023-10-09T09:14:24.106190+0000 osd.1 (osd.1) 95 : cluster 3 failed to encode map e592 with expected crc 2023-10-09T09:14:25.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: audit 2023-10-09T09:14:24.581992+0000 mon.a (mon.0) 626 : audit 0 from='client.? 172.21.15.163:0/155866134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:25.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: cluster 2023-10-09T09:14:25.089980+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: cluster 2023-10-09T09:14:25.091436+0000 osd.2 (osd.2) 187 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: cluster 2023-10-09T09:14:25.094789+0000 mon.a (mon.0) 627 : cluster 0 osdmap e593: 8 total, 8 up, 8 in 2023-10-09T09:14:25.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: cluster 2023-10-09T09:14:25.145766+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: cluster 2023-10-09T09:14:25.161849+0000 osd.3 (osd.3) 187 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: audit 2023-10-09T09:14:25.346600+0000 mon.a (mon.0) 628 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:25.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: audit 2023-10-09T09:14:25.348360+0000 mon.a (mon.0) 629 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:25.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:25 smithi163 bash[75425]: audit 2023-10-09T09:14:25.402121+0000 mon.a (mon.0) 630 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: cluster 2023-10-09T09:14:24.104965+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e592 with expected crc 2023-10-09T09:14:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: cluster 2023-10-09T09:14:24.106190+0000 osd.1 (osd.1) 95 : cluster 3 failed to encode map e592 with expected crc 2023-10-09T09:14:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: audit 2023-10-09T09:14:24.581992+0000 mon.a (mon.0) 626 : audit 0 from='client.? 172.21.15.163:0/155866134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: cluster 2023-10-09T09:14:25.089980+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: cluster 2023-10-09T09:14:25.091436+0000 osd.2 (osd.2) 187 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: cluster 2023-10-09T09:14:25.094789+0000 mon.a (mon.0) 627 : cluster 0 osdmap e593: 8 total, 8 up, 8 in 2023-10-09T09:14:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: cluster 2023-10-09T09:14:25.145766+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: cluster 2023-10-09T09:14:25.161849+0000 osd.3 (osd.3) 187 : cluster 3 failed to encode map e593 with expected crc 2023-10-09T09:14:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: audit 2023-10-09T09:14:25.346600+0000 mon.a (mon.0) 628 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: audit 2023-10-09T09:14:25.348360+0000 mon.a (mon.0) 629 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:25 smithi181 bash[59345]: audit 2023-10-09T09:14:25.402121+0000 mon.a (mon.0) 630 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:26.095 INFO:tasks.rados.rados.0.smithi163.stdout:2075: done (0 left) 2023-10-09T09:14:26.096 INFO:tasks.rados.rados.0.smithi163.stdout:2076: write oid 20 current snap is 268 2023-10-09T09:14:26.096 INFO:tasks.rados.rados.0.smithi163.stdout:2076: seq_num 777 ranges {659039=736479,2001450=643934} 2023-10-09T09:14:26.107 INFO:tasks.rados.rados.0.smithi163.stdout:2076: writing smithi16367185-20 from 659039 to 1395518 tid 1 2023-10-09T09:14:26.115 INFO:tasks.rados.rados.0.smithi163.stdout:2076: writing smithi16367185-20 from 2001450 to 2645384 tid 2 2023-10-09T09:14:26.115 INFO:tasks.rados.rados.0.smithi163.stdout:2077: snap_create 2023-10-09T09:14:26.121 INFO:tasks.rados.rados.0.smithi163.stdout:2076: finishing write tid 1 to smithi16367185-20 2023-10-09T09:14:26.123 INFO:tasks.rados.rados.0.smithi163.stdout:2076: finishing write tid 2 to smithi16367185-20 2023-10-09T09:14:26.123 INFO:tasks.rados.rados.0.smithi163.stdout:2076: finishing write tid 3 to smithi16367185-20 2023-10-09T09:14:26.123 INFO:tasks.rados.rados.0.smithi163.stdout:2076: finishing write tid 4 to smithi16367185-20 2023-10-09T09:14:26.123 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 236 (ObjNum 777 snap 268 seq_num 777) dirty exists 2023-10-09T09:14:26.123 INFO:tasks.rados.rados.0.smithi163.stdout:2076: left oid 20 (ObjNum 777 snap 268 seq_num 777) 2023-10-09T09:14:26.144 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:25 smithi163 systemd[1]: Stopping Ceph osd.1 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:14:26.468 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[41298]: debug 2023-10-09T09:14:26.139+0000 7fbd87ded700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:14:26.468 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[41298]: debug 2023-10-09T09:14:26.139+0000 7fbd87ded700 -1 osd.1 594 *** Got signal Terminated *** 2023-10-09T09:14:26.468 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[41298]: debug 2023-10-09T09:14:26.139+0000 7fbd87ded700 -1 osd.1 594 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:14:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:26 smithi181 bash[59345]: cluster 2023-10-09T09:14:25.317936+0000 mgr.y (mgr.34107) 224 : cluster 0 pgmap v442: 89 pgs: 89 active+clean; 203 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 259 KiB/s wr, 1 op/s 2023-10-09T09:14:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:26 smithi181 bash[59345]: cluster 2023-10-09T09:14:26.091488+0000 osd.2 (osd.2) 188 : cluster 3 failed to encode map e594 with expected crc 2023-10-09T09:14:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:26 smithi181 bash[59345]: cluster 2023-10-09T09:14:26.091654+0000 osd.3 (osd.3) 188 : cluster 3 failed to encode map e594 with expected crc 2023-10-09T09:14:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:26 smithi181 bash[59345]: cluster 2023-10-09T09:14:26.095576+0000 mon.a (mon.0) 631 : cluster 0 osdmap e594: 8 total, 8 up, 8 in 2023-10-09T09:14:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:26 smithi181 bash[59345]: cluster 2023-10-09T09:14:26.144482+0000 mon.a (mon.0) 632 : cluster 1 osd.1 marked itself down and dead 2023-10-09T09:14:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[78107]: cluster 2023-10-09T09:14:25.317936+0000 mgr.y (mgr.34107) 224 : cluster 0 pgmap v442: 89 pgs: 89 active+clean; 203 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 259 KiB/s wr, 1 op/s 2023-10-09T09:14:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[78107]: cluster 2023-10-09T09:14:26.091488+0000 osd.2 (osd.2) 188 : cluster 3 failed to encode map e594 with expected crc 2023-10-09T09:14:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[78107]: cluster 2023-10-09T09:14:26.091654+0000 osd.3 (osd.3) 188 : cluster 3 failed to encode map e594 with expected crc 2023-10-09T09:14:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[78107]: cluster 2023-10-09T09:14:26.095576+0000 mon.a (mon.0) 631 : cluster 0 osdmap e594: 8 total, 8 up, 8 in 2023-10-09T09:14:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[78107]: cluster 2023-10-09T09:14:26.144482+0000 mon.a (mon.0) 632 : cluster 1 osd.1 marked itself down and dead 2023-10-09T09:14:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[75425]: cluster 2023-10-09T09:14:25.317936+0000 mgr.y (mgr.34107) 224 : cluster 0 pgmap v442: 89 pgs: 89 active+clean; 203 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 259 KiB/s wr, 1 op/s 2023-10-09T09:14:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[75425]: cluster 2023-10-09T09:14:26.091488+0000 osd.2 (osd.2) 188 : cluster 3 failed to encode map e594 with expected crc 2023-10-09T09:14:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[75425]: cluster 2023-10-09T09:14:26.091654+0000 osd.3 (osd.3) 188 : cluster 3 failed to encode map e594 with expected crc 2023-10-09T09:14:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[75425]: cluster 2023-10-09T09:14:26.095576+0000 mon.a (mon.0) 631 : cluster 0 osdmap e594: 8 total, 8 up, 8 in 2023-10-09T09:14:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:26 smithi163 bash[75425]: cluster 2023-10-09T09:14:26.144482+0000 mon.a (mon.0) 632 : cluster 1 osd.1 marked itself down and dead 2023-10-09T09:14:27.108 INFO:tasks.rados.rados.0.smithi163.stdout:2076: done (1 left) 2023-10-09T09:14:27.109 INFO:tasks.rados.rados.0.smithi163.stdout:2077: done (0 left) 2023-10-09T09:14:27.109 INFO:tasks.rados.rados.0.smithi163.stdout:2078: snap_remove snap 219 2023-10-09T09:14:27.672 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[78107]: cluster 2023-10-09T09:14:26.110746+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e594 with expected crc 2023-10-09T09:14:27.672 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[78107]: audit 2023-10-09T09:14:26.941345+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.163:0/1711744825' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:27.672 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[78107]: cluster 2023-10-09T09:14:27.100669+0000 mon.a (mon.0) 633 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:14:27.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[78107]: cluster 2023-10-09T09:14:27.104380+0000 osd.3 (osd.3) 189 : cluster 3 failed to encode map e595 with expected crc 2023-10-09T09:14:27.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[78107]: cluster 2023-10-09T09:14:27.107035+0000 osd.2 (osd.2) 189 : cluster 3 failed to encode map e595 with expected crc 2023-10-09T09:14:27.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[78107]: cluster 2023-10-09T09:14:27.108076+0000 mon.a (mon.0) 634 : cluster 0 osdmap e595: 8 total, 7 up, 8 in 2023-10-09T09:14:27.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[75425]: cluster 2023-10-09T09:14:26.110746+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e594 with expected crc 2023-10-09T09:14:27.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[75425]: audit 2023-10-09T09:14:26.941345+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.163:0/1711744825' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:27.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[75425]: cluster 2023-10-09T09:14:27.100669+0000 mon.a (mon.0) 633 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:14:27.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[75425]: cluster 2023-10-09T09:14:27.104380+0000 osd.3 (osd.3) 189 : cluster 3 failed to encode map e595 with expected crc 2023-10-09T09:14:27.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[75425]: cluster 2023-10-09T09:14:27.107035+0000 osd.2 (osd.2) 189 : cluster 3 failed to encode map e595 with expected crc 2023-10-09T09:14:27.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[75425]: cluster 2023-10-09T09:14:27.108076+0000 mon.a (mon.0) 634 : cluster 0 osdmap e595: 8 total, 7 up, 8 in 2023-10-09T09:14:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:27 smithi181 bash[59345]: cluster 2023-10-09T09:14:26.110746+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e594 with expected crc 2023-10-09T09:14:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:27 smithi181 bash[59345]: audit 2023-10-09T09:14:26.941345+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.163:0/1711744825' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:27 smithi181 bash[59345]: cluster 2023-10-09T09:14:27.100669+0000 mon.a (mon.0) 633 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:14:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:27 smithi181 bash[59345]: cluster 2023-10-09T09:14:27.104380+0000 osd.3 (osd.3) 189 : cluster 3 failed to encode map e595 with expected crc 2023-10-09T09:14:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:27 smithi181 bash[59345]: cluster 2023-10-09T09:14:27.107035+0000 osd.2 (osd.2) 189 : cluster 3 failed to encode map e595 with expected crc 2023-10-09T09:14:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:27 smithi181 bash[59345]: cluster 2023-10-09T09:14:27.108076+0000 mon.a (mon.0) 634 : cluster 0 osdmap e595: 8 total, 7 up, 8 in 2023-10-09T09:14:27.857 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[86203]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T09:14:28.118 INFO:tasks.rados.rados.0.smithi163.stdout:2078: done (0 left) 2023-10-09T09:14:28.118 INFO:tasks.rados.rados.0.smithi163.stdout:2079: write oid 8 current snap is 269 2023-10-09T09:14:28.118 INFO:tasks.rados.rados.0.smithi163.stdout:2079: seq_num 778 ranges {675846=411728,1543579=787592,2848853=226993} 2023-10-09T09:14:28.125 INFO:tasks.rados.rados.0.smithi163.stdout:2079: writing smithi16367185-8 from 675846 to 1087574 tid 1 2023-10-09T09:14:28.134 INFO:tasks.rados.rados.0.smithi163.stdout:2079: writing smithi16367185-8 from 1543579 to 2331171 tid 2 2023-10-09T09:14:28.138 INFO:tasks.rados.rados.0.smithi163.stdout:2079: writing smithi16367185-8 from 2848853 to 3075846 tid 3 2023-10-09T09:14:28.138 INFO:tasks.rados.rados.0.smithi163.stdout:2080: delete oid 15 current snap is 269 2023-10-09T09:14:28.147 INFO:tasks.rados.rados.0.smithi163.stdout:2079: finishing write tid 1 to smithi16367185-8 2023-10-09T09:14:28.147 INFO:tasks.rados.rados.0.smithi163.stdout:2079: finishing write tid 2 to smithi16367185-8 2023-10-09T09:14:28.148 INFO:tasks.rados.rados.0.smithi163.stdout:2079: finishing write tid 3 to smithi16367185-8 2023-10-09T09:14:28.148 INFO:tasks.rados.rados.0.smithi163.stdout:2079: finishing write tid 4 to smithi16367185-8 2023-10-09T09:14:28.148 INFO:tasks.rados.rados.0.smithi163.stdout:2079: finishing write tid 5 to smithi16367185-8 2023-10-09T09:14:28.148 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1037 (ObjNum 778 snap 269 seq_num 778) dirty exists 2023-10-09T09:14:28.149 INFO:tasks.rados.rados.0.smithi163.stdout:2079: left oid 8 (ObjNum 778 snap 269 seq_num 778) 2023-10-09T09:14:28.149 INFO:tasks.rados.rados.0.smithi163.stdout:2079: done (1 left) 2023-10-09T09:14:28.149 INFO:tasks.rados.rados.0.smithi163.stdout:2080: done (0 left) 2023-10-09T09:14:28.149 INFO:tasks.rados.rados.0.smithi163.stdout:2081: delete oid 30 current snap is 269 2023-10-09T09:14:28.150 INFO:tasks.rados.rados.0.smithi163.stdout:2081: done (0 left) 2023-10-09T09:14:28.151 INFO:tasks.rados.rados.0.smithi163.stdout:2082: write oid 3 current snap is 269 2023-10-09T09:14:28.151 INFO:tasks.rados.rados.0.smithi163.stdout:2082: seq_num 781 ranges {726267=637475,2119025=651272} 2023-10-09T09:14:28.163 INFO:tasks.rados.rados.0.smithi163.stdout:2082: writing smithi16367185-3 from 726267 to 1363742 tid 1 2023-10-09T09:14:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:2082: writing smithi16367185-3 from 2119025 to 2770297 tid 2 2023-10-09T09:14:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:2083: snap_create 2023-10-09T09:14:28.176 INFO:tasks.rados.rados.0.smithi163.stdout:2082: finishing write tid 1 to smithi16367185-3 2023-10-09T09:14:28.180 INFO:tasks.rados.rados.0.smithi163.stdout:2082: finishing write tid 2 to smithi16367185-3 2023-10-09T09:14:28.181 INFO:tasks.rados.rados.0.smithi163.stdout:2082: finishing write tid 3 to smithi16367185-3 2023-10-09T09:14:28.181 INFO:tasks.rados.rados.0.smithi163.stdout:2082: finishing write tid 4 to smithi16367185-3 2023-10-09T09:14:28.181 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 547 (ObjNum 781 snap 269 seq_num 781) dirty exists 2023-10-09T09:14:28.181 INFO:tasks.rados.rados.0.smithi163.stdout:2082: left oid 3 (ObjNum 781 snap 269 seq_num 781) 2023-10-09T09:14:28.221 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:27 smithi163 bash[86293]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T09:14:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:28 smithi181 bash[59345]: cluster 2023-10-09T09:14:27.318808+0000 mgr.y (mgr.34107) 225 : cluster 0 pgmap v445: 89 pgs: 4 peering, 10 stale+active+clean, 75 active+clean; 210 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 979 KiB/s wr, 6 op/s 2023-10-09T09:14:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:28 smithi181 bash[59345]: cluster 2023-10-09T09:14:28.112474+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:28 smithi181 bash[59345]: cluster 2023-10-09T09:14:28.113092+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:28 smithi181 bash[59345]: cluster 2023-10-09T09:14:28.116634+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:28 smithi181 bash[59345]: cluster 2023-10-09T09:14:28.116759+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:28 smithi181 bash[59345]: cluster 2023-10-09T09:14:28.117813+0000 mon.a (mon.0) 635 : cluster 0 osdmap e596: 8 total, 7 up, 8 in 2023-10-09T09:14:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:28 smithi181 bash[59345]: cluster 2023-10-09T09:14:28.118165+0000 osd.3 (osd.3) 190 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:28 smithi181 bash[59345]: cluster 2023-10-09T09:14:28.118443+0000 osd.2 (osd.2) 190 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:28 smithi181 bash[59345]: cluster 2023-10-09T09:14:28.119272+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[75425]: cluster 2023-10-09T09:14:27.318808+0000 mgr.y (mgr.34107) 225 : cluster 0 pgmap v445: 89 pgs: 4 peering, 10 stale+active+clean, 75 active+clean; 210 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 979 KiB/s wr, 6 op/s 2023-10-09T09:14:28.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[75425]: cluster 2023-10-09T09:14:28.112474+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[75425]: cluster 2023-10-09T09:14:28.113092+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[75425]: cluster 2023-10-09T09:14:28.116634+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[75425]: cluster 2023-10-09T09:14:28.116759+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[75425]: cluster 2023-10-09T09:14:28.117813+0000 mon.a (mon.0) 635 : cluster 0 osdmap e596: 8 total, 7 up, 8 in 2023-10-09T09:14:28.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[75425]: cluster 2023-10-09T09:14:28.118165+0000 osd.3 (osd.3) 190 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[75425]: cluster 2023-10-09T09:14:28.118443+0000 osd.2 (osd.2) 190 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[75425]: cluster 2023-10-09T09:14:28.119272+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[78107]: cluster 2023-10-09T09:14:27.318808+0000 mgr.y (mgr.34107) 225 : cluster 0 pgmap v445: 89 pgs: 4 peering, 10 stale+active+clean, 75 active+clean; 210 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 979 KiB/s wr, 6 op/s 2023-10-09T09:14:28.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[78107]: cluster 2023-10-09T09:14:28.112474+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[78107]: cluster 2023-10-09T09:14:28.113092+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[78107]: cluster 2023-10-09T09:14:28.116634+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[78107]: cluster 2023-10-09T09:14:28.116759+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[78107]: cluster 2023-10-09T09:14:28.117813+0000 mon.a (mon.0) 635 : cluster 0 osdmap e596: 8 total, 7 up, 8 in 2023-10-09T09:14:28.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[78107]: cluster 2023-10-09T09:14:28.118165+0000 osd.3 (osd.3) 190 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[78107]: cluster 2023-10-09T09:14:28.118443+0000 osd.2 (osd.2) 190 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:28.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:28 smithi163 bash[78107]: cluster 2023-10-09T09:14:28.119272+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:29.119 INFO:tasks.rados.rados.0.smithi163.stdout:2082: done (1 left) 2023-10-09T09:14:29.119 INFO:tasks.rados.rados.0.smithi163.stdout:2083: done (0 left) 2023-10-09T09:14:29.120 INFO:tasks.rados.rados.0.smithi163.stdout:2084: write oid 18 current snap is 270 2023-10-09T09:14:29.120 INFO:tasks.rados.rados.0.smithi163.stdout:2084: seq_num 782 ranges {743074=712724,2061154=794930} 2023-10-09T09:14:29.130 INFO:tasks.rados.rados.0.smithi163.stdout:2084: writing smithi16367185-18 from 743074 to 1455798 tid 1 2023-10-09T09:14:29.140 INFO:tasks.rados.rados.0.smithi163.stdout:2084: writing smithi16367185-18 from 2061154 to 2856084 tid 2 2023-10-09T09:14:29.141 INFO:tasks.rados.rados.0.smithi163.stdout:2085: snap_remove snap 264 2023-10-09T09:14:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[75425]: cluster 2023-10-09T09:14:28.118162+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[75425]: cluster 2023-10-09T09:14:29.115122+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e597 with expected crc 2023-10-09T09:14:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[75425]: cluster 2023-10-09T09:14:29.115501+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e597 with expected crc 2023-10-09T09:14:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[75425]: cluster 2023-10-09T09:14:29.119258+0000 mon.a (mon.0) 636 : cluster 0 osdmap e597: 8 total, 7 up, 8 in 2023-10-09T09:14:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[75425]: cluster 2023-10-09T09:14:29.133129+0000 osd.2 (osd.2) 191 : cluster 3 failed to encode map e597 with expected crc 2023-10-09T09:14:29.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[75425]: audit 2023-10-09T09:14:29.314418+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.163:0/2415142744' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:29.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[78107]: cluster 2023-10-09T09:14:28.118162+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:29.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[78107]: cluster 2023-10-09T09:14:29.115122+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e597 with expected crc 2023-10-09T09:14:29.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[78107]: cluster 2023-10-09T09:14:29.115501+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e597 with expected crc 2023-10-09T09:14:29.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[78107]: cluster 2023-10-09T09:14:29.119258+0000 mon.a (mon.0) 636 : cluster 0 osdmap e597: 8 total, 7 up, 8 in 2023-10-09T09:14:29.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[78107]: cluster 2023-10-09T09:14:29.133129+0000 osd.2 (osd.2) 191 : cluster 3 failed to encode map e597 with expected crc 2023-10-09T09:14:29.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:29 smithi163 bash[78107]: audit 2023-10-09T09:14:29.314418+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.163:0/2415142744' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:29 smithi181 bash[59345]: cluster 2023-10-09T09:14:28.118162+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e596 with expected crc 2023-10-09T09:14:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:29 smithi181 bash[59345]: cluster 2023-10-09T09:14:29.115122+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e597 with expected crc 2023-10-09T09:14:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:29 smithi181 bash[59345]: cluster 2023-10-09T09:14:29.115501+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e597 with expected crc 2023-10-09T09:14:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:29 smithi181 bash[59345]: cluster 2023-10-09T09:14:29.119258+0000 mon.a (mon.0) 636 : cluster 0 osdmap e597: 8 total, 7 up, 8 in 2023-10-09T09:14:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:29 smithi181 bash[59345]: cluster 2023-10-09T09:14:29.133129+0000 osd.2 (osd.2) 191 : cluster 3 failed to encode map e597 with expected crc 2023-10-09T09:14:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:29 smithi181 bash[59345]: audit 2023-10-09T09:14:29.314418+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.163:0/2415142744' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:30.098 INFO:tasks.rados.rados.0.smithi163.stdout:2085: done (1 left) 2023-10-09T09:14:30.098 INFO:tasks.rados.rados.0.smithi163.stdout:2086: delete oid 38 current snap is 270 2023-10-09T09:14:30.098 INFO:tasks.rados.rados.0.smithi163.stdout:2084: finishing write tid 1 to smithi16367185-18 2023-10-09T09:14:30.098 INFO:tasks.rados.rados.0.smithi163.stdout:2084: finishing write tid 2 to smithi16367185-18 2023-10-09T09:14:30.098 INFO:tasks.rados.rados.0.smithi163.stdout:2084: finishing write tid 3 to smithi16367185-18 2023-10-09T09:14:30.099 INFO:tasks.rados.rados.0.smithi163.stdout:2084: finishing write tid 4 to smithi16367185-18 2023-10-09T09:14:30.099 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 685 (ObjNum 782 snap 270 seq_num 782) dirty exists 2023-10-09T09:14:30.099 INFO:tasks.rados.rados.0.smithi163.stdout:2084: left oid 18 (ObjNum 782 snap 270 seq_num 782) 2023-10-09T09:14:30.113 INFO:tasks.rados.rados.0.smithi163.stdout:2084: done (1 left) 2023-10-09T09:14:30.113 INFO:tasks.rados.rados.0.smithi163.stdout:2086: done (0 left) 2023-10-09T09:14:30.113 INFO:tasks.rados.rados.0.smithi163.stdout:2087: read oid 43 snap -1 2023-10-09T09:14:30.113 INFO:tasks.rados.rados.0.smithi163.stdout:2087: expect (ObjNum 733 snap 246 seq_num 733) 2023-10-09T09:14:30.113 INFO:tasks.rados.rados.0.smithi163.stdout:2088: snap_remove snap 248 2023-10-09T09:14:30.244 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:29 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T09:14:30.244 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:29 smithi163 systemd[1]: Stopped Ceph osd.1 for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:14:30.531 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:30 smithi163 systemd[1]: Started Ceph osd.1 for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:14:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[75425]: cluster 2023-10-09T09:14:29.319608+0000 mgr.y (mgr.34107) 226 : cluster 0 pgmap v448: 89 pgs: 4 peering, 10 stale+active+clean, 75 active+clean; 210 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 979 KiB/s wr, 6 op/s 2023-10-09T09:14:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[75425]: cluster 2023-10-09T09:14:30.091305+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e598 with expected crc 2023-10-09T09:14:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[75425]: cluster 2023-10-09T09:14:30.097852+0000 mon.a (mon.0) 637 : cluster 0 osdmap e598: 8 total, 7 up, 8 in 2023-10-09T09:14:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[75425]: cluster 2023-10-09T09:14:30.098543+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e598 with expected crc 2023-10-09T09:14:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[75425]: cluster 2023-10-09T09:14:30.138750+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e598 with expected crc 2023-10-09T09:14:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[75425]: audit 2023-10-09T09:14:30.278154+0000 mon.a (mon.0) 638 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[75425]: audit 2023-10-09T09:14:30.287275+0000 mon.a (mon.0) 639 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[75425]: audit 2023-10-09T09:14:30.297311+0000 mon.a (mon.0) 640 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[78107]: cluster 2023-10-09T09:14:29.319608+0000 mgr.y (mgr.34107) 226 : cluster 0 pgmap v448: 89 pgs: 4 peering, 10 stale+active+clean, 75 active+clean; 210 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 979 KiB/s wr, 6 op/s 2023-10-09T09:14:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[78107]: cluster 2023-10-09T09:14:30.091305+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e598 with expected crc 2023-10-09T09:14:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[78107]: cluster 2023-10-09T09:14:30.097852+0000 mon.a (mon.0) 637 : cluster 0 osdmap e598: 8 total, 7 up, 8 in 2023-10-09T09:14:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[78107]: cluster 2023-10-09T09:14:30.098543+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e598 with expected crc 2023-10-09T09:14:30.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[78107]: cluster 2023-10-09T09:14:30.138750+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e598 with expected crc 2023-10-09T09:14:30.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[78107]: audit 2023-10-09T09:14:30.278154+0000 mon.a (mon.0) 638 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:30.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[78107]: audit 2023-10-09T09:14:30.287275+0000 mon.a (mon.0) 639 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:30.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:30 smithi163 bash[78107]: audit 2023-10-09T09:14:30.297311+0000 mon.a (mon.0) 640 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:30 smithi181 bash[59345]: cluster 2023-10-09T09:14:29.319608+0000 mgr.y (mgr.34107) 226 : cluster 0 pgmap v448: 89 pgs: 4 peering, 10 stale+active+clean, 75 active+clean; 210 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 979 KiB/s wr, 6 op/s 2023-10-09T09:14:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:30 smithi181 bash[59345]: cluster 2023-10-09T09:14:30.091305+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e598 with expected crc 2023-10-09T09:14:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:30 smithi181 bash[59345]: cluster 2023-10-09T09:14:30.097852+0000 mon.a (mon.0) 637 : cluster 0 osdmap e598: 8 total, 7 up, 8 in 2023-10-09T09:14:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:30 smithi181 bash[59345]: cluster 2023-10-09T09:14:30.098543+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e598 with expected crc 2023-10-09T09:14:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:30 smithi181 bash[59345]: cluster 2023-10-09T09:14:30.138750+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e598 with expected crc 2023-10-09T09:14:30.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:30 smithi181 bash[59345]: audit 2023-10-09T09:14:30.278154+0000 mon.a (mon.0) 638 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:30.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:30 smithi181 bash[59345]: audit 2023-10-09T09:14:30.287275+0000 mon.a (mon.0) 639 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:30.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:30 smithi181 bash[59345]: audit 2023-10-09T09:14:30.297311+0000 mon.a (mon.0) 640 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:31.108 INFO:tasks.rados.rados.0.smithi163.stdout:2088: done (1 left) 2023-10-09T09:14:31.108 INFO:tasks.rados.rados.0.smithi163.stdout:2089: rollback oid 46 current snap is 270 2023-10-09T09:14:31.108 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 46 to 258 2023-10-09T09:14:31.109 INFO:tasks.rados.rados.0.smithi163.stdout:2087: expect (ObjNum 733 snap 246 seq_num 733) 2023-10-09T09:14:31.256 INFO:tasks.rados.rados.0.smithi163.stdout:2089: finishing rollback tid 0 to smithi16367185-46 2023-10-09T09:14:31.257 INFO:tasks.rados.rados.0.smithi163.stdout:2089: finishing rollback tid 1 to smithi16367185-46 2023-10-09T09:14:31.257 INFO:tasks.rados.rados.0.smithi163.stdout:2089: finishing rollback tid 2 to smithi16367185-46 2023-10-09T09:14:31.257 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 853 (ObjNum 538 snap 168 seq_num 538) dirty exists 2023-10-09T09:14:31.257 INFO:tasks.rados.rados.0.smithi163.stdout:2087: done (1 left) 2023-10-09T09:14:31.257 INFO:tasks.rados.rados.0.smithi163.stdout:2089: done (0 left) 2023-10-09T09:14:31.257 INFO:tasks.rados.rados.0.smithi163.stdout:2090: snap_create 2023-10-09T09:14:32.121 INFO:tasks.rados.rados.0.smithi163.stdout:2090: done (0 left) 2023-10-09T09:14:32.121 INFO:tasks.rados.rados.0.smithi163.stdout:2091: read oid 24 snap -1 2023-10-09T09:14:32.121 INFO:tasks.rados.rados.0.smithi163.stdout:2091: expect (ObjNum 755 snap 256 seq_num 755) 2023-10-09T09:14:32.121 INFO:tasks.rados.rados.0.smithi163.stdout:2092: read oid 23 snap -1 2023-10-09T09:14:32.121 INFO:tasks.rados.rados.0.smithi163.stdout:2092: expect (ObjNum 775 snap 268 seq_num 775) 2023-10-09T09:14:32.121 INFO:tasks.rados.rados.0.smithi163.stdout:2093: delete oid 46 current snap is 271 2023-10-09T09:14:32.128 INFO:tasks.rados.rados.0.smithi163.stdout:2091: expect (ObjNum 755 snap 256 seq_num 755) 2023-10-09T09:14:32.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[75425]: cluster 2023-10-09T09:14:31.103793+0000 osd.3 (osd.3) 191 : cluster 3 failed to encode map e599 with expected crc 2023-10-09T09:14:32.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[75425]: cluster 2023-10-09T09:14:31.104306+0000 osd.2 (osd.2) 192 : cluster 3 failed to encode map e599 with expected crc 2023-10-09T09:14:32.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[75425]: cluster 2023-10-09T09:14:31.108236+0000 mon.a (mon.0) 641 : cluster 0 osdmap e599: 8 total, 7 up, 8 in 2023-10-09T09:14:32.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[75425]: cluster 2023-10-09T09:14:31.110219+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e599 with expected crc 2023-10-09T09:14:32.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[75425]: cluster 2023-10-09T09:14:31.321219+0000 mgr.y (mgr.34107) 227 : cluster 0 pgmap v451: 89 pgs: 6 active+undersized+degraded, 4 peering, 27 active+undersized, 52 active+clean; 204 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 717 KiB/s wr, 6 op/s; 44/411 objects degraded (10.706%) 2023-10-09T09:14:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[75425]: audit 2023-10-09T09:14:31.691223+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.163:0/3984683791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:32.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[78107]: cluster 2023-10-09T09:14:31.103793+0000 osd.3 (osd.3) 191 : cluster 3 failed to encode map e599 with expected crc 2023-10-09T09:14:32.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[78107]: cluster 2023-10-09T09:14:31.104306+0000 osd.2 (osd.2) 192 : cluster 3 failed to encode map e599 with expected crc 2023-10-09T09:14:32.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[78107]: cluster 2023-10-09T09:14:31.108236+0000 mon.a (mon.0) 641 : cluster 0 osdmap e599: 8 total, 7 up, 8 in 2023-10-09T09:14:32.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[78107]: cluster 2023-10-09T09:14:31.110219+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e599 with expected crc 2023-10-09T09:14:32.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[78107]: cluster 2023-10-09T09:14:31.321219+0000 mgr.y (mgr.34107) 227 : cluster 0 pgmap v451: 89 pgs: 6 active+undersized+degraded, 4 peering, 27 active+undersized, 52 active+clean; 204 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 717 KiB/s wr, 6 op/s; 44/411 objects degraded (10.706%) 2023-10-09T09:14:32.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:32 smithi163 bash[78107]: audit 2023-10-09T09:14:31.691223+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.163:0/3984683791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:32.220 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:31 smithi163 bash[86478]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T09:14:32.220 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:31 smithi163 bash[86478]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T09:14:32.220 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:31 smithi163 bash[86478]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T09:14:32.220 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:31 smithi163 bash[86478]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T09:14:32.220 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:31 smithi163 bash[86478]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T09:14:32.220 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:31 smithi163 bash[86478]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T09:14:32.220 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:31 smithi163 bash[86478]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T09:14:32.220 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:31 smithi163 bash[86478]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T09:14:32.285 INFO:tasks.rados.rados.0.smithi163.stdout:2092: expect (ObjNum 775 snap 268 seq_num 775) 2023-10-09T09:14:32.460 INFO:tasks.rados.rados.0.smithi163.stdout:2091: done (2 left) 2023-10-09T09:14:32.460 INFO:tasks.rados.rados.0.smithi163.stdout:2092: done (1 left) 2023-10-09T09:14:32.460 INFO:tasks.rados.rados.0.smithi163.stdout:2093: done (0 left) 2023-10-09T09:14:32.460 INFO:tasks.rados.rados.0.smithi163.stdout:2094: write oid 24 current snap is 271 2023-10-09T09:14:32.460 INFO:tasks.rados.rados.0.smithi163.stdout:2094: seq_num 785 ranges {793495=654824,1952953=658610,3157167=36328} 2023-10-09T09:14:32.475 INFO:tasks.rados.rados.0.smithi163.stdout:2094: writing smithi16367185-24 from 793495 to 1448319 tid 1 2023-10-09T09:14:32.482 INFO:tasks.rados.rados.0.smithi163.stdout:2094: writing smithi16367185-24 from 1952953 to 2611563 tid 2 2023-10-09T09:14:32.485 INFO:tasks.rados.rados.0.smithi163.stdout:2094: writing smithi16367185-24 from 3157167 to 3193495 tid 3 2023-10-09T09:14:32.485 INFO:tasks.rados.rados.0.smithi163.stdout:2095: snap_remove snap 265 2023-10-09T09:14:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:32 smithi181 bash[59345]: cluster 2023-10-09T09:14:31.103793+0000 osd.3 (osd.3) 191 : cluster 3 failed to encode map e599 with expected crc 2023-10-09T09:14:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:32 smithi181 bash[59345]: cluster 2023-10-09T09:14:31.104306+0000 osd.2 (osd.2) 192 : cluster 3 failed to encode map e599 with expected crc 2023-10-09T09:14:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:32 smithi181 bash[59345]: cluster 2023-10-09T09:14:31.108236+0000 mon.a (mon.0) 641 : cluster 0 osdmap e599: 8 total, 7 up, 8 in 2023-10-09T09:14:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:32 smithi181 bash[59345]: cluster 2023-10-09T09:14:31.110219+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e599 with expected crc 2023-10-09T09:14:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:32 smithi181 bash[59345]: cluster 2023-10-09T09:14:31.321219+0000 mgr.y (mgr.34107) 227 : cluster 0 pgmap v451: 89 pgs: 6 active+undersized+degraded, 4 peering, 27 active+undersized, 52 active+clean; 204 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 717 KiB/s wr, 6 op/s; 44/411 objects degraded (10.706%) 2023-10-09T09:14:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:32 smithi181 bash[59345]: audit 2023-10-09T09:14:31.691223+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.163:0/3984683791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:33.128 INFO:tasks.rados.rados.0.smithi163.stdout:2095: done (1 left) 2023-10-09T09:14:33.128 INFO:tasks.rados.rados.0.smithi163.stdout:2096: snap_remove snap 261 2023-10-09T09:14:33.467 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:14:33] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.33.4" 2023-10-09T09:14:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[75425]: cluster 2023-10-09T09:14:32.102529+0000 mon.a (mon.0) 642 : cluster 3 Health check failed: Degraded data redundancy: 44/411 objects degraded (10.706%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[75425]: cluster 2023-10-09T09:14:32.114688+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[75425]: cluster 2023-10-09T09:14:32.116460+0000 osd.3 (osd.3) 192 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[75425]: cluster 2023-10-09T09:14:32.120889+0000 mon.a (mon.0) 643 : cluster 0 osdmap e600: 8 total, 7 up, 8 in 2023-10-09T09:14:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[75425]: cluster 2023-10-09T09:14:32.121793+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[75425]: cluster 2023-10-09T09:14:32.122566+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[78107]: cluster 2023-10-09T09:14:32.102529+0000 mon.a (mon.0) 642 : cluster 3 Health check failed: Degraded data redundancy: 44/411 objects degraded (10.706%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:33.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[78107]: cluster 2023-10-09T09:14:32.114688+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[78107]: cluster 2023-10-09T09:14:32.116460+0000 osd.3 (osd.3) 192 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[78107]: cluster 2023-10-09T09:14:32.120889+0000 mon.a (mon.0) 643 : cluster 0 osdmap e600: 8 total, 7 up, 8 in 2023-10-09T09:14:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[78107]: cluster 2023-10-09T09:14:32.121793+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[78107]: cluster 2023-10-09T09:14:32.122566+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:33 smithi181 bash[59345]: cluster 2023-10-09T09:14:32.102529+0000 mon.a (mon.0) 642 : cluster 3 Health check failed: Degraded data redundancy: 44/411 objects degraded (10.706%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:33.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:33 smithi181 bash[59345]: cluster 2023-10-09T09:14:32.114688+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:33 smithi181 bash[59345]: cluster 2023-10-09T09:14:32.116460+0000 osd.3 (osd.3) 192 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:33 smithi181 bash[59345]: cluster 2023-10-09T09:14:32.120889+0000 mon.a (mon.0) 643 : cluster 0 osdmap e600: 8 total, 7 up, 8 in 2023-10-09T09:14:33.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:33 smithi181 bash[59345]: cluster 2023-10-09T09:14:32.121793+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:33 smithi181 bash[59345]: cluster 2023-10-09T09:14:32.122566+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e600 with expected crc 2023-10-09T09:14:33.718 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:33 smithi163 bash[86800]: debug 2023-10-09T09:14:33.587+0000 7f655e38d700 1 -- 172.21.15.163:0/3385775296 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55eb3c9904e0 con 0x55eb3c9eac00 2023-10-09T09:14:34.139 INFO:tasks.rados.rados.0.smithi163.stdout:2096: done (1 left) 2023-10-09T09:14:34.139 INFO:tasks.rados.rados.0.smithi163.stdout:2097: rollback oid 33 current snap is 271 2023-10-09T09:14:34.139 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 235 2023-10-09T09:14:34.139 INFO:tasks.rados.rados.0.smithi163.stdout:2094: finishing write tid 1 to smithi16367185-24 2023-10-09T09:14:34.139 INFO:tasks.rados.rados.0.smithi163.stdout:2094: finishing write tid 2 to smithi16367185-24 2023-10-09T09:14:34.139 INFO:tasks.rados.rados.0.smithi163.stdout:2094: finishing write tid 3 to smithi16367185-24 2023-10-09T09:14:34.140 INFO:tasks.rados.rados.0.smithi163.stdout:2094: finishing write tid 4 to smithi16367185-24 2023-10-09T09:14:34.140 INFO:tasks.rados.rados.0.smithi163.stdout:2094: finishing write tid 5 to smithi16367185-24 2023-10-09T09:14:34.140 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 522 (ObjNum 785 snap 271 seq_num 785) dirty exists 2023-10-09T09:14:34.140 INFO:tasks.rados.rados.0.smithi163.stdout:2094: left oid 24 (ObjNum 785 snap 271 seq_num 785) 2023-10-09T09:14:34.140 INFO:tasks.rados.rados.0.smithi163.stdout:2094: done (1 left) 2023-10-09T09:14:34.140 INFO:tasks.rados.rados.0.smithi163.stdout:2098: write oid 45 current snap is 271 2023-10-09T09:14:34.140 INFO:tasks.rados.rados.0.smithi163.stdout:2098: seq_num 786 ranges {410302=730073,1895082=402268,2951884=258418} 2023-10-09T09:14:34.149 INFO:tasks.rados.rados.0.smithi163.stdout:2098: writing smithi16367185-45 from 410302 to 1140375 tid 1 2023-10-09T09:14:34.157 INFO:tasks.rados.rados.0.smithi163.stdout:2098: writing smithi16367185-45 from 1895082 to 2297350 tid 2 2023-10-09T09:14:34.163 INFO:tasks.rados.rados.0.smithi163.stdout:2098: writing smithi16367185-45 from 2951884 to 3210302 tid 3 2023-10-09T09:14:34.163 INFO:tasks.rados.rados.0.smithi163.stdout:2099: snap_remove snap 267 2023-10-09T09:14:34.468 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[86800]: debug 2023-10-09T09:14:34.143+0000 7f6567513640 -1 Falling back to public interface 2023-10-09T09:14:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[78107]: cluster 2023-10-09T09:14:33.124917+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[78107]: cluster 2023-10-09T09:14:33.126457+0000 osd.3 (osd.3) 193 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[78107]: cluster 2023-10-09T09:14:33.128465+0000 mon.a (mon.0) 644 : cluster 0 osdmap e601: 8 total, 7 up, 8 in 2023-10-09T09:14:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[78107]: cluster 2023-10-09T09:14:33.321887+0000 mgr.y (mgr.34107) 228 : cluster 0 pgmap v454: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 205 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.0 MiB/s wr, 7 op/s; 51/411 objects degraded (12.409%) 2023-10-09T09:14:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[78107]: audit 2023-10-09T09:14:34.062274+0000 mon.a (mon.0) 645 : audit 0 from='client.? 172.21.15.163:0/1123371709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[75425]: cluster 2023-10-09T09:14:33.124917+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[75425]: cluster 2023-10-09T09:14:33.126457+0000 osd.3 (osd.3) 193 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[75425]: cluster 2023-10-09T09:14:33.128465+0000 mon.a (mon.0) 644 : cluster 0 osdmap e601: 8 total, 7 up, 8 in 2023-10-09T09:14:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[75425]: cluster 2023-10-09T09:14:33.321887+0000 mgr.y (mgr.34107) 228 : cluster 0 pgmap v454: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 205 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.0 MiB/s wr, 7 op/s; 51/411 objects degraded (12.409%) 2023-10-09T09:14:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:34 smithi163 bash[75425]: audit 2023-10-09T09:14:34.062274+0000 mon.a (mon.0) 645 : audit 0 from='client.? 172.21.15.163:0/1123371709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:34 smithi181 bash[59345]: cluster 2023-10-09T09:14:33.124917+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:34 smithi181 bash[59345]: cluster 2023-10-09T09:14:33.126457+0000 osd.3 (osd.3) 193 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:34 smithi181 bash[59345]: cluster 2023-10-09T09:14:33.128465+0000 mon.a (mon.0) 644 : cluster 0 osdmap e601: 8 total, 7 up, 8 in 2023-10-09T09:14:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:34 smithi181 bash[59345]: cluster 2023-10-09T09:14:33.321887+0000 mgr.y (mgr.34107) 228 : cluster 0 pgmap v454: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 205 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.0 MiB/s wr, 7 op/s; 51/411 objects degraded (12.409%) 2023-10-09T09:14:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:34 smithi181 bash[59345]: audit 2023-10-09T09:14:34.062274+0000 mon.a (mon.0) 645 : audit 0 from='client.? 172.21.15.163:0/1123371709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:35.147 INFO:tasks.rados.rados.0.smithi163.stdout:2099: done (2 left) 2023-10-09T09:14:35.147 INFO:tasks.rados.rados.0.smithi163.stdout:2100: read oid 5 snap -1 2023-10-09T09:14:35.147 INFO:tasks.rados.rados.0.smithi163.stdout:2100: expect (ObjNum 710 snap 237 seq_num 710) 2023-10-09T09:14:35.148 INFO:tasks.rados.rados.0.smithi163.stdout:2101: snap_remove snap 269 2023-10-09T09:14:35.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[78107]: cluster 2023-10-09T09:14:33.811846+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[78107]: cluster 2023-10-09T09:14:33.812052+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[78107]: cluster 2023-10-09T09:14:33.812131+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[78107]: cluster 2023-10-09T09:14:33.812932+0000 osd.2 (osd.2) 193 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[78107]: cluster 2023-10-09T09:14:34.135652+0000 osd.3 (osd.3) 194 : cluster 3 failed to encode map e602 with expected crc 2023-10-09T09:14:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[78107]: cluster 2023-10-09T09:14:34.136773+0000 osd.2 (osd.2) 194 : cluster 3 failed to encode map e602 with expected crc 2023-10-09T09:14:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[78107]: cluster 2023-10-09T09:14:34.138867+0000 mon.a (mon.0) 646 : cluster 0 osdmap e602: 8 total, 7 up, 8 in 2023-10-09T09:14:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[78107]: cluster 2023-10-09T09:14:34.167134+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e602 with expected crc 2023-10-09T09:14:35.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[75425]: cluster 2023-10-09T09:14:33.811846+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[75425]: cluster 2023-10-09T09:14:33.812052+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[75425]: cluster 2023-10-09T09:14:33.812131+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[75425]: cluster 2023-10-09T09:14:33.812932+0000 osd.2 (osd.2) 193 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[75425]: cluster 2023-10-09T09:14:34.135652+0000 osd.3 (osd.3) 194 : cluster 3 failed to encode map e602 with expected crc 2023-10-09T09:14:35.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[75425]: cluster 2023-10-09T09:14:34.136773+0000 osd.2 (osd.2) 194 : cluster 3 failed to encode map e602 with expected crc 2023-10-09T09:14:35.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[75425]: cluster 2023-10-09T09:14:34.138867+0000 mon.a (mon.0) 646 : cluster 0 osdmap e602: 8 total, 7 up, 8 in 2023-10-09T09:14:35.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:35 smithi163 bash[75425]: cluster 2023-10-09T09:14:34.167134+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e602 with expected crc 2023-10-09T09:14:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:35 smithi181 bash[59345]: cluster 2023-10-09T09:14:33.811846+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:35 smithi181 bash[59345]: cluster 2023-10-09T09:14:33.812052+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:35 smithi181 bash[59345]: cluster 2023-10-09T09:14:33.812131+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:35 smithi181 bash[59345]: cluster 2023-10-09T09:14:33.812932+0000 osd.2 (osd.2) 193 : cluster 3 failed to encode map e601 with expected crc 2023-10-09T09:14:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:35 smithi181 bash[59345]: cluster 2023-10-09T09:14:34.135652+0000 osd.3 (osd.3) 194 : cluster 3 failed to encode map e602 with expected crc 2023-10-09T09:14:35.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:35 smithi181 bash[59345]: cluster 2023-10-09T09:14:34.136773+0000 osd.2 (osd.2) 194 : cluster 3 failed to encode map e602 with expected crc 2023-10-09T09:14:35.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:35 smithi181 bash[59345]: cluster 2023-10-09T09:14:34.138867+0000 mon.a (mon.0) 646 : cluster 0 osdmap e602: 8 total, 7 up, 8 in 2023-10-09T09:14:35.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:35 smithi181 bash[59345]: cluster 2023-10-09T09:14:34.167134+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e602 with expected crc 2023-10-09T09:14:36.155 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[75425]: cluster 2023-10-09T09:14:35.143090+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.155 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[75425]: cluster 2023-10-09T09:14:35.143566+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.155 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[75425]: cluster 2023-10-09T09:14:35.147257+0000 mon.a (mon.0) 647 : cluster 0 osdmap e603: 8 total, 7 up, 8 in 2023-10-09T09:14:36.155 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[75425]: cluster 2023-10-09T09:14:35.148983+0000 osd.2 (osd.2) 195 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.155 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[75425]: cluster 2023-10-09T09:14:35.210167+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.155 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[75425]: cluster 2023-10-09T09:14:35.322616+0000 mgr.y (mgr.34107) 229 : cluster 0 pgmap v457: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 205 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 315 KiB/s wr, 1 op/s; 51/411 objects degraded (12.409%) 2023-10-09T09:14:36.163 INFO:tasks.rados.rados.0.smithi163.stdout:2101: done (3 left) 2023-10-09T09:14:36.163 INFO:tasks.rados.rados.0.smithi163.stdout:2102: read oid 4 snap -1 2023-10-09T09:14:36.163 INFO:tasks.rados.rados.0.smithi163.stdout:2102: expect (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:14:36.163 INFO:tasks.rados.rados.0.smithi163.stdout:2103: write oid 46 current snap is 271 2023-10-09T09:14:36.163 INFO:tasks.rados.rados.0.smithi163.stdout:2103: seq_num 787 ranges {427109=405322,1553564=545926,2579307=647802} 2023-10-09T09:14:36.170 INFO:tasks.rados.rados.0.smithi163.stdout:2103: writing smithi16367185-46 from 427109 to 832431 tid 1 2023-10-09T09:14:36.178 INFO:tasks.rados.rados.0.smithi163.stdout:2103: writing smithi16367185-46 from 1553564 to 2099490 tid 2 2023-10-09T09:14:36.185 INFO:tasks.rados.rados.0.smithi163.stdout:2103: writing smithi16367185-46 from 2579307 to 3227109 tid 3 2023-10-09T09:14:36.185 INFO:tasks.rados.rados.0.smithi163.stdout:2104: read oid 35 snap -1 2023-10-09T09:14:36.185 INFO:tasks.rados.rados.0.smithi163.stdout:2104: expect deleted 2023-10-09T09:14:36.185 INFO:tasks.rados.rados.0.smithi163.stdout:2105: delete oid 1 current snap is 271 2023-10-09T09:14:36.185 INFO:tasks.rados.rados.0.smithi163.stdout:2098: finishing write tid 1 to smithi16367185-45 2023-10-09T09:14:36.185 INFO:tasks.rados.rados.0.smithi163.stdout:2097: finishing rollback tid 0 to smithi16367185-33 2023-10-09T09:14:36.185 INFO:tasks.rados.rados.0.smithi163.stdout:2097: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:14:36.186 INFO:tasks.rados.rados.0.smithi163.stdout:2097: finishing rollback tid 2 to smithi16367185-33 2023-10-09T09:14:36.186 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 981 (ObjNum 634 snap 202 seq_num 634) dirty exists 2023-10-09T09:14:36.186 INFO:tasks.rados.rados.0.smithi163.stdout:2098: finishing write tid 2 to smithi16367185-45 2023-10-09T09:14:36.186 INFO:tasks.rados.rados.0.smithi163.stdout:2098: finishing write tid 3 to smithi16367185-45 2023-10-09T09:14:36.186 INFO:tasks.rados.rados.0.smithi163.stdout:2098: finishing write tid 4 to smithi16367185-45 2023-10-09T09:14:36.186 INFO:tasks.rados.rados.0.smithi163.stdout:2098: finishing write tid 5 to smithi16367185-45 2023-10-09T09:14:36.186 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 1516 (ObjNum 786 snap 271 seq_num 786) dirty exists 2023-10-09T09:14:36.186 INFO:tasks.rados.rados.0.smithi163.stdout:2098: left oid 45 (ObjNum 786 snap 271 seq_num 786) 2023-10-09T09:14:36.186 INFO:tasks.rados.rados.0.smithi163.stdout:2100: expect (ObjNum 710 snap 237 seq_num 710) 2023-10-09T09:14:36.333 INFO:tasks.rados.rados.0.smithi163.stdout:2102: expect (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:14:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[78107]: cluster 2023-10-09T09:14:35.143090+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[78107]: cluster 2023-10-09T09:14:35.143566+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[78107]: cluster 2023-10-09T09:14:35.147257+0000 mon.a (mon.0) 647 : cluster 0 osdmap e603: 8 total, 7 up, 8 in 2023-10-09T09:14:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[78107]: cluster 2023-10-09T09:14:35.148983+0000 osd.2 (osd.2) 195 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[78107]: cluster 2023-10-09T09:14:35.210167+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:36 smithi163 bash[78107]: cluster 2023-10-09T09:14:35.322616+0000 mgr.y (mgr.34107) 229 : cluster 0 pgmap v457: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 205 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 315 KiB/s wr, 1 op/s; 51/411 objects degraded (12.409%) 2023-10-09T09:14:36.488 INFO:tasks.rados.rados.0.smithi163.stdout:2103: finishing write tid 1 to smithi16367185-46 2023-10-09T09:14:36.488 INFO:tasks.rados.rados.0.smithi163.stdout:2103: finishing write tid 2 to smithi16367185-46 2023-10-09T09:14:36.488 INFO:tasks.rados.rados.0.smithi163.stdout:2103: finishing write tid 3 to smithi16367185-46 2023-10-09T09:14:36.488 INFO:tasks.rados.rados.0.smithi163.stdout:2097: done (6 left) 2023-10-09T09:14:36.488 INFO:tasks.rados.rados.0.smithi163.stdout:2098: done (5 left) 2023-10-09T09:14:36.489 INFO:tasks.rados.rados.0.smithi163.stdout:2100: done (4 left) 2023-10-09T09:14:36.489 INFO:tasks.rados.rados.0.smithi163.stdout:2102: done (3 left) 2023-10-09T09:14:36.489 INFO:tasks.rados.rados.0.smithi163.stdout:2104: done (2 left) 2023-10-09T09:14:36.489 INFO:tasks.rados.rados.0.smithi163.stdout:2105: done (1 left) 2023-10-09T09:14:36.489 INFO:tasks.rados.rados.0.smithi163.stdout:2106: write oid 3 current snap is 271 2023-10-09T09:14:36.489 INFO:tasks.rados.rados.0.smithi163.stdout:2103: finishing write tid 4 to smithi16367185-46 2023-10-09T09:14:36.489 INFO:tasks.rados.rados.0.smithi163.stdout:2106: seq_num 789 ranges {460723=555820,1670528=549595,2634153=626570} 2023-10-09T09:14:36.494 INFO:tasks.rados.rados.0.smithi163.stdout:2106: writing smithi16367185-3 from 460723 to 1016543 tid 1 2023-10-09T09:14:36.501 INFO:tasks.rados.rados.0.smithi163.stdout:2106: writing smithi16367185-3 from 1670528 to 2220123 tid 2 2023-10-09T09:14:36.507 INFO:tasks.rados.rados.0.smithi163.stdout:2106: writing smithi16367185-3 from 2634153 to 3260723 tid 3 2023-10-09T09:14:36.507 INFO:tasks.rados.rados.0.smithi163.stdout:2107: snap_remove snap 268 2023-10-09T09:14:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:36 smithi181 bash[59345]: cluster 2023-10-09T09:14:35.143090+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:36 smithi181 bash[59345]: cluster 2023-10-09T09:14:35.143566+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:36 smithi181 bash[59345]: cluster 2023-10-09T09:14:35.147257+0000 mon.a (mon.0) 647 : cluster 0 osdmap e603: 8 total, 7 up, 8 in 2023-10-09T09:14:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:36 smithi181 bash[59345]: cluster 2023-10-09T09:14:35.148983+0000 osd.2 (osd.2) 195 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:36 smithi181 bash[59345]: cluster 2023-10-09T09:14:35.210167+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e603 with expected crc 2023-10-09T09:14:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:36 smithi181 bash[59345]: cluster 2023-10-09T09:14:35.322616+0000 mgr.y (mgr.34107) 229 : cluster 0 pgmap v457: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 205 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 315 KiB/s wr, 1 op/s; 51/411 objects degraded (12.409%) 2023-10-09T09:14:37.456 INFO:tasks.rados.rados.0.smithi163.stdout:2107: done (2 left) 2023-10-09T09:14:37.456 INFO:tasks.rados.rados.0.smithi163.stdout:2108: delete oid 37 current snap is 271 2023-10-09T09:14:37.456 INFO:tasks.rados.rados.0.smithi163.stdout:2103: finishing write tid 5 to smithi16367185-46 2023-10-09T09:14:37.457 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 881 (ObjNum 787 snap 271 seq_num 787) dirty exists 2023-10-09T09:14:37.457 INFO:tasks.rados.rados.0.smithi163.stdout:2103: left oid 46 (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:14:37.457 INFO:tasks.rados.rados.0.smithi163.stdout:2106: finishing write tid 1 to smithi16367185-3 2023-10-09T09:14:37.457 INFO:tasks.rados.rados.0.smithi163.stdout:2106: finishing write tid 2 to smithi16367185-3 2023-10-09T09:14:37.457 INFO:tasks.rados.rados.0.smithi163.stdout:2106: finishing write tid 3 to smithi16367185-3 2023-10-09T09:14:37.457 INFO:tasks.rados.rados.0.smithi163.stdout:2106: finishing write tid 4 to smithi16367185-3 2023-10-09T09:14:37.457 INFO:tasks.rados.rados.0.smithi163.stdout:2106: finishing write tid 5 to smithi16367185-3 2023-10-09T09:14:37.457 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 552 (ObjNum 789 snap 271 seq_num 789) dirty exists 2023-10-09T09:14:37.457 INFO:tasks.rados.rados.0.smithi163.stdout:2106: left oid 3 (ObjNum 789 snap 271 seq_num 789) 2023-10-09T09:14:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[75425]: cluster 2023-10-09T09:14:36.154303+0000 osd.3 (osd.3) 195 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[75425]: cluster 2023-10-09T09:14:36.159997+0000 mon.a (mon.0) 648 : cluster 0 osdmap e604: 8 total, 7 up, 8 in 2023-10-09T09:14:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[75425]: cluster 2023-10-09T09:14:36.160912+0000 osd.2 (osd.2) 196 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[75425]: cluster 2023-10-09T09:14:36.177577+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[75425]: cluster 2023-10-09T09:14:36.185863+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[75425]: audit 2023-10-09T09:14:36.443452+0000 mon.a (mon.0) 649 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[75425]: audit 2023-10-09T09:14:36.452566+0000 mon.a (mon.0) 650 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[75425]: audit 2023-10-09T09:14:36.488863+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.163:0/948633451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[78107]: cluster 2023-10-09T09:14:36.154303+0000 osd.3 (osd.3) 195 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[78107]: cluster 2023-10-09T09:14:36.159997+0000 mon.a (mon.0) 648 : cluster 0 osdmap e604: 8 total, 7 up, 8 in 2023-10-09T09:14:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[78107]: cluster 2023-10-09T09:14:36.160912+0000 osd.2 (osd.2) 196 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[78107]: cluster 2023-10-09T09:14:36.177577+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[78107]: cluster 2023-10-09T09:14:36.185863+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[78107]: audit 2023-10-09T09:14:36.443452+0000 mon.a (mon.0) 649 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[78107]: audit 2023-10-09T09:14:36.452566+0000 mon.a (mon.0) 650 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[78107]: audit 2023-10-09T09:14:36.488863+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.163:0/948633451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:37.470 INFO:tasks.rados.rados.0.smithi163.stdout:2103: done (2 left) 2023-10-09T09:14:37.470 INFO:tasks.rados.rados.0.smithi163.stdout:2106: done (1 left) 2023-10-09T09:14:37.470 INFO:tasks.rados.rados.0.smithi163.stdout:2108: done (0 left) 2023-10-09T09:14:37.470 INFO:tasks.rados.rados.0.smithi163.stdout:2109: read oid 20 snap -1 2023-10-09T09:14:37.470 INFO:tasks.rados.rados.0.smithi163.stdout:2109: expect (ObjNum 777 snap 268 seq_num 777) 2023-10-09T09:14:37.470 INFO:tasks.rados.rados.0.smithi163.stdout:2110: read oid 33 snap -1 2023-10-09T09:14:37.470 INFO:tasks.rados.rados.0.smithi163.stdout:2110: expect (ObjNum 634 snap 202 seq_num 634) 2023-10-09T09:14:37.470 INFO:tasks.rados.rados.0.smithi163.stdout:2111: read oid 42 snap -1 2023-10-09T09:14:37.471 INFO:tasks.rados.rados.0.smithi163.stdout:2111: expect deleted 2023-10-09T09:14:37.471 INFO:tasks.rados.rados.0.smithi163.stdout:2112: snap_remove snap 258 2023-10-09T09:14:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:37 smithi181 bash[59345]: cluster 2023-10-09T09:14:36.154303+0000 osd.3 (osd.3) 195 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:37 smithi181 bash[59345]: cluster 2023-10-09T09:14:36.159997+0000 mon.a (mon.0) 648 : cluster 0 osdmap e604: 8 total, 7 up, 8 in 2023-10-09T09:14:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:37 smithi181 bash[59345]: cluster 2023-10-09T09:14:36.160912+0000 osd.2 (osd.2) 196 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:37 smithi181 bash[59345]: cluster 2023-10-09T09:14:36.177577+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:37 smithi181 bash[59345]: cluster 2023-10-09T09:14:36.185863+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e604 with expected crc 2023-10-09T09:14:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:37 smithi181 bash[59345]: audit 2023-10-09T09:14:36.443452+0000 mon.a (mon.0) 649 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:37 smithi181 bash[59345]: audit 2023-10-09T09:14:36.452566+0000 mon.a (mon.0) 650 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:37.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:37 smithi181 bash[59345]: audit 2023-10-09T09:14:36.488863+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.163:0/948633451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:38.218 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:37 smithi163 bash[86800]: debug 2023-10-09T09:14:37.803+0000 7f6567513640 -1 osd.1 594 log_to_monitors true 2023-10-09T09:14:38.460 INFO:tasks.rados.rados.0.smithi163.stdout:2112: done (3 left) 2023-10-09T09:14:38.460 INFO:tasks.rados.rados.0.smithi163.stdout:2113: delete oid 17 current snap is 271 2023-10-09T09:14:38.460 INFO:tasks.rados.rados.0.smithi163.stdout:2110: expect (ObjNum 634 snap 202 seq_num 634) 2023-10-09T09:14:38.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[78107]: audit 2023-10-09T09:14:37.245478+0000 mon.a (mon.0) 651 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:38.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[78107]: audit 2023-10-09T09:14:37.253207+0000 mon.a (mon.0) 652 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:38.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[78107]: cluster 2023-10-09T09:14:37.323994+0000 mgr.y (mgr.34107) 230 : cluster 0 pgmap v459: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 212 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.4 MiB/s wr, 15 op/s; 54/420 objects degraded (12.857%) 2023-10-09T09:14:38.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[78107]: cluster 2023-10-09T09:14:37.454092+0000 osd.3 (osd.3) 196 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:38.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[78107]: cluster 2023-10-09T09:14:37.456135+0000 mon.a (mon.0) 653 : cluster 0 osdmap e605: 8 total, 7 up, 8 in 2023-10-09T09:14:38.508 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[78107]: cluster 2023-10-09T09:14:37.470917+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:38.508 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[78107]: audit 2023-10-09T09:14:37.811522+0000 mon.c (mon.2) 15 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:14:38.508 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[78107]: audit 2023-10-09T09:14:37.812017+0000 mon.a (mon.0) 654 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:14:38.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[75425]: audit 2023-10-09T09:14:37.245478+0000 mon.a (mon.0) 651 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:38.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[75425]: audit 2023-10-09T09:14:37.253207+0000 mon.a (mon.0) 652 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:38.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[75425]: cluster 2023-10-09T09:14:37.323994+0000 mgr.y (mgr.34107) 230 : cluster 0 pgmap v459: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 212 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.4 MiB/s wr, 15 op/s; 54/420 objects degraded (12.857%) 2023-10-09T09:14:38.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[75425]: cluster 2023-10-09T09:14:37.454092+0000 osd.3 (osd.3) 196 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:38.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[75425]: cluster 2023-10-09T09:14:37.456135+0000 mon.a (mon.0) 653 : cluster 0 osdmap e605: 8 total, 7 up, 8 in 2023-10-09T09:14:38.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[75425]: cluster 2023-10-09T09:14:37.470917+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:38.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[75425]: audit 2023-10-09T09:14:37.811522+0000 mon.c (mon.2) 15 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:14:38.509 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[75425]: audit 2023-10-09T09:14:37.812017+0000 mon.a (mon.0) 654 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:14:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:38 smithi181 bash[59345]: audit 2023-10-09T09:14:37.245478+0000 mon.a (mon.0) 651 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:38 smithi181 bash[59345]: audit 2023-10-09T09:14:37.253207+0000 mon.a (mon.0) 652 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:38 smithi181 bash[59345]: cluster 2023-10-09T09:14:37.323994+0000 mgr.y (mgr.34107) 230 : cluster 0 pgmap v459: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 212 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.4 MiB/s wr, 15 op/s; 54/420 objects degraded (12.857%) 2023-10-09T09:14:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:38 smithi181 bash[59345]: cluster 2023-10-09T09:14:37.454092+0000 osd.3 (osd.3) 196 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:38 smithi181 bash[59345]: cluster 2023-10-09T09:14:37.456135+0000 mon.a (mon.0) 653 : cluster 0 osdmap e605: 8 total, 7 up, 8 in 2023-10-09T09:14:38.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:38 smithi181 bash[59345]: cluster 2023-10-09T09:14:37.470917+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:38.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:38 smithi181 bash[59345]: audit 2023-10-09T09:14:37.811522+0000 mon.c (mon.2) 15 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:14:38.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:38 smithi181 bash[59345]: audit 2023-10-09T09:14:37.812017+0000 mon.a (mon.0) 654 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:14:38.604 INFO:tasks.rados.rados.0.smithi163.stdout:2109: expect (ObjNum 777 snap 268 seq_num 777) 2023-10-09T09:14:38.715 INFO:tasks.rados.rados.0.smithi163.stdout:2109: done (3 left) 2023-10-09T09:14:38.716 INFO:tasks.rados.rados.0.smithi163.stdout:2110: done (2 left) 2023-10-09T09:14:38.716 INFO:tasks.rados.rados.0.smithi163.stdout:2111: done (1 left) 2023-10-09T09:14:38.716 INFO:tasks.rados.rados.0.smithi163.stdout:2113: done (0 left) 2023-10-09T09:14:38.716 INFO:tasks.rados.rados.0.smithi163.stdout:2114: snap_remove snap 254 2023-10-09T09:14:38.968 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:14:38 smithi163 bash[86800]: debug 2023-10-09T09:14:38.503+0000 7f655897f700 -1 osd.1 594 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:14:39.465 INFO:tasks.rados.rados.0.smithi163.stdout:2114: done (0 left) 2023-10-09T09:14:39.465 INFO:tasks.rados.rados.0.smithi163.stdout:2115: snap_create 2023-10-09T09:14:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:39 smithi181 bash[59345]: cluster 2023-10-09T09:14:37.452676+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:39 smithi181 bash[59345]: cluster 2023-10-09T09:14:37.456887+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:39 smithi181 bash[59345]: cluster 2023-10-09T09:14:38.251003+0000 mon.a (mon.0) 655 : cluster 3 Health check update: Degraded data redundancy: 54/420 objects degraded (12.857%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:39 smithi181 bash[59345]: cluster 2023-10-09T09:14:38.456481+0000 osd.3 (osd.3) 197 : cluster 3 failed to encode map e606 with expected crc 2023-10-09T09:14:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:39 smithi181 bash[59345]: cluster 2023-10-09T09:14:38.457865+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e606 with expected crc 2023-10-09T09:14:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:39 smithi181 bash[59345]: audit 2023-10-09T09:14:38.459741+0000 mon.a (mon.0) 656 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:14:39.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:39 smithi181 bash[59345]: cluster 2023-10-09T09:14:38.459870+0000 mon.a (mon.0) 657 : cluster 0 osdmap e606: 8 total, 7 up, 8 in 2023-10-09T09:14:39.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:39 smithi181 bash[59345]: audit 2023-10-09T09:14:38.464549+0000 mon.c (mon.2) 16 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:39.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:39 smithi181 bash[59345]: audit 2023-10-09T09:14:38.465056+0000 mon.a (mon.0) 658 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:39.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:39 smithi181 bash[59345]: audit 2023-10-09T09:14:38.861179+0000 mon.a (mon.0) 659 : audit 0 from='client.? 172.21.15.163:0/3648609475' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[78107]: cluster 2023-10-09T09:14:37.452676+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[78107]: cluster 2023-10-09T09:14:37.456887+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[78107]: cluster 2023-10-09T09:14:38.251003+0000 mon.a (mon.0) 655 : cluster 3 Health check update: Degraded data redundancy: 54/420 objects degraded (12.857%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[78107]: cluster 2023-10-09T09:14:38.456481+0000 osd.3 (osd.3) 197 : cluster 3 failed to encode map e606 with expected crc 2023-10-09T09:14:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[78107]: cluster 2023-10-09T09:14:38.457865+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e606 with expected crc 2023-10-09T09:14:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[78107]: audit 2023-10-09T09:14:38.459741+0000 mon.a (mon.0) 656 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:14:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[78107]: cluster 2023-10-09T09:14:38.459870+0000 mon.a (mon.0) 657 : cluster 0 osdmap e606: 8 total, 7 up, 8 in 2023-10-09T09:14:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[78107]: audit 2023-10-09T09:14:38.464549+0000 mon.c (mon.2) 16 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[78107]: audit 2023-10-09T09:14:38.465056+0000 mon.a (mon.0) 658 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[78107]: audit 2023-10-09T09:14:38.861179+0000 mon.a (mon.0) 659 : audit 0 from='client.? 172.21.15.163:0/3648609475' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[75425]: cluster 2023-10-09T09:14:37.452676+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[75425]: cluster 2023-10-09T09:14:37.456887+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e605 with expected crc 2023-10-09T09:14:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[75425]: cluster 2023-10-09T09:14:38.251003+0000 mon.a (mon.0) 655 : cluster 3 Health check update: Degraded data redundancy: 54/420 objects degraded (12.857%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[75425]: cluster 2023-10-09T09:14:38.456481+0000 osd.3 (osd.3) 197 : cluster 3 failed to encode map e606 with expected crc 2023-10-09T09:14:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[75425]: cluster 2023-10-09T09:14:38.457865+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e606 with expected crc 2023-10-09T09:14:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[75425]: audit 2023-10-09T09:14:38.459741+0000 mon.a (mon.0) 656 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:14:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[75425]: cluster 2023-10-09T09:14:38.459870+0000 mon.a (mon.0) 657 : cluster 0 osdmap e606: 8 total, 7 up, 8 in 2023-10-09T09:14:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[75425]: audit 2023-10-09T09:14:38.464549+0000 mon.c (mon.2) 16 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[75425]: audit 2023-10-09T09:14:38.465056+0000 mon.a (mon.0) 658 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:14:39.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:39 smithi163 bash[75425]: audit 2023-10-09T09:14:38.861179+0000 mon.a (mon.0) 659 : audit 0 from='client.? 172.21.15.163:0/3648609475' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:40.466 INFO:tasks.rados.rados.0.smithi163.stdout:2115: done (0 left) 2023-10-09T09:14:40.466 INFO:tasks.rados.rados.0.smithi163.stdout:2116: write oid 17 current snap is 272 2023-10-09T09:14:40.466 INFO:tasks.rados.rados.0.smithi163.stdout:2116: seq_num 792 ranges {511144=497920,1445974=413275,2316422=656493} 2023-10-09T09:14:40.474 INFO:tasks.rados.rados.0.smithi163.stdout:2116: writing smithi16367185-17 from 511144 to 1009064 tid 1 2023-10-09T09:14:40.480 INFO:tasks.rados.rados.0.smithi163.stdout:2116: writing smithi16367185-17 from 1445974 to 1859249 tid 2 2023-10-09T09:14:40.488 INFO:tasks.rados.rados.0.smithi163.stdout:2116: writing smithi16367185-17 from 2316422 to 2972915 tid 3 2023-10-09T09:14:40.488 INFO:tasks.rados.rados.0.smithi163.stdout:2117: write oid 49 current snap is 272 2023-10-09T09:14:40.489 INFO:tasks.rados.rados.0.smithi163.stdout:2117: seq_num 793 ranges {527951=573169,1504456=556933,2743845=467765} 2023-10-09T09:14:40.496 INFO:tasks.rados.rados.0.smithi163.stdout:2117: writing smithi16367185-49 from 527951 to 1101120 tid 1 2023-10-09T09:14:40.502 INFO:tasks.rados.rados.0.smithi163.stdout:2117: writing smithi16367185-49 from 1504456 to 2061389 tid 2 2023-10-09T09:14:40.510 INFO:tasks.rados.rados.0.smithi163.stdout:2117: writing smithi16367185-49 from 2743845 to 3211610 tid 3 2023-10-09T09:14:40.510 INFO:tasks.rados.rados.0.smithi163.stdout:2118: delete oid 4 current snap is 272 2023-10-09T09:14:40.510 INFO:tasks.rados.rados.0.smithi163.stdout:2116: finishing write tid 1 to smithi16367185-17 2023-10-09T09:14:40.510 INFO:tasks.rados.rados.0.smithi163.stdout:2116: finishing write tid 2 to smithi16367185-17 2023-10-09T09:14:40.510 INFO:tasks.rados.rados.0.smithi163.stdout:2116: finishing write tid 3 to smithi16367185-17 2023-10-09T09:14:40.510 INFO:tasks.rados.rados.0.smithi163.stdout:2116: finishing write tid 4 to smithi16367185-17 2023-10-09T09:14:40.510 INFO:tasks.rados.rados.0.smithi163.stdout:2116: finishing write tid 5 to smithi16367185-17 2023-10-09T09:14:40.510 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1063 (ObjNum 792 snap 272 seq_num 792) dirty exists 2023-10-09T09:14:40.510 INFO:tasks.rados.rados.0.smithi163.stdout:2116: left oid 17 (ObjNum 792 snap 272 seq_num 792) 2023-10-09T09:14:40.510 INFO:tasks.rados.rados.0.smithi163.stdout:2117: finishing write tid 1 to smithi16367185-49 2023-10-09T09:14:40.511 INFO:tasks.rados.rados.0.smithi163.stdout:2117: finishing write tid 2 to smithi16367185-49 2023-10-09T09:14:40.513 INFO:tasks.rados.rados.0.smithi163.stdout:2116: done (2 left) 2023-10-09T09:14:40.513 INFO:tasks.rados.rados.0.smithi163.stdout:2118: done (1 left) 2023-10-09T09:14:40.513 INFO:tasks.rados.rados.0.smithi163.stdout:2119: snap_create 2023-10-09T09:14:40.513 INFO:tasks.rados.rados.0.smithi163.stdout:2117: finishing write tid 3 to smithi16367185-49 2023-10-09T09:14:40.514 INFO:tasks.rados.rados.0.smithi163.stdout:2117: finishing write tid 4 to smithi16367185-49 2023-10-09T09:14:40.514 INFO:tasks.rados.rados.0.smithi163.stdout:2117: finishing write tid 5 to smithi16367185-49 2023-10-09T09:14:40.514 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 313 (ObjNum 793 snap 272 seq_num 793) dirty exists 2023-10-09T09:14:40.514 INFO:tasks.rados.rados.0.smithi163.stdout:2117: left oid 49 (ObjNum 793 snap 272 seq_num 793) 2023-10-09T09:14:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:40 smithi181 bash[59345]: cluster 2023-10-09T09:14:38.463236+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e606 with expected crc 2023-10-09T09:14:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:40 smithi181 bash[59345]: cluster 2023-10-09T09:14:39.324839+0000 mgr.y (mgr.34107) 231 : cluster 0 pgmap v462: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 212 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.4 MiB/s wr, 16 op/s; 54/420 objects degraded (12.857%) 2023-10-09T09:14:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:40 smithi181 bash[59345]: cluster 2023-10-09T09:14:39.455935+0000 mon.a (mon.0) 660 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:14:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:40 smithi181 bash[59345]: cluster 2023-10-09T09:14:39.460722+0000 osd.3 (osd.3) 198 : cluster 3 failed to encode map e607 with expected crc 2023-10-09T09:14:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:40 smithi181 bash[59345]: cluster 2023-10-09T09:14:39.464523+0000 mon.a (mon.0) 661 : cluster 1 osd.1 [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458] boot 2023-10-09T09:14:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:40 smithi181 bash[59345]: cluster 2023-10-09T09:14:39.464637+0000 mon.a (mon.0) 662 : cluster 0 osdmap e607: 8 total, 8 up, 8 in 2023-10-09T09:14:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:40 smithi181 bash[59345]: audit 2023-10-09T09:14:39.465395+0000 mon.a (mon.0) 663 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:14:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[78107]: cluster 2023-10-09T09:14:38.463236+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e606 with expected crc 2023-10-09T09:14:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[78107]: cluster 2023-10-09T09:14:39.324839+0000 mgr.y (mgr.34107) 231 : cluster 0 pgmap v462: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 212 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.4 MiB/s wr, 16 op/s; 54/420 objects degraded (12.857%) 2023-10-09T09:14:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[78107]: cluster 2023-10-09T09:14:39.455935+0000 mon.a (mon.0) 660 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:14:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[78107]: cluster 2023-10-09T09:14:39.460722+0000 osd.3 (osd.3) 198 : cluster 3 failed to encode map e607 with expected crc 2023-10-09T09:14:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[78107]: cluster 2023-10-09T09:14:39.464523+0000 mon.a (mon.0) 661 : cluster 1 osd.1 [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458] boot 2023-10-09T09:14:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[78107]: cluster 2023-10-09T09:14:39.464637+0000 mon.a (mon.0) 662 : cluster 0 osdmap e607: 8 total, 8 up, 8 in 2023-10-09T09:14:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[78107]: audit 2023-10-09T09:14:39.465395+0000 mon.a (mon.0) 663 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:14:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[75425]: cluster 2023-10-09T09:14:38.463236+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e606 with expected crc 2023-10-09T09:14:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[75425]: cluster 2023-10-09T09:14:39.324839+0000 mgr.y (mgr.34107) 231 : cluster 0 pgmap v462: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 212 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.4 MiB/s wr, 16 op/s; 54/420 objects degraded (12.857%) 2023-10-09T09:14:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[75425]: cluster 2023-10-09T09:14:39.455935+0000 mon.a (mon.0) 660 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:14:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[75425]: cluster 2023-10-09T09:14:39.460722+0000 osd.3 (osd.3) 198 : cluster 3 failed to encode map e607 with expected crc 2023-10-09T09:14:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[75425]: cluster 2023-10-09T09:14:39.464523+0000 mon.a (mon.0) 661 : cluster 1 osd.1 [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458] boot 2023-10-09T09:14:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[75425]: cluster 2023-10-09T09:14:39.464637+0000 mon.a (mon.0) 662 : cluster 0 osdmap e607: 8 total, 8 up, 8 in 2023-10-09T09:14:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:40 smithi163 bash[75425]: audit 2023-10-09T09:14:39.465395+0000 mon.a (mon.0) 663 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:14:41.477 INFO:tasks.rados.rados.0.smithi163.stdout:2117: done (1 left) 2023-10-09T09:14:41.477 INFO:tasks.rados.rados.0.smithi163.stdout:2119: done (0 left) 2023-10-09T09:14:41.477 INFO:tasks.rados.rados.0.smithi163.stdout:2120: read oid 4 snap 270 2023-10-09T09:14:41.477 INFO:tasks.rados.rados.0.smithi163.stdout:2120: expect (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:14:41.477 INFO:tasks.rados.rados.0.smithi163.stdout:2121: snap_create 2023-10-09T09:14:41.487 INFO:tasks.rados.rados.0.smithi163.stdout:2120: expect (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:14:41.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[75425]: cluster 2023-10-09T09:14:39.462490+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e607 with expected crc 2023-10-09T09:14:41.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[75425]: audit 2023-10-09T09:14:40.348092+0000 mon.a (mon.0) 664 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[75425]: audit 2023-10-09T09:14:40.350132+0000 mon.a (mon.0) 665 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.465815+0000 mon.a (mon.0) 666 : cluster 0 osdmap e608: 8 total, 8 up, 8 in 2023-10-09T09:14:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.466285+0000 osd.3 (osd.3) 199 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.466345+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.466573+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.470389+0000 osd.3 (osd.3) 200 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.473159+0000 osd.3 (osd.3) 201 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[75425]: audit 2023-10-09T09:14:41.227268+0000 mon.a (mon.0) 667 : audit 0 from='client.? 172.21.15.163:0/3956146976' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[78107]: cluster 2023-10-09T09:14:39.462490+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e607 with expected crc 2023-10-09T09:14:41.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[78107]: audit 2023-10-09T09:14:40.348092+0000 mon.a (mon.0) 664 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:41.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[78107]: audit 2023-10-09T09:14:40.350132+0000 mon.a (mon.0) 665 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:41.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.465815+0000 mon.a (mon.0) 666 : cluster 0 osdmap e608: 8 total, 8 up, 8 in 2023-10-09T09:14:41.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.466285+0000 osd.3 (osd.3) 199 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.466345+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.466573+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.470389+0000 osd.3 (osd.3) 200 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.473159+0000 osd.3 (osd.3) 201 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:41 smithi163 bash[78107]: audit 2023-10-09T09:14:41.227268+0000 mon.a (mon.0) 667 : audit 0 from='client.? 172.21.15.163:0/3956146976' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:41 smithi181 bash[59345]: cluster 2023-10-09T09:14:39.462490+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e607 with expected crc 2023-10-09T09:14:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:41 smithi181 bash[59345]: audit 2023-10-09T09:14:40.348092+0000 mon.a (mon.0) 664 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:41 smithi181 bash[59345]: audit 2023-10-09T09:14:40.350132+0000 mon.a (mon.0) 665 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:41 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.465815+0000 mon.a (mon.0) 666 : cluster 0 osdmap e608: 8 total, 8 up, 8 in 2023-10-09T09:14:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:41 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.466285+0000 osd.3 (osd.3) 199 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:41 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.466345+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:41 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.466573+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:41 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.470389+0000 osd.3 (osd.3) 200 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:41 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.473159+0000 osd.3 (osd.3) 201 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:41.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:41 smithi181 bash[59345]: audit 2023-10-09T09:14:41.227268+0000 mon.a (mon.0) 667 : audit 0 from='client.? 172.21.15.163:0/3956146976' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.460824+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.461505+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.464637+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.466378+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.467729+0000 osd.2 (osd.2) 197 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[75425]: cluster 2023-10-09T09:14:40.468302+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[75425]: cluster 2023-10-09T09:14:41.326002+0000 mgr.y (mgr.34107) 232 : cluster 0 pgmap v465: 89 pgs: 3 active+undersized+degraded, 11 active+undersized, 75 active+clean; 214 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 383 KiB/s wr, 6 op/s; 14/414 objects degraded (3.382%); 1.2 MiB/s, 1 objects/s recovering 2023-10-09T09:14:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[75425]: cluster 2023-10-09T09:14:41.472343+0000 osd.3 (osd.3) 202 : cluster 3 failed to encode map e609 with expected crc 2023-10-09T09:14:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[75425]: cluster 2023-10-09T09:14:41.476410+0000 mon.a (mon.0) 668 : cluster 0 osdmap e609: 8 total, 8 up, 8 in 2023-10-09T09:14:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.460824+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.461505+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.464637+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.466378+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.467729+0000 osd.2 (osd.2) 197 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[78107]: cluster 2023-10-09T09:14:40.468302+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[78107]: cluster 2023-10-09T09:14:41.326002+0000 mgr.y (mgr.34107) 232 : cluster 0 pgmap v465: 89 pgs: 3 active+undersized+degraded, 11 active+undersized, 75 active+clean; 214 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 383 KiB/s wr, 6 op/s; 14/414 objects degraded (3.382%); 1.2 MiB/s, 1 objects/s recovering 2023-10-09T09:14:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[78107]: cluster 2023-10-09T09:14:41.472343+0000 osd.3 (osd.3) 202 : cluster 3 failed to encode map e609 with expected crc 2023-10-09T09:14:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:42 smithi163 bash[78107]: cluster 2023-10-09T09:14:41.476410+0000 mon.a (mon.0) 668 : cluster 0 osdmap e609: 8 total, 8 up, 8 in 2023-10-09T09:14:42.480 INFO:tasks.rados.rados.0.smithi163.stdout:2120: done (1 left) 2023-10-09T09:14:42.480 INFO:tasks.rados.rados.0.smithi163.stdout:2121: done (0 left) 2023-10-09T09:14:42.480 INFO:tasks.rados.rados.0.smithi163.stdout:2122: snap_remove snap 235 2023-10-09T09:14:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:42 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.460824+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:42 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.461505+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:42 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.464637+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:42 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.466378+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:42 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.467729+0000 osd.2 (osd.2) 197 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:42 smithi181 bash[59345]: cluster 2023-10-09T09:14:40.468302+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e608 with expected crc 2023-10-09T09:14:42.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:42 smithi181 bash[59345]: cluster 2023-10-09T09:14:41.326002+0000 mgr.y (mgr.34107) 232 : cluster 0 pgmap v465: 89 pgs: 3 active+undersized+degraded, 11 active+undersized, 75 active+clean; 214 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 383 KiB/s wr, 6 op/s; 14/414 objects degraded (3.382%); 1.2 MiB/s, 1 objects/s recovering 2023-10-09T09:14:42.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:42 smithi181 bash[59345]: cluster 2023-10-09T09:14:41.472343+0000 osd.3 (osd.3) 202 : cluster 3 failed to encode map e609 with expected crc 2023-10-09T09:14:42.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:42 smithi181 bash[59345]: cluster 2023-10-09T09:14:41.476410+0000 mon.a (mon.0) 668 : cluster 0 osdmap e609: 8 total, 8 up, 8 in 2023-10-09T09:14:43.491 INFO:tasks.rados.rados.0.smithi163.stdout:2122: done (0 left) 2023-10-09T09:14:43.491 INFO:tasks.rados.rados.0.smithi163.stdout:2123: snap_remove snap 270 2023-10-09T09:14:43.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:43 smithi163 bash[75425]: cluster 2023-10-09T09:14:41.479050+0000 osd.2 (osd.2) 198 : cluster 3 failed to encode map e609 with expected crc 2023-10-09T09:14:43.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:43 smithi163 bash[75425]: cluster 2023-10-09T09:14:42.479967+0000 mon.a (mon.0) 669 : cluster 0 osdmap e610: 8 total, 8 up, 8 in 2023-10-09T09:14:43.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:43 smithi163 bash[75425]: cluster 2023-10-09T09:14:42.680339+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:43.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:14:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:14:43] "GET /metrics HTTP/1.1" 200 36056 "" "Prometheus/2.33.4" 2023-10-09T09:14:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:43 smithi163 bash[78107]: cluster 2023-10-09T09:14:41.479050+0000 osd.2 (osd.2) 198 : cluster 3 failed to encode map e609 with expected crc 2023-10-09T09:14:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:43 smithi163 bash[78107]: cluster 2023-10-09T09:14:42.479967+0000 mon.a (mon.0) 669 : cluster 0 osdmap e610: 8 total, 8 up, 8 in 2023-10-09T09:14:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:43 smithi163 bash[78107]: cluster 2023-10-09T09:14:42.680339+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:43 smithi181 bash[59345]: cluster 2023-10-09T09:14:41.479050+0000 osd.2 (osd.2) 198 : cluster 3 failed to encode map e609 with expected crc 2023-10-09T09:14:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:43 smithi181 bash[59345]: cluster 2023-10-09T09:14:42.479967+0000 mon.a (mon.0) 669 : cluster 0 osdmap e610: 8 total, 8 up, 8 in 2023-10-09T09:14:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:43 smithi181 bash[59345]: cluster 2023-10-09T09:14:42.680339+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[78107]: cluster 2023-10-09T09:14:42.475565+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[78107]: cluster 2023-10-09T09:14:43.326656+0000 mgr.y (mgr.34107) 233 : cluster 0 pgmap v468: 89 pgs: 1 active+undersized+degraded, 10 active+undersized, 78 active+clean; 217 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 773 KiB/s wr, 7 op/s; 6/414 objects degraded (1.449%); 2.1 MiB/s, 1 objects/s recovering 2023-10-09T09:14:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[78107]: cluster 2023-10-09T09:14:43.475797+0000 mon.a (mon.0) 670 : cluster 3 Health check update: Degraded data redundancy: 6/414 objects degraded (1.449%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:14:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[78107]: cluster 2023-10-09T09:14:43.488499+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e611 with expected crc 2023-10-09T09:14:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[78107]: cluster 2023-10-09T09:14:43.490532+0000 mon.a (mon.0) 671 : cluster 0 osdmap e611: 8 total, 8 up, 8 in 2023-10-09T09:14:44.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[78107]: cluster 2023-10-09T09:14:43.494384+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[78107]: audit 2023-10-09T09:14:43.628526+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.163:0/3009900131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:44.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[75425]: cluster 2023-10-09T09:14:42.475565+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:44.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[75425]: cluster 2023-10-09T09:14:43.326656+0000 mgr.y (mgr.34107) 233 : cluster 0 pgmap v468: 89 pgs: 1 active+undersized+degraded, 10 active+undersized, 78 active+clean; 217 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 773 KiB/s wr, 7 op/s; 6/414 objects degraded (1.449%); 2.1 MiB/s, 1 objects/s recovering 2023-10-09T09:14:44.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[75425]: cluster 2023-10-09T09:14:43.475797+0000 mon.a (mon.0) 670 : cluster 3 Health check update: Degraded data redundancy: 6/414 objects degraded (1.449%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:14:44.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[75425]: cluster 2023-10-09T09:14:43.488499+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e611 with expected crc 2023-10-09T09:14:44.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[75425]: cluster 2023-10-09T09:14:43.490532+0000 mon.a (mon.0) 671 : cluster 0 osdmap e611: 8 total, 8 up, 8 in 2023-10-09T09:14:44.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[75425]: cluster 2023-10-09T09:14:43.494384+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:44.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:44 smithi163 bash[75425]: audit 2023-10-09T09:14:43.628526+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.163:0/3009900131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:44.494 INFO:tasks.rados.rados.0.smithi163.stdout:2123: done (0 left) 2023-10-09T09:14:44.494 INFO:tasks.rados.rados.0.smithi163.stdout:2124: delete oid 34 current snap is 274 2023-10-09T09:14:44.503 INFO:tasks.rados.rados.0.smithi163.stdout:2124: done (0 left) 2023-10-09T09:14:44.503 INFO:tasks.rados.rados.0.smithi163.stdout:2125: write oid 31 current snap is 274 2023-10-09T09:14:44.503 INFO:tasks.rados.rados.0.smithi163.stdout:2125: seq_num 796 ranges {578372=798916,1963549=420613,2826114=417934} 2023-10-09T09:14:44.511 INFO:tasks.rados.rados.0.smithi163.stdout:2125: writing smithi16367185-31 from 578372 to 1377288 tid 1 2023-10-09T09:14:44.517 INFO:tasks.rados.rados.0.smithi163.stdout:2125: writing smithi16367185-31 from 1963549 to 2384162 tid 2 2023-10-09T09:14:44.522 INFO:tasks.rados.rados.0.smithi163.stdout:2125: writing smithi16367185-31 from 2826114 to 3244048 tid 3 2023-10-09T09:14:44.522 INFO:tasks.rados.rados.0.smithi163.stdout:2126: read oid 23 snap 256 2023-10-09T09:14:44.523 INFO:tasks.rados.rados.0.smithi163.stdout:2126: expect (ObjNum 740 snap 250 seq_num 740) 2023-10-09T09:14:44.523 INFO:tasks.rados.rados.0.smithi163.stdout:2127: snap_create 2023-10-09T09:14:44.523 INFO:tasks.rados.rados.0.smithi163.stdout:2125: finishing write tid 1 to smithi16367185-31 2023-10-09T09:14:44.523 INFO:tasks.rados.rados.0.smithi163.stdout:2125: finishing write tid 2 to smithi16367185-31 2023-10-09T09:14:44.526 INFO:tasks.rados.rados.0.smithi163.stdout:2125: finishing write tid 3 to smithi16367185-31 2023-10-09T09:14:44.526 INFO:tasks.rados.rados.0.smithi163.stdout:2125: finishing write tid 4 to smithi16367185-31 2023-10-09T09:14:44.527 INFO:tasks.rados.rados.0.smithi163.stdout:2125: finishing write tid 5 to smithi16367185-31 2023-10-09T09:14:44.527 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 566 (ObjNum 796 snap 274 seq_num 796) dirty exists 2023-10-09T09:14:44.527 INFO:tasks.rados.rados.0.smithi163.stdout:2125: left oid 31 (ObjNum 796 snap 274 seq_num 796) 2023-10-09T09:14:44.531 INFO:tasks.rados.rados.0.smithi163.stdout:2126: expect (ObjNum 740 snap 250 seq_num 740) 2023-10-09T09:14:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:44 smithi181 bash[59345]: cluster 2023-10-09T09:14:42.475565+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:44 smithi181 bash[59345]: cluster 2023-10-09T09:14:43.326656+0000 mgr.y (mgr.34107) 233 : cluster 0 pgmap v468: 89 pgs: 1 active+undersized+degraded, 10 active+undersized, 78 active+clean; 217 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 773 KiB/s wr, 7 op/s; 6/414 objects degraded (1.449%); 2.1 MiB/s, 1 objects/s recovering 2023-10-09T09:14:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:44 smithi181 bash[59345]: cluster 2023-10-09T09:14:43.475797+0000 mon.a (mon.0) 670 : cluster 3 Health check update: Degraded data redundancy: 6/414 objects degraded (1.449%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:14:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:44 smithi181 bash[59345]: cluster 2023-10-09T09:14:43.488499+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e611 with expected crc 2023-10-09T09:14:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:44 smithi181 bash[59345]: cluster 2023-10-09T09:14:43.490532+0000 mon.a (mon.0) 671 : cluster 0 osdmap e611: 8 total, 8 up, 8 in 2023-10-09T09:14:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:44 smithi181 bash[59345]: cluster 2023-10-09T09:14:43.494384+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:44 smithi181 bash[59345]: audit 2023-10-09T09:14:43.628526+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.163:0/3009900131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:45.498 INFO:tasks.rados.rados.0.smithi163.stdout:2125: done (2 left) 2023-10-09T09:14:45.499 INFO:tasks.rados.rados.0.smithi163.stdout:2126: done (1 left) 2023-10-09T09:14:45.499 INFO:tasks.rados.rados.0.smithi163.stdout:2127: done (0 left) 2023-10-09T09:14:45.499 INFO:tasks.rados.rados.0.smithi163.stdout:2128: write oid 9 current snap is 275 2023-10-09T09:14:45.499 INFO:tasks.rados.rados.0.smithi163.stdout:2128: seq_num 797 ranges {595179=474165,1622031=564271,2737184=629206} 2023-10-09T09:14:45.508 INFO:tasks.rados.rados.0.smithi163.stdout:2128: writing smithi16367185-9 from 595179 to 1069344 tid 1 2023-10-09T09:14:45.516 INFO:tasks.rados.rados.0.smithi163.stdout:2128: writing smithi16367185-9 from 1622031 to 2186302 tid 2 2023-10-09T09:14:45.525 INFO:tasks.rados.rados.0.smithi163.stdout:2128: writing smithi16367185-9 from 2737184 to 3366390 tid 3 2023-10-09T09:14:45.525 INFO:tasks.rados.rados.0.smithi163.stdout:2129: read oid 10 snap 266 2023-10-09T09:14:45.525 INFO:tasks.rados.rados.0.smithi163.stdout:2129: expect deleted 2023-10-09T09:14:45.525 INFO:tasks.rados.rados.0.smithi163.stdout:2130: snap_create 2023-10-09T09:14:45.525 INFO:tasks.rados.rados.0.smithi163.stdout:2128: finishing write tid 1 to smithi16367185-9 2023-10-09T09:14:45.525 INFO:tasks.rados.rados.0.smithi163.stdout:2128: finishing write tid 2 to smithi16367185-9 2023-10-09T09:14:45.529 INFO:tasks.rados.rados.0.smithi163.stdout:2128: finishing write tid 3 to smithi16367185-9 2023-10-09T09:14:45.529 INFO:tasks.rados.rados.0.smithi163.stdout:2128: finishing write tid 4 to smithi16367185-9 2023-10-09T09:14:45.530 INFO:tasks.rados.rados.0.smithi163.stdout:2128: finishing write tid 5 to smithi16367185-9 2023-10-09T09:14:45.530 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 539 (ObjNum 797 snap 275 seq_num 797) dirty exists 2023-10-09T09:14:45.530 INFO:tasks.rados.rados.0.smithi163.stdout:2128: left oid 9 (ObjNum 797 snap 275 seq_num 797) 2023-10-09T09:14:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[78107]: cluster 2023-10-09T09:14:43.485275+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e611 with expected crc 2023-10-09T09:14:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[78107]: cluster 2023-10-09T09:14:43.527054+0000 osd.3 (osd.3) 203 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[78107]: cluster 2023-10-09T09:14:43.528306+0000 osd.3 (osd.3) 204 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[78107]: cluster 2023-10-09T09:14:43.615016+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[78107]: cluster 2023-10-09T09:14:43.616011+0000 osd.2 (osd.2) 199 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[78107]: cluster 2023-10-09T09:14:44.493913+0000 mon.a (mon.0) 672 : cluster 0 osdmap e612: 8 total, 8 up, 8 in 2023-10-09T09:14:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[75425]: cluster 2023-10-09T09:14:43.485275+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e611 with expected crc 2023-10-09T09:14:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[75425]: cluster 2023-10-09T09:14:43.527054+0000 osd.3 (osd.3) 203 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[75425]: cluster 2023-10-09T09:14:43.528306+0000 osd.3 (osd.3) 204 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[75425]: cluster 2023-10-09T09:14:43.615016+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[75425]: cluster 2023-10-09T09:14:43.616011+0000 osd.2 (osd.2) 199 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:45 smithi163 bash[75425]: cluster 2023-10-09T09:14:44.493913+0000 mon.a (mon.0) 672 : cluster 0 osdmap e612: 8 total, 8 up, 8 in 2023-10-09T09:14:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:45 smithi181 bash[59345]: cluster 2023-10-09T09:14:43.485275+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e611 with expected crc 2023-10-09T09:14:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:45 smithi181 bash[59345]: cluster 2023-10-09T09:14:43.527054+0000 osd.3 (osd.3) 203 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:45 smithi181 bash[59345]: cluster 2023-10-09T09:14:43.528306+0000 osd.3 (osd.3) 204 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:45 smithi181 bash[59345]: cluster 2023-10-09T09:14:43.615016+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:45 smithi181 bash[59345]: cluster 2023-10-09T09:14:43.616011+0000 osd.2 (osd.2) 199 : cluster 3 failed to encode map e610 with expected crc 2023-10-09T09:14:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:45 smithi181 bash[59345]: cluster 2023-10-09T09:14:44.493913+0000 mon.a (mon.0) 672 : cluster 0 osdmap e612: 8 total, 8 up, 8 in 2023-10-09T09:14:46.512 INFO:tasks.rados.rados.0.smithi163.stdout:2128: done (2 left) 2023-10-09T09:14:46.512 INFO:tasks.rados.rados.0.smithi163.stdout:2129: done (1 left) 2023-10-09T09:14:46.512 INFO:tasks.rados.rados.0.smithi163.stdout:2130: done (0 left) 2023-10-09T09:14:46.513 INFO:tasks.rados.rados.0.smithi163.stdout:2131: read oid 3 snap -1 2023-10-09T09:14:46.513 INFO:tasks.rados.rados.0.smithi163.stdout:2131: expect (ObjNum 789 snap 271 seq_num 789) 2023-10-09T09:14:46.513 INFO:tasks.rados.rados.0.smithi163.stdout:2132: write oid 39 current snap is 276 2023-10-09T09:14:46.513 INFO:tasks.rados.rados.0.smithi163.stdout:2132: seq_num 798 ranges {611986=549414,1564160=424282,2764607=440478} 2023-10-09T09:14:46.521 INFO:tasks.rados.rados.0.smithi163.stdout:2132: writing smithi16367185-39 from 611986 to 1161400 tid 1 2023-10-09T09:14:46.527 INFO:tasks.rados.rados.0.smithi163.stdout:2132: writing smithi16367185-39 from 1564160 to 1988442 tid 2 2023-10-09T09:14:46.534 INFO:tasks.rados.rados.0.smithi163.stdout:2132: writing smithi16367185-39 from 2764607 to 3205085 tid 3 2023-10-09T09:14:46.535 INFO:tasks.rados.rados.0.smithi163.stdout:2133: read oid 5 snap 272 2023-10-09T09:14:46.535 INFO:tasks.rados.rados.0.smithi163.stdout:2133: expect (ObjNum 710 snap 237 seq_num 710) 2023-10-09T09:14:46.535 INFO:tasks.rados.rados.0.smithi163.stdout:2134: snap_create 2023-10-09T09:14:46.535 INFO:tasks.rados.rados.0.smithi163.stdout:2131: expect (ObjNum 789 snap 271 seq_num 789) 2023-10-09T09:14:46.712 INFO:tasks.rados.rados.0.smithi163.stdout:2132: finishing write tid 1 to smithi16367185-39 2023-10-09T09:14:46.712 INFO:tasks.rados.rados.0.smithi163.stdout:2133: expect (ObjNum 710 snap 237 seq_num 710) 2023-10-09T09:14:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[78107]: cluster 2023-10-09T09:14:44.488293+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e612 with expected crc 2023-10-09T09:14:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[78107]: cluster 2023-10-09T09:14:44.490260+0000 osd.2 (osd.2) 200 : cluster 3 failed to encode map e612 with expected crc 2023-10-09T09:14:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[78107]: cluster 2023-10-09T09:14:45.154955+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e612 with expected crc 2023-10-09T09:14:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[78107]: cluster 2023-10-09T09:14:45.327360+0000 mgr.y (mgr.34107) 234 : cluster 0 pgmap v471: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 213 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 390 KiB/s wr, 1 op/s; 996 KiB/s, 0 objects/s recovering 2023-10-09T09:14:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[78107]: cluster 2023-10-09T09:14:45.494335+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[78107]: cluster 2023-10-09T09:14:45.497614+0000 mon.a (mon.0) 673 : cluster 0 osdmap e613: 8 total, 8 up, 8 in 2023-10-09T09:14:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[78107]: cluster 2023-10-09T09:14:45.526806+0000 osd.2 (osd.2) 201 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[78107]: audit 2023-10-09T09:14:45.988039+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.163:0/2474368117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[78107]: cluster 2023-10-09T09:14:46.096469+0000 mon.a (mon.0) 674 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/414 objects degraded (1.449%), 1 pg degraded) 2023-10-09T09:14:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[78107]: cluster 2023-10-09T09:14:46.096521+0000 mon.a (mon.0) 675 : cluster 1 Cluster is now healthy 2023-10-09T09:14:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[75425]: cluster 2023-10-09T09:14:44.488293+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e612 with expected crc 2023-10-09T09:14:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[75425]: cluster 2023-10-09T09:14:44.490260+0000 osd.2 (osd.2) 200 : cluster 3 failed to encode map e612 with expected crc 2023-10-09T09:14:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[75425]: cluster 2023-10-09T09:14:45.154955+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e612 with expected crc 2023-10-09T09:14:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[75425]: cluster 2023-10-09T09:14:45.327360+0000 mgr.y (mgr.34107) 234 : cluster 0 pgmap v471: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 213 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 390 KiB/s wr, 1 op/s; 996 KiB/s, 0 objects/s recovering 2023-10-09T09:14:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[75425]: cluster 2023-10-09T09:14:45.494335+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[75425]: cluster 2023-10-09T09:14:45.497614+0000 mon.a (mon.0) 673 : cluster 0 osdmap e613: 8 total, 8 up, 8 in 2023-10-09T09:14:46.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[75425]: cluster 2023-10-09T09:14:45.526806+0000 osd.2 (osd.2) 201 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:46.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[75425]: audit 2023-10-09T09:14:45.988039+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.163:0/2474368117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:46.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[75425]: cluster 2023-10-09T09:14:46.096469+0000 mon.a (mon.0) 674 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/414 objects degraded (1.449%), 1 pg degraded) 2023-10-09T09:14:46.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:46 smithi163 bash[75425]: cluster 2023-10-09T09:14:46.096521+0000 mon.a (mon.0) 675 : cluster 1 Cluster is now healthy 2023-10-09T09:14:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:46 smithi181 bash[59345]: cluster 2023-10-09T09:14:44.488293+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e612 with expected crc 2023-10-09T09:14:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:46 smithi181 bash[59345]: cluster 2023-10-09T09:14:44.490260+0000 osd.2 (osd.2) 200 : cluster 3 failed to encode map e612 with expected crc 2023-10-09T09:14:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:46 smithi181 bash[59345]: cluster 2023-10-09T09:14:45.154955+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e612 with expected crc 2023-10-09T09:14:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:46 smithi181 bash[59345]: cluster 2023-10-09T09:14:45.327360+0000 mgr.y (mgr.34107) 234 : cluster 0 pgmap v471: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 213 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 390 KiB/s wr, 1 op/s; 996 KiB/s, 0 objects/s recovering 2023-10-09T09:14:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:46 smithi181 bash[59345]: cluster 2023-10-09T09:14:45.494335+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:46 smithi181 bash[59345]: cluster 2023-10-09T09:14:45.497614+0000 mon.a (mon.0) 673 : cluster 0 osdmap e613: 8 total, 8 up, 8 in 2023-10-09T09:14:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:46 smithi181 bash[59345]: cluster 2023-10-09T09:14:45.526806+0000 osd.2 (osd.2) 201 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:46 smithi181 bash[59345]: audit 2023-10-09T09:14:45.988039+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.163:0/2474368117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:46.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:46 smithi181 bash[59345]: cluster 2023-10-09T09:14:46.096469+0000 mon.a (mon.0) 674 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/414 objects degraded (1.449%), 1 pg degraded) 2023-10-09T09:14:46.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:46 smithi181 bash[59345]: cluster 2023-10-09T09:14:46.096521+0000 mon.a (mon.0) 675 : cluster 1 Cluster is now healthy 2023-10-09T09:14:46.847 INFO:tasks.rados.rados.0.smithi163.stdout:2132: finishing write tid 2 to smithi16367185-39 2023-10-09T09:14:46.848 INFO:tasks.rados.rados.0.smithi163.stdout:2132: finishing write tid 3 to smithi16367185-39 2023-10-09T09:14:46.848 INFO:tasks.rados.rados.0.smithi163.stdout:2132: finishing write tid 4 to smithi16367185-39 2023-10-09T09:14:46.848 INFO:tasks.rados.rados.0.smithi163.stdout:2132: finishing write tid 5 to smithi16367185-39 2023-10-09T09:14:46.848 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 901 (ObjNum 798 snap 276 seq_num 798) dirty exists 2023-10-09T09:14:46.848 INFO:tasks.rados.rados.0.smithi163.stdout:2132: left oid 39 (ObjNum 798 snap 276 seq_num 798) 2023-10-09T09:14:47.539 INFO:tasks.rados.rados.0.smithi163.stdout:2131: done (3 left) 2023-10-09T09:14:47.539 INFO:tasks.rados.rados.0.smithi163.stdout:2132: done (2 left) 2023-10-09T09:14:47.539 INFO:tasks.rados.rados.0.smithi163.stdout:2133: done (1 left) 2023-10-09T09:14:47.539 INFO:tasks.rados.rados.0.smithi163.stdout:2134: done (0 left) 2023-10-09T09:14:47.539 INFO:tasks.rados.rados.0.smithi163.stdout:2135: delete oid 11 current snap is 277 2023-10-09T09:14:47.547 INFO:tasks.rados.rados.0.smithi163.stdout:2135: done (0 left) 2023-10-09T09:14:47.547 INFO:tasks.rados.rados.0.smithi163.stdout:2136: snap_create 2023-10-09T09:14:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: cluster 2023-10-09T09:14:45.492564+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: cluster 2023-10-09T09:14:45.510853+0000 osd.3 (osd.3) 205 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: audit 2023-10-09T09:14:46.502634+0000 mon.a (mon.0) 676 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: cluster 2023-10-09T09:14:46.512149+0000 mon.a (mon.0) 677 : cluster 0 osdmap e614: 8 total, 8 up, 8 in 2023-10-09T09:14:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: cluster 2023-10-09T09:14:46.512405+0000 osd.2 (osd.2) 202 : cluster 3 failed to encode map e614 with expected crc 2023-10-09T09:14:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: audit 2023-10-09T09:14:46.520911+0000 mon.a (mon.0) 678 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: audit 2023-10-09T09:14:46.522276+0000 mon.a (mon.0) 679 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: audit 2023-10-09T09:14:46.523707+0000 mon.a (mon.0) 680 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:14:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: audit 2023-10-09T09:14:46.530745+0000 mon.a (mon.0) 681 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: audit 2023-10-09T09:14:46.612236+0000 mon.a (mon.0) 682 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: audit 2023-10-09T09:14:46.617503+0000 mon.a (mon.0) 683 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:47.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: audit 2023-10-09T09:14:46.620640+0000 mon.a (mon.0) 684 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:47.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: audit 2023-10-09T09:14:46.621883+0000 mon.a (mon.0) 685 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:47.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[75425]: audit 2023-10-09T09:14:46.623511+0000 mon.a (mon.0) 686 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:14:47.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: cluster 2023-10-09T09:14:45.492564+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:47.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: cluster 2023-10-09T09:14:45.510853+0000 osd.3 (osd.3) 205 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:47.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: audit 2023-10-09T09:14:46.502634+0000 mon.a (mon.0) 676 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:47.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: cluster 2023-10-09T09:14:46.512149+0000 mon.a (mon.0) 677 : cluster 0 osdmap e614: 8 total, 8 up, 8 in 2023-10-09T09:14:47.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: cluster 2023-10-09T09:14:46.512405+0000 osd.2 (osd.2) 202 : cluster 3 failed to encode map e614 with expected crc 2023-10-09T09:14:47.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: audit 2023-10-09T09:14:46.520911+0000 mon.a (mon.0) 678 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:47.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: audit 2023-10-09T09:14:46.522276+0000 mon.a (mon.0) 679 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:47.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: audit 2023-10-09T09:14:46.523707+0000 mon.a (mon.0) 680 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:14:47.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: audit 2023-10-09T09:14:46.530745+0000 mon.a (mon.0) 681 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:47.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: audit 2023-10-09T09:14:46.612236+0000 mon.a (mon.0) 682 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:47.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: audit 2023-10-09T09:14:46.617503+0000 mon.a (mon.0) 683 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:47.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: audit 2023-10-09T09:14:46.620640+0000 mon.a (mon.0) 684 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:47.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: audit 2023-10-09T09:14:46.621883+0000 mon.a (mon.0) 685 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:47.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:47 smithi163 bash[78107]: audit 2023-10-09T09:14:46.623511+0000 mon.a (mon.0) 686 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:14:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: cluster 2023-10-09T09:14:45.492564+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: cluster 2023-10-09T09:14:45.510853+0000 osd.3 (osd.3) 205 : cluster 3 failed to encode map e613 with expected crc 2023-10-09T09:14:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: audit 2023-10-09T09:14:46.502634+0000 mon.a (mon.0) 676 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: cluster 2023-10-09T09:14:46.512149+0000 mon.a (mon.0) 677 : cluster 0 osdmap e614: 8 total, 8 up, 8 in 2023-10-09T09:14:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: cluster 2023-10-09T09:14:46.512405+0000 osd.2 (osd.2) 202 : cluster 3 failed to encode map e614 with expected crc 2023-10-09T09:14:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: audit 2023-10-09T09:14:46.520911+0000 mon.a (mon.0) 678 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: audit 2023-10-09T09:14:46.522276+0000 mon.a (mon.0) 679 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: audit 2023-10-09T09:14:46.523707+0000 mon.a (mon.0) 680 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:14:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: audit 2023-10-09T09:14:46.530745+0000 mon.a (mon.0) 681 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: audit 2023-10-09T09:14:46.612236+0000 mon.a (mon.0) 682 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: audit 2023-10-09T09:14:46.617503+0000 mon.a (mon.0) 683 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: audit 2023-10-09T09:14:46.620640+0000 mon.a (mon.0) 684 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:47.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: audit 2023-10-09T09:14:46.621883+0000 mon.a (mon.0) 685 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:14:47.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:47 smithi181 bash[59345]: audit 2023-10-09T09:14:46.623511+0000 mon.a (mon.0) 686 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:14:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:2136: done (0 left) 2023-10-09T09:14:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:2137: read oid 43 snap -1 2023-10-09T09:14:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:2137: expect (ObjNum 733 snap 246 seq_num 733) 2023-10-09T09:14:48.546 INFO:tasks.rados.rados.0.smithi163.stdout:2138: read oid 1 snap -1 2023-10-09T09:14:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:2138: expect deleted 2023-10-09T09:14:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:2139: write oid 42 current snap is 278 2023-10-09T09:14:48.547 INFO:tasks.rados.rados.0.smithi163.stdout:2139: seq_num 800 ranges {645600=416265,1797477=711598,3219453=226147} 2023-10-09T09:14:48.554 INFO:tasks.rados.rados.0.smithi163.stdout:2139: writing smithi16367185-42 from 645600 to 1061865 tid 1 2023-10-09T09:14:48.565 INFO:tasks.rados.rados.0.smithi163.stdout:2139: writing smithi16367185-42 from 1797477 to 2509075 tid 2 2023-10-09T09:14:48.571 INFO:tasks.rados.rados.0.smithi163.stdout:2139: writing smithi16367185-42 from 3219453 to 3445600 tid 3 2023-10-09T09:14:48.571 INFO:tasks.rados.rados.0.smithi163.stdout:2140: delete oid 34 current snap is 278 2023-10-09T09:14:48.571 INFO:tasks.rados.rados.0.smithi163.stdout:2137: expect (ObjNum 733 snap 246 seq_num 733) 2023-10-09T09:14:48.698 INFO:tasks.rados.rados.0.smithi163.stdout:2139: finishing write tid 1 to smithi16367185-42 2023-10-09T09:14:48.698 INFO:tasks.rados.rados.0.smithi163.stdout:2139: finishing write tid 2 to smithi16367185-42 2023-10-09T09:14:48.698 INFO:tasks.rados.rados.0.smithi163.stdout:2137: done (3 left) 2023-10-09T09:14:48.698 INFO:tasks.rados.rados.0.smithi163.stdout:2138: done (2 left) 2023-10-09T09:14:48.699 INFO:tasks.rados.rados.0.smithi163.stdout:2140: done (1 left) 2023-10-09T09:14:48.699 INFO:tasks.rados.rados.0.smithi163.stdout:2141: rollback oid 8 current snap is 278 2023-10-09T09:14:48.699 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 8 to 239 2023-10-09T09:14:48.699 INFO:tasks.rados.rados.0.smithi163.stdout:2139: finishing write tid 3 to smithi16367185-42 2023-10-09T09:14:48.699 INFO:tasks.rados.rados.0.smithi163.stdout:2139: finishing write tid 4 to smithi16367185-42 2023-10-09T09:14:48.699 INFO:tasks.rados.rados.0.smithi163.stdout:2139: finishing write tid 5 to smithi16367185-42 2023-10-09T09:14:48.699 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 1542 (ObjNum 800 snap 278 seq_num 800) dirty exists 2023-10-09T09:14:48.699 INFO:tasks.rados.rados.0.smithi163.stdout:2139: left oid 42 (ObjNum 800 snap 278 seq_num 800) 2023-10-09T09:14:48.699 INFO:tasks.rados.rados.0.smithi163.stdout:2139: done (1 left) 2023-10-09T09:14:48.699 INFO:tasks.rados.rados.0.smithi163.stdout:2142: snap_remove snap 252 2023-10-09T09:14:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[75425]: cluster 2023-10-09T09:14:46.509955+0000 osd.3 (osd.3) 206 : cluster 3 failed to encode map e614 with expected crc 2023-10-09T09:14:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[75425]: cluster 2023-10-09T09:14:46.524026+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e614 with expected crc 2023-10-09T09:14:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[75425]: audit 2023-10-09T09:14:46.623721+0000 mgr.y (mgr.34107) 235 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:14:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[75425]: cephadm 2023-10-09T09:14:46.624704+0000 mgr.y (mgr.34107) 236 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T09:14:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[75425]: cluster 2023-10-09T09:14:47.328267+0000 mgr.y (mgr.34107) 237 : cluster 0 pgmap v474: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 808 KiB/s wr, 9 op/s 2023-10-09T09:14:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[75425]: cluster 2023-10-09T09:14:47.536570+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e615 with expected crc 2023-10-09T09:14:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[75425]: cluster 2023-10-09T09:14:47.539003+0000 mon.a (mon.0) 687 : cluster 0 osdmap e615: 8 total, 8 up, 8 in 2023-10-09T09:14:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[75425]: audit 2023-10-09T09:14:48.338176+0000 mon.a (mon.0) 688 : audit 0 from='client.? 172.21.15.163:0/1412419416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[78107]: cluster 2023-10-09T09:14:46.509955+0000 osd.3 (osd.3) 206 : cluster 3 failed to encode map e614 with expected crc 2023-10-09T09:14:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[78107]: cluster 2023-10-09T09:14:46.524026+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e614 with expected crc 2023-10-09T09:14:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[78107]: audit 2023-10-09T09:14:46.623721+0000 mgr.y (mgr.34107) 235 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:14:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[78107]: cephadm 2023-10-09T09:14:46.624704+0000 mgr.y (mgr.34107) 236 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T09:14:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[78107]: cluster 2023-10-09T09:14:47.328267+0000 mgr.y (mgr.34107) 237 : cluster 0 pgmap v474: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 808 KiB/s wr, 9 op/s 2023-10-09T09:14:48.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[78107]: cluster 2023-10-09T09:14:47.536570+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e615 with expected crc 2023-10-09T09:14:48.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[78107]: cluster 2023-10-09T09:14:47.539003+0000 mon.a (mon.0) 687 : cluster 0 osdmap e615: 8 total, 8 up, 8 in 2023-10-09T09:14:48.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:48 smithi163 bash[78107]: audit 2023-10-09T09:14:48.338176+0000 mon.a (mon.0) 688 : audit 0 from='client.? 172.21.15.163:0/1412419416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:48 smithi181 bash[59345]: cluster 2023-10-09T09:14:46.509955+0000 osd.3 (osd.3) 206 : cluster 3 failed to encode map e614 with expected crc 2023-10-09T09:14:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:48 smithi181 bash[59345]: cluster 2023-10-09T09:14:46.524026+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e614 with expected crc 2023-10-09T09:14:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:48 smithi181 bash[59345]: audit 2023-10-09T09:14:46.623721+0000 mgr.y (mgr.34107) 235 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:14:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:48 smithi181 bash[59345]: cephadm 2023-10-09T09:14:46.624704+0000 mgr.y (mgr.34107) 236 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T09:14:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:48 smithi181 bash[59345]: cluster 2023-10-09T09:14:47.328267+0000 mgr.y (mgr.34107) 237 : cluster 0 pgmap v474: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 808 KiB/s wr, 9 op/s 2023-10-09T09:14:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:48 smithi181 bash[59345]: cluster 2023-10-09T09:14:47.536570+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e615 with expected crc 2023-10-09T09:14:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:48 smithi181 bash[59345]: cluster 2023-10-09T09:14:47.539003+0000 mon.a (mon.0) 687 : cluster 0 osdmap e615: 8 total, 8 up, 8 in 2023-10-09T09:14:48.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:48 smithi181 bash[59345]: audit 2023-10-09T09:14:48.338176+0000 mon.a (mon.0) 688 : audit 0 from='client.? 172.21.15.163:0/1412419416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:49.548 INFO:tasks.rados.rados.0.smithi163.stdout:2142: done (1 left) 2023-10-09T09:14:49.548 INFO:tasks.rados.rados.0.smithi163.stdout:2143: read oid 2 snap -1 2023-10-09T09:14:49.548 INFO:tasks.rados.rados.0.smithi163.stdout:2143: expect deleted 2023-10-09T09:14:49.548 INFO:tasks.rados.rados.0.smithi163.stdout:2144: snap_create 2023-10-09T09:14:49.548 INFO:tasks.rados.rados.0.smithi163.stdout:2141: finishing rollback tid 0 to smithi16367185-8 2023-10-09T09:14:49.548 INFO:tasks.rados.rados.0.smithi163.stdout:2141: finishing rollback tid 1 to smithi16367185-8 2023-10-09T09:14:49.548 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1076 (ObjNum 698735184 snap 21875 seq_num 698865568) dirty dne 2023-10-09T09:14:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[78107]: cluster 2023-10-09T09:14:47.535567+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e615 with expected crc 2023-10-09T09:14:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[78107]: cluster 2023-10-09T09:14:47.539342+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e615 with expected crc 2023-10-09T09:14:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[78107]: cluster 2023-10-09T09:14:48.542806+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[78107]: cluster 2023-10-09T09:14:48.545660+0000 mon.a (mon.0) 689 : cluster 0 osdmap e616: 8 total, 8 up, 8 in 2023-10-09T09:14:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[78107]: cluster 2023-10-09T09:14:48.548524+0000 osd.2 (osd.2) 203 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[78107]: audit 2023-10-09T09:14:48.802845+0000 mon.a (mon.0) 690 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[78107]: audit 2023-10-09T09:14:48.804362+0000 mon.a (mon.0) 691 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:14:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[78107]: audit 2023-10-09T09:14:48.805670+0000 mon.a (mon.0) 692 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[75425]: cluster 2023-10-09T09:14:47.535567+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e615 with expected crc 2023-10-09T09:14:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[75425]: cluster 2023-10-09T09:14:47.539342+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e615 with expected crc 2023-10-09T09:14:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[75425]: cluster 2023-10-09T09:14:48.542806+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[75425]: cluster 2023-10-09T09:14:48.545660+0000 mon.a (mon.0) 689 : cluster 0 osdmap e616: 8 total, 8 up, 8 in 2023-10-09T09:14:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[75425]: cluster 2023-10-09T09:14:48.548524+0000 osd.2 (osd.2) 203 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[75425]: audit 2023-10-09T09:14:48.802845+0000 mon.a (mon.0) 690 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:49.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[75425]: audit 2023-10-09T09:14:48.804362+0000 mon.a (mon.0) 691 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:14:49.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:49 smithi163 bash[75425]: audit 2023-10-09T09:14:48.805670+0000 mon.a (mon.0) 692 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:49 smithi181 bash[59345]: cluster 2023-10-09T09:14:47.535567+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e615 with expected crc 2023-10-09T09:14:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:49 smithi181 bash[59345]: cluster 2023-10-09T09:14:47.539342+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e615 with expected crc 2023-10-09T09:14:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:49 smithi181 bash[59345]: cluster 2023-10-09T09:14:48.542806+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:49 smithi181 bash[59345]: cluster 2023-10-09T09:14:48.545660+0000 mon.a (mon.0) 689 : cluster 0 osdmap e616: 8 total, 8 up, 8 in 2023-10-09T09:14:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:49 smithi181 bash[59345]: cluster 2023-10-09T09:14:48.548524+0000 osd.2 (osd.2) 203 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:49 smithi181 bash[59345]: audit 2023-10-09T09:14:48.802845+0000 mon.a (mon.0) 690 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:49 smithi181 bash[59345]: audit 2023-10-09T09:14:48.804362+0000 mon.a (mon.0) 691 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:14:49.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:49 smithi181 bash[59345]: audit 2023-10-09T09:14:48.805670+0000 mon.a (mon.0) 692 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:14:50.110 INFO:tasks.rados.rados.0.smithi163.stdout:2141: done (2 left) 2023-10-09T09:14:50.110 INFO:tasks.rados.rados.0.smithi163.stdout:2143: done (1 left) 2023-10-09T09:14:50.110 INFO:tasks.rados.rados.0.smithi163.stdout:2144: done (0 left) 2023-10-09T09:14:50.111 INFO:tasks.rados.rados.0.smithi163.stdout:2145: delete oid 13 current snap is 279 2023-10-09T09:14:50.118 INFO:tasks.rados.rados.0.smithi163.stdout:2145: done (0 left) 2023-10-09T09:14:50.118 INFO:tasks.rados.rados.0.smithi163.stdout:2146: write oid 20 current snap is 279 2023-10-09T09:14:50.119 INFO:tasks.rados.rados.0.smithi163.stdout:2146: seq_num 803 ranges {696021=642012,1856570=575278,3185369=310652} 2023-10-09T09:14:50.128 INFO:tasks.rados.rados.0.smithi163.stdout:2146: writing smithi16367185-20 from 696021 to 1338033 tid 1 2023-10-09T09:14:50.134 INFO:tasks.rados.rados.0.smithi163.stdout:2146: writing smithi16367185-20 from 1856570 to 2431848 tid 2 2023-10-09T09:14:50.142 INFO:tasks.rados.rados.0.smithi163.stdout:2146: writing smithi16367185-20 from 3185369 to 3496021 tid 3 2023-10-09T09:14:50.142 INFO:tasks.rados.rados.0.smithi163.stdout:2147: rollback oid 43 current snap is 279 2023-10-09T09:14:50.142 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 277 2023-10-09T09:14:50.142 INFO:tasks.rados.rados.0.smithi163.stdout:2146: finishing write tid 1 to smithi16367185-20 2023-10-09T09:14:50.142 INFO:tasks.rados.rados.0.smithi163.stdout:2146: finishing write tid 2 to smithi16367185-20 2023-10-09T09:14:50.142 INFO:tasks.rados.rados.0.smithi163.stdout:2148: rollback oid 5 current snap is 279 2023-10-09T09:14:50.142 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 266 2023-10-09T09:14:50.143 INFO:tasks.rados.rados.0.smithi163.stdout:2149: write oid 16 current snap is 279 2023-10-09T09:14:50.143 INFO:tasks.rados.rados.0.smithi163.stdout:2149: seq_num 804 ranges {712828=717261,1915052=718936,3212792=300036} 2023-10-09T09:14:50.152 INFO:tasks.rados.rados.0.smithi163.stdout:2149: writing smithi16367185-16 from 712828 to 1430089 tid 1 2023-10-09T09:14:50.161 INFO:tasks.rados.rados.0.smithi163.stdout:2149: writing smithi16367185-16 from 1915052 to 2633988 tid 2 2023-10-09T09:14:50.166 INFO:tasks.rados.rados.0.smithi163.stdout:2149: writing smithi16367185-16 from 3212792 to 3512828 tid 3 2023-10-09T09:14:50.167 INFO:tasks.rados.rados.0.smithi163.stdout:2150: read oid 42 snap 266 2023-10-09T09:14:50.167 INFO:tasks.rados.rados.0.smithi163.stdout:2150: expect deleted 2023-10-09T09:14:50.167 INFO:tasks.rados.rados.0.smithi163.stdout:2151: write oid 17 current snap is 279 2023-10-09T09:14:50.167 INFO:tasks.rados.rados.0.smithi163.stdout:2146: finishing write tid 3 to smithi16367185-20 2023-10-09T09:14:50.167 INFO:tasks.rados.rados.0.smithi163.stdout:2146: finishing write tid 4 to smithi16367185-20 2023-10-09T09:14:50.167 INFO:tasks.rados.rados.0.smithi163.stdout:2146: finishing write tid 5 to smithi16367185-20 2023-10-09T09:14:50.167 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 249 (ObjNum 803 snap 279 seq_num 803) dirty exists 2023-10-09T09:14:50.167 INFO:tasks.rados.rados.0.smithi163.stdout:2146: left oid 20 (ObjNum 803 snap 279 seq_num 803) 2023-10-09T09:14:50.168 INFO:tasks.rados.rados.0.smithi163.stdout:2147: finishing rollback tid 0 to smithi16367185-43 2023-10-09T09:14:50.168 INFO:tasks.rados.rados.0.smithi163.stdout:2147: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:14:50.168 INFO:tasks.rados.rados.0.smithi163.stdout:2147: finishing rollback tid 2 to smithi16367185-43 2023-10-09T09:14:50.168 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 925 (ObjNum 733 snap 246 seq_num 733) dirty exists 2023-10-09T09:14:50.168 INFO:tasks.rados.rados.0.smithi163.stdout:2148: finishing rollback tid 0 to smithi16367185-5 2023-10-09T09:14:50.168 INFO:tasks.rados.rados.0.smithi163.stdout:2151: seq_num 805 ranges {729635=792510,1973534=578947,3240215=289420} 2023-10-09T09:14:50.175 INFO:tasks.rados.rados.0.smithi163.stdout:2151: writing smithi16367185-17 from 729635 to 1522145 tid 1 2023-10-09T09:14:50.180 INFO:tasks.rados.rados.0.smithi163.stdout:2151: writing smithi16367185-17 from 1973534 to 2552481 tid 2 2023-10-09T09:14:50.187 INFO:tasks.rados.rados.0.smithi163.stdout:2151: writing smithi16367185-17 from 3240215 to 3529635 tid 3 2023-10-09T09:14:50.187 INFO:tasks.rados.rados.0.smithi163.stdout:2146: done (5 left) 2023-10-09T09:14:50.187 INFO:tasks.rados.rados.0.smithi163.stdout:2147: done (4 left) 2023-10-09T09:14:50.187 INFO:tasks.rados.rados.0.smithi163.stdout:2152: write oid 34 current snap is 279 2023-10-09T09:14:50.187 INFO:tasks.rados.rados.0.smithi163.stdout:2148: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:14:50.187 INFO:tasks.rados.rados.0.smithi163.stdout:2148: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:14:50.187 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1083 (ObjNum 710 snap 237 seq_num 710) dirty exists 2023-10-09T09:14:50.187 INFO:tasks.rados.rados.0.smithi163.stdout:2149: finishing write tid 1 to smithi16367185-16 2023-10-09T09:14:50.187 INFO:tasks.rados.rados.0.smithi163.stdout:2152: seq_num 806 ranges {746442=584112,1748369=722605,2983991=562451} 2023-10-09T09:14:50.194 INFO:tasks.rados.rados.0.smithi163.stdout:2152: writing smithi16367185-34 from 746442 to 1330554 tid 1 2023-10-09T09:14:50.200 INFO:tasks.rados.rados.0.smithi163.stdout:2152: writing smithi16367185-34 from 1748369 to 2470974 tid 2 2023-10-09T09:14:50.206 INFO:tasks.rados.rados.0.smithi163.stdout:2152: writing smithi16367185-34 from 2983991 to 3546442 tid 3 2023-10-09T09:14:50.206 INFO:tasks.rados.rados.0.smithi163.stdout:2148: done (4 left) 2023-10-09T09:14:50.207 INFO:tasks.rados.rados.0.smithi163.stdout:2153: write oid 20 current snap is 279 2023-10-09T09:14:50.207 INFO:tasks.rados.rados.0.smithi163.stdout:2153: seq_num 807 ranges {763249=659361,2090498=582616,3295061=268188} 2023-10-09T09:14:50.214 INFO:tasks.rados.rados.0.smithi163.stdout:2153: writing smithi16367185-20 from 763249 to 1422610 tid 1 2023-10-09T09:14:50.221 INFO:tasks.rados.rados.0.smithi163.stdout:2153: writing smithi16367185-20 from 2090498 to 2673114 tid 2 2023-10-09T09:14:50.227 INFO:tasks.rados.rados.0.smithi163.stdout:2153: writing smithi16367185-20 from 3295061 to 3563249 tid 3 2023-10-09T09:14:50.227 INFO:tasks.rados.rados.0.smithi163.stdout:2154: delete oid 5 current snap is 279 2023-10-09T09:14:50.227 INFO:tasks.rados.rados.0.smithi163.stdout:2149: finishing write tid 2 to smithi16367185-16 2023-10-09T09:14:50.227 INFO:tasks.rados.rados.0.smithi163.stdout:2149: finishing write tid 3 to smithi16367185-16 2023-10-09T09:14:50.227 INFO:tasks.rados.rados.0.smithi163.stdout:2149: finishing write tid 4 to smithi16367185-16 2023-10-09T09:14:50.227 INFO:tasks.rados.rados.0.smithi163.stdout:2149: finishing write tid 5 to smithi16367185-16 2023-10-09T09:14:50.227 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1088 (ObjNum 804 snap 279 seq_num 804) dirty exists 2023-10-09T09:14:50.227 INFO:tasks.rados.rados.0.smithi163.stdout:2149: left oid 16 (ObjNum 804 snap 279 seq_num 804) 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2151: finishing write tid 1 to smithi16367185-17 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2151: finishing write tid 2 to smithi16367185-17 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2151: finishing write tid 3 to smithi16367185-17 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2151: finishing write tid 4 to smithi16367185-17 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2151: finishing write tid 5 to smithi16367185-17 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1085 (ObjNum 805 snap 279 seq_num 805) dirty exists 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2151: left oid 17 (ObjNum 805 snap 279 seq_num 805) 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2152: finishing write tid 1 to smithi16367185-34 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2152: finishing write tid 2 to smithi16367185-34 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2152: finishing write tid 3 to smithi16367185-34 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2152: finishing write tid 4 to smithi16367185-34 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:2152: finishing write tid 5 to smithi16367185-34 2023-10-09T09:14:50.228 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 914 (ObjNum 806 snap 279 seq_num 806) dirty exists 2023-10-09T09:14:50.229 INFO:tasks.rados.rados.0.smithi163.stdout:2152: left oid 34 (ObjNum 806 snap 279 seq_num 806) 2023-10-09T09:14:50.229 INFO:tasks.rados.rados.0.smithi163.stdout:2153: finishing write tid 1 to smithi16367185-20 2023-10-09T09:14:50.229 INFO:tasks.rados.rados.0.smithi163.stdout:2153: finishing write tid 2 to smithi16367185-20 2023-10-09T09:14:50.229 INFO:tasks.rados.rados.0.smithi163.stdout:2149: done (5 left) 2023-10-09T09:14:50.229 INFO:tasks.rados.rados.0.smithi163.stdout:2150: done (4 left) 2023-10-09T09:14:50.230 INFO:tasks.rados.rados.0.smithi163.stdout:2151: done (3 left) 2023-10-09T09:14:50.230 INFO:tasks.rados.rados.0.smithi163.stdout:2152: done (2 left) 2023-10-09T09:14:50.230 INFO:tasks.rados.rados.0.smithi163.stdout:2154: done (1 left) 2023-10-09T09:14:50.230 INFO:tasks.rados.rados.0.smithi163.stdout:2155: delete oid 9 current snap is 279 2023-10-09T09:14:50.230 INFO:tasks.rados.rados.0.smithi163.stdout:2153: finishing write tid 3 to smithi16367185-20 2023-10-09T09:14:50.230 INFO:tasks.rados.rados.0.smithi163.stdout:2153: finishing write tid 4 to smithi16367185-20 2023-10-09T09:14:50.231 INFO:tasks.rados.rados.0.smithi163.stdout:2153: finishing write tid 5 to smithi16367185-20 2023-10-09T09:14:50.231 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 253 (ObjNum 807 snap 279 seq_num 807) dirty exists 2023-10-09T09:14:50.231 INFO:tasks.rados.rados.0.smithi163.stdout:2153: left oid 20 (ObjNum 807 snap 279 seq_num 807) 2023-10-09T09:14:50.233 INFO:tasks.rados.rados.0.smithi163.stdout:2153: done (1 left) 2023-10-09T09:14:50.233 INFO:tasks.rados.rados.0.smithi163.stdout:2155: done (0 left) 2023-10-09T09:14:50.233 INFO:tasks.rados.rados.0.smithi163.stdout:2156: read oid 43 snap -1 2023-10-09T09:14:50.233 INFO:tasks.rados.rados.0.smithi163.stdout:2156: expect (ObjNum 733 snap 246 seq_num 733) 2023-10-09T09:14:50.234 INFO:tasks.rados.rados.0.smithi163.stdout:2157: read oid 19 snap -1 2023-10-09T09:14:50.234 INFO:tasks.rados.rados.0.smithi163.stdout:2157: expect deleted 2023-10-09T09:14:50.234 INFO:tasks.rados.rados.0.smithi163.stdout:2158: read oid 35 snap -1 2023-10-09T09:14:50.234 INFO:tasks.rados.rados.0.smithi163.stdout:2158: expect deleted 2023-10-09T09:14:50.234 INFO:tasks.rados.rados.0.smithi163.stdout:2159: rollback oid 22 current snap is 279 2023-10-09T09:14:50.234 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 22 to 266 2023-10-09T09:14:50.234 INFO:tasks.rados.rados.0.smithi163.stdout:2160: read oid 31 snap -1 2023-10-09T09:14:50.234 INFO:tasks.rados.rados.0.smithi163.stdout:2160: expect (ObjNum 796 snap 274 seq_num 796) 2023-10-09T09:14:50.234 INFO:tasks.rados.rados.0.smithi163.stdout:2161: snap_remove snap 275 2023-10-09T09:14:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[75425]: cluster 2023-10-09T09:14:48.540332+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[75425]: cluster 2023-10-09T09:14:48.699167+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[75425]: cephadm 2023-10-09T09:14:48.793290+0000 mgr.y (mgr.34107) 238 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T09:14:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[75425]: cephadm 2023-10-09T09:14:48.808579+0000 mgr.y (mgr.34107) 239 : cephadm 1 Deploying daemon osd.2 on smithi163 2023-10-09T09:14:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[75425]: cluster 2023-10-09T09:14:49.328897+0000 mgr.y (mgr.34107) 240 : cluster 0 pgmap v477: 89 pgs: 89 active+clean; 217 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 808 KiB/s wr, 9 op/s 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[75425]: cluster 2023-10-09T09:14:49.546546+0000 mon.a (mon.0) 693 : cluster 0 osdmap e617: 8 total, 8 up, 8 in 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[75425]: cluster 2023-10-09T09:14:49.547409+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[75425]: cluster 2023-10-09T09:14:50.109513+0000 mon.a (mon.0) 694 : cluster 0 osdmap e618: 8 total, 8 up, 8 in 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[75425]: cluster 2023-10-09T09:14:50.142317+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[75425]: cluster 2023-10-09T09:14:50.143520+0000 osd.2 (osd.2) 204 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[78107]: cluster 2023-10-09T09:14:48.540332+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[78107]: cluster 2023-10-09T09:14:48.699167+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[78107]: cephadm 2023-10-09T09:14:48.793290+0000 mgr.y (mgr.34107) 238 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[78107]: cephadm 2023-10-09T09:14:48.808579+0000 mgr.y (mgr.34107) 239 : cephadm 1 Deploying daemon osd.2 on smithi163 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[78107]: cluster 2023-10-09T09:14:49.328897+0000 mgr.y (mgr.34107) 240 : cluster 0 pgmap v477: 89 pgs: 89 active+clean; 217 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 808 KiB/s wr, 9 op/s 2023-10-09T09:14:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[78107]: cluster 2023-10-09T09:14:49.546546+0000 mon.a (mon.0) 693 : cluster 0 osdmap e617: 8 total, 8 up, 8 in 2023-10-09T09:14:50.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[78107]: cluster 2023-10-09T09:14:49.547409+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:50.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[78107]: cluster 2023-10-09T09:14:50.109513+0000 mon.a (mon.0) 694 : cluster 0 osdmap e618: 8 total, 8 up, 8 in 2023-10-09T09:14:50.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[78107]: cluster 2023-10-09T09:14:50.142317+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:50.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:50 smithi163 bash[78107]: cluster 2023-10-09T09:14:50.143520+0000 osd.2 (osd.2) 204 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:50 smithi181 bash[59345]: cluster 2023-10-09T09:14:48.540332+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:50 smithi181 bash[59345]: cluster 2023-10-09T09:14:48.699167+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e616 with expected crc 2023-10-09T09:14:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:50 smithi181 bash[59345]: cephadm 2023-10-09T09:14:48.793290+0000 mgr.y (mgr.34107) 238 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T09:14:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:50 smithi181 bash[59345]: cephadm 2023-10-09T09:14:48.808579+0000 mgr.y (mgr.34107) 239 : cephadm 1 Deploying daemon osd.2 on smithi163 2023-10-09T09:14:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:50 smithi181 bash[59345]: cluster 2023-10-09T09:14:49.328897+0000 mgr.y (mgr.34107) 240 : cluster 0 pgmap v477: 89 pgs: 89 active+clean; 217 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 808 KiB/s wr, 9 op/s 2023-10-09T09:14:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:50 smithi181 bash[59345]: cluster 2023-10-09T09:14:49.546546+0000 mon.a (mon.0) 693 : cluster 0 osdmap e617: 8 total, 8 up, 8 in 2023-10-09T09:14:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:50 smithi181 bash[59345]: cluster 2023-10-09T09:14:49.547409+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:50 smithi181 bash[59345]: cluster 2023-10-09T09:14:50.109513+0000 mon.a (mon.0) 694 : cluster 0 osdmap e618: 8 total, 8 up, 8 in 2023-10-09T09:14:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:50 smithi181 bash[59345]: cluster 2023-10-09T09:14:50.142317+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:50.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:50 smithi181 bash[59345]: cluster 2023-10-09T09:14:50.143520+0000 osd.2 (osd.2) 204 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:51.119 INFO:tasks.rados.rados.0.smithi163.stdout:2161: done (5 left) 2023-10-09T09:14:51.119 INFO:tasks.rados.rados.0.smithi163.stdout:2162: read oid 17 snap -1 2023-10-09T09:14:51.119 INFO:tasks.rados.rados.0.smithi163.stdout:2162: expect (ObjNum 805 snap 279 seq_num 805) 2023-10-09T09:14:51.119 INFO:tasks.rados.rados.0.smithi163.stdout:2163: write oid 2 current snap is 279 2023-10-09T09:14:51.119 INFO:tasks.rados.rados.0.smithi163.stdout:2163: seq_num 810 ranges {413670=485108,1465944=729943,2977330=524801} 2023-10-09T09:14:51.124 INFO:tasks.rados.rados.0.smithi163.stdout:2163: writing smithi16367185-2 from 413670 to 898778 tid 1 2023-10-09T09:14:51.132 INFO:tasks.rados.rados.0.smithi163.stdout:2163: writing smithi16367185-2 from 1465944 to 2195887 tid 2 2023-10-09T09:14:51.140 INFO:tasks.rados.rados.0.smithi163.stdout:2163: writing smithi16367185-2 from 2977330 to 3502131 tid 3 2023-10-09T09:14:51.140 INFO:tasks.rados.rados.0.smithi163.stdout:2164: snap_remove snap 271 2023-10-09T09:14:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[75425]: cluster 2023-10-09T09:14:49.542307+0000 osd.3 (osd.3) 207 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[75425]: cluster 2023-10-09T09:14:49.546185+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[75425]: cluster 2023-10-09T09:14:49.546327+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[75425]: cluster 2023-10-09T09:14:49.548499+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[75425]: cluster 2023-10-09T09:14:50.104448+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[75425]: cluster 2023-10-09T09:14:50.111030+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[75425]: cluster 2023-10-09T09:14:50.176782+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[75425]: audit 2023-10-09T09:14:50.415718+0000 mon.a (mon.0) 695 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[75425]: audit 2023-10-09T09:14:50.693191+0000 mon.a (mon.0) 696 : audit 0 from='client.? 172.21.15.163:0/2495809422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:51.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[75425]: cluster 2023-10-09T09:14:51.118659+0000 mon.a (mon.0) 697 : cluster 0 osdmap e619: 8 total, 8 up, 8 in 2023-10-09T09:14:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[78107]: cluster 2023-10-09T09:14:49.542307+0000 osd.3 (osd.3) 207 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[78107]: cluster 2023-10-09T09:14:49.546185+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[78107]: cluster 2023-10-09T09:14:49.546327+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[78107]: cluster 2023-10-09T09:14:49.548499+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[78107]: cluster 2023-10-09T09:14:50.104448+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[78107]: cluster 2023-10-09T09:14:50.111030+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[78107]: cluster 2023-10-09T09:14:50.176782+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:51.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[78107]: audit 2023-10-09T09:14:50.415718+0000 mon.a (mon.0) 695 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:51.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[78107]: audit 2023-10-09T09:14:50.693191+0000 mon.a (mon.0) 696 : audit 0 from='client.? 172.21.15.163:0/2495809422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:51.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:51 smithi163 bash[78107]: cluster 2023-10-09T09:14:51.118659+0000 mon.a (mon.0) 697 : cluster 0 osdmap e619: 8 total, 8 up, 8 in 2023-10-09T09:14:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:51 smithi181 bash[59345]: cluster 2023-10-09T09:14:49.542307+0000 osd.3 (osd.3) 207 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:51 smithi181 bash[59345]: cluster 2023-10-09T09:14:49.546185+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:51 smithi181 bash[59345]: cluster 2023-10-09T09:14:49.546327+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:51 smithi181 bash[59345]: cluster 2023-10-09T09:14:49.548499+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e617 with expected crc 2023-10-09T09:14:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:51 smithi181 bash[59345]: cluster 2023-10-09T09:14:50.104448+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:51 smithi181 bash[59345]: cluster 2023-10-09T09:14:50.111030+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:51 smithi181 bash[59345]: cluster 2023-10-09T09:14:50.176782+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e618 with expected crc 2023-10-09T09:14:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:51 smithi181 bash[59345]: audit 2023-10-09T09:14:50.415718+0000 mon.a (mon.0) 695 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:51.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:51 smithi181 bash[59345]: audit 2023-10-09T09:14:50.693191+0000 mon.a (mon.0) 696 : audit 0 from='client.? 172.21.15.163:0/2495809422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:51.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:51 smithi181 bash[59345]: cluster 2023-10-09T09:14:51.118659+0000 mon.a (mon.0) 697 : cluster 0 osdmap e619: 8 total, 8 up, 8 in 2023-10-09T09:14:52.128 INFO:tasks.rados.rados.0.smithi163.stdout:2164: done (7 left) 2023-10-09T09:14:52.128 INFO:tasks.rados.rados.0.smithi163.stdout:2165: delete oid 24 current snap is 279 2023-10-09T09:14:52.128 INFO:tasks.rados.rados.0.smithi163.stdout:2159: finishing rollback tid 0 to smithi16367185-22 2023-10-09T09:14:52.128 INFO:tasks.rados.rados.0.smithi163.stdout:2159: finishing rollback tid 1 to smithi16367185-22 2023-10-09T09:14:52.128 INFO:tasks.rados.rados.0.smithi163.stdout:2160: expect (ObjNum 796 snap 274 seq_num 796) 2023-10-09T09:14:52.249 INFO:tasks.rados.rados.0.smithi163.stdout:2159: finishing rollback tid 2 to smithi16367185-22 2023-10-09T09:14:52.249 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 1554 (ObjNum 624 snap 198 seq_num 624) dirty exists 2023-10-09T09:14:52.249 INFO:tasks.rados.rados.0.smithi163.stdout:2156: expect (ObjNum 733 snap 246 seq_num 733) 2023-10-09T09:14:52.373 INFO:tasks.rados.rados.0.smithi163.stdout:2163: finishing write tid 1 to smithi16367185-2 2023-10-09T09:14:52.373 INFO:tasks.rados.rados.0.smithi163.stdout:2162: expect (ObjNum 805 snap 279 seq_num 805) 2023-10-09T09:14:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:52 smithi163 bash[75425]: cluster 2023-10-09T09:14:51.115796+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e619 with expected crc 2023-10-09T09:14:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:52 smithi163 bash[75425]: cluster 2023-10-09T09:14:51.116433+0000 osd.3 (osd.3) 208 : cluster 3 failed to encode map e619 with expected crc 2023-10-09T09:14:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:52 smithi163 bash[75425]: cluster 2023-10-09T09:14:51.329980+0000 mgr.y (mgr.34107) 241 : cluster 0 pgmap v481: 89 pgs: 89 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.2 MiB/s wr, 25 op/s 2023-10-09T09:14:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:52 smithi163 bash[75425]: cluster 2023-10-09T09:14:52.123554+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e620 with expected crc 2023-10-09T09:14:52.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:52 smithi163 bash[75425]: cluster 2023-10-09T09:14:52.127989+0000 mon.a (mon.0) 698 : cluster 0 osdmap e620: 8 total, 8 up, 8 in 2023-10-09T09:14:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:52 smithi163 bash[78107]: cluster 2023-10-09T09:14:51.115796+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e619 with expected crc 2023-10-09T09:14:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:52 smithi163 bash[78107]: cluster 2023-10-09T09:14:51.116433+0000 osd.3 (osd.3) 208 : cluster 3 failed to encode map e619 with expected crc 2023-10-09T09:14:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:52 smithi163 bash[78107]: cluster 2023-10-09T09:14:51.329980+0000 mgr.y (mgr.34107) 241 : cluster 0 pgmap v481: 89 pgs: 89 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.2 MiB/s wr, 25 op/s 2023-10-09T09:14:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:52 smithi163 bash[78107]: cluster 2023-10-09T09:14:52.123554+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e620 with expected crc 2023-10-09T09:14:52.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:52 smithi163 bash[78107]: cluster 2023-10-09T09:14:52.127989+0000 mon.a (mon.0) 698 : cluster 0 osdmap e620: 8 total, 8 up, 8 in 2023-10-09T09:14:52.526 INFO:tasks.rados.rados.0.smithi163.stdout:2163: finishing write tid 2 to smithi16367185-2 2023-10-09T09:14:52.526 INFO:tasks.rados.rados.0.smithi163.stdout:2163: finishing write tid 3 to smithi16367185-2 2023-10-09T09:14:52.526 INFO:tasks.rados.rados.0.smithi163.stdout:2163: finishing write tid 4 to smithi16367185-2 2023-10-09T09:14:52.527 INFO:tasks.rados.rados.0.smithi163.stdout:2163: finishing write tid 5 to smithi16367185-2 2023-10-09T09:14:52.527 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 597 (ObjNum 810 snap 279 seq_num 810) dirty exists 2023-10-09T09:14:52.527 INFO:tasks.rados.rados.0.smithi163.stdout:2163: left oid 2 (ObjNum 810 snap 279 seq_num 810) 2023-10-09T09:14:52.527 INFO:tasks.rados.rados.0.smithi163.stdout:2156: done (7 left) 2023-10-09T09:14:52.527 INFO:tasks.rados.rados.0.smithi163.stdout:2157: done (6 left) 2023-10-09T09:14:52.527 INFO:tasks.rados.rados.0.smithi163.stdout:2158: done (5 left) 2023-10-09T09:14:52.528 INFO:tasks.rados.rados.0.smithi163.stdout:2159: done (4 left) 2023-10-09T09:14:52.528 INFO:tasks.rados.rados.0.smithi163.stdout:2160: done (3 left) 2023-10-09T09:14:52.528 INFO:tasks.rados.rados.0.smithi163.stdout:2162: done (2 left) 2023-10-09T09:14:52.528 INFO:tasks.rados.rados.0.smithi163.stdout:2163: done (1 left) 2023-10-09T09:14:52.528 INFO:tasks.rados.rados.0.smithi163.stdout:2165: done (0 left) 2023-10-09T09:14:52.528 INFO:tasks.rados.rados.0.smithi163.stdout:2166: rollback oid 13 current snap is 279 2023-10-09T09:14:52.528 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 13 to 239 2023-10-09T09:14:52.529 INFO:tasks.rados.rados.0.smithi163.stdout:2167: delete oid 33 current snap is 279 2023-10-09T09:14:52.529 INFO:tasks.rados.rados.0.smithi163.stdout:2167: done (1 left) 2023-10-09T09:14:52.530 INFO:tasks.rados.rados.0.smithi163.stdout:2168: snap_create 2023-10-09T09:14:52.530 INFO:tasks.rados.rados.0.smithi163.stdout:2166: finishing rollback tid 1 to smithi16367185-13 2023-10-09T09:14:52.530 INFO:tasks.rados.rados.0.smithi163.stdout:2166: finishing rollback tid 2 to smithi16367185-13 2023-10-09T09:14:52.531 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 263 (ObjNum 688 snap 219 seq_num 688) dirty exists 2023-10-09T09:14:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:52 smithi181 bash[59345]: cluster 2023-10-09T09:14:51.115796+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e619 with expected crc 2023-10-09T09:14:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:52 smithi181 bash[59345]: cluster 2023-10-09T09:14:51.116433+0000 osd.3 (osd.3) 208 : cluster 3 failed to encode map e619 with expected crc 2023-10-09T09:14:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:52 smithi181 bash[59345]: cluster 2023-10-09T09:14:51.329980+0000 mgr.y (mgr.34107) 241 : cluster 0 pgmap v481: 89 pgs: 89 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.2 MiB/s wr, 25 op/s 2023-10-09T09:14:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:52 smithi181 bash[59345]: cluster 2023-10-09T09:14:52.123554+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e620 with expected crc 2023-10-09T09:14:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:52 smithi181 bash[59345]: cluster 2023-10-09T09:14:52.127989+0000 mon.a (mon.0) 698 : cluster 0 osdmap e620: 8 total, 8 up, 8 in 2023-10-09T09:14:53.360 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:53 smithi163 systemd[1]: Stopping Ceph osd.2 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:14:53.436 INFO:tasks.rados.rados.0.smithi163.stdout:2166: done (1 left) 2023-10-09T09:14:53.436 INFO:tasks.rados.rados.0.smithi163.stdout:2168: done (0 left) 2023-10-09T09:14:53.436 INFO:tasks.rados.rados.0.smithi163.stdout:2169: write oid 4 current snap is 280 2023-10-09T09:14:53.436 INFO:tasks.rados.rados.0.smithi163.stdout:2169: seq_num 813 ranges {464091=710855,1925037=477270,2943246=474970} 2023-10-09T09:14:53.446 INFO:tasks.rados.rados.0.smithi163.stdout:2169: writing smithi16367185-4 from 464091 to 1174946 tid 1 2023-10-09T09:14:53.454 INFO:tasks.rados.rados.0.smithi163.stdout:2169: writing smithi16367185-4 from 1925037 to 2402307 tid 2 2023-10-09T09:14:53.460 INFO:tasks.rados.rados.0.smithi163.stdout:2169: writing smithi16367185-4 from 2943246 to 3418216 tid 3 2023-10-09T09:14:53.460 INFO:tasks.rados.rados.0.smithi163.stdout:2170: read oid 37 snap -1 2023-10-09T09:14:53.460 INFO:tasks.rados.rados.0.smithi163.stdout:2170: expect deleted 2023-10-09T09:14:53.460 INFO:tasks.rados.rados.0.smithi163.stdout:2171: snap_remove snap 266 2023-10-09T09:14:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[78107]: cluster 2023-10-09T09:14:52.124818+0000 osd.3 (osd.3) 209 : cluster 3 failed to encode map e620 with expected crc 2023-10-09T09:14:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[78107]: cluster 2023-10-09T09:14:52.173018+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e620 with expected crc 2023-10-09T09:14:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[78107]: audit 2023-10-09T09:14:53.060826+0000 mon.a (mon.0) 699 : audit 0 from='client.? 172.21.15.163:0/596858159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[78107]: cluster 2023-10-09T09:14:53.397696+0000 mon.a (mon.0) 700 : cluster 1 osd.2 marked itself down and dead 2023-10-09T09:14:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:14:53] "GET /metrics HTTP/1.1" 200 36056 "" "Prometheus/2.33.4" 2023-10-09T09:14:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[75425]: cluster 2023-10-09T09:14:52.124818+0000 osd.3 (osd.3) 209 : cluster 3 failed to encode map e620 with expected crc 2023-10-09T09:14:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[75425]: cluster 2023-10-09T09:14:52.173018+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e620 with expected crc 2023-10-09T09:14:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[75425]: audit 2023-10-09T09:14:53.060826+0000 mon.a (mon.0) 699 : audit 0 from='client.? 172.21.15.163:0/596858159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[75425]: cluster 2023-10-09T09:14:53.397696+0000 mon.a (mon.0) 700 : cluster 1 osd.2 marked itself down and dead 2023-10-09T09:14:53.719 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[45486]: debug 2023-10-09T09:14:53.391+0000 7f8a3afcf700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:14:53.719 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[45486]: debug 2023-10-09T09:14:53.391+0000 7f8a3afcf700 -1 osd.2 620 *** Got signal Terminated *** 2023-10-09T09:14:53.719 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:53 smithi163 bash[45486]: debug 2023-10-09T09:14:53.391+0000 7f8a3afcf700 -1 osd.2 620 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:14:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:53 smithi181 bash[59345]: cluster 2023-10-09T09:14:52.124818+0000 osd.3 (osd.3) 209 : cluster 3 failed to encode map e620 with expected crc 2023-10-09T09:14:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:53 smithi181 bash[59345]: cluster 2023-10-09T09:14:52.173018+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e620 with expected crc 2023-10-09T09:14:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:53 smithi181 bash[59345]: audit 2023-10-09T09:14:53.060826+0000 mon.a (mon.0) 699 : audit 0 from='client.? 172.21.15.163:0/596858159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:53 smithi181 bash[59345]: cluster 2023-10-09T09:14:53.397696+0000 mon.a (mon.0) 700 : cluster 1 osd.2 marked itself down and dead 2023-10-09T09:14:54.401 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[89067]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T09:14:54.401 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[89136]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T09:14:54.443 INFO:tasks.rados.rados.0.smithi163.stdout:2171: done (2 left) 2023-10-09T09:14:54.443 INFO:tasks.rados.rados.0.smithi163.stdout:2172: write oid 33 current snap is 280 2023-10-09T09:14:54.443 INFO:tasks.rados.rados.0.smithi163.stdout:2172: seq_num 814 ranges {480898=502457,1699872=737281,3087022=593876} 2023-10-09T09:14:54.450 INFO:tasks.rados.rados.0.smithi163.stdout:2172: writing smithi16367185-33 from 480898 to 983355 tid 1 2023-10-09T09:14:54.460 INFO:tasks.rados.rados.0.smithi163.stdout:2172: writing smithi16367185-33 from 1699872 to 2437153 tid 2 2023-10-09T09:14:54.468 INFO:tasks.rados.rados.0.smithi163.stdout:2172: writing smithi16367185-33 from 3087022 to 3680898 tid 3 2023-10-09T09:14:54.468 INFO:tasks.rados.rados.0.smithi163.stdout:2173: rollback oid 32 current snap is 280 2023-10-09T09:14:54.468 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 279 2023-10-09T09:14:54.468 INFO:tasks.rados.rados.0.smithi163.stdout:2170: done (3 left) 2023-10-09T09:14:54.469 INFO:tasks.rados.rados.0.smithi163.stdout:2174: snap_create 2023-10-09T09:14:54.472 INFO:tasks.rados.rados.0.smithi163.stdout:2172: finishing write tid 1 to smithi16367185-33 2023-10-09T09:14:54.474 INFO:tasks.rados.rados.0.smithi163.stdout:2173: finishing rollback tid 0 to smithi16367185-32 2023-10-09T09:14:54.475 INFO:tasks.rados.rados.0.smithi163.stdout:2173: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:14:54.475 INFO:tasks.rados.rados.0.smithi163.stdout:2173: finishing rollback tid 2 to smithi16367185-32 2023-10-09T09:14:54.475 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1566 (ObjNum 750 snap 255 seq_num 750) dirty exists 2023-10-09T09:14:54.475 INFO:tasks.rados.rados.0.smithi163.stdout:2172: finishing write tid 2 to smithi16367185-33 2023-10-09T09:14:54.475 INFO:tasks.rados.rados.0.smithi163.stdout:2169: finishing write tid 1 to smithi16367185-4 2023-10-09T09:14:54.475 INFO:tasks.rados.rados.0.smithi163.stdout:2169: finishing write tid 2 to smithi16367185-4 2023-10-09T09:14:54.475 INFO:tasks.rados.rados.0.smithi163.stdout:2172: finishing write tid 3 to smithi16367185-33 2023-10-09T09:14:54.476 INFO:tasks.rados.rados.0.smithi163.stdout:2169: finishing write tid 3 to smithi16367185-4 2023-10-09T09:14:54.476 INFO:tasks.rados.rados.0.smithi163.stdout:2172: finishing write tid 4 to smithi16367185-33 2023-10-09T09:14:54.476 INFO:tasks.rados.rados.0.smithi163.stdout:2169: finishing write tid 4 to smithi16367185-4 2023-10-09T09:14:54.476 INFO:tasks.rados.rados.0.smithi163.stdout:2172: finishing write tid 5 to smithi16367185-33 2023-10-09T09:14:54.476 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1023 (ObjNum 814 snap 280 seq_num 814) dirty exists 2023-10-09T09:14:54.477 INFO:tasks.rados.rados.0.smithi163.stdout:2172: left oid 33 (ObjNum 814 snap 280 seq_num 814) 2023-10-09T09:14:54.477 INFO:tasks.rados.rados.0.smithi163.stdout:2169: finishing write tid 5 to smithi16367185-4 2023-10-09T09:14:54.477 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1103 (ObjNum 813 snap 280 seq_num 813) dirty exists 2023-10-09T09:14:54.477 INFO:tasks.rados.rados.0.smithi163.stdout:2169: left oid 4 (ObjNum 813 snap 280 seq_num 813) 2023-10-09T09:14:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[75425]: cluster 2023-10-09T09:14:53.330811+0000 mgr.y (mgr.34107) 242 : cluster 0 pgmap v483: 89 pgs: 89 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2023-10-09T09:14:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[75425]: cluster 2023-10-09T09:14:53.418543+0000 mon.a (mon.0) 701 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:14:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[75425]: cluster 2023-10-09T09:14:53.429310+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e621 with expected crc 2023-10-09T09:14:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[75425]: cluster 2023-10-09T09:14:53.430948+0000 osd.3 (osd.3) 210 : cluster 3 failed to encode map e621 with expected crc 2023-10-09T09:14:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[75425]: cluster 2023-10-09T09:14:53.435928+0000 mon.a (mon.0) 702 : cluster 0 osdmap e621: 8 total, 7 up, 8 in 2023-10-09T09:14:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[78107]: cluster 2023-10-09T09:14:53.330811+0000 mgr.y (mgr.34107) 242 : cluster 0 pgmap v483: 89 pgs: 89 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2023-10-09T09:14:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[78107]: cluster 2023-10-09T09:14:53.418543+0000 mon.a (mon.0) 701 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:14:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[78107]: cluster 2023-10-09T09:14:53.429310+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e621 with expected crc 2023-10-09T09:14:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[78107]: cluster 2023-10-09T09:14:53.430948+0000 osd.3 (osd.3) 210 : cluster 3 failed to encode map e621 with expected crc 2023-10-09T09:14:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:54 smithi163 bash[78107]: cluster 2023-10-09T09:14:53.435928+0000 mon.a (mon.0) 702 : cluster 0 osdmap e621: 8 total, 7 up, 8 in 2023-10-09T09:14:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:54 smithi181 bash[59345]: cluster 2023-10-09T09:14:53.330811+0000 mgr.y (mgr.34107) 242 : cluster 0 pgmap v483: 89 pgs: 89 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2023-10-09T09:14:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:54 smithi181 bash[59345]: cluster 2023-10-09T09:14:53.418543+0000 mon.a (mon.0) 701 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:14:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:54 smithi181 bash[59345]: cluster 2023-10-09T09:14:53.429310+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e621 with expected crc 2023-10-09T09:14:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:54 smithi181 bash[59345]: cluster 2023-10-09T09:14:53.430948+0000 osd.3 (osd.3) 210 : cluster 3 failed to encode map e621 with expected crc 2023-10-09T09:14:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:54 smithi181 bash[59345]: cluster 2023-10-09T09:14:53.435928+0000 mon.a (mon.0) 702 : cluster 0 osdmap e621: 8 total, 7 up, 8 in 2023-10-09T09:14:55.469 INFO:tasks.rados.rados.0.smithi163.stdout:2169: done (3 left) 2023-10-09T09:14:55.470 INFO:tasks.rados.rados.0.smithi163.stdout:2172: done (2 left) 2023-10-09T09:14:55.470 INFO:tasks.rados.rados.0.smithi163.stdout:2173: done (1 left) 2023-10-09T09:14:55.470 INFO:tasks.rados.rados.0.smithi163.stdout:2174: done (0 left) 2023-10-09T09:14:55.470 INFO:tasks.rados.rados.0.smithi163.stdout:2175: write oid 29 current snap is 281 2023-10-09T09:14:55.470 INFO:tasks.rados.rados.0.smithi163.stdout:2175: seq_num 815 ranges {497705=577706,1642001=480939,2598092=497514} 2023-10-09T09:14:55.476 INFO:tasks.rados.rados.0.smithi163.stdout:2175: writing smithi16367185-29 from 497705 to 1075411 tid 1 2023-10-09T09:14:55.483 INFO:tasks.rados.rados.0.smithi163.stdout:2175: writing smithi16367185-29 from 1642001 to 2122940 tid 2 2023-10-09T09:14:55.490 INFO:tasks.rados.rados.0.smithi163.stdout:2175: writing smithi16367185-29 from 2598092 to 3095606 tid 3 2023-10-09T09:14:55.490 INFO:tasks.rados.rados.0.smithi163.stdout:2176: read oid 21 snap 246 2023-10-09T09:14:55.491 INFO:tasks.rados.rados.0.smithi163.stdout:2176: expect (ObjNum 707 snap 233 seq_num 707) 2023-10-09T09:14:55.491 INFO:tasks.rados.rados.0.smithi163.stdout:2177: rollback oid 46 current snap is 281 2023-10-09T09:14:55.491 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 46 to 272 2023-10-09T09:14:55.491 INFO:tasks.rados.rados.0.smithi163.stdout:2175: finishing write tid 1 to smithi16367185-29 2023-10-09T09:14:55.491 INFO:tasks.rados.rados.0.smithi163.stdout:2175: finishing write tid 2 to smithi16367185-29 2023-10-09T09:14:55.491 INFO:tasks.rados.rados.0.smithi163.stdout:2178: write oid 34 current snap is 281 2023-10-09T09:14:55.491 INFO:tasks.rados.rados.0.smithi163.stdout:2178: seq_num 816 ranges {514512=652955,1700483=740950,3025515=688997} 2023-10-09T09:14:55.498 INFO:tasks.rados.rados.0.smithi163.stdout:2178: writing smithi16367185-34 from 514512 to 1167467 tid 1 2023-10-09T09:14:55.507 INFO:tasks.rados.rados.0.smithi163.stdout:2178: writing smithi16367185-34 from 1700483 to 2441433 tid 2 2023-10-09T09:14:55.515 INFO:tasks.rados.rados.0.smithi163.stdout:2178: writing smithi16367185-34 from 3025515 to 3714512 tid 3 2023-10-09T09:14:55.515 INFO:tasks.rados.rados.0.smithi163.stdout:2179: delete oid 28 current snap is 281 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:2175: finishing write tid 3 to smithi16367185-29 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:2175: finishing write tid 4 to smithi16367185-29 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:2175: finishing write tid 5 to smithi16367185-29 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1027 (ObjNum 815 snap 281 seq_num 815) dirty exists 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:2175: left oid 29 (ObjNum 815 snap 281 seq_num 815) 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:2177: finishing rollback tid 0 to smithi16367185-46 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:2177: finishing rollback tid 1 to smithi16367185-46 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:2177: finishing rollback tid 2 to smithi16367185-46 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 939 (ObjNum 787 snap 271 seq_num 787) dirty exists 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:2178: finishing write tid 1 to smithi16367185-34 2023-10-09T09:14:55.516 INFO:tasks.rados.rados.0.smithi163.stdout:2176: expect (ObjNum 707 snap 233 seq_num 707) 2023-10-09T09:14:55.670 INFO:tasks.rados.rados.0.smithi163.stdout:2178: finishing write tid 2 to smithi16367185-34 2023-10-09T09:14:55.671 INFO:tasks.rados.rados.0.smithi163.stdout:2178: finishing write tid 3 to smithi16367185-34 2023-10-09T09:14:55.671 INFO:tasks.rados.rados.0.smithi163.stdout:2178: finishing write tid 4 to smithi16367185-34 2023-10-09T09:14:55.671 INFO:tasks.rados.rados.0.smithi163.stdout:2178: finishing write tid 5 to smithi16367185-34 2023-10-09T09:14:55.671 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 931 (ObjNum 816 snap 281 seq_num 816) dirty exists 2023-10-09T09:14:55.671 INFO:tasks.rados.rados.0.smithi163.stdout:2178: left oid 34 (ObjNum 816 snap 281 seq_num 816) 2023-10-09T09:14:55.671 INFO:tasks.rados.rados.0.smithi163.stdout:2175: done (4 left) 2023-10-09T09:14:55.671 INFO:tasks.rados.rados.0.smithi163.stdout:2176: done (3 left) 2023-10-09T09:14:55.671 INFO:tasks.rados.rados.0.smithi163.stdout:2177: done (2 left) 2023-10-09T09:14:55.671 INFO:tasks.rados.rados.0.smithi163.stdout:2178: done (1 left) 2023-10-09T09:14:55.672 INFO:tasks.rados.rados.0.smithi163.stdout:2179: done (0 left) 2023-10-09T09:14:55.672 INFO:tasks.rados.rados.0.smithi163.stdout:2180: snap_create 2023-10-09T09:14:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: cluster 2023-10-09T09:14:54.437558+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: cluster 2023-10-09T09:14:54.438370+0000 osd.3 (osd.3) 211 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: cluster 2023-10-09T09:14:54.441407+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: cluster 2023-10-09T09:14:54.442240+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: cluster 2023-10-09T09:14:54.442284+0000 mon.a (mon.0) 703 : cluster 0 osdmap e622: 8 total, 7 up, 8 in 2023-10-09T09:14:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: cluster 2023-10-09T09:14:54.442313+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: cluster 2023-10-09T09:14:54.442395+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: cluster 2023-10-09T09:14:54.442529+0000 osd.3 (osd.3) 212 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: cluster 2023-10-09T09:14:54.453582+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: audit 2023-10-09T09:14:55.350837+0000 mon.a (mon.0) 704 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: audit 2023-10-09T09:14:55.352047+0000 mon.a (mon.0) 705 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[78107]: audit 2023-10-09T09:14:55.427227+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.163:0/2130563191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:55.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: cluster 2023-10-09T09:14:54.437558+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: cluster 2023-10-09T09:14:54.438370+0000 osd.3 (osd.3) 211 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: cluster 2023-10-09T09:14:54.441407+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: cluster 2023-10-09T09:14:54.442240+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: cluster 2023-10-09T09:14:54.442284+0000 mon.a (mon.0) 703 : cluster 0 osdmap e622: 8 total, 7 up, 8 in 2023-10-09T09:14:55.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: cluster 2023-10-09T09:14:54.442313+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: cluster 2023-10-09T09:14:54.442395+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: cluster 2023-10-09T09:14:54.442529+0000 osd.3 (osd.3) 212 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: cluster 2023-10-09T09:14:54.453582+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: audit 2023-10-09T09:14:55.350837+0000 mon.a (mon.0) 704 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:55.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: audit 2023-10-09T09:14:55.352047+0000 mon.a (mon.0) 705 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:55.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:55 smithi163 bash[75425]: audit 2023-10-09T09:14:55.427227+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.163:0/2130563191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: cluster 2023-10-09T09:14:54.437558+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: cluster 2023-10-09T09:14:54.438370+0000 osd.3 (osd.3) 211 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: cluster 2023-10-09T09:14:54.441407+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: cluster 2023-10-09T09:14:54.442240+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: cluster 2023-10-09T09:14:54.442284+0000 mon.a (mon.0) 703 : cluster 0 osdmap e622: 8 total, 7 up, 8 in 2023-10-09T09:14:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: cluster 2023-10-09T09:14:54.442313+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: cluster 2023-10-09T09:14:54.442395+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: cluster 2023-10-09T09:14:54.442529+0000 osd.3 (osd.3) 212 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: cluster 2023-10-09T09:14:54.453582+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e622 with expected crc 2023-10-09T09:14:55.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: audit 2023-10-09T09:14:55.350837+0000 mon.a (mon.0) 704 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:55.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: audit 2023-10-09T09:14:55.352047+0000 mon.a (mon.0) 705 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:14:55.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:55 smithi181 bash[59345]: audit 2023-10-09T09:14:55.427227+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.163:0/2130563191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:56.487 INFO:tasks.rados.rados.0.smithi163.stdout:2180: done (0 left) 2023-10-09T09:14:56.487 INFO:tasks.rados.rados.0.smithi163.stdout:2181: read oid 40 snap -1 2023-10-09T09:14:56.488 INFO:tasks.rados.rados.0.smithi163.stdout:2181: expect (ObjNum 742 snap 252 seq_num 742) 2023-10-09T09:14:56.488 INFO:tasks.rados.rados.0.smithi163.stdout:2182: delete oid 33 current snap is 282 2023-10-09T09:14:56.494 INFO:tasks.rados.rados.0.smithi163.stdout:2182: done (1 left) 2023-10-09T09:14:56.494 INFO:tasks.rados.rados.0.smithi163.stdout:2183: write oid 32 current snap is 282 2023-10-09T09:14:56.494 INFO:tasks.rados.rados.0.smithi163.stdout:2183: seq_num 819 ranges {564933=478702,1759576=488277,2991431=658955} 2023-10-09T09:14:56.500 INFO:tasks.rados.rados.0.smithi163.stdout:2183: writing smithi16367185-32 from 564933 to 1043635 tid 1 2023-10-09T09:14:56.508 INFO:tasks.rados.rados.0.smithi163.stdout:2183: writing smithi16367185-32 from 1759576 to 2247853 tid 2 2023-10-09T09:14:56.516 INFO:tasks.rados.rados.0.smithi163.stdout:2183: writing smithi16367185-32 from 2991431 to 3650386 tid 3 2023-10-09T09:14:56.516 INFO:tasks.rados.rados.0.smithi163.stdout:2184: snap_remove snap 273 2023-10-09T09:14:56.516 INFO:tasks.rados.rados.0.smithi163.stdout:2181: expect (ObjNum 742 snap 252 seq_num 742) 2023-10-09T09:14:56.609 INFO:tasks.rados.rados.0.smithi163.stdout:2183: finishing write tid 1 to smithi16367185-32 2023-10-09T09:14:56.609 INFO:tasks.rados.rados.0.smithi163.stdout:2183: finishing write tid 2 to smithi16367185-32 2023-10-09T09:14:56.609 INFO:tasks.rados.rados.0.smithi163.stdout:2183: finishing write tid 3 to smithi16367185-32 2023-10-09T09:14:56.609 INFO:tasks.rados.rados.0.smithi163.stdout:2183: finishing write tid 4 to smithi16367185-32 2023-10-09T09:14:56.609 INFO:tasks.rados.rados.0.smithi163.stdout:2183: finishing write tid 5 to smithi16367185-32 2023-10-09T09:14:56.609 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1571 (ObjNum 819 snap 282 seq_num 819) dirty exists 2023-10-09T09:14:56.609 INFO:tasks.rados.rados.0.smithi163.stdout:2183: left oid 32 (ObjNum 819 snap 282 seq_num 819) 2023-10-09T09:14:56.718 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:56 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T09:14:56.718 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:56 smithi163 systemd[1]: Stopped Ceph osd.2 for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:14:56.718 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:56 smithi163 systemd[1]: Started Ceph osd.2 for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:14:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:56 smithi163 bash[75425]: cluster 2023-10-09T09:14:55.331368+0000 mgr.y (mgr.34107) 243 : cluster 0 pgmap v486: 89 pgs: 5 peering, 6 stale+active+clean, 78 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 24 op/s 2023-10-09T09:14:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:56 smithi163 bash[75425]: cluster 2023-10-09T09:14:55.467524+0000 osd.3 (osd.3) 213 : cluster 3 failed to encode map e623 with expected crc 2023-10-09T09:14:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:56 smithi163 bash[75425]: cluster 2023-10-09T09:14:55.468333+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e623 with expected crc 2023-10-09T09:14:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:56 smithi163 bash[75425]: cluster 2023-10-09T09:14:55.469783+0000 mon.a (mon.0) 706 : cluster 0 osdmap e623: 8 total, 7 up, 8 in 2023-10-09T09:14:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:56 smithi163 bash[75425]: cluster 2023-10-09T09:14:55.817860+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e623 with expected crc 2023-10-09T09:14:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:56 smithi163 bash[78107]: cluster 2023-10-09T09:14:55.331368+0000 mgr.y (mgr.34107) 243 : cluster 0 pgmap v486: 89 pgs: 5 peering, 6 stale+active+clean, 78 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 24 op/s 2023-10-09T09:14:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:56 smithi163 bash[78107]: cluster 2023-10-09T09:14:55.467524+0000 osd.3 (osd.3) 213 : cluster 3 failed to encode map e623 with expected crc 2023-10-09T09:14:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:56 smithi163 bash[78107]: cluster 2023-10-09T09:14:55.468333+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e623 with expected crc 2023-10-09T09:14:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:56 smithi163 bash[78107]: cluster 2023-10-09T09:14:55.469783+0000 mon.a (mon.0) 706 : cluster 0 osdmap e623: 8 total, 7 up, 8 in 2023-10-09T09:14:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:56 smithi163 bash[78107]: cluster 2023-10-09T09:14:55.817860+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e623 with expected crc 2023-10-09T09:14:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:56 smithi181 bash[59345]: cluster 2023-10-09T09:14:55.331368+0000 mgr.y (mgr.34107) 243 : cluster 0 pgmap v486: 89 pgs: 5 peering, 6 stale+active+clean, 78 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 24 op/s 2023-10-09T09:14:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:56 smithi181 bash[59345]: cluster 2023-10-09T09:14:55.467524+0000 osd.3 (osd.3) 213 : cluster 3 failed to encode map e623 with expected crc 2023-10-09T09:14:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:56 smithi181 bash[59345]: cluster 2023-10-09T09:14:55.468333+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e623 with expected crc 2023-10-09T09:14:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:56 smithi181 bash[59345]: cluster 2023-10-09T09:14:55.469783+0000 mon.a (mon.0) 706 : cluster 0 osdmap e623: 8 total, 7 up, 8 in 2023-10-09T09:14:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:56 smithi181 bash[59345]: cluster 2023-10-09T09:14:55.817860+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e623 with expected crc 2023-10-09T09:14:57.491 INFO:tasks.rados.rados.0.smithi163.stdout:2181: done (2 left) 2023-10-09T09:14:57.491 INFO:tasks.rados.rados.0.smithi163.stdout:2183: done (1 left) 2023-10-09T09:14:57.491 INFO:tasks.rados.rados.0.smithi163.stdout:2184: done (0 left) 2023-10-09T09:14:57.491 INFO:tasks.rados.rados.0.smithi163.stdout:2185: read oid 33 snap 279 2023-10-09T09:14:57.491 INFO:tasks.rados.rados.0.smithi163.stdout:2185: expect deleted 2023-10-09T09:14:57.491 INFO:tasks.rados.rados.0.smithi163.stdout:2186: write oid 14 current snap is 282 2023-10-09T09:14:57.491 INFO:tasks.rados.rados.0.smithi163.stdout:2186: seq_num 820 ranges {581740=553951,1818058=748288,3018854=470227} 2023-10-09T09:14:57.497 INFO:tasks.rados.rados.0.smithi163.stdout:2186: writing smithi16367185-14 from 581740 to 1135691 tid 1 2023-10-09T09:14:57.507 INFO:tasks.rados.rados.0.smithi163.stdout:2186: writing smithi16367185-14 from 1818058 to 2566346 tid 2 2023-10-09T09:14:57.514 INFO:tasks.rados.rados.0.smithi163.stdout:2186: writing smithi16367185-14 from 3018854 to 3489081 tid 3 2023-10-09T09:14:57.514 INFO:tasks.rados.rados.0.smithi163.stdout:2186: finishing write tid 1 to smithi16367185-14 2023-10-09T09:14:57.514 INFO:tasks.rados.rados.0.smithi163.stdout:2186: finishing write tid 2 to smithi16367185-14 2023-10-09T09:14:57.514 INFO:tasks.rados.rados.0.smithi163.stdout:2185: done (1 left) 2023-10-09T09:14:57.514 INFO:tasks.rados.rados.0.smithi163.stdout:2187: read oid 10 snap -1 2023-10-09T09:14:57.514 INFO:tasks.rados.rados.0.smithi163.stdout:2187: expect (ObjNum 773 snap 267 seq_num 773) 2023-10-09T09:14:57.514 INFO:tasks.rados.rados.0.smithi163.stdout:2188: write oid 44 current snap is 282 2023-10-09T09:14:57.514 INFO:tasks.rados.rados.0.smithi163.stdout:2188: seq_num 821 ranges {598547=629200,1876540=491946,3046277=681499} 2023-10-09T09:14:57.519 INFO:tasks.rados.rados.0.smithi163.stdout:2188: writing smithi16367185-44 from 598547 to 1227747 tid 1 2023-10-09T09:14:57.526 INFO:tasks.rados.rados.0.smithi163.stdout:2188: writing smithi16367185-44 from 1876540 to 2368486 tid 2 2023-10-09T09:14:57.534 INFO:tasks.rados.rados.0.smithi163.stdout:2188: writing smithi16367185-44 from 3046277 to 3727776 tid 3 2023-10-09T09:14:57.535 INFO:tasks.rados.rados.0.smithi163.stdout:2189: delete oid 42 current snap is 282 2023-10-09T09:14:57.535 INFO:tasks.rados.rados.0.smithi163.stdout:2186: finishing write tid 3 to smithi16367185-14 2023-10-09T09:14:57.535 INFO:tasks.rados.rados.0.smithi163.stdout:2186: finishing write tid 4 to smithi16367185-14 2023-10-09T09:14:57.535 INFO:tasks.rados.rados.0.smithi163.stdout:2186: finishing write tid 5 to smithi16367185-14 2023-10-09T09:14:57.535 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 948 (ObjNum 820 snap 282 seq_num 820) dirty exists 2023-10-09T09:14:57.535 INFO:tasks.rados.rados.0.smithi163.stdout:2186: left oid 14 (ObjNum 820 snap 282 seq_num 820) 2023-10-09T09:14:57.536 INFO:tasks.rados.rados.0.smithi163.stdout:2187: expect (ObjNum 773 snap 267 seq_num 773) 2023-10-09T09:14:57.643 INFO:tasks.rados.rados.0.smithi163.stdout:2188: finishing write tid 1 to smithi16367185-44 2023-10-09T09:14:57.643 INFO:tasks.rados.rados.0.smithi163.stdout:2188: finishing write tid 2 to smithi16367185-44 2023-10-09T09:14:57.643 INFO:tasks.rados.rados.0.smithi163.stdout:2188: finishing write tid 3 to smithi16367185-44 2023-10-09T09:14:57.643 INFO:tasks.rados.rados.0.smithi163.stdout:2188: finishing write tid 4 to smithi16367185-44 2023-10-09T09:14:57.643 INFO:tasks.rados.rados.0.smithi163.stdout:2188: finishing write tid 5 to smithi16367185-44 2023-10-09T09:14:57.644 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1579 (ObjNum 821 snap 282 seq_num 821) dirty exists 2023-10-09T09:14:57.644 INFO:tasks.rados.rados.0.smithi163.stdout:2188: left oid 44 (ObjNum 821 snap 282 seq_num 821) 2023-10-09T09:14:57.644 INFO:tasks.rados.rados.0.smithi163.stdout:2186: done (3 left) 2023-10-09T09:14:57.644 INFO:tasks.rados.rados.0.smithi163.stdout:2187: done (2 left) 2023-10-09T09:14:57.644 INFO:tasks.rados.rados.0.smithi163.stdout:2188: done (1 left) 2023-10-09T09:14:57.644 INFO:tasks.rados.rados.0.smithi163.stdout:2189: done (0 left) 2023-10-09T09:14:57.644 INFO:tasks.rados.rados.0.smithi163.stdout:2190: snap_remove snap 239 2023-10-09T09:14:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: cluster 2023-10-09T09:14:56.477210+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: cluster 2023-10-09T09:14:56.481481+0000 mon.a (mon.0) 707 : cluster 0 osdmap e624: 8 total, 7 up, 8 in 2023-10-09T09:14:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: cluster 2023-10-09T09:14:56.489133+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: cluster 2023-10-09T09:14:56.502042+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: cluster 2023-10-09T09:14:56.700008+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: audit 2023-10-09T09:14:56.723238+0000 mon.a (mon.0) 708 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: audit 2023-10-09T09:14:56.733695+0000 mon.a (mon.0) 709 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: audit 2023-10-09T09:14:56.738682+0000 mon.a (mon.0) 710 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:57.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: audit 2023-10-09T09:14:57.114119+0000 mon.a (mon.0) 711 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: audit 2023-10-09T09:14:57.120232+0000 mon.a (mon.0) 712 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: audit 2023-10-09T09:14:57.126656+0000 mon.a (mon.0) 713 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:57 smithi181 bash[59345]: audit 2023-10-09T09:14:57.134038+0000 mon.a (mon.0) 714 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: cluster 2023-10-09T09:14:56.477210+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: cluster 2023-10-09T09:14:56.481481+0000 mon.a (mon.0) 707 : cluster 0 osdmap e624: 8 total, 7 up, 8 in 2023-10-09T09:14:57.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: cluster 2023-10-09T09:14:56.489133+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: cluster 2023-10-09T09:14:56.502042+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: cluster 2023-10-09T09:14:56.700008+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: audit 2023-10-09T09:14:56.723238+0000 mon.a (mon.0) 708 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: audit 2023-10-09T09:14:56.733695+0000 mon.a (mon.0) 709 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: audit 2023-10-09T09:14:56.738682+0000 mon.a (mon.0) 710 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: audit 2023-10-09T09:14:57.114119+0000 mon.a (mon.0) 711 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: audit 2023-10-09T09:14:57.120232+0000 mon.a (mon.0) 712 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: audit 2023-10-09T09:14:57.126656+0000 mon.a (mon.0) 713 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[75425]: audit 2023-10-09T09:14:57.134038+0000 mon.a (mon.0) 714 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: cluster 2023-10-09T09:14:56.477210+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: cluster 2023-10-09T09:14:56.481481+0000 mon.a (mon.0) 707 : cluster 0 osdmap e624: 8 total, 7 up, 8 in 2023-10-09T09:14:57.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: cluster 2023-10-09T09:14:56.489133+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: cluster 2023-10-09T09:14:56.502042+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: cluster 2023-10-09T09:14:56.700008+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:57.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: audit 2023-10-09T09:14:56.723238+0000 mon.a (mon.0) 708 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: audit 2023-10-09T09:14:56.733695+0000 mon.a (mon.0) 709 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: audit 2023-10-09T09:14:56.738682+0000 mon.a (mon.0) 710 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:14:57.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: audit 2023-10-09T09:14:57.114119+0000 mon.a (mon.0) 711 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: audit 2023-10-09T09:14:57.120232+0000 mon.a (mon.0) 712 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: audit 2023-10-09T09:14:57.126656+0000 mon.a (mon.0) 713 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:57.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:57 smithi163 bash[78107]: audit 2023-10-09T09:14:57.134038+0000 mon.a (mon.0) 714 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:14:58.468 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[89321]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T09:14:58.468 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[89321]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T09:14:58.469 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[89321]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T09:14:58.469 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[89321]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T09:14:58.469 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[89321]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T09:14:58.469 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[89321]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T09:14:58.469 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[89321]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T09:14:58.470 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[89321]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T09:14:58.491 INFO:tasks.rados.rados.0.smithi163.stdout:2190: done (0 left) 2023-10-09T09:14:58.491 INFO:tasks.rados.rados.0.smithi163.stdout:2191: write oid 25 current snap is 282 2023-10-09T09:14:58.492 INFO:tasks.rados.rados.0.smithi163.stdout:2191: seq_num 823 ranges {632161=496051,1593504=495615,2701123=420396,3611645=220516} 2023-10-09T09:14:58.497 INFO:tasks.rados.rados.0.smithi163.stdout:2191: writing smithi16367185-25 from 632161 to 1128212 tid 1 2023-10-09T09:14:58.503 INFO:tasks.rados.rados.0.smithi163.stdout:2191: writing smithi16367185-25 from 1593504 to 2089119 tid 2 2023-10-09T09:14:58.509 INFO:tasks.rados.rados.0.smithi163.stdout:2191: writing smithi16367185-25 from 2701123 to 3121519 tid 3 2023-10-09T09:14:58.514 INFO:tasks.rados.rados.0.smithi163.stdout:2191: writing smithi16367185-25 from 3611645 to 3832161 tid 4 2023-10-09T09:14:58.514 INFO:tasks.rados.rados.0.smithi163.stdout:2192: read oid 33 snap -1 2023-10-09T09:14:58.515 INFO:tasks.rados.rados.0.smithi163.stdout:2192: expect deleted 2023-10-09T09:14:58.515 INFO:tasks.rados.rados.0.smithi163.stdout:2193: write oid 50 current snap is 282 2023-10-09T09:14:58.515 INFO:tasks.rados.rados.0.smithi163.stdout:2191: finishing write tid 1 to smithi16367185-25 2023-10-09T09:14:58.515 INFO:tasks.rados.rados.0.smithi163.stdout:2191: finishing write tid 2 to smithi16367185-25 2023-10-09T09:14:58.515 INFO:tasks.rados.rados.0.smithi163.stdout:2191: finishing write tid 3 to smithi16367185-25 2023-10-09T09:14:58.516 INFO:tasks.rados.rados.0.smithi163.stdout:2193: seq_num 824 ranges {648968=571300,1651986=639273,2728546=631668} 2023-10-09T09:14:58.521 INFO:tasks.rados.rados.0.smithi163.stdout:2193: writing smithi16367185-50 from 648968 to 1220268 tid 1 2023-10-09T09:14:58.527 INFO:tasks.rados.rados.0.smithi163.stdout:2193: writing smithi16367185-50 from 1651986 to 2291259 tid 2 2023-10-09T09:14:58.534 INFO:tasks.rados.rados.0.smithi163.stdout:2193: writing smithi16367185-50 from 2728546 to 3360214 tid 3 2023-10-09T09:14:58.534 INFO:tasks.rados.rados.0.smithi163.stdout:2194: delete oid 42 current snap is 282 2023-10-09T09:14:58.534 INFO:tasks.rados.rados.0.smithi163.stdout:2191: finishing write tid 4 to smithi16367185-25 2023-10-09T09:14:58.537 INFO:tasks.rados.rados.0.smithi163.stdout:2191: finishing write tid 5 to smithi16367185-25 2023-10-09T09:14:58.537 INFO:tasks.rados.rados.0.smithi163.stdout:2191: finishing write tid 6 to smithi16367185-25 2023-10-09T09:14:58.537 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1035 (ObjNum 823 snap 282 seq_num 823) dirty exists 2023-10-09T09:14:58.538 INFO:tasks.rados.rados.0.smithi163.stdout:2191: left oid 25 (ObjNum 823 snap 282 seq_num 823) 2023-10-09T09:14:58.538 INFO:tasks.rados.rados.0.smithi163.stdout:2193: finishing write tid 1 to smithi16367185-50 2023-10-09T09:14:58.538 INFO:tasks.rados.rados.0.smithi163.stdout:2193: finishing write tid 2 to smithi16367185-50 2023-10-09T09:14:58.539 INFO:tasks.rados.rados.0.smithi163.stdout:2193: finishing write tid 3 to smithi16367185-50 2023-10-09T09:14:58.540 INFO:tasks.rados.rados.0.smithi163.stdout:2193: finishing write tid 4 to smithi16367185-50 2023-10-09T09:14:58.540 INFO:tasks.rados.rados.0.smithi163.stdout:2193: finishing write tid 5 to smithi16367185-50 2023-10-09T09:14:58.540 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 948 (ObjNum 824 snap 282 seq_num 824) dirty exists 2023-10-09T09:14:58.540 INFO:tasks.rados.rados.0.smithi163.stdout:2193: left oid 50 (ObjNum 824 snap 282 seq_num 824) 2023-10-09T09:14:58.540 INFO:tasks.rados.rados.0.smithi163.stdout:2191: done (3 left) 2023-10-09T09:14:58.540 INFO:tasks.rados.rados.0.smithi163.stdout:2192: done (2 left) 2023-10-09T09:14:58.540 INFO:tasks.rados.rados.0.smithi163.stdout:2193: done (1 left) 2023-10-09T09:14:58.540 INFO:tasks.rados.rados.0.smithi163.stdout:2194: done (0 left) 2023-10-09T09:14:58.540 INFO:tasks.rados.rados.0.smithi163.stdout:2195: write oid 12 current snap is 282 2023-10-09T09:14:58.541 INFO:tasks.rados.rados.0.smithi163.stdout:2195: seq_num 826 ranges {682582=721798,2168950=642942,3583392=299190} 2023-10-09T09:14:58.547 INFO:tasks.rados.rados.0.smithi163.stdout:2195: writing smithi16367185-12 from 682582 to 1404380 tid 1 2023-10-09T09:14:58.555 INFO:tasks.rados.rados.0.smithi163.stdout:2195: writing smithi16367185-12 from 2168950 to 2811892 tid 2 2023-10-09T09:14:58.562 INFO:tasks.rados.rados.0.smithi163.stdout:2195: writing smithi16367185-12 from 3583392 to 3882582 tid 3 2023-10-09T09:14:58.562 INFO:tasks.rados.rados.0.smithi163.stdout:2196: rollback oid 38 current snap is 282 2023-10-09T09:14:58.562 INFO:tasks.rados.rados.0.smithi163.stdout:2195: finishing write tid 1 to smithi16367185-12 2023-10-09T09:14:58.562 INFO:tasks.rados.rados.0.smithi163.stdout:2195: finishing write tid 2 to smithi16367185-12 2023-10-09T09:14:58.562 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 281 2023-10-09T09:14:58.562 INFO:tasks.rados.rados.0.smithi163.stdout:2197: read oid 34 snap 277 2023-10-09T09:14:58.563 INFO:tasks.rados.rados.0.smithi163.stdout:2197: expect deleted 2023-10-09T09:14:58.563 INFO:tasks.rados.rados.0.smithi163.stdout:2198: snap_create 2023-10-09T09:14:58.565 INFO:tasks.rados.rados.0.smithi163.stdout:2195: finishing write tid 3 to smithi16367185-12 2023-10-09T09:14:58.565 INFO:tasks.rados.rados.0.smithi163.stdout:2196: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:14:58.565 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 849 (ObjNum 1073749520 snap 32603 seq_num 1073744080) dirty dne 2023-10-09T09:14:58.566 INFO:tasks.rados.rados.0.smithi163.stdout:2195: finishing write tid 4 to smithi16367185-12 2023-10-09T09:14:58.566 INFO:tasks.rados.rados.0.smithi163.stdout:2195: finishing write tid 5 to smithi16367185-12 2023-10-09T09:14:58.566 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 1040 (ObjNum 826 snap 282 seq_num 826) dirty exists 2023-10-09T09:14:58.566 INFO:tasks.rados.rados.0.smithi163.stdout:2195: left oid 12 (ObjNum 826 snap 282 seq_num 826) 2023-10-09T09:14:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[75425]: cluster 2023-10-09T09:14:56.478260+0000 osd.3 (osd.3) 214 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[75425]: cluster 2023-10-09T09:14:57.332242+0000 mgr.y (mgr.34107) 244 : cluster 0 pgmap v489: 89 pgs: 8 active+undersized+degraded, 5 peering, 12 active+undersized, 64 active+clean; 247 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.7 MiB/s wr, 8 op/s; 53/453 objects degraded (11.700%) 2023-10-09T09:14:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[75425]: cluster 2023-10-09T09:14:57.487350+0000 osd.3 (osd.3) 215 : cluster 3 failed to encode map e625 with expected crc 2023-10-09T09:14:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[75425]: cluster 2023-10-09T09:14:57.490024+0000 mon.a (mon.0) 715 : cluster 0 osdmap e625: 8 total, 7 up, 8 in 2023-10-09T09:14:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[75425]: audit 2023-10-09T09:14:57.798935+0000 mon.a (mon.0) 716 : audit 0 from='client.? 172.21.15.163:0/291740669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[75425]: cluster 2023-10-09T09:14:58.131490+0000 mon.a (mon.0) 717 : cluster 3 Health check failed: Degraded data redundancy: 53/453 objects degraded (11.700%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[78107]: cluster 2023-10-09T09:14:56.478260+0000 osd.3 (osd.3) 214 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[78107]: cluster 2023-10-09T09:14:57.332242+0000 mgr.y (mgr.34107) 244 : cluster 0 pgmap v489: 89 pgs: 8 active+undersized+degraded, 5 peering, 12 active+undersized, 64 active+clean; 247 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.7 MiB/s wr, 8 op/s; 53/453 objects degraded (11.700%) 2023-10-09T09:14:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[78107]: cluster 2023-10-09T09:14:57.487350+0000 osd.3 (osd.3) 215 : cluster 3 failed to encode map e625 with expected crc 2023-10-09T09:14:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[78107]: cluster 2023-10-09T09:14:57.490024+0000 mon.a (mon.0) 715 : cluster 0 osdmap e625: 8 total, 7 up, 8 in 2023-10-09T09:14:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[78107]: audit 2023-10-09T09:14:57.798935+0000 mon.a (mon.0) 716 : audit 0 from='client.? 172.21.15.163:0/291740669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:58 smithi163 bash[78107]: cluster 2023-10-09T09:14:58.131490+0000 mon.a (mon.0) 717 : cluster 3 Health check failed: Degraded data redundancy: 53/453 objects degraded (11.700%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:58 smithi181 bash[59345]: cluster 2023-10-09T09:14:56.478260+0000 osd.3 (osd.3) 214 : cluster 3 failed to encode map e624 with expected crc 2023-10-09T09:14:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:58 smithi181 bash[59345]: cluster 2023-10-09T09:14:57.332242+0000 mgr.y (mgr.34107) 244 : cluster 0 pgmap v489: 89 pgs: 8 active+undersized+degraded, 5 peering, 12 active+undersized, 64 active+clean; 247 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.7 MiB/s wr, 8 op/s; 53/453 objects degraded (11.700%) 2023-10-09T09:14:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:58 smithi181 bash[59345]: cluster 2023-10-09T09:14:57.487350+0000 osd.3 (osd.3) 215 : cluster 3 failed to encode map e625 with expected crc 2023-10-09T09:14:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:58 smithi181 bash[59345]: cluster 2023-10-09T09:14:57.490024+0000 mon.a (mon.0) 715 : cluster 0 osdmap e625: 8 total, 7 up, 8 in 2023-10-09T09:14:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:58 smithi181 bash[59345]: audit 2023-10-09T09:14:57.798935+0000 mon.a (mon.0) 716 : audit 0 from='client.? 172.21.15.163:0/291740669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:14:58.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:58 smithi181 bash[59345]: cluster 2023-10-09T09:14:58.131490+0000 mon.a (mon.0) 717 : cluster 3 Health check failed: Degraded data redundancy: 53/453 objects degraded (11.700%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T09:14:59.503 INFO:tasks.rados.rados.0.smithi163.stdout:2195: done (3 left) 2023-10-09T09:14:59.503 INFO:tasks.rados.rados.0.smithi163.stdout:2196: done (2 left) 2023-10-09T09:14:59.503 INFO:tasks.rados.rados.0.smithi163.stdout:2197: done (1 left) 2023-10-09T09:14:59.503 INFO:tasks.rados.rados.0.smithi163.stdout:2198: done (0 left) 2023-10-09T09:14:59.503 INFO:tasks.rados.rados.0.smithi163.stdout:2199: rollback oid 48 current snap is 283 2023-10-09T09:14:59.503 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 48 to 280 2023-10-09T09:14:59.504 INFO:tasks.rados.rados.0.smithi163.stdout:2200: snap_remove snap 274 2023-10-09T09:14:59.772 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:59 smithi163 bash[75425]: cluster 2023-10-09T09:14:58.487513+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e626 with expected crc 2023-10-09T09:14:59.772 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:59 smithi163 bash[75425]: cluster 2023-10-09T09:14:58.487983+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e626 with expected crc 2023-10-09T09:14:59.772 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:14:59 smithi163 bash[75425]: cluster 2023-10-09T09:14:58.490686+0000 mon.a (mon.0) 718 : cluster 0 osdmap e626: 8 total, 7 up, 8 in 2023-10-09T09:14:59.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:59 smithi163 bash[78107]: cluster 2023-10-09T09:14:58.487513+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e626 with expected crc 2023-10-09T09:14:59.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:59 smithi163 bash[78107]: cluster 2023-10-09T09:14:58.487983+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e626 with expected crc 2023-10-09T09:14:59.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:14:59 smithi163 bash[78107]: cluster 2023-10-09T09:14:58.490686+0000 mon.a (mon.0) 718 : cluster 0 osdmap e626: 8 total, 7 up, 8 in 2023-10-09T09:14:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:59 smithi181 bash[59345]: cluster 2023-10-09T09:14:58.487513+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e626 with expected crc 2023-10-09T09:14:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:59 smithi181 bash[59345]: cluster 2023-10-09T09:14:58.487983+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e626 with expected crc 2023-10-09T09:14:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:14:59 smithi181 bash[59345]: cluster 2023-10-09T09:14:58.490686+0000 mon.a (mon.0) 718 : cluster 0 osdmap e626: 8 total, 7 up, 8 in 2023-10-09T09:15:00.110 INFO:tasks.rados.rados.0.smithi163.stdout:2200: done (1 left) 2023-10-09T09:15:00.110 INFO:tasks.rados.rados.0.smithi163.stdout:2201: delete oid 33 current snap is 283 2023-10-09T09:15:00.110 INFO:tasks.rados.rados.0.smithi163.stdout:2199: finishing rollback tid 0 to smithi16367185-48 2023-10-09T09:15:00.110 INFO:tasks.rados.rados.0.smithi163.stdout:2199: finishing rollback tid 1 to smithi16367185-48 2023-10-09T09:15:00.110 INFO:tasks.rados.rados.0.smithi163.stdout:2199: finishing rollback tid 2 to smithi16367185-48 2023-10-09T09:15:00.110 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 1594 (ObjNum 751 snap 255 seq_num 751) dirty exists 2023-10-09T09:15:00.117 INFO:tasks.rados.rados.0.smithi163.stdout:2199: done (1 left) 2023-10-09T09:15:00.118 INFO:tasks.rados.rados.0.smithi163.stdout:2201: done (0 left) 2023-10-09T09:15:00.118 INFO:tasks.rados.rados.0.smithi163.stdout:2202: read oid 35 snap -1 2023-10-09T09:15:00.118 INFO:tasks.rados.rados.0.smithi163.stdout:2202: expect deleted 2023-10-09T09:15:00.118 INFO:tasks.rados.rados.0.smithi163.stdout:2203: read oid 30 snap -1 2023-10-09T09:15:00.118 INFO:tasks.rados.rados.0.smithi163.stdout:2203: expect deleted 2023-10-09T09:15:00.118 INFO:tasks.rados.rados.0.smithi163.stdout:2204: snap_create 2023-10-09T09:15:00.218 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:14:59 smithi163 bash[89638]: debug 2023-10-09T09:14:59.835+0000 7f4fac978700 1 -- 172.21.15.163:0/2774750827 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x565421bfa4e0 con 0x565421c55000 2023-10-09T09:15:00.718 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[89638]: debug 2023-10-09T09:15:00.379+0000 7f4fb5afe640 -1 Falling back to public interface 2023-10-09T09:15:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: cluster 2023-10-09T09:14:59.333204+0000 mgr.y (mgr.34107) 245 : cluster 0 pgmap v492: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 247 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.7 MiB/s wr, 10 op/s; 53/453 objects degraded (11.700%) 2023-10-09T09:15:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: cluster 2023-10-09T09:14:59.497903+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e627 with expected crc 2023-10-09T09:15:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: cluster 2023-10-09T09:14:59.498606+0000 osd.3 (osd.3) 216 : cluster 3 failed to encode map e627 with expected crc 2023-10-09T09:15:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: cluster 2023-10-09T09:14:59.503037+0000 mon.a (mon.0) 719 : cluster 0 osdmap e627: 8 total, 7 up, 8 in 2023-10-09T09:15:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: cluster 2023-10-09T09:14:59.503826+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e627 with expected crc 2023-10-09T09:15:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: cluster 2023-10-09T09:15:00.104631+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: cluster 2023-10-09T09:15:00.106980+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: cluster 2023-10-09T09:15:00.108585+0000 mon.a (mon.0) 720 : cluster 0 osdmap e628: 8 total, 7 up, 8 in 2023-10-09T09:15:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: cluster 2023-10-09T09:15:00.118680+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: audit 2023-10-09T09:15:00.164711+0000 mon.a (mon.0) 721 : audit 0 from='client.? 172.21.15.163:0/4084506522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[75425]: cluster 2023-10-09T09:15:00.375297+0000 osd.3 (osd.3) 217 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: cluster 2023-10-09T09:14:59.333204+0000 mgr.y (mgr.34107) 245 : cluster 0 pgmap v492: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 247 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.7 MiB/s wr, 10 op/s; 53/453 objects degraded (11.700%) 2023-10-09T09:15:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: cluster 2023-10-09T09:14:59.497903+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e627 with expected crc 2023-10-09T09:15:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: cluster 2023-10-09T09:14:59.498606+0000 osd.3 (osd.3) 216 : cluster 3 failed to encode map e627 with expected crc 2023-10-09T09:15:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: cluster 2023-10-09T09:14:59.503037+0000 mon.a (mon.0) 719 : cluster 0 osdmap e627: 8 total, 7 up, 8 in 2023-10-09T09:15:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: cluster 2023-10-09T09:14:59.503826+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e627 with expected crc 2023-10-09T09:15:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: cluster 2023-10-09T09:15:00.104631+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: cluster 2023-10-09T09:15:00.106980+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: cluster 2023-10-09T09:15:00.108585+0000 mon.a (mon.0) 720 : cluster 0 osdmap e628: 8 total, 7 up, 8 in 2023-10-09T09:15:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: cluster 2023-10-09T09:15:00.118680+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: audit 2023-10-09T09:15:00.164711+0000 mon.a (mon.0) 721 : audit 0 from='client.? 172.21.15.163:0/4084506522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:00 smithi163 bash[78107]: cluster 2023-10-09T09:15:00.375297+0000 osd.3 (osd.3) 217 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: cluster 2023-10-09T09:14:59.333204+0000 mgr.y (mgr.34107) 245 : cluster 0 pgmap v492: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 247 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.7 MiB/s wr, 10 op/s; 53/453 objects degraded (11.700%) 2023-10-09T09:15:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: cluster 2023-10-09T09:14:59.497903+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e627 with expected crc 2023-10-09T09:15:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: cluster 2023-10-09T09:14:59.498606+0000 osd.3 (osd.3) 216 : cluster 3 failed to encode map e627 with expected crc 2023-10-09T09:15:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: cluster 2023-10-09T09:14:59.503037+0000 mon.a (mon.0) 719 : cluster 0 osdmap e627: 8 total, 7 up, 8 in 2023-10-09T09:15:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: cluster 2023-10-09T09:14:59.503826+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e627 with expected crc 2023-10-09T09:15:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: cluster 2023-10-09T09:15:00.104631+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: cluster 2023-10-09T09:15:00.106980+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: cluster 2023-10-09T09:15:00.108585+0000 mon.a (mon.0) 720 : cluster 0 osdmap e628: 8 total, 7 up, 8 in 2023-10-09T09:15:00.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: cluster 2023-10-09T09:15:00.118680+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:00.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: audit 2023-10-09T09:15:00.164711+0000 mon.a (mon.0) 721 : audit 0 from='client.? 172.21.15.163:0/4084506522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:00.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:00 smithi181 bash[59345]: cluster 2023-10-09T09:15:00.375297+0000 osd.3 (osd.3) 217 : cluster 3 failed to encode map e628 with expected crc 2023-10-09T09:15:01.118 INFO:tasks.rados.rados.0.smithi163.stdout:2202: done (2 left) 2023-10-09T09:15:01.118 INFO:tasks.rados.rados.0.smithi163.stdout:2203: done (1 left) 2023-10-09T09:15:01.118 INFO:tasks.rados.rados.0.smithi163.stdout:2204: done (0 left) 2023-10-09T09:15:01.119 INFO:tasks.rados.rados.0.smithi163.stdout:2205: snap_create 2023-10-09T09:15:02.131 INFO:tasks.rados.rados.0.smithi163.stdout:2205: done (0 left) 2023-10-09T09:15:02.132 INFO:tasks.rados.rados.0.smithi163.stdout:2206: rollback oid 14 current snap is 285 2023-10-09T09:15:02.132 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 282 2023-10-09T09:15:02.132 INFO:tasks.rados.rados.0.smithi163.stdout:2207: snap_create 2023-10-09T09:15:02.140 INFO:tasks.rados.rados.0.smithi163.stdout:2206: finishing rollback tid 0 to smithi16367185-14 2023-10-09T09:15:02.141 INFO:tasks.rados.rados.0.smithi163.stdout:2206: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:15:02.141 INFO:tasks.rados.rados.0.smithi163.stdout:2206: finishing rollback tid 2 to smithi16367185-14 2023-10-09T09:15:02.141 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 967 (ObjNum 820 snap 282 seq_num 820) dirty exists 2023-10-09T09:15:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:02 smithi163 bash[78107]: cluster 2023-10-09T09:15:01.114398+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:02 smithi163 bash[78107]: cluster 2023-10-09T09:15:01.115165+0000 osd.3 (osd.3) 218 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:02 smithi163 bash[78107]: cluster 2023-10-09T09:15:01.118410+0000 mon.a (mon.0) 722 : cluster 0 osdmap e629: 8 total, 7 up, 8 in 2023-10-09T09:15:02.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:02 smithi163 bash[78107]: cluster 2023-10-09T09:15:01.334548+0000 mgr.y (mgr.34107) 246 : cluster 0 pgmap v496: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 239 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.7 MiB/s wr, 11 op/s; 53/432 objects degraded (12.269%) 2023-10-09T09:15:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:02 smithi163 bash[75425]: cluster 2023-10-09T09:15:01.114398+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:02 smithi163 bash[75425]: cluster 2023-10-09T09:15:01.115165+0000 osd.3 (osd.3) 218 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:02 smithi163 bash[75425]: cluster 2023-10-09T09:15:01.118410+0000 mon.a (mon.0) 722 : cluster 0 osdmap e629: 8 total, 7 up, 8 in 2023-10-09T09:15:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:02 smithi163 bash[75425]: cluster 2023-10-09T09:15:01.334548+0000 mgr.y (mgr.34107) 246 : cluster 0 pgmap v496: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 239 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.7 MiB/s wr, 11 op/s; 53/432 objects degraded (12.269%) 2023-10-09T09:15:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:02 smithi181 bash[59345]: cluster 2023-10-09T09:15:01.114398+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:02 smithi181 bash[59345]: cluster 2023-10-09T09:15:01.115165+0000 osd.3 (osd.3) 218 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:02 smithi181 bash[59345]: cluster 2023-10-09T09:15:01.118410+0000 mon.a (mon.0) 722 : cluster 0 osdmap e629: 8 total, 7 up, 8 in 2023-10-09T09:15:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:02 smithi181 bash[59345]: cluster 2023-10-09T09:15:01.334548+0000 mgr.y (mgr.34107) 246 : cluster 0 pgmap v496: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 239 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.7 MiB/s wr, 11 op/s; 53/432 objects degraded (12.269%) 2023-10-09T09:15:03.131 INFO:tasks.rados.rados.0.smithi163.stdout:2206: done (1 left) 2023-10-09T09:15:03.131 INFO:tasks.rados.rados.0.smithi163.stdout:2207: done (0 left) 2023-10-09T09:15:03.131 INFO:tasks.rados.rados.0.smithi163.stdout:2208: rollback oid 41 current snap is 286 2023-10-09T09:15:03.131 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 41 to 281 2023-10-09T09:15:03.131 INFO:tasks.rados.rados.0.smithi163.stdout:2209: snap_remove snap 276 2023-10-09T09:15:03.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:15:03] "GET /metrics HTTP/1.1" 200 36202 "" "Prometheus/2.33.4" 2023-10-09T09:15:03.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[78107]: cluster 2023-10-09T09:15:01.670345+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:03.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[78107]: cluster 2023-10-09T09:15:02.124405+0000 osd.3 (osd.3) 219 : cluster 3 failed to encode map e630 with expected crc 2023-10-09T09:15:03.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[78107]: cluster 2023-10-09T09:15:02.125886+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e630 with expected crc 2023-10-09T09:15:03.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[78107]: cluster 2023-10-09T09:15:02.131370+0000 mon.a (mon.0) 723 : cluster 0 osdmap e630: 8 total, 7 up, 8 in 2023-10-09T09:15:03.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[78107]: cluster 2023-10-09T09:15:02.197542+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:03.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[78107]: audit 2023-10-09T09:15:02.547082+0000 mon.a (mon.0) 724 : audit 0 from='client.? 172.21.15.163:0/2770486556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[75425]: cluster 2023-10-09T09:15:01.670345+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[75425]: cluster 2023-10-09T09:15:02.124405+0000 osd.3 (osd.3) 219 : cluster 3 failed to encode map e630 with expected crc 2023-10-09T09:15:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[75425]: cluster 2023-10-09T09:15:02.125886+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e630 with expected crc 2023-10-09T09:15:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[75425]: cluster 2023-10-09T09:15:02.131370+0000 mon.a (mon.0) 723 : cluster 0 osdmap e630: 8 total, 7 up, 8 in 2023-10-09T09:15:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[75425]: cluster 2023-10-09T09:15:02.197542+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:03 smithi163 bash[75425]: audit 2023-10-09T09:15:02.547082+0000 mon.a (mon.0) 724 : audit 0 from='client.? 172.21.15.163:0/2770486556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:03.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:03 smithi181 bash[59345]: cluster 2023-10-09T09:15:01.670345+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:03 smithi181 bash[59345]: cluster 2023-10-09T09:15:02.124405+0000 osd.3 (osd.3) 219 : cluster 3 failed to encode map e630 with expected crc 2023-10-09T09:15:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:03 smithi181 bash[59345]: cluster 2023-10-09T09:15:02.125886+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e630 with expected crc 2023-10-09T09:15:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:03 smithi181 bash[59345]: cluster 2023-10-09T09:15:02.131370+0000 mon.a (mon.0) 723 : cluster 0 osdmap e630: 8 total, 7 up, 8 in 2023-10-09T09:15:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:03 smithi181 bash[59345]: cluster 2023-10-09T09:15:02.197542+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e629 with expected crc 2023-10-09T09:15:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:03 smithi181 bash[59345]: audit 2023-10-09T09:15:02.547082+0000 mon.a (mon.0) 724 : audit 0 from='client.? 172.21.15.163:0/2770486556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:04.141 INFO:tasks.rados.rados.0.smithi163.stdout:2209: done (1 left) 2023-10-09T09:15:04.141 INFO:tasks.rados.rados.0.smithi163.stdout:2210: read oid 21 snap -1 2023-10-09T09:15:04.141 INFO:tasks.rados.rados.0.smithi163.stdout:2210: expect deleted 2023-10-09T09:15:04.146 INFO:tasks.rados.rados.0.smithi163.stdout:2211: write oid 44 current snap is 286 2023-10-09T09:15:04.146 INFO:tasks.rados.rados.0.smithi163.stdout:2211: seq_num 828 ranges {716196=472296,1769561=646611,3121885=793109} 2023-10-09T09:15:04.151 INFO:tasks.rados.rados.0.smithi163.stdout:2211: writing smithi16367185-44 from 716196 to 1188492 tid 1 2023-10-09T09:15:04.160 INFO:tasks.rados.rados.0.smithi163.stdout:2211: writing smithi16367185-44 from 1769561 to 2416172 tid 2 2023-10-09T09:15:04.169 INFO:tasks.rados.rados.0.smithi163.stdout:2211: writing smithi16367185-44 from 3121885 to 3914994 tid 3 2023-10-09T09:15:04.170 INFO:tasks.rados.rados.0.smithi163.stdout:2212: rollback oid 29 current snap is 286 2023-10-09T09:15:04.170 INFO:tasks.rados.rados.0.smithi163.stdout:2208: finishing rollback tid 1 to smithi16367185-41 2023-10-09T09:15:04.170 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 136 (ObjNum 288 snap 0 seq_num 273) dirty dne 2023-10-09T09:15:04.170 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 29 to 283 2023-10-09T09:15:04.170 INFO:tasks.rados.rados.0.smithi163.stdout:2211: finishing write tid 1 to smithi16367185-44 2023-10-09T09:15:04.170 INFO:tasks.rados.rados.0.smithi163.stdout:2211: finishing write tid 2 to smithi16367185-44 2023-10-09T09:15:04.170 INFO:tasks.rados.rados.0.smithi163.stdout:2208: done (3 left) 2023-10-09T09:15:04.170 INFO:tasks.rados.rados.0.smithi163.stdout:2210: done (2 left) 2023-10-09T09:15:04.170 INFO:tasks.rados.rados.0.smithi163.stdout:2213: write oid 43 current snap is 286 2023-10-09T09:15:04.171 INFO:tasks.rados.rados.0.smithi163.stdout:2213: seq_num 829 ranges {733003=663898,1944396=506622,2865661=604381} 2023-10-09T09:15:04.178 INFO:tasks.rados.rados.0.smithi163.stdout:2213: writing smithi16367185-43 from 733003 to 1396901 tid 1 2023-10-09T09:15:04.184 INFO:tasks.rados.rados.0.smithi163.stdout:2213: writing smithi16367185-43 from 1944396 to 2451018 tid 2 2023-10-09T09:15:04.190 INFO:tasks.rados.rados.0.smithi163.stdout:2213: writing smithi16367185-43 from 2865661 to 3470042 tid 3 2023-10-09T09:15:04.190 INFO:tasks.rados.rados.0.smithi163.stdout:2212: finishing rollback tid 0 to smithi16367185-29 2023-10-09T09:15:04.190 INFO:tasks.rados.rados.0.smithi163.stdout:2214: read oid 46 snap -1 2023-10-09T09:15:04.191 INFO:tasks.rados.rados.0.smithi163.stdout:2214: expect (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:15:04.191 INFO:tasks.rados.rados.0.smithi163.stdout:2215: snap_create 2023-10-09T09:15:04.191 INFO:tasks.rados.rados.0.smithi163.stdout:2211: finishing write tid 3 to smithi16367185-44 2023-10-09T09:15:04.191 INFO:tasks.rados.rados.0.smithi163.stdout:2211: finishing write tid 4 to smithi16367185-44 2023-10-09T09:15:04.191 INFO:tasks.rados.rados.0.smithi163.stdout:2212: finishing rollback tid 1 to smithi16367185-29 2023-10-09T09:15:04.191 INFO:tasks.rados.rados.0.smithi163.stdout:2211: finishing write tid 5 to smithi16367185-44 2023-10-09T09:15:04.191 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1611 (ObjNum 828 snap 286 seq_num 828) dirty exists 2023-10-09T09:15:04.191 INFO:tasks.rados.rados.0.smithi163.stdout:2211: left oid 44 (ObjNum 828 snap 286 seq_num 828) 2023-10-09T09:15:04.192 INFO:tasks.rados.rados.0.smithi163.stdout:2212: finishing rollback tid 2 to smithi16367185-29 2023-10-09T09:15:04.192 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1045 (ObjNum 815 snap 281 seq_num 815) dirty exists 2023-10-09T09:15:04.192 INFO:tasks.rados.rados.0.smithi163.stdout:2213: finishing write tid 1 to smithi16367185-43 2023-10-09T09:15:04.192 INFO:tasks.rados.rados.0.smithi163.stdout:2213: finishing write tid 2 to smithi16367185-43 2023-10-09T09:15:04.193 INFO:tasks.rados.rados.0.smithi163.stdout:2213: finishing write tid 3 to smithi16367185-43 2023-10-09T09:15:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:2213: finishing write tid 4 to smithi16367185-43 2023-10-09T09:15:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:2213: finishing write tid 5 to smithi16367185-43 2023-10-09T09:15:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 978 (ObjNum 829 snap 286 seq_num 829) dirty exists 2023-10-09T09:15:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:2213: left oid 43 (ObjNum 829 snap 286 seq_num 829) 2023-10-09T09:15:04.202 INFO:tasks.rados.rados.0.smithi163.stdout:2214: expect (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:15:04.455 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[78107]: cluster 2023-10-09T09:15:02.857333+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e630 with expected crc 2023-10-09T09:15:04.455 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[78107]: cluster 2023-10-09T09:15:03.126504+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e631 with expected crc 2023-10-09T09:15:04.455 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[78107]: cluster 2023-10-09T09:15:03.126505+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e631 with expected crc 2023-10-09T09:15:04.456 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[78107]: cluster 2023-10-09T09:15:03.130634+0000 mon.a (mon.0) 725 : cluster 0 osdmap e631: 8 total, 7 up, 8 in 2023-10-09T09:15:04.456 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[78107]: cluster 2023-10-09T09:15:03.335312+0000 mgr.y (mgr.34107) 247 : cluster 0 pgmap v499: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.6 MiB/s wr, 14 op/s; 55/435 objects degraded (12.644%) 2023-10-09T09:15:04.456 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[78107]: audit 2023-10-09T09:15:03.372944+0000 mon.a (mon.0) 726 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.456 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[78107]: audit 2023-10-09T09:15:03.381946+0000 mon.a (mon.0) 727 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.456 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[78107]: audit 2023-10-09T09:15:03.518746+0000 mon.a (mon.0) 728 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.456 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[78107]: audit 2023-10-09T09:15:03.527659+0000 mon.a (mon.0) 729 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.456 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[75425]: cluster 2023-10-09T09:15:02.857333+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e630 with expected crc 2023-10-09T09:15:04.456 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[75425]: cluster 2023-10-09T09:15:03.126504+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e631 with expected crc 2023-10-09T09:15:04.456 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[75425]: cluster 2023-10-09T09:15:03.126505+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e631 with expected crc 2023-10-09T09:15:04.456 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[75425]: cluster 2023-10-09T09:15:03.130634+0000 mon.a (mon.0) 725 : cluster 0 osdmap e631: 8 total, 7 up, 8 in 2023-10-09T09:15:04.457 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[75425]: cluster 2023-10-09T09:15:03.335312+0000 mgr.y (mgr.34107) 247 : cluster 0 pgmap v499: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.6 MiB/s wr, 14 op/s; 55/435 objects degraded (12.644%) 2023-10-09T09:15:04.457 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[75425]: audit 2023-10-09T09:15:03.372944+0000 mon.a (mon.0) 726 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.457 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[75425]: audit 2023-10-09T09:15:03.381946+0000 mon.a (mon.0) 727 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.457 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[75425]: audit 2023-10-09T09:15:03.518746+0000 mon.a (mon.0) 728 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.457 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[75425]: audit 2023-10-09T09:15:03.527659+0000 mon.a (mon.0) 729 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:04 smithi181 bash[59345]: cluster 2023-10-09T09:15:02.857333+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e630 with expected crc 2023-10-09T09:15:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:04 smithi181 bash[59345]: cluster 2023-10-09T09:15:03.126504+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e631 with expected crc 2023-10-09T09:15:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:04 smithi181 bash[59345]: cluster 2023-10-09T09:15:03.126505+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e631 with expected crc 2023-10-09T09:15:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:04 smithi181 bash[59345]: cluster 2023-10-09T09:15:03.130634+0000 mon.a (mon.0) 725 : cluster 0 osdmap e631: 8 total, 7 up, 8 in 2023-10-09T09:15:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:04 smithi181 bash[59345]: cluster 2023-10-09T09:15:03.335312+0000 mgr.y (mgr.34107) 247 : cluster 0 pgmap v499: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.6 MiB/s wr, 14 op/s; 55/435 objects degraded (12.644%) 2023-10-09T09:15:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:04 smithi181 bash[59345]: audit 2023-10-09T09:15:03.372944+0000 mon.a (mon.0) 726 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:04 smithi181 bash[59345]: audit 2023-10-09T09:15:03.381946+0000 mon.a (mon.0) 727 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:04 smithi181 bash[59345]: audit 2023-10-09T09:15:03.518746+0000 mon.a (mon.0) 728 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:04 smithi181 bash[59345]: audit 2023-10-09T09:15:03.527659+0000 mon.a (mon.0) 729 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:04.718 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:15:04 smithi163 bash[89638]: debug 2023-10-09T09:15:04.451+0000 7f4fb5afe640 -1 osd.2 620 log_to_monitors true 2023-10-09T09:15:05.164 INFO:tasks.rados.rados.0.smithi163.stdout:2211: done (4 left) 2023-10-09T09:15:05.165 INFO:tasks.rados.rados.0.smithi163.stdout:2212: done (3 left) 2023-10-09T09:15:05.165 INFO:tasks.rados.rados.0.smithi163.stdout:2213: done (2 left) 2023-10-09T09:15:05.165 INFO:tasks.rados.rados.0.smithi163.stdout:2214: done (1 left) 2023-10-09T09:15:05.165 INFO:tasks.rados.rados.0.smithi163.stdout:2215: done (0 left) 2023-10-09T09:15:05.165 INFO:tasks.rados.rados.0.smithi163.stdout:2216: write oid 44 current snap is 287 2023-10-09T09:15:05.165 INFO:tasks.rados.rados.0.smithi163.stdout:2216: seq_num 830 ranges {749810=739147,2002878=650280,3293084=415653} 2023-10-09T09:15:05.174 INFO:tasks.rados.rados.0.smithi163.stdout:2216: writing smithi16367185-44 from 749810 to 1488957 tid 1 2023-10-09T09:15:05.182 INFO:tasks.rados.rados.0.smithi163.stdout:2216: writing smithi16367185-44 from 2002878 to 2653158 tid 2 2023-10-09T09:15:05.189 INFO:tasks.rados.rados.0.smithi163.stdout:2216: writing smithi16367185-44 from 3293084 to 3708737 tid 3 2023-10-09T09:15:05.189 INFO:tasks.rados.rados.0.smithi163.stdout:2217: delete oid 13 current snap is 287 2023-10-09T09:15:05.189 INFO:tasks.rados.rados.0.smithi163.stdout:2216: finishing write tid 1 to smithi16367185-44 2023-10-09T09:15:05.190 INFO:tasks.rados.rados.0.smithi163.stdout:2216: finishing write tid 2 to smithi16367185-44 2023-10-09T09:15:05.192 INFO:tasks.rados.rados.0.smithi163.stdout:2217: done (1 left) 2023-10-09T09:15:05.193 INFO:tasks.rados.rados.0.smithi163.stdout:2218: read oid 46 snap -1 2023-10-09T09:15:05.193 INFO:tasks.rados.rados.0.smithi163.stdout:2218: expect (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:15:05.193 INFO:tasks.rados.rados.0.smithi163.stdout:2219: read oid 4 snap -1 2023-10-09T09:15:05.193 INFO:tasks.rados.rados.0.smithi163.stdout:2219: expect (ObjNum 813 snap 280 seq_num 813) 2023-10-09T09:15:05.193 INFO:tasks.rados.rados.0.smithi163.stdout:2220: snap_remove snap 279 2023-10-09T09:15:05.468 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[89638]: debug 2023-10-09T09:15:05.211+0000 7f4fa4f66700 -1 osd.2 620 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:15:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[75425]: cluster 2023-10-09T09:15:04.131249+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e632 with expected crc 2023-10-09T09:15:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[75425]: cluster 2023-10-09T09:15:04.132420+0000 osd.3 (osd.3) 220 : cluster 3 failed to encode map e632 with expected crc 2023-10-09T09:15:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[75425]: cluster 2023-10-09T09:15:04.135067+0000 mon.a (mon.0) 730 : cluster 0 osdmap e632: 8 total, 7 up, 8 in 2023-10-09T09:15:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[75425]: cluster 2023-10-09T09:15:04.135361+0000 mon.a (mon.0) 731 : cluster 3 Health check update: Degraded data redundancy: 55/435 objects degraded (12.644%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[75425]: cluster 2023-10-09T09:15:04.158166+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e632 with expected crc 2023-10-09T09:15:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[75425]: audit 2023-10-09T09:15:04.326732+0000 mon.a (mon.0) 732 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[75425]: audit 2023-10-09T09:15:04.333202+0000 mon.a (mon.0) 733 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[75425]: audit 2023-10-09T09:15:04.459670+0000 mon.c (mon.2) 20 : audit 1 from='osd.2 [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:15:05.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[75425]: audit 2023-10-09T09:15:04.460066+0000 mon.a (mon.0) 734 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:15:05.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[75425]: audit 2023-10-09T09:15:04.905190+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.163:0/2991854823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[78107]: cluster 2023-10-09T09:15:04.131249+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e632 with expected crc 2023-10-09T09:15:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[78107]: cluster 2023-10-09T09:15:04.132420+0000 osd.3 (osd.3) 220 : cluster 3 failed to encode map e632 with expected crc 2023-10-09T09:15:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[78107]: cluster 2023-10-09T09:15:04.135067+0000 mon.a (mon.0) 730 : cluster 0 osdmap e632: 8 total, 7 up, 8 in 2023-10-09T09:15:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[78107]: cluster 2023-10-09T09:15:04.135361+0000 mon.a (mon.0) 731 : cluster 3 Health check update: Degraded data redundancy: 55/435 objects degraded (12.644%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[78107]: cluster 2023-10-09T09:15:04.158166+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e632 with expected crc 2023-10-09T09:15:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[78107]: audit 2023-10-09T09:15:04.326732+0000 mon.a (mon.0) 732 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:05.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[78107]: audit 2023-10-09T09:15:04.333202+0000 mon.a (mon.0) 733 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:05.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[78107]: audit 2023-10-09T09:15:04.459670+0000 mon.c (mon.2) 20 : audit 1 from='osd.2 [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:15:05.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[78107]: audit 2023-10-09T09:15:04.460066+0000 mon.a (mon.0) 734 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:15:05.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:05 smithi163 bash[78107]: audit 2023-10-09T09:15:04.905190+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.163:0/2991854823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:05 smithi181 bash[59345]: cluster 2023-10-09T09:15:04.131249+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e632 with expected crc 2023-10-09T09:15:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:05 smithi181 bash[59345]: cluster 2023-10-09T09:15:04.132420+0000 osd.3 (osd.3) 220 : cluster 3 failed to encode map e632 with expected crc 2023-10-09T09:15:05.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:05 smithi181 bash[59345]: cluster 2023-10-09T09:15:04.135067+0000 mon.a (mon.0) 730 : cluster 0 osdmap e632: 8 total, 7 up, 8 in 2023-10-09T09:15:05.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:05 smithi181 bash[59345]: cluster 2023-10-09T09:15:04.135361+0000 mon.a (mon.0) 731 : cluster 3 Health check update: Degraded data redundancy: 55/435 objects degraded (12.644%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:05.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:05 smithi181 bash[59345]: cluster 2023-10-09T09:15:04.158166+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e632 with expected crc 2023-10-09T09:15:05.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:05 smithi181 bash[59345]: audit 2023-10-09T09:15:04.326732+0000 mon.a (mon.0) 732 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:05.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:05 smithi181 bash[59345]: audit 2023-10-09T09:15:04.333202+0000 mon.a (mon.0) 733 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:05.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:05 smithi181 bash[59345]: audit 2023-10-09T09:15:04.459670+0000 mon.c (mon.2) 20 : audit 1 from='osd.2 [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:15:05.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:05 smithi181 bash[59345]: audit 2023-10-09T09:15:04.460066+0000 mon.a (mon.0) 734 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:15:05.569 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:05 smithi181 bash[59345]: audit 2023-10-09T09:15:04.905190+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.163:0/2991854823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:06.181 INFO:tasks.rados.rados.0.smithi163.stdout:2220: done (3 left) 2023-10-09T09:15:06.181 INFO:tasks.rados.rados.0.smithi163.stdout:2221: snap_remove snap 246 2023-10-09T09:15:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[75425]: cluster 2023-10-09T09:15:05.161464+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e633 with expected crc 2023-10-09T09:15:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[75425]: cluster 2023-10-09T09:15:05.161950+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e633 with expected crc 2023-10-09T09:15:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[75425]: audit 2023-10-09T09:15:05.164475+0000 mon.a (mon.0) 735 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T09:15:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[75425]: cluster 2023-10-09T09:15:05.164592+0000 mon.a (mon.0) 736 : cluster 0 osdmap e633: 8 total, 7 up, 8 in 2023-10-09T09:15:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[75425]: audit 2023-10-09T09:15:05.166390+0000 mon.c (mon.2) 22 : audit 1 from='osd.2 [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[75425]: audit 2023-10-09T09:15:05.166922+0000 mon.a (mon.0) 737 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[75425]: cluster 2023-10-09T09:15:05.193625+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e633 with expected crc 2023-10-09T09:15:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[75425]: cluster 2023-10-09T09:15:05.336169+0000 mgr.y (mgr.34107) 248 : cluster 0 pgmap v502: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 805 KiB/s wr, 5 op/s; 55/432 objects degraded (12.731%) 2023-10-09T09:15:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[78107]: cluster 2023-10-09T09:15:05.161464+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e633 with expected crc 2023-10-09T09:15:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[78107]: cluster 2023-10-09T09:15:05.161950+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e633 with expected crc 2023-10-09T09:15:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[78107]: audit 2023-10-09T09:15:05.164475+0000 mon.a (mon.0) 735 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T09:15:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[78107]: cluster 2023-10-09T09:15:05.164592+0000 mon.a (mon.0) 736 : cluster 0 osdmap e633: 8 total, 7 up, 8 in 2023-10-09T09:15:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[78107]: audit 2023-10-09T09:15:05.166390+0000 mon.c (mon.2) 22 : audit 1 from='osd.2 [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[78107]: audit 2023-10-09T09:15:05.166922+0000 mon.a (mon.0) 737 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[78107]: cluster 2023-10-09T09:15:05.193625+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e633 with expected crc 2023-10-09T09:15:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:06 smithi163 bash[78107]: cluster 2023-10-09T09:15:05.336169+0000 mgr.y (mgr.34107) 248 : cluster 0 pgmap v502: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 805 KiB/s wr, 5 op/s; 55/432 objects degraded (12.731%) 2023-10-09T09:15:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:06 smithi181 bash[59345]: cluster 2023-10-09T09:15:05.161464+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e633 with expected crc 2023-10-09T09:15:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:06 smithi181 bash[59345]: cluster 2023-10-09T09:15:05.161950+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e633 with expected crc 2023-10-09T09:15:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:06 smithi181 bash[59345]: audit 2023-10-09T09:15:05.164475+0000 mon.a (mon.0) 735 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T09:15:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:06 smithi181 bash[59345]: cluster 2023-10-09T09:15:05.164592+0000 mon.a (mon.0) 736 : cluster 0 osdmap e633: 8 total, 7 up, 8 in 2023-10-09T09:15:06.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:06 smithi181 bash[59345]: audit 2023-10-09T09:15:05.166390+0000 mon.c (mon.2) 22 : audit 1 from='osd.2 [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:06.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:06 smithi181 bash[59345]: audit 2023-10-09T09:15:05.166922+0000 mon.a (mon.0) 737 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:06.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:06 smithi181 bash[59345]: cluster 2023-10-09T09:15:05.193625+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e633 with expected crc 2023-10-09T09:15:06.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:06 smithi181 bash[59345]: cluster 2023-10-09T09:15:05.336169+0000 mgr.y (mgr.34107) 248 : cluster 0 pgmap v502: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 805 KiB/s wr, 5 op/s; 55/432 objects degraded (12.731%) 2023-10-09T09:15:07.184 INFO:tasks.rados.rados.0.smithi163.stdout:2221: done (3 left) 2023-10-09T09:15:07.185 INFO:tasks.rados.rados.0.smithi163.stdout:2222: write oid 38 current snap is 287 2023-10-09T09:15:07.185 INFO:tasks.rados.rados.0.smithi163.stdout:2216: finishing write tid 3 to smithi16367185-44 2023-10-09T09:15:07.185 INFO:tasks.rados.rados.0.smithi163.stdout:2216: finishing write tid 4 to smithi16367185-44 2023-10-09T09:15:07.185 INFO:tasks.rados.rados.0.smithi163.stdout:2216: finishing write tid 5 to smithi16367185-44 2023-10-09T09:15:07.185 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1616 (ObjNum 830 snap 287 seq_num 830) dirty exists 2023-10-09T09:15:07.185 INFO:tasks.rados.rados.0.smithi163.stdout:2216: left oid 44 (ObjNum 830 snap 287 seq_num 830) 2023-10-09T09:15:07.185 INFO:tasks.rados.rados.0.smithi163.stdout:2222: seq_num 832 ranges {783424=489645,2003489=653949,3231577=554550} 2023-10-09T09:15:07.214 INFO:tasks.rados.rados.0.smithi163.stdout:2222: writing smithi16367185-38 from 783424 to 1273069 tid 1 2023-10-09T09:15:07.223 INFO:tasks.rados.rados.0.smithi163.stdout:2222: writing smithi16367185-38 from 2003489 to 2657438 tid 2 2023-10-09T09:15:07.232 INFO:tasks.rados.rados.0.smithi163.stdout:2222: writing smithi16367185-38 from 3231577 to 3786127 tid 3 2023-10-09T09:15:07.232 INFO:tasks.rados.rados.0.smithi163.stdout:2216: done (3 left) 2023-10-09T09:15:07.233 INFO:tasks.rados.rados.0.smithi163.stdout:2223: snap_remove snap 277 2023-10-09T09:15:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[78107]: cluster 2023-10-09T09:15:06.163155+0000 mon.a (mon.0) 738 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:15:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[78107]: cluster 2023-10-09T09:15:06.174932+0000 osd.3 (osd.3) 221 : cluster 3 failed to encode map e634 with expected crc 2023-10-09T09:15:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[78107]: cluster 2023-10-09T09:15:06.176014+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e634 with expected crc 2023-10-09T09:15:07.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[78107]: cluster 2023-10-09T09:15:06.180708+0000 mon.a (mon.0) 739 : cluster 1 osd.2 [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903] boot 2023-10-09T09:15:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[78107]: cluster 2023-10-09T09:15:06.180782+0000 mon.a (mon.0) 740 : cluster 0 osdmap e634: 8 total, 8 up, 8 in 2023-10-09T09:15:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[78107]: audit 2023-10-09T09:15:06.181394+0000 mon.a (mon.0) 741 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:15:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[75425]: cluster 2023-10-09T09:15:06.163155+0000 mon.a (mon.0) 738 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:15:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[75425]: cluster 2023-10-09T09:15:06.174932+0000 osd.3 (osd.3) 221 : cluster 3 failed to encode map e634 with expected crc 2023-10-09T09:15:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[75425]: cluster 2023-10-09T09:15:06.176014+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e634 with expected crc 2023-10-09T09:15:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[75425]: cluster 2023-10-09T09:15:06.180708+0000 mon.a (mon.0) 739 : cluster 1 osd.2 [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903] boot 2023-10-09T09:15:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[75425]: cluster 2023-10-09T09:15:06.180782+0000 mon.a (mon.0) 740 : cluster 0 osdmap e634: 8 total, 8 up, 8 in 2023-10-09T09:15:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:07 smithi163 bash[75425]: audit 2023-10-09T09:15:06.181394+0000 mon.a (mon.0) 741 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:15:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:07 smithi181 bash[59345]: cluster 2023-10-09T09:15:06.163155+0000 mon.a (mon.0) 738 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:15:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:07 smithi181 bash[59345]: cluster 2023-10-09T09:15:06.174932+0000 osd.3 (osd.3) 221 : cluster 3 failed to encode map e634 with expected crc 2023-10-09T09:15:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:07 smithi181 bash[59345]: cluster 2023-10-09T09:15:06.176014+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e634 with expected crc 2023-10-09T09:15:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:07 smithi181 bash[59345]: cluster 2023-10-09T09:15:06.180708+0000 mon.a (mon.0) 739 : cluster 1 osd.2 [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903] boot 2023-10-09T09:15:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:07 smithi181 bash[59345]: cluster 2023-10-09T09:15:06.180782+0000 mon.a (mon.0) 740 : cluster 0 osdmap e634: 8 total, 8 up, 8 in 2023-10-09T09:15:07.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:07 smithi181 bash[59345]: audit 2023-10-09T09:15:06.181394+0000 mon.a (mon.0) 741 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:15:08.196 INFO:tasks.rados.rados.0.smithi163.stdout:2223: done (3 left) 2023-10-09T09:15:08.196 INFO:tasks.rados.rados.0.smithi163.stdout:2224: write oid 27 current snap is 287 2023-10-09T09:15:08.196 INFO:tasks.rados.rados.0.smithi163.stdout:2224: seq_num 833 ranges {400231=564894,1661971=338260} 2023-10-09T09:15:08.202 INFO:tasks.rados.rados.0.smithi163.stdout:2224: writing smithi16367185-27 from 400231 to 965125 tid 1 2023-10-09T09:15:08.209 INFO:tasks.rados.rados.0.smithi163.stdout:2224: writing smithi16367185-27 from 1661971 to 2000231 tid 2 2023-10-09T09:15:08.210 INFO:tasks.rados.rados.0.smithi163.stdout:2225: delete oid 34 current snap is 287 2023-10-09T09:15:08.218 INFO:tasks.rados.rados.0.smithi163.stdout:2219: expect (ObjNum 813 snap 280 seq_num 813) 2023-10-09T09:15:08.334 INFO:tasks.rados.rados.0.smithi163.stdout:2218: expect (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:15:08.444 INFO:tasks.rados.rados.0.smithi163.stdout:2222: finishing write tid 1 to smithi16367185-38 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:2222: finishing write tid 2 to smithi16367185-38 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:2222: finishing write tid 3 to smithi16367185-38 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:2222: finishing write tid 4 to smithi16367185-38 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:2222: finishing write tid 5 to smithi16367185-38 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 996 (ObjNum 832 snap 287 seq_num 832) dirty exists 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:2222: left oid 38 (ObjNum 832 snap 287 seq_num 832) 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:2224: finishing write tid 1 to smithi16367185-27 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:2224: finishing write tid 2 to smithi16367185-27 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:2224: finishing write tid 3 to smithi16367185-27 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:2224: finishing write tid 4 to smithi16367185-27 2023-10-09T09:15:08.445 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 618 (ObjNum 833 snap 287 seq_num 833) dirty exists 2023-10-09T09:15:08.446 INFO:tasks.rados.rados.0.smithi163.stdout:2224: left oid 27 (ObjNum 833 snap 287 seq_num 833) 2023-10-09T09:15:08.446 INFO:tasks.rados.rados.0.smithi163.stdout:2218: done (4 left) 2023-10-09T09:15:08.446 INFO:tasks.rados.rados.0.smithi163.stdout:2219: done (3 left) 2023-10-09T09:15:08.446 INFO:tasks.rados.rados.0.smithi163.stdout:2222: done (2 left) 2023-10-09T09:15:08.446 INFO:tasks.rados.rados.0.smithi163.stdout:2224: done (1 left) 2023-10-09T09:15:08.446 INFO:tasks.rados.rados.0.smithi163.stdout:2225: done (0 left) 2023-10-09T09:15:08.446 INFO:tasks.rados.rados.0.smithi163.stdout:2226: read oid 16 snap -1 2023-10-09T09:15:08.446 INFO:tasks.rados.rados.0.smithi163.stdout:2226: expect (ObjNum 804 snap 279 seq_num 804) 2023-10-09T09:15:08.446 INFO:tasks.rados.rados.0.smithi163.stdout:2227: write oid 27 current snap is 287 2023-10-09T09:15:08.446 INFO:tasks.rados.rados.0.smithi163.stdout:2227: seq_num 835 ranges {433845=715392,1662582=371263} 2023-10-09T09:15:08.455 INFO:tasks.rados.rados.0.smithi163.stdout:2227: writing smithi16367185-27 from 433845 to 1149237 tid 1 2023-10-09T09:15:08.461 INFO:tasks.rados.rados.0.smithi163.stdout:2227: writing smithi16367185-27 from 1662582 to 2033845 tid 2 2023-10-09T09:15:08.461 INFO:tasks.rados.rados.0.smithi163.stdout:2228: snap_create 2023-10-09T09:15:08.462 INFO:tasks.rados.rados.0.smithi163.stdout:2226: expect (ObjNum 804 snap 279 seq_num 804) 2023-10-09T09:15:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: cluster 2023-10-09T09:15:07.180508+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: cluster 2023-10-09T09:15:07.180914+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: cluster 2023-10-09T09:15:07.183509+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: cluster 2023-10-09T09:15:07.184544+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: cluster 2023-10-09T09:15:07.184618+0000 mon.a (mon.0) 742 : cluster 0 osdmap e635: 8 total, 8 up, 8 in 2023-10-09T09:15:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: cluster 2023-10-09T09:15:07.184833+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: cluster 2023-10-09T09:15:07.184863+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: cluster 2023-10-09T09:15:07.185121+0000 osd.3 (osd.3) 222 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: cluster 2023-10-09T09:15:07.187699+0000 osd.3 (osd.3) 223 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: audit 2023-10-09T09:15:07.299943+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.163:0/1736774517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[75425]: cluster 2023-10-09T09:15:07.336809+0000 mgr.y (mgr.34107) 249 : cluster 0 pgmap v505: 89 pgs: 14 peering, 5 active+undersized+degraded, 6 active+undersized, 64 active+clean; 246 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 38/447 objects degraded (8.501%) 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: cluster 2023-10-09T09:15:07.180508+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: cluster 2023-10-09T09:15:07.180914+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: cluster 2023-10-09T09:15:07.183509+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: cluster 2023-10-09T09:15:07.184544+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: cluster 2023-10-09T09:15:07.184618+0000 mon.a (mon.0) 742 : cluster 0 osdmap e635: 8 total, 8 up, 8 in 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: cluster 2023-10-09T09:15:07.184833+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: cluster 2023-10-09T09:15:07.184863+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: cluster 2023-10-09T09:15:07.185121+0000 osd.3 (osd.3) 222 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: cluster 2023-10-09T09:15:07.187699+0000 osd.3 (osd.3) 223 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: audit 2023-10-09T09:15:07.299943+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.163:0/1736774517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:08 smithi163 bash[78107]: cluster 2023-10-09T09:15:07.336809+0000 mgr.y (mgr.34107) 249 : cluster 0 pgmap v505: 89 pgs: 14 peering, 5 active+undersized+degraded, 6 active+undersized, 64 active+clean; 246 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 38/447 objects degraded (8.501%) 2023-10-09T09:15:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: cluster 2023-10-09T09:15:07.180508+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: cluster 2023-10-09T09:15:07.180914+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: cluster 2023-10-09T09:15:07.183509+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: cluster 2023-10-09T09:15:07.184544+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: cluster 2023-10-09T09:15:07.184618+0000 mon.a (mon.0) 742 : cluster 0 osdmap e635: 8 total, 8 up, 8 in 2023-10-09T09:15:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: cluster 2023-10-09T09:15:07.184833+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: cluster 2023-10-09T09:15:07.184863+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: cluster 2023-10-09T09:15:07.185121+0000 osd.3 (osd.3) 222 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: cluster 2023-10-09T09:15:07.187699+0000 osd.3 (osd.3) 223 : cluster 3 failed to encode map e635 with expected crc 2023-10-09T09:15:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: audit 2023-10-09T09:15:07.299943+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.163:0/1736774517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:08.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:08 smithi181 bash[59345]: cluster 2023-10-09T09:15:07.336809+0000 mgr.y (mgr.34107) 249 : cluster 0 pgmap v505: 89 pgs: 14 peering, 5 active+undersized+degraded, 6 active+undersized, 64 active+clean; 246 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 38/447 objects degraded (8.501%) 2023-10-09T09:15:08.615 INFO:tasks.rados.rados.0.smithi163.stdout:2227: finishing write tid 1 to smithi16367185-27 2023-10-09T09:15:08.615 INFO:tasks.rados.rados.0.smithi163.stdout:2227: finishing write tid 2 to smithi16367185-27 2023-10-09T09:15:08.615 INFO:tasks.rados.rados.0.smithi163.stdout:2227: finishing write tid 3 to smithi16367185-27 2023-10-09T09:15:08.615 INFO:tasks.rados.rados.0.smithi163.stdout:2227: finishing write tid 4 to smithi16367185-27 2023-10-09T09:15:08.615 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 621 (ObjNum 835 snap 287 seq_num 835) dirty exists 2023-10-09T09:15:08.615 INFO:tasks.rados.rados.0.smithi163.stdout:2227: left oid 27 (ObjNum 835 snap 287 seq_num 835) 2023-10-09T09:15:09.204 INFO:tasks.rados.rados.0.smithi163.stdout:2226: done (2 left) 2023-10-09T09:15:09.204 INFO:tasks.rados.rados.0.smithi163.stdout:2227: done (1 left) 2023-10-09T09:15:09.204 INFO:tasks.rados.rados.0.smithi163.stdout:2228: done (0 left) 2023-10-09T09:15:09.204 INFO:tasks.rados.rados.0.smithi163.stdout:2229: write oid 12 current snap is 288 2023-10-09T09:15:09.204 INFO:tasks.rados.rados.0.smithi163.stdout:2229: seq_num 836 ranges {450652=790641,1721064=329588} 2023-10-09T09:15:09.212 INFO:tasks.rados.rados.0.smithi163.stdout:2229: writing smithi16367185-12 from 450652 to 1241293 tid 1 2023-10-09T09:15:09.218 INFO:tasks.rados.rados.0.smithi163.stdout:2229: writing smithi16367185-12 from 1721064 to 2050652 tid 2 2023-10-09T09:15:09.218 INFO:tasks.rados.rados.0.smithi163.stdout:2230: write oid 38 current snap is 288 2023-10-09T09:15:09.218 INFO:tasks.rados.rados.0.smithi163.stdout:2230: seq_num 837 ranges {467459=582243,1495899=404945} 2023-10-09T09:15:09.225 INFO:tasks.rados.rados.0.smithi163.stdout:2230: writing smithi16367185-38 from 467459 to 1049702 tid 1 2023-10-09T09:15:09.231 INFO:tasks.rados.rados.0.smithi163.stdout:2230: writing smithi16367185-38 from 1495899 to 1900844 tid 2 2023-10-09T09:15:09.231 INFO:tasks.rados.rados.0.smithi163.stdout:2231: delete oid 39 current snap is 288 2023-10-09T09:15:09.231 INFO:tasks.rados.rados.0.smithi163.stdout:2229: finishing write tid 1 to smithi16367185-12 2023-10-09T09:15:09.231 INFO:tasks.rados.rados.0.smithi163.stdout:2229: finishing write tid 2 to smithi16367185-12 2023-10-09T09:15:09.231 INFO:tasks.rados.rados.0.smithi163.stdout:2229: finishing write tid 3 to smithi16367185-12 2023-10-09T09:15:09.232 INFO:tasks.rados.rados.0.smithi163.stdout:2229: finishing write tid 4 to smithi16367185-12 2023-10-09T09:15:09.232 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 1075 (ObjNum 836 snap 288 seq_num 836) dirty exists 2023-10-09T09:15:09.232 INFO:tasks.rados.rados.0.smithi163.stdout:2229: left oid 12 (ObjNum 836 snap 288 seq_num 836) 2023-10-09T09:15:09.234 INFO:tasks.rados.rados.0.smithi163.stdout:2230: finishing write tid 1 to smithi16367185-38 2023-10-09T09:15:09.236 INFO:tasks.rados.rados.0.smithi163.stdout:2230: finishing write tid 2 to smithi16367185-38 2023-10-09T09:15:09.237 INFO:tasks.rados.rados.0.smithi163.stdout:2230: finishing write tid 3 to smithi16367185-38 2023-10-09T09:15:09.237 INFO:tasks.rados.rados.0.smithi163.stdout:2230: finishing write tid 4 to smithi16367185-38 2023-10-09T09:15:09.237 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1006 (ObjNum 837 snap 288 seq_num 837) dirty exists 2023-10-09T09:15:09.237 INFO:tasks.rados.rados.0.smithi163.stdout:2230: left oid 38 (ObjNum 837 snap 288 seq_num 837) 2023-10-09T09:15:09.238 INFO:tasks.rados.rados.0.smithi163.stdout:2229: done (2 left) 2023-10-09T09:15:09.238 INFO:tasks.rados.rados.0.smithi163.stdout:2230: done (1 left) 2023-10-09T09:15:09.238 INFO:tasks.rados.rados.0.smithi163.stdout:2231: done (0 left) 2023-10-09T09:15:09.238 INFO:tasks.rados.rados.0.smithi163.stdout:2232: write oid 42 current snap is 288 2023-10-09T09:15:09.238 INFO:tasks.rados.rados.0.smithi163.stdout:2232: seq_num 839 ranges {501073=732741,1896510=204563} 2023-10-09T09:15:09.245 INFO:tasks.rados.rados.0.smithi163.stdout:2232: writing smithi16367185-42 from 501073 to 1233814 tid 1 2023-10-09T09:15:09.251 INFO:tasks.rados.rados.0.smithi163.stdout:2232: writing smithi16367185-42 from 1896510 to 2101073 tid 2 2023-10-09T09:15:09.252 INFO:tasks.rados.rados.0.smithi163.stdout:2233: snap_create 2023-10-09T09:15:09.252 INFO:tasks.rados.rados.0.smithi163.stdout:2232: finishing write tid 1 to smithi16367185-42 2023-10-09T09:15:09.256 INFO:tasks.rados.rados.0.smithi163.stdout:2232: finishing write tid 2 to smithi16367185-42 2023-10-09T09:15:09.256 INFO:tasks.rados.rados.0.smithi163.stdout:2232: finishing write tid 3 to smithi16367185-42 2023-10-09T09:15:09.257 INFO:tasks.rados.rados.0.smithi163.stdout:2232: finishing write tid 4 to smithi16367185-42 2023-10-09T09:15:09.257 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 1639 (ObjNum 839 snap 288 seq_num 839) dirty exists 2023-10-09T09:15:09.257 INFO:tasks.rados.rados.0.smithi163.stdout:2232: left oid 42 (ObjNum 839 snap 288 seq_num 839) 2023-10-09T09:15:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:09 smithi163 bash[78107]: cluster 2023-10-09T09:15:08.190105+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e636 with expected crc 2023-10-09T09:15:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:09 smithi163 bash[78107]: cluster 2023-10-09T09:15:08.195290+0000 mon.a (mon.0) 743 : cluster 0 osdmap e636: 8 total, 8 up, 8 in 2023-10-09T09:15:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:09 smithi163 bash[78107]: cluster 2023-10-09T09:15:08.209433+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e636 with expected crc 2023-10-09T09:15:09.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:09 smithi163 bash[78107]: cluster 2023-10-09T09:15:08.209550+0000 osd.3 (osd.3) 224 : cluster 3 failed to encode map e636 with expected crc 2023-10-09T09:15:09.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:09 smithi163 bash[75425]: cluster 2023-10-09T09:15:08.190105+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e636 with expected crc 2023-10-09T09:15:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:09 smithi163 bash[75425]: cluster 2023-10-09T09:15:08.195290+0000 mon.a (mon.0) 743 : cluster 0 osdmap e636: 8 total, 8 up, 8 in 2023-10-09T09:15:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:09 smithi163 bash[75425]: cluster 2023-10-09T09:15:08.209433+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e636 with expected crc 2023-10-09T09:15:09.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:09 smithi163 bash[75425]: cluster 2023-10-09T09:15:08.209550+0000 osd.3 (osd.3) 224 : cluster 3 failed to encode map e636 with expected crc 2023-10-09T09:15:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:09 smithi181 bash[59345]: cluster 2023-10-09T09:15:08.190105+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e636 with expected crc 2023-10-09T09:15:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:09 smithi181 bash[59345]: cluster 2023-10-09T09:15:08.195290+0000 mon.a (mon.0) 743 : cluster 0 osdmap e636: 8 total, 8 up, 8 in 2023-10-09T09:15:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:09 smithi181 bash[59345]: cluster 2023-10-09T09:15:08.209433+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e636 with expected crc 2023-10-09T09:15:09.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:09 smithi181 bash[59345]: cluster 2023-10-09T09:15:08.209550+0000 osd.3 (osd.3) 224 : cluster 3 failed to encode map e636 with expected crc 2023-10-09T09:15:10.210 INFO:tasks.rados.rados.0.smithi163.stdout:2232: done (1 left) 2023-10-09T09:15:10.210 INFO:tasks.rados.rados.0.smithi163.stdout:2233: done (0 left) 2023-10-09T09:15:10.210 INFO:tasks.rados.rados.0.smithi163.stdout:2234: read oid 46 snap -1 2023-10-09T09:15:10.211 INFO:tasks.rados.rados.0.smithi163.stdout:2234: expect (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:15:10.211 INFO:tasks.rados.rados.0.smithi163.stdout:2235: write oid 5 current snap is 289 2023-10-09T09:15:10.211 INFO:tasks.rados.rados.0.smithi163.stdout:2235: seq_num 840 ranges {517880=407990,1554992=562888} 2023-10-09T09:15:10.216 INFO:tasks.rados.rados.0.smithi163.stdout:2235: writing smithi16367185-5 from 517880 to 925870 tid 1 2023-10-09T09:15:10.226 INFO:tasks.rados.rados.0.smithi163.stdout:2235: writing smithi16367185-5 from 1554992 to 2117880 tid 2 2023-10-09T09:15:10.227 INFO:tasks.rados.rados.0.smithi163.stdout:2236: write oid 37 current snap is 289 2023-10-09T09:15:10.227 INFO:tasks.rados.rados.0.smithi163.stdout:2236: seq_num 841 ranges {534687=483239,1613474=412283} 2023-10-09T09:15:10.232 INFO:tasks.rados.rados.0.smithi163.stdout:2236: writing smithi16367185-37 from 534687 to 1017926 tid 1 2023-10-09T09:15:10.239 INFO:tasks.rados.rados.0.smithi163.stdout:2236: writing smithi16367185-37 from 1613474 to 2025757 tid 2 2023-10-09T09:15:10.240 INFO:tasks.rados.rados.0.smithi163.stdout:2237: delete oid 16 current snap is 289 2023-10-09T09:15:10.240 INFO:tasks.rados.rados.0.smithi163.stdout:2235: finishing write tid 1 to smithi16367185-5 2023-10-09T09:15:10.240 INFO:tasks.rados.rados.0.smithi163.stdout:2234: expect (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:15:10.353 INFO:tasks.rados.rados.0.smithi163.stdout:2235: finishing write tid 2 to smithi16367185-5 2023-10-09T09:15:10.353 INFO:tasks.rados.rados.0.smithi163.stdout:2235: finishing write tid 3 to smithi16367185-5 2023-10-09T09:15:10.354 INFO:tasks.rados.rados.0.smithi163.stdout:2235: finishing write tid 4 to smithi16367185-5 2023-10-09T09:15:10.354 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1125 (ObjNum 840 snap 289 seq_num 840) dirty exists 2023-10-09T09:15:10.354 INFO:tasks.rados.rados.0.smithi163.stdout:2235: left oid 5 (ObjNum 840 snap 289 seq_num 840) 2023-10-09T09:15:10.354 INFO:tasks.rados.rados.0.smithi163.stdout:2236: finishing write tid 1 to smithi16367185-37 2023-10-09T09:15:10.354 INFO:tasks.rados.rados.0.smithi163.stdout:2236: finishing write tid 2 to smithi16367185-37 2023-10-09T09:15:10.354 INFO:tasks.rados.rados.0.smithi163.stdout:2236: finishing write tid 3 to smithi16367185-37 2023-10-09T09:15:10.354 INFO:tasks.rados.rados.0.smithi163.stdout:2236: finishing write tid 4 to smithi16367185-37 2023-10-09T09:15:10.354 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 1114 (ObjNum 841 snap 289 seq_num 841) dirty exists 2023-10-09T09:15:10.354 INFO:tasks.rados.rados.0.smithi163.stdout:2236: left oid 37 (ObjNum 841 snap 289 seq_num 841) 2023-10-09T09:15:10.354 INFO:tasks.rados.rados.0.smithi163.stdout:2234: done (3 left) 2023-10-09T09:15:10.355 INFO:tasks.rados.rados.0.smithi163.stdout:2235: done (2 left) 2023-10-09T09:15:10.355 INFO:tasks.rados.rados.0.smithi163.stdout:2236: done (1 left) 2023-10-09T09:15:10.355 INFO:tasks.rados.rados.0.smithi163.stdout:2237: done (0 left) 2023-10-09T09:15:10.355 INFO:tasks.rados.rados.0.smithi163.stdout:2238: read oid 4 snap 251 2023-10-09T09:15:10.355 INFO:tasks.rados.rados.0.smithi163.stdout:2238: expect (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:15:10.355 INFO:tasks.rados.rados.0.smithi163.stdout:2239: snap_remove snap 256 2023-10-09T09:15:10.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[78107]: cluster 2023-10-09T09:15:09.199902+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e637 with expected crc 2023-10-09T09:15:10.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[78107]: cluster 2023-10-09T09:15:09.200360+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e637 with expected crc 2023-10-09T09:15:10.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[78107]: cluster 2023-10-09T09:15:09.204259+0000 mon.a (mon.0) 744 : cluster 0 osdmap e637: 8 total, 8 up, 8 in 2023-10-09T09:15:10.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[78107]: cluster 2023-10-09T09:15:09.231301+0000 osd.3 (osd.3) 225 : cluster 3 failed to encode map e637 with expected crc 2023-10-09T09:15:10.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[78107]: cluster 2023-10-09T09:15:09.337415+0000 mgr.y (mgr.34107) 250 : cluster 0 pgmap v508: 89 pgs: 14 peering, 5 active+undersized+degraded, 2 active+undersized, 68 active+clean; 246 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 38/447 objects degraded (8.501%) 2023-10-09T09:15:10.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[78107]: audit 2023-10-09T09:15:09.666270+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.163:0/351733231' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:10.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[78107]: cluster 2023-10-09T09:15:10.103649+0000 mon.a (mon.0) 745 : cluster 3 Health check update: Degraded data redundancy: 38/447 objects degraded (8.501%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[75425]: cluster 2023-10-09T09:15:09.199902+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e637 with expected crc 2023-10-09T09:15:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[75425]: cluster 2023-10-09T09:15:09.200360+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e637 with expected crc 2023-10-09T09:15:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[75425]: cluster 2023-10-09T09:15:09.204259+0000 mon.a (mon.0) 744 : cluster 0 osdmap e637: 8 total, 8 up, 8 in 2023-10-09T09:15:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[75425]: cluster 2023-10-09T09:15:09.231301+0000 osd.3 (osd.3) 225 : cluster 3 failed to encode map e637 with expected crc 2023-10-09T09:15:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[75425]: cluster 2023-10-09T09:15:09.337415+0000 mgr.y (mgr.34107) 250 : cluster 0 pgmap v508: 89 pgs: 14 peering, 5 active+undersized+degraded, 2 active+undersized, 68 active+clean; 246 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 38/447 objects degraded (8.501%) 2023-10-09T09:15:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[75425]: audit 2023-10-09T09:15:09.666270+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.163:0/351733231' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:10 smithi163 bash[75425]: cluster 2023-10-09T09:15:10.103649+0000 mon.a (mon.0) 745 : cluster 3 Health check update: Degraded data redundancy: 38/447 objects degraded (8.501%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:10 smithi181 bash[59345]: cluster 2023-10-09T09:15:09.199902+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e637 with expected crc 2023-10-09T09:15:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:10 smithi181 bash[59345]: cluster 2023-10-09T09:15:09.200360+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e637 with expected crc 2023-10-09T09:15:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:10 smithi181 bash[59345]: cluster 2023-10-09T09:15:09.204259+0000 mon.a (mon.0) 744 : cluster 0 osdmap e637: 8 total, 8 up, 8 in 2023-10-09T09:15:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:10 smithi181 bash[59345]: cluster 2023-10-09T09:15:09.231301+0000 osd.3 (osd.3) 225 : cluster 3 failed to encode map e637 with expected crc 2023-10-09T09:15:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:10 smithi181 bash[59345]: cluster 2023-10-09T09:15:09.337415+0000 mgr.y (mgr.34107) 250 : cluster 0 pgmap v508: 89 pgs: 14 peering, 5 active+undersized+degraded, 2 active+undersized, 68 active+clean; 246 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 38/447 objects degraded (8.501%) 2023-10-09T09:15:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:10 smithi181 bash[59345]: audit 2023-10-09T09:15:09.666270+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.163:0/351733231' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:10.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:10 smithi181 bash[59345]: cluster 2023-10-09T09:15:10.103649+0000 mon.a (mon.0) 745 : cluster 3 Health check update: Degraded data redundancy: 38/447 objects degraded (8.501%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:11.353 INFO:tasks.rados.rados.0.smithi163.stdout:2239: done (1 left) 2023-10-09T09:15:11.353 INFO:tasks.rados.rados.0.smithi163.stdout:2240: delete oid 45 current snap is 289 2023-10-09T09:15:11.353 INFO:tasks.rados.rados.0.smithi163.stdout:2238: expect (ObjNum 588 snap 186 seq_num 588) 2023-10-09T09:15:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[75425]: cluster 2023-10-09T09:15:10.205745+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[75425]: cluster 2023-10-09T09:15:10.206900+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[75425]: cluster 2023-10-09T09:15:10.210225+0000 mon.a (mon.0) 746 : cluster 0 osdmap e638: 8 total, 8 up, 8 in 2023-10-09T09:15:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[75425]: cluster 2023-10-09T09:15:10.221070+0000 osd.3 (osd.3) 226 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[75425]: cluster 2023-10-09T09:15:10.233748+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[75425]: audit 2023-10-09T09:15:10.346951+0000 mon.a (mon.0) 747 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[75425]: audit 2023-10-09T09:15:10.348188+0000 mon.a (mon.0) 748 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[78107]: cluster 2023-10-09T09:15:10.205745+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[78107]: cluster 2023-10-09T09:15:10.206900+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[78107]: cluster 2023-10-09T09:15:10.210225+0000 mon.a (mon.0) 746 : cluster 0 osdmap e638: 8 total, 8 up, 8 in 2023-10-09T09:15:11.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[78107]: cluster 2023-10-09T09:15:10.221070+0000 osd.3 (osd.3) 226 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[78107]: cluster 2023-10-09T09:15:10.233748+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[78107]: audit 2023-10-09T09:15:10.346951+0000 mon.a (mon.0) 747 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:11.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:11 smithi163 bash[78107]: audit 2023-10-09T09:15:10.348188+0000 mon.a (mon.0) 748 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:11.489 INFO:tasks.rados.rados.0.smithi163.stdout:2238: done (1 left) 2023-10-09T09:15:11.489 INFO:tasks.rados.rados.0.smithi163.stdout:2240: done (0 left) 2023-10-09T09:15:11.489 INFO:tasks.rados.rados.0.smithi163.stdout:2241: rollback oid 10 current snap is 289 2023-10-09T09:15:11.489 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 284 2023-10-09T09:15:11.489 INFO:tasks.rados.rados.0.smithi163.stdout:2242: read oid 16 snap 251 2023-10-09T09:15:11.489 INFO:tasks.rados.rados.0.smithi163.stdout:2242: expect (ObjNum 686 snap 219 seq_num 686) 2023-10-09T09:15:11.489 INFO:tasks.rados.rados.0.smithi163.stdout:2243: snap_remove snap 288 2023-10-09T09:15:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:11 smithi181 bash[59345]: cluster 2023-10-09T09:15:10.205745+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:11 smithi181 bash[59345]: cluster 2023-10-09T09:15:10.206900+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:11 smithi181 bash[59345]: cluster 2023-10-09T09:15:10.210225+0000 mon.a (mon.0) 746 : cluster 0 osdmap e638: 8 total, 8 up, 8 in 2023-10-09T09:15:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:11 smithi181 bash[59345]: cluster 2023-10-09T09:15:10.221070+0000 osd.3 (osd.3) 226 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:11 smithi181 bash[59345]: cluster 2023-10-09T09:15:10.233748+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e638 with expected crc 2023-10-09T09:15:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:11 smithi181 bash[59345]: audit 2023-10-09T09:15:10.346951+0000 mon.a (mon.0) 747 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:11 smithi181 bash[59345]: audit 2023-10-09T09:15:10.348188+0000 mon.a (mon.0) 748 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:12.359 INFO:tasks.rados.rados.0.smithi163.stdout:2243: done (2 left) 2023-10-09T09:15:12.359 INFO:tasks.rados.rados.0.smithi163.stdout:2244: delete oid 33 current snap is 289 2023-10-09T09:15:12.359 INFO:tasks.rados.rados.0.smithi163.stdout:2241: finishing rollback tid 0 to smithi16367185-10 2023-10-09T09:15:12.359 INFO:tasks.rados.rados.0.smithi163.stdout:2241: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:15:12.359 INFO:tasks.rados.rados.0.smithi163.stdout:2241: finishing rollback tid 2 to smithi16367185-10 2023-10-09T09:15:12.360 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 735 (ObjNum 773 snap 267 seq_num 773) dirty exists 2023-10-09T09:15:12.360 INFO:tasks.rados.rados.0.smithi163.stdout:2242: expect (ObjNum 686 snap 219 seq_num 686) 2023-10-09T09:15:12.485 INFO:tasks.rados.rados.0.smithi163.stdout:2241: done (2 left) 2023-10-09T09:15:12.485 INFO:tasks.rados.rados.0.smithi163.stdout:2242: done (1 left) 2023-10-09T09:15:12.485 INFO:tasks.rados.rados.0.smithi163.stdout:2244: done (0 left) 2023-10-09T09:15:12.486 INFO:tasks.rados.rados.0.smithi163.stdout:2245: write oid 39 current snap is 289 2023-10-09T09:15:12.486 INFO:tasks.rados.rados.0.smithi163.stdout:2245: seq_num 845 ranges {601915=500588,1847402=354513} 2023-10-09T09:15:12.492 INFO:tasks.rados.rados.0.smithi163.stdout:2245: writing smithi16367185-39 from 601915 to 1102503 tid 1 2023-10-09T09:15:12.499 INFO:tasks.rados.rados.0.smithi163.stdout:2245: writing smithi16367185-39 from 1847402 to 2201915 tid 2 2023-10-09T09:15:12.499 INFO:tasks.rados.rados.0.smithi163.stdout:2246: rollback oid 17 current snap is 289 2023-10-09T09:15:12.499 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 287 2023-10-09T09:15:12.499 INFO:tasks.rados.rados.0.smithi163.stdout:2245: finishing write tid 1 to smithi16367185-39 2023-10-09T09:15:12.499 INFO:tasks.rados.rados.0.smithi163.stdout:2247: snap_remove snap 281 2023-10-09T09:15:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[78107]: cluster 2023-10-09T09:15:11.338498+0000 mgr.y (mgr.34107) 251 : cluster 0 pgmap v510: 89 pgs: 2 peering, 87 active+clean; 226 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.8 MiB/s wr, 13 op/s; 2.8 MiB/s, 2 objects/s recovering 2023-10-09T09:15:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[78107]: cluster 2023-10-09T09:15:11.347631+0000 osd.3 (osd.3) 227 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[78107]: cluster 2023-10-09T09:15:11.351172+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[78107]: cluster 2023-10-09T09:15:11.351654+0000 mon.a (mon.0) 749 : cluster 0 osdmap e639: 8 total, 8 up, 8 in 2023-10-09T09:15:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[78107]: cluster 2023-10-09T09:15:11.354879+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[78107]: cluster 2023-10-09T09:15:11.355455+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[78107]: cluster 2023-10-09T09:15:11.359833+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[78107]: audit 2023-10-09T09:15:12.027468+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.163:0/3396532601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[78107]: cluster 2023-10-09T09:15:12.208363+0000 mon.a (mon.0) 750 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/447 objects degraded (8.501%), 5 pgs degraded) 2023-10-09T09:15:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[78107]: cluster 2023-10-09T09:15:12.208434+0000 mon.a (mon.0) 751 : cluster 1 Cluster is now healthy 2023-10-09T09:15:12.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[75425]: cluster 2023-10-09T09:15:11.338498+0000 mgr.y (mgr.34107) 251 : cluster 0 pgmap v510: 89 pgs: 2 peering, 87 active+clean; 226 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.8 MiB/s wr, 13 op/s; 2.8 MiB/s, 2 objects/s recovering 2023-10-09T09:15:12.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[75425]: cluster 2023-10-09T09:15:11.347631+0000 osd.3 (osd.3) 227 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[75425]: cluster 2023-10-09T09:15:11.351172+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[75425]: cluster 2023-10-09T09:15:11.351654+0000 mon.a (mon.0) 749 : cluster 0 osdmap e639: 8 total, 8 up, 8 in 2023-10-09T09:15:12.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[75425]: cluster 2023-10-09T09:15:11.354879+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[75425]: cluster 2023-10-09T09:15:11.355455+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[75425]: cluster 2023-10-09T09:15:11.359833+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[75425]: audit 2023-10-09T09:15:12.027468+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.163:0/3396532601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:12.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[75425]: cluster 2023-10-09T09:15:12.208363+0000 mon.a (mon.0) 750 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/447 objects degraded (8.501%), 5 pgs degraded) 2023-10-09T09:15:12.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:12 smithi163 bash[75425]: cluster 2023-10-09T09:15:12.208434+0000 mon.a (mon.0) 751 : cluster 1 Cluster is now healthy 2023-10-09T09:15:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:12 smithi181 bash[59345]: cluster 2023-10-09T09:15:11.338498+0000 mgr.y (mgr.34107) 251 : cluster 0 pgmap v510: 89 pgs: 2 peering, 87 active+clean; 226 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.8 MiB/s wr, 13 op/s; 2.8 MiB/s, 2 objects/s recovering 2023-10-09T09:15:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:12 smithi181 bash[59345]: cluster 2023-10-09T09:15:11.347631+0000 osd.3 (osd.3) 227 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:12 smithi181 bash[59345]: cluster 2023-10-09T09:15:11.351172+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:12 smithi181 bash[59345]: cluster 2023-10-09T09:15:11.351654+0000 mon.a (mon.0) 749 : cluster 0 osdmap e639: 8 total, 8 up, 8 in 2023-10-09T09:15:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:12 smithi181 bash[59345]: cluster 2023-10-09T09:15:11.354879+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:12 smithi181 bash[59345]: cluster 2023-10-09T09:15:11.355455+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:12 smithi181 bash[59345]: cluster 2023-10-09T09:15:11.359833+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e639 with expected crc 2023-10-09T09:15:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:12 smithi181 bash[59345]: audit 2023-10-09T09:15:12.027468+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.163:0/3396532601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:12 smithi181 bash[59345]: cluster 2023-10-09T09:15:12.208363+0000 mon.a (mon.0) 750 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/447 objects degraded (8.501%), 5 pgs degraded) 2023-10-09T09:15:12.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:12 smithi181 bash[59345]: cluster 2023-10-09T09:15:12.208434+0000 mon.a (mon.0) 751 : cluster 1 Cluster is now healthy 2023-10-09T09:15:13.370 INFO:tasks.rados.rados.0.smithi163.stdout:2247: done (2 left) 2023-10-09T09:15:13.370 INFO:tasks.rados.rados.0.smithi163.stdout:2248: read oid 2 snap -1 2023-10-09T09:15:13.371 INFO:tasks.rados.rados.0.smithi163.stdout:2248: expect (ObjNum 810 snap 279 seq_num 810) 2023-10-09T09:15:13.371 INFO:tasks.rados.rados.0.smithi163.stdout:2249: write oid 9 current snap is 289 2023-10-09T09:15:13.371 INFO:tasks.rados.rados.0.smithi163.stdout:2245: finishing write tid 2 to smithi16367185-39 2023-10-09T09:15:13.371 INFO:tasks.rados.rados.0.smithi163.stdout:2249: seq_num 846 ranges {618722=575837,1905884=312838} 2023-10-09T09:15:13.378 INFO:tasks.rados.rados.0.smithi163.stdout:2249: writing smithi16367185-9 from 618722 to 1194559 tid 1 2023-10-09T09:15:13.385 INFO:tasks.rados.rados.0.smithi163.stdout:2249: writing smithi16367185-9 from 1905884 to 2218722 tid 2 2023-10-09T09:15:13.385 INFO:tasks.rados.rados.0.smithi163.stdout:2250: write oid 35 current snap is 289 2023-10-09T09:15:13.385 INFO:tasks.rados.rados.0.smithi163.stdout:2246: finishing rollback tid 0 to smithi16367185-17 2023-10-09T09:15:13.385 INFO:tasks.rados.rados.0.smithi163.stdout:2245: finishing write tid 3 to smithi16367185-39 2023-10-09T09:15:13.385 INFO:tasks.rados.rados.0.smithi163.stdout:2246: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:15:13.386 INFO:tasks.rados.rados.0.smithi163.stdout:2246: finishing rollback tid 2 to smithi16367185-17 2023-10-09T09:15:13.386 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1120 (ObjNum 805 snap 279 seq_num 805) dirty exists 2023-10-09T09:15:13.386 INFO:tasks.rados.rados.0.smithi163.stdout:2245: finishing write tid 4 to smithi16367185-39 2023-10-09T09:15:13.386 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 983 (ObjNum 845 snap 289 seq_num 845) dirty exists 2023-10-09T09:15:13.386 INFO:tasks.rados.rados.0.smithi163.stdout:2245: left oid 39 (ObjNum 845 snap 289 seq_num 845) 2023-10-09T09:15:13.386 INFO:tasks.rados.rados.0.smithi163.stdout:2249: finishing write tid 1 to smithi16367185-9 2023-10-09T09:15:13.386 INFO:tasks.rados.rados.0.smithi163.stdout:2250: seq_num 847 ranges {635529=651086,1848013=387516} 2023-10-09T09:15:13.393 INFO:tasks.rados.rados.0.smithi163.stdout:2250: writing smithi16367185-35 from 635529 to 1286615 tid 1 2023-10-09T09:15:13.399 INFO:tasks.rados.rados.0.smithi163.stdout:2250: writing smithi16367185-35 from 1848013 to 2235529 tid 2 2023-10-09T09:15:13.399 INFO:tasks.rados.rados.0.smithi163.stdout:2245: done (4 left) 2023-10-09T09:15:13.399 INFO:tasks.rados.rados.0.smithi163.stdout:2246: done (3 left) 2023-10-09T09:15:13.399 INFO:tasks.rados.rados.0.smithi163.stdout:2251: read oid 48 snap 284 2023-10-09T09:15:13.399 INFO:tasks.rados.rados.0.smithi163.stdout:2251: expect (ObjNum 751 snap 255 seq_num 751) 2023-10-09T09:15:13.399 INFO:tasks.rados.rados.0.smithi163.stdout:2252: rollback oid 30 current snap is 289 2023-10-09T09:15:13.399 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 287 2023-10-09T09:15:13.400 INFO:tasks.rados.rados.0.smithi163.stdout:2249: finishing write tid 2 to smithi16367185-9 2023-10-09T09:15:13.400 INFO:tasks.rados.rados.0.smithi163.stdout:2249: finishing write tid 3 to smithi16367185-9 2023-10-09T09:15:13.400 INFO:tasks.rados.rados.0.smithi163.stdout:2249: finishing write tid 4 to smithi16367185-9 2023-10-09T09:15:13.400 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 586 (ObjNum 846 snap 289 seq_num 846) dirty exists 2023-10-09T09:15:13.400 INFO:tasks.rados.rados.0.smithi163.stdout:2249: left oid 9 (ObjNum 846 snap 289 seq_num 846) 2023-10-09T09:15:13.400 INFO:tasks.rados.rados.0.smithi163.stdout:2248: expect (ObjNum 810 snap 279 seq_num 810) 2023-10-09T09:15:13.538 INFO:tasks.rados.rados.0.smithi163.stdout:2250: finishing write tid 1 to smithi16367185-35 2023-10-09T09:15:13.538 INFO:tasks.rados.rados.0.smithi163.stdout:2252: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:15:13.538 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 681 (ObjNum 699172864 snap 21875 seq_num 1618308304) dirty dne 2023-10-09T09:15:13.538 INFO:tasks.rados.rados.0.smithi163.stdout:2250: finishing write tid 2 to smithi16367185-35 2023-10-09T09:15:13.538 INFO:tasks.rados.rados.0.smithi163.stdout:2250: finishing write tid 3 to smithi16367185-35 2023-10-09T09:15:13.538 INFO:tasks.rados.rados.0.smithi163.stdout:2250: finishing write tid 4 to smithi16367185-35 2023-10-09T09:15:13.539 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 608 (ObjNum 847 snap 289 seq_num 847) dirty exists 2023-10-09T09:15:13.539 INFO:tasks.rados.rados.0.smithi163.stdout:2250: left oid 35 (ObjNum 847 snap 289 seq_num 847) 2023-10-09T09:15:13.539 INFO:tasks.rados.rados.0.smithi163.stdout:2251: expect (ObjNum 751 snap 255 seq_num 751) 2023-10-09T09:15:13.684 INFO:tasks.rados.rados.0.smithi163.stdout:2248: done (4 left) 2023-10-09T09:15:13.684 INFO:tasks.rados.rados.0.smithi163.stdout:2249: done (3 left) 2023-10-09T09:15:13.684 INFO:tasks.rados.rados.0.smithi163.stdout:2250: done (2 left) 2023-10-09T09:15:13.684 INFO:tasks.rados.rados.0.smithi163.stdout:2251: done (1 left) 2023-10-09T09:15:13.684 INFO:tasks.rados.rados.0.smithi163.stdout:2252: done (0 left) 2023-10-09T09:15:13.685 INFO:tasks.rados.rados.0.smithi163.stdout:2253: read oid 24 snap -1 2023-10-09T09:15:13.685 INFO:tasks.rados.rados.0.smithi163.stdout:2253: expect deleted 2023-10-09T09:15:13.685 INFO:tasks.rados.rados.0.smithi163.stdout:2254: read oid 15 snap -1 2023-10-09T09:15:13.685 INFO:tasks.rados.rados.0.smithi163.stdout:2254: expect deleted 2023-10-09T09:15:13.685 INFO:tasks.rados.rados.0.smithi163.stdout:2255: read oid 18 snap 272 2023-10-09T09:15:13.685 INFO:tasks.rados.rados.0.smithi163.stdout:2255: expect (ObjNum 782 snap 270 seq_num 782) 2023-10-09T09:15:13.685 INFO:tasks.rados.rados.0.smithi163.stdout:2256: rollback oid 32 current snap is 289 2023-10-09T09:15:13.685 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 282 2023-10-09T09:15:13.685 INFO:tasks.rados.rados.0.smithi163.stdout:2257: write oid 41 current snap is 289 2023-10-09T09:15:13.685 INFO:tasks.rados.rados.0.smithi163.stdout:2257: seq_num 848 ranges {652336=726335,1906495=345841} 2023-10-09T09:15:13.696 INFO:tasks.rados.rados.0.smithi163.stdout:2257: writing smithi16367185-41 from 652336 to 1378671 tid 1 2023-10-09T09:15:13.702 INFO:tasks.rados.rados.0.smithi163.stdout:2257: writing smithi16367185-41 from 1906495 to 2252336 tid 2 2023-10-09T09:15:13.702 INFO:tasks.rados.rados.0.smithi163.stdout:2258: read oid 19 snap 284 2023-10-09T09:15:13.703 INFO:tasks.rados.rados.0.smithi163.stdout:2258: expect deleted 2023-10-09T09:15:13.703 INFO:tasks.rados.rados.0.smithi163.stdout:2259: read oid 26 snap -1 2023-10-09T09:15:13.703 INFO:tasks.rados.rados.0.smithi163.stdout:2259: expect (ObjNum 731 snap 246 seq_num 731) 2023-10-09T09:15:13.703 INFO:tasks.rados.rados.0.smithi163.stdout:2260: delete oid 38 current snap is 289 2023-10-09T09:15:13.703 INFO:tasks.rados.rados.0.smithi163.stdout:2255: expect (ObjNum 782 snap 270 seq_num 782) 2023-10-09T09:15:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:15:13] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.33.4" 2023-10-09T09:15:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[75425]: cluster 2023-10-09T09:15:12.355920+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[75425]: cluster 2023-10-09T09:15:12.356712+0000 osd.3 (osd.3) 228 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[75425]: cluster 2023-10-09T09:15:12.359265+0000 mon.a (mon.0) 752 : cluster 0 osdmap e640: 8 total, 8 up, 8 in 2023-10-09T09:15:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[75425]: cluster 2023-10-09T09:15:12.366552+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[75425]: cluster 2023-10-09T09:15:12.483341+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[75425]: cluster 2023-10-09T09:15:12.488085+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[78107]: cluster 2023-10-09T09:15:12.355920+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[78107]: cluster 2023-10-09T09:15:12.356712+0000 osd.3 (osd.3) 228 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[78107]: cluster 2023-10-09T09:15:12.359265+0000 mon.a (mon.0) 752 : cluster 0 osdmap e640: 8 total, 8 up, 8 in 2023-10-09T09:15:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[78107]: cluster 2023-10-09T09:15:12.366552+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[78107]: cluster 2023-10-09T09:15:12.483341+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:13 smithi163 bash[78107]: cluster 2023-10-09T09:15:12.488085+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:13 smithi181 bash[59345]: cluster 2023-10-09T09:15:12.355920+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:13 smithi181 bash[59345]: cluster 2023-10-09T09:15:12.356712+0000 osd.3 (osd.3) 228 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:13 smithi181 bash[59345]: cluster 2023-10-09T09:15:12.359265+0000 mon.a (mon.0) 752 : cluster 0 osdmap e640: 8 total, 8 up, 8 in 2023-10-09T09:15:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:13 smithi181 bash[59345]: cluster 2023-10-09T09:15:12.366552+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:13 smithi181 bash[59345]: cluster 2023-10-09T09:15:12.483341+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:13 smithi181 bash[59345]: cluster 2023-10-09T09:15:12.488085+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e640 with expected crc 2023-10-09T09:15:13.848 INFO:tasks.rados.rados.0.smithi163.stdout:2256: finishing rollback tid 0 to smithi16367185-32 2023-10-09T09:15:13.848 INFO:tasks.rados.rados.0.smithi163.stdout:2256: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:15:13.848 INFO:tasks.rados.rados.0.smithi163.stdout:2256: finishing rollback tid 2 to smithi16367185-32 2023-10-09T09:15:13.848 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1667 (ObjNum 819 snap 282 seq_num 819) dirty exists 2023-10-09T09:15:13.848 INFO:tasks.rados.rados.0.smithi163.stdout:2257: finishing write tid 1 to smithi16367185-41 2023-10-09T09:15:13.848 INFO:tasks.rados.rados.0.smithi163.stdout:2257: finishing write tid 2 to smithi16367185-41 2023-10-09T09:15:13.848 INFO:tasks.rados.rados.0.smithi163.stdout:2257: finishing write tid 3 to smithi16367185-41 2023-10-09T09:15:13.848 INFO:tasks.rados.rados.0.smithi163.stdout:2257: finishing write tid 4 to smithi16367185-41 2023-10-09T09:15:13.848 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 160 (ObjNum 848 snap 289 seq_num 848) dirty exists 2023-10-09T09:15:13.848 INFO:tasks.rados.rados.0.smithi163.stdout:2257: left oid 41 (ObjNum 848 snap 289 seq_num 848) 2023-10-09T09:15:13.849 INFO:tasks.rados.rados.0.smithi163.stdout:2259: expect (ObjNum 731 snap 246 seq_num 731) 2023-10-09T09:15:14.074 INFO:tasks.rados.rados.0.smithi163.stdout:2253: done (7 left) 2023-10-09T09:15:14.074 INFO:tasks.rados.rados.0.smithi163.stdout:2254: done (6 left) 2023-10-09T09:15:14.074 INFO:tasks.rados.rados.0.smithi163.stdout:2255: done (5 left) 2023-10-09T09:15:14.074 INFO:tasks.rados.rados.0.smithi163.stdout:2256: done (4 left) 2023-10-09T09:15:14.074 INFO:tasks.rados.rados.0.smithi163.stdout:2257: done (3 left) 2023-10-09T09:15:14.074 INFO:tasks.rados.rados.0.smithi163.stdout:2258: done (2 left) 2023-10-09T09:15:14.074 INFO:tasks.rados.rados.0.smithi163.stdout:2259: done (1 left) 2023-10-09T09:15:14.074 INFO:tasks.rados.rados.0.smithi163.stdout:2260: done (0 left) 2023-10-09T09:15:14.074 INFO:tasks.rados.rados.0.smithi163.stdout:2261: write oid 14 current snap is 289 2023-10-09T09:15:14.075 INFO:tasks.rados.rados.0.smithi163.stdout:2261: seq_num 850 ranges {685950=476833,1623459=570617} 2023-10-09T09:15:14.079 INFO:tasks.rados.rados.0.smithi163.stdout:2261: writing smithi16367185-14 from 685950 to 1162783 tid 1 2023-10-09T09:15:14.085 INFO:tasks.rados.rados.0.smithi163.stdout:2261: writing smithi16367185-14 from 1623459 to 2194076 tid 2 2023-10-09T09:15:14.085 INFO:tasks.rados.rados.0.smithi163.stdout:2262: delete oid 41 current snap is 289 2023-10-09T09:15:14.085 INFO:tasks.rados.rados.0.smithi163.stdout:2261: finishing write tid 1 to smithi16367185-14 2023-10-09T09:15:14.088 INFO:tasks.rados.rados.0.smithi163.stdout:2262: done (1 left) 2023-10-09T09:15:14.088 INFO:tasks.rados.rados.0.smithi163.stdout:2263: read oid 46 snap -1 2023-10-09T09:15:14.088 INFO:tasks.rados.rados.0.smithi163.stdout:2263: expect (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:15:14.088 INFO:tasks.rados.rados.0.smithi163.stdout:2264: rollback oid 43 current snap is 289 2023-10-09T09:15:14.088 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 287 2023-10-09T09:15:14.088 INFO:tasks.rados.rados.0.smithi163.stdout:2265: read oid 13 snap 285 2023-10-09T09:15:14.088 INFO:tasks.rados.rados.0.smithi163.stdout:2265: expect (ObjNum 688 snap 219 seq_num 688) 2023-10-09T09:15:14.088 INFO:tasks.rados.rados.0.smithi163.stdout:2266: write oid 27 current snap is 289 2023-10-09T09:15:14.088 INFO:tasks.rados.rados.0.smithi163.stdout:2266: seq_num 852 ranges {719564=743684,2140423=179141} 2023-10-09T09:15:14.096 INFO:tasks.rados.rados.0.smithi163.stdout:2266: writing smithi16367185-27 from 719564 to 1463248 tid 1 2023-10-09T09:15:14.101 INFO:tasks.rados.rados.0.smithi163.stdout:2266: writing smithi16367185-27 from 2140423 to 2319564 tid 2 2023-10-09T09:15:14.101 INFO:tasks.rados.rados.0.smithi163.stdout:2267: write oid 29 current snap is 289 2023-10-09T09:15:14.101 INFO:tasks.rados.rados.0.smithi163.stdout:2261: finishing write tid 2 to smithi16367185-14 2023-10-09T09:15:14.101 INFO:tasks.rados.rados.0.smithi163.stdout:2267: seq_num 853 ranges {736371=418933,1798905=434297} 2023-10-09T09:15:14.108 INFO:tasks.rados.rados.0.smithi163.stdout:2267: writing smithi16367185-29 from 736371 to 1155304 tid 1 2023-10-09T09:15:14.114 INFO:tasks.rados.rados.0.smithi163.stdout:2267: writing smithi16367185-29 from 1798905 to 2233202 tid 2 2023-10-09T09:15:14.114 INFO:tasks.rados.rados.0.smithi163.stdout:2268: snap_create 2023-10-09T09:15:14.115 INFO:tasks.rados.rados.0.smithi163.stdout:2261: finishing write tid 3 to smithi16367185-14 2023-10-09T09:15:14.115 INFO:tasks.rados.rados.0.smithi163.stdout:2261: finishing write tid 4 to smithi16367185-14 2023-10-09T09:15:14.115 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1021 (ObjNum 850 snap 289 seq_num 850) dirty exists 2023-10-09T09:15:14.115 INFO:tasks.rados.rados.0.smithi163.stdout:2261: left oid 14 (ObjNum 850 snap 289 seq_num 850) 2023-10-09T09:15:14.115 INFO:tasks.rados.rados.0.smithi163.stdout:2266: finishing write tid 1 to smithi16367185-27 2023-10-09T09:15:14.115 INFO:tasks.rados.rados.0.smithi163.stdout:2266: finishing write tid 2 to smithi16367185-27 2023-10-09T09:15:14.116 INFO:tasks.rados.rados.0.smithi163.stdout:2266: finishing write tid 3 to smithi16367185-27 2023-10-09T09:15:14.116 INFO:tasks.rados.rados.0.smithi163.stdout:2266: finishing write tid 4 to smithi16367185-27 2023-10-09T09:15:14.116 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 625 (ObjNum 852 snap 289 seq_num 852) dirty exists 2023-10-09T09:15:14.116 INFO:tasks.rados.rados.0.smithi163.stdout:2266: left oid 27 (ObjNum 852 snap 289 seq_num 852) 2023-10-09T09:15:14.116 INFO:tasks.rados.rados.0.smithi163.stdout:2263: expect (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:15:14.239 INFO:tasks.rados.rados.0.smithi163.stdout:2264: finishing rollback tid 0 to smithi16367185-43 2023-10-09T09:15:14.239 INFO:tasks.rados.rados.0.smithi163.stdout:2264: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:15:14.239 INFO:tasks.rados.rados.0.smithi163.stdout:2264: finishing rollback tid 2 to smithi16367185-43 2023-10-09T09:15:14.240 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 1025 (ObjNum 829 snap 286 seq_num 829) dirty exists 2023-10-09T09:15:14.240 INFO:tasks.rados.rados.0.smithi163.stdout:2267: finishing write tid 1 to smithi16367185-29 2023-10-09T09:15:14.240 INFO:tasks.rados.rados.0.smithi163.stdout:2265: expect (ObjNum 688 snap 219 seq_num 688) 2023-10-09T09:15:14.359 INFO:tasks.rados.rados.0.smithi163.stdout:2267: finishing write tid 2 to smithi16367185-29 2023-10-09T09:15:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:2267: finishing write tid 3 to smithi16367185-29 2023-10-09T09:15:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:2267: finishing write tid 4 to smithi16367185-29 2023-10-09T09:15:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1096 (ObjNum 853 snap 289 seq_num 853) dirty exists 2023-10-09T09:15:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:2267: left oid 29 (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:15:14.506 INFO:tasks.rados.rados.0.smithi163.stdout:2261: done (6 left) 2023-10-09T09:15:14.506 INFO:tasks.rados.rados.0.smithi163.stdout:2263: done (5 left) 2023-10-09T09:15:14.506 INFO:tasks.rados.rados.0.smithi163.stdout:2264: done (4 left) 2023-10-09T09:15:14.506 INFO:tasks.rados.rados.0.smithi163.stdout:2265: done (3 left) 2023-10-09T09:15:14.506 INFO:tasks.rados.rados.0.smithi163.stdout:2266: done (2 left) 2023-10-09T09:15:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:2267: done (1 left) 2023-10-09T09:15:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:2268: done (0 left) 2023-10-09T09:15:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:2269: snap_create 2023-10-09T09:15:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: cluster 2023-10-09T09:15:13.339286+0000 mgr.y (mgr.34107) 252 : cluster 0 pgmap v513: 89 pgs: 89 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.8 MiB/s wr, 14 op/s; 2.8 MiB/s, 2 objects/s recovering 2023-10-09T09:15:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: cluster 2023-10-09T09:15:13.366918+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e641 with expected crc 2023-10-09T09:15:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: cluster 2023-10-09T09:15:13.368155+0000 osd.3 (osd.3) 229 : cluster 3 failed to encode map e641 with expected crc 2023-10-09T09:15:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: cluster 2023-10-09T09:15:13.370317+0000 mon.a (mon.0) 753 : cluster 0 osdmap e641: 8 total, 8 up, 8 in 2023-10-09T09:15:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: cluster 2023-10-09T09:15:13.370984+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e641 with expected crc 2023-10-09T09:15:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: audit 2023-10-09T09:15:13.483484+0000 mon.a (mon.0) 754 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: audit 2023-10-09T09:15:13.491312+0000 mon.a (mon.0) 755 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: audit 2023-10-09T09:15:13.492598+0000 mon.a (mon.0) 756 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: audit 2023-10-09T09:15:13.493654+0000 mon.a (mon.0) 757 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: audit 2023-10-09T09:15:13.500589+0000 mon.a (mon.0) 758 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: audit 2023-10-09T09:15:13.582185+0000 mon.a (mon.0) 759 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: audit 2023-10-09T09:15:13.587331+0000 mon.a (mon.0) 760 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: audit 2023-10-09T09:15:13.590097+0000 mon.a (mon.0) 761 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: audit 2023-10-09T09:15:13.592007+0000 mon.a (mon.0) 762 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[78107]: audit 2023-10-09T09:15:13.594309+0000 mon.a (mon.0) 763 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: cluster 2023-10-09T09:15:13.339286+0000 mgr.y (mgr.34107) 252 : cluster 0 pgmap v513: 89 pgs: 89 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.8 MiB/s wr, 14 op/s; 2.8 MiB/s, 2 objects/s recovering 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: cluster 2023-10-09T09:15:13.366918+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e641 with expected crc 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: cluster 2023-10-09T09:15:13.368155+0000 osd.3 (osd.3) 229 : cluster 3 failed to encode map e641 with expected crc 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: cluster 2023-10-09T09:15:13.370317+0000 mon.a (mon.0) 753 : cluster 0 osdmap e641: 8 total, 8 up, 8 in 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: cluster 2023-10-09T09:15:13.370984+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e641 with expected crc 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: audit 2023-10-09T09:15:13.483484+0000 mon.a (mon.0) 754 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: audit 2023-10-09T09:15:13.491312+0000 mon.a (mon.0) 755 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: audit 2023-10-09T09:15:13.492598+0000 mon.a (mon.0) 756 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: audit 2023-10-09T09:15:13.493654+0000 mon.a (mon.0) 757 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: audit 2023-10-09T09:15:13.500589+0000 mon.a (mon.0) 758 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:14.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: audit 2023-10-09T09:15:13.582185+0000 mon.a (mon.0) 759 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:14.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: audit 2023-10-09T09:15:13.587331+0000 mon.a (mon.0) 760 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:14.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: audit 2023-10-09T09:15:13.590097+0000 mon.a (mon.0) 761 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:14.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: audit 2023-10-09T09:15:13.592007+0000 mon.a (mon.0) 762 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:14.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:14 smithi163 bash[75425]: audit 2023-10-09T09:15:13.594309+0000 mon.a (mon.0) 763 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:15:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: cluster 2023-10-09T09:15:13.339286+0000 mgr.y (mgr.34107) 252 : cluster 0 pgmap v513: 89 pgs: 89 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.8 MiB/s wr, 14 op/s; 2.8 MiB/s, 2 objects/s recovering 2023-10-09T09:15:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: cluster 2023-10-09T09:15:13.366918+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e641 with expected crc 2023-10-09T09:15:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: cluster 2023-10-09T09:15:13.368155+0000 osd.3 (osd.3) 229 : cluster 3 failed to encode map e641 with expected crc 2023-10-09T09:15:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: cluster 2023-10-09T09:15:13.370317+0000 mon.a (mon.0) 753 : cluster 0 osdmap e641: 8 total, 8 up, 8 in 2023-10-09T09:15:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: cluster 2023-10-09T09:15:13.370984+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e641 with expected crc 2023-10-09T09:15:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: audit 2023-10-09T09:15:13.483484+0000 mon.a (mon.0) 754 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: audit 2023-10-09T09:15:13.491312+0000 mon.a (mon.0) 755 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: audit 2023-10-09T09:15:13.492598+0000 mon.a (mon.0) 756 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: audit 2023-10-09T09:15:13.493654+0000 mon.a (mon.0) 757 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: audit 2023-10-09T09:15:13.500589+0000 mon.a (mon.0) 758 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: audit 2023-10-09T09:15:13.582185+0000 mon.a (mon.0) 759 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: audit 2023-10-09T09:15:13.587331+0000 mon.a (mon.0) 760 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: audit 2023-10-09T09:15:13.590097+0000 mon.a (mon.0) 761 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: audit 2023-10-09T09:15:13.592007+0000 mon.a (mon.0) 762 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:14 smithi181 bash[59345]: audit 2023-10-09T09:15:13.594309+0000 mon.a (mon.0) 763 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:15:15.115 INFO:tasks.rados.rados.0.smithi163.stdout:2269: done (0 left) 2023-10-09T09:15:15.115 INFO:tasks.rados.rados.0.smithi163.stdout:2270: delete oid 9 current snap is 291 2023-10-09T09:15:15.123 INFO:tasks.rados.rados.0.smithi163.stdout:2270: done (0 left) 2023-10-09T09:15:15.123 INFO:tasks.rados.rados.0.smithi163.stdout:2271: write oid 48 current snap is 291 2023-10-09T09:15:15.123 INFO:tasks.rados.rados.0.smithi163.stdout:2271: seq_num 855 ranges {769985=569431,1915869=437966} 2023-10-09T09:15:15.132 INFO:tasks.rados.rados.0.smithi163.stdout:2271: writing smithi16367185-48 from 769985 to 1339416 tid 1 2023-10-09T09:15:15.141 INFO:tasks.rados.rados.0.smithi163.stdout:2271: writing smithi16367185-48 from 1915869 to 2353835 tid 2 2023-10-09T09:15:15.141 INFO:tasks.rados.rados.0.smithi163.stdout:2272: snap_remove snap 284 2023-10-09T09:15:15.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:15 smithi163 bash[78107]: audit 2023-10-09T09:15:13.594546+0000 mgr.y (mgr.34107) 253 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:15:15.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:15 smithi163 bash[78107]: cephadm 2023-10-09T09:15:13.595868+0000 mgr.y (mgr.34107) 254 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T09:15:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:15 smithi163 bash[78107]: audit 2023-10-09T09:15:14.415986+0000 mon.a (mon.0) 764 : audit 0 from='client.? 172.21.15.163:0/3955803998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:15 smithi163 bash[78107]: cluster 2023-10-09T09:15:14.506093+0000 mon.a (mon.0) 765 : cluster 0 osdmap e642: 8 total, 8 up, 8 in 2023-10-09T09:15:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:15 smithi163 bash[78107]: cluster 2023-10-09T09:15:15.114721+0000 mon.a (mon.0) 766 : cluster 0 osdmap e643: 8 total, 8 up, 8 in 2023-10-09T09:15:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:15 smithi163 bash[75425]: audit 2023-10-09T09:15:13.594546+0000 mgr.y (mgr.34107) 253 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:15:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:15 smithi163 bash[75425]: cephadm 2023-10-09T09:15:13.595868+0000 mgr.y (mgr.34107) 254 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T09:15:15.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:15 smithi163 bash[75425]: audit 2023-10-09T09:15:14.415986+0000 mon.a (mon.0) 764 : audit 0 from='client.? 172.21.15.163:0/3955803998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:15.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:15 smithi163 bash[75425]: cluster 2023-10-09T09:15:14.506093+0000 mon.a (mon.0) 765 : cluster 0 osdmap e642: 8 total, 8 up, 8 in 2023-10-09T09:15:15.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:15 smithi163 bash[75425]: cluster 2023-10-09T09:15:15.114721+0000 mon.a (mon.0) 766 : cluster 0 osdmap e643: 8 total, 8 up, 8 in 2023-10-09T09:15:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:15 smithi181 bash[59345]: audit 2023-10-09T09:15:13.594546+0000 mgr.y (mgr.34107) 253 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:15:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:15 smithi181 bash[59345]: cephadm 2023-10-09T09:15:13.595868+0000 mgr.y (mgr.34107) 254 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T09:15:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:15 smithi181 bash[59345]: audit 2023-10-09T09:15:14.415986+0000 mon.a (mon.0) 764 : audit 0 from='client.? 172.21.15.163:0/3955803998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:15 smithi181 bash[59345]: cluster 2023-10-09T09:15:14.506093+0000 mon.a (mon.0) 765 : cluster 0 osdmap e642: 8 total, 8 up, 8 in 2023-10-09T09:15:15.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:15 smithi181 bash[59345]: cluster 2023-10-09T09:15:15.114721+0000 mon.a (mon.0) 766 : cluster 0 osdmap e643: 8 total, 8 up, 8 in 2023-10-09T09:15:16.135 INFO:tasks.rados.rados.0.smithi163.stdout:2272: done (1 left) 2023-10-09T09:15:16.135 INFO:tasks.rados.rados.0.smithi163.stdout:2273: read oid 34 snap -1 2023-10-09T09:15:16.136 INFO:tasks.rados.rados.0.smithi163.stdout:2273: expect deleted 2023-10-09T09:15:16.136 INFO:tasks.rados.rados.0.smithi163.stdout:2274: snap_remove snap 278 2023-10-09T09:15:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[78107]: cluster 2023-10-09T09:15:14.501304+0000 osd.3 (osd.3) 230 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[78107]: cluster 2023-10-09T09:15:15.109571+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[78107]: cluster 2023-10-09T09:15:15.110173+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[78107]: cluster 2023-10-09T09:15:15.116620+0000 osd.3 (osd.3) 231 : cluster 3 failed to encode map e643 with expected crc 2023-10-09T09:15:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[78107]: cluster 2023-10-09T09:15:15.182225+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[78107]: cluster 2023-10-09T09:15:15.340030+0000 mgr.y (mgr.34107) 255 : cluster 0 pgmap v517: 89 pgs: 89 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 935 KiB/s wr, 14 op/s 2023-10-09T09:15:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[78107]: audit 2023-10-09T09:15:15.774046+0000 mon.a (mon.0) 767 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[78107]: audit 2023-10-09T09:15:15.775495+0000 mon.a (mon.0) 768 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:15:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[78107]: audit 2023-10-09T09:15:15.776675+0000 mon.a (mon.0) 769 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[78107]: cluster 2023-10-09T09:15:16.135546+0000 mon.a (mon.0) 770 : cluster 0 osdmap e644: 8 total, 8 up, 8 in 2023-10-09T09:15:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[75425]: cluster 2023-10-09T09:15:14.501304+0000 osd.3 (osd.3) 230 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[75425]: cluster 2023-10-09T09:15:15.109571+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[75425]: cluster 2023-10-09T09:15:15.110173+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[75425]: cluster 2023-10-09T09:15:15.116620+0000 osd.3 (osd.3) 231 : cluster 3 failed to encode map e643 with expected crc 2023-10-09T09:15:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[75425]: cluster 2023-10-09T09:15:15.182225+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[75425]: cluster 2023-10-09T09:15:15.340030+0000 mgr.y (mgr.34107) 255 : cluster 0 pgmap v517: 89 pgs: 89 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 935 KiB/s wr, 14 op/s 2023-10-09T09:15:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[75425]: audit 2023-10-09T09:15:15.774046+0000 mon.a (mon.0) 767 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:16.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[75425]: audit 2023-10-09T09:15:15.775495+0000 mon.a (mon.0) 768 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:15:16.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[75425]: audit 2023-10-09T09:15:15.776675+0000 mon.a (mon.0) 769 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:16.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:16 smithi163 bash[75425]: cluster 2023-10-09T09:15:16.135546+0000 mon.a (mon.0) 770 : cluster 0 osdmap e644: 8 total, 8 up, 8 in 2023-10-09T09:15:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:16 smithi181 bash[59345]: cluster 2023-10-09T09:15:14.501304+0000 osd.3 (osd.3) 230 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:16 smithi181 bash[59345]: cluster 2023-10-09T09:15:15.109571+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:16 smithi181 bash[59345]: cluster 2023-10-09T09:15:15.110173+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:16 smithi181 bash[59345]: cluster 2023-10-09T09:15:15.116620+0000 osd.3 (osd.3) 231 : cluster 3 failed to encode map e643 with expected crc 2023-10-09T09:15:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:16 smithi181 bash[59345]: cluster 2023-10-09T09:15:15.182225+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e642 with expected crc 2023-10-09T09:15:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:16 smithi181 bash[59345]: cluster 2023-10-09T09:15:15.340030+0000 mgr.y (mgr.34107) 255 : cluster 0 pgmap v517: 89 pgs: 89 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 935 KiB/s wr, 14 op/s 2023-10-09T09:15:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:16 smithi181 bash[59345]: audit 2023-10-09T09:15:15.774046+0000 mon.a (mon.0) 767 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:16 smithi181 bash[59345]: audit 2023-10-09T09:15:15.775495+0000 mon.a (mon.0) 768 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:15:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:16 smithi181 bash[59345]: audit 2023-10-09T09:15:15.776675+0000 mon.a (mon.0) 769 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:16.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:16 smithi181 bash[59345]: cluster 2023-10-09T09:15:16.135546+0000 mon.a (mon.0) 770 : cluster 0 osdmap e644: 8 total, 8 up, 8 in 2023-10-09T09:15:17.138 INFO:tasks.rados.rados.0.smithi163.stdout:2274: done (2 left) 2023-10-09T09:15:17.138 INFO:tasks.rados.rados.0.smithi163.stdout:2275: snap_remove snap 283 2023-10-09T09:15:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[75425]: cephadm 2023-10-09T09:15:15.764542+0000 mgr.y (mgr.34107) 256 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T09:15:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[75425]: cephadm 2023-10-09T09:15:15.779620+0000 mgr.y (mgr.34107) 257 : cephadm 1 Deploying daemon osd.3 on smithi163 2023-10-09T09:15:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[75425]: cluster 2023-10-09T09:15:16.131007+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e644 with expected crc 2023-10-09T09:15:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[75425]: cluster 2023-10-09T09:15:16.189632+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e644 with expected crc 2023-10-09T09:15:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[75425]: audit 2023-10-09T09:15:16.785556+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.163:0/4224595629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:17.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[75425]: cluster 2023-10-09T09:15:17.133850+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e645 with expected crc 2023-10-09T09:15:17.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[75425]: cluster 2023-10-09T09:15:17.138225+0000 mon.a (mon.0) 771 : cluster 0 osdmap e645: 8 total, 8 up, 8 in 2023-10-09T09:15:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[78107]: cephadm 2023-10-09T09:15:15.764542+0000 mgr.y (mgr.34107) 256 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T09:15:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[78107]: cephadm 2023-10-09T09:15:15.779620+0000 mgr.y (mgr.34107) 257 : cephadm 1 Deploying daemon osd.3 on smithi163 2023-10-09T09:15:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[78107]: cluster 2023-10-09T09:15:16.131007+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e644 with expected crc 2023-10-09T09:15:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[78107]: cluster 2023-10-09T09:15:16.189632+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e644 with expected crc 2023-10-09T09:15:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[78107]: audit 2023-10-09T09:15:16.785556+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.163:0/4224595629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[78107]: cluster 2023-10-09T09:15:17.133850+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e645 with expected crc 2023-10-09T09:15:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:17 smithi163 bash[78107]: cluster 2023-10-09T09:15:17.138225+0000 mon.a (mon.0) 771 : cluster 0 osdmap e645: 8 total, 8 up, 8 in 2023-10-09T09:15:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:17 smithi181 bash[59345]: cephadm 2023-10-09T09:15:15.764542+0000 mgr.y (mgr.34107) 256 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T09:15:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:17 smithi181 bash[59345]: cephadm 2023-10-09T09:15:15.779620+0000 mgr.y (mgr.34107) 257 : cephadm 1 Deploying daemon osd.3 on smithi163 2023-10-09T09:15:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:17 smithi181 bash[59345]: cluster 2023-10-09T09:15:16.131007+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e644 with expected crc 2023-10-09T09:15:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:17 smithi181 bash[59345]: cluster 2023-10-09T09:15:16.189632+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e644 with expected crc 2023-10-09T09:15:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:17 smithi181 bash[59345]: audit 2023-10-09T09:15:16.785556+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.163:0/4224595629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:17 smithi181 bash[59345]: cluster 2023-10-09T09:15:17.133850+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e645 with expected crc 2023-10-09T09:15:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:17 smithi181 bash[59345]: cluster 2023-10-09T09:15:17.138225+0000 mon.a (mon.0) 771 : cluster 0 osdmap e645: 8 total, 8 up, 8 in 2023-10-09T09:15:18.148 INFO:tasks.rados.rados.0.smithi163.stdout:2275: done (2 left) 2023-10-09T09:15:18.148 INFO:tasks.rados.rados.0.smithi163.stdout:2276: write oid 7 current snap is 291 2023-10-09T09:15:18.148 INFO:tasks.rados.rados.0.smithi163.stdout:2276: seq_num 856 ranges {786792=644680,1857998=528794} 2023-10-09T09:15:18.157 INFO:tasks.rados.rados.0.smithi163.stdout:2276: writing smithi16367185-7 from 786792 to 1431472 tid 1 2023-10-09T09:15:18.164 INFO:tasks.rados.rados.0.smithi163.stdout:2276: writing smithi16367185-7 from 1857998 to 2386792 tid 2 2023-10-09T09:15:18.164 INFO:tasks.rados.rados.0.smithi163.stdout:2277: snap_remove snap 251 2023-10-09T09:15:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:18 smithi163 bash[78107]: cluster 2023-10-09T09:15:17.134598+0000 osd.3 (osd.3) 232 : cluster 3 failed to encode map e645 with expected crc 2023-10-09T09:15:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:18 smithi163 bash[78107]: cluster 2023-10-09T09:15:17.227101+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e645 with expected crc 2023-10-09T09:15:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:18 smithi163 bash[78107]: cluster 2023-10-09T09:15:17.340743+0000 mgr.y (mgr.34107) 258 : cluster 0 pgmap v520: 89 pgs: 89 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.1 MiB/s wr, 31 op/s 2023-10-09T09:15:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:18 smithi163 bash[78107]: cluster 2023-10-09T09:15:18.144509+0000 mon.a (mon.0) 772 : cluster 0 osdmap e646: 8 total, 8 up, 8 in 2023-10-09T09:15:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:18 smithi163 bash[78107]: cluster 2023-10-09T09:15:18.145701+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:18 smithi163 bash[75425]: cluster 2023-10-09T09:15:17.134598+0000 osd.3 (osd.3) 232 : cluster 3 failed to encode map e645 with expected crc 2023-10-09T09:15:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:18 smithi163 bash[75425]: cluster 2023-10-09T09:15:17.227101+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e645 with expected crc 2023-10-09T09:15:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:18 smithi163 bash[75425]: cluster 2023-10-09T09:15:17.340743+0000 mgr.y (mgr.34107) 258 : cluster 0 pgmap v520: 89 pgs: 89 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.1 MiB/s wr, 31 op/s 2023-10-09T09:15:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:18 smithi163 bash[75425]: cluster 2023-10-09T09:15:18.144509+0000 mon.a (mon.0) 772 : cluster 0 osdmap e646: 8 total, 8 up, 8 in 2023-10-09T09:15:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:18 smithi163 bash[75425]: cluster 2023-10-09T09:15:18.145701+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:18 smithi181 bash[59345]: cluster 2023-10-09T09:15:17.134598+0000 osd.3 (osd.3) 232 : cluster 3 failed to encode map e645 with expected crc 2023-10-09T09:15:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:18 smithi181 bash[59345]: cluster 2023-10-09T09:15:17.227101+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e645 with expected crc 2023-10-09T09:15:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:18 smithi181 bash[59345]: cluster 2023-10-09T09:15:17.340743+0000 mgr.y (mgr.34107) 258 : cluster 0 pgmap v520: 89 pgs: 89 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.1 MiB/s wr, 31 op/s 2023-10-09T09:15:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:18 smithi181 bash[59345]: cluster 2023-10-09T09:15:18.144509+0000 mon.a (mon.0) 772 : cluster 0 osdmap e646: 8 total, 8 up, 8 in 2023-10-09T09:15:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:18 smithi181 bash[59345]: cluster 2023-10-09T09:15:18.145701+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:19.153 INFO:tasks.rados.rados.0.smithi163.stdout:2277: done (3 left) 2023-10-09T09:15:19.153 INFO:tasks.rados.rados.0.smithi163.stdout:2278: write oid 19 current snap is 291 2023-10-09T09:15:19.153 INFO:tasks.rados.rados.0.smithi163.stdout:2271: finishing write tid 1 to smithi16367185-48 2023-10-09T09:15:19.153 INFO:tasks.rados.rados.0.smithi163.stdout:2271: finishing write tid 2 to smithi16367185-48 2023-10-09T09:15:19.153 INFO:tasks.rados.rados.0.smithi163.stdout:2271: finishing write tid 3 to smithi16367185-48 2023-10-09T09:15:19.154 INFO:tasks.rados.rados.0.smithi163.stdout:2271: finishing write tid 4 to smithi16367185-48 2023-10-09T09:15:19.154 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 1671 (ObjNum 855 snap 291 seq_num 855) dirty exists 2023-10-09T09:15:19.154 INFO:tasks.rados.rados.0.smithi163.stdout:2271: left oid 48 (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:15:19.154 INFO:tasks.rados.rados.0.smithi163.stdout:2276: finishing write tid 1 to smithi16367185-7 2023-10-09T09:15:19.154 INFO:tasks.rados.rados.0.smithi163.stdout:2278: seq_num 857 ranges {403599=719929,1916480=441635} 2023-10-09T09:15:19.160 INFO:tasks.rados.rados.0.smithi163.stdout:2278: writing smithi16367185-19 from 403599 to 1123528 tid 1 2023-10-09T09:15:19.170 INFO:tasks.rados.rados.0.smithi163.stdout:2278: writing smithi16367185-19 from 1916480 to 2358115 tid 2 2023-10-09T09:15:19.170 INFO:tasks.rados.rados.0.smithi163.stdout:2271: done (3 left) 2023-10-09T09:15:19.170 INFO:tasks.rados.rados.0.smithi163.stdout:2273: done (2 left) 2023-10-09T09:15:19.170 INFO:tasks.rados.rados.0.smithi163.stdout:2279: snap_create 2023-10-09T09:15:19.170 INFO:tasks.rados.rados.0.smithi163.stdout:2276: finishing write tid 2 to smithi16367185-7 2023-10-09T09:15:19.171 INFO:tasks.rados.rados.0.smithi163.stdout:2276: finishing write tid 3 to smithi16367185-7 2023-10-09T09:15:19.171 INFO:tasks.rados.rados.0.smithi163.stdout:2276: finishing write tid 4 to smithi16367185-7 2023-10-09T09:15:19.171 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 519 (ObjNum 856 snap 291 seq_num 856) dirty exists 2023-10-09T09:15:19.171 INFO:tasks.rados.rados.0.smithi163.stdout:2276: left oid 7 (ObjNum 856 snap 291 seq_num 856) 2023-10-09T09:15:19.171 INFO:tasks.rados.rados.0.smithi163.stdout:2278: finishing write tid 1 to smithi16367185-19 2023-10-09T09:15:19.174 INFO:tasks.rados.rados.0.smithi163.stdout:2278: finishing write tid 2 to smithi16367185-19 2023-10-09T09:15:19.175 INFO:tasks.rados.rados.0.smithi163.stdout:2278: finishing write tid 3 to smithi16367185-19 2023-10-09T09:15:19.175 INFO:tasks.rados.rados.0.smithi163.stdout:2278: finishing write tid 4 to smithi16367185-19 2023-10-09T09:15:19.175 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 628 (ObjNum 857 snap 291 seq_num 857) dirty exists 2023-10-09T09:15:19.175 INFO:tasks.rados.rados.0.smithi163.stdout:2278: left oid 19 (ObjNum 857 snap 291 seq_num 857) 2023-10-09T09:15:19.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[78107]: cluster 2023-10-09T09:15:18.142389+0000 osd.3 (osd.3) 233 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:19.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[78107]: cluster 2023-10-09T09:15:18.145713+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:19.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[78107]: cluster 2023-10-09T09:15:18.250370+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:19.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[78107]: audit 2023-10-09T09:15:19.140138+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.163:0/34536709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:19.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[78107]: cluster 2023-10-09T09:15:19.146824+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e647 with expected crc 2023-10-09T09:15:19.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[78107]: cluster 2023-10-09T09:15:19.152999+0000 mon.a (mon.0) 773 : cluster 0 osdmap e647: 8 total, 8 up, 8 in 2023-10-09T09:15:19.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[75425]: cluster 2023-10-09T09:15:18.142389+0000 osd.3 (osd.3) 233 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:19.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[75425]: cluster 2023-10-09T09:15:18.145713+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:19.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[75425]: cluster 2023-10-09T09:15:18.250370+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:19.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[75425]: audit 2023-10-09T09:15:19.140138+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.163:0/34536709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:19.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[75425]: cluster 2023-10-09T09:15:19.146824+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e647 with expected crc 2023-10-09T09:15:19.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:19 smithi163 bash[75425]: cluster 2023-10-09T09:15:19.152999+0000 mon.a (mon.0) 773 : cluster 0 osdmap e647: 8 total, 8 up, 8 in 2023-10-09T09:15:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:19 smithi181 bash[59345]: cluster 2023-10-09T09:15:18.142389+0000 osd.3 (osd.3) 233 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:19 smithi181 bash[59345]: cluster 2023-10-09T09:15:18.145713+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:19 smithi181 bash[59345]: cluster 2023-10-09T09:15:18.250370+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e646 with expected crc 2023-10-09T09:15:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:19 smithi181 bash[59345]: audit 2023-10-09T09:15:19.140138+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.163:0/34536709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:19 smithi181 bash[59345]: cluster 2023-10-09T09:15:19.146824+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e647 with expected crc 2023-10-09T09:15:19.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:19 smithi181 bash[59345]: cluster 2023-10-09T09:15:19.152999+0000 mon.a (mon.0) 773 : cluster 0 osdmap e647: 8 total, 8 up, 8 in 2023-10-09T09:15:20.132 INFO:tasks.rados.rados.0.smithi163.stdout:2276: done (2 left) 2023-10-09T09:15:20.132 INFO:tasks.rados.rados.0.smithi163.stdout:2278: done (1 left) 2023-10-09T09:15:20.132 INFO:tasks.rados.rados.0.smithi163.stdout:2279: done (0 left) 2023-10-09T09:15:20.133 INFO:tasks.rados.rados.0.smithi163.stdout:2280: delete oid 11 current snap is 292 2023-10-09T09:15:20.145 INFO:tasks.rados.rados.0.smithi163.stdout:2280: done (0 left) 2023-10-09T09:15:20.146 INFO:tasks.rados.rados.0.smithi163.stdout:2281: rollback oid 44 current snap is 292 2023-10-09T09:15:20.146 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 44 to 272 2023-10-09T09:15:20.146 INFO:tasks.rados.rados.0.smithi163.stdout:2282: write oid 45 current snap is 292 2023-10-09T09:15:20.146 INFO:tasks.rados.rados.0.smithi163.stdout:2282: seq_num 859 ranges {437213=470427,1633444=728951} 2023-10-09T09:15:20.152 INFO:tasks.rados.rados.0.smithi163.stdout:2282: writing smithi16367185-45 from 437213 to 907640 tid 1 2023-10-09T09:15:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:2282: writing smithi16367185-45 from 1633444 to 2362395 tid 2 2023-10-09T09:15:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:2281: finishing rollback tid 0 to smithi16367185-44 2023-10-09T09:15:20.163 INFO:tasks.rados.rados.0.smithi163.stdout:2281: finishing rollback tid 1 to smithi16367185-44 2023-10-09T09:15:20.163 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1708 (ObjNum 1632014808 snap 32603 seq_num 7) dirty dne 2023-10-09T09:15:20.163 INFO:tasks.rados.rados.0.smithi163.stdout:2282: finishing write tid 1 to smithi16367185-45 2023-10-09T09:15:20.164 INFO:tasks.rados.rados.0.smithi163.stdout:2281: done (1 left) 2023-10-09T09:15:20.164 INFO:tasks.rados.rados.0.smithi163.stdout:2283: rollback oid 13 current snap is 292 2023-10-09T09:15:20.164 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 13 to 285 2023-10-09T09:15:20.165 INFO:tasks.rados.rados.0.smithi163.stdout:2284: delete oid 18 current snap is 292 2023-10-09T09:15:20.166 INFO:tasks.rados.rados.0.smithi163.stdout:2283: finishing rollback tid 1 to smithi16367185-13 2023-10-09T09:15:20.166 INFO:tasks.rados.rados.0.smithi163.stdout:2283: finishing rollback tid 2 to smithi16367185-13 2023-10-09T09:15:20.166 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 307 (ObjNum 688 snap 219 seq_num 688) dirty exists 2023-10-09T09:15:20.170 INFO:tasks.rados.rados.0.smithi163.stdout:2283: done (2 left) 2023-10-09T09:15:20.171 INFO:tasks.rados.rados.0.smithi163.stdout:2284: done (1 left) 2023-10-09T09:15:20.171 INFO:tasks.rados.rados.0.smithi163.stdout:2285: snap_create 2023-10-09T09:15:20.172 INFO:tasks.rados.rados.0.smithi163.stdout:2282: finishing write tid 2 to smithi16367185-45 2023-10-09T09:15:20.172 INFO:tasks.rados.rados.0.smithi163.stdout:2282: finishing write tid 3 to smithi16367185-45 2023-10-09T09:15:20.172 INFO:tasks.rados.rados.0.smithi163.stdout:2282: finishing write tid 4 to smithi16367185-45 2023-10-09T09:15:20.173 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 1711 (ObjNum 859 snap 292 seq_num 859) dirty exists 2023-10-09T09:15:20.173 INFO:tasks.rados.rados.0.smithi163.stdout:2282: left oid 45 (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:15:20.218 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:20 smithi163 systemd[1]: Stopping Ceph osd.3 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:15:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[78107]: cluster 2023-10-09T09:15:19.149828+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e647 with expected crc 2023-10-09T09:15:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[78107]: cluster 2023-10-09T09:15:19.341417+0000 mgr.y (mgr.34107) 259 : cluster 0 pgmap v523: 89 pgs: 89 active+clean; 226 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2023-10-09T09:15:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[78107]: cluster 2023-10-09T09:15:20.126776+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[78107]: cluster 2023-10-09T09:15:20.131307+0000 mon.a (mon.0) 774 : cluster 0 osdmap e648: 8 total, 8 up, 8 in 2023-10-09T09:15:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[78107]: cluster 2023-10-09T09:15:20.239388+0000 mon.a (mon.0) 775 : cluster 1 osd.3 marked itself down and dead 2023-10-09T09:15:20.719 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[49675]: debug 2023-10-09T09:15:20.238+0000 7ff081aa6700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:15:20.719 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[49675]: debug 2023-10-09T09:15:20.238+0000 7ff081aa6700 -1 osd.3 648 *** Got signal Terminated *** 2023-10-09T09:15:20.719 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[49675]: debug 2023-10-09T09:15:20.238+0000 7ff081aa6700 -1 osd.3 648 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:15:20.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[75425]: cluster 2023-10-09T09:15:19.149828+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e647 with expected crc 2023-10-09T09:15:20.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[75425]: cluster 2023-10-09T09:15:19.341417+0000 mgr.y (mgr.34107) 259 : cluster 0 pgmap v523: 89 pgs: 89 active+clean; 226 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2023-10-09T09:15:20.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[75425]: cluster 2023-10-09T09:15:20.126776+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:20.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[75425]: cluster 2023-10-09T09:15:20.131307+0000 mon.a (mon.0) 774 : cluster 0 osdmap e648: 8 total, 8 up, 8 in 2023-10-09T09:15:20.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:20 smithi163 bash[75425]: cluster 2023-10-09T09:15:20.239388+0000 mon.a (mon.0) 775 : cluster 1 osd.3 marked itself down and dead 2023-10-09T09:15:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:20 smithi181 bash[59345]: cluster 2023-10-09T09:15:19.149828+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e647 with expected crc 2023-10-09T09:15:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:20 smithi181 bash[59345]: cluster 2023-10-09T09:15:19.341417+0000 mgr.y (mgr.34107) 259 : cluster 0 pgmap v523: 89 pgs: 89 active+clean; 226 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2023-10-09T09:15:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:20 smithi181 bash[59345]: cluster 2023-10-09T09:15:20.126776+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:20.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:20 smithi181 bash[59345]: cluster 2023-10-09T09:15:20.131307+0000 mon.a (mon.0) 774 : cluster 0 osdmap e648: 8 total, 8 up, 8 in 2023-10-09T09:15:20.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:20 smithi181 bash[59345]: cluster 2023-10-09T09:15:20.239388+0000 mon.a (mon.0) 775 : cluster 1 osd.3 marked itself down and dead 2023-10-09T09:15:21.158 INFO:tasks.rados.rados.0.smithi163.stdout:2282: done (1 left) 2023-10-09T09:15:21.159 INFO:tasks.rados.rados.0.smithi163.stdout:2285: done (0 left) 2023-10-09T09:15:21.159 INFO:tasks.rados.rados.0.smithi163.stdout:2286: write oid 22 current snap is 293 2023-10-09T09:15:21.159 INFO:tasks.rados.rados.0.smithi163.stdout:2286: seq_num 861 ranges {470827=737278,1750408=720419} 2023-10-09T09:15:21.167 INFO:tasks.rados.rados.0.smithi163.stdout:2286: writing smithi16367185-22 from 470827 to 1208105 tid 1 2023-10-09T09:15:21.175 INFO:tasks.rados.rados.0.smithi163.stdout:2286: writing smithi16367185-22 from 1750408 to 2470827 tid 2 2023-10-09T09:15:21.175 INFO:tasks.rados.rados.0.smithi163.stdout:2287: rollback oid 4 current snap is 293 2023-10-09T09:15:21.175 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 272 2023-10-09T09:15:21.175 INFO:tasks.rados.rados.0.smithi163.stdout:2288: delete oid 42 current snap is 293 2023-10-09T09:15:21.179 INFO:tasks.rados.rados.0.smithi163.stdout:2286: finishing write tid 1 to smithi16367185-22 2023-10-09T09:15:21.181 INFO:tasks.rados.rados.0.smithi163.stdout:2286: finishing write tid 2 to smithi16367185-22 2023-10-09T09:15:21.181 INFO:tasks.rados.rados.0.smithi163.stdout:2286: finishing write tid 3 to smithi16367185-22 2023-10-09T09:15:21.181 INFO:tasks.rados.rados.0.smithi163.stdout:2286: finishing write tid 4 to smithi16367185-22 2023-10-09T09:15:21.181 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 1715 (ObjNum 861 snap 293 seq_num 861) dirty exists 2023-10-09T09:15:21.182 INFO:tasks.rados.rados.0.smithi163.stdout:2286: left oid 22 (ObjNum 861 snap 293 seq_num 861) 2023-10-09T09:15:21.182 INFO:tasks.rados.rados.0.smithi163.stdout:2286: done (2 left) 2023-10-09T09:15:21.182 INFO:tasks.rados.rados.0.smithi163.stdout:2288: done (1 left) 2023-10-09T09:15:21.182 INFO:tasks.rados.rados.0.smithi163.stdout:2289: write oid 10 current snap is 293 2023-10-09T09:15:21.182 INFO:tasks.rados.rados.0.smithi163.stdout:2289: seq_num 863 ranges {504441=487776,1467372=736289} 2023-10-09T09:15:21.188 INFO:tasks.rados.rados.0.smithi163.stdout:2289: writing smithi16367185-10 from 504441 to 992217 tid 1 2023-10-09T09:15:21.196 INFO:tasks.rados.rados.0.smithi163.stdout:2289: writing smithi16367185-10 from 1467372 to 2203661 tid 2 2023-10-09T09:15:21.196 INFO:tasks.rados.rados.0.smithi163.stdout:2290: snap_create 2023-10-09T09:15:21.196 INFO:tasks.rados.rados.0.smithi163.stdout:2289: finishing write tid 1 to smithi16367185-10 2023-10-09T09:15:21.201 INFO:tasks.rados.rados.0.smithi163.stdout:2289: finishing write tid 2 to smithi16367185-10 2023-10-09T09:15:21.202 INFO:tasks.rados.rados.0.smithi163.stdout:2289: finishing write tid 3 to smithi16367185-10 2023-10-09T09:15:21.202 INFO:tasks.rados.rados.0.smithi163.stdout:2289: finishing write tid 4 to smithi16367185-10 2023-10-09T09:15:21.202 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 756 (ObjNum 863 snap 293 seq_num 863) dirty exists 2023-10-09T09:15:21.202 INFO:tasks.rados.rados.0.smithi163.stdout:2289: left oid 10 (ObjNum 863 snap 293 seq_num 863) 2023-10-09T09:15:21.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[75425]: cluster 2023-10-09T09:15:20.128321+0000 osd.3 (osd.3) 234 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:21.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[75425]: cluster 2023-10-09T09:15:20.132665+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:21.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[75425]: cluster 2023-10-09T09:15:20.145244+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[75425]: audit 2023-10-09T09:15:20.436618+0000 mon.a (mon.0) 776 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[75425]: cluster 2023-10-09T09:15:21.147344+0000 mon.a (mon.0) 777 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:15:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[75425]: cluster 2023-10-09T09:15:21.154693+0000 mon.a (mon.0) 778 : cluster 0 osdmap e649: 8 total, 7 up, 8 in 2023-10-09T09:15:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[75425]: cluster 2023-10-09T09:15:21.154766+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[78107]: cluster 2023-10-09T09:15:20.128321+0000 osd.3 (osd.3) 234 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[78107]: cluster 2023-10-09T09:15:20.132665+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[78107]: cluster 2023-10-09T09:15:20.145244+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[78107]: audit 2023-10-09T09:15:20.436618+0000 mon.a (mon.0) 776 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[78107]: cluster 2023-10-09T09:15:21.147344+0000 mon.a (mon.0) 777 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:15:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[78107]: cluster 2023-10-09T09:15:21.154693+0000 mon.a (mon.0) 778 : cluster 0 osdmap e649: 8 total, 7 up, 8 in 2023-10-09T09:15:21.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[78107]: cluster 2023-10-09T09:15:21.154766+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:21 smithi181 bash[59345]: cluster 2023-10-09T09:15:20.128321+0000 osd.3 (osd.3) 234 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:21 smithi181 bash[59345]: cluster 2023-10-09T09:15:20.132665+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:21 smithi181 bash[59345]: cluster 2023-10-09T09:15:20.145244+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e648 with expected crc 2023-10-09T09:15:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:21 smithi181 bash[59345]: audit 2023-10-09T09:15:20.436618+0000 mon.a (mon.0) 776 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:21 smithi181 bash[59345]: cluster 2023-10-09T09:15:21.147344+0000 mon.a (mon.0) 777 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:15:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:21 smithi181 bash[59345]: cluster 2023-10-09T09:15:21.154693+0000 mon.a (mon.0) 778 : cluster 0 osdmap e649: 8 total, 7 up, 8 in 2023-10-09T09:15:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:21 smithi181 bash[59345]: cluster 2023-10-09T09:15:21.154766+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.037 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[91827]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T09:15:22.037 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:21 smithi163 bash[91919]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T09:15:22.162 INFO:tasks.rados.rados.0.smithi163.stdout:2289: done (2 left) 2023-10-09T09:15:22.162 INFO:tasks.rados.rados.0.smithi163.stdout:2290: done (1 left) 2023-10-09T09:15:22.162 INFO:tasks.rados.rados.0.smithi163.stdout:2291: read oid 46 snap -1 2023-10-09T09:15:22.163 INFO:tasks.rados.rados.0.smithi163.stdout:2291: expect (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:15:22.163 INFO:tasks.rados.rados.0.smithi163.stdout:2292: read oid 9 snap -1 2023-10-09T09:15:22.163 INFO:tasks.rados.rados.0.smithi163.stdout:2292: expect deleted 2023-10-09T09:15:22.163 INFO:tasks.rados.rados.0.smithi163.stdout:2293: rollback oid 28 current snap is 294 2023-10-09T09:15:22.163 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 286 2023-10-09T09:15:22.163 INFO:tasks.rados.rados.0.smithi163.stdout:2294: read oid 42 snap -1 2023-10-09T09:15:22.163 INFO:tasks.rados.rados.0.smithi163.stdout:2294: expect deleted 2023-10-09T09:15:22.163 INFO:tasks.rados.rados.0.smithi163.stdout:2295: read oid 7 snap 280 2023-10-09T09:15:22.163 INFO:tasks.rados.rados.0.smithi163.stdout:2295: expect (ObjNum 763 snap 259 seq_num 763) 2023-10-09T09:15:22.163 INFO:tasks.rados.rados.0.smithi163.stdout:2296: snap_create 2023-10-09T09:15:22.176 INFO:tasks.rados.rados.0.smithi163.stdout:2295: expect (ObjNum 763 snap 259 seq_num 763) 2023-10-09T09:15:22.328 INFO:tasks.rados.rados.0.smithi163.stdout:2287: finishing rollback tid 0 to smithi16367185-4 2023-10-09T09:15:22.328 INFO:tasks.rados.rados.0.smithi163.stdout:2287: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:15:22.328 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1158 (ObjNum 698933504 snap 21875 seq_num 699109760) dirty dne 2023-10-09T09:15:22.328 INFO:tasks.rados.rados.0.smithi163.stdout:2293: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:15:22.329 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 0 (ObjNum 0 snap 32603 seq_num 698729280) dirty dne 2023-10-09T09:15:22.329 INFO:tasks.rados.rados.0.smithi163.stdout:2291: expect (ObjNum 787 snap 271 seq_num 787) 2023-10-09T09:15:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: cluster 2023-10-09T09:15:21.149407+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: cluster 2023-10-09T09:15:21.154755+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: cluster 2023-10-09T09:15:21.155994+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: cluster 2023-10-09T09:15:21.156328+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: cluster 2023-10-09T09:15:21.342536+0000 mgr.y (mgr.34107) 260 : cluster 0 pgmap v526: 89 pgs: 15 peering, 8 stale+active+clean, 66 active+clean; 191 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2023-10-09T09:15:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: audit 2023-10-09T09:15:21.509892+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.163:0/708233103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: cluster 2023-10-09T09:15:22.149536+0000 mon.a (mon.0) 779 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T09:15:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: cluster 2023-10-09T09:15:22.161756+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: cluster 2023-10-09T09:15:22.161762+0000 mon.a (mon.0) 780 : cluster 0 osdmap e650: 8 total, 7 up, 8 in 2023-10-09T09:15:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: cluster 2023-10-09T09:15:22.165861+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[78107]: cluster 2023-10-09T09:15:22.167834+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: cluster 2023-10-09T09:15:21.149407+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: cluster 2023-10-09T09:15:21.154755+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: cluster 2023-10-09T09:15:21.155994+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: cluster 2023-10-09T09:15:21.156328+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: cluster 2023-10-09T09:15:21.342536+0000 mgr.y (mgr.34107) 260 : cluster 0 pgmap v526: 89 pgs: 15 peering, 8 stale+active+clean, 66 active+clean; 191 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2023-10-09T09:15:22.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: audit 2023-10-09T09:15:21.509892+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.163:0/708233103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:22.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: cluster 2023-10-09T09:15:22.149536+0000 mon.a (mon.0) 779 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T09:15:22.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: cluster 2023-10-09T09:15:22.161756+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:22.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: cluster 2023-10-09T09:15:22.161762+0000 mon.a (mon.0) 780 : cluster 0 osdmap e650: 8 total, 7 up, 8 in 2023-10-09T09:15:22.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: cluster 2023-10-09T09:15:22.165861+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:22.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:22 smithi163 bash[75425]: cluster 2023-10-09T09:15:22.167834+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: cluster 2023-10-09T09:15:21.149407+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: cluster 2023-10-09T09:15:21.154755+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: cluster 2023-10-09T09:15:21.155994+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: cluster 2023-10-09T09:15:21.156328+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e649 with expected crc 2023-10-09T09:15:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: cluster 2023-10-09T09:15:21.342536+0000 mgr.y (mgr.34107) 260 : cluster 0 pgmap v526: 89 pgs: 15 peering, 8 stale+active+clean, 66 active+clean; 191 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2023-10-09T09:15:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: audit 2023-10-09T09:15:21.509892+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.163:0/708233103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: cluster 2023-10-09T09:15:22.149536+0000 mon.a (mon.0) 779 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T09:15:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: cluster 2023-10-09T09:15:22.161756+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: cluster 2023-10-09T09:15:22.161762+0000 mon.a (mon.0) 780 : cluster 0 osdmap e650: 8 total, 7 up, 8 in 2023-10-09T09:15:22.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: cluster 2023-10-09T09:15:22.165861+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:22.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:22 smithi181 bash[59345]: cluster 2023-10-09T09:15:22.167834+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.166 INFO:tasks.rados.rados.0.smithi163.stdout:2287: done (6 left) 2023-10-09T09:15:23.166 INFO:tasks.rados.rados.0.smithi163.stdout:2291: done (5 left) 2023-10-09T09:15:23.167 INFO:tasks.rados.rados.0.smithi163.stdout:2292: done (4 left) 2023-10-09T09:15:23.167 INFO:tasks.rados.rados.0.smithi163.stdout:2293: done (3 left) 2023-10-09T09:15:23.167 INFO:tasks.rados.rados.0.smithi163.stdout:2294: done (2 left) 2023-10-09T09:15:23.167 INFO:tasks.rados.rados.0.smithi163.stdout:2295: done (1 left) 2023-10-09T09:15:23.167 INFO:tasks.rados.rados.0.smithi163.stdout:2296: done (0 left) 2023-10-09T09:15:23.167 INFO:tasks.rados.rados.0.smithi163.stdout:2297: read oid 48 snap 285 2023-10-09T09:15:23.167 INFO:tasks.rados.rados.0.smithi163.stdout:2297: expect (ObjNum 751 snap 255 seq_num 751) 2023-10-09T09:15:23.167 INFO:tasks.rados.rados.0.smithi163.stdout:2298: delete oid 12 current snap is 295 2023-10-09T09:15:23.171 INFO:tasks.rados.rados.0.smithi163.stdout:2298: done (1 left) 2023-10-09T09:15:23.171 INFO:tasks.rados.rados.0.smithi163.stdout:2299: snap_create 2023-10-09T09:15:23.178 INFO:tasks.rados.rados.0.smithi163.stdout:2297: expect (ObjNum 751 snap 255 seq_num 751) 2023-10-09T09:15:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:15:23] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.33.4" 2023-10-09T09:15:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[78107]: cluster 2023-10-09T09:15:22.158834+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[78107]: cluster 2023-10-09T09:15:22.161049+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[78107]: cluster 2023-10-09T09:15:22.165596+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[78107]: cluster 2023-10-09T09:15:22.165959+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[78107]: cluster 2023-10-09T09:15:22.168473+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[78107]: cluster 2023-10-09T09:15:23.166212+0000 mon.a (mon.0) 781 : cluster 0 osdmap e651: 8 total, 7 up, 8 in 2023-10-09T09:15:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[75425]: cluster 2023-10-09T09:15:22.158834+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[75425]: cluster 2023-10-09T09:15:22.161049+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[75425]: cluster 2023-10-09T09:15:22.165596+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[75425]: cluster 2023-10-09T09:15:22.165959+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[75425]: cluster 2023-10-09T09:15:22.168473+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:23 smithi163 bash[75425]: cluster 2023-10-09T09:15:23.166212+0000 mon.a (mon.0) 781 : cluster 0 osdmap e651: 8 total, 7 up, 8 in 2023-10-09T09:15:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:23 smithi181 bash[59345]: cluster 2023-10-09T09:15:22.158834+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:23 smithi181 bash[59345]: cluster 2023-10-09T09:15:22.161049+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:23 smithi181 bash[59345]: cluster 2023-10-09T09:15:22.165596+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:23 smithi181 bash[59345]: cluster 2023-10-09T09:15:22.165959+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:23 smithi181 bash[59345]: cluster 2023-10-09T09:15:22.168473+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e650 with expected crc 2023-10-09T09:15:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:23 smithi181 bash[59345]: cluster 2023-10-09T09:15:23.166212+0000 mon.a (mon.0) 781 : cluster 0 osdmap e651: 8 total, 7 up, 8 in 2023-10-09T09:15:24.172 INFO:tasks.rados.rados.0.smithi163.stdout:2297: done (1 left) 2023-10-09T09:15:24.172 INFO:tasks.rados.rados.0.smithi163.stdout:2299: done (0 left) 2023-10-09T09:15:24.172 INFO:tasks.rados.rados.0.smithi163.stdout:2300: read oid 45 snap 280 2023-10-09T09:15:24.172 INFO:tasks.rados.rados.0.smithi163.stdout:2300: expect (ObjNum 786 snap 271 seq_num 786) 2023-10-09T09:15:24.172 INFO:tasks.rados.rados.0.smithi163.stdout:2301: delete oid 47 current snap is 296 2023-10-09T09:15:24.183 INFO:tasks.rados.rados.0.smithi163.stdout:2300: expect (ObjNum 786 snap 271 seq_num 786) 2023-10-09T09:15:24.301 INFO:tasks.rados.rados.0.smithi163.stdout:2300: done (1 left) 2023-10-09T09:15:24.302 INFO:tasks.rados.rados.0.smithi163.stdout:2301: done (0 left) 2023-10-09T09:15:24.302 INFO:tasks.rados.rados.0.smithi163.stdout:2302: snap_create 2023-10-09T09:15:24.348 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:24 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T09:15:24.348 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:24 smithi163 systemd[1]: Stopped Ceph osd.3 for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:15:24.348 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:24 smithi163 systemd[1]: Started Ceph osd.3 for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:15:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[75425]: cluster 2023-10-09T09:15:23.159632+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[75425]: cluster 2023-10-09T09:15:23.287493+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[75425]: cluster 2023-10-09T09:15:23.343431+0000 mgr.y (mgr.34107) 261 : cluster 0 pgmap v529: 89 pgs: 22 peering, 6 stale+active+clean, 61 active+clean; 187 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 933 KiB/s wr, 5 op/s 2023-10-09T09:15:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[75425]: cluster 2023-10-09T09:15:23.824666+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[75425]: audit 2023-10-09T09:15:23.867061+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.163:0/1346040337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[75425]: cluster 2023-10-09T09:15:24.171828+0000 mon.a (mon.0) 782 : cluster 0 osdmap e652: 8 total, 7 up, 8 in 2023-10-09T09:15:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[75425]: audit 2023-10-09T09:15:24.328282+0000 mon.a (mon.0) 783 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[75425]: audit 2023-10-09T09:15:24.335054+0000 mon.a (mon.0) 784 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[75425]: audit 2023-10-09T09:15:24.362546+0000 mon.a (mon.0) 785 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[78107]: cluster 2023-10-09T09:15:23.159632+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[78107]: cluster 2023-10-09T09:15:23.287493+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:24.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[78107]: cluster 2023-10-09T09:15:23.343431+0000 mgr.y (mgr.34107) 261 : cluster 0 pgmap v529: 89 pgs: 22 peering, 6 stale+active+clean, 61 active+clean; 187 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 933 KiB/s wr, 5 op/s 2023-10-09T09:15:24.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[78107]: cluster 2023-10-09T09:15:23.824666+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:24.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[78107]: audit 2023-10-09T09:15:23.867061+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.163:0/1346040337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:24.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[78107]: cluster 2023-10-09T09:15:24.171828+0000 mon.a (mon.0) 782 : cluster 0 osdmap e652: 8 total, 7 up, 8 in 2023-10-09T09:15:24.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[78107]: audit 2023-10-09T09:15:24.328282+0000 mon.a (mon.0) 783 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:24.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[78107]: audit 2023-10-09T09:15:24.335054+0000 mon.a (mon.0) 784 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:24.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:24 smithi163 bash[78107]: audit 2023-10-09T09:15:24.362546+0000 mon.a (mon.0) 785 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:24 smithi181 bash[59345]: cluster 2023-10-09T09:15:23.159632+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:24 smithi181 bash[59345]: cluster 2023-10-09T09:15:23.287493+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:24 smithi181 bash[59345]: cluster 2023-10-09T09:15:23.343431+0000 mgr.y (mgr.34107) 261 : cluster 0 pgmap v529: 89 pgs: 22 peering, 6 stale+active+clean, 61 active+clean; 187 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 933 KiB/s wr, 5 op/s 2023-10-09T09:15:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:24 smithi181 bash[59345]: cluster 2023-10-09T09:15:23.824666+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:24 smithi181 bash[59345]: audit 2023-10-09T09:15:23.867061+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.163:0/1346040337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:24 smithi181 bash[59345]: cluster 2023-10-09T09:15:24.171828+0000 mon.a (mon.0) 782 : cluster 0 osdmap e652: 8 total, 7 up, 8 in 2023-10-09T09:15:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:24 smithi181 bash[59345]: audit 2023-10-09T09:15:24.328282+0000 mon.a (mon.0) 783 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:24.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:24 smithi181 bash[59345]: audit 2023-10-09T09:15:24.335054+0000 mon.a (mon.0) 784 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:24.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:24 smithi181 bash[59345]: audit 2023-10-09T09:15:24.362546+0000 mon.a (mon.0) 785 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:25.135 INFO:tasks.rados.rados.0.smithi163.stdout:2302: done (0 left) 2023-10-09T09:15:25.136 INFO:tasks.rados.rados.0.smithi163.stdout:2303: snap_create 2023-10-09T09:15:25.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[78107]: cluster 2023-10-09T09:15:23.883472+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:25.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[78107]: cluster 2023-10-09T09:15:24.167324+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:25.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[78107]: cluster 2023-10-09T09:15:24.167513+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:25.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[78107]: cluster 2023-10-09T09:15:24.775469+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:25.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[78107]: cluster 2023-10-09T09:15:25.131037+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e653 with expected crc 2023-10-09T09:15:25.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[78107]: cluster 2023-10-09T09:15:25.135665+0000 mon.a (mon.0) 786 : cluster 0 osdmap e653: 8 total, 7 up, 8 in 2023-10-09T09:15:25.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[78107]: audit 2023-10-09T09:15:25.350260+0000 mon.a (mon.0) 787 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:25.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[78107]: audit 2023-10-09T09:15:25.351995+0000 mon.a (mon.0) 788 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[75425]: cluster 2023-10-09T09:15:23.883472+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[75425]: cluster 2023-10-09T09:15:24.167324+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[75425]: cluster 2023-10-09T09:15:24.167513+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[75425]: cluster 2023-10-09T09:15:24.775469+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[75425]: cluster 2023-10-09T09:15:25.131037+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e653 with expected crc 2023-10-09T09:15:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[75425]: cluster 2023-10-09T09:15:25.135665+0000 mon.a (mon.0) 786 : cluster 0 osdmap e653: 8 total, 7 up, 8 in 2023-10-09T09:15:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[75425]: audit 2023-10-09T09:15:25.350260+0000 mon.a (mon.0) 787 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:25.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[75425]: audit 2023-10-09T09:15:25.351995+0000 mon.a (mon.0) 788 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:25 smithi181 bash[59345]: cluster 2023-10-09T09:15:23.883472+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e651 with expected crc 2023-10-09T09:15:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:25 smithi181 bash[59345]: cluster 2023-10-09T09:15:24.167324+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:25 smithi181 bash[59345]: cluster 2023-10-09T09:15:24.167513+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:25 smithi181 bash[59345]: cluster 2023-10-09T09:15:24.775469+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:25 smithi181 bash[59345]: cluster 2023-10-09T09:15:25.131037+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e653 with expected crc 2023-10-09T09:15:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:25 smithi181 bash[59345]: cluster 2023-10-09T09:15:25.135665+0000 mon.a (mon.0) 786 : cluster 0 osdmap e653: 8 total, 7 up, 8 in 2023-10-09T09:15:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:25 smithi181 bash[59345]: audit 2023-10-09T09:15:25.350260+0000 mon.a (mon.0) 787 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:25.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:25 smithi181 bash[59345]: audit 2023-10-09T09:15:25.351995+0000 mon.a (mon.0) 788 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:26.139 INFO:tasks.rados.rados.0.smithi163.stdout:2303: done (0 left) 2023-10-09T09:15:26.139 INFO:tasks.rados.rados.0.smithi163.stdout:2304: read oid 34 snap -1 2023-10-09T09:15:26.139 INFO:tasks.rados.rados.0.smithi163.stdout:2304: expect deleted 2023-10-09T09:15:26.140 INFO:tasks.rados.rados.0.smithi163.stdout:2305: read oid 49 snap -1 2023-10-09T09:15:26.140 INFO:tasks.rados.rados.0.smithi163.stdout:2305: expect (ObjNum 793 snap 272 seq_num 793) 2023-10-09T09:15:26.140 INFO:tasks.rados.rados.0.smithi163.stdout:2306: snap_create 2023-10-09T09:15:26.149 INFO:tasks.rados.rados.0.smithi163.stdout:2305: expect (ObjNum 793 snap 272 seq_num 793) 2023-10-09T09:15:26.218 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[92100]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T09:15:26.218 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[92100]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T09:15:26.218 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[92100]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T09:15:26.218 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[92100]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T09:15:26.218 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[92100]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T09:15:26.219 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[92100]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T09:15:26.219 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[92100]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T09:15:26.219 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:25 smithi163 bash[92100]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T09:15:26.640 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[75425]: cluster 2023-10-09T09:15:24.892402+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:26.640 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[75425]: cluster 2023-10-09T09:15:25.129632+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e653 with expected crc 2023-10-09T09:15:26.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[75425]: cluster 2023-10-09T09:15:25.308617+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e653 with expected crc 2023-10-09T09:15:26.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[75425]: cluster 2023-10-09T09:15:25.344459+0000 mgr.y (mgr.34107) 262 : cluster 0 pgmap v532: 89 pgs: 2 active+undersized+degraded, 22 peering, 10 active+undersized, 55 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 299 KiB/s wr, 3 op/s; 12/357 objects degraded (3.361%) 2023-10-09T09:15:26.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[75425]: cluster 2023-10-09T09:15:26.136696+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e654 with expected crc 2023-10-09T09:15:26.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[75425]: cluster 2023-10-09T09:15:26.138751+0000 mon.a (mon.0) 789 : cluster 0 osdmap e654: 8 total, 7 up, 8 in 2023-10-09T09:15:26.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[75425]: cluster 2023-10-09T09:15:26.139062+0000 mon.a (mon.0) 790 : cluster 3 Health check failed: Degraded data redundancy: 12/357 objects degraded (3.361%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:26.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[75425]: audit 2023-10-09T09:15:26.228059+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.163:0/2016985440' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:26.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[78107]: cluster 2023-10-09T09:15:24.892402+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:26.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[78107]: cluster 2023-10-09T09:15:25.129632+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e653 with expected crc 2023-10-09T09:15:26.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[78107]: cluster 2023-10-09T09:15:25.308617+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e653 with expected crc 2023-10-09T09:15:26.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[78107]: cluster 2023-10-09T09:15:25.344459+0000 mgr.y (mgr.34107) 262 : cluster 0 pgmap v532: 89 pgs: 2 active+undersized+degraded, 22 peering, 10 active+undersized, 55 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 299 KiB/s wr, 3 op/s; 12/357 objects degraded (3.361%) 2023-10-09T09:15:26.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[78107]: cluster 2023-10-09T09:15:26.136696+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e654 with expected crc 2023-10-09T09:15:26.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[78107]: cluster 2023-10-09T09:15:26.138751+0000 mon.a (mon.0) 789 : cluster 0 osdmap e654: 8 total, 7 up, 8 in 2023-10-09T09:15:26.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[78107]: cluster 2023-10-09T09:15:26.139062+0000 mon.a (mon.0) 790 : cluster 3 Health check failed: Degraded data redundancy: 12/357 objects degraded (3.361%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:26.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:26 smithi163 bash[78107]: audit 2023-10-09T09:15:26.228059+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.163:0/2016985440' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:26 smithi181 bash[59345]: cluster 2023-10-09T09:15:24.892402+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e652 with expected crc 2023-10-09T09:15:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:26 smithi181 bash[59345]: cluster 2023-10-09T09:15:25.129632+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e653 with expected crc 2023-10-09T09:15:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:26 smithi181 bash[59345]: cluster 2023-10-09T09:15:25.308617+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e653 with expected crc 2023-10-09T09:15:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:26 smithi181 bash[59345]: cluster 2023-10-09T09:15:25.344459+0000 mgr.y (mgr.34107) 262 : cluster 0 pgmap v532: 89 pgs: 2 active+undersized+degraded, 22 peering, 10 active+undersized, 55 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 299 KiB/s wr, 3 op/s; 12/357 objects degraded (3.361%) 2023-10-09T09:15:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:26 smithi181 bash[59345]: cluster 2023-10-09T09:15:26.136696+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e654 with expected crc 2023-10-09T09:15:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:26 smithi181 bash[59345]: cluster 2023-10-09T09:15:26.138751+0000 mon.a (mon.0) 789 : cluster 0 osdmap e654: 8 total, 7 up, 8 in 2023-10-09T09:15:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:26 smithi181 bash[59345]: cluster 2023-10-09T09:15:26.139062+0000 mon.a (mon.0) 790 : cluster 3 Health check failed: Degraded data redundancy: 12/357 objects degraded (3.361%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:26 smithi181 bash[59345]: audit 2023-10-09T09:15:26.228059+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.163:0/2016985440' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:27.137 INFO:tasks.rados.rados.0.smithi163.stdout:2304: done (2 left) 2023-10-09T09:15:27.138 INFO:tasks.rados.rados.0.smithi163.stdout:2305: done (1 left) 2023-10-09T09:15:27.138 INFO:tasks.rados.rados.0.smithi163.stdout:2306: done (0 left) 2023-10-09T09:15:27.138 INFO:tasks.rados.rados.0.smithi163.stdout:2307: read oid 40 snap 296 2023-10-09T09:15:27.138 INFO:tasks.rados.rados.0.smithi163.stdout:2307: expect (ObjNum 742 snap 252 seq_num 742) 2023-10-09T09:15:27.138 INFO:tasks.rados.rados.0.smithi163.stdout:2308: write oid 38 current snap is 299 2023-10-09T09:15:27.138 INFO:tasks.rados.rados.0.smithi163.stdout:2308: seq_num 866 ranges {554862=713523,1926465=599969} 2023-10-09T09:15:27.148 INFO:tasks.rados.rados.0.smithi163.stdout:2308: writing smithi16367185-38 from 554862 to 1268385 tid 1 2023-10-09T09:15:27.155 INFO:tasks.rados.rados.0.smithi163.stdout:2308: writing smithi16367185-38 from 1926465 to 2526434 tid 2 2023-10-09T09:15:27.155 INFO:tasks.rados.rados.0.smithi163.stdout:2309: read oid 44 snap 295 2023-10-09T09:15:27.155 INFO:tasks.rados.rados.0.smithi163.stdout:2309: expect deleted 2023-10-09T09:15:27.155 INFO:tasks.rados.rados.0.smithi163.stdout:2310: snap_create 2023-10-09T09:15:27.156 INFO:tasks.rados.rados.0.smithi163.stdout:2307: expect (ObjNum 742 snap 252 seq_num 742) 2023-10-09T09:15:27.239 INFO:tasks.rados.rados.0.smithi163.stdout:2308: finishing write tid 1 to smithi16367185-38 2023-10-09T09:15:27.239 INFO:tasks.rados.rados.0.smithi163.stdout:2308: finishing write tid 2 to smithi16367185-38 2023-10-09T09:15:27.239 INFO:tasks.rados.rados.0.smithi163.stdout:2308: finishing write tid 3 to smithi16367185-38 2023-10-09T09:15:27.239 INFO:tasks.rados.rados.0.smithi163.stdout:2308: finishing write tid 4 to smithi16367185-38 2023-10-09T09:15:27.239 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1050 (ObjNum 866 snap 299 seq_num 866) dirty exists 2023-10-09T09:15:27.239 INFO:tasks.rados.rados.0.smithi163.stdout:2308: left oid 38 (ObjNum 866 snap 299 seq_num 866) 2023-10-09T09:15:27.718 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:27 smithi163 bash[92392]: debug 2023-10-09T09:15:27.514+0000 7fb3861c7700 1 -- 172.21.15.163:0/1084282884 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5563753144e0 con 0x55637536f000 2023-10-09T09:15:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:27 smithi163 bash[75425]: cluster 2023-10-09T09:15:26.134080+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e654 with expected crc 2023-10-09T09:15:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:27 smithi163 bash[75425]: cluster 2023-10-09T09:15:26.315330+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e654 with expected crc 2023-10-09T09:15:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:27 smithi163 bash[75425]: cluster 2023-10-09T09:15:27.137180+0000 mon.a (mon.0) 791 : cluster 0 osdmap e655: 8 total, 7 up, 8 in 2023-10-09T09:15:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:27 smithi163 bash[75425]: cluster 2023-10-09T09:15:27.149820+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:27.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:27 smithi163 bash[78107]: cluster 2023-10-09T09:15:26.134080+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e654 with expected crc 2023-10-09T09:15:27.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:27 smithi163 bash[78107]: cluster 2023-10-09T09:15:26.315330+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e654 with expected crc 2023-10-09T09:15:27.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:27 smithi163 bash[78107]: cluster 2023-10-09T09:15:27.137180+0000 mon.a (mon.0) 791 : cluster 0 osdmap e655: 8 total, 7 up, 8 in 2023-10-09T09:15:27.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:27 smithi163 bash[78107]: cluster 2023-10-09T09:15:27.149820+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:27 smithi181 bash[59345]: cluster 2023-10-09T09:15:26.134080+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e654 with expected crc 2023-10-09T09:15:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:27 smithi181 bash[59345]: cluster 2023-10-09T09:15:26.315330+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e654 with expected crc 2023-10-09T09:15:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:27 smithi181 bash[59345]: cluster 2023-10-09T09:15:27.137180+0000 mon.a (mon.0) 791 : cluster 0 osdmap e655: 8 total, 7 up, 8 in 2023-10-09T09:15:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:27 smithi181 bash[59345]: cluster 2023-10-09T09:15:27.149820+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:28.151 INFO:tasks.rados.rados.0.smithi163.stdout:2307: done (3 left) 2023-10-09T09:15:28.152 INFO:tasks.rados.rados.0.smithi163.stdout:2308: done (2 left) 2023-10-09T09:15:28.152 INFO:tasks.rados.rados.0.smithi163.stdout:2309: done (1 left) 2023-10-09T09:15:28.152 INFO:tasks.rados.rados.0.smithi163.stdout:2310: done (0 left) 2023-10-09T09:15:28.152 INFO:tasks.rados.rados.0.smithi163.stdout:2311: snap_create 2023-10-09T09:15:28.398 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[92392]: debug 2023-10-09T09:15:28.078+0000 7fb38f34d640 -1 Falling back to public interface 2023-10-09T09:15:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[75425]: cluster 2023-10-09T09:15:27.133866+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[75425]: cluster 2023-10-09T09:15:27.137055+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[75425]: cluster 2023-10-09T09:15:27.142163+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[75425]: cluster 2023-10-09T09:15:27.345360+0000 mgr.y (mgr.34107) 263 : cluster 0 pgmap v535: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 299 KiB/s wr, 12 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[75425]: cluster 2023-10-09T09:15:28.134240+0000 mon.a (mon.0) 792 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-10-09T09:15:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[75425]: cluster 2023-10-09T09:15:28.146366+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[75425]: cluster 2023-10-09T09:15:28.151768+0000 mon.a (mon.0) 793 : cluster 0 osdmap e656: 8 total, 7 up, 8 in 2023-10-09T09:15:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[78107]: cluster 2023-10-09T09:15:27.133866+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[78107]: cluster 2023-10-09T09:15:27.137055+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[78107]: cluster 2023-10-09T09:15:27.142163+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:28.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[78107]: cluster 2023-10-09T09:15:27.345360+0000 mgr.y (mgr.34107) 263 : cluster 0 pgmap v535: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 299 KiB/s wr, 12 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:28.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[78107]: cluster 2023-10-09T09:15:28.134240+0000 mon.a (mon.0) 792 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-10-09T09:15:28.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[78107]: cluster 2023-10-09T09:15:28.146366+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:28.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:28 smithi163 bash[78107]: cluster 2023-10-09T09:15:28.151768+0000 mon.a (mon.0) 793 : cluster 0 osdmap e656: 8 total, 7 up, 8 in 2023-10-09T09:15:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:28 smithi181 bash[59345]: cluster 2023-10-09T09:15:27.133866+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:28 smithi181 bash[59345]: cluster 2023-10-09T09:15:27.137055+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:28 smithi181 bash[59345]: cluster 2023-10-09T09:15:27.142163+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e655 with expected crc 2023-10-09T09:15:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:28 smithi181 bash[59345]: cluster 2023-10-09T09:15:27.345360+0000 mgr.y (mgr.34107) 263 : cluster 0 pgmap v535: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 299 KiB/s wr, 12 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:28 smithi181 bash[59345]: cluster 2023-10-09T09:15:28.134240+0000 mon.a (mon.0) 792 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-10-09T09:15:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:28 smithi181 bash[59345]: cluster 2023-10-09T09:15:28.146366+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:28 smithi181 bash[59345]: cluster 2023-10-09T09:15:28.151768+0000 mon.a (mon.0) 793 : cluster 0 osdmap e656: 8 total, 7 up, 8 in 2023-10-09T09:15:29.157 INFO:tasks.rados.rados.0.smithi163.stdout:2311: done (0 left) 2023-10-09T09:15:29.157 INFO:tasks.rados.rados.0.smithi163.stdout:2312: read oid 36 snap 294 2023-10-09T09:15:29.157 INFO:tasks.rados.rados.0.smithi163.stdout:2312: expect (ObjNum 765 snap 260 seq_num 765) 2023-10-09T09:15:29.158 INFO:tasks.rados.rados.0.smithi163.stdout:2313: write oid 21 current snap is 301 2023-10-09T09:15:29.158 INFO:tasks.rados.rados.0.smithi163.stdout:2313: seq_num 867 ranges {571669=505125,1701300=743627} 2023-10-09T09:15:29.165 INFO:tasks.rados.rados.0.smithi163.stdout:2313: writing smithi16367185-21 from 571669 to 1076794 tid 1 2023-10-09T09:15:29.173 INFO:tasks.rados.rados.0.smithi163.stdout:2313: writing smithi16367185-21 from 1701300 to 2444927 tid 2 2023-10-09T09:15:29.174 INFO:tasks.rados.rados.0.smithi163.stdout:2314: snap_create 2023-10-09T09:15:29.174 INFO:tasks.rados.rados.0.smithi163.stdout:2312: expect (ObjNum 765 snap 260 seq_num 765) 2023-10-09T09:15:29.307 INFO:tasks.rados.rados.0.smithi163.stdout:2313: finishing write tid 1 to smithi16367185-21 2023-10-09T09:15:29.308 INFO:tasks.rados.rados.0.smithi163.stdout:2313: finishing write tid 2 to smithi16367185-21 2023-10-09T09:15:29.308 INFO:tasks.rados.rados.0.smithi163.stdout:2313: finishing write tid 3 to smithi16367185-21 2023-10-09T09:15:29.308 INFO:tasks.rados.rados.0.smithi163.stdout:2313: finishing write tid 4 to smithi16367185-21 2023-10-09T09:15:29.308 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 526 (ObjNum 867 snap 301 seq_num 867) dirty exists 2023-10-09T09:15:29.308 INFO:tasks.rados.rados.0.smithi163.stdout:2313: left oid 21 (ObjNum 867 snap 301 seq_num 867) 2023-10-09T09:15:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:29 smithi163 bash[78107]: cluster 2023-10-09T09:15:28.145839+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:29 smithi163 bash[78107]: audit 2023-10-09T09:15:28.586698+0000 mon.a (mon.0) 794 : audit 0 from='client.? 172.21.15.163:0/1274888719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:29 smithi163 bash[78107]: cluster 2023-10-09T09:15:29.156931+0000 mon.a (mon.0) 795 : cluster 0 osdmap e657: 8 total, 7 up, 8 in 2023-10-09T09:15:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:29 smithi163 bash[78107]: cluster 2023-10-09T09:15:29.170052+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e657 with expected crc 2023-10-09T09:15:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:29 smithi163 bash[75425]: cluster 2023-10-09T09:15:28.145839+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:29 smithi163 bash[75425]: audit 2023-10-09T09:15:28.586698+0000 mon.a (mon.0) 794 : audit 0 from='client.? 172.21.15.163:0/1274888719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:29 smithi163 bash[75425]: cluster 2023-10-09T09:15:29.156931+0000 mon.a (mon.0) 795 : cluster 0 osdmap e657: 8 total, 7 up, 8 in 2023-10-09T09:15:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:29 smithi163 bash[75425]: cluster 2023-10-09T09:15:29.170052+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e657 with expected crc 2023-10-09T09:15:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:29 smithi181 bash[59345]: cluster 2023-10-09T09:15:28.145839+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:29 smithi181 bash[59345]: audit 2023-10-09T09:15:28.586698+0000 mon.a (mon.0) 794 : audit 0 from='client.? 172.21.15.163:0/1274888719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:29 smithi181 bash[59345]: cluster 2023-10-09T09:15:29.156931+0000 mon.a (mon.0) 795 : cluster 0 osdmap e657: 8 total, 7 up, 8 in 2023-10-09T09:15:29.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:29 smithi181 bash[59345]: cluster 2023-10-09T09:15:29.170052+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e657 with expected crc 2023-10-09T09:15:30.138 INFO:tasks.rados.rados.0.smithi163.stdout:2312: done (2 left) 2023-10-09T09:15:30.138 INFO:tasks.rados.rados.0.smithi163.stdout:2313: done (1 left) 2023-10-09T09:15:30.138 INFO:tasks.rados.rados.0.smithi163.stdout:2314: done (0 left) 2023-10-09T09:15:30.138 INFO:tasks.rados.rados.0.smithi163.stdout:2315: snap_create 2023-10-09T09:15:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:30 smithi163 bash[75425]: cluster 2023-10-09T09:15:28.942324+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:30 smithi163 bash[75425]: cluster 2023-10-09T09:15:29.152090+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e657 with expected crc 2023-10-09T09:15:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:30 smithi163 bash[75425]: cluster 2023-10-09T09:15:29.333756+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:30 smithi163 bash[75425]: cluster 2023-10-09T09:15:29.346007+0000 mgr.y (mgr.34107) 264 : cluster 0 pgmap v538: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:30 smithi163 bash[75425]: cluster 2023-10-09T09:15:30.137872+0000 mon.a (mon.0) 796 : cluster 0 osdmap e658: 8 total, 7 up, 8 in 2023-10-09T09:15:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:30 smithi163 bash[78107]: cluster 2023-10-09T09:15:28.942324+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:30 smithi163 bash[78107]: cluster 2023-10-09T09:15:29.152090+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e657 with expected crc 2023-10-09T09:15:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:30 smithi163 bash[78107]: cluster 2023-10-09T09:15:29.333756+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:30 smithi163 bash[78107]: cluster 2023-10-09T09:15:29.346007+0000 mgr.y (mgr.34107) 264 : cluster 0 pgmap v538: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:30.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:30 smithi163 bash[78107]: cluster 2023-10-09T09:15:30.137872+0000 mon.a (mon.0) 796 : cluster 0 osdmap e658: 8 total, 7 up, 8 in 2023-10-09T09:15:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:30 smithi181 bash[59345]: cluster 2023-10-09T09:15:28.942324+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:30 smithi181 bash[59345]: cluster 2023-10-09T09:15:29.152090+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e657 with expected crc 2023-10-09T09:15:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:30 smithi181 bash[59345]: cluster 2023-10-09T09:15:29.333756+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e656 with expected crc 2023-10-09T09:15:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:30 smithi181 bash[59345]: cluster 2023-10-09T09:15:29.346007+0000 mgr.y (mgr.34107) 264 : cluster 0 pgmap v538: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:30 smithi181 bash[59345]: cluster 2023-10-09T09:15:30.137872+0000 mon.a (mon.0) 796 : cluster 0 osdmap e658: 8 total, 7 up, 8 in 2023-10-09T09:15:31.138 INFO:tasks.rados.rados.0.smithi163.stdout:2315: done (0 left) 2023-10-09T09:15:31.138 INFO:tasks.rados.rados.0.smithi163.stdout:2316: snap_remove snap 289 2023-10-09T09:15:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[78107]: cluster 2023-10-09T09:15:29.624115+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e657 with expected crc 2023-10-09T09:15:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[78107]: cluster 2023-10-09T09:15:30.132887+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[78107]: cluster 2023-10-09T09:15:30.345141+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[78107]: audit 2023-10-09T09:15:30.406134+0000 mon.a (mon.0) 797 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[78107]: audit 2023-10-09T09:15:30.417205+0000 mon.a (mon.0) 798 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[78107]: cluster 2023-10-09T09:15:30.489848+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[78107]: audit 2023-10-09T09:15:30.978069+0000 mon.a (mon.0) 799 : audit 0 from='client.? 172.21.15.163:0/2086925187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[78107]: cluster 2023-10-09T09:15:31.137861+0000 mon.a (mon.0) 800 : cluster 0 osdmap e659: 8 total, 7 up, 8 in 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[78107]: audit 2023-10-09T09:15:31.267060+0000 mon.a (mon.0) 801 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[78107]: audit 2023-10-09T09:15:31.276684+0000 mon.a (mon.0) 802 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[75425]: cluster 2023-10-09T09:15:29.624115+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e657 with expected crc 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[75425]: cluster 2023-10-09T09:15:30.132887+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[75425]: cluster 2023-10-09T09:15:30.345141+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[75425]: audit 2023-10-09T09:15:30.406134+0000 mon.a (mon.0) 797 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[75425]: audit 2023-10-09T09:15:30.417205+0000 mon.a (mon.0) 798 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[75425]: cluster 2023-10-09T09:15:30.489848+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[75425]: audit 2023-10-09T09:15:30.978069+0000 mon.a (mon.0) 799 : audit 0 from='client.? 172.21.15.163:0/2086925187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:31.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[75425]: cluster 2023-10-09T09:15:31.137861+0000 mon.a (mon.0) 800 : cluster 0 osdmap e659: 8 total, 7 up, 8 in 2023-10-09T09:15:31.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[75425]: audit 2023-10-09T09:15:31.267060+0000 mon.a (mon.0) 801 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:31 smithi163 bash[75425]: audit 2023-10-09T09:15:31.276684+0000 mon.a (mon.0) 802 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:31 smithi181 bash[59345]: cluster 2023-10-09T09:15:29.624115+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e657 with expected crc 2023-10-09T09:15:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:31 smithi181 bash[59345]: cluster 2023-10-09T09:15:30.132887+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:31 smithi181 bash[59345]: cluster 2023-10-09T09:15:30.345141+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:31 smithi181 bash[59345]: audit 2023-10-09T09:15:30.406134+0000 mon.a (mon.0) 797 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:31 smithi181 bash[59345]: audit 2023-10-09T09:15:30.417205+0000 mon.a (mon.0) 798 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:31 smithi181 bash[59345]: cluster 2023-10-09T09:15:30.489848+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:31 smithi181 bash[59345]: audit 2023-10-09T09:15:30.978069+0000 mon.a (mon.0) 799 : audit 0 from='client.? 172.21.15.163:0/2086925187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:31 smithi181 bash[59345]: cluster 2023-10-09T09:15:31.137861+0000 mon.a (mon.0) 800 : cluster 0 osdmap e659: 8 total, 7 up, 8 in 2023-10-09T09:15:31.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:31 smithi181 bash[59345]: audit 2023-10-09T09:15:31.267060+0000 mon.a (mon.0) 801 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:31.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:31 smithi181 bash[59345]: audit 2023-10-09T09:15:31.276684+0000 mon.a (mon.0) 802 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:32.140 INFO:tasks.rados.rados.0.smithi163.stdout:2316: done (0 left) 2023-10-09T09:15:32.140 INFO:tasks.rados.rados.0.smithi163.stdout:2317: snap_remove snap 292 2023-10-09T09:15:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:32 smithi163 bash[75425]: cluster 2023-10-09T09:15:30.988826+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:32 smithi163 bash[75425]: cluster 2023-10-09T09:15:31.134068+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e659 with expected crc 2023-10-09T09:15:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:32 smithi163 bash[75425]: cluster 2023-10-09T09:15:31.136722+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e659 with expected crc 2023-10-09T09:15:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:32 smithi163 bash[75425]: cluster 2023-10-09T09:15:31.347202+0000 mgr.y (mgr.34107) 265 : cluster 0 pgmap v541: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 182 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 321 KiB/s wr, 3 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:32 smithi163 bash[75425]: cluster 2023-10-09T09:15:32.140684+0000 mon.a (mon.0) 803 : cluster 0 osdmap e660: 8 total, 7 up, 8 in 2023-10-09T09:15:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:32 smithi163 bash[78107]: cluster 2023-10-09T09:15:30.988826+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:32 smithi163 bash[78107]: cluster 2023-10-09T09:15:31.134068+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e659 with expected crc 2023-10-09T09:15:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:32 smithi163 bash[78107]: cluster 2023-10-09T09:15:31.136722+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e659 with expected crc 2023-10-09T09:15:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:32 smithi163 bash[78107]: cluster 2023-10-09T09:15:31.347202+0000 mgr.y (mgr.34107) 265 : cluster 0 pgmap v541: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 182 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 321 KiB/s wr, 3 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:32 smithi163 bash[78107]: cluster 2023-10-09T09:15:32.140684+0000 mon.a (mon.0) 803 : cluster 0 osdmap e660: 8 total, 7 up, 8 in 2023-10-09T09:15:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:32 smithi181 bash[59345]: cluster 2023-10-09T09:15:30.988826+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e658 with expected crc 2023-10-09T09:15:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:32 smithi181 bash[59345]: cluster 2023-10-09T09:15:31.134068+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e659 with expected crc 2023-10-09T09:15:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:32 smithi181 bash[59345]: cluster 2023-10-09T09:15:31.136722+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e659 with expected crc 2023-10-09T09:15:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:32 smithi181 bash[59345]: cluster 2023-10-09T09:15:31.347202+0000 mgr.y (mgr.34107) 265 : cluster 0 pgmap v541: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 182 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 321 KiB/s wr, 3 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:32.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:32 smithi181 bash[59345]: cluster 2023-10-09T09:15:32.140684+0000 mon.a (mon.0) 803 : cluster 0 osdmap e660: 8 total, 7 up, 8 in 2023-10-09T09:15:33.147 INFO:tasks.rados.rados.0.smithi163.stdout:2317: done (0 left) 2023-10-09T09:15:33.147 INFO:tasks.rados.rados.0.smithi163.stdout:2318: snap_create 2023-10-09T09:15:33.413 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:15:33] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.33.4" 2023-10-09T09:15:33.413 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[92392]: debug 2023-10-09T09:15:33.146+0000 7fb38f34d640 -1 osd.3 648 log_to_monitors true 2023-10-09T09:15:33.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[75425]: cluster 2023-10-09T09:15:32.136567+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e660 with expected crc 2023-10-09T09:15:33.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[75425]: cluster 2023-10-09T09:15:32.139503+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e660 with expected crc 2023-10-09T09:15:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[75425]: cluster 2023-10-09T09:15:32.334910+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e660 with expected crc 2023-10-09T09:15:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[75425]: cluster 2023-10-09T09:15:33.144521+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e661 with expected crc 2023-10-09T09:15:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[75425]: cluster 2023-10-09T09:15:33.146957+0000 mon.a (mon.0) 804 : cluster 0 osdmap e661: 8 total, 7 up, 8 in 2023-10-09T09:15:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[75425]: audit 2023-10-09T09:15:33.157604+0000 mon.c (mon.2) 31 : audit 1 from='osd.3 [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:15:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[75425]: audit 2023-10-09T09:15:33.157970+0000 mon.a (mon.0) 805 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:15:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[75425]: audit 2023-10-09T09:15:33.334753+0000 mon.a (mon.0) 806 : audit 0 from='client.? 172.21.15.163:0/3105331257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[78107]: cluster 2023-10-09T09:15:32.136567+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e660 with expected crc 2023-10-09T09:15:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[78107]: cluster 2023-10-09T09:15:32.139503+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e660 with expected crc 2023-10-09T09:15:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[78107]: cluster 2023-10-09T09:15:32.334910+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e660 with expected crc 2023-10-09T09:15:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[78107]: cluster 2023-10-09T09:15:33.144521+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e661 with expected crc 2023-10-09T09:15:33.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[78107]: cluster 2023-10-09T09:15:33.146957+0000 mon.a (mon.0) 804 : cluster 0 osdmap e661: 8 total, 7 up, 8 in 2023-10-09T09:15:33.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[78107]: audit 2023-10-09T09:15:33.157604+0000 mon.c (mon.2) 31 : audit 1 from='osd.3 [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:15:33.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[78107]: audit 2023-10-09T09:15:33.157970+0000 mon.a (mon.0) 805 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:15:33.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:33 smithi163 bash[78107]: audit 2023-10-09T09:15:33.334753+0000 mon.a (mon.0) 806 : audit 0 from='client.? 172.21.15.163:0/3105331257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:33 smithi181 bash[59345]: cluster 2023-10-09T09:15:32.136567+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e660 with expected crc 2023-10-09T09:15:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:33 smithi181 bash[59345]: cluster 2023-10-09T09:15:32.139503+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e660 with expected crc 2023-10-09T09:15:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:33 smithi181 bash[59345]: cluster 2023-10-09T09:15:32.334910+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e660 with expected crc 2023-10-09T09:15:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:33 smithi181 bash[59345]: cluster 2023-10-09T09:15:33.144521+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e661 with expected crc 2023-10-09T09:15:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:33 smithi181 bash[59345]: cluster 2023-10-09T09:15:33.146957+0000 mon.a (mon.0) 804 : cluster 0 osdmap e661: 8 total, 7 up, 8 in 2023-10-09T09:15:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:33 smithi181 bash[59345]: audit 2023-10-09T09:15:33.157604+0000 mon.c (mon.2) 31 : audit 1 from='osd.3 [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:15:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:33 smithi181 bash[59345]: audit 2023-10-09T09:15:33.157970+0000 mon.a (mon.0) 805 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:15:33.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:33 smithi181 bash[59345]: audit 2023-10-09T09:15:33.334753+0000 mon.a (mon.0) 806 : audit 0 from='client.? 172.21.15.163:0/3105331257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:34.147 INFO:tasks.rados.rados.0.smithi163.stdout:2318: done (0 left) 2023-10-09T09:15:34.147 INFO:tasks.rados.rados.0.smithi163.stdout:2319: snap_remove snap 287 2023-10-09T09:15:34.468 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[92392]: debug 2023-10-09T09:15:34.194+0000 7fb37efb6700 -1 osd.3 648 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:15:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[78107]: cluster 2023-10-09T09:15:33.142044+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e661 with expected crc 2023-10-09T09:15:34.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[78107]: cluster 2023-10-09T09:15:33.347884+0000 mgr.y (mgr.34107) 266 : cluster 0 pgmap v544: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 182 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 321 KiB/s wr, 5 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[78107]: audit 2023-10-09T09:15:34.147113+0000 mon.a (mon.0) 807 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T09:15:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[78107]: cluster 2023-10-09T09:15:34.147248+0000 mon.a (mon.0) 808 : cluster 0 osdmap e662: 8 total, 7 up, 8 in 2023-10-09T09:15:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[78107]: audit 2023-10-09T09:15:34.149022+0000 mon.c (mon.2) 32 : audit 1 from='osd.3 [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[78107]: audit 2023-10-09T09:15:34.154146+0000 mon.a (mon.0) 809 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[75425]: cluster 2023-10-09T09:15:33.142044+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e661 with expected crc 2023-10-09T09:15:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[75425]: cluster 2023-10-09T09:15:33.347884+0000 mgr.y (mgr.34107) 266 : cluster 0 pgmap v544: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 182 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 321 KiB/s wr, 5 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[75425]: audit 2023-10-09T09:15:34.147113+0000 mon.a (mon.0) 807 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T09:15:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[75425]: cluster 2023-10-09T09:15:34.147248+0000 mon.a (mon.0) 808 : cluster 0 osdmap e662: 8 total, 7 up, 8 in 2023-10-09T09:15:34.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[75425]: audit 2023-10-09T09:15:34.149022+0000 mon.c (mon.2) 32 : audit 1 from='osd.3 [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:34.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:34 smithi163 bash[75425]: audit 2023-10-09T09:15:34.154146+0000 mon.a (mon.0) 809 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:34 smithi181 bash[59345]: cluster 2023-10-09T09:15:33.142044+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e661 with expected crc 2023-10-09T09:15:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:34 smithi181 bash[59345]: cluster 2023-10-09T09:15:33.347884+0000 mgr.y (mgr.34107) 266 : cluster 0 pgmap v544: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 182 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 321 KiB/s wr, 5 op/s; 17/357 objects degraded (4.762%) 2023-10-09T09:15:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:34 smithi181 bash[59345]: audit 2023-10-09T09:15:34.147113+0000 mon.a (mon.0) 807 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T09:15:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:34 smithi181 bash[59345]: cluster 2023-10-09T09:15:34.147248+0000 mon.a (mon.0) 808 : cluster 0 osdmap e662: 8 total, 7 up, 8 in 2023-10-09T09:15:34.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:34 smithi181 bash[59345]: audit 2023-10-09T09:15:34.149022+0000 mon.c (mon.2) 32 : audit 1 from='osd.3 [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:34.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:34 smithi181 bash[59345]: audit 2023-10-09T09:15:34.154146+0000 mon.a (mon.0) 809 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:15:35.149 INFO:tasks.rados.rados.0.smithi163.stdout:2319: done (0 left) 2023-10-09T09:15:35.149 INFO:tasks.rados.rados.0.smithi163.stdout:2320: write oid 50 current snap is 304 2023-10-09T09:15:35.150 INFO:tasks.rados.rados.0.smithi163.stdout:2320: seq_num 868 ranges {588476=580374,1643429=603638} 2023-10-09T09:15:35.159 INFO:tasks.rados.rados.0.smithi163.stdout:2320: writing smithi16367185-50 from 588476 to 1168850 tid 1 2023-10-09T09:15:35.167 INFO:tasks.rados.rados.0.smithi163.stdout:2320: writing smithi16367185-50 from 1643429 to 2247067 tid 2 2023-10-09T09:15:35.168 INFO:tasks.rados.rados.0.smithi163.stdout:2321: read oid 4 snap -1 2023-10-09T09:15:35.168 INFO:tasks.rados.rados.0.smithi163.stdout:2321: expect deleted 2023-10-09T09:15:35.168 INFO:tasks.rados.rados.0.smithi163.stdout:2322: delete oid 12 current snap is 304 2023-10-09T09:15:35.169 INFO:tasks.rados.rados.0.smithi163.stdout:2320: finishing write tid 1 to smithi16367185-50 2023-10-09T09:15:35.172 INFO:tasks.rados.rados.0.smithi163.stdout:2320: finishing write tid 2 to smithi16367185-50 2023-10-09T09:15:35.172 INFO:tasks.rados.rados.0.smithi163.stdout:2320: finishing write tid 3 to smithi16367185-50 2023-10-09T09:15:35.173 INFO:tasks.rados.rados.0.smithi163.stdout:2320: finishing write tid 4 to smithi16367185-50 2023-10-09T09:15:35.173 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1015 (ObjNum 868 snap 304 seq_num 868) dirty exists 2023-10-09T09:15:35.173 INFO:tasks.rados.rados.0.smithi163.stdout:2320: left oid 50 (ObjNum 868 snap 304 seq_num 868) 2023-10-09T09:15:35.178 INFO:tasks.rados.rados.0.smithi163.stdout:2320: done (2 left) 2023-10-09T09:15:35.178 INFO:tasks.rados.rados.0.smithi163.stdout:2322: done (1 left) 2023-10-09T09:15:35.178 INFO:tasks.rados.rados.0.smithi163.stdout:2323: read oid 23 snap -1 2023-10-09T09:15:35.179 INFO:tasks.rados.rados.0.smithi163.stdout:2323: expect (ObjNum 775 snap 268 seq_num 775) 2023-10-09T09:15:35.179 INFO:tasks.rados.rados.0.smithi163.stdout:2324: read oid 5 snap -1 2023-10-09T09:15:35.179 INFO:tasks.rados.rados.0.smithi163.stdout:2324: expect (ObjNum 840 snap 289 seq_num 840) 2023-10-09T09:15:35.179 INFO:tasks.rados.rados.0.smithi163.stdout:2325: read oid 21 snap -1 2023-10-09T09:15:35.179 INFO:tasks.rados.rados.0.smithi163.stdout:2325: expect (ObjNum 867 snap 301 seq_num 867) 2023-10-09T09:15:35.179 INFO:tasks.rados.rados.0.smithi163.stdout:2326: write oid 8 current snap is 304 2023-10-09T09:15:35.179 INFO:tasks.rados.rados.0.smithi163.stdout:2326: seq_num 870 ranges {622090=730872,1760393=607307} 2023-10-09T09:15:35.187 INFO:tasks.rados.rados.0.smithi163.stdout:2326: writing smithi16367185-8 from 622090 to 1352962 tid 1 2023-10-09T09:15:35.194 INFO:tasks.rados.rados.0.smithi163.stdout:2326: writing smithi16367185-8 from 1760393 to 2367700 tid 2 2023-10-09T09:15:35.195 INFO:tasks.rados.rados.0.smithi163.stdout:2327: write oid 9 current snap is 304 2023-10-09T09:15:35.195 INFO:tasks.rados.rados.0.smithi163.stdout:2327: seq_num 871 ranges {638897=406121,1818875=750965} 2023-10-09T09:15:35.202 INFO:tasks.rados.rados.0.smithi163.stdout:2327: writing smithi16367185-9 from 638897 to 1045018 tid 1 2023-10-09T09:15:35.212 INFO:tasks.rados.rados.0.smithi163.stdout:2327: writing smithi16367185-9 from 1818875 to 2569840 tid 2 2023-10-09T09:15:35.212 INFO:tasks.rados.rados.0.smithi163.stdout:2328: snap_remove snap 302 2023-10-09T09:15:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[78107]: cluster 2023-10-09T09:15:34.142638+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e662 with expected crc 2023-10-09T09:15:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[78107]: cluster 2023-10-09T09:15:34.715321+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e662 with expected crc 2023-10-09T09:15:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[78107]: cluster 2023-10-09T09:15:35.131080+0000 mon.a (mon.0) 810 : cluster 3 Health check update: Degraded data redundancy: 17/357 objects degraded (4.762%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[78107]: cluster 2023-10-09T09:15:35.143448+0000 mon.a (mon.0) 811 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:15:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[78107]: cluster 2023-10-09T09:15:35.148576+0000 mon.a (mon.0) 812 : cluster 1 osd.3 [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121] boot 2023-10-09T09:15:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[78107]: cluster 2023-10-09T09:15:35.148617+0000 mon.a (mon.0) 813 : cluster 0 osdmap e663: 8 total, 8 up, 8 in 2023-10-09T09:15:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[78107]: audit 2023-10-09T09:15:35.149414+0000 mon.a (mon.0) 814 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:15:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[75425]: cluster 2023-10-09T09:15:34.142638+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e662 with expected crc 2023-10-09T09:15:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[75425]: cluster 2023-10-09T09:15:34.715321+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e662 with expected crc 2023-10-09T09:15:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[75425]: cluster 2023-10-09T09:15:35.131080+0000 mon.a (mon.0) 810 : cluster 3 Health check update: Degraded data redundancy: 17/357 objects degraded (4.762%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[75425]: cluster 2023-10-09T09:15:35.143448+0000 mon.a (mon.0) 811 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:15:35.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[75425]: cluster 2023-10-09T09:15:35.148576+0000 mon.a (mon.0) 812 : cluster 1 osd.3 [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121] boot 2023-10-09T09:15:35.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[75425]: cluster 2023-10-09T09:15:35.148617+0000 mon.a (mon.0) 813 : cluster 0 osdmap e663: 8 total, 8 up, 8 in 2023-10-09T09:15:35.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:35 smithi163 bash[75425]: audit 2023-10-09T09:15:35.149414+0000 mon.a (mon.0) 814 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:15:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T09:15:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:35 smithi181 bash[59345]: cluster 2023-10-09T09:15:34.142638+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e662 with expected crc 2023-10-09T09:15:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:35 smithi181 bash[59345]: cluster 2023-10-09T09:15:34.715321+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e662 with expected crc 2023-10-09T09:15:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:35 smithi181 bash[59345]: cluster 2023-10-09T09:15:35.131080+0000 mon.a (mon.0) 810 : cluster 3 Health check update: Degraded data redundancy: 17/357 objects degraded (4.762%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:15:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:35 smithi181 bash[59345]: cluster 2023-10-09T09:15:35.143448+0000 mon.a (mon.0) 811 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:15:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:35 smithi181 bash[59345]: cluster 2023-10-09T09:15:35.148576+0000 mon.a (mon.0) 812 : cluster 1 osd.3 [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121] boot 2023-10-09T09:15:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:35 smithi181 bash[59345]: cluster 2023-10-09T09:15:35.148617+0000 mon.a (mon.0) 813 : cluster 0 osdmap e663: 8 total, 8 up, 8 in 2023-10-09T09:15:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:35 smithi181 bash[59345]: audit 2023-10-09T09:15:35.149414+0000 mon.a (mon.0) 814 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:15:36.167 INFO:tasks.rados.rados.0.smithi163.stdout:2328: done (6 left) 2023-10-09T09:15:36.167 INFO:tasks.rados.rados.0.smithi163.stdout:2329: rollback oid 16 current snap is 304 2023-10-09T09:15:36.168 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 16 to 290 2023-10-09T09:15:36.168 INFO:tasks.rados.rados.0.smithi163.stdout:2325: expect (ObjNum 867 snap 301 seq_num 867) 2023-10-09T09:15:36.262 INFO:tasks.rados.rados.0.smithi163.stdout:2323: expect (ObjNum 775 snap 268 seq_num 775) 2023-10-09T09:15:36.435 INFO:tasks.rados.rados.0.smithi163.stdout:2326: finishing write tid 1 to smithi16367185-8 2023-10-09T09:15:36.435 INFO:tasks.rados.rados.0.smithi163.stdout:2326: finishing write tid 2 to smithi16367185-8 2023-10-09T09:15:36.435 INFO:tasks.rados.rados.0.smithi163.stdout:2326: finishing write tid 3 to smithi16367185-8 2023-10-09T09:15:36.436 INFO:tasks.rados.rados.0.smithi163.stdout:2326: finishing write tid 4 to smithi16367185-8 2023-10-09T09:15:36.436 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1137 (ObjNum 870 snap 304 seq_num 870) dirty exists 2023-10-09T09:15:36.436 INFO:tasks.rados.rados.0.smithi163.stdout:2326: left oid 8 (ObjNum 870 snap 304 seq_num 870) 2023-10-09T09:15:36.436 INFO:tasks.rados.rados.0.smithi163.stdout:2327: finishing write tid 1 to smithi16367185-9 2023-10-09T09:15:36.436 INFO:tasks.rados.rados.0.smithi163.stdout:2324: expect (ObjNum 840 snap 289 seq_num 840) 2023-10-09T09:15:36.507 INFO:tasks.rados.rados.0.smithi163.stdout:2327: finishing write tid 2 to smithi16367185-9 2023-10-09T09:15:36.507 INFO:tasks.rados.rados.0.smithi163.stdout:2327: finishing write tid 3 to smithi16367185-9 2023-10-09T09:15:36.507 INFO:tasks.rados.rados.0.smithi163.stdout:2321: done (6 left) 2023-10-09T09:15:36.507 INFO:tasks.rados.rados.0.smithi163.stdout:2323: done (5 left) 2023-10-09T09:15:36.507 INFO:tasks.rados.rados.0.smithi163.stdout:2324: done (4 left) 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:2325: done (3 left) 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:2326: done (2 left) 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:2330: rollback oid 23 current snap is 304 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 23 to 301 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:2327: finishing write tid 4 to smithi16367185-9 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 599 (ObjNum 871 snap 304 seq_num 871) dirty exists 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:2327: left oid 9 (ObjNum 871 snap 304 seq_num 871) 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:2329: finishing rollback tid 1 to smithi16367185-16 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1127 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:2327: done (2 left) 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:2329: done (1 left) 2023-10-09T09:15:36.508 INFO:tasks.rados.rados.0.smithi163.stdout:2331: rollback oid 4 current snap is 304 2023-10-09T09:15:36.509 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 298 2023-10-09T09:15:36.509 INFO:tasks.rados.rados.0.smithi163.stdout:2332: read oid 48 snap 293 2023-10-09T09:15:36.509 INFO:tasks.rados.rados.0.smithi163.stdout:2332: expect (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:15:36.509 INFO:tasks.rados.rados.0.smithi163.stdout:2333: read oid 10 snap 291 2023-10-09T09:15:36.509 INFO:tasks.rados.rados.0.smithi163.stdout:2333: expect (ObjNum 773 snap 267 seq_num 773) 2023-10-09T09:15:36.509 INFO:tasks.rados.rados.0.smithi163.stdout:2334: rollback oid 16 current snap is 304 2023-10-09T09:15:36.509 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 16 to 290 2023-10-09T09:15:36.509 INFO:tasks.rados.rados.0.smithi163.stdout:2335: delete oid 18 current snap is 304 2023-10-09T09:15:36.510 INFO:tasks.rados.rados.0.smithi163.stdout:2331: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:15:36.510 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1158 (ObjNum 0 snap 32603 seq_num 698928128) dirty dne 2023-10-09T09:15:36.510 INFO:tasks.rados.rados.0.smithi163.stdout:2334: finishing rollback tid 1 to smithi16367185-16 2023-10-09T09:15:36.510 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1127 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:15:36.514 INFO:tasks.rados.rados.0.smithi163.stdout:2333: expect (ObjNum 773 snap 267 seq_num 773) 2023-10-09T09:15:36.613 INFO:tasks.rados.rados.0.smithi163.stdout:2330: finishing rollback tid 0 to smithi16367185-23 2023-10-09T09:15:36.613 INFO:tasks.rados.rados.0.smithi163.stdout:2330: finishing rollback tid 1 to smithi16367185-23 2023-10-09T09:15:36.613 INFO:tasks.rados.rados.0.smithi163.stdout:2330: finishing rollback tid 2 to smithi16367185-23 2023-10-09T09:15:36.613 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1021 (ObjNum 775 snap 268 seq_num 775) dirty exists 2023-10-09T09:15:36.613 INFO:tasks.rados.rados.0.smithi163.stdout:2332: expect (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:15:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[78107]: cluster 2023-10-09T09:15:35.145249+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e663 with expected crc 2023-10-09T09:15:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[78107]: cluster 2023-10-09T09:15:35.145892+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e663 with expected crc 2023-10-09T09:15:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[78107]: cluster 2023-10-09T09:15:35.348848+0000 mgr.y (mgr.34107) 267 : cluster 0 pgmap v547: 89 pgs: 3 peering, 5 active+undersized+degraded, 26 active+undersized, 55 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 305 KiB/s wr, 3 op/s; 10/357 objects degraded (2.801%) 2023-10-09T09:15:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[78107]: audit 2023-10-09T09:15:35.709089+0000 mon.a (mon.0) 815 : audit 0 from='client.? 172.21.15.163:0/3094764501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[78107]: cluster 2023-10-09T09:15:36.156639+0000 mon.a (mon.0) 816 : cluster 0 osdmap e664: 8 total, 8 up, 8 in 2023-10-09T09:15:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[78107]: cluster 2023-10-09T09:15:36.159966+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[78107]: cluster 2023-10-09T09:15:36.163241+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[78107]: cluster 2023-10-09T09:15:36.166413+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[78107]: cluster 2023-10-09T09:15:36.169504+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[75425]: cluster 2023-10-09T09:15:35.145249+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e663 with expected crc 2023-10-09T09:15:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[75425]: cluster 2023-10-09T09:15:35.145892+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e663 with expected crc 2023-10-09T09:15:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[75425]: cluster 2023-10-09T09:15:35.348848+0000 mgr.y (mgr.34107) 267 : cluster 0 pgmap v547: 89 pgs: 3 peering, 5 active+undersized+degraded, 26 active+undersized, 55 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 305 KiB/s wr, 3 op/s; 10/357 objects degraded (2.801%) 2023-10-09T09:15:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[75425]: audit 2023-10-09T09:15:35.709089+0000 mon.a (mon.0) 815 : audit 0 from='client.? 172.21.15.163:0/3094764501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[75425]: cluster 2023-10-09T09:15:36.156639+0000 mon.a (mon.0) 816 : cluster 0 osdmap e664: 8 total, 8 up, 8 in 2023-10-09T09:15:36.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[75425]: cluster 2023-10-09T09:15:36.159966+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[75425]: cluster 2023-10-09T09:15:36.163241+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[75425]: cluster 2023-10-09T09:15:36.166413+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:36 smithi163 bash[75425]: cluster 2023-10-09T09:15:36.169504+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.777 INFO:tasks.rados.rados.0.smithi163.stdout:2330: done (5 left) 2023-10-09T09:15:36.778 INFO:tasks.rados.rados.0.smithi163.stdout:2331: done (4 left) 2023-10-09T09:15:36.778 INFO:tasks.rados.rados.0.smithi163.stdout:2332: done (3 left) 2023-10-09T09:15:36.778 INFO:tasks.rados.rados.0.smithi163.stdout:2333: done (2 left) 2023-10-09T09:15:36.778 INFO:tasks.rados.rados.0.smithi163.stdout:2334: done (1 left) 2023-10-09T09:15:36.778 INFO:tasks.rados.rados.0.smithi163.stdout:2335: done (0 left) 2023-10-09T09:15:36.778 INFO:tasks.rados.rados.0.smithi163.stdout:2336: write oid 6 current snap is 304 2023-10-09T09:15:36.778 INFO:tasks.rados.rados.0.smithi163.stdout:2336: seq_num 873 ranges {672511=556619,1819486=754634} 2023-10-09T09:15:36.788 INFO:tasks.rados.rados.0.smithi163.stdout:2336: writing smithi16367185-6 from 672511 to 1229130 tid 1 2023-10-09T09:15:36.796 INFO:tasks.rados.rados.0.smithi163.stdout:2336: writing smithi16367185-6 from 1819486 to 2574120 tid 2 2023-10-09T09:15:36.796 INFO:tasks.rados.rados.0.smithi163.stdout:2337: rollback oid 24 current snap is 304 2023-10-09T09:15:36.796 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 24 to 290 2023-10-09T09:15:36.796 INFO:tasks.rados.rados.0.smithi163.stdout:2336: finishing write tid 1 to smithi16367185-6 2023-10-09T09:15:36.796 INFO:tasks.rados.rados.0.smithi163.stdout:2338: snap_remove snap 297 2023-10-09T09:15:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:36 smithi181 bash[59345]: cluster 2023-10-09T09:15:35.145249+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e663 with expected crc 2023-10-09T09:15:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:36 smithi181 bash[59345]: cluster 2023-10-09T09:15:35.145892+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e663 with expected crc 2023-10-09T09:15:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:36 smithi181 bash[59345]: cluster 2023-10-09T09:15:35.348848+0000 mgr.y (mgr.34107) 267 : cluster 0 pgmap v547: 89 pgs: 3 peering, 5 active+undersized+degraded, 26 active+undersized, 55 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 305 KiB/s wr, 3 op/s; 10/357 objects degraded (2.801%) 2023-10-09T09:15:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:36 smithi181 bash[59345]: audit 2023-10-09T09:15:35.709089+0000 mon.a (mon.0) 815 : audit 0 from='client.? 172.21.15.163:0/3094764501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:15:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:36 smithi181 bash[59345]: cluster 2023-10-09T09:15:36.156639+0000 mon.a (mon.0) 816 : cluster 0 osdmap e664: 8 total, 8 up, 8 in 2023-10-09T09:15:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:36 smithi181 bash[59345]: cluster 2023-10-09T09:15:36.159966+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:36 smithi181 bash[59345]: cluster 2023-10-09T09:15:36.163241+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:36 smithi181 bash[59345]: cluster 2023-10-09T09:15:36.166413+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:36 smithi181 bash[59345]: cluster 2023-10-09T09:15:36.169504+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T09:15:37.430 INFO:tasks.rados.rados.0.smithi163.stdout:2338: done (2 left) 2023-10-09T09:15:37.431 INFO:tasks.rados.rados.0.smithi163.stdout:2339: write oid 50 current snap is 304 2023-10-09T09:15:37.431 INFO:tasks.rados.rados.0.smithi163.stdout:2337: finishing rollback tid 1 to smithi16367185-24 2023-10-09T09:15:37.431 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 549 (ObjNum 698836960 snap 21875 seq_num 698953808) dirty dne 2023-10-09T09:15:37.431 INFO:tasks.rados.rados.0.smithi163.stdout:2339: seq_num 874 ranges {689318=631868,1877968=498292} 2023-10-09T09:15:37.438 INFO:tasks.rados.rados.0.smithi163.stdout:2339: writing smithi16367185-50 from 689318 to 1321186 tid 1 2023-10-09T09:15:37.444 INFO:tasks.rados.rados.0.smithi163.stdout:2339: writing smithi16367185-50 from 1877968 to 2376260 tid 2 2023-10-09T09:15:37.444 INFO:tasks.rados.rados.0.smithi163.stdout:2337: done (2 left) 2023-10-09T09:15:37.444 INFO:tasks.rados.rados.0.smithi163.stdout:2340: snap_remove snap 296 2023-10-09T09:15:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:37 smithi163 bash[75425]: cluster 2023-10-09T09:15:36.152757+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:37 smithi163 bash[75425]: cluster 2023-10-09T09:15:36.155490+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:37 smithi163 bash[75425]: cluster 2023-10-09T09:15:36.158236+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:37 smithi163 bash[75425]: cluster 2023-10-09T09:15:36.166011+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:37 smithi163 bash[75425]: cluster 2023-10-09T09:15:36.168121+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:37 smithi163 bash[78107]: cluster 2023-10-09T09:15:36.152757+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:37 smithi163 bash[78107]: cluster 2023-10-09T09:15:36.155490+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:37 smithi163 bash[78107]: cluster 2023-10-09T09:15:36.158236+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:37 smithi163 bash[78107]: cluster 2023-10-09T09:15:36.166011+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:37 smithi163 bash[78107]: cluster 2023-10-09T09:15:36.168121+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:37 smithi181 bash[59345]: cluster 2023-10-09T09:15:36.152757+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:37 smithi181 bash[59345]: cluster 2023-10-09T09:15:36.155490+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:37 smithi181 bash[59345]: cluster 2023-10-09T09:15:36.158236+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:37 smithi181 bash[59345]: cluster 2023-10-09T09:15:36.166011+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:37 smithi181 bash[59345]: cluster 2023-10-09T09:15:36.168121+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e664 with expected crc 2023-10-09T09:15:38.438 INFO:tasks.rados.rados.0.smithi163.stdout:2340: done (2 left) 2023-10-09T09:15:38.438 INFO:tasks.rados.rados.0.smithi163.stdout:2341: snap_create 2023-10-09T09:15:38.438 INFO:tasks.rados.rados.0.smithi163.stdout:2336: finishing write tid 2 to smithi16367185-6 2023-10-09T09:15:38.438 INFO:tasks.rados.rados.0.smithi163.stdout:2336: finishing write tid 3 to smithi16367185-6 2023-10-09T09:15:38.438 INFO:tasks.rados.rados.0.smithi163.stdout:2336: finishing write tid 4 to smithi16367185-6 2023-10-09T09:15:38.438 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 218 (ObjNum 873 snap 304 seq_num 873) dirty exists 2023-10-09T09:15:38.439 INFO:tasks.rados.rados.0.smithi163.stdout:2336: left oid 6 (ObjNum 873 snap 304 seq_num 873) 2023-10-09T09:15:38.439 INFO:tasks.rados.rados.0.smithi163.stdout:2339: finishing write tid 1 to smithi16367185-50 2023-10-09T09:15:38.439 INFO:tasks.rados.rados.0.smithi163.stdout:2339: finishing write tid 2 to smithi16367185-50 2023-10-09T09:15:38.439 INFO:tasks.rados.rados.0.smithi163.stdout:2339: finishing write tid 3 to smithi16367185-50 2023-10-09T09:15:38.439 INFO:tasks.rados.rados.0.smithi163.stdout:2339: finishing write tid 4 to smithi16367185-50 2023-10-09T09:15:38.439 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1028 (ObjNum 874 snap 304 seq_num 874) dirty exists 2023-10-09T09:15:38.439 INFO:tasks.rados.rados.0.smithi163.stdout:2339: left oid 50 (ObjNum 874 snap 304 seq_num 874) 2023-10-09T09:15:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:38 smithi163 bash[78107]: cluster 2023-10-09T09:15:37.349565+0000 mgr.y (mgr.34107) 268 : cluster 0 pgmap v549: 89 pgs: 3 peering, 3 active+undersized+degraded, 14 active+undersized, 69 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 876 KiB/s wr, 5 op/s; 7/357 objects degraded (1.961%) 2023-10-09T09:15:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:38 smithi163 bash[78107]: cluster 2023-10-09T09:15:37.426346+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e665 with expected crc 2023-10-09T09:15:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:38 smithi163 bash[78107]: cluster 2023-10-09T09:15:37.430736+0000 mon.a (mon.0) 817 : cluster 0 osdmap e665: 8 total, 8 up, 8 in 2023-10-09T09:15:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:38 smithi163 bash[75425]: cluster 2023-10-09T09:15:37.349565+0000 mgr.y (mgr.34107) 268 : cluster 0 pgmap v549: 89 pgs: 3 peering, 3 active+undersized+degraded, 14 active+undersized, 69 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 876 KiB/s wr, 5 op/s; 7/357 objects degraded (1.961%) 2023-10-09T09:15:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:38 smithi163 bash[75425]: cluster 2023-10-09T09:15:37.426346+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e665 with expected crc 2023-10-09T09:15:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:38 smithi163 bash[75425]: cluster 2023-10-09T09:15:37.430736+0000 mon.a (mon.0) 817 : cluster 0 osdmap e665: 8 total, 8 up, 8 in 2023-10-09T09:15:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:38 smithi181 bash[59345]: cluster 2023-10-09T09:15:37.349565+0000 mgr.y (mgr.34107) 268 : cluster 0 pgmap v549: 89 pgs: 3 peering, 3 active+undersized+degraded, 14 active+undersized, 69 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 876 KiB/s wr, 5 op/s; 7/357 objects degraded (1.961%) 2023-10-09T09:15:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:38 smithi181 bash[59345]: cluster 2023-10-09T09:15:37.426346+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e665 with expected crc 2023-10-09T09:15:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:38 smithi181 bash[59345]: cluster 2023-10-09T09:15:37.430736+0000 mon.a (mon.0) 817 : cluster 0 osdmap e665: 8 total, 8 up, 8 in 2023-10-09T09:15:39.450 INFO:tasks.rados.rados.0.smithi163.stdout:2336: done (2 left) 2023-10-09T09:15:39.451 INFO:tasks.rados.rados.0.smithi163.stdout:2339: done (1 left) 2023-10-09T09:15:39.451 INFO:tasks.rados.rados.0.smithi163.stdout:2341: done (0 left) 2023-10-09T09:15:39.451 INFO:tasks.rados.rados.0.smithi163.stdout:2342: delete oid 31 current snap is 305 2023-10-09T09:15:39.456 INFO:tasks.rados.rados.0.smithi163.stdout:2342: done (0 left) 2023-10-09T09:15:39.456 INFO:tasks.rados.rados.0.smithi163.stdout:2343: write oid 40 current snap is 305 2023-10-09T09:15:39.456 INFO:tasks.rados.rados.0.smithi163.stdout:2343: seq_num 876 ranges {722932=498719,1994932=501961} 2023-10-09T09:15:39.464 INFO:tasks.rados.rados.0.smithi163.stdout:2343: writing smithi16367185-40 from 722932 to 1221651 tid 1 2023-10-09T09:15:39.472 INFO:tasks.rados.rados.0.smithi163.stdout:2343: writing smithi16367185-40 from 1994932 to 2496893 tid 2 2023-10-09T09:15:39.472 INFO:tasks.rados.rados.0.smithi163.stdout:2344: read oid 6 snap -1 2023-10-09T09:15:39.473 INFO:tasks.rados.rados.0.smithi163.stdout:2344: expect (ObjNum 873 snap 304 seq_num 873) 2023-10-09T09:15:39.473 INFO:tasks.rados.rados.0.smithi163.stdout:2345: write oid 34 current snap is 305 2023-10-09T09:15:39.473 INFO:tasks.rados.rados.0.smithi163.stdout:2345: seq_num 877 ranges {739739=573968,2053414=686325} 2023-10-09T09:15:39.479 INFO:tasks.rados.rados.0.smithi163.stdout:2345: writing smithi16367185-34 from 739739 to 1313707 tid 1 2023-10-09T09:15:39.489 INFO:tasks.rados.rados.0.smithi163.stdout:2345: writing smithi16367185-34 from 2053414 to 2739739 tid 2 2023-10-09T09:15:39.489 INFO:tasks.rados.rados.0.smithi163.stdout:2346: read oid 29 snap 291 2023-10-09T09:15:39.489 INFO:tasks.rados.rados.0.smithi163.stdout:2346: expect (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:15:39.489 INFO:tasks.rados.rados.0.smithi163.stdout:2347: rollback oid 30 current snap is 305 2023-10-09T09:15:39.489 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 301 2023-10-09T09:15:39.489 INFO:tasks.rados.rados.0.smithi163.stdout:2343: finishing write tid 1 to smithi16367185-40 2023-10-09T09:15:39.489 INFO:tasks.rados.rados.0.smithi163.stdout:2343: finishing write tid 2 to smithi16367185-40 2023-10-09T09:15:39.489 INFO:tasks.rados.rados.0.smithi163.stdout:2343: finishing write tid 3 to smithi16367185-40 2023-10-09T09:15:39.490 INFO:tasks.rados.rados.0.smithi163.stdout:2348: delete oid 46 current snap is 305 2023-10-09T09:15:39.490 INFO:tasks.rados.rados.0.smithi163.stdout:2343: finishing write tid 4 to smithi16367185-40 2023-10-09T09:15:39.490 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1141 (ObjNum 876 snap 305 seq_num 876) dirty exists 2023-10-09T09:15:39.490 INFO:tasks.rados.rados.0.smithi163.stdout:2343: left oid 40 (ObjNum 876 snap 305 seq_num 876) 2023-10-09T09:15:39.490 INFO:tasks.rados.rados.0.smithi163.stdout:2344: expect (ObjNum 873 snap 304 seq_num 873) 2023-10-09T09:15:39.684 INFO:tasks.rados.rados.0.smithi163.stdout:2345: finishing write tid 1 to smithi16367185-34 2023-10-09T09:15:39.684 INFO:tasks.rados.rados.0.smithi163.stdout:2347: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:15:39.684 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 0 (ObjNum 669362160 snap 21875 seq_num 1) dirty dne 2023-10-09T09:15:39.685 INFO:tasks.rados.rados.0.smithi163.stdout:2345: finishing write tid 2 to smithi16367185-34 2023-10-09T09:15:39.685 INFO:tasks.rados.rados.0.smithi163.stdout:2345: finishing write tid 3 to smithi16367185-34 2023-10-09T09:15:39.685 INFO:tasks.rados.rados.0.smithi163.stdout:2346: expect (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:15:39.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:39 smithi163 bash[75425]: cluster 2023-10-09T09:15:38.434408+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e666 with expected crc 2023-10-09T09:15:39.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:39 smithi163 bash[75425]: cluster 2023-10-09T09:15:38.438191+0000 mon.a (mon.0) 818 : cluster 0 osdmap e666: 8 total, 8 up, 8 in 2023-10-09T09:15:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:39 smithi163 bash[78107]: cluster 2023-10-09T09:15:38.434408+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e666 with expected crc 2023-10-09T09:15:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:39 smithi163 bash[78107]: cluster 2023-10-09T09:15:38.438191+0000 mon.a (mon.0) 818 : cluster 0 osdmap e666: 8 total, 8 up, 8 in 2023-10-09T09:15:39.790 INFO:tasks.rados.rados.0.smithi163.stdout:2345: finishing write tid 4 to smithi16367185-34 2023-10-09T09:15:39.790 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1035 (ObjNum 877 snap 305 seq_num 877) dirty exists 2023-10-09T09:15:39.790 INFO:tasks.rados.rados.0.smithi163.stdout:2345: left oid 34 (ObjNum 877 snap 305 seq_num 877) 2023-10-09T09:15:39.790 INFO:tasks.rados.rados.0.smithi163.stdout:2343: done (5 left) 2023-10-09T09:15:39.790 INFO:tasks.rados.rados.0.smithi163.stdout:2344: done (4 left) 2023-10-09T09:15:39.790 INFO:tasks.rados.rados.0.smithi163.stdout:2345: done (3 left) 2023-10-09T09:15:39.790 INFO:tasks.rados.rados.0.smithi163.stdout:2346: done (2 left) 2023-10-09T09:15:39.791 INFO:tasks.rados.rados.0.smithi163.stdout:2347: done (1 left) 2023-10-09T09:15:39.791 INFO:tasks.rados.rados.0.smithi163.stdout:2348: done (0 left) 2023-10-09T09:15:39.791 INFO:tasks.rados.rados.0.smithi163.stdout:2349: snap_create 2023-10-09T09:15:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:39 smithi181 bash[59345]: cluster 2023-10-09T09:15:38.434408+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e666 with expected crc 2023-10-09T09:15:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:39 smithi181 bash[59345]: cluster 2023-10-09T09:15:38.438191+0000 mon.a (mon.0) 818 : cluster 0 osdmap e666: 8 total, 8 up, 8 in 2023-10-09T09:15:40.466 INFO:tasks.rados.rados.0.smithi163.stdout:2349: done (0 left) 2023-10-09T09:15:40.469 INFO:tasks.rados.rados.0.smithi163.stdout:2350: read oid 3 snap -1 2023-10-09T09:15:40.469 INFO:tasks.rados.rados.0.smithi163.stdout:2350: expect (ObjNum 789 snap 271 seq_num 789) 2023-10-09T09:15:40.469 INFO:tasks.rados.rados.0.smithi163.stdout:2351: snap_remove snap 285 2023-10-09T09:15:40.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[75425]: cluster 2023-10-09T09:15:39.350479+0000 mgr.y (mgr.34107) 269 : cluster 0 pgmap v552: 89 pgs: 3 peering, 86 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 855 KiB/s wr, 7 op/s 2023-10-09T09:15:40.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[75425]: cluster 2023-10-09T09:15:39.434851+0000 mon.a (mon.0) 819 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/357 objects degraded (1.961%), 3 pgs degraded) 2023-10-09T09:15:40.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[75425]: cluster 2023-10-09T09:15:39.434934+0000 mon.a (mon.0) 820 : cluster 1 Cluster is now healthy 2023-10-09T09:15:40.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[75425]: cluster 2023-10-09T09:15:39.445528+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e667 with expected crc 2023-10-09T09:15:40.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[75425]: cluster 2023-10-09T09:15:39.450551+0000 mon.a (mon.0) 821 : cluster 0 osdmap e667: 8 total, 8 up, 8 in 2023-10-09T09:15:40.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[75425]: cluster 2023-10-09T09:15:39.453860+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e667 with expected crc 2023-10-09T09:15:40.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[75425]: cluster 2023-10-09T09:15:39.465958+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e667 with expected crc 2023-10-09T09:15:40.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[75425]: audit 2023-10-09T09:15:40.351644+0000 mon.a (mon.0) 822 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:40.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[75425]: audit 2023-10-09T09:15:40.352857+0000 mon.a (mon.0) 823 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[78107]: cluster 2023-10-09T09:15:39.350479+0000 mgr.y (mgr.34107) 269 : cluster 0 pgmap v552: 89 pgs: 3 peering, 86 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 855 KiB/s wr, 7 op/s 2023-10-09T09:15:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[78107]: cluster 2023-10-09T09:15:39.434851+0000 mon.a (mon.0) 819 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/357 objects degraded (1.961%), 3 pgs degraded) 2023-10-09T09:15:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[78107]: cluster 2023-10-09T09:15:39.434934+0000 mon.a (mon.0) 820 : cluster 1 Cluster is now healthy 2023-10-09T09:15:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[78107]: cluster 2023-10-09T09:15:39.445528+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e667 with expected crc 2023-10-09T09:15:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[78107]: cluster 2023-10-09T09:15:39.450551+0000 mon.a (mon.0) 821 : cluster 0 osdmap e667: 8 total, 8 up, 8 in 2023-10-09T09:15:40.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[78107]: cluster 2023-10-09T09:15:39.453860+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e667 with expected crc 2023-10-09T09:15:40.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[78107]: cluster 2023-10-09T09:15:39.465958+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e667 with expected crc 2023-10-09T09:15:40.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[78107]: audit 2023-10-09T09:15:40.351644+0000 mon.a (mon.0) 822 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:40.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:40 smithi163 bash[78107]: audit 2023-10-09T09:15:40.352857+0000 mon.a (mon.0) 823 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:40 smithi181 bash[59345]: cluster 2023-10-09T09:15:39.350479+0000 mgr.y (mgr.34107) 269 : cluster 0 pgmap v552: 89 pgs: 3 peering, 86 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 855 KiB/s wr, 7 op/s 2023-10-09T09:15:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:40 smithi181 bash[59345]: cluster 2023-10-09T09:15:39.434851+0000 mon.a (mon.0) 819 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/357 objects degraded (1.961%), 3 pgs degraded) 2023-10-09T09:15:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:40 smithi181 bash[59345]: cluster 2023-10-09T09:15:39.434934+0000 mon.a (mon.0) 820 : cluster 1 Cluster is now healthy 2023-10-09T09:15:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:40 smithi181 bash[59345]: cluster 2023-10-09T09:15:39.445528+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e667 with expected crc 2023-10-09T09:15:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:40 smithi181 bash[59345]: cluster 2023-10-09T09:15:39.450551+0000 mon.a (mon.0) 821 : cluster 0 osdmap e667: 8 total, 8 up, 8 in 2023-10-09T09:15:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:40 smithi181 bash[59345]: cluster 2023-10-09T09:15:39.453860+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e667 with expected crc 2023-10-09T09:15:40.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:40 smithi181 bash[59345]: cluster 2023-10-09T09:15:39.465958+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e667 with expected crc 2023-10-09T09:15:40.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:40 smithi181 bash[59345]: audit 2023-10-09T09:15:40.351644+0000 mon.a (mon.0) 822 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:40.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:40 smithi181 bash[59345]: audit 2023-10-09T09:15:40.352857+0000 mon.a (mon.0) 823 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:41.467 INFO:tasks.rados.rados.0.smithi163.stdout:2351: done (1 left) 2023-10-09T09:15:41.467 INFO:tasks.rados.rados.0.smithi163.stdout:2352: read oid 10 snap 305 2023-10-09T09:15:41.467 INFO:tasks.rados.rados.0.smithi163.stdout:2352: expect (ObjNum 863 snap 293 seq_num 863) 2023-10-09T09:15:41.467 INFO:tasks.rados.rados.0.smithi163.stdout:2353: snap_remove snap 291 2023-10-09T09:15:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:41 smithi163 bash[78107]: cluster 2023-10-09T09:15:40.462863+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e668 with expected crc 2023-10-09T09:15:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:41 smithi163 bash[78107]: cluster 2023-10-09T09:15:40.466345+0000 mon.a (mon.0) 824 : cluster 0 osdmap e668: 8 total, 8 up, 8 in 2023-10-09T09:15:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:41 smithi163 bash[75425]: cluster 2023-10-09T09:15:40.462863+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e668 with expected crc 2023-10-09T09:15:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:41 smithi163 bash[75425]: cluster 2023-10-09T09:15:40.466345+0000 mon.a (mon.0) 824 : cluster 0 osdmap e668: 8 total, 8 up, 8 in 2023-10-09T09:15:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:41 smithi181 bash[59345]: cluster 2023-10-09T09:15:40.462863+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e668 with expected crc 2023-10-09T09:15:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:41 smithi181 bash[59345]: cluster 2023-10-09T09:15:40.466345+0000 mon.a (mon.0) 824 : cluster 0 osdmap e668: 8 total, 8 up, 8 in 2023-10-09T09:15:41.823 INFO:teuthology.orchestra.run.smithi163.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:15:42.465 INFO:tasks.rados.rados.0.smithi163.stdout:2353: done (2 left) 2023-10-09T09:15:42.466 INFO:tasks.rados.rados.0.smithi163.stdout:2354: snap_remove snap 298 2023-10-09T09:15:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:42 smithi181 bash[59345]: cluster 2023-10-09T09:15:40.462863+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e668 with expected crc 2023-10-09T09:15:42.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:42 smithi181 bash[59345]: cluster 2023-10-09T09:15:41.351660+0000 mgr.y (mgr.34107) 270 : cluster 0 pgmap v555: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 19 op/s; 406 KiB/s, 0 objects/s recovering 2023-10-09T09:15:42.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:42 smithi181 bash[59345]: cluster 2023-10-09T09:15:41.460312+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e669 with expected crc 2023-10-09T09:15:42.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:42 smithi181 bash[59345]: cluster 2023-10-09T09:15:41.463482+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e669 with expected crc 2023-10-09T09:15:42.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:42 smithi181 bash[59345]: cluster 2023-10-09T09:15:41.464181+0000 mon.a (mon.0) 825 : cluster 0 osdmap e669: 8 total, 8 up, 8 in 2023-10-09T09:15:42.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:42 smithi181 bash[59345]: audit 2023-10-09T09:15:41.823272+0000 mon.a (mon.0) 826 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:42.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:42 smithi181 bash[59345]: cluster 2023-10-09T09:15:42.465882+0000 mon.a (mon.0) 827 : cluster 0 osdmap e670: 8 total, 8 up, 8 in 2023-10-09T09:15:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[78107]: cluster 2023-10-09T09:15:40.462863+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e668 with expected crc 2023-10-09T09:15:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[78107]: cluster 2023-10-09T09:15:41.351660+0000 mgr.y (mgr.34107) 270 : cluster 0 pgmap v555: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 19 op/s; 406 KiB/s, 0 objects/s recovering 2023-10-09T09:15:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[78107]: cluster 2023-10-09T09:15:41.460312+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e669 with expected crc 2023-10-09T09:15:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[78107]: cluster 2023-10-09T09:15:41.463482+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e669 with expected crc 2023-10-09T09:15:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[78107]: cluster 2023-10-09T09:15:41.464181+0000 mon.a (mon.0) 825 : cluster 0 osdmap e669: 8 total, 8 up, 8 in 2023-10-09T09:15:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[78107]: audit 2023-10-09T09:15:41.823272+0000 mon.a (mon.0) 826 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[78107]: cluster 2023-10-09T09:15:42.465882+0000 mon.a (mon.0) 827 : cluster 0 osdmap e670: 8 total, 8 up, 8 in 2023-10-09T09:15:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[75425]: cluster 2023-10-09T09:15:40.462863+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e668 with expected crc 2023-10-09T09:15:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[75425]: cluster 2023-10-09T09:15:41.351660+0000 mgr.y (mgr.34107) 270 : cluster 0 pgmap v555: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 19 op/s; 406 KiB/s, 0 objects/s recovering 2023-10-09T09:15:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[75425]: cluster 2023-10-09T09:15:41.460312+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e669 with expected crc 2023-10-09T09:15:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[75425]: cluster 2023-10-09T09:15:41.463482+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e669 with expected crc 2023-10-09T09:15:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[75425]: cluster 2023-10-09T09:15:41.464181+0000 mon.a (mon.0) 825 : cluster 0 osdmap e669: 8 total, 8 up, 8 in 2023-10-09T09:15:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[75425]: audit 2023-10-09T09:15:41.823272+0000 mon.a (mon.0) 826 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:42 smithi163 bash[75425]: cluster 2023-10-09T09:15:42.465882+0000 mon.a (mon.0) 827 : cluster 0 osdmap e670: 8 total, 8 up, 8 in 2023-10-09T09:15:43.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:15:43.472 INFO:tasks.rados.rados.0.smithi163.stdout:2354: done (2 left) 2023-10-09T09:15:43.472 INFO:tasks.rados.rados.0.smithi163.stdout:2355: snap_create 2023-10-09T09:15:43.473 INFO:tasks.rados.rados.0.smithi163.stdout:2350: expect (ObjNum 789 snap 271 seq_num 789) 2023-10-09T09:15:43.648 INFO:tasks.rados.rados.0.smithi163.stdout:2352: expect (ObjNum 863 snap 293 seq_num 863) 2023-10-09T09:15:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:15:43] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.33.4" 2023-10-09T09:15:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: audit 2023-10-09T09:15:41.816833+0000 mgr.y (mgr.34107) 271 : audit 0 from='client.44447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:15:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: cephadm 2023-10-09T09:15:41.817392+0000 mgr.y (mgr.34107) 272 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: cluster 2023-10-09T09:15:42.462734+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e670 with expected crc 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: audit 2023-10-09T09:15:43.078554+0000 mon.a (mon.0) 828 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: audit 2023-10-09T09:15:43.089439+0000 mon.a (mon.0) 829 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: audit 2023-10-09T09:15:43.091113+0000 mon.a (mon.0) 830 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: audit 2023-10-09T09:15:43.092315+0000 mon.a (mon.0) 831 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: audit 2023-10-09T09:15:43.101665+0000 mon.a (mon.0) 832 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: audit 2023-10-09T09:15:43.184628+0000 mon.a (mon.0) 833 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: audit 2023-10-09T09:15:43.187625+0000 mon.a (mon.0) 834 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: audit 2023-10-09T09:15:43.188874+0000 mon.a (mon.0) 835 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: audit 2023-10-09T09:15:43.195420+0000 mon.a (mon.0) 836 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[75425]: cluster 2023-10-09T09:15:43.472465+0000 mon.a (mon.0) 837 : cluster 0 osdmap e671: 8 total, 8 up, 8 in 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: audit 2023-10-09T09:15:41.816833+0000 mgr.y (mgr.34107) 271 : audit 0 from='client.44447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: cephadm 2023-10-09T09:15:41.817392+0000 mgr.y (mgr.34107) 272 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: cluster 2023-10-09T09:15:42.462734+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e670 with expected crc 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: audit 2023-10-09T09:15:43.078554+0000 mon.a (mon.0) 828 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: audit 2023-10-09T09:15:43.089439+0000 mon.a (mon.0) 829 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: audit 2023-10-09T09:15:43.091113+0000 mon.a (mon.0) 830 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: audit 2023-10-09T09:15:43.092315+0000 mon.a (mon.0) 831 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: audit 2023-10-09T09:15:43.101665+0000 mon.a (mon.0) 832 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: audit 2023-10-09T09:15:43.184628+0000 mon.a (mon.0) 833 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: audit 2023-10-09T09:15:43.187625+0000 mon.a (mon.0) 834 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:43.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: audit 2023-10-09T09:15:43.188874+0000 mon.a (mon.0) 835 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:43.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: audit 2023-10-09T09:15:43.195420+0000 mon.a (mon.0) 836 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:43 smithi163 bash[78107]: cluster 2023-10-09T09:15:43.472465+0000 mon.a (mon.0) 837 : cluster 0 osdmap e671: 8 total, 8 up, 8 in 2023-10-09T09:15:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: audit 2023-10-09T09:15:41.816833+0000 mgr.y (mgr.34107) 271 : audit 0 from='client.44447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:15:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: cephadm 2023-10-09T09:15:41.817392+0000 mgr.y (mgr.34107) 272 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:15:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: cluster 2023-10-09T09:15:42.462734+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e670 with expected crc 2023-10-09T09:15:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: audit 2023-10-09T09:15:43.078554+0000 mon.a (mon.0) 828 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: audit 2023-10-09T09:15:43.089439+0000 mon.a (mon.0) 829 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: audit 2023-10-09T09:15:43.091113+0000 mon.a (mon.0) 830 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: audit 2023-10-09T09:15:43.092315+0000 mon.a (mon.0) 831 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:43.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: audit 2023-10-09T09:15:43.101665+0000 mon.a (mon.0) 832 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: audit 2023-10-09T09:15:43.184628+0000 mon.a (mon.0) 833 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:15:43.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: audit 2023-10-09T09:15:43.187625+0000 mon.a (mon.0) 834 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:15:43.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: audit 2023-10-09T09:15:43.188874+0000 mon.a (mon.0) 835 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:15:43.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: audit 2023-10-09T09:15:43.195420+0000 mon.a (mon.0) 836 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:43.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:43 smithi181 bash[59345]: cluster 2023-10-09T09:15:43.472465+0000 mon.a (mon.0) 837 : cluster 0 osdmap e671: 8 total, 8 up, 8 in 2023-10-09T09:15:44.471 INFO:tasks.rados.rados.0.smithi163.stdout:2350: done (2 left) 2023-10-09T09:15:44.471 INFO:tasks.rados.rados.0.smithi163.stdout:2352: done (1 left) 2023-10-09T09:15:44.471 INFO:tasks.rados.rados.0.smithi163.stdout:2355: done (0 left) 2023-10-09T09:15:44.471 INFO:tasks.rados.rados.0.smithi163.stdout:2356: write oid 41 current snap is 307 2023-10-09T09:15:44.472 INFO:tasks.rados.rados.0.smithi163.stdout:2356: seq_num 879 ranges {773353=724466,2170378=602975} 2023-10-09T09:15:44.479 INFO:tasks.rados.rados.0.smithi163.stdout:2356: writing smithi16367185-41 from 773353 to 1497819 tid 1 2023-10-09T09:15:44.487 INFO:tasks.rados.rados.0.smithi163.stdout:2356: writing smithi16367185-41 from 2170378 to 2773353 tid 2 2023-10-09T09:15:44.487 INFO:tasks.rados.rados.0.smithi163.stdout:2357: write oid 32 current snap is 307 2023-10-09T09:15:44.487 INFO:tasks.rados.rados.0.smithi163.stdout:2356: finishing write tid 1 to smithi16367185-41 2023-10-09T09:15:44.487 INFO:tasks.rados.rados.0.smithi163.stdout:2357: seq_num 880 ranges {790160=799715,2112507=509299} 2023-10-09T09:15:44.496 INFO:tasks.rados.rados.0.smithi163.stdout:2357: writing smithi16367185-32 from 790160 to 1589875 tid 1 2023-10-09T09:15:44.503 INFO:tasks.rados.rados.0.smithi163.stdout:2357: writing smithi16367185-32 from 2112507 to 2621806 tid 2 2023-10-09T09:15:44.503 INFO:tasks.rados.rados.0.smithi163.stdout:2358: write oid 15 current snap is 307 2023-10-09T09:15:44.503 INFO:tasks.rados.rados.0.smithi163.stdout:2356: finishing write tid 2 to smithi16367185-41 2023-10-09T09:15:44.503 INFO:tasks.rados.rados.0.smithi163.stdout:2358: seq_num 881 ranges {406967=474964,1370989=769310,2677186=129781} 2023-10-09T09:15:44.508 INFO:tasks.rados.rados.0.smithi163.stdout:2358: writing smithi16367185-15 from 406967 to 881931 tid 1 2023-10-09T09:15:44.515 INFO:tasks.rados.rados.0.smithi163.stdout:2358: writing smithi16367185-15 from 1370989 to 2140299 tid 2 2023-10-09T09:15:44.520 INFO:tasks.rados.rados.0.smithi163.stdout:2358: writing smithi16367185-15 from 2677186 to 2806967 tid 3 2023-10-09T09:15:44.520 INFO:tasks.rados.rados.0.smithi163.stdout:2359: read oid 38 snap -1 2023-10-09T09:15:44.520 INFO:tasks.rados.rados.0.smithi163.stdout:2359: expect (ObjNum 866 snap 299 seq_num 866) 2023-10-09T09:15:44.520 INFO:tasks.rados.rados.0.smithi163.stdout:2360: snap_create 2023-10-09T09:15:44.520 INFO:tasks.rados.rados.0.smithi163.stdout:2356: finishing write tid 3 to smithi16367185-41 2023-10-09T09:15:44.520 INFO:tasks.rados.rados.0.smithi163.stdout:2356: finishing write tid 4 to smithi16367185-41 2023-10-09T09:15:44.520 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 166 (ObjNum 879 snap 307 seq_num 879) dirty exists 2023-10-09T09:15:44.520 INFO:tasks.rados.rados.0.smithi163.stdout:2356: left oid 41 (ObjNum 879 snap 307 seq_num 879) 2023-10-09T09:15:44.520 INFO:tasks.rados.rados.0.smithi163.stdout:2357: finishing write tid 1 to smithi16367185-32 2023-10-09T09:15:44.521 INFO:tasks.rados.rados.0.smithi163.stdout:2357: finishing write tid 2 to smithi16367185-32 2023-10-09T09:15:44.521 INFO:tasks.rados.rados.0.smithi163.stdout:2357: finishing write tid 3 to smithi16367185-32 2023-10-09T09:15:44.521 INFO:tasks.rados.rados.0.smithi163.stdout:2357: finishing write tid 4 to smithi16367185-32 2023-10-09T09:15:44.521 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1760 (ObjNum 880 snap 307 seq_num 880) dirty exists 2023-10-09T09:15:44.521 INFO:tasks.rados.rados.0.smithi163.stdout:2357: left oid 32 (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:15:44.521 INFO:tasks.rados.rados.0.smithi163.stdout:2358: finishing write tid 1 to smithi16367185-15 2023-10-09T09:15:44.523 INFO:tasks.rados.rados.0.smithi163.stdout:2358: finishing write tid 2 to smithi16367185-15 2023-10-09T09:15:44.524 INFO:tasks.rados.rados.0.smithi163.stdout:2358: finishing write tid 3 to smithi16367185-15 2023-10-09T09:15:44.524 INFO:tasks.rados.rados.0.smithi163.stdout:2358: finishing write tid 4 to smithi16367185-15 2023-10-09T09:15:44.524 INFO:tasks.rados.rados.0.smithi163.stdout:2358: finishing write tid 5 to smithi16367185-15 2023-10-09T09:15:44.525 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 1153 (ObjNum 881 snap 307 seq_num 881) dirty exists 2023-10-09T09:15:44.525 INFO:tasks.rados.rados.0.smithi163.stdout:2358: left oid 15 (ObjNum 881 snap 307 seq_num 881) 2023-10-09T09:15:44.537 INFO:tasks.rados.rados.0.smithi163.stdout:2359: expect (ObjNum 866 snap 299 seq_num 866) 2023-10-09T09:15:44.552 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:44 smithi163 bash[75425]: cluster 2023-10-09T09:15:43.352213+0000 mgr.y (mgr.34107) 273 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 18 op/s; 406 KiB/s, 0 objects/s recovering 2023-10-09T09:15:44.552 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:44 smithi163 bash[75425]: cluster 2023-10-09T09:15:43.467194+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e671 with expected crc 2023-10-09T09:15:44.553 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:44 smithi163 bash[75425]: cluster 2023-10-09T09:15:44.470604+0000 mon.a (mon.0) 838 : cluster 0 osdmap e672: 8 total, 8 up, 8 in 2023-10-09T09:15:44.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:44 smithi163 bash[78107]: cluster 2023-10-09T09:15:43.352213+0000 mgr.y (mgr.34107) 273 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 18 op/s; 406 KiB/s, 0 objects/s recovering 2023-10-09T09:15:44.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:44 smithi163 bash[78107]: cluster 2023-10-09T09:15:43.467194+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e671 with expected crc 2023-10-09T09:15:44.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:44 smithi163 bash[78107]: cluster 2023-10-09T09:15:44.470604+0000 mon.a (mon.0) 838 : cluster 0 osdmap e672: 8 total, 8 up, 8 in 2023-10-09T09:15:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:44 smithi181 bash[59345]: cluster 2023-10-09T09:15:43.352213+0000 mgr.y (mgr.34107) 273 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 18 op/s; 406 KiB/s, 0 objects/s recovering 2023-10-09T09:15:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:44 smithi181 bash[59345]: cluster 2023-10-09T09:15:43.467194+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e671 with expected crc 2023-10-09T09:15:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:44 smithi181 bash[59345]: cluster 2023-10-09T09:15:44.470604+0000 mon.a (mon.0) 838 : cluster 0 osdmap e672: 8 total, 8 up, 8 in 2023-10-09T09:15:45.145 INFO:tasks.rados.rados.0.smithi163.stdout:2356: done (4 left) 2023-10-09T09:15:45.146 INFO:tasks.rados.rados.0.smithi163.stdout:2357: done (3 left) 2023-10-09T09:15:45.146 INFO:tasks.rados.rados.0.smithi163.stdout:2358: done (2 left) 2023-10-09T09:15:45.146 INFO:tasks.rados.rados.0.smithi163.stdout:2359: done (1 left) 2023-10-09T09:15:45.146 INFO:tasks.rados.rados.0.smithi163.stdout:2360: done (0 left) 2023-10-09T09:15:45.146 INFO:tasks.rados.rados.0.smithi163.stdout:2361: write oid 34 current snap is 308 2023-10-09T09:15:45.146 INFO:tasks.rados.rados.0.smithi163.stdout:2361: seq_num 882 ranges {423774=666566,1545824=512968,2704609=119165} 2023-10-09T09:15:45.151 INFO:tasks.rados.rados.0.smithi163.stdout:2361: writing smithi16367185-34 from 423774 to 1090340 tid 1 2023-10-09T09:15:45.157 INFO:tasks.rados.rados.0.smithi163.stdout:2361: writing smithi16367185-34 from 1545824 to 2058792 tid 2 2023-10-09T09:15:45.162 INFO:tasks.rados.rados.0.smithi163.stdout:2361: writing smithi16367185-34 from 2704609 to 2823774 tid 3 2023-10-09T09:15:45.162 INFO:tasks.rados.rados.0.smithi163.stdout:2362: write oid 13 current snap is 308 2023-10-09T09:15:45.162 INFO:tasks.rados.rados.0.smithi163.stdout:2361: finishing write tid 1 to smithi16367185-34 2023-10-09T09:15:45.162 INFO:tasks.rados.rados.0.smithi163.stdout:2361: finishing write tid 2 to smithi16367185-34 2023-10-09T09:15:45.162 INFO:tasks.rados.rados.0.smithi163.stdout:2362: seq_num 883 ranges {440581=741815,1604306=656626,2732032=108549} 2023-10-09T09:15:45.168 INFO:tasks.rados.rados.0.smithi163.stdout:2362: writing smithi16367185-13 from 440581 to 1182396 tid 1 2023-10-09T09:15:45.175 INFO:tasks.rados.rados.0.smithi163.stdout:2362: writing smithi16367185-13 from 1604306 to 2260932 tid 2 2023-10-09T09:15:45.179 INFO:tasks.rados.rados.0.smithi163.stdout:2362: writing smithi16367185-13 from 2732032 to 2840581 tid 3 2023-10-09T09:15:45.179 INFO:tasks.rados.rados.0.smithi163.stdout:2363: write oid 36 current snap is 308 2023-10-09T09:15:45.179 INFO:tasks.rados.rados.0.smithi163.stdout:2361: finishing write tid 3 to smithi16367185-34 2023-10-09T09:15:45.179 INFO:tasks.rados.rados.0.smithi163.stdout:2363: seq_num 884 ranges {457388=417064,1546435=516637,2643102=214286} 2023-10-09T09:15:45.183 INFO:tasks.rados.rados.0.smithi163.stdout:2363: writing smithi16367185-36 from 457388 to 874452 tid 1 2023-10-09T09:15:45.191 INFO:tasks.rados.rados.0.smithi163.stdout:2363: writing smithi16367185-36 from 1546435 to 2063072 tid 2 2023-10-09T09:15:45.196 INFO:tasks.rados.rados.0.smithi163.stdout:2363: writing smithi16367185-36 from 2643102 to 2857388 tid 3 2023-10-09T09:15:45.196 INFO:tasks.rados.rados.0.smithi163.stdout:2364: read oid 15 snap -1 2023-10-09T09:15:45.196 INFO:tasks.rados.rados.0.smithi163.stdout:2364: expect (ObjNum 881 snap 307 seq_num 881) 2023-10-09T09:15:45.196 INFO:tasks.rados.rados.0.smithi163.stdout:2365: write oid 17 current snap is 308 2023-10-09T09:15:45.196 INFO:tasks.rados.rados.0.smithi163.stdout:2361: finishing write tid 4 to smithi16367185-34 2023-10-09T09:15:45.196 INFO:tasks.rados.rados.0.smithi163.stdout:2361: finishing write tid 5 to smithi16367185-34 2023-10-09T09:15:45.196 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1056 (ObjNum 882 snap 308 seq_num 882) dirty exists 2023-10-09T09:15:45.196 INFO:tasks.rados.rados.0.smithi163.stdout:2361: left oid 34 (ObjNum 882 snap 308 seq_num 882) 2023-10-09T09:15:45.197 INFO:tasks.rados.rados.0.smithi163.stdout:2362: finishing write tid 1 to smithi16367185-13 2023-10-09T09:15:45.197 INFO:tasks.rados.rados.0.smithi163.stdout:2365: seq_num 885 ranges {474195=492313,1604917=660295,2670525=203670} 2023-10-09T09:15:45.201 INFO:tasks.rados.rados.0.smithi163.stdout:2365: writing smithi16367185-17 from 474195 to 966508 tid 1 2023-10-09T09:15:45.209 INFO:tasks.rados.rados.0.smithi163.stdout:2365: writing smithi16367185-17 from 1604917 to 2265212 tid 2 2023-10-09T09:15:45.213 INFO:tasks.rados.rados.0.smithi163.stdout:2365: writing smithi16367185-17 from 2670525 to 2874195 tid 3 2023-10-09T09:15:45.214 INFO:tasks.rados.rados.0.smithi163.stdout:2361: done (4 left) 2023-10-09T09:15:45.214 INFO:tasks.rados.rados.0.smithi163.stdout:2366: write oid 8 current snap is 308 2023-10-09T09:15:45.214 INFO:tasks.rados.rados.0.smithi163.stdout:2366: seq_num 886 ranges {491002=567562,1663399=520306,2814301=76701} 2023-10-09T09:15:45.219 INFO:tasks.rados.rados.0.smithi163.stdout:2366: writing smithi16367185-8 from 491002 to 1058564 tid 1 2023-10-09T09:15:45.226 INFO:tasks.rados.rados.0.smithi163.stdout:2366: writing smithi16367185-8 from 1663399 to 2183705 tid 2 2023-10-09T09:15:45.230 INFO:tasks.rados.rados.0.smithi163.stdout:2366: writing smithi16367185-8 from 2814301 to 2891002 tid 3 2023-10-09T09:15:45.230 INFO:tasks.rados.rados.0.smithi163.stdout:2367: snap_create 2023-10-09T09:15:45.230 INFO:tasks.rados.rados.0.smithi163.stdout:2362: finishing write tid 2 to smithi16367185-13 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:2362: finishing write tid 3 to smithi16367185-13 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:2362: finishing write tid 4 to smithi16367185-13 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:2362: finishing write tid 5 to smithi16367185-13 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 314 (ObjNum 883 snap 308 seq_num 883) dirty exists 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:2362: left oid 13 (ObjNum 883 snap 308 seq_num 883) 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:2363: finishing write tid 1 to smithi16367185-36 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:2363: finishing write tid 2 to smithi16367185-36 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:2363: finishing write tid 3 to smithi16367185-36 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:2363: finishing write tid 4 to smithi16367185-36 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:2363: finishing write tid 5 to smithi16367185-36 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 593 (ObjNum 884 snap 308 seq_num 884) dirty exists 2023-10-09T09:15:45.231 INFO:tasks.rados.rados.0.smithi163.stdout:2363: left oid 36 (ObjNum 884 snap 308 seq_num 884) 2023-10-09T09:15:45.232 INFO:tasks.rados.rados.0.smithi163.stdout:2364: expect (ObjNum 881 snap 307 seq_num 881) 2023-10-09T09:15:45.323 INFO:tasks.rados.rados.0.smithi163.stdout:2365: finishing write tid 1 to smithi16367185-17 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:2365: finishing write tid 2 to smithi16367185-17 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:2365: finishing write tid 3 to smithi16367185-17 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:2365: finishing write tid 4 to smithi16367185-17 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:2365: finishing write tid 5 to smithi16367185-17 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1158 (ObjNum 885 snap 308 seq_num 885) dirty exists 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:2365: left oid 17 (ObjNum 885 snap 308 seq_num 885) 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:2366: finishing write tid 1 to smithi16367185-8 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:2366: finishing write tid 2 to smithi16367185-8 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:2366: finishing write tid 3 to smithi16367185-8 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:2366: finishing write tid 4 to smithi16367185-8 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:2366: finishing write tid 5 to smithi16367185-8 2023-10-09T09:15:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1163 (ObjNum 886 snap 308 seq_num 886) dirty exists 2023-10-09T09:15:45.325 INFO:tasks.rados.rados.0.smithi163.stdout:2366: left oid 8 (ObjNum 886 snap 308 seq_num 886) 2023-10-09T09:15:46.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[75425]: cluster 2023-10-09T09:15:44.467183+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e672 with expected crc 2023-10-09T09:15:46.141 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[75425]: cluster 2023-10-09T09:15:44.481596+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e672 with expected crc 2023-10-09T09:15:46.141 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[75425]: cluster 2023-10-09T09:15:44.520635+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e672 with expected crc 2023-10-09T09:15:46.141 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[75425]: cluster 2023-10-09T09:15:45.141848+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e673 with expected crc 2023-10-09T09:15:46.141 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[75425]: cluster 2023-10-09T09:15:45.141953+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e673 with expected crc 2023-10-09T09:15:46.141 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[75425]: cluster 2023-10-09T09:15:45.145158+0000 mon.a (mon.0) 839 : cluster 0 osdmap e673: 8 total, 8 up, 8 in 2023-10-09T09:15:46.141 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[75425]: cluster 2023-10-09T09:15:45.196577+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e673 with expected crc 2023-10-09T09:15:46.141 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[75425]: cluster 2023-10-09T09:15:45.352948+0000 mgr.y (mgr.34107) 274 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 333 KiB/s wr, 5 op/s 2023-10-09T09:15:46.156 INFO:tasks.rados.rados.0.smithi163.stdout:2362: done (5 left) 2023-10-09T09:15:46.157 INFO:tasks.rados.rados.0.smithi163.stdout:2363: done (4 left) 2023-10-09T09:15:46.157 INFO:tasks.rados.rados.0.smithi163.stdout:2364: done (3 left) 2023-10-09T09:15:46.157 INFO:tasks.rados.rados.0.smithi163.stdout:2365: done (2 left) 2023-10-09T09:15:46.157 INFO:tasks.rados.rados.0.smithi163.stdout:2366: done (1 left) 2023-10-09T09:15:46.157 INFO:tasks.rados.rados.0.smithi163.stdout:2367: done (0 left) 2023-10-09T09:15:46.157 INFO:tasks.rados.rados.0.smithi163.stdout:2368: read oid 25 snap 301 2023-10-09T09:15:46.157 INFO:tasks.rados.rados.0.smithi163.stdout:2368: expect (ObjNum 823 snap 282 seq_num 823) 2023-10-09T09:15:46.157 INFO:tasks.rados.rados.0.smithi163.stdout:2369: write oid 39 current snap is 309 2023-10-09T09:15:46.157 INFO:tasks.rados.rados.0.smithi163.stdout:2369: seq_num 887 ranges {507809=642811,1721881=663964} 2023-10-09T09:15:46.165 INFO:tasks.rados.rados.0.smithi163.stdout:2369: writing smithi16367185-39 from 507809 to 1150620 tid 1 2023-10-09T09:15:46.174 INFO:tasks.rados.rados.0.smithi163.stdout:2369: writing smithi16367185-39 from 1721881 to 2385845 tid 2 2023-10-09T09:15:46.174 INFO:tasks.rados.rados.0.smithi163.stdout:2370: read oid 42 snap -1 2023-10-09T09:15:46.174 INFO:tasks.rados.rados.0.smithi163.stdout:2370: expect deleted 2023-10-09T09:15:46.174 INFO:tasks.rados.rados.0.smithi163.stdout:2371: write oid 33 current snap is 309 2023-10-09T09:15:46.175 INFO:tasks.rados.rados.0.smithi163.stdout:2371: seq_num 888 ranges {524616=718060,1664010=523975,2752794=171822} 2023-10-09T09:15:46.181 INFO:tasks.rados.rados.0.smithi163.stdout:2371: writing smithi16367185-33 from 524616 to 1242676 tid 1 2023-10-09T09:15:46.206 INFO:tasks.rados.rados.0.smithi163.stdout:2371: writing smithi16367185-33 from 1664010 to 2187985 tid 2 2023-10-09T09:15:46.206 INFO:tasks.rados.rados.0.smithi163.stdout:2371: writing smithi16367185-33 from 2752794 to 2924616 tid 3 2023-10-09T09:15:46.206 INFO:tasks.rados.rados.0.smithi163.stdout:2372: write oid 23 current snap is 309 2023-10-09T09:15:46.207 INFO:tasks.rados.rados.0.smithi163.stdout:2372: seq_num 889 ranges {541423=793309,2122492=667633} 2023-10-09T09:15:46.207 INFO:tasks.rados.rados.0.smithi163.stdout:2372: writing smithi16367185-23 from 541423 to 1334732 tid 1 2023-10-09T09:15:46.208 INFO:tasks.rados.rados.0.smithi163.stdout:2372: writing smithi16367185-23 from 2122492 to 2790125 tid 2 2023-10-09T09:15:46.208 INFO:tasks.rados.rados.0.smithi163.stdout:2373: read oid 8 snap -1 2023-10-09T09:15:46.208 INFO:tasks.rados.rados.0.smithi163.stdout:2373: expect (ObjNum 886 snap 308 seq_num 886) 2023-10-09T09:15:46.208 INFO:tasks.rados.rados.0.smithi163.stdout:2374: delete oid 21 current snap is 309 2023-10-09T09:15:46.208 INFO:tasks.rados.rados.0.smithi163.stdout:2368: expect (ObjNum 823 snap 282 seq_num 823) 2023-10-09T09:15:46.286 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (8m) - 14m 13.7M - 0.23.0 ba2b418f427c 82a8abc84166 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (7m) - 13m 41.4M - 8.3.5 dad864ee21e9 9d80a71ddd1d 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.vnodzj smithi163 running (12m) - 12m 19.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 892077b22450 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.ajqgsc smithi181 running (12m) - 12m 14.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 01d89554089b 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443,9283,8765 running (6m) - 20m 428M - 18.2.0-598-gb4fa663f 3060897fdf97 ca7a91dd3dab 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (7m) - 22m 485M - 18.2.0-598-gb4fa663f 3060897fdf97 fc427389eb0c 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (6m) - 22m 87.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 097aa5987723 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (3m) - 20m 42.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 76abe6c0dfe2 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (4m) - 20m 55.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 adc323592e84 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (8m) - 14m 9511k - 1.3.1 1dbe0e931976 8a6fec29baf3 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (8m) - 14m 8928k - 1.3.1 1dbe0e931976 3f766d093db8 2023-10-09T09:15:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (100s) - 20m 93.4M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 242f1e42dfa2 2023-10-09T09:15:46.288 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (73s) - 19m 70.4M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 26f120975fe0 2023-10-09T09:15:46.288 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (47s) - 18m 75.3M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 f218a8a2ac28 2023-10-09T09:15:46.288 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (19s) - 18m - 2524M 18.2.0-598-gb4fa663f 3060897fdf97 13ae6c567443 2023-10-09T09:15:46.288 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (17m) - 17m 116M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5dd6bc4b279 2023-10-09T09:15:46.288 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (16m) - 16m 127M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9f3e4fbba907 2023-10-09T09:15:46.288 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (15m) - 16m 81.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 30f1ac2f4524 2023-10-09T09:15:46.289 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (15m) - 15m 102M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 54105d827fdd 2023-10-09T09:15:46.289 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (7m) - 14m 58.2M - 2.33.4 514e6a882f6e cc81501e414f 2023-10-09T09:15:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[78107]: cluster 2023-10-09T09:15:44.467183+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e672 with expected crc 2023-10-09T09:15:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[78107]: cluster 2023-10-09T09:15:44.481596+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e672 with expected crc 2023-10-09T09:15:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[78107]: cluster 2023-10-09T09:15:44.520635+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e672 with expected crc 2023-10-09T09:15:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[78107]: cluster 2023-10-09T09:15:45.141848+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e673 with expected crc 2023-10-09T09:15:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[78107]: cluster 2023-10-09T09:15:45.141953+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e673 with expected crc 2023-10-09T09:15:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[78107]: cluster 2023-10-09T09:15:45.145158+0000 mon.a (mon.0) 839 : cluster 0 osdmap e673: 8 total, 8 up, 8 in 2023-10-09T09:15:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[78107]: cluster 2023-10-09T09:15:45.196577+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e673 with expected crc 2023-10-09T09:15:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:46 smithi163 bash[78107]: cluster 2023-10-09T09:15:45.352948+0000 mgr.y (mgr.34107) 274 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 333 KiB/s wr, 5 op/s 2023-10-09T09:15:46.478 INFO:tasks.rados.rados.0.smithi163.stdout:2369: finishing write tid 1 to smithi16367185-39 2023-10-09T09:15:46.478 INFO:tasks.rados.rados.0.smithi163.stdout:2369: finishing write tid 2 to smithi16367185-39 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:2369: finishing write tid 3 to smithi16367185-39 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:2369: finishing write tid 4 to smithi16367185-39 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1060 (ObjNum 887 snap 309 seq_num 887) dirty exists 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:2369: left oid 39 (ObjNum 887 snap 309 seq_num 887) 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:2371: finishing write tid 1 to smithi16367185-33 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:2371: finishing write tid 2 to smithi16367185-33 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:2371: finishing write tid 3 to smithi16367185-33 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:2371: finishing write tid 4 to smithi16367185-33 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:2371: finishing write tid 5 to smithi16367185-33 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1139 (ObjNum 888 snap 309 seq_num 888) dirty exists 2023-10-09T09:15:46.479 INFO:tasks.rados.rados.0.smithi163.stdout:2371: left oid 33 (ObjNum 888 snap 309 seq_num 888) 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:2372: finishing write tid 1 to smithi16367185-23 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:2368: done (6 left) 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:2369: done (5 left) 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:2370: done (4 left) 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:2371: done (3 left) 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:2374: done (2 left) 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:2375: rollback oid 39 current snap is 309 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 290 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:2372: finishing write tid 2 to smithi16367185-23 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:2372: finishing write tid 3 to smithi16367185-23 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:2372: finishing write tid 4 to smithi16367185-23 2023-10-09T09:15:46.480 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1064 (ObjNum 889 snap 309 seq_num 889) dirty exists 2023-10-09T09:15:46.481 INFO:tasks.rados.rados.0.smithi163.stdout:2372: left oid 23 (ObjNum 889 snap 309 seq_num 889) 2023-10-09T09:15:46.481 INFO:tasks.rados.rados.0.smithi163.stdout:2373: expect (ObjNum 886 snap 308 seq_num 886) 2023-10-09T09:15:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:46 smithi181 bash[59345]: cluster 2023-10-09T09:15:44.467183+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e672 with expected crc 2023-10-09T09:15:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:46 smithi181 bash[59345]: cluster 2023-10-09T09:15:44.481596+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e672 with expected crc 2023-10-09T09:15:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:46 smithi181 bash[59345]: cluster 2023-10-09T09:15:44.520635+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e672 with expected crc 2023-10-09T09:15:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:46 smithi181 bash[59345]: cluster 2023-10-09T09:15:45.141848+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e673 with expected crc 2023-10-09T09:15:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:46 smithi181 bash[59345]: cluster 2023-10-09T09:15:45.141953+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e673 with expected crc 2023-10-09T09:15:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:46 smithi181 bash[59345]: cluster 2023-10-09T09:15:45.145158+0000 mon.a (mon.0) 839 : cluster 0 osdmap e673: 8 total, 8 up, 8 in 2023-10-09T09:15:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:46 smithi181 bash[59345]: cluster 2023-10-09T09:15:45.196577+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e673 with expected crc 2023-10-09T09:15:46.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:46 smithi181 bash[59345]: cluster 2023-10-09T09:15:45.352948+0000 mgr.y (mgr.34107) 274 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 333 KiB/s wr, 5 op/s 2023-10-09T09:15:46.621 INFO:tasks.rados.rados.0.smithi163.stdout:2375: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:15:46.621 INFO:tasks.rados.rados.0.smithi163.stdout:2375: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:15:46.621 INFO:tasks.rados.rados.0.smithi163.stdout:2375: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:15:46.621 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1067 (ObjNum 845 snap 289 seq_num 845) dirty exists 2023-10-09T09:15:46.621 INFO:tasks.rados.rados.0.smithi163.stdout:2372: done (2 left) 2023-10-09T09:15:46.622 INFO:tasks.rados.rados.0.smithi163.stdout:2373: done (1 left) 2023-10-09T09:15:46.622 INFO:tasks.rados.rados.0.smithi163.stdout:2375: done (0 left) 2023-10-09T09:15:46.622 INFO:tasks.rados.rados.0.smithi163.stdout:2376: rollback oid 33 current snap is 309 2023-10-09T09:15:46.622 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 305 2023-10-09T09:15:46.622 INFO:tasks.rados.rados.0.smithi163.stdout:2377: write oid 10 current snap is 309 2023-10-09T09:15:46.622 INFO:tasks.rados.rados.0.smithi163.stdout:2377: seq_num 891 ranges {575037=660160,1955809=671302} 2023-10-09T09:15:46.634 INFO:tasks.rados.rados.0.smithi163.stdout:2377: writing smithi16367185-10 from 575037 to 1235197 tid 1 2023-10-09T09:15:46.643 INFO:tasks.rados.rados.0.smithi163.stdout:2377: writing smithi16367185-10 from 1955809 to 2627111 tid 2 2023-10-09T09:15:46.643 INFO:tasks.rados.rados.0.smithi163.stdout:2378: delete oid 16 current snap is 309 2023-10-09T09:15:46.644 INFO:tasks.rados.rados.0.smithi163.stdout:2376: finishing rollback tid 0 to smithi16367185-33 2023-10-09T09:15:46.644 INFO:tasks.rados.rados.0.smithi163.stdout:2376: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:15:46.644 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1141 (ObjNum 699112672 snap 21875 seq_num 1618308144) dirty dne 2023-10-09T09:15:46.644 INFO:tasks.rados.rados.0.smithi163.stdout:2377: finishing write tid 1 to smithi16367185-10 2023-10-09T09:15:46.646 INFO:tasks.rados.rados.0.smithi163.stdout:2376: done (2 left) 2023-10-09T09:15:46.646 INFO:tasks.rados.rados.0.smithi163.stdout:2378: done (1 left) 2023-10-09T09:15:46.646 INFO:tasks.rados.rados.0.smithi163.stdout:2379: write oid 19 current snap is 309 2023-10-09T09:15:46.646 INFO:tasks.rados.rados.0.smithi163.stdout:2379: seq_num 893 ranges {608651=410658,1556420=674971,2773556=235095} 2023-10-09T09:15:46.652 INFO:tasks.rados.rados.0.smithi163.stdout:2379: writing smithi16367185-19 from 608651 to 1019309 tid 1 2023-10-09T09:15:46.658 INFO:tasks.rados.rados.0.smithi163.stdout:2379: writing smithi16367185-19 from 1556420 to 2231391 tid 2 2023-10-09T09:15:46.663 INFO:tasks.rados.rados.0.smithi163.stdout:2379: writing smithi16367185-19 from 2773556 to 3008651 tid 3 2023-10-09T09:15:46.663 INFO:tasks.rados.rados.0.smithi163.stdout:2380: read oid 41 snap -1 2023-10-09T09:15:46.663 INFO:tasks.rados.rados.0.smithi163.stdout:2380: expect (ObjNum 879 snap 307 seq_num 879) 2023-10-09T09:15:46.664 INFO:tasks.rados.rados.0.smithi163.stdout:2381: write oid 22 current snap is 309 2023-10-09T09:15:46.664 INFO:tasks.rados.rados.0.smithi163.stdout:2381: seq_num 894 ranges {625458=485907,1614902=534982,2917332=108126} 2023-10-09T09:15:46.669 INFO:tasks.rados.rados.0.smithi163.stdout:2381: writing smithi16367185-22 from 625458 to 1111365 tid 1 2023-10-09T09:15:46.676 INFO:tasks.rados.rados.0.smithi163.stdout:2381: writing smithi16367185-22 from 1614902 to 2149884 tid 2 2023-10-09T09:15:46.681 INFO:tasks.rados.rados.0.smithi163.stdout:2381: writing smithi16367185-22 from 2917332 to 3025458 tid 3 2023-10-09T09:15:46.681 INFO:tasks.rados.rados.0.smithi163.stdout:2382: snap_create 2023-10-09T09:15:46.681 INFO:tasks.rados.rados.0.smithi163.stdout:2377: finishing write tid 2 to smithi16367185-10 2023-10-09T09:15:46.681 INFO:tasks.rados.rados.0.smithi163.stdout:2377: finishing write tid 3 to smithi16367185-10 2023-10-09T09:15:46.681 INFO:tasks.rados.rados.0.smithi163.stdout:2377: finishing write tid 4 to smithi16367185-10 2023-10-09T09:15:46.681 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 779 (ObjNum 891 snap 309 seq_num 891) dirty exists 2023-10-09T09:15:46.681 INFO:tasks.rados.rados.0.smithi163.stdout:2377: left oid 10 (ObjNum 891 snap 309 seq_num 891) 2023-10-09T09:15:46.682 INFO:tasks.rados.rados.0.smithi163.stdout:2379: finishing write tid 1 to smithi16367185-19 2023-10-09T09:15:46.682 INFO:tasks.rados.rados.0.smithi163.stdout:2379: finishing write tid 2 to smithi16367185-19 2023-10-09T09:15:46.682 INFO:tasks.rados.rados.0.smithi163.stdout:2379: finishing write tid 3 to smithi16367185-19 2023-10-09T09:15:46.682 INFO:tasks.rados.rados.0.smithi163.stdout:2379: finishing write tid 4 to smithi16367185-19 2023-10-09T09:15:46.682 INFO:tasks.rados.rados.0.smithi163.stdout:2379: finishing write tid 5 to smithi16367185-19 2023-10-09T09:15:46.682 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 635 (ObjNum 893 snap 309 seq_num 893) dirty exists 2023-10-09T09:15:46.682 INFO:tasks.rados.rados.0.smithi163.stdout:2379: left oid 19 (ObjNum 893 snap 309 seq_num 893) 2023-10-09T09:15:46.682 INFO:tasks.rados.rados.0.smithi163.stdout:2380: expect (ObjNum 879 snap 307 seq_num 879) 2023-10-09T09:15:46.773 INFO:tasks.rados.rados.0.smithi163.stdout:2381: finishing write tid 1 to smithi16367185-22 2023-10-09T09:15:46.773 INFO:tasks.rados.rados.0.smithi163.stdout:2381: finishing write tid 2 to smithi16367185-22 2023-10-09T09:15:46.773 INFO:tasks.rados.rados.0.smithi163.stdout:2381: finishing write tid 3 to smithi16367185-22 2023-10-09T09:15:46.773 INFO:tasks.rados.rados.0.smithi163.stdout:2381: finishing write tid 4 to smithi16367185-22 2023-10-09T09:15:46.774 INFO:tasks.rados.rados.0.smithi163.stdout:2381: finishing write tid 5 to smithi16367185-22 2023-10-09T09:15:46.774 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 1765 (ObjNum 894 snap 309 seq_num 894) dirty exists 2023-10-09T09:15:46.774 INFO:tasks.rados.rados.0.smithi163.stdout:2381: left oid 22 (ObjNum 894 snap 309 seq_num 894) 2023-10-09T09:15:47.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:15:47.169 INFO:tasks.rados.rados.0.smithi163.stdout:2377: done (4 left) 2023-10-09T09:15:47.170 INFO:tasks.rados.rados.0.smithi163.stdout:2379: done (3 left) 2023-10-09T09:15:47.170 INFO:tasks.rados.rados.0.smithi163.stdout:2380: done (2 left) 2023-10-09T09:15:47.170 INFO:tasks.rados.rados.0.smithi163.stdout:2381: done (1 left) 2023-10-09T09:15:47.170 INFO:tasks.rados.rados.0.smithi163.stdout:2382: done (0 left) 2023-10-09T09:15:47.170 INFO:tasks.rados.rados.0.smithi163.stdout:2383: write oid 9 current snap is 310 2023-10-09T09:15:47.170 INFO:tasks.rados.rados.0.smithi163.stdout:2383: seq_num 895 ranges {642265=561156,1673384=678640,2828402=213863} 2023-10-09T09:15:47.180 INFO:tasks.rados.rados.0.smithi163.stdout:2383: writing smithi16367185-9 from 642265 to 1203421 tid 1 2023-10-09T09:15:47.188 INFO:tasks.rados.rados.0.smithi163.stdout:2383: writing smithi16367185-9 from 1673384 to 2352024 tid 2 2023-10-09T09:15:47.192 INFO:tasks.rados.rados.0.smithi163.stdout:2383: writing smithi16367185-9 from 2828402 to 3042265 tid 3 2023-10-09T09:15:47.192 INFO:tasks.rados.rados.0.smithi163.stdout:2384: delete oid 25 current snap is 310 2023-10-09T09:15:47.192 INFO:tasks.rados.rados.0.smithi163.stdout:2383: finishing write tid 1 to smithi16367185-9 2023-10-09T09:15:47.192 INFO:tasks.rados.rados.0.smithi163.stdout:2383: finishing write tid 2 to smithi16367185-9 2023-10-09T09:15:47.196 INFO:tasks.rados.rados.0.smithi163.stdout:2383: finishing write tid 3 to smithi16367185-9 2023-10-09T09:15:47.196 INFO:tasks.rados.rados.0.smithi163.stdout:2383: finishing write tid 4 to smithi16367185-9 2023-10-09T09:15:47.196 INFO:tasks.rados.rados.0.smithi163.stdout:2383: finishing write tid 5 to smithi16367185-9 2023-10-09T09:15:47.196 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 607 (ObjNum 895 snap 310 seq_num 895) dirty exists 2023-10-09T09:15:47.197 INFO:tasks.rados.rados.0.smithi163.stdout:2383: left oid 9 (ObjNum 895 snap 310 seq_num 895) 2023-10-09T09:15:47.198 INFO:tasks.rados.rados.0.smithi163.stdout:2383: done (1 left) 2023-10-09T09:15:47.198 INFO:tasks.rados.rados.0.smithi163.stdout:2384: done (0 left) 2023-10-09T09:15:47.198 INFO:tasks.rados.rados.0.smithi163.stdout:2385: read oid 14 snap -1 2023-10-09T09:15:47.199 INFO:tasks.rados.rados.0.smithi163.stdout:2385: expect (ObjNum 850 snap 289 seq_num 850) 2023-10-09T09:15:47.199 INFO:tasks.rados.rados.0.smithi163.stdout:2386: write oid 2 current snap is 310 2023-10-09T09:15:47.199 INFO:tasks.rados.rados.0.smithi163.stdout:2386: seq_num 897 ranges {675879=711654,2073995=682309} 2023-10-09T09:15:47.207 INFO:tasks.rados.rados.0.smithi163.stdout:2386: writing smithi16367185-2 from 675879 to 1387533 tid 1 2023-10-09T09:15:47.215 INFO:tasks.rados.rados.0.smithi163.stdout:2386: writing smithi16367185-2 from 2073995 to 2756304 tid 2 2023-10-09T09:15:47.215 INFO:tasks.rados.rados.0.smithi163.stdout:2387: write oid 19 current snap is 310 2023-10-09T09:15:47.215 INFO:tasks.rados.rados.0.smithi163.stdout:2387: seq_num 898 ranges {692686=503256,1848830=425967,2910671=182015} 2023-10-09T09:15:47.223 INFO:tasks.rados.rados.0.smithi163.stdout:2387: writing smithi16367185-19 from 692686 to 1195942 tid 1 2023-10-09T09:15:47.229 INFO:tasks.rados.rados.0.smithi163.stdout:2387: writing smithi16367185-19 from 1848830 to 2274797 tid 2 2023-10-09T09:15:47.235 INFO:tasks.rados.rados.0.smithi163.stdout:2387: writing smithi16367185-19 from 2910671 to 3092686 tid 3 2023-10-09T09:15:47.235 INFO:tasks.rados.rados.0.smithi163.stdout:2388: read oid 46 snap -1 2023-10-09T09:15:47.235 INFO:tasks.rados.rados.0.smithi163.stdout:2388: expect deleted 2023-10-09T09:15:47.235 INFO:tasks.rados.rados.0.smithi163.stdout:2389: read oid 32 snap -1 2023-10-09T09:15:47.235 INFO:tasks.rados.rados.0.smithi163.stdout:2389: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:15:47.235 INFO:tasks.rados.rados.0.smithi163.stdout:2390: read oid 45 snap -1 2023-10-09T09:15:47.235 INFO:tasks.rados.rados.0.smithi163.stdout:2390: expect (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:15:47.235 INFO:tasks.rados.rados.0.smithi163.stdout:2391: write oid 8 current snap is 310 2023-10-09T09:15:47.235 INFO:tasks.rados.rados.0.smithi163.stdout:2391: seq_num 899 ranges {709493=578505,1907312=685978} 2023-10-09T09:15:47.242 INFO:tasks.rados.rados.0.smithi163.stdout:2391: writing smithi16367185-8 from 709493 to 1287998 tid 1 2023-10-09T09:15:47.252 INFO:tasks.rados.rados.0.smithi163.stdout:2391: writing smithi16367185-8 from 1907312 to 2593290 tid 2 2023-10-09T09:15:47.253 INFO:tasks.rados.rados.0.smithi163.stdout:2392: delete oid 31 current snap is 310 2023-10-09T09:15:47.253 INFO:tasks.rados.rados.0.smithi163.stdout:2385: expect (ObjNum 850 snap 289 seq_num 850) 2023-10-09T09:15:47.395 INFO:tasks.rados.rados.0.smithi163.stdout:2386: finishing write tid 1 to smithi16367185-2 2023-10-09T09:15:47.396 INFO:tasks.rados.rados.0.smithi163.stdout:2386: finishing write tid 2 to smithi16367185-2 2023-10-09T09:15:47.396 INFO:tasks.rados.rados.0.smithi163.stdout:2385: done (7 left) 2023-10-09T09:15:47.396 INFO:tasks.rados.rados.0.smithi163.stdout:2392: done (6 left) 2023-10-09T09:15:47.396 INFO:tasks.rados.rados.0.smithi163.stdout:2393: snap_remove snap 280 2023-10-09T09:15:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:47 smithi163 bash[75425]: cluster 2023-10-09T09:15:46.153730+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e674 with expected crc 2023-10-09T09:15:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:47 smithi163 bash[75425]: cluster 2023-10-09T09:15:46.156582+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e674 with expected crc 2023-10-09T09:15:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:47 smithi163 bash[75425]: cluster 2023-10-09T09:15:46.156652+0000 mon.a (mon.0) 840 : cluster 0 osdmap e674: 8 total, 8 up, 8 in 2023-10-09T09:15:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:47 smithi163 bash[75425]: audit 2023-10-09T09:15:46.267863+0000 mgr.y (mgr.34107) 275 : audit 0 from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:15:47.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:47 smithi163 bash[78107]: cluster 2023-10-09T09:15:46.153730+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e674 with expected crc 2023-10-09T09:15:47.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:47 smithi163 bash[78107]: cluster 2023-10-09T09:15:46.156582+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e674 with expected crc 2023-10-09T09:15:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:47 smithi163 bash[78107]: cluster 2023-10-09T09:15:46.156652+0000 mon.a (mon.0) 840 : cluster 0 osdmap e674: 8 total, 8 up, 8 in 2023-10-09T09:15:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:47 smithi163 bash[78107]: audit 2023-10-09T09:15:46.267863+0000 mgr.y (mgr.34107) 275 : audit 0 from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:15:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:47 smithi181 bash[59345]: cluster 2023-10-09T09:15:46.153730+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e674 with expected crc 2023-10-09T09:15:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:47 smithi181 bash[59345]: cluster 2023-10-09T09:15:46.156582+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e674 with expected crc 2023-10-09T09:15:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:47 smithi181 bash[59345]: cluster 2023-10-09T09:15:46.156652+0000 mon.a (mon.0) 840 : cluster 0 osdmap e674: 8 total, 8 up, 8 in 2023-10-09T09:15:47.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:47 smithi181 bash[59345]: audit 2023-10-09T09:15:46.267863+0000 mgr.y (mgr.34107) 275 : audit 0 from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:15:48.187 INFO:tasks.rados.rados.0.smithi163.stdout:2393: done (6 left) 2023-10-09T09:15:48.187 INFO:tasks.rados.rados.0.smithi163.stdout:2394: read oid 17 snap -1 2023-10-09T09:15:48.187 INFO:tasks.rados.rados.0.smithi163.stdout:2394: expect (ObjNum 885 snap 308 seq_num 885) 2023-10-09T09:15:48.187 INFO:tasks.rados.rados.0.smithi163.stdout:2395: read oid 27 snap -1 2023-10-09T09:15:48.187 INFO:tasks.rados.rados.0.smithi163.stdout:2395: expect (ObjNum 852 snap 289 seq_num 852) 2023-10-09T09:15:48.187 INFO:tasks.rados.rados.0.smithi163.stdout:2396: read oid 23 snap 299 2023-10-09T09:15:48.187 INFO:tasks.rados.rados.0.smithi163.stdout:2396: expect (ObjNum 775 snap 268 seq_num 775) 2023-10-09T09:15:48.187 INFO:tasks.rados.rados.0.smithi163.stdout:2397: delete oid 9 current snap is 310 2023-10-09T09:15:48.187 INFO:tasks.rados.rados.0.smithi163.stdout:2386: finishing write tid 3 to smithi16367185-2 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:2386: finishing write tid 4 to smithi16367185-2 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 615 (ObjNum 897 snap 310 seq_num 897) dirty exists 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:2386: left oid 2 (ObjNum 897 snap 310 seq_num 897) 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:2387: finishing write tid 1 to smithi16367185-19 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:2387: finishing write tid 2 to smithi16367185-19 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:2387: finishing write tid 3 to smithi16367185-19 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:2387: finishing write tid 4 to smithi16367185-19 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:2387: finishing write tid 5 to smithi16367185-19 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 640 (ObjNum 898 snap 310 seq_num 898) dirty exists 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:2387: left oid 19 (ObjNum 898 snap 310 seq_num 898) 2023-10-09T09:15:48.188 INFO:tasks.rados.rados.0.smithi163.stdout:2389: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:15:48.381 INFO:tasks.rados.rados.0.smithi163.stdout:2391: finishing write tid 1 to smithi16367185-8 2023-10-09T09:15:48.381 INFO:tasks.rados.rados.0.smithi163.stdout:2390: expect (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:15:48.401 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:48 smithi163 bash[75425]: cluster 2023-10-09T09:15:47.166193+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e675 with expected crc 2023-10-09T09:15:48.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:48 smithi163 bash[75425]: cluster 2023-10-09T09:15:47.169285+0000 mon.a (mon.0) 841 : cluster 0 osdmap e675: 8 total, 8 up, 8 in 2023-10-09T09:15:48.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:48 smithi163 bash[75425]: cluster 2023-10-09T09:15:47.181930+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e675 with expected crc 2023-10-09T09:15:48.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:48 smithi163 bash[75425]: cluster 2023-10-09T09:15:47.209073+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e675 with expected crc 2023-10-09T09:15:48.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:48 smithi163 bash[75425]: cluster 2023-10-09T09:15:47.353727+0000 mgr.y (mgr.34107) 276 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2023-10-09T09:15:48.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:48 smithi163 bash[78107]: cluster 2023-10-09T09:15:47.166193+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e675 with expected crc 2023-10-09T09:15:48.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:48 smithi163 bash[78107]: cluster 2023-10-09T09:15:47.169285+0000 mon.a (mon.0) 841 : cluster 0 osdmap e675: 8 total, 8 up, 8 in 2023-10-09T09:15:48.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:48 smithi163 bash[78107]: cluster 2023-10-09T09:15:47.181930+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e675 with expected crc 2023-10-09T09:15:48.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:48 smithi163 bash[78107]: cluster 2023-10-09T09:15:47.209073+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e675 with expected crc 2023-10-09T09:15:48.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:48 smithi163 bash[78107]: cluster 2023-10-09T09:15:47.353727+0000 mgr.y (mgr.34107) 276 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2023-10-09T09:15:48.465 INFO:tasks.rados.rados.0.smithi163.stdout:2391: finishing write tid 2 to smithi16367185-8 2023-10-09T09:15:48.465 INFO:tasks.rados.rados.0.smithi163.stdout:2391: finishing write tid 3 to smithi16367185-8 2023-10-09T09:15:48.465 INFO:tasks.rados.rados.0.smithi163.stdout:2391: finishing write tid 4 to smithi16367185-8 2023-10-09T09:15:48.465 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1167 (ObjNum 899 snap 310 seq_num 899) dirty exists 2023-10-09T09:15:48.465 INFO:tasks.rados.rados.0.smithi163.stdout:2391: left oid 8 (ObjNum 899 snap 310 seq_num 899) 2023-10-09T09:15:48.465 INFO:tasks.rados.rados.0.smithi163.stdout:2386: done (9 left) 2023-10-09T09:15:48.465 INFO:tasks.rados.rados.0.smithi163.stdout:2387: done (8 left) 2023-10-09T09:15:48.466 INFO:tasks.rados.rados.0.smithi163.stdout:2388: done (7 left) 2023-10-09T09:15:48.466 INFO:tasks.rados.rados.0.smithi163.stdout:2389: done (6 left) 2023-10-09T09:15:48.466 INFO:tasks.rados.rados.0.smithi163.stdout:2390: done (5 left) 2023-10-09T09:15:48.466 INFO:tasks.rados.rados.0.smithi163.stdout:2391: done (4 left) 2023-10-09T09:15:48.466 INFO:tasks.rados.rados.0.smithi163.stdout:2397: done (3 left) 2023-10-09T09:15:48.466 INFO:tasks.rados.rados.0.smithi163.stdout:2398: write oid 36 current snap is 310 2023-10-09T09:15:48.467 INFO:tasks.rados.rados.0.smithi163.stdout:2396: expect (ObjNum 775 snap 268 seq_num 775) 2023-10-09T09:15:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:48 smithi181 bash[59345]: cluster 2023-10-09T09:15:47.166193+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e675 with expected crc 2023-10-09T09:15:48.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:48 smithi181 bash[59345]: cluster 2023-10-09T09:15:47.169285+0000 mon.a (mon.0) 841 : cluster 0 osdmap e675: 8 total, 8 up, 8 in 2023-10-09T09:15:48.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:48 smithi181 bash[59345]: cluster 2023-10-09T09:15:47.181930+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e675 with expected crc 2023-10-09T09:15:48.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:48 smithi181 bash[59345]: cluster 2023-10-09T09:15:47.209073+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e675 with expected crc 2023-10-09T09:15:48.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:48 smithi181 bash[59345]: cluster 2023-10-09T09:15:47.353727+0000 mgr.y (mgr.34107) 276 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2023-10-09T09:15:48.638 INFO:tasks.rados.rados.0.smithi163.stdout:2395: expect (ObjNum 852 snap 289 seq_num 852) 2023-10-09T09:15:48.764 INFO:tasks.rados.rados.0.smithi163.stdout:2394: expect (ObjNum 885 snap 308 seq_num 885) 2023-10-09T09:15:48.923 INFO:tasks.rados.rados.0.smithi163.stdout:2398: seq_num 902 ranges {759914=404252,1566405=433305,2504010=655904} 2023-10-09T09:15:48.935 INFO:tasks.rados.rados.0.smithi163.stdout:2398: writing smithi16367185-36 from 759914 to 1164166 tid 1 2023-10-09T09:15:48.941 INFO:tasks.rados.rados.0.smithi163.stdout:2398: writing smithi16367185-36 from 1566405 to 1999710 tid 2 2023-10-09T09:15:48.948 INFO:tasks.rados.rados.0.smithi163.stdout:2398: writing smithi16367185-36 from 2504010 to 3159914 tid 3 2023-10-09T09:15:48.949 INFO:tasks.rados.rados.0.smithi163.stdout:2394: done (3 left) 2023-10-09T09:15:48.949 INFO:tasks.rados.rados.0.smithi163.stdout:2395: done (2 left) 2023-10-09T09:15:48.949 INFO:tasks.rados.rados.0.smithi163.stdout:2396: done (1 left) 2023-10-09T09:15:48.949 INFO:tasks.rados.rados.0.smithi163.stdout:2399: read oid 26 snap 303 2023-10-09T09:15:48.949 INFO:tasks.rados.rados.0.smithi163.stdout:2399: expect (ObjNum 731 snap 246 seq_num 731) 2023-10-09T09:15:48.949 INFO:tasks.rados.rados.0.smithi163.stdout:2400: write oid 12 current snap is 310 2023-10-09T09:15:48.949 INFO:tasks.rados.rados.0.smithi163.stdout:2398: finishing write tid 1 to smithi16367185-36 2023-10-09T09:15:48.949 INFO:tasks.rados.rados.0.smithi163.stdout:2400: seq_num 903 ranges {776721=479501,2024887=693316} 2023-10-09T09:15:48.956 INFO:tasks.rados.rados.0.smithi163.stdout:2400: writing smithi16367185-12 from 776721 to 1256222 tid 1 2023-10-09T09:15:48.965 INFO:tasks.rados.rados.0.smithi163.stdout:2400: writing smithi16367185-12 from 2024887 to 2718203 tid 2 2023-10-09T09:15:48.965 INFO:tasks.rados.rados.0.smithi163.stdout:2401: rollback oid 50 current snap is 310 2023-10-09T09:15:48.965 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 50 to 295 2023-10-09T09:15:48.965 INFO:tasks.rados.rados.0.smithi163.stdout:2398: finishing write tid 2 to smithi16367185-36 2023-10-09T09:15:48.965 INFO:tasks.rados.rados.0.smithi163.stdout:2402: read oid 47 snap -1 2023-10-09T09:15:48.965 INFO:tasks.rados.rados.0.smithi163.stdout:2402: expect deleted 2023-10-09T09:15:48.965 INFO:tasks.rados.rados.0.smithi163.stdout:2403: delete oid 40 current snap is 310 2023-10-09T09:15:48.965 INFO:tasks.rados.rados.0.smithi163.stdout:2399: expect (ObjNum 731 snap 246 seq_num 731) 2023-10-09T09:15:49.196 INFO:tasks.rados.rados.0.smithi163.stdout:2398: finishing write tid 3 to smithi16367185-36 2023-10-09T09:15:49.196 INFO:tasks.rados.rados.0.smithi163.stdout:2398: finishing write tid 4 to smithi16367185-36 2023-10-09T09:15:49.196 INFO:tasks.rados.rados.0.smithi163.stdout:2398: finishing write tid 5 to smithi16367185-36 2023-10-09T09:15:49.196 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 603 (ObjNum 902 snap 310 seq_num 902) dirty exists 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2398: left oid 36 (ObjNum 902 snap 310 seq_num 902) 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2400: finishing write tid 1 to smithi16367185-12 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2400: finishing write tid 2 to smithi16367185-12 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2400: finishing write tid 3 to smithi16367185-12 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2400: finishing write tid 4 to smithi16367185-12 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 1152 (ObjNum 903 snap 310 seq_num 903) dirty exists 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2400: left oid 12 (ObjNum 903 snap 310 seq_num 903) 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2398: done (5 left) 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2399: done (4 left) 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2400: done (3 left) 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2403: done (2 left) 2023-10-09T09:15:49.197 INFO:tasks.rados.rados.0.smithi163.stdout:2404: snap_remove snap 290 2023-10-09T09:15:49.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:49 smithi163 bash[78107]: cluster 2023-10-09T09:15:48.176457+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:49 smithi163 bash[78107]: cluster 2023-10-09T09:15:48.186141+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:49 smithi163 bash[78107]: cluster 2023-10-09T09:15:48.186179+0000 mon.a (mon.0) 842 : cluster 0 osdmap e676: 8 total, 8 up, 8 in 2023-10-09T09:15:49.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:49 smithi163 bash[78107]: cluster 2023-10-09T09:15:48.187793+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:49 smithi163 bash[78107]: cluster 2023-10-09T09:15:48.190381+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:49 smithi163 bash[75425]: cluster 2023-10-09T09:15:48.176457+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:49 smithi163 bash[75425]: cluster 2023-10-09T09:15:48.186141+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:49 smithi163 bash[75425]: cluster 2023-10-09T09:15:48.186179+0000 mon.a (mon.0) 842 : cluster 0 osdmap e676: 8 total, 8 up, 8 in 2023-10-09T09:15:49.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:49 smithi163 bash[75425]: cluster 2023-10-09T09:15:48.187793+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:49 smithi163 bash[75425]: cluster 2023-10-09T09:15:48.190381+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:49 smithi181 bash[59345]: cluster 2023-10-09T09:15:48.176457+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:49 smithi181 bash[59345]: cluster 2023-10-09T09:15:48.186141+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:49 smithi181 bash[59345]: cluster 2023-10-09T09:15:48.186179+0000 mon.a (mon.0) 842 : cluster 0 osdmap e676: 8 total, 8 up, 8 in 2023-10-09T09:15:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:49 smithi181 bash[59345]: cluster 2023-10-09T09:15:48.187793+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:49.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:49 smithi181 bash[59345]: cluster 2023-10-09T09:15:48.190381+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e676 with expected crc 2023-10-09T09:15:50.147 INFO:tasks.rados.rados.0.smithi163.stdout:2404: done (2 left) 2023-10-09T09:15:50.148 INFO:tasks.rados.rados.0.smithi163.stdout:2405: read oid 21 snap 308 2023-10-09T09:15:50.148 INFO:tasks.rados.rados.0.smithi163.stdout:2405: expect (ObjNum 867 snap 301 seq_num 867) 2023-10-09T09:15:50.148 INFO:tasks.rados.rados.0.smithi163.stdout:2406: delete oid 1 current snap is 310 2023-10-09T09:15:50.148 INFO:tasks.rados.rados.0.smithi163.stdout:2401: finishing rollback tid 0 to smithi16367185-50 2023-10-09T09:15:50.148 INFO:tasks.rados.rados.0.smithi163.stdout:2401: finishing rollback tid 1 to smithi16367185-50 2023-10-09T09:15:50.148 INFO:tasks.rados.rados.0.smithi163.stdout:2401: finishing rollback tid 2 to smithi16367185-50 2023-10-09T09:15:50.148 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1079 (ObjNum 824 snap 282 seq_num 824) dirty exists 2023-10-09T09:15:50.152 INFO:tasks.rados.rados.0.smithi163.stdout:2405: expect (ObjNum 867 snap 301 seq_num 867) 2023-10-09T09:15:50.246 INFO:tasks.rados.rados.0.smithi163.stdout:2401: done (3 left) 2023-10-09T09:15:50.246 INFO:tasks.rados.rados.0.smithi163.stdout:2402: done (2 left) 2023-10-09T09:15:50.246 INFO:tasks.rados.rados.0.smithi163.stdout:2405: done (1 left) 2023-10-09T09:15:50.246 INFO:tasks.rados.rados.0.smithi163.stdout:2406: done (0 left) 2023-10-09T09:15:50.246 INFO:tasks.rados.rados.0.smithi163.stdout:2407: rollback oid 37 current snap is 310 2023-10-09T09:15:50.246 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 286 2023-10-09T09:15:50.246 INFO:tasks.rados.rados.0.smithi163.stdout:2408: rollback oid 34 current snap is 310 2023-10-09T09:15:50.247 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 34 to 309 2023-10-09T09:15:50.247 INFO:tasks.rados.rados.0.smithi163.stdout:2409: write oid 50 current snap is 310 2023-10-09T09:15:50.247 INFO:tasks.rados.rados.0.smithi163.stdout:2409: seq_num 906 ranges {427142=421601,1400333=440643,2613702=450326} 2023-10-09T09:15:50.257 INFO:tasks.rados.rados.0.smithi163.stdout:2409: writing smithi16367185-50 from 427142 to 848743 tid 1 2023-10-09T09:15:50.263 INFO:tasks.rados.rados.0.smithi163.stdout:2409: writing smithi16367185-50 from 1400333 to 1840976 tid 2 2023-10-09T09:15:50.270 INFO:tasks.rados.rados.0.smithi163.stdout:2409: writing smithi16367185-50 from 2613702 to 3064028 tid 3 2023-10-09T09:15:50.270 INFO:tasks.rados.rados.0.smithi163.stdout:2410: snap_create 2023-10-09T09:15:50.270 INFO:tasks.rados.rados.0.smithi163.stdout:2407: finishing rollback tid 0 to smithi16367185-37 2023-10-09T09:15:50.270 INFO:tasks.rados.rados.0.smithi163.stdout:2407: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:15:50.271 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 1180 (ObjNum 698870832 snap 21875 seq_num 699299088) dirty dne 2023-10-09T09:15:50.271 INFO:tasks.rados.rados.0.smithi163.stdout:2408: finishing rollback tid 0 to smithi16367185-34 2023-10-09T09:15:50.271 INFO:tasks.rados.rados.0.smithi163.stdout:2408: finishing rollback tid 1 to smithi16367185-34 2023-10-09T09:15:50.271 INFO:tasks.rados.rados.0.smithi163.stdout:2408: finishing rollback tid 2 to smithi16367185-34 2023-10-09T09:15:50.271 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1089 (ObjNum 882 snap 308 seq_num 882) dirty exists 2023-10-09T09:15:50.271 INFO:tasks.rados.rados.0.smithi163.stdout:2409: finishing write tid 1 to smithi16367185-50 2023-10-09T09:15:50.271 INFO:tasks.rados.rados.0.smithi163.stdout:2409: finishing write tid 2 to smithi16367185-50 2023-10-09T09:15:50.275 INFO:tasks.rados.rados.0.smithi163.stdout:2409: finishing write tid 3 to smithi16367185-50 2023-10-09T09:15:50.275 INFO:tasks.rados.rados.0.smithi163.stdout:2409: finishing write tid 4 to smithi16367185-50 2023-10-09T09:15:50.276 INFO:tasks.rados.rados.0.smithi163.stdout:2409: finishing write tid 5 to smithi16367185-50 2023-10-09T09:15:50.276 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1093 (ObjNum 906 snap 310 seq_num 906) dirty exists 2023-10-09T09:15:50.276 INFO:tasks.rados.rados.0.smithi163.stdout:2409: left oid 50 (ObjNum 906 snap 310 seq_num 906) 2023-10-09T09:15:50.327 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:15:50.327 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (8m) - 14m 13.7M - 0.23.0 ba2b418f427c 82a8abc84166 2023-10-09T09:15:50.327 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (7m) - 13m 41.4M - 8.3.5 dad864ee21e9 9d80a71ddd1d 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.vnodzj smithi163 running (12m) - 12m 19.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 892077b22450 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.ajqgsc smithi181 running (12m) - 12m 14.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 01d89554089b 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443,9283,8765 running (6m) - 20m 428M - 18.2.0-598-gb4fa663f 3060897fdf97 ca7a91dd3dab 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (7m) - 22m 485M - 18.2.0-598-gb4fa663f 3060897fdf97 fc427389eb0c 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (6m) - 22m 87.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 097aa5987723 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (3m) - 21m 42.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 76abe6c0dfe2 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (4m) - 20m 55.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 adc323592e84 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (8m) - 14m 9511k - 1.3.1 1dbe0e931976 8a6fec29baf3 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (8m) - 14m 8928k - 1.3.1 1dbe0e931976 3f766d093db8 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (104s) - 20m 93.4M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 242f1e42dfa2 2023-10-09T09:15:50.328 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (77s) - 19m 70.4M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 26f120975fe0 2023-10-09T09:15:50.329 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (51s) - 18m 75.3M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 f218a8a2ac28 2023-10-09T09:15:50.329 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (23s) - 18m - 2524M 18.2.0-598-gb4fa663f 3060897fdf97 13ae6c567443 2023-10-09T09:15:50.329 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (17m) - 17m 116M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5dd6bc4b279 2023-10-09T09:15:50.329 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (16m) - 16m 127M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9f3e4fbba907 2023-10-09T09:15:50.329 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (16m) - 16m 81.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 30f1ac2f4524 2023-10-09T09:15:50.329 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (15m) - 15m 102M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 54105d827fdd 2023-10-09T09:15:50.329 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (7m) - 14m 58.2M - 2.33.4 514e6a882f6e cc81501e414f 2023-10-09T09:15:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:50 smithi163 bash[78107]: cluster 2023-10-09T09:15:49.354665+0000 mgr.y (mgr.34107) 277 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2023-10-09T09:15:50.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:50 smithi163 bash[78107]: cluster 2023-10-09T09:15:50.143906+0000 mon.a (mon.0) 843 : cluster 0 osdmap e677: 8 total, 8 up, 8 in 2023-10-09T09:15:50.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:50 smithi163 bash[75425]: cluster 2023-10-09T09:15:49.354665+0000 mgr.y (mgr.34107) 277 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2023-10-09T09:15:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:50 smithi163 bash[75425]: cluster 2023-10-09T09:15:50.143906+0000 mon.a (mon.0) 843 : cluster 0 osdmap e677: 8 total, 8 up, 8 in 2023-10-09T09:15:50.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:50 smithi181 bash[59345]: cluster 2023-10-09T09:15:49.354665+0000 mgr.y (mgr.34107) 277 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2023-10-09T09:15:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:50 smithi181 bash[59345]: cluster 2023-10-09T09:15:50.143906+0000 mon.a (mon.0) 843 : cluster 0 osdmap e677: 8 total, 8 up, 8 in 2023-10-09T09:15:50.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T09:15:51.195 INFO:tasks.rados.rados.0.smithi163.stdout:2407: done (3 left) 2023-10-09T09:15:51.195 INFO:tasks.rados.rados.0.smithi163.stdout:2408: done (2 left) 2023-10-09T09:15:51.195 INFO:tasks.rados.rados.0.smithi163.stdout:2409: done (1 left) 2023-10-09T09:15:51.195 INFO:tasks.rados.rados.0.smithi163.stdout:2410: done (0 left) 2023-10-09T09:15:51.195 INFO:tasks.rados.rados.0.smithi163.stdout:2411: delete oid 24 current snap is 311 2023-10-09T09:15:51.202 INFO:tasks.rados.rados.0.smithi163.stdout:2411: done (0 left) 2023-10-09T09:15:51.202 INFO:tasks.rados.rados.0.smithi163.stdout:2412: rollback oid 18 current snap is 311 2023-10-09T09:15:51.202 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 18 to 307 2023-10-09T09:15:51.202 INFO:tasks.rados.rados.0.smithi163.stdout:2413: snap_remove snap 299 2023-10-09T09:15:51.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:51 smithi163 bash[78107]: cluster 2023-10-09T09:15:50.140305+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e677 with expected crc 2023-10-09T09:15:51.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:51 smithi163 bash[78107]: cluster 2023-10-09T09:15:50.146562+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e677 with expected crc 2023-10-09T09:15:51.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:51 smithi163 bash[78107]: audit 2023-10-09T09:15:50.307650+0000 mgr.y (mgr.34107) 278 : audit 0 from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:15:51.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:51 smithi163 bash[78107]: audit 2023-10-09T09:15:50.458031+0000 mon.a (mon.0) 844 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:51.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:51 smithi163 bash[75425]: cluster 2023-10-09T09:15:50.140305+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e677 with expected crc 2023-10-09T09:15:51.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:51 smithi163 bash[75425]: cluster 2023-10-09T09:15:50.146562+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e677 with expected crc 2023-10-09T09:15:51.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:51 smithi163 bash[75425]: audit 2023-10-09T09:15:50.307650+0000 mgr.y (mgr.34107) 278 : audit 0 from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:15:51.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:51 smithi163 bash[75425]: audit 2023-10-09T09:15:50.458031+0000 mon.a (mon.0) 844 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:51 smithi181 bash[59345]: cluster 2023-10-09T09:15:50.140305+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e677 with expected crc 2023-10-09T09:15:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:51 smithi181 bash[59345]: cluster 2023-10-09T09:15:50.146562+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e677 with expected crc 2023-10-09T09:15:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:51 smithi181 bash[59345]: audit 2023-10-09T09:15:50.307650+0000 mgr.y (mgr.34107) 278 : audit 0 from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:15:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:51 smithi181 bash[59345]: audit 2023-10-09T09:15:50.458031+0000 mon.a (mon.0) 844 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:15:52.209 INFO:tasks.rados.rados.0.smithi163.stdout:2413: done (1 left) 2023-10-09T09:15:52.209 INFO:tasks.rados.rados.0.smithi163.stdout:2414: write oid 19 current snap is 311 2023-10-09T09:15:52.209 INFO:tasks.rados.rados.0.smithi163.stdout:2412: finishing rollback tid 1 to smithi16367185-18 2023-10-09T09:15:52.209 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 752 (ObjNum 1 snap 0 seq_num 699172608) dirty dne 2023-10-09T09:15:52.209 INFO:tasks.rados.rados.0.smithi163.stdout:2414: seq_num 908 ranges {460756=572099,1517297=444312,2668548=472870} 2023-10-09T09:15:52.218 INFO:tasks.rados.rados.0.smithi163.stdout:2414: writing smithi16367185-19 from 460756 to 1032855 tid 1 2023-10-09T09:15:52.225 INFO:tasks.rados.rados.0.smithi163.stdout:2414: writing smithi16367185-19 from 1517297 to 1961609 tid 2 2023-10-09T09:15:52.232 INFO:tasks.rados.rados.0.smithi163.stdout:2414: writing smithi16367185-19 from 2668548 to 3141418 tid 3 2023-10-09T09:15:52.232 INFO:tasks.rados.rados.0.smithi163.stdout:2412: done (1 left) 2023-10-09T09:15:52.232 INFO:tasks.rados.rados.0.smithi163.stdout:2415: write oid 14 current snap is 311 2023-10-09T09:15:52.232 INFO:tasks.rados.rados.0.smithi163.stdout:2414: finishing write tid 1 to smithi16367185-19 2023-10-09T09:15:52.232 INFO:tasks.rados.rados.0.smithi163.stdout:2414: finishing write tid 2 to smithi16367185-19 2023-10-09T09:15:52.233 INFO:tasks.rados.rados.0.smithi163.stdout:2415: seq_num 909 ranges {477563=647348,1859426=587970,2979618=297945} 2023-10-09T09:15:52.238 INFO:tasks.rados.rados.0.smithi163.stdout:2415: writing smithi16367185-14 from 477563 to 1124911 tid 1 2023-10-09T09:15:52.247 INFO:tasks.rados.rados.0.smithi163.stdout:2415: writing smithi16367185-14 from 1859426 to 2447396 tid 2 2023-10-09T09:15:52.252 INFO:tasks.rados.rados.0.smithi163.stdout:2415: writing smithi16367185-14 from 2979618 to 3277563 tid 3 2023-10-09T09:15:52.252 INFO:tasks.rados.rados.0.smithi163.stdout:2416: read oid 10 snap 295 2023-10-09T09:15:52.252 INFO:tasks.rados.rados.0.smithi163.stdout:2416: expect (ObjNum 863 snap 293 seq_num 863) 2023-10-09T09:15:52.252 INFO:tasks.rados.rados.0.smithi163.stdout:2417: rollback oid 44 current snap is 311 2023-10-09T09:15:52.252 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 44 to 294 2023-10-09T09:15:52.253 INFO:tasks.rados.rados.0.smithi163.stdout:2414: finishing write tid 3 to smithi16367185-19 2023-10-09T09:15:52.253 INFO:tasks.rados.rados.0.smithi163.stdout:2418: snap_remove snap 301 2023-10-09T09:15:52.253 INFO:tasks.rados.rados.0.smithi163.stdout:2414: finishing write tid 4 to smithi16367185-19 2023-10-09T09:15:52.253 INFO:tasks.rados.rados.0.smithi163.stdout:2414: finishing write tid 5 to smithi16367185-19 2023-10-09T09:15:52.253 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 647 (ObjNum 908 snap 311 seq_num 908) dirty exists 2023-10-09T09:15:52.253 INFO:tasks.rados.rados.0.smithi163.stdout:2414: left oid 19 (ObjNum 908 snap 311 seq_num 908) 2023-10-09T09:15:52.253 INFO:tasks.rados.rados.0.smithi163.stdout:2415: finishing write tid 1 to smithi16367185-14 2023-10-09T09:15:52.253 INFO:tasks.rados.rados.0.smithi163.stdout:2415: finishing write tid 2 to smithi16367185-14 2023-10-09T09:15:52.419 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:52 smithi163 bash[78107]: cluster 2023-10-09T09:15:51.191438+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:52.419 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:52 smithi163 bash[78107]: cluster 2023-10-09T09:15:51.194924+0000 mon.a (mon.0) 845 : cluster 0 osdmap e678: 8 total, 8 up, 8 in 2023-10-09T09:15:52.419 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:52 smithi163 bash[78107]: cluster 2023-10-09T09:15:51.200405+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:52.419 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:52 smithi163 bash[78107]: cluster 2023-10-09T09:15:51.355429+0000 mgr.y (mgr.34107) 279 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 209 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.2 MiB/s wr, 37 op/s 2023-10-09T09:15:52.419 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:52 smithi163 bash[78107]: cluster 2023-10-09T09:15:51.467661+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:52.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:52 smithi163 bash[75425]: cluster 2023-10-09T09:15:51.191438+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:52.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:52 smithi163 bash[75425]: cluster 2023-10-09T09:15:51.194924+0000 mon.a (mon.0) 845 : cluster 0 osdmap e678: 8 total, 8 up, 8 in 2023-10-09T09:15:52.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:52 smithi163 bash[75425]: cluster 2023-10-09T09:15:51.200405+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:52.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:52 smithi163 bash[75425]: cluster 2023-10-09T09:15:51.355429+0000 mgr.y (mgr.34107) 279 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 209 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.2 MiB/s wr, 37 op/s 2023-10-09T09:15:52.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:52 smithi163 bash[75425]: cluster 2023-10-09T09:15:51.467661+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:52.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:52 smithi181 bash[59345]: cluster 2023-10-09T09:15:51.191438+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:52 smithi181 bash[59345]: cluster 2023-10-09T09:15:51.194924+0000 mon.a (mon.0) 845 : cluster 0 osdmap e678: 8 total, 8 up, 8 in 2023-10-09T09:15:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:52 smithi181 bash[59345]: cluster 2023-10-09T09:15:51.200405+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:52 smithi181 bash[59345]: cluster 2023-10-09T09:15:51.355429+0000 mgr.y (mgr.34107) 279 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 209 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.2 MiB/s wr, 37 op/s 2023-10-09T09:15:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:52 smithi181 bash[59345]: cluster 2023-10-09T09:15:51.467661+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:53.209 INFO:tasks.rados.rados.0.smithi163.stdout:2414: done (4 left) 2023-10-09T09:15:53.209 INFO:tasks.rados.rados.0.smithi163.stdout:2418: done (3 left) 2023-10-09T09:15:53.209 INFO:tasks.rados.rados.0.smithi163.stdout:2419: snap_remove snap 308 2023-10-09T09:15:53.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:53 smithi163 bash[75425]: cluster 2023-10-09T09:15:51.467639+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:53.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:53 smithi163 bash[75425]: cluster 2023-10-09T09:15:52.200757+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e679 with expected crc 2023-10-09T09:15:53.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:53 smithi163 bash[75425]: cluster 2023-10-09T09:15:52.205200+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e679 with expected crc 2023-10-09T09:15:53.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:53 smithi163 bash[75425]: cluster 2023-10-09T09:15:52.208585+0000 mon.a (mon.0) 846 : cluster 0 osdmap e679: 8 total, 8 up, 8 in 2023-10-09T09:15:53.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:53 smithi163 bash[78107]: cluster 2023-10-09T09:15:51.467639+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:53.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:53 smithi163 bash[78107]: cluster 2023-10-09T09:15:52.200757+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e679 with expected crc 2023-10-09T09:15:53.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:53 smithi163 bash[78107]: cluster 2023-10-09T09:15:52.205200+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e679 with expected crc 2023-10-09T09:15:53.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:53 smithi163 bash[78107]: cluster 2023-10-09T09:15:52.208585+0000 mon.a (mon.0) 846 : cluster 0 osdmap e679: 8 total, 8 up, 8 in 2023-10-09T09:15:53.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:15:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:15:53] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.33.4" 2023-10-09T09:15:53.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:53 smithi181 bash[59345]: cluster 2023-10-09T09:15:51.467639+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e678 with expected crc 2023-10-09T09:15:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:53 smithi181 bash[59345]: cluster 2023-10-09T09:15:52.200757+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e679 with expected crc 2023-10-09T09:15:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:53 smithi181 bash[59345]: cluster 2023-10-09T09:15:52.205200+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e679 with expected crc 2023-10-09T09:15:53.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:53 smithi181 bash[59345]: cluster 2023-10-09T09:15:52.208585+0000 mon.a (mon.0) 846 : cluster 0 osdmap e679: 8 total, 8 up, 8 in 2023-10-09T09:15:54.130 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:15:54.130 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2023-10-09T09:15:54.130 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T09:15:54.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:15:54.130 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2023-10-09T09:15:54.130 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 4, 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": { 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 2 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 6, 2023-10-09T09:15:54.131 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T09:15:54.132 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:15:54.132 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:15:54.215 INFO:tasks.rados.rados.0.smithi163.stdout:2416: expect (ObjNum 863 snap 293 seq_num 863) 2023-10-09T09:15:54.341 INFO:tasks.rados.rados.0.smithi163.stdout:2417: finishing rollback tid 1 to smithi16367185-44 2023-10-09T09:15:54.341 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1708 (ObjNum 699242192 snap 21875 seq_num 699314208) dirty dne 2023-10-09T09:15:54.341 INFO:tasks.rados.rados.0.smithi163.stdout:2415: finishing write tid 3 to smithi16367185-14 2023-10-09T09:15:54.341 INFO:tasks.rados.rados.0.smithi163.stdout:2415: finishing write tid 4 to smithi16367185-14 2023-10-09T09:15:54.342 INFO:tasks.rados.rados.0.smithi163.stdout:2415: finishing write tid 5 to smithi16367185-14 2023-10-09T09:15:54.342 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1083 (ObjNum 909 snap 311 seq_num 909) dirty exists 2023-10-09T09:15:54.342 INFO:tasks.rados.rados.0.smithi163.stdout:2415: left oid 14 (ObjNum 909 snap 311 seq_num 909) 2023-10-09T09:15:54.342 INFO:tasks.rados.rados.0.smithi163.stdout:2415: done (3 left) 2023-10-09T09:15:54.342 INFO:tasks.rados.rados.0.smithi163.stdout:2416: done (2 left) 2023-10-09T09:15:54.342 INFO:tasks.rados.rados.0.smithi163.stdout:2417: done (1 left) 2023-10-09T09:15:54.342 INFO:tasks.rados.rados.0.smithi163.stdout:2419: done (0 left) 2023-10-09T09:15:54.342 INFO:tasks.rados.rados.0.smithi163.stdout:2420: rollback oid 20 current snap is 311 2023-10-09T09:15:54.342 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 309 2023-10-09T09:15:54.343 INFO:tasks.rados.rados.0.smithi163.stdout:2421: snap_create 2023-10-09T09:15:54.348 INFO:tasks.rados.rados.0.smithi163.stdout:2420: finishing rollback tid 0 to smithi16367185-20 2023-10-09T09:15:54.348 INFO:tasks.rados.rados.0.smithi163.stdout:2420: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:15:54.348 INFO:tasks.rados.rados.0.smithi163.stdout:2420: finishing rollback tid 2 to smithi16367185-20 2023-10-09T09:15:54.348 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 324 (ObjNum 807 snap 279 seq_num 807) dirty exists 2023-10-09T09:15:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:54 smithi163 bash[75425]: cluster 2023-10-09T09:15:53.205753+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e680 with expected crc 2023-10-09T09:15:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:54 smithi163 bash[75425]: cluster 2023-10-09T09:15:53.209010+0000 mon.a (mon.0) 847 : cluster 0 osdmap e680: 8 total, 8 up, 8 in 2023-10-09T09:15:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:54 smithi163 bash[75425]: cluster 2023-10-09T09:15:53.356009+0000 mgr.y (mgr.34107) 280 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.1 MiB/s wr, 36 op/s 2023-10-09T09:15:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:54 smithi163 bash[75425]: audit 2023-10-09T09:15:54.129696+0000 mon.a (mon.0) 848 : audit 0 from='client.? 172.21.15.163:0/2075471393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:54.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:54 smithi163 bash[75425]: cluster 2023-10-09T09:15:54.212850+0000 mon.a (mon.0) 849 : cluster 0 osdmap e681: 8 total, 8 up, 8 in 2023-10-09T09:15:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:54 smithi163 bash[78107]: cluster 2023-10-09T09:15:53.205753+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e680 with expected crc 2023-10-09T09:15:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:54 smithi163 bash[78107]: cluster 2023-10-09T09:15:53.209010+0000 mon.a (mon.0) 847 : cluster 0 osdmap e680: 8 total, 8 up, 8 in 2023-10-09T09:15:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:54 smithi163 bash[78107]: cluster 2023-10-09T09:15:53.356009+0000 mgr.y (mgr.34107) 280 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.1 MiB/s wr, 36 op/s 2023-10-09T09:15:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:54 smithi163 bash[78107]: audit 2023-10-09T09:15:54.129696+0000 mon.a (mon.0) 848 : audit 0 from='client.? 172.21.15.163:0/2075471393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:54 smithi163 bash[78107]: cluster 2023-10-09T09:15:54.212850+0000 mon.a (mon.0) 849 : cluster 0 osdmap e681: 8 total, 8 up, 8 in 2023-10-09T09:15:54.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:54 smithi181 bash[59345]: cluster 2023-10-09T09:15:53.205753+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e680 with expected crc 2023-10-09T09:15:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:54 smithi181 bash[59345]: cluster 2023-10-09T09:15:53.209010+0000 mon.a (mon.0) 847 : cluster 0 osdmap e680: 8 total, 8 up, 8 in 2023-10-09T09:15:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:54 smithi181 bash[59345]: cluster 2023-10-09T09:15:53.356009+0000 mgr.y (mgr.34107) 280 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.1 MiB/s wr, 36 op/s 2023-10-09T09:15:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:54 smithi181 bash[59345]: audit 2023-10-09T09:15:54.129696+0000 mon.a (mon.0) 848 : audit 0 from='client.? 172.21.15.163:0/2075471393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:15:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:54 smithi181 bash[59345]: cluster 2023-10-09T09:15:54.212850+0000 mon.a (mon.0) 849 : cluster 0 osdmap e681: 8 total, 8 up, 8 in 2023-10-09T09:15:54.809 DEBUG:teuthology.parallel:result is None 2023-10-09T09:15:55.148 INFO:tasks.rados.rados.0.smithi163.stdout:2420: done (1 left) 2023-10-09T09:15:55.149 INFO:tasks.rados.rados.0.smithi163.stdout:2421: done (0 left) 2023-10-09T09:15:55.149 INFO:tasks.rados.rados.0.smithi163.stdout:2422: snap_create 2023-10-09T09:15:55.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:55 smithi181 bash[59345]: cluster 2023-10-09T09:15:54.207661+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:55 smithi181 bash[59345]: cluster 2023-10-09T09:15:54.211519+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:55 smithi181 bash[59345]: cluster 2023-10-09T09:15:54.214518+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:55 smithi181 bash[59345]: cluster 2023-10-09T09:15:54.246399+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:55 smithi181 bash[59345]: cluster 2023-10-09T09:15:55.148112+0000 mon.a (mon.0) 850 : cluster 0 osdmap e682: 8 total, 8 up, 8 in 2023-10-09T09:15:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:55 smithi163 bash[75425]: cluster 2023-10-09T09:15:54.207661+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:55 smithi163 bash[75425]: cluster 2023-10-09T09:15:54.211519+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:55 smithi163 bash[75425]: cluster 2023-10-09T09:15:54.214518+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:55 smithi163 bash[75425]: cluster 2023-10-09T09:15:54.246399+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:55 smithi163 bash[75425]: cluster 2023-10-09T09:15:55.148112+0000 mon.a (mon.0) 850 : cluster 0 osdmap e682: 8 total, 8 up, 8 in 2023-10-09T09:15:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:55 smithi163 bash[78107]: cluster 2023-10-09T09:15:54.207661+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:55 smithi163 bash[78107]: cluster 2023-10-09T09:15:54.211519+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:55 smithi163 bash[78107]: cluster 2023-10-09T09:15:54.214518+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:55 smithi163 bash[78107]: cluster 2023-10-09T09:15:54.246399+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e681 with expected crc 2023-10-09T09:15:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:55 smithi163 bash[78107]: cluster 2023-10-09T09:15:55.148112+0000 mon.a (mon.0) 850 : cluster 0 osdmap e682: 8 total, 8 up, 8 in 2023-10-09T09:15:56.151 INFO:tasks.rados.rados.0.smithi163.stdout:2422: done (0 left) 2023-10-09T09:15:56.151 INFO:tasks.rados.rados.0.smithi163.stdout:2423: read oid 23 snap -1 2023-10-09T09:15:56.151 INFO:tasks.rados.rados.0.smithi163.stdout:2423: expect (ObjNum 889 snap 309 seq_num 889) 2023-10-09T09:15:56.151 INFO:tasks.rados.rados.0.smithi163.stdout:2424: rollback oid 29 current snap is 313 2023-10-09T09:15:56.151 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 29 to 303 2023-10-09T09:15:56.151 INFO:tasks.rados.rados.0.smithi163.stdout:2425: write oid 33 current snap is 313 2023-10-09T09:15:56.151 INFO:tasks.rados.rados.0.smithi163.stdout:2425: seq_num 910 ranges {494370=722597,1917908=447981,3007041=287329} 2023-10-09T09:15:56.159 INFO:tasks.rados.rados.0.smithi163.stdout:2425: writing smithi16367185-33 from 494370 to 1216967 tid 1 2023-10-09T09:15:56.167 INFO:tasks.rados.rados.0.smithi163.stdout:2425: writing smithi16367185-33 from 1917908 to 2365889 tid 2 2023-10-09T09:15:56.173 INFO:tasks.rados.rados.0.smithi163.stdout:2425: writing smithi16367185-33 from 3007041 to 3294370 tid 3 2023-10-09T09:15:56.174 INFO:tasks.rados.rados.0.smithi163.stdout:2426: write oid 3 current snap is 313 2023-10-09T09:15:56.174 INFO:tasks.rados.rados.0.smithi163.stdout:2426: seq_num 911 ranges {511177=797846,1976390=591639,3034464=276713} 2023-10-09T09:15:56.181 INFO:tasks.rados.rados.0.smithi163.stdout:2426: writing smithi16367185-3 from 511177 to 1309023 tid 1 2023-10-09T09:15:56.189 INFO:tasks.rados.rados.0.smithi163.stdout:2426: writing smithi16367185-3 from 1976390 to 2568029 tid 2 2023-10-09T09:15:56.194 INFO:tasks.rados.rados.0.smithi163.stdout:2426: writing smithi16367185-3 from 3034464 to 3311177 tid 3 2023-10-09T09:15:56.194 INFO:tasks.rados.rados.0.smithi163.stdout:2427: rollback oid 36 current snap is 313 2023-10-09T09:15:56.194 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 36 to 309 2023-10-09T09:15:56.194 INFO:tasks.rados.rados.0.smithi163.stdout:2424: finishing rollback tid 0 to smithi16367185-29 2023-10-09T09:15:56.194 INFO:tasks.rados.rados.0.smithi163.stdout:2424: finishing rollback tid 1 to smithi16367185-29 2023-10-09T09:15:56.194 INFO:tasks.rados.rados.0.smithi163.stdout:2424: finishing rollback tid 2 to smithi16367185-29 2023-10-09T09:15:56.194 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1166 (ObjNum 853 snap 289 seq_num 853) dirty exists 2023-10-09T09:15:56.194 INFO:tasks.rados.rados.0.smithi163.stdout:2423: expect (ObjNum 889 snap 309 seq_num 889) 2023-10-09T09:15:56.390 INFO:tasks.rados.rados.0.smithi163.stdout:2425: finishing write tid 1 to smithi16367185-33 2023-10-09T09:15:56.390 INFO:tasks.rados.rados.0.smithi163.stdout:2425: finishing write tid 2 to smithi16367185-33 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:2425: finishing write tid 3 to smithi16367185-33 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:2425: finishing write tid 4 to smithi16367185-33 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:2425: finishing write tid 5 to smithi16367185-33 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1170 (ObjNum 910 snap 313 seq_num 910) dirty exists 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:2425: left oid 33 (ObjNum 910 snap 313 seq_num 910) 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:2426: finishing write tid 1 to smithi16367185-3 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:2426: finishing write tid 2 to smithi16367185-3 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:2426: finishing write tid 3 to smithi16367185-3 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:2426: finishing write tid 4 to smithi16367185-3 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:2426: finishing write tid 5 to smithi16367185-3 2023-10-09T09:15:56.391 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 622 (ObjNum 911 snap 313 seq_num 911) dirty exists 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:2426: left oid 3 (ObjNum 911 snap 313 seq_num 911) 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:2427: finishing rollback tid 0 to smithi16367185-36 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:2427: finishing rollback tid 1 to smithi16367185-36 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:2427: finishing rollback tid 2 to smithi16367185-36 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 626 (ObjNum 884 snap 308 seq_num 884) dirty exists 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:2423: done (4 left) 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:2424: done (3 left) 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:2425: done (2 left) 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:2426: done (1 left) 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:2427: done (0 left) 2023-10-09T09:15:56.392 INFO:tasks.rados.rados.0.smithi163.stdout:2428: snap_create 2023-10-09T09:15:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:56 smithi181 bash[59345]: cluster 2023-10-09T09:15:55.143318+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e682 with expected crc 2023-10-09T09:15:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:56 smithi181 bash[59345]: cluster 2023-10-09T09:15:55.143369+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e682 with expected crc 2023-10-09T09:15:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:56 smithi181 bash[59345]: cluster 2023-10-09T09:15:55.236135+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e682 with expected crc 2023-10-09T09:15:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:56 smithi181 bash[59345]: audit 2023-10-09T09:15:55.344079+0000 mon.a (mon.0) 851 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:56 smithi181 bash[59345]: cluster 2023-10-09T09:15:55.356623+0000 mgr.y (mgr.34107) 281 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 729 KiB/s wr, 6 op/s 2023-10-09T09:15:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:56 smithi181 bash[59345]: cluster 2023-10-09T09:15:56.150543+0000 mon.a (mon.0) 852 : cluster 0 osdmap e683: 8 total, 8 up, 8 in 2023-10-09T09:15:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[78107]: cluster 2023-10-09T09:15:55.143318+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e682 with expected crc 2023-10-09T09:15:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[78107]: cluster 2023-10-09T09:15:55.143369+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e682 with expected crc 2023-10-09T09:15:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[78107]: cluster 2023-10-09T09:15:55.236135+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e682 with expected crc 2023-10-09T09:15:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[78107]: audit 2023-10-09T09:15:55.344079+0000 mon.a (mon.0) 851 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[78107]: cluster 2023-10-09T09:15:55.356623+0000 mgr.y (mgr.34107) 281 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 729 KiB/s wr, 6 op/s 2023-10-09T09:15:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[78107]: cluster 2023-10-09T09:15:56.150543+0000 mon.a (mon.0) 852 : cluster 0 osdmap e683: 8 total, 8 up, 8 in 2023-10-09T09:15:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[75425]: cluster 2023-10-09T09:15:55.143318+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e682 with expected crc 2023-10-09T09:15:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[75425]: cluster 2023-10-09T09:15:55.143369+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e682 with expected crc 2023-10-09T09:15:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[75425]: cluster 2023-10-09T09:15:55.236135+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e682 with expected crc 2023-10-09T09:15:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[75425]: audit 2023-10-09T09:15:55.344079+0000 mon.a (mon.0) 851 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:15:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[75425]: cluster 2023-10-09T09:15:55.356623+0000 mgr.y (mgr.34107) 281 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 729 KiB/s wr, 6 op/s 2023-10-09T09:15:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:56 smithi163 bash[75425]: cluster 2023-10-09T09:15:56.150543+0000 mon.a (mon.0) 852 : cluster 0 osdmap e683: 8 total, 8 up, 8 in 2023-10-09T09:15:57.248 INFO:tasks.rados.rados.0.smithi163.stdout:2428: done (0 left) 2023-10-09T09:15:57.248 INFO:tasks.rados.rados.0.smithi163.stdout:2429: read oid 24 snap -1 2023-10-09T09:15:57.248 INFO:tasks.rados.rados.0.smithi163.stdout:2429: expect deleted 2023-10-09T09:15:57.248 INFO:tasks.rados.rados.0.smithi163.stdout:2430: delete oid 2 current snap is 314 2023-10-09T09:15:57.259 INFO:tasks.rados.rados.0.smithi163.stdout:2429: done (1 left) 2023-10-09T09:15:57.259 INFO:tasks.rados.rados.0.smithi163.stdout:2430: done (0 left) 2023-10-09T09:15:57.259 INFO:tasks.rados.rados.0.smithi163.stdout:2431: delete oid 40 current snap is 314 2023-10-09T09:15:57.262 INFO:tasks.rados.rados.0.smithi163.stdout:2431: done (0 left) 2023-10-09T09:15:57.262 INFO:tasks.rados.rados.0.smithi163.stdout:2432: read oid 48 snap -1 2023-10-09T09:15:57.262 INFO:tasks.rados.rados.0.smithi163.stdout:2432: expect (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:15:57.262 INFO:tasks.rados.rados.0.smithi163.stdout:2433: write oid 41 current snap is 314 2023-10-09T09:15:57.262 INFO:tasks.rados.rados.0.smithi163.stdout:2433: seq_num 914 ranges {561598=739946,1751836=455319,2716733=644865} 2023-10-09T09:15:57.272 INFO:tasks.rados.rados.0.smithi163.stdout:2433: writing smithi16367185-41 from 561598 to 1301544 tid 1 2023-10-09T09:15:57.278 INFO:tasks.rados.rados.0.smithi163.stdout:2433: writing smithi16367185-41 from 1751836 to 2207155 tid 2 2023-10-09T09:15:57.285 INFO:tasks.rados.rados.0.smithi163.stdout:2433: writing smithi16367185-41 from 2716733 to 3361598 tid 3 2023-10-09T09:15:57.285 INFO:tasks.rados.rados.0.smithi163.stdout:2434: write oid 3 current snap is 314 2023-10-09T09:15:57.285 INFO:tasks.rados.rados.0.smithi163.stdout:2434: seq_num 915 ranges {578405=415195,1410318=598977,2744156=584480} 2023-10-09T09:15:57.290 INFO:tasks.rados.rados.0.smithi163.stdout:2434: writing smithi16367185-3 from 578405 to 993600 tid 1 2023-10-09T09:15:57.296 INFO:tasks.rados.rados.0.smithi163.stdout:2434: writing smithi16367185-3 from 1410318 to 2009295 tid 2 2023-10-09T09:15:57.304 INFO:tasks.rados.rados.0.smithi163.stdout:2434: writing smithi16367185-3 from 2744156 to 3328636 tid 3 2023-10-09T09:15:57.304 INFO:tasks.rados.rados.0.smithi163.stdout:2435: read oid 33 snap -1 2023-10-09T09:15:57.304 INFO:tasks.rados.rados.0.smithi163.stdout:2435: expect (ObjNum 910 snap 313 seq_num 910) 2023-10-09T09:15:57.305 INFO:tasks.rados.rados.0.smithi163.stdout:2436: read oid 32 snap -1 2023-10-09T09:15:57.305 INFO:tasks.rados.rados.0.smithi163.stdout:2436: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:15:57.305 INFO:tasks.rados.rados.0.smithi163.stdout:2437: rollback oid 43 current snap is 314 2023-10-09T09:15:57.305 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 286 2023-10-09T09:15:57.305 INFO:tasks.rados.rados.0.smithi163.stdout:2432: expect (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:15:57.479 INFO:tasks.rados.rados.0.smithi163.stdout:2433: finishing write tid 1 to smithi16367185-41 2023-10-09T09:15:57.480 INFO:tasks.rados.rados.0.smithi163.stdout:2433: finishing write tid 2 to smithi16367185-41 2023-10-09T09:15:57.480 INFO:tasks.rados.rados.0.smithi163.stdout:2432: done (5 left) 2023-10-09T09:15:57.480 INFO:tasks.rados.rados.0.smithi163.stdout:2438: delete oid 50 current snap is 314 2023-10-09T09:15:57.480 INFO:tasks.rados.rados.0.smithi163.stdout:2433: finishing write tid 3 to smithi16367185-41 2023-10-09T09:15:57.480 INFO:tasks.rados.rados.0.smithi163.stdout:2433: finishing write tid 4 to smithi16367185-41 2023-10-09T09:15:57.480 INFO:tasks.rados.rados.0.smithi163.stdout:2433: finishing write tid 5 to smithi16367185-41 2023-10-09T09:15:57.480 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 171 (ObjNum 914 snap 314 seq_num 914) dirty exists 2023-10-09T09:15:57.480 INFO:tasks.rados.rados.0.smithi163.stdout:2433: left oid 41 (ObjNum 914 snap 314 seq_num 914) 2023-10-09T09:15:57.480 INFO:tasks.rados.rados.0.smithi163.stdout:2434: finishing write tid 1 to smithi16367185-3 2023-10-09T09:15:57.480 INFO:tasks.rados.rados.0.smithi163.stdout:2434: finishing write tid 2 to smithi16367185-3 2023-10-09T09:15:57.481 INFO:tasks.rados.rados.0.smithi163.stdout:2437: finishing rollback tid 0 to smithi16367185-43 2023-10-09T09:15:57.481 INFO:tasks.rados.rados.0.smithi163.stdout:2437: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:15:57.481 INFO:tasks.rados.rados.0.smithi163.stdout:2437: finishing rollback tid 2 to smithi16367185-43 2023-10-09T09:15:57.481 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 1093 (ObjNum 829 snap 286 seq_num 829) dirty exists 2023-10-09T09:15:57.481 INFO:tasks.rados.rados.0.smithi163.stdout:2435: expect (ObjNum 910 snap 313 seq_num 910) 2023-10-09T09:15:57.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:57 smithi181 bash[59345]: cluster 2023-10-09T09:15:56.145130+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e683 with expected crc 2023-10-09T09:15:57.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:57 smithi181 bash[59345]: cluster 2023-10-09T09:15:56.183745+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e683 with expected crc 2023-10-09T09:15:57.591 INFO:tasks.rados.rados.0.smithi163.stdout:2434: finishing write tid 3 to smithi16367185-3 2023-10-09T09:15:57.591 INFO:tasks.rados.rados.0.smithi163.stdout:2434: finishing write tid 4 to smithi16367185-3 2023-10-09T09:15:57.591 INFO:tasks.rados.rados.0.smithi163.stdout:2434: finishing write tid 5 to smithi16367185-3 2023-10-09T09:15:57.591 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 631 (ObjNum 915 snap 314 seq_num 915) dirty exists 2023-10-09T09:15:57.591 INFO:tasks.rados.rados.0.smithi163.stdout:2434: left oid 3 (ObjNum 915 snap 314 seq_num 915) 2023-10-09T09:15:57.592 INFO:tasks.rados.rados.0.smithi163.stdout:2436: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:15:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:57 smithi163 bash[75425]: cluster 2023-10-09T09:15:56.145130+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e683 with expected crc 2023-10-09T09:15:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:57 smithi163 bash[75425]: cluster 2023-10-09T09:15:56.183745+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e683 with expected crc 2023-10-09T09:15:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:57 smithi163 bash[78107]: cluster 2023-10-09T09:15:56.145130+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e683 with expected crc 2023-10-09T09:15:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:57 smithi163 bash[78107]: cluster 2023-10-09T09:15:56.183745+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e683 with expected crc 2023-10-09T09:15:57.780 INFO:tasks.rados.rados.0.smithi163.stdout:2433: done (5 left) 2023-10-09T09:15:57.780 INFO:tasks.rados.rados.0.smithi163.stdout:2434: done (4 left) 2023-10-09T09:15:57.780 INFO:tasks.rados.rados.0.smithi163.stdout:2435: done (3 left) 2023-10-09T09:15:57.780 INFO:tasks.rados.rados.0.smithi163.stdout:2436: done (2 left) 2023-10-09T09:15:57.780 INFO:tasks.rados.rados.0.smithi163.stdout:2437: done (1 left) 2023-10-09T09:15:57.780 INFO:tasks.rados.rados.0.smithi163.stdout:2438: done (0 left) 2023-10-09T09:15:57.780 INFO:tasks.rados.rados.0.smithi163.stdout:2439: read oid 42 snap -1 2023-10-09T09:15:57.780 INFO:tasks.rados.rados.0.smithi163.stdout:2439: expect deleted 2023-10-09T09:15:57.781 INFO:tasks.rados.rados.0.smithi163.stdout:2440: read oid 44 snap 313 2023-10-09T09:15:57.781 INFO:tasks.rados.rados.0.smithi163.stdout:2440: expect deleted 2023-10-09T09:15:57.781 INFO:tasks.rados.rados.0.smithi163.stdout:2441: snap_create 2023-10-09T09:15:58.270 INFO:tasks.rados.rados.0.smithi163.stdout:2439: done (2 left) 2023-10-09T09:15:58.271 INFO:tasks.rados.rados.0.smithi163.stdout:2440: done (1 left) 2023-10-09T09:15:58.271 INFO:tasks.rados.rados.0.smithi163.stdout:2441: done (0 left) 2023-10-09T09:15:58.271 INFO:tasks.rados.rados.0.smithi163.stdout:2442: write oid 17 current snap is 315 2023-10-09T09:15:58.271 INFO:tasks.rados.rados.0.smithi163.stdout:2442: seq_num 917 ranges {612019=565693,1810929=602646,3082649=329370} 2023-10-09T09:15:58.278 INFO:tasks.rados.rados.0.smithi163.stdout:2442: writing smithi16367185-17 from 612019 to 1177712 tid 1 2023-10-09T09:15:58.285 INFO:tasks.rados.rados.0.smithi163.stdout:2442: writing smithi16367185-17 from 1810929 to 2413575 tid 2 2023-10-09T09:15:58.291 INFO:tasks.rados.rados.0.smithi163.stdout:2442: writing smithi16367185-17 from 3082649 to 3412019 tid 3 2023-10-09T09:15:58.291 INFO:tasks.rados.rados.0.smithi163.stdout:2443: read oid 39 snap -1 2023-10-09T09:15:58.291 INFO:tasks.rados.rados.0.smithi163.stdout:2443: expect (ObjNum 845 snap 289 seq_num 845) 2023-10-09T09:15:58.292 INFO:tasks.rados.rados.0.smithi163.stdout:2444: read oid 42 snap -1 2023-10-09T09:15:58.292 INFO:tasks.rados.rados.0.smithi163.stdout:2444: expect deleted 2023-10-09T09:15:58.292 INFO:tasks.rados.rados.0.smithi163.stdout:2445: delete oid 1 current snap is 315 2023-10-09T09:15:58.292 INFO:tasks.rados.rados.0.smithi163.stdout:2442: finishing write tid 1 to smithi16367185-17 2023-10-09T09:15:58.292 INFO:tasks.rados.rados.0.smithi163.stdout:2442: finishing write tid 2 to smithi16367185-17 2023-10-09T09:15:58.295 INFO:tasks.rados.rados.0.smithi163.stdout:2445: done (3 left) 2023-10-09T09:15:58.295 INFO:tasks.rados.rados.0.smithi163.stdout:2446: delete oid 13 current snap is 315 2023-10-09T09:15:58.297 INFO:tasks.rados.rados.0.smithi163.stdout:2442: finishing write tid 3 to smithi16367185-17 2023-10-09T09:15:58.298 INFO:tasks.rados.rados.0.smithi163.stdout:2442: finishing write tid 4 to smithi16367185-17 2023-10-09T09:15:58.298 INFO:tasks.rados.rados.0.smithi163.stdout:2442: finishing write tid 5 to smithi16367185-17 2023-10-09T09:15:58.298 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1204 (ObjNum 917 snap 315 seq_num 917) dirty exists 2023-10-09T09:15:58.298 INFO:tasks.rados.rados.0.smithi163.stdout:2442: left oid 17 (ObjNum 917 snap 315 seq_num 917) 2023-10-09T09:15:58.300 INFO:tasks.rados.rados.0.smithi163.stdout:2443: expect (ObjNum 845 snap 289 seq_num 845) 2023-10-09T09:15:58.383 INFO:tasks.rados.rados.0.smithi163.stdout:2442: done (3 left) 2023-10-09T09:15:58.384 INFO:tasks.rados.rados.0.smithi163.stdout:2443: done (2 left) 2023-10-09T09:15:58.384 INFO:tasks.rados.rados.0.smithi163.stdout:2444: done (1 left) 2023-10-09T09:15:58.384 INFO:tasks.rados.rados.0.smithi163.stdout:2446: done (0 left) 2023-10-09T09:15:58.384 INFO:tasks.rados.rados.0.smithi163.stdout:2447: snap_remove snap 282 2023-10-09T09:15:58.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:58 smithi181 bash[59345]: cluster 2023-10-09T09:15:57.242124+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e684 with expected crc 2023-10-09T09:15:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:58 smithi181 bash[59345]: cluster 2023-10-09T09:15:57.244764+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e684 with expected crc 2023-10-09T09:15:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:58 smithi181 bash[59345]: cluster 2023-10-09T09:15:57.247490+0000 mon.a (mon.0) 853 : cluster 0 osdmap e684: 8 total, 8 up, 8 in 2023-10-09T09:15:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:58 smithi181 bash[59345]: cluster 2023-10-09T09:15:57.248313+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e684 with expected crc 2023-10-09T09:15:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:58 smithi181 bash[59345]: cluster 2023-10-09T09:15:57.357355+0000 mgr.y (mgr.34107) 282 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 214 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2023-10-09T09:15:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:58 smithi163 bash[75425]: cluster 2023-10-09T09:15:57.242124+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e684 with expected crc 2023-10-09T09:15:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:58 smithi163 bash[75425]: cluster 2023-10-09T09:15:57.244764+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e684 with expected crc 2023-10-09T09:15:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:58 smithi163 bash[75425]: cluster 2023-10-09T09:15:57.247490+0000 mon.a (mon.0) 853 : cluster 0 osdmap e684: 8 total, 8 up, 8 in 2023-10-09T09:15:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:58 smithi163 bash[75425]: cluster 2023-10-09T09:15:57.248313+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e684 with expected crc 2023-10-09T09:15:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:58 smithi163 bash[75425]: cluster 2023-10-09T09:15:57.357355+0000 mgr.y (mgr.34107) 282 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 214 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2023-10-09T09:15:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:58 smithi163 bash[78107]: cluster 2023-10-09T09:15:57.242124+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e684 with expected crc 2023-10-09T09:15:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:58 smithi163 bash[78107]: cluster 2023-10-09T09:15:57.244764+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e684 with expected crc 2023-10-09T09:15:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:58 smithi163 bash[78107]: cluster 2023-10-09T09:15:57.247490+0000 mon.a (mon.0) 853 : cluster 0 osdmap e684: 8 total, 8 up, 8 in 2023-10-09T09:15:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:58 smithi163 bash[78107]: cluster 2023-10-09T09:15:57.248313+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e684 with expected crc 2023-10-09T09:15:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:58 smithi163 bash[78107]: cluster 2023-10-09T09:15:57.357355+0000 mgr.y (mgr.34107) 282 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 214 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2023-10-09T09:15:59.275 INFO:tasks.rados.rados.0.smithi163.stdout:2447: done (0 left) 2023-10-09T09:15:59.275 INFO:tasks.rados.rados.0.smithi163.stdout:2448: snap_remove snap 310 2023-10-09T09:15:59.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:59 smithi181 bash[59345]: cluster 2023-10-09T09:15:58.263763+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e685 with expected crc 2023-10-09T09:15:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:59 smithi181 bash[59345]: cluster 2023-10-09T09:15:58.270549+0000 mon.a (mon.0) 854 : cluster 0 osdmap e685: 8 total, 8 up, 8 in 2023-10-09T09:15:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:59 smithi181 bash[59345]: cluster 2023-10-09T09:15:58.279472+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e685 with expected crc 2023-10-09T09:15:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:15:59 smithi181 bash[59345]: cluster 2023-10-09T09:15:58.296673+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e685 with expected crc 2023-10-09T09:15:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:59 smithi163 bash[75425]: cluster 2023-10-09T09:15:58.263763+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e685 with expected crc 2023-10-09T09:15:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:59 smithi163 bash[75425]: cluster 2023-10-09T09:15:58.270549+0000 mon.a (mon.0) 854 : cluster 0 osdmap e685: 8 total, 8 up, 8 in 2023-10-09T09:15:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:59 smithi163 bash[75425]: cluster 2023-10-09T09:15:58.279472+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e685 with expected crc 2023-10-09T09:15:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:15:59 smithi163 bash[75425]: cluster 2023-10-09T09:15:58.296673+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e685 with expected crc 2023-10-09T09:15:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:59 smithi163 bash[78107]: cluster 2023-10-09T09:15:58.263763+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e685 with expected crc 2023-10-09T09:15:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:59 smithi163 bash[78107]: cluster 2023-10-09T09:15:58.270549+0000 mon.a (mon.0) 854 : cluster 0 osdmap e685: 8 total, 8 up, 8 in 2023-10-09T09:15:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:59 smithi163 bash[78107]: cluster 2023-10-09T09:15:58.279472+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e685 with expected crc 2023-10-09T09:15:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:15:59 smithi163 bash[78107]: cluster 2023-10-09T09:15:58.296673+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e685 with expected crc 2023-10-09T09:16:00.153 INFO:tasks.rados.rados.0.smithi163.stdout:2448: done (0 left) 2023-10-09T09:16:00.153 INFO:tasks.rados.rados.0.smithi163.stdout:2449: read oid 43 snap -1 2023-10-09T09:16:00.153 INFO:tasks.rados.rados.0.smithi163.stdout:2449: expect (ObjNum 829 snap 286 seq_num 829) 2023-10-09T09:16:00.153 INFO:tasks.rados.rados.0.smithi163.stdout:2450: read oid 19 snap -1 2023-10-09T09:16:00.154 INFO:tasks.rados.rados.0.smithi163.stdout:2450: expect (ObjNum 908 snap 311 seq_num 908) 2023-10-09T09:16:00.154 INFO:tasks.rados.rados.0.smithi163.stdout:2451: read oid 29 snap 309 2023-10-09T09:16:00.154 INFO:tasks.rados.rados.0.smithi163.stdout:2451: expect (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:16:00.154 INFO:tasks.rados.rados.0.smithi163.stdout:2452: write oid 14 current snap is 315 2023-10-09T09:16:00.154 INFO:tasks.rados.rados.0.smithi163.stdout:2452: seq_num 920 ranges {662440=507793,1702728=749973,2881271=557193} 2023-10-09T09:16:00.161 INFO:tasks.rados.rados.0.smithi163.stdout:2452: writing smithi16367185-14 from 662440 to 1170233 tid 1 2023-10-09T09:16:00.170 INFO:tasks.rados.rados.0.smithi163.stdout:2452: writing smithi16367185-14 from 1702728 to 2452701 tid 2 2023-10-09T09:16:00.179 INFO:tasks.rados.rados.0.smithi163.stdout:2452: writing smithi16367185-14 from 2881271 to 3438464 tid 3 2023-10-09T09:16:00.179 INFO:tasks.rados.rados.0.smithi163.stdout:2453: snap_remove snap 305 2023-10-09T09:16:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:00 smithi181 bash[59345]: cluster 2023-10-09T09:15:59.271537+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e686 with expected crc 2023-10-09T09:16:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:00 smithi181 bash[59345]: cluster 2023-10-09T09:15:59.275341+0000 mon.a (mon.0) 855 : cluster 0 osdmap e686: 8 total, 8 up, 8 in 2023-10-09T09:16:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:00 smithi181 bash[59345]: cluster 2023-10-09T09:15:59.276134+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e686 with expected crc 2023-10-09T09:16:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:00 smithi181 bash[59345]: cluster 2023-10-09T09:15:59.278834+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e686 with expected crc 2023-10-09T09:16:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:00 smithi181 bash[59345]: cluster 2023-10-09T09:15:59.358197+0000 mgr.y (mgr.34107) 283 : cluster 0 pgmap v582: 89 pgs: 89 active+clean; 214 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 782 KiB/s wr, 5 op/s 2023-10-09T09:16:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:00 smithi181 bash[59345]: cluster 2023-10-09T09:16:00.151932+0000 mon.a (mon.0) 856 : cluster 0 osdmap e687: 8 total, 8 up, 8 in 2023-10-09T09:16:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[78107]: cluster 2023-10-09T09:15:59.271537+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e686 with expected crc 2023-10-09T09:16:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[78107]: cluster 2023-10-09T09:15:59.275341+0000 mon.a (mon.0) 855 : cluster 0 osdmap e686: 8 total, 8 up, 8 in 2023-10-09T09:16:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[78107]: cluster 2023-10-09T09:15:59.276134+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e686 with expected crc 2023-10-09T09:16:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[78107]: cluster 2023-10-09T09:15:59.278834+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e686 with expected crc 2023-10-09T09:16:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[78107]: cluster 2023-10-09T09:15:59.358197+0000 mgr.y (mgr.34107) 283 : cluster 0 pgmap v582: 89 pgs: 89 active+clean; 214 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 782 KiB/s wr, 5 op/s 2023-10-09T09:16:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[78107]: cluster 2023-10-09T09:16:00.151932+0000 mon.a (mon.0) 856 : cluster 0 osdmap e687: 8 total, 8 up, 8 in 2023-10-09T09:16:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[75425]: cluster 2023-10-09T09:15:59.271537+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e686 with expected crc 2023-10-09T09:16:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[75425]: cluster 2023-10-09T09:15:59.275341+0000 mon.a (mon.0) 855 : cluster 0 osdmap e686: 8 total, 8 up, 8 in 2023-10-09T09:16:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[75425]: cluster 2023-10-09T09:15:59.276134+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e686 with expected crc 2023-10-09T09:16:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[75425]: cluster 2023-10-09T09:15:59.278834+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e686 with expected crc 2023-10-09T09:16:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[75425]: cluster 2023-10-09T09:15:59.358197+0000 mgr.y (mgr.34107) 283 : cluster 0 pgmap v582: 89 pgs: 89 active+clean; 214 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 782 KiB/s wr, 5 op/s 2023-10-09T09:16:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:00 smithi163 bash[75425]: cluster 2023-10-09T09:16:00.151932+0000 mon.a (mon.0) 856 : cluster 0 osdmap e687: 8 total, 8 up, 8 in 2023-10-09T09:16:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:2453: done (4 left) 2023-10-09T09:16:01.165 INFO:tasks.rados.rados.0.smithi163.stdout:2454: delete oid 25 current snap is 315 2023-10-09T09:16:01.166 INFO:tasks.rados.rados.0.smithi163.stdout:2451: expect (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:16:01.276 INFO:tasks.rados.rados.0.smithi163.stdout:2450: expect (ObjNum 908 snap 311 seq_num 908) 2023-10-09T09:16:01.477 INFO:tasks.rados.rados.0.smithi163.stdout:2449: expect (ObjNum 829 snap 286 seq_num 829) 2023-10-09T09:16:01.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:01 smithi181 bash[59345]: cluster 2023-10-09T09:16:00.145449+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e687 with expected crc 2023-10-09T09:16:01.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:01 smithi181 bash[59345]: cluster 2023-10-09T09:16:00.154056+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e687 with expected crc 2023-10-09T09:16:01.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:01 smithi181 bash[59345]: cluster 2023-10-09T09:16:00.167581+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e687 with expected crc 2023-10-09T09:16:01.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:01 smithi181 bash[59345]: cluster 2023-10-09T09:16:01.161757+0000 mon.a (mon.0) 857 : cluster 0 osdmap e688: 8 total, 8 up, 8 in 2023-10-09T09:16:01.711 INFO:tasks.rados.rados.0.smithi163.stdout:2452: finishing write tid 1 to smithi16367185-14 2023-10-09T09:16:01.711 INFO:tasks.rados.rados.0.smithi163.stdout:2452: finishing write tid 2 to smithi16367185-14 2023-10-09T09:16:01.711 INFO:tasks.rados.rados.0.smithi163.stdout:2452: finishing write tid 3 to smithi16367185-14 2023-10-09T09:16:01.711 INFO:tasks.rados.rados.0.smithi163.stdout:2452: finishing write tid 4 to smithi16367185-14 2023-10-09T09:16:01.711 INFO:tasks.rados.rados.0.smithi163.stdout:2452: finishing write tid 5 to smithi16367185-14 2023-10-09T09:16:01.712 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1108 (ObjNum 920 snap 315 seq_num 920) dirty exists 2023-10-09T09:16:01.712 INFO:tasks.rados.rados.0.smithi163.stdout:2452: left oid 14 (ObjNum 920 snap 315 seq_num 920) 2023-10-09T09:16:01.712 INFO:tasks.rados.rados.0.smithi163.stdout:2449: done (4 left) 2023-10-09T09:16:01.712 INFO:tasks.rados.rados.0.smithi163.stdout:2450: done (3 left) 2023-10-09T09:16:01.712 INFO:tasks.rados.rados.0.smithi163.stdout:2451: done (2 left) 2023-10-09T09:16:01.712 INFO:tasks.rados.rados.0.smithi163.stdout:2452: done (1 left) 2023-10-09T09:16:01.712 INFO:tasks.rados.rados.0.smithi163.stdout:2454: done (0 left) 2023-10-09T09:16:01.712 INFO:tasks.rados.rados.0.smithi163.stdout:2455: write oid 15 current snap is 315 2023-10-09T09:16:01.712 INFO:tasks.rados.rados.0.smithi163.stdout:2455: seq_num 922 ranges {696054=658291,2103339=753642} 2023-10-09T09:16:01.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:01 smithi163 bash[78107]: cluster 2023-10-09T09:16:00.145449+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e687 with expected crc 2023-10-09T09:16:01.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:01 smithi163 bash[78107]: cluster 2023-10-09T09:16:00.154056+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e687 with expected crc 2023-10-09T09:16:01.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:01 smithi163 bash[78107]: cluster 2023-10-09T09:16:00.167581+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e687 with expected crc 2023-10-09T09:16:01.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:01 smithi163 bash[78107]: cluster 2023-10-09T09:16:01.161757+0000 mon.a (mon.0) 857 : cluster 0 osdmap e688: 8 total, 8 up, 8 in 2023-10-09T09:16:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:01 smithi163 bash[75425]: cluster 2023-10-09T09:16:00.145449+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e687 with expected crc 2023-10-09T09:16:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:01 smithi163 bash[75425]: cluster 2023-10-09T09:16:00.154056+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e687 with expected crc 2023-10-09T09:16:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:01 smithi163 bash[75425]: cluster 2023-10-09T09:16:00.167581+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e687 with expected crc 2023-10-09T09:16:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:01 smithi163 bash[75425]: cluster 2023-10-09T09:16:01.161757+0000 mon.a (mon.0) 857 : cluster 0 osdmap e688: 8 total, 8 up, 8 in 2023-10-09T09:16:01.728 INFO:tasks.rados.rados.0.smithi163.stdout:2455: writing smithi16367185-15 from 696054 to 1354345 tid 1 2023-10-09T09:16:01.736 INFO:tasks.rados.rados.0.smithi163.stdout:2455: writing smithi16367185-15 from 2103339 to 2856981 tid 2 2023-10-09T09:16:01.737 INFO:tasks.rados.rados.0.smithi163.stdout:2456: snap_remove snap 312 2023-10-09T09:16:02.304 INFO:tasks.rados.rados.0.smithi163.stdout:2456: done (1 left) 2023-10-09T09:16:02.305 INFO:tasks.rados.rados.0.smithi163.stdout:2457: read oid 11 snap -1 2023-10-09T09:16:02.305 INFO:tasks.rados.rados.0.smithi163.stdout:2457: expect deleted 2023-10-09T09:16:02.305 INFO:tasks.rados.rados.0.smithi163.stdout:2458: delete oid 27 current snap is 315 2023-10-09T09:16:02.305 INFO:tasks.rados.rados.0.smithi163.stdout:2455: finishing write tid 1 to smithi16367185-15 2023-10-09T09:16:02.305 INFO:tasks.rados.rados.0.smithi163.stdout:2455: finishing write tid 2 to smithi16367185-15 2023-10-09T09:16:02.305 INFO:tasks.rados.rados.0.smithi163.stdout:2455: finishing write tid 3 to smithi16367185-15 2023-10-09T09:16:02.305 INFO:tasks.rados.rados.0.smithi163.stdout:2455: finishing write tid 4 to smithi16367185-15 2023-10-09T09:16:02.305 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 1222 (ObjNum 922 snap 315 seq_num 922) dirty exists 2023-10-09T09:16:02.305 INFO:tasks.rados.rados.0.smithi163.stdout:2455: left oid 15 (ObjNum 922 snap 315 seq_num 922) 2023-10-09T09:16:02.311 INFO:tasks.rados.rados.0.smithi163.stdout:2455: done (2 left) 2023-10-09T09:16:02.311 INFO:tasks.rados.rados.0.smithi163.stdout:2458: done (1 left) 2023-10-09T09:16:02.312 INFO:tasks.rados.rados.0.smithi163.stdout:2459: write oid 33 current snap is 315 2023-10-09T09:16:02.312 INFO:tasks.rados.rados.0.smithi163.stdout:2459: seq_num 924 ranges {729668=408789,1820303=757311,3274610=255058} 2023-10-09T09:16:02.318 INFO:tasks.rados.rados.0.smithi163.stdout:2459: writing smithi16367185-33 from 729668 to 1138457 tid 1 2023-10-09T09:16:02.327 INFO:tasks.rados.rados.0.smithi163.stdout:2459: writing smithi16367185-33 from 1820303 to 2577614 tid 2 2023-10-09T09:16:02.333 INFO:tasks.rados.rados.0.smithi163.stdout:2459: writing smithi16367185-33 from 3274610 to 3529668 tid 3 2023-10-09T09:16:02.333 INFO:tasks.rados.rados.0.smithi163.stdout:2460: delete oid 47 current snap is 315 2023-10-09T09:16:02.333 INFO:tasks.rados.rados.0.smithi163.stdout:2459: finishing write tid 1 to smithi16367185-33 2023-10-09T09:16:02.333 INFO:tasks.rados.rados.0.smithi163.stdout:2459: finishing write tid 2 to smithi16367185-33 2023-10-09T09:16:02.336 INFO:tasks.rados.rados.0.smithi163.stdout:2459: finishing write tid 3 to smithi16367185-33 2023-10-09T09:16:02.337 INFO:tasks.rados.rados.0.smithi163.stdout:2459: finishing write tid 4 to smithi16367185-33 2023-10-09T09:16:02.337 INFO:tasks.rados.rados.0.smithi163.stdout:2459: finishing write tid 5 to smithi16367185-33 2023-10-09T09:16:02.337 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1190 (ObjNum 924 snap 315 seq_num 924) dirty exists 2023-10-09T09:16:02.337 INFO:tasks.rados.rados.0.smithi163.stdout:2459: left oid 33 (ObjNum 924 snap 315 seq_num 924) 2023-10-09T09:16:02.338 INFO:tasks.rados.rados.0.smithi163.stdout:2457: done (2 left) 2023-10-09T09:16:02.338 INFO:tasks.rados.rados.0.smithi163.stdout:2459: done (1 left) 2023-10-09T09:16:02.338 INFO:tasks.rados.rados.0.smithi163.stdout:2460: done (0 left) 2023-10-09T09:16:02.338 INFO:tasks.rados.rados.0.smithi163.stdout:2461: read oid 2 snap -1 2023-10-09T09:16:02.338 INFO:tasks.rados.rados.0.smithi163.stdout:2461: expect deleted 2023-10-09T09:16:02.339 INFO:tasks.rados.rados.0.smithi163.stdout:2462: delete oid 35 current snap is 315 2023-10-09T09:16:02.343 INFO:tasks.rados.rados.0.smithi163.stdout:2461: done (1 left) 2023-10-09T09:16:02.343 INFO:tasks.rados.rados.0.smithi163.stdout:2462: done (0 left) 2023-10-09T09:16:02.343 INFO:tasks.rados.rados.0.smithi163.stdout:2463: delete oid 40 current snap is 315 2023-10-09T09:16:02.345 INFO:tasks.rados.rados.0.smithi163.stdout:2463: done (0 left) 2023-10-09T09:16:02.345 INFO:tasks.rados.rados.0.smithi163.stdout:2464: rollback oid 15 current snap is 315 2023-10-09T09:16:02.346 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 15 to 295 2023-10-09T09:16:02.346 INFO:tasks.rados.rados.0.smithi163.stdout:2465: delete oid 33 current snap is 315 2023-10-09T09:16:02.348 INFO:tasks.rados.rados.0.smithi163.stdout:2465: done (1 left) 2023-10-09T09:16:02.348 INFO:tasks.rados.rados.0.smithi163.stdout:2466: write oid 8 current snap is 315 2023-10-09T09:16:02.348 INFO:tasks.rados.rados.0.smithi163.stdout:2466: seq_num 929 ranges {413703=501387,1596360=624660,2895372=691347} 2023-10-09T09:16:02.358 INFO:tasks.rados.rados.0.smithi163.stdout:2466: writing smithi16367185-8 from 413703 to 915090 tid 1 2023-10-09T09:16:02.366 INFO:tasks.rados.rados.0.smithi163.stdout:2466: writing smithi16367185-8 from 1596360 to 2221020 tid 2 2023-10-09T09:16:02.375 INFO:tasks.rados.rados.0.smithi163.stdout:2466: writing smithi16367185-8 from 2895372 to 3586719 tid 3 2023-10-09T09:16:02.375 INFO:tasks.rados.rados.0.smithi163.stdout:2467: delete oid 5 current snap is 315 2023-10-09T09:16:02.375 INFO:tasks.rados.rados.0.smithi163.stdout:2464: finishing rollback tid 0 to smithi16367185-15 2023-10-09T09:16:02.375 INFO:tasks.rados.rados.0.smithi163.stdout:2464: finishing rollback tid 1 to smithi16367185-15 2023-10-09T09:16:02.375 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 1229 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:16:02.375 INFO:tasks.rados.rados.0.smithi163.stdout:2466: finishing write tid 1 to smithi16367185-8 2023-10-09T09:16:02.375 INFO:tasks.rados.rados.0.smithi163.stdout:2466: finishing write tid 2 to smithi16367185-8 2023-10-09T09:16:02.379 INFO:tasks.rados.rados.0.smithi163.stdout:2464: done (2 left) 2023-10-09T09:16:02.379 INFO:tasks.rados.rados.0.smithi163.stdout:2467: done (1 left) 2023-10-09T09:16:02.379 INFO:tasks.rados.rados.0.smithi163.stdout:2468: rollback oid 21 current snap is 315 2023-10-09T09:16:02.379 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 306 2023-10-09T09:16:02.379 INFO:tasks.rados.rados.0.smithi163.stdout:2469: read oid 20 snap 293 2023-10-09T09:16:02.379 INFO:tasks.rados.rados.0.smithi163.stdout:2469: expect (ObjNum 807 snap 279 seq_num 807) 2023-10-09T09:16:02.379 INFO:tasks.rados.rados.0.smithi163.stdout:2470: read oid 17 snap 295 2023-10-09T09:16:02.379 INFO:tasks.rados.rados.0.smithi163.stdout:2470: expect (ObjNum 805 snap 279 seq_num 805) 2023-10-09T09:16:02.380 INFO:tasks.rados.rados.0.smithi163.stdout:2471: write oid 10 current snap is 315 2023-10-09T09:16:02.380 INFO:tasks.rados.rados.0.smithi163.stdout:2471: seq_num 931 ranges {447317=651885,1713324=628329,3066571=580746} 2023-10-09T09:16:02.385 INFO:tasks.rados.rados.0.smithi163.stdout:2471: writing smithi16367185-10 from 447317 to 1099202 tid 1 2023-10-09T09:16:02.393 INFO:tasks.rados.rados.0.smithi163.stdout:2471: writing smithi16367185-10 from 1713324 to 2341653 tid 2 2023-10-09T09:16:02.401 INFO:tasks.rados.rados.0.smithi163.stdout:2471: writing smithi16367185-10 from 3066571 to 3647317 tid 3 2023-10-09T09:16:02.401 INFO:tasks.rados.rados.0.smithi163.stdout:2472: snap_remove snap 294 2023-10-09T09:16:02.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:02 smithi181 bash[59345]: cluster 2023-10-09T09:16:01.157850+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e688 with expected crc 2023-10-09T09:16:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:02 smithi181 bash[59345]: cluster 2023-10-09T09:16:01.165233+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e688 with expected crc 2023-10-09T09:16:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:02 smithi181 bash[59345]: cluster 2023-10-09T09:16:01.176337+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e688 with expected crc 2023-10-09T09:16:02.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:02 smithi181 bash[59345]: cluster 2023-10-09T09:16:01.359206+0000 mgr.y (mgr.34107) 284 : cluster 0 pgmap v585: 89 pgs: 89 active+clean; 220 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2023-10-09T09:16:02.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:02 smithi163 bash[78107]: cluster 2023-10-09T09:16:01.157850+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e688 with expected crc 2023-10-09T09:16:02.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:02 smithi163 bash[78107]: cluster 2023-10-09T09:16:01.165233+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e688 with expected crc 2023-10-09T09:16:02.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:02 smithi163 bash[78107]: cluster 2023-10-09T09:16:01.176337+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e688 with expected crc 2023-10-09T09:16:02.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:02 smithi163 bash[78107]: cluster 2023-10-09T09:16:01.359206+0000 mgr.y (mgr.34107) 284 : cluster 0 pgmap v585: 89 pgs: 89 active+clean; 220 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2023-10-09T09:16:02.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:02 smithi163 bash[75425]: cluster 2023-10-09T09:16:01.157850+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e688 with expected crc 2023-10-09T09:16:02.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:02 smithi163 bash[75425]: cluster 2023-10-09T09:16:01.165233+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e688 with expected crc 2023-10-09T09:16:02.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:02 smithi163 bash[75425]: cluster 2023-10-09T09:16:01.176337+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e688 with expected crc 2023-10-09T09:16:02.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:02 smithi163 bash[75425]: cluster 2023-10-09T09:16:01.359206+0000 mgr.y (mgr.34107) 284 : cluster 0 pgmap v585: 89 pgs: 89 active+clean; 220 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2023-10-09T09:16:03.307 INFO:tasks.rados.rados.0.smithi163.stdout:2472: done (5 left) 2023-10-09T09:16:03.307 INFO:tasks.rados.rados.0.smithi163.stdout:2473: snap_create 2023-10-09T09:16:03.307 INFO:tasks.rados.rados.0.smithi163.stdout:2466: finishing write tid 3 to smithi16367185-8 2023-10-09T09:16:03.307 INFO:tasks.rados.rados.0.smithi163.stdout:2468: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:16:03.307 INFO:tasks.rados.rados.0.smithi163.stdout:2468: finishing rollback tid 2 to smithi16367185-21 2023-10-09T09:16:03.308 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 548 (ObjNum 867 snap 301 seq_num 867) dirty exists 2023-10-09T09:16:03.308 INFO:tasks.rados.rados.0.smithi163.stdout:2471: finishing write tid 1 to smithi16367185-10 2023-10-09T09:16:03.308 INFO:tasks.rados.rados.0.smithi163.stdout:2466: finishing write tid 4 to smithi16367185-8 2023-10-09T09:16:03.308 INFO:tasks.rados.rados.0.smithi163.stdout:2466: finishing write tid 5 to smithi16367185-8 2023-10-09T09:16:03.308 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1234 (ObjNum 929 snap 315 seq_num 929) dirty exists 2023-10-09T09:16:03.308 INFO:tasks.rados.rados.0.smithi163.stdout:2466: left oid 8 (ObjNum 929 snap 315 seq_num 929) 2023-10-09T09:16:03.308 INFO:tasks.rados.rados.0.smithi163.stdout:2471: finishing write tid 2 to smithi16367185-10 2023-10-09T09:16:03.308 INFO:tasks.rados.rados.0.smithi163.stdout:2470: expect (ObjNum 805 snap 279 seq_num 805) 2023-10-09T09:16:03.466 INFO:tasks.rados.rados.0.smithi163.stdout:2469: expect (ObjNum 807 snap 279 seq_num 807) 2023-10-09T09:16:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:03 smithi181 bash[59345]: cluster 2023-10-09T09:16:02.296609+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:03 smithi181 bash[59345]: cluster 2023-10-09T09:16:02.301852+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:03 smithi181 bash[59345]: cluster 2023-10-09T09:16:02.302853+0000 mon.a (mon.0) 858 : cluster 0 osdmap e689: 8 total, 8 up, 8 in 2023-10-09T09:16:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:03 smithi181 bash[59345]: cluster 2023-10-09T09:16:02.305918+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:03 smithi181 bash[59345]: cluster 2023-10-09T09:16:02.308589+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.690 INFO:tasks.rados.rados.0.smithi163.stdout:2471: finishing write tid 3 to smithi16367185-10 2023-10-09T09:16:03.690 INFO:tasks.rados.rados.0.smithi163.stdout:2471: finishing write tid 4 to smithi16367185-10 2023-10-09T09:16:03.690 INFO:tasks.rados.rados.0.smithi163.stdout:2471: finishing write tid 5 to smithi16367185-10 2023-10-09T09:16:03.690 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 805 (ObjNum 931 snap 315 seq_num 931) dirty exists 2023-10-09T09:16:03.690 INFO:tasks.rados.rados.0.smithi163.stdout:2471: left oid 10 (ObjNum 931 snap 315 seq_num 931) 2023-10-09T09:16:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:16:03] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:16:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[78107]: cluster 2023-10-09T09:16:02.296609+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[78107]: cluster 2023-10-09T09:16:02.301852+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[78107]: cluster 2023-10-09T09:16:02.302853+0000 mon.a (mon.0) 858 : cluster 0 osdmap e689: 8 total, 8 up, 8 in 2023-10-09T09:16:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[78107]: cluster 2023-10-09T09:16:02.305918+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[78107]: cluster 2023-10-09T09:16:02.308589+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[75425]: cluster 2023-10-09T09:16:02.296609+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[75425]: cluster 2023-10-09T09:16:02.301852+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[75425]: cluster 2023-10-09T09:16:02.302853+0000 mon.a (mon.0) 858 : cluster 0 osdmap e689: 8 total, 8 up, 8 in 2023-10-09T09:16:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[75425]: cluster 2023-10-09T09:16:02.305918+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:03 smithi163 bash[75425]: cluster 2023-10-09T09:16:02.308589+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e689 with expected crc 2023-10-09T09:16:04.320 INFO:tasks.rados.rados.0.smithi163.stdout:2466: done (5 left) 2023-10-09T09:16:04.320 INFO:tasks.rados.rados.0.smithi163.stdout:2468: done (4 left) 2023-10-09T09:16:04.320 INFO:tasks.rados.rados.0.smithi163.stdout:2469: done (3 left) 2023-10-09T09:16:04.320 INFO:tasks.rados.rados.0.smithi163.stdout:2470: done (2 left) 2023-10-09T09:16:04.320 INFO:tasks.rados.rados.0.smithi163.stdout:2471: done (1 left) 2023-10-09T09:16:04.320 INFO:tasks.rados.rados.0.smithi163.stdout:2473: done (0 left) 2023-10-09T09:16:04.320 INFO:tasks.rados.rados.0.smithi163.stdout:2474: read oid 7 snap -1 2023-10-09T09:16:04.321 INFO:tasks.rados.rados.0.smithi163.stdout:2474: expect (ObjNum 856 snap 291 seq_num 856) 2023-10-09T09:16:04.321 INFO:tasks.rados.rados.0.smithi163.stdout:2475: rollback oid 40 current snap is 316 2023-10-09T09:16:04.321 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 40 to 309 2023-10-09T09:16:04.321 INFO:tasks.rados.rados.0.smithi163.stdout:2476: snap_remove snap 295 2023-10-09T09:16:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:04 smithi181 bash[59345]: cluster 2023-10-09T09:16:03.303492+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e690 with expected crc 2023-10-09T09:16:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:04 smithi181 bash[59345]: cluster 2023-10-09T09:16:03.306748+0000 mon.a (mon.0) 859 : cluster 0 osdmap e690: 8 total, 8 up, 8 in 2023-10-09T09:16:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:04 smithi181 bash[59345]: cluster 2023-10-09T09:16:03.359871+0000 mgr.y (mgr.34107) 285 : cluster 0 pgmap v588: 89 pgs: 89 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 24 op/s 2023-10-09T09:16:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:04 smithi163 bash[75425]: cluster 2023-10-09T09:16:03.303492+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e690 with expected crc 2023-10-09T09:16:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:04 smithi163 bash[75425]: cluster 2023-10-09T09:16:03.306748+0000 mon.a (mon.0) 859 : cluster 0 osdmap e690: 8 total, 8 up, 8 in 2023-10-09T09:16:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:04 smithi163 bash[75425]: cluster 2023-10-09T09:16:03.359871+0000 mgr.y (mgr.34107) 285 : cluster 0 pgmap v588: 89 pgs: 89 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 24 op/s 2023-10-09T09:16:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:04 smithi163 bash[78107]: cluster 2023-10-09T09:16:03.303492+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e690 with expected crc 2023-10-09T09:16:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:04 smithi163 bash[78107]: cluster 2023-10-09T09:16:03.306748+0000 mon.a (mon.0) 859 : cluster 0 osdmap e690: 8 total, 8 up, 8 in 2023-10-09T09:16:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:04 smithi163 bash[78107]: cluster 2023-10-09T09:16:03.359871+0000 mgr.y (mgr.34107) 285 : cluster 0 pgmap v588: 89 pgs: 89 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 24 op/s 2023-10-09T09:16:05.153 INFO:tasks.rados.rados.0.smithi163.stdout:2476: done (2 left) 2023-10-09T09:16:05.153 INFO:tasks.rados.rados.0.smithi163.stdout:2477: snap_remove snap 286 2023-10-09T09:16:05.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:05 smithi181 bash[59345]: cluster 2023-10-09T09:16:04.315701+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:05 smithi181 bash[59345]: cluster 2023-10-09T09:16:04.316478+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:05 smithi181 bash[59345]: cluster 2023-10-09T09:16:04.319856+0000 mon.a (mon.0) 860 : cluster 0 osdmap e691: 8 total, 8 up, 8 in 2023-10-09T09:16:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:05 smithi181 bash[59345]: cluster 2023-10-09T09:16:04.321011+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:05 smithi181 bash[59345]: cluster 2023-10-09T09:16:05.152354+0000 mon.a (mon.0) 861 : cluster 0 osdmap e692: 8 total, 8 up, 8 in 2023-10-09T09:16:05.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:05 smithi163 bash[75425]: cluster 2023-10-09T09:16:04.315701+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:05.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:05 smithi163 bash[75425]: cluster 2023-10-09T09:16:04.316478+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:05.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:05 smithi163 bash[75425]: cluster 2023-10-09T09:16:04.319856+0000 mon.a (mon.0) 860 : cluster 0 osdmap e691: 8 total, 8 up, 8 in 2023-10-09T09:16:05.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:05 smithi163 bash[75425]: cluster 2023-10-09T09:16:04.321011+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:05 smithi163 bash[75425]: cluster 2023-10-09T09:16:05.152354+0000 mon.a (mon.0) 861 : cluster 0 osdmap e692: 8 total, 8 up, 8 in 2023-10-09T09:16:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:05 smithi163 bash[78107]: cluster 2023-10-09T09:16:04.315701+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:05 smithi163 bash[78107]: cluster 2023-10-09T09:16:04.316478+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:05 smithi163 bash[78107]: cluster 2023-10-09T09:16:04.319856+0000 mon.a (mon.0) 860 : cluster 0 osdmap e691: 8 total, 8 up, 8 in 2023-10-09T09:16:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:05 smithi163 bash[78107]: cluster 2023-10-09T09:16:04.321011+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:05 smithi163 bash[78107]: cluster 2023-10-09T09:16:05.152354+0000 mon.a (mon.0) 861 : cluster 0 osdmap e692: 8 total, 8 up, 8 in 2023-10-09T09:16:06.155 INFO:tasks.rados.rados.0.smithi163.stdout:2477: done (2 left) 2023-10-09T09:16:06.156 INFO:tasks.rados.rados.0.smithi163.stdout:2478: rollback oid 41 current snap is 316 2023-10-09T09:16:06.156 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 41 to 311 2023-10-09T09:16:06.156 INFO:tasks.rados.rados.0.smithi163.stdout:2474: expect (ObjNum 856 snap 291 seq_num 856) 2023-10-09T09:16:06.287 INFO:tasks.rados.rados.0.smithi163.stdout:2475: finishing rollback tid 1 to smithi16367185-40 2023-10-09T09:16:06.288 INFO:tasks.rados.rados.0.smithi163.stdout:2475: finishing rollback tid 2 to smithi16367185-40 2023-10-09T09:16:06.288 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1242 (ObjNum 876 snap 305 seq_num 876) dirty exists 2023-10-09T09:16:06.288 INFO:tasks.rados.rados.0.smithi163.stdout:2478: finishing rollback tid 0 to smithi16367185-41 2023-10-09T09:16:06.288 INFO:tasks.rados.rados.0.smithi163.stdout:2478: finishing rollback tid 1 to smithi16367185-41 2023-10-09T09:16:06.288 INFO:tasks.rados.rados.0.smithi163.stdout:2478: finishing rollback tid 2 to smithi16367185-41 2023-10-09T09:16:06.288 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 179 (ObjNum 879 snap 307 seq_num 879) dirty exists 2023-10-09T09:16:06.288 INFO:tasks.rados.rados.0.smithi163.stdout:2474: done (2 left) 2023-10-09T09:16:06.288 INFO:tasks.rados.rados.0.smithi163.stdout:2475: done (1 left) 2023-10-09T09:16:06.288 INFO:tasks.rados.rados.0.smithi163.stdout:2478: done (0 left) 2023-10-09T09:16:06.288 INFO:tasks.rados.rados.0.smithi163.stdout:2479: read oid 20 snap -1 2023-10-09T09:16:06.289 INFO:tasks.rados.rados.0.smithi163.stdout:2479: expect (ObjNum 807 snap 279 seq_num 807) 2023-10-09T09:16:06.289 INFO:tasks.rados.rados.0.smithi163.stdout:2480: snap_remove snap 307 2023-10-09T09:16:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:06 smithi163 bash[75425]: cluster 2023-10-09T09:16:04.603312+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:06 smithi163 bash[75425]: cluster 2023-10-09T09:16:05.147757+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e692 with expected crc 2023-10-09T09:16:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:06 smithi163 bash[75425]: cluster 2023-10-09T09:16:05.150588+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e692 with expected crc 2023-10-09T09:16:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:06 smithi163 bash[75425]: cluster 2023-10-09T09:16:05.360639+0000 mgr.y (mgr.34107) 286 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:16:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:06 smithi163 bash[75425]: cluster 2023-10-09T09:16:06.155316+0000 mon.a (mon.0) 862 : cluster 0 osdmap e693: 8 total, 8 up, 8 in 2023-10-09T09:16:06.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:06 smithi163 bash[78107]: cluster 2023-10-09T09:16:04.603312+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:06.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:06 smithi163 bash[78107]: cluster 2023-10-09T09:16:05.147757+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e692 with expected crc 2023-10-09T09:16:06.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:06 smithi163 bash[78107]: cluster 2023-10-09T09:16:05.150588+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e692 with expected crc 2023-10-09T09:16:06.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:06 smithi163 bash[78107]: cluster 2023-10-09T09:16:05.360639+0000 mgr.y (mgr.34107) 286 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:16:06.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:06 smithi163 bash[78107]: cluster 2023-10-09T09:16:06.155316+0000 mon.a (mon.0) 862 : cluster 0 osdmap e693: 8 total, 8 up, 8 in 2023-10-09T09:16:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:06 smithi181 bash[59345]: cluster 2023-10-09T09:16:04.603312+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e691 with expected crc 2023-10-09T09:16:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:06 smithi181 bash[59345]: cluster 2023-10-09T09:16:05.147757+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e692 with expected crc 2023-10-09T09:16:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:06 smithi181 bash[59345]: cluster 2023-10-09T09:16:05.150588+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e692 with expected crc 2023-10-09T09:16:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:06 smithi181 bash[59345]: cluster 2023-10-09T09:16:05.360639+0000 mgr.y (mgr.34107) 286 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:16:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:06 smithi181 bash[59345]: cluster 2023-10-09T09:16:06.155316+0000 mon.a (mon.0) 862 : cluster 0 osdmap e693: 8 total, 8 up, 8 in 2023-10-09T09:16:07.158 INFO:tasks.rados.rados.0.smithi163.stdout:2480: done (1 left) 2023-10-09T09:16:07.158 INFO:tasks.rados.rados.0.smithi163.stdout:2481: snap_create 2023-10-09T09:16:07.158 INFO:tasks.rados.rados.0.smithi163.stdout:2479: expect (ObjNum 807 snap 279 seq_num 807) 2023-10-09T09:16:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:07 smithi163 bash[75425]: cluster 2023-10-09T09:16:06.150253+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e693 with expected crc 2023-10-09T09:16:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:07 smithi163 bash[75425]: cluster 2023-10-09T09:16:06.151625+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e693 with expected crc 2023-10-09T09:16:07.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:07 smithi163 bash[75425]: cluster 2023-10-09T09:16:07.157953+0000 mon.a (mon.0) 863 : cluster 0 osdmap e694: 8 total, 8 up, 8 in 2023-10-09T09:16:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:07 smithi163 bash[78107]: cluster 2023-10-09T09:16:06.150253+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e693 with expected crc 2023-10-09T09:16:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:07 smithi163 bash[78107]: cluster 2023-10-09T09:16:06.151625+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e693 with expected crc 2023-10-09T09:16:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:07 smithi163 bash[78107]: cluster 2023-10-09T09:16:07.157953+0000 mon.a (mon.0) 863 : cluster 0 osdmap e694: 8 total, 8 up, 8 in 2023-10-09T09:16:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:07 smithi181 bash[59345]: cluster 2023-10-09T09:16:06.150253+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e693 with expected crc 2023-10-09T09:16:07.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:07 smithi181 bash[59345]: cluster 2023-10-09T09:16:06.151625+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e693 with expected crc 2023-10-09T09:16:07.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:07 smithi181 bash[59345]: cluster 2023-10-09T09:16:07.157953+0000 mon.a (mon.0) 863 : cluster 0 osdmap e694: 8 total, 8 up, 8 in 2023-10-09T09:16:08.160 INFO:tasks.rados.rados.0.smithi163.stdout:2479: done (1 left) 2023-10-09T09:16:08.160 INFO:tasks.rados.rados.0.smithi163.stdout:2481: done (0 left) 2023-10-09T09:16:08.160 INFO:tasks.rados.rados.0.smithi163.stdout:2482: write oid 39 current snap is 317 2023-10-09T09:16:08.160 INFO:tasks.rados.rados.0.smithi163.stdout:2482: seq_num 932 ranges {464124=727134,1771806=771987,2977641=641516} 2023-10-09T09:16:08.170 INFO:tasks.rados.rados.0.smithi163.stdout:2482: writing smithi16367185-39 from 464124 to 1191258 tid 1 2023-10-09T09:16:08.178 INFO:tasks.rados.rados.0.smithi163.stdout:2482: writing smithi16367185-39 from 1771806 to 2543793 tid 2 2023-10-09T09:16:08.184 INFO:tasks.rados.rados.0.smithi163.stdout:2482: writing smithi16367185-39 from 2977641 to 3619157 tid 3 2023-10-09T09:16:08.185 INFO:tasks.rados.rados.0.smithi163.stdout:2483: snap_remove snap 309 2023-10-09T09:16:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:08 smithi163 bash[78107]: cluster 2023-10-09T09:16:07.152970+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e694 with expected crc 2023-10-09T09:16:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:08 smithi163 bash[78107]: cluster 2023-10-09T09:16:07.153756+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e694 with expected crc 2023-10-09T09:16:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:08 smithi163 bash[78107]: cluster 2023-10-09T09:16:07.337379+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e694 with expected crc 2023-10-09T09:16:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:08 smithi163 bash[78107]: cluster 2023-10-09T09:16:07.361133+0000 mgr.y (mgr.34107) 287 : cluster 0 pgmap v594: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 216 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2023-10-09T09:16:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:08 smithi163 bash[78107]: cluster 2023-10-09T09:16:08.159683+0000 mon.a (mon.0) 864 : cluster 0 osdmap e695: 8 total, 8 up, 8 in 2023-10-09T09:16:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:08 smithi163 bash[75425]: cluster 2023-10-09T09:16:07.152970+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e694 with expected crc 2023-10-09T09:16:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:08 smithi163 bash[75425]: cluster 2023-10-09T09:16:07.153756+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e694 with expected crc 2023-10-09T09:16:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:08 smithi163 bash[75425]: cluster 2023-10-09T09:16:07.337379+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e694 with expected crc 2023-10-09T09:16:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:08 smithi163 bash[75425]: cluster 2023-10-09T09:16:07.361133+0000 mgr.y (mgr.34107) 287 : cluster 0 pgmap v594: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 216 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2023-10-09T09:16:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:08 smithi163 bash[75425]: cluster 2023-10-09T09:16:08.159683+0000 mon.a (mon.0) 864 : cluster 0 osdmap e695: 8 total, 8 up, 8 in 2023-10-09T09:16:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:08 smithi181 bash[59345]: cluster 2023-10-09T09:16:07.152970+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e694 with expected crc 2023-10-09T09:16:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:08 smithi181 bash[59345]: cluster 2023-10-09T09:16:07.153756+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e694 with expected crc 2023-10-09T09:16:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:08 smithi181 bash[59345]: cluster 2023-10-09T09:16:07.337379+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e694 with expected crc 2023-10-09T09:16:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:08 smithi181 bash[59345]: cluster 2023-10-09T09:16:07.361133+0000 mgr.y (mgr.34107) 287 : cluster 0 pgmap v594: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 216 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2023-10-09T09:16:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:08 smithi181 bash[59345]: cluster 2023-10-09T09:16:08.159683+0000 mon.a (mon.0) 864 : cluster 0 osdmap e695: 8 total, 8 up, 8 in 2023-10-09T09:16:09.163 INFO:tasks.rados.rados.0.smithi163.stdout:2483: done (1 left) 2023-10-09T09:16:09.163 INFO:tasks.rados.rados.0.smithi163.stdout:2484: snap_create 2023-10-09T09:16:09.163 INFO:tasks.rados.rados.0.smithi163.stdout:2482: finishing write tid 1 to smithi16367185-39 2023-10-09T09:16:09.163 INFO:tasks.rados.rados.0.smithi163.stdout:2482: finishing write tid 2 to smithi16367185-39 2023-10-09T09:16:09.164 INFO:tasks.rados.rados.0.smithi163.stdout:2482: finishing write tid 3 to smithi16367185-39 2023-10-09T09:16:09.164 INFO:tasks.rados.rados.0.smithi163.stdout:2482: finishing write tid 4 to smithi16367185-39 2023-10-09T09:16:09.164 INFO:tasks.rados.rados.0.smithi163.stdout:2482: finishing write tid 5 to smithi16367185-39 2023-10-09T09:16:09.164 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1168 (ObjNum 932 snap 317 seq_num 932) dirty exists 2023-10-09T09:16:09.164 INFO:tasks.rados.rados.0.smithi163.stdout:2482: left oid 39 (ObjNum 932 snap 317 seq_num 932) 2023-10-09T09:16:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:09 smithi163 bash[78107]: cluster 2023-10-09T09:16:08.155583+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e695 with expected crc 2023-10-09T09:16:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:09 smithi163 bash[78107]: cluster 2023-10-09T09:16:08.156255+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e695 with expected crc 2023-10-09T09:16:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:09 smithi163 bash[78107]: cluster 2023-10-09T09:16:09.163161+0000 mon.a (mon.0) 865 : cluster 0 osdmap e696: 8 total, 8 up, 8 in 2023-10-09T09:16:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:09 smithi163 bash[75425]: cluster 2023-10-09T09:16:08.155583+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e695 with expected crc 2023-10-09T09:16:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:09 smithi163 bash[75425]: cluster 2023-10-09T09:16:08.156255+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e695 with expected crc 2023-10-09T09:16:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:09 smithi163 bash[75425]: cluster 2023-10-09T09:16:09.163161+0000 mon.a (mon.0) 865 : cluster 0 osdmap e696: 8 total, 8 up, 8 in 2023-10-09T09:16:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:09 smithi181 bash[59345]: cluster 2023-10-09T09:16:08.155583+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e695 with expected crc 2023-10-09T09:16:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:09 smithi181 bash[59345]: cluster 2023-10-09T09:16:08.156255+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e695 with expected crc 2023-10-09T09:16:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:09 smithi181 bash[59345]: cluster 2023-10-09T09:16:09.163161+0000 mon.a (mon.0) 865 : cluster 0 osdmap e696: 8 total, 8 up, 8 in 2023-10-09T09:16:10.156 INFO:tasks.rados.rados.0.smithi163.stdout:2482: done (1 left) 2023-10-09T09:16:10.157 INFO:tasks.rados.rados.0.smithi163.stdout:2484: done (0 left) 2023-10-09T09:16:10.157 INFO:tasks.rados.rados.0.smithi163.stdout:2485: write oid 13 current snap is 318 2023-10-09T09:16:10.157 INFO:tasks.rados.rados.0.smithi163.stdout:2485: seq_num 933 ranges {480931=402383,1313935=515645,2488711=452788,3443230=237701} 2023-10-09T09:16:10.163 INFO:tasks.rados.rados.0.smithi163.stdout:2485: writing smithi16367185-13 from 480931 to 883314 tid 1 2023-10-09T09:16:10.169 INFO:tasks.rados.rados.0.smithi163.stdout:2485: writing smithi16367185-13 from 1313935 to 1829580 tid 2 2023-10-09T09:16:10.176 INFO:tasks.rados.rados.0.smithi163.stdout:2485: writing smithi16367185-13 from 2488711 to 2941499 tid 3 2023-10-09T09:16:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:2485: writing smithi16367185-13 from 3443230 to 3680931 tid 4 2023-10-09T09:16:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:2486: delete oid 35 current snap is 318 2023-10-09T09:16:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:2485: finishing write tid 1 to smithi16367185-13 2023-10-09T09:16:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:2485: finishing write tid 2 to smithi16367185-13 2023-10-09T09:16:10.181 INFO:tasks.rados.rados.0.smithi163.stdout:2485: finishing write tid 3 to smithi16367185-13 2023-10-09T09:16:10.186 INFO:tasks.rados.rados.0.smithi163.stdout:2485: finishing write tid 4 to smithi16367185-13 2023-10-09T09:16:10.187 INFO:tasks.rados.rados.0.smithi163.stdout:2485: finishing write tid 5 to smithi16367185-13 2023-10-09T09:16:10.187 INFO:tasks.rados.rados.0.smithi163.stdout:2485: finishing write tid 6 to smithi16367185-13 2023-10-09T09:16:10.187 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 365 (ObjNum 933 snap 318 seq_num 933) dirty exists 2023-10-09T09:16:10.187 INFO:tasks.rados.rados.0.smithi163.stdout:2485: left oid 13 (ObjNum 933 snap 318 seq_num 933) 2023-10-09T09:16:10.187 INFO:tasks.rados.rados.0.smithi163.stdout:2485: done (1 left) 2023-10-09T09:16:10.188 INFO:tasks.rados.rados.0.smithi163.stdout:2486: done (0 left) 2023-10-09T09:16:10.188 INFO:tasks.rados.rados.0.smithi163.stdout:2487: snap_create 2023-10-09T09:16:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:10 smithi163 bash[78107]: cluster 2023-10-09T09:16:09.158171+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e696 with expected crc 2023-10-09T09:16:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:10 smithi163 bash[78107]: cluster 2023-10-09T09:16:09.161012+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e696 with expected crc 2023-10-09T09:16:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:10 smithi163 bash[78107]: cluster 2023-10-09T09:16:09.361888+0000 mgr.y (mgr.34107) 288 : cluster 0 pgmap v597: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 216 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 789 KiB/s wr, 10 op/s 2023-10-09T09:16:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:10 smithi163 bash[78107]: cluster 2023-10-09T09:16:10.156423+0000 mon.a (mon.0) 866 : cluster 0 osdmap e697: 8 total, 8 up, 8 in 2023-10-09T09:16:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:10 smithi163 bash[75425]: cluster 2023-10-09T09:16:09.158171+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e696 with expected crc 2023-10-09T09:16:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:10 smithi163 bash[75425]: cluster 2023-10-09T09:16:09.161012+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e696 with expected crc 2023-10-09T09:16:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:10 smithi163 bash[75425]: cluster 2023-10-09T09:16:09.361888+0000 mgr.y (mgr.34107) 288 : cluster 0 pgmap v597: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 216 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 789 KiB/s wr, 10 op/s 2023-10-09T09:16:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:10 smithi163 bash[75425]: cluster 2023-10-09T09:16:10.156423+0000 mon.a (mon.0) 866 : cluster 0 osdmap e697: 8 total, 8 up, 8 in 2023-10-09T09:16:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:10 smithi181 bash[59345]: cluster 2023-10-09T09:16:09.158171+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e696 with expected crc 2023-10-09T09:16:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:10 smithi181 bash[59345]: cluster 2023-10-09T09:16:09.161012+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e696 with expected crc 2023-10-09T09:16:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:10 smithi181 bash[59345]: cluster 2023-10-09T09:16:09.361888+0000 mgr.y (mgr.34107) 288 : cluster 0 pgmap v597: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 216 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 789 KiB/s wr, 10 op/s 2023-10-09T09:16:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:10 smithi181 bash[59345]: cluster 2023-10-09T09:16:10.156423+0000 mon.a (mon.0) 866 : cluster 0 osdmap e697: 8 total, 8 up, 8 in 2023-10-09T09:16:11.167 INFO:tasks.rados.rados.0.smithi163.stdout:2487: done (0 left) 2023-10-09T09:16:11.168 INFO:tasks.rados.rados.0.smithi163.stdout:2488: snap_remove snap 315 2023-10-09T09:16:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:11 smithi163 bash[78107]: cluster 2023-10-09T09:16:10.151164+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e697 with expected crc 2023-10-09T09:16:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:11 smithi163 bash[78107]: cluster 2023-10-09T09:16:10.151656+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e697 with expected crc 2023-10-09T09:16:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:11 smithi163 bash[78107]: cluster 2023-10-09T09:16:10.164399+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e697 with expected crc 2023-10-09T09:16:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:11 smithi163 bash[78107]: audit 2023-10-09T09:16:10.344737+0000 mon.a (mon.0) 867 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:11 smithi163 bash[78107]: cluster 2023-10-09T09:16:11.167732+0000 mon.a (mon.0) 868 : cluster 0 osdmap e698: 8 total, 8 up, 8 in 2023-10-09T09:16:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:11 smithi163 bash[75425]: cluster 2023-10-09T09:16:10.151164+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e697 with expected crc 2023-10-09T09:16:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:11 smithi163 bash[75425]: cluster 2023-10-09T09:16:10.151656+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e697 with expected crc 2023-10-09T09:16:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:11 smithi163 bash[75425]: cluster 2023-10-09T09:16:10.164399+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e697 with expected crc 2023-10-09T09:16:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:11 smithi163 bash[75425]: audit 2023-10-09T09:16:10.344737+0000 mon.a (mon.0) 867 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:11 smithi163 bash[75425]: cluster 2023-10-09T09:16:11.167732+0000 mon.a (mon.0) 868 : cluster 0 osdmap e698: 8 total, 8 up, 8 in 2023-10-09T09:16:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:11 smithi181 bash[59345]: cluster 2023-10-09T09:16:10.151164+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e697 with expected crc 2023-10-09T09:16:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:11 smithi181 bash[59345]: cluster 2023-10-09T09:16:10.151656+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e697 with expected crc 2023-10-09T09:16:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:11 smithi181 bash[59345]: cluster 2023-10-09T09:16:10.164399+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e697 with expected crc 2023-10-09T09:16:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:11 smithi181 bash[59345]: audit 2023-10-09T09:16:10.344737+0000 mon.a (mon.0) 867 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:11 smithi181 bash[59345]: cluster 2023-10-09T09:16:11.167732+0000 mon.a (mon.0) 868 : cluster 0 osdmap e698: 8 total, 8 up, 8 in 2023-10-09T09:16:12.169 INFO:tasks.rados.rados.0.smithi163.stdout:2488: done (0 left) 2023-10-09T09:16:12.169 INFO:tasks.rados.rados.0.smithi163.stdout:2489: read oid 30 snap -1 2023-10-09T09:16:12.169 INFO:tasks.rados.rados.0.smithi163.stdout:2489: expect deleted 2023-10-09T09:16:12.170 INFO:tasks.rados.rados.0.smithi163.stdout:2490: read oid 44 snap -1 2023-10-09T09:16:12.170 INFO:tasks.rados.rados.0.smithi163.stdout:2490: expect deleted 2023-10-09T09:16:12.170 INFO:tasks.rados.rados.0.smithi163.stdout:2491: rollback oid 20 current snap is 319 2023-10-09T09:16:12.170 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 300 2023-10-09T09:16:12.170 INFO:tasks.rados.rados.0.smithi163.stdout:2492: delete oid 23 current snap is 319 2023-10-09T09:16:12.182 INFO:tasks.rados.rados.0.smithi163.stdout:2489: done (3 left) 2023-10-09T09:16:12.182 INFO:tasks.rados.rados.0.smithi163.stdout:2490: done (2 left) 2023-10-09T09:16:12.182 INFO:tasks.rados.rados.0.smithi163.stdout:2492: done (1 left) 2023-10-09T09:16:12.182 INFO:tasks.rados.rados.0.smithi163.stdout:2493: read oid 26 snap 300 2023-10-09T09:16:12.182 INFO:tasks.rados.rados.0.smithi163.stdout:2493: expect (ObjNum 731 snap 246 seq_num 731) 2023-10-09T09:16:12.182 INFO:tasks.rados.rados.0.smithi163.stdout:2494: snap_create 2023-10-09T09:16:12.183 INFO:tasks.rados.rados.0.smithi163.stdout:2491: finishing rollback tid 0 to smithi16367185-20 2023-10-09T09:16:12.183 INFO:tasks.rados.rados.0.smithi163.stdout:2491: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:16:12.184 INFO:tasks.rados.rados.0.smithi163.stdout:2491: finishing rollback tid 2 to smithi16367185-20 2023-10-09T09:16:12.184 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 369 (ObjNum 807 snap 279 seq_num 807) dirty exists 2023-10-09T09:16:12.187 INFO:tasks.rados.rados.0.smithi163.stdout:2493: expect (ObjNum 731 snap 246 seq_num 731) 2023-10-09T09:16:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[75425]: cluster 2023-10-09T09:16:11.163056+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[75425]: cluster 2023-10-09T09:16:11.362571+0000 mgr.y (mgr.34107) 289 : cluster 0 pgmap v600: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 212 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 916 KiB/s wr, 8 op/s 2023-10-09T09:16:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[75425]: cluster 2023-10-09T09:16:11.430158+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[75425]: cluster 2023-10-09T09:16:11.430158+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[75425]: cluster 2023-10-09T09:16:11.430382+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[75425]: cluster 2023-10-09T09:16:12.168137+0000 mon.a (mon.0) 869 : cluster 0 osdmap e699: 8 total, 8 up, 8 in 2023-10-09T09:16:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[78107]: cluster 2023-10-09T09:16:11.163056+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[78107]: cluster 2023-10-09T09:16:11.362571+0000 mgr.y (mgr.34107) 289 : cluster 0 pgmap v600: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 212 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 916 KiB/s wr, 8 op/s 2023-10-09T09:16:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[78107]: cluster 2023-10-09T09:16:11.430158+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[78107]: cluster 2023-10-09T09:16:11.430158+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[78107]: cluster 2023-10-09T09:16:11.430382+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:12 smithi163 bash[78107]: cluster 2023-10-09T09:16:12.168137+0000 mon.a (mon.0) 869 : cluster 0 osdmap e699: 8 total, 8 up, 8 in 2023-10-09T09:16:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:12 smithi181 bash[59345]: cluster 2023-10-09T09:16:11.163056+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:12 smithi181 bash[59345]: cluster 2023-10-09T09:16:11.362571+0000 mgr.y (mgr.34107) 289 : cluster 0 pgmap v600: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 212 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 916 KiB/s wr, 8 op/s 2023-10-09T09:16:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:12 smithi181 bash[59345]: cluster 2023-10-09T09:16:11.430158+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:12 smithi181 bash[59345]: cluster 2023-10-09T09:16:11.430158+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:12 smithi181 bash[59345]: cluster 2023-10-09T09:16:11.430382+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e698 with expected crc 2023-10-09T09:16:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:12 smithi181 bash[59345]: cluster 2023-10-09T09:16:12.168137+0000 mon.a (mon.0) 869 : cluster 0 osdmap e699: 8 total, 8 up, 8 in 2023-10-09T09:16:13.173 INFO:tasks.rados.rados.0.smithi163.stdout:2491: done (2 left) 2023-10-09T09:16:13.173 INFO:tasks.rados.rados.0.smithi163.stdout:2493: done (1 left) 2023-10-09T09:16:13.173 INFO:tasks.rados.rados.0.smithi163.stdout:2494: done (0 left) 2023-10-09T09:16:13.173 INFO:tasks.rados.rados.0.smithi163.stdout:2495: write oid 21 current snap is 320 2023-10-09T09:16:13.173 INFO:tasks.rados.rados.0.smithi163.stdout:2495: seq_num 936 ranges {531352=744483,2005734=779325,3487333=244019} 2023-10-09T09:16:13.184 INFO:tasks.rados.rados.0.smithi163.stdout:2495: writing smithi16367185-21 from 531352 to 1275835 tid 1 2023-10-09T09:16:13.194 INFO:tasks.rados.rados.0.smithi163.stdout:2495: writing smithi16367185-21 from 2005734 to 2785059 tid 2 2023-10-09T09:16:13.200 INFO:tasks.rados.rados.0.smithi163.stdout:2495: writing smithi16367185-21 from 3487333 to 3731352 tid 3 2023-10-09T09:16:13.200 INFO:tasks.rados.rados.0.smithi163.stdout:2496: write oid 30 current snap is 320 2023-10-09T09:16:13.200 INFO:tasks.rados.rados.0.smithi163.stdout:2495: finishing write tid 1 to smithi16367185-21 2023-10-09T09:16:13.201 INFO:tasks.rados.rados.0.smithi163.stdout:2495: finishing write tid 2 to smithi16367185-21 2023-10-09T09:16:13.201 INFO:tasks.rados.rados.0.smithi163.stdout:2496: seq_num 937 ranges {548159=419732,1664216=522983,2714756=614229} 2023-10-09T09:16:13.206 INFO:tasks.rados.rados.0.smithi163.stdout:2496: writing smithi16367185-30 from 548159 to 967891 tid 1 2023-10-09T09:16:13.213 INFO:tasks.rados.rados.0.smithi163.stdout:2496: writing smithi16367185-30 from 1664216 to 2187199 tid 2 2023-10-09T09:16:13.219 INFO:tasks.rados.rados.0.smithi163.stdout:2496: writing smithi16367185-30 from 2714756 to 3328985 tid 3 2023-10-09T09:16:13.220 INFO:tasks.rados.rados.0.smithi163.stdout:2497: write oid 17 current snap is 320 2023-10-09T09:16:13.220 INFO:tasks.rados.rados.0.smithi163.stdout:2497: seq_num 938 ranges {564966=494981,1606345=782994,3025826=425501} 2023-10-09T09:16:13.226 INFO:tasks.rados.rados.0.smithi163.stdout:2497: writing smithi16367185-17 from 564966 to 1059947 tid 1 2023-10-09T09:16:13.233 INFO:tasks.rados.rados.0.smithi163.stdout:2497: writing smithi16367185-17 from 1606345 to 2389339 tid 2 2023-10-09T09:16:13.240 INFO:tasks.rados.rados.0.smithi163.stdout:2497: writing smithi16367185-17 from 3025826 to 3451327 tid 3 2023-10-09T09:16:13.240 INFO:tasks.rados.rados.0.smithi163.stdout:2498: rollback oid 46 current snap is 320 2023-10-09T09:16:13.240 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 46 to 313 2023-10-09T09:16:13.240 INFO:tasks.rados.rados.0.smithi163.stdout:2499: snap_create 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:2495: finishing write tid 3 to smithi16367185-21 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:2495: finishing write tid 4 to smithi16367185-21 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:2495: finishing write tid 5 to smithi16367185-21 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 557 (ObjNum 936 snap 320 seq_num 936) dirty exists 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:2495: left oid 21 (ObjNum 936 snap 320 seq_num 936) 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:2496: finishing write tid 1 to smithi16367185-30 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:2496: finishing write tid 2 to smithi16367185-30 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:2496: finishing write tid 3 to smithi16367185-30 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:2496: finishing write tid 4 to smithi16367185-30 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:2496: finishing write tid 5 to smithi16367185-30 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 821 (ObjNum 937 snap 320 seq_num 937) dirty exists 2023-10-09T09:16:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:2496: left oid 30 (ObjNum 937 snap 320 seq_num 937) 2023-10-09T09:16:13.242 INFO:tasks.rados.rados.0.smithi163.stdout:2497: finishing write tid 1 to smithi16367185-17 2023-10-09T09:16:13.242 INFO:tasks.rados.rados.0.smithi163.stdout:2497: finishing write tid 2 to smithi16367185-17 2023-10-09T09:16:13.243 INFO:tasks.rados.rados.0.smithi163.stdout:2498: finishing rollback tid 1 to smithi16367185-46 2023-10-09T09:16:13.243 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 1058 (ObjNum 698892704 snap 21875 seq_num 697217040) dirty dne 2023-10-09T09:16:13.244 INFO:tasks.rados.rados.0.smithi163.stdout:2497: finishing write tid 3 to smithi16367185-17 2023-10-09T09:16:13.244 INFO:tasks.rados.rados.0.smithi163.stdout:2497: finishing write tid 4 to smithi16367185-17 2023-10-09T09:16:13.244 INFO:tasks.rados.rados.0.smithi163.stdout:2497: finishing write tid 5 to smithi16367185-17 2023-10-09T09:16:13.244 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1277 (ObjNum 938 snap 320 seq_num 938) dirty exists 2023-10-09T09:16:13.244 INFO:tasks.rados.rados.0.smithi163.stdout:2497: left oid 17 (ObjNum 938 snap 320 seq_num 938) 2023-10-09T09:16:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:16:13] "GET /metrics HTTP/1.1" 200 36268 "" "Prometheus/2.33.4" 2023-10-09T09:16:13.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[78107]: cluster 2023-10-09T09:16:12.163524+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e699 with expected crc 2023-10-09T09:16:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[78107]: cluster 2023-10-09T09:16:12.171587+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e699 with expected crc 2023-10-09T09:16:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[78107]: cluster 2023-10-09T09:16:12.175161+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e699 with expected crc 2023-10-09T09:16:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[78107]: cluster 2023-10-09T09:16:13.170496+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[78107]: cluster 2023-10-09T09:16:13.172755+0000 mon.a (mon.0) 870 : cluster 0 osdmap e700: 8 total, 8 up, 8 in 2023-10-09T09:16:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[75425]: cluster 2023-10-09T09:16:12.163524+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e699 with expected crc 2023-10-09T09:16:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[75425]: cluster 2023-10-09T09:16:12.171587+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e699 with expected crc 2023-10-09T09:16:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[75425]: cluster 2023-10-09T09:16:12.175161+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e699 with expected crc 2023-10-09T09:16:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[75425]: cluster 2023-10-09T09:16:13.170496+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:13 smithi163 bash[75425]: cluster 2023-10-09T09:16:13.172755+0000 mon.a (mon.0) 870 : cluster 0 osdmap e700: 8 total, 8 up, 8 in 2023-10-09T09:16:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:13 smithi181 bash[59345]: cluster 2023-10-09T09:16:12.163524+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e699 with expected crc 2023-10-09T09:16:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:13 smithi181 bash[59345]: cluster 2023-10-09T09:16:12.171587+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e699 with expected crc 2023-10-09T09:16:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:13 smithi181 bash[59345]: cluster 2023-10-09T09:16:12.175161+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e699 with expected crc 2023-10-09T09:16:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:13 smithi181 bash[59345]: cluster 2023-10-09T09:16:13.170496+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:13 smithi181 bash[59345]: cluster 2023-10-09T09:16:13.172755+0000 mon.a (mon.0) 870 : cluster 0 osdmap e700: 8 total, 8 up, 8 in 2023-10-09T09:16:14.174 INFO:tasks.rados.rados.0.smithi163.stdout:2495: done (4 left) 2023-10-09T09:16:14.175 INFO:tasks.rados.rados.0.smithi163.stdout:2496: done (3 left) 2023-10-09T09:16:14.175 INFO:tasks.rados.rados.0.smithi163.stdout:2497: done (2 left) 2023-10-09T09:16:14.175 INFO:tasks.rados.rados.0.smithi163.stdout:2498: done (1 left) 2023-10-09T09:16:14.175 INFO:tasks.rados.rados.0.smithi163.stdout:2499: done (0 left) 2023-10-09T09:16:14.175 INFO:tasks.rados.rados.0.smithi163.stdout:2500: snap_create 2023-10-09T09:16:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[75425]: cluster 2023-10-09T09:16:13.168039+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[75425]: cluster 2023-10-09T09:16:13.194544+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[75425]: cluster 2023-10-09T09:16:13.196580+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[75425]: cluster 2023-10-09T09:16:13.227816+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[75425]: cluster 2023-10-09T09:16:13.363293+0000 mgr.y (mgr.34107) 290 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 211 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 916 KiB/s wr, 6 op/s 2023-10-09T09:16:14.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[75425]: cluster 2023-10-09T09:16:14.174053+0000 mon.a (mon.0) 871 : cluster 0 osdmap e701: 8 total, 8 up, 8 in 2023-10-09T09:16:14.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[78107]: cluster 2023-10-09T09:16:13.168039+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[78107]: cluster 2023-10-09T09:16:13.194544+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[78107]: cluster 2023-10-09T09:16:13.196580+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[78107]: cluster 2023-10-09T09:16:13.227816+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[78107]: cluster 2023-10-09T09:16:13.363293+0000 mgr.y (mgr.34107) 290 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 211 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 916 KiB/s wr, 6 op/s 2023-10-09T09:16:14.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:14 smithi163 bash[78107]: cluster 2023-10-09T09:16:14.174053+0000 mon.a (mon.0) 871 : cluster 0 osdmap e701: 8 total, 8 up, 8 in 2023-10-09T09:16:14.864 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:14 smithi181 bash[59345]: cluster 2023-10-09T09:16:13.168039+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.864 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:14 smithi181 bash[59345]: cluster 2023-10-09T09:16:13.194544+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.864 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:14 smithi181 bash[59345]: cluster 2023-10-09T09:16:13.196580+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.865 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:14 smithi181 bash[59345]: cluster 2023-10-09T09:16:13.227816+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e700 with expected crc 2023-10-09T09:16:14.865 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:14 smithi181 bash[59345]: cluster 2023-10-09T09:16:13.363293+0000 mgr.y (mgr.34107) 290 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 211 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 916 KiB/s wr, 6 op/s 2023-10-09T09:16:14.865 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:14 smithi181 bash[59345]: cluster 2023-10-09T09:16:14.174053+0000 mon.a (mon.0) 871 : cluster 0 osdmap e701: 8 total, 8 up, 8 in 2023-10-09T09:16:15.159 INFO:tasks.rados.rados.0.smithi163.stdout:2500: done (0 left) 2023-10-09T09:16:15.159 INFO:tasks.rados.rados.0.smithi163.stdout:2501: write oid 35 current snap is 322 2023-10-09T09:16:15.159 INFO:tasks.rados.rados.0.smithi163.stdout:2501: seq_num 939 ranges {581773=570230,1664827=526652,2653249=636773} 2023-10-09T09:16:15.173 INFO:tasks.rados.rados.0.smithi163.stdout:2501: writing smithi16367185-35 from 581773 to 1152003 tid 1 2023-10-09T09:16:15.179 INFO:tasks.rados.rados.0.smithi163.stdout:2501: writing smithi16367185-35 from 1664827 to 2191479 tid 2 2023-10-09T09:16:15.186 INFO:tasks.rados.rados.0.smithi163.stdout:2501: writing smithi16367185-35 from 2653249 to 3290022 tid 3 2023-10-09T09:16:15.186 INFO:tasks.rados.rados.0.smithi163.stdout:2502: write oid 26 current snap is 322 2023-10-09T09:16:15.186 INFO:tasks.rados.rados.0.smithi163.stdout:2501: finishing write tid 1 to smithi16367185-35 2023-10-09T09:16:15.186 INFO:tasks.rados.rados.0.smithi163.stdout:2502: seq_num 940 ranges {598580=645479,1723309=786663,3080672=448045} 2023-10-09T09:16:15.193 INFO:tasks.rados.rados.0.smithi163.stdout:2502: writing smithi16367185-26 from 598580 to 1244059 tid 1 2023-10-09T09:16:15.201 INFO:tasks.rados.rados.0.smithi163.stdout:2502: writing smithi16367185-26 from 1723309 to 2509972 tid 2 2023-10-09T09:16:15.208 INFO:tasks.rados.rados.0.smithi163.stdout:2502: writing smithi16367185-26 from 3080672 to 3528717 tid 3 2023-10-09T09:16:15.208 INFO:tasks.rados.rados.0.smithi163.stdout:2501: finishing write tid 2 to smithi16367185-35 2023-10-09T09:16:15.208 INFO:tasks.rados.rados.0.smithi163.stdout:2501: finishing write tid 3 to smithi16367185-35 2023-10-09T09:16:15.208 INFO:tasks.rados.rados.0.smithi163.stdout:2501: finishing write tid 4 to smithi16367185-35 2023-10-09T09:16:15.208 INFO:tasks.rados.rados.0.smithi163.stdout:2503: read oid 34 snap 306 2023-10-09T09:16:15.208 INFO:tasks.rados.rados.0.smithi163.stdout:2503: expect (ObjNum 877 snap 305 seq_num 877) 2023-10-09T09:16:15.208 INFO:tasks.rados.rados.0.smithi163.stdout:2504: snap_remove snap 316 2023-10-09T09:16:15.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:15 smithi163 bash[78107]: cluster 2023-10-09T09:16:14.168975+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:15.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:15 smithi163 bash[78107]: cluster 2023-10-09T09:16:14.169000+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:15.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:15 smithi163 bash[78107]: cluster 2023-10-09T09:16:14.550544+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:15 smithi163 bash[78107]: cluster 2023-10-09T09:16:15.154625+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e702 with expected crc 2023-10-09T09:16:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:15 smithi163 bash[78107]: cluster 2023-10-09T09:16:15.159043+0000 mon.a (mon.0) 872 : cluster 0 osdmap e702: 8 total, 8 up, 8 in 2023-10-09T09:16:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:15 smithi163 bash[75425]: cluster 2023-10-09T09:16:14.168975+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:15 smithi163 bash[75425]: cluster 2023-10-09T09:16:14.169000+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:15 smithi163 bash[75425]: cluster 2023-10-09T09:16:14.550544+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:15 smithi163 bash[75425]: cluster 2023-10-09T09:16:15.154625+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e702 with expected crc 2023-10-09T09:16:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:15 smithi163 bash[75425]: cluster 2023-10-09T09:16:15.159043+0000 mon.a (mon.0) 872 : cluster 0 osdmap e702: 8 total, 8 up, 8 in 2023-10-09T09:16:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:15 smithi181 bash[59345]: cluster 2023-10-09T09:16:14.168975+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:15 smithi181 bash[59345]: cluster 2023-10-09T09:16:14.169000+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:15 smithi181 bash[59345]: cluster 2023-10-09T09:16:14.550544+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:15 smithi181 bash[59345]: cluster 2023-10-09T09:16:15.154625+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e702 with expected crc 2023-10-09T09:16:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:15 smithi181 bash[59345]: cluster 2023-10-09T09:16:15.159043+0000 mon.a (mon.0) 872 : cluster 0 osdmap e702: 8 total, 8 up, 8 in 2023-10-09T09:16:16.179 INFO:tasks.rados.rados.0.smithi163.stdout:2504: done (3 left) 2023-10-09T09:16:16.179 INFO:tasks.rados.rados.0.smithi163.stdout:2505: rollback oid 14 current snap is 322 2023-10-09T09:16:16.179 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 314 2023-10-09T09:16:16.179 INFO:tasks.rados.rados.0.smithi163.stdout:2501: finishing write tid 5 to smithi16367185-35 2023-10-09T09:16:16.179 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 660 (ObjNum 939 snap 322 seq_num 939) dirty exists 2023-10-09T09:16:16.180 INFO:tasks.rados.rados.0.smithi163.stdout:2501: left oid 35 (ObjNum 939 snap 322 seq_num 939) 2023-10-09T09:16:16.180 INFO:tasks.rados.rados.0.smithi163.stdout:2502: finishing write tid 1 to smithi16367185-26 2023-10-09T09:16:16.180 INFO:tasks.rados.rados.0.smithi163.stdout:2502: finishing write tid 2 to smithi16367185-26 2023-10-09T09:16:16.180 INFO:tasks.rados.rados.0.smithi163.stdout:2502: finishing write tid 3 to smithi16367185-26 2023-10-09T09:16:16.180 INFO:tasks.rados.rados.0.smithi163.stdout:2502: finishing write tid 4 to smithi16367185-26 2023-10-09T09:16:16.180 INFO:tasks.rados.rados.0.smithi163.stdout:2502: finishing write tid 5 to smithi16367185-26 2023-10-09T09:16:16.180 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 678 (ObjNum 940 snap 322 seq_num 940) dirty exists 2023-10-09T09:16:16.181 INFO:tasks.rados.rados.0.smithi163.stdout:2502: left oid 26 (ObjNum 940 snap 322 seq_num 940) 2023-10-09T09:16:16.181 INFO:tasks.rados.rados.0.smithi163.stdout:2503: expect (ObjNum 877 snap 305 seq_num 877) 2023-10-09T09:16:16.276 INFO:tasks.rados.rados.0.smithi163.stdout:2505: finishing rollback tid 0 to smithi16367185-14 2023-10-09T09:16:16.276 INFO:tasks.rados.rados.0.smithi163.stdout:2505: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:16:16.276 INFO:tasks.rados.rados.0.smithi163.stdout:2505: finishing rollback tid 2 to smithi16367185-14 2023-10-09T09:16:16.276 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1147 (ObjNum 909 snap 311 seq_num 909) dirty exists 2023-10-09T09:16:16.276 INFO:tasks.rados.rados.0.smithi163.stdout:2501: done (3 left) 2023-10-09T09:16:16.276 INFO:tasks.rados.rados.0.smithi163.stdout:2502: done (2 left) 2023-10-09T09:16:16.276 INFO:tasks.rados.rados.0.smithi163.stdout:2503: done (1 left) 2023-10-09T09:16:16.276 INFO:tasks.rados.rados.0.smithi163.stdout:2505: done (0 left) 2023-10-09T09:16:16.276 INFO:tasks.rados.rados.0.smithi163.stdout:2506: read oid 36 snap -1 2023-10-09T09:16:16.277 INFO:tasks.rados.rados.0.smithi163.stdout:2506: expect (ObjNum 884 snap 308 seq_num 884) 2023-10-09T09:16:16.277 INFO:tasks.rados.rados.0.smithi163.stdout:2507: rollback oid 7 current snap is 322 2023-10-09T09:16:16.277 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 318 2023-10-09T09:16:16.277 INFO:tasks.rados.rados.0.smithi163.stdout:2508: write oid 38 current snap is 322 2023-10-09T09:16:16.277 INFO:tasks.rados.rados.0.smithi163.stdout:2508: seq_num 941 ranges {615387=720728,1781791=530321,3108095=707292} 2023-10-09T09:16:16.288 INFO:tasks.rados.rados.0.smithi163.stdout:2508: writing smithi16367185-38 from 615387 to 1336115 tid 1 2023-10-09T09:16:16.295 INFO:tasks.rados.rados.0.smithi163.stdout:2508: writing smithi16367185-38 from 1781791 to 2312112 tid 2 2023-10-09T09:16:16.303 INFO:tasks.rados.rados.0.smithi163.stdout:2508: writing smithi16367185-38 from 3108095 to 3815387 tid 3 2023-10-09T09:16:16.304 INFO:tasks.rados.rados.0.smithi163.stdout:2509: read oid 25 snap -1 2023-10-09T09:16:16.304 INFO:tasks.rados.rados.0.smithi163.stdout:2509: expect deleted 2023-10-09T09:16:16.304 INFO:tasks.rados.rados.0.smithi163.stdout:2510: read oid 48 snap -1 2023-10-09T09:16:16.304 INFO:tasks.rados.rados.0.smithi163.stdout:2510: expect (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:16:16.304 INFO:tasks.rados.rados.0.smithi163.stdout:2511: delete oid 1 current snap is 322 2023-10-09T09:16:16.304 INFO:tasks.rados.rados.0.smithi163.stdout:2506: expect (ObjNum 884 snap 308 seq_num 884) 2023-10-09T09:16:16.465 INFO:tasks.rados.rados.0.smithi163.stdout:2507: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:16:16.465 INFO:tasks.rados.rados.0.smithi163.stdout:2507: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:16:16.465 INFO:tasks.rados.rados.0.smithi163.stdout:2507: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:16:16.465 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 561 (ObjNum 856 snap 291 seq_num 856) dirty exists 2023-10-09T09:16:16.466 INFO:tasks.rados.rados.0.smithi163.stdout:2508: finishing write tid 1 to smithi16367185-38 2023-10-09T09:16:16.466 INFO:tasks.rados.rados.0.smithi163.stdout:2508: finishing write tid 2 to smithi16367185-38 2023-10-09T09:16:16.466 INFO:tasks.rados.rados.0.smithi163.stdout:2506: done (5 left) 2023-10-09T09:16:16.466 INFO:tasks.rados.rados.0.smithi163.stdout:2507: done (4 left) 2023-10-09T09:16:16.466 INFO:tasks.rados.rados.0.smithi163.stdout:2509: done (3 left) 2023-10-09T09:16:16.466 INFO:tasks.rados.rados.0.smithi163.stdout:2511: done (2 left) 2023-10-09T09:16:16.466 INFO:tasks.rados.rados.0.smithi163.stdout:2512: write oid 1 current snap is 322 2023-10-09T09:16:16.466 INFO:tasks.rados.rados.0.smithi163.stdout:2508: finishing write tid 3 to smithi16367185-38 2023-10-09T09:16:16.466 INFO:tasks.rados.rados.0.smithi163.stdout:2512: seq_num 943 ranges {649001=587579,1898755=533990,3162941=686060} 2023-10-09T09:16:16.477 INFO:tasks.rados.rados.0.smithi163.stdout:2512: writing smithi16367185-1 from 649001 to 1236580 tid 1 2023-10-09T09:16:16.484 INFO:tasks.rados.rados.0.smithi163.stdout:2512: writing smithi16367185-1 from 1898755 to 2432745 tid 2 2023-10-09T09:16:16.492 INFO:tasks.rados.rados.0.smithi163.stdout:2512: writing smithi16367185-1 from 3162941 to 3849001 tid 3 2023-10-09T09:16:16.493 INFO:tasks.rados.rados.0.smithi163.stdout:2513: delete oid 11 current snap is 322 2023-10-09T09:16:16.493 INFO:tasks.rados.rados.0.smithi163.stdout:2508: finishing write tid 4 to smithi16367185-38 2023-10-09T09:16:16.493 INFO:tasks.rados.rados.0.smithi163.stdout:2508: finishing write tid 5 to smithi16367185-38 2023-10-09T09:16:16.493 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1152 (ObjNum 941 snap 322 seq_num 941) dirty exists 2023-10-09T09:16:16.493 INFO:tasks.rados.rados.0.smithi163.stdout:2508: left oid 38 (ObjNum 941 snap 322 seq_num 941) 2023-10-09T09:16:16.493 INFO:tasks.rados.rados.0.smithi163.stdout:2510: expect (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:16:16.665 INFO:tasks.rados.rados.0.smithi163.stdout:2512: finishing write tid 1 to smithi16367185-1 2023-10-09T09:16:16.665 INFO:tasks.rados.rados.0.smithi163.stdout:2512: finishing write tid 2 to smithi16367185-1 2023-10-09T09:16:16.665 INFO:tasks.rados.rados.0.smithi163.stdout:2512: finishing write tid 3 to smithi16367185-1 2023-10-09T09:16:16.665 INFO:tasks.rados.rados.0.smithi163.stdout:2512: finishing write tid 4 to smithi16367185-1 2023-10-09T09:16:16.665 INFO:tasks.rados.rados.0.smithi163.stdout:2512: finishing write tid 5 to smithi16367185-1 2023-10-09T09:16:16.666 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1204 (ObjNum 943 snap 322 seq_num 943) dirty exists 2023-10-09T09:16:16.666 INFO:tasks.rados.rados.0.smithi163.stdout:2512: left oid 1 (ObjNum 943 snap 322 seq_num 943) 2023-10-09T09:16:16.666 INFO:tasks.rados.rados.0.smithi163.stdout:2508: done (3 left) 2023-10-09T09:16:16.666 INFO:tasks.rados.rados.0.smithi163.stdout:2510: done (2 left) 2023-10-09T09:16:16.666 INFO:tasks.rados.rados.0.smithi163.stdout:2512: done (1 left) 2023-10-09T09:16:16.666 INFO:tasks.rados.rados.0.smithi163.stdout:2513: done (0 left) 2023-10-09T09:16:16.666 INFO:tasks.rados.rados.0.smithi163.stdout:2514: delete oid 11 current snap is 322 2023-10-09T09:16:16.669 INFO:tasks.rados.rados.0.smithi163.stdout:2514: done (0 left) 2023-10-09T09:16:16.669 INFO:tasks.rados.rados.0.smithi163.stdout:2515: write oid 36 current snap is 322 2023-10-09T09:16:16.669 INFO:tasks.rados.rados.0.smithi163.stdout:2515: seq_num 946 ranges {699422=413326,1557848=681317,3012504=748383} 2023-10-09T09:16:16.680 INFO:tasks.rados.rados.0.smithi163.stdout:2515: writing smithi16367185-36 from 699422 to 1112748 tid 1 2023-10-09T09:16:16.687 INFO:tasks.rados.rados.0.smithi163.stdout:2515: writing smithi16367185-36 from 1557848 to 2239165 tid 2 2023-10-09T09:16:16.695 INFO:tasks.rados.rados.0.smithi163.stdout:2515: writing smithi16367185-36 from 3012504 to 3760887 tid 3 2023-10-09T09:16:16.696 INFO:tasks.rados.rados.0.smithi163.stdout:2516: delete oid 13 current snap is 322 2023-10-09T09:16:16.696 INFO:tasks.rados.rados.0.smithi163.stdout:2515: finishing write tid 1 to smithi16367185-36 2023-10-09T09:16:16.696 INFO:tasks.rados.rados.0.smithi163.stdout:2515: finishing write tid 2 to smithi16367185-36 2023-10-09T09:16:16.699 INFO:tasks.rados.rados.0.smithi163.stdout:2516: done (1 left) 2023-10-09T09:16:16.699 INFO:tasks.rados.rados.0.smithi163.stdout:2517: read oid 45 snap -1 2023-10-09T09:16:16.699 INFO:tasks.rados.rados.0.smithi163.stdout:2517: expect (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:16:16.699 INFO:tasks.rados.rados.0.smithi163.stdout:2518: snap_remove snap 300 2023-10-09T09:16:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[78107]: cluster 2023-10-09T09:16:14.543381+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[78107]: cluster 2023-10-09T09:16:15.154064+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e702 with expected crc 2023-10-09T09:16:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[78107]: cluster 2023-10-09T09:16:15.174846+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e702 with expected crc 2023-10-09T09:16:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[78107]: cluster 2023-10-09T09:16:15.364113+0000 mgr.y (mgr.34107) 291 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 217 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 812 KiB/s wr, 4 op/s 2023-10-09T09:16:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[78107]: cluster 2023-10-09T09:16:16.178037+0000 mon.a (mon.0) 873 : cluster 0 osdmap e703: 8 total, 8 up, 8 in 2023-10-09T09:16:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[78107]: cluster 2023-10-09T09:16:16.179380+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[75425]: cluster 2023-10-09T09:16:14.543381+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[75425]: cluster 2023-10-09T09:16:15.154064+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e702 with expected crc 2023-10-09T09:16:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[75425]: cluster 2023-10-09T09:16:15.174846+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e702 with expected crc 2023-10-09T09:16:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[75425]: cluster 2023-10-09T09:16:15.364113+0000 mgr.y (mgr.34107) 291 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 217 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 812 KiB/s wr, 4 op/s 2023-10-09T09:16:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[75425]: cluster 2023-10-09T09:16:16.178037+0000 mon.a (mon.0) 873 : cluster 0 osdmap e703: 8 total, 8 up, 8 in 2023-10-09T09:16:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:16 smithi163 bash[75425]: cluster 2023-10-09T09:16:16.179380+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:16 smithi181 bash[59345]: cluster 2023-10-09T09:16:14.543381+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e701 with expected crc 2023-10-09T09:16:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:16 smithi181 bash[59345]: cluster 2023-10-09T09:16:15.154064+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e702 with expected crc 2023-10-09T09:16:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:16 smithi181 bash[59345]: cluster 2023-10-09T09:16:15.174846+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e702 with expected crc 2023-10-09T09:16:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:16 smithi181 bash[59345]: cluster 2023-10-09T09:16:15.364113+0000 mgr.y (mgr.34107) 291 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 217 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 812 KiB/s wr, 4 op/s 2023-10-09T09:16:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:16 smithi181 bash[59345]: cluster 2023-10-09T09:16:16.178037+0000 mon.a (mon.0) 873 : cluster 0 osdmap e703: 8 total, 8 up, 8 in 2023-10-09T09:16:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:16 smithi181 bash[59345]: cluster 2023-10-09T09:16:16.179380+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:17.386 INFO:tasks.rados.rados.0.smithi163.stdout:2518: done (2 left) 2023-10-09T09:16:17.386 INFO:tasks.rados.rados.0.smithi163.stdout:2519: delete oid 34 current snap is 322 2023-10-09T09:16:17.386 INFO:tasks.rados.rados.0.smithi163.stdout:2515: finishing write tid 3 to smithi16367185-36 2023-10-09T09:16:17.387 INFO:tasks.rados.rados.0.smithi163.stdout:2515: finishing write tid 4 to smithi16367185-36 2023-10-09T09:16:17.387 INFO:tasks.rados.rados.0.smithi163.stdout:2515: finishing write tid 5 to smithi16367185-36 2023-10-09T09:16:17.387 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 680 (ObjNum 946 snap 322 seq_num 946) dirty exists 2023-10-09T09:16:17.387 INFO:tasks.rados.rados.0.smithi163.stdout:2515: left oid 36 (ObjNum 946 snap 322 seq_num 946) 2023-10-09T09:16:17.387 INFO:tasks.rados.rados.0.smithi163.stdout:2517: expect (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:16:17.470 INFO:tasks.rados.rados.0.smithi163.stdout:2515: done (2 left) 2023-10-09T09:16:17.471 INFO:tasks.rados.rados.0.smithi163.stdout:2517: done (1 left) 2023-10-09T09:16:17.471 INFO:tasks.rados.rados.0.smithi163.stdout:2519: done (0 left) 2023-10-09T09:16:17.471 INFO:tasks.rados.rados.0.smithi163.stdout:2520: snap_create 2023-10-09T09:16:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:17 smithi163 bash[78107]: cluster 2023-10-09T09:16:16.174117+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:17 smithi163 bash[78107]: cluster 2023-10-09T09:16:16.174994+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:17 smithi163 bash[78107]: cluster 2023-10-09T09:16:16.313987+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:17 smithi163 bash[75425]: cluster 2023-10-09T09:16:16.174117+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:17 smithi163 bash[75425]: cluster 2023-10-09T09:16:16.174994+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:17 smithi163 bash[75425]: cluster 2023-10-09T09:16:16.313987+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:17 smithi181 bash[59345]: cluster 2023-10-09T09:16:16.174117+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:17 smithi181 bash[59345]: cluster 2023-10-09T09:16:16.174994+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:17 smithi181 bash[59345]: cluster 2023-10-09T09:16:16.313987+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e703 with expected crc 2023-10-09T09:16:18.396 INFO:tasks.rados.rados.0.smithi163.stdout:2520: done (0 left) 2023-10-09T09:16:18.396 INFO:tasks.rados.rados.0.smithi163.stdout:2521: delete oid 11 current snap is 323 2023-10-09T09:16:18.404 INFO:tasks.rados.rados.0.smithi163.stdout:2521: done (0 left) 2023-10-09T09:16:18.404 INFO:tasks.rados.rados.0.smithi163.stdout:2522: read oid 13 snap -1 2023-10-09T09:16:18.405 INFO:tasks.rados.rados.0.smithi163.stdout:2522: expect deleted 2023-10-09T09:16:18.405 INFO:tasks.rados.rados.0.smithi163.stdout:2523: snap_create 2023-10-09T09:16:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:18 smithi163 bash[78107]: cluster 2023-10-09T09:16:17.365166+0000 mgr.y (mgr.34107) 292 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 227 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2023-10-09T09:16:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:18 smithi163 bash[78107]: cluster 2023-10-09T09:16:17.380019+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e704 with expected crc 2023-10-09T09:16:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:18 smithi163 bash[78107]: cluster 2023-10-09T09:16:17.384232+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e704 with expected crc 2023-10-09T09:16:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:18 smithi163 bash[78107]: cluster 2023-10-09T09:16:17.384882+0000 mon.a (mon.0) 874 : cluster 0 osdmap e704: 8 total, 8 up, 8 in 2023-10-09T09:16:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:18 smithi163 bash[75425]: cluster 2023-10-09T09:16:17.365166+0000 mgr.y (mgr.34107) 292 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 227 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2023-10-09T09:16:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:18 smithi163 bash[75425]: cluster 2023-10-09T09:16:17.380019+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e704 with expected crc 2023-10-09T09:16:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:18 smithi163 bash[75425]: cluster 2023-10-09T09:16:17.384232+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e704 with expected crc 2023-10-09T09:16:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:18 smithi163 bash[75425]: cluster 2023-10-09T09:16:17.384882+0000 mon.a (mon.0) 874 : cluster 0 osdmap e704: 8 total, 8 up, 8 in 2023-10-09T09:16:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:18 smithi181 bash[59345]: cluster 2023-10-09T09:16:17.365166+0000 mgr.y (mgr.34107) 292 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 227 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2023-10-09T09:16:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:18 smithi181 bash[59345]: cluster 2023-10-09T09:16:17.380019+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e704 with expected crc 2023-10-09T09:16:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:18 smithi181 bash[59345]: cluster 2023-10-09T09:16:17.384232+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e704 with expected crc 2023-10-09T09:16:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:18 smithi181 bash[59345]: cluster 2023-10-09T09:16:17.384882+0000 mon.a (mon.0) 874 : cluster 0 osdmap e704: 8 total, 8 up, 8 in 2023-10-09T09:16:19.411 INFO:tasks.rados.rados.0.smithi163.stdout:2522: done (1 left) 2023-10-09T09:16:19.411 INFO:tasks.rados.rados.0.smithi163.stdout:2523: done (0 left) 2023-10-09T09:16:19.412 INFO:tasks.rados.rados.0.smithi163.stdout:2524: write oid 7 current snap is 324 2023-10-09T09:16:19.412 INFO:tasks.rados.rados.0.smithi163.stdout:2524: seq_num 950 ranges {766650=714322,2075423=688655,3522196=444454} 2023-10-09T09:16:19.422 INFO:tasks.rados.rados.0.smithi163.stdout:2524: writing smithi16367185-7 from 766650 to 1480972 tid 1 2023-10-09T09:16:19.430 INFO:tasks.rados.rados.0.smithi163.stdout:2524: writing smithi16367185-7 from 2075423 to 2764078 tid 2 2023-10-09T09:16:19.438 INFO:tasks.rados.rados.0.smithi163.stdout:2524: writing smithi16367185-7 from 3522196 to 3966650 tid 3 2023-10-09T09:16:19.438 INFO:tasks.rados.rados.0.smithi163.stdout:2524: finishing write tid 1 to smithi16367185-7 2023-10-09T09:16:19.438 INFO:tasks.rados.rados.0.smithi163.stdout:2524: finishing write tid 2 to smithi16367185-7 2023-10-09T09:16:19.438 INFO:tasks.rados.rados.0.smithi163.stdout:2525: read oid 10 snap -1 2023-10-09T09:16:19.438 INFO:tasks.rados.rados.0.smithi163.stdout:2525: expect (ObjNum 931 snap 315 seq_num 931) 2023-10-09T09:16:19.438 INFO:tasks.rados.rados.0.smithi163.stdout:2526: snap_remove snap 321 2023-10-09T09:16:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:19 smithi163 bash[78107]: cluster 2023-10-09T09:16:18.390639+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e705 with expected crc 2023-10-09T09:16:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:19 smithi163 bash[78107]: cluster 2023-10-09T09:16:18.395830+0000 mon.a (mon.0) 875 : cluster 0 osdmap e705: 8 total, 8 up, 8 in 2023-10-09T09:16:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:19 smithi163 bash[78107]: cluster 2023-10-09T09:16:18.396535+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e705 with expected crc 2023-10-09T09:16:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:19 smithi163 bash[75425]: cluster 2023-10-09T09:16:18.390639+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e705 with expected crc 2023-10-09T09:16:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:19 smithi163 bash[75425]: cluster 2023-10-09T09:16:18.395830+0000 mon.a (mon.0) 875 : cluster 0 osdmap e705: 8 total, 8 up, 8 in 2023-10-09T09:16:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:19 smithi163 bash[75425]: cluster 2023-10-09T09:16:18.396535+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e705 with expected crc 2023-10-09T09:16:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:19 smithi181 bash[59345]: cluster 2023-10-09T09:16:18.390639+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e705 with expected crc 2023-10-09T09:16:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:19 smithi181 bash[59345]: cluster 2023-10-09T09:16:18.395830+0000 mon.a (mon.0) 875 : cluster 0 osdmap e705: 8 total, 8 up, 8 in 2023-10-09T09:16:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:19 smithi181 bash[59345]: cluster 2023-10-09T09:16:18.396535+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e705 with expected crc 2023-10-09T09:16:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:2526: done (2 left) 2023-10-09T09:16:20.162 INFO:tasks.rados.rados.0.smithi163.stdout:2527: snap_remove snap 272 2023-10-09T09:16:20.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:20 smithi163 bash[75425]: cluster 2023-10-09T09:16:19.366320+0000 mgr.y (mgr.34107) 293 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 227 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2023-10-09T09:16:20.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:20 smithi163 bash[75425]: cluster 2023-10-09T09:16:19.403623+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:20.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:20 smithi163 bash[75425]: cluster 2023-10-09T09:16:19.411402+0000 mon.a (mon.0) 876 : cluster 0 osdmap e706: 8 total, 8 up, 8 in 2023-10-09T09:16:20.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:20 smithi163 bash[75425]: cluster 2023-10-09T09:16:19.428338+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:20 smithi163 bash[75425]: cluster 2023-10-09T09:16:20.161908+0000 mon.a (mon.0) 877 : cluster 0 osdmap e707: 8 total, 8 up, 8 in 2023-10-09T09:16:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:20 smithi163 bash[78107]: cluster 2023-10-09T09:16:19.366320+0000 mgr.y (mgr.34107) 293 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 227 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2023-10-09T09:16:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:20 smithi163 bash[78107]: cluster 2023-10-09T09:16:19.403623+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:20 smithi163 bash[78107]: cluster 2023-10-09T09:16:19.411402+0000 mon.a (mon.0) 876 : cluster 0 osdmap e706: 8 total, 8 up, 8 in 2023-10-09T09:16:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:20 smithi163 bash[78107]: cluster 2023-10-09T09:16:19.428338+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:20 smithi163 bash[78107]: cluster 2023-10-09T09:16:20.161908+0000 mon.a (mon.0) 877 : cluster 0 osdmap e707: 8 total, 8 up, 8 in 2023-10-09T09:16:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:20 smithi181 bash[59345]: cluster 2023-10-09T09:16:19.366320+0000 mgr.y (mgr.34107) 293 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 227 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2023-10-09T09:16:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:20 smithi181 bash[59345]: cluster 2023-10-09T09:16:19.403623+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:20 smithi181 bash[59345]: cluster 2023-10-09T09:16:19.411402+0000 mon.a (mon.0) 876 : cluster 0 osdmap e706: 8 total, 8 up, 8 in 2023-10-09T09:16:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:20 smithi181 bash[59345]: cluster 2023-10-09T09:16:19.428338+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:20 smithi181 bash[59345]: cluster 2023-10-09T09:16:20.161908+0000 mon.a (mon.0) 877 : cluster 0 osdmap e707: 8 total, 8 up, 8 in 2023-10-09T09:16:21.165 INFO:tasks.rados.rados.0.smithi163.stdout:2527: done (2 left) 2023-10-09T09:16:21.166 INFO:tasks.rados.rados.0.smithi163.stdout:2528: write oid 34 current snap is 324 2023-10-09T09:16:21.166 INFO:tasks.rados.rados.0.smithi163.stdout:2524: finishing write tid 3 to smithi16367185-7 2023-10-09T09:16:21.166 INFO:tasks.rados.rados.0.smithi163.stdout:2528: seq_num 951 ranges {783457=505924,1850258=548666,2865972=721096} 2023-10-09T09:16:21.178 INFO:tasks.rados.rados.0.smithi163.stdout:2528: writing smithi16367185-34 from 783457 to 1289381 tid 1 2023-10-09T09:16:21.185 INFO:tasks.rados.rados.0.smithi163.stdout:2528: writing smithi16367185-34 from 1850258 to 2398924 tid 2 2023-10-09T09:16:21.193 INFO:tasks.rados.rados.0.smithi163.stdout:2528: writing smithi16367185-34 from 2865972 to 3587068 tid 3 2023-10-09T09:16:21.193 INFO:tasks.rados.rados.0.smithi163.stdout:2529: write oid 20 current snap is 324 2023-10-09T09:16:21.193 INFO:tasks.rados.rados.0.smithi163.stdout:2524: finishing write tid 4 to smithi16367185-7 2023-10-09T09:16:21.193 INFO:tasks.rados.rados.0.smithi163.stdout:2524: finishing write tid 5 to smithi16367185-7 2023-10-09T09:16:21.193 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 570 (ObjNum 950 snap 324 seq_num 950) dirty exists 2023-10-09T09:16:21.193 INFO:tasks.rados.rados.0.smithi163.stdout:2524: left oid 7 (ObjNum 950 snap 324 seq_num 950) 2023-10-09T09:16:21.193 INFO:tasks.rados.rados.0.smithi163.stdout:2525: expect (ObjNum 931 snap 315 seq_num 931) 2023-10-09T09:16:21.424 INFO:tasks.rados.rados.0.smithi163.stdout:2528: finishing write tid 1 to smithi16367185-34 2023-10-09T09:16:21.425 INFO:tasks.rados.rados.0.smithi163.stdout:2528: finishing write tid 2 to smithi16367185-34 2023-10-09T09:16:21.425 INFO:tasks.rados.rados.0.smithi163.stdout:2528: finishing write tid 3 to smithi16367185-34 2023-10-09T09:16:21.425 INFO:tasks.rados.rados.0.smithi163.stdout:2528: finishing write tid 4 to smithi16367185-34 2023-10-09T09:16:21.425 INFO:tasks.rados.rados.0.smithi163.stdout:2528: finishing write tid 5 to smithi16367185-34 2023-10-09T09:16:21.425 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1204 (ObjNum 951 snap 324 seq_num 951) dirty exists 2023-10-09T09:16:21.425 INFO:tasks.rados.rados.0.smithi163.stdout:2528: left oid 34 (ObjNum 951 snap 324 seq_num 951) 2023-10-09T09:16:21.425 INFO:tasks.rados.rados.0.smithi163.stdout:2529: seq_num 952 ranges {400264=581173,1508740=491524} 2023-10-09T09:16:21.435 INFO:tasks.rados.rados.0.smithi163.stdout:2529: writing smithi16367185-20 from 400264 to 981437 tid 1 2023-10-09T09:16:21.444 INFO:tasks.rados.rados.0.smithi163.stdout:2529: writing smithi16367185-20 from 1508740 to 2000264 tid 2 2023-10-09T09:16:21.444 INFO:tasks.rados.rados.0.smithi163.stdout:2524: done (3 left) 2023-10-09T09:16:21.444 INFO:tasks.rados.rados.0.smithi163.stdout:2525: done (2 left) 2023-10-09T09:16:21.444 INFO:tasks.rados.rados.0.smithi163.stdout:2528: done (1 left) 2023-10-09T09:16:21.444 INFO:tasks.rados.rados.0.smithi163.stdout:2530: read oid 37 snap -1 2023-10-09T09:16:21.444 INFO:tasks.rados.rados.0.smithi163.stdout:2530: expect deleted 2023-10-09T09:16:21.444 INFO:tasks.rados.rados.0.smithi163.stdout:2531: snap_remove snap 311 2023-10-09T09:16:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:21 smithi163 bash[78107]: cluster 2023-10-09T09:16:20.033746+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:21 smithi163 bash[78107]: cluster 2023-10-09T09:16:20.033798+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:21 smithi163 bash[78107]: cluster 2023-10-09T09:16:20.156092+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e707 with expected crc 2023-10-09T09:16:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:21 smithi163 bash[78107]: cluster 2023-10-09T09:16:20.163848+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e707 with expected crc 2023-10-09T09:16:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:21 smithi163 bash[78107]: cluster 2023-10-09T09:16:21.165489+0000 mon.a (mon.0) 878 : cluster 0 osdmap e708: 8 total, 8 up, 8 in 2023-10-09T09:16:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:21 smithi163 bash[75425]: cluster 2023-10-09T09:16:20.033746+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:21 smithi163 bash[75425]: cluster 2023-10-09T09:16:20.033798+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:21 smithi163 bash[75425]: cluster 2023-10-09T09:16:20.156092+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e707 with expected crc 2023-10-09T09:16:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:21 smithi163 bash[75425]: cluster 2023-10-09T09:16:20.163848+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e707 with expected crc 2023-10-09T09:16:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:21 smithi163 bash[75425]: cluster 2023-10-09T09:16:21.165489+0000 mon.a (mon.0) 878 : cluster 0 osdmap e708: 8 total, 8 up, 8 in 2023-10-09T09:16:21.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:21 smithi181 bash[59345]: cluster 2023-10-09T09:16:20.033746+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:21 smithi181 bash[59345]: cluster 2023-10-09T09:16:20.033798+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e706 with expected crc 2023-10-09T09:16:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:21 smithi181 bash[59345]: cluster 2023-10-09T09:16:20.156092+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e707 with expected crc 2023-10-09T09:16:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:21 smithi181 bash[59345]: cluster 2023-10-09T09:16:20.163848+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e707 with expected crc 2023-10-09T09:16:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:21 smithi181 bash[59345]: cluster 2023-10-09T09:16:21.165489+0000 mon.a (mon.0) 878 : cluster 0 osdmap e708: 8 total, 8 up, 8 in 2023-10-09T09:16:22.414 INFO:tasks.rados.rados.0.smithi163.stdout:2531: done (2 left) 2023-10-09T09:16:22.414 INFO:tasks.rados.rados.0.smithi163.stdout:2532: write oid 12 current snap is 324 2023-10-09T09:16:22.415 INFO:tasks.rados.rados.0.smithi163.stdout:2529: finishing write tid 1 to smithi16367185-20 2023-10-09T09:16:22.415 INFO:tasks.rados.rados.0.smithi163.stdout:2532: seq_num 953 ranges {417071=656422,1567222=449849} 2023-10-09T09:16:22.424 INFO:tasks.rados.rados.0.smithi163.stdout:2532: writing smithi16367185-12 from 417071 to 1073493 tid 1 2023-10-09T09:16:22.431 INFO:tasks.rados.rados.0.smithi163.stdout:2532: writing smithi16367185-12 from 1567222 to 2017071 tid 2 2023-10-09T09:16:22.431 INFO:tasks.rados.rados.0.smithi163.stdout:2533: write oid 5 current snap is 324 2023-10-09T09:16:22.431 INFO:tasks.rados.rados.0.smithi163.stdout:2533: seq_num 954 ranges {433878=731671,1909351=124527} 2023-10-09T09:16:22.438 INFO:tasks.rados.rados.0.smithi163.stdout:2533: writing smithi16367185-5 from 433878 to 1165549 tid 1 2023-10-09T09:16:22.443 INFO:tasks.rados.rados.0.smithi163.stdout:2533: writing smithi16367185-5 from 1909351 to 2033878 tid 2 2023-10-09T09:16:22.443 INFO:tasks.rados.rados.0.smithi163.stdout:2534: snap_remove snap 317 2023-10-09T09:16:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:22 smithi163 bash[78107]: cluster 2023-10-09T09:16:21.161428+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e708 with expected crc 2023-10-09T09:16:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:22 smithi163 bash[78107]: cluster 2023-10-09T09:16:21.161640+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e708 with expected crc 2023-10-09T09:16:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:22 smithi163 bash[78107]: cluster 2023-10-09T09:16:21.366978+0000 mgr.y (mgr.34107) 294 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2023-10-09T09:16:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:22 smithi163 bash[75425]: cluster 2023-10-09T09:16:21.161428+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e708 with expected crc 2023-10-09T09:16:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:22 smithi163 bash[75425]: cluster 2023-10-09T09:16:21.161640+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e708 with expected crc 2023-10-09T09:16:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:22 smithi163 bash[75425]: cluster 2023-10-09T09:16:21.366978+0000 mgr.y (mgr.34107) 294 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2023-10-09T09:16:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:22 smithi181 bash[59345]: cluster 2023-10-09T09:16:21.161428+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e708 with expected crc 2023-10-09T09:16:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:22 smithi181 bash[59345]: cluster 2023-10-09T09:16:21.161640+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e708 with expected crc 2023-10-09T09:16:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:22 smithi181 bash[59345]: cluster 2023-10-09T09:16:21.366978+0000 mgr.y (mgr.34107) 294 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2023-10-09T09:16:23.439 INFO:tasks.rados.rados.0.smithi163.stdout:2534: done (4 left) 2023-10-09T09:16:23.439 INFO:tasks.rados.rados.0.smithi163.stdout:2535: read oid 40 snap -1 2023-10-09T09:16:23.439 INFO:tasks.rados.rados.0.smithi163.stdout:2535: expect (ObjNum 876 snap 305 seq_num 876) 2023-10-09T09:16:23.439 INFO:tasks.rados.rados.0.smithi163.stdout:2536: read oid 7 snap 304 2023-10-09T09:16:23.440 INFO:tasks.rados.rados.0.smithi163.stdout:2536: expect (ObjNum 856 snap 291 seq_num 856) 2023-10-09T09:16:23.440 INFO:tasks.rados.rados.0.smithi163.stdout:2537: write oid 1 current snap is 324 2023-10-09T09:16:23.440 INFO:tasks.rados.rados.0.smithi163.stdout:2529: finishing write tid 2 to smithi16367185-20 2023-10-09T09:16:23.440 INFO:tasks.rados.rados.0.smithi163.stdout:2537: seq_num 955 ranges {450685=406920,1567833=482852} 2023-10-09T09:16:23.447 INFO:tasks.rados.rados.0.smithi163.stdout:2537: writing smithi16367185-1 from 450685 to 857605 tid 1 2023-10-09T09:16:23.456 INFO:tasks.rados.rados.0.smithi163.stdout:2537: writing smithi16367185-1 from 1567833 to 2050685 tid 2 2023-10-09T09:16:23.456 INFO:tasks.rados.rados.0.smithi163.stdout:2529: finishing write tid 3 to smithi16367185-20 2023-10-09T09:16:23.456 INFO:tasks.rados.rados.0.smithi163.stdout:2529: finishing write tid 4 to smithi16367185-20 2023-10-09T09:16:23.456 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 387 (ObjNum 952 snap 324 seq_num 952) dirty exists 2023-10-09T09:16:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:2529: left oid 20 (ObjNum 952 snap 324 seq_num 952) 2023-10-09T09:16:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:2532: finishing write tid 1 to smithi16367185-12 2023-10-09T09:16:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:2532: finishing write tid 2 to smithi16367185-12 2023-10-09T09:16:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:2532: finishing write tid 3 to smithi16367185-12 2023-10-09T09:16:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:2532: finishing write tid 4 to smithi16367185-12 2023-10-09T09:16:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 1231 (ObjNum 953 snap 324 seq_num 953) dirty exists 2023-10-09T09:16:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:2532: left oid 12 (ObjNum 953 snap 324 seq_num 953) 2023-10-09T09:16:23.457 INFO:tasks.rados.rados.0.smithi163.stdout:2533: finishing write tid 1 to smithi16367185-5 2023-10-09T09:16:23.458 INFO:tasks.rados.rados.0.smithi163.stdout:2533: finishing write tid 2 to smithi16367185-5 2023-10-09T09:16:23.458 INFO:tasks.rados.rados.0.smithi163.stdout:2533: finishing write tid 3 to smithi16367185-5 2023-10-09T09:16:23.458 INFO:tasks.rados.rados.0.smithi163.stdout:2533: finishing write tid 4 to smithi16367185-5 2023-10-09T09:16:23.458 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1215 (ObjNum 954 snap 324 seq_num 954) dirty exists 2023-10-09T09:16:23.458 INFO:tasks.rados.rados.0.smithi163.stdout:2533: left oid 5 (ObjNum 954 snap 324 seq_num 954) 2023-10-09T09:16:23.458 INFO:tasks.rados.rados.0.smithi163.stdout:2536: expect (ObjNum 856 snap 291 seq_num 856) 2023-10-09T09:16:23.588 INFO:tasks.rados.rados.0.smithi163.stdout:2535: expect (ObjNum 876 snap 305 seq_num 876) 2023-10-09T09:16:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:23 smithi163 bash[78107]: cluster 2023-10-09T09:16:22.409526+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e709 with expected crc 2023-10-09T09:16:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:23 smithi163 bash[78107]: cluster 2023-10-09T09:16:22.413817+0000 mon.a (mon.0) 879 : cluster 0 osdmap e709: 8 total, 8 up, 8 in 2023-10-09T09:16:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:23 smithi163 bash[78107]: cluster 2023-10-09T09:16:22.416257+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e709 with expected crc 2023-10-09T09:16:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:16:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:16:23] "GET /metrics HTTP/1.1" 200 36268 "" "Prometheus/2.33.4" 2023-10-09T09:16:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:23 smithi163 bash[75425]: cluster 2023-10-09T09:16:22.409526+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e709 with expected crc 2023-10-09T09:16:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:23 smithi163 bash[75425]: cluster 2023-10-09T09:16:22.413817+0000 mon.a (mon.0) 879 : cluster 0 osdmap e709: 8 total, 8 up, 8 in 2023-10-09T09:16:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:23 smithi163 bash[75425]: cluster 2023-10-09T09:16:22.416257+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e709 with expected crc 2023-10-09T09:16:23.721 INFO:tasks.rados.rados.0.smithi163.stdout:2537: finishing write tid 1 to smithi16367185-1 2023-10-09T09:16:23.721 INFO:tasks.rados.rados.0.smithi163.stdout:2537: finishing write tid 2 to smithi16367185-1 2023-10-09T09:16:23.721 INFO:tasks.rados.rados.0.smithi163.stdout:2537: finishing write tid 3 to smithi16367185-1 2023-10-09T09:16:23.721 INFO:tasks.rados.rados.0.smithi163.stdout:2537: finishing write tid 4 to smithi16367185-1 2023-10-09T09:16:23.721 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1219 (ObjNum 955 snap 324 seq_num 955) dirty exists 2023-10-09T09:16:23.722 INFO:tasks.rados.rados.0.smithi163.stdout:2537: left oid 1 (ObjNum 955 snap 324 seq_num 955) 2023-10-09T09:16:23.722 INFO:tasks.rados.rados.0.smithi163.stdout:2529: done (6 left) 2023-10-09T09:16:23.722 INFO:tasks.rados.rados.0.smithi163.stdout:2530: done (5 left) 2023-10-09T09:16:23.722 INFO:tasks.rados.rados.0.smithi163.stdout:2532: done (4 left) 2023-10-09T09:16:23.722 INFO:tasks.rados.rados.0.smithi163.stdout:2533: done (3 left) 2023-10-09T09:16:23.722 INFO:tasks.rados.rados.0.smithi163.stdout:2535: done (2 left) 2023-10-09T09:16:23.722 INFO:tasks.rados.rados.0.smithi163.stdout:2536: done (1 left) 2023-10-09T09:16:23.722 INFO:tasks.rados.rados.0.smithi163.stdout:2537: done (0 left) 2023-10-09T09:16:23.722 INFO:tasks.rados.rados.0.smithi163.stdout:2538: write oid 6 current snap is 324 2023-10-09T09:16:23.722 INFO:tasks.rados.rados.0.smithi163.stdout:2538: seq_num 956 ranges {467492=482169,1626315=441177} 2023-10-09T09:16:23.731 INFO:tasks.rados.rados.0.smithi163.stdout:2538: writing smithi16367185-6 from 467492 to 949661 tid 1 2023-10-09T09:16:23.738 INFO:tasks.rados.rados.0.smithi163.stdout:2538: writing smithi16367185-6 from 1626315 to 2067492 tid 2 2023-10-09T09:16:23.738 INFO:tasks.rados.rados.0.smithi163.stdout:2539: delete oid 23 current snap is 324 2023-10-09T09:16:23.743 INFO:tasks.rados.rados.0.smithi163.stdout:2538: finishing write tid 1 to smithi16367185-6 2023-10-09T09:16:23.743 INFO:tasks.rados.rados.0.smithi163.stdout:2539: done (1 left) 2023-10-09T09:16:23.743 INFO:tasks.rados.rados.0.smithi163.stdout:2540: read oid 3 snap 306 2023-10-09T09:16:23.743 INFO:tasks.rados.rados.0.smithi163.stdout:2540: expect (ObjNum 789 snap 271 seq_num 789) 2023-10-09T09:16:23.743 INFO:tasks.rados.rados.0.smithi163.stdout:2541: delete oid 43 current snap is 324 2023-10-09T09:16:23.743 INFO:tasks.rados.rados.0.smithi163.stdout:2538: finishing write tid 2 to smithi16367185-6 2023-10-09T09:16:23.744 INFO:tasks.rados.rados.0.smithi163.stdout:2538: finishing write tid 3 to smithi16367185-6 2023-10-09T09:16:23.744 INFO:tasks.rados.rados.0.smithi163.stdout:2538: finishing write tid 4 to smithi16367185-6 2023-10-09T09:16:23.744 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 252 (ObjNum 956 snap 324 seq_num 956) dirty exists 2023-10-09T09:16:23.744 INFO:tasks.rados.rados.0.smithi163.stdout:2538: left oid 6 (ObjNum 956 snap 324 seq_num 956) 2023-10-09T09:16:23.745 INFO:tasks.rados.rados.0.smithi163.stdout:2538: done (2 left) 2023-10-09T09:16:23.745 INFO:tasks.rados.rados.0.smithi163.stdout:2541: done (1 left) 2023-10-09T09:16:23.745 INFO:tasks.rados.rados.0.smithi163.stdout:2542: read oid 25 snap 320 2023-10-09T09:16:23.745 INFO:tasks.rados.rados.0.smithi163.stdout:2542: expect deleted 2023-10-09T09:16:23.746 INFO:tasks.rados.rados.0.smithi163.stdout:2543: snap_remove snap 303 2023-10-09T09:16:23.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:23 smithi181 bash[59345]: cluster 2023-10-09T09:16:22.409526+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e709 with expected crc 2023-10-09T09:16:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:23 smithi181 bash[59345]: cluster 2023-10-09T09:16:22.413817+0000 mon.a (mon.0) 879 : cluster 0 osdmap e709: 8 total, 8 up, 8 in 2023-10-09T09:16:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:23 smithi181 bash[59345]: cluster 2023-10-09T09:16:22.416257+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e709 with expected crc 2023-10-09T09:16:24.445 INFO:tasks.rados.rados.0.smithi163.stdout:2543: done (2 left) 2023-10-09T09:16:24.450 INFO:tasks.rados.rados.0.smithi163.stdout:2544: snap_create 2023-10-09T09:16:24.450 INFO:tasks.rados.rados.0.smithi163.stdout:2540: expect (ObjNum 789 snap 271 seq_num 789) 2023-10-09T09:16:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:24 smithi163 bash[78107]: cluster 2023-10-09T09:16:23.367569+0000 mgr.y (mgr.34107) 295 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2023-10-09T09:16:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:24 smithi163 bash[78107]: cluster 2023-10-09T09:16:23.432690+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e710 with expected crc 2023-10-09T09:16:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:24 smithi163 bash[78107]: cluster 2023-10-09T09:16:23.435489+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e710 with expected crc 2023-10-09T09:16:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:24 smithi163 bash[78107]: cluster 2023-10-09T09:16:23.438495+0000 mon.a (mon.0) 880 : cluster 0 osdmap e710: 8 total, 8 up, 8 in 2023-10-09T09:16:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:24 smithi163 bash[75425]: cluster 2023-10-09T09:16:23.367569+0000 mgr.y (mgr.34107) 295 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2023-10-09T09:16:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:24 smithi163 bash[75425]: cluster 2023-10-09T09:16:23.432690+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e710 with expected crc 2023-10-09T09:16:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:24 smithi163 bash[75425]: cluster 2023-10-09T09:16:23.435489+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e710 with expected crc 2023-10-09T09:16:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:24 smithi163 bash[75425]: cluster 2023-10-09T09:16:23.438495+0000 mon.a (mon.0) 880 : cluster 0 osdmap e710: 8 total, 8 up, 8 in 2023-10-09T09:16:24.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:24 smithi181 bash[59345]: cluster 2023-10-09T09:16:23.367569+0000 mgr.y (mgr.34107) 295 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2023-10-09T09:16:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:24 smithi181 bash[59345]: cluster 2023-10-09T09:16:23.432690+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e710 with expected crc 2023-10-09T09:16:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:24 smithi181 bash[59345]: cluster 2023-10-09T09:16:23.435489+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e710 with expected crc 2023-10-09T09:16:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:24 smithi181 bash[59345]: cluster 2023-10-09T09:16:23.438495+0000 mon.a (mon.0) 880 : cluster 0 osdmap e710: 8 total, 8 up, 8 in 2023-10-09T09:16:25.166 INFO:tasks.rados.rados.0.smithi163.stdout:2540: done (2 left) 2023-10-09T09:16:25.166 INFO:tasks.rados.rados.0.smithi163.stdout:2542: done (1 left) 2023-10-09T09:16:25.166 INFO:tasks.rados.rados.0.smithi163.stdout:2544: done (0 left) 2023-10-09T09:16:25.166 INFO:tasks.rados.rados.0.smithi163.stdout:2545: snap_create 2023-10-09T09:16:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[78107]: cluster 2023-10-09T09:16:23.441227+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e710 with expected crc 2023-10-09T09:16:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[78107]: cluster 2023-10-09T09:16:24.440099+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e711 with expected crc 2023-10-09T09:16:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[78107]: cluster 2023-10-09T09:16:24.443914+0000 mon.a (mon.0) 881 : cluster 0 osdmap e711: 8 total, 8 up, 8 in 2023-10-09T09:16:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[78107]: cluster 2023-10-09T09:16:24.444721+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e711 with expected crc 2023-10-09T09:16:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[78107]: cluster 2023-10-09T09:16:25.165842+0000 mon.a (mon.0) 882 : cluster 0 osdmap e712: 8 total, 8 up, 8 in 2023-10-09T09:16:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[78107]: audit 2023-10-09T09:16:25.345040+0000 mon.a (mon.0) 883 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[75425]: cluster 2023-10-09T09:16:23.441227+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e710 with expected crc 2023-10-09T09:16:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[75425]: cluster 2023-10-09T09:16:24.440099+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e711 with expected crc 2023-10-09T09:16:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[75425]: cluster 2023-10-09T09:16:24.443914+0000 mon.a (mon.0) 881 : cluster 0 osdmap e711: 8 total, 8 up, 8 in 2023-10-09T09:16:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[75425]: cluster 2023-10-09T09:16:24.444721+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e711 with expected crc 2023-10-09T09:16:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[75425]: cluster 2023-10-09T09:16:25.165842+0000 mon.a (mon.0) 882 : cluster 0 osdmap e712: 8 total, 8 up, 8 in 2023-10-09T09:16:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:25 smithi163 bash[75425]: audit 2023-10-09T09:16:25.345040+0000 mon.a (mon.0) 883 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:25 smithi181 bash[59345]: cluster 2023-10-09T09:16:23.441227+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e710 with expected crc 2023-10-09T09:16:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:25 smithi181 bash[59345]: cluster 2023-10-09T09:16:24.440099+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e711 with expected crc 2023-10-09T09:16:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:25 smithi181 bash[59345]: cluster 2023-10-09T09:16:24.443914+0000 mon.a (mon.0) 881 : cluster 0 osdmap e711: 8 total, 8 up, 8 in 2023-10-09T09:16:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:25 smithi181 bash[59345]: cluster 2023-10-09T09:16:24.444721+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e711 with expected crc 2023-10-09T09:16:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:25 smithi181 bash[59345]: cluster 2023-10-09T09:16:25.165842+0000 mon.a (mon.0) 882 : cluster 0 osdmap e712: 8 total, 8 up, 8 in 2023-10-09T09:16:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:25 smithi181 bash[59345]: audit 2023-10-09T09:16:25.345040+0000 mon.a (mon.0) 883 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:26.166 INFO:tasks.rados.rados.0.smithi163.stdout:2545: done (0 left) 2023-10-09T09:16:26.167 INFO:tasks.rados.rados.0.smithi163.stdout:2546: snap_remove snap 304 2023-10-09T09:16:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:26 smithi163 bash[78107]: cluster 2023-10-09T09:16:25.160595+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e712 with expected crc 2023-10-09T09:16:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:26 smithi163 bash[78107]: cluster 2023-10-09T09:16:25.160610+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e712 with expected crc 2023-10-09T09:16:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:26 smithi163 bash[78107]: cluster 2023-10-09T09:16:25.368302+0000 mgr.y (mgr.34107) 296 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 195 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2023-10-09T09:16:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:26 smithi163 bash[78107]: cluster 2023-10-09T09:16:25.381163+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e712 with expected crc 2023-10-09T09:16:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:26 smithi163 bash[78107]: cluster 2023-10-09T09:16:26.166630+0000 mon.a (mon.0) 884 : cluster 0 osdmap e713: 8 total, 8 up, 8 in 2023-10-09T09:16:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:26 smithi163 bash[75425]: cluster 2023-10-09T09:16:25.160595+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e712 with expected crc 2023-10-09T09:16:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:26 smithi163 bash[75425]: cluster 2023-10-09T09:16:25.160610+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e712 with expected crc 2023-10-09T09:16:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:26 smithi163 bash[75425]: cluster 2023-10-09T09:16:25.368302+0000 mgr.y (mgr.34107) 296 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 195 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2023-10-09T09:16:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:26 smithi163 bash[75425]: cluster 2023-10-09T09:16:25.381163+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e712 with expected crc 2023-10-09T09:16:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:26 smithi163 bash[75425]: cluster 2023-10-09T09:16:26.166630+0000 mon.a (mon.0) 884 : cluster 0 osdmap e713: 8 total, 8 up, 8 in 2023-10-09T09:16:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:26 smithi181 bash[59345]: cluster 2023-10-09T09:16:25.160595+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e712 with expected crc 2023-10-09T09:16:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:26 smithi181 bash[59345]: cluster 2023-10-09T09:16:25.160610+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e712 with expected crc 2023-10-09T09:16:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:26 smithi181 bash[59345]: cluster 2023-10-09T09:16:25.368302+0000 mgr.y (mgr.34107) 296 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 195 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2023-10-09T09:16:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:26 smithi181 bash[59345]: cluster 2023-10-09T09:16:25.381163+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e712 with expected crc 2023-10-09T09:16:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:26 smithi181 bash[59345]: cluster 2023-10-09T09:16:26.166630+0000 mon.a (mon.0) 884 : cluster 0 osdmap e713: 8 total, 8 up, 8 in 2023-10-09T09:16:27.173 INFO:tasks.rados.rados.0.smithi163.stdout:2546: done (0 left) 2023-10-09T09:16:27.174 INFO:tasks.rados.rados.0.smithi163.stdout:2547: delete oid 4 current snap is 326 2023-10-09T09:16:27.184 INFO:tasks.rados.rados.0.smithi163.stdout:2547: done (0 left) 2023-10-09T09:16:27.185 INFO:tasks.rados.rados.0.smithi163.stdout:2548: snap_remove snap 293 2023-10-09T09:16:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:27 smithi163 bash[78107]: cluster 2023-10-09T09:16:26.160795+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e713 with expected crc 2023-10-09T09:16:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:27 smithi163 bash[78107]: cluster 2023-10-09T09:16:27.172204+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e714 with expected crc 2023-10-09T09:16:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:27 smithi163 bash[78107]: cluster 2023-10-09T09:16:27.173027+0000 mon.a (mon.0) 885 : cluster 0 osdmap e714: 8 total, 8 up, 8 in 2023-10-09T09:16:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:27 smithi163 bash[78107]: cluster 2023-10-09T09:16:27.175560+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e714 with expected crc 2023-10-09T09:16:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:27 smithi163 bash[75425]: cluster 2023-10-09T09:16:26.160795+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e713 with expected crc 2023-10-09T09:16:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:27 smithi163 bash[75425]: cluster 2023-10-09T09:16:27.172204+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e714 with expected crc 2023-10-09T09:16:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:27 smithi163 bash[75425]: cluster 2023-10-09T09:16:27.173027+0000 mon.a (mon.0) 885 : cluster 0 osdmap e714: 8 total, 8 up, 8 in 2023-10-09T09:16:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:27 smithi163 bash[75425]: cluster 2023-10-09T09:16:27.175560+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e714 with expected crc 2023-10-09T09:16:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:27 smithi181 bash[59345]: cluster 2023-10-09T09:16:26.160795+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e713 with expected crc 2023-10-09T09:16:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:27 smithi181 bash[59345]: cluster 2023-10-09T09:16:27.172204+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e714 with expected crc 2023-10-09T09:16:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:27 smithi181 bash[59345]: cluster 2023-10-09T09:16:27.173027+0000 mon.a (mon.0) 885 : cluster 0 osdmap e714: 8 total, 8 up, 8 in 2023-10-09T09:16:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:27 smithi181 bash[59345]: cluster 2023-10-09T09:16:27.175560+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e714 with expected crc 2023-10-09T09:16:28.174 INFO:tasks.rados.rados.0.smithi163.stdout:2548: done (0 left) 2023-10-09T09:16:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:2549: rollback oid 35 current snap is 326 2023-10-09T09:16:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 35 to 323 2023-10-09T09:16:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:2550: read oid 29 snap -1 2023-10-09T09:16:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:2550: expect (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:16:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:2551: write oid 44 current snap is 326 2023-10-09T09:16:28.175 INFO:tasks.rados.rados.0.smithi163.stdout:2551: seq_num 960 ranges {534720=499518,1460243=674477} 2023-10-09T09:16:28.183 INFO:tasks.rados.rados.0.smithi163.stdout:2551: writing smithi16367185-44 from 534720 to 1034238 tid 1 2023-10-09T09:16:28.191 INFO:tasks.rados.rados.0.smithi163.stdout:2551: writing smithi16367185-44 from 1460243 to 2134720 tid 2 2023-10-09T09:16:28.192 INFO:tasks.rados.rados.0.smithi163.stdout:2552: write oid 7 current snap is 326 2023-10-09T09:16:28.192 INFO:tasks.rados.rados.0.smithi163.stdout:2550: expect (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:16:28.300 INFO:tasks.rados.rados.0.smithi163.stdout:2549: finishing rollback tid 0 to smithi16367185-35 2023-10-09T09:16:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:2549: finishing rollback tid 1 to smithi16367185-35 2023-10-09T09:16:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:2549: finishing rollback tid 2 to smithi16367185-35 2023-10-09T09:16:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 683 (ObjNum 939 snap 322 seq_num 939) dirty exists 2023-10-09T09:16:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:2551: finishing write tid 1 to smithi16367185-44 2023-10-09T09:16:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:2551: finishing write tid 2 to smithi16367185-44 2023-10-09T09:16:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:2551: finishing write tid 3 to smithi16367185-44 2023-10-09T09:16:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:2551: finishing write tid 4 to smithi16367185-44 2023-10-09T09:16:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1858 (ObjNum 960 snap 326 seq_num 960) dirty exists 2023-10-09T09:16:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:2551: left oid 44 (ObjNum 960 snap 326 seq_num 960) 2023-10-09T09:16:28.301 INFO:tasks.rados.rados.0.smithi163.stdout:2552: seq_num 961 ranges {551527=574767,1918725=232802} 2023-10-09T09:16:28.311 INFO:tasks.rados.rados.0.smithi163.stdout:2552: writing smithi16367185-7 from 551527 to 1126294 tid 1 2023-10-09T09:16:28.318 INFO:tasks.rados.rados.0.smithi163.stdout:2552: writing smithi16367185-7 from 1918725 to 2151527 tid 2 2023-10-09T09:16:28.318 INFO:tasks.rados.rados.0.smithi163.stdout:2549: done (3 left) 2023-10-09T09:16:28.318 INFO:tasks.rados.rados.0.smithi163.stdout:2550: done (2 left) 2023-10-09T09:16:28.318 INFO:tasks.rados.rados.0.smithi163.stdout:2551: done (1 left) 2023-10-09T09:16:28.318 INFO:tasks.rados.rados.0.smithi163.stdout:2553: delete oid 41 current snap is 326 2023-10-09T09:16:28.319 INFO:tasks.rados.rados.0.smithi163.stdout:2552: finishing write tid 1 to smithi16367185-7 2023-10-09T09:16:28.321 INFO:tasks.rados.rados.0.smithi163.stdout:2553: done (1 left) 2023-10-09T09:16:28.321 INFO:tasks.rados.rados.0.smithi163.stdout:2554: snap_create 2023-10-09T09:16:28.322 INFO:tasks.rados.rados.0.smithi163.stdout:2552: finishing write tid 2 to smithi16367185-7 2023-10-09T09:16:28.322 INFO:tasks.rados.rados.0.smithi163.stdout:2552: finishing write tid 3 to smithi16367185-7 2023-10-09T09:16:28.323 INFO:tasks.rados.rados.0.smithi163.stdout:2552: finishing write tid 4 to smithi16367185-7 2023-10-09T09:16:28.323 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 594 (ObjNum 961 snap 326 seq_num 961) dirty exists 2023-10-09T09:16:28.323 INFO:tasks.rados.rados.0.smithi163.stdout:2552: left oid 7 (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:16:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[78107]: cluster 2023-10-09T09:16:26.963231+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e713 with expected crc 2023-10-09T09:16:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[78107]: cluster 2023-10-09T09:16:27.168008+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e714 with expected crc 2023-10-09T09:16:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[78107]: cluster 2023-10-09T09:16:27.369106+0000 mgr.y (mgr.34107) 297 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2023-10-09T09:16:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[78107]: cluster 2023-10-09T09:16:28.171667+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e715 with expected crc 2023-10-09T09:16:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[78107]: cluster 2023-10-09T09:16:28.174183+0000 mon.a (mon.0) 886 : cluster 0 osdmap e715: 8 total, 8 up, 8 in 2023-10-09T09:16:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[78107]: cluster 2023-10-09T09:16:28.175480+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e715 with expected crc 2023-10-09T09:16:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[75425]: cluster 2023-10-09T09:16:26.963231+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e713 with expected crc 2023-10-09T09:16:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[75425]: cluster 2023-10-09T09:16:27.168008+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e714 with expected crc 2023-10-09T09:16:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[75425]: cluster 2023-10-09T09:16:27.369106+0000 mgr.y (mgr.34107) 297 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2023-10-09T09:16:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[75425]: cluster 2023-10-09T09:16:28.171667+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e715 with expected crc 2023-10-09T09:16:28.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[75425]: cluster 2023-10-09T09:16:28.174183+0000 mon.a (mon.0) 886 : cluster 0 osdmap e715: 8 total, 8 up, 8 in 2023-10-09T09:16:28.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:28 smithi163 bash[75425]: cluster 2023-10-09T09:16:28.175480+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e715 with expected crc 2023-10-09T09:16:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:28 smithi181 bash[59345]: cluster 2023-10-09T09:16:26.963231+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e713 with expected crc 2023-10-09T09:16:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:28 smithi181 bash[59345]: cluster 2023-10-09T09:16:27.168008+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e714 with expected crc 2023-10-09T09:16:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:28 smithi181 bash[59345]: cluster 2023-10-09T09:16:27.369106+0000 mgr.y (mgr.34107) 297 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2023-10-09T09:16:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:28 smithi181 bash[59345]: cluster 2023-10-09T09:16:28.171667+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e715 with expected crc 2023-10-09T09:16:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:28 smithi181 bash[59345]: cluster 2023-10-09T09:16:28.174183+0000 mon.a (mon.0) 886 : cluster 0 osdmap e715: 8 total, 8 up, 8 in 2023-10-09T09:16:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:28 smithi181 bash[59345]: cluster 2023-10-09T09:16:28.175480+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e715 with expected crc 2023-10-09T09:16:29.178 INFO:tasks.rados.rados.0.smithi163.stdout:2552: done (1 left) 2023-10-09T09:16:29.179 INFO:tasks.rados.rados.0.smithi163.stdout:2554: done (0 left) 2023-10-09T09:16:29.179 INFO:tasks.rados.rados.0.smithi163.stdout:2555: write oid 4 current snap is 327 2023-10-09T09:16:29.179 INFO:tasks.rados.rados.0.smithi163.stdout:2555: seq_num 963 ranges {585141=725265,1919336=265805} 2023-10-09T09:16:29.187 INFO:tasks.rados.rados.0.smithi163.stdout:2555: writing smithi16367185-4 from 585141 to 1310406 tid 1 2023-10-09T09:16:29.192 INFO:tasks.rados.rados.0.smithi163.stdout:2555: writing smithi16367185-4 from 1919336 to 2185141 tid 2 2023-10-09T09:16:29.192 INFO:tasks.rados.rados.0.smithi163.stdout:2556: snap_remove snap 323 2023-10-09T09:16:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:29 smithi163 bash[78107]: cluster 2023-10-09T09:16:28.171205+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e715 with expected crc 2023-10-09T09:16:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:29 smithi163 bash[78107]: cluster 2023-10-09T09:16:29.174352+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:29 smithi163 bash[78107]: cluster 2023-10-09T09:16:29.178832+0000 mon.a (mon.0) 887 : cluster 0 osdmap e716: 8 total, 8 up, 8 in 2023-10-09T09:16:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:29 smithi163 bash[78107]: cluster 2023-10-09T09:16:29.392685+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:29 smithi163 bash[75425]: cluster 2023-10-09T09:16:28.171205+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e715 with expected crc 2023-10-09T09:16:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:29 smithi163 bash[75425]: cluster 2023-10-09T09:16:29.174352+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:29 smithi163 bash[75425]: cluster 2023-10-09T09:16:29.178832+0000 mon.a (mon.0) 887 : cluster 0 osdmap e716: 8 total, 8 up, 8 in 2023-10-09T09:16:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:29 smithi163 bash[75425]: cluster 2023-10-09T09:16:29.392685+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:29.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:29 smithi181 bash[59345]: cluster 2023-10-09T09:16:28.171205+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e715 with expected crc 2023-10-09T09:16:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:29 smithi181 bash[59345]: cluster 2023-10-09T09:16:29.174352+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:29 smithi181 bash[59345]: cluster 2023-10-09T09:16:29.178832+0000 mon.a (mon.0) 887 : cluster 0 osdmap e716: 8 total, 8 up, 8 in 2023-10-09T09:16:29.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:29 smithi181 bash[59345]: cluster 2023-10-09T09:16:29.392685+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:30.167 INFO:tasks.rados.rados.0.smithi163.stdout:2556: done (1 left) 2023-10-09T09:16:30.167 INFO:tasks.rados.rados.0.smithi163.stdout:2557: snap_remove snap 318 2023-10-09T09:16:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:30 smithi163 bash[78107]: cluster 2023-10-09T09:16:29.173913+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:30 smithi163 bash[78107]: cluster 2023-10-09T09:16:29.369956+0000 mgr.y (mgr.34107) 298 : cluster 0 pgmap v627: 89 pgs: 89 active+clean; 196 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 685 KiB/s wr, 8 op/s 2023-10-09T09:16:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:30 smithi163 bash[78107]: cluster 2023-10-09T09:16:30.166513+0000 mon.a (mon.0) 888 : cluster 0 osdmap e717: 8 total, 8 up, 8 in 2023-10-09T09:16:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:30 smithi163 bash[78107]: cluster 2023-10-09T09:16:30.252191+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e717 with expected crc 2023-10-09T09:16:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:30 smithi163 bash[75425]: cluster 2023-10-09T09:16:29.173913+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:30 smithi163 bash[75425]: cluster 2023-10-09T09:16:29.369956+0000 mgr.y (mgr.34107) 298 : cluster 0 pgmap v627: 89 pgs: 89 active+clean; 196 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 685 KiB/s wr, 8 op/s 2023-10-09T09:16:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:30 smithi163 bash[75425]: cluster 2023-10-09T09:16:30.166513+0000 mon.a (mon.0) 888 : cluster 0 osdmap e717: 8 total, 8 up, 8 in 2023-10-09T09:16:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:30 smithi163 bash[75425]: cluster 2023-10-09T09:16:30.252191+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e717 with expected crc 2023-10-09T09:16:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:30 smithi181 bash[59345]: cluster 2023-10-09T09:16:29.173913+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:30 smithi181 bash[59345]: cluster 2023-10-09T09:16:29.369956+0000 mgr.y (mgr.34107) 298 : cluster 0 pgmap v627: 89 pgs: 89 active+clean; 196 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 685 KiB/s wr, 8 op/s 2023-10-09T09:16:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:30 smithi181 bash[59345]: cluster 2023-10-09T09:16:30.166513+0000 mon.a (mon.0) 888 : cluster 0 osdmap e717: 8 total, 8 up, 8 in 2023-10-09T09:16:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:30 smithi181 bash[59345]: cluster 2023-10-09T09:16:30.252191+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e717 with expected crc 2023-10-09T09:16:31.169 INFO:tasks.rados.rados.0.smithi163.stdout:2557: done (1 left) 2023-10-09T09:16:31.169 INFO:tasks.rados.rados.0.smithi163.stdout:2558: snap_remove snap 313 2023-10-09T09:16:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:31 smithi163 bash[78107]: cluster 2023-10-09T09:16:29.894003+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:31 smithi163 bash[78107]: cluster 2023-10-09T09:16:30.161387+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e717 with expected crc 2023-10-09T09:16:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:31 smithi163 bash[78107]: cluster 2023-10-09T09:16:31.168853+0000 mon.a (mon.0) 889 : cluster 0 osdmap e718: 8 total, 8 up, 8 in 2023-10-09T09:16:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:31 smithi163 bash[75425]: cluster 2023-10-09T09:16:29.894003+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:31 smithi163 bash[75425]: cluster 2023-10-09T09:16:30.161387+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e717 with expected crc 2023-10-09T09:16:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:31 smithi163 bash[75425]: cluster 2023-10-09T09:16:31.168853+0000 mon.a (mon.0) 889 : cluster 0 osdmap e718: 8 total, 8 up, 8 in 2023-10-09T09:16:31.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:31 smithi181 bash[59345]: cluster 2023-10-09T09:16:29.894003+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e716 with expected crc 2023-10-09T09:16:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:31 smithi181 bash[59345]: cluster 2023-10-09T09:16:30.161387+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e717 with expected crc 2023-10-09T09:16:31.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:31 smithi181 bash[59345]: cluster 2023-10-09T09:16:31.168853+0000 mon.a (mon.0) 889 : cluster 0 osdmap e718: 8 total, 8 up, 8 in 2023-10-09T09:16:32.181 INFO:tasks.rados.rados.0.smithi163.stdout:2558: done (1 left) 2023-10-09T09:16:32.181 INFO:tasks.rados.rados.0.smithi163.stdout:2559: write oid 45 current snap is 327 2023-10-09T09:16:32.181 INFO:tasks.rados.rados.0.smithi163.stdout:2555: finishing write tid 1 to smithi16367185-4 2023-10-09T09:16:32.181 INFO:tasks.rados.rados.0.smithi163.stdout:2555: finishing write tid 2 to smithi16367185-4 2023-10-09T09:16:32.182 INFO:tasks.rados.rados.0.smithi163.stdout:2555: finishing write tid 3 to smithi16367185-4 2023-10-09T09:16:32.182 INFO:tasks.rados.rados.0.smithi163.stdout:2555: finishing write tid 4 to smithi16367185-4 2023-10-09T09:16:32.182 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1229 (ObjNum 963 snap 327 seq_num 963) dirty exists 2023-10-09T09:16:32.182 INFO:tasks.rados.rados.0.smithi163.stdout:2555: left oid 4 (ObjNum 963 snap 327 seq_num 963) 2023-10-09T09:16:32.182 INFO:tasks.rados.rados.0.smithi163.stdout:2559: seq_num 964 ranges {601948=400514,1577818=624130} 2023-10-09T09:16:32.188 INFO:tasks.rados.rados.0.smithi163.stdout:2559: writing smithi16367185-45 from 601948 to 1002462 tid 1 2023-10-09T09:16:32.196 INFO:tasks.rados.rados.0.smithi163.stdout:2559: writing smithi16367185-45 from 1577818 to 2201948 tid 2 2023-10-09T09:16:32.197 INFO:tasks.rados.rados.0.smithi163.stdout:2555: done (1 left) 2023-10-09T09:16:32.197 INFO:tasks.rados.rados.0.smithi163.stdout:2560: snap_create 2023-10-09T09:16:32.201 INFO:tasks.rados.rados.0.smithi163.stdout:2559: finishing write tid 1 to smithi16367185-45 2023-10-09T09:16:32.203 INFO:tasks.rados.rados.0.smithi163.stdout:2559: finishing write tid 2 to smithi16367185-45 2023-10-09T09:16:32.203 INFO:tasks.rados.rados.0.smithi163.stdout:2559: finishing write tid 3 to smithi16367185-45 2023-10-09T09:16:32.204 INFO:tasks.rados.rados.0.smithi163.stdout:2559: finishing write tid 4 to smithi16367185-45 2023-10-09T09:16:32.204 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 1862 (ObjNum 964 snap 327 seq_num 964) dirty exists 2023-10-09T09:16:32.204 INFO:tasks.rados.rados.0.smithi163.stdout:2559: left oid 45 (ObjNum 964 snap 327 seq_num 964) 2023-10-09T09:16:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:32 smithi163 bash[78107]: cluster 2023-10-09T09:16:31.163743+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e718 with expected crc 2023-10-09T09:16:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:32 smithi163 bash[78107]: cluster 2023-10-09T09:16:31.164352+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e718 with expected crc 2023-10-09T09:16:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:32 smithi163 bash[78107]: cluster 2023-10-09T09:16:31.370841+0000 mgr.y (mgr.34107) 299 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 726 KiB/s wr, 6 op/s 2023-10-09T09:16:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:32 smithi163 bash[78107]: cluster 2023-10-09T09:16:32.177112+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e719 with expected crc 2023-10-09T09:16:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:32 smithi163 bash[78107]: cluster 2023-10-09T09:16:32.180947+0000 mon.a (mon.0) 890 : cluster 0 osdmap e719: 8 total, 8 up, 8 in 2023-10-09T09:16:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:32 smithi163 bash[75425]: cluster 2023-10-09T09:16:31.163743+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e718 with expected crc 2023-10-09T09:16:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:32 smithi163 bash[75425]: cluster 2023-10-09T09:16:31.164352+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e718 with expected crc 2023-10-09T09:16:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:32 smithi163 bash[75425]: cluster 2023-10-09T09:16:31.370841+0000 mgr.y (mgr.34107) 299 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 726 KiB/s wr, 6 op/s 2023-10-09T09:16:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:32 smithi163 bash[75425]: cluster 2023-10-09T09:16:32.177112+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e719 with expected crc 2023-10-09T09:16:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:32 smithi163 bash[75425]: cluster 2023-10-09T09:16:32.180947+0000 mon.a (mon.0) 890 : cluster 0 osdmap e719: 8 total, 8 up, 8 in 2023-10-09T09:16:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:32 smithi181 bash[59345]: cluster 2023-10-09T09:16:31.163743+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e718 with expected crc 2023-10-09T09:16:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:32 smithi181 bash[59345]: cluster 2023-10-09T09:16:31.164352+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e718 with expected crc 2023-10-09T09:16:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:32 smithi181 bash[59345]: cluster 2023-10-09T09:16:31.370841+0000 mgr.y (mgr.34107) 299 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 726 KiB/s wr, 6 op/s 2023-10-09T09:16:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:32 smithi181 bash[59345]: cluster 2023-10-09T09:16:32.177112+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e719 with expected crc 2023-10-09T09:16:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:32 smithi181 bash[59345]: cluster 2023-10-09T09:16:32.180947+0000 mon.a (mon.0) 890 : cluster 0 osdmap e719: 8 total, 8 up, 8 in 2023-10-09T09:16:33.181 INFO:tasks.rados.rados.0.smithi163.stdout:2559: done (1 left) 2023-10-09T09:16:33.182 INFO:tasks.rados.rados.0.smithi163.stdout:2560: done (0 left) 2023-10-09T09:16:33.182 INFO:tasks.rados.rados.0.smithi163.stdout:2561: delete oid 44 current snap is 328 2023-10-09T09:16:33.189 INFO:tasks.rados.rados.0.smithi163.stdout:2561: done (0 left) 2023-10-09T09:16:33.189 INFO:tasks.rados.rados.0.smithi163.stdout:2562: write oid 25 current snap is 328 2023-10-09T09:16:33.189 INFO:tasks.rados.rados.0.smithi163.stdout:2562: seq_num 966 ranges {635562=667365,2094782=140780} 2023-10-09T09:16:33.200 INFO:tasks.rados.rados.0.smithi163.stdout:2562: writing smithi16367185-25 from 635562 to 1302927 tid 1 2023-10-09T09:16:33.206 INFO:tasks.rados.rados.0.smithi163.stdout:2562: writing smithi16367185-25 from 2094782 to 2235562 tid 2 2023-10-09T09:16:33.206 INFO:tasks.rados.rados.0.smithi163.stdout:2563: read oid 49 snap -1 2023-10-09T09:16:33.206 INFO:tasks.rados.rados.0.smithi163.stdout:2563: expect (ObjNum 793 snap 272 seq_num 793) 2023-10-09T09:16:33.206 INFO:tasks.rados.rados.0.smithi163.stdout:2564: snap_create 2023-10-09T09:16:33.210 INFO:tasks.rados.rados.0.smithi163.stdout:2562: finishing write tid 1 to smithi16367185-25 2023-10-09T09:16:33.214 INFO:tasks.rados.rados.0.smithi163.stdout:2563: expect (ObjNum 793 snap 272 seq_num 793) 2023-10-09T09:16:33.330 INFO:tasks.rados.rados.0.smithi163.stdout:2562: finishing write tid 2 to smithi16367185-25 2023-10-09T09:16:33.331 INFO:tasks.rados.rados.0.smithi163.stdout:2562: finishing write tid 3 to smithi16367185-25 2023-10-09T09:16:33.331 INFO:tasks.rados.rados.0.smithi163.stdout:2562: finishing write tid 4 to smithi16367185-25 2023-10-09T09:16:33.331 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1258 (ObjNum 966 snap 328 seq_num 966) dirty exists 2023-10-09T09:16:33.331 INFO:tasks.rados.rados.0.smithi163.stdout:2562: left oid 25 (ObjNum 966 snap 328 seq_num 966) 2023-10-09T09:16:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:16:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:16:33] "GET /metrics HTTP/1.1" 200 36267 "" "Prometheus/2.33.4" 2023-10-09T09:16:33.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:33 smithi163 bash[75425]: cluster 2023-10-09T09:16:32.177085+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e719 with expected crc 2023-10-09T09:16:33.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:33 smithi163 bash[75425]: cluster 2023-10-09T09:16:33.181646+0000 mon.a (mon.0) 891 : cluster 0 osdmap e720: 8 total, 8 up, 8 in 2023-10-09T09:16:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:33 smithi163 bash[78107]: cluster 2023-10-09T09:16:32.177085+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e719 with expected crc 2023-10-09T09:16:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:33 smithi163 bash[78107]: cluster 2023-10-09T09:16:33.181646+0000 mon.a (mon.0) 891 : cluster 0 osdmap e720: 8 total, 8 up, 8 in 2023-10-09T09:16:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:33 smithi181 bash[59345]: cluster 2023-10-09T09:16:32.177085+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e719 with expected crc 2023-10-09T09:16:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:33 smithi181 bash[59345]: cluster 2023-10-09T09:16:33.181646+0000 mon.a (mon.0) 891 : cluster 0 osdmap e720: 8 total, 8 up, 8 in 2023-10-09T09:16:34.190 INFO:tasks.rados.rados.0.smithi163.stdout:2562: done (2 left) 2023-10-09T09:16:34.190 INFO:tasks.rados.rados.0.smithi163.stdout:2563: done (1 left) 2023-10-09T09:16:34.190 INFO:tasks.rados.rados.0.smithi163.stdout:2564: done (0 left) 2023-10-09T09:16:34.191 INFO:tasks.rados.rados.0.smithi163.stdout:2565: snap_create 2023-10-09T09:16:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[75425]: cluster 2023-10-09T09:16:33.175926+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e720 with expected crc 2023-10-09T09:16:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[75425]: cluster 2023-10-09T09:16:33.371555+0000 mgr.y (mgr.34107) 300 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 187 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 529 KiB/s wr, 5 op/s 2023-10-09T09:16:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[75425]: cluster 2023-10-09T09:16:33.679723+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e720 with expected crc 2023-10-09T09:16:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[75425]: cluster 2023-10-09T09:16:33.679950+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e720 with expected crc 2023-10-09T09:16:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[75425]: cluster 2023-10-09T09:16:34.189701+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e721 with expected crc 2023-10-09T09:16:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[75425]: cluster 2023-10-09T09:16:34.190240+0000 mon.a (mon.0) 892 : cluster 0 osdmap e721: 8 total, 8 up, 8 in 2023-10-09T09:16:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[78107]: cluster 2023-10-09T09:16:33.175926+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e720 with expected crc 2023-10-09T09:16:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[78107]: cluster 2023-10-09T09:16:33.371555+0000 mgr.y (mgr.34107) 300 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 187 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 529 KiB/s wr, 5 op/s 2023-10-09T09:16:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[78107]: cluster 2023-10-09T09:16:33.679723+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e720 with expected crc 2023-10-09T09:16:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[78107]: cluster 2023-10-09T09:16:33.679950+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e720 with expected crc 2023-10-09T09:16:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[78107]: cluster 2023-10-09T09:16:34.189701+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e721 with expected crc 2023-10-09T09:16:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:34 smithi163 bash[78107]: cluster 2023-10-09T09:16:34.190240+0000 mon.a (mon.0) 892 : cluster 0 osdmap e721: 8 total, 8 up, 8 in 2023-10-09T09:16:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:34 smithi181 bash[59345]: cluster 2023-10-09T09:16:33.175926+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e720 with expected crc 2023-10-09T09:16:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:34 smithi181 bash[59345]: cluster 2023-10-09T09:16:33.371555+0000 mgr.y (mgr.34107) 300 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 187 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 529 KiB/s wr, 5 op/s 2023-10-09T09:16:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:34 smithi181 bash[59345]: cluster 2023-10-09T09:16:33.679723+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e720 with expected crc 2023-10-09T09:16:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:34 smithi181 bash[59345]: cluster 2023-10-09T09:16:33.679950+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e720 with expected crc 2023-10-09T09:16:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:34 smithi181 bash[59345]: cluster 2023-10-09T09:16:34.189701+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e721 with expected crc 2023-10-09T09:16:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:34 smithi181 bash[59345]: cluster 2023-10-09T09:16:34.190240+0000 mon.a (mon.0) 892 : cluster 0 osdmap e721: 8 total, 8 up, 8 in 2023-10-09T09:16:35.171 INFO:tasks.rados.rados.0.smithi163.stdout:2565: done (0 left) 2023-10-09T09:16:35.171 INFO:tasks.rados.rados.0.smithi163.stdout:2566: snap_create 2023-10-09T09:16:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:35 smithi163 bash[78107]: cluster 2023-10-09T09:16:34.187093+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e721 with expected crc 2023-10-09T09:16:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:35 smithi163 bash[78107]: cluster 2023-10-09T09:16:35.168847+0000 mon.a (mon.0) 893 : cluster 0 osdmap e722: 8 total, 8 up, 8 in 2023-10-09T09:16:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:35 smithi163 bash[78107]: cluster 2023-10-09T09:16:35.335057+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:35 smithi163 bash[75425]: cluster 2023-10-09T09:16:34.187093+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e721 with expected crc 2023-10-09T09:16:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:35 smithi163 bash[75425]: cluster 2023-10-09T09:16:35.168847+0000 mon.a (mon.0) 893 : cluster 0 osdmap e722: 8 total, 8 up, 8 in 2023-10-09T09:16:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:35 smithi163 bash[75425]: cluster 2023-10-09T09:16:35.335057+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:35.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:35 smithi181 bash[59345]: cluster 2023-10-09T09:16:34.187093+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e721 with expected crc 2023-10-09T09:16:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:35 smithi181 bash[59345]: cluster 2023-10-09T09:16:35.168847+0000 mon.a (mon.0) 893 : cluster 0 osdmap e722: 8 total, 8 up, 8 in 2023-10-09T09:16:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:35 smithi181 bash[59345]: cluster 2023-10-09T09:16:35.335057+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:36.173 INFO:tasks.rados.rados.0.smithi163.stdout:2566: done (0 left) 2023-10-09T09:16:36.173 INFO:tasks.rados.rados.0.smithi163.stdout:2567: read oid 32 snap -1 2023-10-09T09:16:36.173 INFO:tasks.rados.rados.0.smithi163.stdout:2567: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:16:36.173 INFO:tasks.rados.rados.0.smithi163.stdout:2568: write oid 41 current snap is 331 2023-10-09T09:16:36.173 INFO:tasks.rados.rados.0.smithi163.stdout:2568: seq_num 967 ranges {652369=742614,2153264=99105} 2023-10-09T09:16:36.183 INFO:tasks.rados.rados.0.smithi163.stdout:2568: writing smithi16367185-41 from 652369 to 1394983 tid 1 2023-10-09T09:16:36.188 INFO:tasks.rados.rados.0.smithi163.stdout:2568: writing smithi16367185-41 from 2153264 to 2252369 tid 2 2023-10-09T09:16:36.188 INFO:tasks.rados.rados.0.smithi163.stdout:2569: snap_create 2023-10-09T09:16:36.189 INFO:tasks.rados.rados.0.smithi163.stdout:2567: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:16:36.384 INFO:tasks.rados.rados.0.smithi163.stdout:2568: finishing write tid 1 to smithi16367185-41 2023-10-09T09:16:36.385 INFO:tasks.rados.rados.0.smithi163.stdout:2568: finishing write tid 2 to smithi16367185-41 2023-10-09T09:16:36.385 INFO:tasks.rados.rados.0.smithi163.stdout:2568: finishing write tid 3 to smithi16367185-41 2023-10-09T09:16:36.385 INFO:tasks.rados.rados.0.smithi163.stdout:2568: finishing write tid 4 to smithi16367185-41 2023-10-09T09:16:36.385 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 198 (ObjNum 967 snap 331 seq_num 967) dirty exists 2023-10-09T09:16:36.385 INFO:tasks.rados.rados.0.smithi163.stdout:2568: left oid 41 (ObjNum 967 snap 331 seq_num 967) 2023-10-09T09:16:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[78107]: cluster 2023-10-09T09:16:34.875741+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e721 with expected crc 2023-10-09T09:16:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[78107]: cluster 2023-10-09T09:16:35.163560+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[78107]: cluster 2023-10-09T09:16:35.372303+0000 mgr.y (mgr.34107) 301 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 188 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 197 KiB/s wr, 1 op/s 2023-10-09T09:16:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[78107]: cluster 2023-10-09T09:16:35.738365+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[78107]: cluster 2023-10-09T09:16:36.171601+0000 mon.a (mon.0) 894 : cluster 0 osdmap e723: 8 total, 8 up, 8 in 2023-10-09T09:16:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[78107]: cluster 2023-10-09T09:16:36.187897+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[78107]: cluster 2023-10-09T09:16:36.238583+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[75425]: cluster 2023-10-09T09:16:34.875741+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e721 with expected crc 2023-10-09T09:16:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[75425]: cluster 2023-10-09T09:16:35.163560+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[75425]: cluster 2023-10-09T09:16:35.372303+0000 mgr.y (mgr.34107) 301 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 188 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 197 KiB/s wr, 1 op/s 2023-10-09T09:16:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[75425]: cluster 2023-10-09T09:16:35.738365+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[75425]: cluster 2023-10-09T09:16:36.171601+0000 mon.a (mon.0) 894 : cluster 0 osdmap e723: 8 total, 8 up, 8 in 2023-10-09T09:16:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[75425]: cluster 2023-10-09T09:16:36.187897+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:36 smithi163 bash[75425]: cluster 2023-10-09T09:16:36.238583+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:36 smithi181 bash[59345]: cluster 2023-10-09T09:16:34.875741+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e721 with expected crc 2023-10-09T09:16:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:36 smithi181 bash[59345]: cluster 2023-10-09T09:16:35.163560+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:36 smithi181 bash[59345]: cluster 2023-10-09T09:16:35.372303+0000 mgr.y (mgr.34107) 301 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 188 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 197 KiB/s wr, 1 op/s 2023-10-09T09:16:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:36 smithi181 bash[59345]: cluster 2023-10-09T09:16:35.738365+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:36 smithi181 bash[59345]: cluster 2023-10-09T09:16:36.171601+0000 mon.a (mon.0) 894 : cluster 0 osdmap e723: 8 total, 8 up, 8 in 2023-10-09T09:16:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:36 smithi181 bash[59345]: cluster 2023-10-09T09:16:36.187897+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:36 smithi181 bash[59345]: cluster 2023-10-09T09:16:36.238583+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:37.183 INFO:tasks.rados.rados.0.smithi163.stdout:2567: done (2 left) 2023-10-09T09:16:37.183 INFO:tasks.rados.rados.0.smithi163.stdout:2568: done (1 left) 2023-10-09T09:16:37.183 INFO:tasks.rados.rados.0.smithi163.stdout:2569: done (0 left) 2023-10-09T09:16:37.183 INFO:tasks.rados.rados.0.smithi163.stdout:2570: write oid 16 current snap is 332 2023-10-09T09:16:37.183 INFO:tasks.rados.rados.0.smithi163.stdout:2570: seq_num 968 ranges {669176=417863,1811746=457430} 2023-10-09T09:16:37.190 INFO:tasks.rados.rados.0.smithi163.stdout:2570: writing smithi16367185-16 from 669176 to 1087039 tid 1 2023-10-09T09:16:37.201 INFO:tasks.rados.rados.0.smithi163.stdout:2570: writing smithi16367185-16 from 1811746 to 2269176 tid 2 2023-10-09T09:16:37.201 INFO:tasks.rados.rados.0.smithi163.stdout:2571: read oid 5 snap -1 2023-10-09T09:16:37.201 INFO:tasks.rados.rados.0.smithi163.stdout:2571: expect (ObjNum 954 snap 324 seq_num 954) 2023-10-09T09:16:37.201 INFO:tasks.rados.rados.0.smithi163.stdout:2572: snap_remove snap 324 2023-10-09T09:16:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:37 smithi163 bash[78107]: cluster 2023-10-09T09:16:35.865279+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:37 smithi163 bash[78107]: cluster 2023-10-09T09:16:36.166247+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:37 smithi163 bash[78107]: cluster 2023-10-09T09:16:36.238789+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:37 smithi163 bash[78107]: cluster 2023-10-09T09:16:37.178615+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e724 with expected crc 2023-10-09T09:16:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:37 smithi163 bash[78107]: cluster 2023-10-09T09:16:37.183001+0000 mon.a (mon.0) 895 : cluster 0 osdmap e724: 8 total, 8 up, 8 in 2023-10-09T09:16:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:37 smithi163 bash[75425]: cluster 2023-10-09T09:16:35.865279+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:37 smithi163 bash[75425]: cluster 2023-10-09T09:16:36.166247+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:37 smithi163 bash[75425]: cluster 2023-10-09T09:16:36.238789+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:37 smithi163 bash[75425]: cluster 2023-10-09T09:16:37.178615+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e724 with expected crc 2023-10-09T09:16:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:37 smithi163 bash[75425]: cluster 2023-10-09T09:16:37.183001+0000 mon.a (mon.0) 895 : cluster 0 osdmap e724: 8 total, 8 up, 8 in 2023-10-09T09:16:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:37 smithi181 bash[59345]: cluster 2023-10-09T09:16:35.865279+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e722 with expected crc 2023-10-09T09:16:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:37 smithi181 bash[59345]: cluster 2023-10-09T09:16:36.166247+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:37 smithi181 bash[59345]: cluster 2023-10-09T09:16:36.238789+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e723 with expected crc 2023-10-09T09:16:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:37 smithi181 bash[59345]: cluster 2023-10-09T09:16:37.178615+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e724 with expected crc 2023-10-09T09:16:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:37 smithi181 bash[59345]: cluster 2023-10-09T09:16:37.183001+0000 mon.a (mon.0) 895 : cluster 0 osdmap e724: 8 total, 8 up, 8 in 2023-10-09T09:16:38.189 INFO:tasks.rados.rados.0.smithi163.stdout:2572: done (2 left) 2023-10-09T09:16:38.190 INFO:tasks.rados.rados.0.smithi163.stdout:2573: write oid 34 current snap is 332 2023-10-09T09:16:38.190 INFO:tasks.rados.rados.0.smithi163.stdout:2570: finishing write tid 1 to smithi16367185-16 2023-10-09T09:16:38.190 INFO:tasks.rados.rados.0.smithi163.stdout:2573: seq_num 969 ranges {685983=493112,1870228=415755} 2023-10-09T09:16:38.197 INFO:tasks.rados.rados.0.smithi163.stdout:2573: writing smithi16367185-34 from 685983 to 1179095 tid 1 2023-10-09T09:16:38.204 INFO:tasks.rados.rados.0.smithi163.stdout:2573: writing smithi16367185-34 from 1870228 to 2285983 tid 2 2023-10-09T09:16:38.205 INFO:tasks.rados.rados.0.smithi163.stdout:2574: snap_create 2023-10-09T09:16:38.205 INFO:tasks.rados.rados.0.smithi163.stdout:2570: finishing write tid 2 to smithi16367185-16 2023-10-09T09:16:38.205 INFO:tasks.rados.rados.0.smithi163.stdout:2570: finishing write tid 3 to smithi16367185-16 2023-10-09T09:16:38.205 INFO:tasks.rados.rados.0.smithi163.stdout:2570: finishing write tid 4 to smithi16367185-16 2023-10-09T09:16:38.205 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1236 (ObjNum 968 snap 332 seq_num 968) dirty exists 2023-10-09T09:16:38.205 INFO:tasks.rados.rados.0.smithi163.stdout:2570: left oid 16 (ObjNum 968 snap 332 seq_num 968) 2023-10-09T09:16:38.205 INFO:tasks.rados.rados.0.smithi163.stdout:2571: expect (ObjNum 954 snap 324 seq_num 954) 2023-10-09T09:16:38.282 INFO:tasks.rados.rados.0.smithi163.stdout:2573: finishing write tid 1 to smithi16367185-34 2023-10-09T09:16:38.283 INFO:tasks.rados.rados.0.smithi163.stdout:2573: finishing write tid 2 to smithi16367185-34 2023-10-09T09:16:38.283 INFO:tasks.rados.rados.0.smithi163.stdout:2573: finishing write tid 3 to smithi16367185-34 2023-10-09T09:16:38.283 INFO:tasks.rados.rados.0.smithi163.stdout:2573: finishing write tid 4 to smithi16367185-34 2023-10-09T09:16:38.283 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1251 (ObjNum 969 snap 332 seq_num 969) dirty exists 2023-10-09T09:16:38.283 INFO:tasks.rados.rados.0.smithi163.stdout:2573: left oid 34 (ObjNum 969 snap 332 seq_num 969) 2023-10-09T09:16:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:38 smithi163 bash[75425]: cluster 2023-10-09T09:16:37.177058+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e724 with expected crc 2023-10-09T09:16:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:38 smithi163 bash[75425]: cluster 2023-10-09T09:16:37.373004+0000 mgr.y (mgr.34107) 302 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 448 KiB/s wr, 6 op/s 2023-10-09T09:16:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:38 smithi163 bash[75425]: cluster 2023-10-09T09:16:38.189804+0000 mon.a (mon.0) 896 : cluster 0 osdmap e725: 8 total, 8 up, 8 in 2023-10-09T09:16:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:38 smithi163 bash[78107]: cluster 2023-10-09T09:16:37.177058+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e724 with expected crc 2023-10-09T09:16:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:38 smithi163 bash[78107]: cluster 2023-10-09T09:16:37.373004+0000 mgr.y (mgr.34107) 302 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 448 KiB/s wr, 6 op/s 2023-10-09T09:16:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:38 smithi163 bash[78107]: cluster 2023-10-09T09:16:38.189804+0000 mon.a (mon.0) 896 : cluster 0 osdmap e725: 8 total, 8 up, 8 in 2023-10-09T09:16:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:38 smithi181 bash[59345]: cluster 2023-10-09T09:16:37.177058+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e724 with expected crc 2023-10-09T09:16:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:38 smithi181 bash[59345]: cluster 2023-10-09T09:16:37.373004+0000 mgr.y (mgr.34107) 302 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 448 KiB/s wr, 6 op/s 2023-10-09T09:16:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:38 smithi181 bash[59345]: cluster 2023-10-09T09:16:38.189804+0000 mon.a (mon.0) 896 : cluster 0 osdmap e725: 8 total, 8 up, 8 in 2023-10-09T09:16:39.196 INFO:tasks.rados.rados.0.smithi163.stdout:2570: done (3 left) 2023-10-09T09:16:39.196 INFO:tasks.rados.rados.0.smithi163.stdout:2571: done (2 left) 2023-10-09T09:16:39.197 INFO:tasks.rados.rados.0.smithi163.stdout:2573: done (1 left) 2023-10-09T09:16:39.197 INFO:tasks.rados.rados.0.smithi163.stdout:2574: done (0 left) 2023-10-09T09:16:39.197 INFO:tasks.rados.rados.0.smithi163.stdout:2575: write oid 33 current snap is 333 2023-10-09T09:16:39.197 INFO:tasks.rados.rados.0.smithi163.stdout:2575: seq_num 970 ranges {702790=568361,1812357=490433} 2023-10-09T09:16:39.210 INFO:tasks.rados.rados.0.smithi163.stdout:2575: writing smithi16367185-33 from 702790 to 1271151 tid 1 2023-10-09T09:16:39.216 INFO:tasks.rados.rados.0.smithi163.stdout:2575: writing smithi16367185-33 from 1812357 to 2302790 tid 2 2023-10-09T09:16:39.216 INFO:tasks.rados.rados.0.smithi163.stdout:2576: delete oid 20 current snap is 333 2023-10-09T09:16:39.221 INFO:tasks.rados.rados.0.smithi163.stdout:2575: finishing write tid 1 to smithi16367185-33 2023-10-09T09:16:39.222 INFO:tasks.rados.rados.0.smithi163.stdout:2575: finishing write tid 2 to smithi16367185-33 2023-10-09T09:16:39.223 INFO:tasks.rados.rados.0.smithi163.stdout:2575: finishing write tid 3 to smithi16367185-33 2023-10-09T09:16:39.223 INFO:tasks.rados.rados.0.smithi163.stdout:2575: finishing write tid 4 to smithi16367185-33 2023-10-09T09:16:39.223 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1261 (ObjNum 970 snap 333 seq_num 970) dirty exists 2023-10-09T09:16:39.223 INFO:tasks.rados.rados.0.smithi163.stdout:2575: left oid 33 (ObjNum 970 snap 333 seq_num 970) 2023-10-09T09:16:39.227 INFO:tasks.rados.rados.0.smithi163.stdout:2575: done (1 left) 2023-10-09T09:16:39.227 INFO:tasks.rados.rados.0.smithi163.stdout:2576: done (0 left) 2023-10-09T09:16:39.227 INFO:tasks.rados.rados.0.smithi163.stdout:2577: snap_create 2023-10-09T09:16:39.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:39 smithi181 bash[59345]: cluster 2023-10-09T09:16:38.186906+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e725 with expected crc 2023-10-09T09:16:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:39 smithi181 bash[59345]: cluster 2023-10-09T09:16:38.189616+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e725 with expected crc 2023-10-09T09:16:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:39 smithi181 bash[59345]: cluster 2023-10-09T09:16:39.192379+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:39 smithi181 bash[59345]: cluster 2023-10-09T09:16:39.196216+0000 mon.a (mon.0) 897 : cluster 0 osdmap e726: 8 total, 8 up, 8 in 2023-10-09T09:16:39.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:39 smithi163 bash[75425]: cluster 2023-10-09T09:16:38.186906+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e725 with expected crc 2023-10-09T09:16:39.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:39 smithi163 bash[75425]: cluster 2023-10-09T09:16:38.189616+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e725 with expected crc 2023-10-09T09:16:39.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:39 smithi163 bash[75425]: cluster 2023-10-09T09:16:39.192379+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:39.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:39 smithi163 bash[75425]: cluster 2023-10-09T09:16:39.196216+0000 mon.a (mon.0) 897 : cluster 0 osdmap e726: 8 total, 8 up, 8 in 2023-10-09T09:16:39.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:39 smithi163 bash[78107]: cluster 2023-10-09T09:16:38.186906+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e725 with expected crc 2023-10-09T09:16:39.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:39 smithi163 bash[78107]: cluster 2023-10-09T09:16:38.189616+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e725 with expected crc 2023-10-09T09:16:39.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:39 smithi163 bash[78107]: cluster 2023-10-09T09:16:39.192379+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:39.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:39 smithi163 bash[78107]: cluster 2023-10-09T09:16:39.196216+0000 mon.a (mon.0) 897 : cluster 0 osdmap e726: 8 total, 8 up, 8 in 2023-10-09T09:16:40.173 INFO:tasks.rados.rados.0.smithi163.stdout:2577: done (0 left) 2023-10-09T09:16:40.173 INFO:tasks.rados.rados.0.smithi163.stdout:2578: rollback oid 15 current snap is 334 2023-10-09T09:16:40.173 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 15 to 333 2023-10-09T09:16:40.173 INFO:tasks.rados.rados.0.smithi163.stdout:2579: read oid 28 snap -1 2023-10-09T09:16:40.173 INFO:tasks.rados.rados.0.smithi163.stdout:2579: expect deleted 2023-10-09T09:16:40.173 INFO:tasks.rados.rados.0.smithi163.stdout:2580: read oid 11 snap -1 2023-10-09T09:16:40.174 INFO:tasks.rados.rados.0.smithi163.stdout:2580: expect deleted 2023-10-09T09:16:40.174 INFO:tasks.rados.rados.0.smithi163.stdout:2581: snap_create 2023-10-09T09:16:40.186 INFO:tasks.rados.rados.0.smithi163.stdout:2578: finishing rollback tid 1 to smithi16367185-15 2023-10-09T09:16:40.186 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 1229 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:16:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:40 smithi181 bash[59345]: cluster 2023-10-09T09:16:39.191985+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:40 smithi181 bash[59345]: cluster 2023-10-09T09:16:39.217843+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:40 smithi181 bash[59345]: cluster 2023-10-09T09:16:39.373806+0000 mgr.y (mgr.34107) 303 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 250 KiB/s wr, 5 op/s 2023-10-09T09:16:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:40 smithi181 bash[59345]: cluster 2023-10-09T09:16:39.981582+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:40 smithi181 bash[59345]: cluster 2023-10-09T09:16:40.172637+0000 mon.a (mon.0) 898 : cluster 0 osdmap e727: 8 total, 8 up, 8 in 2023-10-09T09:16:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:40 smithi181 bash[59345]: cluster 2023-10-09T09:16:40.177697+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:40 smithi181 bash[59345]: cluster 2023-10-09T09:16:40.240226+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:40 smithi181 bash[59345]: audit 2023-10-09T09:16:40.345502+0000 mon.a (mon.0) 899 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[78107]: cluster 2023-10-09T09:16:39.191985+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[78107]: cluster 2023-10-09T09:16:39.217843+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[78107]: cluster 2023-10-09T09:16:39.373806+0000 mgr.y (mgr.34107) 303 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 250 KiB/s wr, 5 op/s 2023-10-09T09:16:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[78107]: cluster 2023-10-09T09:16:39.981582+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[78107]: cluster 2023-10-09T09:16:40.172637+0000 mon.a (mon.0) 898 : cluster 0 osdmap e727: 8 total, 8 up, 8 in 2023-10-09T09:16:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[78107]: cluster 2023-10-09T09:16:40.177697+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[78107]: cluster 2023-10-09T09:16:40.240226+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[78107]: audit 2023-10-09T09:16:40.345502+0000 mon.a (mon.0) 899 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[75425]: cluster 2023-10-09T09:16:39.191985+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[75425]: cluster 2023-10-09T09:16:39.217843+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[75425]: cluster 2023-10-09T09:16:39.373806+0000 mgr.y (mgr.34107) 303 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 250 KiB/s wr, 5 op/s 2023-10-09T09:16:40.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[75425]: cluster 2023-10-09T09:16:39.981582+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e726 with expected crc 2023-10-09T09:16:40.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[75425]: cluster 2023-10-09T09:16:40.172637+0000 mon.a (mon.0) 898 : cluster 0 osdmap e727: 8 total, 8 up, 8 in 2023-10-09T09:16:40.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[75425]: cluster 2023-10-09T09:16:40.177697+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:40.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[75425]: cluster 2023-10-09T09:16:40.240226+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:40.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:40 smithi163 bash[75425]: audit 2023-10-09T09:16:40.345502+0000 mon.a (mon.0) 899 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:41.182 INFO:tasks.rados.rados.0.smithi163.stdout:2578: done (3 left) 2023-10-09T09:16:41.182 INFO:tasks.rados.rados.0.smithi163.stdout:2579: done (2 left) 2023-10-09T09:16:41.182 INFO:tasks.rados.rados.0.smithi163.stdout:2580: done (1 left) 2023-10-09T09:16:41.182 INFO:tasks.rados.rados.0.smithi163.stdout:2581: done (0 left) 2023-10-09T09:16:41.182 INFO:tasks.rados.rados.0.smithi163.stdout:2582: snap_create 2023-10-09T09:16:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:41 smithi181 bash[59345]: cluster 2023-10-09T09:16:40.167627+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:41 smithi181 bash[59345]: cluster 2023-10-09T09:16:40.240362+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:41 smithi181 bash[59345]: cluster 2023-10-09T09:16:41.177035+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e728 with expected crc 2023-10-09T09:16:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:41 smithi181 bash[59345]: cluster 2023-10-09T09:16:41.181693+0000 mon.a (mon.0) 900 : cluster 0 osdmap e728: 8 total, 8 up, 8 in 2023-10-09T09:16:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:41 smithi163 bash[75425]: cluster 2023-10-09T09:16:40.167627+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:41 smithi163 bash[75425]: cluster 2023-10-09T09:16:40.240362+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:41 smithi163 bash[75425]: cluster 2023-10-09T09:16:41.177035+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e728 with expected crc 2023-10-09T09:16:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:41 smithi163 bash[75425]: cluster 2023-10-09T09:16:41.181693+0000 mon.a (mon.0) 900 : cluster 0 osdmap e728: 8 total, 8 up, 8 in 2023-10-09T09:16:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:41 smithi163 bash[78107]: cluster 2023-10-09T09:16:40.167627+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:41 smithi163 bash[78107]: cluster 2023-10-09T09:16:40.240362+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e727 with expected crc 2023-10-09T09:16:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:41 smithi163 bash[78107]: cluster 2023-10-09T09:16:41.177035+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e728 with expected crc 2023-10-09T09:16:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:41 smithi163 bash[78107]: cluster 2023-10-09T09:16:41.181693+0000 mon.a (mon.0) 900 : cluster 0 osdmap e728: 8 total, 8 up, 8 in 2023-10-09T09:16:42.191 INFO:tasks.rados.rados.0.smithi163.stdout:2582: done (0 left) 2023-10-09T09:16:42.191 INFO:tasks.rados.rados.0.smithi163.stdout:2583: read oid 7 snap -1 2023-10-09T09:16:42.192 INFO:tasks.rados.rados.0.smithi163.stdout:2583: expect (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:16:42.192 INFO:tasks.rados.rados.0.smithi163.stdout:2584: read oid 5 snap -1 2023-10-09T09:16:42.192 INFO:tasks.rados.rados.0.smithi163.stdout:2584: expect (ObjNum 954 snap 324 seq_num 954) 2023-10-09T09:16:42.192 INFO:tasks.rados.rados.0.smithi163.stdout:2585: write oid 35 current snap is 336 2023-10-09T09:16:42.192 INFO:tasks.rados.rados.0.smithi163.stdout:2585: seq_num 972 ranges {736404=718859,1929321=407083} 2023-10-09T09:16:42.204 INFO:tasks.rados.rados.0.smithi163.stdout:2585: writing smithi16367185-35 from 736404 to 1455263 tid 1 2023-10-09T09:16:42.209 INFO:tasks.rados.rados.0.smithi163.stdout:2585: writing smithi16367185-35 from 1929321 to 2336404 tid 2 2023-10-09T09:16:42.209 INFO:tasks.rados.rados.0.smithi163.stdout:2586: write oid 17 current snap is 336 2023-10-09T09:16:42.209 INFO:tasks.rados.rados.0.smithi163.stdout:2586: seq_num 973 ranges {753211=794108,1987803=365408} 2023-10-09T09:16:42.218 INFO:tasks.rados.rados.0.smithi163.stdout:2586: writing smithi16367185-17 from 753211 to 1547319 tid 1 2023-10-09T09:16:42.224 INFO:tasks.rados.rados.0.smithi163.stdout:2586: writing smithi16367185-17 from 1987803 to 2353211 tid 2 2023-10-09T09:16:42.224 INFO:tasks.rados.rados.0.smithi163.stdout:2587: delete oid 33 current snap is 336 2023-10-09T09:16:42.224 INFO:tasks.rados.rados.0.smithi163.stdout:2583: expect (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:16:42.406 INFO:tasks.rados.rados.0.smithi163.stdout:2584: expect (ObjNum 954 snap 324 seq_num 954) 2023-10-09T09:16:42.478 INFO:tasks.rados.rados.0.smithi163.stdout:2585: finishing write tid 1 to smithi16367185-35 2023-10-09T09:16:42.478 INFO:tasks.rados.rados.0.smithi163.stdout:2585: finishing write tid 2 to smithi16367185-35 2023-10-09T09:16:42.478 INFO:tasks.rados.rados.0.smithi163.stdout:2585: finishing write tid 3 to smithi16367185-35 2023-10-09T09:16:42.478 INFO:tasks.rados.rados.0.smithi163.stdout:2583: done (4 left) 2023-10-09T09:16:42.478 INFO:tasks.rados.rados.0.smithi163.stdout:2584: done (3 left) 2023-10-09T09:16:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:2587: done (2 left) 2023-10-09T09:16:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:2588: read oid 1 snap -1 2023-10-09T09:16:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:2588: expect (ObjNum 955 snap 324 seq_num 955) 2023-10-09T09:16:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:2589: read oid 44 snap -1 2023-10-09T09:16:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:2589: expect deleted 2023-10-09T09:16:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:2590: read oid 15 snap -1 2023-10-09T09:16:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:2590: expect deleted 2023-10-09T09:16:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:2591: read oid 11 snap 322 2023-10-09T09:16:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:2591: expect deleted 2023-10-09T09:16:42.479 INFO:tasks.rados.rados.0.smithi163.stdout:2592: read oid 32 snap 330 2023-10-09T09:16:42.480 INFO:tasks.rados.rados.0.smithi163.stdout:2592: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:16:42.480 INFO:tasks.rados.rados.0.smithi163.stdout:2593: read oid 37 snap -1 2023-10-09T09:16:42.480 INFO:tasks.rados.rados.0.smithi163.stdout:2593: expect deleted 2023-10-09T09:16:42.480 INFO:tasks.rados.rados.0.smithi163.stdout:2594: snap_remove snap 329 2023-10-09T09:16:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:42 smithi181 bash[59345]: cluster 2023-10-09T09:16:41.177493+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e728 with expected crc 2023-10-09T09:16:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:42 smithi181 bash[59345]: cluster 2023-10-09T09:16:41.374852+0000 mgr.y (mgr.34107) 304 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 900 KiB/s wr, 7 op/s 2023-10-09T09:16:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:42 smithi181 bash[59345]: cluster 2023-10-09T09:16:42.188337+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e729 with expected crc 2023-10-09T09:16:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:42 smithi181 bash[59345]: cluster 2023-10-09T09:16:42.191147+0000 mon.a (mon.0) 901 : cluster 0 osdmap e729: 8 total, 8 up, 8 in 2023-10-09T09:16:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:42 smithi181 bash[59345]: cluster 2023-10-09T09:16:42.206262+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e729 with expected crc 2023-10-09T09:16:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:42 smithi163 bash[78107]: cluster 2023-10-09T09:16:41.177493+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e728 with expected crc 2023-10-09T09:16:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:42 smithi163 bash[78107]: cluster 2023-10-09T09:16:41.374852+0000 mgr.y (mgr.34107) 304 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 900 KiB/s wr, 7 op/s 2023-10-09T09:16:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:42 smithi163 bash[78107]: cluster 2023-10-09T09:16:42.188337+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e729 with expected crc 2023-10-09T09:16:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:42 smithi163 bash[78107]: cluster 2023-10-09T09:16:42.191147+0000 mon.a (mon.0) 901 : cluster 0 osdmap e729: 8 total, 8 up, 8 in 2023-10-09T09:16:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:42 smithi163 bash[78107]: cluster 2023-10-09T09:16:42.206262+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e729 with expected crc 2023-10-09T09:16:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:42 smithi163 bash[75425]: cluster 2023-10-09T09:16:41.177493+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e728 with expected crc 2023-10-09T09:16:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:42 smithi163 bash[75425]: cluster 2023-10-09T09:16:41.374852+0000 mgr.y (mgr.34107) 304 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 900 KiB/s wr, 7 op/s 2023-10-09T09:16:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:42 smithi163 bash[75425]: cluster 2023-10-09T09:16:42.188337+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e729 with expected crc 2023-10-09T09:16:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:42 smithi163 bash[75425]: cluster 2023-10-09T09:16:42.191147+0000 mon.a (mon.0) 901 : cluster 0 osdmap e729: 8 total, 8 up, 8 in 2023-10-09T09:16:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:42 smithi163 bash[75425]: cluster 2023-10-09T09:16:42.206262+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e729 with expected crc 2023-10-09T09:16:43.492 INFO:tasks.rados.rados.0.smithi163.stdout:2594: done (8 left) 2023-10-09T09:16:43.492 INFO:tasks.rados.rados.0.smithi163.stdout:2595: read oid 6 snap -1 2023-10-09T09:16:43.492 INFO:tasks.rados.rados.0.smithi163.stdout:2595: expect (ObjNum 956 snap 324 seq_num 956) 2023-10-09T09:16:43.492 INFO:tasks.rados.rados.0.smithi163.stdout:2596: write oid 7 current snap is 336 2023-10-09T09:16:43.492 INFO:tasks.rados.rados.0.smithi163.stdout:2585: finishing write tid 4 to smithi16367185-35 2023-10-09T09:16:43.492 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 699 (ObjNum 972 snap 336 seq_num 972) dirty exists 2023-10-09T09:16:43.492 INFO:tasks.rados.rados.0.smithi163.stdout:2585: left oid 35 (ObjNum 972 snap 336 seq_num 972) 2023-10-09T09:16:43.492 INFO:tasks.rados.rados.0.smithi163.stdout:2586: finishing write tid 1 to smithi16367185-17 2023-10-09T09:16:43.493 INFO:tasks.rados.rados.0.smithi163.stdout:2586: finishing write tid 2 to smithi16367185-17 2023-10-09T09:16:43.493 INFO:tasks.rados.rados.0.smithi163.stdout:2586: finishing write tid 3 to smithi16367185-17 2023-10-09T09:16:43.493 INFO:tasks.rados.rados.0.smithi163.stdout:2586: finishing write tid 4 to smithi16367185-17 2023-10-09T09:16:43.493 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1332 (ObjNum 973 snap 336 seq_num 973) dirty exists 2023-10-09T09:16:43.493 INFO:tasks.rados.rados.0.smithi163.stdout:2586: left oid 17 (ObjNum 973 snap 336 seq_num 973) 2023-10-09T09:16:43.493 INFO:tasks.rados.rados.0.smithi163.stdout:2596: seq_num 975 ranges {786825=660959,2104767=282058} 2023-10-09T09:16:43.505 INFO:tasks.rados.rados.0.smithi163.stdout:2596: writing smithi16367185-7 from 786825 to 1447784 tid 1 2023-10-09T09:16:43.515 INFO:tasks.rados.rados.0.smithi163.stdout:2596: writing smithi16367185-7 from 2104767 to 2386825 tid 2 2023-10-09T09:16:43.516 INFO:tasks.rados.rados.0.smithi163.stdout:2585: done (9 left) 2023-10-09T09:16:43.516 INFO:tasks.rados.rados.0.smithi163.stdout:2586: done (8 left) 2023-10-09T09:16:43.516 INFO:tasks.rados.rados.0.smithi163.stdout:2597: read oid 23 snap -1 2023-10-09T09:16:43.516 INFO:tasks.rados.rados.0.smithi163.stdout:2597: expect deleted 2023-10-09T09:16:43.516 INFO:tasks.rados.rados.0.smithi163.stdout:2598: rollback oid 21 current snap is 336 2023-10-09T09:16:43.516 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 322 2023-10-09T09:16:43.516 INFO:tasks.rados.rados.0.smithi163.stdout:2588: expect (ObjNum 955 snap 324 seq_num 955) 2023-10-09T09:16:43.617 INFO:tasks.rados.rados.0.smithi163.stdout:2592: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:16:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:16:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:16:43] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.33.4" 2023-10-09T09:16:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:43 smithi163 bash[78107]: audit 2023-10-09T09:16:43.276624+0000 mon.a (mon.0) 902 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:16:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:43 smithi163 bash[75425]: audit 2023-10-09T09:16:43.276624+0000 mon.a (mon.0) 902 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:16:43.804 INFO:tasks.rados.rados.0.smithi163.stdout:2595: expect (ObjNum 956 snap 324 seq_num 956) 2023-10-09T09:16:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:43 smithi181 bash[59345]: audit 2023-10-09T09:16:43.276624+0000 mon.a (mon.0) 902 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:16:43.952 INFO:tasks.rados.rados.0.smithi163.stdout:2596: finishing write tid 1 to smithi16367185-7 2023-10-09T09:16:43.953 INFO:tasks.rados.rados.0.smithi163.stdout:2596: finishing write tid 2 to smithi16367185-7 2023-10-09T09:16:43.953 INFO:tasks.rados.rados.0.smithi163.stdout:2596: finishing write tid 3 to smithi16367185-7 2023-10-09T09:16:43.953 INFO:tasks.rados.rados.0.smithi163.stdout:2596: finishing write tid 4 to smithi16367185-7 2023-10-09T09:16:43.953 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 608 (ObjNum 975 snap 336 seq_num 975) dirty exists 2023-10-09T09:16:43.953 INFO:tasks.rados.rados.0.smithi163.stdout:2596: left oid 7 (ObjNum 975 snap 336 seq_num 975) 2023-10-09T09:16:43.953 INFO:tasks.rados.rados.0.smithi163.stdout:2598: finishing rollback tid 0 to smithi16367185-21 2023-10-09T09:16:43.953 INFO:tasks.rados.rados.0.smithi163.stdout:2598: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:16:43.953 INFO:tasks.rados.rados.0.smithi163.stdout:2598: finishing rollback tid 2 to smithi16367185-21 2023-10-09T09:16:43.954 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 612 (ObjNum 936 snap 320 seq_num 936) dirty exists 2023-10-09T09:16:43.954 INFO:tasks.rados.rados.0.smithi163.stdout:2588: done (9 left) 2023-10-09T09:16:43.954 INFO:tasks.rados.rados.0.smithi163.stdout:2589: done (8 left) 2023-10-09T09:16:43.954 INFO:tasks.rados.rados.0.smithi163.stdout:2590: done (7 left) 2023-10-09T09:16:43.954 INFO:tasks.rados.rados.0.smithi163.stdout:2591: done (6 left) 2023-10-09T09:16:43.954 INFO:tasks.rados.rados.0.smithi163.stdout:2592: done (5 left) 2023-10-09T09:16:43.954 INFO:tasks.rados.rados.0.smithi163.stdout:2593: done (4 left) 2023-10-09T09:16:43.955 INFO:tasks.rados.rados.0.smithi163.stdout:2595: done (3 left) 2023-10-09T09:16:43.955 INFO:tasks.rados.rados.0.smithi163.stdout:2596: done (2 left) 2023-10-09T09:16:43.955 INFO:tasks.rados.rados.0.smithi163.stdout:2597: done (1 left) 2023-10-09T09:16:43.955 INFO:tasks.rados.rados.0.smithi163.stdout:2598: done (0 left) 2023-10-09T09:16:43.955 INFO:tasks.rados.rados.0.smithi163.stdout:2599: snap_remove snap 335 2023-10-09T09:16:44.498 INFO:tasks.rados.rados.0.smithi163.stdout:2599: done (0 left) 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:2600: read oid 46 snap 320 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:2600: expect deleted 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:2601: rollback oid 44 current snap is 336 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 44 to 330 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:2602: read oid 27 snap -1 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:2602: expect deleted 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:2603: read oid 42 snap -1 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:2603: expect deleted 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:2604: rollback oid 20 current snap is 336 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 322 2023-10-09T09:16:44.499 INFO:tasks.rados.rados.0.smithi163.stdout:2605: write oid 41 current snap is 336 2023-10-09T09:16:44.500 INFO:tasks.rados.rados.0.smithi163.stdout:2605: seq_num 976 ranges {403632=736208,1763249=619999} 2023-10-09T09:16:44.505 INFO:tasks.rados.rados.0.smithi163.stdout:2605: writing smithi16367185-41 from 403632 to 1139840 tid 1 2023-10-09T09:16:44.513 INFO:tasks.rados.rados.0.smithi163.stdout:2605: writing smithi16367185-41 from 1763249 to 2383248 tid 2 2023-10-09T09:16:44.513 INFO:tasks.rados.rados.0.smithi163.stdout:2606: snap_create 2023-10-09T09:16:44.513 INFO:tasks.rados.rados.0.smithi163.stdout:2601: finishing rollback tid 1 to smithi16367185-44 2023-10-09T09:16:44.513 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1864 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:16:44.513 INFO:tasks.rados.rados.0.smithi163.stdout:2604: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:16:44.514 INFO:tasks.rados.rados.0.smithi163.stdout:2604: finishing rollback tid 2 to smithi16367185-20 2023-10-09T09:16:44.514 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 421 (ObjNum 807 snap 279 seq_num 807) dirty exists 2023-10-09T09:16:44.515 INFO:tasks.rados.rados.0.smithi163.stdout:2605: finishing write tid 1 to smithi16367185-41 2023-10-09T09:16:44.517 INFO:tasks.rados.rados.0.smithi163.stdout:2605: finishing write tid 2 to smithi16367185-41 2023-10-09T09:16:44.517 INFO:tasks.rados.rados.0.smithi163.stdout:2605: finishing write tid 3 to smithi16367185-41 2023-10-09T09:16:44.518 INFO:tasks.rados.rados.0.smithi163.stdout:2605: finishing write tid 4 to smithi16367185-41 2023-10-09T09:16:44.518 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 204 (ObjNum 976 snap 336 seq_num 976) dirty exists 2023-10-09T09:16:44.518 INFO:tasks.rados.rados.0.smithi163.stdout:2605: left oid 41 (ObjNum 976 snap 336 seq_num 976) 2023-10-09T09:16:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:44 smithi181 bash[59345]: cluster 2023-10-09T09:16:43.375454+0000 mgr.y (mgr.34107) 305 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 861 KiB/s wr, 7 op/s 2023-10-09T09:16:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:44 smithi181 bash[59345]: cluster 2023-10-09T09:16:43.487408+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:44 smithi181 bash[59345]: cluster 2023-10-09T09:16:43.487804+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:44 smithi181 bash[59345]: cluster 2023-10-09T09:16:43.491605+0000 mon.a (mon.0) 903 : cluster 0 osdmap e730: 8 total, 8 up, 8 in 2023-10-09T09:16:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:44 smithi181 bash[59345]: cluster 2023-10-09T09:16:43.492536+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:44 smithi181 bash[59345]: cluster 2023-10-09T09:16:43.576977+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[75425]: cluster 2023-10-09T09:16:43.375454+0000 mgr.y (mgr.34107) 305 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 861 KiB/s wr, 7 op/s 2023-10-09T09:16:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[75425]: cluster 2023-10-09T09:16:43.487408+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[75425]: cluster 2023-10-09T09:16:43.487804+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[75425]: cluster 2023-10-09T09:16:43.491605+0000 mon.a (mon.0) 903 : cluster 0 osdmap e730: 8 total, 8 up, 8 in 2023-10-09T09:16:44.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[75425]: cluster 2023-10-09T09:16:43.492536+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[75425]: cluster 2023-10-09T09:16:43.576977+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[78107]: cluster 2023-10-09T09:16:43.375454+0000 mgr.y (mgr.34107) 305 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 861 KiB/s wr, 7 op/s 2023-10-09T09:16:44.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[78107]: cluster 2023-10-09T09:16:43.487408+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[78107]: cluster 2023-10-09T09:16:43.487804+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[78107]: cluster 2023-10-09T09:16:43.491605+0000 mon.a (mon.0) 903 : cluster 0 osdmap e730: 8 total, 8 up, 8 in 2023-10-09T09:16:44.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[78107]: cluster 2023-10-09T09:16:43.492536+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:44.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:44 smithi163 bash[78107]: cluster 2023-10-09T09:16:43.576977+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e730 with expected crc 2023-10-09T09:16:45.173 INFO:tasks.rados.rados.0.smithi163.stdout:2600: done (6 left) 2023-10-09T09:16:45.173 INFO:tasks.rados.rados.0.smithi163.stdout:2601: done (5 left) 2023-10-09T09:16:45.174 INFO:tasks.rados.rados.0.smithi163.stdout:2602: done (4 left) 2023-10-09T09:16:45.174 INFO:tasks.rados.rados.0.smithi163.stdout:2603: done (3 left) 2023-10-09T09:16:45.174 INFO:tasks.rados.rados.0.smithi163.stdout:2604: done (2 left) 2023-10-09T09:16:45.174 INFO:tasks.rados.rados.0.smithi163.stdout:2605: done (1 left) 2023-10-09T09:16:45.174 INFO:tasks.rados.rados.0.smithi163.stdout:2606: done (0 left) 2023-10-09T09:16:45.174 INFO:tasks.rados.rados.0.smithi163.stdout:2607: write oid 10 current snap is 337 2023-10-09T09:16:45.174 INFO:tasks.rados.rados.0.smithi163.stdout:2607: seq_num 977 ranges {420439=411457,1421731=480010} 2023-10-09T09:16:45.181 INFO:tasks.rados.rados.0.smithi163.stdout:2607: writing smithi16367185-10 from 420439 to 831896 tid 1 2023-10-09T09:16:45.189 INFO:tasks.rados.rados.0.smithi163.stdout:2607: writing smithi16367185-10 from 1421731 to 1901741 tid 2 2023-10-09T09:16:45.189 INFO:tasks.rados.rados.0.smithi163.stdout:2608: read oid 7 snap 330 2023-10-09T09:16:45.189 INFO:tasks.rados.rados.0.smithi163.stdout:2608: expect (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:16:45.189 INFO:tasks.rados.rados.0.smithi163.stdout:2609: rollback oid 15 current snap is 337 2023-10-09T09:16:45.189 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 15 to 330 2023-10-09T09:16:45.189 INFO:tasks.rados.rados.0.smithi163.stdout:2610: read oid 29 snap 320 2023-10-09T09:16:45.189 INFO:tasks.rados.rados.0.smithi163.stdout:2610: expect (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:16:45.189 INFO:tasks.rados.rados.0.smithi163.stdout:2611: read oid 35 snap 319 2023-10-09T09:16:45.189 INFO:tasks.rados.rados.0.smithi163.stdout:2611: expect deleted 2023-10-09T09:16:45.189 INFO:tasks.rados.rados.0.smithi163.stdout:2612: delete oid 13 current snap is 337 2023-10-09T09:16:45.194 INFO:tasks.rados.rados.0.smithi163.stdout:2610: expect (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:16:45.299 INFO:tasks.rados.rados.0.smithi163.stdout:2607: finishing write tid 1 to smithi16367185-10 2023-10-09T09:16:45.299 INFO:tasks.rados.rados.0.smithi163.stdout:2607: finishing write tid 2 to smithi16367185-10 2023-10-09T09:16:45.299 INFO:tasks.rados.rados.0.smithi163.stdout:2607: finishing write tid 3 to smithi16367185-10 2023-10-09T09:16:45.300 INFO:tasks.rados.rados.0.smithi163.stdout:2607: finishing write tid 4 to smithi16367185-10 2023-10-09T09:16:45.300 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 841 (ObjNum 977 snap 337 seq_num 977) dirty exists 2023-10-09T09:16:45.300 INFO:tasks.rados.rados.0.smithi163.stdout:2607: left oid 10 (ObjNum 977 snap 337 seq_num 977) 2023-10-09T09:16:45.300 INFO:tasks.rados.rados.0.smithi163.stdout:2608: expect (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:16:45.478 INFO:tasks.rados.rados.0.smithi163.stdout:2609: finishing rollback tid 1 to smithi16367185-15 2023-10-09T09:16:45.478 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 1229 (ObjNum 699128208 snap 21875 seq_num 699195984) dirty dne 2023-10-09T09:16:45.478 INFO:tasks.rados.rados.0.smithi163.stdout:2607: done (5 left) 2023-10-09T09:16:45.478 INFO:tasks.rados.rados.0.smithi163.stdout:2608: done (4 left) 2023-10-09T09:16:45.479 INFO:tasks.rados.rados.0.smithi163.stdout:2609: done (3 left) 2023-10-09T09:16:45.479 INFO:tasks.rados.rados.0.smithi163.stdout:2610: done (2 left) 2023-10-09T09:16:45.479 INFO:tasks.rados.rados.0.smithi163.stdout:2611: done (1 left) 2023-10-09T09:16:45.479 INFO:tasks.rados.rados.0.smithi163.stdout:2612: done (0 left) 2023-10-09T09:16:45.479 INFO:tasks.rados.rados.0.smithi163.stdout:2613: snap_remove snap 332 2023-10-09T09:16:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:45 smithi181 bash[59345]: cluster 2023-10-09T09:16:44.494063+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:45 smithi181 bash[59345]: cluster 2023-10-09T09:16:44.497110+0000 mon.a (mon.0) 904 : cluster 0 osdmap e731: 8 total, 8 up, 8 in 2023-10-09T09:16:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:45 smithi181 bash[59345]: cluster 2023-10-09T09:16:44.499135+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:45 smithi181 bash[59345]: cluster 2023-10-09T09:16:44.502837+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:45 smithi181 bash[59345]: cluster 2023-10-09T09:16:44.507444+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:45 smithi181 bash[59345]: cluster 2023-10-09T09:16:45.168702+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:45 smithi181 bash[59345]: cluster 2023-10-09T09:16:45.173040+0000 mon.a (mon.0) 905 : cluster 0 osdmap e732: 8 total, 8 up, 8 in 2023-10-09T09:16:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:45 smithi181 bash[59345]: cluster 2023-10-09T09:16:45.189872+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[75425]: cluster 2023-10-09T09:16:44.494063+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[75425]: cluster 2023-10-09T09:16:44.497110+0000 mon.a (mon.0) 904 : cluster 0 osdmap e731: 8 total, 8 up, 8 in 2023-10-09T09:16:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[75425]: cluster 2023-10-09T09:16:44.499135+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[75425]: cluster 2023-10-09T09:16:44.502837+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[75425]: cluster 2023-10-09T09:16:44.507444+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[75425]: cluster 2023-10-09T09:16:45.168702+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:45.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[75425]: cluster 2023-10-09T09:16:45.173040+0000 mon.a (mon.0) 905 : cluster 0 osdmap e732: 8 total, 8 up, 8 in 2023-10-09T09:16:45.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[75425]: cluster 2023-10-09T09:16:45.189872+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[78107]: cluster 2023-10-09T09:16:44.494063+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[78107]: cluster 2023-10-09T09:16:44.497110+0000 mon.a (mon.0) 904 : cluster 0 osdmap e731: 8 total, 8 up, 8 in 2023-10-09T09:16:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[78107]: cluster 2023-10-09T09:16:44.499135+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[78107]: cluster 2023-10-09T09:16:44.502837+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[78107]: cluster 2023-10-09T09:16:44.507444+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e731 with expected crc 2023-10-09T09:16:45.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[78107]: cluster 2023-10-09T09:16:45.168702+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:45.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[78107]: cluster 2023-10-09T09:16:45.173040+0000 mon.a (mon.0) 905 : cluster 0 osdmap e732: 8 total, 8 up, 8 in 2023-10-09T09:16:45.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:45 smithi163 bash[78107]: cluster 2023-10-09T09:16:45.189872+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:46.189 INFO:tasks.rados.rados.0.smithi163.stdout:2613: done (0 left) 2023-10-09T09:16:46.189 INFO:tasks.rados.rados.0.smithi163.stdout:2614: delete oid 44 current snap is 337 2023-10-09T09:16:46.194 INFO:tasks.rados.rados.0.smithi163.stdout:2614: done (0 left) 2023-10-09T09:16:46.194 INFO:tasks.rados.rados.0.smithi163.stdout:2615: read oid 39 snap -1 2023-10-09T09:16:46.194 INFO:tasks.rados.rados.0.smithi163.stdout:2615: expect (ObjNum 932 snap 317 seq_num 932) 2023-10-09T09:16:46.194 INFO:tasks.rados.rados.0.smithi163.stdout:2616: delete oid 24 current snap is 337 2023-10-09T09:16:46.198 INFO:tasks.rados.rados.0.smithi163.stdout:2616: done (1 left) 2023-10-09T09:16:46.198 INFO:tasks.rados.rados.0.smithi163.stdout:2617: delete oid 28 current snap is 337 2023-10-09T09:16:46.204 INFO:tasks.rados.rados.0.smithi163.stdout:2617: done (1 left) 2023-10-09T09:16:46.204 INFO:tasks.rados.rados.0.smithi163.stdout:2618: snap_remove snap 320 2023-10-09T09:16:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:46 smithi181 bash[59345]: cluster 2023-10-09T09:16:45.169280+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:46 smithi181 bash[59345]: cluster 2023-10-09T09:16:45.366819+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:46 smithi181 bash[59345]: cluster 2023-10-09T09:16:45.375922+0000 mgr.y (mgr.34107) 306 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 710 KiB/s wr, 11 op/s 2023-10-09T09:16:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:46 smithi181 bash[59345]: cluster 2023-10-09T09:16:46.184223+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e733 with expected crc 2023-10-09T09:16:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:46 smithi181 bash[59345]: cluster 2023-10-09T09:16:46.187739+0000 mon.a (mon.0) 906 : cluster 0 osdmap e733: 8 total, 8 up, 8 in 2023-10-09T09:16:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:46 smithi181 bash[59345]: cluster 2023-10-09T09:16:46.188298+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e733 with expected crc 2023-10-09T09:16:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:46 smithi181 bash[59345]: cluster 2023-10-09T09:16:46.203653+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e733 with expected crc 2023-10-09T09:16:46.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[78107]: cluster 2023-10-09T09:16:45.169280+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:46.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[78107]: cluster 2023-10-09T09:16:45.366819+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:46.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[78107]: cluster 2023-10-09T09:16:45.375922+0000 mgr.y (mgr.34107) 306 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 710 KiB/s wr, 11 op/s 2023-10-09T09:16:46.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[78107]: cluster 2023-10-09T09:16:46.184223+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e733 with expected crc 2023-10-09T09:16:46.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[78107]: cluster 2023-10-09T09:16:46.187739+0000 mon.a (mon.0) 906 : cluster 0 osdmap e733: 8 total, 8 up, 8 in 2023-10-09T09:16:46.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[78107]: cluster 2023-10-09T09:16:46.188298+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e733 with expected crc 2023-10-09T09:16:46.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[78107]: cluster 2023-10-09T09:16:46.203653+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e733 with expected crc 2023-10-09T09:16:46.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[75425]: cluster 2023-10-09T09:16:45.169280+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:46.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[75425]: cluster 2023-10-09T09:16:45.366819+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e732 with expected crc 2023-10-09T09:16:46.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[75425]: cluster 2023-10-09T09:16:45.375922+0000 mgr.y (mgr.34107) 306 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 710 KiB/s wr, 11 op/s 2023-10-09T09:16:46.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[75425]: cluster 2023-10-09T09:16:46.184223+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e733 with expected crc 2023-10-09T09:16:46.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[75425]: cluster 2023-10-09T09:16:46.187739+0000 mon.a (mon.0) 906 : cluster 0 osdmap e733: 8 total, 8 up, 8 in 2023-10-09T09:16:46.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[75425]: cluster 2023-10-09T09:16:46.188298+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e733 with expected crc 2023-10-09T09:16:46.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:46 smithi163 bash[75425]: cluster 2023-10-09T09:16:46.203653+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e733 with expected crc 2023-10-09T09:16:47.192 INFO:tasks.rados.rados.0.smithi163.stdout:2618: done (1 left) 2023-10-09T09:16:47.192 INFO:tasks.rados.rados.0.smithi163.stdout:2619: snap_create 2023-10-09T09:16:47.192 INFO:tasks.rados.rados.0.smithi163.stdout:2615: expect (ObjNum 932 snap 317 seq_num 932) 2023-10-09T09:16:48.200 INFO:tasks.rados.rados.0.smithi163.stdout:2615: done (1 left) 2023-10-09T09:16:48.335 INFO:tasks.rados.rados.0.smithi163.stdout:2619: done (0 left) 2023-10-09T09:16:48.335 INFO:tasks.rados.rados.0.smithi163.stdout:2620: snap_remove snap 325 2023-10-09T09:16:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:48 smithi163 bash[75425]: cluster 2023-10-09T09:16:47.187311+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e734 with expected crc 2023-10-09T09:16:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:48 smithi163 bash[75425]: cluster 2023-10-09T09:16:47.191952+0000 mon.a (mon.0) 907 : cluster 0 osdmap e734: 8 total, 8 up, 8 in 2023-10-09T09:16:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:48 smithi163 bash[75425]: cluster 2023-10-09T09:16:47.192486+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e734 with expected crc 2023-10-09T09:16:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:48 smithi163 bash[75425]: cluster 2023-10-09T09:16:47.196355+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e734 with expected crc 2023-10-09T09:16:48.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:48 smithi163 bash[75425]: cluster 2023-10-09T09:16:47.376442+0000 mgr.y (mgr.34107) 307 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.3 MiB/s wr, 21 op/s 2023-10-09T09:16:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:48 smithi163 bash[78107]: cluster 2023-10-09T09:16:47.187311+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e734 with expected crc 2023-10-09T09:16:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:48 smithi163 bash[78107]: cluster 2023-10-09T09:16:47.191952+0000 mon.a (mon.0) 907 : cluster 0 osdmap e734: 8 total, 8 up, 8 in 2023-10-09T09:16:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:48 smithi163 bash[78107]: cluster 2023-10-09T09:16:47.192486+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e734 with expected crc 2023-10-09T09:16:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:48 smithi163 bash[78107]: cluster 2023-10-09T09:16:47.196355+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e734 with expected crc 2023-10-09T09:16:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:48 smithi163 bash[78107]: cluster 2023-10-09T09:16:47.376442+0000 mgr.y (mgr.34107) 307 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.3 MiB/s wr, 21 op/s 2023-10-09T09:16:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:48 smithi181 bash[59345]: cluster 2023-10-09T09:16:47.187311+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e734 with expected crc 2023-10-09T09:16:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:48 smithi181 bash[59345]: cluster 2023-10-09T09:16:47.191952+0000 mon.a (mon.0) 907 : cluster 0 osdmap e734: 8 total, 8 up, 8 in 2023-10-09T09:16:48.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:48 smithi181 bash[59345]: cluster 2023-10-09T09:16:47.192486+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e734 with expected crc 2023-10-09T09:16:48.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:48 smithi181 bash[59345]: cluster 2023-10-09T09:16:47.196355+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e734 with expected crc 2023-10-09T09:16:48.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:48 smithi181 bash[59345]: cluster 2023-10-09T09:16:47.376442+0000 mgr.y (mgr.34107) 307 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.3 MiB/s wr, 21 op/s 2023-10-09T09:16:49.209 INFO:tasks.rados.rados.0.smithi163.stdout:2620: done (0 left) 2023-10-09T09:16:49.209 INFO:tasks.rados.rados.0.smithi163.stdout:2621: rollback oid 42 current snap is 338 2023-10-09T09:16:49.209 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 306 2023-10-09T09:16:49.209 INFO:tasks.rados.rados.0.smithi163.stdout:2622: rollback oid 5 current snap is 338 2023-10-09T09:16:49.209 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 328 2023-10-09T09:16:49.209 INFO:tasks.rados.rados.0.smithi163.stdout:2623: delete oid 47 current snap is 338 2023-10-09T09:16:49.214 INFO:tasks.rados.rados.0.smithi163.stdout:2621: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:16:49.214 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 0 (ObjNum 3 snap 1 seq_num 4) dirty dne 2023-10-09T09:16:49.216 INFO:tasks.rados.rados.0.smithi163.stdout:2622: finishing rollback tid 0 to smithi16367185-5 2023-10-09T09:16:49.218 INFO:tasks.rados.rados.0.smithi163.stdout:2622: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:16:49.218 INFO:tasks.rados.rados.0.smithi163.stdout:2622: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:16:49.218 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1241 (ObjNum 954 snap 324 seq_num 954) dirty exists 2023-10-09T09:16:49.226 INFO:tasks.rados.rados.0.smithi163.stdout:2621: done (2 left) 2023-10-09T09:16:49.226 INFO:tasks.rados.rados.0.smithi163.stdout:2622: done (1 left) 2023-10-09T09:16:49.226 INFO:tasks.rados.rados.0.smithi163.stdout:2623: done (0 left) 2023-10-09T09:16:49.226 INFO:tasks.rados.rados.0.smithi163.stdout:2624: write oid 1 current snap is 338 2023-10-09T09:16:49.226 INFO:tasks.rados.rados.0.smithi163.stdout:2624: seq_num 983 ranges {521281=579304,1656270=774664} 2023-10-09T09:16:49.227 INFO:tasks.rados.rados.0.smithi163.stdout:2624: writing smithi16367185-1 from 521281 to 1100585 tid 1 2023-10-09T09:16:49.235 INFO:tasks.rados.rados.0.smithi163.stdout:2624: writing smithi16367185-1 from 1656270 to 2430934 tid 2 2023-10-09T09:16:49.235 INFO:tasks.rados.rados.0.smithi163.stdout:2625: rollback oid 35 current snap is 338 2023-10-09T09:16:49.235 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 35 to 333 2023-10-09T09:16:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:2624: finishing write tid 1 to smithi16367185-1 2023-10-09T09:16:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:2626: rollback oid 14 current snap is 338 2023-10-09T09:16:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 334 2023-10-09T09:16:49.236 INFO:tasks.rados.rados.0.smithi163.stdout:2627: delete oid 16 current snap is 338 2023-10-09T09:16:49.241 INFO:tasks.rados.rados.0.smithi163.stdout:2625: finishing rollback tid 0 to smithi16367185-35 2023-10-09T09:16:49.242 INFO:tasks.rados.rados.0.smithi163.stdout:2625: finishing rollback tid 1 to smithi16367185-35 2023-10-09T09:16:49.243 INFO:tasks.rados.rados.0.smithi163.stdout:2624: finishing write tid 2 to smithi16367185-1 2023-10-09T09:16:49.243 INFO:tasks.rados.rados.0.smithi163.stdout:2624: finishing write tid 3 to smithi16367185-1 2023-10-09T09:16:49.243 INFO:tasks.rados.rados.0.smithi163.stdout:2625: finishing rollback tid 2 to smithi16367185-35 2023-10-09T09:16:49.243 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 711 (ObjNum 939 snap 322 seq_num 939) dirty exists 2023-10-09T09:16:49.244 INFO:tasks.rados.rados.0.smithi163.stdout:2624: finishing write tid 4 to smithi16367185-1 2023-10-09T09:16:49.244 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1245 (ObjNum 983 snap 338 seq_num 983) dirty exists 2023-10-09T09:16:49.244 INFO:tasks.rados.rados.0.smithi163.stdout:2624: left oid 1 (ObjNum 983 snap 338 seq_num 983) 2023-10-09T09:16:49.244 INFO:tasks.rados.rados.0.smithi163.stdout:2626: finishing rollback tid 0 to smithi16367185-14 2023-10-09T09:16:49.244 INFO:tasks.rados.rados.0.smithi163.stdout:2624: done (3 left) 2023-10-09T09:16:49.244 INFO:tasks.rados.rados.0.smithi163.stdout:2625: done (2 left) 2023-10-09T09:16:49.244 INFO:tasks.rados.rados.0.smithi163.stdout:2627: done (1 left) 2023-10-09T09:16:49.244 INFO:tasks.rados.rados.0.smithi163.stdout:2628: read oid 38 snap -1 2023-10-09T09:16:49.245 INFO:tasks.rados.rados.0.smithi163.stdout:2628: expect (ObjNum 941 snap 322 seq_num 941) 2023-10-09T09:16:49.245 INFO:tasks.rados.rados.0.smithi163.stdout:2629: write oid 36 current snap is 338 2023-10-09T09:16:49.245 INFO:tasks.rados.rados.0.smithi163.stdout:2629: seq_num 985 ranges {554895=729802,1773234=778333} 2023-10-09T09:16:49.251 INFO:tasks.rados.rados.0.smithi163.stdout:2629: writing smithi16367185-36 from 554895 to 1284697 tid 1 2023-10-09T09:16:49.262 INFO:tasks.rados.rados.0.smithi163.stdout:2629: writing smithi16367185-36 from 1773234 to 2551567 tid 2 2023-10-09T09:16:49.262 INFO:tasks.rados.rados.0.smithi163.stdout:2630: read oid 4 snap -1 2023-10-09T09:16:49.262 INFO:tasks.rados.rados.0.smithi163.stdout:2630: expect (ObjNum 963 snap 327 seq_num 963) 2023-10-09T09:16:49.262 INFO:tasks.rados.rados.0.smithi163.stdout:2631: read oid 34 snap -1 2023-10-09T09:16:49.262 INFO:tasks.rados.rados.0.smithi163.stdout:2631: expect (ObjNum 969 snap 332 seq_num 969) 2023-10-09T09:16:49.262 INFO:tasks.rados.rados.0.smithi163.stdout:2632: snap_create 2023-10-09T09:16:49.262 INFO:tasks.rados.rados.0.smithi163.stdout:2626: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:16:49.262 INFO:tasks.rados.rados.0.smithi163.stdout:2626: finishing rollback tid 2 to smithi16367185-14 2023-10-09T09:16:49.262 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1228 (ObjNum 909 snap 311 seq_num 909) dirty exists 2023-10-09T09:16:49.262 INFO:tasks.rados.rados.0.smithi163.stdout:2628: expect (ObjNum 941 snap 322 seq_num 941) 2023-10-09T09:16:49.444 INFO:tasks.rados.rados.0.smithi163.stdout:2629: finishing write tid 1 to smithi16367185-36 2023-10-09T09:16:49.445 INFO:tasks.rados.rados.0.smithi163.stdout:2630: expect (ObjNum 963 snap 327 seq_num 963) 2023-10-09T09:16:49.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:49 smithi163 bash[78107]: cluster 2023-10-09T09:16:48.195403+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e735 with expected crc 2023-10-09T09:16:49.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:49 smithi163 bash[78107]: cluster 2023-10-09T09:16:48.200656+0000 mon.a (mon.0) 908 : cluster 0 osdmap e735: 8 total, 8 up, 8 in 2023-10-09T09:16:49.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:49 smithi163 bash[75425]: cluster 2023-10-09T09:16:48.195403+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e735 with expected crc 2023-10-09T09:16:49.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:49 smithi163 bash[75425]: cluster 2023-10-09T09:16:48.200656+0000 mon.a (mon.0) 908 : cluster 0 osdmap e735: 8 total, 8 up, 8 in 2023-10-09T09:16:49.518 INFO:tasks.rados.rados.0.smithi163.stdout:2629: finishing write tid 2 to smithi16367185-36 2023-10-09T09:16:49.518 INFO:tasks.rados.rados.0.smithi163.stdout:2629: finishing write tid 3 to smithi16367185-36 2023-10-09T09:16:49.518 INFO:tasks.rados.rados.0.smithi163.stdout:2629: finishing write tid 4 to smithi16367185-36 2023-10-09T09:16:49.518 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 728 (ObjNum 985 snap 338 seq_num 985) dirty exists 2023-10-09T09:16:49.518 INFO:tasks.rados.rados.0.smithi163.stdout:2629: left oid 36 (ObjNum 985 snap 338 seq_num 985) 2023-10-09T09:16:49.518 INFO:tasks.rados.rados.0.smithi163.stdout:2631: expect (ObjNum 969 snap 332 seq_num 969) 2023-10-09T09:16:49.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:49 smithi181 bash[59345]: cluster 2023-10-09T09:16:48.195403+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e735 with expected crc 2023-10-09T09:16:49.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:49 smithi181 bash[59345]: cluster 2023-10-09T09:16:48.200656+0000 mon.a (mon.0) 908 : cluster 0 osdmap e735: 8 total, 8 up, 8 in 2023-10-09T09:16:50.182 INFO:tasks.rados.rados.0.smithi163.stdout:2626: done (5 left) 2023-10-09T09:16:50.182 INFO:tasks.rados.rados.0.smithi163.stdout:2628: done (4 left) 2023-10-09T09:16:50.182 INFO:tasks.rados.rados.0.smithi163.stdout:2629: done (3 left) 2023-10-09T09:16:50.182 INFO:tasks.rados.rados.0.smithi163.stdout:2630: done (2 left) 2023-10-09T09:16:50.182 INFO:tasks.rados.rados.0.smithi163.stdout:2631: done (1 left) 2023-10-09T09:16:50.182 INFO:tasks.rados.rados.0.smithi163.stdout:2632: done (0 left) 2023-10-09T09:16:50.182 INFO:tasks.rados.rados.0.smithi163.stdout:2633: write oid 20 current snap is 339 2023-10-09T09:16:50.182 INFO:tasks.rados.rados.0.smithi163.stdout:2633: seq_num 986 ranges {571702=405051,1431716=638344,2560365=11337} 2023-10-09T09:16:50.191 INFO:tasks.rados.rados.0.smithi163.stdout:2633: writing smithi16367185-20 from 571702 to 976753 tid 1 2023-10-09T09:16:50.197 INFO:tasks.rados.rados.0.smithi163.stdout:2633: writing smithi16367185-20 from 1431716 to 2070060 tid 2 2023-10-09T09:16:50.200 INFO:tasks.rados.rados.0.smithi163.stdout:2633: writing smithi16367185-20 from 2560365 to 2571702 tid 3 2023-10-09T09:16:50.200 INFO:tasks.rados.rados.0.smithi163.stdout:2634: rollback oid 39 current snap is 339 2023-10-09T09:16:50.200 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 337 2023-10-09T09:16:50.200 INFO:tasks.rados.rados.0.smithi163.stdout:2635: delete oid 9 current snap is 339 2023-10-09T09:16:50.205 INFO:tasks.rados.rados.0.smithi163.stdout:2633: finishing write tid 1 to smithi16367185-20 2023-10-09T09:16:50.207 INFO:tasks.rados.rados.0.smithi163.stdout:2635: done (2 left) 2023-10-09T09:16:50.207 INFO:tasks.rados.rados.0.smithi163.stdout:2636: read oid 25 snap -1 2023-10-09T09:16:50.207 INFO:tasks.rados.rados.0.smithi163.stdout:2636: expect (ObjNum 966 snap 328 seq_num 966) 2023-10-09T09:16:50.207 INFO:tasks.rados.rados.0.smithi163.stdout:2637: write oid 14 current snap is 339 2023-10-09T09:16:50.207 INFO:tasks.rados.rados.0.smithi163.stdout:2637: seq_num 988 ranges {605316=555549,1832327=642013} 2023-10-09T09:16:50.214 INFO:tasks.rados.rados.0.smithi163.stdout:2637: writing smithi16367185-14 from 605316 to 1160865 tid 1 2023-10-09T09:16:50.224 INFO:tasks.rados.rados.0.smithi163.stdout:2637: writing smithi16367185-14 from 1832327 to 2474340 tid 2 2023-10-09T09:16:50.224 INFO:tasks.rados.rados.0.smithi163.stdout:2638: read oid 31 snap -1 2023-10-09T09:16:50.224 INFO:tasks.rados.rados.0.smithi163.stdout:2638: expect deleted 2023-10-09T09:16:50.224 INFO:tasks.rados.rados.0.smithi163.stdout:2639: snap_remove snap 326 2023-10-09T09:16:50.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: cluster 2023-10-09T09:16:48.564381+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e735 with expected crc 2023-10-09T09:16:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: cluster 2023-10-09T09:16:48.845151+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e735 with expected crc 2023-10-09T09:16:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: cluster 2023-10-09T09:16:49.205179+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e736 with expected crc 2023-10-09T09:16:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: cluster 2023-10-09T09:16:49.205606+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e736 with expected crc 2023-10-09T09:16:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: cluster 2023-10-09T09:16:49.208872+0000 mon.a (mon.0) 909 : cluster 0 osdmap e736: 8 total, 8 up, 8 in 2023-10-09T09:16:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: cluster 2023-10-09T09:16:49.212927+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e736 with expected crc 2023-10-09T09:16:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: audit 2023-10-09T09:16:49.374118+0000 mon.a (mon.0) 910 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: cluster 2023-10-09T09:16:49.376834+0000 mgr.y (mgr.34107) 308 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 627 KiB/s wr, 12 op/s 2023-10-09T09:16:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: audit 2023-10-09T09:16:49.383547+0000 mon.a (mon.0) 911 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: audit 2023-10-09T09:16:49.578006+0000 mon.a (mon.0) 912 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: audit 2023-10-09T09:16:49.587542+0000 mon.a (mon.0) 913 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: audit 2023-10-09T09:16:50.063034+0000 mon.a (mon.0) 914 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:16:50.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: audit 2023-10-09T09:16:50.064475+0000 mon.a (mon.0) 915 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:16:50.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: audit 2023-10-09T09:16:50.073739+0000 mon.a (mon.0) 916 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[75425]: cluster 2023-10-09T09:16:50.181512+0000 mon.a (mon.0) 917 : cluster 0 osdmap e737: 8 total, 8 up, 8 in 2023-10-09T09:16:50.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: cluster 2023-10-09T09:16:48.564381+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e735 with expected crc 2023-10-09T09:16:50.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: cluster 2023-10-09T09:16:48.845151+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e735 with expected crc 2023-10-09T09:16:50.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: cluster 2023-10-09T09:16:49.205179+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e736 with expected crc 2023-10-09T09:16:50.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: cluster 2023-10-09T09:16:49.205606+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e736 with expected crc 2023-10-09T09:16:50.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: cluster 2023-10-09T09:16:49.208872+0000 mon.a (mon.0) 909 : cluster 0 osdmap e736: 8 total, 8 up, 8 in 2023-10-09T09:16:50.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: cluster 2023-10-09T09:16:49.212927+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e736 with expected crc 2023-10-09T09:16:50.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: audit 2023-10-09T09:16:49.374118+0000 mon.a (mon.0) 910 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: cluster 2023-10-09T09:16:49.376834+0000 mgr.y (mgr.34107) 308 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 627 KiB/s wr, 12 op/s 2023-10-09T09:16:50.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: audit 2023-10-09T09:16:49.383547+0000 mon.a (mon.0) 911 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: audit 2023-10-09T09:16:49.578006+0000 mon.a (mon.0) 912 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: audit 2023-10-09T09:16:49.587542+0000 mon.a (mon.0) 913 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: audit 2023-10-09T09:16:50.063034+0000 mon.a (mon.0) 914 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:16:50.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: audit 2023-10-09T09:16:50.064475+0000 mon.a (mon.0) 915 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:16:50.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: audit 2023-10-09T09:16:50.073739+0000 mon.a (mon.0) 916 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:50 smithi163 bash[78107]: cluster 2023-10-09T09:16:50.181512+0000 mon.a (mon.0) 917 : cluster 0 osdmap e737: 8 total, 8 up, 8 in 2023-10-09T09:16:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: cluster 2023-10-09T09:16:48.564381+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e735 with expected crc 2023-10-09T09:16:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: cluster 2023-10-09T09:16:48.845151+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e735 with expected crc 2023-10-09T09:16:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: cluster 2023-10-09T09:16:49.205179+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e736 with expected crc 2023-10-09T09:16:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: cluster 2023-10-09T09:16:49.205606+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e736 with expected crc 2023-10-09T09:16:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: cluster 2023-10-09T09:16:49.208872+0000 mon.a (mon.0) 909 : cluster 0 osdmap e736: 8 total, 8 up, 8 in 2023-10-09T09:16:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: cluster 2023-10-09T09:16:49.212927+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e736 with expected crc 2023-10-09T09:16:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: audit 2023-10-09T09:16:49.374118+0000 mon.a (mon.0) 910 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: cluster 2023-10-09T09:16:49.376834+0000 mgr.y (mgr.34107) 308 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 627 KiB/s wr, 12 op/s 2023-10-09T09:16:50.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: audit 2023-10-09T09:16:49.383547+0000 mon.a (mon.0) 911 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: audit 2023-10-09T09:16:49.578006+0000 mon.a (mon.0) 912 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: audit 2023-10-09T09:16:49.587542+0000 mon.a (mon.0) 913 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: audit 2023-10-09T09:16:50.063034+0000 mon.a (mon.0) 914 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:16:50.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: audit 2023-10-09T09:16:50.064475+0000 mon.a (mon.0) 915 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:16:50.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: audit 2023-10-09T09:16:50.073739+0000 mon.a (mon.0) 916 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:16:50.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:50 smithi181 bash[59345]: cluster 2023-10-09T09:16:50.181512+0000 mon.a (mon.0) 917 : cluster 0 osdmap e737: 8 total, 8 up, 8 in 2023-10-09T09:16:51.221 INFO:tasks.rados.rados.0.smithi163.stdout:2639: done (5 left) 2023-10-09T09:16:51.222 INFO:tasks.rados.rados.0.smithi163.stdout:2640: write oid 11 current snap is 339 2023-10-09T09:16:51.222 INFO:tasks.rados.rados.0.smithi163.stdout:2633: finishing write tid 2 to smithi16367185-20 2023-10-09T09:16:51.222 INFO:tasks.rados.rados.0.smithi163.stdout:2633: finishing write tid 3 to smithi16367185-20 2023-10-09T09:16:51.222 INFO:tasks.rados.rados.0.smithi163.stdout:2633: finishing write tid 4 to smithi16367185-20 2023-10-09T09:16:51.222 INFO:tasks.rados.rados.0.smithi163.stdout:2633: finishing write tid 5 to smithi16367185-20 2023-10-09T09:16:51.222 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 435 (ObjNum 986 snap 339 seq_num 986) dirty exists 2023-10-09T09:16:51.222 INFO:tasks.rados.rados.0.smithi163.stdout:2633: left oid 20 (ObjNum 986 snap 339 seq_num 986) 2023-10-09T09:16:51.222 INFO:tasks.rados.rados.0.smithi163.stdout:2634: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:16:51.222 INFO:tasks.rados.rados.0.smithi163.stdout:2634: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:16:51.222 INFO:tasks.rados.rados.0.smithi163.stdout:2634: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:16:51.223 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1261 (ObjNum 932 snap 317 seq_num 932) dirty exists 2023-10-09T09:16:51.223 INFO:tasks.rados.rados.0.smithi163.stdout:2636: expect (ObjNum 966 snap 328 seq_num 966) 2023-10-09T09:16:51.301 INFO:tasks.rados.rados.0.smithi163.stdout:2637: finishing write tid 1 to smithi16367185-14 2023-10-09T09:16:51.301 INFO:tasks.rados.rados.0.smithi163.stdout:2637: finishing write tid 2 to smithi16367185-14 2023-10-09T09:16:51.301 INFO:tasks.rados.rados.0.smithi163.stdout:2637: finishing write tid 3 to smithi16367185-14 2023-10-09T09:16:51.301 INFO:tasks.rados.rados.0.smithi163.stdout:2637: finishing write tid 4 to smithi16367185-14 2023-10-09T09:16:51.301 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1232 (ObjNum 988 snap 339 seq_num 988) dirty exists 2023-10-09T09:16:51.302 INFO:tasks.rados.rados.0.smithi163.stdout:2637: left oid 14 (ObjNum 988 snap 339 seq_num 988) 2023-10-09T09:16:51.302 INFO:tasks.rados.rados.0.smithi163.stdout:2640: seq_num 989 ranges {622123=747151,2007162=614961} 2023-10-09T09:16:51.314 INFO:tasks.rados.rados.0.smithi163.stdout:2640: writing smithi16367185-11 from 622123 to 1369274 tid 1 2023-10-09T09:16:51.322 INFO:tasks.rados.rados.0.smithi163.stdout:2640: writing smithi16367185-11 from 2007162 to 2622123 tid 2 2023-10-09T09:16:51.322 INFO:tasks.rados.rados.0.smithi163.stdout:2633: done (5 left) 2023-10-09T09:16:51.323 INFO:tasks.rados.rados.0.smithi163.stdout:2634: done (4 left) 2023-10-09T09:16:51.323 INFO:tasks.rados.rados.0.smithi163.stdout:2636: done (3 left) 2023-10-09T09:16:51.323 INFO:tasks.rados.rados.0.smithi163.stdout:2637: done (2 left) 2023-10-09T09:16:51.323 INFO:tasks.rados.rados.0.smithi163.stdout:2638: done (1 left) 2023-10-09T09:16:51.323 INFO:tasks.rados.rados.0.smithi163.stdout:2641: write oid 41 current snap is 339 2023-10-09T09:16:51.323 INFO:tasks.rados.rados.0.smithi163.stdout:2641: seq_num 990 ranges {638930=422400,1665644=645682} 2023-10-09T09:16:51.328 INFO:tasks.rados.rados.0.smithi163.stdout:2641: writing smithi16367185-41 from 638930 to 1061330 tid 1 2023-10-09T09:16:51.335 INFO:tasks.rados.rados.0.smithi163.stdout:2641: writing smithi16367185-41 from 1665644 to 2311326 tid 2 2023-10-09T09:16:51.336 INFO:tasks.rados.rados.0.smithi163.stdout:2642: snap_create 2023-10-09T09:16:51.336 INFO:tasks.rados.rados.0.smithi163.stdout:2640: finishing write tid 1 to smithi16367185-11 2023-10-09T09:16:51.336 INFO:tasks.rados.rados.0.smithi163.stdout:2640: finishing write tid 2 to smithi16367185-11 2023-10-09T09:16:51.336 INFO:tasks.rados.rados.0.smithi163.stdout:2640: finishing write tid 3 to smithi16367185-11 2023-10-09T09:16:51.336 INFO:tasks.rados.rados.0.smithi163.stdout:2640: finishing write tid 4 to smithi16367185-11 2023-10-09T09:16:51.336 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 714 (ObjNum 989 snap 339 seq_num 989) dirty exists 2023-10-09T09:16:51.336 INFO:tasks.rados.rados.0.smithi163.stdout:2640: left oid 11 (ObjNum 989 snap 339 seq_num 989) 2023-10-09T09:16:51.336 INFO:tasks.rados.rados.0.smithi163.stdout:2641: finishing write tid 1 to smithi16367185-41 2023-10-09T09:16:51.339 INFO:tasks.rados.rados.0.smithi163.stdout:2641: finishing write tid 2 to smithi16367185-41 2023-10-09T09:16:51.339 INFO:tasks.rados.rados.0.smithi163.stdout:2641: finishing write tid 3 to smithi16367185-41 2023-10-09T09:16:51.340 INFO:tasks.rados.rados.0.smithi163.stdout:2641: finishing write tid 4 to smithi16367185-41 2023-10-09T09:16:51.340 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 214 (ObjNum 990 snap 339 seq_num 990) dirty exists 2023-10-09T09:16:51.340 INFO:tasks.rados.rados.0.smithi163.stdout:2641: left oid 41 (ObjNum 990 snap 339 seq_num 990) 2023-10-09T09:16:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:51 smithi163 bash[75425]: cluster 2023-10-09T09:16:50.176893+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e737 with expected crc 2023-10-09T09:16:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:51 smithi163 bash[75425]: cluster 2023-10-09T09:16:50.192357+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e737 with expected crc 2023-10-09T09:16:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:51 smithi163 bash[75425]: cluster 2023-10-09T09:16:51.216702+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:51 smithi163 bash[75425]: cluster 2023-10-09T09:16:51.216733+0000 mon.a (mon.0) 918 : cluster 0 osdmap e738: 8 total, 8 up, 8 in 2023-10-09T09:16:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:51 smithi163 bash[78107]: cluster 2023-10-09T09:16:50.176893+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e737 with expected crc 2023-10-09T09:16:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:51 smithi163 bash[78107]: cluster 2023-10-09T09:16:50.192357+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e737 with expected crc 2023-10-09T09:16:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:51 smithi163 bash[78107]: cluster 2023-10-09T09:16:51.216702+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:51 smithi163 bash[78107]: cluster 2023-10-09T09:16:51.216733+0000 mon.a (mon.0) 918 : cluster 0 osdmap e738: 8 total, 8 up, 8 in 2023-10-09T09:16:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:51 smithi181 bash[59345]: cluster 2023-10-09T09:16:50.176893+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e737 with expected crc 2023-10-09T09:16:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:51 smithi181 bash[59345]: cluster 2023-10-09T09:16:50.192357+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e737 with expected crc 2023-10-09T09:16:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:51 smithi181 bash[59345]: cluster 2023-10-09T09:16:51.216702+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:51 smithi181 bash[59345]: cluster 2023-10-09T09:16:51.216733+0000 mon.a (mon.0) 918 : cluster 0 osdmap e738: 8 total, 8 up, 8 in 2023-10-09T09:16:52.222 INFO:tasks.rados.rados.0.smithi163.stdout:2640: done (2 left) 2023-10-09T09:16:52.222 INFO:tasks.rados.rados.0.smithi163.stdout:2641: done (1 left) 2023-10-09T09:16:52.222 INFO:tasks.rados.rados.0.smithi163.stdout:2642: done (0 left) 2023-10-09T09:16:52.222 INFO:tasks.rados.rados.0.smithi163.stdout:2643: delete oid 37 current snap is 340 2023-10-09T09:16:52.232 INFO:tasks.rados.rados.0.smithi163.stdout:2643: done (0 left) 2023-10-09T09:16:52.232 INFO:tasks.rados.rados.0.smithi163.stdout:2644: snap_create 2023-10-09T09:16:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[75425]: cluster 2023-10-09T09:16:50.507182+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e737 with expected crc 2023-10-09T09:16:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[75425]: cluster 2023-10-09T09:16:51.211786+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[75425]: cluster 2023-10-09T09:16:51.216239+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:52.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[75425]: cluster 2023-10-09T09:16:51.317410+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:52.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[75425]: cluster 2023-10-09T09:16:51.378007+0000 mgr.y (mgr.34107) 309 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 206 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 950 KiB/s wr, 16 op/s 2023-10-09T09:16:52.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[75425]: cluster 2023-10-09T09:16:52.221376+0000 mon.a (mon.0) 919 : cluster 0 osdmap e739: 8 total, 8 up, 8 in 2023-10-09T09:16:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[78107]: cluster 2023-10-09T09:16:50.507182+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e737 with expected crc 2023-10-09T09:16:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[78107]: cluster 2023-10-09T09:16:51.211786+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[78107]: cluster 2023-10-09T09:16:51.216239+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[78107]: cluster 2023-10-09T09:16:51.317410+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[78107]: cluster 2023-10-09T09:16:51.378007+0000 mgr.y (mgr.34107) 309 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 206 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 950 KiB/s wr, 16 op/s 2023-10-09T09:16:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:52 smithi163 bash[78107]: cluster 2023-10-09T09:16:52.221376+0000 mon.a (mon.0) 919 : cluster 0 osdmap e739: 8 total, 8 up, 8 in 2023-10-09T09:16:52.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:52 smithi181 bash[59345]: cluster 2023-10-09T09:16:50.507182+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e737 with expected crc 2023-10-09T09:16:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:52 smithi181 bash[59345]: cluster 2023-10-09T09:16:51.211786+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:52 smithi181 bash[59345]: cluster 2023-10-09T09:16:51.216239+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:52 smithi181 bash[59345]: cluster 2023-10-09T09:16:51.317410+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e738 with expected crc 2023-10-09T09:16:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:52 smithi181 bash[59345]: cluster 2023-10-09T09:16:51.378007+0000 mgr.y (mgr.34107) 309 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 206 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 950 KiB/s wr, 16 op/s 2023-10-09T09:16:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:52 smithi181 bash[59345]: cluster 2023-10-09T09:16:52.221376+0000 mon.a (mon.0) 919 : cluster 0 osdmap e739: 8 total, 8 up, 8 in 2023-10-09T09:16:53.221 INFO:tasks.rados.rados.0.smithi163.stdout:2644: done (0 left) 2023-10-09T09:16:53.222 INFO:tasks.rados.rados.0.smithi163.stdout:2645: write oid 43 current snap is 341 2023-10-09T09:16:53.222 INFO:tasks.rados.rados.0.smithi163.stdout:2645: seq_num 992 ranges {672544=572898,1666255=532998} 2023-10-09T09:16:53.233 INFO:tasks.rados.rados.0.smithi163.stdout:2645: writing smithi16367185-43 from 672544 to 1245442 tid 1 2023-10-09T09:16:53.239 INFO:tasks.rados.rados.0.smithi163.stdout:2645: writing smithi16367185-43 from 1666255 to 2199253 tid 2 2023-10-09T09:16:53.239 INFO:tasks.rados.rados.0.smithi163.stdout:2646: delete oid 14 current snap is 341 2023-10-09T09:16:53.240 INFO:tasks.rados.rados.0.smithi163.stdout:2645: finishing write tid 1 to smithi16367185-43 2023-10-09T09:16:53.245 INFO:tasks.rados.rados.0.smithi163.stdout:2645: finishing write tid 2 to smithi16367185-43 2023-10-09T09:16:53.245 INFO:tasks.rados.rados.0.smithi163.stdout:2645: finishing write tid 3 to smithi16367185-43 2023-10-09T09:16:53.246 INFO:tasks.rados.rados.0.smithi163.stdout:2645: finishing write tid 4 to smithi16367185-43 2023-10-09T09:16:53.246 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 1237 (ObjNum 992 snap 341 seq_num 992) dirty exists 2023-10-09T09:16:53.246 INFO:tasks.rados.rados.0.smithi163.stdout:2645: left oid 43 (ObjNum 992 snap 341 seq_num 992) 2023-10-09T09:16:53.247 INFO:tasks.rados.rados.0.smithi163.stdout:2645: done (1 left) 2023-10-09T09:16:53.247 INFO:tasks.rados.rados.0.smithi163.stdout:2646: done (0 left) 2023-10-09T09:16:53.247 INFO:tasks.rados.rados.0.smithi163.stdout:2647: read oid 2 snap -1 2023-10-09T09:16:53.247 INFO:tasks.rados.rados.0.smithi163.stdout:2647: expect deleted 2023-10-09T09:16:53.247 INFO:tasks.rados.rados.0.smithi163.stdout:2648: read oid 44 snap -1 2023-10-09T09:16:53.247 INFO:tasks.rados.rados.0.smithi163.stdout:2648: expect deleted 2023-10-09T09:16:53.247 INFO:tasks.rados.rados.0.smithi163.stdout:2649: write oid 20 current snap is 341 2023-10-09T09:16:53.247 INFO:tasks.rados.rados.0.smithi163.stdout:2649: seq_num 994 ranges {706158=723396,2183219=522939} 2023-10-09T09:16:53.260 INFO:tasks.rados.rados.0.smithi163.stdout:2649: writing smithi16367185-20 from 706158 to 1429554 tid 1 2023-10-09T09:16:53.267 INFO:tasks.rados.rados.0.smithi163.stdout:2649: writing smithi16367185-20 from 2183219 to 2706158 tid 2 2023-10-09T09:16:53.267 INFO:tasks.rados.rados.0.smithi163.stdout:2650: delete oid 5 current snap is 341 2023-10-09T09:16:53.275 INFO:tasks.rados.rados.0.smithi163.stdout:2647: done (3 left) 2023-10-09T09:16:53.275 INFO:tasks.rados.rados.0.smithi163.stdout:2648: done (2 left) 2023-10-09T09:16:53.275 INFO:tasks.rados.rados.0.smithi163.stdout:2650: done (1 left) 2023-10-09T09:16:53.275 INFO:tasks.rados.rados.0.smithi163.stdout:2651: rollback oid 13 current snap is 341 2023-10-09T09:16:53.275 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 13 to 322 2023-10-09T09:16:53.276 INFO:tasks.rados.rados.0.smithi163.stdout:2649: finishing write tid 1 to smithi16367185-20 2023-10-09T09:16:53.276 INFO:tasks.rados.rados.0.smithi163.stdout:2652: snap_create 2023-10-09T09:16:53.276 INFO:tasks.rados.rados.0.smithi163.stdout:2649: finishing write tid 2 to smithi16367185-20 2023-10-09T09:16:53.276 INFO:tasks.rados.rados.0.smithi163.stdout:2649: finishing write tid 3 to smithi16367185-20 2023-10-09T09:16:53.277 INFO:tasks.rados.rados.0.smithi163.stdout:2649: finishing write tid 4 to smithi16367185-20 2023-10-09T09:16:53.277 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 441 (ObjNum 994 snap 341 seq_num 994) dirty exists 2023-10-09T09:16:53.277 INFO:tasks.rados.rados.0.smithi163.stdout:2649: left oid 20 (ObjNum 994 snap 341 seq_num 994) 2023-10-09T09:16:53.277 INFO:tasks.rados.rados.0.smithi163.stdout:2651: finishing rollback tid 1 to smithi16367185-13 2023-10-09T09:16:53.277 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 373 (ObjNum 0 snap 0 seq_num 698772896) dirty dne 2023-10-09T09:16:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:16:53] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.33.4" 2023-10-09T09:16:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[75425]: cluster 2023-10-09T09:16:52.216128+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[75425]: cluster 2023-10-09T09:16:52.222888+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[75425]: cluster 2023-10-09T09:16:52.338469+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[75425]: cluster 2023-10-09T09:16:52.338783+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[75425]: cluster 2023-10-09T09:16:53.217329+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[75425]: cluster 2023-10-09T09:16:53.221253+0000 mon.a (mon.0) 920 : cluster 0 osdmap e740: 8 total, 8 up, 8 in 2023-10-09T09:16:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[78107]: cluster 2023-10-09T09:16:52.216128+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[78107]: cluster 2023-10-09T09:16:52.222888+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[78107]: cluster 2023-10-09T09:16:52.338469+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[78107]: cluster 2023-10-09T09:16:52.338783+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[78107]: cluster 2023-10-09T09:16:53.217329+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:53.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:53 smithi163 bash[78107]: cluster 2023-10-09T09:16:53.221253+0000 mon.a (mon.0) 920 : cluster 0 osdmap e740: 8 total, 8 up, 8 in 2023-10-09T09:16:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:53 smithi181 bash[59345]: cluster 2023-10-09T09:16:52.216128+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:53 smithi181 bash[59345]: cluster 2023-10-09T09:16:52.222888+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:53 smithi181 bash[59345]: cluster 2023-10-09T09:16:52.338469+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:53 smithi181 bash[59345]: cluster 2023-10-09T09:16:52.338783+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e739 with expected crc 2023-10-09T09:16:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:53 smithi181 bash[59345]: cluster 2023-10-09T09:16:53.217329+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:53 smithi181 bash[59345]: cluster 2023-10-09T09:16:53.221253+0000 mon.a (mon.0) 920 : cluster 0 osdmap e740: 8 total, 8 up, 8 in 2023-10-09T09:16:54.237 INFO:tasks.rados.rados.0.smithi163.stdout:2649: done (2 left) 2023-10-09T09:16:54.237 INFO:tasks.rados.rados.0.smithi163.stdout:2651: done (1 left) 2023-10-09T09:16:54.237 INFO:tasks.rados.rados.0.smithi163.stdout:2652: done (0 left) 2023-10-09T09:16:54.237 INFO:tasks.rados.rados.0.smithi163.stdout:2653: read oid 46 snap -1 2023-10-09T09:16:54.237 INFO:tasks.rados.rados.0.smithi163.stdout:2653: expect deleted 2023-10-09T09:16:54.237 INFO:tasks.rados.rados.0.smithi163.stdout:2654: delete oid 17 current snap is 342 2023-10-09T09:16:54.247 INFO:tasks.rados.rados.0.smithi163.stdout:2653: done (1 left) 2023-10-09T09:16:54.247 INFO:tasks.rados.rados.0.smithi163.stdout:2654: done (0 left) 2023-10-09T09:16:54.247 INFO:tasks.rados.rados.0.smithi163.stdout:2655: write oid 38 current snap is 342 2023-10-09T09:16:54.247 INFO:tasks.rados.rados.0.smithi163.stdout:2655: seq_num 997 ranges {756579=665496,1958665=400347} 2023-10-09T09:16:54.256 INFO:tasks.rados.rados.0.smithi163.stdout:2655: writing smithi16367185-38 from 756579 to 1422075 tid 1 2023-10-09T09:16:54.262 INFO:tasks.rados.rados.0.smithi163.stdout:2655: writing smithi16367185-38 from 1958665 to 2359012 tid 2 2023-10-09T09:16:54.263 INFO:tasks.rados.rados.0.smithi163.stdout:2656: write oid 16 current snap is 342 2023-10-09T09:16:54.263 INFO:tasks.rados.rados.0.smithi163.stdout:2656: seq_num 998 ranges {773386=740745,2017147=544005} 2023-10-09T09:16:54.273 INFO:tasks.rados.rados.0.smithi163.stdout:2656: writing smithi16367185-16 from 773386 to 1514131 tid 1 2023-10-09T09:16:54.281 INFO:tasks.rados.rados.0.smithi163.stdout:2656: writing smithi16367185-16 from 2017147 to 2561152 tid 2 2023-10-09T09:16:54.281 INFO:tasks.rados.rados.0.smithi163.stdout:2657: rollback oid 4 current snap is 342 2023-10-09T09:16:54.281 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 327 2023-10-09T09:16:54.282 INFO:tasks.rados.rados.0.smithi163.stdout:2655: finishing write tid 1 to smithi16367185-38 2023-10-09T09:16:54.282 INFO:tasks.rados.rados.0.smithi163.stdout:2658: read oid 34 snap 340 2023-10-09T09:16:54.282 INFO:tasks.rados.rados.0.smithi163.stdout:2658: expect (ObjNum 969 snap 332 seq_num 969) 2023-10-09T09:16:54.282 INFO:tasks.rados.rados.0.smithi163.stdout:2659: delete oid 7 current snap is 342 2023-10-09T09:16:54.282 INFO:tasks.rados.rados.0.smithi163.stdout:2655: finishing write tid 2 to smithi16367185-38 2023-10-09T09:16:54.282 INFO:tasks.rados.rados.0.smithi163.stdout:2655: finishing write tid 3 to smithi16367185-38 2023-10-09T09:16:54.282 INFO:tasks.rados.rados.0.smithi163.stdout:2655: finishing write tid 4 to smithi16367185-38 2023-10-09T09:16:54.283 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1243 (ObjNum 997 snap 342 seq_num 997) dirty exists 2023-10-09T09:16:54.283 INFO:tasks.rados.rados.0.smithi163.stdout:2655: left oid 38 (ObjNum 997 snap 342 seq_num 997) 2023-10-09T09:16:54.283 INFO:tasks.rados.rados.0.smithi163.stdout:2656: finishing write tid 1 to smithi16367185-16 2023-10-09T09:16:54.287 INFO:tasks.rados.rados.0.smithi163.stdout:2658: expect (ObjNum 969 snap 332 seq_num 969) 2023-10-09T09:16:54.399 INFO:tasks.rados.rados.0.smithi163.stdout:2656: finishing write tid 2 to smithi16367185-16 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:2656: finishing write tid 3 to smithi16367185-16 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:2656: finishing write tid 4 to smithi16367185-16 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1256 (ObjNum 998 snap 342 seq_num 998) dirty exists 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:2656: left oid 16 (ObjNum 998 snap 342 seq_num 998) 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:2657: finishing rollback tid 0 to smithi16367185-4 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:2657: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:2657: finishing rollback tid 2 to smithi16367185-4 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1260 (ObjNum 963 snap 327 seq_num 963) dirty exists 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:2655: done (4 left) 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:2656: done (3 left) 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:2657: done (2 left) 2023-10-09T09:16:54.400 INFO:tasks.rados.rados.0.smithi163.stdout:2658: done (1 left) 2023-10-09T09:16:54.401 INFO:tasks.rados.rados.0.smithi163.stdout:2659: done (0 left) 2023-10-09T09:16:54.401 INFO:tasks.rados.rados.0.smithi163.stdout:2660: read oid 18 snap -1 2023-10-09T09:16:54.401 INFO:tasks.rados.rados.0.smithi163.stdout:2660: expect deleted 2023-10-09T09:16:54.401 INFO:tasks.rados.rados.0.smithi163.stdout:2661: read oid 16 snap -1 2023-10-09T09:16:54.401 INFO:tasks.rados.rados.0.smithi163.stdout:2661: expect (ObjNum 998 snap 342 seq_num 998) 2023-10-09T09:16:54.401 INFO:tasks.rados.rados.0.smithi163.stdout:2662: write oid 45 current snap is 342 2023-10-09T09:16:54.401 INFO:tasks.rados.rados.0.smithi163.stdout:2662: seq_num 1000 ranges {407000=491243,1617758=547674,2595228=211772} 2023-10-09T09:16:54.407 INFO:tasks.rados.rados.0.smithi163.stdout:2662: writing smithi16367185-45 from 407000 to 898243 tid 1 2023-10-09T09:16:54.415 INFO:tasks.rados.rados.0.smithi163.stdout:2662: writing smithi16367185-45 from 1617758 to 2165432 tid 2 2023-10-09T09:16:54.419 INFO:tasks.rados.rados.0.smithi163.stdout:2662: writing smithi16367185-45 from 2595228 to 2807000 tid 3 2023-10-09T09:16:54.419 INFO:tasks.rados.rados.0.smithi163.stdout:2663: write oid 27 current snap is 342 2023-10-09T09:16:54.419 INFO:tasks.rados.rados.0.smithi163.stdout:2663: seq_num 1001 ranges {423807=566492,1676240=407685,2739004=84803} 2023-10-09T09:16:54.425 INFO:tasks.rados.rados.0.smithi163.stdout:2663: writing smithi16367185-27 from 423807 to 990299 tid 1 2023-10-09T09:16:54.431 INFO:tasks.rados.rados.0.smithi163.stdout:2663: writing smithi16367185-27 from 1676240 to 2083925 tid 2 2023-10-09T09:16:54.436 INFO:tasks.rados.rados.0.smithi163.stdout:2663: writing smithi16367185-27 from 2739004 to 2823807 tid 3 2023-10-09T09:16:54.436 INFO:tasks.rados.rados.0.smithi163.stdout:2664: write oid 6 current snap is 342 2023-10-09T09:16:54.436 INFO:tasks.rados.rados.0.smithi163.stdout:2664: seq_num 1002 ranges {440614=641741,1734722=551343} 2023-10-09T09:16:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:2664: writing smithi16367185-6 from 440614 to 1082355 tid 1 2023-10-09T09:16:54.449 INFO:tasks.rados.rados.0.smithi163.stdout:2664: writing smithi16367185-6 from 1734722 to 2286065 tid 2 2023-10-09T09:16:54.449 INFO:tasks.rados.rados.0.smithi163.stdout:2665: snap_remove snap 333 2023-10-09T09:16:54.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[78107]: cluster 2023-10-09T09:16:53.216472+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:54.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[78107]: cluster 2023-10-09T09:16:53.247273+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:54.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[78107]: cluster 2023-10-09T09:16:53.262165+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[78107]: cluster 2023-10-09T09:16:53.378746+0000 mgr.y (mgr.34107) 310 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:16:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[78107]: cluster 2023-10-09T09:16:54.236555+0000 mon.a (mon.0) 921 : cluster 0 osdmap e741: 8 total, 8 up, 8 in 2023-10-09T09:16:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[78107]: cluster 2023-10-09T09:16:54.237480+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[75425]: cluster 2023-10-09T09:16:53.216472+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[75425]: cluster 2023-10-09T09:16:53.247273+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[75425]: cluster 2023-10-09T09:16:53.262165+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[75425]: cluster 2023-10-09T09:16:53.378746+0000 mgr.y (mgr.34107) 310 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:16:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[75425]: cluster 2023-10-09T09:16:54.236555+0000 mon.a (mon.0) 921 : cluster 0 osdmap e741: 8 total, 8 up, 8 in 2023-10-09T09:16:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:54 smithi163 bash[75425]: cluster 2023-10-09T09:16:54.237480+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:54 smithi181 bash[59345]: cluster 2023-10-09T09:16:53.216472+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:54 smithi181 bash[59345]: cluster 2023-10-09T09:16:53.247273+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:54 smithi181 bash[59345]: cluster 2023-10-09T09:16:53.262165+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e740 with expected crc 2023-10-09T09:16:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:54 smithi181 bash[59345]: cluster 2023-10-09T09:16:53.378746+0000 mgr.y (mgr.34107) 310 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:16:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:54 smithi181 bash[59345]: cluster 2023-10-09T09:16:54.236555+0000 mon.a (mon.0) 921 : cluster 0 osdmap e741: 8 total, 8 up, 8 in 2023-10-09T09:16:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:54 smithi181 bash[59345]: cluster 2023-10-09T09:16:54.237480+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:55.180 INFO:tasks.rados.rados.0.smithi163.stdout:2665: done (5 left) 2023-10-09T09:16:55.180 INFO:tasks.rados.rados.0.smithi163.stdout:2666: rollback oid 7 current snap is 342 2023-10-09T09:16:55.180 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 341 2023-10-09T09:16:55.181 INFO:tasks.rados.rados.0.smithi163.stdout:2661: expect (ObjNum 998 snap 342 seq_num 998) 2023-10-09T09:16:55.290 INFO:tasks.rados.rados.0.smithi163.stdout:2662: finishing write tid 1 to smithi16367185-45 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:2662: finishing write tid 2 to smithi16367185-45 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:2662: finishing write tid 3 to smithi16367185-45 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:2662: finishing write tid 4 to smithi16367185-45 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:2662: finishing write tid 5 to smithi16367185-45 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 1882 (ObjNum 1000 snap 342 seq_num 1000) dirty exists 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:2662: left oid 45 (ObjNum 1000 snap 342 seq_num 1000) 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:2663: finishing write tid 1 to smithi16367185-27 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:2663: finishing write tid 2 to smithi16367185-27 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:2663: finishing write tid 3 to smithi16367185-27 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:2660: done (5 left) 2023-10-09T09:16:55.291 INFO:tasks.rados.rados.0.smithi163.stdout:2661: done (4 left) 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:2662: done (3 left) 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:2667: write oid 33 current snap is 342 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:2663: finishing write tid 4 to smithi16367185-27 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:2663: finishing write tid 5 to smithi16367185-27 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 726 (ObjNum 1001 snap 342 seq_num 1001) dirty exists 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:2663: left oid 27 (ObjNum 1001 snap 342 seq_num 1001) 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:2664: finishing write tid 1 to smithi16367185-6 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:2664: finishing write tid 2 to smithi16367185-6 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:2664: finishing write tid 3 to smithi16367185-6 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:2664: finishing write tid 4 to smithi16367185-6 2023-10-09T09:16:55.292 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 270 (ObjNum 1002 snap 342 seq_num 1002) dirty exists 2023-10-09T09:16:55.293 INFO:tasks.rados.rados.0.smithi163.stdout:2664: left oid 6 (ObjNum 1002 snap 342 seq_num 1002) 2023-10-09T09:16:55.293 INFO:tasks.rados.rados.0.smithi163.stdout:2666: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:16:55.293 INFO:tasks.rados.rados.0.smithi163.stdout:2667: seq_num 1003 ranges {457421=716990,1676851=411354,2677497=179924} 2023-10-09T09:16:55.304 INFO:tasks.rados.rados.0.smithi163.stdout:2667: writing smithi16367185-33 from 457421 to 1174411 tid 1 2023-10-09T09:16:55.310 INFO:tasks.rados.rados.0.smithi163.stdout:2667: writing smithi16367185-33 from 1676851 to 2088205 tid 2 2023-10-09T09:16:55.314 INFO:tasks.rados.rados.0.smithi163.stdout:2667: writing smithi16367185-33 from 2677497 to 2857421 tid 3 2023-10-09T09:16:55.314 INFO:tasks.rados.rados.0.smithi163.stdout:2663: done (3 left) 2023-10-09T09:16:55.314 INFO:tasks.rados.rados.0.smithi163.stdout:2664: done (2 left) 2023-10-09T09:16:55.314 INFO:tasks.rados.rados.0.smithi163.stdout:2668: snap_remove snap 334 2023-10-09T09:16:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:55 smithi163 bash[75425]: cluster 2023-10-09T09:16:54.231447+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:55 smithi163 bash[75425]: cluster 2023-10-09T09:16:54.237944+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:55 smithi163 bash[75425]: cluster 2023-10-09T09:16:55.179614+0000 mon.a (mon.0) 922 : cluster 0 osdmap e742: 8 total, 8 up, 8 in 2023-10-09T09:16:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:55 smithi163 bash[75425]: cluster 2023-10-09T09:16:55.185811+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:55 smithi163 bash[75425]: audit 2023-10-09T09:16:55.345735+0000 mon.a (mon.0) 923 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:55 smithi163 bash[78107]: cluster 2023-10-09T09:16:54.231447+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:55 smithi163 bash[78107]: cluster 2023-10-09T09:16:54.237944+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:55 smithi163 bash[78107]: cluster 2023-10-09T09:16:55.179614+0000 mon.a (mon.0) 922 : cluster 0 osdmap e742: 8 total, 8 up, 8 in 2023-10-09T09:16:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:55 smithi163 bash[78107]: cluster 2023-10-09T09:16:55.185811+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:55 smithi163 bash[78107]: audit 2023-10-09T09:16:55.345735+0000 mon.a (mon.0) 923 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:55.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:55 smithi181 bash[59345]: cluster 2023-10-09T09:16:54.231447+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:55 smithi181 bash[59345]: cluster 2023-10-09T09:16:54.237944+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:55 smithi181 bash[59345]: cluster 2023-10-09T09:16:55.179614+0000 mon.a (mon.0) 922 : cluster 0 osdmap e742: 8 total, 8 up, 8 in 2023-10-09T09:16:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:55 smithi181 bash[59345]: cluster 2023-10-09T09:16:55.185811+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:55 smithi181 bash[59345]: audit 2023-10-09T09:16:55.345735+0000 mon.a (mon.0) 923 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:16:56.196 INFO:tasks.rados.rados.0.smithi163.stdout:2668: done (2 left) 2023-10-09T09:16:56.197 INFO:tasks.rados.rados.0.smithi163.stdout:2669: read oid 19 snap -1 2023-10-09T09:16:56.197 INFO:tasks.rados.rados.0.smithi163.stdout:2669: expect (ObjNum 908 snap 311 seq_num 908) 2023-10-09T09:16:56.197 INFO:tasks.rados.rados.0.smithi163.stdout:2670: write oid 25 current snap is 342 2023-10-09T09:16:56.197 INFO:tasks.rados.rados.0.smithi163.stdout:2670: seq_num 1004 ranges {474228=508592,1451686=555012,2704920=169308} 2023-10-09T09:16:56.203 INFO:tasks.rados.rados.0.smithi163.stdout:2670: writing smithi16367185-25 from 474228 to 982820 tid 1 2023-10-09T09:16:56.209 INFO:tasks.rados.rados.0.smithi163.stdout:2670: writing smithi16367185-25 from 1451686 to 2006698 tid 2 2023-10-09T09:16:56.218 INFO:tasks.rados.rados.0.smithi163.stdout:2670: writing smithi16367185-25 from 2704920 to 2874228 tid 3 2023-10-09T09:16:56.218 INFO:tasks.rados.rados.0.smithi163.stdout:2671: read oid 32 snap 339 2023-10-09T09:16:56.218 INFO:tasks.rados.rados.0.smithi163.stdout:2671: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:16:56.218 INFO:tasks.rados.rados.0.smithi163.stdout:2672: write oid 28 current snap is 342 2023-10-09T09:16:56.218 INFO:tasks.rados.rados.0.smithi163.stdout:2666: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:16:56.218 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 636 (ObjNum 975 snap 336 seq_num 975) dirty exists 2023-10-09T09:16:56.218 INFO:tasks.rados.rados.0.smithi163.stdout:2667: finishing write tid 1 to smithi16367185-33 2023-10-09T09:16:56.218 INFO:tasks.rados.rados.0.smithi163.stdout:2667: finishing write tid 2 to smithi16367185-33 2023-10-09T09:16:56.218 INFO:tasks.rados.rados.0.smithi163.stdout:2672: seq_num 1005 ranges {491035=583841,1510168=698670,2732343=158692} 2023-10-09T09:16:56.221 INFO:tasks.rados.rados.0.smithi163.stdout:2672: writing smithi16367185-28 from 491035 to 1074876 tid 1 2023-10-09T09:16:56.228 INFO:tasks.rados.rados.0.smithi163.stdout:2672: writing smithi16367185-28 from 1510168 to 2208838 tid 2 2023-10-09T09:16:56.232 INFO:tasks.rados.rados.0.smithi163.stdout:2672: writing smithi16367185-28 from 2732343 to 2891035 tid 3 2023-10-09T09:16:56.232 INFO:tasks.rados.rados.0.smithi163.stdout:2666: done (5 left) 2023-10-09T09:16:56.233 INFO:tasks.rados.rados.0.smithi163.stdout:2673: snap_create 2023-10-09T09:16:56.233 INFO:tasks.rados.rados.0.smithi163.stdout:2667: finishing write tid 3 to smithi16367185-33 2023-10-09T09:16:56.233 INFO:tasks.rados.rados.0.smithi163.stdout:2667: finishing write tid 4 to smithi16367185-33 2023-10-09T09:16:56.233 INFO:tasks.rados.rados.0.smithi163.stdout:2667: finishing write tid 5 to smithi16367185-33 2023-10-09T09:16:56.233 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1273 (ObjNum 1003 snap 342 seq_num 1003) dirty exists 2023-10-09T09:16:56.233 INFO:tasks.rados.rados.0.smithi163.stdout:2667: left oid 33 (ObjNum 1003 snap 342 seq_num 1003) 2023-10-09T09:16:56.233 INFO:tasks.rados.rados.0.smithi163.stdout:2669: expect (ObjNum 908 snap 311 seq_num 908) 2023-10-09T09:16:56.432 INFO:tasks.rados.rados.0.smithi163.stdout:2670: finishing write tid 1 to smithi16367185-25 2023-10-09T09:16:56.432 INFO:tasks.rados.rados.0.smithi163.stdout:2670: finishing write tid 2 to smithi16367185-25 2023-10-09T09:16:56.432 INFO:tasks.rados.rados.0.smithi163.stdout:2670: finishing write tid 3 to smithi16367185-25 2023-10-09T09:16:56.432 INFO:tasks.rados.rados.0.smithi163.stdout:2670: finishing write tid 4 to smithi16367185-25 2023-10-09T09:16:56.432 INFO:tasks.rados.rados.0.smithi163.stdout:2670: finishing write tid 5 to smithi16367185-25 2023-10-09T09:16:56.432 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1280 (ObjNum 1004 snap 342 seq_num 1004) dirty exists 2023-10-09T09:16:56.432 INFO:tasks.rados.rados.0.smithi163.stdout:2670: left oid 25 (ObjNum 1004 snap 342 seq_num 1004) 2023-10-09T09:16:56.432 INFO:tasks.rados.rados.0.smithi163.stdout:2671: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:16:56.621 INFO:tasks.rados.rados.0.smithi163.stdout:2672: finishing write tid 1 to smithi16367185-28 2023-10-09T09:16:56.621 INFO:tasks.rados.rados.0.smithi163.stdout:2672: finishing write tid 2 to smithi16367185-28 2023-10-09T09:16:56.621 INFO:tasks.rados.rados.0.smithi163.stdout:2672: finishing write tid 3 to smithi16367185-28 2023-10-09T09:16:56.621 INFO:tasks.rados.rados.0.smithi163.stdout:2672: finishing write tid 4 to smithi16367185-28 2023-10-09T09:16:56.621 INFO:tasks.rados.rados.0.smithi163.stdout:2672: finishing write tid 5 to smithi16367185-28 2023-10-09T09:16:56.621 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 1280 (ObjNum 1005 snap 342 seq_num 1005) dirty exists 2023-10-09T09:16:56.621 INFO:tasks.rados.rados.0.smithi163.stdout:2672: left oid 28 (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:16:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[75425]: cluster 2023-10-09T09:16:54.743463+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[75425]: cluster 2023-10-09T09:16:55.173598+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[75425]: cluster 2023-10-09T09:16:55.179174+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[75425]: cluster 2023-10-09T09:16:55.179182+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[75425]: cluster 2023-10-09T09:16:55.379433+0000 mgr.y (mgr.34107) 311 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 212 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:16:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[75425]: cluster 2023-10-09T09:16:56.192434+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e743 with expected crc 2023-10-09T09:16:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[75425]: cluster 2023-10-09T09:16:56.194631+0000 mon.a (mon.0) 924 : cluster 0 osdmap e743: 8 total, 8 up, 8 in 2023-10-09T09:16:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[75425]: cluster 2023-10-09T09:16:56.195925+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e743 with expected crc 2023-10-09T09:16:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[78107]: cluster 2023-10-09T09:16:54.743463+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[78107]: cluster 2023-10-09T09:16:55.173598+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[78107]: cluster 2023-10-09T09:16:55.179174+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[78107]: cluster 2023-10-09T09:16:55.179182+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[78107]: cluster 2023-10-09T09:16:55.379433+0000 mgr.y (mgr.34107) 311 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 212 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:16:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[78107]: cluster 2023-10-09T09:16:56.192434+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e743 with expected crc 2023-10-09T09:16:56.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[78107]: cluster 2023-10-09T09:16:56.194631+0000 mon.a (mon.0) 924 : cluster 0 osdmap e743: 8 total, 8 up, 8 in 2023-10-09T09:16:56.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:56 smithi163 bash[78107]: cluster 2023-10-09T09:16:56.195925+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e743 with expected crc 2023-10-09T09:16:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:56 smithi181 bash[59345]: cluster 2023-10-09T09:16:54.743463+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e741 with expected crc 2023-10-09T09:16:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:56 smithi181 bash[59345]: cluster 2023-10-09T09:16:55.173598+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:56 smithi181 bash[59345]: cluster 2023-10-09T09:16:55.179174+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:56 smithi181 bash[59345]: cluster 2023-10-09T09:16:55.179182+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e742 with expected crc 2023-10-09T09:16:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:56 smithi181 bash[59345]: cluster 2023-10-09T09:16:55.379433+0000 mgr.y (mgr.34107) 311 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 212 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:16:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:56 smithi181 bash[59345]: cluster 2023-10-09T09:16:56.192434+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e743 with expected crc 2023-10-09T09:16:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:56 smithi181 bash[59345]: cluster 2023-10-09T09:16:56.194631+0000 mon.a (mon.0) 924 : cluster 0 osdmap e743: 8 total, 8 up, 8 in 2023-10-09T09:16:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:56 smithi181 bash[59345]: cluster 2023-10-09T09:16:56.195925+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e743 with expected crc 2023-10-09T09:16:57.198 INFO:tasks.rados.rados.0.smithi163.stdout:2667: done (5 left) 2023-10-09T09:16:57.199 INFO:tasks.rados.rados.0.smithi163.stdout:2669: done (4 left) 2023-10-09T09:16:57.199 INFO:tasks.rados.rados.0.smithi163.stdout:2670: done (3 left) 2023-10-09T09:16:57.199 INFO:tasks.rados.rados.0.smithi163.stdout:2671: done (2 left) 2023-10-09T09:16:57.199 INFO:tasks.rados.rados.0.smithi163.stdout:2672: done (1 left) 2023-10-09T09:16:57.199 INFO:tasks.rados.rados.0.smithi163.stdout:2673: done (0 left) 2023-10-09T09:16:57.199 INFO:tasks.rados.rados.0.smithi163.stdout:2674: delete oid 4 current snap is 343 2023-10-09T09:16:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:2674: done (0 left) 2023-10-09T09:16:57.203 INFO:tasks.rados.rados.0.smithi163.stdout:2675: rollback oid 2 current snap is 343 2023-10-09T09:16:57.204 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 2 to 336 2023-10-09T09:16:57.204 INFO:tasks.rados.rados.0.smithi163.stdout:2676: snap_create 2023-10-09T09:16:57.211 INFO:tasks.rados.rados.0.smithi163.stdout:2675: finishing rollback tid 1 to smithi16367185-2 2023-10-09T09:16:57.212 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 627 (ObjNum 669367424 snap 21875 seq_num 4000000) dirty dne 2023-10-09T09:16:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:57 smithi163 bash[78107]: cluster 2023-10-09T09:16:56.190063+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e743 with expected crc 2023-10-09T09:16:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:57 smithi163 bash[78107]: cluster 2023-10-09T09:16:57.192971+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e744 with expected crc 2023-10-09T09:16:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:57 smithi163 bash[78107]: cluster 2023-10-09T09:16:57.198314+0000 mon.a (mon.0) 925 : cluster 0 osdmap e744: 8 total, 8 up, 8 in 2023-10-09T09:16:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:57 smithi163 bash[75425]: cluster 2023-10-09T09:16:56.190063+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e743 with expected crc 2023-10-09T09:16:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:57 smithi163 bash[75425]: cluster 2023-10-09T09:16:57.192971+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e744 with expected crc 2023-10-09T09:16:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:57 smithi163 bash[75425]: cluster 2023-10-09T09:16:57.198314+0000 mon.a (mon.0) 925 : cluster 0 osdmap e744: 8 total, 8 up, 8 in 2023-10-09T09:16:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:57 smithi181 bash[59345]: cluster 2023-10-09T09:16:56.190063+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e743 with expected crc 2023-10-09T09:16:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:57 smithi181 bash[59345]: cluster 2023-10-09T09:16:57.192971+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e744 with expected crc 2023-10-09T09:16:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:57 smithi181 bash[59345]: cluster 2023-10-09T09:16:57.198314+0000 mon.a (mon.0) 925 : cluster 0 osdmap e744: 8 total, 8 up, 8 in 2023-10-09T09:16:58.205 INFO:tasks.rados.rados.0.smithi163.stdout:2675: done (1 left) 2023-10-09T09:16:58.205 INFO:tasks.rados.rados.0.smithi163.stdout:2676: done (0 left) 2023-10-09T09:16:58.206 INFO:tasks.rados.rados.0.smithi163.stdout:2677: read oid 35 snap -1 2023-10-09T09:16:58.206 INFO:tasks.rados.rados.0.smithi163.stdout:2677: expect (ObjNum 939 snap 322 seq_num 939) 2023-10-09T09:16:58.206 INFO:tasks.rados.rados.0.smithi163.stdout:2678: write oid 26 current snap is 344 2023-10-09T09:16:58.206 INFO:tasks.rados.rados.0.smithi163.stdout:2678: seq_num 1007 ranges {524649=734339,1910779=702339} 2023-10-09T09:16:58.215 INFO:tasks.rados.rados.0.smithi163.stdout:2678: writing smithi16367185-26 from 524649 to 1258988 tid 1 2023-10-09T09:16:58.224 INFO:tasks.rados.rados.0.smithi163.stdout:2678: writing smithi16367185-26 from 1910779 to 2613118 tid 2 2023-10-09T09:16:58.224 INFO:tasks.rados.rados.0.smithi163.stdout:2679: write oid 41 current snap is 344 2023-10-09T09:16:58.224 INFO:tasks.rados.rados.0.smithi163.stdout:2679: seq_num 1008 ranges {541456=409588,1569261=562350,2814612=126844} 2023-10-09T09:16:58.229 INFO:tasks.rados.rados.0.smithi163.stdout:2679: writing smithi16367185-41 from 541456 to 951044 tid 1 2023-10-09T09:16:58.236 INFO:tasks.rados.rados.0.smithi163.stdout:2679: writing smithi16367185-41 from 1569261 to 2131611 tid 2 2023-10-09T09:16:58.241 INFO:tasks.rados.rados.0.smithi163.stdout:2679: writing smithi16367185-41 from 2814612 to 2941456 tid 3 2023-10-09T09:16:58.241 INFO:tasks.rados.rados.0.smithi163.stdout:2680: read oid 32 snap 339 2023-10-09T09:16:58.241 INFO:tasks.rados.rados.0.smithi163.stdout:2680: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:16:58.241 INFO:tasks.rados.rados.0.smithi163.stdout:2681: write oid 19 current snap is 344 2023-10-09T09:16:58.241 INFO:tasks.rados.rados.0.smithi163.stdout:2681: seq_num 1009 ranges {558263=484837,1627743=706008} 2023-10-09T09:16:58.247 INFO:tasks.rados.rados.0.smithi163.stdout:2681: writing smithi16367185-19 from 558263 to 1043100 tid 1 2023-10-09T09:16:58.255 INFO:tasks.rados.rados.0.smithi163.stdout:2681: writing smithi16367185-19 from 1627743 to 2333751 tid 2 2023-10-09T09:16:58.255 INFO:tasks.rados.rados.0.smithi163.stdout:2682: read oid 39 snap -1 2023-10-09T09:16:58.255 INFO:tasks.rados.rados.0.smithi163.stdout:2682: expect (ObjNum 932 snap 317 seq_num 932) 2023-10-09T09:16:58.255 INFO:tasks.rados.rados.0.smithi163.stdout:2683: rollback oid 7 current snap is 344 2023-10-09T09:16:58.255 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 328 2023-10-09T09:16:58.255 INFO:tasks.rados.rados.0.smithi163.stdout:2677: expect (ObjNum 939 snap 322 seq_num 939) 2023-10-09T09:16:58.385 INFO:tasks.rados.rados.0.smithi163.stdout:2678: finishing write tid 1 to smithi16367185-26 2023-10-09T09:16:58.385 INFO:tasks.rados.rados.0.smithi163.stdout:2678: finishing write tid 2 to smithi16367185-26 2023-10-09T09:16:58.385 INFO:tasks.rados.rados.0.smithi163.stdout:2678: finishing write tid 3 to smithi16367185-26 2023-10-09T09:16:58.385 INFO:tasks.rados.rados.0.smithi163.stdout:2678: finishing write tid 4 to smithi16367185-26 2023-10-09T09:16:58.386 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 730 (ObjNum 1007 snap 344 seq_num 1007) dirty exists 2023-10-09T09:16:58.386 INFO:tasks.rados.rados.0.smithi163.stdout:2678: left oid 26 (ObjNum 1007 snap 344 seq_num 1007) 2023-10-09T09:16:58.386 INFO:tasks.rados.rados.0.smithi163.stdout:2679: finishing write tid 1 to smithi16367185-41 2023-10-09T09:16:58.386 INFO:tasks.rados.rados.0.smithi163.stdout:2679: finishing write tid 2 to smithi16367185-41 2023-10-09T09:16:58.386 INFO:tasks.rados.rados.0.smithi163.stdout:2679: finishing write tid 3 to smithi16367185-41 2023-10-09T09:16:58.386 INFO:tasks.rados.rados.0.smithi163.stdout:2679: finishing write tid 4 to smithi16367185-41 2023-10-09T09:16:58.386 INFO:tasks.rados.rados.0.smithi163.stdout:2679: finishing write tid 5 to smithi16367185-41 2023-10-09T09:16:58.386 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 223 (ObjNum 1008 snap 344 seq_num 1008) dirty exists 2023-10-09T09:16:58.386 INFO:tasks.rados.rados.0.smithi163.stdout:2679: left oid 41 (ObjNum 1008 snap 344 seq_num 1008) 2023-10-09T09:16:58.386 INFO:tasks.rados.rados.0.smithi163.stdout:2680: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:16:58.578 INFO:tasks.rados.rados.0.smithi163.stdout:2681: finishing write tid 1 to smithi16367185-19 2023-10-09T09:16:58.578 INFO:tasks.rados.rados.0.smithi163.stdout:2681: finishing write tid 2 to smithi16367185-19 2023-10-09T09:16:58.578 INFO:tasks.rados.rados.0.smithi163.stdout:2681: finishing write tid 3 to smithi16367185-19 2023-10-09T09:16:58.578 INFO:tasks.rados.rados.0.smithi163.stdout:2681: finishing write tid 4 to smithi16367185-19 2023-10-09T09:16:58.578 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 734 (ObjNum 1009 snap 344 seq_num 1009) dirty exists 2023-10-09T09:16:58.578 INFO:tasks.rados.rados.0.smithi163.stdout:2681: left oid 19 (ObjNum 1009 snap 344 seq_num 1009) 2023-10-09T09:16:58.578 INFO:tasks.rados.rados.0.smithi163.stdout:2682: expect (ObjNum 932 snap 317 seq_num 932) 2023-10-09T09:16:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:58 smithi163 bash[75425]: cluster 2023-10-09T09:16:57.192224+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e744 with expected crc 2023-10-09T09:16:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:58 smithi163 bash[75425]: cluster 2023-10-09T09:16:57.204594+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e744 with expected crc 2023-10-09T09:16:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:58 smithi163 bash[75425]: cluster 2023-10-09T09:16:57.380126+0000 mgr.y (mgr.34107) 312 : cluster 0 pgmap v669: 89 pgs: 89 active+clean; 223 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.5 MiB/s wr, 21 op/s 2023-10-09T09:16:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:58 smithi163 bash[75425]: cluster 2023-10-09T09:16:58.205047+0000 mon.a (mon.0) 926 : cluster 0 osdmap e745: 8 total, 8 up, 8 in 2023-10-09T09:16:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:58 smithi163 bash[75425]: cluster 2023-10-09T09:16:58.259133+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:58 smithi163 bash[78107]: cluster 2023-10-09T09:16:57.192224+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e744 with expected crc 2023-10-09T09:16:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:58 smithi163 bash[78107]: cluster 2023-10-09T09:16:57.204594+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e744 with expected crc 2023-10-09T09:16:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:58 smithi163 bash[78107]: cluster 2023-10-09T09:16:57.380126+0000 mgr.y (mgr.34107) 312 : cluster 0 pgmap v669: 89 pgs: 89 active+clean; 223 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.5 MiB/s wr, 21 op/s 2023-10-09T09:16:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:58 smithi163 bash[78107]: cluster 2023-10-09T09:16:58.205047+0000 mon.a (mon.0) 926 : cluster 0 osdmap e745: 8 total, 8 up, 8 in 2023-10-09T09:16:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:58 smithi163 bash[78107]: cluster 2023-10-09T09:16:58.259133+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:58.755 INFO:tasks.rados.rados.0.smithi163.stdout:2683: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:16:58.755 INFO:tasks.rados.rados.0.smithi163.stdout:2683: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:16:58.755 INFO:tasks.rados.rados.0.smithi163.stdout:2683: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:16:58.755 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 648 (ObjNum 961 snap 326 seq_num 961) dirty exists 2023-10-09T09:16:58.755 INFO:tasks.rados.rados.0.smithi163.stdout:2677: done (6 left) 2023-10-09T09:16:58.755 INFO:tasks.rados.rados.0.smithi163.stdout:2678: done (5 left) 2023-10-09T09:16:58.755 INFO:tasks.rados.rados.0.smithi163.stdout:2679: done (4 left) 2023-10-09T09:16:58.755 INFO:tasks.rados.rados.0.smithi163.stdout:2680: done (3 left) 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2681: done (2 left) 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2682: done (1 left) 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2683: done (0 left) 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2684: read oid 20 snap -1 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2684: expect (ObjNum 994 snap 341 seq_num 994) 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2685: read oid 47 snap 328 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2685: expect deleted 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2686: read oid 17 snap -1 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2686: expect deleted 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2687: read oid 50 snap 322 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2687: expect deleted 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2688: write oid 32 current snap is 344 2023-10-09T09:16:58.756 INFO:tasks.rados.rados.0.smithi163.stdout:2688: seq_num 1010 ranges {575070=560086,1686225=566019,2869458=105612} 2023-10-09T09:16:58.765 INFO:tasks.rados.rados.0.smithi163.stdout:2688: writing smithi16367185-32 from 575070 to 1135156 tid 1 2023-10-09T09:16:58.772 INFO:tasks.rados.rados.0.smithi163.stdout:2688: writing smithi16367185-32 from 1686225 to 2252244 tid 2 2023-10-09T09:16:58.776 INFO:tasks.rados.rados.0.smithi163.stdout:2688: writing smithi16367185-32 from 2869458 to 2975070 tid 3 2023-10-09T09:16:58.776 INFO:tasks.rados.rados.0.smithi163.stdout:2689: snap_remove snap 339 2023-10-09T09:16:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:58 smithi181 bash[59345]: cluster 2023-10-09T09:16:57.192224+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e744 with expected crc 2023-10-09T09:16:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:58 smithi181 bash[59345]: cluster 2023-10-09T09:16:57.204594+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e744 with expected crc 2023-10-09T09:16:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:58 smithi181 bash[59345]: cluster 2023-10-09T09:16:57.380126+0000 mgr.y (mgr.34107) 312 : cluster 0 pgmap v669: 89 pgs: 89 active+clean; 223 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.5 MiB/s wr, 21 op/s 2023-10-09T09:16:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:58 smithi181 bash[59345]: cluster 2023-10-09T09:16:58.205047+0000 mon.a (mon.0) 926 : cluster 0 osdmap e745: 8 total, 8 up, 8 in 2023-10-09T09:16:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:58 smithi181 bash[59345]: cluster 2023-10-09T09:16:58.259133+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:59.416 INFO:tasks.rados.rados.0.smithi163.stdout:2689: done (5 left) 2023-10-09T09:16:59.416 INFO:tasks.rados.rados.0.smithi163.stdout:2690: read oid 45 snap -1 2023-10-09T09:16:59.416 INFO:tasks.rados.rados.0.smithi163.stdout:2690: expect (ObjNum 1000 snap 342 seq_num 1000) 2023-10-09T09:16:59.416 INFO:tasks.rados.rados.0.smithi163.stdout:2691: read oid 14 snap -1 2023-10-09T09:16:59.416 INFO:tasks.rados.rados.0.smithi163.stdout:2691: expect deleted 2023-10-09T09:16:59.416 INFO:tasks.rados.rados.0.smithi163.stdout:2692: snap_create 2023-10-09T09:16:59.416 INFO:tasks.rados.rados.0.smithi163.stdout:2684: expect (ObjNum 994 snap 341 seq_num 994) 2023-10-09T09:16:59.605 INFO:tasks.rados.rados.0.smithi163.stdout:2688: finishing write tid 1 to smithi16367185-32 2023-10-09T09:16:59.606 INFO:tasks.rados.rados.0.smithi163.stdout:2688: finishing write tid 2 to smithi16367185-32 2023-10-09T09:16:59.606 INFO:tasks.rados.rados.0.smithi163.stdout:2688: finishing write tid 3 to smithi16367185-32 2023-10-09T09:16:59.606 INFO:tasks.rados.rados.0.smithi163.stdout:2688: finishing write tid 4 to smithi16367185-32 2023-10-09T09:16:59.606 INFO:tasks.rados.rados.0.smithi163.stdout:2688: finishing write tid 5 to smithi16367185-32 2023-10-09T09:16:59.606 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1891 (ObjNum 1010 snap 344 seq_num 1010) dirty exists 2023-10-09T09:16:59.606 INFO:tasks.rados.rados.0.smithi163.stdout:2688: left oid 32 (ObjNum 1010 snap 344 seq_num 1010) 2023-10-09T09:16:59.606 INFO:tasks.rados.rados.0.smithi163.stdout:2690: expect (ObjNum 1000 snap 342 seq_num 1000) 2023-10-09T09:16:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:59 smithi163 bash[78107]: cluster 2023-10-09T09:16:58.202421+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:59 smithi163 bash[78107]: cluster 2023-10-09T09:16:58.204969+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:16:59 smithi163 bash[78107]: cluster 2023-10-09T09:16:58.207847+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:59 smithi163 bash[75425]: cluster 2023-10-09T09:16:58.202421+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:59 smithi163 bash[75425]: cluster 2023-10-09T09:16:58.204969+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:59.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:16:59 smithi163 bash[75425]: cluster 2023-10-09T09:16:58.207847+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:59 smithi181 bash[59345]: cluster 2023-10-09T09:16:58.202421+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:59 smithi181 bash[59345]: cluster 2023-10-09T09:16:58.204969+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:16:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:16:59 smithi181 bash[59345]: cluster 2023-10-09T09:16:58.207847+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e745 with expected crc 2023-10-09T09:17:00.182 INFO:tasks.rados.rados.0.smithi163.stdout:2684: done (7 left) 2023-10-09T09:17:00.182 INFO:tasks.rados.rados.0.smithi163.stdout:2685: done (6 left) 2023-10-09T09:17:00.182 INFO:tasks.rados.rados.0.smithi163.stdout:2686: done (5 left) 2023-10-09T09:17:00.182 INFO:tasks.rados.rados.0.smithi163.stdout:2687: done (4 left) 2023-10-09T09:17:00.182 INFO:tasks.rados.rados.0.smithi163.stdout:2688: done (3 left) 2023-10-09T09:17:00.182 INFO:tasks.rados.rados.0.smithi163.stdout:2690: done (2 left) 2023-10-09T09:17:00.182 INFO:tasks.rados.rados.0.smithi163.stdout:2691: done (1 left) 2023-10-09T09:17:00.182 INFO:tasks.rados.rados.0.smithi163.stdout:2692: done (0 left) 2023-10-09T09:17:00.183 INFO:tasks.rados.rados.0.smithi163.stdout:2693: write oid 44 current snap is 345 2023-10-09T09:17:00.183 INFO:tasks.rados.rados.0.smithi163.stdout:2693: seq_num 1011 ranges {591877=635335,1628354=709677} 2023-10-09T09:17:00.191 INFO:tasks.rados.rados.0.smithi163.stdout:2693: writing smithi16367185-44 from 591877 to 1227212 tid 1 2023-10-09T09:17:00.198 INFO:tasks.rados.rados.0.smithi163.stdout:2693: writing smithi16367185-44 from 1628354 to 2338031 tid 2 2023-10-09T09:17:00.198 INFO:tasks.rados.rados.0.smithi163.stdout:2694: write oid 47 current snap is 345 2023-10-09T09:17:00.198 INFO:tasks.rados.rados.0.smithi163.stdout:2693: finishing write tid 1 to smithi16367185-44 2023-10-09T09:17:00.198 INFO:tasks.rados.rados.0.smithi163.stdout:2694: seq_num 1012 ranges {608684=426937,1803189=569688,2924304=84380} 2023-10-09T09:17:00.204 INFO:tasks.rados.rados.0.smithi163.stdout:2694: writing smithi16367185-47 from 608684 to 1035621 tid 1 2023-10-09T09:17:00.212 INFO:tasks.rados.rados.0.smithi163.stdout:2694: writing smithi16367185-47 from 1803189 to 2372877 tid 2 2023-10-09T09:17:00.216 INFO:tasks.rados.rados.0.smithi163.stdout:2694: writing smithi16367185-47 from 2924304 to 3008684 tid 3 2023-10-09T09:17:00.216 INFO:tasks.rados.rados.0.smithi163.stdout:2695: read oid 12 snap -1 2023-10-09T09:17:00.216 INFO:tasks.rados.rados.0.smithi163.stdout:2695: expect (ObjNum 953 snap 324 seq_num 953) 2023-10-09T09:17:00.216 INFO:tasks.rados.rados.0.smithi163.stdout:2696: write oid 14 current snap is 345 2023-10-09T09:17:00.217 INFO:tasks.rados.rados.0.smithi163.stdout:2693: finishing write tid 2 to smithi16367185-44 2023-10-09T09:17:00.217 INFO:tasks.rados.rados.0.smithi163.stdout:2693: finishing write tid 3 to smithi16367185-44 2023-10-09T09:17:00.217 INFO:tasks.rados.rados.0.smithi163.stdout:2696: seq_num 1013 ranges {625491=502186,1861671=713346} 2023-10-09T09:17:00.223 INFO:tasks.rados.rados.0.smithi163.stdout:2696: writing smithi16367185-14 from 625491 to 1127677 tid 1 2023-10-09T09:17:00.232 INFO:tasks.rados.rados.0.smithi163.stdout:2696: writing smithi16367185-14 from 1861671 to 2575017 tid 2 2023-10-09T09:17:00.232 INFO:tasks.rados.rados.0.smithi163.stdout:2697: rollback oid 16 current snap is 345 2023-10-09T09:17:00.232 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 16 to 314 2023-10-09T09:17:00.232 INFO:tasks.rados.rados.0.smithi163.stdout:2693: finishing write tid 4 to smithi16367185-44 2023-10-09T09:17:00.232 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1898 (ObjNum 1011 snap 345 seq_num 1011) dirty exists 2023-10-09T09:17:00.232 INFO:tasks.rados.rados.0.smithi163.stdout:2693: left oid 44 (ObjNum 1011 snap 345 seq_num 1011) 2023-10-09T09:17:00.232 INFO:tasks.rados.rados.0.smithi163.stdout:2694: finishing write tid 1 to smithi16367185-47 2023-10-09T09:17:00.232 INFO:tasks.rados.rados.0.smithi163.stdout:2693: done (4 left) 2023-10-09T09:17:00.233 INFO:tasks.rados.rados.0.smithi163.stdout:2698: snap_remove snap 343 2023-10-09T09:17:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[75425]: cluster 2023-10-09T09:16:59.381044+0000 mgr.y (mgr.34107) 313 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 227 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2023-10-09T09:17:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[75425]: cluster 2023-10-09T09:16:59.409638+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e746 with expected crc 2023-10-09T09:17:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[75425]: cluster 2023-10-09T09:16:59.412169+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e746 with expected crc 2023-10-09T09:17:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[75425]: cluster 2023-10-09T09:16:59.415625+0000 mon.a (mon.0) 927 : cluster 0 osdmap e746: 8 total, 8 up, 8 in 2023-10-09T09:17:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[75425]: cluster 2023-10-09T09:16:59.418272+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e746 with expected crc 2023-10-09T09:17:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[75425]: cluster 2023-10-09T09:17:00.181016+0000 mon.a (mon.0) 928 : cluster 0 osdmap e747: 8 total, 8 up, 8 in 2023-10-09T09:17:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[75425]: cluster 2023-10-09T09:17:00.225009+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[78107]: cluster 2023-10-09T09:16:59.381044+0000 mgr.y (mgr.34107) 313 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 227 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2023-10-09T09:17:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[78107]: cluster 2023-10-09T09:16:59.409638+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e746 with expected crc 2023-10-09T09:17:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[78107]: cluster 2023-10-09T09:16:59.412169+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e746 with expected crc 2023-10-09T09:17:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[78107]: cluster 2023-10-09T09:16:59.415625+0000 mon.a (mon.0) 927 : cluster 0 osdmap e746: 8 total, 8 up, 8 in 2023-10-09T09:17:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[78107]: cluster 2023-10-09T09:16:59.418272+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e746 with expected crc 2023-10-09T09:17:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[78107]: cluster 2023-10-09T09:17:00.181016+0000 mon.a (mon.0) 928 : cluster 0 osdmap e747: 8 total, 8 up, 8 in 2023-10-09T09:17:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:00 smithi163 bash[78107]: cluster 2023-10-09T09:17:00.225009+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:00 smithi181 bash[59345]: cluster 2023-10-09T09:16:59.381044+0000 mgr.y (mgr.34107) 313 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 227 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2023-10-09T09:17:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:00 smithi181 bash[59345]: cluster 2023-10-09T09:16:59.409638+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e746 with expected crc 2023-10-09T09:17:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:00 smithi181 bash[59345]: cluster 2023-10-09T09:16:59.412169+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e746 with expected crc 2023-10-09T09:17:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:00 smithi181 bash[59345]: cluster 2023-10-09T09:16:59.415625+0000 mon.a (mon.0) 927 : cluster 0 osdmap e746: 8 total, 8 up, 8 in 2023-10-09T09:17:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:00 smithi181 bash[59345]: cluster 2023-10-09T09:16:59.418272+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e746 with expected crc 2023-10-09T09:17:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:00 smithi181 bash[59345]: cluster 2023-10-09T09:17:00.181016+0000 mon.a (mon.0) 928 : cluster 0 osdmap e747: 8 total, 8 up, 8 in 2023-10-09T09:17:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:00 smithi181 bash[59345]: cluster 2023-10-09T09:17:00.225009+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:01.193 INFO:tasks.rados.rados.0.smithi163.stdout:2698: done (4 left) 2023-10-09T09:17:02.264 INFO:tasks.rados.rados.0.smithi163.stdout:2699: rollback oid 42 current snap is 345 2023-10-09T09:17:02.271 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 344 2023-10-09T09:17:02.271 INFO:tasks.rados.rados.0.smithi163.stdout:2694: finishing write tid 2 to smithi16367185-47 2023-10-09T09:17:02.271 INFO:tasks.rados.rados.0.smithi163.stdout:2694: finishing write tid 3 to smithi16367185-47 2023-10-09T09:17:02.272 INFO:tasks.rados.rados.0.smithi163.stdout:2694: finishing write tid 4 to smithi16367185-47 2023-10-09T09:17:02.272 INFO:tasks.rados.rados.0.smithi163.stdout:2694: finishing write tid 5 to smithi16367185-47 2023-10-09T09:17:02.272 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 1902 (ObjNum 1012 snap 345 seq_num 1012) dirty exists 2023-10-09T09:17:02.272 INFO:tasks.rados.rados.0.smithi163.stdout:2694: left oid 47 (ObjNum 1012 snap 345 seq_num 1012) 2023-10-09T09:17:02.272 INFO:tasks.rados.rados.0.smithi163.stdout:2695: expect (ObjNum 953 snap 324 seq_num 953) 2023-10-09T09:17:02.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:01 smithi181 bash[59345]: cluster 2023-10-09T09:17:00.176392+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:02.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:01 smithi181 bash[59345]: cluster 2023-10-09T09:17:00.358188+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:02.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:01 smithi181 bash[59345]: cluster 2023-10-09T09:17:01.193445+0000 mon.a (mon.0) 929 : cluster 0 osdmap e748: 8 total, 8 up, 8 in 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:2696: finishing write tid 1 to smithi16367185-14 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:2696: finishing write tid 2 to smithi16367185-14 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:2697: finishing rollback tid 0 to smithi16367185-16 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:2696: finishing write tid 3 to smithi16367185-14 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:2697: finishing rollback tid 1 to smithi16367185-16 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1289 (ObjNum 699152000 snap 21875 seq_num 1618308112) dirty dne 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:2696: finishing write tid 4 to smithi16367185-14 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1258 (ObjNum 1013 snap 345 seq_num 1013) dirty exists 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:2696: left oid 14 (ObjNum 1013 snap 345 seq_num 1013) 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:2699: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 0 (ObjNum 669367424 snap 21875 seq_num 4000000) dirty dne 2023-10-09T09:17:02.274 INFO:tasks.rados.rados.0.smithi163.stdout:2694: done (4 left) 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:2695: done (3 left) 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:2696: done (2 left) 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:2697: done (1 left) 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:2699: done (0 left) 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:2700: read oid 34 snap -1 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:2700: expect (ObjNum 969 snap 332 seq_num 969) 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:2701: read oid 11 snap -1 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:2701: expect (ObjNum 989 snap 339 seq_num 989) 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:2702: rollback oid 8 current snap is 345 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 8 to 327 2023-10-09T09:17:02.275 INFO:tasks.rados.rados.0.smithi163.stdout:2703: snap_remove snap 342 2023-10-09T09:17:02.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:01 smithi163 bash[75425]: cluster 2023-10-09T09:17:00.176392+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:02.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:01 smithi163 bash[75425]: cluster 2023-10-09T09:17:00.358188+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:02.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:01 smithi163 bash[75425]: cluster 2023-10-09T09:17:01.193445+0000 mon.a (mon.0) 929 : cluster 0 osdmap e748: 8 total, 8 up, 8 in 2023-10-09T09:17:02.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:01 smithi163 bash[78107]: cluster 2023-10-09T09:17:00.176392+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:02.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:01 smithi163 bash[78107]: cluster 2023-10-09T09:17:00.358188+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:02.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:01 smithi163 bash[78107]: cluster 2023-10-09T09:17:01.193445+0000 mon.a (mon.0) 929 : cluster 0 osdmap e748: 8 total, 8 up, 8 in 2023-10-09T09:17:02.277 INFO:tasks.rados.rados.0.smithi163.stdout:2703: done (3 left) 2023-10-09T09:17:02.277 INFO:tasks.rados.rados.0.smithi163.stdout:2704: rollback oid 45 current snap is 345 2023-10-09T09:17:02.277 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 45 to 319 2023-10-09T09:17:02.277 INFO:tasks.rados.rados.0.smithi163.stdout:2700: expect (ObjNum 969 snap 332 seq_num 969) 2023-10-09T09:17:02.310 INFO:tasks.rados.rados.0.smithi163.stdout:2701: expect (ObjNum 989 snap 339 seq_num 989) 2023-10-09T09:17:02.398 INFO:tasks.rados.rados.0.smithi163.stdout:2702: finishing rollback tid 0 to smithi16367185-8 2023-10-09T09:17:02.415 INFO:tasks.rados.rados.0.smithi163.stdout:2702: finishing rollback tid 1 to smithi16367185-8 2023-10-09T09:17:02.415 INFO:tasks.rados.rados.0.smithi163.stdout:2702: finishing rollback tid 2 to smithi16367185-8 2023-10-09T09:17:02.416 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1358 (ObjNum 929 snap 315 seq_num 929) dirty exists 2023-10-09T09:17:02.416 INFO:tasks.rados.rados.0.smithi163.stdout:2700: done (3 left) 2023-10-09T09:17:02.416 INFO:tasks.rados.rados.0.smithi163.stdout:2701: done (2 left) 2023-10-09T09:17:02.416 INFO:tasks.rados.rados.0.smithi163.stdout:2702: done (1 left) 2023-10-09T09:17:02.416 INFO:tasks.rados.rados.0.smithi163.stdout:2705: read oid 20 snap -1 2023-10-09T09:17:02.416 INFO:tasks.rados.rados.0.smithi163.stdout:2705: expect (ObjNum 994 snap 341 seq_num 994) 2023-10-09T09:17:02.416 INFO:tasks.rados.rados.0.smithi163.stdout:2706: write oid 4 current snap is 345 2023-10-09T09:17:02.416 INFO:tasks.rados.rados.0.smithi163.stdout:2704: finishing rollback tid 0 to smithi16367185-45 2023-10-09T09:17:02.417 INFO:tasks.rados.rados.0.smithi163.stdout:2704: finishing rollback tid 1 to smithi16367185-45 2023-10-09T09:17:02.417 INFO:tasks.rados.rados.0.smithi163.stdout:2704: finishing rollback tid 2 to smithi16367185-45 2023-10-09T09:17:02.417 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 1909 (ObjNum 859 snap 292 seq_num 859) dirty exists 2023-10-09T09:17:02.417 INFO:tasks.rados.rados.0.smithi163.stdout:2706: seq_num 1014 ranges {642298=577435,1920153=573357,2979150=63148} 2023-10-09T09:17:02.417 INFO:tasks.rados.rados.0.smithi163.stdout:2706: writing smithi16367185-4 from 642298 to 1219733 tid 1 2023-10-09T09:17:02.419 INFO:tasks.rados.rados.0.smithi163.stdout:2706: writing smithi16367185-4 from 1920153 to 2493510 tid 2 2023-10-09T09:17:02.422 INFO:tasks.rados.rados.0.smithi163.stdout:2706: writing smithi16367185-4 from 2979150 to 3042298 tid 3 2023-10-09T09:17:02.422 INFO:tasks.rados.rados.0.smithi163.stdout:2704: done (2 left) 2023-10-09T09:17:02.422 INFO:tasks.rados.rados.0.smithi163.stdout:2707: snap_remove snap 337 2023-10-09T09:17:02.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[78107]: cluster 2023-10-09T09:17:00.832892+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:02.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[78107]: cluster 2023-10-09T09:17:01.187368+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e748 with expected crc 2023-10-09T09:17:02.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[78107]: cluster 2023-10-09T09:17:01.287844+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e748 with expected crc 2023-10-09T09:17:02.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[78107]: cluster 2023-10-09T09:17:01.302328+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e748 with expected crc 2023-10-09T09:17:02.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[78107]: cluster 2023-10-09T09:17:01.382067+0000 mgr.y (mgr.34107) 314 : cluster 0 pgmap v675: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 27 op/s 2023-10-09T09:17:02.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[78107]: cluster 2023-10-09T09:17:02.194209+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:02.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[78107]: cluster 2023-10-09T09:17:02.196811+0000 mon.a (mon.0) 930 : cluster 0 osdmap e749: 8 total, 8 up, 8 in 2023-10-09T09:17:02.724 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[75425]: cluster 2023-10-09T09:17:00.832892+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:02.724 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[75425]: cluster 2023-10-09T09:17:01.187368+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e748 with expected crc 2023-10-09T09:17:02.724 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[75425]: cluster 2023-10-09T09:17:01.287844+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e748 with expected crc 2023-10-09T09:17:02.724 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[75425]: cluster 2023-10-09T09:17:01.302328+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e748 with expected crc 2023-10-09T09:17:02.724 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[75425]: cluster 2023-10-09T09:17:01.382067+0000 mgr.y (mgr.34107) 314 : cluster 0 pgmap v675: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 27 op/s 2023-10-09T09:17:02.724 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[75425]: cluster 2023-10-09T09:17:02.194209+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:02.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:02 smithi163 bash[75425]: cluster 2023-10-09T09:17:02.196811+0000 mon.a (mon.0) 930 : cluster 0 osdmap e749: 8 total, 8 up, 8 in 2023-10-09T09:17:02.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:02 smithi181 bash[59345]: cluster 2023-10-09T09:17:00.832892+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e747 with expected crc 2023-10-09T09:17:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:02 smithi181 bash[59345]: cluster 2023-10-09T09:17:01.187368+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e748 with expected crc 2023-10-09T09:17:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:02 smithi181 bash[59345]: cluster 2023-10-09T09:17:01.287844+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e748 with expected crc 2023-10-09T09:17:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:02 smithi181 bash[59345]: cluster 2023-10-09T09:17:01.302328+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e748 with expected crc 2023-10-09T09:17:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:02 smithi181 bash[59345]: cluster 2023-10-09T09:17:01.382067+0000 mgr.y (mgr.34107) 314 : cluster 0 pgmap v675: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 27 op/s 2023-10-09T09:17:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:02 smithi181 bash[59345]: cluster 2023-10-09T09:17:02.194209+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:02 smithi181 bash[59345]: cluster 2023-10-09T09:17:02.196811+0000 mon.a (mon.0) 930 : cluster 0 osdmap e749: 8 total, 8 up, 8 in 2023-10-09T09:17:03.426 INFO:tasks.rados.rados.0.smithi163.stdout:2707: done (2 left) 2023-10-09T09:17:03.426 INFO:tasks.rados.rados.0.smithi163.stdout:2708: delete oid 50 current snap is 345 2023-10-09T09:17:03.426 INFO:tasks.rados.rados.0.smithi163.stdout:2706: finishing write tid 1 to smithi16367185-4 2023-10-09T09:17:03.426 INFO:tasks.rados.rados.0.smithi163.stdout:2705: expect (ObjNum 994 snap 341 seq_num 994) 2023-10-09T09:17:03.627 INFO:tasks.rados.rados.0.smithi163.stdout:2706: finishing write tid 2 to smithi16367185-4 2023-10-09T09:17:03.627 INFO:tasks.rados.rados.0.smithi163.stdout:2706: finishing write tid 3 to smithi16367185-4 2023-10-09T09:17:03.628 INFO:tasks.rados.rados.0.smithi163.stdout:2705: done (2 left) 2023-10-09T09:17:03.628 INFO:tasks.rados.rados.0.smithi163.stdout:2708: done (1 left) 2023-10-09T09:17:03.628 INFO:tasks.rados.rados.0.smithi163.stdout:2709: read oid 37 snap 330 2023-10-09T09:17:03.628 INFO:tasks.rados.rados.0.smithi163.stdout:2709: expect deleted 2023-10-09T09:17:03.628 INFO:tasks.rados.rados.0.smithi163.stdout:2710: write oid 43 current snap is 345 2023-10-09T09:17:03.628 INFO:tasks.rados.rados.0.smithi163.stdout:2706: finishing write tid 4 to smithi16367185-4 2023-10-09T09:17:03.628 INFO:tasks.rados.rados.0.smithi163.stdout:2706: finishing write tid 5 to smithi16367185-4 2023-10-09T09:17:03.628 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1299 (ObjNum 1014 snap 345 seq_num 1014) dirty exists 2023-10-09T09:17:03.628 INFO:tasks.rados.rados.0.smithi163.stdout:2706: left oid 4 (ObjNum 1014 snap 345 seq_num 1014) 2023-10-09T09:17:03.628 INFO:tasks.rados.rados.0.smithi163.stdout:2710: seq_num 1016 ranges {675912=727933,1920764=460673,2801290=274622} 2023-10-09T09:17:03.640 INFO:tasks.rados.rados.0.smithi163.stdout:2710: writing smithi16367185-43 from 675912 to 1403845 tid 1 2023-10-09T09:17:03.647 INFO:tasks.rados.rados.0.smithi163.stdout:2710: writing smithi16367185-43 from 1920764 to 2381437 tid 2 2023-10-09T09:17:03.651 INFO:tasks.rados.rados.0.smithi163.stdout:2710: writing smithi16367185-43 from 2801290 to 3075912 tid 3 2023-10-09T09:17:03.651 INFO:tasks.rados.rados.0.smithi163.stdout:2706: done (2 left) 2023-10-09T09:17:03.651 INFO:tasks.rados.rados.0.smithi163.stdout:2711: read oid 35 snap -1 2023-10-09T09:17:03.651 INFO:tasks.rados.rados.0.smithi163.stdout:2711: expect (ObjNum 939 snap 322 seq_num 939) 2023-10-09T09:17:03.651 INFO:tasks.rados.rados.0.smithi163.stdout:2712: snap_create 2023-10-09T09:17:03.652 INFO:tasks.rados.rados.0.smithi163.stdout:2710: finishing write tid 1 to smithi16367185-43 2023-10-09T09:17:03.652 INFO:tasks.rados.rados.0.smithi163.stdout:2710: finishing write tid 2 to smithi16367185-43 2023-10-09T09:17:03.656 INFO:tasks.rados.rados.0.smithi163.stdout:2710: finishing write tid 3 to smithi16367185-43 2023-10-09T09:17:03.656 INFO:tasks.rados.rados.0.smithi163.stdout:2710: finishing write tid 4 to smithi16367185-43 2023-10-09T09:17:03.656 INFO:tasks.rados.rados.0.smithi163.stdout:2710: finishing write tid 5 to smithi16367185-43 2023-10-09T09:17:03.656 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 1267 (ObjNum 1016 snap 345 seq_num 1016) dirty exists 2023-10-09T09:17:03.657 INFO:tasks.rados.rados.0.smithi163.stdout:2710: left oid 43 (ObjNum 1016 snap 345 seq_num 1016) 2023-10-09T09:17:03.666 INFO:tasks.rados.rados.0.smithi163.stdout:2711: expect (ObjNum 939 snap 322 seq_num 939) 2023-10-09T09:17:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:17:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:17:03] "GET /metrics HTTP/1.1" 200 36267 "" "Prometheus/2.33.4" 2023-10-09T09:17:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:03 smithi163 bash[78107]: cluster 2023-10-09T09:17:02.193568+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:03 smithi163 bash[78107]: cluster 2023-10-09T09:17:02.366434+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:03 smithi163 bash[78107]: cluster 2023-10-09T09:17:02.399732+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:03 smithi163 bash[75425]: cluster 2023-10-09T09:17:02.193568+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:03 smithi163 bash[75425]: cluster 2023-10-09T09:17:02.366434+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:03 smithi163 bash[75425]: cluster 2023-10-09T09:17:02.399732+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:03.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:03 smithi181 bash[59345]: cluster 2023-10-09T09:17:02.193568+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:03 smithi181 bash[59345]: cluster 2023-10-09T09:17:02.366434+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:03 smithi181 bash[59345]: cluster 2023-10-09T09:17:02.399732+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e749 with expected crc 2023-10-09T09:17:04.437 INFO:tasks.rados.rados.0.smithi163.stdout:2709: done (3 left) 2023-10-09T09:17:04.437 INFO:tasks.rados.rados.0.smithi163.stdout:2710: done (2 left) 2023-10-09T09:17:04.438 INFO:tasks.rados.rados.0.smithi163.stdout:2711: done (1 left) 2023-10-09T09:17:04.438 INFO:tasks.rados.rados.0.smithi163.stdout:2712: done (0 left) 2023-10-09T09:17:04.438 INFO:tasks.rados.rados.0.smithi163.stdout:2713: read oid 4 snap -1 2023-10-09T09:17:04.438 INFO:tasks.rados.rados.0.smithi163.stdout:2713: expect (ObjNum 1014 snap 345 seq_num 1014) 2023-10-09T09:17:04.438 INFO:tasks.rados.rados.0.smithi163.stdout:2714: snap_remove snap 341 2023-10-09T09:17:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:04 smithi163 bash[75425]: cluster 2023-10-09T09:17:03.382752+0000 mgr.y (mgr.34107) 315 : cluster 0 pgmap v677: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2023-10-09T09:17:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:04 smithi163 bash[75425]: cluster 2023-10-09T09:17:03.421488+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e750 with expected crc 2023-10-09T09:17:04.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:04 smithi163 bash[75425]: cluster 2023-10-09T09:17:03.422899+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e750 with expected crc 2023-10-09T09:17:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:04 smithi163 bash[75425]: cluster 2023-10-09T09:17:03.426048+0000 mon.a (mon.0) 931 : cluster 0 osdmap e750: 8 total, 8 up, 8 in 2023-10-09T09:17:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:04 smithi163 bash[78107]: cluster 2023-10-09T09:17:03.382752+0000 mgr.y (mgr.34107) 315 : cluster 0 pgmap v677: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2023-10-09T09:17:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:04 smithi163 bash[78107]: cluster 2023-10-09T09:17:03.421488+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e750 with expected crc 2023-10-09T09:17:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:04 smithi163 bash[78107]: cluster 2023-10-09T09:17:03.422899+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e750 with expected crc 2023-10-09T09:17:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:04 smithi163 bash[78107]: cluster 2023-10-09T09:17:03.426048+0000 mon.a (mon.0) 931 : cluster 0 osdmap e750: 8 total, 8 up, 8 in 2023-10-09T09:17:04.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:04 smithi181 bash[59345]: cluster 2023-10-09T09:17:03.382752+0000 mgr.y (mgr.34107) 315 : cluster 0 pgmap v677: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2023-10-09T09:17:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:04 smithi181 bash[59345]: cluster 2023-10-09T09:17:03.421488+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e750 with expected crc 2023-10-09T09:17:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:04 smithi181 bash[59345]: cluster 2023-10-09T09:17:03.422899+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e750 with expected crc 2023-10-09T09:17:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:04 smithi181 bash[59345]: cluster 2023-10-09T09:17:03.426048+0000 mon.a (mon.0) 931 : cluster 0 osdmap e750: 8 total, 8 up, 8 in 2023-10-09T09:17:05.188 INFO:tasks.rados.rados.0.smithi163.stdout:2714: done (1 left) 2023-10-09T09:17:05.189 INFO:tasks.rados.rados.0.smithi163.stdout:2715: snap_remove snap 340 2023-10-09T09:17:05.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:05 smithi163 bash[75425]: cluster 2023-10-09T09:17:04.435054+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:05.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:05 smithi163 bash[75425]: cluster 2023-10-09T09:17:04.437455+0000 mon.a (mon.0) 932 : cluster 0 osdmap e751: 8 total, 8 up, 8 in 2023-10-09T09:17:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:05 smithi163 bash[75425]: cluster 2023-10-09T09:17:05.171809+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:05 smithi163 bash[75425]: cluster 2023-10-09T09:17:05.188040+0000 mon.a (mon.0) 933 : cluster 0 osdmap e752: 8 total, 8 up, 8 in 2023-10-09T09:17:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:05 smithi163 bash[78107]: cluster 2023-10-09T09:17:04.435054+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:05 smithi163 bash[78107]: cluster 2023-10-09T09:17:04.437455+0000 mon.a (mon.0) 932 : cluster 0 osdmap e751: 8 total, 8 up, 8 in 2023-10-09T09:17:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:05 smithi163 bash[78107]: cluster 2023-10-09T09:17:05.171809+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:05 smithi163 bash[78107]: cluster 2023-10-09T09:17:05.188040+0000 mon.a (mon.0) 933 : cluster 0 osdmap e752: 8 total, 8 up, 8 in 2023-10-09T09:17:05.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:05 smithi181 bash[59345]: cluster 2023-10-09T09:17:04.435054+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:05.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:05 smithi181 bash[59345]: cluster 2023-10-09T09:17:04.437455+0000 mon.a (mon.0) 932 : cluster 0 osdmap e751: 8 total, 8 up, 8 in 2023-10-09T09:17:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:05 smithi181 bash[59345]: cluster 2023-10-09T09:17:05.171809+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:05 smithi181 bash[59345]: cluster 2023-10-09T09:17:05.188040+0000 mon.a (mon.0) 933 : cluster 0 osdmap e752: 8 total, 8 up, 8 in 2023-10-09T09:17:06.187 INFO:tasks.rados.rados.0.smithi163.stdout:2715: done (1 left) 2023-10-09T09:17:06.187 INFO:tasks.rados.rados.0.smithi163.stdout:2716: write oid 16 current snap is 346 2023-10-09T09:17:06.187 INFO:tasks.rados.rados.0.smithi163.stdout:2716: seq_num 1017 ranges {692719=403182,1579246=720684,2828713=264006} 2023-10-09T09:17:06.197 INFO:tasks.rados.rados.0.smithi163.stdout:2716: writing smithi16367185-16 from 692719 to 1095901 tid 1 2023-10-09T09:17:06.205 INFO:tasks.rados.rados.0.smithi163.stdout:2716: writing smithi16367185-16 from 1579246 to 2299930 tid 2 2023-10-09T09:17:06.210 INFO:tasks.rados.rados.0.smithi163.stdout:2716: writing smithi16367185-16 from 2828713 to 3092719 tid 3 2023-10-09T09:17:06.211 INFO:tasks.rados.rados.0.smithi163.stdout:2717: snap_create 2023-10-09T09:17:06.211 INFO:tasks.rados.rados.0.smithi163.stdout:2713: expect (ObjNum 1014 snap 345 seq_num 1014) 2023-10-09T09:17:06.317 INFO:tasks.rados.rados.0.smithi163.stdout:2716: finishing write tid 1 to smithi16367185-16 2023-10-09T09:17:06.317 INFO:tasks.rados.rados.0.smithi163.stdout:2716: finishing write tid 2 to smithi16367185-16 2023-10-09T09:17:06.317 INFO:tasks.rados.rados.0.smithi163.stdout:2716: finishing write tid 3 to smithi16367185-16 2023-10-09T09:17:06.318 INFO:tasks.rados.rados.0.smithi163.stdout:2716: finishing write tid 4 to smithi16367185-16 2023-10-09T09:17:06.318 INFO:tasks.rados.rados.0.smithi163.stdout:2716: finishing write tid 5 to smithi16367185-16 2023-10-09T09:17:06.318 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1317 (ObjNum 1017 snap 346 seq_num 1017) dirty exists 2023-10-09T09:17:06.318 INFO:tasks.rados.rados.0.smithi163.stdout:2716: left oid 16 (ObjNum 1017 snap 346 seq_num 1017) 2023-10-09T09:17:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:06 smithi163 bash[75425]: cluster 2023-10-09T09:17:04.811308+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:06 smithi163 bash[75425]: cluster 2023-10-09T09:17:05.172097+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:06 smithi163 bash[75425]: cluster 2023-10-09T09:17:05.182907+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e752 with expected crc 2023-10-09T09:17:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:06 smithi163 bash[75425]: cluster 2023-10-09T09:17:05.383508+0000 mgr.y (mgr.34107) 316 : cluster 0 pgmap v681: 89 pgs: 3 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 85 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 296 KiB/s wr, 5 op/s 2023-10-09T09:17:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:06 smithi163 bash[75425]: cluster 2023-10-09T09:17:06.187145+0000 mon.a (mon.0) 934 : cluster 0 osdmap e753: 8 total, 8 up, 8 in 2023-10-09T09:17:06.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:06 smithi163 bash[78107]: cluster 2023-10-09T09:17:04.811308+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:06.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:06 smithi163 bash[78107]: cluster 2023-10-09T09:17:05.172097+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:06.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:06 smithi163 bash[78107]: cluster 2023-10-09T09:17:05.182907+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e752 with expected crc 2023-10-09T09:17:06.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:06 smithi163 bash[78107]: cluster 2023-10-09T09:17:05.383508+0000 mgr.y (mgr.34107) 316 : cluster 0 pgmap v681: 89 pgs: 3 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 85 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 296 KiB/s wr, 5 op/s 2023-10-09T09:17:06.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:06 smithi163 bash[78107]: cluster 2023-10-09T09:17:06.187145+0000 mon.a (mon.0) 934 : cluster 0 osdmap e753: 8 total, 8 up, 8 in 2023-10-09T09:17:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:06 smithi181 bash[59345]: cluster 2023-10-09T09:17:04.811308+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:06 smithi181 bash[59345]: cluster 2023-10-09T09:17:05.172097+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e751 with expected crc 2023-10-09T09:17:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:06 smithi181 bash[59345]: cluster 2023-10-09T09:17:05.182907+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e752 with expected crc 2023-10-09T09:17:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:06 smithi181 bash[59345]: cluster 2023-10-09T09:17:05.383508+0000 mgr.y (mgr.34107) 316 : cluster 0 pgmap v681: 89 pgs: 3 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 85 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 296 KiB/s wr, 5 op/s 2023-10-09T09:17:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:06 smithi181 bash[59345]: cluster 2023-10-09T09:17:06.187145+0000 mon.a (mon.0) 934 : cluster 0 osdmap e753: 8 total, 8 up, 8 in 2023-10-09T09:17:07.202 INFO:tasks.rados.rados.0.smithi163.stdout:2713: done (2 left) 2023-10-09T09:17:07.202 INFO:tasks.rados.rados.0.smithi163.stdout:2716: done (1 left) 2023-10-09T09:17:07.202 INFO:tasks.rados.rados.0.smithi163.stdout:2717: done (0 left) 2023-10-09T09:17:07.202 INFO:tasks.rados.rados.0.smithi163.stdout:2718: read oid 27 snap 331 2023-10-09T09:17:07.202 INFO:tasks.rados.rados.0.smithi163.stdout:2718: expect deleted 2023-10-09T09:17:07.202 INFO:tasks.rados.rados.0.smithi163.stdout:2719: write oid 30 current snap is 347 2023-10-09T09:17:07.202 INFO:tasks.rados.rados.0.smithi163.stdout:2719: seq_num 1018 ranges {709526=478431,1637728=464342,2856136=253390} 2023-10-09T09:17:07.213 INFO:tasks.rados.rados.0.smithi163.stdout:2719: writing smithi16367185-30 from 709526 to 1187957 tid 1 2023-10-09T09:17:07.220 INFO:tasks.rados.rados.0.smithi163.stdout:2719: writing smithi16367185-30 from 1637728 to 2102070 tid 2 2023-10-09T09:17:07.225 INFO:tasks.rados.rados.0.smithi163.stdout:2719: writing smithi16367185-30 from 2856136 to 3109526 tid 3 2023-10-09T09:17:07.226 INFO:tasks.rados.rados.0.smithi163.stdout:2720: write oid 24 current snap is 347 2023-10-09T09:17:07.226 INFO:tasks.rados.rados.0.smithi163.stdout:2720: seq_num 1019 ranges {726333=670033,2096210=724353} 2023-10-09T09:17:07.233 INFO:tasks.rados.rados.0.smithi163.stdout:2720: writing smithi16367185-24 from 726333 to 1396366 tid 1 2023-10-09T09:17:07.242 INFO:tasks.rados.rados.0.smithi163.stdout:2720: writing smithi16367185-24 from 2096210 to 2820563 tid 2 2023-10-09T09:17:07.242 INFO:tasks.rados.rados.0.smithi163.stdout:2721: snap_remove snap 330 2023-10-09T09:17:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:07 smithi163 bash[75425]: cluster 2023-10-09T09:17:06.182443+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e753 with expected crc 2023-10-09T09:17:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:07 smithi163 bash[75425]: cluster 2023-10-09T09:17:06.182443+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e753 with expected crc 2023-10-09T09:17:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:07 smithi163 bash[75425]: cluster 2023-10-09T09:17:07.197552+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:07 smithi163 bash[75425]: cluster 2023-10-09T09:17:07.201487+0000 mon.a (mon.0) 935 : cluster 0 osdmap e754: 8 total, 8 up, 8 in 2023-10-09T09:17:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:07 smithi163 bash[78107]: cluster 2023-10-09T09:17:06.182443+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e753 with expected crc 2023-10-09T09:17:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:07 smithi163 bash[78107]: cluster 2023-10-09T09:17:06.182443+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e753 with expected crc 2023-10-09T09:17:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:07 smithi163 bash[78107]: cluster 2023-10-09T09:17:07.197552+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:07 smithi163 bash[78107]: cluster 2023-10-09T09:17:07.201487+0000 mon.a (mon.0) 935 : cluster 0 osdmap e754: 8 total, 8 up, 8 in 2023-10-09T09:17:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:07 smithi181 bash[59345]: cluster 2023-10-09T09:17:06.182443+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e753 with expected crc 2023-10-09T09:17:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:07 smithi181 bash[59345]: cluster 2023-10-09T09:17:06.182443+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e753 with expected crc 2023-10-09T09:17:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:07 smithi181 bash[59345]: cluster 2023-10-09T09:17:07.197552+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:07 smithi181 bash[59345]: cluster 2023-10-09T09:17:07.201487+0000 mon.a (mon.0) 935 : cluster 0 osdmap e754: 8 total, 8 up, 8 in 2023-10-09T09:17:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:2721: done (3 left) 2023-10-09T09:17:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:2722: read oid 19 snap -1 2023-10-09T09:17:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:2722: expect (ObjNum 1009 snap 344 seq_num 1009) 2023-10-09T09:17:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:2723: read oid 34 snap 314 2023-10-09T09:17:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:2723: expect (ObjNum 882 snap 308 seq_num 882) 2023-10-09T09:17:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:2724: rollback oid 5 current snap is 347 2023-10-09T09:17:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 336 2023-10-09T09:17:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:2719: finishing write tid 1 to smithi16367185-30 2023-10-09T09:17:08.207 INFO:tasks.rados.rados.0.smithi163.stdout:2719: finishing write tid 2 to smithi16367185-30 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2718: done (5 left) 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2725: snap_create 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2719: finishing write tid 3 to smithi16367185-30 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2719: finishing write tid 4 to smithi16367185-30 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2719: finishing write tid 5 to smithi16367185-30 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 858 (ObjNum 1018 snap 347 seq_num 1018) dirty exists 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2719: left oid 30 (ObjNum 1018 snap 347 seq_num 1018) 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2720: finishing write tid 1 to smithi16367185-24 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2720: finishing write tid 2 to smithi16367185-24 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2720: finishing write tid 3 to smithi16367185-24 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2720: finishing write tid 4 to smithi16367185-24 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 630 (ObjNum 1019 snap 347 seq_num 1019) dirty exists 2023-10-09T09:17:08.208 INFO:tasks.rados.rados.0.smithi163.stdout:2720: left oid 24 (ObjNum 1019 snap 347 seq_num 1019) 2023-10-09T09:17:08.224 INFO:tasks.rados.rados.0.smithi163.stdout:2722: expect (ObjNum 1009 snap 344 seq_num 1009) 2023-10-09T09:17:08.430 INFO:tasks.rados.rados.0.smithi163.stdout:2724: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:17:08.430 INFO:tasks.rados.rados.0.smithi163.stdout:2723: expect (ObjNum 882 snap 308 seq_num 882) 2023-10-09T09:17:08.555 INFO:tasks.rados.rados.0.smithi163.stdout:2724: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:17:08.555 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1319 (ObjNum 954 snap 324 seq_num 954) dirty exists 2023-10-09T09:17:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[78107]: cluster 2023-10-09T09:17:07.196764+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[78107]: cluster 2023-10-09T09:17:07.300814+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[78107]: cluster 2023-10-09T09:17:07.303730+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[78107]: cluster 2023-10-09T09:17:07.384227+0000 mgr.y (mgr.34107) 317 : cluster 0 pgmap v684: 89 pgs: 3 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 85 active+clean; 248 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 660 KiB/s wr, 17 op/s 2023-10-09T09:17:08.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[78107]: cluster 2023-10-09T09:17:08.206289+0000 mon.a (mon.0) 936 : cluster 0 osdmap e755: 8 total, 8 up, 8 in 2023-10-09T09:17:08.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[78107]: cluster 2023-10-09T09:17:08.209535+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e755 with expected crc 2023-10-09T09:17:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[75425]: cluster 2023-10-09T09:17:07.196764+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[75425]: cluster 2023-10-09T09:17:07.300814+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[75425]: cluster 2023-10-09T09:17:07.303730+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[75425]: cluster 2023-10-09T09:17:07.384227+0000 mgr.y (mgr.34107) 317 : cluster 0 pgmap v684: 89 pgs: 3 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 85 active+clean; 248 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 660 KiB/s wr, 17 op/s 2023-10-09T09:17:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[75425]: cluster 2023-10-09T09:17:08.206289+0000 mon.a (mon.0) 936 : cluster 0 osdmap e755: 8 total, 8 up, 8 in 2023-10-09T09:17:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:08 smithi163 bash[75425]: cluster 2023-10-09T09:17:08.209535+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e755 with expected crc 2023-10-09T09:17:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:08 smithi181 bash[59345]: cluster 2023-10-09T09:17:07.196764+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:08 smithi181 bash[59345]: cluster 2023-10-09T09:17:07.300814+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:08 smithi181 bash[59345]: cluster 2023-10-09T09:17:07.303730+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e754 with expected crc 2023-10-09T09:17:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:08 smithi181 bash[59345]: cluster 2023-10-09T09:17:07.384227+0000 mgr.y (mgr.34107) 317 : cluster 0 pgmap v684: 89 pgs: 3 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 85 active+clean; 248 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 660 KiB/s wr, 17 op/s 2023-10-09T09:17:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:08 smithi181 bash[59345]: cluster 2023-10-09T09:17:08.206289+0000 mon.a (mon.0) 936 : cluster 0 osdmap e755: 8 total, 8 up, 8 in 2023-10-09T09:17:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:08 smithi181 bash[59345]: cluster 2023-10-09T09:17:08.209535+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e755 with expected crc 2023-10-09T09:17:09.211 INFO:tasks.rados.rados.0.smithi163.stdout:2719: done (5 left) 2023-10-09T09:17:09.211 INFO:tasks.rados.rados.0.smithi163.stdout:2720: done (4 left) 2023-10-09T09:17:09.211 INFO:tasks.rados.rados.0.smithi163.stdout:2722: done (3 left) 2023-10-09T09:17:09.211 INFO:tasks.rados.rados.0.smithi163.stdout:2723: done (2 left) 2023-10-09T09:17:09.211 INFO:tasks.rados.rados.0.smithi163.stdout:2724: done (1 left) 2023-10-09T09:17:09.211 INFO:tasks.rados.rados.0.smithi163.stdout:2725: done (0 left) 2023-10-09T09:17:09.211 INFO:tasks.rados.rados.0.smithi163.stdout:2726: rollback oid 32 current snap is 348 2023-10-09T09:17:09.211 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 306 2023-10-09T09:17:09.211 INFO:tasks.rados.rados.0.smithi163.stdout:2727: delete oid 43 current snap is 348 2023-10-09T09:17:09.220 INFO:tasks.rados.rados.0.smithi163.stdout:2726: finishing rollback tid 0 to smithi16367185-32 2023-10-09T09:17:09.220 INFO:tasks.rados.rados.0.smithi163.stdout:2726: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:17:09.220 INFO:tasks.rados.rados.0.smithi163.stdout:2727: done (1 left) 2023-10-09T09:17:09.220 INFO:tasks.rados.rados.0.smithi163.stdout:2728: read oid 11 snap -1 2023-10-09T09:17:09.220 INFO:tasks.rados.rados.0.smithi163.stdout:2728: expect (ObjNum 989 snap 339 seq_num 989) 2023-10-09T09:17:09.220 INFO:tasks.rados.rados.0.smithi163.stdout:2729: snap_create 2023-10-09T09:17:09.220 INFO:tasks.rados.rados.0.smithi163.stdout:2726: finishing rollback tid 2 to smithi16367185-32 2023-10-09T09:17:09.221 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1931 (ObjNum 819 snap 282 seq_num 819) dirty exists 2023-10-09T09:17:09.229 INFO:tasks.rados.rados.0.smithi163.stdout:2728: expect (ObjNum 989 snap 339 seq_num 989) 2023-10-09T09:17:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:09 smithi163 bash[78107]: cluster 2023-10-09T09:17:08.203632+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e755 with expected crc 2023-10-09T09:17:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:09 smithi163 bash[78107]: cluster 2023-10-09T09:17:08.203670+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e755 with expected crc 2023-10-09T09:17:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:09 smithi163 bash[78107]: cluster 2023-10-09T09:17:09.210463+0000 mon.a (mon.0) 937 : cluster 0 osdmap e756: 8 total, 8 up, 8 in 2023-10-09T09:17:09.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:09 smithi163 bash[78107]: cluster 2023-10-09T09:17:09.211756+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:09 smithi163 bash[75425]: cluster 2023-10-09T09:17:08.203632+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e755 with expected crc 2023-10-09T09:17:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:09 smithi163 bash[75425]: cluster 2023-10-09T09:17:08.203670+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e755 with expected crc 2023-10-09T09:17:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:09 smithi163 bash[75425]: cluster 2023-10-09T09:17:09.210463+0000 mon.a (mon.0) 937 : cluster 0 osdmap e756: 8 total, 8 up, 8 in 2023-10-09T09:17:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:09 smithi163 bash[75425]: cluster 2023-10-09T09:17:09.211756+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:09 smithi181 bash[59345]: cluster 2023-10-09T09:17:08.203632+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e755 with expected crc 2023-10-09T09:17:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:09 smithi181 bash[59345]: cluster 2023-10-09T09:17:08.203670+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e755 with expected crc 2023-10-09T09:17:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:09 smithi181 bash[59345]: cluster 2023-10-09T09:17:09.210463+0000 mon.a (mon.0) 937 : cluster 0 osdmap e756: 8 total, 8 up, 8 in 2023-10-09T09:17:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:09 smithi181 bash[59345]: cluster 2023-10-09T09:17:09.211756+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:10.190 INFO:tasks.rados.rados.0.smithi163.stdout:2726: done (2 left) 2023-10-09T09:17:10.191 INFO:tasks.rados.rados.0.smithi163.stdout:2728: done (1 left) 2023-10-09T09:17:10.191 INFO:tasks.rados.rados.0.smithi163.stdout:2729: done (0 left) 2023-10-09T09:17:10.191 INFO:tasks.rados.rados.0.smithi163.stdout:2730: read oid 7 snap 345 2023-10-09T09:17:10.191 INFO:tasks.rados.rados.0.smithi163.stdout:2730: expect (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:17:10.191 INFO:tasks.rados.rados.0.smithi163.stdout:2731: write oid 17 current snap is 349 2023-10-09T09:17:10.191 INFO:tasks.rados.rados.0.smithi163.stdout:2731: seq_num 1021 ranges {759947=420531,1813174=728022} 2023-10-09T09:17:10.198 INFO:tasks.rados.rados.0.smithi163.stdout:2731: writing smithi16367185-17 from 759947 to 1180478 tid 1 2023-10-09T09:17:10.207 INFO:tasks.rados.rados.0.smithi163.stdout:2731: writing smithi16367185-17 from 1813174 to 2541196 tid 2 2023-10-09T09:17:10.207 INFO:tasks.rados.rados.0.smithi163.stdout:2732: read oid 13 snap -1 2023-10-09T09:17:10.207 INFO:tasks.rados.rados.0.smithi163.stdout:2732: expect deleted 2023-10-09T09:17:10.207 INFO:tasks.rados.rados.0.smithi163.stdout:2733: read oid 40 snap -1 2023-10-09T09:17:10.208 INFO:tasks.rados.rados.0.smithi163.stdout:2733: expect (ObjNum 876 snap 305 seq_num 876) 2023-10-09T09:17:10.208 INFO:tasks.rados.rados.0.smithi163.stdout:2734: write oid 11 current snap is 349 2023-10-09T09:17:10.208 INFO:tasks.rados.rados.0.smithi163.stdout:2734: seq_num 1022 ranges {776754=495780,1871656=471680,2965828=210926} 2023-10-09T09:17:10.215 INFO:tasks.rados.rados.0.smithi163.stdout:2734: writing smithi16367185-11 from 776754 to 1272534 tid 1 2023-10-09T09:17:10.223 INFO:tasks.rados.rados.0.smithi163.stdout:2734: writing smithi16367185-11 from 1871656 to 2343336 tid 2 2023-10-09T09:17:10.228 INFO:tasks.rados.rados.0.smithi163.stdout:2734: writing smithi16367185-11 from 2965828 to 3176754 tid 3 2023-10-09T09:17:10.228 INFO:tasks.rados.rados.0.smithi163.stdout:2735: read oid 34 snap -1 2023-10-09T09:17:10.229 INFO:tasks.rados.rados.0.smithi163.stdout:2735: expect (ObjNum 969 snap 332 seq_num 969) 2023-10-09T09:17:10.229 INFO:tasks.rados.rados.0.smithi163.stdout:2736: read oid 35 snap 331 2023-10-09T09:17:10.229 INFO:tasks.rados.rados.0.smithi163.stdout:2736: expect (ObjNum 939 snap 322 seq_num 939) 2023-10-09T09:17:10.229 INFO:tasks.rados.rados.0.smithi163.stdout:2737: write oid 19 current snap is 349 2023-10-09T09:17:10.229 INFO:tasks.rados.rados.0.smithi163.stdout:2737: seq_num 1023 ranges {793561=571029,1813785=731691,2993251=200310} 2023-10-09T09:17:10.237 INFO:tasks.rados.rados.0.smithi163.stdout:2737: writing smithi16367185-19 from 793561 to 1364590 tid 1 2023-10-09T09:17:10.244 INFO:tasks.rados.rados.0.smithi163.stdout:2737: writing smithi16367185-19 from 1813785 to 2545476 tid 2 2023-10-09T09:17:10.248 INFO:tasks.rados.rados.0.smithi163.stdout:2737: writing smithi16367185-19 from 2993251 to 3193561 tid 3 2023-10-09T09:17:10.248 INFO:tasks.rados.rados.0.smithi163.stdout:2738: snap_remove snap 314 2023-10-09T09:17:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[78107]: cluster 2023-10-09T09:17:09.205266+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[78107]: cluster 2023-10-09T09:17:09.206014+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[78107]: cluster 2023-10-09T09:17:09.220711+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[78107]: cluster 2023-10-09T09:17:09.384940+0000 mgr.y (mgr.34107) 318 : cluster 0 pgmap v687: 89 pgs: 3 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 85 active+clean; 251 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 698 KiB/s wr, 12 op/s 2023-10-09T09:17:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[78107]: cluster 2023-10-09T09:17:10.190133+0000 mon.a (mon.0) 938 : cluster 0 osdmap e757: 8 total, 8 up, 8 in 2023-10-09T09:17:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[78107]: audit 2023-10-09T09:17:10.346285+0000 mon.a (mon.0) 939 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[75425]: cluster 2023-10-09T09:17:09.205266+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[75425]: cluster 2023-10-09T09:17:09.206014+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[75425]: cluster 2023-10-09T09:17:09.220711+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[75425]: cluster 2023-10-09T09:17:09.384940+0000 mgr.y (mgr.34107) 318 : cluster 0 pgmap v687: 89 pgs: 3 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 85 active+clean; 251 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 698 KiB/s wr, 12 op/s 2023-10-09T09:17:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[75425]: cluster 2023-10-09T09:17:10.190133+0000 mon.a (mon.0) 938 : cluster 0 osdmap e757: 8 total, 8 up, 8 in 2023-10-09T09:17:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:10 smithi163 bash[75425]: audit 2023-10-09T09:17:10.346285+0000 mon.a (mon.0) 939 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:10 smithi181 bash[59345]: cluster 2023-10-09T09:17:09.205266+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:10 smithi181 bash[59345]: cluster 2023-10-09T09:17:09.206014+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:10 smithi181 bash[59345]: cluster 2023-10-09T09:17:09.220711+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e756 with expected crc 2023-10-09T09:17:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:10 smithi181 bash[59345]: cluster 2023-10-09T09:17:09.384940+0000 mgr.y (mgr.34107) 318 : cluster 0 pgmap v687: 89 pgs: 3 active+clean+snaptrim_wait, 1 active+clean+snaptrim, 85 active+clean; 251 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 698 KiB/s wr, 12 op/s 2023-10-09T09:17:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:10 smithi181 bash[59345]: cluster 2023-10-09T09:17:10.190133+0000 mon.a (mon.0) 938 : cluster 0 osdmap e757: 8 total, 8 up, 8 in 2023-10-09T09:17:10.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:10 smithi181 bash[59345]: audit 2023-10-09T09:17:10.346285+0000 mon.a (mon.0) 939 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:11.218 INFO:tasks.rados.rados.0.smithi163.stdout:2730: expect (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:17:11.402 INFO:tasks.rados.rados.0.smithi163.stdout:2731: finishing write tid 1 to smithi16367185-17 2023-10-09T09:17:11.402 INFO:tasks.rados.rados.0.smithi163.stdout:2731: finishing write tid 2 to smithi16367185-17 2023-10-09T09:17:11.402 INFO:tasks.rados.rados.0.smithi163.stdout:2731: finishing write tid 3 to smithi16367185-17 2023-10-09T09:17:11.402 INFO:tasks.rados.rados.0.smithi163.stdout:2731: finishing write tid 4 to smithi16367185-17 2023-10-09T09:17:11.402 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1379 (ObjNum 1021 snap 349 seq_num 1021) dirty exists 2023-10-09T09:17:11.402 INFO:tasks.rados.rados.0.smithi163.stdout:2731: left oid 17 (ObjNum 1021 snap 349 seq_num 1021) 2023-10-09T09:17:11.402 INFO:tasks.rados.rados.0.smithi163.stdout:2733: expect (ObjNum 876 snap 305 seq_num 876) 2023-10-09T09:17:11.535 INFO:tasks.rados.rados.0.smithi163.stdout:2734: finishing write tid 1 to smithi16367185-11 2023-10-09T09:17:11.535 INFO:tasks.rados.rados.0.smithi163.stdout:2734: finishing write tid 2 to smithi16367185-11 2023-10-09T09:17:11.535 INFO:tasks.rados.rados.0.smithi163.stdout:2730: done (8 left) 2023-10-09T09:17:11.535 INFO:tasks.rados.rados.0.smithi163.stdout:2731: done (7 left) 2023-10-09T09:17:11.535 INFO:tasks.rados.rados.0.smithi163.stdout:2732: done (6 left) 2023-10-09T09:17:11.535 INFO:tasks.rados.rados.0.smithi163.stdout:2733: done (5 left) 2023-10-09T09:17:11.535 INFO:tasks.rados.rados.0.smithi163.stdout:2738: done (4 left) 2023-10-09T09:17:11.535 INFO:tasks.rados.rados.0.smithi163.stdout:2739: write oid 5 current snap is 349 2023-10-09T09:17:11.535 INFO:tasks.rados.rados.0.smithi163.stdout:2735: expect (ObjNum 969 snap 332 seq_num 969) 2023-10-09T09:17:11.642 INFO:tasks.rados.rados.0.smithi163.stdout:2734: finishing write tid 3 to smithi16367185-11 2023-10-09T09:17:11.642 INFO:tasks.rados.rados.0.smithi163.stdout:2734: finishing write tid 4 to smithi16367185-11 2023-10-09T09:17:11.642 INFO:tasks.rados.rados.0.smithi163.stdout:2734: finishing write tid 5 to smithi16367185-11 2023-10-09T09:17:11.642 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 730 (ObjNum 1022 snap 349 seq_num 1022) dirty exists 2023-10-09T09:17:11.642 INFO:tasks.rados.rados.0.smithi163.stdout:2734: left oid 11 (ObjNum 1022 snap 349 seq_num 1022) 2023-10-09T09:17:11.642 INFO:tasks.rados.rados.0.smithi163.stdout:2736: expect (ObjNum 939 snap 322 seq_num 939) 2023-10-09T09:17:11.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:11 smithi163 bash[75425]: cluster 2023-10-09T09:17:10.186019+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e757 with expected crc 2023-10-09T09:17:11.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:11 smithi163 bash[75425]: cluster 2023-10-09T09:17:10.200123+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e757 with expected crc 2023-10-09T09:17:11.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:11 smithi163 bash[75425]: cluster 2023-10-09T09:17:10.213554+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e757 with expected crc 2023-10-09T09:17:11.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:11 smithi163 bash[75425]: cluster 2023-10-09T09:17:11.214104+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e758 with expected crc 2023-10-09T09:17:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:11 smithi163 bash[75425]: cluster 2023-10-09T09:17:11.214109+0000 mon.a (mon.0) 940 : cluster 0 osdmap e758: 8 total, 8 up, 8 in 2023-10-09T09:17:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:11 smithi163 bash[78107]: cluster 2023-10-09T09:17:10.186019+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e757 with expected crc 2023-10-09T09:17:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:11 smithi163 bash[78107]: cluster 2023-10-09T09:17:10.200123+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e757 with expected crc 2023-10-09T09:17:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:11 smithi163 bash[78107]: cluster 2023-10-09T09:17:10.213554+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e757 with expected crc 2023-10-09T09:17:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:11 smithi163 bash[78107]: cluster 2023-10-09T09:17:11.214104+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e758 with expected crc 2023-10-09T09:17:11.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:11 smithi163 bash[78107]: cluster 2023-10-09T09:17:11.214109+0000 mon.a (mon.0) 940 : cluster 0 osdmap e758: 8 total, 8 up, 8 in 2023-10-09T09:17:11.761 INFO:tasks.rados.rados.0.smithi163.stdout:2737: finishing write tid 1 to smithi16367185-19 2023-10-09T09:17:11.761 INFO:tasks.rados.rados.0.smithi163.stdout:2737: finishing write tid 2 to smithi16367185-19 2023-10-09T09:17:11.761 INFO:tasks.rados.rados.0.smithi163.stdout:2737: finishing write tid 3 to smithi16367185-19 2023-10-09T09:17:11.761 INFO:tasks.rados.rados.0.smithi163.stdout:2737: finishing write tid 4 to smithi16367185-19 2023-10-09T09:17:11.762 INFO:tasks.rados.rados.0.smithi163.stdout:2737: finishing write tid 5 to smithi16367185-19 2023-10-09T09:17:11.762 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 767 (ObjNum 1023 snap 349 seq_num 1023) dirty exists 2023-10-09T09:17:11.762 INFO:tasks.rados.rados.0.smithi163.stdout:2737: left oid 19 (ObjNum 1023 snap 349 seq_num 1023) 2023-10-09T09:17:11.762 INFO:tasks.rados.rados.0.smithi163.stdout:2739: seq_num 1024 ranges {410368=646278,1472267=475349,2504321=405600} 2023-10-09T09:17:11.771 INFO:tasks.rados.rados.0.smithi163.stdout:2739: writing smithi16367185-5 from 410368 to 1056646 tid 1 2023-10-09T09:17:11.778 INFO:tasks.rados.rados.0.smithi163.stdout:2739: writing smithi16367185-5 from 1472267 to 1947616 tid 2 2023-10-09T09:17:11.784 INFO:tasks.rados.rados.0.smithi163.stdout:2739: writing smithi16367185-5 from 2504321 to 2909921 tid 3 2023-10-09T09:17:11.784 INFO:tasks.rados.rados.0.smithi163.stdout:2734: done (4 left) 2023-10-09T09:17:11.784 INFO:tasks.rados.rados.0.smithi163.stdout:2735: done (3 left) 2023-10-09T09:17:11.784 INFO:tasks.rados.rados.0.smithi163.stdout:2736: done (2 left) 2023-10-09T09:17:11.784 INFO:tasks.rados.rados.0.smithi163.stdout:2737: done (1 left) 2023-10-09T09:17:11.784 INFO:tasks.rados.rados.0.smithi163.stdout:2740: rollback oid 1 current snap is 349 2023-10-09T09:17:11.785 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 1 to 348 2023-10-09T09:17:11.785 INFO:tasks.rados.rados.0.smithi163.stdout:2739: finishing write tid 1 to smithi16367185-5 2023-10-09T09:17:11.785 INFO:tasks.rados.rados.0.smithi163.stdout:2739: finishing write tid 2 to smithi16367185-5 2023-10-09T09:17:11.785 INFO:tasks.rados.rados.0.smithi163.stdout:2741: read oid 29 snap -1 2023-10-09T09:17:11.785 INFO:tasks.rados.rados.0.smithi163.stdout:2741: expect (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:17:11.785 INFO:tasks.rados.rados.0.smithi163.stdout:2742: snap_remove snap 345 2023-10-09T09:17:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:11 smithi181 bash[59345]: cluster 2023-10-09T09:17:10.186019+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e757 with expected crc 2023-10-09T09:17:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:11 smithi181 bash[59345]: cluster 2023-10-09T09:17:10.200123+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e757 with expected crc 2023-10-09T09:17:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:11 smithi181 bash[59345]: cluster 2023-10-09T09:17:10.213554+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e757 with expected crc 2023-10-09T09:17:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:11 smithi181 bash[59345]: cluster 2023-10-09T09:17:11.214104+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e758 with expected crc 2023-10-09T09:17:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:11 smithi181 bash[59345]: cluster 2023-10-09T09:17:11.214109+0000 mon.a (mon.0) 940 : cluster 0 osdmap e758: 8 total, 8 up, 8 in 2023-10-09T09:17:12.462 INFO:tasks.rados.rados.0.smithi163.stdout:2742: done (3 left) 2023-10-09T09:17:12.462 INFO:tasks.rados.rados.0.smithi163.stdout:2743: delete oid 31 current snap is 349 2023-10-09T09:17:12.463 INFO:tasks.rados.rados.0.smithi163.stdout:2741: expect (ObjNum 853 snap 289 seq_num 853) 2023-10-09T09:17:12.573 INFO:tasks.rados.rados.0.smithi163.stdout:2739: finishing write tid 3 to smithi16367185-5 2023-10-09T09:17:12.573 INFO:tasks.rados.rados.0.smithi163.stdout:2739: finishing write tid 4 to smithi16367185-5 2023-10-09T09:17:12.573 INFO:tasks.rados.rados.0.smithi163.stdout:2739: finishing write tid 5 to smithi16367185-5 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1332 (ObjNum 1024 snap 349 seq_num 1024) dirty exists 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:2739: left oid 5 (ObjNum 1024 snap 349 seq_num 1024) 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:2740: finishing rollback tid 0 to smithi16367185-1 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:2740: finishing rollback tid 1 to smithi16367185-1 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:2740: finishing rollback tid 2 to smithi16367185-1 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1336 (ObjNum 983 snap 338 seq_num 983) dirty exists 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:2739: done (3 left) 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:2740: done (2 left) 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:2741: done (1 left) 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:2743: done (0 left) 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:2744: write oid 22 current snap is 349 2023-10-09T09:17:12.574 INFO:tasks.rados.rados.0.smithi163.stdout:2744: seq_num 1026 ranges {443982=796776,1989231=479018,2959167=284815} 2023-10-09T09:17:12.580 INFO:tasks.rados.rados.0.smithi163.stdout:2744: writing smithi16367185-22 from 443982 to 1240758 tid 1 2023-10-09T09:17:12.587 INFO:tasks.rados.rados.0.smithi163.stdout:2744: writing smithi16367185-22 from 1989231 to 2468249 tid 2 2023-10-09T09:17:12.592 INFO:tasks.rados.rados.0.smithi163.stdout:2744: writing smithi16367185-22 from 2959167 to 3243982 tid 3 2023-10-09T09:17:12.592 INFO:tasks.rados.rados.0.smithi163.stdout:2745: write oid 40 current snap is 349 2023-10-09T09:17:12.592 INFO:tasks.rados.rados.0.smithi163.stdout:2745: seq_num 1027 ranges {460789=588378,1764066=739029,3219296=41493} 2023-10-09T09:17:12.597 INFO:tasks.rados.rados.0.smithi163.stdout:2745: writing smithi16367185-40 from 460789 to 1049167 tid 1 2023-10-09T09:17:12.606 INFO:tasks.rados.rados.0.smithi163.stdout:2745: writing smithi16367185-40 from 1764066 to 2503095 tid 2 2023-10-09T09:17:12.611 INFO:tasks.rados.rados.0.smithi163.stdout:2745: writing smithi16367185-40 from 3219296 to 3260789 tid 3 2023-10-09T09:17:12.611 INFO:tasks.rados.rados.0.smithi163.stdout:2746: delete oid 17 current snap is 349 2023-10-09T09:17:12.611 INFO:tasks.rados.rados.0.smithi163.stdout:2744: finishing write tid 1 to smithi16367185-22 2023-10-09T09:17:12.611 INFO:tasks.rados.rados.0.smithi163.stdout:2744: finishing write tid 2 to smithi16367185-22 2023-10-09T09:17:12.611 INFO:tasks.rados.rados.0.smithi163.stdout:2744: finishing write tid 3 to smithi16367185-22 2023-10-09T09:17:12.612 INFO:tasks.rados.rados.0.smithi163.stdout:2744: finishing write tid 4 to smithi16367185-22 2023-10-09T09:17:12.612 INFO:tasks.rados.rados.0.smithi163.stdout:2744: finishing write tid 5 to smithi16367185-22 2023-10-09T09:17:12.612 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 1942 (ObjNum 1026 snap 349 seq_num 1026) dirty exists 2023-10-09T09:17:12.612 INFO:tasks.rados.rados.0.smithi163.stdout:2744: left oid 22 (ObjNum 1026 snap 349 seq_num 1026) 2023-10-09T09:17:12.612 INFO:tasks.rados.rados.0.smithi163.stdout:2745: finishing write tid 1 to smithi16367185-40 2023-10-09T09:17:12.612 INFO:tasks.rados.rados.0.smithi163.stdout:2745: finishing write tid 2 to smithi16367185-40 2023-10-09T09:17:12.616 INFO:tasks.rados.rados.0.smithi163.stdout:2745: finishing write tid 3 to smithi16367185-40 2023-10-09T09:17:12.616 INFO:tasks.rados.rados.0.smithi163.stdout:2745: finishing write tid 4 to smithi16367185-40 2023-10-09T09:17:12.616 INFO:tasks.rados.rados.0.smithi163.stdout:2745: finishing write tid 5 to smithi16367185-40 2023-10-09T09:17:12.616 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1390 (ObjNum 1027 snap 349 seq_num 1027) dirty exists 2023-10-09T09:17:12.617 INFO:tasks.rados.rados.0.smithi163.stdout:2745: left oid 40 (ObjNum 1027 snap 349 seq_num 1027) 2023-10-09T09:17:12.617 INFO:tasks.rados.rados.0.smithi163.stdout:2744: done (2 left) 2023-10-09T09:17:12.617 INFO:tasks.rados.rados.0.smithi163.stdout:2745: done (1 left) 2023-10-09T09:17:12.617 INFO:tasks.rados.rados.0.smithi163.stdout:2746: done (0 left) 2023-10-09T09:17:12.617 INFO:tasks.rados.rados.0.smithi163.stdout:2747: delete oid 17 current snap is 349 2023-10-09T09:17:12.619 INFO:tasks.rados.rados.0.smithi163.stdout:2747: done (0 left) 2023-10-09T09:17:12.619 INFO:tasks.rados.rados.0.smithi163.stdout:2748: snap_remove snap 344 2023-10-09T09:17:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:12 smithi163 bash[78107]: cluster 2023-10-09T09:17:11.209594+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e758 with expected crc 2023-10-09T09:17:12.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:12 smithi163 bash[78107]: cluster 2023-10-09T09:17:11.385854+0000 mgr.y (mgr.34107) 319 : cluster 0 pgmap v690: 89 pgs: 89 active+clean; 253 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 19 op/s 2023-10-09T09:17:12.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:12 smithi163 bash[75425]: cluster 2023-10-09T09:17:11.209594+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e758 with expected crc 2023-10-09T09:17:12.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:12 smithi163 bash[75425]: cluster 2023-10-09T09:17:11.385854+0000 mgr.y (mgr.34107) 319 : cluster 0 pgmap v690: 89 pgs: 89 active+clean; 253 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 19 op/s 2023-10-09T09:17:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:12 smithi181 bash[59345]: cluster 2023-10-09T09:17:11.209594+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e758 with expected crc 2023-10-09T09:17:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:12 smithi181 bash[59345]: cluster 2023-10-09T09:17:11.385854+0000 mgr.y (mgr.34107) 319 : cluster 0 pgmap v690: 89 pgs: 89 active+clean; 253 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 19 op/s 2023-10-09T09:17:13.478 INFO:tasks.rados.rados.0.smithi163.stdout:2748: done (0 left) 2023-10-09T09:17:13.562 INFO:tasks.rados.rados.0.smithi163.stdout:2749: snap_remove snap 322 2023-10-09T09:17:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:17:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:17:13] "GET /metrics HTTP/1.1" 200 36263 "" "Prometheus/2.33.4" 2023-10-09T09:17:13.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:13 smithi163 bash[78107]: cluster 2023-10-09T09:17:12.459728+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e759 with expected crc 2023-10-09T09:17:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:13 smithi163 bash[78107]: cluster 2023-10-09T09:17:12.462319+0000 mon.a (mon.0) 941 : cluster 0 osdmap e759: 8 total, 8 up, 8 in 2023-10-09T09:17:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:13 smithi163 bash[78107]: cluster 2023-10-09T09:17:12.470567+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e759 with expected crc 2023-10-09T09:17:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:13 smithi163 bash[75425]: cluster 2023-10-09T09:17:12.459728+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e759 with expected crc 2023-10-09T09:17:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:13 smithi163 bash[75425]: cluster 2023-10-09T09:17:12.462319+0000 mon.a (mon.0) 941 : cluster 0 osdmap e759: 8 total, 8 up, 8 in 2023-10-09T09:17:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:13 smithi163 bash[75425]: cluster 2023-10-09T09:17:12.470567+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e759 with expected crc 2023-10-09T09:17:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:13 smithi181 bash[59345]: cluster 2023-10-09T09:17:12.459728+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e759 with expected crc 2023-10-09T09:17:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:13 smithi181 bash[59345]: cluster 2023-10-09T09:17:12.462319+0000 mon.a (mon.0) 941 : cluster 0 osdmap e759: 8 total, 8 up, 8 in 2023-10-09T09:17:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:13 smithi181 bash[59345]: cluster 2023-10-09T09:17:12.470567+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e759 with expected crc 2023-10-09T09:17:14.477 INFO:tasks.rados.rados.0.smithi163.stdout:2749: done (0 left) 2023-10-09T09:17:14.477 INFO:tasks.rados.rados.0.smithi163.stdout:2750: snap_create 2023-10-09T09:17:14.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:14 smithi181 bash[59345]: cluster 2023-10-09T09:17:13.386461+0000 mgr.y (mgr.34107) 320 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 253 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.1 MiB/s wr, 17 op/s 2023-10-09T09:17:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:14 smithi181 bash[59345]: cluster 2023-10-09T09:17:13.472086+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e760 with expected crc 2023-10-09T09:17:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:14 smithi181 bash[59345]: cluster 2023-10-09T09:17:13.474289+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e760 with expected crc 2023-10-09T09:17:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:14 smithi181 bash[59345]: cluster 2023-10-09T09:17:13.478709+0000 mon.a (mon.0) 942 : cluster 0 osdmap e760: 8 total, 8 up, 8 in 2023-10-09T09:17:14.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:14 smithi163 bash[78107]: cluster 2023-10-09T09:17:13.386461+0000 mgr.y (mgr.34107) 320 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 253 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.1 MiB/s wr, 17 op/s 2023-10-09T09:17:14.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:14 smithi163 bash[78107]: cluster 2023-10-09T09:17:13.472086+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e760 with expected crc 2023-10-09T09:17:14.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:14 smithi163 bash[78107]: cluster 2023-10-09T09:17:13.474289+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e760 with expected crc 2023-10-09T09:17:14.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:14 smithi163 bash[78107]: cluster 2023-10-09T09:17:13.478709+0000 mon.a (mon.0) 942 : cluster 0 osdmap e760: 8 total, 8 up, 8 in 2023-10-09T09:17:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:14 smithi163 bash[75425]: cluster 2023-10-09T09:17:13.386461+0000 mgr.y (mgr.34107) 320 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 253 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.1 MiB/s wr, 17 op/s 2023-10-09T09:17:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:14 smithi163 bash[75425]: cluster 2023-10-09T09:17:13.472086+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e760 with expected crc 2023-10-09T09:17:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:14 smithi163 bash[75425]: cluster 2023-10-09T09:17:13.474289+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e760 with expected crc 2023-10-09T09:17:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:14 smithi163 bash[75425]: cluster 2023-10-09T09:17:13.478709+0000 mon.a (mon.0) 942 : cluster 0 osdmap e760: 8 total, 8 up, 8 in 2023-10-09T09:17:15.190 INFO:tasks.rados.rados.0.smithi163.stdout:2750: done (0 left) 2023-10-09T09:17:15.191 INFO:tasks.rados.rados.0.smithi163.stdout:2751: read oid 24 snap -1 2023-10-09T09:17:15.191 INFO:tasks.rados.rados.0.smithi163.stdout:2751: expect (ObjNum 1019 snap 347 seq_num 1019) 2023-10-09T09:17:15.191 INFO:tasks.rados.rados.0.smithi163.stdout:2752: snap_remove snap 328 2023-10-09T09:17:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:15 smithi181 bash[59345]: cluster 2023-10-09T09:17:14.472388+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e761 with expected crc 2023-10-09T09:17:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:15 smithi181 bash[59345]: cluster 2023-10-09T09:17:14.473101+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e761 with expected crc 2023-10-09T09:17:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:15 smithi181 bash[59345]: cluster 2023-10-09T09:17:14.476393+0000 mon.a (mon.0) 943 : cluster 0 osdmap e761: 8 total, 8 up, 8 in 2023-10-09T09:17:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:15 smithi181 bash[59345]: cluster 2023-10-09T09:17:15.185537+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e762 with expected crc 2023-10-09T09:17:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:15 smithi181 bash[59345]: cluster 2023-10-09T09:17:15.185565+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e762 with expected crc 2023-10-09T09:17:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:15 smithi181 bash[59345]: cluster 2023-10-09T09:17:15.189912+0000 mon.a (mon.0) 944 : cluster 0 osdmap e762: 8 total, 8 up, 8 in 2023-10-09T09:17:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:15 smithi181 bash[59345]: cluster 2023-10-09T09:17:15.353568+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e762 with expected crc 2023-10-09T09:17:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[78107]: cluster 2023-10-09T09:17:14.472388+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e761 with expected crc 2023-10-09T09:17:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[78107]: cluster 2023-10-09T09:17:14.473101+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e761 with expected crc 2023-10-09T09:17:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[78107]: cluster 2023-10-09T09:17:14.476393+0000 mon.a (mon.0) 943 : cluster 0 osdmap e761: 8 total, 8 up, 8 in 2023-10-09T09:17:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[78107]: cluster 2023-10-09T09:17:15.185537+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e762 with expected crc 2023-10-09T09:17:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[78107]: cluster 2023-10-09T09:17:15.185565+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e762 with expected crc 2023-10-09T09:17:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[78107]: cluster 2023-10-09T09:17:15.189912+0000 mon.a (mon.0) 944 : cluster 0 osdmap e762: 8 total, 8 up, 8 in 2023-10-09T09:17:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[78107]: cluster 2023-10-09T09:17:15.353568+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e762 with expected crc 2023-10-09T09:17:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[75425]: cluster 2023-10-09T09:17:14.472388+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e761 with expected crc 2023-10-09T09:17:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[75425]: cluster 2023-10-09T09:17:14.473101+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e761 with expected crc 2023-10-09T09:17:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[75425]: cluster 2023-10-09T09:17:14.476393+0000 mon.a (mon.0) 943 : cluster 0 osdmap e761: 8 total, 8 up, 8 in 2023-10-09T09:17:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[75425]: cluster 2023-10-09T09:17:15.185537+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e762 with expected crc 2023-10-09T09:17:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[75425]: cluster 2023-10-09T09:17:15.185565+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e762 with expected crc 2023-10-09T09:17:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[75425]: cluster 2023-10-09T09:17:15.189912+0000 mon.a (mon.0) 944 : cluster 0 osdmap e762: 8 total, 8 up, 8 in 2023-10-09T09:17:15.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:15 smithi163 bash[75425]: cluster 2023-10-09T09:17:15.353568+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e762 with expected crc 2023-10-09T09:17:16.195 INFO:tasks.rados.rados.0.smithi163.stdout:2752: done (1 left) 2023-10-09T09:17:16.195 INFO:tasks.rados.rados.0.smithi163.stdout:2753: write oid 41 current snap is 350 2023-10-09T09:17:16.195 INFO:tasks.rados.rados.0.smithi163.stdout:2751: expect (ObjNum 1019 snap 347 seq_num 1019) 2023-10-09T09:17:16.307 INFO:tasks.rados.rados.0.smithi163.stdout:2753: seq_num 1030 ranges {511210=414125,1423159=486356,2668859=589585} 2023-10-09T09:17:16.318 INFO:tasks.rados.rados.0.smithi163.stdout:2753: writing smithi16367185-41 from 511210 to 925335 tid 1 2023-10-09T09:17:16.324 INFO:tasks.rados.rados.0.smithi163.stdout:2753: writing smithi16367185-41 from 1423159 to 1909515 tid 2 2023-10-09T09:17:16.331 INFO:tasks.rados.rados.0.smithi163.stdout:2753: writing smithi16367185-41 from 2668859 to 3258444 tid 3 2023-10-09T09:17:16.332 INFO:tasks.rados.rados.0.smithi163.stdout:2751: done (1 left) 2023-10-09T09:17:16.332 INFO:tasks.rados.rados.0.smithi163.stdout:2754: delete oid 29 current snap is 350 2023-10-09T09:17:16.332 INFO:tasks.rados.rados.0.smithi163.stdout:2753: finishing write tid 1 to smithi16367185-41 2023-10-09T09:17:16.332 INFO:tasks.rados.rados.0.smithi163.stdout:2753: finishing write tid 2 to smithi16367185-41 2023-10-09T09:17:16.335 INFO:tasks.rados.rados.0.smithi163.stdout:2754: done (1 left) 2023-10-09T09:17:16.336 INFO:tasks.rados.rados.0.smithi163.stdout:2755: read oid 17 snap -1 2023-10-09T09:17:16.336 INFO:tasks.rados.rados.0.smithi163.stdout:2755: expect deleted 2023-10-09T09:17:16.336 INFO:tasks.rados.rados.0.smithi163.stdout:2756: read oid 44 snap -1 2023-10-09T09:17:16.336 INFO:tasks.rados.rados.0.smithi163.stdout:2756: expect (ObjNum 1011 snap 345 seq_num 1011) 2023-10-09T09:17:16.336 INFO:tasks.rados.rados.0.smithi163.stdout:2757: rollback oid 10 current snap is 350 2023-10-09T09:17:16.336 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 348 2023-10-09T09:17:16.336 INFO:tasks.rados.rados.0.smithi163.stdout:2758: snap_create 2023-10-09T09:17:16.337 INFO:tasks.rados.rados.0.smithi163.stdout:2753: finishing write tid 3 to smithi16367185-41 2023-10-09T09:17:16.337 INFO:tasks.rados.rados.0.smithi163.stdout:2753: finishing write tid 4 to smithi16367185-41 2023-10-09T09:17:16.337 INFO:tasks.rados.rados.0.smithi163.stdout:2753: finishing write tid 5 to smithi16367185-41 2023-10-09T09:17:16.337 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 244 (ObjNum 1030 snap 350 seq_num 1030) dirty exists 2023-10-09T09:17:16.337 INFO:tasks.rados.rados.0.smithi163.stdout:2753: left oid 41 (ObjNum 1030 snap 350 seq_num 1030) 2023-10-09T09:17:16.339 INFO:tasks.rados.rados.0.smithi163.stdout:2757: finishing rollback tid 0 to smithi16367185-10 2023-10-09T09:17:16.340 INFO:tasks.rados.rados.0.smithi163.stdout:2757: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:17:16.340 INFO:tasks.rados.rados.0.smithi163.stdout:2757: finishing rollback tid 2 to smithi16367185-10 2023-10-09T09:17:16.340 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 880 (ObjNum 977 snap 337 seq_num 977) dirty exists 2023-10-09T09:17:16.344 INFO:tasks.rados.rados.0.smithi163.stdout:2756: expect (ObjNum 1011 snap 345 seq_num 1011) 2023-10-09T09:17:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:16 smithi181 bash[59345]: cluster 2023-10-09T09:17:15.387226+0000 mgr.y (mgr.34107) 321 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 740 KiB/s wr, 7 op/s 2023-10-09T09:17:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:16 smithi181 bash[59345]: cluster 2023-10-09T09:17:16.189237+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e763 with expected crc 2023-10-09T09:17:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:16 smithi181 bash[59345]: cluster 2023-10-09T09:17:16.193970+0000 mon.a (mon.0) 945 : cluster 0 osdmap e763: 8 total, 8 up, 8 in 2023-10-09T09:17:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:16 smithi163 bash[78107]: cluster 2023-10-09T09:17:15.387226+0000 mgr.y (mgr.34107) 321 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 740 KiB/s wr, 7 op/s 2023-10-09T09:17:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:16 smithi163 bash[78107]: cluster 2023-10-09T09:17:16.189237+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e763 with expected crc 2023-10-09T09:17:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:16 smithi163 bash[78107]: cluster 2023-10-09T09:17:16.193970+0000 mon.a (mon.0) 945 : cluster 0 osdmap e763: 8 total, 8 up, 8 in 2023-10-09T09:17:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:16 smithi163 bash[75425]: cluster 2023-10-09T09:17:15.387226+0000 mgr.y (mgr.34107) 321 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 740 KiB/s wr, 7 op/s 2023-10-09T09:17:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:16 smithi163 bash[75425]: cluster 2023-10-09T09:17:16.189237+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e763 with expected crc 2023-10-09T09:17:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:16 smithi163 bash[75425]: cluster 2023-10-09T09:17:16.193970+0000 mon.a (mon.0) 945 : cluster 0 osdmap e763: 8 total, 8 up, 8 in 2023-10-09T09:17:17.205 INFO:tasks.rados.rados.0.smithi163.stdout:2753: done (4 left) 2023-10-09T09:17:17.205 INFO:tasks.rados.rados.0.smithi163.stdout:2755: done (3 left) 2023-10-09T09:17:17.205 INFO:tasks.rados.rados.0.smithi163.stdout:2756: done (2 left) 2023-10-09T09:17:17.205 INFO:tasks.rados.rados.0.smithi163.stdout:2757: done (1 left) 2023-10-09T09:17:17.205 INFO:tasks.rados.rados.0.smithi163.stdout:2758: done (0 left) 2023-10-09T09:17:17.205 INFO:tasks.rados.rados.0.smithi163.stdout:2759: read oid 42 snap -1 2023-10-09T09:17:17.206 INFO:tasks.rados.rados.0.smithi163.stdout:2759: expect deleted 2023-10-09T09:17:17.206 INFO:tasks.rados.rados.0.smithi163.stdout:2760: read oid 29 snap -1 2023-10-09T09:17:17.206 INFO:tasks.rados.rados.0.smithi163.stdout:2760: expect deleted 2023-10-09T09:17:17.206 INFO:tasks.rados.rados.0.smithi163.stdout:2761: read oid 21 snap -1 2023-10-09T09:17:17.206 INFO:tasks.rados.rados.0.smithi163.stdout:2761: expect (ObjNum 936 snap 320 seq_num 936) 2023-10-09T09:17:17.206 INFO:tasks.rados.rados.0.smithi163.stdout:2762: read oid 45 snap -1 2023-10-09T09:17:17.206 INFO:tasks.rados.rados.0.smithi163.stdout:2762: expect (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:17:17.206 INFO:tasks.rados.rados.0.smithi163.stdout:2763: rollback oid 34 current snap is 351 2023-10-09T09:17:17.206 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 34 to 347 2023-10-09T09:17:17.206 INFO:tasks.rados.rados.0.smithi163.stdout:2764: snap_remove snap 348 2023-10-09T09:17:18.211 INFO:tasks.rados.rados.0.smithi163.stdout:2764: done (5 left) 2023-10-09T09:17:18.211 INFO:tasks.rados.rados.0.smithi163.stdout:2765: read oid 28 snap -1 2023-10-09T09:17:18.211 INFO:tasks.rados.rados.0.smithi163.stdout:2765: expect (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:17:18.212 INFO:tasks.rados.rados.0.smithi163.stdout:2766: write oid 41 current snap is 351 2023-10-09T09:17:18.212 INFO:tasks.rados.rados.0.smithi163.stdout:2766: seq_num 1032 ranges {544824=564623,1823770=490025,3007352=337472} 2023-10-09T09:17:18.219 INFO:tasks.rados.rados.0.smithi163.stdout:2766: writing smithi16367185-41 from 544824 to 1109447 tid 1 2023-10-09T09:17:18.226 INFO:tasks.rados.rados.0.smithi163.stdout:2766: writing smithi16367185-41 from 1823770 to 2313795 tid 2 2023-10-09T09:17:18.233 INFO:tasks.rados.rados.0.smithi163.stdout:2766: writing smithi16367185-41 from 3007352 to 3344824 tid 3 2023-10-09T09:17:18.233 INFO:tasks.rados.rados.0.smithi163.stdout:2767: write oid 47 current snap is 351 2023-10-09T09:17:18.233 INFO:tasks.rados.rados.0.smithi163.stdout:2767: seq_num 1033 ranges {561631=639872,1882252=633683,3034775=326856} 2023-10-09T09:17:18.240 INFO:tasks.rados.rados.0.smithi163.stdout:2767: writing smithi16367185-47 from 561631 to 1201503 tid 1 2023-10-09T09:17:18.248 INFO:tasks.rados.rados.0.smithi163.stdout:2767: writing smithi16367185-47 from 1882252 to 2515935 tid 2 2023-10-09T09:17:18.253 INFO:tasks.rados.rados.0.smithi163.stdout:2767: writing smithi16367185-47 from 3034775 to 3361631 tid 3 2023-10-09T09:17:18.253 INFO:tasks.rados.rados.0.smithi163.stdout:2768: rollback oid 2 current snap is 351 2023-10-09T09:17:18.253 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 2 to 346 2023-10-09T09:17:18.253 INFO:tasks.rados.rados.0.smithi163.stdout:2760: done (8 left) 2023-10-09T09:17:18.253 INFO:tasks.rados.rados.0.smithi163.stdout:2769: rollback oid 44 current snap is 351 2023-10-09T09:17:18.253 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 44 to 338 2023-10-09T09:17:18.254 INFO:tasks.rados.rados.0.smithi163.stdout:2762: expect (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:17:18.339 INFO:tasks.rados.rados.0.smithi163.stdout:2763: finishing rollback tid 0 to smithi16367185-34 2023-10-09T09:17:18.340 INFO:tasks.rados.rados.0.smithi163.stdout:2763: finishing rollback tid 1 to smithi16367185-34 2023-10-09T09:17:18.340 INFO:tasks.rados.rados.0.smithi163.stdout:2763: finishing rollback tid 2 to smithi16367185-34 2023-10-09T09:17:18.340 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1292 (ObjNum 969 snap 332 seq_num 969) dirty exists 2023-10-09T09:17:18.340 INFO:tasks.rados.rados.0.smithi163.stdout:2761: expect (ObjNum 936 snap 320 seq_num 936) 2023-10-09T09:17:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:18 smithi163 bash[75425]: cluster 2023-10-09T09:17:17.199660+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e764 with expected crc 2023-10-09T09:17:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:18 smithi163 bash[75425]: cluster 2023-10-09T09:17:17.199984+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e764 with expected crc 2023-10-09T09:17:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:18 smithi163 bash[75425]: cluster 2023-10-09T09:17:17.204569+0000 mon.a (mon.0) 946 : cluster 0 osdmap e764: 8 total, 8 up, 8 in 2023-10-09T09:17:18.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:18 smithi163 bash[75425]: cluster 2023-10-09T09:17:17.387943+0000 mgr.y (mgr.34107) 322 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 244 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2023-10-09T09:17:18.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:18 smithi163 bash[78107]: cluster 2023-10-09T09:17:17.199660+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e764 with expected crc 2023-10-09T09:17:18.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:18 smithi163 bash[78107]: cluster 2023-10-09T09:17:17.199984+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e764 with expected crc 2023-10-09T09:17:18.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:18 smithi163 bash[78107]: cluster 2023-10-09T09:17:17.204569+0000 mon.a (mon.0) 946 : cluster 0 osdmap e764: 8 total, 8 up, 8 in 2023-10-09T09:17:18.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:18 smithi163 bash[78107]: cluster 2023-10-09T09:17:17.387943+0000 mgr.y (mgr.34107) 322 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 244 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2023-10-09T09:17:18.507 INFO:tasks.rados.rados.0.smithi163.stdout:2765: expect (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:17:18.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:18 smithi181 bash[59345]: cluster 2023-10-09T09:17:17.199660+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e764 with expected crc 2023-10-09T09:17:18.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:18 smithi181 bash[59345]: cluster 2023-10-09T09:17:17.199984+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e764 with expected crc 2023-10-09T09:17:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:18 smithi181 bash[59345]: cluster 2023-10-09T09:17:17.204569+0000 mon.a (mon.0) 946 : cluster 0 osdmap e764: 8 total, 8 up, 8 in 2023-10-09T09:17:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:18 smithi181 bash[59345]: cluster 2023-10-09T09:17:17.387943+0000 mgr.y (mgr.34107) 322 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 244 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2023-10-09T09:17:18.601 INFO:tasks.rados.rados.0.smithi163.stdout:2766: finishing write tid 1 to smithi16367185-41 2023-10-09T09:17:18.601 INFO:tasks.rados.rados.0.smithi163.stdout:2766: finishing write tid 2 to smithi16367185-41 2023-10-09T09:17:18.601 INFO:tasks.rados.rados.0.smithi163.stdout:2766: finishing write tid 3 to smithi16367185-41 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:2766: finishing write tid 4 to smithi16367185-41 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:2766: finishing write tid 5 to smithi16367185-41 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 251 (ObjNum 1032 snap 351 seq_num 1032) dirty exists 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:2766: left oid 41 (ObjNum 1032 snap 351 seq_num 1032) 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:2767: finishing write tid 1 to smithi16367185-47 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:2767: finishing write tid 2 to smithi16367185-47 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:2768: finishing rollback tid 1 to smithi16367185-2 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 627 (ObjNum 669362160 snap 21875 seq_num 1) dirty dne 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:2767: finishing write tid 3 to smithi16367185-47 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:2767: finishing write tid 4 to smithi16367185-47 2023-10-09T09:17:18.602 INFO:tasks.rados.rados.0.smithi163.stdout:2767: finishing write tid 5 to smithi16367185-47 2023-10-09T09:17:18.603 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 1967 (ObjNum 1033 snap 351 seq_num 1033) dirty exists 2023-10-09T09:17:18.603 INFO:tasks.rados.rados.0.smithi163.stdout:2767: left oid 47 (ObjNum 1033 snap 351 seq_num 1033) 2023-10-09T09:17:18.603 INFO:tasks.rados.rados.0.smithi163.stdout:2769: finishing rollback tid 0 to smithi16367185-44 2023-10-09T09:17:18.603 INFO:tasks.rados.rados.0.smithi163.stdout:2769: finishing rollback tid 1 to smithi16367185-44 2023-10-09T09:17:18.603 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1970 (ObjNum 699301264 snap 21875 seq_num 699307888) dirty dne 2023-10-09T09:17:18.603 INFO:tasks.rados.rados.0.smithi163.stdout:2759: done (8 left) 2023-10-09T09:17:18.603 INFO:tasks.rados.rados.0.smithi163.stdout:2761: done (7 left) 2023-10-09T09:17:18.603 INFO:tasks.rados.rados.0.smithi163.stdout:2762: done (6 left) 2023-10-09T09:17:18.604 INFO:tasks.rados.rados.0.smithi163.stdout:2763: done (5 left) 2023-10-09T09:17:18.604 INFO:tasks.rados.rados.0.smithi163.stdout:2765: done (4 left) 2023-10-09T09:17:18.604 INFO:tasks.rados.rados.0.smithi163.stdout:2766: done (3 left) 2023-10-09T09:17:18.604 INFO:tasks.rados.rados.0.smithi163.stdout:2767: done (2 left) 2023-10-09T09:17:18.604 INFO:tasks.rados.rados.0.smithi163.stdout:2768: done (1 left) 2023-10-09T09:17:18.604 INFO:tasks.rados.rados.0.smithi163.stdout:2769: done (0 left) 2023-10-09T09:17:18.604 INFO:tasks.rados.rados.0.smithi163.stdout:2770: read oid 2 snap -1 2023-10-09T09:17:18.604 INFO:tasks.rados.rados.0.smithi163.stdout:2770: expect deleted 2023-10-09T09:17:18.605 INFO:tasks.rados.rados.0.smithi163.stdout:2771: write oid 30 current snap is 351 2023-10-09T09:17:18.605 INFO:tasks.rados.rados.0.smithi163.stdout:2771: seq_num 1034 ranges {578438=431474,1657087=493694,2778551=599887} 2023-10-09T09:17:18.613 INFO:tasks.rados.rados.0.smithi163.stdout:2771: writing smithi16367185-30 from 578438 to 1009912 tid 1 2023-10-09T09:17:18.619 INFO:tasks.rados.rados.0.smithi163.stdout:2771: writing smithi16367185-30 from 1657087 to 2150781 tid 2 2023-10-09T09:17:18.626 INFO:tasks.rados.rados.0.smithi163.stdout:2771: writing smithi16367185-30 from 2778551 to 3378438 tid 3 2023-10-09T09:17:18.626 INFO:tasks.rados.rados.0.smithi163.stdout:2772: read oid 35 snap -1 2023-10-09T09:17:18.627 INFO:tasks.rados.rados.0.smithi163.stdout:2772: expect (ObjNum 939 snap 322 seq_num 939) 2023-10-09T09:17:18.627 INFO:tasks.rados.rados.0.smithi163.stdout:2773: write oid 33 current snap is 351 2023-10-09T09:17:18.627 INFO:tasks.rados.rados.0.smithi163.stdout:2773: seq_num 1035 ranges {595245=506723,1715569=637352,2805974=562298} 2023-10-09T09:17:18.632 INFO:tasks.rados.rados.0.smithi163.stdout:2773: writing smithi16367185-33 from 595245 to 1101968 tid 1 2023-10-09T09:17:18.640 INFO:tasks.rados.rados.0.smithi163.stdout:2773: writing smithi16367185-33 from 1715569 to 2352921 tid 2 2023-10-09T09:17:18.646 INFO:tasks.rados.rados.0.smithi163.stdout:2773: writing smithi16367185-33 from 2805974 to 3368272 tid 3 2023-10-09T09:17:18.646 INFO:tasks.rados.rados.0.smithi163.stdout:2774: write oid 41 current snap is 351 2023-10-09T09:17:18.646 INFO:tasks.rados.rados.0.smithi163.stdout:2771: finishing write tid 1 to smithi16367185-30 2023-10-09T09:17:18.647 INFO:tasks.rados.rados.0.smithi163.stdout:2771: finishing write tid 2 to smithi16367185-30 2023-10-09T09:17:18.647 INFO:tasks.rados.rados.0.smithi163.stdout:2771: finishing write tid 3 to smithi16367185-30 2023-10-09T09:17:18.647 INFO:tasks.rados.rados.0.smithi163.stdout:2771: finishing write tid 4 to smithi16367185-30 2023-10-09T09:17:18.647 INFO:tasks.rados.rados.0.smithi163.stdout:2771: finishing write tid 5 to smithi16367185-30 2023-10-09T09:17:18.647 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 887 (ObjNum 1034 snap 351 seq_num 1034) dirty exists 2023-10-09T09:17:18.647 INFO:tasks.rados.rados.0.smithi163.stdout:2771: left oid 30 (ObjNum 1034 snap 351 seq_num 1034) 2023-10-09T09:17:18.647 INFO:tasks.rados.rados.0.smithi163.stdout:2773: finishing write tid 1 to smithi16367185-33 2023-10-09T09:17:18.648 INFO:tasks.rados.rados.0.smithi163.stdout:2772: expect (ObjNum 939 snap 322 seq_num 939) 2023-10-09T09:17:18.773 INFO:tasks.rados.rados.0.smithi163.stdout:2773: finishing write tid 2 to smithi16367185-33 2023-10-09T09:17:18.774 INFO:tasks.rados.rados.0.smithi163.stdout:2773: finishing write tid 3 to smithi16367185-33 2023-10-09T09:17:18.774 INFO:tasks.rados.rados.0.smithi163.stdout:2773: finishing write tid 4 to smithi16367185-33 2023-10-09T09:17:18.774 INFO:tasks.rados.rados.0.smithi163.stdout:2773: finishing write tid 5 to smithi16367185-33 2023-10-09T09:17:18.774 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1307 (ObjNum 1035 snap 351 seq_num 1035) dirty exists 2023-10-09T09:17:18.774 INFO:tasks.rados.rados.0.smithi163.stdout:2773: left oid 33 (ObjNum 1035 snap 351 seq_num 1035) 2023-10-09T09:17:18.774 INFO:tasks.rados.rados.0.smithi163.stdout:2774: seq_num 1036 ranges {612052=581972,1657698=497363,2717044=695008} 2023-10-09T09:17:18.781 INFO:tasks.rados.rados.0.smithi163.stdout:2774: writing smithi16367185-41 from 612052 to 1194024 tid 1 2023-10-09T09:17:18.787 INFO:tasks.rados.rados.0.smithi163.stdout:2774: writing smithi16367185-41 from 1657698 to 2155061 tid 2 2023-10-09T09:17:18.796 INFO:tasks.rados.rados.0.smithi163.stdout:2774: writing smithi16367185-41 from 2717044 to 3412052 tid 3 2023-10-09T09:17:18.796 INFO:tasks.rados.rados.0.smithi163.stdout:2770: done (4 left) 2023-10-09T09:17:18.796 INFO:tasks.rados.rados.0.smithi163.stdout:2771: done (3 left) 2023-10-09T09:17:18.796 INFO:tasks.rados.rados.0.smithi163.stdout:2772: done (2 left) 2023-10-09T09:17:18.796 INFO:tasks.rados.rados.0.smithi163.stdout:2773: done (1 left) 2023-10-09T09:17:18.796 INFO:tasks.rados.rados.0.smithi163.stdout:2775: read oid 25 snap -1 2023-10-09T09:17:18.797 INFO:tasks.rados.rados.0.smithi163.stdout:2775: expect (ObjNum 1004 snap 342 seq_num 1004) 2023-10-09T09:17:18.797 INFO:tasks.rados.rados.0.smithi163.stdout:2776: rollback oid 38 current snap is 351 2023-10-09T09:17:18.797 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 349 2023-10-09T09:17:18.797 INFO:tasks.rados.rados.0.smithi163.stdout:2774: finishing write tid 1 to smithi16367185-41 2023-10-09T09:17:18.797 INFO:tasks.rados.rados.0.smithi163.stdout:2777: read oid 5 snap -1 2023-10-09T09:17:18.797 INFO:tasks.rados.rados.0.smithi163.stdout:2777: expect (ObjNum 1024 snap 349 seq_num 1024) 2023-10-09T09:17:18.797 INFO:tasks.rados.rados.0.smithi163.stdout:2778: snap_create 2023-10-09T09:17:18.797 INFO:tasks.rados.rados.0.smithi163.stdout:2774: finishing write tid 2 to smithi16367185-41 2023-10-09T09:17:18.802 INFO:tasks.rados.rados.0.smithi163.stdout:2774: finishing write tid 3 to smithi16367185-41 2023-10-09T09:17:18.803 INFO:tasks.rados.rados.0.smithi163.stdout:2776: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:17:18.803 INFO:tasks.rados.rados.0.smithi163.stdout:2776: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:17:18.803 INFO:tasks.rados.rados.0.smithi163.stdout:2776: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:17:18.803 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1305 (ObjNum 997 snap 342 seq_num 997) dirty exists 2023-10-09T09:17:18.804 INFO:tasks.rados.rados.0.smithi163.stdout:2775: expect (ObjNum 1004 snap 342 seq_num 1004) 2023-10-09T09:17:18.933 INFO:tasks.rados.rados.0.smithi163.stdout:2777: expect (ObjNum 1024 snap 349 seq_num 1024) 2023-10-09T09:17:19.084 INFO:tasks.rados.rados.0.smithi163.stdout:2774: finishing write tid 4 to smithi16367185-41 2023-10-09T09:17:19.084 INFO:tasks.rados.rados.0.smithi163.stdout:2774: finishing write tid 5 to smithi16367185-41 2023-10-09T09:17:19.084 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 255 (ObjNum 1036 snap 351 seq_num 1036) dirty exists 2023-10-09T09:17:19.085 INFO:tasks.rados.rados.0.smithi163.stdout:2774: left oid 41 (ObjNum 1036 snap 351 seq_num 1036) 2023-10-09T09:17:19.216 INFO:tasks.rados.rados.0.smithi163.stdout:2774: done (4 left) 2023-10-09T09:17:19.216 INFO:tasks.rados.rados.0.smithi163.stdout:2775: done (3 left) 2023-10-09T09:17:19.216 INFO:tasks.rados.rados.0.smithi163.stdout:2776: done (2 left) 2023-10-09T09:17:19.216 INFO:tasks.rados.rados.0.smithi163.stdout:2777: done (1 left) 2023-10-09T09:17:19.216 INFO:tasks.rados.rados.0.smithi163.stdout:2778: done (0 left) 2023-10-09T09:17:19.216 INFO:tasks.rados.rados.0.smithi163.stdout:2779: delete oid 30 current snap is 352 2023-10-09T09:17:19.223 INFO:tasks.rados.rados.0.smithi163.stdout:2779: done (0 left) 2023-10-09T09:17:19.223 INFO:tasks.rados.rados.0.smithi163.stdout:2780: rollback oid 50 current snap is 352 2023-10-09T09:17:19.223 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 50 to 319 2023-10-09T09:17:19.224 INFO:tasks.rados.rados.0.smithi163.stdout:2781: read oid 9 snap -1 2023-10-09T09:17:19.224 INFO:tasks.rados.rados.0.smithi163.stdout:2781: expect deleted 2023-10-09T09:17:19.224 INFO:tasks.rados.rados.0.smithi163.stdout:2782: write oid 23 current snap is 352 2023-10-09T09:17:19.224 INFO:tasks.rados.rados.0.smithi163.stdout:2782: seq_num 1038 ranges {645666=732470,2174662=501032,3288243=157423} 2023-10-09T09:17:19.232 INFO:tasks.rados.rados.0.smithi163.stdout:2782: writing smithi16367185-23 from 645666 to 1378136 tid 1 2023-10-09T09:17:19.253 INFO:tasks.rados.rados.0.smithi163.stdout:2782: writing smithi16367185-23 from 2174662 to 2675694 tid 2 2023-10-09T09:17:19.253 INFO:tasks.rados.rados.0.smithi163.stdout:2782: writing smithi16367185-23 from 3288243 to 3445666 tid 3 2023-10-09T09:17:19.253 INFO:tasks.rados.rados.0.smithi163.stdout:2783: write oid 13 current snap is 352 2023-10-09T09:17:19.253 INFO:tasks.rados.rados.0.smithi163.stdout:2783: seq_num 1039 ranges {662473=407719,1833144=644690,3199313=263160} 2023-10-09T09:17:19.253 INFO:tasks.rados.rados.0.smithi163.stdout:2783: writing smithi16367185-13 from 662473 to 1070192 tid 1 2023-10-09T09:17:19.259 INFO:tasks.rados.rados.0.smithi163.stdout:2783: writing smithi16367185-13 from 1833144 to 2477834 tid 2 2023-10-09T09:17:19.265 INFO:tasks.rados.rados.0.smithi163.stdout:2783: writing smithi16367185-13 from 3199313 to 3462473 tid 3 2023-10-09T09:17:19.265 INFO:tasks.rados.rados.0.smithi163.stdout:2784: rollback oid 20 current snap is 352 2023-10-09T09:17:19.265 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 349 2023-10-09T09:17:19.265 INFO:tasks.rados.rados.0.smithi163.stdout:2780: finishing rollback tid 1 to smithi16367185-50 2023-10-09T09:17:19.265 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1115 (ObjNum 669367496 snap 21875 seq_num 2) dirty dne 2023-10-09T09:17:19.265 INFO:tasks.rados.rados.0.smithi163.stdout:2782: finishing write tid 1 to smithi16367185-23 2023-10-09T09:17:19.265 INFO:tasks.rados.rados.0.smithi163.stdout:2782: finishing write tid 2 to smithi16367185-23 2023-10-09T09:17:19.265 INFO:tasks.rados.rados.0.smithi163.stdout:2782: finishing write tid 3 to smithi16367185-23 2023-10-09T09:17:19.266 INFO:tasks.rados.rados.0.smithi163.stdout:2782: finishing write tid 4 to smithi16367185-23 2023-10-09T09:17:19.266 INFO:tasks.rados.rados.0.smithi163.stdout:2782: finishing write tid 5 to smithi16367185-23 2023-10-09T09:17:19.266 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1299 (ObjNum 1038 snap 352 seq_num 1038) dirty exists 2023-10-09T09:17:19.266 INFO:tasks.rados.rados.0.smithi163.stdout:2782: left oid 23 (ObjNum 1038 snap 352 seq_num 1038) 2023-10-09T09:17:19.266 INFO:tasks.rados.rados.0.smithi163.stdout:2783: finishing write tid 1 to smithi16367185-13 2023-10-09T09:17:19.266 INFO:tasks.rados.rados.0.smithi163.stdout:2780: done (4 left) 2023-10-09T09:17:19.266 INFO:tasks.rados.rados.0.smithi163.stdout:2781: done (3 left) 2023-10-09T09:17:19.266 INFO:tasks.rados.rados.0.smithi163.stdout:2782: done (2 left) 2023-10-09T09:17:19.266 INFO:tasks.rados.rados.0.smithi163.stdout:2785: write oid 42 current snap is 352 2023-10-09T09:17:19.266 INFO:tasks.rados.rados.0.smithi163.stdout:2785: seq_num 1040 ranges {679280=482968,1775273=788348,3110383=368897} 2023-10-09T09:17:19.271 INFO:tasks.rados.rados.0.smithi163.stdout:2785: writing smithi16367185-42 from 679280 to 1162248 tid 1 2023-10-09T09:17:19.279 INFO:tasks.rados.rados.0.smithi163.stdout:2785: writing smithi16367185-42 from 1775273 to 2563621 tid 2 2023-10-09T09:17:19.285 INFO:tasks.rados.rados.0.smithi163.stdout:2785: writing smithi16367185-42 from 3110383 to 3479280 tid 3 2023-10-09T09:17:19.285 INFO:tasks.rados.rados.0.smithi163.stdout:2786: snap_remove snap 319 2023-10-09T09:17:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:19 smithi181 bash[59345]: cluster 2023-10-09T09:17:18.206728+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e765 with expected crc 2023-10-09T09:17:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:19 smithi181 bash[59345]: cluster 2023-10-09T09:17:18.208076+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e765 with expected crc 2023-10-09T09:17:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:19 smithi181 bash[59345]: cluster 2023-10-09T09:17:18.211200+0000 mon.a (mon.0) 947 : cluster 0 osdmap e765: 8 total, 8 up, 8 in 2023-10-09T09:17:19.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:19 smithi181 bash[59345]: cluster 2023-10-09T09:17:18.224086+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e765 with expected crc 2023-10-09T09:17:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:19 smithi163 bash[78107]: cluster 2023-10-09T09:17:18.206728+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e765 with expected crc 2023-10-09T09:17:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:19 smithi163 bash[78107]: cluster 2023-10-09T09:17:18.208076+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e765 with expected crc 2023-10-09T09:17:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:19 smithi163 bash[78107]: cluster 2023-10-09T09:17:18.211200+0000 mon.a (mon.0) 947 : cluster 0 osdmap e765: 8 total, 8 up, 8 in 2023-10-09T09:17:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:19 smithi163 bash[78107]: cluster 2023-10-09T09:17:18.224086+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e765 with expected crc 2023-10-09T09:17:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:19 smithi163 bash[75425]: cluster 2023-10-09T09:17:18.206728+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e765 with expected crc 2023-10-09T09:17:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:19 smithi163 bash[75425]: cluster 2023-10-09T09:17:18.208076+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e765 with expected crc 2023-10-09T09:17:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:19 smithi163 bash[75425]: cluster 2023-10-09T09:17:18.211200+0000 mon.a (mon.0) 947 : cluster 0 osdmap e765: 8 total, 8 up, 8 in 2023-10-09T09:17:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:19 smithi163 bash[75425]: cluster 2023-10-09T09:17:18.224086+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e765 with expected crc 2023-10-09T09:17:20.195 INFO:tasks.rados.rados.0.smithi163.stdout:2786: done (3 left) 2023-10-09T09:17:20.196 INFO:tasks.rados.rados.0.smithi163.stdout:2787: snap_remove snap 346 2023-10-09T09:17:20.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:20 smithi181 bash[59345]: cluster 2023-10-09T09:17:19.209750+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e766 with expected crc 2023-10-09T09:17:20.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:20 smithi181 bash[59345]: cluster 2023-10-09T09:17:19.215445+0000 mon.a (mon.0) 948 : cluster 0 osdmap e766: 8 total, 8 up, 8 in 2023-10-09T09:17:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:20 smithi181 bash[59345]: cluster 2023-10-09T09:17:19.248892+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e766 with expected crc 2023-10-09T09:17:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:20 smithi181 bash[59345]: cluster 2023-10-09T09:17:19.388692+0000 mgr.y (mgr.34107) 323 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 244 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 716 KiB/s wr, 7 op/s 2023-10-09T09:17:20.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:20 smithi181 bash[59345]: cluster 2023-10-09T09:17:20.195330+0000 mon.a (mon.0) 949 : cluster 0 osdmap e767: 8 total, 8 up, 8 in 2023-10-09T09:17:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:20 smithi163 bash[78107]: cluster 2023-10-09T09:17:19.209750+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e766 with expected crc 2023-10-09T09:17:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:20 smithi163 bash[78107]: cluster 2023-10-09T09:17:19.215445+0000 mon.a (mon.0) 948 : cluster 0 osdmap e766: 8 total, 8 up, 8 in 2023-10-09T09:17:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:20 smithi163 bash[78107]: cluster 2023-10-09T09:17:19.248892+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e766 with expected crc 2023-10-09T09:17:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:20 smithi163 bash[78107]: cluster 2023-10-09T09:17:19.388692+0000 mgr.y (mgr.34107) 323 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 244 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 716 KiB/s wr, 7 op/s 2023-10-09T09:17:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:20 smithi163 bash[78107]: cluster 2023-10-09T09:17:20.195330+0000 mon.a (mon.0) 949 : cluster 0 osdmap e767: 8 total, 8 up, 8 in 2023-10-09T09:17:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:20 smithi163 bash[75425]: cluster 2023-10-09T09:17:19.209750+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e766 with expected crc 2023-10-09T09:17:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:20 smithi163 bash[75425]: cluster 2023-10-09T09:17:19.215445+0000 mon.a (mon.0) 948 : cluster 0 osdmap e766: 8 total, 8 up, 8 in 2023-10-09T09:17:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:20 smithi163 bash[75425]: cluster 2023-10-09T09:17:19.248892+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e766 with expected crc 2023-10-09T09:17:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:20 smithi163 bash[75425]: cluster 2023-10-09T09:17:19.388692+0000 mgr.y (mgr.34107) 323 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 244 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 716 KiB/s wr, 7 op/s 2023-10-09T09:17:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:20 smithi163 bash[75425]: cluster 2023-10-09T09:17:20.195330+0000 mon.a (mon.0) 949 : cluster 0 osdmap e767: 8 total, 8 up, 8 in 2023-10-09T09:17:21.194 INFO:tasks.rados.rados.0.smithi163.stdout:2787: done (3 left) 2023-10-09T09:17:21.194 INFO:tasks.rados.rados.0.smithi163.stdout:2788: write oid 14 current snap is 352 2023-10-09T09:17:21.194 INFO:tasks.rados.rados.0.smithi163.stdout:2783: finishing write tid 2 to smithi16367185-13 2023-10-09T09:17:21.194 INFO:tasks.rados.rados.0.smithi163.stdout:2788: seq_num 1041 ranges {696087=558217,1833755=648359,3137806=358281} 2023-10-09T09:17:21.206 INFO:tasks.rados.rados.0.smithi163.stdout:2788: writing smithi16367185-14 from 696087 to 1254304 tid 1 2023-10-09T09:17:21.213 INFO:tasks.rados.rados.0.smithi163.stdout:2788: writing smithi16367185-14 from 1833755 to 2482114 tid 2 2023-10-09T09:17:21.220 INFO:tasks.rados.rados.0.smithi163.stdout:2788: writing smithi16367185-14 from 3137806 to 3496087 tid 3 2023-10-09T09:17:21.220 INFO:tasks.rados.rados.0.smithi163.stdout:2789: snap_create 2023-10-09T09:17:21.220 INFO:tasks.rados.rados.0.smithi163.stdout:2783: finishing write tid 3 to smithi16367185-13 2023-10-09T09:17:21.220 INFO:tasks.rados.rados.0.smithi163.stdout:2783: finishing write tid 4 to smithi16367185-13 2023-10-09T09:17:21.220 INFO:tasks.rados.rados.0.smithi163.stdout:2783: finishing write tid 5 to smithi16367185-13 2023-10-09T09:17:21.220 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 462 (ObjNum 1039 snap 352 seq_num 1039) dirty exists 2023-10-09T09:17:21.221 INFO:tasks.rados.rados.0.smithi163.stdout:2783: left oid 13 (ObjNum 1039 snap 352 seq_num 1039) 2023-10-09T09:17:21.221 INFO:tasks.rados.rados.0.smithi163.stdout:2784: finishing rollback tid 0 to smithi16367185-20 2023-10-09T09:17:21.221 INFO:tasks.rados.rados.0.smithi163.stdout:2784: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:17:21.221 INFO:tasks.rados.rados.0.smithi163.stdout:2784: finishing rollback tid 2 to smithi16367185-20 2023-10-09T09:17:21.221 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 466 (ObjNum 994 snap 341 seq_num 994) dirty exists 2023-10-09T09:17:21.221 INFO:tasks.rados.rados.0.smithi163.stdout:2785: finishing write tid 1 to smithi16367185-42 2023-10-09T09:17:21.221 INFO:tasks.rados.rados.0.smithi163.stdout:2785: finishing write tid 2 to smithi16367185-42 2023-10-09T09:17:21.222 INFO:tasks.rados.rados.0.smithi163.stdout:2785: finishing write tid 3 to smithi16367185-42 2023-10-09T09:17:21.222 INFO:tasks.rados.rados.0.smithi163.stdout:2785: finishing write tid 4 to smithi16367185-42 2023-10-09T09:17:21.222 INFO:tasks.rados.rados.0.smithi163.stdout:2785: finishing write tid 5 to smithi16367185-42 2023-10-09T09:17:21.222 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 1974 (ObjNum 1040 snap 352 seq_num 1040) dirty exists 2023-10-09T09:17:21.222 INFO:tasks.rados.rados.0.smithi163.stdout:2785: left oid 42 (ObjNum 1040 snap 352 seq_num 1040) 2023-10-09T09:17:21.223 INFO:tasks.rados.rados.0.smithi163.stdout:2788: finishing write tid 1 to smithi16367185-14 2023-10-09T09:17:21.223 INFO:tasks.rados.rados.0.smithi163.stdout:2788: finishing write tid 2 to smithi16367185-14 2023-10-09T09:17:21.224 INFO:tasks.rados.rados.0.smithi163.stdout:2788: finishing write tid 3 to smithi16367185-14 2023-10-09T09:17:21.224 INFO:tasks.rados.rados.0.smithi163.stdout:2788: finishing write tid 4 to smithi16367185-14 2023-10-09T09:17:21.224 INFO:tasks.rados.rados.0.smithi163.stdout:2788: finishing write tid 5 to smithi16367185-14 2023-10-09T09:17:21.224 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1320 (ObjNum 1041 snap 352 seq_num 1041) dirty exists 2023-10-09T09:17:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:2788: left oid 14 (ObjNum 1041 snap 352 seq_num 1041) 2023-10-09T09:17:21.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:21 smithi181 bash[59345]: cluster 2023-10-09T09:17:20.192080+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e767 with expected crc 2023-10-09T09:17:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:21 smithi181 bash[59345]: cluster 2023-10-09T09:17:20.192130+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e767 with expected crc 2023-10-09T09:17:21.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:21 smithi181 bash[59345]: cluster 2023-10-09T09:17:21.194057+0000 mon.a (mon.0) 950 : cluster 0 osdmap e768: 8 total, 8 up, 8 in 2023-10-09T09:17:21.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:21 smithi163 bash[75425]: cluster 2023-10-09T09:17:20.192080+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e767 with expected crc 2023-10-09T09:17:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:21 smithi163 bash[75425]: cluster 2023-10-09T09:17:20.192130+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e767 with expected crc 2023-10-09T09:17:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:21 smithi163 bash[75425]: cluster 2023-10-09T09:17:21.194057+0000 mon.a (mon.0) 950 : cluster 0 osdmap e768: 8 total, 8 up, 8 in 2023-10-09T09:17:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:21 smithi163 bash[78107]: cluster 2023-10-09T09:17:20.192080+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e767 with expected crc 2023-10-09T09:17:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:21 smithi163 bash[78107]: cluster 2023-10-09T09:17:20.192130+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e767 with expected crc 2023-10-09T09:17:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:21 smithi163 bash[78107]: cluster 2023-10-09T09:17:21.194057+0000 mon.a (mon.0) 950 : cluster 0 osdmap e768: 8 total, 8 up, 8 in 2023-10-09T09:17:22.197 INFO:tasks.rados.rados.0.smithi163.stdout:2783: done (4 left) 2023-10-09T09:17:22.197 INFO:tasks.rados.rados.0.smithi163.stdout:2784: done (3 left) 2023-10-09T09:17:22.198 INFO:tasks.rados.rados.0.smithi163.stdout:2785: done (2 left) 2023-10-09T09:17:22.198 INFO:tasks.rados.rados.0.smithi163.stdout:2788: done (1 left) 2023-10-09T09:17:22.198 INFO:tasks.rados.rados.0.smithi163.stdout:2789: done (0 left) 2023-10-09T09:17:22.198 INFO:tasks.rados.rados.0.smithi163.stdout:2790: delete oid 8 current snap is 353 2023-10-09T09:17:22.207 INFO:tasks.rados.rados.0.smithi163.stdout:2790: done (0 left) 2023-10-09T09:17:22.207 INFO:tasks.rados.rados.0.smithi163.stdout:2791: read oid 29 snap -1 2023-10-09T09:17:22.208 INFO:tasks.rados.rados.0.smithi163.stdout:2791: expect deleted 2023-10-09T09:17:22.208 INFO:tasks.rados.rados.0.smithi163.stdout:2792: read oid 43 snap -1 2023-10-09T09:17:22.208 INFO:tasks.rados.rados.0.smithi163.stdout:2792: expect deleted 2023-10-09T09:17:22.208 INFO:tasks.rados.rados.0.smithi163.stdout:2793: rollback oid 3 current snap is 353 2023-10-09T09:17:22.208 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 306 2023-10-09T09:17:22.208 INFO:tasks.rados.rados.0.smithi163.stdout:2794: write oid 44 current snap is 353 2023-10-09T09:17:22.208 INFO:tasks.rados.rados.0.smithi163.stdout:2794: seq_num 1043 ranges {729701=425068,1667072=652028,2909005=485180} 2023-10-09T09:17:22.218 INFO:tasks.rados.rados.0.smithi163.stdout:2794: writing smithi16367185-44 from 729701 to 1154769 tid 1 2023-10-09T09:17:22.225 INFO:tasks.rados.rados.0.smithi163.stdout:2794: writing smithi16367185-44 from 1667072 to 2319100 tid 2 2023-10-09T09:17:22.232 INFO:tasks.rados.rados.0.smithi163.stdout:2794: writing smithi16367185-44 from 2909005 to 3394185 tid 3 2023-10-09T09:17:22.233 INFO:tasks.rados.rados.0.smithi163.stdout:2795: delete oid 37 current snap is 353 2023-10-09T09:17:22.233 INFO:tasks.rados.rados.0.smithi163.stdout:2793: finishing rollback tid 0 to smithi16367185-3 2023-10-09T09:17:22.233 INFO:tasks.rados.rados.0.smithi163.stdout:2793: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:17:22.233 INFO:tasks.rados.rados.0.smithi163.stdout:2793: finishing rollback tid 2 to smithi16367185-3 2023-10-09T09:17:22.233 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 751 (ObjNum 789 snap 271 seq_num 789) dirty exists 2023-10-09T09:17:22.233 INFO:tasks.rados.rados.0.smithi163.stdout:2794: finishing write tid 1 to smithi16367185-44 2023-10-09T09:17:22.233 INFO:tasks.rados.rados.0.smithi163.stdout:2794: finishing write tid 2 to smithi16367185-44 2023-10-09T09:17:22.238 INFO:tasks.rados.rados.0.smithi163.stdout:2791: done (4 left) 2023-10-09T09:17:22.238 INFO:tasks.rados.rados.0.smithi163.stdout:2792: done (3 left) 2023-10-09T09:17:22.238 INFO:tasks.rados.rados.0.smithi163.stdout:2793: done (2 left) 2023-10-09T09:17:22.238 INFO:tasks.rados.rados.0.smithi163.stdout:2795: done (1 left) 2023-10-09T09:17:22.238 INFO:tasks.rados.rados.0.smithi163.stdout:2796: write oid 32 current snap is 353 2023-10-09T09:17:22.238 INFO:tasks.rados.rados.0.smithi163.stdout:2796: seq_num 1045 ranges {763315=575566,2067683=655697,3247498=315817} 2023-10-09T09:17:22.246 INFO:tasks.rados.rados.0.smithi163.stdout:2796: writing smithi16367185-32 from 763315 to 1338881 tid 1 2023-10-09T09:17:22.254 INFO:tasks.rados.rados.0.smithi163.stdout:2796: writing smithi16367185-32 from 2067683 to 2723380 tid 2 2023-10-09T09:17:22.260 INFO:tasks.rados.rados.0.smithi163.stdout:2796: writing smithi16367185-32 from 3247498 to 3563315 tid 3 2023-10-09T09:17:22.260 INFO:tasks.rados.rados.0.smithi163.stdout:2797: write oid 9 current snap is 353 2023-10-09T09:17:22.260 INFO:tasks.rados.rados.0.smithi163.stdout:2794: finishing write tid 3 to smithi16367185-44 2023-10-09T09:17:22.260 INFO:tasks.rados.rados.0.smithi163.stdout:2797: seq_num 1046 ranges {780122=650815,2126165=799355} 2023-10-09T09:17:22.268 INFO:tasks.rados.rados.0.smithi163.stdout:2797: writing smithi16367185-9 from 780122 to 1430937 tid 1 2023-10-09T09:17:22.277 INFO:tasks.rados.rados.0.smithi163.stdout:2797: writing smithi16367185-9 from 2126165 to 2925520 tid 2 2023-10-09T09:17:22.277 INFO:tasks.rados.rados.0.smithi163.stdout:2798: delete oid 19 current snap is 353 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:2794: finishing write tid 4 to smithi16367185-44 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:2794: finishing write tid 5 to smithi16367185-44 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 1992 (ObjNum 1043 snap 353 seq_num 1043) dirty exists 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:2794: left oid 44 (ObjNum 1043 snap 353 seq_num 1043) 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:2796: finishing write tid 1 to smithi16367185-32 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:2796: finishing write tid 2 to smithi16367185-32 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:2796: finishing write tid 3 to smithi16367185-32 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:2796: finishing write tid 4 to smithi16367185-32 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:2796: finishing write tid 5 to smithi16367185-32 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 1997 (ObjNum 1045 snap 353 seq_num 1045) dirty exists 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:2796: left oid 32 (ObjNum 1045 snap 353 seq_num 1045) 2023-10-09T09:17:22.278 INFO:tasks.rados.rados.0.smithi163.stdout:2797: finishing write tid 1 to smithi16367185-9 2023-10-09T09:17:22.281 INFO:tasks.rados.rados.0.smithi163.stdout:2794: done (3 left) 2023-10-09T09:17:22.281 INFO:tasks.rados.rados.0.smithi163.stdout:2796: done (2 left) 2023-10-09T09:17:22.281 INFO:tasks.rados.rados.0.smithi163.stdout:2798: done (1 left) 2023-10-09T09:17:22.281 INFO:tasks.rados.rados.0.smithi163.stdout:2799: read oid 47 snap -1 2023-10-09T09:17:22.281 INFO:tasks.rados.rados.0.smithi163.stdout:2799: expect (ObjNum 1033 snap 351 seq_num 1033) 2023-10-09T09:17:22.281 INFO:tasks.rados.rados.0.smithi163.stdout:2800: delete oid 34 current snap is 353 2023-10-09T09:17:22.282 INFO:tasks.rados.rados.0.smithi163.stdout:2797: finishing write tid 2 to smithi16367185-9 2023-10-09T09:17:22.282 INFO:tasks.rados.rados.0.smithi163.stdout:2797: finishing write tid 3 to smithi16367185-9 2023-10-09T09:17:22.283 INFO:tasks.rados.rados.0.smithi163.stdout:2797: finishing write tid 4 to smithi16367185-9 2023-10-09T09:17:22.283 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 633 (ObjNum 1046 snap 353 seq_num 1046) dirty exists 2023-10-09T09:17:22.283 INFO:tasks.rados.rados.0.smithi163.stdout:2797: left oid 9 (ObjNum 1046 snap 353 seq_num 1046) 2023-10-09T09:17:22.297 INFO:tasks.rados.rados.0.smithi163.stdout:2799: expect (ObjNum 1033 snap 351 seq_num 1033) 2023-10-09T09:17:22.466 INFO:tasks.rados.rados.0.smithi163.stdout:2797: done (2 left) 2023-10-09T09:17:22.466 INFO:tasks.rados.rados.0.smithi163.stdout:2799: done (1 left) 2023-10-09T09:17:22.466 INFO:tasks.rados.rados.0.smithi163.stdout:2800: done (0 left) 2023-10-09T09:17:22.466 INFO:tasks.rados.rados.0.smithi163.stdout:2801: snap_create 2023-10-09T09:17:22.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:22 smithi181 bash[59345]: cluster 2023-10-09T09:17:21.190142+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e768 with expected crc 2023-10-09T09:17:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:22 smithi181 bash[59345]: cluster 2023-10-09T09:17:21.190550+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e768 with expected crc 2023-10-09T09:17:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:22 smithi181 bash[59345]: cluster 2023-10-09T09:17:21.389642+0000 mgr.y (mgr.34107) 324 : cluster 0 pgmap v705: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.7 MiB/s wr, 34 op/s 2023-10-09T09:17:22.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:22 smithi181 bash[59345]: cluster 2023-10-09T09:17:22.197421+0000 mon.a (mon.0) 951 : cluster 0 osdmap e769: 8 total, 8 up, 8 in 2023-10-09T09:17:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:22 smithi163 bash[78107]: cluster 2023-10-09T09:17:21.190142+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e768 with expected crc 2023-10-09T09:17:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:22 smithi163 bash[78107]: cluster 2023-10-09T09:17:21.190550+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e768 with expected crc 2023-10-09T09:17:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:22 smithi163 bash[78107]: cluster 2023-10-09T09:17:21.389642+0000 mgr.y (mgr.34107) 324 : cluster 0 pgmap v705: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.7 MiB/s wr, 34 op/s 2023-10-09T09:17:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:22 smithi163 bash[78107]: cluster 2023-10-09T09:17:22.197421+0000 mon.a (mon.0) 951 : cluster 0 osdmap e769: 8 total, 8 up, 8 in 2023-10-09T09:17:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:22 smithi163 bash[75425]: cluster 2023-10-09T09:17:21.190142+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e768 with expected crc 2023-10-09T09:17:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:22 smithi163 bash[75425]: cluster 2023-10-09T09:17:21.190550+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e768 with expected crc 2023-10-09T09:17:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:22 smithi163 bash[75425]: cluster 2023-10-09T09:17:21.389642+0000 mgr.y (mgr.34107) 324 : cluster 0 pgmap v705: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.7 MiB/s wr, 34 op/s 2023-10-09T09:17:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:22 smithi163 bash[75425]: cluster 2023-10-09T09:17:22.197421+0000 mon.a (mon.0) 951 : cluster 0 osdmap e769: 8 total, 8 up, 8 in 2023-10-09T09:17:23.244 INFO:tasks.rados.rados.0.smithi163.stdout:2801: done (0 left) 2023-10-09T09:17:23.244 INFO:tasks.rados.rados.0.smithi163.stdout:2802: snap_remove snap 349 2023-10-09T09:17:23.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:23 smithi181 bash[59345]: cluster 2023-10-09T09:17:22.192474+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:23 smithi181 bash[59345]: cluster 2023-10-09T09:17:22.193017+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:23 smithi181 bash[59345]: cluster 2023-10-09T09:17:22.198932+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:23 smithi181 bash[59345]: cluster 2023-10-09T09:17:22.208450+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:17:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:17:23] "GET /metrics HTTP/1.1" 200 36263 "" "Prometheus/2.33.4" 2023-10-09T09:17:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:23 smithi163 bash[78107]: cluster 2023-10-09T09:17:22.192474+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:23 smithi163 bash[78107]: cluster 2023-10-09T09:17:22.193017+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:23 smithi163 bash[78107]: cluster 2023-10-09T09:17:22.198932+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:23 smithi163 bash[78107]: cluster 2023-10-09T09:17:22.208450+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:23 smithi163 bash[75425]: cluster 2023-10-09T09:17:22.192474+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:23 smithi163 bash[75425]: cluster 2023-10-09T09:17:22.193017+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:23 smithi163 bash[75425]: cluster 2023-10-09T09:17:22.198932+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:23 smithi163 bash[75425]: cluster 2023-10-09T09:17:22.208450+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e769 with expected crc 2023-10-09T09:17:24.247 INFO:tasks.rados.rados.0.smithi163.stdout:2802: done (0 left) 2023-10-09T09:17:24.247 INFO:tasks.rados.rados.0.smithi163.stdout:2803: write oid 4 current snap is 354 2023-10-09T09:17:24.247 INFO:tasks.rados.rados.0.smithi163.stdout:2803: seq_num 1049 ranges {430543=476562,1385258=663035,2840837=669165} 2023-10-09T09:17:24.254 INFO:tasks.rados.rados.0.smithi163.stdout:2803: writing smithi16367185-4 from 430543 to 907105 tid 1 2023-10-09T09:17:24.261 INFO:tasks.rados.rados.0.smithi163.stdout:2803: writing smithi16367185-4 from 1385258 to 2048293 tid 2 2023-10-09T09:17:24.271 INFO:tasks.rados.rados.0.smithi163.stdout:2803: writing smithi16367185-4 from 2840837 to 3510002 tid 3 2023-10-09T09:17:24.271 INFO:tasks.rados.rados.0.smithi163.stdout:2804: write oid 11 current snap is 354 2023-10-09T09:17:24.271 INFO:tasks.rados.rados.0.smithi163.stdout:2804: seq_num 1050 ranges {447350=668164,1560093=406693,2584613=480437} 2023-10-09T09:17:24.278 INFO:tasks.rados.rados.0.smithi163.stdout:2804: writing smithi16367185-11 from 447350 to 1115514 tid 1 2023-10-09T09:17:24.283 INFO:tasks.rados.rados.0.smithi163.stdout:2804: writing smithi16367185-11 from 1560093 to 1966786 tid 2 2023-10-09T09:17:24.289 INFO:tasks.rados.rados.0.smithi163.stdout:2804: writing smithi16367185-11 from 2584613 to 3065050 tid 3 2023-10-09T09:17:24.289 INFO:tasks.rados.rados.0.smithi163.stdout:2805: write oid 39 current snap is 354 2023-10-09T09:17:24.289 INFO:tasks.rados.rados.0.smithi163.stdout:2805: seq_num 1051 ranges {464157=743413,1618575=666704,3012036=408062} 2023-10-09T09:17:24.296 INFO:tasks.rados.rados.0.smithi163.stdout:2805: writing smithi16367185-39 from 464157 to 1207570 tid 1 2023-10-09T09:17:24.302 INFO:tasks.rados.rados.0.smithi163.stdout:2805: writing smithi16367185-39 from 1618575 to 2285279 tid 2 2023-10-09T09:17:24.309 INFO:tasks.rados.rados.0.smithi163.stdout:2805: writing smithi16367185-39 from 3012036 to 3420098 tid 3 2023-10-09T09:17:24.309 INFO:tasks.rados.rados.0.smithi163.stdout:2806: delete oid 36 current snap is 354 2023-10-09T09:17:24.309 INFO:tasks.rados.rados.0.smithi163.stdout:2803: finishing write tid 1 to smithi16367185-4 2023-10-09T09:17:24.309 INFO:tasks.rados.rados.0.smithi163.stdout:2803: finishing write tid 2 to smithi16367185-4 2023-10-09T09:17:24.309 INFO:tasks.rados.rados.0.smithi163.stdout:2803: finishing write tid 3 to smithi16367185-4 2023-10-09T09:17:24.309 INFO:tasks.rados.rados.0.smithi163.stdout:2803: finishing write tid 4 to smithi16367185-4 2023-10-09T09:17:24.309 INFO:tasks.rados.rados.0.smithi163.stdout:2803: finishing write tid 5 to smithi16367185-4 2023-10-09T09:17:24.309 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1361 (ObjNum 1049 snap 354 seq_num 1049) dirty exists 2023-10-09T09:17:24.309 INFO:tasks.rados.rados.0.smithi163.stdout:2803: left oid 4 (ObjNum 1049 snap 354 seq_num 1049) 2023-10-09T09:17:24.309 INFO:tasks.rados.rados.0.smithi163.stdout:2804: finishing write tid 1 to smithi16367185-11 2023-10-09T09:17:24.310 INFO:tasks.rados.rados.0.smithi163.stdout:2804: finishing write tid 2 to smithi16367185-11 2023-10-09T09:17:24.310 INFO:tasks.rados.rados.0.smithi163.stdout:2804: finishing write tid 3 to smithi16367185-11 2023-10-09T09:17:24.310 INFO:tasks.rados.rados.0.smithi163.stdout:2804: finishing write tid 4 to smithi16367185-11 2023-10-09T09:17:24.310 INFO:tasks.rados.rados.0.smithi163.stdout:2804: finishing write tid 5 to smithi16367185-11 2023-10-09T09:17:24.310 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 750 (ObjNum 1050 snap 354 seq_num 1050) dirty exists 2023-10-09T09:17:24.310 INFO:tasks.rados.rados.0.smithi163.stdout:2804: left oid 11 (ObjNum 1050 snap 354 seq_num 1050) 2023-10-09T09:17:24.310 INFO:tasks.rados.rados.0.smithi163.stdout:2805: finishing write tid 1 to smithi16367185-39 2023-10-09T09:17:24.310 INFO:tasks.rados.rados.0.smithi163.stdout:2805: finishing write tid 2 to smithi16367185-39 2023-10-09T09:17:24.312 INFO:tasks.rados.rados.0.smithi163.stdout:2803: done (3 left) 2023-10-09T09:17:24.312 INFO:tasks.rados.rados.0.smithi163.stdout:2804: done (2 left) 2023-10-09T09:17:24.312 INFO:tasks.rados.rados.0.smithi163.stdout:2806: done (1 left) 2023-10-09T09:17:24.313 INFO:tasks.rados.rados.0.smithi163.stdout:2807: read oid 50 snap -1 2023-10-09T09:17:24.313 INFO:tasks.rados.rados.0.smithi163.stdout:2807: expect deleted 2023-10-09T09:17:24.313 INFO:tasks.rados.rados.0.smithi163.stdout:2808: write oid 24 current snap is 354 2023-10-09T09:17:24.313 INFO:tasks.rados.rados.0.smithi163.stdout:2808: seq_num 1053 ranges {497771=493911,1619186=554020,2950529=430606} 2023-10-09T09:17:24.318 INFO:tasks.rados.rados.0.smithi163.stdout:2808: writing smithi16367185-24 from 497771 to 991682 tid 1 2023-10-09T09:17:24.325 INFO:tasks.rados.rados.0.smithi163.stdout:2808: writing smithi16367185-24 from 1619186 to 2173206 tid 2 2023-10-09T09:17:24.332 INFO:tasks.rados.rados.0.smithi163.stdout:2808: writing smithi16367185-24 from 2950529 to 3381135 tid 3 2023-10-09T09:17:24.333 INFO:tasks.rados.rados.0.smithi163.stdout:2809: write oid 33 current snap is 354 2023-10-09T09:17:24.333 INFO:tasks.rados.rados.0.smithi163.stdout:2809: seq_num 1054 ranges {514578=569160,1677668=414031,2577952=641878} 2023-10-09T09:17:24.338 INFO:tasks.rados.rados.0.smithi163.stdout:2809: writing smithi16367185-33 from 514578 to 1083738 tid 1 2023-10-09T09:17:24.344 INFO:tasks.rados.rados.0.smithi163.stdout:2809: writing smithi16367185-33 from 1677668 to 2091699 tid 2 2023-10-09T09:17:24.352 INFO:tasks.rados.rados.0.smithi163.stdout:2809: writing smithi16367185-33 from 2577952 to 3219830 tid 3 2023-10-09T09:17:24.352 INFO:tasks.rados.rados.0.smithi163.stdout:2810: read oid 8 snap -1 2023-10-09T09:17:24.352 INFO:tasks.rados.rados.0.smithi163.stdout:2810: expect deleted 2023-10-09T09:17:24.352 INFO:tasks.rados.rados.0.smithi163.stdout:2811: rollback oid 32 current snap is 354 2023-10-09T09:17:24.352 INFO:tasks.rados.rados.0.smithi163.stdout:2805: finishing write tid 3 to smithi16367185-39 2023-10-09T09:17:24.352 INFO:tasks.rados.rados.0.smithi163.stdout:2805: finishing write tid 4 to smithi16367185-39 2023-10-09T09:17:24.352 INFO:tasks.rados.rados.0.smithi163.stdout:2805: finishing write tid 5 to smithi16367185-39 2023-10-09T09:17:24.352 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1314 (ObjNum 1051 snap 354 seq_num 1051) dirty exists 2023-10-09T09:17:24.352 INFO:tasks.rados.rados.0.smithi163.stdout:2805: left oid 39 (ObjNum 1051 snap 354 seq_num 1051) 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 338 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:2805: done (5 left) 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:2807: done (4 left) 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:2812: snap_create 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:2808: finishing write tid 1 to smithi16367185-24 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:2808: finishing write tid 2 to smithi16367185-24 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:2808: finishing write tid 3 to smithi16367185-24 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:2808: finishing write tid 4 to smithi16367185-24 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:2808: finishing write tid 5 to smithi16367185-24 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 638 (ObjNum 1053 snap 354 seq_num 1053) dirty exists 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:2808: left oid 24 (ObjNum 1053 snap 354 seq_num 1053) 2023-10-09T09:17:24.353 INFO:tasks.rados.rados.0.smithi163.stdout:2809: finishing write tid 1 to smithi16367185-33 2023-10-09T09:17:24.354 INFO:tasks.rados.rados.0.smithi163.stdout:2809: finishing write tid 2 to smithi16367185-33 2023-10-09T09:17:24.356 INFO:tasks.rados.rados.0.smithi163.stdout:2809: finishing write tid 3 to smithi16367185-33 2023-10-09T09:17:24.356 INFO:tasks.rados.rados.0.smithi163.stdout:2809: finishing write tid 4 to smithi16367185-33 2023-10-09T09:17:24.357 INFO:tasks.rados.rados.0.smithi163.stdout:2809: finishing write tid 5 to smithi16367185-33 2023-10-09T09:17:24.357 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1324 (ObjNum 1054 snap 354 seq_num 1054) dirty exists 2023-10-09T09:17:24.357 INFO:tasks.rados.rados.0.smithi163.stdout:2809: left oid 33 (ObjNum 1054 snap 354 seq_num 1054) 2023-10-09T09:17:24.358 INFO:tasks.rados.rados.0.smithi163.stdout:2811: finishing rollback tid 0 to smithi16367185-32 2023-10-09T09:17:24.358 INFO:tasks.rados.rados.0.smithi163.stdout:2811: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:17:24.358 INFO:tasks.rados.rados.0.smithi163.stdout:2811: finishing rollback tid 2 to smithi16367185-32 2023-10-09T09:17:24.358 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 2007 (ObjNum 880 snap 307 seq_num 880) dirty exists 2023-10-09T09:17:24.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:24 smithi181 bash[59345]: cluster 2023-10-09T09:17:23.240207+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e770 with expected crc 2023-10-09T09:17:24.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:24 smithi181 bash[59345]: cluster 2023-10-09T09:17:23.240212+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e770 with expected crc 2023-10-09T09:17:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:24 smithi181 bash[59345]: cluster 2023-10-09T09:17:23.244140+0000 mon.a (mon.0) 952 : cluster 0 osdmap e770: 8 total, 8 up, 8 in 2023-10-09T09:17:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:24 smithi181 bash[59345]: cluster 2023-10-09T09:17:23.390314+0000 mgr.y (mgr.34107) 325 : cluster 0 pgmap v708: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 254 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2023-10-09T09:17:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:24 smithi181 bash[59345]: cluster 2023-10-09T09:17:24.247303+0000 mon.a (mon.0) 953 : cluster 0 osdmap e771: 8 total, 8 up, 8 in 2023-10-09T09:17:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:24 smithi163 bash[75425]: cluster 2023-10-09T09:17:23.240207+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e770 with expected crc 2023-10-09T09:17:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:24 smithi163 bash[75425]: cluster 2023-10-09T09:17:23.240212+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e770 with expected crc 2023-10-09T09:17:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:24 smithi163 bash[75425]: cluster 2023-10-09T09:17:23.244140+0000 mon.a (mon.0) 952 : cluster 0 osdmap e770: 8 total, 8 up, 8 in 2023-10-09T09:17:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:24 smithi163 bash[75425]: cluster 2023-10-09T09:17:23.390314+0000 mgr.y (mgr.34107) 325 : cluster 0 pgmap v708: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 254 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2023-10-09T09:17:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:24 smithi163 bash[75425]: cluster 2023-10-09T09:17:24.247303+0000 mon.a (mon.0) 953 : cluster 0 osdmap e771: 8 total, 8 up, 8 in 2023-10-09T09:17:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:24 smithi163 bash[78107]: cluster 2023-10-09T09:17:23.240207+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e770 with expected crc 2023-10-09T09:17:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:24 smithi163 bash[78107]: cluster 2023-10-09T09:17:23.240212+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e770 with expected crc 2023-10-09T09:17:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:24 smithi163 bash[78107]: cluster 2023-10-09T09:17:23.244140+0000 mon.a (mon.0) 952 : cluster 0 osdmap e770: 8 total, 8 up, 8 in 2023-10-09T09:17:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:24 smithi163 bash[78107]: cluster 2023-10-09T09:17:23.390314+0000 mgr.y (mgr.34107) 325 : cluster 0 pgmap v708: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 254 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2023-10-09T09:17:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:24 smithi163 bash[78107]: cluster 2023-10-09T09:17:24.247303+0000 mon.a (mon.0) 953 : cluster 0 osdmap e771: 8 total, 8 up, 8 in 2023-10-09T09:17:25.199 INFO:tasks.rados.rados.0.smithi163.stdout:2808: done (4 left) 2023-10-09T09:17:25.199 INFO:tasks.rados.rados.0.smithi163.stdout:2809: done (3 left) 2023-10-09T09:17:25.199 INFO:tasks.rados.rados.0.smithi163.stdout:2810: done (2 left) 2023-10-09T09:17:25.200 INFO:tasks.rados.rados.0.smithi163.stdout:2811: done (1 left) 2023-10-09T09:17:25.200 INFO:tasks.rados.rados.0.smithi163.stdout:2812: done (0 left) 2023-10-09T09:17:25.200 INFO:tasks.rados.rados.0.smithi163.stdout:2813: snap_remove snap 350 2023-10-09T09:17:25.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:25 smithi181 bash[59345]: cluster 2023-10-09T09:17:24.242407+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e771 with expected crc 2023-10-09T09:17:25.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:25 smithi181 bash[59345]: cluster 2023-10-09T09:17:24.245146+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e771 with expected crc 2023-10-09T09:17:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:25 smithi181 bash[59345]: cluster 2023-10-09T09:17:24.264449+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e770 with expected crc 2023-10-09T09:17:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:25 smithi181 bash[59345]: cluster 2023-10-09T09:17:25.199003+0000 mon.a (mon.0) 954 : cluster 0 osdmap e772: 8 total, 8 up, 8 in 2023-10-09T09:17:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:25 smithi163 bash[75425]: cluster 2023-10-09T09:17:24.242407+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e771 with expected crc 2023-10-09T09:17:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:25 smithi163 bash[75425]: cluster 2023-10-09T09:17:24.245146+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e771 with expected crc 2023-10-09T09:17:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:25 smithi163 bash[75425]: cluster 2023-10-09T09:17:24.264449+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e770 with expected crc 2023-10-09T09:17:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:25 smithi163 bash[75425]: cluster 2023-10-09T09:17:25.199003+0000 mon.a (mon.0) 954 : cluster 0 osdmap e772: 8 total, 8 up, 8 in 2023-10-09T09:17:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:25 smithi163 bash[78107]: cluster 2023-10-09T09:17:24.242407+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e771 with expected crc 2023-10-09T09:17:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:25 smithi163 bash[78107]: cluster 2023-10-09T09:17:24.245146+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e771 with expected crc 2023-10-09T09:17:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:25 smithi163 bash[78107]: cluster 2023-10-09T09:17:24.264449+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e770 with expected crc 2023-10-09T09:17:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:25 smithi163 bash[78107]: cluster 2023-10-09T09:17:25.199003+0000 mon.a (mon.0) 954 : cluster 0 osdmap e772: 8 total, 8 up, 8 in 2023-10-09T09:17:26.209 INFO:tasks.rados.rados.0.smithi163.stdout:2813: done (0 left) 2023-10-09T09:17:26.209 INFO:tasks.rados.rados.0.smithi163.stdout:2814: rollback oid 26 current snap is 355 2023-10-09T09:17:26.209 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 26 to 347 2023-10-09T09:17:26.210 INFO:tasks.rados.rados.0.smithi163.stdout:2815: delete oid 4 current snap is 355 2023-10-09T09:17:26.221 INFO:tasks.rados.rados.0.smithi163.stdout:2814: finishing rollback tid 0 to smithi16367185-26 2023-10-09T09:17:26.224 INFO:tasks.rados.rados.0.smithi163.stdout:2814: finishing rollback tid 1 to smithi16367185-26 2023-10-09T09:17:26.224 INFO:tasks.rados.rados.0.smithi163.stdout:2814: finishing rollback tid 2 to smithi16367185-26 2023-10-09T09:17:26.224 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 801 (ObjNum 1007 snap 344 seq_num 1007) dirty exists 2023-10-09T09:17:26.225 INFO:tasks.rados.rados.0.smithi163.stdout:2814: done (1 left) 2023-10-09T09:17:26.225 INFO:tasks.rados.rados.0.smithi163.stdout:2815: done (0 left) 2023-10-09T09:17:26.225 INFO:tasks.rados.rados.0.smithi163.stdout:2816: read oid 28 snap 347 2023-10-09T09:17:26.225 INFO:tasks.rados.rados.0.smithi163.stdout:2816: expect (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:17:26.225 INFO:tasks.rados.rados.0.smithi163.stdout:2817: write oid 44 current snap is 355 2023-10-09T09:17:26.225 INFO:tasks.rados.rados.0.smithi163.stdout:2817: seq_num 1056 ranges {548192=719658,1678279=417700,2516445=780775} 2023-10-09T09:17:26.233 INFO:tasks.rados.rados.0.smithi163.stdout:2817: writing smithi16367185-44 from 548192 to 1267850 tid 1 2023-10-09T09:17:26.238 INFO:tasks.rados.rados.0.smithi163.stdout:2817: writing smithi16367185-44 from 1678279 to 2095979 tid 2 2023-10-09T09:17:26.246 INFO:tasks.rados.rados.0.smithi163.stdout:2817: writing smithi16367185-44 from 2516445 to 3297220 tid 3 2023-10-09T09:17:26.246 INFO:tasks.rados.rados.0.smithi163.stdout:2818: snap_create 2023-10-09T09:17:26.246 INFO:tasks.rados.rados.0.smithi163.stdout:2816: expect (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:17:26.347 INFO:tasks.rados.rados.0.smithi163.stdout:2817: finishing write tid 1 to smithi16367185-44 2023-10-09T09:17:26.348 INFO:tasks.rados.rados.0.smithi163.stdout:2817: finishing write tid 2 to smithi16367185-44 2023-10-09T09:17:26.348 INFO:tasks.rados.rados.0.smithi163.stdout:2817: finishing write tid 3 to smithi16367185-44 2023-10-09T09:17:26.348 INFO:tasks.rados.rados.0.smithi163.stdout:2817: finishing write tid 4 to smithi16367185-44 2023-10-09T09:17:26.348 INFO:tasks.rados.rados.0.smithi163.stdout:2817: finishing write tid 5 to smithi16367185-44 2023-10-09T09:17:26.348 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 2018 (ObjNum 1056 snap 355 seq_num 1056) dirty exists 2023-10-09T09:17:26.349 INFO:tasks.rados.rados.0.smithi163.stdout:2817: left oid 44 (ObjNum 1056 snap 355 seq_num 1056) 2023-10-09T09:17:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:26 smithi181 bash[59345]: cluster 2023-10-09T09:17:25.194953+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e772 with expected crc 2023-10-09T09:17:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:26 smithi181 bash[59345]: cluster 2023-10-09T09:17:25.194953+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e772 with expected crc 2023-10-09T09:17:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:26 smithi181 bash[59345]: audit 2023-10-09T09:17:25.346584+0000 mon.a (mon.0) 955 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:26 smithi181 bash[59345]: cluster 2023-10-09T09:17:25.391091+0000 mgr.y (mgr.34107) 326 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 4 op/s 2023-10-09T09:17:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:26 smithi181 bash[59345]: cluster 2023-10-09T09:17:26.209054+0000 mon.a (mon.0) 956 : cluster 0 osdmap e773: 8 total, 8 up, 8 in 2023-10-09T09:17:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:26 smithi163 bash[78107]: cluster 2023-10-09T09:17:25.194953+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e772 with expected crc 2023-10-09T09:17:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:26 smithi163 bash[78107]: cluster 2023-10-09T09:17:25.194953+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e772 with expected crc 2023-10-09T09:17:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:26 smithi163 bash[78107]: audit 2023-10-09T09:17:25.346584+0000 mon.a (mon.0) 955 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:26 smithi163 bash[78107]: cluster 2023-10-09T09:17:25.391091+0000 mgr.y (mgr.34107) 326 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 4 op/s 2023-10-09T09:17:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:26 smithi163 bash[78107]: cluster 2023-10-09T09:17:26.209054+0000 mon.a (mon.0) 956 : cluster 0 osdmap e773: 8 total, 8 up, 8 in 2023-10-09T09:17:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:26 smithi163 bash[75425]: cluster 2023-10-09T09:17:25.194953+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e772 with expected crc 2023-10-09T09:17:26.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:26 smithi163 bash[75425]: cluster 2023-10-09T09:17:25.194953+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e772 with expected crc 2023-10-09T09:17:26.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:26 smithi163 bash[75425]: audit 2023-10-09T09:17:25.346584+0000 mon.a (mon.0) 955 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:26.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:26 smithi163 bash[75425]: cluster 2023-10-09T09:17:25.391091+0000 mgr.y (mgr.34107) 326 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 4 op/s 2023-10-09T09:17:26.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:26 smithi163 bash[75425]: cluster 2023-10-09T09:17:26.209054+0000 mon.a (mon.0) 956 : cluster 0 osdmap e773: 8 total, 8 up, 8 in 2023-10-09T09:17:27.211 INFO:tasks.rados.rados.0.smithi163.stdout:2816: done (2 left) 2023-10-09T09:17:27.211 INFO:tasks.rados.rados.0.smithi163.stdout:2817: done (1 left) 2023-10-09T09:17:27.212 INFO:tasks.rados.rados.0.smithi163.stdout:2818: done (0 left) 2023-10-09T09:17:27.212 INFO:tasks.rados.rados.0.smithi163.stdout:2819: delete oid 25 current snap is 356 2023-10-09T09:17:27.220 INFO:tasks.rados.rados.0.smithi163.stdout:2819: done (0 left) 2023-10-09T09:17:27.220 INFO:tasks.rados.rados.0.smithi163.stdout:2820: snap_create 2023-10-09T09:17:27.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:27 smithi181 bash[59345]: cluster 2023-10-09T09:17:25.629011+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e772 with expected crc 2023-10-09T09:17:27.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:27 smithi181 bash[59345]: cluster 2023-10-09T09:17:26.203528+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e773 with expected crc 2023-10-09T09:17:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:27 smithi181 bash[59345]: cluster 2023-10-09T09:17:26.206090+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e773 with expected crc 2023-10-09T09:17:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:27 smithi181 bash[59345]: cluster 2023-10-09T09:17:26.215367+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e773 with expected crc 2023-10-09T09:17:27.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:27 smithi181 bash[59345]: cluster 2023-10-09T09:17:27.211451+0000 mon.a (mon.0) 957 : cluster 0 osdmap e774: 8 total, 8 up, 8 in 2023-10-09T09:17:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:27 smithi163 bash[75425]: cluster 2023-10-09T09:17:25.629011+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e772 with expected crc 2023-10-09T09:17:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:27 smithi163 bash[75425]: cluster 2023-10-09T09:17:26.203528+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e773 with expected crc 2023-10-09T09:17:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:27 smithi163 bash[75425]: cluster 2023-10-09T09:17:26.206090+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e773 with expected crc 2023-10-09T09:17:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:27 smithi163 bash[75425]: cluster 2023-10-09T09:17:26.215367+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e773 with expected crc 2023-10-09T09:17:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:27 smithi163 bash[75425]: cluster 2023-10-09T09:17:27.211451+0000 mon.a (mon.0) 957 : cluster 0 osdmap e774: 8 total, 8 up, 8 in 2023-10-09T09:17:27.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:27 smithi163 bash[78107]: cluster 2023-10-09T09:17:25.629011+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e772 with expected crc 2023-10-09T09:17:27.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:27 smithi163 bash[78107]: cluster 2023-10-09T09:17:26.203528+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e773 with expected crc 2023-10-09T09:17:27.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:27 smithi163 bash[78107]: cluster 2023-10-09T09:17:26.206090+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e773 with expected crc 2023-10-09T09:17:27.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:27 smithi163 bash[78107]: cluster 2023-10-09T09:17:26.215367+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e773 with expected crc 2023-10-09T09:17:27.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:27 smithi163 bash[78107]: cluster 2023-10-09T09:17:27.211451+0000 mon.a (mon.0) 957 : cluster 0 osdmap e774: 8 total, 8 up, 8 in 2023-10-09T09:17:28.217 INFO:tasks.rados.rados.0.smithi163.stdout:2820: done (0 left) 2023-10-09T09:17:28.217 INFO:tasks.rados.rados.0.smithi163.stdout:2821: rollback oid 38 current snap is 357 2023-10-09T09:17:28.217 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 327 2023-10-09T09:17:28.217 INFO:tasks.rados.rados.0.smithi163.stdout:2822: read oid 16 snap 351 2023-10-09T09:17:28.217 INFO:tasks.rados.rados.0.smithi163.stdout:2822: expect (ObjNum 1017 snap 346 seq_num 1017) 2023-10-09T09:17:28.217 INFO:tasks.rados.rados.0.smithi163.stdout:2823: read oid 33 snap -1 2023-10-09T09:17:28.217 INFO:tasks.rados.rados.0.smithi163.stdout:2823: expect (ObjNum 1054 snap 354 seq_num 1054) 2023-10-09T09:17:28.218 INFO:tasks.rados.rados.0.smithi163.stdout:2824: rollback oid 34 current snap is 357 2023-10-09T09:17:28.218 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 34 to 347 2023-10-09T09:17:28.218 INFO:tasks.rados.rados.0.smithi163.stdout:2825: write oid 3 current snap is 357 2023-10-09T09:17:28.218 INFO:tasks.rados.rados.0.smithi163.stdout:2825: seq_num 1058 ranges {581806=586509,1911596=421369,3087644=403319} 2023-10-09T09:17:28.227 INFO:tasks.rados.rados.0.smithi163.stdout:2825: writing smithi16367185-3 from 581806 to 1168315 tid 1 2023-10-09T09:17:28.234 INFO:tasks.rados.rados.0.smithi163.stdout:2825: writing smithi16367185-3 from 1911596 to 2332965 tid 2 2023-10-09T09:17:28.241 INFO:tasks.rados.rados.0.smithi163.stdout:2825: writing smithi16367185-3 from 3087644 to 3490963 tid 3 2023-10-09T09:17:28.241 INFO:tasks.rados.rados.0.smithi163.stdout:2826: write oid 37 current snap is 357 2023-10-09T09:17:28.241 INFO:tasks.rados.rados.0.smithi163.stdout:2826: seq_num 1059 ranges {598613=661758,1970078=565027,3115067=614591} 2023-10-09T09:17:28.248 INFO:tasks.rados.rados.0.smithi163.stdout:2826: writing smithi16367185-37 from 598613 to 1260371 tid 1 2023-10-09T09:17:28.255 INFO:tasks.rados.rados.0.smithi163.stdout:2826: writing smithi16367185-37 from 1970078 to 2535105 tid 2 2023-10-09T09:17:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:2826: writing smithi16367185-37 from 3115067 to 3729658 tid 3 2023-10-09T09:17:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:2827: write oid 35 current snap is 357 2023-10-09T09:17:28.262 INFO:tasks.rados.rados.0.smithi163.stdout:2827: seq_num 1060 ranges {615420=737007,1912207=425038,3026137=542216} 2023-10-09T09:17:28.271 INFO:tasks.rados.rados.0.smithi163.stdout:2827: writing smithi16367185-35 from 615420 to 1352427 tid 1 2023-10-09T09:17:28.278 INFO:tasks.rados.rados.0.smithi163.stdout:2827: writing smithi16367185-35 from 1912207 to 2337245 tid 2 2023-10-09T09:17:28.287 INFO:tasks.rados.rados.0.smithi163.stdout:2827: writing smithi16367185-35 from 3026137 to 3568353 tid 3 2023-10-09T09:17:28.287 INFO:tasks.rados.rados.0.smithi163.stdout:2828: snap_create 2023-10-09T09:17:28.287 INFO:tasks.rados.rados.0.smithi163.stdout:2824: finishing rollback tid 1 to smithi16367185-34 2023-10-09T09:17:28.287 INFO:tasks.rados.rados.0.smithi163.stdout:2824: finishing rollback tid 2 to smithi16367185-34 2023-10-09T09:17:28.287 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1320 (ObjNum 969 snap 332 seq_num 969) dirty exists 2023-10-09T09:17:28.287 INFO:tasks.rados.rados.0.smithi163.stdout:2823: expect (ObjNum 1054 snap 354 seq_num 1054) 2023-10-09T09:17:28.522 INFO:tasks.rados.rados.0.smithi163.stdout:2821: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:17:28.522 INFO:tasks.rados.rados.0.smithi163.stdout:2821: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:17:28.522 INFO:tasks.rados.rados.0.smithi163.stdout:2821: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:17:28.523 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1332 (ObjNum 941 snap 322 seq_num 941) dirty exists 2023-10-09T09:17:28.523 INFO:tasks.rados.rados.0.smithi163.stdout:2822: expect (ObjNum 1017 snap 346 seq_num 1017) 2023-10-09T09:17:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:28 smithi181 bash[59345]: cluster 2023-10-09T09:17:27.206001+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:28 smithi181 bash[59345]: cluster 2023-10-09T09:17:27.209149+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:28 smithi181 bash[59345]: cluster 2023-10-09T09:17:27.391719+0000 mgr.y (mgr.34107) 327 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.2 MiB/s wr, 16 op/s 2023-10-09T09:17:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:28 smithi181 bash[59345]: cluster 2023-10-09T09:17:28.216474+0000 mon.a (mon.0) 958 : cluster 0 osdmap e775: 8 total, 8 up, 8 in 2023-10-09T09:17:28.626 INFO:tasks.rados.rados.0.smithi163.stdout:2825: finishing write tid 1 to smithi16367185-3 2023-10-09T09:17:28.626 INFO:tasks.rados.rados.0.smithi163.stdout:2825: finishing write tid 2 to smithi16367185-3 2023-10-09T09:17:28.626 INFO:tasks.rados.rados.0.smithi163.stdout:2825: finishing write tid 3 to smithi16367185-3 2023-10-09T09:17:28.626 INFO:tasks.rados.rados.0.smithi163.stdout:2825: finishing write tid 4 to smithi16367185-3 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:2825: finishing write tid 5 to smithi16367185-3 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 764 (ObjNum 1058 snap 357 seq_num 1058) dirty exists 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:2825: left oid 3 (ObjNum 1058 snap 357 seq_num 1058) 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:2826: finishing write tid 1 to smithi16367185-37 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:2826: finishing write tid 2 to smithi16367185-37 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:2826: finishing write tid 3 to smithi16367185-37 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:2826: finishing write tid 4 to smithi16367185-37 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:2826: finishing write tid 5 to smithi16367185-37 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 1429 (ObjNum 1059 snap 357 seq_num 1059) dirty exists 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:2826: left oid 37 (ObjNum 1059 snap 357 seq_num 1059) 2023-10-09T09:17:28.627 INFO:tasks.rados.rados.0.smithi163.stdout:2827: finishing write tid 1 to smithi16367185-35 2023-10-09T09:17:28.628 INFO:tasks.rados.rados.0.smithi163.stdout:2827: finishing write tid 2 to smithi16367185-35 2023-10-09T09:17:28.628 INFO:tasks.rados.rados.0.smithi163.stdout:2827: finishing write tid 3 to smithi16367185-35 2023-10-09T09:17:28.628 INFO:tasks.rados.rados.0.smithi163.stdout:2827: finishing write tid 4 to smithi16367185-35 2023-10-09T09:17:28.628 INFO:tasks.rados.rados.0.smithi163.stdout:2827: finishing write tid 5 to smithi16367185-35 2023-10-09T09:17:28.628 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 757 (ObjNum 1060 snap 357 seq_num 1060) dirty exists 2023-10-09T09:17:28.628 INFO:tasks.rados.rados.0.smithi163.stdout:2827: left oid 35 (ObjNum 1060 snap 357 seq_num 1060) 2023-10-09T09:17:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:28 smithi163 bash[78107]: cluster 2023-10-09T09:17:27.206001+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:28 smithi163 bash[78107]: cluster 2023-10-09T09:17:27.209149+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:28 smithi163 bash[78107]: cluster 2023-10-09T09:17:27.391719+0000 mgr.y (mgr.34107) 327 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.2 MiB/s wr, 16 op/s 2023-10-09T09:17:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:28 smithi163 bash[78107]: cluster 2023-10-09T09:17:28.216474+0000 mon.a (mon.0) 958 : cluster 0 osdmap e775: 8 total, 8 up, 8 in 2023-10-09T09:17:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:28 smithi163 bash[75425]: cluster 2023-10-09T09:17:27.206001+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:28 smithi163 bash[75425]: cluster 2023-10-09T09:17:27.209149+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:28 smithi163 bash[75425]: cluster 2023-10-09T09:17:27.391719+0000 mgr.y (mgr.34107) 327 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.2 MiB/s wr, 16 op/s 2023-10-09T09:17:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:28 smithi163 bash[75425]: cluster 2023-10-09T09:17:28.216474+0000 mon.a (mon.0) 958 : cluster 0 osdmap e775: 8 total, 8 up, 8 in 2023-10-09T09:17:29.289 INFO:tasks.rados.rados.0.smithi163.stdout:2821: done (7 left) 2023-10-09T09:17:29.289 INFO:tasks.rados.rados.0.smithi163.stdout:2822: done (6 left) 2023-10-09T09:17:29.289 INFO:tasks.rados.rados.0.smithi163.stdout:2823: done (5 left) 2023-10-09T09:17:29.289 INFO:tasks.rados.rados.0.smithi163.stdout:2824: done (4 left) 2023-10-09T09:17:29.289 INFO:tasks.rados.rados.0.smithi163.stdout:2825: done (3 left) 2023-10-09T09:17:29.289 INFO:tasks.rados.rados.0.smithi163.stdout:2826: done (2 left) 2023-10-09T09:17:29.289 INFO:tasks.rados.rados.0.smithi163.stdout:2827: done (1 left) 2023-10-09T09:17:29.289 INFO:tasks.rados.rados.0.smithi163.stdout:2828: done (0 left) 2023-10-09T09:17:29.290 INFO:tasks.rados.rados.0.smithi163.stdout:2829: read oid 45 snap -1 2023-10-09T09:17:29.290 INFO:tasks.rados.rados.0.smithi163.stdout:2829: expect (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:17:29.290 INFO:tasks.rados.rados.0.smithi163.stdout:2830: write oid 8 current snap is 358 2023-10-09T09:17:29.290 INFO:tasks.rados.rados.0.smithi163.stdout:2830: seq_num 1061 ranges {632227=412256,1570689=568696,2653560=753488} 2023-10-09T09:17:29.298 INFO:tasks.rados.rados.0.smithi163.stdout:2830: writing smithi16367185-8 from 632227 to 1044483 tid 1 2023-10-09T09:17:29.305 INFO:tasks.rados.rados.0.smithi163.stdout:2830: writing smithi16367185-8 from 1570689 to 2139385 tid 2 2023-10-09T09:17:29.312 INFO:tasks.rados.rados.0.smithi163.stdout:2830: writing smithi16367185-8 from 2653560 to 3407048 tid 3 2023-10-09T09:17:29.312 INFO:tasks.rados.rados.0.smithi163.stdout:2831: read oid 39 snap 338 2023-10-09T09:17:29.312 INFO:tasks.rados.rados.0.smithi163.stdout:2831: expect (ObjNum 932 snap 317 seq_num 932) 2023-10-09T09:17:29.313 INFO:tasks.rados.rados.0.smithi163.stdout:2832: rollback oid 40 current snap is 358 2023-10-09T09:17:29.313 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 40 to 353 2023-10-09T09:17:29.313 INFO:tasks.rados.rados.0.smithi163.stdout:2829: expect (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:17:29.400 INFO:tasks.rados.rados.0.smithi163.stdout:2830: finishing write tid 1 to smithi16367185-8 2023-10-09T09:17:29.400 INFO:tasks.rados.rados.0.smithi163.stdout:2830: finishing write tid 2 to smithi16367185-8 2023-10-09T09:17:29.401 INFO:tasks.rados.rados.0.smithi163.stdout:2830: finishing write tid 3 to smithi16367185-8 2023-10-09T09:17:29.401 INFO:tasks.rados.rados.0.smithi163.stdout:2830: finishing write tid 4 to smithi16367185-8 2023-10-09T09:17:29.401 INFO:tasks.rados.rados.0.smithi163.stdout:2830: finishing write tid 5 to smithi16367185-8 2023-10-09T09:17:29.401 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1433 (ObjNum 1061 snap 358 seq_num 1061) dirty exists 2023-10-09T09:17:29.401 INFO:tasks.rados.rados.0.smithi163.stdout:2830: left oid 8 (ObjNum 1061 snap 358 seq_num 1061) 2023-10-09T09:17:29.401 INFO:tasks.rados.rados.0.smithi163.stdout:2831: expect (ObjNum 932 snap 317 seq_num 932) 2023-10-09T09:17:29.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:29 smithi181 bash[59345]: cluster 2023-10-09T09:17:27.518360+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:29.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:29 smithi181 bash[59345]: cluster 2023-10-09T09:17:28.050942+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:29 smithi181 bash[59345]: cluster 2023-10-09T09:17:28.211389+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e775 with expected crc 2023-10-09T09:17:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:29 smithi181 bash[59345]: cluster 2023-10-09T09:17:28.217392+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e775 with expected crc 2023-10-09T09:17:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:29 smithi181 bash[59345]: cluster 2023-10-09T09:17:28.250380+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e775 with expected crc 2023-10-09T09:17:29.577 INFO:tasks.rados.rados.0.smithi163.stdout:2832: finishing rollback tid 0 to smithi16367185-40 2023-10-09T09:17:29.577 INFO:tasks.rados.rados.0.smithi163.stdout:2832: finishing rollback tid 1 to smithi16367185-40 2023-10-09T09:17:29.577 INFO:tasks.rados.rados.0.smithi163.stdout:2832: finishing rollback tid 2 to smithi16367185-40 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1437 (ObjNum 1027 snap 349 seq_num 1027) dirty exists 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:2829: done (3 left) 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:2830: done (2 left) 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:2831: done (1 left) 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:2832: done (0 left) 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:2833: read oid 14 snap -1 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:2833: expect (ObjNum 1041 snap 352 seq_num 1041) 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:2834: read oid 44 snap -1 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:2834: expect (ObjNum 1056 snap 355 seq_num 1056) 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:2835: rollback oid 34 current snap is 358 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 34 to 352 2023-10-09T09:17:29.578 INFO:tasks.rados.rados.0.smithi163.stdout:2836: read oid 43 snap 352 2023-10-09T09:17:29.579 INFO:tasks.rados.rados.0.smithi163.stdout:2836: expect deleted 2023-10-09T09:17:29.579 INFO:tasks.rados.rados.0.smithi163.stdout:2837: read oid 11 snap -1 2023-10-09T09:17:29.579 INFO:tasks.rados.rados.0.smithi163.stdout:2837: expect (ObjNum 1050 snap 354 seq_num 1050) 2023-10-09T09:17:29.579 INFO:tasks.rados.rados.0.smithi163.stdout:2838: snap_remove snap 347 2023-10-09T09:17:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:29 smithi163 bash[75425]: cluster 2023-10-09T09:17:27.518360+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:29 smithi163 bash[75425]: cluster 2023-10-09T09:17:28.050942+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:29 smithi163 bash[75425]: cluster 2023-10-09T09:17:28.211389+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e775 with expected crc 2023-10-09T09:17:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:29 smithi163 bash[75425]: cluster 2023-10-09T09:17:28.217392+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e775 with expected crc 2023-10-09T09:17:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:29 smithi163 bash[75425]: cluster 2023-10-09T09:17:28.250380+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e775 with expected crc 2023-10-09T09:17:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:29 smithi163 bash[78107]: cluster 2023-10-09T09:17:27.518360+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:29 smithi163 bash[78107]: cluster 2023-10-09T09:17:28.050942+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e774 with expected crc 2023-10-09T09:17:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:29 smithi163 bash[78107]: cluster 2023-10-09T09:17:28.211389+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e775 with expected crc 2023-10-09T09:17:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:29 smithi163 bash[78107]: cluster 2023-10-09T09:17:28.217392+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e775 with expected crc 2023-10-09T09:17:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:29 smithi163 bash[78107]: cluster 2023-10-09T09:17:28.250380+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e775 with expected crc 2023-10-09T09:17:30.199 INFO:tasks.rados.rados.0.smithi163.stdout:2838: done (5 left) 2023-10-09T09:17:30.199 INFO:tasks.rados.rados.0.smithi163.stdout:2839: snap_remove snap 357 2023-10-09T09:17:30.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:30 smithi181 bash[59345]: cluster 2023-10-09T09:17:29.283397+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e776 with expected crc 2023-10-09T09:17:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:30 smithi181 bash[59345]: cluster 2023-10-09T09:17:29.283997+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e776 with expected crc 2023-10-09T09:17:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:30 smithi181 bash[59345]: cluster 2023-10-09T09:17:29.288280+0000 mon.a (mon.0) 959 : cluster 0 osdmap e776: 8 total, 8 up, 8 in 2023-10-09T09:17:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:30 smithi181 bash[59345]: cluster 2023-10-09T09:17:29.301443+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e776 with expected crc 2023-10-09T09:17:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:30 smithi181 bash[59345]: cluster 2023-10-09T09:17:29.392384+0000 mgr.y (mgr.34107) 328 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 272 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2023-10-09T09:17:30.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:30 smithi181 bash[59345]: cluster 2023-10-09T09:17:30.198616+0000 mon.a (mon.0) 960 : cluster 0 osdmap e777: 8 total, 8 up, 8 in 2023-10-09T09:17:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[78107]: cluster 2023-10-09T09:17:29.283397+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e776 with expected crc 2023-10-09T09:17:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[78107]: cluster 2023-10-09T09:17:29.283997+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e776 with expected crc 2023-10-09T09:17:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[78107]: cluster 2023-10-09T09:17:29.288280+0000 mon.a (mon.0) 959 : cluster 0 osdmap e776: 8 total, 8 up, 8 in 2023-10-09T09:17:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[78107]: cluster 2023-10-09T09:17:29.301443+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e776 with expected crc 2023-10-09T09:17:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[78107]: cluster 2023-10-09T09:17:29.392384+0000 mgr.y (mgr.34107) 328 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 272 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2023-10-09T09:17:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[78107]: cluster 2023-10-09T09:17:30.198616+0000 mon.a (mon.0) 960 : cluster 0 osdmap e777: 8 total, 8 up, 8 in 2023-10-09T09:17:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[75425]: cluster 2023-10-09T09:17:29.283397+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e776 with expected crc 2023-10-09T09:17:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[75425]: cluster 2023-10-09T09:17:29.283997+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e776 with expected crc 2023-10-09T09:17:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[75425]: cluster 2023-10-09T09:17:29.288280+0000 mon.a (mon.0) 959 : cluster 0 osdmap e776: 8 total, 8 up, 8 in 2023-10-09T09:17:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[75425]: cluster 2023-10-09T09:17:29.301443+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e776 with expected crc 2023-10-09T09:17:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[75425]: cluster 2023-10-09T09:17:29.392384+0000 mgr.y (mgr.34107) 328 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 272 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2023-10-09T09:17:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:30 smithi163 bash[75425]: cluster 2023-10-09T09:17:30.198616+0000 mon.a (mon.0) 960 : cluster 0 osdmap e777: 8 total, 8 up, 8 in 2023-10-09T09:17:31.202 INFO:tasks.rados.rados.0.smithi163.stdout:2839: done (5 left) 2023-10-09T09:17:31.202 INFO:tasks.rados.rados.0.smithi163.stdout:2840: read oid 16 snap -1 2023-10-09T09:17:31.202 INFO:tasks.rados.rados.0.smithi163.stdout:2840: expect (ObjNum 1017 snap 346 seq_num 1017) 2023-10-09T09:17:31.202 INFO:tasks.rados.rados.0.smithi163.stdout:2841: write oid 7 current snap is 358 2023-10-09T09:17:31.202 INFO:tasks.rados.rados.0.smithi163.stdout:2841: seq_num 1062 ranges {649034=487505,1629171=428707,2680983=564760,3798768=50266} 2023-10-09T09:17:31.211 INFO:tasks.rados.rados.0.smithi163.stdout:2841: writing smithi16367185-7 from 649034 to 1136539 tid 1 2023-10-09T09:17:31.217 INFO:tasks.rados.rados.0.smithi163.stdout:2841: writing smithi16367185-7 from 1629171 to 2057878 tid 2 2023-10-09T09:17:31.224 INFO:tasks.rados.rados.0.smithi163.stdout:2841: writing smithi16367185-7 from 2680983 to 3245743 tid 3 2023-10-09T09:17:31.228 INFO:tasks.rados.rados.0.smithi163.stdout:2841: writing smithi16367185-7 from 3798768 to 3849034 tid 4 2023-10-09T09:17:31.228 INFO:tasks.rados.rados.0.smithi163.stdout:2842: snap_create 2023-10-09T09:17:31.228 INFO:tasks.rados.rados.0.smithi163.stdout:2834: expect (ObjNum 1056 snap 355 seq_num 1056) 2023-10-09T09:17:31.409 INFO:tasks.rados.rados.0.smithi163.stdout:2835: finishing rollback tid 0 to smithi16367185-34 2023-10-09T09:17:31.409 INFO:tasks.rados.rados.0.smithi163.stdout:2835: finishing rollback tid 1 to smithi16367185-34 2023-10-09T09:17:31.409 INFO:tasks.rados.rados.0.smithi163.stdout:2835: finishing rollback tid 2 to smithi16367185-34 2023-10-09T09:17:31.409 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1324 (ObjNum 969 snap 332 seq_num 969) dirty exists 2023-10-09T09:17:31.410 INFO:tasks.rados.rados.0.smithi163.stdout:2833: expect (ObjNum 1041 snap 352 seq_num 1041) 2023-10-09T09:17:31.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:31 smithi181 bash[59345]: cluster 2023-10-09T09:17:30.194652+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e777 with expected crc 2023-10-09T09:17:31.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:31 smithi181 bash[59345]: cluster 2023-10-09T09:17:30.196595+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e777 with expected crc 2023-10-09T09:17:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:31 smithi181 bash[59345]: cluster 2023-10-09T09:17:30.401049+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e777 with expected crc 2023-10-09T09:17:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:31 smithi181 bash[59345]: cluster 2023-10-09T09:17:31.201662+0000 mon.a (mon.0) 961 : cluster 0 osdmap e778: 8 total, 8 up, 8 in 2023-10-09T09:17:31.574 INFO:tasks.rados.rados.0.smithi163.stdout:2837: expect (ObjNum 1050 snap 354 seq_num 1050) 2023-10-09T09:17:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:31 smithi163 bash[78107]: cluster 2023-10-09T09:17:30.194652+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e777 with expected crc 2023-10-09T09:17:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:31 smithi163 bash[78107]: cluster 2023-10-09T09:17:30.196595+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e777 with expected crc 2023-10-09T09:17:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:31 smithi163 bash[78107]: cluster 2023-10-09T09:17:30.401049+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e777 with expected crc 2023-10-09T09:17:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:31 smithi163 bash[78107]: cluster 2023-10-09T09:17:31.201662+0000 mon.a (mon.0) 961 : cluster 0 osdmap e778: 8 total, 8 up, 8 in 2023-10-09T09:17:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:31 smithi163 bash[75425]: cluster 2023-10-09T09:17:30.194652+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e777 with expected crc 2023-10-09T09:17:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:31 smithi163 bash[75425]: cluster 2023-10-09T09:17:30.196595+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e777 with expected crc 2023-10-09T09:17:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:31 smithi163 bash[75425]: cluster 2023-10-09T09:17:30.401049+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e777 with expected crc 2023-10-09T09:17:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:31 smithi163 bash[75425]: cluster 2023-10-09T09:17:31.201662+0000 mon.a (mon.0) 961 : cluster 0 osdmap e778: 8 total, 8 up, 8 in 2023-10-09T09:17:31.772 INFO:tasks.rados.rados.0.smithi163.stdout:2840: expect (ObjNum 1017 snap 346 seq_num 1017) 2023-10-09T09:17:31.868 INFO:tasks.rados.rados.0.smithi163.stdout:2841: finishing write tid 1 to smithi16367185-7 2023-10-09T09:17:31.868 INFO:tasks.rados.rados.0.smithi163.stdout:2841: finishing write tid 2 to smithi16367185-7 2023-10-09T09:17:31.868 INFO:tasks.rados.rados.0.smithi163.stdout:2841: finishing write tid 3 to smithi16367185-7 2023-10-09T09:17:31.868 INFO:tasks.rados.rados.0.smithi163.stdout:2841: finishing write tid 4 to smithi16367185-7 2023-10-09T09:17:31.869 INFO:tasks.rados.rados.0.smithi163.stdout:2841: finishing write tid 5 to smithi16367185-7 2023-10-09T09:17:31.869 INFO:tasks.rados.rados.0.smithi163.stdout:2841: finishing write tid 6 to smithi16367185-7 2023-10-09T09:17:31.869 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 684 (ObjNum 1062 snap 358 seq_num 1062) dirty exists 2023-10-09T09:17:31.869 INFO:tasks.rados.rados.0.smithi163.stdout:2841: left oid 7 (ObjNum 1062 snap 358 seq_num 1062) 2023-10-09T09:17:32.205 INFO:tasks.rados.rados.0.smithi163.stdout:2833: done (7 left) 2023-10-09T09:17:32.205 INFO:tasks.rados.rados.0.smithi163.stdout:2834: done (6 left) 2023-10-09T09:17:32.205 INFO:tasks.rados.rados.0.smithi163.stdout:2835: done (5 left) 2023-10-09T09:17:32.205 INFO:tasks.rados.rados.0.smithi163.stdout:2836: done (4 left) 2023-10-09T09:17:32.205 INFO:tasks.rados.rados.0.smithi163.stdout:2837: done (3 left) 2023-10-09T09:17:32.206 INFO:tasks.rados.rados.0.smithi163.stdout:2840: done (2 left) 2023-10-09T09:17:32.206 INFO:tasks.rados.rados.0.smithi163.stdout:2841: done (1 left) 2023-10-09T09:17:32.206 INFO:tasks.rados.rados.0.smithi163.stdout:2842: done (0 left) 2023-10-09T09:17:32.206 INFO:tasks.rados.rados.0.smithi163.stdout:2843: write oid 12 current snap is 359 2023-10-09T09:17:32.206 INFO:tasks.rados.rados.0.smithi163.stdout:2843: seq_num 1063 ranges {665841=562754,1687653=572365,2708406=776032} 2023-10-09T09:17:32.216 INFO:tasks.rados.rados.0.smithi163.stdout:2843: writing smithi16367185-12 from 665841 to 1228595 tid 1 2023-10-09T09:17:32.222 INFO:tasks.rados.rados.0.smithi163.stdout:2843: writing smithi16367185-12 from 1687653 to 2260018 tid 2 2023-10-09T09:17:32.230 INFO:tasks.rados.rados.0.smithi163.stdout:2843: writing smithi16367185-12 from 2708406 to 3484438 tid 3 2023-10-09T09:17:32.231 INFO:tasks.rados.rados.0.smithi163.stdout:2844: rollback oid 13 current snap is 359 2023-10-09T09:17:32.231 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 13 to 306 2023-10-09T09:17:32.231 INFO:tasks.rados.rados.0.smithi163.stdout:2843: finishing write tid 1 to smithi16367185-12 2023-10-09T09:17:32.231 INFO:tasks.rados.rados.0.smithi163.stdout:2843: finishing write tid 2 to smithi16367185-12 2023-10-09T09:17:32.231 INFO:tasks.rados.rados.0.smithi163.stdout:2845: delete oid 41 current snap is 359 2023-10-09T09:17:32.234 INFO:tasks.rados.rados.0.smithi163.stdout:2845: done (2 left) 2023-10-09T09:17:32.234 INFO:tasks.rados.rados.0.smithi163.stdout:2846: snap_create 2023-10-09T09:17:32.238 INFO:tasks.rados.rados.0.smithi163.stdout:2843: finishing write tid 3 to smithi16367185-12 2023-10-09T09:17:32.238 INFO:tasks.rados.rados.0.smithi163.stdout:2844: finishing rollback tid 0 to smithi16367185-13 2023-10-09T09:17:32.238 INFO:tasks.rados.rados.0.smithi163.stdout:2843: finishing write tid 4 to smithi16367185-12 2023-10-09T09:17:32.238 INFO:tasks.rados.rados.0.smithi163.stdout:2844: finishing rollback tid 1 to smithi16367185-13 2023-10-09T09:17:32.238 INFO:tasks.rados.rados.0.smithi163.stdout:2843: finishing write tid 5 to smithi16367185-12 2023-10-09T09:17:32.238 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 1339 (ObjNum 1063 snap 359 seq_num 1063) dirty exists 2023-10-09T09:17:32.238 INFO:tasks.rados.rados.0.smithi163.stdout:2843: left oid 12 (ObjNum 1063 snap 359 seq_num 1063) 2023-10-09T09:17:32.238 INFO:tasks.rados.rados.0.smithi163.stdout:2844: finishing rollback tid 2 to smithi16367185-13 2023-10-09T09:17:32.238 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 482 (ObjNum 688 snap 219 seq_num 688) dirty exists 2023-10-09T09:17:32.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:32 smithi181 bash[59345]: cluster 2023-10-09T09:17:31.197525+0000 osd.5 (osd.5) 310 : cluster 3 failed to encode map e778 with expected crc 2023-10-09T09:17:32.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:32 smithi181 bash[59345]: cluster 2023-10-09T09:17:31.393467+0000 mgr.y (mgr.34107) 329 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2023-10-09T09:17:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:32 smithi181 bash[59345]: cluster 2023-10-09T09:17:31.437922+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e778 with expected crc 2023-10-09T09:17:32.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:32 smithi181 bash[59345]: cluster 2023-10-09T09:17:32.204946+0000 mon.a (mon.0) 962 : cluster 0 osdmap e779: 8 total, 8 up, 8 in 2023-10-09T09:17:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:32 smithi163 bash[78107]: cluster 2023-10-09T09:17:31.197525+0000 osd.5 (osd.5) 310 : cluster 3 failed to encode map e778 with expected crc 2023-10-09T09:17:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:32 smithi163 bash[78107]: cluster 2023-10-09T09:17:31.393467+0000 mgr.y (mgr.34107) 329 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2023-10-09T09:17:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:32 smithi163 bash[78107]: cluster 2023-10-09T09:17:31.437922+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e778 with expected crc 2023-10-09T09:17:32.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:32 smithi163 bash[78107]: cluster 2023-10-09T09:17:32.204946+0000 mon.a (mon.0) 962 : cluster 0 osdmap e779: 8 total, 8 up, 8 in 2023-10-09T09:17:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:32 smithi163 bash[75425]: cluster 2023-10-09T09:17:31.197525+0000 osd.5 (osd.5) 310 : cluster 3 failed to encode map e778 with expected crc 2023-10-09T09:17:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:32 smithi163 bash[75425]: cluster 2023-10-09T09:17:31.393467+0000 mgr.y (mgr.34107) 329 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2023-10-09T09:17:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:32 smithi163 bash[75425]: cluster 2023-10-09T09:17:31.437922+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e778 with expected crc 2023-10-09T09:17:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:32 smithi163 bash[75425]: cluster 2023-10-09T09:17:32.204946+0000 mon.a (mon.0) 962 : cluster 0 osdmap e779: 8 total, 8 up, 8 in 2023-10-09T09:17:33.208 INFO:tasks.rados.rados.0.smithi163.stdout:2843: done (2 left) 2023-10-09T09:17:33.209 INFO:tasks.rados.rados.0.smithi163.stdout:2844: done (1 left) 2023-10-09T09:17:33.209 INFO:tasks.rados.rados.0.smithi163.stdout:2846: done (0 left) 2023-10-09T09:17:33.209 INFO:tasks.rados.rados.0.smithi163.stdout:2847: read oid 48 snap -1 2023-10-09T09:17:33.209 INFO:tasks.rados.rados.0.smithi163.stdout:2847: expect (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:17:33.209 INFO:tasks.rados.rados.0.smithi163.stdout:2848: delete oid 23 current snap is 360 2023-10-09T09:17:33.220 INFO:tasks.rados.rados.0.smithi163.stdout:2847: expect (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:17:33.396 INFO:tasks.rados.rados.0.smithi163.stdout:2847: done (1 left) 2023-10-09T09:17:33.396 INFO:tasks.rados.rados.0.smithi163.stdout:2848: done (0 left) 2023-10-09T09:17:33.396 INFO:tasks.rados.rados.0.smithi163.stdout:2849: read oid 47 snap -1 2023-10-09T09:17:33.396 INFO:tasks.rados.rados.0.smithi163.stdout:2849: expect (ObjNum 1033 snap 351 seq_num 1033) 2023-10-09T09:17:33.396 INFO:tasks.rados.rados.0.smithi163.stdout:2850: read oid 2 snap 338 2023-10-09T09:17:33.397 INFO:tasks.rados.rados.0.smithi163.stdout:2850: expect deleted 2023-10-09T09:17:33.397 INFO:tasks.rados.rados.0.smithi163.stdout:2851: snap_remove snap 331 2023-10-09T09:17:33.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:33 smithi181 bash[59345]: cluster 2023-10-09T09:17:32.200042+0000 osd.5 (osd.5) 311 : cluster 3 failed to encode map e779 with expected crc 2023-10-09T09:17:33.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:33 smithi181 bash[59345]: cluster 2023-10-09T09:17:32.200543+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e779 with expected crc 2023-10-09T09:17:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:33 smithi181 bash[59345]: cluster 2023-10-09T09:17:33.208315+0000 mon.a (mon.0) 963 : cluster 0 osdmap e780: 8 total, 8 up, 8 in 2023-10-09T09:17:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:17:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:17:33] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.33.4" 2023-10-09T09:17:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:33 smithi163 bash[78107]: cluster 2023-10-09T09:17:32.200042+0000 osd.5 (osd.5) 311 : cluster 3 failed to encode map e779 with expected crc 2023-10-09T09:17:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:33 smithi163 bash[78107]: cluster 2023-10-09T09:17:32.200543+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e779 with expected crc 2023-10-09T09:17:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:33 smithi163 bash[78107]: cluster 2023-10-09T09:17:33.208315+0000 mon.a (mon.0) 963 : cluster 0 osdmap e780: 8 total, 8 up, 8 in 2023-10-09T09:17:33.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:33 smithi163 bash[75425]: cluster 2023-10-09T09:17:32.200042+0000 osd.5 (osd.5) 311 : cluster 3 failed to encode map e779 with expected crc 2023-10-09T09:17:33.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:33 smithi163 bash[75425]: cluster 2023-10-09T09:17:32.200543+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e779 with expected crc 2023-10-09T09:17:33.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:33 smithi163 bash[75425]: cluster 2023-10-09T09:17:33.208315+0000 mon.a (mon.0) 963 : cluster 0 osdmap e780: 8 total, 8 up, 8 in 2023-10-09T09:17:34.329 INFO:tasks.rados.rados.0.smithi163.stdout:2851: done (2 left) 2023-10-09T09:17:34.329 INFO:tasks.rados.rados.0.smithi163.stdout:2852: read oid 25 snap -1 2023-10-09T09:17:34.329 INFO:tasks.rados.rados.0.smithi163.stdout:2852: expect deleted 2023-10-09T09:17:34.329 INFO:tasks.rados.rados.0.smithi163.stdout:2853: write oid 35 current snap is 360 2023-10-09T09:17:34.329 INFO:tasks.rados.rados.0.smithi163.stdout:2853: seq_num 1066 ranges {716262=504854,1863099=719692,3074322=726201} 2023-10-09T09:17:34.339 INFO:tasks.rados.rados.0.smithi163.stdout:2853: writing smithi16367185-35 from 716262 to 1221116 tid 1 2023-10-09T09:17:34.348 INFO:tasks.rados.rados.0.smithi163.stdout:2853: writing smithi16367185-35 from 1863099 to 2582791 tid 2 2023-10-09T09:17:34.356 INFO:tasks.rados.rados.0.smithi163.stdout:2853: writing smithi16367185-35 from 3074322 to 3800523 tid 3 2023-10-09T09:17:34.356 INFO:tasks.rados.rados.0.smithi163.stdout:2854: read oid 1 snap -1 2023-10-09T09:17:34.356 INFO:tasks.rados.rados.0.smithi163.stdout:2854: expect (ObjNum 983 snap 338 seq_num 983) 2023-10-09T09:17:34.356 INFO:tasks.rados.rados.0.smithi163.stdout:2855: snap_remove snap 306 2023-10-09T09:17:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:34 smithi163 bash[75425]: cluster 2023-10-09T09:17:33.202589+0000 osd.5 (osd.5) 312 : cluster 3 failed to encode map e780 with expected crc 2023-10-09T09:17:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:34 smithi163 bash[75425]: cluster 2023-10-09T09:17:33.394137+0000 mgr.y (mgr.34107) 330 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 276 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2023-10-09T09:17:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:34 smithi163 bash[75425]: cluster 2023-10-09T09:17:33.397573+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e780 with expected crc 2023-10-09T09:17:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:34 smithi163 bash[78107]: cluster 2023-10-09T09:17:33.202589+0000 osd.5 (osd.5) 312 : cluster 3 failed to encode map e780 with expected crc 2023-10-09T09:17:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:34 smithi163 bash[78107]: cluster 2023-10-09T09:17:33.394137+0000 mgr.y (mgr.34107) 330 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 276 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2023-10-09T09:17:34.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:34 smithi163 bash[78107]: cluster 2023-10-09T09:17:33.397573+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e780 with expected crc 2023-10-09T09:17:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:34 smithi181 bash[59345]: cluster 2023-10-09T09:17:33.202589+0000 osd.5 (osd.5) 312 : cluster 3 failed to encode map e780 with expected crc 2023-10-09T09:17:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:34 smithi181 bash[59345]: cluster 2023-10-09T09:17:33.394137+0000 mgr.y (mgr.34107) 330 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 276 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2023-10-09T09:17:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:34 smithi181 bash[59345]: cluster 2023-10-09T09:17:33.397573+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e780 with expected crc 2023-10-09T09:17:35.202 INFO:tasks.rados.rados.0.smithi163.stdout:2855: done (5 left) 2023-10-09T09:17:35.202 INFO:tasks.rados.rados.0.smithi163.stdout:2856: write oid 7 current snap is 360 2023-10-09T09:17:35.202 INFO:tasks.rados.rados.0.smithi163.stdout:2856: seq_num 1067 ranges {733069=580103,1921581=579703,3218098=537473} 2023-10-09T09:17:35.210 INFO:tasks.rados.rados.0.smithi163.stdout:2856: writing smithi16367185-7 from 733069 to 1313172 tid 1 2023-10-09T09:17:35.217 INFO:tasks.rados.rados.0.smithi163.stdout:2856: writing smithi16367185-7 from 1921581 to 2501284 tid 2 2023-10-09T09:17:35.225 INFO:tasks.rados.rados.0.smithi163.stdout:2856: writing smithi16367185-7 from 3218098 to 3755571 tid 3 2023-10-09T09:17:35.225 INFO:tasks.rados.rados.0.smithi163.stdout:2857: read oid 30 snap -1 2023-10-09T09:17:35.225 INFO:tasks.rados.rados.0.smithi163.stdout:2857: expect deleted 2023-10-09T09:17:35.230 INFO:tasks.rados.rados.0.smithi163.stdout:2858: snap_create 2023-10-09T09:17:35.230 INFO:tasks.rados.rados.0.smithi163.stdout:2849: expect (ObjNum 1033 snap 351 seq_num 1033) 2023-10-09T09:17:35.393 INFO:tasks.rados.rados.0.smithi163.stdout:2853: finishing write tid 1 to smithi16367185-35 2023-10-09T09:17:35.393 INFO:tasks.rados.rados.0.smithi163.stdout:2853: finishing write tid 2 to smithi16367185-35 2023-10-09T09:17:35.393 INFO:tasks.rados.rados.0.smithi163.stdout:2854: expect (ObjNum 983 snap 338 seq_num 983) 2023-10-09T09:17:35.533 INFO:tasks.rados.rados.0.smithi163.stdout:2853: finishing write tid 3 to smithi16367185-35 2023-10-09T09:17:35.533 INFO:tasks.rados.rados.0.smithi163.stdout:2853: finishing write tid 4 to smithi16367185-35 2023-10-09T09:17:35.533 INFO:tasks.rados.rados.0.smithi163.stdout:2853: finishing write tid 5 to smithi16367185-35 2023-10-09T09:17:35.533 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 768 (ObjNum 1066 snap 360 seq_num 1066) dirty exists 2023-10-09T09:17:35.534 INFO:tasks.rados.rados.0.smithi163.stdout:2853: left oid 35 (ObjNum 1066 snap 360 seq_num 1066) 2023-10-09T09:17:35.534 INFO:tasks.rados.rados.0.smithi163.stdout:2856: finishing write tid 1 to smithi16367185-7 2023-10-09T09:17:35.534 INFO:tasks.rados.rados.0.smithi163.stdout:2856: finishing write tid 2 to smithi16367185-7 2023-10-09T09:17:35.534 INFO:tasks.rados.rados.0.smithi163.stdout:2856: finishing write tid 3 to smithi16367185-7 2023-10-09T09:17:35.534 INFO:tasks.rados.rados.0.smithi163.stdout:2856: finishing write tid 4 to smithi16367185-7 2023-10-09T09:17:35.534 INFO:tasks.rados.rados.0.smithi163.stdout:2856: finishing write tid 5 to smithi16367185-7 2023-10-09T09:17:35.534 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 697 (ObjNum 1067 snap 360 seq_num 1067) dirty exists 2023-10-09T09:17:35.534 INFO:tasks.rados.rados.0.smithi163.stdout:2856: left oid 7 (ObjNum 1067 snap 360 seq_num 1067) 2023-10-09T09:17:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:35 smithi163 bash[78107]: cluster 2023-10-09T09:17:34.321960+0000 osd.5 (osd.5) 313 : cluster 3 failed to encode map e781 with expected crc 2023-10-09T09:17:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:35 smithi163 bash[78107]: cluster 2023-10-09T09:17:34.324712+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e781 with expected crc 2023-10-09T09:17:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:35 smithi163 bash[78107]: cluster 2023-10-09T09:17:34.327332+0000 mon.a (mon.0) 964 : cluster 0 osdmap e781: 8 total, 8 up, 8 in 2023-10-09T09:17:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:35 smithi163 bash[78107]: cluster 2023-10-09T09:17:34.332630+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e781 with expected crc 2023-10-09T09:17:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:35 smithi163 bash[78107]: cluster 2023-10-09T09:17:35.201583+0000 mon.a (mon.0) 965 : cluster 0 osdmap e782: 8 total, 8 up, 8 in 2023-10-09T09:17:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:35 smithi163 bash[75425]: cluster 2023-10-09T09:17:34.321960+0000 osd.5 (osd.5) 313 : cluster 3 failed to encode map e781 with expected crc 2023-10-09T09:17:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:35 smithi163 bash[75425]: cluster 2023-10-09T09:17:34.324712+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e781 with expected crc 2023-10-09T09:17:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:35 smithi163 bash[75425]: cluster 2023-10-09T09:17:34.327332+0000 mon.a (mon.0) 964 : cluster 0 osdmap e781: 8 total, 8 up, 8 in 2023-10-09T09:17:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:35 smithi163 bash[75425]: cluster 2023-10-09T09:17:34.332630+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e781 with expected crc 2023-10-09T09:17:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:35 smithi163 bash[75425]: cluster 2023-10-09T09:17:35.201583+0000 mon.a (mon.0) 965 : cluster 0 osdmap e782: 8 total, 8 up, 8 in 2023-10-09T09:17:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:35 smithi181 bash[59345]: cluster 2023-10-09T09:17:34.321960+0000 osd.5 (osd.5) 313 : cluster 3 failed to encode map e781 with expected crc 2023-10-09T09:17:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:35 smithi181 bash[59345]: cluster 2023-10-09T09:17:34.324712+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e781 with expected crc 2023-10-09T09:17:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:35 smithi181 bash[59345]: cluster 2023-10-09T09:17:34.327332+0000 mon.a (mon.0) 964 : cluster 0 osdmap e781: 8 total, 8 up, 8 in 2023-10-09T09:17:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:35 smithi181 bash[59345]: cluster 2023-10-09T09:17:34.332630+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e781 with expected crc 2023-10-09T09:17:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:35 smithi181 bash[59345]: cluster 2023-10-09T09:17:35.201583+0000 mon.a (mon.0) 965 : cluster 0 osdmap e782: 8 total, 8 up, 8 in 2023-10-09T09:17:36.216 INFO:tasks.rados.rados.0.smithi163.stdout:2849: done (7 left) 2023-10-09T09:17:36.216 INFO:tasks.rados.rados.0.smithi163.stdout:2850: done (6 left) 2023-10-09T09:17:36.216 INFO:tasks.rados.rados.0.smithi163.stdout:2852: done (5 left) 2023-10-09T09:17:36.216 INFO:tasks.rados.rados.0.smithi163.stdout:2853: done (4 left) 2023-10-09T09:17:36.216 INFO:tasks.rados.rados.0.smithi163.stdout:2854: done (3 left) 2023-10-09T09:17:36.216 INFO:tasks.rados.rados.0.smithi163.stdout:2856: done (2 left) 2023-10-09T09:17:36.216 INFO:tasks.rados.rados.0.smithi163.stdout:2857: done (1 left) 2023-10-09T09:17:36.216 INFO:tasks.rados.rados.0.smithi163.stdout:2858: done (0 left) 2023-10-09T09:17:36.216 INFO:tasks.rados.rados.0.smithi163.stdout:2859: delete oid 12 current snap is 361 2023-10-09T09:17:36.222 INFO:tasks.rados.rados.0.smithi163.stdout:2859: done (0 left) 2023-10-09T09:17:36.222 INFO:tasks.rados.rados.0.smithi163.stdout:2860: write oid 33 current snap is 361 2023-10-09T09:17:36.223 INFO:tasks.rados.rados.0.smithi163.stdout:2860: seq_num 1069 ranges {766683=730601,1922192=583372,3156591=676370} 2023-10-09T09:17:36.235 INFO:tasks.rados.rados.0.smithi163.stdout:2860: writing smithi16367185-33 from 766683 to 1497284 tid 1 2023-10-09T09:17:36.243 INFO:tasks.rados.rados.0.smithi163.stdout:2860: writing smithi16367185-33 from 1922192 to 2505564 tid 2 2023-10-09T09:17:36.252 INFO:tasks.rados.rados.0.smithi163.stdout:2860: writing smithi16367185-33 from 3156591 to 3832961 tid 3 2023-10-09T09:17:36.252 INFO:tasks.rados.rados.0.smithi163.stdout:2861: read oid 21 snap -1 2023-10-09T09:17:36.252 INFO:tasks.rados.rados.0.smithi163.stdout:2861: expect (ObjNum 936 snap 320 seq_num 936) 2023-10-09T09:17:36.252 INFO:tasks.rados.rados.0.smithi163.stdout:2862: snap_remove snap 360 2023-10-09T09:17:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:36 smithi163 bash[78107]: cluster 2023-10-09T09:17:35.195649+0000 osd.5 (osd.5) 314 : cluster 3 failed to encode map e782 with expected crc 2023-10-09T09:17:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:36 smithi163 bash[78107]: cluster 2023-10-09T09:17:35.200975+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e782 with expected crc 2023-10-09T09:17:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:36 smithi163 bash[78107]: cluster 2023-10-09T09:17:35.205252+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e782 with expected crc 2023-10-09T09:17:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:36 smithi163 bash[78107]: cluster 2023-10-09T09:17:35.394928+0000 mgr.y (mgr.34107) 331 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2023-10-09T09:17:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:36 smithi163 bash[78107]: cluster 2023-10-09T09:17:36.215612+0000 mon.a (mon.0) 966 : cluster 0 osdmap e783: 8 total, 8 up, 8 in 2023-10-09T09:17:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:36 smithi163 bash[75425]: cluster 2023-10-09T09:17:35.195649+0000 osd.5 (osd.5) 314 : cluster 3 failed to encode map e782 with expected crc 2023-10-09T09:17:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:36 smithi163 bash[75425]: cluster 2023-10-09T09:17:35.200975+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e782 with expected crc 2023-10-09T09:17:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:36 smithi163 bash[75425]: cluster 2023-10-09T09:17:35.205252+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e782 with expected crc 2023-10-09T09:17:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:36 smithi163 bash[75425]: cluster 2023-10-09T09:17:35.394928+0000 mgr.y (mgr.34107) 331 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2023-10-09T09:17:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:36 smithi163 bash[75425]: cluster 2023-10-09T09:17:36.215612+0000 mon.a (mon.0) 966 : cluster 0 osdmap e783: 8 total, 8 up, 8 in 2023-10-09T09:17:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:36 smithi181 bash[59345]: cluster 2023-10-09T09:17:35.195649+0000 osd.5 (osd.5) 314 : cluster 3 failed to encode map e782 with expected crc 2023-10-09T09:17:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:36 smithi181 bash[59345]: cluster 2023-10-09T09:17:35.200975+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e782 with expected crc 2023-10-09T09:17:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:36 smithi181 bash[59345]: cluster 2023-10-09T09:17:35.205252+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e782 with expected crc 2023-10-09T09:17:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:36 smithi181 bash[59345]: cluster 2023-10-09T09:17:35.394928+0000 mgr.y (mgr.34107) 331 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2023-10-09T09:17:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:36 smithi181 bash[59345]: cluster 2023-10-09T09:17:36.215612+0000 mon.a (mon.0) 966 : cluster 0 osdmap e783: 8 total, 8 up, 8 in 2023-10-09T09:17:37.218 INFO:tasks.rados.rados.0.smithi163.stdout:2862: done (2 left) 2023-10-09T09:17:37.218 INFO:tasks.rados.rados.0.smithi163.stdout:2863: read oid 38 snap -1 2023-10-09T09:17:37.218 INFO:tasks.rados.rados.0.smithi163.stdout:2863: expect (ObjNum 941 snap 322 seq_num 941) 2023-10-09T09:17:37.218 INFO:tasks.rados.rados.0.smithi163.stdout:2864: write oid 27 current snap is 361 2023-10-09T09:17:37.218 INFO:tasks.rados.rados.0.smithi163.stdout:2860: finishing write tid 1 to smithi16367185-33 2023-10-09T09:17:37.219 INFO:tasks.rados.rados.0.smithi163.stdout:2860: finishing write tid 2 to smithi16367185-33 2023-10-09T09:17:37.219 INFO:tasks.rados.rados.0.smithi163.stdout:2861: expect (ObjNum 936 snap 320 seq_num 936) 2023-10-09T09:17:37.391 INFO:tasks.rados.rados.0.smithi163.stdout:2860: finishing write tid 3 to smithi16367185-33 2023-10-09T09:17:37.391 INFO:tasks.rados.rados.0.smithi163.stdout:2860: finishing write tid 4 to smithi16367185-33 2023-10-09T09:17:37.391 INFO:tasks.rados.rados.0.smithi163.stdout:2860: finishing write tid 5 to smithi16367185-33 2023-10-09T09:17:37.392 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1356 (ObjNum 1069 snap 361 seq_num 1069) dirty exists 2023-10-09T09:17:37.392 INFO:tasks.rados.rados.0.smithi163.stdout:2860: left oid 33 (ObjNum 1069 snap 361 seq_num 1069) 2023-10-09T09:17:37.392 INFO:tasks.rados.rados.0.smithi163.stdout:2863: expect (ObjNum 941 snap 322 seq_num 941) 2023-10-09T09:17:37.559 INFO:tasks.rados.rados.0.smithi163.stdout:2864: seq_num 1070 ranges {783490=405850,1980674=727030,3184014=487642} 2023-10-09T09:17:37.571 INFO:tasks.rados.rados.0.smithi163.stdout:2864: writing smithi16367185-27 from 783490 to 1189340 tid 1 2023-10-09T09:17:37.581 INFO:tasks.rados.rados.0.smithi163.stdout:2864: writing smithi16367185-27 from 1980674 to 2707704 tid 2 2023-10-09T09:17:37.587 INFO:tasks.rados.rados.0.smithi163.stdout:2864: writing smithi16367185-27 from 3184014 to 3671656 tid 3 2023-10-09T09:17:37.587 INFO:tasks.rados.rados.0.smithi163.stdout:2860: done (3 left) 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:2861: done (2 left) 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:2863: done (1 left) 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:2865: read oid 33 snap 353 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:2865: expect (ObjNum 1035 snap 351 seq_num 1035) 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:2866: read oid 50 snap -1 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:2866: expect deleted 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:2867: rollback oid 39 current snap is 361 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 355 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:2864: finishing write tid 1 to smithi16367185-27 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:2868: write oid 18 current snap is 361 2023-10-09T09:17:37.588 INFO:tasks.rados.rados.0.smithi163.stdout:2868: seq_num 1071 ranges {400297=481099,1639156=361141} 2023-10-09T09:17:37.592 INFO:tasks.rados.rados.0.smithi163.stdout:2868: writing smithi16367185-18 from 400297 to 881396 tid 1 2023-10-09T09:17:37.599 INFO:tasks.rados.rados.0.smithi163.stdout:2868: writing smithi16367185-18 from 1639156 to 2000297 tid 2 2023-10-09T09:17:37.599 INFO:tasks.rados.rados.0.smithi163.stdout:2869: read oid 23 snap -1 2023-10-09T09:17:37.599 INFO:tasks.rados.rados.0.smithi163.stdout:2869: expect deleted 2023-10-09T09:17:37.599 INFO:tasks.rados.rados.0.smithi163.stdout:2870: delete oid 42 current snap is 361 2023-10-09T09:17:37.599 INFO:tasks.rados.rados.0.smithi163.stdout:2864: finishing write tid 2 to smithi16367185-27 2023-10-09T09:17:37.600 INFO:tasks.rados.rados.0.smithi163.stdout:2865: expect (ObjNum 1035 snap 351 seq_num 1035) 2023-10-09T09:17:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:37 smithi163 bash[78107]: cluster 2023-10-09T09:17:36.209957+0000 osd.5 (osd.5) 315 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:37 smithi163 bash[78107]: cluster 2023-10-09T09:17:36.277418+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:37 smithi163 bash[78107]: cluster 2023-10-09T09:17:36.277494+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:37 smithi163 bash[78107]: cluster 2023-10-09T09:17:36.277544+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:37 smithi163 bash[78107]: cluster 2023-10-09T09:17:37.217736+0000 mon.a (mon.0) 967 : cluster 0 osdmap e784: 8 total, 8 up, 8 in 2023-10-09T09:17:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:37 smithi163 bash[75425]: cluster 2023-10-09T09:17:36.209957+0000 osd.5 (osd.5) 315 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:37 smithi163 bash[75425]: cluster 2023-10-09T09:17:36.277418+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:37 smithi163 bash[75425]: cluster 2023-10-09T09:17:36.277494+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:37 smithi163 bash[75425]: cluster 2023-10-09T09:17:36.277544+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:37 smithi163 bash[75425]: cluster 2023-10-09T09:17:37.217736+0000 mon.a (mon.0) 967 : cluster 0 osdmap e784: 8 total, 8 up, 8 in 2023-10-09T09:17:37.766 INFO:tasks.rados.rados.0.smithi163.stdout:2867: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:17:37.766 INFO:tasks.rados.rados.0.smithi163.stdout:2867: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:17:37.766 INFO:tasks.rados.rados.0.smithi163.stdout:2867: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:17:37.766 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1348 (ObjNum 1051 snap 354 seq_num 1051) dirty exists 2023-10-09T09:17:37.766 INFO:tasks.rados.rados.0.smithi163.stdout:2864: finishing write tid 3 to smithi16367185-27 2023-10-09T09:17:37.766 INFO:tasks.rados.rados.0.smithi163.stdout:2864: finishing write tid 4 to smithi16367185-27 2023-10-09T09:17:37.766 INFO:tasks.rados.rados.0.smithi163.stdout:2864: finishing write tid 5 to smithi16367185-27 2023-10-09T09:17:37.766 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 822 (ObjNum 1070 snap 361 seq_num 1070) dirty exists 2023-10-09T09:17:37.766 INFO:tasks.rados.rados.0.smithi163.stdout:2864: left oid 27 (ObjNum 1070 snap 361 seq_num 1070) 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2868: finishing write tid 1 to smithi16367185-18 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2868: finishing write tid 2 to smithi16367185-18 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2868: finishing write tid 3 to smithi16367185-18 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2868: finishing write tid 4 to smithi16367185-18 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 914 (ObjNum 1071 snap 361 seq_num 1071) dirty exists 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2868: left oid 18 (ObjNum 1071 snap 361 seq_num 1071) 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2864: done (6 left) 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2865: done (5 left) 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2866: done (4 left) 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2867: done (3 left) 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2868: done (2 left) 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2869: done (1 left) 2023-10-09T09:17:37.767 INFO:tasks.rados.rados.0.smithi163.stdout:2870: done (0 left) 2023-10-09T09:17:37.768 INFO:tasks.rados.rados.0.smithi163.stdout:2871: rollback oid 38 current snap is 361 2023-10-09T09:17:37.768 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 353 2023-10-09T09:17:37.768 INFO:tasks.rados.rados.0.smithi163.stdout:2872: write oid 1 current snap is 361 2023-10-09T09:17:37.768 INFO:tasks.rados.rados.0.smithi163.stdout:2872: seq_num 1073 ranges {433911=747950,1756120=277791} 2023-10-09T09:17:37.777 INFO:tasks.rados.rados.0.smithi163.stdout:2872: writing smithi16367185-1 from 433911 to 1181861 tid 1 2023-10-09T09:17:37.783 INFO:tasks.rados.rados.0.smithi163.stdout:2872: writing smithi16367185-1 from 1756120 to 2033911 tid 2 2023-10-09T09:17:37.784 INFO:tasks.rados.rados.0.smithi163.stdout:2873: snap_remove snap 336 2023-10-09T09:17:37.784 INFO:tasks.rados.rados.0.smithi163.stdout:2871: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:17:37.784 INFO:tasks.rados.rados.0.smithi163.stdout:2871: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:17:37.784 INFO:tasks.rados.rados.0.smithi163.stdout:2871: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:17:37.784 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1352 (ObjNum 997 snap 342 seq_num 997) dirty exists 2023-10-09T09:17:37.784 INFO:tasks.rados.rados.0.smithi163.stdout:2872: finishing write tid 1 to smithi16367185-1 2023-10-09T09:17:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:37 smithi181 bash[59345]: cluster 2023-10-09T09:17:36.209957+0000 osd.5 (osd.5) 315 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:37 smithi181 bash[59345]: cluster 2023-10-09T09:17:36.277418+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:37 smithi181 bash[59345]: cluster 2023-10-09T09:17:36.277494+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:37 smithi181 bash[59345]: cluster 2023-10-09T09:17:36.277544+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e783 with expected crc 2023-10-09T09:17:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:37 smithi181 bash[59345]: cluster 2023-10-09T09:17:37.217736+0000 mon.a (mon.0) 967 : cluster 0 osdmap e784: 8 total, 8 up, 8 in 2023-10-09T09:17:38.348 INFO:tasks.rados.rados.0.smithi163.stdout:2871: done (2 left) 2023-10-09T09:17:38.348 INFO:tasks.rados.rados.0.smithi163.stdout:2873: done (1 left) 2023-10-09T09:17:38.348 INFO:tasks.rados.rados.0.smithi163.stdout:2874: delete oid 41 current snap is 361 2023-10-09T09:17:38.348 INFO:tasks.rados.rados.0.smithi163.stdout:2872: finishing write tid 2 to smithi16367185-1 2023-10-09T09:17:38.348 INFO:tasks.rados.rados.0.smithi163.stdout:2872: finishing write tid 3 to smithi16367185-1 2023-10-09T09:17:38.348 INFO:tasks.rados.rados.0.smithi163.stdout:2872: finishing write tid 4 to smithi16367185-1 2023-10-09T09:17:38.349 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1383 (ObjNum 1073 snap 361 seq_num 1073) dirty exists 2023-10-09T09:17:38.349 INFO:tasks.rados.rados.0.smithi163.stdout:2872: left oid 1 (ObjNum 1073 snap 361 seq_num 1073) 2023-10-09T09:17:38.364 INFO:tasks.rados.rados.0.smithi163.stdout:2872: done (1 left) 2023-10-09T09:17:38.365 INFO:tasks.rados.rados.0.smithi163.stdout:2874: done (0 left) 2023-10-09T09:17:38.365 INFO:tasks.rados.rados.0.smithi163.stdout:2875: snap_remove snap 356 2023-10-09T09:17:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:38 smithi163 bash[75425]: cluster 2023-10-09T09:17:37.212541+0000 osd.5 (osd.5) 316 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:38 smithi163 bash[75425]: cluster 2023-10-09T09:17:37.213306+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:38 smithi163 bash[75425]: cluster 2023-10-09T09:17:37.219115+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:38 smithi163 bash[75425]: cluster 2023-10-09T09:17:37.395679+0000 mgr.y (mgr.34107) 332 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 235 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2023-10-09T09:17:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:38 smithi163 bash[78107]: cluster 2023-10-09T09:17:37.212541+0000 osd.5 (osd.5) 316 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:38 smithi163 bash[78107]: cluster 2023-10-09T09:17:37.213306+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:38 smithi163 bash[78107]: cluster 2023-10-09T09:17:37.219115+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:38 smithi163 bash[78107]: cluster 2023-10-09T09:17:37.395679+0000 mgr.y (mgr.34107) 332 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 235 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2023-10-09T09:17:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:38 smithi181 bash[59345]: cluster 2023-10-09T09:17:37.212541+0000 osd.5 (osd.5) 316 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:38 smithi181 bash[59345]: cluster 2023-10-09T09:17:37.213306+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:38 smithi181 bash[59345]: cluster 2023-10-09T09:17:37.219115+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:38 smithi181 bash[59345]: cluster 2023-10-09T09:17:37.395679+0000 mgr.y (mgr.34107) 332 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 235 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2023-10-09T09:17:39.354 INFO:tasks.rados.rados.0.smithi163.stdout:2875: done (0 left) 2023-10-09T09:17:39.354 INFO:tasks.rados.rados.0.smithi163.stdout:2876: delete oid 43 current snap is 361 2023-10-09T09:17:39.366 INFO:tasks.rados.rados.0.smithi163.stdout:2876: done (0 left) 2023-10-09T09:17:39.366 INFO:tasks.rados.rados.0.smithi163.stdout:2877: write oid 30 current snap is 361 2023-10-09T09:17:39.367 INFO:tasks.rados.rados.0.smithi163.stdout:2877: seq_num 1076 ranges {484332=573697,1531566=552766} 2023-10-09T09:17:39.372 INFO:tasks.rados.rados.0.smithi163.stdout:2877: writing smithi16367185-30 from 484332 to 1058029 tid 1 2023-10-09T09:17:39.379 INFO:tasks.rados.rados.0.smithi163.stdout:2877: writing smithi16367185-30 from 1531566 to 2084332 tid 2 2023-10-09T09:17:39.379 INFO:tasks.rados.rados.0.smithi163.stdout:2878: write oid 26 current snap is 361 2023-10-09T09:17:39.379 INFO:tasks.rados.rados.0.smithi163.stdout:2878: seq_num 1077 ranges {501139=648946,1873695=227444} 2023-10-09T09:17:39.386 INFO:tasks.rados.rados.0.smithi163.stdout:2878: writing smithi16367185-26 from 501139 to 1150085 tid 1 2023-10-09T09:17:39.392 INFO:tasks.rados.rados.0.smithi163.stdout:2878: writing smithi16367185-26 from 1873695 to 2101139 tid 2 2023-10-09T09:17:39.392 INFO:tasks.rados.rados.0.smithi163.stdout:2879: snap_create 2023-10-09T09:17:39.392 INFO:tasks.rados.rados.0.smithi163.stdout:2877: finishing write tid 1 to smithi16367185-30 2023-10-09T09:17:39.392 INFO:tasks.rados.rados.0.smithi163.stdout:2877: finishing write tid 2 to smithi16367185-30 2023-10-09T09:17:39.392 INFO:tasks.rados.rados.0.smithi163.stdout:2877: finishing write tid 3 to smithi16367185-30 2023-10-09T09:17:39.392 INFO:tasks.rados.rados.0.smithi163.stdout:2877: finishing write tid 4 to smithi16367185-30 2023-10-09T09:17:39.392 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 921 (ObjNum 1076 snap 361 seq_num 1076) dirty exists 2023-10-09T09:17:39.393 INFO:tasks.rados.rados.0.smithi163.stdout:2877: left oid 30 (ObjNum 1076 snap 361 seq_num 1076) 2023-10-09T09:17:39.393 INFO:tasks.rados.rados.0.smithi163.stdout:2878: finishing write tid 1 to smithi16367185-26 2023-10-09T09:17:39.394 INFO:tasks.rados.rados.0.smithi163.stdout:2878: finishing write tid 2 to smithi16367185-26 2023-10-09T09:17:39.395 INFO:tasks.rados.rados.0.smithi163.stdout:2878: finishing write tid 3 to smithi16367185-26 2023-10-09T09:17:39.395 INFO:tasks.rados.rados.0.smithi163.stdout:2878: finishing write tid 4 to smithi16367185-26 2023-10-09T09:17:39.395 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 832 (ObjNum 1077 snap 361 seq_num 1077) dirty exists 2023-10-09T09:17:39.395 INFO:tasks.rados.rados.0.smithi163.stdout:2878: left oid 26 (ObjNum 1077 snap 361 seq_num 1077) 2023-10-09T09:17:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:39 smithi163 bash[78107]: cluster 2023-10-09T09:17:37.484065+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:39 smithi163 bash[78107]: cluster 2023-10-09T09:17:38.342683+0000 osd.5 (osd.5) 317 : cluster 3 failed to encode map e785 with expected crc 2023-10-09T09:17:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:39 smithi163 bash[78107]: cluster 2023-10-09T09:17:38.346983+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e785 with expected crc 2023-10-09T09:17:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:39 smithi163 bash[78107]: cluster 2023-10-09T09:17:38.347560+0000 mon.a (mon.0) 968 : cluster 0 osdmap e785: 8 total, 8 up, 8 in 2023-10-09T09:17:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:39 smithi163 bash[75425]: cluster 2023-10-09T09:17:37.484065+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:39 smithi163 bash[75425]: cluster 2023-10-09T09:17:38.342683+0000 osd.5 (osd.5) 317 : cluster 3 failed to encode map e785 with expected crc 2023-10-09T09:17:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:39 smithi163 bash[75425]: cluster 2023-10-09T09:17:38.346983+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e785 with expected crc 2023-10-09T09:17:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:39 smithi163 bash[75425]: cluster 2023-10-09T09:17:38.347560+0000 mon.a (mon.0) 968 : cluster 0 osdmap e785: 8 total, 8 up, 8 in 2023-10-09T09:17:39.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:39 smithi181 bash[59345]: cluster 2023-10-09T09:17:37.484065+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e784 with expected crc 2023-10-09T09:17:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:39 smithi181 bash[59345]: cluster 2023-10-09T09:17:38.342683+0000 osd.5 (osd.5) 317 : cluster 3 failed to encode map e785 with expected crc 2023-10-09T09:17:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:39 smithi181 bash[59345]: cluster 2023-10-09T09:17:38.346983+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e785 with expected crc 2023-10-09T09:17:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:39 smithi181 bash[59345]: cluster 2023-10-09T09:17:38.347560+0000 mon.a (mon.0) 968 : cluster 0 osdmap e785: 8 total, 8 up, 8 in 2023-10-09T09:17:40.204 INFO:tasks.rados.rados.0.smithi163.stdout:2877: done (2 left) 2023-10-09T09:17:40.204 INFO:tasks.rados.rados.0.smithi163.stdout:2878: done (1 left) 2023-10-09T09:17:40.204 INFO:tasks.rados.rados.0.smithi163.stdout:2879: done (0 left) 2023-10-09T09:17:40.204 INFO:tasks.rados.rados.0.smithi163.stdout:2880: delete oid 34 current snap is 362 2023-10-09T09:17:40.214 INFO:tasks.rados.rados.0.smithi163.stdout:2880: done (0 left) 2023-10-09T09:17:40.214 INFO:tasks.rados.rados.0.smithi163.stdout:2881: read oid 43 snap -1 2023-10-09T09:17:40.214 INFO:tasks.rados.rados.0.smithi163.stdout:2881: expect deleted 2023-10-09T09:17:40.214 INFO:tasks.rados.rados.0.smithi163.stdout:2882: delete oid 36 current snap is 362 2023-10-09T09:17:40.220 INFO:tasks.rados.rados.0.smithi163.stdout:2881: done (1 left) 2023-10-09T09:17:40.220 INFO:tasks.rados.rados.0.smithi163.stdout:2882: done (0 left) 2023-10-09T09:17:40.220 INFO:tasks.rados.rados.0.smithi163.stdout:2883: rollback oid 17 current snap is 362 2023-10-09T09:17:40.221 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 353 2023-10-09T09:17:40.221 INFO:tasks.rados.rados.0.smithi163.stdout:2884: read oid 28 snap 352 2023-10-09T09:17:40.221 INFO:tasks.rados.rados.0.smithi163.stdout:2884: expect (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:17:40.221 INFO:tasks.rados.rados.0.smithi163.stdout:2885: rollback oid 31 current snap is 362 2023-10-09T09:17:40.221 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 31 to 361 2023-10-09T09:17:40.221 INFO:tasks.rados.rados.0.smithi163.stdout:2886: rollback oid 30 current snap is 362 2023-10-09T09:17:40.221 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 351 2023-10-09T09:17:40.221 INFO:tasks.rados.rados.0.smithi163.stdout:2887: write oid 42 current snap is 362 2023-10-09T09:17:40.221 INFO:tasks.rados.rados.0.smithi163.stdout:2887: seq_num 1080 ranges {551560=591046,1765494=386066} 2023-10-09T09:17:40.230 INFO:tasks.rados.rados.0.smithi163.stdout:2887: writing smithi16367185-42 from 551560 to 1142606 tid 1 2023-10-09T09:17:40.237 INFO:tasks.rados.rados.0.smithi163.stdout:2887: writing smithi16367185-42 from 1765494 to 2151560 tid 2 2023-10-09T09:17:40.237 INFO:tasks.rados.rados.0.smithi163.stdout:2888: read oid 18 snap -1 2023-10-09T09:17:40.237 INFO:tasks.rados.rados.0.smithi163.stdout:2888: expect (ObjNum 1071 snap 361 seq_num 1071) 2023-10-09T09:17:40.237 INFO:tasks.rados.rados.0.smithi163.stdout:2889: write oid 38 current snap is 362 2023-10-09T09:17:40.237 INFO:tasks.rados.rados.0.smithi163.stdout:2885: finishing rollback tid 1 to smithi16367185-31 2023-10-09T09:17:40.237 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:17:40.237 INFO:tasks.rados.rados.0.smithi163.stdout:2884: expect (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:17:40.336 INFO:tasks.rados.rados.0.smithi163.stdout:2886: finishing rollback tid 0 to smithi16367185-30 2023-10-09T09:17:40.336 INFO:tasks.rados.rados.0.smithi163.stdout:2886: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:17:40.336 INFO:tasks.rados.rados.0.smithi163.stdout:2886: finishing rollback tid 2 to smithi16367185-30 2023-10-09T09:17:40.336 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 925 (ObjNum 1034 snap 351 seq_num 1034) dirty exists 2023-10-09T09:17:40.336 INFO:tasks.rados.rados.0.smithi163.stdout:2887: finishing write tid 1 to smithi16367185-42 2023-10-09T09:17:40.336 INFO:tasks.rados.rados.0.smithi163.stdout:2888: expect (ObjNum 1071 snap 361 seq_num 1071) 2023-10-09T09:17:40.417 INFO:tasks.rados.rados.0.smithi163.stdout:2887: finishing write tid 2 to smithi16367185-42 2023-10-09T09:17:40.417 INFO:tasks.rados.rados.0.smithi163.stdout:2887: finishing write tid 3 to smithi16367185-42 2023-10-09T09:17:40.417 INFO:tasks.rados.rados.0.smithi163.stdout:2887: finishing write tid 4 to smithi16367185-42 2023-10-09T09:17:40.417 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 2051 (ObjNum 1080 snap 362 seq_num 1080) dirty exists 2023-10-09T09:17:40.418 INFO:tasks.rados.rados.0.smithi163.stdout:2887: left oid 42 (ObjNum 1080 snap 362 seq_num 1080) 2023-10-09T09:17:40.418 INFO:tasks.rados.rados.0.smithi163.stdout:2883: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:17:40.418 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1391 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:17:40.418 INFO:tasks.rados.rados.0.smithi163.stdout:2889: seq_num 1081 ranges {568367=666295,1707623=460744} 2023-10-09T09:17:40.430 INFO:tasks.rados.rados.0.smithi163.stdout:2889: writing smithi16367185-38 from 568367 to 1234662 tid 1 2023-10-09T09:17:40.435 INFO:tasks.rados.rados.0.smithi163.stdout:2889: writing smithi16367185-38 from 1707623 to 2168367 tid 2 2023-10-09T09:17:40.435 INFO:tasks.rados.rados.0.smithi163.stdout:2883: done (6 left) 2023-10-09T09:17:40.435 INFO:tasks.rados.rados.0.smithi163.stdout:2884: done (5 left) 2023-10-09T09:17:40.436 INFO:tasks.rados.rados.0.smithi163.stdout:2885: done (4 left) 2023-10-09T09:17:40.436 INFO:tasks.rados.rados.0.smithi163.stdout:2886: done (3 left) 2023-10-09T09:17:40.436 INFO:tasks.rados.rados.0.smithi163.stdout:2887: done (2 left) 2023-10-09T09:17:40.436 INFO:tasks.rados.rados.0.smithi163.stdout:2888: done (1 left) 2023-10-09T09:17:40.436 INFO:tasks.rados.rados.0.smithi163.stdout:2890: delete oid 11 current snap is 362 2023-10-09T09:17:40.439 INFO:tasks.rados.rados.0.smithi163.stdout:2889: finishing write tid 1 to smithi16367185-38 2023-10-09T09:17:40.441 INFO:tasks.rados.rados.0.smithi163.stdout:2890: done (1 left) 2023-10-09T09:17:40.441 INFO:tasks.rados.rados.0.smithi163.stdout:2891: write oid 22 current snap is 362 2023-10-09T09:17:40.441 INFO:tasks.rados.rados.0.smithi163.stdout:2891: seq_num 1083 ranges {601981=416793,1424587=492702} 2023-10-09T09:17:40.452 INFO:tasks.rados.rados.0.smithi163.stdout:2891: writing smithi16367185-22 from 601981 to 1018774 tid 1 2023-10-09T09:17:40.458 INFO:tasks.rados.rados.0.smithi163.stdout:2891: writing smithi16367185-22 from 1424587 to 1917289 tid 2 2023-10-09T09:17:40.459 INFO:tasks.rados.rados.0.smithi163.stdout:2892: read oid 3 snap -1 2023-10-09T09:17:40.459 INFO:tasks.rados.rados.0.smithi163.stdout:2892: expect (ObjNum 1058 snap 357 seq_num 1058) 2023-10-09T09:17:40.459 INFO:tasks.rados.rados.0.smithi163.stdout:2893: write oid 49 current snap is 362 2023-10-09T09:17:40.459 INFO:tasks.rados.rados.0.smithi163.stdout:2889: finishing write tid 2 to smithi16367185-38 2023-10-09T09:17:40.459 INFO:tasks.rados.rados.0.smithi163.stdout:2889: finishing write tid 3 to smithi16367185-38 2023-10-09T09:17:40.459 INFO:tasks.rados.rados.0.smithi163.stdout:2889: finishing write tid 4 to smithi16367185-38 2023-10-09T09:17:40.459 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1363 (ObjNum 1081 snap 362 seq_num 1081) dirty exists 2023-10-09T09:17:40.459 INFO:tasks.rados.rados.0.smithi163.stdout:2889: left oid 38 (ObjNum 1081 snap 362 seq_num 1081) 2023-10-09T09:17:40.459 INFO:tasks.rados.rados.0.smithi163.stdout:2891: finishing write tid 1 to smithi16367185-22 2023-10-09T09:17:40.459 INFO:tasks.rados.rados.0.smithi163.stdout:2893: seq_num 1084 ranges {618788=492042,1883069=335719} 2023-10-09T09:17:40.464 INFO:tasks.rados.rados.0.smithi163.stdout:2893: writing smithi16367185-49 from 618788 to 1110830 tid 1 2023-10-09T09:17:40.471 INFO:tasks.rados.rados.0.smithi163.stdout:2893: writing smithi16367185-49 from 1883069 to 2218788 tid 2 2023-10-09T09:17:40.471 INFO:tasks.rados.rados.0.smithi163.stdout:2889: done (3 left) 2023-10-09T09:17:40.471 INFO:tasks.rados.rados.0.smithi163.stdout:2894: snap_remove snap 361 2023-10-09T09:17:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[75425]: cluster 2023-10-09T09:17:39.348106+0000 osd.5 (osd.5) 318 : cluster 3 failed to encode map e786 with expected crc 2023-10-09T09:17:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[75425]: cluster 2023-10-09T09:17:39.354051+0000 mon.a (mon.0) 969 : cluster 0 osdmap e786: 8 total, 8 up, 8 in 2023-10-09T09:17:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[75425]: cluster 2023-10-09T09:17:39.380366+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e786 with expected crc 2023-10-09T09:17:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[75425]: cluster 2023-10-09T09:17:39.396308+0000 mgr.y (mgr.34107) 333 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 230 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 891 KiB/s wr, 10 op/s 2023-10-09T09:17:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[75425]: cluster 2023-10-09T09:17:40.203244+0000 mon.a (mon.0) 970 : cluster 0 osdmap e787: 8 total, 8 up, 8 in 2023-10-09T09:17:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[75425]: audit 2023-10-09T09:17:40.347013+0000 mon.a (mon.0) 971 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[78107]: cluster 2023-10-09T09:17:39.348106+0000 osd.5 (osd.5) 318 : cluster 3 failed to encode map e786 with expected crc 2023-10-09T09:17:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[78107]: cluster 2023-10-09T09:17:39.354051+0000 mon.a (mon.0) 969 : cluster 0 osdmap e786: 8 total, 8 up, 8 in 2023-10-09T09:17:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[78107]: cluster 2023-10-09T09:17:39.380366+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e786 with expected crc 2023-10-09T09:17:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[78107]: cluster 2023-10-09T09:17:39.396308+0000 mgr.y (mgr.34107) 333 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 230 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 891 KiB/s wr, 10 op/s 2023-10-09T09:17:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[78107]: cluster 2023-10-09T09:17:40.203244+0000 mon.a (mon.0) 970 : cluster 0 osdmap e787: 8 total, 8 up, 8 in 2023-10-09T09:17:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:40 smithi163 bash[78107]: audit 2023-10-09T09:17:40.347013+0000 mon.a (mon.0) 971 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:40 smithi181 bash[59345]: cluster 2023-10-09T09:17:39.348106+0000 osd.5 (osd.5) 318 : cluster 3 failed to encode map e786 with expected crc 2023-10-09T09:17:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:40 smithi181 bash[59345]: cluster 2023-10-09T09:17:39.354051+0000 mon.a (mon.0) 969 : cluster 0 osdmap e786: 8 total, 8 up, 8 in 2023-10-09T09:17:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:40 smithi181 bash[59345]: cluster 2023-10-09T09:17:39.380366+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e786 with expected crc 2023-10-09T09:17:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:40 smithi181 bash[59345]: cluster 2023-10-09T09:17:39.396308+0000 mgr.y (mgr.34107) 333 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 230 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 891 KiB/s wr, 10 op/s 2023-10-09T09:17:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:40 smithi181 bash[59345]: cluster 2023-10-09T09:17:40.203244+0000 mon.a (mon.0) 970 : cluster 0 osdmap e787: 8 total, 8 up, 8 in 2023-10-09T09:17:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:40 smithi181 bash[59345]: audit 2023-10-09T09:17:40.347013+0000 mon.a (mon.0) 971 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:41.372 INFO:tasks.rados.rados.0.smithi163.stdout:2894: done (3 left) 2023-10-09T09:17:41.372 INFO:tasks.rados.rados.0.smithi163.stdout:2895: snap_remove snap 355 2023-10-09T09:17:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:41 smithi163 bash[78107]: cluster 2023-10-09T09:17:40.197696+0000 osd.5 (osd.5) 319 : cluster 3 failed to encode map e787 with expected crc 2023-10-09T09:17:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:41 smithi163 bash[78107]: cluster 2023-10-09T09:17:40.223773+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e787 with expected crc 2023-10-09T09:17:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:41 smithi163 bash[75425]: cluster 2023-10-09T09:17:40.197696+0000 osd.5 (osd.5) 319 : cluster 3 failed to encode map e787 with expected crc 2023-10-09T09:17:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:41 smithi163 bash[75425]: cluster 2023-10-09T09:17:40.223773+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e787 with expected crc 2023-10-09T09:17:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:41 smithi181 bash[59345]: cluster 2023-10-09T09:17:40.197696+0000 osd.5 (osd.5) 319 : cluster 3 failed to encode map e787 with expected crc 2023-10-09T09:17:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:41 smithi181 bash[59345]: cluster 2023-10-09T09:17:40.223773+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e787 with expected crc 2023-10-09T09:17:42.375 INFO:tasks.rados.rados.0.smithi163.stdout:2895: done (3 left) 2023-10-09T09:17:42.376 INFO:tasks.rados.rados.0.smithi163.stdout:2896: delete oid 13 current snap is 362 2023-10-09T09:17:42.376 INFO:tasks.rados.rados.0.smithi163.stdout:2891: finishing write tid 2 to smithi16367185-22 2023-10-09T09:17:42.376 INFO:tasks.rados.rados.0.smithi163.stdout:2891: finishing write tid 3 to smithi16367185-22 2023-10-09T09:17:42.376 INFO:tasks.rados.rados.0.smithi163.stdout:2891: finishing write tid 4 to smithi16367185-22 2023-10-09T09:17:42.376 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 2055 (ObjNum 1083 snap 362 seq_num 1083) dirty exists 2023-10-09T09:17:42.376 INFO:tasks.rados.rados.0.smithi163.stdout:2891: left oid 22 (ObjNum 1083 snap 362 seq_num 1083) 2023-10-09T09:17:42.376 INFO:tasks.rados.rados.0.smithi163.stdout:2892: expect (ObjNum 1058 snap 357 seq_num 1058) 2023-10-09T09:17:42.628 INFO:tasks.rados.rados.0.smithi163.stdout:2893: finishing write tid 1 to smithi16367185-49 2023-10-09T09:17:42.921 INFO:tasks.rados.rados.0.smithi163.stdout:2893: finishing write tid 2 to smithi16367185-49 2023-10-09T09:17:42.921 INFO:tasks.rados.rados.0.smithi163.stdout:2893: finishing write tid 3 to smithi16367185-49 2023-10-09T09:17:42.921 INFO:tasks.rados.rados.0.smithi163.stdout:2893: finishing write tid 4 to smithi16367185-49 2023-10-09T09:17:42.921 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 327 (ObjNum 1084 snap 362 seq_num 1084) dirty exists 2023-10-09T09:17:42.921 INFO:tasks.rados.rados.0.smithi163.stdout:2893: left oid 49 (ObjNum 1084 snap 362 seq_num 1084) 2023-10-09T09:17:42.921 INFO:tasks.rados.rados.0.smithi163.stdout:2891: done (3 left) 2023-10-09T09:17:42.921 INFO:tasks.rados.rados.0.smithi163.stdout:2892: done (2 left) 2023-10-09T09:17:42.921 INFO:tasks.rados.rados.0.smithi163.stdout:2893: done (1 left) 2023-10-09T09:17:42.922 INFO:tasks.rados.rados.0.smithi163.stdout:2896: done (0 left) 2023-10-09T09:17:42.922 INFO:tasks.rados.rados.0.smithi163.stdout:2897: read oid 32 snap -1 2023-10-09T09:17:42.922 INFO:tasks.rados.rados.0.smithi163.stdout:2897: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:17:42.922 INFO:tasks.rados.rados.0.smithi163.stdout:2898: read oid 12 snap -1 2023-10-09T09:17:42.922 INFO:tasks.rados.rados.0.smithi163.stdout:2898: expect deleted 2023-10-09T09:17:42.922 INFO:tasks.rados.rados.0.smithi163.stdout:2899: read oid 39 snap -1 2023-10-09T09:17:42.922 INFO:tasks.rados.rados.0.smithi163.stdout:2899: expect (ObjNum 1051 snap 354 seq_num 1051) 2023-10-09T09:17:42.922 INFO:tasks.rados.rados.0.smithi163.stdout:2900: write oid 50 current snap is 362 2023-10-09T09:17:42.922 INFO:tasks.rados.rados.0.smithi163.stdout:2900: seq_num 1086 ranges {652402=642540,1883680=368722} 2023-10-09T09:17:42.923 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:42 smithi181 bash[59345]: cluster 2023-10-09T09:17:41.366928+0000 osd.5 (osd.5) 320 : cluster 3 failed to encode map e788 with expected crc 2023-10-09T09:17:42.923 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:42 smithi181 bash[59345]: cluster 2023-10-09T09:17:41.366929+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e788 with expected crc 2023-10-09T09:17:42.923 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:42 smithi181 bash[59345]: cluster 2023-10-09T09:17:41.371711+0000 mon.a (mon.0) 972 : cluster 0 osdmap e788: 8 total, 8 up, 8 in 2023-10-09T09:17:42.923 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:42 smithi181 bash[59345]: cluster 2023-10-09T09:17:41.396855+0000 mgr.y (mgr.34107) 334 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 230 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2023-10-09T09:17:42.924 INFO:tasks.rados.rados.0.smithi163.stdout:2900: writing smithi16367185-50 from 652402 to 1294942 tid 1 2023-10-09T09:17:42.924 INFO:tasks.rados.rados.0.smithi163.stdout:2900: writing smithi16367185-50 from 1883680 to 2252402 tid 2 2023-10-09T09:17:42.924 INFO:tasks.rados.rados.0.smithi163.stdout:2901: delete oid 11 current snap is 362 2023-10-09T09:17:42.924 INFO:tasks.rados.rados.0.smithi163.stdout:2897: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:17:42.925 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:42 smithi163 bash[75425]: cluster 2023-10-09T09:17:41.366928+0000 osd.5 (osd.5) 320 : cluster 3 failed to encode map e788 with expected crc 2023-10-09T09:17:42.925 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:42 smithi163 bash[75425]: cluster 2023-10-09T09:17:41.366929+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e788 with expected crc 2023-10-09T09:17:42.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:42 smithi163 bash[75425]: cluster 2023-10-09T09:17:41.371711+0000 mon.a (mon.0) 972 : cluster 0 osdmap e788: 8 total, 8 up, 8 in 2023-10-09T09:17:42.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:42 smithi163 bash[75425]: cluster 2023-10-09T09:17:41.396855+0000 mgr.y (mgr.34107) 334 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 230 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2023-10-09T09:17:42.926 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:42 smithi163 bash[78107]: cluster 2023-10-09T09:17:41.366928+0000 osd.5 (osd.5) 320 : cluster 3 failed to encode map e788 with expected crc 2023-10-09T09:17:43.077 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:42 smithi163 bash[78107]: cluster 2023-10-09T09:17:41.366929+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e788 with expected crc 2023-10-09T09:17:43.077 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:42 smithi163 bash[78107]: cluster 2023-10-09T09:17:41.371711+0000 mon.a (mon.0) 972 : cluster 0 osdmap e788: 8 total, 8 up, 8 in 2023-10-09T09:17:43.080 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:42 smithi163 bash[78107]: cluster 2023-10-09T09:17:41.396855+0000 mgr.y (mgr.34107) 334 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 230 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2023-10-09T09:17:43.081 INFO:tasks.rados.rados.0.smithi163.stdout:2899: expect (ObjNum 1051 snap 354 seq_num 1051) 2023-10-09T09:17:43.081 INFO:tasks.rados.rados.0.smithi163.stdout:2900: finishing write tid 1 to smithi16367185-50 2023-10-09T09:17:43.081 INFO:tasks.rados.rados.0.smithi163.stdout:2900: finishing write tid 2 to smithi16367185-50 2023-10-09T09:17:43.081 INFO:tasks.rados.rados.0.smithi163.stdout:2900: finishing write tid 3 to smithi16367185-50 2023-10-09T09:17:43.081 INFO:tasks.rados.rados.0.smithi163.stdout:2900: finishing write tid 4 to smithi16367185-50 2023-10-09T09:17:43.081 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1367 (ObjNum 1086 snap 362 seq_num 1086) dirty exists 2023-10-09T09:17:43.082 INFO:tasks.rados.rados.0.smithi163.stdout:2900: left oid 50 (ObjNum 1086 snap 362 seq_num 1086) 2023-10-09T09:17:43.082 INFO:tasks.rados.rados.0.smithi163.stdout:2897: done (4 left) 2023-10-09T09:17:43.082 INFO:tasks.rados.rados.0.smithi163.stdout:2898: done (3 left) 2023-10-09T09:17:43.082 INFO:tasks.rados.rados.0.smithi163.stdout:2899: done (2 left) 2023-10-09T09:17:43.082 INFO:tasks.rados.rados.0.smithi163.stdout:2900: done (1 left) 2023-10-09T09:17:43.082 INFO:tasks.rados.rados.0.smithi163.stdout:2901: done (0 left) 2023-10-09T09:17:43.082 INFO:tasks.rados.rados.0.smithi163.stdout:2902: write oid 13 current snap is 362 2023-10-09T09:17:43.082 INFO:tasks.rados.rados.0.smithi163.stdout:2902: seq_num 1088 ranges {686016=509391,1716997=569019} 2023-10-09T09:17:43.094 INFO:tasks.rados.rados.0.smithi163.stdout:2902: writing smithi16367185-13 from 686016 to 1195407 tid 1 2023-10-09T09:17:43.152 INFO:tasks.rados.rados.0.smithi163.stdout:2902: writing smithi16367185-13 from 1716997 to 2286016 tid 2 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:2903: write oid 41 current snap is 362 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:2903: seq_num 1089 ranges {702823=584640,2059126=243697} 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:2903: writing smithi16367185-41 from 702823 to 1287463 tid 1 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:2903: writing smithi16367185-41 from 2059126 to 2302823 tid 2 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:2904: rollback oid 12 current snap is 362 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 12 to 351 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:2902: finishing write tid 1 to smithi16367185-13 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:2905: read oid 11 snap -1 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:2905: expect deleted 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:2906: rollback oid 16 current snap is 362 2023-10-09T09:17:43.153 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 16 to 351 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2902: finishing write tid 2 to smithi16367185-13 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2902: finishing write tid 3 to smithi16367185-13 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2902: finishing write tid 4 to smithi16367185-13 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 499 (ObjNum 1088 snap 362 seq_num 1088) dirty exists 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2902: left oid 13 (ObjNum 1088 snap 362 seq_num 1088) 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2903: finishing write tid 1 to smithi16367185-41 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2902: done (4 left) 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2907: write oid 39 current snap is 362 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2907: seq_num 1090 ranges {719630=659889,2117608=202022} 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2907: writing smithi16367185-39 from 719630 to 1379519 tid 1 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2907: writing smithi16367185-39 from 2117608 to 2319630 tid 2 2023-10-09T09:17:43.154 INFO:tasks.rados.rados.0.smithi163.stdout:2908: rollback oid 25 current snap is 362 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 25 to 353 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:2904: finishing rollback tid 1 to smithi16367185-12 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:2903: finishing write tid 2 to smithi16367185-41 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:2905: done (5 left) 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:2909: write oid 43 current snap is 362 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:2903: finishing write tid 3 to smithi16367185-41 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:2904: finishing rollback tid 2 to smithi16367185-12 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 1378 (ObjNum 953 snap 324 seq_num 953) dirty exists 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:2906: finishing rollback tid 0 to smithi16367185-16 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:2903: finishing write tid 4 to smithi16367185-41 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 279 (ObjNum 1089 snap 362 seq_num 1089) dirty exists 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:2903: left oid 41 (ObjNum 1089 snap 362 seq_num 1089) 2023-10-09T09:17:43.155 INFO:tasks.rados.rados.0.smithi163.stdout:2906: finishing rollback tid 1 to smithi16367185-16 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2906: finishing rollback tid 2 to smithi16367185-16 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1399 (ObjNum 1017 snap 346 seq_num 1017) dirty exists 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2907: finishing write tid 1 to smithi16367185-39 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2909: seq_num 1091 ranges {736437=735138,2176090=160347} 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2909: writing smithi16367185-43 from 736437 to 1471575 tid 1 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2909: writing smithi16367185-43 from 2176090 to 2336437 tid 2 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2903: done (5 left) 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2904: done (4 left) 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2906: done (3 left) 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2910: write oid 8 current snap is 362 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2910: seq_num 1092 ranges {753244=410387,1834572=518672} 2023-10-09T09:17:43.156 INFO:tasks.rados.rados.0.smithi163.stdout:2910: writing smithi16367185-8 from 753244 to 1163631 tid 1 2023-10-09T09:17:43.157 INFO:tasks.rados.rados.0.smithi163.stdout:2910: writing smithi16367185-8 from 1834572 to 2353244 tid 2 2023-10-09T09:17:43.157 INFO:tasks.rados.rados.0.smithi163.stdout:2911: rollback oid 20 current snap is 362 2023-10-09T09:17:43.157 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 354 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:2908: finishing rollback tid 1 to smithi16367185-25 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:2912: delete oid 6 current snap is 362 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:2908: finishing rollback tid 2 to smithi16367185-25 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1380 (ObjNum 1004 snap 342 seq_num 1004) dirty exists 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:2907: finishing write tid 2 to smithi16367185-39 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:2907: finishing write tid 3 to smithi16367185-39 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:2907: finishing write tid 4 to smithi16367185-39 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1370 (ObjNum 1090 snap 362 seq_num 1090) dirty exists 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:2907: left oid 39 (ObjNum 1090 snap 362 seq_num 1090) 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:2909: finishing write tid 1 to smithi16367185-43 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:2909: finishing write tid 2 to smithi16367185-43 2023-10-09T09:17:43.158 INFO:tasks.rados.rados.0.smithi163.stdout:2909: finishing write tid 3 to smithi16367185-43 2023-10-09T09:17:43.159 INFO:tasks.rados.rados.0.smithi163.stdout:2909: finishing write tid 4 to smithi16367185-43 2023-10-09T09:17:43.159 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 1370 (ObjNum 1091 snap 362 seq_num 1091) dirty exists 2023-10-09T09:17:43.159 INFO:tasks.rados.rados.0.smithi163.stdout:2909: left oid 43 (ObjNum 1091 snap 362 seq_num 1091) 2023-10-09T09:17:43.159 INFO:tasks.rados.rados.0.smithi163.stdout:2910: finishing write tid 1 to smithi16367185-8 2023-10-09T09:17:43.160 INFO:tasks.rados.rados.0.smithi163.stdout:2911: finishing rollback tid 0 to smithi16367185-20 2023-10-09T09:17:43.160 INFO:tasks.rados.rados.0.smithi163.stdout:2907: done (5 left) 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:2908: done (4 left) 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:2909: done (3 left) 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:2912: done (2 left) 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:2913: rollback oid 7 current snap is 362 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 351 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:2914: read oid 32 snap -1 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:2914: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:2915: delete oid 12 current snap is 362 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:2911: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:2911: finishing rollback tid 2 to smithi16367185-20 2023-10-09T09:17:43.161 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 503 (ObjNum 994 snap 341 seq_num 994) dirty exists 2023-10-09T09:17:43.162 INFO:tasks.rados.rados.0.smithi163.stdout:2910: finishing write tid 2 to smithi16367185-8 2023-10-09T09:17:43.162 INFO:tasks.rados.rados.0.smithi163.stdout:2910: finishing write tid 3 to smithi16367185-8 2023-10-09T09:17:43.162 INFO:tasks.rados.rados.0.smithi163.stdout:2910: finishing write tid 4 to smithi16367185-8 2023-10-09T09:17:43.162 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1472 (ObjNum 1092 snap 362 seq_num 1092) dirty exists 2023-10-09T09:17:43.162 INFO:tasks.rados.rados.0.smithi163.stdout:2910: left oid 8 (ObjNum 1092 snap 362 seq_num 1092) 2023-10-09T09:17:43.163 INFO:tasks.rados.rados.0.smithi163.stdout:2910: done (4 left) 2023-10-09T09:17:43.164 INFO:tasks.rados.rados.0.smithi163.stdout:2911: done (3 left) 2023-10-09T09:17:43.164 INFO:tasks.rados.rados.0.smithi163.stdout:2915: done (2 left) 2023-10-09T09:17:43.164 INFO:tasks.rados.rados.0.smithi163.stdout:2916: snap_create 2023-10-09T09:17:43.164 INFO:tasks.rados.rados.0.smithi163.stdout:2913: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:17:43.165 INFO:tasks.rados.rados.0.smithi163.stdout:2913: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:17:43.165 INFO:tasks.rados.rados.0.smithi163.stdout:2913: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:17:43.165 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 706 (ObjNum 961 snap 326 seq_num 961) dirty exists 2023-10-09T09:17:43.169 INFO:tasks.rados.rados.0.smithi163.stdout:2914: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:17:43.398 INFO:tasks.rados.rados.0.smithi163.stdout:2913: done (2 left) 2023-10-09T09:17:43.621 INFO:tasks.rados.rados.0.smithi163.stdout:2914: done (1 left) 2023-10-09T09:17:43.621 INFO:tasks.rados.rados.0.smithi163.stdout:2916: done (0 left) 2023-10-09T09:17:43.621 INFO:tasks.rados.rados.0.smithi163.stdout:2917: write oid 3 current snap is 363 2023-10-09T09:17:43.621 INFO:tasks.rados.rados.0.smithi163.stdout:2917: seq_num 1095 ranges {403665=752487,1610018=514716} 2023-10-09T09:17:43.623 INFO:tasks.rados.rados.0.smithi163.stdout:2917: writing smithi16367185-3 from 403665 to 1156152 tid 1 2023-10-09T09:17:43.623 INFO:tasks.rados.rados.0.smithi163.stdout:2917: writing smithi16367185-3 from 1610018 to 2124734 tid 2 2023-10-09T09:17:43.624 INFO:tasks.rados.rados.0.smithi163.stdout:2918: snap_create 2023-10-09T09:17:43.624 INFO:tasks.rados.rados.0.smithi163.stdout:2917: finishing write tid 1 to smithi16367185-3 2023-10-09T09:17:43.624 INFO:tasks.rados.rados.0.smithi163.stdout:2917: finishing write tid 2 to smithi16367185-3 2023-10-09T09:17:43.624 INFO:tasks.rados.rados.0.smithi163.stdout:2917: finishing write tid 3 to smithi16367185-3 2023-10-09T09:17:43.624 INFO:tasks.rados.rados.0.smithi163.stdout:2917: finishing write tid 4 to smithi16367185-3 2023-10-09T09:17:43.624 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 789 (ObjNum 1095 snap 363 seq_num 1095) dirty exists 2023-10-09T09:17:43.624 INFO:tasks.rados.rados.0.smithi163.stdout:2917: left oid 3 (ObjNum 1095 snap 363 seq_num 1095) 2023-10-09T09:17:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:17:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:17:43] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:17:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:43 smithi163 bash[75425]: cluster 2023-10-09T09:17:42.368823+0000 osd.5 (osd.5) 321 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:43 smithi163 bash[75425]: cluster 2023-10-09T09:17:42.369649+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:43 smithi163 bash[75425]: cluster 2023-10-09T09:17:42.374948+0000 mon.a (mon.0) 973 : cluster 0 osdmap e789: 8 total, 8 up, 8 in 2023-10-09T09:17:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:43 smithi163 bash[75425]: cluster 2023-10-09T09:17:42.379093+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:43 smithi163 bash[78107]: cluster 2023-10-09T09:17:42.368823+0000 osd.5 (osd.5) 321 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:43 smithi163 bash[78107]: cluster 2023-10-09T09:17:42.369649+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:43 smithi163 bash[78107]: cluster 2023-10-09T09:17:42.374948+0000 mon.a (mon.0) 973 : cluster 0 osdmap e789: 8 total, 8 up, 8 in 2023-10-09T09:17:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:43 smithi163 bash[78107]: cluster 2023-10-09T09:17:42.379093+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:43 smithi181 bash[59345]: cluster 2023-10-09T09:17:42.368823+0000 osd.5 (osd.5) 321 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:43 smithi181 bash[59345]: cluster 2023-10-09T09:17:42.369649+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:43 smithi181 bash[59345]: cluster 2023-10-09T09:17:42.374948+0000 mon.a (mon.0) 973 : cluster 0 osdmap e789: 8 total, 8 up, 8 in 2023-10-09T09:17:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:43 smithi181 bash[59345]: cluster 2023-10-09T09:17:42.379093+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:44.414 INFO:tasks.rados.rados.0.smithi163.stdout:2917: done (1 left) 2023-10-09T09:17:44.414 INFO:tasks.rados.rados.0.smithi163.stdout:2918: done (0 left) 2023-10-09T09:17:44.415 INFO:tasks.rados.rados.0.smithi163.stdout:2919: delete oid 47 current snap is 364 2023-10-09T09:17:44.422 INFO:tasks.rados.rados.0.smithi163.stdout:2919: done (0 left) 2023-10-09T09:17:44.422 INFO:tasks.rados.rados.0.smithi163.stdout:2920: rollback oid 10 current snap is 364 2023-10-09T09:17:44.422 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 351 2023-10-09T09:17:44.422 INFO:tasks.rados.rados.0.smithi163.stdout:2921: snap_create 2023-10-09T09:17:44.429 INFO:tasks.rados.rados.0.smithi163.stdout:2920: finishing rollback tid 0 to smithi16367185-10 2023-10-09T09:17:44.429 INFO:tasks.rados.rados.0.smithi163.stdout:2920: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:17:44.429 INFO:tasks.rados.rados.0.smithi163.stdout:2920: finishing rollback tid 2 to smithi16367185-10 2023-10-09T09:17:44.429 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 931 (ObjNum 977 snap 337 seq_num 977) dirty exists 2023-10-09T09:17:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:44 smithi163 bash[75425]: cluster 2023-10-09T09:17:42.510466+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:44 smithi163 bash[75425]: cluster 2023-10-09T09:17:43.396164+0000 osd.5 (osd.5) 322 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:44 smithi163 bash[75425]: cluster 2023-10-09T09:17:43.397255+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:44 smithi163 bash[75425]: cluster 2023-10-09T09:17:43.397349+0000 mgr.y (mgr.34107) 335 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 231 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.7 MiB/s wr, 21 op/s 2023-10-09T09:17:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:44 smithi163 bash[75425]: cluster 2023-10-09T09:17:43.398227+0000 mon.a (mon.0) 974 : cluster 0 osdmap e790: 8 total, 8 up, 8 in 2023-10-09T09:17:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:44 smithi163 bash[78107]: cluster 2023-10-09T09:17:42.510466+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:44 smithi163 bash[78107]: cluster 2023-10-09T09:17:43.396164+0000 osd.5 (osd.5) 322 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:44 smithi163 bash[78107]: cluster 2023-10-09T09:17:43.397255+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:44 smithi163 bash[78107]: cluster 2023-10-09T09:17:43.397349+0000 mgr.y (mgr.34107) 335 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 231 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.7 MiB/s wr, 21 op/s 2023-10-09T09:17:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:44 smithi163 bash[78107]: cluster 2023-10-09T09:17:43.398227+0000 mon.a (mon.0) 974 : cluster 0 osdmap e790: 8 total, 8 up, 8 in 2023-10-09T09:17:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:44 smithi181 bash[59345]: cluster 2023-10-09T09:17:42.510466+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e789 with expected crc 2023-10-09T09:17:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:44 smithi181 bash[59345]: cluster 2023-10-09T09:17:43.396164+0000 osd.5 (osd.5) 322 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:44 smithi181 bash[59345]: cluster 2023-10-09T09:17:43.397255+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:44 smithi181 bash[59345]: cluster 2023-10-09T09:17:43.397349+0000 mgr.y (mgr.34107) 335 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 231 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.7 MiB/s wr, 21 op/s 2023-10-09T09:17:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:44 smithi181 bash[59345]: cluster 2023-10-09T09:17:43.398227+0000 mon.a (mon.0) 974 : cluster 0 osdmap e790: 8 total, 8 up, 8 in 2023-10-09T09:17:45.205 INFO:tasks.rados.rados.0.smithi163.stdout:2920: done (1 left) 2023-10-09T09:17:45.206 INFO:tasks.rados.rados.0.smithi163.stdout:2921: done (0 left) 2023-10-09T09:17:45.206 INFO:tasks.rados.rados.0.smithi163.stdout:2922: read oid 35 snap -1 2023-10-09T09:17:45.206 INFO:tasks.rados.rados.0.smithi163.stdout:2922: expect (ObjNum 1066 snap 360 seq_num 1066) 2023-10-09T09:17:45.206 INFO:tasks.rados.rados.0.smithi163.stdout:2923: read oid 48 snap 353 2023-10-09T09:17:45.206 INFO:tasks.rados.rados.0.smithi163.stdout:2923: expect (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:17:45.206 INFO:tasks.rados.rados.0.smithi163.stdout:2924: write oid 46 current snap is 365 2023-10-09T09:17:45.206 INFO:tasks.rados.rados.0.smithi163.stdout:2924: seq_num 1097 ranges {437279=502985,1610629=518385} 2023-10-09T09:17:45.213 INFO:tasks.rados.rados.0.smithi163.stdout:2924: writing smithi16367185-46 from 437279 to 940264 tid 1 2023-10-09T09:17:45.220 INFO:tasks.rados.rados.0.smithi163.stdout:2924: writing smithi16367185-46 from 1610629 to 2129014 tid 2 2023-10-09T09:17:45.220 INFO:tasks.rados.rados.0.smithi163.stdout:2925: read oid 4 snap 354 2023-10-09T09:17:45.220 INFO:tasks.rados.rados.0.smithi163.stdout:2925: expect (ObjNum 1049 snap 354 seq_num 1049) 2023-10-09T09:17:45.220 INFO:tasks.rados.rados.0.smithi163.stdout:2926: write oid 22 current snap is 365 2023-10-09T09:17:45.221 INFO:tasks.rados.rados.0.smithi163.stdout:2926: seq_num 1098 ranges {454086=578234,1669111=662043} 2023-10-09T09:17:45.227 INFO:tasks.rados.rados.0.smithi163.stdout:2926: writing smithi16367185-22 from 454086 to 1032320 tid 1 2023-10-09T09:17:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:2926: writing smithi16367185-22 from 1669111 to 2331154 tid 2 2023-10-09T09:17:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:2927: write oid 49 current snap is 365 2023-10-09T09:17:45.235 INFO:tasks.rados.rados.0.smithi163.stdout:2927: seq_num 1099 ranges {470893=653483,1727593=522054} 2023-10-09T09:17:45.242 INFO:tasks.rados.rados.0.smithi163.stdout:2927: writing smithi16367185-49 from 470893 to 1124376 tid 1 2023-10-09T09:17:45.248 INFO:tasks.rados.rados.0.smithi163.stdout:2927: writing smithi16367185-49 from 1727593 to 2249647 tid 2 2023-10-09T09:17:45.248 INFO:tasks.rados.rados.0.smithi163.stdout:2928: read oid 44 snap -1 2023-10-09T09:17:45.248 INFO:tasks.rados.rados.0.smithi163.stdout:2928: expect (ObjNum 1056 snap 355 seq_num 1056) 2023-10-09T09:17:45.248 INFO:tasks.rados.rados.0.smithi163.stdout:2929: snap_remove snap 351 2023-10-09T09:17:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[75425]: cluster 2023-10-09T09:17:44.409504+0000 osd.5 (osd.5) 323 : cluster 3 failed to encode map e791 with expected crc 2023-10-09T09:17:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[75425]: cluster 2023-10-09T09:17:44.409530+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[75425]: cluster 2023-10-09T09:17:44.409967+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e791 with expected crc 2023-10-09T09:17:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[75425]: cluster 2023-10-09T09:17:44.410976+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[75425]: cluster 2023-10-09T09:17:44.412255+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[75425]: cluster 2023-10-09T09:17:44.414426+0000 mon.a (mon.0) 975 : cluster 0 osdmap e791: 8 total, 8 up, 8 in 2023-10-09T09:17:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[75425]: cluster 2023-10-09T09:17:45.204914+0000 mon.a (mon.0) 976 : cluster 0 osdmap e792: 8 total, 8 up, 8 in 2023-10-09T09:17:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[75425]: cluster 2023-10-09T09:17:45.214974+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e792 with expected crc 2023-10-09T09:17:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[78107]: cluster 2023-10-09T09:17:44.409504+0000 osd.5 (osd.5) 323 : cluster 3 failed to encode map e791 with expected crc 2023-10-09T09:17:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[78107]: cluster 2023-10-09T09:17:44.409530+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[78107]: cluster 2023-10-09T09:17:44.409967+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e791 with expected crc 2023-10-09T09:17:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[78107]: cluster 2023-10-09T09:17:44.410976+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[78107]: cluster 2023-10-09T09:17:44.412255+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[78107]: cluster 2023-10-09T09:17:44.414426+0000 mon.a (mon.0) 975 : cluster 0 osdmap e791: 8 total, 8 up, 8 in 2023-10-09T09:17:45.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[78107]: cluster 2023-10-09T09:17:45.204914+0000 mon.a (mon.0) 976 : cluster 0 osdmap e792: 8 total, 8 up, 8 in 2023-10-09T09:17:45.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:45 smithi163 bash[78107]: cluster 2023-10-09T09:17:45.214974+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e792 with expected crc 2023-10-09T09:17:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:45 smithi181 bash[59345]: cluster 2023-10-09T09:17:44.409504+0000 osd.5 (osd.5) 323 : cluster 3 failed to encode map e791 with expected crc 2023-10-09T09:17:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:45 smithi181 bash[59345]: cluster 2023-10-09T09:17:44.409530+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:45 smithi181 bash[59345]: cluster 2023-10-09T09:17:44.409967+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e791 with expected crc 2023-10-09T09:17:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:45 smithi181 bash[59345]: cluster 2023-10-09T09:17:44.410976+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:45 smithi181 bash[59345]: cluster 2023-10-09T09:17:44.412255+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:45 smithi181 bash[59345]: cluster 2023-10-09T09:17:44.414426+0000 mon.a (mon.0) 975 : cluster 0 osdmap e791: 8 total, 8 up, 8 in 2023-10-09T09:17:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:45 smithi181 bash[59345]: cluster 2023-10-09T09:17:45.204914+0000 mon.a (mon.0) 976 : cluster 0 osdmap e792: 8 total, 8 up, 8 in 2023-10-09T09:17:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:45 smithi181 bash[59345]: cluster 2023-10-09T09:17:45.214974+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e792 with expected crc 2023-10-09T09:17:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:2929: done (7 left) 2023-10-09T09:17:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:2930: rollback oid 31 current snap is 365 2023-10-09T09:17:46.219 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 31 to 338 2023-10-09T09:17:46.220 INFO:tasks.rados.rados.0.smithi163.stdout:2931: snap_remove snap 363 2023-10-09T09:17:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[75425]: cluster 2023-10-09T09:17:44.569437+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[75425]: cluster 2023-10-09T09:17:45.200196+0000 osd.5 (osd.5) 324 : cluster 3 failed to encode map e792 with expected crc 2023-10-09T09:17:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[75425]: cluster 2023-10-09T09:17:45.206675+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e792 with expected crc 2023-10-09T09:17:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[75425]: cluster 2023-10-09T09:17:45.398138+0000 mgr.y (mgr.34107) 336 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 236 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 404 KiB/s wr, 11 op/s 2023-10-09T09:17:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[75425]: cluster 2023-10-09T09:17:46.218679+0000 mon.a (mon.0) 977 : cluster 0 osdmap e793: 8 total, 8 up, 8 in 2023-10-09T09:17:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[75425]: cluster 2023-10-09T09:17:46.220617+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e793 with expected crc 2023-10-09T09:17:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[78107]: cluster 2023-10-09T09:17:44.569437+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[78107]: cluster 2023-10-09T09:17:45.200196+0000 osd.5 (osd.5) 324 : cluster 3 failed to encode map e792 with expected crc 2023-10-09T09:17:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[78107]: cluster 2023-10-09T09:17:45.206675+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e792 with expected crc 2023-10-09T09:17:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[78107]: cluster 2023-10-09T09:17:45.398138+0000 mgr.y (mgr.34107) 336 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 236 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 404 KiB/s wr, 11 op/s 2023-10-09T09:17:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[78107]: cluster 2023-10-09T09:17:46.218679+0000 mon.a (mon.0) 977 : cluster 0 osdmap e793: 8 total, 8 up, 8 in 2023-10-09T09:17:46.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:46 smithi163 bash[78107]: cluster 2023-10-09T09:17:46.220617+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e793 with expected crc 2023-10-09T09:17:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:46 smithi181 bash[59345]: cluster 2023-10-09T09:17:44.569437+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e790 with expected crc 2023-10-09T09:17:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:46 smithi181 bash[59345]: cluster 2023-10-09T09:17:45.200196+0000 osd.5 (osd.5) 324 : cluster 3 failed to encode map e792 with expected crc 2023-10-09T09:17:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:46 smithi181 bash[59345]: cluster 2023-10-09T09:17:45.206675+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e792 with expected crc 2023-10-09T09:17:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:46 smithi181 bash[59345]: cluster 2023-10-09T09:17:45.398138+0000 mgr.y (mgr.34107) 336 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 236 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 404 KiB/s wr, 11 op/s 2023-10-09T09:17:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:46 smithi181 bash[59345]: cluster 2023-10-09T09:17:46.218679+0000 mon.a (mon.0) 977 : cluster 0 osdmap e793: 8 total, 8 up, 8 in 2023-10-09T09:17:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:46 smithi181 bash[59345]: cluster 2023-10-09T09:17:46.220617+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e793 with expected crc 2023-10-09T09:17:47.225 INFO:tasks.rados.rados.0.smithi163.stdout:2931: done (8 left) 2023-10-09T09:17:47.225 INFO:tasks.rados.rados.0.smithi163.stdout:2932: read oid 7 snap -1 2023-10-09T09:17:47.225 INFO:tasks.rados.rados.0.smithi163.stdout:2932: expect (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:17:47.225 INFO:tasks.rados.rados.0.smithi163.stdout:2933: read oid 38 snap -1 2023-10-09T09:17:47.225 INFO:tasks.rados.rados.0.smithi163.stdout:2933: expect (ObjNum 1081 snap 362 seq_num 1081) 2023-10-09T09:17:47.226 INFO:tasks.rados.rados.0.smithi163.stdout:2934: delete oid 14 current snap is 365 2023-10-09T09:17:47.226 INFO:tasks.rados.rados.0.smithi163.stdout:2923: expect (ObjNum 855 snap 291 seq_num 855) 2023-10-09T09:17:47.397 INFO:tasks.rados.rados.0.smithi163.stdout:2924: finishing write tid 1 to smithi16367185-46 2023-10-09T09:17:47.397 INFO:tasks.rados.rados.0.smithi163.stdout:2924: finishing write tid 2 to smithi16367185-46 2023-10-09T09:17:47.398 INFO:tasks.rados.rados.0.smithi163.stdout:2924: finishing write tid 3 to smithi16367185-46 2023-10-09T09:17:47.398 INFO:tasks.rados.rados.0.smithi163.stdout:2924: finishing write tid 4 to smithi16367185-46 2023-10-09T09:17:47.398 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 1373 (ObjNum 1097 snap 365 seq_num 1097) dirty exists 2023-10-09T09:17:47.398 INFO:tasks.rados.rados.0.smithi163.stdout:2924: left oid 46 (ObjNum 1097 snap 365 seq_num 1097) 2023-10-09T09:17:47.398 INFO:tasks.rados.rados.0.smithi163.stdout:2922: expect (ObjNum 1066 snap 360 seq_num 1066) 2023-10-09T09:17:47.643 INFO:tasks.rados.rados.0.smithi163.stdout:2926: finishing write tid 1 to smithi16367185-22 2023-10-09T09:17:47.643 INFO:tasks.rados.rados.0.smithi163.stdout:2925: expect (ObjNum 1049 snap 354 seq_num 1049) 2023-10-09T09:17:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:47 smithi163 bash[75425]: cluster 2023-10-09T09:17:46.215897+0000 osd.5 (osd.5) 325 : cluster 3 failed to encode map e793 with expected crc 2023-10-09T09:17:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:47 smithi163 bash[75425]: cluster 2023-10-09T09:17:47.223178+0000 mon.a (mon.0) 978 : cluster 0 osdmap e794: 8 total, 8 up, 8 in 2023-10-09T09:17:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:47 smithi163 bash[75425]: cluster 2023-10-09T09:17:47.226378+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:47 smithi163 bash[78107]: cluster 2023-10-09T09:17:46.215897+0000 osd.5 (osd.5) 325 : cluster 3 failed to encode map e793 with expected crc 2023-10-09T09:17:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:47 smithi163 bash[78107]: cluster 2023-10-09T09:17:47.223178+0000 mon.a (mon.0) 978 : cluster 0 osdmap e794: 8 total, 8 up, 8 in 2023-10-09T09:17:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:47 smithi163 bash[78107]: cluster 2023-10-09T09:17:47.226378+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:47.804 INFO:tasks.rados.rados.0.smithi163.stdout:2926: finishing write tid 2 to smithi16367185-22 2023-10-09T09:17:47.804 INFO:tasks.rados.rados.0.smithi163.stdout:2926: finishing write tid 3 to smithi16367185-22 2023-10-09T09:17:47.804 INFO:tasks.rados.rados.0.smithi163.stdout:2926: finishing write tid 4 to smithi16367185-22 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 2067 (ObjNum 1098 snap 365 seq_num 1098) dirty exists 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:2926: left oid 22 (ObjNum 1098 snap 365 seq_num 1098) 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:2927: finishing write tid 1 to smithi16367185-49 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:2922: done (10 left) 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:2923: done (9 left) 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:2924: done (8 left) 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:2925: done (7 left) 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:2926: done (6 left) 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:2934: done (5 left) 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:2935: write oid 26 current snap is 365 2023-10-09T09:17:47.805 INFO:tasks.rados.rados.0.smithi163.stdout:2935: seq_num 1101 ranges {504507=403981,1328204=409370,2252362=252145} 2023-10-09T09:17:47.809 INFO:tasks.rados.rados.0.smithi163.stdout:2935: writing smithi16367185-26 from 504507 to 908488 tid 1 2023-10-09T09:17:47.815 INFO:tasks.rados.rados.0.smithi163.stdout:2935: writing smithi16367185-26 from 1328204 to 1737574 tid 2 2023-10-09T09:17:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:47 smithi181 bash[59345]: cluster 2023-10-09T09:17:46.215897+0000 osd.5 (osd.5) 325 : cluster 3 failed to encode map e793 with expected crc 2023-10-09T09:17:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:47 smithi181 bash[59345]: cluster 2023-10-09T09:17:47.223178+0000 mon.a (mon.0) 978 : cluster 0 osdmap e794: 8 total, 8 up, 8 in 2023-10-09T09:17:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:47 smithi181 bash[59345]: cluster 2023-10-09T09:17:47.226378+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:47.819 INFO:tasks.rados.rados.0.smithi163.stdout:2935: writing smithi16367185-26 from 2252362 to 2504507 tid 3 2023-10-09T09:17:47.820 INFO:tasks.rados.rados.0.smithi163.stdout:2936: delete oid 28 current snap is 365 2023-10-09T09:17:47.820 INFO:tasks.rados.rados.0.smithi163.stdout:2927: finishing write tid 2 to smithi16367185-49 2023-10-09T09:17:47.820 INFO:tasks.rados.rados.0.smithi163.stdout:2927: finishing write tid 3 to smithi16367185-49 2023-10-09T09:17:47.820 INFO:tasks.rados.rados.0.smithi163.stdout:2927: finishing write tid 4 to smithi16367185-49 2023-10-09T09:17:47.820 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 335 (ObjNum 1099 snap 365 seq_num 1099) dirty exists 2023-10-09T09:17:47.820 INFO:tasks.rados.rados.0.smithi163.stdout:2927: left oid 49 (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:17:47.820 INFO:tasks.rados.rados.0.smithi163.stdout:2928: expect (ObjNum 1056 snap 355 seq_num 1056) 2023-10-09T09:17:47.990 INFO:tasks.rados.rados.0.smithi163.stdout:2930: finishing rollback tid 1 to smithi16367185-31 2023-10-09T09:17:47.990 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 0 (ObjNum 699377888 snap 21875 seq_num 699173760) dirty dne 2023-10-09T09:17:47.990 INFO:tasks.rados.rados.0.smithi163.stdout:2932: expect (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:17:48.156 INFO:tasks.rados.rados.0.smithi163.stdout:2933: expect (ObjNum 1081 snap 362 seq_num 1081) 2023-10-09T09:17:48.251 INFO:tasks.rados.rados.0.smithi163.stdout:2935: finishing write tid 1 to smithi16367185-26 2023-10-09T09:17:48.252 INFO:tasks.rados.rados.0.smithi163.stdout:2935: finishing write tid 2 to smithi16367185-26 2023-10-09T09:17:48.252 INFO:tasks.rados.rados.0.smithi163.stdout:2935: finishing write tid 3 to smithi16367185-26 2023-10-09T09:17:48.252 INFO:tasks.rados.rados.0.smithi163.stdout:2935: finishing write tid 4 to smithi16367185-26 2023-10-09T09:17:48.252 INFO:tasks.rados.rados.0.smithi163.stdout:2935: finishing write tid 5 to smithi16367185-26 2023-10-09T09:17:48.252 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 847 (ObjNum 1101 snap 365 seq_num 1101) dirty exists 2023-10-09T09:17:48.252 INFO:tasks.rados.rados.0.smithi163.stdout:2935: left oid 26 (ObjNum 1101 snap 365 seq_num 1101) 2023-10-09T09:17:48.252 INFO:tasks.rados.rados.0.smithi163.stdout:2927: done (6 left) 2023-10-09T09:17:48.252 INFO:tasks.rados.rados.0.smithi163.stdout:2928: done (5 left) 2023-10-09T09:17:48.252 INFO:tasks.rados.rados.0.smithi163.stdout:2930: done (4 left) 2023-10-09T09:17:48.253 INFO:tasks.rados.rados.0.smithi163.stdout:2932: done (3 left) 2023-10-09T09:17:48.253 INFO:tasks.rados.rados.0.smithi163.stdout:2933: done (2 left) 2023-10-09T09:17:48.253 INFO:tasks.rados.rados.0.smithi163.stdout:2935: done (1 left) 2023-10-09T09:17:48.253 INFO:tasks.rados.rados.0.smithi163.stdout:2936: done (0 left) 2023-10-09T09:17:48.253 INFO:tasks.rados.rados.0.smithi163.stdout:2937: write oid 31 current snap is 365 2023-10-09T09:17:48.253 INFO:tasks.rados.rados.0.smithi163.stdout:2937: seq_num 1103 ranges {538121=670832,1961521=413039} 2023-10-09T09:17:48.264 INFO:tasks.rados.rados.0.smithi163.stdout:2937: writing smithi16367185-31 from 538121 to 1208953 tid 1 2023-10-09T09:17:48.272 INFO:tasks.rados.rados.0.smithi163.stdout:2937: writing smithi16367185-31 from 1961521 to 2374560 tid 2 2023-10-09T09:17:48.272 INFO:tasks.rados.rados.0.smithi163.stdout:2938: snap_create 2023-10-09T09:17:48.272 INFO:tasks.rados.rados.0.smithi163.stdout:2937: finishing write tid 1 to smithi16367185-31 2023-10-09T09:17:48.276 INFO:tasks.rados.rados.0.smithi163.stdout:2937: finishing write tid 2 to smithi16367185-31 2023-10-09T09:17:48.276 INFO:tasks.rados.rados.0.smithi163.stdout:2937: finishing write tid 3 to smithi16367185-31 2023-10-09T09:17:48.276 INFO:tasks.rados.rados.0.smithi163.stdout:2937: finishing write tid 4 to smithi16367185-31 2023-10-09T09:17:48.276 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 797 (ObjNum 1103 snap 365 seq_num 1103) dirty exists 2023-10-09T09:17:48.276 INFO:tasks.rados.rados.0.smithi163.stdout:2937: left oid 31 (ObjNum 1103 snap 365 seq_num 1103) 2023-10-09T09:17:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:48 smithi163 bash[78107]: cluster 2023-10-09T09:17:47.218035+0000 osd.5 (osd.5) 326 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:48 smithi163 bash[78107]: cluster 2023-10-09T09:17:47.224517+0000 osd.5 (osd.5) 327 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:48 smithi163 bash[78107]: cluster 2023-10-09T09:17:47.224609+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:48 smithi163 bash[78107]: cluster 2023-10-09T09:17:47.398779+0000 mgr.y (mgr.34107) 337 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.5 MiB/s wr, 33 op/s 2023-10-09T09:17:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:48 smithi163 bash[75425]: cluster 2023-10-09T09:17:47.218035+0000 osd.5 (osd.5) 326 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:48 smithi163 bash[75425]: cluster 2023-10-09T09:17:47.224517+0000 osd.5 (osd.5) 327 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:48 smithi163 bash[75425]: cluster 2023-10-09T09:17:47.224609+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:48 smithi163 bash[75425]: cluster 2023-10-09T09:17:47.398779+0000 mgr.y (mgr.34107) 337 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.5 MiB/s wr, 33 op/s 2023-10-09T09:17:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:48 smithi181 bash[59345]: cluster 2023-10-09T09:17:47.218035+0000 osd.5 (osd.5) 326 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:48 smithi181 bash[59345]: cluster 2023-10-09T09:17:47.224517+0000 osd.5 (osd.5) 327 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:48 smithi181 bash[59345]: cluster 2023-10-09T09:17:47.224609+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e794 with expected crc 2023-10-09T09:17:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:48 smithi181 bash[59345]: cluster 2023-10-09T09:17:47.398779+0000 mgr.y (mgr.34107) 337 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.5 MiB/s wr, 33 op/s 2023-10-09T09:17:49.227 INFO:tasks.rados.rados.0.smithi163.stdout:2937: done (1 left) 2023-10-09T09:17:49.228 INFO:tasks.rados.rados.0.smithi163.stdout:2938: done (0 left) 2023-10-09T09:17:49.228 INFO:tasks.rados.rados.0.smithi163.stdout:2939: write oid 23 current snap is 366 2023-10-09T09:17:49.228 INFO:tasks.rados.rados.0.smithi163.stdout:2939: seq_num 1104 ranges {554928=746081,2020003=534925} 2023-10-09T09:17:49.238 INFO:tasks.rados.rados.0.smithi163.stdout:2939: writing smithi16367185-23 from 554928 to 1301009 tid 1 2023-10-09T09:17:49.248 INFO:tasks.rados.rados.0.smithi163.stdout:2939: writing smithi16367185-23 from 2020003 to 2554928 tid 2 2023-10-09T09:17:49.248 INFO:tasks.rados.rados.0.smithi163.stdout:2940: snap_remove snap 327 2023-10-09T09:17:50.204 INFO:tasks.rados.rados.0.smithi163.stdout:2940: done (1 left) 2023-10-09T09:17:50.204 INFO:tasks.rados.rados.0.smithi163.stdout:2941: snap_create 2023-10-09T09:17:50.204 INFO:tasks.rados.rados.0.smithi163.stdout:2939: finishing write tid 1 to smithi16367185-23 2023-10-09T09:17:50.204 INFO:tasks.rados.rados.0.smithi163.stdout:2939: finishing write tid 2 to smithi16367185-23 2023-10-09T09:17:50.204 INFO:tasks.rados.rados.0.smithi163.stdout:2939: finishing write tid 3 to smithi16367185-23 2023-10-09T09:17:50.205 INFO:tasks.rados.rados.0.smithi163.stdout:2939: finishing write tid 4 to smithi16367185-23 2023-10-09T09:17:50.205 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1377 (ObjNum 1104 snap 366 seq_num 1104) dirty exists 2023-10-09T09:17:50.205 INFO:tasks.rados.rados.0.smithi163.stdout:2939: left oid 23 (ObjNum 1104 snap 366 seq_num 1104) 2023-10-09T09:17:50.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:50 smithi181 bash[59345]: cluster 2023-10-09T09:17:49.222340+0000 osd.5 (osd.5) 328 : cluster 3 failed to encode map e795 with expected crc 2023-10-09T09:17:50.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:50 smithi181 bash[59345]: cluster 2023-10-09T09:17:49.227658+0000 mon.a (mon.0) 979 : cluster 0 osdmap e795: 8 total, 8 up, 8 in 2023-10-09T09:17:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:50 smithi181 bash[59345]: cluster 2023-10-09T09:17:49.399485+0000 mgr.y (mgr.34107) 338 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 2.3 MiB/s wr, 34 op/s 2023-10-09T09:17:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:50 smithi181 bash[59345]: audit 2023-10-09T09:17:50.154686+0000 mon.a (mon.0) 980 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:17:50.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:50 smithi181 bash[59345]: cluster 2023-10-09T09:17:50.204043+0000 mon.a (mon.0) 981 : cluster 0 osdmap e796: 8 total, 8 up, 8 in 2023-10-09T09:17:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:50 smithi163 bash[78107]: cluster 2023-10-09T09:17:49.222340+0000 osd.5 (osd.5) 328 : cluster 3 failed to encode map e795 with expected crc 2023-10-09T09:17:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:50 smithi163 bash[78107]: cluster 2023-10-09T09:17:49.227658+0000 mon.a (mon.0) 979 : cluster 0 osdmap e795: 8 total, 8 up, 8 in 2023-10-09T09:17:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:50 smithi163 bash[78107]: cluster 2023-10-09T09:17:49.399485+0000 mgr.y (mgr.34107) 338 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 2.3 MiB/s wr, 34 op/s 2023-10-09T09:17:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:50 smithi163 bash[78107]: audit 2023-10-09T09:17:50.154686+0000 mon.a (mon.0) 980 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:17:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:50 smithi163 bash[78107]: cluster 2023-10-09T09:17:50.204043+0000 mon.a (mon.0) 981 : cluster 0 osdmap e796: 8 total, 8 up, 8 in 2023-10-09T09:17:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:50 smithi163 bash[75425]: cluster 2023-10-09T09:17:49.222340+0000 osd.5 (osd.5) 328 : cluster 3 failed to encode map e795 with expected crc 2023-10-09T09:17:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:50 smithi163 bash[75425]: cluster 2023-10-09T09:17:49.227658+0000 mon.a (mon.0) 979 : cluster 0 osdmap e795: 8 total, 8 up, 8 in 2023-10-09T09:17:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:50 smithi163 bash[75425]: cluster 2023-10-09T09:17:49.399485+0000 mgr.y (mgr.34107) 338 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 2.3 MiB/s wr, 34 op/s 2023-10-09T09:17:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:50 smithi163 bash[75425]: audit 2023-10-09T09:17:50.154686+0000 mon.a (mon.0) 980 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:17:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:50 smithi163 bash[75425]: cluster 2023-10-09T09:17:50.204043+0000 mon.a (mon.0) 981 : cluster 0 osdmap e796: 8 total, 8 up, 8 in 2023-10-09T09:17:51.211 INFO:tasks.rados.rados.0.smithi163.stdout:2939: done (1 left) 2023-10-09T09:17:51.211 INFO:tasks.rados.rados.0.smithi163.stdout:2941: done (0 left) 2023-10-09T09:17:51.211 INFO:tasks.rados.rados.0.smithi163.stdout:2942: rollback oid 14 current snap is 367 2023-10-09T09:17:51.211 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 358 2023-10-09T09:17:51.211 INFO:tasks.rados.rados.0.smithi163.stdout:2943: read oid 17 snap -1 2023-10-09T09:17:51.211 INFO:tasks.rados.rados.0.smithi163.stdout:2943: expect deleted 2023-10-09T09:17:51.211 INFO:tasks.rados.rados.0.smithi163.stdout:2944: snap_remove snap 359 2023-10-09T09:17:51.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:51 smithi181 bash[59345]: cluster 2023-10-09T09:17:50.201570+0000 osd.5 (osd.5) 329 : cluster 3 failed to encode map e796 with expected crc 2023-10-09T09:17:51.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:51 smithi181 bash[59345]: cluster 2023-10-09T09:17:50.202460+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e796 with expected crc 2023-10-09T09:17:51.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:51 smithi181 bash[59345]: cluster 2023-10-09T09:17:51.210022+0000 mon.a (mon.0) 982 : cluster 0 osdmap e797: 8 total, 8 up, 8 in 2023-10-09T09:17:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:51 smithi163 bash[78107]: cluster 2023-10-09T09:17:50.201570+0000 osd.5 (osd.5) 329 : cluster 3 failed to encode map e796 with expected crc 2023-10-09T09:17:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:51 smithi163 bash[78107]: cluster 2023-10-09T09:17:50.202460+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e796 with expected crc 2023-10-09T09:17:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:51 smithi163 bash[78107]: cluster 2023-10-09T09:17:51.210022+0000 mon.a (mon.0) 982 : cluster 0 osdmap e797: 8 total, 8 up, 8 in 2023-10-09T09:17:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:51 smithi163 bash[75425]: cluster 2023-10-09T09:17:50.201570+0000 osd.5 (osd.5) 329 : cluster 3 failed to encode map e796 with expected crc 2023-10-09T09:17:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:51 smithi163 bash[75425]: cluster 2023-10-09T09:17:50.202460+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e796 with expected crc 2023-10-09T09:17:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:51 smithi163 bash[75425]: cluster 2023-10-09T09:17:51.210022+0000 mon.a (mon.0) 982 : cluster 0 osdmap e797: 8 total, 8 up, 8 in 2023-10-09T09:17:52.212 INFO:tasks.rados.rados.0.smithi163.stdout:2944: done (2 left) 2023-10-09T09:17:52.212 INFO:tasks.rados.rados.0.smithi163.stdout:2945: delete oid 46 current snap is 367 2023-10-09T09:17:52.212 INFO:tasks.rados.rados.0.smithi163.stdout:2942: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:17:52.212 INFO:tasks.rados.rados.0.smithi163.stdout:2942: finishing rollback tid 2 to smithi16367185-14 2023-10-09T09:17:52.212 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1385 (ObjNum 1041 snap 352 seq_num 1041) dirty exists 2023-10-09T09:17:52.226 INFO:tasks.rados.rados.0.smithi163.stdout:2942: done (2 left) 2023-10-09T09:17:52.226 INFO:tasks.rados.rados.0.smithi163.stdout:2943: done (1 left) 2023-10-09T09:17:52.226 INFO:tasks.rados.rados.0.smithi163.stdout:2945: done (0 left) 2023-10-09T09:17:52.226 INFO:tasks.rados.rados.0.smithi163.stdout:2946: delete oid 24 current snap is 367 2023-10-09T09:17:52.230 INFO:tasks.rados.rados.0.smithi163.stdout:2946: done (0 left) 2023-10-09T09:17:52.230 INFO:tasks.rados.rados.0.smithi163.stdout:2947: delete oid 15 current snap is 367 2023-10-09T09:17:52.233 INFO:tasks.rados.rados.0.smithi163.stdout:2947: done (0 left) 2023-10-09T09:17:52.233 INFO:tasks.rados.rados.0.smithi163.stdout:2948: rollback oid 32 current snap is 367 2023-10-09T09:17:52.233 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 358 2023-10-09T09:17:52.233 INFO:tasks.rados.rados.0.smithi163.stdout:2949: write oid 48 current snap is 367 2023-10-09T09:17:52.233 INFO:tasks.rados.rados.0.smithi163.stdout:2949: seq_num 1108 ranges {622156=647077,1737578=680388} 2023-10-09T09:17:52.240 INFO:tasks.rados.rados.0.smithi163.stdout:2949: writing smithi16367185-48 from 622156 to 1269233 tid 1 2023-10-09T09:17:52.247 INFO:tasks.rados.rados.0.smithi163.stdout:2949: writing smithi16367185-48 from 1737578 to 2417966 tid 2 2023-10-09T09:17:52.247 INFO:tasks.rados.rados.0.smithi163.stdout:2950: write oid 28 current snap is 367 2023-10-09T09:17:52.248 INFO:tasks.rados.rados.0.smithi163.stdout:2948: finishing rollback tid 0 to smithi16367185-32 2023-10-09T09:17:52.248 INFO:tasks.rados.rados.0.smithi163.stdout:2948: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:17:52.248 INFO:tasks.rados.rados.0.smithi163.stdout:2950: seq_num 1109 ranges {638963=722326,2079707=424046} 2023-10-09T09:17:52.257 INFO:tasks.rados.rados.0.smithi163.stdout:2950: writing smithi16367185-28 from 638963 to 1361289 tid 1 2023-10-09T09:17:52.264 INFO:tasks.rados.rados.0.smithi163.stdout:2950: writing smithi16367185-28 from 2079707 to 2503753 tid 2 2023-10-09T09:17:52.264 INFO:tasks.rados.rados.0.smithi163.stdout:2951: read oid 10 snap -1 2023-10-09T09:17:52.264 INFO:tasks.rados.rados.0.smithi163.stdout:2951: expect (ObjNum 977 snap 337 seq_num 977) 2023-10-09T09:17:52.265 INFO:tasks.rados.rados.0.smithi163.stdout:2952: snap_create 2023-10-09T09:17:52.265 INFO:tasks.rados.rados.0.smithi163.stdout:2948: finishing rollback tid 2 to smithi16367185-32 2023-10-09T09:17:52.265 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 2095 (ObjNum 880 snap 307 seq_num 880) dirty exists 2023-10-09T09:17:52.265 INFO:tasks.rados.rados.0.smithi163.stdout:2949: finishing write tid 1 to smithi16367185-48 2023-10-09T09:17:52.265 INFO:tasks.rados.rados.0.smithi163.stdout:2949: finishing write tid 2 to smithi16367185-48 2023-10-09T09:17:52.265 INFO:tasks.rados.rados.0.smithi163.stdout:2949: finishing write tid 3 to smithi16367185-48 2023-10-09T09:17:52.265 INFO:tasks.rados.rados.0.smithi163.stdout:2949: finishing write tid 4 to smithi16367185-48 2023-10-09T09:17:52.265 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 2099 (ObjNum 1108 snap 367 seq_num 1108) dirty exists 2023-10-09T09:17:52.265 INFO:tasks.rados.rados.0.smithi163.stdout:2949: left oid 48 (ObjNum 1108 snap 367 seq_num 1108) 2023-10-09T09:17:52.265 INFO:tasks.rados.rados.0.smithi163.stdout:2950: finishing write tid 1 to smithi16367185-28 2023-10-09T09:17:52.269 INFO:tasks.rados.rados.0.smithi163.stdout:2951: expect (ObjNum 977 snap 337 seq_num 977) 2023-10-09T09:17:52.443 INFO:tasks.rados.rados.0.smithi163.stdout:2950: finishing write tid 2 to smithi16367185-28 2023-10-09T09:17:52.444 INFO:tasks.rados.rados.0.smithi163.stdout:2950: finishing write tid 3 to smithi16367185-28 2023-10-09T09:17:52.444 INFO:tasks.rados.rados.0.smithi163.stdout:2950: finishing write tid 4 to smithi16367185-28 2023-10-09T09:17:52.444 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 1422 (ObjNum 1109 snap 367 seq_num 1109) dirty exists 2023-10-09T09:17:52.444 INFO:tasks.rados.rados.0.smithi163.stdout:2950: left oid 28 (ObjNum 1109 snap 367 seq_num 1109) 2023-10-09T09:17:52.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:52 smithi181 bash[59345]: cluster 2023-10-09T09:17:51.205236+0000 osd.5 (osd.5) 330 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:52.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:52 smithi181 bash[59345]: cluster 2023-10-09T09:17:51.211675+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:52 smithi181 bash[59345]: cluster 2023-10-09T09:17:51.211840+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:52 smithi181 bash[59345]: cluster 2023-10-09T09:17:51.400530+0000 mgr.y (mgr.34107) 339 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 243 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 549 KiB/s wr, 10 op/s 2023-10-09T09:17:52.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:52 smithi181 bash[59345]: cluster 2023-10-09T09:17:52.211499+0000 mon.a (mon.0) 983 : cluster 0 osdmap e798: 8 total, 8 up, 8 in 2023-10-09T09:17:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:52 smithi163 bash[75425]: cluster 2023-10-09T09:17:51.205236+0000 osd.5 (osd.5) 330 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:52 smithi163 bash[75425]: cluster 2023-10-09T09:17:51.211675+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:52 smithi163 bash[75425]: cluster 2023-10-09T09:17:51.211840+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:52.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:52 smithi163 bash[75425]: cluster 2023-10-09T09:17:51.400530+0000 mgr.y (mgr.34107) 339 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 243 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 549 KiB/s wr, 10 op/s 2023-10-09T09:17:52.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:52 smithi163 bash[75425]: cluster 2023-10-09T09:17:52.211499+0000 mon.a (mon.0) 983 : cluster 0 osdmap e798: 8 total, 8 up, 8 in 2023-10-09T09:17:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:52 smithi163 bash[78107]: cluster 2023-10-09T09:17:51.205236+0000 osd.5 (osd.5) 330 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:52 smithi163 bash[78107]: cluster 2023-10-09T09:17:51.211675+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:52 smithi163 bash[78107]: cluster 2023-10-09T09:17:51.211840+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:52 smithi163 bash[78107]: cluster 2023-10-09T09:17:51.400530+0000 mgr.y (mgr.34107) 339 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 243 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 549 KiB/s wr, 10 op/s 2023-10-09T09:17:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:52 smithi163 bash[78107]: cluster 2023-10-09T09:17:52.211499+0000 mon.a (mon.0) 983 : cluster 0 osdmap e798: 8 total, 8 up, 8 in 2023-10-09T09:17:53.263 INFO:tasks.rados.rados.0.smithi163.stdout:2948: done (4 left) 2023-10-09T09:17:53.263 INFO:tasks.rados.rados.0.smithi163.stdout:2949: done (3 left) 2023-10-09T09:17:53.263 INFO:tasks.rados.rados.0.smithi163.stdout:2950: done (2 left) 2023-10-09T09:17:53.263 INFO:tasks.rados.rados.0.smithi163.stdout:2951: done (1 left) 2023-10-09T09:17:53.263 INFO:tasks.rados.rados.0.smithi163.stdout:2952: done (0 left) 2023-10-09T09:17:53.263 INFO:tasks.rados.rados.0.smithi163.stdout:2953: write oid 45 current snap is 368 2023-10-09T09:17:53.263 INFO:tasks.rados.rados.0.smithi163.stdout:2953: seq_num 1110 ranges {655770=513928,1854542=684057} 2023-10-09T09:17:53.275 INFO:tasks.rados.rados.0.smithi163.stdout:2953: writing smithi16367185-45 from 655770 to 1169698 tid 1 2023-10-09T09:17:53.284 INFO:tasks.rados.rados.0.smithi163.stdout:2953: writing smithi16367185-45 from 1854542 to 2538599 tid 2 2023-10-09T09:17:53.284 INFO:tasks.rados.rados.0.smithi163.stdout:2954: write oid 4 current snap is 368 2023-10-09T09:17:53.284 INFO:tasks.rados.rados.0.smithi163.stdout:2954: seq_num 1111 ranges {672577=589177,1913024=427715} 2023-10-09T09:17:53.291 INFO:tasks.rados.rados.0.smithi163.stdout:2954: writing smithi16367185-4 from 672577 to 1261754 tid 1 2023-10-09T09:17:53.298 INFO:tasks.rados.rados.0.smithi163.stdout:2954: writing smithi16367185-4 from 1913024 to 2340739 tid 2 2023-10-09T09:17:53.298 INFO:tasks.rados.rados.0.smithi163.stdout:2955: read oid 26 snap -1 2023-10-09T09:17:53.298 INFO:tasks.rados.rados.0.smithi163.stdout:2955: expect (ObjNum 1101 snap 365 seq_num 1101) 2023-10-09T09:17:53.298 INFO:tasks.rados.rados.0.smithi163.stdout:2956: delete oid 13 current snap is 368 2023-10-09T09:17:53.299 INFO:tasks.rados.rados.0.smithi163.stdout:2953: finishing write tid 1 to smithi16367185-45 2023-10-09T09:17:53.299 INFO:tasks.rados.rados.0.smithi163.stdout:2953: finishing write tid 2 to smithi16367185-45 2023-10-09T09:17:53.299 INFO:tasks.rados.rados.0.smithi163.stdout:2953: finishing write tid 3 to smithi16367185-45 2023-10-09T09:17:53.299 INFO:tasks.rados.rados.0.smithi163.stdout:2953: finishing write tid 4 to smithi16367185-45 2023-10-09T09:17:53.299 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 2103 (ObjNum 1110 snap 368 seq_num 1110) dirty exists 2023-10-09T09:17:53.299 INFO:tasks.rados.rados.0.smithi163.stdout:2953: left oid 45 (ObjNum 1110 snap 368 seq_num 1110) 2023-10-09T09:17:53.299 INFO:tasks.rados.rados.0.smithi163.stdout:2954: finishing write tid 1 to smithi16367185-4 2023-10-09T09:17:53.303 INFO:tasks.rados.rados.0.smithi163.stdout:2953: done (3 left) 2023-10-09T09:17:53.303 INFO:tasks.rados.rados.0.smithi163.stdout:2956: done (2 left) 2023-10-09T09:17:53.303 INFO:tasks.rados.rados.0.smithi163.stdout:2957: rollback oid 6 current snap is 368 2023-10-09T09:17:53.303 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 6 to 338 2023-10-09T09:17:53.303 INFO:tasks.rados.rados.0.smithi163.stdout:2958: delete oid 8 current snap is 368 2023-10-09T09:17:53.304 INFO:tasks.rados.rados.0.smithi163.stdout:2955: expect (ObjNum 1101 snap 365 seq_num 1101) 2023-10-09T09:17:53.545 INFO:tasks.rados.rados.0.smithi163.stdout:2954: finishing write tid 2 to smithi16367185-4 2023-10-09T09:17:53.545 INFO:tasks.rados.rados.0.smithi163.stdout:2954: finishing write tid 3 to smithi16367185-4 2023-10-09T09:17:53.545 INFO:tasks.rados.rados.0.smithi163.stdout:2954: finishing write tid 4 to smithi16367185-4 2023-10-09T09:17:53.545 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1425 (ObjNum 1111 snap 368 seq_num 1111) dirty exists 2023-10-09T09:17:53.546 INFO:tasks.rados.rados.0.smithi163.stdout:2954: left oid 4 (ObjNum 1111 snap 368 seq_num 1111) 2023-10-09T09:17:53.546 INFO:tasks.rados.rados.0.smithi163.stdout:2957: finishing rollback tid 1 to smithi16367185-6 2023-10-09T09:17:53.546 INFO:tasks.rados.rados.0.smithi163.stdout:2957: finishing rollback tid 2 to smithi16367185-6 2023-10-09T09:17:53.546 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 308 (ObjNum 956 snap 324 seq_num 956) dirty exists 2023-10-09T09:17:53.546 INFO:tasks.rados.rados.0.smithi163.stdout:2954: done (3 left) 2023-10-09T09:17:53.546 INFO:tasks.rados.rados.0.smithi163.stdout:2955: done (2 left) 2023-10-09T09:17:53.546 INFO:tasks.rados.rados.0.smithi163.stdout:2957: done (1 left) 2023-10-09T09:17:53.546 INFO:tasks.rados.rados.0.smithi163.stdout:2958: done (0 left) 2023-10-09T09:17:53.546 INFO:tasks.rados.rados.0.smithi163.stdout:2959: delete oid 2 current snap is 368 2023-10-09T09:17:53.549 INFO:tasks.rados.rados.0.smithi163.stdout:2959: done (0 left) 2023-10-09T09:17:53.549 INFO:tasks.rados.rados.0.smithi163.stdout:2960: read oid 48 snap -1 2023-10-09T09:17:53.549 INFO:tasks.rados.rados.0.smithi163.stdout:2960: expect (ObjNum 1108 snap 367 seq_num 1108) 2023-10-09T09:17:53.549 INFO:tasks.rados.rados.0.smithi163.stdout:2961: write oid 22 current snap is 368 2023-10-09T09:17:53.549 INFO:tasks.rados.rados.0.smithi163.stdout:2961: seq_num 1115 ranges {739805=490173,1630599=435053,2636284=103521} 2023-10-09T09:17:53.564 INFO:tasks.rados.rados.0.smithi163.stdout:2961: writing smithi16367185-22 from 739805 to 1229978 tid 1 2023-10-09T09:17:53.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:53 smithi181 bash[59345]: cluster 2023-10-09T09:17:51.882090+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:53.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:53 smithi181 bash[59345]: cluster 2023-10-09T09:17:52.206822+0000 osd.5 (osd.5) 331 : cluster 3 failed to encode map e798 with expected crc 2023-10-09T09:17:53.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:53 smithi181 bash[59345]: cluster 2023-10-09T09:17:52.209403+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e798 with expected crc 2023-10-09T09:17:53.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:53 smithi181 bash[59345]: cluster 2023-10-09T09:17:52.213863+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e798 with expected crc 2023-10-09T09:17:53.569 INFO:tasks.rados.rados.0.smithi163.stdout:2961: writing smithi16367185-22 from 1630599 to 2065652 tid 2 2023-10-09T09:17:53.573 INFO:tasks.rados.rados.0.smithi163.stdout:2961: writing smithi16367185-22 from 2636284 to 2739805 tid 3 2023-10-09T09:17:53.573 INFO:tasks.rados.rados.0.smithi163.stdout:2962: snap_create 2023-10-09T09:17:53.573 INFO:tasks.rados.rados.0.smithi163.stdout:2960: expect (ObjNum 1108 snap 367 seq_num 1108) 2023-10-09T09:17:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:17:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:17:53] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:17:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:53 smithi163 bash[75425]: cluster 2023-10-09T09:17:51.882090+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:53 smithi163 bash[75425]: cluster 2023-10-09T09:17:52.206822+0000 osd.5 (osd.5) 331 : cluster 3 failed to encode map e798 with expected crc 2023-10-09T09:17:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:53 smithi163 bash[75425]: cluster 2023-10-09T09:17:52.209403+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e798 with expected crc 2023-10-09T09:17:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:53 smithi163 bash[75425]: cluster 2023-10-09T09:17:52.213863+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e798 with expected crc 2023-10-09T09:17:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:53 smithi163 bash[78107]: cluster 2023-10-09T09:17:51.882090+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e797 with expected crc 2023-10-09T09:17:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:53 smithi163 bash[78107]: cluster 2023-10-09T09:17:52.206822+0000 osd.5 (osd.5) 331 : cluster 3 failed to encode map e798 with expected crc 2023-10-09T09:17:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:53 smithi163 bash[78107]: cluster 2023-10-09T09:17:52.209403+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e798 with expected crc 2023-10-09T09:17:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:53 smithi163 bash[78107]: cluster 2023-10-09T09:17:52.213863+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e798 with expected crc 2023-10-09T09:17:53.763 INFO:tasks.rados.rados.0.smithi163.stdout:2961: finishing write tid 1 to smithi16367185-22 2023-10-09T09:17:53.763 INFO:tasks.rados.rados.0.smithi163.stdout:2961: finishing write tid 2 to smithi16367185-22 2023-10-09T09:17:53.764 INFO:tasks.rados.rados.0.smithi163.stdout:2961: finishing write tid 3 to smithi16367185-22 2023-10-09T09:17:53.764 INFO:tasks.rados.rados.0.smithi163.stdout:2961: finishing write tid 4 to smithi16367185-22 2023-10-09T09:17:53.764 INFO:tasks.rados.rados.0.smithi163.stdout:2961: finishing write tid 5 to smithi16367185-22 2023-10-09T09:17:53.764 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 2108 (ObjNum 1115 snap 368 seq_num 1115) dirty exists 2023-10-09T09:17:53.764 INFO:tasks.rados.rados.0.smithi163.stdout:2961: left oid 22 (ObjNum 1115 snap 368 seq_num 1115) 2023-10-09T09:17:54.276 INFO:tasks.rados.rados.0.smithi163.stdout:2960: done (2 left) 2023-10-09T09:17:54.276 INFO:tasks.rados.rados.0.smithi163.stdout:2961: done (1 left) 2023-10-09T09:17:54.277 INFO:tasks.rados.rados.0.smithi163.stdout:2962: done (0 left) 2023-10-09T09:17:54.277 INFO:tasks.rados.rados.0.smithi163.stdout:2963: delete oid 36 current snap is 369 2023-10-09T09:17:54.281 INFO:tasks.rados.rados.0.smithi163.stdout:2963: done (0 left) 2023-10-09T09:17:54.281 INFO:tasks.rados.rados.0.smithi163.stdout:2964: delete oid 42 current snap is 369 2023-10-09T09:17:54.284 INFO:tasks.rados.rados.0.smithi163.stdout:2964: done (0 left) 2023-10-09T09:17:54.284 INFO:tasks.rados.rados.0.smithi163.stdout:2965: write oid 47 current snap is 369 2023-10-09T09:17:54.284 INFO:tasks.rados.rados.0.smithi163.stdout:2965: seq_num 1118 ranges {790226=432273,1806045=582380} 2023-10-09T09:17:54.293 INFO:tasks.rados.rados.0.smithi163.stdout:2965: writing smithi16367185-47 from 790226 to 1222499 tid 1 2023-10-09T09:17:54.301 INFO:tasks.rados.rados.0.smithi163.stdout:2965: writing smithi16367185-47 from 1806045 to 2388425 tid 2 2023-10-09T09:17:54.301 INFO:tasks.rados.rados.0.smithi163.stdout:2966: write oid 20 current snap is 369 2023-10-09T09:17:54.301 INFO:tasks.rados.rados.0.smithi163.stdout:2965: finishing write tid 1 to smithi16367185-47 2023-10-09T09:17:54.301 INFO:tasks.rados.rados.0.smithi163.stdout:2966: seq_num 1119 ranges {407033=507522,1464527=442391,2345976=461057} 2023-10-09T09:17:54.307 INFO:tasks.rados.rados.0.smithi163.stdout:2966: writing smithi16367185-20 from 407033 to 914555 tid 1 2023-10-09T09:17:54.312 INFO:tasks.rados.rados.0.smithi163.stdout:2966: writing smithi16367185-20 from 1464527 to 1906918 tid 2 2023-10-09T09:17:54.318 INFO:tasks.rados.rados.0.smithi163.stdout:2966: writing smithi16367185-20 from 2345976 to 2807033 tid 3 2023-10-09T09:17:54.318 INFO:tasks.rados.rados.0.smithi163.stdout:2967: read oid 50 snap -1 2023-10-09T09:17:54.318 INFO:tasks.rados.rados.0.smithi163.stdout:2967: expect (ObjNum 1086 snap 362 seq_num 1086) 2023-10-09T09:17:54.318 INFO:tasks.rados.rados.0.smithi163.stdout:2968: write oid 46 current snap is 369 2023-10-09T09:17:54.318 INFO:tasks.rados.rados.0.smithi163.stdout:2965: finishing write tid 2 to smithi16367185-47 2023-10-09T09:17:54.319 INFO:tasks.rados.rados.0.smithi163.stdout:2965: finishing write tid 3 to smithi16367185-47 2023-10-09T09:17:54.319 INFO:tasks.rados.rados.0.smithi163.stdout:2965: finishing write tid 4 to smithi16367185-47 2023-10-09T09:17:54.319 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 2113 (ObjNum 1118 snap 369 seq_num 1118) dirty exists 2023-10-09T09:17:54.319 INFO:tasks.rados.rados.0.smithi163.stdout:2965: left oid 47 (ObjNum 1118 snap 369 seq_num 1118) 2023-10-09T09:17:54.319 INFO:tasks.rados.rados.0.smithi163.stdout:2966: finishing write tid 1 to smithi16367185-20 2023-10-09T09:17:54.319 INFO:tasks.rados.rados.0.smithi163.stdout:2966: finishing write tid 2 to smithi16367185-20 2023-10-09T09:17:54.319 INFO:tasks.rados.rados.0.smithi163.stdout:2968: seq_num 1120 ranges {423840=582771,1523009=586049,2657046=166794} 2023-10-09T09:17:54.324 INFO:tasks.rados.rados.0.smithi163.stdout:2968: writing smithi16367185-46 from 423840 to 1006611 tid 1 2023-10-09T09:17:54.330 INFO:tasks.rados.rados.0.smithi163.stdout:2968: writing smithi16367185-46 from 1523009 to 2109058 tid 2 2023-10-09T09:17:54.335 INFO:tasks.rados.rados.0.smithi163.stdout:2968: writing smithi16367185-46 from 2657046 to 2823840 tid 3 2023-10-09T09:17:54.335 INFO:tasks.rados.rados.0.smithi163.stdout:2965: done (3 left) 2023-10-09T09:17:54.335 INFO:tasks.rados.rados.0.smithi163.stdout:2969: snap_create 2023-10-09T09:17:54.335 INFO:tasks.rados.rados.0.smithi163.stdout:2966: finishing write tid 3 to smithi16367185-20 2023-10-09T09:17:54.335 INFO:tasks.rados.rados.0.smithi163.stdout:2966: finishing write tid 4 to smithi16367185-20 2023-10-09T09:17:54.335 INFO:tasks.rados.rados.0.smithi163.stdout:2966: finishing write tid 5 to smithi16367185-20 2023-10-09T09:17:54.335 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 518 (ObjNum 1119 snap 369 seq_num 1119) dirty exists 2023-10-09T09:17:54.335 INFO:tasks.rados.rados.0.smithi163.stdout:2966: left oid 20 (ObjNum 1119 snap 369 seq_num 1119) 2023-10-09T09:17:54.335 INFO:tasks.rados.rados.0.smithi163.stdout:2967: expect (ObjNum 1086 snap 362 seq_num 1086) 2023-10-09T09:17:54.416 INFO:tasks.rados.rados.0.smithi163.stdout:2968: finishing write tid 1 to smithi16367185-46 2023-10-09T09:17:54.416 INFO:tasks.rados.rados.0.smithi163.stdout:2968: finishing write tid 2 to smithi16367185-46 2023-10-09T09:17:54.417 INFO:tasks.rados.rados.0.smithi163.stdout:2968: finishing write tid 3 to smithi16367185-46 2023-10-09T09:17:54.417 INFO:tasks.rados.rados.0.smithi163.stdout:2968: finishing write tid 4 to smithi16367185-46 2023-10-09T09:17:54.417 INFO:tasks.rados.rados.0.smithi163.stdout:2968: finishing write tid 5 to smithi16367185-46 2023-10-09T09:17:54.417 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 1395 (ObjNum 1120 snap 369 seq_num 1120) dirty exists 2023-10-09T09:17:54.417 INFO:tasks.rados.rados.0.smithi163.stdout:2968: left oid 46 (ObjNum 1120 snap 369 seq_num 1120) 2023-10-09T09:17:54.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:54 smithi181 bash[59345]: cluster 2023-10-09T09:17:53.259213+0000 osd.5 (osd.5) 332 : cluster 3 failed to encode map e799 with expected crc 2023-10-09T09:17:54.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:54 smithi181 bash[59345]: cluster 2023-10-09T09:17:53.259704+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e799 with expected crc 2023-10-09T09:17:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:54 smithi181 bash[59345]: cluster 2023-10-09T09:17:53.262284+0000 mon.a (mon.0) 984 : cluster 0 osdmap e799: 8 total, 8 up, 8 in 2023-10-09T09:17:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:54 smithi181 bash[59345]: cluster 2023-10-09T09:17:53.294923+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e799 with expected crc 2023-10-09T09:17:54.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:54 smithi181 bash[59345]: cluster 2023-10-09T09:17:53.401305+0000 mgr.y (mgr.34107) 340 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 838 KiB/s wr, 9 op/s 2023-10-09T09:17:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:54 smithi163 bash[75425]: cluster 2023-10-09T09:17:53.259213+0000 osd.5 (osd.5) 332 : cluster 3 failed to encode map e799 with expected crc 2023-10-09T09:17:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:54 smithi163 bash[75425]: cluster 2023-10-09T09:17:53.259704+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e799 with expected crc 2023-10-09T09:17:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:54 smithi163 bash[75425]: cluster 2023-10-09T09:17:53.262284+0000 mon.a (mon.0) 984 : cluster 0 osdmap e799: 8 total, 8 up, 8 in 2023-10-09T09:17:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:54 smithi163 bash[75425]: cluster 2023-10-09T09:17:53.294923+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e799 with expected crc 2023-10-09T09:17:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:54 smithi163 bash[75425]: cluster 2023-10-09T09:17:53.401305+0000 mgr.y (mgr.34107) 340 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 838 KiB/s wr, 9 op/s 2023-10-09T09:17:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:54 smithi163 bash[78107]: cluster 2023-10-09T09:17:53.259213+0000 osd.5 (osd.5) 332 : cluster 3 failed to encode map e799 with expected crc 2023-10-09T09:17:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:54 smithi163 bash[78107]: cluster 2023-10-09T09:17:53.259704+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e799 with expected crc 2023-10-09T09:17:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:54 smithi163 bash[78107]: cluster 2023-10-09T09:17:53.262284+0000 mon.a (mon.0) 984 : cluster 0 osdmap e799: 8 total, 8 up, 8 in 2023-10-09T09:17:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:54 smithi163 bash[78107]: cluster 2023-10-09T09:17:53.294923+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e799 with expected crc 2023-10-09T09:17:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:54 smithi163 bash[78107]: cluster 2023-10-09T09:17:53.401305+0000 mgr.y (mgr.34107) 340 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 838 KiB/s wr, 9 op/s 2023-10-09T09:17:55.209 INFO:tasks.rados.rados.0.smithi163.stdout:2966: done (3 left) 2023-10-09T09:17:55.210 INFO:tasks.rados.rados.0.smithi163.stdout:2967: done (2 left) 2023-10-09T09:17:55.210 INFO:tasks.rados.rados.0.smithi163.stdout:2968: done (1 left) 2023-10-09T09:17:55.210 INFO:tasks.rados.rados.0.smithi163.stdout:2969: done (0 left) 2023-10-09T09:17:55.210 INFO:tasks.rados.rados.0.smithi163.stdout:2970: snap_remove snap 364 2023-10-09T09:17:55.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:55 smithi181 bash[59345]: cluster 2023-10-09T09:17:54.269391+0000 osd.5 (osd.5) 333 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:55 smithi181 bash[59345]: cluster 2023-10-09T09:17:54.276280+0000 mon.a (mon.0) 985 : cluster 0 osdmap e800: 8 total, 8 up, 8 in 2023-10-09T09:17:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:55 smithi181 bash[59345]: cluster 2023-10-09T09:17:54.280099+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:55 smithi181 bash[59345]: cluster 2023-10-09T09:17:54.308558+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:55 smithi181 bash[59345]: cluster 2023-10-09T09:17:54.410707+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:55 smithi181 bash[59345]: cluster 2023-10-09T09:17:54.411917+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:55 smithi181 bash[59345]: cluster 2023-10-09T09:17:55.208567+0000 mon.a (mon.0) 986 : cluster 0 osdmap e801: 8 total, 8 up, 8 in 2023-10-09T09:17:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[78107]: cluster 2023-10-09T09:17:54.269391+0000 osd.5 (osd.5) 333 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[78107]: cluster 2023-10-09T09:17:54.276280+0000 mon.a (mon.0) 985 : cluster 0 osdmap e800: 8 total, 8 up, 8 in 2023-10-09T09:17:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[78107]: cluster 2023-10-09T09:17:54.280099+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[78107]: cluster 2023-10-09T09:17:54.308558+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[78107]: cluster 2023-10-09T09:17:54.410707+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[78107]: cluster 2023-10-09T09:17:54.411917+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[78107]: cluster 2023-10-09T09:17:55.208567+0000 mon.a (mon.0) 986 : cluster 0 osdmap e801: 8 total, 8 up, 8 in 2023-10-09T09:17:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[75425]: cluster 2023-10-09T09:17:54.269391+0000 osd.5 (osd.5) 333 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[75425]: cluster 2023-10-09T09:17:54.276280+0000 mon.a (mon.0) 985 : cluster 0 osdmap e800: 8 total, 8 up, 8 in 2023-10-09T09:17:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[75425]: cluster 2023-10-09T09:17:54.280099+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[75425]: cluster 2023-10-09T09:17:54.308558+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[75425]: cluster 2023-10-09T09:17:54.410707+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[75425]: cluster 2023-10-09T09:17:54.411917+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e800 with expected crc 2023-10-09T09:17:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:55 smithi163 bash[75425]: cluster 2023-10-09T09:17:55.208567+0000 mon.a (mon.0) 986 : cluster 0 osdmap e801: 8 total, 8 up, 8 in 2023-10-09T09:17:56.216 INFO:tasks.rados.rados.0.smithi163.stdout:2970: done (0 left) 2023-10-09T09:17:56.216 INFO:tasks.rados.rados.0.smithi163.stdout:2971: read oid 11 snap -1 2023-10-09T09:17:56.217 INFO:tasks.rados.rados.0.smithi163.stdout:2971: expect deleted 2023-10-09T09:17:56.217 INFO:tasks.rados.rados.0.smithi163.stdout:2972: rollback oid 23 current snap is 370 2023-10-09T09:17:56.217 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 23 to 365 2023-10-09T09:17:56.217 INFO:tasks.rados.rados.0.smithi163.stdout:2973: snap_create 2023-10-09T09:17:56.228 INFO:tasks.rados.rados.0.smithi163.stdout:2972: finishing rollback tid 0 to smithi16367185-23 2023-10-09T09:17:56.228 INFO:tasks.rados.rados.0.smithi163.stdout:2972: finishing rollback tid 1 to smithi16367185-23 2023-10-09T09:17:56.228 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1390 (ObjNum 698985200 snap 21875 seq_num 1618308176) dirty dne 2023-10-09T09:17:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:56 smithi181 bash[59345]: cluster 2023-10-09T09:17:55.204267+0000 osd.5 (osd.5) 334 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:56 smithi181 bash[59345]: cluster 2023-10-09T09:17:55.207328+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:56 smithi181 bash[59345]: audit 2023-10-09T09:17:55.347249+0000 mon.a (mon.0) 987 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:56 smithi181 bash[59345]: cluster 2023-10-09T09:17:55.401998+0000 mgr.y (mgr.34107) 341 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 241 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 793 KiB/s wr, 7 op/s 2023-10-09T09:17:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:56 smithi181 bash[59345]: cluster 2023-10-09T09:17:55.544616+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:56 smithi181 bash[59345]: audit 2023-10-09T09:17:56.199304+0000 mon.a (mon.0) 988 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:56 smithi181 bash[59345]: audit 2023-10-09T09:17:56.208312+0000 mon.a (mon.0) 989 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:56.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:56 smithi181 bash[59345]: cluster 2023-10-09T09:17:56.213963+0000 mon.a (mon.0) 990 : cluster 0 osdmap e802: 8 total, 8 up, 8 in 2023-10-09T09:17:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[75425]: cluster 2023-10-09T09:17:55.204267+0000 osd.5 (osd.5) 334 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:56.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[75425]: cluster 2023-10-09T09:17:55.207328+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[75425]: audit 2023-10-09T09:17:55.347249+0000 mon.a (mon.0) 987 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[75425]: cluster 2023-10-09T09:17:55.401998+0000 mgr.y (mgr.34107) 341 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 241 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 793 KiB/s wr, 7 op/s 2023-10-09T09:17:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[75425]: cluster 2023-10-09T09:17:55.544616+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[75425]: audit 2023-10-09T09:17:56.199304+0000 mon.a (mon.0) 988 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[75425]: audit 2023-10-09T09:17:56.208312+0000 mon.a (mon.0) 989 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[75425]: cluster 2023-10-09T09:17:56.213963+0000 mon.a (mon.0) 990 : cluster 0 osdmap e802: 8 total, 8 up, 8 in 2023-10-09T09:17:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[78107]: cluster 2023-10-09T09:17:55.204267+0000 osd.5 (osd.5) 334 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[78107]: cluster 2023-10-09T09:17:55.207328+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[78107]: audit 2023-10-09T09:17:55.347249+0000 mon.a (mon.0) 987 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:17:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[78107]: cluster 2023-10-09T09:17:55.401998+0000 mgr.y (mgr.34107) 341 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 241 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 793 KiB/s wr, 7 op/s 2023-10-09T09:17:56.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[78107]: cluster 2023-10-09T09:17:55.544616+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:56.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[78107]: audit 2023-10-09T09:17:56.199304+0000 mon.a (mon.0) 988 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:56.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[78107]: audit 2023-10-09T09:17:56.208312+0000 mon.a (mon.0) 989 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:56.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:56 smithi163 bash[78107]: cluster 2023-10-09T09:17:56.213963+0000 mon.a (mon.0) 990 : cluster 0 osdmap e802: 8 total, 8 up, 8 in 2023-10-09T09:17:57.218 INFO:tasks.rados.rados.0.smithi163.stdout:2971: done (2 left) 2023-10-09T09:17:57.218 INFO:tasks.rados.rados.0.smithi163.stdout:2972: done (1 left) 2023-10-09T09:17:57.219 INFO:tasks.rados.rados.0.smithi163.stdout:2973: done (0 left) 2023-10-09T09:17:57.219 INFO:tasks.rados.rados.0.smithi163.stdout:2974: rollback oid 9 current snap is 371 2023-10-09T09:17:57.219 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 367 2023-10-09T09:17:57.219 INFO:tasks.rados.rados.0.smithi163.stdout:2975: write oid 29 current snap is 371 2023-10-09T09:17:57.219 INFO:tasks.rados.rados.0.smithi163.stdout:2975: seq_num 1121 ranges {440647=658020,1581491=446060,2800822=39825} 2023-10-09T09:17:57.226 INFO:tasks.rados.rados.0.smithi163.stdout:2975: writing smithi16367185-29 from 440647 to 1098667 tid 1 2023-10-09T09:17:57.232 INFO:tasks.rados.rados.0.smithi163.stdout:2975: writing smithi16367185-29 from 1581491 to 2027551 tid 2 2023-10-09T09:17:57.237 INFO:tasks.rados.rados.0.smithi163.stdout:2975: writing smithi16367185-29 from 2800822 to 2840647 tid 3 2023-10-09T09:17:57.237 INFO:tasks.rados.rados.0.smithi163.stdout:2976: write oid 44 current snap is 371 2023-10-09T09:17:57.238 INFO:tasks.rados.rados.0.smithi163.stdout:2976: seq_num 1122 ranges {457454=733269,1923620=589718} 2023-10-09T09:17:57.244 INFO:tasks.rados.rados.0.smithi163.stdout:2976: writing smithi16367185-44 from 457454 to 1190723 tid 1 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:2976: writing smithi16367185-44 from 1923620 to 2513338 tid 2 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:2977: snap_create 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:2974: finishing rollback tid 0 to smithi16367185-9 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:2974: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:2974: finishing rollback tid 2 to smithi16367185-9 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 654 (ObjNum 1046 snap 353 seq_num 1046) dirty exists 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:2975: finishing write tid 1 to smithi16367185-29 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:2975: finishing write tid 2 to smithi16367185-29 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:2975: finishing write tid 3 to smithi16367185-29 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:2975: finishing write tid 4 to smithi16367185-29 2023-10-09T09:17:57.253 INFO:tasks.rados.rados.0.smithi163.stdout:2975: finishing write tid 5 to smithi16367185-29 2023-10-09T09:17:57.254 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1399 (ObjNum 1121 snap 371 seq_num 1121) dirty exists 2023-10-09T09:17:57.254 INFO:tasks.rados.rados.0.smithi163.stdout:2975: left oid 29 (ObjNum 1121 snap 371 seq_num 1121) 2023-10-09T09:17:57.254 INFO:tasks.rados.rados.0.smithi163.stdout:2976: finishing write tid 1 to smithi16367185-44 2023-10-09T09:17:57.258 INFO:tasks.rados.rados.0.smithi163.stdout:2976: finishing write tid 2 to smithi16367185-44 2023-10-09T09:17:57.258 INFO:tasks.rados.rados.0.smithi163.stdout:2976: finishing write tid 3 to smithi16367185-44 2023-10-09T09:17:57.259 INFO:tasks.rados.rados.0.smithi163.stdout:2976: finishing write tid 4 to smithi16367185-44 2023-10-09T09:17:57.259 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 2127 (ObjNum 1122 snap 371 seq_num 1122) dirty exists 2023-10-09T09:17:57.259 INFO:tasks.rados.rados.0.smithi163.stdout:2976: left oid 44 (ObjNum 1122 snap 371 seq_num 1122) 2023-10-09T09:17:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[75425]: cluster 2023-10-09T09:17:55.512319+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[75425]: cluster 2023-10-09T09:17:56.210725+0000 osd.5 (osd.5) 335 : cluster 3 failed to encode map e802 with expected crc 2023-10-09T09:17:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[75425]: cluster 2023-10-09T09:17:56.217187+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e802 with expected crc 2023-10-09T09:17:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[75425]: audit 2023-10-09T09:17:56.397759+0000 mon.a (mon.0) 991 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[75425]: audit 2023-10-09T09:17:56.404945+0000 mon.a (mon.0) 992 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[75425]: audit 2023-10-09T09:17:56.884851+0000 mon.a (mon.0) 993 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:17:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[75425]: audit 2023-10-09T09:17:56.886648+0000 mon.a (mon.0) 994 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:17:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[75425]: audit 2023-10-09T09:17:56.895518+0000 mon.a (mon.0) 995 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[75425]: cluster 2023-10-09T09:17:57.217940+0000 mon.a (mon.0) 996 : cluster 0 osdmap e803: 8 total, 8 up, 8 in 2023-10-09T09:17:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[75425]: cluster 2023-10-09T09:17:57.222213+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[78107]: cluster 2023-10-09T09:17:55.512319+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:57.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[78107]: cluster 2023-10-09T09:17:56.210725+0000 osd.5 (osd.5) 335 : cluster 3 failed to encode map e802 with expected crc 2023-10-09T09:17:57.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[78107]: cluster 2023-10-09T09:17:56.217187+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e802 with expected crc 2023-10-09T09:17:57.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[78107]: audit 2023-10-09T09:17:56.397759+0000 mon.a (mon.0) 991 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:57.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[78107]: audit 2023-10-09T09:17:56.404945+0000 mon.a (mon.0) 992 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:57.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[78107]: audit 2023-10-09T09:17:56.884851+0000 mon.a (mon.0) 993 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:17:57.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[78107]: audit 2023-10-09T09:17:56.886648+0000 mon.a (mon.0) 994 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:17:57.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[78107]: audit 2023-10-09T09:17:56.895518+0000 mon.a (mon.0) 995 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:57.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[78107]: cluster 2023-10-09T09:17:57.217940+0000 mon.a (mon.0) 996 : cluster 0 osdmap e803: 8 total, 8 up, 8 in 2023-10-09T09:17:57.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:57 smithi163 bash[78107]: cluster 2023-10-09T09:17:57.222213+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:57 smithi181 bash[59345]: cluster 2023-10-09T09:17:55.512319+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e801 with expected crc 2023-10-09T09:17:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:57 smithi181 bash[59345]: cluster 2023-10-09T09:17:56.210725+0000 osd.5 (osd.5) 335 : cluster 3 failed to encode map e802 with expected crc 2023-10-09T09:17:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:57 smithi181 bash[59345]: cluster 2023-10-09T09:17:56.217187+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e802 with expected crc 2023-10-09T09:17:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:57 smithi181 bash[59345]: audit 2023-10-09T09:17:56.397759+0000 mon.a (mon.0) 991 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:57 smithi181 bash[59345]: audit 2023-10-09T09:17:56.404945+0000 mon.a (mon.0) 992 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:57 smithi181 bash[59345]: audit 2023-10-09T09:17:56.884851+0000 mon.a (mon.0) 993 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:17:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:57 smithi181 bash[59345]: audit 2023-10-09T09:17:56.886648+0000 mon.a (mon.0) 994 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:17:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:57 smithi181 bash[59345]: audit 2023-10-09T09:17:56.895518+0000 mon.a (mon.0) 995 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:17:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:57 smithi181 bash[59345]: cluster 2023-10-09T09:17:57.217940+0000 mon.a (mon.0) 996 : cluster 0 osdmap e803: 8 total, 8 up, 8 in 2023-10-09T09:17:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:57 smithi181 bash[59345]: cluster 2023-10-09T09:17:57.222213+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:58.221 INFO:tasks.rados.rados.0.smithi163.stdout:2974: done (3 left) 2023-10-09T09:17:58.221 INFO:tasks.rados.rados.0.smithi163.stdout:2975: done (2 left) 2023-10-09T09:17:58.222 INFO:tasks.rados.rados.0.smithi163.stdout:2976: done (1 left) 2023-10-09T09:17:58.222 INFO:tasks.rados.rados.0.smithi163.stdout:2977: done (0 left) 2023-10-09T09:17:58.222 INFO:tasks.rados.rados.0.smithi163.stdout:2978: snap_remove snap 367 2023-10-09T09:17:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[78107]: cluster 2023-10-09T09:17:57.213438+0000 osd.5 (osd.5) 336 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[78107]: cluster 2023-10-09T09:17:57.214849+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[78107]: cluster 2023-10-09T09:17:57.402589+0000 mgr.y (mgr.34107) 342 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 2.3 MiB/s wr, 21 op/s 2023-10-09T09:17:58.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[78107]: cluster 2023-10-09T09:17:57.414938+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[78107]: cluster 2023-10-09T09:17:58.221278+0000 mon.a (mon.0) 997 : cluster 0 osdmap e804: 8 total, 8 up, 8 in 2023-10-09T09:17:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[78107]: cluster 2023-10-09T09:17:58.325302+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:17:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[75425]: cluster 2023-10-09T09:17:57.213438+0000 osd.5 (osd.5) 336 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[75425]: cluster 2023-10-09T09:17:57.214849+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[75425]: cluster 2023-10-09T09:17:57.402589+0000 mgr.y (mgr.34107) 342 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 2.3 MiB/s wr, 21 op/s 2023-10-09T09:17:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[75425]: cluster 2023-10-09T09:17:57.414938+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[75425]: cluster 2023-10-09T09:17:58.221278+0000 mon.a (mon.0) 997 : cluster 0 osdmap e804: 8 total, 8 up, 8 in 2023-10-09T09:17:58.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:58 smithi163 bash[75425]: cluster 2023-10-09T09:17:58.325302+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:17:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:58 smithi181 bash[59345]: cluster 2023-10-09T09:17:57.213438+0000 osd.5 (osd.5) 336 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:58 smithi181 bash[59345]: cluster 2023-10-09T09:17:57.214849+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:58 smithi181 bash[59345]: cluster 2023-10-09T09:17:57.402589+0000 mgr.y (mgr.34107) 342 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 2.3 MiB/s wr, 21 op/s 2023-10-09T09:17:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:58 smithi181 bash[59345]: cluster 2023-10-09T09:17:57.414938+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e803 with expected crc 2023-10-09T09:17:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:58 smithi181 bash[59345]: cluster 2023-10-09T09:17:58.221278+0000 mon.a (mon.0) 997 : cluster 0 osdmap e804: 8 total, 8 up, 8 in 2023-10-09T09:17:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:58 smithi181 bash[59345]: cluster 2023-10-09T09:17:58.325302+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:17:59.225 INFO:tasks.rados.rados.0.smithi163.stdout:2978: done (0 left) 2023-10-09T09:17:59.225 INFO:tasks.rados.rados.0.smithi163.stdout:2979: snap_create 2023-10-09T09:17:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:59 smithi163 bash[78107]: cluster 2023-10-09T09:17:58.217731+0000 osd.5 (osd.5) 337 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:17:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:17:59 smithi163 bash[78107]: cluster 2023-10-09T09:17:59.225165+0000 mon.a (mon.0) 998 : cluster 0 osdmap e805: 8 total, 8 up, 8 in 2023-10-09T09:17:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:59 smithi163 bash[75425]: cluster 2023-10-09T09:17:58.217731+0000 osd.5 (osd.5) 337 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:17:59.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:17:59 smithi163 bash[75425]: cluster 2023-10-09T09:17:59.225165+0000 mon.a (mon.0) 998 : cluster 0 osdmap e805: 8 total, 8 up, 8 in 2023-10-09T09:17:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:59 smithi181 bash[59345]: cluster 2023-10-09T09:17:58.217731+0000 osd.5 (osd.5) 337 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:17:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:17:59 smithi181 bash[59345]: cluster 2023-10-09T09:17:59.225165+0000 mon.a (mon.0) 998 : cluster 0 osdmap e805: 8 total, 8 up, 8 in 2023-10-09T09:18:00.215 INFO:tasks.rados.rados.0.smithi163.stdout:2979: done (0 left) 2023-10-09T09:18:00.215 INFO:tasks.rados.rados.0.smithi163.stdout:2980: rollback oid 7 current snap is 373 2023-10-09T09:18:00.215 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 365 2023-10-09T09:18:00.216 INFO:tasks.rados.rados.0.smithi163.stdout:2981: write oid 14 current snap is 373 2023-10-09T09:18:00.216 INFO:tasks.rados.rados.0.smithi163.stdout:2981: seq_num 1123 ranges {474261=408518,1582102=449729,2739315=134946} 2023-10-09T09:18:00.222 INFO:tasks.rados.rados.0.smithi163.stdout:2981: writing smithi16367185-14 from 474261 to 882779 tid 1 2023-10-09T09:18:00.229 INFO:tasks.rados.rados.0.smithi163.stdout:2981: writing smithi16367185-14 from 1582102 to 2031831 tid 2 2023-10-09T09:18:00.234 INFO:tasks.rados.rados.0.smithi163.stdout:2981: writing smithi16367185-14 from 2739315 to 2874261 tid 3 2023-10-09T09:18:00.234 INFO:tasks.rados.rados.0.smithi163.stdout:2982: read oid 39 snap -1 2023-10-09T09:18:00.234 INFO:tasks.rados.rados.0.smithi163.stdout:2982: expect (ObjNum 1090 snap 362 seq_num 1090) 2023-10-09T09:18:00.234 INFO:tasks.rados.rados.0.smithi163.stdout:2983: delete oid 41 current snap is 373 2023-10-09T09:18:00.234 INFO:tasks.rados.rados.0.smithi163.stdout:2980: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:18:00.234 INFO:tasks.rados.rados.0.smithi163.stdout:2980: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:18:00.234 INFO:tasks.rados.rados.0.smithi163.stdout:2980: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:18:00.235 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 718 (ObjNum 961 snap 326 seq_num 961) dirty exists 2023-10-09T09:18:00.240 INFO:tasks.rados.rados.0.smithi163.stdout:2981: finishing write tid 1 to smithi16367185-14 2023-10-09T09:18:00.241 INFO:tasks.rados.rados.0.smithi163.stdout:2981: finishing write tid 2 to smithi16367185-14 2023-10-09T09:18:00.241 INFO:tasks.rados.rados.0.smithi163.stdout:2981: finishing write tid 3 to smithi16367185-14 2023-10-09T09:18:00.241 INFO:tasks.rados.rados.0.smithi163.stdout:2981: finishing write tid 4 to smithi16367185-14 2023-10-09T09:18:00.242 INFO:tasks.rados.rados.0.smithi163.stdout:2980: done (3 left) 2023-10-09T09:18:00.242 INFO:tasks.rados.rados.0.smithi163.stdout:2983: done (2 left) 2023-10-09T09:18:00.242 INFO:tasks.rados.rados.0.smithi163.stdout:2984: write oid 7 current snap is 373 2023-10-09T09:18:00.242 INFO:tasks.rados.rados.0.smithi163.stdout:2984: seq_num 1125 ranges {507875=559016,1699066=737045} 2023-10-09T09:18:00.248 INFO:tasks.rados.rados.0.smithi163.stdout:2984: writing smithi16367185-7 from 507875 to 1066891 tid 1 2023-10-09T09:18:00.258 INFO:tasks.rados.rados.0.smithi163.stdout:2984: writing smithi16367185-7 from 1699066 to 2436111 tid 2 2023-10-09T09:18:00.258 INFO:tasks.rados.rados.0.smithi163.stdout:2985: write oid 37 current snap is 373 2023-10-09T09:18:00.259 INFO:tasks.rados.rados.0.smithi163.stdout:2985: seq_num 1126 ranges {524682=750618,1757548=597056} 2023-10-09T09:18:00.266 INFO:tasks.rados.rados.0.smithi163.stdout:2985: writing smithi16367185-37 from 524682 to 1275300 tid 1 2023-10-09T09:18:00.273 INFO:tasks.rados.rados.0.smithi163.stdout:2985: writing smithi16367185-37 from 1757548 to 2354604 tid 2 2023-10-09T09:18:00.273 INFO:tasks.rados.rados.0.smithi163.stdout:2986: snap_create 2023-10-09T09:18:00.273 INFO:tasks.rados.rados.0.smithi163.stdout:2981: finishing write tid 5 to smithi16367185-14 2023-10-09T09:18:00.273 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1402 (ObjNum 1123 snap 373 seq_num 1123) dirty exists 2023-10-09T09:18:00.274 INFO:tasks.rados.rados.0.smithi163.stdout:2981: left oid 14 (ObjNum 1123 snap 373 seq_num 1123) 2023-10-09T09:18:00.274 INFO:tasks.rados.rados.0.smithi163.stdout:2982: expect (ObjNum 1090 snap 362 seq_num 1090) 2023-10-09T09:18:00.392 INFO:tasks.rados.rados.0.smithi163.stdout:2984: finishing write tid 1 to smithi16367185-7 2023-10-09T09:18:00.393 INFO:tasks.rados.rados.0.smithi163.stdout:2984: finishing write tid 2 to smithi16367185-7 2023-10-09T09:18:00.393 INFO:tasks.rados.rados.0.smithi163.stdout:2984: finishing write tid 3 to smithi16367185-7 2023-10-09T09:18:00.393 INFO:tasks.rados.rados.0.smithi163.stdout:2984: finishing write tid 4 to smithi16367185-7 2023-10-09T09:18:00.393 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 721 (ObjNum 1125 snap 373 seq_num 1125) dirty exists 2023-10-09T09:18:00.393 INFO:tasks.rados.rados.0.smithi163.stdout:2984: left oid 7 (ObjNum 1125 snap 373 seq_num 1125) 2023-10-09T09:18:00.393 INFO:tasks.rados.rados.0.smithi163.stdout:2985: finishing write tid 1 to smithi16367185-37 2023-10-09T09:18:00.393 INFO:tasks.rados.rados.0.smithi163.stdout:2985: finishing write tid 2 to smithi16367185-37 2023-10-09T09:18:00.393 INFO:tasks.rados.rados.0.smithi163.stdout:2985: finishing write tid 3 to smithi16367185-37 2023-10-09T09:18:00.393 INFO:tasks.rados.rados.0.smithi163.stdout:2985: finishing write tid 4 to smithi16367185-37 2023-10-09T09:18:00.393 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 1495 (ObjNum 1126 snap 373 seq_num 1126) dirty exists 2023-10-09T09:18:00.394 INFO:tasks.rados.rados.0.smithi163.stdout:2985: left oid 37 (ObjNum 1126 snap 373 seq_num 1126) 2023-10-09T09:18:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[75425]: cluster 2023-10-09T09:17:59.115641+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:18:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[75425]: cluster 2023-10-09T09:17:59.220370+0000 osd.5 (osd.5) 338 : cluster 3 failed to encode map e805 with expected crc 2023-10-09T09:18:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[75425]: cluster 2023-10-09T09:17:59.223268+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:18:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[75425]: cluster 2023-10-09T09:17:59.403315+0000 mgr.y (mgr.34107) 343 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2023-10-09T09:18:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[75425]: cluster 2023-10-09T09:18:00.214735+0000 mon.a (mon.0) 999 : cluster 0 osdmap e806: 8 total, 8 up, 8 in 2023-10-09T09:18:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[75425]: cluster 2023-10-09T09:18:00.217623+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[75425]: cluster 2023-10-09T09:18:00.223495+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[78107]: cluster 2023-10-09T09:17:59.115641+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:18:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[78107]: cluster 2023-10-09T09:17:59.220370+0000 osd.5 (osd.5) 338 : cluster 3 failed to encode map e805 with expected crc 2023-10-09T09:18:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[78107]: cluster 2023-10-09T09:17:59.223268+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:18:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[78107]: cluster 2023-10-09T09:17:59.403315+0000 mgr.y (mgr.34107) 343 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2023-10-09T09:18:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[78107]: cluster 2023-10-09T09:18:00.214735+0000 mon.a (mon.0) 999 : cluster 0 osdmap e806: 8 total, 8 up, 8 in 2023-10-09T09:18:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[78107]: cluster 2023-10-09T09:18:00.217623+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:00 smithi163 bash[78107]: cluster 2023-10-09T09:18:00.223495+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:00 smithi181 bash[59345]: cluster 2023-10-09T09:17:59.115641+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:18:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:00 smithi181 bash[59345]: cluster 2023-10-09T09:17:59.220370+0000 osd.5 (osd.5) 338 : cluster 3 failed to encode map e805 with expected crc 2023-10-09T09:18:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:00 smithi181 bash[59345]: cluster 2023-10-09T09:17:59.223268+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e804 with expected crc 2023-10-09T09:18:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:00 smithi181 bash[59345]: cluster 2023-10-09T09:17:59.403315+0000 mgr.y (mgr.34107) 343 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2023-10-09T09:18:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:00 smithi181 bash[59345]: cluster 2023-10-09T09:18:00.214735+0000 mon.a (mon.0) 999 : cluster 0 osdmap e806: 8 total, 8 up, 8 in 2023-10-09T09:18:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:00 smithi181 bash[59345]: cluster 2023-10-09T09:18:00.217623+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:00 smithi181 bash[59345]: cluster 2023-10-09T09:18:00.223495+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:01.233 INFO:tasks.rados.rados.0.smithi163.stdout:2981: done (4 left) 2023-10-09T09:18:01.234 INFO:tasks.rados.rados.0.smithi163.stdout:2982: done (3 left) 2023-10-09T09:18:01.234 INFO:tasks.rados.rados.0.smithi163.stdout:2984: done (2 left) 2023-10-09T09:18:01.234 INFO:tasks.rados.rados.0.smithi163.stdout:2985: done (1 left) 2023-10-09T09:18:01.234 INFO:tasks.rados.rados.0.smithi163.stdout:2986: done (0 left) 2023-10-09T09:18:01.234 INFO:tasks.rados.rados.0.smithi163.stdout:2987: rollback oid 7 current snap is 374 2023-10-09T09:18:01.234 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 372 2023-10-09T09:18:01.234 INFO:tasks.rados.rados.0.smithi163.stdout:2988: write oid 26 current snap is 374 2023-10-09T09:18:01.234 INFO:tasks.rados.rados.0.smithi163.stdout:2988: seq_num 1127 ranges {541489=425867,1416030=740714,2732654=208835} 2023-10-09T09:18:01.242 INFO:tasks.rados.rados.0.smithi163.stdout:2988: writing smithi16367185-26 from 541489 to 967356 tid 1 2023-10-09T09:18:01.252 INFO:tasks.rados.rados.0.smithi163.stdout:2988: writing smithi16367185-26 from 1416030 to 2156744 tid 2 2023-10-09T09:18:01.257 INFO:tasks.rados.rados.0.smithi163.stdout:2988: writing smithi16367185-26 from 2732654 to 2941489 tid 3 2023-10-09T09:18:01.257 INFO:tasks.rados.rados.0.smithi163.stdout:2989: snap_create 2023-10-09T09:18:01.257 INFO:tasks.rados.rados.0.smithi163.stdout:2987: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:18:01.258 INFO:tasks.rados.rados.0.smithi163.stdout:2987: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:18:01.258 INFO:tasks.rados.rados.0.smithi163.stdout:2987: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:18:01.258 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 725 (ObjNum 961 snap 326 seq_num 961) dirty exists 2023-10-09T09:18:01.258 INFO:tasks.rados.rados.0.smithi163.stdout:2988: finishing write tid 1 to smithi16367185-26 2023-10-09T09:18:01.258 INFO:tasks.rados.rados.0.smithi163.stdout:2988: finishing write tid 2 to smithi16367185-26 2023-10-09T09:18:01.260 INFO:tasks.rados.rados.0.smithi163.stdout:2988: finishing write tid 3 to smithi16367185-26 2023-10-09T09:18:01.260 INFO:tasks.rados.rados.0.smithi163.stdout:2988: finishing write tid 4 to smithi16367185-26 2023-10-09T09:18:01.261 INFO:tasks.rados.rados.0.smithi163.stdout:2988: finishing write tid 5 to smithi16367185-26 2023-10-09T09:18:01.261 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 862 (ObjNum 1127 snap 374 seq_num 1127) dirty exists 2023-10-09T09:18:01.261 INFO:tasks.rados.rados.0.smithi163.stdout:2988: left oid 26 (ObjNum 1127 snap 374 seq_num 1127) 2023-10-09T09:18:01.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:01 smithi163 bash[78107]: cluster 2023-10-09T09:18:00.208992+0000 osd.5 (osd.5) 339 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:01.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:01 smithi163 bash[78107]: cluster 2023-10-09T09:18:00.216199+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:01 smithi163 bash[78107]: cluster 2023-10-09T09:18:00.238333+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:01 smithi163 bash[78107]: cluster 2023-10-09T09:18:01.232439+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e807: 8 total, 8 up, 8 in 2023-10-09T09:18:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:01 smithi163 bash[78107]: cluster 2023-10-09T09:18:01.237559+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:01 smithi163 bash[75425]: cluster 2023-10-09T09:18:00.208992+0000 osd.5 (osd.5) 339 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:01 smithi163 bash[75425]: cluster 2023-10-09T09:18:00.216199+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:01 smithi163 bash[75425]: cluster 2023-10-09T09:18:00.238333+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:01 smithi163 bash[75425]: cluster 2023-10-09T09:18:01.232439+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e807: 8 total, 8 up, 8 in 2023-10-09T09:18:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:01 smithi163 bash[75425]: cluster 2023-10-09T09:18:01.237559+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:01 smithi181 bash[59345]: cluster 2023-10-09T09:18:00.208992+0000 osd.5 (osd.5) 339 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:01 smithi181 bash[59345]: cluster 2023-10-09T09:18:00.216199+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:01 smithi181 bash[59345]: cluster 2023-10-09T09:18:00.238333+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e806 with expected crc 2023-10-09T09:18:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:01 smithi181 bash[59345]: cluster 2023-10-09T09:18:01.232439+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e807: 8 total, 8 up, 8 in 2023-10-09T09:18:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:01 smithi181 bash[59345]: cluster 2023-10-09T09:18:01.237559+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:02.241 INFO:tasks.rados.rados.0.smithi163.stdout:2987: done (2 left) 2023-10-09T09:18:02.486 INFO:tasks.rados.rados.0.smithi163.stdout:2988: done (1 left) 2023-10-09T09:18:02.486 INFO:tasks.rados.rados.0.smithi163.stdout:2989: done (0 left) 2023-10-09T09:18:02.487 INFO:tasks.rados.rados.0.smithi163.stdout:2990: rollback oid 2 current snap is 375 2023-10-09T09:18:02.487 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 2 to 373 2023-10-09T09:18:02.487 INFO:tasks.rados.rados.0.smithi163.stdout:2991: snap_create 2023-10-09T09:18:02.487 INFO:tasks.rados.rados.0.smithi163.stdout:2990: finishing rollback tid 1 to smithi16367185-2 2023-10-09T09:18:02.487 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 0 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:18:02.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:02 smithi163 bash[78107]: cluster 2023-10-09T09:18:01.227335+0000 osd.5 (osd.5) 340 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:02.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:02 smithi163 bash[78107]: cluster 2023-10-09T09:18:01.245748+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:02.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:02 smithi163 bash[78107]: cluster 2023-10-09T09:18:01.404205+0000 mgr.y (mgr.34107) 344 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 257 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2023-10-09T09:18:02.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:02 smithi163 bash[78107]: cluster 2023-10-09T09:18:01.500629+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:02.898 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:02 smithi163 bash[78107]: cluster 2023-10-09T09:18:02.240988+0000 mon.a (mon.0) 1001 : cluster 0 osdmap e808: 8 total, 8 up, 8 in 2023-10-09T09:18:02.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:02 smithi163 bash[75425]: cluster 2023-10-09T09:18:01.227335+0000 osd.5 (osd.5) 340 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:02.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:02 smithi163 bash[75425]: cluster 2023-10-09T09:18:01.245748+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:02.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:02 smithi163 bash[75425]: cluster 2023-10-09T09:18:01.404205+0000 mgr.y (mgr.34107) 344 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 257 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2023-10-09T09:18:02.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:02 smithi163 bash[75425]: cluster 2023-10-09T09:18:01.500629+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:02.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:02 smithi163 bash[75425]: cluster 2023-10-09T09:18:02.240988+0000 mon.a (mon.0) 1001 : cluster 0 osdmap e808: 8 total, 8 up, 8 in 2023-10-09T09:18:02.899 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:02 smithi181 bash[59345]: cluster 2023-10-09T09:18:01.227335+0000 osd.5 (osd.5) 340 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:02.899 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:02 smithi181 bash[59345]: cluster 2023-10-09T09:18:01.245748+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:02.899 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:02 smithi181 bash[59345]: cluster 2023-10-09T09:18:01.404205+0000 mgr.y (mgr.34107) 344 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 257 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2023-10-09T09:18:02.899 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:02 smithi181 bash[59345]: cluster 2023-10-09T09:18:01.500629+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e807 with expected crc 2023-10-09T09:18:02.899 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:02 smithi181 bash[59345]: cluster 2023-10-09T09:18:02.240988+0000 mon.a (mon.0) 1001 : cluster 0 osdmap e808: 8 total, 8 up, 8 in 2023-10-09T09:18:03.245 INFO:tasks.rados.rados.0.smithi163.stdout:2990: done (1 left) 2023-10-09T09:18:03.245 INFO:tasks.rados.rados.0.smithi163.stdout:2991: done (0 left) 2023-10-09T09:18:03.245 INFO:tasks.rados.rados.0.smithi163.stdout:2992: write oid 18 current snap is 376 2023-10-09T09:18:03.245 INFO:tasks.rados.rados.0.smithi163.stdout:2992: seq_num 1128 ranges {558296=501116,1474512=600725,2760077=198219} 2023-10-09T09:18:03.251 INFO:tasks.rados.rados.0.smithi163.stdout:2992: writing smithi16367185-18 from 558296 to 1059412 tid 1 2023-10-09T09:18:03.258 INFO:tasks.rados.rados.0.smithi163.stdout:2992: writing smithi16367185-18 from 1474512 to 2075237 tid 2 2023-10-09T09:18:03.263 INFO:tasks.rados.rados.0.smithi163.stdout:2992: writing smithi16367185-18 from 2760077 to 2958296 tid 3 2023-10-09T09:18:03.263 INFO:tasks.rados.rados.0.smithi163.stdout:2993: delete oid 49 current snap is 376 2023-10-09T09:18:03.264 INFO:tasks.rados.rados.0.smithi163.stdout:2992: finishing write tid 1 to smithi16367185-18 2023-10-09T09:18:03.264 INFO:tasks.rados.rados.0.smithi163.stdout:2992: finishing write tid 2 to smithi16367185-18 2023-10-09T09:18:03.267 INFO:tasks.rados.rados.0.smithi163.stdout:2992: finishing write tid 3 to smithi16367185-18 2023-10-09T09:18:03.267 INFO:tasks.rados.rados.0.smithi163.stdout:2993: done (1 left) 2023-10-09T09:18:03.267 INFO:tasks.rados.rados.0.smithi163.stdout:2994: snap_remove snap 369 2023-10-09T09:18:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:18:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:18:03] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:18:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:03 smithi163 bash[78107]: cluster 2023-10-09T09:18:02.237766+0000 osd.5 (osd.5) 341 : cluster 3 failed to encode map e808 with expected crc 2023-10-09T09:18:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:03 smithi163 bash[78107]: cluster 2023-10-09T09:18:02.242169+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e808 with expected crc 2023-10-09T09:18:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:03 smithi163 bash[78107]: cluster 2023-10-09T09:18:02.425459+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e808 with expected crc 2023-10-09T09:18:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:03 smithi163 bash[78107]: cluster 2023-10-09T09:18:03.244393+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e809: 8 total, 8 up, 8 in 2023-10-09T09:18:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:03 smithi163 bash[75425]: cluster 2023-10-09T09:18:02.237766+0000 osd.5 (osd.5) 341 : cluster 3 failed to encode map e808 with expected crc 2023-10-09T09:18:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:03 smithi163 bash[75425]: cluster 2023-10-09T09:18:02.242169+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e808 with expected crc 2023-10-09T09:18:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:03 smithi163 bash[75425]: cluster 2023-10-09T09:18:02.425459+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e808 with expected crc 2023-10-09T09:18:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:03 smithi163 bash[75425]: cluster 2023-10-09T09:18:03.244393+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e809: 8 total, 8 up, 8 in 2023-10-09T09:18:03.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:03 smithi181 bash[59345]: cluster 2023-10-09T09:18:02.237766+0000 osd.5 (osd.5) 341 : cluster 3 failed to encode map e808 with expected crc 2023-10-09T09:18:03.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:03 smithi181 bash[59345]: cluster 2023-10-09T09:18:02.242169+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e808 with expected crc 2023-10-09T09:18:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:03 smithi181 bash[59345]: cluster 2023-10-09T09:18:02.425459+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e808 with expected crc 2023-10-09T09:18:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:03 smithi181 bash[59345]: cluster 2023-10-09T09:18:03.244393+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e809: 8 total, 8 up, 8 in 2023-10-09T09:18:04.253 INFO:tasks.rados.rados.0.smithi163.stdout:2994: done (1 left) 2023-10-09T09:18:04.253 INFO:tasks.rados.rados.0.smithi163.stdout:2995: write oid 27 current snap is 376 2023-10-09T09:18:04.253 INFO:tasks.rados.rados.0.smithi163.stdout:2992: finishing write tid 4 to smithi16367185-18 2023-10-09T09:18:04.253 INFO:tasks.rados.rados.0.smithi163.stdout:2992: finishing write tid 5 to smithi16367185-18 2023-10-09T09:18:04.254 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 948 (ObjNum 1128 snap 376 seq_num 1128) dirty exists 2023-10-09T09:18:04.254 INFO:tasks.rados.rados.0.smithi163.stdout:2992: left oid 18 (ObjNum 1128 snap 376 seq_num 1128) 2023-10-09T09:18:04.254 INFO:tasks.rados.rados.0.smithi163.stdout:2995: seq_num 1130 ranges {591910=651614,1875123=604394} 2023-10-09T09:18:04.265 INFO:tasks.rados.rados.0.smithi163.stdout:2995: writing smithi16367185-27 from 591910 to 1243524 tid 1 2023-10-09T09:18:04.274 INFO:tasks.rados.rados.0.smithi163.stdout:2995: writing smithi16367185-27 from 1875123 to 2479517 tid 2 2023-10-09T09:18:04.274 INFO:tasks.rados.rados.0.smithi163.stdout:2992: done (1 left) 2023-10-09T09:18:04.274 INFO:tasks.rados.rados.0.smithi163.stdout:2996: rollback oid 33 current snap is 376 2023-10-09T09:18:04.274 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 370 2023-10-09T09:18:04.274 INFO:tasks.rados.rados.0.smithi163.stdout:2997: delete oid 42 current snap is 376 2023-10-09T09:18:04.277 INFO:tasks.rados.rados.0.smithi163.stdout:2995: finishing write tid 1 to smithi16367185-27 2023-10-09T09:18:04.278 INFO:tasks.rados.rados.0.smithi163.stdout:2996: finishing rollback tid 0 to smithi16367185-33 2023-10-09T09:18:04.280 INFO:tasks.rados.rados.0.smithi163.stdout:2997: done (2 left) 2023-10-09T09:18:04.280 INFO:tasks.rados.rados.0.smithi163.stdout:2998: read oid 25 snap -1 2023-10-09T09:18:04.280 INFO:tasks.rados.rados.0.smithi163.stdout:2998: expect (ObjNum 1004 snap 342 seq_num 1004) 2023-10-09T09:18:04.280 INFO:tasks.rados.rados.0.smithi163.stdout:2999: write oid 46 current snap is 376 2023-10-09T09:18:04.280 INFO:tasks.rados.rados.0.smithi163.stdout:2999: seq_num 1132 ranges {625524=402112,1592087=608063,2869769=155755} 2023-10-09T09:18:04.286 INFO:tasks.rados.rados.0.smithi163.stdout:2999: writing smithi16367185-46 from 625524 to 1027636 tid 1 2023-10-09T09:18:04.293 INFO:tasks.rados.rados.0.smithi163.stdout:2999: writing smithi16367185-46 from 1592087 to 2200150 tid 2 2023-10-09T09:18:04.298 INFO:tasks.rados.rados.0.smithi163.stdout:2999: writing smithi16367185-46 from 2869769 to 3025524 tid 3 2023-10-09T09:18:04.298 INFO:tasks.rados.rados.0.smithi163.stdout:3000: write oid 9 current snap is 376 2023-10-09T09:18:04.298 INFO:tasks.rados.rados.0.smithi163.stdout:2995: finishing write tid 2 to smithi16367185-27 2023-10-09T09:18:04.298 INFO:tasks.rados.rados.0.smithi163.stdout:3000: seq_num 1133 ranges {642331=593714,1766922=751721} 2023-10-09T09:18:04.304 INFO:tasks.rados.rados.0.smithi163.stdout:3000: writing smithi16367185-9 from 642331 to 1236045 tid 1 2023-10-09T09:18:04.312 INFO:tasks.rados.rados.0.smithi163.stdout:3000: writing smithi16367185-9 from 1766922 to 2518643 tid 2 2023-10-09T09:18:04.312 INFO:tasks.rados.rados.0.smithi163.stdout:3001: rollback oid 38 current snap is 376 2023-10-09T09:18:04.312 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 375 2023-10-09T09:18:04.312 INFO:tasks.rados.rados.0.smithi163.stdout:2996: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:18:04.313 INFO:tasks.rados.rados.0.smithi163.stdout:2995: finishing write tid 3 to smithi16367185-27 2023-10-09T09:18:04.313 INFO:tasks.rados.rados.0.smithi163.stdout:2995: finishing write tid 4 to smithi16367185-27 2023-10-09T09:18:04.313 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 868 (ObjNum 1130 snap 376 seq_num 1130) dirty exists 2023-10-09T09:18:04.313 INFO:tasks.rados.rados.0.smithi163.stdout:2995: left oid 27 (ObjNum 1130 snap 376 seq_num 1130) 2023-10-09T09:18:04.313 INFO:tasks.rados.rados.0.smithi163.stdout:2996: finishing rollback tid 2 to smithi16367185-33 2023-10-09T09:18:04.313 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1403 (ObjNum 1069 snap 361 seq_num 1069) dirty exists 2023-10-09T09:18:04.313 INFO:tasks.rados.rados.0.smithi163.stdout:2998: expect (ObjNum 1004 snap 342 seq_num 1004) 2023-10-09T09:18:04.449 INFO:tasks.rados.rados.0.smithi163.stdout:2999: finishing write tid 1 to smithi16367185-46 2023-10-09T09:18:04.449 INFO:tasks.rados.rados.0.smithi163.stdout:2999: finishing write tid 2 to smithi16367185-46 2023-10-09T09:18:04.449 INFO:tasks.rados.rados.0.smithi163.stdout:2999: finishing write tid 3 to smithi16367185-46 2023-10-09T09:18:04.449 INFO:tasks.rados.rados.0.smithi163.stdout:2999: finishing write tid 4 to smithi16367185-46 2023-10-09T09:18:04.449 INFO:tasks.rados.rados.0.smithi163.stdout:2999: finishing write tid 5 to smithi16367185-46 2023-10-09T09:18:04.449 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 1409 (ObjNum 1132 snap 376 seq_num 1132) dirty exists 2023-10-09T09:18:04.449 INFO:tasks.rados.rados.0.smithi163.stdout:2999: left oid 46 (ObjNum 1132 snap 376 seq_num 1132) 2023-10-09T09:18:04.449 INFO:tasks.rados.rados.0.smithi163.stdout:3000: finishing write tid 1 to smithi16367185-9 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:2995: done (5 left) 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:2996: done (4 left) 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:2998: done (3 left) 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:2999: done (2 left) 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:3002: rollback oid 40 current snap is 376 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 40 to 368 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:3000: finishing write tid 2 to smithi16367185-9 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:3000: finishing write tid 3 to smithi16367185-9 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:3000: finishing write tid 4 to smithi16367185-9 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 662 (ObjNum 1133 snap 376 seq_num 1133) dirty exists 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:3000: left oid 9 (ObjNum 1133 snap 376 seq_num 1133) 2023-10-09T09:18:04.450 INFO:tasks.rados.rados.0.smithi163.stdout:3001: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:18:04.451 INFO:tasks.rados.rados.0.smithi163.stdout:3001: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:18:04.451 INFO:tasks.rados.rados.0.smithi163.stdout:3001: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:18:04.451 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1413 (ObjNum 1081 snap 362 seq_num 1081) dirty exists 2023-10-09T09:18:04.451 INFO:tasks.rados.rados.0.smithi163.stdout:3000: done (2 left) 2023-10-09T09:18:04.451 INFO:tasks.rados.rados.0.smithi163.stdout:3001: done (1 left) 2023-10-09T09:18:04.451 INFO:tasks.rados.rados.0.smithi163.stdout:3003: read oid 8 snap -1 2023-10-09T09:18:04.451 INFO:tasks.rados.rados.0.smithi163.stdout:3003: expect deleted 2023-10-09T09:18:04.451 INFO:tasks.rados.rados.0.smithi163.stdout:3004: snap_remove snap 352 2023-10-09T09:18:04.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:04 smithi163 bash[78107]: cluster 2023-10-09T09:18:03.239622+0000 osd.5 (osd.5) 342 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:04.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:04 smithi163 bash[78107]: cluster 2023-10-09T09:18:03.265289+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:04.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:04 smithi163 bash[78107]: cluster 2023-10-09T09:18:03.404973+0000 mgr.y (mgr.34107) 345 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 257 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2023-10-09T09:18:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:04 smithi163 bash[78107]: cluster 2023-10-09T09:18:04.250629+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:04 smithi163 bash[78107]: cluster 2023-10-09T09:18:04.253196+0000 mon.a (mon.0) 1003 : cluster 0 osdmap e810: 8 total, 8 up, 8 in 2023-10-09T09:18:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:04 smithi163 bash[75425]: cluster 2023-10-09T09:18:03.239622+0000 osd.5 (osd.5) 342 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:04 smithi163 bash[75425]: cluster 2023-10-09T09:18:03.265289+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:04 smithi163 bash[75425]: cluster 2023-10-09T09:18:03.404973+0000 mgr.y (mgr.34107) 345 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 257 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2023-10-09T09:18:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:04 smithi163 bash[75425]: cluster 2023-10-09T09:18:04.250629+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:04 smithi163 bash[75425]: cluster 2023-10-09T09:18:04.253196+0000 mon.a (mon.0) 1003 : cluster 0 osdmap e810: 8 total, 8 up, 8 in 2023-10-09T09:18:04.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:04 smithi181 bash[59345]: cluster 2023-10-09T09:18:03.239622+0000 osd.5 (osd.5) 342 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:04.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:04 smithi181 bash[59345]: cluster 2023-10-09T09:18:03.265289+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:04 smithi181 bash[59345]: cluster 2023-10-09T09:18:03.404973+0000 mgr.y (mgr.34107) 345 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 257 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2023-10-09T09:18:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:04 smithi181 bash[59345]: cluster 2023-10-09T09:18:04.250629+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:04 smithi181 bash[59345]: cluster 2023-10-09T09:18:04.253196+0000 mon.a (mon.0) 1003 : cluster 0 osdmap e810: 8 total, 8 up, 8 in 2023-10-09T09:18:05.219 INFO:tasks.rados.rados.0.smithi163.stdout:3004: done (2 left) 2023-10-09T09:18:05.219 INFO:tasks.rados.rados.0.smithi163.stdout:3002: finishing rollback tid 0 to smithi16367185-40 2023-10-09T09:18:05.219 INFO:tasks.rados.rados.0.smithi163.stdout:3002: finishing rollback tid 1 to smithi16367185-40 2023-10-09T09:18:05.219 INFO:tasks.rados.rados.0.smithi163.stdout:3002: finishing rollback tid 2 to smithi16367185-40 2023-10-09T09:18:05.219 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1501 (ObjNum 1027 snap 349 seq_num 1027) dirty exists 2023-10-09T09:18:05.219 INFO:tasks.rados.rados.0.smithi163.stdout:3005: read oid 32 snap -1 2023-10-09T09:18:05.220 INFO:tasks.rados.rados.0.smithi163.stdout:3005: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:18:05.220 INFO:tasks.rados.rados.0.smithi163.stdout:3002: done (2 left) 2023-10-09T09:18:05.220 INFO:tasks.rados.rados.0.smithi163.stdout:3003: done (1 left) 2023-10-09T09:18:05.220 INFO:tasks.rados.rados.0.smithi163.stdout:3006: read oid 27 snap -1 2023-10-09T09:18:05.220 INFO:tasks.rados.rados.0.smithi163.stdout:3006: expect (ObjNum 1130 snap 376 seq_num 1130) 2023-10-09T09:18:05.220 INFO:tasks.rados.rados.0.smithi163.stdout:3007: read oid 47 snap -1 2023-10-09T09:18:05.220 INFO:tasks.rados.rados.0.smithi163.stdout:3007: expect (ObjNum 1118 snap 369 seq_num 1118) 2023-10-09T09:18:05.220 INFO:tasks.rados.rados.0.smithi163.stdout:3008: delete oid 16 current snap is 376 2023-10-09T09:18:05.234 INFO:tasks.rados.rados.0.smithi163.stdout:3005: expect (ObjNum 880 snap 307 seq_num 880) 2023-10-09T09:18:05.427 INFO:tasks.rados.rados.0.smithi163.stdout:3006: expect (ObjNum 1130 snap 376 seq_num 1130) 2023-10-09T09:18:05.613 INFO:tasks.rados.rados.0.smithi163.stdout:3007: expect (ObjNum 1118 snap 369 seq_num 1118) 2023-10-09T09:18:05.710 INFO:tasks.rados.rados.0.smithi163.stdout:3005: done (3 left) 2023-10-09T09:18:05.710 INFO:tasks.rados.rados.0.smithi163.stdout:3006: done (2 left) 2023-10-09T09:18:05.710 INFO:tasks.rados.rados.0.smithi163.stdout:3007: done (1 left) 2023-10-09T09:18:05.710 INFO:tasks.rados.rados.0.smithi163.stdout:3008: done (0 left) 2023-10-09T09:18:05.710 INFO:tasks.rados.rados.0.smithi163.stdout:3009: snap_create 2023-10-09T09:18:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[78107]: cluster 2023-10-09T09:18:03.713580+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[78107]: cluster 2023-10-09T09:18:03.716600+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[78107]: cluster 2023-10-09T09:18:04.250173+0000 osd.5 (osd.5) 343 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[78107]: cluster 2023-10-09T09:18:04.269778+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[78107]: cluster 2023-10-09T09:18:04.272358+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[78107]: cluster 2023-10-09T09:18:05.217792+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e811: 8 total, 8 up, 8 in 2023-10-09T09:18:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[78107]: cluster 2023-10-09T09:18:05.218828+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[78107]: cluster 2023-10-09T09:18:05.224499+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[75425]: cluster 2023-10-09T09:18:03.713580+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[75425]: cluster 2023-10-09T09:18:03.716600+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[75425]: cluster 2023-10-09T09:18:04.250173+0000 osd.5 (osd.5) 343 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[75425]: cluster 2023-10-09T09:18:04.269778+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[75425]: cluster 2023-10-09T09:18:04.272358+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[75425]: cluster 2023-10-09T09:18:05.217792+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e811: 8 total, 8 up, 8 in 2023-10-09T09:18:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[75425]: cluster 2023-10-09T09:18:05.218828+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:05 smithi163 bash[75425]: cluster 2023-10-09T09:18:05.224499+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:05.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:05 smithi181 bash[59345]: cluster 2023-10-09T09:18:03.713580+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:05.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:05 smithi181 bash[59345]: cluster 2023-10-09T09:18:03.716600+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e809 with expected crc 2023-10-09T09:18:05.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:05 smithi181 bash[59345]: cluster 2023-10-09T09:18:04.250173+0000 osd.5 (osd.5) 343 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:05 smithi181 bash[59345]: cluster 2023-10-09T09:18:04.269778+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:05 smithi181 bash[59345]: cluster 2023-10-09T09:18:04.272358+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e810 with expected crc 2023-10-09T09:18:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:05 smithi181 bash[59345]: cluster 2023-10-09T09:18:05.217792+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e811: 8 total, 8 up, 8 in 2023-10-09T09:18:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:05 smithi181 bash[59345]: cluster 2023-10-09T09:18:05.218828+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:05 smithi181 bash[59345]: cluster 2023-10-09T09:18:05.224499+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:06.432 INFO:tasks.rados.rados.0.smithi163.stdout:3009: done (0 left) 2023-10-09T09:18:06.432 INFO:tasks.rados.rados.0.smithi163.stdout:3010: rollback oid 13 current snap is 377 2023-10-09T09:18:06.432 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 13 to 362 2023-10-09T09:18:06.432 INFO:tasks.rados.rados.0.smithi163.stdout:3011: snap_create 2023-10-09T09:18:06.442 INFO:tasks.rados.rados.0.smithi163.stdout:3010: finishing rollback tid 1 to smithi16367185-13 2023-10-09T09:18:06.443 INFO:tasks.rados.rados.0.smithi163.stdout:3010: finishing rollback tid 2 to smithi16367185-13 2023-10-09T09:18:06.444 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 532 (ObjNum 1088 snap 362 seq_num 1088) dirty exists 2023-10-09T09:18:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:06 smithi163 bash[78107]: cluster 2023-10-09T09:18:05.213419+0000 osd.5 (osd.5) 344 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:06 smithi163 bash[78107]: cluster 2023-10-09T09:18:05.230617+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:06 smithi163 bash[78107]: cluster 2023-10-09T09:18:05.405692+0000 mgr.y (mgr.34107) 346 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 264 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2023-10-09T09:18:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:06 smithi163 bash[75425]: cluster 2023-10-09T09:18:05.213419+0000 osd.5 (osd.5) 344 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:06 smithi163 bash[75425]: cluster 2023-10-09T09:18:05.230617+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:06 smithi163 bash[75425]: cluster 2023-10-09T09:18:05.405692+0000 mgr.y (mgr.34107) 346 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 264 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2023-10-09T09:18:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:06 smithi181 bash[59345]: cluster 2023-10-09T09:18:05.213419+0000 osd.5 (osd.5) 344 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:06 smithi181 bash[59345]: cluster 2023-10-09T09:18:05.230617+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e811 with expected crc 2023-10-09T09:18:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:06 smithi181 bash[59345]: cluster 2023-10-09T09:18:05.405692+0000 mgr.y (mgr.34107) 346 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 264 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2023-10-09T09:18:07.445 INFO:tasks.rados.rados.0.smithi163.stdout:3010: done (1 left) 2023-10-09T09:18:07.445 INFO:tasks.rados.rados.0.smithi163.stdout:3011: done (0 left) 2023-10-09T09:18:07.445 INFO:tasks.rados.rados.0.smithi163.stdout:3012: read oid 9 snap -1 2023-10-09T09:18:07.445 INFO:tasks.rados.rados.0.smithi163.stdout:3012: expect (ObjNum 1133 snap 376 seq_num 1133) 2023-10-09T09:18:07.445 INFO:tasks.rados.rados.0.smithi163.stdout:3013: write oid 11 current snap is 378 2023-10-09T09:18:07.446 INFO:tasks.rados.rados.0.smithi163.stdout:3013: seq_num 1135 ranges {675945=744212,2167533=755390} 2023-10-09T09:18:07.457 INFO:tasks.rados.rados.0.smithi163.stdout:3013: writing smithi16367185-11 from 675945 to 1420157 tid 1 2023-10-09T09:18:07.467 INFO:tasks.rados.rados.0.smithi163.stdout:3013: writing smithi16367185-11 from 2167533 to 2922923 tid 2 2023-10-09T09:18:07.467 INFO:tasks.rados.rados.0.smithi163.stdout:3014: snap_remove snap 372 2023-10-09T09:18:07.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:07 smithi163 bash[78107]: cluster 2023-10-09T09:18:06.425018+0000 osd.5 (osd.5) 345 : cluster 3 failed to encode map e812 with expected crc 2023-10-09T09:18:07.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:07 smithi163 bash[78107]: cluster 2023-10-09T09:18:06.429187+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e812: 8 total, 8 up, 8 in 2023-10-09T09:18:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:07 smithi163 bash[78107]: cluster 2023-10-09T09:18:06.435489+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e812 with expected crc 2023-10-09T09:18:07.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:07 smithi163 bash[75425]: cluster 2023-10-09T09:18:06.425018+0000 osd.5 (osd.5) 345 : cluster 3 failed to encode map e812 with expected crc 2023-10-09T09:18:07.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:07 smithi163 bash[75425]: cluster 2023-10-09T09:18:06.429187+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e812: 8 total, 8 up, 8 in 2023-10-09T09:18:07.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:07 smithi163 bash[75425]: cluster 2023-10-09T09:18:06.435489+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e812 with expected crc 2023-10-09T09:18:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:07 smithi181 bash[59345]: cluster 2023-10-09T09:18:06.425018+0000 osd.5 (osd.5) 345 : cluster 3 failed to encode map e812 with expected crc 2023-10-09T09:18:07.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:07 smithi181 bash[59345]: cluster 2023-10-09T09:18:06.429187+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e812: 8 total, 8 up, 8 in 2023-10-09T09:18:07.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:07 smithi181 bash[59345]: cluster 2023-10-09T09:18:06.435489+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e812 with expected crc 2023-10-09T09:18:08.455 INFO:tasks.rados.rados.0.smithi163.stdout:3014: done (2 left) 2023-10-09T09:18:08.456 INFO:tasks.rados.rados.0.smithi163.stdout:3015: write oid 27 current snap is 378 2023-10-09T09:18:08.456 INFO:tasks.rados.rados.0.smithi163.stdout:3015: seq_num 1136 ranges {692752=419461,1826015=615401,2979461=113291} 2023-10-09T09:18:08.465 INFO:tasks.rados.rados.0.smithi163.stdout:3015: writing smithi16367185-27 from 692752 to 1112213 tid 1 2023-10-09T09:18:08.475 INFO:tasks.rados.rados.0.smithi163.stdout:3015: writing smithi16367185-27 from 1826015 to 2441416 tid 2 2023-10-09T09:18:08.479 INFO:tasks.rados.rados.0.smithi163.stdout:3015: writing smithi16367185-27 from 2979461 to 3092752 tid 3 2023-10-09T09:18:08.479 INFO:tasks.rados.rados.0.smithi163.stdout:3016: write oid 5 current snap is 378 2023-10-09T09:18:08.479 INFO:tasks.rados.rados.0.smithi163.stdout:3016: seq_num 1137 ranges {709559=494710,1884497=759059} 2023-10-09T09:18:08.486 INFO:tasks.rados.rados.0.smithi163.stdout:3016: writing smithi16367185-5 from 709559 to 1204269 tid 1 2023-10-09T09:18:08.494 INFO:tasks.rados.rados.0.smithi163.stdout:3016: writing smithi16367185-5 from 1884497 to 2643556 tid 2 2023-10-09T09:18:08.494 INFO:tasks.rados.rados.0.smithi163.stdout:3017: read oid 1 snap -1 2023-10-09T09:18:08.494 INFO:tasks.rados.rados.0.smithi163.stdout:3017: expect (ObjNum 1073 snap 361 seq_num 1073) 2023-10-09T09:18:08.495 INFO:tasks.rados.rados.0.smithi163.stdout:3018: snap_remove snap 375 2023-10-09T09:18:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:08 smithi163 bash[78107]: cluster 2023-10-09T09:18:07.406505+0000 mgr.y (mgr.34107) 347 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 263 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2023-10-09T09:18:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:08 smithi163 bash[78107]: cluster 2023-10-09T09:18:07.442527+0000 osd.5 (osd.5) 346 : cluster 3 failed to encode map e813 with expected crc 2023-10-09T09:18:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:08 smithi163 bash[78107]: cluster 2023-10-09T09:18:07.443409+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e813 with expected crc 2023-10-09T09:18:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:08 smithi163 bash[78107]: cluster 2023-10-09T09:18:07.444912+0000 mon.a (mon.0) 1006 : cluster 0 osdmap e813: 8 total, 8 up, 8 in 2023-10-09T09:18:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:08 smithi163 bash[75425]: cluster 2023-10-09T09:18:07.406505+0000 mgr.y (mgr.34107) 347 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 263 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2023-10-09T09:18:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:08 smithi163 bash[75425]: cluster 2023-10-09T09:18:07.442527+0000 osd.5 (osd.5) 346 : cluster 3 failed to encode map e813 with expected crc 2023-10-09T09:18:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:08 smithi163 bash[75425]: cluster 2023-10-09T09:18:07.443409+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e813 with expected crc 2023-10-09T09:18:08.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:08 smithi163 bash[75425]: cluster 2023-10-09T09:18:07.444912+0000 mon.a (mon.0) 1006 : cluster 0 osdmap e813: 8 total, 8 up, 8 in 2023-10-09T09:18:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:08 smithi181 bash[59345]: cluster 2023-10-09T09:18:07.406505+0000 mgr.y (mgr.34107) 347 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 263 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2023-10-09T09:18:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:08 smithi181 bash[59345]: cluster 2023-10-09T09:18:07.442527+0000 osd.5 (osd.5) 346 : cluster 3 failed to encode map e813 with expected crc 2023-10-09T09:18:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:08 smithi181 bash[59345]: cluster 2023-10-09T09:18:07.443409+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e813 with expected crc 2023-10-09T09:18:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:08 smithi181 bash[59345]: cluster 2023-10-09T09:18:07.444912+0000 mon.a (mon.0) 1006 : cluster 0 osdmap e813: 8 total, 8 up, 8 in 2023-10-09T09:18:09.473 INFO:tasks.rados.rados.0.smithi163.stdout:3018: done (5 left) 2023-10-09T09:18:09.473 INFO:tasks.rados.rados.0.smithi163.stdout:3019: snap_create 2023-10-09T09:18:09.473 INFO:tasks.rados.rados.0.smithi163.stdout:3012: expect (ObjNum 1133 snap 376 seq_num 1133) 2023-10-09T09:18:09.626 INFO:tasks.rados.rados.0.smithi163.stdout:3013: finishing write tid 1 to smithi16367185-11 2023-10-09T09:18:09.626 INFO:tasks.rados.rados.0.smithi163.stdout:3013: finishing write tid 2 to smithi16367185-11 2023-10-09T09:18:09.626 INFO:tasks.rados.rados.0.smithi163.stdout:3013: finishing write tid 3 to smithi16367185-11 2023-10-09T09:18:09.626 INFO:tasks.rados.rados.0.smithi163.stdout:3013: finishing write tid 4 to smithi16367185-11 2023-10-09T09:18:09.626 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 803 (ObjNum 1135 snap 378 seq_num 1135) dirty exists 2023-10-09T09:18:09.626 INFO:tasks.rados.rados.0.smithi163.stdout:3013: left oid 11 (ObjNum 1135 snap 378 seq_num 1135) 2023-10-09T09:18:09.626 INFO:tasks.rados.rados.0.smithi163.stdout:3015: finishing write tid 1 to smithi16367185-27 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:3015: finishing write tid 2 to smithi16367185-27 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:3015: finishing write tid 3 to smithi16367185-27 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:3015: finishing write tid 4 to smithi16367185-27 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:3015: finishing write tid 5 to smithi16367185-27 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 883 (ObjNum 1136 snap 378 seq_num 1136) dirty exists 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:3015: left oid 27 (ObjNum 1136 snap 378 seq_num 1136) 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:3016: finishing write tid 1 to smithi16367185-5 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:3016: finishing write tid 2 to smithi16367185-5 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:3016: finishing write tid 3 to smithi16367185-5 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:3016: finishing write tid 4 to smithi16367185-5 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1443 (ObjNum 1137 snap 378 seq_num 1137) dirty exists 2023-10-09T09:18:09.627 INFO:tasks.rados.rados.0.smithi163.stdout:3016: left oid 5 (ObjNum 1137 snap 378 seq_num 1137) 2023-10-09T09:18:09.628 INFO:tasks.rados.rados.0.smithi163.stdout:3017: expect (ObjNum 1073 snap 361 seq_num 1073) 2023-10-09T09:18:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:09 smithi181 bash[59345]: cluster 2023-10-09T09:18:07.459339+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e813 with expected crc 2023-10-09T09:18:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:09 smithi181 bash[59345]: cluster 2023-10-09T09:18:08.453524+0000 osd.5 (osd.5) 347 : cluster 3 failed to encode map e814 with expected crc 2023-10-09T09:18:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:09 smithi181 bash[59345]: cluster 2023-10-09T09:18:08.455808+0000 mon.a (mon.0) 1007 : cluster 0 osdmap e814: 8 total, 8 up, 8 in 2023-10-09T09:18:09.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:09 smithi181 bash[59345]: cluster 2023-10-09T09:18:08.456303+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e814 with expected crc 2023-10-09T09:18:09.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:09 smithi163 bash[75425]: cluster 2023-10-09T09:18:07.459339+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e813 with expected crc 2023-10-09T09:18:09.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:09 smithi163 bash[75425]: cluster 2023-10-09T09:18:08.453524+0000 osd.5 (osd.5) 347 : cluster 3 failed to encode map e814 with expected crc 2023-10-09T09:18:09.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:09 smithi163 bash[75425]: cluster 2023-10-09T09:18:08.455808+0000 mon.a (mon.0) 1007 : cluster 0 osdmap e814: 8 total, 8 up, 8 in 2023-10-09T09:18:09.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:09 smithi163 bash[75425]: cluster 2023-10-09T09:18:08.456303+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e814 with expected crc 2023-10-09T09:18:09.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:09 smithi163 bash[78107]: cluster 2023-10-09T09:18:07.459339+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e813 with expected crc 2023-10-09T09:18:09.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:09 smithi163 bash[78107]: cluster 2023-10-09T09:18:08.453524+0000 osd.5 (osd.5) 347 : cluster 3 failed to encode map e814 with expected crc 2023-10-09T09:18:09.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:09 smithi163 bash[78107]: cluster 2023-10-09T09:18:08.455808+0000 mon.a (mon.0) 1007 : cluster 0 osdmap e814: 8 total, 8 up, 8 in 2023-10-09T09:18:09.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:09 smithi163 bash[78107]: cluster 2023-10-09T09:18:08.456303+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e814 with expected crc 2023-10-09T09:18:10.220 INFO:tasks.rados.rados.0.smithi163.stdout:3012: done (5 left) 2023-10-09T09:18:10.220 INFO:tasks.rados.rados.0.smithi163.stdout:3013: done (4 left) 2023-10-09T09:18:10.220 INFO:tasks.rados.rados.0.smithi163.stdout:3015: done (3 left) 2023-10-09T09:18:10.220 INFO:tasks.rados.rados.0.smithi163.stdout:3016: done (2 left) 2023-10-09T09:18:10.220 INFO:tasks.rados.rados.0.smithi163.stdout:3017: done (1 left) 2023-10-09T09:18:10.220 INFO:tasks.rados.rados.0.smithi163.stdout:3019: done (0 left) 2023-10-09T09:18:10.220 INFO:tasks.rados.rados.0.smithi163.stdout:3020: write oid 49 current snap is 379 2023-10-09T09:18:10.220 INFO:tasks.rados.rados.0.smithi163.stdout:3020: seq_num 1138 ranges {726366=569959,1826626=502717,2801601=324765} 2023-10-09T09:18:10.227 INFO:tasks.rados.rados.0.smithi163.stdout:3020: writing smithi16367185-49 from 726366 to 1296325 tid 1 2023-10-09T09:18:10.233 INFO:tasks.rados.rados.0.smithi163.stdout:3020: writing smithi16367185-49 from 1826626 to 2329343 tid 2 2023-10-09T09:18:10.238 INFO:tasks.rados.rados.0.smithi163.stdout:3020: writing smithi16367185-49 from 2801601 to 3126366 tid 3 2023-10-09T09:18:10.238 INFO:tasks.rados.rados.0.smithi163.stdout:3021: read oid 38 snap -1 2023-10-09T09:18:10.238 INFO:tasks.rados.rados.0.smithi163.stdout:3021: expect (ObjNum 1081 snap 362 seq_num 1081) 2023-10-09T09:18:10.238 INFO:tasks.rados.rados.0.smithi163.stdout:3022: rollback oid 21 current snap is 379 2023-10-09T09:18:10.238 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 366 2023-10-09T09:18:10.238 INFO:tasks.rados.rados.0.smithi163.stdout:3020: finishing write tid 1 to smithi16367185-49 2023-10-09T09:18:10.238 INFO:tasks.rados.rados.0.smithi163.stdout:3020: finishing write tid 2 to smithi16367185-49 2023-10-09T09:18:10.239 INFO:tasks.rados.rados.0.smithi163.stdout:3023: snap_create 2023-10-09T09:18:10.241 INFO:tasks.rados.rados.0.smithi163.stdout:3020: finishing write tid 3 to smithi16367185-49 2023-10-09T09:18:10.242 INFO:tasks.rados.rados.0.smithi163.stdout:3020: finishing write tid 4 to smithi16367185-49 2023-10-09T09:18:10.242 INFO:tasks.rados.rados.0.smithi163.stdout:3020: finishing write tid 5 to smithi16367185-49 2023-10-09T09:18:10.242 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 359 (ObjNum 1138 snap 379 seq_num 1138) dirty exists 2023-10-09T09:18:10.242 INFO:tasks.rados.rados.0.smithi163.stdout:3020: left oid 49 (ObjNum 1138 snap 379 seq_num 1138) 2023-10-09T09:18:10.245 INFO:tasks.rados.rados.0.smithi163.stdout:3022: finishing rollback tid 0 to smithi16367185-21 2023-10-09T09:18:10.245 INFO:tasks.rados.rados.0.smithi163.stdout:3022: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:18:10.246 INFO:tasks.rados.rados.0.smithi163.stdout:3022: finishing rollback tid 2 to smithi16367185-21 2023-10-09T09:18:10.246 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 735 (ObjNum 936 snap 320 seq_num 936) dirty exists 2023-10-09T09:18:10.249 INFO:tasks.rados.rados.0.smithi163.stdout:3021: expect (ObjNum 1081 snap 362 seq_num 1081) 2023-10-09T09:18:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[75425]: cluster 2023-10-09T09:18:09.407359+0000 mgr.y (mgr.34107) 348 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2023-10-09T09:18:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[75425]: cluster 2023-10-09T09:18:09.468571+0000 osd.5 (osd.5) 348 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[75425]: cluster 2023-10-09T09:18:09.468673+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[75425]: cluster 2023-10-09T09:18:09.473081+0000 mon.a (mon.0) 1008 : cluster 0 osdmap e815: 8 total, 8 up, 8 in 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[75425]: cluster 2023-10-09T09:18:09.534731+0000 osd.4 (osd.4) 330 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[75425]: cluster 2023-10-09T09:18:09.552106+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[75425]: cluster 2023-10-09T09:18:10.219261+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e816: 8 total, 8 up, 8 in 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[75425]: cluster 2023-10-09T09:18:10.239481+0000 osd.4 (osd.4) 331 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[75425]: audit 2023-10-09T09:18:10.347633+0000 mon.a (mon.0) 1010 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[78107]: cluster 2023-10-09T09:18:09.407359+0000 mgr.y (mgr.34107) 348 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[78107]: cluster 2023-10-09T09:18:09.468571+0000 osd.5 (osd.5) 348 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[78107]: cluster 2023-10-09T09:18:09.468673+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[78107]: cluster 2023-10-09T09:18:09.473081+0000 mon.a (mon.0) 1008 : cluster 0 osdmap e815: 8 total, 8 up, 8 in 2023-10-09T09:18:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[78107]: cluster 2023-10-09T09:18:09.534731+0000 osd.4 (osd.4) 330 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[78107]: cluster 2023-10-09T09:18:09.552106+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[78107]: cluster 2023-10-09T09:18:10.219261+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e816: 8 total, 8 up, 8 in 2023-10-09T09:18:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[78107]: cluster 2023-10-09T09:18:10.239481+0000 osd.4 (osd.4) 331 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:10 smithi163 bash[78107]: audit 2023-10-09T09:18:10.347633+0000 mon.a (mon.0) 1010 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:10.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:10 smithi181 bash[59345]: cluster 2023-10-09T09:18:09.407359+0000 mgr.y (mgr.34107) 348 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2023-10-09T09:18:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:10 smithi181 bash[59345]: cluster 2023-10-09T09:18:09.468571+0000 osd.5 (osd.5) 348 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:10 smithi181 bash[59345]: cluster 2023-10-09T09:18:09.468673+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:10 smithi181 bash[59345]: cluster 2023-10-09T09:18:09.473081+0000 mon.a (mon.0) 1008 : cluster 0 osdmap e815: 8 total, 8 up, 8 in 2023-10-09T09:18:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:10 smithi181 bash[59345]: cluster 2023-10-09T09:18:09.534731+0000 osd.4 (osd.4) 330 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:10 smithi181 bash[59345]: cluster 2023-10-09T09:18:09.552106+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e815 with expected crc 2023-10-09T09:18:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:10 smithi181 bash[59345]: cluster 2023-10-09T09:18:10.219261+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e816: 8 total, 8 up, 8 in 2023-10-09T09:18:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:10 smithi181 bash[59345]: cluster 2023-10-09T09:18:10.239481+0000 osd.4 (osd.4) 331 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:10 smithi181 bash[59345]: audit 2023-10-09T09:18:10.347633+0000 mon.a (mon.0) 1010 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:11.227 INFO:tasks.rados.rados.0.smithi163.stdout:3020: done (3 left) 2023-10-09T09:18:11.227 INFO:tasks.rados.rados.0.smithi163.stdout:3021: done (2 left) 2023-10-09T09:18:11.228 INFO:tasks.rados.rados.0.smithi163.stdout:3022: done (1 left) 2023-10-09T09:18:11.228 INFO:tasks.rados.rados.0.smithi163.stdout:3023: done (0 left) 2023-10-09T09:18:11.228 INFO:tasks.rados.rados.0.smithi163.stdout:3024: rollback oid 10 current snap is 380 2023-10-09T09:18:11.228 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 377 2023-10-09T09:18:11.228 INFO:tasks.rados.rados.0.smithi163.stdout:3025: snap_remove snap 374 2023-10-09T09:18:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:11 smithi163 bash[78107]: cluster 2023-10-09T09:18:10.213340+0000 osd.5 (osd.5) 349 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:11 smithi163 bash[78107]: cluster 2023-10-09T09:18:10.229001+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:11 smithi163 bash[78107]: cluster 2023-10-09T09:18:10.534602+0000 osd.7 (osd.7) 305 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:11 smithi163 bash[78107]: cluster 2023-10-09T09:18:11.227460+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e817: 8 total, 8 up, 8 in 2023-10-09T09:18:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:11 smithi163 bash[75425]: cluster 2023-10-09T09:18:10.213340+0000 osd.5 (osd.5) 349 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:11 smithi163 bash[75425]: cluster 2023-10-09T09:18:10.229001+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:11 smithi163 bash[75425]: cluster 2023-10-09T09:18:10.534602+0000 osd.7 (osd.7) 305 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:11 smithi163 bash[75425]: cluster 2023-10-09T09:18:11.227460+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e817: 8 total, 8 up, 8 in 2023-10-09T09:18:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:11 smithi181 bash[59345]: cluster 2023-10-09T09:18:10.213340+0000 osd.5 (osd.5) 349 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:11 smithi181 bash[59345]: cluster 2023-10-09T09:18:10.229001+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:11 smithi181 bash[59345]: cluster 2023-10-09T09:18:10.534602+0000 osd.7 (osd.7) 305 : cluster 3 failed to encode map e816 with expected crc 2023-10-09T09:18:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:11 smithi181 bash[59345]: cluster 2023-10-09T09:18:11.227460+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e817: 8 total, 8 up, 8 in 2023-10-09T09:18:12.238 INFO:tasks.rados.rados.0.smithi163.stdout:3025: done (1 left) 2023-10-09T09:18:12.238 INFO:tasks.rados.rados.0.smithi163.stdout:3026: read oid 47 snap -1 2023-10-09T09:18:12.238 INFO:tasks.rados.rados.0.smithi163.stdout:3026: expect (ObjNum 1118 snap 369 seq_num 1118) 2023-10-09T09:18:12.239 INFO:tasks.rados.rados.0.smithi163.stdout:3027: read oid 31 snap 338 2023-10-09T09:18:12.239 INFO:tasks.rados.rados.0.smithi163.stdout:3027: expect deleted 2023-10-09T09:18:12.239 INFO:tasks.rados.rados.0.smithi163.stdout:3028: write oid 20 current snap is 380 2023-10-09T09:18:12.239 INFO:tasks.rados.rados.0.smithi163.stdout:3024: finishing rollback tid 0 to smithi16367185-10 2023-10-09T09:18:12.239 INFO:tasks.rados.rados.0.smithi163.stdout:3024: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:18:12.239 INFO:tasks.rados.rados.0.smithi163.stdout:3028: seq_num 1139 ranges {743173=645208,1885108=762728} 2023-10-09T09:18:12.250 INFO:tasks.rados.rados.0.smithi163.stdout:3028: writing smithi16367185-20 from 743173 to 1388381 tid 1 2023-10-09T09:18:12.260 INFO:tasks.rados.rados.0.smithi163.stdout:3028: writing smithi16367185-20 from 1885108 to 2647836 tid 2 2023-10-09T09:18:12.260 INFO:tasks.rados.rados.0.smithi163.stdout:3029: rollback oid 8 current snap is 380 2023-10-09T09:18:12.260 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 8 to 371 2023-10-09T09:18:12.260 INFO:tasks.rados.rados.0.smithi163.stdout:3024: finishing rollback tid 2 to smithi16367185-10 2023-10-09T09:18:12.260 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 960 (ObjNum 977 snap 337 seq_num 977) dirty exists 2023-10-09T09:18:12.260 INFO:tasks.rados.rados.0.smithi163.stdout:3024: done (4 left) 2023-10-09T09:18:12.261 INFO:tasks.rados.rados.0.smithi163.stdout:3030: rollback oid 2 current snap is 380 2023-10-09T09:18:12.261 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 2 to 358 2023-10-09T09:18:12.261 INFO:tasks.rados.rados.0.smithi163.stdout:3028: finishing write tid 1 to smithi16367185-20 2023-10-09T09:18:12.261 INFO:tasks.rados.rados.0.smithi163.stdout:3026: expect (ObjNum 1118 snap 369 seq_num 1118) 2023-10-09T09:18:12.371 INFO:tasks.rados.rados.0.smithi163.stdout:3028: finishing write tid 2 to smithi16367185-20 2023-10-09T09:18:12.371 INFO:tasks.rados.rados.0.smithi163.stdout:3028: finishing write tid 3 to smithi16367185-20 2023-10-09T09:18:12.371 INFO:tasks.rados.rados.0.smithi163.stdout:3028: finishing write tid 4 to smithi16367185-20 2023-10-09T09:18:12.371 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 536 (ObjNum 1139 snap 380 seq_num 1139) dirty exists 2023-10-09T09:18:12.371 INFO:tasks.rados.rados.0.smithi163.stdout:3028: left oid 20 (ObjNum 1139 snap 380 seq_num 1139) 2023-10-09T09:18:12.371 INFO:tasks.rados.rados.0.smithi163.stdout:3030: finishing rollback tid 1 to smithi16367185-2 2023-10-09T09:18:12.371 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:18:12.371 INFO:tasks.rados.rados.0.smithi163.stdout:3029: finishing rollback tid 1 to smithi16367185-8 2023-10-09T09:18:12.371 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1487 (ObjNum 699732432 snap 21875 seq_num 1618308256) dirty dne 2023-10-09T09:18:12.372 INFO:tasks.rados.rados.0.smithi163.stdout:3026: done (4 left) 2023-10-09T09:18:12.372 INFO:tasks.rados.rados.0.smithi163.stdout:3027: done (3 left) 2023-10-09T09:18:12.372 INFO:tasks.rados.rados.0.smithi163.stdout:3028: done (2 left) 2023-10-09T09:18:12.372 INFO:tasks.rados.rados.0.smithi163.stdout:3029: done (1 left) 2023-10-09T09:18:12.372 INFO:tasks.rados.rados.0.smithi163.stdout:3030: done (0 left) 2023-10-09T09:18:12.372 INFO:tasks.rados.rados.0.smithi163.stdout:3031: delete oid 45 current snap is 380 2023-10-09T09:18:12.376 INFO:tasks.rados.rados.0.smithi163.stdout:3031: done (0 left) 2023-10-09T09:18:12.376 INFO:tasks.rados.rados.0.smithi163.stdout:3032: read oid 29 snap -1 2023-10-09T09:18:12.376 INFO:tasks.rados.rados.0.smithi163.stdout:3032: expect (ObjNum 1121 snap 371 seq_num 1121) 2023-10-09T09:18:12.376 INFO:tasks.rados.rados.0.smithi163.stdout:3033: write oid 23 current snap is 380 2023-10-09T09:18:12.376 INFO:tasks.rados.rados.0.smithi163.stdout:3033: seq_num 1141 ranges {776787=512059,1718425=766397,3000223=176564} 2023-10-09T09:18:12.388 INFO:tasks.rados.rados.0.smithi163.stdout:3033: writing smithi16367185-23 from 776787 to 1288846 tid 1 2023-10-09T09:18:12.396 INFO:tasks.rados.rados.0.smithi163.stdout:3033: writing smithi16367185-23 from 1718425 to 2484822 tid 2 2023-10-09T09:18:12.400 INFO:tasks.rados.rados.0.smithi163.stdout:3033: writing smithi16367185-23 from 3000223 to 3176787 tid 3 2023-10-09T09:18:12.400 INFO:tasks.rados.rados.0.smithi163.stdout:3034: read oid 15 snap -1 2023-10-09T09:18:12.400 INFO:tasks.rados.rados.0.smithi163.stdout:3034: expect deleted 2023-10-09T09:18:12.400 INFO:tasks.rados.rados.0.smithi163.stdout:3035: snap_create 2023-10-09T09:18:12.400 INFO:tasks.rados.rados.0.smithi163.stdout:3032: expect (ObjNum 1121 snap 371 seq_num 1121) 2023-10-09T09:18:12.506 INFO:tasks.rados.rados.0.smithi163.stdout:3033: finishing write tid 1 to smithi16367185-23 2023-10-09T09:18:12.507 INFO:tasks.rados.rados.0.smithi163.stdout:3033: finishing write tid 2 to smithi16367185-23 2023-10-09T09:18:12.507 INFO:tasks.rados.rados.0.smithi163.stdout:3033: finishing write tid 3 to smithi16367185-23 2023-10-09T09:18:12.507 INFO:tasks.rados.rados.0.smithi163.stdout:3033: finishing write tid 4 to smithi16367185-23 2023-10-09T09:18:12.507 INFO:tasks.rados.rados.0.smithi163.stdout:3033: finishing write tid 5 to smithi16367185-23 2023-10-09T09:18:12.507 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1404 (ObjNum 1141 snap 380 seq_num 1141) dirty exists 2023-10-09T09:18:12.507 INFO:tasks.rados.rados.0.smithi163.stdout:3033: left oid 23 (ObjNum 1141 snap 380 seq_num 1141) 2023-10-09T09:18:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:12 smithi181 bash[59345]: cluster 2023-10-09T09:18:11.222888+0000 osd.5 (osd.5) 350 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:12 smithi181 bash[59345]: cluster 2023-10-09T09:18:11.408396+0000 mgr.y (mgr.34107) 349 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2023-10-09T09:18:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:12 smithi181 bash[59345]: cluster 2023-10-09T09:18:11.915386+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:12 smithi181 bash[59345]: cluster 2023-10-09T09:18:11.918923+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:12 smithi181 bash[59345]: cluster 2023-10-09T09:18:12.161807+0000 osd.4 (osd.4) 332 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:12 smithi181 bash[59345]: cluster 2023-10-09T09:18:12.237493+0000 mon.a (mon.0) 1012 : cluster 0 osdmap e818: 8 total, 8 up, 8 in 2023-10-09T09:18:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:12 smithi181 bash[59345]: cluster 2023-10-09T09:18:12.261150+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e818 with expected crc 2023-10-09T09:18:12.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[75425]: cluster 2023-10-09T09:18:11.222888+0000 osd.5 (osd.5) 350 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[75425]: cluster 2023-10-09T09:18:11.408396+0000 mgr.y (mgr.34107) 349 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2023-10-09T09:18:12.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[75425]: cluster 2023-10-09T09:18:11.915386+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[75425]: cluster 2023-10-09T09:18:11.918923+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[75425]: cluster 2023-10-09T09:18:12.161807+0000 osd.4 (osd.4) 332 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[75425]: cluster 2023-10-09T09:18:12.237493+0000 mon.a (mon.0) 1012 : cluster 0 osdmap e818: 8 total, 8 up, 8 in 2023-10-09T09:18:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[75425]: cluster 2023-10-09T09:18:12.261150+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e818 with expected crc 2023-10-09T09:18:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[78107]: cluster 2023-10-09T09:18:11.222888+0000 osd.5 (osd.5) 350 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[78107]: cluster 2023-10-09T09:18:11.408396+0000 mgr.y (mgr.34107) 349 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2023-10-09T09:18:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[78107]: cluster 2023-10-09T09:18:11.915386+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[78107]: cluster 2023-10-09T09:18:11.918923+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[78107]: cluster 2023-10-09T09:18:12.161807+0000 osd.4 (osd.4) 332 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[78107]: cluster 2023-10-09T09:18:12.237493+0000 mon.a (mon.0) 1012 : cluster 0 osdmap e818: 8 total, 8 up, 8 in 2023-10-09T09:18:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:12 smithi163 bash[78107]: cluster 2023-10-09T09:18:12.261150+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e818 with expected crc 2023-10-09T09:18:13.241 INFO:tasks.rados.rados.0.smithi163.stdout:3032: done (3 left) 2023-10-09T09:18:13.242 INFO:tasks.rados.rados.0.smithi163.stdout:3033: done (2 left) 2023-10-09T09:18:13.242 INFO:tasks.rados.rados.0.smithi163.stdout:3034: done (1 left) 2023-10-09T09:18:13.242 INFO:tasks.rados.rados.0.smithi163.stdout:3035: done (0 left) 2023-10-09T09:18:13.242 INFO:tasks.rados.rados.0.smithi163.stdout:3036: snap_create 2023-10-09T09:18:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:18:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:18:13] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.33.4" 2023-10-09T09:18:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:13 smithi163 bash[78107]: cluster 2023-10-09T09:18:11.918677+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:13 smithi163 bash[78107]: cluster 2023-10-09T09:18:12.234868+0000 osd.5 (osd.5) 351 : cluster 3 failed to encode map e818 with expected crc 2023-10-09T09:18:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:13 smithi163 bash[78107]: cluster 2023-10-09T09:18:12.242074+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e818 with expected crc 2023-10-09T09:18:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:13 smithi163 bash[78107]: cluster 2023-10-09T09:18:13.241380+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e819: 8 total, 8 up, 8 in 2023-10-09T09:18:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:13 smithi163 bash[75425]: cluster 2023-10-09T09:18:11.918677+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:13 smithi163 bash[75425]: cluster 2023-10-09T09:18:12.234868+0000 osd.5 (osd.5) 351 : cluster 3 failed to encode map e818 with expected crc 2023-10-09T09:18:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:13 smithi163 bash[75425]: cluster 2023-10-09T09:18:12.242074+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e818 with expected crc 2023-10-09T09:18:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:13 smithi163 bash[75425]: cluster 2023-10-09T09:18:13.241380+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e819: 8 total, 8 up, 8 in 2023-10-09T09:18:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:13 smithi181 bash[59345]: cluster 2023-10-09T09:18:11.918677+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e817 with expected crc 2023-10-09T09:18:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:13 smithi181 bash[59345]: cluster 2023-10-09T09:18:12.234868+0000 osd.5 (osd.5) 351 : cluster 3 failed to encode map e818 with expected crc 2023-10-09T09:18:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:13 smithi181 bash[59345]: cluster 2023-10-09T09:18:12.242074+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e818 with expected crc 2023-10-09T09:18:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:13 smithi181 bash[59345]: cluster 2023-10-09T09:18:13.241380+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e819: 8 total, 8 up, 8 in 2023-10-09T09:18:14.247 INFO:tasks.rados.rados.0.smithi163.stdout:3036: done (0 left) 2023-10-09T09:18:14.247 INFO:tasks.rados.rados.0.smithi163.stdout:3037: read oid 22 snap -1 2023-10-09T09:18:14.247 INFO:tasks.rados.rados.0.smithi163.stdout:3037: expect (ObjNum 1115 snap 368 seq_num 1115) 2023-10-09T09:18:14.247 INFO:tasks.rados.rados.0.smithi163.stdout:3038: snap_create 2023-10-09T09:18:14.261 INFO:tasks.rados.rados.0.smithi163.stdout:3037: expect (ObjNum 1115 snap 368 seq_num 1115) 2023-10-09T09:18:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:14 smithi181 bash[59345]: cluster 2023-10-09T09:18:13.235465+0000 osd.5 (osd.5) 352 : cluster 3 failed to encode map e819 with expected crc 2023-10-09T09:18:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:14 smithi181 bash[59345]: cluster 2023-10-09T09:18:13.409067+0000 mgr.y (mgr.34107) 350 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 273 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2023-10-09T09:18:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:14 smithi181 bash[59345]: cluster 2023-10-09T09:18:13.521978+0000 osd.4 (osd.4) 333 : cluster 3 failed to encode map e819 with expected crc 2023-10-09T09:18:14.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:14 smithi181 bash[59345]: cluster 2023-10-09T09:18:14.246728+0000 mon.a (mon.0) 1014 : cluster 0 osdmap e820: 8 total, 8 up, 8 in 2023-10-09T09:18:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:14 smithi163 bash[75425]: cluster 2023-10-09T09:18:13.235465+0000 osd.5 (osd.5) 352 : cluster 3 failed to encode map e819 with expected crc 2023-10-09T09:18:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:14 smithi163 bash[75425]: cluster 2023-10-09T09:18:13.409067+0000 mgr.y (mgr.34107) 350 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 273 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2023-10-09T09:18:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:14 smithi163 bash[75425]: cluster 2023-10-09T09:18:13.521978+0000 osd.4 (osd.4) 333 : cluster 3 failed to encode map e819 with expected crc 2023-10-09T09:18:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:14 smithi163 bash[75425]: cluster 2023-10-09T09:18:14.246728+0000 mon.a (mon.0) 1014 : cluster 0 osdmap e820: 8 total, 8 up, 8 in 2023-10-09T09:18:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:14 smithi163 bash[78107]: cluster 2023-10-09T09:18:13.235465+0000 osd.5 (osd.5) 352 : cluster 3 failed to encode map e819 with expected crc 2023-10-09T09:18:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:14 smithi163 bash[78107]: cluster 2023-10-09T09:18:13.409067+0000 mgr.y (mgr.34107) 350 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 273 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2023-10-09T09:18:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:14 smithi163 bash[78107]: cluster 2023-10-09T09:18:13.521978+0000 osd.4 (osd.4) 333 : cluster 3 failed to encode map e819 with expected crc 2023-10-09T09:18:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:14 smithi163 bash[78107]: cluster 2023-10-09T09:18:14.246728+0000 mon.a (mon.0) 1014 : cluster 0 osdmap e820: 8 total, 8 up, 8 in 2023-10-09T09:18:15.219 INFO:tasks.rados.rados.0.smithi163.stdout:3037: done (1 left) 2023-10-09T09:18:15.219 INFO:tasks.rados.rados.0.smithi163.stdout:3038: done (0 left) 2023-10-09T09:18:15.219 INFO:tasks.rados.rados.0.smithi163.stdout:3039: read oid 29 snap -1 2023-10-09T09:18:15.219 INFO:tasks.rados.rados.0.smithi163.stdout:3039: expect (ObjNum 1121 snap 371 seq_num 1121) 2023-10-09T09:18:15.219 INFO:tasks.rados.rados.0.smithi163.stdout:3040: snap_create 2023-10-09T09:18:15.229 INFO:tasks.rados.rados.0.smithi163.stdout:3039: expect (ObjNum 1121 snap 371 seq_num 1121) 2023-10-09T09:18:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:15 smithi181 bash[59345]: cluster 2023-10-09T09:18:14.242913+0000 osd.5 (osd.5) 353 : cluster 3 failed to encode map e820 with expected crc 2023-10-09T09:18:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:15 smithi181 bash[59345]: cluster 2023-10-09T09:18:14.243263+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e820 with expected crc 2023-10-09T09:18:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:15 smithi181 bash[59345]: cluster 2023-10-09T09:18:15.218554+0000 mon.a (mon.0) 1015 : cluster 0 osdmap e821: 8 total, 8 up, 8 in 2023-10-09T09:18:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:15 smithi163 bash[78107]: cluster 2023-10-09T09:18:14.242913+0000 osd.5 (osd.5) 353 : cluster 3 failed to encode map e820 with expected crc 2023-10-09T09:18:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:15 smithi163 bash[78107]: cluster 2023-10-09T09:18:14.243263+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e820 with expected crc 2023-10-09T09:18:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:15 smithi163 bash[78107]: cluster 2023-10-09T09:18:15.218554+0000 mon.a (mon.0) 1015 : cluster 0 osdmap e821: 8 total, 8 up, 8 in 2023-10-09T09:18:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:15 smithi163 bash[75425]: cluster 2023-10-09T09:18:14.242913+0000 osd.5 (osd.5) 353 : cluster 3 failed to encode map e820 with expected crc 2023-10-09T09:18:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:15 smithi163 bash[75425]: cluster 2023-10-09T09:18:14.243263+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e820 with expected crc 2023-10-09T09:18:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:15 smithi163 bash[75425]: cluster 2023-10-09T09:18:15.218554+0000 mon.a (mon.0) 1015 : cluster 0 osdmap e821: 8 total, 8 up, 8 in 2023-10-09T09:18:16.221 INFO:tasks.rados.rados.0.smithi163.stdout:3039: done (1 left) 2023-10-09T09:18:16.221 INFO:tasks.rados.rados.0.smithi163.stdout:3040: done (0 left) 2023-10-09T09:18:16.221 INFO:tasks.rados.rados.0.smithi163.stdout:3041: write oid 19 current snap is 384 2023-10-09T09:18:16.221 INFO:tasks.rados.rados.0.smithi163.stdout:3041: seq_num 1142 ranges {793594=587308,2060554=510055} 2023-10-09T09:18:16.232 INFO:tasks.rados.rados.0.smithi163.stdout:3041: writing smithi16367185-19 from 793594 to 1380902 tid 1 2023-10-09T09:18:16.240 INFO:tasks.rados.rados.0.smithi163.stdout:3041: writing smithi16367185-19 from 2060554 to 2570609 tid 2 2023-10-09T09:18:16.240 INFO:tasks.rados.rados.0.smithi163.stdout:3042: delete oid 35 current snap is 384 2023-10-09T09:18:16.244 INFO:tasks.rados.rados.0.smithi163.stdout:3041: finishing write tid 1 to smithi16367185-19 2023-10-09T09:18:16.247 INFO:tasks.rados.rados.0.smithi163.stdout:3042: done (1 left) 2023-10-09T09:18:16.247 INFO:tasks.rados.rados.0.smithi163.stdout:3043: write oid 45 current snap is 384 2023-10-09T09:18:16.247 INFO:tasks.rados.rados.0.smithi163.stdout:3041: finishing write tid 2 to smithi16367185-19 2023-10-09T09:18:16.247 INFO:tasks.rados.rados.0.smithi163.stdout:3043: seq_num 1144 ranges {427208=737806,1777518=513724,2966139=261069} 2023-10-09T09:18:16.258 INFO:tasks.rados.rados.0.smithi163.stdout:3043: writing smithi16367185-45 from 427208 to 1165014 tid 1 2023-10-09T09:18:16.265 INFO:tasks.rados.rados.0.smithi163.stdout:3043: writing smithi16367185-45 from 1777518 to 2291242 tid 2 2023-10-09T09:18:16.271 INFO:tasks.rados.rados.0.smithi163.stdout:3043: writing smithi16367185-45 from 2966139 to 3227208 tid 3 2023-10-09T09:18:16.271 INFO:tasks.rados.rados.0.smithi163.stdout:3044: write oid 46 current snap is 384 2023-10-09T09:18:16.271 INFO:tasks.rados.rados.0.smithi163.stdout:3041: finishing write tid 3 to smithi16367185-19 2023-10-09T09:18:16.271 INFO:tasks.rados.rados.0.smithi163.stdout:3041: finishing write tid 4 to smithi16367185-19 2023-10-09T09:18:16.271 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 890 (ObjNum 1142 snap 384 seq_num 1142) dirty exists 2023-10-09T09:18:16.271 INFO:tasks.rados.rados.0.smithi163.stdout:3041: left oid 19 (ObjNum 1142 snap 384 seq_num 1142) 2023-10-09T09:18:16.271 INFO:tasks.rados.rados.0.smithi163.stdout:3043: finishing write tid 1 to smithi16367185-45 2023-10-09T09:18:16.272 INFO:tasks.rados.rados.0.smithi163.stdout:3043: finishing write tid 2 to smithi16367185-45 2023-10-09T09:18:16.272 INFO:tasks.rados.rados.0.smithi163.stdout:3044: seq_num 1145 ranges {444015=413055,1436000=773735,2709915=534100} 2023-10-09T09:18:16.276 INFO:tasks.rados.rados.0.smithi163.stdout:3044: writing smithi16367185-46 from 444015 to 857070 tid 1 2023-10-09T09:18:16.283 INFO:tasks.rados.rados.0.smithi163.stdout:3044: writing smithi16367185-46 from 1436000 to 2209735 tid 2 2023-10-09T09:18:16.289 INFO:tasks.rados.rados.0.smithi163.stdout:3044: writing smithi16367185-46 from 2709915 to 3244015 tid 3 2023-10-09T09:18:16.290 INFO:tasks.rados.rados.0.smithi163.stdout:3041: done (2 left) 2023-10-09T09:18:16.290 INFO:tasks.rados.rados.0.smithi163.stdout:3045: snap_create 2023-10-09T09:18:16.290 INFO:tasks.rados.rados.0.smithi163.stdout:3043: finishing write tid 3 to smithi16367185-45 2023-10-09T09:18:16.290 INFO:tasks.rados.rados.0.smithi163.stdout:3043: finishing write tid 4 to smithi16367185-45 2023-10-09T09:18:16.290 INFO:tasks.rados.rados.0.smithi163.stdout:3043: finishing write tid 5 to smithi16367185-45 2023-10-09T09:18:16.290 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 2156 (ObjNum 1144 snap 384 seq_num 1144) dirty exists 2023-10-09T09:18:16.290 INFO:tasks.rados.rados.0.smithi163.stdout:3043: left oid 45 (ObjNum 1144 snap 384 seq_num 1144) 2023-10-09T09:18:16.290 INFO:tasks.rados.rados.0.smithi163.stdout:3044: finishing write tid 1 to smithi16367185-46 2023-10-09T09:18:16.290 INFO:tasks.rados.rados.0.smithi163.stdout:3044: finishing write tid 2 to smithi16367185-46 2023-10-09T09:18:16.294 INFO:tasks.rados.rados.0.smithi163.stdout:3044: finishing write tid 3 to smithi16367185-46 2023-10-09T09:18:16.294 INFO:tasks.rados.rados.0.smithi163.stdout:3044: finishing write tid 4 to smithi16367185-46 2023-10-09T09:18:16.295 INFO:tasks.rados.rados.0.smithi163.stdout:3044: finishing write tid 5 to smithi16367185-46 2023-10-09T09:18:16.295 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 1436 (ObjNum 1145 snap 384 seq_num 1145) dirty exists 2023-10-09T09:18:16.295 INFO:tasks.rados.rados.0.smithi163.stdout:3044: left oid 46 (ObjNum 1145 snap 384 seq_num 1145) 2023-10-09T09:18:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:16 smithi181 bash[59345]: cluster 2023-10-09T09:18:15.213793+0000 osd.5 (osd.5) 354 : cluster 3 failed to encode map e821 with expected crc 2023-10-09T09:18:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:16 smithi181 bash[59345]: cluster 2023-10-09T09:18:15.215021+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e821 with expected crc 2023-10-09T09:18:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:16 smithi181 bash[59345]: cluster 2023-10-09T09:18:15.409787+0000 mgr.y (mgr.34107) 351 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 273 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 341 KiB/s wr, 6 op/s 2023-10-09T09:18:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:16 smithi181 bash[59345]: cluster 2023-10-09T09:18:16.217267+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e822 with expected crc 2023-10-09T09:18:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:16 smithi181 bash[59345]: cluster 2023-10-09T09:18:16.220559+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e822: 8 total, 8 up, 8 in 2023-10-09T09:18:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:16 smithi181 bash[59345]: cluster 2023-10-09T09:18:16.237890+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e822 with expected crc 2023-10-09T09:18:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[75425]: cluster 2023-10-09T09:18:15.213793+0000 osd.5 (osd.5) 354 : cluster 3 failed to encode map e821 with expected crc 2023-10-09T09:18:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[75425]: cluster 2023-10-09T09:18:15.215021+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e821 with expected crc 2023-10-09T09:18:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[75425]: cluster 2023-10-09T09:18:15.409787+0000 mgr.y (mgr.34107) 351 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 273 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 341 KiB/s wr, 6 op/s 2023-10-09T09:18:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[75425]: cluster 2023-10-09T09:18:16.217267+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e822 with expected crc 2023-10-09T09:18:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[75425]: cluster 2023-10-09T09:18:16.220559+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e822: 8 total, 8 up, 8 in 2023-10-09T09:18:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[75425]: cluster 2023-10-09T09:18:16.237890+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e822 with expected crc 2023-10-09T09:18:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[78107]: cluster 2023-10-09T09:18:15.213793+0000 osd.5 (osd.5) 354 : cluster 3 failed to encode map e821 with expected crc 2023-10-09T09:18:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[78107]: cluster 2023-10-09T09:18:15.215021+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e821 with expected crc 2023-10-09T09:18:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[78107]: cluster 2023-10-09T09:18:15.409787+0000 mgr.y (mgr.34107) 351 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 273 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 341 KiB/s wr, 6 op/s 2023-10-09T09:18:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[78107]: cluster 2023-10-09T09:18:16.217267+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e822 with expected crc 2023-10-09T09:18:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[78107]: cluster 2023-10-09T09:18:16.220559+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e822: 8 total, 8 up, 8 in 2023-10-09T09:18:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:16 smithi163 bash[78107]: cluster 2023-10-09T09:18:16.237890+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e822 with expected crc 2023-10-09T09:18:17.232 INFO:tasks.rados.rados.0.smithi163.stdout:3043: done (2 left) 2023-10-09T09:18:17.232 INFO:tasks.rados.rados.0.smithi163.stdout:3044: done (1 left) 2023-10-09T09:18:17.232 INFO:tasks.rados.rados.0.smithi163.stdout:3045: done (0 left) 2023-10-09T09:18:17.232 INFO:tasks.rados.rados.0.smithi163.stdout:3046: snap_create 2023-10-09T09:18:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:17 smithi181 bash[59345]: cluster 2023-10-09T09:18:16.216088+0000 osd.5 (osd.5) 355 : cluster 3 failed to encode map e822 with expected crc 2023-10-09T09:18:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:17 smithi181 bash[59345]: cluster 2023-10-09T09:18:17.232146+0000 mon.a (mon.0) 1017 : cluster 0 osdmap e823: 8 total, 8 up, 8 in 2023-10-09T09:18:17.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:17 smithi163 bash[78107]: cluster 2023-10-09T09:18:16.216088+0000 osd.5 (osd.5) 355 : cluster 3 failed to encode map e822 with expected crc 2023-10-09T09:18:17.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:17 smithi163 bash[78107]: cluster 2023-10-09T09:18:17.232146+0000 mon.a (mon.0) 1017 : cluster 0 osdmap e823: 8 total, 8 up, 8 in 2023-10-09T09:18:17.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:17 smithi163 bash[75425]: cluster 2023-10-09T09:18:16.216088+0000 osd.5 (osd.5) 355 : cluster 3 failed to encode map e822 with expected crc 2023-10-09T09:18:17.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:17 smithi163 bash[75425]: cluster 2023-10-09T09:18:17.232146+0000 mon.a (mon.0) 1017 : cluster 0 osdmap e823: 8 total, 8 up, 8 in 2023-10-09T09:18:18.236 INFO:tasks.rados.rados.0.smithi163.stdout:3046: done (0 left) 2023-10-09T09:18:18.236 INFO:tasks.rados.rados.0.smithi163.stdout:3047: snap_create 2023-10-09T09:18:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:18 smithi181 bash[59345]: cluster 2023-10-09T09:18:17.226785+0000 osd.5 (osd.5) 356 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:18 smithi181 bash[59345]: cluster 2023-10-09T09:18:17.410542+0000 mgr.y (mgr.34107) 352 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 281 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2023-10-09T09:18:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:18 smithi181 bash[59345]: cluster 2023-10-09T09:18:18.236221+0000 mon.a (mon.0) 1018 : cluster 0 osdmap e824: 8 total, 8 up, 8 in 2023-10-09T09:18:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:18 smithi181 bash[59345]: cluster 2023-10-09T09:18:18.361297+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:18 smithi181 bash[59345]: cluster 2023-10-09T09:18:18.367349+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:18 smithi163 bash[78107]: cluster 2023-10-09T09:18:17.226785+0000 osd.5 (osd.5) 356 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:18 smithi163 bash[78107]: cluster 2023-10-09T09:18:17.410542+0000 mgr.y (mgr.34107) 352 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 281 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2023-10-09T09:18:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:18 smithi163 bash[78107]: cluster 2023-10-09T09:18:18.236221+0000 mon.a (mon.0) 1018 : cluster 0 osdmap e824: 8 total, 8 up, 8 in 2023-10-09T09:18:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:18 smithi163 bash[78107]: cluster 2023-10-09T09:18:18.361297+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:18 smithi163 bash[78107]: cluster 2023-10-09T09:18:18.367349+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:18 smithi163 bash[75425]: cluster 2023-10-09T09:18:17.226785+0000 osd.5 (osd.5) 356 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:18 smithi163 bash[75425]: cluster 2023-10-09T09:18:17.410542+0000 mgr.y (mgr.34107) 352 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 281 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2023-10-09T09:18:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:18 smithi163 bash[75425]: cluster 2023-10-09T09:18:18.236221+0000 mon.a (mon.0) 1018 : cluster 0 osdmap e824: 8 total, 8 up, 8 in 2023-10-09T09:18:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:18 smithi163 bash[75425]: cluster 2023-10-09T09:18:18.361297+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:18 smithi163 bash[75425]: cluster 2023-10-09T09:18:18.367349+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:19.240 INFO:tasks.rados.rados.0.smithi163.stdout:3047: done (0 left) 2023-10-09T09:18:19.240 INFO:tasks.rados.rados.0.smithi163.stdout:3048: delete oid 36 current snap is 387 2023-10-09T09:18:19.246 INFO:tasks.rados.rados.0.smithi163.stdout:3048: done (0 left) 2023-10-09T09:18:19.246 INFO:tasks.rados.rados.0.smithi163.stdout:3049: read oid 20 snap -1 2023-10-09T09:18:19.246 INFO:tasks.rados.rados.0.smithi163.stdout:3049: expect (ObjNum 1139 snap 380 seq_num 1139) 2023-10-09T09:18:19.246 INFO:tasks.rados.rados.0.smithi163.stdout:3050: snap_remove snap 380 2023-10-09T09:18:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:19 smithi181 bash[59345]: cluster 2023-10-09T09:18:17.819868+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:19 smithi181 bash[59345]: cluster 2023-10-09T09:18:18.233618+0000 osd.5 (osd.5) 357 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:19 smithi181 bash[59345]: cluster 2023-10-09T09:18:18.234305+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:19 smithi181 bash[59345]: cluster 2023-10-09T09:18:18.361097+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:19 smithi181 bash[59345]: cluster 2023-10-09T09:18:18.368795+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:19 smithi181 bash[59345]: cluster 2023-10-09T09:18:19.236534+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e825 with expected crc 2023-10-09T09:18:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:19 smithi181 bash[59345]: cluster 2023-10-09T09:18:19.240090+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e825: 8 total, 8 up, 8 in 2023-10-09T09:18:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[78107]: cluster 2023-10-09T09:18:17.819868+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[78107]: cluster 2023-10-09T09:18:18.233618+0000 osd.5 (osd.5) 357 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[78107]: cluster 2023-10-09T09:18:18.234305+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[78107]: cluster 2023-10-09T09:18:18.361097+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:19.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[78107]: cluster 2023-10-09T09:18:18.368795+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:19.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[78107]: cluster 2023-10-09T09:18:19.236534+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e825 with expected crc 2023-10-09T09:18:19.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[78107]: cluster 2023-10-09T09:18:19.240090+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e825: 8 total, 8 up, 8 in 2023-10-09T09:18:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[75425]: cluster 2023-10-09T09:18:17.819868+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[75425]: cluster 2023-10-09T09:18:18.233618+0000 osd.5 (osd.5) 357 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[75425]: cluster 2023-10-09T09:18:18.234305+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[75425]: cluster 2023-10-09T09:18:18.361097+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e823 with expected crc 2023-10-09T09:18:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[75425]: cluster 2023-10-09T09:18:18.368795+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e824 with expected crc 2023-10-09T09:18:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[75425]: cluster 2023-10-09T09:18:19.236534+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e825 with expected crc 2023-10-09T09:18:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:19 smithi163 bash[75425]: cluster 2023-10-09T09:18:19.240090+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e825: 8 total, 8 up, 8 in 2023-10-09T09:18:20.220 INFO:tasks.rados.rados.0.smithi163.stdout:3050: done (1 left) 2023-10-09T09:18:20.220 INFO:tasks.rados.rados.0.smithi163.stdout:3051: snap_remove snap 381 2023-10-09T09:18:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:20 smithi181 bash[59345]: cluster 2023-10-09T09:18:19.235784+0000 osd.5 (osd.5) 358 : cluster 3 failed to encode map e825 with expected crc 2023-10-09T09:18:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:20 smithi181 bash[59345]: cluster 2023-10-09T09:18:19.247434+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e825 with expected crc 2023-10-09T09:18:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:20 smithi181 bash[59345]: cluster 2023-10-09T09:18:19.411317+0000 mgr.y (mgr.34107) 353 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 281 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.0 MiB/s wr, 9 op/s 2023-10-09T09:18:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:20 smithi181 bash[59345]: cluster 2023-10-09T09:18:20.216916+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e826 with expected crc 2023-10-09T09:18:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:20 smithi181 bash[59345]: cluster 2023-10-09T09:18:20.220239+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e826: 8 total, 8 up, 8 in 2023-10-09T09:18:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:20 smithi163 bash[75425]: cluster 2023-10-09T09:18:19.235784+0000 osd.5 (osd.5) 358 : cluster 3 failed to encode map e825 with expected crc 2023-10-09T09:18:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:20 smithi163 bash[75425]: cluster 2023-10-09T09:18:19.247434+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e825 with expected crc 2023-10-09T09:18:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:20 smithi163 bash[75425]: cluster 2023-10-09T09:18:19.411317+0000 mgr.y (mgr.34107) 353 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 281 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.0 MiB/s wr, 9 op/s 2023-10-09T09:18:20.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:20 smithi163 bash[75425]: cluster 2023-10-09T09:18:20.216916+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e826 with expected crc 2023-10-09T09:18:20.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:20 smithi163 bash[75425]: cluster 2023-10-09T09:18:20.220239+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e826: 8 total, 8 up, 8 in 2023-10-09T09:18:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:20 smithi163 bash[78107]: cluster 2023-10-09T09:18:19.235784+0000 osd.5 (osd.5) 358 : cluster 3 failed to encode map e825 with expected crc 2023-10-09T09:18:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:20 smithi163 bash[78107]: cluster 2023-10-09T09:18:19.247434+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e825 with expected crc 2023-10-09T09:18:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:20 smithi163 bash[78107]: cluster 2023-10-09T09:18:19.411317+0000 mgr.y (mgr.34107) 353 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 281 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.0 MiB/s wr, 9 op/s 2023-10-09T09:18:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:20 smithi163 bash[78107]: cluster 2023-10-09T09:18:20.216916+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e826 with expected crc 2023-10-09T09:18:20.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:20 smithi163 bash[78107]: cluster 2023-10-09T09:18:20.220239+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e826: 8 total, 8 up, 8 in 2023-10-09T09:18:21.224 INFO:tasks.rados.rados.0.smithi163.stdout:3051: done (1 left) 2023-10-09T09:18:21.224 INFO:tasks.rados.rados.0.smithi163.stdout:3052: read oid 48 snap -1 2023-10-09T09:18:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:3052: expect (ObjNum 1108 snap 367 seq_num 1108) 2023-10-09T09:18:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:3053: write oid 11 current snap is 387 2023-10-09T09:18:21.225 INFO:tasks.rados.rados.0.smithi163.stdout:3053: seq_num 1147 ranges {477629=563553,1836611=777404,3048408=229221} 2023-10-09T09:18:21.235 INFO:tasks.rados.rados.0.smithi163.stdout:3053: writing smithi16367185-11 from 477629 to 1041182 tid 1 2023-10-09T09:18:21.244 INFO:tasks.rados.rados.0.smithi163.stdout:3053: writing smithi16367185-11 from 1836611 to 2614015 tid 2 2023-10-09T09:18:21.248 INFO:tasks.rados.rados.0.smithi163.stdout:3053: writing smithi16367185-11 from 3048408 to 3277629 tid 3 2023-10-09T09:18:21.249 INFO:tasks.rados.rados.0.smithi163.stdout:3054: read oid 50 snap -1 2023-10-09T09:18:21.249 INFO:tasks.rados.rados.0.smithi163.stdout:3054: expect (ObjNum 1086 snap 362 seq_num 1086) 2023-10-09T09:18:21.249 INFO:tasks.rados.rados.0.smithi163.stdout:3055: delete oid 29 current snap is 387 2023-10-09T09:18:21.249 INFO:tasks.rados.rados.0.smithi163.stdout:3049: expect (ObjNum 1139 snap 380 seq_num 1139) 2023-10-09T09:18:21.441 INFO:tasks.rados.rados.0.smithi163.stdout:3052: expect (ObjNum 1108 snap 367 seq_num 1108) 2023-10-09T09:18:21.629 INFO:tasks.rados.rados.0.smithi163.stdout:3053: finishing write tid 1 to smithi16367185-11 2023-10-09T09:18:21.630 INFO:tasks.rados.rados.0.smithi163.stdout:3053: finishing write tid 2 to smithi16367185-11 2023-10-09T09:18:21.630 INFO:tasks.rados.rados.0.smithi163.stdout:3053: finishing write tid 3 to smithi16367185-11 2023-10-09T09:18:21.630 INFO:tasks.rados.rados.0.smithi163.stdout:3053: finishing write tid 4 to smithi16367185-11 2023-10-09T09:18:21.630 INFO:tasks.rados.rados.0.smithi163.stdout:3053: finishing write tid 5 to smithi16367185-11 2023-10-09T09:18:21.630 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 815 (ObjNum 1147 snap 387 seq_num 1147) dirty exists 2023-10-09T09:18:21.630 INFO:tasks.rados.rados.0.smithi163.stdout:3053: left oid 11 (ObjNum 1147 snap 387 seq_num 1147) 2023-10-09T09:18:21.630 INFO:tasks.rados.rados.0.smithi163.stdout:3054: expect (ObjNum 1086 snap 362 seq_num 1086) 2023-10-09T09:18:21.705 INFO:tasks.rados.rados.0.smithi163.stdout:3049: done (4 left) 2023-10-09T09:18:21.705 INFO:tasks.rados.rados.0.smithi163.stdout:3052: done (3 left) 2023-10-09T09:18:21.705 INFO:tasks.rados.rados.0.smithi163.stdout:3053: done (2 left) 2023-10-09T09:18:21.705 INFO:tasks.rados.rados.0.smithi163.stdout:3054: done (1 left) 2023-10-09T09:18:21.705 INFO:tasks.rados.rados.0.smithi163.stdout:3055: done (0 left) 2023-10-09T09:18:21.705 INFO:tasks.rados.rados.0.smithi163.stdout:3056: read oid 25 snap -1 2023-10-09T09:18:21.705 INFO:tasks.rados.rados.0.smithi163.stdout:3056: expect (ObjNum 1004 snap 342 seq_num 1004) 2023-10-09T09:18:21.706 INFO:tasks.rados.rados.0.smithi163.stdout:3057: snap_create 2023-10-09T09:18:21.709 INFO:tasks.rados.rados.0.smithi163.stdout:3056: expect (ObjNum 1004 snap 342 seq_num 1004) 2023-10-09T09:18:21.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:21 smithi181 bash[59345]: cluster 2023-10-09T09:18:20.216255+0000 osd.5 (osd.5) 359 : cluster 3 failed to encode map e826 with expected crc 2023-10-09T09:18:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:21 smithi181 bash[59345]: cluster 2023-10-09T09:18:21.219546+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e827 with expected crc 2023-10-09T09:18:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:21 smithi181 bash[59345]: cluster 2023-10-09T09:18:21.223481+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e827: 8 total, 8 up, 8 in 2023-10-09T09:18:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:21 smithi181 bash[59345]: cluster 2023-10-09T09:18:21.237838+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e827 with expected crc 2023-10-09T09:18:21.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:21 smithi163 bash[78107]: cluster 2023-10-09T09:18:20.216255+0000 osd.5 (osd.5) 359 : cluster 3 failed to encode map e826 with expected crc 2023-10-09T09:18:21.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:21 smithi163 bash[78107]: cluster 2023-10-09T09:18:21.219546+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e827 with expected crc 2023-10-09T09:18:21.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:21 smithi163 bash[78107]: cluster 2023-10-09T09:18:21.223481+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e827: 8 total, 8 up, 8 in 2023-10-09T09:18:21.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:21 smithi163 bash[78107]: cluster 2023-10-09T09:18:21.237838+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e827 with expected crc 2023-10-09T09:18:21.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:21 smithi163 bash[75425]: cluster 2023-10-09T09:18:20.216255+0000 osd.5 (osd.5) 359 : cluster 3 failed to encode map e826 with expected crc 2023-10-09T09:18:21.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:21 smithi163 bash[75425]: cluster 2023-10-09T09:18:21.219546+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e827 with expected crc 2023-10-09T09:18:21.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:21 smithi163 bash[75425]: cluster 2023-10-09T09:18:21.223481+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e827: 8 total, 8 up, 8 in 2023-10-09T09:18:21.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:21 smithi163 bash[75425]: cluster 2023-10-09T09:18:21.237838+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e827 with expected crc 2023-10-09T09:18:22.508 INFO:tasks.rados.rados.0.smithi163.stdout:3056: done (1 left) 2023-10-09T09:18:22.508 INFO:tasks.rados.rados.0.smithi163.stdout:3057: done (0 left) 2023-10-09T09:18:22.508 INFO:tasks.rados.rados.0.smithi163.stdout:3058: rollback oid 39 current snap is 388 2023-10-09T09:18:22.508 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 365 2023-10-09T09:18:22.509 INFO:tasks.rados.rados.0.smithi163.stdout:3059: delete oid 40 current snap is 388 2023-10-09T09:18:22.517 INFO:tasks.rados.rados.0.smithi163.stdout:3058: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:18:22.517 INFO:tasks.rados.rados.0.smithi163.stdout:3058: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:18:22.517 INFO:tasks.rados.rados.0.smithi163.stdout:3058: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:18:22.517 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1408 (ObjNum 1090 snap 362 seq_num 1090) dirty exists 2023-10-09T09:18:22.522 INFO:tasks.rados.rados.0.smithi163.stdout:3058: done (1 left) 2023-10-09T09:18:22.522 INFO:tasks.rados.rados.0.smithi163.stdout:3059: done (0 left) 2023-10-09T09:18:22.522 INFO:tasks.rados.rados.0.smithi163.stdout:3060: write oid 14 current snap is 388 2023-10-09T09:18:22.522 INFO:tasks.rados.rados.0.smithi163.stdout:3060: seq_num 1150 ranges {528050=505653,1612057=524731,2614324=567403} 2023-10-09T09:18:22.530 INFO:tasks.rados.rados.0.smithi163.stdout:3060: writing smithi16367185-14 from 528050 to 1033703 tid 1 2023-10-09T09:18:22.537 INFO:tasks.rados.rados.0.smithi163.stdout:3060: writing smithi16367185-14 from 1612057 to 2136788 tid 2 2023-10-09T09:18:22.544 INFO:tasks.rados.rados.0.smithi163.stdout:3060: writing smithi16367185-14 from 2614324 to 3181727 tid 3 2023-10-09T09:18:22.544 INFO:tasks.rados.rados.0.smithi163.stdout:3061: read oid 25 snap -1 2023-10-09T09:18:22.544 INFO:tasks.rados.rados.0.smithi163.stdout:3061: expect (ObjNum 1004 snap 342 seq_num 1004) 2023-10-09T09:18:22.544 INFO:tasks.rados.rados.0.smithi163.stdout:3062: read oid 8 snap -1 2023-10-09T09:18:22.544 INFO:tasks.rados.rados.0.smithi163.stdout:3062: expect deleted 2023-10-09T09:18:22.544 INFO:tasks.rados.rados.0.smithi163.stdout:3063: write oid 6 current snap is 388 2023-10-09T09:18:22.544 INFO:tasks.rados.rados.0.smithi163.stdout:3060: finishing write tid 1 to smithi16367185-14 2023-10-09T09:18:22.545 INFO:tasks.rados.rados.0.smithi163.stdout:3063: seq_num 1151 ranges {544857=580902,1670539=668389,3041747=303110} 2023-10-09T09:18:22.550 INFO:tasks.rados.rados.0.smithi163.stdout:3063: writing smithi16367185-6 from 544857 to 1125759 tid 1 2023-10-09T09:18:22.559 INFO:tasks.rados.rados.0.smithi163.stdout:3063: writing smithi16367185-6 from 1670539 to 2338928 tid 2 2023-10-09T09:18:22.565 INFO:tasks.rados.rados.0.smithi163.stdout:3063: writing smithi16367185-6 from 3041747 to 3344857 tid 3 2023-10-09T09:18:22.565 INFO:tasks.rados.rados.0.smithi163.stdout:3064: write oid 49 current snap is 388 2023-10-09T09:18:22.565 INFO:tasks.rados.rados.0.smithi163.stdout:3064: seq_num 1152 ranges {561664=656151,1729021=528400,2669170=692494} 2023-10-09T09:18:22.572 INFO:tasks.rados.rados.0.smithi163.stdout:3064: writing smithi16367185-49 from 561664 to 1217815 tid 1 2023-10-09T09:18:22.578 INFO:tasks.rados.rados.0.smithi163.stdout:3064: writing smithi16367185-49 from 1729021 to 2257421 tid 2 2023-10-09T09:18:22.584 INFO:tasks.rados.rados.0.smithi163.stdout:3064: writing smithi16367185-49 from 2669170 to 3361664 tid 3 2023-10-09T09:18:22.584 INFO:tasks.rados.rados.0.smithi163.stdout:3065: snap_remove snap 384 2023-10-09T09:18:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:22 smithi181 bash[59345]: cluster 2023-10-09T09:18:21.218983+0000 osd.5 (osd.5) 360 : cluster 3 failed to encode map e827 with expected crc 2023-10-09T09:18:22.993 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:22 smithi181 bash[59345]: cluster 2023-10-09T09:18:21.412163+0000 mgr.y (mgr.34107) 354 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.0 MiB/s wr, 10 op/s 2023-10-09T09:18:22.994 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:22 smithi163 bash[78107]: cluster 2023-10-09T09:18:21.218983+0000 osd.5 (osd.5) 360 : cluster 3 failed to encode map e827 with expected crc 2023-10-09T09:18:22.994 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:22 smithi163 bash[78107]: cluster 2023-10-09T09:18:21.412163+0000 mgr.y (mgr.34107) 354 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.0 MiB/s wr, 10 op/s 2023-10-09T09:18:22.999 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:22 smithi163 bash[75425]: cluster 2023-10-09T09:18:21.218983+0000 osd.5 (osd.5) 360 : cluster 3 failed to encode map e827 with expected crc 2023-10-09T09:18:22.999 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:22 smithi163 bash[75425]: cluster 2023-10-09T09:18:21.412163+0000 mgr.y (mgr.34107) 354 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.0 MiB/s wr, 10 op/s 2023-10-09T09:18:23.520 INFO:tasks.rados.rados.0.smithi163.stdout:3065: done (5 left) 2023-10-09T09:18:23.520 INFO:tasks.rados.rados.0.smithi163.stdout:3066: snap_create 2023-10-09T09:18:23.520 INFO:tasks.rados.rados.0.smithi163.stdout:3060: finishing write tid 2 to smithi16367185-14 2023-10-09T09:18:23.520 INFO:tasks.rados.rados.0.smithi163.stdout:3060: finishing write tid 3 to smithi16367185-14 2023-10-09T09:18:23.521 INFO:tasks.rados.rados.0.smithi163.stdout:3060: finishing write tid 4 to smithi16367185-14 2023-10-09T09:18:23.521 INFO:tasks.rados.rados.0.smithi163.stdout:3061: expect (ObjNum 1004 snap 342 seq_num 1004) 2023-10-09T09:18:23.669 INFO:tasks.rados.rados.0.smithi163.stdout:3060: finishing write tid 5 to smithi16367185-14 2023-10-09T09:18:23.669 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1445 (ObjNum 1150 snap 388 seq_num 1150) dirty exists 2023-10-09T09:18:23.669 INFO:tasks.rados.rados.0.smithi163.stdout:3060: left oid 14 (ObjNum 1150 snap 388 seq_num 1150) 2023-10-09T09:18:23.669 INFO:tasks.rados.rados.0.smithi163.stdout:3063: finishing write tid 1 to smithi16367185-6 2023-10-09T09:18:23.669 INFO:tasks.rados.rados.0.smithi163.stdout:3063: finishing write tid 2 to smithi16367185-6 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:3063: finishing write tid 3 to smithi16367185-6 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:3063: finishing write tid 4 to smithi16367185-6 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:3063: finishing write tid 5 to smithi16367185-6 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 315 (ObjNum 1151 snap 388 seq_num 1151) dirty exists 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:3063: left oid 6 (ObjNum 1151 snap 388 seq_num 1151) 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:3064: finishing write tid 1 to smithi16367185-49 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:3064: finishing write tid 2 to smithi16367185-49 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:3064: finishing write tid 3 to smithi16367185-49 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:3064: finishing write tid 4 to smithi16367185-49 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:3064: finishing write tid 5 to smithi16367185-49 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 366 (ObjNum 1152 snap 388 seq_num 1152) dirty exists 2023-10-09T09:18:23.670 INFO:tasks.rados.rados.0.smithi163.stdout:3064: left oid 49 (ObjNum 1152 snap 388 seq_num 1152) 2023-10-09T09:18:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:23 smithi163 bash[75425]: cluster 2023-10-09T09:18:22.504128+0000 osd.5 (osd.5) 361 : cluster 3 failed to encode map e828 with expected crc 2023-10-09T09:18:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:23 smithi163 bash[75425]: cluster 2023-10-09T09:18:22.507730+0000 mon.a (mon.0) 1022 : cluster 0 osdmap e828: 8 total, 8 up, 8 in 2023-10-09T09:18:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:23 smithi163 bash[75425]: cluster 2023-10-09T09:18:22.509741+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e828 with expected crc 2023-10-09T09:18:23.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:18:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:18:23] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.33.4" 2023-10-09T09:18:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:23 smithi163 bash[78107]: cluster 2023-10-09T09:18:22.504128+0000 osd.5 (osd.5) 361 : cluster 3 failed to encode map e828 with expected crc 2023-10-09T09:18:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:23 smithi163 bash[78107]: cluster 2023-10-09T09:18:22.507730+0000 mon.a (mon.0) 1022 : cluster 0 osdmap e828: 8 total, 8 up, 8 in 2023-10-09T09:18:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:23 smithi163 bash[78107]: cluster 2023-10-09T09:18:22.509741+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e828 with expected crc 2023-10-09T09:18:23.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:23 smithi181 bash[59345]: cluster 2023-10-09T09:18:22.504128+0000 osd.5 (osd.5) 361 : cluster 3 failed to encode map e828 with expected crc 2023-10-09T09:18:23.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:23 smithi181 bash[59345]: cluster 2023-10-09T09:18:22.507730+0000 mon.a (mon.0) 1022 : cluster 0 osdmap e828: 8 total, 8 up, 8 in 2023-10-09T09:18:23.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:23 smithi181 bash[59345]: cluster 2023-10-09T09:18:22.509741+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e828 with expected crc 2023-10-09T09:18:24.519 INFO:tasks.rados.rados.0.smithi163.stdout:3060: done (5 left) 2023-10-09T09:18:24.519 INFO:tasks.rados.rados.0.smithi163.stdout:3061: done (4 left) 2023-10-09T09:18:24.519 INFO:tasks.rados.rados.0.smithi163.stdout:3062: done (3 left) 2023-10-09T09:18:24.520 INFO:tasks.rados.rados.0.smithi163.stdout:3063: done (2 left) 2023-10-09T09:18:24.520 INFO:tasks.rados.rados.0.smithi163.stdout:3064: done (1 left) 2023-10-09T09:18:24.520 INFO:tasks.rados.rados.0.smithi163.stdout:3066: done (0 left) 2023-10-09T09:18:24.520 INFO:tasks.rados.rados.0.smithi163.stdout:3067: write oid 7 current snap is 389 2023-10-09T09:18:24.520 INFO:tasks.rados.rados.0.smithi163.stdout:3067: seq_num 1153 ranges {578471=731400,1787503=672058,3096593=281878} 2023-10-09T09:18:24.530 INFO:tasks.rados.rados.0.smithi163.stdout:3067: writing smithi16367185-7 from 578471 to 1309871 tid 1 2023-10-09T09:18:24.539 INFO:tasks.rados.rados.0.smithi163.stdout:3067: writing smithi16367185-7 from 1787503 to 2459561 tid 2 2023-10-09T09:18:24.545 INFO:tasks.rados.rados.0.smithi163.stdout:3067: writing smithi16367185-7 from 3096593 to 3378471 tid 3 2023-10-09T09:18:24.545 INFO:tasks.rados.rados.0.smithi163.stdout:3067: finishing write tid 1 to smithi16367185-7 2023-10-09T09:18:24.545 INFO:tasks.rados.rados.0.smithi163.stdout:3068: read oid 21 snap -1 2023-10-09T09:18:24.545 INFO:tasks.rados.rados.0.smithi163.stdout:3068: expect (ObjNum 936 snap 320 seq_num 936) 2023-10-09T09:18:24.545 INFO:tasks.rados.rados.0.smithi163.stdout:3069: rollback oid 17 current snap is 389 2023-10-09T09:18:24.545 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 368 2023-10-09T09:18:24.545 INFO:tasks.rados.rados.0.smithi163.stdout:3067: finishing write tid 2 to smithi16367185-7 2023-10-09T09:18:24.545 INFO:tasks.rados.rados.0.smithi163.stdout:3070: read oid 44 snap -1 2023-10-09T09:18:24.545 INFO:tasks.rados.rados.0.smithi163.stdout:3070: expect (ObjNum 1122 snap 371 seq_num 1122) 2023-10-09T09:18:24.545 INFO:tasks.rados.rados.0.smithi163.stdout:3071: snap_remove snap 379 2023-10-09T09:18:24.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:24 smithi181 bash[59345]: cluster 2023-10-09T09:18:23.412835+0000 mgr.y (mgr.34107) 355 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.0 MiB/s wr, 9 op/s 2023-10-09T09:18:24.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:24 smithi181 bash[59345]: cluster 2023-10-09T09:18:23.511993+0000 osd.5 (osd.5) 362 : cluster 3 failed to encode map e829 with expected crc 2023-10-09T09:18:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:24 smithi181 bash[59345]: cluster 2023-10-09T09:18:23.519156+0000 mon.a (mon.0) 1023 : cluster 0 osdmap e829: 8 total, 8 up, 8 in 2023-10-09T09:18:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:24 smithi181 bash[59345]: cluster 2023-10-09T09:18:23.520946+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e829 with expected crc 2023-10-09T09:18:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:24 smithi163 bash[75425]: cluster 2023-10-09T09:18:23.412835+0000 mgr.y (mgr.34107) 355 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.0 MiB/s wr, 9 op/s 2023-10-09T09:18:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:24 smithi163 bash[75425]: cluster 2023-10-09T09:18:23.511993+0000 osd.5 (osd.5) 362 : cluster 3 failed to encode map e829 with expected crc 2023-10-09T09:18:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:24 smithi163 bash[75425]: cluster 2023-10-09T09:18:23.519156+0000 mon.a (mon.0) 1023 : cluster 0 osdmap e829: 8 total, 8 up, 8 in 2023-10-09T09:18:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:24 smithi163 bash[75425]: cluster 2023-10-09T09:18:23.520946+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e829 with expected crc 2023-10-09T09:18:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:24 smithi163 bash[78107]: cluster 2023-10-09T09:18:23.412835+0000 mgr.y (mgr.34107) 355 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.0 MiB/s wr, 9 op/s 2023-10-09T09:18:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:24 smithi163 bash[78107]: cluster 2023-10-09T09:18:23.511993+0000 osd.5 (osd.5) 362 : cluster 3 failed to encode map e829 with expected crc 2023-10-09T09:18:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:24 smithi163 bash[78107]: cluster 2023-10-09T09:18:23.519156+0000 mon.a (mon.0) 1023 : cluster 0 osdmap e829: 8 total, 8 up, 8 in 2023-10-09T09:18:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:24 smithi163 bash[78107]: cluster 2023-10-09T09:18:23.520946+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e829 with expected crc 2023-10-09T09:18:25.533 INFO:tasks.rados.rados.0.smithi163.stdout:3071: done (4 left) 2023-10-09T09:18:25.533 INFO:tasks.rados.rados.0.smithi163.stdout:3072: read oid 9 snap -1 2023-10-09T09:18:25.533 INFO:tasks.rados.rados.0.smithi163.stdout:3072: expect (ObjNum 1133 snap 376 seq_num 1133) 2023-10-09T09:18:25.533 INFO:tasks.rados.rados.0.smithi163.stdout:3073: rollback oid 6 current snap is 389 2023-10-09T09:18:25.533 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 6 to 386 2023-10-09T09:18:25.533 INFO:tasks.rados.rados.0.smithi163.stdout:3068: expect (ObjNum 936 snap 320 seq_num 936) 2023-10-09T09:18:25.705 INFO:tasks.rados.rados.0.smithi163.stdout:3067: finishing write tid 3 to smithi16367185-7 2023-10-09T09:18:25.705 INFO:tasks.rados.rados.0.smithi163.stdout:3067: finishing write tid 4 to smithi16367185-7 2023-10-09T09:18:25.705 INFO:tasks.rados.rados.0.smithi163.stdout:3067: finishing write tid 5 to smithi16367185-7 2023-10-09T09:18:25.705 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 742 (ObjNum 1153 snap 389 seq_num 1153) dirty exists 2023-10-09T09:18:25.705 INFO:tasks.rados.rados.0.smithi163.stdout:3067: left oid 7 (ObjNum 1153 snap 389 seq_num 1153) 2023-10-09T09:18:25.705 INFO:tasks.rados.rados.0.smithi163.stdout:3069: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:18:25.705 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1391 (ObjNum 699348816 snap 21875 seq_num 0) dirty dne 2023-10-09T09:18:25.705 INFO:tasks.rados.rados.0.smithi163.stdout:3070: expect (ObjNum 1122 snap 371 seq_num 1122) 2023-10-09T09:18:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:25 smithi181 bash[59345]: cluster 2023-10-09T09:18:24.513895+0000 osd.5 (osd.5) 363 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:25 smithi181 bash[59345]: cluster 2023-10-09T09:18:24.518427+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e830: 8 total, 8 up, 8 in 2023-10-09T09:18:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:25 smithi181 bash[59345]: cluster 2023-10-09T09:18:24.532898+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:25 smithi181 bash[59345]: cluster 2023-10-09T09:18:24.545405+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:25 smithi181 bash[59345]: audit 2023-10-09T09:18:25.348069+0000 mon.a (mon.0) 1025 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:25.866 INFO:tasks.rados.rados.0.smithi163.stdout:3072: expect (ObjNum 1133 snap 376 seq_num 1133) 2023-10-09T09:18:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:25 smithi163 bash[78107]: cluster 2023-10-09T09:18:24.513895+0000 osd.5 (osd.5) 363 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:25 smithi163 bash[78107]: cluster 2023-10-09T09:18:24.518427+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e830: 8 total, 8 up, 8 in 2023-10-09T09:18:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:25 smithi163 bash[78107]: cluster 2023-10-09T09:18:24.532898+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:25 smithi163 bash[78107]: cluster 2023-10-09T09:18:24.545405+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:25 smithi163 bash[78107]: audit 2023-10-09T09:18:25.348069+0000 mon.a (mon.0) 1025 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:25 smithi163 bash[75425]: cluster 2023-10-09T09:18:24.513895+0000 osd.5 (osd.5) 363 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:25 smithi163 bash[75425]: cluster 2023-10-09T09:18:24.518427+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e830: 8 total, 8 up, 8 in 2023-10-09T09:18:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:25 smithi163 bash[75425]: cluster 2023-10-09T09:18:24.532898+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:25 smithi163 bash[75425]: cluster 2023-10-09T09:18:24.545405+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:25 smithi163 bash[75425]: audit 2023-10-09T09:18:25.348069+0000 mon.a (mon.0) 1025 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:26.008 INFO:tasks.rados.rados.0.smithi163.stdout:3073: finishing rollback tid 0 to smithi16367185-6 2023-10-09T09:18:26.008 INFO:tasks.rados.rados.0.smithi163.stdout:3073: finishing rollback tid 1 to smithi16367185-6 2023-10-09T09:18:26.008 INFO:tasks.rados.rados.0.smithi163.stdout:3073: finishing rollback tid 2 to smithi16367185-6 2023-10-09T09:18:26.008 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 321 (ObjNum 956 snap 324 seq_num 956) dirty exists 2023-10-09T09:18:26.008 INFO:tasks.rados.rados.0.smithi163.stdout:3067: done (5 left) 2023-10-09T09:18:26.008 INFO:tasks.rados.rados.0.smithi163.stdout:3068: done (4 left) 2023-10-09T09:18:26.008 INFO:tasks.rados.rados.0.smithi163.stdout:3069: done (3 left) 2023-10-09T09:18:26.009 INFO:tasks.rados.rados.0.smithi163.stdout:3070: done (2 left) 2023-10-09T09:18:26.009 INFO:tasks.rados.rados.0.smithi163.stdout:3072: done (1 left) 2023-10-09T09:18:26.009 INFO:tasks.rados.rados.0.smithi163.stdout:3073: done (0 left) 2023-10-09T09:18:26.009 INFO:tasks.rados.rados.0.smithi163.stdout:3074: snap_create 2023-10-09T09:18:26.547 INFO:tasks.rados.rados.0.smithi163.stdout:3074: done (0 left) 2023-10-09T09:18:26.547 INFO:tasks.rados.rados.0.smithi163.stdout:3075: delete oid 11 current snap is 390 2023-10-09T09:18:26.558 INFO:tasks.rados.rados.0.smithi163.stdout:3075: done (0 left) 2023-10-09T09:18:26.558 INFO:tasks.rados.rados.0.smithi163.stdout:3076: snap_remove snap 368 2023-10-09T09:18:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:26 smithi181 bash[59345]: cluster 2023-10-09T09:18:25.064627+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:26 smithi181 bash[59345]: cluster 2023-10-09T09:18:25.413600+0000 mgr.y (mgr.34107) 356 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 767 KiB/s wr, 12 op/s 2023-10-09T09:18:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:26 smithi181 bash[59345]: cluster 2023-10-09T09:18:25.529711+0000 osd.5 (osd.5) 364 : cluster 3 failed to encode map e831 with expected crc 2023-10-09T09:18:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:26 smithi181 bash[59345]: cluster 2023-10-09T09:18:25.532872+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e831: 8 total, 8 up, 8 in 2023-10-09T09:18:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:26 smithi181 bash[59345]: cluster 2023-10-09T09:18:25.535312+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e831 with expected crc 2023-10-09T09:18:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:26 smithi181 bash[59345]: cluster 2023-10-09T09:18:25.538015+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e831 with expected crc 2023-10-09T09:18:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[75425]: cluster 2023-10-09T09:18:25.064627+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[75425]: cluster 2023-10-09T09:18:25.413600+0000 mgr.y (mgr.34107) 356 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 767 KiB/s wr, 12 op/s 2023-10-09T09:18:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[75425]: cluster 2023-10-09T09:18:25.529711+0000 osd.5 (osd.5) 364 : cluster 3 failed to encode map e831 with expected crc 2023-10-09T09:18:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[75425]: cluster 2023-10-09T09:18:25.532872+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e831: 8 total, 8 up, 8 in 2023-10-09T09:18:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[75425]: cluster 2023-10-09T09:18:25.535312+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e831 with expected crc 2023-10-09T09:18:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[75425]: cluster 2023-10-09T09:18:25.538015+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e831 with expected crc 2023-10-09T09:18:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[78107]: cluster 2023-10-09T09:18:25.064627+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e830 with expected crc 2023-10-09T09:18:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[78107]: cluster 2023-10-09T09:18:25.413600+0000 mgr.y (mgr.34107) 356 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 767 KiB/s wr, 12 op/s 2023-10-09T09:18:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[78107]: cluster 2023-10-09T09:18:25.529711+0000 osd.5 (osd.5) 364 : cluster 3 failed to encode map e831 with expected crc 2023-10-09T09:18:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[78107]: cluster 2023-10-09T09:18:25.532872+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e831: 8 total, 8 up, 8 in 2023-10-09T09:18:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[78107]: cluster 2023-10-09T09:18:25.535312+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e831 with expected crc 2023-10-09T09:18:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:26 smithi163 bash[78107]: cluster 2023-10-09T09:18:25.538015+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e831 with expected crc 2023-10-09T09:18:27.562 INFO:tasks.rados.rados.0.smithi163.stdout:3076: done (0 left) 2023-10-09T09:18:27.563 INFO:tasks.rados.rados.0.smithi163.stdout:3077: rollback oid 30 current snap is 390 2023-10-09T09:18:27.563 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 353 2023-10-09T09:18:27.563 INFO:tasks.rados.rados.0.smithi163.stdout:3078: delete oid 35 current snap is 390 2023-10-09T09:18:27.578 INFO:tasks.rados.rados.0.smithi163.stdout:3077: finishing rollback tid 0 to smithi16367185-30 2023-10-09T09:18:27.578 INFO:tasks.rados.rados.0.smithi163.stdout:3077: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:18:27.578 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 969 (ObjNum 699340688 snap 21875 seq_num 1618308320) dirty dne 2023-10-09T09:18:27.580 INFO:tasks.rados.rados.0.smithi163.stdout:3077: done (1 left) 2023-10-09T09:18:27.580 INFO:tasks.rados.rados.0.smithi163.stdout:3078: done (0 left) 2023-10-09T09:18:27.580 INFO:tasks.rados.rados.0.smithi163.stdout:3079: snap_create 2023-10-09T09:18:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:27 smithi181 bash[59345]: cluster 2023-10-09T09:18:26.543382+0000 osd.5 (osd.5) 365 : cluster 3 failed to encode map e832 with expected crc 2023-10-09T09:18:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:27 smithi181 bash[59345]: cluster 2023-10-09T09:18:26.545913+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e832 with expected crc 2023-10-09T09:18:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:27 smithi181 bash[59345]: cluster 2023-10-09T09:18:26.547271+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e832: 8 total, 8 up, 8 in 2023-10-09T09:18:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:27 smithi181 bash[59345]: cluster 2023-10-09T09:18:26.549440+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e832 with expected crc 2023-10-09T09:18:27.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:27 smithi163 bash[75425]: cluster 2023-10-09T09:18:26.543382+0000 osd.5 (osd.5) 365 : cluster 3 failed to encode map e832 with expected crc 2023-10-09T09:18:27.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:27 smithi163 bash[75425]: cluster 2023-10-09T09:18:26.545913+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e832 with expected crc 2023-10-09T09:18:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:27 smithi163 bash[75425]: cluster 2023-10-09T09:18:26.547271+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e832: 8 total, 8 up, 8 in 2023-10-09T09:18:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:27 smithi163 bash[75425]: cluster 2023-10-09T09:18:26.549440+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e832 with expected crc 2023-10-09T09:18:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:27 smithi163 bash[78107]: cluster 2023-10-09T09:18:26.543382+0000 osd.5 (osd.5) 365 : cluster 3 failed to encode map e832 with expected crc 2023-10-09T09:18:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:27 smithi163 bash[78107]: cluster 2023-10-09T09:18:26.545913+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e832 with expected crc 2023-10-09T09:18:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:27 smithi163 bash[78107]: cluster 2023-10-09T09:18:26.547271+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e832: 8 total, 8 up, 8 in 2023-10-09T09:18:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:27 smithi163 bash[78107]: cluster 2023-10-09T09:18:26.549440+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e832 with expected crc 2023-10-09T09:18:28.569 INFO:tasks.rados.rados.0.smithi163.stdout:3079: done (0 left) 2023-10-09T09:18:28.569 INFO:tasks.rados.rados.0.smithi163.stdout:3080: read oid 34 snap -1 2023-10-09T09:18:28.570 INFO:tasks.rados.rados.0.smithi163.stdout:3080: expect deleted 2023-10-09T09:18:28.570 INFO:tasks.rados.rados.0.smithi163.stdout:3081: write oid 10 current snap is 391 2023-10-09T09:18:28.570 INFO:tasks.rados.rados.0.smithi163.stdout:3081: seq_num 1156 ranges {628892=673500,1962949=535738,3178862=250030} 2023-10-09T09:18:28.580 INFO:tasks.rados.rados.0.smithi163.stdout:3081: writing smithi16367185-10 from 628892 to 1302392 tid 1 2023-10-09T09:18:28.588 INFO:tasks.rados.rados.0.smithi163.stdout:3081: writing smithi16367185-10 from 1962949 to 2498687 tid 2 2023-10-09T09:18:28.593 INFO:tasks.rados.rados.0.smithi163.stdout:3081: writing smithi16367185-10 from 3178862 to 3428892 tid 3 2023-10-09T09:18:28.594 INFO:tasks.rados.rados.0.smithi163.stdout:3082: rollback oid 26 current snap is 391 2023-10-09T09:18:28.594 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 26 to 362 2023-10-09T09:18:28.594 INFO:tasks.rados.rados.0.smithi163.stdout:3081: finishing write tid 1 to smithi16367185-10 2023-10-09T09:18:28.594 INFO:tasks.rados.rados.0.smithi163.stdout:3081: finishing write tid 2 to smithi16367185-10 2023-10-09T09:18:28.594 INFO:tasks.rados.rados.0.smithi163.stdout:3080: done (2 left) 2023-10-09T09:18:28.594 INFO:tasks.rados.rados.0.smithi163.stdout:3083: read oid 50 snap -1 2023-10-09T09:18:28.594 INFO:tasks.rados.rados.0.smithi163.stdout:3083: expect (ObjNum 1086 snap 362 seq_num 1086) 2023-10-09T09:18:28.594 INFO:tasks.rados.rados.0.smithi163.stdout:3084: write oid 25 current snap is 391 2023-10-09T09:18:28.594 INFO:tasks.rados.rados.0.smithi163.stdout:3084: seq_num 1157 ranges {645699=748749,2021431=679396,3206285=239414} 2023-10-09T09:18:28.601 INFO:tasks.rados.rados.0.smithi163.stdout:3084: writing smithi16367185-25 from 645699 to 1394448 tid 1 2023-10-09T09:18:28.609 INFO:tasks.rados.rados.0.smithi163.stdout:3084: writing smithi16367185-25 from 2021431 to 2700827 tid 2 2023-10-09T09:18:28.614 INFO:tasks.rados.rados.0.smithi163.stdout:3084: writing smithi16367185-25 from 3206285 to 3445699 tid 3 2023-10-09T09:18:28.614 INFO:tasks.rados.rados.0.smithi163.stdout:3085: read oid 43 snap -1 2023-10-09T09:18:28.614 INFO:tasks.rados.rados.0.smithi163.stdout:3085: expect (ObjNum 1091 snap 362 seq_num 1091) 2023-10-09T09:18:28.615 INFO:tasks.rados.rados.0.smithi163.stdout:3086: write oid 9 current snap is 391 2023-10-09T09:18:28.615 INFO:tasks.rados.rados.0.smithi163.stdout:3086: seq_num 1158 ranges {662506=423998,1563560=539407,2717355=490285} 2023-10-09T09:18:28.620 INFO:tasks.rados.rados.0.smithi163.stdout:3086: writing smithi16367185-9 from 662506 to 1086504 tid 1 2023-10-09T09:18:28.628 INFO:tasks.rados.rados.0.smithi163.stdout:3086: writing smithi16367185-9 from 1563560 to 2102967 tid 2 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:3086: writing smithi16367185-9 from 2717355 to 3207640 tid 3 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:3087: read oid 17 snap 354 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:3087: expect deleted 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:3088: rollback oid 49 current snap is 391 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 49 to 358 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:3081: finishing write tid 3 to smithi16367185-10 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:3089: read oid 7 snap 378 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:3089: expect (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:3090: write oid 18 current snap is 391 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:3081: finishing write tid 4 to smithi16367185-10 2023-10-09T09:18:28.635 INFO:tasks.rados.rados.0.smithi163.stdout:3081: finishing write tid 5 to smithi16367185-10 2023-10-09T09:18:28.636 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 978 (ObjNum 1156 snap 391 seq_num 1156) dirty exists 2023-10-09T09:18:28.636 INFO:tasks.rados.rados.0.smithi163.stdout:3081: left oid 10 (ObjNum 1156 snap 391 seq_num 1156) 2023-10-09T09:18:28.636 INFO:tasks.rados.rados.0.smithi163.stdout:3083: expect (ObjNum 1086 snap 362 seq_num 1086) 2023-10-09T09:18:28.711 INFO:tasks.rados.rados.0.smithi163.stdout:3082: finishing rollback tid 0 to smithi16367185-26 2023-10-09T09:18:28.711 INFO:tasks.rados.rados.0.smithi163.stdout:3082: finishing rollback tid 1 to smithi16367185-26 2023-10-09T09:18:28.711 INFO:tasks.rados.rados.0.smithi163.stdout:3082: finishing rollback tid 2 to smithi16367185-26 2023-10-09T09:18:28.712 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 898 (ObjNum 1077 snap 361 seq_num 1077) dirty exists 2023-10-09T09:18:28.712 INFO:tasks.rados.rados.0.smithi163.stdout:3084: finishing write tid 1 to smithi16367185-25 2023-10-09T09:18:28.712 INFO:tasks.rados.rados.0.smithi163.stdout:3090: seq_num 1159 ranges {679313=499247,1622042=683065,2744778=701557} 2023-10-09T09:18:28.718 INFO:tasks.rados.rados.0.smithi163.stdout:3090: writing smithi16367185-18 from 679313 to 1178560 tid 1 2023-10-09T09:18:28.725 INFO:tasks.rados.rados.0.smithi163.stdout:3090: writing smithi16367185-18 from 1622042 to 2305107 tid 2 2023-10-09T09:18:28.732 INFO:tasks.rados.rados.0.smithi163.stdout:3090: writing smithi16367185-18 from 2744778 to 3446335 tid 3 2023-10-09T09:18:28.732 INFO:tasks.rados.rados.0.smithi163.stdout:3081: done (9 left) 2023-10-09T09:18:28.732 INFO:tasks.rados.rados.0.smithi163.stdout:3082: done (8 left) 2023-10-09T09:18:28.732 INFO:tasks.rados.rados.0.smithi163.stdout:3083: done (7 left) 2023-10-09T09:18:28.732 INFO:tasks.rados.rados.0.smithi163.stdout:3091: rollback oid 4 current snap is 391 2023-10-09T09:18:28.732 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 358 2023-10-09T09:18:28.732 INFO:tasks.rados.rados.0.smithi163.stdout:3084: finishing write tid 2 to smithi16367185-25 2023-10-09T09:18:28.732 INFO:tasks.rados.rados.0.smithi163.stdout:3084: finishing write tid 3 to smithi16367185-25 2023-10-09T09:18:28.732 INFO:tasks.rados.rados.0.smithi163.stdout:3084: finishing write tid 4 to smithi16367185-25 2023-10-09T09:18:28.732 INFO:tasks.rados.rados.0.smithi163.stdout:3084: finishing write tid 5 to smithi16367185-25 2023-10-09T09:18:28.733 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1428 (ObjNum 1157 snap 391 seq_num 1157) dirty exists 2023-10-09T09:18:28.733 INFO:tasks.rados.rados.0.smithi163.stdout:3084: left oid 25 (ObjNum 1157 snap 391 seq_num 1157) 2023-10-09T09:18:28.733 INFO:tasks.rados.rados.0.smithi163.stdout:3084: done (7 left) 2023-10-09T09:18:28.733 INFO:tasks.rados.rados.0.smithi163.stdout:3092: snap_remove snap 366 2023-10-09T09:18:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:28 smithi181 bash[59345]: cluster 2023-10-09T09:18:27.414530+0000 mgr.y (mgr.34107) 357 : cluster 0 pgmap v802: 89 pgs: 89 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2023-10-09T09:18:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:28 smithi181 bash[59345]: cluster 2023-10-09T09:18:27.553452+0000 osd.5 (osd.5) 366 : cluster 3 failed to encode map e833 with expected crc 2023-10-09T09:18:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:28 smithi181 bash[59345]: cluster 2023-10-09T09:18:27.560606+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e833: 8 total, 8 up, 8 in 2023-10-09T09:18:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:28 smithi181 bash[59345]: cluster 2023-10-09T09:18:27.560889+0000 osd.4 (osd.4) 343 : cluster 3 failed to encode map e833 with expected crc 2023-10-09T09:18:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:28 smithi181 bash[59345]: cluster 2023-10-09T09:18:27.563789+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e833 with expected crc 2023-10-09T09:18:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:28 smithi163 bash[75425]: cluster 2023-10-09T09:18:27.414530+0000 mgr.y (mgr.34107) 357 : cluster 0 pgmap v802: 89 pgs: 89 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2023-10-09T09:18:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:28 smithi163 bash[75425]: cluster 2023-10-09T09:18:27.553452+0000 osd.5 (osd.5) 366 : cluster 3 failed to encode map e833 with expected crc 2023-10-09T09:18:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:28 smithi163 bash[75425]: cluster 2023-10-09T09:18:27.560606+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e833: 8 total, 8 up, 8 in 2023-10-09T09:18:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:28 smithi163 bash[75425]: cluster 2023-10-09T09:18:27.560889+0000 osd.4 (osd.4) 343 : cluster 3 failed to encode map e833 with expected crc 2023-10-09T09:18:28.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:28 smithi163 bash[75425]: cluster 2023-10-09T09:18:27.563789+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e833 with expected crc 2023-10-09T09:18:28.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:28 smithi163 bash[78107]: cluster 2023-10-09T09:18:27.414530+0000 mgr.y (mgr.34107) 357 : cluster 0 pgmap v802: 89 pgs: 89 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2023-10-09T09:18:28.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:28 smithi163 bash[78107]: cluster 2023-10-09T09:18:27.553452+0000 osd.5 (osd.5) 366 : cluster 3 failed to encode map e833 with expected crc 2023-10-09T09:18:28.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:28 smithi163 bash[78107]: cluster 2023-10-09T09:18:27.560606+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e833: 8 total, 8 up, 8 in 2023-10-09T09:18:28.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:28 smithi163 bash[78107]: cluster 2023-10-09T09:18:27.560889+0000 osd.4 (osd.4) 343 : cluster 3 failed to encode map e833 with expected crc 2023-10-09T09:18:28.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:28 smithi163 bash[78107]: cluster 2023-10-09T09:18:27.563789+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e833 with expected crc 2023-10-09T09:18:29.587 INFO:tasks.rados.rados.0.smithi163.stdout:3092: done (7 left) 2023-10-09T09:18:29.587 INFO:tasks.rados.rados.0.smithi163.stdout:3093: write oid 45 current snap is 391 2023-10-09T09:18:29.587 INFO:tasks.rados.rados.0.smithi163.stdout:3093: seq_num 1160 ranges {696120=574496,1680524=543076,2888554=512829} 2023-10-09T09:18:29.596 INFO:tasks.rados.rados.0.smithi163.stdout:3093: writing smithi16367185-45 from 696120 to 1270616 tid 1 2023-10-09T09:18:29.602 INFO:tasks.rados.rados.0.smithi163.stdout:3093: writing smithi16367185-45 from 1680524 to 2223600 tid 2 2023-10-09T09:18:29.610 INFO:tasks.rados.rados.0.smithi163.stdout:3093: writing smithi16367185-45 from 2888554 to 3401383 tid 3 2023-10-09T09:18:29.610 INFO:tasks.rados.rados.0.smithi163.stdout:3094: write oid 48 current snap is 391 2023-10-09T09:18:29.610 INFO:tasks.rados.rados.0.smithi163.stdout:3085: expect (ObjNum 1091 snap 362 seq_num 1091) 2023-10-09T09:18:29.696 INFO:tasks.rados.rados.0.smithi163.stdout:3086: finishing write tid 1 to smithi16367185-9 2023-10-09T09:18:29.696 INFO:tasks.rados.rados.0.smithi163.stdout:3086: finishing write tid 2 to smithi16367185-9 2023-10-09T09:18:29.696 INFO:tasks.rados.rados.0.smithi163.stdout:3094: seq_num 1161 ranges {712927=649745,2139006=686734} 2023-10-09T09:18:29.703 INFO:tasks.rados.rados.0.smithi163.stdout:3094: writing smithi16367185-48 from 712927 to 1362672 tid 1 2023-10-09T09:18:29.712 INFO:tasks.rados.rados.0.smithi163.stdout:3094: writing smithi16367185-48 from 2139006 to 2825740 tid 2 2023-10-09T09:18:29.712 INFO:tasks.rados.rados.0.smithi163.stdout:3085: done (8 left) 2023-10-09T09:18:29.713 INFO:tasks.rados.rados.0.smithi163.stdout:3095: read oid 29 snap -1 2023-10-09T09:18:29.713 INFO:tasks.rados.rados.0.smithi163.stdout:3095: expect deleted 2023-10-09T09:18:29.713 INFO:tasks.rados.rados.0.smithi163.stdout:3096: rollback oid 40 current snap is 391 2023-10-09T09:18:29.713 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 40 to 390 2023-10-09T09:18:29.713 INFO:tasks.rados.rados.0.smithi163.stdout:3097: write oid 25 current snap is 391 2023-10-09T09:18:29.713 INFO:tasks.rados.rados.0.smithi163.stdout:3089: expect (ObjNum 961 snap 326 seq_num 961) 2023-10-09T09:18:29.894 INFO:tasks.rados.rados.0.smithi163.stdout:3088: finishing rollback tid 0 to smithi16367185-49 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:3088: finishing rollback tid 1 to smithi16367185-49 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:3088: finishing rollback tid 2 to smithi16367185-49 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 376 (ObjNum 793 snap 272 seq_num 793) dirty exists 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:3086: finishing write tid 3 to smithi16367185-9 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:3086: finishing write tid 4 to smithi16367185-9 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:3086: finishing write tid 5 to smithi16367185-9 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 677 (ObjNum 1158 snap 391 seq_num 1158) dirty exists 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:3086: left oid 9 (ObjNum 1158 snap 391 seq_num 1158) 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:3090: finishing write tid 1 to smithi16367185-18 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:3090: finishing write tid 2 to smithi16367185-18 2023-10-09T09:18:29.895 INFO:tasks.rados.rados.0.smithi163.stdout:3090: finishing write tid 3 to smithi16367185-18 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:3090: finishing write tid 4 to smithi16367185-18 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:3090: finishing write tid 5 to smithi16367185-18 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 983 (ObjNum 1159 snap 391 seq_num 1159) dirty exists 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:3090: left oid 18 (ObjNum 1159 snap 391 seq_num 1159) 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:3091: finishing rollback tid 0 to smithi16367185-4 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:3091: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1458 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:3093: finishing write tid 1 to smithi16367185-45 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:3093: finishing write tid 2 to smithi16367185-45 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:3093: finishing write tid 3 to smithi16367185-45 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:3093: finishing write tid 4 to smithi16367185-45 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:3093: finishing write tid 5 to smithi16367185-45 2023-10-09T09:18:29.896 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 2177 (ObjNum 1160 snap 391 seq_num 1160) dirty exists 2023-10-09T09:18:29.897 INFO:tasks.rados.rados.0.smithi163.stdout:3093: left oid 45 (ObjNum 1160 snap 391 seq_num 1160) 2023-10-09T09:18:29.897 INFO:tasks.rados.rados.0.smithi163.stdout:3094: finishing write tid 1 to smithi16367185-48 2023-10-09T09:18:29.897 INFO:tasks.rados.rados.0.smithi163.stdout:3096: finishing rollback tid 1 to smithi16367185-40 2023-10-09T09:18:29.897 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1516 (ObjNum 699648144 snap 21875 seq_num 699758032) dirty dne 2023-10-09T09:18:29.897 INFO:tasks.rados.rados.0.smithi163.stdout:3094: finishing write tid 2 to smithi16367185-48 2023-10-09T09:18:29.897 INFO:tasks.rados.rados.0.smithi163.stdout:3094: finishing write tid 3 to smithi16367185-48 2023-10-09T09:18:29.897 INFO:tasks.rados.rados.0.smithi163.stdout:3094: finishing write tid 4 to smithi16367185-48 2023-10-09T09:18:29.897 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 2185 (ObjNum 1161 snap 391 seq_num 1161) dirty exists 2023-10-09T09:18:29.897 INFO:tasks.rados.rados.0.smithi163.stdout:3094: left oid 48 (ObjNum 1161 snap 391 seq_num 1161) 2023-10-09T09:18:29.897 INFO:tasks.rados.rados.0.smithi163.stdout:3097: seq_num 1162 ranges {729734=724994,2197488=430392,3227047=302687} 2023-10-09T09:18:29.910 INFO:tasks.rados.rados.0.smithi163.stdout:3097: writing smithi16367185-25 from 729734 to 1454728 tid 1 2023-10-09T09:18:29.917 INFO:tasks.rados.rados.0.smithi163.stdout:3097: writing smithi16367185-25 from 2197488 to 2627880 tid 2 2023-10-09T09:18:29.922 INFO:tasks.rados.rados.0.smithi163.stdout:3097: writing smithi16367185-25 from 3227047 to 3529734 tid 3 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3086: done (10 left) 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3087: done (9 left) 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3088: done (8 left) 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3089: done (7 left) 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3090: done (6 left) 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3091: done (5 left) 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3093: done (4 left) 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3094: done (3 left) 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3095: done (2 left) 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3096: done (1 left) 2023-10-09T09:18:29.923 INFO:tasks.rados.rados.0.smithi163.stdout:3098: write oid 33 current snap is 391 2023-10-09T09:18:29.924 INFO:tasks.rados.rados.0.smithi163.stdout:3097: finishing write tid 1 to smithi16367185-25 2023-10-09T09:18:29.924 INFO:tasks.rados.rados.0.smithi163.stdout:3098: seq_num 1163 ranges {746541=400243,1739617=690403,3138117=408424} 2023-10-09T09:18:29.929 INFO:tasks.rados.rados.0.smithi163.stdout:3098: writing smithi16367185-33 from 746541 to 1146784 tid 1 2023-10-09T09:18:29.936 INFO:tasks.rados.rados.0.smithi163.stdout:3098: writing smithi16367185-33 from 1739617 to 2430020 tid 2 2023-10-09T09:18:29.943 INFO:tasks.rados.rados.0.smithi163.stdout:3098: writing smithi16367185-33 from 3138117 to 3546541 tid 3 2023-10-09T09:18:29.943 INFO:tasks.rados.rados.0.smithi163.stdout:3097: finishing write tid 2 to smithi16367185-25 2023-10-09T09:18:29.943 INFO:tasks.rados.rados.0.smithi163.stdout:3097: finishing write tid 3 to smithi16367185-25 2023-10-09T09:18:29.943 INFO:tasks.rados.rados.0.smithi163.stdout:3097: finishing write tid 4 to smithi16367185-25 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:3099: read oid 22 snap -1 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:3099: expect (ObjNum 1115 snap 368 seq_num 1115) 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:3100: rollback oid 16 current snap is 391 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 16 to 376 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:3097: finishing write tid 5 to smithi16367185-25 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1436 (ObjNum 1162 snap 391 seq_num 1162) dirty exists 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:3097: left oid 25 (ObjNum 1162 snap 391 seq_num 1162) 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:3097: done (3 left) 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:3101: rollback oid 28 current snap is 391 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 362 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:3098: finishing write tid 1 to smithi16367185-33 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:3098: finishing write tid 2 to smithi16367185-33 2023-10-09T09:18:29.944 INFO:tasks.rados.rados.0.smithi163.stdout:3102: snap_create 2023-10-09T09:18:29.947 INFO:tasks.rados.rados.0.smithi163.stdout:3098: finishing write tid 3 to smithi16367185-33 2023-10-09T09:18:29.947 INFO:tasks.rados.rados.0.smithi163.stdout:3098: finishing write tid 4 to smithi16367185-33 2023-10-09T09:18:29.948 INFO:tasks.rados.rados.0.smithi163.stdout:3098: finishing write tid 5 to smithi16367185-33 2023-10-09T09:18:29.948 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1441 (ObjNum 1163 snap 391 seq_num 1163) dirty exists 2023-10-09T09:18:29.948 INFO:tasks.rados.rados.0.smithi163.stdout:3098: left oid 33 (ObjNum 1163 snap 391 seq_num 1163) 2023-10-09T09:18:29.950 INFO:tasks.rados.rados.0.smithi163.stdout:3100: finishing rollback tid 1 to smithi16367185-16 2023-10-09T09:18:29.950 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1429 (ObjNum 762 snap 0 seq_num 760) dirty dne 2023-10-09T09:18:29.953 INFO:tasks.rados.rados.0.smithi163.stdout:3101: finishing rollback tid 0 to smithi16367185-28 2023-10-09T09:18:29.953 INFO:tasks.rados.rados.0.smithi163.stdout:3099: expect (ObjNum 1115 snap 368 seq_num 1115) 2023-10-09T09:18:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:29 smithi163 bash[78107]: cluster 2023-10-09T09:18:28.566720+0000 osd.5 (osd.5) 367 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:29 smithi163 bash[78107]: cluster 2023-10-09T09:18:28.569013+0000 mon.a (mon.0) 1029 : cluster 0 osdmap e834: 8 total, 8 up, 8 in 2023-10-09T09:18:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:29 smithi163 bash[78107]: cluster 2023-10-09T09:18:28.594709+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:29 smithi163 bash[78107]: cluster 2023-10-09T09:18:28.615423+0000 osd.4 (osd.4) 344 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:29 smithi163 bash[75425]: cluster 2023-10-09T09:18:28.566720+0000 osd.5 (osd.5) 367 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:29 smithi163 bash[75425]: cluster 2023-10-09T09:18:28.569013+0000 mon.a (mon.0) 1029 : cluster 0 osdmap e834: 8 total, 8 up, 8 in 2023-10-09T09:18:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:29 smithi163 bash[75425]: cluster 2023-10-09T09:18:28.594709+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:29 smithi163 bash[75425]: cluster 2023-10-09T09:18:28.615423+0000 osd.4 (osd.4) 344 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:29 smithi181 bash[59345]: cluster 2023-10-09T09:18:28.566720+0000 osd.5 (osd.5) 367 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:29 smithi181 bash[59345]: cluster 2023-10-09T09:18:28.569013+0000 mon.a (mon.0) 1029 : cluster 0 osdmap e834: 8 total, 8 up, 8 in 2023-10-09T09:18:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:29 smithi181 bash[59345]: cluster 2023-10-09T09:18:28.594709+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:29 smithi181 bash[59345]: cluster 2023-10-09T09:18:28.615423+0000 osd.4 (osd.4) 344 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:30.157 INFO:tasks.rados.rados.0.smithi163.stdout:3101: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:18:30.158 INFO:tasks.rados.rados.0.smithi163.stdout:3101: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:18:30.158 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 1467 (ObjNum 1005 snap 342 seq_num 1005) dirty exists 2023-10-09T09:18:30.238 INFO:tasks.rados.rados.0.smithi163.stdout:3098: done (4 left) 2023-10-09T09:18:30.238 INFO:tasks.rados.rados.0.smithi163.stdout:3099: done (3 left) 2023-10-09T09:18:30.238 INFO:tasks.rados.rados.0.smithi163.stdout:3100: done (2 left) 2023-10-09T09:18:30.238 INFO:tasks.rados.rados.0.smithi163.stdout:3101: done (1 left) 2023-10-09T09:18:30.238 INFO:tasks.rados.rados.0.smithi163.stdout:3102: done (0 left) 2023-10-09T09:18:30.238 INFO:tasks.rados.rados.0.smithi163.stdout:3103: snap_remove snap 354 2023-10-09T09:18:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:28.635867+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:29.415437+0000 mgr.y (mgr.34107) 358 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 298 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2023-10-09T09:18:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:29.577471+0000 osd.5 (osd.5) 368 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:29.584419+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e835: 8 total, 8 up, 8 in 2023-10-09T09:18:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:29.586534+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:29.586588+0000 osd.5 (osd.5) 369 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:29.590312+0000 osd.5 (osd.5) 370 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:29.590444+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:29.592181+0000 osd.7 (osd.7) 319 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:29.596677+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:30.233639+0000 osd.5 (osd.5) 371 : cluster 3 failed to encode map e836 with expected crc 2023-10-09T09:18:30.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[78107]: cluster 2023-10-09T09:18:30.237759+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e836: 8 total, 8 up, 8 in 2023-10-09T09:18:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:28.635867+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:30.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:29.415437+0000 mgr.y (mgr.34107) 358 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 298 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2023-10-09T09:18:30.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:29.577471+0000 osd.5 (osd.5) 368 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:29.584419+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e835: 8 total, 8 up, 8 in 2023-10-09T09:18:30.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:29.586534+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:29.586588+0000 osd.5 (osd.5) 369 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:29.590312+0000 osd.5 (osd.5) 370 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:29.590444+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:29.592181+0000 osd.7 (osd.7) 319 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:29.596677+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:30.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:30.233639+0000 osd.5 (osd.5) 371 : cluster 3 failed to encode map e836 with expected crc 2023-10-09T09:18:30.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:30 smithi163 bash[75425]: cluster 2023-10-09T09:18:30.237759+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e836: 8 total, 8 up, 8 in 2023-10-09T09:18:31.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:28.635867+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e834 with expected crc 2023-10-09T09:18:31.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:29.415437+0000 mgr.y (mgr.34107) 358 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 298 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2023-10-09T09:18:31.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:29.577471+0000 osd.5 (osd.5) 368 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:31.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:29.584419+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e835: 8 total, 8 up, 8 in 2023-10-09T09:18:31.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:29.586534+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:31.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:29.586588+0000 osd.5 (osd.5) 369 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:31.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:29.590312+0000 osd.5 (osd.5) 370 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:31.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:29.590444+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:31.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:29.592181+0000 osd.7 (osd.7) 319 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:31.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:29.596677+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:31.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:30.233639+0000 osd.5 (osd.5) 371 : cluster 3 failed to encode map e836 with expected crc 2023-10-09T09:18:31.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:30 smithi181 bash[59345]: cluster 2023-10-09T09:18:30.237759+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e836: 8 total, 8 up, 8 in 2023-10-09T09:18:31.248 INFO:tasks.rados.rados.0.smithi163.stdout:3103: done (0 left) 2023-10-09T09:18:31.249 INFO:tasks.rados.rados.0.smithi163.stdout:3104: read oid 35 snap -1 2023-10-09T09:18:31.249 INFO:tasks.rados.rados.0.smithi163.stdout:3104: expect deleted 2023-10-09T09:18:31.249 INFO:tasks.rados.rados.0.smithi163.stdout:3105: read oid 15 snap 376 2023-10-09T09:18:31.249 INFO:tasks.rados.rados.0.smithi163.stdout:3105: expect deleted 2023-10-09T09:18:31.249 INFO:tasks.rados.rados.0.smithi163.stdout:3106: rollback oid 13 current snap is 392 2023-10-09T09:18:31.249 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 13 to 383 2023-10-09T09:18:31.249 INFO:tasks.rados.rados.0.smithi163.stdout:3107: rollback oid 37 current snap is 392 2023-10-09T09:18:31.249 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 37 to 353 2023-10-09T09:18:31.249 INFO:tasks.rados.rados.0.smithi163.stdout:3108: read oid 46 snap -1 2023-10-09T09:18:31.249 INFO:tasks.rados.rados.0.smithi163.stdout:3108: expect (ObjNum 1145 snap 384 seq_num 1145) 2023-10-09T09:18:31.250 INFO:tasks.rados.rados.0.smithi163.stdout:3109: rollback oid 8 current snap is 392 2023-10-09T09:18:31.250 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 8 to 365 2023-10-09T09:18:31.250 INFO:tasks.rados.rados.0.smithi163.stdout:3110: write oid 12 current snap is 392 2023-10-09T09:18:31.250 INFO:tasks.rados.rados.0.smithi163.stdout:3110: seq_num 1164 ranges {763348=591845,1914452=434061,2881893=674270} 2023-10-09T09:18:31.257 INFO:tasks.rados.rados.0.smithi163.stdout:3110: writing smithi16367185-12 from 763348 to 1355193 tid 1 2023-10-09T09:18:31.263 INFO:tasks.rados.rados.0.smithi163.stdout:3110: writing smithi16367185-12 from 1914452 to 2348513 tid 2 2023-10-09T09:18:31.272 INFO:tasks.rados.rados.0.smithi163.stdout:3110: writing smithi16367185-12 from 2881893 to 3556163 tid 3 2023-10-09T09:18:31.272 INFO:tasks.rados.rados.0.smithi163.stdout:3111: delete oid 3 current snap is 392 2023-10-09T09:18:31.272 INFO:tasks.rados.rados.0.smithi163.stdout:3107: finishing rollback tid 0 to smithi16367185-37 2023-10-09T09:18:31.272 INFO:tasks.rados.rados.0.smithi163.stdout:3107: finishing rollback tid 1 to smithi16367185-37 2023-10-09T09:18:31.273 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 1535 (ObjNum 764 snap 0 seq_num 762) dirty dne 2023-10-09T09:18:31.273 INFO:tasks.rados.rados.0.smithi163.stdout:3109: finishing rollback tid 1 to smithi16367185-8 2023-10-09T09:18:31.273 INFO:tasks.rados.rados.0.smithi163.stdout:3106: finishing rollback tid 0 to smithi16367185-13 2023-10-09T09:18:31.273 INFO:tasks.rados.rados.0.smithi163.stdout:3109: finishing rollback tid 2 to smithi16367185-8 2023-10-09T09:18:31.273 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1537 (ObjNum 1092 snap 362 seq_num 1092) dirty exists 2023-10-09T09:18:31.273 INFO:tasks.rados.rados.0.smithi163.stdout:3106: finishing rollback tid 1 to smithi16367185-13 2023-10-09T09:18:31.273 INFO:tasks.rados.rados.0.smithi163.stdout:3106: finishing rollback tid 2 to smithi16367185-13 2023-10-09T09:18:31.273 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 548 (ObjNum 1088 snap 362 seq_num 1088) dirty exists 2023-10-09T09:18:31.276 INFO:tasks.rados.rados.0.smithi163.stdout:3108: expect (ObjNum 1145 snap 384 seq_num 1145) 2023-10-09T09:18:31.445 INFO:tasks.rados.rados.0.smithi163.stdout:3110: finishing write tid 1 to smithi16367185-12 2023-10-09T09:18:31.445 INFO:tasks.rados.rados.0.smithi163.stdout:3110: finishing write tid 2 to smithi16367185-12 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:3110: finishing write tid 3 to smithi16367185-12 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:3110: finishing write tid 4 to smithi16367185-12 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:3110: finishing write tid 5 to smithi16367185-12 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 1449 (ObjNum 1164 snap 392 seq_num 1164) dirty exists 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:3110: left oid 12 (ObjNum 1164 snap 392 seq_num 1164) 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:3104: done (7 left) 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:3105: done (6 left) 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:3106: done (5 left) 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:3107: done (4 left) 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:3108: done (3 left) 2023-10-09T09:18:31.446 INFO:tasks.rados.rados.0.smithi163.stdout:3109: done (2 left) 2023-10-09T09:18:31.447 INFO:tasks.rados.rados.0.smithi163.stdout:3110: done (1 left) 2023-10-09T09:18:31.447 INFO:tasks.rados.rados.0.smithi163.stdout:3111: done (0 left) 2023-10-09T09:18:31.447 INFO:tasks.rados.rados.0.smithi163.stdout:3112: snap_remove snap 362 2023-10-09T09:18:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[75425]: cluster 2023-10-09T09:18:29.590579+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[75425]: cluster 2023-10-09T09:18:30.457648+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e836 with expected crc 2023-10-09T09:18:31.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[75425]: cluster 2023-10-09T09:18:30.572555+0000 osd.7 (osd.7) 320 : cluster 3 failed to encode map e836 with expected crc 2023-10-09T09:18:31.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[75425]: cluster 2023-10-09T09:18:31.242336+0000 osd.5 (osd.5) 372 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:31.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[75425]: cluster 2023-10-09T09:18:31.246995+0000 mon.a (mon.0) 1032 : cluster 0 osdmap e837: 8 total, 8 up, 8 in 2023-10-09T09:18:31.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[75425]: cluster 2023-10-09T09:18:31.249330+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:31.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[75425]: cluster 2023-10-09T09:18:31.249442+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:31.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[75425]: cluster 2023-10-09T09:18:31.249786+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:31.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[75425]: cluster 2023-10-09T09:18:31.253529+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:31.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[78107]: cluster 2023-10-09T09:18:29.590579+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:31.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[78107]: cluster 2023-10-09T09:18:30.457648+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e836 with expected crc 2023-10-09T09:18:31.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[78107]: cluster 2023-10-09T09:18:30.572555+0000 osd.7 (osd.7) 320 : cluster 3 failed to encode map e836 with expected crc 2023-10-09T09:18:31.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[78107]: cluster 2023-10-09T09:18:31.242336+0000 osd.5 (osd.5) 372 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:31.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[78107]: cluster 2023-10-09T09:18:31.246995+0000 mon.a (mon.0) 1032 : cluster 0 osdmap e837: 8 total, 8 up, 8 in 2023-10-09T09:18:31.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[78107]: cluster 2023-10-09T09:18:31.249330+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:31.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[78107]: cluster 2023-10-09T09:18:31.249442+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:31.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[78107]: cluster 2023-10-09T09:18:31.249786+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:31.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:31 smithi163 bash[78107]: cluster 2023-10-09T09:18:31.253529+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:31 smithi181 bash[59345]: cluster 2023-10-09T09:18:29.590579+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e835 with expected crc 2023-10-09T09:18:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:31 smithi181 bash[59345]: cluster 2023-10-09T09:18:30.457648+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e836 with expected crc 2023-10-09T09:18:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:31 smithi181 bash[59345]: cluster 2023-10-09T09:18:30.572555+0000 osd.7 (osd.7) 320 : cluster 3 failed to encode map e836 with expected crc 2023-10-09T09:18:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:31 smithi181 bash[59345]: cluster 2023-10-09T09:18:31.242336+0000 osd.5 (osd.5) 372 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:31 smithi181 bash[59345]: cluster 2023-10-09T09:18:31.246995+0000 mon.a (mon.0) 1032 : cluster 0 osdmap e837: 8 total, 8 up, 8 in 2023-10-09T09:18:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:31 smithi181 bash[59345]: cluster 2023-10-09T09:18:31.249330+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:31 smithi181 bash[59345]: cluster 2023-10-09T09:18:31.249442+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:31 smithi181 bash[59345]: cluster 2023-10-09T09:18:31.249786+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:32.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:31 smithi181 bash[59345]: cluster 2023-10-09T09:18:31.253529+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e837 with expected crc 2023-10-09T09:18:32.254 INFO:tasks.rados.rados.0.smithi163.stdout:3112: done (0 left) 2023-10-09T09:18:32.254 INFO:tasks.rados.rados.0.smithi163.stdout:3113: read oid 42 snap 353 2023-10-09T09:18:32.254 INFO:tasks.rados.rados.0.smithi163.stdout:3113: expect (ObjNum 1040 snap 352 seq_num 1040) 2023-10-09T09:18:32.254 INFO:tasks.rados.rados.0.smithi163.stdout:3114: write oid 14 current snap is 392 2023-10-09T09:18:32.254 INFO:tasks.rados.rados.0.smithi163.stdout:3114: seq_num 1166 ranges {796962=742343,2031416=437730,2936739=413167} 2023-10-09T09:18:32.263 INFO:tasks.rados.rados.0.smithi163.stdout:3114: writing smithi16367185-14 from 796962 to 1539305 tid 1 2023-10-09T09:18:32.271 INFO:tasks.rados.rados.0.smithi163.stdout:3114: writing smithi16367185-14 from 2031416 to 2469146 tid 2 2023-10-09T09:18:32.276 INFO:tasks.rados.rados.0.smithi163.stdout:3114: writing smithi16367185-14 from 2936739 to 3349906 tid 3 2023-10-09T09:18:32.277 INFO:tasks.rados.rados.0.smithi163.stdout:3115: rollback oid 5 current snap is 392 2023-10-09T09:18:32.277 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 377 2023-10-09T09:18:32.277 INFO:tasks.rados.rados.0.smithi163.stdout:3113: expect (ObjNum 1040 snap 352 seq_num 1040) 2023-10-09T09:18:32.393 INFO:tasks.rados.rados.0.smithi163.stdout:3114: finishing write tid 1 to smithi16367185-14 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:3114: finishing write tid 2 to smithi16367185-14 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:3114: finishing write tid 3 to smithi16367185-14 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:3114: finishing write tid 4 to smithi16367185-14 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:3114: finishing write tid 5 to smithi16367185-14 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1472 (ObjNum 1166 snap 392 seq_num 1166) dirty exists 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:3114: left oid 14 (ObjNum 1166 snap 392 seq_num 1166) 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:3115: finishing rollback tid 0 to smithi16367185-5 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:3115: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:3115: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1487 (ObjNum 1024 snap 349 seq_num 1024) dirty exists 2023-10-09T09:18:32.394 INFO:tasks.rados.rados.0.smithi163.stdout:3113: done (2 left) 2023-10-09T09:18:32.395 INFO:tasks.rados.rados.0.smithi163.stdout:3114: done (1 left) 2023-10-09T09:18:32.395 INFO:tasks.rados.rados.0.smithi163.stdout:3115: done (0 left) 2023-10-09T09:18:32.395 INFO:tasks.rados.rados.0.smithi163.stdout:3116: snap_create 2023-10-09T09:18:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:32 smithi163 bash[78107]: cluster 2023-10-09T09:18:31.416433+0000 mgr.y (mgr.34107) 359 : cluster 0 pgmap v809: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 312 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.5 MiB/s wr, 29 op/s 2023-10-09T09:18:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:32 smithi163 bash[78107]: cluster 2023-10-09T09:18:32.250518+0000 osd.5 (osd.5) 373 : cluster 3 failed to encode map e838 with expected crc 2023-10-09T09:18:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:32 smithi163 bash[78107]: cluster 2023-10-09T09:18:32.253290+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e838: 8 total, 8 up, 8 in 2023-10-09T09:18:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:32 smithi163 bash[78107]: cluster 2023-10-09T09:18:32.255527+0000 osd.7 (osd.7) 322 : cluster 3 failed to encode map e838 with expected crc 2023-10-09T09:18:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:32 smithi163 bash[78107]: cluster 2023-10-09T09:18:32.258885+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e838 with expected crc 2023-10-09T09:18:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:32 smithi163 bash[75425]: cluster 2023-10-09T09:18:31.416433+0000 mgr.y (mgr.34107) 359 : cluster 0 pgmap v809: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 312 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.5 MiB/s wr, 29 op/s 2023-10-09T09:18:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:32 smithi163 bash[75425]: cluster 2023-10-09T09:18:32.250518+0000 osd.5 (osd.5) 373 : cluster 3 failed to encode map e838 with expected crc 2023-10-09T09:18:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:32 smithi163 bash[75425]: cluster 2023-10-09T09:18:32.253290+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e838: 8 total, 8 up, 8 in 2023-10-09T09:18:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:32 smithi163 bash[75425]: cluster 2023-10-09T09:18:32.255527+0000 osd.7 (osd.7) 322 : cluster 3 failed to encode map e838 with expected crc 2023-10-09T09:18:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:32 smithi163 bash[75425]: cluster 2023-10-09T09:18:32.258885+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e838 with expected crc 2023-10-09T09:18:33.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:32 smithi181 bash[59345]: cluster 2023-10-09T09:18:31.416433+0000 mgr.y (mgr.34107) 359 : cluster 0 pgmap v809: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 312 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.5 MiB/s wr, 29 op/s 2023-10-09T09:18:33.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:32 smithi181 bash[59345]: cluster 2023-10-09T09:18:32.250518+0000 osd.5 (osd.5) 373 : cluster 3 failed to encode map e838 with expected crc 2023-10-09T09:18:33.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:32 smithi181 bash[59345]: cluster 2023-10-09T09:18:32.253290+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e838: 8 total, 8 up, 8 in 2023-10-09T09:18:33.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:32 smithi181 bash[59345]: cluster 2023-10-09T09:18:32.255527+0000 osd.7 (osd.7) 322 : cluster 3 failed to encode map e838 with expected crc 2023-10-09T09:18:33.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:32 smithi181 bash[59345]: cluster 2023-10-09T09:18:32.258885+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e838 with expected crc 2023-10-09T09:18:33.264 INFO:tasks.rados.rados.0.smithi163.stdout:3116: done (0 left) 2023-10-09T09:18:33.264 INFO:tasks.rados.rados.0.smithi163.stdout:3117: read oid 8 snap -1 2023-10-09T09:18:33.264 INFO:tasks.rados.rados.0.smithi163.stdout:3117: expect (ObjNum 1092 snap 362 seq_num 1092) 2023-10-09T09:18:33.264 INFO:tasks.rados.rados.0.smithi163.stdout:3118: write oid 31 current snap is 393 2023-10-09T09:18:33.264 INFO:tasks.rados.rados.0.smithi163.stdout:3118: seq_num 1167 ranges {413769=417592,1573545=697741,2847809=624439} 2023-10-09T09:18:33.273 INFO:tasks.rados.rados.0.smithi163.stdout:3118: writing smithi16367185-31 from 413769 to 831361 tid 1 2023-10-09T09:18:33.282 INFO:tasks.rados.rados.0.smithi163.stdout:3118: writing smithi16367185-31 from 1573545 to 2271286 tid 2 2023-10-09T09:18:33.290 INFO:tasks.rados.rados.0.smithi163.stdout:3118: writing smithi16367185-31 from 2847809 to 3472248 tid 3 2023-10-09T09:18:33.290 INFO:tasks.rados.rados.0.smithi163.stdout:3119: snap_create 2023-10-09T09:18:33.290 INFO:tasks.rados.rados.0.smithi163.stdout:3117: expect (ObjNum 1092 snap 362 seq_num 1092) 2023-10-09T09:18:33.399 INFO:tasks.rados.rados.0.smithi163.stdout:3118: finishing write tid 1 to smithi16367185-31 2023-10-09T09:18:33.400 INFO:tasks.rados.rados.0.smithi163.stdout:3118: finishing write tid 2 to smithi16367185-31 2023-10-09T09:18:33.400 INFO:tasks.rados.rados.0.smithi163.stdout:3118: finishing write tid 3 to smithi16367185-31 2023-10-09T09:18:33.400 INFO:tasks.rados.rados.0.smithi163.stdout:3118: finishing write tid 4 to smithi16367185-31 2023-10-09T09:18:33.400 INFO:tasks.rados.rados.0.smithi163.stdout:3118: finishing write tid 5 to smithi16367185-31 2023-10-09T09:18:33.400 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 820 (ObjNum 1167 snap 393 seq_num 1167) dirty exists 2023-10-09T09:18:33.400 INFO:tasks.rados.rados.0.smithi163.stdout:3118: left oid 31 (ObjNum 1167 snap 393 seq_num 1167) 2023-10-09T09:18:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:18:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:18:33] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.33.4" 2023-10-09T09:18:34.265 INFO:tasks.rados.rados.0.smithi163.stdout:3117: done (2 left) 2023-10-09T09:18:34.265 INFO:tasks.rados.rados.0.smithi163.stdout:3118: done (1 left) 2023-10-09T09:18:34.265 INFO:tasks.rados.rados.0.smithi163.stdout:3119: done (0 left) 2023-10-09T09:18:34.265 INFO:tasks.rados.rados.0.smithi163.stdout:3120: snap_remove snap 388 2023-10-09T09:18:34.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:34 smithi181 bash[59345]: cluster 2023-10-09T09:18:33.260195+0000 osd.5 (osd.5) 374 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:34 smithi181 bash[59345]: cluster 2023-10-09T09:18:33.260196+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:34 smithi181 bash[59345]: cluster 2023-10-09T09:18:33.263355+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e839: 8 total, 8 up, 8 in 2023-10-09T09:18:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:34 smithi181 bash[59345]: cluster 2023-10-09T09:18:33.264436+0000 osd.7 (osd.7) 323 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:34 smithi181 bash[59345]: cluster 2023-10-09T09:18:33.281164+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:34 smithi181 bash[59345]: cluster 2023-10-09T09:18:33.417126+0000 mgr.y (mgr.34107) 360 : cluster 0 pgmap v812: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 315 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2023-10-09T09:18:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[75425]: cluster 2023-10-09T09:18:33.260195+0000 osd.5 (osd.5) 374 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[75425]: cluster 2023-10-09T09:18:33.260196+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[75425]: cluster 2023-10-09T09:18:33.263355+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e839: 8 total, 8 up, 8 in 2023-10-09T09:18:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[75425]: cluster 2023-10-09T09:18:33.264436+0000 osd.7 (osd.7) 323 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[75425]: cluster 2023-10-09T09:18:33.281164+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[75425]: cluster 2023-10-09T09:18:33.417126+0000 mgr.y (mgr.34107) 360 : cluster 0 pgmap v812: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 315 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2023-10-09T09:18:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[78107]: cluster 2023-10-09T09:18:33.260195+0000 osd.5 (osd.5) 374 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[78107]: cluster 2023-10-09T09:18:33.260196+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[78107]: cluster 2023-10-09T09:18:33.263355+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e839: 8 total, 8 up, 8 in 2023-10-09T09:18:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[78107]: cluster 2023-10-09T09:18:33.264436+0000 osd.7 (osd.7) 323 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[78107]: cluster 2023-10-09T09:18:33.281164+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e839 with expected crc 2023-10-09T09:18:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:34 smithi163 bash[78107]: cluster 2023-10-09T09:18:33.417126+0000 mgr.y (mgr.34107) 360 : cluster 0 pgmap v812: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 315 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2023-10-09T09:18:35.238 INFO:tasks.rados.rados.0.smithi163.stdout:3120: done (0 left) 2023-10-09T09:18:35.239 INFO:tasks.rados.rados.0.smithi163.stdout:3121: read oid 35 snap -1 2023-10-09T09:18:35.239 INFO:tasks.rados.rados.0.smithi163.stdout:3121: expect deleted 2023-10-09T09:18:35.239 INFO:tasks.rados.rados.0.smithi163.stdout:3122: snap_remove snap 383 2023-10-09T09:18:35.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:35 smithi181 bash[59345]: cluster 2023-10-09T09:18:34.260715+0000 osd.5 (osd.5) 375 : cluster 3 failed to encode map e840 with expected crc 2023-10-09T09:18:35.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:35 smithi181 bash[59345]: cluster 2023-10-09T09:18:34.261309+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e840 with expected crc 2023-10-09T09:18:35.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:35 smithi181 bash[59345]: cluster 2023-10-09T09:18:34.265245+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e840: 8 total, 8 up, 8 in 2023-10-09T09:18:35.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:35 smithi181 bash[59345]: cluster 2023-10-09T09:18:35.237959+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e841: 8 total, 8 up, 8 in 2023-10-09T09:18:35.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:35 smithi163 bash[75425]: cluster 2023-10-09T09:18:34.260715+0000 osd.5 (osd.5) 375 : cluster 3 failed to encode map e840 with expected crc 2023-10-09T09:18:35.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:35 smithi163 bash[75425]: cluster 2023-10-09T09:18:34.261309+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e840 with expected crc 2023-10-09T09:18:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:35 smithi163 bash[75425]: cluster 2023-10-09T09:18:34.265245+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e840: 8 total, 8 up, 8 in 2023-10-09T09:18:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:35 smithi163 bash[75425]: cluster 2023-10-09T09:18:35.237959+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e841: 8 total, 8 up, 8 in 2023-10-09T09:18:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:35 smithi163 bash[78107]: cluster 2023-10-09T09:18:34.260715+0000 osd.5 (osd.5) 375 : cluster 3 failed to encode map e840 with expected crc 2023-10-09T09:18:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:35 smithi163 bash[78107]: cluster 2023-10-09T09:18:34.261309+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e840 with expected crc 2023-10-09T09:18:35.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:35 smithi163 bash[78107]: cluster 2023-10-09T09:18:34.265245+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e840: 8 total, 8 up, 8 in 2023-10-09T09:18:35.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:35 smithi163 bash[78107]: cluster 2023-10-09T09:18:35.237959+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e841: 8 total, 8 up, 8 in 2023-10-09T09:18:36.242 INFO:tasks.rados.rados.0.smithi163.stdout:3122: done (1 left) 2023-10-09T09:18:36.242 INFO:tasks.rados.rados.0.smithi163.stdout:3123: write oid 42 current snap is 394 2023-10-09T09:18:36.243 INFO:tasks.rados.rados.0.smithi163.stdout:3123: seq_num 1168 ranges {430576=492841,1632027=441399,2475232=435711,3365397=265179} 2023-10-09T09:18:36.250 INFO:tasks.rados.rados.0.smithi163.stdout:3123: writing smithi16367185-42 from 430576 to 923417 tid 1 2023-10-09T09:18:36.258 INFO:tasks.rados.rados.0.smithi163.stdout:3123: writing smithi16367185-42 from 1632027 to 2073426 tid 2 2023-10-09T09:18:36.263 INFO:tasks.rados.rados.0.smithi163.stdout:3123: writing smithi16367185-42 from 2475232 to 2910943 tid 3 2023-10-09T09:18:36.268 INFO:tasks.rados.rados.0.smithi163.stdout:3123: writing smithi16367185-42 from 3365397 to 3630576 tid 4 2023-10-09T09:18:36.268 INFO:tasks.rados.rados.0.smithi163.stdout:3121: done (1 left) 2023-10-09T09:18:36.269 INFO:tasks.rados.rados.0.smithi163.stdout:3124: read oid 35 snap -1 2023-10-09T09:18:36.269 INFO:tasks.rados.rados.0.smithi163.stdout:3124: expect deleted 2023-10-09T09:18:36.269 INFO:tasks.rados.rados.0.smithi163.stdout:3125: read oid 22 snap -1 2023-10-09T09:18:36.269 INFO:tasks.rados.rados.0.smithi163.stdout:3125: expect (ObjNum 1115 snap 368 seq_num 1115) 2023-10-09T09:18:36.269 INFO:tasks.rados.rados.0.smithi163.stdout:3126: write oid 23 current snap is 394 2023-10-09T09:18:36.269 INFO:tasks.rados.rados.0.smithi163.stdout:3123: finishing write tid 1 to smithi16367185-42 2023-10-09T09:18:36.269 INFO:tasks.rados.rados.0.smithi163.stdout:3126: seq_num 1169 ranges {447383=568090,1690509=701410,2902655=646983} 2023-10-09T09:18:36.275 INFO:tasks.rados.rados.0.smithi163.stdout:3126: writing smithi16367185-23 from 447383 to 1015473 tid 1 2023-10-09T09:18:36.283 INFO:tasks.rados.rados.0.smithi163.stdout:3126: writing smithi16367185-23 from 1690509 to 2391919 tid 2 2023-10-09T09:18:36.290 INFO:tasks.rados.rados.0.smithi163.stdout:3126: writing smithi16367185-23 from 2902655 to 3549638 tid 3 2023-10-09T09:18:36.290 INFO:tasks.rados.rados.0.smithi163.stdout:3127: read oid 45 snap 353 2023-10-09T09:18:36.290 INFO:tasks.rados.rados.0.smithi163.stdout:3127: expect (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:18:36.291 INFO:tasks.rados.rados.0.smithi163.stdout:3128: rollback oid 5 current snap is 394 2023-10-09T09:18:36.291 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 390 2023-10-09T09:18:36.291 INFO:tasks.rados.rados.0.smithi163.stdout:3123: finishing write tid 2 to smithi16367185-42 2023-10-09T09:18:36.291 INFO:tasks.rados.rados.0.smithi163.stdout:3123: finishing write tid 3 to smithi16367185-42 2023-10-09T09:18:36.291 INFO:tasks.rados.rados.0.smithi163.stdout:3123: finishing write tid 4 to smithi16367185-42 2023-10-09T09:18:36.291 INFO:tasks.rados.rados.0.smithi163.stdout:3123: finishing write tid 5 to smithi16367185-42 2023-10-09T09:18:36.291 INFO:tasks.rados.rados.0.smithi163.stdout:3123: finishing write tid 6 to smithi16367185-42 2023-10-09T09:18:36.291 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 2224 (ObjNum 1168 snap 394 seq_num 1168) dirty exists 2023-10-09T09:18:36.291 INFO:tasks.rados.rados.0.smithi163.stdout:3123: left oid 42 (ObjNum 1168 snap 394 seq_num 1168) 2023-10-09T09:18:36.291 INFO:tasks.rados.rados.0.smithi163.stdout:3125: expect (ObjNum 1115 snap 368 seq_num 1115) 2023-10-09T09:18:36.503 INFO:tasks.rados.rados.0.smithi163.stdout:3126: finishing write tid 1 to smithi16367185-23 2023-10-09T09:18:36.503 INFO:tasks.rados.rados.0.smithi163.stdout:3126: finishing write tid 2 to smithi16367185-23 2023-10-09T09:18:36.503 INFO:tasks.rados.rados.0.smithi163.stdout:3123: done (5 left) 2023-10-09T09:18:36.503 INFO:tasks.rados.rados.0.smithi163.stdout:3124: done (4 left) 2023-10-09T09:18:36.503 INFO:tasks.rados.rados.0.smithi163.stdout:3125: done (3 left) 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3129: read oid 40 snap 386 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3129: expect (ObjNum 1027 snap 349 seq_num 1027) 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3130: rollback oid 15 current snap is 394 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 15 to 373 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3128: finishing rollback tid 0 to smithi16367185-5 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3128: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3128: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1503 (ObjNum 1137 snap 378 seq_num 1137) dirty exists 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3126: finishing write tid 3 to smithi16367185-23 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3128: done (4 left) 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3131: read oid 1 snap -1 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3131: expect (ObjNum 1073 snap 361 seq_num 1073) 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3132: delete oid 9 current snap is 394 2023-10-09T09:18:36.504 INFO:tasks.rados.rados.0.smithi163.stdout:3126: finishing write tid 4 to smithi16367185-23 2023-10-09T09:18:36.505 INFO:tasks.rados.rados.0.smithi163.stdout:3126: finishing write tid 5 to smithi16367185-23 2023-10-09T09:18:36.505 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1433 (ObjNum 1169 snap 394 seq_num 1169) dirty exists 2023-10-09T09:18:36.505 INFO:tasks.rados.rados.0.smithi163.stdout:3126: left oid 23 (ObjNum 1169 snap 394 seq_num 1169) 2023-10-09T09:18:36.505 INFO:tasks.rados.rados.0.smithi163.stdout:3127: expect (ObjNum 859 snap 292 seq_num 859) 2023-10-09T09:18:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:36 smithi181 bash[59345]: cluster 2023-10-09T09:18:35.234026+0000 osd.5 (osd.5) 376 : cluster 3 failed to encode map e841 with expected crc 2023-10-09T09:18:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:36 smithi181 bash[59345]: cluster 2023-10-09T09:18:35.234398+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e841 with expected crc 2023-10-09T09:18:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:36 smithi181 bash[59345]: cluster 2023-10-09T09:18:35.239808+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e841 with expected crc 2023-10-09T09:18:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:36 smithi181 bash[59345]: cluster 2023-10-09T09:18:35.417841+0000 mgr.y (mgr.34107) 361 : cluster 0 pgmap v815: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 318 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 415 KiB/s wr, 4 op/s 2023-10-09T09:18:36.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:36 smithi181 bash[59345]: cluster 2023-10-09T09:18:36.241635+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e842: 8 total, 8 up, 8 in 2023-10-09T09:18:36.587 INFO:tasks.rados.rados.0.smithi163.stdout:3131: expect (ObjNum 1073 snap 361 seq_num 1073) 2023-10-09T09:18:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:36 smithi163 bash[75425]: cluster 2023-10-09T09:18:35.234026+0000 osd.5 (osd.5) 376 : cluster 3 failed to encode map e841 with expected crc 2023-10-09T09:18:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:36 smithi163 bash[75425]: cluster 2023-10-09T09:18:35.234398+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e841 with expected crc 2023-10-09T09:18:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:36 smithi163 bash[75425]: cluster 2023-10-09T09:18:35.239808+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e841 with expected crc 2023-10-09T09:18:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:36 smithi163 bash[75425]: cluster 2023-10-09T09:18:35.417841+0000 mgr.y (mgr.34107) 361 : cluster 0 pgmap v815: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 318 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 415 KiB/s wr, 4 op/s 2023-10-09T09:18:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:36 smithi163 bash[75425]: cluster 2023-10-09T09:18:36.241635+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e842: 8 total, 8 up, 8 in 2023-10-09T09:18:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:36 smithi163 bash[78107]: cluster 2023-10-09T09:18:35.234026+0000 osd.5 (osd.5) 376 : cluster 3 failed to encode map e841 with expected crc 2023-10-09T09:18:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:36 smithi163 bash[78107]: cluster 2023-10-09T09:18:35.234398+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e841 with expected crc 2023-10-09T09:18:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:36 smithi163 bash[78107]: cluster 2023-10-09T09:18:35.239808+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e841 with expected crc 2023-10-09T09:18:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:36 smithi163 bash[78107]: cluster 2023-10-09T09:18:35.417841+0000 mgr.y (mgr.34107) 361 : cluster 0 pgmap v815: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 318 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 415 KiB/s wr, 4 op/s 2023-10-09T09:18:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:36 smithi163 bash[78107]: cluster 2023-10-09T09:18:36.241635+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e842: 8 total, 8 up, 8 in 2023-10-09T09:18:36.728 INFO:tasks.rados.rados.0.smithi163.stdout:3129: expect (ObjNum 1027 snap 349 seq_num 1027) 2023-10-09T09:18:36.924 INFO:tasks.rados.rados.0.smithi163.stdout:3130: finishing rollback tid 1 to smithi16367185-15 2023-10-09T09:18:36.924 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 0 (ObjNum 1 snap 21875 seq_num 698976832) dirty dne 2023-10-09T09:18:36.924 INFO:tasks.rados.rados.0.smithi163.stdout:3126: done (5 left) 2023-10-09T09:18:36.924 INFO:tasks.rados.rados.0.smithi163.stdout:3127: done (4 left) 2023-10-09T09:18:36.924 INFO:tasks.rados.rados.0.smithi163.stdout:3129: done (3 left) 2023-10-09T09:18:36.924 INFO:tasks.rados.rados.0.smithi163.stdout:3130: done (2 left) 2023-10-09T09:18:36.924 INFO:tasks.rados.rados.0.smithi163.stdout:3131: done (1 left) 2023-10-09T09:18:36.925 INFO:tasks.rados.rados.0.smithi163.stdout:3132: done (0 left) 2023-10-09T09:18:36.925 INFO:tasks.rados.rados.0.smithi163.stdout:3133: snap_remove snap 387 2023-10-09T09:18:37.282 INFO:tasks.rados.rados.0.smithi163.stdout:3133: done (0 left) 2023-10-09T09:18:37.282 INFO:tasks.rados.rados.0.smithi163.stdout:3134: snap_remove snap 378 2023-10-09T09:18:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:37 smithi181 bash[59345]: cluster 2023-10-09T09:18:36.236613+0000 osd.5 (osd.5) 377 : cluster 3 failed to encode map e842 with expected crc 2023-10-09T09:18:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:37 smithi181 bash[59345]: cluster 2023-10-09T09:18:36.244067+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e842 with expected crc 2023-10-09T09:18:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:37 smithi163 bash[75425]: cluster 2023-10-09T09:18:36.236613+0000 osd.5 (osd.5) 377 : cluster 3 failed to encode map e842 with expected crc 2023-10-09T09:18:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:37 smithi163 bash[75425]: cluster 2023-10-09T09:18:36.244067+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e842 with expected crc 2023-10-09T09:18:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:37 smithi163 bash[78107]: cluster 2023-10-09T09:18:36.236613+0000 osd.5 (osd.5) 377 : cluster 3 failed to encode map e842 with expected crc 2023-10-09T09:18:37.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:37 smithi163 bash[78107]: cluster 2023-10-09T09:18:36.244067+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e842 with expected crc 2023-10-09T09:18:38.299 INFO:tasks.rados.rados.0.smithi163.stdout:3134: done (0 left) 2023-10-09T09:18:38.299 INFO:tasks.rados.rados.0.smithi163.stdout:3135: write oid 40 current snap is 394 2023-10-09T09:18:38.300 INFO:tasks.rados.rados.0.smithi163.stdout:3135: seq_num 1171 ranges {480997=434941,1407473=705079,2557501=785880} 2023-10-09T09:18:38.310 INFO:tasks.rados.rados.0.smithi163.stdout:3135: writing smithi16367185-40 from 480997 to 915938 tid 1 2023-10-09T09:18:38.315 INFO:tasks.rados.rados.0.smithi163.stdout:3135: writing smithi16367185-40 from 1407473 to 2112552 tid 2 2023-10-09T09:18:38.323 INFO:tasks.rados.rados.0.smithi163.stdout:3135: writing smithi16367185-40 from 2557501 to 3343381 tid 3 2023-10-09T09:18:38.323 INFO:tasks.rados.rados.0.smithi163.stdout:3136: delete oid 19 current snap is 394 2023-10-09T09:18:38.323 INFO:tasks.rados.rados.0.smithi163.stdout:3135: finishing write tid 1 to smithi16367185-40 2023-10-09T09:18:38.323 INFO:tasks.rados.rados.0.smithi163.stdout:3135: finishing write tid 2 to smithi16367185-40 2023-10-09T09:18:38.326 INFO:tasks.rados.rados.0.smithi163.stdout:3136: done (1 left) 2023-10-09T09:18:38.326 INFO:tasks.rados.rados.0.smithi163.stdout:3137: delete oid 33 current snap is 394 2023-10-09T09:18:38.328 INFO:tasks.rados.rados.0.smithi163.stdout:3135: finishing write tid 3 to smithi16367185-40 2023-10-09T09:18:38.328 INFO:tasks.rados.rados.0.smithi163.stdout:3135: finishing write tid 4 to smithi16367185-40 2023-10-09T09:18:38.328 INFO:tasks.rados.rados.0.smithi163.stdout:3135: finishing write tid 5 to smithi16367185-40 2023-10-09T09:18:38.328 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1563 (ObjNum 1171 snap 394 seq_num 1171) dirty exists 2023-10-09T09:18:38.329 INFO:tasks.rados.rados.0.smithi163.stdout:3135: left oid 40 (ObjNum 1171 snap 394 seq_num 1171) 2023-10-09T09:18:38.329 INFO:tasks.rados.rados.0.smithi163.stdout:3135: done (1 left) 2023-10-09T09:18:38.329 INFO:tasks.rados.rados.0.smithi163.stdout:3137: done (0 left) 2023-10-09T09:18:38.329 INFO:tasks.rados.rados.0.smithi163.stdout:3138: snap_create 2023-10-09T09:18:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:38 smithi181 bash[59345]: cluster 2023-10-09T09:18:37.279877+0000 osd.5 (osd.5) 378 : cluster 3 failed to encode map e843 with expected crc 2023-10-09T09:18:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:38 smithi181 bash[59345]: cluster 2023-10-09T09:18:37.282123+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e843: 8 total, 8 up, 8 in 2023-10-09T09:18:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:38 smithi181 bash[59345]: cluster 2023-10-09T09:18:37.282872+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e843 with expected crc 2023-10-09T09:18:38.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:38 smithi181 bash[59345]: cluster 2023-10-09T09:18:37.418533+0000 mgr.y (mgr.34107) 362 : cluster 0 pgmap v818: 89 pgs: 89 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2023-10-09T09:18:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:38 smithi163 bash[75425]: cluster 2023-10-09T09:18:37.279877+0000 osd.5 (osd.5) 378 : cluster 3 failed to encode map e843 with expected crc 2023-10-09T09:18:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:38 smithi163 bash[75425]: cluster 2023-10-09T09:18:37.282123+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e843: 8 total, 8 up, 8 in 2023-10-09T09:18:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:38 smithi163 bash[75425]: cluster 2023-10-09T09:18:37.282872+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e843 with expected crc 2023-10-09T09:18:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:38 smithi163 bash[75425]: cluster 2023-10-09T09:18:37.418533+0000 mgr.y (mgr.34107) 362 : cluster 0 pgmap v818: 89 pgs: 89 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2023-10-09T09:18:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:38 smithi163 bash[78107]: cluster 2023-10-09T09:18:37.279877+0000 osd.5 (osd.5) 378 : cluster 3 failed to encode map e843 with expected crc 2023-10-09T09:18:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:38 smithi163 bash[78107]: cluster 2023-10-09T09:18:37.282123+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e843: 8 total, 8 up, 8 in 2023-10-09T09:18:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:38 smithi163 bash[78107]: cluster 2023-10-09T09:18:37.282872+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e843 with expected crc 2023-10-09T09:18:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:38 smithi163 bash[78107]: cluster 2023-10-09T09:18:37.418533+0000 mgr.y (mgr.34107) 362 : cluster 0 pgmap v818: 89 pgs: 89 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2023-10-09T09:18:39.301 INFO:tasks.rados.rados.0.smithi163.stdout:3138: done (0 left) 2023-10-09T09:18:39.301 INFO:tasks.rados.rados.0.smithi163.stdout:3139: rollback oid 4 current snap is 395 2023-10-09T09:18:39.301 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 393 2023-10-09T09:18:39.302 INFO:tasks.rados.rados.0.smithi163.stdout:3140: snap_create 2023-10-09T09:18:39.306 INFO:tasks.rados.rados.0.smithi163.stdout:3139: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:18:39.307 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1458 (ObjNum 0 snap 32603 seq_num 699665712) dirty dne 2023-10-09T09:18:39.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:39 smithi181 bash[59345]: cluster 2023-10-09T09:18:38.291563+0000 osd.5 (osd.5) 379 : cluster 3 failed to encode map e844 with expected crc 2023-10-09T09:18:39.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:39 smithi181 bash[59345]: cluster 2023-10-09T09:18:38.298247+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e844 with expected crc 2023-10-09T09:18:39.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:39 smithi181 bash[59345]: cluster 2023-10-09T09:18:38.298248+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e844: 8 total, 8 up, 8 in 2023-10-09T09:18:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:39 smithi163 bash[78107]: cluster 2023-10-09T09:18:38.291563+0000 osd.5 (osd.5) 379 : cluster 3 failed to encode map e844 with expected crc 2023-10-09T09:18:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:39 smithi163 bash[78107]: cluster 2023-10-09T09:18:38.298247+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e844 with expected crc 2023-10-09T09:18:39.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:39 smithi163 bash[78107]: cluster 2023-10-09T09:18:38.298248+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e844: 8 total, 8 up, 8 in 2023-10-09T09:18:39.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:39 smithi163 bash[75425]: cluster 2023-10-09T09:18:38.291563+0000 osd.5 (osd.5) 379 : cluster 3 failed to encode map e844 with expected crc 2023-10-09T09:18:39.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:39 smithi163 bash[75425]: cluster 2023-10-09T09:18:38.298247+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e844 with expected crc 2023-10-09T09:18:39.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:39 smithi163 bash[75425]: cluster 2023-10-09T09:18:38.298248+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e844: 8 total, 8 up, 8 in 2023-10-09T09:18:40.241 INFO:tasks.rados.rados.0.smithi163.stdout:3139: done (1 left) 2023-10-09T09:18:40.242 INFO:tasks.rados.rados.0.smithi163.stdout:3140: done (0 left) 2023-10-09T09:18:40.242 INFO:tasks.rados.rados.0.smithi163.stdout:3141: rollback oid 18 current snap is 396 2023-10-09T09:18:40.242 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 18 to 382 2023-10-09T09:18:40.242 INFO:tasks.rados.rados.0.smithi163.stdout:3142: read oid 29 snap 394 2023-10-09T09:18:40.242 INFO:tasks.rados.rados.0.smithi163.stdout:3142: expect deleted 2023-10-09T09:18:40.242 INFO:tasks.rados.rados.0.smithi163.stdout:3143: snap_create 2023-10-09T09:18:40.251 INFO:tasks.rados.rados.0.smithi163.stdout:3141: finishing rollback tid 0 to smithi16367185-18 2023-10-09T09:18:40.251 INFO:tasks.rados.rados.0.smithi163.stdout:3141: finishing rollback tid 1 to smithi16367185-18 2023-10-09T09:18:40.251 INFO:tasks.rados.rados.0.smithi163.stdout:3141: finishing rollback tid 2 to smithi16367185-18 2023-10-09T09:18:40.251 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1025 (ObjNum 1128 snap 376 seq_num 1128) dirty exists 2023-10-09T09:18:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:40 smithi181 bash[59345]: cluster 2023-10-09T09:18:39.298461+0000 osd.5 (osd.5) 380 : cluster 3 failed to encode map e845 with expected crc 2023-10-09T09:18:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:40 smithi181 bash[59345]: cluster 2023-10-09T09:18:39.301420+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e845: 8 total, 8 up, 8 in 2023-10-09T09:18:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:40 smithi181 bash[59345]: cluster 2023-10-09T09:18:39.304945+0000 osd.4 (osd.4) 355 : cluster 3 failed to encode map e845 with expected crc 2023-10-09T09:18:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:40 smithi181 bash[59345]: cluster 2023-10-09T09:18:39.419295+0000 mgr.y (mgr.34107) 363 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:18:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:40 smithi181 bash[59345]: cluster 2023-10-09T09:18:40.241118+0000 mon.a (mon.0) 1041 : cluster 0 osdmap e846: 8 total, 8 up, 8 in 2023-10-09T09:18:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:40 smithi163 bash[78107]: cluster 2023-10-09T09:18:39.298461+0000 osd.5 (osd.5) 380 : cluster 3 failed to encode map e845 with expected crc 2023-10-09T09:18:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:40 smithi163 bash[78107]: cluster 2023-10-09T09:18:39.301420+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e845: 8 total, 8 up, 8 in 2023-10-09T09:18:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:40 smithi163 bash[78107]: cluster 2023-10-09T09:18:39.304945+0000 osd.4 (osd.4) 355 : cluster 3 failed to encode map e845 with expected crc 2023-10-09T09:18:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:40 smithi163 bash[78107]: cluster 2023-10-09T09:18:39.419295+0000 mgr.y (mgr.34107) 363 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:18:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:40 smithi163 bash[78107]: cluster 2023-10-09T09:18:40.241118+0000 mon.a (mon.0) 1041 : cluster 0 osdmap e846: 8 total, 8 up, 8 in 2023-10-09T09:18:40.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:40 smithi163 bash[75425]: cluster 2023-10-09T09:18:39.298461+0000 osd.5 (osd.5) 380 : cluster 3 failed to encode map e845 with expected crc 2023-10-09T09:18:40.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:40 smithi163 bash[75425]: cluster 2023-10-09T09:18:39.301420+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e845: 8 total, 8 up, 8 in 2023-10-09T09:18:40.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:40 smithi163 bash[75425]: cluster 2023-10-09T09:18:39.304945+0000 osd.4 (osd.4) 355 : cluster 3 failed to encode map e845 with expected crc 2023-10-09T09:18:40.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:40 smithi163 bash[75425]: cluster 2023-10-09T09:18:39.419295+0000 mgr.y (mgr.34107) 363 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:18:40.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:40 smithi163 bash[75425]: cluster 2023-10-09T09:18:40.241118+0000 mon.a (mon.0) 1041 : cluster 0 osdmap e846: 8 total, 8 up, 8 in 2023-10-09T09:18:41.243 INFO:tasks.rados.rados.0.smithi163.stdout:3141: done (2 left) 2023-10-09T09:18:41.243 INFO:tasks.rados.rados.0.smithi163.stdout:3142: done (1 left) 2023-10-09T09:18:41.243 INFO:tasks.rados.rados.0.smithi163.stdout:3143: done (0 left) 2023-10-09T09:18:41.243 INFO:tasks.rados.rados.0.smithi163.stdout:3144: read oid 26 snap -1 2023-10-09T09:18:41.244 INFO:tasks.rados.rados.0.smithi163.stdout:3144: expect (ObjNum 1077 snap 361 seq_num 1077) 2023-10-09T09:18:41.244 INFO:tasks.rados.rados.0.smithi163.stdout:3145: snap_remove snap 391 2023-10-09T09:18:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:41 smithi181 bash[59345]: cluster 2023-10-09T09:18:40.236684+0000 osd.5 (osd.5) 381 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:41 smithi181 bash[59345]: cluster 2023-10-09T09:18:40.237151+0000 osd.4 (osd.4) 356 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:41 smithi181 bash[59345]: audit 2023-10-09T09:18:40.348408+0000 mon.a (mon.0) 1042 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:41 smithi181 bash[59345]: cluster 2023-10-09T09:18:40.477321+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:41 smithi181 bash[59345]: cluster 2023-10-09T09:18:41.242998+0000 mon.a (mon.0) 1043 : cluster 0 osdmap e847: 8 total, 8 up, 8 in 2023-10-09T09:18:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:41 smithi163 bash[78107]: cluster 2023-10-09T09:18:40.236684+0000 osd.5 (osd.5) 381 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:41 smithi163 bash[78107]: cluster 2023-10-09T09:18:40.237151+0000 osd.4 (osd.4) 356 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:41 smithi163 bash[78107]: audit 2023-10-09T09:18:40.348408+0000 mon.a (mon.0) 1042 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:41 smithi163 bash[78107]: cluster 2023-10-09T09:18:40.477321+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:41 smithi163 bash[78107]: cluster 2023-10-09T09:18:41.242998+0000 mon.a (mon.0) 1043 : cluster 0 osdmap e847: 8 total, 8 up, 8 in 2023-10-09T09:18:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:41 smithi163 bash[75425]: cluster 2023-10-09T09:18:40.236684+0000 osd.5 (osd.5) 381 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:41 smithi163 bash[75425]: cluster 2023-10-09T09:18:40.237151+0000 osd.4 (osd.4) 356 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:41 smithi163 bash[75425]: audit 2023-10-09T09:18:40.348408+0000 mon.a (mon.0) 1042 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:41 smithi163 bash[75425]: cluster 2023-10-09T09:18:40.477321+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:41 smithi163 bash[75425]: cluster 2023-10-09T09:18:41.242998+0000 mon.a (mon.0) 1043 : cluster 0 osdmap e847: 8 total, 8 up, 8 in 2023-10-09T09:18:42.247 INFO:tasks.rados.rados.0.smithi163.stdout:3144: expect (ObjNum 1077 snap 361 seq_num 1077) 2023-10-09T09:18:42.522 INFO:tasks.rados.rados.0.smithi163.stdout:3144: done (1 left) 2023-10-09T09:18:42.522 INFO:tasks.rados.rados.0.smithi163.stdout:3145: done (0 left) 2023-10-09T09:18:42.522 INFO:tasks.rados.rados.0.smithi163.stdout:3146: write oid 1 current snap is 397 2023-10-09T09:18:42.523 INFO:tasks.rados.rados.0.smithi163.stdout:3146: seq_num 1174 ranges {531418=660688,1982919=452406,3039770=691648} 2023-10-09T09:18:42.536 INFO:tasks.rados.rados.0.smithi163.stdout:3146: writing smithi16367185-1 from 531418 to 1192106 tid 1 2023-10-09T09:18:42.543 INFO:tasks.rados.rados.0.smithi163.stdout:3146: writing smithi16367185-1 from 1982919 to 2435325 tid 2 2023-10-09T09:18:42.551 INFO:tasks.rados.rados.0.smithi163.stdout:3146: writing smithi16367185-1 from 3039770 to 3731418 tid 3 2023-10-09T09:18:42.551 INFO:tasks.rados.rados.0.smithi163.stdout:3147: snap_remove snap 390 2023-10-09T09:18:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:42 smithi163 bash[78107]: cluster 2023-10-09T09:18:40.640189+0000 osd.7 (osd.7) 325 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:42 smithi163 bash[78107]: cluster 2023-10-09T09:18:41.238564+0000 osd.5 (osd.5) 382 : cluster 3 failed to encode map e847 with expected crc 2023-10-09T09:18:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:42 smithi163 bash[78107]: cluster 2023-10-09T09:18:41.420233+0000 mgr.y (mgr.34107) 364 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 938 KiB/s wr, 11 op/s 2023-10-09T09:18:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:42 smithi163 bash[78107]: cluster 2023-10-09T09:18:42.246026+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e848: 8 total, 8 up, 8 in 2023-10-09T09:18:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:42 smithi163 bash[75425]: cluster 2023-10-09T09:18:40.640189+0000 osd.7 (osd.7) 325 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:42 smithi163 bash[75425]: cluster 2023-10-09T09:18:41.238564+0000 osd.5 (osd.5) 382 : cluster 3 failed to encode map e847 with expected crc 2023-10-09T09:18:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:42 smithi163 bash[75425]: cluster 2023-10-09T09:18:41.420233+0000 mgr.y (mgr.34107) 364 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 938 KiB/s wr, 11 op/s 2023-10-09T09:18:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:42 smithi163 bash[75425]: cluster 2023-10-09T09:18:42.246026+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e848: 8 total, 8 up, 8 in 2023-10-09T09:18:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:42 smithi181 bash[59345]: cluster 2023-10-09T09:18:40.640189+0000 osd.7 (osd.7) 325 : cluster 3 failed to encode map e846 with expected crc 2023-10-09T09:18:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:42 smithi181 bash[59345]: cluster 2023-10-09T09:18:41.238564+0000 osd.5 (osd.5) 382 : cluster 3 failed to encode map e847 with expected crc 2023-10-09T09:18:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:42 smithi181 bash[59345]: cluster 2023-10-09T09:18:41.420233+0000 mgr.y (mgr.34107) 364 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 938 KiB/s wr, 11 op/s 2023-10-09T09:18:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:42 smithi181 bash[59345]: cluster 2023-10-09T09:18:42.246026+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e848: 8 total, 8 up, 8 in 2023-10-09T09:18:43.338 INFO:tasks.rados.rados.0.smithi163.stdout:3147: done (1 left) 2023-10-09T09:18:43.338 INFO:tasks.rados.rados.0.smithi163.stdout:3148: snap_remove snap 394 2023-10-09T09:18:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:18:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:18:43] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.33.4" 2023-10-09T09:18:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:43 smithi163 bash[75425]: cluster 2023-10-09T09:18:42.241382+0000 osd.5 (osd.5) 383 : cluster 3 failed to encode map e848 with expected crc 2023-10-09T09:18:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:43 smithi163 bash[75425]: cluster 2023-10-09T09:18:42.248194+0000 osd.4 (osd.4) 357 : cluster 3 failed to encode map e847 with expected crc 2023-10-09T09:18:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:43 smithi163 bash[75425]: cluster 2023-10-09T09:18:42.252906+0000 osd.7 (osd.7) 326 : cluster 3 failed to encode map e847 with expected crc 2023-10-09T09:18:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:43 smithi163 bash[78107]: cluster 2023-10-09T09:18:42.241382+0000 osd.5 (osd.5) 383 : cluster 3 failed to encode map e848 with expected crc 2023-10-09T09:18:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:43 smithi163 bash[78107]: cluster 2023-10-09T09:18:42.248194+0000 osd.4 (osd.4) 357 : cluster 3 failed to encode map e847 with expected crc 2023-10-09T09:18:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:43 smithi163 bash[78107]: cluster 2023-10-09T09:18:42.252906+0000 osd.7 (osd.7) 326 : cluster 3 failed to encode map e847 with expected crc 2023-10-09T09:18:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:43 smithi181 bash[59345]: cluster 2023-10-09T09:18:42.241382+0000 osd.5 (osd.5) 383 : cluster 3 failed to encode map e848 with expected crc 2023-10-09T09:18:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:43 smithi181 bash[59345]: cluster 2023-10-09T09:18:42.248194+0000 osd.4 (osd.4) 357 : cluster 3 failed to encode map e847 with expected crc 2023-10-09T09:18:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:43 smithi181 bash[59345]: cluster 2023-10-09T09:18:42.252906+0000 osd.7 (osd.7) 326 : cluster 3 failed to encode map e847 with expected crc 2023-10-09T09:18:44.353 INFO:tasks.rados.rados.0.smithi163.stdout:3148: done (1 left) 2023-10-09T09:18:44.353 INFO:tasks.rados.rados.0.smithi163.stdout:3149: delete oid 30 current snap is 397 2023-10-09T09:18:44.353 INFO:tasks.rados.rados.0.smithi163.stdout:3146: finishing write tid 1 to smithi16367185-1 2023-10-09T09:18:44.353 INFO:tasks.rados.rados.0.smithi163.stdout:3146: finishing write tid 2 to smithi16367185-1 2023-10-09T09:18:44.353 INFO:tasks.rados.rados.0.smithi163.stdout:3146: finishing write tid 3 to smithi16367185-1 2023-10-09T09:18:44.354 INFO:tasks.rados.rados.0.smithi163.stdout:3146: finishing write tid 4 to smithi16367185-1 2023-10-09T09:18:44.354 INFO:tasks.rados.rados.0.smithi163.stdout:3146: finishing write tid 5 to smithi16367185-1 2023-10-09T09:18:44.354 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1523 (ObjNum 1174 snap 397 seq_num 1174) dirty exists 2023-10-09T09:18:44.354 INFO:tasks.rados.rados.0.smithi163.stdout:3146: left oid 1 (ObjNum 1174 snap 397 seq_num 1174) 2023-10-09T09:18:44.360 INFO:tasks.rados.rados.0.smithi163.stdout:3146: done (1 left) 2023-10-09T09:18:44.360 INFO:tasks.rados.rados.0.smithi163.stdout:3149: done (0 left) 2023-10-09T09:18:44.360 INFO:tasks.rados.rados.0.smithi163.stdout:3150: rollback oid 49 current snap is 397 2023-10-09T09:18:44.360 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 49 to 371 2023-10-09T09:18:44.360 INFO:tasks.rados.rados.0.smithi163.stdout:3151: rollback oid 18 current snap is 397 2023-10-09T09:18:44.360 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 18 to 376 2023-10-09T09:18:44.360 INFO:tasks.rados.rados.0.smithi163.stdout:3152: rollback oid 41 current snap is 397 2023-10-09T09:18:44.360 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 41 to 377 2023-10-09T09:18:44.360 INFO:tasks.rados.rados.0.smithi163.stdout:3153: read oid 30 snap -1 2023-10-09T09:18:44.361 INFO:tasks.rados.rados.0.smithi163.stdout:3153: expect deleted 2023-10-09T09:18:44.361 INFO:tasks.rados.rados.0.smithi163.stdout:3154: read oid 25 snap -1 2023-10-09T09:18:44.361 INFO:tasks.rados.rados.0.smithi163.stdout:3154: expect (ObjNum 1162 snap 391 seq_num 1162) 2023-10-09T09:18:44.361 INFO:tasks.rados.rados.0.smithi163.stdout:3155: write oid 37 current snap is 397 2023-10-09T09:18:44.361 INFO:tasks.rados.rados.0.smithi163.stdout:3155: seq_num 1176 ranges {565032=411186,1583530=456075,2578263=758593} 2023-10-09T09:18:44.367 INFO:tasks.rados.rados.0.smithi163.stdout:3155: writing smithi16367185-37 from 565032 to 976218 tid 1 2023-10-09T09:18:44.373 INFO:tasks.rados.rados.0.smithi163.stdout:3155: writing smithi16367185-37 from 1583530 to 2039605 tid 2 2023-10-09T09:18:44.380 INFO:tasks.rados.rados.0.smithi163.stdout:3155: writing smithi16367185-37 from 2578263 to 3336856 tid 3 2023-10-09T09:18:44.381 INFO:tasks.rados.rados.0.smithi163.stdout:3156: write oid 23 current snap is 397 2023-10-09T09:18:44.381 INFO:tasks.rados.rados.0.smithi163.stdout:3156: seq_num 1177 ranges {581839=486435,1642012=599733,3005686=569865} 2023-10-09T09:18:44.387 INFO:tasks.rados.rados.0.smithi163.stdout:3156: writing smithi16367185-23 from 581839 to 1068274 tid 1 2023-10-09T09:18:44.393 INFO:tasks.rados.rados.0.smithi163.stdout:3156: writing smithi16367185-23 from 1642012 to 2241745 tid 2 2023-10-09T09:18:44.401 INFO:tasks.rados.rados.0.smithi163.stdout:3156: writing smithi16367185-23 from 3005686 to 3575551 tid 3 2023-10-09T09:18:44.401 INFO:tasks.rados.rados.0.smithi163.stdout:3157: write oid 7 current snap is 397 2023-10-09T09:18:44.401 INFO:tasks.rados.rados.0.smithi163.stdout:3157: seq_num 1178 ranges {598646=561684,1700494=459744,2633109=781137} 2023-10-09T09:18:44.407 INFO:tasks.rados.rados.0.smithi163.stdout:3157: writing smithi16367185-7 from 598646 to 1160330 tid 1 2023-10-09T09:18:44.413 INFO:tasks.rados.rados.0.smithi163.stdout:3157: writing smithi16367185-7 from 1700494 to 2160238 tid 2 2023-10-09T09:18:44.421 INFO:tasks.rados.rados.0.smithi163.stdout:3157: writing smithi16367185-7 from 2633109 to 3414246 tid 3 2023-10-09T09:18:44.421 INFO:tasks.rados.rados.0.smithi163.stdout:3158: read oid 38 snap 386 2023-10-09T09:18:44.421 INFO:tasks.rados.rados.0.smithi163.stdout:3158: expect (ObjNum 1081 snap 362 seq_num 1081) 2023-10-09T09:18:44.421 INFO:tasks.rados.rados.0.smithi163.stdout:3159: delete oid 2 current snap is 397 2023-10-09T09:18:44.421 INFO:tasks.rados.rados.0.smithi163.stdout:3152: finishing rollback tid 1 to smithi16367185-41 2023-10-09T09:18:44.421 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 283 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:18:44.421 INFO:tasks.rados.rados.0.smithi163.stdout:3151: finishing rollback tid 0 to smithi16367185-18 2023-10-09T09:18:44.421 INFO:tasks.rados.rados.0.smithi163.stdout:3151: finishing rollback tid 1 to smithi16367185-18 2023-10-09T09:18:44.422 INFO:tasks.rados.rados.0.smithi163.stdout:3151: finishing rollback tid 2 to smithi16367185-18 2023-10-09T09:18:44.422 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1038 (ObjNum 1128 snap 376 seq_num 1128) dirty exists 2023-10-09T09:18:44.422 INFO:tasks.rados.rados.0.smithi163.stdout:3150: finishing rollback tid 0 to smithi16367185-49 2023-10-09T09:18:44.422 INFO:tasks.rados.rados.0.smithi163.stdout:3150: finishing rollback tid 1 to smithi16367185-49 2023-10-09T09:18:44.422 INFO:tasks.rados.rados.0.smithi163.stdout:3154: expect (ObjNum 1162 snap 391 seq_num 1162) 2023-10-09T09:18:44.675 INFO:tasks.rados.rados.0.smithi163.stdout:3150: finishing rollback tid 2 to smithi16367185-49 2023-10-09T09:18:44.675 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 394 (ObjNum 1099 snap 365 seq_num 1099) dirty exists 2023-10-09T09:18:44.675 INFO:tasks.rados.rados.0.smithi163.stdout:3155: finishing write tid 1 to smithi16367185-37 2023-10-09T09:18:44.675 INFO:tasks.rados.rados.0.smithi163.stdout:3155: finishing write tid 2 to smithi16367185-37 2023-10-09T09:18:44.675 INFO:tasks.rados.rados.0.smithi163.stdout:3155: finishing write tid 3 to smithi16367185-37 2023-10-09T09:18:44.675 INFO:tasks.rados.rados.0.smithi163.stdout:3155: finishing write tid 4 to smithi16367185-37 2023-10-09T09:18:44.675 INFO:tasks.rados.rados.0.smithi163.stdout:3155: finishing write tid 5 to smithi16367185-37 2023-10-09T09:18:44.676 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 1571 (ObjNum 1176 snap 397 seq_num 1176) dirty exists 2023-10-09T09:18:44.676 INFO:tasks.rados.rados.0.smithi163.stdout:3155: left oid 37 (ObjNum 1176 snap 397 seq_num 1176) 2023-10-09T09:18:44.676 INFO:tasks.rados.rados.0.smithi163.stdout:3156: finishing write tid 1 to smithi16367185-23 2023-10-09T09:18:44.676 INFO:tasks.rados.rados.0.smithi163.stdout:3156: finishing write tid 2 to smithi16367185-23 2023-10-09T09:18:44.676 INFO:tasks.rados.rados.0.smithi163.stdout:3156: finishing write tid 3 to smithi16367185-23 2023-10-09T09:18:44.676 INFO:tasks.rados.rados.0.smithi163.stdout:3156: finishing write tid 4 to smithi16367185-23 2023-10-09T09:18:44.676 INFO:tasks.rados.rados.0.smithi163.stdout:3156: finishing write tid 5 to smithi16367185-23 2023-10-09T09:18:44.676 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1448 (ObjNum 1177 snap 397 seq_num 1177) dirty exists 2023-10-09T09:18:44.676 INFO:tasks.rados.rados.0.smithi163.stdout:3156: left oid 23 (ObjNum 1177 snap 397 seq_num 1177) 2023-10-09T09:18:44.676 INFO:tasks.rados.rados.0.smithi163.stdout:3150: done (9 left) 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3151: done (8 left) 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3152: done (7 left) 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3153: done (6 left) 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3154: done (5 left) 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3155: done (4 left) 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3156: done (3 left) 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3159: done (2 left) 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3160: write oid 5 current snap is 397 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3157: finishing write tid 1 to smithi16367185-7 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3157: finishing write tid 2 to smithi16367185-7 2023-10-09T09:18:44.677 INFO:tasks.rados.rados.0.smithi163.stdout:3160: seq_num 1180 ranges {632260=428535,1817458=463413,2687955=520034,3709324=122936} 2023-10-09T09:18:44.685 INFO:tasks.rados.rados.0.smithi163.stdout:3160: writing smithi16367185-5 from 632260 to 1060795 tid 1 2023-10-09T09:18:44.692 INFO:tasks.rados.rados.0.smithi163.stdout:3160: writing smithi16367185-5 from 1817458 to 2280871 tid 2 2023-10-09T09:18:44.698 INFO:tasks.rados.rados.0.smithi163.stdout:3160: writing smithi16367185-5 from 2687955 to 3207989 tid 3 2023-10-09T09:18:44.702 INFO:tasks.rados.rados.0.smithi163.stdout:3160: writing smithi16367185-5 from 3709324 to 3832260 tid 4 2023-10-09T09:18:44.702 INFO:tasks.rados.rados.0.smithi163.stdout:3161: read oid 35 snap 395 2023-10-09T09:18:44.702 INFO:tasks.rados.rados.0.smithi163.stdout:3161: expect deleted 2023-10-09T09:18:44.702 INFO:tasks.rados.rados.0.smithi163.stdout:3162: delete oid 18 current snap is 397 2023-10-09T09:18:44.702 INFO:tasks.rados.rados.0.smithi163.stdout:3157: finishing write tid 3 to smithi16367185-7 2023-10-09T09:18:44.702 INFO:tasks.rados.rados.0.smithi163.stdout:3157: finishing write tid 4 to smithi16367185-7 2023-10-09T09:18:44.703 INFO:tasks.rados.rados.0.smithi163.stdout:3157: finishing write tid 5 to smithi16367185-7 2023-10-09T09:18:44.703 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 775 (ObjNum 1178 snap 397 seq_num 1178) dirty exists 2023-10-09T09:18:44.703 INFO:tasks.rados.rados.0.smithi163.stdout:3157: left oid 7 (ObjNum 1178 snap 397 seq_num 1178) 2023-10-09T09:18:44.703 INFO:tasks.rados.rados.0.smithi163.stdout:3158: expect (ObjNum 1081 snap 362 seq_num 1081) 2023-10-09T09:18:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:44 smithi163 bash[78107]: cluster 2023-10-09T09:18:43.335198+0000 osd.5 (osd.5) 384 : cluster 3 failed to encode map e849 with expected crc 2023-10-09T09:18:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:44 smithi163 bash[78107]: cluster 2023-10-09T09:18:43.337852+0000 mon.a (mon.0) 1045 : cluster 0 osdmap e849: 8 total, 8 up, 8 in 2023-10-09T09:18:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:44 smithi163 bash[78107]: cluster 2023-10-09T09:18:43.338019+0000 osd.4 (osd.4) 358 : cluster 3 failed to encode map e849 with expected crc 2023-10-09T09:18:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:44 smithi163 bash[78107]: cluster 2023-10-09T09:18:43.421076+0000 mgr.y (mgr.34107) 365 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 938 KiB/s wr, 11 op/s 2023-10-09T09:18:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:44 smithi163 bash[75425]: cluster 2023-10-09T09:18:43.335198+0000 osd.5 (osd.5) 384 : cluster 3 failed to encode map e849 with expected crc 2023-10-09T09:18:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:44 smithi163 bash[75425]: cluster 2023-10-09T09:18:43.337852+0000 mon.a (mon.0) 1045 : cluster 0 osdmap e849: 8 total, 8 up, 8 in 2023-10-09T09:18:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:44 smithi163 bash[75425]: cluster 2023-10-09T09:18:43.338019+0000 osd.4 (osd.4) 358 : cluster 3 failed to encode map e849 with expected crc 2023-10-09T09:18:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:44 smithi163 bash[75425]: cluster 2023-10-09T09:18:43.421076+0000 mgr.y (mgr.34107) 365 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 938 KiB/s wr, 11 op/s 2023-10-09T09:18:44.809 INFO:tasks.rados.rados.0.smithi163.stdout:3160: finishing write tid 1 to smithi16367185-5 2023-10-09T09:18:44.809 INFO:tasks.rados.rados.0.smithi163.stdout:3160: finishing write tid 2 to smithi16367185-5 2023-10-09T09:18:44.809 INFO:tasks.rados.rados.0.smithi163.stdout:3160: finishing write tid 3 to smithi16367185-5 2023-10-09T09:18:44.810 INFO:tasks.rados.rados.0.smithi163.stdout:3160: finishing write tid 4 to smithi16367185-5 2023-10-09T09:18:44.810 INFO:tasks.rados.rados.0.smithi163.stdout:3160: finishing write tid 5 to smithi16367185-5 2023-10-09T09:18:44.810 INFO:tasks.rados.rados.0.smithi163.stdout:3160: finishing write tid 6 to smithi16367185-5 2023-10-09T09:18:44.810 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1539 (ObjNum 1180 snap 397 seq_num 1180) dirty exists 2023-10-09T09:18:44.810 INFO:tasks.rados.rados.0.smithi163.stdout:3160: left oid 5 (ObjNum 1180 snap 397 seq_num 1180) 2023-10-09T09:18:44.810 INFO:tasks.rados.rados.0.smithi163.stdout:3157: done (4 left) 2023-10-09T09:18:44.810 INFO:tasks.rados.rados.0.smithi163.stdout:3158: done (3 left) 2023-10-09T09:18:44.810 INFO:tasks.rados.rados.0.smithi163.stdout:3160: done (2 left) 2023-10-09T09:18:44.811 INFO:tasks.rados.rados.0.smithi163.stdout:3161: done (1 left) 2023-10-09T09:18:44.811 INFO:tasks.rados.rados.0.smithi163.stdout:3162: done (0 left) 2023-10-09T09:18:44.811 INFO:tasks.rados.rados.0.smithi163.stdout:3163: read oid 7 snap -1 2023-10-09T09:18:44.811 INFO:tasks.rados.rados.0.smithi163.stdout:3163: expect (ObjNum 1178 snap 397 seq_num 1178) 2023-10-09T09:18:44.811 INFO:tasks.rados.rados.0.smithi163.stdout:3164: write oid 4 current snap is 397 2023-10-09T09:18:44.811 INFO:tasks.rados.rados.0.smithi163.stdout:3164: seq_num 1182 ranges {665874=579033,1934422=467082,3142801=542578} 2023-10-09T09:18:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:44 smithi181 bash[59345]: cluster 2023-10-09T09:18:43.335198+0000 osd.5 (osd.5) 384 : cluster 3 failed to encode map e849 with expected crc 2023-10-09T09:18:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:44 smithi181 bash[59345]: cluster 2023-10-09T09:18:43.337852+0000 mon.a (mon.0) 1045 : cluster 0 osdmap e849: 8 total, 8 up, 8 in 2023-10-09T09:18:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:44 smithi181 bash[59345]: cluster 2023-10-09T09:18:43.338019+0000 osd.4 (osd.4) 358 : cluster 3 failed to encode map e849 with expected crc 2023-10-09T09:18:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:44 smithi181 bash[59345]: cluster 2023-10-09T09:18:43.421076+0000 mgr.y (mgr.34107) 365 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 938 KiB/s wr, 11 op/s 2023-10-09T09:18:44.822 INFO:tasks.rados.rados.0.smithi163.stdout:3164: writing smithi16367185-4 from 665874 to 1244907 tid 1 2023-10-09T09:18:44.828 INFO:tasks.rados.rados.0.smithi163.stdout:3164: writing smithi16367185-4 from 1934422 to 2401504 tid 2 2023-10-09T09:18:44.836 INFO:tasks.rados.rados.0.smithi163.stdout:3164: writing smithi16367185-4 from 3142801 to 3685379 tid 3 2023-10-09T09:18:44.836 INFO:tasks.rados.rados.0.smithi163.stdout:3165: write oid 39 current snap is 397 2023-10-09T09:18:44.836 INFO:tasks.rados.rados.0.smithi163.stdout:3165: seq_num 1183 ranges {682681=654282,1876551=610740,3053871=470203} 2023-10-09T09:18:44.843 INFO:tasks.rados.rados.0.smithi163.stdout:3165: writing smithi16367185-39 from 682681 to 1336963 tid 1 2023-10-09T09:18:44.850 INFO:tasks.rados.rados.0.smithi163.stdout:3165: writing smithi16367185-39 from 1876551 to 2487291 tid 2 2023-10-09T09:18:44.856 INFO:tasks.rados.rados.0.smithi163.stdout:3165: writing smithi16367185-39 from 3053871 to 3524074 tid 3 2023-10-09T09:18:44.856 INFO:tasks.rados.rados.0.smithi163.stdout:3166: read oid 26 snap -1 2023-10-09T09:18:44.856 INFO:tasks.rados.rados.0.smithi163.stdout:3166: expect (ObjNum 1077 snap 361 seq_num 1077) 2023-10-09T09:18:44.856 INFO:tasks.rados.rados.0.smithi163.stdout:3167: write oid 2 current snap is 397 2023-10-09T09:18:44.856 INFO:tasks.rados.rados.0.smithi163.stdout:3167: seq_num 1184 ranges {699488=729531,1935033=470751,3081294=681475} 2023-10-09T09:18:44.864 INFO:tasks.rados.rados.0.smithi163.stdout:3167: writing smithi16367185-2 from 699488 to 1429019 tid 1 2023-10-09T09:18:44.869 INFO:tasks.rados.rados.0.smithi163.stdout:3167: writing smithi16367185-2 from 1935033 to 2405784 tid 2 2023-10-09T09:18:44.877 INFO:tasks.rados.rados.0.smithi163.stdout:3167: writing smithi16367185-2 from 3081294 to 3762769 tid 3 2023-10-09T09:18:44.877 INFO:tasks.rados.rados.0.smithi163.stdout:3168: read oid 18 snap 393 2023-10-09T09:18:44.877 INFO:tasks.rados.rados.0.smithi163.stdout:3168: expect (ObjNum 1159 snap 391 seq_num 1159) 2023-10-09T09:18:44.877 INFO:tasks.rados.rados.0.smithi163.stdout:3169: snap_create 2023-10-09T09:18:44.877 INFO:tasks.rados.rados.0.smithi163.stdout:3163: expect (ObjNum 1178 snap 397 seq_num 1178) 2023-10-09T09:18:45.170 INFO:tasks.rados.rados.0.smithi163.stdout:3164: finishing write tid 1 to smithi16367185-4 2023-10-09T09:18:45.170 INFO:tasks.rados.rados.0.smithi163.stdout:3164: finishing write tid 2 to smithi16367185-4 2023-10-09T09:18:45.171 INFO:tasks.rados.rados.0.smithi163.stdout:3164: finishing write tid 3 to smithi16367185-4 2023-10-09T09:18:45.171 INFO:tasks.rados.rados.0.smithi163.stdout:3164: finishing write tid 4 to smithi16367185-4 2023-10-09T09:18:45.171 INFO:tasks.rados.rados.0.smithi163.stdout:3164: finishing write tid 5 to smithi16367185-4 2023-10-09T09:18:45.171 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1543 (ObjNum 1182 snap 397 seq_num 1182) dirty exists 2023-10-09T09:18:45.171 INFO:tasks.rados.rados.0.smithi163.stdout:3164: left oid 4 (ObjNum 1182 snap 397 seq_num 1182) 2023-10-09T09:18:45.171 INFO:tasks.rados.rados.0.smithi163.stdout:3165: finishing write tid 1 to smithi16367185-39 2023-10-09T09:18:45.171 INFO:tasks.rados.rados.0.smithi163.stdout:3165: finishing write tid 2 to smithi16367185-39 2023-10-09T09:18:45.171 INFO:tasks.rados.rados.0.smithi163.stdout:3166: expect (ObjNum 1077 snap 361 seq_num 1077) 2023-10-09T09:18:45.426 INFO:tasks.rados.rados.0.smithi163.stdout:3165: finishing write tid 3 to smithi16367185-39 2023-10-09T09:18:45.426 INFO:tasks.rados.rados.0.smithi163.stdout:3165: finishing write tid 4 to smithi16367185-39 2023-10-09T09:18:45.426 INFO:tasks.rados.rados.0.smithi163.stdout:3165: finishing write tid 5 to smithi16367185-39 2023-10-09T09:18:45.426 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1453 (ObjNum 1183 snap 397 seq_num 1183) dirty exists 2023-10-09T09:18:45.426 INFO:tasks.rados.rados.0.smithi163.stdout:3165: left oid 39 (ObjNum 1183 snap 397 seq_num 1183) 2023-10-09T09:18:45.426 INFO:tasks.rados.rados.0.smithi163.stdout:3167: finishing write tid 1 to smithi16367185-2 2023-10-09T09:18:45.426 INFO:tasks.rados.rados.0.smithi163.stdout:3167: finishing write tid 2 to smithi16367185-2 2023-10-09T09:18:45.426 INFO:tasks.rados.rados.0.smithi163.stdout:3167: finishing write tid 3 to smithi16367185-2 2023-10-09T09:18:45.426 INFO:tasks.rados.rados.0.smithi163.stdout:3167: finishing write tid 4 to smithi16367185-2 2023-10-09T09:18:45.427 INFO:tasks.rados.rados.0.smithi163.stdout:3167: finishing write tid 5 to smithi16367185-2 2023-10-09T09:18:45.427 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 851 (ObjNum 1184 snap 397 seq_num 1184) dirty exists 2023-10-09T09:18:45.427 INFO:tasks.rados.rados.0.smithi163.stdout:3167: left oid 2 (ObjNum 1184 snap 397 seq_num 1184) 2023-10-09T09:18:45.427 INFO:tasks.rados.rados.0.smithi163.stdout:3168: expect (ObjNum 1159 snap 391 seq_num 1159) 2023-10-09T09:18:45.615 INFO:tasks.rados.rados.0.smithi163.stdout:3163: done (6 left) 2023-10-09T09:18:45.616 INFO:tasks.rados.rados.0.smithi163.stdout:3164: done (5 left) 2023-10-09T09:18:45.616 INFO:tasks.rados.rados.0.smithi163.stdout:3165: done (4 left) 2023-10-09T09:18:45.616 INFO:tasks.rados.rados.0.smithi163.stdout:3166: done (3 left) 2023-10-09T09:18:45.616 INFO:tasks.rados.rados.0.smithi163.stdout:3167: done (2 left) 2023-10-09T09:18:45.616 INFO:tasks.rados.rados.0.smithi163.stdout:3168: done (1 left) 2023-10-09T09:18:45.616 INFO:tasks.rados.rados.0.smithi163.stdout:3169: done (0 left) 2023-10-09T09:18:45.616 INFO:tasks.rados.rados.0.smithi163.stdout:3170: write oid 1 current snap is 398 2023-10-09T09:18:45.616 INFO:tasks.rados.rados.0.smithi163.stdout:3170: seq_num 1185 ranges {716295=404780,1593515=614409,2708717=492747,3640519=275776} 2023-10-09T09:18:45.628 INFO:tasks.rados.rados.0.smithi163.stdout:3170: writing smithi16367185-1 from 716295 to 1121075 tid 1 2023-10-09T09:18:45.638 INFO:tasks.rados.rados.0.smithi163.stdout:3170: writing smithi16367185-1 from 1593515 to 2207924 tid 2 2023-10-09T09:18:45.641 INFO:tasks.rados.rados.0.smithi163.stdout:3170: writing smithi16367185-1 from 2708717 to 3201464 tid 3 2023-10-09T09:18:45.645 INFO:tasks.rados.rados.0.smithi163.stdout:3170: writing smithi16367185-1 from 3640519 to 3916295 tid 4 2023-10-09T09:18:45.646 INFO:tasks.rados.rados.0.smithi163.stdout:3171: write oid 38 current snap is 398 2023-10-09T09:18:45.646 INFO:tasks.rados.rados.0.smithi163.stdout:3170: finishing write tid 1 to smithi16367185-1 2023-10-09T09:18:45.646 INFO:tasks.rados.rados.0.smithi163.stdout:3170: finishing write tid 2 to smithi16367185-1 2023-10-09T09:18:45.646 INFO:tasks.rados.rados.0.smithi163.stdout:3170: finishing write tid 3 to smithi16367185-1 2023-10-09T09:18:45.646 INFO:tasks.rados.rados.0.smithi163.stdout:3171: seq_num 1186 ranges {733102=596382,1768350=758067,3136140=704019} 2023-10-09T09:18:45.653 INFO:tasks.rados.rados.0.smithi163.stdout:3171: writing smithi16367185-38 from 733102 to 1329484 tid 1 2023-10-09T09:18:45.660 INFO:tasks.rados.rados.0.smithi163.stdout:3171: writing smithi16367185-38 from 1768350 to 2526417 tid 2 2023-10-09T09:18:45.668 INFO:tasks.rados.rados.0.smithi163.stdout:3171: writing smithi16367185-38 from 3136140 to 3840159 tid 3 2023-10-09T09:18:45.668 INFO:tasks.rados.rados.0.smithi163.stdout:3172: read oid 48 snap -1 2023-10-09T09:18:45.668 INFO:tasks.rados.rados.0.smithi163.stdout:3172: expect (ObjNum 1161 snap 391 seq_num 1161) 2023-10-09T09:18:45.668 INFO:tasks.rados.rados.0.smithi163.stdout:3173: write oid 13 current snap is 398 2023-10-09T09:18:45.668 INFO:tasks.rados.rados.0.smithi163.stdout:3170: finishing write tid 4 to smithi16367185-1 2023-10-09T09:18:45.669 INFO:tasks.rados.rados.0.smithi163.stdout:3170: finishing write tid 5 to smithi16367185-1 2023-10-09T09:18:45.669 INFO:tasks.rados.rados.0.smithi163.stdout:3170: finishing write tid 6 to smithi16367185-1 2023-10-09T09:18:45.669 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1549 (ObjNum 1185 snap 398 seq_num 1185) dirty exists 2023-10-09T09:18:45.669 INFO:tasks.rados.rados.0.smithi163.stdout:3170: left oid 1 (ObjNum 1185 snap 398 seq_num 1185) 2023-10-09T09:18:45.669 INFO:tasks.rados.rados.0.smithi163.stdout:3171: finishing write tid 1 to smithi16367185-38 2023-10-09T09:18:45.669 INFO:tasks.rados.rados.0.smithi163.stdout:3171: finishing write tid 2 to smithi16367185-38 2023-10-09T09:18:45.669 INFO:tasks.rados.rados.0.smithi163.stdout:3173: seq_num 1187 ranges {749909=671631,2110479=618078,3163563=515291} 2023-10-09T09:18:45.679 INFO:tasks.rados.rados.0.smithi163.stdout:3173: writing smithi16367185-13 from 749909 to 1421540 tid 1 2023-10-09T09:18:45.687 INFO:tasks.rados.rados.0.smithi163.stdout:3173: writing smithi16367185-13 from 2110479 to 2728557 tid 2 2023-10-09T09:18:45.693 INFO:tasks.rados.rados.0.smithi163.stdout:3173: writing smithi16367185-13 from 3163563 to 3678854 tid 3 2023-10-09T09:18:45.693 INFO:tasks.rados.rados.0.smithi163.stdout:3170: done (3 left) 2023-10-09T09:18:45.693 INFO:tasks.rados.rados.0.smithi163.stdout:3174: read oid 4 snap -1 2023-10-09T09:18:45.693 INFO:tasks.rados.rados.0.smithi163.stdout:3174: expect (ObjNum 1182 snap 397 seq_num 1182) 2023-10-09T09:18:45.693 INFO:tasks.rados.rados.0.smithi163.stdout:3175: snap_create 2023-10-09T09:18:45.693 INFO:tasks.rados.rados.0.smithi163.stdout:3171: finishing write tid 3 to smithi16367185-38 2023-10-09T09:18:45.693 INFO:tasks.rados.rados.0.smithi163.stdout:3171: finishing write tid 4 to smithi16367185-38 2023-10-09T09:18:45.693 INFO:tasks.rados.rados.0.smithi163.stdout:3171: finishing write tid 5 to smithi16367185-38 2023-10-09T09:18:45.694 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1493 (ObjNum 1186 snap 398 seq_num 1186) dirty exists 2023-10-09T09:18:45.694 INFO:tasks.rados.rados.0.smithi163.stdout:3171: left oid 38 (ObjNum 1186 snap 398 seq_num 1186) 2023-10-09T09:18:45.694 INFO:tasks.rados.rados.0.smithi163.stdout:3172: expect (ObjNum 1161 snap 391 seq_num 1161) 2023-10-09T09:18:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[75425]: cluster 2023-10-09T09:18:44.347587+0000 osd.5 (osd.5) 385 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[75425]: cluster 2023-10-09T09:18:44.353333+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e850: 8 total, 8 up, 8 in 2023-10-09T09:18:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[75425]: cluster 2023-10-09T09:18:44.357184+0000 osd.4 (osd.4) 359 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[75425]: cluster 2023-10-09T09:18:44.361486+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[75425]: cluster 2023-10-09T09:18:44.362609+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[75425]: cluster 2023-10-09T09:18:45.241073+0000 mon.a (mon.0) 1047 : cluster 0 osdmap e851: 8 total, 8 up, 8 in 2023-10-09T09:18:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[78107]: cluster 2023-10-09T09:18:44.347587+0000 osd.5 (osd.5) 385 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[78107]: cluster 2023-10-09T09:18:44.353333+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e850: 8 total, 8 up, 8 in 2023-10-09T09:18:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[78107]: cluster 2023-10-09T09:18:44.357184+0000 osd.4 (osd.4) 359 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[78107]: cluster 2023-10-09T09:18:44.361486+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[78107]: cluster 2023-10-09T09:18:44.362609+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:45 smithi163 bash[78107]: cluster 2023-10-09T09:18:45.241073+0000 mon.a (mon.0) 1047 : cluster 0 osdmap e851: 8 total, 8 up, 8 in 2023-10-09T09:18:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:45 smithi181 bash[59345]: cluster 2023-10-09T09:18:44.347587+0000 osd.5 (osd.5) 385 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:45 smithi181 bash[59345]: cluster 2023-10-09T09:18:44.353333+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e850: 8 total, 8 up, 8 in 2023-10-09T09:18:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:45 smithi181 bash[59345]: cluster 2023-10-09T09:18:44.357184+0000 osd.4 (osd.4) 359 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:45 smithi181 bash[59345]: cluster 2023-10-09T09:18:44.361486+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:45 smithi181 bash[59345]: cluster 2023-10-09T09:18:44.362609+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e850 with expected crc 2023-10-09T09:18:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:45 smithi181 bash[59345]: cluster 2023-10-09T09:18:45.241073+0000 mon.a (mon.0) 1047 : cluster 0 osdmap e851: 8 total, 8 up, 8 in 2023-10-09T09:18:45.923 INFO:tasks.rados.rados.0.smithi163.stdout:3173: finishing write tid 1 to smithi16367185-13 2023-10-09T09:18:45.924 INFO:tasks.rados.rados.0.smithi163.stdout:3173: finishing write tid 2 to smithi16367185-13 2023-10-09T09:18:45.924 INFO:tasks.rados.rados.0.smithi163.stdout:3173: finishing write tid 3 to smithi16367185-13 2023-10-09T09:18:45.924 INFO:tasks.rados.rados.0.smithi163.stdout:3173: finishing write tid 4 to smithi16367185-13 2023-10-09T09:18:45.924 INFO:tasks.rados.rados.0.smithi163.stdout:3173: finishing write tid 5 to smithi16367185-13 2023-10-09T09:18:45.924 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 575 (ObjNum 1187 snap 398 seq_num 1187) dirty exists 2023-10-09T09:18:45.924 INFO:tasks.rados.rados.0.smithi163.stdout:3173: left oid 13 (ObjNum 1187 snap 398 seq_num 1187) 2023-10-09T09:18:45.924 INFO:tasks.rados.rados.0.smithi163.stdout:3174: expect (ObjNum 1182 snap 397 seq_num 1182) 2023-10-09T09:18:46.367 INFO:tasks.rados.rados.0.smithi163.stdout:3171: done (4 left) 2023-10-09T09:18:46.367 INFO:tasks.rados.rados.0.smithi163.stdout:3172: done (3 left) 2023-10-09T09:18:46.367 INFO:tasks.rados.rados.0.smithi163.stdout:3173: done (2 left) 2023-10-09T09:18:46.367 INFO:tasks.rados.rados.0.smithi163.stdout:3174: done (1 left) 2023-10-09T09:18:46.367 INFO:tasks.rados.rados.0.smithi163.stdout:3175: done (0 left) 2023-10-09T09:18:46.367 INFO:tasks.rados.rados.0.smithi163.stdout:3176: write oid 19 current snap is 399 2023-10-09T09:18:46.368 INFO:tasks.rados.rados.0.smithi163.stdout:3176: seq_num 1188 ranges {766716=746880,2168961=761736,3474633=442916} 2023-10-09T09:18:46.378 INFO:tasks.rados.rados.0.smithi163.stdout:3176: writing smithi16367185-19 from 766716 to 1513596 tid 1 2023-10-09T09:18:46.386 INFO:tasks.rados.rados.0.smithi163.stdout:3176: writing smithi16367185-19 from 2168961 to 2930697 tid 2 2023-10-09T09:18:46.392 INFO:tasks.rados.rados.0.smithi163.stdout:3176: writing smithi16367185-19 from 3474633 to 3917549 tid 3 2023-10-09T09:18:46.392 INFO:tasks.rados.rados.0.smithi163.stdout:3177: rollback oid 2 current snap is 399 2023-10-09T09:18:46.392 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 2 to 398 2023-10-09T09:18:46.392 INFO:tasks.rados.rados.0.smithi163.stdout:3176: finishing write tid 1 to smithi16367185-19 2023-10-09T09:18:46.392 INFO:tasks.rados.rados.0.smithi163.stdout:3178: delete oid 15 current snap is 399 2023-10-09T09:18:46.394 INFO:tasks.rados.rados.0.smithi163.stdout:3176: finishing write tid 2 to smithi16367185-19 2023-10-09T09:18:46.397 INFO:tasks.rados.rados.0.smithi163.stdout:3177: finishing rollback tid 0 to smithi16367185-2 2023-10-09T09:18:46.398 INFO:tasks.rados.rados.0.smithi163.stdout:3176: finishing write tid 3 to smithi16367185-19 2023-10-09T09:18:46.398 INFO:tasks.rados.rados.0.smithi163.stdout:3176: finishing write tid 4 to smithi16367185-19 2023-10-09T09:18:46.399 INFO:tasks.rados.rados.0.smithi163.stdout:3178: done (2 left) 2023-10-09T09:18:46.399 INFO:tasks.rados.rados.0.smithi163.stdout:3179: delete oid 8 current snap is 399 2023-10-09T09:18:46.399 INFO:tasks.rados.rados.0.smithi163.stdout:3176: finishing write tid 5 to smithi16367185-19 2023-10-09T09:18:46.399 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 930 (ObjNum 1188 snap 399 seq_num 1188) dirty exists 2023-10-09T09:18:46.399 INFO:tasks.rados.rados.0.smithi163.stdout:3176: left oid 19 (ObjNum 1188 snap 399 seq_num 1188) 2023-10-09T09:18:46.399 INFO:tasks.rados.rados.0.smithi163.stdout:3177: finishing rollback tid 1 to smithi16367185-2 2023-10-09T09:18:46.400 INFO:tasks.rados.rados.0.smithi163.stdout:3177: finishing rollback tid 2 to smithi16367185-2 2023-10-09T09:18:46.400 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 855 (ObjNum 1184 snap 397 seq_num 1184) dirty exists 2023-10-09T09:18:46.401 INFO:tasks.rados.rados.0.smithi163.stdout:3176: done (2 left) 2023-10-09T09:18:46.401 INFO:tasks.rados.rados.0.smithi163.stdout:3177: done (1 left) 2023-10-09T09:18:46.401 INFO:tasks.rados.rados.0.smithi163.stdout:3179: done (0 left) 2023-10-09T09:18:46.401 INFO:tasks.rados.rados.0.smithi163.stdout:3180: write oid 32 current snap is 399 2023-10-09T09:18:46.401 INFO:tasks.rados.rados.0.smithi163.stdout:3180: seq_num 1191 ranges {417137=572627,1428054=589083} 2023-10-09T09:18:46.411 INFO:tasks.rados.rados.0.smithi163.stdout:3180: writing smithi16367185-32 from 417137 to 989764 tid 1 2023-10-09T09:18:46.418 INFO:tasks.rados.rados.0.smithi163.stdout:3180: writing smithi16367185-32 from 1428054 to 2017137 tid 2 2023-10-09T09:18:46.418 INFO:tasks.rados.rados.0.smithi163.stdout:3181: rollback oid 35 current snap is 399 2023-10-09T09:18:46.418 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 35 to 382 2023-10-09T09:18:46.418 INFO:tasks.rados.rados.0.smithi163.stdout:3182: rollback oid 38 current snap is 399 2023-10-09T09:18:46.418 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 398 2023-10-09T09:18:46.418 INFO:tasks.rados.rados.0.smithi163.stdout:3183: snap_remove snap 365 2023-10-09T09:18:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:46 smithi163 bash[78107]: cluster 2023-10-09T09:18:45.238361+0000 osd.5 (osd.5) 386 : cluster 3 failed to encode map e851 with expected crc 2023-10-09T09:18:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:46 smithi163 bash[78107]: cluster 2023-10-09T09:18:45.421789+0000 mgr.y (mgr.34107) 366 : cluster 0 pgmap v830: 89 pgs: 89 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:18:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:46 smithi163 bash[78107]: cluster 2023-10-09T09:18:45.545564+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e851 with expected crc 2023-10-09T09:18:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:46 smithi163 bash[75425]: cluster 2023-10-09T09:18:45.238361+0000 osd.5 (osd.5) 386 : cluster 3 failed to encode map e851 with expected crc 2023-10-09T09:18:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:46 smithi163 bash[75425]: cluster 2023-10-09T09:18:45.421789+0000 mgr.y (mgr.34107) 366 : cluster 0 pgmap v830: 89 pgs: 89 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:18:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:46 smithi163 bash[75425]: cluster 2023-10-09T09:18:45.545564+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e851 with expected crc 2023-10-09T09:18:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:46 smithi181 bash[59345]: cluster 2023-10-09T09:18:45.238361+0000 osd.5 (osd.5) 386 : cluster 3 failed to encode map e851 with expected crc 2023-10-09T09:18:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:46 smithi181 bash[59345]: cluster 2023-10-09T09:18:45.421789+0000 mgr.y (mgr.34107) 366 : cluster 0 pgmap v830: 89 pgs: 89 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2023-10-09T09:18:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:46 smithi181 bash[59345]: cluster 2023-10-09T09:18:45.545564+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e851 with expected crc 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:3183: done (3 left) 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:3184: write oid 44 current snap is 399 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:3180: finishing write tid 1 to smithi16367185-32 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:3181: finishing rollback tid 1 to smithi16367185-35 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:3181: finishing rollback tid 2 to smithi16367185-35 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 857 (ObjNum 1066 snap 360 seq_num 1066) dirty exists 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:3180: finishing write tid 2 to smithi16367185-32 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:3182: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:3182: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:3180: finishing write tid 3 to smithi16367185-32 2023-10-09T09:18:47.370 INFO:tasks.rados.rados.0.smithi163.stdout:3182: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:18:47.371 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1497 (ObjNum 1186 snap 398 seq_num 1186) dirty exists 2023-10-09T09:18:47.371 INFO:tasks.rados.rados.0.smithi163.stdout:3180: finishing write tid 4 to smithi16367185-32 2023-10-09T09:18:47.371 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 2240 (ObjNum 1191 snap 399 seq_num 1191) dirty exists 2023-10-09T09:18:47.371 INFO:tasks.rados.rados.0.smithi163.stdout:3180: left oid 32 (ObjNum 1191 snap 399 seq_num 1191) 2023-10-09T09:18:47.371 INFO:tasks.rados.rados.0.smithi163.stdout:3184: seq_num 1192 ranges {433944=647876,1486536=547408} 2023-10-09T09:18:47.377 INFO:tasks.rados.rados.0.smithi163.stdout:3184: writing smithi16367185-44 from 433944 to 1081820 tid 1 2023-10-09T09:18:47.383 INFO:tasks.rados.rados.0.smithi163.stdout:3184: writing smithi16367185-44 from 1486536 to 2033944 tid 2 2023-10-09T09:18:47.383 INFO:tasks.rados.rados.0.smithi163.stdout:3180: done (3 left) 2023-10-09T09:18:47.383 INFO:tasks.rados.rados.0.smithi163.stdout:3181: done (2 left) 2023-10-09T09:18:47.384 INFO:tasks.rados.rados.0.smithi163.stdout:3182: done (1 left) 2023-10-09T09:18:47.384 INFO:tasks.rados.rados.0.smithi163.stdout:3185: snap_remove snap 382 2023-10-09T09:18:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:47 smithi163 bash[78107]: cluster 2023-10-09T09:18:45.633711+0000 osd.4 (osd.4) 360 : cluster 3 failed to encode map e851 with expected crc 2023-10-09T09:18:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:47 smithi163 bash[78107]: cluster 2023-10-09T09:18:46.361932+0000 osd.5 (osd.5) 387 : cluster 3 failed to encode map e852 with expected crc 2023-10-09T09:18:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:47 smithi163 bash[78107]: cluster 2023-10-09T09:18:46.366986+0000 mon.a (mon.0) 1048 : cluster 0 osdmap e852: 8 total, 8 up, 8 in 2023-10-09T09:18:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:47 smithi163 bash[78107]: cluster 2023-10-09T09:18:46.383555+0000 osd.7 (osd.7) 328 : cluster 3 failed to encode map e852 with expected crc 2023-10-09T09:18:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:47 smithi163 bash[75425]: cluster 2023-10-09T09:18:45.633711+0000 osd.4 (osd.4) 360 : cluster 3 failed to encode map e851 with expected crc 2023-10-09T09:18:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:47 smithi163 bash[75425]: cluster 2023-10-09T09:18:46.361932+0000 osd.5 (osd.5) 387 : cluster 3 failed to encode map e852 with expected crc 2023-10-09T09:18:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:47 smithi163 bash[75425]: cluster 2023-10-09T09:18:46.366986+0000 mon.a (mon.0) 1048 : cluster 0 osdmap e852: 8 total, 8 up, 8 in 2023-10-09T09:18:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:47 smithi163 bash[75425]: cluster 2023-10-09T09:18:46.383555+0000 osd.7 (osd.7) 328 : cluster 3 failed to encode map e852 with expected crc 2023-10-09T09:18:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:47 smithi181 bash[59345]: cluster 2023-10-09T09:18:45.633711+0000 osd.4 (osd.4) 360 : cluster 3 failed to encode map e851 with expected crc 2023-10-09T09:18:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:47 smithi181 bash[59345]: cluster 2023-10-09T09:18:46.361932+0000 osd.5 (osd.5) 387 : cluster 3 failed to encode map e852 with expected crc 2023-10-09T09:18:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:47 smithi181 bash[59345]: cluster 2023-10-09T09:18:46.366986+0000 mon.a (mon.0) 1048 : cluster 0 osdmap e852: 8 total, 8 up, 8 in 2023-10-09T09:18:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:47 smithi181 bash[59345]: cluster 2023-10-09T09:18:46.383555+0000 osd.7 (osd.7) 328 : cluster 3 failed to encode map e852 with expected crc 2023-10-09T09:18:48.381 INFO:tasks.rados.rados.0.smithi163.stdout:3184: finishing write tid 1 to smithi16367185-44 2023-10-09T09:18:48.382 INFO:tasks.rados.rados.0.smithi163.stdout:3184: finishing write tid 2 to smithi16367185-44 2023-10-09T09:18:48.382 INFO:tasks.rados.rados.0.smithi163.stdout:3184: finishing write tid 3 to smithi16367185-44 2023-10-09T09:18:48.382 INFO:tasks.rados.rados.0.smithi163.stdout:3184: finishing write tid 4 to smithi16367185-44 2023-10-09T09:18:48.382 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 2252 (ObjNum 1192 snap 399 seq_num 1192) dirty exists 2023-10-09T09:18:48.382 INFO:tasks.rados.rados.0.smithi163.stdout:3184: left oid 44 (ObjNum 1192 snap 399 seq_num 1192) 2023-10-09T09:18:48.382 INFO:tasks.rados.rados.0.smithi163.stdout:3184: done (1 left) 2023-10-09T09:18:48.382 INFO:tasks.rados.rados.0.smithi163.stdout:3185: done (0 left) 2023-10-09T09:18:48.382 INFO:tasks.rados.rados.0.smithi163.stdout:3186: read oid 9 snap -1 2023-10-09T09:18:48.382 INFO:tasks.rados.rados.0.smithi163.stdout:3186: expect deleted 2023-10-09T09:18:48.382 INFO:tasks.rados.rados.0.smithi163.stdout:3187: snap_create 2023-10-09T09:18:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:48 smithi163 bash[78107]: cluster 2023-10-09T09:18:47.366061+0000 osd.5 (osd.5) 388 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:48 smithi163 bash[78107]: cluster 2023-10-09T09:18:47.369269+0000 mon.a (mon.0) 1049 : cluster 0 osdmap e853: 8 total, 8 up, 8 in 2023-10-09T09:18:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:48 smithi163 bash[78107]: cluster 2023-10-09T09:18:47.370967+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:48 smithi163 bash[78107]: cluster 2023-10-09T09:18:47.371556+0000 osd.4 (osd.4) 361 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:48 smithi163 bash[78107]: cluster 2023-10-09T09:18:47.422462+0000 mgr.y (mgr.34107) 367 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 356 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.7 MiB/s wr, 28 op/s 2023-10-09T09:18:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:48 smithi163 bash[75425]: cluster 2023-10-09T09:18:47.366061+0000 osd.5 (osd.5) 388 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:48 smithi163 bash[75425]: cluster 2023-10-09T09:18:47.369269+0000 mon.a (mon.0) 1049 : cluster 0 osdmap e853: 8 total, 8 up, 8 in 2023-10-09T09:18:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:48 smithi163 bash[75425]: cluster 2023-10-09T09:18:47.370967+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:48 smithi163 bash[75425]: cluster 2023-10-09T09:18:47.371556+0000 osd.4 (osd.4) 361 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:48 smithi163 bash[75425]: cluster 2023-10-09T09:18:47.422462+0000 mgr.y (mgr.34107) 367 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 356 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.7 MiB/s wr, 28 op/s 2023-10-09T09:18:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:48 smithi181 bash[59345]: cluster 2023-10-09T09:18:47.366061+0000 osd.5 (osd.5) 388 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:48 smithi181 bash[59345]: cluster 2023-10-09T09:18:47.369269+0000 mon.a (mon.0) 1049 : cluster 0 osdmap e853: 8 total, 8 up, 8 in 2023-10-09T09:18:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:48 smithi181 bash[59345]: cluster 2023-10-09T09:18:47.370967+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:48 smithi181 bash[59345]: cluster 2023-10-09T09:18:47.371556+0000 osd.4 (osd.4) 361 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:48 smithi181 bash[59345]: cluster 2023-10-09T09:18:47.422462+0000 mgr.y (mgr.34107) 367 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 356 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.7 MiB/s wr, 28 op/s 2023-10-09T09:18:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:3186: done (1 left) 2023-10-09T09:18:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:3187: done (0 left) 2023-10-09T09:18:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:3188: write oid 31 current snap is 400 2023-10-09T09:18:49.382 INFO:tasks.rados.rados.0.smithi163.stdout:3188: seq_num 1193 ranges {450751=723125,1945018=105733} 2023-10-09T09:18:49.389 INFO:tasks.rados.rados.0.smithi163.stdout:3188: writing smithi16367185-31 from 450751 to 1173876 tid 1 2023-10-09T09:18:49.394 INFO:tasks.rados.rados.0.smithi163.stdout:3188: writing smithi16367185-31 from 1945018 to 2050751 tid 2 2023-10-09T09:18:49.394 INFO:tasks.rados.rados.0.smithi163.stdout:3189: delete oid 2 current snap is 400 2023-10-09T09:18:49.399 INFO:tasks.rados.rados.0.smithi163.stdout:3188: finishing write tid 1 to smithi16367185-31 2023-10-09T09:18:49.399 INFO:tasks.rados.rados.0.smithi163.stdout:3188: finishing write tid 2 to smithi16367185-31 2023-10-09T09:18:49.400 INFO:tasks.rados.rados.0.smithi163.stdout:3189: done (1 left) 2023-10-09T09:18:49.400 INFO:tasks.rados.rados.0.smithi163.stdout:3190: snap_create 2023-10-09T09:18:49.401 INFO:tasks.rados.rados.0.smithi163.stdout:3188: finishing write tid 3 to smithi16367185-31 2023-10-09T09:18:49.401 INFO:tasks.rados.rados.0.smithi163.stdout:3188: finishing write tid 4 to smithi16367185-31 2023-10-09T09:18:49.402 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 856 (ObjNum 1193 snap 400 seq_num 1193) dirty exists 2023-10-09T09:18:49.402 INFO:tasks.rados.rados.0.smithi163.stdout:3188: left oid 31 (ObjNum 1193 snap 400 seq_num 1193) 2023-10-09T09:18:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:49 smithi163 bash[78107]: cluster 2023-10-09T09:18:47.504004+0000 osd.7 (osd.7) 329 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:49 smithi163 bash[78107]: cluster 2023-10-09T09:18:48.374569+0000 osd.5 (osd.5) 389 : cluster 3 failed to encode map e854 with expected crc 2023-10-09T09:18:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:49 smithi163 bash[78107]: cluster 2023-10-09T09:18:48.378525+0000 osd.5 (osd.5) 390 : cluster 3 failed to encode map e854 with expected crc 2023-10-09T09:18:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:49 smithi163 bash[78107]: cluster 2023-10-09T09:18:48.379484+0000 mon.a (mon.0) 1050 : cluster 0 osdmap e854: 8 total, 8 up, 8 in 2023-10-09T09:18:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:49 smithi163 bash[75425]: cluster 2023-10-09T09:18:47.504004+0000 osd.7 (osd.7) 329 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:49 smithi163 bash[75425]: cluster 2023-10-09T09:18:48.374569+0000 osd.5 (osd.5) 389 : cluster 3 failed to encode map e854 with expected crc 2023-10-09T09:18:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:49 smithi163 bash[75425]: cluster 2023-10-09T09:18:48.378525+0000 osd.5 (osd.5) 390 : cluster 3 failed to encode map e854 with expected crc 2023-10-09T09:18:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:49 smithi163 bash[75425]: cluster 2023-10-09T09:18:48.379484+0000 mon.a (mon.0) 1050 : cluster 0 osdmap e854: 8 total, 8 up, 8 in 2023-10-09T09:18:49.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:49 smithi181 bash[59345]: cluster 2023-10-09T09:18:47.504004+0000 osd.7 (osd.7) 329 : cluster 3 failed to encode map e853 with expected crc 2023-10-09T09:18:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:49 smithi181 bash[59345]: cluster 2023-10-09T09:18:48.374569+0000 osd.5 (osd.5) 389 : cluster 3 failed to encode map e854 with expected crc 2023-10-09T09:18:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:49 smithi181 bash[59345]: cluster 2023-10-09T09:18:48.378525+0000 osd.5 (osd.5) 390 : cluster 3 failed to encode map e854 with expected crc 2023-10-09T09:18:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:49 smithi181 bash[59345]: cluster 2023-10-09T09:18:48.379484+0000 mon.a (mon.0) 1050 : cluster 0 osdmap e854: 8 total, 8 up, 8 in 2023-10-09T09:18:50.250 INFO:tasks.rados.rados.0.smithi163.stdout:3188: done (1 left) 2023-10-09T09:18:50.250 INFO:tasks.rados.rados.0.smithi163.stdout:3190: done (0 left) 2023-10-09T09:18:50.250 INFO:tasks.rados.rados.0.smithi163.stdout:3191: write oid 13 current snap is 401 2023-10-09T09:18:50.250 INFO:tasks.rados.rados.0.smithi163.stdout:3191: seq_num 1195 ranges {484365=589976,1661982=422383} 2023-10-09T09:18:50.258 INFO:tasks.rados.rados.0.smithi163.stdout:3191: writing smithi16367185-13 from 484365 to 1074341 tid 1 2023-10-09T09:18:50.264 INFO:tasks.rados.rados.0.smithi163.stdout:3191: writing smithi16367185-13 from 1661982 to 2084365 tid 2 2023-10-09T09:18:50.264 INFO:tasks.rados.rados.0.smithi163.stdout:3192: snap_create 2023-10-09T09:18:50.269 INFO:tasks.rados.rados.0.smithi163.stdout:3191: finishing write tid 1 to smithi16367185-13 2023-10-09T09:18:50.270 INFO:tasks.rados.rados.0.smithi163.stdout:3191: finishing write tid 2 to smithi16367185-13 2023-10-09T09:18:50.270 INFO:tasks.rados.rados.0.smithi163.stdout:3191: finishing write tid 3 to smithi16367185-13 2023-10-09T09:18:50.271 INFO:tasks.rados.rados.0.smithi163.stdout:3191: finishing write tid 4 to smithi16367185-13 2023-10-09T09:18:50.271 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 585 (ObjNum 1195 snap 401 seq_num 1195) dirty exists 2023-10-09T09:18:50.271 INFO:tasks.rados.rados.0.smithi163.stdout:3191: left oid 13 (ObjNum 1195 snap 401 seq_num 1195) 2023-10-09T09:18:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[78107]: cluster 2023-10-09T09:18:49.376846+0000 osd.5 (osd.5) 391 : cluster 3 failed to encode map e855 with expected crc 2023-10-09T09:18:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[78107]: cluster 2023-10-09T09:18:49.377732+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e855 with expected crc 2023-10-09T09:18:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[78107]: cluster 2023-10-09T09:18:49.381636+0000 mon.a (mon.0) 1051 : cluster 0 osdmap e855: 8 total, 8 up, 8 in 2023-10-09T09:18:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[78107]: cluster 2023-10-09T09:18:49.393596+0000 osd.4 (osd.4) 362 : cluster 3 failed to encode map e855 with expected crc 2023-10-09T09:18:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[78107]: cluster 2023-10-09T09:18:49.422938+0000 mgr.y (mgr.34107) 368 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 359 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.0 MiB/s wr, 17 op/s 2023-10-09T09:18:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[78107]: cluster 2023-10-09T09:18:50.246246+0000 osd.4 (osd.4) 363 : cluster 3 failed to encode map e856 with expected crc 2023-10-09T09:18:50.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[78107]: cluster 2023-10-09T09:18:50.249752+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e856: 8 total, 8 up, 8 in 2023-10-09T09:18:50.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[75425]: cluster 2023-10-09T09:18:49.376846+0000 osd.5 (osd.5) 391 : cluster 3 failed to encode map e855 with expected crc 2023-10-09T09:18:50.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[75425]: cluster 2023-10-09T09:18:49.377732+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e855 with expected crc 2023-10-09T09:18:50.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[75425]: cluster 2023-10-09T09:18:49.381636+0000 mon.a (mon.0) 1051 : cluster 0 osdmap e855: 8 total, 8 up, 8 in 2023-10-09T09:18:50.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[75425]: cluster 2023-10-09T09:18:49.393596+0000 osd.4 (osd.4) 362 : cluster 3 failed to encode map e855 with expected crc 2023-10-09T09:18:50.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[75425]: cluster 2023-10-09T09:18:49.422938+0000 mgr.y (mgr.34107) 368 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 359 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.0 MiB/s wr, 17 op/s 2023-10-09T09:18:50.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[75425]: cluster 2023-10-09T09:18:50.246246+0000 osd.4 (osd.4) 363 : cluster 3 failed to encode map e856 with expected crc 2023-10-09T09:18:50.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:50 smithi163 bash[75425]: cluster 2023-10-09T09:18:50.249752+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e856: 8 total, 8 up, 8 in 2023-10-09T09:18:50.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:50 smithi181 bash[59345]: cluster 2023-10-09T09:18:49.376846+0000 osd.5 (osd.5) 391 : cluster 3 failed to encode map e855 with expected crc 2023-10-09T09:18:50.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:50 smithi181 bash[59345]: cluster 2023-10-09T09:18:49.377732+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e855 with expected crc 2023-10-09T09:18:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:50 smithi181 bash[59345]: cluster 2023-10-09T09:18:49.381636+0000 mon.a (mon.0) 1051 : cluster 0 osdmap e855: 8 total, 8 up, 8 in 2023-10-09T09:18:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:50 smithi181 bash[59345]: cluster 2023-10-09T09:18:49.393596+0000 osd.4 (osd.4) 362 : cluster 3 failed to encode map e855 with expected crc 2023-10-09T09:18:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:50 smithi181 bash[59345]: cluster 2023-10-09T09:18:49.422938+0000 mgr.y (mgr.34107) 368 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 359 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.0 MiB/s wr, 17 op/s 2023-10-09T09:18:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:50 smithi181 bash[59345]: cluster 2023-10-09T09:18:50.246246+0000 osd.4 (osd.4) 363 : cluster 3 failed to encode map e856 with expected crc 2023-10-09T09:18:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:50 smithi181 bash[59345]: cluster 2023-10-09T09:18:50.249752+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e856: 8 total, 8 up, 8 in 2023-10-09T09:18:51.259 INFO:tasks.rados.rados.0.smithi163.stdout:3191: done (1 left) 2023-10-09T09:18:51.260 INFO:tasks.rados.rados.0.smithi163.stdout:3192: done (0 left) 2023-10-09T09:18:51.260 INFO:tasks.rados.rados.0.smithi163.stdout:3193: write oid 38 current snap is 402 2023-10-09T09:18:51.260 INFO:tasks.rados.rados.0.smithi163.stdout:3193: seq_num 1196 ranges {501172=665225,1720464=380708} 2023-10-09T09:18:51.270 INFO:tasks.rados.rados.0.smithi163.stdout:3193: writing smithi16367185-38 from 501172 to 1166397 tid 1 2023-10-09T09:18:51.276 INFO:tasks.rados.rados.0.smithi163.stdout:3193: writing smithi16367185-38 from 1720464 to 2101172 tid 2 2023-10-09T09:18:51.276 INFO:tasks.rados.rados.0.smithi163.stdout:3194: write oid 18 current snap is 402 2023-10-09T09:18:51.276 INFO:tasks.rados.rados.0.smithi163.stdout:3194: seq_num 1197 ranges {517979=740474,1778946=339033} 2023-10-09T09:18:51.283 INFO:tasks.rados.rados.0.smithi163.stdout:3194: writing smithi16367185-18 from 517979 to 1258453 tid 1 2023-10-09T09:18:51.290 INFO:tasks.rados.rados.0.smithi163.stdout:3194: writing smithi16367185-18 from 1778946 to 2117979 tid 2 2023-10-09T09:18:51.290 INFO:tasks.rados.rados.0.smithi163.stdout:3195: rollback oid 40 current snap is 402 2023-10-09T09:18:51.290 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 40 to 370 2023-10-09T09:18:51.290 INFO:tasks.rados.rados.0.smithi163.stdout:3193: finishing write tid 1 to smithi16367185-38 2023-10-09T09:18:51.290 INFO:tasks.rados.rados.0.smithi163.stdout:3193: finishing write tid 2 to smithi16367185-38 2023-10-09T09:18:51.290 INFO:tasks.rados.rados.0.smithi163.stdout:3193: finishing write tid 3 to smithi16367185-38 2023-10-09T09:18:51.290 INFO:tasks.rados.rados.0.smithi163.stdout:3193: finishing write tid 4 to smithi16367185-38 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1511 (ObjNum 1196 snap 402 seq_num 1196) dirty exists 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:3193: left oid 38 (ObjNum 1196 snap 402 seq_num 1196) 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:3193: done (2 left) 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:3196: read oid 7 snap 397 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:3196: expect (ObjNum 1178 snap 397 seq_num 1178) 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:3197: read oid 10 snap -1 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:3197: expect (ObjNum 1156 snap 391 seq_num 1156) 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:3198: rollback oid 15 current snap is 402 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 15 to 358 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:3199: write oid 1 current snap is 402 2023-10-09T09:18:51.291 INFO:tasks.rados.rados.0.smithi163.stdout:3199: seq_num 1198 ranges {534786=415723,1437428=697358} 2023-10-09T09:18:51.295 INFO:tasks.rados.rados.0.smithi163.stdout:3199: writing smithi16367185-1 from 534786 to 950509 tid 1 2023-10-09T09:18:51.303 INFO:tasks.rados.rados.0.smithi163.stdout:3199: writing smithi16367185-1 from 1437428 to 2134786 tid 2 2023-10-09T09:18:51.303 INFO:tasks.rados.rados.0.smithi163.stdout:3200: delete oid 19 current snap is 402 2023-10-09T09:18:51.303 INFO:tasks.rados.rados.0.smithi163.stdout:3194: finishing write tid 1 to smithi16367185-18 2023-10-09T09:18:51.304 INFO:tasks.rados.rados.0.smithi163.stdout:3194: finishing write tid 2 to smithi16367185-18 2023-10-09T09:18:51.304 INFO:tasks.rados.rados.0.smithi163.stdout:3194: finishing write tid 3 to smithi16367185-18 2023-10-09T09:18:51.304 INFO:tasks.rados.rados.0.smithi163.stdout:3194: finishing write tid 4 to smithi16367185-18 2023-10-09T09:18:51.304 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1054 (ObjNum 1197 snap 402 seq_num 1197) dirty exists 2023-10-09T09:18:51.304 INFO:tasks.rados.rados.0.smithi163.stdout:3194: left oid 18 (ObjNum 1197 snap 402 seq_num 1197) 2023-10-09T09:18:51.304 INFO:tasks.rados.rados.0.smithi163.stdout:3197: expect (ObjNum 1156 snap 391 seq_num 1156) 2023-10-09T09:18:51.506 INFO:tasks.rados.rados.0.smithi163.stdout:3196: expect (ObjNum 1178 snap 397 seq_num 1178) 2023-10-09T09:18:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:51 smithi163 bash[75425]: cluster 2023-10-09T09:18:50.246209+0000 osd.5 (osd.5) 392 : cluster 3 failed to encode map e856 with expected crc 2023-10-09T09:18:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:51 smithi163 bash[75425]: cluster 2023-10-09T09:18:50.259770+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e856 with expected crc 2023-10-09T09:18:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:51 smithi163 bash[75425]: cluster 2023-10-09T09:18:51.259075+0000 mon.a (mon.0) 1053 : cluster 0 osdmap e857: 8 total, 8 up, 8 in 2023-10-09T09:18:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:51 smithi163 bash[75425]: cluster 2023-10-09T09:18:51.273057+0000 osd.4 (osd.4) 364 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:51 smithi163 bash[78107]: cluster 2023-10-09T09:18:50.246209+0000 osd.5 (osd.5) 392 : cluster 3 failed to encode map e856 with expected crc 2023-10-09T09:18:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:51 smithi163 bash[78107]: cluster 2023-10-09T09:18:50.259770+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e856 with expected crc 2023-10-09T09:18:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:51 smithi163 bash[78107]: cluster 2023-10-09T09:18:51.259075+0000 mon.a (mon.0) 1053 : cluster 0 osdmap e857: 8 total, 8 up, 8 in 2023-10-09T09:18:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:51 smithi163 bash[78107]: cluster 2023-10-09T09:18:51.273057+0000 osd.4 (osd.4) 364 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:51.795 INFO:tasks.rados.rados.0.smithi163.stdout:3195: finishing rollback tid 0 to smithi16367185-40 2023-10-09T09:18:51.795 INFO:tasks.rados.rados.0.smithi163.stdout:3195: finishing rollback tid 1 to smithi16367185-40 2023-10-09T09:18:51.795 INFO:tasks.rados.rados.0.smithi163.stdout:3199: finishing write tid 1 to smithi16367185-1 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:3195: finishing rollback tid 2 to smithi16367185-40 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1588 (ObjNum 1027 snap 349 seq_num 1027) dirty exists 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:3198: finishing rollback tid 1 to smithi16367185-15 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 0 (ObjNum 669367496 snap 21875 seq_num 1) dirty dne 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:3199: finishing write tid 2 to smithi16367185-1 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:3199: finishing write tid 3 to smithi16367185-1 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:3199: finishing write tid 4 to smithi16367185-1 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1567 (ObjNum 1198 snap 402 seq_num 1198) dirty exists 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:3199: left oid 1 (ObjNum 1198 snap 402 seq_num 1198) 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:3194: done (6 left) 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:3195: done (5 left) 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:3196: done (4 left) 2023-10-09T09:18:51.796 INFO:tasks.rados.rados.0.smithi163.stdout:3197: done (3 left) 2023-10-09T09:18:51.797 INFO:tasks.rados.rados.0.smithi163.stdout:3198: done (2 left) 2023-10-09T09:18:51.797 INFO:tasks.rados.rados.0.smithi163.stdout:3199: done (1 left) 2023-10-09T09:18:51.797 INFO:tasks.rados.rados.0.smithi163.stdout:3200: done (0 left) 2023-10-09T09:18:51.797 INFO:tasks.rados.rados.0.smithi163.stdout:3201: snap_remove snap 389 2023-10-09T09:18:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:51 smithi181 bash[59345]: cluster 2023-10-09T09:18:50.246209+0000 osd.5 (osd.5) 392 : cluster 3 failed to encode map e856 with expected crc 2023-10-09T09:18:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:51 smithi181 bash[59345]: cluster 2023-10-09T09:18:50.259770+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e856 with expected crc 2023-10-09T09:18:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:51 smithi181 bash[59345]: cluster 2023-10-09T09:18:51.259075+0000 mon.a (mon.0) 1053 : cluster 0 osdmap e857: 8 total, 8 up, 8 in 2023-10-09T09:18:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:51 smithi181 bash[59345]: cluster 2023-10-09T09:18:51.273057+0000 osd.4 (osd.4) 364 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:52.418 INFO:tasks.rados.rados.0.smithi163.stdout:3201: done (0 left) 2023-10-09T09:18:52.418 INFO:tasks.rados.rados.0.smithi163.stdout:3202: rollback oid 8 current snap is 402 2023-10-09T09:18:52.418 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 8 to 393 2023-10-09T09:18:52.419 INFO:tasks.rados.rados.0.smithi163.stdout:3203: read oid 20 snap -1 2023-10-09T09:18:52.419 INFO:tasks.rados.rados.0.smithi163.stdout:3203: expect (ObjNum 1139 snap 380 seq_num 1139) 2023-10-09T09:18:52.419 INFO:tasks.rados.rados.0.smithi163.stdout:3204: delete oid 9 current snap is 402 2023-10-09T09:18:52.430 INFO:tasks.rados.rados.0.smithi163.stdout:3204: done (2 left) 2023-10-09T09:18:52.430 INFO:tasks.rados.rados.0.smithi163.stdout:3205: snap_remove snap 396 2023-10-09T09:18:52.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:52 smithi163 bash[78107]: cluster 2023-10-09T09:18:51.253910+0000 osd.5 (osd.5) 393 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:52.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:52 smithi163 bash[78107]: cluster 2023-10-09T09:18:51.254624+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:52.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:52 smithi163 bash[78107]: cluster 2023-10-09T09:18:51.290663+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:52.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:52 smithi163 bash[78107]: cluster 2023-10-09T09:18:51.423758+0000 mgr.y (mgr.34107) 369 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 363 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2023-10-09T09:18:52.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:52 smithi163 bash[75425]: cluster 2023-10-09T09:18:51.253910+0000 osd.5 (osd.5) 393 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:52.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:52 smithi163 bash[75425]: cluster 2023-10-09T09:18:51.254624+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:52.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:52 smithi163 bash[75425]: cluster 2023-10-09T09:18:51.290663+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:52.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:52 smithi163 bash[75425]: cluster 2023-10-09T09:18:51.423758+0000 mgr.y (mgr.34107) 369 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 363 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2023-10-09T09:18:52.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:52 smithi181 bash[59345]: cluster 2023-10-09T09:18:51.253910+0000 osd.5 (osd.5) 393 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:52.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:52 smithi181 bash[59345]: cluster 2023-10-09T09:18:51.254624+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:52 smithi181 bash[59345]: cluster 2023-10-09T09:18:51.290663+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e857 with expected crc 2023-10-09T09:18:52.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:52 smithi181 bash[59345]: cluster 2023-10-09T09:18:51.423758+0000 mgr.y (mgr.34107) 369 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 363 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2023-10-09T09:18:53.415 INFO:tasks.rados.rados.0.smithi163.stdout:3205: done (2 left) 2023-10-09T09:18:53.416 INFO:tasks.rados.rados.0.smithi163.stdout:3206: read oid 2 snap -1 2023-10-09T09:18:53.416 INFO:tasks.rados.rados.0.smithi163.stdout:3206: expect deleted 2023-10-09T09:18:53.416 INFO:tasks.rados.rados.0.smithi163.stdout:3207: snap_remove snap 386 2023-10-09T09:18:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:18:53] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.33.4" 2023-10-09T09:18:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[78107]: cluster 2023-10-09T09:18:52.410650+0000 osd.5 (osd.5) 394 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[78107]: cluster 2023-10-09T09:18:52.417573+0000 osd.4 (osd.4) 365 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[78107]: cluster 2023-10-09T09:18:52.417672+0000 mon.a (mon.0) 1054 : cluster 0 osdmap e858: 8 total, 8 up, 8 in 2023-10-09T09:18:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[78107]: cluster 2023-10-09T09:18:52.419526+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[78107]: cluster 2023-10-09T09:18:52.419751+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[78107]: cluster 2023-10-09T09:18:52.424599+0000 osd.4 (osd.4) 366 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[75425]: cluster 2023-10-09T09:18:52.410650+0000 osd.5 (osd.5) 394 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[75425]: cluster 2023-10-09T09:18:52.417573+0000 osd.4 (osd.4) 365 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[75425]: cluster 2023-10-09T09:18:52.417672+0000 mon.a (mon.0) 1054 : cluster 0 osdmap e858: 8 total, 8 up, 8 in 2023-10-09T09:18:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[75425]: cluster 2023-10-09T09:18:52.419526+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[75425]: cluster 2023-10-09T09:18:52.419751+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:53 smithi163 bash[75425]: cluster 2023-10-09T09:18:52.424599+0000 osd.4 (osd.4) 366 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:53 smithi181 bash[59345]: cluster 2023-10-09T09:18:52.410650+0000 osd.5 (osd.5) 394 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:53 smithi181 bash[59345]: cluster 2023-10-09T09:18:52.417573+0000 osd.4 (osd.4) 365 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:53 smithi181 bash[59345]: cluster 2023-10-09T09:18:52.417672+0000 mon.a (mon.0) 1054 : cluster 0 osdmap e858: 8 total, 8 up, 8 in 2023-10-09T09:18:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:53 smithi181 bash[59345]: cluster 2023-10-09T09:18:52.419526+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:53 smithi181 bash[59345]: cluster 2023-10-09T09:18:52.419751+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:53 smithi181 bash[59345]: cluster 2023-10-09T09:18:52.424599+0000 osd.4 (osd.4) 366 : cluster 3 failed to encode map e858 with expected crc 2023-10-09T09:18:54.421 INFO:tasks.rados.rados.0.smithi163.stdout:3207: done (3 left) 2023-10-09T09:18:54.421 INFO:tasks.rados.rados.0.smithi163.stdout:3208: write oid 31 current snap is 402 2023-10-09T09:18:54.421 INFO:tasks.rados.rados.0.smithi163.stdout:3208: seq_num 1201 ranges {585207=641470,1896521=288686} 2023-10-09T09:18:54.430 INFO:tasks.rados.rados.0.smithi163.stdout:3208: writing smithi16367185-31 from 585207 to 1226677 tid 1 2023-10-09T09:18:54.436 INFO:tasks.rados.rados.0.smithi163.stdout:3208: writing smithi16367185-31 from 1896521 to 2185207 tid 2 2023-10-09T09:18:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:3203: expect (ObjNum 1139 snap 380 seq_num 1139) 2023-10-09T09:18:54.623 INFO:tasks.rados.rados.0.smithi163.stdout:3202: finishing rollback tid 1 to smithi16367185-8 2023-10-09T09:18:54.624 INFO:tasks.rados.rados.0.smithi163.stdout:3202: finishing rollback tid 2 to smithi16367185-8 2023-10-09T09:18:54.624 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1593 (ObjNum 1092 snap 362 seq_num 1092) dirty exists 2023-10-09T09:18:54.624 INFO:tasks.rados.rados.0.smithi163.stdout:3202: done (3 left) 2023-10-09T09:18:54.624 INFO:tasks.rados.rados.0.smithi163.stdout:3203: done (2 left) 2023-10-09T09:18:54.624 INFO:tasks.rados.rados.0.smithi163.stdout:3206: done (1 left) 2023-10-09T09:18:54.624 INFO:tasks.rados.rados.0.smithi163.stdout:3209: delete oid 9 current snap is 402 2023-10-09T09:18:54.624 INFO:tasks.rados.rados.0.smithi163.stdout:3208: finishing write tid 1 to smithi16367185-31 2023-10-09T09:18:54.624 INFO:tasks.rados.rados.0.smithi163.stdout:3208: finishing write tid 2 to smithi16367185-31 2023-10-09T09:18:54.624 INFO:tasks.rados.rados.0.smithi163.stdout:3208: finishing write tid 3 to smithi16367185-31 2023-10-09T09:18:54.624 INFO:tasks.rados.rados.0.smithi163.stdout:3208: finishing write tid 4 to smithi16367185-31 2023-10-09T09:18:54.625 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 870 (ObjNum 1201 snap 402 seq_num 1201) dirty exists 2023-10-09T09:18:54.625 INFO:tasks.rados.rados.0.smithi163.stdout:3208: left oid 31 (ObjNum 1201 snap 402 seq_num 1201) 2023-10-09T09:18:54.627 INFO:tasks.rados.rados.0.smithi163.stdout:3208: done (1 left) 2023-10-09T09:18:54.627 INFO:tasks.rados.rados.0.smithi163.stdout:3209: done (0 left) 2023-10-09T09:18:54.627 INFO:tasks.rados.rados.0.smithi163.stdout:3210: write oid 9 current snap is 402 2023-10-09T09:18:54.627 INFO:tasks.rados.rados.0.smithi163.stdout:3210: seq_num 1203 ranges {618821=508321,1613485=531077} 2023-10-09T09:18:54.640 INFO:tasks.rados.rados.0.smithi163.stdout:3210: writing smithi16367185-9 from 618821 to 1127142 tid 1 2023-10-09T09:18:54.645 INFO:tasks.rados.rados.0.smithi163.stdout:3210: writing smithi16367185-9 from 1613485 to 2144562 tid 2 2023-10-09T09:18:54.646 INFO:tasks.rados.rados.0.smithi163.stdout:3211: rollback oid 19 current snap is 402 2023-10-09T09:18:54.646 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 19 to 371 2023-10-09T09:18:54.646 INFO:tasks.rados.rados.0.smithi163.stdout:3210: finishing write tid 1 to smithi16367185-9 2023-10-09T09:18:54.646 INFO:tasks.rados.rados.0.smithi163.stdout:3212: read oid 7 snap -1 2023-10-09T09:18:54.646 INFO:tasks.rados.rados.0.smithi163.stdout:3212: expect (ObjNum 1178 snap 397 seq_num 1178) 2023-10-09T09:18:54.646 INFO:tasks.rados.rados.0.smithi163.stdout:3213: write oid 31 current snap is 402 2023-10-09T09:18:54.646 INFO:tasks.rados.rados.0.smithi163.stdout:3213: seq_num 1204 ranges {635628=583570,1671967=563661} 2023-10-09T09:18:54.652 INFO:tasks.rados.rados.0.smithi163.stdout:3213: writing smithi16367185-31 from 635628 to 1219198 tid 1 2023-10-09T09:18:54.658 INFO:tasks.rados.rados.0.smithi163.stdout:3213: writing smithi16367185-31 from 1671967 to 2235628 tid 2 2023-10-09T09:18:54.659 INFO:tasks.rados.rados.0.smithi163.stdout:3214: read oid 40 snap -1 2023-10-09T09:18:54.659 INFO:tasks.rados.rados.0.smithi163.stdout:3214: expect (ObjNum 1027 snap 349 seq_num 1027) 2023-10-09T09:18:54.659 INFO:tasks.rados.rados.0.smithi163.stdout:3215: snap_remove snap 353 2023-10-09T09:18:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:54 smithi163 bash[75425]: cluster 2023-10-09T09:18:53.411251+0000 osd.5 (osd.5) 395 : cluster 3 failed to encode map e859 with expected crc 2023-10-09T09:18:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:54 smithi163 bash[75425]: cluster 2023-10-09T09:18:53.411989+0000 osd.4 (osd.4) 367 : cluster 3 failed to encode map e859 with expected crc 2023-10-09T09:18:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:54 smithi163 bash[75425]: cluster 2023-10-09T09:18:53.415309+0000 mon.a (mon.0) 1055 : cluster 0 osdmap e859: 8 total, 8 up, 8 in 2023-10-09T09:18:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:54 smithi163 bash[75425]: cluster 2023-10-09T09:18:53.417026+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e859 with expected crc 2023-10-09T09:18:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:54 smithi163 bash[75425]: cluster 2023-10-09T09:18:53.424279+0000 mgr.y (mgr.34107) 370 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 364 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2023-10-09T09:18:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:54 smithi163 bash[78107]: cluster 2023-10-09T09:18:53.411251+0000 osd.5 (osd.5) 395 : cluster 3 failed to encode map e859 with expected crc 2023-10-09T09:18:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:54 smithi163 bash[78107]: cluster 2023-10-09T09:18:53.411989+0000 osd.4 (osd.4) 367 : cluster 3 failed to encode map e859 with expected crc 2023-10-09T09:18:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:54 smithi163 bash[78107]: cluster 2023-10-09T09:18:53.415309+0000 mon.a (mon.0) 1055 : cluster 0 osdmap e859: 8 total, 8 up, 8 in 2023-10-09T09:18:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:54 smithi163 bash[78107]: cluster 2023-10-09T09:18:53.417026+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e859 with expected crc 2023-10-09T09:18:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:54 smithi163 bash[78107]: cluster 2023-10-09T09:18:53.424279+0000 mgr.y (mgr.34107) 370 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 364 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2023-10-09T09:18:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:54 smithi181 bash[59345]: cluster 2023-10-09T09:18:53.411251+0000 osd.5 (osd.5) 395 : cluster 3 failed to encode map e859 with expected crc 2023-10-09T09:18:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:54 smithi181 bash[59345]: cluster 2023-10-09T09:18:53.411989+0000 osd.4 (osd.4) 367 : cluster 3 failed to encode map e859 with expected crc 2023-10-09T09:18:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:54 smithi181 bash[59345]: cluster 2023-10-09T09:18:53.415309+0000 mon.a (mon.0) 1055 : cluster 0 osdmap e859: 8 total, 8 up, 8 in 2023-10-09T09:18:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:54 smithi181 bash[59345]: cluster 2023-10-09T09:18:53.417026+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e859 with expected crc 2023-10-09T09:18:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:54 smithi181 bash[59345]: cluster 2023-10-09T09:18:53.424279+0000 mgr.y (mgr.34107) 370 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 364 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2023-10-09T09:18:55.247 INFO:tasks.rados.rados.0.smithi163.stdout:3215: done (5 left) 2023-10-09T09:18:55.248 INFO:tasks.rados.rados.0.smithi163.stdout:3216: delete oid 3 current snap is 402 2023-10-09T09:18:55.248 INFO:tasks.rados.rados.0.smithi163.stdout:3211: finishing rollback tid 1 to smithi16367185-19 2023-10-09T09:18:55.248 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 938 (ObjNum 1618308304 snap 32603 seq_num 699274160) dirty dne 2023-10-09T09:18:55.248 INFO:tasks.rados.rados.0.smithi163.stdout:3210: finishing write tid 2 to smithi16367185-9 2023-10-09T09:18:55.248 INFO:tasks.rados.rados.0.smithi163.stdout:3212: expect (ObjNum 1178 snap 397 seq_num 1178) 2023-10-09T09:18:55.554 INFO:tasks.rados.rados.0.smithi163.stdout:3210: finishing write tid 3 to smithi16367185-9 2023-10-09T09:18:55.554 INFO:tasks.rados.rados.0.smithi163.stdout:3210: finishing write tid 4 to smithi16367185-9 2023-10-09T09:18:55.554 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 718 (ObjNum 1203 snap 402 seq_num 1203) dirty exists 2023-10-09T09:18:55.554 INFO:tasks.rados.rados.0.smithi163.stdout:3210: left oid 9 (ObjNum 1203 snap 402 seq_num 1203) 2023-10-09T09:18:55.554 INFO:tasks.rados.rados.0.smithi163.stdout:3213: finishing write tid 1 to smithi16367185-31 2023-10-09T09:18:55.555 INFO:tasks.rados.rados.0.smithi163.stdout:3213: finishing write tid 2 to smithi16367185-31 2023-10-09T09:18:55.555 INFO:tasks.rados.rados.0.smithi163.stdout:3213: finishing write tid 3 to smithi16367185-31 2023-10-09T09:18:55.555 INFO:tasks.rados.rados.0.smithi163.stdout:3213: finishing write tid 4 to smithi16367185-31 2023-10-09T09:18:55.555 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 873 (ObjNum 1204 snap 402 seq_num 1204) dirty exists 2023-10-09T09:18:55.555 INFO:tasks.rados.rados.0.smithi163.stdout:3213: left oid 31 (ObjNum 1204 snap 402 seq_num 1204) 2023-10-09T09:18:55.555 INFO:tasks.rados.rados.0.smithi163.stdout:3214: expect (ObjNum 1027 snap 349 seq_num 1027) 2023-10-09T09:18:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[75425]: cluster 2023-10-09T09:18:54.414287+0000 osd.5 (osd.5) 396 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[75425]: cluster 2023-10-09T09:18:54.418617+0000 osd.4 (osd.4) 368 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[75425]: cluster 2023-10-09T09:18:54.420167+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e860: 8 total, 8 up, 8 in 2023-10-09T09:18:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[75425]: cluster 2023-10-09T09:18:54.421710+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[75425]: cluster 2023-10-09T09:18:54.422338+0000 osd.4 (osd.4) 369 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[75425]: cluster 2023-10-09T09:18:55.247347+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e861: 8 total, 8 up, 8 in 2023-10-09T09:18:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[75425]: audit 2023-10-09T09:18:55.347866+0000 mon.a (mon.0) 1058 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[78107]: cluster 2023-10-09T09:18:54.414287+0000 osd.5 (osd.5) 396 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[78107]: cluster 2023-10-09T09:18:54.418617+0000 osd.4 (osd.4) 368 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[78107]: cluster 2023-10-09T09:18:54.420167+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e860: 8 total, 8 up, 8 in 2023-10-09T09:18:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[78107]: cluster 2023-10-09T09:18:54.421710+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[78107]: cluster 2023-10-09T09:18:54.422338+0000 osd.4 (osd.4) 369 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[78107]: cluster 2023-10-09T09:18:55.247347+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e861: 8 total, 8 up, 8 in 2023-10-09T09:18:55.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:55 smithi163 bash[78107]: audit 2023-10-09T09:18:55.347866+0000 mon.a (mon.0) 1058 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:55.749 INFO:tasks.rados.rados.0.smithi163.stdout:3210: done (5 left) 2023-10-09T09:18:55.749 INFO:tasks.rados.rados.0.smithi163.stdout:3211: done (4 left) 2023-10-09T09:18:55.749 INFO:tasks.rados.rados.0.smithi163.stdout:3212: done (3 left) 2023-10-09T09:18:55.749 INFO:tasks.rados.rados.0.smithi163.stdout:3213: done (2 left) 2023-10-09T09:18:55.749 INFO:tasks.rados.rados.0.smithi163.stdout:3214: done (1 left) 2023-10-09T09:18:55.749 INFO:tasks.rados.rados.0.smithi163.stdout:3216: done (0 left) 2023-10-09T09:18:55.749 INFO:tasks.rados.rados.0.smithi163.stdout:3217: write oid 50 current snap is 402 2023-10-09T09:18:55.749 INFO:tasks.rados.rados.0.smithi163.stdout:3217: seq_num 1206 ranges {669242=734068,2188931=80311} 2023-10-09T09:18:55.763 INFO:tasks.rados.rados.0.smithi163.stdout:3217: writing smithi16367185-50 from 669242 to 1403310 tid 1 2023-10-09T09:18:55.768 INFO:tasks.rados.rados.0.smithi163.stdout:3217: writing smithi16367185-50 from 2188931 to 2269242 tid 2 2023-10-09T09:18:55.768 INFO:tasks.rados.rados.0.smithi163.stdout:3218: snap_create 2023-10-09T09:18:55.774 INFO:tasks.rados.rados.0.smithi163.stdout:3217: finishing write tid 1 to smithi16367185-50 2023-10-09T09:18:55.775 INFO:tasks.rados.rados.0.smithi163.stdout:3217: finishing write tid 2 to smithi16367185-50 2023-10-09T09:18:55.775 INFO:tasks.rados.rados.0.smithi163.stdout:3217: finishing write tid 3 to smithi16367185-50 2023-10-09T09:18:55.775 INFO:tasks.rados.rados.0.smithi163.stdout:3217: finishing write tid 4 to smithi16367185-50 2023-10-09T09:18:55.775 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1479 (ObjNum 1206 snap 402 seq_num 1206) dirty exists 2023-10-09T09:18:55.775 INFO:tasks.rados.rados.0.smithi163.stdout:3217: left oid 50 (ObjNum 1206 snap 402 seq_num 1206) 2023-10-09T09:18:55.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:55 smithi181 bash[59345]: cluster 2023-10-09T09:18:54.414287+0000 osd.5 (osd.5) 396 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:55 smithi181 bash[59345]: cluster 2023-10-09T09:18:54.418617+0000 osd.4 (osd.4) 368 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:55 smithi181 bash[59345]: cluster 2023-10-09T09:18:54.420167+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e860: 8 total, 8 up, 8 in 2023-10-09T09:18:55.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:55 smithi181 bash[59345]: cluster 2023-10-09T09:18:54.421710+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:55 smithi181 bash[59345]: cluster 2023-10-09T09:18:54.422338+0000 osd.4 (osd.4) 369 : cluster 3 failed to encode map e860 with expected crc 2023-10-09T09:18:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:55 smithi181 bash[59345]: cluster 2023-10-09T09:18:55.247347+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e861: 8 total, 8 up, 8 in 2023-10-09T09:18:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:55 smithi181 bash[59345]: audit 2023-10-09T09:18:55.347866+0000 mon.a (mon.0) 1058 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:18:56.437 INFO:tasks.rados.rados.0.smithi163.stdout:3217: done (1 left) 2023-10-09T09:18:56.437 INFO:tasks.rados.rados.0.smithi163.stdout:3218: done (0 left) 2023-10-09T09:18:56.437 INFO:tasks.rados.rados.0.smithi163.stdout:3219: rollback oid 11 current snap is 403 2023-10-09T09:18:56.438 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 11 to 393 2023-10-09T09:18:56.438 INFO:tasks.rados.rados.0.smithi163.stdout:3220: read oid 22 snap 397 2023-10-09T09:18:56.438 INFO:tasks.rados.rados.0.smithi163.stdout:3220: expect (ObjNum 1115 snap 368 seq_num 1115) 2023-10-09T09:18:56.438 INFO:tasks.rados.rados.0.smithi163.stdout:3221: snap_remove snap 377 2023-10-09T09:18:56.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:56 smithi163 bash[78107]: cluster 2023-10-09T09:18:55.244274+0000 osd.5 (osd.5) 397 : cluster 3 failed to encode map e861 with expected crc 2023-10-09T09:18:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:56 smithi163 bash[78107]: cluster 2023-10-09T09:18:55.246953+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e861 with expected crc 2023-10-09T09:18:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:56 smithi163 bash[78107]: cluster 2023-10-09T09:18:55.424655+0000 mgr.y (mgr.34107) 371 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 365 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 739 KiB/s wr, 7 op/s 2023-10-09T09:18:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:56 smithi163 bash[75425]: cluster 2023-10-09T09:18:55.244274+0000 osd.5 (osd.5) 397 : cluster 3 failed to encode map e861 with expected crc 2023-10-09T09:18:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:56 smithi163 bash[75425]: cluster 2023-10-09T09:18:55.246953+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e861 with expected crc 2023-10-09T09:18:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:56 smithi163 bash[75425]: cluster 2023-10-09T09:18:55.424655+0000 mgr.y (mgr.34107) 371 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 365 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 739 KiB/s wr, 7 op/s 2023-10-09T09:18:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:56 smithi181 bash[59345]: cluster 2023-10-09T09:18:55.244274+0000 osd.5 (osd.5) 397 : cluster 3 failed to encode map e861 with expected crc 2023-10-09T09:18:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:56 smithi181 bash[59345]: cluster 2023-10-09T09:18:55.246953+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e861 with expected crc 2023-10-09T09:18:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:56 smithi181 bash[59345]: cluster 2023-10-09T09:18:55.424655+0000 mgr.y (mgr.34107) 371 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 365 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 739 KiB/s wr, 7 op/s 2023-10-09T09:18:57.440 INFO:tasks.rados.rados.0.smithi163.stdout:3221: done (2 left) 2023-10-09T09:18:57.440 INFO:tasks.rados.rados.0.smithi163.stdout:3222: snap_remove snap 395 2023-10-09T09:18:57.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[78107]: cluster 2023-10-09T09:18:56.433589+0000 osd.5 (osd.5) 398 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[78107]: cluster 2023-10-09T09:18:56.434021+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[78107]: cluster 2023-10-09T09:18:56.437125+0000 mon.a (mon.0) 1059 : cluster 0 osdmap e862: 8 total, 8 up, 8 in 2023-10-09T09:18:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[78107]: cluster 2023-10-09T09:18:56.438355+0000 osd.5 (osd.5) 399 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[78107]: cluster 2023-10-09T09:18:56.438361+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[78107]: audit 2023-10-09T09:18:56.975791+0000 mon.a (mon.0) 1060 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:18:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[75425]: cluster 2023-10-09T09:18:56.433589+0000 osd.5 (osd.5) 398 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[75425]: cluster 2023-10-09T09:18:56.434021+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[75425]: cluster 2023-10-09T09:18:56.437125+0000 mon.a (mon.0) 1059 : cluster 0 osdmap e862: 8 total, 8 up, 8 in 2023-10-09T09:18:57.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[75425]: cluster 2023-10-09T09:18:56.438355+0000 osd.5 (osd.5) 399 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[75425]: cluster 2023-10-09T09:18:56.438361+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:57 smithi163 bash[75425]: audit 2023-10-09T09:18:56.975791+0000 mon.a (mon.0) 1060 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:18:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:57 smithi181 bash[59345]: cluster 2023-10-09T09:18:56.433589+0000 osd.5 (osd.5) 398 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:57 smithi181 bash[59345]: cluster 2023-10-09T09:18:56.434021+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:57 smithi181 bash[59345]: cluster 2023-10-09T09:18:56.437125+0000 mon.a (mon.0) 1059 : cluster 0 osdmap e862: 8 total, 8 up, 8 in 2023-10-09T09:18:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:57 smithi181 bash[59345]: cluster 2023-10-09T09:18:56.438355+0000 osd.5 (osd.5) 399 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:57 smithi181 bash[59345]: cluster 2023-10-09T09:18:56.438361+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e862 with expected crc 2023-10-09T09:18:57.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:57 smithi181 bash[59345]: audit 2023-10-09T09:18:56.975791+0000 mon.a (mon.0) 1060 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:18:58.447 INFO:tasks.rados.rados.0.smithi163.stdout:3222: done (2 left) 2023-10-09T09:18:58.447 INFO:tasks.rados.rados.0.smithi163.stdout:3223: read oid 48 snap -1 2023-10-09T09:18:58.448 INFO:tasks.rados.rados.0.smithi163.stdout:3223: expect (ObjNum 1161 snap 391 seq_num 1161) 2023-10-09T09:18:58.448 INFO:tasks.rados.rados.0.smithi163.stdout:3224: rollback oid 9 current snap is 403 2023-10-09T09:18:58.448 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 338 2023-10-09T09:18:58.448 INFO:tasks.rados.rados.0.smithi163.stdout:3219: finishing rollback tid 1 to smithi16367185-11 2023-10-09T09:18:58.448 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 823 (ObjNum 669362840 snap 21875 seq_num 3219) dirty dne 2023-10-09T09:18:58.448 INFO:tasks.rados.rados.0.smithi163.stdout:3220: expect (ObjNum 1115 snap 368 seq_num 1115) 2023-10-09T09:18:58.659 INFO:tasks.rados.rados.0.smithi163.stdout:3224: finishing rollback tid 0 to smithi16367185-9 2023-10-09T09:18:58.659 INFO:tasks.rados.rados.0.smithi163.stdout:3224: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:18:58.659 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 727 (ObjNum 0 snap 0 seq_num 697217040) dirty dne 2023-10-09T09:18:58.659 INFO:tasks.rados.rados.0.smithi163.stdout:3223: expect (ObjNum 1161 snap 391 seq_num 1161) 2023-10-09T09:18:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:58 smithi163 bash[75425]: cluster 2023-10-09T09:18:57.425466+0000 mgr.y (mgr.34107) 372 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 362 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2023-10-09T09:18:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:58 smithi163 bash[75425]: cluster 2023-10-09T09:18:57.436157+0000 osd.5 (osd.5) 400 : cluster 3 failed to encode map e863 with expected crc 2023-10-09T09:18:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:58 smithi163 bash[75425]: cluster 2023-10-09T09:18:57.436787+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e863 with expected crc 2023-10-09T09:18:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:58 smithi163 bash[75425]: cluster 2023-10-09T09:18:57.440376+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e863: 8 total, 8 up, 8 in 2023-10-09T09:18:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:58 smithi163 bash[78107]: cluster 2023-10-09T09:18:57.425466+0000 mgr.y (mgr.34107) 372 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 362 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2023-10-09T09:18:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:58 smithi163 bash[78107]: cluster 2023-10-09T09:18:57.436157+0000 osd.5 (osd.5) 400 : cluster 3 failed to encode map e863 with expected crc 2023-10-09T09:18:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:58 smithi163 bash[78107]: cluster 2023-10-09T09:18:57.436787+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e863 with expected crc 2023-10-09T09:18:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:58 smithi163 bash[78107]: cluster 2023-10-09T09:18:57.440376+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e863: 8 total, 8 up, 8 in 2023-10-09T09:18:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:58 smithi181 bash[59345]: cluster 2023-10-09T09:18:57.425466+0000 mgr.y (mgr.34107) 372 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 362 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2023-10-09T09:18:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:58 smithi181 bash[59345]: cluster 2023-10-09T09:18:57.436157+0000 osd.5 (osd.5) 400 : cluster 3 failed to encode map e863 with expected crc 2023-10-09T09:18:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:58 smithi181 bash[59345]: cluster 2023-10-09T09:18:57.436787+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e863 with expected crc 2023-10-09T09:18:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:58 smithi181 bash[59345]: cluster 2023-10-09T09:18:57.440376+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e863: 8 total, 8 up, 8 in 2023-10-09T09:18:58.877 INFO:tasks.rados.rados.0.smithi163.stdout:3219: done (3 left) 2023-10-09T09:18:58.877 INFO:tasks.rados.rados.0.smithi163.stdout:3220: done (2 left) 2023-10-09T09:18:58.878 INFO:tasks.rados.rados.0.smithi163.stdout:3223: done (1 left) 2023-10-09T09:18:58.878 INFO:tasks.rados.rados.0.smithi163.stdout:3224: done (0 left) 2023-10-09T09:18:58.878 INFO:tasks.rados.rados.0.smithi163.stdout:3225: rollback oid 10 current snap is 403 2023-10-09T09:18:58.878 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 10 to 401 2023-10-09T09:18:58.878 INFO:tasks.rados.rados.0.smithi163.stdout:3226: read oid 24 snap -1 2023-10-09T09:18:58.878 INFO:tasks.rados.rados.0.smithi163.stdout:3226: expect deleted 2023-10-09T09:18:58.878 INFO:tasks.rados.rados.0.smithi163.stdout:3227: delete oid 47 current snap is 403 2023-10-09T09:18:58.882 INFO:tasks.rados.rados.0.smithi163.stdout:3226: done (2 left) 2023-10-09T09:18:58.882 INFO:tasks.rados.rados.0.smithi163.stdout:3227: done (1 left) 2023-10-09T09:18:58.882 INFO:tasks.rados.rados.0.smithi163.stdout:3228: delete oid 23 current snap is 403 2023-10-09T09:18:58.883 INFO:tasks.rados.rados.0.smithi163.stdout:3225: finishing rollback tid 0 to smithi16367185-10 2023-10-09T09:18:58.884 INFO:tasks.rados.rados.0.smithi163.stdout:3225: finishing rollback tid 1 to smithi16367185-10 2023-10-09T09:18:58.884 INFO:tasks.rados.rados.0.smithi163.stdout:3225: finishing rollback tid 2 to smithi16367185-10 2023-10-09T09:18:58.885 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 1082 (ObjNum 1156 snap 391 seq_num 1156) dirty exists 2023-10-09T09:18:58.887 INFO:tasks.rados.rados.0.smithi163.stdout:3225: done (1 left) 2023-10-09T09:18:58.888 INFO:tasks.rados.rados.0.smithi163.stdout:3228: done (0 left) 2023-10-09T09:18:58.888 INFO:tasks.rados.rados.0.smithi163.stdout:3229: snap_remove snap 400 2023-10-09T09:18:59.470 INFO:tasks.rados.rados.0.smithi163.stdout:3229: done (0 left) 2023-10-09T09:18:59.470 INFO:tasks.rados.rados.0.smithi163.stdout:3230: snap_create 2023-10-09T09:18:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[78107]: cluster 2023-10-09T09:18:58.441105+0000 osd.5 (osd.5) 401 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[78107]: cluster 2023-10-09T09:18:58.445513+0000 osd.4 (osd.4) 370 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[78107]: cluster 2023-10-09T09:18:58.446099+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[78107]: cluster 2023-10-09T09:18:58.446738+0000 mon.a (mon.0) 1062 : cluster 0 osdmap e864: 8 total, 8 up, 8 in 2023-10-09T09:18:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[78107]: cluster 2023-10-09T09:18:58.446973+0000 osd.5 (osd.5) 402 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[78107]: cluster 2023-10-09T09:18:58.449672+0000 osd.4 (osd.4) 371 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[78107]: cluster 2023-10-09T09:18:58.454395+0000 osd.4 (osd.4) 372 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[75425]: cluster 2023-10-09T09:18:58.441105+0000 osd.5 (osd.5) 401 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[75425]: cluster 2023-10-09T09:18:58.445513+0000 osd.4 (osd.4) 370 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[75425]: cluster 2023-10-09T09:18:58.446099+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[75425]: cluster 2023-10-09T09:18:58.446738+0000 mon.a (mon.0) 1062 : cluster 0 osdmap e864: 8 total, 8 up, 8 in 2023-10-09T09:18:59.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[75425]: cluster 2023-10-09T09:18:58.446973+0000 osd.5 (osd.5) 402 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[75425]: cluster 2023-10-09T09:18:58.449672+0000 osd.4 (osd.4) 371 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:18:59 smithi163 bash[75425]: cluster 2023-10-09T09:18:58.454395+0000 osd.4 (osd.4) 372 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:59 smithi181 bash[59345]: cluster 2023-10-09T09:18:58.441105+0000 osd.5 (osd.5) 401 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:59 smithi181 bash[59345]: cluster 2023-10-09T09:18:58.445513+0000 osd.4 (osd.4) 370 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:59 smithi181 bash[59345]: cluster 2023-10-09T09:18:58.446099+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:59 smithi181 bash[59345]: cluster 2023-10-09T09:18:58.446738+0000 mon.a (mon.0) 1062 : cluster 0 osdmap e864: 8 total, 8 up, 8 in 2023-10-09T09:18:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:59 smithi181 bash[59345]: cluster 2023-10-09T09:18:58.446973+0000 osd.5 (osd.5) 402 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:59 smithi181 bash[59345]: cluster 2023-10-09T09:18:58.449672+0000 osd.4 (osd.4) 371 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:18:59.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:18:59 smithi181 bash[59345]: cluster 2023-10-09T09:18:58.454395+0000 osd.4 (osd.4) 372 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:19:00.252 INFO:tasks.rados.rados.0.smithi163.stdout:3230: done (0 left) 2023-10-09T09:19:00.252 INFO:tasks.rados.rados.0.smithi163.stdout:3231: write oid 30 current snap is 404 2023-10-09T09:19:00.252 INFO:tasks.rados.rados.0.smithi163.stdout:3231: seq_num 1209 ranges {719663=676168,1964377=355286} 2023-10-09T09:19:00.260 INFO:tasks.rados.rados.0.smithi163.stdout:3231: writing smithi16367185-30 from 719663 to 1395831 tid 1 2023-10-09T09:19:00.266 INFO:tasks.rados.rados.0.smithi163.stdout:3231: writing smithi16367185-30 from 1964377 to 2319663 tid 2 2023-10-09T09:19:00.267 INFO:tasks.rados.rados.0.smithi163.stdout:3232: read oid 44 snap 358 2023-10-09T09:19:00.267 INFO:tasks.rados.rados.0.smithi163.stdout:3232: expect (ObjNum 1056 snap 355 seq_num 1056) 2023-10-09T09:19:00.267 INFO:tasks.rados.rados.0.smithi163.stdout:3233: write oid 24 current snap is 404 2023-10-09T09:19:00.267 INFO:tasks.rados.rados.0.smithi163.stdout:3233: seq_num 1210 ranges {736470=751417,2022859=313611} 2023-10-09T09:19:00.275 INFO:tasks.rados.rados.0.smithi163.stdout:3233: writing smithi16367185-24 from 736470 to 1487887 tid 1 2023-10-09T09:19:00.281 INFO:tasks.rados.rados.0.smithi163.stdout:3233: writing smithi16367185-24 from 2022859 to 2336470 tid 2 2023-10-09T09:19:00.281 INFO:tasks.rados.rados.0.smithi163.stdout:3234: read oid 36 snap -1 2023-10-09T09:19:00.281 INFO:tasks.rados.rados.0.smithi163.stdout:3234: expect deleted 2023-10-09T09:19:00.281 INFO:tasks.rados.rados.0.smithi163.stdout:3235: delete oid 16 current snap is 404 2023-10-09T09:19:00.281 INFO:tasks.rados.rados.0.smithi163.stdout:3231: finishing write tid 1 to smithi16367185-30 2023-10-09T09:19:00.281 INFO:tasks.rados.rados.0.smithi163.stdout:3231: finishing write tid 2 to smithi16367185-30 2023-10-09T09:19:00.281 INFO:tasks.rados.rados.0.smithi163.stdout:3231: finishing write tid 3 to smithi16367185-30 2023-10-09T09:19:00.281 INFO:tasks.rados.rados.0.smithi163.stdout:3231: finishing write tid 4 to smithi16367185-30 2023-10-09T09:19:00.282 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1087 (ObjNum 1209 snap 404 seq_num 1209) dirty exists 2023-10-09T09:19:00.282 INFO:tasks.rados.rados.0.smithi163.stdout:3231: left oid 30 (ObjNum 1209 snap 404 seq_num 1209) 2023-10-09T09:19:00.283 INFO:tasks.rados.rados.0.smithi163.stdout:3232: expect (ObjNum 1056 snap 355 seq_num 1056) 2023-10-09T09:19:00.461 INFO:tasks.rados.rados.0.smithi163.stdout:3233: finishing write tid 1 to smithi16367185-24 2023-10-09T09:19:00.461 INFO:tasks.rados.rados.0.smithi163.stdout:3233: finishing write tid 2 to smithi16367185-24 2023-10-09T09:19:00.461 INFO:tasks.rados.rados.0.smithi163.stdout:3233: finishing write tid 3 to smithi16367185-24 2023-10-09T09:19:00.461 INFO:tasks.rados.rados.0.smithi163.stdout:3233: finishing write tid 4 to smithi16367185-24 2023-10-09T09:19:00.461 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 730 (ObjNum 1210 snap 404 seq_num 1210) dirty exists 2023-10-09T09:19:00.461 INFO:tasks.rados.rados.0.smithi163.stdout:3233: left oid 24 (ObjNum 1210 snap 404 seq_num 1210) 2023-10-09T09:19:00.462 INFO:tasks.rados.rados.0.smithi163.stdout:3231: done (4 left) 2023-10-09T09:19:00.462 INFO:tasks.rados.rados.0.smithi163.stdout:3232: done (3 left) 2023-10-09T09:19:00.462 INFO:tasks.rados.rados.0.smithi163.stdout:3233: done (2 left) 2023-10-09T09:19:00.462 INFO:tasks.rados.rados.0.smithi163.stdout:3234: done (1 left) 2023-10-09T09:19:00.462 INFO:tasks.rados.rados.0.smithi163.stdout:3235: done (0 left) 2023-10-09T09:19:00.462 INFO:tasks.rados.rados.0.smithi163.stdout:3236: snap_remove snap 358 2023-10-09T09:19:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[75425]: cluster 2023-10-09T09:18:58.588120+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:19:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[75425]: cluster 2023-10-09T09:18:59.426308+0000 mgr.y (mgr.34107) 373 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 362 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2023-10-09T09:19:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[75425]: cluster 2023-10-09T09:18:59.463980+0000 osd.5 (osd.5) 403 : cluster 3 failed to encode map e865 with expected crc 2023-10-09T09:19:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[75425]: cluster 2023-10-09T09:18:59.464529+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e865 with expected crc 2023-10-09T09:19:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[75425]: cluster 2023-10-09T09:18:59.469440+0000 mon.a (mon.0) 1063 : cluster 0 osdmap e865: 8 total, 8 up, 8 in 2023-10-09T09:19:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[75425]: cluster 2023-10-09T09:19:00.251501+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e866: 8 total, 8 up, 8 in 2023-10-09T09:19:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[75425]: cluster 2023-10-09T09:19:00.280898+0000 osd.4 (osd.4) 373 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[78107]: cluster 2023-10-09T09:18:58.588120+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:19:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[78107]: cluster 2023-10-09T09:18:59.426308+0000 mgr.y (mgr.34107) 373 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 362 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2023-10-09T09:19:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[78107]: cluster 2023-10-09T09:18:59.463980+0000 osd.5 (osd.5) 403 : cluster 3 failed to encode map e865 with expected crc 2023-10-09T09:19:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[78107]: cluster 2023-10-09T09:18:59.464529+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e865 with expected crc 2023-10-09T09:19:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[78107]: cluster 2023-10-09T09:18:59.469440+0000 mon.a (mon.0) 1063 : cluster 0 osdmap e865: 8 total, 8 up, 8 in 2023-10-09T09:19:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[78107]: cluster 2023-10-09T09:19:00.251501+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e866: 8 total, 8 up, 8 in 2023-10-09T09:19:00.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:00 smithi163 bash[78107]: cluster 2023-10-09T09:19:00.280898+0000 osd.4 (osd.4) 373 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:00 smithi181 bash[59345]: cluster 2023-10-09T09:18:58.588120+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e864 with expected crc 2023-10-09T09:19:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:00 smithi181 bash[59345]: cluster 2023-10-09T09:18:59.426308+0000 mgr.y (mgr.34107) 373 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 362 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2023-10-09T09:19:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:00 smithi181 bash[59345]: cluster 2023-10-09T09:18:59.463980+0000 osd.5 (osd.5) 403 : cluster 3 failed to encode map e865 with expected crc 2023-10-09T09:19:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:00 smithi181 bash[59345]: cluster 2023-10-09T09:18:59.464529+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e865 with expected crc 2023-10-09T09:19:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:00 smithi181 bash[59345]: cluster 2023-10-09T09:18:59.469440+0000 mon.a (mon.0) 1063 : cluster 0 osdmap e865: 8 total, 8 up, 8 in 2023-10-09T09:19:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:00 smithi181 bash[59345]: cluster 2023-10-09T09:19:00.251501+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e866: 8 total, 8 up, 8 in 2023-10-09T09:19:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:00 smithi181 bash[59345]: cluster 2023-10-09T09:19:00.280898+0000 osd.4 (osd.4) 373 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.263 INFO:tasks.rados.rados.0.smithi163.stdout:3236: done (0 left) 2023-10-09T09:19:01.264 INFO:tasks.rados.rados.0.smithi163.stdout:3237: write oid 44 current snap is 404 2023-10-09T09:19:01.264 INFO:tasks.rados.rados.0.smithi163.stdout:3237: seq_num 1212 ranges {770084=501915,2023470=346614} 2023-10-09T09:19:01.272 INFO:tasks.rados.rados.0.smithi163.stdout:3237: writing smithi16367185-44 from 770084 to 1271999 tid 1 2023-10-09T09:19:01.279 INFO:tasks.rados.rados.0.smithi163.stdout:3237: writing smithi16367185-44 from 2023470 to 2370084 tid 2 2023-10-09T09:19:01.280 INFO:tasks.rados.rados.0.smithi163.stdout:3238: write oid 17 current snap is 404 2023-10-09T09:19:01.280 INFO:tasks.rados.rados.0.smithi163.stdout:3238: seq_num 1213 ranges {786891=577164,2081952=304939} 2023-10-09T09:19:01.288 INFO:tasks.rados.rados.0.smithi163.stdout:3238: writing smithi16367185-17 from 786891 to 1364055 tid 1 2023-10-09T09:19:01.294 INFO:tasks.rados.rados.0.smithi163.stdout:3238: writing smithi16367185-17 from 2081952 to 2386891 tid 2 2023-10-09T09:19:01.295 INFO:tasks.rados.rados.0.smithi163.stdout:3239: read oid 28 snap -1 2023-10-09T09:19:01.295 INFO:tasks.rados.rados.0.smithi163.stdout:3239: expect (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:19:01.295 INFO:tasks.rados.rados.0.smithi163.stdout:3240: write oid 21 current snap is 404 2023-10-09T09:19:01.295 INFO:tasks.rados.rados.0.smithi163.stdout:3240: seq_num 1214 ranges {403698=652413,1740434=663264} 2023-10-09T09:19:01.301 INFO:tasks.rados.rados.0.smithi163.stdout:3240: writing smithi16367185-21 from 403698 to 1056111 tid 1 2023-10-09T09:19:01.309 INFO:tasks.rados.rados.0.smithi163.stdout:3240: writing smithi16367185-21 from 1740434 to 2403698 tid 2 2023-10-09T09:19:01.309 INFO:tasks.rados.rados.0.smithi163.stdout:3241: write oid 8 current snap is 404 2023-10-09T09:19:01.309 INFO:tasks.rados.rados.0.smithi163.stdout:3237: finishing write tid 1 to smithi16367185-44 2023-10-09T09:19:01.309 INFO:tasks.rados.rados.0.smithi163.stdout:3241: seq_num 1215 ranges {420505=727662,1798916=553091} 2023-10-09T09:19:01.315 INFO:tasks.rados.rados.0.smithi163.stdout:3241: writing smithi16367185-8 from 420505 to 1148167 tid 1 2023-10-09T09:19:01.323 INFO:tasks.rados.rados.0.smithi163.stdout:3241: writing smithi16367185-8 from 1798916 to 2352007 tid 2 2023-10-09T09:19:01.323 INFO:tasks.rados.rados.0.smithi163.stdout:3242: delete oid 26 current snap is 404 2023-10-09T09:19:01.323 INFO:tasks.rados.rados.0.smithi163.stdout:3237: finishing write tid 2 to smithi16367185-44 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:3237: finishing write tid 3 to smithi16367185-44 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:3237: finishing write tid 4 to smithi16367185-44 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 2320 (ObjNum 1212 snap 404 seq_num 1212) dirty exists 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:3237: left oid 44 (ObjNum 1212 snap 404 seq_num 1212) 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:3238: finishing write tid 1 to smithi16367185-17 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:3238: finishing write tid 2 to smithi16367185-17 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:3238: finishing write tid 3 to smithi16367185-17 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:3238: finishing write tid 4 to smithi16367185-17 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1622 (ObjNum 1213 snap 404 seq_num 1213) dirty exists 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:3238: left oid 17 (ObjNum 1213 snap 404 seq_num 1213) 2023-10-09T09:19:01.324 INFO:tasks.rados.rados.0.smithi163.stdout:3239: expect (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:19:01.422 INFO:tasks.rados.rados.0.smithi163.stdout:3240: finishing write tid 1 to smithi16367185-21 2023-10-09T09:19:01.422 INFO:tasks.rados.rados.0.smithi163.stdout:3240: finishing write tid 2 to smithi16367185-21 2023-10-09T09:19:01.422 INFO:tasks.rados.rados.0.smithi163.stdout:3240: finishing write tid 3 to smithi16367185-21 2023-10-09T09:19:01.422 INFO:tasks.rados.rados.0.smithi163.stdout:3240: finishing write tid 4 to smithi16367185-21 2023-10-09T09:19:01.422 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 805 (ObjNum 1214 snap 404 seq_num 1214) dirty exists 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3240: left oid 21 (ObjNum 1214 snap 404 seq_num 1214) 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3241: finishing write tid 1 to smithi16367185-8 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3241: finishing write tid 2 to smithi16367185-8 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3241: finishing write tid 3 to smithi16367185-8 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3241: finishing write tid 4 to smithi16367185-8 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1626 (ObjNum 1215 snap 404 seq_num 1215) dirty exists 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3241: left oid 8 (ObjNum 1215 snap 404 seq_num 1215) 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3237: done (5 left) 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3238: done (4 left) 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3239: done (3 left) 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3240: done (2 left) 2023-10-09T09:19:01.423 INFO:tasks.rados.rados.0.smithi163.stdout:3241: done (1 left) 2023-10-09T09:19:01.424 INFO:tasks.rados.rados.0.smithi163.stdout:3242: done (0 left) 2023-10-09T09:19:01.424 INFO:tasks.rados.rados.0.smithi163.stdout:3243: write oid 3 current snap is 404 2023-10-09T09:19:01.424 INFO:tasks.rados.rados.0.smithi163.stdout:3243: seq_num 1217 ranges {454119=594513,1515880=556760} 2023-10-09T09:19:01.433 INFO:tasks.rados.rados.0.smithi163.stdout:3243: writing smithi16367185-3 from 454119 to 1048632 tid 1 2023-10-09T09:19:01.441 INFO:tasks.rados.rados.0.smithi163.stdout:3243: writing smithi16367185-3 from 1515880 to 2072640 tid 2 2023-10-09T09:19:01.441 INFO:tasks.rados.rados.0.smithi163.stdout:3244: snap_remove snap 376 2023-10-09T09:19:01.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[78107]: cluster 2023-10-09T09:18:59.605583+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e865 with expected crc 2023-10-09T09:19:01.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[78107]: cluster 2023-10-09T09:19:00.246521+0000 osd.5 (osd.5) 404 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[78107]: cluster 2023-10-09T09:19:00.334068+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[78107]: cluster 2023-10-09T09:19:00.336758+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[78107]: cluster 2023-10-09T09:19:00.339473+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[78107]: cluster 2023-10-09T09:19:00.342247+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[78107]: cluster 2023-10-09T09:19:01.263447+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e867: 8 total, 8 up, 8 in 2023-10-09T09:19:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[75425]: cluster 2023-10-09T09:18:59.605583+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e865 with expected crc 2023-10-09T09:19:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[75425]: cluster 2023-10-09T09:19:00.246521+0000 osd.5 (osd.5) 404 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[75425]: cluster 2023-10-09T09:19:00.334068+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[75425]: cluster 2023-10-09T09:19:00.336758+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[75425]: cluster 2023-10-09T09:19:00.339473+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[75425]: cluster 2023-10-09T09:19:00.342247+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:01 smithi163 bash[75425]: cluster 2023-10-09T09:19:01.263447+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e867: 8 total, 8 up, 8 in 2023-10-09T09:19:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:01 smithi181 bash[59345]: cluster 2023-10-09T09:18:59.605583+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e865 with expected crc 2023-10-09T09:19:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:01 smithi181 bash[59345]: cluster 2023-10-09T09:19:00.246521+0000 osd.5 (osd.5) 404 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:01 smithi181 bash[59345]: cluster 2023-10-09T09:19:00.334068+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:01 smithi181 bash[59345]: cluster 2023-10-09T09:19:00.336758+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:01 smithi181 bash[59345]: cluster 2023-10-09T09:19:00.339473+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:01 smithi181 bash[59345]: cluster 2023-10-09T09:19:00.342247+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:01 smithi181 bash[59345]: cluster 2023-10-09T09:19:01.263447+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e867: 8 total, 8 up, 8 in 2023-10-09T09:19:02.269 INFO:tasks.rados.rados.0.smithi163.stdout:3244: done (1 left) 2023-10-09T09:19:02.269 INFO:tasks.rados.rados.0.smithi163.stdout:3245: write oid 32 current snap is 404 2023-10-09T09:19:02.269 INFO:tasks.rados.rados.0.smithi163.stdout:3245: seq_num 1218 ranges {470926=669762,1574362=700418} 2023-10-09T09:19:02.280 INFO:tasks.rados.rados.0.smithi163.stdout:3245: writing smithi16367185-32 from 470926 to 1140688 tid 1 2023-10-09T09:19:02.290 INFO:tasks.rados.rados.0.smithi163.stdout:3245: writing smithi16367185-32 from 1574362 to 2274780 tid 2 2023-10-09T09:19:02.290 INFO:tasks.rados.rados.0.smithi163.stdout:3246: snap_remove snap 401 2023-10-09T09:19:02.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:02 smithi181 bash[59345]: cluster 2023-10-09T09:19:00.635279+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:02.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:02 smithi181 bash[59345]: cluster 2023-10-09T09:19:01.259284+0000 osd.5 (osd.5) 405 : cluster 3 failed to encode map e867 with expected crc 2023-10-09T09:19:02.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:02 smithi181 bash[59345]: cluster 2023-10-09T09:19:01.265328+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e867 with expected crc 2023-10-09T09:19:02.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:02 smithi181 bash[59345]: cluster 2023-10-09T09:19:01.427335+0000 mgr.y (mgr.34107) 374 : cluster 0 pgmap v854: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 255 KiB/s wr, 11 op/s 2023-10-09T09:19:02.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:02 smithi181 bash[59345]: cluster 2023-10-09T09:19:02.268860+0000 mon.a (mon.0) 1066 : cluster 0 osdmap e868: 8 total, 8 up, 8 in 2023-10-09T09:19:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:02 smithi163 bash[78107]: cluster 2023-10-09T09:19:00.635279+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:02 smithi163 bash[78107]: cluster 2023-10-09T09:19:01.259284+0000 osd.5 (osd.5) 405 : cluster 3 failed to encode map e867 with expected crc 2023-10-09T09:19:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:02 smithi163 bash[78107]: cluster 2023-10-09T09:19:01.265328+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e867 with expected crc 2023-10-09T09:19:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:02 smithi163 bash[78107]: cluster 2023-10-09T09:19:01.427335+0000 mgr.y (mgr.34107) 374 : cluster 0 pgmap v854: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 255 KiB/s wr, 11 op/s 2023-10-09T09:19:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:02 smithi163 bash[78107]: cluster 2023-10-09T09:19:02.268860+0000 mon.a (mon.0) 1066 : cluster 0 osdmap e868: 8 total, 8 up, 8 in 2023-10-09T09:19:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:02 smithi163 bash[75425]: cluster 2023-10-09T09:19:00.635279+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e866 with expected crc 2023-10-09T09:19:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:02 smithi163 bash[75425]: cluster 2023-10-09T09:19:01.259284+0000 osd.5 (osd.5) 405 : cluster 3 failed to encode map e867 with expected crc 2023-10-09T09:19:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:02 smithi163 bash[75425]: cluster 2023-10-09T09:19:01.265328+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e867 with expected crc 2023-10-09T09:19:02.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:02 smithi163 bash[75425]: cluster 2023-10-09T09:19:01.427335+0000 mgr.y (mgr.34107) 374 : cluster 0 pgmap v854: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 255 KiB/s wr, 11 op/s 2023-10-09T09:19:02.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:02 smithi163 bash[75425]: cluster 2023-10-09T09:19:02.268860+0000 mon.a (mon.0) 1066 : cluster 0 osdmap e868: 8 total, 8 up, 8 in 2023-10-09T09:19:03.279 INFO:tasks.rados.rados.0.smithi163.stdout:3246: done (2 left) 2023-10-09T09:19:03.279 INFO:tasks.rados.rados.0.smithi163.stdout:3247: read oid 50 snap -1 2023-10-09T09:19:03.279 INFO:tasks.rados.rados.0.smithi163.stdout:3247: expect (ObjNum 1206 snap 402 seq_num 1206) 2023-10-09T09:19:03.279 INFO:tasks.rados.rados.0.smithi163.stdout:3248: write oid 18 current snap is 404 2023-10-09T09:19:03.279 INFO:tasks.rados.rados.0.smithi163.stdout:3243: finishing write tid 1 to smithi16367185-3 2023-10-09T09:19:03.279 INFO:tasks.rados.rados.0.smithi163.stdout:3248: seq_num 1219 ranges {487733=745011,1632844=560429} 2023-10-09T09:19:03.289 INFO:tasks.rados.rados.0.smithi163.stdout:3248: writing smithi16367185-18 from 487733 to 1232744 tid 1 2023-10-09T09:19:03.295 INFO:tasks.rados.rados.0.smithi163.stdout:3248: writing smithi16367185-18 from 1632844 to 2193273 tid 2 2023-10-09T09:19:03.301 INFO:tasks.rados.rados.0.smithi163.stdout:3243: finishing write tid 2 to smithi16367185-3 2023-10-09T09:19:03.302 INFO:tasks.rados.rados.0.smithi163.stdout:3243: finishing write tid 3 to smithi16367185-3 2023-10-09T09:19:03.302 INFO:tasks.rados.rados.0.smithi163.stdout:3243: finishing write tid 4 to smithi16367185-3 2023-10-09T09:19:03.302 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 891 (ObjNum 1217 snap 404 seq_num 1217) dirty exists 2023-10-09T09:19:03.302 INFO:tasks.rados.rados.0.smithi163.stdout:3243: left oid 3 (ObjNum 1217 snap 404 seq_num 1217) 2023-10-09T09:19:03.302 INFO:tasks.rados.rados.0.smithi163.stdout:3245: finishing write tid 1 to smithi16367185-32 2023-10-09T09:19:03.302 INFO:tasks.rados.rados.0.smithi163.stdout:3245: finishing write tid 2 to smithi16367185-32 2023-10-09T09:19:03.302 INFO:tasks.rados.rados.0.smithi163.stdout:3245: finishing write tid 3 to smithi16367185-32 2023-10-09T09:19:03.302 INFO:tasks.rados.rados.0.smithi163.stdout:3245: finishing write tid 4 to smithi16367185-32 2023-10-09T09:19:03.302 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 2342 (ObjNum 1218 snap 404 seq_num 1218) dirty exists 2023-10-09T09:19:03.303 INFO:tasks.rados.rados.0.smithi163.stdout:3245: left oid 32 (ObjNum 1218 snap 404 seq_num 1218) 2023-10-09T09:19:03.303 INFO:tasks.rados.rados.0.smithi163.stdout:3247: expect (ObjNum 1206 snap 402 seq_num 1206) 2023-10-09T09:19:03.411 INFO:tasks.rados.rados.0.smithi163.stdout:3248: finishing write tid 1 to smithi16367185-18 2023-10-09T09:19:03.411 INFO:tasks.rados.rados.0.smithi163.stdout:3248: finishing write tid 2 to smithi16367185-18 2023-10-09T09:19:03.411 INFO:tasks.rados.rados.0.smithi163.stdout:3248: finishing write tid 3 to smithi16367185-18 2023-10-09T09:19:03.411 INFO:tasks.rados.rados.0.smithi163.stdout:3248: finishing write tid 4 to smithi16367185-18 2023-10-09T09:19:03.411 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1101 (ObjNum 1219 snap 404 seq_num 1219) dirty exists 2023-10-09T09:19:03.412 INFO:tasks.rados.rados.0.smithi163.stdout:3248: left oid 18 (ObjNum 1219 snap 404 seq_num 1219) 2023-10-09T09:19:03.412 INFO:tasks.rados.rados.0.smithi163.stdout:3243: done (3 left) 2023-10-09T09:19:03.412 INFO:tasks.rados.rados.0.smithi163.stdout:3245: done (2 left) 2023-10-09T09:19:03.412 INFO:tasks.rados.rados.0.smithi163.stdout:3247: done (1 left) 2023-10-09T09:19:03.412 INFO:tasks.rados.rados.0.smithi163.stdout:3248: done (0 left) 2023-10-09T09:19:03.412 INFO:tasks.rados.rados.0.smithi163.stdout:3249: rollback oid 21 current snap is 404 2023-10-09T09:19:03.412 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 21 to 370 2023-10-09T09:19:03.413 INFO:tasks.rados.rados.0.smithi163.stdout:3250: read oid 42 snap -1 2023-10-09T09:19:03.413 INFO:tasks.rados.rados.0.smithi163.stdout:3250: expect (ObjNum 1168 snap 394 seq_num 1168) 2023-10-09T09:19:03.413 INFO:tasks.rados.rados.0.smithi163.stdout:3251: snap_create 2023-10-09T09:19:03.414 INFO:tasks.rados.rados.0.smithi163.stdout:3249: finishing rollback tid 0 to smithi16367185-21 2023-10-09T09:19:03.415 INFO:tasks.rados.rados.0.smithi163.stdout:3249: finishing rollback tid 1 to smithi16367185-21 2023-10-09T09:19:03.415 INFO:tasks.rados.rados.0.smithi163.stdout:3249: finishing rollback tid 2 to smithi16367185-21 2023-10-09T09:19:03.415 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 814 (ObjNum 936 snap 320 seq_num 936) dirty exists 2023-10-09T09:19:03.425 INFO:tasks.rados.rados.0.smithi163.stdout:3250: expect (ObjNum 1168 snap 394 seq_num 1168) 2023-10-09T09:19:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:19:03] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:19:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[78107]: cluster 2023-10-09T09:19:02.265778+0000 osd.5 (osd.5) 406 : cluster 3 failed to encode map e868 with expected crc 2023-10-09T09:19:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[78107]: cluster 2023-10-09T09:19:02.265913+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e868 with expected crc 2023-10-09T09:19:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[78107]: audit 2023-10-09T09:19:03.025120+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[78107]: audit 2023-10-09T09:19:03.033175+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[78107]: audit 2023-10-09T09:19:03.227999+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[78107]: audit 2023-10-09T09:19:03.238020+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[78107]: cluster 2023-10-09T09:19:03.278683+0000 mon.a (mon.0) 1071 : cluster 0 osdmap e869: 8 total, 8 up, 8 in 2023-10-09T09:19:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[75425]: cluster 2023-10-09T09:19:02.265778+0000 osd.5 (osd.5) 406 : cluster 3 failed to encode map e868 with expected crc 2023-10-09T09:19:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[75425]: cluster 2023-10-09T09:19:02.265913+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e868 with expected crc 2023-10-09T09:19:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[75425]: audit 2023-10-09T09:19:03.025120+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[75425]: audit 2023-10-09T09:19:03.033175+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[75425]: audit 2023-10-09T09:19:03.227999+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[75425]: audit 2023-10-09T09:19:03.238020+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:03 smithi163 bash[75425]: cluster 2023-10-09T09:19:03.278683+0000 mon.a (mon.0) 1071 : cluster 0 osdmap e869: 8 total, 8 up, 8 in 2023-10-09T09:19:03.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:03 smithi181 bash[59345]: cluster 2023-10-09T09:19:02.265778+0000 osd.5 (osd.5) 406 : cluster 3 failed to encode map e868 with expected crc 2023-10-09T09:19:03.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:03 smithi181 bash[59345]: cluster 2023-10-09T09:19:02.265913+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e868 with expected crc 2023-10-09T09:19:03.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:03 smithi181 bash[59345]: audit 2023-10-09T09:19:03.025120+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:03 smithi181 bash[59345]: audit 2023-10-09T09:19:03.033175+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:03 smithi181 bash[59345]: audit 2023-10-09T09:19:03.227999+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:03 smithi181 bash[59345]: audit 2023-10-09T09:19:03.238020+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:03.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:03 smithi181 bash[59345]: cluster 2023-10-09T09:19:03.278683+0000 mon.a (mon.0) 1071 : cluster 0 osdmap e869: 8 total, 8 up, 8 in 2023-10-09T09:19:04.281 INFO:tasks.rados.rados.0.smithi163.stdout:3249: done (2 left) 2023-10-09T09:19:04.282 INFO:tasks.rados.rados.0.smithi163.stdout:3250: done (1 left) 2023-10-09T09:19:04.282 INFO:tasks.rados.rados.0.smithi163.stdout:3251: done (0 left) 2023-10-09T09:19:04.282 INFO:tasks.rados.rados.0.smithi163.stdout:3252: snap_create 2023-10-09T09:19:04.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:04 smithi181 bash[59345]: cluster 2023-10-09T09:19:03.272529+0000 osd.5 (osd.5) 407 : cluster 3 failed to encode map e869 with expected crc 2023-10-09T09:19:04.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:04 smithi181 bash[59345]: cluster 2023-10-09T09:19:03.298820+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e869 with expected crc 2023-10-09T09:19:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:04 smithi181 bash[59345]: cluster 2023-10-09T09:19:03.427845+0000 mgr.y (mgr.34107) 375 : cluster 0 pgmap v857: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 284 KiB/s wr, 9 op/s 2023-10-09T09:19:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:04 smithi181 bash[59345]: audit 2023-10-09T09:19:03.763002+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:19:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:04 smithi181 bash[59345]: audit 2023-10-09T09:19:03.764697+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:19:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:04 smithi181 bash[59345]: audit 2023-10-09T09:19:03.773588+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:04 smithi181 bash[59345]: cluster 2023-10-09T09:19:04.280854+0000 mon.a (mon.0) 1075 : cluster 0 osdmap e870: 8 total, 8 up, 8 in 2023-10-09T09:19:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[78107]: cluster 2023-10-09T09:19:03.272529+0000 osd.5 (osd.5) 407 : cluster 3 failed to encode map e869 with expected crc 2023-10-09T09:19:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[78107]: cluster 2023-10-09T09:19:03.298820+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e869 with expected crc 2023-10-09T09:19:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[78107]: cluster 2023-10-09T09:19:03.427845+0000 mgr.y (mgr.34107) 375 : cluster 0 pgmap v857: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 284 KiB/s wr, 9 op/s 2023-10-09T09:19:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[78107]: audit 2023-10-09T09:19:03.763002+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:19:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[78107]: audit 2023-10-09T09:19:03.764697+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:19:04.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[78107]: audit 2023-10-09T09:19:03.773588+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:04.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[78107]: cluster 2023-10-09T09:19:04.280854+0000 mon.a (mon.0) 1075 : cluster 0 osdmap e870: 8 total, 8 up, 8 in 2023-10-09T09:19:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[75425]: cluster 2023-10-09T09:19:03.272529+0000 osd.5 (osd.5) 407 : cluster 3 failed to encode map e869 with expected crc 2023-10-09T09:19:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[75425]: cluster 2023-10-09T09:19:03.298820+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e869 with expected crc 2023-10-09T09:19:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[75425]: cluster 2023-10-09T09:19:03.427845+0000 mgr.y (mgr.34107) 375 : cluster 0 pgmap v857: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 284 KiB/s wr, 9 op/s 2023-10-09T09:19:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[75425]: audit 2023-10-09T09:19:03.763002+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:19:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[75425]: audit 2023-10-09T09:19:03.764697+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:19:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[75425]: audit 2023-10-09T09:19:03.773588+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:19:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:04 smithi163 bash[75425]: cluster 2023-10-09T09:19:04.280854+0000 mon.a (mon.0) 1075 : cluster 0 osdmap e870: 8 total, 8 up, 8 in 2023-10-09T09:19:05.257 INFO:tasks.rados.rados.0.smithi163.stdout:3252: done (0 left) 2023-10-09T09:19:05.257 INFO:tasks.rados.rados.0.smithi163.stdout:3253: read oid 12 snap -1 2023-10-09T09:19:05.257 INFO:tasks.rados.rados.0.smithi163.stdout:3253: expect (ObjNum 1164 snap 392 seq_num 1164) 2023-10-09T09:19:05.257 INFO:tasks.rados.rados.0.smithi163.stdout:3254: write oid 27 current snap is 406 2023-10-09T09:19:05.257 INFO:tasks.rados.rados.0.smithi163.stdout:3254: seq_num 1220 ranges {504540=420260,1574973=704087} 2023-10-09T09:19:05.264 INFO:tasks.rados.rados.0.smithi163.stdout:3254: writing smithi16367185-27 from 504540 to 924800 tid 1 2023-10-09T09:19:05.272 INFO:tasks.rados.rados.0.smithi163.stdout:3254: writing smithi16367185-27 from 1574973 to 2279060 tid 2 2023-10-09T09:19:05.272 INFO:tasks.rados.rados.0.smithi163.stdout:3255: rollback oid 3 current snap is 406 2023-10-09T09:19:05.273 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 405 2023-10-09T09:19:05.273 INFO:tasks.rados.rados.0.smithi163.stdout:3253: expect (ObjNum 1164 snap 392 seq_num 1164) 2023-10-09T09:19:05.398 INFO:tasks.rados.rados.0.smithi163.stdout:3254: finishing write tid 1 to smithi16367185-27 2023-10-09T09:19:05.398 INFO:tasks.rados.rados.0.smithi163.stdout:3254: finishing write tid 2 to smithi16367185-27 2023-10-09T09:19:05.398 INFO:tasks.rados.rados.0.smithi163.stdout:3254: finishing write tid 3 to smithi16367185-27 2023-10-09T09:19:05.398 INFO:tasks.rados.rados.0.smithi163.stdout:3254: finishing write tid 4 to smithi16367185-27 2023-10-09T09:19:05.398 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 975 (ObjNum 1220 snap 406 seq_num 1220) dirty exists 2023-10-09T09:19:05.398 INFO:tasks.rados.rados.0.smithi163.stdout:3254: left oid 27 (ObjNum 1220 snap 406 seq_num 1220) 2023-10-09T09:19:05.399 INFO:tasks.rados.rados.0.smithi163.stdout:3255: finishing rollback tid 0 to smithi16367185-3 2023-10-09T09:19:05.399 INFO:tasks.rados.rados.0.smithi163.stdout:3255: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:19:05.399 INFO:tasks.rados.rados.0.smithi163.stdout:3255: finishing rollback tid 2 to smithi16367185-3 2023-10-09T09:19:05.399 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 901 (ObjNum 1217 snap 404 seq_num 1217) dirty exists 2023-10-09T09:19:05.399 INFO:tasks.rados.rados.0.smithi163.stdout:3253: done (2 left) 2023-10-09T09:19:05.399 INFO:tasks.rados.rados.0.smithi163.stdout:3254: done (1 left) 2023-10-09T09:19:05.399 INFO:tasks.rados.rados.0.smithi163.stdout:3255: done (0 left) 2023-10-09T09:19:05.399 INFO:tasks.rados.rados.0.smithi163.stdout:3256: rollback oid 39 current snap is 406 2023-10-09T09:19:05.399 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 338 2023-10-09T09:19:05.399 INFO:tasks.rados.rados.0.smithi163.stdout:3257: snap_remove snap 399 2023-10-09T09:19:05.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:05 smithi181 bash[59345]: cluster 2023-10-09T09:19:04.278062+0000 osd.5 (osd.5) 408 : cluster 3 failed to encode map e870 with expected crc 2023-10-09T09:19:05.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:05 smithi181 bash[59345]: cluster 2023-10-09T09:19:04.376262+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e870 with expected crc 2023-10-09T09:19:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:05 smithi181 bash[59345]: cluster 2023-10-09T09:19:04.376313+0000 osd.4 (osd.4) 374 : cluster 3 failed to encode map e870 with expected crc 2023-10-09T09:19:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:05 smithi181 bash[59345]: cluster 2023-10-09T09:19:05.255581+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e871: 8 total, 8 up, 8 in 2023-10-09T09:19:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:05 smithi181 bash[59345]: cluster 2023-10-09T09:19:05.268800+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e871 with expected crc 2023-10-09T09:19:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:05 smithi181 bash[59345]: cluster 2023-10-09T09:19:05.276752+0000 osd.4 (osd.4) 375 : cluster 3 failed to encode map e871 with expected crc 2023-10-09T09:19:05.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[78107]: cluster 2023-10-09T09:19:04.278062+0000 osd.5 (osd.5) 408 : cluster 3 failed to encode map e870 with expected crc 2023-10-09T09:19:05.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[78107]: cluster 2023-10-09T09:19:04.376262+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e870 with expected crc 2023-10-09T09:19:05.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[78107]: cluster 2023-10-09T09:19:04.376313+0000 osd.4 (osd.4) 374 : cluster 3 failed to encode map e870 with expected crc 2023-10-09T09:19:05.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[78107]: cluster 2023-10-09T09:19:05.255581+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e871: 8 total, 8 up, 8 in 2023-10-09T09:19:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[78107]: cluster 2023-10-09T09:19:05.268800+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e871 with expected crc 2023-10-09T09:19:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[78107]: cluster 2023-10-09T09:19:05.276752+0000 osd.4 (osd.4) 375 : cluster 3 failed to encode map e871 with expected crc 2023-10-09T09:19:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[75425]: cluster 2023-10-09T09:19:04.278062+0000 osd.5 (osd.5) 408 : cluster 3 failed to encode map e870 with expected crc 2023-10-09T09:19:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[75425]: cluster 2023-10-09T09:19:04.376262+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e870 with expected crc 2023-10-09T09:19:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[75425]: cluster 2023-10-09T09:19:04.376313+0000 osd.4 (osd.4) 374 : cluster 3 failed to encode map e870 with expected crc 2023-10-09T09:19:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[75425]: cluster 2023-10-09T09:19:05.255581+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e871: 8 total, 8 up, 8 in 2023-10-09T09:19:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[75425]: cluster 2023-10-09T09:19:05.268800+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e871 with expected crc 2023-10-09T09:19:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:05 smithi163 bash[75425]: cluster 2023-10-09T09:19:05.276752+0000 osd.4 (osd.4) 375 : cluster 3 failed to encode map e871 with expected crc 2023-10-09T09:19:06.275 INFO:tasks.rados.rados.0.smithi163.stdout:3257: done (1 left) 2023-10-09T09:19:06.275 INFO:tasks.rados.rados.0.smithi163.stdout:3258: read oid 49 snap -1 2023-10-09T09:19:06.275 INFO:tasks.rados.rados.0.smithi163.stdout:3258: expect (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:19:06.275 INFO:tasks.rados.rados.0.smithi163.stdout:3259: read oid 26 snap -1 2023-10-09T09:19:06.275 INFO:tasks.rados.rados.0.smithi163.stdout:3259: expect deleted 2023-10-09T09:19:06.275 INFO:tasks.rados.rados.0.smithi163.stdout:3260: rollback oid 38 current snap is 406 2023-10-09T09:19:06.275 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 403 2023-10-09T09:19:06.275 INFO:tasks.rados.rados.0.smithi163.stdout:3261: read oid 10 snap 402 2023-10-09T09:19:06.275 INFO:tasks.rados.rados.0.smithi163.stdout:3261: expect (ObjNum 1156 snap 391 seq_num 1156) 2023-10-09T09:19:06.276 INFO:tasks.rados.rados.0.smithi163.stdout:3262: read oid 20 snap -1 2023-10-09T09:19:06.276 INFO:tasks.rados.rados.0.smithi163.stdout:3262: expect (ObjNum 1139 snap 380 seq_num 1139) 2023-10-09T09:19:06.276 INFO:tasks.rados.rados.0.smithi163.stdout:3263: snap_create 2023-10-09T09:19:06.276 INFO:tasks.rados.rados.0.smithi163.stdout:3256: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:19:06.276 INFO:tasks.rados.rados.0.smithi163.stdout:3256: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:19:06.276 INFO:tasks.rados.rados.0.smithi163.stdout:3256: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:19:06.277 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1513 (ObjNum 932 snap 317 seq_num 932) dirty exists 2023-10-09T09:19:06.282 INFO:tasks.rados.rados.0.smithi163.stdout:3258: expect (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:19:06.431 INFO:tasks.rados.rados.0.smithi163.stdout:3261: expect (ObjNum 1156 snap 391 seq_num 1156) 2023-10-09T09:19:06.634 INFO:tasks.rados.rados.0.smithi163.stdout:3262: expect (ObjNum 1139 snap 380 seq_num 1139) 2023-10-09T09:19:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:06 smithi181 bash[59345]: cluster 2023-10-09T09:19:05.251838+0000 osd.5 (osd.5) 409 : cluster 3 failed to encode map e871 with expected crc 2023-10-09T09:19:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:06 smithi181 bash[59345]: cluster 2023-10-09T09:19:05.428407+0000 mgr.y (mgr.34107) 376 : cluster 0 pgmap v860: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2023-10-09T09:19:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:06 smithi181 bash[59345]: cluster 2023-10-09T09:19:06.270593+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e872: 8 total, 8 up, 8 in 2023-10-09T09:19:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:06 smithi181 bash[59345]: cluster 2023-10-09T09:19:06.272356+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:06 smithi181 bash[59345]: cluster 2023-10-09T09:19:06.275318+0000 osd.4 (osd.4) 376 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:06 smithi181 bash[59345]: cluster 2023-10-09T09:19:06.281736+0000 osd.4 (osd.4) 377 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:06.821 INFO:tasks.rados.rados.0.smithi163.stdout:3260: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:19:06.821 INFO:tasks.rados.rados.0.smithi163.stdout:3260: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:19:06.821 INFO:tasks.rados.rados.0.smithi163.stdout:3260: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:19:06.821 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1551 (ObjNum 1196 snap 402 seq_num 1196) dirty exists 2023-10-09T09:19:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[78107]: cluster 2023-10-09T09:19:05.251838+0000 osd.5 (osd.5) 409 : cluster 3 failed to encode map e871 with expected crc 2023-10-09T09:19:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[78107]: cluster 2023-10-09T09:19:05.428407+0000 mgr.y (mgr.34107) 376 : cluster 0 pgmap v860: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2023-10-09T09:19:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[78107]: cluster 2023-10-09T09:19:06.270593+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e872: 8 total, 8 up, 8 in 2023-10-09T09:19:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[78107]: cluster 2023-10-09T09:19:06.272356+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[78107]: cluster 2023-10-09T09:19:06.275318+0000 osd.4 (osd.4) 376 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:06.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[78107]: cluster 2023-10-09T09:19:06.281736+0000 osd.4 (osd.4) 377 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[75425]: cluster 2023-10-09T09:19:05.251838+0000 osd.5 (osd.5) 409 : cluster 3 failed to encode map e871 with expected crc 2023-10-09T09:19:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[75425]: cluster 2023-10-09T09:19:05.428407+0000 mgr.y (mgr.34107) 376 : cluster 0 pgmap v860: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2023-10-09T09:19:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[75425]: cluster 2023-10-09T09:19:06.270593+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e872: 8 total, 8 up, 8 in 2023-10-09T09:19:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[75425]: cluster 2023-10-09T09:19:06.272356+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[75425]: cluster 2023-10-09T09:19:06.275318+0000 osd.4 (osd.4) 376 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:06 smithi163 bash[75425]: cluster 2023-10-09T09:19:06.281736+0000 osd.4 (osd.4) 377 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:07.276 INFO:tasks.rados.rados.0.smithi163.stdout:3256: done (6 left) 2023-10-09T09:19:07.276 INFO:tasks.rados.rados.0.smithi163.stdout:3258: done (5 left) 2023-10-09T09:19:07.276 INFO:tasks.rados.rados.0.smithi163.stdout:3259: done (4 left) 2023-10-09T09:19:07.276 INFO:tasks.rados.rados.0.smithi163.stdout:3260: done (3 left) 2023-10-09T09:19:07.276 INFO:tasks.rados.rados.0.smithi163.stdout:3261: done (2 left) 2023-10-09T09:19:07.276 INFO:tasks.rados.rados.0.smithi163.stdout:3262: done (1 left) 2023-10-09T09:19:07.276 INFO:tasks.rados.rados.0.smithi163.stdout:3263: done (0 left) 2023-10-09T09:19:07.276 INFO:tasks.rados.rados.0.smithi163.stdout:3264: delete oid 47 current snap is 407 2023-10-09T09:19:07.283 INFO:tasks.rados.rados.0.smithi163.stdout:3264: done (0 left) 2023-10-09T09:19:07.283 INFO:tasks.rados.rados.0.smithi163.stdout:3265: rollback oid 17 current snap is 407 2023-10-09T09:19:07.283 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 406 2023-10-09T09:19:07.283 INFO:tasks.rados.rados.0.smithi163.stdout:3266: read oid 18 snap -1 2023-10-09T09:19:07.283 INFO:tasks.rados.rados.0.smithi163.stdout:3266: expect (ObjNum 1219 snap 404 seq_num 1219) 2023-10-09T09:19:07.284 INFO:tasks.rados.rados.0.smithi163.stdout:3267: read oid 19 snap -1 2023-10-09T09:19:07.284 INFO:tasks.rados.rados.0.smithi163.stdout:3267: expect deleted 2023-10-09T09:19:07.284 INFO:tasks.rados.rados.0.smithi163.stdout:3268: read oid 23 snap -1 2023-10-09T09:19:07.284 INFO:tasks.rados.rados.0.smithi163.stdout:3268: expect deleted 2023-10-09T09:19:07.284 INFO:tasks.rados.rados.0.smithi163.stdout:3269: rollback oid 40 current snap is 407 2023-10-09T09:19:07.284 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 40 to 398 2023-10-09T09:19:07.284 INFO:tasks.rados.rados.0.smithi163.stdout:3270: snap_create 2023-10-09T09:19:07.290 INFO:tasks.rados.rados.0.smithi163.stdout:3266: expect (ObjNum 1219 snap 404 seq_num 1219) 2023-10-09T09:19:07.411 INFO:tasks.rados.rados.0.smithi163.stdout:3265: finishing rollback tid 0 to smithi16367185-17 2023-10-09T09:19:07.411 INFO:tasks.rados.rados.0.smithi163.stdout:3265: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:19:07.411 INFO:tasks.rados.rados.0.smithi163.stdout:3265: finishing rollback tid 2 to smithi16367185-17 2023-10-09T09:19:07.411 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1638 (ObjNum 1213 snap 404 seq_num 1213) dirty exists 2023-10-09T09:19:07.411 INFO:tasks.rados.rados.0.smithi163.stdout:3269: finishing rollback tid 0 to smithi16367185-40 2023-10-09T09:19:07.411 INFO:tasks.rados.rados.0.smithi163.stdout:3269: finishing rollback tid 1 to smithi16367185-40 2023-10-09T09:19:07.411 INFO:tasks.rados.rados.0.smithi163.stdout:3269: finishing rollback tid 2 to smithi16367185-40 2023-10-09T09:19:07.412 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1642 (ObjNum 1171 snap 394 seq_num 1171) dirty exists 2023-10-09T09:19:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:07 smithi181 bash[59345]: cluster 2023-10-09T09:19:06.267869+0000 osd.5 (osd.5) 410 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:07 smithi181 bash[59345]: cluster 2023-10-09T09:19:06.271341+0000 osd.5 (osd.5) 411 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:07 smithi181 bash[59345]: cluster 2023-10-09T09:19:06.274874+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:07 smithi181 bash[59345]: cluster 2023-10-09T09:19:07.275872+0000 mon.a (mon.0) 1078 : cluster 0 osdmap e873: 8 total, 8 up, 8 in 2023-10-09T09:19:07.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:07 smithi181 bash[59345]: cluster 2023-10-09T09:19:07.284133+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e873 with expected crc 2023-10-09T09:19:07.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:07 smithi163 bash[78107]: cluster 2023-10-09T09:19:06.267869+0000 osd.5 (osd.5) 410 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:07.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:07 smithi163 bash[78107]: cluster 2023-10-09T09:19:06.271341+0000 osd.5 (osd.5) 411 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:07.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:07 smithi163 bash[78107]: cluster 2023-10-09T09:19:06.274874+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:07.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:07 smithi163 bash[78107]: cluster 2023-10-09T09:19:07.275872+0000 mon.a (mon.0) 1078 : cluster 0 osdmap e873: 8 total, 8 up, 8 in 2023-10-09T09:19:07.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:07 smithi163 bash[78107]: cluster 2023-10-09T09:19:07.284133+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e873 with expected crc 2023-10-09T09:19:07.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:07 smithi163 bash[75425]: cluster 2023-10-09T09:19:06.267869+0000 osd.5 (osd.5) 410 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:07.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:07 smithi163 bash[75425]: cluster 2023-10-09T09:19:06.271341+0000 osd.5 (osd.5) 411 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:07.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:07 smithi163 bash[75425]: cluster 2023-10-09T09:19:06.274874+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e872 with expected crc 2023-10-09T09:19:07.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:07 smithi163 bash[75425]: cluster 2023-10-09T09:19:07.275872+0000 mon.a (mon.0) 1078 : cluster 0 osdmap e873: 8 total, 8 up, 8 in 2023-10-09T09:19:07.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:07 smithi163 bash[75425]: cluster 2023-10-09T09:19:07.284133+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e873 with expected crc 2023-10-09T09:19:08.281 INFO:tasks.rados.rados.0.smithi163.stdout:3265: done (5 left) 2023-10-09T09:19:08.469 INFO:tasks.rados.rados.0.smithi163.stdout:3266: done (4 left) 2023-10-09T09:19:08.469 INFO:tasks.rados.rados.0.smithi163.stdout:3267: done (3 left) 2023-10-09T09:19:08.469 INFO:tasks.rados.rados.0.smithi163.stdout:3268: done (2 left) 2023-10-09T09:19:08.469 INFO:tasks.rados.rados.0.smithi163.stdout:3269: done (1 left) 2023-10-09T09:19:08.469 INFO:tasks.rados.rados.0.smithi163.stdout:3270: done (0 left) 2023-10-09T09:19:08.469 INFO:tasks.rados.rados.0.smithi163.stdout:3271: write oid 17 current snap is 408 2023-10-09T09:19:08.470 INFO:tasks.rados.rados.0.smithi163.stdout:3271: seq_num 1222 ranges {538154=570758,1691937=707756} 2023-10-09T09:19:08.472 INFO:tasks.rados.rados.0.smithi163.stdout:3271: writing smithi16367185-17 from 538154 to 1108912 tid 1 2023-10-09T09:19:08.472 INFO:tasks.rados.rados.0.smithi163.stdout:3271: writing smithi16367185-17 from 1691937 to 2399693 tid 2 2023-10-09T09:19:08.472 INFO:tasks.rados.rados.0.smithi163.stdout:3272: write oid 21 current snap is 408 2023-10-09T09:19:08.472 INFO:tasks.rados.rados.0.smithi163.stdout:3272: seq_num 1223 ranges {554961=646007,1750419=451414} 2023-10-09T09:19:08.472 INFO:tasks.rados.rados.0.smithi163.stdout:3272: writing smithi16367185-21 from 554961 to 1200968 tid 1 2023-10-09T09:19:08.472 INFO:tasks.rados.rados.0.smithi163.stdout:3272: writing smithi16367185-21 from 1750419 to 2201833 tid 2 2023-10-09T09:19:08.472 INFO:tasks.rados.rados.0.smithi163.stdout:3273: write oid 47 current snap is 408 2023-10-09T09:19:08.472 INFO:tasks.rados.rados.0.smithi163.stdout:3271: finishing write tid 1 to smithi16367185-17 2023-10-09T09:19:08.472 INFO:tasks.rados.rados.0.smithi163.stdout:3271: finishing write tid 2 to smithi16367185-17 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3271: finishing write tid 3 to smithi16367185-17 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3271: finishing write tid 4 to smithi16367185-17 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1646 (ObjNum 1222 snap 408 seq_num 1222) dirty exists 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3271: left oid 17 (ObjNum 1222 snap 408 seq_num 1222) 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3273: seq_num 1224 ranges {571768=437609,1808901=711425} 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3273: writing smithi16367185-47 from 571768 to 1009377 tid 1 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3273: writing smithi16367185-47 from 1808901 to 2520326 tid 2 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3271: done (2 left) 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3274: delete oid 10 current snap is 408 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3272: finishing write tid 1 to smithi16367185-21 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3272: finishing write tid 2 to smithi16367185-21 2023-10-09T09:19:08.473 INFO:tasks.rados.rados.0.smithi163.stdout:3272: finishing write tid 3 to smithi16367185-21 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:3272: finishing write tid 4 to smithi16367185-21 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 820 (ObjNum 1223 snap 408 seq_num 1223) dirty exists 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:3272: left oid 21 (ObjNum 1223 snap 408 seq_num 1223) 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:3273: finishing write tid 1 to smithi16367185-47 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:3272: done (2 left) 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:3274: done (1 left) 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:3275: delete oid 31 current snap is 408 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:3275: done (1 left) 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:3276: snap_create 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:3273: finishing write tid 2 to smithi16367185-47 2023-10-09T09:19:08.474 INFO:tasks.rados.rados.0.smithi163.stdout:3273: finishing write tid 3 to smithi16367185-47 2023-10-09T09:19:08.475 INFO:tasks.rados.rados.0.smithi163.stdout:3273: finishing write tid 4 to smithi16367185-47 2023-10-09T09:19:08.475 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 2360 (ObjNum 1224 snap 408 seq_num 1224) dirty exists 2023-10-09T09:19:08.475 INFO:tasks.rados.rados.0.smithi163.stdout:3273: left oid 47 (ObjNum 1224 snap 408 seq_num 1224) 2023-10-09T09:19:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:08 smithi181 bash[59345]: cluster 2023-10-09T09:19:07.270707+0000 osd.5 (osd.5) 412 : cluster 3 failed to encode map e873 with expected crc 2023-10-09T09:19:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:08 smithi181 bash[59345]: cluster 2023-10-09T09:19:07.429037+0000 mgr.y (mgr.34107) 377 : cluster 0 pgmap v863: 89 pgs: 4 active+clean+snaptrim, 85 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.2 MiB/s wr, 20 op/s 2023-10-09T09:19:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:08 smithi181 bash[59345]: cluster 2023-10-09T09:19:08.281601+0000 mon.a (mon.0) 1079 : cluster 0 osdmap e874: 8 total, 8 up, 8 in 2023-10-09T09:19:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:08 smithi181 bash[59345]: cluster 2023-10-09T09:19:08.290632+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e874 with expected crc 2023-10-09T09:19:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:08 smithi181 bash[59345]: cluster 2023-10-09T09:19:08.307969+0000 osd.4 (osd.4) 378 : cluster 3 failed to encode map e874 with expected crc 2023-10-09T09:19:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:08 smithi163 bash[78107]: cluster 2023-10-09T09:19:07.270707+0000 osd.5 (osd.5) 412 : cluster 3 failed to encode map e873 with expected crc 2023-10-09T09:19:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:08 smithi163 bash[78107]: cluster 2023-10-09T09:19:07.429037+0000 mgr.y (mgr.34107) 377 : cluster 0 pgmap v863: 89 pgs: 4 active+clean+snaptrim, 85 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.2 MiB/s wr, 20 op/s 2023-10-09T09:19:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:08 smithi163 bash[78107]: cluster 2023-10-09T09:19:08.281601+0000 mon.a (mon.0) 1079 : cluster 0 osdmap e874: 8 total, 8 up, 8 in 2023-10-09T09:19:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:08 smithi163 bash[78107]: cluster 2023-10-09T09:19:08.290632+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e874 with expected crc 2023-10-09T09:19:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:08 smithi163 bash[78107]: cluster 2023-10-09T09:19:08.307969+0000 osd.4 (osd.4) 378 : cluster 3 failed to encode map e874 with expected crc 2023-10-09T09:19:08.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:08 smithi163 bash[75425]: cluster 2023-10-09T09:19:07.270707+0000 osd.5 (osd.5) 412 : cluster 3 failed to encode map e873 with expected crc 2023-10-09T09:19:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:08 smithi163 bash[75425]: cluster 2023-10-09T09:19:07.429037+0000 mgr.y (mgr.34107) 377 : cluster 0 pgmap v863: 89 pgs: 4 active+clean+snaptrim, 85 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.2 MiB/s wr, 20 op/s 2023-10-09T09:19:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:08 smithi163 bash[75425]: cluster 2023-10-09T09:19:08.281601+0000 mon.a (mon.0) 1079 : cluster 0 osdmap e874: 8 total, 8 up, 8 in 2023-10-09T09:19:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:08 smithi163 bash[75425]: cluster 2023-10-09T09:19:08.290632+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e874 with expected crc 2023-10-09T09:19:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:08 smithi163 bash[75425]: cluster 2023-10-09T09:19:08.307969+0000 osd.4 (osd.4) 378 : cluster 3 failed to encode map e874 with expected crc 2023-10-09T09:19:09.287 INFO:tasks.rados.rados.0.smithi163.stdout:3273: done (1 left) 2023-10-09T09:19:09.288 INFO:tasks.rados.rados.0.smithi163.stdout:3276: done (0 left) 2023-10-09T09:19:09.288 INFO:tasks.rados.rados.0.smithi163.stdout:3277: snap_create 2023-10-09T09:19:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:09 smithi181 bash[59345]: cluster 2023-10-09T09:19:08.278516+0000 osd.5 (osd.5) 413 : cluster 3 failed to encode map e874 with expected crc 2023-10-09T09:19:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:09 smithi181 bash[59345]: cluster 2023-10-09T09:19:09.287398+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e875: 8 total, 8 up, 8 in 2023-10-09T09:19:09.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:09 smithi163 bash[75425]: cluster 2023-10-09T09:19:08.278516+0000 osd.5 (osd.5) 413 : cluster 3 failed to encode map e874 with expected crc 2023-10-09T09:19:09.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:09 smithi163 bash[75425]: cluster 2023-10-09T09:19:09.287398+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e875: 8 total, 8 up, 8 in 2023-10-09T09:19:09.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:09 smithi163 bash[78107]: cluster 2023-10-09T09:19:08.278516+0000 osd.5 (osd.5) 413 : cluster 3 failed to encode map e874 with expected crc 2023-10-09T09:19:09.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:09 smithi163 bash[78107]: cluster 2023-10-09T09:19:09.287398+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e875: 8 total, 8 up, 8 in 2023-10-09T09:19:10.258 INFO:tasks.rados.rados.0.smithi163.stdout:3277: done (0 left) 2023-10-09T09:19:10.258 INFO:tasks.rados.rados.0.smithi163.stdout:3278: read oid 26 snap -1 2023-10-09T09:19:10.258 INFO:tasks.rados.rados.0.smithi163.stdout:3278: expect deleted 2023-10-09T09:19:10.258 INFO:tasks.rados.rados.0.smithi163.stdout:3279: snap_remove snap 370 2023-10-09T09:19:10.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:10 smithi181 bash[59345]: cluster 2023-10-09T09:19:09.281237+0000 osd.5 (osd.5) 414 : cluster 3 failed to encode map e875 with expected crc 2023-10-09T09:19:10.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:10 smithi181 bash[59345]: cluster 2023-10-09T09:19:09.281971+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e875 with expected crc 2023-10-09T09:19:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:10 smithi181 bash[59345]: cluster 2023-10-09T09:19:09.429683+0000 mgr.y (mgr.34107) 378 : cluster 0 pgmap v866: 89 pgs: 4 active+clean+snaptrim, 85 active+clean; 318 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2023-10-09T09:19:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:10 smithi181 bash[59345]: cluster 2023-10-09T09:19:10.257850+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e876: 8 total, 8 up, 8 in 2023-10-09T09:19:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:10 smithi181 bash[59345]: audit 2023-10-09T09:19:10.349075+0000 mon.a (mon.0) 1082 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:10 smithi163 bash[75425]: cluster 2023-10-09T09:19:09.281237+0000 osd.5 (osd.5) 414 : cluster 3 failed to encode map e875 with expected crc 2023-10-09T09:19:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:10 smithi163 bash[75425]: cluster 2023-10-09T09:19:09.281971+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e875 with expected crc 2023-10-09T09:19:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:10 smithi163 bash[75425]: cluster 2023-10-09T09:19:09.429683+0000 mgr.y (mgr.34107) 378 : cluster 0 pgmap v866: 89 pgs: 4 active+clean+snaptrim, 85 active+clean; 318 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2023-10-09T09:19:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:10 smithi163 bash[75425]: cluster 2023-10-09T09:19:10.257850+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e876: 8 total, 8 up, 8 in 2023-10-09T09:19:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:10 smithi163 bash[75425]: audit 2023-10-09T09:19:10.349075+0000 mon.a (mon.0) 1082 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:10.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:10 smithi163 bash[78107]: cluster 2023-10-09T09:19:09.281237+0000 osd.5 (osd.5) 414 : cluster 3 failed to encode map e875 with expected crc 2023-10-09T09:19:10.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:10 smithi163 bash[78107]: cluster 2023-10-09T09:19:09.281971+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e875 with expected crc 2023-10-09T09:19:10.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:10 smithi163 bash[78107]: cluster 2023-10-09T09:19:09.429683+0000 mgr.y (mgr.34107) 378 : cluster 0 pgmap v866: 89 pgs: 4 active+clean+snaptrim, 85 active+clean; 318 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2023-10-09T09:19:10.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:10 smithi163 bash[78107]: cluster 2023-10-09T09:19:10.257850+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e876: 8 total, 8 up, 8 in 2023-10-09T09:19:10.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:10 smithi163 bash[78107]: audit 2023-10-09T09:19:10.349075+0000 mon.a (mon.0) 1082 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:11.266 INFO:tasks.rados.rados.0.smithi163.stdout:3279: done (1 left) 2023-10-09T09:19:11.267 INFO:tasks.rados.rados.0.smithi163.stdout:3280: write oid 35 current snap is 410 2023-10-09T09:19:11.267 INFO:tasks.rados.rados.0.smithi163.stdout:3280: seq_num 1227 ranges {622189=663356,1984347=458752} 2023-10-09T09:19:11.277 INFO:tasks.rados.rados.0.smithi163.stdout:3280: writing smithi16367185-35 from 622189 to 1285545 tid 1 2023-10-09T09:19:11.286 INFO:tasks.rados.rados.0.smithi163.stdout:3280: writing smithi16367185-35 from 1984347 to 2443099 tid 2 2023-10-09T09:19:11.286 INFO:tasks.rados.rados.0.smithi163.stdout:3281: rollback oid 46 current snap is 410 2023-10-09T09:19:11.286 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 46 to 398 2023-10-09T09:19:11.286 INFO:tasks.rados.rados.0.smithi163.stdout:3282: rollback oid 2 current snap is 410 2023-10-09T09:19:11.286 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 2 to 385 2023-10-09T09:19:11.286 INFO:tasks.rados.rados.0.smithi163.stdout:3283: delete oid 40 current snap is 410 2023-10-09T09:19:11.288 INFO:tasks.rados.rados.0.smithi163.stdout:3280: finishing write tid 1 to smithi16367185-35 2023-10-09T09:19:11.294 INFO:tasks.rados.rados.0.smithi163.stdout:3281: finishing rollback tid 0 to smithi16367185-46 2023-10-09T09:19:11.294 INFO:tasks.rados.rados.0.smithi163.stdout:3278: done (4 left) 2023-10-09T09:19:11.294 INFO:tasks.rados.rados.0.smithi163.stdout:3283: done (3 left) 2023-10-09T09:19:11.294 INFO:tasks.rados.rados.0.smithi163.stdout:3284: snap_remove snap 392 2023-10-09T09:19:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:11 smithi181 bash[59345]: cluster 2023-10-09T09:19:10.251599+0000 osd.5 (osd.5) 415 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:11 smithi181 bash[59345]: cluster 2023-10-09T09:19:10.508510+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:11 smithi181 bash[59345]: cluster 2023-10-09T09:19:10.625043+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:11 smithi181 bash[59345]: cluster 2023-10-09T09:19:10.706841+0000 osd.4 (osd.4) 379 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:11 smithi181 bash[59345]: cluster 2023-10-09T09:19:11.263125+0000 osd.4 (osd.4) 380 : cluster 3 failed to encode map e877 with expected crc 2023-10-09T09:19:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:11 smithi181 bash[59345]: cluster 2023-10-09T09:19:11.266403+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e877: 8 total, 8 up, 8 in 2023-10-09T09:19:11.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[75425]: cluster 2023-10-09T09:19:10.251599+0000 osd.5 (osd.5) 415 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[75425]: cluster 2023-10-09T09:19:10.508510+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[75425]: cluster 2023-10-09T09:19:10.625043+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[75425]: cluster 2023-10-09T09:19:10.706841+0000 osd.4 (osd.4) 379 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[75425]: cluster 2023-10-09T09:19:11.263125+0000 osd.4 (osd.4) 380 : cluster 3 failed to encode map e877 with expected crc 2023-10-09T09:19:11.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[75425]: cluster 2023-10-09T09:19:11.266403+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e877: 8 total, 8 up, 8 in 2023-10-09T09:19:11.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[78107]: cluster 2023-10-09T09:19:10.251599+0000 osd.5 (osd.5) 415 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[78107]: cluster 2023-10-09T09:19:10.508510+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[78107]: cluster 2023-10-09T09:19:10.625043+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[78107]: cluster 2023-10-09T09:19:10.706841+0000 osd.4 (osd.4) 379 : cluster 3 failed to encode map e876 with expected crc 2023-10-09T09:19:11.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[78107]: cluster 2023-10-09T09:19:11.263125+0000 osd.4 (osd.4) 380 : cluster 3 failed to encode map e877 with expected crc 2023-10-09T09:19:11.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:11 smithi163 bash[78107]: cluster 2023-10-09T09:19:11.266403+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e877: 8 total, 8 up, 8 in 2023-10-09T09:19:12.275 INFO:tasks.rados.rados.0.smithi163.stdout:3284: done (3 left) 2023-10-09T09:19:12.275 INFO:tasks.rados.rados.0.smithi163.stdout:3285: snap_remove snap 371 2023-10-09T09:19:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:12 smithi181 bash[59345]: cluster 2023-10-09T09:19:11.262499+0000 osd.5 (osd.5) 416 : cluster 3 failed to encode map e877 with expected crc 2023-10-09T09:19:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:12 smithi181 bash[59345]: cluster 2023-10-09T09:19:11.430567+0000 mgr.y (mgr.34107) 379 : cluster 0 pgmap v869: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 15 op/s 2023-10-09T09:19:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:12 smithi181 bash[59345]: cluster 2023-10-09T09:19:12.273543+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e878: 8 total, 8 up, 8 in 2023-10-09T09:19:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:12 smithi181 bash[59345]: cluster 2023-10-09T09:19:12.274736+0000 osd.4 (osd.4) 381 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:12 smithi181 bash[59345]: cluster 2023-10-09T09:19:12.277822+0000 osd.4 (osd.4) 382 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:12 smithi163 bash[78107]: cluster 2023-10-09T09:19:11.262499+0000 osd.5 (osd.5) 416 : cluster 3 failed to encode map e877 with expected crc 2023-10-09T09:19:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:12 smithi163 bash[78107]: cluster 2023-10-09T09:19:11.430567+0000 mgr.y (mgr.34107) 379 : cluster 0 pgmap v869: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 15 op/s 2023-10-09T09:19:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:12 smithi163 bash[78107]: cluster 2023-10-09T09:19:12.273543+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e878: 8 total, 8 up, 8 in 2023-10-09T09:19:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:12 smithi163 bash[78107]: cluster 2023-10-09T09:19:12.274736+0000 osd.4 (osd.4) 381 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:12 smithi163 bash[78107]: cluster 2023-10-09T09:19:12.277822+0000 osd.4 (osd.4) 382 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:12 smithi163 bash[75425]: cluster 2023-10-09T09:19:11.262499+0000 osd.5 (osd.5) 416 : cluster 3 failed to encode map e877 with expected crc 2023-10-09T09:19:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:12 smithi163 bash[75425]: cluster 2023-10-09T09:19:11.430567+0000 mgr.y (mgr.34107) 379 : cluster 0 pgmap v869: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 15 op/s 2023-10-09T09:19:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:12 smithi163 bash[75425]: cluster 2023-10-09T09:19:12.273543+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e878: 8 total, 8 up, 8 in 2023-10-09T09:19:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:12 smithi163 bash[75425]: cluster 2023-10-09T09:19:12.274736+0000 osd.4 (osd.4) 381 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:12 smithi163 bash[75425]: cluster 2023-10-09T09:19:12.277822+0000 osd.4 (osd.4) 382 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.278 INFO:tasks.rados.rados.0.smithi163.stdout:3285: done (3 left) 2023-10-09T09:19:13.278 INFO:tasks.rados.rados.0.smithi163.stdout:3286: read oid 17 snap -1 2023-10-09T09:19:13.278 INFO:tasks.rados.rados.0.smithi163.stdout:3286: expect (ObjNum 1222 snap 408 seq_num 1222) 2023-10-09T09:19:13.278 INFO:tasks.rados.rados.0.smithi163.stdout:3287: write oid 42 current snap is 410 2023-10-09T09:19:13.278 INFO:tasks.rados.rados.0.smithi163.stdout:3287: seq_num 1229 ranges {655803=413854,1584958=462421} 2023-10-09T09:19:13.288 INFO:tasks.rados.rados.0.smithi163.stdout:3287: writing smithi16367185-42 from 655803 to 1069657 tid 1 2023-10-09T09:19:13.294 INFO:tasks.rados.rados.0.smithi163.stdout:3287: writing smithi16367185-42 from 1584958 to 2047379 tid 2 2023-10-09T09:19:13.294 INFO:tasks.rados.rados.0.smithi163.stdout:3288: write oid 30 current snap is 410 2023-10-09T09:19:13.294 INFO:tasks.rados.rados.0.smithi163.stdout:3288: seq_num 1230 ranges {672610=489103,1643440=722432} 2023-10-09T09:19:13.301 INFO:tasks.rados.rados.0.smithi163.stdout:3288: writing smithi16367185-30 from 672610 to 1161713 tid 1 2023-10-09T09:19:13.308 INFO:tasks.rados.rados.0.smithi163.stdout:3288: writing smithi16367185-30 from 1643440 to 2365872 tid 2 2023-10-09T09:19:13.309 INFO:tasks.rados.rados.0.smithi163.stdout:3289: snap_remove snap 338 2023-10-09T09:19:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:19:13] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.33.4" 2023-10-09T09:19:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[75425]: cluster 2023-10-09T09:19:12.270907+0000 osd.5 (osd.5) 417 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[75425]: cluster 2023-10-09T09:19:12.274634+0000 osd.5 (osd.5) 418 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[75425]: cluster 2023-10-09T09:19:12.275523+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[75425]: cluster 2023-10-09T09:19:12.581945+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[75425]: cluster 2023-10-09T09:19:13.277977+0000 mon.a (mon.0) 1085 : cluster 0 osdmap e879: 8 total, 8 up, 8 in 2023-10-09T09:19:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[75425]: cluster 2023-10-09T09:19:13.279167+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e879 with expected crc 2023-10-09T09:19:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[78107]: cluster 2023-10-09T09:19:12.270907+0000 osd.5 (osd.5) 417 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[78107]: cluster 2023-10-09T09:19:12.274634+0000 osd.5 (osd.5) 418 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[78107]: cluster 2023-10-09T09:19:12.275523+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[78107]: cluster 2023-10-09T09:19:12.581945+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[78107]: cluster 2023-10-09T09:19:13.277977+0000 mon.a (mon.0) 1085 : cluster 0 osdmap e879: 8 total, 8 up, 8 in 2023-10-09T09:19:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:13 smithi163 bash[78107]: cluster 2023-10-09T09:19:13.279167+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e879 with expected crc 2023-10-09T09:19:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:13 smithi181 bash[59345]: cluster 2023-10-09T09:19:12.270907+0000 osd.5 (osd.5) 417 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:13 smithi181 bash[59345]: cluster 2023-10-09T09:19:12.274634+0000 osd.5 (osd.5) 418 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:13 smithi181 bash[59345]: cluster 2023-10-09T09:19:12.275523+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:13 smithi181 bash[59345]: cluster 2023-10-09T09:19:12.581945+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e878 with expected crc 2023-10-09T09:19:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:13 smithi181 bash[59345]: cluster 2023-10-09T09:19:13.277977+0000 mon.a (mon.0) 1085 : cluster 0 osdmap e879: 8 total, 8 up, 8 in 2023-10-09T09:19:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:13 smithi181 bash[59345]: cluster 2023-10-09T09:19:13.279167+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e879 with expected crc 2023-10-09T09:19:14.287 INFO:tasks.rados.rados.0.smithi163.stdout:3289: done (6 left) 2023-10-09T09:19:14.287 INFO:tasks.rados.rados.0.smithi163.stdout:3290: delete oid 15 current snap is 410 2023-10-09T09:19:14.287 INFO:tasks.rados.rados.0.smithi163.stdout:3280: finishing write tid 2 to smithi16367185-35 2023-10-09T09:19:14.287 INFO:tasks.rados.rados.0.smithi163.stdout:3281: finishing rollback tid 1 to smithi16367185-46 2023-10-09T09:19:14.287 INFO:tasks.rados.rados.0.smithi163.stdout:3280: finishing write tid 3 to smithi16367185-35 2023-10-09T09:19:14.287 INFO:tasks.rados.rados.0.smithi163.stdout:3280: finishing write tid 4 to smithi16367185-35 2023-10-09T09:19:14.287 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 890 (ObjNum 1227 snap 410 seq_num 1227) dirty exists 2023-10-09T09:19:14.288 INFO:tasks.rados.rados.0.smithi163.stdout:3280: left oid 35 (ObjNum 1227 snap 410 seq_num 1227) 2023-10-09T09:19:14.288 INFO:tasks.rados.rados.0.smithi163.stdout:3282: finishing rollback tid 1 to smithi16367185-2 2023-10-09T09:19:14.288 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 865 (ObjNum 1211805728 snap 32603 seq_num 1207961808) dirty dne 2023-10-09T09:19:14.288 INFO:tasks.rados.rados.0.smithi163.stdout:3281: finishing rollback tid 2 to smithi16367185-46 2023-10-09T09:19:14.288 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 1563 (ObjNum 1145 snap 384 seq_num 1145) dirty exists 2023-10-09T09:19:14.288 INFO:tasks.rados.rados.0.smithi163.stdout:3287: finishing write tid 1 to smithi16367185-42 2023-10-09T09:19:14.288 INFO:tasks.rados.rados.0.smithi163.stdout:3288: finishing write tid 1 to smithi16367185-30 2023-10-09T09:19:14.288 INFO:tasks.rados.rados.0.smithi163.stdout:3287: finishing write tid 2 to smithi16367185-42 2023-10-09T09:19:14.288 INFO:tasks.rados.rados.0.smithi163.stdout:3287: finishing write tid 3 to smithi16367185-42 2023-10-09T09:19:14.289 INFO:tasks.rados.rados.0.smithi163.stdout:3287: finishing write tid 4 to smithi16367185-42 2023-10-09T09:19:14.289 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 2388 (ObjNum 1229 snap 410 seq_num 1229) dirty exists 2023-10-09T09:19:14.289 INFO:tasks.rados.rados.0.smithi163.stdout:3287: left oid 42 (ObjNum 1229 snap 410 seq_num 1229) 2023-10-09T09:19:14.289 INFO:tasks.rados.rados.0.smithi163.stdout:3288: finishing write tid 2 to smithi16367185-30 2023-10-09T09:19:14.289 INFO:tasks.rados.rados.0.smithi163.stdout:3288: finishing write tid 3 to smithi16367185-30 2023-10-09T09:19:14.289 INFO:tasks.rados.rados.0.smithi163.stdout:3288: finishing write tid 4 to smithi16367185-30 2023-10-09T09:19:14.289 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1119 (ObjNum 1230 snap 410 seq_num 1230) dirty exists 2023-10-09T09:19:14.290 INFO:tasks.rados.rados.0.smithi163.stdout:3288: left oid 30 (ObjNum 1230 snap 410 seq_num 1230) 2023-10-09T09:19:14.290 INFO:tasks.rados.rados.0.smithi163.stdout:3286: expect (ObjNum 1222 snap 408 seq_num 1222) 2023-10-09T09:19:14.414 INFO:tasks.rados.rados.0.smithi163.stdout:3280: done (6 left) 2023-10-09T09:19:14.415 INFO:tasks.rados.rados.0.smithi163.stdout:3281: done (5 left) 2023-10-09T09:19:14.415 INFO:tasks.rados.rados.0.smithi163.stdout:3282: done (4 left) 2023-10-09T09:19:14.415 INFO:tasks.rados.rados.0.smithi163.stdout:3286: done (3 left) 2023-10-09T09:19:14.415 INFO:tasks.rados.rados.0.smithi163.stdout:3287: done (2 left) 2023-10-09T09:19:14.415 INFO:tasks.rados.rados.0.smithi163.stdout:3288: done (1 left) 2023-10-09T09:19:14.415 INFO:tasks.rados.rados.0.smithi163.stdout:3290: done (0 left) 2023-10-09T09:19:14.415 INFO:tasks.rados.rados.0.smithi163.stdout:3291: snap_remove snap 398 2023-10-09T09:19:14.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:14 smithi181 bash[59345]: cluster 2023-10-09T09:19:13.273428+0000 osd.5 (osd.5) 419 : cluster 3 failed to encode map e879 with expected crc 2023-10-09T09:19:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:14 smithi181 bash[59345]: cluster 2023-10-09T09:19:13.274751+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e879 with expected crc 2023-10-09T09:19:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:14 smithi181 bash[59345]: cluster 2023-10-09T09:19:13.431290+0000 mgr.y (mgr.34107) 380 : cluster 0 pgmap v872: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 868 KiB/s wr, 14 op/s 2023-10-09T09:19:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:14 smithi181 bash[59345]: cluster 2023-10-09T09:19:14.286297+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e880: 8 total, 8 up, 8 in 2023-10-09T09:19:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:14 smithi181 bash[59345]: cluster 2023-10-09T09:19:14.292892+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e880 with expected crc 2023-10-09T09:19:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:14 smithi163 bash[75425]: cluster 2023-10-09T09:19:13.273428+0000 osd.5 (osd.5) 419 : cluster 3 failed to encode map e879 with expected crc 2023-10-09T09:19:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:14 smithi163 bash[75425]: cluster 2023-10-09T09:19:13.274751+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e879 with expected crc 2023-10-09T09:19:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:14 smithi163 bash[75425]: cluster 2023-10-09T09:19:13.431290+0000 mgr.y (mgr.34107) 380 : cluster 0 pgmap v872: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 868 KiB/s wr, 14 op/s 2023-10-09T09:19:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:14 smithi163 bash[75425]: cluster 2023-10-09T09:19:14.286297+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e880: 8 total, 8 up, 8 in 2023-10-09T09:19:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:14 smithi163 bash[75425]: cluster 2023-10-09T09:19:14.292892+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e880 with expected crc 2023-10-09T09:19:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:14 smithi163 bash[78107]: cluster 2023-10-09T09:19:13.273428+0000 osd.5 (osd.5) 419 : cluster 3 failed to encode map e879 with expected crc 2023-10-09T09:19:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:14 smithi163 bash[78107]: cluster 2023-10-09T09:19:13.274751+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e879 with expected crc 2023-10-09T09:19:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:14 smithi163 bash[78107]: cluster 2023-10-09T09:19:13.431290+0000 mgr.y (mgr.34107) 380 : cluster 0 pgmap v872: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 868 KiB/s wr, 14 op/s 2023-10-09T09:19:14.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:14 smithi163 bash[78107]: cluster 2023-10-09T09:19:14.286297+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e880: 8 total, 8 up, 8 in 2023-10-09T09:19:14.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:14 smithi163 bash[78107]: cluster 2023-10-09T09:19:14.292892+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e880 with expected crc 2023-10-09T09:19:15.259 INFO:tasks.rados.rados.0.smithi163.stdout:3291: done (0 left) 2023-10-09T09:19:15.259 INFO:tasks.rados.rados.0.smithi163.stdout:3292: rollback oid 4 current snap is 410 2023-10-09T09:19:15.259 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 403 2023-10-09T09:19:15.259 INFO:tasks.rados.rados.0.smithi163.stdout:3293: rollback oid 39 current snap is 410 2023-10-09T09:19:15.259 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 407 2023-10-09T09:19:15.259 INFO:tasks.rados.rados.0.smithi163.stdout:3294: write oid 29 current snap is 410 2023-10-09T09:19:15.259 INFO:tasks.rados.rados.0.smithi163.stdout:3294: seq_num 1232 ranges {706224=755954,2160404=545820} 2023-10-09T09:19:15.272 INFO:tasks.rados.rados.0.smithi163.stdout:3294: writing smithi16367185-29 from 706224 to 1462178 tid 1 2023-10-09T09:19:15.280 INFO:tasks.rados.rados.0.smithi163.stdout:3294: writing smithi16367185-29 from 2160404 to 2706224 tid 2 2023-10-09T09:19:15.280 INFO:tasks.rados.rados.0.smithi163.stdout:3295: delete oid 45 current snap is 410 2023-10-09T09:19:15.281 INFO:tasks.rados.rados.0.smithi163.stdout:3293: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:19:15.281 INFO:tasks.rados.rados.0.smithi163.stdout:3293: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:19:15.281 INFO:tasks.rados.rados.0.smithi163.stdout:3293: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:19:15.281 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1541 (ObjNum 932 snap 317 seq_num 932) dirty exists 2023-10-09T09:19:15.281 INFO:tasks.rados.rados.0.smithi163.stdout:3292: finishing rollback tid 0 to smithi16367185-4 2023-10-09T09:19:15.283 INFO:tasks.rados.rados.0.smithi163.stdout:3292: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:19:15.283 INFO:tasks.rados.rados.0.smithi163.stdout:3292: finishing rollback tid 2 to smithi16367185-4 2023-10-09T09:19:15.283 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1666 (ObjNum 1182 snap 397 seq_num 1182) dirty exists 2023-10-09T09:19:15.285 INFO:tasks.rados.rados.0.smithi163.stdout:3294: finishing write tid 1 to smithi16367185-29 2023-10-09T09:19:15.285 INFO:tasks.rados.rados.0.smithi163.stdout:3292: done (3 left) 2023-10-09T09:19:15.285 INFO:tasks.rados.rados.0.smithi163.stdout:3293: done (2 left) 2023-10-09T09:19:15.285 INFO:tasks.rados.rados.0.smithi163.stdout:3295: done (1 left) 2023-10-09T09:19:15.285 INFO:tasks.rados.rados.0.smithi163.stdout:3296: write oid 27 current snap is 410 2023-10-09T09:19:15.285 INFO:tasks.rados.rados.0.smithi163.stdout:3296: seq_num 1234 ranges {739838=506452,1877368=613417} 2023-10-09T09:19:15.292 INFO:tasks.rados.rados.0.smithi163.stdout:3296: writing smithi16367185-27 from 739838 to 1246290 tid 1 2023-10-09T09:19:15.299 INFO:tasks.rados.rados.0.smithi163.stdout:3296: writing smithi16367185-27 from 1877368 to 2490785 tid 2 2023-10-09T09:19:15.300 INFO:tasks.rados.rados.0.smithi163.stdout:3297: delete oid 7 current snap is 410 2023-10-09T09:19:15.300 INFO:tasks.rados.rados.0.smithi163.stdout:3294: finishing write tid 2 to smithi16367185-29 2023-10-09T09:19:15.300 INFO:tasks.rados.rados.0.smithi163.stdout:3294: finishing write tid 3 to smithi16367185-29 2023-10-09T09:19:15.300 INFO:tasks.rados.rados.0.smithi163.stdout:3294: finishing write tid 4 to smithi16367185-29 2023-10-09T09:19:15.300 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1546 (ObjNum 1232 snap 410 seq_num 1232) dirty exists 2023-10-09T09:19:15.300 INFO:tasks.rados.rados.0.smithi163.stdout:3294: left oid 29 (ObjNum 1232 snap 410 seq_num 1232) 2023-10-09T09:19:15.300 INFO:tasks.rados.rados.0.smithi163.stdout:3296: finishing write tid 1 to smithi16367185-27 2023-10-09T09:19:15.303 INFO:tasks.rados.rados.0.smithi163.stdout:3294: done (2 left) 2023-10-09T09:19:15.303 INFO:tasks.rados.rados.0.smithi163.stdout:3297: done (1 left) 2023-10-09T09:19:15.303 INFO:tasks.rados.rados.0.smithi163.stdout:3298: snap_remove snap 393 2023-10-09T09:19:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:15 smithi181 bash[59345]: cluster 2023-10-09T09:19:14.283376+0000 osd.5 (osd.5) 420 : cluster 3 failed to encode map e880 with expected crc 2023-10-09T09:19:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:15 smithi181 bash[59345]: cluster 2023-10-09T09:19:15.254787+0000 osd.4 (osd.4) 383 : cluster 3 failed to encode map e881 with expected crc 2023-10-09T09:19:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:15 smithi181 bash[59345]: cluster 2023-10-09T09:19:15.258216+0000 mon.a (mon.0) 1087 : cluster 0 osdmap e881: 8 total, 8 up, 8 in 2023-10-09T09:19:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:15 smithi181 bash[59345]: cluster 2023-10-09T09:19:15.265040+0000 osd.7 (osd.7) 349 : cluster 3 failed to encode map e881 with expected crc 2023-10-09T09:19:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:15 smithi163 bash[75425]: cluster 2023-10-09T09:19:14.283376+0000 osd.5 (osd.5) 420 : cluster 3 failed to encode map e880 with expected crc 2023-10-09T09:19:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:15 smithi163 bash[75425]: cluster 2023-10-09T09:19:15.254787+0000 osd.4 (osd.4) 383 : cluster 3 failed to encode map e881 with expected crc 2023-10-09T09:19:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:15 smithi163 bash[75425]: cluster 2023-10-09T09:19:15.258216+0000 mon.a (mon.0) 1087 : cluster 0 osdmap e881: 8 total, 8 up, 8 in 2023-10-09T09:19:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:15 smithi163 bash[75425]: cluster 2023-10-09T09:19:15.265040+0000 osd.7 (osd.7) 349 : cluster 3 failed to encode map e881 with expected crc 2023-10-09T09:19:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:15 smithi163 bash[78107]: cluster 2023-10-09T09:19:14.283376+0000 osd.5 (osd.5) 420 : cluster 3 failed to encode map e880 with expected crc 2023-10-09T09:19:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:15 smithi163 bash[78107]: cluster 2023-10-09T09:19:15.254787+0000 osd.4 (osd.4) 383 : cluster 3 failed to encode map e881 with expected crc 2023-10-09T09:19:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:15 smithi163 bash[78107]: cluster 2023-10-09T09:19:15.258216+0000 mon.a (mon.0) 1087 : cluster 0 osdmap e881: 8 total, 8 up, 8 in 2023-10-09T09:19:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:15 smithi163 bash[78107]: cluster 2023-10-09T09:19:15.265040+0000 osd.7 (osd.7) 349 : cluster 3 failed to encode map e881 with expected crc 2023-10-09T09:19:16.272 INFO:tasks.rados.rados.0.smithi163.stdout:3298: done (1 left) 2023-10-09T09:19:16.272 INFO:tasks.rados.rados.0.smithi163.stdout:3299: snap_remove snap 405 2023-10-09T09:19:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:16 smithi181 bash[59345]: cluster 2023-10-09T09:19:15.254087+0000 osd.5 (osd.5) 421 : cluster 3 failed to encode map e881 with expected crc 2023-10-09T09:19:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:16 smithi181 bash[59345]: cluster 2023-10-09T09:19:15.432094+0000 mgr.y (mgr.34107) 381 : cluster 0 pgmap v875: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 295 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 296 KiB/s wr, 1 op/s 2023-10-09T09:19:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:16 smithi181 bash[59345]: cluster 2023-10-09T09:19:16.272531+0000 mon.a (mon.0) 1088 : cluster 0 osdmap e882: 8 total, 8 up, 8 in 2023-10-09T09:19:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:16 smithi163 bash[75425]: cluster 2023-10-09T09:19:15.254087+0000 osd.5 (osd.5) 421 : cluster 3 failed to encode map e881 with expected crc 2023-10-09T09:19:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:16 smithi163 bash[75425]: cluster 2023-10-09T09:19:15.432094+0000 mgr.y (mgr.34107) 381 : cluster 0 pgmap v875: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 295 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 296 KiB/s wr, 1 op/s 2023-10-09T09:19:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:16 smithi163 bash[75425]: cluster 2023-10-09T09:19:16.272531+0000 mon.a (mon.0) 1088 : cluster 0 osdmap e882: 8 total, 8 up, 8 in 2023-10-09T09:19:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:16 smithi163 bash[78107]: cluster 2023-10-09T09:19:15.254087+0000 osd.5 (osd.5) 421 : cluster 3 failed to encode map e881 with expected crc 2023-10-09T09:19:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:16 smithi163 bash[78107]: cluster 2023-10-09T09:19:15.432094+0000 mgr.y (mgr.34107) 381 : cluster 0 pgmap v875: 89 pgs: 5 active+clean+snaptrim, 84 active+clean; 295 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 296 KiB/s wr, 1 op/s 2023-10-09T09:19:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:16 smithi163 bash[78107]: cluster 2023-10-09T09:19:16.272531+0000 mon.a (mon.0) 1088 : cluster 0 osdmap e882: 8 total, 8 up, 8 in 2023-10-09T09:19:17.276 INFO:tasks.rados.rados.0.smithi163.stdout:3299: done (1 left) 2023-10-09T09:19:17.276 INFO:tasks.rados.rados.0.smithi163.stdout:3300: snap_create 2023-10-09T09:19:17.277 INFO:tasks.rados.rados.0.smithi163.stdout:3296: finishing write tid 2 to smithi16367185-27 2023-10-09T09:19:17.277 INFO:tasks.rados.rados.0.smithi163.stdout:3296: finishing write tid 3 to smithi16367185-27 2023-10-09T09:19:17.277 INFO:tasks.rados.rados.0.smithi163.stdout:3296: finishing write tid 4 to smithi16367185-27 2023-10-09T09:19:17.277 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 1007 (ObjNum 1234 snap 410 seq_num 1234) dirty exists 2023-10-09T09:19:17.277 INFO:tasks.rados.rados.0.smithi163.stdout:3296: left oid 27 (ObjNum 1234 snap 410 seq_num 1234) 2023-10-09T09:19:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:17 smithi181 bash[59345]: cluster 2023-10-09T09:19:16.270003+0000 osd.5 (osd.5) 422 : cluster 3 failed to encode map e882 with expected crc 2023-10-09T09:19:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:17 smithi181 bash[59345]: cluster 2023-10-09T09:19:16.273129+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e882 with expected crc 2023-10-09T09:19:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:17 smithi181 bash[59345]: cluster 2023-10-09T09:19:17.275830+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e883: 8 total, 8 up, 8 in 2023-10-09T09:19:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:17 smithi181 bash[59345]: cluster 2023-10-09T09:19:17.277305+0000 osd.4 (osd.4) 384 : cluster 3 failed to encode map e883 with expected crc 2023-10-09T09:19:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:17 smithi181 bash[59345]: cluster 2023-10-09T09:19:17.278995+0000 osd.7 (osd.7) 350 : cluster 3 failed to encode map e883 with expected crc 2023-10-09T09:19:17.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:17 smithi163 bash[75425]: cluster 2023-10-09T09:19:16.270003+0000 osd.5 (osd.5) 422 : cluster 3 failed to encode map e882 with expected crc 2023-10-09T09:19:17.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:17 smithi163 bash[75425]: cluster 2023-10-09T09:19:16.273129+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e882 with expected crc 2023-10-09T09:19:17.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:17 smithi163 bash[75425]: cluster 2023-10-09T09:19:17.275830+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e883: 8 total, 8 up, 8 in 2023-10-09T09:19:17.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:17 smithi163 bash[75425]: cluster 2023-10-09T09:19:17.277305+0000 osd.4 (osd.4) 384 : cluster 3 failed to encode map e883 with expected crc 2023-10-09T09:19:17.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:17 smithi163 bash[75425]: cluster 2023-10-09T09:19:17.278995+0000 osd.7 (osd.7) 350 : cluster 3 failed to encode map e883 with expected crc 2023-10-09T09:19:17.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:17 smithi163 bash[78107]: cluster 2023-10-09T09:19:16.270003+0000 osd.5 (osd.5) 422 : cluster 3 failed to encode map e882 with expected crc 2023-10-09T09:19:17.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:17 smithi163 bash[78107]: cluster 2023-10-09T09:19:16.273129+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e882 with expected crc 2023-10-09T09:19:17.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:17 smithi163 bash[78107]: cluster 2023-10-09T09:19:17.275830+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e883: 8 total, 8 up, 8 in 2023-10-09T09:19:17.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:17 smithi163 bash[78107]: cluster 2023-10-09T09:19:17.277305+0000 osd.4 (osd.4) 384 : cluster 3 failed to encode map e883 with expected crc 2023-10-09T09:19:17.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:17 smithi163 bash[78107]: cluster 2023-10-09T09:19:17.278995+0000 osd.7 (osd.7) 350 : cluster 3 failed to encode map e883 with expected crc 2023-10-09T09:19:18.282 INFO:tasks.rados.rados.0.smithi163.stdout:3296: done (1 left) 2023-10-09T09:19:18.282 INFO:tasks.rados.rados.0.smithi163.stdout:3300: done (0 left) 2023-10-09T09:19:18.282 INFO:tasks.rados.rados.0.smithi163.stdout:3301: snap_create 2023-10-09T09:19:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:18 smithi181 bash[59345]: cluster 2023-10-09T09:19:17.271020+0000 osd.5 (osd.5) 423 : cluster 3 failed to encode map e883 with expected crc 2023-10-09T09:19:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:18 smithi181 bash[59345]: cluster 2023-10-09T09:19:17.432862+0000 mgr.y (mgr.34107) 382 : cluster 0 pgmap v878: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 247 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 784 KiB/s wr, 7 op/s 2023-10-09T09:19:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:18 smithi181 bash[59345]: cluster 2023-10-09T09:19:18.279764+0000 osd.4 (osd.4) 385 : cluster 3 failed to encode map e884 with expected crc 2023-10-09T09:19:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:18 smithi181 bash[59345]: cluster 2023-10-09T09:19:18.281873+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e884: 8 total, 8 up, 8 in 2023-10-09T09:19:18.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:18 smithi163 bash[75425]: cluster 2023-10-09T09:19:17.271020+0000 osd.5 (osd.5) 423 : cluster 3 failed to encode map e883 with expected crc 2023-10-09T09:19:18.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:18 smithi163 bash[75425]: cluster 2023-10-09T09:19:17.432862+0000 mgr.y (mgr.34107) 382 : cluster 0 pgmap v878: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 247 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 784 KiB/s wr, 7 op/s 2023-10-09T09:19:18.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:18 smithi163 bash[75425]: cluster 2023-10-09T09:19:18.279764+0000 osd.4 (osd.4) 385 : cluster 3 failed to encode map e884 with expected crc 2023-10-09T09:19:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:18 smithi163 bash[75425]: cluster 2023-10-09T09:19:18.281873+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e884: 8 total, 8 up, 8 in 2023-10-09T09:19:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:18 smithi163 bash[78107]: cluster 2023-10-09T09:19:17.271020+0000 osd.5 (osd.5) 423 : cluster 3 failed to encode map e883 with expected crc 2023-10-09T09:19:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:18 smithi163 bash[78107]: cluster 2023-10-09T09:19:17.432862+0000 mgr.y (mgr.34107) 382 : cluster 0 pgmap v878: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 247 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 784 KiB/s wr, 7 op/s 2023-10-09T09:19:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:18 smithi163 bash[78107]: cluster 2023-10-09T09:19:18.279764+0000 osd.4 (osd.4) 385 : cluster 3 failed to encode map e884 with expected crc 2023-10-09T09:19:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:18 smithi163 bash[78107]: cluster 2023-10-09T09:19:18.281873+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e884: 8 total, 8 up, 8 in 2023-10-09T09:19:19.287 INFO:tasks.rados.rados.0.smithi163.stdout:3301: done (0 left) 2023-10-09T09:19:19.287 INFO:tasks.rados.rados.0.smithi163.stdout:3302: rollback oid 7 current snap is 412 2023-10-09T09:19:19.288 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 404 2023-10-09T09:19:19.288 INFO:tasks.rados.rados.0.smithi163.stdout:3303: read oid 21 snap -1 2023-10-09T09:19:19.288 INFO:tasks.rados.rados.0.smithi163.stdout:3303: expect (ObjNum 1223 snap 408 seq_num 1223) 2023-10-09T09:19:19.288 INFO:tasks.rados.rados.0.smithi163.stdout:3304: read oid 27 snap -1 2023-10-09T09:19:19.288 INFO:tasks.rados.rados.0.smithi163.stdout:3304: expect (ObjNum 1234 snap 410 seq_num 1234) 2023-10-09T09:19:19.288 INFO:tasks.rados.rados.0.smithi163.stdout:3305: write oid 15 current snap is 412 2023-10-09T09:19:19.288 INFO:tasks.rados.rados.0.smithi163.stdout:3305: seq_num 1236 ranges {773452=656950,1877979=617086} 2023-10-09T09:19:19.296 INFO:tasks.rados.rados.0.smithi163.stdout:3305: writing smithi16367185-15 from 773452 to 1430402 tid 1 2023-10-09T09:19:19.302 INFO:tasks.rados.rados.0.smithi163.stdout:3305: writing smithi16367185-15 from 1877979 to 2495065 tid 2 2023-10-09T09:19:19.303 INFO:tasks.rados.rados.0.smithi163.stdout:3306: snap_create 2023-10-09T09:19:19.303 INFO:tasks.rados.rados.0.smithi163.stdout:3304: expect (ObjNum 1234 snap 410 seq_num 1234) 2023-10-09T09:19:19.485 INFO:tasks.rados.rados.0.smithi163.stdout:3302: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:19:19.485 INFO:tasks.rados.rados.0.smithi163.stdout:3302: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:19:19.485 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 850 (ObjNum 1178 snap 397 seq_num 1178) dirty exists 2023-10-09T09:19:19.485 INFO:tasks.rados.rados.0.smithi163.stdout:3303: expect (ObjNum 1223 snap 408 seq_num 1223) 2023-10-09T09:19:19.636 INFO:tasks.rados.rados.0.smithi163.stdout:3305: finishing write tid 1 to smithi16367185-15 2023-10-09T09:19:19.637 INFO:tasks.rados.rados.0.smithi163.stdout:3305: finishing write tid 2 to smithi16367185-15 2023-10-09T09:19:19.637 INFO:tasks.rados.rados.0.smithi163.stdout:3305: finishing write tid 3 to smithi16367185-15 2023-10-09T09:19:19.637 INFO:tasks.rados.rados.0.smithi163.stdout:3305: finishing write tid 4 to smithi16367185-15 2023-10-09T09:19:19.637 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 1678 (ObjNum 1236 snap 412 seq_num 1236) dirty exists 2023-10-09T09:19:19.637 INFO:tasks.rados.rados.0.smithi163.stdout:3305: left oid 15 (ObjNum 1236 snap 412 seq_num 1236) 2023-10-09T09:19:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:19 smithi181 bash[59345]: cluster 2023-10-09T09:19:18.278790+0000 osd.5 (osd.5) 424 : cluster 3 failed to encode map e884 with expected crc 2023-10-09T09:19:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:19 smithi181 bash[59345]: cluster 2023-10-09T09:19:19.287288+0000 mon.a (mon.0) 1091 : cluster 0 osdmap e885: 8 total, 8 up, 8 in 2023-10-09T09:19:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:19 smithi181 bash[59345]: cluster 2023-10-09T09:19:19.292044+0000 osd.4 (osd.4) 386 : cluster 3 failed to encode map e885 with expected crc 2023-10-09T09:19:19.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:19 smithi181 bash[59345]: cluster 2023-10-09T09:19:19.300442+0000 osd.7 (osd.7) 351 : cluster 3 failed to encode map e885 with expected crc 2023-10-09T09:19:19.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:19 smithi163 bash[75425]: cluster 2023-10-09T09:19:18.278790+0000 osd.5 (osd.5) 424 : cluster 3 failed to encode map e884 with expected crc 2023-10-09T09:19:19.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:19 smithi163 bash[75425]: cluster 2023-10-09T09:19:19.287288+0000 mon.a (mon.0) 1091 : cluster 0 osdmap e885: 8 total, 8 up, 8 in 2023-10-09T09:19:19.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:19 smithi163 bash[75425]: cluster 2023-10-09T09:19:19.292044+0000 osd.4 (osd.4) 386 : cluster 3 failed to encode map e885 with expected crc 2023-10-09T09:19:19.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:19 smithi163 bash[75425]: cluster 2023-10-09T09:19:19.300442+0000 osd.7 (osd.7) 351 : cluster 3 failed to encode map e885 with expected crc 2023-10-09T09:19:19.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:19 smithi163 bash[78107]: cluster 2023-10-09T09:19:18.278790+0000 osd.5 (osd.5) 424 : cluster 3 failed to encode map e884 with expected crc 2023-10-09T09:19:19.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:19 smithi163 bash[78107]: cluster 2023-10-09T09:19:19.287288+0000 mon.a (mon.0) 1091 : cluster 0 osdmap e885: 8 total, 8 up, 8 in 2023-10-09T09:19:19.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:19 smithi163 bash[78107]: cluster 2023-10-09T09:19:19.292044+0000 osd.4 (osd.4) 386 : cluster 3 failed to encode map e885 with expected crc 2023-10-09T09:19:19.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:19 smithi163 bash[78107]: cluster 2023-10-09T09:19:19.300442+0000 osd.7 (osd.7) 351 : cluster 3 failed to encode map e885 with expected crc 2023-10-09T09:19:20.265 INFO:tasks.rados.rados.0.smithi163.stdout:3302: done (4 left) 2023-10-09T09:19:20.266 INFO:tasks.rados.rados.0.smithi163.stdout:3303: done (3 left) 2023-10-09T09:19:20.266 INFO:tasks.rados.rados.0.smithi163.stdout:3304: done (2 left) 2023-10-09T09:19:20.266 INFO:tasks.rados.rados.0.smithi163.stdout:3305: done (1 left) 2023-10-09T09:19:20.266 INFO:tasks.rados.rados.0.smithi163.stdout:3306: done (0 left) 2023-10-09T09:19:20.266 INFO:tasks.rados.rados.0.smithi163.stdout:3307: write oid 20 current snap is 413 2023-10-09T09:19:20.266 INFO:tasks.rados.rados.0.smithi163.stdout:3307: seq_num 1237 ranges {790259=732199,1936461=477097} 2023-10-09T09:19:20.275 INFO:tasks.rados.rados.0.smithi163.stdout:3307: writing smithi16367185-20 from 790259 to 1522458 tid 1 2023-10-09T09:19:20.281 INFO:tasks.rados.rados.0.smithi163.stdout:3307: writing smithi16367185-20 from 1936461 to 2413558 tid 2 2023-10-09T09:19:20.281 INFO:tasks.rados.rados.0.smithi163.stdout:3308: read oid 32 snap -1 2023-10-09T09:19:20.281 INFO:tasks.rados.rados.0.smithi163.stdout:3308: expect (ObjNum 1218 snap 404 seq_num 1218) 2023-10-09T09:19:20.281 INFO:tasks.rados.rados.0.smithi163.stdout:3309: snap_create 2023-10-09T09:19:20.290 INFO:tasks.rados.rados.0.smithi163.stdout:3307: finishing write tid 1 to smithi16367185-20 2023-10-09T09:19:20.290 INFO:tasks.rados.rados.0.smithi163.stdout:3307: finishing write tid 2 to smithi16367185-20 2023-10-09T09:19:20.290 INFO:tasks.rados.rados.0.smithi163.stdout:3307: finishing write tid 3 to smithi16367185-20 2023-10-09T09:19:20.291 INFO:tasks.rados.rados.0.smithi163.stdout:3307: finishing write tid 4 to smithi16367185-20 2023-10-09T09:19:20.291 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 627 (ObjNum 1237 snap 413 seq_num 1237) dirty exists 2023-10-09T09:19:20.291 INFO:tasks.rados.rados.0.smithi163.stdout:3307: left oid 20 (ObjNum 1237 snap 413 seq_num 1237) 2023-10-09T09:19:20.294 INFO:tasks.rados.rados.0.smithi163.stdout:3308: expect (ObjNum 1218 snap 404 seq_num 1218) 2023-10-09T09:19:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:20 smithi181 bash[59345]: cluster 2023-10-09T09:19:19.281481+0000 osd.5 (osd.5) 425 : cluster 3 failed to encode map e885 with expected crc 2023-10-09T09:19:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:20 smithi181 bash[59345]: cluster 2023-10-09T09:19:19.433528+0000 mgr.y (mgr.34107) 383 : cluster 0 pgmap v881: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 241 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 488 KiB/s wr, 6 op/s 2023-10-09T09:19:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:20 smithi181 bash[59345]: cluster 2023-10-09T09:19:20.265365+0000 mon.a (mon.0) 1092 : cluster 0 osdmap e886: 8 total, 8 up, 8 in 2023-10-09T09:19:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:20 smithi163 bash[75425]: cluster 2023-10-09T09:19:19.281481+0000 osd.5 (osd.5) 425 : cluster 3 failed to encode map e885 with expected crc 2023-10-09T09:19:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:20 smithi163 bash[75425]: cluster 2023-10-09T09:19:19.433528+0000 mgr.y (mgr.34107) 383 : cluster 0 pgmap v881: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 241 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 488 KiB/s wr, 6 op/s 2023-10-09T09:19:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:20 smithi163 bash[75425]: cluster 2023-10-09T09:19:20.265365+0000 mon.a (mon.0) 1092 : cluster 0 osdmap e886: 8 total, 8 up, 8 in 2023-10-09T09:19:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:20 smithi163 bash[78107]: cluster 2023-10-09T09:19:19.281481+0000 osd.5 (osd.5) 425 : cluster 3 failed to encode map e885 with expected crc 2023-10-09T09:19:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:20 smithi163 bash[78107]: cluster 2023-10-09T09:19:19.433528+0000 mgr.y (mgr.34107) 383 : cluster 0 pgmap v881: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 241 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 488 KiB/s wr, 6 op/s 2023-10-09T09:19:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:20 smithi163 bash[78107]: cluster 2023-10-09T09:19:20.265365+0000 mon.a (mon.0) 1092 : cluster 0 osdmap e886: 8 total, 8 up, 8 in 2023-10-09T09:19:21.278 INFO:tasks.rados.rados.0.smithi163.stdout:3307: done (2 left) 2023-10-09T09:19:21.278 INFO:tasks.rados.rados.0.smithi163.stdout:3308: done (1 left) 2023-10-09T09:19:21.279 INFO:tasks.rados.rados.0.smithi163.stdout:3309: done (0 left) 2023-10-09T09:19:21.279 INFO:tasks.rados.rados.0.smithi163.stdout:3310: rollback oid 49 current snap is 414 2023-10-09T09:19:21.279 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 49 to 402 2023-10-09T09:19:21.279 INFO:tasks.rados.rados.0.smithi163.stdout:3311: delete oid 35 current snap is 414 2023-10-09T09:19:21.283 INFO:tasks.rados.rados.0.smithi163.stdout:3310: finishing rollback tid 0 to smithi16367185-49 2023-10-09T09:19:21.283 INFO:tasks.rados.rados.0.smithi163.stdout:3310: finishing rollback tid 1 to smithi16367185-49 2023-10-09T09:19:21.284 INFO:tasks.rados.rados.0.smithi163.stdout:3310: finishing rollback tid 2 to smithi16367185-49 2023-10-09T09:19:21.284 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 424 (ObjNum 1099 snap 365 seq_num 1099) dirty exists 2023-10-09T09:19:21.289 INFO:tasks.rados.rados.0.smithi163.stdout:3310: done (1 left) 2023-10-09T09:19:21.289 INFO:tasks.rados.rados.0.smithi163.stdout:3311: done (0 left) 2023-10-09T09:19:21.289 INFO:tasks.rados.rados.0.smithi163.stdout:3312: rollback oid 46 current snap is 414 2023-10-09T09:19:21.289 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 46 to 411 2023-10-09T09:19:21.289 INFO:tasks.rados.rados.0.smithi163.stdout:3313: snap_create 2023-10-09T09:19:21.293 INFO:tasks.rados.rados.0.smithi163.stdout:3312: finishing rollback tid 0 to smithi16367185-46 2023-10-09T09:19:21.294 INFO:tasks.rados.rados.0.smithi163.stdout:3312: finishing rollback tid 1 to smithi16367185-46 2023-10-09T09:19:21.294 INFO:tasks.rados.rados.0.smithi163.stdout:3312: finishing rollback tid 2 to smithi16367185-46 2023-10-09T09:19:21.294 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 1593 (ObjNum 1145 snap 384 seq_num 1145) dirty exists 2023-10-09T09:19:21.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:21 smithi181 bash[59345]: cluster 2023-10-09T09:19:20.261071+0000 osd.5 (osd.5) 426 : cluster 3 failed to encode map e886 with expected crc 2023-10-09T09:19:21.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:21 smithi181 bash[59345]: cluster 2023-10-09T09:19:20.277603+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e886 with expected crc 2023-10-09T09:19:21.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:21 smithi181 bash[59345]: cluster 2023-10-09T09:19:21.271634+0000 osd.5 (osd.5) 427 : cluster 3 failed to encode map e887 with expected crc 2023-10-09T09:19:21.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:21 smithi181 bash[59345]: cluster 2023-10-09T09:19:21.276120+0000 mon.a (mon.0) 1093 : cluster 0 osdmap e887: 8 total, 8 up, 8 in 2023-10-09T09:19:21.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:21 smithi181 bash[59345]: cluster 2023-10-09T09:19:21.279819+0000 osd.7 (osd.7) 352 : cluster 3 failed to encode map e887 with expected crc 2023-10-09T09:19:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:21 smithi163 bash[75425]: cluster 2023-10-09T09:19:20.261071+0000 osd.5 (osd.5) 426 : cluster 3 failed to encode map e886 with expected crc 2023-10-09T09:19:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:21 smithi163 bash[75425]: cluster 2023-10-09T09:19:20.277603+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e886 with expected crc 2023-10-09T09:19:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:21 smithi163 bash[75425]: cluster 2023-10-09T09:19:21.271634+0000 osd.5 (osd.5) 427 : cluster 3 failed to encode map e887 with expected crc 2023-10-09T09:19:21.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:21 smithi163 bash[75425]: cluster 2023-10-09T09:19:21.276120+0000 mon.a (mon.0) 1093 : cluster 0 osdmap e887: 8 total, 8 up, 8 in 2023-10-09T09:19:21.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:21 smithi163 bash[75425]: cluster 2023-10-09T09:19:21.279819+0000 osd.7 (osd.7) 352 : cluster 3 failed to encode map e887 with expected crc 2023-10-09T09:19:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:21 smithi163 bash[78107]: cluster 2023-10-09T09:19:20.261071+0000 osd.5 (osd.5) 426 : cluster 3 failed to encode map e886 with expected crc 2023-10-09T09:19:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:21 smithi163 bash[78107]: cluster 2023-10-09T09:19:20.277603+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e886 with expected crc 2023-10-09T09:19:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:21 smithi163 bash[78107]: cluster 2023-10-09T09:19:21.271634+0000 osd.5 (osd.5) 427 : cluster 3 failed to encode map e887 with expected crc 2023-10-09T09:19:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:21 smithi163 bash[78107]: cluster 2023-10-09T09:19:21.276120+0000 mon.a (mon.0) 1093 : cluster 0 osdmap e887: 8 total, 8 up, 8 in 2023-10-09T09:19:21.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:21 smithi163 bash[78107]: cluster 2023-10-09T09:19:21.279819+0000 osd.7 (osd.7) 352 : cluster 3 failed to encode map e887 with expected crc 2023-10-09T09:19:22.285 INFO:tasks.rados.rados.0.smithi163.stdout:3312: done (1 left) 2023-10-09T09:19:22.285 INFO:tasks.rados.rados.0.smithi163.stdout:3313: done (0 left) 2023-10-09T09:19:22.285 INFO:tasks.rados.rados.0.smithi163.stdout:3314: snap_create 2023-10-09T09:19:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:22 smithi181 bash[59345]: cluster 2023-10-09T09:19:21.279857+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e887 with expected crc 2023-10-09T09:19:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:22 smithi181 bash[59345]: cluster 2023-10-09T09:19:21.434512+0000 mgr.y (mgr.34107) 384 : cluster 0 pgmap v884: 89 pgs: 89 active+clean; 243 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2023-10-09T09:19:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:22 smithi181 bash[59345]: cluster 2023-10-09T09:19:22.282589+0000 osd.5 (osd.5) 428 : cluster 3 failed to encode map e888 with expected crc 2023-10-09T09:19:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:22 smithi181 bash[59345]: cluster 2023-10-09T09:19:22.284003+0000 osd.7 (osd.7) 353 : cluster 3 failed to encode map e888 with expected crc 2023-10-09T09:19:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:22 smithi181 bash[59345]: cluster 2023-10-09T09:19:22.285122+0000 mon.a (mon.0) 1094 : cluster 0 osdmap e888: 8 total, 8 up, 8 in 2023-10-09T09:19:22.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:22 smithi163 bash[75425]: cluster 2023-10-09T09:19:21.279857+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e887 with expected crc 2023-10-09T09:19:22.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:22 smithi163 bash[75425]: cluster 2023-10-09T09:19:21.434512+0000 mgr.y (mgr.34107) 384 : cluster 0 pgmap v884: 89 pgs: 89 active+clean; 243 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2023-10-09T09:19:22.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:22 smithi163 bash[75425]: cluster 2023-10-09T09:19:22.282589+0000 osd.5 (osd.5) 428 : cluster 3 failed to encode map e888 with expected crc 2023-10-09T09:19:22.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:22 smithi163 bash[75425]: cluster 2023-10-09T09:19:22.284003+0000 osd.7 (osd.7) 353 : cluster 3 failed to encode map e888 with expected crc 2023-10-09T09:19:22.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:22 smithi163 bash[75425]: cluster 2023-10-09T09:19:22.285122+0000 mon.a (mon.0) 1094 : cluster 0 osdmap e888: 8 total, 8 up, 8 in 2023-10-09T09:19:22.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:22 smithi163 bash[78107]: cluster 2023-10-09T09:19:21.279857+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e887 with expected crc 2023-10-09T09:19:22.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:22 smithi163 bash[78107]: cluster 2023-10-09T09:19:21.434512+0000 mgr.y (mgr.34107) 384 : cluster 0 pgmap v884: 89 pgs: 89 active+clean; 243 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2023-10-09T09:19:22.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:22 smithi163 bash[78107]: cluster 2023-10-09T09:19:22.282589+0000 osd.5 (osd.5) 428 : cluster 3 failed to encode map e888 with expected crc 2023-10-09T09:19:22.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:22 smithi163 bash[78107]: cluster 2023-10-09T09:19:22.284003+0000 osd.7 (osd.7) 353 : cluster 3 failed to encode map e888 with expected crc 2023-10-09T09:19:22.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:22 smithi163 bash[78107]: cluster 2023-10-09T09:19:22.285122+0000 mon.a (mon.0) 1094 : cluster 0 osdmap e888: 8 total, 8 up, 8 in 2023-10-09T09:19:23.289 INFO:tasks.rados.rados.0.smithi163.stdout:3314: done (0 left) 2023-10-09T09:19:23.289 INFO:tasks.rados.rados.0.smithi163.stdout:3315: snap_remove snap 408 2023-10-09T09:19:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:19:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:19:23] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.33.4" 2023-10-09T09:19:24.302 INFO:tasks.rados.rados.0.smithi163.stdout:3315: done (0 left) 2023-10-09T09:19:24.302 INFO:tasks.rados.rados.0.smithi163.stdout:3316: snap_remove snap 407 2023-10-09T09:19:24.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:24 smithi181 bash[59345]: cluster 2023-10-09T09:19:22.787719+0000 osd.4 (osd.4) 387 : cluster 3 failed to encode map e888 with expected crc 2023-10-09T09:19:24.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:24 smithi181 bash[59345]: cluster 2023-10-09T09:19:23.282819+0000 osd.5 (osd.5) 429 : cluster 3 failed to encode map e889 with expected crc 2023-10-09T09:19:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:24 smithi181 bash[59345]: cluster 2023-10-09T09:19:23.283309+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e889 with expected crc 2023-10-09T09:19:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:24 smithi181 bash[59345]: cluster 2023-10-09T09:19:23.289272+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e889: 8 total, 8 up, 8 in 2023-10-09T09:19:24.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:24 smithi181 bash[59345]: cluster 2023-10-09T09:19:23.435174+0000 mgr.y (mgr.34107) 385 : cluster 0 pgmap v887: 89 pgs: 89 active+clean; 243 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:19:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:24 smithi163 bash[78107]: cluster 2023-10-09T09:19:22.787719+0000 osd.4 (osd.4) 387 : cluster 3 failed to encode map e888 with expected crc 2023-10-09T09:19:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:24 smithi163 bash[78107]: cluster 2023-10-09T09:19:23.282819+0000 osd.5 (osd.5) 429 : cluster 3 failed to encode map e889 with expected crc 2023-10-09T09:19:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:24 smithi163 bash[78107]: cluster 2023-10-09T09:19:23.283309+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e889 with expected crc 2023-10-09T09:19:24.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:24 smithi163 bash[78107]: cluster 2023-10-09T09:19:23.289272+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e889: 8 total, 8 up, 8 in 2023-10-09T09:19:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:24 smithi163 bash[78107]: cluster 2023-10-09T09:19:23.435174+0000 mgr.y (mgr.34107) 385 : cluster 0 pgmap v887: 89 pgs: 89 active+clean; 243 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:19:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:24 smithi163 bash[75425]: cluster 2023-10-09T09:19:22.787719+0000 osd.4 (osd.4) 387 : cluster 3 failed to encode map e888 with expected crc 2023-10-09T09:19:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:24 smithi163 bash[75425]: cluster 2023-10-09T09:19:23.282819+0000 osd.5 (osd.5) 429 : cluster 3 failed to encode map e889 with expected crc 2023-10-09T09:19:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:24 smithi163 bash[75425]: cluster 2023-10-09T09:19:23.283309+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e889 with expected crc 2023-10-09T09:19:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:24 smithi163 bash[75425]: cluster 2023-10-09T09:19:23.289272+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e889: 8 total, 8 up, 8 in 2023-10-09T09:19:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:24 smithi163 bash[75425]: cluster 2023-10-09T09:19:23.435174+0000 mgr.y (mgr.34107) 385 : cluster 0 pgmap v887: 89 pgs: 89 active+clean; 243 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:19:25.265 INFO:tasks.rados.rados.0.smithi163.stdout:3316: done (0 left) 2023-10-09T09:19:25.265 INFO:tasks.rados.rados.0.smithi163.stdout:3317: rollback oid 50 current snap is 416 2023-10-09T09:19:25.266 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 50 to 385 2023-10-09T09:19:25.266 INFO:tasks.rados.rados.0.smithi163.stdout:3318: rollback oid 12 current snap is 416 2023-10-09T09:19:25.266 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 12 to 403 2023-10-09T09:19:25.266 INFO:tasks.rados.rados.0.smithi163.stdout:3319: delete oid 33 current snap is 416 2023-10-09T09:19:25.275 INFO:tasks.rados.rados.0.smithi163.stdout:3318: finishing rollback tid 0 to smithi16367185-12 2023-10-09T09:19:25.275 INFO:tasks.rados.rados.0.smithi163.stdout:3318: finishing rollback tid 1 to smithi16367185-12 2023-10-09T09:19:25.275 INFO:tasks.rados.rados.0.smithi163.stdout:3318: finishing rollback tid 2 to smithi16367185-12 2023-10-09T09:19:25.275 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 1552 (ObjNum 1164 snap 392 seq_num 1164) dirty exists 2023-10-09T09:19:25.277 INFO:tasks.rados.rados.0.smithi163.stdout:3318: done (2 left) 2023-10-09T09:19:25.277 INFO:tasks.rados.rados.0.smithi163.stdout:3319: done (1 left) 2023-10-09T09:19:25.277 INFO:tasks.rados.rados.0.smithi163.stdout:3320: snap_create 2023-10-09T09:19:25.284 INFO:tasks.rados.rados.0.smithi163.stdout:3317: finishing rollback tid 0 to smithi16367185-50 2023-10-09T09:19:25.284 INFO:tasks.rados.rados.0.smithi163.stdout:3317: finishing rollback tid 1 to smithi16367185-50 2023-10-09T09:19:25.284 INFO:tasks.rados.rados.0.smithi163.stdout:3317: finishing rollback tid 2 to smithi16367185-50 2023-10-09T09:19:25.284 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1561 (ObjNum 1086 snap 362 seq_num 1086) dirty exists 2023-10-09T09:19:25.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:25 smithi181 bash[59345]: cluster 2023-10-09T09:19:24.295443+0000 osd.5 (osd.5) 430 : cluster 3 failed to encode map e890 with expected crc 2023-10-09T09:19:25.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:25 smithi181 bash[59345]: cluster 2023-10-09T09:19:24.298046+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e890 with expected crc 2023-10-09T09:19:25.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:25 smithi181 bash[59345]: cluster 2023-10-09T09:19:24.302100+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e890: 8 total, 8 up, 8 in 2023-10-09T09:19:25.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:25 smithi181 bash[59345]: cluster 2023-10-09T09:19:25.264210+0000 mon.a (mon.0) 1097 : cluster 0 osdmap e891: 8 total, 8 up, 8 in 2023-10-09T09:19:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:25 smithi163 bash[75425]: cluster 2023-10-09T09:19:24.295443+0000 osd.5 (osd.5) 430 : cluster 3 failed to encode map e890 with expected crc 2023-10-09T09:19:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:25 smithi163 bash[75425]: cluster 2023-10-09T09:19:24.298046+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e890 with expected crc 2023-10-09T09:19:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:25 smithi163 bash[75425]: cluster 2023-10-09T09:19:24.302100+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e890: 8 total, 8 up, 8 in 2023-10-09T09:19:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:25 smithi163 bash[75425]: cluster 2023-10-09T09:19:25.264210+0000 mon.a (mon.0) 1097 : cluster 0 osdmap e891: 8 total, 8 up, 8 in 2023-10-09T09:19:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:25 smithi163 bash[78107]: cluster 2023-10-09T09:19:24.295443+0000 osd.5 (osd.5) 430 : cluster 3 failed to encode map e890 with expected crc 2023-10-09T09:19:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:25 smithi163 bash[78107]: cluster 2023-10-09T09:19:24.298046+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e890 with expected crc 2023-10-09T09:19:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:25 smithi163 bash[78107]: cluster 2023-10-09T09:19:24.302100+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e890: 8 total, 8 up, 8 in 2023-10-09T09:19:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:25 smithi163 bash[78107]: cluster 2023-10-09T09:19:25.264210+0000 mon.a (mon.0) 1097 : cluster 0 osdmap e891: 8 total, 8 up, 8 in 2023-10-09T09:19:26.275 INFO:tasks.rados.rados.0.smithi163.stdout:3317: done (1 left) 2023-10-09T09:19:26.275 INFO:tasks.rados.rados.0.smithi163.stdout:3320: done (0 left) 2023-10-09T09:19:26.275 INFO:tasks.rados.rados.0.smithi163.stdout:3321: snap_create 2023-10-09T09:19:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:26 smithi181 bash[59345]: cluster 2023-10-09T09:19:25.259152+0000 osd.5 (osd.5) 431 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:26 smithi181 bash[59345]: cluster 2023-10-09T09:19:25.264694+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:26 smithi181 bash[59345]: cluster 2023-10-09T09:19:25.264914+0000 osd.5 (osd.5) 432 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:26 smithi181 bash[59345]: cluster 2023-10-09T09:19:25.264954+0000 osd.4 (osd.4) 388 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:26 smithi181 bash[59345]: cluster 2023-10-09T09:19:25.279749+0000 osd.7 (osd.7) 354 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:26 smithi181 bash[59345]: audit 2023-10-09T09:19:25.349205+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:26 smithi181 bash[59345]: cluster 2023-10-09T09:19:25.435851+0000 mgr.y (mgr.34107) 386 : cluster 0 pgmap v890: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 2 op/s 2023-10-09T09:19:26.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:26 smithi181 bash[59345]: cluster 2023-10-09T09:19:26.274915+0000 mon.a (mon.0) 1099 : cluster 0 osdmap e892: 8 total, 8 up, 8 in 2023-10-09T09:19:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[75425]: cluster 2023-10-09T09:19:25.259152+0000 osd.5 (osd.5) 431 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[75425]: cluster 2023-10-09T09:19:25.264694+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[75425]: cluster 2023-10-09T09:19:25.264914+0000 osd.5 (osd.5) 432 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[75425]: cluster 2023-10-09T09:19:25.264954+0000 osd.4 (osd.4) 388 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[75425]: cluster 2023-10-09T09:19:25.279749+0000 osd.7 (osd.7) 354 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[75425]: audit 2023-10-09T09:19:25.349205+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[75425]: cluster 2023-10-09T09:19:25.435851+0000 mgr.y (mgr.34107) 386 : cluster 0 pgmap v890: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 2 op/s 2023-10-09T09:19:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[75425]: cluster 2023-10-09T09:19:26.274915+0000 mon.a (mon.0) 1099 : cluster 0 osdmap e892: 8 total, 8 up, 8 in 2023-10-09T09:19:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[78107]: cluster 2023-10-09T09:19:25.259152+0000 osd.5 (osd.5) 431 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[78107]: cluster 2023-10-09T09:19:25.264694+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[78107]: cluster 2023-10-09T09:19:25.264914+0000 osd.5 (osd.5) 432 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[78107]: cluster 2023-10-09T09:19:25.264954+0000 osd.4 (osd.4) 388 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[78107]: cluster 2023-10-09T09:19:25.279749+0000 osd.7 (osd.7) 354 : cluster 3 failed to encode map e891 with expected crc 2023-10-09T09:19:26.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[78107]: audit 2023-10-09T09:19:25.349205+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:26.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[78107]: cluster 2023-10-09T09:19:25.435851+0000 mgr.y (mgr.34107) 386 : cluster 0 pgmap v890: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 2 op/s 2023-10-09T09:19:26.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:26 smithi163 bash[78107]: cluster 2023-10-09T09:19:26.274915+0000 mon.a (mon.0) 1099 : cluster 0 osdmap e892: 8 total, 8 up, 8 in 2023-10-09T09:19:27.278 INFO:tasks.rados.rados.0.smithi163.stdout:3321: done (0 left) 2023-10-09T09:19:27.278 INFO:tasks.rados.rados.0.smithi163.stdout:3322: snap_remove snap 413 2023-10-09T09:19:27.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:27 smithi181 bash[59345]: cluster 2023-10-09T09:19:26.269710+0000 osd.5 (osd.5) 433 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:27.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:27 smithi181 bash[59345]: cluster 2023-10-09T09:19:27.277450+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e893: 8 total, 8 up, 8 in 2023-10-09T09:19:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:27 smithi163 bash[78107]: cluster 2023-10-09T09:19:26.269710+0000 osd.5 (osd.5) 433 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:27 smithi163 bash[78107]: cluster 2023-10-09T09:19:27.277450+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e893: 8 total, 8 up, 8 in 2023-10-09T09:19:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:27 smithi163 bash[75425]: cluster 2023-10-09T09:19:26.269710+0000 osd.5 (osd.5) 433 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:27 smithi163 bash[75425]: cluster 2023-10-09T09:19:27.277450+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e893: 8 total, 8 up, 8 in 2023-10-09T09:19:28.282 INFO:tasks.rados.rados.0.smithi163.stdout:3322: done (0 left) 2023-10-09T09:19:28.282 INFO:tasks.rados.rados.0.smithi163.stdout:3323: snap_remove snap 411 2023-10-09T09:19:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:28 smithi181 bash[59345]: cluster 2023-10-09T09:19:26.834484+0000 osd.7 (osd.7) 355 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:28 smithi181 bash[59345]: cluster 2023-10-09T09:19:26.837209+0000 osd.7 (osd.7) 356 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:28 smithi181 bash[59345]: cluster 2023-10-09T09:19:27.272234+0000 osd.5 (osd.5) 434 : cluster 3 failed to encode map e893 with expected crc 2023-10-09T09:19:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:28 smithi181 bash[59345]: cluster 2023-10-09T09:19:27.436569+0000 mgr.y (mgr.34107) 387 : cluster 0 pgmap v893: 89 pgs: 89 active+clean; 247 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:19:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:28 smithi181 bash[59345]: cluster 2023-10-09T09:19:28.282147+0000 mon.a (mon.0) 1101 : cluster 0 osdmap e894: 8 total, 8 up, 8 in 2023-10-09T09:19:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:28 smithi163 bash[78107]: cluster 2023-10-09T09:19:26.834484+0000 osd.7 (osd.7) 355 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:28 smithi163 bash[78107]: cluster 2023-10-09T09:19:26.837209+0000 osd.7 (osd.7) 356 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:28 smithi163 bash[78107]: cluster 2023-10-09T09:19:27.272234+0000 osd.5 (osd.5) 434 : cluster 3 failed to encode map e893 with expected crc 2023-10-09T09:19:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:28 smithi163 bash[78107]: cluster 2023-10-09T09:19:27.436569+0000 mgr.y (mgr.34107) 387 : cluster 0 pgmap v893: 89 pgs: 89 active+clean; 247 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:19:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:28 smithi163 bash[78107]: cluster 2023-10-09T09:19:28.282147+0000 mon.a (mon.0) 1101 : cluster 0 osdmap e894: 8 total, 8 up, 8 in 2023-10-09T09:19:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:28 smithi163 bash[75425]: cluster 2023-10-09T09:19:26.834484+0000 osd.7 (osd.7) 355 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:28 smithi163 bash[75425]: cluster 2023-10-09T09:19:26.837209+0000 osd.7 (osd.7) 356 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:28 smithi163 bash[75425]: cluster 2023-10-09T09:19:27.272234+0000 osd.5 (osd.5) 434 : cluster 3 failed to encode map e893 with expected crc 2023-10-09T09:19:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:28 smithi163 bash[75425]: cluster 2023-10-09T09:19:27.436569+0000 mgr.y (mgr.34107) 387 : cluster 0 pgmap v893: 89 pgs: 89 active+clean; 247 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:19:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:28 smithi163 bash[75425]: cluster 2023-10-09T09:19:28.282147+0000 mon.a (mon.0) 1101 : cluster 0 osdmap e894: 8 total, 8 up, 8 in 2023-10-09T09:19:29.284 INFO:tasks.rados.rados.0.smithi163.stdout:3323: done (0 left) 2023-10-09T09:19:29.285 INFO:tasks.rados.rados.0.smithi163.stdout:3324: snap_create 2023-10-09T09:19:29.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:29 smithi181 bash[59345]: cluster 2023-10-09T09:19:27.805197+0000 osd.4 (osd.4) 389 : cluster 3 failed to encode map e893 with expected crc 2023-10-09T09:19:29.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:29 smithi181 bash[59345]: cluster 2023-10-09T09:19:27.805266+0000 osd.7 (osd.7) 357 : cluster 3 failed to encode map e893 with expected crc 2023-10-09T09:19:29.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:29 smithi181 bash[59345]: cluster 2023-10-09T09:19:27.805362+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:29.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:29 smithi181 bash[59345]: cluster 2023-10-09T09:19:28.277328+0000 osd.5 (osd.5) 435 : cluster 3 failed to encode map e894 with expected crc 2023-10-09T09:19:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:29 smithi181 bash[59345]: cluster 2023-10-09T09:19:28.277670+0000 osd.7 (osd.7) 358 : cluster 3 failed to encode map e894 with expected crc 2023-10-09T09:19:29.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:29 smithi181 bash[59345]: cluster 2023-10-09T09:19:29.283566+0000 mon.a (mon.0) 1102 : cluster 0 osdmap e895: 8 total, 8 up, 8 in 2023-10-09T09:19:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[78107]: cluster 2023-10-09T09:19:27.805197+0000 osd.4 (osd.4) 389 : cluster 3 failed to encode map e893 with expected crc 2023-10-09T09:19:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[78107]: cluster 2023-10-09T09:19:27.805266+0000 osd.7 (osd.7) 357 : cluster 3 failed to encode map e893 with expected crc 2023-10-09T09:19:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[78107]: cluster 2023-10-09T09:19:27.805362+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[78107]: cluster 2023-10-09T09:19:28.277328+0000 osd.5 (osd.5) 435 : cluster 3 failed to encode map e894 with expected crc 2023-10-09T09:19:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[78107]: cluster 2023-10-09T09:19:28.277670+0000 osd.7 (osd.7) 358 : cluster 3 failed to encode map e894 with expected crc 2023-10-09T09:19:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[78107]: cluster 2023-10-09T09:19:29.283566+0000 mon.a (mon.0) 1102 : cluster 0 osdmap e895: 8 total, 8 up, 8 in 2023-10-09T09:19:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[75425]: cluster 2023-10-09T09:19:27.805197+0000 osd.4 (osd.4) 389 : cluster 3 failed to encode map e893 with expected crc 2023-10-09T09:19:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[75425]: cluster 2023-10-09T09:19:27.805266+0000 osd.7 (osd.7) 357 : cluster 3 failed to encode map e893 with expected crc 2023-10-09T09:19:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[75425]: cluster 2023-10-09T09:19:27.805362+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e892 with expected crc 2023-10-09T09:19:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[75425]: cluster 2023-10-09T09:19:28.277328+0000 osd.5 (osd.5) 435 : cluster 3 failed to encode map e894 with expected crc 2023-10-09T09:19:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[75425]: cluster 2023-10-09T09:19:28.277670+0000 osd.7 (osd.7) 358 : cluster 3 failed to encode map e894 with expected crc 2023-10-09T09:19:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:29 smithi163 bash[75425]: cluster 2023-10-09T09:19:29.283566+0000 mon.a (mon.0) 1102 : cluster 0 osdmap e895: 8 total, 8 up, 8 in 2023-10-09T09:19:30.285 INFO:tasks.rados.rados.0.smithi163.stdout:3324: done (0 left) 2023-10-09T09:19:30.285 INFO:tasks.rados.rados.0.smithi163.stdout:3325: snap_remove snap 412 2023-10-09T09:19:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:30 smithi163 bash[78107]: cluster 2023-10-09T09:19:29.277539+0000 osd.5 (osd.5) 436 : cluster 3 failed to encode map e895 with expected crc 2023-10-09T09:19:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:30 smithi163 bash[78107]: cluster 2023-10-09T09:19:29.286264+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e895 with expected crc 2023-10-09T09:19:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:30 smithi163 bash[78107]: cluster 2023-10-09T09:19:29.437258+0000 mgr.y (mgr.34107) 388 : cluster 0 pgmap v896: 89 pgs: 89 active+clean; 247 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2023-10-09T09:19:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:30 smithi163 bash[78107]: cluster 2023-10-09T09:19:30.284313+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e896: 8 total, 8 up, 8 in 2023-10-09T09:19:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:30 smithi163 bash[75425]: cluster 2023-10-09T09:19:29.277539+0000 osd.5 (osd.5) 436 : cluster 3 failed to encode map e895 with expected crc 2023-10-09T09:19:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:30 smithi163 bash[75425]: cluster 2023-10-09T09:19:29.286264+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e895 with expected crc 2023-10-09T09:19:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:30 smithi163 bash[75425]: cluster 2023-10-09T09:19:29.437258+0000 mgr.y (mgr.34107) 388 : cluster 0 pgmap v896: 89 pgs: 89 active+clean; 247 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2023-10-09T09:19:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:30 smithi163 bash[75425]: cluster 2023-10-09T09:19:30.284313+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e896: 8 total, 8 up, 8 in 2023-10-09T09:19:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:30 smithi181 bash[59345]: cluster 2023-10-09T09:19:29.277539+0000 osd.5 (osd.5) 436 : cluster 3 failed to encode map e895 with expected crc 2023-10-09T09:19:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:30 smithi181 bash[59345]: cluster 2023-10-09T09:19:29.286264+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e895 with expected crc 2023-10-09T09:19:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:30 smithi181 bash[59345]: cluster 2023-10-09T09:19:29.437258+0000 mgr.y (mgr.34107) 388 : cluster 0 pgmap v896: 89 pgs: 89 active+clean; 247 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2023-10-09T09:19:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:30 smithi181 bash[59345]: cluster 2023-10-09T09:19:30.284313+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e896: 8 total, 8 up, 8 in 2023-10-09T09:19:31.287 INFO:tasks.rados.rados.0.smithi163.stdout:3325: done (0 left) 2023-10-09T09:19:31.288 INFO:tasks.rados.rados.0.smithi163.stdout:3326: read oid 42 snap -1 2023-10-09T09:19:31.288 INFO:tasks.rados.rados.0.smithi163.stdout:3326: expect (ObjNum 1229 snap 410 seq_num 1229) 2023-10-09T09:19:31.288 INFO:tasks.rados.rados.0.smithi163.stdout:3327: rollback oid 23 current snap is 419 2023-10-09T09:19:31.288 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 23 to 416 2023-10-09T09:19:31.288 INFO:tasks.rados.rados.0.smithi163.stdout:3328: delete oid 41 current snap is 419 2023-10-09T09:19:31.297 INFO:tasks.rados.rados.0.smithi163.stdout:3328: done (2 left) 2023-10-09T09:19:31.297 INFO:tasks.rados.rados.0.smithi163.stdout:3329: delete oid 22 current snap is 419 2023-10-09T09:19:31.302 INFO:tasks.rados.rados.0.smithi163.stdout:3326: expect (ObjNum 1229 snap 410 seq_num 1229) 2023-10-09T09:19:31.438 INFO:tasks.rados.rados.0.smithi163.stdout:3327: finishing rollback tid 1 to smithi16367185-23 2023-10-09T09:19:31.439 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1491 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:19:31.439 INFO:tasks.rados.rados.0.smithi163.stdout:3326: done (2 left) 2023-10-09T09:19:31.439 INFO:tasks.rados.rados.0.smithi163.stdout:3327: done (1 left) 2023-10-09T09:19:31.439 INFO:tasks.rados.rados.0.smithi163.stdout:3329: done (0 left) 2023-10-09T09:19:31.439 INFO:tasks.rados.rados.0.smithi163.stdout:3330: read oid 31 snap 402 2023-10-09T09:19:31.439 INFO:tasks.rados.rados.0.smithi163.stdout:3330: expect (ObjNum 1204 snap 402 seq_num 1204) 2023-10-09T09:19:31.439 INFO:tasks.rados.rados.0.smithi163.stdout:3331: delete oid 8 current snap is 419 2023-10-09T09:19:31.444 INFO:tasks.rados.rados.0.smithi163.stdout:3330: expect (ObjNum 1204 snap 402 seq_num 1204) 2023-10-09T09:19:31.577 INFO:tasks.rados.rados.0.smithi163.stdout:3330: done (1 left) 2023-10-09T09:19:31.577 INFO:tasks.rados.rados.0.smithi163.stdout:3331: done (0 left) 2023-10-09T09:19:31.577 INFO:tasks.rados.rados.0.smithi163.stdout:3332: write oid 23 current snap is 419 2023-10-09T09:19:31.577 INFO:tasks.rados.rados.0.smithi163.stdout:3332: seq_num 1243 ranges {491101=500046,1487353=488104,2401133=489968} 2023-10-09T09:19:31.587 INFO:tasks.rados.rados.0.smithi163.stdout:3332: writing smithi16367185-23 from 491101 to 991147 tid 1 2023-10-09T09:19:31.593 INFO:tasks.rados.rados.0.smithi163.stdout:3332: writing smithi16367185-23 from 1487353 to 1975457 tid 2 2023-10-09T09:19:31.598 INFO:tasks.rados.rados.0.smithi163.stdout:3332: writing smithi16367185-23 from 2401133 to 2891101 tid 3 2023-10-09T09:19:31.598 INFO:tasks.rados.rados.0.smithi163.stdout:3333: snap_create 2023-10-09T09:19:31.598 INFO:tasks.rados.rados.0.smithi163.stdout:3332: finishing write tid 1 to smithi16367185-23 2023-10-09T09:19:31.599 INFO:tasks.rados.rados.0.smithi163.stdout:3332: finishing write tid 2 to smithi16367185-23 2023-10-09T09:19:31.603 INFO:tasks.rados.rados.0.smithi163.stdout:3332: finishing write tid 3 to smithi16367185-23 2023-10-09T09:19:31.603 INFO:tasks.rados.rados.0.smithi163.stdout:3332: finishing write tid 4 to smithi16367185-23 2023-10-09T09:19:31.604 INFO:tasks.rados.rados.0.smithi163.stdout:3332: finishing write tid 5 to smithi16367185-23 2023-10-09T09:19:31.604 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1574 (ObjNum 1243 snap 419 seq_num 1243) dirty exists 2023-10-09T09:19:31.604 INFO:tasks.rados.rados.0.smithi163.stdout:3332: left oid 23 (ObjNum 1243 snap 419 seq_num 1243) 2023-10-09T09:19:31.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:31 smithi163 bash[75425]: cluster 2023-10-09T09:19:29.782206+0000 osd.7 (osd.7) 359 : cluster 3 failed to encode map e895 with expected crc 2023-10-09T09:19:31.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:31 smithi163 bash[75425]: cluster 2023-10-09T09:19:30.279575+0000 osd.5 (osd.5) 437 : cluster 3 failed to encode map e896 with expected crc 2023-10-09T09:19:31.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:31 smithi163 bash[75425]: cluster 2023-10-09T09:19:31.287122+0000 mon.a (mon.0) 1104 : cluster 0 osdmap e897: 8 total, 8 up, 8 in 2023-10-09T09:19:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:31 smithi163 bash[78107]: cluster 2023-10-09T09:19:29.782206+0000 osd.7 (osd.7) 359 : cluster 3 failed to encode map e895 with expected crc 2023-10-09T09:19:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:31 smithi163 bash[78107]: cluster 2023-10-09T09:19:30.279575+0000 osd.5 (osd.5) 437 : cluster 3 failed to encode map e896 with expected crc 2023-10-09T09:19:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:31 smithi163 bash[78107]: cluster 2023-10-09T09:19:31.287122+0000 mon.a (mon.0) 1104 : cluster 0 osdmap e897: 8 total, 8 up, 8 in 2023-10-09T09:19:31.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:31 smithi181 bash[59345]: cluster 2023-10-09T09:19:29.782206+0000 osd.7 (osd.7) 359 : cluster 3 failed to encode map e895 with expected crc 2023-10-09T09:19:31.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:31 smithi181 bash[59345]: cluster 2023-10-09T09:19:30.279575+0000 osd.5 (osd.5) 437 : cluster 3 failed to encode map e896 with expected crc 2023-10-09T09:19:31.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:31 smithi181 bash[59345]: cluster 2023-10-09T09:19:31.287122+0000 mon.a (mon.0) 1104 : cluster 0 osdmap e897: 8 total, 8 up, 8 in 2023-10-09T09:19:32.344 INFO:tasks.rados.rados.0.smithi163.stdout:3332: done (1 left) 2023-10-09T09:19:32.344 INFO:tasks.rados.rados.0.smithi163.stdout:3333: done (0 left) 2023-10-09T09:19:32.344 INFO:tasks.rados.rados.0.smithi163.stdout:3334: read oid 33 snap 416 2023-10-09T09:19:32.344 INFO:tasks.rados.rados.0.smithi163.stdout:3334: expect deleted 2023-10-09T09:19:32.344 INFO:tasks.rados.rados.0.smithi163.stdout:3335: read oid 22 snap -1 2023-10-09T09:19:32.344 INFO:tasks.rados.rados.0.smithi163.stdout:3335: expect deleted 2023-10-09T09:19:32.344 INFO:tasks.rados.rados.0.smithi163.stdout:3336: write oid 8 current snap is 420 2023-10-09T09:19:32.344 INFO:tasks.rados.rados.0.smithi163.stdout:3336: seq_num 1244 ranges {507908=575295,1829482=631762} 2023-10-09T09:19:32.354 INFO:tasks.rados.rados.0.smithi163.stdout:3336: writing smithi16367185-8 from 507908 to 1083203 tid 1 2023-10-09T09:19:32.362 INFO:tasks.rados.rados.0.smithi163.stdout:3336: writing smithi16367185-8 from 1829482 to 2461244 tid 2 2023-10-09T09:19:32.362 INFO:tasks.rados.rados.0.smithi163.stdout:3337: rollback oid 42 current snap is 420 2023-10-09T09:19:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 42 to 409 2023-10-09T09:19:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3334: done (3 left) 2023-10-09T09:19:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3335: done (2 left) 2023-10-09T09:19:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3338: read oid 38 snap -1 2023-10-09T09:19:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3338: expect (ObjNum 1196 snap 402 seq_num 1196) 2023-10-09T09:19:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3339: snap_remove snap 403 2023-10-09T09:19:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:32 smithi163 bash[75425]: cluster 2023-10-09T09:19:30.611803+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e896 with expected crc 2023-10-09T09:19:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:32 smithi163 bash[75425]: cluster 2023-10-09T09:19:31.282188+0000 osd.5 (osd.5) 438 : cluster 3 failed to encode map e897 with expected crc 2023-10-09T09:19:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:32 smithi163 bash[75425]: cluster 2023-10-09T09:19:31.291646+0000 osd.7 (osd.7) 360 : cluster 3 failed to encode map e897 with expected crc 2023-10-09T09:19:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:32 smithi163 bash[75425]: cluster 2023-10-09T09:19:31.437881+0000 mgr.y (mgr.34107) 389 : cluster 0 pgmap v899: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 247 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 320 KiB/s rd, 0 B/s wr, 1 op/s 2023-10-09T09:19:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:32 smithi163 bash[78107]: cluster 2023-10-09T09:19:30.611803+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e896 with expected crc 2023-10-09T09:19:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:32 smithi163 bash[78107]: cluster 2023-10-09T09:19:31.282188+0000 osd.5 (osd.5) 438 : cluster 3 failed to encode map e897 with expected crc 2023-10-09T09:19:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:32 smithi163 bash[78107]: cluster 2023-10-09T09:19:31.291646+0000 osd.7 (osd.7) 360 : cluster 3 failed to encode map e897 with expected crc 2023-10-09T09:19:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:32 smithi163 bash[78107]: cluster 2023-10-09T09:19:31.437881+0000 mgr.y (mgr.34107) 389 : cluster 0 pgmap v899: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 247 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 320 KiB/s rd, 0 B/s wr, 1 op/s 2023-10-09T09:19:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:32 smithi181 bash[59345]: cluster 2023-10-09T09:19:30.611803+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e896 with expected crc 2023-10-09T09:19:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:32 smithi181 bash[59345]: cluster 2023-10-09T09:19:31.282188+0000 osd.5 (osd.5) 438 : cluster 3 failed to encode map e897 with expected crc 2023-10-09T09:19:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:32 smithi181 bash[59345]: cluster 2023-10-09T09:19:31.291646+0000 osd.7 (osd.7) 360 : cluster 3 failed to encode map e897 with expected crc 2023-10-09T09:19:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:32 smithi181 bash[59345]: cluster 2023-10-09T09:19:31.437881+0000 mgr.y (mgr.34107) 389 : cluster 0 pgmap v899: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 247 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 320 KiB/s rd, 0 B/s wr, 1 op/s 2023-10-09T09:19:33.355 INFO:tasks.rados.rados.0.smithi163.stdout:3339: done (3 left) 2023-10-09T09:19:33.355 INFO:tasks.rados.rados.0.smithi163.stdout:3340: rollback oid 11 current snap is 420 2023-10-09T09:19:33.355 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 11 to 404 2023-10-09T09:19:33.355 INFO:tasks.rados.rados.0.smithi163.stdout:3336: finishing write tid 1 to smithi16367185-8 2023-10-09T09:19:33.355 INFO:tasks.rados.rados.0.smithi163.stdout:3337: finishing rollback tid 0 to smithi16367185-42 2023-10-09T09:19:33.355 INFO:tasks.rados.rados.0.smithi163.stdout:3336: finishing write tid 2 to smithi16367185-8 2023-10-09T09:19:33.356 INFO:tasks.rados.rados.0.smithi163.stdout:3336: finishing write tid 3 to smithi16367185-8 2023-10-09T09:19:33.356 INFO:tasks.rados.rados.0.smithi163.stdout:3336: finishing write tid 4 to smithi16367185-8 2023-10-09T09:19:33.356 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1689 (ObjNum 1244 snap 420 seq_num 1244) dirty exists 2023-10-09T09:19:33.356 INFO:tasks.rados.rados.0.smithi163.stdout:3336: left oid 8 (ObjNum 1244 snap 420 seq_num 1244) 2023-10-09T09:19:33.356 INFO:tasks.rados.rados.0.smithi163.stdout:3337: finishing rollback tid 1 to smithi16367185-42 2023-10-09T09:19:33.356 INFO:tasks.rados.rados.0.smithi163.stdout:3337: finishing rollback tid 2 to smithi16367185-42 2023-10-09T09:19:33.356 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 2434 (ObjNum 1168 snap 394 seq_num 1168) dirty exists 2023-10-09T09:19:33.356 INFO:tasks.rados.rados.0.smithi163.stdout:3338: expect (ObjNum 1196 snap 402 seq_num 1196) 2023-10-09T09:19:33.502 INFO:tasks.rados.rados.0.smithi163.stdout:3340: finishing rollback tid 1 to smithi16367185-11 2023-10-09T09:19:33.502 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 823 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:19:33.502 INFO:tasks.rados.rados.0.smithi163.stdout:3336: done (3 left) 2023-10-09T09:19:33.502 INFO:tasks.rados.rados.0.smithi163.stdout:3337: done (2 left) 2023-10-09T09:19:33.502 INFO:tasks.rados.rados.0.smithi163.stdout:3338: done (1 left) 2023-10-09T09:19:33.502 INFO:tasks.rados.rados.0.smithi163.stdout:3340: done (0 left) 2023-10-09T09:19:33.502 INFO:tasks.rados.rados.0.smithi163.stdout:3341: read oid 38 snap -1 2023-10-09T09:19:33.502 INFO:tasks.rados.rados.0.smithi163.stdout:3341: expect (ObjNum 1196 snap 402 seq_num 1196) 2023-10-09T09:19:33.503 INFO:tasks.rados.rados.0.smithi163.stdout:3342: rollback oid 28 current snap is 420 2023-10-09T09:19:33.503 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 402 2023-10-09T09:19:33.503 INFO:tasks.rados.rados.0.smithi163.stdout:3343: read oid 45 snap -1 2023-10-09T09:19:33.503 INFO:tasks.rados.rados.0.smithi163.stdout:3343: expect deleted 2023-10-09T09:19:33.503 INFO:tasks.rados.rados.0.smithi163.stdout:3344: read oid 34 snap -1 2023-10-09T09:19:33.503 INFO:tasks.rados.rados.0.smithi163.stdout:3344: expect deleted 2023-10-09T09:19:33.503 INFO:tasks.rados.rados.0.smithi163.stdout:3345: delete oid 31 current snap is 420 2023-10-09T09:19:33.506 INFO:tasks.rados.rados.0.smithi163.stdout:3342: finishing rollback tid 0 to smithi16367185-28 2023-10-09T09:19:33.507 INFO:tasks.rados.rados.0.smithi163.stdout:3343: done (4 left) 2023-10-09T09:19:33.507 INFO:tasks.rados.rados.0.smithi163.stdout:3344: done (3 left) 2023-10-09T09:19:33.507 INFO:tasks.rados.rados.0.smithi163.stdout:3345: done (2 left) 2023-10-09T09:19:33.507 INFO:tasks.rados.rados.0.smithi163.stdout:3346: delete oid 12 current snap is 420 2023-10-09T09:19:33.507 INFO:tasks.rados.rados.0.smithi163.stdout:3342: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:19:33.507 INFO:tasks.rados.rados.0.smithi163.stdout:3341: expect (ObjNum 1196 snap 402 seq_num 1196) 2023-10-09T09:19:33.656 INFO:tasks.rados.rados.0.smithi163.stdout:3342: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:19:33.656 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 1684 (ObjNum 1005 snap 342 seq_num 1005) dirty exists 2023-10-09T09:19:33.656 INFO:tasks.rados.rados.0.smithi163.stdout:3341: done (2 left) 2023-10-09T09:19:33.656 INFO:tasks.rados.rados.0.smithi163.stdout:3342: done (1 left) 2023-10-09T09:19:33.656 INFO:tasks.rados.rados.0.smithi163.stdout:3346: done (0 left) 2023-10-09T09:19:33.656 INFO:tasks.rados.rados.0.smithi163.stdout:3347: snap_remove snap 414 2023-10-09T09:19:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:19:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:19:33] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.33.4" 2023-10-09T09:19:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:33 smithi163 bash[78107]: cluster 2023-10-09T09:19:32.336263+0000 osd.5 (osd.5) 439 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:33 smithi163 bash[78107]: cluster 2023-10-09T09:19:32.339382+0000 osd.4 (osd.4) 390 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:33 smithi163 bash[78107]: cluster 2023-10-09T09:19:32.343320+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e898: 8 total, 8 up, 8 in 2023-10-09T09:19:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:33 smithi163 bash[78107]: cluster 2023-10-09T09:19:32.356024+0000 osd.7 (osd.7) 361 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:33 smithi163 bash[75425]: cluster 2023-10-09T09:19:32.336263+0000 osd.5 (osd.5) 439 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:33 smithi163 bash[75425]: cluster 2023-10-09T09:19:32.339382+0000 osd.4 (osd.4) 390 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:33 smithi163 bash[75425]: cluster 2023-10-09T09:19:32.343320+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e898: 8 total, 8 up, 8 in 2023-10-09T09:19:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:33 smithi163 bash[75425]: cluster 2023-10-09T09:19:32.356024+0000 osd.7 (osd.7) 361 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:33 smithi181 bash[59345]: cluster 2023-10-09T09:19:32.336263+0000 osd.5 (osd.5) 439 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:33 smithi181 bash[59345]: cluster 2023-10-09T09:19:32.339382+0000 osd.4 (osd.4) 390 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:33 smithi181 bash[59345]: cluster 2023-10-09T09:19:32.343320+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e898: 8 total, 8 up, 8 in 2023-10-09T09:19:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:33 smithi181 bash[59345]: cluster 2023-10-09T09:19:32.356024+0000 osd.7 (osd.7) 361 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:34.362 INFO:tasks.rados.rados.0.smithi163.stdout:3347: done (0 left) 2023-10-09T09:19:34.362 INFO:tasks.rados.rados.0.smithi163.stdout:3348: snap_remove snap 397 2023-10-09T09:19:34.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:34 smithi163 bash[78107]: cluster 2023-10-09T09:19:33.351336+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:34.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:34 smithi163 bash[78107]: cluster 2023-10-09T09:19:33.351352+0000 osd.5 (osd.5) 440 : cluster 3 failed to encode map e899 with expected crc 2023-10-09T09:19:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:34 smithi163 bash[78107]: cluster 2023-10-09T09:19:33.354772+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e899: 8 total, 8 up, 8 in 2023-10-09T09:19:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:34 smithi163 bash[78107]: cluster 2023-10-09T09:19:33.356311+0000 osd.7 (osd.7) 362 : cluster 3 failed to encode map e899 with expected crc 2023-10-09T09:19:34.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:34 smithi163 bash[78107]: cluster 2023-10-09T09:19:33.438485+0000 mgr.y (mgr.34107) 390 : cluster 0 pgmap v902: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 247 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:19:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:34 smithi163 bash[75425]: cluster 2023-10-09T09:19:33.351336+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:34 smithi163 bash[75425]: cluster 2023-10-09T09:19:33.351352+0000 osd.5 (osd.5) 440 : cluster 3 failed to encode map e899 with expected crc 2023-10-09T09:19:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:34 smithi163 bash[75425]: cluster 2023-10-09T09:19:33.354772+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e899: 8 total, 8 up, 8 in 2023-10-09T09:19:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:34 smithi163 bash[75425]: cluster 2023-10-09T09:19:33.356311+0000 osd.7 (osd.7) 362 : cluster 3 failed to encode map e899 with expected crc 2023-10-09T09:19:34.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:34 smithi163 bash[75425]: cluster 2023-10-09T09:19:33.438485+0000 mgr.y (mgr.34107) 390 : cluster 0 pgmap v902: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 247 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:19:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:34 smithi181 bash[59345]: cluster 2023-10-09T09:19:33.351336+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e898 with expected crc 2023-10-09T09:19:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:34 smithi181 bash[59345]: cluster 2023-10-09T09:19:33.351352+0000 osd.5 (osd.5) 440 : cluster 3 failed to encode map e899 with expected crc 2023-10-09T09:19:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:34 smithi181 bash[59345]: cluster 2023-10-09T09:19:33.354772+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e899: 8 total, 8 up, 8 in 2023-10-09T09:19:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:34 smithi181 bash[59345]: cluster 2023-10-09T09:19:33.356311+0000 osd.7 (osd.7) 362 : cluster 3 failed to encode map e899 with expected crc 2023-10-09T09:19:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:34 smithi181 bash[59345]: cluster 2023-10-09T09:19:33.438485+0000 mgr.y (mgr.34107) 390 : cluster 0 pgmap v902: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 247 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:19:35.268 INFO:tasks.rados.rados.0.smithi163.stdout:3348: done (0 left) 2023-10-09T09:19:35.268 INFO:tasks.rados.rados.0.smithi163.stdout:3349: delete oid 3 current snap is 420 2023-10-09T09:19:35.278 INFO:tasks.rados.rados.0.smithi163.stdout:3349: done (0 left) 2023-10-09T09:19:35.278 INFO:tasks.rados.rados.0.smithi163.stdout:3350: write oid 22 current snap is 420 2023-10-09T09:19:35.278 INFO:tasks.rados.rados.0.smithi163.stdout:3350: seq_num 1248 ranges {575136=592644,1663410=639100,2705542=269594} 2023-10-09T09:19:35.285 INFO:tasks.rados.rados.0.smithi163.stdout:3350: writing smithi16367185-22 from 575136 to 1167780 tid 1 2023-10-09T09:19:35.292 INFO:tasks.rados.rados.0.smithi163.stdout:3350: writing smithi16367185-22 from 1663410 to 2302510 tid 2 2023-10-09T09:19:35.296 INFO:tasks.rados.rados.0.smithi163.stdout:3350: writing smithi16367185-22 from 2705542 to 2975136 tid 3 2023-10-09T09:19:35.297 INFO:tasks.rados.rados.0.smithi163.stdout:3351: write oid 45 current snap is 420 2023-10-09T09:19:35.297 INFO:tasks.rados.rados.0.smithi163.stdout:3350: finishing write tid 1 to smithi16367185-22 2023-10-09T09:19:35.297 INFO:tasks.rados.rados.0.smithi163.stdout:3351: seq_num 1249 ranges {591943=667893,1721892=782758} 2023-10-09T09:19:35.304 INFO:tasks.rados.rados.0.smithi163.stdout:3351: writing smithi16367185-45 from 591943 to 1259836 tid 1 2023-10-09T09:19:35.312 INFO:tasks.rados.rados.0.smithi163.stdout:3351: writing smithi16367185-45 from 1721892 to 2504650 tid 2 2023-10-09T09:19:35.312 INFO:tasks.rados.rados.0.smithi163.stdout:3352: write oid 10 current snap is 420 2023-10-09T09:19:35.312 INFO:tasks.rados.rados.0.smithi163.stdout:3352: seq_num 1250 ranges {608750=743142,1780374=642769} 2023-10-09T09:19:35.320 INFO:tasks.rados.rados.0.smithi163.stdout:3352: writing smithi16367185-10 from 608750 to 1351892 tid 1 2023-10-09T09:19:35.327 INFO:tasks.rados.rados.0.smithi163.stdout:3352: writing smithi16367185-10 from 1780374 to 2423143 tid 2 2023-10-09T09:19:35.327 INFO:tasks.rados.rados.0.smithi163.stdout:3353: write oid 36 current snap is 420 2023-10-09T09:19:35.327 INFO:tasks.rados.rados.0.smithi163.stdout:3353: seq_num 1251 ranges {625557=418391,1838856=786427} 2023-10-09T09:19:35.334 INFO:tasks.rados.rados.0.smithi163.stdout:3353: writing smithi16367185-36 from 625557 to 1043948 tid 1 2023-10-09T09:19:35.343 INFO:tasks.rados.rados.0.smithi163.stdout:3353: writing smithi16367185-36 from 1838856 to 2625283 tid 2 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:3354: read oid 5 snap 419 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:3354: expect (ObjNum 1180 snap 397 seq_num 1180) 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:3355: delete oid 38 current snap is 420 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:3350: finishing write tid 2 to smithi16367185-22 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:3350: finishing write tid 3 to smithi16367185-22 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:3350: finishing write tid 4 to smithi16367185-22 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:3350: finishing write tid 5 to smithi16367185-22 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 2464 (ObjNum 1248 snap 420 seq_num 1248) dirty exists 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:3350: left oid 22 (ObjNum 1248 snap 420 seq_num 1248) 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:3351: finishing write tid 1 to smithi16367185-45 2023-10-09T09:19:35.344 INFO:tasks.rados.rados.0.smithi163.stdout:3351: finishing write tid 2 to smithi16367185-45 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:3351: finishing write tid 3 to smithi16367185-45 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:3351: finishing write tid 4 to smithi16367185-45 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 2467 (ObjNum 1249 snap 420 seq_num 1249) dirty exists 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:3351: left oid 45 (ObjNum 1249 snap 420 seq_num 1249) 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:3352: finishing write tid 1 to smithi16367185-10 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:3352: finishing write tid 2 to smithi16367185-10 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:3352: finishing write tid 3 to smithi16367185-10 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:3352: finishing write tid 4 to smithi16367185-10 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 1154 (ObjNum 1250 snap 420 seq_num 1250) dirty exists 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:3352: left oid 10 (ObjNum 1250 snap 420 seq_num 1250) 2023-10-09T09:19:35.345 INFO:tasks.rados.rados.0.smithi163.stdout:3353: finishing write tid 1 to smithi16367185-36 2023-10-09T09:19:35.348 INFO:tasks.rados.rados.0.smithi163.stdout:3350: done (5 left) 2023-10-09T09:19:35.348 INFO:tasks.rados.rados.0.smithi163.stdout:3351: done (4 left) 2023-10-09T09:19:35.348 INFO:tasks.rados.rados.0.smithi163.stdout:3352: done (3 left) 2023-10-09T09:19:35.348 INFO:tasks.rados.rados.0.smithi163.stdout:3355: done (2 left) 2023-10-09T09:19:35.348 INFO:tasks.rados.rados.0.smithi163.stdout:3356: read oid 2 snap 404 2023-10-09T09:19:35.348 INFO:tasks.rados.rados.0.smithi163.stdout:3356: expect deleted 2023-10-09T09:19:35.348 INFO:tasks.rados.rados.0.smithi163.stdout:3357: write oid 3 current snap is 420 2023-10-09T09:19:35.348 INFO:tasks.rados.rados.0.smithi163.stdout:3353: finishing write tid 2 to smithi16367185-36 2023-10-09T09:19:35.348 INFO:tasks.rados.rados.0.smithi163.stdout:3357: seq_num 1253 ranges {659171=568889,1839467=790096} 2023-10-09T09:19:35.356 INFO:tasks.rados.rados.0.smithi163.stdout:3357: writing smithi16367185-3 from 659171 to 1228060 tid 1 2023-10-09T09:19:35.366 INFO:tasks.rados.rados.0.smithi163.stdout:3357: writing smithi16367185-3 from 1839467 to 2629563 tid 2 2023-10-09T09:19:35.366 INFO:tasks.rados.rados.0.smithi163.stdout:3358: snap_create 2023-10-09T09:19:35.366 INFO:tasks.rados.rados.0.smithi163.stdout:3353: finishing write tid 3 to smithi16367185-36 2023-10-09T09:19:35.366 INFO:tasks.rados.rados.0.smithi163.stdout:3353: finishing write tid 4 to smithi16367185-36 2023-10-09T09:19:35.366 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 939 (ObjNum 1251 snap 420 seq_num 1251) dirty exists 2023-10-09T09:19:35.366 INFO:tasks.rados.rados.0.smithi163.stdout:3353: left oid 36 (ObjNum 1251 snap 420 seq_num 1251) 2023-10-09T09:19:35.366 INFO:tasks.rados.rados.0.smithi163.stdout:3354: expect (ObjNum 1180 snap 397 seq_num 1180) 2023-10-09T09:19:35.589 INFO:tasks.rados.rados.0.smithi163.stdout:3357: finishing write tid 1 to smithi16367185-3 2023-10-09T09:19:35.589 INFO:tasks.rados.rados.0.smithi163.stdout:3357: finishing write tid 2 to smithi16367185-3 2023-10-09T09:19:35.589 INFO:tasks.rados.rados.0.smithi163.stdout:3357: finishing write tid 3 to smithi16367185-3 2023-10-09T09:19:35.589 INFO:tasks.rados.rados.0.smithi163.stdout:3357: finishing write tid 4 to smithi16367185-3 2023-10-09T09:19:35.589 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 942 (ObjNum 1253 snap 420 seq_num 1253) dirty exists 2023-10-09T09:19:35.589 INFO:tasks.rados.rados.0.smithi163.stdout:3357: left oid 3 (ObjNum 1253 snap 420 seq_num 1253) 2023-10-09T09:19:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:35 smithi163 bash[78107]: cluster 2023-10-09T09:19:34.357529+0000 osd.5 (osd.5) 441 : cluster 3 failed to encode map e900 with expected crc 2023-10-09T09:19:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:35 smithi163 bash[78107]: cluster 2023-10-09T09:19:34.357790+0000 osd.4 (osd.4) 391 : cluster 3 failed to encode map e900 with expected crc 2023-10-09T09:19:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:35 smithi163 bash[78107]: cluster 2023-10-09T09:19:34.361796+0000 mon.a (mon.0) 1107 : cluster 0 osdmap e900: 8 total, 8 up, 8 in 2023-10-09T09:19:35.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:35 smithi163 bash[78107]: cluster 2023-10-09T09:19:35.267974+0000 mon.a (mon.0) 1108 : cluster 0 osdmap e901: 8 total, 8 up, 8 in 2023-10-09T09:19:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:35 smithi163 bash[75425]: cluster 2023-10-09T09:19:34.357529+0000 osd.5 (osd.5) 441 : cluster 3 failed to encode map e900 with expected crc 2023-10-09T09:19:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:35 smithi163 bash[75425]: cluster 2023-10-09T09:19:34.357790+0000 osd.4 (osd.4) 391 : cluster 3 failed to encode map e900 with expected crc 2023-10-09T09:19:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:35 smithi163 bash[75425]: cluster 2023-10-09T09:19:34.361796+0000 mon.a (mon.0) 1107 : cluster 0 osdmap e900: 8 total, 8 up, 8 in 2023-10-09T09:19:35.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:35 smithi163 bash[75425]: cluster 2023-10-09T09:19:35.267974+0000 mon.a (mon.0) 1108 : cluster 0 osdmap e901: 8 total, 8 up, 8 in 2023-10-09T09:19:35.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:35 smithi181 bash[59345]: cluster 2023-10-09T09:19:34.357529+0000 osd.5 (osd.5) 441 : cluster 3 failed to encode map e900 with expected crc 2023-10-09T09:19:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:35 smithi181 bash[59345]: cluster 2023-10-09T09:19:34.357790+0000 osd.4 (osd.4) 391 : cluster 3 failed to encode map e900 with expected crc 2023-10-09T09:19:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:35 smithi181 bash[59345]: cluster 2023-10-09T09:19:34.361796+0000 mon.a (mon.0) 1107 : cluster 0 osdmap e900: 8 total, 8 up, 8 in 2023-10-09T09:19:35.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:35 smithi181 bash[59345]: cluster 2023-10-09T09:19:35.267974+0000 mon.a (mon.0) 1108 : cluster 0 osdmap e901: 8 total, 8 up, 8 in 2023-10-09T09:19:36.371 INFO:tasks.rados.rados.0.smithi163.stdout:3353: done (4 left) 2023-10-09T09:19:36.371 INFO:tasks.rados.rados.0.smithi163.stdout:3354: done (3 left) 2023-10-09T09:19:36.371 INFO:tasks.rados.rados.0.smithi163.stdout:3356: done (2 left) 2023-10-09T09:19:36.371 INFO:tasks.rados.rados.0.smithi163.stdout:3357: done (1 left) 2023-10-09T09:19:36.372 INFO:tasks.rados.rados.0.smithi163.stdout:3358: done (0 left) 2023-10-09T09:19:36.372 INFO:tasks.rados.rados.0.smithi163.stdout:3359: snap_remove snap 402 2023-10-09T09:19:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[75425]: cluster 2023-10-09T09:19:34.777105+0000 osd.7 (osd.7) 363 : cluster 3 failed to encode map e900 with expected crc 2023-10-09T09:19:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[75425]: cluster 2023-10-09T09:19:35.263331+0000 osd.5 (osd.5) 442 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[75425]: cluster 2023-10-09T09:19:35.263894+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[75425]: cluster 2023-10-09T09:19:35.274262+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[75425]: cluster 2023-10-09T09:19:35.335226+0000 osd.7 (osd.7) 364 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[75425]: cluster 2023-10-09T09:19:35.438945+0000 mgr.y (mgr.34107) 391 : cluster 0 pgmap v905: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 247 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:19:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[78107]: cluster 2023-10-09T09:19:34.777105+0000 osd.7 (osd.7) 363 : cluster 3 failed to encode map e900 with expected crc 2023-10-09T09:19:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[78107]: cluster 2023-10-09T09:19:35.263331+0000 osd.5 (osd.5) 442 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[78107]: cluster 2023-10-09T09:19:35.263894+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[78107]: cluster 2023-10-09T09:19:35.274262+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[78107]: cluster 2023-10-09T09:19:35.335226+0000 osd.7 (osd.7) 364 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:36 smithi163 bash[78107]: cluster 2023-10-09T09:19:35.438945+0000 mgr.y (mgr.34107) 391 : cluster 0 pgmap v905: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 247 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:19:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:36 smithi181 bash[59345]: cluster 2023-10-09T09:19:34.777105+0000 osd.7 (osd.7) 363 : cluster 3 failed to encode map e900 with expected crc 2023-10-09T09:19:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:36 smithi181 bash[59345]: cluster 2023-10-09T09:19:35.263331+0000 osd.5 (osd.5) 442 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:36 smithi181 bash[59345]: cluster 2023-10-09T09:19:35.263894+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:36 smithi181 bash[59345]: cluster 2023-10-09T09:19:35.274262+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:36 smithi181 bash[59345]: cluster 2023-10-09T09:19:35.335226+0000 osd.7 (osd.7) 364 : cluster 3 failed to encode map e901 with expected crc 2023-10-09T09:19:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:36 smithi181 bash[59345]: cluster 2023-10-09T09:19:35.438945+0000 mgr.y (mgr.34107) 391 : cluster 0 pgmap v905: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 247 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 3 op/s 2023-10-09T09:19:37.375 INFO:tasks.rados.rados.0.smithi163.stdout:3359: done (0 left) 2023-10-09T09:19:37.375 INFO:tasks.rados.rados.0.smithi163.stdout:3360: read oid 7 snap 416 2023-10-09T09:19:37.375 INFO:tasks.rados.rados.0.smithi163.stdout:3360: expect (ObjNum 1178 snap 397 seq_num 1178) 2023-10-09T09:19:37.375 INFO:tasks.rados.rados.0.smithi163.stdout:3361: snap_create 2023-10-09T09:19:37.394 INFO:tasks.rados.rados.0.smithi163.stdout:3360: expect (ObjNum 1178 snap 397 seq_num 1178) 2023-10-09T09:19:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:37 smithi163 bash[75425]: cluster 2023-10-09T09:19:36.366689+0000 osd.5 (osd.5) 443 : cluster 3 failed to encode map e902 with expected crc 2023-10-09T09:19:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:37 smithi163 bash[75425]: cluster 2023-10-09T09:19:36.371074+0000 mon.a (mon.0) 1109 : cluster 0 osdmap e902: 8 total, 8 up, 8 in 2023-10-09T09:19:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:37 smithi163 bash[78107]: cluster 2023-10-09T09:19:36.366689+0000 osd.5 (osd.5) 443 : cluster 3 failed to encode map e902 with expected crc 2023-10-09T09:19:37.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:37 smithi163 bash[78107]: cluster 2023-10-09T09:19:36.371074+0000 mon.a (mon.0) 1109 : cluster 0 osdmap e902: 8 total, 8 up, 8 in 2023-10-09T09:19:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:37 smithi181 bash[59345]: cluster 2023-10-09T09:19:36.366689+0000 osd.5 (osd.5) 443 : cluster 3 failed to encode map e902 with expected crc 2023-10-09T09:19:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:37 smithi181 bash[59345]: cluster 2023-10-09T09:19:36.371074+0000 mon.a (mon.0) 1109 : cluster 0 osdmap e902: 8 total, 8 up, 8 in 2023-10-09T09:19:38.395 INFO:tasks.rados.rados.0.smithi163.stdout:3360: done (1 left) 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3361: done (0 left) 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3362: read oid 18 snap -1 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3362: expect (ObjNum 1219 snap 404 seq_num 1219) 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3363: read oid 27 snap -1 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3363: expect (ObjNum 1234 snap 410 seq_num 1234) 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3364: read oid 38 snap -1 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3364: expect deleted 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3365: read oid 25 snap 406 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3365: expect (ObjNum 1162 snap 391 seq_num 1162) 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3366: read oid 49 snap 406 2023-10-09T09:19:38.396 INFO:tasks.rados.rados.0.smithi163.stdout:3366: expect (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:19:38.397 INFO:tasks.rados.rados.0.smithi163.stdout:3367: snap_remove snap 417 2023-10-09T09:19:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[78107]: cluster 2023-10-09T09:19:36.811170+0000 osd.7 (osd.7) 365 : cluster 3 failed to encode map e902 with expected crc 2023-10-09T09:19:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[78107]: cluster 2023-10-09T09:19:36.838186+0000 osd.4 (osd.4) 392 : cluster 3 failed to encode map e902 with expected crc 2023-10-09T09:19:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[78107]: cluster 2023-10-09T09:19:37.369269+0000 osd.5 (osd.5) 444 : cluster 3 failed to encode map e903 with expected crc 2023-10-09T09:19:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[78107]: cluster 2023-10-09T09:19:37.369818+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e903 with expected crc 2023-10-09T09:19:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[78107]: cluster 2023-10-09T09:19:37.374083+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e903: 8 total, 8 up, 8 in 2023-10-09T09:19:38.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[78107]: cluster 2023-10-09T09:19:37.439375+0000 mgr.y (mgr.34107) 392 : cluster 0 pgmap v908: 89 pgs: 89 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2023-10-09T09:19:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[75425]: cluster 2023-10-09T09:19:36.811170+0000 osd.7 (osd.7) 365 : cluster 3 failed to encode map e902 with expected crc 2023-10-09T09:19:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[75425]: cluster 2023-10-09T09:19:36.838186+0000 osd.4 (osd.4) 392 : cluster 3 failed to encode map e902 with expected crc 2023-10-09T09:19:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[75425]: cluster 2023-10-09T09:19:37.369269+0000 osd.5 (osd.5) 444 : cluster 3 failed to encode map e903 with expected crc 2023-10-09T09:19:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[75425]: cluster 2023-10-09T09:19:37.369818+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e903 with expected crc 2023-10-09T09:19:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[75425]: cluster 2023-10-09T09:19:37.374083+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e903: 8 total, 8 up, 8 in 2023-10-09T09:19:38.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:38 smithi163 bash[75425]: cluster 2023-10-09T09:19:37.439375+0000 mgr.y (mgr.34107) 392 : cluster 0 pgmap v908: 89 pgs: 89 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2023-10-09T09:19:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:38 smithi181 bash[59345]: cluster 2023-10-09T09:19:36.811170+0000 osd.7 (osd.7) 365 : cluster 3 failed to encode map e902 with expected crc 2023-10-09T09:19:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:38 smithi181 bash[59345]: cluster 2023-10-09T09:19:36.838186+0000 osd.4 (osd.4) 392 : cluster 3 failed to encode map e902 with expected crc 2023-10-09T09:19:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:38 smithi181 bash[59345]: cluster 2023-10-09T09:19:37.369269+0000 osd.5 (osd.5) 444 : cluster 3 failed to encode map e903 with expected crc 2023-10-09T09:19:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:38 smithi181 bash[59345]: cluster 2023-10-09T09:19:37.369818+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e903 with expected crc 2023-10-09T09:19:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:38 smithi181 bash[59345]: cluster 2023-10-09T09:19:37.374083+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e903: 8 total, 8 up, 8 in 2023-10-09T09:19:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:38 smithi181 bash[59345]: cluster 2023-10-09T09:19:37.439375+0000 mgr.y (mgr.34107) 392 : cluster 0 pgmap v908: 89 pgs: 89 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2023-10-09T09:19:39.397 INFO:tasks.rados.rados.0.smithi163.stdout:3367: done (5 left) 2023-10-09T09:19:39.398 INFO:tasks.rados.rados.0.smithi163.stdout:3368: rollback oid 20 current snap is 422 2023-10-09T09:19:39.398 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 373 2023-10-09T09:19:39.398 INFO:tasks.rados.rados.0.smithi163.stdout:3365: expect (ObjNum 1162 snap 391 seq_num 1162) 2023-10-09T09:19:39.653 INFO:tasks.rados.rados.0.smithi163.stdout:3363: expect (ObjNum 1234 snap 410 seq_num 1234) 2023-10-09T09:19:39.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:39 smithi163 bash[75425]: cluster 2023-10-09T09:19:38.388807+0000 osd.7 (osd.7) 366 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:39.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:39 smithi163 bash[75425]: cluster 2023-10-09T09:19:38.388822+0000 osd.5 (osd.5) 445 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:39.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:39 smithi163 bash[75425]: cluster 2023-10-09T09:19:38.394817+0000 mon.a (mon.0) 1111 : cluster 0 osdmap e904: 8 total, 8 up, 8 in 2023-10-09T09:19:39.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:39 smithi163 bash[75425]: cluster 2023-10-09T09:19:38.396557+0000 osd.4 (osd.4) 393 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:39 smithi163 bash[78107]: cluster 2023-10-09T09:19:38.388807+0000 osd.7 (osd.7) 366 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:39 smithi163 bash[78107]: cluster 2023-10-09T09:19:38.388822+0000 osd.5 (osd.5) 445 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:39 smithi163 bash[78107]: cluster 2023-10-09T09:19:38.394817+0000 mon.a (mon.0) 1111 : cluster 0 osdmap e904: 8 total, 8 up, 8 in 2023-10-09T09:19:39.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:39 smithi163 bash[78107]: cluster 2023-10-09T09:19:38.396557+0000 osd.4 (osd.4) 393 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:39.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:39 smithi181 bash[59345]: cluster 2023-10-09T09:19:38.388807+0000 osd.7 (osd.7) 366 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:39.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:39 smithi181 bash[59345]: cluster 2023-10-09T09:19:38.388822+0000 osd.5 (osd.5) 445 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:39.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:39 smithi181 bash[59345]: cluster 2023-10-09T09:19:38.394817+0000 mon.a (mon.0) 1111 : cluster 0 osdmap e904: 8 total, 8 up, 8 in 2023-10-09T09:19:39.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:39 smithi181 bash[59345]: cluster 2023-10-09T09:19:38.396557+0000 osd.4 (osd.4) 393 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:39.827 INFO:tasks.rados.rados.0.smithi163.stdout:3366: expect (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:19:39.962 INFO:tasks.rados.rados.0.smithi163.stdout:3362: expect (ObjNum 1219 snap 404 seq_num 1219) 2023-10-09T09:19:40.071 INFO:tasks.rados.rados.0.smithi163.stdout:3368: finishing rollback tid 0 to smithi16367185-20 2023-10-09T09:19:40.071 INFO:tasks.rados.rados.0.smithi163.stdout:3368: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:19:40.071 INFO:tasks.rados.rados.0.smithi163.stdout:3368: finishing rollback tid 2 to smithi16367185-20 2023-10-09T09:19:40.071 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 647 (ObjNum 1119 snap 369 seq_num 1119) dirty exists 2023-10-09T09:19:40.071 INFO:tasks.rados.rados.0.smithi163.stdout:3362: done (5 left) 2023-10-09T09:19:40.072 INFO:tasks.rados.rados.0.smithi163.stdout:3363: done (4 left) 2023-10-09T09:19:40.072 INFO:tasks.rados.rados.0.smithi163.stdout:3364: done (3 left) 2023-10-09T09:19:40.072 INFO:tasks.rados.rados.0.smithi163.stdout:3365: done (2 left) 2023-10-09T09:19:40.072 INFO:tasks.rados.rados.0.smithi163.stdout:3366: done (1 left) 2023-10-09T09:19:40.072 INFO:tasks.rados.rados.0.smithi163.stdout:3368: done (0 left) 2023-10-09T09:19:40.072 INFO:tasks.rados.rados.0.smithi163.stdout:3369: rollback oid 31 current snap is 422 2023-10-09T09:19:40.073 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 31 to 410 2023-10-09T09:19:40.073 INFO:tasks.rados.rados.0.smithi163.stdout:3370: snap_remove snap 416 2023-10-09T09:19:40.284 INFO:tasks.rados.rados.0.smithi163.stdout:3370: done (1 left) 2023-10-09T09:19:40.284 INFO:tasks.rados.rados.0.smithi163.stdout:3371: snap_remove snap 373 2023-10-09T09:19:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[75425]: cluster 2023-10-09T09:19:38.602930+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[75425]: cluster 2023-10-09T09:19:39.392267+0000 osd.5 (osd.5) 446 : cluster 3 failed to encode map e905 with expected crc 2023-10-09T09:19:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[75425]: cluster 2023-10-09T09:19:39.397203+0000 mon.a (mon.0) 1112 : cluster 0 osdmap e905: 8 total, 8 up, 8 in 2023-10-09T09:19:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[75425]: cluster 2023-10-09T09:19:39.398947+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e905 with expected crc 2023-10-09T09:19:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[75425]: cluster 2023-10-09T09:19:39.439845+0000 mgr.y (mgr.34107) 393 : cluster 0 pgmap v911: 89 pgs: 89 active+clean; 252 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2023-10-09T09:19:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[75425]: cluster 2023-10-09T09:19:40.282486+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e906: 8 total, 8 up, 8 in 2023-10-09T09:19:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[75425]: audit 2023-10-09T09:19:40.348831+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[78107]: cluster 2023-10-09T09:19:38.602930+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[78107]: cluster 2023-10-09T09:19:39.392267+0000 osd.5 (osd.5) 446 : cluster 3 failed to encode map e905 with expected crc 2023-10-09T09:19:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[78107]: cluster 2023-10-09T09:19:39.397203+0000 mon.a (mon.0) 1112 : cluster 0 osdmap e905: 8 total, 8 up, 8 in 2023-10-09T09:19:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[78107]: cluster 2023-10-09T09:19:39.398947+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e905 with expected crc 2023-10-09T09:19:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[78107]: cluster 2023-10-09T09:19:39.439845+0000 mgr.y (mgr.34107) 393 : cluster 0 pgmap v911: 89 pgs: 89 active+clean; 252 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2023-10-09T09:19:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[78107]: cluster 2023-10-09T09:19:40.282486+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e906: 8 total, 8 up, 8 in 2023-10-09T09:19:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:40 smithi163 bash[78107]: audit 2023-10-09T09:19:40.348831+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:40 smithi181 bash[59345]: cluster 2023-10-09T09:19:38.602930+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e904 with expected crc 2023-10-09T09:19:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:40 smithi181 bash[59345]: cluster 2023-10-09T09:19:39.392267+0000 osd.5 (osd.5) 446 : cluster 3 failed to encode map e905 with expected crc 2023-10-09T09:19:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:40 smithi181 bash[59345]: cluster 2023-10-09T09:19:39.397203+0000 mon.a (mon.0) 1112 : cluster 0 osdmap e905: 8 total, 8 up, 8 in 2023-10-09T09:19:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:40 smithi181 bash[59345]: cluster 2023-10-09T09:19:39.398947+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e905 with expected crc 2023-10-09T09:19:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:40 smithi181 bash[59345]: cluster 2023-10-09T09:19:39.439845+0000 mgr.y (mgr.34107) 393 : cluster 0 pgmap v911: 89 pgs: 89 active+clean; 252 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2023-10-09T09:19:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:40 smithi181 bash[59345]: cluster 2023-10-09T09:19:40.282486+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e906: 8 total, 8 up, 8 in 2023-10-09T09:19:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:40 smithi181 bash[59345]: audit 2023-10-09T09:19:40.348831+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:41.286 INFO:tasks.rados.rados.0.smithi163.stdout:3371: done (1 left) 2023-10-09T09:19:41.286 INFO:tasks.rados.rados.0.smithi163.stdout:3372: read oid 28 snap -1 2023-10-09T09:19:41.286 INFO:tasks.rados.rados.0.smithi163.stdout:3372: expect (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:19:41.287 INFO:tasks.rados.rados.0.smithi163.stdout:3373: write oid 38 current snap is 422 2023-10-09T09:19:41.287 INFO:tasks.rados.rados.0.smithi163.stdout:3369: finishing rollback tid 1 to smithi16367185-31 2023-10-09T09:19:41.287 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 905 (ObjNum 669367496 snap 21875 seq_num 1) dirty dne 2023-10-09T09:19:41.287 INFO:tasks.rados.rados.0.smithi163.stdout:3373: seq_num 1254 ranges {675978=644138,1897949=650107} 2023-10-09T09:19:41.295 INFO:tasks.rados.rados.0.smithi163.stdout:3373: writing smithi16367185-38 from 675978 to 1320116 tid 1 2023-10-09T09:19:41.303 INFO:tasks.rados.rados.0.smithi163.stdout:3373: writing smithi16367185-38 from 1897949 to 2548056 tid 2 2023-10-09T09:19:41.303 INFO:tasks.rados.rados.0.smithi163.stdout:3369: done (2 left) 2023-10-09T09:19:41.304 INFO:tasks.rados.rados.0.smithi163.stdout:3374: rollback oid 7 current snap is 422 2023-10-09T09:19:41.304 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 421 2023-10-09T09:19:41.304 INFO:tasks.rados.rados.0.smithi163.stdout:3372: expect (ObjNum 1005 snap 342 seq_num 1005) 2023-10-09T09:19:41.404 INFO:tasks.rados.rados.0.smithi163.stdout:3373: finishing write tid 1 to smithi16367185-38 2023-10-09T09:19:41.404 INFO:tasks.rados.rados.0.smithi163.stdout:3374: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:19:41.404 INFO:tasks.rados.rados.0.smithi163.stdout:3374: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:19:41.404 INFO:tasks.rados.rados.0.smithi163.stdout:3372: done (2 left) 2023-10-09T09:19:41.404 INFO:tasks.rados.rados.0.smithi163.stdout:3375: rollback oid 30 current snap is 422 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 409 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:3373: finishing write tid 2 to smithi16367185-38 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:3373: finishing write tid 3 to smithi16367185-38 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:3374: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 876 (ObjNum 1178 snap 397 seq_num 1178) dirty exists 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:3373: finishing write tid 4 to smithi16367185-38 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1628 (ObjNum 1254 snap 422 seq_num 1254) dirty exists 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:3373: left oid 38 (ObjNum 1254 snap 422 seq_num 1254) 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:3373: done (2 left) 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:3374: done (1 left) 2023-10-09T09:19:41.405 INFO:tasks.rados.rados.0.smithi163.stdout:3376: snap_remove snap 421 2023-10-09T09:19:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:41 smithi163 bash[78107]: cluster 2023-10-09T09:19:39.848205+0000 osd.7 (osd.7) 367 : cluster 3 failed to encode map e905 with expected crc 2023-10-09T09:19:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:41 smithi163 bash[78107]: cluster 2023-10-09T09:19:40.280268+0000 osd.5 (osd.5) 447 : cluster 3 failed to encode map e906 with expected crc 2023-10-09T09:19:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:41 smithi163 bash[78107]: cluster 2023-10-09T09:19:41.285514+0000 mon.a (mon.0) 1115 : cluster 0 osdmap e907: 8 total, 8 up, 8 in 2023-10-09T09:19:41.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:41 smithi163 bash[78107]: cluster 2023-10-09T09:19:41.286004+0000 osd.4 (osd.4) 394 : cluster 3 failed to encode map e907 with expected crc 2023-10-09T09:19:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:41 smithi163 bash[75425]: cluster 2023-10-09T09:19:39.848205+0000 osd.7 (osd.7) 367 : cluster 3 failed to encode map e905 with expected crc 2023-10-09T09:19:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:41 smithi163 bash[75425]: cluster 2023-10-09T09:19:40.280268+0000 osd.5 (osd.5) 447 : cluster 3 failed to encode map e906 with expected crc 2023-10-09T09:19:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:41 smithi163 bash[75425]: cluster 2023-10-09T09:19:41.285514+0000 mon.a (mon.0) 1115 : cluster 0 osdmap e907: 8 total, 8 up, 8 in 2023-10-09T09:19:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:41 smithi163 bash[75425]: cluster 2023-10-09T09:19:41.286004+0000 osd.4 (osd.4) 394 : cluster 3 failed to encode map e907 with expected crc 2023-10-09T09:19:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:41 smithi181 bash[59345]: cluster 2023-10-09T09:19:39.848205+0000 osd.7 (osd.7) 367 : cluster 3 failed to encode map e905 with expected crc 2023-10-09T09:19:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:41 smithi181 bash[59345]: cluster 2023-10-09T09:19:40.280268+0000 osd.5 (osd.5) 447 : cluster 3 failed to encode map e906 with expected crc 2023-10-09T09:19:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:41 smithi181 bash[59345]: cluster 2023-10-09T09:19:41.285514+0000 mon.a (mon.0) 1115 : cluster 0 osdmap e907: 8 total, 8 up, 8 in 2023-10-09T09:19:41.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:41 smithi181 bash[59345]: cluster 2023-10-09T09:19:41.286004+0000 osd.4 (osd.4) 394 : cluster 3 failed to encode map e907 with expected crc 2023-10-09T09:19:42.293 INFO:tasks.rados.rados.0.smithi163.stdout:3376: done (1 left) 2023-10-09T09:19:42.293 INFO:tasks.rados.rados.0.smithi163.stdout:3377: snap_remove snap 410 2023-10-09T09:19:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[75425]: cluster 2023-10-09T09:19:40.594647+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e906 with expected crc 2023-10-09T09:19:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[75425]: cluster 2023-10-09T09:19:40.707063+0000 osd.7 (osd.7) 368 : cluster 3 failed to encode map e906 with expected crc 2023-10-09T09:19:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[75425]: cluster 2023-10-09T09:19:41.281039+0000 osd.5 (osd.5) 448 : cluster 3 failed to encode map e907 with expected crc 2023-10-09T09:19:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[75425]: cluster 2023-10-09T09:19:41.289804+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e907 with expected crc 2023-10-09T09:19:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[75425]: cluster 2023-10-09T09:19:41.440792+0000 mgr.y (mgr.34107) 394 : cluster 0 pgmap v914: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 248 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 338 KiB/s wr, 12 op/s 2023-10-09T09:19:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[75425]: cluster 2023-10-09T09:19:42.289392+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e908: 8 total, 8 up, 8 in 2023-10-09T09:19:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[78107]: cluster 2023-10-09T09:19:40.594647+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e906 with expected crc 2023-10-09T09:19:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[78107]: cluster 2023-10-09T09:19:40.707063+0000 osd.7 (osd.7) 368 : cluster 3 failed to encode map e906 with expected crc 2023-10-09T09:19:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[78107]: cluster 2023-10-09T09:19:41.281039+0000 osd.5 (osd.5) 448 : cluster 3 failed to encode map e907 with expected crc 2023-10-09T09:19:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[78107]: cluster 2023-10-09T09:19:41.289804+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e907 with expected crc 2023-10-09T09:19:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[78107]: cluster 2023-10-09T09:19:41.440792+0000 mgr.y (mgr.34107) 394 : cluster 0 pgmap v914: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 248 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 338 KiB/s wr, 12 op/s 2023-10-09T09:19:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:42 smithi163 bash[78107]: cluster 2023-10-09T09:19:42.289392+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e908: 8 total, 8 up, 8 in 2023-10-09T09:19:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:42 smithi181 bash[59345]: cluster 2023-10-09T09:19:40.594647+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e906 with expected crc 2023-10-09T09:19:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:42 smithi181 bash[59345]: cluster 2023-10-09T09:19:40.707063+0000 osd.7 (osd.7) 368 : cluster 3 failed to encode map e906 with expected crc 2023-10-09T09:19:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:42 smithi181 bash[59345]: cluster 2023-10-09T09:19:41.281039+0000 osd.5 (osd.5) 448 : cluster 3 failed to encode map e907 with expected crc 2023-10-09T09:19:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:42 smithi181 bash[59345]: cluster 2023-10-09T09:19:41.289804+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e907 with expected crc 2023-10-09T09:19:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:42 smithi181 bash[59345]: cluster 2023-10-09T09:19:41.440792+0000 mgr.y (mgr.34107) 394 : cluster 0 pgmap v914: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 248 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 338 KiB/s wr, 12 op/s 2023-10-09T09:19:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:42 smithi181 bash[59345]: cluster 2023-10-09T09:19:42.289392+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e908: 8 total, 8 up, 8 in 2023-10-09T09:19:43.290 INFO:tasks.rados.rados.0.smithi163.stdout:3377: done (1 left) 2023-10-09T09:19:43.290 INFO:tasks.rados.rados.0.smithi163.stdout:3378: delete oid 4 current snap is 422 2023-10-09T09:19:43.290 INFO:tasks.rados.rados.0.smithi163.stdout:3375: finishing rollback tid 0 to smithi16367185-30 2023-10-09T09:19:43.290 INFO:tasks.rados.rados.0.smithi163.stdout:3375: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:19:43.290 INFO:tasks.rados.rados.0.smithi163.stdout:3375: finishing rollback tid 2 to smithi16367185-30 2023-10-09T09:19:43.290 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1168 (ObjNum 1209 snap 404 seq_num 1209) dirty exists 2023-10-09T09:19:43.300 INFO:tasks.rados.rados.0.smithi163.stdout:3375: done (1 left) 2023-10-09T09:19:43.300 INFO:tasks.rados.rados.0.smithi163.stdout:3378: done (0 left) 2023-10-09T09:19:43.300 INFO:tasks.rados.rados.0.smithi163.stdout:3379: delete oid 27 current snap is 422 2023-10-09T09:19:43.304 INFO:tasks.rados.rados.0.smithi163.stdout:3379: done (0 left) 2023-10-09T09:19:43.304 INFO:tasks.rados.rados.0.smithi163.stdout:3380: snap_create 2023-10-09T09:19:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:19:43] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.33.4" 2023-10-09T09:19:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[78107]: cluster 2023-10-09T09:19:42.285511+0000 osd.5 (osd.5) 449 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[78107]: cluster 2023-10-09T09:19:42.286285+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[78107]: cluster 2023-10-09T09:19:42.388212+0000 osd.4 (osd.4) 395 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[78107]: cluster 2023-10-09T09:19:43.286601+0000 osd.4 (osd.4) 396 : cluster 3 failed to encode map e909 with expected crc 2023-10-09T09:19:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[78107]: cluster 2023-10-09T09:19:43.289528+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e909: 8 total, 8 up, 8 in 2023-10-09T09:19:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[75425]: cluster 2023-10-09T09:19:42.285511+0000 osd.5 (osd.5) 449 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[75425]: cluster 2023-10-09T09:19:42.286285+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[75425]: cluster 2023-10-09T09:19:42.388212+0000 osd.4 (osd.4) 395 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[75425]: cluster 2023-10-09T09:19:43.286601+0000 osd.4 (osd.4) 396 : cluster 3 failed to encode map e909 with expected crc 2023-10-09T09:19:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:43 smithi163 bash[75425]: cluster 2023-10-09T09:19:43.289528+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e909: 8 total, 8 up, 8 in 2023-10-09T09:19:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:43 smithi181 bash[59345]: cluster 2023-10-09T09:19:42.285511+0000 osd.5 (osd.5) 449 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:43 smithi181 bash[59345]: cluster 2023-10-09T09:19:42.286285+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:43 smithi181 bash[59345]: cluster 2023-10-09T09:19:42.388212+0000 osd.4 (osd.4) 395 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:43 smithi181 bash[59345]: cluster 2023-10-09T09:19:43.286601+0000 osd.4 (osd.4) 396 : cluster 3 failed to encode map e909 with expected crc 2023-10-09T09:19:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:43 smithi181 bash[59345]: cluster 2023-10-09T09:19:43.289528+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e909: 8 total, 8 up, 8 in 2023-10-09T09:19:44.295 INFO:tasks.rados.rados.0.smithi163.stdout:3380: done (0 left) 2023-10-09T09:19:44.295 INFO:tasks.rados.rados.0.smithi163.stdout:3381: write oid 4 current snap is 423 2023-10-09T09:19:44.295 INFO:tasks.rados.rados.0.smithi163.stdout:3381: seq_num 1257 ranges {726399=586238,2073395=797434} 2023-10-09T09:19:44.302 INFO:tasks.rados.rados.0.smithi163.stdout:3381: writing smithi16367185-4 from 726399 to 1312637 tid 1 2023-10-09T09:19:44.312 INFO:tasks.rados.rados.0.smithi163.stdout:3381: writing smithi16367185-4 from 2073395 to 2870829 tid 2 2023-10-09T09:19:44.312 INFO:tasks.rados.rados.0.smithi163.stdout:3382: read oid 29 snap -1 2023-10-09T09:19:44.312 INFO:tasks.rados.rados.0.smithi163.stdout:3382: expect (ObjNum 1232 snap 410 seq_num 1232) 2023-10-09T09:19:44.312 INFO:tasks.rados.rados.0.smithi163.stdout:3383: write oid 19 current snap is 423 2023-10-09T09:19:44.312 INFO:tasks.rados.rados.0.smithi163.stdout:3381: finishing write tid 1 to smithi16367185-4 2023-10-09T09:19:44.312 INFO:tasks.rados.rados.0.smithi163.stdout:3383: seq_num 1258 ranges {743206=661487,2131877=541092} 2023-10-09T09:19:44.320 INFO:tasks.rados.rados.0.smithi163.stdout:3383: writing smithi16367185-19 from 743206 to 1404693 tid 1 2023-10-09T09:19:44.327 INFO:tasks.rados.rados.0.smithi163.stdout:3383: writing smithi16367185-19 from 2131877 to 2672969 tid 2 2023-10-09T09:19:44.327 INFO:tasks.rados.rados.0.smithi163.stdout:3384: read oid 49 snap -1 2023-10-09T09:19:44.327 INFO:tasks.rados.rados.0.smithi163.stdout:3384: expect (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:19:44.328 INFO:tasks.rados.rados.0.smithi163.stdout:3385: rollback oid 30 current snap is 423 2023-10-09T09:19:44.328 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 418 2023-10-09T09:19:44.328 INFO:tasks.rados.rados.0.smithi163.stdout:3381: finishing write tid 2 to smithi16367185-4 2023-10-09T09:19:44.328 INFO:tasks.rados.rados.0.smithi163.stdout:3382: expect (ObjNum 1232 snap 410 seq_num 1232) 2023-10-09T09:19:44.425 INFO:tasks.rados.rados.0.smithi163.stdout:3381: finishing write tid 3 to smithi16367185-4 2023-10-09T09:19:44.425 INFO:tasks.rados.rados.0.smithi163.stdout:3381: finishing write tid 4 to smithi16367185-4 2023-10-09T09:19:44.425 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1717 (ObjNum 1257 snap 423 seq_num 1257) dirty exists 2023-10-09T09:19:44.425 INFO:tasks.rados.rados.0.smithi163.stdout:3381: left oid 4 (ObjNum 1257 snap 423 seq_num 1257) 2023-10-09T09:19:44.425 INFO:tasks.rados.rados.0.smithi163.stdout:3383: finishing write tid 1 to smithi16367185-19 2023-10-09T09:19:44.425 INFO:tasks.rados.rados.0.smithi163.stdout:3381: done (4 left) 2023-10-09T09:19:44.426 INFO:tasks.rados.rados.0.smithi163.stdout:3382: done (3 left) 2023-10-09T09:19:44.426 INFO:tasks.rados.rados.0.smithi163.stdout:3386: snap_create 2023-10-09T09:19:44.426 INFO:tasks.rados.rados.0.smithi163.stdout:3383: finishing write tid 2 to smithi16367185-19 2023-10-09T09:19:44.426 INFO:tasks.rados.rados.0.smithi163.stdout:3383: finishing write tid 3 to smithi16367185-19 2023-10-09T09:19:44.426 INFO:tasks.rados.rados.0.smithi163.stdout:3383: finishing write tid 4 to smithi16367185-19 2023-10-09T09:19:44.426 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 1040 (ObjNum 1258 snap 423 seq_num 1258) dirty exists 2023-10-09T09:19:44.426 INFO:tasks.rados.rados.0.smithi163.stdout:3383: left oid 19 (ObjNum 1258 snap 423 seq_num 1258) 2023-10-09T09:19:44.426 INFO:tasks.rados.rados.0.smithi163.stdout:3384: expect (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:19:44.572 INFO:tasks.rados.rados.0.smithi163.stdout:3385: finishing rollback tid 0 to smithi16367185-30 2023-10-09T09:19:44.572 INFO:tasks.rados.rados.0.smithi163.stdout:3385: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:19:44.572 INFO:tasks.rados.rados.0.smithi163.stdout:3385: finishing rollback tid 2 to smithi16367185-30 2023-10-09T09:19:44.572 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1176 (ObjNum 1230 snap 410 seq_num 1230) dirty exists 2023-10-09T09:19:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[78107]: cluster 2023-10-09T09:19:42.743910+0000 osd.7 (osd.7) 369 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[78107]: cluster 2023-10-09T09:19:43.285883+0000 osd.5 (osd.5) 450 : cluster 3 failed to encode map e909 with expected crc 2023-10-09T09:19:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[78107]: cluster 2023-10-09T09:19:43.303067+0000 osd.7 (osd.7) 370 : cluster 3 failed to encode map e909 with expected crc 2023-10-09T09:19:44.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[78107]: cluster 2023-10-09T09:19:43.441489+0000 mgr.y (mgr.34107) 395 : cluster 0 pgmap v917: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 245 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 338 KiB/s wr, 11 op/s 2023-10-09T09:19:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[78107]: cluster 2023-10-09T09:19:44.294715+0000 mon.a (mon.0) 1118 : cluster 0 osdmap e910: 8 total, 8 up, 8 in 2023-10-09T09:19:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[78107]: cluster 2023-10-09T09:19:44.307468+0000 osd.4 (osd.4) 397 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[75425]: cluster 2023-10-09T09:19:42.743910+0000 osd.7 (osd.7) 369 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[75425]: cluster 2023-10-09T09:19:43.285883+0000 osd.5 (osd.5) 450 : cluster 3 failed to encode map e909 with expected crc 2023-10-09T09:19:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[75425]: cluster 2023-10-09T09:19:43.303067+0000 osd.7 (osd.7) 370 : cluster 3 failed to encode map e909 with expected crc 2023-10-09T09:19:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[75425]: cluster 2023-10-09T09:19:43.441489+0000 mgr.y (mgr.34107) 395 : cluster 0 pgmap v917: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 245 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 338 KiB/s wr, 11 op/s 2023-10-09T09:19:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[75425]: cluster 2023-10-09T09:19:44.294715+0000 mon.a (mon.0) 1118 : cluster 0 osdmap e910: 8 total, 8 up, 8 in 2023-10-09T09:19:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:44 smithi163 bash[75425]: cluster 2023-10-09T09:19:44.307468+0000 osd.4 (osd.4) 397 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:44 smithi181 bash[59345]: cluster 2023-10-09T09:19:42.743910+0000 osd.7 (osd.7) 369 : cluster 3 failed to encode map e908 with expected crc 2023-10-09T09:19:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:44 smithi181 bash[59345]: cluster 2023-10-09T09:19:43.285883+0000 osd.5 (osd.5) 450 : cluster 3 failed to encode map e909 with expected crc 2023-10-09T09:19:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:44 smithi181 bash[59345]: cluster 2023-10-09T09:19:43.303067+0000 osd.7 (osd.7) 370 : cluster 3 failed to encode map e909 with expected crc 2023-10-09T09:19:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:44 smithi181 bash[59345]: cluster 2023-10-09T09:19:43.441489+0000 mgr.y (mgr.34107) 395 : cluster 0 pgmap v917: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 245 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 338 KiB/s wr, 11 op/s 2023-10-09T09:19:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:44 smithi181 bash[59345]: cluster 2023-10-09T09:19:44.294715+0000 mon.a (mon.0) 1118 : cluster 0 osdmap e910: 8 total, 8 up, 8 in 2023-10-09T09:19:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:44 smithi181 bash[59345]: cluster 2023-10-09T09:19:44.307468+0000 osd.4 (osd.4) 397 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:45.288 INFO:tasks.rados.rados.0.smithi163.stdout:3383: done (3 left) 2023-10-09T09:19:45.288 INFO:tasks.rados.rados.0.smithi163.stdout:3384: done (2 left) 2023-10-09T09:19:45.288 INFO:tasks.rados.rados.0.smithi163.stdout:3385: done (1 left) 2023-10-09T09:19:45.288 INFO:tasks.rados.rados.0.smithi163.stdout:3386: done (0 left) 2023-10-09T09:19:45.288 INFO:tasks.rados.rados.0.smithi163.stdout:3387: read oid 3 snap 420 2023-10-09T09:19:45.288 INFO:tasks.rados.rados.0.smithi163.stdout:3387: expect (ObjNum 1253 snap 420 seq_num 1253) 2023-10-09T09:19:45.288 INFO:tasks.rados.rados.0.smithi163.stdout:3388: snap_create 2023-10-09T09:19:45.300 INFO:tasks.rados.rados.0.smithi163.stdout:3387: expect (ObjNum 1253 snap 420 seq_num 1253) 2023-10-09T09:19:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:45 smithi163 bash[75425]: cluster 2023-10-09T09:19:44.288339+0000 osd.5 (osd.5) 451 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:45 smithi163 bash[75425]: cluster 2023-10-09T09:19:44.288858+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:45 smithi163 bash[75425]: cluster 2023-10-09T09:19:44.321653+0000 osd.7 (osd.7) 371 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:45 smithi163 bash[75425]: cluster 2023-10-09T09:19:45.287461+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e911: 8 total, 8 up, 8 in 2023-10-09T09:19:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:45 smithi163 bash[78107]: cluster 2023-10-09T09:19:44.288339+0000 osd.5 (osd.5) 451 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:45 smithi163 bash[78107]: cluster 2023-10-09T09:19:44.288858+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:45 smithi163 bash[78107]: cluster 2023-10-09T09:19:44.321653+0000 osd.7 (osd.7) 371 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:45 smithi163 bash[78107]: cluster 2023-10-09T09:19:45.287461+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e911: 8 total, 8 up, 8 in 2023-10-09T09:19:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:45 smithi181 bash[59345]: cluster 2023-10-09T09:19:44.288339+0000 osd.5 (osd.5) 451 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:45 smithi181 bash[59345]: cluster 2023-10-09T09:19:44.288858+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:45 smithi181 bash[59345]: cluster 2023-10-09T09:19:44.321653+0000 osd.7 (osd.7) 371 : cluster 3 failed to encode map e910 with expected crc 2023-10-09T09:19:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:45 smithi181 bash[59345]: cluster 2023-10-09T09:19:45.287461+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e911: 8 total, 8 up, 8 in 2023-10-09T09:19:46.288 INFO:tasks.rados.rados.0.smithi163.stdout:3387: done (1 left) 2023-10-09T09:19:46.288 INFO:tasks.rados.rados.0.smithi163.stdout:3388: done (0 left) 2023-10-09T09:19:46.288 INFO:tasks.rados.rados.0.smithi163.stdout:3389: read oid 2 snap -1 2023-10-09T09:19:46.288 INFO:tasks.rados.rados.0.smithi163.stdout:3389: expect deleted 2023-10-09T09:19:46.289 INFO:tasks.rados.rados.0.smithi163.stdout:3390: delete oid 48 current snap is 425 2023-10-09T09:19:46.295 INFO:tasks.rados.rados.0.smithi163.stdout:3390: done (1 left) 2023-10-09T09:19:46.295 INFO:tasks.rados.rados.0.smithi163.stdout:3391: delete oid 12 current snap is 425 2023-10-09T09:19:46.301 INFO:tasks.rados.rados.0.smithi163.stdout:3389: done (1 left) 2023-10-09T09:19:46.302 INFO:tasks.rados.rados.0.smithi163.stdout:3391: done (0 left) 2023-10-09T09:19:46.302 INFO:tasks.rados.rados.0.smithi163.stdout:3392: rollback oid 39 current snap is 425 2023-10-09T09:19:46.302 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 419 2023-10-09T09:19:46.302 INFO:tasks.rados.rados.0.smithi163.stdout:3393: write oid 5 current snap is 425 2023-10-09T09:19:46.302 INFO:tasks.rados.rados.0.smithi163.stdout:3393: seq_num 1261 ranges {793627=487234,1790970=404772,2829335=364292} 2023-10-09T09:19:46.309 INFO:tasks.rados.rados.0.smithi163.stdout:3393: writing smithi16367185-5 from 793627 to 1280861 tid 1 2023-10-09T09:19:46.315 INFO:tasks.rados.rados.0.smithi163.stdout:3393: writing smithi16367185-5 from 1790970 to 2195742 tid 2 2023-10-09T09:19:46.321 INFO:tasks.rados.rados.0.smithi163.stdout:3393: writing smithi16367185-5 from 2829335 to 3193627 tid 3 2023-10-09T09:19:46.321 INFO:tasks.rados.rados.0.smithi163.stdout:3394: snap_create 2023-10-09T09:19:46.321 INFO:tasks.rados.rados.0.smithi163.stdout:3392: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:19:46.321 INFO:tasks.rados.rados.0.smithi163.stdout:3392: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:19:46.321 INFO:tasks.rados.rados.0.smithi163.stdout:3392: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:19:46.321 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1602 (ObjNum 932 snap 317 seq_num 932) dirty exists 2023-10-09T09:19:46.321 INFO:tasks.rados.rados.0.smithi163.stdout:3393: finishing write tid 1 to smithi16367185-5 2023-10-09T09:19:46.321 INFO:tasks.rados.rados.0.smithi163.stdout:3393: finishing write tid 2 to smithi16367185-5 2023-10-09T09:19:46.325 INFO:tasks.rados.rados.0.smithi163.stdout:3393: finishing write tid 3 to smithi16367185-5 2023-10-09T09:19:46.325 INFO:tasks.rados.rados.0.smithi163.stdout:3393: finishing write tid 4 to smithi16367185-5 2023-10-09T09:19:46.325 INFO:tasks.rados.rados.0.smithi163.stdout:3393: finishing write tid 5 to smithi16367185-5 2023-10-09T09:19:46.325 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1722 (ObjNum 1261 snap 425 seq_num 1261) dirty exists 2023-10-09T09:19:46.325 INFO:tasks.rados.rados.0.smithi163.stdout:3393: left oid 5 (ObjNum 1261 snap 425 seq_num 1261) 2023-10-09T09:19:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:46 smithi163 bash[78107]: cluster 2023-10-09T09:19:45.283068+0000 osd.5 (osd.5) 452 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:46 smithi163 bash[78107]: cluster 2023-10-09T09:19:45.283476+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:46 smithi163 bash[78107]: cluster 2023-10-09T09:19:45.442090+0000 mgr.y (mgr.34107) 396 : cluster 0 pgmap v920: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 242 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 631 KiB/s wr, 11 op/s 2023-10-09T09:19:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:46 smithi163 bash[78107]: cluster 2023-10-09T09:19:45.780233+0000 osd.4 (osd.4) 398 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:46 smithi163 bash[78107]: cluster 2023-10-09T09:19:46.287568+0000 mon.a (mon.0) 1120 : cluster 0 osdmap e912: 8 total, 8 up, 8 in 2023-10-09T09:19:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:46 smithi163 bash[75425]: cluster 2023-10-09T09:19:45.283068+0000 osd.5 (osd.5) 452 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:46 smithi163 bash[75425]: cluster 2023-10-09T09:19:45.283476+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:46 smithi163 bash[75425]: cluster 2023-10-09T09:19:45.442090+0000 mgr.y (mgr.34107) 396 : cluster 0 pgmap v920: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 242 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 631 KiB/s wr, 11 op/s 2023-10-09T09:19:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:46 smithi163 bash[75425]: cluster 2023-10-09T09:19:45.780233+0000 osd.4 (osd.4) 398 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:46 smithi163 bash[75425]: cluster 2023-10-09T09:19:46.287568+0000 mon.a (mon.0) 1120 : cluster 0 osdmap e912: 8 total, 8 up, 8 in 2023-10-09T09:19:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:46 smithi181 bash[59345]: cluster 2023-10-09T09:19:45.283068+0000 osd.5 (osd.5) 452 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:46 smithi181 bash[59345]: cluster 2023-10-09T09:19:45.283476+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:46 smithi181 bash[59345]: cluster 2023-10-09T09:19:45.442090+0000 mgr.y (mgr.34107) 396 : cluster 0 pgmap v920: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 242 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 631 KiB/s wr, 11 op/s 2023-10-09T09:19:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:46 smithi181 bash[59345]: cluster 2023-10-09T09:19:45.780233+0000 osd.4 (osd.4) 398 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:46 smithi181 bash[59345]: cluster 2023-10-09T09:19:46.287568+0000 mon.a (mon.0) 1120 : cluster 0 osdmap e912: 8 total, 8 up, 8 in 2023-10-09T09:19:47.291 INFO:tasks.rados.rados.0.smithi163.stdout:3392: done (2 left) 2023-10-09T09:19:47.291 INFO:tasks.rados.rados.0.smithi163.stdout:3393: done (1 left) 2023-10-09T09:19:47.291 INFO:tasks.rados.rados.0.smithi163.stdout:3394: done (0 left) 2023-10-09T09:19:47.292 INFO:tasks.rados.rados.0.smithi163.stdout:3395: read oid 46 snap -1 2023-10-09T09:19:47.292 INFO:tasks.rados.rados.0.smithi163.stdout:3395: expect (ObjNum 1145 snap 384 seq_num 1145) 2023-10-09T09:19:47.292 INFO:tasks.rados.rados.0.smithi163.stdout:3396: snap_create 2023-10-09T09:19:47.305 INFO:tasks.rados.rados.0.smithi163.stdout:3395: expect (ObjNum 1145 snap 384 seq_num 1145) 2023-10-09T09:19:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:47 smithi163 bash[78107]: cluster 2023-10-09T09:19:46.283508+0000 osd.5 (osd.5) 453 : cluster 3 failed to encode map e912 with expected crc 2023-10-09T09:19:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:47 smithi163 bash[78107]: cluster 2023-10-09T09:19:46.291938+0000 osd.7 (osd.7) 372 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:47 smithi163 bash[78107]: cluster 2023-10-09T09:19:47.286708+0000 osd.4 (osd.4) 399 : cluster 3 failed to encode map e913 with expected crc 2023-10-09T09:19:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:47 smithi163 bash[78107]: cluster 2023-10-09T09:19:47.291152+0000 mon.a (mon.0) 1121 : cluster 0 osdmap e913: 8 total, 8 up, 8 in 2023-10-09T09:19:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:47 smithi163 bash[78107]: cluster 2023-10-09T09:19:47.292356+0000 osd.4 (osd.4) 400 : cluster 3 failed to encode map e913 with expected crc 2023-10-09T09:19:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:47 smithi163 bash[75425]: cluster 2023-10-09T09:19:46.283508+0000 osd.5 (osd.5) 453 : cluster 3 failed to encode map e912 with expected crc 2023-10-09T09:19:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:47 smithi163 bash[75425]: cluster 2023-10-09T09:19:46.291938+0000 osd.7 (osd.7) 372 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:47 smithi163 bash[75425]: cluster 2023-10-09T09:19:47.286708+0000 osd.4 (osd.4) 399 : cluster 3 failed to encode map e913 with expected crc 2023-10-09T09:19:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:47 smithi163 bash[75425]: cluster 2023-10-09T09:19:47.291152+0000 mon.a (mon.0) 1121 : cluster 0 osdmap e913: 8 total, 8 up, 8 in 2023-10-09T09:19:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:47 smithi163 bash[75425]: cluster 2023-10-09T09:19:47.292356+0000 osd.4 (osd.4) 400 : cluster 3 failed to encode map e913 with expected crc 2023-10-09T09:19:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:47 smithi181 bash[59345]: cluster 2023-10-09T09:19:46.283508+0000 osd.5 (osd.5) 453 : cluster 3 failed to encode map e912 with expected crc 2023-10-09T09:19:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:47 smithi181 bash[59345]: cluster 2023-10-09T09:19:46.291938+0000 osd.7 (osd.7) 372 : cluster 3 failed to encode map e911 with expected crc 2023-10-09T09:19:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:47 smithi181 bash[59345]: cluster 2023-10-09T09:19:47.286708+0000 osd.4 (osd.4) 399 : cluster 3 failed to encode map e913 with expected crc 2023-10-09T09:19:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:47 smithi181 bash[59345]: cluster 2023-10-09T09:19:47.291152+0000 mon.a (mon.0) 1121 : cluster 0 osdmap e913: 8 total, 8 up, 8 in 2023-10-09T09:19:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:47 smithi181 bash[59345]: cluster 2023-10-09T09:19:47.292356+0000 osd.4 (osd.4) 400 : cluster 3 failed to encode map e913 with expected crc 2023-10-09T09:19:48.293 INFO:tasks.rados.rados.0.smithi163.stdout:3395: done (1 left) 2023-10-09T09:19:48.293 INFO:tasks.rados.rados.0.smithi163.stdout:3396: done (0 left) 2023-10-09T09:19:48.293 INFO:tasks.rados.rados.0.smithi163.stdout:3397: rollback oid 32 current snap is 427 2023-10-09T09:19:48.293 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 415 2023-10-09T09:19:48.294 INFO:tasks.rados.rados.0.smithi163.stdout:3398: delete oid 6 current snap is 427 2023-10-09T09:19:48.303 INFO:tasks.rados.rados.0.smithi163.stdout:3398: done (1 left) 2023-10-09T09:19:48.303 INFO:tasks.rados.rados.0.smithi163.stdout:3399: snap_create 2023-10-09T09:19:48.303 INFO:tasks.rados.rados.0.smithi163.stdout:3397: finishing rollback tid 0 to smithi16367185-32 2023-10-09T09:19:48.306 INFO:tasks.rados.rados.0.smithi163.stdout:3397: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:19:48.306 INFO:tasks.rados.rados.0.smithi163.stdout:3397: finishing rollback tid 2 to smithi16367185-32 2023-10-09T09:19:48.306 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 2509 (ObjNum 1218 snap 404 seq_num 1218) dirty exists 2023-10-09T09:19:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:48 smithi163 bash[78107]: cluster 2023-10-09T09:19:47.286120+0000 osd.5 (osd.5) 454 : cluster 3 failed to encode map e913 with expected crc 2023-10-09T09:19:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:48 smithi163 bash[78107]: cluster 2023-10-09T09:19:47.442530+0000 mgr.y (mgr.34107) 397 : cluster 0 pgmap v923: 89 pgs: 89 active+clean; 235 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 948 KiB/s wr, 14 op/s 2023-10-09T09:19:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:48 smithi163 bash[78107]: cluster 2023-10-09T09:19:48.293115+0000 mon.a (mon.0) 1122 : cluster 0 osdmap e914: 8 total, 8 up, 8 in 2023-10-09T09:19:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:48 smithi163 bash[78107]: cluster 2023-10-09T09:19:48.294616+0000 osd.4 (osd.4) 401 : cluster 3 failed to encode map e914 with expected crc 2023-10-09T09:19:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:48 smithi163 bash[75425]: cluster 2023-10-09T09:19:47.286120+0000 osd.5 (osd.5) 454 : cluster 3 failed to encode map e913 with expected crc 2023-10-09T09:19:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:48 smithi163 bash[75425]: cluster 2023-10-09T09:19:47.442530+0000 mgr.y (mgr.34107) 397 : cluster 0 pgmap v923: 89 pgs: 89 active+clean; 235 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 948 KiB/s wr, 14 op/s 2023-10-09T09:19:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:48 smithi163 bash[75425]: cluster 2023-10-09T09:19:48.293115+0000 mon.a (mon.0) 1122 : cluster 0 osdmap e914: 8 total, 8 up, 8 in 2023-10-09T09:19:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:48 smithi163 bash[75425]: cluster 2023-10-09T09:19:48.294616+0000 osd.4 (osd.4) 401 : cluster 3 failed to encode map e914 with expected crc 2023-10-09T09:19:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:48 smithi181 bash[59345]: cluster 2023-10-09T09:19:47.286120+0000 osd.5 (osd.5) 454 : cluster 3 failed to encode map e913 with expected crc 2023-10-09T09:19:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:48 smithi181 bash[59345]: cluster 2023-10-09T09:19:47.442530+0000 mgr.y (mgr.34107) 397 : cluster 0 pgmap v923: 89 pgs: 89 active+clean; 235 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 948 KiB/s wr, 14 op/s 2023-10-09T09:19:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:48 smithi181 bash[59345]: cluster 2023-10-09T09:19:48.293115+0000 mon.a (mon.0) 1122 : cluster 0 osdmap e914: 8 total, 8 up, 8 in 2023-10-09T09:19:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:48 smithi181 bash[59345]: cluster 2023-10-09T09:19:48.294616+0000 osd.4 (osd.4) 401 : cluster 3 failed to encode map e914 with expected crc 2023-10-09T09:19:49.296 INFO:tasks.rados.rados.0.smithi163.stdout:3397: done (1 left) 2023-10-09T09:19:49.297 INFO:tasks.rados.rados.0.smithi163.stdout:3399: done (0 left) 2023-10-09T09:19:49.297 INFO:tasks.rados.rados.0.smithi163.stdout:3400: read oid 2 snap -1 2023-10-09T09:19:49.297 INFO:tasks.rados.rados.0.smithi163.stdout:3400: expect deleted 2023-10-09T09:19:49.297 INFO:tasks.rados.rados.0.smithi163.stdout:3401: read oid 23 snap 418 2023-10-09T09:19:49.297 INFO:tasks.rados.rados.0.smithi163.stdout:3401: expect deleted 2023-10-09T09:19:49.298 INFO:tasks.rados.rados.0.smithi163.stdout:3402: delete oid 47 current snap is 428 2023-10-09T09:19:49.303 INFO:tasks.rados.rados.0.smithi163.stdout:3400: done (2 left) 2023-10-09T09:19:49.304 INFO:tasks.rados.rados.0.smithi163.stdout:3401: done (1 left) 2023-10-09T09:19:49.304 INFO:tasks.rados.rados.0.smithi163.stdout:3402: done (0 left) 2023-10-09T09:19:49.304 INFO:tasks.rados.rados.0.smithi163.stdout:3403: read oid 32 snap 422 2023-10-09T09:19:49.304 INFO:tasks.rados.rados.0.smithi163.stdout:3403: expect (ObjNum 1218 snap 404 seq_num 1218) 2023-10-09T09:19:49.304 INFO:tasks.rados.rados.0.smithi163.stdout:3404: snap_remove snap 406 2023-10-09T09:19:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:49 smithi163 bash[78107]: cluster 2023-10-09T09:19:48.288616+0000 osd.5 (osd.5) 455 : cluster 3 failed to encode map e914 with expected crc 2023-10-09T09:19:49.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:49 smithi163 bash[78107]: cluster 2023-10-09T09:19:48.289320+0000 osd.7 (osd.7) 373 : cluster 3 failed to encode map e914 with expected crc 2023-10-09T09:19:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:49 smithi163 bash[78107]: cluster 2023-10-09T09:19:49.291516+0000 osd.4 (osd.4) 402 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:49 smithi163 bash[78107]: cluster 2023-10-09T09:19:49.295757+0000 mon.a (mon.0) 1123 : cluster 0 osdmap e915: 8 total, 8 up, 8 in 2023-10-09T09:19:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:49 smithi163 bash[75425]: cluster 2023-10-09T09:19:48.288616+0000 osd.5 (osd.5) 455 : cluster 3 failed to encode map e914 with expected crc 2023-10-09T09:19:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:49 smithi163 bash[75425]: cluster 2023-10-09T09:19:48.289320+0000 osd.7 (osd.7) 373 : cluster 3 failed to encode map e914 with expected crc 2023-10-09T09:19:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:49 smithi163 bash[75425]: cluster 2023-10-09T09:19:49.291516+0000 osd.4 (osd.4) 402 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:49 smithi163 bash[75425]: cluster 2023-10-09T09:19:49.295757+0000 mon.a (mon.0) 1123 : cluster 0 osdmap e915: 8 total, 8 up, 8 in 2023-10-09T09:19:49.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:49 smithi181 bash[59345]: cluster 2023-10-09T09:19:48.288616+0000 osd.5 (osd.5) 455 : cluster 3 failed to encode map e914 with expected crc 2023-10-09T09:19:49.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:49 smithi181 bash[59345]: cluster 2023-10-09T09:19:48.289320+0000 osd.7 (osd.7) 373 : cluster 3 failed to encode map e914 with expected crc 2023-10-09T09:19:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:49 smithi181 bash[59345]: cluster 2023-10-09T09:19:49.291516+0000 osd.4 (osd.4) 402 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:49.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:49 smithi181 bash[59345]: cluster 2023-10-09T09:19:49.295757+0000 mon.a (mon.0) 1123 : cluster 0 osdmap e915: 8 total, 8 up, 8 in 2023-10-09T09:19:50.290 INFO:tasks.rados.rados.0.smithi163.stdout:3404: done (1 left) 2023-10-09T09:19:50.290 INFO:tasks.rados.rados.0.smithi163.stdout:3405: snap_remove snap 423 2023-10-09T09:19:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:50 smithi163 bash[78107]: cluster 2023-10-09T09:19:49.291002+0000 osd.5 (osd.5) 456 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:50 smithi163 bash[78107]: cluster 2023-10-09T09:19:49.297480+0000 osd.7 (osd.7) 374 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:50 smithi163 bash[78107]: cluster 2023-10-09T09:19:49.443223+0000 mgr.y (mgr.34107) 398 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 235 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 316 KiB/s wr, 5 op/s 2023-10-09T09:19:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:50 smithi163 bash[78107]: cluster 2023-10-09T09:19:49.447833+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:50 smithi163 bash[78107]: cluster 2023-10-09T09:19:50.289834+0000 mon.a (mon.0) 1124 : cluster 0 osdmap e916: 8 total, 8 up, 8 in 2023-10-09T09:19:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:50 smithi163 bash[75425]: cluster 2023-10-09T09:19:49.291002+0000 osd.5 (osd.5) 456 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:50 smithi163 bash[75425]: cluster 2023-10-09T09:19:49.297480+0000 osd.7 (osd.7) 374 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:50 smithi163 bash[75425]: cluster 2023-10-09T09:19:49.443223+0000 mgr.y (mgr.34107) 398 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 235 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 316 KiB/s wr, 5 op/s 2023-10-09T09:19:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:50 smithi163 bash[75425]: cluster 2023-10-09T09:19:49.447833+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:50 smithi163 bash[75425]: cluster 2023-10-09T09:19:50.289834+0000 mon.a (mon.0) 1124 : cluster 0 osdmap e916: 8 total, 8 up, 8 in 2023-10-09T09:19:50.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:50 smithi181 bash[59345]: cluster 2023-10-09T09:19:49.291002+0000 osd.5 (osd.5) 456 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:50.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:50 smithi181 bash[59345]: cluster 2023-10-09T09:19:49.297480+0000 osd.7 (osd.7) 374 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:50.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:50 smithi181 bash[59345]: cluster 2023-10-09T09:19:49.443223+0000 mgr.y (mgr.34107) 398 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 235 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 316 KiB/s wr, 5 op/s 2023-10-09T09:19:50.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:50 smithi181 bash[59345]: cluster 2023-10-09T09:19:49.447833+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e915 with expected crc 2023-10-09T09:19:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:50 smithi181 bash[59345]: cluster 2023-10-09T09:19:50.289834+0000 mon.a (mon.0) 1124 : cluster 0 osdmap e916: 8 total, 8 up, 8 in 2023-10-09T09:19:51.292 INFO:tasks.rados.rados.0.smithi163.stdout:3405: done (1 left) 2023-10-09T09:19:51.292 INFO:tasks.rados.rados.0.smithi163.stdout:3406: read oid 49 snap -1 2023-10-09T09:19:51.292 INFO:tasks.rados.rados.0.smithi163.stdout:3406: expect (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:19:51.292 INFO:tasks.rados.rados.0.smithi163.stdout:3407: snap_create 2023-10-09T09:19:51.292 INFO:tasks.rados.rados.0.smithi163.stdout:3403: expect (ObjNum 1218 snap 404 seq_num 1218) 2023-10-09T09:19:51.432 INFO:tasks.rados.rados.0.smithi163.stdout:3406: expect (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:19:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:51 smithi163 bash[75425]: cluster 2023-10-09T09:19:50.285509+0000 osd.5 (osd.5) 457 : cluster 3 failed to encode map e916 with expected crc 2023-10-09T09:19:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:51 smithi163 bash[75425]: cluster 2023-10-09T09:19:50.286245+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e916 with expected crc 2023-10-09T09:19:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:51 smithi163 bash[75425]: cluster 2023-10-09T09:19:51.291401+0000 mon.a (mon.0) 1125 : cluster 0 osdmap e917: 8 total, 8 up, 8 in 2023-10-09T09:19:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:51 smithi163 bash[78107]: cluster 2023-10-09T09:19:50.285509+0000 osd.5 (osd.5) 457 : cluster 3 failed to encode map e916 with expected crc 2023-10-09T09:19:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:51 smithi163 bash[78107]: cluster 2023-10-09T09:19:50.286245+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e916 with expected crc 2023-10-09T09:19:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:51 smithi163 bash[78107]: cluster 2023-10-09T09:19:51.291401+0000 mon.a (mon.0) 1125 : cluster 0 osdmap e917: 8 total, 8 up, 8 in 2023-10-09T09:19:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:51 smithi181 bash[59345]: cluster 2023-10-09T09:19:50.285509+0000 osd.5 (osd.5) 457 : cluster 3 failed to encode map e916 with expected crc 2023-10-09T09:19:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:51 smithi181 bash[59345]: cluster 2023-10-09T09:19:50.286245+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e916 with expected crc 2023-10-09T09:19:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:51 smithi181 bash[59345]: cluster 2023-10-09T09:19:51.291401+0000 mon.a (mon.0) 1125 : cluster 0 osdmap e917: 8 total, 8 up, 8 in 2023-10-09T09:19:52.296 INFO:tasks.rados.rados.0.smithi163.stdout:3403: done (2 left) 2023-10-09T09:19:52.297 INFO:tasks.rados.rados.0.smithi163.stdout:3406: done (1 left) 2023-10-09T09:19:52.297 INFO:tasks.rados.rados.0.smithi163.stdout:3407: done (0 left) 2023-10-09T09:19:52.297 INFO:tasks.rados.rados.0.smithi163.stdout:3408: read oid 6 snap -1 2023-10-09T09:19:52.297 INFO:tasks.rados.rados.0.smithi163.stdout:3408: expect deleted 2023-10-09T09:19:52.297 INFO:tasks.rados.rados.0.smithi163.stdout:3409: delete oid 45 current snap is 429 2023-10-09T09:19:52.303 INFO:tasks.rados.rados.0.smithi163.stdout:3408: done (1 left) 2023-10-09T09:19:52.303 INFO:tasks.rados.rados.0.smithi163.stdout:3409: done (0 left) 2023-10-09T09:19:52.303 INFO:tasks.rados.rados.0.smithi163.stdout:3410: read oid 44 snap -1 2023-10-09T09:19:52.304 INFO:tasks.rados.rados.0.smithi163.stdout:3410: expect (ObjNum 1212 snap 404 seq_num 1212) 2023-10-09T09:19:52.304 INFO:tasks.rados.rados.0.smithi163.stdout:3411: snap_create 2023-10-09T09:19:52.311 INFO:tasks.rados.rados.0.smithi163.stdout:3410: expect (ObjNum 1212 snap 404 seq_num 1212) 2023-10-09T09:19:52.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:52 smithi163 bash[75425]: cluster 2023-10-09T09:19:51.286219+0000 osd.5 (osd.5) 458 : cluster 3 failed to encode map e917 with expected crc 2023-10-09T09:19:52.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:52 smithi163 bash[75425]: cluster 2023-10-09T09:19:51.444186+0000 mgr.y (mgr.34107) 399 : cluster 0 pgmap v929: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 236 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 307 KiB/s wr, 7 op/s 2023-10-09T09:19:52.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:52 smithi163 bash[75425]: cluster 2023-10-09T09:19:52.293774+0000 osd.4 (osd.4) 403 : cluster 3 failed to encode map e918 with expected crc 2023-10-09T09:19:52.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:52 smithi163 bash[75425]: cluster 2023-10-09T09:19:52.296454+0000 mon.a (mon.0) 1126 : cluster 0 osdmap e918: 8 total, 8 up, 8 in 2023-10-09T09:19:52.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:52 smithi163 bash[78107]: cluster 2023-10-09T09:19:51.286219+0000 osd.5 (osd.5) 458 : cluster 3 failed to encode map e917 with expected crc 2023-10-09T09:19:52.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:52 smithi163 bash[78107]: cluster 2023-10-09T09:19:51.444186+0000 mgr.y (mgr.34107) 399 : cluster 0 pgmap v929: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 236 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 307 KiB/s wr, 7 op/s 2023-10-09T09:19:52.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:52 smithi163 bash[78107]: cluster 2023-10-09T09:19:52.293774+0000 osd.4 (osd.4) 403 : cluster 3 failed to encode map e918 with expected crc 2023-10-09T09:19:52.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:52 smithi163 bash[78107]: cluster 2023-10-09T09:19:52.296454+0000 mon.a (mon.0) 1126 : cluster 0 osdmap e918: 8 total, 8 up, 8 in 2023-10-09T09:19:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:52 smithi181 bash[59345]: cluster 2023-10-09T09:19:51.286219+0000 osd.5 (osd.5) 458 : cluster 3 failed to encode map e917 with expected crc 2023-10-09T09:19:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:52 smithi181 bash[59345]: cluster 2023-10-09T09:19:51.444186+0000 mgr.y (mgr.34107) 399 : cluster 0 pgmap v929: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 236 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 307 KiB/s wr, 7 op/s 2023-10-09T09:19:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:52 smithi181 bash[59345]: cluster 2023-10-09T09:19:52.293774+0000 osd.4 (osd.4) 403 : cluster 3 failed to encode map e918 with expected crc 2023-10-09T09:19:52.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:52 smithi181 bash[59345]: cluster 2023-10-09T09:19:52.296454+0000 mon.a (mon.0) 1126 : cluster 0 osdmap e918: 8 total, 8 up, 8 in 2023-10-09T09:19:53.301 INFO:tasks.rados.rados.0.smithi163.stdout:3410: done (1 left) 2023-10-09T09:19:53.302 INFO:tasks.rados.rados.0.smithi163.stdout:3411: done (0 left) 2023-10-09T09:19:53.302 INFO:tasks.rados.rados.0.smithi163.stdout:3412: delete oid 8 current snap is 430 2023-10-09T09:19:53.312 INFO:tasks.rados.rados.0.smithi163.stdout:3412: done (0 left) 2023-10-09T09:19:53.312 INFO:tasks.rados.rados.0.smithi163.stdout:3413: delete oid 35 current snap is 430 2023-10-09T09:19:53.314 INFO:tasks.rados.rados.0.smithi163.stdout:3413: done (0 left) 2023-10-09T09:19:53.314 INFO:tasks.rados.rados.0.smithi163.stdout:3414: write oid 45 current snap is 430 2023-10-09T09:19:53.315 INFO:tasks.rados.rados.0.smithi163.stdout:3414: seq_num 1267 ranges {494469=655081,1741862=415779,2710226=584243} 2023-10-09T09:19:53.324 INFO:tasks.rados.rados.0.smithi163.stdout:3414: writing smithi16367185-45 from 494469 to 1149550 tid 1 2023-10-09T09:19:53.332 INFO:tasks.rados.rados.0.smithi163.stdout:3414: writing smithi16367185-45 from 1741862 to 2157641 tid 2 2023-10-09T09:19:53.339 INFO:tasks.rados.rados.0.smithi163.stdout:3414: writing smithi16367185-45 from 2710226 to 3294469 tid 3 2023-10-09T09:19:53.339 INFO:tasks.rados.rados.0.smithi163.stdout:3415: snap_create 2023-10-09T09:19:53.340 INFO:tasks.rados.rados.0.smithi163.stdout:3414: finishing write tid 1 to smithi16367185-45 2023-10-09T09:19:53.340 INFO:tasks.rados.rados.0.smithi163.stdout:3414: finishing write tid 2 to smithi16367185-45 2023-10-09T09:19:53.345 INFO:tasks.rados.rados.0.smithi163.stdout:3414: finishing write tid 3 to smithi16367185-45 2023-10-09T09:19:53.346 INFO:tasks.rados.rados.0.smithi163.stdout:3414: finishing write tid 4 to smithi16367185-45 2023-10-09T09:19:53.346 INFO:tasks.rados.rados.0.smithi163.stdout:3414: finishing write tid 5 to smithi16367185-45 2023-10-09T09:19:53.346 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 2533 (ObjNum 1267 snap 430 seq_num 1267) dirty exists 2023-10-09T09:19:53.346 INFO:tasks.rados.rados.0.smithi163.stdout:3414: left oid 45 (ObjNum 1267 snap 430 seq_num 1267) 2023-10-09T09:19:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:19:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:19:53] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.33.4" 2023-10-09T09:19:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:53 smithi163 bash[75425]: cluster 2023-10-09T09:19:52.293673+0000 osd.5 (osd.5) 459 : cluster 3 failed to encode map e918 with expected crc 2023-10-09T09:19:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:53 smithi163 bash[75425]: cluster 2023-10-09T09:19:53.298076+0000 osd.4 (osd.4) 404 : cluster 3 failed to encode map e919 with expected crc 2023-10-09T09:19:53.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:53 smithi163 bash[75425]: cluster 2023-10-09T09:19:53.301129+0000 mon.a (mon.0) 1127 : cluster 0 osdmap e919: 8 total, 8 up, 8 in 2023-10-09T09:19:53.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:53 smithi163 bash[78107]: cluster 2023-10-09T09:19:52.293673+0000 osd.5 (osd.5) 459 : cluster 3 failed to encode map e918 with expected crc 2023-10-09T09:19:53.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:53 smithi163 bash[78107]: cluster 2023-10-09T09:19:53.298076+0000 osd.4 (osd.4) 404 : cluster 3 failed to encode map e919 with expected crc 2023-10-09T09:19:53.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:53 smithi163 bash[78107]: cluster 2023-10-09T09:19:53.301129+0000 mon.a (mon.0) 1127 : cluster 0 osdmap e919: 8 total, 8 up, 8 in 2023-10-09T09:19:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:53 smithi181 bash[59345]: cluster 2023-10-09T09:19:52.293673+0000 osd.5 (osd.5) 459 : cluster 3 failed to encode map e918 with expected crc 2023-10-09T09:19:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:53 smithi181 bash[59345]: cluster 2023-10-09T09:19:53.298076+0000 osd.4 (osd.4) 404 : cluster 3 failed to encode map e919 with expected crc 2023-10-09T09:19:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:53 smithi181 bash[59345]: cluster 2023-10-09T09:19:53.301129+0000 mon.a (mon.0) 1127 : cluster 0 osdmap e919: 8 total, 8 up, 8 in 2023-10-09T09:19:54.310 INFO:tasks.rados.rados.0.smithi163.stdout:3414: done (1 left) 2023-10-09T09:19:54.310 INFO:tasks.rados.rados.0.smithi163.stdout:3415: done (0 left) 2023-10-09T09:19:54.310 INFO:tasks.rados.rados.0.smithi163.stdout:3416: write oid 37 current snap is 431 2023-10-09T09:19:54.310 INFO:tasks.rados.rados.0.smithi163.stdout:3416: seq_num 1268 ranges {511276=730330,1800344=559437,3021296=289980} 2023-10-09T09:19:54.320 INFO:tasks.rados.rados.0.smithi163.stdout:3416: writing smithi16367185-37 from 511276 to 1241606 tid 1 2023-10-09T09:19:54.327 INFO:tasks.rados.rados.0.smithi163.stdout:3416: writing smithi16367185-37 from 1800344 to 2359781 tid 2 2023-10-09T09:19:54.333 INFO:tasks.rados.rados.0.smithi163.stdout:3416: writing smithi16367185-37 from 3021296 to 3311276 tid 3 2023-10-09T09:19:54.333 INFO:tasks.rados.rados.0.smithi163.stdout:3417: write oid 25 current snap is 431 2023-10-09T09:19:54.333 INFO:tasks.rados.rados.0.smithi163.stdout:3417: seq_num 1269 ranges {528083=405579,1342473=419448,2248719=733949} 2023-10-09T09:19:54.338 INFO:tasks.rados.rados.0.smithi163.stdout:3417: writing smithi16367185-25 from 528083 to 933662 tid 1 2023-10-09T09:19:54.343 INFO:tasks.rados.rados.0.smithi163.stdout:3417: writing smithi16367185-25 from 1342473 to 1761921 tid 2 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:3417: writing smithi16367185-25 from 2248719 to 2982668 tid 3 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:3418: rollback oid 18 current snap is 431 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 18 to 420 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:3416: finishing write tid 1 to smithi16367185-37 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:3416: finishing write tid 2 to smithi16367185-37 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:3416: finishing write tid 3 to smithi16367185-37 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:3416: finishing write tid 4 to smithi16367185-37 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:3416: finishing write tid 5 to smithi16367185-37 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 1726 (ObjNum 1268 snap 431 seq_num 1268) dirty exists 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:3416: left oid 37 (ObjNum 1268 snap 431 seq_num 1268) 2023-10-09T09:19:54.351 INFO:tasks.rados.rados.0.smithi163.stdout:3416: done (2 left) 2023-10-09T09:19:54.352 INFO:tasks.rados.rados.0.smithi163.stdout:3419: read oid 13 snap -1 2023-10-09T09:19:54.352 INFO:tasks.rados.rados.0.smithi163.stdout:3419: expect (ObjNum 1195 snap 401 seq_num 1195) 2023-10-09T09:19:54.352 INFO:tasks.rados.rados.0.smithi163.stdout:3420: write oid 3 current snap is 431 2023-10-09T09:19:54.352 INFO:tasks.rados.rados.0.smithi163.stdout:3417: finishing write tid 1 to smithi16367185-25 2023-10-09T09:19:54.352 INFO:tasks.rados.rados.0.smithi163.stdout:3417: finishing write tid 2 to smithi16367185-25 2023-10-09T09:19:54.352 INFO:tasks.rados.rados.0.smithi163.stdout:3420: seq_num 1270 ranges {544890=597181,1917308=563106,3076142=268748} 2023-10-09T09:19:54.357 INFO:tasks.rados.rados.0.smithi163.stdout:3420: writing smithi16367185-3 from 544890 to 1142071 tid 1 2023-10-09T09:19:54.365 INFO:tasks.rados.rados.0.smithi163.stdout:3420: writing smithi16367185-3 from 1917308 to 2480414 tid 2 2023-10-09T09:19:54.371 INFO:tasks.rados.rados.0.smithi163.stdout:3420: writing smithi16367185-3 from 3076142 to 3344890 tid 3 2023-10-09T09:19:54.371 INFO:tasks.rados.rados.0.smithi163.stdout:3421: write oid 5 current snap is 431 2023-10-09T09:19:54.371 INFO:tasks.rados.rados.0.smithi163.stdout:3421: seq_num 1271 ranges {561697=672430,1975790=706764,3103565=258132} 2023-10-09T09:19:54.377 INFO:tasks.rados.rados.0.smithi163.stdout:3421: writing smithi16367185-5 from 561697 to 1234127 tid 1 2023-10-09T09:19:54.385 INFO:tasks.rados.rados.0.smithi163.stdout:3421: writing smithi16367185-5 from 1975790 to 2682554 tid 2 2023-10-09T09:19:54.390 INFO:tasks.rados.rados.0.smithi163.stdout:3421: writing smithi16367185-5 from 3103565 to 3361697 tid 3 2023-10-09T09:19:54.390 INFO:tasks.rados.rados.0.smithi163.stdout:3422: rollback oid 38 current snap is 431 2023-10-09T09:19:54.390 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 38 to 424 2023-10-09T09:19:54.390 INFO:tasks.rados.rados.0.smithi163.stdout:3417: finishing write tid 3 to smithi16367185-25 2023-10-09T09:19:54.390 INFO:tasks.rados.rados.0.smithi163.stdout:3418: finishing rollback tid 0 to smithi16367185-18 2023-10-09T09:19:54.391 INFO:tasks.rados.rados.0.smithi163.stdout:3417: finishing write tid 4 to smithi16367185-25 2023-10-09T09:19:54.391 INFO:tasks.rados.rados.0.smithi163.stdout:3418: finishing rollback tid 1 to smithi16367185-18 2023-10-09T09:19:54.391 INFO:tasks.rados.rados.0.smithi163.stdout:3419: expect (ObjNum 1195 snap 401 seq_num 1195) 2023-10-09T09:19:54.483 INFO:tasks.rados.rados.0.smithi163.stdout:3417: finishing write tid 5 to smithi16367185-25 2023-10-09T09:19:54.483 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1589 (ObjNum 1269 snap 431 seq_num 1269) dirty exists 2023-10-09T09:19:54.483 INFO:tasks.rados.rados.0.smithi163.stdout:3417: left oid 25 (ObjNum 1269 snap 431 seq_num 1269) 2023-10-09T09:19:54.483 INFO:tasks.rados.rados.0.smithi163.stdout:3418: finishing rollback tid 2 to smithi16367185-18 2023-10-09T09:19:54.483 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1184 (ObjNum 1219 snap 404 seq_num 1219) dirty exists 2023-10-09T09:19:54.483 INFO:tasks.rados.rados.0.smithi163.stdout:3420: finishing write tid 1 to smithi16367185-3 2023-10-09T09:19:54.483 INFO:tasks.rados.rados.0.smithi163.stdout:3420: finishing write tid 2 to smithi16367185-3 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:3420: finishing write tid 3 to smithi16367185-3 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:3420: finishing write tid 4 to smithi16367185-3 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:3420: finishing write tid 5 to smithi16367185-3 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 964 (ObjNum 1270 snap 431 seq_num 1270) dirty exists 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:3420: left oid 3 (ObjNum 1270 snap 431 seq_num 1270) 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:3421: finishing write tid 1 to smithi16367185-5 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:3421: finishing write tid 2 to smithi16367185-5 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:3421: finishing write tid 3 to smithi16367185-5 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:3421: finishing write tid 4 to smithi16367185-5 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:3421: finishing write tid 5 to smithi16367185-5 2023-10-09T09:19:54.484 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1735 (ObjNum 1271 snap 431 seq_num 1271) dirty exists 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3421: left oid 5 (ObjNum 1271 snap 431 seq_num 1271) 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3422: finishing rollback tid 0 to smithi16367185-38 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3422: finishing rollback tid 1 to smithi16367185-38 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3422: finishing rollback tid 2 to smithi16367185-38 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 38 v 1642 (ObjNum 1254 snap 422 seq_num 1254) dirty exists 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3417: done (5 left) 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3418: done (4 left) 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3419: done (3 left) 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3420: done (2 left) 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3421: done (1 left) 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3422: done (0 left) 2023-10-09T09:19:54.485 INFO:tasks.rados.rados.0.smithi163.stdout:3423: read oid 17 snap -1 2023-10-09T09:19:54.486 INFO:tasks.rados.rados.0.smithi163.stdout:3423: expect (ObjNum 1222 snap 408 seq_num 1222) 2023-10-09T09:19:54.486 INFO:tasks.rados.rados.0.smithi163.stdout:3424: delete oid 18 current snap is 431 2023-10-09T09:19:54.486 INFO:tasks.rados.rados.0.smithi163.stdout:3424: done (1 left) 2023-10-09T09:19:54.486 INFO:tasks.rados.rados.0.smithi163.stdout:3425: write oid 6 current snap is 431 2023-10-09T09:19:54.486 INFO:tasks.rados.rados.0.smithi163.stdout:3425: seq_num 1273 ranges {595311=422928,1576401=710433,3042058=353253} 2023-10-09T09:19:54.498 INFO:tasks.rados.rados.0.smithi163.stdout:3425: writing smithi16367185-6 from 595311 to 1018239 tid 1 2023-10-09T09:19:54.507 INFO:tasks.rados.rados.0.smithi163.stdout:3425: writing smithi16367185-6 from 1576401 to 2286834 tid 2 2023-10-09T09:19:54.513 INFO:tasks.rados.rados.0.smithi163.stdout:3425: writing smithi16367185-6 from 3042058 to 3395311 tid 3 2023-10-09T09:19:54.514 INFO:tasks.rados.rados.0.smithi163.stdout:3426: read oid 49 snap 424 2023-10-09T09:19:54.514 INFO:tasks.rados.rados.0.smithi163.stdout:3426: expect (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:19:54.514 INFO:tasks.rados.rados.0.smithi163.stdout:3427: write oid 2 current snap is 431 2023-10-09T09:19:54.514 INFO:tasks.rados.rados.0.smithi163.stdout:3427: seq_num 1274 ranges {612118=498177,1634883=570444,2669481=706662} 2023-10-09T09:19:54.520 INFO:tasks.rados.rados.0.smithi163.stdout:3427: writing smithi16367185-2 from 612118 to 1110295 tid 1 2023-10-09T09:19:54.526 INFO:tasks.rados.rados.0.smithi163.stdout:3427: writing smithi16367185-2 from 1634883 to 2205327 tid 2 2023-10-09T09:19:54.533 INFO:tasks.rados.rados.0.smithi163.stdout:3427: writing smithi16367185-2 from 2669481 to 3376143 tid 3 2023-10-09T09:19:54.533 INFO:tasks.rados.rados.0.smithi163.stdout:3428: snap_remove snap 429 2023-10-09T09:19:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:54 smithi163 bash[75425]: cluster 2023-10-09T09:19:53.296729+0000 osd.5 (osd.5) 460 : cluster 3 failed to encode map e919 with expected crc 2023-10-09T09:19:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:54 smithi163 bash[75425]: cluster 2023-10-09T09:19:53.302874+0000 osd.7 (osd.7) 375 : cluster 3 failed to encode map e919 with expected crc 2023-10-09T09:19:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:54 smithi163 bash[75425]: cluster 2023-10-09T09:19:53.444968+0000 mgr.y (mgr.34107) 400 : cluster 0 pgmap v932: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 236 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 307 KiB/s wr, 9 op/s 2023-10-09T09:19:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:54 smithi163 bash[75425]: cluster 2023-10-09T09:19:54.309852+0000 mon.a (mon.0) 1128 : cluster 0 osdmap e920: 8 total, 8 up, 8 in 2023-10-09T09:19:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:54 smithi163 bash[78107]: cluster 2023-10-09T09:19:53.296729+0000 osd.5 (osd.5) 460 : cluster 3 failed to encode map e919 with expected crc 2023-10-09T09:19:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:54 smithi163 bash[78107]: cluster 2023-10-09T09:19:53.302874+0000 osd.7 (osd.7) 375 : cluster 3 failed to encode map e919 with expected crc 2023-10-09T09:19:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:54 smithi163 bash[78107]: cluster 2023-10-09T09:19:53.444968+0000 mgr.y (mgr.34107) 400 : cluster 0 pgmap v932: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 236 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 307 KiB/s wr, 9 op/s 2023-10-09T09:19:54.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:54 smithi163 bash[78107]: cluster 2023-10-09T09:19:54.309852+0000 mon.a (mon.0) 1128 : cluster 0 osdmap e920: 8 total, 8 up, 8 in 2023-10-09T09:19:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:54 smithi181 bash[59345]: cluster 2023-10-09T09:19:53.296729+0000 osd.5 (osd.5) 460 : cluster 3 failed to encode map e919 with expected crc 2023-10-09T09:19:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:54 smithi181 bash[59345]: cluster 2023-10-09T09:19:53.302874+0000 osd.7 (osd.7) 375 : cluster 3 failed to encode map e919 with expected crc 2023-10-09T09:19:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:54 smithi181 bash[59345]: cluster 2023-10-09T09:19:53.444968+0000 mgr.y (mgr.34107) 400 : cluster 0 pgmap v932: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 236 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 307 KiB/s wr, 9 op/s 2023-10-09T09:19:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:54 smithi181 bash[59345]: cluster 2023-10-09T09:19:54.309852+0000 mon.a (mon.0) 1128 : cluster 0 osdmap e920: 8 total, 8 up, 8 in 2023-10-09T09:19:55.473 INFO:tasks.rados.rados.0.smithi163.stdout:3428: done (4 left) 2023-10-09T09:19:55.474 INFO:tasks.rados.rados.0.smithi163.stdout:3429: write oid 24 current snap is 431 2023-10-09T09:19:55.474 INFO:tasks.rados.rados.0.smithi163.stdout:3423: expect (ObjNum 1222 snap 408 seq_num 1222) 2023-10-09T09:19:55.598 INFO:tasks.rados.rados.0.smithi163.stdout:3425: finishing write tid 1 to smithi16367185-6 2023-10-09T09:19:55.599 INFO:tasks.rados.rados.0.smithi163.stdout:3425: finishing write tid 2 to smithi16367185-6 2023-10-09T09:19:55.599 INFO:tasks.rados.rados.0.smithi163.stdout:3425: finishing write tid 3 to smithi16367185-6 2023-10-09T09:19:55.599 INFO:tasks.rados.rados.0.smithi163.stdout:3429: seq_num 1275 ranges {628925=573426,1693365=714102,3096904=332021} 2023-10-09T09:19:55.610 INFO:tasks.rados.rados.0.smithi163.stdout:3429: writing smithi16367185-24 from 628925 to 1202351 tid 1 2023-10-09T09:19:55.618 INFO:tasks.rados.rados.0.smithi163.stdout:3429: writing smithi16367185-24 from 1693365 to 2407467 tid 2 2023-10-09T09:19:55.624 INFO:tasks.rados.rados.0.smithi163.stdout:3429: writing smithi16367185-24 from 3096904 to 3428925 tid 3 2023-10-09T09:19:55.624 INFO:tasks.rados.rados.0.smithi163.stdout:3423: done (4 left) 2023-10-09T09:19:55.624 INFO:tasks.rados.rados.0.smithi163.stdout:3430: snap_create 2023-10-09T09:19:55.625 INFO:tasks.rados.rados.0.smithi163.stdout:3425: finishing write tid 4 to smithi16367185-6 2023-10-09T09:19:55.625 INFO:tasks.rados.rados.0.smithi163.stdout:3425: finishing write tid 5 to smithi16367185-6 2023-10-09T09:19:55.625 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 365 (ObjNum 1273 snap 431 seq_num 1273) dirty exists 2023-10-09T09:19:55.625 INFO:tasks.rados.rados.0.smithi163.stdout:3425: left oid 6 (ObjNum 1273 snap 431 seq_num 1273) 2023-10-09T09:19:55.625 INFO:tasks.rados.rados.0.smithi163.stdout:3426: expect (ObjNum 1099 snap 365 seq_num 1099) 2023-10-09T09:19:55.777 INFO:tasks.rados.rados.0.smithi163.stdout:3427: finishing write tid 1 to smithi16367185-2 2023-10-09T09:19:55.777 INFO:tasks.rados.rados.0.smithi163.stdout:3427: finishing write tid 2 to smithi16367185-2 2023-10-09T09:19:55.777 INFO:tasks.rados.rados.0.smithi163.stdout:3427: finishing write tid 3 to smithi16367185-2 2023-10-09T09:19:55.777 INFO:tasks.rados.rados.0.smithi163.stdout:3427: finishing write tid 4 to smithi16367185-2 2023-10-09T09:19:55.777 INFO:tasks.rados.rados.0.smithi163.stdout:3427: finishing write tid 5 to smithi16367185-2 2023-10-09T09:19:55.777 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 929 (ObjNum 1274 snap 431 seq_num 1274) dirty exists 2023-10-09T09:19:55.777 INFO:tasks.rados.rados.0.smithi163.stdout:3427: left oid 2 (ObjNum 1274 snap 431 seq_num 1274) 2023-10-09T09:19:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:3429: finishing write tid 1 to smithi16367185-24 2023-10-09T09:19:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:3429: finishing write tid 2 to smithi16367185-24 2023-10-09T09:19:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:3429: finishing write tid 3 to smithi16367185-24 2023-10-09T09:19:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:3429: finishing write tid 4 to smithi16367185-24 2023-10-09T09:19:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:3429: finishing write tid 5 to smithi16367185-24 2023-10-09T09:19:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 753 (ObjNum 1275 snap 431 seq_num 1275) dirty exists 2023-10-09T09:19:55.778 INFO:tasks.rados.rados.0.smithi163.stdout:3429: left oid 24 (ObjNum 1275 snap 431 seq_num 1275) 2023-10-09T09:19:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:55 smithi181 bash[59345]: cluster 2023-10-09T09:19:54.307329+0000 osd.5 (osd.5) 461 : cluster 3 failed to encode map e920 with expected crc 2023-10-09T09:19:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:55 smithi181 bash[59345]: cluster 2023-10-09T09:19:54.308087+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e920 with expected crc 2023-10-09T09:19:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:55 smithi181 bash[59345]: cluster 2023-10-09T09:19:54.323281+0000 osd.7 (osd.7) 376 : cluster 3 failed to encode map e920 with expected crc 2023-10-09T09:19:55.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:55 smithi181 bash[59345]: audit 2023-10-09T09:19:55.350052+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:55.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:55 smithi163 bash[75425]: cluster 2023-10-09T09:19:54.307329+0000 osd.5 (osd.5) 461 : cluster 3 failed to encode map e920 with expected crc 2023-10-09T09:19:55.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:55 smithi163 bash[75425]: cluster 2023-10-09T09:19:54.308087+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e920 with expected crc 2023-10-09T09:19:55.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:55 smithi163 bash[75425]: cluster 2023-10-09T09:19:54.323281+0000 osd.7 (osd.7) 376 : cluster 3 failed to encode map e920 with expected crc 2023-10-09T09:19:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:55 smithi163 bash[75425]: audit 2023-10-09T09:19:55.350052+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:55 smithi163 bash[78107]: cluster 2023-10-09T09:19:54.307329+0000 osd.5 (osd.5) 461 : cluster 3 failed to encode map e920 with expected crc 2023-10-09T09:19:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:55 smithi163 bash[78107]: cluster 2023-10-09T09:19:54.308087+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e920 with expected crc 2023-10-09T09:19:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:55 smithi163 bash[78107]: cluster 2023-10-09T09:19:54.323281+0000 osd.7 (osd.7) 376 : cluster 3 failed to encode map e920 with expected crc 2023-10-09T09:19:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:55 smithi163 bash[78107]: audit 2023-10-09T09:19:55.350052+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:19:56.490 INFO:tasks.rados.rados.0.smithi163.stdout:3425: done (4 left) 2023-10-09T09:19:56.490 INFO:tasks.rados.rados.0.smithi163.stdout:3426: done (3 left) 2023-10-09T09:19:56.490 INFO:tasks.rados.rados.0.smithi163.stdout:3427: done (2 left) 2023-10-09T09:19:56.490 INFO:tasks.rados.rados.0.smithi163.stdout:3429: done (1 left) 2023-10-09T09:19:56.490 INFO:tasks.rados.rados.0.smithi163.stdout:3430: done (0 left) 2023-10-09T09:19:56.490 INFO:tasks.rados.rados.0.smithi163.stdout:3431: write oid 16 current snap is 432 2023-10-09T09:19:56.490 INFO:tasks.rados.rados.0.smithi163.stdout:3431: seq_num 1276 ranges {645732=648675,1751847=574113,3124327=321405} 2023-10-09T09:19:56.501 INFO:tasks.rados.rados.0.smithi163.stdout:3431: writing smithi16367185-16 from 645732 to 1294407 tid 1 2023-10-09T09:19:56.507 INFO:tasks.rados.rados.0.smithi163.stdout:3431: writing smithi16367185-16 from 1751847 to 2325960 tid 2 2023-10-09T09:19:56.514 INFO:tasks.rados.rados.0.smithi163.stdout:3431: writing smithi16367185-16 from 3124327 to 3445732 tid 3 2023-10-09T09:19:56.514 INFO:tasks.rados.rados.0.smithi163.stdout:3432: write oid 24 current snap is 432 2023-10-09T09:19:56.514 INFO:tasks.rados.rados.0.smithi163.stdout:3431: finishing write tid 1 to smithi16367185-16 2023-10-09T09:19:56.514 INFO:tasks.rados.rados.0.smithi163.stdout:3431: finishing write tid 2 to smithi16367185-16 2023-10-09T09:19:56.514 INFO:tasks.rados.rados.0.smithi163.stdout:3432: seq_num 1277 ranges {662539=723924,2093976=717771,3435397=27142} 2023-10-09T09:19:56.522 INFO:tasks.rados.rados.0.smithi163.stdout:3432: writing smithi16367185-24 from 662539 to 1386463 tid 1 2023-10-09T09:19:56.530 INFO:tasks.rados.rados.0.smithi163.stdout:3432: writing smithi16367185-24 from 2093976 to 2811747 tid 2 2023-10-09T09:19:56.534 INFO:tasks.rados.rados.0.smithi163.stdout:3432: writing smithi16367185-24 from 3435397 to 3462539 tid 3 2023-10-09T09:19:56.534 INFO:tasks.rados.rados.0.smithi163.stdout:3433: rollback oid 35 current snap is 432 2023-10-09T09:19:56.535 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 35 to 427 2023-10-09T09:19:56.535 INFO:tasks.rados.rados.0.smithi163.stdout:3431: finishing write tid 3 to smithi16367185-16 2023-10-09T09:19:56.535 INFO:tasks.rados.rados.0.smithi163.stdout:3431: finishing write tid 4 to smithi16367185-16 2023-10-09T09:19:56.535 INFO:tasks.rados.rados.0.smithi163.stdout:3431: finishing write tid 5 to smithi16367185-16 2023-10-09T09:19:56.535 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1741 (ObjNum 1276 snap 432 seq_num 1276) dirty exists 2023-10-09T09:19:56.535 INFO:tasks.rados.rados.0.smithi163.stdout:3431: left oid 16 (ObjNum 1276 snap 432 seq_num 1276) 2023-10-09T09:19:56.535 INFO:tasks.rados.rados.0.smithi163.stdout:3431: done (2 left) 2023-10-09T09:19:56.535 INFO:tasks.rados.rados.0.smithi163.stdout:3434: write oid 45 current snap is 432 2023-10-09T09:19:56.535 INFO:tasks.rados.rados.0.smithi163.stdout:3432: finishing write tid 1 to smithi16367185-24 2023-10-09T09:19:56.535 INFO:tasks.rados.rados.0.smithi163.stdout:3434: seq_num 1278 ranges {679346=515526,1868811=577782,3179173=300173} 2023-10-09T09:19:56.541 INFO:tasks.rados.rados.0.smithi163.stdout:3434: writing smithi16367185-45 from 679346 to 1194872 tid 1 2023-10-09T09:19:56.548 INFO:tasks.rados.rados.0.smithi163.stdout:3434: writing smithi16367185-45 from 1868811 to 2446593 tid 2 2023-10-09T09:19:56.555 INFO:tasks.rados.rados.0.smithi163.stdout:3434: writing smithi16367185-45 from 3179173 to 3479346 tid 3 2023-10-09T09:19:56.555 INFO:tasks.rados.rados.0.smithi163.stdout:3435: snap_create 2023-10-09T09:19:56.555 INFO:tasks.rados.rados.0.smithi163.stdout:3432: finishing write tid 2 to smithi16367185-24 2023-10-09T09:19:56.555 INFO:tasks.rados.rados.0.smithi163.stdout:3432: finishing write tid 3 to smithi16367185-24 2023-10-09T09:19:56.555 INFO:tasks.rados.rados.0.smithi163.stdout:3432: finishing write tid 4 to smithi16367185-24 2023-10-09T09:19:56.555 INFO:tasks.rados.rados.0.smithi163.stdout:3432: finishing write tid 5 to smithi16367185-24 2023-10-09T09:19:56.555 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 758 (ObjNum 1277 snap 432 seq_num 1277) dirty exists 2023-10-09T09:19:56.555 INFO:tasks.rados.rados.0.smithi163.stdout:3432: left oid 24 (ObjNum 1277 snap 432 seq_num 1277) 2023-10-09T09:19:56.555 INFO:tasks.rados.rados.0.smithi163.stdout:3433: finishing rollback tid 1 to smithi16367185-35 2023-10-09T09:19:56.555 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 901 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:19:56.556 INFO:tasks.rados.rados.0.smithi163.stdout:3434: finishing write tid 1 to smithi16367185-45 2023-10-09T09:19:56.556 INFO:tasks.rados.rados.0.smithi163.stdout:3434: finishing write tid 2 to smithi16367185-45 2023-10-09T09:19:56.560 INFO:tasks.rados.rados.0.smithi163.stdout:3434: finishing write tid 3 to smithi16367185-45 2023-10-09T09:19:56.560 INFO:tasks.rados.rados.0.smithi163.stdout:3434: finishing write tid 4 to smithi16367185-45 2023-10-09T09:19:56.560 INFO:tasks.rados.rados.0.smithi163.stdout:3434: finishing write tid 5 to smithi16367185-45 2023-10-09T09:19:56.560 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 2538 (ObjNum 1278 snap 432 seq_num 1278) dirty exists 2023-10-09T09:19:56.560 INFO:tasks.rados.rados.0.smithi163.stdout:3434: left oid 45 (ObjNum 1278 snap 432 seq_num 1278) 2023-10-09T09:19:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:56 smithi181 bash[59345]: cluster 2023-10-09T09:19:55.445694+0000 mgr.y (mgr.34107) 401 : cluster 0 pgmap v934: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 751 KiB/s wr, 11 op/s 2023-10-09T09:19:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:56 smithi181 bash[59345]: cluster 2023-10-09T09:19:55.468707+0000 osd.5 (osd.5) 462 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:56 smithi181 bash[59345]: cluster 2023-10-09T09:19:55.473064+0000 mon.a (mon.0) 1130 : cluster 0 osdmap e921: 8 total, 8 up, 8 in 2023-10-09T09:19:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:56 smithi181 bash[59345]: cluster 2023-10-09T09:19:55.589071+0000 osd.4 (osd.4) 405 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:56.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:56 smithi163 bash[75425]: cluster 2023-10-09T09:19:55.445694+0000 mgr.y (mgr.34107) 401 : cluster 0 pgmap v934: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 751 KiB/s wr, 11 op/s 2023-10-09T09:19:56.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:56 smithi163 bash[75425]: cluster 2023-10-09T09:19:55.468707+0000 osd.5 (osd.5) 462 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:56.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:56 smithi163 bash[75425]: cluster 2023-10-09T09:19:55.473064+0000 mon.a (mon.0) 1130 : cluster 0 osdmap e921: 8 total, 8 up, 8 in 2023-10-09T09:19:56.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:56 smithi163 bash[75425]: cluster 2023-10-09T09:19:55.589071+0000 osd.4 (osd.4) 405 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:56.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:56 smithi163 bash[78107]: cluster 2023-10-09T09:19:55.445694+0000 mgr.y (mgr.34107) 401 : cluster 0 pgmap v934: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 751 KiB/s wr, 11 op/s 2023-10-09T09:19:56.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:56 smithi163 bash[78107]: cluster 2023-10-09T09:19:55.468707+0000 osd.5 (osd.5) 462 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:56.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:56 smithi163 bash[78107]: cluster 2023-10-09T09:19:55.473064+0000 mon.a (mon.0) 1130 : cluster 0 osdmap e921: 8 total, 8 up, 8 in 2023-10-09T09:19:56.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:56 smithi163 bash[78107]: cluster 2023-10-09T09:19:55.589071+0000 osd.4 (osd.4) 405 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:57.500 INFO:tasks.rados.rados.0.smithi163.stdout:3432: done (3 left) 2023-10-09T09:19:57.500 INFO:tasks.rados.rados.0.smithi163.stdout:3433: done (2 left) 2023-10-09T09:19:57.501 INFO:tasks.rados.rados.0.smithi163.stdout:3434: done (1 left) 2023-10-09T09:19:57.501 INFO:tasks.rados.rados.0.smithi163.stdout:3435: done (0 left) 2023-10-09T09:19:57.501 INFO:tasks.rados.rados.0.smithi163.stdout:3436: snap_remove snap 426 2023-10-09T09:19:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:57 smithi181 bash[59345]: cluster 2023-10-09T09:19:55.855925+0000 osd.7 (osd.7) 377 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:57 smithi181 bash[59345]: cluster 2023-10-09T09:19:55.886417+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:57 smithi181 bash[59345]: cluster 2023-10-09T09:19:56.486887+0000 osd.4 (osd.4) 406 : cluster 3 failed to encode map e922 with expected crc 2023-10-09T09:19:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:57 smithi181 bash[59345]: cluster 2023-10-09T09:19:56.486893+0000 osd.5 (osd.5) 463 : cluster 3 failed to encode map e922 with expected crc 2023-10-09T09:19:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:57 smithi181 bash[59345]: cluster 2023-10-09T09:19:56.489903+0000 mon.a (mon.0) 1131 : cluster 0 osdmap e922: 8 total, 8 up, 8 in 2023-10-09T09:19:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:57 smithi181 bash[59345]: cluster 2023-10-09T09:19:56.535681+0000 osd.7 (osd.7) 378 : cluster 3 failed to encode map e922 with expected crc 2023-10-09T09:19:57.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[78107]: cluster 2023-10-09T09:19:55.855925+0000 osd.7 (osd.7) 377 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:57.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[78107]: cluster 2023-10-09T09:19:55.886417+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:57.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[78107]: cluster 2023-10-09T09:19:56.486887+0000 osd.4 (osd.4) 406 : cluster 3 failed to encode map e922 with expected crc 2023-10-09T09:19:57.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[78107]: cluster 2023-10-09T09:19:56.486893+0000 osd.5 (osd.5) 463 : cluster 3 failed to encode map e922 with expected crc 2023-10-09T09:19:57.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[78107]: cluster 2023-10-09T09:19:56.489903+0000 mon.a (mon.0) 1131 : cluster 0 osdmap e922: 8 total, 8 up, 8 in 2023-10-09T09:19:57.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[78107]: cluster 2023-10-09T09:19:56.535681+0000 osd.7 (osd.7) 378 : cluster 3 failed to encode map e922 with expected crc 2023-10-09T09:19:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[75425]: cluster 2023-10-09T09:19:55.855925+0000 osd.7 (osd.7) 377 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[75425]: cluster 2023-10-09T09:19:55.886417+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e921 with expected crc 2023-10-09T09:19:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[75425]: cluster 2023-10-09T09:19:56.486887+0000 osd.4 (osd.4) 406 : cluster 3 failed to encode map e922 with expected crc 2023-10-09T09:19:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[75425]: cluster 2023-10-09T09:19:56.486893+0000 osd.5 (osd.5) 463 : cluster 3 failed to encode map e922 with expected crc 2023-10-09T09:19:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[75425]: cluster 2023-10-09T09:19:56.489903+0000 mon.a (mon.0) 1131 : cluster 0 osdmap e922: 8 total, 8 up, 8 in 2023-10-09T09:19:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:57 smithi163 bash[75425]: cluster 2023-10-09T09:19:56.535681+0000 osd.7 (osd.7) 378 : cluster 3 failed to encode map e922 with expected crc 2023-10-09T09:19:58.504 INFO:tasks.rados.rados.0.smithi163.stdout:3436: done (0 left) 2023-10-09T09:19:58.504 INFO:tasks.rados.rados.0.smithi163.stdout:3437: read oid 31 snap 418 2023-10-09T09:19:58.504 INFO:tasks.rados.rados.0.smithi163.stdout:3437: expect deleted 2023-10-09T09:19:58.504 INFO:tasks.rados.rados.0.smithi163.stdout:3438: write oid 25 current snap is 433 2023-10-09T09:19:58.504 INFO:tasks.rados.rados.0.smithi163.stdout:3438: seq_num 1279 ranges {696153=590775,1927293=721440,3206596=289557} 2023-10-09T09:19:58.515 INFO:tasks.rados.rados.0.smithi163.stdout:3438: writing smithi16367185-25 from 696153 to 1286928 tid 1 2023-10-09T09:19:58.524 INFO:tasks.rados.rados.0.smithi163.stdout:3438: writing smithi16367185-25 from 1927293 to 2648733 tid 2 2023-10-09T09:19:58.529 INFO:tasks.rados.rados.0.smithi163.stdout:3438: writing smithi16367185-25 from 3206596 to 3496153 tid 3 2023-10-09T09:19:58.529 INFO:tasks.rados.rados.0.smithi163.stdout:3439: snap_remove snap 385 2023-10-09T09:19:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:58 smithi181 bash[59345]: cluster 2023-10-09T09:19:57.446581+0000 mgr.y (mgr.34107) 402 : cluster 0 pgmap v937: 89 pgs: 89 active+clean; 251 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2023-10-09T09:19:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:58 smithi181 bash[59345]: cluster 2023-10-09T09:19:57.494038+0000 osd.5 (osd.5) 464 : cluster 3 failed to encode map e923 with expected crc 2023-10-09T09:19:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:58 smithi181 bash[59345]: cluster 2023-10-09T09:19:57.499097+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e923: 8 total, 8 up, 8 in 2023-10-09T09:19:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:58 smithi181 bash[59345]: cluster 2023-10-09T09:19:57.867738+0000 osd.7 (osd.7) 379 : cluster 3 failed to encode map e923 with expected crc 2023-10-09T09:19:58.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:58 smithi163 bash[78107]: cluster 2023-10-09T09:19:57.446581+0000 mgr.y (mgr.34107) 402 : cluster 0 pgmap v937: 89 pgs: 89 active+clean; 251 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2023-10-09T09:19:58.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:58 smithi163 bash[78107]: cluster 2023-10-09T09:19:57.494038+0000 osd.5 (osd.5) 464 : cluster 3 failed to encode map e923 with expected crc 2023-10-09T09:19:58.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:58 smithi163 bash[78107]: cluster 2023-10-09T09:19:57.499097+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e923: 8 total, 8 up, 8 in 2023-10-09T09:19:58.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:58 smithi163 bash[78107]: cluster 2023-10-09T09:19:57.867738+0000 osd.7 (osd.7) 379 : cluster 3 failed to encode map e923 with expected crc 2023-10-09T09:19:58.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:58 smithi163 bash[75425]: cluster 2023-10-09T09:19:57.446581+0000 mgr.y (mgr.34107) 402 : cluster 0 pgmap v937: 89 pgs: 89 active+clean; 251 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2023-10-09T09:19:58.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:58 smithi163 bash[75425]: cluster 2023-10-09T09:19:57.494038+0000 osd.5 (osd.5) 464 : cluster 3 failed to encode map e923 with expected crc 2023-10-09T09:19:58.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:58 smithi163 bash[75425]: cluster 2023-10-09T09:19:57.499097+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e923: 8 total, 8 up, 8 in 2023-10-09T09:19:58.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:58 smithi163 bash[75425]: cluster 2023-10-09T09:19:57.867738+0000 osd.7 (osd.7) 379 : cluster 3 failed to encode map e923 with expected crc 2023-10-09T09:19:59.526 INFO:tasks.rados.rados.0.smithi163.stdout:3439: done (2 left) 2023-10-09T09:19:59.526 INFO:tasks.rados.rados.0.smithi163.stdout:3440: read oid 21 snap -1 2023-10-09T09:19:59.527 INFO:tasks.rados.rados.0.smithi163.stdout:3440: expect (ObjNum 1223 snap 408 seq_num 1223) 2023-10-09T09:19:59.527 INFO:tasks.rados.rados.0.smithi163.stdout:3441: snap_create 2023-10-09T09:19:59.527 INFO:tasks.rados.rados.0.smithi163.stdout:3438: finishing write tid 1 to smithi16367185-25 2023-10-09T09:19:59.527 INFO:tasks.rados.rados.0.smithi163.stdout:3438: finishing write tid 2 to smithi16367185-25 2023-10-09T09:19:59.527 INFO:tasks.rados.rados.0.smithi163.stdout:3438: finishing write tid 3 to smithi16367185-25 2023-10-09T09:19:59.527 INFO:tasks.rados.rados.0.smithi163.stdout:3438: finishing write tid 4 to smithi16367185-25 2023-10-09T09:19:59.527 INFO:tasks.rados.rados.0.smithi163.stdout:3438: finishing write tid 5 to smithi16367185-25 2023-10-09T09:19:59.527 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1598 (ObjNum 1279 snap 433 seq_num 1279) dirty exists 2023-10-09T09:19:59.527 INFO:tasks.rados.rados.0.smithi163.stdout:3438: left oid 25 (ObjNum 1279 snap 433 seq_num 1279) 2023-10-09T09:19:59.542 INFO:tasks.rados.rados.0.smithi163.stdout:3440: expect (ObjNum 1223 snap 408 seq_num 1223) 2023-10-09T09:19:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:59 smithi181 bash[59345]: cluster 2023-10-09T09:19:58.498906+0000 osd.5 (osd.5) 465 : cluster 3 failed to encode map e924 with expected crc 2023-10-09T09:19:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:59 smithi181 bash[59345]: cluster 2023-10-09T09:19:58.499424+0000 osd.4 (osd.4) 407 : cluster 3 failed to encode map e924 with expected crc 2023-10-09T09:19:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:19:59 smithi181 bash[59345]: cluster 2023-10-09T09:19:58.503496+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e924: 8 total, 8 up, 8 in 2023-10-09T09:19:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:59 smithi163 bash[78107]: cluster 2023-10-09T09:19:58.498906+0000 osd.5 (osd.5) 465 : cluster 3 failed to encode map e924 with expected crc 2023-10-09T09:19:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:59 smithi163 bash[78107]: cluster 2023-10-09T09:19:58.499424+0000 osd.4 (osd.4) 407 : cluster 3 failed to encode map e924 with expected crc 2023-10-09T09:19:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:19:59 smithi163 bash[78107]: cluster 2023-10-09T09:19:58.503496+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e924: 8 total, 8 up, 8 in 2023-10-09T09:19:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:59 smithi163 bash[75425]: cluster 2023-10-09T09:19:58.498906+0000 osd.5 (osd.5) 465 : cluster 3 failed to encode map e924 with expected crc 2023-10-09T09:19:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:59 smithi163 bash[75425]: cluster 2023-10-09T09:19:58.499424+0000 osd.4 (osd.4) 407 : cluster 3 failed to encode map e924 with expected crc 2023-10-09T09:19:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:19:59 smithi163 bash[75425]: cluster 2023-10-09T09:19:58.503496+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e924: 8 total, 8 up, 8 in 2023-10-09T09:20:00.294 INFO:tasks.rados.rados.0.smithi163.stdout:3437: done (3 left) 2023-10-09T09:20:00.295 INFO:tasks.rados.rados.0.smithi163.stdout:3438: done (2 left) 2023-10-09T09:20:00.295 INFO:tasks.rados.rados.0.smithi163.stdout:3440: done (1 left) 2023-10-09T09:20:00.295 INFO:tasks.rados.rados.0.smithi163.stdout:3441: done (0 left) 2023-10-09T09:20:00.295 INFO:tasks.rados.rados.0.smithi163.stdout:3442: snap_remove snap 419 2023-10-09T09:20:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:00 smithi181 bash[59345]: cluster 2023-10-09T09:19:58.687046+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e924 with expected crc 2023-10-09T09:20:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:00 smithi181 bash[59345]: cluster 2023-10-09T09:19:59.447503+0000 mgr.y (mgr.34107) 403 : cluster 0 pgmap v940: 89 pgs: 89 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2023-10-09T09:20:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:00 smithi181 bash[59345]: cluster 2023-10-09T09:19:59.518261+0000 osd.5 (osd.5) 466 : cluster 3 failed to encode map e925 with expected crc 2023-10-09T09:20:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:00 smithi181 bash[59345]: cluster 2023-10-09T09:19:59.522018+0000 osd.4 (osd.4) 408 : cluster 3 failed to encode map e925 with expected crc 2023-10-09T09:20:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:00 smithi181 bash[59345]: cluster 2023-10-09T09:19:59.525301+0000 mon.a (mon.0) 1134 : cluster 0 osdmap e925: 8 total, 8 up, 8 in 2023-10-09T09:20:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:00 smithi181 bash[59345]: cluster 2023-10-09T09:19:59.529216+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e925 with expected crc 2023-10-09T09:20:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:00 smithi181 bash[59345]: cluster 2023-10-09T09:20:00.000146+0000 mon.a (mon.0) 1135 : cluster 1 overall HEALTH_OK 2023-10-09T09:20:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:00 smithi181 bash[59345]: cluster 2023-10-09T09:20:00.288465+0000 osd.5 (osd.5) 467 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:00 smithi181 bash[59345]: cluster 2023-10-09T09:20:00.294407+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e926: 8 total, 8 up, 8 in 2023-10-09T09:20:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[78107]: cluster 2023-10-09T09:19:58.687046+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e924 with expected crc 2023-10-09T09:20:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[78107]: cluster 2023-10-09T09:19:59.447503+0000 mgr.y (mgr.34107) 403 : cluster 0 pgmap v940: 89 pgs: 89 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2023-10-09T09:20:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[78107]: cluster 2023-10-09T09:19:59.518261+0000 osd.5 (osd.5) 466 : cluster 3 failed to encode map e925 with expected crc 2023-10-09T09:20:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[78107]: cluster 2023-10-09T09:19:59.522018+0000 osd.4 (osd.4) 408 : cluster 3 failed to encode map e925 with expected crc 2023-10-09T09:20:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[78107]: cluster 2023-10-09T09:19:59.525301+0000 mon.a (mon.0) 1134 : cluster 0 osdmap e925: 8 total, 8 up, 8 in 2023-10-09T09:20:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[78107]: cluster 2023-10-09T09:19:59.529216+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e925 with expected crc 2023-10-09T09:20:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[78107]: cluster 2023-10-09T09:20:00.000146+0000 mon.a (mon.0) 1135 : cluster 1 overall HEALTH_OK 2023-10-09T09:20:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[78107]: cluster 2023-10-09T09:20:00.288465+0000 osd.5 (osd.5) 467 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:00.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[78107]: cluster 2023-10-09T09:20:00.294407+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e926: 8 total, 8 up, 8 in 2023-10-09T09:20:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[75425]: cluster 2023-10-09T09:19:58.687046+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e924 with expected crc 2023-10-09T09:20:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[75425]: cluster 2023-10-09T09:19:59.447503+0000 mgr.y (mgr.34107) 403 : cluster 0 pgmap v940: 89 pgs: 89 active+clean; 255 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2023-10-09T09:20:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[75425]: cluster 2023-10-09T09:19:59.518261+0000 osd.5 (osd.5) 466 : cluster 3 failed to encode map e925 with expected crc 2023-10-09T09:20:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[75425]: cluster 2023-10-09T09:19:59.522018+0000 osd.4 (osd.4) 408 : cluster 3 failed to encode map e925 with expected crc 2023-10-09T09:20:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[75425]: cluster 2023-10-09T09:19:59.525301+0000 mon.a (mon.0) 1134 : cluster 0 osdmap e925: 8 total, 8 up, 8 in 2023-10-09T09:20:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[75425]: cluster 2023-10-09T09:19:59.529216+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e925 with expected crc 2023-10-09T09:20:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[75425]: cluster 2023-10-09T09:20:00.000146+0000 mon.a (mon.0) 1135 : cluster 1 overall HEALTH_OK 2023-10-09T09:20:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[75425]: cluster 2023-10-09T09:20:00.288465+0000 osd.5 (osd.5) 467 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:00 smithi163 bash[75425]: cluster 2023-10-09T09:20:00.294407+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e926: 8 total, 8 up, 8 in 2023-10-09T09:20:01.296 INFO:tasks.rados.rados.0.smithi163.stdout:3442: done (0 left) 2023-10-09T09:20:01.296 INFO:tasks.rados.rados.0.smithi163.stdout:3443: rollback oid 11 current snap is 434 2023-10-09T09:20:01.296 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 11 to 424 2023-10-09T09:20:01.296 INFO:tasks.rados.rados.0.smithi163.stdout:3444: snap_remove snap 415 2023-10-09T09:20:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:01 smithi181 bash[59345]: cluster 2023-10-09T09:20:00.428731+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:01 smithi181 bash[59345]: cluster 2023-10-09T09:20:00.655964+0000 osd.4 (osd.4) 409 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:01 smithi181 bash[59345]: cluster 2023-10-09T09:20:00.939083+0000 osd.7 (osd.7) 380 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:01 smithi181 bash[59345]: cluster 2023-10-09T09:20:01.290926+0000 osd.5 (osd.5) 468 : cluster 3 failed to encode map e927 with expected crc 2023-10-09T09:20:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:01 smithi181 bash[59345]: cluster 2023-10-09T09:20:01.291767+0000 osd.4 (osd.4) 410 : cluster 3 failed to encode map e927 with expected crc 2023-10-09T09:20:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:01 smithi181 bash[59345]: cluster 2023-10-09T09:20:01.295620+0000 mon.a (mon.0) 1137 : cluster 0 osdmap e927: 8 total, 8 up, 8 in 2023-10-09T09:20:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:01 smithi181 bash[59345]: cluster 2023-10-09T09:20:01.296917+0000 osd.7 (osd.7) 381 : cluster 3 failed to encode map e927 with expected crc 2023-10-09T09:20:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[75425]: cluster 2023-10-09T09:20:00.428731+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[75425]: cluster 2023-10-09T09:20:00.655964+0000 osd.4 (osd.4) 409 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[75425]: cluster 2023-10-09T09:20:00.939083+0000 osd.7 (osd.7) 380 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:01.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[75425]: cluster 2023-10-09T09:20:01.290926+0000 osd.5 (osd.5) 468 : cluster 3 failed to encode map e927 with expected crc 2023-10-09T09:20:01.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[75425]: cluster 2023-10-09T09:20:01.291767+0000 osd.4 (osd.4) 410 : cluster 3 failed to encode map e927 with expected crc 2023-10-09T09:20:01.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[75425]: cluster 2023-10-09T09:20:01.295620+0000 mon.a (mon.0) 1137 : cluster 0 osdmap e927: 8 total, 8 up, 8 in 2023-10-09T09:20:01.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[75425]: cluster 2023-10-09T09:20:01.296917+0000 osd.7 (osd.7) 381 : cluster 3 failed to encode map e927 with expected crc 2023-10-09T09:20:01.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[78107]: cluster 2023-10-09T09:20:00.428731+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:01.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[78107]: cluster 2023-10-09T09:20:00.655964+0000 osd.4 (osd.4) 409 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:01.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[78107]: cluster 2023-10-09T09:20:00.939083+0000 osd.7 (osd.7) 380 : cluster 3 failed to encode map e926 with expected crc 2023-10-09T09:20:01.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[78107]: cluster 2023-10-09T09:20:01.290926+0000 osd.5 (osd.5) 468 : cluster 3 failed to encode map e927 with expected crc 2023-10-09T09:20:01.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[78107]: cluster 2023-10-09T09:20:01.291767+0000 osd.4 (osd.4) 410 : cluster 3 failed to encode map e927 with expected crc 2023-10-09T09:20:01.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[78107]: cluster 2023-10-09T09:20:01.295620+0000 mon.a (mon.0) 1137 : cluster 0 osdmap e927: 8 total, 8 up, 8 in 2023-10-09T09:20:01.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:01 smithi163 bash[78107]: cluster 2023-10-09T09:20:01.296917+0000 osd.7 (osd.7) 381 : cluster 3 failed to encode map e927 with expected crc 2023-10-09T09:20:02.302 INFO:tasks.rados.rados.0.smithi163.stdout:3444: done (1 left) 2023-10-09T09:20:02.669 INFO:tasks.rados.rados.0.smithi163.stdout:3445: rollback oid 47 current snap is 434 2023-10-09T09:20:02.669 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 47 to 418 2023-10-09T09:20:02.669 INFO:tasks.rados.rados.0.smithi163.stdout:3443: finishing rollback tid 1 to smithi16367185-11 2023-10-09T09:20:02.669 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 0 (ObjNum 669367424 snap 21875 seq_num 4000000) dirty dne 2023-10-09T09:20:02.669 INFO:tasks.rados.rados.0.smithi163.stdout:3443: done (1 left) 2023-10-09T09:20:02.669 INFO:tasks.rados.rados.0.smithi163.stdout:3446: delete oid 2 current snap is 434 2023-10-09T09:20:02.670 INFO:tasks.rados.rados.0.smithi163.stdout:3446: done (1 left) 2023-10-09T09:20:02.670 INFO:tasks.rados.rados.0.smithi163.stdout:3447: snap_create 2023-10-09T09:20:02.670 INFO:tasks.rados.rados.0.smithi163.stdout:3445: finishing rollback tid 1 to smithi16367185-47 2023-10-09T09:20:02.670 INFO:tasks.rados.rados.0.smithi163.stdout:3445: finishing rollback tid 2 to smithi16367185-47 2023-10-09T09:20:02.670 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 2566 (ObjNum 1224 snap 408 seq_num 1224) dirty exists 2023-10-09T09:20:02.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:02 smithi181 bash[59345]: cluster 2023-10-09T09:20:01.448402+0000 mgr.y (mgr.34107) 404 : cluster 0 pgmap v944: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 204 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2023-10-09T09:20:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:02 smithi181 bash[59345]: cluster 2023-10-09T09:20:02.298528+0000 osd.5 (osd.5) 469 : cluster 3 failed to encode map e928 with expected crc 2023-10-09T09:20:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:02 smithi181 bash[59345]: cluster 2023-10-09T09:20:02.298644+0000 osd.4 (osd.4) 411 : cluster 3 failed to encode map e928 with expected crc 2023-10-09T09:20:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:02 smithi181 bash[59345]: cluster 2023-10-09T09:20:02.302445+0000 mon.a (mon.0) 1138 : cluster 0 osdmap e928: 8 total, 8 up, 8 in 2023-10-09T09:20:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:02 smithi181 bash[59345]: cluster 2023-10-09T09:20:02.303587+0000 osd.7 (osd.7) 382 : cluster 3 failed to encode map e928 with expected crc 2023-10-09T09:20:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:02 smithi163 bash[78107]: cluster 2023-10-09T09:20:01.448402+0000 mgr.y (mgr.34107) 404 : cluster 0 pgmap v944: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 204 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2023-10-09T09:20:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:02 smithi163 bash[78107]: cluster 2023-10-09T09:20:02.298528+0000 osd.5 (osd.5) 469 : cluster 3 failed to encode map e928 with expected crc 2023-10-09T09:20:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:02 smithi163 bash[78107]: cluster 2023-10-09T09:20:02.298644+0000 osd.4 (osd.4) 411 : cluster 3 failed to encode map e928 with expected crc 2023-10-09T09:20:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:02 smithi163 bash[78107]: cluster 2023-10-09T09:20:02.302445+0000 mon.a (mon.0) 1138 : cluster 0 osdmap e928: 8 total, 8 up, 8 in 2023-10-09T09:20:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:02 smithi163 bash[78107]: cluster 2023-10-09T09:20:02.303587+0000 osd.7 (osd.7) 382 : cluster 3 failed to encode map e928 with expected crc 2023-10-09T09:20:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:02 smithi163 bash[75425]: cluster 2023-10-09T09:20:01.448402+0000 mgr.y (mgr.34107) 404 : cluster 0 pgmap v944: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 204 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2023-10-09T09:20:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:02 smithi163 bash[75425]: cluster 2023-10-09T09:20:02.298528+0000 osd.5 (osd.5) 469 : cluster 3 failed to encode map e928 with expected crc 2023-10-09T09:20:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:02 smithi163 bash[75425]: cluster 2023-10-09T09:20:02.298644+0000 osd.4 (osd.4) 411 : cluster 3 failed to encode map e928 with expected crc 2023-10-09T09:20:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:02 smithi163 bash[75425]: cluster 2023-10-09T09:20:02.302445+0000 mon.a (mon.0) 1138 : cluster 0 osdmap e928: 8 total, 8 up, 8 in 2023-10-09T09:20:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:02 smithi163 bash[75425]: cluster 2023-10-09T09:20:02.303587+0000 osd.7 (osd.7) 382 : cluster 3 failed to encode map e928 with expected crc 2023-10-09T09:20:03.306 INFO:tasks.rados.rados.0.smithi163.stdout:3445: done (1 left) 2023-10-09T09:20:03.306 INFO:tasks.rados.rados.0.smithi163.stdout:3447: done (0 left) 2023-10-09T09:20:03.306 INFO:tasks.rados.rados.0.smithi163.stdout:3448: snap_remove snap 428 2023-10-09T09:20:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:20:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:20:03] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:20:04.318 INFO:tasks.rados.rados.0.smithi163.stdout:3448: done (0 left) 2023-10-09T09:20:04.319 INFO:tasks.rados.rados.0.smithi163.stdout:3449: rollback oid 36 current snap is 435 2023-10-09T09:20:04.319 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 36 to 404 2023-10-09T09:20:04.319 INFO:tasks.rados.rados.0.smithi163.stdout:3450: snap_create 2023-10-09T09:20:04.322 INFO:tasks.rados.rados.0.smithi163.stdout:3449: finishing rollback tid 0 to smithi16367185-36 2023-10-09T09:20:04.324 INFO:tasks.rados.rados.0.smithi163.stdout:3449: finishing rollback tid 1 to smithi16367185-36 2023-10-09T09:20:04.324 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 981 (ObjNum 669367496 snap 21875 seq_num 3) dirty dne 2023-10-09T09:20:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:04 smithi181 bash[59345]: cluster 2023-10-09T09:20:03.300995+0000 osd.5 (osd.5) 470 : cluster 3 failed to encode map e929 with expected crc 2023-10-09T09:20:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:04 smithi181 bash[59345]: cluster 2023-10-09T09:20:03.301798+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e929 with expected crc 2023-10-09T09:20:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:04 smithi181 bash[59345]: cluster 2023-10-09T09:20:03.305495+0000 mon.a (mon.0) 1139 : cluster 0 osdmap e929: 8 total, 8 up, 8 in 2023-10-09T09:20:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:04 smithi181 bash[59345]: cluster 2023-10-09T09:20:03.449094+0000 mgr.y (mgr.34107) 405 : cluster 0 pgmap v947: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 781 B/s rd, 1.1 MiB/s wr, 4 op/s 2023-10-09T09:20:04.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:04 smithi181 bash[59345]: audit 2023-10-09T09:20:03.856553+0000 mon.a (mon.0) 1140 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:20:04.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:04 smithi163 bash[78107]: cluster 2023-10-09T09:20:03.300995+0000 osd.5 (osd.5) 470 : cluster 3 failed to encode map e929 with expected crc 2023-10-09T09:20:04.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:04 smithi163 bash[78107]: cluster 2023-10-09T09:20:03.301798+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e929 with expected crc 2023-10-09T09:20:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:04 smithi163 bash[78107]: cluster 2023-10-09T09:20:03.305495+0000 mon.a (mon.0) 1139 : cluster 0 osdmap e929: 8 total, 8 up, 8 in 2023-10-09T09:20:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:04 smithi163 bash[78107]: cluster 2023-10-09T09:20:03.449094+0000 mgr.y (mgr.34107) 405 : cluster 0 pgmap v947: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 781 B/s rd, 1.1 MiB/s wr, 4 op/s 2023-10-09T09:20:04.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:04 smithi163 bash[78107]: audit 2023-10-09T09:20:03.856553+0000 mon.a (mon.0) 1140 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:20:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:04 smithi163 bash[75425]: cluster 2023-10-09T09:20:03.300995+0000 osd.5 (osd.5) 470 : cluster 3 failed to encode map e929 with expected crc 2023-10-09T09:20:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:04 smithi163 bash[75425]: cluster 2023-10-09T09:20:03.301798+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e929 with expected crc 2023-10-09T09:20:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:04 smithi163 bash[75425]: cluster 2023-10-09T09:20:03.305495+0000 mon.a (mon.0) 1139 : cluster 0 osdmap e929: 8 total, 8 up, 8 in 2023-10-09T09:20:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:04 smithi163 bash[75425]: cluster 2023-10-09T09:20:03.449094+0000 mgr.y (mgr.34107) 405 : cluster 0 pgmap v947: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 781 B/s rd, 1.1 MiB/s wr, 4 op/s 2023-10-09T09:20:04.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:04 smithi163 bash[75425]: audit 2023-10-09T09:20:03.856553+0000 mon.a (mon.0) 1140 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:20:05.296 INFO:tasks.rados.rados.0.smithi163.stdout:3449: done (1 left) 2023-10-09T09:20:05.297 INFO:tasks.rados.rados.0.smithi163.stdout:3450: done (0 left) 2023-10-09T09:20:05.297 INFO:tasks.rados.rados.0.smithi163.stdout:3451: rollback oid 33 current snap is 436 2023-10-09T09:20:05.297 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 420 2023-10-09T09:20:05.297 INFO:tasks.rados.rados.0.smithi163.stdout:3452: write oid 20 current snap is 436 2023-10-09T09:20:05.297 INFO:tasks.rados.rados.0.smithi163.stdout:3452: seq_num 1281 ranges {729767=741273,1927904=725109,3145089=384678} 2023-10-09T09:20:05.307 INFO:tasks.rados.rados.0.smithi163.stdout:3452: writing smithi16367185-20 from 729767 to 1471040 tid 1 2023-10-09T09:20:05.314 INFO:tasks.rados.rados.0.smithi163.stdout:3452: writing smithi16367185-20 from 1927904 to 2653013 tid 2 2023-10-09T09:20:05.320 INFO:tasks.rados.rados.0.smithi163.stdout:3452: writing smithi16367185-20 from 3145089 to 3529767 tid 3 2023-10-09T09:20:05.320 INFO:tasks.rados.rados.0.smithi163.stdout:3453: write oid 18 current snap is 436 2023-10-09T09:20:05.320 INFO:tasks.rados.rados.0.smithi163.stdout:3451: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:20:05.320 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 0 (ObjNum 699378976 snap 21875 seq_num 699091600) dirty dne 2023-10-09T09:20:05.320 INFO:tasks.rados.rados.0.smithi163.stdout:3453: seq_num 1282 ranges {746574=416522,1586386=468767,2772512=629544} 2023-10-09T09:20:05.327 INFO:tasks.rados.rados.0.smithi163.stdout:3453: writing smithi16367185-18 from 746574 to 1163096 tid 1 2023-10-09T09:20:05.333 INFO:tasks.rados.rados.0.smithi163.stdout:3453: writing smithi16367185-18 from 1586386 to 2055153 tid 2 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:3453: writing smithi16367185-18 from 2772512 to 3402056 tid 3 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:3451: done (2 left) 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:3454: snap_create 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:3452: finishing write tid 1 to smithi16367185-20 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:3452: finishing write tid 2 to smithi16367185-20 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:3452: finishing write tid 3 to smithi16367185-20 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:3452: finishing write tid 4 to smithi16367185-20 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:3452: finishing write tid 5 to smithi16367185-20 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 670 (ObjNum 1281 snap 436 seq_num 1281) dirty exists 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:3452: left oid 20 (ObjNum 1281 snap 436 seq_num 1281) 2023-10-09T09:20:05.344 INFO:tasks.rados.rados.0.smithi163.stdout:3453: finishing write tid 1 to smithi16367185-18 2023-10-09T09:20:05.345 INFO:tasks.rados.rados.0.smithi163.stdout:3453: finishing write tid 2 to smithi16367185-18 2023-10-09T09:20:05.348 INFO:tasks.rados.rados.0.smithi163.stdout:3453: finishing write tid 3 to smithi16367185-18 2023-10-09T09:20:05.348 INFO:tasks.rados.rados.0.smithi163.stdout:3453: finishing write tid 4 to smithi16367185-18 2023-10-09T09:20:05.348 INFO:tasks.rados.rados.0.smithi163.stdout:3453: finishing write tid 5 to smithi16367185-18 2023-10-09T09:20:05.349 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1209 (ObjNum 1282 snap 436 seq_num 1282) dirty exists 2023-10-09T09:20:05.349 INFO:tasks.rados.rados.0.smithi163.stdout:3453: left oid 18 (ObjNum 1282 snap 436 seq_num 1282) 2023-10-09T09:20:05.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:05 smithi181 bash[59345]: cluster 2023-10-09T09:20:04.309334+0000 osd.5 (osd.5) 471 : cluster 3 failed to encode map e930 with expected crc 2023-10-09T09:20:05.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:05 smithi181 bash[59345]: cluster 2023-10-09T09:20:04.313705+0000 mon.a (mon.0) 1141 : cluster 0 osdmap e930: 8 total, 8 up, 8 in 2023-10-09T09:20:05.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:05 smithi181 bash[59345]: cluster 2023-10-09T09:20:04.317707+0000 osd.4 (osd.4) 412 : cluster 3 failed to encode map e929 with expected crc 2023-10-09T09:20:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:05 smithi181 bash[59345]: cluster 2023-10-09T09:20:04.348054+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e930 with expected crc 2023-10-09T09:20:05.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:05 smithi181 bash[59345]: cluster 2023-10-09T09:20:05.296210+0000 mon.a (mon.0) 1142 : cluster 0 osdmap e931: 8 total, 8 up, 8 in 2023-10-09T09:20:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:05 smithi163 bash[78107]: cluster 2023-10-09T09:20:04.309334+0000 osd.5 (osd.5) 471 : cluster 3 failed to encode map e930 with expected crc 2023-10-09T09:20:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:05 smithi163 bash[78107]: cluster 2023-10-09T09:20:04.313705+0000 mon.a (mon.0) 1141 : cluster 0 osdmap e930: 8 total, 8 up, 8 in 2023-10-09T09:20:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:05 smithi163 bash[78107]: cluster 2023-10-09T09:20:04.317707+0000 osd.4 (osd.4) 412 : cluster 3 failed to encode map e929 with expected crc 2023-10-09T09:20:05.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:05 smithi163 bash[78107]: cluster 2023-10-09T09:20:04.348054+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e930 with expected crc 2023-10-09T09:20:05.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:05 smithi163 bash[78107]: cluster 2023-10-09T09:20:05.296210+0000 mon.a (mon.0) 1142 : cluster 0 osdmap e931: 8 total, 8 up, 8 in 2023-10-09T09:20:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:05 smithi163 bash[75425]: cluster 2023-10-09T09:20:04.309334+0000 osd.5 (osd.5) 471 : cluster 3 failed to encode map e930 with expected crc 2023-10-09T09:20:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:05 smithi163 bash[75425]: cluster 2023-10-09T09:20:04.313705+0000 mon.a (mon.0) 1141 : cluster 0 osdmap e930: 8 total, 8 up, 8 in 2023-10-09T09:20:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:05 smithi163 bash[75425]: cluster 2023-10-09T09:20:04.317707+0000 osd.4 (osd.4) 412 : cluster 3 failed to encode map e929 with expected crc 2023-10-09T09:20:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:05 smithi163 bash[75425]: cluster 2023-10-09T09:20:04.348054+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e930 with expected crc 2023-10-09T09:20:05.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:05 smithi163 bash[75425]: cluster 2023-10-09T09:20:05.296210+0000 mon.a (mon.0) 1142 : cluster 0 osdmap e931: 8 total, 8 up, 8 in 2023-10-09T09:20:06.324 INFO:tasks.rados.rados.0.smithi163.stdout:3452: done (2 left) 2023-10-09T09:20:06.324 INFO:tasks.rados.rados.0.smithi163.stdout:3453: done (1 left) 2023-10-09T09:20:06.324 INFO:tasks.rados.rados.0.smithi163.stdout:3454: done (0 left) 2023-10-09T09:20:06.324 INFO:tasks.rados.rados.0.smithi163.stdout:3455: write oid 49 current snap is 437 2023-10-09T09:20:06.324 INFO:tasks.rados.rados.0.smithi163.stdout:3455: seq_num 1283 ranges {763381=491771,2044868=728778,3199935=363446} 2023-10-09T09:20:06.330 INFO:tasks.rados.rados.0.smithi163.stdout:3455: writing smithi16367185-49 from 763381 to 1255152 tid 1 2023-10-09T09:20:06.339 INFO:tasks.rados.rados.0.smithi163.stdout:3455: writing smithi16367185-49 from 2044868 to 2773646 tid 2 2023-10-09T09:20:06.344 INFO:tasks.rados.rados.0.smithi163.stdout:3455: writing smithi16367185-49 from 3199935 to 3563381 tid 3 2023-10-09T09:20:06.344 INFO:tasks.rados.rados.0.smithi163.stdout:3456: write oid 11 current snap is 437 2023-10-09T09:20:06.344 INFO:tasks.rados.rados.0.smithi163.stdout:3455: finishing write tid 1 to smithi16367185-49 2023-10-09T09:20:06.345 INFO:tasks.rados.rados.0.smithi163.stdout:3456: seq_num 1284 ranges {780188=567020,2103350=472436,3227358=352830} 2023-10-09T09:20:06.352 INFO:tasks.rados.rados.0.smithi163.stdout:3456: writing smithi16367185-11 from 780188 to 1347208 tid 1 2023-10-09T09:20:06.359 INFO:tasks.rados.rados.0.smithi163.stdout:3456: writing smithi16367185-11 from 2103350 to 2575786 tid 2 2023-10-09T09:20:06.364 INFO:tasks.rados.rados.0.smithi163.stdout:3456: writing smithi16367185-11 from 3227358 to 3580188 tid 3 2023-10-09T09:20:06.365 INFO:tasks.rados.rados.0.smithi163.stdout:3457: write oid 7 current snap is 437 2023-10-09T09:20:06.365 INFO:tasks.rados.rados.0.smithi163.stdout:3457: seq_num 1285 ranges {796995=758622,2161832=732447} 2023-10-09T09:20:06.373 INFO:tasks.rados.rados.0.smithi163.stdout:3457: writing smithi16367185-7 from 796995 to 1555617 tid 1 2023-10-09T09:20:06.381 INFO:tasks.rados.rados.0.smithi163.stdout:3457: writing smithi16367185-7 from 2161832 to 2894279 tid 2 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:3458: delete oid 28 current snap is 437 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:3455: finishing write tid 2 to smithi16367185-49 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:3455: finishing write tid 3 to smithi16367185-49 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:3455: finishing write tid 4 to smithi16367185-49 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:3455: finishing write tid 5 to smithi16367185-49 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 49 v 453 (ObjNum 1283 snap 437 seq_num 1283) dirty exists 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:3455: left oid 49 (ObjNum 1283 snap 437 seq_num 1283) 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:3456: finishing write tid 1 to smithi16367185-11 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:3456: finishing write tid 2 to smithi16367185-11 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:3456: finishing write tid 3 to smithi16367185-11 2023-10-09T09:20:06.382 INFO:tasks.rados.rados.0.smithi163.stdout:3456: finishing write tid 4 to smithi16367185-11 2023-10-09T09:20:06.383 INFO:tasks.rados.rados.0.smithi163.stdout:3456: finishing write tid 5 to smithi16367185-11 2023-10-09T09:20:06.383 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 938 (ObjNum 1284 snap 437 seq_num 1284) dirty exists 2023-10-09T09:20:06.383 INFO:tasks.rados.rados.0.smithi163.stdout:3456: left oid 11 (ObjNum 1284 snap 437 seq_num 1284) 2023-10-09T09:20:06.387 INFO:tasks.rados.rados.0.smithi163.stdout:3457: finishing write tid 1 to smithi16367185-7 2023-10-09T09:20:06.387 INFO:tasks.rados.rados.0.smithi163.stdout:3457: finishing write tid 2 to smithi16367185-7 2023-10-09T09:20:06.387 INFO:tasks.rados.rados.0.smithi163.stdout:3455: done (3 left) 2023-10-09T09:20:06.387 INFO:tasks.rados.rados.0.smithi163.stdout:3456: done (2 left) 2023-10-09T09:20:06.388 INFO:tasks.rados.rados.0.smithi163.stdout:3458: done (1 left) 2023-10-09T09:20:06.388 INFO:tasks.rados.rados.0.smithi163.stdout:3459: snap_create 2023-10-09T09:20:06.388 INFO:tasks.rados.rados.0.smithi163.stdout:3457: finishing write tid 3 to smithi16367185-7 2023-10-09T09:20:06.388 INFO:tasks.rados.rados.0.smithi163.stdout:3457: finishing write tid 4 to smithi16367185-7 2023-10-09T09:20:06.388 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 894 (ObjNum 1285 snap 437 seq_num 1285) dirty exists 2023-10-09T09:20:06.388 INFO:tasks.rados.rados.0.smithi163.stdout:3457: left oid 7 (ObjNum 1285 snap 437 seq_num 1285) 2023-10-09T09:20:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:06 smithi163 bash[78107]: cluster 2023-10-09T09:20:04.627319+0000 osd.7 (osd.7) 383 : cluster 3 failed to encode map e930 with expected crc 2023-10-09T09:20:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:06 smithi163 bash[78107]: cluster 2023-10-09T09:20:05.291067+0000 osd.5 (osd.5) 472 : cluster 3 failed to encode map e931 with expected crc 2023-10-09T09:20:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:06 smithi163 bash[78107]: cluster 2023-10-09T09:20:05.309672+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e931 with expected crc 2023-10-09T09:20:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:06 smithi163 bash[78107]: cluster 2023-10-09T09:20:05.449721+0000 mgr.y (mgr.34107) 406 : cluster 0 pgmap v950: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 2 op/s 2023-10-09T09:20:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:06 smithi163 bash[75425]: cluster 2023-10-09T09:20:04.627319+0000 osd.7 (osd.7) 383 : cluster 3 failed to encode map e930 with expected crc 2023-10-09T09:20:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:06 smithi163 bash[75425]: cluster 2023-10-09T09:20:05.291067+0000 osd.5 (osd.5) 472 : cluster 3 failed to encode map e931 with expected crc 2023-10-09T09:20:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:06 smithi163 bash[75425]: cluster 2023-10-09T09:20:05.309672+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e931 with expected crc 2023-10-09T09:20:06.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:06 smithi163 bash[75425]: cluster 2023-10-09T09:20:05.449721+0000 mgr.y (mgr.34107) 406 : cluster 0 pgmap v950: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 2 op/s 2023-10-09T09:20:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:06 smithi181 bash[59345]: cluster 2023-10-09T09:20:04.627319+0000 osd.7 (osd.7) 383 : cluster 3 failed to encode map e930 with expected crc 2023-10-09T09:20:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:06 smithi181 bash[59345]: cluster 2023-10-09T09:20:05.291067+0000 osd.5 (osd.5) 472 : cluster 3 failed to encode map e931 with expected crc 2023-10-09T09:20:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:06 smithi181 bash[59345]: cluster 2023-10-09T09:20:05.309672+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e931 with expected crc 2023-10-09T09:20:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:06 smithi181 bash[59345]: cluster 2023-10-09T09:20:05.449721+0000 mgr.y (mgr.34107) 406 : cluster 0 pgmap v950: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 2 op/s 2023-10-09T09:20:07.326 INFO:tasks.rados.rados.0.smithi163.stdout:3457: done (1 left) 2023-10-09T09:20:07.326 INFO:tasks.rados.rados.0.smithi163.stdout:3459: done (0 left) 2023-10-09T09:20:07.326 INFO:tasks.rados.rados.0.smithi163.stdout:3460: snap_remove snap 436 2023-10-09T09:20:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:07 smithi163 bash[75425]: cluster 2023-10-09T09:20:06.318288+0000 osd.5 (osd.5) 473 : cluster 3 failed to encode map e932 with expected crc 2023-10-09T09:20:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:07 smithi163 bash[75425]: cluster 2023-10-09T09:20:06.323568+0000 mon.a (mon.0) 1143 : cluster 0 osdmap e932: 8 total, 8 up, 8 in 2023-10-09T09:20:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:07 smithi163 bash[75425]: cluster 2023-10-09T09:20:06.335214+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e932 with expected crc 2023-10-09T09:20:07.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:07 smithi163 bash[75425]: cluster 2023-10-09T09:20:06.354382+0000 osd.7 (osd.7) 384 : cluster 3 failed to encode map e932 with expected crc 2023-10-09T09:20:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:07 smithi163 bash[78107]: cluster 2023-10-09T09:20:06.318288+0000 osd.5 (osd.5) 473 : cluster 3 failed to encode map e932 with expected crc 2023-10-09T09:20:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:07 smithi163 bash[78107]: cluster 2023-10-09T09:20:06.323568+0000 mon.a (mon.0) 1143 : cluster 0 osdmap e932: 8 total, 8 up, 8 in 2023-10-09T09:20:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:07 smithi163 bash[78107]: cluster 2023-10-09T09:20:06.335214+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e932 with expected crc 2023-10-09T09:20:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:07 smithi163 bash[78107]: cluster 2023-10-09T09:20:06.354382+0000 osd.7 (osd.7) 384 : cluster 3 failed to encode map e932 with expected crc 2023-10-09T09:20:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:07 smithi181 bash[59345]: cluster 2023-10-09T09:20:06.318288+0000 osd.5 (osd.5) 473 : cluster 3 failed to encode map e932 with expected crc 2023-10-09T09:20:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:07 smithi181 bash[59345]: cluster 2023-10-09T09:20:06.323568+0000 mon.a (mon.0) 1143 : cluster 0 osdmap e932: 8 total, 8 up, 8 in 2023-10-09T09:20:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:07 smithi181 bash[59345]: cluster 2023-10-09T09:20:06.335214+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e932 with expected crc 2023-10-09T09:20:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:07 smithi181 bash[59345]: cluster 2023-10-09T09:20:06.354382+0000 osd.7 (osd.7) 384 : cluster 3 failed to encode map e932 with expected crc 2023-10-09T09:20:08.329 INFO:tasks.rados.rados.0.smithi163.stdout:3460: done (0 left) 2023-10-09T09:20:08.329 INFO:tasks.rados.rados.0.smithi163.stdout:3461: snap_remove snap 404 2023-10-09T09:20:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:08 smithi163 bash[78107]: cluster 2023-10-09T09:20:07.320672+0000 osd.5 (osd.5) 474 : cluster 3 failed to encode map e933 with expected crc 2023-10-09T09:20:08.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:08 smithi163 bash[78107]: cluster 2023-10-09T09:20:07.325973+0000 mon.a (mon.0) 1144 : cluster 0 osdmap e933: 8 total, 8 up, 8 in 2023-10-09T09:20:08.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:08 smithi163 bash[78107]: cluster 2023-10-09T09:20:07.450466+0000 mgr.y (mgr.34107) 407 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 839 KiB/s wr, 6 op/s 2023-10-09T09:20:08.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:08 smithi163 bash[75425]: cluster 2023-10-09T09:20:07.320672+0000 osd.5 (osd.5) 474 : cluster 3 failed to encode map e933 with expected crc 2023-10-09T09:20:08.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:08 smithi163 bash[75425]: cluster 2023-10-09T09:20:07.325973+0000 mon.a (mon.0) 1144 : cluster 0 osdmap e933: 8 total, 8 up, 8 in 2023-10-09T09:20:08.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:08 smithi163 bash[75425]: cluster 2023-10-09T09:20:07.450466+0000 mgr.y (mgr.34107) 407 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 839 KiB/s wr, 6 op/s 2023-10-09T09:20:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:08 smithi181 bash[59345]: cluster 2023-10-09T09:20:07.320672+0000 osd.5 (osd.5) 474 : cluster 3 failed to encode map e933 with expected crc 2023-10-09T09:20:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:08 smithi181 bash[59345]: cluster 2023-10-09T09:20:07.325973+0000 mon.a (mon.0) 1144 : cluster 0 osdmap e933: 8 total, 8 up, 8 in 2023-10-09T09:20:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:08 smithi181 bash[59345]: cluster 2023-10-09T09:20:07.450466+0000 mgr.y (mgr.34107) 407 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 839 KiB/s wr, 6 op/s 2023-10-09T09:20:09.334 INFO:tasks.rados.rados.0.smithi163.stdout:3461: done (0 left) 2023-10-09T09:20:09.334 INFO:tasks.rados.rados.0.smithi163.stdout:3462: delete oid 7 current snap is 438 2023-10-09T09:20:09.358 INFO:tasks.rados.rados.0.smithi163.stdout:3462: done (0 left) 2023-10-09T09:20:09.358 INFO:tasks.rados.rados.0.smithi163.stdout:3463: write oid 7 current snap is 438 2023-10-09T09:20:09.358 INFO:tasks.rados.rados.0.smithi163.stdout:3463: seq_num 1288 ranges {447416=584369,1537278=479774,2537050=529882,3524784=122632} 2023-10-09T09:20:09.359 INFO:tasks.rados.rados.0.smithi163.stdout:3463: writing smithi16367185-7 from 447416 to 1031785 tid 1 2023-10-09T09:20:09.365 INFO:tasks.rados.rados.0.smithi163.stdout:3463: writing smithi16367185-7 from 1537278 to 2017052 tid 2 2023-10-09T09:20:09.371 INFO:tasks.rados.rados.0.smithi163.stdout:3463: writing smithi16367185-7 from 2537050 to 3066932 tid 3 2023-10-09T09:20:09.375 INFO:tasks.rados.rados.0.smithi163.stdout:3463: writing smithi16367185-7 from 3524784 to 3647416 tid 4 2023-10-09T09:20:09.375 INFO:tasks.rados.rados.0.smithi163.stdout:3464: snap_remove snap 424 2023-10-09T09:20:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:09 smithi163 bash[78107]: cluster 2023-10-09T09:20:08.323660+0000 osd.5 (osd.5) 475 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:09 smithi163 bash[78107]: cluster 2023-10-09T09:20:08.324230+0000 osd.7 (osd.7) 385 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:09.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:09 smithi163 bash[78107]: cluster 2023-10-09T09:20:08.328375+0000 mon.a (mon.0) 1145 : cluster 0 osdmap e934: 8 total, 8 up, 8 in 2023-10-09T09:20:09.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:09 smithi163 bash[78107]: cluster 2023-10-09T09:20:08.510102+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:09.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:09 smithi163 bash[78107]: cluster 2023-10-09T09:20:09.331593+0000 mon.a (mon.0) 1146 : cluster 0 osdmap e935: 8 total, 8 up, 8 in 2023-10-09T09:20:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:09 smithi163 bash[75425]: cluster 2023-10-09T09:20:08.323660+0000 osd.5 (osd.5) 475 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:09 smithi163 bash[75425]: cluster 2023-10-09T09:20:08.324230+0000 osd.7 (osd.7) 385 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:09 smithi163 bash[75425]: cluster 2023-10-09T09:20:08.328375+0000 mon.a (mon.0) 1145 : cluster 0 osdmap e934: 8 total, 8 up, 8 in 2023-10-09T09:20:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:09 smithi163 bash[75425]: cluster 2023-10-09T09:20:08.510102+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:09.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:09 smithi163 bash[75425]: cluster 2023-10-09T09:20:09.331593+0000 mon.a (mon.0) 1146 : cluster 0 osdmap e935: 8 total, 8 up, 8 in 2023-10-09T09:20:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:09 smithi181 bash[59345]: cluster 2023-10-09T09:20:08.323660+0000 osd.5 (osd.5) 475 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:09 smithi181 bash[59345]: cluster 2023-10-09T09:20:08.324230+0000 osd.7 (osd.7) 385 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:09 smithi181 bash[59345]: cluster 2023-10-09T09:20:08.328375+0000 mon.a (mon.0) 1145 : cluster 0 osdmap e934: 8 total, 8 up, 8 in 2023-10-09T09:20:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:09 smithi181 bash[59345]: cluster 2023-10-09T09:20:08.510102+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:09 smithi181 bash[59345]: cluster 2023-10-09T09:20:09.331593+0000 mon.a (mon.0) 1146 : cluster 0 osdmap e935: 8 total, 8 up, 8 in 2023-10-09T09:20:10.297 INFO:tasks.rados.rados.0.smithi163.stdout:3464: done (1 left) 2023-10-09T09:20:10.297 INFO:tasks.rados.rados.0.smithi163.stdout:3465: write oid 5 current snap is 438 2023-10-09T09:20:10.297 INFO:tasks.rados.rados.0.smithi163.stdout:3463: finishing write tid 1 to smithi16367185-7 2023-10-09T09:20:10.297 INFO:tasks.rados.rados.0.smithi163.stdout:3463: finishing write tid 2 to smithi16367185-7 2023-10-09T09:20:10.297 INFO:tasks.rados.rados.0.smithi163.stdout:3465: seq_num 1289 ranges {464223=659618,1879407=739785,3248120=416103} 2023-10-09T09:20:10.307 INFO:tasks.rados.rados.0.smithi163.stdout:3465: writing smithi16367185-5 from 464223 to 1123841 tid 1 2023-10-09T09:20:10.318 INFO:tasks.rados.rados.0.smithi163.stdout:3465: writing smithi16367185-5 from 1879407 to 2619192 tid 2 2023-10-09T09:20:10.325 INFO:tasks.rados.rados.0.smithi163.stdout:3465: writing smithi16367185-5 from 3248120 to 3664223 tid 3 2023-10-09T09:20:10.325 INFO:tasks.rados.rados.0.smithi163.stdout:3466: snap_remove snap 430 2023-10-09T09:20:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[78107]: cluster 2023-10-09T09:20:08.506303+0000 osd.4 (osd.4) 413 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[78107]: cluster 2023-10-09T09:20:09.327202+0000 osd.5 (osd.5) 476 : cluster 3 failed to encode map e935 with expected crc 2023-10-09T09:20:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[78107]: cluster 2023-10-09T09:20:09.333490+0000 osd.4 (osd.4) 414 : cluster 3 failed to encode map e935 with expected crc 2023-10-09T09:20:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[78107]: cluster 2023-10-09T09:20:09.451091+0000 mgr.y (mgr.34107) 408 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 198 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:20:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[78107]: audit 2023-10-09T09:20:09.952871+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[78107]: audit 2023-10-09T09:20:09.959969+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[78107]: audit 2023-10-09T09:20:10.050118+0000 mon.a (mon.0) 1149 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[78107]: audit 2023-10-09T09:20:10.057118+0000 mon.a (mon.0) 1150 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[78107]: cluster 2023-10-09T09:20:10.296643+0000 mon.a (mon.0) 1151 : cluster 0 osdmap e936: 8 total, 8 up, 8 in 2023-10-09T09:20:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[75425]: cluster 2023-10-09T09:20:08.506303+0000 osd.4 (osd.4) 413 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[75425]: cluster 2023-10-09T09:20:09.327202+0000 osd.5 (osd.5) 476 : cluster 3 failed to encode map e935 with expected crc 2023-10-09T09:20:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[75425]: cluster 2023-10-09T09:20:09.333490+0000 osd.4 (osd.4) 414 : cluster 3 failed to encode map e935 with expected crc 2023-10-09T09:20:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[75425]: cluster 2023-10-09T09:20:09.451091+0000 mgr.y (mgr.34107) 408 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 198 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:20:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[75425]: audit 2023-10-09T09:20:09.952871+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[75425]: audit 2023-10-09T09:20:09.959969+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[75425]: audit 2023-10-09T09:20:10.050118+0000 mon.a (mon.0) 1149 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[75425]: audit 2023-10-09T09:20:10.057118+0000 mon.a (mon.0) 1150 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:10 smithi163 bash[75425]: cluster 2023-10-09T09:20:10.296643+0000 mon.a (mon.0) 1151 : cluster 0 osdmap e936: 8 total, 8 up, 8 in 2023-10-09T09:20:10.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:10 smithi181 bash[59345]: cluster 2023-10-09T09:20:08.506303+0000 osd.4 (osd.4) 413 : cluster 3 failed to encode map e934 with expected crc 2023-10-09T09:20:10.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:10 smithi181 bash[59345]: cluster 2023-10-09T09:20:09.327202+0000 osd.5 (osd.5) 476 : cluster 3 failed to encode map e935 with expected crc 2023-10-09T09:20:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:10 smithi181 bash[59345]: cluster 2023-10-09T09:20:09.333490+0000 osd.4 (osd.4) 414 : cluster 3 failed to encode map e935 with expected crc 2023-10-09T09:20:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:10 smithi181 bash[59345]: cluster 2023-10-09T09:20:09.451091+0000 mgr.y (mgr.34107) 408 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 198 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:20:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:10 smithi181 bash[59345]: audit 2023-10-09T09:20:09.952871+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:10 smithi181 bash[59345]: audit 2023-10-09T09:20:09.959969+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:10 smithi181 bash[59345]: audit 2023-10-09T09:20:10.050118+0000 mon.a (mon.0) 1149 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:10 smithi181 bash[59345]: audit 2023-10-09T09:20:10.057118+0000 mon.a (mon.0) 1150 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:10 smithi181 bash[59345]: cluster 2023-10-09T09:20:10.296643+0000 mon.a (mon.0) 1151 : cluster 0 osdmap e936: 8 total, 8 up, 8 in 2023-10-09T09:20:11.315 INFO:tasks.rados.rados.0.smithi163.stdout:3463: finishing write tid 3 to smithi16367185-7 2023-10-09T09:20:11.315 INFO:tasks.rados.rados.0.smithi163.stdout:3463: finishing write tid 4 to smithi16367185-7 2023-10-09T09:20:11.315 INFO:tasks.rados.rados.0.smithi163.stdout:3463: finishing write tid 5 to smithi16367185-7 2023-10-09T09:20:11.315 INFO:tasks.rados.rados.0.smithi163.stdout:3463: finishing write tid 6 to smithi16367185-7 2023-10-09T09:20:11.315 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 907 (ObjNum 1288 snap 438 seq_num 1288) dirty exists 2023-10-09T09:20:11.315 INFO:tasks.rados.rados.0.smithi163.stdout:3463: left oid 7 (ObjNum 1288 snap 438 seq_num 1288) 2023-10-09T09:20:11.315 INFO:tasks.rados.rados.0.smithi163.stdout:3465: finishing write tid 1 to smithi16367185-5 2023-10-09T09:20:11.316 INFO:tasks.rados.rados.0.smithi163.stdout:3465: finishing write tid 2 to smithi16367185-5 2023-10-09T09:20:11.316 INFO:tasks.rados.rados.0.smithi163.stdout:3465: finishing write tid 3 to smithi16367185-5 2023-10-09T09:20:11.316 INFO:tasks.rados.rados.0.smithi163.stdout:3465: finishing write tid 4 to smithi16367185-5 2023-10-09T09:20:11.316 INFO:tasks.rados.rados.0.smithi163.stdout:3465: finishing write tid 5 to smithi16367185-5 2023-10-09T09:20:11.316 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1784 (ObjNum 1289 snap 438 seq_num 1289) dirty exists 2023-10-09T09:20:11.316 INFO:tasks.rados.rados.0.smithi163.stdout:3465: left oid 5 (ObjNum 1289 snap 438 seq_num 1289) 2023-10-09T09:20:11.316 INFO:tasks.rados.rados.0.smithi163.stdout:3463: done (2 left) 2023-10-09T09:20:11.316 INFO:tasks.rados.rados.0.smithi163.stdout:3465: done (1 left) 2023-10-09T09:20:11.316 INFO:tasks.rados.rados.0.smithi163.stdout:3466: done (0 left) 2023-10-09T09:20:11.316 INFO:tasks.rados.rados.0.smithi163.stdout:3467: snap_create 2023-10-09T09:20:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[78107]: cluster 2023-10-09T09:20:10.293115+0000 osd.5 (osd.5) 477 : cluster 3 failed to encode map e936 with expected crc 2023-10-09T09:20:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[78107]: cluster 2023-10-09T09:20:10.293492+0000 osd.4 (osd.4) 415 : cluster 3 failed to encode map e936 with expected crc 2023-10-09T09:20:11.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[78107]: audit 2023-10-09T09:20:10.355742+0000 mon.a (mon.0) 1152 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[78107]: audit 2023-10-09T09:20:10.556372+0000 mon.a (mon.0) 1153 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:20:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[78107]: audit 2023-10-09T09:20:10.558131+0000 mon.a (mon.0) 1154 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:20:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[78107]: audit 2023-10-09T09:20:10.567057+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[78107]: cluster 2023-10-09T09:20:11.313471+0000 mon.a (mon.0) 1156 : cluster 0 osdmap e937: 8 total, 8 up, 8 in 2023-10-09T09:20:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[75425]: cluster 2023-10-09T09:20:10.293115+0000 osd.5 (osd.5) 477 : cluster 3 failed to encode map e936 with expected crc 2023-10-09T09:20:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[75425]: cluster 2023-10-09T09:20:10.293492+0000 osd.4 (osd.4) 415 : cluster 3 failed to encode map e936 with expected crc 2023-10-09T09:20:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[75425]: audit 2023-10-09T09:20:10.355742+0000 mon.a (mon.0) 1152 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[75425]: audit 2023-10-09T09:20:10.556372+0000 mon.a (mon.0) 1153 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:20:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[75425]: audit 2023-10-09T09:20:10.558131+0000 mon.a (mon.0) 1154 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:20:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[75425]: audit 2023-10-09T09:20:10.567057+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:11.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:11 smithi163 bash[75425]: cluster 2023-10-09T09:20:11.313471+0000 mon.a (mon.0) 1156 : cluster 0 osdmap e937: 8 total, 8 up, 8 in 2023-10-09T09:20:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:11 smithi181 bash[59345]: cluster 2023-10-09T09:20:10.293115+0000 osd.5 (osd.5) 477 : cluster 3 failed to encode map e936 with expected crc 2023-10-09T09:20:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:11 smithi181 bash[59345]: cluster 2023-10-09T09:20:10.293492+0000 osd.4 (osd.4) 415 : cluster 3 failed to encode map e936 with expected crc 2023-10-09T09:20:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:11 smithi181 bash[59345]: audit 2023-10-09T09:20:10.355742+0000 mon.a (mon.0) 1152 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:11 smithi181 bash[59345]: audit 2023-10-09T09:20:10.556372+0000 mon.a (mon.0) 1153 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:20:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:11 smithi181 bash[59345]: audit 2023-10-09T09:20:10.558131+0000 mon.a (mon.0) 1154 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:20:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:11 smithi181 bash[59345]: audit 2023-10-09T09:20:10.567057+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:20:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:11 smithi181 bash[59345]: cluster 2023-10-09T09:20:11.313471+0000 mon.a (mon.0) 1156 : cluster 0 osdmap e937: 8 total, 8 up, 8 in 2023-10-09T09:20:12.320 INFO:tasks.rados.rados.0.smithi163.stdout:3467: done (0 left) 2023-10-09T09:20:12.320 INFO:tasks.rados.rados.0.smithi163.stdout:3468: read oid 25 snap -1 2023-10-09T09:20:12.320 INFO:tasks.rados.rados.0.smithi163.stdout:3468: expect (ObjNum 1279 snap 433 seq_num 1279) 2023-10-09T09:20:12.320 INFO:tasks.rados.rados.0.smithi163.stdout:3469: read oid 23 snap -1 2023-10-09T09:20:12.320 INFO:tasks.rados.rados.0.smithi163.stdout:3469: expect (ObjNum 1243 snap 419 seq_num 1243) 2023-10-09T09:20:12.320 INFO:tasks.rados.rados.0.smithi163.stdout:3470: delete oid 45 current snap is 439 2023-10-09T09:20:12.329 INFO:tasks.rados.rados.0.smithi163.stdout:3469: expect (ObjNum 1243 snap 419 seq_num 1243) 2023-10-09T09:20:12.430 INFO:tasks.rados.rados.0.smithi163.stdout:3468: expect (ObjNum 1279 snap 433 seq_num 1279) 2023-10-09T09:20:12.709 INFO:tasks.rados.rados.0.smithi163.stdout:3468: done (2 left) 2023-10-09T09:20:12.709 INFO:tasks.rados.rados.0.smithi163.stdout:3469: done (1 left) 2023-10-09T09:20:12.709 INFO:tasks.rados.rados.0.smithi163.stdout:3470: done (0 left) 2023-10-09T09:20:12.709 INFO:tasks.rados.rados.0.smithi163.stdout:3471: write oid 34 current snap is 439 2023-10-09T09:20:12.709 INFO:tasks.rados.rados.0.smithi163.stdout:3471: seq_num 1291 ranges {497837=410116,1596371=743454,2902966=763698} 2023-10-09T09:20:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:12 smithi163 bash[78107]: cluster 2023-10-09T09:20:11.308949+0000 osd.5 (osd.5) 478 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:12 smithi163 bash[78107]: cluster 2023-10-09T09:20:11.316254+0000 osd.4 (osd.4) 416 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:12 smithi163 bash[78107]: cluster 2023-10-09T09:20:11.321623+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:12 smithi163 bash[78107]: cluster 2023-10-09T09:20:11.451941+0000 mgr.y (mgr.34107) 409 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.0 MiB/s wr, 4 op/s 2023-10-09T09:20:12.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:12 smithi163 bash[78107]: cluster 2023-10-09T09:20:12.319505+0000 mon.a (mon.0) 1157 : cluster 0 osdmap e938: 8 total, 8 up, 8 in 2023-10-09T09:20:12.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:12 smithi163 bash[75425]: cluster 2023-10-09T09:20:11.308949+0000 osd.5 (osd.5) 478 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:12 smithi163 bash[75425]: cluster 2023-10-09T09:20:11.316254+0000 osd.4 (osd.4) 416 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:12 smithi163 bash[75425]: cluster 2023-10-09T09:20:11.321623+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:12 smithi163 bash[75425]: cluster 2023-10-09T09:20:11.451941+0000 mgr.y (mgr.34107) 409 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.0 MiB/s wr, 4 op/s 2023-10-09T09:20:12.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:12 smithi163 bash[75425]: cluster 2023-10-09T09:20:12.319505+0000 mon.a (mon.0) 1157 : cluster 0 osdmap e938: 8 total, 8 up, 8 in 2023-10-09T09:20:12.719 INFO:tasks.rados.rados.0.smithi163.stdout:3471: writing smithi16367185-34 from 497837 to 907953 tid 1 2023-10-09T09:20:12.727 INFO:tasks.rados.rados.0.smithi163.stdout:3471: writing smithi16367185-34 from 1596371 to 2339825 tid 2 2023-10-09T09:20:12.735 INFO:tasks.rados.rados.0.smithi163.stdout:3471: writing smithi16367185-34 from 2902966 to 3666664 tid 3 2023-10-09T09:20:12.735 INFO:tasks.rados.rados.0.smithi163.stdout:3472: read oid 6 snap -1 2023-10-09T09:20:12.735 INFO:tasks.rados.rados.0.smithi163.stdout:3472: expect (ObjNum 1273 snap 431 seq_num 1273) 2023-10-09T09:20:12.735 INFO:tasks.rados.rados.0.smithi163.stdout:3473: write oid 36 current snap is 439 2023-10-09T09:20:12.735 INFO:tasks.rados.rados.0.smithi163.stdout:3471: finishing write tid 1 to smithi16367185-34 2023-10-09T09:20:12.735 INFO:tasks.rados.rados.0.smithi163.stdout:3471: finishing write tid 2 to smithi16367185-34 2023-10-09T09:20:12.735 INFO:tasks.rados.rados.0.smithi163.stdout:3473: seq_num 1292 ranges {514644=485365,1654853=487112,2930389=574970} 2023-10-09T09:20:12.740 INFO:tasks.rados.rados.0.smithi163.stdout:3473: writing smithi16367185-36 from 514644 to 1000009 tid 1 2023-10-09T09:20:12.746 INFO:tasks.rados.rados.0.smithi163.stdout:3473: writing smithi16367185-36 from 1654853 to 2141965 tid 2 2023-10-09T09:20:12.755 INFO:tasks.rados.rados.0.smithi163.stdout:3473: writing smithi16367185-36 from 2930389 to 3505359 tid 3 2023-10-09T09:20:12.755 INFO:tasks.rados.rados.0.smithi163.stdout:3474: write oid 1 current snap is 439 2023-10-09T09:20:12.755 INFO:tasks.rados.rados.0.smithi163.stdout:3471: finishing write tid 3 to smithi16367185-34 2023-10-09T09:20:12.755 INFO:tasks.rados.rados.0.smithi163.stdout:3471: finishing write tid 4 to smithi16367185-34 2023-10-09T09:20:12.755 INFO:tasks.rados.rados.0.smithi163.stdout:3471: finishing write tid 5 to smithi16367185-34 2023-10-09T09:20:12.755 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1630 (ObjNum 1291 snap 439 seq_num 1291) dirty exists 2023-10-09T09:20:12.755 INFO:tasks.rados.rados.0.smithi163.stdout:3471: left oid 34 (ObjNum 1291 snap 439 seq_num 1291) 2023-10-09T09:20:12.755 INFO:tasks.rados.rados.0.smithi163.stdout:3473: finishing write tid 1 to smithi16367185-36 2023-10-09T09:20:12.756 INFO:tasks.rados.rados.0.smithi163.stdout:3474: seq_num 1293 ranges {531451=676967,1713335=747123,2957812=502595} 2023-10-09T09:20:12.764 INFO:tasks.rados.rados.0.smithi163.stdout:3474: writing smithi16367185-1 from 531451 to 1208418 tid 1 2023-10-09T09:20:12.771 INFO:tasks.rados.rados.0.smithi163.stdout:3474: writing smithi16367185-1 from 1713335 to 2460458 tid 2 2023-10-09T09:20:12.777 INFO:tasks.rados.rados.0.smithi163.stdout:3474: writing smithi16367185-1 from 2957812 to 3460407 tid 3 2023-10-09T09:20:12.778 INFO:tasks.rados.rados.0.smithi163.stdout:3471: done (3 left) 2023-10-09T09:20:12.778 INFO:tasks.rados.rados.0.smithi163.stdout:3475: write oid 7 current snap is 439 2023-10-09T09:20:12.778 INFO:tasks.rados.rados.0.smithi163.stdout:3473: finishing write tid 2 to smithi16367185-36 2023-10-09T09:20:12.778 INFO:tasks.rados.rados.0.smithi163.stdout:3472: expect (ObjNum 1273 snap 431 seq_num 1273) 2023-10-09T09:20:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:12 smithi181 bash[59345]: cluster 2023-10-09T09:20:11.308949+0000 osd.5 (osd.5) 478 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:12 smithi181 bash[59345]: cluster 2023-10-09T09:20:11.316254+0000 osd.4 (osd.4) 416 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:12 smithi181 bash[59345]: cluster 2023-10-09T09:20:11.321623+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:12 smithi181 bash[59345]: cluster 2023-10-09T09:20:11.451941+0000 mgr.y (mgr.34107) 409 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.0 MiB/s wr, 4 op/s 2023-10-09T09:20:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:12 smithi181 bash[59345]: cluster 2023-10-09T09:20:12.319505+0000 mon.a (mon.0) 1157 : cluster 0 osdmap e938: 8 total, 8 up, 8 in 2023-10-09T09:20:12.898 INFO:tasks.rados.rados.0.smithi163.stdout:3473: finishing write tid 3 to smithi16367185-36 2023-10-09T09:20:12.898 INFO:tasks.rados.rados.0.smithi163.stdout:3473: finishing write tid 4 to smithi16367185-36 2023-10-09T09:20:12.898 INFO:tasks.rados.rados.0.smithi163.stdout:3473: finishing write tid 5 to smithi16367185-36 2023-10-09T09:20:12.898 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 997 (ObjNum 1292 snap 439 seq_num 1292) dirty exists 2023-10-09T09:20:12.898 INFO:tasks.rados.rados.0.smithi163.stdout:3473: left oid 36 (ObjNum 1292 snap 439 seq_num 1292) 2023-10-09T09:20:12.899 INFO:tasks.rados.rados.0.smithi163.stdout:3474: finishing write tid 1 to smithi16367185-1 2023-10-09T09:20:12.899 INFO:tasks.rados.rados.0.smithi163.stdout:3474: finishing write tid 2 to smithi16367185-1 2023-10-09T09:20:12.899 INFO:tasks.rados.rados.0.smithi163.stdout:3474: finishing write tid 3 to smithi16367185-1 2023-10-09T09:20:12.899 INFO:tasks.rados.rados.0.smithi163.stdout:3474: finishing write tid 4 to smithi16367185-1 2023-10-09T09:20:12.899 INFO:tasks.rados.rados.0.smithi163.stdout:3474: finishing write tid 5 to smithi16367185-1 2023-10-09T09:20:12.899 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1793 (ObjNum 1293 snap 439 seq_num 1293) dirty exists 2023-10-09T09:20:12.899 INFO:tasks.rados.rados.0.smithi163.stdout:3474: left oid 1 (ObjNum 1293 snap 439 seq_num 1293) 2023-10-09T09:20:12.899 INFO:tasks.rados.rados.0.smithi163.stdout:3475: seq_num 1294 ranges {548258=752216,1771817=490781,2985235=713867} 2023-10-09T09:20:12.911 INFO:tasks.rados.rados.0.smithi163.stdout:3475: writing smithi16367185-7 from 548258 to 1300474 tid 1 2023-10-09T09:20:12.918 INFO:tasks.rados.rados.0.smithi163.stdout:3475: writing smithi16367185-7 from 1771817 to 2262598 tid 2 2023-10-09T09:20:12.926 INFO:tasks.rados.rados.0.smithi163.stdout:3475: writing smithi16367185-7 from 2985235 to 3699102 tid 3 2023-10-09T09:20:12.926 INFO:tasks.rados.rados.0.smithi163.stdout:3472: done (3 left) 2023-10-09T09:20:12.926 INFO:tasks.rados.rados.0.smithi163.stdout:3473: done (2 left) 2023-10-09T09:20:12.926 INFO:tasks.rados.rados.0.smithi163.stdout:3474: done (1 left) 2023-10-09T09:20:12.926 INFO:tasks.rados.rados.0.smithi163.stdout:3476: snap_remove snap 433 2023-10-09T09:20:13.378 INFO:tasks.rados.rados.0.smithi163.stdout:3476: done (1 left) 2023-10-09T09:20:13.379 INFO:tasks.rados.rados.0.smithi163.stdout:3477: read oid 38 snap -1 2023-10-09T09:20:13.379 INFO:tasks.rados.rados.0.smithi163.stdout:3477: expect (ObjNum 1254 snap 422 seq_num 1254) 2023-10-09T09:20:13.379 INFO:tasks.rados.rados.0.smithi163.stdout:3478: read oid 17 snap -1 2023-10-09T09:20:13.379 INFO:tasks.rados.rados.0.smithi163.stdout:3478: expect (ObjNum 1222 snap 408 seq_num 1222) 2023-10-09T09:20:13.379 INFO:tasks.rados.rados.0.smithi163.stdout:3475: finishing write tid 1 to smithi16367185-7 2023-10-09T09:20:13.379 INFO:tasks.rados.rados.0.smithi163.stdout:3475: finishing write tid 2 to smithi16367185-7 2023-10-09T09:20:13.380 INFO:tasks.rados.rados.0.smithi163.stdout:3475: finishing write tid 3 to smithi16367185-7 2023-10-09T09:20:13.380 INFO:tasks.rados.rados.0.smithi163.stdout:3475: finishing write tid 4 to smithi16367185-7 2023-10-09T09:20:13.380 INFO:tasks.rados.rados.0.smithi163.stdout:3475: finishing write tid 5 to smithi16367185-7 2023-10-09T09:20:13.380 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 916 (ObjNum 1294 snap 439 seq_num 1294) dirty exists 2023-10-09T09:20:13.380 INFO:tasks.rados.rados.0.smithi163.stdout:3475: left oid 7 (ObjNum 1294 snap 439 seq_num 1294) 2023-10-09T09:20:13.380 INFO:tasks.rados.rados.0.smithi163.stdout:3475: done (2 left) 2023-10-09T09:20:13.380 INFO:tasks.rados.rados.0.smithi163.stdout:3479: read oid 23 snap -1 2023-10-09T09:20:13.381 INFO:tasks.rados.rados.0.smithi163.stdout:3479: expect (ObjNum 1243 snap 419 seq_num 1243) 2023-10-09T09:20:13.381 INFO:tasks.rados.rados.0.smithi163.stdout:3480: rollback oid 45 current snap is 439 2023-10-09T09:20:13.381 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 45 to 425 2023-10-09T09:20:13.381 INFO:tasks.rados.rados.0.smithi163.stdout:3481: read oid 16 snap -1 2023-10-09T09:20:13.381 INFO:tasks.rados.rados.0.smithi163.stdout:3481: expect (ObjNum 1276 snap 432 seq_num 1276) 2023-10-09T09:20:13.382 INFO:tasks.rados.rados.0.smithi163.stdout:3482: write oid 37 current snap is 439 2023-10-09T09:20:13.382 INFO:tasks.rados.rados.0.smithi163.stdout:3482: seq_num 1295 ranges {565065=427465,1430299=634439,2496305=525139,3472104=292961} 2023-10-09T09:20:13.385 INFO:tasks.rados.rados.0.smithi163.stdout:3482: writing smithi16367185-37 from 565065 to 992530 tid 1 2023-10-09T09:20:13.392 INFO:tasks.rados.rados.0.smithi163.stdout:3482: writing smithi16367185-37 from 1430299 to 2064738 tid 2 2023-10-09T09:20:13.396 INFO:tasks.rados.rados.0.smithi163.stdout:3482: writing smithi16367185-37 from 2496305 to 3021444 tid 3 2023-10-09T09:20:13.401 INFO:tasks.rados.rados.0.smithi163.stdout:3482: writing smithi16367185-37 from 3472104 to 3765065 tid 4 2023-10-09T09:20:13.402 INFO:tasks.rados.rados.0.smithi163.stdout:3483: delete oid 46 current snap is 439 2023-10-09T09:20:13.402 INFO:tasks.rados.rados.0.smithi163.stdout:3481: expect (ObjNum 1276 snap 432 seq_num 1276) 2023-10-09T09:20:13.521 INFO:tasks.rados.rados.0.smithi163.stdout:3479: expect (ObjNum 1243 snap 419 seq_num 1243) 2023-10-09T09:20:13.625 INFO:tasks.rados.rados.0.smithi163.stdout:3480: finishing rollback tid 1 to smithi16367185-45 2023-10-09T09:20:13.625 INFO:tasks.rados.rados.0.smithi163.stdout:3480: finishing rollback tid 2 to smithi16367185-45 2023-10-09T09:20:13.625 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 2602 (ObjNum 1249 snap 420 seq_num 1249) dirty exists 2023-10-09T09:20:13.625 INFO:tasks.rados.rados.0.smithi163.stdout:3478: expect (ObjNum 1222 snap 408 seq_num 1222) 2023-10-09T09:20:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:20:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:20:13] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:20:13.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:13 smithi163 bash[78107]: cluster 2023-10-09T09:20:11.967383+0000 osd.7 (osd.7) 386 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:13.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:13 smithi163 bash[78107]: cluster 2023-10-09T09:20:12.313488+0000 osd.5 (osd.5) 479 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:13 smithi163 bash[78107]: cluster 2023-10-09T09:20:12.313966+0000 osd.4 (osd.4) 417 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:13 smithi163 bash[78107]: cluster 2023-10-09T09:20:12.477254+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:13 smithi163 bash[75425]: cluster 2023-10-09T09:20:11.967383+0000 osd.7 (osd.7) 386 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:13 smithi163 bash[75425]: cluster 2023-10-09T09:20:12.313488+0000 osd.5 (osd.5) 479 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:13 smithi163 bash[75425]: cluster 2023-10-09T09:20:12.313966+0000 osd.4 (osd.4) 417 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:13 smithi163 bash[75425]: cluster 2023-10-09T09:20:12.477254+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:13.746 INFO:tasks.rados.rados.0.smithi163.stdout:3482: finishing write tid 1 to smithi16367185-37 2023-10-09T09:20:13.747 INFO:tasks.rados.rados.0.smithi163.stdout:3482: finishing write tid 2 to smithi16367185-37 2023-10-09T09:20:13.747 INFO:tasks.rados.rados.0.smithi163.stdout:3478: done (6 left) 2023-10-09T09:20:13.747 INFO:tasks.rados.rados.0.smithi163.stdout:3479: done (5 left) 2023-10-09T09:20:13.747 INFO:tasks.rados.rados.0.smithi163.stdout:3480: done (4 left) 2023-10-09T09:20:13.747 INFO:tasks.rados.rados.0.smithi163.stdout:3481: done (3 left) 2023-10-09T09:20:13.747 INFO:tasks.rados.rados.0.smithi163.stdout:3483: done (2 left) 2023-10-09T09:20:13.747 INFO:tasks.rados.rados.0.smithi163.stdout:3484: read oid 32 snap -1 2023-10-09T09:20:13.747 INFO:tasks.rados.rados.0.smithi163.stdout:3484: expect (ObjNum 1218 snap 404 seq_num 1218) 2023-10-09T09:20:13.747 INFO:tasks.rados.rados.0.smithi163.stdout:3485: write oid 20 current snap is 439 2023-10-09T09:20:13.747 INFO:tasks.rados.rados.0.smithi163.stdout:3485: seq_num 1297 ranges {598679=577963,1947263=638108,3351151=447528} 2023-10-09T09:20:13.758 INFO:tasks.rados.rados.0.smithi163.stdout:3485: writing smithi16367185-20 from 598679 to 1176642 tid 1 2023-10-09T09:20:13.767 INFO:tasks.rados.rados.0.smithi163.stdout:3485: writing smithi16367185-20 from 1947263 to 2585371 tid 2 2023-10-09T09:20:13.774 INFO:tasks.rados.rados.0.smithi163.stdout:3485: writing smithi16367185-20 from 3351151 to 3798679 tid 3 2023-10-09T09:20:13.774 INFO:tasks.rados.rados.0.smithi163.stdout:3486: write oid 41 current snap is 439 2023-10-09T09:20:13.774 INFO:tasks.rados.rados.0.smithi163.stdout:3486: seq_num 1298 ranges {615486=653212,1889392=498119,2978574=475308} 2023-10-09T09:20:13.781 INFO:tasks.rados.rados.0.smithi163.stdout:3486: writing smithi16367185-41 from 615486 to 1268698 tid 1 2023-10-09T09:20:13.787 INFO:tasks.rados.rados.0.smithi163.stdout:3486: writing smithi16367185-41 from 1889392 to 2387511 tid 2 2023-10-09T09:20:13.793 INFO:tasks.rados.rados.0.smithi163.stdout:3486: writing smithi16367185-41 from 2978574 to 3453882 tid 3 2023-10-09T09:20:13.794 INFO:tasks.rados.rados.0.smithi163.stdout:3487: write oid 2 current snap is 439 2023-10-09T09:20:13.794 INFO:tasks.rados.rados.0.smithi163.stdout:3487: seq_num 1299 ranges {632293=728461,1947874=641777,3005997=686580} 2023-10-09T09:20:13.801 INFO:tasks.rados.rados.0.smithi163.stdout:3487: writing smithi16367185-2 from 632293 to 1360754 tid 1 2023-10-09T09:20:13.808 INFO:tasks.rados.rados.0.smithi163.stdout:3487: writing smithi16367185-2 from 1947874 to 2589651 tid 2 2023-10-09T09:20:13.815 INFO:tasks.rados.rados.0.smithi163.stdout:3487: writing smithi16367185-2 from 3005997 to 3692577 tid 3 2023-10-09T09:20:13.815 INFO:tasks.rados.rados.0.smithi163.stdout:3488: read oid 36 snap -1 2023-10-09T09:20:13.816 INFO:tasks.rados.rados.0.smithi163.stdout:3488: expect (ObjNum 1292 snap 439 seq_num 1292) 2023-10-09T09:20:13.816 INFO:tasks.rados.rados.0.smithi163.stdout:3489: snap_create 2023-10-09T09:20:13.816 INFO:tasks.rados.rados.0.smithi163.stdout:3482: finishing write tid 3 to smithi16367185-37 2023-10-09T09:20:13.816 INFO:tasks.rados.rados.0.smithi163.stdout:3482: finishing write tid 4 to smithi16367185-37 2023-10-09T09:20:13.816 INFO:tasks.rados.rados.0.smithi163.stdout:3482: finishing write tid 5 to smithi16367185-37 2023-10-09T09:20:13.816 INFO:tasks.rados.rados.0.smithi163.stdout:3482: finishing write tid 6 to smithi16367185-37 2023-10-09T09:20:13.816 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 1768 (ObjNum 1295 snap 439 seq_num 1295) dirty exists 2023-10-09T09:20:13.817 INFO:tasks.rados.rados.0.smithi163.stdout:3482: left oid 37 (ObjNum 1295 snap 439 seq_num 1295) 2023-10-09T09:20:13.817 INFO:tasks.rados.rados.0.smithi163.stdout:3477: expect (ObjNum 1254 snap 422 seq_num 1254) 2023-10-09T09:20:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:13 smithi181 bash[59345]: cluster 2023-10-09T09:20:11.967383+0000 osd.7 (osd.7) 386 : cluster 3 failed to encode map e937 with expected crc 2023-10-09T09:20:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:13 smithi181 bash[59345]: cluster 2023-10-09T09:20:12.313488+0000 osd.5 (osd.5) 479 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:13 smithi181 bash[59345]: cluster 2023-10-09T09:20:12.313966+0000 osd.4 (osd.4) 417 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:13.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:13 smithi181 bash[59345]: cluster 2023-10-09T09:20:12.477254+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:13.930 INFO:tasks.rados.rados.0.smithi163.stdout:3484: expect (ObjNum 1218 snap 404 seq_num 1218) 2023-10-09T09:20:14.063 INFO:tasks.rados.rados.0.smithi163.stdout:3485: finishing write tid 1 to smithi16367185-20 2023-10-09T09:20:14.063 INFO:tasks.rados.rados.0.smithi163.stdout:3485: finishing write tid 2 to smithi16367185-20 2023-10-09T09:20:14.063 INFO:tasks.rados.rados.0.smithi163.stdout:3485: finishing write tid 3 to smithi16367185-20 2023-10-09T09:20:14.063 INFO:tasks.rados.rados.0.smithi163.stdout:3485: finishing write tid 4 to smithi16367185-20 2023-10-09T09:20:14.063 INFO:tasks.rados.rados.0.smithi163.stdout:3485: finishing write tid 5 to smithi16367185-20 2023-10-09T09:20:14.063 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 683 (ObjNum 1297 snap 439 seq_num 1297) dirty exists 2023-10-09T09:20:14.063 INFO:tasks.rados.rados.0.smithi163.stdout:3485: left oid 20 (ObjNum 1297 snap 439 seq_num 1297) 2023-10-09T09:20:14.064 INFO:tasks.rados.rados.0.smithi163.stdout:3486: finishing write tid 1 to smithi16367185-41 2023-10-09T09:20:14.064 INFO:tasks.rados.rados.0.smithi163.stdout:3486: finishing write tid 2 to smithi16367185-41 2023-10-09T09:20:14.064 INFO:tasks.rados.rados.0.smithi163.stdout:3486: finishing write tid 3 to smithi16367185-41 2023-10-09T09:20:14.064 INFO:tasks.rados.rados.0.smithi163.stdout:3486: finishing write tid 4 to smithi16367185-41 2023-10-09T09:20:14.064 INFO:tasks.rados.rados.0.smithi163.stdout:3486: finishing write tid 5 to smithi16367185-41 2023-10-09T09:20:14.064 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 315 (ObjNum 1298 snap 439 seq_num 1298) dirty exists 2023-10-09T09:20:14.064 INFO:tasks.rados.rados.0.smithi163.stdout:3486: left oid 41 (ObjNum 1298 snap 439 seq_num 1298) 2023-10-09T09:20:14.064 INFO:tasks.rados.rados.0.smithi163.stdout:3487: finishing write tid 1 to smithi16367185-2 2023-10-09T09:20:14.064 INFO:tasks.rados.rados.0.smithi163.stdout:3487: finishing write tid 2 to smithi16367185-2 2023-10-09T09:20:14.064 INFO:tasks.rados.rados.0.smithi163.stdout:3487: finishing write tid 3 to smithi16367185-2 2023-10-09T09:20:14.065 INFO:tasks.rados.rados.0.smithi163.stdout:3487: finishing write tid 4 to smithi16367185-2 2023-10-09T09:20:14.065 INFO:tasks.rados.rados.0.smithi163.stdout:3487: finishing write tid 5 to smithi16367185-2 2023-10-09T09:20:14.065 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 946 (ObjNum 1299 snap 439 seq_num 1299) dirty exists 2023-10-09T09:20:14.065 INFO:tasks.rados.rados.0.smithi163.stdout:3487: left oid 2 (ObjNum 1299 snap 439 seq_num 1299) 2023-10-09T09:20:14.065 INFO:tasks.rados.rados.0.smithi163.stdout:3488: expect (ObjNum 1292 snap 439 seq_num 1292) 2023-10-09T09:20:14.386 INFO:tasks.rados.rados.0.smithi163.stdout:3477: done (7 left) 2023-10-09T09:20:14.386 INFO:tasks.rados.rados.0.smithi163.stdout:3482: done (6 left) 2023-10-09T09:20:14.386 INFO:tasks.rados.rados.0.smithi163.stdout:3484: done (5 left) 2023-10-09T09:20:14.386 INFO:tasks.rados.rados.0.smithi163.stdout:3485: done (4 left) 2023-10-09T09:20:14.386 INFO:tasks.rados.rados.0.smithi163.stdout:3486: done (3 left) 2023-10-09T09:20:14.386 INFO:tasks.rados.rados.0.smithi163.stdout:3487: done (2 left) 2023-10-09T09:20:14.386 INFO:tasks.rados.rados.0.smithi163.stdout:3488: done (1 left) 2023-10-09T09:20:14.386 INFO:tasks.rados.rados.0.smithi163.stdout:3489: done (0 left) 2023-10-09T09:20:14.387 INFO:tasks.rados.rados.0.smithi163.stdout:3490: write oid 42 current snap is 440 2023-10-09T09:20:14.387 INFO:tasks.rados.rados.0.smithi163.stdout:3490: seq_num 1300 ranges {649100=403710,1606356=501788,2633420=497852,3636005=213095} 2023-10-09T09:20:14.394 INFO:tasks.rados.rados.0.smithi163.stdout:3490: writing smithi16367185-42 from 649100 to 1052810 tid 1 2023-10-09T09:20:14.401 INFO:tasks.rados.rados.0.smithi163.stdout:3490: writing smithi16367185-42 from 1606356 to 2108144 tid 2 2023-10-09T09:20:14.407 INFO:tasks.rados.rados.0.smithi163.stdout:3490: writing smithi16367185-42 from 2633420 to 3131272 tid 3 2023-10-09T09:20:14.411 INFO:tasks.rados.rados.0.smithi163.stdout:3490: writing smithi16367185-42 from 3636005 to 3849100 tid 4 2023-10-09T09:20:14.411 INFO:tasks.rados.rados.0.smithi163.stdout:3491: read oid 26 snap -1 2023-10-09T09:20:14.411 INFO:tasks.rados.rados.0.smithi163.stdout:3491: expect deleted 2023-10-09T09:20:14.412 INFO:tasks.rados.rados.0.smithi163.stdout:3492: write oid 2 current snap is 440 2023-10-09T09:20:14.412 INFO:tasks.rados.rados.0.smithi163.stdout:3490: finishing write tid 1 to smithi16367185-42 2023-10-09T09:20:14.412 INFO:tasks.rados.rados.0.smithi163.stdout:3492: seq_num 1301 ranges {665907=595312,1781191=645446,3177196=688711} 2023-10-09T09:20:14.418 INFO:tasks.rados.rados.0.smithi163.stdout:3492: writing smithi16367185-2 from 665907 to 1261219 tid 1 2023-10-09T09:20:14.425 INFO:tasks.rados.rados.0.smithi163.stdout:3492: writing smithi16367185-2 from 1781191 to 2426637 tid 2 2023-10-09T09:20:14.433 INFO:tasks.rados.rados.0.smithi163.stdout:3492: writing smithi16367185-2 from 3177196 to 3865907 tid 3 2023-10-09T09:20:14.433 INFO:tasks.rados.rados.0.smithi163.stdout:3493: write oid 36 current snap is 440 2023-10-09T09:20:14.434 INFO:tasks.rados.rados.0.smithi163.stdout:3490: finishing write tid 2 to smithi16367185-42 2023-10-09T09:20:14.434 INFO:tasks.rados.rados.0.smithi163.stdout:3490: finishing write tid 3 to smithi16367185-42 2023-10-09T09:20:14.434 INFO:tasks.rados.rados.0.smithi163.stdout:3493: seq_num 1302 ranges {682714=670561,2123320=505457,3088266=636749} 2023-10-09T09:20:14.440 INFO:tasks.rados.rados.0.smithi163.stdout:3493: writing smithi16367185-36 from 682714 to 1353275 tid 1 2023-10-09T09:20:14.448 INFO:tasks.rados.rados.0.smithi163.stdout:3493: writing smithi16367185-36 from 2123320 to 2628777 tid 2 2023-10-09T09:20:14.455 INFO:tasks.rados.rados.0.smithi163.stdout:3493: writing smithi16367185-36 from 3088266 to 3725015 tid 3 2023-10-09T09:20:14.455 INFO:tasks.rados.rados.0.smithi163.stdout:3490: finishing write tid 4 to smithi16367185-42 2023-10-09T09:20:14.455 INFO:tasks.rados.rados.0.smithi163.stdout:3490: finishing write tid 5 to smithi16367185-42 2023-10-09T09:20:14.455 INFO:tasks.rados.rados.0.smithi163.stdout:3494: read oid 14 snap -1 2023-10-09T09:20:14.455 INFO:tasks.rados.rados.0.smithi163.stdout:3494: expect (ObjNum 1166 snap 392 seq_num 1166) 2023-10-09T09:20:14.455 INFO:tasks.rados.rados.0.smithi163.stdout:3491: done (4 left) 2023-10-09T09:20:14.455 INFO:tasks.rados.rados.0.smithi163.stdout:3495: rollback oid 28 current snap is 440 2023-10-09T09:20:14.455 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 434 2023-10-09T09:20:14.455 INFO:tasks.rados.rados.0.smithi163.stdout:3490: finishing write tid 6 to smithi16367185-42 2023-10-09T09:20:14.456 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 2610 (ObjNum 1300 snap 440 seq_num 1300) dirty exists 2023-10-09T09:20:14.456 INFO:tasks.rados.rados.0.smithi163.stdout:3490: left oid 42 (ObjNum 1300 snap 440 seq_num 1300) 2023-10-09T09:20:14.456 INFO:tasks.rados.rados.0.smithi163.stdout:3492: finishing write tid 1 to smithi16367185-2 2023-10-09T09:20:14.456 INFO:tasks.rados.rados.0.smithi163.stdout:3492: finishing write tid 2 to smithi16367185-2 2023-10-09T09:20:14.456 INFO:tasks.rados.rados.0.smithi163.stdout:3490: done (4 left) 2023-10-09T09:20:14.456 INFO:tasks.rados.rados.0.smithi163.stdout:3496: read oid 31 snap -1 2023-10-09T09:20:14.456 INFO:tasks.rados.rados.0.smithi163.stdout:3496: expect deleted 2023-10-09T09:20:14.456 INFO:tasks.rados.rados.0.smithi163.stdout:3497: write oid 15 current snap is 440 2023-10-09T09:20:14.456 INFO:tasks.rados.rados.0.smithi163.stdout:3492: finishing write tid 3 to smithi16367185-2 2023-10-09T09:20:14.457 INFO:tasks.rados.rados.0.smithi163.stdout:3497: seq_num 1303 ranges {699521=745810,2181802=649115,3515689=383832} 2023-10-09T09:20:14.463 INFO:tasks.rados.rados.0.smithi163.stdout:3497: writing smithi16367185-15 from 699521 to 1445331 tid 1 2023-10-09T09:20:14.471 INFO:tasks.rados.rados.0.smithi163.stdout:3497: writing smithi16367185-15 from 2181802 to 2830917 tid 2 2023-10-09T09:20:14.478 INFO:tasks.rados.rados.0.smithi163.stdout:3497: writing smithi16367185-15 from 3515689 to 3899521 tid 3 2023-10-09T09:20:14.478 INFO:tasks.rados.rados.0.smithi163.stdout:3492: finishing write tid 4 to smithi16367185-2 2023-10-09T09:20:14.478 INFO:tasks.rados.rados.0.smithi163.stdout:3492: finishing write tid 5 to smithi16367185-2 2023-10-09T09:20:14.478 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 951 (ObjNum 1301 snap 440 seq_num 1301) dirty exists 2023-10-09T09:20:14.478 INFO:tasks.rados.rados.0.smithi163.stdout:3492: left oid 2 (ObjNum 1301 snap 440 seq_num 1301) 2023-10-09T09:20:14.478 INFO:tasks.rados.rados.0.smithi163.stdout:3493: finishing write tid 1 to smithi16367185-36 2023-10-09T09:20:14.478 INFO:tasks.rados.rados.0.smithi163.stdout:3498: read oid 20 snap -1 2023-10-09T09:20:14.478 INFO:tasks.rados.rados.0.smithi163.stdout:3498: expect (ObjNum 1297 snap 439 seq_num 1297) 2023-10-09T09:20:14.478 INFO:tasks.rados.rados.0.smithi163.stdout:3492: done (6 left) 2023-10-09T09:20:14.479 INFO:tasks.rados.rados.0.smithi163.stdout:3499: snap_remove snap 437 2023-10-09T09:20:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[75425]: cluster 2023-10-09T09:20:12.647067+0000 osd.7 (osd.7) 387 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[75425]: cluster 2023-10-09T09:20:13.372627+0000 osd.5 (osd.5) 480 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[75425]: cluster 2023-10-09T09:20:13.375758+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e939: 8 total, 8 up, 8 in 2023-10-09T09:20:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[75425]: cluster 2023-10-09T09:20:13.378440+0000 osd.7 (osd.7) 388 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[75425]: cluster 2023-10-09T09:20:13.380748+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[75425]: cluster 2023-10-09T09:20:13.452612+0000 mgr.y (mgr.34107) 410 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 191 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 710 KiB/s wr, 3 op/s 2023-10-09T09:20:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[78107]: cluster 2023-10-09T09:20:12.647067+0000 osd.7 (osd.7) 387 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[78107]: cluster 2023-10-09T09:20:13.372627+0000 osd.5 (osd.5) 480 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[78107]: cluster 2023-10-09T09:20:13.375758+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e939: 8 total, 8 up, 8 in 2023-10-09T09:20:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[78107]: cluster 2023-10-09T09:20:13.378440+0000 osd.7 (osd.7) 388 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[78107]: cluster 2023-10-09T09:20:13.380748+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:14.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:14 smithi163 bash[78107]: cluster 2023-10-09T09:20:13.452612+0000 mgr.y (mgr.34107) 410 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 191 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 710 KiB/s wr, 3 op/s 2023-10-09T09:20:14.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:14 smithi181 bash[59345]: cluster 2023-10-09T09:20:12.647067+0000 osd.7 (osd.7) 387 : cluster 3 failed to encode map e938 with expected crc 2023-10-09T09:20:14.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:14 smithi181 bash[59345]: cluster 2023-10-09T09:20:13.372627+0000 osd.5 (osd.5) 480 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:14 smithi181 bash[59345]: cluster 2023-10-09T09:20:13.375758+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e939: 8 total, 8 up, 8 in 2023-10-09T09:20:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:14 smithi181 bash[59345]: cluster 2023-10-09T09:20:13.378440+0000 osd.7 (osd.7) 388 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:14 smithi181 bash[59345]: cluster 2023-10-09T09:20:13.380748+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:14 smithi181 bash[59345]: cluster 2023-10-09T09:20:13.452612+0000 mgr.y (mgr.34107) 410 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 191 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 710 KiB/s wr, 3 op/s 2023-10-09T09:20:15.301 INFO:tasks.rados.rados.0.smithi163.stdout:3499: done (6 left) 2023-10-09T09:20:15.301 INFO:tasks.rados.rados.0.smithi163.stdout:3500: write oid 25 current snap is 440 2023-10-09T09:20:15.301 INFO:tasks.rados.rados.0.smithi163.stdout:3500: seq_num 1304 ranges {716328=421059,1840284=509126,3143112=659293} 2023-10-09T09:20:15.311 INFO:tasks.rados.rados.0.smithi163.stdout:3500: writing smithi16367185-25 from 716328 to 1137387 tid 1 2023-10-09T09:20:15.318 INFO:tasks.rados.rados.0.smithi163.stdout:3500: writing smithi16367185-25 from 1840284 to 2349410 tid 2 2023-10-09T09:20:15.327 INFO:tasks.rados.rados.0.smithi163.stdout:3500: writing smithi16367185-25 from 3143112 to 3802405 tid 3 2023-10-09T09:20:15.327 INFO:tasks.rados.rados.0.smithi163.stdout:3501: write oid 22 current snap is 440 2023-10-09T09:20:15.327 INFO:tasks.rados.rados.0.smithi163.stdout:3501: seq_num 1305 ranges {733135=496308,1898766=652784,3170535=470565} 2023-10-09T09:20:15.334 INFO:tasks.rados.rados.0.smithi163.stdout:3501: writing smithi16367185-22 from 733135 to 1229443 tid 1 2023-10-09T09:20:15.341 INFO:tasks.rados.rados.0.smithi163.stdout:3501: writing smithi16367185-22 from 1898766 to 2551550 tid 2 2023-10-09T09:20:15.348 INFO:tasks.rados.rados.0.smithi163.stdout:3501: writing smithi16367185-22 from 3170535 to 3641100 tid 3 2023-10-09T09:20:15.348 INFO:tasks.rados.rados.0.smithi163.stdout:3502: write oid 41 current snap is 440 2023-10-09T09:20:15.348 INFO:tasks.rados.rados.0.smithi163.stdout:3493: finishing write tid 2 to smithi16367185-36 2023-10-09T09:20:15.348 INFO:tasks.rados.rados.0.smithi163.stdout:3502: seq_num 1306 ranges {749942=571557,1840895=796442,3365252=584690} 2023-10-09T09:20:15.355 INFO:tasks.rados.rados.0.smithi163.stdout:3502: writing smithi16367185-41 from 749942 to 1321499 tid 1 2023-10-09T09:20:15.363 INFO:tasks.rados.rados.0.smithi163.stdout:3502: writing smithi16367185-41 from 1840895 to 2637337 tid 2 2023-10-09T09:20:15.371 INFO:tasks.rados.rados.0.smithi163.stdout:3502: writing smithi16367185-41 from 3365252 to 3949942 tid 3 2023-10-09T09:20:15.371 INFO:tasks.rados.rados.0.smithi163.stdout:3503: write oid 13 current snap is 440 2023-10-09T09:20:15.371 INFO:tasks.rados.rados.0.smithi163.stdout:3503: seq_num 1307 ranges {766749=646806,1899377=656453,3109028=609462} 2023-10-09T09:20:15.379 INFO:tasks.rados.rados.0.smithi163.stdout:3503: writing smithi16367185-13 from 766749 to 1413555 tid 1 2023-10-09T09:20:15.386 INFO:tasks.rados.rados.0.smithi163.stdout:3503: writing smithi16367185-13 from 1899377 to 2555830 tid 2 2023-10-09T09:20:15.393 INFO:tasks.rados.rados.0.smithi163.stdout:3503: writing smithi16367185-13 from 3109028 to 3718490 tid 3 2023-10-09T09:20:15.393 INFO:tasks.rados.rados.0.smithi163.stdout:3504: rollback oid 24 current snap is 440 2023-10-09T09:20:15.393 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 24 to 409 2023-10-09T09:20:15.394 INFO:tasks.rados.rados.0.smithi163.stdout:3493: finishing write tid 3 to smithi16367185-36 2023-10-09T09:20:15.394 INFO:tasks.rados.rados.0.smithi163.stdout:3493: finishing write tid 4 to smithi16367185-36 2023-10-09T09:20:15.394 INFO:tasks.rados.rados.0.smithi163.stdout:3493: finishing write tid 5 to smithi16367185-36 2023-10-09T09:20:15.394 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 1004 (ObjNum 1302 snap 440 seq_num 1302) dirty exists 2023-10-09T09:20:15.394 INFO:tasks.rados.rados.0.smithi163.stdout:3493: left oid 36 (ObjNum 1302 snap 440 seq_num 1302) 2023-10-09T09:20:15.394 INFO:tasks.rados.rados.0.smithi163.stdout:3495: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:20:15.394 INFO:tasks.rados.rados.0.smithi163.stdout:3493: done (10 left) 2023-10-09T09:20:15.394 INFO:tasks.rados.rados.0.smithi163.stdout:3496: done (9 left) 2023-10-09T09:20:15.394 INFO:tasks.rados.rados.0.smithi163.stdout:3505: write oid 18 current snap is 440 2023-10-09T09:20:15.394 INFO:tasks.rados.rados.0.smithi163.stdout:3505: seq_num 1308 ranges {783556=438408,1674212=400111,2736451=420734,3882270=101286} 2023-10-09T09:20:15.400 INFO:tasks.rados.rados.0.smithi163.stdout:3505: writing smithi16367185-18 from 783556 to 1221964 tid 1 2023-10-09T09:20:15.405 INFO:tasks.rados.rados.0.smithi163.stdout:3505: writing smithi16367185-18 from 1674212 to 2074323 tid 2 2023-10-09T09:20:15.411 INFO:tasks.rados.rados.0.smithi163.stdout:3505: writing smithi16367185-18 from 2736451 to 3157185 tid 3 2023-10-09T09:20:15.416 INFO:tasks.rados.rados.0.smithi163.stdout:3505: writing smithi16367185-18 from 3882270 to 3983556 tid 4 2023-10-09T09:20:15.416 INFO:tasks.rados.rados.0.smithi163.stdout:3506: rollback oid 9 current snap is 440 2023-10-09T09:20:15.417 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 9 to 431 2023-10-09T09:20:15.417 INFO:tasks.rados.rados.0.smithi163.stdout:3495: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:20:15.417 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 1801 (ObjNum 1005 snap 342 seq_num 1005) dirty exists 2023-10-09T09:20:15.417 INFO:tasks.rados.rados.0.smithi163.stdout:3495: done (10 left) 2023-10-09T09:20:15.417 INFO:tasks.rados.rados.0.smithi163.stdout:3507: read oid 26 snap -1 2023-10-09T09:20:15.417 INFO:tasks.rados.rados.0.smithi163.stdout:3507: expect deleted 2023-10-09T09:20:15.417 INFO:tasks.rados.rados.0.smithi163.stdout:3508: snap_create 2023-10-09T09:20:15.417 INFO:tasks.rados.rados.0.smithi163.stdout:3494: expect (ObjNum 1166 snap 392 seq_num 1166) 2023-10-09T09:20:15.692 INFO:tasks.rados.rados.0.smithi163.stdout:3497: finishing write tid 1 to smithi16367185-15 2023-10-09T09:20:15.692 INFO:tasks.rados.rados.0.smithi163.stdout:3497: finishing write tid 2 to smithi16367185-15 2023-10-09T09:20:15.692 INFO:tasks.rados.rados.0.smithi163.stdout:3497: finishing write tid 3 to smithi16367185-15 2023-10-09T09:20:15.693 INFO:tasks.rados.rados.0.smithi163.stdout:3497: finishing write tid 4 to smithi16367185-15 2023-10-09T09:20:15.693 INFO:tasks.rados.rados.0.smithi163.stdout:3497: finishing write tid 5 to smithi16367185-15 2023-10-09T09:20:15.693 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 1773 (ObjNum 1303 snap 440 seq_num 1303) dirty exists 2023-10-09T09:20:15.693 INFO:tasks.rados.rados.0.smithi163.stdout:3497: left oid 15 (ObjNum 1303 snap 440 seq_num 1303) 2023-10-09T09:20:15.693 INFO:tasks.rados.rados.0.smithi163.stdout:3498: expect (ObjNum 1297 snap 439 seq_num 1297) 2023-10-09T09:20:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[75425]: cluster 2023-10-09T09:20:13.376862+0000 osd.4 (osd.4) 418 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[75425]: cluster 2023-10-09T09:20:13.379922+0000 osd.4 (osd.4) 419 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[75425]: cluster 2023-10-09T09:20:13.382749+0000 osd.4 (osd.4) 420 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[75425]: cluster 2023-10-09T09:20:14.380811+0000 osd.5 (osd.5) 481 : cluster 3 failed to encode map e940 with expected crc 2023-10-09T09:20:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[75425]: cluster 2023-10-09T09:20:14.381256+0000 osd.7 (osd.7) 389 : cluster 3 failed to encode map e940 with expected crc 2023-10-09T09:20:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[75425]: cluster 2023-10-09T09:20:14.385777+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e940: 8 total, 8 up, 8 in 2023-10-09T09:20:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[75425]: cluster 2023-10-09T09:20:15.299843+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e941: 8 total, 8 up, 8 in 2023-10-09T09:20:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[75425]: cluster 2023-10-09T09:20:15.302416+0000 osd.4 (osd.4) 421 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[78107]: cluster 2023-10-09T09:20:13.376862+0000 osd.4 (osd.4) 418 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[78107]: cluster 2023-10-09T09:20:13.379922+0000 osd.4 (osd.4) 419 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[78107]: cluster 2023-10-09T09:20:13.382749+0000 osd.4 (osd.4) 420 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[78107]: cluster 2023-10-09T09:20:14.380811+0000 osd.5 (osd.5) 481 : cluster 3 failed to encode map e940 with expected crc 2023-10-09T09:20:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[78107]: cluster 2023-10-09T09:20:14.381256+0000 osd.7 (osd.7) 389 : cluster 3 failed to encode map e940 with expected crc 2023-10-09T09:20:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[78107]: cluster 2023-10-09T09:20:14.385777+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e940: 8 total, 8 up, 8 in 2023-10-09T09:20:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[78107]: cluster 2023-10-09T09:20:15.299843+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e941: 8 total, 8 up, 8 in 2023-10-09T09:20:15.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:15 smithi163 bash[78107]: cluster 2023-10-09T09:20:15.302416+0000 osd.4 (osd.4) 421 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:15 smithi181 bash[59345]: cluster 2023-10-09T09:20:13.376862+0000 osd.4 (osd.4) 418 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:15 smithi181 bash[59345]: cluster 2023-10-09T09:20:13.379922+0000 osd.4 (osd.4) 419 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:15 smithi181 bash[59345]: cluster 2023-10-09T09:20:13.382749+0000 osd.4 (osd.4) 420 : cluster 3 failed to encode map e939 with expected crc 2023-10-09T09:20:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:15 smithi181 bash[59345]: cluster 2023-10-09T09:20:14.380811+0000 osd.5 (osd.5) 481 : cluster 3 failed to encode map e940 with expected crc 2023-10-09T09:20:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:15 smithi181 bash[59345]: cluster 2023-10-09T09:20:14.381256+0000 osd.7 (osd.7) 389 : cluster 3 failed to encode map e940 with expected crc 2023-10-09T09:20:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:15 smithi181 bash[59345]: cluster 2023-10-09T09:20:14.385777+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e940: 8 total, 8 up, 8 in 2023-10-09T09:20:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:15 smithi181 bash[59345]: cluster 2023-10-09T09:20:15.299843+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e941: 8 total, 8 up, 8 in 2023-10-09T09:20:15.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:15 smithi181 bash[59345]: cluster 2023-10-09T09:20:15.302416+0000 osd.4 (osd.4) 421 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:15.935 INFO:tasks.rados.rados.0.smithi163.stdout:3500: finishing write tid 1 to smithi16367185-25 2023-10-09T09:20:15.935 INFO:tasks.rados.rados.0.smithi163.stdout:3500: finishing write tid 2 to smithi16367185-25 2023-10-09T09:20:15.935 INFO:tasks.rados.rados.0.smithi163.stdout:3500: finishing write tid 3 to smithi16367185-25 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:3500: finishing write tid 4 to smithi16367185-25 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:3500: finishing write tid 5 to smithi16367185-25 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1627 (ObjNum 1304 snap 440 seq_num 1304) dirty exists 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:3500: left oid 25 (ObjNum 1304 snap 440 seq_num 1304) 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:3501: finishing write tid 1 to smithi16367185-22 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:3501: finishing write tid 2 to smithi16367185-22 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:3501: finishing write tid 3 to smithi16367185-22 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:3501: finishing write tid 4 to smithi16367185-22 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:3501: finishing write tid 5 to smithi16367185-22 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 2619 (ObjNum 1305 snap 440 seq_num 1305) dirty exists 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:3501: left oid 22 (ObjNum 1305 snap 440 seq_num 1305) 2023-10-09T09:20:15.936 INFO:tasks.rados.rados.0.smithi163.stdout:3502: finishing write tid 1 to smithi16367185-41 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:3502: finishing write tid 2 to smithi16367185-41 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:3502: finishing write tid 3 to smithi16367185-41 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:3502: finishing write tid 4 to smithi16367185-41 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:3502: finishing write tid 5 to smithi16367185-41 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 320 (ObjNum 1306 snap 440 seq_num 1306) dirty exists 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:3502: left oid 41 (ObjNum 1306 snap 440 seq_num 1306) 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:3503: finishing write tid 1 to smithi16367185-13 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:3503: finishing write tid 2 to smithi16367185-13 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:3504: finishing rollback tid 0 to smithi16367185-24 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:3504: finishing rollback tid 1 to smithi16367185-24 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:3504: finishing rollback tid 2 to smithi16367185-24 2023-10-09T09:20:15.937 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 778 (ObjNum 1210 snap 404 seq_num 1210) dirty exists 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:3503: finishing write tid 3 to smithi16367185-13 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:3503: finishing write tid 4 to smithi16367185-13 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:3503: finishing write tid 5 to smithi16367185-13 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 690 (ObjNum 1307 snap 440 seq_num 1307) dirty exists 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:3503: left oid 13 (ObjNum 1307 snap 440 seq_num 1307) 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:3505: finishing write tid 1 to smithi16367185-18 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:3505: finishing write tid 2 to smithi16367185-18 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:3505: finishing write tid 3 to smithi16367185-18 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:3506: finishing rollback tid 1 to smithi16367185-9 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 0 (ObjNum 669362160 snap 21875 seq_num 1) dirty dne 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:3505: finishing write tid 4 to smithi16367185-18 2023-10-09T09:20:15.938 INFO:tasks.rados.rados.0.smithi163.stdout:3505: finishing write tid 5 to smithi16367185-18 2023-10-09T09:20:15.939 INFO:tasks.rados.rados.0.smithi163.stdout:3505: finishing write tid 6 to smithi16367185-18 2023-10-09T09:20:15.939 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1225 (ObjNum 1308 snap 440 seq_num 1308) dirty exists 2023-10-09T09:20:15.939 INFO:tasks.rados.rados.0.smithi163.stdout:3505: left oid 18 (ObjNum 1308 snap 440 seq_num 1308) 2023-10-09T09:20:16.395 INFO:tasks.rados.rados.0.smithi163.stdout:3494: done (11 left) 2023-10-09T09:20:16.395 INFO:tasks.rados.rados.0.smithi163.stdout:3497: done (10 left) 2023-10-09T09:20:16.395 INFO:tasks.rados.rados.0.smithi163.stdout:3498: done (9 left) 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3500: done (8 left) 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3501: done (7 left) 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3502: done (6 left) 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3503: done (5 left) 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3504: done (4 left) 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3505: done (3 left) 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3506: done (2 left) 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3507: done (1 left) 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3508: done (0 left) 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3509: rollback oid 36 current snap is 441 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 36 to 438 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:3510: rollback oid 29 current snap is 441 2023-10-09T09:20:16.396 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 29 to 431 2023-10-09T09:20:16.397 INFO:tasks.rados.rados.0.smithi163.stdout:3511: snap_create 2023-10-09T09:20:16.402 INFO:tasks.rados.rados.0.smithi163.stdout:3509: finishing rollback tid 0 to smithi16367185-36 2023-10-09T09:20:16.402 INFO:tasks.rados.rados.0.smithi163.stdout:3509: finishing rollback tid 1 to smithi16367185-36 2023-10-09T09:20:16.402 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 1007 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:20:16.405 INFO:tasks.rados.rados.0.smithi163.stdout:3510: finishing rollback tid 0 to smithi16367185-29 2023-10-09T09:20:16.405 INFO:tasks.rados.rados.0.smithi163.stdout:3510: finishing rollback tid 1 to smithi16367185-29 2023-10-09T09:20:16.406 INFO:tasks.rados.rados.0.smithi163.stdout:3510: finishing rollback tid 2 to smithi16367185-29 2023-10-09T09:20:16.406 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1631 (ObjNum 1232 snap 410 seq_num 1232) dirty exists 2023-10-09T09:20:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:16 smithi163 bash[78107]: cluster 2023-10-09T09:20:15.295390+0000 osd.5 (osd.5) 482 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:16 smithi163 bash[78107]: cluster 2023-10-09T09:20:15.357821+0000 osd.7 (osd.7) 390 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:16 smithi163 bash[78107]: cluster 2023-10-09T09:20:15.380976+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:16 smithi163 bash[78107]: cluster 2023-10-09T09:20:15.453229+0000 mgr.y (mgr.34107) 411 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2023-10-09T09:20:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:16 smithi163 bash[75425]: cluster 2023-10-09T09:20:15.295390+0000 osd.5 (osd.5) 482 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:16 smithi163 bash[75425]: cluster 2023-10-09T09:20:15.357821+0000 osd.7 (osd.7) 390 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:16 smithi163 bash[75425]: cluster 2023-10-09T09:20:15.380976+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:16.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:16 smithi163 bash[75425]: cluster 2023-10-09T09:20:15.453229+0000 mgr.y (mgr.34107) 411 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2023-10-09T09:20:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:16 smithi181 bash[59345]: cluster 2023-10-09T09:20:15.295390+0000 osd.5 (osd.5) 482 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:16 smithi181 bash[59345]: cluster 2023-10-09T09:20:15.357821+0000 osd.7 (osd.7) 390 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:16 smithi181 bash[59345]: cluster 2023-10-09T09:20:15.380976+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e941 with expected crc 2023-10-09T09:20:16.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:16 smithi181 bash[59345]: cluster 2023-10-09T09:20:15.453229+0000 mgr.y (mgr.34107) 411 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2023-10-09T09:20:17.397 INFO:tasks.rados.rados.0.smithi163.stdout:3509: done (2 left) 2023-10-09T09:20:17.397 INFO:tasks.rados.rados.0.smithi163.stdout:3510: done (1 left) 2023-10-09T09:20:17.398 INFO:tasks.rados.rados.0.smithi163.stdout:3511: done (0 left) 2023-10-09T09:20:17.398 INFO:tasks.rados.rados.0.smithi163.stdout:3512: snap_create 2023-10-09T09:20:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:17 smithi163 bash[78107]: cluster 2023-10-09T09:20:16.390262+0000 osd.5 (osd.5) 483 : cluster 3 failed to encode map e942 with expected crc 2023-10-09T09:20:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:17 smithi163 bash[78107]: cluster 2023-10-09T09:20:16.390904+0000 osd.7 (osd.7) 391 : cluster 3 failed to encode map e942 with expected crc 2023-10-09T09:20:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:17 smithi163 bash[78107]: cluster 2023-10-09T09:20:16.394785+0000 mon.a (mon.0) 1161 : cluster 0 osdmap e942: 8 total, 8 up, 8 in 2023-10-09T09:20:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:17 smithi163 bash[78107]: cluster 2023-10-09T09:20:16.398926+0000 osd.4 (osd.4) 422 : cluster 3 failed to encode map e942 with expected crc 2023-10-09T09:20:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:17 smithi163 bash[75425]: cluster 2023-10-09T09:20:16.390262+0000 osd.5 (osd.5) 483 : cluster 3 failed to encode map e942 with expected crc 2023-10-09T09:20:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:17 smithi163 bash[75425]: cluster 2023-10-09T09:20:16.390904+0000 osd.7 (osd.7) 391 : cluster 3 failed to encode map e942 with expected crc 2023-10-09T09:20:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:17 smithi163 bash[75425]: cluster 2023-10-09T09:20:16.394785+0000 mon.a (mon.0) 1161 : cluster 0 osdmap e942: 8 total, 8 up, 8 in 2023-10-09T09:20:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:17 smithi163 bash[75425]: cluster 2023-10-09T09:20:16.398926+0000 osd.4 (osd.4) 422 : cluster 3 failed to encode map e942 with expected crc 2023-10-09T09:20:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:17 smithi181 bash[59345]: cluster 2023-10-09T09:20:16.390262+0000 osd.5 (osd.5) 483 : cluster 3 failed to encode map e942 with expected crc 2023-10-09T09:20:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:17 smithi181 bash[59345]: cluster 2023-10-09T09:20:16.390904+0000 osd.7 (osd.7) 391 : cluster 3 failed to encode map e942 with expected crc 2023-10-09T09:20:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:17 smithi181 bash[59345]: cluster 2023-10-09T09:20:16.394785+0000 mon.a (mon.0) 1161 : cluster 0 osdmap e942: 8 total, 8 up, 8 in 2023-10-09T09:20:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:17 smithi181 bash[59345]: cluster 2023-10-09T09:20:16.398926+0000 osd.4 (osd.4) 422 : cluster 3 failed to encode map e942 with expected crc 2023-10-09T09:20:18.400 INFO:tasks.rados.rados.0.smithi163.stdout:3512: done (0 left) 2023-10-09T09:20:18.400 INFO:tasks.rados.rados.0.smithi163.stdout:3513: delete oid 2 current snap is 443 2023-10-09T09:20:18.413 INFO:tasks.rados.rados.0.smithi163.stdout:3513: done (0 left) 2023-10-09T09:20:18.413 INFO:tasks.rados.rados.0.smithi163.stdout:3514: rollback oid 20 current snap is 443 2023-10-09T09:20:18.413 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 422 2023-10-09T09:20:18.413 INFO:tasks.rados.rados.0.smithi163.stdout:3515: write oid 39 current snap is 443 2023-10-09T09:20:18.413 INFO:tasks.rados.rados.0.smithi163.stdout:3515: seq_num 1310 ranges {417170=588906,1674823=342347} 2023-10-09T09:20:18.422 INFO:tasks.rados.rados.0.smithi163.stdout:3515: writing smithi16367185-39 from 417170 to 1006076 tid 1 2023-10-09T09:20:18.429 INFO:tasks.rados.rados.0.smithi163.stdout:3515: writing smithi16367185-39 from 1674823 to 2017170 tid 2 2023-10-09T09:20:18.429 INFO:tasks.rados.rados.0.smithi163.stdout:3516: snap_remove snap 440 2023-10-09T09:20:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:18 smithi163 bash[78107]: cluster 2023-10-09T09:20:17.392362+0000 osd.5 (osd.5) 484 : cluster 3 failed to encode map e943 with expected crc 2023-10-09T09:20:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:18 smithi163 bash[78107]: cluster 2023-10-09T09:20:17.393819+0000 osd.7 (osd.7) 392 : cluster 3 failed to encode map e943 with expected crc 2023-10-09T09:20:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:18 smithi163 bash[78107]: cluster 2023-10-09T09:20:17.397041+0000 mon.a (mon.0) 1162 : cluster 0 osdmap e943: 8 total, 8 up, 8 in 2023-10-09T09:20:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:18 smithi163 bash[78107]: cluster 2023-10-09T09:20:17.453855+0000 mgr.y (mgr.34107) 412 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 6.9 MiB/s wr, 50 op/s 2023-10-09T09:20:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:18 smithi163 bash[75425]: cluster 2023-10-09T09:20:17.392362+0000 osd.5 (osd.5) 484 : cluster 3 failed to encode map e943 with expected crc 2023-10-09T09:20:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:18 smithi163 bash[75425]: cluster 2023-10-09T09:20:17.393819+0000 osd.7 (osd.7) 392 : cluster 3 failed to encode map e943 with expected crc 2023-10-09T09:20:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:18 smithi163 bash[75425]: cluster 2023-10-09T09:20:17.397041+0000 mon.a (mon.0) 1162 : cluster 0 osdmap e943: 8 total, 8 up, 8 in 2023-10-09T09:20:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:18 smithi163 bash[75425]: cluster 2023-10-09T09:20:17.453855+0000 mgr.y (mgr.34107) 412 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 6.9 MiB/s wr, 50 op/s 2023-10-09T09:20:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:18 smithi181 bash[59345]: cluster 2023-10-09T09:20:17.392362+0000 osd.5 (osd.5) 484 : cluster 3 failed to encode map e943 with expected crc 2023-10-09T09:20:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:18 smithi181 bash[59345]: cluster 2023-10-09T09:20:17.393819+0000 osd.7 (osd.7) 392 : cluster 3 failed to encode map e943 with expected crc 2023-10-09T09:20:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:18 smithi181 bash[59345]: cluster 2023-10-09T09:20:17.397041+0000 mon.a (mon.0) 1162 : cluster 0 osdmap e943: 8 total, 8 up, 8 in 2023-10-09T09:20:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:18 smithi181 bash[59345]: cluster 2023-10-09T09:20:17.453855+0000 mgr.y (mgr.34107) 412 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 6.9 MiB/s wr, 50 op/s 2023-10-09T09:20:19.415 INFO:tasks.rados.rados.0.smithi163.stdout:3514: finishing rollback tid 0 to smithi16367185-20 2023-10-09T09:20:19.415 INFO:tasks.rados.rados.0.smithi163.stdout:3514: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:20:19.415 INFO:tasks.rados.rados.0.smithi163.stdout:3516: done (2 left) 2023-10-09T09:20:19.415 INFO:tasks.rados.rados.0.smithi163.stdout:3517: write oid 7 current snap is 443 2023-10-09T09:20:19.415 INFO:tasks.rados.rados.0.smithi163.stdout:3517: seq_num 1311 ranges {433977=664155,1733305=300672} 2023-10-09T09:20:19.426 INFO:tasks.rados.rados.0.smithi163.stdout:3517: writing smithi16367185-7 from 433977 to 1098132 tid 1 2023-10-09T09:20:19.433 INFO:tasks.rados.rados.0.smithi163.stdout:3517: writing smithi16367185-7 from 1733305 to 2033977 tid 2 2023-10-09T09:20:19.433 INFO:tasks.rados.rados.0.smithi163.stdout:3518: rollback oid 26 current snap is 443 2023-10-09T09:20:19.433 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 26 to 432 2023-10-09T09:20:19.433 INFO:tasks.rados.rados.0.smithi163.stdout:3514: finishing rollback tid 2 to smithi16367185-20 2023-10-09T09:20:19.433 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 694 (ObjNum 1119 snap 369 seq_num 1119) dirty exists 2023-10-09T09:20:19.433 INFO:tasks.rados.rados.0.smithi163.stdout:3515: finishing write tid 1 to smithi16367185-39 2023-10-09T09:20:19.433 INFO:tasks.rados.rados.0.smithi163.stdout:3514: done (3 left) 2023-10-09T09:20:19.434 INFO:tasks.rados.rados.0.smithi163.stdout:3519: snap_create 2023-10-09T09:20:19.434 INFO:tasks.rados.rados.0.smithi163.stdout:3515: finishing write tid 2 to smithi16367185-39 2023-10-09T09:20:19.434 INFO:tasks.rados.rados.0.smithi163.stdout:3515: finishing write tid 3 to smithi16367185-39 2023-10-09T09:20:19.434 INFO:tasks.rados.rados.0.smithi163.stdout:3515: finishing write tid 4 to smithi16367185-39 2023-10-09T09:20:19.434 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1634 (ObjNum 1310 snap 443 seq_num 1310) dirty exists 2023-10-09T09:20:19.434 INFO:tasks.rados.rados.0.smithi163.stdout:3515: left oid 39 (ObjNum 1310 snap 443 seq_num 1310) 2023-10-09T09:20:19.434 INFO:tasks.rados.rados.0.smithi163.stdout:3517: finishing write tid 1 to smithi16367185-7 2023-10-09T09:20:19.436 INFO:tasks.rados.rados.0.smithi163.stdout:3518: finishing rollback tid 1 to smithi16367185-26 2023-10-09T09:20:19.436 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:20:19.437 INFO:tasks.rados.rados.0.smithi163.stdout:3517: finishing write tid 2 to smithi16367185-7 2023-10-09T09:20:19.438 INFO:tasks.rados.rados.0.smithi163.stdout:3517: finishing write tid 3 to smithi16367185-7 2023-10-09T09:20:19.438 INFO:tasks.rados.rados.0.smithi163.stdout:3517: finishing write tid 4 to smithi16367185-7 2023-10-09T09:20:19.438 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 924 (ObjNum 1311 snap 443 seq_num 1311) dirty exists 2023-10-09T09:20:19.439 INFO:tasks.rados.rados.0.smithi163.stdout:3517: left oid 7 (ObjNum 1311 snap 443 seq_num 1311) 2023-10-09T09:20:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:19 smithi163 bash[78107]: cluster 2023-10-09T09:20:18.395373+0000 osd.5 (osd.5) 485 : cluster 3 failed to encode map e944 with expected crc 2023-10-09T09:20:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:19 smithi163 bash[78107]: cluster 2023-10-09T09:20:18.396081+0000 osd.7 (osd.7) 393 : cluster 3 failed to encode map e944 with expected crc 2023-10-09T09:20:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:19 smithi163 bash[78107]: cluster 2023-10-09T09:20:18.399920+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e944: 8 total, 8 up, 8 in 2023-10-09T09:20:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:19 smithi163 bash[78107]: cluster 2023-10-09T09:20:18.401590+0000 osd.7 (osd.7) 394 : cluster 3 failed to encode map e944 with expected crc 2023-10-09T09:20:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:19 smithi163 bash[75425]: cluster 2023-10-09T09:20:18.395373+0000 osd.5 (osd.5) 485 : cluster 3 failed to encode map e944 with expected crc 2023-10-09T09:20:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:19 smithi163 bash[75425]: cluster 2023-10-09T09:20:18.396081+0000 osd.7 (osd.7) 393 : cluster 3 failed to encode map e944 with expected crc 2023-10-09T09:20:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:19 smithi163 bash[75425]: cluster 2023-10-09T09:20:18.399920+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e944: 8 total, 8 up, 8 in 2023-10-09T09:20:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:19 smithi163 bash[75425]: cluster 2023-10-09T09:20:18.401590+0000 osd.7 (osd.7) 394 : cluster 3 failed to encode map e944 with expected crc 2023-10-09T09:20:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:19 smithi181 bash[59345]: cluster 2023-10-09T09:20:18.395373+0000 osd.5 (osd.5) 485 : cluster 3 failed to encode map e944 with expected crc 2023-10-09T09:20:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:19 smithi181 bash[59345]: cluster 2023-10-09T09:20:18.396081+0000 osd.7 (osd.7) 393 : cluster 3 failed to encode map e944 with expected crc 2023-10-09T09:20:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:19 smithi181 bash[59345]: cluster 2023-10-09T09:20:18.399920+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e944: 8 total, 8 up, 8 in 2023-10-09T09:20:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:19 smithi181 bash[59345]: cluster 2023-10-09T09:20:18.401590+0000 osd.7 (osd.7) 394 : cluster 3 failed to encode map e944 with expected crc 2023-10-09T09:20:20.307 INFO:tasks.rados.rados.0.smithi163.stdout:3515: done (3 left) 2023-10-09T09:20:20.308 INFO:tasks.rados.rados.0.smithi163.stdout:3517: done (2 left) 2023-10-09T09:20:20.308 INFO:tasks.rados.rados.0.smithi163.stdout:3518: done (1 left) 2023-10-09T09:20:20.308 INFO:tasks.rados.rados.0.smithi163.stdout:3519: done (0 left) 2023-10-09T09:20:20.308 INFO:tasks.rados.rados.0.smithi163.stdout:3520: delete oid 23 current snap is 444 2023-10-09T09:20:20.317 INFO:tasks.rados.rados.0.smithi163.stdout:3520: done (0 left) 2023-10-09T09:20:20.318 INFO:tasks.rados.rados.0.smithi163.stdout:3521: read oid 34 snap -1 2023-10-09T09:20:20.318 INFO:tasks.rados.rados.0.smithi163.stdout:3521: expect (ObjNum 1291 snap 439 seq_num 1291) 2023-10-09T09:20:20.318 INFO:tasks.rados.rados.0.smithi163.stdout:3522: snap_remove snap 432 2023-10-09T09:20:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:20 smithi163 bash[78107]: cluster 2023-10-09T09:20:19.409421+0000 osd.5 (osd.5) 486 : cluster 3 failed to encode map e945 with expected crc 2023-10-09T09:20:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:20 smithi163 bash[78107]: cluster 2023-10-09T09:20:19.409749+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e945 with expected crc 2023-10-09T09:20:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:20 smithi163 bash[78107]: cluster 2023-10-09T09:20:19.414210+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e945: 8 total, 8 up, 8 in 2023-10-09T09:20:20.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:20 smithi163 bash[78107]: cluster 2023-10-09T09:20:19.454429+0000 mgr.y (mgr.34107) 413 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 4.7 MiB/s wr, 38 op/s 2023-10-09T09:20:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:20 smithi163 bash[78107]: cluster 2023-10-09T09:20:20.307297+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e946: 8 total, 8 up, 8 in 2023-10-09T09:20:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:20 smithi163 bash[75425]: cluster 2023-10-09T09:20:19.409421+0000 osd.5 (osd.5) 486 : cluster 3 failed to encode map e945 with expected crc 2023-10-09T09:20:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:20 smithi163 bash[75425]: cluster 2023-10-09T09:20:19.409749+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e945 with expected crc 2023-10-09T09:20:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:20 smithi163 bash[75425]: cluster 2023-10-09T09:20:19.414210+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e945: 8 total, 8 up, 8 in 2023-10-09T09:20:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:20 smithi163 bash[75425]: cluster 2023-10-09T09:20:19.454429+0000 mgr.y (mgr.34107) 413 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 4.7 MiB/s wr, 38 op/s 2023-10-09T09:20:20.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:20 smithi163 bash[75425]: cluster 2023-10-09T09:20:20.307297+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e946: 8 total, 8 up, 8 in 2023-10-09T09:20:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:20 smithi181 bash[59345]: cluster 2023-10-09T09:20:19.409421+0000 osd.5 (osd.5) 486 : cluster 3 failed to encode map e945 with expected crc 2023-10-09T09:20:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:20 smithi181 bash[59345]: cluster 2023-10-09T09:20:19.409749+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e945 with expected crc 2023-10-09T09:20:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:20 smithi181 bash[59345]: cluster 2023-10-09T09:20:19.414210+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e945: 8 total, 8 up, 8 in 2023-10-09T09:20:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:20 smithi181 bash[59345]: cluster 2023-10-09T09:20:19.454429+0000 mgr.y (mgr.34107) 413 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 4.7 MiB/s wr, 38 op/s 2023-10-09T09:20:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:20 smithi181 bash[59345]: cluster 2023-10-09T09:20:20.307297+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e946: 8 total, 8 up, 8 in 2023-10-09T09:20:21.317 INFO:tasks.rados.rados.0.smithi163.stdout:3522: done (1 left) 2023-10-09T09:20:21.317 INFO:tasks.rados.rados.0.smithi163.stdout:3523: write oid 22 current snap is 444 2023-10-09T09:20:21.317 INFO:tasks.rados.rados.0.smithi163.stdout:3523: seq_num 1313 ranges {467591=414653,1450269=617322} 2023-10-09T09:20:21.326 INFO:tasks.rados.rados.0.smithi163.stdout:3523: writing smithi16367185-22 from 467591 to 882244 tid 1 2023-10-09T09:20:21.335 INFO:tasks.rados.rados.0.smithi163.stdout:3523: writing smithi16367185-22 from 1450269 to 2067591 tid 2 2023-10-09T09:20:21.335 INFO:tasks.rados.rados.0.smithi163.stdout:3524: delete oid 7 current snap is 444 2023-10-09T09:20:21.335 INFO:tasks.rados.rados.0.smithi163.stdout:3521: expect (ObjNum 1291 snap 439 seq_num 1291) 2023-10-09T09:20:21.456 INFO:tasks.rados.rados.0.smithi163.stdout:3523: finishing write tid 1 to smithi16367185-22 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:3523: finishing write tid 2 to smithi16367185-22 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:3521: done (2 left) 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:3524: done (1 left) 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:3525: read oid 35 snap -1 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:3525: expect deleted 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:3526: write oid 31 current snap is 444 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:3523: finishing write tid 3 to smithi16367185-22 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:3523: finishing write tid 4 to smithi16367185-22 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 2629 (ObjNum 1313 snap 444 seq_num 1313) dirty exists 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:3523: left oid 22 (ObjNum 1313 snap 444 seq_num 1313) 2023-10-09T09:20:21.457 INFO:tasks.rados.rados.0.smithi163.stdout:3526: seq_num 1315 ranges {501205=565151,1850880=250325} 2023-10-09T09:20:21.464 INFO:tasks.rados.rados.0.smithi163.stdout:3526: writing smithi16367185-31 from 501205 to 1066356 tid 1 2023-10-09T09:20:21.471 INFO:tasks.rados.rados.0.smithi163.stdout:3526: writing smithi16367185-31 from 1850880 to 2101205 tid 2 2023-10-09T09:20:21.472 INFO:tasks.rados.rados.0.smithi163.stdout:3523: done (2 left) 2023-10-09T09:20:21.472 INFO:tasks.rados.rados.0.smithi163.stdout:3527: delete oid 4 current snap is 444 2023-10-09T09:20:21.472 INFO:tasks.rados.rados.0.smithi163.stdout:3526: finishing write tid 1 to smithi16367185-31 2023-10-09T09:20:21.477 INFO:tasks.rados.rados.0.smithi163.stdout:3525: done (2 left) 2023-10-09T09:20:21.477 INFO:tasks.rados.rados.0.smithi163.stdout:3527: done (1 left) 2023-10-09T09:20:21.477 INFO:tasks.rados.rados.0.smithi163.stdout:3528: write oid 42 current snap is 444 2023-10-09T09:20:21.477 INFO:tasks.rados.rados.0.smithi163.stdout:3528: seq_num 1317 ranges {534819=432002,1684197=450622} 2023-10-09T09:20:21.482 INFO:tasks.rados.rados.0.smithi163.stdout:3528: writing smithi16367185-42 from 534819 to 966821 tid 1 2023-10-09T09:20:21.490 INFO:tasks.rados.rados.0.smithi163.stdout:3528: writing smithi16367185-42 from 1684197 to 2134819 tid 2 2023-10-09T09:20:21.490 INFO:tasks.rados.rados.0.smithi163.stdout:3529: read oid 26 snap 418 2023-10-09T09:20:21.490 INFO:tasks.rados.rados.0.smithi163.stdout:3529: expect deleted 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:3530: rollback oid 5 current snap is 444 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 427 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:3526: finishing write tid 2 to smithi16367185-31 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:3526: finishing write tid 3 to smithi16367185-31 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:3531: read oid 22 snap 422 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:3531: expect (ObjNum 1248 snap 420 seq_num 1248) 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:3532: rollback oid 27 current snap is 444 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 27 to 439 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:3526: finishing write tid 4 to smithi16367185-31 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 1014 (ObjNum 1315 snap 444 seq_num 1315) dirty exists 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:3526: left oid 31 (ObjNum 1315 snap 444 seq_num 1315) 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:3528: finishing write tid 1 to smithi16367185-42 2023-10-09T09:20:21.491 INFO:tasks.rados.rados.0.smithi163.stdout:3526: done (5 left) 2023-10-09T09:20:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:3533: read oid 11 snap -1 2023-10-09T09:20:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:3533: expect (ObjNum 1284 snap 437 seq_num 1284) 2023-10-09T09:20:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:3534: rollback oid 3 current snap is 444 2023-10-09T09:20:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 420 2023-10-09T09:20:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:3535: write oid 1 current snap is 444 2023-10-09T09:20:21.492 INFO:tasks.rados.rados.0.smithi163.stdout:3535: seq_num 1318 ranges {551626=507251,1626326=418456} 2023-10-09T09:20:21.497 INFO:tasks.rados.rados.0.smithi163.stdout:3535: writing smithi16367185-1 from 551626 to 1058877 tid 1 2023-10-09T09:20:21.504 INFO:tasks.rados.rados.0.smithi163.stdout:3535: writing smithi16367185-1 from 1626326 to 2044782 tid 2 2023-10-09T09:20:21.504 INFO:tasks.rados.rados.0.smithi163.stdout:3536: snap_remove snap 425 2023-10-09T09:20:21.504 INFO:tasks.rados.rados.0.smithi163.stdout:3532: finishing rollback tid 1 to smithi16367185-27 2023-10-09T09:20:21.504 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 1037 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:20:21.505 INFO:tasks.rados.rados.0.smithi163.stdout:3534: finishing rollback tid 0 to smithi16367185-3 2023-10-09T09:20:21.505 INFO:tasks.rados.rados.0.smithi163.stdout:3534: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:20:21.505 INFO:tasks.rados.rados.0.smithi163.stdout:3530: finishing rollback tid 0 to smithi16367185-5 2023-10-09T09:20:21.505 INFO:tasks.rados.rados.0.smithi163.stdout:3530: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:20:21.505 INFO:tasks.rados.rados.0.smithi163.stdout:3534: finishing rollback tid 2 to smithi16367185-3 2023-10-09T09:20:21.505 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 1018 (ObjNum 1253 snap 420 seq_num 1253) dirty exists 2023-10-09T09:20:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:21 smithi163 bash[78107]: cluster 2023-10-09T09:20:20.301785+0000 osd.5 (osd.5) 487 : cluster 3 failed to encode map e946 with expected crc 2023-10-09T09:20:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:21 smithi163 bash[78107]: cluster 2023-10-09T09:20:21.312898+0000 osd.4 (osd.4) 423 : cluster 3 failed to encode map e947 with expected crc 2023-10-09T09:20:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:21 smithi163 bash[78107]: cluster 2023-10-09T09:20:21.317181+0000 mon.a (mon.0) 1166 : cluster 0 osdmap e947: 8 total, 8 up, 8 in 2023-10-09T09:20:21.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:21 smithi163 bash[75425]: cluster 2023-10-09T09:20:20.301785+0000 osd.5 (osd.5) 487 : cluster 3 failed to encode map e946 with expected crc 2023-10-09T09:20:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:21 smithi163 bash[75425]: cluster 2023-10-09T09:20:21.312898+0000 osd.4 (osd.4) 423 : cluster 3 failed to encode map e947 with expected crc 2023-10-09T09:20:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:21 smithi163 bash[75425]: cluster 2023-10-09T09:20:21.317181+0000 mon.a (mon.0) 1166 : cluster 0 osdmap e947: 8 total, 8 up, 8 in 2023-10-09T09:20:21.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:21 smithi181 bash[59345]: cluster 2023-10-09T09:20:20.301785+0000 osd.5 (osd.5) 487 : cluster 3 failed to encode map e946 with expected crc 2023-10-09T09:20:21.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:21 smithi181 bash[59345]: cluster 2023-10-09T09:20:21.312898+0000 osd.4 (osd.4) 423 : cluster 3 failed to encode map e947 with expected crc 2023-10-09T09:20:21.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:21 smithi181 bash[59345]: cluster 2023-10-09T09:20:21.317181+0000 mon.a (mon.0) 1166 : cluster 0 osdmap e947: 8 total, 8 up, 8 in 2023-10-09T09:20:22.443 INFO:tasks.rados.rados.0.smithi163.stdout:3529: done (8 left) 2023-10-09T09:20:22.444 INFO:tasks.rados.rados.0.smithi163.stdout:3532: done (7 left) 2023-10-09T09:20:22.444 INFO:tasks.rados.rados.0.smithi163.stdout:3534: done (6 left) 2023-10-09T09:20:22.444 INFO:tasks.rados.rados.0.smithi163.stdout:3536: done (5 left) 2023-10-09T09:20:22.444 INFO:tasks.rados.rados.0.smithi163.stdout:3537: read oid 43 snap -1 2023-10-09T09:20:22.444 INFO:tasks.rados.rados.0.smithi163.stdout:3537: expect (ObjNum 1091 snap 362 seq_num 1091) 2023-10-09T09:20:22.445 INFO:tasks.rados.rados.0.smithi163.stdout:3538: snap_create 2023-10-09T09:20:22.445 INFO:tasks.rados.rados.0.smithi163.stdout:3530: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:20:22.445 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1817 (ObjNum 1261 snap 425 seq_num 1261) dirty exists 2023-10-09T09:20:22.445 INFO:tasks.rados.rados.0.smithi163.stdout:3535: finishing write tid 1 to smithi16367185-1 2023-10-09T09:20:22.445 INFO:tasks.rados.rados.0.smithi163.stdout:3535: finishing write tid 2 to smithi16367185-1 2023-10-09T09:20:22.445 INFO:tasks.rados.rados.0.smithi163.stdout:3535: finishing write tid 3 to smithi16367185-1 2023-10-09T09:20:22.446 INFO:tasks.rados.rados.0.smithi163.stdout:3535: finishing write tid 4 to smithi16367185-1 2023-10-09T09:20:22.446 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1821 (ObjNum 1318 snap 444 seq_num 1318) dirty exists 2023-10-09T09:20:22.446 INFO:tasks.rados.rados.0.smithi163.stdout:3535: left oid 1 (ObjNum 1318 snap 444 seq_num 1318) 2023-10-09T09:20:22.446 INFO:tasks.rados.rados.0.smithi163.stdout:3533: expect (ObjNum 1284 snap 437 seq_num 1284) 2023-10-09T09:20:22.578 INFO:tasks.rados.rados.0.smithi163.stdout:3528: finishing write tid 2 to smithi16367185-42 2023-10-09T09:20:22.579 INFO:tasks.rados.rados.0.smithi163.stdout:3528: finishing write tid 3 to smithi16367185-42 2023-10-09T09:20:22.579 INFO:tasks.rados.rados.0.smithi163.stdout:3528: finishing write tid 4 to smithi16367185-42 2023-10-09T09:20:22.579 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 2633 (ObjNum 1317 snap 444 seq_num 1317) dirty exists 2023-10-09T09:20:22.579 INFO:tasks.rados.rados.0.smithi163.stdout:3528: left oid 42 (ObjNum 1317 snap 444 seq_num 1317) 2023-10-09T09:20:22.580 INFO:tasks.rados.rados.0.smithi163.stdout:3531: expect (ObjNum 1248 snap 420 seq_num 1248) 2023-10-09T09:20:22.675 INFO:tasks.rados.rados.0.smithi163.stdout:3537: expect (ObjNum 1091 snap 362 seq_num 1091) 2023-10-09T09:20:22.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:22 smithi163 bash[75425]: cluster 2023-10-09T09:20:20.934098+0000 osd.7 (osd.7) 395 : cluster 3 failed to encode map e946 with expected crc 2023-10-09T09:20:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:22 smithi163 bash[75425]: cluster 2023-10-09T09:20:21.312171+0000 osd.5 (osd.5) 488 : cluster 3 failed to encode map e947 with expected crc 2023-10-09T09:20:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:22 smithi163 bash[75425]: cluster 2023-10-09T09:20:21.454987+0000 mgr.y (mgr.34107) 414 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 241 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2023-10-09T09:20:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:22 smithi163 bash[75425]: cluster 2023-10-09T09:20:21.457989+0000 osd.7 (osd.7) 396 : cluster 3 failed to encode map e947 with expected crc 2023-10-09T09:20:22.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:22 smithi163 bash[78107]: cluster 2023-10-09T09:20:20.934098+0000 osd.7 (osd.7) 395 : cluster 3 failed to encode map e946 with expected crc 2023-10-09T09:20:22.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:22 smithi163 bash[78107]: cluster 2023-10-09T09:20:21.312171+0000 osd.5 (osd.5) 488 : cluster 3 failed to encode map e947 with expected crc 2023-10-09T09:20:22.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:22 smithi163 bash[78107]: cluster 2023-10-09T09:20:21.454987+0000 mgr.y (mgr.34107) 414 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 241 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2023-10-09T09:20:22.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:22 smithi163 bash[78107]: cluster 2023-10-09T09:20:21.457989+0000 osd.7 (osd.7) 396 : cluster 3 failed to encode map e947 with expected crc 2023-10-09T09:20:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:22 smithi181 bash[59345]: cluster 2023-10-09T09:20:20.934098+0000 osd.7 (osd.7) 395 : cluster 3 failed to encode map e946 with expected crc 2023-10-09T09:20:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:22 smithi181 bash[59345]: cluster 2023-10-09T09:20:21.312171+0000 osd.5 (osd.5) 488 : cluster 3 failed to encode map e947 with expected crc 2023-10-09T09:20:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:22 smithi181 bash[59345]: cluster 2023-10-09T09:20:21.454987+0000 mgr.y (mgr.34107) 414 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 241 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2023-10-09T09:20:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:22 smithi181 bash[59345]: cluster 2023-10-09T09:20:21.457989+0000 osd.7 (osd.7) 396 : cluster 3 failed to encode map e947 with expected crc 2023-10-09T09:20:23.448 INFO:tasks.rados.rados.0.smithi163.stdout:3528: done (6 left) 2023-10-09T09:20:23.448 INFO:tasks.rados.rados.0.smithi163.stdout:3530: done (5 left) 2023-10-09T09:20:23.448 INFO:tasks.rados.rados.0.smithi163.stdout:3531: done (4 left) 2023-10-09T09:20:23.449 INFO:tasks.rados.rados.0.smithi163.stdout:3533: done (3 left) 2023-10-09T09:20:23.449 INFO:tasks.rados.rados.0.smithi163.stdout:3535: done (2 left) 2023-10-09T09:20:23.449 INFO:tasks.rados.rados.0.smithi163.stdout:3537: done (1 left) 2023-10-09T09:20:23.449 INFO:tasks.rados.rados.0.smithi163.stdout:3538: done (0 left) 2023-10-09T09:20:23.449 INFO:tasks.rados.rados.0.smithi163.stdout:3539: read oid 49 snap -1 2023-10-09T09:20:23.450 INFO:tasks.rados.rados.0.smithi163.stdout:3539: expect (ObjNum 1283 snap 437 seq_num 1283) 2023-10-09T09:20:23.450 INFO:tasks.rados.rados.0.smithi163.stdout:3540: snap_create 2023-10-09T09:20:23.460 INFO:tasks.rados.rados.0.smithi163.stdout:3539: expect (ObjNum 1283 snap 437 seq_num 1283) 2023-10-09T09:20:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:20:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:20:23] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:20:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:23 smithi163 bash[75425]: cluster 2023-10-09T09:20:22.439113+0000 osd.5 (osd.5) 489 : cluster 3 failed to encode map e948 with expected crc 2023-10-09T09:20:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:23 smithi163 bash[75425]: cluster 2023-10-09T09:20:22.440072+0000 osd.7 (osd.7) 397 : cluster 3 failed to encode map e948 with expected crc 2023-10-09T09:20:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:23 smithi163 bash[75425]: cluster 2023-10-09T09:20:22.442613+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e948: 8 total, 8 up, 8 in 2023-10-09T09:20:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:23 smithi163 bash[75425]: cluster 2023-10-09T09:20:22.444086+0000 osd.4 (osd.4) 424 : cluster 3 failed to encode map e948 with expected crc 2023-10-09T09:20:23.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:23 smithi163 bash[78107]: cluster 2023-10-09T09:20:22.439113+0000 osd.5 (osd.5) 489 : cluster 3 failed to encode map e948 with expected crc 2023-10-09T09:20:23.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:23 smithi163 bash[78107]: cluster 2023-10-09T09:20:22.440072+0000 osd.7 (osd.7) 397 : cluster 3 failed to encode map e948 with expected crc 2023-10-09T09:20:23.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:23 smithi163 bash[78107]: cluster 2023-10-09T09:20:22.442613+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e948: 8 total, 8 up, 8 in 2023-10-09T09:20:23.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:23 smithi163 bash[78107]: cluster 2023-10-09T09:20:22.444086+0000 osd.4 (osd.4) 424 : cluster 3 failed to encode map e948 with expected crc 2023-10-09T09:20:23.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:23 smithi181 bash[59345]: cluster 2023-10-09T09:20:22.439113+0000 osd.5 (osd.5) 489 : cluster 3 failed to encode map e948 with expected crc 2023-10-09T09:20:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:23 smithi181 bash[59345]: cluster 2023-10-09T09:20:22.440072+0000 osd.7 (osd.7) 397 : cluster 3 failed to encode map e948 with expected crc 2023-10-09T09:20:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:23 smithi181 bash[59345]: cluster 2023-10-09T09:20:22.442613+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e948: 8 total, 8 up, 8 in 2023-10-09T09:20:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:23 smithi181 bash[59345]: cluster 2023-10-09T09:20:22.444086+0000 osd.4 (osd.4) 424 : cluster 3 failed to encode map e948 with expected crc 2023-10-09T09:20:24.449 INFO:tasks.rados.rados.0.smithi163.stdout:3539: done (1 left) 2023-10-09T09:20:24.449 INFO:tasks.rados.rados.0.smithi163.stdout:3540: done (0 left) 2023-10-09T09:20:24.450 INFO:tasks.rados.rados.0.smithi163.stdout:3541: write oid 33 current snap is 446 2023-10-09T09:20:24.450 INFO:tasks.rados.rados.0.smithi163.stdout:3541: seq_num 1319 ranges {568433=582500,1684808=483625} 2023-10-09T09:20:24.458 INFO:tasks.rados.rados.0.smithi163.stdout:3541: writing smithi16367185-33 from 568433 to 1150933 tid 1 2023-10-09T09:20:24.465 INFO:tasks.rados.rados.0.smithi163.stdout:3541: writing smithi16367185-33 from 1684808 to 2168433 tid 2 2023-10-09T09:20:24.466 INFO:tasks.rados.rados.0.smithi163.stdout:3542: write oid 11 current snap is 446 2023-10-09T09:20:24.466 INFO:tasks.rados.rados.0.smithi163.stdout:3542: seq_num 1320 ranges {585240=657749,1743290=422125} 2023-10-09T09:20:24.473 INFO:tasks.rados.rados.0.smithi163.stdout:3542: writing smithi16367185-11 from 585240 to 1242989 tid 1 2023-10-09T09:20:24.479 INFO:tasks.rados.rados.0.smithi163.stdout:3542: writing smithi16367185-11 from 1743290 to 2165415 tid 2 2023-10-09T09:20:24.479 INFO:tasks.rados.rados.0.smithi163.stdout:3543: rollback oid 15 current snap is 446 2023-10-09T09:20:24.480 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 15 to 443 2023-10-09T09:20:24.480 INFO:tasks.rados.rados.0.smithi163.stdout:3541: finishing write tid 1 to smithi16367185-33 2023-10-09T09:20:24.480 INFO:tasks.rados.rados.0.smithi163.stdout:3541: finishing write tid 2 to smithi16367185-33 2023-10-09T09:20:24.480 INFO:tasks.rados.rados.0.smithi163.stdout:3541: finishing write tid 3 to smithi16367185-33 2023-10-09T09:20:24.480 INFO:tasks.rados.rados.0.smithi163.stdout:3541: finishing write tid 4 to smithi16367185-33 2023-10-09T09:20:24.480 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1644 (ObjNum 1319 snap 446 seq_num 1319) dirty exists 2023-10-09T09:20:24.480 INFO:tasks.rados.rados.0.smithi163.stdout:3541: left oid 33 (ObjNum 1319 snap 446 seq_num 1319) 2023-10-09T09:20:24.481 INFO:tasks.rados.rados.0.smithi163.stdout:3541: done (2 left) 2023-10-09T09:20:24.481 INFO:tasks.rados.rados.0.smithi163.stdout:3544: write oid 13 current snap is 446 2023-10-09T09:20:24.481 INFO:tasks.rados.rados.0.smithi163.stdout:3544: seq_num 1321 ranges {602047=732998,1801772=400275} 2023-10-09T09:20:24.486 INFO:tasks.rados.rados.0.smithi163.stdout:3544: writing smithi16367185-13 from 602047 to 1335045 tid 1 2023-10-09T09:20:24.491 INFO:tasks.rados.rados.0.smithi163.stdout:3544: writing smithi16367185-13 from 1801772 to 2202047 tid 2 2023-10-09T09:20:24.491 INFO:tasks.rados.rados.0.smithi163.stdout:3545: rollback oid 6 current snap is 446 2023-10-09T09:20:24.491 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 6 to 427 2023-10-09T09:20:24.491 INFO:tasks.rados.rados.0.smithi163.stdout:3542: finishing write tid 1 to smithi16367185-11 2023-10-09T09:20:24.491 INFO:tasks.rados.rados.0.smithi163.stdout:3543: finishing rollback tid 0 to smithi16367185-15 2023-10-09T09:20:24.491 INFO:tasks.rados.rados.0.smithi163.stdout:3542: finishing write tid 2 to smithi16367185-11 2023-10-09T09:20:24.492 INFO:tasks.rados.rados.0.smithi163.stdout:3543: finishing rollback tid 1 to smithi16367185-15 2023-10-09T09:20:24.492 INFO:tasks.rados.rados.0.smithi163.stdout:3543: finishing rollback tid 2 to smithi16367185-15 2023-10-09T09:20:24.492 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 1791 (ObjNum 1303 snap 440 seq_num 1303) dirty exists 2023-10-09T09:20:24.492 INFO:tasks.rados.rados.0.smithi163.stdout:3542: finishing write tid 3 to smithi16367185-11 2023-10-09T09:20:24.492 INFO:tasks.rados.rados.0.smithi163.stdout:3542: finishing write tid 4 to smithi16367185-11 2023-10-09T09:20:24.492 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 961 (ObjNum 1320 snap 446 seq_num 1320) dirty exists 2023-10-09T09:20:24.492 INFO:tasks.rados.rados.0.smithi163.stdout:3542: left oid 11 (ObjNum 1320 snap 446 seq_num 1320) 2023-10-09T09:20:24.492 INFO:tasks.rados.rados.0.smithi163.stdout:3542: done (3 left) 2023-10-09T09:20:24.492 INFO:tasks.rados.rados.0.smithi163.stdout:3543: done (2 left) 2023-10-09T09:20:24.492 INFO:tasks.rados.rados.0.smithi163.stdout:3546: delete oid 31 current snap is 446 2023-10-09T09:20:24.493 INFO:tasks.rados.rados.0.smithi163.stdout:3546: done (2 left) 2023-10-09T09:20:24.493 INFO:tasks.rados.rados.0.smithi163.stdout:3547: write oid 8 current snap is 446 2023-10-09T09:20:24.493 INFO:tasks.rados.rados.0.smithi163.stdout:3547: seq_num 1323 ranges {635661=599849,1918736=316925} 2023-10-09T09:20:24.502 INFO:tasks.rados.rados.0.smithi163.stdout:3547: writing smithi16367185-8 from 635661 to 1235510 tid 1 2023-10-09T09:20:24.508 INFO:tasks.rados.rados.0.smithi163.stdout:3547: writing smithi16367185-8 from 1918736 to 2235661 tid 2 2023-10-09T09:20:24.508 INFO:tasks.rados.rados.0.smithi163.stdout:3548: write oid 40 current snap is 446 2023-10-09T09:20:24.508 INFO:tasks.rados.rados.0.smithi163.stdout:3544: finishing write tid 1 to smithi16367185-13 2023-10-09T09:20:24.508 INFO:tasks.rados.rados.0.smithi163.stdout:3548: seq_num 1324 ranges {652468=675098,1977218=275250} 2023-10-09T09:20:24.515 INFO:tasks.rados.rados.0.smithi163.stdout:3548: writing smithi16367185-40 from 652468 to 1327566 tid 1 2023-10-09T09:20:24.521 INFO:tasks.rados.rados.0.smithi163.stdout:3548: writing smithi16367185-40 from 1977218 to 2252468 tid 2 2023-10-09T09:20:24.521 INFO:tasks.rados.rados.0.smithi163.stdout:3549: read oid 41 snap -1 2023-10-09T09:20:24.521 INFO:tasks.rados.rados.0.smithi163.stdout:3549: expect (ObjNum 1306 snap 440 seq_num 1306) 2023-10-09T09:20:24.521 INFO:tasks.rados.rados.0.smithi163.stdout:3550: snap_remove snap 435 2023-10-09T09:20:24.521 INFO:tasks.rados.rados.0.smithi163.stdout:3545: finishing rollback tid 0 to smithi16367185-6 2023-10-09T09:20:24.521 INFO:tasks.rados.rados.0.smithi163.stdout:3544: finishing write tid 2 to smithi16367185-13 2023-10-09T09:20:24.521 INFO:tasks.rados.rados.0.smithi163.stdout:3544: finishing write tid 3 to smithi16367185-13 2023-10-09T09:20:24.522 INFO:tasks.rados.rados.0.smithi163.stdout:3545: finishing rollback tid 1 to smithi16367185-6 2023-10-09T09:20:24.522 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 382 (ObjNum 699737696 snap 21875 seq_num 698725104) dirty dne 2023-10-09T09:20:24.522 INFO:tasks.rados.rados.0.smithi163.stdout:3544: finishing write tid 4 to smithi16367185-13 2023-10-09T09:20:24.522 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 708 (ObjNum 1321 snap 446 seq_num 1321) dirty exists 2023-10-09T09:20:24.522 INFO:tasks.rados.rados.0.smithi163.stdout:3544: left oid 13 (ObjNum 1321 snap 446 seq_num 1321) 2023-10-09T09:20:24.522 INFO:tasks.rados.rados.0.smithi163.stdout:3547: finishing write tid 1 to smithi16367185-8 2023-10-09T09:20:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:24 smithi163 bash[75425]: cluster 2023-10-09T09:20:23.441727+0000 osd.5 (osd.5) 490 : cluster 3 failed to encode map e949 with expected crc 2023-10-09T09:20:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:24 smithi163 bash[75425]: cluster 2023-10-09T09:20:23.444475+0000 osd.7 (osd.7) 398 : cluster 3 failed to encode map e949 with expected crc 2023-10-09T09:20:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:24 smithi163 bash[75425]: cluster 2023-10-09T09:20:23.445999+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e949: 8 total, 8 up, 8 in 2023-10-09T09:20:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:24 smithi163 bash[75425]: cluster 2023-10-09T09:20:23.455666+0000 mgr.y (mgr.34107) 415 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 242 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2023-10-09T09:20:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:24 smithi163 bash[75425]: cluster 2023-10-09T09:20:24.448841+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e950: 8 total, 8 up, 8 in 2023-10-09T09:20:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:24 smithi163 bash[78107]: cluster 2023-10-09T09:20:23.441727+0000 osd.5 (osd.5) 490 : cluster 3 failed to encode map e949 with expected crc 2023-10-09T09:20:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:24 smithi163 bash[78107]: cluster 2023-10-09T09:20:23.444475+0000 osd.7 (osd.7) 398 : cluster 3 failed to encode map e949 with expected crc 2023-10-09T09:20:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:24 smithi163 bash[78107]: cluster 2023-10-09T09:20:23.445999+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e949: 8 total, 8 up, 8 in 2023-10-09T09:20:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:24 smithi163 bash[78107]: cluster 2023-10-09T09:20:23.455666+0000 mgr.y (mgr.34107) 415 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 242 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2023-10-09T09:20:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:24 smithi163 bash[78107]: cluster 2023-10-09T09:20:24.448841+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e950: 8 total, 8 up, 8 in 2023-10-09T09:20:24.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:24 smithi181 bash[59345]: cluster 2023-10-09T09:20:23.441727+0000 osd.5 (osd.5) 490 : cluster 3 failed to encode map e949 with expected crc 2023-10-09T09:20:24.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:24 smithi181 bash[59345]: cluster 2023-10-09T09:20:23.444475+0000 osd.7 (osd.7) 398 : cluster 3 failed to encode map e949 with expected crc 2023-10-09T09:20:24.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:24 smithi181 bash[59345]: cluster 2023-10-09T09:20:23.445999+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e949: 8 total, 8 up, 8 in 2023-10-09T09:20:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:24 smithi181 bash[59345]: cluster 2023-10-09T09:20:23.455666+0000 mgr.y (mgr.34107) 415 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 242 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2023-10-09T09:20:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:24 smithi181 bash[59345]: cluster 2023-10-09T09:20:24.448841+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e950: 8 total, 8 up, 8 in 2023-10-09T09:20:25.311 INFO:tasks.rados.rados.0.smithi163.stdout:3544: done (5 left) 2023-10-09T09:20:25.312 INFO:tasks.rados.rados.0.smithi163.stdout:3545: done (4 left) 2023-10-09T09:20:25.312 INFO:tasks.rados.rados.0.smithi163.stdout:3550: done (3 left) 2023-10-09T09:20:25.312 INFO:tasks.rados.rados.0.smithi163.stdout:3551: snap_remove snap 438 2023-10-09T09:20:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:25 smithi163 bash[78107]: cluster 2023-10-09T09:20:24.444112+0000 osd.5 (osd.5) 491 : cluster 3 failed to encode map e950 with expected crc 2023-10-09T09:20:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:25 smithi163 bash[78107]: cluster 2023-10-09T09:20:24.469930+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e949 with expected crc 2023-10-09T09:20:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:25 smithi163 bash[78107]: cluster 2023-10-09T09:20:24.475767+0000 osd.7 (osd.7) 399 : cluster 3 failed to encode map e950 with expected crc 2023-10-09T09:20:25.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:25 smithi163 bash[78107]: cluster 2023-10-09T09:20:25.311244+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e951: 8 total, 8 up, 8 in 2023-10-09T09:20:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:25 smithi163 bash[78107]: audit 2023-10-09T09:20:25.351648+0000 mon.a (mon.0) 1171 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:25 smithi163 bash[75425]: cluster 2023-10-09T09:20:24.444112+0000 osd.5 (osd.5) 491 : cluster 3 failed to encode map e950 with expected crc 2023-10-09T09:20:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:25 smithi163 bash[75425]: cluster 2023-10-09T09:20:24.469930+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e949 with expected crc 2023-10-09T09:20:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:25 smithi163 bash[75425]: cluster 2023-10-09T09:20:24.475767+0000 osd.7 (osd.7) 399 : cluster 3 failed to encode map e950 with expected crc 2023-10-09T09:20:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:25 smithi163 bash[75425]: cluster 2023-10-09T09:20:25.311244+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e951: 8 total, 8 up, 8 in 2023-10-09T09:20:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:25 smithi163 bash[75425]: audit 2023-10-09T09:20:25.351648+0000 mon.a (mon.0) 1171 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:25 smithi181 bash[59345]: cluster 2023-10-09T09:20:24.444112+0000 osd.5 (osd.5) 491 : cluster 3 failed to encode map e950 with expected crc 2023-10-09T09:20:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:25 smithi181 bash[59345]: cluster 2023-10-09T09:20:24.469930+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e949 with expected crc 2023-10-09T09:20:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:25 smithi181 bash[59345]: cluster 2023-10-09T09:20:24.475767+0000 osd.7 (osd.7) 399 : cluster 3 failed to encode map e950 with expected crc 2023-10-09T09:20:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:25 smithi181 bash[59345]: cluster 2023-10-09T09:20:25.311244+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e951: 8 total, 8 up, 8 in 2023-10-09T09:20:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:25 smithi181 bash[59345]: audit 2023-10-09T09:20:25.351648+0000 mon.a (mon.0) 1171 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:26.313 INFO:tasks.rados.rados.0.smithi163.stdout:3551: done (3 left) 2023-10-09T09:20:26.313 INFO:tasks.rados.rados.0.smithi163.stdout:3552: write oid 10 current snap is 446 2023-10-09T09:20:26.313 INFO:tasks.rados.rados.0.smithi163.stdout:3552: seq_num 1325 ranges {669275=750347,2035700=233575} 2023-10-09T09:20:26.324 INFO:tasks.rados.rados.0.smithi163.stdout:3552: writing smithi16367185-10 from 669275 to 1419622 tid 1 2023-10-09T09:20:26.330 INFO:tasks.rados.rados.0.smithi163.stdout:3552: writing smithi16367185-10 from 2035700 to 2269275 tid 2 2023-10-09T09:20:26.330 INFO:tasks.rados.rados.0.smithi163.stdout:3553: write oid 35 current snap is 446 2023-10-09T09:20:26.330 INFO:tasks.rados.rados.0.smithi163.stdout:3547: finishing write tid 2 to smithi16367185-8 2023-10-09T09:20:26.330 INFO:tasks.rados.rados.0.smithi163.stdout:3547: finishing write tid 3 to smithi16367185-8 2023-10-09T09:20:26.330 INFO:tasks.rados.rados.0.smithi163.stdout:3547: finishing write tid 4 to smithi16367185-8 2023-10-09T09:20:26.330 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1794 (ObjNum 1323 snap 446 seq_num 1323) dirty exists 2023-10-09T09:20:26.330 INFO:tasks.rados.rados.0.smithi163.stdout:3547: left oid 8 (ObjNum 1323 snap 446 seq_num 1323) 2023-10-09T09:20:26.330 INFO:tasks.rados.rados.0.smithi163.stdout:3548: finishing write tid 1 to smithi16367185-40 2023-10-09T09:20:26.331 INFO:tasks.rados.rados.0.smithi163.stdout:3553: seq_num 1326 ranges {686082=425596,1577829=433132} 2023-10-09T09:20:26.336 INFO:tasks.rados.rados.0.smithi163.stdout:3553: writing smithi16367185-35 from 686082 to 1111678 tid 1 2023-10-09T09:20:26.342 INFO:tasks.rados.rados.0.smithi163.stdout:3553: writing smithi16367185-35 from 1577829 to 2010961 tid 2 2023-10-09T09:20:26.342 INFO:tasks.rados.rados.0.smithi163.stdout:3547: done (4 left) 2023-10-09T09:20:26.343 INFO:tasks.rados.rados.0.smithi163.stdout:3554: delete oid 45 current snap is 446 2023-10-09T09:20:26.343 INFO:tasks.rados.rados.0.smithi163.stdout:3548: finishing write tid 2 to smithi16367185-40 2023-10-09T09:20:26.343 INFO:tasks.rados.rados.0.smithi163.stdout:3548: finishing write tid 3 to smithi16367185-40 2023-10-09T09:20:26.343 INFO:tasks.rados.rados.0.smithi163.stdout:3548: finishing write tid 4 to smithi16367185-40 2023-10-09T09:20:26.343 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1797 (ObjNum 1324 snap 446 seq_num 1324) dirty exists 2023-10-09T09:20:26.343 INFO:tasks.rados.rados.0.smithi163.stdout:3548: left oid 40 (ObjNum 1324 snap 446 seq_num 1324) 2023-10-09T09:20:26.343 INFO:tasks.rados.rados.0.smithi163.stdout:3549: expect (ObjNum 1306 snap 440 seq_num 1306) 2023-10-09T09:20:26.525 INFO:tasks.rados.rados.0.smithi163.stdout:3552: finishing write tid 1 to smithi16367185-10 2023-10-09T09:20:26.525 INFO:tasks.rados.rados.0.smithi163.stdout:3552: finishing write tid 2 to smithi16367185-10 2023-10-09T09:20:26.525 INFO:tasks.rados.rados.0.smithi163.stdout:3552: finishing write tid 3 to smithi16367185-10 2023-10-09T09:20:26.526 INFO:tasks.rados.rados.0.smithi163.stdout:3552: finishing write tid 4 to smithi16367185-10 2023-10-09T09:20:26.526 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 10 v 1231 (ObjNum 1325 snap 446 seq_num 1325) dirty exists 2023-10-09T09:20:26.526 INFO:tasks.rados.rados.0.smithi163.stdout:3552: left oid 10 (ObjNum 1325 snap 446 seq_num 1325) 2023-10-09T09:20:26.526 INFO:tasks.rados.rados.0.smithi163.stdout:3553: finishing write tid 1 to smithi16367185-35 2023-10-09T09:20:26.526 INFO:tasks.rados.rados.0.smithi163.stdout:3553: finishing write tid 2 to smithi16367185-35 2023-10-09T09:20:26.526 INFO:tasks.rados.rados.0.smithi163.stdout:3553: finishing write tid 3 to smithi16367185-35 2023-10-09T09:20:26.526 INFO:tasks.rados.rados.0.smithi163.stdout:3553: finishing write tid 4 to smithi16367185-35 2023-10-09T09:20:26.526 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 966 (ObjNum 1326 snap 446 seq_num 1326) dirty exists 2023-10-09T09:20:26.527 INFO:tasks.rados.rados.0.smithi163.stdout:3553: left oid 35 (ObjNum 1326 snap 446 seq_num 1326) 2023-10-09T09:20:26.527 INFO:tasks.rados.rados.0.smithi163.stdout:3548: done (4 left) 2023-10-09T09:20:26.527 INFO:tasks.rados.rados.0.smithi163.stdout:3549: done (3 left) 2023-10-09T09:20:26.527 INFO:tasks.rados.rados.0.smithi163.stdout:3552: done (2 left) 2023-10-09T09:20:26.527 INFO:tasks.rados.rados.0.smithi163.stdout:3553: done (1 left) 2023-10-09T09:20:26.527 INFO:tasks.rados.rados.0.smithi163.stdout:3554: done (0 left) 2023-10-09T09:20:26.527 INFO:tasks.rados.rados.0.smithi163.stdout:3555: write oid 26 current snap is 446 2023-10-09T09:20:26.527 INFO:tasks.rados.rados.0.smithi163.stdout:3555: seq_num 1328 ranges {719696=576094,2094793=224903} 2023-10-09T09:20:26.538 INFO:tasks.rados.rados.0.smithi163.stdout:3555: writing smithi16367185-26 from 719696 to 1295790 tid 1 2023-10-09T09:20:26.545 INFO:tasks.rados.rados.0.smithi163.stdout:3555: writing smithi16367185-26 from 2094793 to 2319696 tid 2 2023-10-09T09:20:26.545 INFO:tasks.rados.rados.0.smithi163.stdout:3556: delete oid 37 current snap is 446 2023-10-09T09:20:26.545 INFO:tasks.rados.rados.0.smithi163.stdout:3555: finishing write tid 1 to smithi16367185-26 2023-10-09T09:20:26.550 INFO:tasks.rados.rados.0.smithi163.stdout:3555: finishing write tid 2 to smithi16367185-26 2023-10-09T09:20:26.550 INFO:tasks.rados.rados.0.smithi163.stdout:3556: done (1 left) 2023-10-09T09:20:26.550 INFO:tasks.rados.rados.0.smithi163.stdout:3557: snap_create 2023-10-09T09:20:26.550 INFO:tasks.rados.rados.0.smithi163.stdout:3555: finishing write tid 3 to smithi16367185-26 2023-10-09T09:20:26.551 INFO:tasks.rados.rados.0.smithi163.stdout:3555: finishing write tid 4 to smithi16367185-26 2023-10-09T09:20:26.551 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 1058 (ObjNum 1328 snap 446 seq_num 1328) dirty exists 2023-10-09T09:20:26.551 INFO:tasks.rados.rados.0.smithi163.stdout:3555: left oid 26 (ObjNum 1328 snap 446 seq_num 1328) 2023-10-09T09:20:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:26 smithi163 bash[78107]: cluster 2023-10-09T09:20:25.307503+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e951 with expected crc 2023-10-09T09:20:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:26 smithi163 bash[78107]: cluster 2023-10-09T09:20:25.307615+0000 osd.5 (osd.5) 492 : cluster 3 failed to encode map e951 with expected crc 2023-10-09T09:20:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:26 smithi163 bash[78107]: cluster 2023-10-09T09:20:25.456312+0000 mgr.y (mgr.34107) 416 : cluster 0 pgmap v980: 89 pgs: 89 active+clean; 248 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 921 KiB/s wr, 8 op/s 2023-10-09T09:20:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:26 smithi163 bash[78107]: cluster 2023-10-09T09:20:26.312037+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e952: 8 total, 8 up, 8 in 2023-10-09T09:20:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:26 smithi163 bash[78107]: cluster 2023-10-09T09:20:26.313375+0000 osd.4 (osd.4) 425 : cluster 3 failed to encode map e952 with expected crc 2023-10-09T09:20:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:26 smithi163 bash[75425]: cluster 2023-10-09T09:20:25.307503+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e951 with expected crc 2023-10-09T09:20:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:26 smithi163 bash[75425]: cluster 2023-10-09T09:20:25.307615+0000 osd.5 (osd.5) 492 : cluster 3 failed to encode map e951 with expected crc 2023-10-09T09:20:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:26 smithi163 bash[75425]: cluster 2023-10-09T09:20:25.456312+0000 mgr.y (mgr.34107) 416 : cluster 0 pgmap v980: 89 pgs: 89 active+clean; 248 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 921 KiB/s wr, 8 op/s 2023-10-09T09:20:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:26 smithi163 bash[75425]: cluster 2023-10-09T09:20:26.312037+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e952: 8 total, 8 up, 8 in 2023-10-09T09:20:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:26 smithi163 bash[75425]: cluster 2023-10-09T09:20:26.313375+0000 osd.4 (osd.4) 425 : cluster 3 failed to encode map e952 with expected crc 2023-10-09T09:20:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:26 smithi181 bash[59345]: cluster 2023-10-09T09:20:25.307503+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e951 with expected crc 2023-10-09T09:20:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:26 smithi181 bash[59345]: cluster 2023-10-09T09:20:25.307615+0000 osd.5 (osd.5) 492 : cluster 3 failed to encode map e951 with expected crc 2023-10-09T09:20:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:26 smithi181 bash[59345]: cluster 2023-10-09T09:20:25.456312+0000 mgr.y (mgr.34107) 416 : cluster 0 pgmap v980: 89 pgs: 89 active+clean; 248 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 921 KiB/s wr, 8 op/s 2023-10-09T09:20:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:26 smithi181 bash[59345]: cluster 2023-10-09T09:20:26.312037+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e952: 8 total, 8 up, 8 in 2023-10-09T09:20:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:26 smithi181 bash[59345]: cluster 2023-10-09T09:20:26.313375+0000 osd.4 (osd.4) 425 : cluster 3 failed to encode map e952 with expected crc 2023-10-09T09:20:27.481 INFO:tasks.rados.rados.0.smithi163.stdout:3555: done (1 left) 2023-10-09T09:20:27.481 INFO:tasks.rados.rados.0.smithi163.stdout:3557: done (0 left) 2023-10-09T09:20:27.481 INFO:tasks.rados.rados.0.smithi163.stdout:3558: snap_create 2023-10-09T09:20:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:27 smithi163 bash[75425]: cluster 2023-10-09T09:20:26.307430+0000 osd.5 (osd.5) 493 : cluster 3 failed to encode map e952 with expected crc 2023-10-09T09:20:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:27 smithi163 bash[78107]: cluster 2023-10-09T09:20:26.307430+0000 osd.5 (osd.5) 493 : cluster 3 failed to encode map e952 with expected crc 2023-10-09T09:20:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:27 smithi181 bash[59345]: cluster 2023-10-09T09:20:26.307430+0000 osd.5 (osd.5) 493 : cluster 3 failed to encode map e952 with expected crc 2023-10-09T09:20:28.485 INFO:tasks.rados.rados.0.smithi163.stdout:3558: done (0 left) 2023-10-09T09:20:28.485 INFO:tasks.rados.rados.0.smithi163.stdout:3559: write oid 17 current snap is 448 2023-10-09T09:20:28.485 INFO:tasks.rados.rados.0.smithi163.stdout:3559: seq_num 1330 ranges {753310=726592,2095404=257906} 2023-10-09T09:20:28.498 INFO:tasks.rados.rados.0.smithi163.stdout:3559: writing smithi16367185-17 from 753310 to 1479902 tid 1 2023-10-09T09:20:28.503 INFO:tasks.rados.rados.0.smithi163.stdout:3559: writing smithi16367185-17 from 2095404 to 2353310 tid 2 2023-10-09T09:20:28.503 INFO:tasks.rados.rados.0.smithi163.stdout:3560: rollback oid 8 current snap is 448 2023-10-09T09:20:28.503 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 8 to 409 2023-10-09T09:20:28.503 INFO:tasks.rados.rados.0.smithi163.stdout:3561: write oid 34 current snap is 448 2023-10-09T09:20:28.503 INFO:tasks.rados.rados.0.smithi163.stdout:3561: seq_num 1331 ranges {770117=518194,1870239=499878} 2023-10-09T09:20:28.511 INFO:tasks.rados.rados.0.smithi163.stdout:3561: writing smithi16367185-34 from 770117 to 1288311 tid 1 2023-10-09T09:20:28.517 INFO:tasks.rados.rados.0.smithi163.stdout:3561: writing smithi16367185-34 from 1870239 to 2370117 tid 2 2023-10-09T09:20:28.517 INFO:tasks.rados.rados.0.smithi163.stdout:3562: delete oid 11 current snap is 448 2023-10-09T09:20:28.517 INFO:tasks.rados.rados.0.smithi163.stdout:3559: finishing write tid 1 to smithi16367185-17 2023-10-09T09:20:28.517 INFO:tasks.rados.rados.0.smithi163.stdout:3559: finishing write tid 2 to smithi16367185-17 2023-10-09T09:20:28.518 INFO:tasks.rados.rados.0.smithi163.stdout:3559: finishing write tid 3 to smithi16367185-17 2023-10-09T09:20:28.518 INFO:tasks.rados.rados.0.smithi163.stdout:3559: finishing write tid 4 to smithi16367185-17 2023-10-09T09:20:28.518 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1811 (ObjNum 1330 snap 448 seq_num 1330) dirty exists 2023-10-09T09:20:28.518 INFO:tasks.rados.rados.0.smithi163.stdout:3559: left oid 17 (ObjNum 1330 snap 448 seq_num 1330) 2023-10-09T09:20:28.518 INFO:tasks.rados.rados.0.smithi163.stdout:3560: finishing rollback tid 0 to smithi16367185-8 2023-10-09T09:20:28.518 INFO:tasks.rados.rados.0.smithi163.stdout:3561: finishing write tid 1 to smithi16367185-34 2023-10-09T09:20:28.521 INFO:tasks.rados.rados.0.smithi163.stdout:3559: done (3 left) 2023-10-09T09:20:28.521 INFO:tasks.rados.rados.0.smithi163.stdout:3562: done (2 left) 2023-10-09T09:20:28.522 INFO:tasks.rados.rados.0.smithi163.stdout:3563: read oid 39 snap -1 2023-10-09T09:20:28.522 INFO:tasks.rados.rados.0.smithi163.stdout:3563: expect (ObjNum 1310 snap 443 seq_num 1310) 2023-10-09T09:20:28.522 INFO:tasks.rados.rados.0.smithi163.stdout:3564: snap_remove snap 442 2023-10-09T09:20:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:28 smithi181 bash[59345]: cluster 2023-10-09T09:20:27.457278+0000 mgr.y (mgr.34107) 417 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 256 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2023-10-09T09:20:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:28 smithi181 bash[59345]: cluster 2023-10-09T09:20:27.477410+0000 osd.5 (osd.5) 494 : cluster 3 failed to encode map e953 with expected crc 2023-10-09T09:20:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:28 smithi181 bash[59345]: cluster 2023-10-09T09:20:27.477828+0000 osd.4 (osd.4) 426 : cluster 3 failed to encode map e953 with expected crc 2023-10-09T09:20:28.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:28 smithi181 bash[59345]: cluster 2023-10-09T09:20:27.480604+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e953: 8 total, 8 up, 8 in 2023-10-09T09:20:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:28 smithi163 bash[75425]: cluster 2023-10-09T09:20:27.457278+0000 mgr.y (mgr.34107) 417 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 256 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2023-10-09T09:20:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:28 smithi163 bash[75425]: cluster 2023-10-09T09:20:27.477410+0000 osd.5 (osd.5) 494 : cluster 3 failed to encode map e953 with expected crc 2023-10-09T09:20:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:28 smithi163 bash[75425]: cluster 2023-10-09T09:20:27.477828+0000 osd.4 (osd.4) 426 : cluster 3 failed to encode map e953 with expected crc 2023-10-09T09:20:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:28 smithi163 bash[75425]: cluster 2023-10-09T09:20:27.480604+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e953: 8 total, 8 up, 8 in 2023-10-09T09:20:28.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:28 smithi163 bash[78107]: cluster 2023-10-09T09:20:27.457278+0000 mgr.y (mgr.34107) 417 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 256 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2023-10-09T09:20:28.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:28 smithi163 bash[78107]: cluster 2023-10-09T09:20:27.477410+0000 osd.5 (osd.5) 494 : cluster 3 failed to encode map e953 with expected crc 2023-10-09T09:20:28.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:28 smithi163 bash[78107]: cluster 2023-10-09T09:20:27.477828+0000 osd.4 (osd.4) 426 : cluster 3 failed to encode map e953 with expected crc 2023-10-09T09:20:28.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:28 smithi163 bash[78107]: cluster 2023-10-09T09:20:27.480604+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e953: 8 total, 8 up, 8 in 2023-10-09T09:20:29.511 INFO:tasks.rados.rados.0.smithi163.stdout:3564: done (3 left) 2023-10-09T09:20:29.511 INFO:tasks.rados.rados.0.smithi163.stdout:3565: delete oid 40 current snap is 448 2023-10-09T09:20:29.511 INFO:tasks.rados.rados.0.smithi163.stdout:3561: finishing write tid 2 to smithi16367185-34 2023-10-09T09:20:29.511 INFO:tasks.rados.rados.0.smithi163.stdout:3560: finishing rollback tid 1 to smithi16367185-8 2023-10-09T09:20:29.511 INFO:tasks.rados.rados.0.smithi163.stdout:3560: finishing rollback tid 2 to smithi16367185-8 2023-10-09T09:20:29.511 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1815 (ObjNum 1215 snap 404 seq_num 1215) dirty exists 2023-10-09T09:20:29.511 INFO:tasks.rados.rados.0.smithi163.stdout:3561: finishing write tid 3 to smithi16367185-34 2023-10-09T09:20:29.511 INFO:tasks.rados.rados.0.smithi163.stdout:3561: finishing write tid 4 to smithi16367185-34 2023-10-09T09:20:29.511 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1658 (ObjNum 1331 snap 448 seq_num 1331) dirty exists 2023-10-09T09:20:29.512 INFO:tasks.rados.rados.0.smithi163.stdout:3561: left oid 34 (ObjNum 1331 snap 448 seq_num 1331) 2023-10-09T09:20:29.512 INFO:tasks.rados.rados.0.smithi163.stdout:3563: expect (ObjNum 1310 snap 443 seq_num 1310) 2023-10-09T09:20:29.604 INFO:tasks.rados.rados.0.smithi163.stdout:3560: done (3 left) 2023-10-09T09:20:29.604 INFO:tasks.rados.rados.0.smithi163.stdout:3561: done (2 left) 2023-10-09T09:20:29.604 INFO:tasks.rados.rados.0.smithi163.stdout:3563: done (1 left) 2023-10-09T09:20:29.604 INFO:tasks.rados.rados.0.smithi163.stdout:3565: done (0 left) 2023-10-09T09:20:29.604 INFO:tasks.rados.rados.0.smithi163.stdout:3566: read oid 21 snap -1 2023-10-09T09:20:29.604 INFO:tasks.rados.rados.0.smithi163.stdout:3566: expect (ObjNum 1223 snap 408 seq_num 1223) 2023-10-09T09:20:29.604 INFO:tasks.rados.rados.0.smithi163.stdout:3567: delete oid 36 current snap is 448 2023-10-09T09:20:29.608 INFO:tasks.rados.rados.0.smithi163.stdout:3567: done (1 left) 2023-10-09T09:20:29.608 INFO:tasks.rados.rados.0.smithi163.stdout:3568: read oid 31 snap -1 2023-10-09T09:20:29.608 INFO:tasks.rados.rados.0.smithi163.stdout:3568: expect deleted 2023-10-09T09:20:29.608 INFO:tasks.rados.rados.0.smithi163.stdout:3569: write oid 44 current snap is 448 2023-10-09T09:20:29.608 INFO:tasks.rados.rados.0.smithi163.stdout:3569: seq_num 1335 ranges {437345=419190,1587814=591466} 2023-10-09T09:20:29.616 INFO:tasks.rados.rados.0.smithi163.stdout:3569: writing smithi16367185-44 from 437345 to 856535 tid 1 2023-10-09T09:20:29.625 INFO:tasks.rados.rados.0.smithi163.stdout:3569: writing smithi16367185-44 from 1587814 to 2179280 tid 2 2023-10-09T09:20:29.625 INFO:tasks.rados.rados.0.smithi163.stdout:3570: read oid 24 snap -1 2023-10-09T09:20:29.625 INFO:tasks.rados.rados.0.smithi163.stdout:3570: expect (ObjNum 1210 snap 404 seq_num 1210) 2023-10-09T09:20:29.625 INFO:tasks.rados.rados.0.smithi163.stdout:3571: read oid 17 snap -1 2023-10-09T09:20:29.625 INFO:tasks.rados.rados.0.smithi163.stdout:3571: expect (ObjNum 1330 snap 448 seq_num 1330) 2023-10-09T09:20:29.625 INFO:tasks.rados.rados.0.smithi163.stdout:3572: read oid 32 snap -1 2023-10-09T09:20:29.626 INFO:tasks.rados.rados.0.smithi163.stdout:3572: expect (ObjNum 1218 snap 404 seq_num 1218) 2023-10-09T09:20:29.626 INFO:tasks.rados.rados.0.smithi163.stdout:3573: read oid 9 snap -1 2023-10-09T09:20:29.626 INFO:tasks.rados.rados.0.smithi163.stdout:3573: expect deleted 2023-10-09T09:20:29.626 INFO:tasks.rados.rados.0.smithi163.stdout:3574: rollback oid 47 current snap is 448 2023-10-09T09:20:29.626 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 47 to 443 2023-10-09T09:20:29.626 INFO:tasks.rados.rados.0.smithi163.stdout:3566: expect (ObjNum 1223 snap 408 seq_num 1223) 2023-10-09T09:20:29.784 INFO:tasks.rados.rados.0.smithi163.stdout:3569: finishing write tid 1 to smithi16367185-44 2023-10-09T09:20:29.784 INFO:tasks.rados.rados.0.smithi163.stdout:3570: expect (ObjNum 1210 snap 404 seq_num 1210) 2023-10-09T09:20:29.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:29 smithi181 bash[59345]: cluster 2023-10-09T09:20:28.480163+0000 osd.5 (osd.5) 495 : cluster 3 failed to encode map e954 with expected crc 2023-10-09T09:20:29.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:29 smithi181 bash[59345]: cluster 2023-10-09T09:20:28.480753+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e954 with expected crc 2023-10-09T09:20:29.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:29 smithi181 bash[59345]: cluster 2023-10-09T09:20:28.484916+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e954: 8 total, 8 up, 8 in 2023-10-09T09:20:29.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:29 smithi181 bash[59345]: cluster 2023-10-09T09:20:28.500350+0000 osd.7 (osd.7) 400 : cluster 3 failed to encode map e954 with expected crc 2023-10-09T09:20:29.863 INFO:tasks.rados.rados.0.smithi163.stdout:3571: expect (ObjNum 1330 snap 448 seq_num 1330) 2023-10-09T09:20:29.968 INFO:tasks.rados.rados.0.smithi163.stdout:3569: finishing write tid 2 to smithi16367185-44 2023-10-09T09:20:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:29 smithi163 bash[75425]: cluster 2023-10-09T09:20:28.480163+0000 osd.5 (osd.5) 495 : cluster 3 failed to encode map e954 with expected crc 2023-10-09T09:20:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:29 smithi163 bash[75425]: cluster 2023-10-09T09:20:28.480753+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e954 with expected crc 2023-10-09T09:20:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:29 smithi163 bash[75425]: cluster 2023-10-09T09:20:28.484916+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e954: 8 total, 8 up, 8 in 2023-10-09T09:20:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:29 smithi163 bash[75425]: cluster 2023-10-09T09:20:28.500350+0000 osd.7 (osd.7) 400 : cluster 3 failed to encode map e954 with expected crc 2023-10-09T09:20:29.969 INFO:tasks.rados.rados.0.smithi163.stdout:3569: finishing write tid 3 to smithi16367185-44 2023-10-09T09:20:29.969 INFO:tasks.rados.rados.0.smithi163.stdout:3569: finishing write tid 4 to smithi16367185-44 2023-10-09T09:20:29.969 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 2667 (ObjNum 1335 snap 448 seq_num 1335) dirty exists 2023-10-09T09:20:29.969 INFO:tasks.rados.rados.0.smithi163.stdout:3569: left oid 44 (ObjNum 1335 snap 448 seq_num 1335) 2023-10-09T09:20:29.969 INFO:tasks.rados.rados.0.smithi163.stdout:3572: expect (ObjNum 1218 snap 404 seq_num 1218) 2023-10-09T09:20:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:29 smithi163 bash[78107]: cluster 2023-10-09T09:20:28.480163+0000 osd.5 (osd.5) 495 : cluster 3 failed to encode map e954 with expected crc 2023-10-09T09:20:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:29 smithi163 bash[78107]: cluster 2023-10-09T09:20:28.480753+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e954 with expected crc 2023-10-09T09:20:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:29 smithi163 bash[78107]: cluster 2023-10-09T09:20:28.484916+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e954: 8 total, 8 up, 8 in 2023-10-09T09:20:29.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:29 smithi163 bash[78107]: cluster 2023-10-09T09:20:28.500350+0000 osd.7 (osd.7) 400 : cluster 3 failed to encode map e954 with expected crc 2023-10-09T09:20:30.103 INFO:tasks.rados.rados.0.smithi163.stdout:3574: finishing rollback tid 0 to smithi16367185-47 2023-10-09T09:20:30.103 INFO:tasks.rados.rados.0.smithi163.stdout:3574: finishing rollback tid 1 to smithi16367185-47 2023-10-09T09:20:30.103 INFO:tasks.rados.rados.0.smithi163.stdout:3574: finishing rollback tid 2 to smithi16367185-47 2023-10-09T09:20:30.103 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 2671 (ObjNum 1224 snap 408 seq_num 1224) dirty exists 2023-10-09T09:20:30.103 INFO:tasks.rados.rados.0.smithi163.stdout:3566: done (7 left) 2023-10-09T09:20:30.103 INFO:tasks.rados.rados.0.smithi163.stdout:3568: done (6 left) 2023-10-09T09:20:30.104 INFO:tasks.rados.rados.0.smithi163.stdout:3569: done (5 left) 2023-10-09T09:20:30.104 INFO:tasks.rados.rados.0.smithi163.stdout:3570: done (4 left) 2023-10-09T09:20:30.104 INFO:tasks.rados.rados.0.smithi163.stdout:3571: done (3 left) 2023-10-09T09:20:30.104 INFO:tasks.rados.rados.0.smithi163.stdout:3572: done (2 left) 2023-10-09T09:20:30.104 INFO:tasks.rados.rados.0.smithi163.stdout:3573: done (1 left) 2023-10-09T09:20:30.104 INFO:tasks.rados.rados.0.smithi163.stdout:3574: done (0 left) 2023-10-09T09:20:30.104 INFO:tasks.rados.rados.0.smithi163.stdout:3575: read oid 22 snap -1 2023-10-09T09:20:30.104 INFO:tasks.rados.rados.0.smithi163.stdout:3575: expect (ObjNum 1313 snap 444 seq_num 1313) 2023-10-09T09:20:30.104 INFO:tasks.rados.rados.0.smithi163.stdout:3576: snap_remove snap 418 2023-10-09T09:20:30.312 INFO:tasks.rados.rados.0.smithi163.stdout:3576: done (1 left) 2023-10-09T09:20:30.313 INFO:tasks.rados.rados.0.smithi163.stdout:3577: write oid 40 current snap is 448 2023-10-09T09:20:30.313 INFO:tasks.rados.rados.0.smithi163.stdout:3577: seq_num 1336 ranges {454152=494439,1646296=735124} 2023-10-09T09:20:30.318 INFO:tasks.rados.rados.0.smithi163.stdout:3577: writing smithi16367185-40 from 454152 to 948591 tid 1 2023-10-09T09:20:30.328 INFO:tasks.rados.rados.0.smithi163.stdout:3577: writing smithi16367185-40 from 1646296 to 2381420 tid 2 2023-10-09T09:20:30.328 INFO:tasks.rados.rados.0.smithi163.stdout:3578: rollback oid 3 current snap is 448 2023-10-09T09:20:30.328 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 447 2023-10-09T09:20:30.336 INFO:tasks.rados.rados.0.smithi163.stdout:3575: expect (ObjNum 1313 snap 444 seq_num 1313) 2023-10-09T09:20:30.415 INFO:tasks.rados.rados.0.smithi163.stdout:3577: finishing write tid 1 to smithi16367185-40 2023-10-09T09:20:30.415 INFO:tasks.rados.rados.0.smithi163.stdout:3577: finishing write tid 2 to smithi16367185-40 2023-10-09T09:20:30.415 INFO:tasks.rados.rados.0.smithi163.stdout:3577: finishing write tid 3 to smithi16367185-40 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:3577: finishing write tid 4 to smithi16367185-40 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1834 (ObjNum 1336 snap 448 seq_num 1336) dirty exists 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:3577: left oid 40 (ObjNum 1336 snap 448 seq_num 1336) 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:3578: finishing rollback tid 0 to smithi16367185-3 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:3578: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:3578: finishing rollback tid 2 to smithi16367185-3 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 1037 (ObjNum 1253 snap 420 seq_num 1253) dirty exists 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:3575: done (2 left) 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:3577: done (1 left) 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:3578: done (0 left) 2023-10-09T09:20:30.416 INFO:tasks.rados.rados.0.smithi163.stdout:3579: snap_remove snap 434 2023-10-09T09:20:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:30 smithi181 bash[59345]: cluster 2023-10-09T09:20:29.458173+0000 mgr.y (mgr.34107) 418 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 254 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2023-10-09T09:20:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:30 smithi181 bash[59345]: cluster 2023-10-09T09:20:29.505245+0000 osd.5 (osd.5) 496 : cluster 3 failed to encode map e955 with expected crc 2023-10-09T09:20:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:30 smithi181 bash[59345]: cluster 2023-10-09T09:20:29.505779+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e955 with expected crc 2023-10-09T09:20:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:30 smithi181 bash[59345]: cluster 2023-10-09T09:20:29.510377+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e955: 8 total, 8 up, 8 in 2023-10-09T09:20:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:30 smithi181 bash[59345]: cluster 2023-10-09T09:20:29.511630+0000 osd.7 (osd.7) 401 : cluster 3 failed to encode map e955 with expected crc 2023-10-09T09:20:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:30 smithi181 bash[59345]: cluster 2023-10-09T09:20:30.307568+0000 osd.7 (osd.7) 402 : cluster 3 failed to encode map e956 with expected crc 2023-10-09T09:20:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:30 smithi181 bash[59345]: cluster 2023-10-09T09:20:30.312568+0000 mon.a (mon.0) 1176 : cluster 0 osdmap e956: 8 total, 8 up, 8 in 2023-10-09T09:20:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[78107]: cluster 2023-10-09T09:20:29.458173+0000 mgr.y (mgr.34107) 418 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 254 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2023-10-09T09:20:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[78107]: cluster 2023-10-09T09:20:29.505245+0000 osd.5 (osd.5) 496 : cluster 3 failed to encode map e955 with expected crc 2023-10-09T09:20:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[78107]: cluster 2023-10-09T09:20:29.505779+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e955 with expected crc 2023-10-09T09:20:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[78107]: cluster 2023-10-09T09:20:29.510377+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e955: 8 total, 8 up, 8 in 2023-10-09T09:20:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[78107]: cluster 2023-10-09T09:20:29.511630+0000 osd.7 (osd.7) 401 : cluster 3 failed to encode map e955 with expected crc 2023-10-09T09:20:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[78107]: cluster 2023-10-09T09:20:30.307568+0000 osd.7 (osd.7) 402 : cluster 3 failed to encode map e956 with expected crc 2023-10-09T09:20:30.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[78107]: cluster 2023-10-09T09:20:30.312568+0000 mon.a (mon.0) 1176 : cluster 0 osdmap e956: 8 total, 8 up, 8 in 2023-10-09T09:20:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[75425]: cluster 2023-10-09T09:20:29.458173+0000 mgr.y (mgr.34107) 418 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 254 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2023-10-09T09:20:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[75425]: cluster 2023-10-09T09:20:29.505245+0000 osd.5 (osd.5) 496 : cluster 3 failed to encode map e955 with expected crc 2023-10-09T09:20:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[75425]: cluster 2023-10-09T09:20:29.505779+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e955 with expected crc 2023-10-09T09:20:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[75425]: cluster 2023-10-09T09:20:29.510377+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e955: 8 total, 8 up, 8 in 2023-10-09T09:20:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[75425]: cluster 2023-10-09T09:20:29.511630+0000 osd.7 (osd.7) 401 : cluster 3 failed to encode map e955 with expected crc 2023-10-09T09:20:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[75425]: cluster 2023-10-09T09:20:30.307568+0000 osd.7 (osd.7) 402 : cluster 3 failed to encode map e956 with expected crc 2023-10-09T09:20:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:30 smithi163 bash[75425]: cluster 2023-10-09T09:20:30.312568+0000 mon.a (mon.0) 1176 : cluster 0 osdmap e956: 8 total, 8 up, 8 in 2023-10-09T09:20:31.322 INFO:tasks.rados.rados.0.smithi163.stdout:3579: done (0 left) 2023-10-09T09:20:31.322 INFO:tasks.rados.rados.0.smithi163.stdout:3580: write oid 47 current snap is 448 2023-10-09T09:20:31.322 INFO:tasks.rados.rados.0.smithi163.stdout:3580: seq_num 1337 ranges {470959=569688,1704778=595135} 2023-10-09T09:20:31.328 INFO:tasks.rados.rados.0.smithi163.stdout:3580: writing smithi16367185-47 from 470959 to 1040647 tid 1 2023-10-09T09:20:31.336 INFO:tasks.rados.rados.0.smithi163.stdout:3580: writing smithi16367185-47 from 1704778 to 2299913 tid 2 2023-10-09T09:20:31.337 INFO:tasks.rados.rados.0.smithi163.stdout:3581: snap_remove snap 420 2023-10-09T09:20:31.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:31 smithi181 bash[59345]: cluster 2023-10-09T09:20:30.306996+0000 osd.5 (osd.5) 497 : cluster 3 failed to encode map e956 with expected crc 2023-10-09T09:20:31.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:31 smithi181 bash[59345]: cluster 2023-10-09T09:20:31.317673+0000 osd.7 (osd.7) 403 : cluster 3 failed to encode map e957 with expected crc 2023-10-09T09:20:31.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:31 smithi181 bash[59345]: cluster 2023-10-09T09:20:31.321744+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e957: 8 total, 8 up, 8 in 2023-10-09T09:20:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:31 smithi163 bash[75425]: cluster 2023-10-09T09:20:30.306996+0000 osd.5 (osd.5) 497 : cluster 3 failed to encode map e956 with expected crc 2023-10-09T09:20:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:31 smithi163 bash[75425]: cluster 2023-10-09T09:20:31.317673+0000 osd.7 (osd.7) 403 : cluster 3 failed to encode map e957 with expected crc 2023-10-09T09:20:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:31 smithi163 bash[75425]: cluster 2023-10-09T09:20:31.321744+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e957: 8 total, 8 up, 8 in 2023-10-09T09:20:31.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:31 smithi163 bash[78107]: cluster 2023-10-09T09:20:30.306996+0000 osd.5 (osd.5) 497 : cluster 3 failed to encode map e956 with expected crc 2023-10-09T09:20:31.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:31 smithi163 bash[78107]: cluster 2023-10-09T09:20:31.317673+0000 osd.7 (osd.7) 403 : cluster 3 failed to encode map e957 with expected crc 2023-10-09T09:20:31.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:31 smithi163 bash[78107]: cluster 2023-10-09T09:20:31.321744+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e957: 8 total, 8 up, 8 in 2023-10-09T09:20:32.329 INFO:tasks.rados.rados.0.smithi163.stdout:3581: done (1 left) 2023-10-09T09:20:32.329 INFO:tasks.rados.rados.0.smithi163.stdout:3582: write oid 33 current snap is 448 2023-10-09T09:20:32.329 INFO:tasks.rados.rados.0.smithi163.stdout:3580: finishing write tid 1 to smithi16367185-47 2023-10-09T09:20:32.330 INFO:tasks.rados.rados.0.smithi163.stdout:3582: seq_num 1338 ranges {487766=644937,1646907=738793} 2023-10-09T09:20:32.336 INFO:tasks.rados.rados.0.smithi163.stdout:3582: writing smithi16367185-33 from 487766 to 1132703 tid 1 2023-10-09T09:20:32.345 INFO:tasks.rados.rados.0.smithi163.stdout:3582: writing smithi16367185-33 from 1646907 to 2385700 tid 2 2023-10-09T09:20:32.345 INFO:tasks.rados.rados.0.smithi163.stdout:3583: read oid 37 snap -1 2023-10-09T09:20:32.346 INFO:tasks.rados.rados.0.smithi163.stdout:3583: expect deleted 2023-10-09T09:20:32.346 INFO:tasks.rados.rados.0.smithi163.stdout:3584: write oid 42 current snap is 448 2023-10-09T09:20:32.346 INFO:tasks.rados.rados.0.smithi163.stdout:3584: seq_num 1339 ranges {504573=436539,1421742=598804,2437505=67068} 2023-10-09T09:20:32.351 INFO:tasks.rados.rados.0.smithi163.stdout:3584: writing smithi16367185-42 from 504573 to 941112 tid 1 2023-10-09T09:20:32.359 INFO:tasks.rados.rados.0.smithi163.stdout:3584: writing smithi16367185-42 from 1421742 to 2020546 tid 2 2023-10-09T09:20:32.362 INFO:tasks.rados.rados.0.smithi163.stdout:3584: writing smithi16367185-42 from 2437505 to 2504573 tid 3 2023-10-09T09:20:32.362 INFO:tasks.rados.rados.0.smithi163.stdout:3585: write oid 5 current snap is 448 2023-10-09T09:20:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3580: finishing write tid 2 to smithi16367185-47 2023-10-09T09:20:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3580: finishing write tid 3 to smithi16367185-47 2023-10-09T09:20:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3580: finishing write tid 4 to smithi16367185-47 2023-10-09T09:20:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 2690 (ObjNum 1337 snap 448 seq_num 1337) dirty exists 2023-10-09T09:20:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3580: left oid 47 (ObjNum 1337 snap 448 seq_num 1337) 2023-10-09T09:20:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3582: finishing write tid 1 to smithi16367185-33 2023-10-09T09:20:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3585: seq_num 1340 ranges {521380=511788,1480224=742462} 2023-10-09T09:20:32.368 INFO:tasks.rados.rados.0.smithi163.stdout:3585: writing smithi16367185-5 from 521380 to 1033168 tid 1 2023-10-09T09:20:32.376 INFO:tasks.rados.rados.0.smithi163.stdout:3585: writing smithi16367185-5 from 1480224 to 2222686 tid 2 2023-10-09T09:20:32.376 INFO:tasks.rados.rados.0.smithi163.stdout:3580: done (4 left) 2023-10-09T09:20:32.376 INFO:tasks.rados.rados.0.smithi163.stdout:3586: delete oid 8 current snap is 448 2023-10-09T09:20:32.376 INFO:tasks.rados.rados.0.smithi163.stdout:3582: finishing write tid 2 to smithi16367185-33 2023-10-09T09:20:32.376 INFO:tasks.rados.rados.0.smithi163.stdout:3582: finishing write tid 3 to smithi16367185-33 2023-10-09T09:20:32.376 INFO:tasks.rados.rados.0.smithi163.stdout:3582: finishing write tid 4 to smithi16367185-33 2023-10-09T09:20:32.376 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1666 (ObjNum 1338 snap 448 seq_num 1338) dirty exists 2023-10-09T09:20:32.376 INFO:tasks.rados.rados.0.smithi163.stdout:3582: left oid 33 (ObjNum 1338 snap 448 seq_num 1338) 2023-10-09T09:20:32.376 INFO:tasks.rados.rados.0.smithi163.stdout:3584: finishing write tid 1 to smithi16367185-42 2023-10-09T09:20:32.377 INFO:tasks.rados.rados.0.smithi163.stdout:3584: finishing write tid 2 to smithi16367185-42 2023-10-09T09:20:32.377 INFO:tasks.rados.rados.0.smithi163.stdout:3584: finishing write tid 3 to smithi16367185-42 2023-10-09T09:20:32.377 INFO:tasks.rados.rados.0.smithi163.stdout:3584: finishing write tid 4 to smithi16367185-42 2023-10-09T09:20:32.377 INFO:tasks.rados.rados.0.smithi163.stdout:3584: finishing write tid 5 to smithi16367185-42 2023-10-09T09:20:32.377 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 2715 (ObjNum 1339 snap 448 seq_num 1339) dirty exists 2023-10-09T09:20:32.377 INFO:tasks.rados.rados.0.smithi163.stdout:3584: left oid 42 (ObjNum 1339 snap 448 seq_num 1339) 2023-10-09T09:20:32.377 INFO:tasks.rados.rados.0.smithi163.stdout:3585: finishing write tid 1 to smithi16367185-5 2023-10-09T09:20:32.379 INFO:tasks.rados.rados.0.smithi163.stdout:3582: done (4 left) 2023-10-09T09:20:32.379 INFO:tasks.rados.rados.0.smithi163.stdout:3583: done (3 left) 2023-10-09T09:20:32.379 INFO:tasks.rados.rados.0.smithi163.stdout:3584: done (2 left) 2023-10-09T09:20:32.380 INFO:tasks.rados.rados.0.smithi163.stdout:3586: done (1 left) 2023-10-09T09:20:32.380 INFO:tasks.rados.rados.0.smithi163.stdout:3587: snap_create 2023-10-09T09:20:32.381 INFO:tasks.rados.rados.0.smithi163.stdout:3585: finishing write tid 2 to smithi16367185-5 2023-10-09T09:20:32.381 INFO:tasks.rados.rados.0.smithi163.stdout:3585: finishing write tid 3 to smithi16367185-5 2023-10-09T09:20:32.382 INFO:tasks.rados.rados.0.smithi163.stdout:3585: finishing write tid 4 to smithi16367185-5 2023-10-09T09:20:32.382 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1877 (ObjNum 1340 snap 448 seq_num 1340) dirty exists 2023-10-09T09:20:32.382 INFO:tasks.rados.rados.0.smithi163.stdout:3585: left oid 5 (ObjNum 1340 snap 448 seq_num 1340) 2023-10-09T09:20:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:32 smithi181 bash[59345]: cluster 2023-10-09T09:20:31.317293+0000 osd.5 (osd.5) 498 : cluster 3 failed to encode map e957 with expected crc 2023-10-09T09:20:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:32 smithi181 bash[59345]: cluster 2023-10-09T09:20:31.458867+0000 mgr.y (mgr.34107) 419 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2023-10-09T09:20:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:32 smithi181 bash[59345]: cluster 2023-10-09T09:20:32.329010+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e958: 8 total, 8 up, 8 in 2023-10-09T09:20:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:32 smithi181 bash[59345]: cluster 2023-10-09T09:20:32.330188+0000 osd.4 (osd.4) 427 : cluster 3 failed to encode map e958 with expected crc 2023-10-09T09:20:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:32 smithi163 bash[75425]: cluster 2023-10-09T09:20:31.317293+0000 osd.5 (osd.5) 498 : cluster 3 failed to encode map e957 with expected crc 2023-10-09T09:20:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:32 smithi163 bash[75425]: cluster 2023-10-09T09:20:31.458867+0000 mgr.y (mgr.34107) 419 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2023-10-09T09:20:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:32 smithi163 bash[75425]: cluster 2023-10-09T09:20:32.329010+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e958: 8 total, 8 up, 8 in 2023-10-09T09:20:32.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:32 smithi163 bash[75425]: cluster 2023-10-09T09:20:32.330188+0000 osd.4 (osd.4) 427 : cluster 3 failed to encode map e958 with expected crc 2023-10-09T09:20:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:32 smithi163 bash[78107]: cluster 2023-10-09T09:20:31.317293+0000 osd.5 (osd.5) 498 : cluster 3 failed to encode map e957 with expected crc 2023-10-09T09:20:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:32 smithi163 bash[78107]: cluster 2023-10-09T09:20:31.458867+0000 mgr.y (mgr.34107) 419 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2023-10-09T09:20:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:32 smithi163 bash[78107]: cluster 2023-10-09T09:20:32.329010+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e958: 8 total, 8 up, 8 in 2023-10-09T09:20:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:32 smithi163 bash[78107]: cluster 2023-10-09T09:20:32.330188+0000 osd.4 (osd.4) 427 : cluster 3 failed to encode map e958 with expected crc 2023-10-09T09:20:33.335 INFO:tasks.rados.rados.0.smithi163.stdout:3585: done (1 left) 2023-10-09T09:20:33.335 INFO:tasks.rados.rados.0.smithi163.stdout:3587: done (0 left) 2023-10-09T09:20:33.335 INFO:tasks.rados.rados.0.smithi163.stdout:3588: read oid 9 snap -1 2023-10-09T09:20:33.335 INFO:tasks.rados.rados.0.smithi163.stdout:3588: expect deleted 2023-10-09T09:20:33.335 INFO:tasks.rados.rados.0.smithi163.stdout:3589: snap_create 2023-10-09T09:20:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:20:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:20:33] "GET /metrics HTTP/1.1" 200 36263 "" "Prometheus/2.33.4" 2023-10-09T09:20:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:33 smithi163 bash[78107]: cluster 2023-10-09T09:20:32.325074+0000 osd.5 (osd.5) 499 : cluster 3 failed to encode map e958 with expected crc 2023-10-09T09:20:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:33 smithi163 bash[78107]: cluster 2023-10-09T09:20:33.330966+0000 osd.4 (osd.4) 428 : cluster 3 failed to encode map e959 with expected crc 2023-10-09T09:20:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:33 smithi163 bash[78107]: cluster 2023-10-09T09:20:33.334722+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e959: 8 total, 8 up, 8 in 2023-10-09T09:20:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:33 smithi163 bash[75425]: cluster 2023-10-09T09:20:32.325074+0000 osd.5 (osd.5) 499 : cluster 3 failed to encode map e958 with expected crc 2023-10-09T09:20:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:33 smithi163 bash[75425]: cluster 2023-10-09T09:20:33.330966+0000 osd.4 (osd.4) 428 : cluster 3 failed to encode map e959 with expected crc 2023-10-09T09:20:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:33 smithi163 bash[75425]: cluster 2023-10-09T09:20:33.334722+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e959: 8 total, 8 up, 8 in 2023-10-09T09:20:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:33 smithi181 bash[59345]: cluster 2023-10-09T09:20:32.325074+0000 osd.5 (osd.5) 499 : cluster 3 failed to encode map e958 with expected crc 2023-10-09T09:20:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:33 smithi181 bash[59345]: cluster 2023-10-09T09:20:33.330966+0000 osd.4 (osd.4) 428 : cluster 3 failed to encode map e959 with expected crc 2023-10-09T09:20:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:33 smithi181 bash[59345]: cluster 2023-10-09T09:20:33.334722+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e959: 8 total, 8 up, 8 in 2023-10-09T09:20:34.340 INFO:tasks.rados.rados.0.smithi163.stdout:3588: done (1 left) 2023-10-09T09:20:34.340 INFO:tasks.rados.rados.0.smithi163.stdout:3589: done (0 left) 2023-10-09T09:20:34.340 INFO:tasks.rados.rados.0.smithi163.stdout:3590: snap_create 2023-10-09T09:20:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:34 smithi181 bash[59345]: cluster 2023-10-09T09:20:33.330572+0000 osd.5 (osd.5) 500 : cluster 3 failed to encode map e959 with expected crc 2023-10-09T09:20:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:34 smithi181 bash[59345]: cluster 2023-10-09T09:20:33.459545+0000 mgr.y (mgr.34107) 420 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2023-10-09T09:20:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:34 smithi181 bash[59345]: cluster 2023-10-09T09:20:34.340121+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e960: 8 total, 8 up, 8 in 2023-10-09T09:20:34.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:34 smithi163 bash[75425]: cluster 2023-10-09T09:20:33.330572+0000 osd.5 (osd.5) 500 : cluster 3 failed to encode map e959 with expected crc 2023-10-09T09:20:34.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:34 smithi163 bash[75425]: cluster 2023-10-09T09:20:33.459545+0000 mgr.y (mgr.34107) 420 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2023-10-09T09:20:34.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:34 smithi163 bash[75425]: cluster 2023-10-09T09:20:34.340121+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e960: 8 total, 8 up, 8 in 2023-10-09T09:20:34.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:34 smithi163 bash[78107]: cluster 2023-10-09T09:20:33.330572+0000 osd.5 (osd.5) 500 : cluster 3 failed to encode map e959 with expected crc 2023-10-09T09:20:34.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:34 smithi163 bash[78107]: cluster 2023-10-09T09:20:33.459545+0000 mgr.y (mgr.34107) 420 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2023-10-09T09:20:34.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:34 smithi163 bash[78107]: cluster 2023-10-09T09:20:34.340121+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e960: 8 total, 8 up, 8 in 2023-10-09T09:20:35.314 INFO:tasks.rados.rados.0.smithi163.stdout:3590: done (0 left) 2023-10-09T09:20:35.314 INFO:tasks.rados.rados.0.smithi163.stdout:3591: snap_remove snap 427 2023-10-09T09:20:35.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:35 smithi181 bash[59345]: cluster 2023-10-09T09:20:34.337339+0000 osd.5 (osd.5) 501 : cluster 3 failed to encode map e960 with expected crc 2023-10-09T09:20:35.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:35 smithi181 bash[59345]: cluster 2023-10-09T09:20:34.340412+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e960 with expected crc 2023-10-09T09:20:35.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:35 smithi181 bash[59345]: cluster 2023-10-09T09:20:35.313768+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e961: 8 total, 8 up, 8 in 2023-10-09T09:20:35.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:35 smithi163 bash[75425]: cluster 2023-10-09T09:20:34.337339+0000 osd.5 (osd.5) 501 : cluster 3 failed to encode map e960 with expected crc 2023-10-09T09:20:35.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:35 smithi163 bash[75425]: cluster 2023-10-09T09:20:34.340412+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e960 with expected crc 2023-10-09T09:20:35.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:35 smithi163 bash[75425]: cluster 2023-10-09T09:20:35.313768+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e961: 8 total, 8 up, 8 in 2023-10-09T09:20:35.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:35 smithi163 bash[78107]: cluster 2023-10-09T09:20:34.337339+0000 osd.5 (osd.5) 501 : cluster 3 failed to encode map e960 with expected crc 2023-10-09T09:20:35.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:35 smithi163 bash[78107]: cluster 2023-10-09T09:20:34.340412+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e960 with expected crc 2023-10-09T09:20:35.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:35 smithi163 bash[78107]: cluster 2023-10-09T09:20:35.313768+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e961: 8 total, 8 up, 8 in 2023-10-09T09:20:36.317 INFO:tasks.rados.rados.0.smithi163.stdout:3591: done (0 left) 2023-10-09T09:20:36.317 INFO:tasks.rados.rados.0.smithi163.stdout:3592: write oid 1 current snap is 451 2023-10-09T09:20:36.317 INFO:tasks.rados.rados.0.smithi163.stdout:3592: seq_num 1342 ranges {554994=662286,1997188=557806} 2023-10-09T09:20:36.329 INFO:tasks.rados.rados.0.smithi163.stdout:3592: writing smithi16367185-1 from 554994 to 1217280 tid 1 2023-10-09T09:20:36.338 INFO:tasks.rados.rados.0.smithi163.stdout:3592: writing smithi16367185-1 from 1997188 to 2554994 tid 2 2023-10-09T09:20:36.338 INFO:tasks.rados.rados.0.smithi163.stdout:3593: read oid 2 snap -1 2023-10-09T09:20:36.338 INFO:tasks.rados.rados.0.smithi163.stdout:3593: expect deleted 2023-10-09T09:20:36.338 INFO:tasks.rados.rados.0.smithi163.stdout:3594: write oid 13 current snap is 451 2023-10-09T09:20:36.338 INFO:tasks.rados.rados.0.smithi163.stdout:3594: seq_num 1343 ranges {571801=737535,1939317=489789} 2023-10-09T09:20:36.346 INFO:tasks.rados.rados.0.smithi163.stdout:3594: writing smithi16367185-13 from 571801 to 1309336 tid 1 2023-10-09T09:20:36.353 INFO:tasks.rados.rados.0.smithi163.stdout:3594: writing smithi16367185-13 from 1939317 to 2429106 tid 2 2023-10-09T09:20:36.353 INFO:tasks.rados.rados.0.smithi163.stdout:3595: delete oid 34 current snap is 451 2023-10-09T09:20:36.353 INFO:tasks.rados.rados.0.smithi163.stdout:3592: finishing write tid 1 to smithi16367185-1 2023-10-09T09:20:36.354 INFO:tasks.rados.rados.0.smithi163.stdout:3592: finishing write tid 2 to smithi16367185-1 2023-10-09T09:20:36.354 INFO:tasks.rados.rados.0.smithi163.stdout:3592: finishing write tid 3 to smithi16367185-1 2023-10-09T09:20:36.354 INFO:tasks.rados.rados.0.smithi163.stdout:3592: finishing write tid 4 to smithi16367185-1 2023-10-09T09:20:36.354 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 1885 (ObjNum 1342 snap 451 seq_num 1342) dirty exists 2023-10-09T09:20:36.354 INFO:tasks.rados.rados.0.smithi163.stdout:3592: left oid 1 (ObjNum 1342 snap 451 seq_num 1342) 2023-10-09T09:20:36.354 INFO:tasks.rados.rados.0.smithi163.stdout:3594: finishing write tid 1 to smithi16367185-13 2023-10-09T09:20:36.356 INFO:tasks.rados.rados.0.smithi163.stdout:3592: done (3 left) 2023-10-09T09:20:36.356 INFO:tasks.rados.rados.0.smithi163.stdout:3593: done (2 left) 2023-10-09T09:20:36.356 INFO:tasks.rados.rados.0.smithi163.stdout:3595: done (1 left) 2023-10-09T09:20:36.356 INFO:tasks.rados.rados.0.smithi163.stdout:3596: snap_remove snap 450 2023-10-09T09:20:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:36 smithi181 bash[59345]: cluster 2023-10-09T09:20:35.309023+0000 osd.5 (osd.5) 502 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:36 smithi181 bash[59345]: cluster 2023-10-09T09:20:35.460236+0000 mgr.y (mgr.34107) 421 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 644 KiB/s wr, 7 op/s 2023-10-09T09:20:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:36 smithi181 bash[59345]: cluster 2023-10-09T09:20:35.480276+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:36 smithi181 bash[59345]: cluster 2023-10-09T09:20:35.882619+0000 osd.4 (osd.4) 429 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:36 smithi181 bash[59345]: cluster 2023-10-09T09:20:36.001302+0000 osd.7 (osd.7) 404 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:36 smithi181 bash[59345]: cluster 2023-10-09T09:20:36.316804+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e962: 8 total, 8 up, 8 in 2023-10-09T09:20:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:36 smithi181 bash[59345]: cluster 2023-10-09T09:20:36.339465+0000 osd.7 (osd.7) 405 : cluster 3 failed to encode map e962 with expected crc 2023-10-09T09:20:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[75425]: cluster 2023-10-09T09:20:35.309023+0000 osd.5 (osd.5) 502 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[75425]: cluster 2023-10-09T09:20:35.460236+0000 mgr.y (mgr.34107) 421 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 644 KiB/s wr, 7 op/s 2023-10-09T09:20:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[75425]: cluster 2023-10-09T09:20:35.480276+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[75425]: cluster 2023-10-09T09:20:35.882619+0000 osd.4 (osd.4) 429 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[75425]: cluster 2023-10-09T09:20:36.001302+0000 osd.7 (osd.7) 404 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[75425]: cluster 2023-10-09T09:20:36.316804+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e962: 8 total, 8 up, 8 in 2023-10-09T09:20:36.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[75425]: cluster 2023-10-09T09:20:36.339465+0000 osd.7 (osd.7) 405 : cluster 3 failed to encode map e962 with expected crc 2023-10-09T09:20:36.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[78107]: cluster 2023-10-09T09:20:35.309023+0000 osd.5 (osd.5) 502 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[78107]: cluster 2023-10-09T09:20:35.460236+0000 mgr.y (mgr.34107) 421 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 644 KiB/s wr, 7 op/s 2023-10-09T09:20:36.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[78107]: cluster 2023-10-09T09:20:35.480276+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[78107]: cluster 2023-10-09T09:20:35.882619+0000 osd.4 (osd.4) 429 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[78107]: cluster 2023-10-09T09:20:36.001302+0000 osd.7 (osd.7) 404 : cluster 3 failed to encode map e961 with expected crc 2023-10-09T09:20:36.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[78107]: cluster 2023-10-09T09:20:36.316804+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e962: 8 total, 8 up, 8 in 2023-10-09T09:20:36.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:36 smithi163 bash[78107]: cluster 2023-10-09T09:20:36.339465+0000 osd.7 (osd.7) 405 : cluster 3 failed to encode map e962 with expected crc 2023-10-09T09:20:37.328 INFO:tasks.rados.rados.0.smithi163.stdout:3596: done (1 left) 2023-10-09T09:20:37.328 INFO:tasks.rados.rados.0.smithi163.stdout:3597: rollback oid 41 current snap is 451 2023-10-09T09:20:37.328 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 41 to 444 2023-10-09T09:20:37.328 INFO:tasks.rados.rados.0.smithi163.stdout:3594: finishing write tid 2 to smithi16367185-13 2023-10-09T09:20:37.328 INFO:tasks.rados.rados.0.smithi163.stdout:3594: finishing write tid 3 to smithi16367185-13 2023-10-09T09:20:37.328 INFO:tasks.rados.rados.0.smithi163.stdout:3594: finishing write tid 4 to smithi16367185-13 2023-10-09T09:20:37.328 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 740 (ObjNum 1343 snap 451 seq_num 1343) dirty exists 2023-10-09T09:20:37.329 INFO:tasks.rados.rados.0.smithi163.stdout:3594: left oid 13 (ObjNum 1343 snap 451 seq_num 1343) 2023-10-09T09:20:37.329 INFO:tasks.rados.rados.0.smithi163.stdout:3594: done (1 left) 2023-10-09T09:20:37.329 INFO:tasks.rados.rados.0.smithi163.stdout:3598: write oid 16 current snap is 451 2023-10-09T09:20:37.329 INFO:tasks.rados.rados.0.smithi163.stdout:3598: seq_num 1345 ranges {605415=488033,1656281=493458} 2023-10-09T09:20:37.336 INFO:tasks.rados.rados.0.smithi163.stdout:3598: writing smithi16367185-16 from 605415 to 1093448 tid 1 2023-10-09T09:20:37.343 INFO:tasks.rados.rados.0.smithi163.stdout:3598: writing smithi16367185-16 from 1656281 to 2149739 tid 2 2023-10-09T09:20:37.343 INFO:tasks.rados.rados.0.smithi163.stdout:3599: delete oid 22 current snap is 451 2023-10-09T09:20:37.343 INFO:tasks.rados.rados.0.smithi163.stdout:3597: finishing rollback tid 0 to smithi16367185-41 2023-10-09T09:20:37.344 INFO:tasks.rados.rados.0.smithi163.stdout:3597: finishing rollback tid 1 to smithi16367185-41 2023-10-09T09:20:37.344 INFO:tasks.rados.rados.0.smithi163.stdout:3597: finishing rollback tid 2 to smithi16367185-41 2023-10-09T09:20:37.344 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 324 (ObjNum 1306 snap 440 seq_num 1306) dirty exists 2023-10-09T09:20:37.344 INFO:tasks.rados.rados.0.smithi163.stdout:3598: finishing write tid 1 to smithi16367185-16 2023-10-09T09:20:37.347 INFO:tasks.rados.rados.0.smithi163.stdout:3597: done (2 left) 2023-10-09T09:20:37.347 INFO:tasks.rados.rados.0.smithi163.stdout:3599: done (1 left) 2023-10-09T09:20:37.347 INFO:tasks.rados.rados.0.smithi163.stdout:3600: read oid 19 snap 447 2023-10-09T09:20:37.347 INFO:tasks.rados.rados.0.smithi163.stdout:3600: expect (ObjNum 1258 snap 423 seq_num 1258) 2023-10-09T09:20:37.347 INFO:tasks.rados.rados.0.smithi163.stdout:3601: snap_remove snap 443 2023-10-09T09:20:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:37 smithi181 bash[59345]: cluster 2023-10-09T09:20:36.312102+0000 osd.5 (osd.5) 503 : cluster 3 failed to encode map e962 with expected crc 2023-10-09T09:20:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:37 smithi181 bash[59345]: cluster 2023-10-09T09:20:36.313014+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e962 with expected crc 2023-10-09T09:20:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:37 smithi181 bash[59345]: cluster 2023-10-09T09:20:37.322218+0000 osd.5 (osd.5) 504 : cluster 3 failed to encode map e963 with expected crc 2023-10-09T09:20:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:37 smithi181 bash[59345]: cluster 2023-10-09T09:20:37.327498+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e963: 8 total, 8 up, 8 in 2023-10-09T09:20:37.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:37 smithi181 bash[59345]: cluster 2023-10-09T09:20:37.331190+0000 osd.7 (osd.7) 406 : cluster 3 failed to encode map e963 with expected crc 2023-10-09T09:20:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:37 smithi163 bash[75425]: cluster 2023-10-09T09:20:36.312102+0000 osd.5 (osd.5) 503 : cluster 3 failed to encode map e962 with expected crc 2023-10-09T09:20:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:37 smithi163 bash[75425]: cluster 2023-10-09T09:20:36.313014+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e962 with expected crc 2023-10-09T09:20:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:37 smithi163 bash[75425]: cluster 2023-10-09T09:20:37.322218+0000 osd.5 (osd.5) 504 : cluster 3 failed to encode map e963 with expected crc 2023-10-09T09:20:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:37 smithi163 bash[75425]: cluster 2023-10-09T09:20:37.327498+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e963: 8 total, 8 up, 8 in 2023-10-09T09:20:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:37 smithi163 bash[75425]: cluster 2023-10-09T09:20:37.331190+0000 osd.7 (osd.7) 406 : cluster 3 failed to encode map e963 with expected crc 2023-10-09T09:20:37.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:37 smithi163 bash[78107]: cluster 2023-10-09T09:20:36.312102+0000 osd.5 (osd.5) 503 : cluster 3 failed to encode map e962 with expected crc 2023-10-09T09:20:37.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:37 smithi163 bash[78107]: cluster 2023-10-09T09:20:36.313014+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e962 with expected crc 2023-10-09T09:20:37.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:37 smithi163 bash[78107]: cluster 2023-10-09T09:20:37.322218+0000 osd.5 (osd.5) 504 : cluster 3 failed to encode map e963 with expected crc 2023-10-09T09:20:37.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:37 smithi163 bash[78107]: cluster 2023-10-09T09:20:37.327498+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e963: 8 total, 8 up, 8 in 2023-10-09T09:20:37.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:37 smithi163 bash[78107]: cluster 2023-10-09T09:20:37.331190+0000 osd.7 (osd.7) 406 : cluster 3 failed to encode map e963 with expected crc 2023-10-09T09:20:38.334 INFO:tasks.rados.rados.0.smithi163.stdout:3601: done (2 left) 2023-10-09T09:20:38.334 INFO:tasks.rados.rados.0.smithi163.stdout:3602: read oid 31 snap -1 2023-10-09T09:20:38.334 INFO:tasks.rados.rados.0.smithi163.stdout:3602: expect deleted 2023-10-09T09:20:38.334 INFO:tasks.rados.rados.0.smithi163.stdout:3603: read oid 26 snap -1 2023-10-09T09:20:38.334 INFO:tasks.rados.rados.0.smithi163.stdout:3603: expect (ObjNum 1328 snap 446 seq_num 1328) 2023-10-09T09:20:38.334 INFO:tasks.rados.rados.0.smithi163.stdout:3604: write oid 6 current snap is 451 2023-10-09T09:20:38.334 INFO:tasks.rados.rados.0.smithi163.stdout:3598: finishing write tid 2 to smithi16367185-16 2023-10-09T09:20:38.334 INFO:tasks.rados.rados.0.smithi163.stdout:3598: finishing write tid 3 to smithi16367185-16 2023-10-09T09:20:38.335 INFO:tasks.rados.rados.0.smithi163.stdout:3598: finishing write tid 4 to smithi16367185-16 2023-10-09T09:20:38.335 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1895 (ObjNum 1345 snap 451 seq_num 1345) dirty exists 2023-10-09T09:20:38.335 INFO:tasks.rados.rados.0.smithi163.stdout:3598: left oid 16 (ObjNum 1345 snap 451 seq_num 1345) 2023-10-09T09:20:38.335 INFO:tasks.rados.rados.0.smithi163.stdout:3600: expect (ObjNum 1258 snap 423 seq_num 1258) 2023-10-09T09:20:38.450 INFO:tasks.rados.rados.0.smithi163.stdout:3603: expect (ObjNum 1328 snap 446 seq_num 1328) 2023-10-09T09:20:38.539 INFO:tasks.rados.rados.0.smithi163.stdout:3604: seq_num 1347 ranges {639029=754884,2173245=465784} 2023-10-09T09:20:38.553 INFO:tasks.rados.rados.0.smithi163.stdout:3604: writing smithi16367185-6 from 639029 to 1393913 tid 1 2023-10-09T09:20:38.561 INFO:tasks.rados.rados.0.smithi163.stdout:3604: writing smithi16367185-6 from 2173245 to 2639029 tid 2 2023-10-09T09:20:38.561 INFO:tasks.rados.rados.0.smithi163.stdout:3598: done (4 left) 2023-10-09T09:20:38.561 INFO:tasks.rados.rados.0.smithi163.stdout:3600: done (3 left) 2023-10-09T09:20:38.561 INFO:tasks.rados.rados.0.smithi163.stdout:3602: done (2 left) 2023-10-09T09:20:38.561 INFO:tasks.rados.rados.0.smithi163.stdout:3603: done (1 left) 2023-10-09T09:20:38.561 INFO:tasks.rados.rados.0.smithi163.stdout:3605: write oid 4 current snap is 451 2023-10-09T09:20:38.561 INFO:tasks.rados.rados.0.smithi163.stdout:3604: finishing write tid 1 to smithi16367185-6 2023-10-09T09:20:38.561 INFO:tasks.rados.rados.0.smithi163.stdout:3605: seq_num 1348 ranges {655836=430133,1831727=757138} 2023-10-09T09:20:38.572 INFO:tasks.rados.rados.0.smithi163.stdout:3605: writing smithi16367185-4 from 655836 to 1085969 tid 1 2023-10-09T09:20:38.581 INFO:tasks.rados.rados.0.smithi163.stdout:3605: writing smithi16367185-4 from 1831727 to 2588865 tid 2 2023-10-09T09:20:38.581 INFO:tasks.rados.rados.0.smithi163.stdout:3606: delete oid 42 current snap is 451 2023-10-09T09:20:38.582 INFO:tasks.rados.rados.0.smithi163.stdout:3604: finishing write tid 2 to smithi16367185-6 2023-10-09T09:20:38.582 INFO:tasks.rados.rados.0.smithi163.stdout:3604: finishing write tid 3 to smithi16367185-6 2023-10-09T09:20:38.582 INFO:tasks.rados.rados.0.smithi163.stdout:3604: finishing write tid 4 to smithi16367185-6 2023-10-09T09:20:38.582 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 399 (ObjNum 1347 snap 451 seq_num 1347) dirty exists 2023-10-09T09:20:38.582 INFO:tasks.rados.rados.0.smithi163.stdout:3604: left oid 6 (ObjNum 1347 snap 451 seq_num 1347) 2023-10-09T09:20:38.582 INFO:tasks.rados.rados.0.smithi163.stdout:3605: finishing write tid 1 to smithi16367185-4 2023-10-09T09:20:38.587 INFO:tasks.rados.rados.0.smithi163.stdout:3604: done (2 left) 2023-10-09T09:20:38.592 INFO:tasks.rados.rados.0.smithi163.stdout:3606: done (1 left) 2023-10-09T09:20:38.592 INFO:tasks.rados.rados.0.smithi163.stdout:3607: read oid 10 snap -1 2023-10-09T09:20:38.592 INFO:tasks.rados.rados.0.smithi163.stdout:3607: expect (ObjNum 1325 snap 446 seq_num 1325) 2023-10-09T09:20:38.592 INFO:tasks.rados.rados.0.smithi163.stdout:3608: rollback oid 43 current snap is 451 2023-10-09T09:20:38.592 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 446 2023-10-09T09:20:38.592 INFO:tasks.rados.rados.0.smithi163.stdout:3609: snap_create 2023-10-09T09:20:38.593 INFO:tasks.rados.rados.0.smithi163.stdout:3605: finishing write tid 2 to smithi16367185-4 2023-10-09T09:20:38.593 INFO:tasks.rados.rados.0.smithi163.stdout:3605: finishing write tid 3 to smithi16367185-4 2023-10-09T09:20:38.593 INFO:tasks.rados.rados.0.smithi163.stdout:3605: finishing write tid 4 to smithi16367185-4 2023-10-09T09:20:38.593 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1906 (ObjNum 1348 snap 451 seq_num 1348) dirty exists 2023-10-09T09:20:38.593 INFO:tasks.rados.rados.0.smithi163.stdout:3605: left oid 4 (ObjNum 1348 snap 451 seq_num 1348) 2023-10-09T09:20:38.593 INFO:tasks.rados.rados.0.smithi163.stdout:3607: expect (ObjNum 1325 snap 446 seq_num 1325) 2023-10-09T09:20:38.700 INFO:tasks.rados.rados.0.smithi163.stdout:3608: finishing rollback tid 0 to smithi16367185-43 2023-10-09T09:20:38.700 INFO:tasks.rados.rados.0.smithi163.stdout:3608: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:20:38.700 INFO:tasks.rados.rados.0.smithi163.stdout:3608: finishing rollback tid 2 to smithi16367185-43 2023-10-09T09:20:38.700 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 1696 (ObjNum 1091 snap 362 seq_num 1091) dirty exists 2023-10-09T09:20:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:38 smithi181 bash[59345]: cluster 2023-10-09T09:20:37.461001+0000 mgr.y (mgr.34107) 422 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 263 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:20:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:38 smithi181 bash[59345]: cluster 2023-10-09T09:20:38.331528+0000 osd.5 (osd.5) 505 : cluster 3 failed to encode map e964 with expected crc 2023-10-09T09:20:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:38 smithi181 bash[59345]: cluster 2023-10-09T09:20:38.333774+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e964: 8 total, 8 up, 8 in 2023-10-09T09:20:38.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:38 smithi181 bash[59345]: cluster 2023-10-09T09:20:38.334549+0000 osd.7 (osd.7) 407 : cluster 3 failed to encode map e964 with expected crc 2023-10-09T09:20:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:38 smithi163 bash[78107]: cluster 2023-10-09T09:20:37.461001+0000 mgr.y (mgr.34107) 422 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 263 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:20:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:38 smithi163 bash[78107]: cluster 2023-10-09T09:20:38.331528+0000 osd.5 (osd.5) 505 : cluster 3 failed to encode map e964 with expected crc 2023-10-09T09:20:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:38 smithi163 bash[78107]: cluster 2023-10-09T09:20:38.333774+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e964: 8 total, 8 up, 8 in 2023-10-09T09:20:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:38 smithi163 bash[78107]: cluster 2023-10-09T09:20:38.334549+0000 osd.7 (osd.7) 407 : cluster 3 failed to encode map e964 with expected crc 2023-10-09T09:20:38.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:38 smithi163 bash[75425]: cluster 2023-10-09T09:20:37.461001+0000 mgr.y (mgr.34107) 422 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 263 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:20:38.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:38 smithi163 bash[75425]: cluster 2023-10-09T09:20:38.331528+0000 osd.5 (osd.5) 505 : cluster 3 failed to encode map e964 with expected crc 2023-10-09T09:20:38.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:38 smithi163 bash[75425]: cluster 2023-10-09T09:20:38.333774+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e964: 8 total, 8 up, 8 in 2023-10-09T09:20:38.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:38 smithi163 bash[75425]: cluster 2023-10-09T09:20:38.334549+0000 osd.7 (osd.7) 407 : cluster 3 failed to encode map e964 with expected crc 2023-10-09T09:20:39.540 INFO:tasks.rados.rados.0.smithi163.stdout:3605: done (3 left) 2023-10-09T09:20:39.541 INFO:tasks.rados.rados.0.smithi163.stdout:3607: done (2 left) 2023-10-09T09:20:39.541 INFO:tasks.rados.rados.0.smithi163.stdout:3608: done (1 left) 2023-10-09T09:20:39.541 INFO:tasks.rados.rados.0.smithi163.stdout:3609: done (0 left) 2023-10-09T09:20:39.541 INFO:tasks.rados.rados.0.smithi163.stdout:3610: read oid 2 snap -1 2023-10-09T09:20:39.541 INFO:tasks.rados.rados.0.smithi163.stdout:3610: expect deleted 2023-10-09T09:20:39.541 INFO:tasks.rados.rados.0.smithi163.stdout:3611: snap_remove snap 446 2023-10-09T09:20:40.317 INFO:tasks.rados.rados.0.smithi163.stdout:3611: done (1 left) 2023-10-09T09:20:40.317 INFO:tasks.rados.rados.0.smithi163.stdout:3612: write oid 18 current snap is 452 2023-10-09T09:20:40.317 INFO:tasks.rados.rados.0.smithi163.stdout:3612: seq_num 1350 ranges {689450=580631,1948691=740759} 2023-10-09T09:20:40.326 INFO:tasks.rados.rados.0.smithi163.stdout:3612: writing smithi16367185-18 from 689450 to 1270081 tid 1 2023-10-09T09:20:40.336 INFO:tasks.rados.rados.0.smithi163.stdout:3612: writing smithi16367185-18 from 1948691 to 2689450 tid 2 2023-10-09T09:20:40.336 INFO:tasks.rados.rados.0.smithi163.stdout:3613: delete oid 32 current snap is 452 2023-10-09T09:20:40.342 INFO:tasks.rados.rados.0.smithi163.stdout:3612: finishing write tid 1 to smithi16367185-18 2023-10-09T09:20:40.343 INFO:tasks.rados.rados.0.smithi163.stdout:3612: finishing write tid 2 to smithi16367185-18 2023-10-09T09:20:40.344 INFO:tasks.rados.rados.0.smithi163.stdout:3612: finishing write tid 3 to smithi16367185-18 2023-10-09T09:20:40.345 INFO:tasks.rados.rados.0.smithi163.stdout:3612: finishing write tid 4 to smithi16367185-18 2023-10-09T09:20:40.345 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1257 (ObjNum 1350 snap 452 seq_num 1350) dirty exists 2023-10-09T09:20:40.345 INFO:tasks.rados.rados.0.smithi163.stdout:3612: left oid 18 (ObjNum 1350 snap 452 seq_num 1350) 2023-10-09T09:20:40.345 INFO:tasks.rados.rados.0.smithi163.stdout:3610: done (2 left) 2023-10-09T09:20:40.345 INFO:tasks.rados.rados.0.smithi163.stdout:3612: done (1 left) 2023-10-09T09:20:40.345 INFO:tasks.rados.rados.0.smithi163.stdout:3613: done (0 left) 2023-10-09T09:20:40.346 INFO:tasks.rados.rados.0.smithi163.stdout:3614: read oid 7 snap -1 2023-10-09T09:20:40.346 INFO:tasks.rados.rados.0.smithi163.stdout:3614: expect deleted 2023-10-09T09:20:40.346 INFO:tasks.rados.rados.0.smithi163.stdout:3615: write oid 37 current snap is 452 2023-10-09T09:20:40.346 INFO:tasks.rados.rados.0.smithi163.stdout:3615: seq_num 1352 ranges {723064=731129,1949302=764476} 2023-10-09T09:20:40.358 INFO:tasks.rados.rados.0.smithi163.stdout:3615: writing smithi16367185-37 from 723064 to 1454193 tid 1 2023-10-09T09:20:40.372 INFO:tasks.rados.rados.0.smithi163.stdout:3615: writing smithi16367185-37 from 1949302 to 2713778 tid 2 2023-10-09T09:20:40.372 INFO:tasks.rados.rados.0.smithi163.stdout:3616: delete oid 17 current snap is 452 2023-10-09T09:20:40.372 INFO:tasks.rados.rados.0.smithi163.stdout:3615: finishing write tid 1 to smithi16367185-37 2023-10-09T09:20:40.380 INFO:tasks.rados.rados.0.smithi163.stdout:3615: finishing write tid 2 to smithi16367185-37 2023-10-09T09:20:40.381 INFO:tasks.rados.rados.0.smithi163.stdout:3615: finishing write tid 3 to smithi16367185-37 2023-10-09T09:20:40.381 INFO:tasks.rados.rados.0.smithi163.stdout:3615: finishing write tid 4 to smithi16367185-37 2023-10-09T09:20:40.381 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 1872 (ObjNum 1352 snap 452 seq_num 1352) dirty exists 2023-10-09T09:20:40.381 INFO:tasks.rados.rados.0.smithi163.stdout:3615: left oid 37 (ObjNum 1352 snap 452 seq_num 1352) 2023-10-09T09:20:40.382 INFO:tasks.rados.rados.0.smithi163.stdout:3614: done (2 left) 2023-10-09T09:20:40.382 INFO:tasks.rados.rados.0.smithi163.stdout:3615: done (1 left) 2023-10-09T09:20:40.382 INFO:tasks.rados.rados.0.smithi163.stdout:3616: done (0 left) 2023-10-09T09:20:40.382 INFO:tasks.rados.rados.0.smithi163.stdout:3617: delete oid 5 current snap is 452 2023-10-09T09:20:40.385 INFO:tasks.rados.rados.0.smithi163.stdout:3617: done (0 left) 2023-10-09T09:20:40.385 INFO:tasks.rados.rados.0.smithi163.stdout:3618: write oid 32 current snap is 452 2023-10-09T09:20:40.385 INFO:tasks.rados.rados.0.smithi163.stdout:3618: seq_num 1355 ranges {773485=673229,2124748=511803} 2023-10-09T09:20:40.397 INFO:tasks.rados.rados.0.smithi163.stdout:3618: writing smithi16367185-32 from 773485 to 1446714 tid 1 2023-10-09T09:20:40.404 INFO:tasks.rados.rados.0.smithi163.stdout:3618: writing smithi16367185-32 from 2124748 to 2636551 tid 2 2023-10-09T09:20:40.404 INFO:tasks.rados.rados.0.smithi163.stdout:3619: write oid 41 current snap is 452 2023-10-09T09:20:40.404 INFO:tasks.rados.rados.0.smithi163.stdout:3618: finishing write tid 1 to smithi16367185-32 2023-10-09T09:20:40.404 INFO:tasks.rados.rados.0.smithi163.stdout:3619: seq_num 1356 ranges {790292=748478,2183230=607062} 2023-10-09T09:20:40.417 INFO:tasks.rados.rados.0.smithi163.stdout:3619: writing smithi16367185-41 from 790292 to 1538770 tid 1 2023-10-09T09:20:40.425 INFO:tasks.rados.rados.0.smithi163.stdout:3619: writing smithi16367185-41 from 2183230 to 2790292 tid 2 2023-10-09T09:20:40.425 INFO:tasks.rados.rados.0.smithi163.stdout:3620: snap_remove snap 431 2023-10-09T09:20:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:40 smithi181 bash[59345]: cluster 2023-10-09T09:20:39.461845+0000 mgr.y (mgr.34107) 423 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 262 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 493 B/s rd, 534 KiB/s wr, 3 op/s 2023-10-09T09:20:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:40 smithi181 bash[59345]: cluster 2023-10-09T09:20:39.537867+0000 osd.5 (osd.5) 506 : cluster 3 failed to encode map e965 with expected crc 2023-10-09T09:20:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:40 smithi181 bash[59345]: cluster 2023-10-09T09:20:39.538174+0000 osd.4 (osd.4) 430 : cluster 3 failed to encode map e965 with expected crc 2023-10-09T09:20:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:40 smithi181 bash[59345]: cluster 2023-10-09T09:20:39.540536+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e965: 8 total, 8 up, 8 in 2023-10-09T09:20:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:40 smithi181 bash[59345]: cluster 2023-10-09T09:20:39.541599+0000 osd.7 (osd.7) 408 : cluster 3 failed to encode map e965 with expected crc 2023-10-09T09:20:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:40 smithi181 bash[59345]: cluster 2023-10-09T09:20:40.311964+0000 osd.5 (osd.5) 507 : cluster 3 failed to encode map e966 with expected crc 2023-10-09T09:20:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:40 smithi181 bash[59345]: cluster 2023-10-09T09:20:40.315111+0000 osd.4 (osd.4) 431 : cluster 3 failed to encode map e966 with expected crc 2023-10-09T09:20:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:40 smithi181 bash[59345]: cluster 2023-10-09T09:20:40.316402+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e966: 8 total, 8 up, 8 in 2023-10-09T09:20:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:40 smithi181 bash[59345]: cluster 2023-10-09T09:20:40.344278+0000 osd.7 (osd.7) 409 : cluster 3 failed to encode map e966 with expected crc 2023-10-09T09:20:40.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:40 smithi181 bash[59345]: audit 2023-10-09T09:20:40.351576+0000 mon.a (mon.0) 1187 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[75425]: cluster 2023-10-09T09:20:39.461845+0000 mgr.y (mgr.34107) 423 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 262 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 493 B/s rd, 534 KiB/s wr, 3 op/s 2023-10-09T09:20:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[75425]: cluster 2023-10-09T09:20:39.537867+0000 osd.5 (osd.5) 506 : cluster 3 failed to encode map e965 with expected crc 2023-10-09T09:20:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[75425]: cluster 2023-10-09T09:20:39.538174+0000 osd.4 (osd.4) 430 : cluster 3 failed to encode map e965 with expected crc 2023-10-09T09:20:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[75425]: cluster 2023-10-09T09:20:39.540536+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e965: 8 total, 8 up, 8 in 2023-10-09T09:20:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[75425]: cluster 2023-10-09T09:20:39.541599+0000 osd.7 (osd.7) 408 : cluster 3 failed to encode map e965 with expected crc 2023-10-09T09:20:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[75425]: cluster 2023-10-09T09:20:40.311964+0000 osd.5 (osd.5) 507 : cluster 3 failed to encode map e966 with expected crc 2023-10-09T09:20:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[75425]: cluster 2023-10-09T09:20:40.315111+0000 osd.4 (osd.4) 431 : cluster 3 failed to encode map e966 with expected crc 2023-10-09T09:20:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[75425]: cluster 2023-10-09T09:20:40.316402+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e966: 8 total, 8 up, 8 in 2023-10-09T09:20:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[75425]: cluster 2023-10-09T09:20:40.344278+0000 osd.7 (osd.7) 409 : cluster 3 failed to encode map e966 with expected crc 2023-10-09T09:20:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[75425]: audit 2023-10-09T09:20:40.351576+0000 mon.a (mon.0) 1187 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[78107]: cluster 2023-10-09T09:20:39.461845+0000 mgr.y (mgr.34107) 423 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 262 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 493 B/s rd, 534 KiB/s wr, 3 op/s 2023-10-09T09:20:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[78107]: cluster 2023-10-09T09:20:39.537867+0000 osd.5 (osd.5) 506 : cluster 3 failed to encode map e965 with expected crc 2023-10-09T09:20:40.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[78107]: cluster 2023-10-09T09:20:39.538174+0000 osd.4 (osd.4) 430 : cluster 3 failed to encode map e965 with expected crc 2023-10-09T09:20:40.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[78107]: cluster 2023-10-09T09:20:39.540536+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e965: 8 total, 8 up, 8 in 2023-10-09T09:20:40.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[78107]: cluster 2023-10-09T09:20:39.541599+0000 osd.7 (osd.7) 408 : cluster 3 failed to encode map e965 with expected crc 2023-10-09T09:20:40.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[78107]: cluster 2023-10-09T09:20:40.311964+0000 osd.5 (osd.5) 507 : cluster 3 failed to encode map e966 with expected crc 2023-10-09T09:20:40.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[78107]: cluster 2023-10-09T09:20:40.315111+0000 osd.4 (osd.4) 431 : cluster 3 failed to encode map e966 with expected crc 2023-10-09T09:20:40.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[78107]: cluster 2023-10-09T09:20:40.316402+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e966: 8 total, 8 up, 8 in 2023-10-09T09:20:40.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[78107]: cluster 2023-10-09T09:20:40.344278+0000 osd.7 (osd.7) 409 : cluster 3 failed to encode map e966 with expected crc 2023-10-09T09:20:40.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:40 smithi163 bash[78107]: audit 2023-10-09T09:20:40.351576+0000 mon.a (mon.0) 1187 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:41.330 INFO:tasks.rados.rados.0.smithi163.stdout:3620: done (2 left) 2023-10-09T09:20:41.330 INFO:tasks.rados.rados.0.smithi163.stdout:3621: snap_create 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:3618: finishing write tid 2 to smithi16367185-32 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:3618: finishing write tid 3 to smithi16367185-32 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:3618: finishing write tid 4 to smithi16367185-32 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 2752 (ObjNum 1355 snap 452 seq_num 1355) dirty exists 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:3618: left oid 32 (ObjNum 1355 snap 452 seq_num 1355) 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:3619: finishing write tid 1 to smithi16367185-41 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:3619: finishing write tid 2 to smithi16367185-41 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:3619: finishing write tid 3 to smithi16367185-41 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:3619: finishing write tid 4 to smithi16367185-41 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 332 (ObjNum 1356 snap 452 seq_num 1356) dirty exists 2023-10-09T09:20:41.331 INFO:tasks.rados.rados.0.smithi163.stdout:3619: left oid 41 (ObjNum 1356 snap 452 seq_num 1356) 2023-10-09T09:20:42.338 INFO:tasks.rados.rados.0.smithi163.stdout:3618: done (2 left) 2023-10-09T09:20:42.339 INFO:tasks.rados.rados.0.smithi163.stdout:3619: done (1 left) 2023-10-09T09:20:42.339 INFO:tasks.rados.rados.0.smithi163.stdout:3621: done (0 left) 2023-10-09T09:20:42.339 INFO:tasks.rados.rados.0.smithi163.stdout:3622: snap_create 2023-10-09T09:20:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:42 smithi163 bash[75425]: cluster 2023-10-09T09:20:41.323919+0000 osd.5 (osd.5) 508 : cluster 3 failed to encode map e967 with expected crc 2023-10-09T09:20:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:42 smithi163 bash[75425]: cluster 2023-10-09T09:20:41.329162+0000 osd.4 (osd.4) 432 : cluster 3 failed to encode map e967 with expected crc 2023-10-09T09:20:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:42 smithi163 bash[75425]: cluster 2023-10-09T09:20:41.329675+0000 mon.a (mon.0) 1188 : cluster 0 osdmap e967: 8 total, 8 up, 8 in 2023-10-09T09:20:42.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:42 smithi163 bash[75425]: cluster 2023-10-09T09:20:41.462518+0000 mgr.y (mgr.34107) 424 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.0 MiB/s wr, 17 op/s 2023-10-09T09:20:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:42 smithi163 bash[78107]: cluster 2023-10-09T09:20:41.323919+0000 osd.5 (osd.5) 508 : cluster 3 failed to encode map e967 with expected crc 2023-10-09T09:20:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:42 smithi163 bash[78107]: cluster 2023-10-09T09:20:41.329162+0000 osd.4 (osd.4) 432 : cluster 3 failed to encode map e967 with expected crc 2023-10-09T09:20:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:42 smithi163 bash[78107]: cluster 2023-10-09T09:20:41.329675+0000 mon.a (mon.0) 1188 : cluster 0 osdmap e967: 8 total, 8 up, 8 in 2023-10-09T09:20:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:42 smithi163 bash[78107]: cluster 2023-10-09T09:20:41.462518+0000 mgr.y (mgr.34107) 424 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.0 MiB/s wr, 17 op/s 2023-10-09T09:20:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:42 smithi181 bash[59345]: cluster 2023-10-09T09:20:41.323919+0000 osd.5 (osd.5) 508 : cluster 3 failed to encode map e967 with expected crc 2023-10-09T09:20:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:42 smithi181 bash[59345]: cluster 2023-10-09T09:20:41.329162+0000 osd.4 (osd.4) 432 : cluster 3 failed to encode map e967 with expected crc 2023-10-09T09:20:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:42 smithi181 bash[59345]: cluster 2023-10-09T09:20:41.329675+0000 mon.a (mon.0) 1188 : cluster 0 osdmap e967: 8 total, 8 up, 8 in 2023-10-09T09:20:42.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:42 smithi181 bash[59345]: cluster 2023-10-09T09:20:41.462518+0000 mgr.y (mgr.34107) 424 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.0 MiB/s wr, 17 op/s 2023-10-09T09:20:43.346 INFO:tasks.rados.rados.0.smithi163.stdout:3622: done (0 left) 2023-10-09T09:20:43.346 INFO:tasks.rados.rados.0.smithi163.stdout:3623: write oid 9 current snap is 454 2023-10-09T09:20:43.346 INFO:tasks.rados.rados.0.smithi163.stdout:3623: seq_num 1357 ranges {407099=423727,1441712=515472,2582060=225039} 2023-10-09T09:20:43.354 INFO:tasks.rados.rados.0.smithi163.stdout:3623: writing smithi16367185-9 from 407099 to 830826 tid 1 2023-10-09T09:20:43.362 INFO:tasks.rados.rados.0.smithi163.stdout:3623: writing smithi16367185-9 from 1441712 to 1957184 tid 2 2023-10-09T09:20:43.367 INFO:tasks.rados.rados.0.smithi163.stdout:3623: writing smithi16367185-9 from 2582060 to 2807099 tid 3 2023-10-09T09:20:43.367 INFO:tasks.rados.rados.0.smithi163.stdout:3624: write oid 7 current snap is 454 2023-10-09T09:20:43.367 INFO:tasks.rados.rados.0.smithi163.stdout:3623: finishing write tid 1 to smithi16367185-9 2023-10-09T09:20:43.367 INFO:tasks.rados.rados.0.smithi163.stdout:3623: finishing write tid 2 to smithi16367185-9 2023-10-09T09:20:43.367 INFO:tasks.rados.rados.0.smithi163.stdout:3624: seq_num 1358 ranges {423906=498976,1500194=659130,2609483=214423} 2023-10-09T09:20:43.372 INFO:tasks.rados.rados.0.smithi163.stdout:3624: writing smithi16367185-7 from 423906 to 922882 tid 1 2023-10-09T09:20:43.379 INFO:tasks.rados.rados.0.smithi163.stdout:3624: writing smithi16367185-7 from 1500194 to 2159324 tid 2 2023-10-09T09:20:43.383 INFO:tasks.rados.rados.0.smithi163.stdout:3624: writing smithi16367185-7 from 2609483 to 2823906 tid 3 2023-10-09T09:20:43.383 INFO:tasks.rados.rados.0.smithi163.stdout:3625: delete oid 18 current snap is 454 2023-10-09T09:20:43.383 INFO:tasks.rados.rados.0.smithi163.stdout:3623: finishing write tid 3 to smithi16367185-9 2023-10-09T09:20:43.383 INFO:tasks.rados.rados.0.smithi163.stdout:3623: finishing write tid 4 to smithi16367185-9 2023-10-09T09:20:43.384 INFO:tasks.rados.rados.0.smithi163.stdout:3623: finishing write tid 5 to smithi16367185-9 2023-10-09T09:20:43.384 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 800 (ObjNum 1357 snap 454 seq_num 1357) dirty exists 2023-10-09T09:20:43.384 INFO:tasks.rados.rados.0.smithi163.stdout:3623: left oid 9 (ObjNum 1357 snap 454 seq_num 1357) 2023-10-09T09:20:43.384 INFO:tasks.rados.rados.0.smithi163.stdout:3624: finishing write tid 1 to smithi16367185-7 2023-10-09T09:20:43.384 INFO:tasks.rados.rados.0.smithi163.stdout:3624: finishing write tid 2 to smithi16367185-7 2023-10-09T09:20:43.386 INFO:tasks.rados.rados.0.smithi163.stdout:3624: finishing write tid 3 to smithi16367185-7 2023-10-09T09:20:43.386 INFO:tasks.rados.rados.0.smithi163.stdout:3624: finishing write tid 4 to smithi16367185-7 2023-10-09T09:20:43.387 INFO:tasks.rados.rados.0.smithi163.stdout:3624: finishing write tid 5 to smithi16367185-7 2023-10-09T09:20:43.387 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 952 (ObjNum 1358 snap 454 seq_num 1358) dirty exists 2023-10-09T09:20:43.387 INFO:tasks.rados.rados.0.smithi163.stdout:3624: left oid 7 (ObjNum 1358 snap 454 seq_num 1358) 2023-10-09T09:20:43.389 INFO:tasks.rados.rados.0.smithi163.stdout:3623: done (2 left) 2023-10-09T09:20:43.389 INFO:tasks.rados.rados.0.smithi163.stdout:3624: done (1 left) 2023-10-09T09:20:43.389 INFO:tasks.rados.rados.0.smithi163.stdout:3625: done (0 left) 2023-10-09T09:20:43.389 INFO:tasks.rados.rados.0.smithi163.stdout:3626: delete oid 35 current snap is 454 2023-10-09T09:20:43.399 INFO:tasks.rados.rados.0.smithi163.stdout:3626: done (0 left) 2023-10-09T09:20:43.399 INFO:tasks.rados.rados.0.smithi163.stdout:3627: snap_create 2023-10-09T09:20:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:20:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:20:43] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:20:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:43 smithi163 bash[78107]: cluster 2023-10-09T09:20:42.335156+0000 osd.5 (osd.5) 509 : cluster 3 failed to encode map e968 with expected crc 2023-10-09T09:20:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:43 smithi163 bash[78107]: cluster 2023-10-09T09:20:42.335524+0000 osd.7 (osd.7) 410 : cluster 3 failed to encode map e968 with expected crc 2023-10-09T09:20:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:43 smithi163 bash[78107]: cluster 2023-10-09T09:20:42.338615+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e968: 8 total, 8 up, 8 in 2023-10-09T09:20:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:43 smithi163 bash[78107]: cluster 2023-10-09T09:20:42.505610+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e968 with expected crc 2023-10-09T09:20:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:43 smithi163 bash[75425]: cluster 2023-10-09T09:20:42.335156+0000 osd.5 (osd.5) 509 : cluster 3 failed to encode map e968 with expected crc 2023-10-09T09:20:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:43 smithi163 bash[75425]: cluster 2023-10-09T09:20:42.335524+0000 osd.7 (osd.7) 410 : cluster 3 failed to encode map e968 with expected crc 2023-10-09T09:20:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:43 smithi163 bash[75425]: cluster 2023-10-09T09:20:42.338615+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e968: 8 total, 8 up, 8 in 2023-10-09T09:20:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:43 smithi163 bash[75425]: cluster 2023-10-09T09:20:42.505610+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e968 with expected crc 2023-10-09T09:20:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:43 smithi181 bash[59345]: cluster 2023-10-09T09:20:42.335156+0000 osd.5 (osd.5) 509 : cluster 3 failed to encode map e968 with expected crc 2023-10-09T09:20:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:43 smithi181 bash[59345]: cluster 2023-10-09T09:20:42.335524+0000 osd.7 (osd.7) 410 : cluster 3 failed to encode map e968 with expected crc 2023-10-09T09:20:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:43 smithi181 bash[59345]: cluster 2023-10-09T09:20:42.338615+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e968: 8 total, 8 up, 8 in 2023-10-09T09:20:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:43 smithi181 bash[59345]: cluster 2023-10-09T09:20:42.505610+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e968 with expected crc 2023-10-09T09:20:44.366 INFO:tasks.rados.rados.0.smithi163.stdout:3627: done (0 left) 2023-10-09T09:20:44.367 INFO:tasks.rados.rados.0.smithi163.stdout:3628: snap_remove snap 452 2023-10-09T09:20:44.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:44 smithi163 bash[75425]: cluster 2023-10-09T09:20:43.340043+0000 osd.5 (osd.5) 510 : cluster 3 failed to encode map e969 with expected crc 2023-10-09T09:20:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:44 smithi163 bash[75425]: cluster 2023-10-09T09:20:43.345541+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e969: 8 total, 8 up, 8 in 2023-10-09T09:20:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:44 smithi163 bash[75425]: cluster 2023-10-09T09:20:43.358983+0000 osd.4 (osd.4) 433 : cluster 3 failed to encode map e969 with expected crc 2023-10-09T09:20:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:44 smithi163 bash[75425]: cluster 2023-10-09T09:20:43.390848+0000 osd.7 (osd.7) 411 : cluster 3 failed to encode map e969 with expected crc 2023-10-09T09:20:44.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:44 smithi163 bash[75425]: cluster 2023-10-09T09:20:43.463264+0000 mgr.y (mgr.34107) 425 : cluster 0 pgmap v1007: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2023-10-09T09:20:44.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:44 smithi163 bash[78107]: cluster 2023-10-09T09:20:43.340043+0000 osd.5 (osd.5) 510 : cluster 3 failed to encode map e969 with expected crc 2023-10-09T09:20:44.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:44 smithi163 bash[78107]: cluster 2023-10-09T09:20:43.345541+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e969: 8 total, 8 up, 8 in 2023-10-09T09:20:44.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:44 smithi163 bash[78107]: cluster 2023-10-09T09:20:43.358983+0000 osd.4 (osd.4) 433 : cluster 3 failed to encode map e969 with expected crc 2023-10-09T09:20:44.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:44 smithi163 bash[78107]: cluster 2023-10-09T09:20:43.390848+0000 osd.7 (osd.7) 411 : cluster 3 failed to encode map e969 with expected crc 2023-10-09T09:20:44.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:44 smithi163 bash[78107]: cluster 2023-10-09T09:20:43.463264+0000 mgr.y (mgr.34107) 425 : cluster 0 pgmap v1007: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2023-10-09T09:20:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:44 smithi181 bash[59345]: cluster 2023-10-09T09:20:43.340043+0000 osd.5 (osd.5) 510 : cluster 3 failed to encode map e969 with expected crc 2023-10-09T09:20:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:44 smithi181 bash[59345]: cluster 2023-10-09T09:20:43.345541+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e969: 8 total, 8 up, 8 in 2023-10-09T09:20:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:44 smithi181 bash[59345]: cluster 2023-10-09T09:20:43.358983+0000 osd.4 (osd.4) 433 : cluster 3 failed to encode map e969 with expected crc 2023-10-09T09:20:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:44 smithi181 bash[59345]: cluster 2023-10-09T09:20:43.390848+0000 osd.7 (osd.7) 411 : cluster 3 failed to encode map e969 with expected crc 2023-10-09T09:20:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:44 smithi181 bash[59345]: cluster 2023-10-09T09:20:43.463264+0000 mgr.y (mgr.34107) 425 : cluster 0 pgmap v1007: 89 pgs: 89 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2023-10-09T09:20:45.324 INFO:tasks.rados.rados.0.smithi163.stdout:3628: done (0 left) 2023-10-09T09:20:45.325 INFO:tasks.rados.rados.0.smithi163.stdout:3629: rollback oid 39 current snap is 455 2023-10-09T09:20:45.325 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 454 2023-10-09T09:20:45.325 INFO:tasks.rados.rados.0.smithi163.stdout:3630: delete oid 14 current snap is 455 2023-10-09T09:20:45.338 INFO:tasks.rados.rados.0.smithi163.stdout:3629: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:20:45.338 INFO:tasks.rados.rados.0.smithi163.stdout:3629: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:20:45.338 INFO:tasks.rados.rados.0.smithi163.stdout:3629: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:20:45.338 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1696 (ObjNum 1310 snap 443 seq_num 1310) dirty exists 2023-10-09T09:20:45.338 INFO:tasks.rados.rados.0.smithi163.stdout:3629: done (1 left) 2023-10-09T09:20:45.338 INFO:tasks.rados.rados.0.smithi163.stdout:3630: done (0 left) 2023-10-09T09:20:45.338 INFO:tasks.rados.rados.0.smithi163.stdout:3631: snap_create 2023-10-09T09:20:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:45 smithi163 bash[75425]: cluster 2023-10-09T09:20:44.363945+0000 osd.5 (osd.5) 511 : cluster 3 failed to encode map e970 with expected crc 2023-10-09T09:20:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:45 smithi163 bash[75425]: cluster 2023-10-09T09:20:44.366697+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e970: 8 total, 8 up, 8 in 2023-10-09T09:20:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:45 smithi163 bash[75425]: cluster 2023-10-09T09:20:44.367048+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e970 with expected crc 2023-10-09T09:20:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:45 smithi163 bash[75425]: cluster 2023-10-09T09:20:45.323749+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e971: 8 total, 8 up, 8 in 2023-10-09T09:20:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:45 smithi163 bash[78107]: cluster 2023-10-09T09:20:44.363945+0000 osd.5 (osd.5) 511 : cluster 3 failed to encode map e970 with expected crc 2023-10-09T09:20:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:45 smithi163 bash[78107]: cluster 2023-10-09T09:20:44.366697+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e970: 8 total, 8 up, 8 in 2023-10-09T09:20:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:45 smithi163 bash[78107]: cluster 2023-10-09T09:20:44.367048+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e970 with expected crc 2023-10-09T09:20:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:45 smithi163 bash[78107]: cluster 2023-10-09T09:20:45.323749+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e971: 8 total, 8 up, 8 in 2023-10-09T09:20:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:45 smithi181 bash[59345]: cluster 2023-10-09T09:20:44.363945+0000 osd.5 (osd.5) 511 : cluster 3 failed to encode map e970 with expected crc 2023-10-09T09:20:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:45 smithi181 bash[59345]: cluster 2023-10-09T09:20:44.366697+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e970: 8 total, 8 up, 8 in 2023-10-09T09:20:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:45 smithi181 bash[59345]: cluster 2023-10-09T09:20:44.367048+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e970 with expected crc 2023-10-09T09:20:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:45 smithi181 bash[59345]: cluster 2023-10-09T09:20:45.323749+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e971: 8 total, 8 up, 8 in 2023-10-09T09:20:46.336 INFO:tasks.rados.rados.0.smithi163.stdout:3631: done (0 left) 2023-10-09T09:20:46.336 INFO:tasks.rados.rados.0.smithi163.stdout:3632: read oid 43 snap 448 2023-10-09T09:20:46.336 INFO:tasks.rados.rados.0.smithi163.stdout:3632: expect (ObjNum 1091 snap 362 seq_num 1091) 2023-10-09T09:20:46.336 INFO:tasks.rados.rados.0.smithi163.stdout:3633: read oid 9 snap -1 2023-10-09T09:20:46.336 INFO:tasks.rados.rados.0.smithi163.stdout:3633: expect (ObjNum 1357 snap 454 seq_num 1357) 2023-10-09T09:20:46.336 INFO:tasks.rados.rados.0.smithi163.stdout:3634: read oid 10 snap 449 2023-10-09T09:20:46.336 INFO:tasks.rados.rados.0.smithi163.stdout:3634: expect (ObjNum 1325 snap 446 seq_num 1325) 2023-10-09T09:20:46.336 INFO:tasks.rados.rados.0.smithi163.stdout:3635: snap_create 2023-10-09T09:20:46.341 INFO:tasks.rados.rados.0.smithi163.stdout:3634: expect (ObjNum 1325 snap 446 seq_num 1325) 2023-10-09T09:20:46.453 INFO:tasks.rados.rados.0.smithi163.stdout:3633: expect (ObjNum 1357 snap 454 seq_num 1357) 2023-10-09T09:20:46.547 INFO:tasks.rados.rados.0.smithi163.stdout:3632: expect (ObjNum 1091 snap 362 seq_num 1091) 2023-10-09T09:20:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:46 smithi163 bash[78107]: cluster 2023-10-09T09:20:45.319292+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e971 with expected crc 2023-10-09T09:20:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:46 smithi163 bash[78107]: cluster 2023-10-09T09:20:45.319310+0000 osd.5 (osd.5) 512 : cluster 3 failed to encode map e971 with expected crc 2023-10-09T09:20:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:46 smithi163 bash[78107]: cluster 2023-10-09T09:20:45.463922+0000 mgr.y (mgr.34107) 426 : cluster 0 pgmap v1010: 89 pgs: 89 active+clean; 280 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2023-10-09T09:20:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:46 smithi163 bash[78107]: cluster 2023-10-09T09:20:46.335611+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e972: 8 total, 8 up, 8 in 2023-10-09T09:20:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:46 smithi163 bash[75425]: cluster 2023-10-09T09:20:45.319292+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e971 with expected crc 2023-10-09T09:20:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:46 smithi163 bash[75425]: cluster 2023-10-09T09:20:45.319310+0000 osd.5 (osd.5) 512 : cluster 3 failed to encode map e971 with expected crc 2023-10-09T09:20:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:46 smithi163 bash[75425]: cluster 2023-10-09T09:20:45.463922+0000 mgr.y (mgr.34107) 426 : cluster 0 pgmap v1010: 89 pgs: 89 active+clean; 280 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2023-10-09T09:20:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:46 smithi163 bash[75425]: cluster 2023-10-09T09:20:46.335611+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e972: 8 total, 8 up, 8 in 2023-10-09T09:20:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:46 smithi181 bash[59345]: cluster 2023-10-09T09:20:45.319292+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e971 with expected crc 2023-10-09T09:20:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:46 smithi181 bash[59345]: cluster 2023-10-09T09:20:45.319310+0000 osd.5 (osd.5) 512 : cluster 3 failed to encode map e971 with expected crc 2023-10-09T09:20:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:46 smithi181 bash[59345]: cluster 2023-10-09T09:20:45.463922+0000 mgr.y (mgr.34107) 426 : cluster 0 pgmap v1010: 89 pgs: 89 active+clean; 280 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2023-10-09T09:20:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:46 smithi181 bash[59345]: cluster 2023-10-09T09:20:46.335611+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e972: 8 total, 8 up, 8 in 2023-10-09T09:20:47.338 INFO:tasks.rados.rados.0.smithi163.stdout:3632: done (3 left) 2023-10-09T09:20:47.339 INFO:tasks.rados.rados.0.smithi163.stdout:3633: done (2 left) 2023-10-09T09:20:47.339 INFO:tasks.rados.rados.0.smithi163.stdout:3634: done (1 left) 2023-10-09T09:20:47.339 INFO:tasks.rados.rados.0.smithi163.stdout:3635: done (0 left) 2023-10-09T09:20:47.339 INFO:tasks.rados.rados.0.smithi163.stdout:3636: read oid 34 snap -1 2023-10-09T09:20:47.339 INFO:tasks.rados.rados.0.smithi163.stdout:3636: expect deleted 2023-10-09T09:20:47.339 INFO:tasks.rados.rados.0.smithi163.stdout:3637: delete oid 28 current snap is 457 2023-10-09T09:20:47.345 INFO:tasks.rados.rados.0.smithi163.stdout:3636: done (1 left) 2023-10-09T09:20:47.345 INFO:tasks.rados.rados.0.smithi163.stdout:3637: done (0 left) 2023-10-09T09:20:47.345 INFO:tasks.rados.rados.0.smithi163.stdout:3638: write oid 5 current snap is 457 2023-10-09T09:20:47.345 INFO:tasks.rados.rados.0.smithi163.stdout:3638: seq_num 1363 ranges {507941=591574,1676251=526479,2746598=161343} 2023-10-09T09:20:47.352 INFO:tasks.rados.rados.0.smithi163.stdout:3638: writing smithi16367185-5 from 507941 to 1099515 tid 1 2023-10-09T09:20:47.359 INFO:tasks.rados.rados.0.smithi163.stdout:3638: writing smithi16367185-5 from 1676251 to 2202730 tid 2 2023-10-09T09:20:47.363 INFO:tasks.rados.rados.0.smithi163.stdout:3638: writing smithi16367185-5 from 2746598 to 2907941 tid 3 2023-10-09T09:20:47.363 INFO:tasks.rados.rados.0.smithi163.stdout:3639: read oid 15 snap -1 2023-10-09T09:20:47.363 INFO:tasks.rados.rados.0.smithi163.stdout:3639: expect (ObjNum 1303 snap 440 seq_num 1303) 2023-10-09T09:20:47.363 INFO:tasks.rados.rados.0.smithi163.stdout:3640: snap_remove snap 447 2023-10-09T09:20:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:47 smithi163 bash[78107]: cluster 2023-10-09T09:20:46.330618+0000 osd.5 (osd.5) 513 : cluster 3 failed to encode map e972 with expected crc 2023-10-09T09:20:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:47 smithi163 bash[78107]: cluster 2023-10-09T09:20:46.337013+0000 osd.4 (osd.4) 434 : cluster 3 failed to encode map e972 with expected crc 2023-10-09T09:20:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:47 smithi163 bash[78107]: cluster 2023-10-09T09:20:47.338183+0000 mon.a (mon.0) 1194 : cluster 0 osdmap e973: 8 total, 8 up, 8 in 2023-10-09T09:20:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:47 smithi163 bash[75425]: cluster 2023-10-09T09:20:46.330618+0000 osd.5 (osd.5) 513 : cluster 3 failed to encode map e972 with expected crc 2023-10-09T09:20:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:47 smithi163 bash[75425]: cluster 2023-10-09T09:20:46.337013+0000 osd.4 (osd.4) 434 : cluster 3 failed to encode map e972 with expected crc 2023-10-09T09:20:47.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:47 smithi163 bash[75425]: cluster 2023-10-09T09:20:47.338183+0000 mon.a (mon.0) 1194 : cluster 0 osdmap e973: 8 total, 8 up, 8 in 2023-10-09T09:20:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:47 smithi181 bash[59345]: cluster 2023-10-09T09:20:46.330618+0000 osd.5 (osd.5) 513 : cluster 3 failed to encode map e972 with expected crc 2023-10-09T09:20:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:47 smithi181 bash[59345]: cluster 2023-10-09T09:20:46.337013+0000 osd.4 (osd.4) 434 : cluster 3 failed to encode map e972 with expected crc 2023-10-09T09:20:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:47 smithi181 bash[59345]: cluster 2023-10-09T09:20:47.338183+0000 mon.a (mon.0) 1194 : cluster 0 osdmap e973: 8 total, 8 up, 8 in 2023-10-09T09:20:48.345 INFO:tasks.rados.rados.0.smithi163.stdout:3640: done (2 left) 2023-10-09T09:20:48.346 INFO:tasks.rados.rados.0.smithi163.stdout:3641: snap_create 2023-10-09T09:20:48.346 INFO:tasks.rados.rados.0.smithi163.stdout:3638: finishing write tid 1 to smithi16367185-5 2023-10-09T09:20:48.346 INFO:tasks.rados.rados.0.smithi163.stdout:3638: finishing write tid 2 to smithi16367185-5 2023-10-09T09:20:48.346 INFO:tasks.rados.rados.0.smithi163.stdout:3638: finishing write tid 3 to smithi16367185-5 2023-10-09T09:20:48.346 INFO:tasks.rados.rados.0.smithi163.stdout:3638: finishing write tid 4 to smithi16367185-5 2023-10-09T09:20:48.346 INFO:tasks.rados.rados.0.smithi163.stdout:3638: finishing write tid 5 to smithi16367185-5 2023-10-09T09:20:48.346 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 1928 (ObjNum 1363 snap 457 seq_num 1363) dirty exists 2023-10-09T09:20:48.346 INFO:tasks.rados.rados.0.smithi163.stdout:3638: left oid 5 (ObjNum 1363 snap 457 seq_num 1363) 2023-10-09T09:20:48.346 INFO:tasks.rados.rados.0.smithi163.stdout:3639: expect (ObjNum 1303 snap 440 seq_num 1303) 2023-10-09T09:20:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:48 smithi163 bash[78107]: cluster 2023-10-09T09:20:47.333038+0000 osd.5 (osd.5) 514 : cluster 3 failed to encode map e973 with expected crc 2023-10-09T09:20:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:48 smithi163 bash[78107]: cluster 2023-10-09T09:20:47.333773+0000 osd.4 (osd.4) 435 : cluster 3 failed to encode map e973 with expected crc 2023-10-09T09:20:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:48 smithi163 bash[78107]: cluster 2023-10-09T09:20:47.364884+0000 osd.7 (osd.7) 412 : cluster 3 failed to encode map e973 with expected crc 2023-10-09T09:20:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:48 smithi163 bash[78107]: cluster 2023-10-09T09:20:47.464562+0000 mgr.y (mgr.34107) 427 : cluster 0 pgmap v1013: 89 pgs: 89 active+clean; 272 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 7 op/s 2023-10-09T09:20:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:48 smithi163 bash[78107]: cluster 2023-10-09T09:20:48.344610+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e974: 8 total, 8 up, 8 in 2023-10-09T09:20:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:48 smithi163 bash[75425]: cluster 2023-10-09T09:20:47.333038+0000 osd.5 (osd.5) 514 : cluster 3 failed to encode map e973 with expected crc 2023-10-09T09:20:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:48 smithi163 bash[75425]: cluster 2023-10-09T09:20:47.333773+0000 osd.4 (osd.4) 435 : cluster 3 failed to encode map e973 with expected crc 2023-10-09T09:20:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:48 smithi163 bash[75425]: cluster 2023-10-09T09:20:47.364884+0000 osd.7 (osd.7) 412 : cluster 3 failed to encode map e973 with expected crc 2023-10-09T09:20:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:48 smithi163 bash[75425]: cluster 2023-10-09T09:20:47.464562+0000 mgr.y (mgr.34107) 427 : cluster 0 pgmap v1013: 89 pgs: 89 active+clean; 272 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 7 op/s 2023-10-09T09:20:48.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:48 smithi163 bash[75425]: cluster 2023-10-09T09:20:48.344610+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e974: 8 total, 8 up, 8 in 2023-10-09T09:20:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:48 smithi181 bash[59345]: cluster 2023-10-09T09:20:47.333038+0000 osd.5 (osd.5) 514 : cluster 3 failed to encode map e973 with expected crc 2023-10-09T09:20:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:48 smithi181 bash[59345]: cluster 2023-10-09T09:20:47.333773+0000 osd.4 (osd.4) 435 : cluster 3 failed to encode map e973 with expected crc 2023-10-09T09:20:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:48 smithi181 bash[59345]: cluster 2023-10-09T09:20:47.364884+0000 osd.7 (osd.7) 412 : cluster 3 failed to encode map e973 with expected crc 2023-10-09T09:20:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:48 smithi181 bash[59345]: cluster 2023-10-09T09:20:47.464562+0000 mgr.y (mgr.34107) 427 : cluster 0 pgmap v1013: 89 pgs: 89 active+clean; 272 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 7 op/s 2023-10-09T09:20:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:48 smithi181 bash[59345]: cluster 2023-10-09T09:20:48.344610+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e974: 8 total, 8 up, 8 in 2023-10-09T09:20:49.344 INFO:tasks.rados.rados.0.smithi163.stdout:3638: done (2 left) 2023-10-09T09:20:49.344 INFO:tasks.rados.rados.0.smithi163.stdout:3639: done (1 left) 2023-10-09T09:20:49.344 INFO:tasks.rados.rados.0.smithi163.stdout:3641: done (0 left) 2023-10-09T09:20:49.344 INFO:tasks.rados.rados.0.smithi163.stdout:3642: delete oid 6 current snap is 458 2023-10-09T09:20:49.353 INFO:tasks.rados.rados.0.smithi163.stdout:3642: done (0 left) 2023-10-09T09:20:49.353 INFO:tasks.rados.rados.0.smithi163.stdout:3643: delete oid 32 current snap is 458 2023-10-09T09:20:49.358 INFO:tasks.rados.rados.0.smithi163.stdout:3643: done (0 left) 2023-10-09T09:20:49.358 INFO:tasks.rados.rados.0.smithi163.stdout:3644: read oid 33 snap -1 2023-10-09T09:20:49.358 INFO:tasks.rados.rados.0.smithi163.stdout:3644: expect (ObjNum 1338 snap 448 seq_num 1338) 2023-10-09T09:20:49.358 INFO:tasks.rados.rados.0.smithi163.stdout:3645: write oid 16 current snap is 458 2023-10-09T09:20:49.358 INFO:tasks.rados.rados.0.smithi163.stdout:3645: seq_num 1366 ranges {558362=417321,1451697=673806,2712514=245848} 2023-10-09T09:20:49.366 INFO:tasks.rados.rados.0.smithi163.stdout:3645: writing smithi16367185-16 from 558362 to 975683 tid 1 2023-10-09T09:20:49.374 INFO:tasks.rados.rados.0.smithi163.stdout:3645: writing smithi16367185-16 from 1451697 to 2125503 tid 2 2023-10-09T09:20:49.379 INFO:tasks.rados.rados.0.smithi163.stdout:3645: writing smithi16367185-16 from 2712514 to 2958362 tid 3 2023-10-09T09:20:49.379 INFO:tasks.rados.rados.0.smithi163.stdout:3646: snap_remove snap 456 2023-10-09T09:20:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:49 smithi163 bash[75425]: cluster 2023-10-09T09:20:48.340392+0000 osd.5 (osd.5) 515 : cluster 3 failed to encode map e974 with expected crc 2023-10-09T09:20:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:49 smithi163 bash[75425]: cluster 2023-10-09T09:20:48.340552+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e974 with expected crc 2023-10-09T09:20:49.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:49 smithi163 bash[75425]: cluster 2023-10-09T09:20:49.343607+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e975: 8 total, 8 up, 8 in 2023-10-09T09:20:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:49 smithi163 bash[78107]: cluster 2023-10-09T09:20:48.340392+0000 osd.5 (osd.5) 515 : cluster 3 failed to encode map e974 with expected crc 2023-10-09T09:20:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:49 smithi163 bash[78107]: cluster 2023-10-09T09:20:48.340552+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e974 with expected crc 2023-10-09T09:20:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:49 smithi163 bash[78107]: cluster 2023-10-09T09:20:49.343607+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e975: 8 total, 8 up, 8 in 2023-10-09T09:20:49.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:49 smithi181 bash[59345]: cluster 2023-10-09T09:20:48.340392+0000 osd.5 (osd.5) 515 : cluster 3 failed to encode map e974 with expected crc 2023-10-09T09:20:49.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:49 smithi181 bash[59345]: cluster 2023-10-09T09:20:48.340552+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e974 with expected crc 2023-10-09T09:20:49.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:49 smithi181 bash[59345]: cluster 2023-10-09T09:20:49.343607+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e975: 8 total, 8 up, 8 in 2023-10-09T09:20:50.323 INFO:tasks.rados.rados.0.smithi163.stdout:3646: done (2 left) 2023-10-09T09:20:50.323 INFO:tasks.rados.rados.0.smithi163.stdout:3647: snap_remove snap 445 2023-10-09T09:20:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[75425]: cluster 2023-10-09T09:20:49.004722+0000 osd.7 (osd.7) 413 : cluster 3 failed to encode map e974 with expected crc 2023-10-09T09:20:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[75425]: cluster 2023-10-09T09:20:49.338306+0000 osd.5 (osd.5) 516 : cluster 3 failed to encode map e975 with expected crc 2023-10-09T09:20:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[75425]: cluster 2023-10-09T09:20:49.345239+0000 osd.4 (osd.4) 436 : cluster 3 failed to encode map e975 with expected crc 2023-10-09T09:20:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[75425]: cluster 2023-10-09T09:20:49.465250+0000 mgr.y (mgr.34107) 428 : cluster 0 pgmap v1016: 89 pgs: 89 active+clean; 272 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 853 KiB/s rd, 0 B/s wr, 2 op/s 2023-10-09T09:20:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[75425]: cluster 2023-10-09T09:20:50.323371+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e976: 8 total, 8 up, 8 in 2023-10-09T09:20:50.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[75425]: cluster 2023-10-09T09:20:50.326328+0000 osd.4 (osd.4) 437 : cluster 3 failed to encode map e976 with expected crc 2023-10-09T09:20:50.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[78107]: cluster 2023-10-09T09:20:49.004722+0000 osd.7 (osd.7) 413 : cluster 3 failed to encode map e974 with expected crc 2023-10-09T09:20:50.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[78107]: cluster 2023-10-09T09:20:49.338306+0000 osd.5 (osd.5) 516 : cluster 3 failed to encode map e975 with expected crc 2023-10-09T09:20:50.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[78107]: cluster 2023-10-09T09:20:49.345239+0000 osd.4 (osd.4) 436 : cluster 3 failed to encode map e975 with expected crc 2023-10-09T09:20:50.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[78107]: cluster 2023-10-09T09:20:49.465250+0000 mgr.y (mgr.34107) 428 : cluster 0 pgmap v1016: 89 pgs: 89 active+clean; 272 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 853 KiB/s rd, 0 B/s wr, 2 op/s 2023-10-09T09:20:50.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[78107]: cluster 2023-10-09T09:20:50.323371+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e976: 8 total, 8 up, 8 in 2023-10-09T09:20:50.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:50 smithi163 bash[78107]: cluster 2023-10-09T09:20:50.326328+0000 osd.4 (osd.4) 437 : cluster 3 failed to encode map e976 with expected crc 2023-10-09T09:20:50.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:50 smithi181 bash[59345]: cluster 2023-10-09T09:20:49.004722+0000 osd.7 (osd.7) 413 : cluster 3 failed to encode map e974 with expected crc 2023-10-09T09:20:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:50 smithi181 bash[59345]: cluster 2023-10-09T09:20:49.338306+0000 osd.5 (osd.5) 516 : cluster 3 failed to encode map e975 with expected crc 2023-10-09T09:20:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:50 smithi181 bash[59345]: cluster 2023-10-09T09:20:49.345239+0000 osd.4 (osd.4) 436 : cluster 3 failed to encode map e975 with expected crc 2023-10-09T09:20:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:50 smithi181 bash[59345]: cluster 2023-10-09T09:20:49.465250+0000 mgr.y (mgr.34107) 428 : cluster 0 pgmap v1016: 89 pgs: 89 active+clean; 272 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 853 KiB/s rd, 0 B/s wr, 2 op/s 2023-10-09T09:20:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:50 smithi181 bash[59345]: cluster 2023-10-09T09:20:50.323371+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e976: 8 total, 8 up, 8 in 2023-10-09T09:20:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:50 smithi181 bash[59345]: cluster 2023-10-09T09:20:50.326328+0000 osd.4 (osd.4) 437 : cluster 3 failed to encode map e976 with expected crc 2023-10-09T09:20:51.326 INFO:tasks.rados.rados.0.smithi163.stdout:3644: expect (ObjNum 1338 snap 448 seq_num 1338) 2023-10-09T09:20:51.459 INFO:tasks.rados.rados.0.smithi163.stdout:3645: finishing write tid 1 to smithi16367185-16 2023-10-09T09:20:51.459 INFO:tasks.rados.rados.0.smithi163.stdout:3645: finishing write tid 2 to smithi16367185-16 2023-10-09T09:20:51.459 INFO:tasks.rados.rados.0.smithi163.stdout:3645: finishing write tid 3 to smithi16367185-16 2023-10-09T09:20:51.459 INFO:tasks.rados.rados.0.smithi163.stdout:3645: finishing write tid 4 to smithi16367185-16 2023-10-09T09:20:51.459 INFO:tasks.rados.rados.0.smithi163.stdout:3645: finishing write tid 5 to smithi16367185-16 2023-10-09T09:20:51.459 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1941 (ObjNum 1366 snap 458 seq_num 1366) dirty exists 2023-10-09T09:20:51.459 INFO:tasks.rados.rados.0.smithi163.stdout:3645: left oid 16 (ObjNum 1366 snap 458 seq_num 1366) 2023-10-09T09:20:51.460 INFO:tasks.rados.rados.0.smithi163.stdout:3644: done (2 left) 2023-10-09T09:20:51.460 INFO:tasks.rados.rados.0.smithi163.stdout:3645: done (1 left) 2023-10-09T09:20:51.460 INFO:tasks.rados.rados.0.smithi163.stdout:3647: done (0 left) 2023-10-09T09:20:51.460 INFO:tasks.rados.rados.0.smithi163.stdout:3648: delete oid 48 current snap is 458 2023-10-09T09:20:51.465 INFO:tasks.rados.rados.0.smithi163.stdout:3648: done (0 left) 2023-10-09T09:20:51.465 INFO:tasks.rados.rados.0.smithi163.stdout:3649: snap_create 2023-10-09T09:20:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:51 smithi163 bash[75425]: cluster 2023-10-09T09:20:50.318878+0000 osd.5 (osd.5) 517 : cluster 3 failed to encode map e976 with expected crc 2023-10-09T09:20:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:51 smithi163 bash[75425]: cluster 2023-10-09T09:20:51.325757+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e977: 8 total, 8 up, 8 in 2023-10-09T09:20:51.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:51 smithi163 bash[75425]: cluster 2023-10-09T09:20:51.326222+0000 osd.4 (osd.4) 438 : cluster 3 failed to encode map e977 with expected crc 2023-10-09T09:20:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:51 smithi163 bash[78107]: cluster 2023-10-09T09:20:50.318878+0000 osd.5 (osd.5) 517 : cluster 3 failed to encode map e976 with expected crc 2023-10-09T09:20:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:51 smithi163 bash[78107]: cluster 2023-10-09T09:20:51.325757+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e977: 8 total, 8 up, 8 in 2023-10-09T09:20:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:51 smithi163 bash[78107]: cluster 2023-10-09T09:20:51.326222+0000 osd.4 (osd.4) 438 : cluster 3 failed to encode map e977 with expected crc 2023-10-09T09:20:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:51 smithi181 bash[59345]: cluster 2023-10-09T09:20:50.318878+0000 osd.5 (osd.5) 517 : cluster 3 failed to encode map e976 with expected crc 2023-10-09T09:20:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:51 smithi181 bash[59345]: cluster 2023-10-09T09:20:51.325757+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e977: 8 total, 8 up, 8 in 2023-10-09T09:20:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:51 smithi181 bash[59345]: cluster 2023-10-09T09:20:51.326222+0000 osd.4 (osd.4) 438 : cluster 3 failed to encode map e977 with expected crc 2023-10-09T09:20:52.414 INFO:tasks.rados.rados.0.smithi163.stdout:3649: done (0 left) 2023-10-09T09:20:52.414 INFO:tasks.rados.rados.0.smithi163.stdout:3650: read oid 47 snap -1 2023-10-09T09:20:52.414 INFO:tasks.rados.rados.0.smithi163.stdout:3650: expect (ObjNum 1337 snap 448 seq_num 1337) 2023-10-09T09:20:52.414 INFO:tasks.rados.rados.0.smithi163.stdout:3651: write oid 4 current snap is 459 2023-10-09T09:20:52.414 INFO:tasks.rados.rados.0.smithi163.stdout:3651: seq_num 1368 ranges {591976=567819,1852308=677475} 2023-10-09T09:20:52.425 INFO:tasks.rados.rados.0.smithi163.stdout:3651: writing smithi16367185-4 from 591976 to 1159795 tid 1 2023-10-09T09:20:52.436 INFO:tasks.rados.rados.0.smithi163.stdout:3651: writing smithi16367185-4 from 1852308 to 2529783 tid 2 2023-10-09T09:20:52.436 INFO:tasks.rados.rados.0.smithi163.stdout:3652: write oid 13 current snap is 459 2023-10-09T09:20:52.436 INFO:tasks.rados.rados.0.smithi163.stdout:3652: seq_num 1369 ranges {608783=759421,2027143=421133} 2023-10-09T09:20:52.444 INFO:tasks.rados.rados.0.smithi163.stdout:3652: writing smithi16367185-13 from 608783 to 1368204 tid 1 2023-10-09T09:20:52.451 INFO:tasks.rados.rados.0.smithi163.stdout:3652: writing smithi16367185-13 from 2027143 to 2448276 tid 2 2023-10-09T09:20:52.451 INFO:tasks.rados.rados.0.smithi163.stdout:3653: write oid 14 current snap is 459 2023-10-09T09:20:52.451 INFO:tasks.rados.rados.0.smithi163.stdout:3653: seq_num 1370 ranges {625590=434670,1685625=681144,2822206=203384} 2023-10-09T09:20:52.457 INFO:tasks.rados.rados.0.smithi163.stdout:3653: writing smithi16367185-14 from 625590 to 1060260 tid 1 2023-10-09T09:20:52.464 INFO:tasks.rados.rados.0.smithi163.stdout:3653: writing smithi16367185-14 from 1685625 to 2366769 tid 2 2023-10-09T09:20:52.468 INFO:tasks.rados.rados.0.smithi163.stdout:3653: writing smithi16367185-14 from 2822206 to 3025590 tid 3 2023-10-09T09:20:52.468 INFO:tasks.rados.rados.0.smithi163.stdout:3654: write oid 16 current snap is 459 2023-10-09T09:20:52.468 INFO:tasks.rados.rados.0.smithi163.stdout:3650: expect (ObjNum 1337 snap 448 seq_num 1337) 2023-10-09T09:20:52.593 INFO:tasks.rados.rados.0.smithi163.stdout:3651: finishing write tid 1 to smithi16367185-4 2023-10-09T09:20:52.594 INFO:tasks.rados.rados.0.smithi163.stdout:3651: finishing write tid 2 to smithi16367185-4 2023-10-09T09:20:52.594 INFO:tasks.rados.rados.0.smithi163.stdout:3651: finishing write tid 3 to smithi16367185-4 2023-10-09T09:20:52.594 INFO:tasks.rados.rados.0.smithi163.stdout:3651: finishing write tid 4 to smithi16367185-4 2023-10-09T09:20:52.594 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 1957 (ObjNum 1368 snap 459 seq_num 1368) dirty exists 2023-10-09T09:20:52.594 INFO:tasks.rados.rados.0.smithi163.stdout:3651: left oid 4 (ObjNum 1368 snap 459 seq_num 1368) 2023-10-09T09:20:52.594 INFO:tasks.rados.rados.0.smithi163.stdout:3652: finishing write tid 1 to smithi16367185-13 2023-10-09T09:20:52.594 INFO:tasks.rados.rados.0.smithi163.stdout:3652: finishing write tid 2 to smithi16367185-13 2023-10-09T09:20:52.594 INFO:tasks.rados.rados.0.smithi163.stdout:3652: finishing write tid 3 to smithi16367185-13 2023-10-09T09:20:52.594 INFO:tasks.rados.rados.0.smithi163.stdout:3652: finishing write tid 4 to smithi16367185-13 2023-10-09T09:20:52.594 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 758 (ObjNum 1369 snap 459 seq_num 1369) dirty exists 2023-10-09T09:20:52.595 INFO:tasks.rados.rados.0.smithi163.stdout:3652: left oid 13 (ObjNum 1369 snap 459 seq_num 1369) 2023-10-09T09:20:52.595 INFO:tasks.rados.rados.0.smithi163.stdout:3653: finishing write tid 1 to smithi16367185-14 2023-10-09T09:20:52.595 INFO:tasks.rados.rados.0.smithi163.stdout:3653: finishing write tid 2 to smithi16367185-14 2023-10-09T09:20:52.595 INFO:tasks.rados.rados.0.smithi163.stdout:3653: finishing write tid 3 to smithi16367185-14 2023-10-09T09:20:52.595 INFO:tasks.rados.rados.0.smithi163.stdout:3653: finishing write tid 4 to smithi16367185-14 2023-10-09T09:20:52.595 INFO:tasks.rados.rados.0.smithi163.stdout:3653: finishing write tid 5 to smithi16367185-14 2023-10-09T09:20:52.595 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1716 (ObjNum 1370 snap 459 seq_num 1370) dirty exists 2023-10-09T09:20:52.595 INFO:tasks.rados.rados.0.smithi163.stdout:3653: left oid 14 (ObjNum 1370 snap 459 seq_num 1370) 2023-10-09T09:20:52.595 INFO:tasks.rados.rados.0.smithi163.stdout:3654: seq_num 1371 ranges {642397=509919,1627754=424802,2733276=309121} 2023-10-09T09:20:52.606 INFO:tasks.rados.rados.0.smithi163.stdout:3654: writing smithi16367185-16 from 642397 to 1152316 tid 1 2023-10-09T09:20:52.611 INFO:tasks.rados.rados.0.smithi163.stdout:3654: writing smithi16367185-16 from 1627754 to 2052556 tid 2 2023-10-09T09:20:52.617 INFO:tasks.rados.rados.0.smithi163.stdout:3654: writing smithi16367185-16 from 2733276 to 3042397 tid 3 2023-10-09T09:20:52.617 INFO:tasks.rados.rados.0.smithi163.stdout:3650: done (4 left) 2023-10-09T09:20:52.617 INFO:tasks.rados.rados.0.smithi163.stdout:3651: done (3 left) 2023-10-09T09:20:52.617 INFO:tasks.rados.rados.0.smithi163.stdout:3652: done (2 left) 2023-10-09T09:20:52.617 INFO:tasks.rados.rados.0.smithi163.stdout:3653: done (1 left) 2023-10-09T09:20:52.617 INFO:tasks.rados.rados.0.smithi163.stdout:3655: snap_create 2023-10-09T09:20:52.617 INFO:tasks.rados.rados.0.smithi163.stdout:3654: finishing write tid 1 to smithi16367185-16 2023-10-09T09:20:52.617 INFO:tasks.rados.rados.0.smithi163.stdout:3654: finishing write tid 2 to smithi16367185-16 2023-10-09T09:20:52.620 INFO:tasks.rados.rados.0.smithi163.stdout:3654: finishing write tid 3 to smithi16367185-16 2023-10-09T09:20:52.621 INFO:tasks.rados.rados.0.smithi163.stdout:3654: finishing write tid 4 to smithi16367185-16 2023-10-09T09:20:52.621 INFO:tasks.rados.rados.0.smithi163.stdout:3654: finishing write tid 5 to smithi16367185-16 2023-10-09T09:20:52.621 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1962 (ObjNum 1371 snap 459 seq_num 1371) dirty exists 2023-10-09T09:20:52.621 INFO:tasks.rados.rados.0.smithi163.stdout:3654: left oid 16 (ObjNum 1371 snap 459 seq_num 1371) 2023-10-09T09:20:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:52 smithi163 bash[75425]: cluster 2023-10-09T09:20:51.321811+0000 osd.5 (osd.5) 518 : cluster 3 failed to encode map e977 with expected crc 2023-10-09T09:20:52.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:52 smithi163 bash[75425]: cluster 2023-10-09T09:20:51.465875+0000 mgr.y (mgr.34107) 429 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 640 KiB/s wr, 13 op/s 2023-10-09T09:20:52.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:52 smithi163 bash[78107]: cluster 2023-10-09T09:20:51.321811+0000 osd.5 (osd.5) 518 : cluster 3 failed to encode map e977 with expected crc 2023-10-09T09:20:52.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:52 smithi163 bash[78107]: cluster 2023-10-09T09:20:51.465875+0000 mgr.y (mgr.34107) 429 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 640 KiB/s wr, 13 op/s 2023-10-09T09:20:52.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:52 smithi181 bash[59345]: cluster 2023-10-09T09:20:51.321811+0000 osd.5 (osd.5) 518 : cluster 3 failed to encode map e977 with expected crc 2023-10-09T09:20:52.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:52 smithi181 bash[59345]: cluster 2023-10-09T09:20:51.465875+0000 mgr.y (mgr.34107) 429 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 640 KiB/s wr, 13 op/s 2023-10-09T09:20:53.417 INFO:tasks.rados.rados.0.smithi163.stdout:3654: done (1 left) 2023-10-09T09:20:53.418 INFO:tasks.rados.rados.0.smithi163.stdout:3655: done (0 left) 2023-10-09T09:20:53.418 INFO:tasks.rados.rados.0.smithi163.stdout:3656: delete oid 14 current snap is 460 2023-10-09T09:20:53.432 INFO:tasks.rados.rados.0.smithi163.stdout:3656: done (0 left) 2023-10-09T09:20:53.519 INFO:tasks.rados.rados.0.smithi163.stdout:3657: rollback oid 7 current snap is 460 2023-10-09T09:20:53.519 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 7 to 422 2023-10-09T09:20:53.519 INFO:tasks.rados.rados.0.smithi163.stdout:3658: write oid 20 current snap is 460 2023-10-09T09:20:53.519 INFO:tasks.rados.rados.0.smithi163.stdout:3658: seq_num 1373 ranges {676011=660417,1744718=428471,2788122=287889} 2023-10-09T09:20:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:3658: writing smithi16367185-20 from 676011 to 1336428 tid 1 2023-10-09T09:20:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:3658: writing smithi16367185-20 from 1744718 to 2173189 tid 2 2023-10-09T09:20:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:3658: writing smithi16367185-20 from 2788122 to 3076011 tid 3 2023-10-09T09:20:53.522 INFO:tasks.rados.rados.0.smithi163.stdout:3659: snap_remove snap 453 2023-10-09T09:20:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:20:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:20:53] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:20:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:53 smithi163 bash[75425]: cluster 2023-10-09T09:20:52.407437+0000 osd.5 (osd.5) 519 : cluster 3 failed to encode map e978 with expected crc 2023-10-09T09:20:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:53 smithi163 bash[75425]: cluster 2023-10-09T09:20:52.410252+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e978 with expected crc 2023-10-09T09:20:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:53 smithi163 bash[75425]: cluster 2023-10-09T09:20:52.413119+0000 mon.a (mon.0) 1199 : cluster 0 osdmap e978: 8 total, 8 up, 8 in 2023-10-09T09:20:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:53 smithi163 bash[75425]: cluster 2023-10-09T09:20:52.430273+0000 osd.4 (osd.4) 439 : cluster 3 failed to encode map e978 with expected crc 2023-10-09T09:20:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:53 smithi163 bash[78107]: cluster 2023-10-09T09:20:52.407437+0000 osd.5 (osd.5) 519 : cluster 3 failed to encode map e978 with expected crc 2023-10-09T09:20:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:53 smithi163 bash[78107]: cluster 2023-10-09T09:20:52.410252+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e978 with expected crc 2023-10-09T09:20:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:53 smithi163 bash[78107]: cluster 2023-10-09T09:20:52.413119+0000 mon.a (mon.0) 1199 : cluster 0 osdmap e978: 8 total, 8 up, 8 in 2023-10-09T09:20:53.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:53 smithi163 bash[78107]: cluster 2023-10-09T09:20:52.430273+0000 osd.4 (osd.4) 439 : cluster 3 failed to encode map e978 with expected crc 2023-10-09T09:20:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:53 smithi181 bash[59345]: cluster 2023-10-09T09:20:52.407437+0000 osd.5 (osd.5) 519 : cluster 3 failed to encode map e978 with expected crc 2023-10-09T09:20:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:53 smithi181 bash[59345]: cluster 2023-10-09T09:20:52.410252+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e978 with expected crc 2023-10-09T09:20:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:53 smithi181 bash[59345]: cluster 2023-10-09T09:20:52.413119+0000 mon.a (mon.0) 1199 : cluster 0 osdmap e978: 8 total, 8 up, 8 in 2023-10-09T09:20:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:53 smithi181 bash[59345]: cluster 2023-10-09T09:20:52.430273+0000 osd.4 (osd.4) 439 : cluster 3 failed to encode map e978 with expected crc 2023-10-09T09:20:54.441 INFO:tasks.rados.rados.0.smithi163.stdout:3659: done (2 left) 2023-10-09T09:20:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:3660: write oid 18 current snap is 460 2023-10-09T09:20:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:3657: finishing rollback tid 0 to smithi16367185-7 2023-10-09T09:20:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:3657: finishing rollback tid 1 to smithi16367185-7 2023-10-09T09:20:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:3657: finishing rollback tid 2 to smithi16367185-7 2023-10-09T09:20:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 7 v 956 (ObjNum 1178 snap 397 seq_num 1178) dirty exists 2023-10-09T09:20:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:3658: finishing write tid 1 to smithi16367185-20 2023-10-09T09:20:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:3658: finishing write tid 2 to smithi16367185-20 2023-10-09T09:20:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:3658: finishing write tid 3 to smithi16367185-20 2023-10-09T09:20:54.442 INFO:tasks.rados.rados.0.smithi163.stdout:3658: finishing write tid 4 to smithi16367185-20 2023-10-09T09:20:54.443 INFO:tasks.rados.rados.0.smithi163.stdout:3658: finishing write tid 5 to smithi16367185-20 2023-10-09T09:20:54.443 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 763 (ObjNum 1373 snap 460 seq_num 1373) dirty exists 2023-10-09T09:20:54.443 INFO:tasks.rados.rados.0.smithi163.stdout:3658: left oid 20 (ObjNum 1373 snap 460 seq_num 1373) 2023-10-09T09:20:54.443 INFO:tasks.rados.rados.0.smithi163.stdout:3660: seq_num 1374 ranges {692818=735666,2203200=688482} 2023-10-09T09:20:54.451 INFO:tasks.rados.rados.0.smithi163.stdout:3660: writing smithi16367185-18 from 692818 to 1428484 tid 1 2023-10-09T09:20:54.460 INFO:tasks.rados.rados.0.smithi163.stdout:3660: writing smithi16367185-18 from 2203200 to 2891682 tid 2 2023-10-09T09:20:54.461 INFO:tasks.rados.rados.0.smithi163.stdout:3657: done (2 left) 2023-10-09T09:20:54.461 INFO:tasks.rados.rados.0.smithi163.stdout:3658: done (1 left) 2023-10-09T09:20:54.461 INFO:tasks.rados.rados.0.smithi163.stdout:3661: delete oid 29 current snap is 460 2023-10-09T09:20:54.461 INFO:tasks.rados.rados.0.smithi163.stdout:3660: finishing write tid 1 to smithi16367185-18 2023-10-09T09:20:54.465 INFO:tasks.rados.rados.0.smithi163.stdout:3661: done (1 left) 2023-10-09T09:20:54.465 INFO:tasks.rados.rados.0.smithi163.stdout:3662: write oid 22 current snap is 460 2023-10-09T09:20:54.465 INFO:tasks.rados.rados.0.smithi163.stdout:3662: seq_num 1376 ranges {726432=486164,1803811=692151} 2023-10-09T09:20:54.472 INFO:tasks.rados.rados.0.smithi163.stdout:3662: writing smithi16367185-22 from 726432 to 1212596 tid 1 2023-10-09T09:20:54.480 INFO:tasks.rados.rados.0.smithi163.stdout:3662: writing smithi16367185-22 from 1803811 to 2495962 tid 2 2023-10-09T09:20:54.480 INFO:tasks.rados.rados.0.smithi163.stdout:3663: read oid 33 snap 409 2023-10-09T09:20:54.480 INFO:tasks.rados.rados.0.smithi163.stdout:3663: expect deleted 2023-10-09T09:20:54.480 INFO:tasks.rados.rados.0.smithi163.stdout:3664: rollback oid 44 current snap is 460 2023-10-09T09:20:54.481 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 44 to 458 2023-10-09T09:20:54.481 INFO:tasks.rados.rados.0.smithi163.stdout:3660: finishing write tid 2 to smithi16367185-18 2023-10-09T09:20:54.481 INFO:tasks.rados.rados.0.smithi163.stdout:3665: write oid 6 current snap is 460 2023-10-09T09:20:54.481 INFO:tasks.rados.rados.0.smithi163.stdout:3665: seq_num 1377 ranges {743239=677766,1978646=435809,2897814=245425} 2023-10-09T09:20:54.488 INFO:tasks.rados.rados.0.smithi163.stdout:3665: writing smithi16367185-6 from 743239 to 1421005 tid 1 2023-10-09T09:20:54.494 INFO:tasks.rados.rados.0.smithi163.stdout:3665: writing smithi16367185-6 from 1978646 to 2414455 tid 2 2023-10-09T09:20:54.498 INFO:tasks.rados.rados.0.smithi163.stdout:3665: writing smithi16367185-6 from 2897814 to 3143239 tid 3 2023-10-09T09:20:54.499 INFO:tasks.rados.rados.0.smithi163.stdout:3666: read oid 31 snap -1 2023-10-09T09:20:54.499 INFO:tasks.rados.rados.0.smithi163.stdout:3666: expect deleted 2023-10-09T09:20:54.499 INFO:tasks.rados.rados.0.smithi163.stdout:3667: write oid 36 current snap is 460 2023-10-09T09:20:54.499 INFO:tasks.rados.rados.0.smithi163.stdout:3660: finishing write tid 3 to smithi16367185-18 2023-10-09T09:20:54.499 INFO:tasks.rados.rados.0.smithi163.stdout:3667: seq_num 1378 ranges {760046=753015,2037128=695820} 2023-10-09T09:20:54.506 INFO:tasks.rados.rados.0.smithi163.stdout:3667: writing smithi16367185-36 from 760046 to 1513061 tid 1 2023-10-09T09:20:54.514 INFO:tasks.rados.rados.0.smithi163.stdout:3667: writing smithi16367185-36 from 2037128 to 2732948 tid 2 2023-10-09T09:20:54.514 INFO:tasks.rados.rados.0.smithi163.stdout:3668: read oid 25 snap -1 2023-10-09T09:20:54.514 INFO:tasks.rados.rados.0.smithi163.stdout:3668: expect (ObjNum 1304 snap 440 seq_num 1304) 2023-10-09T09:20:54.515 INFO:tasks.rados.rados.0.smithi163.stdout:3669: snap_create 2023-10-09T09:20:54.515 INFO:tasks.rados.rados.0.smithi163.stdout:3660: finishing write tid 4 to smithi16367185-18 2023-10-09T09:20:54.515 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1274 (ObjNum 1374 snap 460 seq_num 1374) dirty exists 2023-10-09T09:20:54.515 INFO:tasks.rados.rados.0.smithi163.stdout:3660: left oid 18 (ObjNum 1374 snap 460 seq_num 1374) 2023-10-09T09:20:54.515 INFO:tasks.rados.rados.0.smithi163.stdout:3662: finishing write tid 1 to smithi16367185-22 2023-10-09T09:20:54.515 INFO:tasks.rados.rados.0.smithi163.stdout:3662: finishing write tid 2 to smithi16367185-22 2023-10-09T09:20:54.515 INFO:tasks.rados.rados.0.smithi163.stdout:3662: finishing write tid 3 to smithi16367185-22 2023-10-09T09:20:54.515 INFO:tasks.rados.rados.0.smithi163.stdout:3662: finishing write tid 4 to smithi16367185-22 2023-10-09T09:20:54.515 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 2794 (ObjNum 1376 snap 460 seq_num 1376) dirty exists 2023-10-09T09:20:54.515 INFO:tasks.rados.rados.0.smithi163.stdout:3662: left oid 22 (ObjNum 1376 snap 460 seq_num 1376) 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:3664: finishing rollback tid 0 to smithi16367185-44 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:3664: finishing rollback tid 1 to smithi16367185-44 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:3665: finishing write tid 1 to smithi16367185-6 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:3664: finishing rollback tid 2 to smithi16367185-44 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 2798 (ObjNum 1335 snap 448 seq_num 1335) dirty exists 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:3665: finishing write tid 2 to smithi16367185-6 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:3665: finishing write tid 3 to smithi16367185-6 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:3665: finishing write tid 4 to smithi16367185-6 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:3665: finishing write tid 5 to smithi16367185-6 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 413 (ObjNum 1377 snap 460 seq_num 1377) dirty exists 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:3665: left oid 6 (ObjNum 1377 snap 460 seq_num 1377) 2023-10-09T09:20:54.516 INFO:tasks.rados.rados.0.smithi163.stdout:3667: finishing write tid 1 to smithi16367185-36 2023-10-09T09:20:54.518 INFO:tasks.rados.rados.0.smithi163.stdout:3667: finishing write tid 2 to smithi16367185-36 2023-10-09T09:20:54.518 INFO:tasks.rados.rados.0.smithi163.stdout:3667: finishing write tid 3 to smithi16367185-36 2023-10-09T09:20:54.518 INFO:tasks.rados.rados.0.smithi163.stdout:3667: finishing write tid 4 to smithi16367185-36 2023-10-09T09:20:54.518 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 1066 (ObjNum 1378 snap 460 seq_num 1378) dirty exists 2023-10-09T09:20:54.518 INFO:tasks.rados.rados.0.smithi163.stdout:3667: left oid 36 (ObjNum 1378 snap 460 seq_num 1378) 2023-10-09T09:20:54.522 INFO:tasks.rados.rados.0.smithi163.stdout:3668: expect (ObjNum 1304 snap 440 seq_num 1304) 2023-10-09T09:20:54.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:54 smithi163 bash[78107]: cluster 2023-10-09T09:20:53.412555+0000 osd.5 (osd.5) 520 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:54.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:54 smithi163 bash[78107]: cluster 2023-10-09T09:20:53.416730+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e979: 8 total, 8 up, 8 in 2023-10-09T09:20:54.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:54 smithi163 bash[78107]: cluster 2023-10-09T09:20:53.419628+0000 osd.4 (osd.4) 440 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:54.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:54 smithi163 bash[78107]: cluster 2023-10-09T09:20:53.446314+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:54.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:54 smithi163 bash[78107]: cluster 2023-10-09T09:20:53.466498+0000 mgr.y (mgr.34107) 430 : cluster 0 pgmap v1022: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 640 KiB/s wr, 11 op/s 2023-10-09T09:20:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:54 smithi163 bash[75425]: cluster 2023-10-09T09:20:53.412555+0000 osd.5 (osd.5) 520 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:54 smithi163 bash[75425]: cluster 2023-10-09T09:20:53.416730+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e979: 8 total, 8 up, 8 in 2023-10-09T09:20:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:54 smithi163 bash[75425]: cluster 2023-10-09T09:20:53.419628+0000 osd.4 (osd.4) 440 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:54 smithi163 bash[75425]: cluster 2023-10-09T09:20:53.446314+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:54 smithi163 bash[75425]: cluster 2023-10-09T09:20:53.466498+0000 mgr.y (mgr.34107) 430 : cluster 0 pgmap v1022: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 640 KiB/s wr, 11 op/s 2023-10-09T09:20:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:54 smithi181 bash[59345]: cluster 2023-10-09T09:20:53.412555+0000 osd.5 (osd.5) 520 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:54 smithi181 bash[59345]: cluster 2023-10-09T09:20:53.416730+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e979: 8 total, 8 up, 8 in 2023-10-09T09:20:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:54 smithi181 bash[59345]: cluster 2023-10-09T09:20:53.419628+0000 osd.4 (osd.4) 440 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:54 smithi181 bash[59345]: cluster 2023-10-09T09:20:53.446314+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:54.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:54 smithi181 bash[59345]: cluster 2023-10-09T09:20:53.466498+0000 mgr.y (mgr.34107) 430 : cluster 0 pgmap v1022: 89 pgs: 89 active+clean; 270 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 640 KiB/s wr, 11 op/s 2023-10-09T09:20:55.325 INFO:tasks.rados.rados.0.smithi163.stdout:3660: done (8 left) 2023-10-09T09:20:55.325 INFO:tasks.rados.rados.0.smithi163.stdout:3662: done (7 left) 2023-10-09T09:20:55.325 INFO:tasks.rados.rados.0.smithi163.stdout:3663: done (6 left) 2023-10-09T09:20:55.326 INFO:tasks.rados.rados.0.smithi163.stdout:3664: done (5 left) 2023-10-09T09:20:55.326 INFO:tasks.rados.rados.0.smithi163.stdout:3665: done (4 left) 2023-10-09T09:20:55.326 INFO:tasks.rados.rados.0.smithi163.stdout:3666: done (3 left) 2023-10-09T09:20:55.326 INFO:tasks.rados.rados.0.smithi163.stdout:3667: done (2 left) 2023-10-09T09:20:55.326 INFO:tasks.rados.rados.0.smithi163.stdout:3668: done (1 left) 2023-10-09T09:20:55.326 INFO:tasks.rados.rados.0.smithi163.stdout:3669: done (0 left) 2023-10-09T09:20:55.327 INFO:tasks.rados.rados.0.smithi163.stdout:3670: write oid 16 current snap is 461 2023-10-09T09:20:55.327 INFO:tasks.rados.rados.0.smithi163.stdout:3670: seq_num 1379 ranges {776853=428264,1979257=439478,2836307=340546} 2023-10-09T09:20:55.341 INFO:tasks.rados.rados.0.smithi163.stdout:3670: writing smithi16367185-16 from 776853 to 1205117 tid 1 2023-10-09T09:20:55.346 INFO:tasks.rados.rados.0.smithi163.stdout:3670: writing smithi16367185-16 from 1979257 to 2418735 tid 2 2023-10-09T09:20:55.358 INFO:tasks.rados.rados.0.smithi163.stdout:3670: writing smithi16367185-16 from 2836307 to 3176853 tid 3 2023-10-09T09:20:55.358 INFO:tasks.rados.rados.0.smithi163.stdout:3671: snap_remove snap 449 2023-10-09T09:20:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[75425]: cluster 2023-10-09T09:20:54.035623+0000 osd.7 (osd.7) 414 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[75425]: cluster 2023-10-09T09:20:54.436434+0000 osd.5 (osd.5) 521 : cluster 3 failed to encode map e980 with expected crc 2023-10-09T09:20:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[75425]: cluster 2023-10-09T09:20:54.441313+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e980: 8 total, 8 up, 8 in 2023-10-09T09:20:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[75425]: cluster 2023-10-09T09:20:55.323975+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e981: 8 total, 8 up, 8 in 2023-10-09T09:20:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[75425]: cluster 2023-10-09T09:20:55.342849+0000 osd.4 (osd.4) 441 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[75425]: audit 2023-10-09T09:20:55.351894+0000 mon.a (mon.0) 1203 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[78107]: cluster 2023-10-09T09:20:54.035623+0000 osd.7 (osd.7) 414 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[78107]: cluster 2023-10-09T09:20:54.436434+0000 osd.5 (osd.5) 521 : cluster 3 failed to encode map e980 with expected crc 2023-10-09T09:20:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[78107]: cluster 2023-10-09T09:20:54.441313+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e980: 8 total, 8 up, 8 in 2023-10-09T09:20:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[78107]: cluster 2023-10-09T09:20:55.323975+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e981: 8 total, 8 up, 8 in 2023-10-09T09:20:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[78107]: cluster 2023-10-09T09:20:55.342849+0000 osd.4 (osd.4) 441 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:55 smithi163 bash[78107]: audit 2023-10-09T09:20:55.351894+0000 mon.a (mon.0) 1203 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:55.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:55 smithi181 bash[59345]: cluster 2023-10-09T09:20:54.035623+0000 osd.7 (osd.7) 414 : cluster 3 failed to encode map e979 with expected crc 2023-10-09T09:20:55.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:55 smithi181 bash[59345]: cluster 2023-10-09T09:20:54.436434+0000 osd.5 (osd.5) 521 : cluster 3 failed to encode map e980 with expected crc 2023-10-09T09:20:55.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:55 smithi181 bash[59345]: cluster 2023-10-09T09:20:54.441313+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e980: 8 total, 8 up, 8 in 2023-10-09T09:20:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:55 smithi181 bash[59345]: cluster 2023-10-09T09:20:55.323975+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e981: 8 total, 8 up, 8 in 2023-10-09T09:20:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:55 smithi181 bash[59345]: cluster 2023-10-09T09:20:55.342849+0000 osd.4 (osd.4) 441 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:55.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:55 smithi181 bash[59345]: audit 2023-10-09T09:20:55.351894+0000 mon.a (mon.0) 1203 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:20:56.337 INFO:tasks.rados.rados.0.smithi163.stdout:3671: done (1 left) 2023-10-09T09:20:56.337 INFO:tasks.rados.rados.0.smithi163.stdout:3672: delete oid 28 current snap is 461 2023-10-09T09:20:56.337 INFO:tasks.rados.rados.0.smithi163.stdout:3670: finishing write tid 1 to smithi16367185-16 2023-10-09T09:20:56.337 INFO:tasks.rados.rados.0.smithi163.stdout:3670: finishing write tid 2 to smithi16367185-16 2023-10-09T09:20:56.337 INFO:tasks.rados.rados.0.smithi163.stdout:3670: finishing write tid 3 to smithi16367185-16 2023-10-09T09:20:56.337 INFO:tasks.rados.rados.0.smithi163.stdout:3670: finishing write tid 4 to smithi16367185-16 2023-10-09T09:20:56.337 INFO:tasks.rados.rados.0.smithi163.stdout:3670: finishing write tid 5 to smithi16367185-16 2023-10-09T09:20:56.338 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 1973 (ObjNum 1379 snap 461 seq_num 1379) dirty exists 2023-10-09T09:20:56.338 INFO:tasks.rados.rados.0.smithi163.stdout:3670: left oid 16 (ObjNum 1379 snap 461 seq_num 1379) 2023-10-09T09:20:56.349 INFO:tasks.rados.rados.0.smithi163.stdout:3670: done (1 left) 2023-10-09T09:20:56.350 INFO:tasks.rados.rados.0.smithi163.stdout:3672: done (0 left) 2023-10-09T09:20:56.350 INFO:tasks.rados.rados.0.smithi163.stdout:3673: snap_create 2023-10-09T09:20:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:56 smithi163 bash[78107]: cluster 2023-10-09T09:20:55.319560+0000 osd.5 (osd.5) 522 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:56 smithi163 bash[78107]: cluster 2023-10-09T09:20:55.320544+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:56 smithi163 bash[78107]: cluster 2023-10-09T09:20:55.467197+0000 mgr.y (mgr.34107) 431 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 278 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 956 KiB/s wr, 7 op/s 2023-10-09T09:20:56.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:56 smithi163 bash[78107]: cluster 2023-10-09T09:20:56.336835+0000 mon.a (mon.0) 1204 : cluster 0 osdmap e982: 8 total, 8 up, 8 in 2023-10-09T09:20:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:56 smithi163 bash[75425]: cluster 2023-10-09T09:20:55.319560+0000 osd.5 (osd.5) 522 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:56 smithi163 bash[75425]: cluster 2023-10-09T09:20:55.320544+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:56 smithi163 bash[75425]: cluster 2023-10-09T09:20:55.467197+0000 mgr.y (mgr.34107) 431 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 278 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 956 KiB/s wr, 7 op/s 2023-10-09T09:20:56.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:56 smithi163 bash[75425]: cluster 2023-10-09T09:20:56.336835+0000 mon.a (mon.0) 1204 : cluster 0 osdmap e982: 8 total, 8 up, 8 in 2023-10-09T09:20:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:56 smithi181 bash[59345]: cluster 2023-10-09T09:20:55.319560+0000 osd.5 (osd.5) 522 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:56 smithi181 bash[59345]: cluster 2023-10-09T09:20:55.320544+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:56 smithi181 bash[59345]: cluster 2023-10-09T09:20:55.467197+0000 mgr.y (mgr.34107) 431 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 278 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 956 KiB/s wr, 7 op/s 2023-10-09T09:20:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:56 smithi181 bash[59345]: cluster 2023-10-09T09:20:56.336835+0000 mon.a (mon.0) 1204 : cluster 0 osdmap e982: 8 total, 8 up, 8 in 2023-10-09T09:20:57.339 INFO:tasks.rados.rados.0.smithi163.stdout:3673: done (0 left) 2023-10-09T09:20:57.339 INFO:tasks.rados.rados.0.smithi163.stdout:3674: rollback oid 30 current snap is 462 2023-10-09T09:20:57.339 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 458 2023-10-09T09:20:57.340 INFO:tasks.rados.rados.0.smithi163.stdout:3675: snap_remove snap 439 2023-10-09T09:20:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:57 smithi163 bash[75425]: cluster 2023-10-09T09:20:55.641713+0000 osd.7 (osd.7) 415 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:57 smithi163 bash[75425]: cluster 2023-10-09T09:20:56.331489+0000 osd.5 (osd.5) 523 : cluster 3 failed to encode map e982 with expected crc 2023-10-09T09:20:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:57 smithi163 bash[75425]: cluster 2023-10-09T09:20:56.331942+0000 osd.7 (osd.7) 416 : cluster 3 failed to encode map e982 with expected crc 2023-10-09T09:20:57.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:57 smithi163 bash[75425]: cluster 2023-10-09T09:20:57.339282+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e983: 8 total, 8 up, 8 in 2023-10-09T09:20:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:57 smithi163 bash[78107]: cluster 2023-10-09T09:20:55.641713+0000 osd.7 (osd.7) 415 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:57 smithi163 bash[78107]: cluster 2023-10-09T09:20:56.331489+0000 osd.5 (osd.5) 523 : cluster 3 failed to encode map e982 with expected crc 2023-10-09T09:20:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:57 smithi163 bash[78107]: cluster 2023-10-09T09:20:56.331942+0000 osd.7 (osd.7) 416 : cluster 3 failed to encode map e982 with expected crc 2023-10-09T09:20:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:57 smithi163 bash[78107]: cluster 2023-10-09T09:20:57.339282+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e983: 8 total, 8 up, 8 in 2023-10-09T09:20:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:57 smithi181 bash[59345]: cluster 2023-10-09T09:20:55.641713+0000 osd.7 (osd.7) 415 : cluster 3 failed to encode map e981 with expected crc 2023-10-09T09:20:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:57 smithi181 bash[59345]: cluster 2023-10-09T09:20:56.331489+0000 osd.5 (osd.5) 523 : cluster 3 failed to encode map e982 with expected crc 2023-10-09T09:20:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:57 smithi181 bash[59345]: cluster 2023-10-09T09:20:56.331942+0000 osd.7 (osd.7) 416 : cluster 3 failed to encode map e982 with expected crc 2023-10-09T09:20:57.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:57 smithi181 bash[59345]: cluster 2023-10-09T09:20:57.339282+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e983: 8 total, 8 up, 8 in 2023-10-09T09:20:58.343 INFO:tasks.rados.rados.0.smithi163.stdout:3675: done (1 left) 2023-10-09T09:20:58.343 INFO:tasks.rados.rados.0.smithi163.stdout:3676: read oid 9 snap 459 2023-10-09T09:20:58.343 INFO:tasks.rados.rados.0.smithi163.stdout:3676: expect (ObjNum 1357 snap 454 seq_num 1357) 2023-10-09T09:20:58.343 INFO:tasks.rados.rados.0.smithi163.stdout:3677: rollback oid 3 current snap is 462 2023-10-09T09:20:58.343 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 461 2023-10-09T09:20:58.343 INFO:tasks.rados.rados.0.smithi163.stdout:3678: rollback oid 29 current snap is 462 2023-10-09T09:20:58.344 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 29 to 409 2023-10-09T09:20:58.344 INFO:tasks.rados.rados.0.smithi163.stdout:3679: snap_remove snap 454 2023-10-09T09:20:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:58 smithi163 bash[75425]: cluster 2023-10-09T09:20:57.333956+0000 osd.5 (osd.5) 524 : cluster 3 failed to encode map e983 with expected crc 2023-10-09T09:20:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:58 smithi163 bash[75425]: cluster 2023-10-09T09:20:57.467829+0000 mgr.y (mgr.34107) 432 : cluster 0 pgmap v1028: 89 pgs: 89 active+clean; 293 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.3 MiB/s rd, 2.8 MiB/s wr, 18 op/s 2023-10-09T09:20:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:58 smithi163 bash[75425]: cluster 2023-10-09T09:20:58.340750+0000 mon.a (mon.0) 1206 : cluster 0 osdmap e984: 8 total, 8 up, 8 in 2023-10-09T09:20:58.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:58 smithi163 bash[75425]: cluster 2023-10-09T09:20:58.341622+0000 osd.4 (osd.4) 442 : cluster 3 failed to encode map e984 with expected crc 2023-10-09T09:20:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:58 smithi163 bash[78107]: cluster 2023-10-09T09:20:57.333956+0000 osd.5 (osd.5) 524 : cluster 3 failed to encode map e983 with expected crc 2023-10-09T09:20:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:58 smithi163 bash[78107]: cluster 2023-10-09T09:20:57.467829+0000 mgr.y (mgr.34107) 432 : cluster 0 pgmap v1028: 89 pgs: 89 active+clean; 293 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.3 MiB/s rd, 2.8 MiB/s wr, 18 op/s 2023-10-09T09:20:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:58 smithi163 bash[78107]: cluster 2023-10-09T09:20:58.340750+0000 mon.a (mon.0) 1206 : cluster 0 osdmap e984: 8 total, 8 up, 8 in 2023-10-09T09:20:58.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:58 smithi163 bash[78107]: cluster 2023-10-09T09:20:58.341622+0000 osd.4 (osd.4) 442 : cluster 3 failed to encode map e984 with expected crc 2023-10-09T09:20:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:58 smithi181 bash[59345]: cluster 2023-10-09T09:20:57.333956+0000 osd.5 (osd.5) 524 : cluster 3 failed to encode map e983 with expected crc 2023-10-09T09:20:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:58 smithi181 bash[59345]: cluster 2023-10-09T09:20:57.467829+0000 mgr.y (mgr.34107) 432 : cluster 0 pgmap v1028: 89 pgs: 89 active+clean; 293 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.3 MiB/s rd, 2.8 MiB/s wr, 18 op/s 2023-10-09T09:20:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:58 smithi181 bash[59345]: cluster 2023-10-09T09:20:58.340750+0000 mon.a (mon.0) 1206 : cluster 0 osdmap e984: 8 total, 8 up, 8 in 2023-10-09T09:20:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:58 smithi181 bash[59345]: cluster 2023-10-09T09:20:58.341622+0000 osd.4 (osd.4) 442 : cluster 3 failed to encode map e984 with expected crc 2023-10-09T09:20:59.343 INFO:tasks.rados.rados.0.smithi163.stdout:3679: done (4 left) 2023-10-09T09:20:59.343 INFO:tasks.rados.rados.0.smithi163.stdout:3680: rollback oid 22 current snap is 462 2023-10-09T09:20:59.343 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 22 to 459 2023-10-09T09:20:59.343 INFO:tasks.rados.rados.0.smithi163.stdout:3674: finishing rollback tid 0 to smithi16367185-30 2023-10-09T09:20:59.343 INFO:tasks.rados.rados.0.smithi163.stdout:3674: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:20:59.343 INFO:tasks.rados.rados.0.smithi163.stdout:3674: finishing rollback tid 2 to smithi16367185-30 2023-10-09T09:20:59.343 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1280 (ObjNum 1230 snap 410 seq_num 1230) dirty exists 2023-10-09T09:20:59.344 INFO:tasks.rados.rados.0.smithi163.stdout:3676: expect (ObjNum 1357 snap 454 seq_num 1357) 2023-10-09T09:20:59.445 INFO:tasks.rados.rados.0.smithi163.stdout:3678: finishing rollback tid 1 to smithi16367185-29 2023-10-09T09:20:59.445 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1684 (ObjNum 699319568 snap 21875 seq_num 699052848) dirty dne 2023-10-09T09:20:59.445 INFO:tasks.rados.rados.0.smithi163.stdout:3677: finishing rollback tid 0 to smithi16367185-3 2023-10-09T09:20:59.445 INFO:tasks.rados.rados.0.smithi163.stdout:3677: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:20:59.445 INFO:tasks.rados.rados.0.smithi163.stdout:3677: finishing rollback tid 2 to smithi16367185-3 2023-10-09T09:20:59.445 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 1070 (ObjNum 1253 snap 420 seq_num 1253) dirty exists 2023-10-09T09:20:59.445 INFO:tasks.rados.rados.0.smithi163.stdout:3680: finishing rollback tid 0 to smithi16367185-22 2023-10-09T09:20:59.445 INFO:tasks.rados.rados.0.smithi163.stdout:3680: finishing rollback tid 1 to smithi16367185-22 2023-10-09T09:20:59.446 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 2821 (ObjNum 699751776 snap 21875 seq_num 699636688) dirty dne 2023-10-09T09:20:59.446 INFO:tasks.rados.rados.0.smithi163.stdout:3674: done (4 left) 2023-10-09T09:20:59.446 INFO:tasks.rados.rados.0.smithi163.stdout:3676: done (3 left) 2023-10-09T09:20:59.446 INFO:tasks.rados.rados.0.smithi163.stdout:3677: done (2 left) 2023-10-09T09:20:59.446 INFO:tasks.rados.rados.0.smithi163.stdout:3678: done (1 left) 2023-10-09T09:20:59.446 INFO:tasks.rados.rados.0.smithi163.stdout:3680: done (0 left) 2023-10-09T09:20:59.446 INFO:tasks.rados.rados.0.smithi163.stdout:3681: read oid 15 snap 459 2023-10-09T09:20:59.446 INFO:tasks.rados.rados.0.smithi163.stdout:3681: expect (ObjNum 1303 snap 440 seq_num 1303) 2023-10-09T09:20:59.446 INFO:tasks.rados.rados.0.smithi163.stdout:3682: write oid 13 current snap is 462 2023-10-09T09:20:59.446 INFO:tasks.rados.rados.0.smithi163.stdout:3682: seq_num 1381 ranges {410467=578762,1696221=443147,2891153=319314} 2023-10-09T09:20:59.457 INFO:tasks.rados.rados.0.smithi163.stdout:3682: writing smithi16367185-13 from 410467 to 989229 tid 1 2023-10-09T09:20:59.464 INFO:tasks.rados.rados.0.smithi163.stdout:3682: writing smithi16367185-13 from 1696221 to 2139368 tid 2 2023-10-09T09:20:59.470 INFO:tasks.rados.rados.0.smithi163.stdout:3682: writing smithi16367185-13 from 2891153 to 3210467 tid 3 2023-10-09T09:20:59.470 INFO:tasks.rados.rados.0.smithi163.stdout:3683: write oid 41 current snap is 462 2023-10-09T09:20:59.470 INFO:tasks.rados.rados.0.smithi163.stdout:3681: expect (ObjNum 1303 snap 440 seq_num 1303) 2023-10-09T09:20:59.648 INFO:tasks.rados.rados.0.smithi163.stdout:3682: finishing write tid 1 to smithi16367185-13 2023-10-09T09:20:59.648 INFO:tasks.rados.rados.0.smithi163.stdout:3682: finishing write tid 2 to smithi16367185-13 2023-10-09T09:20:59.648 INFO:tasks.rados.rados.0.smithi163.stdout:3682: finishing write tid 3 to smithi16367185-13 2023-10-09T09:20:59.649 INFO:tasks.rados.rados.0.smithi163.stdout:3682: finishing write tid 4 to smithi16367185-13 2023-10-09T09:20:59.649 INFO:tasks.rados.rados.0.smithi163.stdout:3682: finishing write tid 5 to smithi16367185-13 2023-10-09T09:20:59.649 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 784 (ObjNum 1381 snap 462 seq_num 1381) dirty exists 2023-10-09T09:20:59.649 INFO:tasks.rados.rados.0.smithi163.stdout:3682: left oid 13 (ObjNum 1381 snap 462 seq_num 1381) 2023-10-09T09:20:59.649 INFO:tasks.rados.rados.0.smithi163.stdout:3683: seq_num 1382 ranges {427274=654011,1754703=586805,2918576=308698} 2023-10-09T09:20:59.658 INFO:tasks.rados.rados.0.smithi163.stdout:3683: writing smithi16367185-41 from 427274 to 1081285 tid 1 2023-10-09T09:20:59.667 INFO:tasks.rados.rados.0.smithi163.stdout:3683: writing smithi16367185-41 from 1754703 to 2341508 tid 2 2023-10-09T09:20:59.673 INFO:tasks.rados.rados.0.smithi163.stdout:3683: writing smithi16367185-41 from 2918576 to 3227274 tid 3 2023-10-09T09:20:59.674 INFO:tasks.rados.rados.0.smithi163.stdout:3681: done (2 left) 2023-10-09T09:20:59.674 INFO:tasks.rados.rados.0.smithi163.stdout:3682: done (1 left) 2023-10-09T09:20:59.674 INFO:tasks.rados.rados.0.smithi163.stdout:3684: write oid 5 current snap is 462 2023-10-09T09:20:59.674 INFO:tasks.rados.rados.0.smithi163.stdout:3683: finishing write tid 1 to smithi16367185-41 2023-10-09T09:20:59.674 INFO:tasks.rados.rados.0.smithi163.stdout:3683: finishing write tid 2 to smithi16367185-41 2023-10-09T09:20:59.674 INFO:tasks.rados.rados.0.smithi163.stdout:3684: seq_num 1383 ranges {444081=729260,1813185=446816,2945999=298082} 2023-10-09T09:20:59.680 INFO:tasks.rados.rados.0.smithi163.stdout:3684: writing smithi16367185-5 from 444081 to 1173341 tid 1 2023-10-09T09:20:59.686 INFO:tasks.rados.rados.0.smithi163.stdout:3684: writing smithi16367185-5 from 1813185 to 2260001 tid 2 2023-10-09T09:20:59.692 INFO:tasks.rados.rados.0.smithi163.stdout:3684: writing smithi16367185-5 from 2945999 to 3244081 tid 3 2023-10-09T09:20:59.692 INFO:tasks.rados.rados.0.smithi163.stdout:3685: read oid 23 snap 451 2023-10-09T09:20:59.692 INFO:tasks.rados.rados.0.smithi163.stdout:3685: expect deleted 2023-10-09T09:20:59.692 INFO:tasks.rados.rados.0.smithi163.stdout:3686: snap_remove snap 459 2023-10-09T09:20:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:59 smithi163 bash[75425]: cluster 2023-10-09T09:20:58.336822+0000 osd.5 (osd.5) 525 : cluster 3 failed to encode map e984 with expected crc 2023-10-09T09:20:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:20:59 smithi163 bash[75425]: cluster 2023-10-09T09:20:59.342661+0000 mon.a (mon.0) 1207 : cluster 0 osdmap e985: 8 total, 8 up, 8 in 2023-10-09T09:20:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:59 smithi163 bash[78107]: cluster 2023-10-09T09:20:58.336822+0000 osd.5 (osd.5) 525 : cluster 3 failed to encode map e984 with expected crc 2023-10-09T09:20:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:20:59 smithi163 bash[78107]: cluster 2023-10-09T09:20:59.342661+0000 mon.a (mon.0) 1207 : cluster 0 osdmap e985: 8 total, 8 up, 8 in 2023-10-09T09:20:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:59 smithi181 bash[59345]: cluster 2023-10-09T09:20:58.336822+0000 osd.5 (osd.5) 525 : cluster 3 failed to encode map e984 with expected crc 2023-10-09T09:20:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:20:59 smithi181 bash[59345]: cluster 2023-10-09T09:20:59.342661+0000 mon.a (mon.0) 1207 : cluster 0 osdmap e985: 8 total, 8 up, 8 in 2023-10-09T09:21:00.328 INFO:tasks.rados.rados.0.smithi163.stdout:3686: done (3 left) 2023-10-09T09:21:00.328 INFO:tasks.rados.rados.0.smithi163.stdout:3687: write oid 45 current snap is 462 2023-10-09T09:21:00.328 INFO:tasks.rados.rados.0.smithi163.stdout:3683: finishing write tid 3 to smithi16367185-41 2023-10-09T09:21:00.328 INFO:tasks.rados.rados.0.smithi163.stdout:3687: seq_num 1384 ranges {460888=520862,1471667=590474,2573422=455407} 2023-10-09T09:21:00.333 INFO:tasks.rados.rados.0.smithi163.stdout:3687: writing smithi16367185-45 from 460888 to 981750 tid 1 2023-10-09T09:21:00.341 INFO:tasks.rados.rados.0.smithi163.stdout:3687: writing smithi16367185-45 from 1471667 to 2062141 tid 2 2023-10-09T09:21:00.347 INFO:tasks.rados.rados.0.smithi163.stdout:3687: writing smithi16367185-45 from 2573422 to 3028829 tid 3 2023-10-09T09:21:00.347 INFO:tasks.rados.rados.0.smithi163.stdout:3688: delete oid 30 current snap is 462 2023-10-09T09:21:00.347 INFO:tasks.rados.rados.0.smithi163.stdout:3683: finishing write tid 4 to smithi16367185-41 2023-10-09T09:21:00.347 INFO:tasks.rados.rados.0.smithi163.stdout:3683: finishing write tid 5 to smithi16367185-41 2023-10-09T09:21:00.347 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 345 (ObjNum 1382 snap 462 seq_num 1382) dirty exists 2023-10-09T09:21:00.348 INFO:tasks.rados.rados.0.smithi163.stdout:3683: left oid 41 (ObjNum 1382 snap 462 seq_num 1382) 2023-10-09T09:21:00.348 INFO:tasks.rados.rados.0.smithi163.stdout:3684: finishing write tid 1 to smithi16367185-5 2023-10-09T09:21:00.348 INFO:tasks.rados.rados.0.smithi163.stdout:3684: finishing write tid 2 to smithi16367185-5 2023-10-09T09:21:00.348 INFO:tasks.rados.rados.0.smithi163.stdout:3684: finishing write tid 3 to smithi16367185-5 2023-10-09T09:21:00.348 INFO:tasks.rados.rados.0.smithi163.stdout:3684: finishing write tid 4 to smithi16367185-5 2023-10-09T09:21:00.348 INFO:tasks.rados.rados.0.smithi163.stdout:3684: finishing write tid 5 to smithi16367185-5 2023-10-09T09:21:00.348 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 2001 (ObjNum 1383 snap 462 seq_num 1383) dirty exists 2023-10-09T09:21:00.348 INFO:tasks.rados.rados.0.smithi163.stdout:3684: left oid 5 (ObjNum 1383 snap 462 seq_num 1383) 2023-10-09T09:21:00.348 INFO:tasks.rados.rados.0.smithi163.stdout:3687: finishing write tid 1 to smithi16367185-45 2023-10-09T09:21:00.348 INFO:tasks.rados.rados.0.smithi163.stdout:3687: finishing write tid 2 to smithi16367185-45 2023-10-09T09:21:00.353 INFO:tasks.rados.rados.0.smithi163.stdout:3687: finishing write tid 3 to smithi16367185-45 2023-10-09T09:21:00.355 INFO:tasks.rados.rados.0.smithi163.stdout:3687: finishing write tid 4 to smithi16367185-45 2023-10-09T09:21:00.355 INFO:tasks.rados.rados.0.smithi163.stdout:3687: finishing write tid 5 to smithi16367185-45 2023-10-09T09:21:00.355 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 2831 (ObjNum 1384 snap 462 seq_num 1384) dirty exists 2023-10-09T09:21:00.355 INFO:tasks.rados.rados.0.smithi163.stdout:3687: left oid 45 (ObjNum 1384 snap 462 seq_num 1384) 2023-10-09T09:21:00.357 INFO:tasks.rados.rados.0.smithi163.stdout:3683: done (4 left) 2023-10-09T09:21:00.357 INFO:tasks.rados.rados.0.smithi163.stdout:3684: done (3 left) 2023-10-09T09:21:00.358 INFO:tasks.rados.rados.0.smithi163.stdout:3685: done (2 left) 2023-10-09T09:21:00.358 INFO:tasks.rados.rados.0.smithi163.stdout:3687: done (1 left) 2023-10-09T09:21:00.358 INFO:tasks.rados.rados.0.smithi163.stdout:3688: done (0 left) 2023-10-09T09:21:00.359 INFO:tasks.rados.rados.0.smithi163.stdout:3689: delete oid 41 current snap is 462 2023-10-09T09:21:00.360 INFO:tasks.rados.rados.0.smithi163.stdout:3689: done (0 left) 2023-10-09T09:21:00.360 INFO:tasks.rados.rados.0.smithi163.stdout:3690: read oid 39 snap -1 2023-10-09T09:21:00.360 INFO:tasks.rados.rados.0.smithi163.stdout:3690: expect (ObjNum 1310 snap 443 seq_num 1310) 2023-10-09T09:21:00.360 INFO:tasks.rados.rados.0.smithi163.stdout:3691: write oid 45 current snap is 462 2023-10-09T09:21:00.360 INFO:tasks.rados.rados.0.smithi163.stdout:3691: seq_num 1387 ranges {511309=746609,2047113=454154,3055691=255618} 2023-10-09T09:21:00.368 INFO:tasks.rados.rados.0.smithi163.stdout:3691: writing smithi16367185-45 from 511309 to 1257918 tid 1 2023-10-09T09:21:00.375 INFO:tasks.rados.rados.0.smithi163.stdout:3691: writing smithi16367185-45 from 2047113 to 2501267 tid 2 2023-10-09T09:21:00.380 INFO:tasks.rados.rados.0.smithi163.stdout:3691: writing smithi16367185-45 from 3055691 to 3311309 tid 3 2023-10-09T09:21:00.380 INFO:tasks.rados.rados.0.smithi163.stdout:3692: delete oid 43 current snap is 462 2023-10-09T09:21:00.380 INFO:tasks.rados.rados.0.smithi163.stdout:3690: expect (ObjNum 1310 snap 443 seq_num 1310) 2023-10-09T09:21:00.468 INFO:tasks.rados.rados.0.smithi163.stdout:3691: finishing write tid 1 to smithi16367185-45 2023-10-09T09:21:00.468 INFO:tasks.rados.rados.0.smithi163.stdout:3691: finishing write tid 2 to smithi16367185-45 2023-10-09T09:21:00.468 INFO:tasks.rados.rados.0.smithi163.stdout:3691: finishing write tid 3 to smithi16367185-45 2023-10-09T09:21:00.468 INFO:tasks.rados.rados.0.smithi163.stdout:3691: finishing write tid 4 to smithi16367185-45 2023-10-09T09:21:00.468 INFO:tasks.rados.rados.0.smithi163.stdout:3690: done (2 left) 2023-10-09T09:21:00.468 INFO:tasks.rados.rados.0.smithi163.stdout:3692: done (1 left) 2023-10-09T09:21:00.468 INFO:tasks.rados.rados.0.smithi163.stdout:3693: delete oid 25 current snap is 462 2023-10-09T09:21:00.468 INFO:tasks.rados.rados.0.smithi163.stdout:3691: finishing write tid 5 to smithi16367185-45 2023-10-09T09:21:00.469 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 2835 (ObjNum 1387 snap 462 seq_num 1387) dirty exists 2023-10-09T09:21:00.469 INFO:tasks.rados.rados.0.smithi163.stdout:3691: left oid 45 (ObjNum 1387 snap 462 seq_num 1387) 2023-10-09T09:21:00.471 INFO:tasks.rados.rados.0.smithi163.stdout:3691: done (1 left) 2023-10-09T09:21:00.471 INFO:tasks.rados.rados.0.smithi163.stdout:3693: done (0 left) 2023-10-09T09:21:00.471 INFO:tasks.rados.rados.0.smithi163.stdout:3694: delete oid 40 current snap is 462 2023-10-09T09:21:00.475 INFO:tasks.rados.rados.0.smithi163.stdout:3694: done (0 left) 2023-10-09T09:21:00.475 INFO:tasks.rados.rados.0.smithi163.stdout:3695: snap_remove snap 455 2023-10-09T09:21:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:00 smithi163 bash[78107]: cluster 2023-10-09T09:20:59.338009+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e985 with expected crc 2023-10-09T09:21:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:00 smithi163 bash[78107]: cluster 2023-10-09T09:20:59.338100+0000 osd.5 (osd.5) 526 : cluster 3 failed to encode map e985 with expected crc 2023-10-09T09:21:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:00 smithi163 bash[78107]: cluster 2023-10-09T09:20:59.468277+0000 mgr.y (mgr.34107) 433 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2023-10-09T09:21:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:00 smithi163 bash[78107]: cluster 2023-10-09T09:21:00.327003+0000 mon.a (mon.0) 1208 : cluster 0 osdmap e986: 8 total, 8 up, 8 in 2023-10-09T09:21:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:00 smithi163 bash[75425]: cluster 2023-10-09T09:20:59.338009+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e985 with expected crc 2023-10-09T09:21:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:00 smithi163 bash[75425]: cluster 2023-10-09T09:20:59.338100+0000 osd.5 (osd.5) 526 : cluster 3 failed to encode map e985 with expected crc 2023-10-09T09:21:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:00 smithi163 bash[75425]: cluster 2023-10-09T09:20:59.468277+0000 mgr.y (mgr.34107) 433 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2023-10-09T09:21:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:00 smithi163 bash[75425]: cluster 2023-10-09T09:21:00.327003+0000 mon.a (mon.0) 1208 : cluster 0 osdmap e986: 8 total, 8 up, 8 in 2023-10-09T09:21:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:00 smithi181 bash[59345]: cluster 2023-10-09T09:20:59.338009+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e985 with expected crc 2023-10-09T09:21:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:00 smithi181 bash[59345]: cluster 2023-10-09T09:20:59.338100+0000 osd.5 (osd.5) 526 : cluster 3 failed to encode map e985 with expected crc 2023-10-09T09:21:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:00 smithi181 bash[59345]: cluster 2023-10-09T09:20:59.468277+0000 mgr.y (mgr.34107) 433 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2023-10-09T09:21:00.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:00 smithi181 bash[59345]: cluster 2023-10-09T09:21:00.327003+0000 mon.a (mon.0) 1208 : cluster 0 osdmap e986: 8 total, 8 up, 8 in 2023-10-09T09:21:01.480 INFO:tasks.rados.rados.0.smithi163.stdout:3695: done (0 left) 2023-10-09T09:21:01.480 INFO:tasks.rados.rados.0.smithi163.stdout:3696: snap_remove snap 422 2023-10-09T09:21:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:01 smithi181 bash[59345]: cluster 2023-10-09T09:21:00.322121+0000 osd.5 (osd.5) 527 : cluster 3 failed to encode map e986 with expected crc 2023-10-09T09:21:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:01 smithi181 bash[59345]: cluster 2023-10-09T09:21:00.322577+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e986 with expected crc 2023-10-09T09:21:01.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:01 smithi181 bash[59345]: cluster 2023-10-09T09:21:00.356100+0000 osd.7 (osd.7) 417 : cluster 3 failed to encode map e986 with expected crc 2023-10-09T09:21:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:01 smithi163 bash[75425]: cluster 2023-10-09T09:21:00.322121+0000 osd.5 (osd.5) 527 : cluster 3 failed to encode map e986 with expected crc 2023-10-09T09:21:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:01 smithi163 bash[75425]: cluster 2023-10-09T09:21:00.322577+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e986 with expected crc 2023-10-09T09:21:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:01 smithi163 bash[75425]: cluster 2023-10-09T09:21:00.356100+0000 osd.7 (osd.7) 417 : cluster 3 failed to encode map e986 with expected crc 2023-10-09T09:21:01.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:01 smithi163 bash[78107]: cluster 2023-10-09T09:21:00.322121+0000 osd.5 (osd.5) 527 : cluster 3 failed to encode map e986 with expected crc 2023-10-09T09:21:01.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:01 smithi163 bash[78107]: cluster 2023-10-09T09:21:00.322577+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e986 with expected crc 2023-10-09T09:21:01.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:01 smithi163 bash[78107]: cluster 2023-10-09T09:21:00.356100+0000 osd.7 (osd.7) 417 : cluster 3 failed to encode map e986 with expected crc 2023-10-09T09:21:02.481 INFO:tasks.rados.rados.0.smithi163.stdout:3696: done (0 left) 2023-10-09T09:21:02.481 INFO:tasks.rados.rados.0.smithi163.stdout:3697: write oid 31 current snap is 462 2023-10-09T09:21:02.481 INFO:tasks.rados.rados.0.smithi163.stdout:3697: seq_num 1391 ranges {578537=647605,1764688=745139,3049030=329507} 2023-10-09T09:21:02.489 INFO:tasks.rados.rados.0.smithi163.stdout:3697: writing smithi16367185-31 from 578537 to 1226142 tid 1 2023-10-09T09:21:02.497 INFO:tasks.rados.rados.0.smithi163.stdout:3697: writing smithi16367185-31 from 1764688 to 2509827 tid 2 2023-10-09T09:21:02.502 INFO:tasks.rados.rados.0.smithi163.stdout:3697: writing smithi16367185-31 from 3049030 to 3378537 tid 3 2023-10-09T09:21:02.503 INFO:tasks.rados.rados.0.smithi163.stdout:3698: snap_create 2023-10-09T09:21:02.505 INFO:tasks.rados.rados.0.smithi163.stdout:3697: finishing write tid 1 to smithi16367185-31 2023-10-09T09:21:02.508 INFO:tasks.rados.rados.0.smithi163.stdout:3697: finishing write tid 2 to smithi16367185-31 2023-10-09T09:21:02.509 INFO:tasks.rados.rados.0.smithi163.stdout:3697: finishing write tid 3 to smithi16367185-31 2023-10-09T09:21:02.511 INFO:tasks.rados.rados.0.smithi163.stdout:3697: finishing write tid 4 to smithi16367185-31 2023-10-09T09:21:02.511 INFO:tasks.rados.rados.0.smithi163.stdout:3697: finishing write tid 5 to smithi16367185-31 2023-10-09T09:21:02.511 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 1088 (ObjNum 1391 snap 462 seq_num 1391) dirty exists 2023-10-09T09:21:02.511 INFO:tasks.rados.rados.0.smithi163.stdout:3697: left oid 31 (ObjNum 1391 snap 462 seq_num 1391) 2023-10-09T09:21:02.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:02 smithi181 bash[59345]: cluster 2023-10-09T09:21:01.469261+0000 mgr.y (mgr.34107) 434 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 290 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2023-10-09T09:21:02.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:02 smithi181 bash[59345]: cluster 2023-10-09T09:21:01.474068+0000 osd.5 (osd.5) 528 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:02 smithi181 bash[59345]: cluster 2023-10-09T09:21:01.478639+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e987: 8 total, 8 up, 8 in 2023-10-09T09:21:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:02 smithi181 bash[59345]: cluster 2023-10-09T09:21:01.479629+0000 osd.4 (osd.4) 443 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:02 smithi181 bash[59345]: cluster 2023-10-09T09:21:01.483303+0000 osd.4 (osd.4) 444 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:02 smithi181 bash[59345]: cluster 2023-10-09T09:21:01.521976+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:02 smithi181 bash[59345]: cluster 2023-10-09T09:21:02.480759+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e988: 8 total, 8 up, 8 in 2023-10-09T09:21:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[75425]: cluster 2023-10-09T09:21:01.469261+0000 mgr.y (mgr.34107) 434 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 290 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2023-10-09T09:21:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[75425]: cluster 2023-10-09T09:21:01.474068+0000 osd.5 (osd.5) 528 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[75425]: cluster 2023-10-09T09:21:01.478639+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e987: 8 total, 8 up, 8 in 2023-10-09T09:21:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[75425]: cluster 2023-10-09T09:21:01.479629+0000 osd.4 (osd.4) 443 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[75425]: cluster 2023-10-09T09:21:01.483303+0000 osd.4 (osd.4) 444 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[75425]: cluster 2023-10-09T09:21:01.521976+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[75425]: cluster 2023-10-09T09:21:02.480759+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e988: 8 total, 8 up, 8 in 2023-10-09T09:21:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[78107]: cluster 2023-10-09T09:21:01.469261+0000 mgr.y (mgr.34107) 434 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 290 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2023-10-09T09:21:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[78107]: cluster 2023-10-09T09:21:01.474068+0000 osd.5 (osd.5) 528 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[78107]: cluster 2023-10-09T09:21:01.478639+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e987: 8 total, 8 up, 8 in 2023-10-09T09:21:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[78107]: cluster 2023-10-09T09:21:01.479629+0000 osd.4 (osd.4) 443 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[78107]: cluster 2023-10-09T09:21:01.483303+0000 osd.4 (osd.4) 444 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[78107]: cluster 2023-10-09T09:21:01.521976+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e987 with expected crc 2023-10-09T09:21:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:02 smithi163 bash[78107]: cluster 2023-10-09T09:21:02.480759+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e988: 8 total, 8 up, 8 in 2023-10-09T09:21:03.500 INFO:tasks.rados.rados.0.smithi163.stdout:3697: done (1 left) 2023-10-09T09:21:03.501 INFO:tasks.rados.rados.0.smithi163.stdout:3698: done (0 left) 2023-10-09T09:21:03.501 INFO:tasks.rados.rados.0.smithi163.stdout:3699: rollback oid 14 current snap is 463 2023-10-09T09:21:03.501 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 461 2023-10-09T09:21:03.501 INFO:tasks.rados.rados.0.smithi163.stdout:3700: write oid 3 current snap is 463 2023-10-09T09:21:03.501 INFO:tasks.rados.rados.0.smithi163.stdout:3700: seq_num 1392 ranges {595344=439207,1823170=605150,3076453=318891} 2023-10-09T09:21:03.510 INFO:tasks.rados.rados.0.smithi163.stdout:3700: writing smithi16367185-3 from 595344 to 1034551 tid 1 2023-10-09T09:21:03.519 INFO:tasks.rados.rados.0.smithi163.stdout:3700: writing smithi16367185-3 from 1823170 to 2428320 tid 2 2023-10-09T09:21:03.525 INFO:tasks.rados.rados.0.smithi163.stdout:3700: writing smithi16367185-3 from 3076453 to 3395344 tid 3 2023-10-09T09:21:03.525 INFO:tasks.rados.rados.0.smithi163.stdout:3701: delete oid 10 current snap is 463 2023-10-09T09:21:03.525 INFO:tasks.rados.rados.0.smithi163.stdout:3699: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:21:03.525 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 1718 (ObjNum 669367496 snap 21875 seq_num 2) dirty dne 2023-10-09T09:21:03.526 INFO:tasks.rados.rados.0.smithi163.stdout:3700: finishing write tid 1 to smithi16367185-3 2023-10-09T09:21:03.526 INFO:tasks.rados.rados.0.smithi163.stdout:3700: finishing write tid 2 to smithi16367185-3 2023-10-09T09:21:03.528 INFO:tasks.rados.rados.0.smithi163.stdout:3700: finishing write tid 3 to smithi16367185-3 2023-10-09T09:21:03.529 INFO:tasks.rados.rados.0.smithi163.stdout:3700: finishing write tid 4 to smithi16367185-3 2023-10-09T09:21:03.529 INFO:tasks.rados.rados.0.smithi163.stdout:3700: finishing write tid 5 to smithi16367185-3 2023-10-09T09:21:03.529 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 1093 (ObjNum 1392 snap 463 seq_num 1392) dirty exists 2023-10-09T09:21:03.529 INFO:tasks.rados.rados.0.smithi163.stdout:3700: left oid 3 (ObjNum 1392 snap 463 seq_num 1392) 2023-10-09T09:21:03.530 INFO:tasks.rados.rados.0.smithi163.stdout:3699: done (2 left) 2023-10-09T09:21:03.531 INFO:tasks.rados.rados.0.smithi163.stdout:3700: done (1 left) 2023-10-09T09:21:03.531 INFO:tasks.rados.rados.0.smithi163.stdout:3701: done (0 left) 2023-10-09T09:21:03.531 INFO:tasks.rados.rados.0.smithi163.stdout:3702: snap_create 2023-10-09T09:21:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:21:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:21:03] "GET /metrics HTTP/1.1" 200 36263 "" "Prometheus/2.33.4" 2023-10-09T09:21:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:03 smithi163 bash[78107]: cluster 2023-10-09T09:21:02.474537+0000 osd.5 (osd.5) 529 : cluster 3 failed to encode map e988 with expected crc 2023-10-09T09:21:03.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:03 smithi163 bash[78107]: cluster 2023-10-09T09:21:02.475311+0000 osd.4 (osd.4) 445 : cluster 3 failed to encode map e988 with expected crc 2023-10-09T09:21:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:03 smithi163 bash[75425]: cluster 2023-10-09T09:21:02.474537+0000 osd.5 (osd.5) 529 : cluster 3 failed to encode map e988 with expected crc 2023-10-09T09:21:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:03 smithi163 bash[75425]: cluster 2023-10-09T09:21:02.475311+0000 osd.4 (osd.4) 445 : cluster 3 failed to encode map e988 with expected crc 2023-10-09T09:21:03.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:03 smithi181 bash[59345]: cluster 2023-10-09T09:21:02.474537+0000 osd.5 (osd.5) 529 : cluster 3 failed to encode map e988 with expected crc 2023-10-09T09:21:03.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:03 smithi181 bash[59345]: cluster 2023-10-09T09:21:02.475311+0000 osd.4 (osd.4) 445 : cluster 3 failed to encode map e988 with expected crc 2023-10-09T09:21:04.503 INFO:tasks.rados.rados.0.smithi163.stdout:3702: done (0 left) 2023-10-09T09:21:04.503 INFO:tasks.rados.rados.0.smithi163.stdout:3703: read oid 18 snap -1 2023-10-09T09:21:04.503 INFO:tasks.rados.rados.0.smithi163.stdout:3703: expect (ObjNum 1374 snap 460 seq_num 1374) 2023-10-09T09:21:04.503 INFO:tasks.rados.rados.0.smithi163.stdout:3704: rollback oid 32 current snap is 464 2023-10-09T09:21:04.503 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 32 to 462 2023-10-09T09:21:04.503 INFO:tasks.rados.rados.0.smithi163.stdout:3705: write oid 8 current snap is 464 2023-10-09T09:21:04.504 INFO:tasks.rados.rados.0.smithi163.stdout:3705: seq_num 1394 ranges {628958=589705,1940134=608819,3131299=297659} 2023-10-09T09:21:04.510 INFO:tasks.rados.rados.0.smithi163.stdout:3705: writing smithi16367185-8 from 628958 to 1218663 tid 1 2023-10-09T09:21:04.518 INFO:tasks.rados.rados.0.smithi163.stdout:3705: writing smithi16367185-8 from 1940134 to 2548953 tid 2 2023-10-09T09:21:04.524 INFO:tasks.rados.rados.0.smithi163.stdout:3705: writing smithi16367185-8 from 3131299 to 3428958 tid 3 2023-10-09T09:21:04.524 INFO:tasks.rados.rados.0.smithi163.stdout:3706: read oid 16 snap -1 2023-10-09T09:21:04.524 INFO:tasks.rados.rados.0.smithi163.stdout:3706: expect (ObjNum 1379 snap 461 seq_num 1379) 2023-10-09T09:21:04.524 INFO:tasks.rados.rados.0.smithi163.stdout:3707: delete oid 24 current snap is 464 2023-10-09T09:21:04.524 INFO:tasks.rados.rados.0.smithi163.stdout:3704: finishing rollback tid 1 to smithi16367185-32 2023-10-09T09:21:04.524 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 32 v 2774 (ObjNum 2 snap 0 seq_num 1) dirty dne 2023-10-09T09:21:04.524 INFO:tasks.rados.rados.0.smithi163.stdout:3703: expect (ObjNum 1374 snap 460 seq_num 1374) 2023-10-09T09:21:04.631 INFO:tasks.rados.rados.0.smithi163.stdout:3705: finishing write tid 1 to smithi16367185-8 2023-10-09T09:21:04.631 INFO:tasks.rados.rados.0.smithi163.stdout:3705: finishing write tid 2 to smithi16367185-8 2023-10-09T09:21:04.631 INFO:tasks.rados.rados.0.smithi163.stdout:3705: finishing write tid 3 to smithi16367185-8 2023-10-09T09:21:04.631 INFO:tasks.rados.rados.0.smithi163.stdout:3705: finishing write tid 4 to smithi16367185-8 2023-10-09T09:21:04.631 INFO:tasks.rados.rados.0.smithi163.stdout:3705: finishing write tid 5 to smithi16367185-8 2023-10-09T09:21:04.631 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 8 v 1916 (ObjNum 1394 snap 464 seq_num 1394) dirty exists 2023-10-09T09:21:04.631 INFO:tasks.rados.rados.0.smithi163.stdout:3705: left oid 8 (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:21:04.632 INFO:tasks.rados.rados.0.smithi163.stdout:3706: expect (ObjNum 1379 snap 461 seq_num 1379) 2023-10-09T09:21:04.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:04 smithi181 bash[59345]: cluster 2023-10-09T09:21:03.469892+0000 mgr.y (mgr.34107) 435 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2023-10-09T09:21:04.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:04 smithi181 bash[59345]: cluster 2023-10-09T09:21:03.495763+0000 osd.5 (osd.5) 530 : cluster 3 failed to encode map e989 with expected crc 2023-10-09T09:21:04.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:04 smithi181 bash[59345]: cluster 2023-10-09T09:21:03.498331+0000 osd.7 (osd.7) 418 : cluster 3 failed to encode map e989 with expected crc 2023-10-09T09:21:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:04 smithi181 bash[59345]: cluster 2023-10-09T09:21:03.500544+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e989: 8 total, 8 up, 8 in 2023-10-09T09:21:04.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:04 smithi181 bash[59345]: cluster 2023-10-09T09:21:03.503938+0000 osd.4 (osd.4) 446 : cluster 3 failed to encode map e989 with expected crc 2023-10-09T09:21:04.884 INFO:tasks.rados.rados.0.smithi163.stdout:3703: done (4 left) 2023-10-09T09:21:04.884 INFO:tasks.rados.rados.0.smithi163.stdout:3704: done (3 left) 2023-10-09T09:21:04.884 INFO:tasks.rados.rados.0.smithi163.stdout:3705: done (2 left) 2023-10-09T09:21:04.884 INFO:tasks.rados.rados.0.smithi163.stdout:3706: done (1 left) 2023-10-09T09:21:04.884 INFO:tasks.rados.rados.0.smithi163.stdout:3707: done (0 left) 2023-10-09T09:21:04.884 INFO:tasks.rados.rados.0.smithi163.stdout:3708: rollback oid 5 current snap is 464 2023-10-09T09:21:04.884 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 461 2023-10-09T09:21:04.885 INFO:tasks.rados.rados.0.smithi163.stdout:3709: rollback oid 20 current snap is 464 2023-10-09T09:21:04.885 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 20 to 458 2023-10-09T09:21:04.885 INFO:tasks.rados.rados.0.smithi163.stdout:3710: write oid 23 current snap is 464 2023-10-09T09:21:04.885 INFO:tasks.rados.rados.0.smithi163.stdout:3710: seq_num 1396 ranges {662572=740203,1940745=612488,3069792=392780} 2023-10-09T09:21:04.897 INFO:tasks.rados.rados.0.smithi163.stdout:3710: writing smithi16367185-23 from 662572 to 1402775 tid 1 2023-10-09T09:21:04.904 INFO:tasks.rados.rados.0.smithi163.stdout:3710: writing smithi16367185-23 from 1940745 to 2553233 tid 2 2023-10-09T09:21:04.909 INFO:tasks.rados.rados.0.smithi163.stdout:3710: writing smithi16367185-23 from 3069792 to 3462572 tid 3 2023-10-09T09:21:04.909 INFO:tasks.rados.rados.0.smithi163.stdout:3711: read oid 34 snap -1 2023-10-09T09:21:04.909 INFO:tasks.rados.rados.0.smithi163.stdout:3711: expect deleted 2023-10-09T09:21:04.910 INFO:tasks.rados.rados.0.smithi163.stdout:3712: write oid 44 current snap is 464 2023-10-09T09:21:04.910 INFO:tasks.rados.rados.0.smithi163.stdout:3712: seq_num 1397 ranges {679379=415452,1599227=756146,3097215=382164} 2023-10-09T09:21:04.915 INFO:tasks.rados.rados.0.smithi163.stdout:3712: writing smithi16367185-44 from 679379 to 1094831 tid 1 2023-10-09T09:21:04.922 INFO:tasks.rados.rados.0.smithi163.stdout:3712: writing smithi16367185-44 from 1599227 to 2355373 tid 2 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:3712: writing smithi16367185-44 from 3097215 to 3479379 tid 3 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:3713: delete oid 7 current snap is 464 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:3709: finishing rollback tid 0 to smithi16367185-20 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:3708: finishing rollback tid 0 to smithi16367185-5 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:3708: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:3708: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 2021 (ObjNum 1363 snap 457 seq_num 1363) dirty exists 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:3709: finishing rollback tid 1 to smithi16367185-20 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:3709: finishing rollback tid 2 to smithi16367185-20 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 800 (ObjNum 1119 snap 369 seq_num 1119) dirty exists 2023-10-09T09:21:04.930 INFO:tasks.rados.rados.0.smithi163.stdout:3710: finishing write tid 1 to smithi16367185-23 2023-10-09T09:21:04.931 INFO:tasks.rados.rados.0.smithi163.stdout:3710: finishing write tid 2 to smithi16367185-23 2023-10-09T09:21:04.931 INFO:tasks.rados.rados.0.smithi163.stdout:3710: finishing write tid 3 to smithi16367185-23 2023-10-09T09:21:04.931 INFO:tasks.rados.rados.0.smithi163.stdout:3710: finishing write tid 4 to smithi16367185-23 2023-10-09T09:21:04.931 INFO:tasks.rados.rados.0.smithi163.stdout:3710: finishing write tid 5 to smithi16367185-23 2023-10-09T09:21:04.931 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1726 (ObjNum 1396 snap 464 seq_num 1396) dirty exists 2023-10-09T09:21:04.931 INFO:tasks.rados.rados.0.smithi163.stdout:3710: left oid 23 (ObjNum 1396 snap 464 seq_num 1396) 2023-10-09T09:21:04.931 INFO:tasks.rados.rados.0.smithi163.stdout:3712: finishing write tid 1 to smithi16367185-44 2023-10-09T09:21:04.931 INFO:tasks.rados.rados.0.smithi163.stdout:3712: finishing write tid 2 to smithi16367185-44 2023-10-09T09:21:04.933 INFO:tasks.rados.rados.0.smithi163.stdout:3708: done (5 left) 2023-10-09T09:21:04.933 INFO:tasks.rados.rados.0.smithi163.stdout:3709: done (4 left) 2023-10-09T09:21:04.933 INFO:tasks.rados.rados.0.smithi163.stdout:3710: done (3 left) 2023-10-09T09:21:04.933 INFO:tasks.rados.rados.0.smithi163.stdout:3711: done (2 left) 2023-10-09T09:21:04.933 INFO:tasks.rados.rados.0.smithi163.stdout:3713: done (1 left) 2023-10-09T09:21:04.933 INFO:tasks.rados.rados.0.smithi163.stdout:3714: rollback oid 17 current snap is 464 2023-10-09T09:21:04.933 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 460 2023-10-09T09:21:04.933 INFO:tasks.rados.rados.0.smithi163.stdout:3715: snap_remove snap 441 2023-10-09T09:21:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:04 smithi163 bash[78107]: cluster 2023-10-09T09:21:03.469892+0000 mgr.y (mgr.34107) 435 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2023-10-09T09:21:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:04 smithi163 bash[78107]: cluster 2023-10-09T09:21:03.495763+0000 osd.5 (osd.5) 530 : cluster 3 failed to encode map e989 with expected crc 2023-10-09T09:21:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:04 smithi163 bash[78107]: cluster 2023-10-09T09:21:03.498331+0000 osd.7 (osd.7) 418 : cluster 3 failed to encode map e989 with expected crc 2023-10-09T09:21:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:04 smithi163 bash[78107]: cluster 2023-10-09T09:21:03.500544+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e989: 8 total, 8 up, 8 in 2023-10-09T09:21:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:04 smithi163 bash[78107]: cluster 2023-10-09T09:21:03.503938+0000 osd.4 (osd.4) 446 : cluster 3 failed to encode map e989 with expected crc 2023-10-09T09:21:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:04 smithi163 bash[75425]: cluster 2023-10-09T09:21:03.469892+0000 mgr.y (mgr.34107) 435 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 287 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2023-10-09T09:21:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:04 smithi163 bash[75425]: cluster 2023-10-09T09:21:03.495763+0000 osd.5 (osd.5) 530 : cluster 3 failed to encode map e989 with expected crc 2023-10-09T09:21:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:04 smithi163 bash[75425]: cluster 2023-10-09T09:21:03.498331+0000 osd.7 (osd.7) 418 : cluster 3 failed to encode map e989 with expected crc 2023-10-09T09:21:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:04 smithi163 bash[75425]: cluster 2023-10-09T09:21:03.500544+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e989: 8 total, 8 up, 8 in 2023-10-09T09:21:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:04 smithi163 bash[75425]: cluster 2023-10-09T09:21:03.503938+0000 osd.4 (osd.4) 446 : cluster 3 failed to encode map e989 with expected crc 2023-10-09T09:21:05.331 INFO:tasks.rados.rados.0.smithi163.stdout:3715: done (2 left) 2023-10-09T09:21:05.331 INFO:tasks.rados.rados.0.smithi163.stdout:3716: read oid 46 snap -1 2023-10-09T09:21:05.331 INFO:tasks.rados.rados.0.smithi163.stdout:3716: expect deleted 2023-10-09T09:21:05.331 INFO:tasks.rados.rados.0.smithi163.stdout:3717: snap_remove snap 448 2023-10-09T09:21:05.331 INFO:tasks.rados.rados.0.smithi163.stdout:3712: finishing write tid 3 to smithi16367185-44 2023-10-09T09:21:05.331 INFO:tasks.rados.rados.0.smithi163.stdout:3712: finishing write tid 4 to smithi16367185-44 2023-10-09T09:21:05.332 INFO:tasks.rados.rados.0.smithi163.stdout:3712: finishing write tid 5 to smithi16367185-44 2023-10-09T09:21:05.332 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 2859 (ObjNum 1397 snap 464 seq_num 1397) dirty exists 2023-10-09T09:21:05.332 INFO:tasks.rados.rados.0.smithi163.stdout:3712: left oid 44 (ObjNum 1397 snap 464 seq_num 1397) 2023-10-09T09:21:05.332 INFO:tasks.rados.rados.0.smithi163.stdout:3714: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:21:05.332 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1874 (ObjNum 1618308128 snap 32603 seq_num 699633168) dirty dne 2023-10-09T09:21:05.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:05 smithi181 bash[59345]: cluster 2023-10-09T09:21:04.498130+0000 osd.5 (osd.5) 531 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:05 smithi181 bash[59345]: cluster 2023-10-09T09:21:04.498767+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:05 smithi181 bash[59345]: cluster 2023-10-09T09:21:04.502671+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e990: 8 total, 8 up, 8 in 2023-10-09T09:21:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:05 smithi181 bash[59345]: cluster 2023-10-09T09:21:04.513660+0000 osd.7 (osd.7) 419 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:05 smithi181 bash[59345]: cluster 2023-10-09T09:21:04.527415+0000 osd.4 (osd.4) 447 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:05 smithi181 bash[59345]: cluster 2023-10-09T09:21:05.330157+0000 mon.a (mon.0) 1213 : cluster 0 osdmap e991: 8 total, 8 up, 8 in 2023-10-09T09:21:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:05 smithi181 bash[59345]: cluster 2023-10-09T09:21:05.332199+0000 osd.4 (osd.4) 448 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:05 smithi181 bash[59345]: cluster 2023-10-09T09:21:05.333227+0000 osd.7 (osd.7) 420 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:05.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:05 smithi181 bash[59345]: cluster 2023-10-09T09:21:05.337713+0000 osd.4 (osd.4) 449 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:05.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[78107]: cluster 2023-10-09T09:21:04.498130+0000 osd.5 (osd.5) 531 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[78107]: cluster 2023-10-09T09:21:04.498767+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[78107]: cluster 2023-10-09T09:21:04.502671+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e990: 8 total, 8 up, 8 in 2023-10-09T09:21:05.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[78107]: cluster 2023-10-09T09:21:04.513660+0000 osd.7 (osd.7) 419 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[78107]: cluster 2023-10-09T09:21:04.527415+0000 osd.4 (osd.4) 447 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[78107]: cluster 2023-10-09T09:21:05.330157+0000 mon.a (mon.0) 1213 : cluster 0 osdmap e991: 8 total, 8 up, 8 in 2023-10-09T09:21:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[78107]: cluster 2023-10-09T09:21:05.332199+0000 osd.4 (osd.4) 448 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[78107]: cluster 2023-10-09T09:21:05.333227+0000 osd.7 (osd.7) 420 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[78107]: cluster 2023-10-09T09:21:05.337713+0000 osd.4 (osd.4) 449 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[75425]: cluster 2023-10-09T09:21:04.498130+0000 osd.5 (osd.5) 531 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[75425]: cluster 2023-10-09T09:21:04.498767+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[75425]: cluster 2023-10-09T09:21:04.502671+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e990: 8 total, 8 up, 8 in 2023-10-09T09:21:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[75425]: cluster 2023-10-09T09:21:04.513660+0000 osd.7 (osd.7) 419 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[75425]: cluster 2023-10-09T09:21:04.527415+0000 osd.4 (osd.4) 447 : cluster 3 failed to encode map e990 with expected crc 2023-10-09T09:21:05.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[75425]: cluster 2023-10-09T09:21:05.330157+0000 mon.a (mon.0) 1213 : cluster 0 osdmap e991: 8 total, 8 up, 8 in 2023-10-09T09:21:05.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[75425]: cluster 2023-10-09T09:21:05.332199+0000 osd.4 (osd.4) 448 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:05.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[75425]: cluster 2023-10-09T09:21:05.333227+0000 osd.7 (osd.7) 420 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:05.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:05 smithi163 bash[75425]: cluster 2023-10-09T09:21:05.337713+0000 osd.4 (osd.4) 449 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:06.335 INFO:tasks.rados.rados.0.smithi163.stdout:3712: done (3 left) 2023-10-09T09:21:06.335 INFO:tasks.rados.rados.0.smithi163.stdout:3714: done (2 left) 2023-10-09T09:21:06.335 INFO:tasks.rados.rados.0.smithi163.stdout:3717: done (1 left) 2023-10-09T09:21:06.335 INFO:tasks.rados.rados.0.smithi163.stdout:3718: delete oid 38 current snap is 464 2023-10-09T09:21:06.353 INFO:tasks.rados.rados.0.smithi163.stdout:3716: done (1 left) 2023-10-09T09:21:06.354 INFO:tasks.rados.rados.0.smithi163.stdout:3718: done (0 left) 2023-10-09T09:21:06.354 INFO:tasks.rados.rados.0.smithi163.stdout:3719: snap_create 2023-10-09T09:21:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:06 smithi181 bash[59345]: cluster 2023-10-09T09:21:05.324622+0000 osd.5 (osd.5) 532 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:06 smithi181 bash[59345]: cluster 2023-10-09T09:21:05.470479+0000 mgr.y (mgr.34107) 436 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 282 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 0 B/s wr, 6 op/s 2023-10-09T09:21:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:06 smithi181 bash[59345]: cluster 2023-10-09T09:21:06.334840+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e992: 8 total, 8 up, 8 in 2023-10-09T09:21:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:06 smithi181 bash[59345]: cluster 2023-10-09T09:21:06.337187+0000 osd.4 (osd.4) 450 : cluster 3 failed to encode map e992 with expected crc 2023-10-09T09:21:06.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:06 smithi181 bash[59345]: cluster 2023-10-09T09:21:06.348861+0000 osd.7 (osd.7) 421 : cluster 3 failed to encode map e992 with expected crc 2023-10-09T09:21:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:06 smithi163 bash[78107]: cluster 2023-10-09T09:21:05.324622+0000 osd.5 (osd.5) 532 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:06 smithi163 bash[78107]: cluster 2023-10-09T09:21:05.470479+0000 mgr.y (mgr.34107) 436 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 282 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 0 B/s wr, 6 op/s 2023-10-09T09:21:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:06 smithi163 bash[78107]: cluster 2023-10-09T09:21:06.334840+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e992: 8 total, 8 up, 8 in 2023-10-09T09:21:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:06 smithi163 bash[78107]: cluster 2023-10-09T09:21:06.337187+0000 osd.4 (osd.4) 450 : cluster 3 failed to encode map e992 with expected crc 2023-10-09T09:21:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:06 smithi163 bash[78107]: cluster 2023-10-09T09:21:06.348861+0000 osd.7 (osd.7) 421 : cluster 3 failed to encode map e992 with expected crc 2023-10-09T09:21:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:06 smithi163 bash[75425]: cluster 2023-10-09T09:21:05.324622+0000 osd.5 (osd.5) 532 : cluster 3 failed to encode map e991 with expected crc 2023-10-09T09:21:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:06 smithi163 bash[75425]: cluster 2023-10-09T09:21:05.470479+0000 mgr.y (mgr.34107) 436 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 282 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 0 B/s wr, 6 op/s 2023-10-09T09:21:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:06 smithi163 bash[75425]: cluster 2023-10-09T09:21:06.334840+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e992: 8 total, 8 up, 8 in 2023-10-09T09:21:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:06 smithi163 bash[75425]: cluster 2023-10-09T09:21:06.337187+0000 osd.4 (osd.4) 450 : cluster 3 failed to encode map e992 with expected crc 2023-10-09T09:21:06.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:06 smithi163 bash[75425]: cluster 2023-10-09T09:21:06.348861+0000 osd.7 (osd.7) 421 : cluster 3 failed to encode map e992 with expected crc 2023-10-09T09:21:07.347 INFO:tasks.rados.rados.0.smithi163.stdout:3719: done (0 left) 2023-10-09T09:21:07.347 INFO:tasks.rados.rados.0.smithi163.stdout:3720: rollback oid 3 current snap is 465 2023-10-09T09:21:07.348 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 3 to 457 2023-10-09T09:21:07.348 INFO:tasks.rados.rados.0.smithi163.stdout:3721: rollback oid 16 current snap is 465 2023-10-09T09:21:07.348 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 16 to 460 2023-10-09T09:21:07.348 INFO:tasks.rados.rados.0.smithi163.stdout:3722: rollback oid 33 current snap is 465 2023-10-09T09:21:07.348 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 409 2023-10-09T09:21:07.348 INFO:tasks.rados.rados.0.smithi163.stdout:3723: write oid 27 current snap is 465 2023-10-09T09:21:07.348 INFO:tasks.rados.rados.0.smithi163.stdout:3723: seq_num 1400 ranges {729800=757552,2174673=619826} 2023-10-09T09:21:07.356 INFO:tasks.rados.rados.0.smithi163.stdout:3723: writing smithi16367185-27 from 729800 to 1487352 tid 1 2023-10-09T09:21:07.364 INFO:tasks.rados.rados.0.smithi163.stdout:3723: writing smithi16367185-27 from 2174673 to 2794499 tid 2 2023-10-09T09:21:07.365 INFO:tasks.rados.rados.0.smithi163.stdout:3724: snap_create 2023-10-09T09:21:07.365 INFO:tasks.rados.rados.0.smithi163.stdout:3721: finishing rollback tid 0 to smithi16367185-16 2023-10-09T09:21:07.365 INFO:tasks.rados.rados.0.smithi163.stdout:3721: finishing rollback tid 1 to smithi16367185-16 2023-10-09T09:21:07.365 INFO:tasks.rados.rados.0.smithi163.stdout:3721: finishing rollback tid 2 to smithi16367185-16 2023-10-09T09:21:07.365 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 2043 (ObjNum 1371 snap 459 seq_num 1371) dirty exists 2023-10-09T09:21:07.365 INFO:tasks.rados.rados.0.smithi163.stdout:3722: finishing rollback tid 0 to smithi16367185-33 2023-10-09T09:21:07.365 INFO:tasks.rados.rados.0.smithi163.stdout:3722: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:21:07.365 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1716 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:21:07.366 INFO:tasks.rados.rados.0.smithi163.stdout:3720: finishing rollback tid 0 to smithi16367185-3 2023-10-09T09:21:07.366 INFO:tasks.rados.rados.0.smithi163.stdout:3720: finishing rollback tid 1 to smithi16367185-3 2023-10-09T09:21:07.366 INFO:tasks.rados.rados.0.smithi163.stdout:3720: finishing rollback tid 2 to smithi16367185-3 2023-10-09T09:21:07.366 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 1101 (ObjNum 1253 snap 420 seq_num 1253) dirty exists 2023-10-09T09:21:07.370 INFO:tasks.rados.rados.0.smithi163.stdout:3723: finishing write tid 1 to smithi16367185-27 2023-10-09T09:21:07.371 INFO:tasks.rados.rados.0.smithi163.stdout:3723: finishing write tid 2 to smithi16367185-27 2023-10-09T09:21:07.372 INFO:tasks.rados.rados.0.smithi163.stdout:3723: finishing write tid 3 to smithi16367185-27 2023-10-09T09:21:07.372 INFO:tasks.rados.rados.0.smithi163.stdout:3723: finishing write tid 4 to smithi16367185-27 2023-10-09T09:21:07.372 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 1065 (ObjNum 1400 snap 465 seq_num 1400) dirty exists 2023-10-09T09:21:07.372 INFO:tasks.rados.rados.0.smithi163.stdout:3723: left oid 27 (ObjNum 1400 snap 465 seq_num 1400) 2023-10-09T09:21:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:07 smithi181 bash[59345]: cluster 2023-10-09T09:21:06.329418+0000 osd.5 (osd.5) 533 : cluster 3 failed to encode map e992 with expected crc 2023-10-09T09:21:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:07 smithi181 bash[59345]: cluster 2023-10-09T09:21:07.346960+0000 mon.a (mon.0) 1215 : cluster 0 osdmap e993: 8 total, 8 up, 8 in 2023-10-09T09:21:07.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:07 smithi181 bash[59345]: cluster 2023-10-09T09:21:07.348990+0000 osd.4 (osd.4) 451 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:07.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:07 smithi181 bash[59345]: cluster 2023-10-09T09:21:07.359076+0000 osd.7 (osd.7) 422 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:07.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:07 smithi163 bash[75425]: cluster 2023-10-09T09:21:06.329418+0000 osd.5 (osd.5) 533 : cluster 3 failed to encode map e992 with expected crc 2023-10-09T09:21:07.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:07 smithi163 bash[75425]: cluster 2023-10-09T09:21:07.346960+0000 mon.a (mon.0) 1215 : cluster 0 osdmap e993: 8 total, 8 up, 8 in 2023-10-09T09:21:07.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:07 smithi163 bash[75425]: cluster 2023-10-09T09:21:07.348990+0000 osd.4 (osd.4) 451 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:07.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:07 smithi163 bash[75425]: cluster 2023-10-09T09:21:07.359076+0000 osd.7 (osd.7) 422 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:07.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:07 smithi163 bash[78107]: cluster 2023-10-09T09:21:06.329418+0000 osd.5 (osd.5) 533 : cluster 3 failed to encode map e992 with expected crc 2023-10-09T09:21:07.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:07 smithi163 bash[78107]: cluster 2023-10-09T09:21:07.346960+0000 mon.a (mon.0) 1215 : cluster 0 osdmap e993: 8 total, 8 up, 8 in 2023-10-09T09:21:07.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:07 smithi163 bash[78107]: cluster 2023-10-09T09:21:07.348990+0000 osd.4 (osd.4) 451 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:07.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:07 smithi163 bash[78107]: cluster 2023-10-09T09:21:07.359076+0000 osd.7 (osd.7) 422 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:08.352 INFO:tasks.rados.rados.0.smithi163.stdout:3720: done (4 left) 2023-10-09T09:21:08.353 INFO:tasks.rados.rados.0.smithi163.stdout:3721: done (3 left) 2023-10-09T09:21:08.353 INFO:tasks.rados.rados.0.smithi163.stdout:3722: done (2 left) 2023-10-09T09:21:08.353 INFO:tasks.rados.rados.0.smithi163.stdout:3723: done (1 left) 2023-10-09T09:21:08.353 INFO:tasks.rados.rados.0.smithi163.stdout:3724: done (0 left) 2023-10-09T09:21:08.353 INFO:tasks.rados.rados.0.smithi163.stdout:3725: snap_create 2023-10-09T09:21:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:08 smithi181 bash[59345]: cluster 2023-10-09T09:21:07.341602+0000 osd.5 (osd.5) 534 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:08 smithi181 bash[59345]: cluster 2023-10-09T09:21:07.471182+0000 mgr.y (mgr.34107) 437 : cluster 0 pgmap v1043: 89 pgs: 89 active+clean; 266 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2023-10-09T09:21:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:08 smithi181 bash[59345]: cluster 2023-10-09T09:21:08.350112+0000 osd.4 (osd.4) 452 : cluster 3 failed to encode map e994 with expected crc 2023-10-09T09:21:08.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:08 smithi181 bash[59345]: cluster 2023-10-09T09:21:08.352382+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e994: 8 total, 8 up, 8 in 2023-10-09T09:21:08.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:08 smithi163 bash[75425]: cluster 2023-10-09T09:21:07.341602+0000 osd.5 (osd.5) 534 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:08.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:08 smithi163 bash[75425]: cluster 2023-10-09T09:21:07.471182+0000 mgr.y (mgr.34107) 437 : cluster 0 pgmap v1043: 89 pgs: 89 active+clean; 266 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2023-10-09T09:21:08.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:08 smithi163 bash[75425]: cluster 2023-10-09T09:21:08.350112+0000 osd.4 (osd.4) 452 : cluster 3 failed to encode map e994 with expected crc 2023-10-09T09:21:08.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:08 smithi163 bash[75425]: cluster 2023-10-09T09:21:08.352382+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e994: 8 total, 8 up, 8 in 2023-10-09T09:21:08.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:08 smithi163 bash[78107]: cluster 2023-10-09T09:21:07.341602+0000 osd.5 (osd.5) 534 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:08.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:08 smithi163 bash[78107]: cluster 2023-10-09T09:21:07.471182+0000 mgr.y (mgr.34107) 437 : cluster 0 pgmap v1043: 89 pgs: 89 active+clean; 266 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2023-10-09T09:21:08.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:08 smithi163 bash[78107]: cluster 2023-10-09T09:21:08.350112+0000 osd.4 (osd.4) 452 : cluster 3 failed to encode map e994 with expected crc 2023-10-09T09:21:08.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:08 smithi163 bash[78107]: cluster 2023-10-09T09:21:08.352382+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e994: 8 total, 8 up, 8 in 2023-10-09T09:21:09.359 INFO:tasks.rados.rados.0.smithi163.stdout:3725: done (0 left) 2023-10-09T09:21:09.359 INFO:tasks.rados.rados.0.smithi163.stdout:3726: write oid 6 current snap is 467 2023-10-09T09:21:09.359 INFO:tasks.rados.rados.0.smithi163.stdout:3726: seq_num 1401 ranges {746607=432801,1833155=763484,3206907=339700} 2023-10-09T09:21:09.368 INFO:tasks.rados.rados.0.smithi163.stdout:3726: writing smithi16367185-6 from 746607 to 1179408 tid 1 2023-10-09T09:21:09.377 INFO:tasks.rados.rados.0.smithi163.stdout:3726: writing smithi16367185-6 from 1833155 to 2596639 tid 2 2023-10-09T09:21:09.382 INFO:tasks.rados.rados.0.smithi163.stdout:3726: writing smithi16367185-6 from 3206907 to 3546607 tid 3 2023-10-09T09:21:09.383 INFO:tasks.rados.rados.0.smithi163.stdout:3727: read oid 23 snap -1 2023-10-09T09:21:09.383 INFO:tasks.rados.rados.0.smithi163.stdout:3727: expect (ObjNum 1396 snap 464 seq_num 1396) 2023-10-09T09:21:09.383 INFO:tasks.rados.rados.0.smithi163.stdout:3728: snap_remove snap 466 2023-10-09T09:21:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:09 smithi181 bash[59345]: cluster 2023-10-09T09:21:07.845090+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:09 smithi181 bash[59345]: cluster 2023-10-09T09:21:08.349619+0000 osd.5 (osd.5) 535 : cluster 3 failed to encode map e994 with expected crc 2023-10-09T09:21:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:09 smithi181 bash[59345]: cluster 2023-10-09T09:21:09.359279+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e995: 8 total, 8 up, 8 in 2023-10-09T09:21:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:09 smithi181 bash[59345]: cluster 2023-10-09T09:21:09.370660+0000 osd.4 (osd.4) 453 : cluster 3 failed to encode map e995 with expected crc 2023-10-09T09:21:09.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:09 smithi163 bash[78107]: cluster 2023-10-09T09:21:07.845090+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:09.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:09 smithi163 bash[78107]: cluster 2023-10-09T09:21:08.349619+0000 osd.5 (osd.5) 535 : cluster 3 failed to encode map e994 with expected crc 2023-10-09T09:21:09.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:09 smithi163 bash[78107]: cluster 2023-10-09T09:21:09.359279+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e995: 8 total, 8 up, 8 in 2023-10-09T09:21:09.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:09 smithi163 bash[78107]: cluster 2023-10-09T09:21:09.370660+0000 osd.4 (osd.4) 453 : cluster 3 failed to encode map e995 with expected crc 2023-10-09T09:21:09.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:09 smithi163 bash[75425]: cluster 2023-10-09T09:21:07.845090+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e993 with expected crc 2023-10-09T09:21:09.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:09 smithi163 bash[75425]: cluster 2023-10-09T09:21:08.349619+0000 osd.5 (osd.5) 535 : cluster 3 failed to encode map e994 with expected crc 2023-10-09T09:21:09.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:09 smithi163 bash[75425]: cluster 2023-10-09T09:21:09.359279+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e995: 8 total, 8 up, 8 in 2023-10-09T09:21:09.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:09 smithi163 bash[75425]: cluster 2023-10-09T09:21:09.370660+0000 osd.4 (osd.4) 453 : cluster 3 failed to encode map e995 with expected crc 2023-10-09T09:21:10.332 INFO:tasks.rados.rados.0.smithi163.stdout:3728: done (2 left) 2023-10-09T09:21:10.332 INFO:tasks.rados.rados.0.smithi163.stdout:3729: rollback oid 48 current snap is 467 2023-10-09T09:21:10.333 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 48 to 462 2023-10-09T09:21:10.333 INFO:tasks.rados.rados.0.smithi163.stdout:3726: finishing write tid 1 to smithi16367185-6 2023-10-09T09:21:10.333 INFO:tasks.rados.rados.0.smithi163.stdout:3730: snap_remove snap 461 2023-10-09T09:21:10.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:10 smithi181 bash[59345]: cluster 2023-10-09T09:21:09.352406+0000 osd.5 (osd.5) 536 : cluster 3 failed to encode map e995 with expected crc 2023-10-09T09:21:10.875 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:10 smithi181 bash[59345]: cluster 2023-10-09T09:21:09.471948+0000 mgr.y (mgr.34107) 438 : cluster 0 pgmap v1046: 89 pgs: 89 active+clean; 263 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.9 MiB/s wr, 8 op/s 2023-10-09T09:21:10.875 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:10 smithi181 bash[59345]: cluster 2023-10-09T09:21:10.331982+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e996: 8 total, 8 up, 8 in 2023-10-09T09:21:10.875 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:10 smithi181 bash[59345]: audit 2023-10-09T09:21:10.352477+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:10 smithi163 bash[75425]: cluster 2023-10-09T09:21:09.352406+0000 osd.5 (osd.5) 536 : cluster 3 failed to encode map e995 with expected crc 2023-10-09T09:21:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:10 smithi163 bash[75425]: cluster 2023-10-09T09:21:09.471948+0000 mgr.y (mgr.34107) 438 : cluster 0 pgmap v1046: 89 pgs: 89 active+clean; 263 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.9 MiB/s wr, 8 op/s 2023-10-09T09:21:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:10 smithi163 bash[75425]: cluster 2023-10-09T09:21:10.331982+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e996: 8 total, 8 up, 8 in 2023-10-09T09:21:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:10 smithi163 bash[75425]: audit 2023-10-09T09:21:10.352477+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:10.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:10 smithi163 bash[78107]: cluster 2023-10-09T09:21:09.352406+0000 osd.5 (osd.5) 536 : cluster 3 failed to encode map e995 with expected crc 2023-10-09T09:21:10.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:10 smithi163 bash[78107]: cluster 2023-10-09T09:21:09.471948+0000 mgr.y (mgr.34107) 438 : cluster 0 pgmap v1046: 89 pgs: 89 active+clean; 263 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.9 MiB/s wr, 8 op/s 2023-10-09T09:21:10.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:10 smithi163 bash[78107]: cluster 2023-10-09T09:21:10.331982+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e996: 8 total, 8 up, 8 in 2023-10-09T09:21:10.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:10 smithi163 bash[78107]: audit 2023-10-09T09:21:10.352477+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:11.345 INFO:tasks.rados.rados.0.smithi163.stdout:3730: done (3 left) 2023-10-09T09:21:11.345 INFO:tasks.rados.rados.0.smithi163.stdout:3731: rollback oid 30 current snap is 467 2023-10-09T09:21:11.345 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 464 2023-10-09T09:21:11.345 INFO:tasks.rados.rados.0.smithi163.stdout:3726: finishing write tid 2 to smithi16367185-6 2023-10-09T09:21:11.345 INFO:tasks.rados.rados.0.smithi163.stdout:3726: finishing write tid 3 to smithi16367185-6 2023-10-09T09:21:11.345 INFO:tasks.rados.rados.0.smithi163.stdout:3726: finishing write tid 4 to smithi16367185-6 2023-10-09T09:21:11.345 INFO:tasks.rados.rados.0.smithi163.stdout:3726: finishing write tid 5 to smithi16367185-6 2023-10-09T09:21:11.345 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 428 (ObjNum 1401 snap 467 seq_num 1401) dirty exists 2023-10-09T09:21:11.346 INFO:tasks.rados.rados.0.smithi163.stdout:3726: left oid 6 (ObjNum 1401 snap 467 seq_num 1401) 2023-10-09T09:21:11.346 INFO:tasks.rados.rados.0.smithi163.stdout:3727: expect (ObjNum 1396 snap 464 seq_num 1396) 2023-10-09T09:21:11.464 INFO:tasks.rados.rados.0.smithi163.stdout:3729: finishing rollback tid 1 to smithi16367185-48 2023-10-09T09:21:11.464 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 2505 (ObjNum 2 snap 0 seq_num 5) dirty dne 2023-10-09T09:21:11.464 INFO:tasks.rados.rados.0.smithi163.stdout:3731: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:21:11.464 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1289 (ObjNum 1632017880 snap 32603 seq_num 698436240) dirty dne 2023-10-09T09:21:11.464 INFO:tasks.rados.rados.0.smithi163.stdout:3726: done (3 left) 2023-10-09T09:21:11.465 INFO:tasks.rados.rados.0.smithi163.stdout:3727: done (2 left) 2023-10-09T09:21:11.465 INFO:tasks.rados.rados.0.smithi163.stdout:3729: done (1 left) 2023-10-09T09:21:11.465 INFO:tasks.rados.rados.0.smithi163.stdout:3731: done (0 left) 2023-10-09T09:21:11.465 INFO:tasks.rados.rados.0.smithi163.stdout:3732: snap_create 2023-10-09T09:21:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:11 smithi181 bash[59345]: cluster 2023-10-09T09:21:10.327553+0000 osd.5 (osd.5) 537 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:11 smithi181 bash[59345]: cluster 2023-10-09T09:21:10.328176+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:11 smithi181 bash[59345]: audit 2023-10-09T09:21:10.651581+0000 mon.a (mon.0) 1220 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:21:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:11 smithi181 bash[59345]: cluster 2023-10-09T09:21:10.822043+0000 osd.7 (osd.7) 423 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:11 smithi181 bash[59345]: cluster 2023-10-09T09:21:11.127042+0000 osd.4 (osd.4) 454 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:11 smithi181 bash[59345]: cluster 2023-10-09T09:21:11.344288+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e997: 8 total, 8 up, 8 in 2023-10-09T09:21:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:11 smithi181 bash[59345]: cluster 2023-10-09T09:21:11.345151+0000 osd.4 (osd.4) 455 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:11 smithi181 bash[59345]: cluster 2023-10-09T09:21:11.349208+0000 osd.4 (osd.4) 456 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:11.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[78107]: cluster 2023-10-09T09:21:10.327553+0000 osd.5 (osd.5) 537 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[78107]: cluster 2023-10-09T09:21:10.328176+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[78107]: audit 2023-10-09T09:21:10.651581+0000 mon.a (mon.0) 1220 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:21:11.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[78107]: cluster 2023-10-09T09:21:10.822043+0000 osd.7 (osd.7) 423 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[78107]: cluster 2023-10-09T09:21:11.127042+0000 osd.4 (osd.4) 454 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[78107]: cluster 2023-10-09T09:21:11.344288+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e997: 8 total, 8 up, 8 in 2023-10-09T09:21:11.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[78107]: cluster 2023-10-09T09:21:11.345151+0000 osd.4 (osd.4) 455 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:11.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[78107]: cluster 2023-10-09T09:21:11.349208+0000 osd.4 (osd.4) 456 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:11.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[75425]: cluster 2023-10-09T09:21:10.327553+0000 osd.5 (osd.5) 537 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[75425]: cluster 2023-10-09T09:21:10.328176+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[75425]: audit 2023-10-09T09:21:10.651581+0000 mon.a (mon.0) 1220 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:21:11.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[75425]: cluster 2023-10-09T09:21:10.822043+0000 osd.7 (osd.7) 423 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[75425]: cluster 2023-10-09T09:21:11.127042+0000 osd.4 (osd.4) 454 : cluster 3 failed to encode map e996 with expected crc 2023-10-09T09:21:11.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[75425]: cluster 2023-10-09T09:21:11.344288+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e997: 8 total, 8 up, 8 in 2023-10-09T09:21:11.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[75425]: cluster 2023-10-09T09:21:11.345151+0000 osd.4 (osd.4) 455 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:11.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:11 smithi163 bash[75425]: cluster 2023-10-09T09:21:11.349208+0000 osd.4 (osd.4) 456 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:12.347 INFO:tasks.rados.rados.0.smithi163.stdout:3732: done (0 left) 2023-10-09T09:21:12.347 INFO:tasks.rados.rados.0.smithi163.stdout:3733: snap_remove snap 451 2023-10-09T09:21:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:12 smithi181 bash[59345]: cluster 2023-10-09T09:21:11.339441+0000 osd.5 (osd.5) 538 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:12 smithi181 bash[59345]: cluster 2023-10-09T09:21:11.472642+0000 mgr.y (mgr.34107) 439 : cluster 0 pgmap v1049: 89 pgs: 89 active+clean; 258 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 711 KiB/s wr, 7 op/s 2023-10-09T09:21:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:12 smithi181 bash[59345]: cluster 2023-10-09T09:21:11.531564+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:12.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:12 smithi181 bash[59345]: cluster 2023-10-09T09:21:12.347556+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e998: 8 total, 8 up, 8 in 2023-10-09T09:21:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:12 smithi163 bash[78107]: cluster 2023-10-09T09:21:11.339441+0000 osd.5 (osd.5) 538 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:12 smithi163 bash[78107]: cluster 2023-10-09T09:21:11.472642+0000 mgr.y (mgr.34107) 439 : cluster 0 pgmap v1049: 89 pgs: 89 active+clean; 258 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 711 KiB/s wr, 7 op/s 2023-10-09T09:21:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:12 smithi163 bash[78107]: cluster 2023-10-09T09:21:11.531564+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:12 smithi163 bash[78107]: cluster 2023-10-09T09:21:12.347556+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e998: 8 total, 8 up, 8 in 2023-10-09T09:21:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:12 smithi163 bash[75425]: cluster 2023-10-09T09:21:11.339441+0000 osd.5 (osd.5) 538 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:12 smithi163 bash[75425]: cluster 2023-10-09T09:21:11.472642+0000 mgr.y (mgr.34107) 439 : cluster 0 pgmap v1049: 89 pgs: 89 active+clean; 258 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 711 KiB/s wr, 7 op/s 2023-10-09T09:21:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:12 smithi163 bash[75425]: cluster 2023-10-09T09:21:11.531564+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e997 with expected crc 2023-10-09T09:21:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:12 smithi163 bash[75425]: cluster 2023-10-09T09:21:12.347556+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e998: 8 total, 8 up, 8 in 2023-10-09T09:21:13.353 INFO:tasks.rados.rados.0.smithi163.stdout:3733: done (0 left) 2023-10-09T09:21:13.353 INFO:tasks.rados.rados.0.smithi163.stdout:3734: delete oid 41 current snap is 468 2023-10-09T09:21:13.361 INFO:tasks.rados.rados.0.smithi163.stdout:3734: done (0 left) 2023-10-09T09:21:13.362 INFO:tasks.rados.rados.0.smithi163.stdout:3735: write oid 36 current snap is 468 2023-10-09T09:21:13.362 INFO:tasks.rados.rados.0.smithi163.stdout:3735: seq_num 1403 ranges {780221=583299,1950119=767153,3261753=318468} 2023-10-09T09:21:13.370 INFO:tasks.rados.rados.0.smithi163.stdout:3735: writing smithi16367185-36 from 780221 to 1363520 tid 1 2023-10-09T09:21:13.378 INFO:tasks.rados.rados.0.smithi163.stdout:3735: writing smithi16367185-36 from 1950119 to 2717272 tid 2 2023-10-09T09:21:13.383 INFO:tasks.rados.rados.0.smithi163.stdout:3735: writing smithi16367185-36 from 3261753 to 3580221 tid 3 2023-10-09T09:21:13.383 INFO:tasks.rados.rados.0.smithi163.stdout:3736: write oid 9 current snap is 468 2023-10-09T09:21:13.384 INFO:tasks.rados.rados.0.smithi163.stdout:3735: finishing write tid 1 to smithi16367185-36 2023-10-09T09:21:13.384 INFO:tasks.rados.rados.0.smithi163.stdout:3736: seq_num 1404 ranges {797028=658548,1892248=510811,3172823=424205} 2023-10-09T09:21:13.392 INFO:tasks.rados.rados.0.smithi163.stdout:3736: writing smithi16367185-9 from 797028 to 1455576 tid 1 2023-10-09T09:21:13.399 INFO:tasks.rados.rados.0.smithi163.stdout:3736: writing smithi16367185-9 from 1892248 to 2403059 tid 2 2023-10-09T09:21:13.405 INFO:tasks.rados.rados.0.smithi163.stdout:3736: writing smithi16367185-9 from 3172823 to 3597028 tid 3 2023-10-09T09:21:13.406 INFO:tasks.rados.rados.0.smithi163.stdout:3737: write oid 3 current snap is 468 2023-10-09T09:21:13.406 INFO:tasks.rados.rados.0.smithi163.stdout:3737: seq_num 1405 ranges {413835=733797,1550730=770822,2800246=557531} 2023-10-09T09:21:13.412 INFO:tasks.rados.rados.0.smithi163.stdout:3737: writing smithi16367185-3 from 413835 to 1147632 tid 1 2023-10-09T09:21:13.418 INFO:tasks.rados.rados.0.smithi163.stdout:3737: writing smithi16367185-3 from 1550730 to 2321552 tid 2 2023-10-09T09:21:13.424 INFO:tasks.rados.rados.0.smithi163.stdout:3737: writing smithi16367185-3 from 2800246 to 3357777 tid 3 2023-10-09T09:21:13.425 INFO:tasks.rados.rados.0.smithi163.stdout:3738: write oid 27 current snap is 468 2023-10-09T09:21:13.425 INFO:tasks.rados.rados.0.smithi163.stdout:3735: finishing write tid 2 to smithi16367185-36 2023-10-09T09:21:13.425 INFO:tasks.rados.rados.0.smithi163.stdout:3735: finishing write tid 3 to smithi16367185-36 2023-10-09T09:21:13.425 INFO:tasks.rados.rados.0.smithi163.stdout:3735: finishing write tid 4 to smithi16367185-36 2023-10-09T09:21:13.425 INFO:tasks.rados.rados.0.smithi163.stdout:3738: seq_num 1406 ranges {430642=409046,1609212=514480,2827669=768803} 2023-10-09T09:21:13.429 INFO:tasks.rados.rados.0.smithi163.stdout:3738: writing smithi16367185-27 from 430642 to 839688 tid 1 2023-10-09T09:21:13.436 INFO:tasks.rados.rados.0.smithi163.stdout:3738: writing smithi16367185-27 from 1609212 to 2123692 tid 2 2023-10-09T09:21:13.445 INFO:tasks.rados.rados.0.smithi163.stdout:3738: writing smithi16367185-27 from 2827669 to 3596472 tid 3 2023-10-09T09:21:13.445 INFO:tasks.rados.rados.0.smithi163.stdout:3739: delete oid 4 current snap is 468 2023-10-09T09:21:13.445 INFO:tasks.rados.rados.0.smithi163.stdout:3735: finishing write tid 5 to smithi16367185-36 2023-10-09T09:21:13.445 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 1110 (ObjNum 1403 snap 468 seq_num 1403) dirty exists 2023-10-09T09:21:13.445 INFO:tasks.rados.rados.0.smithi163.stdout:3735: left oid 36 (ObjNum 1403 snap 468 seq_num 1403) 2023-10-09T09:21:13.445 INFO:tasks.rados.rados.0.smithi163.stdout:3736: finishing write tid 1 to smithi16367185-9 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:3736: finishing write tid 2 to smithi16367185-9 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:3736: finishing write tid 3 to smithi16367185-9 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:3736: finishing write tid 4 to smithi16367185-9 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:3736: finishing write tid 5 to smithi16367185-9 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 819 (ObjNum 1404 snap 468 seq_num 1404) dirty exists 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:3736: left oid 9 (ObjNum 1404 snap 468 seq_num 1404) 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:3737: finishing write tid 1 to smithi16367185-3 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:3737: finishing write tid 2 to smithi16367185-3 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:3737: finishing write tid 3 to smithi16367185-3 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:3737: finishing write tid 4 to smithi16367185-3 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:3737: finishing write tid 5 to smithi16367185-3 2023-10-09T09:21:13.446 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 1115 (ObjNum 1405 snap 468 seq_num 1405) dirty exists 2023-10-09T09:21:13.447 INFO:tasks.rados.rados.0.smithi163.stdout:3737: left oid 3 (ObjNum 1405 snap 468 seq_num 1405) 2023-10-09T09:21:13.447 INFO:tasks.rados.rados.0.smithi163.stdout:3738: finishing write tid 1 to smithi16367185-27 2023-10-09T09:21:13.447 INFO:tasks.rados.rados.0.smithi163.stdout:3738: finishing write tid 2 to smithi16367185-27 2023-10-09T09:21:13.450 INFO:tasks.rados.rados.0.smithi163.stdout:3735: done (4 left) 2023-10-09T09:21:13.450 INFO:tasks.rados.rados.0.smithi163.stdout:3736: done (3 left) 2023-10-09T09:21:13.450 INFO:tasks.rados.rados.0.smithi163.stdout:3737: done (2 left) 2023-10-09T09:21:13.450 INFO:tasks.rados.rados.0.smithi163.stdout:3739: done (1 left) 2023-10-09T09:21:13.450 INFO:tasks.rados.rados.0.smithi163.stdout:3740: snap_remove snap 458 2023-10-09T09:21:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:21:13] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.33.4" 2023-10-09T09:21:13.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[78107]: cluster 2023-10-09T09:21:12.344963+0000 osd.5 (osd.5) 539 : cluster 3 failed to encode map e998 with expected crc 2023-10-09T09:21:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[78107]: cluster 2023-10-09T09:21:12.345130+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e998 with expected crc 2023-10-09T09:21:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[78107]: cluster 2023-10-09T09:21:12.869652+0000 osd.7 (osd.7) 424 : cluster 3 failed to encode map e998 with expected crc 2023-10-09T09:21:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[78107]: cluster 2023-10-09T09:21:13.351983+0000 mon.a (mon.0) 1223 : cluster 0 osdmap e999: 8 total, 8 up, 8 in 2023-10-09T09:21:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[78107]: cluster 2023-10-09T09:21:13.358158+0000 osd.7 (osd.7) 425 : cluster 3 failed to encode map e999 with expected crc 2023-10-09T09:21:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[75425]: cluster 2023-10-09T09:21:12.344963+0000 osd.5 (osd.5) 539 : cluster 3 failed to encode map e998 with expected crc 2023-10-09T09:21:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[75425]: cluster 2023-10-09T09:21:12.345130+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e998 with expected crc 2023-10-09T09:21:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[75425]: cluster 2023-10-09T09:21:12.869652+0000 osd.7 (osd.7) 424 : cluster 3 failed to encode map e998 with expected crc 2023-10-09T09:21:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[75425]: cluster 2023-10-09T09:21:13.351983+0000 mon.a (mon.0) 1223 : cluster 0 osdmap e999: 8 total, 8 up, 8 in 2023-10-09T09:21:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:13 smithi163 bash[75425]: cluster 2023-10-09T09:21:13.358158+0000 osd.7 (osd.7) 425 : cluster 3 failed to encode map e999 with expected crc 2023-10-09T09:21:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:13 smithi181 bash[59345]: cluster 2023-10-09T09:21:12.344963+0000 osd.5 (osd.5) 539 : cluster 3 failed to encode map e998 with expected crc 2023-10-09T09:21:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:13 smithi181 bash[59345]: cluster 2023-10-09T09:21:12.345130+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e998 with expected crc 2023-10-09T09:21:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:13 smithi181 bash[59345]: cluster 2023-10-09T09:21:12.869652+0000 osd.7 (osd.7) 424 : cluster 3 failed to encode map e998 with expected crc 2023-10-09T09:21:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:13 smithi181 bash[59345]: cluster 2023-10-09T09:21:13.351983+0000 mon.a (mon.0) 1223 : cluster 0 osdmap e999: 8 total, 8 up, 8 in 2023-10-09T09:21:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:13 smithi181 bash[59345]: cluster 2023-10-09T09:21:13.358158+0000 osd.7 (osd.7) 425 : cluster 3 failed to encode map e999 with expected crc 2023-10-09T09:21:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:3740: done (1 left) 2023-10-09T09:21:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:3741: read oid 16 snap -1 2023-10-09T09:21:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:3741: expect (ObjNum 1371 snap 459 seq_num 1371) 2023-10-09T09:21:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:3742: read oid 25 snap -1 2023-10-09T09:21:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:3742: expect deleted 2023-10-09T09:21:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:3743: delete oid 42 current snap is 468 2023-10-09T09:21:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:3738: finishing write tid 3 to smithi16367185-27 2023-10-09T09:21:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:3738: finishing write tid 4 to smithi16367185-27 2023-10-09T09:21:14.360 INFO:tasks.rados.rados.0.smithi163.stdout:3738: finishing write tid 5 to smithi16367185-27 2023-10-09T09:21:14.361 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 1070 (ObjNum 1406 snap 468 seq_num 1406) dirty exists 2023-10-09T09:21:14.361 INFO:tasks.rados.rados.0.smithi163.stdout:3738: left oid 27 (ObjNum 1406 snap 468 seq_num 1406) 2023-10-09T09:21:14.378 INFO:tasks.rados.rados.0.smithi163.stdout:3741: expect (ObjNum 1371 snap 459 seq_num 1371) 2023-10-09T09:21:14.602 INFO:tasks.rados.rados.0.smithi163.stdout:3738: done (3 left) 2023-10-09T09:21:14.602 INFO:tasks.rados.rados.0.smithi163.stdout:3741: done (2 left) 2023-10-09T09:21:14.602 INFO:tasks.rados.rados.0.smithi163.stdout:3742: done (1 left) 2023-10-09T09:21:14.602 INFO:tasks.rados.rados.0.smithi163.stdout:3743: done (0 left) 2023-10-09T09:21:14.603 INFO:tasks.rados.rados.0.smithi163.stdout:3744: read oid 33 snap -1 2023-10-09T09:21:14.603 INFO:tasks.rados.rados.0.smithi163.stdout:3744: expect deleted 2023-10-09T09:21:14.603 INFO:tasks.rados.rados.0.smithi163.stdout:3745: write oid 11 current snap is 468 2023-10-09T09:21:14.603 INFO:tasks.rados.rados.0.smithi163.stdout:3745: seq_num 1409 ranges {481063=751146,1784658=778160,3309938=371125} 2023-10-09T09:21:14.614 INFO:tasks.rados.rados.0.smithi163.stdout:3745: writing smithi16367185-11 from 481063 to 1232209 tid 1 2023-10-09T09:21:14.623 INFO:tasks.rados.rados.0.smithi163.stdout:3745: writing smithi16367185-11 from 1784658 to 2562818 tid 2 2023-10-09T09:21:14.629 INFO:tasks.rados.rados.0.smithi163.stdout:3745: writing smithi16367185-11 from 3309938 to 3681063 tid 3 2023-10-09T09:21:14.629 INFO:tasks.rados.rados.0.smithi163.stdout:3746: delete oid 34 current snap is 468 2023-10-09T09:21:14.629 INFO:tasks.rados.rados.0.smithi163.stdout:3745: finishing write tid 1 to smithi16367185-11 2023-10-09T09:21:14.630 INFO:tasks.rados.rados.0.smithi163.stdout:3745: finishing write tid 2 to smithi16367185-11 2023-10-09T09:21:14.634 INFO:tasks.rados.rados.0.smithi163.stdout:3745: finishing write tid 3 to smithi16367185-11 2023-10-09T09:21:14.634 INFO:tasks.rados.rados.0.smithi163.stdout:3744: done (2 left) 2023-10-09T09:21:14.634 INFO:tasks.rados.rados.0.smithi163.stdout:3746: done (1 left) 2023-10-09T09:21:14.635 INFO:tasks.rados.rados.0.smithi163.stdout:3747: read oid 44 snap -1 2023-10-09T09:21:14.635 INFO:tasks.rados.rados.0.smithi163.stdout:3747: expect (ObjNum 1397 snap 464 seq_num 1397) 2023-10-09T09:21:14.635 INFO:tasks.rados.rados.0.smithi163.stdout:3748: read oid 33 snap 409 2023-10-09T09:21:14.635 INFO:tasks.rados.rados.0.smithi163.stdout:3748: expect deleted 2023-10-09T09:21:14.635 INFO:tasks.rados.rados.0.smithi163.stdout:3749: snap_remove snap 462 2023-10-09T09:21:14.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:14 smithi181 bash[59345]: cluster 2023-10-09T09:21:13.347463+0000 osd.5 (osd.5) 540 : cluster 3 failed to encode map e999 with expected crc 2023-10-09T09:21:14.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:14 smithi181 bash[59345]: cluster 2023-10-09T09:21:13.347938+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e999 with expected crc 2023-10-09T09:21:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:14 smithi181 bash[59345]: cluster 2023-10-09T09:21:13.473263+0000 mgr.y (mgr.34107) 440 : cluster 0 pgmap v1052: 89 pgs: 89 active+clean; 259 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 712 KiB/s wr, 7 op/s 2023-10-09T09:21:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:14 smithi181 bash[59345]: cluster 2023-10-09T09:21:14.359785+0000 mon.a (mon.0) 1224 : cluster 0 osdmap e1000: 8 total, 8 up, 8 in 2023-10-09T09:21:14.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:14 smithi181 bash[59345]: cluster 2023-10-09T09:21:14.363936+0000 osd.4 (osd.4) 457 : cluster 3 failed to encode map e1000 with expected crc 2023-10-09T09:21:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:14 smithi163 bash[75425]: cluster 2023-10-09T09:21:13.347463+0000 osd.5 (osd.5) 540 : cluster 3 failed to encode map e999 with expected crc 2023-10-09T09:21:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:14 smithi163 bash[75425]: cluster 2023-10-09T09:21:13.347938+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e999 with expected crc 2023-10-09T09:21:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:14 smithi163 bash[75425]: cluster 2023-10-09T09:21:13.473263+0000 mgr.y (mgr.34107) 440 : cluster 0 pgmap v1052: 89 pgs: 89 active+clean; 259 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 712 KiB/s wr, 7 op/s 2023-10-09T09:21:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:14 smithi163 bash[75425]: cluster 2023-10-09T09:21:14.359785+0000 mon.a (mon.0) 1224 : cluster 0 osdmap e1000: 8 total, 8 up, 8 in 2023-10-09T09:21:14.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:14 smithi163 bash[75425]: cluster 2023-10-09T09:21:14.363936+0000 osd.4 (osd.4) 457 : cluster 3 failed to encode map e1000 with expected crc 2023-10-09T09:21:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:14 smithi163 bash[78107]: cluster 2023-10-09T09:21:13.347463+0000 osd.5 (osd.5) 540 : cluster 3 failed to encode map e999 with expected crc 2023-10-09T09:21:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:14 smithi163 bash[78107]: cluster 2023-10-09T09:21:13.347938+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e999 with expected crc 2023-10-09T09:21:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:14 smithi163 bash[78107]: cluster 2023-10-09T09:21:13.473263+0000 mgr.y (mgr.34107) 440 : cluster 0 pgmap v1052: 89 pgs: 89 active+clean; 259 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 712 KiB/s wr, 7 op/s 2023-10-09T09:21:14.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:14 smithi163 bash[78107]: cluster 2023-10-09T09:21:14.359785+0000 mon.a (mon.0) 1224 : cluster 0 osdmap e1000: 8 total, 8 up, 8 in 2023-10-09T09:21:14.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:14 smithi163 bash[78107]: cluster 2023-10-09T09:21:14.363936+0000 osd.4 (osd.4) 457 : cluster 3 failed to encode map e1000 with expected crc 2023-10-09T09:21:15.340 INFO:tasks.rados.rados.0.smithi163.stdout:3745: finishing write tid 4 to smithi16367185-11 2023-10-09T09:21:15.340 INFO:tasks.rados.rados.0.smithi163.stdout:3745: finishing write tid 5 to smithi16367185-11 2023-10-09T09:21:15.340 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 1008 (ObjNum 1409 snap 468 seq_num 1409) dirty exists 2023-10-09T09:21:15.340 INFO:tasks.rados.rados.0.smithi163.stdout:3745: left oid 11 (ObjNum 1409 snap 468 seq_num 1409) 2023-10-09T09:21:15.340 INFO:tasks.rados.rados.0.smithi163.stdout:3747: expect (ObjNum 1397 snap 464 seq_num 1397) 2023-10-09T09:21:15.557 INFO:tasks.rados.rados.0.smithi163.stdout:3745: done (3 left) 2023-10-09T09:21:15.558 INFO:tasks.rados.rados.0.smithi163.stdout:3747: done (2 left) 2023-10-09T09:21:15.558 INFO:tasks.rados.rados.0.smithi163.stdout:3748: done (1 left) 2023-10-09T09:21:15.558 INFO:tasks.rados.rados.0.smithi163.stdout:3749: done (0 left) 2023-10-09T09:21:15.558 INFO:tasks.rados.rados.0.smithi163.stdout:3750: read oid 42 snap -1 2023-10-09T09:21:15.558 INFO:tasks.rados.rados.0.smithi163.stdout:3750: expect deleted 2023-10-09T09:21:15.558 INFO:tasks.rados.rados.0.smithi163.stdout:3751: write oid 3 current snap is 468 2023-10-09T09:21:15.558 INFO:tasks.rados.rados.0.smithi163.stdout:3751: seq_num 1411 ranges {514677=501644,1501622=781829,2964784=741516} 2023-10-09T09:21:15.566 INFO:tasks.rados.rados.0.smithi163.stdout:3751: writing smithi16367185-3 from 514677 to 1016321 tid 1 2023-10-09T09:21:15.575 INFO:tasks.rados.rados.0.smithi163.stdout:3751: writing smithi16367185-3 from 1501622 to 2283451 tid 2 2023-10-09T09:21:15.583 INFO:tasks.rados.rados.0.smithi163.stdout:3751: writing smithi16367185-3 from 2964784 to 3706300 tid 3 2023-10-09T09:21:15.583 INFO:tasks.rados.rados.0.smithi163.stdout:3752: snap_remove snap 465 2023-10-09T09:21:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:15 smithi181 bash[59345]: cluster 2023-10-09T09:21:14.357187+0000 osd.5 (osd.5) 541 : cluster 3 failed to encode map e1000 with expected crc 2023-10-09T09:21:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:15 smithi181 bash[59345]: cluster 2023-10-09T09:21:15.336637+0000 mon.a (mon.0) 1225 : cluster 0 osdmap e1001: 8 total, 8 up, 8 in 2023-10-09T09:21:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:15 smithi181 bash[59345]: cluster 2023-10-09T09:21:15.340619+0000 osd.4 (osd.4) 458 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:15 smithi181 bash[59345]: cluster 2023-10-09T09:21:15.343415+0000 osd.4 (osd.4) 459 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:15 smithi181 bash[59345]: cluster 2023-10-09T09:21:15.345307+0000 osd.4 (osd.4) 460 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:15 smithi163 bash[78107]: cluster 2023-10-09T09:21:14.357187+0000 osd.5 (osd.5) 541 : cluster 3 failed to encode map e1000 with expected crc 2023-10-09T09:21:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:15 smithi163 bash[78107]: cluster 2023-10-09T09:21:15.336637+0000 mon.a (mon.0) 1225 : cluster 0 osdmap e1001: 8 total, 8 up, 8 in 2023-10-09T09:21:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:15 smithi163 bash[78107]: cluster 2023-10-09T09:21:15.340619+0000 osd.4 (osd.4) 458 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:15 smithi163 bash[78107]: cluster 2023-10-09T09:21:15.343415+0000 osd.4 (osd.4) 459 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:15 smithi163 bash[78107]: cluster 2023-10-09T09:21:15.345307+0000 osd.4 (osd.4) 460 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:15 smithi163 bash[75425]: cluster 2023-10-09T09:21:14.357187+0000 osd.5 (osd.5) 541 : cluster 3 failed to encode map e1000 with expected crc 2023-10-09T09:21:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:15 smithi163 bash[75425]: cluster 2023-10-09T09:21:15.336637+0000 mon.a (mon.0) 1225 : cluster 0 osdmap e1001: 8 total, 8 up, 8 in 2023-10-09T09:21:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:15 smithi163 bash[75425]: cluster 2023-10-09T09:21:15.340619+0000 osd.4 (osd.4) 458 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:15 smithi163 bash[75425]: cluster 2023-10-09T09:21:15.343415+0000 osd.4 (osd.4) 459 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:15 smithi163 bash[75425]: cluster 2023-10-09T09:21:15.345307+0000 osd.4 (osd.4) 460 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:16.542 INFO:tasks.rados.rados.0.smithi163.stdout:3752: done (2 left) 2023-10-09T09:21:16.542 INFO:tasks.rados.rados.0.smithi163.stdout:3753: snap_create 2023-10-09T09:21:16.542 INFO:tasks.rados.rados.0.smithi163.stdout:3751: finishing write tid 1 to smithi16367185-3 2023-10-09T09:21:16.542 INFO:tasks.rados.rados.0.smithi163.stdout:3751: finishing write tid 2 to smithi16367185-3 2023-10-09T09:21:16.542 INFO:tasks.rados.rados.0.smithi163.stdout:3751: finishing write tid 3 to smithi16367185-3 2023-10-09T09:21:16.542 INFO:tasks.rados.rados.0.smithi163.stdout:3751: finishing write tid 4 to smithi16367185-3 2023-10-09T09:21:16.543 INFO:tasks.rados.rados.0.smithi163.stdout:3751: finishing write tid 5 to smithi16367185-3 2023-10-09T09:21:16.543 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 1125 (ObjNum 1411 snap 468 seq_num 1411) dirty exists 2023-10-09T09:21:16.543 INFO:tasks.rados.rados.0.smithi163.stdout:3751: left oid 3 (ObjNum 1411 snap 468 seq_num 1411) 2023-10-09T09:21:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:16 smithi181 bash[59345]: cluster 2023-10-09T09:21:15.331272+0000 osd.5 (osd.5) 542 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:16 smithi181 bash[59345]: cluster 2023-10-09T09:21:15.473890+0000 mgr.y (mgr.34107) 441 : cluster 0 pgmap v1055: 89 pgs: 89 active+clean; 261 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 803 KiB/s wr, 5 op/s 2023-10-09T09:21:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:16 smithi181 bash[59345]: cluster 2023-10-09T09:21:15.552559+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:16 smithi163 bash[75425]: cluster 2023-10-09T09:21:15.331272+0000 osd.5 (osd.5) 542 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:16 smithi163 bash[75425]: cluster 2023-10-09T09:21:15.473890+0000 mgr.y (mgr.34107) 441 : cluster 0 pgmap v1055: 89 pgs: 89 active+clean; 261 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 803 KiB/s wr, 5 op/s 2023-10-09T09:21:16.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:16 smithi163 bash[75425]: cluster 2023-10-09T09:21:15.552559+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:16 smithi163 bash[78107]: cluster 2023-10-09T09:21:15.331272+0000 osd.5 (osd.5) 542 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:16 smithi163 bash[78107]: cluster 2023-10-09T09:21:15.473890+0000 mgr.y (mgr.34107) 441 : cluster 0 pgmap v1055: 89 pgs: 89 active+clean; 261 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 803 KiB/s wr, 5 op/s 2023-10-09T09:21:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:16 smithi163 bash[78107]: cluster 2023-10-09T09:21:15.552559+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e1001 with expected crc 2023-10-09T09:21:17.552 INFO:tasks.rados.rados.0.smithi163.stdout:3750: done (2 left) 2023-10-09T09:21:17.552 INFO:tasks.rados.rados.0.smithi163.stdout:3751: done (1 left) 2023-10-09T09:21:17.552 INFO:tasks.rados.rados.0.smithi163.stdout:3753: done (0 left) 2023-10-09T09:21:17.552 INFO:tasks.rados.rados.0.smithi163.stdout:3754: read oid 2 snap -1 2023-10-09T09:21:17.552 INFO:tasks.rados.rados.0.smithi163.stdout:3754: expect deleted 2023-10-09T09:21:17.552 INFO:tasks.rados.rados.0.smithi163.stdout:3755: snap_create 2023-10-09T09:21:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:17 smithi181 bash[59345]: cluster 2023-10-09T09:21:16.539275+0000 osd.5 (osd.5) 543 : cluster 3 failed to encode map e1002 with expected crc 2023-10-09T09:21:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:17 smithi181 bash[59345]: cluster 2023-10-09T09:21:16.539962+0000 osd.4 (osd.4) 461 : cluster 3 failed to encode map e1002 with expected crc 2023-10-09T09:21:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:17 smithi181 bash[59345]: cluster 2023-10-09T09:21:16.541925+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e1002: 8 total, 8 up, 8 in 2023-10-09T09:21:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:17 smithi181 bash[59345]: audit 2023-10-09T09:21:16.758122+0000 mon.a (mon.0) 1227 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:17 smithi181 bash[59345]: audit 2023-10-09T09:21:16.765314+0000 mon.a (mon.0) 1228 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:17 smithi181 bash[59345]: audit 2023-10-09T09:21:16.967196+0000 mon.a (mon.0) 1229 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:17 smithi181 bash[59345]: audit 2023-10-09T09:21:16.975167+0000 mon.a (mon.0) 1230 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:17 smithi181 bash[59345]: audit 2023-10-09T09:21:17.456989+0000 mon.a (mon.0) 1231 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:21:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:17 smithi181 bash[59345]: audit 2023-10-09T09:21:17.459014+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:21:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:17 smithi181 bash[59345]: audit 2023-10-09T09:21:17.467827+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[75425]: cluster 2023-10-09T09:21:16.539275+0000 osd.5 (osd.5) 543 : cluster 3 failed to encode map e1002 with expected crc 2023-10-09T09:21:17.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[75425]: cluster 2023-10-09T09:21:16.539962+0000 osd.4 (osd.4) 461 : cluster 3 failed to encode map e1002 with expected crc 2023-10-09T09:21:17.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[75425]: cluster 2023-10-09T09:21:16.541925+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e1002: 8 total, 8 up, 8 in 2023-10-09T09:21:17.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[75425]: audit 2023-10-09T09:21:16.758122+0000 mon.a (mon.0) 1227 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[75425]: audit 2023-10-09T09:21:16.765314+0000 mon.a (mon.0) 1228 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[75425]: audit 2023-10-09T09:21:16.967196+0000 mon.a (mon.0) 1229 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[75425]: audit 2023-10-09T09:21:16.975167+0000 mon.a (mon.0) 1230 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[75425]: audit 2023-10-09T09:21:17.456989+0000 mon.a (mon.0) 1231 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:21:17.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[75425]: audit 2023-10-09T09:21:17.459014+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:21:17.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[75425]: audit 2023-10-09T09:21:17.467827+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[78107]: cluster 2023-10-09T09:21:16.539275+0000 osd.5 (osd.5) 543 : cluster 3 failed to encode map e1002 with expected crc 2023-10-09T09:21:17.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[78107]: cluster 2023-10-09T09:21:16.539962+0000 osd.4 (osd.4) 461 : cluster 3 failed to encode map e1002 with expected crc 2023-10-09T09:21:17.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[78107]: cluster 2023-10-09T09:21:16.541925+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e1002: 8 total, 8 up, 8 in 2023-10-09T09:21:17.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[78107]: audit 2023-10-09T09:21:16.758122+0000 mon.a (mon.0) 1227 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[78107]: audit 2023-10-09T09:21:16.765314+0000 mon.a (mon.0) 1228 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[78107]: audit 2023-10-09T09:21:16.967196+0000 mon.a (mon.0) 1229 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[78107]: audit 2023-10-09T09:21:16.975167+0000 mon.a (mon.0) 1230 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:17.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[78107]: audit 2023-10-09T09:21:17.456989+0000 mon.a (mon.0) 1231 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:21:17.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[78107]: audit 2023-10-09T09:21:17.459014+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:21:17.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:17 smithi163 bash[78107]: audit 2023-10-09T09:21:17.467827+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:21:18.557 INFO:tasks.rados.rados.0.smithi163.stdout:3754: done (1 left) 2023-10-09T09:21:18.558 INFO:tasks.rados.rados.0.smithi163.stdout:3755: done (0 left) 2023-10-09T09:21:18.558 INFO:tasks.rados.rados.0.smithi163.stdout:3756: snap_create 2023-10-09T09:21:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:18 smithi181 bash[59345]: cluster 2023-10-09T09:21:17.474388+0000 mgr.y (mgr.34107) 442 : cluster 0 pgmap v1057: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.5 MiB/s wr, 12 op/s 2023-10-09T09:21:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:18 smithi181 bash[59345]: cluster 2023-10-09T09:21:17.548796+0000 osd.5 (osd.5) 544 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:18 smithi181 bash[59345]: cluster 2023-10-09T09:21:17.551859+0000 mon.a (mon.0) 1234 : cluster 0 osdmap e1003: 8 total, 8 up, 8 in 2023-10-09T09:21:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:18 smithi181 bash[59345]: cluster 2023-10-09T09:21:17.552746+0000 osd.7 (osd.7) 426 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:18 smithi181 bash[59345]: cluster 2023-10-09T09:21:17.648291+0000 osd.4 (osd.4) 462 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:18 smithi163 bash[78107]: cluster 2023-10-09T09:21:17.474388+0000 mgr.y (mgr.34107) 442 : cluster 0 pgmap v1057: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.5 MiB/s wr, 12 op/s 2023-10-09T09:21:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:18 smithi163 bash[78107]: cluster 2023-10-09T09:21:17.548796+0000 osd.5 (osd.5) 544 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:18 smithi163 bash[78107]: cluster 2023-10-09T09:21:17.551859+0000 mon.a (mon.0) 1234 : cluster 0 osdmap e1003: 8 total, 8 up, 8 in 2023-10-09T09:21:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:18 smithi163 bash[78107]: cluster 2023-10-09T09:21:17.552746+0000 osd.7 (osd.7) 426 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:18 smithi163 bash[78107]: cluster 2023-10-09T09:21:17.648291+0000 osd.4 (osd.4) 462 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:18 smithi163 bash[75425]: cluster 2023-10-09T09:21:17.474388+0000 mgr.y (mgr.34107) 442 : cluster 0 pgmap v1057: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.5 MiB/s wr, 12 op/s 2023-10-09T09:21:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:18 smithi163 bash[75425]: cluster 2023-10-09T09:21:17.548796+0000 osd.5 (osd.5) 544 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:18 smithi163 bash[75425]: cluster 2023-10-09T09:21:17.551859+0000 mon.a (mon.0) 1234 : cluster 0 osdmap e1003: 8 total, 8 up, 8 in 2023-10-09T09:21:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:18 smithi163 bash[75425]: cluster 2023-10-09T09:21:17.552746+0000 osd.7 (osd.7) 426 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:18 smithi163 bash[75425]: cluster 2023-10-09T09:21:17.648291+0000 osd.4 (osd.4) 462 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:19.557 INFO:tasks.rados.rados.0.smithi163.stdout:3756: done (0 left) 2023-10-09T09:21:19.557 INFO:tasks.rados.rados.0.smithi163.stdout:3757: snap_remove snap 444 2023-10-09T09:21:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:19 smithi163 bash[78107]: cluster 2023-10-09T09:21:17.648453+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:19 smithi163 bash[78107]: cluster 2023-10-09T09:21:18.552609+0000 osd.5 (osd.5) 545 : cluster 3 failed to encode map e1004 with expected crc 2023-10-09T09:21:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:19 smithi163 bash[78107]: cluster 2023-10-09T09:21:18.552698+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e1004 with expected crc 2023-10-09T09:21:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:19 smithi163 bash[78107]: cluster 2023-10-09T09:21:18.557060+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e1004: 8 total, 8 up, 8 in 2023-10-09T09:21:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:19 smithi163 bash[78107]: cluster 2023-10-09T09:21:19.556426+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e1005: 8 total, 8 up, 8 in 2023-10-09T09:21:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:19 smithi163 bash[75425]: cluster 2023-10-09T09:21:17.648453+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:19 smithi163 bash[75425]: cluster 2023-10-09T09:21:18.552609+0000 osd.5 (osd.5) 545 : cluster 3 failed to encode map e1004 with expected crc 2023-10-09T09:21:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:19 smithi163 bash[75425]: cluster 2023-10-09T09:21:18.552698+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e1004 with expected crc 2023-10-09T09:21:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:19 smithi163 bash[75425]: cluster 2023-10-09T09:21:18.557060+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e1004: 8 total, 8 up, 8 in 2023-10-09T09:21:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:19 smithi163 bash[75425]: cluster 2023-10-09T09:21:19.556426+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e1005: 8 total, 8 up, 8 in 2023-10-09T09:21:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:19 smithi181 bash[59345]: cluster 2023-10-09T09:21:17.648453+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e1003 with expected crc 2023-10-09T09:21:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:19 smithi181 bash[59345]: cluster 2023-10-09T09:21:18.552609+0000 osd.5 (osd.5) 545 : cluster 3 failed to encode map e1004 with expected crc 2023-10-09T09:21:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:19 smithi181 bash[59345]: cluster 2023-10-09T09:21:18.552698+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e1004 with expected crc 2023-10-09T09:21:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:19 smithi181 bash[59345]: cluster 2023-10-09T09:21:18.557060+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e1004: 8 total, 8 up, 8 in 2023-10-09T09:21:20.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:19 smithi181 bash[59345]: cluster 2023-10-09T09:21:19.556426+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e1005: 8 total, 8 up, 8 in 2023-10-09T09:21:20.338 INFO:tasks.rados.rados.0.smithi163.stdout:3757: done (0 left) 2023-10-09T09:21:20.339 INFO:tasks.rados.rados.0.smithi163.stdout:3758: snap_remove snap 460 2023-10-09T09:21:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[75425]: cluster 2023-10-09T09:21:19.475264+0000 mgr.y (mgr.34107) 443 : cluster 0 pgmap v1060: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.8 MiB/s wr, 7 op/s 2023-10-09T09:21:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[75425]: cluster 2023-10-09T09:21:19.553431+0000 osd.5 (osd.5) 546 : cluster 3 failed to encode map e1005 with expected crc 2023-10-09T09:21:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[75425]: cluster 2023-10-09T09:21:20.205477+0000 osd.4 (osd.4) 463 : cluster 3 failed to encode map e1005 with expected crc 2023-10-09T09:21:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[75425]: cluster 2023-10-09T09:21:20.333661+0000 osd.5 (osd.5) 547 : cluster 3 failed to encode map e1006 with expected crc 2023-10-09T09:21:20.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[75425]: cluster 2023-10-09T09:21:20.334212+0000 osd.4 (osd.4) 464 : cluster 3 failed to encode map e1006 with expected crc 2023-10-09T09:21:20.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[75425]: cluster 2023-10-09T09:21:20.337905+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e1006: 8 total, 8 up, 8 in 2023-10-09T09:21:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[78107]: cluster 2023-10-09T09:21:19.475264+0000 mgr.y (mgr.34107) 443 : cluster 0 pgmap v1060: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.8 MiB/s wr, 7 op/s 2023-10-09T09:21:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[78107]: cluster 2023-10-09T09:21:19.553431+0000 osd.5 (osd.5) 546 : cluster 3 failed to encode map e1005 with expected crc 2023-10-09T09:21:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[78107]: cluster 2023-10-09T09:21:20.205477+0000 osd.4 (osd.4) 463 : cluster 3 failed to encode map e1005 with expected crc 2023-10-09T09:21:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[78107]: cluster 2023-10-09T09:21:20.333661+0000 osd.5 (osd.5) 547 : cluster 3 failed to encode map e1006 with expected crc 2023-10-09T09:21:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[78107]: cluster 2023-10-09T09:21:20.334212+0000 osd.4 (osd.4) 464 : cluster 3 failed to encode map e1006 with expected crc 2023-10-09T09:21:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:20 smithi163 bash[78107]: cluster 2023-10-09T09:21:20.337905+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e1006: 8 total, 8 up, 8 in 2023-10-09T09:21:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:20 smithi181 bash[59345]: cluster 2023-10-09T09:21:19.475264+0000 mgr.y (mgr.34107) 443 : cluster 0 pgmap v1060: 89 pgs: 89 active+clean; 265 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.8 MiB/s wr, 7 op/s 2023-10-09T09:21:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:20 smithi181 bash[59345]: cluster 2023-10-09T09:21:19.553431+0000 osd.5 (osd.5) 546 : cluster 3 failed to encode map e1005 with expected crc 2023-10-09T09:21:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:20 smithi181 bash[59345]: cluster 2023-10-09T09:21:20.205477+0000 osd.4 (osd.4) 463 : cluster 3 failed to encode map e1005 with expected crc 2023-10-09T09:21:21.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:20 smithi181 bash[59345]: cluster 2023-10-09T09:21:20.333661+0000 osd.5 (osd.5) 547 : cluster 3 failed to encode map e1006 with expected crc 2023-10-09T09:21:21.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:20 smithi181 bash[59345]: cluster 2023-10-09T09:21:20.334212+0000 osd.4 (osd.4) 464 : cluster 3 failed to encode map e1006 with expected crc 2023-10-09T09:21:21.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:20 smithi181 bash[59345]: cluster 2023-10-09T09:21:20.337905+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e1006: 8 total, 8 up, 8 in 2023-10-09T09:21:21.343 INFO:tasks.rados.rados.0.smithi163.stdout:3758: done (0 left) 2023-10-09T09:21:21.343 INFO:tasks.rados.rados.0.smithi163.stdout:3759: write oid 39 current snap is 471 2023-10-09T09:21:21.343 INFO:tasks.rados.rados.0.smithi163.stdout:3759: seq_num 1412 ranges {531484=576893,1843751=525487,2875854=669141} 2023-10-09T09:21:21.353 INFO:tasks.rados.rados.0.smithi163.stdout:3759: writing smithi16367185-39 from 531484 to 1108377 tid 1 2023-10-09T09:21:21.359 INFO:tasks.rados.rados.0.smithi163.stdout:3759: writing smithi16367185-39 from 1843751 to 2369238 tid 2 2023-10-09T09:21:21.366 INFO:tasks.rados.rados.0.smithi163.stdout:3759: writing smithi16367185-39 from 2875854 to 3544995 tid 3 2023-10-09T09:21:21.367 INFO:tasks.rados.rados.0.smithi163.stdout:3760: write oid 30 current snap is 471 2023-10-09T09:21:21.367 INFO:tasks.rados.rados.0.smithi163.stdout:3760: seq_num 1413 ranges {548291=652142,1902233=785498,3303277=445014} 2023-10-09T09:21:21.374 INFO:tasks.rados.rados.0.smithi163.stdout:3760: writing smithi16367185-30 from 548291 to 1200433 tid 1 2023-10-09T09:21:21.386 INFO:tasks.rados.rados.0.smithi163.stdout:3760: writing smithi16367185-30 from 1902233 to 2687731 tid 2 2023-10-09T09:21:21.390 INFO:tasks.rados.rados.0.smithi163.stdout:3760: writing smithi16367185-30 from 3303277 to 3748291 tid 3 2023-10-09T09:21:21.390 INFO:tasks.rados.rados.0.smithi163.stdout:3761: write oid 1 current snap is 471 2023-10-09T09:21:21.390 INFO:tasks.rados.rados.0.smithi163.stdout:3761: seq_num 1414 ranges {565098=727391,1960715=529156,2930700=691685} 2023-10-09T09:21:21.401 INFO:tasks.rados.rados.0.smithi163.stdout:3761: writing smithi16367185-1 from 565098 to 1292489 tid 1 2023-10-09T09:21:21.404 INFO:tasks.rados.rados.0.smithi163.stdout:3761: writing smithi16367185-1 from 1960715 to 2489871 tid 2 2023-10-09T09:21:21.414 INFO:tasks.rados.rados.0.smithi163.stdout:3761: writing smithi16367185-1 from 2930700 to 3622385 tid 3 2023-10-09T09:21:21.414 INFO:tasks.rados.rados.0.smithi163.stdout:3762: snap_remove snap 468 2023-10-09T09:21:21.414 INFO:tasks.rados.rados.0.smithi163.stdout:3759: finishing write tid 1 to smithi16367185-39 2023-10-09T09:21:21.414 INFO:tasks.rados.rados.0.smithi163.stdout:3759: finishing write tid 2 to smithi16367185-39 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:3759: finishing write tid 3 to smithi16367185-39 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:3759: finishing write tid 4 to smithi16367185-39 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:3759: finishing write tid 5 to smithi16367185-39 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1748 (ObjNum 1412 snap 471 seq_num 1412) dirty exists 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:3759: left oid 39 (ObjNum 1412 snap 471 seq_num 1412) 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:3760: finishing write tid 1 to smithi16367185-30 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:3760: finishing write tid 2 to smithi16367185-30 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:3760: finishing write tid 3 to smithi16367185-30 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:3760: finishing write tid 4 to smithi16367185-30 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:3760: finishing write tid 5 to smithi16367185-30 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1344 (ObjNum 1413 snap 471 seq_num 1413) dirty exists 2023-10-09T09:21:21.415 INFO:tasks.rados.rados.0.smithi163.stdout:3760: left oid 30 (ObjNum 1413 snap 471 seq_num 1413) 2023-10-09T09:21:21.416 INFO:tasks.rados.rados.0.smithi163.stdout:3761: finishing write tid 1 to smithi16367185-1 2023-10-09T09:21:21.416 INFO:tasks.rados.rados.0.smithi163.stdout:3761: finishing write tid 2 to smithi16367185-1 2023-10-09T09:21:22.352 INFO:tasks.rados.rados.0.smithi163.stdout:3761: finishing write tid 3 to smithi16367185-1 2023-10-09T09:21:22.352 INFO:tasks.rados.rados.0.smithi163.stdout:3761: finishing write tid 4 to smithi16367185-1 2023-10-09T09:21:22.352 INFO:tasks.rados.rados.0.smithi163.stdout:3761: finishing write tid 5 to smithi16367185-1 2023-10-09T09:21:22.352 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 2090 (ObjNum 1414 snap 471 seq_num 1414) dirty exists 2023-10-09T09:21:22.352 INFO:tasks.rados.rados.0.smithi163.stdout:3761: left oid 1 (ObjNum 1414 snap 471 seq_num 1414) 2023-10-09T09:21:22.352 INFO:tasks.rados.rados.0.smithi163.stdout:3759: done (3 left) 2023-10-09T09:21:22.352 INFO:tasks.rados.rados.0.smithi163.stdout:3760: done (2 left) 2023-10-09T09:21:22.352 INFO:tasks.rados.rados.0.smithi163.stdout:3761: done (1 left) 2023-10-09T09:21:22.353 INFO:tasks.rados.rados.0.smithi163.stdout:3762: done (0 left) 2023-10-09T09:21:22.353 INFO:tasks.rados.rados.0.smithi163.stdout:3763: snap_create 2023-10-09T09:21:22.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:22 smithi163 bash[75425]: cluster 2023-10-09T09:21:21.336517+0000 osd.5 (osd.5) 548 : cluster 3 failed to encode map e1007 with expected crc 2023-10-09T09:21:22.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:22 smithi163 bash[75425]: cluster 2023-10-09T09:21:21.341165+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e1007: 8 total, 8 up, 8 in 2023-10-09T09:21:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:22 smithi163 bash[75425]: cluster 2023-10-09T09:21:21.341847+0000 osd.4 (osd.4) 465 : cluster 3 failed to encode map e1007 with expected crc 2023-10-09T09:21:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:22 smithi163 bash[75425]: cluster 2023-10-09T09:21:21.345513+0000 osd.4 (osd.4) 466 : cluster 3 failed to encode map e1007 with expected crc 2023-10-09T09:21:22.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:22 smithi163 bash[75425]: cluster 2023-10-09T09:21:21.475885+0000 mgr.y (mgr.34107) 444 : cluster 0 pgmap v1064: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:21:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:22 smithi163 bash[78107]: cluster 2023-10-09T09:21:21.336517+0000 osd.5 (osd.5) 548 : cluster 3 failed to encode map e1007 with expected crc 2023-10-09T09:21:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:22 smithi163 bash[78107]: cluster 2023-10-09T09:21:21.341165+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e1007: 8 total, 8 up, 8 in 2023-10-09T09:21:22.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:22 smithi163 bash[78107]: cluster 2023-10-09T09:21:21.341847+0000 osd.4 (osd.4) 465 : cluster 3 failed to encode map e1007 with expected crc 2023-10-09T09:21:22.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:22 smithi163 bash[78107]: cluster 2023-10-09T09:21:21.345513+0000 osd.4 (osd.4) 466 : cluster 3 failed to encode map e1007 with expected crc 2023-10-09T09:21:22.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:22 smithi163 bash[78107]: cluster 2023-10-09T09:21:21.475885+0000 mgr.y (mgr.34107) 444 : cluster 0 pgmap v1064: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:21:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:22 smithi181 bash[59345]: cluster 2023-10-09T09:21:21.336517+0000 osd.5 (osd.5) 548 : cluster 3 failed to encode map e1007 with expected crc 2023-10-09T09:21:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:22 smithi181 bash[59345]: cluster 2023-10-09T09:21:21.341165+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e1007: 8 total, 8 up, 8 in 2023-10-09T09:21:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:22 smithi181 bash[59345]: cluster 2023-10-09T09:21:21.341847+0000 osd.4 (osd.4) 465 : cluster 3 failed to encode map e1007 with expected crc 2023-10-09T09:21:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:22 smithi181 bash[59345]: cluster 2023-10-09T09:21:21.345513+0000 osd.4 (osd.4) 466 : cluster 3 failed to encode map e1007 with expected crc 2023-10-09T09:21:22.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:22 smithi181 bash[59345]: cluster 2023-10-09T09:21:21.475885+0000 mgr.y (mgr.34107) 444 : cluster 0 pgmap v1064: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:21:23.358 INFO:tasks.rados.rados.0.smithi163.stdout:3763: done (0 left) 2023-10-09T09:21:23.358 INFO:tasks.rados.rados.0.smithi163.stdout:3764: write oid 24 current snap is 472 2023-10-09T09:21:23.358 INFO:tasks.rados.rados.0.smithi163.stdout:3764: seq_num 1415 ranges {581905=518993,1735550=672814,2958123=502957} 2023-10-09T09:21:23.366 INFO:tasks.rados.rados.0.smithi163.stdout:3764: writing smithi16367185-24 from 581905 to 1100898 tid 1 2023-10-09T09:21:23.375 INFO:tasks.rados.rados.0.smithi163.stdout:3764: writing smithi16367185-24 from 1735550 to 2408364 tid 2 2023-10-09T09:21:23.382 INFO:tasks.rados.rados.0.smithi163.stdout:3764: writing smithi16367185-24 from 2958123 to 3461080 tid 3 2023-10-09T09:21:23.382 INFO:tasks.rados.rados.0.smithi163.stdout:3765: rollback oid 4 current snap is 472 2023-10-09T09:21:23.382 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 4 to 409 2023-10-09T09:21:23.382 INFO:tasks.rados.rados.0.smithi163.stdout:3764: finishing write tid 1 to smithi16367185-24 2023-10-09T09:21:23.382 INFO:tasks.rados.rados.0.smithi163.stdout:3764: finishing write tid 2 to smithi16367185-24 2023-10-09T09:21:23.382 INFO:tasks.rados.rados.0.smithi163.stdout:3766: write oid 12 current snap is 472 2023-10-09T09:21:23.382 INFO:tasks.rados.rados.0.smithi163.stdout:3766: seq_num 1416 ranges {598712=594242,1677679=532825,2985546=430582} 2023-10-09T09:21:23.389 INFO:tasks.rados.rados.0.smithi163.stdout:3766: writing smithi16367185-12 from 598712 to 1192954 tid 1 2023-10-09T09:21:23.395 INFO:tasks.rados.rados.0.smithi163.stdout:3766: writing smithi16367185-12 from 1677679 to 2210504 tid 2 2023-10-09T09:21:23.402 INFO:tasks.rados.rados.0.smithi163.stdout:3766: writing smithi16367185-12 from 2985546 to 3416128 tid 3 2023-10-09T09:21:23.402 INFO:tasks.rados.rados.0.smithi163.stdout:3767: write oid 29 current snap is 472 2023-10-09T09:21:23.402 INFO:tasks.rados.rados.0.smithi163.stdout:3764: finishing write tid 3 to smithi16367185-24 2023-10-09T09:21:23.402 INFO:tasks.rados.rados.0.smithi163.stdout:3764: finishing write tid 4 to smithi16367185-24 2023-10-09T09:21:23.403 INFO:tasks.rados.rados.0.smithi163.stdout:3764: finishing write tid 5 to smithi16367185-24 2023-10-09T09:21:23.403 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 839 (ObjNum 1415 snap 472 seq_num 1415) dirty exists 2023-10-09T09:21:23.403 INFO:tasks.rados.rados.0.smithi163.stdout:3764: left oid 24 (ObjNum 1415 snap 472 seq_num 1415) 2023-10-09T09:21:23.403 INFO:tasks.rados.rados.0.smithi163.stdout:3765: finishing rollback tid 1 to smithi16367185-4 2023-10-09T09:21:23.403 INFO:tasks.rados.rados.0.smithi163.stdout:3765: finishing rollback tid 2 to smithi16367185-4 2023-10-09T09:21:23.403 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 4 v 2094 (ObjNum 1182 snap 397 seq_num 1182) dirty exists 2023-10-09T09:21:23.403 INFO:tasks.rados.rados.0.smithi163.stdout:3766: finishing write tid 1 to smithi16367185-12 2023-10-09T09:21:23.403 INFO:tasks.rados.rados.0.smithi163.stdout:3767: seq_num 1417 ranges {615519=669491,1736161=676483,2896616=641854} 2023-10-09T09:21:23.409 INFO:tasks.rados.rados.0.smithi163.stdout:3767: writing smithi16367185-29 from 615519 to 1285010 tid 1 2023-10-09T09:21:23.416 INFO:tasks.rados.rados.0.smithi163.stdout:3767: writing smithi16367185-29 from 1736161 to 2412644 tid 2 2023-10-09T09:21:23.423 INFO:tasks.rados.rados.0.smithi163.stdout:3767: writing smithi16367185-29 from 2896616 to 3538470 tid 3 2023-10-09T09:21:23.423 INFO:tasks.rados.rados.0.smithi163.stdout:3764: done (3 left) 2023-10-09T09:21:23.423 INFO:tasks.rados.rados.0.smithi163.stdout:3765: done (2 left) 2023-10-09T09:21:23.423 INFO:tasks.rados.rados.0.smithi163.stdout:3768: write oid 40 current snap is 472 2023-10-09T09:21:23.423 INFO:tasks.rados.rados.0.smithi163.stdout:3768: seq_num 1418 ranges {632326=744740,1794643=536494,3040392=453126} 2023-10-09T09:21:23.431 INFO:tasks.rados.rados.0.smithi163.stdout:3768: writing smithi16367185-40 from 632326 to 1377066 tid 1 2023-10-09T09:21:23.436 INFO:tasks.rados.rados.0.smithi163.stdout:3768: writing smithi16367185-40 from 1794643 to 2331137 tid 2 2023-10-09T09:21:23.443 INFO:tasks.rados.rados.0.smithi163.stdout:3768: writing smithi16367185-40 from 3040392 to 3493518 tid 3 2023-10-09T09:21:23.444 INFO:tasks.rados.rados.0.smithi163.stdout:3769: read oid 25 snap -1 2023-10-09T09:21:23.444 INFO:tasks.rados.rados.0.smithi163.stdout:3769: expect deleted 2023-10-09T09:21:23.444 INFO:tasks.rados.rados.0.smithi163.stdout:3770: write oid 47 current snap is 472 2023-10-09T09:21:23.444 INFO:tasks.rados.rados.0.smithi163.stdout:3766: finishing write tid 2 to smithi16367185-12 2023-10-09T09:21:23.444 INFO:tasks.rados.rados.0.smithi163.stdout:3770: seq_num 1419 ranges {649133=419989,1853125=680152,2951462=664398} 2023-10-09T09:21:23.449 INFO:tasks.rados.rados.0.smithi163.stdout:3770: writing smithi16367185-47 from 649133 to 1069122 tid 1 2023-10-09T09:21:23.459 INFO:tasks.rados.rados.0.smithi163.stdout:3770: writing smithi16367185-47 from 1853125 to 2533277 tid 2 2023-10-09T09:21:23.466 INFO:tasks.rados.rados.0.smithi163.stdout:3770: writing smithi16367185-47 from 2951462 to 3615860 tid 3 2023-10-09T09:21:23.466 INFO:tasks.rados.rados.0.smithi163.stdout:3771: snap_create 2023-10-09T09:21:23.466 INFO:tasks.rados.rados.0.smithi163.stdout:3766: finishing write tid 3 to smithi16367185-12 2023-10-09T09:21:23.466 INFO:tasks.rados.rados.0.smithi163.stdout:3766: finishing write tid 4 to smithi16367185-12 2023-10-09T09:21:23.466 INFO:tasks.rados.rados.0.smithi163.stdout:3766: finishing write tid 5 to smithi16367185-12 2023-10-09T09:21:23.466 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 12 v 1746 (ObjNum 1416 snap 472 seq_num 1416) dirty exists 2023-10-09T09:21:23.466 INFO:tasks.rados.rados.0.smithi163.stdout:3766: left oid 12 (ObjNum 1416 snap 472 seq_num 1416) 2023-10-09T09:21:23.467 INFO:tasks.rados.rados.0.smithi163.stdout:3767: finishing write tid 1 to smithi16367185-29 2023-10-09T09:21:23.467 INFO:tasks.rados.rados.0.smithi163.stdout:3767: finishing write tid 2 to smithi16367185-29 2023-10-09T09:21:23.467 INFO:tasks.rados.rados.0.smithi163.stdout:3767: finishing write tid 3 to smithi16367185-29 2023-10-09T09:21:23.467 INFO:tasks.rados.rados.0.smithi163.stdout:3767: finishing write tid 4 to smithi16367185-29 2023-10-09T09:21:23.467 INFO:tasks.rados.rados.0.smithi163.stdout:3767: finishing write tid 5 to smithi16367185-29 2023-10-09T09:21:23.467 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1750 (ObjNum 1417 snap 472 seq_num 1417) dirty exists 2023-10-09T09:21:23.468 INFO:tasks.rados.rados.0.smithi163.stdout:3767: left oid 29 (ObjNum 1417 snap 472 seq_num 1417) 2023-10-09T09:21:23.468 INFO:tasks.rados.rados.0.smithi163.stdout:3768: finishing write tid 1 to smithi16367185-40 2023-10-09T09:21:23.468 INFO:tasks.rados.rados.0.smithi163.stdout:3768: finishing write tid 2 to smithi16367185-40 2023-10-09T09:21:23.468 INFO:tasks.rados.rados.0.smithi163.stdout:3768: finishing write tid 3 to smithi16367185-40 2023-10-09T09:21:23.469 INFO:tasks.rados.rados.0.smithi163.stdout:3768: finishing write tid 4 to smithi16367185-40 2023-10-09T09:21:23.469 INFO:tasks.rados.rados.0.smithi163.stdout:3768: finishing write tid 5 to smithi16367185-40 2023-10-09T09:21:23.469 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1947 (ObjNum 1418 snap 472 seq_num 1418) dirty exists 2023-10-09T09:21:23.469 INFO:tasks.rados.rados.0.smithi163.stdout:3768: left oid 40 (ObjNum 1418 snap 472 seq_num 1418) 2023-10-09T09:21:23.469 INFO:tasks.rados.rados.0.smithi163.stdout:3770: finishing write tid 1 to smithi16367185-47 2023-10-09T09:21:23.469 INFO:tasks.rados.rados.0.smithi163.stdout:3770: finishing write tid 2 to smithi16367185-47 2023-10-09T09:21:23.473 INFO:tasks.rados.rados.0.smithi163.stdout:3770: finishing write tid 3 to smithi16367185-47 2023-10-09T09:21:23.473 INFO:tasks.rados.rados.0.smithi163.stdout:3770: finishing write tid 4 to smithi16367185-47 2023-10-09T09:21:23.473 INFO:tasks.rados.rados.0.smithi163.stdout:3770: finishing write tid 5 to smithi16367185-47 2023-10-09T09:21:23.473 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 2914 (ObjNum 1419 snap 472 seq_num 1419) dirty exists 2023-10-09T09:21:23.473 INFO:tasks.rados.rados.0.smithi163.stdout:3770: left oid 47 (ObjNum 1419 snap 472 seq_num 1419) 2023-10-09T09:21:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:21:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:21:23] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.33.4" 2023-10-09T09:21:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:23 smithi163 bash[75425]: cluster 2023-10-09T09:21:22.345029+0000 osd.5 (osd.5) 549 : cluster 3 failed to encode map e1008 with expected crc 2023-10-09T09:21:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:23 smithi163 bash[75425]: cluster 2023-10-09T09:21:22.349463+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e1008: 8 total, 8 up, 8 in 2023-10-09T09:21:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:23 smithi163 bash[75425]: cluster 2023-10-09T09:21:22.352643+0000 osd.4 (osd.4) 467 : cluster 3 failed to encode map e1008 with expected crc 2023-10-09T09:21:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:23 smithi163 bash[78107]: cluster 2023-10-09T09:21:22.345029+0000 osd.5 (osd.5) 549 : cluster 3 failed to encode map e1008 with expected crc 2023-10-09T09:21:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:23 smithi163 bash[78107]: cluster 2023-10-09T09:21:22.349463+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e1008: 8 total, 8 up, 8 in 2023-10-09T09:21:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:23 smithi163 bash[78107]: cluster 2023-10-09T09:21:22.352643+0000 osd.4 (osd.4) 467 : cluster 3 failed to encode map e1008 with expected crc 2023-10-09T09:21:23.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:23 smithi181 bash[59345]: cluster 2023-10-09T09:21:22.345029+0000 osd.5 (osd.5) 549 : cluster 3 failed to encode map e1008 with expected crc 2023-10-09T09:21:23.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:23 smithi181 bash[59345]: cluster 2023-10-09T09:21:22.349463+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e1008: 8 total, 8 up, 8 in 2023-10-09T09:21:23.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:23 smithi181 bash[59345]: cluster 2023-10-09T09:21:22.352643+0000 osd.4 (osd.4) 467 : cluster 3 failed to encode map e1008 with expected crc 2023-10-09T09:21:24.373 INFO:tasks.rados.rados.0.smithi163.stdout:3766: done (5 left) 2023-10-09T09:21:24.373 INFO:tasks.rados.rados.0.smithi163.stdout:3767: done (4 left) 2023-10-09T09:21:24.373 INFO:tasks.rados.rados.0.smithi163.stdout:3768: done (3 left) 2023-10-09T09:21:24.373 INFO:tasks.rados.rados.0.smithi163.stdout:3769: done (2 left) 2023-10-09T09:21:24.373 INFO:tasks.rados.rados.0.smithi163.stdout:3770: done (1 left) 2023-10-09T09:21:24.373 INFO:tasks.rados.rados.0.smithi163.stdout:3771: done (0 left) 2023-10-09T09:21:24.373 INFO:tasks.rados.rados.0.smithi163.stdout:3772: read oid 33 snap -1 2023-10-09T09:21:24.373 INFO:tasks.rados.rados.0.smithi163.stdout:3772: expect deleted 2023-10-09T09:21:24.373 INFO:tasks.rados.rados.0.smithi163.stdout:3773: snap_remove snap 471 2023-10-09T09:21:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[75425]: cluster 2023-10-09T09:21:22.628751+0000 osd.7 (osd.7) 427 : cluster 3 failed to encode map e1008 with expected crc 2023-10-09T09:21:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[75425]: cluster 2023-10-09T09:21:23.354361+0000 osd.5 (osd.5) 550 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:24.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[75425]: cluster 2023-10-09T09:21:23.357410+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e1009: 8 total, 8 up, 8 in 2023-10-09T09:21:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[75425]: cluster 2023-10-09T09:21:23.370899+0000 osd.4 (osd.4) 468 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[75425]: cluster 2023-10-09T09:21:23.433415+0000 osd.7 (osd.7) 428 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:24.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[75425]: cluster 2023-10-09T09:21:23.476437+0000 mgr.y (mgr.34107) 445 : cluster 0 pgmap v1067: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:21:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[78107]: cluster 2023-10-09T09:21:22.628751+0000 osd.7 (osd.7) 427 : cluster 3 failed to encode map e1008 with expected crc 2023-10-09T09:21:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[78107]: cluster 2023-10-09T09:21:23.354361+0000 osd.5 (osd.5) 550 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[78107]: cluster 2023-10-09T09:21:23.357410+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e1009: 8 total, 8 up, 8 in 2023-10-09T09:21:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[78107]: cluster 2023-10-09T09:21:23.370899+0000 osd.4 (osd.4) 468 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[78107]: cluster 2023-10-09T09:21:23.433415+0000 osd.7 (osd.7) 428 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:24.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:24 smithi163 bash[78107]: cluster 2023-10-09T09:21:23.476437+0000 mgr.y (mgr.34107) 445 : cluster 0 pgmap v1067: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:21:24.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:24 smithi181 bash[59345]: cluster 2023-10-09T09:21:22.628751+0000 osd.7 (osd.7) 427 : cluster 3 failed to encode map e1008 with expected crc 2023-10-09T09:21:24.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:24 smithi181 bash[59345]: cluster 2023-10-09T09:21:23.354361+0000 osd.5 (osd.5) 550 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:24.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:24 smithi181 bash[59345]: cluster 2023-10-09T09:21:23.357410+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e1009: 8 total, 8 up, 8 in 2023-10-09T09:21:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:24 smithi181 bash[59345]: cluster 2023-10-09T09:21:23.370899+0000 osd.4 (osd.4) 468 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:24 smithi181 bash[59345]: cluster 2023-10-09T09:21:23.433415+0000 osd.7 (osd.7) 428 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:24 smithi181 bash[59345]: cluster 2023-10-09T09:21:23.476437+0000 mgr.y (mgr.34107) 445 : cluster 0 pgmap v1067: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:21:25.342 INFO:tasks.rados.rados.0.smithi163.stdout:3773: done (1 left) 2023-10-09T09:21:25.343 INFO:tasks.rados.rados.0.smithi163.stdout:3774: snap_remove snap 457 2023-10-09T09:21:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[75425]: cluster 2023-10-09T09:21:23.910954+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[75425]: cluster 2023-10-09T09:21:24.366607+0000 osd.5 (osd.5) 551 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[75425]: cluster 2023-10-09T09:21:24.372667+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e1010: 8 total, 8 up, 8 in 2023-10-09T09:21:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[75425]: cluster 2023-10-09T09:21:24.550511+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:25.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[75425]: cluster 2023-10-09T09:21:25.341700+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e1011: 8 total, 8 up, 8 in 2023-10-09T09:21:25.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[75425]: audit 2023-10-09T09:21:25.352854+0000 mon.a (mon.0) 1243 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[78107]: cluster 2023-10-09T09:21:23.910954+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[78107]: cluster 2023-10-09T09:21:24.366607+0000 osd.5 (osd.5) 551 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[78107]: cluster 2023-10-09T09:21:24.372667+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e1010: 8 total, 8 up, 8 in 2023-10-09T09:21:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[78107]: cluster 2023-10-09T09:21:24.550511+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[78107]: cluster 2023-10-09T09:21:25.341700+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e1011: 8 total, 8 up, 8 in 2023-10-09T09:21:25.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:25 smithi163 bash[78107]: audit 2023-10-09T09:21:25.352854+0000 mon.a (mon.0) 1243 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:25 smithi181 bash[59345]: cluster 2023-10-09T09:21:23.910954+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e1009 with expected crc 2023-10-09T09:21:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:25 smithi181 bash[59345]: cluster 2023-10-09T09:21:24.366607+0000 osd.5 (osd.5) 551 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:25 smithi181 bash[59345]: cluster 2023-10-09T09:21:24.372667+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e1010: 8 total, 8 up, 8 in 2023-10-09T09:21:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:25 smithi181 bash[59345]: cluster 2023-10-09T09:21:24.550511+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:25 smithi181 bash[59345]: cluster 2023-10-09T09:21:25.341700+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e1011: 8 total, 8 up, 8 in 2023-10-09T09:21:25.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:25 smithi181 bash[59345]: audit 2023-10-09T09:21:25.352854+0000 mon.a (mon.0) 1243 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:26.345 INFO:tasks.rados.rados.0.smithi163.stdout:3774: done (1 left) 2023-10-09T09:21:26.345 INFO:tasks.rados.rados.0.smithi163.stdout:3775: read oid 4 snap -1 2023-10-09T09:21:26.345 INFO:tasks.rados.rados.0.smithi163.stdout:3775: expect (ObjNum 1182 snap 397 seq_num 1182) 2023-10-09T09:21:26.345 INFO:tasks.rados.rados.0.smithi163.stdout:3776: write oid 1 current snap is 473 2023-10-09T09:21:26.345 INFO:tasks.rados.rados.0.smithi163.stdout:3776: seq_num 1420 ranges {665940=495238,1795254=540163,2978885=475670} 2023-10-09T09:21:26.356 INFO:tasks.rados.rados.0.smithi163.stdout:3776: writing smithi16367185-1 from 665940 to 1161178 tid 1 2023-10-09T09:21:26.363 INFO:tasks.rados.rados.0.smithi163.stdout:3776: writing smithi16367185-1 from 1795254 to 2335417 tid 2 2023-10-09T09:21:26.371 INFO:tasks.rados.rados.0.smithi163.stdout:3776: writing smithi16367185-1 from 2978885 to 3454555 tid 3 2023-10-09T09:21:26.373 INFO:tasks.rados.rados.0.smithi163.stdout:3772: done (2 left) 2023-10-09T09:21:26.373 INFO:tasks.rados.rados.0.smithi163.stdout:3777: rollback oid 50 current snap is 473 2023-10-09T09:21:26.373 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 50 to 464 2023-10-09T09:21:26.373 INFO:tasks.rados.rados.0.smithi163.stdout:3775: expect (ObjNum 1182 snap 397 seq_num 1182) 2023-10-09T09:21:26.507 INFO:tasks.rados.rados.0.smithi163.stdout:3776: finishing write tid 1 to smithi16367185-1 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:3776: finishing write tid 2 to smithi16367185-1 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:3777: finishing rollback tid 0 to smithi16367185-50 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:3776: finishing write tid 3 to smithi16367185-1 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:3777: finishing rollback tid 1 to smithi16367185-50 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:3777: finishing rollback tid 2 to smithi16367185-50 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1756 (ObjNum 1086 snap 362 seq_num 1086) dirty exists 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:3776: finishing write tid 4 to smithi16367185-1 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:3776: finishing write tid 5 to smithi16367185-1 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 1 v 2107 (ObjNum 1420 snap 473 seq_num 1420) dirty exists 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:3776: left oid 1 (ObjNum 1420 snap 473 seq_num 1420) 2023-10-09T09:21:26.508 INFO:tasks.rados.rados.0.smithi163.stdout:3775: done (2 left) 2023-10-09T09:21:26.509 INFO:tasks.rados.rados.0.smithi163.stdout:3776: done (1 left) 2023-10-09T09:21:26.509 INFO:tasks.rados.rados.0.smithi163.stdout:3777: done (0 left) 2023-10-09T09:21:26.509 INFO:tasks.rados.rados.0.smithi163.stdout:3778: write oid 11 current snap is 473 2023-10-09T09:21:26.509 INFO:tasks.rados.rados.0.smithi163.stdout:3778: seq_num 1421 ranges {682747=570487,1853736=683821,3006308=403295} 2023-10-09T09:21:26.521 INFO:tasks.rados.rados.0.smithi163.stdout:3778: writing smithi16367185-11 from 682747 to 1253234 tid 1 2023-10-09T09:21:26.528 INFO:tasks.rados.rados.0.smithi163.stdout:3778: writing smithi16367185-11 from 1853736 to 2537557 tid 2 2023-10-09T09:21:26.533 INFO:tasks.rados.rados.0.smithi163.stdout:3778: writing smithi16367185-11 from 3006308 to 3409603 tid 3 2023-10-09T09:21:26.533 INFO:tasks.rados.rados.0.smithi163.stdout:3779: write oid 15 current snap is 473 2023-10-09T09:21:26.533 INFO:tasks.rados.rados.0.smithi163.stdout:3779: seq_num 1422 ranges {699554=762089,2028571=543832,3150084=614567} 2023-10-09T09:21:26.541 INFO:tasks.rados.rados.0.smithi163.stdout:3779: writing smithi16367185-15 from 699554 to 1461643 tid 1 2023-10-09T09:21:26.547 INFO:tasks.rados.rados.0.smithi163.stdout:3779: writing smithi16367185-15 from 2028571 to 2572403 tid 2 2023-10-09T09:21:26.555 INFO:tasks.rados.rados.0.smithi163.stdout:3779: writing smithi16367185-15 from 3150084 to 3764651 tid 3 2023-10-09T09:21:26.556 INFO:tasks.rados.rados.0.smithi163.stdout:3780: snap_create 2023-10-09T09:21:26.556 INFO:tasks.rados.rados.0.smithi163.stdout:3778: finishing write tid 1 to smithi16367185-11 2023-10-09T09:21:26.556 INFO:tasks.rados.rados.0.smithi163.stdout:3778: finishing write tid 2 to smithi16367185-11 2023-10-09T09:21:26.556 INFO:tasks.rados.rados.0.smithi163.stdout:3778: finishing write tid 3 to smithi16367185-11 2023-10-09T09:21:26.556 INFO:tasks.rados.rados.0.smithi163.stdout:3778: finishing write tid 4 to smithi16367185-11 2023-10-09T09:21:26.556 INFO:tasks.rados.rados.0.smithi163.stdout:3778: finishing write tid 5 to smithi16367185-11 2023-10-09T09:21:26.556 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 1015 (ObjNum 1421 snap 473 seq_num 1421) dirty exists 2023-10-09T09:21:26.556 INFO:tasks.rados.rados.0.smithi163.stdout:3778: left oid 11 (ObjNum 1421 snap 473 seq_num 1421) 2023-10-09T09:21:26.557 INFO:tasks.rados.rados.0.smithi163.stdout:3779: finishing write tid 1 to smithi16367185-15 2023-10-09T09:21:26.557 INFO:tasks.rados.rados.0.smithi163.stdout:3779: finishing write tid 2 to smithi16367185-15 2023-10-09T09:21:26.561 INFO:tasks.rados.rados.0.smithi163.stdout:3779: finishing write tid 3 to smithi16367185-15 2023-10-09T09:21:26.561 INFO:tasks.rados.rados.0.smithi163.stdout:3779: finishing write tid 4 to smithi16367185-15 2023-10-09T09:21:26.562 INFO:tasks.rados.rados.0.smithi163.stdout:3779: finishing write tid 5 to smithi16367185-15 2023-10-09T09:21:26.562 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 1954 (ObjNum 1422 snap 473 seq_num 1422) dirty exists 2023-10-09T09:21:26.562 INFO:tasks.rados.rados.0.smithi163.stdout:3779: left oid 15 (ObjNum 1422 snap 473 seq_num 1422) 2023-10-09T09:21:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[78107]: cluster 2023-10-09T09:21:24.550621+0000 osd.4 (osd.4) 469 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[78107]: cluster 2023-10-09T09:21:24.550712+0000 osd.7 (osd.7) 429 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[78107]: cluster 2023-10-09T09:21:25.336539+0000 osd.5 (osd.5) 552 : cluster 3 failed to encode map e1011 with expected crc 2023-10-09T09:21:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[78107]: cluster 2023-10-09T09:21:25.477188+0000 mgr.y (mgr.34107) 446 : cluster 0 pgmap v1070: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.2 MiB/s wr, 7 op/s 2023-10-09T09:21:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[78107]: cluster 2023-10-09T09:21:25.537207+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e1011 with expected crc 2023-10-09T09:21:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[78107]: cluster 2023-10-09T09:21:26.344336+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e1012: 8 total, 8 up, 8 in 2023-10-09T09:21:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[75425]: cluster 2023-10-09T09:21:24.550621+0000 osd.4 (osd.4) 469 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[75425]: cluster 2023-10-09T09:21:24.550712+0000 osd.7 (osd.7) 429 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[75425]: cluster 2023-10-09T09:21:25.336539+0000 osd.5 (osd.5) 552 : cluster 3 failed to encode map e1011 with expected crc 2023-10-09T09:21:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[75425]: cluster 2023-10-09T09:21:25.477188+0000 mgr.y (mgr.34107) 446 : cluster 0 pgmap v1070: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.2 MiB/s wr, 7 op/s 2023-10-09T09:21:26.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[75425]: cluster 2023-10-09T09:21:25.537207+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e1011 with expected crc 2023-10-09T09:21:26.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:26 smithi163 bash[75425]: cluster 2023-10-09T09:21:26.344336+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e1012: 8 total, 8 up, 8 in 2023-10-09T09:21:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:26 smithi181 bash[59345]: cluster 2023-10-09T09:21:24.550621+0000 osd.4 (osd.4) 469 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:26 smithi181 bash[59345]: cluster 2023-10-09T09:21:24.550712+0000 osd.7 (osd.7) 429 : cluster 3 failed to encode map e1010 with expected crc 2023-10-09T09:21:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:26 smithi181 bash[59345]: cluster 2023-10-09T09:21:25.336539+0000 osd.5 (osd.5) 552 : cluster 3 failed to encode map e1011 with expected crc 2023-10-09T09:21:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:26 smithi181 bash[59345]: cluster 2023-10-09T09:21:25.477188+0000 mgr.y (mgr.34107) 446 : cluster 0 pgmap v1070: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.2 MiB/s wr, 7 op/s 2023-10-09T09:21:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:26 smithi181 bash[59345]: cluster 2023-10-09T09:21:25.537207+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e1011 with expected crc 2023-10-09T09:21:26.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:26 smithi181 bash[59345]: cluster 2023-10-09T09:21:26.344336+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e1012: 8 total, 8 up, 8 in 2023-10-09T09:21:27.388 INFO:tasks.rados.rados.0.smithi163.stdout:3778: done (2 left) 2023-10-09T09:21:27.389 INFO:tasks.rados.rados.0.smithi163.stdout:3779: done (1 left) 2023-10-09T09:21:27.389 INFO:tasks.rados.rados.0.smithi163.stdout:3780: done (0 left) 2023-10-09T09:21:27.389 INFO:tasks.rados.rados.0.smithi163.stdout:3781: read oid 30 snap 467 2023-10-09T09:21:27.389 INFO:tasks.rados.rados.0.smithi163.stdout:3781: expect deleted 2023-10-09T09:21:27.389 INFO:tasks.rados.rados.0.smithi163.stdout:3782: write oid 29 current snap is 474 2023-10-09T09:21:27.389 INFO:tasks.rados.rados.0.smithi163.stdout:3782: seq_num 1423 ranges {716361=437338,1687053=687490,2777507=425839,3877756=38605} 2023-10-09T09:21:27.398 INFO:tasks.rados.rados.0.smithi163.stdout:3782: writing smithi16367185-29 from 716361 to 1153699 tid 1 2023-10-09T09:21:27.407 INFO:tasks.rados.rados.0.smithi163.stdout:3782: writing smithi16367185-29 from 1687053 to 2374543 tid 2 2023-10-09T09:21:27.412 INFO:tasks.rados.rados.0.smithi163.stdout:3782: writing smithi16367185-29 from 2777507 to 3203346 tid 3 2023-10-09T09:21:27.416 INFO:tasks.rados.rados.0.smithi163.stdout:3782: writing smithi16367185-29 from 3877756 to 3916361 tid 4 2023-10-09T09:21:27.416 INFO:tasks.rados.rados.0.smithi163.stdout:3783: snap_remove snap 472 2023-10-09T09:21:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:27 smithi163 bash[78107]: cluster 2023-10-09T09:21:26.339856+0000 osd.7 (osd.7) 430 : cluster 3 failed to encode map e1012 with expected crc 2023-10-09T09:21:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:27 smithi163 bash[78107]: cluster 2023-10-09T09:21:26.339921+0000 osd.5 (osd.5) 553 : cluster 3 failed to encode map e1012 with expected crc 2023-10-09T09:21:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:27 smithi163 bash[75425]: cluster 2023-10-09T09:21:26.339856+0000 osd.7 (osd.7) 430 : cluster 3 failed to encode map e1012 with expected crc 2023-10-09T09:21:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:27 smithi163 bash[75425]: cluster 2023-10-09T09:21:26.339921+0000 osd.5 (osd.5) 553 : cluster 3 failed to encode map e1012 with expected crc 2023-10-09T09:21:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:27 smithi181 bash[59345]: cluster 2023-10-09T09:21:26.339856+0000 osd.7 (osd.7) 430 : cluster 3 failed to encode map e1012 with expected crc 2023-10-09T09:21:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:27 smithi181 bash[59345]: cluster 2023-10-09T09:21:26.339921+0000 osd.5 (osd.5) 553 : cluster 3 failed to encode map e1012 with expected crc 2023-10-09T09:21:28.400 INFO:tasks.rados.rados.0.smithi163.stdout:3783: done (2 left) 2023-10-09T09:21:28.400 INFO:tasks.rados.rados.0.smithi163.stdout:3784: write oid 11 current snap is 474 2023-10-09T09:21:28.400 INFO:tasks.rados.rados.0.smithi163.stdout:3782: finishing write tid 1 to smithi16367185-29 2023-10-09T09:21:28.400 INFO:tasks.rados.rados.0.smithi163.stdout:3782: finishing write tid 2 to smithi16367185-29 2023-10-09T09:21:28.400 INFO:tasks.rados.rados.0.smithi163.stdout:3782: finishing write tid 3 to smithi16367185-29 2023-10-09T09:21:28.400 INFO:tasks.rados.rados.0.smithi163.stdout:3782: finishing write tid 4 to smithi16367185-29 2023-10-09T09:21:28.400 INFO:tasks.rados.rados.0.smithi163.stdout:3784: seq_num 1424 ranges {733168=512587,2029182=547501,3088577=637111} 2023-10-09T09:21:28.409 INFO:tasks.rados.rados.0.smithi163.stdout:3784: writing smithi16367185-11 from 733168 to 1245755 tid 1 2023-10-09T09:21:28.417 INFO:tasks.rados.rados.0.smithi163.stdout:3784: writing smithi16367185-11 from 2029182 to 2576683 tid 2 2023-10-09T09:21:28.425 INFO:tasks.rados.rados.0.smithi163.stdout:3784: writing smithi16367185-11 from 3088577 to 3725688 tid 3 2023-10-09T09:21:28.425 INFO:tasks.rados.rados.0.smithi163.stdout:3781: done (2 left) 2023-10-09T09:21:28.425 INFO:tasks.rados.rados.0.smithi163.stdout:3785: write oid 19 current snap is 474 2023-10-09T09:21:28.425 INFO:tasks.rados.rados.0.smithi163.stdout:3782: finishing write tid 5 to smithi16367185-29 2023-10-09T09:21:28.426 INFO:tasks.rados.rados.0.smithi163.stdout:3782: finishing write tid 6 to smithi16367185-29 2023-10-09T09:21:28.426 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1762 (ObjNum 1423 snap 474 seq_num 1423) dirty exists 2023-10-09T09:21:28.426 INFO:tasks.rados.rados.0.smithi163.stdout:3782: left oid 29 (ObjNum 1423 snap 474 seq_num 1423) 2023-10-09T09:21:28.426 INFO:tasks.rados.rados.0.smithi163.stdout:3784: finishing write tid 1 to smithi16367185-11 2023-10-09T09:21:28.426 INFO:tasks.rados.rados.0.smithi163.stdout:3784: finishing write tid 2 to smithi16367185-11 2023-10-09T09:21:28.426 INFO:tasks.rados.rados.0.smithi163.stdout:3785: seq_num 1425 ranges {749975=587836,2087664=691159,3516000=433975} 2023-10-09T09:21:28.432 INFO:tasks.rados.rados.0.smithi163.stdout:3785: writing smithi16367185-19 from 749975 to 1337811 tid 1 2023-10-09T09:21:28.441 INFO:tasks.rados.rados.0.smithi163.stdout:3785: writing smithi16367185-19 from 2087664 to 2778823 tid 2 2023-10-09T09:21:28.447 INFO:tasks.rados.rados.0.smithi163.stdout:3785: writing smithi16367185-19 from 3516000 to 3949975 tid 3 2023-10-09T09:21:28.447 INFO:tasks.rados.rados.0.smithi163.stdout:3782: done (2 left) 2023-10-09T09:21:28.448 INFO:tasks.rados.rados.0.smithi163.stdout:3786: write oid 5 current snap is 474 2023-10-09T09:21:28.448 INFO:tasks.rados.rados.0.smithi163.stdout:3786: seq_num 1426 ranges {766782=663085,2146146=551170,3143423=776008} 2023-10-09T09:21:28.455 INFO:tasks.rados.rados.0.smithi163.stdout:3786: writing smithi16367185-5 from 766782 to 1429867 tid 1 2023-10-09T09:21:28.463 INFO:tasks.rados.rados.0.smithi163.stdout:3786: writing smithi16367185-5 from 2146146 to 2697316 tid 2 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:3786: writing smithi16367185-5 from 3143423 to 3919431 tid 3 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:3787: rollback oid 39 current snap is 474 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 470 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:3784: finishing write tid 3 to smithi16367185-11 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:3784: finishing write tid 4 to smithi16367185-11 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:3784: finishing write tid 5 to smithi16367185-11 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 11 v 1022 (ObjNum 1424 snap 474 seq_num 1424) dirty exists 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:3784: left oid 11 (ObjNum 1424 snap 474 seq_num 1424) 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:3785: finishing write tid 1 to smithi16367185-19 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:3785: finishing write tid 2 to smithi16367185-19 2023-10-09T09:21:28.472 INFO:tasks.rados.rados.0.smithi163.stdout:3785: finishing write tid 3 to smithi16367185-19 2023-10-09T09:21:28.473 INFO:tasks.rados.rados.0.smithi163.stdout:3784: done (3 left) 2023-10-09T09:21:28.473 INFO:tasks.rados.rados.0.smithi163.stdout:3788: delete oid 29 current snap is 474 2023-10-09T09:21:28.473 INFO:tasks.rados.rados.0.smithi163.stdout:3785: finishing write tid 4 to smithi16367185-19 2023-10-09T09:21:28.473 INFO:tasks.rados.rados.0.smithi163.stdout:3785: finishing write tid 5 to smithi16367185-19 2023-10-09T09:21:28.473 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 1077 (ObjNum 1425 snap 474 seq_num 1425) dirty exists 2023-10-09T09:21:28.473 INFO:tasks.rados.rados.0.smithi163.stdout:3785: left oid 19 (ObjNum 1425 snap 474 seq_num 1425) 2023-10-09T09:21:28.473 INFO:tasks.rados.rados.0.smithi163.stdout:3786: finishing write tid 1 to smithi16367185-5 2023-10-09T09:21:28.473 INFO:tasks.rados.rados.0.smithi163.stdout:3786: finishing write tid 2 to smithi16367185-5 2023-10-09T09:21:28.477 INFO:tasks.rados.rados.0.smithi163.stdout:3785: done (3 left) 2023-10-09T09:21:28.477 INFO:tasks.rados.rados.0.smithi163.stdout:3788: done (2 left) 2023-10-09T09:21:28.477 INFO:tasks.rados.rados.0.smithi163.stdout:3789: write oid 22 current snap is 474 2023-10-09T09:21:28.478 INFO:tasks.rados.rados.0.smithi163.stdout:3789: seq_num 1428 ranges {400396=413583,1463110=438486} 2023-10-09T09:21:28.480 INFO:tasks.rados.rados.0.smithi163.stdout:3789: writing smithi16367185-22 from 400396 to 813979 tid 1 2023-10-09T09:21:28.489 INFO:tasks.rados.rados.0.smithi163.stdout:3789: writing smithi16367185-22 from 1463110 to 1901596 tid 2 2023-10-09T09:21:28.489 INFO:tasks.rados.rados.0.smithi163.stdout:3790: rollback oid 50 current snap is 474 2023-10-09T09:21:28.489 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 50 to 473 2023-10-09T09:21:28.489 INFO:tasks.rados.rados.0.smithi163.stdout:3786: finishing write tid 3 to smithi16367185-5 2023-10-09T09:21:28.489 INFO:tasks.rados.rados.0.smithi163.stdout:3786: finishing write tid 4 to smithi16367185-5 2023-10-09T09:21:28.489 INFO:tasks.rados.rados.0.smithi163.stdout:3786: finishing write tid 5 to smithi16367185-5 2023-10-09T09:21:28.489 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 2114 (ObjNum 1426 snap 474 seq_num 1426) dirty exists 2023-10-09T09:21:28.489 INFO:tasks.rados.rados.0.smithi163.stdout:3786: left oid 5 (ObjNum 1426 snap 474 seq_num 1426) 2023-10-09T09:21:28.490 INFO:tasks.rados.rados.0.smithi163.stdout:3787: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:21:28.490 INFO:tasks.rados.rados.0.smithi163.stdout:3787: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:21:28.490 INFO:tasks.rados.rados.0.smithi163.stdout:3787: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:21:28.490 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1762 (ObjNum 1310 snap 443 seq_num 1310) dirty exists 2023-10-09T09:21:28.490 INFO:tasks.rados.rados.0.smithi163.stdout:3789: finishing write tid 1 to smithi16367185-22 2023-10-09T09:21:28.490 INFO:tasks.rados.rados.0.smithi163.stdout:3786: done (3 left) 2023-10-09T09:21:28.490 INFO:tasks.rados.rados.0.smithi163.stdout:3787: done (2 left) 2023-10-09T09:21:28.490 INFO:tasks.rados.rados.0.smithi163.stdout:3791: read oid 11 snap 470 2023-10-09T09:21:28.490 INFO:tasks.rados.rados.0.smithi163.stdout:3791: expect (ObjNum 1409 snap 468 seq_num 1409) 2023-10-09T09:21:28.490 INFO:tasks.rados.rados.0.smithi163.stdout:3792: read oid 8 snap -1 2023-10-09T09:21:28.491 INFO:tasks.rados.rados.0.smithi163.stdout:3792: expect (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:21:28.491 INFO:tasks.rados.rados.0.smithi163.stdout:3793: rollback oid 31 current snap is 474 2023-10-09T09:21:28.491 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 31 to 464 2023-10-09T09:21:28.491 INFO:tasks.rados.rados.0.smithi163.stdout:3794: read oid 9 snap 473 2023-10-09T09:21:28.491 INFO:tasks.rados.rados.0.smithi163.stdout:3794: expect (ObjNum 1404 snap 468 seq_num 1404) 2023-10-09T09:21:28.491 INFO:tasks.rados.rados.0.smithi163.stdout:3795: snap_create 2023-10-09T09:21:28.494 INFO:tasks.rados.rados.0.smithi163.stdout:3793: finishing rollback tid 0 to smithi16367185-31 2023-10-09T09:21:28.494 INFO:tasks.rados.rados.0.smithi163.stdout:3793: finishing rollback tid 1 to smithi16367185-31 2023-10-09T09:21:28.495 INFO:tasks.rados.rados.0.smithi163.stdout:3789: finishing write tid 2 to smithi16367185-22 2023-10-09T09:21:28.496 INFO:tasks.rados.rados.0.smithi163.stdout:3794: expect (ObjNum 1404 snap 468 seq_num 1404) 2023-10-09T09:21:28.648 INFO:tasks.rados.rados.0.smithi163.stdout:3793: finishing rollback tid 2 to smithi16367185-31 2023-10-09T09:21:28.649 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 1143 (ObjNum 1391 snap 462 seq_num 1391) dirty exists 2023-10-09T09:21:28.649 INFO:tasks.rados.rados.0.smithi163.stdout:3789: finishing write tid 3 to smithi16367185-22 2023-10-09T09:21:28.649 INFO:tasks.rados.rados.0.smithi163.stdout:3789: finishing write tid 4 to smithi16367185-22 2023-10-09T09:21:28.649 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 2925 (ObjNum 1428 snap 474 seq_num 1428) dirty exists 2023-10-09T09:21:28.649 INFO:tasks.rados.rados.0.smithi163.stdout:3789: left oid 22 (ObjNum 1428 snap 474 seq_num 1428) 2023-10-09T09:21:28.649 INFO:tasks.rados.rados.0.smithi163.stdout:3791: expect (ObjNum 1409 snap 468 seq_num 1409) 2023-10-09T09:21:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:28 smithi163 bash[78107]: cluster 2023-10-09T09:21:27.384904+0000 osd.5 (osd.5) 554 : cluster 3 failed to encode map e1013 with expected crc 2023-10-09T09:21:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:28 smithi163 bash[78107]: cluster 2023-10-09T09:21:27.388401+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e1013: 8 total, 8 up, 8 in 2023-10-09T09:21:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:28 smithi163 bash[78107]: cluster 2023-10-09T09:21:27.477951+0000 mgr.y (mgr.34107) 447 : cluster 0 pgmap v1073: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.4 MiB/s wr, 12 op/s 2023-10-09T09:21:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:28 smithi163 bash[75425]: cluster 2023-10-09T09:21:27.384904+0000 osd.5 (osd.5) 554 : cluster 3 failed to encode map e1013 with expected crc 2023-10-09T09:21:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:28 smithi163 bash[75425]: cluster 2023-10-09T09:21:27.388401+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e1013: 8 total, 8 up, 8 in 2023-10-09T09:21:28.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:28 smithi163 bash[75425]: cluster 2023-10-09T09:21:27.477951+0000 mgr.y (mgr.34107) 447 : cluster 0 pgmap v1073: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.4 MiB/s wr, 12 op/s 2023-10-09T09:21:28.766 INFO:tasks.rados.rados.0.smithi163.stdout:3790: finishing rollback tid 0 to smithi16367185-50 2023-10-09T09:21:28.766 INFO:tasks.rados.rados.0.smithi163.stdout:3790: finishing rollback tid 1 to smithi16367185-50 2023-10-09T09:21:28.766 INFO:tasks.rados.rados.0.smithi163.stdout:3790: finishing rollback tid 2 to smithi16367185-50 2023-10-09T09:21:28.767 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1766 (ObjNum 1086 snap 362 seq_num 1086) dirty exists 2023-10-09T09:21:28.767 INFO:tasks.rados.rados.0.smithi163.stdout:3792: expect (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:21:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:28 smithi181 bash[59345]: cluster 2023-10-09T09:21:27.384904+0000 osd.5 (osd.5) 554 : cluster 3 failed to encode map e1013 with expected crc 2023-10-09T09:21:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:28 smithi181 bash[59345]: cluster 2023-10-09T09:21:27.388401+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e1013: 8 total, 8 up, 8 in 2023-10-09T09:21:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:28 smithi181 bash[59345]: cluster 2023-10-09T09:21:27.477951+0000 mgr.y (mgr.34107) 447 : cluster 0 pgmap v1073: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.4 MiB/s wr, 12 op/s 2023-10-09T09:21:29.410 INFO:tasks.rados.rados.0.smithi163.stdout:3789: done (6 left) 2023-10-09T09:21:29.410 INFO:tasks.rados.rados.0.smithi163.stdout:3790: done (5 left) 2023-10-09T09:21:29.410 INFO:tasks.rados.rados.0.smithi163.stdout:3791: done (4 left) 2023-10-09T09:21:29.410 INFO:tasks.rados.rados.0.smithi163.stdout:3792: done (3 left) 2023-10-09T09:21:29.410 INFO:tasks.rados.rados.0.smithi163.stdout:3793: done (2 left) 2023-10-09T09:21:29.410 INFO:tasks.rados.rados.0.smithi163.stdout:3794: done (1 left) 2023-10-09T09:21:29.410 INFO:tasks.rados.rados.0.smithi163.stdout:3795: done (0 left) 2023-10-09T09:21:29.410 INFO:tasks.rados.rados.0.smithi163.stdout:3796: snap_create 2023-10-09T09:21:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:29 smithi163 bash[78107]: cluster 2023-10-09T09:21:28.393152+0000 osd.5 (osd.5) 555 : cluster 3 failed to encode map e1014 with expected crc 2023-10-09T09:21:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:29 smithi163 bash[78107]: cluster 2023-10-09T09:21:28.399826+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e1014: 8 total, 8 up, 8 in 2023-10-09T09:21:29.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:29 smithi163 bash[78107]: cluster 2023-10-09T09:21:28.411279+0000 osd.7 (osd.7) 431 : cluster 3 failed to encode map e1014 with expected crc 2023-10-09T09:21:29.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:29 smithi163 bash[75425]: cluster 2023-10-09T09:21:28.393152+0000 osd.5 (osd.5) 555 : cluster 3 failed to encode map e1014 with expected crc 2023-10-09T09:21:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:29 smithi163 bash[75425]: cluster 2023-10-09T09:21:28.399826+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e1014: 8 total, 8 up, 8 in 2023-10-09T09:21:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:29 smithi163 bash[75425]: cluster 2023-10-09T09:21:28.411279+0000 osd.7 (osd.7) 431 : cluster 3 failed to encode map e1014 with expected crc 2023-10-09T09:21:29.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:29 smithi181 bash[59345]: cluster 2023-10-09T09:21:28.393152+0000 osd.5 (osd.5) 555 : cluster 3 failed to encode map e1014 with expected crc 2023-10-09T09:21:29.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:29 smithi181 bash[59345]: cluster 2023-10-09T09:21:28.399826+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e1014: 8 total, 8 up, 8 in 2023-10-09T09:21:29.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:29 smithi181 bash[59345]: cluster 2023-10-09T09:21:28.411279+0000 osd.7 (osd.7) 431 : cluster 3 failed to encode map e1014 with expected crc 2023-10-09T09:21:30.346 INFO:tasks.rados.rados.0.smithi163.stdout:3796: done (0 left) 2023-10-09T09:21:30.346 INFO:tasks.rados.rados.0.smithi163.stdout:3797: delete oid 32 current snap is 476 2023-10-09T09:21:30.356 INFO:tasks.rados.rados.0.smithi163.stdout:3797: done (0 left) 2023-10-09T09:21:30.357 INFO:tasks.rados.rados.0.smithi163.stdout:3798: write oid 40 current snap is 476 2023-10-09T09:21:30.357 INFO:tasks.rados.rados.0.smithi163.stdout:3798: seq_num 1430 ranges {434010=680434,1580074=442155} 2023-10-09T09:21:30.364 INFO:tasks.rados.rados.0.smithi163.stdout:3798: writing smithi16367185-40 from 434010 to 1114444 tid 1 2023-10-09T09:21:30.369 INFO:tasks.rados.rados.0.smithi163.stdout:3798: writing smithi16367185-40 from 1580074 to 2022229 tid 2 2023-10-09T09:21:30.369 INFO:tasks.rados.rados.0.smithi163.stdout:3799: snap_create 2023-10-09T09:21:30.375 INFO:tasks.rados.rados.0.smithi163.stdout:3798: finishing write tid 1 to smithi16367185-40 2023-10-09T09:21:30.375 INFO:tasks.rados.rados.0.smithi163.stdout:3798: finishing write tid 2 to smithi16367185-40 2023-10-09T09:21:30.376 INFO:tasks.rados.rados.0.smithi163.stdout:3798: finishing write tid 3 to smithi16367185-40 2023-10-09T09:21:30.376 INFO:tasks.rados.rados.0.smithi163.stdout:3798: finishing write tid 4 to smithi16367185-40 2023-10-09T09:21:30.376 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 1960 (ObjNum 1430 snap 476 seq_num 1430) dirty exists 2023-10-09T09:21:30.376 INFO:tasks.rados.rados.0.smithi163.stdout:3798: left oid 40 (ObjNum 1430 snap 476 seq_num 1430) 2023-10-09T09:21:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:30 smithi163 bash[75425]: cluster 2023-10-09T09:21:29.406263+0000 osd.5 (osd.5) 556 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:30 smithi163 bash[75425]: cluster 2023-10-09T09:21:29.409549+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e1015: 8 total, 8 up, 8 in 2023-10-09T09:21:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:30 smithi163 bash[75425]: cluster 2023-10-09T09:21:29.478659+0000 mgr.y (mgr.34107) 448 : cluster 0 pgmap v1076: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2023-10-09T09:21:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:30 smithi163 bash[75425]: cluster 2023-10-09T09:21:30.346024+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e1016: 8 total, 8 up, 8 in 2023-10-09T09:21:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:30 smithi163 bash[78107]: cluster 2023-10-09T09:21:29.406263+0000 osd.5 (osd.5) 556 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:30 smithi163 bash[78107]: cluster 2023-10-09T09:21:29.409549+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e1015: 8 total, 8 up, 8 in 2023-10-09T09:21:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:30 smithi163 bash[78107]: cluster 2023-10-09T09:21:29.478659+0000 mgr.y (mgr.34107) 448 : cluster 0 pgmap v1076: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2023-10-09T09:21:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:30 smithi163 bash[78107]: cluster 2023-10-09T09:21:30.346024+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e1016: 8 total, 8 up, 8 in 2023-10-09T09:21:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:30 smithi181 bash[59345]: cluster 2023-10-09T09:21:29.406263+0000 osd.5 (osd.5) 556 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:30 smithi181 bash[59345]: cluster 2023-10-09T09:21:29.409549+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e1015: 8 total, 8 up, 8 in 2023-10-09T09:21:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:30 smithi181 bash[59345]: cluster 2023-10-09T09:21:29.478659+0000 mgr.y (mgr.34107) 448 : cluster 0 pgmap v1076: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2023-10-09T09:21:30.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:30 smithi181 bash[59345]: cluster 2023-10-09T09:21:30.346024+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e1016: 8 total, 8 up, 8 in 2023-10-09T09:21:31.356 INFO:tasks.rados.rados.0.smithi163.stdout:3798: done (1 left) 2023-10-09T09:21:31.357 INFO:tasks.rados.rados.0.smithi163.stdout:3799: done (0 left) 2023-10-09T09:21:31.357 INFO:tasks.rados.rados.0.smithi163.stdout:3800: delete oid 17 current snap is 477 2023-10-09T09:21:31.368 INFO:tasks.rados.rados.0.smithi163.stdout:3800: done (0 left) 2023-10-09T09:21:31.368 INFO:tasks.rados.rados.0.smithi163.stdout:3801: delete oid 17 current snap is 477 2023-10-09T09:21:31.371 INFO:tasks.rados.rados.0.smithi163.stdout:3801: done (0 left) 2023-10-09T09:21:31.371 INFO:tasks.rados.rados.0.smithi163.stdout:3802: read oid 32 snap -1 2023-10-09T09:21:31.371 INFO:tasks.rados.rados.0.smithi163.stdout:3802: expect deleted 2023-10-09T09:21:31.371 INFO:tasks.rados.rados.0.smithi163.stdout:3803: rollback oid 27 current snap is 477 2023-10-09T09:21:31.371 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 27 to 467 2023-10-09T09:21:31.371 INFO:tasks.rados.rados.0.smithi163.stdout:3804: write oid 28 current snap is 477 2023-10-09T09:21:31.371 INFO:tasks.rados.rados.0.smithi163.stdout:3804: seq_num 1433 ranges {484431=506181,1639167=445264} 2023-10-09T09:21:31.380 INFO:tasks.rados.rados.0.smithi163.stdout:3804: writing smithi16367185-28 from 484431 to 990612 tid 1 2023-10-09T09:21:31.387 INFO:tasks.rados.rados.0.smithi163.stdout:3804: writing smithi16367185-28 from 1639167 to 2084431 tid 2 2023-10-09T09:21:31.387 INFO:tasks.rados.rados.0.smithi163.stdout:3805: read oid 38 snap -1 2023-10-09T09:21:31.387 INFO:tasks.rados.rados.0.smithi163.stdout:3805: expect deleted 2023-10-09T09:21:31.387 INFO:tasks.rados.rados.0.smithi163.stdout:3806: snap_remove snap 475 2023-10-09T09:21:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:31 smithi163 bash[78107]: cluster 2023-10-09T09:21:30.343100+0000 osd.5 (osd.5) 557 : cluster 3 failed to encode map e1016 with expected crc 2023-10-09T09:21:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:31 smithi163 bash[78107]: cluster 2023-10-09T09:21:30.343228+0000 osd.7 (osd.7) 432 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:31 smithi163 bash[78107]: cluster 2023-10-09T09:21:30.378446+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:31 smithi163 bash[78107]: cluster 2023-10-09T09:21:30.407658+0000 osd.4 (osd.4) 470 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:31.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:31 smithi163 bash[78107]: cluster 2023-10-09T09:21:31.356226+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e1017: 8 total, 8 up, 8 in 2023-10-09T09:21:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:31 smithi163 bash[75425]: cluster 2023-10-09T09:21:30.343100+0000 osd.5 (osd.5) 557 : cluster 3 failed to encode map e1016 with expected crc 2023-10-09T09:21:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:31 smithi163 bash[75425]: cluster 2023-10-09T09:21:30.343228+0000 osd.7 (osd.7) 432 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:31 smithi163 bash[75425]: cluster 2023-10-09T09:21:30.378446+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:31.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:31 smithi163 bash[75425]: cluster 2023-10-09T09:21:30.407658+0000 osd.4 (osd.4) 470 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:31.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:31 smithi163 bash[75425]: cluster 2023-10-09T09:21:31.356226+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e1017: 8 total, 8 up, 8 in 2023-10-09T09:21:31.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:31 smithi181 bash[59345]: cluster 2023-10-09T09:21:30.343100+0000 osd.5 (osd.5) 557 : cluster 3 failed to encode map e1016 with expected crc 2023-10-09T09:21:31.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:31 smithi181 bash[59345]: cluster 2023-10-09T09:21:30.343228+0000 osd.7 (osd.7) 432 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:31.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:31 smithi181 bash[59345]: cluster 2023-10-09T09:21:30.378446+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:31.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:31 smithi181 bash[59345]: cluster 2023-10-09T09:21:30.407658+0000 osd.4 (osd.4) 470 : cluster 3 failed to encode map e1015 with expected crc 2023-10-09T09:21:31.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:31 smithi181 bash[59345]: cluster 2023-10-09T09:21:31.356226+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e1017: 8 total, 8 up, 8 in 2023-10-09T09:21:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3806: done (4 left) 2023-10-09T09:21:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3807: read oid 34 snap -1 2023-10-09T09:21:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3807: expect deleted 2023-10-09T09:21:32.363 INFO:tasks.rados.rados.0.smithi163.stdout:3808: read oid 44 snap 467 2023-10-09T09:21:32.364 INFO:tasks.rados.rados.0.smithi163.stdout:3808: expect (ObjNum 1397 snap 464 seq_num 1397) 2023-10-09T09:21:32.364 INFO:tasks.rados.rados.0.smithi163.stdout:3809: write oid 17 current snap is 477 2023-10-09T09:21:32.364 INFO:tasks.rados.rados.0.smithi163.stdout:3809: seq_num 1434 ranges {501238=581430,1697649=403589} 2023-10-09T09:21:32.372 INFO:tasks.rados.rados.0.smithi163.stdout:3809: writing smithi16367185-17 from 501238 to 1082668 tid 1 2023-10-09T09:21:32.378 INFO:tasks.rados.rados.0.smithi163.stdout:3809: writing smithi16367185-17 from 1697649 to 2101238 tid 2 2023-10-09T09:21:32.379 INFO:tasks.rados.rados.0.smithi163.stdout:3810: write oid 41 current snap is 477 2023-10-09T09:21:32.379 INFO:tasks.rados.rados.0.smithi163.stdout:3810: seq_num 1435 ranges {518045=656679,1756131=361914} 2023-10-09T09:21:32.385 INFO:tasks.rados.rados.0.smithi163.stdout:3810: writing smithi16367185-41 from 518045 to 1174724 tid 1 2023-10-09T09:21:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:3810: writing smithi16367185-41 from 1756131 to 2118045 tid 2 2023-10-09T09:21:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:3811: delete oid 14 current snap is 477 2023-10-09T09:21:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:3803: finishing rollback tid 0 to smithi16367185-27 2023-10-09T09:21:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:3803: finishing rollback tid 1 to smithi16367185-27 2023-10-09T09:21:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:3803: finishing rollback tid 2 to smithi16367185-27 2023-10-09T09:21:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 1081 (ObjNum 1400 snap 465 seq_num 1400) dirty exists 2023-10-09T09:21:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:3804: finishing write tid 1 to smithi16367185-28 2023-10-09T09:21:32.391 INFO:tasks.rados.rados.0.smithi163.stdout:3804: finishing write tid 2 to smithi16367185-28 2023-10-09T09:21:32.392 INFO:tasks.rados.rados.0.smithi163.stdout:3804: finishing write tid 3 to smithi16367185-28 2023-10-09T09:21:32.392 INFO:tasks.rados.rados.0.smithi163.stdout:3804: finishing write tid 4 to smithi16367185-28 2023-10-09T09:21:32.392 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 2117 (ObjNum 1433 snap 477 seq_num 1433) dirty exists 2023-10-09T09:21:32.392 INFO:tasks.rados.rados.0.smithi163.stdout:3804: left oid 28 (ObjNum 1433 snap 477 seq_num 1433) 2023-10-09T09:21:32.392 INFO:tasks.rados.rados.0.smithi163.stdout:3808: expect (ObjNum 1397 snap 464 seq_num 1397) 2023-10-09T09:21:32.612 INFO:tasks.rados.rados.0.smithi163.stdout:3809: finishing write tid 1 to smithi16367185-17 2023-10-09T09:21:32.612 INFO:tasks.rados.rados.0.smithi163.stdout:3809: finishing write tid 2 to smithi16367185-17 2023-10-09T09:21:32.612 INFO:tasks.rados.rados.0.smithi163.stdout:3809: finishing write tid 3 to smithi16367185-17 2023-10-09T09:21:32.612 INFO:tasks.rados.rados.0.smithi163.stdout:3809: finishing write tid 4 to smithi16367185-17 2023-10-09T09:21:32.612 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 1965 (ObjNum 1434 snap 477 seq_num 1434) dirty exists 2023-10-09T09:21:32.612 INFO:tasks.rados.rados.0.smithi163.stdout:3809: left oid 17 (ObjNum 1434 snap 477 seq_num 1434) 2023-10-09T09:21:32.612 INFO:tasks.rados.rados.0.smithi163.stdout:3810: finishing write tid 1 to smithi16367185-41 2023-10-09T09:21:32.612 INFO:tasks.rados.rados.0.smithi163.stdout:3810: finishing write tid 2 to smithi16367185-41 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:3810: finishing write tid 3 to smithi16367185-41 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:3810: finishing write tid 4 to smithi16367185-41 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 370 (ObjNum 1435 snap 477 seq_num 1435) dirty exists 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:3810: left oid 41 (ObjNum 1435 snap 477 seq_num 1435) 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:3802: done (8 left) 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:3803: done (7 left) 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:3804: done (6 left) 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:3805: done (5 left) 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:3807: done (4 left) 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:3808: done (3 left) 2023-10-09T09:21:32.613 INFO:tasks.rados.rados.0.smithi163.stdout:3809: done (2 left) 2023-10-09T09:21:32.614 INFO:tasks.rados.rados.0.smithi163.stdout:3810: done (1 left) 2023-10-09T09:21:32.614 INFO:tasks.rados.rados.0.smithi163.stdout:3811: done (0 left) 2023-10-09T09:21:32.614 INFO:tasks.rados.rados.0.smithi163.stdout:3812: delete oid 13 current snap is 477 2023-10-09T09:21:32.616 INFO:tasks.rados.rados.0.smithi163.stdout:3812: done (0 left) 2023-10-09T09:21:32.616 INFO:tasks.rados.rados.0.smithi163.stdout:3813: snap_create 2023-10-09T09:21:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:32 smithi163 bash[75425]: cluster 2023-10-09T09:21:31.351428+0000 osd.5 (osd.5) 558 : cluster 3 failed to encode map e1017 with expected crc 2023-10-09T09:21:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:32 smithi163 bash[75425]: cluster 2023-10-09T09:21:31.358186+0000 osd.7 (osd.7) 433 : cluster 3 failed to encode map e1017 with expected crc 2023-10-09T09:21:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:32 smithi163 bash[75425]: cluster 2023-10-09T09:21:31.412985+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e1017 with expected crc 2023-10-09T09:21:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:32 smithi163 bash[75425]: cluster 2023-10-09T09:21:31.479390+0000 mgr.y (mgr.34107) 449 : cluster 0 pgmap v1079: 89 pgs: 89 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 22 op/s 2023-10-09T09:21:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:32 smithi163 bash[75425]: cluster 2023-10-09T09:21:32.362811+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e1018: 8 total, 8 up, 8 in 2023-10-09T09:21:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:32 smithi163 bash[78107]: cluster 2023-10-09T09:21:31.351428+0000 osd.5 (osd.5) 558 : cluster 3 failed to encode map e1017 with expected crc 2023-10-09T09:21:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:32 smithi163 bash[78107]: cluster 2023-10-09T09:21:31.358186+0000 osd.7 (osd.7) 433 : cluster 3 failed to encode map e1017 with expected crc 2023-10-09T09:21:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:32 smithi163 bash[78107]: cluster 2023-10-09T09:21:31.412985+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e1017 with expected crc 2023-10-09T09:21:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:32 smithi163 bash[78107]: cluster 2023-10-09T09:21:31.479390+0000 mgr.y (mgr.34107) 449 : cluster 0 pgmap v1079: 89 pgs: 89 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 22 op/s 2023-10-09T09:21:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:32 smithi163 bash[78107]: cluster 2023-10-09T09:21:32.362811+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e1018: 8 total, 8 up, 8 in 2023-10-09T09:21:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:32 smithi181 bash[59345]: cluster 2023-10-09T09:21:31.351428+0000 osd.5 (osd.5) 558 : cluster 3 failed to encode map e1017 with expected crc 2023-10-09T09:21:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:32 smithi181 bash[59345]: cluster 2023-10-09T09:21:31.358186+0000 osd.7 (osd.7) 433 : cluster 3 failed to encode map e1017 with expected crc 2023-10-09T09:21:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:32 smithi181 bash[59345]: cluster 2023-10-09T09:21:31.412985+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e1017 with expected crc 2023-10-09T09:21:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:32 smithi181 bash[59345]: cluster 2023-10-09T09:21:31.479390+0000 mgr.y (mgr.34107) 449 : cluster 0 pgmap v1079: 89 pgs: 89 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 22 op/s 2023-10-09T09:21:32.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:32 smithi181 bash[59345]: cluster 2023-10-09T09:21:32.362811+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e1018: 8 total, 8 up, 8 in 2023-10-09T09:21:33.438 INFO:tasks.rados.rados.0.smithi163.stdout:3813: done (0 left) 2023-10-09T09:21:33.438 INFO:tasks.rados.rados.0.smithi163.stdout:3814: snap_remove snap 470 2023-10-09T09:21:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:21:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:21:33] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.33.4" 2023-10-09T09:21:33.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:33 smithi163 bash[78107]: cluster 2023-10-09T09:21:32.358509+0000 osd.5 (osd.5) 559 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:33 smithi163 bash[78107]: cluster 2023-10-09T09:21:32.374765+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:33 smithi163 bash[78107]: cluster 2023-10-09T09:21:32.374863+0000 osd.7 (osd.7) 434 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:33 smithi163 bash[78107]: cluster 2023-10-09T09:21:32.391853+0000 osd.4 (osd.4) 471 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:33 smithi163 bash[75425]: cluster 2023-10-09T09:21:32.358509+0000 osd.5 (osd.5) 559 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:33 smithi163 bash[75425]: cluster 2023-10-09T09:21:32.374765+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:33 smithi163 bash[75425]: cluster 2023-10-09T09:21:32.374863+0000 osd.7 (osd.7) 434 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:33 smithi163 bash[75425]: cluster 2023-10-09T09:21:32.391853+0000 osd.4 (osd.4) 471 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:33 smithi181 bash[59345]: cluster 2023-10-09T09:21:32.358509+0000 osd.5 (osd.5) 559 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:33 smithi181 bash[59345]: cluster 2023-10-09T09:21:32.374765+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:33 smithi181 bash[59345]: cluster 2023-10-09T09:21:32.374863+0000 osd.7 (osd.7) 434 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:33 smithi181 bash[59345]: cluster 2023-10-09T09:21:32.391853+0000 osd.4 (osd.4) 471 : cluster 3 failed to encode map e1018 with expected crc 2023-10-09T09:21:34.436 INFO:tasks.rados.rados.0.smithi163.stdout:3814: done (0 left) 2023-10-09T09:21:34.712 INFO:tasks.rados.rados.0.smithi163.stdout:3815: rollback oid 34 current snap is 478 2023-10-09T09:21:34.712 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 34 to 469 2023-10-09T09:21:34.712 INFO:tasks.rados.rados.0.smithi163.stdout:3816: delete oid 9 current snap is 478 2023-10-09T09:21:34.715 INFO:tasks.rados.rados.0.smithi163.stdout:3815: finishing rollback tid 1 to smithi16367185-34 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 0 (ObjNum 669367424 snap 21875 seq_num 4000000) dirty dne 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3815: done (1 left) 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3816: done (0 left) 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3817: write oid 15 current snap is 478 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3817: seq_num 1439 ranges {585273=674028,1990059=195214} 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3817: writing smithi16367185-15 from 585273 to 1259301 tid 1 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3817: writing smithi16367185-15 from 1990059 to 2185273 tid 2 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3818: delete oid 24 current snap is 478 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3818: done (1 left) 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3819: delete oid 3 current snap is 478 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3819: done (1 left) 2023-10-09T09:21:34.721 INFO:tasks.rados.rados.0.smithi163.stdout:3820: read oid 21 snap -1 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3820: expect (ObjNum 1223 snap 408 seq_num 1223) 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3821: read oid 14 snap 473 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3821: expect deleted 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3822: read oid 13 snap -1 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3822: expect deleted 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3823: write oid 48 current snap is 478 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3823: seq_num 1442 ranges {635694=499775,1649152=464169} 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3823: writing smithi16367185-48 from 635694 to 1135469 tid 1 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3823: writing smithi16367185-48 from 1649152 to 2113321 tid 2 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3824: snap_create 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3817: finishing write tid 1 to smithi16367185-15 2023-10-09T09:21:34.722 INFO:tasks.rados.rados.0.smithi163.stdout:3817: finishing write tid 2 to smithi16367185-15 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:3817: finishing write tid 3 to smithi16367185-15 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:3817: finishing write tid 4 to smithi16367185-15 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 1971 (ObjNum 1439 snap 478 seq_num 1439) dirty exists 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:3817: left oid 15 (ObjNum 1439 snap 478 seq_num 1439) 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:3820: expect (ObjNum 1223 snap 408 seq_num 1223) 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:3823: finishing write tid 1 to smithi16367185-48 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:3823: finishing write tid 2 to smithi16367185-48 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:3823: finishing write tid 3 to smithi16367185-48 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:3823: finishing write tid 4 to smithi16367185-48 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 48 v 2931 (ObjNum 1442 snap 478 seq_num 1442) dirty exists 2023-10-09T09:21:34.723 INFO:tasks.rados.rados.0.smithi163.stdout:3823: left oid 48 (ObjNum 1442 snap 478 seq_num 1442) 2023-10-09T09:21:34.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:34 smithi163 bash[78107]: cluster 2023-10-09T09:21:33.432472+0000 osd.5 (osd.5) 560 : cluster 3 failed to encode map e1019 with expected crc 2023-10-09T09:21:34.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:34 smithi163 bash[78107]: cluster 2023-10-09T09:21:33.437533+0000 mon.a (mon.0) 1251 : cluster 0 osdmap e1019: 8 total, 8 up, 8 in 2023-10-09T09:21:34.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:34 smithi163 bash[78107]: cluster 2023-10-09T09:21:33.480046+0000 mgr.y (mgr.34107) 450 : cluster 0 pgmap v1082: 89 pgs: 89 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2023-10-09T09:21:34.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:34 smithi163 bash[78107]: cluster 2023-10-09T09:21:34.435860+0000 mon.a (mon.0) 1252 : cluster 0 osdmap e1020: 8 total, 8 up, 8 in 2023-10-09T09:21:34.724 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:34 smithi163 bash[75425]: cluster 2023-10-09T09:21:33.432472+0000 osd.5 (osd.5) 560 : cluster 3 failed to encode map e1019 with expected crc 2023-10-09T09:21:34.724 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:34 smithi163 bash[75425]: cluster 2023-10-09T09:21:33.437533+0000 mon.a (mon.0) 1251 : cluster 0 osdmap e1019: 8 total, 8 up, 8 in 2023-10-09T09:21:34.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:34 smithi163 bash[75425]: cluster 2023-10-09T09:21:33.480046+0000 mgr.y (mgr.34107) 450 : cluster 0 pgmap v1082: 89 pgs: 89 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2023-10-09T09:21:34.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:34 smithi163 bash[75425]: cluster 2023-10-09T09:21:34.435860+0000 mon.a (mon.0) 1252 : cluster 0 osdmap e1020: 8 total, 8 up, 8 in 2023-10-09T09:21:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:34 smithi181 bash[59345]: cluster 2023-10-09T09:21:33.432472+0000 osd.5 (osd.5) 560 : cluster 3 failed to encode map e1019 with expected crc 2023-10-09T09:21:34.917 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:34 smithi181 bash[59345]: cluster 2023-10-09T09:21:33.437533+0000 mon.a (mon.0) 1251 : cluster 0 osdmap e1019: 8 total, 8 up, 8 in 2023-10-09T09:21:34.917 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:34 smithi181 bash[59345]: cluster 2023-10-09T09:21:33.480046+0000 mgr.y (mgr.34107) 450 : cluster 0 pgmap v1082: 89 pgs: 89 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2023-10-09T09:21:34.917 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:34 smithi181 bash[59345]: cluster 2023-10-09T09:21:34.435860+0000 mon.a (mon.0) 1252 : cluster 0 osdmap e1020: 8 total, 8 up, 8 in 2023-10-09T09:21:35.346 INFO:tasks.rados.rados.0.smithi163.stdout:3817: done (5 left) 2023-10-09T09:21:36.671 INFO:tasks.rados.rados.0.smithi163.stdout:3820: done (4 left) 2023-10-09T09:21:36.672 INFO:tasks.rados.rados.0.smithi163.stdout:3821: done (3 left) 2023-10-09T09:21:36.672 INFO:tasks.rados.rados.0.smithi163.stdout:3822: done (2 left) 2023-10-09T09:21:36.672 INFO:tasks.rados.rados.0.smithi163.stdout:3823: done (1 left) 2023-10-09T09:21:36.672 INFO:tasks.rados.rados.0.smithi163.stdout:3824: done (0 left) 2023-10-09T09:21:36.672 INFO:tasks.rados.rados.0.smithi163.stdout:3825: write oid 25 current snap is 479 2023-10-09T09:21:36.672 INFO:tasks.rados.rados.0.smithi163.stdout:3825: seq_num 1443 ranges {652501=575024,1707634=544867} 2023-10-09T09:21:36.673 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:35 smithi181 bash[59345]: cluster 2023-10-09T09:21:34.430320+0000 osd.5 (osd.5) 561 : cluster 3 failed to encode map e1020 with expected crc 2023-10-09T09:21:36.673 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:35 smithi181 bash[59345]: cluster 2023-10-09T09:21:34.435379+0000 osd.4 (osd.4) 472 : cluster 3 failed to encode map e1020 with expected crc 2023-10-09T09:21:36.673 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:35 smithi181 bash[59345]: cluster 2023-10-09T09:21:34.458387+0000 osd.7 (osd.7) 435 : cluster 3 failed to encode map e1020 with expected crc 2023-10-09T09:21:36.673 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:35 smithi181 bash[59345]: cluster 2023-10-09T09:21:35.346076+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e1021: 8 total, 8 up, 8 in 2023-10-09T09:21:36.673 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:35 smithi181 bash[59345]: cluster 2023-10-09T09:21:35.365907+0000 osd.4 (osd.4) 473 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:36.674 INFO:tasks.rados.rados.0.smithi163.stdout:3825: writing smithi16367185-25 from 652501 to 1227525 tid 1 2023-10-09T09:21:36.674 INFO:tasks.rados.rados.0.smithi163.stdout:3825: writing smithi16367185-25 from 1707634 to 2252501 tid 2 2023-10-09T09:21:36.674 INFO:tasks.rados.rados.0.smithi163.stdout:3826: read oid 43 snap -1 2023-10-09T09:21:36.674 INFO:tasks.rados.rados.0.smithi163.stdout:3826: expect deleted 2023-10-09T09:21:36.674 INFO:tasks.rados.rados.0.smithi163.stdout:3827: snap_remove snap 478 2023-10-09T09:21:36.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:35 smithi163 bash[78107]: cluster 2023-10-09T09:21:34.430320+0000 osd.5 (osd.5) 561 : cluster 3 failed to encode map e1020 with expected crc 2023-10-09T09:21:36.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:35 smithi163 bash[78107]: cluster 2023-10-09T09:21:34.435379+0000 osd.4 (osd.4) 472 : cluster 3 failed to encode map e1020 with expected crc 2023-10-09T09:21:36.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:35 smithi163 bash[78107]: cluster 2023-10-09T09:21:34.458387+0000 osd.7 (osd.7) 435 : cluster 3 failed to encode map e1020 with expected crc 2023-10-09T09:21:36.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:35 smithi163 bash[78107]: cluster 2023-10-09T09:21:35.346076+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e1021: 8 total, 8 up, 8 in 2023-10-09T09:21:36.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:35 smithi163 bash[78107]: cluster 2023-10-09T09:21:35.365907+0000 osd.4 (osd.4) 473 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:36.675 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:35 smithi163 bash[75425]: cluster 2023-10-09T09:21:34.430320+0000 osd.5 (osd.5) 561 : cluster 3 failed to encode map e1020 with expected crc 2023-10-09T09:21:36.675 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:35 smithi163 bash[75425]: cluster 2023-10-09T09:21:34.435379+0000 osd.4 (osd.4) 472 : cluster 3 failed to encode map e1020 with expected crc 2023-10-09T09:21:36.675 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:35 smithi163 bash[75425]: cluster 2023-10-09T09:21:34.458387+0000 osd.7 (osd.7) 435 : cluster 3 failed to encode map e1020 with expected crc 2023-10-09T09:21:36.675 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:35 smithi163 bash[75425]: cluster 2023-10-09T09:21:35.346076+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e1021: 8 total, 8 up, 8 in 2023-10-09T09:21:36.676 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:35 smithi163 bash[75425]: cluster 2023-10-09T09:21:35.365907+0000 osd.4 (osd.4) 473 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:36.677 INFO:tasks.rados.rados.0.smithi163.stdout:3825: finishing write tid 1 to smithi16367185-25 2023-10-09T09:21:36.677 INFO:tasks.rados.rados.0.smithi163.stdout:3827: done (2 left) 2023-10-09T09:21:36.677 INFO:tasks.rados.rados.0.smithi163.stdout:3828: read oid 15 snap -1 2023-10-09T09:21:36.677 INFO:tasks.rados.rados.0.smithi163.stdout:3828: expect (ObjNum 1439 snap 478 seq_num 1439) 2023-10-09T09:21:36.677 INFO:tasks.rados.rados.0.smithi163.stdout:3829: read oid 11 snap -1 2023-10-09T09:21:36.677 INFO:tasks.rados.rados.0.smithi163.stdout:3829: expect (ObjNum 1424 snap 474 seq_num 1424) 2023-10-09T09:21:36.677 INFO:tasks.rados.rados.0.smithi163.stdout:3830: delete oid 1 current snap is 479 2023-10-09T09:21:36.677 INFO:tasks.rados.rados.0.smithi163.stdout:3825: finishing write tid 2 to smithi16367185-25 2023-10-09T09:21:36.677 INFO:tasks.rados.rados.0.smithi163.stdout:3825: finishing write tid 3 to smithi16367185-25 2023-10-09T09:21:36.677 INFO:tasks.rados.rados.0.smithi163.stdout:3825: finishing write tid 4 to smithi16367185-25 2023-10-09T09:21:36.678 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1768 (ObjNum 1443 snap 479 seq_num 1443) dirty exists 2023-10-09T09:21:36.678 INFO:tasks.rados.rados.0.smithi163.stdout:3825: left oid 25 (ObjNum 1443 snap 479 seq_num 1443) 2023-10-09T09:21:36.678 INFO:tasks.rados.rados.0.smithi163.stdout:3825: done (4 left) 2023-10-09T09:21:36.678 INFO:tasks.rados.rados.0.smithi163.stdout:3826: done (3 left) 2023-10-09T09:21:36.678 INFO:tasks.rados.rados.0.smithi163.stdout:3830: done (2 left) 2023-10-09T09:21:36.678 INFO:tasks.rados.rados.0.smithi163.stdout:3831: write oid 13 current snap is 479 2023-10-09T09:21:36.678 INFO:tasks.rados.rados.0.smithi163.stdout:3831: seq_num 1445 ranges {686115=441875,1824598=461517} 2023-10-09T09:21:36.678 INFO:tasks.rados.rados.0.smithi163.stdout:3831: writing smithi16367185-13 from 686115 to 1127990 tid 1 2023-10-09T09:21:36.678 INFO:tasks.rados.rados.0.smithi163.stdout:3831: writing smithi16367185-13 from 1824598 to 2286115 tid 2 2023-10-09T09:21:36.678 INFO:tasks.rados.rados.0.smithi163.stdout:3832: snap_remove snap 463 2023-10-09T09:21:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:36 smithi163 bash[78107]: cluster 2023-10-09T09:21:35.342975+0000 osd.5 (osd.5) 562 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:36.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:36 smithi163 bash[78107]: cluster 2023-10-09T09:21:35.345818+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:36.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:36 smithi163 bash[78107]: cluster 2023-10-09T09:21:35.480734+0000 mgr.y (mgr.34107) 451 : cluster 0 pgmap v1085: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 481 KiB/s wr, 3 op/s 2023-10-09T09:21:36.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:36 smithi163 bash[78107]: cluster 2023-10-09T09:21:36.360248+0000 mon.a (mon.0) 1254 : cluster 0 osdmap e1022: 8 total, 8 up, 8 in 2023-10-09T09:21:36.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:36 smithi163 bash[78107]: cluster 2023-10-09T09:21:36.367369+0000 osd.4 (osd.4) 474 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:36 smithi163 bash[75425]: cluster 2023-10-09T09:21:35.342975+0000 osd.5 (osd.5) 562 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:36 smithi163 bash[75425]: cluster 2023-10-09T09:21:35.345818+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:36 smithi163 bash[75425]: cluster 2023-10-09T09:21:35.480734+0000 mgr.y (mgr.34107) 451 : cluster 0 pgmap v1085: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 481 KiB/s wr, 3 op/s 2023-10-09T09:21:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:36 smithi163 bash[75425]: cluster 2023-10-09T09:21:36.360248+0000 mon.a (mon.0) 1254 : cluster 0 osdmap e1022: 8 total, 8 up, 8 in 2023-10-09T09:21:37.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:36 smithi163 bash[75425]: cluster 2023-10-09T09:21:36.367369+0000 osd.4 (osd.4) 474 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:37.467 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:36 smithi181 bash[59345]: cluster 2023-10-09T09:21:35.342975+0000 osd.5 (osd.5) 562 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:37.467 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:36 smithi181 bash[59345]: cluster 2023-10-09T09:21:35.345818+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:37.467 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:36 smithi181 bash[59345]: cluster 2023-10-09T09:21:35.480734+0000 mgr.y (mgr.34107) 451 : cluster 0 pgmap v1085: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 481 KiB/s wr, 3 op/s 2023-10-09T09:21:37.467 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:36 smithi181 bash[59345]: cluster 2023-10-09T09:21:36.360248+0000 mon.a (mon.0) 1254 : cluster 0 osdmap e1022: 8 total, 8 up, 8 in 2023-10-09T09:21:37.467 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:36 smithi181 bash[59345]: cluster 2023-10-09T09:21:36.367369+0000 osd.4 (osd.4) 474 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:37.467 INFO:tasks.rados.rados.0.smithi163.stdout:3832: done (3 left) 2023-10-09T09:21:37.468 INFO:tasks.rados.rados.0.smithi163.stdout:3829: expect (ObjNum 1424 snap 474 seq_num 1424) 2023-10-09T09:21:37.621 INFO:tasks.rados.rados.0.smithi163.stdout:3828: expect (ObjNum 1439 snap 478 seq_num 1439) 2023-10-09T09:21:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[78107]: cluster 2023-10-09T09:21:35.707880+0000 osd.7 (osd.7) 436 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:37.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[78107]: cluster 2023-10-09T09:21:36.355646+0000 osd.5 (osd.5) 563 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:37.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[78107]: cluster 2023-10-09T09:21:36.356972+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:37.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[78107]: cluster 2023-10-09T09:21:36.359988+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:37.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[78107]: cluster 2023-10-09T09:21:36.361621+0000 osd.7 (osd.7) 437 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:37.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[78107]: cluster 2023-10-09T09:21:37.363529+0000 osd.4 (osd.4) 475 : cluster 3 failed to encode map e1023 with expected crc 2023-10-09T09:21:37.757 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[78107]: cluster 2023-10-09T09:21:37.365799+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e1023: 8 total, 8 up, 8 in 2023-10-09T09:21:37.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[75425]: cluster 2023-10-09T09:21:35.707880+0000 osd.7 (osd.7) 436 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:37.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[75425]: cluster 2023-10-09T09:21:36.355646+0000 osd.5 (osd.5) 563 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:37.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[75425]: cluster 2023-10-09T09:21:36.356972+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:37.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[75425]: cluster 2023-10-09T09:21:36.359988+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:37.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[75425]: cluster 2023-10-09T09:21:36.361621+0000 osd.7 (osd.7) 437 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:37.758 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[75425]: cluster 2023-10-09T09:21:37.363529+0000 osd.4 (osd.4) 475 : cluster 3 failed to encode map e1023 with expected crc 2023-10-09T09:21:37.758 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:37 smithi163 bash[75425]: cluster 2023-10-09T09:21:37.365799+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e1023: 8 total, 8 up, 8 in 2023-10-09T09:21:37.758 INFO:tasks.rados.rados.0.smithi163.stdout:3831: finishing write tid 1 to smithi16367185-13 2023-10-09T09:21:37.758 INFO:tasks.rados.rados.0.smithi163.stdout:3831: finishing write tid 2 to smithi16367185-13 2023-10-09T09:21:37.758 INFO:tasks.rados.rados.0.smithi163.stdout:3831: finishing write tid 3 to smithi16367185-13 2023-10-09T09:21:37.758 INFO:tasks.rados.rados.0.smithi163.stdout:3831: finishing write tid 4 to smithi16367185-13 2023-10-09T09:21:37.758 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 841 (ObjNum 1445 snap 479 seq_num 1445) dirty exists 2023-10-09T09:21:37.758 INFO:tasks.rados.rados.0.smithi163.stdout:3831: left oid 13 (ObjNum 1445 snap 479 seq_num 1445) 2023-10-09T09:21:37.759 INFO:tasks.rados.rados.0.smithi163.stdout:3833: read oid 12 snap 469 2023-10-09T09:21:37.759 INFO:tasks.rados.rados.0.smithi163.stdout:3833: expect deleted 2023-10-09T09:21:37.759 INFO:tasks.rados.rados.0.smithi163.stdout:3828: done (3 left) 2023-10-09T09:21:37.759 INFO:tasks.rados.rados.0.smithi163.stdout:3829: done (2 left) 2023-10-09T09:21:37.759 INFO:tasks.rados.rados.0.smithi163.stdout:3831: done (1 left) 2023-10-09T09:21:37.759 INFO:tasks.rados.rados.0.smithi163.stdout:3834: snap_create 2023-10-09T09:21:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:37 smithi181 bash[59345]: cluster 2023-10-09T09:21:35.707880+0000 osd.7 (osd.7) 436 : cluster 3 failed to encode map e1021 with expected crc 2023-10-09T09:21:38.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:37 smithi181 bash[59345]: cluster 2023-10-09T09:21:36.355646+0000 osd.5 (osd.5) 563 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:38.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:37 smithi181 bash[59345]: cluster 2023-10-09T09:21:36.356972+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:38.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:37 smithi181 bash[59345]: cluster 2023-10-09T09:21:36.359988+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:38.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:37 smithi181 bash[59345]: cluster 2023-10-09T09:21:36.361621+0000 osd.7 (osd.7) 437 : cluster 3 failed to encode map e1022 with expected crc 2023-10-09T09:21:38.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:37 smithi181 bash[59345]: cluster 2023-10-09T09:21:37.363529+0000 osd.4 (osd.4) 475 : cluster 3 failed to encode map e1023 with expected crc 2023-10-09T09:21:38.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:37 smithi181 bash[59345]: cluster 2023-10-09T09:21:37.365799+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e1023: 8 total, 8 up, 8 in 2023-10-09T09:21:38.484 INFO:tasks.rados.rados.0.smithi163.stdout:3833: done (1 left) 2023-10-09T09:21:38.484 INFO:tasks.rados.rados.0.smithi163.stdout:3834: done (0 left) 2023-10-09T09:21:38.485 INFO:tasks.rados.rados.0.smithi163.stdout:3835: read oid 18 snap -1 2023-10-09T09:21:38.485 INFO:tasks.rados.rados.0.smithi163.stdout:3835: expect (ObjNum 1374 snap 460 seq_num 1374) 2023-10-09T09:21:38.485 INFO:tasks.rados.rados.0.smithi163.stdout:3836: snap_create 2023-10-09T09:21:38.493 INFO:tasks.rados.rados.0.smithi163.stdout:3835: expect (ObjNum 1374 snap 460 seq_num 1374) 2023-10-09T09:21:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:38 smithi181 bash[59345]: cluster 2023-10-09T09:21:37.359359+0000 osd.5 (osd.5) 564 : cluster 3 failed to encode map e1023 with expected crc 2023-10-09T09:21:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:38 smithi181 bash[59345]: cluster 2023-10-09T09:21:37.481158+0000 mgr.y (mgr.34107) 452 : cluster 0 pgmap v1088: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2023-10-09T09:21:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:38 smithi163 bash[78107]: cluster 2023-10-09T09:21:37.359359+0000 osd.5 (osd.5) 564 : cluster 3 failed to encode map e1023 with expected crc 2023-10-09T09:21:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:38 smithi163 bash[78107]: cluster 2023-10-09T09:21:37.481158+0000 mgr.y (mgr.34107) 452 : cluster 0 pgmap v1088: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2023-10-09T09:21:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:38 smithi163 bash[75425]: cluster 2023-10-09T09:21:37.359359+0000 osd.5 (osd.5) 564 : cluster 3 failed to encode map e1023 with expected crc 2023-10-09T09:21:38.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:38 smithi163 bash[75425]: cluster 2023-10-09T09:21:37.481158+0000 mgr.y (mgr.34107) 452 : cluster 0 pgmap v1088: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2023-10-09T09:21:39.497 INFO:tasks.rados.rados.0.smithi163.stdout:3835: done (1 left) 2023-10-09T09:21:39.497 INFO:tasks.rados.rados.0.smithi163.stdout:3836: done (0 left) 2023-10-09T09:21:39.498 INFO:tasks.rados.rados.0.smithi163.stdout:3837: write oid 20 current snap is 481 2023-10-09T09:21:39.498 INFO:tasks.rados.rados.0.smithi163.stdout:3837: seq_num 1446 ranges {702922=517124,1883080=419842} 2023-10-09T09:21:39.507 INFO:tasks.rados.rados.0.smithi163.stdout:3837: writing smithi16367185-20 from 702922 to 1220046 tid 1 2023-10-09T09:21:39.513 INFO:tasks.rados.rados.0.smithi163.stdout:3837: writing smithi16367185-20 from 1883080 to 2302922 tid 2 2023-10-09T09:21:39.514 INFO:tasks.rados.rados.0.smithi163.stdout:3838: snap_create 2023-10-09T09:21:39.520 INFO:tasks.rados.rados.0.smithi163.stdout:3837: finishing write tid 1 to smithi16367185-20 2023-10-09T09:21:39.521 INFO:tasks.rados.rados.0.smithi163.stdout:3837: finishing write tid 2 to smithi16367185-20 2023-10-09T09:21:39.521 INFO:tasks.rados.rados.0.smithi163.stdout:3837: finishing write tid 3 to smithi16367185-20 2023-10-09T09:21:39.521 INFO:tasks.rados.rados.0.smithi163.stdout:3837: finishing write tid 4 to smithi16367185-20 2023-10-09T09:21:39.522 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 849 (ObjNum 1446 snap 481 seq_num 1446) dirty exists 2023-10-09T09:21:39.522 INFO:tasks.rados.rados.0.smithi163.stdout:3837: left oid 20 (ObjNum 1446 snap 481 seq_num 1446) 2023-10-09T09:21:39.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:39 smithi181 bash[59345]: cluster 2023-10-09T09:21:38.476605+0000 osd.5 (osd.5) 565 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:39.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:39 smithi181 bash[59345]: cluster 2023-10-09T09:21:38.484322+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e1024: 8 total, 8 up, 8 in 2023-10-09T09:21:39.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:39 smithi181 bash[59345]: cluster 2023-10-09T09:21:39.289377+0000 osd.4 (osd.4) 476 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:39.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:39 smithi163 bash[78107]: cluster 2023-10-09T09:21:38.476605+0000 osd.5 (osd.5) 565 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:39.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:39 smithi163 bash[78107]: cluster 2023-10-09T09:21:38.484322+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e1024: 8 total, 8 up, 8 in 2023-10-09T09:21:39.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:39 smithi163 bash[78107]: cluster 2023-10-09T09:21:39.289377+0000 osd.4 (osd.4) 476 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:40.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:39 smithi163 bash[75425]: cluster 2023-10-09T09:21:38.476605+0000 osd.5 (osd.5) 565 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:40.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:39 smithi163 bash[75425]: cluster 2023-10-09T09:21:38.484322+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e1024: 8 total, 8 up, 8 in 2023-10-09T09:21:40.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:39 smithi163 bash[75425]: cluster 2023-10-09T09:21:39.289377+0000 osd.4 (osd.4) 476 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:40.349 INFO:tasks.rados.rados.0.smithi163.stdout:3837: done (1 left) 2023-10-09T09:21:40.519 INFO:tasks.rados.rados.0.smithi163.stdout:3838: done (0 left) 2023-10-09T09:21:40.519 INFO:tasks.rados.rados.0.smithi163.stdout:3839: write oid 43 current snap is 482 2023-10-09T09:21:40.519 INFO:tasks.rados.rados.0.smithi163.stdout:3839: seq_num 1447 ranges {719729=592373,1941562=378167} 2023-10-09T09:21:40.723 INFO:tasks.rados.rados.0.smithi163.stdout:3839: writing smithi16367185-43 from 719729 to 1312102 tid 1 2023-10-09T09:21:40.724 INFO:tasks.rados.rados.0.smithi163.stdout:3839: writing smithi16367185-43 from 1941562 to 2319729 tid 2 2023-10-09T09:21:40.724 INFO:tasks.rados.rados.0.smithi163.stdout:3840: rollback oid 16 current snap is 482 2023-10-09T09:21:40.854 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 16 to 467 2023-10-09T09:21:40.858 INFO:tasks.rados.rados.0.smithi163.stdout:3841: rollback oid 31 current snap is 482 2023-10-09T09:21:40.858 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 31 to 469 2023-10-09T09:21:40.858 INFO:tasks.rados.rados.0.smithi163.stdout:3842: read oid 4 snap -1 2023-10-09T09:21:40.858 INFO:tasks.rados.rados.0.smithi163.stdout:3842: expect (ObjNum 1182 snap 397 seq_num 1182) 2023-10-09T09:21:40.858 INFO:tasks.rados.rados.0.smithi163.stdout:3843: write oid 29 current snap is 482 2023-10-09T09:21:40.858 INFO:tasks.rados.rados.0.smithi163.stdout:3843: seq_num 1448 ranges {736536=667622,2000044=336492} 2023-10-09T09:21:40.858 INFO:tasks.rados.rados.0.smithi163.stdout:3843: writing smithi16367185-29 from 736536 to 1404158 tid 1 2023-10-09T09:21:40.859 INFO:tasks.rados.rados.0.smithi163.stdout:3843: writing smithi16367185-29 from 2000044 to 2336536 tid 2 2023-10-09T09:21:40.859 INFO:tasks.rados.rados.0.smithi163.stdout:3844: write oid 44 current snap is 482 2023-10-09T09:21:40.859 INFO:tasks.rados.rados.0.smithi163.stdout:3839: finishing write tid 1 to smithi16367185-43 2023-10-09T09:21:40.859 INFO:tasks.rados.rados.0.smithi163.stdout:3839: finishing write tid 2 to smithi16367185-43 2023-10-09T09:21:40.859 INFO:tasks.rados.rados.0.smithi163.stdout:3839: finishing write tid 3 to smithi16367185-43 2023-10-09T09:21:40.859 INFO:tasks.rados.rados.0.smithi163.stdout:3841: finishing rollback tid 0 to smithi16367185-31 2023-10-09T09:21:40.859 INFO:tasks.rados.rados.0.smithi163.stdout:3841: finishing rollback tid 1 to smithi16367185-31 2023-10-09T09:21:40.859 INFO:tasks.rados.rados.0.smithi163.stdout:3839: finishing write tid 4 to smithi16367185-43 2023-10-09T09:21:40.859 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 1791 (ObjNum 1447 snap 482 seq_num 1447) dirty exists 2023-10-09T09:21:40.859 INFO:tasks.rados.rados.0.smithi163.stdout:3839: left oid 43 (ObjNum 1447 snap 482 seq_num 1447) 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3844: seq_num 1449 ranges {753343=742871,1942173=411170} 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3844: writing smithi16367185-44 from 753343 to 1496214 tid 1 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3844: writing smithi16367185-44 from 1942173 to 2353343 tid 2 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3839: done (5 left) 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3845: read oid 9 snap -1 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3845: expect deleted 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3846: snap_create 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3841: finishing rollback tid 2 to smithi16367185-31 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 31 v 1157 (ObjNum 1391 snap 462 seq_num 1391) dirty exists 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3843: finishing write tid 1 to smithi16367185-29 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3843: finishing write tid 2 to smithi16367185-29 2023-10-09T09:21:40.860 INFO:tasks.rados.rados.0.smithi163.stdout:3843: finishing write tid 3 to smithi16367185-29 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:3842: expect (ObjNum 1182 snap 397 seq_num 1182) 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:3840: finishing rollback tid 0 to smithi16367185-16 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:3843: finishing write tid 4 to smithi16367185-29 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1773 (ObjNum 1448 snap 482 seq_num 1448) dirty exists 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:3843: left oid 29 (ObjNum 1448 snap 482 seq_num 1448) 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:3840: finishing rollback tid 1 to smithi16367185-16 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:3840: finishing rollback tid 2 to smithi16367185-16 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 2135 (ObjNum 1371 snap 459 seq_num 1371) dirty exists 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:3844: finishing write tid 1 to smithi16367185-44 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:3844: finishing write tid 2 to smithi16367185-44 2023-10-09T09:21:40.861 INFO:tasks.rados.rados.0.smithi163.stdout:3844: finishing write tid 3 to smithi16367185-44 2023-10-09T09:21:40.862 INFO:tasks.rados.rados.0.smithi163.stdout:3844: finishing write tid 4 to smithi16367185-44 2023-10-09T09:21:40.862 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 2939 (ObjNum 1449 snap 482 seq_num 1449) dirty exists 2023-10-09T09:21:40.862 INFO:tasks.rados.rados.0.smithi163.stdout:3844: left oid 44 (ObjNum 1449 snap 482 seq_num 1449) 2023-10-09T09:21:40.862 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:40 smithi181 bash[59345]: cluster 2023-10-09T09:21:39.340189+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:40.862 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:40 smithi181 bash[59345]: cluster 2023-10-09T09:21:39.455577+0000 osd.7 (osd.7) 438 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:40.862 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:40 smithi181 bash[59345]: cluster 2023-10-09T09:21:39.482196+0000 mgr.y (mgr.34107) 453 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 666 KiB/s wr, 9 op/s 2023-10-09T09:21:40.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:40 smithi181 bash[59345]: cluster 2023-10-09T09:21:39.493764+0000 osd.5 (osd.5) 566 : cluster 3 failed to encode map e1025 with expected crc 2023-10-09T09:21:40.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:40 smithi181 bash[59345]: cluster 2023-10-09T09:21:39.497234+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e1025: 8 total, 8 up, 8 in 2023-10-09T09:21:40.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:40 smithi181 bash[59345]: cluster 2023-10-09T09:21:39.509813+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e1025 with expected crc 2023-10-09T09:21:40.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:40 smithi181 bash[59345]: cluster 2023-10-09T09:21:40.348173+0000 osd.4 (osd.4) 477 : cluster 3 failed to encode map e1025 with expected crc 2023-10-09T09:21:40.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:40 smithi181 bash[59345]: cluster 2023-10-09T09:21:40.349603+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e1026: 8 total, 8 up, 8 in 2023-10-09T09:21:40.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:40 smithi181 bash[59345]: audit 2023-10-09T09:21:40.353355+0000 mon.a (mon.0) 1259 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[78107]: cluster 2023-10-09T09:21:39.340189+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:41.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[78107]: cluster 2023-10-09T09:21:39.455577+0000 osd.7 (osd.7) 438 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:41.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[78107]: cluster 2023-10-09T09:21:39.482196+0000 mgr.y (mgr.34107) 453 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 666 KiB/s wr, 9 op/s 2023-10-09T09:21:41.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[78107]: cluster 2023-10-09T09:21:39.493764+0000 osd.5 (osd.5) 566 : cluster 3 failed to encode map e1025 with expected crc 2023-10-09T09:21:41.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[78107]: cluster 2023-10-09T09:21:39.497234+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e1025: 8 total, 8 up, 8 in 2023-10-09T09:21:41.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[78107]: cluster 2023-10-09T09:21:39.509813+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e1025 with expected crc 2023-10-09T09:21:41.008 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[78107]: cluster 2023-10-09T09:21:40.348173+0000 osd.4 (osd.4) 477 : cluster 3 failed to encode map e1025 with expected crc 2023-10-09T09:21:41.008 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[78107]: cluster 2023-10-09T09:21:40.349603+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e1026: 8 total, 8 up, 8 in 2023-10-09T09:21:41.008 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[78107]: audit 2023-10-09T09:21:40.353355+0000 mon.a (mon.0) 1259 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:41.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[75425]: cluster 2023-10-09T09:21:39.340189+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:41.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[75425]: cluster 2023-10-09T09:21:39.455577+0000 osd.7 (osd.7) 438 : cluster 3 failed to encode map e1024 with expected crc 2023-10-09T09:21:41.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[75425]: cluster 2023-10-09T09:21:39.482196+0000 mgr.y (mgr.34107) 453 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 666 KiB/s wr, 9 op/s 2023-10-09T09:21:41.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[75425]: cluster 2023-10-09T09:21:39.493764+0000 osd.5 (osd.5) 566 : cluster 3 failed to encode map e1025 with expected crc 2023-10-09T09:21:41.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[75425]: cluster 2023-10-09T09:21:39.497234+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e1025: 8 total, 8 up, 8 in 2023-10-09T09:21:41.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[75425]: cluster 2023-10-09T09:21:39.509813+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e1025 with expected crc 2023-10-09T09:21:41.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[75425]: cluster 2023-10-09T09:21:40.348173+0000 osd.4 (osd.4) 477 : cluster 3 failed to encode map e1025 with expected crc 2023-10-09T09:21:41.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[75425]: cluster 2023-10-09T09:21:40.349603+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e1026: 8 total, 8 up, 8 in 2023-10-09T09:21:41.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:40 smithi163 bash[75425]: audit 2023-10-09T09:21:40.353355+0000 mon.a (mon.0) 1259 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:41.363 INFO:tasks.rados.rados.0.smithi163.stdout:3840: done (6 left) 2023-10-09T09:21:41.608 INFO:tasks.rados.rados.0.smithi163.stdout:3841: done (5 left) 2023-10-09T09:21:41.608 INFO:tasks.rados.rados.0.smithi163.stdout:3842: done (4 left) 2023-10-09T09:21:41.608 INFO:tasks.rados.rados.0.smithi163.stdout:3843: done (3 left) 2023-10-09T09:21:41.608 INFO:tasks.rados.rados.0.smithi163.stdout:3844: done (2 left) 2023-10-09T09:21:41.608 INFO:tasks.rados.rados.0.smithi163.stdout:3845: done (1 left) 2023-10-09T09:21:41.608 INFO:tasks.rados.rados.0.smithi163.stdout:3846: done (0 left) 2023-10-09T09:21:41.611 INFO:tasks.rados.rados.0.smithi163.stdout:3847: write oid 44 current snap is 483 2023-10-09T09:21:41.611 INFO:tasks.rados.rados.0.smithi163.stdout:3847: seq_num 1450 ranges {770150=418120,1600655=478845} 2023-10-09T09:21:41.613 INFO:tasks.rados.rados.0.smithi163.stdout:3847: writing smithi16367185-44 from 770150 to 1188270 tid 1 2023-10-09T09:21:41.613 INFO:tasks.rados.rados.0.smithi163.stdout:3847: writing smithi16367185-44 from 1600655 to 2079500 tid 2 2023-10-09T09:21:41.629 INFO:tasks.rados.rados.0.smithi163.stdout:3848: read oid 10 snap 481 2023-10-09T09:21:41.629 INFO:tasks.rados.rados.0.smithi163.stdout:3848: expect deleted 2023-10-09T09:21:41.630 INFO:tasks.rados.rados.0.smithi163.stdout:3849: write oid 41 current snap is 483 2023-10-09T09:21:41.630 INFO:tasks.rados.rados.0.smithi163.stdout:3849: seq_num 1451 ranges {786957=493369,2059137=327820} 2023-10-09T09:21:41.631 INFO:tasks.rados.rados.0.smithi163.stdout:3849: writing smithi16367185-41 from 786957 to 1280326 tid 1 2023-10-09T09:21:41.631 INFO:tasks.rados.rados.0.smithi163.stdout:3849: writing smithi16367185-41 from 2059137 to 2386957 tid 2 2023-10-09T09:21:41.631 INFO:tasks.rados.rados.0.smithi163.stdout:3850: delete oid 37 current snap is 483 2023-10-09T09:21:41.631 INFO:tasks.rados.rados.0.smithi163.stdout:3847: finishing write tid 1 to smithi16367185-44 2023-10-09T09:21:41.631 INFO:tasks.rados.rados.0.smithi163.stdout:3847: finishing write tid 2 to smithi16367185-44 2023-10-09T09:21:41.631 INFO:tasks.rados.rados.0.smithi163.stdout:3847: finishing write tid 3 to smithi16367185-44 2023-10-09T09:21:41.631 INFO:tasks.rados.rados.0.smithi163.stdout:3847: finishing write tid 4 to smithi16367185-44 2023-10-09T09:21:41.632 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 2943 (ObjNum 1450 snap 483 seq_num 1450) dirty exists 2023-10-09T09:21:41.632 INFO:tasks.rados.rados.0.smithi163.stdout:3847: left oid 44 (ObjNum 1450 snap 483 seq_num 1450) 2023-10-09T09:21:41.632 INFO:tasks.rados.rados.0.smithi163.stdout:3849: finishing write tid 1 to smithi16367185-41 2023-10-09T09:21:41.632 INFO:tasks.rados.rados.0.smithi163.stdout:3849: finishing write tid 2 to smithi16367185-41 2023-10-09T09:21:41.632 INFO:tasks.rados.rados.0.smithi163.stdout:3849: finishing write tid 3 to smithi16367185-41 2023-10-09T09:21:41.632 INFO:tasks.rados.rados.0.smithi163.stdout:3849: finishing write tid 4 to smithi16367185-41 2023-10-09T09:21:41.632 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 374 (ObjNum 1451 snap 483 seq_num 1451) dirty exists 2023-10-09T09:21:41.632 INFO:tasks.rados.rados.0.smithi163.stdout:3849: left oid 41 (ObjNum 1451 snap 483 seq_num 1451) 2023-10-09T09:21:41.632 INFO:tasks.rados.rados.0.smithi163.stdout:3847: done (3 left) 2023-10-09T09:21:41.633 INFO:tasks.rados.rados.0.smithi163.stdout:3848: done (2 left) 2023-10-09T09:21:41.633 INFO:tasks.rados.rados.0.smithi163.stdout:3849: done (1 left) 2023-10-09T09:21:41.633 INFO:tasks.rados.rados.0.smithi163.stdout:3850: done (0 left) 2023-10-09T09:21:41.633 INFO:tasks.rados.rados.0.smithi163.stdout:3851: delete oid 18 current snap is 483 2023-10-09T09:21:41.633 INFO:tasks.rados.rados.0.smithi163.stdout:3851: done (0 left) 2023-10-09T09:21:41.633 INFO:tasks.rados.rados.0.smithi163.stdout:3852: delete oid 47 current snap is 483 2023-10-09T09:21:41.633 INFO:tasks.rados.rados.0.smithi163.stdout:3852: done (0 left) 2023-10-09T09:21:41.633 INFO:tasks.rados.rados.0.smithi163.stdout:3853: snap_remove snap 474 2023-10-09T09:21:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:41 smithi181 bash[59345]: cluster 2023-10-09T09:21:40.345334+0000 osd.5 (osd.5) 567 : cluster 3 failed to encode map e1026 with expected crc 2023-10-09T09:21:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:41 smithi181 bash[59345]: cluster 2023-10-09T09:21:40.387113+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e1026 with expected crc 2023-10-09T09:21:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:41 smithi181 bash[59345]: cluster 2023-10-09T09:21:41.362676+0000 mon.a (mon.0) 1260 : cluster 0 osdmap e1027: 8 total, 8 up, 8 in 2023-10-09T09:21:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:41 smithi163 bash[75425]: cluster 2023-10-09T09:21:40.345334+0000 osd.5 (osd.5) 567 : cluster 3 failed to encode map e1026 with expected crc 2023-10-09T09:21:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:41 smithi163 bash[75425]: cluster 2023-10-09T09:21:40.387113+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e1026 with expected crc 2023-10-09T09:21:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:41 smithi163 bash[75425]: cluster 2023-10-09T09:21:41.362676+0000 mon.a (mon.0) 1260 : cluster 0 osdmap e1027: 8 total, 8 up, 8 in 2023-10-09T09:21:41.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:41 smithi163 bash[78107]: cluster 2023-10-09T09:21:40.345334+0000 osd.5 (osd.5) 567 : cluster 3 failed to encode map e1026 with expected crc 2023-10-09T09:21:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:41 smithi163 bash[78107]: cluster 2023-10-09T09:21:40.387113+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e1026 with expected crc 2023-10-09T09:21:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:41 smithi163 bash[78107]: cluster 2023-10-09T09:21:41.362676+0000 mon.a (mon.0) 1260 : cluster 0 osdmap e1027: 8 total, 8 up, 8 in 2023-10-09T09:21:42.373 INFO:tasks.rados.rados.0.smithi163.stdout:3853: done (0 left) 2023-10-09T09:21:42.690 INFO:tasks.rados.rados.0.smithi163.stdout:3854: write oid 30 current snap is 483 2023-10-09T09:21:42.690 INFO:tasks.rados.rados.0.smithi163.stdout:3854: seq_num 1455 ranges {454185=510718,1493065=629841} 2023-10-09T09:21:42.691 INFO:tasks.rados.rados.0.smithi163.stdout:3854: writing smithi16367185-30 from 454185 to 964903 tid 1 2023-10-09T09:21:42.720 INFO:tasks.rados.rados.0.smithi163.stdout:3854: writing smithi16367185-30 from 1493065 to 2122906 tid 2 2023-10-09T09:21:42.720 INFO:tasks.rados.rados.0.smithi163.stdout:3855: snap_remove snap 482 2023-10-09T09:21:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:42 smithi181 bash[59345]: cluster 2023-10-09T09:21:40.978479+0000 osd.7 (osd.7) 439 : cluster 3 failed to encode map e1026 with expected crc 2023-10-09T09:21:43.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:42 smithi181 bash[59345]: cluster 2023-10-09T09:21:41.357922+0000 osd.5 (osd.5) 568 : cluster 3 failed to encode map e1027 with expected crc 2023-10-09T09:21:43.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:42 smithi181 bash[59345]: cluster 2023-10-09T09:21:41.358449+0000 osd.7 (osd.7) 440 : cluster 3 failed to encode map e1027 with expected crc 2023-10-09T09:21:43.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:42 smithi181 bash[59345]: cluster 2023-10-09T09:21:41.483037+0000 mgr.y (mgr.34107) 454 : cluster 0 pgmap v1094: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2023-10-09T09:21:43.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:42 smithi181 bash[59345]: cluster 2023-10-09T09:21:42.373465+0000 mon.a (mon.0) 1261 : cluster 0 osdmap e1028: 8 total, 8 up, 8 in 2023-10-09T09:21:43.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:42 smithi181 bash[59345]: cluster 2023-10-09T09:21:42.382838+0000 osd.4 (osd.4) 478 : cluster 3 failed to encode map e1028 with expected crc 2023-10-09T09:21:43.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[75425]: cluster 2023-10-09T09:21:40.978479+0000 osd.7 (osd.7) 439 : cluster 3 failed to encode map e1026 with expected crc 2023-10-09T09:21:43.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[75425]: cluster 2023-10-09T09:21:41.357922+0000 osd.5 (osd.5) 568 : cluster 3 failed to encode map e1027 with expected crc 2023-10-09T09:21:43.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[75425]: cluster 2023-10-09T09:21:41.358449+0000 osd.7 (osd.7) 440 : cluster 3 failed to encode map e1027 with expected crc 2023-10-09T09:21:43.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[75425]: cluster 2023-10-09T09:21:41.483037+0000 mgr.y (mgr.34107) 454 : cluster 0 pgmap v1094: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2023-10-09T09:21:43.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[75425]: cluster 2023-10-09T09:21:42.373465+0000 mon.a (mon.0) 1261 : cluster 0 osdmap e1028: 8 total, 8 up, 8 in 2023-10-09T09:21:43.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[75425]: cluster 2023-10-09T09:21:42.382838+0000 osd.4 (osd.4) 478 : cluster 3 failed to encode map e1028 with expected crc 2023-10-09T09:21:43.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[78107]: cluster 2023-10-09T09:21:40.978479+0000 osd.7 (osd.7) 439 : cluster 3 failed to encode map e1026 with expected crc 2023-10-09T09:21:43.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[78107]: cluster 2023-10-09T09:21:41.357922+0000 osd.5 (osd.5) 568 : cluster 3 failed to encode map e1027 with expected crc 2023-10-09T09:21:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[78107]: cluster 2023-10-09T09:21:41.358449+0000 osd.7 (osd.7) 440 : cluster 3 failed to encode map e1027 with expected crc 2023-10-09T09:21:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[78107]: cluster 2023-10-09T09:21:41.483037+0000 mgr.y (mgr.34107) 454 : cluster 0 pgmap v1094: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2023-10-09T09:21:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[78107]: cluster 2023-10-09T09:21:42.373465+0000 mon.a (mon.0) 1261 : cluster 0 osdmap e1028: 8 total, 8 up, 8 in 2023-10-09T09:21:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:42 smithi163 bash[78107]: cluster 2023-10-09T09:21:42.382838+0000 osd.4 (osd.4) 478 : cluster 3 failed to encode map e1028 with expected crc 2023-10-09T09:21:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:3855: done (1 left) 2023-10-09T09:21:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:3856: rollback oid 43 current snap is 483 2023-10-09T09:21:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 467 2023-10-09T09:21:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:3854: finishing write tid 1 to smithi16367185-30 2023-10-09T09:21:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:3854: finishing write tid 2 to smithi16367185-30 2023-10-09T09:21:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:3854: finishing write tid 3 to smithi16367185-30 2023-10-09T09:21:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:3854: finishing write tid 4 to smithi16367185-30 2023-10-09T09:21:43.378 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1356 (ObjNum 1455 snap 483 seq_num 1455) dirty exists 2023-10-09T09:21:43.379 INFO:tasks.rados.rados.0.smithi163.stdout:3854: left oid 30 (ObjNum 1455 snap 483 seq_num 1455) 2023-10-09T09:21:43.379 INFO:tasks.rados.rados.0.smithi163.stdout:3854: done (1 left) 2023-10-09T09:21:43.379 INFO:tasks.rados.rados.0.smithi163.stdout:3857: snap_create 2023-10-09T09:21:43.387 INFO:tasks.rados.rados.0.smithi163.stdout:3856: finishing rollback tid 0 to smithi16367185-43 2023-10-09T09:21:43.387 INFO:tasks.rados.rados.0.smithi163.stdout:3856: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:21:43.387 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 1793 (ObjNum 698717024 snap 21875 seq_num 699103536) dirty dne 2023-10-09T09:21:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:21:43] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.33.4" 2023-10-09T09:21:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[78107]: cluster 2023-10-09T09:21:42.369796+0000 osd.5 (osd.5) 569 : cluster 3 failed to encode map e1028 with expected crc 2023-10-09T09:21:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[78107]: cluster 2023-10-09T09:21:42.378473+0000 osd.7 (osd.7) 441 : cluster 3 failed to encode map e1028 with expected crc 2023-10-09T09:21:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[78107]: cluster 2023-10-09T09:21:42.398538+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e1027 with expected crc 2023-10-09T09:21:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[78107]: cluster 2023-10-09T09:21:43.375524+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e1029: 8 total, 8 up, 8 in 2023-10-09T09:21:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[78107]: cluster 2023-10-09T09:21:43.377253+0000 osd.4 (osd.4) 479 : cluster 3 failed to encode map e1029 with expected crc 2023-10-09T09:21:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[75425]: cluster 2023-10-09T09:21:42.369796+0000 osd.5 (osd.5) 569 : cluster 3 failed to encode map e1028 with expected crc 2023-10-09T09:21:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[75425]: cluster 2023-10-09T09:21:42.378473+0000 osd.7 (osd.7) 441 : cluster 3 failed to encode map e1028 with expected crc 2023-10-09T09:21:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[75425]: cluster 2023-10-09T09:21:42.398538+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e1027 with expected crc 2023-10-09T09:21:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[75425]: cluster 2023-10-09T09:21:43.375524+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e1029: 8 total, 8 up, 8 in 2023-10-09T09:21:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:43 smithi163 bash[75425]: cluster 2023-10-09T09:21:43.377253+0000 osd.4 (osd.4) 479 : cluster 3 failed to encode map e1029 with expected crc 2023-10-09T09:21:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:43 smithi181 bash[59345]: cluster 2023-10-09T09:21:42.369796+0000 osd.5 (osd.5) 569 : cluster 3 failed to encode map e1028 with expected crc 2023-10-09T09:21:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:43 smithi181 bash[59345]: cluster 2023-10-09T09:21:42.378473+0000 osd.7 (osd.7) 441 : cluster 3 failed to encode map e1028 with expected crc 2023-10-09T09:21:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:43 smithi181 bash[59345]: cluster 2023-10-09T09:21:42.398538+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e1027 with expected crc 2023-10-09T09:21:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:43 smithi181 bash[59345]: cluster 2023-10-09T09:21:43.375524+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e1029: 8 total, 8 up, 8 in 2023-10-09T09:21:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:43 smithi181 bash[59345]: cluster 2023-10-09T09:21:43.377253+0000 osd.4 (osd.4) 479 : cluster 3 failed to encode map e1029 with expected crc 2023-10-09T09:21:44.385 INFO:tasks.rados.rados.0.smithi163.stdout:3856: done (1 left) 2023-10-09T09:21:44.385 INFO:tasks.rados.rados.0.smithi163.stdout:3857: done (0 left) 2023-10-09T09:21:44.385 INFO:tasks.rados.rados.0.smithi163.stdout:3858: snap_create 2023-10-09T09:21:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:44 smithi181 bash[59345]: cluster 2023-10-09T09:21:43.370665+0000 osd.5 (osd.5) 570 : cluster 3 failed to encode map e1029 with expected crc 2023-10-09T09:21:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:44 smithi181 bash[59345]: cluster 2023-10-09T09:21:43.396385+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e1029 with expected crc 2023-10-09T09:21:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:44 smithi181 bash[59345]: cluster 2023-10-09T09:21:43.483700+0000 mgr.y (mgr.34107) 455 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2023-10-09T09:21:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:44 smithi181 bash[59345]: cluster 2023-10-09T09:21:44.384988+0000 mon.a (mon.0) 1263 : cluster 0 osdmap e1030: 8 total, 8 up, 8 in 2023-10-09T09:21:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:44 smithi163 bash[78107]: cluster 2023-10-09T09:21:43.370665+0000 osd.5 (osd.5) 570 : cluster 3 failed to encode map e1029 with expected crc 2023-10-09T09:21:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:44 smithi163 bash[78107]: cluster 2023-10-09T09:21:43.396385+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e1029 with expected crc 2023-10-09T09:21:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:44 smithi163 bash[78107]: cluster 2023-10-09T09:21:43.483700+0000 mgr.y (mgr.34107) 455 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2023-10-09T09:21:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:44 smithi163 bash[78107]: cluster 2023-10-09T09:21:44.384988+0000 mon.a (mon.0) 1263 : cluster 0 osdmap e1030: 8 total, 8 up, 8 in 2023-10-09T09:21:44.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:44 smithi163 bash[75425]: cluster 2023-10-09T09:21:43.370665+0000 osd.5 (osd.5) 570 : cluster 3 failed to encode map e1029 with expected crc 2023-10-09T09:21:44.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:44 smithi163 bash[75425]: cluster 2023-10-09T09:21:43.396385+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e1029 with expected crc 2023-10-09T09:21:44.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:44 smithi163 bash[75425]: cluster 2023-10-09T09:21:43.483700+0000 mgr.y (mgr.34107) 455 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2023-10-09T09:21:44.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:44 smithi163 bash[75425]: cluster 2023-10-09T09:21:44.384988+0000 mon.a (mon.0) 1263 : cluster 0 osdmap e1030: 8 total, 8 up, 8 in 2023-10-09T09:21:45.351 INFO:tasks.rados.rados.0.smithi163.stdout:3858: done (0 left) 2023-10-09T09:21:45.352 INFO:tasks.rados.rados.0.smithi163.stdout:3859: write oid 3 current snap is 485 2023-10-09T09:21:45.352 INFO:tasks.rados.rados.0.smithi163.stdout:3859: seq_num 1456 ranges {470992=585967,1551547=773499} 2023-10-09T09:21:45.357 INFO:tasks.rados.rados.0.smithi163.stdout:3859: writing smithi16367185-3 from 470992 to 1056959 tid 1 2023-10-09T09:21:45.366 INFO:tasks.rados.rados.0.smithi163.stdout:3859: writing smithi16367185-3 from 1551547 to 2325046 tid 2 2023-10-09T09:21:45.366 INFO:tasks.rados.rados.0.smithi163.stdout:3860: snap_remove snap 479 2023-10-09T09:21:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:45 smithi181 bash[59345]: cluster 2023-10-09T09:21:44.382254+0000 osd.5 (osd.5) 571 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:45 smithi181 bash[59345]: cluster 2023-10-09T09:21:44.382695+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:45 smithi181 bash[59345]: cluster 2023-10-09T09:21:45.226713+0000 osd.4 (osd.4) 480 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:45 smithi181 bash[59345]: cluster 2023-10-09T09:21:45.351341+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e1031: 8 total, 8 up, 8 in 2023-10-09T09:21:45.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:45 smithi181 bash[59345]: cluster 2023-10-09T09:21:45.362985+0000 osd.4 (osd.4) 481 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:45 smithi163 bash[75425]: cluster 2023-10-09T09:21:44.382254+0000 osd.5 (osd.5) 571 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:45 smithi163 bash[75425]: cluster 2023-10-09T09:21:44.382695+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:45 smithi163 bash[75425]: cluster 2023-10-09T09:21:45.226713+0000 osd.4 (osd.4) 480 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:45 smithi163 bash[75425]: cluster 2023-10-09T09:21:45.351341+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e1031: 8 total, 8 up, 8 in 2023-10-09T09:21:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:45 smithi163 bash[75425]: cluster 2023-10-09T09:21:45.362985+0000 osd.4 (osd.4) 481 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:45 smithi163 bash[78107]: cluster 2023-10-09T09:21:44.382254+0000 osd.5 (osd.5) 571 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:45 smithi163 bash[78107]: cluster 2023-10-09T09:21:44.382695+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:45 smithi163 bash[78107]: cluster 2023-10-09T09:21:45.226713+0000 osd.4 (osd.4) 480 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:45 smithi163 bash[78107]: cluster 2023-10-09T09:21:45.351341+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e1031: 8 total, 8 up, 8 in 2023-10-09T09:21:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:45 smithi163 bash[78107]: cluster 2023-10-09T09:21:45.362985+0000 osd.4 (osd.4) 481 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:46.366 INFO:tasks.rados.rados.0.smithi163.stdout:3860: done (1 left) 2023-10-09T09:21:46.367 INFO:tasks.rados.rados.0.smithi163.stdout:3861: snap_remove snap 409 2023-10-09T09:21:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:46 smithi181 bash[59345]: cluster 2023-10-09T09:21:44.903499+0000 osd.7 (osd.7) 442 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:46 smithi181 bash[59345]: cluster 2023-10-09T09:21:45.346015+0000 osd.5 (osd.5) 572 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:46 smithi181 bash[59345]: cluster 2023-10-09T09:21:45.381004+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:46 smithi181 bash[59345]: cluster 2023-10-09T09:21:45.416386+0000 osd.7 (osd.7) 443 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:46 smithi181 bash[59345]: cluster 2023-10-09T09:21:45.484320+0000 mgr.y (mgr.34107) 456 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 724 KiB/s wr, 7 op/s 2023-10-09T09:21:46.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:46 smithi181 bash[59345]: cluster 2023-10-09T09:21:46.366204+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e1032: 8 total, 8 up, 8 in 2023-10-09T09:21:46.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[75425]: cluster 2023-10-09T09:21:44.903499+0000 osd.7 (osd.7) 442 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:46.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[75425]: cluster 2023-10-09T09:21:45.346015+0000 osd.5 (osd.5) 572 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:46.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[75425]: cluster 2023-10-09T09:21:45.381004+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:46.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[75425]: cluster 2023-10-09T09:21:45.416386+0000 osd.7 (osd.7) 443 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:46.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[75425]: cluster 2023-10-09T09:21:45.484320+0000 mgr.y (mgr.34107) 456 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 724 KiB/s wr, 7 op/s 2023-10-09T09:21:46.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[75425]: cluster 2023-10-09T09:21:46.366204+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e1032: 8 total, 8 up, 8 in 2023-10-09T09:21:46.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[78107]: cluster 2023-10-09T09:21:44.903499+0000 osd.7 (osd.7) 442 : cluster 3 failed to encode map e1030 with expected crc 2023-10-09T09:21:46.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[78107]: cluster 2023-10-09T09:21:45.346015+0000 osd.5 (osd.5) 572 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:46.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[78107]: cluster 2023-10-09T09:21:45.381004+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:46.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[78107]: cluster 2023-10-09T09:21:45.416386+0000 osd.7 (osd.7) 443 : cluster 3 failed to encode map e1031 with expected crc 2023-10-09T09:21:46.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[78107]: cluster 2023-10-09T09:21:45.484320+0000 mgr.y (mgr.34107) 456 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 724 KiB/s wr, 7 op/s 2023-10-09T09:21:46.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:46 smithi163 bash[78107]: cluster 2023-10-09T09:21:46.366204+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e1032: 8 total, 8 up, 8 in 2023-10-09T09:21:47.372 INFO:tasks.rados.rados.0.smithi163.stdout:3861: done (1 left) 2023-10-09T09:21:47.372 INFO:tasks.rados.rados.0.smithi163.stdout:3862: delete oid 45 current snap is 485 2023-10-09T09:21:47.372 INFO:tasks.rados.rados.0.smithi163.stdout:3859: finishing write tid 1 to smithi16367185-3 2023-10-09T09:21:47.372 INFO:tasks.rados.rados.0.smithi163.stdout:3859: finishing write tid 2 to smithi16367185-3 2023-10-09T09:21:47.372 INFO:tasks.rados.rados.0.smithi163.stdout:3859: finishing write tid 3 to smithi16367185-3 2023-10-09T09:21:47.373 INFO:tasks.rados.rados.0.smithi163.stdout:3859: finishing write tid 4 to smithi16367185-3 2023-10-09T09:21:47.373 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 1164 (ObjNum 1456 snap 485 seq_num 1456) dirty exists 2023-10-09T09:21:47.373 INFO:tasks.rados.rados.0.smithi163.stdout:3859: left oid 3 (ObjNum 1456 snap 485 seq_num 1456) 2023-10-09T09:21:47.389 INFO:tasks.rados.rados.0.smithi163.stdout:3859: done (1 left) 2023-10-09T09:21:47.389 INFO:tasks.rados.rados.0.smithi163.stdout:3862: done (0 left) 2023-10-09T09:21:47.389 INFO:tasks.rados.rados.0.smithi163.stdout:3863: rollback oid 39 current snap is 485 2023-10-09T09:21:47.389 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 39 to 467 2023-10-09T09:21:47.390 INFO:tasks.rados.rados.0.smithi163.stdout:3864: write oid 50 current snap is 485 2023-10-09T09:21:47.390 INFO:tasks.rados.rados.0.smithi163.stdout:3864: seq_num 1458 ranges {504606=736465,1952158=552448} 2023-10-09T09:21:47.397 INFO:tasks.rados.rados.0.smithi163.stdout:3864: writing smithi16367185-50 from 504606 to 1241071 tid 1 2023-10-09T09:21:47.405 INFO:tasks.rados.rados.0.smithi163.stdout:3864: writing smithi16367185-50 from 1952158 to 2504606 tid 2 2023-10-09T09:21:47.405 INFO:tasks.rados.rados.0.smithi163.stdout:3865: write oid 37 current snap is 485 2023-10-09T09:21:47.406 INFO:tasks.rados.rados.0.smithi163.stdout:3865: seq_num 1459 ranges {521413=411714,1610640=637179} 2023-10-09T09:21:47.411 INFO:tasks.rados.rados.0.smithi163.stdout:3865: writing smithi16367185-37 from 521413 to 933127 tid 1 2023-10-09T09:21:47.420 INFO:tasks.rados.rados.0.smithi163.stdout:3865: writing smithi16367185-37 from 1610640 to 2247819 tid 2 2023-10-09T09:21:47.420 INFO:tasks.rados.rados.0.smithi163.stdout:3866: snap_remove snap 477 2023-10-09T09:21:47.420 INFO:tasks.rados.rados.0.smithi163.stdout:3863: finishing rollback tid 0 to smithi16367185-39 2023-10-09T09:21:47.420 INFO:tasks.rados.rados.0.smithi163.stdout:3863: finishing rollback tid 1 to smithi16367185-39 2023-10-09T09:21:47.420 INFO:tasks.rados.rados.0.smithi163.stdout:3863: finishing rollback tid 2 to smithi16367185-39 2023-10-09T09:21:47.420 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 39 v 1782 (ObjNum 1310 snap 443 seq_num 1310) dirty exists 2023-10-09T09:21:47.420 INFO:tasks.rados.rados.0.smithi163.stdout:3864: finishing write tid 1 to smithi16367185-50 2023-10-09T09:21:47.420 INFO:tasks.rados.rados.0.smithi163.stdout:3864: finishing write tid 2 to smithi16367185-50 2023-10-09T09:21:47.420 INFO:tasks.rados.rados.0.smithi163.stdout:3864: finishing write tid 3 to smithi16367185-50 2023-10-09T09:21:47.421 INFO:tasks.rados.rados.0.smithi163.stdout:3865: finishing write tid 1 to smithi16367185-37 2023-10-09T09:21:47.421 INFO:tasks.rados.rados.0.smithi163.stdout:3864: finishing write tid 4 to smithi16367185-50 2023-10-09T09:21:47.421 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 50 v 1786 (ObjNum 1458 snap 485 seq_num 1458) dirty exists 2023-10-09T09:21:47.421 INFO:tasks.rados.rados.0.smithi163.stdout:3864: left oid 50 (ObjNum 1458 snap 485 seq_num 1458) 2023-10-09T09:21:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:47 smithi181 bash[59345]: cluster 2023-10-09T09:21:46.361495+0000 osd.5 (osd.5) 573 : cluster 3 failed to encode map e1032 with expected crc 2023-10-09T09:21:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:47 smithi181 bash[59345]: cluster 2023-10-09T09:21:46.376247+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e1032 with expected crc 2023-10-09T09:21:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:47 smithi181 bash[59345]: cluster 2023-10-09T09:21:46.914178+0000 osd.7 (osd.7) 444 : cluster 3 failed to encode map e1032 with expected crc 2023-10-09T09:21:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:47 smithi181 bash[59345]: cluster 2023-10-09T09:21:47.371230+0000 mon.a (mon.0) 1266 : cluster 0 osdmap e1033: 8 total, 8 up, 8 in 2023-10-09T09:21:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:47 smithi181 bash[59345]: cluster 2023-10-09T09:21:47.371575+0000 osd.4 (osd.4) 482 : cluster 3 failed to encode map e1033 with expected crc 2023-10-09T09:21:47.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:47 smithi163 bash[78107]: cluster 2023-10-09T09:21:46.361495+0000 osd.5 (osd.5) 573 : cluster 3 failed to encode map e1032 with expected crc 2023-10-09T09:21:47.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:47 smithi163 bash[78107]: cluster 2023-10-09T09:21:46.376247+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e1032 with expected crc 2023-10-09T09:21:47.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:47 smithi163 bash[78107]: cluster 2023-10-09T09:21:46.914178+0000 osd.7 (osd.7) 444 : cluster 3 failed to encode map e1032 with expected crc 2023-10-09T09:21:47.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:47 smithi163 bash[78107]: cluster 2023-10-09T09:21:47.371230+0000 mon.a (mon.0) 1266 : cluster 0 osdmap e1033: 8 total, 8 up, 8 in 2023-10-09T09:21:47.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:47 smithi163 bash[78107]: cluster 2023-10-09T09:21:47.371575+0000 osd.4 (osd.4) 482 : cluster 3 failed to encode map e1033 with expected crc 2023-10-09T09:21:47.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:47 smithi163 bash[75425]: cluster 2023-10-09T09:21:46.361495+0000 osd.5 (osd.5) 573 : cluster 3 failed to encode map e1032 with expected crc 2023-10-09T09:21:47.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:47 smithi163 bash[75425]: cluster 2023-10-09T09:21:46.376247+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e1032 with expected crc 2023-10-09T09:21:47.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:47 smithi163 bash[75425]: cluster 2023-10-09T09:21:46.914178+0000 osd.7 (osd.7) 444 : cluster 3 failed to encode map e1032 with expected crc 2023-10-09T09:21:47.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:47 smithi163 bash[75425]: cluster 2023-10-09T09:21:47.371230+0000 mon.a (mon.0) 1266 : cluster 0 osdmap e1033: 8 total, 8 up, 8 in 2023-10-09T09:21:47.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:47 smithi163 bash[75425]: cluster 2023-10-09T09:21:47.371575+0000 osd.4 (osd.4) 482 : cluster 3 failed to encode map e1033 with expected crc 2023-10-09T09:21:48.379 INFO:tasks.rados.rados.0.smithi163.stdout:3863: done (3 left) 2023-10-09T09:21:48.380 INFO:tasks.rados.rados.0.smithi163.stdout:3864: done (2 left) 2023-10-09T09:21:48.380 INFO:tasks.rados.rados.0.smithi163.stdout:3866: done (1 left) 2023-10-09T09:21:48.380 INFO:tasks.rados.rados.0.smithi163.stdout:3867: snap_create 2023-10-09T09:21:48.380 INFO:tasks.rados.rados.0.smithi163.stdout:3865: finishing write tid 2 to smithi16367185-37 2023-10-09T09:21:48.380 INFO:tasks.rados.rados.0.smithi163.stdout:3865: finishing write tid 3 to smithi16367185-37 2023-10-09T09:21:48.380 INFO:tasks.rados.rados.0.smithi163.stdout:3865: finishing write tid 4 to smithi16367185-37 2023-10-09T09:21:48.380 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 37 v 1998 (ObjNum 1459 snap 485 seq_num 1459) dirty exists 2023-10-09T09:21:48.380 INFO:tasks.rados.rados.0.smithi163.stdout:3865: left oid 37 (ObjNum 1459 snap 485 seq_num 1459) 2023-10-09T09:21:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:48 smithi181 bash[59345]: cluster 2023-10-09T09:21:47.366167+0000 osd.5 (osd.5) 574 : cluster 3 failed to encode map e1033 with expected crc 2023-10-09T09:21:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:48 smithi181 bash[59345]: cluster 2023-10-09T09:21:47.377538+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e1033 with expected crc 2023-10-09T09:21:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:48 smithi181 bash[59345]: cluster 2023-10-09T09:21:47.484997+0000 mgr.y (mgr.34107) 457 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:21:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:48 smithi181 bash[59345]: cluster 2023-10-09T09:21:48.374407+0000 osd.4 (osd.4) 483 : cluster 3 failed to encode map e1034 with expected crc 2023-10-09T09:21:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:48 smithi181 bash[59345]: cluster 2023-10-09T09:21:48.379689+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e1034: 8 total, 8 up, 8 in 2023-10-09T09:21:48.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:48 smithi163 bash[75425]: cluster 2023-10-09T09:21:47.366167+0000 osd.5 (osd.5) 574 : cluster 3 failed to encode map e1033 with expected crc 2023-10-09T09:21:48.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:48 smithi163 bash[75425]: cluster 2023-10-09T09:21:47.377538+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e1033 with expected crc 2023-10-09T09:21:48.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:48 smithi163 bash[75425]: cluster 2023-10-09T09:21:47.484997+0000 mgr.y (mgr.34107) 457 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:21:48.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:48 smithi163 bash[75425]: cluster 2023-10-09T09:21:48.374407+0000 osd.4 (osd.4) 483 : cluster 3 failed to encode map e1034 with expected crc 2023-10-09T09:21:48.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:48 smithi163 bash[75425]: cluster 2023-10-09T09:21:48.379689+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e1034: 8 total, 8 up, 8 in 2023-10-09T09:21:48.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:48 smithi163 bash[78107]: cluster 2023-10-09T09:21:47.366167+0000 osd.5 (osd.5) 574 : cluster 3 failed to encode map e1033 with expected crc 2023-10-09T09:21:48.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:48 smithi163 bash[78107]: cluster 2023-10-09T09:21:47.377538+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e1033 with expected crc 2023-10-09T09:21:48.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:48 smithi163 bash[78107]: cluster 2023-10-09T09:21:47.484997+0000 mgr.y (mgr.34107) 457 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2023-10-09T09:21:48.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:48 smithi163 bash[78107]: cluster 2023-10-09T09:21:48.374407+0000 osd.4 (osd.4) 483 : cluster 3 failed to encode map e1034 with expected crc 2023-10-09T09:21:48.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:48 smithi163 bash[78107]: cluster 2023-10-09T09:21:48.379689+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e1034: 8 total, 8 up, 8 in 2023-10-09T09:21:49.384 INFO:tasks.rados.rados.0.smithi163.stdout:3865: done (1 left) 2023-10-09T09:21:49.384 INFO:tasks.rados.rados.0.smithi163.stdout:3867: done (0 left) 2023-10-09T09:21:49.384 INFO:tasks.rados.rados.0.smithi163.stdout:3868: snap_create 2023-10-09T09:21:49.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:49 smithi181 bash[59345]: cluster 2023-10-09T09:21:48.377029+0000 osd.5 (osd.5) 575 : cluster 3 failed to encode map e1034 with expected crc 2023-10-09T09:21:49.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:49 smithi181 bash[59345]: cluster 2023-10-09T09:21:49.383435+0000 mon.a (mon.0) 1268 : cluster 0 osdmap e1035: 8 total, 8 up, 8 in 2023-10-09T09:21:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:49 smithi163 bash[78107]: cluster 2023-10-09T09:21:48.377029+0000 osd.5 (osd.5) 575 : cluster 3 failed to encode map e1034 with expected crc 2023-10-09T09:21:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:49 smithi163 bash[78107]: cluster 2023-10-09T09:21:49.383435+0000 mon.a (mon.0) 1268 : cluster 0 osdmap e1035: 8 total, 8 up, 8 in 2023-10-09T09:21:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:49 smithi163 bash[75425]: cluster 2023-10-09T09:21:48.377029+0000 osd.5 (osd.5) 575 : cluster 3 failed to encode map e1034 with expected crc 2023-10-09T09:21:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:49 smithi163 bash[75425]: cluster 2023-10-09T09:21:49.383435+0000 mon.a (mon.0) 1268 : cluster 0 osdmap e1035: 8 total, 8 up, 8 in 2023-10-09T09:21:50.354 INFO:tasks.rados.rados.0.smithi163.stdout:3868: done (0 left) 2023-10-09T09:21:50.354 INFO:tasks.rados.rados.0.smithi163.stdout:3869: snap_create 2023-10-09T09:21:50.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:50 smithi181 bash[59345]: cluster 2023-10-09T09:21:49.376945+0000 osd.5 (osd.5) 576 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:50 smithi181 bash[59345]: cluster 2023-10-09T09:21:49.485687+0000 mgr.y (mgr.34107) 458 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 241 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 551 KiB/s wr, 3 op/s 2023-10-09T09:21:50.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:50 smithi181 bash[59345]: cluster 2023-10-09T09:21:49.516211+0000 osd.4 (osd.4) 484 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:50 smithi181 bash[59345]: cluster 2023-10-09T09:21:49.517161+0000 osd.7 (osd.7) 445 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:50 smithi181 bash[59345]: cluster 2023-10-09T09:21:49.517192+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:50 smithi181 bash[59345]: cluster 2023-10-09T09:21:50.353805+0000 mon.a (mon.0) 1269 : cluster 0 osdmap e1036: 8 total, 8 up, 8 in 2023-10-09T09:21:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[78107]: cluster 2023-10-09T09:21:49.376945+0000 osd.5 (osd.5) 576 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[78107]: cluster 2023-10-09T09:21:49.485687+0000 mgr.y (mgr.34107) 458 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 241 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 551 KiB/s wr, 3 op/s 2023-10-09T09:21:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[78107]: cluster 2023-10-09T09:21:49.516211+0000 osd.4 (osd.4) 484 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[78107]: cluster 2023-10-09T09:21:49.517161+0000 osd.7 (osd.7) 445 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[78107]: cluster 2023-10-09T09:21:49.517192+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[78107]: cluster 2023-10-09T09:21:50.353805+0000 mon.a (mon.0) 1269 : cluster 0 osdmap e1036: 8 total, 8 up, 8 in 2023-10-09T09:21:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[75425]: cluster 2023-10-09T09:21:49.376945+0000 osd.5 (osd.5) 576 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[75425]: cluster 2023-10-09T09:21:49.485687+0000 mgr.y (mgr.34107) 458 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 241 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 551 KiB/s wr, 3 op/s 2023-10-09T09:21:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[75425]: cluster 2023-10-09T09:21:49.516211+0000 osd.4 (osd.4) 484 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[75425]: cluster 2023-10-09T09:21:49.517161+0000 osd.7 (osd.7) 445 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[75425]: cluster 2023-10-09T09:21:49.517192+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e1035 with expected crc 2023-10-09T09:21:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:50 smithi163 bash[75425]: cluster 2023-10-09T09:21:50.353805+0000 mon.a (mon.0) 1269 : cluster 0 osdmap e1036: 8 total, 8 up, 8 in 2023-10-09T09:21:51.356 INFO:tasks.rados.rados.0.smithi163.stdout:3869: done (0 left) 2023-10-09T09:21:51.356 INFO:tasks.rados.rados.0.smithi163.stdout:3870: write oid 5 current snap is 488 2023-10-09T09:21:51.356 INFO:tasks.rados.rados.0.smithi163.stdout:3870: seq_num 1460 ranges {538220=603316,1785475=752745} 2023-10-09T09:21:51.365 INFO:tasks.rados.rados.0.smithi163.stdout:3870: writing smithi16367185-5 from 538220 to 1141536 tid 1 2023-10-09T09:21:51.374 INFO:tasks.rados.rados.0.smithi163.stdout:3870: writing smithi16367185-5 from 1785475 to 2538220 tid 2 2023-10-09T09:21:51.374 INFO:tasks.rados.rados.0.smithi163.stdout:3871: write oid 16 current snap is 488 2023-10-09T09:21:51.374 INFO:tasks.rados.rados.0.smithi163.stdout:3870: finishing write tid 1 to smithi16367185-5 2023-10-09T09:21:51.374 INFO:tasks.rados.rados.0.smithi163.stdout:3871: seq_num 1461 ranges {555027=678565,1727604=640848} 2023-10-09T09:21:51.381 INFO:tasks.rados.rados.0.smithi163.stdout:3871: writing smithi16367185-16 from 555027 to 1233592 tid 1 2023-10-09T09:21:51.388 INFO:tasks.rados.rados.0.smithi163.stdout:3871: writing smithi16367185-16 from 1727604 to 2368452 tid 2 2023-10-09T09:21:51.388 INFO:tasks.rados.rados.0.smithi163.stdout:3872: write oid 46 current snap is 488 2023-10-09T09:21:51.388 INFO:tasks.rados.rados.0.smithi163.stdout:3872: seq_num 1462 ranges {571834=753814,1786086=784506} 2023-10-09T09:21:51.396 INFO:tasks.rados.rados.0.smithi163.stdout:3872: writing smithi16367185-46 from 571834 to 1325648 tid 1 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:3872: writing smithi16367185-46 from 1786086 to 2570592 tid 2 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:3870: finishing write tid 2 to smithi16367185-5 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:3870: finishing write tid 3 to smithi16367185-5 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:3870: finishing write tid 4 to smithi16367185-5 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 2157 (ObjNum 1460 snap 488 seq_num 1460) dirty exists 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:3870: left oid 5 (ObjNum 1460 snap 488 seq_num 1460) 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:3873: read oid 38 snap -1 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:3873: expect deleted 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:3870: done (3 left) 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:3874: delete oid 6 current snap is 488 2023-10-09T09:21:51.404 INFO:tasks.rados.rados.0.smithi163.stdout:3871: finishing write tid 1 to smithi16367185-16 2023-10-09T09:21:51.405 INFO:tasks.rados.rados.0.smithi163.stdout:3871: finishing write tid 2 to smithi16367185-16 2023-10-09T09:21:51.405 INFO:tasks.rados.rados.0.smithi163.stdout:3871: finishing write tid 3 to smithi16367185-16 2023-10-09T09:21:51.405 INFO:tasks.rados.rados.0.smithi163.stdout:3871: finishing write tid 4 to smithi16367185-16 2023-10-09T09:21:51.405 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 16 v 2161 (ObjNum 1461 snap 488 seq_num 1461) dirty exists 2023-10-09T09:21:51.405 INFO:tasks.rados.rados.0.smithi163.stdout:3871: left oid 16 (ObjNum 1461 snap 488 seq_num 1461) 2023-10-09T09:21:51.405 INFO:tasks.rados.rados.0.smithi163.stdout:3872: finishing write tid 1 to smithi16367185-46 2023-10-09T09:21:51.411 INFO:tasks.rados.rados.0.smithi163.stdout:3871: done (3 left) 2023-10-09T09:21:51.411 INFO:tasks.rados.rados.0.smithi163.stdout:3874: done (2 left) 2023-10-09T09:21:51.411 INFO:tasks.rados.rados.0.smithi163.stdout:3875: read oid 3 snap -1 2023-10-09T09:21:51.411 INFO:tasks.rados.rados.0.smithi163.stdout:3875: expect (ObjNum 1456 snap 485 seq_num 1456) 2023-10-09T09:21:51.411 INFO:tasks.rados.rados.0.smithi163.stdout:3876: write oid 15 current snap is 488 2023-10-09T09:21:51.411 INFO:tasks.rados.rados.0.smithi163.stdout:3876: seq_num 1464 ranges {605448=504312,1903050=702398} 2023-10-09T09:21:51.417 INFO:tasks.rados.rados.0.smithi163.stdout:3876: writing smithi16367185-15 from 605448 to 1109760 tid 1 2023-10-09T09:21:51.427 INFO:tasks.rados.rados.0.smithi163.stdout:3876: writing smithi16367185-15 from 1903050 to 2605448 tid 2 2023-10-09T09:21:51.427 INFO:tasks.rados.rados.0.smithi163.stdout:3877: delete oid 5 current snap is 488 2023-10-09T09:21:51.427 INFO:tasks.rados.rados.0.smithi163.stdout:3872: finishing write tid 2 to smithi16367185-46 2023-10-09T09:21:51.427 INFO:tasks.rados.rados.0.smithi163.stdout:3872: finishing write tid 3 to smithi16367185-46 2023-10-09T09:21:51.427 INFO:tasks.rados.rados.0.smithi163.stdout:3872: finishing write tid 4 to smithi16367185-46 2023-10-09T09:21:51.427 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 46 v 1804 (ObjNum 1462 snap 488 seq_num 1462) dirty exists 2023-10-09T09:21:51.428 INFO:tasks.rados.rados.0.smithi163.stdout:3872: left oid 46 (ObjNum 1462 snap 488 seq_num 1462) 2023-10-09T09:21:51.428 INFO:tasks.rados.rados.0.smithi163.stdout:3875: expect (ObjNum 1456 snap 485 seq_num 1456) 2023-10-09T09:21:51.507 INFO:tasks.rados.rados.0.smithi163.stdout:3876: finishing write tid 1 to smithi16367185-15 2023-10-09T09:21:51.507 INFO:tasks.rados.rados.0.smithi163.stdout:3876: finishing write tid 2 to smithi16367185-15 2023-10-09T09:21:51.507 INFO:tasks.rados.rados.0.smithi163.stdout:3876: finishing write tid 3 to smithi16367185-15 2023-10-09T09:21:51.507 INFO:tasks.rados.rados.0.smithi163.stdout:3876: finishing write tid 4 to smithi16367185-15 2023-10-09T09:21:51.508 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 2006 (ObjNum 1464 snap 488 seq_num 1464) dirty exists 2023-10-09T09:21:51.508 INFO:tasks.rados.rados.0.smithi163.stdout:3876: left oid 15 (ObjNum 1464 snap 488 seq_num 1464) 2023-10-09T09:21:51.508 INFO:tasks.rados.rados.0.smithi163.stdout:3872: done (4 left) 2023-10-09T09:21:51.508 INFO:tasks.rados.rados.0.smithi163.stdout:3873: done (3 left) 2023-10-09T09:21:51.508 INFO:tasks.rados.rados.0.smithi163.stdout:3875: done (2 left) 2023-10-09T09:21:51.508 INFO:tasks.rados.rados.0.smithi163.stdout:3876: done (1 left) 2023-10-09T09:21:51.508 INFO:tasks.rados.rados.0.smithi163.stdout:3877: done (0 left) 2023-10-09T09:21:51.508 INFO:tasks.rados.rados.0.smithi163.stdout:3878: write oid 47 current snap is 488 2023-10-09T09:21:51.508 INFO:tasks.rados.rados.0.smithi163.stdout:3878: seq_num 1466 ranges {639062=654810,1903661=735401} 2023-10-09T09:21:51.521 INFO:tasks.rados.rados.0.smithi163.stdout:3878: writing smithi16367185-47 from 639062 to 1293872 tid 1 2023-10-09T09:21:51.529 INFO:tasks.rados.rados.0.smithi163.stdout:3878: writing smithi16367185-47 from 1903661 to 2639062 tid 2 2023-10-09T09:21:51.530 INFO:tasks.rados.rados.0.smithi163.stdout:3879: read oid 49 snap -1 2023-10-09T09:21:51.530 INFO:tasks.rados.rados.0.smithi163.stdout:3879: expect (ObjNum 1283 snap 437 seq_num 1283) 2023-10-09T09:21:51.530 INFO:tasks.rados.rados.0.smithi163.stdout:3880: write oid 23 current snap is 488 2023-10-09T09:21:51.530 INFO:tasks.rados.rados.0.smithi163.stdout:3878: finishing write tid 1 to smithi16367185-47 2023-10-09T09:21:51.530 INFO:tasks.rados.rados.0.smithi163.stdout:3880: seq_num 1467 ranges {655869=730059,1962143=535502} 2023-10-09T09:21:51.538 INFO:tasks.rados.rados.0.smithi163.stdout:3880: writing smithi16367185-23 from 655869 to 1385928 tid 1 2023-10-09T09:21:51.544 INFO:tasks.rados.rados.0.smithi163.stdout:3880: writing smithi16367185-23 from 1962143 to 2497645 tid 2 2023-10-09T09:21:51.544 INFO:tasks.rados.rados.0.smithi163.stdout:3881: write oid 44 current snap is 488 2023-10-09T09:21:51.544 INFO:tasks.rados.rados.0.smithi163.stdout:3881: seq_num 1468 ranges {672676=521661,1736978=795513} 2023-10-09T09:21:51.551 INFO:tasks.rados.rados.0.smithi163.stdout:3881: writing smithi16367185-44 from 672676 to 1194337 tid 1 2023-10-09T09:21:51.559 INFO:tasks.rados.rados.0.smithi163.stdout:3881: writing smithi16367185-44 from 1736978 to 2532491 tid 2 2023-10-09T09:21:51.559 INFO:tasks.rados.rados.0.smithi163.stdout:3882: snap_create 2023-10-09T09:21:51.559 INFO:tasks.rados.rados.0.smithi163.stdout:3879: expect (ObjNum 1283 snap 437 seq_num 1283) 2023-10-09T09:21:51.750 INFO:tasks.rados.rados.0.smithi163.stdout:3878: finishing write tid 2 to smithi16367185-47 2023-10-09T09:21:51.750 INFO:tasks.rados.rados.0.smithi163.stdout:3878: finishing write tid 3 to smithi16367185-47 2023-10-09T09:21:51.750 INFO:tasks.rados.rados.0.smithi163.stdout:3878: finishing write tid 4 to smithi16367185-47 2023-10-09T09:21:51.750 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 47 v 2982 (ObjNum 1466 snap 488 seq_num 1466) dirty exists 2023-10-09T09:21:51.750 INFO:tasks.rados.rados.0.smithi163.stdout:3878: left oid 47 (ObjNum 1466 snap 488 seq_num 1466) 2023-10-09T09:21:51.750 INFO:tasks.rados.rados.0.smithi163.stdout:3880: finishing write tid 1 to smithi16367185-23 2023-10-09T09:21:51.750 INFO:tasks.rados.rados.0.smithi163.stdout:3880: finishing write tid 2 to smithi16367185-23 2023-10-09T09:21:51.750 INFO:tasks.rados.rados.0.smithi163.stdout:3880: finishing write tid 3 to smithi16367185-23 2023-10-09T09:21:51.750 INFO:tasks.rados.rados.0.smithi163.stdout:3880: finishing write tid 4 to smithi16367185-23 2023-10-09T09:21:51.751 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1794 (ObjNum 1467 snap 488 seq_num 1467) dirty exists 2023-10-09T09:21:51.751 INFO:tasks.rados.rados.0.smithi163.stdout:3880: left oid 23 (ObjNum 1467 snap 488 seq_num 1467) 2023-10-09T09:21:51.751 INFO:tasks.rados.rados.0.smithi163.stdout:3881: finishing write tid 1 to smithi16367185-44 2023-10-09T09:21:51.751 INFO:tasks.rados.rados.0.smithi163.stdout:3881: finishing write tid 2 to smithi16367185-44 2023-10-09T09:21:51.751 INFO:tasks.rados.rados.0.smithi163.stdout:3881: finishing write tid 3 to smithi16367185-44 2023-10-09T09:21:51.751 INFO:tasks.rados.rados.0.smithi163.stdout:3881: finishing write tid 4 to smithi16367185-44 2023-10-09T09:21:51.751 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 2986 (ObjNum 1468 snap 488 seq_num 1468) dirty exists 2023-10-09T09:21:51.751 INFO:tasks.rados.rados.0.smithi163.stdout:3881: left oid 44 (ObjNum 1468 snap 488 seq_num 1468) 2023-10-09T09:21:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:51 smithi181 bash[59345]: cluster 2023-10-09T09:21:50.348324+0000 osd.5 (osd.5) 577 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:51 smithi181 bash[59345]: cluster 2023-10-09T09:21:50.458092+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:51 smithi181 bash[59345]: cluster 2023-10-09T09:21:50.767456+0000 osd.7 (osd.7) 446 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:51 smithi181 bash[59345]: cluster 2023-10-09T09:21:50.978836+0000 osd.4 (osd.4) 485 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:51 smithi181 bash[59345]: cluster 2023-10-09T09:21:51.355577+0000 mon.a (mon.0) 1270 : cluster 0 osdmap e1037: 8 total, 8 up, 8 in 2023-10-09T09:21:51.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:51 smithi181 bash[59345]: cluster 2023-10-09T09:21:51.369663+0000 osd.4 (osd.4) 486 : cluster 3 failed to encode map e1037 with expected crc 2023-10-09T09:21:51.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[78107]: cluster 2023-10-09T09:21:50.348324+0000 osd.5 (osd.5) 577 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[78107]: cluster 2023-10-09T09:21:50.458092+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[78107]: cluster 2023-10-09T09:21:50.767456+0000 osd.7 (osd.7) 446 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[78107]: cluster 2023-10-09T09:21:50.978836+0000 osd.4 (osd.4) 485 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[78107]: cluster 2023-10-09T09:21:51.355577+0000 mon.a (mon.0) 1270 : cluster 0 osdmap e1037: 8 total, 8 up, 8 in 2023-10-09T09:21:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[78107]: cluster 2023-10-09T09:21:51.369663+0000 osd.4 (osd.4) 486 : cluster 3 failed to encode map e1037 with expected crc 2023-10-09T09:21:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[75425]: cluster 2023-10-09T09:21:50.348324+0000 osd.5 (osd.5) 577 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[75425]: cluster 2023-10-09T09:21:50.458092+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[75425]: cluster 2023-10-09T09:21:50.767456+0000 osd.7 (osd.7) 446 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[75425]: cluster 2023-10-09T09:21:50.978836+0000 osd.4 (osd.4) 485 : cluster 3 failed to encode map e1036 with expected crc 2023-10-09T09:21:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[75425]: cluster 2023-10-09T09:21:51.355577+0000 mon.a (mon.0) 1270 : cluster 0 osdmap e1037: 8 total, 8 up, 8 in 2023-10-09T09:21:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:51 smithi163 bash[75425]: cluster 2023-10-09T09:21:51.369663+0000 osd.4 (osd.4) 486 : cluster 3 failed to encode map e1037 with expected crc 2023-10-09T09:21:52.522 INFO:tasks.rados.rados.0.smithi163.stdout:3878: done (4 left) 2023-10-09T09:21:52.522 INFO:tasks.rados.rados.0.smithi163.stdout:3879: done (3 left) 2023-10-09T09:21:52.522 INFO:tasks.rados.rados.0.smithi163.stdout:3880: done (2 left) 2023-10-09T09:21:52.523 INFO:tasks.rados.rados.0.smithi163.stdout:3881: done (1 left) 2023-10-09T09:21:52.523 INFO:tasks.rados.rados.0.smithi163.stdout:3882: done (0 left) 2023-10-09T09:21:52.523 INFO:tasks.rados.rados.0.smithi163.stdout:3883: read oid 50 snap -1 2023-10-09T09:21:52.523 INFO:tasks.rados.rados.0.smithi163.stdout:3883: expect (ObjNum 1458 snap 485 seq_num 1458) 2023-10-09T09:21:52.523 INFO:tasks.rados.rados.0.smithi163.stdout:3884: write oid 13 current snap is 489 2023-10-09T09:21:52.523 INFO:tasks.rados.rados.0.smithi163.stdout:3884: seq_num 1469 ranges {689483=596910,2079107=539171} 2023-10-09T09:21:52.531 INFO:tasks.rados.rados.0.smithi163.stdout:3884: writing smithi16367185-13 from 689483 to 1286393 tid 1 2023-10-09T09:21:52.540 INFO:tasks.rados.rados.0.smithi163.stdout:3884: writing smithi16367185-13 from 2079107 to 2618278 tid 2 2023-10-09T09:21:52.540 INFO:tasks.rados.rados.0.smithi163.stdout:3885: read oid 30 snap 480 2023-10-09T09:21:52.540 INFO:tasks.rados.rados.0.smithi163.stdout:3885: expect (ObjNum 1413 snap 471 seq_num 1413) 2023-10-09T09:21:52.540 INFO:tasks.rados.rados.0.smithi163.stdout:3886: read oid 5 snap -1 2023-10-09T09:21:52.540 INFO:tasks.rados.rados.0.smithi163.stdout:3886: expect deleted 2023-10-09T09:21:52.540 INFO:tasks.rados.rados.0.smithi163.stdout:3887: delete oid 12 current snap is 489 2023-10-09T09:21:52.540 INFO:tasks.rados.rados.0.smithi163.stdout:3883: expect (ObjNum 1458 snap 485 seq_num 1458) 2023-10-09T09:21:52.660 INFO:tasks.rados.rados.0.smithi163.stdout:3884: finishing write tid 1 to smithi16367185-13 2023-10-09T09:21:52.661 INFO:tasks.rados.rados.0.smithi163.stdout:3884: finishing write tid 2 to smithi16367185-13 2023-10-09T09:21:52.661 INFO:tasks.rados.rados.0.smithi163.stdout:3884: finishing write tid 3 to smithi16367185-13 2023-10-09T09:21:52.661 INFO:tasks.rados.rados.0.smithi163.stdout:3884: finishing write tid 4 to smithi16367185-13 2023-10-09T09:21:52.661 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 13 v 865 (ObjNum 1469 snap 489 seq_num 1469) dirty exists 2023-10-09T09:21:52.661 INFO:tasks.rados.rados.0.smithi163.stdout:3884: left oid 13 (ObjNum 1469 snap 489 seq_num 1469) 2023-10-09T09:21:52.661 INFO:tasks.rados.rados.0.smithi163.stdout:3885: expect (ObjNum 1413 snap 471 seq_num 1413) 2023-10-09T09:21:52.784 INFO:tasks.rados.rados.0.smithi163.stdout:3883: done (4 left) 2023-10-09T09:21:52.785 INFO:tasks.rados.rados.0.smithi163.stdout:3884: done (3 left) 2023-10-09T09:21:52.785 INFO:tasks.rados.rados.0.smithi163.stdout:3885: done (2 left) 2023-10-09T09:21:52.785 INFO:tasks.rados.rados.0.smithi163.stdout:3886: done (1 left) 2023-10-09T09:21:52.785 INFO:tasks.rados.rados.0.smithi163.stdout:3887: done (0 left) 2023-10-09T09:21:52.785 INFO:tasks.rados.rados.0.smithi163.stdout:3888: snap_create 2023-10-09T09:21:52.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:52 smithi181 bash[59345]: cluster 2023-10-09T09:21:51.350632+0000 osd.5 (osd.5) 578 : cluster 3 failed to encode map e1037 with expected crc 2023-10-09T09:21:52.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:52 smithi181 bash[59345]: cluster 2023-10-09T09:21:51.410990+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e1037 with expected crc 2023-10-09T09:21:52.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:52 smithi181 bash[59345]: cluster 2023-10-09T09:21:51.486245+0000 mgr.y (mgr.34107) 459 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 571 KiB/s wr, 3 op/s 2023-10-09T09:21:52.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:52 smithi163 bash[75425]: cluster 2023-10-09T09:21:51.350632+0000 osd.5 (osd.5) 578 : cluster 3 failed to encode map e1037 with expected crc 2023-10-09T09:21:52.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:52 smithi163 bash[75425]: cluster 2023-10-09T09:21:51.410990+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e1037 with expected crc 2023-10-09T09:21:52.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:52 smithi163 bash[75425]: cluster 2023-10-09T09:21:51.486245+0000 mgr.y (mgr.34107) 459 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 571 KiB/s wr, 3 op/s 2023-10-09T09:21:52.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:52 smithi163 bash[78107]: cluster 2023-10-09T09:21:51.350632+0000 osd.5 (osd.5) 578 : cluster 3 failed to encode map e1037 with expected crc 2023-10-09T09:21:52.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:52 smithi163 bash[78107]: cluster 2023-10-09T09:21:51.410990+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e1037 with expected crc 2023-10-09T09:21:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:52 smithi163 bash[78107]: cluster 2023-10-09T09:21:51.486245+0000 mgr.y (mgr.34107) 459 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 571 KiB/s wr, 3 op/s 2023-10-09T09:21:53.543 INFO:tasks.rados.rados.0.smithi163.stdout:3888: done (0 left) 2023-10-09T09:21:53.543 INFO:tasks.rados.rados.0.smithi163.stdout:3889: read oid 3 snap -1 2023-10-09T09:21:53.543 INFO:tasks.rados.rados.0.smithi163.stdout:3889: expect (ObjNum 1456 snap 485 seq_num 1456) 2023-10-09T09:21:53.543 INFO:tasks.rados.rados.0.smithi163.stdout:3890: delete oid 34 current snap is 490 2023-10-09T09:21:53.549 INFO:tasks.rados.rados.0.smithi163.stdout:3889: expect (ObjNum 1456 snap 485 seq_num 1456) 2023-10-09T09:21:53.641 INFO:tasks.rados.rados.0.smithi163.stdout:3889: done (1 left) 2023-10-09T09:21:53.641 INFO:tasks.rados.rados.0.smithi163.stdout:3890: done (0 left) 2023-10-09T09:21:53.641 INFO:tasks.rados.rados.0.smithi163.stdout:3891: rollback oid 14 current snap is 490 2023-10-09T09:21:53.641 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 14 to 489 2023-10-09T09:21:53.641 INFO:tasks.rados.rados.0.smithi163.stdout:3892: write oid 26 current snap is 490 2023-10-09T09:21:53.641 INFO:tasks.rados.rados.0.smithi163.stdout:3892: seq_num 1472 ranges {739904=422657,1854553=402851} 2023-10-09T09:21:53.652 INFO:tasks.rados.rados.0.smithi163.stdout:3892: writing smithi16367185-26 from 739904 to 1162561 tid 1 2023-10-09T09:21:53.659 INFO:tasks.rados.rados.0.smithi163.stdout:3892: writing smithi16367185-26 from 1854553 to 2257404 tid 2 2023-10-09T09:21:53.659 INFO:tasks.rados.rados.0.smithi163.stdout:3893: read oid 38 snap -1 2023-10-09T09:21:53.659 INFO:tasks.rados.rados.0.smithi163.stdout:3893: expect deleted 2023-10-09T09:21:53.659 INFO:tasks.rados.rados.0.smithi163.stdout:3894: delete oid 21 current snap is 490 2023-10-09T09:21:53.659 INFO:tasks.rados.rados.0.smithi163.stdout:3891: finishing rollback tid 1 to smithi16367185-14 2023-10-09T09:21:53.660 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 14 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2023-10-09T09:21:53.664 INFO:tasks.rados.rados.0.smithi163.stdout:3892: finishing write tid 1 to smithi16367185-26 2023-10-09T09:21:53.664 INFO:tasks.rados.rados.0.smithi163.stdout:3892: finishing write tid 2 to smithi16367185-26 2023-10-09T09:21:53.664 INFO:tasks.rados.rados.0.smithi163.stdout:3892: finishing write tid 3 to smithi16367185-26 2023-10-09T09:21:53.665 INFO:tasks.rados.rados.0.smithi163.stdout:3892: finishing write tid 4 to smithi16367185-26 2023-10-09T09:21:53.665 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 1097 (ObjNum 1472 snap 490 seq_num 1472) dirty exists 2023-10-09T09:21:53.665 INFO:tasks.rados.rados.0.smithi163.stdout:3892: left oid 26 (ObjNum 1472 snap 490 seq_num 1472) 2023-10-09T09:21:53.666 INFO:tasks.rados.rados.0.smithi163.stdout:3891: done (3 left) 2023-10-09T09:21:53.666 INFO:tasks.rados.rados.0.smithi163.stdout:3892: done (2 left) 2023-10-09T09:21:53.666 INFO:tasks.rados.rados.0.smithi163.stdout:3893: done (1 left) 2023-10-09T09:21:53.666 INFO:tasks.rados.rados.0.smithi163.stdout:3894: done (0 left) 2023-10-09T09:21:53.666 INFO:tasks.rados.rados.0.smithi163.stdout:3895: write oid 42 current snap is 490 2023-10-09T09:21:53.666 INFO:tasks.rados.rados.0.smithi163.stdout:3895: seq_num 1474 ranges {773518=573155,1855164=406520} 2023-10-09T09:21:53.674 INFO:tasks.rados.rados.0.smithi163.stdout:3895: writing smithi16367185-42 from 773518 to 1346673 tid 1 2023-10-09T09:21:53.680 INFO:tasks.rados.rados.0.smithi163.stdout:3895: writing smithi16367185-42 from 1855164 to 2261684 tid 2 2023-10-09T09:21:53.680 INFO:tasks.rados.rados.0.smithi163.stdout:3896: delete oid 29 current snap is 490 2023-10-09T09:21:53.680 INFO:tasks.rados.rados.0.smithi163.stdout:3895: finishing write tid 1 to smithi16367185-42 2023-10-09T09:21:53.684 INFO:tasks.rados.rados.0.smithi163.stdout:3896: done (1 left) 2023-10-09T09:21:53.684 INFO:tasks.rados.rados.0.smithi163.stdout:3897: write oid 27 current snap is 490 2023-10-09T09:21:53.684 INFO:tasks.rados.rados.0.smithi163.stdout:3897: seq_num 1476 ranges {407132=440006,1288481=693836,2616455=190677} 2023-10-09T09:21:53.690 INFO:tasks.rados.rados.0.smithi163.stdout:3897: writing smithi16367185-27 from 407132 to 847138 tid 1 2023-10-09T09:21:53.698 INFO:tasks.rados.rados.0.smithi163.stdout:3897: writing smithi16367185-27 from 1288481 to 1982317 tid 2 2023-10-09T09:21:53.703 INFO:tasks.rados.rados.0.smithi163.stdout:3897: writing smithi16367185-27 from 2616455 to 2807132 tid 3 2023-10-09T09:21:53.703 INFO:tasks.rados.rados.0.smithi163.stdout:3898: read oid 38 snap -1 2023-10-09T09:21:53.703 INFO:tasks.rados.rados.0.smithi163.stdout:3898: expect deleted 2023-10-09T09:21:53.703 INFO:tasks.rados.rados.0.smithi163.stdout:3899: write oid 24 current snap is 490 2023-10-09T09:21:53.703 INFO:tasks.rados.rados.0.smithi163.stdout:3895: finishing write tid 2 to smithi16367185-42 2023-10-09T09:21:53.703 INFO:tasks.rados.rados.0.smithi163.stdout:3899: seq_num 1477 ranges {423939=515255,1346963=553847,2643878=180061} 2023-10-09T09:21:53.708 INFO:tasks.rados.rados.0.smithi163.stdout:3899: writing smithi16367185-24 from 423939 to 939194 tid 1 2023-10-09T09:21:53.714 INFO:tasks.rados.rados.0.smithi163.stdout:3899: writing smithi16367185-24 from 1346963 to 1900810 tid 2 2023-10-09T09:21:53.717 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:21:53] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.33.4" 2023-10-09T09:21:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[78107]: cluster 2023-10-09T09:21:52.517851+0000 osd.5 (osd.5) 579 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[78107]: cluster 2023-10-09T09:21:52.521932+0000 mon.a (mon.0) 1271 : cluster 0 osdmap e1038: 8 total, 8 up, 8 in 2023-10-09T09:21:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[78107]: cluster 2023-10-09T09:21:52.533701+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[78107]: cluster 2023-10-09T09:21:53.068298+0000 osd.7 (osd.7) 447 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[78107]: cluster 2023-10-09T09:21:53.071139+0000 osd.7 (osd.7) 448 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[75425]: cluster 2023-10-09T09:21:52.517851+0000 osd.5 (osd.5) 579 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[75425]: cluster 2023-10-09T09:21:52.521932+0000 mon.a (mon.0) 1271 : cluster 0 osdmap e1038: 8 total, 8 up, 8 in 2023-10-09T09:21:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[75425]: cluster 2023-10-09T09:21:52.533701+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[75425]: cluster 2023-10-09T09:21:53.068298+0000 osd.7 (osd.7) 447 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:53 smithi163 bash[75425]: cluster 2023-10-09T09:21:53.071139+0000 osd.7 (osd.7) 448 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:3899: writing smithi16367185-24 from 2643878 to 2823939 tid 3 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:3900: delete oid 46 current snap is 490 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:3895: finishing write tid 3 to smithi16367185-42 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:3895: finishing write tid 4 to smithi16367185-42 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 42 v 2989 (ObjNum 1474 snap 490 seq_num 1474) dirty exists 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:3895: left oid 42 (ObjNum 1474 snap 490 seq_num 1474) 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:3897: finishing write tid 1 to smithi16367185-27 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:3897: finishing write tid 2 to smithi16367185-27 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:3897: finishing write tid 3 to smithi16367185-27 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:3897: finishing write tid 4 to smithi16367185-27 2023-10-09T09:21:53.720 INFO:tasks.rados.rados.0.smithi163.stdout:3897: finishing write tid 5 to smithi16367185-27 2023-10-09T09:21:53.721 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 27 v 1102 (ObjNum 1476 snap 490 seq_num 1476) dirty exists 2023-10-09T09:21:53.721 INFO:tasks.rados.rados.0.smithi163.stdout:3897: left oid 27 (ObjNum 1476 snap 490 seq_num 1476) 2023-10-09T09:21:53.721 INFO:tasks.rados.rados.0.smithi163.stdout:3899: finishing write tid 1 to smithi16367185-24 2023-10-09T09:21:53.721 INFO:tasks.rados.rados.0.smithi163.stdout:3899: finishing write tid 2 to smithi16367185-24 2023-10-09T09:21:53.723 INFO:tasks.rados.rados.0.smithi163.stdout:3899: finishing write tid 3 to smithi16367185-24 2023-10-09T09:21:53.723 INFO:tasks.rados.rados.0.smithi163.stdout:3899: finishing write tid 4 to smithi16367185-24 2023-10-09T09:21:53.723 INFO:tasks.rados.rados.0.smithi163.stdout:3899: finishing write tid 5 to smithi16367185-24 2023-10-09T09:21:53.723 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 24 v 865 (ObjNum 1477 snap 490 seq_num 1477) dirty exists 2023-10-09T09:21:53.723 INFO:tasks.rados.rados.0.smithi163.stdout:3899: left oid 24 (ObjNum 1477 snap 490 seq_num 1477) 2023-10-09T09:21:53.724 INFO:tasks.rados.rados.0.smithi163.stdout:3895: done (4 left) 2023-10-09T09:21:53.724 INFO:tasks.rados.rados.0.smithi163.stdout:3897: done (3 left) 2023-10-09T09:21:53.724 INFO:tasks.rados.rados.0.smithi163.stdout:3898: done (2 left) 2023-10-09T09:21:53.724 INFO:tasks.rados.rados.0.smithi163.stdout:3899: done (1 left) 2023-10-09T09:21:53.724 INFO:tasks.rados.rados.0.smithi163.stdout:3900: done (0 left) 2023-10-09T09:21:53.724 INFO:tasks.rados.rados.0.smithi163.stdout:3901: snap_create 2023-10-09T09:21:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:53 smithi181 bash[59345]: cluster 2023-10-09T09:21:52.517851+0000 osd.5 (osd.5) 579 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:53 smithi181 bash[59345]: cluster 2023-10-09T09:21:52.521932+0000 mon.a (mon.0) 1271 : cluster 0 osdmap e1038: 8 total, 8 up, 8 in 2023-10-09T09:21:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:53 smithi181 bash[59345]: cluster 2023-10-09T09:21:52.533701+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:53 smithi181 bash[59345]: cluster 2023-10-09T09:21:53.068298+0000 osd.7 (osd.7) 447 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:53.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:53 smithi181 bash[59345]: cluster 2023-10-09T09:21:53.071139+0000 osd.7 (osd.7) 448 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:54.545 INFO:tasks.rados.rados.0.smithi163.stdout:3901: done (0 left) 2023-10-09T09:21:54.545 INFO:tasks.rados.rados.0.smithi163.stdout:3902: read oid 41 snap 487 2023-10-09T09:21:54.545 INFO:tasks.rados.rados.0.smithi163.stdout:3902: expect (ObjNum 1451 snap 483 seq_num 1451) 2023-10-09T09:21:54.545 INFO:tasks.rados.rados.0.smithi163.stdout:3903: write oid 25 current snap is 491 2023-10-09T09:21:54.545 INFO:tasks.rados.rados.0.smithi163.stdout:3903: seq_num 1479 ranges {457553=665753,1747574=557516} 2023-10-09T09:21:54.552 INFO:tasks.rados.rados.0.smithi163.stdout:3903: writing smithi16367185-25 from 457553 to 1123306 tid 1 2023-10-09T09:21:54.560 INFO:tasks.rados.rados.0.smithi163.stdout:3903: writing smithi16367185-25 from 1747574 to 2305090 tid 2 2023-10-09T09:21:54.560 INFO:tasks.rados.rados.0.smithi163.stdout:3904: snap_remove snap 488 2023-10-09T09:21:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:54 smithi181 bash[59345]: cluster 2023-10-09T09:21:53.458506+0000 osd.4 (osd.4) 487 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:54.841 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:54 smithi181 bash[59345]: cluster 2023-10-09T09:21:53.486831+0000 mgr.y (mgr.34107) 460 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 163 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 556 KiB/s wr, 7 op/s 2023-10-09T09:21:54.841 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:54 smithi181 bash[59345]: cluster 2023-10-09T09:21:53.536441+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e1039 with expected crc 2023-10-09T09:21:54.841 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:54 smithi181 bash[59345]: cluster 2023-10-09T09:21:53.542998+0000 mon.a (mon.0) 1272 : cluster 0 osdmap e1039: 8 total, 8 up, 8 in 2023-10-09T09:21:54.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:54 smithi181 bash[59345]: cluster 2023-10-09T09:21:53.658615+0000 osd.7 (osd.7) 449 : cluster 3 failed to encode map e1039 with expected crc 2023-10-09T09:21:54.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:54 smithi163 bash[78107]: cluster 2023-10-09T09:21:53.458506+0000 osd.4 (osd.4) 487 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:54.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:54 smithi163 bash[78107]: cluster 2023-10-09T09:21:53.486831+0000 mgr.y (mgr.34107) 460 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 163 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 556 KiB/s wr, 7 op/s 2023-10-09T09:21:54.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:54 smithi163 bash[78107]: cluster 2023-10-09T09:21:53.536441+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e1039 with expected crc 2023-10-09T09:21:54.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:54 smithi163 bash[78107]: cluster 2023-10-09T09:21:53.542998+0000 mon.a (mon.0) 1272 : cluster 0 osdmap e1039: 8 total, 8 up, 8 in 2023-10-09T09:21:54.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:54 smithi163 bash[78107]: cluster 2023-10-09T09:21:53.658615+0000 osd.7 (osd.7) 449 : cluster 3 failed to encode map e1039 with expected crc 2023-10-09T09:21:54.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:54 smithi163 bash[75425]: cluster 2023-10-09T09:21:53.458506+0000 osd.4 (osd.4) 487 : cluster 3 failed to encode map e1038 with expected crc 2023-10-09T09:21:54.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:54 smithi163 bash[75425]: cluster 2023-10-09T09:21:53.486831+0000 mgr.y (mgr.34107) 460 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 163 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 556 KiB/s wr, 7 op/s 2023-10-09T09:21:54.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:54 smithi163 bash[75425]: cluster 2023-10-09T09:21:53.536441+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e1039 with expected crc 2023-10-09T09:21:54.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:54 smithi163 bash[75425]: cluster 2023-10-09T09:21:53.542998+0000 mon.a (mon.0) 1272 : cluster 0 osdmap e1039: 8 total, 8 up, 8 in 2023-10-09T09:21:54.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:54 smithi163 bash[75425]: cluster 2023-10-09T09:21:53.658615+0000 osd.7 (osd.7) 449 : cluster 3 failed to encode map e1039 with expected crc 2023-10-09T09:21:55.356 INFO:tasks.rados.rados.0.smithi163.stdout:3904: done (2 left) 2023-10-09T09:21:55.844 INFO:tasks.rados.rados.0.smithi163.stdout:3905: write oid 2 current snap is 491 2023-10-09T09:21:55.844 INFO:tasks.rados.rados.0.smithi163.stdout:3905: seq_num 1480 ranges {474360=741002,1806056=701174} 2023-10-09T09:21:55.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:55 smithi181 bash[59345]: cluster 2023-10-09T09:21:53.535832+0000 osd.5 (osd.5) 580 : cluster 3 failed to encode map e1039 with expected crc 2023-10-09T09:21:55.906 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:55 smithi181 bash[59345]: cluster 2023-10-09T09:21:54.542708+0000 osd.5 (osd.5) 581 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:55.906 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:55 smithi181 bash[59345]: cluster 2023-10-09T09:21:54.544888+0000 mon.a (mon.0) 1273 : cluster 0 osdmap e1040: 8 total, 8 up, 8 in 2023-10-09T09:21:55.907 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:55 smithi181 bash[59345]: cluster 2023-10-09T09:21:55.159286+0000 osd.4 (osd.4) 488 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:55.907 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:55 smithi181 bash[59345]: cluster 2023-10-09T09:21:55.159362+0000 osd.7 (osd.7) 450 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:55.907 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:55 smithi181 bash[59345]: cluster 2023-10-09T09:21:55.351505+0000 osd.5 (osd.5) 582 : cluster 3 failed to encode map e1041 with expected crc 2023-10-09T09:21:55.907 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:55 smithi181 bash[59345]: cluster 2023-10-09T09:21:55.351818+0000 osd.4 (osd.4) 489 : cluster 3 failed to encode map e1041 with expected crc 2023-10-09T09:21:55.911 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:55 smithi181 bash[59345]: cluster 2023-10-09T09:21:55.355990+0000 mon.a (mon.0) 1274 : cluster 0 osdmap e1041: 8 total, 8 up, 8 in 2023-10-09T09:21:55.911 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:55 smithi181 bash[59345]: audit 2023-10-09T09:21:55.357174+0000 mon.a (mon.0) 1275 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:55.911 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:55 smithi181 bash[59345]: cluster 2023-10-09T09:21:55.370634+0000 osd.7 (osd.7) 451 : cluster 3 failed to encode map e1041 with expected crc 2023-10-09T09:21:55.912 INFO:tasks.rados.rados.0.smithi163.stdout:3905: writing smithi16367185-2 from 474360 to 1215362 tid 1 2023-10-09T09:21:55.912 INFO:tasks.rados.rados.0.smithi163.stdout:3905: writing smithi16367185-2 from 1806056 to 2507230 tid 2 2023-10-09T09:21:55.912 INFO:tasks.rados.rados.0.smithi163.stdout:3906: read oid 39 snap -1 2023-10-09T09:21:55.912 INFO:tasks.rados.rados.0.smithi163.stdout:3906: expect (ObjNum 1310 snap 443 seq_num 1310) 2023-10-09T09:21:55.912 INFO:tasks.rados.rados.0.smithi163.stdout:3907: snap_remove snap 469 2023-10-09T09:21:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[78107]: cluster 2023-10-09T09:21:53.535832+0000 osd.5 (osd.5) 580 : cluster 3 failed to encode map e1039 with expected crc 2023-10-09T09:21:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[78107]: cluster 2023-10-09T09:21:54.542708+0000 osd.5 (osd.5) 581 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[78107]: cluster 2023-10-09T09:21:54.544888+0000 mon.a (mon.0) 1273 : cluster 0 osdmap e1040: 8 total, 8 up, 8 in 2023-10-09T09:21:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[78107]: cluster 2023-10-09T09:21:55.159286+0000 osd.4 (osd.4) 488 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[78107]: cluster 2023-10-09T09:21:55.159362+0000 osd.7 (osd.7) 450 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[78107]: cluster 2023-10-09T09:21:55.351505+0000 osd.5 (osd.5) 582 : cluster 3 failed to encode map e1041 with expected crc 2023-10-09T09:21:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[78107]: cluster 2023-10-09T09:21:55.351818+0000 osd.4 (osd.4) 489 : cluster 3 failed to encode map e1041 with expected crc 2023-10-09T09:21:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[78107]: cluster 2023-10-09T09:21:55.355990+0000 mon.a (mon.0) 1274 : cluster 0 osdmap e1041: 8 total, 8 up, 8 in 2023-10-09T09:21:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[78107]: audit 2023-10-09T09:21:55.357174+0000 mon.a (mon.0) 1275 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:55.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[78107]: cluster 2023-10-09T09:21:55.370634+0000 osd.7 (osd.7) 451 : cluster 3 failed to encode map e1041 with expected crc 2023-10-09T09:21:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[75425]: cluster 2023-10-09T09:21:53.535832+0000 osd.5 (osd.5) 580 : cluster 3 failed to encode map e1039 with expected crc 2023-10-09T09:21:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[75425]: cluster 2023-10-09T09:21:54.542708+0000 osd.5 (osd.5) 581 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[75425]: cluster 2023-10-09T09:21:54.544888+0000 mon.a (mon.0) 1273 : cluster 0 osdmap e1040: 8 total, 8 up, 8 in 2023-10-09T09:21:55.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[75425]: cluster 2023-10-09T09:21:55.159286+0000 osd.4 (osd.4) 488 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:55.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[75425]: cluster 2023-10-09T09:21:55.159362+0000 osd.7 (osd.7) 450 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:55.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[75425]: cluster 2023-10-09T09:21:55.351505+0000 osd.5 (osd.5) 582 : cluster 3 failed to encode map e1041 with expected crc 2023-10-09T09:21:55.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[75425]: cluster 2023-10-09T09:21:55.351818+0000 osd.4 (osd.4) 489 : cluster 3 failed to encode map e1041 with expected crc 2023-10-09T09:21:55.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[75425]: cluster 2023-10-09T09:21:55.355990+0000 mon.a (mon.0) 1274 : cluster 0 osdmap e1041: 8 total, 8 up, 8 in 2023-10-09T09:21:55.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[75425]: audit 2023-10-09T09:21:55.357174+0000 mon.a (mon.0) 1275 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:21:55.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:55 smithi163 bash[75425]: cluster 2023-10-09T09:21:55.370634+0000 osd.7 (osd.7) 451 : cluster 3 failed to encode map e1041 with expected crc 2023-10-09T09:21:56.370 INFO:tasks.rados.rados.0.smithi163.stdout:3907: done (4 left) 2023-10-09T09:21:56.370 INFO:tasks.rados.rados.0.smithi163.stdout:3908: read oid 3 snap 476 2023-10-09T09:21:56.370 INFO:tasks.rados.rados.0.smithi163.stdout:3908: expect (ObjNum 1411 snap 468 seq_num 1411) 2023-10-09T09:21:56.370 INFO:tasks.rados.rados.0.smithi163.stdout:3909: read oid 46 snap -1 2023-10-09T09:21:56.370 INFO:tasks.rados.rados.0.smithi163.stdout:3909: expect deleted 2023-10-09T09:21:56.374 INFO:tasks.rados.rados.0.smithi163.stdout:3910: write oid 18 current snap is 491 2023-10-09T09:21:56.374 INFO:tasks.rados.rados.0.smithi163.stdout:3902: expect (ObjNum 1451 snap 483 seq_num 1451) 2023-10-09T09:21:56.485 INFO:tasks.rados.rados.0.smithi163.stdout:3903: finishing write tid 1 to smithi16367185-25 2023-10-09T09:21:56.485 INFO:tasks.rados.rados.0.smithi163.stdout:3903: finishing write tid 2 to smithi16367185-25 2023-10-09T09:21:56.485 INFO:tasks.rados.rados.0.smithi163.stdout:3903: finishing write tid 3 to smithi16367185-25 2023-10-09T09:21:56.486 INFO:tasks.rados.rados.0.smithi163.stdout:3903: finishing write tid 4 to smithi16367185-25 2023-10-09T09:21:56.486 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 25 v 1785 (ObjNum 1479 snap 491 seq_num 1479) dirty exists 2023-10-09T09:21:56.486 INFO:tasks.rados.rados.0.smithi163.stdout:3903: left oid 25 (ObjNum 1479 snap 491 seq_num 1479) 2023-10-09T09:21:56.486 INFO:tasks.rados.rados.0.smithi163.stdout:3905: finishing write tid 1 to smithi16367185-2 2023-10-09T09:21:56.486 INFO:tasks.rados.rados.0.smithi163.stdout:3906: expect (ObjNum 1310 snap 443 seq_num 1310) 2023-10-09T09:21:56.571 INFO:tasks.rados.rados.0.smithi163.stdout:3905: finishing write tid 2 to smithi16367185-2 2023-10-09T09:21:56.572 INFO:tasks.rados.rados.0.smithi163.stdout:3905: finishing write tid 3 to smithi16367185-2 2023-10-09T09:21:56.572 INFO:tasks.rados.rados.0.smithi163.stdout:3905: finishing write tid 4 to smithi16367185-2 2023-10-09T09:21:56.572 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 1029 (ObjNum 1480 snap 491 seq_num 1480) dirty exists 2023-10-09T09:21:56.572 INFO:tasks.rados.rados.0.smithi163.stdout:3905: left oid 2 (ObjNum 1480 snap 491 seq_num 1480) 2023-10-09T09:21:56.572 INFO:tasks.rados.rados.0.smithi163.stdout:3908: expect (ObjNum 1411 snap 468 seq_num 1411) 2023-10-09T09:21:56.785 INFO:tasks.rados.rados.0.smithi163.stdout:3910: seq_num 1481 ranges {491167=416251,1464538=561185,2753570=137597} 2023-10-09T09:21:56.791 INFO:tasks.rados.rados.0.smithi163.stdout:3910: writing smithi16367185-18 from 491167 to 907418 tid 1 2023-10-09T09:21:56.799 INFO:tasks.rados.rados.0.smithi163.stdout:3910: writing smithi16367185-18 from 1464538 to 2025723 tid 2 2023-10-09T09:21:56.805 INFO:tasks.rados.rados.0.smithi163.stdout:3910: writing smithi16367185-18 from 2753570 to 2891167 tid 3 2023-10-09T09:21:56.805 INFO:tasks.rados.rados.0.smithi163.stdout:3902: done (6 left) 2023-10-09T09:21:56.805 INFO:tasks.rados.rados.0.smithi163.stdout:3903: done (5 left) 2023-10-09T09:21:56.805 INFO:tasks.rados.rados.0.smithi163.stdout:3905: done (4 left) 2023-10-09T09:21:56.805 INFO:tasks.rados.rados.0.smithi163.stdout:3906: done (3 left) 2023-10-09T09:21:56.805 INFO:tasks.rados.rados.0.smithi163.stdout:3908: done (2 left) 2023-10-09T09:21:56.805 INFO:tasks.rados.rados.0.smithi163.stdout:3909: done (1 left) 2023-10-09T09:21:56.805 INFO:tasks.rados.rados.0.smithi163.stdout:3911: delete oid 34 current snap is 491 2023-10-09T09:21:56.805 INFO:tasks.rados.rados.0.smithi163.stdout:3910: finishing write tid 1 to smithi16367185-18 2023-10-09T09:21:56.805 INFO:tasks.rados.rados.0.smithi163.stdout:3910: finishing write tid 2 to smithi16367185-18 2023-10-09T09:21:56.808 INFO:tasks.rados.rados.0.smithi163.stdout:3910: finishing write tid 3 to smithi16367185-18 2023-10-09T09:21:56.808 INFO:tasks.rados.rados.0.smithi163.stdout:3910: finishing write tid 4 to smithi16367185-18 2023-10-09T09:21:56.809 INFO:tasks.rados.rados.0.smithi163.stdout:3910: finishing write tid 5 to smithi16367185-18 2023-10-09T09:21:56.809 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1378 (ObjNum 1481 snap 491 seq_num 1481) dirty exists 2023-10-09T09:21:56.809 INFO:tasks.rados.rados.0.smithi163.stdout:3910: left oid 18 (ObjNum 1481 snap 491 seq_num 1481) 2023-10-09T09:21:56.810 INFO:tasks.rados.rados.0.smithi163.stdout:3910: done (1 left) 2023-10-09T09:21:56.810 INFO:tasks.rados.rados.0.smithi163.stdout:3911: done (0 left) 2023-10-09T09:21:56.810 INFO:tasks.rados.rados.0.smithi163.stdout:3912: rollback oid 35 current snap is 491 2023-10-09T09:21:56.810 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 35 to 485 2023-10-09T09:21:56.810 INFO:tasks.rados.rados.0.smithi163.stdout:3913: read oid 23 snap -1 2023-10-09T09:21:56.810 INFO:tasks.rados.rados.0.smithi163.stdout:3913: expect (ObjNum 1467 snap 488 seq_num 1467) 2023-10-09T09:21:56.810 INFO:tasks.rados.rados.0.smithi163.stdout:3914: snap_remove snap 484 2023-10-09T09:21:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:56 smithi181 bash[59345]: cluster 2023-10-09T09:21:54.973488+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:56 smithi181 bash[59345]: cluster 2023-10-09T09:21:54.976141+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:56 smithi181 bash[59345]: cluster 2023-10-09T09:21:55.487515+0000 mgr.y (mgr.34107) 461 : cluster 0 pgmap v1115: 89 pgs: 89 active+clean; 171 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2023-10-09T09:21:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:56 smithi181 bash[59345]: cluster 2023-10-09T09:21:56.366384+0000 osd.5 (osd.5) 583 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:56 smithi181 bash[59345]: cluster 2023-10-09T09:21:56.369199+0000 mon.a (mon.0) 1276 : cluster 0 osdmap e1042: 8 total, 8 up, 8 in 2023-10-09T09:21:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:56 smithi181 bash[59345]: cluster 2023-10-09T09:21:56.371053+0000 osd.4 (osd.4) 490 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:56 smithi181 bash[59345]: cluster 2023-10-09T09:21:56.375223+0000 osd.4 (osd.4) 491 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:56 smithi181 bash[59345]: cluster 2023-10-09T09:21:56.377787+0000 osd.4 (osd.4) 492 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[78107]: cluster 2023-10-09T09:21:54.973488+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:56.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[78107]: cluster 2023-10-09T09:21:54.976141+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:56.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[78107]: cluster 2023-10-09T09:21:55.487515+0000 mgr.y (mgr.34107) 461 : cluster 0 pgmap v1115: 89 pgs: 89 active+clean; 171 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2023-10-09T09:21:56.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[78107]: cluster 2023-10-09T09:21:56.366384+0000 osd.5 (osd.5) 583 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[78107]: cluster 2023-10-09T09:21:56.369199+0000 mon.a (mon.0) 1276 : cluster 0 osdmap e1042: 8 total, 8 up, 8 in 2023-10-09T09:21:56.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[78107]: cluster 2023-10-09T09:21:56.371053+0000 osd.4 (osd.4) 490 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[78107]: cluster 2023-10-09T09:21:56.375223+0000 osd.4 (osd.4) 491 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[78107]: cluster 2023-10-09T09:21:56.377787+0000 osd.4 (osd.4) 492 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[75425]: cluster 2023-10-09T09:21:54.973488+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:56.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[75425]: cluster 2023-10-09T09:21:54.976141+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e1040 with expected crc 2023-10-09T09:21:56.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[75425]: cluster 2023-10-09T09:21:55.487515+0000 mgr.y (mgr.34107) 461 : cluster 0 pgmap v1115: 89 pgs: 89 active+clean; 171 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2023-10-09T09:21:56.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[75425]: cluster 2023-10-09T09:21:56.366384+0000 osd.5 (osd.5) 583 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[75425]: cluster 2023-10-09T09:21:56.369199+0000 mon.a (mon.0) 1276 : cluster 0 osdmap e1042: 8 total, 8 up, 8 in 2023-10-09T09:21:56.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[75425]: cluster 2023-10-09T09:21:56.371053+0000 osd.4 (osd.4) 490 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[75425]: cluster 2023-10-09T09:21:56.375223+0000 osd.4 (osd.4) 491 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:56.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:56 smithi163 bash[75425]: cluster 2023-10-09T09:21:56.377787+0000 osd.4 (osd.4) 492 : cluster 3 failed to encode map e1042 with expected crc 2023-10-09T09:21:57.552 INFO:tasks.rados.rados.0.smithi163.stdout:3914: done (2 left) 2023-10-09T09:21:57.552 INFO:tasks.rados.rados.0.smithi163.stdout:3915: snap_remove snap 483 2023-10-09T09:21:58.561 INFO:tasks.rados.rados.0.smithi163.stdout:3915: done (2 left) 2023-10-09T09:21:58.562 INFO:tasks.rados.rados.0.smithi163.stdout:3916: snap_remove snap 480 2023-10-09T09:21:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:58 smithi181 bash[59345]: cluster 2023-10-09T09:21:57.488274+0000 mgr.y (mgr.34107) 462 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 4.2 MiB/s wr, 32 op/s 2023-10-09T09:21:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:58 smithi181 bash[59345]: cluster 2023-10-09T09:21:57.550114+0000 osd.5 (osd.5) 584 : cluster 3 failed to encode map e1043 with expected crc 2023-10-09T09:21:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:58 smithi181 bash[59345]: cluster 2023-10-09T09:21:57.552250+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e1043: 8 total, 8 up, 8 in 2023-10-09T09:21:58.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:58 smithi181 bash[59345]: cluster 2023-10-09T09:21:57.556268+0000 osd.4 (osd.4) 493 : cluster 3 failed to encode map e1043 with expected crc 2023-10-09T09:21:58.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:58 smithi163 bash[75425]: cluster 2023-10-09T09:21:57.488274+0000 mgr.y (mgr.34107) 462 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 4.2 MiB/s wr, 32 op/s 2023-10-09T09:21:58.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:58 smithi163 bash[75425]: cluster 2023-10-09T09:21:57.550114+0000 osd.5 (osd.5) 584 : cluster 3 failed to encode map e1043 with expected crc 2023-10-09T09:21:58.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:58 smithi163 bash[75425]: cluster 2023-10-09T09:21:57.552250+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e1043: 8 total, 8 up, 8 in 2023-10-09T09:21:58.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:58 smithi163 bash[75425]: cluster 2023-10-09T09:21:57.556268+0000 osd.4 (osd.4) 493 : cluster 3 failed to encode map e1043 with expected crc 2023-10-09T09:21:58.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:58 smithi163 bash[78107]: cluster 2023-10-09T09:21:57.488274+0000 mgr.y (mgr.34107) 462 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 4.2 MiB/s wr, 32 op/s 2023-10-09T09:21:58.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:58 smithi163 bash[78107]: cluster 2023-10-09T09:21:57.550114+0000 osd.5 (osd.5) 584 : cluster 3 failed to encode map e1043 with expected crc 2023-10-09T09:21:58.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:58 smithi163 bash[78107]: cluster 2023-10-09T09:21:57.552250+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e1043: 8 total, 8 up, 8 in 2023-10-09T09:21:58.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:58 smithi163 bash[78107]: cluster 2023-10-09T09:21:57.556268+0000 osd.4 (osd.4) 493 : cluster 3 failed to encode map e1043 with expected crc 2023-10-09T09:21:59.572 INFO:tasks.rados.rados.0.smithi163.stdout:3916: done (2 left) 2023-10-09T09:21:59.572 INFO:tasks.rados.rados.0.smithi163.stdout:3917: rollback oid 18 current snap is 491 2023-10-09T09:21:59.572 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 18 to 487 2023-10-09T09:21:59.572 INFO:tasks.rados.rados.0.smithi163.stdout:3912: finishing rollback tid 1 to smithi16367185-35 2023-10-09T09:21:59.572 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 35 v 0 (ObjNum 699733952 snap 21875 seq_num 0) dirty dne 2023-10-09T09:21:59.573 INFO:tasks.rados.rados.0.smithi163.stdout:3913: expect (ObjNum 1467 snap 488 seq_num 1467) 2023-10-09T09:21:59.706 INFO:tasks.rados.rados.0.smithi163.stdout:3917: finishing rollback tid 0 to smithi16367185-18 2023-10-09T09:21:59.706 INFO:tasks.rados.rados.0.smithi163.stdout:3917: finishing rollback tid 1 to smithi16367185-18 2023-10-09T09:21:59.706 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1380 (ObjNum 699307904 snap 21875 seq_num 699667504) dirty dne 2023-10-09T09:21:59.706 INFO:tasks.rados.rados.0.smithi163.stdout:3912: done (2 left) 2023-10-09T09:21:59.706 INFO:tasks.rados.rados.0.smithi163.stdout:3913: done (1 left) 2023-10-09T09:21:59.706 INFO:tasks.rados.rados.0.smithi163.stdout:3917: done (0 left) 2023-10-09T09:21:59.706 INFO:tasks.rados.rados.0.smithi163.stdout:3918: snap_remove snap 467 2023-10-09T09:21:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:59 smithi163 bash[78107]: cluster 2023-10-09T09:21:58.559179+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e1044 with expected crc 2023-10-09T09:21:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:59 smithi163 bash[78107]: cluster 2023-10-09T09:21:58.559225+0000 osd.5 (osd.5) 585 : cluster 3 failed to encode map e1044 with expected crc 2023-10-09T09:21:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:21:59 smithi163 bash[78107]: cluster 2023-10-09T09:21:58.561743+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e1044: 8 total, 8 up, 8 in 2023-10-09T09:21:59.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:59 smithi163 bash[75425]: cluster 2023-10-09T09:21:58.559179+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e1044 with expected crc 2023-10-09T09:21:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:59 smithi163 bash[75425]: cluster 2023-10-09T09:21:58.559225+0000 osd.5 (osd.5) 585 : cluster 3 failed to encode map e1044 with expected crc 2023-10-09T09:21:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:21:59 smithi163 bash[75425]: cluster 2023-10-09T09:21:58.561743+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e1044: 8 total, 8 up, 8 in 2023-10-09T09:22:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:59 smithi181 bash[59345]: cluster 2023-10-09T09:21:58.559179+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e1044 with expected crc 2023-10-09T09:22:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:59 smithi181 bash[59345]: cluster 2023-10-09T09:21:58.559225+0000 osd.5 (osd.5) 585 : cluster 3 failed to encode map e1044 with expected crc 2023-10-09T09:22:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:21:59 smithi181 bash[59345]: cluster 2023-10-09T09:21:58.561743+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e1044: 8 total, 8 up, 8 in 2023-10-09T09:22:00.581 INFO:tasks.rados.rados.0.smithi163.stdout:3918: done (0 left) 2023-10-09T09:22:00.581 INFO:tasks.rados.rados.0.smithi163.stdout:3919: read oid 50 snap 487 2023-10-09T09:22:00.581 INFO:tasks.rados.rados.0.smithi163.stdout:3919: expect (ObjNum 1458 snap 485 seq_num 1458) 2023-10-09T09:22:00.581 INFO:tasks.rados.rados.0.smithi163.stdout:3920: snap_create 2023-10-09T09:22:00.591 INFO:tasks.rados.rados.0.smithi163.stdout:3919: expect (ObjNum 1458 snap 485 seq_num 1458) 2023-10-09T09:22:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:00 smithi163 bash[78107]: cluster 2023-10-09T09:21:59.489106+0000 mgr.y (mgr.34107) 463 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.4 MiB/s wr, 18 op/s 2023-10-09T09:22:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:00 smithi163 bash[78107]: cluster 2023-10-09T09:21:59.568354+0000 osd.5 (osd.5) 586 : cluster 3 failed to encode map e1045 with expected crc 2023-10-09T09:22:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:00 smithi163 bash[78107]: cluster 2023-10-09T09:21:59.568737+0000 osd.4 (osd.4) 494 : cluster 3 failed to encode map e1045 with expected crc 2023-10-09T09:22:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:00 smithi163 bash[78107]: cluster 2023-10-09T09:21:59.572301+0000 mon.a (mon.0) 1279 : cluster 0 osdmap e1045: 8 total, 8 up, 8 in 2023-10-09T09:22:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:00 smithi163 bash[75425]: cluster 2023-10-09T09:21:59.489106+0000 mgr.y (mgr.34107) 463 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.4 MiB/s wr, 18 op/s 2023-10-09T09:22:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:00 smithi163 bash[75425]: cluster 2023-10-09T09:21:59.568354+0000 osd.5 (osd.5) 586 : cluster 3 failed to encode map e1045 with expected crc 2023-10-09T09:22:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:00 smithi163 bash[75425]: cluster 2023-10-09T09:21:59.568737+0000 osd.4 (osd.4) 494 : cluster 3 failed to encode map e1045 with expected crc 2023-10-09T09:22:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:00 smithi163 bash[75425]: cluster 2023-10-09T09:21:59.572301+0000 mon.a (mon.0) 1279 : cluster 0 osdmap e1045: 8 total, 8 up, 8 in 2023-10-09T09:22:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:00 smithi181 bash[59345]: cluster 2023-10-09T09:21:59.489106+0000 mgr.y (mgr.34107) 463 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.4 MiB/s wr, 18 op/s 2023-10-09T09:22:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:00 smithi181 bash[59345]: cluster 2023-10-09T09:21:59.568354+0000 osd.5 (osd.5) 586 : cluster 3 failed to encode map e1045 with expected crc 2023-10-09T09:22:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:00 smithi181 bash[59345]: cluster 2023-10-09T09:21:59.568737+0000 osd.4 (osd.4) 494 : cluster 3 failed to encode map e1045 with expected crc 2023-10-09T09:22:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:00 smithi181 bash[59345]: cluster 2023-10-09T09:21:59.572301+0000 mon.a (mon.0) 1279 : cluster 0 osdmap e1045: 8 total, 8 up, 8 in 2023-10-09T09:22:01.589 INFO:tasks.rados.rados.0.smithi163.stdout:3919: done (1 left) 2023-10-09T09:22:01.589 INFO:tasks.rados.rados.0.smithi163.stdout:3920: done (0 left) 2023-10-09T09:22:01.590 INFO:tasks.rados.rados.0.smithi163.stdout:3921: snap_remove snap 491 2023-10-09T09:22:01.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:01 smithi163 bash[78107]: cluster 2023-10-09T09:22:00.577102+0000 osd.5 (osd.5) 587 : cluster 3 failed to encode map e1046 with expected crc 2023-10-09T09:22:01.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:01 smithi163 bash[78107]: cluster 2023-10-09T09:22:00.580451+0000 mon.a (mon.0) 1280 : cluster 0 osdmap e1046: 8 total, 8 up, 8 in 2023-10-09T09:22:01.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:01 smithi163 bash[78107]: cluster 2023-10-09T09:22:00.580762+0000 osd.4 (osd.4) 495 : cluster 3 failed to encode map e1046 with expected crc 2023-10-09T09:22:01.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:01 smithi163 bash[78107]: cluster 2023-10-09T09:22:00.584551+0000 osd.4 (osd.4) 496 : cluster 3 failed to encode map e1046 with expected crc 2023-10-09T09:22:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:01 smithi163 bash[75425]: cluster 2023-10-09T09:22:00.577102+0000 osd.5 (osd.5) 587 : cluster 3 failed to encode map e1046 with expected crc 2023-10-09T09:22:01.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:01 smithi163 bash[75425]: cluster 2023-10-09T09:22:00.580451+0000 mon.a (mon.0) 1280 : cluster 0 osdmap e1046: 8 total, 8 up, 8 in 2023-10-09T09:22:01.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:01 smithi163 bash[75425]: cluster 2023-10-09T09:22:00.580762+0000 osd.4 (osd.4) 495 : cluster 3 failed to encode map e1046 with expected crc 2023-10-09T09:22:01.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:01 smithi163 bash[75425]: cluster 2023-10-09T09:22:00.584551+0000 osd.4 (osd.4) 496 : cluster 3 failed to encode map e1046 with expected crc 2023-10-09T09:22:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:01 smithi181 bash[59345]: cluster 2023-10-09T09:22:00.577102+0000 osd.5 (osd.5) 587 : cluster 3 failed to encode map e1046 with expected crc 2023-10-09T09:22:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:01 smithi181 bash[59345]: cluster 2023-10-09T09:22:00.580451+0000 mon.a (mon.0) 1280 : cluster 0 osdmap e1046: 8 total, 8 up, 8 in 2023-10-09T09:22:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:01 smithi181 bash[59345]: cluster 2023-10-09T09:22:00.580762+0000 osd.4 (osd.4) 495 : cluster 3 failed to encode map e1046 with expected crc 2023-10-09T09:22:02.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:01 smithi181 bash[59345]: cluster 2023-10-09T09:22:00.584551+0000 osd.4 (osd.4) 496 : cluster 3 failed to encode map e1046 with expected crc 2023-10-09T09:22:02.593 INFO:tasks.rados.rados.0.smithi163.stdout:3921: done (0 left) 2023-10-09T09:22:02.593 INFO:tasks.rados.rados.0.smithi163.stdout:3922: write oid 9 current snap is 492 2023-10-09T09:22:02.593 INFO:tasks.rados.rados.0.smithi163.stdout:3922: seq_num 1483 ranges {524781=683102,1981502=564854} 2023-10-09T09:22:02.604 INFO:tasks.rados.rados.0.smithi163.stdout:3922: writing smithi16367185-9 from 524781 to 1207883 tid 1 2023-10-09T09:22:02.613 INFO:tasks.rados.rados.0.smithi163.stdout:3922: writing smithi16367185-9 from 1981502 to 2546356 tid 2 2023-10-09T09:22:02.613 INFO:tasks.rados.rados.0.smithi163.stdout:3923: rollback oid 5 current snap is 492 2023-10-09T09:22:02.613 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 5 to 486 2023-10-09T09:22:02.613 INFO:tasks.rados.rados.0.smithi163.stdout:3924: delete oid 39 current snap is 492 2023-10-09T09:22:02.614 INFO:tasks.rados.rados.0.smithi163.stdout:3922: finishing write tid 1 to smithi16367185-9 2023-10-09T09:22:02.618 INFO:tasks.rados.rados.0.smithi163.stdout:3922: finishing write tid 2 to smithi16367185-9 2023-10-09T09:22:02.619 INFO:tasks.rados.rados.0.smithi163.stdout:3922: finishing write tid 3 to smithi16367185-9 2023-10-09T09:22:02.619 INFO:tasks.rados.rados.0.smithi163.stdout:3922: finishing write tid 4 to smithi16367185-9 2023-10-09T09:22:02.619 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 9 v 872 (ObjNum 1483 snap 492 seq_num 1483) dirty exists 2023-10-09T09:22:02.619 INFO:tasks.rados.rados.0.smithi163.stdout:3922: left oid 9 (ObjNum 1483 snap 492 seq_num 1483) 2023-10-09T09:22:02.619 INFO:tasks.rados.rados.0.smithi163.stdout:3922: done (2 left) 2023-10-09T09:22:02.620 INFO:tasks.rados.rados.0.smithi163.stdout:3924: done (1 left) 2023-10-09T09:22:02.620 INFO:tasks.rados.rados.0.smithi163.stdout:3925: snap_remove snap 490 2023-10-09T09:22:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:02 smithi163 bash[78107]: cluster 2023-10-09T09:22:01.490083+0000 mgr.y (mgr.34107) 464 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 273 KiB/s wr, 6 op/s 2023-10-09T09:22:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:02 smithi163 bash[78107]: cluster 2023-10-09T09:22:01.585256+0000 osd.5 (osd.5) 588 : cluster 3 failed to encode map e1047 with expected crc 2023-10-09T09:22:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:02 smithi163 bash[78107]: cluster 2023-10-09T09:22:01.587862+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e1047 with expected crc 2023-10-09T09:22:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:02 smithi163 bash[78107]: cluster 2023-10-09T09:22:01.589467+0000 mon.a (mon.0) 1281 : cluster 0 osdmap e1047: 8 total, 8 up, 8 in 2023-10-09T09:22:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:02 smithi163 bash[75425]: cluster 2023-10-09T09:22:01.490083+0000 mgr.y (mgr.34107) 464 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 273 KiB/s wr, 6 op/s 2023-10-09T09:22:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:02 smithi163 bash[75425]: cluster 2023-10-09T09:22:01.585256+0000 osd.5 (osd.5) 588 : cluster 3 failed to encode map e1047 with expected crc 2023-10-09T09:22:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:02 smithi163 bash[75425]: cluster 2023-10-09T09:22:01.587862+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e1047 with expected crc 2023-10-09T09:22:02.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:02 smithi163 bash[75425]: cluster 2023-10-09T09:22:01.589467+0000 mon.a (mon.0) 1281 : cluster 0 osdmap e1047: 8 total, 8 up, 8 in 2023-10-09T09:22:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:02 smithi181 bash[59345]: cluster 2023-10-09T09:22:01.490083+0000 mgr.y (mgr.34107) 464 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 273 KiB/s wr, 6 op/s 2023-10-09T09:22:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:02 smithi181 bash[59345]: cluster 2023-10-09T09:22:01.585256+0000 osd.5 (osd.5) 588 : cluster 3 failed to encode map e1047 with expected crc 2023-10-09T09:22:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:02 smithi181 bash[59345]: cluster 2023-10-09T09:22:01.587862+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e1047 with expected crc 2023-10-09T09:22:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:02 smithi181 bash[59345]: cluster 2023-10-09T09:22:01.589467+0000 mon.a (mon.0) 1281 : cluster 0 osdmap e1047: 8 total, 8 up, 8 in 2023-10-09T09:22:03.610 INFO:tasks.rados.rados.0.smithi163.stdout:3925: done (1 left) 2023-10-09T09:22:03.610 INFO:tasks.rados.rados.0.smithi163.stdout:3926: write oid 26 current snap is 492 2023-10-09T09:22:03.610 INFO:tasks.rados.rados.0.smithi163.stdout:3926: seq_num 1485 ranges {558395=433600,1698466=568523,2863262=95133} 2023-10-09T09:22:03.615 INFO:tasks.rados.rados.0.smithi163.stdout:3926: writing smithi16367185-26 from 558395 to 991995 tid 1 2023-10-09T09:22:03.624 INFO:tasks.rados.rados.0.smithi163.stdout:3926: writing smithi16367185-26 from 1698466 to 2266989 tid 2 2023-10-09T09:22:03.629 INFO:tasks.rados.rados.0.smithi163.stdout:3926: writing smithi16367185-26 from 2863262 to 2958395 tid 3 2023-10-09T09:22:03.629 INFO:tasks.rados.rados.0.smithi163.stdout:3927: snap_create 2023-10-09T09:22:03.629 INFO:tasks.rados.rados.0.smithi163.stdout:3923: finishing rollback tid 1 to smithi16367185-5 2023-10-09T09:22:03.629 INFO:tasks.rados.rados.0.smithi163.stdout:3923: finishing rollback tid 2 to smithi16367185-5 2023-10-09T09:22:03.629 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 5 v 2190 (ObjNum 1426 snap 474 seq_num 1426) dirty exists 2023-10-09T09:22:03.629 INFO:tasks.rados.rados.0.smithi163.stdout:3926: finishing write tid 1 to smithi16367185-26 2023-10-09T09:22:03.630 INFO:tasks.rados.rados.0.smithi163.stdout:3926: finishing write tid 2 to smithi16367185-26 2023-10-09T09:22:03.631 INFO:tasks.rados.rados.0.smithi163.stdout:3926: finishing write tid 3 to smithi16367185-26 2023-10-09T09:22:03.631 INFO:tasks.rados.rados.0.smithi163.stdout:3926: finishing write tid 4 to smithi16367185-26 2023-10-09T09:22:03.632 INFO:tasks.rados.rados.0.smithi163.stdout:3926: finishing write tid 5 to smithi16367185-26 2023-10-09T09:22:03.632 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 1134 (ObjNum 1485 snap 492 seq_num 1485) dirty exists 2023-10-09T09:22:03.632 INFO:tasks.rados.rados.0.smithi163.stdout:3926: left oid 26 (ObjNum 1485 snap 492 seq_num 1485) 2023-10-09T09:22:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:22:03] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.33.4" 2023-10-09T09:22:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[75425]: cluster 2023-10-09T09:22:02.587761+0000 osd.5 (osd.5) 589 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:03.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[75425]: cluster 2023-10-09T09:22:02.590334+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[75425]: cluster 2023-10-09T09:22:02.592343+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e1048: 8 total, 8 up, 8 in 2023-10-09T09:22:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[75425]: cluster 2023-10-09T09:22:02.609363+0000 osd.4 (osd.4) 497 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:03.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[75425]: cluster 2023-10-09T09:22:02.760571+0000 osd.7 (osd.7) 452 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[78107]: cluster 2023-10-09T09:22:02.587761+0000 osd.5 (osd.5) 589 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[78107]: cluster 2023-10-09T09:22:02.590334+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[78107]: cluster 2023-10-09T09:22:02.592343+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e1048: 8 total, 8 up, 8 in 2023-10-09T09:22:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[78107]: cluster 2023-10-09T09:22:02.609363+0000 osd.4 (osd.4) 497 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:03.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:03 smithi163 bash[78107]: cluster 2023-10-09T09:22:02.760571+0000 osd.7 (osd.7) 452 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:03 smithi181 bash[59345]: cluster 2023-10-09T09:22:02.587761+0000 osd.5 (osd.5) 589 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:03 smithi181 bash[59345]: cluster 2023-10-09T09:22:02.590334+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:03 smithi181 bash[59345]: cluster 2023-10-09T09:22:02.592343+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e1048: 8 total, 8 up, 8 in 2023-10-09T09:22:04.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:03 smithi181 bash[59345]: cluster 2023-10-09T09:22:02.609363+0000 osd.4 (osd.4) 497 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:04.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:03 smithi181 bash[59345]: cluster 2023-10-09T09:22:02.760571+0000 osd.7 (osd.7) 452 : cluster 3 failed to encode map e1048 with expected crc 2023-10-09T09:22:04.621 INFO:tasks.rados.rados.0.smithi163.stdout:3923: done (2 left) 2023-10-09T09:22:04.622 INFO:tasks.rados.rados.0.smithi163.stdout:3926: done (1 left) 2023-10-09T09:22:04.622 INFO:tasks.rados.rados.0.smithi163.stdout:3927: done (0 left) 2023-10-09T09:22:04.622 INFO:tasks.rados.rados.0.smithi163.stdout:3928: read oid 13 snap -1 2023-10-09T09:22:04.622 INFO:tasks.rados.rados.0.smithi163.stdout:3928: expect (ObjNum 1469 snap 489 seq_num 1469) 2023-10-09T09:22:04.622 INFO:tasks.rados.rados.0.smithi163.stdout:3929: write oid 45 current snap is 493 2023-10-09T09:22:04.622 INFO:tasks.rados.rados.0.smithi163.stdout:3929: seq_num 1486 ranges {575202=508849,1640595=712181} 2023-10-09T09:22:04.627 INFO:tasks.rados.rados.0.smithi163.stdout:3929: writing smithi16367185-45 from 575202 to 1084051 tid 1 2023-10-09T09:22:04.635 INFO:tasks.rados.rados.0.smithi163.stdout:3929: writing smithi16367185-45 from 1640595 to 2352776 tid 2 2023-10-09T09:22:04.636 INFO:tasks.rados.rados.0.smithi163.stdout:3930: snap_remove snap 481 2023-10-09T09:22:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:04 smithi163 bash[78107]: cluster 2023-10-09T09:22:03.490865+0000 mgr.y (mgr.34107) 465 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 273 KiB/s wr, 5 op/s 2023-10-09T09:22:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:04 smithi163 bash[78107]: cluster 2023-10-09T09:22:03.605901+0000 osd.5 (osd.5) 590 : cluster 3 failed to encode map e1049 with expected crc 2023-10-09T09:22:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:04 smithi163 bash[78107]: cluster 2023-10-09T09:22:03.609310+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e1049: 8 total, 8 up, 8 in 2023-10-09T09:22:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:04 smithi163 bash[78107]: cluster 2023-10-09T09:22:03.620550+0000 osd.7 (osd.7) 453 : cluster 3 failed to encode map e1049 with expected crc 2023-10-09T09:22:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:04 smithi163 bash[75425]: cluster 2023-10-09T09:22:03.490865+0000 mgr.y (mgr.34107) 465 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 273 KiB/s wr, 5 op/s 2023-10-09T09:22:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:04 smithi163 bash[75425]: cluster 2023-10-09T09:22:03.605901+0000 osd.5 (osd.5) 590 : cluster 3 failed to encode map e1049 with expected crc 2023-10-09T09:22:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:04 smithi163 bash[75425]: cluster 2023-10-09T09:22:03.609310+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e1049: 8 total, 8 up, 8 in 2023-10-09T09:22:04.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:04 smithi163 bash[75425]: cluster 2023-10-09T09:22:03.620550+0000 osd.7 (osd.7) 453 : cluster 3 failed to encode map e1049 with expected crc 2023-10-09T09:22:05.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:04 smithi181 bash[59345]: cluster 2023-10-09T09:22:03.490865+0000 mgr.y (mgr.34107) 465 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 273 KiB/s wr, 5 op/s 2023-10-09T09:22:05.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:04 smithi181 bash[59345]: cluster 2023-10-09T09:22:03.605901+0000 osd.5 (osd.5) 590 : cluster 3 failed to encode map e1049 with expected crc 2023-10-09T09:22:05.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:04 smithi181 bash[59345]: cluster 2023-10-09T09:22:03.609310+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e1049: 8 total, 8 up, 8 in 2023-10-09T09:22:05.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:04 smithi181 bash[59345]: cluster 2023-10-09T09:22:03.620550+0000 osd.7 (osd.7) 453 : cluster 3 failed to encode map e1049 with expected crc 2023-10-09T09:22:05.360 INFO:tasks.rados.rados.0.smithi163.stdout:3930: done (2 left) 2023-10-09T09:22:05.361 INFO:tasks.rados.rados.0.smithi163.stdout:3931: write oid 22 current snap is 493 2023-10-09T09:22:05.361 INFO:tasks.rados.rados.0.smithi163.stdout:3931: seq_num 1487 ranges {592009=584098,1699077=572192,2801755=190254} 2023-10-09T09:22:05.370 INFO:tasks.rados.rados.0.smithi163.stdout:3931: writing smithi16367185-22 from 592009 to 1176107 tid 1 2023-10-09T09:22:05.377 INFO:tasks.rados.rados.0.smithi163.stdout:3931: writing smithi16367185-22 from 1699077 to 2271269 tid 2 2023-10-09T09:22:05.382 INFO:tasks.rados.rados.0.smithi163.stdout:3931: writing smithi16367185-22 from 2801755 to 2992009 tid 3 2023-10-09T09:22:05.383 INFO:tasks.rados.rados.0.smithi163.stdout:3932: rollback oid 30 current snap is 493 2023-10-09T09:22:05.383 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 30 to 473 2023-10-09T09:22:05.383 INFO:tasks.rados.rados.0.smithi163.stdout:3929: finishing write tid 1 to smithi16367185-45 2023-10-09T09:22:05.383 INFO:tasks.rados.rados.0.smithi163.stdout:3928: expect (ObjNum 1469 snap 489 seq_num 1469) 2023-10-09T09:22:05.518 INFO:tasks.rados.rados.0.smithi163.stdout:3929: finishing write tid 2 to smithi16367185-45 2023-10-09T09:22:05.518 INFO:tasks.rados.rados.0.smithi163.stdout:3929: finishing write tid 3 to smithi16367185-45 2023-10-09T09:22:05.518 INFO:tasks.rados.rados.0.smithi163.stdout:3929: finishing write tid 4 to smithi16367185-45 2023-10-09T09:22:05.518 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 45 v 3018 (ObjNum 1486 snap 493 seq_num 1486) dirty exists 2023-10-09T09:22:05.518 INFO:tasks.rados.rados.0.smithi163.stdout:3929: left oid 45 (ObjNum 1486 snap 493 seq_num 1486) 2023-10-09T09:22:05.518 INFO:tasks.rados.rados.0.smithi163.stdout:3931: finishing write tid 1 to smithi16367185-22 2023-10-09T09:22:05.518 INFO:tasks.rados.rados.0.smithi163.stdout:3931: finishing write tid 2 to smithi16367185-22 2023-10-09T09:22:05.518 INFO:tasks.rados.rados.0.smithi163.stdout:3931: finishing write tid 3 to smithi16367185-22 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:3932: finishing rollback tid 0 to smithi16367185-30 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:3932: finishing rollback tid 1 to smithi16367185-30 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:3932: finishing rollback tid 2 to smithi16367185-30 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1390 (ObjNum 1413 snap 471 seq_num 1413) dirty exists 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:3931: finishing write tid 4 to smithi16367185-22 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:3931: finishing write tid 5 to smithi16367185-22 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 22 v 3029 (ObjNum 1487 snap 493 seq_num 1487) dirty exists 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:3931: left oid 22 (ObjNum 1487 snap 493 seq_num 1487) 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:3928: done (3 left) 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:3929: done (2 left) 2023-10-09T09:22:05.519 INFO:tasks.rados.rados.0.smithi163.stdout:3931: done (1 left) 2023-10-09T09:22:05.520 INFO:tasks.rados.rados.0.smithi163.stdout:3932: done (0 left) 2023-10-09T09:22:05.520 INFO:tasks.rados.rados.0.smithi163.stdout:3933: snap_create 2023-10-09T09:22:05.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[75425]: cluster 2023-10-09T09:22:04.616020+0000 osd.5 (osd.5) 591 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:05.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[75425]: cluster 2023-10-09T09:22:04.619904+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e1050: 8 total, 8 up, 8 in 2023-10-09T09:22:05.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[75425]: cluster 2023-10-09T09:22:05.172287+0000 osd.7 (osd.7) 454 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:05.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[75425]: cluster 2023-10-09T09:22:05.175310+0000 osd.7 (osd.7) 455 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[75425]: cluster 2023-10-09T09:22:05.355852+0000 osd.5 (osd.5) 592 : cluster 3 failed to encode map e1051 with expected crc 2023-10-09T09:22:05.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[75425]: cluster 2023-10-09T09:22:05.360083+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e1051: 8 total, 8 up, 8 in 2023-10-09T09:22:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[78107]: cluster 2023-10-09T09:22:04.616020+0000 osd.5 (osd.5) 591 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[78107]: cluster 2023-10-09T09:22:04.619904+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e1050: 8 total, 8 up, 8 in 2023-10-09T09:22:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[78107]: cluster 2023-10-09T09:22:05.172287+0000 osd.7 (osd.7) 454 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[78107]: cluster 2023-10-09T09:22:05.175310+0000 osd.7 (osd.7) 455 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[78107]: cluster 2023-10-09T09:22:05.355852+0000 osd.5 (osd.5) 592 : cluster 3 failed to encode map e1051 with expected crc 2023-10-09T09:22:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:05 smithi163 bash[78107]: cluster 2023-10-09T09:22:05.360083+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e1051: 8 total, 8 up, 8 in 2023-10-09T09:22:06.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:05 smithi181 bash[59345]: cluster 2023-10-09T09:22:04.616020+0000 osd.5 (osd.5) 591 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:06.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:05 smithi181 bash[59345]: cluster 2023-10-09T09:22:04.619904+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e1050: 8 total, 8 up, 8 in 2023-10-09T09:22:06.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:05 smithi181 bash[59345]: cluster 2023-10-09T09:22:05.172287+0000 osd.7 (osd.7) 454 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:06.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:05 smithi181 bash[59345]: cluster 2023-10-09T09:22:05.175310+0000 osd.7 (osd.7) 455 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:06.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:05 smithi181 bash[59345]: cluster 2023-10-09T09:22:05.355852+0000 osd.5 (osd.5) 592 : cluster 3 failed to encode map e1051 with expected crc 2023-10-09T09:22:06.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:05 smithi181 bash[59345]: cluster 2023-10-09T09:22:05.360083+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e1051: 8 total, 8 up, 8 in 2023-10-09T09:22:06.377 INFO:tasks.rados.rados.0.smithi163.stdout:3933: done (0 left) 2023-10-09T09:22:06.377 INFO:tasks.rados.rados.0.smithi163.stdout:3934: write oid 20 current snap is 494 2023-10-09T09:22:06.377 INFO:tasks.rados.rados.0.smithi163.stdout:3934: seq_num 1488 ranges {608816=659347,1757559=715850} 2023-10-09T09:22:06.387 INFO:tasks.rados.rados.0.smithi163.stdout:3934: writing smithi16367185-20 from 608816 to 1268163 tid 1 2023-10-09T09:22:06.395 INFO:tasks.rados.rados.0.smithi163.stdout:3934: writing smithi16367185-20 from 1757559 to 2473409 tid 2 2023-10-09T09:22:06.395 INFO:tasks.rados.rados.0.smithi163.stdout:3935: write oid 34 current snap is 494 2023-10-09T09:22:06.395 INFO:tasks.rados.rados.0.smithi163.stdout:3935: seq_num 1489 ranges {625623=734596,1816041=459508,2740248=285375} 2023-10-09T09:22:06.403 INFO:tasks.rados.rados.0.smithi163.stdout:3935: writing smithi16367185-34 from 625623 to 1360219 tid 1 2023-10-09T09:22:06.408 INFO:tasks.rados.rados.0.smithi163.stdout:3935: writing smithi16367185-34 from 1816041 to 2275549 tid 2 2023-10-09T09:22:06.413 INFO:tasks.rados.rados.0.smithi163.stdout:3935: writing smithi16367185-34 from 2740248 to 3025623 tid 3 2023-10-09T09:22:06.413 INFO:tasks.rados.rados.0.smithi163.stdout:3936: read oid 46 snap -1 2023-10-09T09:22:06.413 INFO:tasks.rados.rados.0.smithi163.stdout:3936: expect deleted 2023-10-09T09:22:06.413 INFO:tasks.rados.rados.0.smithi163.stdout:3937: snap_create 2023-10-09T09:22:06.413 INFO:tasks.rados.rados.0.smithi163.stdout:3934: finishing write tid 1 to smithi16367185-20 2023-10-09T09:22:06.413 INFO:tasks.rados.rados.0.smithi163.stdout:3934: finishing write tid 2 to smithi16367185-20 2023-10-09T09:22:06.413 INFO:tasks.rados.rados.0.smithi163.stdout:3934: finishing write tid 3 to smithi16367185-20 2023-10-09T09:22:06.413 INFO:tasks.rados.rados.0.smithi163.stdout:3934: finishing write tid 4 to smithi16367185-20 2023-10-09T09:22:06.414 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 20 v 889 (ObjNum 1488 snap 494 seq_num 1488) dirty exists 2023-10-09T09:22:06.414 INFO:tasks.rados.rados.0.smithi163.stdout:3934: left oid 20 (ObjNum 1488 snap 494 seq_num 1488) 2023-10-09T09:22:06.414 INFO:tasks.rados.rados.0.smithi163.stdout:3935: finishing write tid 1 to smithi16367185-34 2023-10-09T09:22:06.414 INFO:tasks.rados.rados.0.smithi163.stdout:3935: finishing write tid 2 to smithi16367185-34 2023-10-09T09:22:06.417 INFO:tasks.rados.rados.0.smithi163.stdout:3935: finishing write tid 3 to smithi16367185-34 2023-10-09T09:22:06.418 INFO:tasks.rados.rados.0.smithi163.stdout:3935: finishing write tid 4 to smithi16367185-34 2023-10-09T09:22:06.418 INFO:tasks.rados.rados.0.smithi163.stdout:3935: finishing write tid 5 to smithi16367185-34 2023-10-09T09:22:06.418 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 34 v 1840 (ObjNum 1489 snap 494 seq_num 1489) dirty exists 2023-10-09T09:22:06.418 INFO:tasks.rados.rados.0.smithi163.stdout:3935: left oid 34 (ObjNum 1489 snap 494 seq_num 1489) 2023-10-09T09:22:06.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:06 smithi163 bash[75425]: cluster 2023-10-09T09:22:04.620911+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:06.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:06 smithi163 bash[75425]: cluster 2023-10-09T09:22:05.356769+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e1051 with expected crc 2023-10-09T09:22:06.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:06 smithi163 bash[75425]: cluster 2023-10-09T09:22:05.491692+0000 mgr.y (mgr.34107) 466 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 524 B/s rd, 587 KiB/s wr, 3 op/s 2023-10-09T09:22:06.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:06 smithi163 bash[75425]: cluster 2023-10-09T09:22:06.373756+0000 osd.5 (osd.5) 593 : cluster 3 failed to encode map e1052 with expected crc 2023-10-09T09:22:06.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:06 smithi163 bash[75425]: cluster 2023-10-09T09:22:06.376664+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e1052: 8 total, 8 up, 8 in 2023-10-09T09:22:06.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:06 smithi163 bash[78107]: cluster 2023-10-09T09:22:04.620911+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:06.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:06 smithi163 bash[78107]: cluster 2023-10-09T09:22:05.356769+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e1051 with expected crc 2023-10-09T09:22:06.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:06 smithi163 bash[78107]: cluster 2023-10-09T09:22:05.491692+0000 mgr.y (mgr.34107) 466 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 524 B/s rd, 587 KiB/s wr, 3 op/s 2023-10-09T09:22:06.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:06 smithi163 bash[78107]: cluster 2023-10-09T09:22:06.373756+0000 osd.5 (osd.5) 593 : cluster 3 failed to encode map e1052 with expected crc 2023-10-09T09:22:06.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:06 smithi163 bash[78107]: cluster 2023-10-09T09:22:06.376664+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e1052: 8 total, 8 up, 8 in 2023-10-09T09:22:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:06 smithi181 bash[59345]: cluster 2023-10-09T09:22:04.620911+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e1050 with expected crc 2023-10-09T09:22:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:06 smithi181 bash[59345]: cluster 2023-10-09T09:22:05.356769+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e1051 with expected crc 2023-10-09T09:22:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:06 smithi181 bash[59345]: cluster 2023-10-09T09:22:05.491692+0000 mgr.y (mgr.34107) 466 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 524 B/s rd, 587 KiB/s wr, 3 op/s 2023-10-09T09:22:07.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:06 smithi181 bash[59345]: cluster 2023-10-09T09:22:06.373756+0000 osd.5 (osd.5) 593 : cluster 3 failed to encode map e1052 with expected crc 2023-10-09T09:22:07.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:06 smithi181 bash[59345]: cluster 2023-10-09T09:22:06.376664+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e1052: 8 total, 8 up, 8 in 2023-10-09T09:22:07.381 INFO:tasks.rados.rados.0.smithi163.stdout:3934: done (3 left) 2023-10-09T09:22:07.382 INFO:tasks.rados.rados.0.smithi163.stdout:3935: done (2 left) 2023-10-09T09:22:07.382 INFO:tasks.rados.rados.0.smithi163.stdout:3936: done (1 left) 2023-10-09T09:22:07.382 INFO:tasks.rados.rados.0.smithi163.stdout:3937: done (0 left) 2023-10-09T09:22:07.382 INFO:tasks.rados.rados.0.smithi163.stdout:3938: snap_create 2023-10-09T09:22:07.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:07 smithi163 bash[78107]: cluster 2023-10-09T09:22:06.390038+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e1052 with expected crc 2023-10-09T09:22:07.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:07 smithi163 bash[78107]: cluster 2023-10-09T09:22:07.376621+0000 osd.5 (osd.5) 594 : cluster 3 failed to encode map e1053 with expected crc 2023-10-09T09:22:07.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:07 smithi163 bash[78107]: cluster 2023-10-09T09:22:07.377740+0000 osd.7 (osd.7) 456 : cluster 3 failed to encode map e1053 with expected crc 2023-10-09T09:22:07.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:07 smithi163 bash[78107]: cluster 2023-10-09T09:22:07.381785+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e1053: 8 total, 8 up, 8 in 2023-10-09T09:22:07.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:07 smithi163 bash[75425]: cluster 2023-10-09T09:22:06.390038+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e1052 with expected crc 2023-10-09T09:22:07.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:07 smithi163 bash[75425]: cluster 2023-10-09T09:22:07.376621+0000 osd.5 (osd.5) 594 : cluster 3 failed to encode map e1053 with expected crc 2023-10-09T09:22:07.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:07 smithi163 bash[75425]: cluster 2023-10-09T09:22:07.377740+0000 osd.7 (osd.7) 456 : cluster 3 failed to encode map e1053 with expected crc 2023-10-09T09:22:07.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:07 smithi163 bash[75425]: cluster 2023-10-09T09:22:07.381785+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e1053: 8 total, 8 up, 8 in 2023-10-09T09:22:08.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:07 smithi181 bash[59345]: cluster 2023-10-09T09:22:06.390038+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e1052 with expected crc 2023-10-09T09:22:08.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:07 smithi181 bash[59345]: cluster 2023-10-09T09:22:07.376621+0000 osd.5 (osd.5) 594 : cluster 3 failed to encode map e1053 with expected crc 2023-10-09T09:22:08.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:07 smithi181 bash[59345]: cluster 2023-10-09T09:22:07.377740+0000 osd.7 (osd.7) 456 : cluster 3 failed to encode map e1053 with expected crc 2023-10-09T09:22:08.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:07 smithi181 bash[59345]: cluster 2023-10-09T09:22:07.381785+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e1053: 8 total, 8 up, 8 in 2023-10-09T09:22:08.390 INFO:tasks.rados.rados.0.smithi163.stdout:3938: done (0 left) 2023-10-09T09:22:08.391 INFO:tasks.rados.rados.0.smithi163.stdout:3939: snap_remove snap 473 2023-10-09T09:22:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:08 smithi163 bash[78107]: cluster 2023-10-09T09:22:07.492421+0000 mgr.y (mgr.34107) 467 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2023-10-09T09:22:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:08 smithi163 bash[78107]: cluster 2023-10-09T09:22:08.385024+0000 osd.5 (osd.5) 595 : cluster 3 failed to encode map e1054 with expected crc 2023-10-09T09:22:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:08 smithi163 bash[78107]: cluster 2023-10-09T09:22:08.387682+0000 mon.a (mon.0) 1288 : cluster 0 osdmap e1054: 8 total, 8 up, 8 in 2023-10-09T09:22:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:08 smithi163 bash[78107]: cluster 2023-10-09T09:22:08.468983+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e1053 with expected crc 2023-10-09T09:22:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:08 smithi163 bash[75425]: cluster 2023-10-09T09:22:07.492421+0000 mgr.y (mgr.34107) 467 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2023-10-09T09:22:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:08 smithi163 bash[75425]: cluster 2023-10-09T09:22:08.385024+0000 osd.5 (osd.5) 595 : cluster 3 failed to encode map e1054 with expected crc 2023-10-09T09:22:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:08 smithi163 bash[75425]: cluster 2023-10-09T09:22:08.387682+0000 mon.a (mon.0) 1288 : cluster 0 osdmap e1054: 8 total, 8 up, 8 in 2023-10-09T09:22:08.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:08 smithi163 bash[75425]: cluster 2023-10-09T09:22:08.468983+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e1053 with expected crc 2023-10-09T09:22:09.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:08 smithi181 bash[59345]: cluster 2023-10-09T09:22:07.492421+0000 mgr.y (mgr.34107) 467 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2023-10-09T09:22:09.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:08 smithi181 bash[59345]: cluster 2023-10-09T09:22:08.385024+0000 osd.5 (osd.5) 595 : cluster 3 failed to encode map e1054 with expected crc 2023-10-09T09:22:09.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:08 smithi181 bash[59345]: cluster 2023-10-09T09:22:08.387682+0000 mon.a (mon.0) 1288 : cluster 0 osdmap e1054: 8 total, 8 up, 8 in 2023-10-09T09:22:09.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:08 smithi181 bash[59345]: cluster 2023-10-09T09:22:08.468983+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e1053 with expected crc 2023-10-09T09:22:09.406 INFO:tasks.rados.rados.0.smithi163.stdout:3939: done (0 left) 2023-10-09T09:22:09.406 INFO:tasks.rados.rados.0.smithi163.stdout:3940: write oid 6 current snap is 496 2023-10-09T09:22:09.406 INFO:tasks.rados.rados.0.smithi163.stdout:3940: seq_num 1490 ranges {642430=409845,1758170=719519} 2023-10-09T09:22:09.406 INFO:tasks.rados.rados.0.smithi163.stdout:3940: writing smithi16367185-6 from 642430 to 1052275 tid 1 2023-10-09T09:22:09.415 INFO:tasks.rados.rados.0.smithi163.stdout:3940: writing smithi16367185-6 from 1758170 to 2477689 tid 2 2023-10-09T09:22:09.415 INFO:tasks.rados.rados.0.smithi163.stdout:3941: write oid 2 current snap is 496 2023-10-09T09:22:09.415 INFO:tasks.rados.rados.0.smithi163.stdout:3941: seq_num 1491 ranges {659237=601447,1933005=463177} 2023-10-09T09:22:09.422 INFO:tasks.rados.rados.0.smithi163.stdout:3941: writing smithi16367185-2 from 659237 to 1260684 tid 1 2023-10-09T09:22:09.428 INFO:tasks.rados.rados.0.smithi163.stdout:3941: writing smithi16367185-2 from 1933005 to 2396182 tid 2 2023-10-09T09:22:09.429 INFO:tasks.rados.rados.0.smithi163.stdout:3942: snap_remove snap 486 2023-10-09T09:22:10.364 INFO:tasks.rados.rados.0.smithi163.stdout:3942: done (2 left) 2023-10-09T09:22:10.364 INFO:tasks.rados.rados.0.smithi163.stdout:3943: snap_remove snap 495 2023-10-09T09:22:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[75425]: cluster 2023-10-09T09:22:09.388274+0000 osd.5 (osd.5) 596 : cluster 3 failed to encode map e1055 with expected crc 2023-10-09T09:22:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[75425]: cluster 2023-10-09T09:22:09.393149+0000 osd.5 (osd.5) 597 : cluster 3 failed to encode map e1055 with expected crc 2023-10-09T09:22:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[75425]: cluster 2023-10-09T09:22:09.393251+0000 osd.4 (osd.4) 498 : cluster 3 failed to encode map e1055 with expected crc 2023-10-09T09:22:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[75425]: cluster 2023-10-09T09:22:09.394087+0000 mon.a (mon.0) 1289 : cluster 0 osdmap e1055: 8 total, 8 up, 8 in 2023-10-09T09:22:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[75425]: cluster 2023-10-09T09:22:09.493128+0000 mgr.y (mgr.34107) 468 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 627 KiB/s wr, 4 op/s 2023-10-09T09:22:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[75425]: audit 2023-10-09T09:22:10.354537+0000 mon.a (mon.0) 1290 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:10.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[75425]: cluster 2023-10-09T09:22:10.363916+0000 mon.a (mon.0) 1291 : cluster 0 osdmap e1056: 8 total, 8 up, 8 in 2023-10-09T09:22:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[78107]: cluster 2023-10-09T09:22:09.388274+0000 osd.5 (osd.5) 596 : cluster 3 failed to encode map e1055 with expected crc 2023-10-09T09:22:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[78107]: cluster 2023-10-09T09:22:09.393149+0000 osd.5 (osd.5) 597 : cluster 3 failed to encode map e1055 with expected crc 2023-10-09T09:22:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[78107]: cluster 2023-10-09T09:22:09.393251+0000 osd.4 (osd.4) 498 : cluster 3 failed to encode map e1055 with expected crc 2023-10-09T09:22:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[78107]: cluster 2023-10-09T09:22:09.394087+0000 mon.a (mon.0) 1289 : cluster 0 osdmap e1055: 8 total, 8 up, 8 in 2023-10-09T09:22:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[78107]: cluster 2023-10-09T09:22:09.493128+0000 mgr.y (mgr.34107) 468 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 627 KiB/s wr, 4 op/s 2023-10-09T09:22:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[78107]: audit 2023-10-09T09:22:10.354537+0000 mon.a (mon.0) 1290 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:10.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:10 smithi163 bash[78107]: cluster 2023-10-09T09:22:10.363916+0000 mon.a (mon.0) 1291 : cluster 0 osdmap e1056: 8 total, 8 up, 8 in 2023-10-09T09:22:10.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:10 smithi181 bash[59345]: cluster 2023-10-09T09:22:09.388274+0000 osd.5 (osd.5) 596 : cluster 3 failed to encode map e1055 with expected crc 2023-10-09T09:22:10.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:10 smithi181 bash[59345]: cluster 2023-10-09T09:22:09.393149+0000 osd.5 (osd.5) 597 : cluster 3 failed to encode map e1055 with expected crc 2023-10-09T09:22:10.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:10 smithi181 bash[59345]: cluster 2023-10-09T09:22:09.393251+0000 osd.4 (osd.4) 498 : cluster 3 failed to encode map e1055 with expected crc 2023-10-09T09:22:10.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:10 smithi181 bash[59345]: cluster 2023-10-09T09:22:09.394087+0000 mon.a (mon.0) 1289 : cluster 0 osdmap e1055: 8 total, 8 up, 8 in 2023-10-09T09:22:10.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:10 smithi181 bash[59345]: cluster 2023-10-09T09:22:09.493128+0000 mgr.y (mgr.34107) 468 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 627 KiB/s wr, 4 op/s 2023-10-09T09:22:10.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:10 smithi181 bash[59345]: audit 2023-10-09T09:22:10.354537+0000 mon.a (mon.0) 1290 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:10.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:10 smithi181 bash[59345]: cluster 2023-10-09T09:22:10.363916+0000 mon.a (mon.0) 1291 : cluster 0 osdmap e1056: 8 total, 8 up, 8 in 2023-10-09T09:22:11.366 INFO:tasks.rados.rados.0.smithi163.stdout:3943: done (2 left) 2023-10-09T09:22:11.366 INFO:tasks.rados.rados.0.smithi163.stdout:3944: snap_remove snap 493 2023-10-09T09:22:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:11 smithi163 bash[78107]: cluster 2023-10-09T09:22:10.359006+0000 osd.5 (osd.5) 598 : cluster 3 failed to encode map e1056 with expected crc 2023-10-09T09:22:12.929 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:11 smithi163 bash[78107]: cluster 2023-10-09T09:22:10.364666+0000 osd.4 (osd.4) 499 : cluster 3 failed to encode map e1056 with expected crc 2023-10-09T09:22:12.929 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:11 smithi163 bash[78107]: cluster 2023-10-09T09:22:11.365829+0000 mon.a (mon.0) 1292 : cluster 0 osdmap e1057: 8 total, 8 up, 8 in 2023-10-09T09:22:12.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:11 smithi163 bash[75425]: cluster 2023-10-09T09:22:10.359006+0000 osd.5 (osd.5) 598 : cluster 3 failed to encode map e1056 with expected crc 2023-10-09T09:22:12.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:11 smithi163 bash[75425]: cluster 2023-10-09T09:22:10.364666+0000 osd.4 (osd.4) 499 : cluster 3 failed to encode map e1056 with expected crc 2023-10-09T09:22:12.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:11 smithi163 bash[75425]: cluster 2023-10-09T09:22:11.365829+0000 mon.a (mon.0) 1292 : cluster 0 osdmap e1057: 8 total, 8 up, 8 in 2023-10-09T09:22:12.931 INFO:tasks.rados.rados.0.smithi163.stdout:3944: done (2 left) 2023-10-09T09:22:12.931 INFO:tasks.rados.rados.0.smithi163.stdout:3945: delete oid 16 current snap is 496 2023-10-09T09:22:12.931 INFO:tasks.rados.rados.0.smithi163.stdout:3940: finishing write tid 1 to smithi16367185-6 2023-10-09T09:22:12.931 INFO:tasks.rados.rados.0.smithi163.stdout:3940: finishing write tid 2 to smithi16367185-6 2023-10-09T09:22:12.931 INFO:tasks.rados.rados.0.smithi163.stdout:3940: finishing write tid 3 to smithi16367185-6 2023-10-09T09:22:12.931 INFO:tasks.rados.rados.0.smithi163.stdout:3940: finishing write tid 4 to smithi16367185-6 2023-10-09T09:22:12.931 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 6 v 467 (ObjNum 1490 snap 496 seq_num 1490) dirty exists 2023-10-09T09:22:12.931 INFO:tasks.rados.rados.0.smithi163.stdout:3940: left oid 6 (ObjNum 1490 snap 496 seq_num 1490) 2023-10-09T09:22:12.931 INFO:tasks.rados.rados.0.smithi163.stdout:3941: finishing write tid 1 to smithi16367185-2 2023-10-09T09:22:12.932 INFO:tasks.rados.rados.0.smithi163.stdout:3941: finishing write tid 2 to smithi16367185-2 2023-10-09T09:22:12.932 INFO:tasks.rados.rados.0.smithi163.stdout:3941: finishing write tid 3 to smithi16367185-2 2023-10-09T09:22:12.932 INFO:tasks.rados.rados.0.smithi163.stdout:3941: finishing write tid 4 to smithi16367185-2 2023-10-09T09:22:12.932 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 2 v 1037 (ObjNum 1491 snap 496 seq_num 1491) dirty exists 2023-10-09T09:22:12.932 INFO:tasks.rados.rados.0.smithi163.stdout:3941: left oid 2 (ObjNum 1491 snap 496 seq_num 1491) 2023-10-09T09:22:12.932 INFO:tasks.rados.rados.0.smithi163.stdout:3940: done (2 left) 2023-10-09T09:22:12.932 INFO:tasks.rados.rados.0.smithi163.stdout:3941: done (1 left) 2023-10-09T09:22:12.932 INFO:tasks.rados.rados.0.smithi163.stdout:3945: done (0 left) 2023-10-09T09:22:12.932 INFO:tasks.rados.rados.0.smithi163.stdout:3946: write oid 40 current snap is 496 2023-10-09T09:22:12.932 INFO:tasks.rados.rados.0.smithi163.stdout:3946: seq_num 1493 ranges {692851=751945,2049969=466846} 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3946: writing smithi16367185-40 from 692851 to 1444796 tid 1 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3946: writing smithi16367185-40 from 2049969 to 2516815 tid 2 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3947: read oid 20 snap -1 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3947: expect (ObjNum 1488 snap 494 seq_num 1488) 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3948: read oid 23 snap -1 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3948: expect (ObjNum 1467 snap 488 seq_num 1467) 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3949: read oid 8 snap -1 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3949: expect (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3950: read oid 24 snap -1 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3950: expect (ObjNum 1477 snap 490 seq_num 1477) 2023-10-09T09:22:12.933 INFO:tasks.rados.rados.0.smithi163.stdout:3951: snap_remove snap 489 2023-10-09T09:22:12.934 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:12 smithi163 bash[78107]: cluster 2023-10-09T09:22:11.361481+0000 osd.5 (osd.5) 599 : cluster 3 failed to encode map e1057 with expected crc 2023-10-09T09:22:12.934 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:12 smithi163 bash[78107]: cluster 2023-10-09T09:22:11.362128+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e1057 with expected crc 2023-10-09T09:22:12.934 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:12 smithi163 bash[78107]: cluster 2023-10-09T09:22:11.493774+0000 mgr.y (mgr.34107) 469 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:22:12.934 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:12 smithi163 bash[78107]: cluster 2023-10-09T09:22:12.368367+0000 mon.a (mon.0) 1293 : cluster 0 osdmap e1058: 8 total, 8 up, 8 in 2023-10-09T09:22:12.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:12 smithi163 bash[75425]: cluster 2023-10-09T09:22:11.361481+0000 osd.5 (osd.5) 599 : cluster 3 failed to encode map e1057 with expected crc 2023-10-09T09:22:12.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:12 smithi163 bash[75425]: cluster 2023-10-09T09:22:11.362128+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e1057 with expected crc 2023-10-09T09:22:12.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:12 smithi163 bash[75425]: cluster 2023-10-09T09:22:11.493774+0000 mgr.y (mgr.34107) 469 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:22:12.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:12 smithi163 bash[75425]: cluster 2023-10-09T09:22:12.368367+0000 mon.a (mon.0) 1293 : cluster 0 osdmap e1058: 8 total, 8 up, 8 in 2023-10-09T09:22:12.935 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:11 smithi181 bash[59345]: cluster 2023-10-09T09:22:10.359006+0000 osd.5 (osd.5) 598 : cluster 3 failed to encode map e1056 with expected crc 2023-10-09T09:22:12.935 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:11 smithi181 bash[59345]: cluster 2023-10-09T09:22:10.364666+0000 osd.4 (osd.4) 499 : cluster 3 failed to encode map e1056 with expected crc 2023-10-09T09:22:12.935 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:11 smithi181 bash[59345]: cluster 2023-10-09T09:22:11.365829+0000 mon.a (mon.0) 1292 : cluster 0 osdmap e1057: 8 total, 8 up, 8 in 2023-10-09T09:22:12.935 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:12 smithi181 bash[59345]: cluster 2023-10-09T09:22:11.361481+0000 osd.5 (osd.5) 599 : cluster 3 failed to encode map e1057 with expected crc 2023-10-09T09:22:12.936 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:12 smithi181 bash[59345]: cluster 2023-10-09T09:22:11.362128+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e1057 with expected crc 2023-10-09T09:22:12.936 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:12 smithi181 bash[59345]: cluster 2023-10-09T09:22:11.493774+0000 mgr.y (mgr.34107) 469 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:22:12.937 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:12 smithi181 bash[59345]: cluster 2023-10-09T09:22:12.368367+0000 mon.a (mon.0) 1293 : cluster 0 osdmap e1058: 8 total, 8 up, 8 in 2023-10-09T09:22:13.372 INFO:tasks.rados.rados.0.smithi163.stdout:3951: done (5 left) 2023-10-09T09:22:13.385 INFO:tasks.rados.rados.0.smithi163.stdout:3952: read oid 2 snap -1 2023-10-09T09:22:13.385 INFO:tasks.rados.rados.0.smithi163.stdout:3952: expect (ObjNum 1491 snap 496 seq_num 1491) 2023-10-09T09:22:13.385 INFO:tasks.rados.rados.0.smithi163.stdout:3953: delete oid 12 current snap is 496 2023-10-09T09:22:13.385 INFO:tasks.rados.rados.0.smithi163.stdout:3950: expect (ObjNum 1477 snap 490 seq_num 1477) 2023-10-09T09:22:13.475 INFO:tasks.rados.rados.0.smithi163.stdout:3948: expect (ObjNum 1467 snap 488 seq_num 1467) 2023-10-09T09:22:13.611 INFO:tasks.rados.rados.0.smithi163.stdout:3947: expect (ObjNum 1488 snap 494 seq_num 1488) 2023-10-09T09:22:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:22:13] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:22:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[75425]: cluster 2023-10-09T09:22:12.364067+0000 osd.5 (osd.5) 600 : cluster 3 failed to encode map e1058 with expected crc 2023-10-09T09:22:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[75425]: cluster 2023-10-09T09:22:12.376523+0000 osd.4 (osd.4) 500 : cluster 3 failed to encode map e1058 with expected crc 2023-10-09T09:22:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[75425]: cluster 2023-10-09T09:22:12.391491+0000 osd.7 (osd.7) 457 : cluster 3 failed to encode map e1057 with expected crc 2023-10-09T09:22:13.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[75425]: cluster 2023-10-09T09:22:12.396721+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e1058 with expected crc 2023-10-09T09:22:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[75425]: cluster 2023-10-09T09:22:13.372113+0000 mon.a (mon.0) 1294 : cluster 0 osdmap e1059: 8 total, 8 up, 8 in 2023-10-09T09:22:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[78107]: cluster 2023-10-09T09:22:12.364067+0000 osd.5 (osd.5) 600 : cluster 3 failed to encode map e1058 with expected crc 2023-10-09T09:22:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[78107]: cluster 2023-10-09T09:22:12.376523+0000 osd.4 (osd.4) 500 : cluster 3 failed to encode map e1058 with expected crc 2023-10-09T09:22:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[78107]: cluster 2023-10-09T09:22:12.391491+0000 osd.7 (osd.7) 457 : cluster 3 failed to encode map e1057 with expected crc 2023-10-09T09:22:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[78107]: cluster 2023-10-09T09:22:12.396721+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e1058 with expected crc 2023-10-09T09:22:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:13 smithi163 bash[78107]: cluster 2023-10-09T09:22:13.372113+0000 mon.a (mon.0) 1294 : cluster 0 osdmap e1059: 8 total, 8 up, 8 in 2023-10-09T09:22:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:13 smithi181 bash[59345]: cluster 2023-10-09T09:22:12.364067+0000 osd.5 (osd.5) 600 : cluster 3 failed to encode map e1058 with expected crc 2023-10-09T09:22:13.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:13 smithi181 bash[59345]: cluster 2023-10-09T09:22:12.376523+0000 osd.4 (osd.4) 500 : cluster 3 failed to encode map e1058 with expected crc 2023-10-09T09:22:13.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:13 smithi181 bash[59345]: cluster 2023-10-09T09:22:12.391491+0000 osd.7 (osd.7) 457 : cluster 3 failed to encode map e1057 with expected crc 2023-10-09T09:22:13.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:13 smithi181 bash[59345]: cluster 2023-10-09T09:22:12.396721+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e1058 with expected crc 2023-10-09T09:22:13.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:13 smithi181 bash[59345]: cluster 2023-10-09T09:22:13.372113+0000 mon.a (mon.0) 1294 : cluster 0 osdmap e1059: 8 total, 8 up, 8 in 2023-10-09T09:22:13.853 INFO:tasks.rados.rados.0.smithi163.stdout:3946: finishing write tid 1 to smithi16367185-40 2023-10-09T09:22:13.853 INFO:tasks.rados.rados.0.smithi163.stdout:3946: finishing write tid 2 to smithi16367185-40 2023-10-09T09:22:13.853 INFO:tasks.rados.rados.0.smithi163.stdout:3946: finishing write tid 3 to smithi16367185-40 2023-10-09T09:22:13.853 INFO:tasks.rados.rados.0.smithi163.stdout:3946: finishing write tid 4 to smithi16367185-40 2023-10-09T09:22:13.854 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 40 v 2038 (ObjNum 1493 snap 496 seq_num 1493) dirty exists 2023-10-09T09:22:13.854 INFO:tasks.rados.rados.0.smithi163.stdout:3946: left oid 40 (ObjNum 1493 snap 496 seq_num 1493) 2023-10-09T09:22:13.854 INFO:tasks.rados.rados.0.smithi163.stdout:3949: expect (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:22:13.937 INFO:tasks.rados.rados.0.smithi163.stdout:3952: expect (ObjNum 1491 snap 496 seq_num 1491) 2023-10-09T09:22:14.081 INFO:tasks.rados.rados.0.smithi163.stdout:3946: done (6 left) 2023-10-09T09:22:14.350 INFO:tasks.rados.rados.0.smithi163.stdout:3947: done (5 left) 2023-10-09T09:22:14.351 INFO:tasks.rados.rados.0.smithi163.stdout:3948: done (4 left) 2023-10-09T09:22:14.352 INFO:tasks.rados.rados.0.smithi163.stdout:3949: done (3 left) 2023-10-09T09:22:14.352 INFO:tasks.rados.rados.0.smithi163.stdout:3950: done (2 left) 2023-10-09T09:22:14.352 INFO:tasks.rados.rados.0.smithi163.stdout:3952: done (1 left) 2023-10-09T09:22:14.352 INFO:tasks.rados.rados.0.smithi163.stdout:3953: done (0 left) 2023-10-09T09:22:14.352 INFO:tasks.rados.rados.0.smithi163.stdout:3954: rollback oid 43 current snap is 496 2023-10-09T09:22:14.352 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 43 to 487 2023-10-09T09:22:14.352 INFO:tasks.rados.rados.0.smithi163.stdout:3955: delete oid 7 current snap is 496 2023-10-09T09:22:14.354 INFO:tasks.rados.rados.0.smithi163.stdout:3954: finishing rollback tid 1 to smithi16367185-43 2023-10-09T09:22:14.494 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 0 (ObjNum 1953066355 snap 909207912 seq_num 825701939) dirty dne 2023-10-09T09:22:14.494 INFO:tasks.rados.rados.0.smithi163.stdout:3954: done (1 left) 2023-10-09T09:22:14.494 INFO:tasks.rados.rados.0.smithi163.stdout:3955: done (0 left) 2023-10-09T09:22:14.494 INFO:tasks.rados.rados.0.smithi163.stdout:3956: read oid 44 snap -1 2023-10-09T09:22:14.494 INFO:tasks.rados.rados.0.smithi163.stdout:3956: expect (ObjNum 1468 snap 488 seq_num 1468) 2023-10-09T09:22:14.494 INFO:tasks.rados.rados.0.smithi163.stdout:3957: write oid 30 current snap is 496 2023-10-09T09:22:14.494 INFO:tasks.rados.rados.0.smithi163.stdout:3957: seq_num 1496 ranges {743272=577692,2109062=730526} 2023-10-09T09:22:14.497 INFO:tasks.rados.rados.0.smithi163.stdout:3957: writing smithi16367185-30 from 743272 to 1320964 tid 1 2023-10-09T09:22:14.498 INFO:tasks.rados.rados.0.smithi163.stdout:3957: writing smithi16367185-30 from 2109062 to 2839588 tid 2 2023-10-09T09:22:14.498 INFO:tasks.rados.rados.0.smithi163.stdout:3958: read oid 18 snap -1 2023-10-09T09:22:14.498 INFO:tasks.rados.rados.0.smithi163.stdout:3958: expect deleted 2023-10-09T09:22:14.498 INFO:tasks.rados.rados.0.smithi163.stdout:3959: snap_create 2023-10-09T09:22:14.498 INFO:tasks.rados.rados.0.smithi163.stdout:3957: finishing write tid 1 to smithi16367185-30 2023-10-09T09:22:14.498 INFO:tasks.rados.rados.0.smithi163.stdout:3956: expect (ObjNum 1468 snap 488 seq_num 1468) 2023-10-09T09:22:14.499 INFO:tasks.rados.rados.0.smithi163.stdout:3957: finishing write tid 2 to smithi16367185-30 2023-10-09T09:22:14.499 INFO:tasks.rados.rados.0.smithi163.stdout:3957: finishing write tid 3 to smithi16367185-30 2023-10-09T09:22:14.499 INFO:tasks.rados.rados.0.smithi163.stdout:3957: finishing write tid 4 to smithi16367185-30 2023-10-09T09:22:14.499 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1406 (ObjNum 1496 snap 496 seq_num 1496) dirty exists 2023-10-09T09:22:14.499 INFO:tasks.rados.rados.0.smithi163.stdout:3957: left oid 30 (ObjNum 1496 snap 496 seq_num 1496) 2023-10-09T09:22:14.502 INFO:tasks.rados.rados.0.smithi163.stdout:3956: done (3 left) 2023-10-09T09:22:14.506 INFO:tasks.rados.rados.0.smithi163.stdout:3957: done (2 left) 2023-10-09T09:22:14.506 INFO:tasks.rados.rados.0.smithi163.stdout:3958: done (1 left) 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3959: done (0 left) 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3960: write oid 43 current snap is 497 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3960: seq_num 1497 ranges {760079=652941,2167544=474184} 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3960: writing smithi16367185-43 from 760079 to 1413020 tid 1 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3960: writing smithi16367185-43 from 2167544 to 2641728 tid 2 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3961: write oid 18 current snap is 497 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3961: seq_num 1498 ranges {776886=444543,1826026=734195,2987055=189831} 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3961: writing smithi16367185-18 from 776886 to 1221429 tid 1 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3961: writing smithi16367185-18 from 1826026 to 2560221 tid 2 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3961: writing smithi16367185-18 from 2987055 to 3176886 tid 3 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3962: write oid 3 current snap is 497 2023-10-09T09:22:14.507 INFO:tasks.rados.rados.0.smithi163.stdout:3962: seq_num 1499 ranges {793693=519792,1884508=477853,2898125=295568} 2023-10-09T09:22:14.508 INFO:tasks.rados.rados.0.smithi163.stdout:3962: writing smithi16367185-3 from 793693 to 1313485 tid 1 2023-10-09T09:22:14.508 INFO:tasks.rados.rados.0.smithi163.stdout:3962: writing smithi16367185-3 from 1884508 to 2362361 tid 2 2023-10-09T09:22:14.508 INFO:tasks.rados.rados.0.smithi163.stdout:3962: writing smithi16367185-3 from 2898125 to 3193693 tid 3 2023-10-09T09:22:14.508 INFO:tasks.rados.rados.0.smithi163.stdout:3963: read oid 8 snap 496 2023-10-09T09:22:14.508 INFO:tasks.rados.rados.0.smithi163.stdout:3963: expect (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:22:14.508 INFO:tasks.rados.rados.0.smithi163.stdout:3964: read oid 9 snap 485 2023-10-09T09:22:14.508 INFO:tasks.rados.rados.0.smithi163.stdout:3964: expect deleted 2023-10-09T09:22:14.508 INFO:tasks.rados.rados.0.smithi163.stdout:3965: snap_remove snap 476 2023-10-09T09:22:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:14 smithi163 bash[78107]: cluster 2023-10-09T09:22:13.366938+0000 osd.5 (osd.5) 601 : cluster 3 failed to encode map e1059 with expected crc 2023-10-09T09:22:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:14 smithi163 bash[78107]: cluster 2023-10-09T09:22:13.373751+0000 osd.7 (osd.7) 458 : cluster 3 failed to encode map e1059 with expected crc 2023-10-09T09:22:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:14 smithi163 bash[78107]: cluster 2023-10-09T09:22:13.494375+0000 mgr.y (mgr.34107) 470 : cluster 0 pgmap v1142: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:22:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:14 smithi163 bash[75425]: cluster 2023-10-09T09:22:13.366938+0000 osd.5 (osd.5) 601 : cluster 3 failed to encode map e1059 with expected crc 2023-10-09T09:22:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:14 smithi163 bash[75425]: cluster 2023-10-09T09:22:13.373751+0000 osd.7 (osd.7) 458 : cluster 3 failed to encode map e1059 with expected crc 2023-10-09T09:22:14.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:14 smithi163 bash[75425]: cluster 2023-10-09T09:22:13.494375+0000 mgr.y (mgr.34107) 470 : cluster 0 pgmap v1142: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:22:14.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:14 smithi181 bash[59345]: cluster 2023-10-09T09:22:13.366938+0000 osd.5 (osd.5) 601 : cluster 3 failed to encode map e1059 with expected crc 2023-10-09T09:22:14.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:14 smithi181 bash[59345]: cluster 2023-10-09T09:22:13.373751+0000 osd.7 (osd.7) 458 : cluster 3 failed to encode map e1059 with expected crc 2023-10-09T09:22:14.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:14 smithi181 bash[59345]: cluster 2023-10-09T09:22:13.494375+0000 mgr.y (mgr.34107) 470 : cluster 0 pgmap v1142: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 5 op/s 2023-10-09T09:22:15.368 INFO:tasks.rados.rados.0.smithi163.stdout:3965: done (5 left) 2023-10-09T09:22:15.608 INFO:tasks.rados.rados.0.smithi163.stdout:3966: snap_create 2023-10-09T09:22:15.608 INFO:tasks.rados.rados.0.smithi163.stdout:3960: finishing write tid 1 to smithi16367185-43 2023-10-09T09:22:15.608 INFO:tasks.rados.rados.0.smithi163.stdout:3960: finishing write tid 2 to smithi16367185-43 2023-10-09T09:22:15.609 INFO:tasks.rados.rados.0.smithi163.stdout:3960: finishing write tid 3 to smithi16367185-43 2023-10-09T09:22:15.609 INFO:tasks.rados.rados.0.smithi163.stdout:3960: finishing write tid 4 to smithi16367185-43 2023-10-09T09:22:15.609 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 43 v 1813 (ObjNum 1497 snap 497 seq_num 1497) dirty exists 2023-10-09T09:22:15.609 INFO:tasks.rados.rados.0.smithi163.stdout:3960: left oid 43 (ObjNum 1497 snap 497 seq_num 1497) 2023-10-09T09:22:15.609 INFO:tasks.rados.rados.0.smithi163.stdout:3961: finishing write tid 1 to smithi16367185-18 2023-10-09T09:22:15.609 INFO:tasks.rados.rados.0.smithi163.stdout:3961: finishing write tid 2 to smithi16367185-18 2023-10-09T09:22:15.609 INFO:tasks.rados.rados.0.smithi163.stdout:3961: finishing write tid 3 to smithi16367185-18 2023-10-09T09:22:15.609 INFO:tasks.rados.rados.0.smithi163.stdout:3961: finishing write tid 4 to smithi16367185-18 2023-10-09T09:22:15.612 INFO:tasks.rados.rados.0.smithi163.stdout:3961: finishing write tid 5 to smithi16367185-18 2023-10-09T09:22:15.612 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 18 v 1410 (ObjNum 1498 snap 497 seq_num 1498) dirty exists 2023-10-09T09:22:15.612 INFO:tasks.rados.rados.0.smithi163.stdout:3961: left oid 18 (ObjNum 1498 snap 497 seq_num 1498) 2023-10-09T09:22:15.612 INFO:tasks.rados.rados.0.smithi163.stdout:3962: finishing write tid 1 to smithi16367185-3 2023-10-09T09:22:15.612 INFO:tasks.rados.rados.0.smithi163.stdout:3962: finishing write tid 2 to smithi16367185-3 2023-10-09T09:22:15.612 INFO:tasks.rados.rados.0.smithi163.stdout:3962: finishing write tid 3 to smithi16367185-3 2023-10-09T09:22:15.612 INFO:tasks.rados.rados.0.smithi163.stdout:3962: finishing write tid 4 to smithi16367185-3 2023-10-09T09:22:15.612 INFO:tasks.rados.rados.0.smithi163.stdout:3962: finishing write tid 5 to smithi16367185-3 2023-10-09T09:22:15.613 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 3 v 1195 (ObjNum 1499 snap 497 seq_num 1499) dirty exists 2023-10-09T09:22:15.613 INFO:tasks.rados.rados.0.smithi163.stdout:3962: left oid 3 (ObjNum 1499 snap 497 seq_num 1499) 2023-10-09T09:22:15.613 INFO:tasks.rados.rados.0.smithi163.stdout:3963: expect (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:22:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:15 smithi163 bash[75425]: cluster 2023-10-09T09:22:14.420112+0000 osd.5 (osd.5) 602 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:15.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:15 smithi163 bash[75425]: cluster 2023-10-09T09:22:14.420581+0000 osd.7 (osd.7) 459 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:15.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:15 smithi163 bash[75425]: cluster 2023-10-09T09:22:14.423368+0000 mon.a (mon.0) 1295 : cluster 0 osdmap e1060: 8 total, 8 up, 8 in 2023-10-09T09:22:15.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:15 smithi163 bash[75425]: cluster 2023-10-09T09:22:14.436951+0000 osd.4 (osd.4) 501 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:15 smithi163 bash[75425]: cluster 2023-10-09T09:22:15.367583+0000 mon.a (mon.0) 1296 : cluster 0 osdmap e1061: 8 total, 8 up, 8 in 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:15 smithi163 bash[78107]: cluster 2023-10-09T09:22:14.420112+0000 osd.5 (osd.5) 602 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:15 smithi163 bash[78107]: cluster 2023-10-09T09:22:14.420581+0000 osd.7 (osd.7) 459 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:15 smithi163 bash[78107]: cluster 2023-10-09T09:22:14.423368+0000 mon.a (mon.0) 1295 : cluster 0 osdmap e1060: 8 total, 8 up, 8 in 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:15 smithi163 bash[78107]: cluster 2023-10-09T09:22:14.436951+0000 osd.4 (osd.4) 501 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:15.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:15 smithi163 bash[78107]: cluster 2023-10-09T09:22:15.367583+0000 mon.a (mon.0) 1296 : cluster 0 osdmap e1061: 8 total, 8 up, 8 in 2023-10-09T09:22:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:15 smithi181 bash[59345]: cluster 2023-10-09T09:22:14.420112+0000 osd.5 (osd.5) 602 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:15 smithi181 bash[59345]: cluster 2023-10-09T09:22:14.420581+0000 osd.7 (osd.7) 459 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:15 smithi181 bash[59345]: cluster 2023-10-09T09:22:14.423368+0000 mon.a (mon.0) 1295 : cluster 0 osdmap e1060: 8 total, 8 up, 8 in 2023-10-09T09:22:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:15 smithi181 bash[59345]: cluster 2023-10-09T09:22:14.436951+0000 osd.4 (osd.4) 501 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:15 smithi181 bash[59345]: cluster 2023-10-09T09:22:15.367583+0000 mon.a (mon.0) 1296 : cluster 0 osdmap e1061: 8 total, 8 up, 8 in 2023-10-09T09:22:16.377 INFO:tasks.rados.rados.0.smithi163.stdout:3960: done (5 left) 2023-10-09T09:22:16.419 INFO:tasks.rados.rados.0.smithi163.stdout:3961: done (4 left) 2023-10-09T09:22:16.419 INFO:tasks.rados.rados.0.smithi163.stdout:3962: done (3 left) 2023-10-09T09:22:16.419 INFO:tasks.rados.rados.0.smithi163.stdout:3963: done (2 left) 2023-10-09T09:22:16.419 INFO:tasks.rados.rados.0.smithi163.stdout:3964: done (1 left) 2023-10-09T09:22:16.628 INFO:tasks.rados.rados.0.smithi163.stdout:3966: done (0 left) 2023-10-09T09:22:16.628 INFO:tasks.rados.rados.0.smithi163.stdout:3967: write oid 44 current snap is 498 2023-10-09T09:22:16.629 INFO:tasks.rados.rados.0.smithi163.stdout:3967: seq_num 1500 ranges {410500=595041,1542990=621511,2925548=284952} 2023-10-09T09:22:16.630 INFO:tasks.rados.rados.0.smithi163.stdout:3967: writing smithi16367185-44 from 410500 to 1005541 tid 1 2023-10-09T09:22:16.630 INFO:tasks.rados.rados.0.smithi163.stdout:3967: writing smithi16367185-44 from 1542990 to 2164501 tid 2 2023-10-09T09:22:16.630 INFO:tasks.rados.rados.0.smithi163.stdout:3967: writing smithi16367185-44 from 2925548 to 3210500 tid 3 2023-10-09T09:22:16.630 INFO:tasks.rados.rados.0.smithi163.stdout:3968: rollback oid 33 current snap is 498 2023-10-09T09:22:16.630 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 33 to 464 2023-10-09T09:22:16.630 INFO:tasks.rados.rados.0.smithi163.stdout:3967: finishing write tid 1 to smithi16367185-44 2023-10-09T09:22:16.630 INFO:tasks.rados.rados.0.smithi163.stdout:3967: finishing write tid 2 to smithi16367185-44 2023-10-09T09:22:16.630 INFO:tasks.rados.rados.0.smithi163.stdout:3969: snap_remove snap 494 2023-10-09T09:22:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:16 smithi163 bash[78107]: cluster 2023-10-09T09:22:15.361580+0000 osd.5 (osd.5) 603 : cluster 3 failed to encode map e1061 with expected crc 2023-10-09T09:22:16.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:16 smithi163 bash[78107]: cluster 2023-10-09T09:22:15.362227+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:16.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:16 smithi163 bash[78107]: cluster 2023-10-09T09:22:15.495094+0000 mgr.y (mgr.34107) 471 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 198 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 654 KiB/s wr, 5 op/s 2023-10-09T09:22:16.734 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:16 smithi163 bash[78107]: cluster 2023-10-09T09:22:16.376879+0000 mon.a (mon.0) 1297 : cluster 0 osdmap e1062: 8 total, 8 up, 8 in 2023-10-09T09:22:16.734 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:16 smithi163 bash[75425]: cluster 2023-10-09T09:22:15.361580+0000 osd.5 (osd.5) 603 : cluster 3 failed to encode map e1061 with expected crc 2023-10-09T09:22:16.735 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:16 smithi163 bash[75425]: cluster 2023-10-09T09:22:15.362227+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:16.735 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:16 smithi163 bash[75425]: cluster 2023-10-09T09:22:15.495094+0000 mgr.y (mgr.34107) 471 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 198 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 654 KiB/s wr, 5 op/s 2023-10-09T09:22:16.735 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:16 smithi163 bash[75425]: cluster 2023-10-09T09:22:16.376879+0000 mon.a (mon.0) 1297 : cluster 0 osdmap e1062: 8 total, 8 up, 8 in 2023-10-09T09:22:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:16 smithi181 bash[59345]: cluster 2023-10-09T09:22:15.361580+0000 osd.5 (osd.5) 603 : cluster 3 failed to encode map e1061 with expected crc 2023-10-09T09:22:16.887 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:16 smithi181 bash[59345]: cluster 2023-10-09T09:22:15.362227+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e1060 with expected crc 2023-10-09T09:22:16.888 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:16 smithi181 bash[59345]: cluster 2023-10-09T09:22:15.495094+0000 mgr.y (mgr.34107) 471 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 198 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 654 KiB/s wr, 5 op/s 2023-10-09T09:22:16.888 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:16 smithi181 bash[59345]: cluster 2023-10-09T09:22:16.376879+0000 mon.a (mon.0) 1297 : cluster 0 osdmap e1062: 8 total, 8 up, 8 in 2023-10-09T09:22:17.381 INFO:tasks.rados.rados.0.smithi163.stdout:3969: done (2 left) 2023-10-09T09:22:17.381 INFO:tasks.rados.rados.0.smithi163.stdout:3970: read oid 5 snap -1 2023-10-09T09:22:17.381 INFO:tasks.rados.rados.0.smithi163.stdout:3970: expect (ObjNum 1426 snap 474 seq_num 1426) 2023-10-09T09:22:17.381 INFO:tasks.rados.rados.0.smithi163.stdout:3967: finishing write tid 3 to smithi16367185-44 2023-10-09T09:22:17.381 INFO:tasks.rados.rados.0.smithi163.stdout:3968: finishing rollback tid 1 to smithi16367185-33 2023-10-09T09:22:17.382 INFO:tasks.rados.rados.0.smithi163.stdout:3968: finishing rollback tid 2 to smithi16367185-33 2023-10-09T09:22:17.382 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 33 v 1832 (ObjNum 1338 snap 448 seq_num 1338) dirty exists 2023-10-09T09:22:17.382 INFO:tasks.rados.rados.0.smithi163.stdout:3967: finishing write tid 4 to smithi16367185-44 2023-10-09T09:22:17.382 INFO:tasks.rados.rados.0.smithi163.stdout:3967: finishing write tid 5 to smithi16367185-44 2023-10-09T09:22:17.382 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 44 v 3054 (ObjNum 1500 snap 498 seq_num 1500) dirty exists 2023-10-09T09:22:17.382 INFO:tasks.rados.rados.0.smithi163.stdout:3967: left oid 44 (ObjNum 1500 snap 498 seq_num 1500) 2023-10-09T09:22:17.382 INFO:tasks.rados.rados.0.smithi163.stdout:3967: done (2 left) 2023-10-09T09:22:17.382 INFO:tasks.rados.rados.0.smithi163.stdout:3968: done (1 left) 2023-10-09T09:22:17.382 INFO:tasks.rados.rados.0.smithi163.stdout:3971: snap_create 2023-10-09T09:22:17.393 INFO:tasks.rados.rados.0.smithi163.stdout:3970: expect (ObjNum 1426 snap 474 seq_num 1426) 2023-10-09T09:22:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:17 smithi163 bash[78107]: cluster 2023-10-09T09:22:16.372270+0000 osd.5 (osd.5) 604 : cluster 3 failed to encode map e1062 with expected crc 2023-10-09T09:22:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:17 smithi163 bash[78107]: cluster 2023-10-09T09:22:16.372970+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e1062 with expected crc 2023-10-09T09:22:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:17 smithi163 bash[78107]: cluster 2023-10-09T09:22:17.380318+0000 mon.a (mon.0) 1298 : cluster 0 osdmap e1063: 8 total, 8 up, 8 in 2023-10-09T09:22:17.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:17 smithi163 bash[78107]: cluster 2023-10-09T09:22:17.381507+0000 osd.4 (osd.4) 502 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:17.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:17 smithi163 bash[78107]: cluster 2023-10-09T09:22:17.386589+0000 osd.4 (osd.4) 503 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:17 smithi163 bash[75425]: cluster 2023-10-09T09:22:16.372270+0000 osd.5 (osd.5) 604 : cluster 3 failed to encode map e1062 with expected crc 2023-10-09T09:22:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:17 smithi163 bash[75425]: cluster 2023-10-09T09:22:16.372970+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e1062 with expected crc 2023-10-09T09:22:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:17 smithi163 bash[75425]: cluster 2023-10-09T09:22:17.380318+0000 mon.a (mon.0) 1298 : cluster 0 osdmap e1063: 8 total, 8 up, 8 in 2023-10-09T09:22:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:17 smithi163 bash[75425]: cluster 2023-10-09T09:22:17.381507+0000 osd.4 (osd.4) 502 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:17.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:17 smithi163 bash[75425]: cluster 2023-10-09T09:22:17.386589+0000 osd.4 (osd.4) 503 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:17 smithi181 bash[59345]: cluster 2023-10-09T09:22:16.372270+0000 osd.5 (osd.5) 604 : cluster 3 failed to encode map e1062 with expected crc 2023-10-09T09:22:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:17 smithi181 bash[59345]: cluster 2023-10-09T09:22:16.372970+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e1062 with expected crc 2023-10-09T09:22:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:17 smithi181 bash[59345]: cluster 2023-10-09T09:22:17.380318+0000 mon.a (mon.0) 1298 : cluster 0 osdmap e1063: 8 total, 8 up, 8 in 2023-10-09T09:22:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:17 smithi181 bash[59345]: cluster 2023-10-09T09:22:17.381507+0000 osd.4 (osd.4) 502 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:17 smithi181 bash[59345]: cluster 2023-10-09T09:22:17.386589+0000 osd.4 (osd.4) 503 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:18.384 INFO:tasks.rados.rados.0.smithi163.stdout:3970: done (1 left) 2023-10-09T09:22:18.384 INFO:tasks.rados.rados.0.smithi163.stdout:3971: done (0 left) 2023-10-09T09:22:18.384 INFO:tasks.rados.rados.0.smithi163.stdout:3972: write oid 29 current snap is 499 2023-10-09T09:22:18.384 INFO:tasks.rados.rados.0.smithi163.stdout:3972: seq_num 1501 ranges {427307=670290,1601472=481522,2552971=483056} 2023-10-09T09:22:18.391 INFO:tasks.rados.rados.0.smithi163.stdout:3972: writing smithi16367185-29 from 427307 to 1097597 tid 1 2023-10-09T09:22:18.396 INFO:tasks.rados.rados.0.smithi163.stdout:3972: writing smithi16367185-29 from 1601472 to 2082994 tid 2 2023-10-09T09:22:18.403 INFO:tasks.rados.rados.0.smithi163.stdout:3972: writing smithi16367185-29 from 2552971 to 3036027 tid 3 2023-10-09T09:22:18.403 INFO:tasks.rados.rados.0.smithi163.stdout:3973: snap_create 2023-10-09T09:22:18.403 INFO:tasks.rados.rados.0.smithi163.stdout:3972: finishing write tid 1 to smithi16367185-29 2023-10-09T09:22:18.406 INFO:tasks.rados.rados.0.smithi163.stdout:3972: finishing write tid 2 to smithi16367185-29 2023-10-09T09:22:18.406 INFO:tasks.rados.rados.0.smithi163.stdout:3972: finishing write tid 3 to smithi16367185-29 2023-10-09T09:22:18.407 INFO:tasks.rados.rados.0.smithi163.stdout:3972: finishing write tid 4 to smithi16367185-29 2023-10-09T09:22:18.407 INFO:tasks.rados.rados.0.smithi163.stdout:3972: finishing write tid 5 to smithi16367185-29 2023-10-09T09:22:18.407 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 29 v 1836 (ObjNum 1501 snap 499 seq_num 1501) dirty exists 2023-10-09T09:22:18.407 INFO:tasks.rados.rados.0.smithi163.stdout:3972: left oid 29 (ObjNum 1501 snap 499 seq_num 1501) 2023-10-09T09:22:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:18 smithi163 bash[78107]: cluster 2023-10-09T09:22:17.375232+0000 osd.5 (osd.5) 605 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:18 smithi163 bash[78107]: cluster 2023-10-09T09:22:17.473609+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:18 smithi163 bash[78107]: cluster 2023-10-09T09:22:17.495683+0000 mgr.y (mgr.34107) 472 : cluster 0 pgmap v1148: 89 pgs: 89 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.5 MiB/s wr, 22 op/s 2023-10-09T09:22:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:18 smithi163 bash[78107]: audit 2023-10-09T09:22:17.554210+0000 mon.a (mon.0) 1299 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:22:18.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:18 smithi163 bash[78107]: cluster 2023-10-09T09:22:18.383901+0000 mon.a (mon.0) 1300 : cluster 0 osdmap e1064: 8 total, 8 up, 8 in 2023-10-09T09:22:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:18 smithi163 bash[75425]: cluster 2023-10-09T09:22:17.375232+0000 osd.5 (osd.5) 605 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:18 smithi163 bash[75425]: cluster 2023-10-09T09:22:17.473609+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:18 smithi163 bash[75425]: cluster 2023-10-09T09:22:17.495683+0000 mgr.y (mgr.34107) 472 : cluster 0 pgmap v1148: 89 pgs: 89 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.5 MiB/s wr, 22 op/s 2023-10-09T09:22:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:18 smithi163 bash[75425]: audit 2023-10-09T09:22:17.554210+0000 mon.a (mon.0) 1299 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:22:18.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:18 smithi163 bash[75425]: cluster 2023-10-09T09:22:18.383901+0000 mon.a (mon.0) 1300 : cluster 0 osdmap e1064: 8 total, 8 up, 8 in 2023-10-09T09:22:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:18 smithi181 bash[59345]: cluster 2023-10-09T09:22:17.375232+0000 osd.5 (osd.5) 605 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:18 smithi181 bash[59345]: cluster 2023-10-09T09:22:17.473609+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:18 smithi181 bash[59345]: cluster 2023-10-09T09:22:17.495683+0000 mgr.y (mgr.34107) 472 : cluster 0 pgmap v1148: 89 pgs: 89 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.5 MiB/s wr, 22 op/s 2023-10-09T09:22:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:18 smithi181 bash[59345]: audit 2023-10-09T09:22:17.554210+0000 mon.a (mon.0) 1299 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:22:18.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:18 smithi181 bash[59345]: cluster 2023-10-09T09:22:18.383901+0000 mon.a (mon.0) 1300 : cluster 0 osdmap e1064: 8 total, 8 up, 8 in 2023-10-09T09:22:19.385 INFO:tasks.rados.rados.0.smithi163.stdout:3972: done (1 left) 2023-10-09T09:22:19.386 INFO:tasks.rados.rados.0.smithi163.stdout:3973: done (0 left) 2023-10-09T09:22:19.386 INFO:tasks.rados.rados.0.smithi163.stdout:3974: snap_create 2023-10-09T09:22:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:19 smithi163 bash[78107]: cluster 2023-10-09T09:22:17.820419+0000 osd.7 (osd.7) 460 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:19 smithi163 bash[78107]: cluster 2023-10-09T09:22:18.379647+0000 osd.5 (osd.5) 606 : cluster 3 failed to encode map e1064 with expected crc 2023-10-09T09:22:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:19 smithi163 bash[78107]: cluster 2023-10-09T09:22:18.382332+0000 osd.7 (osd.7) 461 : cluster 3 failed to encode map e1064 with expected crc 2023-10-09T09:22:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:19 smithi163 bash[78107]: cluster 2023-10-09T09:22:18.902091+0000 osd.4 (osd.4) 504 : cluster 3 failed to encode map e1064 with expected crc 2023-10-09T09:22:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:19 smithi163 bash[78107]: cluster 2023-10-09T09:22:19.385507+0000 mon.a (mon.0) 1301 : cluster 0 osdmap e1065: 8 total, 8 up, 8 in 2023-10-09T09:22:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:19 smithi163 bash[75425]: cluster 2023-10-09T09:22:17.820419+0000 osd.7 (osd.7) 460 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:19 smithi163 bash[75425]: cluster 2023-10-09T09:22:18.379647+0000 osd.5 (osd.5) 606 : cluster 3 failed to encode map e1064 with expected crc 2023-10-09T09:22:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:19 smithi163 bash[75425]: cluster 2023-10-09T09:22:18.382332+0000 osd.7 (osd.7) 461 : cluster 3 failed to encode map e1064 with expected crc 2023-10-09T09:22:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:19 smithi163 bash[75425]: cluster 2023-10-09T09:22:18.902091+0000 osd.4 (osd.4) 504 : cluster 3 failed to encode map e1064 with expected crc 2023-10-09T09:22:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:19 smithi163 bash[75425]: cluster 2023-10-09T09:22:19.385507+0000 mon.a (mon.0) 1301 : cluster 0 osdmap e1065: 8 total, 8 up, 8 in 2023-10-09T09:22:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:19 smithi181 bash[59345]: cluster 2023-10-09T09:22:17.820419+0000 osd.7 (osd.7) 460 : cluster 3 failed to encode map e1063 with expected crc 2023-10-09T09:22:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:19 smithi181 bash[59345]: cluster 2023-10-09T09:22:18.379647+0000 osd.5 (osd.5) 606 : cluster 3 failed to encode map e1064 with expected crc 2023-10-09T09:22:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:19 smithi181 bash[59345]: cluster 2023-10-09T09:22:18.382332+0000 osd.7 (osd.7) 461 : cluster 3 failed to encode map e1064 with expected crc 2023-10-09T09:22:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:19 smithi181 bash[59345]: cluster 2023-10-09T09:22:18.902091+0000 osd.4 (osd.4) 504 : cluster 3 failed to encode map e1064 with expected crc 2023-10-09T09:22:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:19 smithi181 bash[59345]: cluster 2023-10-09T09:22:19.385507+0000 mon.a (mon.0) 1301 : cluster 0 osdmap e1065: 8 total, 8 up, 8 in 2023-10-09T09:22:20.369 INFO:tasks.rados.rados.0.smithi163.stdout:3974: done (0 left) 2023-10-09T09:22:20.369 INFO:tasks.rados.rados.0.smithi163.stdout:3975: read oid 8 snap -1 2023-10-09T09:22:20.369 INFO:tasks.rados.rados.0.smithi163.stdout:3975: expect (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:22:20.369 INFO:tasks.rados.rados.0.smithi163.stdout:3976: read oid 38 snap -1 2023-10-09T09:22:20.369 INFO:tasks.rados.rados.0.smithi163.stdout:3976: expect deleted 2023-10-09T09:22:20.370 INFO:tasks.rados.rados.0.smithi163.stdout:3977: rollback oid 26 current snap is 501 2023-10-09T09:22:20.370 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 26 to 498 2023-10-09T09:22:20.370 INFO:tasks.rados.rados.0.smithi163.stdout:3978: snap_create 2023-10-09T09:22:20.377 INFO:tasks.rados.rados.0.smithi163.stdout:3977: finishing rollback tid 0 to smithi16367185-26 2023-10-09T09:22:20.377 INFO:tasks.rados.rados.0.smithi163.stdout:3977: finishing rollback tid 1 to smithi16367185-26 2023-10-09T09:22:20.377 INFO:tasks.rados.rados.0.smithi163.stdout:3977: finishing rollback tid 2 to smithi16367185-26 2023-10-09T09:22:20.378 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 26 v 1160 (ObjNum 1485 snap 492 seq_num 1485) dirty exists 2023-10-09T09:22:20.389 INFO:tasks.rados.rados.0.smithi163.stdout:3975: expect (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:22:20.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:20 smithi163 bash[75425]: cluster 2023-10-09T09:22:19.380030+0000 osd.5 (osd.5) 607 : cluster 3 failed to encode map e1065 with expected crc 2023-10-09T09:22:20.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:20 smithi163 bash[75425]: cluster 2023-10-09T09:22:19.496302+0000 mgr.y (mgr.34107) 473 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 889 KiB/s wr, 16 op/s 2023-10-09T09:22:20.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:20 smithi163 bash[75425]: cluster 2023-10-09T09:22:20.364509+0000 osd.4 (osd.4) 505 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:20.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:20 smithi163 bash[75425]: cluster 2023-10-09T09:22:20.368465+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e1066: 8 total, 8 up, 8 in 2023-10-09T09:22:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:20 smithi163 bash[78107]: cluster 2023-10-09T09:22:19.380030+0000 osd.5 (osd.5) 607 : cluster 3 failed to encode map e1065 with expected crc 2023-10-09T09:22:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:20 smithi163 bash[78107]: cluster 2023-10-09T09:22:19.496302+0000 mgr.y (mgr.34107) 473 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 889 KiB/s wr, 16 op/s 2023-10-09T09:22:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:20 smithi163 bash[78107]: cluster 2023-10-09T09:22:20.364509+0000 osd.4 (osd.4) 505 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:20.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:20 smithi163 bash[78107]: cluster 2023-10-09T09:22:20.368465+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e1066: 8 total, 8 up, 8 in 2023-10-09T09:22:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:20 smithi181 bash[59345]: cluster 2023-10-09T09:22:19.380030+0000 osd.5 (osd.5) 607 : cluster 3 failed to encode map e1065 with expected crc 2023-10-09T09:22:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:20 smithi181 bash[59345]: cluster 2023-10-09T09:22:19.496302+0000 mgr.y (mgr.34107) 473 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 889 KiB/s wr, 16 op/s 2023-10-09T09:22:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:20 smithi181 bash[59345]: cluster 2023-10-09T09:22:20.364509+0000 osd.4 (osd.4) 505 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:20.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:20 smithi181 bash[59345]: cluster 2023-10-09T09:22:20.368465+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e1066: 8 total, 8 up, 8 in 2023-10-09T09:22:21.373 INFO:tasks.rados.rados.0.smithi163.stdout:3975: done (3 left) 2023-10-09T09:22:21.373 INFO:tasks.rados.rados.0.smithi163.stdout:3976: done (2 left) 2023-10-09T09:22:21.373 INFO:tasks.rados.rados.0.smithi163.stdout:3977: done (1 left) 2023-10-09T09:22:21.373 INFO:tasks.rados.rados.0.smithi163.stdout:3978: done (0 left) 2023-10-09T09:22:21.373 INFO:tasks.rados.rados.0.smithi163.stdout:3979: read oid 35 snap -1 2023-10-09T09:22:21.373 INFO:tasks.rados.rados.0.smithi163.stdout:3979: expect deleted 2023-10-09T09:22:21.374 INFO:tasks.rados.rados.0.smithi163.stdout:3980: read oid 28 snap -1 2023-10-09T09:22:21.374 INFO:tasks.rados.rados.0.smithi163.stdout:3980: expect (ObjNum 1433 snap 477 seq_num 1433) 2023-10-09T09:22:21.374 INFO:tasks.rados.rados.0.smithi163.stdout:3981: read oid 8 snap 485 2023-10-09T09:22:21.374 INFO:tasks.rados.rados.0.smithi163.stdout:3981: expect (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:22:21.374 INFO:tasks.rados.rados.0.smithi163.stdout:3982: write oid 36 current snap is 502 2023-10-09T09:22:21.374 INFO:tasks.rados.rados.0.smithi163.stdout:3982: seq_num 1502 ranges {444114=745539,1943601=625180} 2023-10-09T09:22:21.380 INFO:tasks.rados.rados.0.smithi163.stdout:3982: writing smithi16367185-36 from 444114 to 1189653 tid 1 2023-10-09T09:22:21.392 INFO:tasks.rados.rados.0.smithi163.stdout:3982: writing smithi16367185-36 from 1943601 to 2568781 tid 2 2023-10-09T09:22:21.392 INFO:tasks.rados.rados.0.smithi163.stdout:3983: write oid 21 current snap is 502 2023-10-09T09:22:21.392 INFO:tasks.rados.rados.0.smithi163.stdout:3983: seq_num 1503 ranges {460921=420788,1602083=485191,2491464=621953} 2023-10-09T09:22:21.397 INFO:tasks.rados.rados.0.smithi163.stdout:3983: writing smithi16367185-21 from 460921 to 881709 tid 1 2023-10-09T09:22:21.404 INFO:tasks.rados.rados.0.smithi163.stdout:3983: writing smithi16367185-21 from 1602083 to 2087274 tid 2 2023-10-09T09:22:21.410 INFO:tasks.rados.rados.0.smithi163.stdout:3983: writing smithi16367185-21 from 2491464 to 3113417 tid 3 2023-10-09T09:22:21.410 INFO:tasks.rados.rados.0.smithi163.stdout:3984: rollback oid 41 current snap is 502 2023-10-09T09:22:21.410 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 41 to 496 2023-10-09T09:22:21.410 INFO:tasks.rados.rados.0.smithi163.stdout:3980: expect (ObjNum 1433 snap 477 seq_num 1433) 2023-10-09T09:22:21.485 INFO:tasks.rados.rados.0.smithi163.stdout:3982: finishing write tid 1 to smithi16367185-36 2023-10-09T09:22:21.485 INFO:tasks.rados.rados.0.smithi163.stdout:3981: expect (ObjNum 1394 snap 464 seq_num 1394) 2023-10-09T09:22:21.602 INFO:tasks.rados.rados.0.smithi163.stdout:3982: finishing write tid 2 to smithi16367185-36 2023-10-09T09:22:21.602 INFO:tasks.rados.rados.0.smithi163.stdout:3982: finishing write tid 3 to smithi16367185-36 2023-10-09T09:22:21.602 INFO:tasks.rados.rados.0.smithi163.stdout:3982: finishing write tid 4 to smithi16367185-36 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 36 v 1205 (ObjNum 1502 snap 502 seq_num 1502) dirty exists 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:3982: left oid 36 (ObjNum 1502 snap 502 seq_num 1502) 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:3983: finishing write tid 1 to smithi16367185-21 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:3983: finishing write tid 2 to smithi16367185-21 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:3984: finishing rollback tid 0 to smithi16367185-41 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:3984: finishing rollback tid 1 to smithi16367185-41 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:3983: finishing write tid 3 to smithi16367185-21 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:3984: finishing rollback tid 2 to smithi16367185-41 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 41 v 388 (ObjNum 1451 snap 483 seq_num 1451) dirty exists 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:3983: finishing write tid 4 to smithi16367185-21 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:3983: finishing write tid 5 to smithi16367185-21 2023-10-09T09:22:21.603 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 21 v 1013 (ObjNum 1503 snap 502 seq_num 1503) dirty exists 2023-10-09T09:22:21.604 INFO:tasks.rados.rados.0.smithi163.stdout:3983: left oid 21 (ObjNum 1503 snap 502 seq_num 1503) 2023-10-09T09:22:21.604 INFO:tasks.rados.rados.0.smithi163.stdout:3979: done (5 left) 2023-10-09T09:22:21.604 INFO:tasks.rados.rados.0.smithi163.stdout:3980: done (4 left) 2023-10-09T09:22:21.604 INFO:tasks.rados.rados.0.smithi163.stdout:3981: done (3 left) 2023-10-09T09:22:21.604 INFO:tasks.rados.rados.0.smithi163.stdout:3982: done (2 left) 2023-10-09T09:22:21.604 INFO:tasks.rados.rados.0.smithi163.stdout:3983: done (1 left) 2023-10-09T09:22:21.604 INFO:tasks.rados.rados.0.smithi163.stdout:3984: done (0 left) 2023-10-09T09:22:21.604 INFO:tasks.rados.rados.0.smithi163.stdout:3985: write oid 23 current snap is 502 2023-10-09T09:22:21.604 INFO:tasks.rados.rados.0.smithi163.stdout:3985: seq_num 1504 ranges {477728=496037,1660565=628849,2918887=358841} 2023-10-09T09:22:21.612 INFO:tasks.rados.rados.0.smithi163.stdout:3985: writing smithi16367185-23 from 477728 to 973765 tid 1 2023-10-09T09:22:21.621 INFO:tasks.rados.rados.0.smithi163.stdout:3985: writing smithi16367185-23 from 1660565 to 2289414 tid 2 2023-10-09T09:22:21.627 INFO:tasks.rados.rados.0.smithi163.stdout:3985: writing smithi16367185-23 from 2918887 to 3277728 tid 3 2023-10-09T09:22:21.627 INFO:tasks.rados.rados.0.smithi163.stdout:3986: rollback oid 28 current snap is 502 2023-10-09T09:22:21.627 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 28 to 485 2023-10-09T09:22:21.627 INFO:tasks.rados.rados.0.smithi163.stdout:3987: rollback oid 17 current snap is 502 2023-10-09T09:22:21.628 INFO:tasks.rados.rados.0.smithi163.stdout:rollback oid 17 to 500 2023-10-09T09:22:21.628 INFO:tasks.rados.rados.0.smithi163.stdout:3985: finishing write tid 1 to smithi16367185-23 2023-10-09T09:22:21.628 INFO:tasks.rados.rados.0.smithi163.stdout:3988: delete oid 25 current snap is 502 2023-10-09T09:22:21.628 INFO:tasks.rados.rados.0.smithi163.stdout:3985: finishing write tid 2 to smithi16367185-23 2023-10-09T09:22:21.632 INFO:tasks.rados.rados.0.smithi163.stdout:3988: done (3 left) 2023-10-09T09:22:21.632 INFO:tasks.rados.rados.0.smithi163.stdout:3989: delete oid 29 current snap is 502 2023-10-09T09:22:21.633 INFO:tasks.rados.rados.0.smithi163.stdout:3986: finishing rollback tid 0 to smithi16367185-28 2023-10-09T09:22:21.633 INFO:tasks.rados.rados.0.smithi163.stdout:3986: finishing rollback tid 1 to smithi16367185-28 2023-10-09T09:22:21.634 INFO:tasks.rados.rados.0.smithi163.stdout:3985: finishing write tid 3 to smithi16367185-23 2023-10-09T09:22:21.634 INFO:tasks.rados.rados.0.smithi163.stdout:3985: finishing write tid 4 to smithi16367185-23 2023-10-09T09:22:21.634 INFO:tasks.rados.rados.0.smithi163.stdout:3985: finishing write tid 5 to smithi16367185-23 2023-10-09T09:22:21.634 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 23 v 1863 (ObjNum 1504 snap 502 seq_num 1504) dirty exists 2023-10-09T09:22:21.634 INFO:tasks.rados.rados.0.smithi163.stdout:3985: left oid 23 (ObjNum 1504 snap 502 seq_num 1504) 2023-10-09T09:22:21.634 INFO:tasks.rados.rados.0.smithi163.stdout:3987: finishing rollback tid 0 to smithi16367185-17 2023-10-09T09:22:21.635 INFO:tasks.rados.rados.0.smithi163.stdout:3987: finishing rollback tid 1 to smithi16367185-17 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:3986: finishing rollback tid 2 to smithi16367185-28 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 28 v 2220 (ObjNum 1433 snap 477 seq_num 1433) dirty exists 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:3987: finishing rollback tid 2 to smithi16367185-17 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 17 v 2052 (ObjNum 1434 snap 477 seq_num 1434) dirty exists 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:3985: done (3 left) 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:3986: done (2 left) 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:3987: done (1 left) 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:3989: done (0 left) 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:3990: read oid 12 snap -1 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:3990: expect deleted 2023-10-09T09:22:21.636 INFO:tasks.rados.rados.0.smithi163.stdout:3991: read oid 28 snap -1 2023-10-09T09:22:21.637 INFO:tasks.rados.rados.0.smithi163.stdout:3991: expect (ObjNum 1433 snap 477 seq_num 1433) 2023-10-09T09:22:21.637 INFO:tasks.rados.rados.0.smithi163.stdout:3992: snap_create 2023-10-09T09:22:21.637 INFO:tasks.rados.rados.0.smithi163.stdout:3991: expect (ObjNum 1433 snap 477 seq_num 1433) 2023-10-09T09:22:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[78107]: cluster 2023-10-09T09:22:19.724518+0000 osd.7 (osd.7) 462 : cluster 3 failed to encode map e1065 with expected crc 2023-10-09T09:22:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[78107]: cluster 2023-10-09T09:22:19.724593+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e1065 with expected crc 2023-10-09T09:22:21.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[78107]: cluster 2023-10-09T09:22:20.363688+0000 osd.5 (osd.5) 608 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[78107]: cluster 2023-10-09T09:22:20.370009+0000 osd.7 (osd.7) 463 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[78107]: cluster 2023-10-09T09:22:20.374022+0000 osd.7 (osd.7) 464 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[78107]: cluster 2023-10-09T09:22:20.397928+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[78107]: cluster 2023-10-09T09:22:21.372675+0000 mon.a (mon.0) 1303 : cluster 0 osdmap e1067: 8 total, 8 up, 8 in 2023-10-09T09:22:21.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[78107]: cluster 2023-10-09T09:22:21.386253+0000 osd.4 (osd.4) 506 : cluster 3 failed to encode map e1067 with expected crc 2023-10-09T09:22:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[75425]: cluster 2023-10-09T09:22:19.724518+0000 osd.7 (osd.7) 462 : cluster 3 failed to encode map e1065 with expected crc 2023-10-09T09:22:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[75425]: cluster 2023-10-09T09:22:19.724593+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e1065 with expected crc 2023-10-09T09:22:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[75425]: cluster 2023-10-09T09:22:20.363688+0000 osd.5 (osd.5) 608 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[75425]: cluster 2023-10-09T09:22:20.370009+0000 osd.7 (osd.7) 463 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[75425]: cluster 2023-10-09T09:22:20.374022+0000 osd.7 (osd.7) 464 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[75425]: cluster 2023-10-09T09:22:20.397928+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[75425]: cluster 2023-10-09T09:22:21.372675+0000 mon.a (mon.0) 1303 : cluster 0 osdmap e1067: 8 total, 8 up, 8 in 2023-10-09T09:22:21.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:21 smithi163 bash[75425]: cluster 2023-10-09T09:22:21.386253+0000 osd.4 (osd.4) 506 : cluster 3 failed to encode map e1067 with expected crc 2023-10-09T09:22:21.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:21 smithi181 bash[59345]: cluster 2023-10-09T09:22:19.724518+0000 osd.7 (osd.7) 462 : cluster 3 failed to encode map e1065 with expected crc 2023-10-09T09:22:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:21 smithi181 bash[59345]: cluster 2023-10-09T09:22:19.724593+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e1065 with expected crc 2023-10-09T09:22:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:21 smithi181 bash[59345]: cluster 2023-10-09T09:22:20.363688+0000 osd.5 (osd.5) 608 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:21 smithi181 bash[59345]: cluster 2023-10-09T09:22:20.370009+0000 osd.7 (osd.7) 463 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:21 smithi181 bash[59345]: cluster 2023-10-09T09:22:20.374022+0000 osd.7 (osd.7) 464 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:21 smithi181 bash[59345]: cluster 2023-10-09T09:22:20.397928+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e1066 with expected crc 2023-10-09T09:22:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:21 smithi181 bash[59345]: cluster 2023-10-09T09:22:21.372675+0000 mon.a (mon.0) 1303 : cluster 0 osdmap e1067: 8 total, 8 up, 8 in 2023-10-09T09:22:21.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:21 smithi181 bash[59345]: cluster 2023-10-09T09:22:21.386253+0000 osd.4 (osd.4) 506 : cluster 3 failed to encode map e1067 with expected crc 2023-10-09T09:22:22.472 INFO:tasks.rados.rados.0.smithi163.stdout:3990: done (2 left) 2023-10-09T09:22:22.472 INFO:tasks.rados.rados.0.smithi163.stdout:3991: done (1 left) 2023-10-09T09:22:22.472 INFO:tasks.rados.rados.0.smithi163.stdout:3992: done (0 left) 2023-10-09T09:22:22.472 INFO:tasks.rados.rados.0.smithi163.stdout:3993: write oid 15 current snap is 503 2023-10-09T09:22:22.472 INFO:tasks.rados.rados.0.smithi163.stdout:3993: seq_num 1507 ranges {528149=438137,1436011=492529,2601156=667041} 2023-10-09T09:22:22.477 INFO:tasks.rados.rados.0.smithi163.stdout:3993: writing smithi16367185-15 from 528149 to 966286 tid 1 2023-10-09T09:22:22.483 INFO:tasks.rados.rados.0.smithi163.stdout:3993: writing smithi16367185-15 from 1436011 to 1928540 tid 2 2023-10-09T09:22:22.492 INFO:tasks.rados.rados.0.smithi163.stdout:3993: writing smithi16367185-15 from 2601156 to 3268197 tid 3 2023-10-09T09:22:22.492 INFO:tasks.rados.rados.0.smithi163.stdout:3994: read oid 41 snap -1 2023-10-09T09:22:22.492 INFO:tasks.rados.rados.0.smithi163.stdout:3994: expect (ObjNum 1451 snap 483 seq_num 1451) 2023-10-09T09:22:22.492 INFO:tasks.rados.rados.0.smithi163.stdout:3995: snap_remove snap 498 2023-10-09T09:22:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:22 smithi163 bash[78107]: cluster 2023-10-09T09:22:21.368118+0000 osd.5 (osd.5) 609 : cluster 3 failed to encode map e1067 with expected crc 2023-10-09T09:22:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:22 smithi163 bash[78107]: cluster 2023-10-09T09:22:21.379322+0000 osd.7 (osd.7) 465 : cluster 3 failed to encode map e1067 with expected crc 2023-10-09T09:22:22.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:22 smithi163 bash[78107]: cluster 2023-10-09T09:22:21.496879+0000 mgr.y (mgr.34107) 474 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 766 KiB/s wr, 7 op/s 2023-10-09T09:22:22.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:22 smithi163 bash[75425]: cluster 2023-10-09T09:22:21.368118+0000 osd.5 (osd.5) 609 : cluster 3 failed to encode map e1067 with expected crc 2023-10-09T09:22:22.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:22 smithi163 bash[75425]: cluster 2023-10-09T09:22:21.379322+0000 osd.7 (osd.7) 465 : cluster 3 failed to encode map e1067 with expected crc 2023-10-09T09:22:22.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:22 smithi163 bash[75425]: cluster 2023-10-09T09:22:21.496879+0000 mgr.y (mgr.34107) 474 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 766 KiB/s wr, 7 op/s 2023-10-09T09:22:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:22 smithi181 bash[59345]: cluster 2023-10-09T09:22:21.368118+0000 osd.5 (osd.5) 609 : cluster 3 failed to encode map e1067 with expected crc 2023-10-09T09:22:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:22 smithi181 bash[59345]: cluster 2023-10-09T09:22:21.379322+0000 osd.7 (osd.7) 465 : cluster 3 failed to encode map e1067 with expected crc 2023-10-09T09:22:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:22 smithi181 bash[59345]: cluster 2023-10-09T09:22:21.496879+0000 mgr.y (mgr.34107) 474 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 766 KiB/s wr, 7 op/s 2023-10-09T09:22:23.477 INFO:tasks.rados.rados.0.smithi163.stdout:3995: done (2 left) 2023-10-09T09:22:23.477 INFO:tasks.rados.rados.0.smithi163.stdout:3996: write oid 30 current snap is 503 2023-10-09T09:22:23.477 INFO:tasks.rados.rados.0.smithi163.stdout:3993: finishing write tid 1 to smithi16367185-15 2023-10-09T09:22:23.477 INFO:tasks.rados.rados.0.smithi163.stdout:3996: seq_num 1508 ranges {544956=513386,1494493=636187,2628579=594666} 2023-10-09T09:22:23.485 INFO:tasks.rados.rados.0.smithi163.stdout:3996: writing smithi16367185-30 from 544956 to 1058342 tid 1 2023-10-09T09:22:23.493 INFO:tasks.rados.rados.0.smithi163.stdout:3996: writing smithi16367185-30 from 1494493 to 2130680 tid 2 2023-10-09T09:22:23.501 INFO:tasks.rados.rados.0.smithi163.stdout:3996: writing smithi16367185-30 from 2628579 to 3223245 tid 3 2023-10-09T09:22:23.503 INFO:tasks.rados.rados.0.smithi163.stdout:3993: finishing write tid 2 to smithi16367185-15 2023-10-09T09:22:23.503 INFO:tasks.rados.rados.0.smithi163.stdout:3994: expect (ObjNum 1451 snap 483 seq_num 1451) 2023-10-09T09:22:23.626 INFO:tasks.rados.rados.0.smithi163.stdout:3993: finishing write tid 3 to smithi16367185-15 2023-10-09T09:22:23.626 INFO:tasks.rados.rados.0.smithi163.stdout:3993: finishing write tid 4 to smithi16367185-15 2023-10-09T09:22:23.626 INFO:tasks.rados.rados.0.smithi163.stdout:3993: finishing write tid 5 to smithi16367185-15 2023-10-09T09:22:23.626 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 15 v 2057 (ObjNum 1507 snap 503 seq_num 1507) dirty exists 2023-10-09T09:22:23.626 INFO:tasks.rados.rados.0.smithi163.stdout:3993: left oid 15 (ObjNum 1507 snap 503 seq_num 1507) 2023-10-09T09:22:23.626 INFO:tasks.rados.rados.0.smithi163.stdout:3996: finishing write tid 1 to smithi16367185-30 2023-10-09T09:22:23.626 INFO:tasks.rados.rados.0.smithi163.stdout:3996: finishing write tid 2 to smithi16367185-30 2023-10-09T09:22:23.626 INFO:tasks.rados.rados.0.smithi163.stdout:3996: finishing write tid 3 to smithi16367185-30 2023-10-09T09:22:23.626 INFO:tasks.rados.rados.0.smithi163.stdout:3996: finishing write tid 4 to smithi16367185-30 2023-10-09T09:22:23.627 INFO:tasks.rados.rados.0.smithi163.stdout:3996: finishing write tid 5 to smithi16367185-30 2023-10-09T09:22:23.627 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 30 v 1421 (ObjNum 1508 snap 503 seq_num 1508) dirty exists 2023-10-09T09:22:23.627 INFO:tasks.rados.rados.0.smithi163.stdout:3996: left oid 30 (ObjNum 1508 snap 503 seq_num 1508) 2023-10-09T09:22:23.627 INFO:tasks.rados.rados.0.smithi163.stdout:3993: done (2 left) 2023-10-09T09:22:23.627 INFO:tasks.rados.rados.0.smithi163.stdout:3994: done (1 left) 2023-10-09T09:22:23.627 INFO:tasks.rados.rados.0.smithi163.stdout:3996: done (0 left) 2023-10-09T09:22:23.627 INFO:tasks.rados.rados.0.smithi163.stdout:3997: write oid 19 current snap is 503 2023-10-09T09:22:23.627 INFO:tasks.rados.rados.0.smithi163.stdout:3997: seq_num 1509 ranges {561763=588635,1552975=496198,2656002=405938} 2023-10-09T09:22:23.637 INFO:tasks.rados.rados.0.smithi163.stdout:3997: writing smithi16367185-19 from 561763 to 1150398 tid 1 2023-10-09T09:22:23.643 INFO:tasks.rados.rados.0.smithi163.stdout:3997: writing smithi16367185-19 from 1552975 to 2049173 tid 2 2023-10-09T09:22:23.649 INFO:tasks.rados.rados.0.smithi163.stdout:3997: writing smithi16367185-19 from 2656002 to 3061940 tid 3 2023-10-09T09:22:23.649 INFO:tasks.rados.rados.0.smithi163.stdout:3998: snap_create 2023-10-09T09:22:23.650 INFO:tasks.rados.rados.0.smithi163.stdout:3997: finishing write tid 1 to smithi16367185-19 2023-10-09T09:22:23.650 INFO:tasks.rados.rados.0.smithi163.stdout:3997: finishing write tid 2 to smithi16367185-19 2023-10-09T09:22:23.653 INFO:tasks.rados.rados.0.smithi163.stdout:3997: finishing write tid 3 to smithi16367185-19 2023-10-09T09:22:23.659 INFO:tasks.rados.rados.0.smithi163.stdout:3997: finishing write tid 4 to smithi16367185-19 2023-10-09T09:22:23.659 INFO:tasks.rados.rados.0.smithi163.stdout:3997: finishing write tid 5 to smithi16367185-19 2023-10-09T09:22:23.659 INFO:tasks.rados.rados.0.smithi163.stdout:update_object_version oid 19 v 1167 (ObjNum 1509 snap 503 seq_num 1509) dirty exists 2023-10-09T09:22:23.660 INFO:tasks.rados.rados.0.smithi163.stdout:3997: left oid 19 (ObjNum 1509 snap 503 seq_num 1509) 2023-10-09T09:22:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:22:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:22:23] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.33.4" 2023-10-09T09:22:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:23 smithi163 bash[75425]: cluster 2023-10-09T09:22:22.464578+0000 osd.5 (osd.5) 610 : cluster 3 failed to encode map e1068 with expected crc 2023-10-09T09:22:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:23 smithi163 bash[75425]: cluster 2023-10-09T09:22:22.471021+0000 mon.a (mon.0) 1304 : cluster 0 osdmap e1068: 8 total, 8 up, 8 in 2023-10-09T09:22:23.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:23 smithi163 bash[75425]: cluster 2023-10-09T09:22:22.479076+0000 osd.7 (osd.7) 466 : cluster 3 failed to encode map e1068 with expected crc 2023-10-09T09:22:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:23 smithi163 bash[78107]: cluster 2023-10-09T09:22:22.464578+0000 osd.5 (osd.5) 610 : cluster 3 failed to encode map e1068 with expected crc 2023-10-09T09:22:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:23 smithi163 bash[78107]: cluster 2023-10-09T09:22:22.471021+0000 mon.a (mon.0) 1304 : cluster 0 osdmap e1068: 8 total, 8 up, 8 in 2023-10-09T09:22:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:23 smithi163 bash[78107]: cluster 2023-10-09T09:22:22.479076+0000 osd.7 (osd.7) 466 : cluster 3 failed to encode map e1068 with expected crc 2023-10-09T09:22:23.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:23 smithi181 bash[59345]: cluster 2023-10-09T09:22:22.464578+0000 osd.5 (osd.5) 610 : cluster 3 failed to encode map e1068 with expected crc 2023-10-09T09:22:23.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:23 smithi181 bash[59345]: cluster 2023-10-09T09:22:22.471021+0000 mon.a (mon.0) 1304 : cluster 0 osdmap e1068: 8 total, 8 up, 8 in 2023-10-09T09:22:23.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:23 smithi181 bash[59345]: cluster 2023-10-09T09:22:22.479076+0000 osd.7 (osd.7) 466 : cluster 3 failed to encode map e1068 with expected crc 2023-10-09T09:22:24.653 INFO:tasks.rados.rados.0.smithi163.stdout:3997: done (1 left) 2023-10-09T09:22:24.653 INFO:tasks.rados.rados.0.smithi163.stdout:3998: done (0 left) 2023-10-09T09:22:24.653 INFO:tasks.rados.rados.0.smithi163.stdout:3999: snap_remove snap 499 2023-10-09T09:22:24.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: cluster 2023-10-09T09:22:23.473638+0000 osd.5 (osd.5) 611 : cluster 3 failed to encode map e1069 with expected crc 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: cluster 2023-10-09T09:22:23.473730+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e1068 with expected crc 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: cluster 2023-10-09T09:22:23.477046+0000 mon.a (mon.0) 1305 : cluster 0 osdmap e1069: 8 total, 8 up, 8 in 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: cluster 2023-10-09T09:22:23.497436+0000 mgr.y (mgr.34107) 475 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: cluster 2023-10-09T09:22:23.505152+0000 osd.7 (osd.7) 467 : cluster 3 failed to encode map e1069 with expected crc 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: audit 2023-10-09T09:22:23.647338+0000 mon.a (mon.0) 1306 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: audit 2023-10-09T09:22:23.654813+0000 mon.a (mon.0) 1307 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: audit 2023-10-09T09:22:23.806355+0000 mon.a (mon.0) 1308 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: audit 2023-10-09T09:22:23.814178+0000 mon.a (mon.0) 1309 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: audit 2023-10-09T09:22:24.292766+0000 mon.a (mon.0) 1310 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: audit 2023-10-09T09:22:24.294231+0000 mon.a (mon.0) 1311 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:22:24.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:24 smithi181 bash[59345]: audit 2023-10-09T09:22:24.302629+0000 mon.a (mon.0) 1312 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: cluster 2023-10-09T09:22:23.473638+0000 osd.5 (osd.5) 611 : cluster 3 failed to encode map e1069 with expected crc 2023-10-09T09:22:24.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: cluster 2023-10-09T09:22:23.473730+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e1068 with expected crc 2023-10-09T09:22:24.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: cluster 2023-10-09T09:22:23.477046+0000 mon.a (mon.0) 1305 : cluster 0 osdmap e1069: 8 total, 8 up, 8 in 2023-10-09T09:22:24.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: cluster 2023-10-09T09:22:23.497436+0000 mgr.y (mgr.34107) 475 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2023-10-09T09:22:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: cluster 2023-10-09T09:22:23.505152+0000 osd.7 (osd.7) 467 : cluster 3 failed to encode map e1069 with expected crc 2023-10-09T09:22:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: audit 2023-10-09T09:22:23.647338+0000 mon.a (mon.0) 1306 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: audit 2023-10-09T09:22:23.654813+0000 mon.a (mon.0) 1307 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: audit 2023-10-09T09:22:23.806355+0000 mon.a (mon.0) 1308 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: audit 2023-10-09T09:22:23.814178+0000 mon.a (mon.0) 1309 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: audit 2023-10-09T09:22:24.292766+0000 mon.a (mon.0) 1310 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:22:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: audit 2023-10-09T09:22:24.294231+0000 mon.a (mon.0) 1311 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:22:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[78107]: audit 2023-10-09T09:22:24.302629+0000 mon.a (mon.0) 1312 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: cluster 2023-10-09T09:22:23.473638+0000 osd.5 (osd.5) 611 : cluster 3 failed to encode map e1069 with expected crc 2023-10-09T09:22:24.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: cluster 2023-10-09T09:22:23.473730+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e1068 with expected crc 2023-10-09T09:22:24.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: cluster 2023-10-09T09:22:23.477046+0000 mon.a (mon.0) 1305 : cluster 0 osdmap e1069: 8 total, 8 up, 8 in 2023-10-09T09:22:24.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: cluster 2023-10-09T09:22:23.497436+0000 mgr.y (mgr.34107) 475 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2023-10-09T09:22:24.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: cluster 2023-10-09T09:22:23.505152+0000 osd.7 (osd.7) 467 : cluster 3 failed to encode map e1069 with expected crc 2023-10-09T09:22:24.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: audit 2023-10-09T09:22:23.647338+0000 mon.a (mon.0) 1306 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: audit 2023-10-09T09:22:23.654813+0000 mon.a (mon.0) 1307 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: audit 2023-10-09T09:22:23.806355+0000 mon.a (mon.0) 1308 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: audit 2023-10-09T09:22:23.814178+0000 mon.a (mon.0) 1309 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:24.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: audit 2023-10-09T09:22:24.292766+0000 mon.a (mon.0) 1310 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:22:24.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: audit 2023-10-09T09:22:24.294231+0000 mon.a (mon.0) 1311 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:22:24.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:24 smithi163 bash[75425]: audit 2023-10-09T09:22:24.302629+0000 mon.a (mon.0) 1312 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:22:25.371 INFO:tasks.rados.rados.0.smithi163.stdout:3999: done (0 left) 2023-10-09T09:22:25.379 INFO:tasks.rados.rados.0.smithi163.stderr:0 errors. 2023-10-09T09:22:25.379 INFO:tasks.rados.rados.0.smithi163.stderr: 2023-10-09T09:22:25.385 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:22:25.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[75425]: cluster 2023-10-09T09:22:24.647278+0000 osd.5 (osd.5) 612 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:25.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[75425]: cluster 2023-10-09T09:22:24.652389+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e1070: 8 total, 8 up, 8 in 2023-10-09T09:22:25.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[75425]: cluster 2023-10-09T09:22:24.976493+0000 osd.7 (osd.7) 468 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:25.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[75425]: cluster 2023-10-09T09:22:25.281484+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:25.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[75425]: audit 2023-10-09T09:22:25.354947+0000 mon.a (mon.0) 1314 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:25.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[75425]: cluster 2023-10-09T09:22:25.366046+0000 osd.5 (osd.5) 613 : cluster 3 failed to encode map e1071 with expected crc 2023-10-09T09:22:25.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[75425]: cluster 2023-10-09T09:22:25.366738+0000 osd.4 (osd.4) 507 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:25.653 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[75425]: cluster 2023-10-09T09:22:25.370454+0000 mon.a (mon.0) 1315 : cluster 0 osdmap e1071: 8 total, 8 up, 8 in 2023-10-09T09:22:25.653 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[75425]: cluster 2023-10-09T09:22:25.497966+0000 mgr.y (mgr.34107) 476 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2023-10-09T09:22:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[78107]: cluster 2023-10-09T09:22:24.647278+0000 osd.5 (osd.5) 612 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[78107]: cluster 2023-10-09T09:22:24.652389+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e1070: 8 total, 8 up, 8 in 2023-10-09T09:22:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[78107]: cluster 2023-10-09T09:22:24.976493+0000 osd.7 (osd.7) 468 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[78107]: cluster 2023-10-09T09:22:25.281484+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[78107]: audit 2023-10-09T09:22:25.354947+0000 mon.a (mon.0) 1314 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[78107]: cluster 2023-10-09T09:22:25.366046+0000 osd.5 (osd.5) 613 : cluster 3 failed to encode map e1071 with expected crc 2023-10-09T09:22:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[78107]: cluster 2023-10-09T09:22:25.366738+0000 osd.4 (osd.4) 507 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[78107]: cluster 2023-10-09T09:22:25.370454+0000 mon.a (mon.0) 1315 : cluster 0 osdmap e1071: 8 total, 8 up, 8 in 2023-10-09T09:22:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:25 smithi163 bash[78107]: cluster 2023-10-09T09:22:25.497966+0000 mgr.y (mgr.34107) 476 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2023-10-09T09:22:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:25 smithi181 bash[59345]: cluster 2023-10-09T09:22:24.647278+0000 osd.5 (osd.5) 612 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:25 smithi181 bash[59345]: cluster 2023-10-09T09:22:24.652389+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e1070: 8 total, 8 up, 8 in 2023-10-09T09:22:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:25 smithi181 bash[59345]: cluster 2023-10-09T09:22:24.976493+0000 osd.7 (osd.7) 468 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:25 smithi181 bash[59345]: cluster 2023-10-09T09:22:25.281484+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:25 smithi181 bash[59345]: audit 2023-10-09T09:22:25.354947+0000 mon.a (mon.0) 1314 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:25 smithi181 bash[59345]: cluster 2023-10-09T09:22:25.366046+0000 osd.5 (osd.5) 613 : cluster 3 failed to encode map e1071 with expected crc 2023-10-09T09:22:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:25 smithi181 bash[59345]: cluster 2023-10-09T09:22:25.366738+0000 osd.4 (osd.4) 507 : cluster 3 failed to encode map e1070 with expected crc 2023-10-09T09:22:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:25 smithi181 bash[59345]: cluster 2023-10-09T09:22:25.370454+0000 mon.a (mon.0) 1315 : cluster 0 osdmap e1071: 8 total, 8 up, 8 in 2023-10-09T09:22:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:25 smithi181 bash[59345]: cluster 2023-10-09T09:22:25.497966+0000 mgr.y (mgr.34107) 476 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2023-10-09T09:22:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:28 smithi181 bash[59345]: cluster 2023-10-09T09:22:27.499117+0000 mgr.y (mgr.34107) 477 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2023-10-09T09:22:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:28 smithi163 bash[75425]: cluster 2023-10-09T09:22:27.499117+0000 mgr.y (mgr.34107) 477 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2023-10-09T09:22:28.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:28 smithi163 bash[78107]: cluster 2023-10-09T09:22:27.499117+0000 mgr.y (mgr.34107) 477 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2023-10-09T09:22:30.164 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:30 smithi163 bash[75425]: cluster 2023-10-09T09:22:29.500025+0000 mgr.y (mgr.34107) 478 : cluster 0 pgmap v1162: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2023-10-09T09:22:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:30 smithi163 bash[75425]: cluster 2023-10-09T09:22:30.368112+0000 osd.5 (osd.5) 614 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:30.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:30 smithi163 bash[75425]: cluster 2023-10-09T09:22:30.372467+0000 mon.a (mon.0) 1316 : cluster 0 osdmap e1072: 8 total, 8 up, 8 in 2023-10-09T09:22:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:30 smithi163 bash[78107]: cluster 2023-10-09T09:22:29.500025+0000 mgr.y (mgr.34107) 478 : cluster 0 pgmap v1162: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2023-10-09T09:22:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:30 smithi163 bash[78107]: cluster 2023-10-09T09:22:30.368112+0000 osd.5 (osd.5) 614 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:30 smithi163 bash[78107]: cluster 2023-10-09T09:22:30.372467+0000 mon.a (mon.0) 1316 : cluster 0 osdmap e1072: 8 total, 8 up, 8 in 2023-10-09T09:22:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:30 smithi181 bash[59345]: cluster 2023-10-09T09:22:29.500025+0000 mgr.y (mgr.34107) 478 : cluster 0 pgmap v1162: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2023-10-09T09:22:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:30 smithi181 bash[59345]: cluster 2023-10-09T09:22:30.368112+0000 osd.5 (osd.5) 614 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:30.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:30 smithi181 bash[59345]: cluster 2023-10-09T09:22:30.372467+0000 mon.a (mon.0) 1316 : cluster 0 osdmap e1072: 8 total, 8 up, 8 in 2023-10-09T09:22:31.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:31 smithi163 bash[75425]: cluster 2023-10-09T09:22:30.545460+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:31.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:31 smithi163 bash[75425]: cluster 2023-10-09T09:22:30.847009+0000 osd.7 (osd.7) 469 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:31.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:31 smithi163 bash[75425]: cluster 2023-10-09T09:22:31.002171+0000 osd.4 (osd.4) 508 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:31.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:31 smithi163 bash[78107]: cluster 2023-10-09T09:22:30.545460+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:31.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:31 smithi163 bash[78107]: cluster 2023-10-09T09:22:30.847009+0000 osd.7 (osd.7) 469 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:31.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:31 smithi163 bash[78107]: cluster 2023-10-09T09:22:31.002171+0000 osd.4 (osd.4) 508 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:31 smithi181 bash[59345]: cluster 2023-10-09T09:22:30.545460+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:31 smithi181 bash[59345]: cluster 2023-10-09T09:22:30.847009+0000 osd.7 (osd.7) 469 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:31 smithi181 bash[59345]: cluster 2023-10-09T09:22:31.002171+0000 osd.4 (osd.4) 508 : cluster 3 failed to encode map e1072 with expected crc 2023-10-09T09:22:32.959 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:32 smithi163 bash[75425]: cluster 2023-10-09T09:22:31.500936+0000 mgr.y (mgr.34107) 479 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 901 KiB/s wr, 9 op/s 2023-10-09T09:22:32.959 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:32 smithi163 bash[78107]: cluster 2023-10-09T09:22:31.500936+0000 mgr.y (mgr.34107) 479 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 901 KiB/s wr, 9 op/s 2023-10-09T09:22:33.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:32 smithi181 bash[59345]: cluster 2023-10-09T09:22:31.500936+0000 mgr.y (mgr.34107) 479 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 901 KiB/s wr, 9 op/s 2023-10-09T09:22:33.419 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:22:33.420 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":1072,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:22:30.365301+0000","last_up_change":"2023-10-09T09:15:35.142507+0000","last_in_change":"2023-10-09T09:00:10.180048+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"152","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:01:33.304980+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:03:23.129371+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:03:23.743990+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2023-10-09T09:05:13.379685+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1071","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":999,"snap_epoch":1071,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":567,"up_from":581,"up_thru":663,"down_at":568,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6803","nonce":2071796309}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6805","nonce":2071796309}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6809","nonce":2071796309}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6807","nonce":2071796309}]},"public_addr":"172.21.15.163:6803/2071796309","cluster_addr":"172.21.15.163:6805/2071796309","heartbeat_back_addr":"172.21.15.163:6809/2071796309","heartbeat_front_addr":"172.21.15.163:6807/2071796309","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":594,"up_from":607,"up_thru":663,"down_at":595,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6811","nonce":3902451458}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6813","nonce":3902451458}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6817","nonce":3902451458}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6815","nonce":3902451458}]},"public_addr":"172.21.15.163:6811/3902451458","cluster_addr":"172.21.15.163:6813/3902451458","heartbeat_back_addr":"172.21.15.163:6817/3902451458","heartbeat_front_addr":"172.21.15.163:6815/3902451458","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":620,"up_from":634,"up_thru":663,"down_at":621,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6819","nonce":2749314903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6821","nonce":2749314903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6825","nonce":2749314903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6823","nonce":2749314903}]},"public_addr":"172.21.15.163:6819/2749314903","cluster_addr":"172.21.15.163:6821/2749314903","heartbeat_back_addr":"172.21.15.163:6825/2749314903","heartbeat_front_addr":"172.21.15.163:6823/2749314903","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":648,"up_from":663,"up_thru":663,"down_at":649,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6827","nonce":198665121}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6829","nonce":198665121}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6833","nonce":198665121}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6831","nonce":198665121}]},"public_addr":"172.21.15.163:6827/198665121","cluster_addr":"172.21.15.163:6829/198665121","heartbeat_back_addr":"172.21.15.163:6833/198665121","heartbeat_front_addr":"172.21.15.163:6831/198665121","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":649,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":567},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":594},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":620},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":648},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/3642753484":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6801/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/1436605859":"2023-10-10T09:09:25.185363+0000","172.21.15.181:0/394087329":"2023-10-10T09:08:23.434950+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/623143106":"2023-10-10T09:06:46.871855+0000","172.21.15.181:0/1788638593":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/689482878":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/3090084368":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.181:0/2794784192":"2023-10-10T09:08:23.434950+0000","172.21.15.181:0/769962302":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/2563190797":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2199287559":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6800/1737049372":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6801/1737049372":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/536711455":"2023-10-10T09:06:46.871855+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6800/3919298633":"2023-10-10T09:06:46.871855+0000","172.21.15.163:6801/3919298633":"2023-10-10T09:06:46.871855+0000","172.21.15.181:6835/3017734555":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/566729834":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/1618377325":"2023-10-10T09:09:25.185363+0000","172.21.15.181:6834/3017734555":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":993,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":5,"snaps":[{"begin":992,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:22:33.622 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:33 smithi163 bash[75425]: audit 2023-10-09T09:22:33.415944+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.163:0/2178809939' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:22:33.622 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:22:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:22:33] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.33.4" 2023-10-09T09:22:33.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:33 smithi163 bash[78107]: audit 2023-10-09T09:22:33.415944+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.163:0/2178809939' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:22:34.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:33 smithi181 bash[59345]: audit 2023-10-09T09:22:33.415944+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.163:0/2178809939' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:22:34.105 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:22:34.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:34 smithi163 bash[75425]: cluster 2023-10-09T09:22:33.501675+0000 mgr.y (mgr.34107) 480 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 370 KiB/s wr, 2 op/s 2023-10-09T09:22:34.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:34 smithi163 bash[78107]: cluster 2023-10-09T09:22:33.501675+0000 mgr.y (mgr.34107) 480 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 370 KiB/s wr, 2 op/s 2023-10-09T09:22:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:34 smithi181 bash[59345]: cluster 2023-10-09T09:22:33.501675+0000 mgr.y (mgr.34107) 480 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 370 KiB/s wr, 2 op/s 2023-10-09T09:22:36.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:36 smithi163 bash[75425]: cluster 2023-10-09T09:22:35.372468+0000 osd.5 (osd.5) 615 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:36.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:36 smithi163 bash[75425]: cluster 2023-10-09T09:22:35.376063+0000 mon.a (mon.0) 1317 : cluster 0 osdmap e1073: 8 total, 8 up, 8 in 2023-10-09T09:22:36.721 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:36 smithi163 bash[75425]: cluster 2023-10-09T09:22:35.427606+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:36.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:36 smithi163 bash[75425]: cluster 2023-10-09T09:22:35.427746+0000 osd.7 (osd.7) 470 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:36.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:36 smithi163 bash[75425]: cluster 2023-10-09T09:22:35.502353+0000 mgr.y (mgr.34107) 481 : cluster 0 pgmap v1167: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:36.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:36 smithi163 bash[78107]: cluster 2023-10-09T09:22:35.372468+0000 osd.5 (osd.5) 615 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:36.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:36 smithi163 bash[78107]: cluster 2023-10-09T09:22:35.376063+0000 mon.a (mon.0) 1317 : cluster 0 osdmap e1073: 8 total, 8 up, 8 in 2023-10-09T09:22:36.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:36 smithi163 bash[78107]: cluster 2023-10-09T09:22:35.427606+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:36.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:36 smithi163 bash[78107]: cluster 2023-10-09T09:22:35.427746+0000 osd.7 (osd.7) 470 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:36.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:36 smithi163 bash[78107]: cluster 2023-10-09T09:22:35.502353+0000 mgr.y (mgr.34107) 481 : cluster 0 pgmap v1167: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:36 smithi181 bash[59345]: cluster 2023-10-09T09:22:35.372468+0000 osd.5 (osd.5) 615 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:36 smithi181 bash[59345]: cluster 2023-10-09T09:22:35.376063+0000 mon.a (mon.0) 1317 : cluster 0 osdmap e1073: 8 total, 8 up, 8 in 2023-10-09T09:22:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:36 smithi181 bash[59345]: cluster 2023-10-09T09:22:35.427606+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:36 smithi181 bash[59345]: cluster 2023-10-09T09:22:35.427746+0000 osd.7 (osd.7) 470 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:36.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:36 smithi181 bash[59345]: cluster 2023-10-09T09:22:35.502353+0000 mgr.y (mgr.34107) 481 : cluster 0 pgmap v1167: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:37 smithi163 bash[75425]: cluster 2023-10-09T09:22:35.427432+0000 osd.4 (osd.4) 509 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:37 smithi163 bash[78107]: cluster 2023-10-09T09:22:35.427432+0000 osd.4 (osd.4) 509 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:37 smithi181 bash[59345]: cluster 2023-10-09T09:22:35.427432+0000 osd.4 (osd.4) 509 : cluster 3 failed to encode map e1073 with expected crc 2023-10-09T09:22:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:38 smithi163 bash[75425]: cluster 2023-10-09T09:22:37.503586+0000 mgr.y (mgr.34107) 482 : cluster 0 pgmap v1168: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:38 smithi163 bash[78107]: cluster 2023-10-09T09:22:37.503586+0000 mgr.y (mgr.34107) 482 : cluster 0 pgmap v1168: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:38 smithi181 bash[59345]: cluster 2023-10-09T09:22:37.503586+0000 mgr.y (mgr.34107) 482 : cluster 0 pgmap v1168: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:38.882 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:40 smithi181 bash[59345]: cluster 2023-10-09T09:22:39.504185+0000 mgr.y (mgr.34107) 483 : cluster 0 pgmap v1169: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:40 smithi181 bash[59345]: audit 2023-10-09T09:22:40.355374+0000 mon.a (mon.0) 1318 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:40 smithi163 bash[78107]: cluster 2023-10-09T09:22:39.504185+0000 mgr.y (mgr.34107) 483 : cluster 0 pgmap v1169: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:40 smithi163 bash[78107]: audit 2023-10-09T09:22:40.355374+0000 mon.a (mon.0) 1318 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:40 smithi163 bash[75425]: cluster 2023-10-09T09:22:39.504185+0000 mgr.y (mgr.34107) 483 : cluster 0 pgmap v1169: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:40 smithi163 bash[75425]: audit 2023-10-09T09:22:40.355374+0000 mon.a (mon.0) 1318 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:41.807 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:22:41.809 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:22:42.498 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1170,"stamp":"2023-10-09T09:22:41.504418+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":209689795,"num_objects":121,"num_object_clones":54,"num_object_copies":363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":121,"num_whiteouts":6,"num_read":7164,"num_read_kb":7348803,"num_write":5906,"num_write_kb":1352806,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":43692882,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22053,"ondisk_log_size":22053,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1779260,"kb_used_data":404284,"kb_used_omap":416,"kb_used_meta":1374495,"kb_avail":748181956,"statfs":{"total":767960285184,"available":766138322944,"internally_reserved":0,"allocated":413986816,"data_stored":485928179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":1407483313},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004133"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1040,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.007856+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:22:31.007856+0000","last_peered":"2023-10-09T09:22:31.007856+0000","last_clean":"2023-10-09T09:22:31.007856+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:22:31.007856+0000","last_undegraded":"2023-10-09T09:22:31.007856+0000","last_fullsized":"2023-10-09T09:22:31.007856+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1002,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.548904+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:22:30.548904+0000","last_peered":"2023-10-09T09:22:30.548904+0000","last_clean":"2023-10-09T09:22:30.548904+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:22:30.548904+0000","last_undegraded":"2023-10-09T09:22:30.548904+0000","last_fullsized":"2023-10-09T09:22:30.548904+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1006,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.761102+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:22:30.761102+0000","last_peered":"2023-10-09T09:22:30.761102+0000","last_clean":"2023-10-09T09:22:30.761102+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:22:30.761102+0000","last_undegraded":"2023-10-09T09:22:30.761102+0000","last_fullsized":"2023-10-09T09:22:30.761102+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":997,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.561302+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:22:30.561302+0000","last_peered":"2023-10-09T09:22:30.561302+0000","last_clean":"2023-10-09T09:22:30.561302+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:22:30.561302+0000","last_undegraded":"2023-10-09T09:22:30.561302+0000","last_fullsized":"2023-10-09T09:22:30.561302+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":983,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.548379+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:22:30.548379+0000","last_peered":"2023-10-09T09:22:30.548379+0000","last_clean":"2023-10-09T09:22:30.548379+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:22:30.548379+0000","last_undegraded":"2023-10-09T09:22:30.548379+0000","last_fullsized":"2023-10-09T09:22:30.548379+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1004,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.551784+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:22:30.551784+0000","last_peered":"2023-10-09T09:22:30.551784+0000","last_clean":"2023-10-09T09:22:30.551784+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:22:30.551784+0000","last_undegraded":"2023-10-09T09:22:30.551784+0000","last_fullsized":"2023-10-09T09:22:30.551784+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":997,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.560803+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:22:30.560803+0000","last_peered":"2023-10-09T09:22:30.560803+0000","last_clean":"2023-10-09T09:22:30.560803+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:22:30.560803+0000","last_undegraded":"2023-10-09T09:22:30.560803+0000","last_fullsized":"2023-10-09T09:22:30.560803+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":968,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:34.253899+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:22:34.253899+0000","last_peered":"2023-10-09T09:22:34.253899+0000","last_clean":"2023-10-09T09:22:34.253899+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:22:34.253899+0000","last_undegraded":"2023-10-09T09:22:34.253899+0000","last_fullsized":"2023-10-09T09:22:34.253899+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1025,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.375244+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:22:30.375244+0000","last_peered":"2023-10-09T09:22:30.375244+0000","last_clean":"2023-10-09T09:22:30.375244+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:22:30.375244+0000","last_undegraded":"2023-10-09T09:22:30.375244+0000","last_fullsized":"2023-10-09T09:22:30.375244+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":979,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:34.254007+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:22:34.254007+0000","last_peered":"2023-10-09T09:22:34.254007+0000","last_clean":"2023-10-09T09:22:34.254007+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:22:34.254007+0000","last_undegraded":"2023-10-09T09:22:34.254007+0000","last_fullsized":"2023-10-09T09:22:34.254007+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1004,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.852574+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:22:30.852574+0000","last_peered":"2023-10-09T09:22:30.852574+0000","last_clean":"2023-10-09T09:22:30.852574+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:22:30.852574+0000","last_undegraded":"2023-10-09T09:22:30.852574+0000","last_fullsized":"2023-10-09T09:22:30.852574+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1006,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.761080+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:22:30.761080+0000","last_peered":"2023-10-09T09:22:30.761080+0000","last_clean":"2023-10-09T09:22:30.761080+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:22:30.761080+0000","last_undegraded":"2023-10-09T09:22:30.761080+0000","last_fullsized":"2023-10-09T09:22:30.761080+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1031,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.008361+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:22:31.008361+0000","last_peered":"2023-10-09T09:22:31.008361+0000","last_clean":"2023-10-09T09:22:31.008361+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:22:31.008361+0000","last_undegraded":"2023-10-09T09:22:31.008361+0000","last_fullsized":"2023-10-09T09:22:31.008361+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1023,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.008609+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:22:31.008609+0000","last_peered":"2023-10-09T09:22:31.008609+0000","last_clean":"2023-10-09T09:22:31.008609+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:22:31.008609+0000","last_undegraded":"2023-10-09T09:22:31.008609+0000","last_fullsized":"2023-10-09T09:22:31.008609+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1003,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.852429+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:22:30.852429+0000","last_peered":"2023-10-09T09:22:30.852429+0000","last_clean":"2023-10-09T09:22:30.852429+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:22:30.852429+0000","last_undegraded":"2023-10-09T09:22:30.852429+0000","last_fullsized":"2023-10-09T09:22:30.852429+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":968,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:34.254103+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:22:34.254103+0000","last_peered":"2023-10-09T09:22:34.254103+0000","last_clean":"2023-10-09T09:22:34.254103+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:22:34.254103+0000","last_undegraded":"2023-10-09T09:22:34.254103+0000","last_fullsized":"2023-10-09T09:22:34.254103+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":997,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.559725+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:22:30.559725+0000","last_peered":"2023-10-09T09:22:30.559725+0000","last_clean":"2023-10-09T09:22:30.559725+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:22:30.559725+0000","last_undegraded":"2023-10-09T09:22:30.559725+0000","last_fullsized":"2023-10-09T09:22:30.559725+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1024,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.375111+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:22:30.375111+0000","last_peered":"2023-10-09T09:22:30.375111+0000","last_clean":"2023-10-09T09:22:30.375111+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:22:30.375111+0000","last_undegraded":"2023-10-09T09:22:30.375111+0000","last_fullsized":"2023-10-09T09:22:30.375111+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":986,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.852237+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:22:30.852237+0000","last_peered":"2023-10-09T09:22:30.852237+0000","last_clean":"2023-10-09T09:22:30.852237+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:22:30.852237+0000","last_undegraded":"2023-10-09T09:22:30.852237+0000","last_fullsized":"2023-10-09T09:22:30.852237+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":996,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.560837+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:22:30.560837+0000","last_peered":"2023-10-09T09:22:30.560837+0000","last_clean":"2023-10-09T09:22:30.560837+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:22:30.560837+0000","last_undegraded":"2023-10-09T09:22:30.560837+0000","last_fullsized":"2023-10-09T09:22:30.560837+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1014,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.551138+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:22:30.551138+0000","last_peered":"2023-10-09T09:22:30.551138+0000","last_clean":"2023-10-09T09:22:30.551138+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:22:30.551138+0000","last_undegraded":"2023-10-09T09:22:30.551138+0000","last_fullsized":"2023-10-09T09:22:30.551138+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1002,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.548938+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:22:30.548938+0000","last_peered":"2023-10-09T09:22:30.548938+0000","last_clean":"2023-10-09T09:22:30.548938+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:22:30.548938+0000","last_undegraded":"2023-10-09T09:22:30.548938+0000","last_fullsized":"2023-10-09T09:22:30.548938+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"1063'1041","reported_seq":4183,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.436026+0000","last_change":"2023-10-09T09:22:25.388100+0000","last_active":"2023-10-09T09:22:35.436026+0000","last_peered":"2023-10-09T09:22:35.436026+0000","last_clean":"2023-10-09T09:22:35.436026+0000","last_became_active":"2023-10-09T09:05:14.451870+0000","last_became_peered":"2023-10-09T09:05:14.451870+0000","last_unstale":"2023-10-09T09:22:35.436026+0000","last_undegraded":"2023-10-09T09:22:35.436026+0000","last_fullsized":"2023-10-09T09:22:35.436026+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":1041,"log_dups_size":0,"ondisk_log_size":1041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8057029,"num_objects":3,"num_object_clones":1,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":371,"num_read_kb":423349,"num_write":275,"num_write_kb":69070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1002,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.853190+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:22:30.853190+0000","last_peered":"2023-10-09T09:22:30.853190+0000","last_clean":"2023-10-09T09:22:30.853190+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:22:30.853190+0000","last_undegraded":"2023-10-09T09:22:30.853190+0000","last_fullsized":"2023-10-09T09:22:30.853190+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1009,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.374494+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:22:30.374494+0000","last_peered":"2023-10-09T09:22:30.374494+0000","last_clean":"2023-10-09T09:22:30.374494+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:22:30.374494+0000","last_undegraded":"2023-10-09T09:22:30.374494+0000","last_fullsized":"2023-10-09T09:22:30.374494+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"1071'1846","reported_seq":4960,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.432904+0000","last_change":"2023-10-09T09:22:25.398936+0000","last_active":"2023-10-09T09:22:35.432904+0000","last_peered":"2023-10-09T09:22:35.432904+0000","last_clean":"2023-10-09T09:22:35.432904+0000","last_became_active":"2023-10-09T09:15:07.216599+0000","last_became_peered":"2023-10-09T09:15:07.216599+0000","last_unstale":"2023-10-09T09:22:35.432904+0000","last_undegraded":"2023-10-09T09:22:35.432904+0000","last_fullsized":"2023-10-09T09:22:35.432904+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":1846,"log_dups_size":0,"ondisk_log_size":1846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:01.102705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.010611697999999999,"stat_sum":{"num_bytes":18366320,"num_objects":10,"num_object_clones":6,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":3,"num_read":587,"num_read_kb":642144,"num_write":513,"num_write_kb":124866,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":15566979,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1003,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.852745+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:22:30.852745+0000","last_peered":"2023-10-09T09:22:30.852745+0000","last_clean":"2023-10-09T09:22:30.852745+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:22:30.852745+0000","last_undegraded":"2023-10-09T09:22:30.852745+0000","last_fullsized":"2023-10-09T09:22:30.852745+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1038,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.007958+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:22:31.007958+0000","last_peered":"2023-10-09T09:22:31.007958+0000","last_clean":"2023-10-09T09:22:31.007958+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:22:31.007958+0000","last_undegraded":"2023-10-09T09:22:31.007958+0000","last_fullsized":"2023-10-09T09:22:31.007958+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"1071'1423","reported_seq":4497,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.432036+0000","last_change":"2023-10-09T09:22:25.400110+0000","last_active":"2023-10-09T09:22:35.432036+0000","last_peered":"2023-10-09T09:22:35.432036+0000","last_clean":"2023-10-09T09:22:35.432036+0000","last_became_active":"2023-10-09T09:15:07.215107+0000","last_became_peered":"2023-10-09T09:15:07.215107+0000","last_unstale":"2023-10-09T09:22:35.432036+0000","last_undegraded":"2023-10-09T09:22:35.432036+0000","last_fullsized":"2023-10-09T09:22:35.432036+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":1423,"log_dups_size":0,"ondisk_log_size":1423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:18.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.011466949000000001,"stat_sum":{"num_bytes":13106825,"num_objects":5,"num_object_clones":3,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":383,"num_read_kb":379801,"num_write":363,"num_write_kb":83556,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":987,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.852893+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:22:30.852893+0000","last_peered":"2023-10-09T09:22:30.852893+0000","last_clean":"2023-10-09T09:22:30.852893+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:22:30.852893+0000","last_undegraded":"2023-10-09T09:22:30.852893+0000","last_fullsized":"2023-10-09T09:22:30.852893+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1005,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.551272+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:22:30.551272+0000","last_peered":"2023-10-09T09:22:30.551272+0000","last_clean":"2023-10-09T09:22:30.551272+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:22:30.551272+0000","last_undegraded":"2023-10-09T09:22:30.551272+0000","last_fullsized":"2023-10-09T09:22:30.551272+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"1071'2224","reported_seq":5263,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434086+0000","last_change":"2023-10-09T09:22:25.403146+0000","last_active":"2023-10-09T09:22:35.434086+0000","last_peered":"2023-10-09T09:22:35.434086+0000","last_clean":"2023-10-09T09:22:35.434086+0000","last_became_active":"2023-10-09T09:15:36.176509+0000","last_became_peered":"2023-10-09T09:15:36.176509+0000","last_unstale":"2023-10-09T09:22:35.434086+0000","last_undegraded":"2023-10-09T09:22:35.434086+0000","last_fullsized":"2023-10-09T09:22:35.434086+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":2224,"log_dups_size":0,"ondisk_log_size":2224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:30.962946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.014198481000000001,"stat_sum":{"num_bytes":26215192,"num_objects":13,"num_object_clones":8,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":2,"num_read":755,"num_read_kb":893028,"num_write":521,"num_write_kb":121451,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":3326190,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1025,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.374987+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:22:30.374987+0000","last_peered":"2023-10-09T09:22:30.374987+0000","last_clean":"2023-10-09T09:22:30.374987+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:22:30.374987+0000","last_undegraded":"2023-10-09T09:22:30.374987+0000","last_fullsized":"2023-10-09T09:22:30.374987+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":996,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.560201+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:22:30.560201+0000","last_peered":"2023-10-09T09:22:30.560201+0000","last_clean":"2023-10-09T09:22:30.560201+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:22:30.560201+0000","last_undegraded":"2023-10-09T09:22:30.560201+0000","last_fullsized":"2023-10-09T09:22:30.560201+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"1071'1171","reported_seq":4282,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434422+0000","last_change":"2023-10-09T09:22:25.405198+0000","last_active":"2023-10-09T09:22:35.434422+0000","last_peered":"2023-10-09T09:22:35.434422+0000","last_clean":"2023-10-09T09:22:35.434422+0000","last_became_active":"2023-10-09T09:15:07.211552+0000","last_became_peered":"2023-10-09T09:15:07.211552+0000","last_unstale":"2023-10-09T09:22:35.434422+0000","last_undegraded":"2023-10-09T09:22:35.434422+0000","last_fullsized":"2023-10-09T09:22:35.434422+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":1171,"log_dups_size":0,"ondisk_log_size":1171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:02.623774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.011515668999999999,"stat_sum":{"num_bytes":15705144,"num_objects":8,"num_object_clones":5,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":397,"num_read_kb":384249,"num_write":320,"num_write_kb":78798,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":986,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.548201+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:22:30.548201+0000","last_peered":"2023-10-09T09:22:30.548201+0000","last_clean":"2023-10-09T09:22:30.548201+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:22:30.548201+0000","last_undegraded":"2023-10-09T09:22:30.548201+0000","last_fullsized":"2023-10-09T09:22:30.548201+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1023,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.008323+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:22:31.008323+0000","last_peered":"2023-10-09T09:22:31.008323+0000","last_clean":"2023-10-09T09:22:31.008323+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:22:31.008323+0000","last_undegraded":"2023-10-09T09:22:31.008323+0000","last_fullsized":"2023-10-09T09:22:31.008323+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"1067'1013","reported_seq":4164,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.379072+0000","last_change":"2023-10-09T09:22:25.394194+0000","last_active":"2023-10-09T09:22:35.379072+0000","last_peered":"2023-10-09T09:22:35.379072+0000","last_clean":"2023-10-09T09:22:35.379072+0000","last_became_active":"2023-10-09T09:14:40.492167+0000","last_became_peered":"2023-10-09T09:14:40.492167+0000","last_unstale":"2023-10-09T09:22:35.379072+0000","last_undegraded":"2023-10-09T09:22:35.379072+0000","last_fullsized":"2023-10-09T09:22:35.379072+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":1013,"log_dups_size":0,"ondisk_log_size":1013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:20.638061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0026651660000000001,"stat_sum":{"num_bytes":5815882,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":356,"num_read_kb":403141,"num_write":253,"num_write_kb":56349,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":3944135,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1006,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.551617+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:22:30.551617+0000","last_peered":"2023-10-09T09:22:30.551617+0000","last_clean":"2023-10-09T09:22:30.551617+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:22:30.551617+0000","last_undegraded":"2023-10-09T09:22:30.551617+0000","last_fullsized":"2023-10-09T09:22:30.551617+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1038,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.008700+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:22:31.008700+0000","last_peered":"2023-10-09T09:22:31.008700+0000","last_clean":"2023-10-09T09:22:31.008700+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:22:31.008700+0000","last_undegraded":"2023-10-09T09:22:31.008700+0000","last_fullsized":"2023-10-09T09:22:31.008700+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"1071'1867","reported_seq":4931,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378321+0000","last_change":"2023-10-09T09:22:25.380178+0000","last_active":"2023-10-09T09:22:35.378321+0000","last_peered":"2023-10-09T09:22:35.378321+0000","last_clean":"2023-10-09T09:22:35.378321+0000","last_became_active":"2023-10-09T09:05:14.450385+0000","last_became_peered":"2023-10-09T09:05:14.450385+0000","last_unstale":"2023-10-09T09:22:35.378321+0000","last_undegraded":"2023-10-09T09:22:35.378321+0000","last_fullsized":"2023-10-09T09:22:35.378321+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":1867,"log_dups_size":0,"ondisk_log_size":1867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15809298,"num_objects":9,"num_object_clones":5,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":1,"num_read":607,"num_read_kb":674009,"num_write":441,"num_write_kb":106953,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1024,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.007505+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:22:31.007505+0000","last_peered":"2023-10-09T09:22:31.007505+0000","last_clean":"2023-10-09T09:22:31.007505+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:22:31.007505+0000","last_undegraded":"2023-10-09T09:22:31.007505+0000","last_fullsized":"2023-10-09T09:22:31.007505+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1023,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.008710+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:22:31.008710+0000","last_peered":"2023-10-09T09:22:31.008710+0000","last_clean":"2023-10-09T09:22:31.008710+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:22:31.008710+0000","last_undegraded":"2023-10-09T09:22:31.008710+0000","last_fullsized":"2023-10-09T09:22:31.008710+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"1060'1813","reported_seq":4925,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434382+0000","last_change":"2023-10-09T09:22:25.379932+0000","last_active":"2023-10-09T09:22:35.434382+0000","last_peered":"2023-10-09T09:22:35.434382+0000","last_clean":"2023-10-09T09:22:35.434382+0000","last_became_active":"2023-10-09T09:14:14.171356+0000","last_became_peered":"2023-10-09T09:14:14.171356+0000","last_unstale":"2023-10-09T09:22:35.434382+0000","last_undegraded":"2023-10-09T09:22:35.434382+0000","last_fullsized":"2023-10-09T09:22:35.434382+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":1813,"log_dups_size":0,"ondisk_log_size":1813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3160079,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":658,"num_read_kb":730590,"num_write":421,"num_write_kb":95922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":2290704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":969,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.547955+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:22:30.547955+0000","last_peered":"2023-10-09T09:22:30.547955+0000","last_clean":"2023-10-09T09:22:30.547955+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:22:30.547955+0000","last_undegraded":"2023-10-09T09:22:30.547955+0000","last_fullsized":"2023-10-09T09:22:30.547955+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1006,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.762074+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:22:30.762074+0000","last_peered":"2023-10-09T09:22:30.762074+0000","last_clean":"2023-10-09T09:22:30.762074+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:22:30.762074+0000","last_undegraded":"2023-10-09T09:22:30.762074+0000","last_fullsized":"2023-10-09T09:22:30.762074+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1001,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.762334+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:22:30.762334+0000","last_peered":"2023-10-09T09:22:30.762334+0000","last_clean":"2023-10-09T09:22:30.762334+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:22:30.762334+0000","last_undegraded":"2023-10-09T09:22:30.762334+0000","last_fullsized":"2023-10-09T09:22:30.762334+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1040,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.375169+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:22:30.375169+0000","last_peered":"2023-10-09T09:22:30.375169+0000","last_clean":"2023-10-09T09:22:30.375169+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:22:30.375169+0000","last_undegraded":"2023-10-09T09:22:30.375169+0000","last_fullsized":"2023-10-09T09:22:30.375169+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":991,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.761112+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:22:30.761112+0000","last_peered":"2023-10-09T09:22:30.761112+0000","last_clean":"2023-10-09T09:22:30.761112+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:22:30.761112+0000","last_undegraded":"2023-10-09T09:22:30.761112+0000","last_fullsized":"2023-10-09T09:22:30.761112+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"1033'483","reported_seq":3666,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.430640+0000","last_change":"2023-10-09T09:22:25.397147+0000","last_active":"2023-10-09T09:22:35.430640+0000","last_peered":"2023-10-09T09:22:35.430640+0000","last_clean":"2023-10-09T09:22:35.430640+0000","last_became_active":"2023-10-09T09:14:14.177346+0000","last_became_peered":"2023-10-09T09:14:14.177346+0000","last_unstale":"2023-10-09T09:22:35.430640+0000","last_undegraded":"2023-10-09T09:22:35.430640+0000","last_fullsized":"2023-10-09T09:22:35.430640+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":483,"log_dups_size":0,"ondisk_log_size":483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:51.832535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000407174,"stat_sum":{"num_bytes":3563381,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":157,"num_read_kb":170640,"num_write":121,"num_write_kb":32645,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"1071'1209","reported_seq":4352,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.432106+0000","last_change":"2023-10-09T09:22:25.397974+0000","last_active":"2023-10-09T09:22:35.432106+0000","last_peered":"2023-10-09T09:22:35.432106+0000","last_clean":"2023-10-09T09:22:35.432106+0000","last_became_active":"2023-10-09T09:14:40.489213+0000","last_became_peered":"2023-10-09T09:14:40.489213+0000","last_unstale":"2023-10-09T09:22:35.432106+0000","last_undegraded":"2023-10-09T09:22:35.432106+0000","last_fullsized":"2023-10-09T09:22:35.432106+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":1209,"log_dups_size":0,"ondisk_log_size":1209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:15.333300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0089171879999999995,"stat_sum":{"num_bytes":17266611,"num_objects":8,"num_object_clones":5,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":354,"num_read_kb":379856,"num_write":351,"num_write_kb":89682,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":6108376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1029,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.374642+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:22:30.374642+0000","last_peered":"2023-10-09T09:22:30.374642+0000","last_clean":"2023-10-09T09:22:30.374642+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:22:30.374642+0000","last_undegraded":"2023-10-09T09:22:30.374642+0000","last_fullsized":"2023-10-09T09:22:30.374642+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1024,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.008424+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:22:31.008424+0000","last_peered":"2023-10-09T09:22:31.008424+0000","last_clean":"2023-10-09T09:22:31.008424+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:22:31.008424+0000","last_undegraded":"2023-10-09T09:22:31.008424+0000","last_fullsized":"2023-10-09T09:22:31.008424+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1021,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.761560+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:22:30.761560+0000","last_peered":"2023-10-09T09:22:30.761560+0000","last_clean":"2023-10-09T09:22:30.761560+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:22:30.761560+0000","last_undegraded":"2023-10-09T09:22:30.761560+0000","last_fullsized":"2023-10-09T09:22:30.761560+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"1071'392","reported_seq":3544,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434312+0000","last_change":"2023-10-09T09:22:25.402324+0000","last_active":"2023-10-09T09:22:35.434312+0000","last_peered":"2023-10-09T09:22:35.434312+0000","last_clean":"2023-10-09T09:22:35.434312+0000","last_became_active":"2023-10-09T09:15:07.211635+0000","last_became_peered":"2023-10-09T09:15:07.211635+0000","last_unstale":"2023-10-09T09:22:35.434312+0000","last_undegraded":"2023-10-09T09:22:35.434312+0000","last_fullsized":"2023-10-09T09:22:35.434312+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":392,"log_dups_size":0,"ondisk_log_size":392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.133850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.011999676000000001,"stat_sum":{"num_bytes":2386957,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":96,"num_read_kb":65603,"num_write":142,"num_write_kb":34455,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":988,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.371159+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:22:30.371159+0000","last_peered":"2023-10-09T09:22:30.371159+0000","last_clean":"2023-10-09T09:22:30.371159+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:22:30.371159+0000","last_undegraded":"2023-10-09T09:22:30.371159+0000","last_fullsized":"2023-10-09T09:22:30.371159+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1007,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378403+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:22:35.378403+0000","last_peered":"2023-10-09T09:22:35.378403+0000","last_clean":"2023-10-09T09:22:35.378403+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:22:35.378403+0000","last_undegraded":"2023-10-09T09:22:35.378403+0000","last_fullsized":"2023-10-09T09:22:35.378403+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":996,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.762266+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:22:30.762266+0000","last_peered":"2023-10-09T09:22:30.762266+0000","last_clean":"2023-10-09T09:22:30.762266+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:22:30.762266+0000","last_undegraded":"2023-10-09T09:22:30.762266+0000","last_fullsized":"2023-10-09T09:22:30.762266+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1029,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.374809+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:22:30.374809+0000","last_peered":"2023-10-09T09:22:30.374809+0000","last_clean":"2023-10-09T09:22:30.374809+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:22:30.374809+0000","last_undegraded":"2023-10-09T09:22:30.374809+0000","last_fullsized":"2023-10-09T09:22:30.374809+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1237,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.851991+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:22:30.851991+0000","last_peered":"2023-10-09T09:22:30.851991+0000","last_clean":"2023-10-09T09:22:30.851991+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:22:30.851991+0000","last_undegraded":"2023-10-09T09:22:30.851991+0000","last_fullsized":"2023-10-09T09:22:30.851991+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":980,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.548675+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:22:30.548675+0000","last_peered":"2023-10-09T09:22:30.548675+0000","last_clean":"2023-10-09T09:22:30.548675+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:22:30.548675+0000","last_undegraded":"2023-10-09T09:22:30.548675+0000","last_fullsized":"2023-10-09T09:22:30.548675+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":990,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.551607+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:22:30.551607+0000","last_peered":"2023-10-09T09:22:30.551607+0000","last_clean":"2023-10-09T09:22:30.551607+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:22:30.551607+0000","last_undegraded":"2023-10-09T09:22:30.551607+0000","last_fullsized":"2023-10-09T09:22:30.551607+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"1071'2065","reported_seq":5107,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.436453+0000","last_change":"2023-10-09T09:22:25.397671+0000","last_active":"2023-10-09T09:22:35.436453+0000","last_peered":"2023-10-09T09:22:35.436453+0000","last_clean":"2023-10-09T09:22:35.436453+0000","last_became_active":"2023-10-09T09:15:07.200667+0000","last_became_peered":"2023-10-09T09:15:07.200667+0000","last_unstale":"2023-10-09T09:22:35.436453+0000","last_undegraded":"2023-10-09T09:22:35.436453+0000","last_fullsized":"2023-10-09T09:22:35.436453+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":2065,"log_dups_size":0,"ondisk_log_size":2065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22454943,"num_objects":11,"num_object_clones":6,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":576,"num_read_kb":593551,"num_write":540,"num_write_kb":125733,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"1063'3056","reported_seq":6072,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378222+0000","last_change":"2023-10-09T09:22:25.374014+0000","last_active":"2023-10-09T09:22:35.378222+0000","last_peered":"2023-10-09T09:22:35.378222+0000","last_clean":"2023-10-09T09:22:35.378222+0000","last_became_active":"2023-10-09T09:14:40.485256+0000","last_became_peered":"2023-10-09T09:14:40.485256+0000","last_unstale":"2023-10-09T09:22:35.378222+0000","last_undegraded":"2023-10-09T09:22:35.378222+0000","last_fullsized":"2023-10-09T09:22:35.378222+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":3056,"log_dups_size":0,"ondisk_log_size":3056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28486650,"num_objects":12,"num_object_clones":6,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":909,"num_read_kb":889818,"num_write":806,"num_write_kb":192452,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":8602705,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1020,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.851777+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:22:30.851777+0000","last_peered":"2023-10-09T09:22:30.851777+0000","last_clean":"2023-10-09T09:22:30.851777+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:22:30.851777+0000","last_undegraded":"2023-10-09T09:22:30.851777+0000","last_fullsized":"2023-10-09T09:22:30.851777+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":998,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.239119+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:22:32.239119+0000","last_peered":"2023-10-09T09:22:32.239119+0000","last_clean":"2023-10-09T09:22:32.239119+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:22:32.239119+0000","last_undegraded":"2023-10-09T09:22:32.239119+0000","last_fullsized":"2023-10-09T09:22:32.239119+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1008,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.560645+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:22:30.560645+0000","last_peered":"2023-10-09T09:22:30.560645+0000","last_clean":"2023-10-09T09:22:30.560645+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:22:30.560645+0000","last_undegraded":"2023-10-09T09:22:30.560645+0000","last_fullsized":"2023-10-09T09:22:30.560645+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1005,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.852718+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:22:30.852718+0000","last_peered":"2023-10-09T09:22:30.852718+0000","last_clean":"2023-10-09T09:22:30.852718+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:22:30.852718+0000","last_undegraded":"2023-10-09T09:22:30.852718+0000","last_fullsized":"2023-10-09T09:22:30.852718+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1007,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.762162+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:22:30.762162+0000","last_peered":"2023-10-09T09:22:30.762162+0000","last_clean":"2023-10-09T09:22:30.762162+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:22:30.762162+0000","last_undegraded":"2023-10-09T09:22:30.762162+0000","last_fullsized":"2023-10-09T09:22:30.762162+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":982,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.239225+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:22:32.239225+0000","last_peered":"2023-10-09T09:22:32.239225+0000","last_clean":"2023-10-09T09:22:32.239225+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:22:32.239225+0000","last_undegraded":"2023-10-09T09:22:32.239225+0000","last_fullsized":"2023-10-09T09:22:32.239225+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"1061'905","reported_seq":4038,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437607+0000","last_change":"2023-10-09T09:22:25.377857+0000","last_active":"2023-10-09T09:22:35.437607+0000","last_peered":"2023-10-09T09:22:35.437607+0000","last_clean":"2023-10-09T09:22:35.437607+0000","last_became_active":"2023-10-09T09:14:14.176008+0000","last_became_peered":"2023-10-09T09:14:14.176008+0000","last_unstale":"2023-10-09T09:22:35.437607+0000","last_undegraded":"2023-10-09T09:22:35.437607+0000","last_fullsized":"2023-10-09T09:22:35.437607+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":905,"log_dups_size":0,"ondisk_log_size":905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12358471,"num_objects":6,"num_object_clones":4,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":240,"num_read_kb":268358,"num_write":221,"num_write_kb":52134,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":1230356,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":986,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.548973+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:22:30.548973+0000","last_peered":"2023-10-09T09:22:30.548973+0000","last_clean":"2023-10-09T09:22:30.548973+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:22:30.548973+0000","last_undegraded":"2023-10-09T09:22:30.548973+0000","last_fullsized":"2023-10-09T09:22:30.548973+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":990,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.761770+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:22:30.761770+0000","last_peered":"2023-10-09T09:22:30.761770+0000","last_clean":"2023-10-09T09:22:30.761770+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:22:30.761770+0000","last_undegraded":"2023-10-09T09:22:30.761770+0000","last_fullsized":"2023-10-09T09:22:30.761770+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":983,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.547973+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:22:30.547973+0000","last_peered":"2023-10-09T09:22:30.547973+0000","last_clean":"2023-10-09T09:22:30.547973+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:22:30.547973+0000","last_undegraded":"2023-10-09T09:22:30.547973+0000","last_fullsized":"2023-10-09T09:22:30.547973+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"1061'471","reported_seq":3663,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433151+0000","last_change":"2023-10-09T09:22:25.379234+0000","last_active":"2023-10-09T09:22:35.433151+0000","last_peered":"2023-10-09T09:22:35.433151+0000","last_clean":"2023-10-09T09:22:35.433151+0000","last_became_active":"2023-10-09T09:15:36.183323+0000","last_became_peered":"2023-10-09T09:15:36.183323+0000","last_unstale":"2023-10-09T09:22:35.433151+0000","last_undegraded":"2023-10-09T09:22:35.433151+0000","last_fullsized":"2023-10-09T09:22:35.433151+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":471,"log_dups_size":0,"ondisk_log_size":471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7785322,"num_objects":3,"num_object_clones":2,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":124,"num_read_kb":161903,"num_write":100,"num_write_kb":22455,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1009,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.551671+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:22:30.551671+0000","last_peered":"2023-10-09T09:22:30.551671+0000","last_clean":"2023-10-09T09:22:30.551671+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:22:30.551671+0000","last_undegraded":"2023-10-09T09:22:30.551671+0000","last_fullsized":"2023-10-09T09:22:30.551671+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1000,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.561327+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:22:30.561327+0000","last_peered":"2023-10-09T09:22:30.561327+0000","last_clean":"2023-10-09T09:22:30.561327+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:22:30.561327+0000","last_undegraded":"2023-10-09T09:22:30.561327+0000","last_fullsized":"2023-10-09T09:22:30.561327+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1008,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.008227+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:22:31.008227+0000","last_peered":"2023-10-09T09:22:31.008227+0000","last_clean":"2023-10-09T09:22:31.008227+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:22:31.008227+0000","last_undegraded":"2023-10-09T09:22:31.008227+0000","last_fullsized":"2023-10-09T09:22:31.008227+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"1061'880","reported_seq":4045,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.430864+0000","last_change":"2023-10-09T09:22:25.393532+0000","last_active":"2023-10-09T09:22:35.430864+0000","last_peered":"2023-10-09T09:22:35.430864+0000","last_clean":"2023-10-09T09:22:35.430864+0000","last_became_active":"2023-10-09T09:15:36.176346+0000","last_became_peered":"2023-10-09T09:15:36.176346+0000","last_unstale":"2023-10-09T09:22:35.430864+0000","last_undegraded":"2023-10-09T09:22:35.430864+0000","last_fullsized":"2023-10-09T09:22:35.430864+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:05:13.429896+0000","last_clean_scrub_stamp":"2023-10-09T09:05:13.429896+0000","objects_scrubbed":0,"log_size":880,"log_dups_size":0,"ondisk_log_size":880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:47.444371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086481399999999999,"stat_sum":{"num_bytes":8555819,"num_objects":3,"num_object_clones":1,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":286,"num_read_kb":288251,"num_write":258,"num_write_kb":63840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":2623437,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":985,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.561490+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:22:30.561490+0000","last_peered":"2023-10-09T09:22:30.561490+0000","last_clean":"2023-10-09T09:22:30.561490+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:22:30.561490+0000","last_undegraded":"2023-10-09T09:22:30.561490+0000","last_fullsized":"2023-10-09T09:22:30.561490+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1007,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.270494+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:22:35.270494+0000","last_peered":"2023-10-09T09:22:35.270494+0000","last_clean":"2023-10-09T09:22:35.270494+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:22:35.270494+0000","last_undegraded":"2023-10-09T09:22:35.270494+0000","last_fullsized":"2023-10-09T09:22:35.270494+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1027,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.375015+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:22:30.375015+0000","last_peered":"2023-10-09T09:22:30.375015+0000","last_clean":"2023-10-09T09:22:30.375015+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:22:30.375015+0000","last_undegraded":"2023-10-09T09:22:30.375015+0000","last_fullsized":"2023-10-09T09:22:30.375015+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1038,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.008885+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:22:31.008885+0000","last_peered":"2023-10-09T09:22:31.008885+0000","last_clean":"2023-10-09T09:22:31.008885+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:22:31.008885+0000","last_undegraded":"2023-10-09T09:22:31.008885+0000","last_fullsized":"2023-10-09T09:22:31.008885+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1012,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.375328+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:22:30.375328+0000","last_peered":"2023-10-09T09:22:30.375328+0000","last_clean":"2023-10-09T09:22:30.375328+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:22:30.375328+0000","last_undegraded":"2023-10-09T09:22:30.375328+0000","last_fullsized":"2023-10-09T09:22:30.375328+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1038,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.007899+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:22:31.007899+0000","last_peered":"2023-10-09T09:22:31.007899+0000","last_clean":"2023-10-09T09:22:31.007899+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:22:31.007899+0000","last_undegraded":"2023-10-09T09:22:31.007899+0000","last_fullsized":"2023-10-09T09:22:31.007899+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":996,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.548790+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:22:30.548790+0000","last_peered":"2023-10-09T09:22:30.548790+0000","last_clean":"2023-10-09T09:22:30.548790+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:22:30.548790+0000","last_undegraded":"2023-10-09T09:22:30.548790+0000","last_fullsized":"2023-10-09T09:22:30.548790+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1008,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:30.548133+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:22:30.548133+0000","last_peered":"2023-10-09T09:22:30.548133+0000","last_clean":"2023-10-09T09:22:30.548133+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:22:30.548133+0000","last_undegraded":"2023-10-09T09:22:30.548133+0000","last_fullsized":"2023-10-09T09:22:30.548133+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1004,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:32.239336+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:22:32.239336+0000","last_peered":"2023-10-09T09:22:32.239336+0000","last_clean":"2023-10-09T09:22:32.239336+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:22:32.239336+0000","last_undegraded":"2023-10-09T09:22:32.239336+0000","last_fullsized":"2023-10-09T09:22:32.239336+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1023,"reported_epoch":1072,"state":"active+clean","last_fresh":"2023-10-09T09:22:31.007363+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:22:31.007363+0000","last_peered":"2023-10-09T09:22:31.007363+0000","last_clean":"2023-10-09T09:22:31.007363+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:22:31.007363+0000","last_undegraded":"2023-10-09T09:22:31.007363+0000","last_fullsized":"2023-10-09T09:22:31.007363+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":209093923,"num_objects":97,"num_object_clones":54,"num_object_copies":291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97,"num_whiteouts":6,"num_read":6856,"num_read_kb":7348291,"num_write":5646,"num_write_kb":1350361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":43692882,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":359190528,"data_stored":436728516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21859,"ondisk_log_size":21859,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397770,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":348568,"kb_used_data":39056,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93396584,"statfs":{"total":95995035648,"available":95638102016,"internally_reserved":0,"allocated":39993344,"data_stored":48493394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:22:24 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.0409999999999999,"15min":1.0429999999999999},"min":{"1min":0.68100000000000005,"5min":0.48599999999999999,"15min":0.41899999999999998},"max":{"1min":1.532,"5min":1.837,"15min":1.837},"last":1.1160000000000001},{"interface":"front","average":{"1min":1.206,"5min":1.179,"15min":1.127},"min":{"1min":0.78400000000000003,"5min":0.60599999999999998,"15min":0.48599999999999999},"max":{"1min":1.722,"5min":1.905,"15min":1.905},"last":1.2050000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:21:58 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.93999999999999995,"15min":0.90100000000000002},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.60699999999999998},"max":{"1min":1.28,"5min":1.5980000000000001,"15min":1.5980000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.069,"15min":0.98499999999999999},"min":{"1min":0.68500000000000005,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.5049999999999999,"5min":1.722,"15min":2.028},"last":0.93300000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:22:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":1.177,"15min":1.103},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.4730000000000001,"5min":3.0059999999999998,"15min":3.0059999999999998},"last":1.0089999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0940000000000001,"15min":0.996},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.4630000000000001,"5min":3.0249999999999999,"15min":3.0249999999999999},"last":1.0369999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:21:47 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.0129999999999999,"15min":0.99299999999999999},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.6220000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":1.163,"5min":1.1779999999999999,"15min":1.1299999999999999},"min":{"1min":0.77500000000000002,"5min":0.74199999999999999,"15min":0.74199999999999999},"max":{"1min":1.5449999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":0.65100000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:22:24 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.97499999999999998,"15min":0.93899999999999995},"min":{"1min":0.72499999999999998,"5min":0.374,"15min":0.374},"max":{"1min":1.1850000000000001,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.96399999999999997},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1,"15min":1.0209999999999999},"min":{"1min":0.69899999999999995,"5min":0.61499999999999999,"15min":0.50900000000000001},"max":{"1min":1.5109999999999999,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.87}]},{"osd":5,"last update":"Mon Oct 9 09:22:24 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.177,"15min":1.1020000000000001},"min":{"1min":0.65200000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.6819999999999999,"5min":1.8819999999999999,"15min":2.2330000000000001},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.095,"5min":1.081,"15min":1.0669999999999999},"min":{"1min":0.66400000000000003,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.4690000000000001,"5min":1.7789999999999999,"15min":2.302},"last":1.0800000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:22:24 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.097,"15min":1.089},"min":{"1min":0.78100000000000003,"5min":0.60999999999999999,"15min":0.54300000000000004},"max":{"1min":1.5409999999999999,"5min":1.5589999999999999,"15min":2.0049999999999999},"last":1.163},{"interface":"front","average":{"1min":1.2350000000000001,"5min":1.2030000000000001,"15min":1.1659999999999999},"min":{"1min":1.0409999999999999,"5min":0.69299999999999995,"15min":0.60999999999999999},"max":{"1min":1.651,"5min":2.0070000000000001,"15min":2.0070000000000001},"last":1.1399999999999999}]}]},{"osd":6,"up_from":43,"seq":184683594003,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":337096,"kb_used_data":31680,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93408056,"statfs":{"total":95995035648,"available":95649849344,"internally_reserved":0,"allocated":32440320,"data_stored":36249745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.92100000000000004,"15min":0.91800000000000004},"min":{"1min":0.64400000000000002,"5min":0.46500000000000002,"15min":0.35499999999999998},"max":{"1min":1.72,"5min":1.72,"15min":2.9060000000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.242,"5min":1.0509999999999999,"15min":0.96199999999999997},"min":{"1min":0.82899999999999996,"5min":0.54200000000000004,"15min":0.38300000000000001},"max":{"1min":2.2669999999999999,"5min":2.2669999999999999,"15min":2.2669999999999999},"last":1.117}]},{"osd":1,"last update":"Mon Oct 9 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.999,"15min":0.91500000000000004},"min":{"1min":0.70399999999999996,"5min":0.53100000000000003,"15min":0.40699999999999997},"max":{"1min":1.1579999999999999,"5min":3.5590000000000002,"15min":3.5590000000000002},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.98399999999999999,"15min":0.91000000000000003},"min":{"1min":0.70499999999999996,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.208,"5min":3.3860000000000001,"15min":3.3860000000000001},"last":1.0920000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.96499999999999997,"15min":0.91700000000000004},"min":{"1min":0.56100000000000005,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.343,"5min":1.548,"15min":1.5640000000000001},"last":0.995},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.097,"15min":0.97799999999999998},"min":{"1min":0.67600000000000005,"5min":0.51800000000000002,"15min":0.51200000000000001},"max":{"1min":2.2120000000000002,"5min":2.2120000000000002,"15min":2.2120000000000002},"last":0.86499999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:21:43 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.026,"15min":0.99399999999999999},"min":{"1min":0.70699999999999996,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.5489999999999999,"5min":4.7430000000000003,"15min":4.7430000000000003},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.081,"5min":1.1299999999999999,"15min":1.0740000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.8620000000000001,"5min":3.4249999999999998,"15min":3.4249999999999998},"last":1.032}]},{"osd":4,"last update":"Mon Oct 9 09:22:38 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":0.99299999999999999,"15min":0.96899999999999997},"min":{"1min":0.67400000000000004,"5min":0.57499999999999996,"15min":0.39900000000000002},"max":{"1min":2.3809999999999998,"5min":2.3809999999999998,"15min":3.5640000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.155,"5min":1.0660000000000001,"15min":1.042},"min":{"1min":0.76300000000000001,"5min":0.57499999999999996,"15min":0.41799999999999998},"max":{"1min":2.5800000000000001,"5min":2.5800000000000001,"15min":4.0629999999999997},"last":0.93300000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:22:38 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.139,"15min":1.0780000000000001},"min":{"1min":0.91200000000000003,"5min":0.56000000000000005,"15min":0.46000000000000002},"max":{"1min":2.7229999999999999,"5min":2.7229999999999999,"15min":2.7229999999999999},"last":1.1799999999999999},{"interface":"front","average":{"1min":1.2629999999999999,"5min":1.177,"15min":1.0900000000000001},"min":{"1min":0.80500000000000005,"5min":0.63300000000000001,"15min":0.48699999999999999},"max":{"1min":2.7570000000000001,"5min":2.7570000000000001,"15min":3.246},"last":1.214}]},{"osd":7,"last update":"Mon Oct 9 09:22:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":1.2010000000000001,"15min":1.1379999999999999},"min":{"1min":0.89200000000000002,"5min":0.68200000000000005,"15min":0.49299999999999999},"max":{"1min":2.8570000000000002,"5min":2.8570000000000002,"15min":3.3340000000000001},"last":1.157},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.198,"15min":1.1639999999999999},"min":{"1min":0.95599999999999996,"5min":0.79300000000000004,"15min":0.46999999999999997},"max":{"1min":1.8580000000000001,"5min":1.8580000000000001,"15min":2.6659999999999999},"last":1.075}]}]},{"osd":2,"up_from":634,"seq":2723009265756,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":96808,"kb_used_data":59808,"kb_used_omap":122,"kb_used_meta":36869,"kb_avail":93648344,"statfs":{"total":95995035648,"available":95895904256,"internally_reserved":0,"allocated":61243392,"data_stored":76097991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":37754443},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.77400000000000002,"15min":0.73799999999999999},"min":{"1min":0.57099999999999995,"5min":0.40799999999999997,"15min":0.39800000000000002},"max":{"1min":1.0369999999999999,"5min":1.22,"15min":1.454},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.76000000000000001,"15min":0.68500000000000005},"min":{"1min":0.47199999999999998,"5min":0.44700000000000001,"15min":0.33600000000000002},"max":{"1min":0.97699999999999998,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.84199999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.75600000000000001,"15min":0.70499999999999996},"min":{"1min":0.58599999999999997,"5min":0.42799999999999999,"15min":0.377},"max":{"1min":1.036,"5min":1.2549999999999999,"15min":1.2809999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.85999999999999999,"15min":0.753},"min":{"1min":0.63,"5min":0.51800000000000002,"15min":0.47499999999999998},"max":{"1min":1.3100000000000001,"5min":1.956,"15min":1.956},"last":1.0860000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:21:44 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.84599999999999997,"15min":0.79200000000000004},"min":{"1min":0.58799999999999997,"5min":0.51700000000000002,"15min":0.40600000000000003},"max":{"1min":1.083,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.91500000000000004,"15min":0.85699999999999998},"min":{"1min":0.54600000000000004,"5min":0.5,"15min":0.44800000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":1.194}]},{"osd":4,"last update":"Mon Oct 9 09:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.091,"15min":1.032},"min":{"1min":0.85499999999999998,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.4630000000000001,"5min":2.2559999999999998,"15min":2.2559999999999998},"last":1.3959999999999999},{"interface":"front","average":{"1min":1.048,"5min":1.0389999999999999,"15min":1.032},"min":{"1min":0.79500000000000004,"5min":0.60999999999999999,"15min":0.59699999999999998},"max":{"1min":1.407,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.2889999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.0920000000000001,"15min":1.036},"min":{"1min":0.79200000000000004,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.4950000000000001,"5min":2.226,"15min":2.226},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.22,"5min":1.1830000000000001,"15min":1.123},"min":{"1min":0.90900000000000003,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.696,"5min":1.696,"15min":1.696},"last":1.4730000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":1.212,"15min":1.147},"min":{"1min":0.86199999999999999,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.6080000000000001,"5min":1.716,"15min":1.716},"last":1.3160000000000001},{"interface":"front","average":{"1min":1.226,"5min":1.1699999999999999,"15min":1.103},"min":{"1min":0.95199999999999996,"5min":0.75800000000000001,"15min":0.68500000000000005},"max":{"1min":1.5509999999999999,"5min":1.7989999999999999,"15min":1.7989999999999999},"last":1.1319999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":1.2170000000000001,"15min":1.149},"min":{"1min":0.95999999999999996,"5min":0.83599999999999997,"15min":0.76700000000000002},"max":{"1min":1.647,"5min":1.748,"15min":1.8080000000000001},"last":1.2330000000000001},{"interface":"front","average":{"1min":1.22,"5min":1.204,"15min":1.143},"min":{"1min":0.877,"5min":0.748,"15min":0.68600000000000005},"max":{"1min":1.581,"5min":1.752,"15min":1.752},"last":1.004}]}]},{"osd":1,"up_from":607,"seq":2607045148769,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":81644,"kb_used_data":48740,"kb_used_omap":107,"kb_used_meta":32788,"kb_avail":93663508,"statfs":{"total":95995035648,"available":95911432192,"internally_reserved":0,"allocated":49909760,"data_stored":55550116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":33575595},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.72199999999999998,"15min":0.64800000000000002},"min":{"1min":0.42899999999999999,"5min":0.37,"15min":0.35599999999999998},"max":{"1min":1.1850000000000001,"5min":1.4410000000000001,"15min":1.4410000000000001},"last":1.2050000000000001},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.78400000000000003,"15min":0.68000000000000005},"min":{"1min":0.53900000000000003,"5min":0.47699999999999998,"15min":0.32600000000000001},"max":{"1min":0.98899999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.69299999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:22:23 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.77700000000000002,"15min":0.72599999999999998},"min":{"1min":0.53600000000000003,"5min":0.41499999999999998,"15min":0.38200000000000001},"max":{"1min":1.363,"5min":3.1749999999999998,"15min":3.1749999999999998},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.84199999999999997,"15min":0.82099999999999995},"min":{"1min":0.68700000000000006,"5min":0.45500000000000002,"15min":0.314},"max":{"1min":1.075,"5min":1.333,"15min":3.484},"last":0.80200000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:21:45 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.86899999999999999,"15min":0.82899999999999996},"min":{"1min":0.47299999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.2809999999999999,"5min":1.3160000000000001,"15min":1.393},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.94799999999999995,"15min":0.89900000000000002},"min":{"1min":0.60299999999999998,"5min":0.56599999999999995,"15min":0.50900000000000001},"max":{"1min":1.417,"5min":1.6830000000000001,"15min":1.6830000000000001},"last":1.167}]},{"osd":4,"last update":"Mon Oct 9 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.069,"15min":1.01},"min":{"1min":0.77100000000000002,"5min":0.62,"15min":0.59399999999999997},"max":{"1min":1.4610000000000001,"5min":1.9079999999999999,"15min":1.9079999999999999},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0609999999999999,"15min":1.032},"min":{"1min":0.749,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.4299999999999999,"5min":1.5860000000000001,"15min":1.601},"last":1.018}]},{"osd":5,"last update":"Mon Oct 9 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.1379999999999999,"15min":1.081},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.56499999999999995},"max":{"1min":1.639,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.075},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.129,"15min":1.05},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.627},"max":{"1min":1.593,"5min":1.833,"15min":1.833},"last":1.1120000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.1240000000000001,"15min":1.0620000000000001},"min":{"1min":0.82399999999999995,"5min":0.68300000000000005,"15min":0.57599999999999996},"max":{"1min":1.806,"5min":1.921,"15min":1.921},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.1890000000000001,"15min":1.083},"min":{"1min":0.84199999999999997,"5min":0.76800000000000002,"15min":0.66500000000000004},"max":{"1min":1.837,"5min":1.837,"15min":1.837},"last":1.137}]},{"osd":7,"last update":"Mon Oct 9 09:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.204,"15min":1.163},"min":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"max":{"1min":1.7629999999999999,"5min":1.9990000000000001,"15min":1.9990000000000001},"last":1.2849999999999999},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.2689999999999999,"15min":1.208},"min":{"1min":0.87,"5min":0.78000000000000003,"15min":0.64200000000000002},"max":{"1min":1.9039999999999999,"5min":1.9039999999999999,"15min":1.9039999999999999},"last":1.2589999999999999}]}]},{"osd":0,"up_from":581,"seq":2495375999078,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":113132,"kb_used_data":65828,"kb_used_omap":131,"kb_used_meta":47164,"kb_avail":93632020,"statfs":{"total":95995035648,"available":95879188480,"internally_reserved":0,"allocated":67407872,"data_stored":75511547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":48296537},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:21:49 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.76600000000000001,"15min":0.70399999999999996},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.40600000000000003},"max":{"1min":0.98499999999999999,"5min":1.218,"15min":1.218},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.84299999999999997,"15min":0.76500000000000001},"min":{"1min":0.58799999999999997,"5min":0.53000000000000003,"15min":0.371},"max":{"1min":1.1850000000000001,"5min":1.393,"15min":1.393},"last":1.0149999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.84199999999999997,"15min":0.76700000000000002},"min":{"1min":0.54100000000000004,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.81200000000000006,"15min":0.76000000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.32800000000000001},"max":{"1min":1.669,"5min":1.669,"15min":1.669},"last":0.69699999999999995}]},{"osd":3,"last update":"Mon Oct 9 09:21:42 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.85199999999999998,"15min":0.79200000000000004},"min":{"1min":0.61399999999999999,"5min":0.47499999999999998,"15min":0.41899999999999998},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.95999999999999996,"15min":0.97099999999999997},"min":{"1min":0.70999999999999996,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.28,"5min":1.292,"15min":1.3799999999999999},"last":1.0569999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.0640000000000001,"15min":0.96499999999999997},"min":{"1min":0.69299999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.5229999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.0309999999999999,"15min":1.0369999999999999},"min":{"1min":0.84999999999999998,"5min":0.70499999999999996,"15min":0.67200000000000004},"max":{"1min":1.6739999999999999,"5min":1.6739999999999999,"15min":1.6739999999999999},"last":0.89400000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.1060000000000001,"15min":1.05},"min":{"1min":0.70899999999999996,"5min":0.66600000000000004,"15min":0.66600000000000004},"max":{"1min":1.591,"5min":1.591,"15min":1.591},"last":0.996},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.109,"15min":1.083},"min":{"1min":0.91600000000000004,"5min":0.73499999999999999,"15min":0.68200000000000005},"max":{"1min":1.7669999999999999,"5min":1.8640000000000001,"15min":1.8640000000000001},"last":1.117}]},{"osd":6,"last update":"Mon Oct 9 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.173,"15min":1.077},"min":{"1min":0.78600000000000003,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.9319999999999999,"5min":1.9319999999999999,"15min":1.9319999999999999},"last":1.55},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.1990000000000001,"15min":1.099},"min":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.70699999999999996},"max":{"1min":1.8859999999999999,"5min":1.8859999999999999,"15min":1.8859999999999999},"last":1.091}]},{"osd":7,"last update":"Mon Oct 9 09:21:35 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":1.236,"15min":1.1399999999999999},"min":{"1min":0.88300000000000001,"5min":0.78700000000000003,"15min":0.76600000000000001},"max":{"1min":1.8460000000000001,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":1.669},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.2689999999999999,"15min":1.1739999999999999},"min":{"1min":1.0129999999999999,"5min":0.76700000000000002,"15min":0.76700000000000002},"max":{"1min":1.8029999999999999,"5min":1.8029999999999999,"15min":1.8029999999999999},"last":1.147}]}]},{"osd":3,"up_from":663,"seq":2847563317334,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":59772,"kb_used_data":27764,"kb_used_omap":55,"kb_used_meta":31944,"kb_avail":93685380,"statfs":{"total":95995035648,"available":95933829120,"internally_reserved":0,"allocated":28430336,"data_stored":33711513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":32711266},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:21:50 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.84299999999999997,"15min":0.77600000000000002},"min":{"1min":0.47099999999999997,"5min":0.44900000000000001,"15min":0.29699999999999999},"max":{"1min":1.4350000000000001,"5min":1.51,"15min":1.5369999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.81599999999999995,"15min":0.751},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":1.7929999999999999,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":0.44600000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:21:50 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.90800000000000003,"15min":0.84899999999999998},"min":{"1min":0.56999999999999995,"5min":0.51800000000000002,"15min":0.442},"max":{"1min":1.4490000000000001,"5min":3.9449999999999998,"15min":3.9449999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.85699999999999998,"15min":0.82299999999999995},"min":{"1min":0.61699999999999999,"5min":0.51400000000000001,"15min":0.50700000000000001},"max":{"1min":1.5880000000000001,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":0.58199999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:21:50 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.88100000000000001,"15min":0.84299999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.39500000000000002},"max":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.97499999999999998,"15min":0.84499999999999997},"min":{"1min":0.61399999999999999,"5min":0.50700000000000001,"15min":0.443},"max":{"1min":1.482,"5min":10.167,"15min":10.167},"last":0.60799999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.105,"15min":1.0029999999999999},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.73,"5min":3.8959999999999999,"15min":3.8959999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.004,"5min":1.016,"15min":0.98699999999999999},"min":{"1min":0.751,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.335,"5min":1.6850000000000001,"15min":1.7270000000000001},"last":0.90500000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.1559999999999999,"15min":1.1180000000000001},"min":{"1min":0.91300000000000003,"5min":0.68600000000000005,"15min":0.59999999999999998},"max":{"1min":1.6499999999999999,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":1.161,"5min":1.2,"15min":1.1060000000000001},"min":{"1min":0.77300000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.6020000000000001,"5min":3.996,"15min":3.996},"last":0.93600000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":1.115,"15min":1.071},"min":{"1min":0.83399999999999996,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.4059999999999999,"5min":1.663,"15min":1.663},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.198,"5min":1.2130000000000001,"15min":1.161},"min":{"1min":0.877,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.5,"5min":3.972,"15min":3.972},"last":0.82799999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.232,"15min":1.1950000000000001},"min":{"1min":1.0389999999999999,"5min":0.65500000000000003,"15min":0.61699999999999999},"max":{"1min":1.821,"5min":1.8260000000000001,"15min":1.8260000000000001},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.1659999999999999,"15min":1.1259999999999999},"min":{"1min":0.88600000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.635,"5min":1.845,"15min":1.845},"last":0.79300000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986468,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":365024,"kb_used_data":55512,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93380128,"statfs":{"total":95995035648,"available":95621251072,"internally_reserved":0,"allocated":56844288,"data_stored":68114932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.90200000000000002,"15min":0.88700000000000001},"min":{"1min":0.59799999999999998,"5min":0.48999999999999999,"15min":0.315},"max":{"1min":1.1970000000000001,"5min":1.45,"15min":2.0659999999999998},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.0129999999999999,"15min":0.91700000000000004},"min":{"1min":0.746,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.381,"5min":1.653,"15min":1.653},"last":0.92700000000000005}]},{"osd":1,"last update":"Mon Oct 9 09:21:55 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.91400000000000003,"15min":0.85899999999999999},"min":{"1min":0.71499999999999997,"5min":0.44400000000000001,"15min":0.42699999999999999},"max":{"1min":1.2929999999999999,"5min":1.395,"15min":1.5229999999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.97899999999999998,"15min":1.0109999999999999},"min":{"1min":0.52200000000000002,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.4159999999999999,"5min":1.595,"15min":1.595},"last":0.879}]},{"osd":2,"last update":"Mon Oct 9 09:22:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.0009999999999999,"15min":1.004},"min":{"1min":0.73699999999999999,"5min":0.58499999999999996,"15min":0.5},"max":{"1min":5.7199999999999998,"5min":5.7199999999999998,"15min":5.7199999999999998},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.97399999999999998,"15min":0.96799999999999997},"min":{"1min":0.68899999999999995,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.61399999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:21:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.034,"15min":1.04},"min":{"1min":0.70099999999999996,"5min":0.57499999999999996,"15min":0.54800000000000004},"max":{"1min":2.488,"5min":2.488,"15min":2.488},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.302,"5min":1.1339999999999999,"15min":1.1459999999999999},"min":{"1min":0.76800000000000002,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":4.9009999999999998,"5min":4.9009999999999998,"15min":4.9009999999999998},"last":0.96299999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:21:49 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.94399999999999995,"15min":0.98299999999999998},"min":{"1min":0.499,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.3460000000000001,"5min":1.3839999999999999,"15min":1.7390000000000001},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.121,"15min":1.085},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":2.411,"5min":2.411,"15min":2.411},"last":0.90000000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:21:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.0620000000000001,"15min":1.038},"min":{"1min":0.81299999999999994,"5min":0.48399999999999999,"15min":0.48199999999999998},"max":{"1min":1.5449999999999999,"5min":1.5449999999999999,"15min":2.7719999999999998},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.091,"15min":1.111},"min":{"1min":0.57299999999999995,"5min":0.55500000000000005,"15min":0.46899999999999997},"max":{"1min":1.282,"5min":1.7849999999999999,"15min":1.8680000000000001},"last":0.85899999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.056,"15min":1.048},"min":{"1min":0.56499999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.841,"5min":1.9099999999999999,"15min":1.925},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.1499999999999999,"15min":1.113},"min":{"1min":0.62,"5min":0.58199999999999996,"15min":0.54200000000000004},"max":{"1min":1.7470000000000001,"5min":2.4529999999999998,"15min":2.4529999999999998},"last":0.94799999999999995}]}]},{"osd":5,"up_from":37,"seq":158913790234,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":377216,"kb_used_data":75896,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93367936,"statfs":{"total":95995035648,"available":95608766464,"internally_reserved":0,"allocated":77717504,"data_stored":92198941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.89500000000000002,"15min":0.84499999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.258,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.004},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.95899999999999996,"15min":0.91200000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.2849999999999999,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":1.2230000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.0009999999999999,"15min":0.92900000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.3240000000000001,"5min":3.3490000000000002,"15min":3.3490000000000002},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.095,"15min":1.048},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.4450000000000001,"5min":3.3730000000000002,"15min":3.3730000000000002},"last":0.93400000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:22:22 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0680000000000001,"15min":1.012},"min":{"1min":0.69099999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.304,"5min":2.9220000000000002,"15min":2.9220000000000002},"last":0.996},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.0760000000000001,"15min":0.97399999999999998},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.649,"5min":1.649,"15min":1.649},"last":1.0289999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:21:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.042,"15min":0.97399999999999998},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.61499999999999999},"max":{"1min":1.464,"5min":1.464,"15min":1.464},"last":1.129},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.153,"15min":1.081},"min":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.72299999999999998},"max":{"1min":1.5449999999999999,"5min":4.258,"15min":4.258},"last":0.81599999999999995}]},{"osd":4,"last update":"Mon Oct 9 09:21:57 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.97199999999999998,"15min":0.93700000000000006},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.911},"last":0.71699999999999997},{"interface":"front","average":{"1min":1.02,"5min":1.095,"15min":1.036},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.41199999999999998},"max":{"1min":1.399,"5min":1.625,"15min":2.4990000000000001},"last":1.1659999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0860000000000001,"15min":1.0349999999999999},"min":{"1min":0.60699999999999998,"5min":0.53400000000000003,"15min":0.50800000000000001},"max":{"1min":1.456,"5min":1.637,"15min":2.0419999999999998},"last":0.89700000000000002},{"interface":"front","average":{"1min":1.248,"5min":1.2889999999999999,"15min":1.202},"min":{"1min":0.89600000000000002,"5min":0.69099999999999995,"15min":0.45900000000000002},"max":{"1min":1.742,"5min":1.917,"15min":2.4079999999999999},"last":1.272}]},{"osd":7,"last update":"Mon Oct 9 09:22:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.2430000000000001,"15min":1.153},"min":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.51800000000000002},"max":{"1min":1.9359999999999999,"5min":2.052,"15min":2.556},"last":0.96099999999999997},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.167,"15min":1.1579999999999999},"min":{"1min":0.75700000000000001,"5min":0.51500000000000001,"15min":0.5},"max":{"1min":1.5800000000000001,"5min":2.3460000000000001,"15min":5.8559999999999999},"last":1.0720000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":59633664,"data_stored":68898166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":42758144,"data_stored":49530820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":54067200,"data_stored":70075109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":21413888,"data_stored":27716699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":50716672,"data_stored":62271728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":71606272,"data_stored":86359993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":25726976,"data_stored":29816249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":33267712,"data_stored":42059752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:22:42.504 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T09:22:42.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T09:22:42.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:42 smithi181 bash[59345]: cluster 2023-10-09T09:22:41.505036+0000 mgr.y (mgr.34107) 484 : cluster 0 pgmap v1170: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:42 smithi163 bash[78107]: cluster 2023-10-09T09:22:41.505036+0000 mgr.y (mgr.34107) 484 : cluster 0 pgmap v1170: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:42 smithi163 bash[75425]: cluster 2023-10-09T09:22:41.505036+0000 mgr.y (mgr.34107) 484 : cluster 0 pgmap v1170: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:43 smithi163 bash[78107]: audit 2023-10-09T09:22:41.788058+0000 mgr.y (mgr.34107) 485 : audit 0 from='client.34572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:22:43.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:43 smithi163 bash[75425]: audit 2023-10-09T09:22:41.788058+0000 mgr.y (mgr.34107) 485 : audit 0 from='client.34572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:22:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:22:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:22:43] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.33.4" 2023-10-09T09:22:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:43 smithi181 bash[59345]: audit 2023-10-09T09:22:41.788058+0000 mgr.y (mgr.34107) 485 : audit 0 from='client.34572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:22:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:44 smithi163 bash[78107]: cluster 2023-10-09T09:22:43.505776+0000 mgr.y (mgr.34107) 486 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:44 smithi163 bash[75425]: cluster 2023-10-09T09:22:43.505776+0000 mgr.y (mgr.34107) 486 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:45.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:44 smithi181 bash[59345]: cluster 2023-10-09T09:22:43.505776+0000 mgr.y (mgr.34107) 486 : cluster 0 pgmap v1171: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:46 smithi163 bash[75425]: cluster 2023-10-09T09:22:45.506687+0000 mgr.y (mgr.34107) 487 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:46 smithi163 bash[78107]: cluster 2023-10-09T09:22:45.506687+0000 mgr.y (mgr.34107) 487 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:46 smithi181 bash[59345]: cluster 2023-10-09T09:22:45.506687+0000 mgr.y (mgr.34107) 487 : cluster 0 pgmap v1172: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:47.315 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:48 smithi181 bash[59345]: cluster 2023-10-09T09:22:47.507800+0000 mgr.y (mgr.34107) 488 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:48.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:48 smithi163 bash[75425]: cluster 2023-10-09T09:22:47.507800+0000 mgr.y (mgr.34107) 488 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:48 smithi163 bash[78107]: cluster 2023-10-09T09:22:47.507800+0000 mgr.y (mgr.34107) 488 : cluster 0 pgmap v1173: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:50.578 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_0' removed 2023-10-09T09:22:50.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:50 smithi163 bash[75425]: cluster 2023-10-09T09:22:49.508678+0000 mgr.y (mgr.34107) 489 : cluster 0 pgmap v1174: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:50.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:50 smithi163 bash[75425]: audit 2023-10-09T09:22:50.225224+0000 mon.a (mon.0) 1319 : audit 1 from='client.? 172.21.15.163:0/2851951353' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:22:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:50 smithi163 bash[78107]: cluster 2023-10-09T09:22:49.508678+0000 mgr.y (mgr.34107) 489 : cluster 0 pgmap v1174: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:50.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:50 smithi163 bash[78107]: audit 2023-10-09T09:22:50.225224+0000 mon.a (mon.0) 1319 : audit 1 from='client.? 172.21.15.163:0/2851951353' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:22:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:50 smithi181 bash[59345]: cluster 2023-10-09T09:22:49.508678+0000 mgr.y (mgr.34107) 489 : cluster 0 pgmap v1174: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:50 smithi181 bash[59345]: audit 2023-10-09T09:22:50.225224+0000 mon.a (mon.0) 1319 : audit 1 from='client.? 172.21.15.163:0/2851951353' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:22:51.338 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T09:22:51.338 INFO:teuthology.task.print:**** done rados/snaps-few-objects 5-workload 2023-10-09T09:22:51.339 DEBUG:teuthology.parallel:result is None 2023-10-09T09:22:51.339 INFO:teuthology.run_tasks:Running task print... 2023-10-09T09:22:51.348 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T09:22:51.349 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T09:22:51.373 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T09:22:51.373 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T09:22:51.373 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T09:22:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:51 smithi163 bash[78107]: cluster 2023-10-09T09:22:50.568771+0000 osd.5 (osd.5) 616 : cluster 3 failed to encode map e1074 with expected crc 2023-10-09T09:22:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:51 smithi163 bash[78107]: cluster 2023-10-09T09:22:50.568859+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e1074 with expected crc 2023-10-09T09:22:51.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:51 smithi163 bash[78107]: audit 2023-10-09T09:22:50.572699+0000 mon.a (mon.0) 1320 : audit 1 from='client.? 172.21.15.163:0/2851951353' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:22:51.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:51 smithi163 bash[78107]: cluster 2023-10-09T09:22:50.572846+0000 mon.a (mon.0) 1321 : cluster 0 osdmap e1074: 8 total, 8 up, 8 in 2023-10-09T09:22:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:51 smithi163 bash[75425]: cluster 2023-10-09T09:22:50.568771+0000 osd.5 (osd.5) 616 : cluster 3 failed to encode map e1074 with expected crc 2023-10-09T09:22:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:51 smithi163 bash[75425]: cluster 2023-10-09T09:22:50.568859+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e1074 with expected crc 2023-10-09T09:22:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:51 smithi163 bash[75425]: audit 2023-10-09T09:22:50.572699+0000 mon.a (mon.0) 1320 : audit 1 from='client.? 172.21.15.163:0/2851951353' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:22:51.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:51 smithi163 bash[75425]: cluster 2023-10-09T09:22:50.572846+0000 mon.a (mon.0) 1321 : cluster 0 osdmap e1074: 8 total, 8 up, 8 in 2023-10-09T09:22:52.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:51 smithi181 bash[59345]: cluster 2023-10-09T09:22:50.568771+0000 osd.5 (osd.5) 616 : cluster 3 failed to encode map e1074 with expected crc 2023-10-09T09:22:52.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:51 smithi181 bash[59345]: cluster 2023-10-09T09:22:50.568859+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e1074 with expected crc 2023-10-09T09:22:52.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:51 smithi181 bash[59345]: audit 2023-10-09T09:22:50.572699+0000 mon.a (mon.0) 1320 : audit 1 from='client.? 172.21.15.163:0/2851951353' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:22:52.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:51 smithi181 bash[59345]: cluster 2023-10-09T09:22:50.572846+0000 mon.a (mon.0) 1321 : cluster 0 osdmap e1074: 8 total, 8 up, 8 in 2023-10-09T09:22:52.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:52 smithi163 bash[75425]: cluster 2023-10-09T09:22:51.509495+0000 mgr.y (mgr.34107) 490 : cluster 0 pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:52.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:52 smithi163 bash[75425]: cluster 2023-10-09T09:22:51.937131+0000 osd.4 (osd.4) 510 : cluster 3 failed to encode map e1074 with expected crc 2023-10-09T09:22:52.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:52 smithi163 bash[78107]: cluster 2023-10-09T09:22:51.509495+0000 mgr.y (mgr.34107) 490 : cluster 0 pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:52 smithi163 bash[78107]: cluster 2023-10-09T09:22:51.937131+0000 osd.4 (osd.4) 510 : cluster 3 failed to encode map e1074 with expected crc 2023-10-09T09:22:53.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:52 smithi181 bash[59345]: cluster 2023-10-09T09:22:51.509495+0000 mgr.y (mgr.34107) 490 : cluster 0 pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:53.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:52 smithi181 bash[59345]: cluster 2023-10-09T09:22:51.937131+0000 osd.4 (osd.4) 510 : cluster 3 failed to encode map e1074 with expected crc 2023-10-09T09:22:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:22:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:22:53] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.33.4" 2023-10-09T09:22:54.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:54 smithi163 bash[75425]: cluster 2023-10-09T09:22:53.510263+0000 mgr.y (mgr.34107) 491 : cluster 0 pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:54.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:54 smithi163 bash[78107]: cluster 2023-10-09T09:22:53.510263+0000 mgr.y (mgr.34107) 491 : cluster 0 pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:54 smithi181 bash[59345]: cluster 2023-10-09T09:22:53.510263+0000 mgr.y (mgr.34107) 491 : cluster 0 pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:55 smithi163 bash[78107]: audit 2023-10-09T09:22:55.355758+0000 mon.a (mon.0) 1322 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:55.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:55 smithi163 bash[75425]: audit 2023-10-09T09:22:55.355758+0000 mon.a (mon.0) 1322 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:55 smithi181 bash[59345]: audit 2023-10-09T09:22:55.355758+0000 mon.a (mon.0) 1322 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:22:56.145 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:22:56.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:56 smithi163 bash[75425]: cluster 2023-10-09T09:22:55.511133+0000 mgr.y (mgr.34107) 492 : cluster 0 pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:56.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:56 smithi163 bash[78107]: cluster 2023-10-09T09:22:55.511133+0000 mgr.y (mgr.34107) 492 : cluster 0 pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:56 smithi181 bash[59345]: cluster 2023-10-09T09:22:55.511133+0000 mgr.y (mgr.34107) 492 : cluster 0 pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:58.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:58 smithi163 bash[78107]: cluster 2023-10-09T09:22:57.511742+0000 mgr.y (mgr.34107) 493 : cluster 0 pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:58.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:58 smithi163 bash[75425]: cluster 2023-10-09T09:22:57.511742+0000 mgr.y (mgr.34107) 493 : cluster 0 pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:59.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:58 smithi181 bash[59345]: cluster 2023-10-09T09:22:57.511742+0000 mgr.y (mgr.34107) 493 : cluster 0 pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:22:59.189 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:22:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:22:59 smithi163 bash[75425]: audit 2023-10-09T09:22:59.179650+0000 mon.a (mon.0) 1323 : audit 0 from='client.? 172.21.15.163:0/2220349984' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:22:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:22:59 smithi163 bash[78107]: audit 2023-10-09T09:22:59.179650+0000 mon.a (mon.0) 1323 : audit 0 from='client.? 172.21.15.163:0/2220349984' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:23:00.005 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":35,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6800","nonce":3184674423},{"type":"v1","addr":"172.21.15.163:6801","nonce":3184674423}]},"active_addr":"172.21.15.163:6801/3184674423","active_change":"2023-10-09T09:09:25.186097+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25165,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":304,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":1697577797}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":3966931257}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":2955693241}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":1575301941}]}]} 2023-10-09T09:23:00.009 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T09:23:00.009 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T09:23:00.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:23:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:22:59 smithi181 bash[59345]: audit 2023-10-09T09:22:59.179650+0000 mon.a (mon.0) 1323 : audit 0 from='client.? 172.21.15.163:0/2220349984' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:23:00.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:00 smithi163 bash[75425]: cluster 2023-10-09T09:22:59.512466+0000 mgr.y (mgr.34107) 494 : cluster 0 pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:00 smithi163 bash[78107]: cluster 2023-10-09T09:22:59.512466+0000 mgr.y (mgr.34107) 494 : cluster 0 pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:00 smithi181 bash[59345]: cluster 2023-10-09T09:22:59.512466+0000 mgr.y (mgr.34107) 494 : cluster 0 pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:02 smithi163 bash[78107]: cluster 2023-10-09T09:23:01.513333+0000 mgr.y (mgr.34107) 495 : cluster 0 pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:02 smithi163 bash[75425]: cluster 2023-10-09T09:23:01.513333+0000 mgr.y (mgr.34107) 495 : cluster 0 pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:02 smithi181 bash[59345]: cluster 2023-10-09T09:23:01.513333+0000 mgr.y (mgr.34107) 495 : cluster 0 pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:23:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:23:03] "GET /metrics HTTP/1.1" 200 33901 "" "Prometheus/2.33.4" 2023-10-09T09:23:04.785 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:04.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:04 smithi163 bash[75425]: cluster 2023-10-09T09:23:03.514180+0000 mgr.y (mgr.34107) 496 : cluster 0 pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:04.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:04 smithi163 bash[78107]: cluster 2023-10-09T09:23:03.514180+0000 mgr.y (mgr.34107) 496 : cluster 0 pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:05.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:04 smithi181 bash[59345]: cluster 2023-10-09T09:23:03.514180+0000 mgr.y (mgr.34107) 496 : cluster 0 pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:06 smithi163 bash[78107]: cluster 2023-10-09T09:23:05.515027+0000 mgr.y (mgr.34107) 497 : cluster 0 pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:06 smithi163 bash[75425]: cluster 2023-10-09T09:23:05.515027+0000 mgr.y (mgr.34107) 497 : cluster 0 pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:06 smithi181 bash[59345]: cluster 2023-10-09T09:23:05.515027+0000 mgr.y (mgr.34107) 497 : cluster 0 pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:07.719 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:23:07.719 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":1074,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:22:50.565311+0000","last_up_change":"2023-10-09T09:15:35.142507+0000","last_in_change":"2023-10-09T09:00:10.180048+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"152","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:01:33.304980+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:03:23.129371+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:03:23.743990+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":567,"up_from":581,"up_thru":663,"down_at":568,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6803","nonce":2071796309}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6805","nonce":2071796309}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6809","nonce":2071796309}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6807","nonce":2071796309}]},"public_addr":"172.21.15.163:6803/2071796309","cluster_addr":"172.21.15.163:6805/2071796309","heartbeat_back_addr":"172.21.15.163:6809/2071796309","heartbeat_front_addr":"172.21.15.163:6807/2071796309","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":594,"up_from":607,"up_thru":663,"down_at":595,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6811","nonce":3902451458}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6813","nonce":3902451458}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6817","nonce":3902451458}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6815","nonce":3902451458}]},"public_addr":"172.21.15.163:6811/3902451458","cluster_addr":"172.21.15.163:6813/3902451458","heartbeat_back_addr":"172.21.15.163:6817/3902451458","heartbeat_front_addr":"172.21.15.163:6815/3902451458","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":620,"up_from":634,"up_thru":663,"down_at":621,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6819","nonce":2749314903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6821","nonce":2749314903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6825","nonce":2749314903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6823","nonce":2749314903}]},"public_addr":"172.21.15.163:6819/2749314903","cluster_addr":"172.21.15.163:6821/2749314903","heartbeat_back_addr":"172.21.15.163:6825/2749314903","heartbeat_front_addr":"172.21.15.163:6823/2749314903","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":648,"up_from":663,"up_thru":663,"down_at":649,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6827","nonce":198665121}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6829","nonce":198665121}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6833","nonce":198665121}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6831","nonce":198665121}]},"public_addr":"172.21.15.163:6827/198665121","cluster_addr":"172.21.15.163:6829/198665121","heartbeat_back_addr":"172.21.15.163:6833/198665121","heartbeat_front_addr":"172.21.15.163:6831/198665121","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":649,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":567},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":594},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":620},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":648},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/3642753484":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6801/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/1436605859":"2023-10-10T09:09:25.185363+0000","172.21.15.181:0/394087329":"2023-10-10T09:08:23.434950+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/623143106":"2023-10-10T09:06:46.871855+0000","172.21.15.181:0/1788638593":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/689482878":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/3090084368":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.181:0/2794784192":"2023-10-10T09:08:23.434950+0000","172.21.15.181:0/769962302":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/2563190797":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2199287559":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6800/1737049372":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6801/1737049372":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/536711455":"2023-10-10T09:06:46.871855+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6800/3919298633":"2023-10-10T09:06:46.871855+0000","172.21.15.163:6801/3919298633":"2023-10-10T09:06:46.871855+0000","172.21.15.181:6835/3017734555":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/566729834":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/1618377325":"2023-10-10T09:09:25.185363+0000","172.21.15.181:6834/3017734555":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:23:08.466 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T09:23:08.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:23:08.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:08 smithi163 bash[75425]: cluster 2023-10-09T09:23:07.515973+0000 mgr.y (mgr.34107) 498 : cluster 0 pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:08.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:08 smithi163 bash[75425]: audit 2023-10-09T09:23:07.715213+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.163:0/1238066994' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:23:08.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:08 smithi163 bash[78107]: cluster 2023-10-09T09:23:07.515973+0000 mgr.y (mgr.34107) 498 : cluster 0 pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:08.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:08 smithi163 bash[78107]: audit 2023-10-09T09:23:07.715213+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.163:0/1238066994' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:23:09.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:08 smithi181 bash[59345]: cluster 2023-10-09T09:23:07.515973+0000 mgr.y (mgr.34107) 498 : cluster 0 pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:09.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:08 smithi181 bash[59345]: audit 2023-10-09T09:23:07.715213+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.163:0/1238066994' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:23:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:10 smithi163 bash[75425]: cluster 2023-10-09T09:23:09.516782+0000 mgr.y (mgr.34107) 499 : cluster 0 pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:10 smithi163 bash[75425]: audit 2023-10-09T09:23:10.356113+0000 mon.a (mon.0) 1324 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:10.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:10 smithi163 bash[78107]: cluster 2023-10-09T09:23:09.516782+0000 mgr.y (mgr.34107) 499 : cluster 0 pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:10.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:10 smithi163 bash[78107]: audit 2023-10-09T09:23:10.356113+0000 mon.a (mon.0) 1324 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:10 smithi181 bash[59345]: cluster 2023-10-09T09:23:09.516782+0000 mgr.y (mgr.34107) 499 : cluster 0 pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:10 smithi181 bash[59345]: audit 2023-10-09T09:23:10.356113+0000 mon.a (mon.0) 1324 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:12 smithi163 bash[78107]: cluster 2023-10-09T09:23:11.517672+0000 mgr.y (mgr.34107) 500 : cluster 0 pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:12.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:12 smithi163 bash[75425]: cluster 2023-10-09T09:23:11.517672+0000 mgr.y (mgr.34107) 500 : cluster 0 pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:13.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:12 smithi181 bash[59345]: cluster 2023-10-09T09:23:11.517672+0000 mgr.y (mgr.34107) 500 : cluster 0 pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:13.241 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:23:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:23:13] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.33.4" 2023-10-09T09:23:14.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:14 smithi163 bash[78107]: cluster 2023-10-09T09:23:13.518408+0000 mgr.y (mgr.34107) 501 : cluster 0 pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:14 smithi163 bash[75425]: cluster 2023-10-09T09:23:13.518408+0000 mgr.y (mgr.34107) 501 : cluster 0 pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:15.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:14 smithi181 bash[59345]: cluster 2023-10-09T09:23:13.518408+0000 mgr.y (mgr.34107) 501 : cluster 0 pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:16.264 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:23:16.265 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":1074,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:22:50.565311+0000","last_up_change":"2023-10-09T09:15:35.142507+0000","last_in_change":"2023-10-09T09:00:10.180048+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"152","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:01:33.304980+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:03:23.129371+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:03:23.743990+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":567,"up_from":581,"up_thru":663,"down_at":568,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6803","nonce":2071796309}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6805","nonce":2071796309}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6809","nonce":2071796309}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":2071796309},{"type":"v1","addr":"172.21.15.163:6807","nonce":2071796309}]},"public_addr":"172.21.15.163:6803/2071796309","cluster_addr":"172.21.15.163:6805/2071796309","heartbeat_back_addr":"172.21.15.163:6809/2071796309","heartbeat_front_addr":"172.21.15.163:6807/2071796309","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":594,"up_from":607,"up_thru":663,"down_at":595,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6811","nonce":3902451458}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6813","nonce":3902451458}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6817","nonce":3902451458}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":3902451458},{"type":"v1","addr":"172.21.15.163:6815","nonce":3902451458}]},"public_addr":"172.21.15.163:6811/3902451458","cluster_addr":"172.21.15.163:6813/3902451458","heartbeat_back_addr":"172.21.15.163:6817/3902451458","heartbeat_front_addr":"172.21.15.163:6815/3902451458","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":620,"up_from":634,"up_thru":663,"down_at":621,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6819","nonce":2749314903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6821","nonce":2749314903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6825","nonce":2749314903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6823","nonce":2749314903}]},"public_addr":"172.21.15.163:6819/2749314903","cluster_addr":"172.21.15.163:6821/2749314903","heartbeat_back_addr":"172.21.15.163:6825/2749314903","heartbeat_front_addr":"172.21.15.163:6823/2749314903","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":648,"up_from":663,"up_thru":663,"down_at":649,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6827","nonce":198665121}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6829","nonce":198665121}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6833","nonce":198665121}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6831","nonce":198665121}]},"public_addr":"172.21.15.163:6827/198665121","cluster_addr":"172.21.15.163:6829/198665121","heartbeat_back_addr":"172.21.15.163:6833/198665121","heartbeat_front_addr":"172.21.15.163:6831/198665121","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":649,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":567},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":594},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":620},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":648},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/3642753484":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/1436605859":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/689482878":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6800/1737049372":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/3090084368":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/2563190797":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6801/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6801/1737049372":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/536711455":"2023-10-10T09:06:46.871855+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.181:0/2794784192":"2023-10-10T09:08:23.434950+0000","172.21.15.181:0/769962302":"2023-10-10T09:08:23.434950+0000","172.21.15.181:0/394087329":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/2199287559":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6800/3919298633":"2023-10-10T09:06:46.871855+0000","172.21.15.163:6801/3919298633":"2023-10-10T09:06:46.871855+0000","172.21.15.163:0/1618377325":"2023-10-10T09:09:25.185363+0000","172.21.15.181:6835/3017734555":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/566729834":"2023-10-10T09:01:07.146560+0000","172.21.15.181:6834/3017734555":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/623143106":"2023-10-10T09:06:46.871855+0000","172.21.15.181:0/1788638593":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1080938935":"2023-10-10T09:01:07.146560+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:23:16.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:16 smithi163 bash[75425]: cluster 2023-10-09T09:23:15.518961+0000 mgr.y (mgr.34107) 502 : cluster 0 pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:16.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:16 smithi163 bash[75425]: audit 2023-10-09T09:23:16.261030+0000 mon.a (mon.0) 1325 : audit 0 from='client.? 172.21.15.163:0/473091665' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:23:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:16 smithi163 bash[78107]: cluster 2023-10-09T09:23:15.518961+0000 mgr.y (mgr.34107) 502 : cluster 0 pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:16 smithi163 bash[78107]: audit 2023-10-09T09:23:16.261030+0000 mon.a (mon.0) 1325 : audit 0 from='client.? 172.21.15.163:0/473091665' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:23:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:16 smithi181 bash[59345]: cluster 2023-10-09T09:23:15.518961+0000 mgr.y (mgr.34107) 502 : cluster 0 pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:16 smithi181 bash[59345]: audit 2023-10-09T09:23:16.261030+0000 mon.a (mon.0) 1325 : audit 0 from='client.? 172.21.15.163:0/473091665' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:23:17.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T09:23:17.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T09:23:17.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T09:23:17.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T09:23:17.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T09:23:17.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T09:23:17.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T09:23:17.039 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T09:23:18.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:18 smithi163 bash[75425]: cluster 2023-10-09T09:23:17.519805+0000 mgr.y (mgr.34107) 503 : cluster 0 pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:18 smithi163 bash[78107]: cluster 2023-10-09T09:23:17.519805+0000 mgr.y (mgr.34107) 503 : cluster 0 pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:19.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:18 smithi181 bash[59345]: cluster 2023-10-09T09:23:17.519805+0000 mgr.y (mgr.34107) 503 : cluster 0 pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:20.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:20 smithi163 bash[78107]: cluster 2023-10-09T09:23:19.520637+0000 mgr.y (mgr.34107) 504 : cluster 0 pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:20 smithi163 bash[75425]: cluster 2023-10-09T09:23:19.520637+0000 mgr.y (mgr.34107) 504 : cluster 0 pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:20 smithi181 bash[59345]: cluster 2023-10-09T09:23:19.520637+0000 mgr.y (mgr.34107) 504 : cluster 0 pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:21.923 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:21.926 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:21.926 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:21.927 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:21.927 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:21.930 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:21.930 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:21.930 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:22.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:22 smithi163 bash[78107]: cluster 2023-10-09T09:23:21.521456+0000 mgr.y (mgr.34107) 505 : cluster 0 pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:22.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:22 smithi163 bash[75425]: cluster 2023-10-09T09:23:21.521456+0000 mgr.y (mgr.34107) 505 : cluster 0 pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:23.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:22 smithi181 bash[59345]: cluster 2023-10-09T09:23:21.521456+0000 mgr.y (mgr.34107) 505 : cluster 0 pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:23:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:23:23] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.33.4" 2023-10-09T09:23:24.805 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:24 smithi163 bash[75425]: cluster 2023-10-09T09:23:23.522143+0000 mgr.y (mgr.34107) 506 : cluster 0 pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:24.806 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:24 smithi163 bash[75425]: audit 2023-10-09T09:23:24.382775+0000 mon.a (mon.0) 1326 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:23:24.806 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:24 smithi163 bash[78107]: cluster 2023-10-09T09:23:23.522143+0000 mgr.y (mgr.34107) 506 : cluster 0 pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:24.806 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:24 smithi163 bash[78107]: audit 2023-10-09T09:23:24.382775+0000 mon.a (mon.0) 1326 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:23:25.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:24 smithi181 bash[59345]: cluster 2023-10-09T09:23:23.522143+0000 mgr.y (mgr.34107) 506 : cluster 0 pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:25.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:24 smithi181 bash[59345]: audit 2023-10-09T09:23:24.382775+0000 mon.a (mon.0) 1326 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:23:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:25 smithi163 bash[78107]: audit 2023-10-09T09:23:25.355708+0000 mon.a (mon.0) 1327 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:25.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:25 smithi163 bash[75425]: audit 2023-10-09T09:23:25.355708+0000 mon.a (mon.0) 1327 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:25 smithi181 bash[59345]: audit 2023-10-09T09:23:25.355708+0000 mon.a (mon.0) 1327 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:26 smithi163 bash[78107]: cluster 2023-10-09T09:23:25.522569+0000 mgr.y (mgr.34107) 507 : cluster 0 pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:26 smithi163 bash[75425]: cluster 2023-10-09T09:23:25.522569+0000 mgr.y (mgr.34107) 507 : cluster 0 pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:26 smithi181 bash[59345]: cluster 2023-10-09T09:23:25.522569+0000 mgr.y (mgr.34107) 507 : cluster 0 pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:28 smithi163 bash[75425]: cluster 2023-10-09T09:23:27.523242+0000 mgr.y (mgr.34107) 508 : cluster 0 pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:28.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:28 smithi163 bash[78107]: cluster 2023-10-09T09:23:27.523242+0000 mgr.y (mgr.34107) 508 : cluster 0 pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:29.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:28 smithi181 bash[59345]: cluster 2023-10-09T09:23:27.523242+0000 mgr.y (mgr.34107) 508 : cluster 0 pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:30.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:30 smithi163 bash[75425]: cluster 2023-10-09T09:23:29.523830+0000 mgr.y (mgr.34107) 509 : cluster 0 pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:30.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:30 smithi163 bash[78107]: cluster 2023-10-09T09:23:29.523830+0000 mgr.y (mgr.34107) 509 : cluster 0 pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:31.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:30 smithi181 bash[59345]: cluster 2023-10-09T09:23:29.523830+0000 mgr.y (mgr.34107) 509 : cluster 0 pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:31 smithi163 bash[75425]: audit 2023-10-09T09:23:30.670527+0000 mon.a (mon.0) 1328 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:31 smithi163 bash[75425]: audit 2023-10-09T09:23:30.676199+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:31 smithi163 bash[75425]: cluster 2023-10-09T09:23:31.524370+0000 mgr.y (mgr.34107) 510 : cluster 0 pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:31.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:31 smithi163 bash[78107]: audit 2023-10-09T09:23:30.670527+0000 mon.a (mon.0) 1328 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:31.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:31 smithi163 bash[78107]: audit 2023-10-09T09:23:30.676199+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:31.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:31 smithi163 bash[78107]: cluster 2023-10-09T09:23:31.524370+0000 mgr.y (mgr.34107) 510 : cluster 0 pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:31 smithi181 bash[59345]: audit 2023-10-09T09:23:30.670527+0000 mon.a (mon.0) 1328 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:31 smithi181 bash[59345]: audit 2023-10-09T09:23:30.676199+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:31 smithi181 bash[59345]: cluster 2023-10-09T09:23:31.524370+0000 mgr.y (mgr.34107) 510 : cluster 0 pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:33.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:23:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:23:33] "GET /metrics HTTP/1.1" 200 33905 "" "Prometheus/2.33.4" 2023-10-09T09:23:34.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:34 smithi163 bash[75425]: cluster 2023-10-09T09:23:33.525072+0000 mgr.y (mgr.34107) 511 : cluster 0 pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:34.868 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:34 smithi163 bash[78107]: cluster 2023-10-09T09:23:33.525072+0000 mgr.y (mgr.34107) 511 : cluster 0 pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:34 smithi181 bash[59345]: cluster 2023-10-09T09:23:33.525072+0000 mgr.y (mgr.34107) 511 : cluster 0 pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:35.405 INFO:teuthology.orchestra.run.smithi163.stdout:2723009265767 2023-10-09T09:23:35.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T09:23:35.943 INFO:teuthology.orchestra.run.smithi163.stdout:210453397781 2023-10-09T09:23:35.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T09:23:36.495 INFO:teuthology.orchestra.run.smithi163.stdout:2495375999090 2023-10-09T09:23:36.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T09:23:36.685 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:36 smithi163 bash[75425]: cluster 2023-10-09T09:23:35.525593+0000 mgr.y (mgr.34107) 512 : cluster 0 pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:36.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:36 smithi163 bash[78107]: cluster 2023-10-09T09:23:35.525593+0000 mgr.y (mgr.34107) 512 : cluster 0 pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:36 smithi181 bash[59345]: cluster 2023-10-09T09:23:35.525593+0000 mgr.y (mgr.34107) 512 : cluster 0 pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:38.036 INFO:teuthology.orchestra.run.smithi163.stdout:158913790245 2023-10-09T09:23:38.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T09:23:38.823 INFO:teuthology.orchestra.run.smithi163.stdout:133143986480 2023-10-09T09:23:38.824 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T09:23:38.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:38 smithi163 bash[78107]: cluster 2023-10-09T09:23:37.526424+0000 mgr.y (mgr.34107) 513 : cluster 0 pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:38.857 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:38 smithi163 bash[75425]: cluster 2023-10-09T09:23:37.526424+0000 mgr.y (mgr.34107) 513 : cluster 0 pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:39.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:38 smithi181 bash[59345]: cluster 2023-10-09T09:23:37.526424+0000 mgr.y (mgr.34107) 513 : cluster 0 pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:39.552 INFO:teuthology.orchestra.run.smithi163.stdout:2847563317346 2023-10-09T09:23:39.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T09:23:39.934 INFO:teuthology.orchestra.run.smithi163.stdout:184683594015 2023-10-09T09:23:39.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T09:23:39.977 INFO:teuthology.orchestra.run.smithi163.stdout:2607045148781 2023-10-09T09:23:39.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T09:23:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:40 smithi163 bash[78107]: cluster 2023-10-09T09:23:39.526935+0000 mgr.y (mgr.34107) 514 : cluster 0 pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:40 smithi163 bash[78107]: audit 2023-10-09T09:23:40.356788+0000 mon.a (mon.0) 1330 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:40 smithi163 bash[75425]: cluster 2023-10-09T09:23:39.526935+0000 mgr.y (mgr.34107) 514 : cluster 0 pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:40 smithi163 bash[75425]: audit 2023-10-09T09:23:40.356788+0000 mon.a (mon.0) 1330 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:40 smithi181 bash[59345]: cluster 2023-10-09T09:23:39.526935+0000 mgr.y (mgr.34107) 514 : cluster 0 pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:40 smithi181 bash[59345]: audit 2023-10-09T09:23:40.356788+0000 mon.a (mon.0) 1330 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:42.060 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:42.061 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:42.061 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:42.063 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[75425]: cluster 2023-10-09T09:23:41.527740+0000 mgr.y (mgr.34107) 515 : cluster 0 pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[75425]: audit 2023-10-09T09:23:41.701692+0000 mon.a (mon.0) 1331 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[75425]: audit 2023-10-09T09:23:41.710232+0000 mon.a (mon.0) 1332 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[75425]: audit 2023-10-09T09:23:42.182486+0000 mon.a (mon.0) 1333 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:23:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[75425]: audit 2023-10-09T09:23:42.183797+0000 mon.a (mon.0) 1334 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:23:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[75425]: audit 2023-10-09T09:23:42.189579+0000 mon.a (mon.0) 1335 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[78107]: cluster 2023-10-09T09:23:41.527740+0000 mgr.y (mgr.34107) 515 : cluster 0 pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[78107]: audit 2023-10-09T09:23:41.701692+0000 mon.a (mon.0) 1331 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[78107]: audit 2023-10-09T09:23:41.710232+0000 mon.a (mon.0) 1332 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[78107]: audit 2023-10-09T09:23:42.182486+0000 mon.a (mon.0) 1333 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:23:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[78107]: audit 2023-10-09T09:23:42.183797+0000 mon.a (mon.0) 1334 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:23:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:42 smithi163 bash[78107]: audit 2023-10-09T09:23:42.189579+0000 mon.a (mon.0) 1335 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:42 smithi181 bash[59345]: cluster 2023-10-09T09:23:41.527740+0000 mgr.y (mgr.34107) 515 : cluster 0 pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:42 smithi181 bash[59345]: audit 2023-10-09T09:23:41.701692+0000 mon.a (mon.0) 1331 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:43.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:42 smithi181 bash[59345]: audit 2023-10-09T09:23:41.710232+0000 mon.a (mon.0) 1332 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:43.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:42 smithi181 bash[59345]: audit 2023-10-09T09:23:42.182486+0000 mon.a (mon.0) 1333 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:23:43.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:42 smithi181 bash[59345]: audit 2023-10-09T09:23:42.183797+0000 mon.a (mon.0) 1334 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:23:43.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:42 smithi181 bash[59345]: audit 2023-10-09T09:23:42.189579+0000 mon.a (mon.0) 1335 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:23:43.073 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:23:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:23:43] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-09T09:23:44.087 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:44.088 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:44.088 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:23:44.972 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:44 smithi163 bash[75425]: cluster 2023-10-09T09:23:43.528420+0000 mgr.y (mgr.34107) 516 : cluster 0 pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:44.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:44 smithi163 bash[78107]: cluster 2023-10-09T09:23:43.528420+0000 mgr.y (mgr.34107) 516 : cluster 0 pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:45.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:44 smithi181 bash[59345]: cluster 2023-10-09T09:23:43.528420+0000 mgr.y (mgr.34107) 516 : cluster 0 pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:46.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:46 smithi163 bash[78107]: cluster 2023-10-09T09:23:45.529018+0000 mgr.y (mgr.34107) 517 : cluster 0 pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:46.668 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:46 smithi163 bash[75425]: cluster 2023-10-09T09:23:45.529018+0000 mgr.y (mgr.34107) 517 : cluster 0 pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:46 smithi181 bash[59345]: cluster 2023-10-09T09:23:45.529018+0000 mgr.y (mgr.34107) 517 : cluster 0 pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:48.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:48 smithi163 bash[78107]: cluster 2023-10-09T09:23:47.529622+0000 mgr.y (mgr.34107) 518 : cluster 0 pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:48.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:48 smithi163 bash[75425]: cluster 2023-10-09T09:23:47.529622+0000 mgr.y (mgr.34107) 518 : cluster 0 pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:48 smithi181 bash[59345]: cluster 2023-10-09T09:23:47.529622+0000 mgr.y (mgr.34107) 518 : cluster 0 pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:50.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:50 smithi163 bash[75425]: cluster 2023-10-09T09:23:49.530232+0000 mgr.y (mgr.34107) 519 : cluster 0 pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:51.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:50 smithi163 bash[78107]: cluster 2023-10-09T09:23:49.530232+0000 mgr.y (mgr.34107) 519 : cluster 0 pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:50 smithi181 bash[59345]: cluster 2023-10-09T09:23:49.530232+0000 mgr.y (mgr.34107) 519 : cluster 0 pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:52 smithi163 bash[78107]: cluster 2023-10-09T09:23:51.531036+0000 mgr.y (mgr.34107) 520 : cluster 0 pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:52.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:52 smithi163 bash[75425]: cluster 2023-10-09T09:23:51.531036+0000 mgr.y (mgr.34107) 520 : cluster 0 pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:53.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:52 smithi181 bash[59345]: cluster 2023-10-09T09:23:51.531036+0000 mgr.y (mgr.34107) 520 : cluster 0 pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:53.668 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:23:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:23:53] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-09T09:23:54.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:54 smithi163 bash[75425]: cluster 2023-10-09T09:23:53.531483+0000 mgr.y (mgr.34107) 521 : cluster 0 pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:54.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:54 smithi163 bash[78107]: cluster 2023-10-09T09:23:53.531483+0000 mgr.y (mgr.34107) 521 : cluster 0 pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:54 smithi181 bash[59345]: cluster 2023-10-09T09:23:53.531483+0000 mgr.y (mgr.34107) 521 : cluster 0 pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:55.163 INFO:teuthology.orchestra.run.smithi163.stdout:210453397785 2023-10-09T09:23:55.516 INFO:teuthology.orchestra.run.smithi163.stdout:158913790249 2023-10-09T09:23:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:55 smithi163 bash[78107]: audit 2023-10-09T09:23:55.164205+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.163:0/2469983465' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:23:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:55 smithi163 bash[78107]: audit 2023-10-09T09:23:55.356170+0000 mon.a (mon.0) 1336 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:55.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:55 smithi163 bash[78107]: audit 2023-10-09T09:23:55.516798+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.163:0/3088719185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:23:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:55 smithi163 bash[75425]: audit 2023-10-09T09:23:55.164205+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.163:0/2469983465' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:23:55.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:55 smithi163 bash[75425]: audit 2023-10-09T09:23:55.356170+0000 mon.a (mon.0) 1336 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:55 smithi163 bash[75425]: audit 2023-10-09T09:23:55.516798+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.163:0/3088719185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:23:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:55 smithi181 bash[59345]: audit 2023-10-09T09:23:55.164205+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.163:0/2469983465' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:23:56.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:55 smithi181 bash[59345]: audit 2023-10-09T09:23:55.356170+0000 mon.a (mon.0) 1336 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:23:56.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:55 smithi181 bash[59345]: audit 2023-10-09T09:23:55.516798+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.163:0/3088719185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:23:56.889 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:56 smithi163 bash[78107]: cluster 2023-10-09T09:23:55.531927+0000 mgr.y (mgr.34107) 522 : cluster 0 pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:58.806 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:56 smithi163 bash[75425]: cluster 2023-10-09T09:23:55.531927+0000 mgr.y (mgr.34107) 522 : cluster 0 pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:58.807 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397781 got 210453397785 for osd.7 2023-10-09T09:23:58.807 INFO:teuthology.orchestra.run.smithi163.stdout:184683594018 2023-10-09T09:23:58.857 INFO:teuthology.orchestra.run.smithi163.stdout:2723009265772 2023-10-09T09:23:58.862 DEBUG:teuthology.parallel:result is None 2023-10-09T09:23:58.863 INFO:teuthology.orchestra.run.smithi163.stdout:2495375999094 2023-10-09T09:23:58.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:57 smithi163 bash[78107]: audit 2023-10-09T09:23:57.355388+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.163:0/4241064497' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:23:58.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:57 smithi163 bash[78107]: audit 2023-10-09T09:23:57.523484+0000 mon.a (mon.0) 1337 : audit 0 from='client.? 172.21.15.163:0/1829210588' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:23:58.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:57 smithi163 bash[78107]: audit 2023-10-09T09:23:57.558885+0000 mon.a (mon.0) 1338 : audit 0 from='client.? 172.21.15.163:0/3474828220' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:23:58.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:57 smithi163 bash[75425]: audit 2023-10-09T09:23:57.355388+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.163:0/4241064497' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:23:58.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:57 smithi163 bash[75425]: audit 2023-10-09T09:23:57.523484+0000 mon.a (mon.0) 1337 : audit 0 from='client.? 172.21.15.163:0/1829210588' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:23:58.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:57 smithi163 bash[75425]: audit 2023-10-09T09:23:57.558885+0000 mon.a (mon.0) 1338 : audit 0 from='client.? 172.21.15.163:0/3474828220' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:23:58.865 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790245 got 158913790249 for osd.5 2023-10-09T09:23:58.865 DEBUG:teuthology.parallel:result is None 2023-10-09T09:23:58.866 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:56 smithi181 bash[59345]: cluster 2023-10-09T09:23:55.531927+0000 mgr.y (mgr.34107) 522 : cluster 0 pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:58.866 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:57 smithi181 bash[59345]: audit 2023-10-09T09:23:57.355388+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.163:0/4241064497' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:23:58.866 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:57 smithi181 bash[59345]: audit 2023-10-09T09:23:57.523484+0000 mon.a (mon.0) 1337 : audit 0 from='client.? 172.21.15.163:0/1829210588' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:23:58.866 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:57 smithi181 bash[59345]: audit 2023-10-09T09:23:57.558885+0000 mon.a (mon.0) 1338 : audit 0 from='client.? 172.21.15.163:0/3474828220' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:23:58.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:58 smithi163 bash[75425]: cluster 2023-10-09T09:23:57.532425+0000 mgr.y (mgr.34107) 523 : cluster 0 pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:58.965 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:58 smithi163 bash[78107]: cluster 2023-10-09T09:23:57.532425+0000 mgr.y (mgr.34107) 523 : cluster 0 pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:59.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:58 smithi181 bash[59345]: cluster 2023-10-09T09:23:57.532425+0000 mgr.y (mgr.34107) 523 : cluster 0 pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:23:59.314 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2723009265767 got 2723009265772 for osd.2 2023-10-09T09:23:59.317 DEBUG:teuthology.parallel:result is None 2023-10-09T09:23:59.374 INFO:teuthology.orchestra.run.smithi163.stdout:133143986484 2023-10-09T09:23:59.637 INFO:teuthology.orchestra.run.smithi163.stdout:2607045148785 2023-10-09T09:23:59.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:59 smithi163 bash[75425]: audit 2023-10-09T09:23:59.375218+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.163:0/2127539240' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:23:59.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:23:59 smithi163 bash[75425]: audit 2023-10-09T09:23:59.601701+0000 mon.a (mon.0) 1339 : audit 0 from='client.? 172.21.15.163:0/994022749' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:23:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:59 smithi163 bash[78107]: audit 2023-10-09T09:23:59.375218+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.163:0/2127539240' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:23:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:23:59 smithi163 bash[78107]: audit 2023-10-09T09:23:59.601701+0000 mon.a (mon.0) 1339 : audit 0 from='client.? 172.21.15.163:0/994022749' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:24:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:59 smithi181 bash[59345]: audit 2023-10-09T09:23:59.375218+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.163:0/2127539240' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:24:00.074 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:23:59 smithi181 bash[59345]: audit 2023-10-09T09:23:59.601701+0000 mon.a (mon.0) 1339 : audit 0 from='client.? 172.21.15.163:0/994022749' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:24:00.344 INFO:teuthology.orchestra.run.smithi163.stdout:2847563317351 2023-10-09T09:24:00.862 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683594015 got 184683594018 for osd.6 2023-10-09T09:24:00.862 DEBUG:teuthology.parallel:result is None 2023-10-09T09:24:00.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:00 smithi163 bash[75425]: cluster 2023-10-09T09:23:59.532880+0000 mgr.y (mgr.34107) 524 : cluster 0 pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:00.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:00 smithi163 bash[75425]: audit 2023-10-09T09:24:00.345118+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.163:0/3293045072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:24:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:00 smithi163 bash[78107]: cluster 2023-10-09T09:23:59.532880+0000 mgr.y (mgr.34107) 524 : cluster 0 pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:00.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:00 smithi163 bash[78107]: audit 2023-10-09T09:24:00.345118+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.163:0/3293045072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:24:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:00 smithi181 bash[59345]: cluster 2023-10-09T09:23:59.532880+0000 mgr.y (mgr.34107) 524 : cluster 0 pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:00 smithi181 bash[59345]: audit 2023-10-09T09:24:00.345118+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.163:0/3293045072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:24:01.567 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2495375999090 got 2495375999094 for osd.0 2023-10-09T09:24:01.567 DEBUG:teuthology.parallel:result is None 2023-10-09T09:24:01.848 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986480 got 133143986484 for osd.4 2023-10-09T09:24:01.848 DEBUG:teuthology.parallel:result is None 2023-10-09T09:24:02.239 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2607045148781 got 2607045148785 for osd.1 2023-10-09T09:24:02.239 DEBUG:teuthology.parallel:result is None 2023-10-09T09:24:02.325 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2847563317346 got 2847563317351 for osd.3 2023-10-09T09:24:02.325 DEBUG:teuthology.parallel:result is None 2023-10-09T09:24:02.325 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T09:24:02.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:24:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:02 smithi163 bash[78107]: cluster 2023-10-09T09:24:01.533514+0000 mgr.y (mgr.34107) 525 : cluster 0 pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:02 smithi163 bash[75425]: cluster 2023-10-09T09:24:01.533514+0000 mgr.y (mgr.34107) 525 : cluster 0 pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:02 smithi181 bash[59345]: cluster 2023-10-09T09:24:01.533514+0000 mgr.y (mgr.34107) 525 : cluster 0 pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:24:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:24:03] "GET /metrics HTTP/1.1" 200 33901 "" "Prometheus/2.33.4" 2023-10-09T09:24:04.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:04 smithi163 bash[75425]: cluster 2023-10-09T09:24:03.534127+0000 mgr.y (mgr.34107) 526 : cluster 0 pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:04 smithi163 bash[78107]: cluster 2023-10-09T09:24:03.534127+0000 mgr.y (mgr.34107) 526 : cluster 0 pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:05.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:04 smithi181 bash[59345]: cluster 2023-10-09T09:24:03.534127+0000 mgr.y (mgr.34107) 526 : cluster 0 pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:06 smithi163 bash[78107]: cluster 2023-10-09T09:24:05.535000+0000 mgr.y (mgr.34107) 527 : cluster 0 pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:06 smithi163 bash[75425]: cluster 2023-10-09T09:24:05.535000+0000 mgr.y (mgr.34107) 527 : cluster 0 pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:06 smithi181 bash[59345]: cluster 2023-10-09T09:24:05.535000+0000 mgr.y (mgr.34107) 527 : cluster 0 pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:07.100 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:08 smithi163 bash[78107]: cluster 2023-10-09T09:24:07.535744+0000 mgr.y (mgr.34107) 528 : cluster 0 pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:08.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:08 smithi163 bash[75425]: cluster 2023-10-09T09:24:07.535744+0000 mgr.y (mgr.34107) 528 : cluster 0 pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:09.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:08 smithi181 bash[59345]: cluster 2023-10-09T09:24:07.535744+0000 mgr.y (mgr.34107) 528 : cluster 0 pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:09.954 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:24:09.955 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:24:10.677 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1215,"stamp":"2023-10-09T09:24:09.535916+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1428592,"kb_used_data":53616,"kb_used_omap":416,"kb_used_meta":1374495,"kb_avail":748532624,"statfs":{"total":767960285184,"available":766497406976,"internally_reserved":0,"allocated":54902784,"data_stored":49262341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":1407483313},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003818"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1041,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433825+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:22:35.433825+0000","last_peered":"2023-10-09T09:22:35.433825+0000","last_clean":"2023-10-09T09:22:35.433825+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:22:35.433825+0000","last_undegraded":"2023-10-09T09:22:35.433825+0000","last_fullsized":"2023-10-09T09:22:35.433825+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1004,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653453+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:22:51.653453+0000","last_peered":"2023-10-09T09:22:51.653453+0000","last_clean":"2023-10-09T09:22:51.653453+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:22:51.653453+0000","last_undegraded":"2023-10-09T09:22:51.653453+0000","last_fullsized":"2023-10-09T09:22:51.653453+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1007,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378499+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:22:35.378499+0000","last_peered":"2023-10-09T09:22:35.378499+0000","last_clean":"2023-10-09T09:22:35.378499+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:22:35.378499+0000","last_undegraded":"2023-10-09T09:22:35.378499+0000","last_fullsized":"2023-10-09T09:22:35.378499+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":998,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431422+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:22:35.431422+0000","last_peered":"2023-10-09T09:22:35.431422+0000","last_clean":"2023-10-09T09:22:35.431422+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:22:35.431422+0000","last_undegraded":"2023-10-09T09:22:35.431422+0000","last_fullsized":"2023-10-09T09:22:35.431422+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":985,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.256827+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:22:49.256827+0000","last_peered":"2023-10-09T09:22:49.256827+0000","last_clean":"2023-10-09T09:22:49.256827+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:22:49.256827+0000","last_undegraded":"2023-10-09T09:22:49.256827+0000","last_fullsized":"2023-10-09T09:22:49.256827+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438338+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:22:35.438338+0000","last_peered":"2023-10-09T09:22:35.438338+0000","last_clean":"2023-10-09T09:22:35.438338+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:22:35.438338+0000","last_undegraded":"2023-10-09T09:22:35.438338+0000","last_fullsized":"2023-10-09T09:22:35.438338+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624927+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:22:51.624927+0000","last_peered":"2023-10-09T09:22:51.624927+0000","last_clean":"2023-10-09T09:22:51.624927+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:22:51.624927+0000","last_undegraded":"2023-10-09T09:22:51.624927+0000","last_fullsized":"2023-10-09T09:22:51.624927+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":969,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431402+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:22:35.431402+0000","last_peered":"2023-10-09T09:22:35.431402+0000","last_clean":"2023-10-09T09:22:35.431402+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:22:35.431402+0000","last_undegraded":"2023-10-09T09:22:35.431402+0000","last_fullsized":"2023-10-09T09:22:35.431402+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1026,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378044+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:22:35.378044+0000","last_peered":"2023-10-09T09:22:35.378044+0000","last_clean":"2023-10-09T09:22:35.378044+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:22:35.378044+0000","last_undegraded":"2023-10-09T09:22:35.378044+0000","last_fullsized":"2023-10-09T09:22:35.378044+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":980,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431472+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:22:35.431472+0000","last_peered":"2023-10-09T09:22:35.431472+0000","last_clean":"2023-10-09T09:22:35.431472+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:22:35.431472+0000","last_undegraded":"2023-10-09T09:22:35.431472+0000","last_fullsized":"2023-10-09T09:22:35.431472+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435642+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:22:35.435642+0000","last_peered":"2023-10-09T09:22:35.435642+0000","last_clean":"2023-10-09T09:22:35.435642+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:22:35.435642+0000","last_undegraded":"2023-10-09T09:22:35.435642+0000","last_fullsized":"2023-10-09T09:22:35.435642+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1008,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.798999+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:22:51.798999+0000","last_peered":"2023-10-09T09:22:51.798999+0000","last_clean":"2023-10-09T09:22:51.798999+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:22:51.798999+0000","last_undegraded":"2023-10-09T09:22:51.798999+0000","last_fullsized":"2023-10-09T09:22:51.798999+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1032,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433801+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:22:35.433801+0000","last_peered":"2023-10-09T09:22:35.433801+0000","last_clean":"2023-10-09T09:22:35.433801+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:22:35.433801+0000","last_undegraded":"2023-10-09T09:22:35.433801+0000","last_fullsized":"2023-10-09T09:22:35.433801+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433513+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:22:35.433513+0000","last_peered":"2023-10-09T09:22:35.433513+0000","last_clean":"2023-10-09T09:22:35.433513+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:22:35.433513+0000","last_undegraded":"2023-10-09T09:22:35.433513+0000","last_fullsized":"2023-10-09T09:22:35.433513+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435712+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:22:35.435712+0000","last_peered":"2023-10-09T09:22:35.435712+0000","last_clean":"2023-10-09T09:22:35.435712+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:22:35.435712+0000","last_undegraded":"2023-10-09T09:22:35.435712+0000","last_fullsized":"2023-10-09T09:22:35.435712+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":969,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433403+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:22:35.433403+0000","last_peered":"2023-10-09T09:22:35.433403+0000","last_clean":"2023-10-09T09:22:35.433403+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:22:35.433403+0000","last_undegraded":"2023-10-09T09:22:35.433403+0000","last_fullsized":"2023-10-09T09:22:35.433403+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.622061+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:22:51.622061+0000","last_peered":"2023-10-09T09:22:51.622061+0000","last_clean":"2023-10-09T09:22:51.622061+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:22:51.622061+0000","last_undegraded":"2023-10-09T09:22:51.622061+0000","last_fullsized":"2023-10-09T09:22:51.622061+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378062+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:22:35.378062+0000","last_peered":"2023-10-09T09:22:35.378062+0000","last_clean":"2023-10-09T09:22:35.378062+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:22:35.378062+0000","last_undegraded":"2023-10-09T09:22:35.378062+0000","last_fullsized":"2023-10-09T09:22:35.378062+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":987,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.436318+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:22:35.436318+0000","last_peered":"2023-10-09T09:22:35.436318+0000","last_clean":"2023-10-09T09:22:35.436318+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:22:35.436318+0000","last_undegraded":"2023-10-09T09:22:35.436318+0000","last_fullsized":"2023-10-09T09:22:35.436318+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624005+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:22:51.624005+0000","last_peered":"2023-10-09T09:22:51.624005+0000","last_clean":"2023-10-09T09:22:51.624005+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:22:51.624005+0000","last_undegraded":"2023-10-09T09:22:51.624005+0000","last_fullsized":"2023-10-09T09:22:51.624005+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1015,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437450+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:22:35.437450+0000","last_peered":"2023-10-09T09:22:35.437450+0000","last_clean":"2023-10-09T09:22:35.437450+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:22:35.437450+0000","last_undegraded":"2023-10-09T09:22:35.437450+0000","last_fullsized":"2023-10-09T09:22:35.437450+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.257124+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:22:49.257124+0000","last_peered":"2023-10-09T09:22:49.257124+0000","last_clean":"2023-10-09T09:22:49.257124+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:22:49.257124+0000","last_undegraded":"2023-10-09T09:22:49.257124+0000","last_fullsized":"2023-10-09T09:22:49.257124+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1003,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435617+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:22:35.435617+0000","last_peered":"2023-10-09T09:22:35.435617+0000","last_clean":"2023-10-09T09:22:35.435617+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:22:35.435617+0000","last_undegraded":"2023-10-09T09:22:35.435617+0000","last_fullsized":"2023-10-09T09:22:35.435617+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377442+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:22:35.377442+0000","last_peered":"2023-10-09T09:22:35.377442+0000","last_clean":"2023-10-09T09:22:35.377442+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:22:35.377442+0000","last_undegraded":"2023-10-09T09:22:35.377442+0000","last_fullsized":"2023-10-09T09:22:35.377442+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435716+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:22:35.435716+0000","last_peered":"2023-10-09T09:22:35.435716+0000","last_clean":"2023-10-09T09:22:35.435716+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:22:35.435716+0000","last_undegraded":"2023-10-09T09:22:35.435716+0000","last_fullsized":"2023-10-09T09:22:35.435716+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433540+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:22:35.433540+0000","last_peered":"2023-10-09T09:22:35.433540+0000","last_clean":"2023-10-09T09:22:35.433540+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:22:35.433540+0000","last_undegraded":"2023-10-09T09:22:35.433540+0000","last_fullsized":"2023-10-09T09:22:35.433540+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":988,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.436871+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:22:35.436871+0000","last_peered":"2023-10-09T09:22:35.436871+0000","last_clean":"2023-10-09T09:22:35.436871+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:22:35.436871+0000","last_undegraded":"2023-10-09T09:22:35.436871+0000","last_fullsized":"2023-10-09T09:22:35.436871+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1006,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437643+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:22:35.437643+0000","last_peered":"2023-10-09T09:22:35.437643+0000","last_clean":"2023-10-09T09:22:35.437643+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:22:35.437643+0000","last_undegraded":"2023-10-09T09:22:35.437643+0000","last_fullsized":"2023-10-09T09:22:35.437643+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1026,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377882+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:22:35.377882+0000","last_peered":"2023-10-09T09:22:35.377882+0000","last_clean":"2023-10-09T09:22:35.377882+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:22:35.377882+0000","last_undegraded":"2023-10-09T09:22:35.377882+0000","last_fullsized":"2023-10-09T09:22:35.377882+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.622439+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:22:51.622439+0000","last_peered":"2023-10-09T09:22:51.622439+0000","last_clean":"2023-10-09T09:22:51.622439+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:22:51.622439+0000","last_undegraded":"2023-10-09T09:22:51.622439+0000","last_fullsized":"2023-10-09T09:22:51.622439+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":987,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.432256+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:22:35.432256+0000","last_peered":"2023-10-09T09:22:35.432256+0000","last_clean":"2023-10-09T09:22:35.432256+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:22:35.432256+0000","last_undegraded":"2023-10-09T09:22:35.432256+0000","last_fullsized":"2023-10-09T09:22:35.432256+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434896+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:22:35.434896+0000","last_peered":"2023-10-09T09:22:35.434896+0000","last_clean":"2023-10-09T09:22:35.434896+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:22:35.434896+0000","last_undegraded":"2023-10-09T09:22:35.434896+0000","last_fullsized":"2023-10-09T09:22:35.434896+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1007,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437878+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:22:35.437878+0000","last_peered":"2023-10-09T09:22:35.437878+0000","last_clean":"2023-10-09T09:22:35.437878+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:22:35.437878+0000","last_undegraded":"2023-10-09T09:22:35.437878+0000","last_fullsized":"2023-10-09T09:22:35.437878+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434259+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:22:35.434259+0000","last_peered":"2023-10-09T09:22:35.434259+0000","last_clean":"2023-10-09T09:22:35.434259+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:22:35.434259+0000","last_undegraded":"2023-10-09T09:22:35.434259+0000","last_fullsized":"2023-10-09T09:22:35.434259+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433074+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:22:35.433074+0000","last_peered":"2023-10-09T09:22:35.433074+0000","last_clean":"2023-10-09T09:22:35.433074+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:22:35.433074+0000","last_undegraded":"2023-10-09T09:22:35.433074+0000","last_fullsized":"2023-10-09T09:22:35.433074+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434199+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:22:35.434199+0000","last_peered":"2023-10-09T09:22:35.434199+0000","last_clean":"2023-10-09T09:22:35.434199+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:22:35.434199+0000","last_undegraded":"2023-10-09T09:22:35.434199+0000","last_fullsized":"2023-10-09T09:22:35.434199+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":970,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.430466+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:22:35.430466+0000","last_peered":"2023-10-09T09:22:35.430466+0000","last_clean":"2023-10-09T09:22:35.430466+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:22:35.430466+0000","last_undegraded":"2023-10-09T09:22:35.430466+0000","last_fullsized":"2023-10-09T09:22:35.430466+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1008,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801270+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:22:51.801270+0000","last_peered":"2023-10-09T09:22:51.801270+0000","last_clean":"2023-10-09T09:22:51.801270+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:22:51.801270+0000","last_undegraded":"2023-10-09T09:22:51.801270+0000","last_fullsized":"2023-10-09T09:22:51.801270+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1003,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801363+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:22:51.801363+0000","last_peered":"2023-10-09T09:22:51.801363+0000","last_clean":"2023-10-09T09:22:51.801363+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:22:51.801363+0000","last_undegraded":"2023-10-09T09:22:51.801363+0000","last_fullsized":"2023-10-09T09:22:51.801363+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1041,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378075+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:22:35.378075+0000","last_peered":"2023-10-09T09:22:35.378075+0000","last_clean":"2023-10-09T09:22:35.378075+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:22:35.378075+0000","last_undegraded":"2023-10-09T09:22:35.378075+0000","last_fullsized":"2023-10-09T09:22:35.378075+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":993,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.799273+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:22:51.799273+0000","last_peered":"2023-10-09T09:22:51.799273+0000","last_clean":"2023-10-09T09:22:51.799273+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:22:51.799273+0000","last_undegraded":"2023-10-09T09:22:51.799273+0000","last_fullsized":"2023-10-09T09:22:51.799273+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1030,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377624+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:22:35.377624+0000","last_peered":"2023-10-09T09:22:35.377624+0000","last_clean":"2023-10-09T09:22:35.377624+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:22:35.377624+0000","last_undegraded":"2023-10-09T09:22:35.377624+0000","last_fullsized":"2023-10-09T09:22:35.377624+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433321+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:22:35.433321+0000","last_peered":"2023-10-09T09:22:35.433321+0000","last_clean":"2023-10-09T09:22:35.433321+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:22:35.433321+0000","last_undegraded":"2023-10-09T09:22:35.433321+0000","last_fullsized":"2023-10-09T09:22:35.433321+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1023,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.800115+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:22:51.800115+0000","last_peered":"2023-10-09T09:22:51.800115+0000","last_clean":"2023-10-09T09:22:51.800115+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:22:51.800115+0000","last_undegraded":"2023-10-09T09:22:51.800115+0000","last_fullsized":"2023-10-09T09:22:51.800115+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":990,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:50.876223+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:22:50.876223+0000","last_peered":"2023-10-09T09:22:50.876223+0000","last_clean":"2023-10-09T09:22:50.876223+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:22:50.876223+0000","last_undegraded":"2023-10-09T09:22:50.876223+0000","last_fullsized":"2023-10-09T09:22:50.876223+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1008,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:38.583196+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:22:38.583196+0000","last_peered":"2023-10-09T09:22:38.583196+0000","last_clean":"2023-10-09T09:22:38.583196+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:22:38.583196+0000","last_undegraded":"2023-10-09T09:22:38.583196+0000","last_fullsized":"2023-10-09T09:22:38.583196+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801584+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:22:51.801584+0000","last_peered":"2023-10-09T09:22:51.801584+0000","last_clean":"2023-10-09T09:22:51.801584+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:22:51.801584+0000","last_undegraded":"2023-10-09T09:22:51.801584+0000","last_fullsized":"2023-10-09T09:22:51.801584+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1030,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377739+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:22:35.377739+0000","last_peered":"2023-10-09T09:22:35.377739+0000","last_clean":"2023-10-09T09:22:35.377739+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:22:35.377739+0000","last_undegraded":"2023-10-09T09:22:35.377739+0000","last_fullsized":"2023-10-09T09:22:35.377739+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1238,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435393+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:22:35.435393+0000","last_peered":"2023-10-09T09:22:35.435393+0000","last_clean":"2023-10-09T09:22:35.435393+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:22:35.435393+0000","last_undegraded":"2023-10-09T09:22:35.435393+0000","last_fullsized":"2023-10-09T09:22:35.435393+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":982,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652484+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:22:51.652484+0000","last_peered":"2023-10-09T09:22:51.652484+0000","last_clean":"2023-10-09T09:22:51.652484+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:22:51.652484+0000","last_undegraded":"2023-10-09T09:22:51.652484+0000","last_fullsized":"2023-10-09T09:22:51.652484+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":991,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438069+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:22:35.438069+0000","last_peered":"2023-10-09T09:22:35.438069+0000","last_clean":"2023-10-09T09:22:35.438069+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:22:35.438069+0000","last_undegraded":"2023-10-09T09:22:35.438069+0000","last_fullsized":"2023-10-09T09:22:35.438069+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1021,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435529+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:22:35.435529+0000","last_peered":"2023-10-09T09:22:35.435529+0000","last_clean":"2023-10-09T09:22:35.435529+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:22:35.435529+0000","last_undegraded":"2023-10-09T09:22:35.435529+0000","last_fullsized":"2023-10-09T09:22:35.435529+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":999,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.429996+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:22:35.429996+0000","last_peered":"2023-10-09T09:22:35.429996+0000","last_clean":"2023-10-09T09:22:35.429996+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:22:35.429996+0000","last_undegraded":"2023-10-09T09:22:35.429996+0000","last_fullsized":"2023-10-09T09:22:35.429996+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1010,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.621540+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:22:51.621540+0000","last_peered":"2023-10-09T09:22:51.621540+0000","last_clean":"2023-10-09T09:22:51.621540+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:22:51.621540+0000","last_undegraded":"2023-10-09T09:22:51.621540+0000","last_fullsized":"2023-10-09T09:22:51.621540+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1006,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435480+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:22:35.435480+0000","last_peered":"2023-10-09T09:22:35.435480+0000","last_clean":"2023-10-09T09:22:35.435480+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:22:35.435480+0000","last_undegraded":"2023-10-09T09:22:35.435480+0000","last_fullsized":"2023-10-09T09:22:35.435480+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1009,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.799014+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:22:51.799014+0000","last_peered":"2023-10-09T09:22:51.799014+0000","last_clean":"2023-10-09T09:22:51.799014+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:22:51.799014+0000","last_undegraded":"2023-10-09T09:22:51.799014+0000","last_fullsized":"2023-10-09T09:22:51.799014+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":983,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431109+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:22:35.431109+0000","last_peered":"2023-10-09T09:22:35.431109+0000","last_clean":"2023-10-09T09:22:35.431109+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:22:35.431109+0000","last_undegraded":"2023-10-09T09:22:35.431109+0000","last_fullsized":"2023-10-09T09:22:35.431109+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":987,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434037+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:22:35.434037+0000","last_peered":"2023-10-09T09:22:35.434037+0000","last_clean":"2023-10-09T09:22:35.434037+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:22:35.434037+0000","last_undegraded":"2023-10-09T09:22:35.434037+0000","last_fullsized":"2023-10-09T09:22:35.434037+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":992,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.800514+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:22:51.800514+0000","last_peered":"2023-10-09T09:22:51.800514+0000","last_clean":"2023-10-09T09:22:51.800514+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:22:51.800514+0000","last_undegraded":"2023-10-09T09:22:51.800514+0000","last_fullsized":"2023-10-09T09:22:51.800514+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":985,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652104+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:22:51.652104+0000","last_peered":"2023-10-09T09:22:51.652104+0000","last_clean":"2023-10-09T09:22:51.652104+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:22:51.652104+0000","last_undegraded":"2023-10-09T09:22:51.652104+0000","last_fullsized":"2023-10-09T09:22:51.652104+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438335+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:22:35.438335+0000","last_peered":"2023-10-09T09:22:35.438335+0000","last_clean":"2023-10-09T09:22:35.438335+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:22:35.438335+0000","last_undegraded":"2023-10-09T09:22:35.438335+0000","last_fullsized":"2023-10-09T09:22:35.438335+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1002,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.625144+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:22:51.625144+0000","last_peered":"2023-10-09T09:22:51.625144+0000","last_clean":"2023-10-09T09:22:51.625144+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:22:51.625144+0000","last_undegraded":"2023-10-09T09:22:51.625144+0000","last_fullsized":"2023-10-09T09:22:51.625144+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1009,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433219+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:22:35.433219+0000","last_peered":"2023-10-09T09:22:35.433219+0000","last_clean":"2023-10-09T09:22:35.433219+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:22:35.433219+0000","last_undegraded":"2023-10-09T09:22:35.433219+0000","last_fullsized":"2023-10-09T09:22:35.433219+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":986,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431007+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:22:35.431007+0000","last_peered":"2023-10-09T09:22:35.431007+0000","last_clean":"2023-10-09T09:22:35.431007+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:22:35.431007+0000","last_undegraded":"2023-10-09T09:22:35.431007+0000","last_fullsized":"2023-10-09T09:22:35.431007+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1008,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433286+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:22:35.433286+0000","last_peered":"2023-10-09T09:22:35.433286+0000","last_clean":"2023-10-09T09:22:35.433286+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:22:35.433286+0000","last_undegraded":"2023-10-09T09:22:35.433286+0000","last_fullsized":"2023-10-09T09:22:35.433286+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1028,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378083+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:22:35.378083+0000","last_peered":"2023-10-09T09:22:35.378083+0000","last_clean":"2023-10-09T09:22:35.378083+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:22:35.378083+0000","last_undegraded":"2023-10-09T09:22:35.378083+0000","last_fullsized":"2023-10-09T09:22:35.378083+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433836+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:22:35.433836+0000","last_peered":"2023-10-09T09:22:35.433836+0000","last_clean":"2023-10-09T09:22:35.433836+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:22:35.433836+0000","last_undegraded":"2023-10-09T09:22:35.433836+0000","last_fullsized":"2023-10-09T09:22:35.433836+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1013,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378205+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:22:35.378205+0000","last_peered":"2023-10-09T09:22:35.378205+0000","last_clean":"2023-10-09T09:22:35.378205+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:22:35.378205+0000","last_undegraded":"2023-10-09T09:22:35.378205+0000","last_fullsized":"2023-10-09T09:22:35.378205+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433894+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:22:35.433894+0000","last_peered":"2023-10-09T09:22:35.433894+0000","last_clean":"2023-10-09T09:22:35.433894+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:22:35.433894+0000","last_undegraded":"2023-10-09T09:22:35.433894+0000","last_fullsized":"2023-10-09T09:22:35.433894+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653248+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:22:51.653248+0000","last_peered":"2023-10-09T09:22:51.653248+0000","last_clean":"2023-10-09T09:22:51.653248+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:22:51.653248+0000","last_undegraded":"2023-10-09T09:22:51.653248+0000","last_fullsized":"2023-10-09T09:22:51.653248+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.256889+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:22:49.256889+0000","last_peered":"2023-10-09T09:22:49.256889+0000","last_clean":"2023-10-09T09:22:49.256889+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:22:49.256889+0000","last_undegraded":"2023-10-09T09:22:49.256889+0000","last_fullsized":"2023-10-09T09:22:49.256889+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433005+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:22:35.433005+0000","last_peered":"2023-10-09T09:22:35.433005+0000","last_clean":"2023-10-09T09:22:35.433005+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:22:35.433005+0000","last_undegraded":"2023-10-09T09:22:35.433005+0000","last_fullsized":"2023-10-09T09:22:35.433005+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.432824+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:22:35.432824+0000","last_peered":"2023-10-09T09:22:35.432824+0000","last_clean":"2023-10-09T09:22:35.432824+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:22:35.432824+0000","last_undegraded":"2023-10-09T09:22:35.432824+0000","last_fullsized":"2023-10-09T09:22:35.432824+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397788,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316092,"kb_used_data":6580,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93429060,"statfs":{"total":95995035648,"available":95671357440,"internally_reserved":0,"allocated":6737920,"data_stored":6441392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.075,"15min":1.0449999999999999},"min":{"1min":0.68799999999999994,"5min":0.48599999999999999,"15min":0.41899999999999998},"max":{"1min":1.363,"5min":1.837,"15min":1.837},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.181,"5min":1.1859999999999999,"15min":1.1319999999999999},"min":{"1min":0.94799999999999995,"5min":0.60599999999999998,"15min":0.48599999999999999},"max":{"1min":1.3959999999999999,"5min":1.905,"15min":1.905},"last":1.004}]},{"osd":1,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.93200000000000005,"15min":0.90200000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.145,"5min":1.4019999999999999,"15min":1.5980000000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.016,"15min":0.98399999999999999},"min":{"1min":0.66600000000000004,"5min":0.66500000000000004,"15min":0.60199999999999998},"max":{"1min":1.3049999999999999,"5min":1.722,"15min":2.028},"last":0.92500000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.196,"15min":1.1080000000000001},"min":{"1min":0.73999999999999999,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.5900000000000001,"5min":3.0059999999999998,"15min":3.0059999999999998},"last":1.2150000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0780000000000001,"15min":0.999},"min":{"1min":0.67600000000000005,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2250000000000001,"5min":3.0249999999999999,"15min":3.0249999999999999},"last":1.018}]},{"osd":3,"last update":"Mon Oct 9 09:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0129999999999999,"15min":0.998},"min":{"1min":0.70899999999999996,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.2729999999999999,"5min":1.825,"15min":1.825},"last":0.69999999999999996},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1799999999999999,"15min":1.1399999999999999},"min":{"1min":0.82899999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.51,"5min":1.673,"15min":1.673},"last":0.81100000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.96699999999999997,"15min":0.94899999999999995},"min":{"1min":0.70399999999999996,"5min":0.374,"15min":0.374},"max":{"1min":1.4650000000000001,"5min":1.4650000000000001,"15min":1.6679999999999999},"last":0.84699999999999998},{"interface":"front","average":{"1min":1.022,"5min":0.999,"15min":1.0169999999999999},"min":{"1min":0.72999999999999998,"5min":0.61499999999999999,"15min":0.50900000000000001},"max":{"1min":1.335,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.95199999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.1899999999999999,"15min":1.113},"min":{"1min":0.84899999999999998,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.9019999999999999,"5min":1.9019999999999999,"15min":2.2330000000000001},"last":1.143},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.079,"15min":1.069},"min":{"1min":0.92500000000000004,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.52,"5min":1.6859999999999999,"15min":2.302},"last":0.97899999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.095,"15min":1.0920000000000001},"min":{"1min":0.85299999999999998,"5min":0.60999999999999999,"15min":0.54300000000000004},"max":{"1min":1.569,"5min":1.569,"15min":2.0049999999999999},"last":1.1679999999999999},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.194,"15min":1.1699999999999999},"min":{"1min":0.84599999999999997,"5min":0.69299999999999995,"15min":0.60999999999999999},"max":{"1min":1.716,"5min":2.0070000000000001,"15min":2.0070000000000001},"last":1.1910000000000001}]}]},{"osd":6,"up_from":43,"seq":184683594021,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311984,"kb_used_data":6568,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93433168,"statfs":{"total":95995035648,"available":95675564032,"internally_reserved":0,"allocated":6725632,"data_stored":6441246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.91700000000000004,"15min":0.91900000000000004},"min":{"1min":0.52500000000000002,"5min":0.46500000000000002,"15min":0.35499999999999998},"max":{"1min":1.4910000000000001,"5min":1.72,"15min":2.9060000000000001},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.036,"15min":0.96399999999999997},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.38300000000000001},"max":{"1min":1.3069999999999999,"5min":2.2669999999999999,"15min":2.2669999999999999},"last":0.98199999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.93200000000000005,"15min":0.92500000000000004},"min":{"1min":0.57799999999999996,"5min":0.54300000000000004,"15min":0.40699999999999997},"max":{"1min":1.2470000000000001,"5min":2.508,"15min":3.5590000000000002},"last":1.052},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.92200000000000004,"15min":0.91200000000000003},"min":{"1min":0.49299999999999999,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.2589999999999999,"5min":1.581,"15min":3.3860000000000001},"last":0.92800000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.96999999999999997,"15min":0.92600000000000005},"min":{"1min":0.60899999999999999,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":3.3210000000000002,"5min":3.3210000000000002,"15min":3.3210000000000002},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.02,"5min":1.083,"15min":0.98599999999999999},"min":{"1min":0.75600000000000001,"5min":0.51800000000000002,"15min":0.51200000000000001},"max":{"1min":1.333,"5min":2.2120000000000002,"15min":2.2120000000000002},"last":1.101}]},{"osd":3,"last update":"Mon Oct 9 09:23:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.004,"15min":1.0069999999999999},"min":{"1min":0.627,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":3.8010000000000002,"5min":3.8010000000000002,"15min":4.7430000000000003},"last":1.153},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.052,"15min":1.0700000000000001},"min":{"1min":0.61899999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.3520000000000001,"5min":2.0649999999999999,"15min":3.4249999999999998},"last":1.0309999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:23:40 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.97999999999999998,"15min":0.96699999999999997},"min":{"1min":0.59699999999999998,"5min":0.57499999999999996,"15min":0.39900000000000002},"max":{"1min":1.204,"5min":2.3809999999999998,"15min":3.5640000000000001},"last":1.179},{"interface":"front","average":{"1min":1.004,"5min":1.0469999999999999,"15min":1.0449999999999999},"min":{"1min":0.65700000000000003,"5min":0.57499999999999996,"15min":0.41799999999999998},"max":{"1min":1.3720000000000001,"5min":2.5800000000000001,"15min":4.0629999999999997},"last":1.091}]},{"osd":5,"last update":"Mon Oct 9 09:23:40 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.113,"15min":1.081},"min":{"1min":0.72399999999999998,"5min":0.56000000000000005,"15min":0.46000000000000002},"max":{"1min":1.3220000000000001,"5min":2.7229999999999999,"15min":2.7229999999999999},"last":1.24},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.149,"15min":1.095},"min":{"1min":0.71499999999999997,"5min":0.63300000000000001,"15min":0.48699999999999999},"max":{"1min":1.3320000000000001,"5min":2.7570000000000001,"15min":3.246},"last":1.2}]},{"osd":7,"last update":"Mon Oct 9 09:23:14 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.1819999999999999,"15min":1.141},"min":{"1min":0.755,"5min":0.68200000000000005,"15min":0.49299999999999999},"max":{"1min":1.401,"5min":2.8570000000000002,"15min":3.3340000000000001},"last":1.133},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.1910000000000001,"15min":1.167},"min":{"1min":0.92300000000000004,"5min":0.79300000000000004,"15min":0.46999999999999997},"max":{"1min":1.575,"5min":1.8580000000000001,"15min":2.6659999999999999},"last":1.26}]}]},{"osd":2,"up_from":634,"seq":2723009265774,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":44020,"kb_used_data":7020,"kb_used_omap":122,"kb_used_meta":36869,"kb_avail":93701132,"statfs":{"total":95995035648,"available":95949959168,"internally_reserved":0,"allocated":7188480,"data_stored":6030632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":37754443},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.77100000000000002,"15min":0.74099999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.39800000000000002},"max":{"1min":1.421,"5min":1.421,"15min":1.454},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.752,"15min":0.69199999999999995},"min":{"1min":0.52500000000000002,"5min":0.44700000000000001,"15min":0.33600000000000002},"max":{"1min":0.98999999999999999,"5min":1.1000000000000001,"15min":1.2809999999999999},"last":0.70899999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":0.755,"15min":0.71199999999999997},"min":{"1min":0.55800000000000005,"5min":0.52100000000000002,"15min":0.377},"max":{"1min":1.0669999999999999,"5min":1.2549999999999999,"15min":1.2809999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.84399999999999997,"15min":0.76000000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.0860000000000001,"5min":1.3100000000000001,"15min":1.956},"last":0.99099999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.82999999999999996,"15min":0.79800000000000004},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.40600000000000003},"max":{"1min":1.0960000000000001,"5min":1.1850000000000001,"15min":1.3169999999999999},"last":0.93999999999999995},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.91300000000000003,"15min":0.86699999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.4530000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.91200000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.0940000000000001,"15min":1.0369999999999999},"min":{"1min":0.73499999999999999,"5min":0.70999999999999996,"15min":0.59399999999999997},"max":{"1min":1.603,"5min":2.2559999999999998,"15min":2.2559999999999998},"last":1.044},{"interface":"front","average":{"1min":1.05,"5min":1.05,"15min":1.0329999999999999},"min":{"1min":0.66700000000000004,"5min":0.60999999999999999,"15min":0.59699999999999998},"max":{"1min":1.3169999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.87}]},{"osd":5,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.091,"15min":1.04},"min":{"1min":0.81000000000000005,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.381,"5min":2.226,"15min":2.226},"last":1.069},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1739999999999999,"15min":1.127},"min":{"1min":0.70799999999999996,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.669,"5min":1.696,"15min":1.696},"last":1.1499999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":1.21,"15min":1.1519999999999999},"min":{"1min":0.877,"5min":0.82199999999999995,"15min":0.67700000000000005},"max":{"1min":1.5309999999999999,"5min":1.6160000000000001,"15min":1.716},"last":1.097},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.175,"15min":1.109},"min":{"1min":0.83999999999999997,"5min":0.81000000000000005,"15min":0.68500000000000005},"max":{"1min":1.4770000000000001,"5min":1.7989999999999999,"15min":1.7989999999999999},"last":1.266}]},{"osd":7,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.22,"15min":1.1559999999999999},"min":{"1min":0.96399999999999997,"5min":0.85199999999999998,"15min":0.76700000000000002},"max":{"1min":1.5580000000000001,"5min":1.748,"15min":1.8080000000000001},"last":1.335},{"interface":"front","average":{"1min":1.2,"5min":1.2010000000000001,"15min":1.1499999999999999},"min":{"1min":1.004,"5min":0.748,"15min":0.68600000000000005},"max":{"1min":1.52,"5min":1.752,"15min":1.752},"last":1.22}]}]},{"osd":1,"up_from":607,"seq":2607045148788,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":39904,"kb_used_data":7000,"kb_used_omap":107,"kb_used_meta":32788,"kb_avail":93705248,"statfs":{"total":95995035648,"available":95954173952,"internally_reserved":0,"allocated":7168000,"data_stored":6027385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":33575595},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.67900000000000005,"15min":0.65200000000000002},"min":{"1min":0.377,"5min":0.37,"15min":0.35599999999999998},"max":{"1min":0.94799999999999995,"5min":1.2050000000000001,"15min":1.4410000000000001},"last":0.754},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.73999999999999999,"15min":0.68799999999999994},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.32600000000000001},"max":{"1min":0.86899999999999999,"5min":1.1679999999999999,"15min":1.2729999999999999},"last":0.67800000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:23:24 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.76800000000000002,"15min":0.72499999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.222,"5min":3.1749999999999998,"15min":3.1749999999999998},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.81699999999999995,"15min":0.81899999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.314},"max":{"1min":1.056,"5min":1.333,"15min":3.484},"last":0.88700000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:23:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.83099999999999996,"15min":0.82299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.089,"5min":1.304,"15min":1.393},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.86699999999999999,"15min":0.88600000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.1080000000000001,"5min":1.417,"15min":1.6830000000000001},"last":0.96299999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.032,"15min":1.0129999999999999},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59399999999999997},"max":{"1min":1.3149999999999999,"5min":1.6850000000000001,"15min":1.9079999999999999},"last":1.0580000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.0109999999999999,"15min":1.0309999999999999},"min":{"1min":0.67000000000000004,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.363,"5min":1.5389999999999999,"15min":1.601},"last":1.091}]},{"osd":5,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.073,"15min":1.075},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.56499999999999995},"max":{"1min":1.252,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":0.998},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.077,"15min":1.054},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2669999999999999,"5min":1.605,"15min":1.833},"last":1.0189999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0620000000000001,"15min":1.0580000000000001},"min":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.57599999999999996},"max":{"1min":1.6319999999999999,"5min":1.806,"15min":1.921},"last":1.1339999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.1379999999999999,"15min":1.091},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.5,"5min":1.837,"15min":1.837},"last":1.1619999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.137,"15min":1.1539999999999999},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.349,"5min":1.7629999999999999,"15min":1.9990000000000001},"last":1.1859999999999999},{"interface":"front","average":{"1min":1.044,"5min":1.212,"15min":1.204},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.64200000000000002},"max":{"1min":1.431,"5min":1.9039999999999999,"15min":1.9039999999999999},"last":1.1160000000000001}]}]},{"osd":0,"up_from":581,"seq":2495375999097,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":54912,"kb_used_data":7608,"kb_used_omap":131,"kb_used_meta":47164,"kb_avail":93690240,"statfs":{"total":95995035648,"available":95938805760,"internally_reserved":0,"allocated":7790592,"data_stored":6621470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":48296537},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:23:57 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.73499999999999999,"15min":0.70699999999999996},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.94099999999999995,"5min":1.091,"15min":1.218},"last":0.751},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.80400000000000005,"15min":0.76700000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.97299999999999998,"5min":1.393,"15min":1.393},"last":0.86699999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.83999999999999997,"15min":0.77300000000000002},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.32900000000000001},"max":{"1min":1.0820000000000001,"5min":1.321,"15min":1.321},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.81599999999999995,"15min":0.76400000000000001},"min":{"1min":0.56699999999999995,"5min":0.35899999999999999,"15min":0.32800000000000001},"max":{"1min":1.222,"5min":1.669,"15min":1.669},"last":0.95899999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.85399999999999998,"15min":0.80200000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.071,"5min":1.712,"15min":1.712},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.93600000000000005,"15min":0.95899999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2829999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.83999999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.04,"15min":0.97699999999999998},"min":{"1min":0.66400000000000003,"5min":0.44,"15min":0.44},"max":{"1min":1.2809999999999999,"5min":1.742,"15min":1.742},"last":1.083},{"interface":"front","average":{"1min":0.89400000000000002,"5min":1.0009999999999999,"15min":1.026},"min":{"1min":0.63,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1359999999999999,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":1.0600000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.0860000000000001,"15min":1.054},"min":{"1min":0.61299999999999999,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.3400000000000001,"5min":1.7150000000000001,"15min":1.7150000000000001},"last":1.028},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.071,"15min":1.075},"min":{"1min":0.68600000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.226,"5min":1.7669999999999999,"15min":1.8640000000000001},"last":0.99299999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.159,"15min":1.0900000000000001},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.61499999999999999},"max":{"1min":1.468,"5min":1.9319999999999999,"15min":1.9319999999999999},"last":1.1899999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.1499999999999999,"15min":1.107},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.4930000000000001,"5min":1.8859999999999999,"15min":1.8859999999999999},"last":1.2689999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.2250000000000001,"15min":1.155},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.5169999999999999,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":1.254},{"interface":"front","average":{"1min":1.099,"5min":1.22,"15min":1.177},"min":{"1min":0.72499999999999998,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.5429999999999999,"5min":1.8029999999999999,"15min":1.8029999999999999},"last":1.3240000000000001}]}]},{"osd":3,"up_from":663,"seq":2847563317353,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":38872,"kb_used_data":6864,"kb_used_omap":55,"kb_used_meta":31944,"kb_avail":93706280,"statfs":{"total":95995035648,"available":95955230720,"internally_reserved":0,"allocated":7028736,"data_stored":6002564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":32711266},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.79800000000000004,"15min":0.77500000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.29699999999999999},"max":{"1min":0.88900000000000001,"5min":1.4350000000000001,"15min":1.5369999999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.81399999999999995,"15min":0.75800000000000001},"min":{"1min":0.42799999999999999,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":0.96299999999999997,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":1.0469999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.85699999999999998,"15min":0.83699999999999997},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.442},"max":{"1min":0.88500000000000001,"5min":3.9449999999999998,"15min":3.9449999999999998},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.83399999999999996,"15min":0.82699999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.05,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":0.97199999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.85499999999999998,"15min":0.84299999999999997},"min":{"1min":0.51000000000000001,"5min":0.42499999999999999,"15min":0.39500000000000002},"max":{"1min":1.1459999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":1.137},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.94199999999999995,"15min":0.84399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85499999999999998,"5min":10.167,"15min":10.167},"last":0.66400000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.0780000000000001,"15min":1.01},"min":{"1min":0.746,"5min":0.62,"15min":0.62},"max":{"1min":1.181,"5min":3.8959999999999999,"15min":3.8959999999999999},"last":1.1839999999999999},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.98899999999999999,"15min":0.98899999999999999},"min":{"1min":0.63100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.214,"5min":1.629,"15min":1.7270000000000001},"last":1.113}]},{"osd":5,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.117,"15min":1.115},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.59999999999999998},"max":{"1min":1.2310000000000001,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":1.089},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.1579999999999999,"15min":1.109},"min":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.65100000000000002},"max":{"1min":1.1890000000000001,"5min":3.996,"15min":3.996},"last":1.359}]},{"osd":6,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.095,"15min":1.073},"min":{"1min":0.69599999999999995,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.2170000000000001,"5min":1.663,"15min":1.663},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.179,"15min":1.1579999999999999},"min":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.68999999999999995},"max":{"1min":1.266,"5min":3.972,"15min":3.972},"last":1.286}]},{"osd":7,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.1879999999999999,"15min":1.1910000000000001},"min":{"1min":0.72599999999999998,"5min":0.65500000000000003,"15min":0.61699999999999999},"max":{"1min":1.385,"5min":1.821,"15min":1.8260000000000001},"last":1.2290000000000001},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.1439999999999999,"15min":1.1319999999999999},"min":{"1min":0.77600000000000002,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2929999999999999,"5min":1.7669999999999999,"15min":1.845},"last":0.999}]}]},{"osd":4,"up_from":31,"seq":133143986486,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315508,"kb_used_data":5996,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93429644,"statfs":{"total":95995035648,"available":95671955456,"internally_reserved":0,"allocated":6139904,"data_stored":5850954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.90000000000000002,"15min":0.88900000000000001},"min":{"1min":0.58799999999999997,"5min":0.48999999999999999,"15min":0.315},"max":{"1min":1.2270000000000001,"5min":1.45,"15min":2.0659999999999998},"last":1.123},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.98299999999999998,"15min":0.91600000000000004},"min":{"1min":0.65800000000000003,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.0469999999999999,"5min":1.653,"15min":1.653},"last":0.79000000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.92300000000000004,"15min":0.872},"min":{"1min":0.67900000000000005,"5min":0.375,"15min":0.375},"max":{"1min":1.2250000000000001,"5min":1.5489999999999999,"15min":1.5489999999999999},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.98799999999999999,"15min":1.0029999999999999},"min":{"1min":0.53600000000000003,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.4590000000000001,"5min":1.722,"15min":1.722},"last":1.0509999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:23:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.97599999999999998,"15min":0.99299999999999999},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.0880000000000001,"5min":5.7199999999999998,"15min":5.7199999999999998},"last":1.0980000000000001},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.96399999999999997,"15min":0.96299999999999997},"min":{"1min":0.61399999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.218,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.875}]},{"osd":3,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.997,"15min":1.03},"min":{"1min":0.59099999999999997,"5min":0.57499999999999996,"15min":0.54800000000000004},"max":{"1min":1.3160000000000001,"5min":2.488,"15min":2.488},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.123,"15min":1.133},"min":{"1min":0.751,"5min":0.54300000000000004,"15min":0.41399999999999998},"max":{"1min":1.4119999999999999,"5min":4.9009999999999998,"15min":4.9009999999999998},"last":1.202}]},{"osd":5,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.98599999999999999},"min":{"1min":0.629,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.1639999999999999,"5min":1.6930000000000001,"15min":1.7390000000000001},"last":1.179},{"interface":"front","average":{"1min":1.099,"5min":1.109,"15min":1.107},"min":{"1min":0.69299999999999995,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.4770000000000001,"5min":2.411,"15min":2.411},"last":0.74299999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0880000000000001,"15min":1.0569999999999999},"min":{"1min":0.66100000000000003,"5min":0.504,"15min":0.48199999999999998},"max":{"1min":1.6479999999999999,"5min":1.6479999999999999,"15min":2.7719999999999998},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.095,"5min":1.099,"15min":1.123},"min":{"1min":0.68500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.5800000000000001,"5min":1.7849999999999999,"15min":1.8680000000000001},"last":1.012}]},{"osd":7,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.0609999999999999,"15min":1.054},"min":{"1min":0.73999999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.325,"5min":1.9099999999999999,"15min":1.925},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.159,"5min":1.1579999999999999,"15min":1.123},"min":{"1min":0.94799999999999995,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.3029999999999999,"5min":2.4529999999999998,"15min":2.4529999999999998},"last":1.272}]}]},{"osd":5,"up_from":37,"seq":158913790252,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307300,"kb_used_data":5980,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93437852,"statfs":{"total":95995035648,"available":95680360448,"internally_reserved":0,"allocated":6123520,"data_stored":5846698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.89100000000000001,"15min":0.85199999999999998},"min":{"1min":0.56000000000000005,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.2350000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.069},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.91900000000000004,"15min":0.91100000000000003},"min":{"1min":0.52900000000000003,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.2549999999999999,"5min":1.4359999999999999,"15min":1.6080000000000001},"last":0.98399999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.92500000000000004,"15min":0.93100000000000005},"min":{"1min":0.61699999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.1890000000000001,"5min":1.3240000000000001,"15min":3.3490000000000002},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.0249999999999999,"15min":1.046},"min":{"1min":0.71299999999999997,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.179,"5min":1.5509999999999999,"15min":3.3730000000000002},"last":1.179}]},{"osd":2,"last update":"Mon Oct 9 09:23:24 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.0389999999999999,"15min":1.01},"min":{"1min":0.63900000000000001,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.1419999999999999,"5min":2.9220000000000002,"15min":2.9220000000000002},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.024,"5min":1.0589999999999999,"15min":0.98199999999999998},"min":{"1min":0.54500000000000004,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.294,"5min":1.649,"15min":1.649},"last":0.82899999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.97199999999999998,"15min":0.96999999999999997},"min":{"1min":0.67400000000000004,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.999,"5min":1.464,"15min":1.464},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.1180000000000001,"15min":1.081},"min":{"1min":0.79500000000000004,"5min":0.72299999999999998,"15min":0.72299999999999998},"max":{"1min":1.222,"5min":4.258,"15min":4.258},"last":1.048}]},{"osd":4,"last update":"Mon Oct 9 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.93100000000000005,"15min":0.92800000000000005},"min":{"1min":0.60199999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.0549999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":0.998},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.034,"15min":1.0389999999999999},"min":{"1min":0.66200000000000003,"5min":0.55400000000000005,"15min":0.41199999999999998},"max":{"1min":1.2450000000000001,"5min":1.6120000000000001,"15min":2.4990000000000001},"last":1.097}]},{"osd":6,"last update":"Mon Oct 9 09:23:37 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.0680000000000001,"15min":1.0389999999999999},"min":{"1min":0.67100000000000004,"5min":0.53400000000000003,"15min":0.50800000000000001},"max":{"1min":1.2250000000000001,"5min":1.637,"15min":2.0419999999999998},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.167,"5min":1.2509999999999999,"15min":1.2090000000000001},"min":{"1min":0.74399999999999999,"5min":0.69099999999999995,"15min":0.45900000000000002},"max":{"1min":1.5029999999999999,"5min":1.9079999999999999,"15min":2.4079999999999999},"last":1.1459999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.214,"15min":1.1619999999999999},"min":{"1min":0.88600000000000001,"5min":0.63900000000000001,"15min":0.51800000000000002},"max":{"1min":1.5820000000000001,"5min":2.052,"15min":2.556},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.1599999999999999,"15min":1.153},"min":{"1min":0.755,"5min":0.51500000000000001,"15min":0.5},"max":{"1min":1.458,"5min":2.3460000000000001,"15min":5.8559999999999999},"last":0.85999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:24:10.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:24:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:10 smithi163 bash[75425]: cluster 2023-10-09T09:24:09.536251+0000 mgr.y (mgr.34107) 529 : cluster 0 pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:10 smithi163 bash[75425]: audit 2023-10-09T09:24:10.356605+0000 mon.a (mon.0) 1340 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:10.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:10 smithi163 bash[78107]: cluster 2023-10-09T09:24:09.536251+0000 mgr.y (mgr.34107) 529 : cluster 0 pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:10.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:10 smithi163 bash[78107]: audit 2023-10-09T09:24:10.356605+0000 mon.a (mon.0) 1340 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:10 smithi181 bash[59345]: cluster 2023-10-09T09:24:09.536251+0000 mgr.y (mgr.34107) 529 : cluster 0 pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:11.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:10 smithi181 bash[59345]: audit 2023-10-09T09:24:10.356605+0000 mon.a (mon.0) 1340 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:11.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:11 smithi163 bash[75425]: audit 2023-10-09T09:24:09.934785+0000 mgr.y (mgr.34107) 530 : audit 0 from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:11.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:11 smithi163 bash[78107]: audit 2023-10-09T09:24:09.934785+0000 mgr.y (mgr.34107) 530 : audit 0 from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:12.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:11 smithi181 bash[59345]: audit 2023-10-09T09:24:09.934785+0000 mgr.y (mgr.34107) 530 : audit 0 from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:12.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:12 smithi163 bash[75425]: cluster 2023-10-09T09:24:11.537109+0000 mgr.y (mgr.34107) 531 : cluster 0 pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:12 smithi163 bash[78107]: cluster 2023-10-09T09:24:11.537109+0000 mgr.y (mgr.34107) 531 : cluster 0 pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:13.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:12 smithi181 bash[59345]: cluster 2023-10-09T09:24:11.537109+0000 mgr.y (mgr.34107) 531 : cluster 0 pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:24:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:24:13] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-09T09:24:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:14 smithi163 bash[75425]: cluster 2023-10-09T09:24:13.537890+0000 mgr.y (mgr.34107) 532 : cluster 0 pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:14.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:14 smithi163 bash[78107]: cluster 2023-10-09T09:24:13.537890+0000 mgr.y (mgr.34107) 532 : cluster 0 pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:15.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:14 smithi181 bash[59345]: cluster 2023-10-09T09:24:13.537890+0000 mgr.y (mgr.34107) 532 : cluster 0 pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:15.464 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:16.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:16 smithi163 bash[78107]: cluster 2023-10-09T09:24:15.538424+0000 mgr.y (mgr.34107) 533 : cluster 0 pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:16.869 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:16 smithi163 bash[75425]: cluster 2023-10-09T09:24:15.538424+0000 mgr.y (mgr.34107) 533 : cluster 0 pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:17.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:16 smithi181 bash[59345]: cluster 2023-10-09T09:24:15.538424+0000 mgr.y (mgr.34107) 533 : cluster 0 pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:18.399 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:24:18.403 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:24:18.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:18 smithi163 bash[75425]: cluster 2023-10-09T09:24:17.539379+0000 mgr.y (mgr.34107) 534 : cluster 0 pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:18.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:18 smithi163 bash[78107]: cluster 2023-10-09T09:24:17.539379+0000 mgr.y (mgr.34107) 534 : cluster 0 pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:19.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:18 smithi181 bash[59345]: cluster 2023-10-09T09:24:17.539379+0000 mgr.y (mgr.34107) 534 : cluster 0 pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:19.173 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1219,"stamp":"2023-10-09T09:24:17.538682+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1428592,"kb_used_data":53616,"kb_used_omap":416,"kb_used_meta":1374495,"kb_avail":748532624,"statfs":{"total":767960285184,"available":766497406976,"internally_reserved":0,"allocated":54902784,"data_stored":49262341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":1407483313},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004254"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1041,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433825+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:22:35.433825+0000","last_peered":"2023-10-09T09:22:35.433825+0000","last_clean":"2023-10-09T09:22:35.433825+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:22:35.433825+0000","last_undegraded":"2023-10-09T09:22:35.433825+0000","last_fullsized":"2023-10-09T09:22:35.433825+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1004,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653453+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:22:51.653453+0000","last_peered":"2023-10-09T09:22:51.653453+0000","last_clean":"2023-10-09T09:22:51.653453+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:22:51.653453+0000","last_undegraded":"2023-10-09T09:22:51.653453+0000","last_fullsized":"2023-10-09T09:22:51.653453+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1007,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378499+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:22:35.378499+0000","last_peered":"2023-10-09T09:22:35.378499+0000","last_clean":"2023-10-09T09:22:35.378499+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:22:35.378499+0000","last_undegraded":"2023-10-09T09:22:35.378499+0000","last_fullsized":"2023-10-09T09:22:35.378499+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624753+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:22:51.624753+0000","last_peered":"2023-10-09T09:22:51.624753+0000","last_clean":"2023-10-09T09:22:51.624753+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:22:51.624753+0000","last_undegraded":"2023-10-09T09:22:51.624753+0000","last_fullsized":"2023-10-09T09:22:51.624753+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":985,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.256827+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:22:49.256827+0000","last_peered":"2023-10-09T09:22:49.256827+0000","last_clean":"2023-10-09T09:22:49.256827+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:22:49.256827+0000","last_undegraded":"2023-10-09T09:22:49.256827+0000","last_fullsized":"2023-10-09T09:22:49.256827+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438338+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:22:35.438338+0000","last_peered":"2023-10-09T09:22:35.438338+0000","last_clean":"2023-10-09T09:22:35.438338+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:22:35.438338+0000","last_undegraded":"2023-10-09T09:22:35.438338+0000","last_fullsized":"2023-10-09T09:22:35.438338+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624927+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:22:51.624927+0000","last_peered":"2023-10-09T09:22:51.624927+0000","last_clean":"2023-10-09T09:22:51.624927+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:22:51.624927+0000","last_undegraded":"2023-10-09T09:22:51.624927+0000","last_fullsized":"2023-10-09T09:22:51.624927+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":969,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431402+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:22:35.431402+0000","last_peered":"2023-10-09T09:22:35.431402+0000","last_clean":"2023-10-09T09:22:35.431402+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:22:35.431402+0000","last_undegraded":"2023-10-09T09:22:35.431402+0000","last_fullsized":"2023-10-09T09:22:35.431402+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1026,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378044+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:22:35.378044+0000","last_peered":"2023-10-09T09:22:35.378044+0000","last_clean":"2023-10-09T09:22:35.378044+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:22:35.378044+0000","last_undegraded":"2023-10-09T09:22:35.378044+0000","last_fullsized":"2023-10-09T09:22:35.378044+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":980,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431472+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:22:35.431472+0000","last_peered":"2023-10-09T09:22:35.431472+0000","last_clean":"2023-10-09T09:22:35.431472+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:22:35.431472+0000","last_undegraded":"2023-10-09T09:22:35.431472+0000","last_fullsized":"2023-10-09T09:22:35.431472+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435642+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:22:35.435642+0000","last_peered":"2023-10-09T09:22:35.435642+0000","last_clean":"2023-10-09T09:22:35.435642+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:22:35.435642+0000","last_undegraded":"2023-10-09T09:22:35.435642+0000","last_fullsized":"2023-10-09T09:22:35.435642+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1008,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.798999+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:22:51.798999+0000","last_peered":"2023-10-09T09:22:51.798999+0000","last_clean":"2023-10-09T09:22:51.798999+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:22:51.798999+0000","last_undegraded":"2023-10-09T09:22:51.798999+0000","last_fullsized":"2023-10-09T09:22:51.798999+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1032,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433801+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:22:35.433801+0000","last_peered":"2023-10-09T09:22:35.433801+0000","last_clean":"2023-10-09T09:22:35.433801+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:22:35.433801+0000","last_undegraded":"2023-10-09T09:22:35.433801+0000","last_fullsized":"2023-10-09T09:22:35.433801+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433513+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:22:35.433513+0000","last_peered":"2023-10-09T09:22:35.433513+0000","last_clean":"2023-10-09T09:22:35.433513+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:22:35.433513+0000","last_undegraded":"2023-10-09T09:22:35.433513+0000","last_fullsized":"2023-10-09T09:22:35.433513+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435712+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:22:35.435712+0000","last_peered":"2023-10-09T09:22:35.435712+0000","last_clean":"2023-10-09T09:22:35.435712+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:22:35.435712+0000","last_undegraded":"2023-10-09T09:22:35.435712+0000","last_fullsized":"2023-10-09T09:22:35.435712+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":969,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433403+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:22:35.433403+0000","last_peered":"2023-10-09T09:22:35.433403+0000","last_clean":"2023-10-09T09:22:35.433403+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:22:35.433403+0000","last_undegraded":"2023-10-09T09:22:35.433403+0000","last_fullsized":"2023-10-09T09:22:35.433403+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.622061+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:22:51.622061+0000","last_peered":"2023-10-09T09:22:51.622061+0000","last_clean":"2023-10-09T09:22:51.622061+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:22:51.622061+0000","last_undegraded":"2023-10-09T09:22:51.622061+0000","last_fullsized":"2023-10-09T09:22:51.622061+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378062+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:22:35.378062+0000","last_peered":"2023-10-09T09:22:35.378062+0000","last_clean":"2023-10-09T09:22:35.378062+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:22:35.378062+0000","last_undegraded":"2023-10-09T09:22:35.378062+0000","last_fullsized":"2023-10-09T09:22:35.378062+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":987,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.436318+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:22:35.436318+0000","last_peered":"2023-10-09T09:22:35.436318+0000","last_clean":"2023-10-09T09:22:35.436318+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:22:35.436318+0000","last_undegraded":"2023-10-09T09:22:35.436318+0000","last_fullsized":"2023-10-09T09:22:35.436318+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624005+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:22:51.624005+0000","last_peered":"2023-10-09T09:22:51.624005+0000","last_clean":"2023-10-09T09:22:51.624005+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:22:51.624005+0000","last_undegraded":"2023-10-09T09:22:51.624005+0000","last_fullsized":"2023-10-09T09:22:51.624005+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1015,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437450+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:22:35.437450+0000","last_peered":"2023-10-09T09:22:35.437450+0000","last_clean":"2023-10-09T09:22:35.437450+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:22:35.437450+0000","last_undegraded":"2023-10-09T09:22:35.437450+0000","last_fullsized":"2023-10-09T09:22:35.437450+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.257124+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:22:49.257124+0000","last_peered":"2023-10-09T09:22:49.257124+0000","last_clean":"2023-10-09T09:22:49.257124+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:22:49.257124+0000","last_undegraded":"2023-10-09T09:22:49.257124+0000","last_fullsized":"2023-10-09T09:22:49.257124+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1003,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435617+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:22:35.435617+0000","last_peered":"2023-10-09T09:22:35.435617+0000","last_clean":"2023-10-09T09:22:35.435617+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:22:35.435617+0000","last_undegraded":"2023-10-09T09:22:35.435617+0000","last_fullsized":"2023-10-09T09:22:35.435617+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377442+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:22:35.377442+0000","last_peered":"2023-10-09T09:22:35.377442+0000","last_clean":"2023-10-09T09:22:35.377442+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:22:35.377442+0000","last_undegraded":"2023-10-09T09:22:35.377442+0000","last_fullsized":"2023-10-09T09:22:35.377442+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435716+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:22:35.435716+0000","last_peered":"2023-10-09T09:22:35.435716+0000","last_clean":"2023-10-09T09:22:35.435716+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:22:35.435716+0000","last_undegraded":"2023-10-09T09:22:35.435716+0000","last_fullsized":"2023-10-09T09:22:35.435716+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433540+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:22:35.433540+0000","last_peered":"2023-10-09T09:22:35.433540+0000","last_clean":"2023-10-09T09:22:35.433540+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:22:35.433540+0000","last_undegraded":"2023-10-09T09:22:35.433540+0000","last_fullsized":"2023-10-09T09:22:35.433540+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":988,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.436871+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:22:35.436871+0000","last_peered":"2023-10-09T09:22:35.436871+0000","last_clean":"2023-10-09T09:22:35.436871+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:22:35.436871+0000","last_undegraded":"2023-10-09T09:22:35.436871+0000","last_fullsized":"2023-10-09T09:22:35.436871+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1006,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437643+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:22:35.437643+0000","last_peered":"2023-10-09T09:22:35.437643+0000","last_clean":"2023-10-09T09:22:35.437643+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:22:35.437643+0000","last_undegraded":"2023-10-09T09:22:35.437643+0000","last_fullsized":"2023-10-09T09:22:35.437643+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1026,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377882+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:22:35.377882+0000","last_peered":"2023-10-09T09:22:35.377882+0000","last_clean":"2023-10-09T09:22:35.377882+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:22:35.377882+0000","last_undegraded":"2023-10-09T09:22:35.377882+0000","last_fullsized":"2023-10-09T09:22:35.377882+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.622439+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:22:51.622439+0000","last_peered":"2023-10-09T09:22:51.622439+0000","last_clean":"2023-10-09T09:22:51.622439+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:22:51.622439+0000","last_undegraded":"2023-10-09T09:22:51.622439+0000","last_fullsized":"2023-10-09T09:22:51.622439+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":987,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.432256+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:22:35.432256+0000","last_peered":"2023-10-09T09:22:35.432256+0000","last_clean":"2023-10-09T09:22:35.432256+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:22:35.432256+0000","last_undegraded":"2023-10-09T09:22:35.432256+0000","last_fullsized":"2023-10-09T09:22:35.432256+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434896+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:22:35.434896+0000","last_peered":"2023-10-09T09:22:35.434896+0000","last_clean":"2023-10-09T09:22:35.434896+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:22:35.434896+0000","last_undegraded":"2023-10-09T09:22:35.434896+0000","last_fullsized":"2023-10-09T09:22:35.434896+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1007,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437878+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:22:35.437878+0000","last_peered":"2023-10-09T09:22:35.437878+0000","last_clean":"2023-10-09T09:22:35.437878+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:22:35.437878+0000","last_undegraded":"2023-10-09T09:22:35.437878+0000","last_fullsized":"2023-10-09T09:22:35.437878+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434259+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:22:35.434259+0000","last_peered":"2023-10-09T09:22:35.434259+0000","last_clean":"2023-10-09T09:22:35.434259+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:22:35.434259+0000","last_undegraded":"2023-10-09T09:22:35.434259+0000","last_fullsized":"2023-10-09T09:22:35.434259+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433074+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:22:35.433074+0000","last_peered":"2023-10-09T09:22:35.433074+0000","last_clean":"2023-10-09T09:22:35.433074+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:22:35.433074+0000","last_undegraded":"2023-10-09T09:22:35.433074+0000","last_fullsized":"2023-10-09T09:22:35.433074+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434199+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:22:35.434199+0000","last_peered":"2023-10-09T09:22:35.434199+0000","last_clean":"2023-10-09T09:22:35.434199+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:22:35.434199+0000","last_undegraded":"2023-10-09T09:22:35.434199+0000","last_fullsized":"2023-10-09T09:22:35.434199+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":971,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652272+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:22:51.652272+0000","last_peered":"2023-10-09T09:22:51.652272+0000","last_clean":"2023-10-09T09:22:51.652272+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:22:51.652272+0000","last_undegraded":"2023-10-09T09:22:51.652272+0000","last_fullsized":"2023-10-09T09:22:51.652272+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1008,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801270+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:22:51.801270+0000","last_peered":"2023-10-09T09:22:51.801270+0000","last_clean":"2023-10-09T09:22:51.801270+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:22:51.801270+0000","last_undegraded":"2023-10-09T09:22:51.801270+0000","last_fullsized":"2023-10-09T09:22:51.801270+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1003,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801363+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:22:51.801363+0000","last_peered":"2023-10-09T09:22:51.801363+0000","last_clean":"2023-10-09T09:22:51.801363+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:22:51.801363+0000","last_undegraded":"2023-10-09T09:22:51.801363+0000","last_fullsized":"2023-10-09T09:22:51.801363+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1041,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378075+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:22:35.378075+0000","last_peered":"2023-10-09T09:22:35.378075+0000","last_clean":"2023-10-09T09:22:35.378075+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:22:35.378075+0000","last_undegraded":"2023-10-09T09:22:35.378075+0000","last_fullsized":"2023-10-09T09:22:35.378075+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":993,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.799273+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:22:51.799273+0000","last_peered":"2023-10-09T09:22:51.799273+0000","last_clean":"2023-10-09T09:22:51.799273+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:22:51.799273+0000","last_undegraded":"2023-10-09T09:22:51.799273+0000","last_fullsized":"2023-10-09T09:22:51.799273+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1030,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377624+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:22:35.377624+0000","last_peered":"2023-10-09T09:22:35.377624+0000","last_clean":"2023-10-09T09:22:35.377624+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:22:35.377624+0000","last_undegraded":"2023-10-09T09:22:35.377624+0000","last_fullsized":"2023-10-09T09:22:35.377624+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433321+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:22:35.433321+0000","last_peered":"2023-10-09T09:22:35.433321+0000","last_clean":"2023-10-09T09:22:35.433321+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:22:35.433321+0000","last_undegraded":"2023-10-09T09:22:35.433321+0000","last_fullsized":"2023-10-09T09:22:35.433321+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1023,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.800115+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:22:51.800115+0000","last_peered":"2023-10-09T09:22:51.800115+0000","last_clean":"2023-10-09T09:22:51.800115+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:22:51.800115+0000","last_undegraded":"2023-10-09T09:22:51.800115+0000","last_fullsized":"2023-10-09T09:22:51.800115+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":990,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:50.876223+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:22:50.876223+0000","last_peered":"2023-10-09T09:22:50.876223+0000","last_clean":"2023-10-09T09:22:50.876223+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:22:50.876223+0000","last_undegraded":"2023-10-09T09:22:50.876223+0000","last_fullsized":"2023-10-09T09:22:50.876223+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1008,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:38.583196+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:22:38.583196+0000","last_peered":"2023-10-09T09:22:38.583196+0000","last_clean":"2023-10-09T09:22:38.583196+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:22:38.583196+0000","last_undegraded":"2023-10-09T09:22:38.583196+0000","last_fullsized":"2023-10-09T09:22:38.583196+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801584+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:22:51.801584+0000","last_peered":"2023-10-09T09:22:51.801584+0000","last_clean":"2023-10-09T09:22:51.801584+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:22:51.801584+0000","last_undegraded":"2023-10-09T09:22:51.801584+0000","last_fullsized":"2023-10-09T09:22:51.801584+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1030,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377739+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:22:35.377739+0000","last_peered":"2023-10-09T09:22:35.377739+0000","last_clean":"2023-10-09T09:22:35.377739+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:22:35.377739+0000","last_undegraded":"2023-10-09T09:22:35.377739+0000","last_fullsized":"2023-10-09T09:22:35.377739+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1238,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435393+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:22:35.435393+0000","last_peered":"2023-10-09T09:22:35.435393+0000","last_clean":"2023-10-09T09:22:35.435393+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:22:35.435393+0000","last_undegraded":"2023-10-09T09:22:35.435393+0000","last_fullsized":"2023-10-09T09:22:35.435393+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":982,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652484+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:22:51.652484+0000","last_peered":"2023-10-09T09:22:51.652484+0000","last_clean":"2023-10-09T09:22:51.652484+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:22:51.652484+0000","last_undegraded":"2023-10-09T09:22:51.652484+0000","last_fullsized":"2023-10-09T09:22:51.652484+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":991,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438069+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:22:35.438069+0000","last_peered":"2023-10-09T09:22:35.438069+0000","last_clean":"2023-10-09T09:22:35.438069+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:22:35.438069+0000","last_undegraded":"2023-10-09T09:22:35.438069+0000","last_fullsized":"2023-10-09T09:22:35.438069+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1021,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435529+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:22:35.435529+0000","last_peered":"2023-10-09T09:22:35.435529+0000","last_clean":"2023-10-09T09:22:35.435529+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:22:35.435529+0000","last_undegraded":"2023-10-09T09:22:35.435529+0000","last_fullsized":"2023-10-09T09:22:35.435529+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":999,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.429996+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:22:35.429996+0000","last_peered":"2023-10-09T09:22:35.429996+0000","last_clean":"2023-10-09T09:22:35.429996+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:22:35.429996+0000","last_undegraded":"2023-10-09T09:22:35.429996+0000","last_fullsized":"2023-10-09T09:22:35.429996+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1010,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.621540+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:22:51.621540+0000","last_peered":"2023-10-09T09:22:51.621540+0000","last_clean":"2023-10-09T09:22:51.621540+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:22:51.621540+0000","last_undegraded":"2023-10-09T09:22:51.621540+0000","last_fullsized":"2023-10-09T09:22:51.621540+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1006,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435480+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:22:35.435480+0000","last_peered":"2023-10-09T09:22:35.435480+0000","last_clean":"2023-10-09T09:22:35.435480+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:22:35.435480+0000","last_undegraded":"2023-10-09T09:22:35.435480+0000","last_fullsized":"2023-10-09T09:22:35.435480+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1009,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.799014+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:22:51.799014+0000","last_peered":"2023-10-09T09:22:51.799014+0000","last_clean":"2023-10-09T09:22:51.799014+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:22:51.799014+0000","last_undegraded":"2023-10-09T09:22:51.799014+0000","last_fullsized":"2023-10-09T09:22:51.799014+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":983,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431109+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:22:35.431109+0000","last_peered":"2023-10-09T09:22:35.431109+0000","last_clean":"2023-10-09T09:22:35.431109+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:22:35.431109+0000","last_undegraded":"2023-10-09T09:22:35.431109+0000","last_fullsized":"2023-10-09T09:22:35.431109+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":988,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653895+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:22:51.653895+0000","last_peered":"2023-10-09T09:22:51.653895+0000","last_clean":"2023-10-09T09:22:51.653895+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:22:51.653895+0000","last_undegraded":"2023-10-09T09:22:51.653895+0000","last_fullsized":"2023-10-09T09:22:51.653895+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":992,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.800514+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:22:51.800514+0000","last_peered":"2023-10-09T09:22:51.800514+0000","last_clean":"2023-10-09T09:22:51.800514+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:22:51.800514+0000","last_undegraded":"2023-10-09T09:22:51.800514+0000","last_fullsized":"2023-10-09T09:22:51.800514+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":985,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652104+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:22:51.652104+0000","last_peered":"2023-10-09T09:22:51.652104+0000","last_clean":"2023-10-09T09:22:51.652104+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:22:51.652104+0000","last_undegraded":"2023-10-09T09:22:51.652104+0000","last_fullsized":"2023-10-09T09:22:51.652104+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438335+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:22:35.438335+0000","last_peered":"2023-10-09T09:22:35.438335+0000","last_clean":"2023-10-09T09:22:35.438335+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:22:35.438335+0000","last_undegraded":"2023-10-09T09:22:35.438335+0000","last_fullsized":"2023-10-09T09:22:35.438335+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1002,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.625144+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:22:51.625144+0000","last_peered":"2023-10-09T09:22:51.625144+0000","last_clean":"2023-10-09T09:22:51.625144+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:22:51.625144+0000","last_undegraded":"2023-10-09T09:22:51.625144+0000","last_fullsized":"2023-10-09T09:22:51.625144+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1009,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433219+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:22:35.433219+0000","last_peered":"2023-10-09T09:22:35.433219+0000","last_clean":"2023-10-09T09:22:35.433219+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:22:35.433219+0000","last_undegraded":"2023-10-09T09:22:35.433219+0000","last_fullsized":"2023-10-09T09:22:35.433219+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":987,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624417+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:22:51.624417+0000","last_peered":"2023-10-09T09:22:51.624417+0000","last_clean":"2023-10-09T09:22:51.624417+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:22:51.624417+0000","last_undegraded":"2023-10-09T09:22:51.624417+0000","last_fullsized":"2023-10-09T09:22:51.624417+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1008,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433286+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:22:35.433286+0000","last_peered":"2023-10-09T09:22:35.433286+0000","last_clean":"2023-10-09T09:22:35.433286+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:22:35.433286+0000","last_undegraded":"2023-10-09T09:22:35.433286+0000","last_fullsized":"2023-10-09T09:22:35.433286+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1028,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378083+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:22:35.378083+0000","last_peered":"2023-10-09T09:22:35.378083+0000","last_clean":"2023-10-09T09:22:35.378083+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:22:35.378083+0000","last_undegraded":"2023-10-09T09:22:35.378083+0000","last_fullsized":"2023-10-09T09:22:35.378083+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433836+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:22:35.433836+0000","last_peered":"2023-10-09T09:22:35.433836+0000","last_clean":"2023-10-09T09:22:35.433836+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:22:35.433836+0000","last_undegraded":"2023-10-09T09:22:35.433836+0000","last_fullsized":"2023-10-09T09:22:35.433836+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1013,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378205+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:22:35.378205+0000","last_peered":"2023-10-09T09:22:35.378205+0000","last_clean":"2023-10-09T09:22:35.378205+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:22:35.378205+0000","last_undegraded":"2023-10-09T09:22:35.378205+0000","last_fullsized":"2023-10-09T09:22:35.378205+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433894+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:22:35.433894+0000","last_peered":"2023-10-09T09:22:35.433894+0000","last_clean":"2023-10-09T09:22:35.433894+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:22:35.433894+0000","last_undegraded":"2023-10-09T09:22:35.433894+0000","last_fullsized":"2023-10-09T09:22:35.433894+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653248+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:22:51.653248+0000","last_peered":"2023-10-09T09:22:51.653248+0000","last_clean":"2023-10-09T09:22:51.653248+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:22:51.653248+0000","last_undegraded":"2023-10-09T09:22:51.653248+0000","last_fullsized":"2023-10-09T09:22:51.653248+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.256889+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:22:49.256889+0000","last_peered":"2023-10-09T09:22:49.256889+0000","last_clean":"2023-10-09T09:22:49.256889+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:22:49.256889+0000","last_undegraded":"2023-10-09T09:22:49.256889+0000","last_fullsized":"2023-10-09T09:22:49.256889+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433005+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:22:35.433005+0000","last_peered":"2023-10-09T09:22:35.433005+0000","last_clean":"2023-10-09T09:22:35.433005+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:22:35.433005+0000","last_undegraded":"2023-10-09T09:22:35.433005+0000","last_fullsized":"2023-10-09T09:22:35.433005+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.432824+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:22:35.432824+0000","last_peered":"2023-10-09T09:22:35.432824+0000","last_clean":"2023-10-09T09:22:35.432824+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:22:35.432824+0000","last_undegraded":"2023-10-09T09:22:35.432824+0000","last_fullsized":"2023-10-09T09:22:35.432824+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397790,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316092,"kb_used_data":6580,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93429060,"statfs":{"total":95995035648,"available":95671357440,"internally_reserved":0,"allocated":6737920,"data_stored":6441392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.075,"15min":1.0449999999999999},"min":{"1min":0.68799999999999994,"5min":0.48599999999999999,"15min":0.41899999999999998},"max":{"1min":1.363,"5min":1.837,"15min":1.837},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.181,"5min":1.1859999999999999,"15min":1.1319999999999999},"min":{"1min":0.94799999999999995,"5min":0.60599999999999998,"15min":0.48599999999999999},"max":{"1min":1.3959999999999999,"5min":1.905,"15min":1.905},"last":1.361}]},{"osd":1,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.93200000000000005,"15min":0.90200000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.145,"5min":1.4019999999999999,"15min":1.5980000000000001},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.016,"15min":0.98399999999999999},"min":{"1min":0.66600000000000004,"5min":0.66500000000000004,"15min":0.60199999999999998},"max":{"1min":1.3049999999999999,"5min":1.722,"15min":2.028},"last":1.204}]},{"osd":2,"last update":"Mon Oct 9 09:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.196,"15min":1.1080000000000001},"min":{"1min":0.73999999999999999,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.5900000000000001,"5min":3.0059999999999998,"15min":3.0059999999999998},"last":1.5009999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0780000000000001,"15min":0.999},"min":{"1min":0.67600000000000005,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2250000000000001,"5min":3.0249999999999999,"15min":3.0249999999999999},"last":1.0109999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0129999999999999,"15min":0.998},"min":{"1min":0.70899999999999996,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.2729999999999999,"5min":1.825,"15min":1.825},"last":1.2989999999999999},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1799999999999999,"15min":1.1399999999999999},"min":{"1min":0.82899999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.51,"5min":1.673,"15min":1.673},"last":1.5640000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.96699999999999997,"15min":0.94899999999999995},"min":{"1min":0.70399999999999996,"5min":0.374,"15min":0.374},"max":{"1min":1.4650000000000001,"5min":1.4650000000000001,"15min":1.6679999999999999},"last":1.0489999999999999},{"interface":"front","average":{"1min":1.022,"5min":0.999,"15min":1.0169999999999999},"min":{"1min":0.72999999999999998,"5min":0.61499999999999999,"15min":0.50900000000000001},"max":{"1min":1.335,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.80100000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.1899999999999999,"15min":1.113},"min":{"1min":0.84899999999999998,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.9019999999999999,"5min":1.9019999999999999,"15min":2.2330000000000001},"last":1.4359999999999999},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.079,"15min":1.069},"min":{"1min":0.92500000000000004,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.52,"5min":1.6859999999999999,"15min":2.302},"last":1.179}]},{"osd":6,"last update":"Mon Oct 9 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.095,"15min":1.0920000000000001},"min":{"1min":0.85299999999999998,"5min":0.60999999999999999,"15min":0.54300000000000004},"max":{"1min":1.569,"5min":1.569,"15min":2.0049999999999999},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.194,"15min":1.1699999999999999},"min":{"1min":0.84599999999999997,"5min":0.69299999999999995,"15min":0.60999999999999999},"max":{"1min":1.716,"5min":2.0070000000000001,"15min":2.0070000000000001},"last":1.0840000000000001}]}]},{"osd":6,"up_from":43,"seq":184683594023,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311984,"kb_used_data":6568,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93433168,"statfs":{"total":95995035648,"available":95675564032,"internally_reserved":0,"allocated":6725632,"data_stored":6441246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.91700000000000004,"15min":0.91900000000000004},"min":{"1min":0.52500000000000002,"5min":0.46500000000000002,"15min":0.35499999999999998},"max":{"1min":1.4910000000000001,"5min":1.72,"15min":2.9060000000000001},"last":0.97499999999999998},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.036,"15min":0.96399999999999997},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.38300000000000001},"max":{"1min":1.3069999999999999,"5min":2.2669999999999999,"15min":2.2669999999999999},"last":0.82399999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.93200000000000005,"15min":0.92500000000000004},"min":{"1min":0.57799999999999996,"5min":0.54300000000000004,"15min":0.40699999999999997},"max":{"1min":1.2470000000000001,"5min":2.508,"15min":3.5590000000000002},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.92200000000000004,"15min":0.91200000000000003},"min":{"1min":0.49299999999999999,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.2589999999999999,"5min":1.581,"15min":3.3860000000000001},"last":0.74399999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.96999999999999997,"15min":0.92600000000000005},"min":{"1min":0.60899999999999999,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":3.3210000000000002,"5min":3.3210000000000002,"15min":3.3210000000000002},"last":1.1299999999999999},{"interface":"front","average":{"1min":1.02,"5min":1.083,"15min":0.98599999999999999},"min":{"1min":0.75600000000000001,"5min":0.51800000000000002,"15min":0.51200000000000001},"max":{"1min":1.333,"5min":2.2120000000000002,"15min":2.2120000000000002},"last":0.92700000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:23:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.004,"15min":1.0069999999999999},"min":{"1min":0.627,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":3.8010000000000002,"5min":3.8010000000000002,"15min":4.7430000000000003},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.052,"15min":1.0700000000000001},"min":{"1min":0.61899999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.3520000000000001,"5min":2.0649999999999999,"15min":3.4249999999999998},"last":0.997}]},{"osd":4,"last update":"Mon Oct 9 09:23:40 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.97999999999999998,"15min":0.96699999999999997},"min":{"1min":0.59699999999999998,"5min":0.57499999999999996,"15min":0.39900000000000002},"max":{"1min":1.204,"5min":2.3809999999999998,"15min":3.5640000000000001},"last":1.036},{"interface":"front","average":{"1min":1.004,"5min":1.0469999999999999,"15min":1.0449999999999999},"min":{"1min":0.65700000000000003,"5min":0.57499999999999996,"15min":0.41799999999999998},"max":{"1min":1.3720000000000001,"5min":2.5800000000000001,"15min":4.0629999999999997},"last":1.0760000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:23:40 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.113,"15min":1.081},"min":{"1min":0.72399999999999998,"5min":0.56000000000000005,"15min":0.46000000000000002},"max":{"1min":1.3220000000000001,"5min":2.7229999999999999,"15min":2.7229999999999999},"last":1.089},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.149,"15min":1.095},"min":{"1min":0.71499999999999997,"5min":0.63300000000000001,"15min":0.48699999999999999},"max":{"1min":1.3320000000000001,"5min":2.7570000000000001,"15min":3.246},"last":1.167}]},{"osd":7,"last update":"Mon Oct 9 09:23:14 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.1819999999999999,"15min":1.141},"min":{"1min":0.755,"5min":0.68200000000000005,"15min":0.49299999999999999},"max":{"1min":1.401,"5min":2.8570000000000002,"15min":3.3340000000000001},"last":1.121},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.1910000000000001,"15min":1.167},"min":{"1min":0.92300000000000004,"5min":0.79300000000000004,"15min":0.46999999999999997},"max":{"1min":1.575,"5min":1.8580000000000001,"15min":2.6659999999999999},"last":1.2250000000000001}]}]},{"osd":2,"up_from":634,"seq":2723009265776,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":44020,"kb_used_data":7020,"kb_used_omap":122,"kb_used_meta":36869,"kb_avail":93701132,"statfs":{"total":95995035648,"available":95949959168,"internally_reserved":0,"allocated":7188480,"data_stored":6030632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":37754443},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.77100000000000002,"15min":0.74099999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.39800000000000002},"max":{"1min":1.421,"5min":1.421,"15min":1.454},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.752,"15min":0.69199999999999995},"min":{"1min":0.52500000000000002,"5min":0.44700000000000001,"15min":0.33600000000000002},"max":{"1min":0.98999999999999999,"5min":1.1000000000000001,"15min":1.2809999999999999},"last":1.0249999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":0.755,"15min":0.71199999999999997},"min":{"1min":0.55800000000000005,"5min":0.52100000000000002,"15min":0.377},"max":{"1min":1.0669999999999999,"5min":1.2549999999999999,"15min":1.2809999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.84399999999999997,"15min":0.76000000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.0860000000000001,"5min":1.3100000000000001,"15min":1.956},"last":0.63400000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.82999999999999996,"15min":0.79800000000000004},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.40600000000000003},"max":{"1min":1.0960000000000001,"5min":1.1850000000000001,"15min":1.3169999999999999},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.91300000000000003,"15min":0.86699999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.4530000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.755}]},{"osd":4,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.0940000000000001,"15min":1.0369999999999999},"min":{"1min":0.73499999999999999,"5min":0.70999999999999996,"15min":0.59399999999999997},"max":{"1min":1.603,"5min":2.2559999999999998,"15min":2.2559999999999998},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.05,"5min":1.05,"15min":1.0329999999999999},"min":{"1min":0.66700000000000004,"5min":0.60999999999999999,"15min":0.59699999999999998},"max":{"1min":1.3169999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.81699999999999995}]},{"osd":5,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.091,"15min":1.04},"min":{"1min":0.81000000000000005,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.381,"5min":2.226,"15min":2.226},"last":0.78000000000000003},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1739999999999999,"15min":1.127},"min":{"1min":0.70799999999999996,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.669,"5min":1.696,"15min":1.696},"last":1.1439999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":1.21,"15min":1.1519999999999999},"min":{"1min":0.877,"5min":0.82199999999999995,"15min":0.67700000000000005},"max":{"1min":1.5309999999999999,"5min":1.6160000000000001,"15min":1.716},"last":1.196},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.175,"15min":1.109},"min":{"1min":0.83999999999999997,"5min":0.81000000000000005,"15min":0.68500000000000005},"max":{"1min":1.4770000000000001,"5min":1.7989999999999999,"15min":1.7989999999999999},"last":1.109}]},{"osd":7,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.22,"15min":1.1559999999999999},"min":{"1min":0.96399999999999997,"5min":0.85199999999999998,"15min":0.76700000000000002},"max":{"1min":1.5580000000000001,"5min":1.748,"15min":1.8080000000000001},"last":1.21},{"interface":"front","average":{"1min":1.2,"5min":1.2010000000000001,"15min":1.1499999999999999},"min":{"1min":1.004,"5min":0.748,"15min":0.68600000000000005},"max":{"1min":1.52,"5min":1.752,"15min":1.752},"last":1.236}]}]},{"osd":1,"up_from":607,"seq":2607045148789,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":39904,"kb_used_data":7000,"kb_used_omap":107,"kb_used_meta":32788,"kb_avail":93705248,"statfs":{"total":95995035648,"available":95954173952,"internally_reserved":0,"allocated":7168000,"data_stored":6027385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":33575595},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.67900000000000005,"15min":0.65200000000000002},"min":{"1min":0.377,"5min":0.37,"15min":0.35599999999999998},"max":{"1min":0.94799999999999995,"5min":1.2050000000000001,"15min":1.4410000000000001},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.73999999999999999,"15min":0.68799999999999994},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.32600000000000001},"max":{"1min":0.86899999999999999,"5min":1.1679999999999999,"15min":1.2729999999999999},"last":0.67400000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:23:24 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.76800000000000002,"15min":0.72499999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.222,"5min":3.1749999999999998,"15min":3.1749999999999998},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.81699999999999995,"15min":0.81899999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.314},"max":{"1min":1.056,"5min":1.333,"15min":3.484},"last":0.73599999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:23:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.83099999999999996,"15min":0.82299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.089,"5min":1.304,"15min":1.393},"last":0.995},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.86699999999999999,"15min":0.88600000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.1080000000000001,"5min":1.417,"15min":1.6830000000000001},"last":0.97399999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.032,"15min":1.0129999999999999},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59399999999999997},"max":{"1min":1.3149999999999999,"5min":1.6850000000000001,"15min":1.9079999999999999},"last":1.04},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.0109999999999999,"15min":1.0309999999999999},"min":{"1min":0.67000000000000004,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.363,"5min":1.5389999999999999,"15min":1.601},"last":1.083}]},{"osd":5,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.073,"15min":1.075},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.56499999999999995},"max":{"1min":1.252,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.119},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.077,"15min":1.054},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2669999999999999,"5min":1.605,"15min":1.833},"last":0.91000000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0620000000000001,"15min":1.0580000000000001},"min":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.57599999999999996},"max":{"1min":1.6319999999999999,"5min":1.806,"15min":1.921},"last":1.1990000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.1379999999999999,"15min":1.091},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.5,"5min":1.837,"15min":1.837},"last":1.179}]},{"osd":7,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.137,"15min":1.1539999999999999},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.349,"5min":1.7629999999999999,"15min":1.9990000000000001},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.044,"5min":1.212,"15min":1.204},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.64200000000000002},"max":{"1min":1.431,"5min":1.9039999999999999,"15min":1.9039999999999999},"last":1.274}]}]},{"osd":0,"up_from":581,"seq":2495375999099,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":54912,"kb_used_data":7608,"kb_used_omap":131,"kb_used_meta":47164,"kb_avail":93690240,"statfs":{"total":95995035648,"available":95938805760,"internally_reserved":0,"allocated":7790592,"data_stored":6621470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":48296537},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:23:57 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.73499999999999999,"15min":0.70699999999999996},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.94099999999999995,"5min":1.091,"15min":1.218},"last":1.036},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.80400000000000005,"15min":0.76700000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.97299999999999998,"5min":1.393,"15min":1.393},"last":0.84499999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.83999999999999997,"15min":0.77300000000000002},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.32900000000000001},"max":{"1min":1.0820000000000001,"5min":1.321,"15min":1.321},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.81599999999999995,"15min":0.76400000000000001},"min":{"1min":0.56699999999999995,"5min":0.35899999999999999,"15min":0.32800000000000001},"max":{"1min":1.222,"5min":1.669,"15min":1.669},"last":0.90900000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.85399999999999998,"15min":0.80200000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.071,"5min":1.712,"15min":1.712},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.93600000000000005,"15min":0.95899999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2829999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.87}]},{"osd":4,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.04,"15min":0.97699999999999998},"min":{"1min":0.66400000000000003,"5min":0.44,"15min":0.44},"max":{"1min":1.2809999999999999,"5min":1.742,"15min":1.742},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.89400000000000002,"5min":1.0009999999999999,"15min":1.026},"min":{"1min":0.63,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1359999999999999,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":0.73999999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.0860000000000001,"15min":1.054},"min":{"1min":0.61299999999999999,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.3400000000000001,"5min":1.7150000000000001,"15min":1.7150000000000001},"last":1.1990000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.071,"15min":1.075},"min":{"1min":0.68600000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.226,"5min":1.7669999999999999,"15min":1.8640000000000001},"last":1.0189999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.159,"15min":1.0900000000000001},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.61499999999999999},"max":{"1min":1.468,"5min":1.9319999999999999,"15min":1.9319999999999999},"last":0.995},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.1499999999999999,"15min":1.107},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.4930000000000001,"5min":1.8859999999999999,"15min":1.8859999999999999},"last":1.3560000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.2250000000000001,"15min":1.155},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.5169999999999999,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.22,"15min":1.177},"min":{"1min":0.72499999999999998,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.5429999999999999,"5min":1.8029999999999999,"15min":1.8029999999999999},"last":1.2789999999999999}]}]},{"osd":3,"up_from":663,"seq":2847563317354,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":38872,"kb_used_data":6864,"kb_used_omap":55,"kb_used_meta":31944,"kb_avail":93706280,"statfs":{"total":95995035648,"available":95955230720,"internally_reserved":0,"allocated":7028736,"data_stored":6002564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":32711266},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.79800000000000004,"15min":0.77500000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.29699999999999999},"max":{"1min":0.88900000000000001,"5min":1.4350000000000001,"15min":1.5369999999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.81399999999999995,"15min":0.75800000000000001},"min":{"1min":0.42799999999999999,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":0.96299999999999997,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":1.121}]},{"osd":1,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.85699999999999998,"15min":0.83699999999999997},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.442},"max":{"1min":0.88500000000000001,"5min":3.9449999999999998,"15min":3.9449999999999998},"last":1.0409999999999999},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.83399999999999996,"15min":0.82699999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.05,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":0.89900000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.85499999999999998,"15min":0.84299999999999997},"min":{"1min":0.51000000000000001,"5min":0.42499999999999999,"15min":0.39500000000000002},"max":{"1min":1.1459999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":1.014},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.94199999999999995,"15min":0.84399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85499999999999998,"5min":10.167,"15min":10.167},"last":0.82299999999999995}]},{"osd":4,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.0780000000000001,"15min":1.01},"min":{"1min":0.746,"5min":0.62,"15min":0.62},"max":{"1min":1.181,"5min":3.8959999999999999,"15min":3.8959999999999999},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.98899999999999999,"15min":0.98899999999999999},"min":{"1min":0.63100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.214,"5min":1.629,"15min":1.7270000000000001},"last":0.93300000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.117,"15min":1.115},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.59999999999999998},"max":{"1min":1.2310000000000001,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":1.081},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.1579999999999999,"15min":1.109},"min":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.65100000000000002},"max":{"1min":1.1890000000000001,"5min":3.996,"15min":3.996},"last":1.2010000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.095,"15min":1.073},"min":{"1min":0.69599999999999995,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.2170000000000001,"5min":1.663,"15min":1.663},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.179,"15min":1.1579999999999999},"min":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.68999999999999995},"max":{"1min":1.266,"5min":3.972,"15min":3.972},"last":0.97199999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.1879999999999999,"15min":1.1910000000000001},"min":{"1min":0.72599999999999998,"5min":0.65500000000000003,"15min":0.61699999999999999},"max":{"1min":1.385,"5min":1.821,"15min":1.8260000000000001},"last":1.1850000000000001},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.1439999999999999,"15min":1.1319999999999999},"min":{"1min":0.77600000000000002,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2929999999999999,"5min":1.7669999999999999,"15min":1.845},"last":1.228}]}]},{"osd":4,"up_from":31,"seq":133143986488,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315508,"kb_used_data":5996,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93429644,"statfs":{"total":95995035648,"available":95671955456,"internally_reserved":0,"allocated":6139904,"data_stored":5850954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.90000000000000002,"15min":0.88900000000000001},"min":{"1min":0.58799999999999997,"5min":0.48999999999999999,"15min":0.315},"max":{"1min":1.2270000000000001,"5min":1.45,"15min":2.0659999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.98299999999999998,"15min":0.91600000000000004},"min":{"1min":0.65800000000000003,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.0469999999999999,"5min":1.653,"15min":1.653},"last":0.64000000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.92300000000000004,"15min":0.872},"min":{"1min":0.67900000000000005,"5min":0.375,"15min":0.375},"max":{"1min":1.2250000000000001,"5min":1.5489999999999999,"15min":1.5489999999999999},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.98799999999999999,"15min":1.0029999999999999},"min":{"1min":0.53600000000000003,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.4590000000000001,"5min":1.722,"15min":1.722},"last":0.82499999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:23:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.97599999999999998,"15min":0.99299999999999999},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.0880000000000001,"5min":5.7199999999999998,"15min":5.7199999999999998},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.96399999999999997,"15min":0.96299999999999997},"min":{"1min":0.61399999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.218,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.93500000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.997,"15min":1.03},"min":{"1min":0.59099999999999997,"5min":0.57499999999999996,"15min":0.54800000000000004},"max":{"1min":1.3160000000000001,"5min":2.488,"15min":2.488},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.123,"15min":1.133},"min":{"1min":0.751,"5min":0.54300000000000004,"15min":0.41399999999999998},"max":{"1min":1.4119999999999999,"5min":4.9009999999999998,"15min":4.9009999999999998},"last":0.749}]},{"osd":5,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.98599999999999999},"min":{"1min":0.629,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.1639999999999999,"5min":1.6930000000000001,"15min":1.7390000000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":1.099,"5min":1.109,"15min":1.107},"min":{"1min":0.69299999999999995,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.4770000000000001,"5min":2.411,"15min":2.411},"last":1.0129999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0880000000000001,"15min":1.0569999999999999},"min":{"1min":0.66100000000000003,"5min":0.504,"15min":0.48199999999999998},"max":{"1min":1.6479999999999999,"5min":1.6479999999999999,"15min":2.7719999999999998},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.095,"5min":1.099,"15min":1.123},"min":{"1min":0.68500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.5800000000000001,"5min":1.7849999999999999,"15min":1.8680000000000001},"last":0.99299999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.0609999999999999,"15min":1.054},"min":{"1min":0.73999999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.325,"5min":1.9099999999999999,"15min":1.925},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.159,"5min":1.1579999999999999,"15min":1.123},"min":{"1min":0.94799999999999995,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.3029999999999999,"5min":2.4529999999999998,"15min":2.4529999999999998},"last":1.145}]}]},{"osd":5,"up_from":37,"seq":158913790254,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307300,"kb_used_data":5980,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93437852,"statfs":{"total":95995035648,"available":95680360448,"internally_reserved":0,"allocated":6123520,"data_stored":5846698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.89100000000000001,"15min":0.85199999999999998},"min":{"1min":0.56000000000000005,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.2350000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.91900000000000004,"15min":0.91100000000000003},"min":{"1min":0.52900000000000003,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.2549999999999999,"5min":1.4359999999999999,"15min":1.6080000000000001},"last":1.198}]},{"osd":1,"last update":"Mon Oct 9 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.92500000000000004,"15min":0.93100000000000005},"min":{"1min":0.61699999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.1890000000000001,"5min":1.3240000000000001,"15min":3.3490000000000002},"last":1.034},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.0249999999999999,"15min":1.046},"min":{"1min":0.71299999999999997,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.179,"5min":1.5509999999999999,"15min":3.3730000000000002},"last":0.89900000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:23:24 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.0389999999999999,"15min":1.01},"min":{"1min":0.63900000000000001,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.1419999999999999,"5min":2.9220000000000002,"15min":2.9220000000000002},"last":0.97599999999999998},{"interface":"front","average":{"1min":1.024,"5min":1.0589999999999999,"15min":0.98199999999999998},"min":{"1min":0.54500000000000004,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.294,"5min":1.649,"15min":1.649},"last":1.0660000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.97199999999999998,"15min":0.96999999999999997},"min":{"1min":0.67400000000000004,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.999,"5min":1.464,"15min":1.464},"last":0.746},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.1180000000000001,"15min":1.081},"min":{"1min":0.79500000000000004,"5min":0.72299999999999998,"15min":0.72299999999999998},"max":{"1min":1.222,"5min":4.258,"15min":4.258},"last":1.296}]},{"osd":4,"last update":"Mon Oct 9 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.93100000000000005,"15min":0.92800000000000005},"min":{"1min":0.60199999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.0549999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.034,"15min":1.0389999999999999},"min":{"1min":0.66200000000000003,"5min":0.55400000000000005,"15min":0.41199999999999998},"max":{"1min":1.2450000000000001,"5min":1.6120000000000001,"15min":2.4990000000000001},"last":1.2470000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:23:37 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.0680000000000001,"15min":1.0389999999999999},"min":{"1min":0.67100000000000004,"5min":0.53400000000000003,"15min":0.50800000000000001},"max":{"1min":1.2250000000000001,"5min":1.637,"15min":2.0419999999999998},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.167,"5min":1.2509999999999999,"15min":1.2090000000000001},"min":{"1min":0.74399999999999999,"5min":0.69099999999999995,"15min":0.45900000000000002},"max":{"1min":1.5029999999999999,"5min":1.9079999999999999,"15min":2.4079999999999999},"last":1.339}]},{"osd":7,"last update":"Mon Oct 9 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.214,"15min":1.1619999999999999},"min":{"1min":0.88600000000000001,"5min":0.63900000000000001,"15min":0.51800000000000002},"max":{"1min":1.5820000000000001,"5min":2.052,"15min":2.556},"last":1.3839999999999999},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.1599999999999999,"15min":1.153},"min":{"1min":0.755,"5min":0.51500000000000001,"15min":0.5},"max":{"1min":1.458,"5min":2.3460000000000001,"15min":5.8559999999999999},"last":1.1020000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:24:19.179 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T09:24:19.179 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T09:24:19.179 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T09:24:19.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T09:24:19.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:19 smithi163 bash[75425]: audit 2023-10-09T09:24:18.379935+0000 mgr.y (mgr.34107) 535 : audit 0 from='client.44609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:19 smithi163 bash[78107]: audit 2023-10-09T09:24:18.379935+0000 mgr.y (mgr.34107) 535 : audit 0 from='client.44609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:19 smithi181 bash[59345]: audit 2023-10-09T09:24:18.379935+0000 mgr.y (mgr.34107) 535 : audit 0 from='client.44609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:20 smithi163 bash[75425]: cluster 2023-10-09T09:24:19.540202+0000 mgr.y (mgr.34107) 536 : cluster 0 pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:20.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:20 smithi163 bash[78107]: cluster 2023-10-09T09:24:19.540202+0000 mgr.y (mgr.34107) 536 : cluster 0 pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:20 smithi181 bash[59345]: cluster 2023-10-09T09:24:19.540202+0000 mgr.y (mgr.34107) 536 : cluster 0 pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:22.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:22 smithi163 bash[75425]: cluster 2023-10-09T09:24:21.541092+0000 mgr.y (mgr.34107) 537 : cluster 0 pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:23.501 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:22 smithi163 bash[78107]: cluster 2023-10-09T09:24:21.541092+0000 mgr.y (mgr.34107) 537 : cluster 0 pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:23.501 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:22 smithi181 bash[59345]: cluster 2023-10-09T09:24:21.541092+0000 mgr.y (mgr.34107) 537 : cluster 0 pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:24:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:24:23] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-09T09:24:23.982 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:24.891 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:24 smithi163 bash[75425]: cluster 2023-10-09T09:24:23.541890+0000 mgr.y (mgr.34107) 538 : cluster 0 pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:24.892 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:24 smithi163 bash[78107]: cluster 2023-10-09T09:24:23.541890+0000 mgr.y (mgr.34107) 538 : cluster 0 pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:25.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:24 smithi181 bash[59345]: cluster 2023-10-09T09:24:23.541890+0000 mgr.y (mgr.34107) 538 : cluster 0 pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:25 smithi163 bash[78107]: audit 2023-10-09T09:24:25.356881+0000 mon.a (mon.0) 1341 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:25.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:25 smithi163 bash[75425]: audit 2023-10-09T09:24:25.356881+0000 mon.a (mon.0) 1341 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:25 smithi181 bash[59345]: audit 2023-10-09T09:24:25.356881+0000 mon.a (mon.0) 1341 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:26 smithi163 bash[78107]: cluster 2023-10-09T09:24:25.542702+0000 mgr.y (mgr.34107) 539 : cluster 0 pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:26 smithi163 bash[75425]: cluster 2023-10-09T09:24:25.542702+0000 mgr.y (mgr.34107) 539 : cluster 0 pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:26 smithi181 bash[59345]: cluster 2023-10-09T09:24:25.542702+0000 mgr.y (mgr.34107) 539 : cluster 0 pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:27.138 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:24:27.138 INFO:teuthology.orchestra.run.smithi163.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T09:24:27.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:27 smithi163 bash[78107]: audit 2023-10-09T09:24:27.138133+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.163:0/2962801654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:24:27.884 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:27 smithi163 bash[75425]: audit 2023-10-09T09:24:27.138133+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.163:0/2962801654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:24:27.884 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T09:24:27.885 INFO:teuthology.run_tasks:Running task print... 2023-10-09T09:24:27.898 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T09:24:27.898 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T09:24:27.918 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T09:24:27.918 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T09:24:27.918 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T09:24:27.920 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T09:24:27.920 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'kernel': {'hwe': True}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T09:24:27.920 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T09:24:27.921 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T09:24:27.921 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T09:24:27.921 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:24:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:27 smithi181 bash[59345]: audit 2023-10-09T09:24:27.138133+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.163:0/2962801654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:24:28.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:28 smithi163 bash[75425]: cluster 2023-10-09T09:24:27.543749+0000 mgr.y (mgr.34107) 540 : cluster 0 pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:28.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:28 smithi163 bash[78107]: cluster 2023-10-09T09:24:27.543749+0000 mgr.y (mgr.34107) 540 : cluster 0 pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:29.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:28 smithi181 bash[59345]: cluster 2023-10-09T09:24:27.543749+0000 mgr.y (mgr.34107) 540 : cluster 0 pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:30.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:30 smithi163 bash[75425]: cluster 2023-10-09T09:24:29.544613+0000 mgr.y (mgr.34107) 541 : cluster 0 pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:30.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:30 smithi163 bash[78107]: cluster 2023-10-09T09:24:29.544613+0000 mgr.y (mgr.34107) 541 : cluster 0 pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:31.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:30 smithi181 bash[59345]: cluster 2023-10-09T09:24:29.544613+0000 mgr.y (mgr.34107) 541 : cluster 0 pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:32.696 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:32 smithi163 bash[78107]: cluster 2023-10-09T09:24:31.545464+0000 mgr.y (mgr.34107) 542 : cluster 0 pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:32 smithi163 bash[75425]: cluster 2023-10-09T09:24:31.545464+0000 mgr.y (mgr.34107) 542 : cluster 0 pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:33.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:32 smithi181 bash[59345]: cluster 2023-10-09T09:24:31.545464+0000 mgr.y (mgr.34107) 542 : cluster 0 pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:33.619 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:24:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:24:33] "GET /metrics HTTP/1.1" 200 33905 "" "Prometheus/2.33.4" 2023-10-09T09:24:34.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:34 smithi163 bash[78107]: cluster 2023-10-09T09:24:33.546174+0000 mgr.y (mgr.34107) 543 : cluster 0 pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:34.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:34 smithi163 bash[75425]: cluster 2023-10-09T09:24:33.546174+0000 mgr.y (mgr.34107) 543 : cluster 0 pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:34 smithi181 bash[59345]: cluster 2023-10-09T09:24:33.546174+0000 mgr.y (mgr.34107) 543 : cluster 0 pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:35.560 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:24:35.563 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:24:36.318 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1227,"stamp":"2023-10-09T09:24:33.545757+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1428592,"kb_used_data":53616,"kb_used_omap":416,"kb_used_meta":1374495,"kb_avail":748532624,"statfs":{"total":767960285184,"available":766497406976,"internally_reserved":0,"allocated":54902784,"data_stored":49262341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":1407483313},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005242"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1041,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433825+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:22:35.433825+0000","last_peered":"2023-10-09T09:22:35.433825+0000","last_clean":"2023-10-09T09:22:35.433825+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:22:35.433825+0000","last_undegraded":"2023-10-09T09:22:35.433825+0000","last_fullsized":"2023-10-09T09:22:35.433825+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1004,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653453+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:22:51.653453+0000","last_peered":"2023-10-09T09:22:51.653453+0000","last_clean":"2023-10-09T09:22:51.653453+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:22:51.653453+0000","last_undegraded":"2023-10-09T09:22:51.653453+0000","last_fullsized":"2023-10-09T09:22:51.653453+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1008,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.799357+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:22:51.799357+0000","last_peered":"2023-10-09T09:22:51.799357+0000","last_clean":"2023-10-09T09:22:51.799357+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:22:51.799357+0000","last_undegraded":"2023-10-09T09:22:51.799357+0000","last_fullsized":"2023-10-09T09:22:51.799357+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624753+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:22:51.624753+0000","last_peered":"2023-10-09T09:22:51.624753+0000","last_clean":"2023-10-09T09:22:51.624753+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:22:51.624753+0000","last_undegraded":"2023-10-09T09:22:51.624753+0000","last_fullsized":"2023-10-09T09:22:51.624753+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":985,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.256827+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:22:49.256827+0000","last_peered":"2023-10-09T09:22:49.256827+0000","last_clean":"2023-10-09T09:22:49.256827+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:22:49.256827+0000","last_undegraded":"2023-10-09T09:22:49.256827+0000","last_fullsized":"2023-10-09T09:22:49.256827+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438338+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:22:35.438338+0000","last_peered":"2023-10-09T09:22:35.438338+0000","last_clean":"2023-10-09T09:22:35.438338+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:22:35.438338+0000","last_undegraded":"2023-10-09T09:22:35.438338+0000","last_fullsized":"2023-10-09T09:22:35.438338+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624927+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:22:51.624927+0000","last_peered":"2023-10-09T09:22:51.624927+0000","last_clean":"2023-10-09T09:22:51.624927+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:22:51.624927+0000","last_undegraded":"2023-10-09T09:22:51.624927+0000","last_fullsized":"2023-10-09T09:22:51.624927+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":969,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431402+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:22:35.431402+0000","last_peered":"2023-10-09T09:22:35.431402+0000","last_clean":"2023-10-09T09:22:35.431402+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:22:35.431402+0000","last_undegraded":"2023-10-09T09:22:35.431402+0000","last_fullsized":"2023-10-09T09:22:35.431402+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1026,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378044+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:22:35.378044+0000","last_peered":"2023-10-09T09:22:35.378044+0000","last_clean":"2023-10-09T09:22:35.378044+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:22:35.378044+0000","last_undegraded":"2023-10-09T09:22:35.378044+0000","last_fullsized":"2023-10-09T09:22:35.378044+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":980,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431472+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:22:35.431472+0000","last_peered":"2023-10-09T09:22:35.431472+0000","last_clean":"2023-10-09T09:22:35.431472+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:22:35.431472+0000","last_undegraded":"2023-10-09T09:22:35.431472+0000","last_fullsized":"2023-10-09T09:22:35.431472+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435642+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:22:35.435642+0000","last_peered":"2023-10-09T09:22:35.435642+0000","last_clean":"2023-10-09T09:22:35.435642+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:22:35.435642+0000","last_undegraded":"2023-10-09T09:22:35.435642+0000","last_fullsized":"2023-10-09T09:22:35.435642+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1008,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.798999+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:22:51.798999+0000","last_peered":"2023-10-09T09:22:51.798999+0000","last_clean":"2023-10-09T09:22:51.798999+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:22:51.798999+0000","last_undegraded":"2023-10-09T09:22:51.798999+0000","last_fullsized":"2023-10-09T09:22:51.798999+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1032,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433801+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:22:35.433801+0000","last_peered":"2023-10-09T09:22:35.433801+0000","last_clean":"2023-10-09T09:22:35.433801+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:22:35.433801+0000","last_undegraded":"2023-10-09T09:22:35.433801+0000","last_fullsized":"2023-10-09T09:22:35.433801+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433513+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:22:35.433513+0000","last_peered":"2023-10-09T09:22:35.433513+0000","last_clean":"2023-10-09T09:22:35.433513+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:22:35.433513+0000","last_undegraded":"2023-10-09T09:22:35.433513+0000","last_fullsized":"2023-10-09T09:22:35.433513+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435712+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:22:35.435712+0000","last_peered":"2023-10-09T09:22:35.435712+0000","last_clean":"2023-10-09T09:22:35.435712+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:22:35.435712+0000","last_undegraded":"2023-10-09T09:22:35.435712+0000","last_fullsized":"2023-10-09T09:22:35.435712+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":969,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433403+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:22:35.433403+0000","last_peered":"2023-10-09T09:22:35.433403+0000","last_clean":"2023-10-09T09:22:35.433403+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:22:35.433403+0000","last_undegraded":"2023-10-09T09:22:35.433403+0000","last_fullsized":"2023-10-09T09:22:35.433403+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.622061+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:22:51.622061+0000","last_peered":"2023-10-09T09:22:51.622061+0000","last_clean":"2023-10-09T09:22:51.622061+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:22:51.622061+0000","last_undegraded":"2023-10-09T09:22:51.622061+0000","last_fullsized":"2023-10-09T09:22:51.622061+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378062+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:22:35.378062+0000","last_peered":"2023-10-09T09:22:35.378062+0000","last_clean":"2023-10-09T09:22:35.378062+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:22:35.378062+0000","last_undegraded":"2023-10-09T09:22:35.378062+0000","last_fullsized":"2023-10-09T09:22:35.378062+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":987,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.436318+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:22:35.436318+0000","last_peered":"2023-10-09T09:22:35.436318+0000","last_clean":"2023-10-09T09:22:35.436318+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:22:35.436318+0000","last_undegraded":"2023-10-09T09:22:35.436318+0000","last_fullsized":"2023-10-09T09:22:35.436318+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624005+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:22:51.624005+0000","last_peered":"2023-10-09T09:22:51.624005+0000","last_clean":"2023-10-09T09:22:51.624005+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:22:51.624005+0000","last_undegraded":"2023-10-09T09:22:51.624005+0000","last_fullsized":"2023-10-09T09:22:51.624005+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1015,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437450+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:22:35.437450+0000","last_peered":"2023-10-09T09:22:35.437450+0000","last_clean":"2023-10-09T09:22:35.437450+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:22:35.437450+0000","last_undegraded":"2023-10-09T09:22:35.437450+0000","last_fullsized":"2023-10-09T09:22:35.437450+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.257124+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:22:49.257124+0000","last_peered":"2023-10-09T09:22:49.257124+0000","last_clean":"2023-10-09T09:22:49.257124+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:22:49.257124+0000","last_undegraded":"2023-10-09T09:22:49.257124+0000","last_fullsized":"2023-10-09T09:22:49.257124+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1003,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435617+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:22:35.435617+0000","last_peered":"2023-10-09T09:22:35.435617+0000","last_clean":"2023-10-09T09:22:35.435617+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:22:35.435617+0000","last_undegraded":"2023-10-09T09:22:35.435617+0000","last_fullsized":"2023-10-09T09:22:35.435617+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377442+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:22:35.377442+0000","last_peered":"2023-10-09T09:22:35.377442+0000","last_clean":"2023-10-09T09:22:35.377442+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:22:35.377442+0000","last_undegraded":"2023-10-09T09:22:35.377442+0000","last_fullsized":"2023-10-09T09:22:35.377442+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435716+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:22:35.435716+0000","last_peered":"2023-10-09T09:22:35.435716+0000","last_clean":"2023-10-09T09:22:35.435716+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:22:35.435716+0000","last_undegraded":"2023-10-09T09:22:35.435716+0000","last_fullsized":"2023-10-09T09:22:35.435716+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433540+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:22:35.433540+0000","last_peered":"2023-10-09T09:22:35.433540+0000","last_clean":"2023-10-09T09:22:35.433540+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:22:35.433540+0000","last_undegraded":"2023-10-09T09:22:35.433540+0000","last_fullsized":"2023-10-09T09:22:35.433540+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":988,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.436871+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:22:35.436871+0000","last_peered":"2023-10-09T09:22:35.436871+0000","last_clean":"2023-10-09T09:22:35.436871+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:22:35.436871+0000","last_undegraded":"2023-10-09T09:22:35.436871+0000","last_fullsized":"2023-10-09T09:22:35.436871+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1006,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437643+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:22:35.437643+0000","last_peered":"2023-10-09T09:22:35.437643+0000","last_clean":"2023-10-09T09:22:35.437643+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:22:35.437643+0000","last_undegraded":"2023-10-09T09:22:35.437643+0000","last_fullsized":"2023-10-09T09:22:35.437643+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1026,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377882+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:22:35.377882+0000","last_peered":"2023-10-09T09:22:35.377882+0000","last_clean":"2023-10-09T09:22:35.377882+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:22:35.377882+0000","last_undegraded":"2023-10-09T09:22:35.377882+0000","last_fullsized":"2023-10-09T09:22:35.377882+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.622439+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:22:51.622439+0000","last_peered":"2023-10-09T09:22:51.622439+0000","last_clean":"2023-10-09T09:22:51.622439+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:22:51.622439+0000","last_undegraded":"2023-10-09T09:22:51.622439+0000","last_fullsized":"2023-10-09T09:22:51.622439+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":987,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.432256+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:22:35.432256+0000","last_peered":"2023-10-09T09:22:35.432256+0000","last_clean":"2023-10-09T09:22:35.432256+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:22:35.432256+0000","last_undegraded":"2023-10-09T09:22:35.432256+0000","last_fullsized":"2023-10-09T09:22:35.432256+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434896+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:22:35.434896+0000","last_peered":"2023-10-09T09:22:35.434896+0000","last_clean":"2023-10-09T09:22:35.434896+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:22:35.434896+0000","last_undegraded":"2023-10-09T09:22:35.434896+0000","last_fullsized":"2023-10-09T09:22:35.434896+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1007,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437878+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:22:35.437878+0000","last_peered":"2023-10-09T09:22:35.437878+0000","last_clean":"2023-10-09T09:22:35.437878+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:22:35.437878+0000","last_undegraded":"2023-10-09T09:22:35.437878+0000","last_fullsized":"2023-10-09T09:22:35.437878+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434259+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:22:35.434259+0000","last_peered":"2023-10-09T09:22:35.434259+0000","last_clean":"2023-10-09T09:22:35.434259+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:22:35.434259+0000","last_undegraded":"2023-10-09T09:22:35.434259+0000","last_fullsized":"2023-10-09T09:22:35.434259+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433074+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:22:35.433074+0000","last_peered":"2023-10-09T09:22:35.433074+0000","last_clean":"2023-10-09T09:22:35.433074+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:22:35.433074+0000","last_undegraded":"2023-10-09T09:22:35.433074+0000","last_fullsized":"2023-10-09T09:22:35.433074+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434199+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:22:35.434199+0000","last_peered":"2023-10-09T09:22:35.434199+0000","last_clean":"2023-10-09T09:22:35.434199+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:22:35.434199+0000","last_undegraded":"2023-10-09T09:22:35.434199+0000","last_fullsized":"2023-10-09T09:22:35.434199+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":971,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652272+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:22:51.652272+0000","last_peered":"2023-10-09T09:22:51.652272+0000","last_clean":"2023-10-09T09:22:51.652272+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:22:51.652272+0000","last_undegraded":"2023-10-09T09:22:51.652272+0000","last_fullsized":"2023-10-09T09:22:51.652272+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1008,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801270+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:22:51.801270+0000","last_peered":"2023-10-09T09:22:51.801270+0000","last_clean":"2023-10-09T09:22:51.801270+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:22:51.801270+0000","last_undegraded":"2023-10-09T09:22:51.801270+0000","last_fullsized":"2023-10-09T09:22:51.801270+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1003,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801363+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:22:51.801363+0000","last_peered":"2023-10-09T09:22:51.801363+0000","last_clean":"2023-10-09T09:22:51.801363+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:22:51.801363+0000","last_undegraded":"2023-10-09T09:22:51.801363+0000","last_fullsized":"2023-10-09T09:22:51.801363+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1041,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378075+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:22:35.378075+0000","last_peered":"2023-10-09T09:22:35.378075+0000","last_clean":"2023-10-09T09:22:35.378075+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:22:35.378075+0000","last_undegraded":"2023-10-09T09:22:35.378075+0000","last_fullsized":"2023-10-09T09:22:35.378075+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":993,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.799273+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:22:51.799273+0000","last_peered":"2023-10-09T09:22:51.799273+0000","last_clean":"2023-10-09T09:22:51.799273+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:22:51.799273+0000","last_undegraded":"2023-10-09T09:22:51.799273+0000","last_fullsized":"2023-10-09T09:22:51.799273+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1030,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377624+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:22:35.377624+0000","last_peered":"2023-10-09T09:22:35.377624+0000","last_clean":"2023-10-09T09:22:35.377624+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:22:35.377624+0000","last_undegraded":"2023-10-09T09:22:35.377624+0000","last_fullsized":"2023-10-09T09:22:35.377624+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433321+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:22:35.433321+0000","last_peered":"2023-10-09T09:22:35.433321+0000","last_clean":"2023-10-09T09:22:35.433321+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:22:35.433321+0000","last_undegraded":"2023-10-09T09:22:35.433321+0000","last_fullsized":"2023-10-09T09:22:35.433321+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1023,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.800115+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:22:51.800115+0000","last_peered":"2023-10-09T09:22:51.800115+0000","last_clean":"2023-10-09T09:22:51.800115+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:22:51.800115+0000","last_undegraded":"2023-10-09T09:22:51.800115+0000","last_fullsized":"2023-10-09T09:22:51.800115+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":990,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:50.876223+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:22:50.876223+0000","last_peered":"2023-10-09T09:22:50.876223+0000","last_clean":"2023-10-09T09:22:50.876223+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:22:50.876223+0000","last_undegraded":"2023-10-09T09:22:50.876223+0000","last_fullsized":"2023-10-09T09:22:50.876223+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1008,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:38.583196+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:22:38.583196+0000","last_peered":"2023-10-09T09:22:38.583196+0000","last_clean":"2023-10-09T09:22:38.583196+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:22:38.583196+0000","last_undegraded":"2023-10-09T09:22:38.583196+0000","last_fullsized":"2023-10-09T09:22:38.583196+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801584+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:22:51.801584+0000","last_peered":"2023-10-09T09:22:51.801584+0000","last_clean":"2023-10-09T09:22:51.801584+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:22:51.801584+0000","last_undegraded":"2023-10-09T09:22:51.801584+0000","last_fullsized":"2023-10-09T09:22:51.801584+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1030,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377739+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:22:35.377739+0000","last_peered":"2023-10-09T09:22:35.377739+0000","last_clean":"2023-10-09T09:22:35.377739+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:22:35.377739+0000","last_undegraded":"2023-10-09T09:22:35.377739+0000","last_fullsized":"2023-10-09T09:22:35.377739+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1238,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435393+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:22:35.435393+0000","last_peered":"2023-10-09T09:22:35.435393+0000","last_clean":"2023-10-09T09:22:35.435393+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:22:35.435393+0000","last_undegraded":"2023-10-09T09:22:35.435393+0000","last_fullsized":"2023-10-09T09:22:35.435393+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":982,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652484+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:22:51.652484+0000","last_peered":"2023-10-09T09:22:51.652484+0000","last_clean":"2023-10-09T09:22:51.652484+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:22:51.652484+0000","last_undegraded":"2023-10-09T09:22:51.652484+0000","last_fullsized":"2023-10-09T09:22:51.652484+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":991,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438069+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:22:35.438069+0000","last_peered":"2023-10-09T09:22:35.438069+0000","last_clean":"2023-10-09T09:22:35.438069+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:22:35.438069+0000","last_undegraded":"2023-10-09T09:22:35.438069+0000","last_fullsized":"2023-10-09T09:22:35.438069+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1021,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435529+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:22:35.435529+0000","last_peered":"2023-10-09T09:22:35.435529+0000","last_clean":"2023-10-09T09:22:35.435529+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:22:35.435529+0000","last_undegraded":"2023-10-09T09:22:35.435529+0000","last_fullsized":"2023-10-09T09:22:35.435529+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":999,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.429996+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:22:35.429996+0000","last_peered":"2023-10-09T09:22:35.429996+0000","last_clean":"2023-10-09T09:22:35.429996+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:22:35.429996+0000","last_undegraded":"2023-10-09T09:22:35.429996+0000","last_fullsized":"2023-10-09T09:22:35.429996+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1010,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.621540+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:22:51.621540+0000","last_peered":"2023-10-09T09:22:51.621540+0000","last_clean":"2023-10-09T09:22:51.621540+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:22:51.621540+0000","last_undegraded":"2023-10-09T09:22:51.621540+0000","last_fullsized":"2023-10-09T09:22:51.621540+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1006,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435480+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:22:35.435480+0000","last_peered":"2023-10-09T09:22:35.435480+0000","last_clean":"2023-10-09T09:22:35.435480+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:22:35.435480+0000","last_undegraded":"2023-10-09T09:22:35.435480+0000","last_fullsized":"2023-10-09T09:22:35.435480+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1009,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.799014+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:22:51.799014+0000","last_peered":"2023-10-09T09:22:51.799014+0000","last_clean":"2023-10-09T09:22:51.799014+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:22:51.799014+0000","last_undegraded":"2023-10-09T09:22:51.799014+0000","last_fullsized":"2023-10-09T09:22:51.799014+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":983,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431109+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:22:35.431109+0000","last_peered":"2023-10-09T09:22:35.431109+0000","last_clean":"2023-10-09T09:22:35.431109+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:22:35.431109+0000","last_undegraded":"2023-10-09T09:22:35.431109+0000","last_fullsized":"2023-10-09T09:22:35.431109+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":988,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653895+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:22:51.653895+0000","last_peered":"2023-10-09T09:22:51.653895+0000","last_clean":"2023-10-09T09:22:51.653895+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:22:51.653895+0000","last_undegraded":"2023-10-09T09:22:51.653895+0000","last_fullsized":"2023-10-09T09:22:51.653895+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":992,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.800514+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:22:51.800514+0000","last_peered":"2023-10-09T09:22:51.800514+0000","last_clean":"2023-10-09T09:22:51.800514+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:22:51.800514+0000","last_undegraded":"2023-10-09T09:22:51.800514+0000","last_fullsized":"2023-10-09T09:22:51.800514+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":985,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652104+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:22:51.652104+0000","last_peered":"2023-10-09T09:22:51.652104+0000","last_clean":"2023-10-09T09:22:51.652104+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:22:51.652104+0000","last_undegraded":"2023-10-09T09:22:51.652104+0000","last_fullsized":"2023-10-09T09:22:51.652104+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438335+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:22:35.438335+0000","last_peered":"2023-10-09T09:22:35.438335+0000","last_clean":"2023-10-09T09:22:35.438335+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:22:35.438335+0000","last_undegraded":"2023-10-09T09:22:35.438335+0000","last_fullsized":"2023-10-09T09:22:35.438335+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1002,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.625144+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:22:51.625144+0000","last_peered":"2023-10-09T09:22:51.625144+0000","last_clean":"2023-10-09T09:22:51.625144+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:22:51.625144+0000","last_undegraded":"2023-10-09T09:22:51.625144+0000","last_fullsized":"2023-10-09T09:22:51.625144+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1009,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433219+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:22:35.433219+0000","last_peered":"2023-10-09T09:22:35.433219+0000","last_clean":"2023-10-09T09:22:35.433219+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:22:35.433219+0000","last_undegraded":"2023-10-09T09:22:35.433219+0000","last_fullsized":"2023-10-09T09:22:35.433219+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":987,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624417+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:22:51.624417+0000","last_peered":"2023-10-09T09:22:51.624417+0000","last_clean":"2023-10-09T09:22:51.624417+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:22:51.624417+0000","last_undegraded":"2023-10-09T09:22:51.624417+0000","last_fullsized":"2023-10-09T09:22:51.624417+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1008,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433286+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:22:35.433286+0000","last_peered":"2023-10-09T09:22:35.433286+0000","last_clean":"2023-10-09T09:22:35.433286+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:22:35.433286+0000","last_undegraded":"2023-10-09T09:22:35.433286+0000","last_fullsized":"2023-10-09T09:22:35.433286+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1028,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378083+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:22:35.378083+0000","last_peered":"2023-10-09T09:22:35.378083+0000","last_clean":"2023-10-09T09:22:35.378083+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:22:35.378083+0000","last_undegraded":"2023-10-09T09:22:35.378083+0000","last_fullsized":"2023-10-09T09:22:35.378083+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433836+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:22:35.433836+0000","last_peered":"2023-10-09T09:22:35.433836+0000","last_clean":"2023-10-09T09:22:35.433836+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:22:35.433836+0000","last_undegraded":"2023-10-09T09:22:35.433836+0000","last_fullsized":"2023-10-09T09:22:35.433836+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1013,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378205+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:22:35.378205+0000","last_peered":"2023-10-09T09:22:35.378205+0000","last_clean":"2023-10-09T09:22:35.378205+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:22:35.378205+0000","last_undegraded":"2023-10-09T09:22:35.378205+0000","last_fullsized":"2023-10-09T09:22:35.378205+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433894+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:22:35.433894+0000","last_peered":"2023-10-09T09:22:35.433894+0000","last_clean":"2023-10-09T09:22:35.433894+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:22:35.433894+0000","last_undegraded":"2023-10-09T09:22:35.433894+0000","last_fullsized":"2023-10-09T09:22:35.433894+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653248+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:22:51.653248+0000","last_peered":"2023-10-09T09:22:51.653248+0000","last_clean":"2023-10-09T09:22:51.653248+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:22:51.653248+0000","last_undegraded":"2023-10-09T09:22:51.653248+0000","last_fullsized":"2023-10-09T09:22:51.653248+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.256889+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:22:49.256889+0000","last_peered":"2023-10-09T09:22:49.256889+0000","last_clean":"2023-10-09T09:22:49.256889+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:22:49.256889+0000","last_undegraded":"2023-10-09T09:22:49.256889+0000","last_fullsized":"2023-10-09T09:22:49.256889+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433005+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:22:35.433005+0000","last_peered":"2023-10-09T09:22:35.433005+0000","last_clean":"2023-10-09T09:22:35.433005+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:22:35.433005+0000","last_undegraded":"2023-10-09T09:22:35.433005+0000","last_fullsized":"2023-10-09T09:22:35.433005+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.432824+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:22:35.432824+0000","last_peered":"2023-10-09T09:22:35.432824+0000","last_clean":"2023-10-09T09:22:35.432824+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:22:35.432824+0000","last_undegraded":"2023-10-09T09:22:35.432824+0000","last_fullsized":"2023-10-09T09:22:35.432824+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397793,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316092,"kb_used_data":6580,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93429060,"statfs":{"total":95995035648,"available":95671357440,"internally_reserved":0,"allocated":6737920,"data_stored":6441392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.079,"15min":1.0449999999999999},"min":{"1min":0.66300000000000003,"5min":0.59699999999999998,"15min":0.41899999999999998},"max":{"1min":1.325,"5min":1.837,"15min":1.837},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.115,"5min":1.1859999999999999,"15min":1.133},"min":{"1min":0.873,"5min":0.747,"15min":0.48599999999999999},"max":{"1min":1.361,"5min":1.905,"15min":1.905},"last":1.0760000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.93200000000000005,"15min":0.90200000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.145,"5min":1.4019999999999999,"15min":1.5980000000000001},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.016,"15min":0.98399999999999999},"min":{"1min":0.66600000000000004,"5min":0.66500000000000004,"15min":0.60199999999999998},"max":{"1min":1.3049999999999999,"5min":1.722,"15min":2.028},"last":1}]},{"osd":2,"last update":"Mon Oct 9 09:24:19 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.1739999999999999,"15min":1.111},"min":{"1min":0.77000000000000002,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.5009999999999999,"5min":2.4710000000000001,"15min":3.0059999999999998},"last":0.89600000000000002},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.032,"15min":0.999},"min":{"1min":0.61699999999999999,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2110000000000001,"5min":2.6539999999999999,"15min":3.0249999999999999},"last":1.054}]},{"osd":3,"last update":"Mon Oct 9 09:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0129999999999999,"15min":0.998},"min":{"1min":0.70899999999999996,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.2729999999999999,"5min":1.825,"15min":1.825},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1799999999999999,"15min":1.1399999999999999},"min":{"1min":0.82899999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.51,"5min":1.673,"15min":1.673},"last":1.21}]},{"osd":4,"last update":"Mon Oct 9 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.95699999999999996,"15min":0.94999999999999996},"min":{"1min":0.69999999999999996,"5min":0.374,"15min":0.374},"max":{"1min":1.157,"5min":1.4650000000000001,"15min":1.6679999999999999},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.98399999999999999,"15min":1.0149999999999999},"min":{"1min":0.70399999999999996,"5min":0.65000000000000002,"15min":0.50900000000000001},"max":{"1min":1.3580000000000001,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.97299999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.1819999999999999,"15min":1.119},"min":{"1min":0.748,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.4359999999999999,"5min":1.9019999999999999,"15min":2.2330000000000001},"last":1.141},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.0740000000000001,"15min":1.069},"min":{"1min":0.72699999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.274,"5min":1.6859999999999999,"15min":2.302},"last":1.179}]},{"osd":6,"last update":"Mon Oct 9 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.087,"15min":1.0900000000000001},"min":{"1min":0.64100000000000001,"5min":0.60999999999999999,"15min":0.54300000000000004},"max":{"1min":1.296,"5min":1.569,"15min":2.0049999999999999},"last":1.2450000000000001},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.173,"15min":1.1699999999999999},"min":{"1min":0.80400000000000005,"5min":0.69299999999999995,"15min":0.60999999999999999},"max":{"1min":1.3069999999999999,"5min":2.0070000000000001,"15min":2.0070000000000001},"last":1.1559999999999999}]}]},{"osd":6,"up_from":43,"seq":184683594026,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311984,"kb_used_data":6568,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93433168,"statfs":{"total":95995035648,"available":95675564032,"internally_reserved":0,"allocated":6725632,"data_stored":6441246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.91700000000000004,"15min":0.91900000000000004},"min":{"1min":0.52500000000000002,"5min":0.46500000000000002,"15min":0.35499999999999998},"max":{"1min":1.4910000000000001,"5min":1.72,"15min":2.9060000000000001},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.036,"15min":0.96399999999999997},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.38300000000000001},"max":{"1min":1.3069999999999999,"5min":2.2669999999999999,"15min":2.2669999999999999},"last":0.93600000000000005}]},{"osd":1,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.93200000000000005,"15min":0.92500000000000004},"min":{"1min":0.57799999999999996,"5min":0.54300000000000004,"15min":0.40699999999999997},"max":{"1min":1.2470000000000001,"5min":2.508,"15min":3.5590000000000002},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.92200000000000004,"15min":0.91200000000000003},"min":{"1min":0.49299999999999999,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.2589999999999999,"5min":1.581,"15min":3.3860000000000001},"last":0.90200000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.96999999999999997,"15min":0.92600000000000005},"min":{"1min":0.60899999999999999,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":3.3210000000000002,"5min":3.3210000000000002,"15min":3.3210000000000002},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.02,"5min":1.083,"15min":0.98599999999999999},"min":{"1min":0.75600000000000001,"5min":0.51800000000000002,"15min":0.51200000000000001},"max":{"1min":1.333,"5min":2.2120000000000002,"15min":2.2120000000000002},"last":0.81200000000000006}]},{"osd":3,"last update":"Mon Oct 9 09:23:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.004,"15min":1.0069999999999999},"min":{"1min":0.627,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":3.8010000000000002,"5min":3.8010000000000002,"15min":4.7430000000000003},"last":1.034},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.052,"15min":1.0700000000000001},"min":{"1min":0.61899999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.3520000000000001,"5min":2.0649999999999999,"15min":3.4249999999999998},"last":1.103}]},{"osd":4,"last update":"Mon Oct 9 09:23:40 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.97999999999999998,"15min":0.96699999999999997},"min":{"1min":0.59699999999999998,"5min":0.57499999999999996,"15min":0.39900000000000002},"max":{"1min":1.204,"5min":2.3809999999999998,"15min":3.5640000000000001},"last":1.016},{"interface":"front","average":{"1min":1.004,"5min":1.0469999999999999,"15min":1.0449999999999999},"min":{"1min":0.65700000000000003,"5min":0.57499999999999996,"15min":0.41799999999999998},"max":{"1min":1.3720000000000001,"5min":2.5800000000000001,"15min":4.0629999999999997},"last":1.206}]},{"osd":5,"last update":"Mon Oct 9 09:23:40 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.113,"15min":1.081},"min":{"1min":0.72399999999999998,"5min":0.56000000000000005,"15min":0.46000000000000002},"max":{"1min":1.3220000000000001,"5min":2.7229999999999999,"15min":2.7229999999999999},"last":1.081},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.149,"15min":1.095},"min":{"1min":0.71499999999999997,"5min":0.63300000000000001,"15min":0.48699999999999999},"max":{"1min":1.3320000000000001,"5min":2.7570000000000001,"15min":3.246},"last":1.2829999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.169,"15min":1.1439999999999999},"min":{"1min":0.86799999999999999,"5min":0.68600000000000005,"15min":0.49299999999999999},"max":{"1min":1.468,"5min":2.8570000000000002,"15min":3.3340000000000001},"last":1.3720000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1759999999999999,"15min":1.1659999999999999},"min":{"1min":0.746,"5min":0.746,"15min":0.46999999999999997},"max":{"1min":1.3720000000000001,"5min":1.8580000000000001,"15min":2.6659999999999999},"last":1.264}]}]},{"osd":2,"up_from":634,"seq":2723009265779,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":44020,"kb_used_data":7020,"kb_used_omap":122,"kb_used_meta":36869,"kb_avail":93701132,"statfs":{"total":95995035648,"available":95949959168,"internally_reserved":0,"allocated":7188480,"data_stored":6030632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":37754443},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.77100000000000002,"15min":0.74099999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.39800000000000002},"max":{"1min":1.421,"5min":1.421,"15min":1.454},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.752,"15min":0.69199999999999995},"min":{"1min":0.52500000000000002,"5min":0.44700000000000001,"15min":0.33600000000000002},"max":{"1min":0.98999999999999999,"5min":1.1000000000000001,"15min":1.2809999999999999},"last":0.84699999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":0.755,"15min":0.71199999999999997},"min":{"1min":0.55800000000000005,"5min":0.52100000000000002,"15min":0.377},"max":{"1min":1.0669999999999999,"5min":1.2549999999999999,"15min":1.2809999999999999},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.84399999999999997,"15min":0.76000000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.0860000000000001,"5min":1.3100000000000001,"15min":1.956},"last":0.90300000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.82999999999999996,"15min":0.79800000000000004},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.40600000000000003},"max":{"1min":1.0960000000000001,"5min":1.1850000000000001,"15min":1.3169999999999999},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.91300000000000003,"15min":0.86699999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.4530000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":1.1559999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.0940000000000001,"15min":1.0369999999999999},"min":{"1min":0.73499999999999999,"5min":0.70999999999999996,"15min":0.59399999999999997},"max":{"1min":1.603,"5min":2.2559999999999998,"15min":2.2559999999999998},"last":1.2549999999999999},{"interface":"front","average":{"1min":1.05,"5min":1.05,"15min":1.0329999999999999},"min":{"1min":0.66700000000000004,"5min":0.60999999999999999,"15min":0.59699999999999998},"max":{"1min":1.3169999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.3320000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.091,"15min":1.04},"min":{"1min":0.81000000000000005,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.381,"5min":2.226,"15min":2.226},"last":1.075},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1739999999999999,"15min":1.127},"min":{"1min":0.70799999999999996,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.669,"5min":1.696,"15min":1.696},"last":1.3089999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":1.21,"15min":1.1519999999999999},"min":{"1min":0.877,"5min":0.82199999999999995,"15min":0.67700000000000005},"max":{"1min":1.5309999999999999,"5min":1.6160000000000001,"15min":1.716},"last":1.355},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.175,"15min":1.109},"min":{"1min":0.83999999999999997,"5min":0.81000000000000005,"15min":0.68500000000000005},"max":{"1min":1.4770000000000001,"5min":1.7989999999999999,"15min":1.7989999999999999},"last":1.2889999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:23:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.22,"15min":1.1559999999999999},"min":{"1min":0.96399999999999997,"5min":0.85199999999999998,"15min":0.76700000000000002},"max":{"1min":1.5580000000000001,"5min":1.748,"15min":1.8080000000000001},"last":1.2190000000000001},{"interface":"front","average":{"1min":1.2,"5min":1.2010000000000001,"15min":1.1499999999999999},"min":{"1min":1.004,"5min":0.748,"15min":0.68600000000000005},"max":{"1min":1.52,"5min":1.752,"15min":1.752},"last":1.1899999999999999}]}]},{"osd":1,"up_from":607,"seq":2607045148792,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":39904,"kb_used_data":7000,"kb_used_omap":107,"kb_used_meta":32788,"kb_avail":93705248,"statfs":{"total":95995035648,"available":95954173952,"internally_reserved":0,"allocated":7168000,"data_stored":6027385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":33575595},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.67900000000000005,"15min":0.65200000000000002},"min":{"1min":0.377,"5min":0.37,"15min":0.35599999999999998},"max":{"1min":0.94799999999999995,"5min":1.2050000000000001,"15min":1.4410000000000001},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.73999999999999999,"15min":0.68799999999999994},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.32600000000000001},"max":{"1min":0.86899999999999999,"5min":1.1679999999999999,"15min":1.2729999999999999},"last":1.2270000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:23:24 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.76800000000000002,"15min":0.72499999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.222,"5min":3.1749999999999998,"15min":3.1749999999999998},"last":1.129},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.81699999999999995,"15min":0.81899999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.314},"max":{"1min":1.056,"5min":1.333,"15min":3.484},"last":1.1659999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:23:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.83099999999999996,"15min":0.82299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.089,"5min":1.304,"15min":1.393},"last":1.056},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.86699999999999999,"15min":0.88600000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.1080000000000001,"5min":1.417,"15min":1.6830000000000001},"last":0.96799999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.032,"15min":1.0129999999999999},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59399999999999997},"max":{"1min":1.3149999999999999,"5min":1.6850000000000001,"15min":1.9079999999999999},"last":1.506},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.0109999999999999,"15min":1.0309999999999999},"min":{"1min":0.67000000000000004,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.363,"5min":1.5389999999999999,"15min":1.601},"last":1.794}]},{"osd":5,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.073,"15min":1.075},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.56499999999999995},"max":{"1min":1.252,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.762},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.077,"15min":1.054},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2669999999999999,"5min":1.605,"15min":1.833},"last":1.8680000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0620000000000001,"15min":1.0580000000000001},"min":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.57599999999999996},"max":{"1min":1.6319999999999999,"5min":1.806,"15min":1.921},"last":1.53},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.1379999999999999,"15min":1.091},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.5,"5min":1.837,"15min":1.837},"last":1.645}]},{"osd":7,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.137,"15min":1.1539999999999999},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.349,"5min":1.7629999999999999,"15min":1.9990000000000001},"last":1.72},{"interface":"front","average":{"1min":1.044,"5min":1.212,"15min":1.204},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.64200000000000002},"max":{"1min":1.431,"5min":1.9039999999999999,"15min":1.9039999999999999},"last":1.8240000000000001}]}]},{"osd":0,"up_from":581,"seq":2495375999102,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":54912,"kb_used_data":7608,"kb_used_omap":131,"kb_used_meta":47164,"kb_avail":93690240,"statfs":{"total":95995035648,"available":95938805760,"internally_reserved":0,"allocated":7790592,"data_stored":6621470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":48296537},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:23:57 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.73499999999999999,"15min":0.70699999999999996},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.94099999999999995,"5min":1.091,"15min":1.218},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.80400000000000005,"15min":0.76700000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.97299999999999998,"5min":1.393,"15min":1.393},"last":0.81200000000000006}]},{"osd":2,"last update":"Mon Oct 9 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.79800000000000004,"15min":0.77000000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.04,"5min":1.321,"15min":1.321},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.78700000000000003,"15min":0.76100000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32800000000000001},"max":{"1min":1.036,"5min":1.669,"15min":1.669},"last":0.91900000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.85399999999999998,"15min":0.80200000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.071,"5min":1.712,"15min":1.712},"last":1.0009999999999999},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.93600000000000005,"15min":0.95899999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2829999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":1.1140000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.04,"15min":0.97699999999999998},"min":{"1min":0.66400000000000003,"5min":0.44,"15min":0.44},"max":{"1min":1.2809999999999999,"5min":1.742,"15min":1.742},"last":1.0349999999999999},{"interface":"front","average":{"1min":0.89400000000000002,"5min":1.0009999999999999,"15min":1.026},"min":{"1min":0.63,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1359999999999999,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":0.96999999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.0860000000000001,"15min":1.054},"min":{"1min":0.61299999999999999,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.3400000000000001,"5min":1.7150000000000001,"15min":1.7150000000000001},"last":1.151},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.071,"15min":1.075},"min":{"1min":0.68600000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.226,"5min":1.7669999999999999,"15min":1.8640000000000001},"last":1.0669999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.159,"15min":1.0900000000000001},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.61499999999999999},"max":{"1min":1.468,"5min":1.9319999999999999,"15min":1.9319999999999999},"last":1.25},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.1499999999999999,"15min":1.107},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.4930000000000001,"5min":1.8859999999999999,"15min":1.8859999999999999},"last":1.3009999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.2250000000000001,"15min":1.155},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.5169999999999999,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":1.181},{"interface":"front","average":{"1min":1.099,"5min":1.22,"15min":1.177},"min":{"1min":0.72499999999999998,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.5429999999999999,"5min":1.8029999999999999,"15min":1.8029999999999999},"last":1.22}]}]},{"osd":3,"up_from":663,"seq":2847563317357,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":38872,"kb_used_data":6864,"kb_used_omap":55,"kb_used_meta":31944,"kb_avail":93706280,"statfs":{"total":95995035648,"available":95955230720,"internally_reserved":0,"allocated":7028736,"data_stored":6002564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":32711266},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.79800000000000004,"15min":0.77500000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.29699999999999999},"max":{"1min":0.88900000000000001,"5min":1.4350000000000001,"15min":1.5369999999999999},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.81399999999999995,"15min":0.75800000000000001},"min":{"1min":0.42799999999999999,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":0.96299999999999997,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":0.54600000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.85699999999999998,"15min":0.83699999999999997},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.442},"max":{"1min":0.88500000000000001,"5min":3.9449999999999998,"15min":3.9449999999999998},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.83399999999999996,"15min":0.82699999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.05,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":0.76900000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.85499999999999998,"15min":0.84299999999999997},"min":{"1min":0.51000000000000001,"5min":0.42499999999999999,"15min":0.39500000000000002},"max":{"1min":1.1459999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":0.95499999999999996},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.94199999999999995,"15min":0.84399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85499999999999998,"5min":10.167,"15min":10.167},"last":0.79900000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.0780000000000001,"15min":1.01},"min":{"1min":0.746,"5min":0.62,"15min":0.62},"max":{"1min":1.181,"5min":3.8959999999999999,"15min":3.8959999999999999},"last":1.004},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.98899999999999999,"15min":0.98899999999999999},"min":{"1min":0.63100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.214,"5min":1.629,"15min":1.7270000000000001},"last":1.0269999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.117,"15min":1.115},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.59999999999999998},"max":{"1min":1.2310000000000001,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":1.1519999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.1579999999999999,"15min":1.109},"min":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.65100000000000002},"max":{"1min":1.1890000000000001,"5min":3.996,"15min":3.996},"last":1.177}]},{"osd":6,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.095,"15min":1.073},"min":{"1min":0.69599999999999995,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.2170000000000001,"5min":1.663,"15min":1.663},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.01,"5min":1.179,"15min":1.1579999999999999},"min":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.68999999999999995},"max":{"1min":1.266,"5min":3.972,"15min":3.972},"last":1.107}]},{"osd":7,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.1879999999999999,"15min":1.1910000000000001},"min":{"1min":0.72599999999999998,"5min":0.65500000000000003,"15min":0.61699999999999999},"max":{"1min":1.385,"5min":1.821,"15min":1.8260000000000001},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.1439999999999999,"15min":1.1319999999999999},"min":{"1min":0.77600000000000002,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2929999999999999,"5min":1.7669999999999999,"15min":1.845},"last":1.1319999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986491,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315508,"kb_used_data":5996,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93429644,"statfs":{"total":95995035648,"available":95671955456,"internally_reserved":0,"allocated":6139904,"data_stored":5850954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.90000000000000002,"15min":0.88900000000000001},"min":{"1min":0.58799999999999997,"5min":0.48999999999999999,"15min":0.315},"max":{"1min":1.2270000000000001,"5min":1.45,"15min":2.0659999999999998},"last":1.091},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.98299999999999998,"15min":0.91600000000000004},"min":{"1min":0.65800000000000003,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.0469999999999999,"5min":1.653,"15min":1.653},"last":1.0329999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.92300000000000004,"15min":0.872},"min":{"1min":0.67900000000000005,"5min":0.375,"15min":0.375},"max":{"1min":1.2250000000000001,"5min":1.5489999999999999,"15min":1.5489999999999999},"last":1.157},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.98799999999999999,"15min":1.0029999999999999},"min":{"1min":0.53600000000000003,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.4590000000000001,"5min":1.722,"15min":1.722},"last":0.90200000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:24:17 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.95299999999999996,"15min":0.98099999999999998},"min":{"1min":0.57099999999999995,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.1259999999999999,"5min":5.7199999999999998,"15min":5.7199999999999998},"last":0.93500000000000005},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.94299999999999995,"15min":0.95799999999999996},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.42999999999999999},"max":{"1min":1.359,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.236}]},{"osd":3,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.997,"15min":1.03},"min":{"1min":0.59099999999999997,"5min":0.57499999999999996,"15min":0.54800000000000004},"max":{"1min":1.3160000000000001,"5min":2.488,"15min":2.488},"last":0.80400000000000005},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.123,"15min":1.133},"min":{"1min":0.751,"5min":0.54300000000000004,"15min":0.41399999999999998},"max":{"1min":1.4119999999999999,"5min":4.9009999999999998,"15min":4.9009999999999998},"last":1.175}]},{"osd":5,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.98599999999999999},"min":{"1min":0.629,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.1639999999999999,"5min":1.6930000000000001,"15min":1.7390000000000001},"last":0.82699999999999996},{"interface":"front","average":{"1min":1.099,"5min":1.109,"15min":1.107},"min":{"1min":0.69299999999999995,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.4770000000000001,"5min":2.411,"15min":2.411},"last":1.0069999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0880000000000001,"15min":1.0569999999999999},"min":{"1min":0.66100000000000003,"5min":0.504,"15min":0.48199999999999998},"max":{"1min":1.6479999999999999,"5min":1.6479999999999999,"15min":2.7719999999999998},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.095,"5min":1.099,"15min":1.123},"min":{"1min":0.68500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.5800000000000001,"5min":1.7849999999999999,"15min":1.8680000000000001},"last":1.129}]},{"osd":7,"last update":"Mon Oct 9 09:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.0740000000000001,"15min":1.0620000000000001},"min":{"1min":0.76500000000000001,"5min":0.56499999999999995,"15min":0.51200000000000001},"max":{"1min":1.476,"5min":1.9099999999999999,"15min":1.925},"last":1.0609999999999999},{"interface":"front","average":{"1min":1.133,"5min":1.155,"15min":1.135},"min":{"1min":0.79800000000000004,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.385,"5min":2.4529999999999998,"15min":2.4529999999999998},"last":1.204}]}]},{"osd":5,"up_from":37,"seq":158913790257,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307300,"kb_used_data":5980,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93437852,"statfs":{"total":95995035648,"available":95680360448,"internally_reserved":0,"allocated":6123520,"data_stored":5846698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.89100000000000001,"15min":0.85199999999999998},"min":{"1min":0.56000000000000005,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.2350000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.91900000000000004,"15min":0.91100000000000003},"min":{"1min":0.52900000000000003,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.2549999999999999,"5min":1.4359999999999999,"15min":1.6080000000000001},"last":0.65000000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.92500000000000004,"15min":0.93100000000000005},"min":{"1min":0.61699999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.1890000000000001,"5min":1.3240000000000001,"15min":3.3490000000000002},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.0249999999999999,"15min":1.046},"min":{"1min":0.71299999999999997,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.179,"5min":1.5509999999999999,"15min":3.3730000000000002},"last":0.92700000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:24:28 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.996,"15min":1.0089999999999999},"min":{"1min":0.624,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.141,"5min":1.4650000000000001,"15min":2.9220000000000002},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.03,"15min":0.98699999999999999},"min":{"1min":0.76000000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3340000000000001,"5min":1.649,"15min":1.649},"last":1.0820000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.97199999999999998,"15min":0.96999999999999997},"min":{"1min":0.67400000000000004,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.999,"5min":1.464,"15min":1.464},"last":0.79400000000000004},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.1180000000000001,"15min":1.081},"min":{"1min":0.79500000000000004,"5min":0.72299999999999998,"15min":0.72299999999999998},"max":{"1min":1.222,"5min":4.258,"15min":4.258},"last":1.216}]},{"osd":4,"last update":"Mon Oct 9 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.93100000000000005,"15min":0.92800000000000005},"min":{"1min":0.60199999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.0549999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.034,"15min":1.0389999999999999},"min":{"1min":0.66200000000000003,"5min":0.55400000000000005,"15min":0.41199999999999998},"max":{"1min":1.2450000000000001,"5min":1.6120000000000001,"15min":2.4990000000000001},"last":1.286}]},{"osd":6,"last update":"Mon Oct 9 09:23:37 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.0680000000000001,"15min":1.0389999999999999},"min":{"1min":0.67100000000000004,"5min":0.53400000000000003,"15min":0.50800000000000001},"max":{"1min":1.2250000000000001,"5min":1.637,"15min":2.0419999999999998},"last":0.98199999999999998},{"interface":"front","average":{"1min":1.167,"5min":1.2509999999999999,"15min":1.2090000000000001},"min":{"1min":0.74399999999999999,"5min":0.69099999999999995,"15min":0.45900000000000002},"max":{"1min":1.5029999999999999,"5min":1.9079999999999999,"15min":2.4079999999999999},"last":1.3759999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:24:17 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.1919999999999999,"15min":1.1659999999999999},"min":{"1min":0.68899999999999995,"5min":0.63900000000000001,"15min":0.55300000000000005},"max":{"1min":1.3839999999999999,"5min":1.9359999999999999,"15min":2.556},"last":1.3380000000000001},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.1359999999999999,"15min":1.1319999999999999},"min":{"1min":0.64300000000000002,"5min":0.51500000000000001,"15min":0.5},"max":{"1min":1.3839999999999999,"5min":1.663,"15min":2.3460000000000001},"last":1.2450000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:24:36.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:24:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:36 smithi163 bash[75425]: audit 2023-10-09T09:24:35.541478+0000 mgr.y (mgr.34107) 544 : audit 0 from='client.44621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:36 smithi163 bash[75425]: cluster 2023-10-09T09:24:35.556939+0000 mgr.y (mgr.34107) 545 : cluster 0 pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:36 smithi163 bash[78107]: audit 2023-10-09T09:24:35.541478+0000 mgr.y (mgr.34107) 544 : audit 0 from='client.44621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:36.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:36 smithi163 bash[78107]: cluster 2023-10-09T09:24:35.556939+0000 mgr.y (mgr.34107) 545 : cluster 0 pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:36 smithi181 bash[59345]: audit 2023-10-09T09:24:35.541478+0000 mgr.y (mgr.34107) 544 : audit 0 from='client.44621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:36 smithi181 bash[59345]: cluster 2023-10-09T09:24:35.556939+0000 mgr.y (mgr.34107) 545 : cluster 0 pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:38 smithi163 bash[78107]: cluster 2023-10-09T09:24:37.557998+0000 mgr.y (mgr.34107) 546 : cluster 0 pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:38 smithi163 bash[75425]: cluster 2023-10-09T09:24:37.557998+0000 mgr.y (mgr.34107) 546 : cluster 0 pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:39.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:38 smithi181 bash[59345]: cluster 2023-10-09T09:24:37.557998+0000 mgr.y (mgr.34107) 546 : cluster 0 pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:40 smithi163 bash[75425]: cluster 2023-10-09T09:24:39.558784+0000 mgr.y (mgr.34107) 547 : cluster 0 pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:40 smithi163 bash[75425]: audit 2023-10-09T09:24:40.358076+0000 mon.a (mon.0) 1342 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:40 smithi163 bash[78107]: cluster 2023-10-09T09:24:39.558784+0000 mgr.y (mgr.34107) 547 : cluster 0 pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:40 smithi163 bash[78107]: audit 2023-10-09T09:24:40.358076+0000 mon.a (mon.0) 1342 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:40 smithi181 bash[59345]: cluster 2023-10-09T09:24:39.558784+0000 mgr.y (mgr.34107) 547 : cluster 0 pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:40 smithi181 bash[59345]: audit 2023-10-09T09:24:40.358076+0000 mon.a (mon.0) 1342 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:41.140 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:42 smithi163 bash[75425]: cluster 2023-10-09T09:24:41.559379+0000 mgr.y (mgr.34107) 548 : cluster 0 pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:42 smithi163 bash[75425]: audit 2023-10-09T09:24:42.254610+0000 mon.a (mon.0) 1343 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:24:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:42 smithi163 bash[78107]: cluster 2023-10-09T09:24:41.559379+0000 mgr.y (mgr.34107) 548 : cluster 0 pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:42 smithi163 bash[78107]: audit 2023-10-09T09:24:42.254610+0000 mon.a (mon.0) 1343 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:24:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:42 smithi181 bash[59345]: cluster 2023-10-09T09:24:41.559379+0000 mgr.y (mgr.34107) 548 : cluster 0 pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:42 smithi181 bash[59345]: audit 2023-10-09T09:24:42.254610+0000 mon.a (mon.0) 1343 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:24:43.620 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:24:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:24:43] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-09T09:24:43.988 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:24:43.992 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:24:44.735 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1232,"stamp":"2023-10-09T09:24:43.559538+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1428592,"kb_used_data":53616,"kb_used_omap":416,"kb_used_meta":1374495,"kb_avail":748532624,"statfs":{"total":767960285184,"available":766497406976,"internally_reserved":0,"allocated":54902784,"data_stored":49262341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":1407483313},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.014678"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1041,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433825+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:22:35.433825+0000","last_peered":"2023-10-09T09:22:35.433825+0000","last_clean":"2023-10-09T09:22:35.433825+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:22:35.433825+0000","last_undegraded":"2023-10-09T09:22:35.433825+0000","last_fullsized":"2023-10-09T09:22:35.433825+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1004,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653453+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:22:51.653453+0000","last_peered":"2023-10-09T09:22:51.653453+0000","last_clean":"2023-10-09T09:22:51.653453+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:22:51.653453+0000","last_undegraded":"2023-10-09T09:22:51.653453+0000","last_fullsized":"2023-10-09T09:22:51.653453+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1008,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.799357+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:22:51.799357+0000","last_peered":"2023-10-09T09:22:51.799357+0000","last_clean":"2023-10-09T09:22:51.799357+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:22:51.799357+0000","last_undegraded":"2023-10-09T09:22:51.799357+0000","last_fullsized":"2023-10-09T09:22:51.799357+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624753+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:22:51.624753+0000","last_peered":"2023-10-09T09:22:51.624753+0000","last_clean":"2023-10-09T09:22:51.624753+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:22:51.624753+0000","last_undegraded":"2023-10-09T09:22:51.624753+0000","last_fullsized":"2023-10-09T09:22:51.624753+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":985,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.256827+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:22:49.256827+0000","last_peered":"2023-10-09T09:22:49.256827+0000","last_clean":"2023-10-09T09:22:49.256827+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:22:49.256827+0000","last_undegraded":"2023-10-09T09:22:49.256827+0000","last_fullsized":"2023-10-09T09:22:49.256827+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438338+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:22:35.438338+0000","last_peered":"2023-10-09T09:22:35.438338+0000","last_clean":"2023-10-09T09:22:35.438338+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:22:35.438338+0000","last_undegraded":"2023-10-09T09:22:35.438338+0000","last_fullsized":"2023-10-09T09:22:35.438338+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624927+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:22:51.624927+0000","last_peered":"2023-10-09T09:22:51.624927+0000","last_clean":"2023-10-09T09:22:51.624927+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:22:51.624927+0000","last_undegraded":"2023-10-09T09:22:51.624927+0000","last_fullsized":"2023-10-09T09:22:51.624927+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":969,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431402+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:22:35.431402+0000","last_peered":"2023-10-09T09:22:35.431402+0000","last_clean":"2023-10-09T09:22:35.431402+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:22:35.431402+0000","last_undegraded":"2023-10-09T09:22:35.431402+0000","last_fullsized":"2023-10-09T09:22:35.431402+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1026,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378044+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:22:35.378044+0000","last_peered":"2023-10-09T09:22:35.378044+0000","last_clean":"2023-10-09T09:22:35.378044+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:22:35.378044+0000","last_undegraded":"2023-10-09T09:22:35.378044+0000","last_fullsized":"2023-10-09T09:22:35.378044+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":980,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.431472+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:22:35.431472+0000","last_peered":"2023-10-09T09:22:35.431472+0000","last_clean":"2023-10-09T09:22:35.431472+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:22:35.431472+0000","last_undegraded":"2023-10-09T09:22:35.431472+0000","last_fullsized":"2023-10-09T09:22:35.431472+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1005,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435642+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:22:35.435642+0000","last_peered":"2023-10-09T09:22:35.435642+0000","last_clean":"2023-10-09T09:22:35.435642+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:22:35.435642+0000","last_undegraded":"2023-10-09T09:22:35.435642+0000","last_fullsized":"2023-10-09T09:22:35.435642+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1008,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.798999+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:22:51.798999+0000","last_peered":"2023-10-09T09:22:51.798999+0000","last_clean":"2023-10-09T09:22:51.798999+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:22:51.798999+0000","last_undegraded":"2023-10-09T09:22:51.798999+0000","last_fullsized":"2023-10-09T09:22:51.798999+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1032,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433801+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:22:35.433801+0000","last_peered":"2023-10-09T09:22:35.433801+0000","last_clean":"2023-10-09T09:22:35.433801+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:22:35.433801+0000","last_undegraded":"2023-10-09T09:22:35.433801+0000","last_fullsized":"2023-10-09T09:22:35.433801+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433513+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:22:35.433513+0000","last_peered":"2023-10-09T09:22:35.433513+0000","last_clean":"2023-10-09T09:22:35.433513+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:22:35.433513+0000","last_undegraded":"2023-10-09T09:22:35.433513+0000","last_fullsized":"2023-10-09T09:22:35.433513+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435712+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:22:35.435712+0000","last_peered":"2023-10-09T09:22:35.435712+0000","last_clean":"2023-10-09T09:22:35.435712+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:22:35.435712+0000","last_undegraded":"2023-10-09T09:22:35.435712+0000","last_fullsized":"2023-10-09T09:22:35.435712+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":969,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433403+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:22:35.433403+0000","last_peered":"2023-10-09T09:22:35.433403+0000","last_clean":"2023-10-09T09:22:35.433403+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:22:35.433403+0000","last_undegraded":"2023-10-09T09:22:35.433403+0000","last_fullsized":"2023-10-09T09:22:35.433403+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":999,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.622061+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:22:51.622061+0000","last_peered":"2023-10-09T09:22:51.622061+0000","last_clean":"2023-10-09T09:22:51.622061+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:22:51.622061+0000","last_undegraded":"2023-10-09T09:22:51.622061+0000","last_fullsized":"2023-10-09T09:22:51.622061+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378062+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:22:35.378062+0000","last_peered":"2023-10-09T09:22:35.378062+0000","last_clean":"2023-10-09T09:22:35.378062+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:22:35.378062+0000","last_undegraded":"2023-10-09T09:22:35.378062+0000","last_fullsized":"2023-10-09T09:22:35.378062+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":987,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.436318+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:22:35.436318+0000","last_peered":"2023-10-09T09:22:35.436318+0000","last_clean":"2023-10-09T09:22:35.436318+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:22:35.436318+0000","last_undegraded":"2023-10-09T09:22:35.436318+0000","last_fullsized":"2023-10-09T09:22:35.436318+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624005+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:22:51.624005+0000","last_peered":"2023-10-09T09:22:51.624005+0000","last_clean":"2023-10-09T09:22:51.624005+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:22:51.624005+0000","last_undegraded":"2023-10-09T09:22:51.624005+0000","last_fullsized":"2023-10-09T09:22:51.624005+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1015,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437450+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:22:35.437450+0000","last_peered":"2023-10-09T09:22:35.437450+0000","last_clean":"2023-10-09T09:22:35.437450+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:22:35.437450+0000","last_undegraded":"2023-10-09T09:22:35.437450+0000","last_fullsized":"2023-10-09T09:22:35.437450+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.257124+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:22:49.257124+0000","last_peered":"2023-10-09T09:22:49.257124+0000","last_clean":"2023-10-09T09:22:49.257124+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:22:49.257124+0000","last_undegraded":"2023-10-09T09:22:49.257124+0000","last_fullsized":"2023-10-09T09:22:49.257124+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1003,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435617+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:22:35.435617+0000","last_peered":"2023-10-09T09:22:35.435617+0000","last_clean":"2023-10-09T09:22:35.435617+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:22:35.435617+0000","last_undegraded":"2023-10-09T09:22:35.435617+0000","last_fullsized":"2023-10-09T09:22:35.435617+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377442+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:22:35.377442+0000","last_peered":"2023-10-09T09:22:35.377442+0000","last_clean":"2023-10-09T09:22:35.377442+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:22:35.377442+0000","last_undegraded":"2023-10-09T09:22:35.377442+0000","last_fullsized":"2023-10-09T09:22:35.377442+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435716+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:22:35.435716+0000","last_peered":"2023-10-09T09:22:35.435716+0000","last_clean":"2023-10-09T09:22:35.435716+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:22:35.435716+0000","last_undegraded":"2023-10-09T09:22:35.435716+0000","last_fullsized":"2023-10-09T09:22:35.435716+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433540+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:22:35.433540+0000","last_peered":"2023-10-09T09:22:35.433540+0000","last_clean":"2023-10-09T09:22:35.433540+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:22:35.433540+0000","last_undegraded":"2023-10-09T09:22:35.433540+0000","last_fullsized":"2023-10-09T09:22:35.433540+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":988,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.436871+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:22:35.436871+0000","last_peered":"2023-10-09T09:22:35.436871+0000","last_clean":"2023-10-09T09:22:35.436871+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:22:35.436871+0000","last_undegraded":"2023-10-09T09:22:35.436871+0000","last_fullsized":"2023-10-09T09:22:35.436871+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1006,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437643+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:22:35.437643+0000","last_peered":"2023-10-09T09:22:35.437643+0000","last_clean":"2023-10-09T09:22:35.437643+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:22:35.437643+0000","last_undegraded":"2023-10-09T09:22:35.437643+0000","last_fullsized":"2023-10-09T09:22:35.437643+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1026,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377882+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:22:35.377882+0000","last_peered":"2023-10-09T09:22:35.377882+0000","last_clean":"2023-10-09T09:22:35.377882+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:22:35.377882+0000","last_undegraded":"2023-10-09T09:22:35.377882+0000","last_fullsized":"2023-10-09T09:22:35.377882+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.622439+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:22:51.622439+0000","last_peered":"2023-10-09T09:22:51.622439+0000","last_clean":"2023-10-09T09:22:51.622439+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:22:51.622439+0000","last_undegraded":"2023-10-09T09:22:51.622439+0000","last_fullsized":"2023-10-09T09:22:51.622439+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":988,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652916+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:22:51.652916+0000","last_peered":"2023-10-09T09:22:51.652916+0000","last_clean":"2023-10-09T09:22:51.652916+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:22:51.652916+0000","last_undegraded":"2023-10-09T09:22:51.652916+0000","last_fullsized":"2023-10-09T09:22:51.652916+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434896+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:22:35.434896+0000","last_peered":"2023-10-09T09:22:35.434896+0000","last_clean":"2023-10-09T09:22:35.434896+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:22:35.434896+0000","last_undegraded":"2023-10-09T09:22:35.434896+0000","last_fullsized":"2023-10-09T09:22:35.434896+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1007,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.437878+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:22:35.437878+0000","last_peered":"2023-10-09T09:22:35.437878+0000","last_clean":"2023-10-09T09:22:35.437878+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:22:35.437878+0000","last_undegraded":"2023-10-09T09:22:35.437878+0000","last_fullsized":"2023-10-09T09:22:35.437878+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434259+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:22:35.434259+0000","last_peered":"2023-10-09T09:22:35.434259+0000","last_clean":"2023-10-09T09:22:35.434259+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:22:35.434259+0000","last_undegraded":"2023-10-09T09:22:35.434259+0000","last_fullsized":"2023-10-09T09:22:35.434259+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433074+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:22:35.433074+0000","last_peered":"2023-10-09T09:22:35.433074+0000","last_clean":"2023-10-09T09:22:35.433074+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:22:35.433074+0000","last_undegraded":"2023-10-09T09:22:35.433074+0000","last_fullsized":"2023-10-09T09:22:35.433074+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.434199+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:22:35.434199+0000","last_peered":"2023-10-09T09:22:35.434199+0000","last_clean":"2023-10-09T09:22:35.434199+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:22:35.434199+0000","last_undegraded":"2023-10-09T09:22:35.434199+0000","last_fullsized":"2023-10-09T09:22:35.434199+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":971,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652272+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:22:51.652272+0000","last_peered":"2023-10-09T09:22:51.652272+0000","last_clean":"2023-10-09T09:22:51.652272+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:22:51.652272+0000","last_undegraded":"2023-10-09T09:22:51.652272+0000","last_fullsized":"2023-10-09T09:22:51.652272+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1008,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801270+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:22:51.801270+0000","last_peered":"2023-10-09T09:22:51.801270+0000","last_clean":"2023-10-09T09:22:51.801270+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:22:51.801270+0000","last_undegraded":"2023-10-09T09:22:51.801270+0000","last_fullsized":"2023-10-09T09:22:51.801270+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1003,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801363+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:22:51.801363+0000","last_peered":"2023-10-09T09:22:51.801363+0000","last_clean":"2023-10-09T09:22:51.801363+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:22:51.801363+0000","last_undegraded":"2023-10-09T09:22:51.801363+0000","last_fullsized":"2023-10-09T09:22:51.801363+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1041,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378075+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:22:35.378075+0000","last_peered":"2023-10-09T09:22:35.378075+0000","last_clean":"2023-10-09T09:22:35.378075+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:22:35.378075+0000","last_undegraded":"2023-10-09T09:22:35.378075+0000","last_fullsized":"2023-10-09T09:22:35.378075+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":993,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.799273+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:22:51.799273+0000","last_peered":"2023-10-09T09:22:51.799273+0000","last_clean":"2023-10-09T09:22:51.799273+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:22:51.799273+0000","last_undegraded":"2023-10-09T09:22:51.799273+0000","last_fullsized":"2023-10-09T09:22:51.799273+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1030,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377624+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:22:35.377624+0000","last_peered":"2023-10-09T09:22:35.377624+0000","last_clean":"2023-10-09T09:22:35.377624+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:22:35.377624+0000","last_undegraded":"2023-10-09T09:22:35.377624+0000","last_fullsized":"2023-10-09T09:22:35.377624+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1025,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433321+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:22:35.433321+0000","last_peered":"2023-10-09T09:22:35.433321+0000","last_clean":"2023-10-09T09:22:35.433321+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:22:35.433321+0000","last_undegraded":"2023-10-09T09:22:35.433321+0000","last_fullsized":"2023-10-09T09:22:35.433321+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1023,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.800115+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:22:51.800115+0000","last_peered":"2023-10-09T09:22:51.800115+0000","last_clean":"2023-10-09T09:22:51.800115+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:22:51.800115+0000","last_undegraded":"2023-10-09T09:22:51.800115+0000","last_fullsized":"2023-10-09T09:22:51.800115+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":990,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:50.876223+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:22:50.876223+0000","last_peered":"2023-10-09T09:22:50.876223+0000","last_clean":"2023-10-09T09:22:50.876223+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:22:50.876223+0000","last_undegraded":"2023-10-09T09:22:50.876223+0000","last_fullsized":"2023-10-09T09:22:50.876223+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1009,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.798918+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:22:51.798918+0000","last_peered":"2023-10-09T09:22:51.798918+0000","last_clean":"2023-10-09T09:22:51.798918+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:22:51.798918+0000","last_undegraded":"2023-10-09T09:22:51.798918+0000","last_fullsized":"2023-10-09T09:22:51.798918+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.801584+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:22:51.801584+0000","last_peered":"2023-10-09T09:22:51.801584+0000","last_clean":"2023-10-09T09:22:51.801584+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:22:51.801584+0000","last_undegraded":"2023-10-09T09:22:51.801584+0000","last_fullsized":"2023-10-09T09:22:51.801584+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1030,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.377739+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:22:35.377739+0000","last_peered":"2023-10-09T09:22:35.377739+0000","last_clean":"2023-10-09T09:22:35.377739+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:22:35.377739+0000","last_undegraded":"2023-10-09T09:22:35.377739+0000","last_fullsized":"2023-10-09T09:22:35.377739+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1238,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435393+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:22:35.435393+0000","last_peered":"2023-10-09T09:22:35.435393+0000","last_clean":"2023-10-09T09:22:35.435393+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:22:35.435393+0000","last_undegraded":"2023-10-09T09:22:35.435393+0000","last_fullsized":"2023-10-09T09:22:35.435393+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":982,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652484+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:22:51.652484+0000","last_peered":"2023-10-09T09:22:51.652484+0000","last_clean":"2023-10-09T09:22:51.652484+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:22:51.652484+0000","last_undegraded":"2023-10-09T09:22:51.652484+0000","last_fullsized":"2023-10-09T09:22:51.652484+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":991,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438069+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:22:35.438069+0000","last_peered":"2023-10-09T09:22:35.438069+0000","last_clean":"2023-10-09T09:22:35.438069+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:22:35.438069+0000","last_undegraded":"2023-10-09T09:22:35.438069+0000","last_fullsized":"2023-10-09T09:22:35.438069+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1021,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435529+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:22:35.435529+0000","last_peered":"2023-10-09T09:22:35.435529+0000","last_clean":"2023-10-09T09:22:35.435529+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:22:35.435529+0000","last_undegraded":"2023-10-09T09:22:35.435529+0000","last_fullsized":"2023-10-09T09:22:35.435529+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1000,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.621936+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:22:51.621936+0000","last_peered":"2023-10-09T09:22:51.621936+0000","last_clean":"2023-10-09T09:22:51.621936+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:22:51.621936+0000","last_undegraded":"2023-10-09T09:22:51.621936+0000","last_fullsized":"2023-10-09T09:22:51.621936+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1010,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.621540+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:22:51.621540+0000","last_peered":"2023-10-09T09:22:51.621540+0000","last_clean":"2023-10-09T09:22:51.621540+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:22:51.621540+0000","last_undegraded":"2023-10-09T09:22:51.621540+0000","last_fullsized":"2023-10-09T09:22:51.621540+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1006,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.435480+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:22:35.435480+0000","last_peered":"2023-10-09T09:22:35.435480+0000","last_clean":"2023-10-09T09:22:35.435480+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:22:35.435480+0000","last_undegraded":"2023-10-09T09:22:35.435480+0000","last_fullsized":"2023-10-09T09:22:35.435480+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1009,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.799014+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:22:51.799014+0000","last_peered":"2023-10-09T09:22:51.799014+0000","last_clean":"2023-10-09T09:22:51.799014+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:22:51.799014+0000","last_undegraded":"2023-10-09T09:22:51.799014+0000","last_fullsized":"2023-10-09T09:22:51.799014+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":984,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624920+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:22:51.624920+0000","last_peered":"2023-10-09T09:22:51.624920+0000","last_clean":"2023-10-09T09:22:51.624920+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:22:51.624920+0000","last_undegraded":"2023-10-09T09:22:51.624920+0000","last_fullsized":"2023-10-09T09:22:51.624920+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":988,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653895+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:22:51.653895+0000","last_peered":"2023-10-09T09:22:51.653895+0000","last_clean":"2023-10-09T09:22:51.653895+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:22:51.653895+0000","last_undegraded":"2023-10-09T09:22:51.653895+0000","last_fullsized":"2023-10-09T09:22:51.653895+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":992,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.800514+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:22:51.800514+0000","last_peered":"2023-10-09T09:22:51.800514+0000","last_clean":"2023-10-09T09:22:51.800514+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:22:51.800514+0000","last_undegraded":"2023-10-09T09:22:51.800514+0000","last_fullsized":"2023-10-09T09:22:51.800514+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":985,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.652104+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:22:51.652104+0000","last_peered":"2023-10-09T09:22:51.652104+0000","last_clean":"2023-10-09T09:22:51.652104+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:22:51.652104+0000","last_undegraded":"2023-10-09T09:22:51.652104+0000","last_fullsized":"2023-10-09T09:22:51.652104+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.438335+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:22:35.438335+0000","last_peered":"2023-10-09T09:22:35.438335+0000","last_clean":"2023-10-09T09:22:35.438335+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:22:35.438335+0000","last_undegraded":"2023-10-09T09:22:35.438335+0000","last_fullsized":"2023-10-09T09:22:35.438335+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1002,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.625144+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:22:51.625144+0000","last_peered":"2023-10-09T09:22:51.625144+0000","last_clean":"2023-10-09T09:22:51.625144+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:22:51.625144+0000","last_undegraded":"2023-10-09T09:22:51.625144+0000","last_fullsized":"2023-10-09T09:22:51.625144+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1009,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433219+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:22:35.433219+0000","last_peered":"2023-10-09T09:22:35.433219+0000","last_clean":"2023-10-09T09:22:35.433219+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:22:35.433219+0000","last_undegraded":"2023-10-09T09:22:35.433219+0000","last_fullsized":"2023-10-09T09:22:35.433219+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":987,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.624417+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:22:51.624417+0000","last_peered":"2023-10-09T09:22:51.624417+0000","last_clean":"2023-10-09T09:22:51.624417+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:22:51.624417+0000","last_undegraded":"2023-10-09T09:22:51.624417+0000","last_fullsized":"2023-10-09T09:22:51.624417+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1008,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433286+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:22:35.433286+0000","last_peered":"2023-10-09T09:22:35.433286+0000","last_clean":"2023-10-09T09:22:35.433286+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:22:35.433286+0000","last_undegraded":"2023-10-09T09:22:35.433286+0000","last_fullsized":"2023-10-09T09:22:35.433286+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1028,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378083+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:22:35.378083+0000","last_peered":"2023-10-09T09:22:35.378083+0000","last_clean":"2023-10-09T09:22:35.378083+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:22:35.378083+0000","last_undegraded":"2023-10-09T09:22:35.378083+0000","last_fullsized":"2023-10-09T09:22:35.378083+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433836+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:22:35.433836+0000","last_peered":"2023-10-09T09:22:35.433836+0000","last_clean":"2023-10-09T09:22:35.433836+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:22:35.433836+0000","last_undegraded":"2023-10-09T09:22:35.433836+0000","last_fullsized":"2023-10-09T09:22:35.433836+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1013,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.378205+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:22:35.378205+0000","last_peered":"2023-10-09T09:22:35.378205+0000","last_clean":"2023-10-09T09:22:35.378205+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:22:35.378205+0000","last_undegraded":"2023-10-09T09:22:35.378205+0000","last_fullsized":"2023-10-09T09:22:35.378205+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1039,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.433894+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:22:35.433894+0000","last_peered":"2023-10-09T09:22:35.433894+0000","last_clean":"2023-10-09T09:22:35.433894+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:22:35.433894+0000","last_undegraded":"2023-10-09T09:22:35.433894+0000","last_fullsized":"2023-10-09T09:22:35.433894+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":998,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.653248+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:22:51.653248+0000","last_peered":"2023-10-09T09:22:51.653248+0000","last_clean":"2023-10-09T09:22:51.653248+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:22:51.653248+0000","last_undegraded":"2023-10-09T09:22:51.653248+0000","last_fullsized":"2023-10-09T09:22:51.653248+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:49.256889+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:22:49.256889+0000","last_peered":"2023-10-09T09:22:49.256889+0000","last_clean":"2023-10-09T09:22:49.256889+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:22:49.256889+0000","last_undegraded":"2023-10-09T09:22:49.256889+0000","last_fullsized":"2023-10-09T09:22:49.256889+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1006,"reported_epoch":1074,"state":"active+clean","last_fresh":"2023-10-09T09:22:51.621680+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:22:51.621680+0000","last_peered":"2023-10-09T09:22:51.621680+0000","last_clean":"2023-10-09T09:22:51.621680+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:22:51.621680+0000","last_undegraded":"2023-10-09T09:22:51.621680+0000","last_fullsized":"2023-10-09T09:22:51.621680+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1024,"reported_epoch":1073,"state":"active+clean","last_fresh":"2023-10-09T09:22:35.432824+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:22:35.432824+0000","last_peered":"2023-10-09T09:22:35.432824+0000","last_clean":"2023-10-09T09:22:35.432824+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:22:35.432824+0000","last_undegraded":"2023-10-09T09:22:35.432824+0000","last_fullsized":"2023-10-09T09:22:35.432824+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397795,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316092,"kb_used_data":6580,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93429060,"statfs":{"total":95995035648,"available":95671357440,"internally_reserved":0,"allocated":6737920,"data_stored":6441392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.079,"15min":1.0449999999999999},"min":{"1min":0.66300000000000003,"5min":0.59699999999999998,"15min":0.41899999999999998},"max":{"1min":1.325,"5min":1.837,"15min":1.837},"last":1.2070000000000001},{"interface":"front","average":{"1min":1.115,"5min":1.1859999999999999,"15min":1.133},"min":{"1min":0.873,"5min":0.747,"15min":0.48599999999999999},"max":{"1min":1.361,"5min":1.905,"15min":1.905},"last":1.3200000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.93200000000000005,"15min":0.90200000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.145,"5min":1.4019999999999999,"15min":1.5980000000000001},"last":1.0429999999999999},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.016,"15min":0.98399999999999999},"min":{"1min":0.66600000000000004,"5min":0.66500000000000004,"15min":0.60199999999999998},"max":{"1min":1.3049999999999999,"5min":1.722,"15min":2.028},"last":0.81399999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:24:19 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.1739999999999999,"15min":1.111},"min":{"1min":0.77000000000000002,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.5009999999999999,"5min":2.4710000000000001,"15min":3.0059999999999998},"last":1.363},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.032,"15min":0.999},"min":{"1min":0.61699999999999999,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2110000000000001,"5min":2.6539999999999999,"15min":3.0249999999999999},"last":1.234}]},{"osd":3,"last update":"Mon Oct 9 09:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0129999999999999,"15min":0.998},"min":{"1min":0.70899999999999996,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.2729999999999999,"5min":1.825,"15min":1.825},"last":0.97599999999999998},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1799999999999999,"15min":1.1399999999999999},"min":{"1min":0.82899999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.51,"5min":1.673,"15min":1.673},"last":1.26}]},{"osd":4,"last update":"Mon Oct 9 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.95699999999999996,"15min":0.94999999999999996},"min":{"1min":0.69999999999999996,"5min":0.374,"15min":0.374},"max":{"1min":1.157,"5min":1.4650000000000001,"15min":1.6679999999999999},"last":1.2829999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.98399999999999999,"15min":1.0149999999999999},"min":{"1min":0.70399999999999996,"5min":0.65000000000000002,"15min":0.50900000000000001},"max":{"1min":1.3580000000000001,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.94399999999999995}]},{"osd":5,"last update":"Mon Oct 9 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.1819999999999999,"15min":1.119},"min":{"1min":0.748,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.4359999999999999,"5min":1.9019999999999999,"15min":2.2330000000000001},"last":1.089},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.0740000000000001,"15min":1.069},"min":{"1min":0.72699999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.274,"5min":1.6859999999999999,"15min":2.302},"last":1.157}]},{"osd":6,"last update":"Mon Oct 9 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.087,"15min":1.0900000000000001},"min":{"1min":0.64100000000000001,"5min":0.60999999999999999,"15min":0.54300000000000004},"max":{"1min":1.296,"5min":1.569,"15min":2.0049999999999999},"last":1.3360000000000001},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.173,"15min":1.1699999999999999},"min":{"1min":0.80400000000000005,"5min":0.69299999999999995,"15min":0.60999999999999999},"max":{"1min":1.3069999999999999,"5min":2.0070000000000001,"15min":2.0070000000000001},"last":1.1319999999999999}]}]},{"osd":6,"up_from":43,"seq":184683594028,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311984,"kb_used_data":6568,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93433168,"statfs":{"total":95995035648,"available":95675564032,"internally_reserved":0,"allocated":6725632,"data_stored":6441246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:24:31 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.91200000000000003,"15min":0.91600000000000004},"min":{"1min":0.496,"5min":0.46500000000000002,"15min":0.35499999999999998},"max":{"1min":1.105,"5min":1.72,"15min":2.9060000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.0209999999999999,"15min":0.96499999999999997},"min":{"1min":0.67100000000000004,"5min":0.51100000000000001,"15min":0.38300000000000001},"max":{"1min":1.321,"5min":2.2669999999999999,"15min":2.2669999999999999},"last":0.38900000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:24:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.93200000000000005,"15min":0.92500000000000004},"min":{"1min":0.57799999999999996,"5min":0.54300000000000004,"15min":0.40699999999999997},"max":{"1min":1.2470000000000001,"5min":2.508,"15min":3.5590000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.92200000000000004,"15min":0.91200000000000003},"min":{"1min":0.49299999999999999,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.2589999999999999,"5min":1.581,"15min":3.3860000000000001},"last":0.77900000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:24:31 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.97899999999999998,"15min":0.93200000000000005},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.232,"5min":3.3210000000000002,"15min":3.3210000000000002},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.0600000000000001,"15min":0.98899999999999999},"min":{"1min":0.621,"5min":0.51800000000000002,"15min":0.51200000000000001},"max":{"1min":1.1879999999999999,"5min":2.2120000000000002,"15min":2.2120000000000002},"last":0.44800000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:23:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.004,"15min":1.0069999999999999},"min":{"1min":0.627,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":3.8010000000000002,"5min":3.8010000000000002,"15min":4.7430000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.052,"15min":1.0700000000000001},"min":{"1min":0.61899999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.3520000000000001,"5min":2.0649999999999999,"15min":3.4249999999999998},"last":0.64000000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:23:40 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.97999999999999998,"15min":0.96699999999999997},"min":{"1min":0.59699999999999998,"5min":0.57499999999999996,"15min":0.39900000000000002},"max":{"1min":1.204,"5min":2.3809999999999998,"15min":3.5640000000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.004,"5min":1.0469999999999999,"15min":1.0449999999999999},"min":{"1min":0.65700000000000003,"5min":0.57499999999999996,"15min":0.41799999999999998},"max":{"1min":1.3720000000000001,"5min":2.5800000000000001,"15min":4.0629999999999997},"last":0.751}]},{"osd":5,"last update":"Mon Oct 9 09:23:40 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.113,"15min":1.081},"min":{"1min":0.72399999999999998,"5min":0.56000000000000005,"15min":0.46000000000000002},"max":{"1min":1.3220000000000001,"5min":2.7229999999999999,"15min":2.7229999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.149,"15min":1.095},"min":{"1min":0.71499999999999997,"5min":0.63300000000000001,"15min":0.48699999999999999},"max":{"1min":1.3320000000000001,"5min":2.7570000000000001,"15min":3.246},"last":0.71099999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.169,"15min":1.1439999999999999},"min":{"1min":0.86799999999999999,"5min":0.68600000000000005,"15min":0.49299999999999999},"max":{"1min":1.468,"5min":2.8570000000000002,"15min":3.3340000000000001},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1759999999999999,"15min":1.1659999999999999},"min":{"1min":0.746,"5min":0.746,"15min":0.46999999999999997},"max":{"1min":1.3720000000000001,"5min":1.8580000000000001,"15min":2.6659999999999999},"last":0.60299999999999998}]}]},{"osd":2,"up_from":634,"seq":2723009265781,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":44020,"kb_used_data":7020,"kb_used_omap":122,"kb_used_meta":36869,"kb_avail":93701132,"statfs":{"total":95995035648,"available":95949959168,"internally_reserved":0,"allocated":7188480,"data_stored":6030632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":37754443},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:24:31 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.746,"15min":0.73699999999999999},"min":{"1min":0.45000000000000001,"5min":0.41799999999999998,"15min":0.39800000000000002},"max":{"1min":1.0640000000000001,"5min":1.421,"15min":1.454},"last":1.1779999999999999},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.72899999999999998,"15min":0.69399999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.33600000000000002},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.2809999999999999},"last":2.3999999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:24:31 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.746,"15min":0.70999999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.377},"max":{"1min":0.98299999999999998,"5min":1.2549999999999999,"15min":1.2809999999999999},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.80400000000000005,"15min":0.76000000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.0549999999999999,"5min":1.3100000000000001,"15min":1.956},"last":2.3570000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.82999999999999996,"15min":0.79800000000000004},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.40600000000000003},"max":{"1min":1.0960000000000001,"5min":1.1850000000000001,"15min":1.3169999999999999},"last":3.5569999999999999},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.91300000000000003,"15min":0.86699999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.4530000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":2.2589999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:24:31 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.083,"15min":1.036},"min":{"1min":0.747,"5min":0.70999999999999996,"15min":0.59399999999999997},"max":{"1min":1.2549999999999999,"5min":2.2559999999999998,"15min":2.2559999999999998},"last":1.466},{"interface":"front","average":{"1min":0.94399999999999995,"5min":1.0269999999999999,"15min":1.0269999999999999},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.3320000000000001,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":2.5619999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:24:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0680000000000001,"15min":1.038},"min":{"1min":0.73799999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.508,"5min":2.226,"15min":2.226},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.137,"15min":1.121},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.3089999999999999,"5min":1.696,"15min":1.696},"last":1.3180000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:24:31 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.179,"15min":1.1459999999999999},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.355,"5min":1.6160000000000001,"15min":1.716},"last":1.397},{"interface":"front","average":{"1min":1.044,"5min":1.1559999999999999,"15min":1.1080000000000001},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.2889999999999999,"5min":1.6879999999999999,"15min":1.7989999999999999},"last":2.6259999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:24:31 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.2,"15min":1.1559999999999999},"min":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.76700000000000002},"max":{"1min":1.335,"5min":1.748,"15min":1.8080000000000001},"last":2.4910000000000001},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.1819999999999999,"15min":1.1499999999999999},"min":{"1min":0.77000000000000002,"5min":0.748,"15min":0.68600000000000005},"max":{"1min":1.45,"5min":1.6890000000000001,"15min":1.752},"last":1.0800000000000001}]}]},{"osd":1,"up_from":607,"seq":2607045148794,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":39904,"kb_used_data":7000,"kb_used_omap":107,"kb_used_meta":32788,"kb_avail":93705248,"statfs":{"total":95995035648,"available":95954173952,"internally_reserved":0,"allocated":7168000,"data_stored":6027385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":33575595},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.67900000000000005,"15min":0.65200000000000002},"min":{"1min":0.377,"5min":0.37,"15min":0.35599999999999998},"max":{"1min":0.94799999999999995,"5min":1.2050000000000001,"15min":1.4410000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.73999999999999999,"15min":0.68799999999999994},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.32600000000000001},"max":{"1min":0.86899999999999999,"5min":1.1679999999999999,"15min":1.2729999999999999},"last":0.52500000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:24:28 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.71899999999999997,"15min":0.72099999999999997},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.129,"5min":1.363,"15min":3.1749999999999998},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.80700000000000005,"15min":0.81499999999999995},"min":{"1min":0.45200000000000001,"5min":0.41999999999999998,"15min":0.314},"max":{"1min":1.2609999999999999,"5min":1.333,"15min":3.484},"last":0.72999999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:23:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.83099999999999996,"15min":0.82299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.089,"5min":1.304,"15min":1.393},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.86699999999999999,"15min":0.88600000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.1080000000000001,"5min":1.417,"15min":1.6830000000000001},"last":0.622}]},{"osd":4,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.032,"15min":1.0129999999999999},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59399999999999997},"max":{"1min":1.3149999999999999,"5min":1.6850000000000001,"15min":1.9079999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.0109999999999999,"15min":1.0309999999999999},"min":{"1min":0.67000000000000004,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.363,"5min":1.5389999999999999,"15min":1.601},"last":0.75}]},{"osd":5,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.073,"15min":1.075},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.56499999999999995},"max":{"1min":1.252,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":0.92300000000000004},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.077,"15min":1.054},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2669999999999999,"5min":1.605,"15min":1.833},"last":0.68500000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0620000000000001,"15min":1.0580000000000001},"min":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.57599999999999996},"max":{"1min":1.6319999999999999,"5min":1.806,"15min":1.921},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.1379999999999999,"15min":1.091},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.5,"5min":1.837,"15min":1.837},"last":0.875}]},{"osd":7,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.137,"15min":1.1539999999999999},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.349,"5min":1.7629999999999999,"15min":1.9990000000000001},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.044,"5min":1.212,"15min":1.204},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.64200000000000002},"max":{"1min":1.431,"5min":1.9039999999999999,"15min":1.9039999999999999},"last":0.82999999999999996}]}]},{"osd":0,"up_from":581,"seq":2495375999104,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":54912,"kb_used_data":7608,"kb_used_omap":131,"kb_used_meta":47164,"kb_avail":93690240,"statfs":{"total":95995035648,"available":95938805760,"internally_reserved":0,"allocated":7790592,"data_stored":6621470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":48296537},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:23:57 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.73499999999999999,"15min":0.70699999999999996},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.94099999999999995,"5min":1.091,"15min":1.218},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.80400000000000005,"15min":0.76700000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.97299999999999998,"5min":1.393,"15min":1.393},"last":0.68799999999999994}]},{"osd":2,"last update":"Mon Oct 9 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.79800000000000004,"15min":0.77000000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.04,"5min":1.321,"15min":1.321},"last":1.046},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.78700000000000003,"15min":0.76100000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32800000000000001},"max":{"1min":1.036,"5min":1.669,"15min":1.669},"last":0.96699999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.85399999999999998,"15min":0.80200000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.071,"5min":1.712,"15min":1.712},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.93600000000000005,"15min":0.95899999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2829999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":1.163}]},{"osd":4,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.04,"15min":0.97699999999999998},"min":{"1min":0.66400000000000003,"5min":0.44,"15min":0.44},"max":{"1min":1.2809999999999999,"5min":1.742,"15min":1.742},"last":1.018},{"interface":"front","average":{"1min":0.89400000000000002,"5min":1.0009999999999999,"15min":1.026},"min":{"1min":0.63,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1359999999999999,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":0.873}]},{"osd":5,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.0860000000000001,"15min":1.054},"min":{"1min":0.61299999999999999,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.3400000000000001,"5min":1.7150000000000001,"15min":1.7150000000000001},"last":1.1319999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.071,"15min":1.075},"min":{"1min":0.68600000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.226,"5min":1.7669999999999999,"15min":1.8640000000000001},"last":0.98899999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.159,"15min":1.0900000000000001},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.61499999999999999},"max":{"1min":1.468,"5min":1.9319999999999999,"15min":1.9319999999999999},"last":1.2490000000000001},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.1499999999999999,"15min":1.107},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.4930000000000001,"5min":1.8859999999999999,"15min":1.8859999999999999},"last":1.2}]},{"osd":7,"last update":"Mon Oct 9 09:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.2250000000000001,"15min":1.155},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.5169999999999999,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":1.327},{"interface":"front","average":{"1min":1.099,"5min":1.22,"15min":1.177},"min":{"1min":0.72499999999999998,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.5429999999999999,"5min":1.8029999999999999,"15min":1.8029999999999999},"last":1.087}]}]},{"osd":3,"up_from":663,"seq":2847563317359,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":38872,"kb_used_data":6864,"kb_used_omap":55,"kb_used_meta":31944,"kb_avail":93706280,"statfs":{"total":95995035648,"available":95955230720,"internally_reserved":0,"allocated":7028736,"data_stored":6002564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":32711266},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.79800000000000004,"15min":0.77500000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.29699999999999999},"max":{"1min":0.88900000000000001,"5min":1.4350000000000001,"15min":1.5369999999999999},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.81399999999999995,"15min":0.75800000000000001},"min":{"1min":0.42799999999999999,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":0.96299999999999997,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":0.40200000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.85699999999999998,"15min":0.83699999999999997},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.442},"max":{"1min":0.88500000000000001,"5min":3.9449999999999998,"15min":3.9449999999999998},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.83399999999999996,"15min":0.82699999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.05,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":0.52200000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.85499999999999998,"15min":0.84299999999999997},"min":{"1min":0.51000000000000001,"5min":0.42499999999999999,"15min":0.39500000000000002},"max":{"1min":1.1459999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.94199999999999995,"15min":0.84399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85499999999999998,"5min":10.167,"15min":10.167},"last":0.70699999999999996}]},{"osd":4,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.0780000000000001,"15min":1.01},"min":{"1min":0.746,"5min":0.62,"15min":0.62},"max":{"1min":1.181,"5min":3.8959999999999999,"15min":3.8959999999999999},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.98899999999999999,"15min":0.98899999999999999},"min":{"1min":0.63100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.214,"5min":1.629,"15min":1.7270000000000001},"last":0.88600000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.117,"15min":1.115},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.59999999999999998},"max":{"1min":1.2310000000000001,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.1579999999999999,"15min":1.109},"min":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.65100000000000002},"max":{"1min":1.1890000000000001,"5min":3.996,"15min":3.996},"last":0.79700000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.095,"15min":1.073},"min":{"1min":0.69599999999999995,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.2170000000000001,"5min":1.663,"15min":1.663},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.01,"5min":1.179,"15min":1.1579999999999999},"min":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.68999999999999995},"max":{"1min":1.266,"5min":3.972,"15min":3.972},"last":0.74099999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.1879999999999999,"15min":1.1910000000000001},"min":{"1min":0.72599999999999998,"5min":0.65500000000000003,"15min":0.61699999999999999},"max":{"1min":1.385,"5min":1.821,"15min":1.8260000000000001},"last":0.84699999999999998},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.1439999999999999,"15min":1.1319999999999999},"min":{"1min":0.77600000000000002,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2929999999999999,"5min":1.7669999999999999,"15min":1.845},"last":1.0569999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986493,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315508,"kb_used_data":5996,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93429644,"statfs":{"total":95995035648,"available":95671955456,"internally_reserved":0,"allocated":6139904,"data_stored":5850954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:24:32 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.873,"15min":0.88600000000000001},"min":{"1min":0.56499999999999995,"5min":0.48999999999999999,"15min":0.315},"max":{"1min":1.167,"5min":1.278,"15min":2.0659999999999998},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.97099999999999997,"15min":0.92000000000000004},"min":{"1min":0.51900000000000002,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.26,"5min":1.653,"15min":1.653},"last":0.92400000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.92300000000000004,"15min":0.872},"min":{"1min":0.67900000000000005,"5min":0.375,"15min":0.375},"max":{"1min":1.2250000000000001,"5min":1.5489999999999999,"15min":1.5489999999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.98799999999999999,"15min":1.0029999999999999},"min":{"1min":0.53600000000000003,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.4590000000000001,"5min":1.722,"15min":1.722},"last":0.67800000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:24:17 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.95299999999999996,"15min":0.98099999999999998},"min":{"1min":0.57099999999999995,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.1259999999999999,"5min":5.7199999999999998,"15min":5.7199999999999998},"last":0.621},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.94299999999999995,"15min":0.95799999999999996},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.42999999999999999},"max":{"1min":1.359,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.999}]},{"osd":3,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.997,"15min":1.03},"min":{"1min":0.59099999999999997,"5min":0.57499999999999996,"15min":0.54800000000000004},"max":{"1min":1.3160000000000001,"5min":2.488,"15min":2.488},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.123,"15min":1.133},"min":{"1min":0.751,"5min":0.54300000000000004,"15min":0.41399999999999998},"max":{"1min":1.4119999999999999,"5min":4.9009999999999998,"15min":4.9009999999999998},"last":0.84999999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.98599999999999999},"min":{"1min":0.629,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.1639999999999999,"5min":1.6930000000000001,"15min":1.7390000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.109,"15min":1.107},"min":{"1min":0.69299999999999995,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.4770000000000001,"5min":2.411,"15min":2.411},"last":0.74299999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:23:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0880000000000001,"15min":1.0569999999999999},"min":{"1min":0.66100000000000003,"5min":0.504,"15min":0.48199999999999998},"max":{"1min":1.6479999999999999,"5min":1.6479999999999999,"15min":2.7719999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.095,"5min":1.099,"15min":1.123},"min":{"1min":0.68500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.5800000000000001,"5min":1.7849999999999999,"15min":1.8680000000000001},"last":0.86599999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.0740000000000001,"15min":1.0620000000000001},"min":{"1min":0.76500000000000001,"5min":0.56499999999999995,"15min":0.51200000000000001},"max":{"1min":1.476,"5min":1.9099999999999999,"15min":1.925},"last":0.77900000000000003},{"interface":"front","average":{"1min":1.133,"5min":1.155,"15min":1.135},"min":{"1min":0.79800000000000004,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.385,"5min":2.4529999999999998,"15min":2.4529999999999998},"last":0.89500000000000002}]}]},{"osd":5,"up_from":37,"seq":158913790259,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307300,"kb_used_data":5980,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93437852,"statfs":{"total":95995035648,"available":95680360448,"internally_reserved":0,"allocated":6123520,"data_stored":5846698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.86599999999999999,"15min":0.85299999999999998},"min":{"1min":0.56100000000000005,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.069,"5min":1.468,"15min":1.7250000000000001},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.81599999999999995,"5min":0.89700000000000002,"15min":0.90600000000000003},"min":{"1min":0.58299999999999996,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.198,"5min":1.4359999999999999,"15min":1.6080000000000001},"last":0.95999999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.92500000000000004,"15min":0.93100000000000005},"min":{"1min":0.61699999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.1890000000000001,"5min":1.3240000000000001,"15min":3.3490000000000002},"last":0.878},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.0249999999999999,"15min":1.046},"min":{"1min":0.71299999999999997,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.179,"5min":1.5509999999999999,"15min":3.3730000000000002},"last":1.079}]},{"osd":2,"last update":"Mon Oct 9 09:24:28 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.996,"15min":1.0089999999999999},"min":{"1min":0.624,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.141,"5min":1.4650000000000001,"15min":2.9220000000000002},"last":0.996},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.03,"15min":0.98699999999999999},"min":{"1min":0.76000000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3340000000000001,"5min":1.649,"15min":1.649},"last":1.175}]},{"osd":3,"last update":"Mon Oct 9 09:23:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.97199999999999998,"15min":0.96999999999999997},"min":{"1min":0.67400000000000004,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.999,"5min":1.464,"15min":1.464},"last":1.113},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.1180000000000001,"15min":1.081},"min":{"1min":0.79500000000000004,"5min":0.72299999999999998,"15min":0.72299999999999998},"max":{"1min":1.222,"5min":4.258,"15min":4.258},"last":1.129}]},{"osd":4,"last update":"Mon Oct 9 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.93100000000000005,"15min":0.92800000000000005},"min":{"1min":0.60199999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.0549999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.034,"15min":1.0389999999999999},"min":{"1min":0.66200000000000003,"5min":0.55400000000000005,"15min":0.41199999999999998},"max":{"1min":1.2450000000000001,"5min":1.6120000000000001,"15min":2.4990000000000001},"last":1.0549999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:24:40 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.052,"15min":1.032},"min":{"1min":0.55000000000000004,"5min":0.53400000000000003,"15min":0.50800000000000001},"max":{"1min":1.1339999999999999,"5min":1.637,"15min":1.637},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.234,"15min":1.2110000000000001},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.45900000000000002},"max":{"1min":1.401,"5min":1.9079999999999999,"15min":2.4079999999999999},"last":1.256}]},{"osd":7,"last update":"Mon Oct 9 09:24:17 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.1919999999999999,"15min":1.1659999999999999},"min":{"1min":0.68899999999999995,"5min":0.63900000000000001,"15min":0.55300000000000005},"max":{"1min":1.3839999999999999,"5min":1.9359999999999999,"15min":2.556},"last":1.216},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.1359999999999999,"15min":1.1319999999999999},"min":{"1min":0.64300000000000002,"5min":0.51500000000000001,"15min":0.5},"max":{"1min":1.3839999999999999,"5min":1.663,"15min":2.3460000000000001},"last":1.1930000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:24:44.741 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T09:24:44.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T09:24:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:44 smithi163 bash[75425]: cluster 2023-10-09T09:24:43.559965+0000 mgr.y (mgr.34107) 549 : cluster 0 pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:44 smithi163 bash[78107]: cluster 2023-10-09T09:24:43.559965+0000 mgr.y (mgr.34107) 549 : cluster 0 pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:45.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:44 smithi181 bash[59345]: cluster 2023-10-09T09:24:43.559965+0000 mgr.y (mgr.34107) 549 : cluster 0 pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:45.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:45 smithi163 bash[75425]: audit 2023-10-09T09:24:43.971946+0000 mgr.y (mgr.34107) 550 : audit 0 from='client.44627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:45.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:45 smithi163 bash[78107]: audit 2023-10-09T09:24:43.971946+0000 mgr.y (mgr.34107) 550 : audit 0 from='client.44627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:46.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:45 smithi181 bash[59345]: audit 2023-10-09T09:24:43.971946+0000 mgr.y (mgr.34107) 550 : audit 0 from='client.44627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:24:46.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:46 smithi163 bash[75425]: cluster 2023-10-09T09:24:45.560738+0000 mgr.y (mgr.34107) 551 : cluster 0 pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:46.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:46 smithi163 bash[78107]: cluster 2023-10-09T09:24:45.560738+0000 mgr.y (mgr.34107) 551 : cluster 0 pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:47.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:46 smithi181 bash[59345]: cluster 2023-10-09T09:24:45.560738+0000 mgr.y (mgr.34107) 551 : cluster 0 pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:48 smithi163 bash[78107]: cluster 2023-10-09T09:24:47.561604+0000 mgr.y (mgr.34107) 552 : cluster 0 pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:48 smithi163 bash[78107]: audit 2023-10-09T09:24:48.503096+0000 mon.a (mon.0) 1344 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:48 smithi163 bash[78107]: audit 2023-10-09T09:24:48.509738+0000 mon.a (mon.0) 1345 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:48 smithi163 bash[78107]: audit 2023-10-09T09:24:48.608863+0000 mon.a (mon.0) 1346 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:48 smithi163 bash[78107]: audit 2023-10-09T09:24:48.619114+0000 mon.a (mon.0) 1347 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:48.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:48 smithi163 bash[75425]: cluster 2023-10-09T09:24:47.561604+0000 mgr.y (mgr.34107) 552 : cluster 0 pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:48.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:48 smithi163 bash[75425]: audit 2023-10-09T09:24:48.503096+0000 mon.a (mon.0) 1344 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:48.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:48 smithi163 bash[75425]: audit 2023-10-09T09:24:48.509738+0000 mon.a (mon.0) 1345 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:48.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:48 smithi163 bash[75425]: audit 2023-10-09T09:24:48.608863+0000 mon.a (mon.0) 1346 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:48.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:48 smithi163 bash[75425]: audit 2023-10-09T09:24:48.619114+0000 mon.a (mon.0) 1347 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:48 smithi181 bash[59345]: cluster 2023-10-09T09:24:47.561604+0000 mgr.y (mgr.34107) 552 : cluster 0 pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:48 smithi181 bash[59345]: audit 2023-10-09T09:24:48.503096+0000 mon.a (mon.0) 1344 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:48 smithi181 bash[59345]: audit 2023-10-09T09:24:48.509738+0000 mon.a (mon.0) 1345 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:48 smithi181 bash[59345]: audit 2023-10-09T09:24:48.608863+0000 mon.a (mon.0) 1346 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:48 smithi181 bash[59345]: audit 2023-10-09T09:24:48.619114+0000 mon.a (mon.0) 1347 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:49.290 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:49 smithi163 bash[75425]: audit 2023-10-09T09:24:49.099932+0000 mon.a (mon.0) 1348 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:24:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:49 smithi163 bash[75425]: audit 2023-10-09T09:24:49.101651+0000 mon.a (mon.0) 1349 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:24:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:49 smithi163 bash[75425]: audit 2023-10-09T09:24:49.111507+0000 mon.a (mon.0) 1350 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:49 smithi163 bash[78107]: audit 2023-10-09T09:24:49.099932+0000 mon.a (mon.0) 1348 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:24:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:49 smithi163 bash[78107]: audit 2023-10-09T09:24:49.101651+0000 mon.a (mon.0) 1349 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:24:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:49 smithi163 bash[78107]: audit 2023-10-09T09:24:49.111507+0000 mon.a (mon.0) 1350 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:49 smithi181 bash[59345]: audit 2023-10-09T09:24:49.099932+0000 mon.a (mon.0) 1348 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:24:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:49 smithi181 bash[59345]: audit 2023-10-09T09:24:49.101651+0000 mon.a (mon.0) 1349 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:24:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:49 smithi181 bash[59345]: audit 2023-10-09T09:24:49.111507+0000 mon.a (mon.0) 1350 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:24:50.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:50 smithi163 bash[75425]: cluster 2023-10-09T09:24:49.562543+0000 mgr.y (mgr.34107) 553 : cluster 0 pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:50 smithi163 bash[78107]: cluster 2023-10-09T09:24:49.562543+0000 mgr.y (mgr.34107) 553 : cluster 0 pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:50 smithi181 bash[59345]: cluster 2023-10-09T09:24:49.562543+0000 mgr.y (mgr.34107) 553 : cluster 0 pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:52.190 INFO:teuthology.orchestra.run.smithi163.stdout:epoch 1074 2023-10-09T09:24:52.190 INFO:teuthology.orchestra.run.smithi163.stdout:fsid 097c0480-6681-11ee-8db5-212e2dc638e7 2023-10-09T09:24:52.190 INFO:teuthology.orchestra.run.smithi163.stdout:created 2023-10-09T08:53:11.193188+0000 2023-10-09T09:24:52.190 INFO:teuthology.orchestra.run.smithi163.stdout:modified 2023-10-09T09:22:50.565311+0000 2023-10-09T09:24:52.190 INFO:teuthology.orchestra.run.smithi163.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T09:24:52.190 INFO:teuthology.orchestra.run.smithi163.stdout:crush_version 22 2023-10-09T09:24:52.190 INFO:teuthology.orchestra.run.smithi163.stdout:full_ratio 0.95 2023-10-09T09:24:52.190 INFO:teuthology.orchestra.run.smithi163.stdout:backfillfull_ratio 0.9 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:nearfull_ratio 0.85 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:require_min_compat_client luminous 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:min_compat_client luminous 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:require_osd_release pacific 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:stretch_mode_enabled false 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 152 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 56 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 61 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:max_osd 8 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 up in weight 1 up_from 581 up_thru 663 down_at 568 last_clean_interval [8,567) [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309] [v2:172.21.15.163:6804/2071796309,v1:172.21.15.163:6805/2071796309] exists,up 09899392-dd0c-4eaf-b933-aafe194ff2b7 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 up in weight 1 up_from 607 up_thru 663 down_at 595 last_clean_interval [13,594) [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458] [v2:172.21.15.163:6812/3902451458,v1:172.21.15.163:6813/3902451458] exists,up fb74bcac-37d3-472b-abae-6c72df5b264b 2023-10-09T09:24:52.191 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 up in weight 1 up_from 634 up_thru 663 down_at 621 last_clean_interval [18,620) [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903] [v2:172.21.15.163:6820/2749314903,v1:172.21.15.163:6821/2749314903] exists,up 3f368e12-b9e7-46dc-93eb-5feee9e4ea2d 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 up in weight 1 up_from 663 up_thru 663 down_at 649 last_clean_interval [25,648) [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121] [v2:172.21.15.163:6828/198665121,v1:172.21.15.163:6829/198665121] exists,up 228a3c3f-20e9-4d74-a360-64c80ceadaf0 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 up in weight 1 up_from 31 up_thru 663 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913] [v2:172.21.15.181:6802/2112712913,v1:172.21.15.181:6803/2112712913] exists,up 70b76f58-2db5-421c-a558-db7c00067177 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 up in weight 1 up_from 37 up_thru 663 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733] [v2:172.21.15.181:6810/1025108733,v1:172.21.15.181:6811/1025108733] exists,up ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 up in weight 1 up_from 43 up_thru 649 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176] [v2:172.21.15.181:6818/1623825176,v1:172.21.15.181:6819/1623825176] exists,up 5de7d354-0821-4d89-ad3e-25c5aaed463a 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 up in weight 1 up_from 49 up_thru 663 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963] [v2:172.21.15.181:6826/800314963,v1:172.21.15.181:6827/800314963] exists,up fbb1831a-b34a-4d3a-b19b-507f65324930 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:pg_upmap_items 4.6 [1,5] 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:pg_upmap_items 4.17 [1,2] 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:pg_upmap_items 4.1d [1,2] 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6800/829587412 expires 2023-10-10T08:54:05.000912+0000 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/3642753484 expires 2023-10-10T09:09:25.185363+0000 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6801/1080938935 expires 2023-10-10T09:01:07.146560+0000 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/625604585 expires 2023-10-10T08:54:05.000912+0000 2023-10-09T09:24:52.192 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6800/1080938935 expires 2023-10-10T09:01:07.146560+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/1436605859 expires 2023-10-10T09:09:25.185363+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:0/394087329 expires 2023-10-10T09:08:23.434950+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6801/829587412 expires 2023-10-10T08:54:05.000912+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/623143106 expires 2023-10-10T09:06:46.871855+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:0/1788638593 expires 2023-10-10T09:08:23.434950+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/689482878 expires 2023-10-10T09:09:25.185363+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/3090084368 expires 2023-10-10T09:09:25.185363+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6801/1232823824 expires 2023-10-10T08:53:37.161178+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:0/2794784192 expires 2023-10-10T09:08:23.434950+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:0/769962302 expires 2023-10-10T09:08:23.434950+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/2563190797 expires 2023-10-10T09:09:25.185363+0000 2023-10-09T09:24:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/2356626184 expires 2023-10-10T08:54:05.000912+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/2199287559 expires 2023-10-10T09:01:07.146560+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6800/1737049372 expires 2023-10-10T09:09:25.185363+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6801/1737049372 expires 2023-10-10T09:09:25.185363+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/536711455 expires 2023-10-10T09:06:46.871855+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/1916578112 expires 2023-10-10T08:53:37.161178+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6800/1232823824 expires 2023-10-10T08:53:37.161178+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6800/3919298633 expires 2023-10-10T09:06:46.871855+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6801/3919298633 expires 2023-10-10T09:06:46.871855+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:6835/3017734555 expires 2023-10-10T09:08:23.434950+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/566729834 expires 2023-10-10T09:01:07.146560+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/1618377325 expires 2023-10-10T09:09:25.185363+0000 2023-10-09T09:24:52.194 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:6834/3017734555 expires 2023-10-10T09:08:23.434950+0000 2023-10-09T09:24:52.195 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/3487497332 expires 2023-10-10T08:53:37.161178+0000 2023-10-09T09:24:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:52 smithi163 bash[75425]: cluster 2023-10-09T09:24:51.563321+0000 mgr.y (mgr.34107) 554 : cluster 0 pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:52.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:52 smithi163 bash[75425]: audit 2023-10-09T09:24:52.188540+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.163:0/3723873357' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:24:52.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:52 smithi163 bash[78107]: cluster 2023-10-09T09:24:51.563321+0000 mgr.y (mgr.34107) 554 : cluster 0 pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:52.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:52 smithi163 bash[78107]: audit 2023-10-09T09:24:52.188540+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.163:0/3723873357' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:24:52.889 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 581 up_thru 663 down_at 568 last_clean_interval [8,567) [v2:172.21.15.163:6802/2071796309,v1:172.21.15.163:6803/2071796309] [v2:172.21.15.163:6804/2071796309,v1:172.21.15.163:6805/2071796309] exists,up 09899392-dd0c-4eaf-b933-aafe194ff2b7', 'osd.1 up in weight 1 up_from 607 up_thru 663 down_at 595 last_clean_interval [13,594) [v2:172.21.15.163:6810/3902451458,v1:172.21.15.163:6811/3902451458] [v2:172.21.15.163:6812/3902451458,v1:172.21.15.163:6813/3902451458] exists,up fb74bcac-37d3-472b-abae-6c72df5b264b', 'osd.2 up in weight 1 up_from 634 up_thru 663 down_at 621 last_clean_interval [18,620) [v2:172.21.15.163:6818/2749314903,v1:172.21.15.163:6819/2749314903] [v2:172.21.15.163:6820/2749314903,v1:172.21.15.163:6821/2749314903] exists,up 3f368e12-b9e7-46dc-93eb-5feee9e4ea2d', 'osd.3 up in weight 1 up_from 663 up_thru 663 down_at 649 last_clean_interval [25,648) [v2:172.21.15.163:6826/198665121,v1:172.21.15.163:6827/198665121] [v2:172.21.15.163:6828/198665121,v1:172.21.15.163:6829/198665121] exists,up 228a3c3f-20e9-4d74-a360-64c80ceadaf0', 'osd.4 up in weight 1 up_from 31 up_thru 663 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6800/2112712913,v1:172.21.15.181:6801/2112712913] [v2:172.21.15.181:6802/2112712913,v1:172.21.15.181:6803/2112712913] exists,up 70b76f58-2db5-421c-a558-db7c00067177', 'osd.5 up in weight 1 up_from 37 up_thru 663 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6808/1025108733,v1:172.21.15.181:6809/1025108733] [v2:172.21.15.181:6810/1025108733,v1:172.21.15.181:6811/1025108733] exists,up ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4', 'osd.6 up in weight 1 up_from 43 up_thru 649 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6816/1623825176,v1:172.21.15.181:6817/1623825176] [v2:172.21.15.181:6818/1623825176,v1:172.21.15.181:6819/1623825176] exists,up 5de7d354-0821-4d89-ad3e-25c5aaed463a', 'osd.7 up in weight 1 up_from 49 up_thru 663 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6824/800314963,v1:172.21.15.181:6825/800314963] [v2:172.21.15.181:6826/800314963,v1:172.21.15.181:6827/800314963] exists,up fbb1831a-b34a-4d3a-b19b-507f65324930'] 2023-10-09T09:24:52.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T09:24:53.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:52 smithi181 bash[59345]: cluster 2023-10-09T09:24:51.563321+0000 mgr.y (mgr.34107) 554 : cluster 0 pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:53.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:52 smithi181 bash[59345]: audit 2023-10-09T09:24:52.188540+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.163:0/3723873357' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:24:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:24:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:24:53] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-09T09:24:54.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:54 smithi163 bash[75425]: cluster 2023-10-09T09:24:53.563912+0000 mgr.y (mgr.34107) 555 : cluster 0 pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:54.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:54 smithi163 bash[78107]: cluster 2023-10-09T09:24:53.563912+0000 mgr.y (mgr.34107) 555 : cluster 0 pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:54 smithi181 bash[59345]: cluster 2023-10-09T09:24:53.563912+0000 mgr.y (mgr.34107) 555 : cluster 0 pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:55.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:55 smithi163 bash[75425]: audit 2023-10-09T09:24:55.358552+0000 mon.a (mon.0) 1351 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:55 smithi163 bash[78107]: audit 2023-10-09T09:24:55.358552+0000 mon.a (mon.0) 1351 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:55 smithi181 bash[59345]: audit 2023-10-09T09:24:55.358552+0000 mon.a (mon.0) 1351 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:24:56.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:56 smithi163 bash[75425]: cluster 2023-10-09T09:24:55.564741+0000 mgr.y (mgr.34107) 556 : cluster 0 pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:56.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:56 smithi163 bash[78107]: cluster 2023-10-09T09:24:55.564741+0000 mgr.y (mgr.34107) 556 : cluster 0 pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:56 smithi181 bash[59345]: cluster 2023-10-09T09:24:55.564741+0000 mgr.y (mgr.34107) 556 : cluster 0 pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:57.665 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:24:57.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:24:57 smithi163 bash[78107]: cluster 2023-10-09T09:24:57.565717+0000 mgr.y (mgr.34107) 557 : cluster 0 pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:57.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:24:57 smithi163 bash[75425]: cluster 2023-10-09T09:24:57.565717+0000 mgr.y (mgr.34107) 557 : cluster 0 pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:24:58.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:24:57 smithi181 bash[59345]: cluster 2023-10-09T09:24:57.565717+0000 mgr.y (mgr.34107) 557 : cluster 0 pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:00.379 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:25:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "name": "mon.a", 2023-10-09T09:25:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "cluster": "ceph", 2023-10-09T09:25:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T09:25:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "admin_socket_mode": "", 2023-10-09T09:25:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T09:25:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_client_required": "cephx, none", 2023-10-09T09:25:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_cluster_required": "cephx", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_debug": "true", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_service_required": "cephx", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_supported": "", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_aio": "true", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_aio_reap_max": "16", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_async_discard": "false", 2023-10-09T09:25:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_block_size": "4096", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_debug_aio": "false", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_enable_discard": "false", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_flock_retry": "3", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_inject_crash": "0", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_ioring": "false", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_ioring_hipri": "false", 2023-10-09T09:25:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_type": "", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_allocator": "hybrid", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_buffered_io": "true", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T09:25:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_replay_recovery": "false", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T09:25:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_sync_write": "false", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_allocator": "hybrid", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T09:25:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_create": "true", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_db_create": "false", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_db_path": "", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_db_size": "0", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_path": "", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_size": "107374182400", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_wal_create": "false", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_wal_path": "", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bluefs": "true", 2023-10-09T09:25:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_autotune": "true", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T09:25:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_size": "0", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_type": "2q", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_clone_cow": "true", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T09:25:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_mode": "none", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_freelist": "false", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T09:25:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T09:25:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fail_eio": "false", 2023-10-09T09:25:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T09:25:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T09:25:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_blob_size": "0", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T09:25:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_spdk_mem": "512", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T09:25:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_tracing": "false", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T09:25:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T09:25:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_cluster_require_version": "2", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_require_signatures": "false", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_require_version": "2", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_service_require_signatures": "false", 2023-10-09T09:25:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_service_require_version": "2", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_sign_messages": "true", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "chdir": "", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "client_acl_type": "", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "client_asio_thread_count": "2", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "client_cache_mid": "0.750000", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "client_cache_size": "16384", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "client_caps_release_delay": "5", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "client_check_pool_perm": "true", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "client_debug_force_sync_read": "false", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "client_debug_getattr_caps": "false", 2023-10-09T09:25:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_die_on_failed_remount": "false", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_dirsize_rbytes": "true", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_force_lazyio": "false", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_fs": "", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_inject_release_failure": "false", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_max_inline_size": "4096", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_mds_namespace": "", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_metadata": "", 2023-10-09T09:25:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_mount_gid": "-1", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_mount_timeout": "300", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_mount_uid": "-1", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_mountpoint": "/", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_notify_timeout": "10", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc": "true", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc_max_objects": "1000", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc_size": "209715200", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_permissions": "true", 2023-10-09T09:25:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_quota": "true", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_quota_df": "true", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_readahead_max_bytes": "0", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_readahead_max_periods": "4", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_readahead_min": "131072", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_reconnect_stale": "false", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_shutdown_timeout": "30", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_snapdir": ".snap", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_tick_interval": "1", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_trace": "", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_use_faked_inos": "false", 2023-10-09T09:25:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_use_random_mds": "false", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_graylog": "false", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_graylog_port": "12201", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_monitors": "default=true", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_syslog": "false", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_syslog_level": "info", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "cluster_addr": "-", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "cluster_network": "", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "cluster_network_interface": "", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "compressor_zlib_isal": "false", 2023-10-09T09:25:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "compressor_zlib_level": "5", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "compressor_zstd_level": "1", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "crimson_seastar_smp": "1", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "crush_location": "", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "crush_location_hook": "", 2023-10-09T09:25:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "crush_location_hook_timeout": "10", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "daemonize": "false", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "daos_pool": "tank", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_alienstore": "0/5", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_asok": "1/5", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_asok_assert_abort": "false", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_auth": "1/5", 2023-10-09T09:25:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_bdev": "1/3", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_bluefs": "1/5", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_bluestore": "1/5", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_buffer": "0/1", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_cephsqlite": "0/5", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_client": "0/5", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_compressor": "1/5", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_context": "0/1", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_crush": "1/1", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_crypto": "1/5", 2023-10-09T09:25:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_cyanstore": "0/5", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_dpdk": "1/5", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_eventtrace": "1/5", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_filer": "0/1", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_filestore": "1/3", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_finisher": "1/1", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_fuse": "1/5", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_javaclient": "1/5", 2023-10-09T09:25:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_journal": "1/3", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_journaler": "0/5", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_kstore": "1/5", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_leveldb": "4/5", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_lockdep": "0/1", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mclock": "1/5", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds": "1/5", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds_balancer": "1/5", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds_locker": "1/5", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds_log": "1/5", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds_migrator": "1/5", 2023-10-09T09:25:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_memstore": "1/5", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mgr": "2/5", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mgrc": "1/5", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mon": "20/20", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_monc": "0/10", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_ms": "1/1", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_none": "0/5", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_objclass": "0/5", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_objectcacher": "0/5", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_objecter": "0/1", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_optracker": "0/5", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_osd": "1/5", 2023-10-09T09:25:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_paxos": "20/20", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_perfcounter": "1/5", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_prioritycache": "1/5", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rados": "0/5", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rbd": "0/5", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rbd_replay": "0/5", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_refs": "0/0", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_reserver": "1/1", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw": "1/5", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw_access": "1/5", 2023-10-09T09:25:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw_flight": "1/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw_sync": "1/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rocksdb": "4/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore": "0/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_backref": "0/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_cache": "0/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_device": "0/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_epm": "0/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T09:25:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_journal": "0/5", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_lba": "0/5", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_odata": "0/5", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_omap": "0/5", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_onode": "0/5", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_t": "0/5", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_tm": "0/5", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_striper": "0/1", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_test": "0/5", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_throttle": "1/1", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_timer": "0/1", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_tp": "0/5", 2023-10-09T09:25:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "device_failure_prediction_mode": "none", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "err_to_graylog": "false", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "err_to_journald": "false", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "err_to_stderr": "true", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "err_to_syslog": "false", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event_tracing": "false", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_http_port": "9926", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_prio_limit": "5", 2023-10-09T09:25:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_sort_metrics": "true", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_stats_period": "5", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "fake_statfs_for_testing": "0", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "fatal_signal_handlers": "true", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "filer_max_purge_ops": "10", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "filer_max_truncate_ops": "128", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_blackhole": "false", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_btrfs_snap": "true", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_caller_concurrency": "10", 2023-10-09T09:25:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_debug_omap_check": "false", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_debug_verify_split": "false", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_dump_file": "", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fadvise": "true", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fail_eio": "true", 2023-10-09T09:25:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fd_cache_size": "128", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fiemap": "false", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_inject_stall": "0", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_journal_parallel": "false", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_journal_trailing": "false", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_journal_writeahead": "false", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_kill_at": "0", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T09:25:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T09:25:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_merge_threshold": "-10", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_odsync_write": "false", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_omap_backend_path": "", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_op_threads": "2", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_punch_hole": "false", 2023-10-09T09:25:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_max_ops": "50", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T09:25:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_seek_data_hole": "false", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_sloppy_crc": "false", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_splice": "false", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_split_multiple": "2", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_split_rand_factor": "20", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_update_to": "1000", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T09:25:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_xfs_extsize": "false", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_zfs_snap": "false", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "fio_dir": "/tmp/fio", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "fsid": "097c0480-6681-11ee-8db5-212e2dc638e7", 2023-10-09T09:25:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_allow_other": "true", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_big_writes": "true", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_debug": "false", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_default_permissions": "false", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_disable_pagecache": "false", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_max_write": "0", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_multithreaded": "true", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_require_active_mds": "true", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_set_user_groups": "true", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_splice_move": "true", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_splice_read": "true", 2023-10-09T09:25:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_splice_write": "true", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "gss_target_name": "ceph", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "heartbeat_file": "", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "heartbeat_inject_failure": "0", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "heartbeat_interval": "5", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "host": "smithi163", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T09:25:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "inject_early_sigterm": "false", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "jaeger_agent_port": "6799", 2023-10-09T09:25:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "jaeger_tracing_enable": "false", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_aio": "true", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_align_min_size": "65536", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_block_align": "true", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_block_size": "4096", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_dio": "true", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_discard": "false", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_force_aio": "false", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_ignore_corruption": "false", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_max_write_entries": "100", 2023-10-09T09:25:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_replay_from": "0", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_write_header_frequency": "0", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_zero_on_create": "false", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "journaler_prefetch_periods": "10", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "journaler_prezero_periods": "5", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "journaler_write_head_interval": "15", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "key": "", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "keyfile": "", 2023-10-09T09:25:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_backend": "rocksdb", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_max_bytes": "67108864", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_max_ops": "512", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_onode_map_size": "1024", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_sync_transaction": "false", 2023-10-09T09:25:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_block_size": "65536", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_bloom_size": "0", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_cache_size": "536870912", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_compression": "false", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_log": "", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_max_open_files": "0", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_paranoid": "false", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "librados_thread_count": "2", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "lockdep": "false", 2023-10-09T09:25:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "lockdep_force_backtrace": "false", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_coarse_timestamps": "true", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_flush_on_exit": "false", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_graylog_port": "12201", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_max_new": "1000", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_max_recent": "10000", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_stderr_prefix": "debug ", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_to_file": "true", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_to_graylog": "false", 2023-10-09T09:25:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "log_to_journald": "false", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "log_to_stderr": "false", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "log_to_syslog": "false", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_action_on_write_error": "1", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_alternate_name_max": "8192", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_asio_thread_count": "2", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_export_pin": "true", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T09:25:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_interval": "10", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_max": "-1", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_max_until": "-1", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_merge_size": "50", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_mode": "0", 2023-10-09T09:25:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_split_bits": "3", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_split_size": "10000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T09:25:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_mid": "0.700000", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_trim_interval": "1", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T09:25:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_debug_auth_pins": "false", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_debug_frag": "false", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_debug_scatterstat": "false", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_debug_subtrees": "false", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_default_dir_hash": "2", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_defer_session_stale": "true", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T09:25:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dir_max_entries": "0", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dir_prefetch": "true", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_early_reply": "true", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_enable_op_tracker": "true", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_enforce_unique_name": "true", 2023-10-09T09:25:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T09:25:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_health_dummy": "false", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_join_fs": "", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_journal_format": "1", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_create_at": "0", 2023-10-09T09:25:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_export_at": "0", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_import_at": "0", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_journal_at": "0", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_link_at": "0", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_openc_at": "0", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_rename_at": "0", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_max_events": "-1", 2023-10-09T09:25:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_max_segments": "128", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_pause": "false", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_segment_size": "0", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_completed_requests": "100000", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_export_size": "20971520", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_file_recover": "32", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_purge_files": "64", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_purge_ops": "8192", 2023-10-09T09:25:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_metrics_update_interval": "2", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_min_caps_per_client": "100", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_numa_node": "-1", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_op_history_duration": "600", 2023-10-09T09:25:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_op_history_size": "20", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_op_log_threshold": "5", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_ping_grace": "15", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_ping_interval": "5", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_max_caps": "30000", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T09:25:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_replay_interval": "1.000000", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_root_ino_gid": "0", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_root_ino_uid": "0", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T09:25:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_shutdown_check": "0", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_skip_ino": "0", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_snap_min_uid": "0", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_snap_rstat": "false", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_symlink_recovery": "true", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T09:25:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_thrash_exports": "0", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_thrash_fragments": "0", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_tick_interval": "5.000000", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_valgrind_exit": "false", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_verify_backtrace": "1", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_verify_scatter": "false", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_wipe_sessions": "false", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "mempool_debug": "false", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "memstore_page_set": "false", 2023-10-09T09:25:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "memstore_page_size": "65536", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_client_bytes": "134217728", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_client_messages": "512", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_disabled_modules": "", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_mds_messages": "128", 2023-10-09T09:25:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_mon_messages": "128", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_osd_messages": "8192", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_pool": "true", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_standby_modules": "true", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_stats_period": "5", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_stats_threshold": "5", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_tick_period": "2", 2023-10-09T09:25:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_allow_pool_delete": "true", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_bytes": "104857600", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T09:25:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T09:25:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_journald": "false", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_stderr": "true", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_compact_on_start": "false", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_compact_on_trim": "true", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T09:25:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cpu_threads": "4", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_data_avail_crit": "5", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_data_avail_warn": "5", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_dump_json": "false", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T09:25:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_extra_checks": "false", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_election_default_strategy": "3", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_election_timeout": "5.000000", 2023-10-09T09:25:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_enable_op_tracker": "true", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_fake_pool_delete": "false", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_force_quorum_join": "false", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_log_update_period": "5", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_max_detail": "50", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_to_clog": "true", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T09:25:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_host": "[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_host_override": "", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_initial_members": "", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_lease": "5.000000", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T09:25:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_log_full_interval": "50", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_log_max": "10000", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_log_max_summary": "50", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_log_epochs": "500", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_osd": "10000", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T09:25:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_memory_autotune": "true", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_memory_target": "2147483648", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_digest_period": "5", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T09:25:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_complaint_time": "30", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_history_duration": "600", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_history_size": "20", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_log_threshold": "5", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T09:25:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_cache_size": "500", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T09:25:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_report_timeout": "900", 2023-10-09T09:25:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T09:25:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_reweight_max_osds": "4", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T09:25:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_scrub_interval": "86400", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_scrub_max_keys": "100", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_scrub_timeout": "300", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_session_timeout": "300", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_smart_report_timeout": "5", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_stretch_pool_size": "4", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_debug": "false", 2023-10-09T09:25:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_tick_interval": "5", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T09:25:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_older_version": "true", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T09:25:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "monmap": "", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T09:25:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_tracing_enabled": "false", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_op_threads": "3", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_cm": "false", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_device_name": "", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T09:25:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_sl": "3", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_type": "ib", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_reap_threshold": "5", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_before_connect": "false", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_ipv4": "true", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_ipv6": "false", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_msgr1": "true", 2023-10-09T09:25:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_msgr2": "true", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_port_max": "7568", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_port_min": "6800", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_retry_count": "3", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_retry_delay": "5", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_blackhole_client": "false", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_blackhole_mds": "false", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_blackhole_mgr": "false", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_blackhole_mon": "false", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_blackhole_osd": "false", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_client_mode": "crc secure", 2023-10-09T09:25:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_cluster_type": "", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_compress_secure": "false", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_crc_data": "true", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_crc_header": "true", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_die_on_bug": "true", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_die_on_old_message": "true", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T09:25:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_hugepages": "", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_lro": "true", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T09:25:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_pmd": "", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_port_id": "0", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dump_on_send": "false", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_delay_type": "", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_network_congestion": "0", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_socket_failures": "0", 2023-10-09T09:25:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_max_accept_failures": "4", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_max_backoff": "15.000000", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_osd_compress_mode": "none", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_pq_min_cost": "65536", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_public_type": "", 2023-10-09T09:25:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_service_mode": "crc secure", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_tcp_nodelay": "true", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_type": "async+posix", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "no_config_file": "false", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_inflight_ops": "1024", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T09:25:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_timeout": "10.000000", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "objectstore_blackhole": "false", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "openssl_engine_opts": "", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_max_ops": "4", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_slop": "0.020000", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T09:25:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_asio_thread_count": "2", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backfill_scan_max": "512", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backfill_scan_min": "64", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backoff_on_peering": "false", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_beacon_report_interval": "300", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_bench_duration": "30", 2023-10-09T09:25:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_blkin_trace_all": "false", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T09:25:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_class_update_on_start": "true", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_client_message_cap": "256", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_client_op_priority": "63", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_client_watch_timeout": "30", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_command_max_records": "256", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_compact_on_start": "false", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_crush_update_on_start": "true", 2023-10-09T09:25:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T09:25:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_op_order": "false", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_shutdown": "false", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T09:25:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_default_notify_timeout": "30", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T09:25:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_enable_op_tracker": "true", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_fast_info": "true", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_fast_shutdown": "true", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T09:25:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_function_tracing": "false", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_grace": "20", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_interval": "6", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_stale": "600", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T09:25:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_journal_size": "5120", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_kill_backfill_at": "0", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_map_cache_size": "50", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_map_dedup": "true", 2023-10-09T09:25:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_map_message_max": "40", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_attr_name_len": "100", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_attr_size": "0", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_backfills": "1", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_markdown_count": "5", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_markdown_period": "600", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_object_name_len": "2048", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_object_size": "134217728", 2023-10-09T09:25:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_pgls": "1024", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_push_cost": "8388608", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_push_objects": "10", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_scrubs": "1", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_write_size": "90", 2023-10-09T09:25:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_profile": "balanced", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T09:25:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_base": "805306368", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T09:25:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_target": "4294967296", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_target_autotune": "false", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_min_recovery_priority": "0", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mon_report_interval": "5", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_num_cache_shards": "32", 2023-10-09T09:25:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_numa_node": "-1", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_objecter_finishers": "1", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_objectstore": "bluestore", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_objectstore_fuse": "false", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_objectstore_tracing": "false", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_history_duration": "600", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_history_size": "20", 2023-10-09T09:25:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_log_threshold": "5", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_shards": "0", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T09:25:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_thread_timeout": "15", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_open_classes_on_start": "true", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_os_flags": "0", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_peering_op_priority": "255", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_delete_priority": "5", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T09:25:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_crimson": "false", 2023-10-09T09:25:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flags": "0", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_min_size": "0", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T09:25:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_size": "3", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_type": "replicated", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_cost": "20971520", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T09:25:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_active": "0", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_op_priority": "3", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_priority": "5", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T09:25:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_repair_during_recovery": "false", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T09:25:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_cost": "52428800", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_end_hour": "0", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T09:25:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_priority": "5", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_skip_data_digest": "false", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_smart_report_timeout": "5", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T09:25:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_priority": "5", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_target_transaction_size": "30", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T09:25:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tracing": "false", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_use_stale_snap": "false", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_kill_at": "0", 2023-10-09T09:25:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_max_join_drift": "10", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_min": "500", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_min_wait": "0.050000", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_service_trim_max": "500", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_service_trim_min": "250", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_stash_full_interval": "25", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_trim_max": "500", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_trim_min": "250", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "perf": "true", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "pid_file": "", 2023-10-09T09:25:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "public_addr": "-", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "public_addrv": "", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "public_bind_addr": "-", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "public_network": "172.21.0.0/20,172.21.15.254/32", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "public_network_interface": "", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "qat_compressor_enabled": "false", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "rados_mon_op_timeout": "0", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "rados_osd_op_timeout": "0", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "rados_tracing": "false", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_atime_update_interval": "60", 2023-10-09T09:25:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache": "true", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_policy": "writearound", 2023-10-09T09:25:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_size": "33554432", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_compression_hint": "none", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_clone_format": "auto", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_data_pool": "", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_features": "61", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_format": "2", 2023-10-09T09:25:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_map_options": "", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_order": "22", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_pool": "rbd", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_stripe_count": "0", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_io_scheduler": "simple", 2023-10-09T09:25:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_order": "24", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_pool": "", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_splay_width": "4", 2023-10-09T09:25:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T09:25:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T09:25:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T09:25:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_op_threads": "1", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_plugins": "", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T09:25:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T09:25:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_tracing": "false", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_validate_names": "true", 2023-10-09T09:25:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_validate_pool": "true", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "restapi_base_url": "", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "restapi_log_level": "", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_admin_entry": "admin", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_backend_store": "rados", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_barbican_url": "", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_beast_enable_async": "true", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T09:25:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_cache_enabled": "true", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_config_store": "rados", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_content_length_compat": "false", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T09:25:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_cross_domain_policy": "", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T09:25:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T09:25:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T09:25:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T09:25:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_log_window": "30", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T09:25:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T09:25:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dns_name": "", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dns_s3website_name": "", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T09:25:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_ops_log": "false", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_static_website": "false", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_usage_log": "false", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_expose_bucket": "false", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_extended_http_attrs": "", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_filter": "none", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T09:25:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_objs": "32", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T09:25:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_init_timeout": "300", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_password": "", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_project": "", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_token": "", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T09:25:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_user": "", 2023-10-09T09:25:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_api_version": "2", 2023-10-09T09:25:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T09:25:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T09:25:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T09:25:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T09:25:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T09:25:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T09:25:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_url": "", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_max_objs": "32", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_max_worker": "3", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T09:25:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_log_http_headers": "", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T09:25:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_attr_size": "0", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_listing_results": "1000", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_notify_retries": "10", 2023-10-09T09:25:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T09:25:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T09:25:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_num_control_oids": "8", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_numa_node": "-1", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T09:25:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_op_tracing": "false", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_opa_token": "", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_opa_url": "", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ops_log_rados": "false", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T09:25:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_print_continue": "true", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_rados_tracing": "false", 2023-10-09T09:25:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_realm": "", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_realm_id": "", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_region": "", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_request_uri": "", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T09:25:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_resolve_cname": "false", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_run_sync_thread": "true", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T09:25:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_script_uri": "", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_service_provider_name": "", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_client_id": "", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_client_secret": "", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_entry": "sts", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_key": "sts", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T09:25:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_auth_url": "", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_custom_header": "", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_need_stats": "true", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_tenant_name": "", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_url": "", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T09:25:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_lease_period": "120", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T09:25:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_thread_pool_size": "512", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_comment": "", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_createby": "", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_encoding": "", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_flag": "false", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_origin": "", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_tracker": "", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T09:25:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_usage_max_shards": "32", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_use_opa_authz": "false", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_unique_email": "true", 2023-10-09T09:25:00.539 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_verify_ssl": "true", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zone": "", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zone_id": "", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zonegroup": "", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zonegroup_id": "", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_block_size": "4096", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T09:25:00.540 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T09:25:00.541 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_partition_filters": "false", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_perf": "false", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "run_dir": "/var/run/ceph", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_block_create": "true", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T09:25:00.542 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_device_size": "53687091200", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_main_device_type": "SSD", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T09:25:00.543 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_segment_size": "67108864", 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout: "set_keepcaps": "false", 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout: "setgroup": "ceph", 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout: "setuser": "ceph", 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout: "setuser_match_path": "", 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout: "thp": "false", 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout: "threadpool_default_timeout": "60", 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout: "throttler_perf_counter": "true" 2023-10-09T09:25:00.544 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:25:00.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:00 smithi163 bash[78107]: cluster 2023-10-09T09:24:59.566622+0000 mgr.y (mgr.34107) 558 : cluster 0 pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:00.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:00 smithi163 bash[75425]: cluster 2023-10-09T09:24:59.566622+0000 mgr.y (mgr.34107) 558 : cluster 0 pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:00 smithi181 bash[59345]: cluster 2023-10-09T09:24:59.566622+0000 mgr.y (mgr.34107) 558 : cluster 0 pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:01.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T09:25:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:02 smithi163 bash[78107]: cluster 2023-10-09T09:25:01.567645+0000 mgr.y (mgr.34107) 559 : cluster 0 pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:02 smithi163 bash[75425]: cluster 2023-10-09T09:25:01.567645+0000 mgr.y (mgr.34107) 559 : cluster 0 pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:02 smithi181 bash[59345]: cluster 2023-10-09T09:25:01.567645+0000 mgr.y (mgr.34107) 559 : cluster 0 pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:25:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:25:03] "GET /metrics HTTP/1.1" 200 33901 "" "Prometheus/2.33.4" 2023-10-09T09:25:04.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:04 smithi163 bash[78107]: cluster 2023-10-09T09:25:03.568207+0000 mgr.y (mgr.34107) 560 : cluster 0 pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:04.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:04 smithi163 bash[75425]: cluster 2023-10-09T09:25:03.568207+0000 mgr.y (mgr.34107) 560 : cluster 0 pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:05.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:04 smithi181 bash[59345]: cluster 2023-10-09T09:25:03.568207+0000 mgr.y (mgr.34107) 560 : cluster 0 pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:05.887 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:06 smithi163 bash[75425]: cluster 2023-10-09T09:25:05.569041+0000 mgr.y (mgr.34107) 561 : cluster 0 pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:06 smithi163 bash[78107]: cluster 2023-10-09T09:25:05.569041+0000 mgr.y (mgr.34107) 561 : cluster 0 pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:06 smithi181 bash[59345]: cluster 2023-10-09T09:25:05.569041+0000 mgr.y (mgr.34107) 561 : cluster 0 pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:08.580 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a: {} 2023-10-09T09:25:08.596 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b: {} 2023-10-09T09:25:08.608 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c: {} 2023-10-09T09:25:08.610 INFO:teuthology.orchestra.run.smithi163.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T09:25:08.610 INFO:teuthology.orchestra.run.smithi163.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T09:25:08.610 INFO:teuthology.orchestra.run.smithi163.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T09:25:08.625 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:08 smithi163 bash[75425]: cluster 2023-10-09T09:25:07.570037+0000 mgr.y (mgr.34107) 562 : cluster 0 pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:08.625 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:08 smithi163 bash[75425]: audit 2023-10-09T09:25:08.567265+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.163:0/2207044400' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:25:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:08 smithi163 bash[78107]: cluster 2023-10-09T09:25:07.570037+0000 mgr.y (mgr.34107) 562 : cluster 0 pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:08.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:08 smithi163 bash[78107]: audit 2023-10-09T09:25:08.567265+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.163:0/2207044400' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:25:09.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:08 smithi181 bash[59345]: cluster 2023-10-09T09:25:07.570037+0000 mgr.y (mgr.34107) 562 : cluster 0 pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:09.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:08 smithi181 bash[59345]: audit 2023-10-09T09:25:08.567265+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.163:0/2207044400' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:25:09.350 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T09:25:09.350 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T09:25:09.351 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:25:09.352 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:25:09.353 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T09:25:09.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T09:25:09.353 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T09:25:09.353 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:25:09.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T09:25:09.354 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T09:25:09.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:09.354 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T09:25:09.355 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:25:09.355 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T09:25:09.355 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:25:09.356 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T09:25:09.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:25:09.389 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:09.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:09.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:09.519 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:09.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:09.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:09.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:09.645 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:09.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:09.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:09.748 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:09.781 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:09.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:09.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:09.886 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:09.922 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:09.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:10.026 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:10.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:10.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:10.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:10.193 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:10.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:10.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:10.330 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:10.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:10.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:10.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:10.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:10.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:10.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:10.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:10.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:10.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:10.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:10.693 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:10 smithi163 bash[78107]: cluster 2023-10-09T09:25:09.570603+0000 mgr.y (mgr.34107) 563 : cluster 0 pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:10.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:10 smithi163 bash[78107]: audit 2023-10-09T09:25:10.359007+0000 mon.a (mon.0) 1352 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:10 smithi163 bash[75425]: cluster 2023-10-09T09:25:09.570603+0000 mgr.y (mgr.34107) 563 : cluster 0 pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:10.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:10 smithi163 bash[75425]: audit 2023-10-09T09:25:10.359007+0000 mon.a (mon.0) 1352 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:10.723 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:10.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:10.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:10.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:10.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:10.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:10.961 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:10.992 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:10.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:10.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:10 smithi181 bash[59345]: cluster 2023-10-09T09:25:09.570603+0000 mgr.y (mgr.34107) 563 : cluster 0 pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:10 smithi181 bash[59345]: audit 2023-10-09T09:25:10.359007+0000 mon.a (mon.0) 1352 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:11.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:11.127 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:11.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:11.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:11.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:11.259 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:11.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:11.362 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:11.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:11.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:11.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:11.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:11.531 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:11.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:11.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:11.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:11.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:11.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:11.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:11.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:11.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:11.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:11.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:11.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:11.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:11.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:12.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:12.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:12.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:12.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:12.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:12.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:12.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:12.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:12.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:12.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:12.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:12.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:12.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:12.480 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:12.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:12.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:12.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:12.616 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:12.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:12.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:12.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:12.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:12.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:12.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:12.855 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:12.887 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:12.889 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:12 smithi163 bash[78107]: cluster 2023-10-09T09:25:11.571511+0000 mgr.y (mgr.34107) 564 : cluster 0 pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:12.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:12 smithi163 bash[75425]: cluster 2023-10-09T09:25:11.571511+0000 mgr.y (mgr.34107) 564 : cluster 0 pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:12.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:12.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:12.991 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:13.024 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:13.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:13.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:13.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:12 smithi181 bash[59345]: cluster 2023-10-09T09:25:11.571511+0000 mgr.y (mgr.34107) 564 : cluster 0 pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:13.128 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:13.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:13.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:13.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:13.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:13.297 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:13.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:13.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:13.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:13.437 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:13.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:13.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:13.571 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:13.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:13.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:13.674 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:13.685 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:25:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:25:13] "GET /metrics HTTP/1.1" 200 33902 "" "Prometheus/2.33.4" 2023-10-09T09:25:13.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:13.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:13.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:13.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:13.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:13.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:13.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:13.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:13.974 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:13.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:13.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:14.077 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:14.109 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:14.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:14.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:14.146 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:14.146 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:14.147 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:14.148 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:14.148 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:14.213 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:14.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:14.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:14.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:14.345 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:14.371 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:14.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:14.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:14.474 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:14.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:14.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:14.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:14.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:14.639 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:14.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:14.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:14.743 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:14.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:14.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:14.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:14.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:14.908 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:14.909 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:14 smithi181 bash[59345]: cluster 2023-10-09T09:25:13.572091+0000 mgr.y (mgr.34107) 565 : cluster 0 pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:14.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:14.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:14.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:14 smithi163 bash[75425]: cluster 2023-10-09T09:25:13.572091+0000 mgr.y (mgr.34107) 565 : cluster 0 pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:14.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:14 smithi163 bash[78107]: cluster 2023-10-09T09:25:13.572091+0000 mgr.y (mgr.34107) 565 : cluster 0 pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:15.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:15.045 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:15.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:15.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:15.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:15.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:15.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:15.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:15.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:15.319 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:15.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:15.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:15.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:15.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:15.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:15.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:15.558 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:15.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:15.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:15.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:15.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:15.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:15.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:15.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:15.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:15.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:15.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:15.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:15.961 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:16.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:16.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:16.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:16.333 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:16.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:16.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:16.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:16.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:16.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:16.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:16.577 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:16 smithi163 bash[78107]: cluster 2023-10-09T09:25:15.572934+0000 mgr.y (mgr.34107) 566 : cluster 0 pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:16.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:16 smithi163 bash[75425]: cluster 2023-10-09T09:25:15.572934+0000 mgr.y (mgr.34107) 566 : cluster 0 pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:16.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:16.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:16.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:16.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:16.719 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:16.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:16.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:16 smithi181 bash[59345]: cluster 2023-10-09T09:25:15.572934+0000 mgr.y (mgr.34107) 566 : cluster 0 pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:16.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:16.854 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:16.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:16.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:16.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:16.988 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:16.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:16.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:17.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:17.123 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:17.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:17.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:17.227 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:17.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:17.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:17.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:17.355 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:17.385 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:17.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:17.488 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:17.519 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:17.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:17.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:17.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:17.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:17.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:17.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:17.752 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:17.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:17.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:17.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:17.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:17.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:17.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:17.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:18.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:18.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:18.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:18.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:18.139 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:18.168 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:18.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:18.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:18.272 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:18.308 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:18.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:18.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:18.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:18.445 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:18.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:18.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:18.549 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:18.577 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:18.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:18.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:18.680 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:18.710 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:18.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:18.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:18.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:18.844 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:18.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:18.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:18.948 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:18.962 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:18 smithi181 bash[59345]: cluster 2023-10-09T09:25:17.573776+0000 mgr.y (mgr.34107) 567 : cluster 0 pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:18.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:18 smithi163 bash[75425]: cluster 2023-10-09T09:25:17.573776+0000 mgr.y (mgr.34107) 567 : cluster 0 pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:18 smithi163 bash[78107]: cluster 2023-10-09T09:25:17.573776+0000 mgr.y (mgr.34107) 567 : cluster 0 pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:18.983 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:18.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:18.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:19.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:19.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:19.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:19.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:19.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:19.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:19.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:19.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:19.347 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:19.377 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:19.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:19.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:19.480 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:19.513 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:19.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:19.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:19.618 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:19.644 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:19.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:19.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:19.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:19.779 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:19.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:19.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:19.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:19.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:19.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:20.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:20.039 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:20.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:20.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:20.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:20.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:20.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:20.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:20.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:20.306 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:20.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:20.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:20.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:20.441 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:20.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:20.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:20.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:20.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:20.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:20.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:20.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:20.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:20 smithi163 bash[78107]: cluster 2023-10-09T09:25:19.574431+0000 mgr.y (mgr.34107) 568 : cluster 0 pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:20.695 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:20 smithi163 bash[75425]: cluster 2023-10-09T09:25:19.574431+0000 mgr.y (mgr.34107) 568 : cluster 0 pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:20.710 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:20.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:20.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:20.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:20 smithi181 bash[59345]: cluster 2023-10-09T09:25:19.574431+0000 mgr.y (mgr.34107) 568 : cluster 0 pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:20.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:20.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:20.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:20.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:20.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:20.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:21.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:21.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:21.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:21.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:21.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:21.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:21.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:21.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:21.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:21.377 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:21.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:21.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:21.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:21.507 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:21.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:21.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:21.610 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:21.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:21.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:21.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:21.734 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:21.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:21.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:21.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:21.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:21.879 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:21.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:21.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:21.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:22.004 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:22.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:22.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:22.143 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:22.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:22.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:22.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:22.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:22.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:22.380 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:22.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:22.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:22.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:22.508 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:22.540 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:22.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:22.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:22.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:22.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:22 smithi163 bash[75425]: cluster 2023-10-09T09:25:21.575052+0000 mgr.y (mgr.34107) 569 : cluster 0 pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:22.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:22 smithi163 bash[75425]: audit 2023-10-09T09:25:22.084969+0000 mon.a (mon.0) 1353 : audit 1 from='client.? 172.21.15.163:0/1598284275' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:22.647 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T09:25:22.672 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:22.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:22.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:22.777 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:22.797 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:22.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:22.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:22 smithi181 bash[59345]: cluster 2023-10-09T09:25:21.575052+0000 mgr.y (mgr.34107) 569 : cluster 0 pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:22 smithi181 bash[59345]: audit 2023-10-09T09:25:22.084969+0000 mon.a (mon.0) 1353 : audit 1 from='client.? 172.21.15.163:0/1598284275' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:22.899 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:22.913 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:22 smithi163 bash[78107]: cluster 2023-10-09T09:25:21.575052+0000 mgr.y (mgr.34107) 569 : cluster 0 pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:22.913 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:22 smithi163 bash[78107]: audit 2023-10-09T09:25:22.084969+0000 mon.a (mon.0) 1353 : audit 1 from='client.? 172.21.15.163:0/1598284275' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:25:22.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:22.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:22.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:23.034 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:23.067 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:23.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:23.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:23.172 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:23.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:23.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:23.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:23.296 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:23.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:23.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:23.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:23.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:23.453 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:23.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:23.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:23.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:23.591 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:23.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:23.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:23.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:25:23] "GET /metrics HTTP/1.1" 200 33902 "" "Prometheus/2.33.4" 2023-10-09T09:25:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[75425]: cluster 2023-10-09T09:25:22.637857+0000 osd.5 (osd.5) 617 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[75425]: audit 2023-10-09T09:25:22.640751+0000 mon.a (mon.0) 1354 : audit 1 from='client.? 172.21.15.163:0/1598284275' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:25:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[75425]: cluster 2023-10-09T09:25:22.640845+0000 mon.a (mon.0) 1355 : cluster 0 osdmap e1075: 8 total, 8 up, 8 in 2023-10-09T09:25:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[75425]: cluster 2023-10-09T09:25:22.765817+0000 osd.7 (osd.7) 471 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:23.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[75425]: cluster 2023-10-09T09:25:22.765997+0000 osd.4 (osd.4) 511 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[78107]: cluster 2023-10-09T09:25:22.637857+0000 osd.5 (osd.5) 617 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[78107]: audit 2023-10-09T09:25:22.640751+0000 mon.a (mon.0) 1354 : audit 1 from='client.? 172.21.15.163:0/1598284275' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:25:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[78107]: cluster 2023-10-09T09:25:22.640845+0000 mon.a (mon.0) 1355 : cluster 0 osdmap e1075: 8 total, 8 up, 8 in 2023-10-09T09:25:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[78107]: cluster 2023-10-09T09:25:22.765817+0000 osd.7 (osd.7) 471 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:23.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:23 smithi163 bash[78107]: cluster 2023-10-09T09:25:22.765997+0000 osd.4 (osd.4) 511 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:23.727 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:23.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:23.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:23.835 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:23.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:23 smithi181 bash[59345]: cluster 2023-10-09T09:25:22.637857+0000 osd.5 (osd.5) 617 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:23.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:23 smithi181 bash[59345]: audit 2023-10-09T09:25:22.640751+0000 mon.a (mon.0) 1354 : audit 1 from='client.? 172.21.15.163:0/1598284275' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:25:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:23 smithi181 bash[59345]: cluster 2023-10-09T09:25:22.640845+0000 mon.a (mon.0) 1355 : cluster 0 osdmap e1075: 8 total, 8 up, 8 in 2023-10-09T09:25:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:23 smithi181 bash[59345]: cluster 2023-10-09T09:25:22.765817+0000 osd.7 (osd.7) 471 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:23.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:23 smithi181 bash[59345]: cluster 2023-10-09T09:25:22.765997+0000 osd.4 (osd.4) 511 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:23.867 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:23.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:23.970 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:24.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:24.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:24.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:24.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T09:25:24.105 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:24.165 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:24.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:24.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:24.221 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:25:24.244 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:25:24.267 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:25:24.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:24.287 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:24.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:24.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:24.295 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:25:24.319 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:25:24.346 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:25:24.371 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:25:24.392 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:24.395 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:25:24.397 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:25:24.397 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:25:24.397 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:25:24.397 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:25:24.397 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:25:24.397 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:25:24.398 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:25:24.398 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:25:24.411 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:24.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:24.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:24.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:24.518 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:25:24.519 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:25:24.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:24.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:24.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:24.684 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:24.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:24.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:24.789 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:24.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:24.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:24.898 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_1' created 2023-10-09T09:25:24.898 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:25:24.920 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:24.939 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:24.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:24.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:24.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:24 smithi163 bash[78107]: cluster 2023-10-09T09:25:22.765719+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:24.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:24 smithi163 bash[78107]: cluster 2023-10-09T09:25:23.575496+0000 mgr.y (mgr.34107) 570 : cluster 0 pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:24.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:24 smithi163 bash[78107]: audit 2023-10-09T09:25:24.200483+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.163:0/3327333011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:25:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:24 smithi163 bash[75425]: cluster 2023-10-09T09:25:22.765719+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:24 smithi163 bash[75425]: cluster 2023-10-09T09:25:23.575496+0000 mgr.y (mgr.34107) 570 : cluster 0 pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:24 smithi163 bash[75425]: audit 2023-10-09T09:25:24.200483+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.163:0/3327333011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:25:25.042 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:25.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:25.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:25.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:25.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:25.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:24 smithi181 bash[59345]: cluster 2023-10-09T09:25:22.765719+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e1075 with expected crc 2023-10-09T09:25:25.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:24 smithi181 bash[59345]: cluster 2023-10-09T09:25:23.575496+0000 mgr.y (mgr.34107) 570 : cluster 0 pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:25.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:24 smithi181 bash[59345]: audit 2023-10-09T09:25:24.200483+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.163:0/3327333011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:25:25.199 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:25.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:25.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:25.302 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:25.332 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:25.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:25.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:25.469 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:25.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:25.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:25.573 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:25.604 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:25.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:25.707 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:25.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:25.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:25.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:25.872 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:25.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: audit 2023-10-09T09:25:24.368033+0000 mon.a (mon.0) 1356 : audit 1 from='client.? 172.21.15.163:0/1790987482' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T09:25:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: audit 2023-10-09T09:25:24.837212+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.163:0/3913800048' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:25:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: audit 2023-10-09T09:25:24.839728+0000 mon.a (mon.0) 1357 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:25:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: cluster 2023-10-09T09:25:24.877666+0000 mon.a (mon.0) 1358 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:25:25.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: cluster 2023-10-09T09:25:24.883788+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: cluster 2023-10-09T09:25:24.886603+0000 osd.5 (osd.5) 618 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: audit 2023-10-09T09:25:24.890966+0000 mon.a (mon.0) 1359 : audit 1 from='client.? 172.21.15.163:0/1790987482' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: audit 2023-10-09T09:25:24.891131+0000 mon.a (mon.0) 1360 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: cluster 2023-10-09T09:25:24.891206+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e1076: 8 total, 8 up, 8 in 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: cluster 2023-10-09T09:25:24.891273+0000 osd.4 (osd.4) 512 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: cluster 2023-10-09T09:25:24.893074+0000 osd.7 (osd.7) 472 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: cluster 2023-10-09T09:25:24.893854+0000 osd.4 (osd.4) 513 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: audit 2023-10-09T09:25:25.365655+0000 mon.a (mon.0) 1362 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: audit 2023-10-09T09:25:25.367479+0000 mon.a (mon.0) 1363 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[78107]: cluster 2023-10-09T09:25:25.576138+0000 mgr.y (mgr.34107) 571 : cluster 0 pgmap v1255: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: audit 2023-10-09T09:25:24.368033+0000 mon.a (mon.0) 1356 : audit 1 from='client.? 172.21.15.163:0/1790987482' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: audit 2023-10-09T09:25:24.837212+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.163:0/3913800048' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: audit 2023-10-09T09:25:24.839728+0000 mon.a (mon.0) 1357 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: cluster 2023-10-09T09:25:24.877666+0000 mon.a (mon.0) 1358 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: cluster 2023-10-09T09:25:24.883788+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: cluster 2023-10-09T09:25:24.886603+0000 osd.5 (osd.5) 618 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: audit 2023-10-09T09:25:24.890966+0000 mon.a (mon.0) 1359 : audit 1 from='client.? 172.21.15.163:0/1790987482' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: audit 2023-10-09T09:25:24.891131+0000 mon.a (mon.0) 1360 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: cluster 2023-10-09T09:25:24.891206+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e1076: 8 total, 8 up, 8 in 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: cluster 2023-10-09T09:25:24.891273+0000 osd.4 (osd.4) 512 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: cluster 2023-10-09T09:25:24.893074+0000 osd.7 (osd.7) 472 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: cluster 2023-10-09T09:25:24.893854+0000 osd.4 (osd.4) 513 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: audit 2023-10-09T09:25:25.365655+0000 mon.a (mon.0) 1362 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:25.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: audit 2023-10-09T09:25:25.367479+0000 mon.a (mon.0) 1363 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:25.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:25 smithi163 bash[75425]: cluster 2023-10-09T09:25:25.576138+0000 mgr.y (mgr.34107) 571 : cluster 0 pgmap v1255: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:25.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:25.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: audit 2023-10-09T09:25:24.368033+0000 mon.a (mon.0) 1356 : audit 1 from='client.? 172.21.15.163:0/1790987482' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T09:25:25.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: audit 2023-10-09T09:25:24.837212+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.163:0/3913800048' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:25:25.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: audit 2023-10-09T09:25:24.839728+0000 mon.a (mon.0) 1357 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:25:25.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: cluster 2023-10-09T09:25:24.877666+0000 mon.a (mon.0) 1358 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:25:25.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: cluster 2023-10-09T09:25:24.883788+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: cluster 2023-10-09T09:25:24.886603+0000 osd.5 (osd.5) 618 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: audit 2023-10-09T09:25:24.890966+0000 mon.a (mon.0) 1359 : audit 1 from='client.? 172.21.15.163:0/1790987482' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T09:25:25.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: audit 2023-10-09T09:25:24.891131+0000 mon.a (mon.0) 1360 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:25:25.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: cluster 2023-10-09T09:25:24.891206+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e1076: 8 total, 8 up, 8 in 2023-10-09T09:25:25.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: cluster 2023-10-09T09:25:24.891273+0000 osd.4 (osd.4) 512 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: cluster 2023-10-09T09:25:24.893074+0000 osd.7 (osd.7) 472 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: cluster 2023-10-09T09:25:24.893854+0000 osd.4 (osd.4) 513 : cluster 3 failed to encode map e1076 with expected crc 2023-10-09T09:25:25.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: audit 2023-10-09T09:25:25.365655+0000 mon.a (mon.0) 1362 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:25.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: audit 2023-10-09T09:25:25.367479+0000 mon.a (mon.0) 1363 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:25.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:25 smithi181 bash[59345]: cluster 2023-10-09T09:25:25.576138+0000 mgr.y (mgr.34107) 571 : cluster 0 pgmap v1255: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:26.012 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:26.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:26.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:26.116 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:26.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:26.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:26.246 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:26.272 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:26.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:26.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:26.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:26.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:26.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:26.505 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:26.536 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:26.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:26.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:26.616 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T09:25:26.639 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:26.670 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:26.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:26.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:26.774 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:26.805 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:26.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:26.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:26.824 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T09:25:26.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:26.938 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:26.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:27.042 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:27.053 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[75425]: cluster 2023-10-09T09:25:25.884304+0000 osd.5 (osd.5) 619 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.053 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[75425]: cluster 2023-10-09T09:25:25.884494+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[75425]: cluster 2023-10-09T09:25:25.887211+0000 osd.5 (osd.5) 620 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[75425]: cluster 2023-10-09T09:25:25.887568+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e1077: 8 total, 8 up, 8 in 2023-10-09T09:25:27.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[75425]: cluster 2023-10-09T09:25:25.887919+0000 osd.4 (osd.4) 514 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[75425]: cluster 2023-10-09T09:25:25.888110+0000 osd.7 (osd.7) 473 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[78107]: cluster 2023-10-09T09:25:25.884304+0000 osd.5 (osd.5) 619 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[78107]: cluster 2023-10-09T09:25:25.884494+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[78107]: cluster 2023-10-09T09:25:25.887211+0000 osd.5 (osd.5) 620 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[78107]: cluster 2023-10-09T09:25:25.887568+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e1077: 8 total, 8 up, 8 in 2023-10-09T09:25:27.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[78107]: cluster 2023-10-09T09:25:25.887919+0000 osd.4 (osd.4) 514 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:26 smithi163 bash[78107]: cluster 2023-10-09T09:25:25.888110+0000 osd.7 (osd.7) 473 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:26 smithi181 bash[59345]: cluster 2023-10-09T09:25:25.884304+0000 osd.5 (osd.5) 619 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:26 smithi181 bash[59345]: cluster 2023-10-09T09:25:25.884494+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:26 smithi181 bash[59345]: cluster 2023-10-09T09:25:25.887211+0000 osd.5 (osd.5) 620 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:26 smithi181 bash[59345]: cluster 2023-10-09T09:25:25.887568+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e1077: 8 total, 8 up, 8 in 2023-10-09T09:25:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:26 smithi181 bash[59345]: cluster 2023-10-09T09:25:25.887919+0000 osd.4 (osd.4) 514 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:26 smithi181 bash[59345]: cluster 2023-10-09T09:25:25.888110+0000 osd.7 (osd.7) 473 : cluster 3 failed to encode map e1077 with expected crc 2023-10-09T09:25:27.074 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:27.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:27.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:27.178 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:27.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:27.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:27.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:27.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:27.350 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:27.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:27.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:27.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:27.482 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:27.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:27.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:27.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:27.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:27.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:27.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:27.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:27.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:27.857 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:27.890 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:27.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:27.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:28.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:27 smithi163 bash[78107]: cluster 2023-10-09T09:25:26.894306+0000 osd.5 (osd.5) 621 : cluster 3 failed to encode map e1078 with expected crc 2023-10-09T09:25:28.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:27 smithi163 bash[78107]: cluster 2023-10-09T09:25:26.894348+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e1078 with expected crc 2023-10-09T09:25:28.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:27 smithi163 bash[78107]: cluster 2023-10-09T09:25:26.897684+0000 mon.a (mon.0) 1365 : cluster 0 osdmap e1078: 8 total, 8 up, 8 in 2023-10-09T09:25:28.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:27 smithi163 bash[78107]: cluster 2023-10-09T09:25:27.577016+0000 mgr.y (mgr.34107) 572 : cluster 0 pgmap v1258: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:28.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:27 smithi163 bash[75425]: cluster 2023-10-09T09:25:26.894306+0000 osd.5 (osd.5) 621 : cluster 3 failed to encode map e1078 with expected crc 2023-10-09T09:25:28.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:27 smithi163 bash[75425]: cluster 2023-10-09T09:25:26.894348+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e1078 with expected crc 2023-10-09T09:25:28.008 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:27 smithi163 bash[75425]: cluster 2023-10-09T09:25:26.897684+0000 mon.a (mon.0) 1365 : cluster 0 osdmap e1078: 8 total, 8 up, 8 in 2023-10-09T09:25:28.008 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:27 smithi163 bash[75425]: cluster 2023-10-09T09:25:27.577016+0000 mgr.y (mgr.34107) 572 : cluster 0 pgmap v1258: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:28.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:28.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:28.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:28.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:28.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:28.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:28.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:28.267 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:28.281 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:27 smithi181 bash[59345]: cluster 2023-10-09T09:25:26.894306+0000 osd.5 (osd.5) 621 : cluster 3 failed to encode map e1078 with expected crc 2023-10-09T09:25:28.281 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:27 smithi181 bash[59345]: cluster 2023-10-09T09:25:26.894348+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e1078 with expected crc 2023-10-09T09:25:28.281 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:27 smithi181 bash[59345]: cluster 2023-10-09T09:25:26.897684+0000 mon.a (mon.0) 1365 : cluster 0 osdmap e1078: 8 total, 8 up, 8 in 2023-10-09T09:25:28.281 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:27 smithi181 bash[59345]: cluster 2023-10-09T09:25:27.577016+0000 mgr.y (mgr.34107) 572 : cluster 0 pgmap v1258: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:28.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:28.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:28.408 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:28.441 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:28.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:28.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:28.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:25:28.543 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:28.571 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:28.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:28.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:28.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:28.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:28.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:28.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:28.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:25:28.811 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:28.846 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:28.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:28.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:28.950 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:28.981 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:28.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:28.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:29.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:29.119 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:29.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:29.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:29.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:29.252 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:29.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:29.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:29.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:29.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:29.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:29.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:29.487 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:29.519 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:29.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:29.622 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:29.635 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:29 smithi163 bash[78107]: cluster 2023-10-09T09:25:29.059631+0000 osd.7 (osd.7) 474 : cluster 3 failed to encode map e1078 with expected crc 2023-10-09T09:25:29.635 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:29 smithi163 bash[75425]: cluster 2023-10-09T09:25:29.059631+0000 osd.7 (osd.7) 474 : cluster 3 failed to encode map e1078 with expected crc 2023-10-09T09:25:29.650 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:29.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:29.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:29.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:29.783 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:29.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:29.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:29.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:29 smithi181 bash[59345]: cluster 2023-10-09T09:25:29.059631+0000 osd.7 (osd.7) 474 : cluster 3 failed to encode map e1078 with expected crc 2023-10-09T09:25:29.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:29.917 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:29.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:30.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:30.054 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:30.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:30.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:30.158 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:30.189 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:30.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:30.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:30.291 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:30.323 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:30.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:30.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:30.360 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:30.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:30.451 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:30.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:30.554 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:30.584 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:30.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:30.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:30.686 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:30.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:30 smithi163 bash[78107]: cluster 2023-10-09T09:25:29.577922+0000 mgr.y (mgr.34107) 573 : cluster 0 pgmap v1259: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:30.701 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:30 smithi163 bash[75425]: cluster 2023-10-09T09:25:29.577922+0000 mgr.y (mgr.34107) 573 : cluster 0 pgmap v1259: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:30.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:30.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:30.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:30.821 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:30.850 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:30.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:30.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:30.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:30.966 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:30 smithi181 bash[59345]: cluster 2023-10-09T09:25:29.577922+0000 mgr.y (mgr.34107) 573 : cluster 0 pgmap v1259: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:30.988 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:30.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:30.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:31.091 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:31.123 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:31.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:31.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:31.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:31.255 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:31.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:31.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:31.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:31.376 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:31.376 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:31.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:31.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:31.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:31.525 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:31.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:31.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:31.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:31.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:31.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:31.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:31.764 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:31.785 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:31.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:31.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:31.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:31.913 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:31.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:31.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:32.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:32.053 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:32.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:32.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:32.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:32.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:32.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:32.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:32.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:32.318 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:32.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:32.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:32.389 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:32.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:32.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:32.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:32.559 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:32.581 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:32.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:32.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:32.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:32.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:32.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:32.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:32 smithi181 bash[59345]: cluster 2023-10-09T09:25:31.578826+0000 mgr.y (mgr.34107) 574 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:32.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:32.828 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:32 smithi163 bash[75425]: cluster 2023-10-09T09:25:31.578826+0000 mgr.y (mgr.34107) 574 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:32.828 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:32 smithi163 bash[78107]: cluster 2023-10-09T09:25:31.578826+0000 mgr.y (mgr.34107) 574 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:32.845 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:32.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:32.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:32.949 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:32.977 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:32.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:32.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:33.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:33.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:33.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:33.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:33.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:33.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:33.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:33.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:33.342 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:33.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:33.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:33.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:33.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:33.493 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:25:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:25:33] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.33.4" 2023-10-09T09:25:33.509 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:33.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:33.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:33.639 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:33.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:33.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:33.744 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:33.775 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:33.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:33.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:33.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:33.909 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:33.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:33.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:34.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:34.045 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:34.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:34.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:34.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:34.174 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:34.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:34.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:34.277 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:34.311 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:34.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:34.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:34.414 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:34.445 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:34.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:34.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:34.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:34.587 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:34.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:34.691 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:34.723 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:34.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:34.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:34.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:34 smithi181 bash[59345]: cluster 2023-10-09T09:25:33.579350+0000 mgr.y (mgr.34107) 575 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:34.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:34.856 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:34.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:34.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:34.893 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:34 smithi163 bash[78107]: cluster 2023-10-09T09:25:33.579350+0000 mgr.y (mgr.34107) 575 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:34.894 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:34 smithi163 bash[75425]: cluster 2023-10-09T09:25:33.579350+0000 mgr.y (mgr.34107) 575 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:34.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:34.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:34.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:34.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:35.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:35.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:35.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:35.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:35.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:35.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:35.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:35.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:35.377 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:35.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:35.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:35.480 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:35.512 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:35.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:35.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:35.616 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:35.648 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:35.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:35.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:35.752 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:35.784 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:35.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:35.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:35.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:35.909 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:35.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:36.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:36.038 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:36.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:36.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:36.141 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:36.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:36.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:36.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:36.265 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:36.286 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:36.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:36.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:36.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:36.413 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:36.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:36.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:36.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:36.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:36 smithi163 bash[78107]: cluster 2023-10-09T09:25:35.580209+0000 mgr.y (mgr.34107) 576 : cluster 0 pgmap v1262: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:36.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:36 smithi163 bash[78107]: audit 2023-10-09T09:25:35.702291+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:36.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:36 smithi163 bash[75425]: cluster 2023-10-09T09:25:35.580209+0000 mgr.y (mgr.34107) 576 : cluster 0 pgmap v1262: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:36.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:36 smithi163 bash[75425]: audit 2023-10-09T09:25:35.702291+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:36.549 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:36.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:36.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:36.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:36.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:36.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:36.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:36.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:36.802 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:36.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:36.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:36 smithi181 bash[59345]: cluster 2023-10-09T09:25:35.580209+0000 mgr.y (mgr.34107) 576 : cluster 0 pgmap v1262: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:36 smithi181 bash[59345]: audit 2023-10-09T09:25:35.702291+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:36.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:36.925 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:36.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:36.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:37.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:37.065 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:37.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:37.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:37.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:37.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:37.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:37.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:37.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:37.327 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:37.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:37.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:37.430 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:37.462 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:37.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:37.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:37.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:37.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:37.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:37.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:37.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:37.728 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:37.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:37.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:37.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:37.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:37.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:37.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:37.959 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:37.994 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:37.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:37.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:38.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:38.130 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:38.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:38.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:38.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:38.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:38.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:38.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:38.371 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:38.399 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:38.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:38.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:38.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:38.533 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:38.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:38.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:38.636 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:38.658 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:38.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:38.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:38.762 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:38.789 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:38.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:38.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:38 smithi181 bash[59345]: cluster 2023-10-09T09:25:37.580928+0000 mgr.y (mgr.34107) 577 : cluster 0 pgmap v1263: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:38.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:38.915 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:38.916 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:38.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:38.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:38.938 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:38 smithi163 bash[75425]: cluster 2023-10-09T09:25:37.580928+0000 mgr.y (mgr.34107) 577 : cluster 0 pgmap v1263: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:38.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:38 smithi163 bash[78107]: cluster 2023-10-09T09:25:37.580928+0000 mgr.y (mgr.34107) 577 : cluster 0 pgmap v1263: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:39.018 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:39.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:39.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:39.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:39.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:39.188 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:39.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:39.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:39.292 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:39.323 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:39.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:39.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:39.427 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:39.458 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:39.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:39.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:39.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:39.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:39.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:39.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:39.659 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T09:25:39.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:39.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:39.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:39.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:39.815 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:39.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:39.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:39.881 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:39 smithi163 bash[75425]: audit 2023-10-09T09:25:39.119855+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.163:0/870157121' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:39.882 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:39 smithi163 bash[75425]: audit 2023-10-09T09:25:39.120201+0000 mon.a (mon.0) 1367 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:39.882 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:39 smithi163 bash[78107]: audit 2023-10-09T09:25:39.119855+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.163:0/870157121' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:39.882 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:39 smithi163 bash[78107]: audit 2023-10-09T09:25:39.120201+0000 mon.a (mon.0) 1367 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:39.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:39.964 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:39.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:39.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:39 smithi181 bash[59345]: audit 2023-10-09T09:25:39.119855+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.163:0/870157121' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:39 smithi181 bash[59345]: audit 2023-10-09T09:25:39.120201+0000 mon.a (mon.0) 1367 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:25:40.066 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:40.089 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:40.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:40.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:40.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:40.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:40.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:40.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:40.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:40.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:40.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:40.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:40.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:40.495 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:40.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:40.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:40.568 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:25:40.568 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:25:40.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:40.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:40.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:40.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:40.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:40.759 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:40.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:40.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:40 smithi181 bash[59345]: cluster 2023-10-09T09:25:39.581468+0000 mgr.y (mgr.34107) 578 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:40 smithi181 bash[59345]: cluster 2023-10-09T09:25:39.651624+0000 osd.5 (osd.5) 622 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:40 smithi181 bash[59345]: audit 2023-10-09T09:25:39.654796+0000 mon.a (mon.0) 1368 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:25:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:40 smithi181 bash[59345]: cluster 2023-10-09T09:25:39.654942+0000 mon.a (mon.0) 1369 : cluster 0 osdmap e1079: 8 total, 8 up, 8 in 2023-10-09T09:25:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:40 smithi181 bash[59345]: audit 2023-10-09T09:25:40.358973+0000 mon.a (mon.0) 1370 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:40.863 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:40.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:40.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:40.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:40 smithi163 bash[75425]: cluster 2023-10-09T09:25:39.581468+0000 mgr.y (mgr.34107) 578 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:40 smithi163 bash[75425]: cluster 2023-10-09T09:25:39.651624+0000 osd.5 (osd.5) 622 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:40 smithi163 bash[75425]: audit 2023-10-09T09:25:39.654796+0000 mon.a (mon.0) 1368 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:25:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:40 smithi163 bash[75425]: cluster 2023-10-09T09:25:39.654942+0000 mon.a (mon.0) 1369 : cluster 0 osdmap e1079: 8 total, 8 up, 8 in 2023-10-09T09:25:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:40 smithi163 bash[75425]: audit 2023-10-09T09:25:40.358973+0000 mon.a (mon.0) 1370 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:40 smithi163 bash[78107]: cluster 2023-10-09T09:25:39.581468+0000 mgr.y (mgr.34107) 578 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:40 smithi163 bash[78107]: cluster 2023-10-09T09:25:39.651624+0000 osd.5 (osd.5) 622 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:40 smithi163 bash[78107]: audit 2023-10-09T09:25:39.654796+0000 mon.a (mon.0) 1368 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:25:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:40 smithi163 bash[78107]: cluster 2023-10-09T09:25:39.654942+0000 mon.a (mon.0) 1369 : cluster 0 osdmap e1079: 8 total, 8 up, 8 in 2023-10-09T09:25:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:40 smithi163 bash[78107]: audit 2023-10-09T09:25:40.358973+0000 mon.a (mon.0) 1370 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:40.989 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:41.014 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:41.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:41.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:41.118 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:41.140 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:41.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:41.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:41.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:41.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:41.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:41.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:41.407 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:41.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:41.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:41.510 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:41.541 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:41.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:41.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:41.645 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:41.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:41.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:41.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:41.678 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T09:25:41.706 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T09:25:41.777 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:41.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:41 smithi163 bash[78107]: cluster 2023-10-09T09:25:41.003702+0000 osd.7 (osd.7) 475 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:41.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:41 smithi163 bash[78107]: audit 2023-10-09T09:25:41.108571+0000 mon.a (mon.0) 1371 : audit 1 from='client.? 172.21.15.163:0/2307474534' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:25:41.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:41 smithi163 bash[78107]: cluster 2023-10-09T09:25:41.190558+0000 osd.4 (osd.4) 515 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:41.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:41 smithi163 bash[78107]: cluster 2023-10-09T09:25:41.304392+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:41.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:41 smithi163 bash[75425]: cluster 2023-10-09T09:25:41.003702+0000 osd.7 (osd.7) 475 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:41.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:41 smithi163 bash[75425]: audit 2023-10-09T09:25:41.108571+0000 mon.a (mon.0) 1371 : audit 1 from='client.? 172.21.15.163:0/2307474534' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:25:41.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:41 smithi163 bash[75425]: cluster 2023-10-09T09:25:41.190558+0000 osd.4 (osd.4) 515 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:41.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:41 smithi163 bash[75425]: cluster 2023-10-09T09:25:41.304392+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:41.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:41 smithi181 bash[59345]: cluster 2023-10-09T09:25:41.003702+0000 osd.7 (osd.7) 475 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:41 smithi181 bash[59345]: audit 2023-10-09T09:25:41.108571+0000 mon.a (mon.0) 1371 : audit 1 from='client.? 172.21.15.163:0/2307474534' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:25:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:41 smithi181 bash[59345]: cluster 2023-10-09T09:25:41.190558+0000 osd.4 (osd.4) 515 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:41 smithi181 bash[59345]: cluster 2023-10-09T09:25:41.304392+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e1079 with expected crc 2023-10-09T09:25:41.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:41.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:41.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:41.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:41.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:41.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:42.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:42.085 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:42.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:42.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:42.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:42.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:42.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:42.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:42.314 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:42.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:42.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:42.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:42.439 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:42.467 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:42.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:42.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:42.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:42.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:42.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:42.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:42.627 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:25:42.628 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T09:25:42.647 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:25:42.671 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:25:42.693 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:25:42.696 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[78107]: cluster 2023-10-09T09:25:41.582167+0000 mgr.y (mgr.34107) 579 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[78107]: cluster 2023-10-09T09:25:41.669000+0000 osd.5 (osd.5) 623 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[78107]: audit 2023-10-09T09:25:41.671640+0000 mon.a (mon.0) 1372 : audit 1 from='client.? 172.21.15.163:0/2307474534' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:25:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[78107]: cluster 2023-10-09T09:25:41.671703+0000 mon.a (mon.0) 1373 : cluster 0 osdmap e1080: 8 total, 8 up, 8 in 2023-10-09T09:25:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[78107]: cluster 2023-10-09T09:25:42.041960+0000 osd.7 (osd.7) 476 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[78107]: cluster 2023-10-09T09:25:42.229643+0000 osd.4 (osd.4) 516 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[78107]: cluster 2023-10-09T09:25:42.329214+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[78107]: audit 2023-10-09T09:25:42.602991+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.163:0/1306081349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:25:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[75425]: cluster 2023-10-09T09:25:41.582167+0000 mgr.y (mgr.34107) 579 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[75425]: cluster 2023-10-09T09:25:41.669000+0000 osd.5 (osd.5) 623 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[75425]: audit 2023-10-09T09:25:41.671640+0000 mon.a (mon.0) 1372 : audit 1 from='client.? 172.21.15.163:0/2307474534' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:25:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[75425]: cluster 2023-10-09T09:25:41.671703+0000 mon.a (mon.0) 1373 : cluster 0 osdmap e1080: 8 total, 8 up, 8 in 2023-10-09T09:25:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[75425]: cluster 2023-10-09T09:25:42.041960+0000 osd.7 (osd.7) 476 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[75425]: cluster 2023-10-09T09:25:42.229643+0000 osd.4 (osd.4) 516 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[75425]: cluster 2023-10-09T09:25:42.329214+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:42 smithi163 bash[75425]: audit 2023-10-09T09:25:42.602991+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.163:0/1306081349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:25:42.724 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:25:42.731 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:42.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:42.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:42.744 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:25:42.766 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:25:42.788 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:25:42.789 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:25:42.789 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:25:42.790 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:25:42.790 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:25:42.790 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:25:42.790 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:25:42.790 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:25:42.790 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:25:42.834 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:42.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:42.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:42.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:42.974 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:42.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:42 smithi181 bash[59345]: cluster 2023-10-09T09:25:41.582167+0000 mgr.y (mgr.34107) 579 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:42.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:42 smithi181 bash[59345]: cluster 2023-10-09T09:25:41.669000+0000 osd.5 (osd.5) 623 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:42 smithi181 bash[59345]: audit 2023-10-09T09:25:41.671640+0000 mon.a (mon.0) 1372 : audit 1 from='client.? 172.21.15.163:0/2307474534' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:25:42.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:42 smithi181 bash[59345]: cluster 2023-10-09T09:25:41.671703+0000 mon.a (mon.0) 1373 : cluster 0 osdmap e1080: 8 total, 8 up, 8 in 2023-10-09T09:25:42.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:42 smithi181 bash[59345]: cluster 2023-10-09T09:25:42.041960+0000 osd.7 (osd.7) 476 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:42 smithi181 bash[59345]: cluster 2023-10-09T09:25:42.229643+0000 osd.4 (osd.4) 516 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:42 smithi181 bash[59345]: cluster 2023-10-09T09:25:42.329214+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e1080 with expected crc 2023-10-09T09:25:42.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:42 smithi181 bash[59345]: audit 2023-10-09T09:25:42.602991+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.163:0/1306081349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:25:43.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:43.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:43.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:43.104 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:43.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:43.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:43.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:43.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:43.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:43.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:43.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:43.397 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:43.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:43.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:43.501 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:43.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:43.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:43.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:43.534 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:25:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:25:43] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.33.4" 2023-10-09T09:25:43.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:43.670 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:43.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:43.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:43.807 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:43.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:43.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:43.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:43.923 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:43 smithi163 bash[78107]: cluster 2023-10-09T09:25:43.582671+0000 mgr.y (mgr.34107) 580 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:43.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:43 smithi163 bash[75425]: cluster 2023-10-09T09:25:43.582671+0000 mgr.y (mgr.34107) 580 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:43.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:43.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:44.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:44.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:43 smithi181 bash[59345]: cluster 2023-10-09T09:25:43.582671+0000 mgr.y (mgr.34107) 580 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:25:44.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:44.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:44.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:44.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:44.215 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:44.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:44.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:44.318 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:44.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:44.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:44.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:44.486 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:44.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:44.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:44.590 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:44.617 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:44.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:44.720 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:44.746 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:44.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:44.754 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T09:25:44.791 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:25:44.792 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:25:44.833 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T09:25:44.833 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:25:44.833 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_107771 2023-10-09T09:25:44.833 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:25:44.833 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:25:44.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:44.877 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:44.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:44.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:45.015 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:45.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:45.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:45.120 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:45.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:45.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:45.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:45.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:45.284 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:45.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:45.387 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:45.422 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:45.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:45.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:45.526 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:45.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:45.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:45.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:45.670 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:45.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:45.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:45.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:45.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:45.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:45.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:45.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:45.833 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 2946 2930 183.115 183.125 0.00653394 0.00544347 2023-10-09T09:25:45.933 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:45.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:45.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:45.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:46.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:46.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:46.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:46.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:46.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:46.225 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:46.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:46.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:46.328 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:46.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:46.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:46.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:46.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:46.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:46 smithi163 bash[78107]: cluster 2023-10-09T09:25:45.583274+0000 mgr.y (mgr.34107) 581 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 5.5 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 632 KiB/s wr, 9 op/s 2023-10-09T09:25:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:46 smithi163 bash[75425]: cluster 2023-10-09T09:25:45.583274+0000 mgr.y (mgr.34107) 581 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 5.5 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 632 KiB/s wr, 9 op/s 2023-10-09T09:25:46.469 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:46 smithi181 bash[59345]: cluster 2023-10-09T09:25:45.583274+0000 mgr.y (mgr.34107) 581 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 5.5 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 632 KiB/s wr, 9 op/s 2023-10-09T09:25:46.485 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:46.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:46.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:46.537 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:25:46.589 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:46.626 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:46.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:46.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:46.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:46.757 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:46.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:46.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:46.833 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 5576 5560 173.738 164.375 0.0050078 0.00573333 2023-10-09T09:25:46.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:46.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:46.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:46.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:46.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:47.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:47.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:47.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:47.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:47.156 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:47.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:47.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:47.259 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:47.286 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:47.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:47.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:47.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:47.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:47.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:47.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:47.478 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:25:47.520 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:47.550 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:47.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:47.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:47.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:47 smithi181 bash[59345]: audit 2023-10-09T09:25:46.969793+0000 mon.a (mon.0) 1374 : audit 1 from='client.? 172.21.15.163:0/3941822908' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:25:47.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:47.679 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:47.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:47.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:47.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:47 smithi163 bash[78107]: audit 2023-10-09T09:25:46.969793+0000 mon.a (mon.0) 1374 : audit 1 from='client.? 172.21.15.163:0/3941822908' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:25:47.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:47 smithi163 bash[75425]: audit 2023-10-09T09:25:46.969793+0000 mon.a (mon.0) 1374 : audit 1 from='client.? 172.21.15.163:0/3941822908' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:25:47.783 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:47.806 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:47.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:47.833 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 8224 8208 170.987 165.5 0.00656051 0.00583707 2023-10-09T09:25:47.910 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:47.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:47.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:47.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:48.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:48.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:48.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:48.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:48.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:48.199 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:48.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:48.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:48.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:48.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:48.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:48.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:48.463 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:48.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:48.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:48.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[75425]: cluster 2023-10-09T09:25:47.454189+0000 mon.a (mon.0) 1375 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:25:48.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[75425]: cluster 2023-10-09T09:25:47.467532+0000 osd.5 (osd.5) 624 : cluster 3 failed to encode map e1081 with expected crc 2023-10-09T09:25:48.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[75425]: cluster 2023-10-09T09:25:47.469697+0000 osd.7 (osd.7) 477 : cluster 3 failed to encode map e1081 with expected crc 2023-10-09T09:25:48.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[75425]: audit 2023-10-09T09:25:47.473761+0000 mon.a (mon.0) 1376 : audit 1 from='client.? 172.21.15.163:0/3941822908' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:25:48.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[75425]: cluster 2023-10-09T09:25:47.474946+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e1081: 8 total, 8 up, 8 in 2023-10-09T09:25:48.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[75425]: cluster 2023-10-09T09:25:47.584328+0000 mgr.y (mgr.34107) 582 : cluster 0 pgmap v1271: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s wr, 374 op/s 2023-10-09T09:25:48.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[78107]: cluster 2023-10-09T09:25:47.454189+0000 mon.a (mon.0) 1375 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:25:48.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[78107]: cluster 2023-10-09T09:25:47.467532+0000 osd.5 (osd.5) 624 : cluster 3 failed to encode map e1081 with expected crc 2023-10-09T09:25:48.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[78107]: cluster 2023-10-09T09:25:47.469697+0000 osd.7 (osd.7) 477 : cluster 3 failed to encode map e1081 with expected crc 2023-10-09T09:25:48.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[78107]: audit 2023-10-09T09:25:47.473761+0000 mon.a (mon.0) 1376 : audit 1 from='client.? 172.21.15.163:0/3941822908' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:25:48.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[78107]: cluster 2023-10-09T09:25:47.474946+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e1081: 8 total, 8 up, 8 in 2023-10-09T09:25:48.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:48 smithi163 bash[78107]: cluster 2023-10-09T09:25:47.584328+0000 mgr.y (mgr.34107) 582 : cluster 0 pgmap v1271: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s wr, 374 op/s 2023-10-09T09:25:48.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:48.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:48.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:48.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:48 smithi181 bash[59345]: cluster 2023-10-09T09:25:47.454189+0000 mon.a (mon.0) 1375 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:25:48.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:48 smithi181 bash[59345]: cluster 2023-10-09T09:25:47.467532+0000 osd.5 (osd.5) 624 : cluster 3 failed to encode map e1081 with expected crc 2023-10-09T09:25:48.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:48 smithi181 bash[59345]: cluster 2023-10-09T09:25:47.469697+0000 osd.7 (osd.7) 477 : cluster 3 failed to encode map e1081 with expected crc 2023-10-09T09:25:48.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:48 smithi181 bash[59345]: audit 2023-10-09T09:25:47.473761+0000 mon.a (mon.0) 1376 : audit 1 from='client.? 172.21.15.163:0/3941822908' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:25:48.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:48 smithi181 bash[59345]: cluster 2023-10-09T09:25:47.474946+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e1081: 8 total, 8 up, 8 in 2023-10-09T09:25:48.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:48 smithi181 bash[59345]: cluster 2023-10-09T09:25:47.584328+0000 mgr.y (mgr.34107) 582 : cluster 0 pgmap v1271: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s wr, 374 op/s 2023-10-09T09:25:48.728 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:48.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:48.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:48.834 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 10807 10791 168.596 161.438 0.0147419 0.00592067 2023-10-09T09:25:48.865 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:48.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:48.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:48.969 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:48.999 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:49.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:49.103 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:49.134 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:49.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:49.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:49.238 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:49.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:49.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:49.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:49.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:49.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:49.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:49.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:49 smithi163 bash[75425]: audit 2023-10-09T09:25:49.192637+0000 mon.a (mon.0) 1378 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:25:49.508 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:49 smithi163 bash[78107]: audit 2023-10-09T09:25:49.192637+0000 mon.a (mon.0) 1378 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:25:49.533 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:49.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:49.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:49.636 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:49.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:49.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:49.783 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:49.792 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:49 smithi181 bash[59345]: audit 2023-10-09T09:25:49.192637+0000 mon.a (mon.0) 1378 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:25:49.825 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:49.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:49.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:49.833 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 13022 13006 162.562 138.438 0.0103751 0.00613903 2023-10-09T09:25:49.929 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:49.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:49.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:49.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:50.066 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:50.105 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:50.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:50.167 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:50.167 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:50.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:50.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:50.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:50.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:50.342 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:50.369 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:50.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:50.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:50.473 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:50.502 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:50.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:50.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:50.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:50.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:50.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:50.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:50 smithi163 bash[78107]: cluster 2023-10-09T09:25:49.584905+0000 mgr.y (mgr.34107) 583 : cluster 0 pgmap v1272: 89 pgs: 89 active+clean; 409 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-09T09:25:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:50 smithi163 bash[78107]: audit 2023-10-09T09:25:49.726198+0000 mon.a (mon.0) 1379 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:50 smithi163 bash[78107]: audit 2023-10-09T09:25:49.743109+0000 mon.a (mon.0) 1380 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:50 smithi163 bash[78107]: audit 2023-10-09T09:25:49.775516+0000 mon.a (mon.0) 1381 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:50 smithi163 bash[78107]: audit 2023-10-09T09:25:49.787716+0000 mon.a (mon.0) 1382 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:50 smithi163 bash[75425]: cluster 2023-10-09T09:25:49.584905+0000 mgr.y (mgr.34107) 583 : cluster 0 pgmap v1272: 89 pgs: 89 active+clean; 409 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-09T09:25:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:50 smithi163 bash[75425]: audit 2023-10-09T09:25:49.726198+0000 mon.a (mon.0) 1379 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:50 smithi163 bash[75425]: audit 2023-10-09T09:25:49.743109+0000 mon.a (mon.0) 1380 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:50 smithi163 bash[75425]: audit 2023-10-09T09:25:49.775516+0000 mon.a (mon.0) 1381 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:50 smithi163 bash[75425]: audit 2023-10-09T09:25:49.787716+0000 mon.a (mon.0) 1382 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.743 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:50.753 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:50 smithi181 bash[59345]: cluster 2023-10-09T09:25:49.584905+0000 mgr.y (mgr.34107) 583 : cluster 0 pgmap v1272: 89 pgs: 89 active+clean; 409 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-09T09:25:50.753 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:50 smithi181 bash[59345]: audit 2023-10-09T09:25:49.726198+0000 mon.a (mon.0) 1379 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.753 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:50 smithi181 bash[59345]: audit 2023-10-09T09:25:49.743109+0000 mon.a (mon.0) 1380 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.754 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:50 smithi181 bash[59345]: audit 2023-10-09T09:25:49.775516+0000 mon.a (mon.0) 1381 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.754 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:50 smithi181 bash[59345]: audit 2023-10-09T09:25:49.787716+0000 mon.a (mon.0) 1382 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:50.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:50.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:50.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:50.834 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 15122 15106 157.341 131.25 0.00698407 0.00634784 2023-10-09T09:25:50.873 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:50.903 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:50.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:50.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:51.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:51.033 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:51.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:51.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:51.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:51.168 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:51.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:51.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:51.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:51.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:51.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:51.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:51.402 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:51.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:51.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:51.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:51.536 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:51.564 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:51.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:51.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:51.668 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:51.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:25:51.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:51.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:51.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:51.834 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 15 17136 17121 152.853 125.938 0.00313288 0.00653126 2023-10-09T09:25:51.835 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:51.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:51.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:51.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:51.965 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:51.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:51.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:52.071 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:52.099 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:52.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:52.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:52.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:52.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:52.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:52.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:52.336 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:52.362 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:52.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:52.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:52.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:52.490 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:52.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:52.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:52.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:52.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:52.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:52.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:52.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:52.737 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:52 smithi181 bash[59345]: cluster 2023-10-09T09:25:51.585589+0000 mgr.y (mgr.34107) 584 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 821 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T09:25:52.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:52.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:52.834 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 19252 19236 150.268 132.188 0.00814492 0.00664794 2023-10-09T09:25:52.856 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:52.883 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:52.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:52.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:52 smithi163 bash[78107]: cluster 2023-10-09T09:25:51.585589+0000 mgr.y (mgr.34107) 584 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 821 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T09:25:52.901 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:52 smithi163 bash[75425]: cluster 2023-10-09T09:25:51.585589+0000 mgr.y (mgr.34107) 584 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 821 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T09:25:52.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:53.024 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:53.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:53.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:53.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:53.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:53.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:53.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:53.233 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:53.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:53.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:53.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:53.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:53.359 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:25:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:25:53] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.33.4" 2023-10-09T09:25:53.388 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:53.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:53.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:53.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:53.523 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:53.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:53.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:53.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:53.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:53.687 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:53.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:53.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:53.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:53.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:53.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:53.834 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 21142 21126 146.696 118.125 0.00625242 0.00681079 2023-10-09T09:25:53.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:53.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:53.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:54.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:54.080 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:54.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:54.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:54.184 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:54.212 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:54.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:54.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:54.316 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:54.348 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:54.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:54.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:54.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:54.483 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:54.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:54.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:54.588 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:54.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:54.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:54.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:54.726 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:54.747 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:54.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:54 smithi181 bash[59345]: cluster 2023-10-09T09:25:53.586163+0000 mgr.y (mgr.34107) 585 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 909 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T09:25:54.834 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 23101 23085 144.269 122.438 0.0100541 0.00692421 2023-10-09T09:25:54.851 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:54.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:54.894 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:54.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:54.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:54 smithi163 bash[75425]: cluster 2023-10-09T09:25:53.586163+0000 mgr.y (mgr.34107) 585 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 909 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T09:25:54.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:54 smithi163 bash[78107]: cluster 2023-10-09T09:25:53.586163+0000 mgr.y (mgr.34107) 585 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 909 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T09:25:54.995 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:55.025 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:55.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:55.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:55.127 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:55.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:55.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:55.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:55.250 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:55.275 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:55.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:55.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:55.379 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:55.419 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:55.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:55.526 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:55.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:55.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:55.659 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:55.684 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:55.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:55.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:55.788 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:55.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:55 smithi163 bash[78107]: audit 2023-10-09T09:25:55.361743+0000 mon.a (mon.0) 1383 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:55.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:55 smithi163 bash[75425]: audit 2023-10-09T09:25:55.361743+0000 mon.a (mon.0) 1383 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:55.822 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:25:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:55.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:25:55.834 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 24706 24690 140.272 100.312 0.00786607 0.00712197 2023-10-09T09:25:55.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:55.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:55.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:55.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:56.057 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:56.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:55 smithi181 bash[59345]: audit 2023-10-09T09:25:55.361743+0000 mon.a (mon.0) 1383 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:25:56.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:56.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:56.189 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:56.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:56.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:56.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:56.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:56.349 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:56.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:56.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:56.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:56.491 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:56.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:56.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:56.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:56.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:56.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:56.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:56.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:56.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:56.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:56.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:56.834 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 26230 26214 136.519 95.25 0.00633768 0.00731872 2023-10-09T09:25:56.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:56.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:56 smithi163 bash[75425]: cluster 2023-10-09T09:25:55.586795+0000 mgr.y (mgr.34107) 586 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T09:25:56.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:56 smithi163 bash[75425]: audit 2023-10-09T09:25:56.192638+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:56.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:56 smithi163 bash[75425]: audit 2023-10-09T09:25:56.202299+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:56.868 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:56 smithi163 bash[78107]: cluster 2023-10-09T09:25:55.586795+0000 mgr.y (mgr.34107) 586 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T09:25:56.869 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:56 smithi163 bash[78107]: audit 2023-10-09T09:25:56.192638+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:56.869 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:56 smithi163 bash[78107]: audit 2023-10-09T09:25:56.202299+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:56.887 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:56.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:56.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:56.990 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:57.030 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:57.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:57.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:56 smithi181 bash[59345]: cluster 2023-10-09T09:25:55.586795+0000 mgr.y (mgr.34107) 586 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T09:25:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:56 smithi181 bash[59345]: audit 2023-10-09T09:25:56.192638+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:56 smithi181 bash[59345]: audit 2023-10-09T09:25:56.202299+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:57.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:57.170 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:57.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:57.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:57.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:57.305 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:57.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:57.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:57.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:57.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:57.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:57.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:57.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:57.568 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:57.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:57.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:57.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:57.703 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:57.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:57.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:57.806 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:57.829 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:57.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:57.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:57.834 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 27825 27809 133.685 99.6875 0.0120698 0.00746745 2023-10-09T09:25:57.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:57.966 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:57.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:57.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:25:58.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:58.108 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:58.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:58.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:58.213 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:58.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:25:58.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:58.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:25:58.343 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:58.367 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:58.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:58.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:58.470 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:58.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:58.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:58.602 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:58.650 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:25:58.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:58.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:25:58.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:58.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:58 smithi181 bash[59345]: cluster 2023-10-09T09:25:57.587552+0000 mgr.y (mgr.34107) 587 : cluster 0 pgmap v1276: 89 pgs: 89 active+clean; 1.5 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-10-09T09:25:58.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:58 smithi181 bash[59345]: audit 2023-10-09T09:25:58.474369+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:58.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:58 smithi181 bash[59345]: audit 2023-10-09T09:25:58.484188+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:58.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:58.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:58.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:58.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 29200 29184 130.27 85.9375 0.00507593 0.00766871 2023-10-09T09:25:58.881 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:58.891 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:58 smithi163 bash[75425]: cluster 2023-10-09T09:25:57.587552+0000 mgr.y (mgr.34107) 587 : cluster 0 pgmap v1276: 89 pgs: 89 active+clean; 1.5 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-10-09T09:25:58.891 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:58 smithi163 bash[75425]: audit 2023-10-09T09:25:58.474369+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:58.891 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:58 smithi163 bash[75425]: audit 2023-10-09T09:25:58.484188+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:58.891 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:58 smithi163 bash[78107]: cluster 2023-10-09T09:25:57.587552+0000 mgr.y (mgr.34107) 587 : cluster 0 pgmap v1276: 89 pgs: 89 active+clean; 1.5 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-10-09T09:25:58.891 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:58 smithi163 bash[78107]: audit 2023-10-09T09:25:58.474369+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:58.891 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:58 smithi163 bash[78107]: audit 2023-10-09T09:25:58.484188+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:58.910 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:58.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:59.014 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:59.038 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:59.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:59.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:59.145 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:59.170 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:25:59.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:59.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:25:59.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:59.302 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:59.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:59.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:59.431 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:25:59.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:59.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:25:59.534 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:59.569 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:59.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:59.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:59.587 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:25:59.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:59.708 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:25:59.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:59.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:25:59.812 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:59 smithi181 bash[59345]: audit 2023-10-09T09:25:59.079532+0000 mon.a (mon.0) 1388 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:25:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:59 smithi181 bash[59345]: audit 2023-10-09T09:25:59.086213+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:25:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:25:59 smithi181 bash[59345]: audit 2023-10-09T09:25:59.095471+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:59.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 15 30726 30711 127.947 95.4375 0.00740683 0.00780979 2023-10-09T09:25:59.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:25:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:59.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:25:59.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:59 smithi163 bash[78107]: audit 2023-10-09T09:25:59.079532+0000 mon.a (mon.0) 1388 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:25:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:59 smithi163 bash[78107]: audit 2023-10-09T09:25:59.086213+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:25:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:25:59 smithi163 bash[78107]: audit 2023-10-09T09:25:59.095471+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:59.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:59 smithi163 bash[75425]: audit 2023-10-09T09:25:59.079532+0000 mon.a (mon.0) 1388 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:25:59.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:59 smithi163 bash[75425]: audit 2023-10-09T09:25:59.086213+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:25:59.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:25:59 smithi163 bash[75425]: audit 2023-10-09T09:25:59.095471+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:25:59.980 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:25:59.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:25:59.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:00.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:00.114 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:00.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:00.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:00.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:00.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:00.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:00.366 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:00.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:00.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:00.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:00.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:00.515 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:00.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:00.617 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:00.650 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:00.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:00.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:00.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:00.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:00.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:00.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 32302 32286 126.102 98.4375 0.00348968 0.00792039 2023-10-09T09:26:00.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:00.917 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:00.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:00.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:00.957 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:00 smithi163 bash[75425]: cluster 2023-10-09T09:25:59.588119+0000 mgr.y (mgr.34107) 588 : cluster 0 pgmap v1277: 89 pgs: 89 active+clean; 1.6 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T09:26:00.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:00 smithi163 bash[78107]: cluster 2023-10-09T09:25:59.588119+0000 mgr.y (mgr.34107) 588 : cluster 0 pgmap v1277: 89 pgs: 89 active+clean; 1.6 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T09:26:01.022 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:01.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:00 smithi181 bash[59345]: cluster 2023-10-09T09:25:59.588119+0000 mgr.y (mgr.34107) 588 : cluster 0 pgmap v1277: 89 pgs: 89 active+clean; 1.6 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T09:26:01.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:01.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:01.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:01.151 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:01.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:01.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:01.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:01.296 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:01.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:01.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:01.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:01.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:01.459 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:01.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:01.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:01.563 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:01.592 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:01.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:01.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:01.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:01.725 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:01.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:01.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:01.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 34020 34004 125 107.375 0.00900366 0.00799363 2023-10-09T09:26:01.838 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:01 smithi181 bash[59345]: cluster 2023-10-09T09:26:01.588865+0000 mgr.y (mgr.34107) 589 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T09:26:01.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:01.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:01.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:01.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:01.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:01.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:01.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:02.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:02.127 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:02.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:02.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:02.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:01 smithi163 bash[75425]: cluster 2023-10-09T09:26:01.588865+0000 mgr.y (mgr.34107) 589 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T09:26:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:01 smithi163 bash[78107]: cluster 2023-10-09T09:26:01.588865+0000 mgr.y (mgr.34107) 589 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T09:26:02.231 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:02.259 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:02.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:02.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:02.363 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:02.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:02.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:02.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:02.492 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:02.519 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:02.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:02.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:02.668 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:02.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:02.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:02.772 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:02.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:02.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:02.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:02.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 16 35395 35379 122.83 85.9375 0.00723009 0.00813406 2023-10-09T09:26:02.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:02.914 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:26:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:26:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:26:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:22613 6.0 6:065eb579:::benchmark_data_smithi163_107771_object22612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:54.523691+0000", 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 8.3890282680000006, 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.085738474999999995, 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 22613 2023-10-09T09:26:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.523691+0000", 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.523691+0000", 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.523690+0000", 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.523712+0000", 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2240999999999999e-05 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.523721+0000", 2023-10-09T09:26:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4169999999999998e-06 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.523727+0000", 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5110000000000003e-06 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.523803+0000", 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6136000000000004e-05 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.523908+0000", 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000104678 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.523992+0000", 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3979000000000004e-05 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.526120+0000", 2023-10-09T09:26:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021287569999999998 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.526620+0000", 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00049955099999999999 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.609325+0000", 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.082704604000000001 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.609384+0000", 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9836000000000001e-05 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.609429+0000", 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4926999999999999e-05 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:22621 6.0 6:0d9b3058:::benchmark_data_smithi163_107771_object22620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:54.529645+0000", 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 8.3830743030000008, 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.080165210000000001, 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 22621 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.529645+0000", 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.529645+0000", 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.529644+0000", 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.529661+0000", 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7156e-05 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.529670+0000", 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6999999999999997e-06 2023-10-09T09:26:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.529675+0000", 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2639999999999999e-06 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.529708+0000", 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2963000000000003e-05 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.529796+0000", 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7986999999999996e-05 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.529887+0000", 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1055000000000001e-05 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.530684+0000", 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00079685499999999996 2023-10-09T09:26:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.530985+0000", 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000301148 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.609639+0000", 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.078653766999999999 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.609721+0000", 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2391000000000004e-05 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:54.609810+0000", 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8774000000000001e-05 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:24580 6.9 6:94e1e7a6:::benchmark_data_smithi163_107771_object24579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:55.700551+0000", 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 7.2121681310000003, 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.070846470999999994, 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 24580 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.700551+0000", 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.700551+0000", 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.700549+0000", 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.700570+0000", 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1027999999999999e-05 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.700581+0000", 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0232e-05 2023-10-09T09:26:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.700587+0000", 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0229999999999998e-06 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.700631+0000", 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4576999999999997e-05 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.700751+0000", 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000119834 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.700877+0000", 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000125878 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.701878+0000", 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001001105 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.706568+0000", 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046903379999999996 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.771290+0000", 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.064721574000000004 2023-10-09T09:26:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.771350+0000", 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0361e-05 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.771397+0000", 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7089000000000003e-05 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.936 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:24582 6.0 6:0c0936c8:::benchmark_data_smithi163_107771_object24581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:55.702972+0000", 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 7.209747159, 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.068485362999999994, 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 24582 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.702972+0000", 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.702972+0000", 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.702971+0000", 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.702993+0000", 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1940999999999999e-05 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.703003+0000", 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0079e-05 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.703009+0000", 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8680000000000004e-06 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.703062+0000", 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2933999999999998e-05 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.703190+0000", 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012846900000000001 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.703322+0000", 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000131679 2023-10-09T09:26:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.704551+0000", 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001228827 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.704558+0000", 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4449999999999999e-06 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.771370+0000", 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.066811623000000001 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.771418+0000", 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8424000000000003e-05 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:55.771457+0000", 2023-10-09T09:26:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9011999999999999e-05 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:26044 6.0 6:07838b38:::benchmark_data_smithi163_107771_object26043:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:56.672362+0000", 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 6.2403569770000002, 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.081644565000000002, 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 26044 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.672362+0000", 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.672362+0000", 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.672360+0000", 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.672388+0000", 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7815999999999999e-05 2023-10-09T09:26:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.672404+0000", 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6160000000000001e-05 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.672415+0000", 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0387000000000001e-05 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.672477+0000", 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1957000000000003e-05 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.672642+0000", 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016543800000000001 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.672832+0000", 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019029 2023-10-09T09:26:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.674154+0000", 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013218819999999999 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.675541+0000", 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013867580000000001 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.753914+0000", 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.078372608999999996 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.753967+0000", 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3526000000000002e-05 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.754007+0000", 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9378999999999997e-05 2023-10-09T09:26:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:26050 6.9 6:99a288ec:::benchmark_data_smithi163_107771_object26049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:56.676427+0000", 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 6.2362919100000003, 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.077137236999999997, 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 26050 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.676427+0000", 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.676427+0000", 2023-10-09T09:26:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.676426+0000", 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.676451+0000", 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5296e-05 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.676463+0000", 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1729000000000001e-05 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.676471+0000", 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6759999999999996e-06 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.676539+0000", 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7300999999999998e-05 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.676689+0000", 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015070900000000001 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.676797+0000", 2023-10-09T09:26:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000107797 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.678291+0000", 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014934760000000001 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.678548+0000", 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00025797299999999999 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.753479+0000", 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.074930418999999998 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.753529+0000", 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9747999999999999e-05 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.753564+0000", 2023-10-09T09:26:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5722999999999997e-05 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:26056 6.0 6:0fa45a73:::benchmark_data_smithi163_107771_object26055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:56.684938+0000", 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 6.2277810039999997, 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.069490653999999999, 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 26056 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.684938+0000", 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.684938+0000", 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.684936+0000", 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.684967+0000", 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0317000000000001e-05 2023-10-09T09:26:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.684982+0000", 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5756999999999998e-05 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.684994+0000", 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2058e-05 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.685046+0000", 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1372000000000002e-05 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.685141+0000", 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5363000000000002e-05 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.685233+0000", 2023-10-09T09:26:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1710000000000001e-05 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.686116+0000", 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088298700000000001 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.686441+0000", 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00032480200000000001 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.754203+0000", 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.067762808999999993 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.754334+0000", 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000130383 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.754429+0000", 2023-10-09T09:26:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4928999999999999e-05 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:27819 6.0 6:0e0f4088:::benchmark_data_smithi163_107771_object27818:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:57.822164+0000", 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 5.0905553799999996, 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11910617599999999, 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 27819 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.822164+0000", 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.822164+0000", 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.822162+0000", 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.822186+0000", 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3742000000000001e-05 2023-10-09T09:26:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.822198+0000", 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2398999999999999e-05 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.822206+0000", 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6729999999999992e-06 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.822255+0000", 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8755999999999999e-05 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.822403+0000", 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014788099999999999 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.822549+0000", 2023-10-09T09:26:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000146203 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.823993+0000", 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001444271 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.824077+0000", 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4017000000000006e-05 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.940978+0000", 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11690062699999999 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.941115+0000", 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013699599999999999 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.941270+0000", 2023-10-09T09:26:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015502699999999999 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:30624 6.9 6:9117b1bf:::benchmark_data_smithi163_107771_object30623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:59.697478+0000", 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 3.2152409209999999, 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.114847474, 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 30624 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.697478+0000", 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.697478+0000", 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.697478+0000", 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.697499+0000", 2023-10-09T09:26:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1401000000000001e-05 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.697504+0000", 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1839999999999998e-06 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.697509+0000", 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2499999999999997e-06 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.698704+0000", 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011942809999999999 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.698777+0000", 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3598999999999998e-05 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.698906+0000", 2023-10-09T09:26:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012909600000000001 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.701352+0000", 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024450980000000001 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.701377+0000", 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5035999999999999e-05 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812143+0000", 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.110766869 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812268+0000", 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000124747 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812326+0000", 2023-10-09T09:26:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7423999999999998e-05 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:30630 6.9 6:91c9ceaa:::benchmark_data_smithi163_107771_object30629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:59.698175+0000", 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 3.2145438940000002, 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11423760400000001, 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 30630 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.698175+0000", 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.698175+0000", 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.698174+0000", 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.698219+0000", 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4690000000000001e-05 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.698229+0000", 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7909999999999999e-06 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.698234+0000", 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4670000000000003e-06 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.699958+0000", 2023-10-09T09:26:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001723739 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.700074+0000", 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115801 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.700161+0000", 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7227999999999999e-05 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.702214+0000", 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020525389999999999 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.703257+0000", 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010432270000000001 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812361+0000", 2023-10-09T09:26:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.109104596 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812389+0000", 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7926e-05 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812413+0000", 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3499e-05 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:30641 6.9 6:9e5c0190:::benchmark_data_smithi163_107771_object30640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:59.705311+0000", 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 3.2074080920000001, 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.107207281, 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 30641 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.705311+0000", 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.705311+0000", 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.705310+0000", 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.705331+0000", 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1895999999999998e-05 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.705342+0000", 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0908e-05 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.705354+0000", 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1939999999999999e-05 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.705401+0000", 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7079000000000001e-05 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.705508+0000", 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000107044 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.705603+0000", 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4114e-05 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.706944+0000", 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001341248 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812041+0000", 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10509769400000001 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812454+0000", 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000412672 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812489+0000", 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4990000000000002e-05 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812518+0000", 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9098000000000002e-05 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:30634 6.a e1081/1076 6:5582f078:::benchmark_data_smithi163_107771_object30633:head v 1081'1947, mlcod=1081'1944)", 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:59.706913+0000", 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 3.2058056960000001, 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10829873499999999, 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.706913+0000", 2023-10-09T09:26:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.706913+0000", 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.706913+0000", 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.706959+0000", 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6693999999999997e-05 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.706970+0000", 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0924000000000001e-05 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.706976+0000", 2023-10-09T09:26:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4280000000000004e-06 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.707014+0000", 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8319999999999999e-05 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.707034+0000", 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0200999999999999e-05 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.815159+0000", 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.108124393 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.815212+0000", 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3421999999999998e-05 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:30637 6.a e1081/1076 6:57cf03b8:::benchmark_data_smithi163_107771_object30636:head v 1081'1948, mlcod=1081'1944)", 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:59.706986+0000", 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 3.2057334179999999, 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11019620400000001, 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.706986+0000", 2023-10-09T09:26:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.706986+0000", 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.706985+0000", 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.707027+0000", 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1430000000000001e-05 2023-10-09T09:26:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.707034+0000", 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.99e-06 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.707039+0000", 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9389999999999996e-06 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.813257+0000", 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10621873 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.813276+0000", 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8873e-05 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.817028+0000", 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037521239999999999 2023-10-09T09:26:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.817182+0000", 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015351000000000001 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.34791.0:30641 6.9 e1081/1076 ondisk, result = 0)", 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:59.707736+0000", 2023-10-09T09:26:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 3.2049830890000002, 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.104322761, 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.707736+0000", 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.707736+0000", 2023-10-09T09:26:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.707735+0000", 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.707739+0000", 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0609999999999997e-06 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.811942+0000", 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10420210000000001 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.811970+0000", 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8102e-05 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812025+0000", 2023-10-09T09:26:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5398000000000001e-05 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812035+0000", 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5400000000000001e-06 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812059+0000", 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4090000000000001e-05 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.34791.0:30624 6.9 e1081/1076 ondisk, result = 0)", 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:59.708454+0000", 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 3.2042645809999999, 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10387880400000001, 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.708454+0000", 2023-10-09T09:26:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.708454+0000", 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.708454+0000", 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.708458+0000", 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5639999999999998e-06 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.811953+0000", 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10349491500000001 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.811979+0000", 2023-10-09T09:26:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5809e-05 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812109+0000", 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013018700000000001 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812129+0000", 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9463999999999999e-05 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:59.812333+0000", 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020456000000000001 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:31835 6.9 6:931dd639:::benchmark_data_smithi163_107771_object31834:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:00.487990+0000", 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.4247287239999999, 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10662021300000001, 2023-10-09T09:26:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 31835 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.487990+0000", 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.487990+0000", 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.487989+0000", 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.488009+0000", 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0251e-05 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.488018+0000", 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6710000000000005e-06 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.488024+0000", 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9039999999999997e-06 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.488622+0000", 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00059825500000000001 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.488711+0000", 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8745999999999994e-05 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.488864+0000", 2023-10-09T09:26:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015307100000000001 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490644+0000", 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017799199999999999 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.594490+0000", 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.103845887 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.594527+0000", 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7268999999999998e-05 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.594572+0000", 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5256000000000002e-05 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.594611+0000", 2023-10-09T09:26:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8444e-05 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:31844 6.9 6:9d8cf579:::benchmark_data_smithi163_107771_object31843:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:00.490900+0000", 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.4218190540000002, 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.107276382, 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 31844 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490900+0000", 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490900+0000", 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490898+0000", 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490923+0000", 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4411000000000001e-05 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490935+0000", 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2493999999999999e-05 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490943+0000", 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8040000000000002e-06 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490987+0000", 2023-10-09T09:26:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.405e-05 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.491135+0000", 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014824200000000001 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.491302+0000", 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000166661 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.596675+0000", 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.105373256 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.596688+0000", 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2869e-05 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.598080+0000", 2023-10-09T09:26:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001392363 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.598134+0000", 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3519e-05 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.598176+0000", 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2432000000000003e-05 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:31839 6.d e1081/1076 6:b675b2b8:::benchmark_data_smithi163_107771_object31838:head v 1081'2039, mlcod=1081'2036)", 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:00.490943+0000", 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.42177599, 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10032355699999999, 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490943+0000", 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490943+0000", 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490942+0000", 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490977+0000", 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4895999999999999e-05 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490988+0000", 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0549e-05 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.490994+0000", 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5189999999999998e-06 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.491088+0000", 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3663000000000001e-05 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.491106+0000", 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8521000000000001e-05 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.591236+0000", 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10013006000000001 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.591267+0000", 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0210999999999999e-05 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.34791.0:31835 6.9 e1081/1076 ondisk, result = 0)", 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:00.492412+0000", 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.4203069880000001, 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.102080489, 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.492412+0000", 2023-10-09T09:26:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.492412+0000", 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.492411+0000", 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.492416+0000", 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7539999999999997e-06 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.591137+0000", 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.098720364000000005 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.591147+0000", 2023-10-09T09:26:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0908e-05 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.594475+0000", 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033280279999999998 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.594483+0000", 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9249999999999995e-06 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.594493+0000", 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1339999999999993e-06 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:31845 6.a e1081/1076 6:5677b941:::benchmark_data_smithi163_107771_object31844:head v 1081'2025, mlcod=1081'2023)", 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:00.492698+0000", 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.4200208160000001, 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10241953199999999, 2023-10-09T09:26:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.492698+0000", 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.492698+0000", 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.492698+0000", 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.492725+0000", 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7727999999999999e-05 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.591430+0000", 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.098704542000000006 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.591435+0000", 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6840000000000001e-06 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.591552+0000", 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000116873 2023-10-09T09:26:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.591625+0000", 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2560999999999999e-05 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.594825+0000", 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031997179999999998 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:26:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:00.595118+0000", 2023-10-09T09:26:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029314500000000003 2023-10-09T09:26:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:03.041 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:26:03.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:03.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:03.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:03.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:03.175 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:03.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:03.278 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:03.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:03.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:03.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:03.335 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T09:26:03.408 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:03.416 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:03 smithi181 bash[59345]: audit 2023-10-09T09:26:03.259844+0000 mon.c (mon.2) 47 : audit 1 from='client.? 172.21.15.163:0/4124372993' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:26:03.416 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:03 smithi181 bash[59345]: audit 2023-10-09T09:26:03.260169+0000 mon.a (mon.0) 1391 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:26:03.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:03.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:03.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:03.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:03.567 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:03.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:03.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:03.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:03 smithi163 bash[78107]: audit 2023-10-09T09:26:03.259844+0000 mon.c (mon.2) 47 : audit 1 from='client.? 172.21.15.163:0/4124372993' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:26:03.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:03 smithi163 bash[78107]: audit 2023-10-09T09:26:03.260169+0000 mon.a (mon.0) 1391 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:26:03.571 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:26:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:26:03] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.33.4" 2023-10-09T09:26:03.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:03 smithi163 bash[75425]: audit 2023-10-09T09:26:03.259844+0000 mon.c (mon.2) 47 : audit 1 from='client.? 172.21.15.163:0/4124372993' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:26:03.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:03 smithi163 bash[75425]: audit 2023-10-09T09:26:03.260169+0000 mon.a (mon.0) 1391 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:26:03.671 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:03.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:03.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:03.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:03.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:03.824 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:03.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:03.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:03.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 36749 36733 120.818 84.625 0.00489318 0.00827074 2023-10-09T09:26:03.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:03.954 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:03.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:03.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:04.061 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:04.084 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:04.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:04.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:04.188 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:04.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:04.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:04.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:04.329 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:04.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:04.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:04.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:04.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[75425]: cluster 2023-10-09T09:26:03.314525+0000 osd.5 (osd.5) 625 : cluster 3 failed to encode map e1082 with expected crc 2023-10-09T09:26:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[75425]: audit 2023-10-09T09:26:03.318021+0000 mon.a (mon.0) 1392 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:26:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[75425]: cluster 2023-10-09T09:26:03.318099+0000 mon.a (mon.0) 1393 : cluster 0 osdmap e1082: 8 total, 8 up, 8 in 2023-10-09T09:26:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[75425]: cluster 2023-10-09T09:26:03.329198+0000 osd.7 (osd.7) 478 : cluster 3 failed to encode map e1082 with expected crc 2023-10-09T09:26:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[75425]: cluster 2023-10-09T09:26:03.331170+0000 osd.7 (osd.7) 479 : cluster 3 failed to encode map e1082 with expected crc 2023-10-09T09:26:04.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[75425]: cluster 2023-10-09T09:26:03.589303+0000 mgr.y (mgr.34107) 590 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T09:26:04.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[78107]: cluster 2023-10-09T09:26:03.314525+0000 osd.5 (osd.5) 625 : cluster 3 failed to encode map e1082 with expected crc 2023-10-09T09:26:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[78107]: audit 2023-10-09T09:26:03.318021+0000 mon.a (mon.0) 1392 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:26:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[78107]: cluster 2023-10-09T09:26:03.318099+0000 mon.a (mon.0) 1393 : cluster 0 osdmap e1082: 8 total, 8 up, 8 in 2023-10-09T09:26:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[78107]: cluster 2023-10-09T09:26:03.329198+0000 osd.7 (osd.7) 478 : cluster 3 failed to encode map e1082 with expected crc 2023-10-09T09:26:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[78107]: cluster 2023-10-09T09:26:03.331170+0000 osd.7 (osd.7) 479 : cluster 3 failed to encode map e1082 with expected crc 2023-10-09T09:26:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:04 smithi163 bash[78107]: cluster 2023-10-09T09:26:03.589303+0000 mgr.y (mgr.34107) 590 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T09:26:04.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:04.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:04.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:04.588 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:04.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:04 smithi181 bash[59345]: cluster 2023-10-09T09:26:03.314525+0000 osd.5 (osd.5) 625 : cluster 3 failed to encode map e1082 with expected crc 2023-10-09T09:26:04.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:04 smithi181 bash[59345]: audit 2023-10-09T09:26:03.318021+0000 mon.a (mon.0) 1392 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:26:04.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:04 smithi181 bash[59345]: cluster 2023-10-09T09:26:03.318099+0000 mon.a (mon.0) 1393 : cluster 0 osdmap e1082: 8 total, 8 up, 8 in 2023-10-09T09:26:04.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:04 smithi181 bash[59345]: cluster 2023-10-09T09:26:03.329198+0000 osd.7 (osd.7) 478 : cluster 3 failed to encode map e1082 with expected crc 2023-10-09T09:26:04.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:04 smithi181 bash[59345]: cluster 2023-10-09T09:26:03.331170+0000 osd.7 (osd.7) 479 : cluster 3 failed to encode map e1082 with expected crc 2023-10-09T09:26:04.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:04 smithi181 bash[59345]: cluster 2023-10-09T09:26:03.589303+0000 mgr.y (mgr.34107) 590 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T09:26:04.613 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:04.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:04.717 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:04.747 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:04.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:04.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:04.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:26:04.836495+0000 min lat: 0.00141645 max lat: 0.152145 avg lat: 0.0083378 2023-10-09T09:26:04.836 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:26:04.836 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 38373 38357 119.852 101.5 0.00585593 0.0083378 2023-10-09T09:26:04.854 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:04.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:04.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:04.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:04.983 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:05.011 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:05.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:05.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:05.114 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:05.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:05.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:05.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:05.242 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:05.271 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:05.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:05.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:05.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:05.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:05.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:05.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:05.508 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:05.533 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:05.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:05.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:05.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:05.667 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:05.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:05.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:05.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:05.798 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:05.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:05.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 39852 39836 118.546 92.4375 0.0115626 0.00839883 2023-10-09T09:26:05.902 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:05.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:05.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:05.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:06.043 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:06.078 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:06.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:06.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:06.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:06.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:06.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:06.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:06.317 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:06.346 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:06.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:06.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:06.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:06.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:06.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:06.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:06.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:06 smithi163 bash[75425]: cluster 2023-10-09T09:26:05.589963+0000 mgr.y (mgr.34107) 591 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T09:26:06.572 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:06.603 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:06.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:06.707 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:06.731 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:06.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:06 smithi163 bash[78107]: cluster 2023-10-09T09:26:05.589963+0000 mgr.y (mgr.34107) 591 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T09:26:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:06.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:06.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:06 smithi181 bash[59345]: cluster 2023-10-09T09:26:05.589963+0000 mgr.y (mgr.34107) 591 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T09:26:06.834 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:06.837 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 41131 41115 116.791 79.9375 0.00521358 0.00855621 2023-10-09T09:26:06.856 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:06.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:06.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:06.962 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:06.984 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:06.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:07.089 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:07.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:07.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:07.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:07.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:07.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:07.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:07.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:07.351 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:07.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:07.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:07.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:07.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:07.504 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:07.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:07.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:07.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:26:07.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:07.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:07.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:07.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:07.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:07.775 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:07.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:07.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:07.836 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 42510 42494 115.46 86.1875 0.0103565 0.00865243 2023-10-09T09:26:07.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:07.910 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:07.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:07.938 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T09:26:08.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:08.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:08.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:08.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:08.073 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:26:08.122 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:26:08.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:08.168 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:26:08.175 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:08.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:08.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:08.204 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:26:08.243 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:26:08.280 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:26:08.280 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:08.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:08.313 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:26:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:08.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:08.349 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:26:08.352 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:26:08.352 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:26:08.352 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:26:08.352 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:26:08.352 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:26:08.352 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:26:08.353 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:26:08.353 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:26:08.416 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:08.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:08.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:08.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:08.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:08.577 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:08.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:08.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:08.710 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:08.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:08.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:08.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:08 smithi181 bash[59345]: cluster 2023-10-09T09:26:07.590785+0000 mgr.y (mgr.34107) 592 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T09:26:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:08 smithi181 bash[59345]: audit 2023-10-09T09:26:08.034275+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.163:0/2142818537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:26:08.836 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 43885 43869 114.229 85.9375 0.00825672 0.00874701 2023-10-09T09:26:08.842 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:08.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:08.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:08.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:08.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:08 smithi163 bash[78107]: cluster 2023-10-09T09:26:07.590785+0000 mgr.y (mgr.34107) 592 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T09:26:08.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:08 smithi163 bash[78107]: audit 2023-10-09T09:26:08.034275+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.163:0/2142818537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:26:08.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:08 smithi163 bash[75425]: cluster 2023-10-09T09:26:07.590785+0000 mgr.y (mgr.34107) 592 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T09:26:08.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:08 smithi163 bash[75425]: audit 2023-10-09T09:26:08.034275+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.163:0/2142818537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:26:08.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:08.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:08.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:09.075 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:09.108 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:09.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:09.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:09.213 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:09.237 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:09.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:09.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:09.367 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:09.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:09.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:09.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:09.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:09.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:09.603 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:09.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:09.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:09.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:09.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:09.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:09.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:09.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:09.836 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 45137 45121 112.79 78.25 0.0186984 0.00885981 2023-10-09T09:26:09.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:09.893 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:09.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:09.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:09.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:10.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:10.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:10.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:10.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:10.166 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:10.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:10.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:10.272 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:10.297 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:10.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:10.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:10.403 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:10.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:10.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:10.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:10.538 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:10.567 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:10.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:10.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:10.671 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:10.712 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:10.714 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:10 smithi163 bash[75425]: cluster 2023-10-09T09:26:09.591625+0000 mgr.y (mgr.34107) 593 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-10-09T09:26:10.714 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:10 smithi163 bash[75425]: audit 2023-10-09T09:26:10.359522+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:10.714 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:10 smithi163 bash[78107]: cluster 2023-10-09T09:26:09.591625+0000 mgr.y (mgr.34107) 593 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-10-09T09:26:10.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:10 smithi163 bash[78107]: audit 2023-10-09T09:26:10.359522+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:10.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:10.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:10 smithi181 bash[59345]: cluster 2023-10-09T09:26:09.591625+0000 mgr.y (mgr.34107) 593 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-10-09T09:26:10.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:10 smithi181 bash[59345]: audit 2023-10-09T09:26:10.359522+0000 mon.a (mon.0) 1394 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:10.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:10.836 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 16 46533 46517 111.807 87.25 0.00522506 0.0089354 2023-10-09T09:26:10.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:10.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:10.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:10.976 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:10.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:10.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:11.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:11.112 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:11.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:11.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:11.216 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:11.256 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:11.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:11.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:11.363 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:11.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:11.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:11.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:11.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:11.521 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:11.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:11.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:11.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:11.655 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:11.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:11.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:11.679 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:26:11.761 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:11.795 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:11.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:11.836 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 47979 47963 111.013 90.375 0.00697114 0.00900202 2023-10-09T09:26:11.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:11 smithi163 bash[78107]: audit 2023-10-09T09:26:11.113433+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.163:0/1436920593' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:26:11.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:11 smithi163 bash[78107]: audit 2023-10-09T09:26:11.114816+0000 mon.a (mon.0) 1395 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:26:11.865 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:11 smithi163 bash[75425]: audit 2023-10-09T09:26:11.113433+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.163:0/1436920593' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:26:11.865 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:11 smithi163 bash[75425]: audit 2023-10-09T09:26:11.114816+0000 mon.a (mon.0) 1395 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:26:11.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:11.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:11.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:12.032 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:12.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:11 smithi181 bash[59345]: audit 2023-10-09T09:26:11.113433+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.163:0/1436920593' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:26:12.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:11 smithi181 bash[59345]: audit 2023-10-09T09:26:11.114816+0000 mon.a (mon.0) 1395 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:26:12.058 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:12.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:12.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:12.162 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:12.192 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:12.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:12.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:12.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:12.327 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:12.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:12.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:12.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:26:12.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:12.457 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:12.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:12.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:12.568 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:12.591 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:12.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:12.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:12.696 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:12.705 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:12 smithi181 bash[59345]: cluster 2023-10-09T09:26:11.592309+0000 mgr.y (mgr.34107) 594 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T09:26:12.705 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:12 smithi181 bash[59345]: cluster 2023-10-09T09:26:11.655333+0000 mon.a (mon.0) 1396 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:26:12.706 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:12 smithi181 bash[59345]: cluster 2023-10-09T09:26:11.665823+0000 osd.5 (osd.5) 626 : cluster 3 failed to encode map e1083 with expected crc 2023-10-09T09:26:12.706 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:12 smithi181 bash[59345]: audit 2023-10-09T09:26:11.673071+0000 mon.a (mon.0) 1397 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:26:12.706 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:12 smithi181 bash[59345]: cluster 2023-10-09T09:26:11.673320+0000 mon.a (mon.0) 1398 : cluster 0 osdmap e1083: 8 total, 8 up, 8 in 2023-10-09T09:26:12.706 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:12 smithi181 bash[59345]: cluster 2023-10-09T09:26:11.682779+0000 osd.7 (osd.7) 480 : cluster 3 failed to encode map e1083 with expected crc 2023-10-09T09:26:12.706 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:12 smithi181 bash[59345]: cluster 2023-10-09T09:26:11.685841+0000 osd.7 (osd.7) 481 : cluster 3 failed to encode map e1083 with expected crc 2023-10-09T09:26:12.732 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:12.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:12.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:12.836 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:12.837 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 49361 49345 110.133 86.375 0.00767138 0.00907439 2023-10-09T09:26:12.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:12.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:12.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:12.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:12.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[75425]: cluster 2023-10-09T09:26:11.592309+0000 mgr.y (mgr.34107) 594 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T09:26:12.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[75425]: cluster 2023-10-09T09:26:11.655333+0000 mon.a (mon.0) 1396 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:26:12.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[75425]: cluster 2023-10-09T09:26:11.665823+0000 osd.5 (osd.5) 626 : cluster 3 failed to encode map e1083 with expected crc 2023-10-09T09:26:12.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[75425]: audit 2023-10-09T09:26:11.673071+0000 mon.a (mon.0) 1397 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:26:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[75425]: cluster 2023-10-09T09:26:11.673320+0000 mon.a (mon.0) 1398 : cluster 0 osdmap e1083: 8 total, 8 up, 8 in 2023-10-09T09:26:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[75425]: cluster 2023-10-09T09:26:11.682779+0000 osd.7 (osd.7) 480 : cluster 3 failed to encode map e1083 with expected crc 2023-10-09T09:26:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[75425]: cluster 2023-10-09T09:26:11.685841+0000 osd.7 (osd.7) 481 : cluster 3 failed to encode map e1083 with expected crc 2023-10-09T09:26:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[78107]: cluster 2023-10-09T09:26:11.592309+0000 mgr.y (mgr.34107) 594 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T09:26:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[78107]: cluster 2023-10-09T09:26:11.655333+0000 mon.a (mon.0) 1396 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:26:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[78107]: cluster 2023-10-09T09:26:11.665823+0000 osd.5 (osd.5) 626 : cluster 3 failed to encode map e1083 with expected crc 2023-10-09T09:26:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[78107]: audit 2023-10-09T09:26:11.673071+0000 mon.a (mon.0) 1397 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:26:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[78107]: cluster 2023-10-09T09:26:11.673320+0000 mon.a (mon.0) 1398 : cluster 0 osdmap e1083: 8 total, 8 up, 8 in 2023-10-09T09:26:12.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[78107]: cluster 2023-10-09T09:26:11.682779+0000 osd.7 (osd.7) 480 : cluster 3 failed to encode map e1083 with expected crc 2023-10-09T09:26:12.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:12 smithi163 bash[78107]: cluster 2023-10-09T09:26:11.685841+0000 osd.7 (osd.7) 481 : cluster 3 failed to encode map e1083 with expected crc 2023-10-09T09:26:13.003 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:13.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:13.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:13.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:13.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:13.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:13.379 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:13.412 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:13.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:13.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:13.516 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:13.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:13.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:13.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:13.659 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:26:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:26:13] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.33.4" 2023-10-09T09:26:13.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:13.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:13.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:13.808 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:13.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:13.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:13.839 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 50767 50751 109.365 87.875 0.00917688 0.00913813 2023-10-09T09:26:13.913 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:13.961 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:13.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:14.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:14.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:14.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:14.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:14.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:14.234 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:14.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:14.339 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:14.372 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:14.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:14.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:14.519 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:14.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:14.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:14.622 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:14.649 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:14.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:14.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:14.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:14.762 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:14 smithi163 bash[78107]: cluster 2023-10-09T09:26:13.592832+0000 mgr.y (mgr.34107) 595 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T09:26:14.762 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:14 smithi163 bash[75425]: cluster 2023-10-09T09:26:13.592832+0000 mgr.y (mgr.34107) 595 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T09:26:14.789 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:14.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:14.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:14.836 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 52098 52082 108.492 83.1875 0.0104388 0.00921105 2023-10-09T09:26:14.894 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:14.927 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:14.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:14.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:15.030 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:15.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:15.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:15.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:15.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:14 smithi181 bash[59345]: cluster 2023-10-09T09:26:13.592832+0000 mgr.y (mgr.34107) 595 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T09:26:15.159 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:15.187 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:15.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:15.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:15.290 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:15.299 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:26:15.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:15.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:15.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:15.420 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:15.471 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:15.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:15.573 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:15.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:15.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:15.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:15.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:15.738 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:15.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:15.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:15.806 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:15.811 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:15.838 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 53383 53367 107.58 80.3125 0.0191742 0.00928714 2023-10-09T09:26:15.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:15.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:15.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:15.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:15.979 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:16.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:16.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:16.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:16.114 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:16.143 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:16.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:16.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:16.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:16.272 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:16.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:16.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:16.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:16.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:16.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:16.507 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:16.539 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:16.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:16.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:16.645 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:16.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:16 smithi163 bash[75425]: cluster 2023-10-09T09:26:15.593416+0000 mgr.y (mgr.34107) 596 : cluster 0 pgmap v1287: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:26:16.659 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:16 smithi163 bash[78107]: cluster 2023-10-09T09:26:15.593416+0000 mgr.y (mgr.34107) 596 : cluster 0 pgmap v1287: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:26:16.679 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:16.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:16.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:16.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:16 smithi181 bash[59345]: cluster 2023-10-09T09:26:15.593416+0000 mgr.y (mgr.34107) 596 : cluster 0 pgmap v1287: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:26:16.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:16.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:16.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:16.837 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 54567 54551 106.531 74 0.00513624 0.00938148 2023-10-09T09:26:16.925 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:16.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:16.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:16.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:17.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:17.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:17.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:17.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:17.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:17.230 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:17.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:17.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:17.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:17.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:17.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:17.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:17.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:17.500 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:17.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:17.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:17.603 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:17.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:17.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:17.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:17.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:17.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:17.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:17.838 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 55886 55870 105.8 82.4375 0.00543111 0.00944513 2023-10-09T09:26:17.850 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:17.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:17.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:17.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:17.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:18.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:18.040 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:18.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:18.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:18.145 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:18.170 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:18.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:18.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:18.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:18.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:18.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:18.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:18.442 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:18.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:18.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:18.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:18.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:18.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:18.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:18.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:18.711 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:18.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:18.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:18.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:18.825 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:18 smithi163 bash[78107]: cluster 2023-10-09T09:26:17.595383+0000 mgr.y (mgr.34107) 597 : cluster 0 pgmap v1288: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T09:26:18.825 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:18 smithi163 bash[75425]: cluster 2023-10-09T09:26:17.595383+0000 mgr.y (mgr.34107) 597 : cluster 0 pgmap v1288: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T09:26:18.837 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 57160 57144 105.03 79.625 0.0145635 0.00951481 2023-10-09T09:26:18.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:18.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:18.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:18.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:18.953 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:18 smithi181 bash[59345]: cluster 2023-10-09T09:26:17.595383+0000 mgr.y (mgr.34107) 597 : cluster 0 pgmap v1288: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T09:26:18.974 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:18.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:18.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:19.079 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:19.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:19.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:19.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:19.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:19.236 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:19.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:19.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:19.344 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:19.373 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:19.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:19.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:19.480 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:19.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:19.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:19.625 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:19.651 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:19.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:19.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:19.756 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:19.796 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:19.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:19.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:19.838 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 58446 58430 104.326 80.375 0.00950898 0.00958006 2023-10-09T09:26:19.880 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:19.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:19.932 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:19.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:19.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:20.038 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:20.066 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:20.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:20.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:20.169 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:20.199 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:20.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:20.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:20.302 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:20.332 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:20.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:20.435 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:20.461 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:20.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:20.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:20.564 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:20.596 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:20.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:20.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:20.700 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:20.727 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:20.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:20.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:20.830 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:20.838 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 15 59517 59502 103.289 67 0.00855844 0.00967596 2023-10-09T09:26:20.839 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:20 smithi181 bash[59345]: cluster 2023-10-09T09:26:19.597711+0000 mgr.y (mgr.34107) 598 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T09:26:20.865 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:20.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:20.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:20.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:20 smithi163 bash[78107]: cluster 2023-10-09T09:26:19.597711+0000 mgr.y (mgr.34107) 598 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T09:26:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:20 smithi163 bash[75425]: cluster 2023-10-09T09:26:19.597711+0000 mgr.y (mgr.34107) 598 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T09:26:20.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:20.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:20.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:20.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:21.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:21.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:21.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:21.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:21.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:21.257 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:21.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:21.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:21.360 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:21.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:21.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:21.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:21.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:21.521 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:21.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:21.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:21.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:21.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:21.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:21.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:21.756 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:21.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:21.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:21.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:21.838 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 60663 60647 102.431 71.5625 0.00340192 0.00974997 2023-10-09T09:26:21.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:21.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:21.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:21.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:22.029 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:22.050 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:22.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:22.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:22.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:22.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:22.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:22.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:22.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:22.315 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:22.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:22.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:22.419 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:22.450 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:22.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:22.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:22.561 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:22.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:22.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:22.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:22.656 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:22 smithi163 bash[78107]: cluster 2023-10-09T09:26:21.598315+0000 mgr.y (mgr.34107) 599 : cluster 0 pgmap v1290: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T09:26:22.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:22 smithi163 bash[75425]: cluster 2023-10-09T09:26:21.598315+0000 mgr.y (mgr.34107) 599 : cluster 0 pgmap v1290: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T09:26:22.693 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:22.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:22.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:22.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:22.825 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:22.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:22 smithi181 bash[59345]: cluster 2023-10-09T09:26:21.598315+0000 mgr.y (mgr.34107) 599 : cluster 0 pgmap v1290: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T09:26:22.838 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 61891 61875 101.755 76.75 0.031793 0.00982111 2023-10-09T09:26:22.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:22.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:22.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:22.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:22.988 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:22.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:23.093 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:23.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:23.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:23.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:23.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:23.245 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:23.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:23.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:23.372 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:23.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:23.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:23.476 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:23.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:23.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:23.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:23.611 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:23.645 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:23.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:23.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:26:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:26:23] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.33.4" 2023-10-09T09:26:23.750 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:23.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:23.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:23.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:23.838 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 63040 63024 100.987 71.8125 0.00622335 0.00989677 2023-10-09T09:26:23.876 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:23.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:23.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:23.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:24.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:24.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:24.040 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:24.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:24.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:24.163 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:24.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:24.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:24.267 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:24.291 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:24.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:24.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:24.415 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:24.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:24.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:24.518 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:24.555 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:24.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:24.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:24.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:24.672 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:24 smithi163 bash[78107]: cluster 2023-10-09T09:26:23.598817+0000 mgr.y (mgr.34107) 600 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T09:26:24.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:24 smithi163 bash[75425]: cluster 2023-10-09T09:26:23.598817+0000 mgr.y (mgr.34107) 600 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T09:26:24.691 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:24.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:24.796 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:24.821 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:24.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:24.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:24.838 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:26:24.839268+0000 min lat: 0.00141645 max lat: 0.152145 avg lat: 0.00997401 2023-10-09T09:26:24.838 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:26:24.838 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 64101 64085 100.12 66.3125 0.00829649 0.00997401 2023-10-09T09:26:24.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:24.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:24 smithi181 bash[59345]: cluster 2023-10-09T09:26:23.598817+0000 mgr.y (mgr.34107) 600 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T09:26:24.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:24.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:24.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:25.055 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:25.083 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:25.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:25.187 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:25.220 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:25.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:25.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:25.323 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:25.346 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:25.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:25.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:25.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:25.479 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:25.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:25.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:25.582 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:25.604 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:25.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:25.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:25 smithi163 bash[75425]: audit 2023-10-09T09:26:25.360083+0000 mon.a (mon.0) 1399 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:25.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:25.731 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:25.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:25.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:25.834 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:25.842 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 65250 65234 99.4256 71.8125 0.0126966 0.0100515 2023-10-09T09:26:25.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:25.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:25.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:25.959 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:25 smithi163 bash[78107]: audit 2023-10-09T09:26:25.360083+0000 mon.a (mon.0) 1399 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:25.966 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:25.988 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:25.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:25.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:25 smithi181 bash[59345]: audit 2023-10-09T09:26:25.360083+0000 mon.a (mon.0) 1399 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:26.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:26.115 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:26.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:26.219 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:26.241 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:26.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:26.345 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:26.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:26.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:26.476 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:26.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:26.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:26.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:26.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:26.629 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:26.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:26.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:26.733 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:26.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:26.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:26.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:26.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:26 smithi181 bash[59345]: cluster 2023-10-09T09:26:25.599387+0000 mgr.y (mgr.34107) 601 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T09:26:26.840 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 66384 66368 98.7457 70.875 0.00829938 0.0101214 2023-10-09T09:26:26.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:26.870 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:26 smithi163 bash[75425]: cluster 2023-10-09T09:26:25.599387+0000 mgr.y (mgr.34107) 601 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T09:26:26.870 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:26 smithi163 bash[78107]: cluster 2023-10-09T09:26:25.599387+0000 mgr.y (mgr.34107) 601 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T09:26:26.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:26.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:26.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:27.029 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:27.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:27.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:27.133 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:27.157 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:27.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:27.263 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:27.293 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:27.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:27.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:27.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:27.426 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:27.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:27.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:27.529 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:27.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:27.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:27.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:27.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:27.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:27.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:27.794 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:27.823 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:27.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:27.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:27.840 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 67485 67469 98.0495 68.8125 0.00602235 0.0101932 2023-10-09T09:26:27.929 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:27.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:27.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:27.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:28.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:28.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:28.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:28.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:28.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:28.213 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:28.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:28.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:28.316 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:28.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:28.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:28.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:28.482 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:28.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:28.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:28.587 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:28.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:28.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:28.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:28.716 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:28.745 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:28.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:28.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:28.840 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 68579 68563 97.375 68.375 0.032147 0.0102625 2023-10-09T09:26:28.848 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:28.874 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:28.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:28.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:28.972 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:28 smithi163 bash[75425]: cluster 2023-10-09T09:26:27.602090+0000 mgr.y (mgr.34107) 602 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T09:26:28.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:28 smithi163 bash[78107]: cluster 2023-10-09T09:26:27.602090+0000 mgr.y (mgr.34107) 602 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T09:26:28.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:29.005 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:29.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:29.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:29.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:28 smithi181 bash[59345]: cluster 2023-10-09T09:26:27.602090+0000 mgr.y (mgr.34107) 602 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T09:26:29.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:29.135 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:29.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:29.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:29.239 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:29.263 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:29.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:29.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:29.437 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:29.459 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:29.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:29.562 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:29.595 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:29.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:29.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:29.702 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:29.735 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:29.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:29.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:29.839 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:29 smithi181 bash[59345]: cluster 2023-10-09T09:26:29.602615+0000 mgr.y (mgr.34107) 603 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T09:26:30.066 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 69604 69588 96.6347 64.0625 0.0157307 0.0103426 2023-10-09T09:26:30.067 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:29 smithi163 bash[78107]: cluster 2023-10-09T09:26:29.602615+0000 mgr.y (mgr.34107) 603 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T09:26:30.067 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:29 smithi163 bash[75425]: cluster 2023-10-09T09:26:29.602615+0000 mgr.y (mgr.34107) 603 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T09:26:30.095 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:30.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:30.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:30.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:30.223 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:30.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:30.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:30.357 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:30.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:30.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:30.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:30.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:30.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:30.840 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 70507 70491 95.7608 56.4375 0.0169742 0.0104359 2023-10-09T09:26:30.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:30.892 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:30.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:30.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:30.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:31.006 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:26:31.025 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:31.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:31.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:31.047 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:26:31.086 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:26:31.119 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:26:31.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:31.162 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:26:31.168 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:31.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:31.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:31.189 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:26:31.204 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:26:31.204 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:26:31.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:70791 6.e e1083/1076 6:72bdd2d8:::benchmark_data_smithi163_107771_object70790:head v 1083'4411, mlcod=1083'4407)", 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:31.160832+0000", 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.031949793999999997, 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031981339999999997, 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.160832+0000", 2023-10-09T09:26:31.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.160832+0000", 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.160830+0000", 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:26:31.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.160932+0000", 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101338 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.160952+0000", 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0086999999999999e-05 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.160962+0000", 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8630000000000003e-06 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.173230+0000", 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012268604000000001 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.173252+0000", 2023-10-09T09:26:31.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1645999999999999e-05 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:70806 6.1 6:8f9f45ce:::benchmark_data_smithi163_107771_object70805:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e1083)", 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:31.173881+0000", 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.018900079, 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019014377999999998, 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:31.209 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 70806 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.173881+0000", 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:31.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.173881+0000", 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.173880+0000", 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.173904+0000", 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3866000000000002e-05 2023-10-09T09:26:31.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.173915+0000", 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1096e-05 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.173923+0000", 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.422e-06 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:31.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.178941+0000", 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050181030000000003 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.179095+0000", 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015382500000000001 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.179247+0000", 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015164900000000001 2023-10-09T09:26:31.213 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.34791.0:70806 6.1 e1083/1076)", 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:31.188990+0000", 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.003791698, 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0039901049999999999, 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:26:31.214 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.188990+0000", 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.188990+0000", 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:31.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.188989+0000", 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.188998+0000", 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9630000000000004e-06 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.189018+0000", 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.018e-05 2023-10-09T09:26:31.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.189028+0000", 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6649999999999998e-06 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:70817 6.b e1083/1076 6:d14c8686:::benchmark_data_smithi163_107771_object70816:head v 1083'4439, mlcod=1083'4437)", 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:31.191452+0000", 2023-10-09T09:26:31.217 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0013292919999999999, 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015865759999999999, 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191452+0000", 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:31.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191452+0000", 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191452+0000", 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191480+0000", 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7911000000000001e-05 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191488+0000", 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5210000000000002e-06 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191494+0000", 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.207e-06 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:31.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191854+0000", 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00035989600000000001 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191877+0000", 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3136e-05 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.221 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:70810 6.4 6:2abb58ed:::benchmark_data_smithi163_107771_object70809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e1083)", 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:31.191531+0000", 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0012509439999999999, 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001577303, 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 70810 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.222 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191531+0000", 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191531+0000", 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:31.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191529+0000", 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191556+0000", 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7441e-05 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191567+0000", 2023-10-09T09:26:31.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0695e-05 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191574+0000", 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0280000000000003e-06 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191639+0000", 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5038999999999999e-05 2023-10-09T09:26:31.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191768+0000", 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000129188 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191904+0000", 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013623300000000001 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.226 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:70816 6.1 6.dd5fbc31 (undecoded) ondisk+write+known_if_redirected e1083)", 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:31.191580+0000", 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0012015260000000001, 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015701599999999999, 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:26:31.227 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 70816 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191580+0000", 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191580+0000", 2023-10-09T09:26:31.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191579+0000", 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191602+0000", 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2558e-05 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:31.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191609+0000", 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1010000000000003e-06 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191618+0000", 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1409999999999994e-06 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.230 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.34791.0:70806 6.1 e1083/1076)", 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:31.191637+0000", 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0011440210000000001, 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015425440000000001, 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191637+0000", 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191637+0000", 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191637+0000", 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:31.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191640+0000", 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2349999999999999e-06 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191648+0000", 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8979999999999996e-06 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191653+0000", 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0350000000000001e-06 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.233 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:70815 6.d e1083/1076)", 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:31.191670+0000", 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.001111248, 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015353319999999999, 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:26:31.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191670+0000", 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191670+0000", 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191670+0000", 2023-10-09T09:26:31.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191711+0000", 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1399000000000001e-05 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191719+0000", 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9270000000000005e-06 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:26:31.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:26:31.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:26:31.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:31.191725+0000", 2023-10-09T09:26:31.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6579999999999999e-06 2023-10-09T09:26:31.237 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.237 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:26:31.237 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.237 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:26:31.237 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T09:26:31.237 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 8 2023-10-09T09:26:31.237 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:26:31.238 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:26:31.265 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:26:31.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:26:31.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:26:31.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:26:31.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:26:31.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:26:31.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:26:31.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:26:31.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:26:31.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:31.302 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:31.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:31.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:31.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:31.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:31.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:31.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:31.477 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T09:26:31.532 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:31.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:31 smithi181 bash[59345]: audit 2023-10-09T09:26:30.930546+0000 mon.a (mon.0) 1400 : audit 1 from='client.? 172.21.15.163:0/2001649553' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:31.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:31 smithi181 bash[59345]: audit 2023-10-09T09:26:30.965280+0000 mon.a (mon.0) 1401 : audit 0 from='client.? 172.21.15.163:0/2423115207' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:26:31.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:31.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:31.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:31.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:31.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:31 smithi163 bash[75425]: audit 2023-10-09T09:26:30.930546+0000 mon.a (mon.0) 1400 : audit 1 from='client.? 172.21.15.163:0/2001649553' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:31.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:31 smithi163 bash[75425]: audit 2023-10-09T09:26:30.965280+0000 mon.a (mon.0) 1401 : audit 0 from='client.? 172.21.15.163:0/2423115207' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:26:31.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:31 smithi163 bash[78107]: audit 2023-10-09T09:26:30.930546+0000 mon.a (mon.0) 1400 : audit 1 from='client.? 172.21.15.163:0/2001649553' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:26:31.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:31 smithi163 bash[78107]: audit 2023-10-09T09:26:30.965280+0000 mon.a (mon.0) 1401 : audit 0 from='client.? 172.21.15.163:0/2423115207' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:26:31.688 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:31.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:31.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:31.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:31.825 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:31.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:31.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:31.841 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 71444 71428 94.9692 58.5625 0.00873123 0.0105233 2023-10-09T09:26:31.929 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:31.958 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:31.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:31.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:32.061 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:32.094 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:32.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:32.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:32.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:32.233 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:32.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:32.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:32.340 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:32.367 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:32.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:32.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:32.473 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:32.499 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:26:32.505 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:32.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:32.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:32.610 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:32.640 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:32.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:32.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[75425]: cluster 2023-10-09T09:26:31.466761+0000 osd.5 (osd.5) 627 : cluster 3 failed to encode map e1084 with expected crc 2023-10-09T09:26:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[75425]: cluster 2023-10-09T09:26:31.467421+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e1084 with expected crc 2023-10-09T09:26:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[75425]: audit 2023-10-09T09:26:31.470388+0000 mon.a (mon.0) 1402 : audit 1 from='client.? 172.21.15.163:0/2001649553' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:26:32.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[75425]: cluster 2023-10-09T09:26:31.470634+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e1084: 8 total, 8 up, 8 in 2023-10-09T09:26:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[75425]: cluster 2023-10-09T09:26:31.603069+0000 mgr.y (mgr.34107) 604 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:26:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[75425]: audit 2023-10-09T09:26:31.769679+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.163:0/3946193571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:26:32.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[75425]: audit 2023-10-09T09:26:31.770679+0000 mon.a (mon.0) 1404 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:26:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[78107]: cluster 2023-10-09T09:26:31.466761+0000 osd.5 (osd.5) 627 : cluster 3 failed to encode map e1084 with expected crc 2023-10-09T09:26:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[78107]: cluster 2023-10-09T09:26:31.467421+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e1084 with expected crc 2023-10-09T09:26:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[78107]: audit 2023-10-09T09:26:31.470388+0000 mon.a (mon.0) 1402 : audit 1 from='client.? 172.21.15.163:0/2001649553' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:26:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[78107]: cluster 2023-10-09T09:26:31.470634+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e1084: 8 total, 8 up, 8 in 2023-10-09T09:26:32.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[78107]: cluster 2023-10-09T09:26:31.603069+0000 mgr.y (mgr.34107) 604 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:26:32.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[78107]: audit 2023-10-09T09:26:31.769679+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.163:0/3946193571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:26:32.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:32 smithi163 bash[78107]: audit 2023-10-09T09:26:31.770679+0000 mon.a (mon.0) 1404 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:26:32.745 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:32.752 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:32 smithi181 bash[59345]: cluster 2023-10-09T09:26:31.466761+0000 osd.5 (osd.5) 627 : cluster 3 failed to encode map e1084 with expected crc 2023-10-09T09:26:32.752 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:32 smithi181 bash[59345]: cluster 2023-10-09T09:26:31.467421+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e1084 with expected crc 2023-10-09T09:26:32.752 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:32 smithi181 bash[59345]: audit 2023-10-09T09:26:31.470388+0000 mon.a (mon.0) 1402 : audit 1 from='client.? 172.21.15.163:0/2001649553' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:26:32.752 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:32 smithi181 bash[59345]: cluster 2023-10-09T09:26:31.470634+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e1084: 8 total, 8 up, 8 in 2023-10-09T09:26:32.753 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:32 smithi181 bash[59345]: cluster 2023-10-09T09:26:31.603069+0000 mgr.y (mgr.34107) 604 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:26:32.753 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:32 smithi181 bash[59345]: audit 2023-10-09T09:26:31.769679+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.163:0/3946193571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:26:32.753 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:32 smithi181 bash[59345]: audit 2023-10-09T09:26:31.770679+0000 mon.a (mon.0) 1404 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:26:32.773 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:32.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:32.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:32.840 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 72448 72432 94.2979 62.75 0.0485197 0.0105919 2023-10-09T09:26:32.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:32.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:32.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:32.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:33.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:33.039 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:33.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:33.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:33.146 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:33.169 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:33.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:33.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:33.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:33.299 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:33.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:33.388 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:26:33] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.33.4" 2023-10-09T09:26:33.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:33.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:33.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:33.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:33.539 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:33.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:33 smithi181 bash[59345]: cluster 2023-10-09T09:26:32.467209+0000 mon.a (mon.0) 1405 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:26:33.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:33 smithi181 bash[59345]: cluster 2023-10-09T09:26:32.467244+0000 mon.a (mon.0) 1406 : cluster 1 Cluster is now healthy 2023-10-09T09:26:33.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:33 smithi181 bash[59345]: cluster 2023-10-09T09:26:32.477018+0000 osd.5 (osd.5) 628 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:33 smithi181 bash[59345]: cluster 2023-10-09T09:26:32.491743+0000 osd.7 (osd.7) 482 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:33 smithi181 bash[59345]: cluster 2023-10-09T09:26:32.491745+0000 osd.4 (osd.4) 517 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:33 smithi181 bash[59345]: cluster 2023-10-09T09:26:32.492863+0000 osd.5 (osd.5) 629 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:33 smithi181 bash[59345]: audit 2023-10-09T09:26:32.493550+0000 mon.a (mon.0) 1407 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:26:33.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:33 smithi181 bash[59345]: cluster 2023-10-09T09:26:32.493777+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e1085: 8 total, 8 up, 8 in 2023-10-09T09:26:33.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:33.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:33.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:33.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:33.684 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[75425]: cluster 2023-10-09T09:26:32.467209+0000 mon.a (mon.0) 1405 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:26:33.684 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[75425]: cluster 2023-10-09T09:26:32.467244+0000 mon.a (mon.0) 1406 : cluster 1 Cluster is now healthy 2023-10-09T09:26:33.684 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[75425]: cluster 2023-10-09T09:26:32.477018+0000 osd.5 (osd.5) 628 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.684 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[75425]: cluster 2023-10-09T09:26:32.491743+0000 osd.7 (osd.7) 482 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.684 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[75425]: cluster 2023-10-09T09:26:32.491745+0000 osd.4 (osd.4) 517 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[75425]: cluster 2023-10-09T09:26:32.492863+0000 osd.5 (osd.5) 629 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[75425]: audit 2023-10-09T09:26:32.493550+0000 mon.a (mon.0) 1407 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[75425]: cluster 2023-10-09T09:26:32.493777+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e1085: 8 total, 8 up, 8 in 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[78107]: cluster 2023-10-09T09:26:32.467209+0000 mon.a (mon.0) 1405 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[78107]: cluster 2023-10-09T09:26:32.467244+0000 mon.a (mon.0) 1406 : cluster 1 Cluster is now healthy 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[78107]: cluster 2023-10-09T09:26:32.477018+0000 osd.5 (osd.5) 628 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[78107]: cluster 2023-10-09T09:26:32.491743+0000 osd.7 (osd.7) 482 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[78107]: cluster 2023-10-09T09:26:32.491745+0000 osd.4 (osd.4) 517 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[78107]: cluster 2023-10-09T09:26:32.492863+0000 osd.5 (osd.5) 629 : cluster 3 failed to encode map e1085 with expected crc 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[78107]: audit 2023-10-09T09:26:32.493550+0000 mon.a (mon.0) 1407 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:26:33.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:33 smithi163 bash[78107]: cluster 2023-10-09T09:26:32.493777+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e1085: 8 total, 8 up, 8 in 2023-10-09T09:26:33.701 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:33.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:33.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:33.804 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:33.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:33.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:33.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:33.840 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 73584 73568 93.8223 71 0.0177264 0.0106529 2023-10-09T09:26:33.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:33.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:33.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:34.063 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:34.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:34.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:34.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:34.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:34.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:34.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:34.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:34.350 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:34.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:34.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:34.482 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:34.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:34.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:34.587 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:34.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:34.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:34.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:34.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:34.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:34 smithi163 bash[78107]: cluster 2023-10-09T09:26:33.603580+0000 mgr.y (mgr.34107) 605 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T09:26:34.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:34 smithi163 bash[75425]: cluster 2023-10-09T09:26:33.603580+0000 mgr.y (mgr.34107) 605 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T09:26:34.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:34.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:34.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T09:26:34.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:34 smithi181 bash[59345]: cluster 2023-10-09T09:26:33.603580+0000 mgr.y (mgr.34107) 605 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T09:26:34.823 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T09:26:34.842 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 74651 74635 93.2777 66.6875 0.0102202 0.0107145 2023-10-09T09:26:34.847 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:34.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:34.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:34.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:35.035 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:35.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:35.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:35.164 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:35.192 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:35.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:35.299 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:35.327 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:35.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:35.430 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:35.457 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:35.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:35.561 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:35.592 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:35.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:35.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:35.695 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:35.726 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:35.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:35.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:35.843 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 75656 75640 92.6778 62.8125 0.0162833 0.010784 2023-10-09T09:26:35.859 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:35.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:35.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:35.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:35.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:35.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:35.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:36.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:36.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:36.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:36.232 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:36.258 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:36.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:36.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:36.362 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:36.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:36.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:36.501 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:36.510 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:36 smithi181 bash[59345]: cluster 2023-10-09T09:26:35.604215+0000 mgr.y (mgr.34107) 606 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T09:26:36.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:36.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:36.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:36.632 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:36.658 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:36.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:36.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:36.765 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:36.805 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:36.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:36.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:36.820 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:26:36.829 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:36 smithi163 bash[75425]: cluster 2023-10-09T09:26:35.604215+0000 mgr.y (mgr.34107) 606 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T09:26:36.829 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:36 smithi163 bash[78107]: cluster 2023-10-09T09:26:35.604215+0000 mgr.y (mgr.34107) 606 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T09:26:36.843 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 76723 76707 92.1776 66.6875 0.0226155 0.0108425 2023-10-09T09:26:36.910 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:36.954 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:36.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:36.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:37.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:26:37.059 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:37.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:37.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:37.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:37.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:37.253 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:37.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:37.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:37.356 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:37.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:37.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:37.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:37.484 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:37.509 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:37.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:37.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:37.617 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:37.642 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:37.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:37.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:37.745 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:37.774 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:37.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:37.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:37.843 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 77671 77655 91.5563 59.25 0.00777176 0.010916 2023-10-09T09:26:37.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:37.911 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:37.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:37.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:38.017 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:38.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:38.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:38.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:38.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:38.180 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:38.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:38.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:38.285 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:38.312 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:38.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:38.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:38.417 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:38.460 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:38.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:38.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:38.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:38.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:38.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:38.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:38.695 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:38.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:38 smithi163 bash[75425]: cluster 2023-10-09T09:26:37.604925+0000 mgr.y (mgr.34107) 607 : cluster 0 pgmap v1300: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:26:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:38 smithi163 bash[78107]: cluster 2023-10-09T09:26:37.604925+0000 mgr.y (mgr.34107) 607 : cluster 0 pgmap v1300: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:26:38.720 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:38.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:38.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:38.823 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:38.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 78700 78684 91.0517 64.3125 0.0111817 0.0109769 2023-10-09T09:26:38.850 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:38.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:38.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:38.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:38.965 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:38 smithi181 bash[59345]: cluster 2023-10-09T09:26:37.604925+0000 mgr.y (mgr.34107) 607 : cluster 0 pgmap v1300: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:26:38.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:38.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:38.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:39.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:39.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:39.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:39.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:39.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:39.256 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:39.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:39.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:39.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:39.388 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:39.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:39.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:39.495 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:39.521 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:39.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:39.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:39.627 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:39.652 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:39.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:39.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:39.701 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:39.704 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:39.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:39.796 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:39.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:39.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 79716 79700 90.5504 63.5 0.0332808 0.0110382 2023-10-09T09:26:39.902 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:39.930 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:39.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:40.034 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:40.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:40.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:40.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:40.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:40.191 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:40.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:40.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:40.296 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:40.327 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:40.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:40.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:40.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:40.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:40.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:40.571 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:40.602 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:40.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:40.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:40.705 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:40 smithi163 bash[75425]: cluster 2023-10-09T09:26:39.605516+0000 mgr.y (mgr.34107) 608 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T09:26:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:40 smithi163 bash[75425]: audit 2023-10-09T09:26:40.367554+0000 mon.a (mon.0) 1409 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:26:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:40 smithi163 bash[75425]: audit 2023-10-09T09:26:40.368859+0000 mon.a (mon.0) 1410 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:40 smithi163 bash[78107]: cluster 2023-10-09T09:26:39.605516+0000 mgr.y (mgr.34107) 608 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T09:26:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:40 smithi163 bash[78107]: audit 2023-10-09T09:26:40.367554+0000 mon.a (mon.0) 1409 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:26:40.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:40 smithi163 bash[78107]: audit 2023-10-09T09:26:40.368859+0000 mon.a (mon.0) 1410 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:40.723 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:40.730 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:40.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:40.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:40.836 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:40.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 80711 80695 90.0439 62.1875 0.0077691 0.0110997 2023-10-09T09:26:40.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:40 smithi181 bash[59345]: cluster 2023-10-09T09:26:39.605516+0000 mgr.y (mgr.34107) 608 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T09:26:40.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:40 smithi181 bash[59345]: audit 2023-10-09T09:26:40.367554+0000 mon.a (mon.0) 1409 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:26:40.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:40 smithi181 bash[59345]: audit 2023-10-09T09:26:40.368859+0000 mon.a (mon.0) 1410 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:40.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:40.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:40.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:40.969 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:41.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:41.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:41.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:41.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:41.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:41.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:41.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:41.271 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:41.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:41.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:41.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:41.400 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:41.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:41.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:41.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:41.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:41.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:41.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:41.639 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:41.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:41.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:41.740 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:26:41.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:41.813 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:41.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:41.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:41.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 81701 81685 89.5496 61.875 0.0127747 0.0111611 2023-10-09T09:26:41.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:41.954 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:41.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:41.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:42.059 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:42.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:42.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:42.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:42.195 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:42.219 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:42.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:42.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:42.322 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:42.360 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:42.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:42.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:42.463 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:42.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:42.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:42.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:42.599 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:42.633 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:42.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:42.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:42.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:42.759 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:42.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:42.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:42.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 82687 82671 89.0681 61.625 0.00462676 0.0112212 2023-10-09T09:26:42.866 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:42.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:42.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:42.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:42 smithi163 bash[75425]: cluster 2023-10-09T09:26:41.606148+0000 mgr.y (mgr.34107) 609 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:26:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:42 smithi163 bash[78107]: cluster 2023-10-09T09:26:41.606148+0000 mgr.y (mgr.34107) 609 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:26:43.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:43.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:42 smithi181 bash[59345]: cluster 2023-10-09T09:26:41.606148+0000 mgr.y (mgr.34107) 609 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:26:43.037 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:43.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:43.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:43.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:43.174 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:43.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:43.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:43.279 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:43.307 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:43.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:43.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:43.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:43.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:43.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:43.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:43.579 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:43.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:43.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:43.685 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:43.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:43.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:43.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:43.717 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:26:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:26:43] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.33.4" 2023-10-09T09:26:43.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:43.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:43.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:43.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:43.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 83738 83722 88.6718 65.6875 0.0247154 0.0112721 2023-10-09T09:26:43.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:43.977 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:43.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:43.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:44.084 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:44.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:44.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:44.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:44.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:44.253 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:44.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:44.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:44.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:44.389 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:44.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:44.492 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:44.522 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:44.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:44.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:44.656 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:44.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:44.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:44.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:44.791 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:44.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:44.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:44.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:26:44.845497+0000 min lat: 0.00141645 max lat: 0.196141 avg lat: 0.0113287 2023-10-09T09:26:44.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:26:44.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 84733 84717 88.2303 62.1875 0.0120833 0.0113287 2023-10-09T09:26:44.899 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:44.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:44 smithi163 bash[75425]: cluster 2023-10-09T09:26:43.608107+0000 mgr.y (mgr.34107) 610 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-09T09:26:44.900 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:44 smithi163 bash[78107]: cluster 2023-10-09T09:26:43.608107+0000 mgr.y (mgr.34107) 610 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-09T09:26:44.924 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:44.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:44 smithi181 bash[59345]: cluster 2023-10-09T09:26:43.608107+0000 mgr.y (mgr.34107) 610 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-09T09:26:44.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:44.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:45.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:45.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:45.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:45.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:45.161 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:45.191 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:45.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:45.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:45.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:45.322 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:45.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:45.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:45.426 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:45.456 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:45.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:45.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:45.559 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:45.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:45.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:45.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:45.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:45.719 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:45.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:45.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:45.822 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:45.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 85629 85613 87.7019 56 0.0242268 0.011396 2023-10-09T09:26:45.846 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:45.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:45.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:45.954 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:45.979 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:45.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:45.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:46.083 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:46.111 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:46.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:46.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:46.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:46.249 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:46.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:46.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:46.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:46.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:46.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:46.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:46.478 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:46.501 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:46.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:46.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:46.607 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:46.630 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:46.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:46.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:46 smithi163 bash[75425]: cluster 2023-10-09T09:26:45.608687+0000 mgr.y (mgr.34107) 611 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-10-09T09:26:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:46 smithi163 bash[78107]: cluster 2023-10-09T09:26:45.608687+0000 mgr.y (mgr.34107) 611 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-10-09T09:26:46.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:46.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:46 smithi181 bash[59345]: cluster 2023-10-09T09:26:45.608687+0000 mgr.y (mgr.34107) 611 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-10-09T09:26:46.767 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:46.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:46.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:46.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 86510 86494 87.1754 55.0625 0.0131478 0.0114661 2023-10-09T09:26:46.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:46.903 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:46.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:46.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:47.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:47.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:47.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:47.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:47.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:47.172 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:47.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:47.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:47.277 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:47.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:47.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:47.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:47.407 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:47.427 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:47.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:47.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:47.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:47.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:47.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:47.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:47.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:47.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:47.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:47.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:47.836 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:47.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:47.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:47.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 87516 87500 86.7896 62.875 0.00705297 0.0115164 2023-10-09T09:26:47.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:47.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:47.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:48.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:48.098 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:48.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:48.200 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:48.224 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:48.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:48.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:48.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:48.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:48.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:48.493 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:48.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:48.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:48.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:48.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:48.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:48.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:48.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:48 smithi163 bash[75425]: cluster 2023-10-09T09:26:47.609259+0000 mgr.y (mgr.34107) 612 : cluster 0 pgmap v1305: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T09:26:48.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:48 smithi163 bash[78107]: cluster 2023-10-09T09:26:47.609259+0000 mgr.y (mgr.34107) 612 : cluster 0 pgmap v1305: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T09:26:48.722 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:48.756 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:48.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:48.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:48.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 88480 88464 86.3749 60.25 0.011951 0.0115721 2023-10-09T09:26:48.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:48.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:48.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:48.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:49.001 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:49.010 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:48 smithi181 bash[59345]: cluster 2023-10-09T09:26:47.609259+0000 mgr.y (mgr.34107) 612 : cluster 0 pgmap v1305: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T09:26:49.025 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:49.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:49.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:49.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:49.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:49.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:49.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:49.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:49.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:49.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:49.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:49.391 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:49.421 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:49.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:49.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:49.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:49.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:49.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:49.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:49.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:49.696 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:49.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:49.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:49.803 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:49.830 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:49.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:49.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:49.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 89425 89409 85.9546 59.0625 0.0081515 0.0116281 2023-10-09T09:26:49.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:49.964 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:49.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:49.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:50.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:50.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:50.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:50.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:50.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:50.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:50.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:50.329 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:50.359 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:50.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:50.463 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:50.490 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:50.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:50.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:50.598 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:50.626 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:50.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:50.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:50.733 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:50 smithi163 bash[75425]: cluster 2023-10-09T09:26:49.609829+0000 mgr.y (mgr.34107) 613 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s 2023-10-09T09:26:50.760 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:50.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:50.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:50.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 90367 90351 85.5443 58.875 0.0434962 0.0116834 2023-10-09T09:26:50.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:50.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:50.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:50.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:51.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:51.017 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:50 smithi163 bash[78107]: cluster 2023-10-09T09:26:49.609829+0000 mgr.y (mgr.34107) 613 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s 2023-10-09T09:26:51.036 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:51.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:51.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:50 smithi181 bash[59345]: cluster 2023-10-09T09:26:49.609829+0000 mgr.y (mgr.34107) 613 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s 2023-10-09T09:26:51.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:51.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:51.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:51.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:51.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:51.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:51.299 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:51.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:51.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:51.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:51.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:51.537 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:51.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:51.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:51.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:51.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:51.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:51.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:51.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:51.841 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:51.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 91340 91324 85.1751 60.8125 0.0153534 0.0117352 2023-10-09T09:26:51.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:51.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:51.947 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:51.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:51 smithi163 bash[78107]: cluster 2023-10-09T09:26:51.610473+0000 mgr.y (mgr.34107) 614 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T09:26:51.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:51 smithi163 bash[75425]: cluster 2023-10-09T09:26:51.610473+0000 mgr.y (mgr.34107) 614 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T09:26:51.974 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:51.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:52.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:51 smithi181 bash[59345]: cluster 2023-10-09T09:26:51.610473+0000 mgr.y (mgr.34107) 614 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T09:26:52.079 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:52.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:52.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:52.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:52.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:52.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:52.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:52.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:52.334 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:52.358 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:52.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:52.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:52.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:52.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:52.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:52.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:52.594 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:52.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:52.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:52.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:52.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:52.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:52.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:52.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 92308 92292 84.8122 60.5 0.0112611 0.0117853 2023-10-09T09:26:52.857 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:52.883 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:52.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:52.987 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:53.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:53.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:53.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:53 smithi163 bash[75425]: audit 2023-10-09T09:26:53.044437+0000 mon.a (mon.0) 1411 : audit 0 from='client.? 172.21.15.163:0/2209316025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:26:53.114 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:26:53.124 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:53.132 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:53 smithi181 bash[59345]: audit 2023-10-09T09:26:53.044437+0000 mon.a (mon.0) 1411 : audit 0 from='client.? 172.21.15.163:0/2209316025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:26:53.152 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:53.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:53.193 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:26:53.241 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:26:53.258 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:53.292 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:53.294 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:26:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:53.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:53.334 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:26:53.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:53 smithi163 bash[78107]: audit 2023-10-09T09:26:53.044437+0000 mon.a (mon.0) 1411 : audit 0 from='client.? 172.21.15.163:0/2209316025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:26:53.386 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:26:53.397 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:53.425 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:53.426 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:26:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:53.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:53.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:26:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:26:53] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.33.4" 2023-10-09T09:26:53.506 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:26:53.516 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:26:53.516 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:26:53.516 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:26:53.516 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:26:53.516 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:26:53.516 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:26:53.516 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:26:53.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:26:53.530 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:53.563 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:53.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:53.624 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:26:53.624 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:26:53.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:53.692 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:53.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:53.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:53.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:53.820 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:53.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:53.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:53.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 93014 92998 84.2225 44.125 0.0103454 0.0118666 2023-10-09T09:26:53.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:53.951 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:53.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:53.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:54.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:54.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:54.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:54.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:54.103 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:54 smithi163 bash[75425]: cluster 2023-10-09T09:26:53.611256+0000 mgr.y (mgr.34107) 615 : cluster 0 pgmap v1308: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-10-09T09:26:54.103 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:54 smithi163 bash[75425]: audit 2023-10-09T09:26:53.900797+0000 mon.a (mon.0) 1412 : audit 1 from='client.? 172.21.15.163:0/3991039058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:26:54.104 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:54 smithi163 bash[75425]: audit 2023-10-09T09:26:53.983798+0000 mon.a (mon.0) 1413 : audit 1 from='client.? 172.21.15.163:0/2415199126' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:54.133 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T09:26:54.135 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:26:54.190 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:54.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:54.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:54.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:54 smithi181 bash[59345]: cluster 2023-10-09T09:26:53.611256+0000 mgr.y (mgr.34107) 615 : cluster 0 pgmap v1308: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-10-09T09:26:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:54 smithi181 bash[59345]: audit 2023-10-09T09:26:53.900797+0000 mon.a (mon.0) 1412 : audit 1 from='client.? 172.21.15.163:0/3991039058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:26:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:54 smithi181 bash[59345]: audit 2023-10-09T09:26:53.983798+0000 mon.a (mon.0) 1413 : audit 1 from='client.? 172.21.15.163:0/2415199126' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:54.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:54.351 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:54.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:54.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:54.421 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:54 smithi163 bash[78107]: cluster 2023-10-09T09:26:53.611256+0000 mgr.y (mgr.34107) 615 : cluster 0 pgmap v1308: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-10-09T09:26:54.422 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:54 smithi163 bash[78107]: audit 2023-10-09T09:26:53.900797+0000 mon.a (mon.0) 1412 : audit 1 from='client.? 172.21.15.163:0/3991039058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:26:54.422 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:54 smithi163 bash[78107]: audit 2023-10-09T09:26:53.983798+0000 mon.a (mon.0) 1413 : audit 1 from='client.? 172.21.15.163:0/2415199126' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:26:54.455 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:54.485 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:54.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:54.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:54.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:54.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:54.622 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:54.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:54.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:54.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:54.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:54.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 93922 93906 83.83 56.75 0.02128 0.011923 2023-10-09T09:26:54.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:54.885 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:54.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:54.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:54.991 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:55.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:55.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:55.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:55.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:55.160 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:55.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:55.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:55.264 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:55.290 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:55.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:55.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:55.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:55 smithi181 bash[59345]: cluster 2023-10-09T09:26:54.102442+0000 mon.a (mon.0) 1414 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:26:55.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:55 smithi181 bash[59345]: cluster 2023-10-09T09:26:54.117261+0000 osd.5 (osd.5) 630 : cluster 3 failed to encode map e1086 with expected crc 2023-10-09T09:26:55.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:55 smithi181 bash[59345]: cluster 2023-10-09T09:26:54.121945+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e1086 with expected crc 2023-10-09T09:26:55.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:55 smithi181 bash[59345]: audit 2023-10-09T09:26:54.126036+0000 mon.a (mon.0) 1415 : audit 1 from='client.? 172.21.15.163:0/3991039058' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:26:55.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:55 smithi181 bash[59345]: audit 2023-10-09T09:26:54.126275+0000 mon.a (mon.0) 1416 : audit 1 from='client.? 172.21.15.163:0/2415199126' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:26:55.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:55 smithi181 bash[59345]: cluster 2023-10-09T09:26:54.126645+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e1086: 8 total, 8 up, 8 in 2023-10-09T09:26:55.395 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:55.419 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:55.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:55.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[75425]: cluster 2023-10-09T09:26:54.102442+0000 mon.a (mon.0) 1414 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:26:55.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[75425]: cluster 2023-10-09T09:26:54.117261+0000 osd.5 (osd.5) 630 : cluster 3 failed to encode map e1086 with expected crc 2023-10-09T09:26:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[75425]: cluster 2023-10-09T09:26:54.121945+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e1086 with expected crc 2023-10-09T09:26:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[75425]: audit 2023-10-09T09:26:54.126036+0000 mon.a (mon.0) 1415 : audit 1 from='client.? 172.21.15.163:0/3991039058' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:26:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[75425]: audit 2023-10-09T09:26:54.126275+0000 mon.a (mon.0) 1416 : audit 1 from='client.? 172.21.15.163:0/2415199126' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:26:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[75425]: cluster 2023-10-09T09:26:54.126645+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e1086: 8 total, 8 up, 8 in 2023-10-09T09:26:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[78107]: cluster 2023-10-09T09:26:54.102442+0000 mon.a (mon.0) 1414 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:26:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[78107]: cluster 2023-10-09T09:26:54.117261+0000 osd.5 (osd.5) 630 : cluster 3 failed to encode map e1086 with expected crc 2023-10-09T09:26:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[78107]: cluster 2023-10-09T09:26:54.121945+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e1086 with expected crc 2023-10-09T09:26:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[78107]: audit 2023-10-09T09:26:54.126036+0000 mon.a (mon.0) 1415 : audit 1 from='client.? 172.21.15.163:0/3991039058' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:26:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[78107]: audit 2023-10-09T09:26:54.126275+0000 mon.a (mon.0) 1416 : audit 1 from='client.? 172.21.15.163:0/2415199126' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:26:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:55 smithi163 bash[78107]: cluster 2023-10-09T09:26:54.126645+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e1086: 8 total, 8 up, 8 in 2023-10-09T09:26:55.522 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:55.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:55.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:55.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:55.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:55.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:55.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:55.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:55.783 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:55.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:55.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:55.845 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 94834 94818 83.4521 57 0.0110345 0.0119765 2023-10-09T09:26:55.916 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:55.938 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:55.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:55.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:56.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:56.074 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:56.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:56.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:56.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:56.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:56.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:56.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:56.313 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:56.346 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:56.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:56.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:56.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:56.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:56 smithi163 bash[75425]: audit 2023-10-09T09:26:55.371611+0000 mon.a (mon.0) 1418 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:26:56.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:56 smithi163 bash[75425]: audit 2023-10-09T09:26:55.376011+0000 mon.a (mon.0) 1419 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:56.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:56 smithi163 bash[75425]: cluster 2023-10-09T09:26:55.611875+0000 mgr.y (mgr.34107) 616 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T09:26:56.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:56 smithi163 bash[78107]: audit 2023-10-09T09:26:55.371611+0000 mon.a (mon.0) 1418 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:26:56.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:56 smithi163 bash[78107]: audit 2023-10-09T09:26:55.376011+0000 mon.a (mon.0) 1419 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:56.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:56 smithi163 bash[78107]: cluster 2023-10-09T09:26:55.611875+0000 mgr.y (mgr.34107) 616 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T09:26:56.487 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:56.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:56.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:56.595 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:56.621 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:56.624 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:56 smithi181 bash[59345]: audit 2023-10-09T09:26:55.371611+0000 mon.a (mon.0) 1418 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:26:56.625 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:56 smithi181 bash[59345]: audit 2023-10-09T09:26:55.376011+0000 mon.a (mon.0) 1419 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:26:56.625 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:56 smithi181 bash[59345]: cluster 2023-10-09T09:26:55.611875+0000 mgr.y (mgr.34107) 616 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T09:26:56.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:56.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:56.726 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:56.751 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:56.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:56.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:56.846 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 95745 95729 83.0838 56.9375 0.0121117 0.0120302 2023-10-09T09:26:56.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:56.894 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:56.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:56.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:57.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:57.023 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:57.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:57.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:57.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:57.153 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:57.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:57.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:57.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:57.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T09:26:57.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:57.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:57.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:57.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T09:26:57.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:57.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:26:57.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:57.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:26:57.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:57.548 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:57.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:57.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:57.651 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:57.690 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:57.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:57.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:57.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:57.816 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:57.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:57.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:57.846 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 96606 96590 82.6828 53.8125 0.0148774 0.0120884 2023-10-09T09:26:57.919 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:57.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:57.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:57.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:58.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:58.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:58.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:58.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:58.195 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:58.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:58.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:58.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:58.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:58.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:58.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:58.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:58.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:58.491 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:26:58.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:58.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:26:58.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:58.635 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:26:58.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:58.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:26:58.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:58.750 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:58 smithi163 bash[75425]: cluster 2023-10-09T09:26:57.612586+0000 mgr.y (mgr.34107) 617 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T09:26:58.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:58 smithi163 bash[78107]: cluster 2023-10-09T09:26:57.612586+0000 mgr.y (mgr.34107) 617 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T09:26:58.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:58.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:58.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:58.846 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 97518 97502 82.3356 57 0.0140147 0.0121406 2023-10-09T09:26:58.870 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:58.878 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:26:58.880 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:58 smithi181 bash[59345]: cluster 2023-10-09T09:26:57.612586+0000 mgr.y (mgr.34107) 617 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T09:26:58.898 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:58.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:58.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:59.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:59.045 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:59.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:59.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:59.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:59.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:59.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:59.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:59.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:59.336 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:26:59.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:59.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:26:59.371 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:26:59.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:59.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:26:59.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:59.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:26:59.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:59.616 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:26:59.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:59.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:26:59.686 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:26:59 smithi163 bash[75425]: audit 2023-10-09T09:26:59.179403+0000 mon.a (mon.0) 1420 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:26:59.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:59.745 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:59.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:59.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:26:59.846 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 98268 98252 81.8628 46.875 0.00545816 0.012209 2023-10-09T09:26:59.848 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:59.884 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:26:59.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:26:59.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:26:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:26:59 smithi163 bash[78107]: audit 2023-10-09T09:26:59.179403+0000 mon.a (mon.0) 1420 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:26:59.987 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:26:59.996 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:26:59 smithi181 bash[59345]: audit 2023-10-09T09:26:59.179403+0000 mon.a (mon.0) 1420 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:27:00.018 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:00.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:00.122 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:00.147 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:00.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:00.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:00.251 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:00.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:00.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:00.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:00.380 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:00.412 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:00.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:00.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:00.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:00.547 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:00.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:00.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:00.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:00.675 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:00.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:00.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:00.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:00.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:00.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:00.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:00 smithi181 bash[59345]: cluster 2023-10-09T09:26:59.613136+0000 mgr.y (mgr.34107) 618 : cluster 0 pgmap v1312: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T09:27:00.846 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 99157 99141 81.5167 55.5625 0.0149325 0.0122619 2023-10-09T09:27:00.914 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:00.923 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:00 smithi163 bash[78107]: cluster 2023-10-09T09:26:59.613136+0000 mgr.y (mgr.34107) 618 : cluster 0 pgmap v1312: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T09:27:00.923 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:00 smithi163 bash[75425]: cluster 2023-10-09T09:26:59.613136+0000 mgr.y (mgr.34107) 618 : cluster 0 pgmap v1312: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T09:27:00.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:00.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:01.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:01.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:01.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:01.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:01.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:01.212 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:01.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:01.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:01.316 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:01.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:01.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:01.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:01.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:01.485 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:01.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:01.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:01.592 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:01.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:01.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:01.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:01.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:01.763 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:01.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:01.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:01.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:01 smithi181 bash[59345]: cluster 2023-10-09T09:27:01.614086+0000 mgr.y (mgr.34107) 619 : cluster 0 pgmap v1313: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T09:27:01.846 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 100026 100010 81.1634 54.3125 0.0151168 0.0123142 2023-10-09T09:27:01.866 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:01.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:01.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:01.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:01 smithi163 bash[75425]: cluster 2023-10-09T09:27:01.614086+0000 mgr.y (mgr.34107) 619 : cluster 0 pgmap v1313: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T09:27:01.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:01 smithi163 bash[78107]: cluster 2023-10-09T09:27:01.614086+0000 mgr.y (mgr.34107) 619 : cluster 0 pgmap v1313: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-09T09:27:01.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:02.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:02.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:02.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:02.117 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:02.119 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:02.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:02.167 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:02.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:02.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:02.271 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:02.295 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:02.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:02.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:02.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:02.443 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:02.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:02.551 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:02.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:02.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:02.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:02.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:02.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:02.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:02.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:02.823 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:02.849 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 100893 100877 80.8153 54.1875 0.0243508 0.0123684 2023-10-09T09:27:02.851 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:02.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:02.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:02.956 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:02.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:02.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:02.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:03.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:03.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:03.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:03.144 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:03.228 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:03.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:03.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:03.371 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:03.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:03.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:03.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:03.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:27:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:27:03] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.33.4" 2023-10-09T09:27:03.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:03.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:03.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:03.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:03.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:03.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:03.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:03.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:03.763 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:03.791 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:03.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:03.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:03.848 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 101751 101735 80.4711 53.625 0.0220686 0.0124209 2023-10-09T09:27:03.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:03.922 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:03.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:03.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:04.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:04.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:04.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:04.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:04.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:04.163 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:04.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:04.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:04.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:04.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:04.327 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:04.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:04.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:04.490 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:04.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:04.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:04.600 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:04.629 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:04.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:04.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:04.734 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:04.785 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:04 smithi181 bash[59345]: cluster 2023-10-09T09:27:03.614605+0000 mgr.y (mgr.34107) 620 : cluster 0 pgmap v1314: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2023-10-09T09:27:04.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:04.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:04.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:04.849 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:27:04.849983+0000 min lat: 0.00141645 max lat: 0.196141 avg lat: 0.0124826 2023-10-09T09:27:04.849 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:27:04.849 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 102534 102518 80.0764 48.9375 0.0288351 0.0124826 2023-10-09T09:27:04.893 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:04.902 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:04 smithi163 bash[78107]: cluster 2023-10-09T09:27:03.614605+0000 mgr.y (mgr.34107) 620 : cluster 0 pgmap v1314: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2023-10-09T09:27:04.902 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:04 smithi163 bash[75425]: cluster 2023-10-09T09:27:03.614605+0000 mgr.y (mgr.34107) 620 : cluster 0 pgmap v1314: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2023-10-09T09:27:04.920 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:04.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:04.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:05.025 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:05.070 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:05.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:05.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:05.173 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:05.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:05.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:05.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:05.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:05.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:05.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:05.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:05.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:05.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:05.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:05.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:05.583 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:05.607 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:05.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:05.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:05.712 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:05.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:05.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:05.849 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 103308 103292 79.6849 48.375 0.015242 0.0125438 2023-10-09T09:27:05.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:05.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:05.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:05.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:05.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:06.034 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:06.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:06.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:06.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:06.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:06.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:06.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:06.295 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:06.324 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:06.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:06.428 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:06.453 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:06.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:06.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:06.560 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:06.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:06.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:06.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:06.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:06.707 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[78107]: audit 2023-10-09T09:27:05.606904+0000 mon.a (mon.0) 1421 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:06.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[78107]: cluster 2023-10-09T09:27:05.615186+0000 mgr.y (mgr.34107) 621 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T09:27:06.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[78107]: audit 2023-10-09T09:27:05.617454+0000 mon.a (mon.0) 1422 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:06.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[78107]: audit 2023-10-09T09:27:05.707078+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:06.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[78107]: audit 2023-10-09T09:27:05.714525+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:06.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[78107]: audit 2023-10-09T09:27:06.355221+0000 mon.a (mon.0) 1425 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:27:06.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[78107]: audit 2023-10-09T09:27:06.356361+0000 mon.a (mon.0) 1426 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:27:06.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[78107]: audit 2023-10-09T09:27:06.362346+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:06.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[75425]: audit 2023-10-09T09:27:05.606904+0000 mon.a (mon.0) 1421 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:06.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[75425]: cluster 2023-10-09T09:27:05.615186+0000 mgr.y (mgr.34107) 621 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T09:27:06.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[75425]: audit 2023-10-09T09:27:05.617454+0000 mon.a (mon.0) 1422 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:06.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[75425]: audit 2023-10-09T09:27:05.707078+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:06.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[75425]: audit 2023-10-09T09:27:05.714525+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:06.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[75425]: audit 2023-10-09T09:27:06.355221+0000 mon.a (mon.0) 1425 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:27:06.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[75425]: audit 2023-10-09T09:27:06.356361+0000 mon.a (mon.0) 1426 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:27:06.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:06 smithi163 bash[75425]: audit 2023-10-09T09:27:06.362346+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:06.725 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:06.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:06.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:06.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:06.849 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 104097 104081 79.3145 49.3125 0.0137007 0.0126008 2023-10-09T09:27:06.851 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:06.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:06.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:06.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:06.984 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:06.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:06.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:06 smithi181 bash[59345]: audit 2023-10-09T09:27:05.606904+0000 mon.a (mon.0) 1421 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:06 smithi181 bash[59345]: cluster 2023-10-09T09:27:05.615186+0000 mgr.y (mgr.34107) 621 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T09:27:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:06 smithi181 bash[59345]: audit 2023-10-09T09:27:05.617454+0000 mon.a (mon.0) 1422 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:06 smithi181 bash[59345]: audit 2023-10-09T09:27:05.707078+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:06 smithi181 bash[59345]: audit 2023-10-09T09:27:05.714525+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:06 smithi181 bash[59345]: audit 2023-10-09T09:27:06.355221+0000 mon.a (mon.0) 1425 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:27:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:06 smithi181 bash[59345]: audit 2023-10-09T09:27:06.356361+0000 mon.a (mon.0) 1426 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:27:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:06 smithi181 bash[59345]: audit 2023-10-09T09:27:06.362346+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:27:07.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:07.114 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:07.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:07.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:07.249 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:07.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:07.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:07.355 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:07.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:07.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:07.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:07.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:07.525 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:07.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:07.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:07.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:07.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:07.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:07.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:07.758 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:07.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:07.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:07.849 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 104919 104903 78.9779 51.375 0.0185335 0.0126562 2023-10-09T09:27:07.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:07.924 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:07.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:08.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:08.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:08.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:08.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:08.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:08.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:08.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:08.300 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:08.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:08.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:08.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:08.429 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:08.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:08.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:08.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:08.558 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:08.591 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:08.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:08.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:08.621 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:08 smithi163 bash[78107]: cluster 2023-10-09T09:27:07.616865+0000 mgr.y (mgr.34107) 622 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-10-09T09:27:08.622 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:08 smithi163 bash[75425]: cluster 2023-10-09T09:27:07.616865+0000 mgr.y (mgr.34107) 622 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-10-09T09:27:08.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:08.725 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:08.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:08.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:08.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:08 smithi181 bash[59345]: cluster 2023-10-09T09:27:07.616865+0000 mgr.y (mgr.34107) 622 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-10-09T09:27:08.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:08.849 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 105786 105770 78.6827 54.1875 0.026786 0.0127032 2023-10-09T09:27:08.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:08.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:08.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:08.998 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:09.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:09.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:09.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:09.139 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:09.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:09.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:09.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:09.277 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:09.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:09.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:09.380 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:09.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:09.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:09.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:09.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:09.535 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:09.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:09.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:09.639 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:09.681 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:09.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:09.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:09.784 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:09.814 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:09.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:09.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:09.849 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 106516 106500 78.2938 45.625 0.0100004 0.0127666 2023-10-09T09:27:09.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:09.944 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:09.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:09.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:10.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:10.073 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:10.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:10.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:10.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:10.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:10.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:10.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:10.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:10.382 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:10.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:10.490 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:10.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:10.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:10.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:10.656 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:10.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:10.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:10.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:10.791 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:10.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:10.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:10.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:10 smithi181 bash[59345]: cluster 2023-10-09T09:27:09.617414+0000 mgr.y (mgr.34107) 623 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2023-10-09T09:27:10.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:10 smithi181 bash[59345]: audit 2023-10-09T09:27:10.361780+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:10.850 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 107348 107332 77.988 52 0.0111853 0.0128168 2023-10-09T09:27:10.894 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:10.920 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:10.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:10.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:10 smithi163 bash[75425]: cluster 2023-10-09T09:27:09.617414+0000 mgr.y (mgr.34107) 623 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2023-10-09T09:27:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:10 smithi163 bash[75425]: audit 2023-10-09T09:27:10.361780+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:10.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:10 smithi163 bash[78107]: cluster 2023-10-09T09:27:09.617414+0000 mgr.y (mgr.34107) 623 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2023-10-09T09:27:10.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:10 smithi163 bash[78107]: audit 2023-10-09T09:27:10.361780+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:11.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:11.063 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:11.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:11.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:11.166 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:11.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:11.202 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:11.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:11.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:11.329 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:11.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:11.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:11.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:11.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:11.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:11.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:11.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:11.591 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:11.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:11.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:11.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:11.733 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:11.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:11.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:11.837 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:11.849 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 108197 108181 77.7015 53.0625 0.0113976 0.0128644 2023-10-09T09:27:11.862 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:11.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:11.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:12.002 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:12.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:12.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:12.109 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:12.137 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:12.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:12.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:12.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:12.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:12.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:12.383 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:12.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:12.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:12.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:12.523 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:12.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:12.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:12.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:12.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:12.681 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:12.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:12.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:12.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:12.808 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:12.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:12.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:12.850 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 109062 109046 77.4329 54.0625 0.011817 0.0129089 2023-10-09T09:27:12.913 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:12.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:12 smithi181 bash[59345]: cluster 2023-10-09T09:27:11.617911+0000 mgr.y (mgr.34107) 624 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T09:27:12.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:12.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:12.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:12.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:12 smithi163 bash[75425]: cluster 2023-10-09T09:27:11.617911+0000 mgr.y (mgr.34107) 624 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T09:27:12.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:12 smithi163 bash[78107]: cluster 2023-10-09T09:27:11.617911+0000 mgr.y (mgr.34107) 624 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T09:27:13.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:13.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:13.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:13.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:13.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:13.214 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:13.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:13.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:13.317 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:13.359 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:13.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:13.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:13.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:13.495 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:27:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:27:13] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.33.4" 2023-10-09T09:27:13.502 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:13.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:13.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:13.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:13.646 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:13.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:13.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:13.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:13.775 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:13.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:13.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:13.850 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 109914 109898 77.1611 53.25 0.00679055 0.0129543 2023-10-09T09:27:13.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:13.909 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:13.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:13.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:14.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:14.041 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:14.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:14.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:14.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:14.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:14.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:14.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:14.284 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:14.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:14.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:14.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:14.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:14.445 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:14.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:14.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:14.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:14.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:14.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:14.679 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:14.705 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:14.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:14.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:14 smithi163 bash[75425]: cluster 2023-10-09T09:27:13.618402+0000 mgr.y (mgr.34107) 625 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s 2023-10-09T09:27:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:14 smithi163 bash[78107]: cluster 2023-10-09T09:27:13.618402+0000 mgr.y (mgr.34107) 625 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s 2023-10-09T09:27:14.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:14.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:14.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:14.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:14.850 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.0093 2023-10-09T09:27:14.850 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 110757 2023-10-09T09:27:14.850 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T09:27:14.851 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T09:27:14.851 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 76.9067 2023-10-09T09:27:14.851 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 29.0083 2023-10-09T09:27:14.851 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 183.125 2023-10-09T09:27:14.851 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 44.125 2023-10-09T09:27:14.851 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1230 2023-10-09T09:27:14.851 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 464.133 2023-10-09T09:27:14.851 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 2930 2023-10-09T09:27:14.851 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 706 2023-10-09T09:27:14.851 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.012998 2023-10-09T09:27:14.852 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0102054 2023-10-09T09:27:14.852 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 0.196141 2023-10-09T09:27:14.852 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.00141645 2023-10-09T09:27:14.857 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T09:27:14.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:14.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:14.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:15.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:14 smithi181 bash[59345]: cluster 2023-10-09T09:27:13.618402+0000 mgr.y (mgr.34107) 625 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s 2023-10-09T09:27:15.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:15.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:15.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:15.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:15.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:15.233 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:15.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:15.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:15.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:15.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:15.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:15.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:15.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:15.489 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:15.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:15.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:15.592 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:15.642 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:15.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:15.743 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:15.782 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:15.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:15.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:15.859 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:27:15.859 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:27:15.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:16.316 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:27:16.316 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:27:16.316 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:27:16.316 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:27:16.317 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:27:16.317 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:27:16.317 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:27:16.317 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:27:16.317 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:27:16.317 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:27:16.317 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:27:16.317 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:27:16.318 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:27:16.318 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:27:16.318 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:27:16.318 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:27:16.348 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:16.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:16.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:16.456 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:16.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:16.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:16.499 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T09:27:16.506 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:27:16.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:16 smithi163 bash[78107]: cluster 2023-10-09T09:27:15.619458+0000 mgr.y (mgr.34107) 626 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T09:27:16.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:16 smithi163 bash[78107]: audit 2023-10-09T09:27:15.673765+0000 mon.a (mon.0) 1429 : audit 1 from='client.? 172.21.15.163:0/3651071120' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:27:16.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:16 smithi163 bash[78107]: audit 2023-10-09T09:27:15.865705+0000 mon.a (mon.0) 1430 : audit 0 from='client.? 172.21.15.163:0/2251046214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:27:16.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:16 smithi163 bash[78107]: audit 2023-10-09T09:27:15.994441+0000 mon.c (mon.2) 49 : audit 1 from='client.? 172.21.15.163:0/2937733165' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:27:16.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:16 smithi163 bash[78107]: audit 2023-10-09T09:27:15.995592+0000 mon.a (mon.0) 1431 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:27:16.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:16 smithi163 bash[75425]: cluster 2023-10-09T09:27:15.619458+0000 mgr.y (mgr.34107) 626 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T09:27:16.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:16 smithi163 bash[75425]: audit 2023-10-09T09:27:15.673765+0000 mon.a (mon.0) 1429 : audit 1 from='client.? 172.21.15.163:0/3651071120' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:27:16.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:16 smithi163 bash[75425]: audit 2023-10-09T09:27:15.865705+0000 mon.a (mon.0) 1430 : audit 0 from='client.? 172.21.15.163:0/2251046214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:27:16.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:16 smithi163 bash[75425]: audit 2023-10-09T09:27:15.994441+0000 mon.c (mon.2) 49 : audit 1 from='client.? 172.21.15.163:0/2937733165' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:27:16.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:16 smithi163 bash[75425]: audit 2023-10-09T09:27:15.995592+0000 mon.a (mon.0) 1431 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:27:16.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:16.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:16.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:16.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:16.715 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:16.738 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:16.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:16.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:16 smithi181 bash[59345]: cluster 2023-10-09T09:27:15.619458+0000 mgr.y (mgr.34107) 626 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T09:27:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:16 smithi181 bash[59345]: audit 2023-10-09T09:27:15.673765+0000 mon.a (mon.0) 1429 : audit 1 from='client.? 172.21.15.163:0/3651071120' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:27:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:16 smithi181 bash[59345]: audit 2023-10-09T09:27:15.865705+0000 mon.a (mon.0) 1430 : audit 0 from='client.? 172.21.15.163:0/2251046214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:27:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:16 smithi181 bash[59345]: audit 2023-10-09T09:27:15.994441+0000 mon.c (mon.2) 49 : audit 1 from='client.? 172.21.15.163:0/2937733165' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:27:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:16 smithi181 bash[59345]: audit 2023-10-09T09:27:15.995592+0000 mon.a (mon.0) 1431 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:27:16.842 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:16.865 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:16.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:16.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:16.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:16.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:17.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:17.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:17.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:17.137 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:17.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:17.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:17.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:17.265 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:17.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:17.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:17.371 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:17.407 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:17.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:17.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:17.516 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:17.548 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:17.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:17.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:17 smithi181 bash[59345]: cluster 2023-10-09T09:27:16.472664+0000 mon.a (mon.0) 1432 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:27:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:17 smithi181 bash[59345]: cluster 2023-10-09T09:27:16.485516+0000 osd.7 (osd.7) 483 : cluster 3 failed to encode map e1087 with expected crc 2023-10-09T09:27:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:17 smithi181 bash[59345]: cluster 2023-10-09T09:27:16.487062+0000 osd.5 (osd.5) 631 : cluster 3 failed to encode map e1087 with expected crc 2023-10-09T09:27:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:17 smithi181 bash[59345]: audit 2023-10-09T09:27:16.490680+0000 mon.a (mon.0) 1433 : audit 1 from='client.? 172.21.15.163:0/3651071120' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:27:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:17 smithi181 bash[59345]: audit 2023-10-09T09:27:16.490798+0000 mon.a (mon.0) 1434 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:27:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:17 smithi181 bash[59345]: cluster 2023-10-09T09:27:16.491194+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e1087: 8 total, 8 up, 8 in 2023-10-09T09:27:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:17 smithi181 bash[59345]: cluster 2023-10-09T09:27:16.498577+0000 osd.4 (osd.4) 518 : cluster 3 failed to encode map e1087 with expected crc 2023-10-09T09:27:17.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:17.664 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[75425]: cluster 2023-10-09T09:27:16.472664+0000 mon.a (mon.0) 1432 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:27:17.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[75425]: cluster 2023-10-09T09:27:16.485516+0000 osd.7 (osd.7) 483 : cluster 3 failed to encode map e1087 with expected crc 2023-10-09T09:27:17.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[75425]: cluster 2023-10-09T09:27:16.487062+0000 osd.5 (osd.5) 631 : cluster 3 failed to encode map e1087 with expected crc 2023-10-09T09:27:17.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[75425]: audit 2023-10-09T09:27:16.490680+0000 mon.a (mon.0) 1433 : audit 1 from='client.? 172.21.15.163:0/3651071120' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:27:17.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[75425]: audit 2023-10-09T09:27:16.490798+0000 mon.a (mon.0) 1434 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:27:17.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[75425]: cluster 2023-10-09T09:27:16.491194+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e1087: 8 total, 8 up, 8 in 2023-10-09T09:27:17.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[75425]: cluster 2023-10-09T09:27:16.498577+0000 osd.4 (osd.4) 518 : cluster 3 failed to encode map e1087 with expected crc 2023-10-09T09:27:17.665 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[78107]: cluster 2023-10-09T09:27:16.472664+0000 mon.a (mon.0) 1432 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:27:17.665 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[78107]: cluster 2023-10-09T09:27:16.485516+0000 osd.7 (osd.7) 483 : cluster 3 failed to encode map e1087 with expected crc 2023-10-09T09:27:17.666 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[78107]: cluster 2023-10-09T09:27:16.487062+0000 osd.5 (osd.5) 631 : cluster 3 failed to encode map e1087 with expected crc 2023-10-09T09:27:17.666 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[78107]: audit 2023-10-09T09:27:16.490680+0000 mon.a (mon.0) 1433 : audit 1 from='client.? 172.21.15.163:0/3651071120' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:27:17.666 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[78107]: audit 2023-10-09T09:27:16.490798+0000 mon.a (mon.0) 1434 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:27:17.666 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[78107]: cluster 2023-10-09T09:27:16.491194+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e1087: 8 total, 8 up, 8 in 2023-10-09T09:27:17.666 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:17 smithi163 bash[78107]: cluster 2023-10-09T09:27:16.498577+0000 osd.4 (osd.4) 518 : cluster 3 failed to encode map e1087 with expected crc 2023-10-09T09:27:17.676 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:17.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:17.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:17.779 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:17.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:17.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:17.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:17.909 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:17.945 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:17.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:17.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:18.048 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:18.080 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:18.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:18.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:18.184 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:18.214 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:18.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:18.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:18.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T09:27:18.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:18.376 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:18.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:18.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:18.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:18.512 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:18.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:18.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:18.617 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:18.665 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:18.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:18.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:18.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:18.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:18 smithi181 bash[59345]: cluster 2023-10-09T09:27:17.619885+0000 mgr.y (mgr.34107) 627 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 46 MiB/s wr, 820 op/s 2023-10-09T09:27:18.797 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:18.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:18.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:18.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:18.913 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:18 smithi163 bash[75425]: cluster 2023-10-09T09:27:17.619885+0000 mgr.y (mgr.34107) 627 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 46 MiB/s wr, 820 op/s 2023-10-09T09:27:18.913 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:18 smithi163 bash[78107]: cluster 2023-10-09T09:27:17.619885+0000 mgr.y (mgr.34107) 627 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 46 MiB/s wr, 820 op/s 2023-10-09T09:27:18.944 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:18.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:19.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:19.078 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:19.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:19.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:19.184 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:19.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:19.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:19.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:19.316 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:19.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:19.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:19.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:19.435 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T09:27:19.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:19.492 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:19.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:19.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:19.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:19.625 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:19.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:19.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:19.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:19.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:19.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:19.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:19.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:19.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:19.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:19.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:19.995 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:20.020 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:20.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:20.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:20.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:20.150 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:20.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:20.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:20.253 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:20.283 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:20.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:20.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:20.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:20.412 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:20.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:20.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:20.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:27:20.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:20.550 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:20.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:20.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:20.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:20.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:20.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:20.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:20.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:20.822 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:20.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:20.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:20.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:20.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:20 smithi163 bash[78107]: cluster 2023-10-09T09:27:19.623349+0000 mgr.y (mgr.34107) 628 : cluster 0 pgmap v1323: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 827 op/s 2023-10-09T09:27:20.937 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:20 smithi163 bash[75425]: cluster 2023-10-09T09:27:19.623349+0000 mgr.y (mgr.34107) 628 : cluster 0 pgmap v1323: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 827 op/s 2023-10-09T09:27:20.959 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:20.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:20.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:21.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:20 smithi181 bash[59345]: cluster 2023-10-09T09:27:19.623349+0000 mgr.y (mgr.34107) 628 : cluster 0 pgmap v1323: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 827 op/s 2023-10-09T09:27:21.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:21.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:21.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:21.193 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:21.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:21.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:21.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:21.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:21.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:21.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:21.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:27:21.473 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:21.512 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:21.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:21.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:21.616 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:21.648 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:21.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:21.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:21.752 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:21.759 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:21 smithi163 bash[78107]: cluster 2023-10-09T09:27:21.623986+0000 mgr.y (mgr.34107) 629 : cluster 0 pgmap v1324: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 25 MiB/s wr, 829 op/s 2023-10-09T09:27:21.760 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:21 smithi163 bash[75425]: cluster 2023-10-09T09:27:21.623986+0000 mgr.y (mgr.34107) 629 : cluster 0 pgmap v1324: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 25 MiB/s wr, 829 op/s 2023-10-09T09:27:21.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:21.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:21.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:21.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:21.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:21.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:21.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:22.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:22.052 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:22.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:22.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:21 smithi181 bash[59345]: cluster 2023-10-09T09:27:21.623986+0000 mgr.y (mgr.34107) 629 : cluster 0 pgmap v1324: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 25 MiB/s wr, 829 op/s 2023-10-09T09:27:22.154 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:22.182 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:22.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:22.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:22.286 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:22.327 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:22.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:22.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:22.430 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:22.461 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:22.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:22.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:22.564 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:22.587 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:22.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:22.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:22.693 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:22.727 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:22.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:22.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:22.832 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:22.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:22.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:22.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:22.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:22.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:22.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:23.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:23.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:23.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:23.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:23.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:23.150 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:23.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:23.258 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:23.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:23.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:23.358 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:27:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:27:23] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.33.4" 2023-10-09T09:27:23.368 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:23.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:23.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:23.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:23.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:23.536 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:23.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:23.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:23.639 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:23.677 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:23.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:23.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:23.786 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:23.818 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:23.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:23.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:23.924 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:23.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:23.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:23.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:24.072 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:24.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:24.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:24.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:24.168 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:24.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:24.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:24.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:24.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:24.350 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:24.376 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:24.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:24.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:24.480 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:24.515 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:24.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:24.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:24.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:24.652 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:24.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:24.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:24.760 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:24.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:24 smithi163 bash[75425]: cluster 2023-10-09T09:27:23.624501+0000 mgr.y (mgr.34107) 630 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 831 op/s 2023-10-09T09:27:24.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:24 smithi163 bash[78107]: cluster 2023-10-09T09:27:23.624501+0000 mgr.y (mgr.34107) 630 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 831 op/s 2023-10-09T09:27:24.790 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:24.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:24.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:24.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:24.920 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:24.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:24.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:25.024 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:25.033 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:24 smithi181 bash[59345]: cluster 2023-10-09T09:27:23.624501+0000 mgr.y (mgr.34107) 630 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 831 op/s 2023-10-09T09:27:25.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:25.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:25.154 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:25.184 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:25.187 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:25.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:25.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:25.291 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:25.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:25.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:25.420 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:25.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:25.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:25.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:25.580 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:25.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:25.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:25.685 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:25.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:25.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:25.827 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:25.856 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:25.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:25.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:25.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:25 smithi163 bash[75425]: audit 2023-10-09T09:27:25.362079+0000 mon.a (mon.0) 1436 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:25.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:25 smithi163 bash[78107]: audit 2023-10-09T09:27:25.362079+0000 mon.a (mon.0) 1436 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:25.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:25.967 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:25 smithi181 bash[59345]: audit 2023-10-09T09:27:25.362079+0000 mon.a (mon.0) 1436 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:25.987 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:25.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:25.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:26.091 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:26.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:26.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:26.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:26.206 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:26.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:26.256 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:26.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:26.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:26.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:26.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:26.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:26.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:26.499 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:26.533 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:26.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:26.643 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:26.684 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:26.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:26.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:26.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:26.795 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:26 smithi181 bash[59345]: cluster 2023-10-09T09:27:25.625062+0000 mgr.y (mgr.34107) 631 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 827 op/s 2023-10-09T09:27:26.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:26.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:26.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:26.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:26.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:26 smithi163 bash[75425]: cluster 2023-10-09T09:27:25.625062+0000 mgr.y (mgr.34107) 631 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 827 op/s 2023-10-09T09:27:26.924 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:26 smithi163 bash[78107]: cluster 2023-10-09T09:27:25.625062+0000 mgr.y (mgr.34107) 631 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 827 op/s 2023-10-09T09:27:26.945 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:26.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:26.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:27.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:27.082 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:27.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:27.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:27.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:27.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:27.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:27.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:27.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:27.358 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:27.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:27.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:27.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:27.487 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:27.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:27.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:27.591 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:27.613 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:27.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:27.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:27.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:27.751 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:27.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:27.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:27 smithi181 bash[59345]: cluster 2023-10-09T09:27:27.625638+0000 mgr.y (mgr.34107) 632 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.5 MiB/s wr, 783 op/s 2023-10-09T09:27:27.859 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:27.889 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:27.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:27.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:27.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:27 smithi163 bash[75425]: cluster 2023-10-09T09:27:27.625638+0000 mgr.y (mgr.34107) 632 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.5 MiB/s wr, 783 op/s 2023-10-09T09:27:27.900 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:27 smithi163 bash[78107]: cluster 2023-10-09T09:27:27.625638+0000 mgr.y (mgr.34107) 632 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.5 MiB/s wr, 783 op/s 2023-10-09T09:27:27.992 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:28.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:28.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:28.130 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:28.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:28.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:28.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:28.259 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:28.284 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:28.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:28.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:28.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:28.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:28.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:28.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:28.523 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:28.550 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:28.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:28.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:28.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:28.685 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:28.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:28.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:28.789 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:28.827 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:28.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:28.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:28.933 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:28.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:28.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:28.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:29.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:29.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:29.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:29.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:29.193 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:29.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:29.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:29.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:29.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:29.351 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:29.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:29.457 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:29.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:29.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:29.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:29.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:29.606 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:29.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:29.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:29.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:29.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:29.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:29.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:29.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:29.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:29.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:29.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:29.971 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:29.992 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:29.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:29.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:30.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:30.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:30.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:30.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:30.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:30.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:30.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:30.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:30.393 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:30.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:30.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:30.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:30.517 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:30.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:30.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:30.620 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:30.644 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:30.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:30.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:30.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:30.812 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:30.813 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:30 smithi163 bash[75425]: cluster 2023-10-09T09:27:29.626164+0000 mgr.y (mgr.34107) 633 : cluster 0 pgmap v1328: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 874 KiB/s wr, 798 op/s 2023-10-09T09:27:30.813 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:30 smithi163 bash[78107]: cluster 2023-10-09T09:27:29.626164+0000 mgr.y (mgr.34107) 633 : cluster 0 pgmap v1328: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 874 KiB/s wr, 798 op/s 2023-10-09T09:27:30.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:30.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:30.915 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:30.940 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:30.943 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:30 smithi181 bash[59345]: cluster 2023-10-09T09:27:29.626164+0000 mgr.y (mgr.34107) 633 : cluster 0 pgmap v1328: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 874 KiB/s wr, 798 op/s 2023-10-09T09:27:30.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:30.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:31.044 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:31.079 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:31.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:31.187 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:31.211 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:31.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:31.318 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:31.351 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:31.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:31.457 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:31.484 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:31.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:31.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:31.587 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:31.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:31.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:31.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:31.724 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:31.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:31.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:31.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:31.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:31.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:31.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:31.997 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:32.020 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:32.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:32.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:32.124 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:32.147 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:32.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:32.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:32.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:32.276 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:32.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:32.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:32.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:32.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:32.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:32.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:32.507 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:32.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:32.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:32.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:32.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:32.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:32.667 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:32.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:32.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:32.776 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:32 smithi181 bash[59345]: cluster 2023-10-09T09:27:31.626899+0000 mgr.y (mgr.34107) 634 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T09:27:32.790 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:32.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:32.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:32.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:32.904 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:32 smithi163 bash[78107]: cluster 2023-10-09T09:27:31.626899+0000 mgr.y (mgr.34107) 634 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T09:27:32.904 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:32 smithi163 bash[75425]: cluster 2023-10-09T09:27:31.626899+0000 mgr.y (mgr.34107) 634 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T09:27:32.921 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:32.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:33.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:33.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:33.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:33.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:33.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:33.188 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:33.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:33.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:33.292 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:33.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:33.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:33.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:33.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:33.433 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:27:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:27:33] "GET /metrics HTTP/1.1" 200 36331 "" "Prometheus/2.33.4" 2023-10-09T09:27:33.451 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:33.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:33.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:33.554 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:33.581 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:33.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:33.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:33.688 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:33.721 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:33.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:33.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:33 smithi181 bash[59345]: cluster 2023-10-09T09:27:33.627403+0000 mgr.y (mgr.34107) 635 : cluster 0 pgmap v1330: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T09:27:33.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:33.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:33.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:33.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:33.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:33 smithi163 bash[75425]: cluster 2023-10-09T09:27:33.627403+0000 mgr.y (mgr.34107) 635 : cluster 0 pgmap v1330: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T09:27:33.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:33 smithi163 bash[78107]: cluster 2023-10-09T09:27:33.627403+0000 mgr.y (mgr.34107) 635 : cluster 0 pgmap v1330: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T09:27:33.990 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:33.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:33.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:34.096 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:34.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:34.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:34.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:34.227 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:34.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:34.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:34.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:34.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:34.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:34.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:34.498 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:34.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:34.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:34.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:34.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:34.666 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:34.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:34.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:34.772 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:34.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:34.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:34.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:34.916 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:34.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:34.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:34.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:35.065 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:35.111 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:35.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:35.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:35.214 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:35.241 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:35.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:35.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:35.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:35.367 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:35.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:35.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:35.470 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:35.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:35.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:35.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:35.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:35.628 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:35.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:35.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:35.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:35.782 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:35.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:35.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:35.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:35.897 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:27:35.897 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:27:35.898 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.898 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:127600 6.f 6:fb06560d:::benchmark_data_smithi163_107771_object16840:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1087)", 2023-10-09T09:27:35.898 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:35.872127+0000", 2023-10-09T09:27:35.898 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.025442187000000002, 2023-10-09T09:27:35.898 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.025472867, 2023-10-09T09:27:35.898 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:27:35.898 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:27:35.898 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:27:35.899 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:27:35.899 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:27:35.899 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 127600 2023-10-09T09:27:35.899 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.899 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:27:35.899 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.899 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:27:35.899 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.872127+0000", 2023-10-09T09:27:35.900 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:27:35.900 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.900 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.900 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:27:35.900 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.872127+0000", 2023-10-09T09:27:35.900 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:27:35.900 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.900 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.900 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:27:35.900 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.872126+0000", 2023-10-09T09:27:35.901 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:27:35.901 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.901 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.901 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:27:35.901 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.872133+0000", 2023-10-09T09:27:35.901 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6000000000000001e-06 2023-10-09T09:27:35.901 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.901 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.901 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:27:35.902 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.872150+0000", 2023-10-09T09:27:35.902 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6589999999999999e-05 2023-10-09T09:27:35.902 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.902 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.902 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:27:35.902 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.872157+0000", 2023-10-09T09:27:35.902 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3610000000000003e-06 2023-10-09T09:27:35.902 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.902 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.872908+0000", 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000750357 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.873077+0000", 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016910800000000001 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.903 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.873216+0000", 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013896700000000001 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.877421+0000", 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042050120000000002 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:27:35.904 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.882200+0000", 2023-10-09T09:27:35.905 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0047796569999999997 2023-10-09T09:27:35.905 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.905 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:27:35.905 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.905 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.905 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.906 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:127611 6.8 e1087/1076 6:15fa6dd7:::benchmark_data_smithi163_107771_object16851:head v 1087'8071, mlcod=1087'8068)", 2023-10-09T09:27:35.906 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:35.889708+0000", 2023-10-09T09:27:35.906 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0078607920000000001, 2023-10-09T09:27:35.906 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079490099999999994, 2023-10-09T09:27:35.906 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:27:35.907 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:27:35.907 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:27:35.907 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.907 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:27:35.907 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.889708+0000", 2023-10-09T09:27:35.908 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:27:35.908 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:27:35.908 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.889708+0000", 2023-10-09T09:27:35.908 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:27:35.908 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:27:35.909 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.889708+0000", 2023-10-09T09:27:35.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:27:35.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.910 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:27:35.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.889712+0000", 2023-10-09T09:27:35.911 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0029999999999996e-06 2023-10-09T09:27:35.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:27:35.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.889718+0000", 2023-10-09T09:27:35.911 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4720000000000003e-06 2023-10-09T09:27:35.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.912 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.912 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:27:35.912 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.889723+0000", 2023-10-09T09:27:35.912 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9110000000000001e-06 2023-10-09T09:27:35.912 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.912 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.912 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:27:35.913 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.889771+0000", 2023-10-09T09:27:35.913 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7975000000000001e-05 2023-10-09T09:27:35.913 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.913 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.913 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:27:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.889803+0000", 2023-10-09T09:27:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2385000000000002e-05 2023-10-09T09:27:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:27:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:127616 6.8 e1087/1076)", 2023-10-09T09:27:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:35.891659+0000", 2023-10-09T09:27:35.914 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0059107400000000003, 2023-10-09T09:27:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060337689999999996, 2023-10-09T09:27:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:27:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:27:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:27:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:27:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.891659+0000", 2023-10-09T09:27:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:27:35.915 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:27:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.891659+0000", 2023-10-09T09:27:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:27:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:27:35.916 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.891658+0000", 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.891664+0000", 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2600000000000002e-06 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.891678+0000", 2023-10-09T09:27:35.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3848e-05 2023-10-09T09:27:35.918 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.918 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.918 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:27:35.918 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.891685+0000", 2023-10-09T09:27:35.918 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4919999999999998e-06 2023-10-09T09:27:35.918 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.918 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:27:35.918 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.919 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:127615 6.0 e1087/1076)", 2023-10-09T09:27:35.919 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:35.893012+0000", 2023-10-09T09:27:35.919 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0045575219999999996, 2023-10-09T09:27:35.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0047072879999999996, 2023-10-09T09:27:35.919 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:27:35.919 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:27:35.919 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:27:35.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:27:35.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.893012+0000", 2023-10-09T09:27:35.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:27:35.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.921 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:27:35.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.893012+0000", 2023-10-09T09:27:35.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:27:35.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.922 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:27:35.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.893011+0000", 2023-10-09T09:27:35.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:27:35.922 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.922 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:27:35.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.893019+0000", 2023-10-09T09:27:35.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8779999999999998e-06 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.893036+0000", 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7325000000000001e-05 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.893044+0000", 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0150000000000003e-06 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.923 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:27:35.924 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.924 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:127619 6.1 e1087/1076 6:86b3df47:::benchmark_data_smithi163_107771_object16859:head v 1087'7887, mlcod=1087'7885)", 2023-10-09T09:27:35.924 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:35.895911+0000", 2023-10-09T09:27:35.924 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.001658484, 2023-10-09T09:27:35.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0018382349999999999, 2023-10-09T09:27:35.925 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:27:35.925 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:27:35.925 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:27:35.925 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.925 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:27:35.925 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.895911+0000", 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.895911+0000", 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.895910+0000", 2023-10-09T09:27:35.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.895917+0000", 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0029999999999997e-06 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.895929+0000", 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1744e-05 2023-10-09T09:27:35.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.895937+0000", 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1890000000000007e-06 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.896008+0000", 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1588000000000006e-05 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:27:35.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:27:35.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:27:35.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:35.896031+0000", 2023-10-09T09:27:35.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2204e-05 2023-10-09T09:27:35.929 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.929 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:27:35.929 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.929 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:27:35.929 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T09:27:35.929 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 5 2023-10-09T09:27:35.930 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:27:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:35.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:35.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:36.032 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:36.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:36.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:36.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:36.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:36.198 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:36.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:36.301 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:36.332 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:36.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:36.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:36.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:36.471 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:36.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:36 smithi181 bash[59345]: cluster 2023-10-09T09:27:35.627957+0000 mgr.y (mgr.34107) 636 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-10-09T09:27:36.573 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:36.599 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:36.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:36.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:36.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:36.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:36.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:36 smithi163 bash[78107]: cluster 2023-10-09T09:27:35.627957+0000 mgr.y (mgr.34107) 636 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-10-09T09:27:36.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:36.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:36.731 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:36 smithi163 bash[75425]: cluster 2023-10-09T09:27:35.627957+0000 mgr.y (mgr.34107) 636 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-10-09T09:27:36.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:36.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:36.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:36.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:36.969 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:36.990 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:27:37.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:37.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:37.029 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:27:37.074 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:27:37.109 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:37.123 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:27:37.131 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:37.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:37.159 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:27:37.192 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:27:37.229 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:27:37.234 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:37.258 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:37.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:37.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:37.266 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:27:37.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:27:37.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:27:37.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:27:37.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:27:37.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:27:37.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:27:37.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:27:37.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:27:37.365 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:37.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:37.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:37.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:37.534 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:37.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:37.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:37 smithi181 bash[59345]: audit 2023-10-09T09:27:36.909634+0000 mon.a (mon.0) 1437 : audit 0 from='client.? 172.21.15.163:0/998063022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:27:37.636 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:37.661 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:37.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:37 smithi163 bash[78107]: audit 2023-10-09T09:27:36.909634+0000 mon.a (mon.0) 1437 : audit 0 from='client.? 172.21.15.163:0/998063022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:27:37.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:37 smithi163 bash[75425]: audit 2023-10-09T09:27:36.909634+0000 mon.a (mon.0) 1437 : audit 0 from='client.? 172.21.15.163:0/998063022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:27:37.766 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:37.968 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:37.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:37.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:38.071 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:38.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:38.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:38.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:38.225 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:38.253 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:38.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:38.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:38.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:38.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:38.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:38.488 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:38.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:38 smithi163 bash[75425]: audit 2023-10-09T09:27:37.616537+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.163:0/3033077864' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:38.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:38 smithi163 bash[75425]: audit 2023-10-09T09:27:37.617313+0000 mon.a (mon.0) 1438 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:38.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:38 smithi163 bash[75425]: cluster 2023-10-09T09:27:37.628604+0000 mgr.y (mgr.34107) 637 : cluster 0 pgmap v1332: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-10-09T09:27:38.515 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:38.515 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T09:27:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:38.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:38.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:38.629 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:38 smithi181 bash[59345]: audit 2023-10-09T09:27:37.616537+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.163:0/3033077864' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:38.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:38 smithi181 bash[59345]: audit 2023-10-09T09:27:37.617313+0000 mon.a (mon.0) 1438 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:38.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:38 smithi181 bash[59345]: cluster 2023-10-09T09:27:37.628604+0000 mgr.y (mgr.34107) 637 : cluster 0 pgmap v1332: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-10-09T09:27:38.646 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:38.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:38.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:38 smithi163 bash[78107]: audit 2023-10-09T09:27:37.616537+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.163:0/3033077864' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:38 smithi163 bash[78107]: audit 2023-10-09T09:27:37.617313+0000 mon.a (mon.0) 1438 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:27:38.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:38 smithi163 bash[78107]: cluster 2023-10-09T09:27:37.628604+0000 mgr.y (mgr.34107) 637 : cluster 0 pgmap v1332: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-10-09T09:27:38.752 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:38.781 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:38.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:38.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:38.884 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:38.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:38.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:38.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:38.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T09:27:39.018 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:39.044 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:39.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:39.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:39.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:39.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:39.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:39.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:39.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:39.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:39.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:39.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:39.447 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:39.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:39.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:39.531 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:27:39.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:39.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[78107]: cluster 2023-10-09T09:27:38.502711+0000 osd.5 (osd.5) 632 : cluster 3 failed to encode map e1088 with expected crc 2023-10-09T09:27:39.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[78107]: cluster 2023-10-09T09:27:38.505889+0000 osd.7 (osd.7) 484 : cluster 3 failed to encode map e1088 with expected crc 2023-10-09T09:27:39.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[78107]: audit 2023-10-09T09:27:38.506235+0000 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:27:39.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[78107]: cluster 2023-10-09T09:27:38.506565+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e1088: 8 total, 8 up, 8 in 2023-10-09T09:27:39.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[78107]: audit 2023-10-09T09:27:39.462069+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.163:0/1357838592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:27:39.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[78107]: audit 2023-10-09T09:27:39.463016+0000 mon.a (mon.0) 1441 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:27:39.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[75425]: cluster 2023-10-09T09:27:38.502711+0000 osd.5 (osd.5) 632 : cluster 3 failed to encode map e1088 with expected crc 2023-10-09T09:27:39.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[75425]: cluster 2023-10-09T09:27:38.505889+0000 osd.7 (osd.7) 484 : cluster 3 failed to encode map e1088 with expected crc 2023-10-09T09:27:39.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[75425]: audit 2023-10-09T09:27:38.506235+0000 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:27:39.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[75425]: cluster 2023-10-09T09:27:38.506565+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e1088: 8 total, 8 up, 8 in 2023-10-09T09:27:39.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[75425]: audit 2023-10-09T09:27:39.462069+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.163:0/1357838592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:27:39.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:39 smithi163 bash[75425]: audit 2023-10-09T09:27:39.463016+0000 mon.a (mon.0) 1441 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:27:39.580 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:39.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:39.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:39.687 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:39.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:39 smithi181 bash[59345]: cluster 2023-10-09T09:27:38.502711+0000 osd.5 (osd.5) 632 : cluster 3 failed to encode map e1088 with expected crc 2023-10-09T09:27:39.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:39 smithi181 bash[59345]: cluster 2023-10-09T09:27:38.505889+0000 osd.7 (osd.7) 484 : cluster 3 failed to encode map e1088 with expected crc 2023-10-09T09:27:39.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:39 smithi181 bash[59345]: audit 2023-10-09T09:27:38.506235+0000 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:27:39.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:39 smithi181 bash[59345]: cluster 2023-10-09T09:27:38.506565+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e1088: 8 total, 8 up, 8 in 2023-10-09T09:27:39.697 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:39 smithi181 bash[59345]: audit 2023-10-09T09:27:39.462069+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.163:0/1357838592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:27:39.697 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:39 smithi181 bash[59345]: audit 2023-10-09T09:27:39.463016+0000 mon.a (mon.0) 1441 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:27:39.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:39.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:39.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:39.829 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:39.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:39.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:39.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:39.952 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:39.973 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:39.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:39.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:40.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:40.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:40.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:40.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:40.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:40.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:40.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:40.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:40.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:40.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:40.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:40.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:40.512 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:40.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:40.616 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:40.652 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:40.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:40.756 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:40.782 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:40.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:40.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:40 smithi181 bash[59345]: cluster 2023-10-09T09:27:39.509703+0000 mon.a (mon.0) 1442 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:27:40.790 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:40 smithi181 bash[59345]: cluster 2023-10-09T09:27:39.517017+0000 osd.5 (osd.5) 633 : cluster 3 failed to encode map e1089 with expected crc 2023-10-09T09:27:40.790 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:40 smithi181 bash[59345]: cluster 2023-10-09T09:27:39.517767+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e1089 with expected crc 2023-10-09T09:27:40.790 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:40 smithi181 bash[59345]: audit 2023-10-09T09:27:39.519775+0000 mon.a (mon.0) 1443 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:27:40.790 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:40 smithi181 bash[59345]: cluster 2023-10-09T09:27:39.519965+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e1089: 8 total, 8 up, 8 in 2023-10-09T09:27:40.790 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:40 smithi181 bash[59345]: cluster 2023-10-09T09:27:39.629050+0000 mgr.y (mgr.34107) 638 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T09:27:40.790 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:40 smithi181 bash[59345]: audit 2023-10-09T09:27:40.363189+0000 mon.a (mon.0) 1445 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:40.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:40.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[75425]: cluster 2023-10-09T09:27:39.509703+0000 mon.a (mon.0) 1442 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:27:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[75425]: cluster 2023-10-09T09:27:39.517017+0000 osd.5 (osd.5) 633 : cluster 3 failed to encode map e1089 with expected crc 2023-10-09T09:27:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[75425]: cluster 2023-10-09T09:27:39.517767+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e1089 with expected crc 2023-10-09T09:27:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[75425]: audit 2023-10-09T09:27:39.519775+0000 mon.a (mon.0) 1443 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:27:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[75425]: cluster 2023-10-09T09:27:39.519965+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e1089: 8 total, 8 up, 8 in 2023-10-09T09:27:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[75425]: cluster 2023-10-09T09:27:39.629050+0000 mgr.y (mgr.34107) 638 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T09:27:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[75425]: audit 2023-10-09T09:27:40.363189+0000 mon.a (mon.0) 1445 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:40.900 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[78107]: cluster 2023-10-09T09:27:39.509703+0000 mon.a (mon.0) 1442 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:27:40.900 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[78107]: cluster 2023-10-09T09:27:39.517017+0000 osd.5 (osd.5) 633 : cluster 3 failed to encode map e1089 with expected crc 2023-10-09T09:27:40.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[78107]: cluster 2023-10-09T09:27:39.517767+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e1089 with expected crc 2023-10-09T09:27:40.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[78107]: audit 2023-10-09T09:27:39.519775+0000 mon.a (mon.0) 1443 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:27:40.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[78107]: cluster 2023-10-09T09:27:39.519965+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e1089: 8 total, 8 up, 8 in 2023-10-09T09:27:40.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[78107]: cluster 2023-10-09T09:27:39.629050+0000 mgr.y (mgr.34107) 638 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T09:27:40.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:40 smithi163 bash[78107]: audit 2023-10-09T09:27:40.363189+0000 mon.a (mon.0) 1445 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:40.919 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:40.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:40.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:41.025 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:41.050 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:41.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:41.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:41.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:41.198 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:41.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:41.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:41.304 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:41.332 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:41.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:41.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T09:27:41.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:41.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:41.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:41.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:41.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:41.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:41.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:41.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:41.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:41.757 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:41.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:41.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:41.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:41.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:41.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:41.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:42.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:42.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:42.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:42.163 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:42.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:42.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:42.184 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:27:42.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:42.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:42.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:42.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:42.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:42.433 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:42.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:42.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:42.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:42.574 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:42.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:42.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:42.679 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:42.684 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:42 smithi181 bash[59345]: cluster 2023-10-09T09:27:41.629682+0000 mgr.y (mgr.34107) 639 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-10-09T09:27:42.703 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:42.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:42.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:42.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:42.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:42.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:42.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:42.938 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:42.962 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:42 smithi163 bash[75425]: cluster 2023-10-09T09:27:41.629682+0000 mgr.y (mgr.34107) 639 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-10-09T09:27:42.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:42 smithi163 bash[78107]: cluster 2023-10-09T09:27:41.629682+0000 mgr.y (mgr.34107) 639 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-10-09T09:27:42.990 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:42.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:42.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:43.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:43.132 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:43.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:43.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:43.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:43.269 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:43.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:43.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:43.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:43.409 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:43.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:43.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:43.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:27:43.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:43.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:43.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:43.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:43.646 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:43.656 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:27:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:27:43] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.33.4" 2023-10-09T09:27:43.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:43.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:43.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:43.788 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:43.824 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:43.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:43.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:43.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:43.951 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:43.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:43.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:44.059 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:44.084 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:44.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:44.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:44.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:44.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:44.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:44.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:44.318 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:44.343 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:44.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:44.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:44.445 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:44.473 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:44.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:44.578 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:44.612 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:44.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:44.717 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:44.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:44.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:44.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:44.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:44 smithi181 bash[59345]: cluster 2023-10-09T09:27:43.630207+0000 mgr.y (mgr.34107) 640 : cluster 0 pgmap v1337: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-09T09:27:44.876 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:44.909 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:44.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:44.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:44.964 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:44 smithi163 bash[75425]: cluster 2023-10-09T09:27:43.630207+0000 mgr.y (mgr.34107) 640 : cluster 0 pgmap v1337: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-09T09:27:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:44 smithi163 bash[78107]: cluster 2023-10-09T09:27:43.630207+0000 mgr.y (mgr.34107) 640 : cluster 0 pgmap v1337: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-09T09:27:45.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:45.037 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:45.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:45.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:45.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:45.168 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:45.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:45.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:45.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:45.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:45.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:45.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:45.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:45.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:45.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:45.547 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:45.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:45.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:45.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:45.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:45.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:45.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:45.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:45.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:45.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:45.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:45.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:45.963 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:45.990 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:45.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:45.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:46.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:46.126 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:46.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:46.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:46.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:46.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:46.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:46.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:46.388 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:46.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:46.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:46.492 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:46.524 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:46.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:46.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:46.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:46.642 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:46 smithi181 bash[59345]: cluster 2023-10-09T09:27:45.630760+0000 mgr.y (mgr.34107) 641 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T09:27:46.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:46.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:46.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:46 smithi163 bash[75425]: cluster 2023-10-09T09:27:45.630760+0000 mgr.y (mgr.34107) 641 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T09:27:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:46 smithi163 bash[78107]: cluster 2023-10-09T09:27:45.630760+0000 mgr.y (mgr.34107) 641 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T09:27:46.769 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:46.803 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:46.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:46.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:46.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:46.932 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:46.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:46.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:47.012 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:47.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:47.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:47.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:47.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:47.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:47.220 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:47.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:47.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:47.324 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:47.348 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:47.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:47.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:47.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:47.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:47.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:47.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:47.572 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:47.603 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:47.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:47.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:47.709 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:47.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:47.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:47.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:47.847 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:47.883 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:47.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:47.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:47.986 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:48.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:48.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:48.022 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:27:48.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:48.152 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:48.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:48.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:48.255 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:48.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:48.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:48.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:48.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:48.414 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:48.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:48.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:48.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:48.562 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:48.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:48.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:48.667 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:48.698 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:48.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:48.803 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:48.824 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:48.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:48.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:48.899 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:48 smithi163 bash[78107]: cluster 2023-10-09T09:27:47.631376+0000 mgr.y (mgr.34107) 642 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T09:27:48.899 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:48 smithi163 bash[75425]: cluster 2023-10-09T09:27:47.631376+0000 mgr.y (mgr.34107) 642 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T09:27:48.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:48.936 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:48 smithi181 bash[59345]: cluster 2023-10-09T09:27:47.631376+0000 mgr.y (mgr.34107) 642 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T09:27:48.959 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:48.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:48.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:49.064 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:49.102 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:49.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:49.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:49.206 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:49.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:49.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:49.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:49.353 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:49.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:49.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:49.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:49.511 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:49.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:49.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:49.616 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:49.644 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:49.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:49.748 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:49.787 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:49.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:49.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:49.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:49.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:49.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:50.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:50.040 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:50.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:50.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:50.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:50.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:50.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:50.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:50.282 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:50.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:50.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:50.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:50.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:50.440 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:50.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:50.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:50.547 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:50.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:50.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:50.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:50.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:50.712 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:50.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:50.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:50.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:50.824 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:50 smithi163 bash[75425]: cluster 2023-10-09T09:27:49.634051+0000 mgr.y (mgr.34107) 643 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-10-09T09:27:50.825 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:50 smithi163 bash[78107]: cluster 2023-10-09T09:27:49.634051+0000 mgr.y (mgr.34107) 643 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-10-09T09:27:50.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:50.851 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:50.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:50.952 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:50.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:50.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:50.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:50 smithi181 bash[59345]: cluster 2023-10-09T09:27:49.634051+0000 mgr.y (mgr.34107) 643 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-10-09T09:27:51.091 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:51.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:51.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:51.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:51.227 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:51.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:51.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:51.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:51.363 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:51.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:51.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:51.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:51.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:51.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:51.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:51.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:51.645 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:51.670 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:51.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:51.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:51.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:51.807 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:51.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:51.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:51.931 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:51.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:51.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:52.034 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:52.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:52.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:52.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:52.184 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:52.220 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:52.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:52.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:52.329 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:52.350 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:52.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:52.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:52.453 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:52.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:52.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:52.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:52.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:52.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:52.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:52.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:52.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:52 smithi163 bash[75425]: cluster 2023-10-09T09:27:51.634701+0000 mgr.y (mgr.34107) 644 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-10-09T09:27:52.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:52.771 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:52.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:52.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:52.883 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:52.892 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:52 smithi181 bash[59345]: cluster 2023-10-09T09:27:51.634701+0000 mgr.y (mgr.34107) 644 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-10-09T09:27:52.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:52.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:52.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:52.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:52 smithi163 bash[78107]: cluster 2023-10-09T09:27:51.634701+0000 mgr.y (mgr.34107) 644 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-10-09T09:27:53.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:53.039 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:53.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:53.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:53.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:53.182 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:53.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:53.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:53.287 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:53.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:53.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:53.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:53.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:53.457 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:53.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:53.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:53.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:53.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:53.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:53.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:53.699 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:27:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:27:53] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.33.4" 2023-10-09T09:27:53.725 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:53.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:53.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:53.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:53.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:53.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:53.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:53.963 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:53.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:53.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:53.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:54.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:54.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:54.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:54.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:54.224 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:54.252 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:54.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:54.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:54.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:54.381 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:54.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:54.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:54.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:54.511 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:54.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:54.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:54.614 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:54.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:54.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:54.762 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:54.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:54.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:54.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:54.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:54 smithi181 bash[59345]: cluster 2023-10-09T09:27:53.635195+0000 mgr.y (mgr.34107) 645 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T09:27:54.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:54.914 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:54 smithi163 bash[75425]: cluster 2023-10-09T09:27:53.635195+0000 mgr.y (mgr.34107) 645 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T09:27:54.914 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:54 smithi163 bash[78107]: cluster 2023-10-09T09:27:53.635195+0000 mgr.y (mgr.34107) 645 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T09:27:54.939 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:54.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:54.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:55.043 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:55.066 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:55.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:55.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:55.169 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:55.196 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:55.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:55.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:55.299 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:55.334 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:55.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:55.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:55.440 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:55.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:55.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:55.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:55.610 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:55.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:55.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:55.715 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:55.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:55.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:55.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:55.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:55.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:55.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:55.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:55 smithi163 bash[78107]: audit 2023-10-09T09:27:55.363187+0000 mon.a (mon.0) 1446 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:55 smithi163 bash[78107]: cluster 2023-10-09T09:27:55.635779+0000 mgr.y (mgr.34107) 646 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-09T09:27:55.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:55 smithi163 bash[75425]: audit 2023-10-09T09:27:55.363187+0000 mon.a (mon.0) 1446 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:55 smithi163 bash[75425]: cluster 2023-10-09T09:27:55.635779+0000 mgr.y (mgr.34107) 646 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-09T09:27:55.980 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:55.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:55 smithi181 bash[59345]: audit 2023-10-09T09:27:55.363187+0000 mon.a (mon.0) 1446 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:27:55.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:55 smithi181 bash[59345]: cluster 2023-10-09T09:27:55.635779+0000 mgr.y (mgr.34107) 646 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-09T09:27:56.009 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:56.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:56.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:56.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:56.141 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:56.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:56.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:56.247 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:56.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:56.277 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:56.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:56.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:56.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:56.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:56.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:56.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:56.537 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:56.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:56.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:56.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:56.666 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:56.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:56.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:56.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:56.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:56.908 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:56.944 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:56.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:56.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:57.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:57.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:57.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:57.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:57.193 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:57.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:57.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:57.324 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:57.346 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:57.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:57.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:57.451 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:57.486 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:57.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:57.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:57.536 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:27:57.537 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:27:57.587 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:57.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:57.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:57.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:57.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:57.748 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:27:57.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:57.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:27:57.852 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:57.879 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:27:57.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:57.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:27:57.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:58.004 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:58.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:58.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:58.115 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:58.147 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:58.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:58.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:58.254 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:58.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:58.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:58.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:58.398 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:58.420 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:58.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:58.528 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:58.559 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:58.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:58.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:58.665 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:58.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:27:58.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:58.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:27:58.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:58.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:58 smithi163 bash[78107]: cluster 2023-10-09T09:27:57.636411+0000 mgr.y (mgr.34107) 647 : cluster 0 pgmap v1344: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T09:27:58.805 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:58 smithi163 bash[75425]: cluster 2023-10-09T09:27:57.636411+0000 mgr.y (mgr.34107) 647 : cluster 0 pgmap v1344: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T09:27:58.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:58.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:58.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:58.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:58.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:58 smithi181 bash[59345]: cluster 2023-10-09T09:27:57.636411+0000 mgr.y (mgr.34107) 647 : cluster 0 pgmap v1344: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T09:27:58.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:58.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:58.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:59.056 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:59.092 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:27:59.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:59.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:27:59.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:59.235 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:59.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:59.342 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:59.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:59.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:27:59.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:59.524 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:27:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:59.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:27:59.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:59.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:27:59.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:59.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:27:59.708 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T09:27:59.763 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:59 smithi163 bash[78107]: audit 2023-10-09T09:27:59.262235+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.163:0/840437948' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:59.764 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:27:59 smithi163 bash[78107]: audit 2023-10-09T09:27:59.263893+0000 mon.a (mon.0) 1447 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:59.764 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:59 smithi163 bash[75425]: audit 2023-10-09T09:27:59.262235+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.163:0/840437948' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:59.764 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:27:59 smithi163 bash[75425]: audit 2023-10-09T09:27:59.263893+0000 mon.a (mon.0) 1447 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:59.765 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:59.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:27:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:59.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:27:59.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:59.904 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:59 smithi181 bash[59345]: audit 2023-10-09T09:27:59.262235+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.163:0/840437948' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:59.904 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:27:59 smithi181 bash[59345]: audit 2023-10-09T09:27:59.263893+0000 mon.a (mon.0) 1447 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:27:59.921 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:27:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:59.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:00.024 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:00.049 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:00.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:00.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:00.155 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:00.183 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:00.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:00.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:00.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:00.323 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:00.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:00.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:00.430 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:00.463 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:00.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:00.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:00.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T09:28:00.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:00.612 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:00.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:00.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:00.706 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[75425]: cluster 2023-10-09T09:27:59.639248+0000 mgr.y (mgr.34107) 648 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T09:28:00.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[75425]: cluster 2023-10-09T09:27:59.695904+0000 osd.5 (osd.5) 634 : cluster 3 failed to encode map e1090 with expected crc 2023-10-09T09:28:00.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[75425]: audit 2023-10-09T09:27:59.698777+0000 mon.a (mon.0) 1448 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:28:00.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[75425]: cluster 2023-10-09T09:27:59.698856+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e1090: 8 total, 8 up, 8 in 2023-10-09T09:28:00.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[75425]: cluster 2023-10-09T09:27:59.702738+0000 osd.4 (osd.4) 519 : cluster 3 failed to encode map e1090 with expected crc 2023-10-09T09:28:00.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[75425]: audit 2023-10-09T09:28:00.657970+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.163:0/170861122' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:28:00.709 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:28:00.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:00.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:00.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:00.760 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:28:00.802 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:28:00.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:28:00.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:00.886 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:28:00.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:00.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:00.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:00.918 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:28:00.939 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:28:00.958 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:28:00.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:28:00.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:28:00.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:28:00.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:28:00.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:28:00.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:28:00.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:28:00.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:28:00.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[78107]: cluster 2023-10-09T09:27:59.639248+0000 mgr.y (mgr.34107) 648 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T09:28:00.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[78107]: cluster 2023-10-09T09:27:59.695904+0000 osd.5 (osd.5) 634 : cluster 3 failed to encode map e1090 with expected crc 2023-10-09T09:28:00.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[78107]: audit 2023-10-09T09:27:59.698777+0000 mon.a (mon.0) 1448 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:28:00.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[78107]: cluster 2023-10-09T09:27:59.698856+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e1090: 8 total, 8 up, 8 in 2023-10-09T09:28:00.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[78107]: cluster 2023-10-09T09:27:59.702738+0000 osd.4 (osd.4) 519 : cluster 3 failed to encode map e1090 with expected crc 2023-10-09T09:28:00.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:00 smithi163 bash[78107]: audit 2023-10-09T09:28:00.657970+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.163:0/170861122' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:28:00.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:01.032 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:01.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:01.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:00 smithi181 bash[59345]: cluster 2023-10-09T09:27:59.639248+0000 mgr.y (mgr.34107) 648 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T09:28:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:00 smithi181 bash[59345]: cluster 2023-10-09T09:27:59.695904+0000 osd.5 (osd.5) 634 : cluster 3 failed to encode map e1090 with expected crc 2023-10-09T09:28:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:00 smithi181 bash[59345]: audit 2023-10-09T09:27:59.698777+0000 mon.a (mon.0) 1448 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:28:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:00 smithi181 bash[59345]: cluster 2023-10-09T09:27:59.698856+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e1090: 8 total, 8 up, 8 in 2023-10-09T09:28:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:00 smithi181 bash[59345]: cluster 2023-10-09T09:27:59.702738+0000 osd.4 (osd.4) 519 : cluster 3 failed to encode map e1090 with expected crc 2023-10-09T09:28:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:00 smithi181 bash[59345]: audit 2023-10-09T09:28:00.657970+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.163:0/170861122' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:28:01.137 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:01.164 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:01.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:01.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:01.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:01.297 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:01.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:01.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:01.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:01.425 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:01.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:01.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:01.628 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:01.656 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:01.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:01.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:01.767 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:01.795 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:01.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:01.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:01.897 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:01.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:01.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:01.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:01 smithi163 bash[75425]: cluster 2023-10-09T09:28:01.639904+0000 mgr.y (mgr.34107) 649 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-10-09T09:28:01.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:01 smithi163 bash[78107]: cluster 2023-10-09T09:28:01.639904+0000 mgr.y (mgr.34107) 649 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-10-09T09:28:02.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:02.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:02.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:02.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:02.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:01 smithi181 bash[59345]: cluster 2023-10-09T09:28:01.639904+0000 mgr.y (mgr.34107) 649 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-10-09T09:28:02.115 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T09:28:02.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:28:02.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:02.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:02.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:02.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:02.309 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:02.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:02.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:02.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:02.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:02.477 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:02.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:02.580 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:02.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:02.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:02.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:02.710 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:02.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:02 smithi181 bash[59345]: audit 2023-10-09T09:28:02.580013+0000 mon.a (mon.0) 1450 : audit 1 from='client.? 172.21.15.163:0/2517168134' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:28:02.735 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:28:02.745 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:02.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:02.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:02.852 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:02.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:02 smithi163 bash[75425]: audit 2023-10-09T09:28:02.580013+0000 mon.a (mon.0) 1450 : audit 1 from='client.? 172.21.15.163:0/2517168134' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:28:02.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:02 smithi163 bash[78107]: audit 2023-10-09T09:28:02.580013+0000 mon.a (mon.0) 1450 : audit 1 from='client.? 172.21.15.163:0/2517168134' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:28:02.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:02.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:03.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:03.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:03.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:03.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:03.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:03.177 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:03.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:03.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:03.286 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:03.313 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:03.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:03.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:03.416 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:03.444 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:03.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:03.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:03.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:28:03] "GET /metrics HTTP/1.1" 200 36331 "" "Prometheus/2.33.4" 2023-10-09T09:28:03.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:03.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:03.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:03.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:03.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:03.709 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:03.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:03.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:03.815 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:03.842 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:03.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:03.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:03.946 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:03.954 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:03 smithi181 bash[59345]: cluster 2023-10-09T09:28:02.710231+0000 mon.a (mon.0) 1451 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:28:03.955 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:03 smithi181 bash[59345]: cluster 2023-10-09T09:28:02.710267+0000 mon.a (mon.0) 1452 : cluster 1 Cluster is now healthy 2023-10-09T09:28:03.955 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:03 smithi181 bash[59345]: audit 2023-10-09T09:28:02.725196+0000 mon.a (mon.0) 1453 : audit 1 from='client.? 172.21.15.163:0/2517168134' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:28:03.955 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:03 smithi181 bash[59345]: cluster 2023-10-09T09:28:02.725322+0000 mon.a (mon.0) 1454 : cluster 0 osdmap e1091: 8 total, 8 up, 8 in 2023-10-09T09:28:03.955 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:03 smithi181 bash[59345]: cluster 2023-10-09T09:28:02.726294+0000 osd.5 (osd.5) 635 : cluster 3 failed to encode map e1091 with expected crc 2023-10-09T09:28:03.955 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:03 smithi181 bash[59345]: cluster 2023-10-09T09:28:02.727922+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e1091 with expected crc 2023-10-09T09:28:03.955 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:03 smithi181 bash[59345]: cluster 2023-10-09T09:28:03.640303+0000 mgr.y (mgr.34107) 650 : cluster 0 pgmap v1349: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-09T09:28:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[75425]: cluster 2023-10-09T09:28:02.710231+0000 mon.a (mon.0) 1451 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:28:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[75425]: cluster 2023-10-09T09:28:02.710267+0000 mon.a (mon.0) 1452 : cluster 1 Cluster is now healthy 2023-10-09T09:28:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[75425]: audit 2023-10-09T09:28:02.725196+0000 mon.a (mon.0) 1453 : audit 1 from='client.? 172.21.15.163:0/2517168134' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:28:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[75425]: cluster 2023-10-09T09:28:02.725322+0000 mon.a (mon.0) 1454 : cluster 0 osdmap e1091: 8 total, 8 up, 8 in 2023-10-09T09:28:03.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[75425]: cluster 2023-10-09T09:28:02.726294+0000 osd.5 (osd.5) 635 : cluster 3 failed to encode map e1091 with expected crc 2023-10-09T09:28:03.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[75425]: cluster 2023-10-09T09:28:02.727922+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e1091 with expected crc 2023-10-09T09:28:03.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[75425]: cluster 2023-10-09T09:28:03.640303+0000 mgr.y (mgr.34107) 650 : cluster 0 pgmap v1349: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-09T09:28:03.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[78107]: cluster 2023-10-09T09:28:02.710231+0000 mon.a (mon.0) 1451 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:28:03.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[78107]: cluster 2023-10-09T09:28:02.710267+0000 mon.a (mon.0) 1452 : cluster 1 Cluster is now healthy 2023-10-09T09:28:03.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[78107]: audit 2023-10-09T09:28:02.725196+0000 mon.a (mon.0) 1453 : audit 1 from='client.? 172.21.15.163:0/2517168134' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:28:03.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[78107]: cluster 2023-10-09T09:28:02.725322+0000 mon.a (mon.0) 1454 : cluster 0 osdmap e1091: 8 total, 8 up, 8 in 2023-10-09T09:28:03.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[78107]: cluster 2023-10-09T09:28:02.726294+0000 osd.5 (osd.5) 635 : cluster 3 failed to encode map e1091 with expected crc 2023-10-09T09:28:03.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[78107]: cluster 2023-10-09T09:28:02.727922+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e1091 with expected crc 2023-10-09T09:28:03.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:03 smithi163 bash[78107]: cluster 2023-10-09T09:28:03.640303+0000 mgr.y (mgr.34107) 650 : cluster 0 pgmap v1349: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-09T09:28:03.976 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:03.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:03.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:04.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:04.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:04.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:04.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:04.218 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:04.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:04.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:04.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:04.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:04.382 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:04.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:04.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:04.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:04.517 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:04.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:04.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:04.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:04.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:04.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:04.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:04.763 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:04.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:04.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:04.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:04.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:28:04.904 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:04.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:04.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:04.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:05.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:05.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:05.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:05.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:05.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:05.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:05.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:05.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:05.323 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:05.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:05.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:05.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:05.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:05.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:05.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:05.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:05.624 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:05.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:28:05.655 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:05.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:05.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:05.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:05.787 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:05.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:05.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:05.893 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:05.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:05.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:05.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:06.039 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:06.069 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:06.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:06.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:06.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:06.218 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:06.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:06.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:06.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:06.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:06.355 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:06.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:06.428 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:06.456 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:06.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:06.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:06.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:06.590 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:06.624 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:06.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:06 smithi163 bash[75425]: cluster 2023-10-09T09:28:05.642959+0000 mgr.y (mgr.34107) 651 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T09:28:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:06 smithi163 bash[75425]: audit 2023-10-09T09:28:06.429928+0000 mon.a (mon.0) 1455 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:28:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:06 smithi163 bash[78107]: cluster 2023-10-09T09:28:05.642959+0000 mgr.y (mgr.34107) 651 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T09:28:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:06 smithi163 bash[78107]: audit 2023-10-09T09:28:06.429928+0000 mon.a (mon.0) 1455 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:28:06.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:06.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:06.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:06.786 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:06 smithi181 bash[59345]: cluster 2023-10-09T09:28:05.642959+0000 mgr.y (mgr.34107) 651 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T09:28:06.786 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:06 smithi181 bash[59345]: audit 2023-10-09T09:28:06.429928+0000 mon.a (mon.0) 1455 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:28:06.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:06.887 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:06.917 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:06.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:06.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:07.024 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:07.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:07.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:07.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:07.179 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:07.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:07.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:07.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:07.329 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:07.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:07.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:07.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:07.442 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:07.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:07.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:07.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:07.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:07.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:07.634 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:07.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:07.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:07.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:07.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:07.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:07.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:07.888 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:07.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:07.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:07.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:08.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:08.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:08.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:08.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:08.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:08.198 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:08.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:08.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:08.304 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:08.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:08.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:08.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:08.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:08.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:08.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:08.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:08.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:08.620 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:08.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:08.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:08.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:08.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:08.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:08.889 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:08.891 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:08 smithi163 bash[78107]: cluster 2023-10-09T09:28:07.643602+0000 mgr.y (mgr.34107) 652 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-10-09T09:28:08.891 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:08 smithi163 bash[75425]: cluster 2023-10-09T09:28:07.643602+0000 mgr.y (mgr.34107) 652 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-10-09T09:28:08.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:08.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:08.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:09.025 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:09.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:09.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:09.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:08 smithi181 bash[59345]: cluster 2023-10-09T09:28:07.643602+0000 mgr.y (mgr.34107) 652 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-10-09T09:28:09.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:09.161 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:09.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:09.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:09.265 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:09.296 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:09.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:09.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:09.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:09.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:09.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:09.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:09.489 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:09.534 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:09.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:09.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:09.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:09.705 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:09.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:09.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:09.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:09.835 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:09.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:09.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:09.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:09.968 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:09.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:10.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:10.095 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:10.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:10.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:10.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:10.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:10.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:10.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:10.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:10.358 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:10.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:10.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:10.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:10.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:10.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:10.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:10.634 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:10.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:10.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:10.743 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:10.753 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:10 smithi181 bash[59345]: cluster 2023-10-09T09:28:09.644100+0000 mgr.y (mgr.34107) 653 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T09:28:10.753 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:10 smithi181 bash[59345]: audit 2023-10-09T09:28:10.371498+0000 mon.a (mon.0) 1456 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:10.753 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:10 smithi181 bash[59345]: audit 2023-10-09T09:28:10.373508+0000 mon.a (mon.0) 1457 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:10.772 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:10.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:10.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:10.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:10.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:10 smithi163 bash[75425]: cluster 2023-10-09T09:28:09.644100+0000 mgr.y (mgr.34107) 653 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T09:28:10.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:10 smithi163 bash[75425]: audit 2023-10-09T09:28:10.371498+0000 mon.a (mon.0) 1456 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:10.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:10 smithi163 bash[75425]: audit 2023-10-09T09:28:10.373508+0000 mon.a (mon.0) 1457 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:10.886 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:10 smithi163 bash[78107]: cluster 2023-10-09T09:28:09.644100+0000 mgr.y (mgr.34107) 653 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T09:28:10.886 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:10 smithi163 bash[78107]: audit 2023-10-09T09:28:10.371498+0000 mon.a (mon.0) 1456 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:10.886 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:10 smithi163 bash[78107]: audit 2023-10-09T09:28:10.373508+0000 mon.a (mon.0) 1457 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:10.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:10.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:11.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:11.031 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:11.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:11.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:11.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:11.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:11.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:11.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:11.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:11.286 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:11.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:11.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:11.295 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:11.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:11.421 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:11.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:11.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:11.525 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:11.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:11.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:11.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:11.667 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:11.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:11.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:11.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:11.801 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:11.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:11.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:11.938 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:11.966 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:11.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:12.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:12.098 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:12.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:12.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:12.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:12.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:12.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:12.352 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:12.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:12.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:12.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:12.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:12.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:12.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:12.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:12.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:12.685 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:12.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:12.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:12.792 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:12.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:12 smithi181 bash[59345]: cluster 2023-10-09T09:28:11.644771+0000 mgr.y (mgr.34107) 654 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T09:28:12.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:12.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:12.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:12.933 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:12.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:12 smithi163 bash[75425]: cluster 2023-10-09T09:28:11.644771+0000 mgr.y (mgr.34107) 654 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T09:28:12.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:12 smithi163 bash[78107]: cluster 2023-10-09T09:28:11.644771+0000 mgr.y (mgr.34107) 654 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T09:28:12.963 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:12.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:12.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:13.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:13.108 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:13.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:13.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:13.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:13.257 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:13.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:13.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:13.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:13.396 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:13.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:13.501 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:13.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:13.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:13.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:13.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:13.665 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:13.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:13.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:13.739 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:28:13] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.33.4" 2023-10-09T09:28:13.769 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:13.805 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:13.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:13.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:13.909 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:13.932 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:13.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:14.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:14.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[75425]: audit 2023-10-09T09:28:12.810395+0000 mon.a (mon.0) 1458 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[75425]: audit 2023-10-09T09:28:12.823424+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[75425]: audit 2023-10-09T09:28:12.905566+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[75425]: audit 2023-10-09T09:28:12.925655+0000 mon.a (mon.0) 1461 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[75425]: audit 2023-10-09T09:28:13.508460+0000 mon.a (mon.0) 1462 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:28:14.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[75425]: audit 2023-10-09T09:28:13.509508+0000 mon.a (mon.0) 1463 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:28:14.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[75425]: audit 2023-10-09T09:28:13.517458+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[75425]: cluster 2023-10-09T09:28:13.645288+0000 mgr.y (mgr.34107) 655 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T09:28:14.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[78107]: audit 2023-10-09T09:28:12.810395+0000 mon.a (mon.0) 1458 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[78107]: audit 2023-10-09T09:28:12.823424+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[78107]: audit 2023-10-09T09:28:12.905566+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[78107]: audit 2023-10-09T09:28:12.925655+0000 mon.a (mon.0) 1461 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[78107]: audit 2023-10-09T09:28:13.508460+0000 mon.a (mon.0) 1462 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:28:14.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[78107]: audit 2023-10-09T09:28:13.509508+0000 mon.a (mon.0) 1463 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:28:14.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[78107]: audit 2023-10-09T09:28:13.517458+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:13 smithi163 bash[78107]: cluster 2023-10-09T09:28:13.645288+0000 mgr.y (mgr.34107) 655 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T09:28:14.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:13 smithi181 bash[59345]: audit 2023-10-09T09:28:12.810395+0000 mon.a (mon.0) 1458 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:13 smithi181 bash[59345]: audit 2023-10-09T09:28:12.823424+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:13 smithi181 bash[59345]: audit 2023-10-09T09:28:12.905566+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:13 smithi181 bash[59345]: audit 2023-10-09T09:28:12.925655+0000 mon.a (mon.0) 1461 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:13 smithi181 bash[59345]: audit 2023-10-09T09:28:13.508460+0000 mon.a (mon.0) 1462 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:28:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:13 smithi181 bash[59345]: audit 2023-10-09T09:28:13.509508+0000 mon.a (mon.0) 1463 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:28:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:13 smithi181 bash[59345]: audit 2023-10-09T09:28:13.517458+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:13 smithi181 bash[59345]: cluster 2023-10-09T09:28:13.645288+0000 mgr.y (mgr.34107) 655 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T09:28:14.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:14.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:14.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:14.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:14.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:14.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:14.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:14.326 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:14.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:14.429 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:14.460 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:14.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:14.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:14.566 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:14.591 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:14.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:14.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:14.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:14.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:14.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:14.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:14.829 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:14.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:14.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:14.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:14.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:15.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:15.100 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:15.120 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:15.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:15.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:15.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:15.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:15.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:15.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:15.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:15.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:15.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:15.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:15.495 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:15.528 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:15.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:15.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:15.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:15.668 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:15.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:15.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:15.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:15.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:15.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:15.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:15.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:15.938 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:15.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:15.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:16.034 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:28:16.034 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:28:16.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:28:16.034 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:28:16.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.034 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:25161 6.f 6:fe65c9c6:::benchmark_data_smithi163_107771_object25160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:28:16.034 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:56.084444+0000", 2023-10-09T09:28:16.034 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.94612156400001, 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.123615006, 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 25161 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.084444+0000", 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.084444+0000", 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.084443+0000", 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:28:16.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.084472+0000", 2023-10-09T09:28:16.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9583000000000001e-05 2023-10-09T09:28:16.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.084485+0000", 2023-10-09T09:28:16.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2789000000000001e-05 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.084493+0000", 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7680000000000008e-06 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.084605+0000", 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000112293 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.084736+0000", 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013095899999999999 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.085874+0000", 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011385 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:28:16.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.087108+0000", 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012339709999999999 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.087706+0000", 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00059786799999999999 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.207947+0000", 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.120240583 2023-10-09T09:28:16.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.208009+0000", 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1885999999999997e-05 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.208059+0000", 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0713e-05 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:25176 6.f 6:fe4fcf9a:::benchmark_data_smithi163_107771_object25175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:56.093241+0000", 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.937324953, 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.121781423, 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:28:16.042 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 25176 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093241+0000", 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093241+0000", 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093239+0000", 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093266+0000", 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6682999999999998e-05 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093280+0000", 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4501999999999999e-05 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093288+0000", 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9529999999999999e-06 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093335+0000", 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7051000000000001e-05 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093483+0000", 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000147081 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093653+0000", 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000170533 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.095706+0000", 2023-10-09T09:28:16.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020527129999999998 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.096750+0000", 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001044248 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.214860+0000", 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.118109859 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.214958+0000", 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7868999999999994e-05 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.215022+0000", 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4690000000000006e-05 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:25177 6.f 6:f4a78ac9:::benchmark_data_smithi163_107771_object25176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:56.093301+0000", 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.93726460299999, 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12192868799999999, 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 25177 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.049 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093301+0000", 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093301+0000", 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093301+0000", 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093322+0000", 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1396e-05 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093331+0000", 2023-10-09T09:28:16.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6270000000000006e-06 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.093336+0000", 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5960000000000003e-06 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.094688+0000", 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013517609999999999 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.094819+0000", 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013107599999999999 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.094976+0000", 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000156599 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.095825+0000", 2023-10-09T09:28:16.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00084890900000000001 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.096858+0000", 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001032792 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.215089+0000", 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.118231605 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.054 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.215177+0000", 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7725000000000005e-05 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:56.215230+0000", 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3199999999999999e-05 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:27804 6.f 6:f317c1d1:::benchmark_data_smithi163_107771_object27803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:25:57.809748+0000", 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 138.22081814800001, 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12440598, 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 27804 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.809748+0000", 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.809748+0000", 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.809746+0000", 2023-10-09T09:28:16.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:28:16.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.809844+0000", 2023-10-09T09:28:16.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8066999999999995e-05 2023-10-09T09:28:16.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.809858+0000", 2023-10-09T09:28:16.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3957e-05 2023-10-09T09:28:16.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.809868+0000", 2023-10-09T09:28:16.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7180000000000008e-06 2023-10-09T09:28:16.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.809920+0000", 2023-10-09T09:28:16.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2160999999999997e-05 2023-10-09T09:28:16.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.810057+0000", 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013715600000000001 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.810207+0000", 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014950400000000001 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:28:16.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.811747+0000", 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001540191 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.815364+0000", 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036168189999999999 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.934057+0000", 2023-10-09T09:28:16.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.118693546 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.934106+0000", 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8959000000000003e-05 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:25:57.934154+0000", 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7540000000000002e-05 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.063 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:40119 6.e e1082/1076 6:76f56fc2:::benchmark_data_smithi163_107771_object40118:head v 1082'2526, mlcod=1082'2523)", 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:06.011516+0000", 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 130.01905019899999, 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13682873700000001, 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.011516+0000", 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.011516+0000", 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.011515+0000", 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:28:16.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.011598+0000", 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3375999999999996e-05 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.011613+0000", 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4747e-05 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.011622+0000", 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6260000000000001e-06 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.011657+0000", 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5395999999999998e-05 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.011676+0000", 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8530999999999999e-05 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014313+0000", 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026375669999999999 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.148345+0000", 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13403119099999999 2023-10-09T09:28:16.068 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:40121 6.e e1082/1076 6:713bf906:::benchmark_data_smithi163_107771_object40120:head v 1082'2527, mlcod=1082'2524)", 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:06.013079+0000", 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 130.01748670200001, 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13530671, 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.013079+0000", 2023-10-09T09:28:16.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.013079+0000", 2023-10-09T09:28:16.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.013079+0000", 2023-10-09T09:28:16.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:28:16.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.013163+0000", 2023-10-09T09:28:16.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4213000000000002e-05 2023-10-09T09:28:16.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.013181+0000", 2023-10-09T09:28:16.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8417999999999999e-05 2023-10-09T09:28:16.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.013189+0000", 2023-10-09T09:28:16.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1009999999999992e-06 2023-10-09T09:28:16.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.013459+0000", 2023-10-09T09:28:16.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026973399999999999 2023-10-09T09:28:16.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.013570+0000", 2023-10-09T09:28:16.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111032 2023-10-09T09:28:16.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.148358+0000", 2023-10-09T09:28:16.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13478796000000001 2023-10-09T09:28:16.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.148386+0000", 2023-10-09T09:28:16.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7796e-05 2023-10-09T09:28:16.077 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.077 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.077 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.078 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:40124 6.f 6:f7490f03:::benchmark_data_smithi163_107771_object40123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1082)", 2023-10-09T09:28:16.078 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:06.014235+0000", 2023-10-09T09:28:16.078 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 130.016330971, 2023-10-09T09:28:16.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14503318900000001, 2023-10-09T09:28:16.079 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.079 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.079 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:28:16.079 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:28:16.079 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:28:16.079 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 40124 2023-10-09T09:28:16.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.080 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014235+0000", 2023-10-09T09:28:16.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014235+0000", 2023-10-09T09:28:16.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014234+0000", 2023-10-09T09:28:16.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:28:16.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014255+0000", 2023-10-09T09:28:16.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1511000000000001e-05 2023-10-09T09:28:16.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014266+0000", 2023-10-09T09:28:16.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.077e-05 2023-10-09T09:28:16.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014272+0000", 2023-10-09T09:28:16.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2079999999999997e-06 2023-10-09T09:28:16.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.147062+0000", 2023-10-09T09:28:16.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13279028700000001 2023-10-09T09:28:16.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.147220+0000", 2023-10-09T09:28:16.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000158188 2023-10-09T09:28:16.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:28:16.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.147392+0000", 2023-10-09T09:28:16.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017140999999999999 2023-10-09T09:28:16.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:28:16.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.149404+0000", 2023-10-09T09:28:16.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020122400000000002 2023-10-09T09:28:16.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.150618+0000", 2023-10-09T09:28:16.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012137210000000001 2023-10-09T09:28:16.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.159179+0000", 2023-10-09T09:28:16.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085606729999999995 2023-10-09T09:28:16.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.159229+0000", 2023-10-09T09:28:16.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0012000000000002e-05 2023-10-09T09:28:16.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.159268+0000", 2023-10-09T09:28:16.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9669999999999998e-05 2023-10-09T09:28:16.093 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.093 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.093 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.094 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:40122 6.8 e1082/1076 6:1cbf8bde:::benchmark_data_smithi163_107771_object40121:head v 1082'2534, mlcod=1082'2530)", 2023-10-09T09:28:16.094 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:06.014295+0000", 2023-10-09T09:28:16.094 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 130.01627131399999, 2023-10-09T09:28:16.094 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13684964899999999, 2023-10-09T09:28:16.094 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.095 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.095 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014295+0000", 2023-10-09T09:28:16.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014295+0000", 2023-10-09T09:28:16.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014294+0000", 2023-10-09T09:28:16.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:28:16.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.014317+0000", 2023-10-09T09:28:16.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.234e-05 2023-10-09T09:28:16.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.098 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.098 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.146258+0000", 2023-10-09T09:28:16.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13194130900000001 2023-10-09T09:28:16.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.098 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.098 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.146273+0000", 2023-10-09T09:28:16.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5438e-05 2023-10-09T09:28:16.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.146370+0000", 2023-10-09T09:28:16.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6864e-05 2023-10-09T09:28:16.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.146394+0000", 2023-10-09T09:28:16.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3805999999999999e-05 2023-10-09T09:28:16.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.151112+0000", 2023-10-09T09:28:16.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0047182830000000002 2023-10-09T09:28:16.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.151144+0000", 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2054999999999997e-05 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:122970 6.f 6:f3bb532d:::benchmark_data_smithi163_107771_object12210:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1087)", 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.768692+0000", 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.261874357000003, 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12772892, 2023-10-09T09:28:16.102 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 122970 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768692+0000", 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768692+0000", 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768690+0000", 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768695+0000", 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.515e-06 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768703+0000", 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4200000000000007e-06 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768710+0000", 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2639999999999997e-06 2023-10-09T09:28:16.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768944+0000", 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023458499999999999 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.769052+0000", 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000107563 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.769151+0000", 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9607999999999999e-05 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.773798+0000", 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046462200000000004 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896076+0000", 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.122278912 2023-10-09T09:28:16.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896335+0000", 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00025894399999999999 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896380+0000", 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5058999999999997e-05 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896421+0000", 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0108000000000003e-05 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:122965 6.1 e1087/1076 6:83184e43:::benchmark_data_smithi163_107771_object12205:head v 1087'7602, mlcod=1087'7597)", 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.768732+0000", 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.261834217000001, 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12622130000000001, 2023-10-09T09:28:16.109 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768732+0000", 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768732+0000", 2023-10-09T09:28:16.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768731+0000", 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768736+0000", 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9089999999999999e-06 2023-10-09T09:28:16.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768743+0000", 2023-10-09T09:28:16.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3740000000000003e-06 2023-10-09T09:28:16.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.768748+0000", 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8790000000000002e-06 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.773782+0000", 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050347639999999997 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.773804+0000", 2023-10-09T09:28:16.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.194e-05 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.894889+0000", 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.121085048 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.894953+0000", 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3796999999999999e-05 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.114 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:122974 6.1 e1087/1076 6:89ac4ba4:::benchmark_data_smithi163_107771_object12214:head v 1087'7603, mlcod=1087'7598)", 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.771438+0000", 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.259128015000002, 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.123715267, 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.771438+0000", 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.771438+0000", 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.771437+0000", 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.771444+0000", 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2859999999999996e-06 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.771455+0000", 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1804000000000001e-05 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.771464+0000", 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7499999999999992e-06 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.777728+0000", 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062638640000000001 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.777747+0000", 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8889000000000002e-05 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.895070+0000", 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.117323573 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.895153+0000", 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2848000000000004e-05 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:122973 6.0 e1087/1076 6:049fe7c3:::benchmark_data_smithi163_107771_object12213:head v 1087'7775, mlcod=1087'7773)", 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.773992+0000", 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.256574276000002, 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.122387915, 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.120 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.773992+0000", 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.773992+0000", 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.773991+0000", 2023-10-09T09:28:16.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:28:16.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.774001+0000", 2023-10-09T09:28:16.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9289999999999993e-06 2023-10-09T09:28:16.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.774013+0000", 2023-10-09T09:28:16.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.234e-05 2023-10-09T09:28:16.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.774021+0000", 2023-10-09T09:28:16.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3739999999999998e-06 2023-10-09T09:28:16.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.774116+0000", 2023-10-09T09:28:16.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4155999999999998e-05 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.774135+0000", 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9848999999999999e-05 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896290+0000", 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.122154793 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896380+0000", 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9431999999999993e-05 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:122979 6.3 e1087/1076 6:c32ff0f5:::benchmark_data_smithi163_107771_object12219:head v 1087'7570, mlcod=1087'7567)", 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.776747+0000", 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.253819174, 2023-10-09T09:28:16.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11952856100000001, 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776747+0000", 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776747+0000", 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776746+0000", 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776753+0000", 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5719999999999997e-06 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776766+0000", 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2330000000000001e-05 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776773+0000", 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7619999999999995e-06 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776816+0000", 2023-10-09T09:28:16.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2261000000000001e-05 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776835+0000", 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9728000000000002e-05 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896238+0000", 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.119402631 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896275+0000", 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7465999999999997e-05 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:122980 6.8 e1087/1076 6:1ccf85ab:::benchmark_data_smithi163_107771_object12220:head v 1087'7784, mlcod=1087'7782)", 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.776783+0000", 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.253782985999997, 2023-10-09T09:28:16.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.119664917, 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776783+0000", 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776783+0000", 2023-10-09T09:28:16.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776783+0000", 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776787+0000", 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9029999999999997e-06 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776793+0000", 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8889999999999999e-06 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776797+0000", 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5210000000000004e-06 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776838+0000", 2023-10-09T09:28:16.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0481000000000001e-05 2023-10-09T09:28:16.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776857+0000", 2023-10-09T09:28:16.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9916e-05 2023-10-09T09:28:16.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896391+0000", 2023-10-09T09:28:16.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.119533308 2023-10-09T09:28:16.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896448+0000", 2023-10-09T09:28:16.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7156e-05 2023-10-09T09:28:16.138 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.138 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.138 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.138 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:122983 6.2 e1087/1076 6:4e4f5c65:::benchmark_data_smithi163_107771_object12223:head v 1087'7760, mlcod=1087'7758)", 2023-10-09T09:28:16.139 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.779114+0000", 2023-10-09T09:28:16.139 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.251452268000001, 2023-10-09T09:28:16.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11628545, 2023-10-09T09:28:16.139 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.139 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.140 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.779114+0000", 2023-10-09T09:28:16.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.779114+0000", 2023-10-09T09:28:16.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.779113+0000", 2023-10-09T09:28:16.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:28:16.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.779120+0000", 2023-10-09T09:28:16.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9820000000000002e-06 2023-10-09T09:28:16.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.779131+0000", 2023-10-09T09:28:16.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1326e-05 2023-10-09T09:28:16.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.779139+0000", 2023-10-09T09:28:16.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0409999999999998e-06 2023-10-09T09:28:16.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.779181+0000", 2023-10-09T09:28:16.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2134e-05 2023-10-09T09:28:16.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.779199+0000", 2023-10-09T09:28:16.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7842999999999998e-05 2023-10-09T09:28:16.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.895362+0000", 2023-10-09T09:28:16.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.116162644 2023-10-09T09:28:16.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.895399+0000", 2023-10-09T09:28:16.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7286999999999997e-05 2023-10-09T09:28:16.148 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.148 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.148 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.149 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.34791.0:122981 6.0 e1087/1076 6:0731e430:::benchmark_data_smithi163_107771_object12221:head v 1087'7776, mlcod=1087'7773)", 2023-10-09T09:28:16.149 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.780526+0000", 2023-10-09T09:28:16.149 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.250040261000002, 2023-10-09T09:28:16.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.115995943, 2023-10-09T09:28:16.149 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.149 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.149 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.780526+0000", 2023-10-09T09:28:16.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.780526+0000", 2023-10-09T09:28:16.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.780525+0000", 2023-10-09T09:28:16.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:28:16.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.780531+0000", 2023-10-09T09:28:16.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2480000000000001e-06 2023-10-09T09:28:16.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.780539+0000", 2023-10-09T09:28:16.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.661e-06 2023-10-09T09:28:16.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.780544+0000", 2023-10-09T09:28:16.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8910000000000003e-06 2023-10-09T09:28:16.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.780587+0000", 2023-10-09T09:28:16.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3148000000000001e-05 2023-10-09T09:28:16.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.780600+0000", 2023-10-09T09:28:16.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3182e-05 2023-10-09T09:28:16.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.156 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896452+0000", 2023-10-09T09:28:16.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.115851915 2023-10-09T09:28:16.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896522+0000", 2023-10-09T09:28:16.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9554999999999994e-05 2023-10-09T09:28:16.157 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.157 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.157 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.158 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.34791.0:122970 6.f e1087/1076 ondisk, result = 0)", 2023-10-09T09:28:16.158 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.781266+0000", 2023-10-09T09:28:16.158 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.249299977, 2023-10-09T09:28:16.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.114819476, 2023-10-09T09:28:16.158 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.159 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:28:16.159 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.781266+0000", 2023-10-09T09:28:16.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.781266+0000", 2023-10-09T09:28:16.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.781265+0000", 2023-10-09T09:28:16.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.781271+0000", 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6069999999999998e-06 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.781282+0000", 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0969e-05 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.781289+0000", 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1010000000000003e-06 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896034+0000", 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.114744576 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896057+0000", 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3805e-05 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896085+0000", 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8187e-05 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.163 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.34791.0:122970 6.f e1087/1076 ondisk, result = 0)", 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.783066+0000", 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.247500303000002, 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.113360164, 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.783066+0000", 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.164 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.783066+0000", 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.783065+0000", 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.783070+0000", 2023-10-09T09:28:16.165 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1510000000000003e-06 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.895647+0000", 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.112576462 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.895658+0000", 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1124999999999999e-05 2023-10-09T09:28:16.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896321+0000", 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00066299700000000002 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896330+0000", 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6929999999999996e-06 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.896426+0000", 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6325999999999999e-05 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:153527 6.f 6:f209ffab:::benchmark_data_smithi163_107771_object42767:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1091)", 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:28:10.958384+0000", 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 5.072182433, 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.120215869, 2023-10-09T09:28:16.168 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 153527 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.958384+0000", 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.958384+0000", 2023-10-09T09:28:16.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.958382+0000", 2023-10-09T09:28:16.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:28:16.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.958390+0000", 2023-10-09T09:28:16.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.32e-06 2023-10-09T09:28:16.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.958408+0000", 2023-10-09T09:28:16.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8369e-05 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.958419+0000", 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0213e-05 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.958480+0000", 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1601999999999997e-05 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.958587+0000", 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000106681 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.958670+0000", 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3448999999999993e-05 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.966757+0000", 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080870899999999999 2023-10-09T09:28:16.173 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.967027+0000", 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026936700000000001 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:11.078356+0000", 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11132924600000001 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.174 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:11.078484+0000", 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012757799999999999 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:11.078599+0000", 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011589100000000001 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.175 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:153535 6.f 6:fbe4abf7:::benchmark_data_smithi163_107771_object42775:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1091)", 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:28:10.967855+0000", 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 5.0627114720000002, 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11087119099999999, 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 153535 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.967855+0000", 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.967855+0000", 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.967853+0000", 2023-10-09T09:28:16.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.967859+0000", 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7740000000000001e-06 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.967872+0000", 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3025e-05 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.967879+0000", 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1169999999999998e-06 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.967906+0000", 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6415999999999999e-05 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.967985+0000", 2023-10-09T09:28:16.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9340999999999998e-05 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.968060+0000", 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4688e-05 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.971530+0000", 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003470215 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:10.972779+0000", 2023-10-09T09:28:16.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012487290000000001 2023-10-09T09:28:16.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:28:16.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:11.078638+0000", 2023-10-09T09:28:16.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.105859067 2023-10-09T09:28:16.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:11.078688+0000", 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0420999999999997e-05 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:11.078726+0000", 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7676999999999999e-05 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.182 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.183 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:28:16.183 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:28:16.183 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:28:16.183 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:16.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:16.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:16.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:16.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:16.334 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:16.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:16.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:16.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:16.463 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:16.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:16.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:16.479 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:28:16.486 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:28:16.573 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:16.616 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:16.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:16.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:16 smithi163 bash[78107]: cluster 2023-10-09T09:28:15.645845+0000 mgr.y (mgr.34107) 656 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T09:28:16.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:16 smithi163 bash[75425]: cluster 2023-10-09T09:28:15.645845+0000 mgr.y (mgr.34107) 656 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T09:28:16.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:16.730 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:16 smithi181 bash[59345]: cluster 2023-10-09T09:28:15.645845+0000 mgr.y (mgr.34107) 656 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T09:28:16.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:16.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:16.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:16.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:16.885 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:16.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:16.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:16.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:17.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:17.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:17.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:17.120 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:17.149 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:17.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:17.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:17.276 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:17.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:17.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:17.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:17.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:17.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:17.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:17.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:17 smithi163 bash[78107]: audit 2023-10-09T09:28:16.457860+0000 mgr.y (mgr.34107) 657 : audit 0 from='client.44786 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:28:17.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:17 smithi163 bash[75425]: audit 2023-10-09T09:28:16.457860+0000 mgr.y (mgr.34107) 657 : audit 0 from='client.44786 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:28:17.567 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:17.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:17.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:17.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:17.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:17.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:17.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:17.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:17.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:17 smithi181 bash[59345]: audit 2023-10-09T09:28:16.457860+0000 mgr.y (mgr.34107) 657 : audit 0 from='client.44786 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:28:17.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:17.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:17.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:17.942 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:17.967 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:17.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:17.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:18.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:18.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:18.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:18.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:18.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:18.228 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:18.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:18.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:18.332 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:18.356 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:28:18.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:18.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:18.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:18.391 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:28:18.426 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:28:18.461 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:28:18.465 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:18.502 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:28:18.502 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:18.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:18.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:18.510 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:18 smithi181 bash[59345]: cluster 2023-10-09T09:28:17.646472+0000 mgr.y (mgr.34107) 658 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-10-09T09:28:18.510 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:18 smithi181 bash[59345]: audit 2023-10-09T09:28:18.305945+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.163:0/2775645042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:28:18.541 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:28:18.580 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:28:18.608 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:18.617 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:18 smithi163 bash[75425]: cluster 2023-10-09T09:28:17.646472+0000 mgr.y (mgr.34107) 658 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-10-09T09:28:18.617 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:18 smithi163 bash[75425]: audit 2023-10-09T09:28:18.305945+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.163:0/2775645042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:28:18.617 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:18 smithi163 bash[78107]: cluster 2023-10-09T09:28:17.646472+0000 mgr.y (mgr.34107) 658 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-10-09T09:28:18.617 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:18 smithi163 bash[78107]: audit 2023-10-09T09:28:18.305945+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.163:0/2775645042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:28:18.627 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:28:18.636 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:28:18.637 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:28:18.637 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:28:18.637 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:28:18.637 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:28:18.637 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:28:18.637 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:28:18.637 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:28:18.637 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:18.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:18.745 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:18.773 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:18.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:18.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:18.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:18.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:18.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:18.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:19.019 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:19.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:19.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:19.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:19.153 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:19.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:19.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:19.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:19.283 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:19.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:19.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:19.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:19.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:19.449 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:19.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:19.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:19.555 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:19.586 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:19.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:19.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:19.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:19.745 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T09:28:19.769 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:19.770 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1355,"stamp":"2023-10-09T09:28:15.645470+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4390000568,"num_objects":67002,"num_object_clones":0,"num_object_copies":201006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":67002,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":154798,"num_write_kb":7090894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137032,"ondisk_log_size":137032,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14581668,"kb_used_data":12919460,"kb_used_omap":416,"kb_used_meta":1661599,"kb_avail":735379548,"statfs":{"total":767960285184,"available":753028657152,"internally_reserved":0,"allocated":13229527040,"data_stored":13223715889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":1701477809},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,4,9,12,5,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-552009728,"num_objects":-8423,"num_object_clones":0,"num_object_copies":-25269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8423,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005389"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1058,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.709916+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:27:59.709916+0000","last_peered":"2023-10-09T09:27:59.709916+0000","last_clean":"2023-10-09T09:27:59.709916+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:27:59.709916+0000","last_undegraded":"2023-10-09T09:27:59.709916+0000","last_fullsized":"2023-10-09T09:27:59.709916+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1023,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.740055+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:28:02.740055+0000","last_peered":"2023-10-09T09:28:02.740055+0000","last_clean":"2023-10-09T09:28:02.740055+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:28:02.740055+0000","last_undegraded":"2023-10-09T09:28:02.740055+0000","last_fullsized":"2023-10-09T09:28:02.740055+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1026,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.722839+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:27:59.722839+0000","last_peered":"2023-10-09T09:27:59.722839+0000","last_clean":"2023-10-09T09:27:59.722839+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:27:59.722839+0000","last_undegraded":"2023-10-09T09:27:59.722839+0000","last_fullsized":"2023-10-09T09:27:59.722839+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1018,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742430+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:28:02.742430+0000","last_peered":"2023-10-09T09:28:02.742430+0000","last_clean":"2023-10-09T09:28:02.742430+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:28:02.742430+0000","last_undegraded":"2023-10-09T09:28:02.742430+0000","last_fullsized":"2023-10-09T09:28:02.742430+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1004,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.708128+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:27:59.708128+0000","last_peered":"2023-10-09T09:27:59.708128+0000","last_clean":"2023-10-09T09:27:59.708128+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:27:59.708128+0000","last_undegraded":"2023-10-09T09:27:59.708128+0000","last_fullsized":"2023-10-09T09:27:59.708128+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1022,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.717893+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:27:59.717893+0000","last_peered":"2023-10-09T09:27:59.717893+0000","last_clean":"2023-10-09T09:27:59.717893+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:27:59.717893+0000","last_undegraded":"2023-10-09T09:27:59.717893+0000","last_fullsized":"2023-10-09T09:27:59.717893+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1017,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.713174+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:27:59.713174+0000","last_peered":"2023-10-09T09:27:59.713174+0000","last_clean":"2023-10-09T09:27:59.713174+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:27:59.713174+0000","last_undegraded":"2023-10-09T09:27:59.713174+0000","last_fullsized":"2023-10-09T09:27:59.713174+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":988,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.708613+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:27:59.708613+0000","last_peered":"2023-10-09T09:27:59.708613+0000","last_clean":"2023-10-09T09:27:59.708613+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:27:59.708613+0000","last_undegraded":"2023-10-09T09:27:59.708613+0000","last_fullsized":"2023-10-09T09:27:59.708613+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1043,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.702056+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:27:59.702056+0000","last_peered":"2023-10-09T09:27:59.702056+0000","last_clean":"2023-10-09T09:27:59.702056+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:27:59.702056+0000","last_undegraded":"2023-10-09T09:27:59.702056+0000","last_fullsized":"2023-10-09T09:27:59.702056+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":999,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.708441+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:27:59.708441+0000","last_peered":"2023-10-09T09:27:59.708441+0000","last_clean":"2023-10-09T09:27:59.708441+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:27:59.708441+0000","last_undegraded":"2023-10-09T09:27:59.708441+0000","last_fullsized":"2023-10-09T09:27:59.708441+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1022,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712424+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:27:59.712424+0000","last_peered":"2023-10-09T09:27:59.712424+0000","last_clean":"2023-10-09T09:27:59.712424+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:27:59.712424+0000","last_undegraded":"2023-10-09T09:27:59.712424+0000","last_fullsized":"2023-10-09T09:27:59.712424+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1026,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.722171+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:27:59.722171+0000","last_peered":"2023-10-09T09:27:59.722171+0000","last_clean":"2023-10-09T09:27:59.722171+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:27:59.722171+0000","last_undegraded":"2023-10-09T09:27:59.722171+0000","last_fullsized":"2023-10-09T09:27:59.722171+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1049,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.710561+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:27:59.710561+0000","last_peered":"2023-10-09T09:27:59.710561+0000","last_clean":"2023-10-09T09:27:59.710561+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:27:59.710561+0000","last_undegraded":"2023-10-09T09:27:59.710561+0000","last_fullsized":"2023-10-09T09:27:59.710561+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1041,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.710098+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:27:59.710098+0000","last_peered":"2023-10-09T09:27:59.710098+0000","last_clean":"2023-10-09T09:27:59.710098+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:27:59.710098+0000","last_undegraded":"2023-10-09T09:27:59.710098+0000","last_fullsized":"2023-10-09T09:27:59.710098+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1021,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712488+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:27:59.712488+0000","last_peered":"2023-10-09T09:27:59.712488+0000","last_clean":"2023-10-09T09:27:59.712488+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:27:59.712488+0000","last_undegraded":"2023-10-09T09:27:59.712488+0000","last_fullsized":"2023-10-09T09:27:59.712488+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":988,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.715296+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:27:59.715296+0000","last_peered":"2023-10-09T09:27:59.715296+0000","last_clean":"2023-10-09T09:27:59.715296+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:27:59.715296+0000","last_undegraded":"2023-10-09T09:27:59.715296+0000","last_fullsized":"2023-10-09T09:27:59.715296+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1018,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742307+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:28:02.742307+0000","last_peered":"2023-10-09T09:28:02.742307+0000","last_clean":"2023-10-09T09:28:02.742307+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:28:02.742307+0000","last_undegraded":"2023-10-09T09:28:02.742307+0000","last_fullsized":"2023-10-09T09:28:02.742307+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1042,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.702175+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:27:59.702175+0000","last_peered":"2023-10-09T09:27:59.702175+0000","last_clean":"2023-10-09T09:27:59.702175+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:27:59.702175+0000","last_undegraded":"2023-10-09T09:27:59.702175+0000","last_fullsized":"2023-10-09T09:27:59.702175+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1004,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.711988+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:27:59.711988+0000","last_peered":"2023-10-09T09:27:59.711988+0000","last_clean":"2023-10-09T09:27:59.711988+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:27:59.711988+0000","last_undegraded":"2023-10-09T09:27:59.711988+0000","last_fullsized":"2023-10-09T09:27:59.711988+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1016,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712808+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:27:59.712808+0000","last_peered":"2023-10-09T09:27:59.712808+0000","last_clean":"2023-10-09T09:27:59.712808+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:27:59.712808+0000","last_undegraded":"2023-10-09T09:27:59.712808+0000","last_fullsized":"2023-10-09T09:27:59.712808+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1032,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.717399+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:27:59.717399+0000","last_peered":"2023-10-09T09:27:59.717399+0000","last_clean":"2023-10-09T09:27:59.717399+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:27:59.717399+0000","last_undegraded":"2023-10-09T09:27:59.717399+0000","last_fullsized":"2023-10-09T09:27:59.717399+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1023,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.724339+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:27:59.724339+0000","last_peered":"2023-10-09T09:27:59.724339+0000","last_clean":"2023-10-09T09:27:59.724339+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:27:59.724339+0000","last_undegraded":"2023-10-09T09:27:59.724339+0000","last_fullsized":"2023-10-09T09:27:59.724339+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"1091'9818","reported_seq":9841,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:11.245297+0000","last_change":"2023-10-09T09:25:25.899629+0000","last_active":"2023-10-09T09:28:11.245297+0000","last_peered":"2023-10-09T09:28:11.245297+0000","last_clean":"2023-10-09T09:28:11.245297+0000","last_became_active":"2023-10-09T09:25:25.899245+0000","last_became_peered":"2023-10-09T09:25:25.899245+0000","last_unstale":"2023-10-09T09:28:11.245297+0000","last_undegraded":"2023-10-09T09:28:11.245297+0000","last_fullsized":"2023-10-09T09:28:11.245297+0000","mapping_epoch":1076,"log_start":"0'0","ondisk_log_start":"0'0","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":9818,"log_dups_size":0,"ondisk_log_size":9818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281935872,"num_objects":4302,"num_object_clones":0,"num_object_copies":12906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9818,"num_write_kb":451840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1020,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712665+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:27:59.712665+0000","last_peered":"2023-10-09T09:27:59.712665+0000","last_clean":"2023-10-09T09:27:59.712665+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:27:59.712665+0000","last_undegraded":"2023-10-09T09:27:59.712665+0000","last_fullsized":"2023-10-09T09:27:59.712665+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1027,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.702119+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:27:59.702119+0000","last_peered":"2023-10-09T09:27:59.702119+0000","last_clean":"2023-10-09T09:27:59.702119+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:27:59.702119+0000","last_undegraded":"2023-10-09T09:27:59.702119+0000","last_fullsized":"2023-10-09T09:27:59.702119+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1091'9763","reported_seq":9788,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:14.709963+0000","last_change":"2023-10-09T09:25:25.902246+0000","last_active":"2023-10-09T09:28:14.709963+0000","last_peered":"2023-10-09T09:28:14.709963+0000","last_clean":"2023-10-09T09:28:14.709963+0000","last_became_active":"2023-10-09T09:25:25.901932+0000","last_became_peered":"2023-10-09T09:25:25.901932+0000","last_unstale":"2023-10-09T09:28:14.709963+0000","last_undegraded":"2023-10-09T09:28:14.709963+0000","last_fullsized":"2023-10-09T09:28:14.709963+0000","mapping_epoch":1076,"log_start":"1082'2400","ondisk_log_start":"1082'2400","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7363,"log_dups_size":0,"ondisk_log_size":7363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:10.795272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273481728,"num_objects":4173,"num_object_clones":0,"num_object_copies":12519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9763,"num_write_kb":445952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1021,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712282+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:27:59.712282+0000","last_peered":"2023-10-09T09:27:59.712282+0000","last_clean":"2023-10-09T09:27:59.712282+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:27:59.712282+0000","last_undegraded":"2023-10-09T09:27:59.712282+0000","last_fullsized":"2023-10-09T09:27:59.712282+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1056,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.709570+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:27:59.709570+0000","last_peered":"2023-10-09T09:27:59.709570+0000","last_clean":"2023-10-09T09:27:59.709570+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:27:59.709570+0000","last_undegraded":"2023-10-09T09:27:59.709570+0000","last_fullsized":"2023-10-09T09:27:59.709570+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1091'9713","reported_seq":9736,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:11.231000+0000","last_change":"2023-10-09T09:25:25.899849+0000","last_active":"2023-10-09T09:28:11.231000+0000","last_peered":"2023-10-09T09:28:11.231000+0000","last_clean":"2023-10-09T09:28:11.231000+0000","last_became_active":"2023-10-09T09:25:25.899350+0000","last_became_peered":"2023-10-09T09:25:25.899350+0000","last_unstale":"2023-10-09T09:28:11.231000+0000","last_undegraded":"2023-10-09T09:28:11.231000+0000","last_fullsized":"2023-10-09T09:28:11.231000+0000","mapping_epoch":1076,"log_start":"1081'1300","ondisk_log_start":"1081'1300","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8413,"log_dups_size":0,"ondisk_log_size":8413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280428544,"num_objects":4279,"num_object_clones":0,"num_object_copies":12837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9713,"num_write_kb":447744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1005,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712724+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:27:59.712724+0000","last_peered":"2023-10-09T09:27:59.712724+0000","last_clean":"2023-10-09T09:27:59.712724+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:27:59.712724+0000","last_undegraded":"2023-10-09T09:27:59.712724+0000","last_fullsized":"2023-10-09T09:27:59.712724+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1023,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.717741+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:27:59.717741+0000","last_peered":"2023-10-09T09:27:59.717741+0000","last_clean":"2023-10-09T09:27:59.717741+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:27:59.717741+0000","last_undegraded":"2023-10-09T09:27:59.717741+0000","last_fullsized":"2023-10-09T09:27:59.717741+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1091'9260","reported_seq":9284,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:12.380174+0000","last_change":"2023-10-09T09:25:25.899129+0000","last_active":"2023-10-09T09:28:12.380174+0000","last_peered":"2023-10-09T09:28:12.380174+0000","last_clean":"2023-10-09T09:28:12.380174+0000","last_became_active":"2023-10-09T09:25:25.898720+0000","last_became_peered":"2023-10-09T09:25:25.898720+0000","last_unstale":"2023-10-09T09:28:12.380174+0000","last_undegraded":"2023-10-09T09:28:12.380174+0000","last_fullsized":"2023-10-09T09:28:12.380174+0000","mapping_epoch":1076,"log_start":"0'0","ondisk_log_start":"0'0","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":9260,"log_dups_size":0,"ondisk_log_size":9260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:06.994778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266600448,"num_objects":4068,"num_object_clones":0,"num_object_copies":12204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9260,"num_write_kb":426496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1043,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.704061+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:27:59.704061+0000","last_peered":"2023-10-09T09:27:59.704061+0000","last_clean":"2023-10-09T09:27:59.704061+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:27:59.704061+0000","last_undegraded":"2023-10-09T09:27:59.704061+0000","last_fullsized":"2023-10-09T09:27:59.704061+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1017,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712376+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:27:59.712376+0000","last_peered":"2023-10-09T09:27:59.712376+0000","last_clean":"2023-10-09T09:27:59.712376+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:27:59.712376+0000","last_undegraded":"2023-10-09T09:27:59.712376+0000","last_fullsized":"2023-10-09T09:27:59.712376+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1091'9623","reported_seq":9646,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:11.279132+0000","last_change":"2023-10-09T09:25:25.902960+0000","last_active":"2023-10-09T09:28:11.279132+0000","last_peered":"2023-10-09T09:28:11.279132+0000","last_clean":"2023-10-09T09:28:11.279132+0000","last_became_active":"2023-10-09T09:25:25.902534+0000","last_became_peered":"2023-10-09T09:25:25.902534+0000","last_unstale":"2023-10-09T09:28:11.279132+0000","last_undegraded":"2023-10-09T09:28:11.279132+0000","last_fullsized":"2023-10-09T09:28:11.279132+0000","mapping_epoch":1076,"log_start":"1081'1700","ondisk_log_start":"1081'1700","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7923,"log_dups_size":0,"ondisk_log_size":7923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278331392,"num_objects":4247,"num_object_clones":0,"num_object_copies":12741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9623,"num_write_kb":443840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":1006,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.708185+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:27:59.708185+0000","last_peered":"2023-10-09T09:27:59.708185+0000","last_clean":"2023-10-09T09:27:59.708185+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:27:59.708185+0000","last_undegraded":"2023-10-09T09:27:59.708185+0000","last_fullsized":"2023-10-09T09:27:59.708185+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1041,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.710639+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:27:59.710639+0000","last_peered":"2023-10-09T09:27:59.710639+0000","last_clean":"2023-10-09T09:27:59.710639+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:27:59.710639+0000","last_undegraded":"2023-10-09T09:27:59.710639+0000","last_fullsized":"2023-10-09T09:27:59.710639+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1091'9587","reported_seq":9612,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:15.462230+0000","last_change":"2023-10-09T09:25:25.901392+0000","last_active":"2023-10-09T09:28:15.462230+0000","last_peered":"2023-10-09T09:28:15.462230+0000","last_clean":"2023-10-09T09:28:15.462230+0000","last_became_active":"2023-10-09T09:25:25.901057+0000","last_became_peered":"2023-10-09T09:25:25.901057+0000","last_unstale":"2023-10-09T09:28:15.462230+0000","last_undegraded":"2023-10-09T09:28:15.462230+0000","last_fullsized":"2023-10-09T09:28:15.462230+0000","mapping_epoch":1076,"log_start":"0'0","ondisk_log_start":"0'0","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":9587,"log_dups_size":0,"ondisk_log_size":9587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:56.342418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266272768,"num_objects":4063,"num_object_clones":0,"num_object_copies":12189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9587,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1024,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.717842+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:27:59.717842+0000","last_peered":"2023-10-09T09:27:59.717842+0000","last_clean":"2023-10-09T09:27:59.717842+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:27:59.717842+0000","last_undegraded":"2023-10-09T09:27:59.717842+0000","last_fullsized":"2023-10-09T09:27:59.717842+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1056,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.709999+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:27:59.709999+0000","last_peered":"2023-10-09T09:27:59.709999+0000","last_clean":"2023-10-09T09:27:59.709999+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:27:59.709999+0000","last_undegraded":"2023-10-09T09:27:59.709999+0000","last_fullsized":"2023-10-09T09:27:59.709999+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1091'9929","reported_seq":9954,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:14.683060+0000","last_change":"2023-10-09T09:25:25.896450+0000","last_active":"2023-10-09T09:28:14.683060+0000","last_peered":"2023-10-09T09:28:14.683060+0000","last_clean":"2023-10-09T09:28:14.683060+0000","last_became_active":"2023-10-09T09:25:25.896015+0000","last_became_peered":"2023-10-09T09:25:25.896015+0000","last_unstale":"2023-10-09T09:28:14.683060+0000","last_undegraded":"2023-10-09T09:28:14.683060+0000","last_fullsized":"2023-10-09T09:28:14.683060+0000","mapping_epoch":1076,"log_start":"1082'2600","ondisk_log_start":"1082'2600","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7329,"log_dups_size":0,"ondisk_log_size":7329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:32.630600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273088512,"num_objects":4167,"num_object_clones":0,"num_object_copies":12501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9929,"num_write_kb":451072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1042,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.709349+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:27:59.709349+0000","last_peered":"2023-10-09T09:27:59.709349+0000","last_clean":"2023-10-09T09:27:59.709349+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:27:59.709349+0000","last_undegraded":"2023-10-09T09:27:59.709349+0000","last_fullsized":"2023-10-09T09:27:59.709349+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1041,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.711567+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:27:59.711567+0000","last_peered":"2023-10-09T09:27:59.711567+0000","last_clean":"2023-10-09T09:27:59.711567+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:27:59.711567+0000","last_undegraded":"2023-10-09T09:27:59.711567+0000","last_fullsized":"2023-10-09T09:27:59.711567+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1091'9646","reported_seq":9670,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:11.278991+0000","last_change":"2023-10-09T09:25:25.898838+0000","last_active":"2023-10-09T09:28:11.278991+0000","last_peered":"2023-10-09T09:28:11.278991+0000","last_clean":"2023-10-09T09:28:11.278991+0000","last_became_active":"2023-10-09T09:25:25.898426+0000","last_became_peered":"2023-10-09T09:25:25.898426+0000","last_unstale":"2023-10-09T09:28:11.278991+0000","last_undegraded":"2023-10-09T09:28:11.278991+0000","last_fullsized":"2023-10-09T09:28:11.278991+0000","mapping_epoch":1076,"log_start":"0'0","ondisk_log_start":"0'0","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":9646,"log_dups_size":0,"ondisk_log_size":9646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275709976,"num_objects":4208,"num_object_clones":0,"num_object_copies":12624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4208,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9646,"num_write_kb":443265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":989,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.707926+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:27:59.707926+0000","last_peered":"2023-10-09T09:27:59.707926+0000","last_clean":"2023-10-09T09:27:59.707926+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:27:59.707926+0000","last_undegraded":"2023-10-09T09:27:59.707926+0000","last_fullsized":"2023-10-09T09:27:59.707926+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1027,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.753330+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:28:02.753330+0000","last_peered":"2023-10-09T09:28:02.753330+0000","last_clean":"2023-10-09T09:28:02.753330+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:28:02.753330+0000","last_undegraded":"2023-10-09T09:28:02.753330+0000","last_fullsized":"2023-10-09T09:28:02.753330+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1091'9790","reported_seq":9815,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:12.402060+0000","last_change":"2023-10-09T09:25:25.899052+0000","last_active":"2023-10-09T09:28:12.402060+0000","last_peered":"2023-10-09T09:28:12.402060+0000","last_clean":"2023-10-09T09:28:12.402060+0000","last_became_active":"2023-10-09T09:25:25.898696+0000","last_became_peered":"2023-10-09T09:25:25.898696+0000","last_unstale":"2023-10-09T09:28:12.402060+0000","last_undegraded":"2023-10-09T09:28:12.402060+0000","last_fullsized":"2023-10-09T09:28:12.402060+0000","mapping_epoch":1076,"log_start":"0'0","ondisk_log_start":"0'0","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":9790,"log_dups_size":0,"ondisk_log_size":9790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:34.741367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276037632,"num_objects":4212,"num_object_clones":0,"num_object_copies":12636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9790,"num_write_kb":448064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1022,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.725902+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:27:59.725902+0000","last_peered":"2023-10-09T09:27:59.725902+0000","last_clean":"2023-10-09T09:27:59.725902+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:27:59.725902+0000","last_undegraded":"2023-10-09T09:27:59.725902+0000","last_fullsized":"2023-10-09T09:27:59.725902+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1058,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.702253+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:27:59.702253+0000","last_peered":"2023-10-09T09:27:59.702253+0000","last_clean":"2023-10-09T09:27:59.702253+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:27:59.702253+0000","last_undegraded":"2023-10-09T09:27:59.702253+0000","last_fullsized":"2023-10-09T09:27:59.702253+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1012,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.722263+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:27:59.722263+0000","last_peered":"2023-10-09T09:27:59.722263+0000","last_clean":"2023-10-09T09:27:59.722263+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:27:59.722263+0000","last_undegraded":"2023-10-09T09:27:59.722263+0000","last_fullsized":"2023-10-09T09:27:59.722263+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1091'9923","reported_seq":9948,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:14.760903+0000","last_change":"2023-10-09T09:25:25.901310+0000","last_active":"2023-10-09T09:28:14.760903+0000","last_peered":"2023-10-09T09:28:14.760903+0000","last_clean":"2023-10-09T09:28:14.760903+0000","last_became_active":"2023-10-09T09:25:25.900980+0000","last_became_peered":"2023-10-09T09:25:25.900980+0000","last_unstale":"2023-10-09T09:28:14.760903+0000","last_undegraded":"2023-10-09T09:28:14.760903+0000","last_fullsized":"2023-10-09T09:28:14.760903+0000","mapping_epoch":1076,"log_start":"1082'2600","ondisk_log_start":"1082'2600","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7323,"log_dups_size":0,"ondisk_log_size":7323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:10.107258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274006016,"num_objects":4181,"num_object_clones":0,"num_object_copies":12543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9923,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1047,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.704242+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:27:59.704242+0000","last_peered":"2023-10-09T09:27:59.704242+0000","last_clean":"2023-10-09T09:27:59.704242+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:27:59.704242+0000","last_undegraded":"2023-10-09T09:27:59.704242+0000","last_fullsized":"2023-10-09T09:27:59.704242+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1042,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.710010+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:27:59.710010+0000","last_peered":"2023-10-09T09:27:59.710010+0000","last_clean":"2023-10-09T09:27:59.710010+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:27:59.710010+0000","last_undegraded":"2023-10-09T09:27:59.710010+0000","last_fullsized":"2023-10-09T09:27:59.710010+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1042,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.725925+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:27:59.725925+0000","last_peered":"2023-10-09T09:27:59.725925+0000","last_clean":"2023-10-09T09:27:59.725925+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:27:59.725925+0000","last_undegraded":"2023-10-09T09:27:59.725925+0000","last_fullsized":"2023-10-09T09:27:59.725925+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"1091'9617","reported_seq":9640,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:11.291468+0000","last_change":"2023-10-09T09:25:25.903443+0000","last_active":"2023-10-09T09:28:11.291468+0000","last_peered":"2023-10-09T09:28:11.291468+0000","last_clean":"2023-10-09T09:28:11.291468+0000","last_became_active":"2023-10-09T09:25:25.903086+0000","last_became_peered":"2023-10-09T09:25:25.903086+0000","last_unstale":"2023-10-09T09:28:11.291468+0000","last_undegraded":"2023-10-09T09:28:11.291468+0000","last_fullsized":"2023-10-09T09:28:11.291468+0000","mapping_epoch":1076,"log_start":"1081'1200","ondisk_log_start":"1081'1200","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8417,"log_dups_size":0,"ondisk_log_size":8417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283181056,"num_objects":4321,"num_object_clones":0,"num_object_copies":12963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9617,"num_write_kb":446016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1009,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.701612+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:27:59.701612+0000","last_peered":"2023-10-09T09:27:59.701612+0000","last_clean":"2023-10-09T09:27:59.701612+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:27:59.701612+0000","last_undegraded":"2023-10-09T09:27:59.701612+0000","last_fullsized":"2023-10-09T09:27:59.701612+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1027,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.722171+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:27:59.722171+0000","last_peered":"2023-10-09T09:27:59.722171+0000","last_clean":"2023-10-09T09:27:59.722171+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:27:59.722171+0000","last_undegraded":"2023-10-09T09:27:59.722171+0000","last_fullsized":"2023-10-09T09:27:59.722171+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1017,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.723533+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:27:59.723533+0000","last_peered":"2023-10-09T09:27:59.723533+0000","last_clean":"2023-10-09T09:27:59.723533+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:27:59.723533+0000","last_undegraded":"2023-10-09T09:27:59.723533+0000","last_fullsized":"2023-10-09T09:27:59.723533+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1091'9403","reported_seq":9426,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:11.269300+0000","last_change":"2023-10-09T09:25:25.900270+0000","last_active":"2023-10-09T09:28:11.269300+0000","last_peered":"2023-10-09T09:28:11.269300+0000","last_clean":"2023-10-09T09:28:11.269300+0000","last_became_active":"2023-10-09T09:25:25.899813+0000","last_became_peered":"2023-10-09T09:25:25.899813+0000","last_unstale":"2023-10-09T09:28:11.269300+0000","last_undegraded":"2023-10-09T09:28:11.269300+0000","last_fullsized":"2023-10-09T09:28:11.269300+0000","mapping_epoch":1076,"log_start":"0'0","ondisk_log_start":"0'0","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":9403,"log_dups_size":0,"ondisk_log_size":9403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274006016,"num_objects":4181,"num_object_clones":0,"num_object_copies":12543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9403,"num_write_kb":434688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1047,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.702275+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:27:59.702275+0000","last_peered":"2023-10-09T09:27:59.702275+0000","last_clean":"2023-10-09T09:27:59.702275+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:27:59.702275+0000","last_undegraded":"2023-10-09T09:27:59.702275+0000","last_fullsized":"2023-10-09T09:27:59.702275+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1255,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712115+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:27:59.712115+0000","last_peered":"2023-10-09T09:27:59.712115+0000","last_clean":"2023-10-09T09:27:59.712115+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:27:59.712115+0000","last_undegraded":"2023-10-09T09:27:59.712115+0000","last_fullsized":"2023-10-09T09:27:59.712115+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1001,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.739618+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:28:02.739618+0000","last_peered":"2023-10-09T09:28:02.739618+0000","last_clean":"2023-10-09T09:28:02.739618+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:28:02.739618+0000","last_undegraded":"2023-10-09T09:28:02.739618+0000","last_fullsized":"2023-10-09T09:28:02.739618+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1008,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.718285+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:27:59.718285+0000","last_peered":"2023-10-09T09:27:59.718285+0000","last_clean":"2023-10-09T09:27:59.718285+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:27:59.718285+0000","last_undegraded":"2023-10-09T09:27:59.718285+0000","last_fullsized":"2023-10-09T09:27:59.718285+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1091'9712","reported_seq":9737,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:13.627783+0000","last_change":"2023-10-09T09:25:25.897749+0000","last_active":"2023-10-09T09:28:13.627783+0000","last_peered":"2023-10-09T09:28:13.627783+0000","last_clean":"2023-10-09T09:28:13.627783+0000","last_became_active":"2023-10-09T09:25:25.897366+0000","last_became_peered":"2023-10-09T09:25:25.897366+0000","last_unstale":"2023-10-09T09:28:13.627783+0000","last_undegraded":"2023-10-09T09:28:13.627783+0000","last_fullsized":"2023-10-09T09:28:13.627783+0000","mapping_epoch":1076,"log_start":"1081'1600","ondisk_log_start":"1081'1600","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8112,"log_dups_size":0,"ondisk_log_size":8112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:21.152296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262537216,"num_objects":4006,"num_object_clones":0,"num_object_copies":12018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9712,"num_write_kb":438976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1038,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712196+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:27:59.712196+0000","last_peered":"2023-10-09T09:27:59.712196+0000","last_clean":"2023-10-09T09:27:59.712196+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:27:59.712196+0000","last_undegraded":"2023-10-09T09:27:59.712196+0000","last_fullsized":"2023-10-09T09:27:59.712196+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1018,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712191+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:27:59.712191+0000","last_peered":"2023-10-09T09:27:59.712191+0000","last_clean":"2023-10-09T09:27:59.712191+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:27:59.712191+0000","last_undegraded":"2023-10-09T09:27:59.712191+0000","last_fullsized":"2023-10-09T09:27:59.712191+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1029,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712744+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:27:59.712744+0000","last_peered":"2023-10-09T09:27:59.712744+0000","last_clean":"2023-10-09T09:27:59.712744+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:27:59.712744+0000","last_undegraded":"2023-10-09T09:27:59.712744+0000","last_fullsized":"2023-10-09T09:27:59.712744+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1091'9587","reported_seq":9612,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:13.658801+0000","last_change":"2023-10-09T09:25:25.895292+0000","last_active":"2023-10-09T09:28:13.658801+0000","last_peered":"2023-10-09T09:28:13.658801+0000","last_clean":"2023-10-09T09:28:13.658801+0000","last_became_active":"2023-10-09T09:25:25.894970+0000","last_became_peered":"2023-10-09T09:25:25.894970+0000","last_unstale":"2023-10-09T09:28:13.658801+0000","last_undegraded":"2023-10-09T09:28:13.658801+0000","last_fullsized":"2023-10-09T09:28:13.658801+0000","mapping_epoch":1076,"log_start":"1081'1400","ondisk_log_start":"1081'1400","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:21.241118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272171008,"num_objects":4153,"num_object_clones":0,"num_object_copies":12459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9587,"num_write_kb":439680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1023,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712527+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:27:59.712527+0000","last_peered":"2023-10-09T09:27:59.712527+0000","last_clean":"2023-10-09T09:27:59.712527+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:27:59.712527+0000","last_undegraded":"2023-10-09T09:27:59.712527+0000","last_fullsized":"2023-10-09T09:27:59.712527+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1028,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.725789+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:27:59.725789+0000","last_peered":"2023-10-09T09:27:59.725789+0000","last_clean":"2023-10-09T09:27:59.725789+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:27:59.725789+0000","last_undegraded":"2023-10-09T09:27:59.725789+0000","last_fullsized":"2023-10-09T09:27:59.725789+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1002,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712594+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:27:59.712594+0000","last_peered":"2023-10-09T09:27:59.712594+0000","last_clean":"2023-10-09T09:27:59.712594+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:27:59.712594+0000","last_undegraded":"2023-10-09T09:27:59.712594+0000","last_fullsized":"2023-10-09T09:27:59.712594+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1091'9728","reported_seq":9751,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:11.306481+0000","last_change":"2023-10-09T09:25:25.903038+0000","last_active":"2023-10-09T09:28:11.306481+0000","last_peered":"2023-10-09T09:28:11.306481+0000","last_clean":"2023-10-09T09:28:11.306481+0000","last_became_active":"2023-10-09T09:25:25.902503+0000","last_became_peered":"2023-10-09T09:25:25.902503+0000","last_unstale":"2023-10-09T09:28:11.306481+0000","last_undegraded":"2023-10-09T09:28:11.306481+0000","last_fullsized":"2023-10-09T09:28:11.306481+0000","mapping_epoch":1076,"log_start":"1081'1800","ondisk_log_start":"1081'1800","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7928,"log_dups_size":0,"ondisk_log_size":7928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278528000,"num_objects":4250,"num_object_clones":0,"num_object_copies":12750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9728,"num_write_kb":447296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":1006,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.724785+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:27:59.724785+0000","last_peered":"2023-10-09T09:27:59.724785+0000","last_clean":"2023-10-09T09:27:59.724785+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:27:59.724785+0000","last_undegraded":"2023-10-09T09:27:59.724785+0000","last_fullsized":"2023-10-09T09:27:59.724785+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1011,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.723066+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:27:59.723066+0000","last_peered":"2023-10-09T09:27:59.723066+0000","last_clean":"2023-10-09T09:27:59.723066+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:27:59.723066+0000","last_undegraded":"2023-10-09T09:27:59.723066+0000","last_fullsized":"2023-10-09T09:27:59.723066+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1004,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.707845+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:27:59.707845+0000","last_peered":"2023-10-09T09:27:59.707845+0000","last_clean":"2023-10-09T09:27:59.707845+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:27:59.707845+0000","last_undegraded":"2023-10-09T09:27:59.707845+0000","last_fullsized":"2023-10-09T09:27:59.707845+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1091'9439","reported_seq":9462,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:11.255287+0000","last_change":"2023-10-09T09:25:25.902231+0000","last_active":"2023-10-09T09:28:11.255287+0000","last_peered":"2023-10-09T09:28:11.255287+0000","last_clean":"2023-10-09T09:28:11.255287+0000","last_became_active":"2023-10-09T09:25:25.901859+0000","last_became_peered":"2023-10-09T09:25:25.901859+0000","last_unstale":"2023-10-09T09:28:11.255287+0000","last_undegraded":"2023-10-09T09:28:11.255287+0000","last_fullsized":"2023-10-09T09:28:11.255287+0000","mapping_epoch":1076,"log_start":"1081'1100","ondisk_log_start":"1081'1100","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8339,"log_dups_size":0,"ondisk_log_size":8339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273088512,"num_objects":4167,"num_object_clones":0,"num_object_copies":12501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9439,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1027,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.717896+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:27:59.717896+0000","last_peered":"2023-10-09T09:27:59.717896+0000","last_clean":"2023-10-09T09:27:59.717896+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:27:59.717896+0000","last_undegraded":"2023-10-09T09:27:59.717896+0000","last_fullsized":"2023-10-09T09:27:59.717896+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1021,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712948+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:27:59.712948+0000","last_peered":"2023-10-09T09:27:59.712948+0000","last_clean":"2023-10-09T09:27:59.712948+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:27:59.712948+0000","last_undegraded":"2023-10-09T09:27:59.712948+0000","last_fullsized":"2023-10-09T09:27:59.712948+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1026,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.709435+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:27:59.709435+0000","last_peered":"2023-10-09T09:27:59.709435+0000","last_clean":"2023-10-09T09:27:59.709435+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:27:59.709435+0000","last_undegraded":"2023-10-09T09:27:59.709435+0000","last_fullsized":"2023-10-09T09:27:59.709435+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1005,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.712525+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:27:59.712525+0000","last_peered":"2023-10-09T09:27:59.712525+0000","last_clean":"2023-10-09T09:27:59.712525+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:27:59.712525+0000","last_undegraded":"2023-10-09T09:27:59.712525+0000","last_fullsized":"2023-10-09T09:27:59.712525+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1027,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.701521+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:27:59.701521+0000","last_peered":"2023-10-09T09:27:59.701521+0000","last_clean":"2023-10-09T09:27:59.701521+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:27:59.701521+0000","last_undegraded":"2023-10-09T09:27:59.701521+0000","last_fullsized":"2023-10-09T09:27:59.701521+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1045,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.704227+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:27:59.704227+0000","last_peered":"2023-10-09T09:27:59.704227+0000","last_clean":"2023-10-09T09:27:59.704227+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:27:59.704227+0000","last_undegraded":"2023-10-09T09:27:59.704227+0000","last_fullsized":"2023-10-09T09:27:59.704227+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1056,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.710261+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:27:59.710261+0000","last_peered":"2023-10-09T09:27:59.710261+0000","last_clean":"2023-10-09T09:27:59.710261+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:27:59.710261+0000","last_undegraded":"2023-10-09T09:27:59.710261+0000","last_fullsized":"2023-10-09T09:27:59.710261+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1030,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.702334+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:27:59.702334+0000","last_peered":"2023-10-09T09:27:59.702334+0000","last_clean":"2023-10-09T09:27:59.702334+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:27:59.702334+0000","last_undegraded":"2023-10-09T09:27:59.702334+0000","last_fullsized":"2023-10-09T09:27:59.702334+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1056,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.710011+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:27:59.710011+0000","last_peered":"2023-10-09T09:27:59.710011+0000","last_clean":"2023-10-09T09:27:59.710011+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:27:59.710011+0000","last_undegraded":"2023-10-09T09:27:59.710011+0000","last_fullsized":"2023-10-09T09:27:59.710011+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":1017,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.708385+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:27:59.708385+0000","last_peered":"2023-10-09T09:27:59.708385+0000","last_clean":"2023-10-09T09:27:59.708385+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:27:59.708385+0000","last_undegraded":"2023-10-09T09:27:59.708385+0000","last_fullsized":"2023-10-09T09:27:59.708385+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1029,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.708001+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:27:59.708001+0000","last_peered":"2023-10-09T09:27:59.708001+0000","last_clean":"2023-10-09T09:27:59.708001+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:27:59.708001+0000","last_undegraded":"2023-10-09T09:27:59.708001+0000","last_fullsized":"2023-10-09T09:27:59.708001+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1024,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.713104+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:27:59.713104+0000","last_peered":"2023-10-09T09:27:59.713104+0000","last_clean":"2023-10-09T09:27:59.713104+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:27:59.713104+0000","last_undegraded":"2023-10-09T09:27:59.713104+0000","last_fullsized":"2023-10-09T09:27:59.713104+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1041,"reported_epoch":1090,"state":"active+clean","last_fresh":"2023-10-09T09:27:59.709321+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:27:59.709321+0000","last_peered":"2023-10-09T09:27:59.709321+0000","last_clean":"2023-10-09T09:27:59.709321+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:27:59.709321+0000","last_undegraded":"2023-10-09T09:27:59.709321+0000","last_fullsized":"2023-10-09T09:27:59.709321+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4389404696,"num_objects":66978,"num_object_clones":0,"num_object_copies":200934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66978,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":154538,"num_write_kb":7088449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13109702656,"data_stored":13109690440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136838,"ondisk_log_size":136838,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397837,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1955012,"kb_used_data":1649596,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91790140,"statfs":{"total":95995035648,"available":93993103360,"internally_reserved":0,"allocated":1689186304,"data_stored":1688881865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.704,"5min":1.9870000000000001,"15min":1.355},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.7289999999999992,"5min":55.296999999999997,"15min":55.296999999999997},"last":9.0950000000000006},{"interface":"front","average":{"1min":1.804,"5min":2.0960000000000001,"15min":1.458},"min":{"1min":0.46400000000000002,"5min":0.442,"15min":0.442},"max":{"1min":9.8490000000000002,"5min":55.369999999999997,"15min":55.369999999999997},"last":9.1460000000000008}]},{"osd":1,"last update":"Mon Oct 9 09:28:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9610000000000001,"5min":2.411,"15min":1.4099999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.0419999999999998,"5min":103.84699999999999,"15min":103.84699999999999},"last":0.69299999999999995},{"interface":"front","average":{"1min":2.1309999999999998,"5min":2.5409999999999999,"15min":1.5189999999999999},"min":{"1min":0.33300000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":9.8190000000000008,"5min":103.955,"15min":103.955},"last":2.262}]},{"osd":2,"last update":"Mon Oct 9 09:27:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.5249999999999999,"15min":1.2589999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.590999999999999,"5min":17.638000000000002,"15min":17.638000000000002},"last":8.4930000000000003},{"interface":"front","average":{"1min":2.1659999999999999,"5min":1.462,"15min":1.171},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":8.1750000000000007,"5min":17.600000000000001,"15min":17.600000000000001},"last":0.53700000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.823,"5min":3.5499999999999998,"15min":1.8520000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":12.978,"5min":107.97499999999999,"15min":107.97499999999999},"last":2.2850000000000001},{"interface":"front","average":{"1min":2.7090000000000001,"5min":3.6280000000000001,"15min":1.976},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":12.817,"5min":105.282,"15min":105.282},"last":2.2469999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":1.099,"15min":1.004},"min":{"1min":0.377,"5min":0.377,"15min":0.374},"max":{"1min":4.9400000000000004,"5min":6.1550000000000002,"15min":6.1550000000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.446,"5min":1.069,"15min":1.0229999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.109,"5min":5.109,"15min":5.109},"last":0.48599999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.974,"5min":1.2629999999999999,"15min":1.1679999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":10.063000000000001,"5min":10.063000000000001,"15min":10.063000000000001},"last":2.2210000000000001},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.1859999999999999,"15min":1.1040000000000001},"min":{"1min":0.45200000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":5.2389999999999999,"5min":6.4359999999999999,"15min":6.4359999999999999},"last":0.65600000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.202,"15min":1.1240000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.8250000000000002,"5min":6.1219999999999999,"15min":6.1219999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.3029999999999999,"15min":1.212},"min":{"1min":0.51500000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":5.1349999999999998,"5min":5.1349999999999998,"15min":5.1349999999999998},"last":0.624}]}]},{"osd":6,"up_from":43,"seq":184683594070,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1687736,"kb_used_data":1382320,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92057416,"statfs":{"total":95995035648,"available":94266793984,"internally_reserved":0,"allocated":1415495680,"data_stored":1415203383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":1.0509999999999999,"15min":0.96899999999999997},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":4.5110000000000001,"5min":4.5110000000000001,"15min":4.5110000000000001},"last":4.7789999999999999},{"interface":"front","average":{"1min":1.2250000000000001,"5min":0.96199999999999997,"15min":0.98499999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.649,"5min":5.649,"15min":5.649},"last":0.76400000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:28:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":1.415,"15min":1.1120000000000001},"min":{"1min":0.33200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.253,"5min":10.253,"15min":10.253},"last":4.8819999999999997},{"interface":"front","average":{"1min":1.679,"5min":1.1899999999999999,"15min":1.022},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.641,"5min":8.641,"15min":8.641},"last":0.52500000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.083,"15min":0.98299999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.1829999999999998,"5min":4.1829999999999998,"15min":4.1829999999999998},"last":1.091},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.3100000000000001,"15min":1.1140000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.7629999999999999,"5min":6.0039999999999996,"15min":6.0039999999999996},"last":0.73399999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:27:53 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":1.28,"15min":1.1000000000000001},"min":{"1min":0.38600000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.04,"5min":7.04,"15min":7.04},"last":4.4610000000000003},{"interface":"front","average":{"1min":1.27,"5min":1.3680000000000001,"15min":1.1859999999999999},"min":{"1min":0.46200000000000002,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":3.4079999999999999,"5min":11.843999999999999,"15min":11.843999999999999},"last":0.67700000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:27:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.173,"15min":1.0349999999999999},"min":{"1min":0.374,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":9.1099999999999994,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":0.68799999999999994},{"interface":"front","average":{"1min":1.222,"5min":1.159,"15min":1.091},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.3759999999999999,"5min":3.5569999999999999,"15min":3.5569999999999999},"last":0.56799999999999995}]},{"osd":5,"last update":"Mon Oct 9 09:27:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.103,"15min":1.0960000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":4.1280000000000001,"5min":4.1280000000000001,"15min":4.1280000000000001},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.617,"5min":1.161,"15min":1.1459999999999999},"min":{"1min":0.47699999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":8.5950000000000006,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.63300000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:27:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":1.2649999999999999,"15min":1.2010000000000001},"min":{"1min":0.436,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":7.3630000000000004,"5min":7.3630000000000004,"15min":7.3630000000000004},"last":7.1100000000000003},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.212,"15min":1.1899999999999999},"min":{"1min":0.45400000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":4.7610000000000001,"5min":4.7610000000000001,"15min":4.7610000000000001},"last":7.6459999999999999}]}]},{"osd":2,"up_from":634,"seq":2723009265824,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2189784,"kb_used_data":2084624,"kb_used_omap":122,"kb_used_meta":104965,"kb_avail":91555368,"statfs":{"total":95995035648,"available":93752696832,"internally_reserved":0,"allocated":2134654976,"data_stored":2133463634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":107484747},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":1.1599999999999999,"15min":0.88800000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.9860000000000007,"5min":8.9860000000000007,"15min":8.9860000000000007},"last":4.7089999999999996},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.266,"15min":0.89500000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.5709999999999997,"5min":7.71,"15min":7.71},"last":0.98599999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.0840000000000001,"15min":0.84299999999999997},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":6.6600000000000001,"5min":6.798,"15min":6.798},"last":0.65200000000000002},{"interface":"front","average":{"1min":2.27,"5min":1.369,"15min":0.98199999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":8.702,"5min":8.702,"15min":8.702},"last":4.6349999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:27:53 2023","interfaces":[{"interface":"back","average":{"1min":2.411,"5min":1.528,"15min":1.0509999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":11.597,"5min":11.597,"15min":11.597},"last":1.7889999999999999},{"interface":"front","average":{"1min":2.262,"5min":1.393,"15min":1.056},"min":{"1min":0.34599999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.0259999999999998,"5min":9.0259999999999998,"15min":9.0259999999999998},"last":4.6820000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":1.169,"15min":1.091},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":4.9630000000000001,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":7.452},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.294,"15min":1.119},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":5.1959999999999997,"5min":7.5060000000000002,"15min":7.5060000000000002},"last":0.70599999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.175,"15min":1.0940000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.7619999999999996,"5min":6.7699999999999996,"15min":6.7699999999999996},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.552,"5min":1.2290000000000001,"15min":1.171},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.9320000000000004,"5min":4.9320000000000004,"15min":4.9320000000000004},"last":10.393000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.224,"15min":1.1870000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.3259999999999996,"5min":5.3259999999999996,"15min":5.3259999999999996},"last":4.8010000000000002},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.353,"15min":1.1990000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.5230000000000001,"5min":7.5860000000000003,"15min":7.5860000000000003},"last":8.577}]},{"osd":7,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.534,"15min":1.292},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":5.1660000000000004,"5min":7.7469999999999999,"15min":7.7469999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.349,"15min":1.2250000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":6.7350000000000003,"5min":6.7350000000000003,"15min":6.7350000000000003},"last":0.63100000000000001}]}]},{"osd":1,"up_from":607,"seq":2607045148837,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1433352,"kb_used_data":1334016,"kb_used_omap":107,"kb_used_meta":99220,"kb_avail":92311800,"statfs":{"total":95995035648,"available":94527283200,"internally_reserved":0,"allocated":1366032384,"data_stored":1364854195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":101601963},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":0.80100000000000005,"15min":0.71699999999999997},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":5.4569999999999999,"5min":5.4569999999999999,"15min":5.4569999999999999},"last":1.04},{"interface":"front","average":{"1min":2.238,"5min":1.1120000000000001,"15min":0.85399999999999998},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":11.845000000000001,"5min":11.845000000000001,"15min":11.845000000000001},"last":0.311}]},{"osd":2,"last update":"Mon Oct 9 09:27:34 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.3220000000000001,"15min":0.93899999999999995},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.6099999999999994,"5min":9.7959999999999994,"15min":9.7959999999999994},"last":4.6909999999999998},{"interface":"front","average":{"1min":1.929,"5min":1.1839999999999999,"15min":0.95199999999999996},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7899999999999991,"5min":9.9260000000000002,"15min":9.9260000000000002},"last":0.33900000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:27:53 2023","interfaces":[{"interface":"back","average":{"1min":8.2530000000000001,"5min":2.9500000000000002,"15min":1.5449999999999999},"min":{"1min":0.26400000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":107.426,"5min":107.426,"15min":107.426},"last":0.90100000000000002},{"interface":"front","average":{"1min":8.8000000000000007,"5min":3.2050000000000001,"15min":1.677},"min":{"1min":0.32000000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":106.15900000000001,"5min":106.15900000000001,"15min":106.15900000000001},"last":3.0609999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.0189999999999999,"15min":1.0329999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.3049999999999997,"5min":7.3049999999999997,"15min":7.3049999999999997},"last":2.169},{"interface":"front","average":{"1min":2.0489999999999999,"5min":1.266,"15min":1.121},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":11.933999999999999,"5min":11.933999999999999,"15min":11.933999999999999},"last":0.38600000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":1.1279999999999999,"15min":1.111},"min":{"1min":0.38800000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.2469999999999999,"5min":7.2469999999999999,"15min":7.2469999999999999},"last":1.0089999999999999},{"interface":"front","average":{"1min":2.0059999999999998,"5min":1.2629999999999999,"15min":1.147},"min":{"1min":0.33100000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":11.907,"5min":11.907,"15min":11.907},"last":0.60099999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.385,"15min":1.1830000000000001},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.379},"max":{"1min":7.476,"5min":8.1340000000000003,"15min":8.1340000000000003},"last":2.9220000000000002},{"interface":"front","average":{"1min":1.2390000000000001,"5min":1.179,"15min":1.1499999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":7.3970000000000002,"5min":7.3970000000000002,"15min":7.3970000000000002},"last":0.97899999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":1.5580000000000001,"15min":1.3029999999999999},"min":{"1min":0.40899999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.5119999999999996,"5min":8.1029999999999998,"15min":8.1029999999999998},"last":3.3820000000000001},{"interface":"front","average":{"1min":1.284,"5min":1.157,"15min":1.2090000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":7.3520000000000003,"5min":7.3520000000000003,"15min":7.3520000000000003},"last":1.0640000000000001}]}]},{"osd":0,"up_from":581,"seq":2495375999146,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1173536,"kb_used_data":1077784,"kb_used_omap":131,"kb_used_meta":95548,"kb_avail":92571616,"statfs":{"total":95995035648,"available":94793334784,"internally_reserved":0,"allocated":1103650816,"data_stored":1102451285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":97841753},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,2,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:28:07 2023","interfaces":[{"interface":"back","average":{"1min":1.915,"5min":1.24,"15min":0.90200000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.1230000000000002,"5min":14.44,"15min":14.44},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.385,"5min":1.349,"15min":0.98199999999999998},"min":{"1min":0.23499999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.4320000000000004,"5min":14.663,"15min":14.663},"last":2.6419999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":1.264,"15min":0.94999999999999996},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":6.976,"5min":21.093,"15min":21.093},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.841,"5min":1.143,"15min":0.89800000000000002},"min":{"1min":0.21099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.9939999999999998,"5min":19.614000000000001,"15min":19.614000000000001},"last":0.54600000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:28:00 2023","interfaces":[{"interface":"back","average":{"1min":9.0609999999999999,"5min":2.5510000000000002,"15min":1.3979999999999999},"min":{"1min":0.34399999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":122.749,"5min":122.749,"15min":122.749},"last":3.7400000000000002},{"interface":"front","average":{"1min":8.6690000000000005,"5min":2.4969999999999999,"15min":1.4770000000000001},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":123.51900000000001,"5min":123.51900000000001,"15min":123.51900000000001},"last":0.53200000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:27:50 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.1870000000000001,"15min":1.0669999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":7.093,"5min":7.093,"15min":7.093},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.508,"5min":1.2230000000000001,"15min":1.095},"min":{"1min":0.35599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3310000000000004,"5min":7.3310000000000004,"15min":7.3310000000000004},"last":0.48799999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:27:50 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.173,"15min":1.1100000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.3620000000000001,"5min":7.3620000000000001,"15min":7.3620000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.9199999999999999,"5min":1.3200000000000001,"15min":1.1639999999999999},"min":{"1min":0.371,"5min":0.312,"15min":0.312},"max":{"1min":8.5609999999999999,"5min":8.5609999999999999,"15min":8.5609999999999999},"last":0.55600000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:27:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":1.3879999999999999,"15min":1.2130000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.202,"5min":8.202,"15min":8.202},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.8879999999999999,"5min":1.272,"15min":1.1830000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.3289999999999997,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":0.70599999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:27:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":1.4450000000000001,"15min":1.274},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":8.1769999999999996,"5min":8.1769999999999996,"15min":8.1769999999999996},"last":2.0800000000000001},{"interface":"front","average":{"1min":1.879,"5min":1.367,"15min":1.26},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":8.2710000000000008,"5min":8.6549999999999994,"15min":8.6549999999999994},"last":0.39300000000000002}]}]},{"osd":3,"up_from":663,"seq":2847563317402,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2810684,"kb_used_data":2670388,"kb_used_omap":55,"kb_used_meta":140232,"kb_avail":90934468,"statfs":{"total":95995035648,"available":93116895232,"internally_reserved":0,"allocated":2734477312,"data_stored":2733417625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":143598178},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,4,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.2230000000000001,"15min":0.93999999999999995},"min":{"1min":0.23000000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":6.1390000000000002,"5min":7.9080000000000004,"15min":7.9080000000000004},"last":0.46200000000000002},{"interface":"front","average":{"1min":2.4620000000000002,"5min":1.3220000000000001,"15min":0.95999999999999996},"min":{"1min":0.28599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":10.938000000000001,"5min":11.507999999999999,"15min":11.507999999999999},"last":0.27600000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.401,"15min":1.04},"min":{"1min":0.36499999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":11.090999999999999,"5min":11.581,"15min":11.581},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.758,"5min":1.597,"15min":1.095},"min":{"1min":0.25800000000000001,"5min":0.249,"15min":0.249},"max":{"1min":6.4660000000000002,"5min":7.6509999999999998,"15min":7.6509999999999998},"last":1.0609999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6349999999999998,"5min":1.54,"15min":1.0880000000000001},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.7370000000000001,"5min":7.867,"15min":7.867},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.347,"5min":1.6479999999999999,"15min":1.1419999999999999},"min":{"1min":0.248,"5min":0.223,"15min":0.223},"max":{"1min":7.3550000000000004,"5min":13.856,"15min":13.856},"last":4.6900000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":1.532,"15min":1.2050000000000001},"min":{"1min":0.42699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":12.629,"5min":12.629,"15min":12.629},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.9339999999999999,"5min":1.4350000000000001,"15min":1.1479999999999999},"min":{"1min":0.313,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.827,"5min":7.827,"15min":7.827},"last":4.6100000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4129999999999998,"5min":1.5940000000000001,"15min":1.2889999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":6.593,"5min":7.6929999999999996,"15min":7.6929999999999996},"last":0.66600000000000004},{"interface":"front","average":{"1min":3.0259999999999998,"5min":1.631,"15min":1.3049999999999999},"min":{"1min":0.40400000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":11.135999999999999,"5min":11.388999999999999,"15min":11.388999999999999},"last":0.55800000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":1.54,"15min":1.242},"min":{"1min":0.33600000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.7830000000000004,"5min":7.7830000000000004,"15min":7.7830000000000004},"last":4.5590000000000002},{"interface":"front","average":{"1min":2.8300000000000001,"5min":1.6739999999999999,"15min":1.3460000000000001},"min":{"1min":0.438,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":11.058999999999999,"5min":17.234999999999999,"15min":17.234999999999999},"last":0.52000000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.7010000000000001,"5min":1.708,"15min":1.3799999999999999},"min":{"1min":0.45900000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.5650000000000004,"5min":7.7320000000000002,"15min":7.7320000000000002},"last":0.63100000000000001},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.6439999999999999,"15min":1.3169999999999999},"min":{"1min":0.375,"5min":0.318,"15min":0.318},"max":{"1min":7.4039999999999999,"5min":7.5869999999999997,"15min":7.5869999999999997},"last":4.6420000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986535,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1672828,"kb_used_data":1363316,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92072324,"statfs":{"total":95995035648,"available":94282059776,"internally_reserved":0,"allocated":1396035584,"data_stored":1395738723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":0.94299999999999995,"15min":0.90900000000000003},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.2800000000000002,"5min":4.2800000000000002,"15min":4.2800000000000002},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.698,"5min":1.0589999999999999,"15min":0.98399999999999999},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.5149999999999997,"5min":7.5149999999999997,"15min":7.5149999999999997},"last":1.4359999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.0289999999999999,"15min":0.93000000000000005},"min":{"1min":0.36499999999999999,"5min":0.312,"15min":0.312},"max":{"1min":4.399,"5min":4.399,"15min":4.399},"last":2.5179999999999998},{"interface":"front","average":{"1min":1.1040000000000001,"5min":0.97099999999999997,"15min":0.99299999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.3679999999999999,"5min":3.3679999999999999,"15min":3.3679999999999999},"last":2.456}]},{"osd":2,"last update":"Mon Oct 9 09:27:21 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":1.093,"15min":1.036},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.4649999999999999,"5min":5.2919999999999998,"15min":5.7199999999999998},"last":1.3580000000000001},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.0489999999999999,"15min":0.995},"min":{"1min":0.35299999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.2850000000000001,"5min":3.2850000000000001,"15min":3.2850000000000001},"last":1.5169999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":3.9180000000000001,"5min":1.7130000000000001,"15min":1.2589999999999999},"min":{"1min":0.38100000000000001,"5min":0.372,"15min":0.372},"max":{"1min":17.376000000000001,"5min":17.376000000000001,"15min":17.376000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":4.2510000000000003,"5min":1.8220000000000001,"15min":1.3660000000000001},"min":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":17.277999999999999,"5min":17.277999999999999,"15min":17.277999999999999},"last":1.48}]},{"osd":5,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.2649999999999999,"15min":1.0740000000000001},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.8319999999999999,"5min":7.2999999999999998,"15min":7.2999999999999998},"last":5.6369999999999996},{"interface":"front","average":{"1min":1.502,"5min":1.4410000000000001,"15min":1.2130000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.165,"5min":7.5819999999999999,"15min":7.5819999999999999},"last":1.4530000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.8,"5min":1.2030000000000001,"15min":1.1020000000000001},"min":{"1min":0.44500000000000001,"5min":0.377,"15min":0.377},"max":{"1min":7.625,"5min":7.625,"15min":7.625},"last":1.5880000000000001},{"interface":"front","average":{"1min":1.9730000000000001,"5min":1.2749999999999999,"15min":1.165},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":7.1420000000000003,"5min":7.1420000000000003,"15min":7.1420000000000003},"last":5.6950000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":1.147,"15min":1.079},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":7.2830000000000004,"5min":7.2830000000000004,"15min":7.2830000000000004},"last":5.7270000000000003},{"interface":"front","average":{"1min":1.748,"5min":1.3440000000000001,"15min":1.1950000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":4.3620000000000001,"5min":4.3620000000000001,"15min":4.3620000000000001},"last":5.2350000000000003}]}]},{"osd":5,"up_from":37,"seq":158913790301,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1658736,"kb_used_data":1357416,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92086416,"statfs":{"total":95995035648,"available":94296489984,"internally_reserved":0,"allocated":1389993984,"data_stored":1389705179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:27:39 2023","interfaces":[{"interface":"back","average":{"1min":1.665,"5min":1.151,"15min":0.95699999999999996},"min":{"1min":0.309,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.5549999999999997,"5min":10.109,"15min":10.109},"last":0.315},{"interface":"front","average":{"1min":1.359,"5min":1.077,"15min":0.97499999999999998},"min":{"1min":0.41599999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.8630000000000004,"5min":4.8630000000000004,"15min":4.8630000000000004},"last":3.3929999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":0.998,"15min":0.96599999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.0270000000000001,"5min":4.7290000000000001,"15min":4.7290000000000001},"last":2.2010000000000001},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.081,"15min":1.0700000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.0590000000000002,"5min":4.766,"15min":4.766},"last":2.2519999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:27:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":1.1879999999999999,"15min":1.0820000000000001},"min":{"1min":0.377,"5min":0.372,"15min":0.372},"max":{"1min":5.6159999999999997,"5min":5.6159999999999997,"15min":5.6159999999999997},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.73,"5min":1.1719999999999999,"15min":1.0629999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.6269999999999998,"5min":5.6269999999999998,"15min":5.6269999999999998},"last":9.0060000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:27:54 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":1.3149999999999999,"15min":1.101},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.798,"5min":10.193,"15min":10.193},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.1419999999999999,"15min":1.1160000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.5579999999999998,"5min":7.5579999999999998,"15min":7.5579999999999998},"last":3.431}]},{"osd":4,"last update":"Mon Oct 9 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":1.532,"5min":1.0149999999999999,"15min":0.96799999999999997},"min":{"1min":0.38800000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":10.164999999999999,"5min":10.164999999999999,"15min":10.164999999999999},"last":0.495},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.0089999999999999,"15min":1.046},"min":{"1min":0.40899999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.427,"5min":3.427,"15min":3.427},"last":3.456}]},{"osd":6,"last update":"Mon Oct 9 09:27:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":1.157,"15min":1.0900000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.6139999999999999,"5min":10.253,"15min":10.253},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.393,"5min":1.181,"15min":1.2250000000000001},"min":{"1min":0.46500000000000002,"5min":0.372,"15min":0.372},"max":{"1min":7.5960000000000001,"5min":7.5960000000000001,"15min":7.5960000000000001},"last":3.7610000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:27:21 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.2130000000000001,"15min":1.204},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.4950000000000001,"5min":4.4950000000000001,"15min":4.4950000000000001},"last":3.6040000000000001},{"interface":"front","average":{"1min":1.952,"5min":1.262,"15min":1.1899999999999999},"min":{"1min":0.41699999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":6.5229999999999997,"5min":6.5229999999999997,"15min":6.5229999999999997},"last":0.58399999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1094586368,"data_stored":1094582296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1353453568,"data_stored":1353449496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2124873728,"data_stored":2124873728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2696544256,"data_stored":2696544256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1389166592,"data_stored":1389166592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1383534592,"data_stored":1383530520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1397751808,"data_stored":1397751808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1669791744,"data_stored":1669791744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:28:19.792 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:28:19.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:19.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:19.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:19.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:19.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:19.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:20.039 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:20.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:20.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:20.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:20.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:20.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:20.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:20.295 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:20.329 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:20.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:20.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:20.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:20.460 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:20.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:20.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:20.564 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:20.596 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:20.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:20.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:20.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:20.744 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:20.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:20.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:20.850 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:20.888 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:20.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:20 smithi163 bash[75425]: cluster 2023-10-09T09:28:19.646985+0000 mgr.y (mgr.34107) 659 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T09:28:20.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:20 smithi163 bash[78107]: cluster 2023-10-09T09:28:19.646985+0000 mgr.y (mgr.34107) 659 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T09:28:20.992 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:21.023 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:21.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:21.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:21.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:20 smithi181 bash[59345]: cluster 2023-10-09T09:28:19.646985+0000 mgr.y (mgr.34107) 659 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T09:28:21.128 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:21.159 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:21.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:21.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:21.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:21.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:21.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:21.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:21.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:21.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:21.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:21.538 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:21.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:21.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:21.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:21.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:21.707 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:21.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:21 smithi181 bash[59345]: audit 2023-10-09T09:28:21.265855+0000 mon.a (mon.0) 1465 : audit 1 from='client.? 172.21.15.163:0/3249450948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:28:21.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:21.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:21.740 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:28:21.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:21.843 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:21.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:21.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:21.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:21.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:21 smithi163 bash[78107]: audit 2023-10-09T09:28:21.265855+0000 mon.a (mon.0) 1465 : audit 1 from='client.? 172.21.15.163:0/3249450948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:28:21.954 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:21 smithi163 bash[75425]: audit 2023-10-09T09:28:21.265855+0000 mon.a (mon.0) 1465 : audit 1 from='client.? 172.21.15.163:0/3249450948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:28:21.982 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:21.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:21.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:22.089 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:22.115 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:22.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:22.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:22.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:22.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:22.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:22.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:22.360 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:22.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:22.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:22.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:22.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:22.509 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:22.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:22.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:22.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:22.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:22.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:22.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:22.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:28:22.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:22.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:22.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:22.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:22.903 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:22.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[75425]: cluster 2023-10-09T09:28:21.647660+0000 mgr.y (mgr.34107) 660 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-10-09T09:28:22.917 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[75425]: cluster 2023-10-09T09:28:21.701414+0000 mon.a (mon.0) 1466 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:28:22.917 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[75425]: cluster 2023-10-09T09:28:21.723248+0000 osd.4 (osd.4) 520 : cluster 3 failed to encode map e1092 with expected crc 2023-10-09T09:28:22.917 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[75425]: cluster 2023-10-09T09:28:21.724164+0000 osd.5 (osd.5) 636 : cluster 3 failed to encode map e1092 with expected crc 2023-10-09T09:28:22.917 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[75425]: audit 2023-10-09T09:28:21.725805+0000 mon.a (mon.0) 1467 : audit 1 from='client.? 172.21.15.163:0/3249450948' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:28:22.917 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[75425]: cluster 2023-10-09T09:28:21.725985+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e1092: 8 total, 8 up, 8 in 2023-10-09T09:28:22.917 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[75425]: cluster 2023-10-09T09:28:21.738729+0000 osd.7 (osd.7) 485 : cluster 3 failed to encode map e1092 with expected crc 2023-10-09T09:28:22.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[78107]: cluster 2023-10-09T09:28:21.647660+0000 mgr.y (mgr.34107) 660 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-10-09T09:28:22.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[78107]: cluster 2023-10-09T09:28:21.701414+0000 mon.a (mon.0) 1466 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:28:22.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[78107]: cluster 2023-10-09T09:28:21.723248+0000 osd.4 (osd.4) 520 : cluster 3 failed to encode map e1092 with expected crc 2023-10-09T09:28:22.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[78107]: cluster 2023-10-09T09:28:21.724164+0000 osd.5 (osd.5) 636 : cluster 3 failed to encode map e1092 with expected crc 2023-10-09T09:28:22.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[78107]: audit 2023-10-09T09:28:21.725805+0000 mon.a (mon.0) 1467 : audit 1 from='client.? 172.21.15.163:0/3249450948' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:28:22.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[78107]: cluster 2023-10-09T09:28:21.725985+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e1092: 8 total, 8 up, 8 in 2023-10-09T09:28:22.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:22 smithi163 bash[78107]: cluster 2023-10-09T09:28:21.738729+0000 osd.7 (osd.7) 485 : cluster 3 failed to encode map e1092 with expected crc 2023-10-09T09:28:22.948 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:22.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:22.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:23.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:23.063 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:22 smithi181 bash[59345]: cluster 2023-10-09T09:28:21.647660+0000 mgr.y (mgr.34107) 660 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-10-09T09:28:23.064 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:22 smithi181 bash[59345]: cluster 2023-10-09T09:28:21.701414+0000 mon.a (mon.0) 1466 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:28:23.064 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:22 smithi181 bash[59345]: cluster 2023-10-09T09:28:21.723248+0000 osd.4 (osd.4) 520 : cluster 3 failed to encode map e1092 with expected crc 2023-10-09T09:28:23.064 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:22 smithi181 bash[59345]: cluster 2023-10-09T09:28:21.724164+0000 osd.5 (osd.5) 636 : cluster 3 failed to encode map e1092 with expected crc 2023-10-09T09:28:23.064 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:22 smithi181 bash[59345]: audit 2023-10-09T09:28:21.725805+0000 mon.a (mon.0) 1467 : audit 1 from='client.? 172.21.15.163:0/3249450948' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:28:23.064 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:22 smithi181 bash[59345]: cluster 2023-10-09T09:28:21.725985+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e1092: 8 total, 8 up, 8 in 2023-10-09T09:28:23.064 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:22 smithi181 bash[59345]: cluster 2023-10-09T09:28:21.738729+0000 osd.7 (osd.7) 485 : cluster 3 failed to encode map e1092 with expected crc 2023-10-09T09:28:23.087 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:23.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:23.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:23.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:23.219 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:23.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:23.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:23.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:23.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:23.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:23.467 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:28:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:28:23] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.33.4" 2023-10-09T09:28:23.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:23.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:23.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:23.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:23.634 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:23.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:23.727 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:23 smithi181 bash[59345]: cluster 2023-10-09T09:28:23.648165+0000 mgr.y (mgr.34107) 661 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T09:28:23.743 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:23.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:23.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:23.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:23.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:23.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:23.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:23.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:24.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:24.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:23 smithi163 bash[75425]: cluster 2023-10-09T09:28:23.648165+0000 mgr.y (mgr.34107) 661 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T09:28:24.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:23 smithi163 bash[78107]: cluster 2023-10-09T09:28:23.648165+0000 mgr.y (mgr.34107) 661 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T09:28:24.038 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:24.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:24.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:24.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:24.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:24.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:24.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:24.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:24.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:24.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:24.415 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:24.444 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:24.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:24.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:24.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:24.582 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:24.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:24.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:24.688 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:24.716 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:24.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:24.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:24.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:24.846 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:24.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:24.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:24.940 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:28:24.951 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:24.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:24.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:24.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:25.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:25.121 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:25.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:25.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:25.225 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:25.273 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:25.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:25.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:25.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:25.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:25.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:25.438 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:25.438 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:25.517 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:25.550 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:25.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:25.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:25.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:25.680 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:25.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:25.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:25.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:25.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:25.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:25.922 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:25.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:25.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:26.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:26.094 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:26.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:26.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:26.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:26.219 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:26.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:26.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:26.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:26.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:26.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:26.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:26.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:26.460 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:26 smithi163 bash[78107]: audit 2023-10-09T09:28:25.371418+0000 mon.a (mon.0) 1469 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:26.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:26 smithi163 bash[78107]: audit 2023-10-09T09:28:25.372808+0000 mon.a (mon.0) 1470 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:26.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:26 smithi163 bash[78107]: cluster 2023-10-09T09:28:25.648791+0000 mgr.y (mgr.34107) 662 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T09:28:26.461 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:26 smithi163 bash[75425]: audit 2023-10-09T09:28:25.371418+0000 mon.a (mon.0) 1469 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:26.461 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:26 smithi163 bash[75425]: audit 2023-10-09T09:28:25.372808+0000 mon.a (mon.0) 1470 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:26.461 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:26 smithi163 bash[75425]: cluster 2023-10-09T09:28:25.648791+0000 mgr.y (mgr.34107) 662 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T09:28:26.480 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:26.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:26.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:26.585 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:26.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:26 smithi181 bash[59345]: audit 2023-10-09T09:28:25.371418+0000 mon.a (mon.0) 1469 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:28:26.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:26 smithi181 bash[59345]: audit 2023-10-09T09:28:25.372808+0000 mon.a (mon.0) 1470 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:26.593 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:26 smithi181 bash[59345]: cluster 2023-10-09T09:28:25.648791+0000 mgr.y (mgr.34107) 662 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T09:28:26.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:26.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:26.720 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:26.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:26.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:26.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:26.862 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:26.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:26.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:26.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:26.999 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:27.029 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:27.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:27.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:27.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:27.166 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:27.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:27.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:27.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:27.295 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:27.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:27.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:27.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:27.430 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:27.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:27.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:27.463 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:27.535 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:27.558 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:27.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:27.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:27.691 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:27.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:27.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:27.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:27.825 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:27.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:27.930 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:27.957 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:27.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:27.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:28.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:28.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:28.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:28.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:28.241 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:28.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:28.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:28.347 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:28.374 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:28.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:28.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:28.503 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:28.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:28.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:28.606 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:28.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:28.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:28.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:28.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:28 smithi181 bash[59345]: cluster 2023-10-09T09:28:27.649373+0000 mgr.y (mgr.34107) 663 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T09:28:28.735 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:28.752 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:28.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:28.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:28.854 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:28.879 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:28.882 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:28 smithi163 bash[75425]: cluster 2023-10-09T09:28:27.649373+0000 mgr.y (mgr.34107) 663 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T09:28:28.882 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:28 smithi163 bash[78107]: cluster 2023-10-09T09:28:27.649373+0000 mgr.y (mgr.34107) 663 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T09:28:28.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:28.983 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:29.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:29.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:29.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:29.113 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:29.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:29.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:29.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:29.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:29.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:29.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:29.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:29.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:29.415 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:29.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:29.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:29.513 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:29.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:29.548 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:29.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:29.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:29.659 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:29.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:29.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:29.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:29.792 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:29.822 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:29.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:29.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:29.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:29.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:30.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:30.094 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:30.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:30.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:30.198 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:30.221 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:30.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:30.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:30.327 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:30.361 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:30.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:30.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:30.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:30.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:30.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:30.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:30.601 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:30.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:30.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:30.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:30.743 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:30 smithi163 bash[75425]: cluster 2023-10-09T09:28:29.649878+0000 mgr.y (mgr.34107) 664 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T09:28:30.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:30 smithi163 bash[78107]: cluster 2023-10-09T09:28:29.649878+0000 mgr.y (mgr.34107) 664 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T09:28:30.764 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:30.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:30.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:30.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:30.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:30.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:30.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:30.992 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:31.016 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:31.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:31.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:31.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:30 smithi181 bash[59345]: cluster 2023-10-09T09:28:29.649878+0000 mgr.y (mgr.34107) 664 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T09:28:31.120 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:31.145 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:31.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:31.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:31.256 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:31.279 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:31.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:31.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:31.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:31.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:31.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:31.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:31.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:31.548 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:31.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:31.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:31.656 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:31.678 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:31.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:31.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:31.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:31.808 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:31.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:31.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:31.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:31.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:31.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:32.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:32.078 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:32.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:32.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:32.183 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:32.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:32.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:32.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:32.349 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:32.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:32.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:32.480 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:32.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:32.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:32.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:32.612 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:32.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:32.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:32.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:32.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:32.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:32.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:32.857 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:32.882 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:32.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:32.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:32 smithi163 bash[78107]: cluster 2023-10-09T09:28:31.654343+0000 mgr.y (mgr.34107) 665 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T09:28:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:32 smithi163 bash[75425]: cluster 2023-10-09T09:28:31.654343+0000 mgr.y (mgr.34107) 665 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T09:28:32.985 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:33.013 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:33.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:33.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:33.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:32 smithi181 bash[59345]: cluster 2023-10-09T09:28:31.654343+0000 mgr.y (mgr.34107) 665 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T09:28:33.117 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:33.145 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:33.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:33.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:33.251 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:33.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:33.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:33.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:33.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:33.407 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:33.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:33.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:33.512 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:33.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:33.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:33.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:33.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:33.680 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:33.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:28:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:28:33] "GET /metrics HTTP/1.1" 200 36331 "" "Prometheus/2.33.4" 2023-10-09T09:28:33.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:33.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:33.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:33.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:33.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:33.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:33.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:33.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:34.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:34.053 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:33 smithi163 bash[78107]: cluster 2023-10-09T09:28:33.654855+0000 mgr.y (mgr.34107) 666 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T09:28:34.053 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:33 smithi163 bash[75425]: cluster 2023-10-09T09:28:33.654855+0000 mgr.y (mgr.34107) 666 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T09:28:34.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:33 smithi181 bash[59345]: cluster 2023-10-09T09:28:33.654855+0000 mgr.y (mgr.34107) 666 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T09:28:34.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:34.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:34.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:34.207 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:34.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:34.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:34.311 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:34.343 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:34.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:34.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:34.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:34.477 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:34.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:34.580 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:34.599 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:34.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:34.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:34.702 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:34.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:34.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:34.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:34.848 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:34.880 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:34.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:34.983 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:35.018 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:35.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:35.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:35.121 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:35.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:35.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:35.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:35.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:35.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:35.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:35.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:35.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:35.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:35.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:35.547 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:35.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:35.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:35.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:35.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:35.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:35.779 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:35.898 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:35.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:35.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:36.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:36.030 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:36.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:36.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:36.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:36.166 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:36.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:36.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:36.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:36.294 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:36.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:36.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:36.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:36 smithi181 bash[59345]: cluster 2023-10-09T09:28:35.655550+0000 mgr.y (mgr.34107) 667 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T09:28:36.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:37.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:36 smithi163 bash[75425]: cluster 2023-10-09T09:28:35.655550+0000 mgr.y (mgr.34107) 667 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T09:28:37.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:36 smithi163 bash[78107]: cluster 2023-10-09T09:28:35.655550+0000 mgr.y (mgr.34107) 667 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T09:28:37.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:37.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:37.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:37.332 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:37.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:37.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:37.439 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:37.920 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:28:37.965 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:28:37.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:37.965 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:37.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:38.017 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:28:38.073 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:28:38.073 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:28:38.074 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:28:38.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:38.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:38.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:38.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:38.139 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:28:38.180 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:28:38.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:38.221 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:28:38.241 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:38.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:38.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:38.262 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:28:38.307 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:28:38.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:38.350 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:28:38.357 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:28:38.357 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:28:38.357 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:28:38.358 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:28:38.358 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:28:38.358 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:28:38.358 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:28:38.358 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:28:38.382 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:38.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:38.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:38.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:38.534 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:38.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:38.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:38.684 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:38.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:38.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:38.744 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:28:38.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:38 smithi163 bash[75425]: cluster 2023-10-09T09:28:37.656158+0000 mgr.y (mgr.34107) 668 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-10-09T09:28:38.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:38 smithi163 bash[75425]: audit 2023-10-09T09:28:37.933783+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.163:0/3944411043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:28:38.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:38 smithi163 bash[75425]: audit 2023-10-09T09:28:38.664628+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.163:0/3189378162' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:28:38.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:38 smithi163 bash[75425]: audit 2023-10-09T09:28:38.668386+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:28:38.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:38 smithi163 bash[78107]: cluster 2023-10-09T09:28:37.656158+0000 mgr.y (mgr.34107) 668 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-10-09T09:28:38.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:38 smithi163 bash[78107]: audit 2023-10-09T09:28:37.933783+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.163:0/3944411043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:28:38.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:38 smithi163 bash[78107]: audit 2023-10-09T09:28:38.664628+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.163:0/3189378162' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:28:38.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:38 smithi163 bash[78107]: audit 2023-10-09T09:28:38.668386+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:28:38.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:38.821 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:38.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:39.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:38 smithi181 bash[59345]: cluster 2023-10-09T09:28:37.656158+0000 mgr.y (mgr.34107) 668 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-10-09T09:28:39.342 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:38 smithi181 bash[59345]: audit 2023-10-09T09:28:37.933783+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.163:0/3944411043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:28:39.342 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:38 smithi181 bash[59345]: audit 2023-10-09T09:28:38.664628+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.163:0/3189378162' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:28:39.342 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:38 smithi181 bash[59345]: audit 2023-10-09T09:28:38.668386+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:28:39.343 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:39.366 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:39.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:39.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:39.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:39.503 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:39.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:39.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:39.607 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:39.646 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:39.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:39.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:39.751 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:39.775 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:39.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:39.884 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:39.917 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:39.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:40.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:40.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:40.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[75425]: audit 2023-10-09T09:28:37.899616+0000 mgr.y (mgr.34107) 669 : audit 0 from='client.54592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:28:40.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[75425]: cluster 2023-10-09T09:28:38.709197+0000 mon.a (mon.0) 1472 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:28:40.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[75425]: cluster 2023-10-09T09:28:38.724421+0000 osd.7 (osd.7) 486 : cluster 3 failed to encode map e1093 with expected crc 2023-10-09T09:28:40.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[75425]: cluster 2023-10-09T09:28:38.724970+0000 osd.5 (osd.5) 637 : cluster 3 failed to encode map e1093 with expected crc 2023-10-09T09:28:40.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[75425]: audit 2023-10-09T09:28:38.732482+0000 mon.a (mon.0) 1473 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:28:40.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[75425]: cluster 2023-10-09T09:28:38.732748+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e1093: 8 total, 8 up, 8 in 2023-10-09T09:28:40.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[75425]: cluster 2023-10-09T09:28:38.744519+0000 osd.4 (osd.4) 521 : cluster 3 failed to encode map e1093 with expected crc 2023-10-09T09:28:40.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[75425]: cluster 2023-10-09T09:28:39.656684+0000 mgr.y (mgr.34107) 670 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T09:28:40.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[78107]: audit 2023-10-09T09:28:37.899616+0000 mgr.y (mgr.34107) 669 : audit 0 from='client.54592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:28:40.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[78107]: cluster 2023-10-09T09:28:38.709197+0000 mon.a (mon.0) 1472 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:28:40.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[78107]: cluster 2023-10-09T09:28:38.724421+0000 osd.7 (osd.7) 486 : cluster 3 failed to encode map e1093 with expected crc 2023-10-09T09:28:40.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[78107]: cluster 2023-10-09T09:28:38.724970+0000 osd.5 (osd.5) 637 : cluster 3 failed to encode map e1093 with expected crc 2023-10-09T09:28:40.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[78107]: audit 2023-10-09T09:28:38.732482+0000 mon.a (mon.0) 1473 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:28:40.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[78107]: cluster 2023-10-09T09:28:38.732748+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e1093: 8 total, 8 up, 8 in 2023-10-09T09:28:40.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[78107]: cluster 2023-10-09T09:28:38.744519+0000 osd.4 (osd.4) 521 : cluster 3 failed to encode map e1093 with expected crc 2023-10-09T09:28:40.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:39 smithi163 bash[78107]: cluster 2023-10-09T09:28:39.656684+0000 mgr.y (mgr.34107) 670 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T09:28:40.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:40.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:40.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:39 smithi181 bash[59345]: audit 2023-10-09T09:28:37.899616+0000 mgr.y (mgr.34107) 669 : audit 0 from='client.54592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:28:40.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:39 smithi181 bash[59345]: cluster 2023-10-09T09:28:38.709197+0000 mon.a (mon.0) 1472 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:28:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:39 smithi181 bash[59345]: cluster 2023-10-09T09:28:38.724421+0000 osd.7 (osd.7) 486 : cluster 3 failed to encode map e1093 with expected crc 2023-10-09T09:28:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:39 smithi181 bash[59345]: cluster 2023-10-09T09:28:38.724970+0000 osd.5 (osd.5) 637 : cluster 3 failed to encode map e1093 with expected crc 2023-10-09T09:28:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:39 smithi181 bash[59345]: audit 2023-10-09T09:28:38.732482+0000 mon.a (mon.0) 1473 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:28:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:39 smithi181 bash[59345]: cluster 2023-10-09T09:28:38.732748+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e1093: 8 total, 8 up, 8 in 2023-10-09T09:28:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:39 smithi181 bash[59345]: cluster 2023-10-09T09:28:38.744519+0000 osd.4 (osd.4) 521 : cluster 3 failed to encode map e1093 with expected crc 2023-10-09T09:28:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:39 smithi181 bash[59345]: cluster 2023-10-09T09:28:39.656684+0000 mgr.y (mgr.34107) 670 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T09:28:40.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:40.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:40.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:40.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:40.322 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:40.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:40.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:40.429 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:40.455 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:40.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:40.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:40.539 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1367,"stamp":"2023-10-09T09:28:37.655750+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3327989688,"num_objects":50797,"num_object_clones":0,"num_object_copies":152391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50797,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":171003,"num_write_kb":7090894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":140037,"ondisk_log_size":140037,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11589284,"kb_used_data":9877156,"kb_used_omap":416,"kb_used_meta":1711647,"kb_avail":738371932,"statfs":{"total":767960285184,"available":756092858368,"internally_reserved":0,"allocated":10114207744,"data_stored":10108393057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":1752726961},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-602865664,"num_objects":-9199,"num_object_clones":0,"num_object_copies":-27597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9199,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.007378"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1059,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.743307+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:28:02.743307+0000","last_peered":"2023-10-09T09:28:02.743307+0000","last_clean":"2023-10-09T09:28:02.743307+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:28:02.743307+0000","last_undegraded":"2023-10-09T09:28:02.743307+0000","last_fullsized":"2023-10-09T09:28:02.743307+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1024,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:04.702264+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:28:04.702264+0000","last_peered":"2023-10-09T09:28:04.702264+0000","last_clean":"2023-10-09T09:28:04.702264+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:28:04.702264+0000","last_undegraded":"2023-10-09T09:28:04.702264+0000","last_fullsized":"2023-10-09T09:28:04.702264+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1027,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.753426+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:28:02.753426+0000","last_peered":"2023-10-09T09:28:02.753426+0000","last_clean":"2023-10-09T09:28:02.753426+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:28:02.753426+0000","last_undegraded":"2023-10-09T09:28:02.753426+0000","last_fullsized":"2023-10-09T09:28:02.753426+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1019,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:12.419330+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:28:12.419330+0000","last_peered":"2023-10-09T09:28:12.419330+0000","last_clean":"2023-10-09T09:28:12.419330+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:28:12.419330+0000","last_undegraded":"2023-10-09T09:28:12.419330+0000","last_fullsized":"2023-10-09T09:28:12.419330+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1005,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.740510+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:28:02.740510+0000","last_peered":"2023-10-09T09:28:02.740510+0000","last_clean":"2023-10-09T09:28:02.740510+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:28:02.740510+0000","last_undegraded":"2023-10-09T09:28:02.740510+0000","last_fullsized":"2023-10-09T09:28:02.740510+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1023,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.734458+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:28:02.734458+0000","last_peered":"2023-10-09T09:28:02.734458+0000","last_clean":"2023-10-09T09:28:02.734458+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:28:02.734458+0000","last_undegraded":"2023-10-09T09:28:02.734458+0000","last_fullsized":"2023-10-09T09:28:02.734458+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1018,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742523+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:28:02.742523+0000","last_peered":"2023-10-09T09:28:02.742523+0000","last_clean":"2023-10-09T09:28:02.742523+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:28:02.742523+0000","last_undegraded":"2023-10-09T09:28:02.742523+0000","last_fullsized":"2023-10-09T09:28:02.742523+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":989,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.740445+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:28:02.740445+0000","last_peered":"2023-10-09T09:28:02.740445+0000","last_clean":"2023-10-09T09:28:02.740445+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:28:02.740445+0000","last_undegraded":"2023-10-09T09:28:02.740445+0000","last_fullsized":"2023-10-09T09:28:02.740445+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1044,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.745105+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:28:02.745105+0000","last_peered":"2023-10-09T09:28:02.745105+0000","last_clean":"2023-10-09T09:28:02.745105+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:28:02.745105+0000","last_undegraded":"2023-10-09T09:28:02.745105+0000","last_fullsized":"2023-10-09T09:28:02.745105+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1000,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.739641+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:28:02.739641+0000","last_peered":"2023-10-09T09:28:02.739641+0000","last_clean":"2023-10-09T09:28:02.739641+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:28:02.739641+0000","last_undegraded":"2023-10-09T09:28:02.739641+0000","last_fullsized":"2023-10-09T09:28:02.739641+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1023,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.745248+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:28:02.745248+0000","last_peered":"2023-10-09T09:28:02.745248+0000","last_clean":"2023-10-09T09:28:02.745248+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:28:02.745248+0000","last_undegraded":"2023-10-09T09:28:02.745248+0000","last_fullsized":"2023-10-09T09:28:02.745248+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1027,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.752886+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:28:02.752886+0000","last_peered":"2023-10-09T09:28:02.752886+0000","last_clean":"2023-10-09T09:28:02.752886+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:28:02.752886+0000","last_undegraded":"2023-10-09T09:28:02.752886+0000","last_fullsized":"2023-10-09T09:28:02.752886+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1050,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.743389+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:28:02.743389+0000","last_peered":"2023-10-09T09:28:02.743389+0000","last_clean":"2023-10-09T09:28:02.743389+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:28:02.743389+0000","last_undegraded":"2023-10-09T09:28:02.743389+0000","last_fullsized":"2023-10-09T09:28:02.743389+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1042,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.743142+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:28:02.743142+0000","last_peered":"2023-10-09T09:28:02.743142+0000","last_clean":"2023-10-09T09:28:02.743142+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:28:02.743142+0000","last_undegraded":"2023-10-09T09:28:02.743142+0000","last_fullsized":"2023-10-09T09:28:02.743142+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1022,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.741945+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:28:02.741945+0000","last_peered":"2023-10-09T09:28:02.741945+0000","last_clean":"2023-10-09T09:28:02.741945+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:28:02.741945+0000","last_undegraded":"2023-10-09T09:28:02.741945+0000","last_fullsized":"2023-10-09T09:28:02.741945+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":989,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.739875+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:28:02.739875+0000","last_peered":"2023-10-09T09:28:02.739875+0000","last_clean":"2023-10-09T09:28:02.739875+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:28:02.739875+0000","last_undegraded":"2023-10-09T09:28:02.739875+0000","last_fullsized":"2023-10-09T09:28:02.739875+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1019,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:12.419290+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:28:12.419290+0000","last_peered":"2023-10-09T09:28:12.419290+0000","last_clean":"2023-10-09T09:28:12.419290+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:28:12.419290+0000","last_undegraded":"2023-10-09T09:28:12.419290+0000","last_fullsized":"2023-10-09T09:28:12.419290+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1043,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.746489+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:28:02.746489+0000","last_peered":"2023-10-09T09:28:02.746489+0000","last_clean":"2023-10-09T09:28:02.746489+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:28:02.746489+0000","last_undegraded":"2023-10-09T09:28:02.746489+0000","last_fullsized":"2023-10-09T09:28:02.746489+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1005,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742808+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:28:02.742808+0000","last_peered":"2023-10-09T09:28:02.742808+0000","last_clean":"2023-10-09T09:28:02.742808+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:28:02.742808+0000","last_undegraded":"2023-10-09T09:28:02.742808+0000","last_fullsized":"2023-10-09T09:28:02.742808+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1017,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742355+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:28:02.742355+0000","last_peered":"2023-10-09T09:28:02.742355+0000","last_clean":"2023-10-09T09:28:02.742355+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:28:02.742355+0000","last_undegraded":"2023-10-09T09:28:02.742355+0000","last_fullsized":"2023-10-09T09:28:02.742355+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1033,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.733242+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:28:02.733242+0000","last_peered":"2023-10-09T09:28:02.733242+0000","last_clean":"2023-10-09T09:28:02.733242+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:28:02.733242+0000","last_undegraded":"2023-10-09T09:28:02.733242+0000","last_fullsized":"2023-10-09T09:28:02.733242+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1024,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.745525+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:28:02.745525+0000","last_peered":"2023-10-09T09:28:02.745525+0000","last_clean":"2023-10-09T09:28:02.745525+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:28:02.745525+0000","last_undegraded":"2023-10-09T09:28:02.745525+0000","last_fullsized":"2023-10-09T09:28:02.745525+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"1092'10887","reported_seq":10911,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:36.205398+0000","last_change":"2023-10-09T09:25:25.899629+0000","last_active":"2023-10-09T09:28:36.205398+0000","last_peered":"2023-10-09T09:28:36.205398+0000","last_clean":"2023-10-09T09:28:36.205398+0000","last_became_active":"2023-10-09T09:25:25.899245+0000","last_became_peered":"2023-10-09T09:25:25.899245+0000","last_unstale":"2023-10-09T09:28:36.205398+0000","last_undegraded":"2023-10-09T09:28:36.205398+0000","last_fullsized":"2023-10-09T09:28:36.205398+0000","mapping_epoch":1076,"log_start":"1081'800","ondisk_log_start":"1081'800","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10087,"log_dups_size":0,"ondisk_log_size":10087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211877888,"num_objects":3233,"num_object_clones":0,"num_object_copies":9699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10887,"num_write_kb":451840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1021,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.741936+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:28:02.741936+0000","last_peered":"2023-10-09T09:28:02.741936+0000","last_clean":"2023-10-09T09:28:02.741936+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:28:02.741936+0000","last_undegraded":"2023-10-09T09:28:02.741936+0000","last_fullsized":"2023-10-09T09:28:02.741936+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1028,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.737245+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:28:02.737245+0000","last_peered":"2023-10-09T09:28:02.737245+0000","last_clean":"2023-10-09T09:28:02.737245+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:28:02.737245+0000","last_undegraded":"2023-10-09T09:28:02.737245+0000","last_fullsized":"2023-10-09T09:28:02.737245+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1092'10660","reported_seq":10686,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:34.939500+0000","last_change":"2023-10-09T09:25:25.902246+0000","last_active":"2023-10-09T09:28:34.939500+0000","last_peered":"2023-10-09T09:28:34.939500+0000","last_clean":"2023-10-09T09:28:34.939500+0000","last_became_active":"2023-10-09T09:25:25.901932+0000","last_became_peered":"2023-10-09T09:25:25.901932+0000","last_unstale":"2023-10-09T09:28:34.939500+0000","last_undegraded":"2023-10-09T09:28:34.939500+0000","last_fullsized":"2023-10-09T09:28:34.939500+0000","mapping_epoch":1076,"log_start":"1083'3300","ondisk_log_start":"1083'3300","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7360,"log_dups_size":0,"ondisk_log_size":7360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:10.795272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214695936,"num_objects":3276,"num_object_clones":0,"num_object_copies":9828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10660,"num_write_kb":445952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1022,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742249+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:28:02.742249+0000","last_peered":"2023-10-09T09:28:02.742249+0000","last_clean":"2023-10-09T09:28:02.742249+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:28:02.742249+0000","last_undegraded":"2023-10-09T09:28:02.742249+0000","last_fullsized":"2023-10-09T09:28:02.742249+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1057,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.743116+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:28:02.743116+0000","last_peered":"2023-10-09T09:28:02.743116+0000","last_clean":"2023-10-09T09:28:02.743116+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:28:02.743116+0000","last_undegraded":"2023-10-09T09:28:02.743116+0000","last_fullsized":"2023-10-09T09:28:02.743116+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1092'10841","reported_seq":10864,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:36.311674+0000","last_change":"2023-10-09T09:25:25.899849+0000","last_active":"2023-10-09T09:28:36.311674+0000","last_peered":"2023-10-09T09:28:36.311674+0000","last_clean":"2023-10-09T09:28:36.311674+0000","last_became_active":"2023-10-09T09:25:25.899350+0000","last_became_peered":"2023-10-09T09:25:25.899350+0000","last_unstale":"2023-10-09T09:28:36.311674+0000","last_undegraded":"2023-10-09T09:28:36.311674+0000","last_fullsized":"2023-10-09T09:28:36.311674+0000","mapping_epoch":1076,"log_start":"1082'2500","ondisk_log_start":"1082'2500","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8341,"log_dups_size":0,"ondisk_log_size":8341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206503936,"num_objects":3151,"num_object_clones":0,"num_object_copies":9453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10841,"num_write_kb":447744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1006,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.745436+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:28:02.745436+0000","last_peered":"2023-10-09T09:28:02.745436+0000","last_clean":"2023-10-09T09:28:02.745436+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:28:02.745436+0000","last_undegraded":"2023-10-09T09:28:02.745436+0000","last_fullsized":"2023-10-09T09:28:02.745436+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1024,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.735776+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:28:02.735776+0000","last_peered":"2023-10-09T09:28:02.735776+0000","last_clean":"2023-10-09T09:28:02.735776+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:28:02.735776+0000","last_undegraded":"2023-10-09T09:28:02.735776+0000","last_fullsized":"2023-10-09T09:28:02.735776+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1092'10298","reported_seq":10324,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:37.312573+0000","last_change":"2023-10-09T09:25:25.899129+0000","last_active":"2023-10-09T09:28:37.312573+0000","last_peered":"2023-10-09T09:28:37.312573+0000","last_clean":"2023-10-09T09:28:37.312573+0000","last_became_active":"2023-10-09T09:25:25.898720+0000","last_became_peered":"2023-10-09T09:25:25.898720+0000","last_unstale":"2023-10-09T09:28:37.312573+0000","last_undegraded":"2023-10-09T09:28:37.312573+0000","last_fullsized":"2023-10-09T09:28:37.312573+0000","mapping_epoch":1076,"log_start":"1080'200","ondisk_log_start":"1080'200","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10098,"log_dups_size":0,"ondisk_log_size":10098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:06.994778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198574080,"num_objects":3030,"num_object_clones":0,"num_object_copies":9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10298,"num_write_kb":426496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1044,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.746315+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:28:02.746315+0000","last_peered":"2023-10-09T09:28:02.746315+0000","last_clean":"2023-10-09T09:28:02.746315+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:28:02.746315+0000","last_undegraded":"2023-10-09T09:28:02.746315+0000","last_fullsized":"2023-10-09T09:28:02.746315+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1018,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742657+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:28:02.742657+0000","last_peered":"2023-10-09T09:28:02.742657+0000","last_clean":"2023-10-09T09:28:02.742657+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:28:02.742657+0000","last_undegraded":"2023-10-09T09:28:02.742657+0000","last_fullsized":"2023-10-09T09:28:02.742657+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1092'10694","reported_seq":10718,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:36.271829+0000","last_change":"2023-10-09T09:25:25.902960+0000","last_active":"2023-10-09T09:28:36.271829+0000","last_peered":"2023-10-09T09:28:36.271829+0000","last_clean":"2023-10-09T09:28:36.271829+0000","last_became_active":"2023-10-09T09:25:25.902534+0000","last_became_peered":"2023-10-09T09:25:25.902534+0000","last_unstale":"2023-10-09T09:28:36.271829+0000","last_undegraded":"2023-10-09T09:28:36.271829+0000","last_fullsized":"2023-10-09T09:28:36.271829+0000","mapping_epoch":1076,"log_start":"1082'2700","ondisk_log_start":"1082'2700","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7994,"log_dups_size":0,"ondisk_log_size":7994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208142336,"num_objects":3176,"num_object_clones":0,"num_object_copies":9528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10694,"num_write_kb":443840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":1007,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.740091+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:28:02.740091+0000","last_peered":"2023-10-09T09:28:02.740091+0000","last_clean":"2023-10-09T09:28:02.740091+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:28:02.740091+0000","last_undegraded":"2023-10-09T09:28:02.740091+0000","last_fullsized":"2023-10-09T09:28:02.740091+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1042,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.743466+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:28:02.743466+0000","last_peered":"2023-10-09T09:28:02.743466+0000","last_clean":"2023-10-09T09:28:02.743466+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:28:02.743466+0000","last_undegraded":"2023-10-09T09:28:02.743466+0000","last_fullsized":"2023-10-09T09:28:02.743466+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1092'10410","reported_seq":10435,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:35.557094+0000","last_change":"2023-10-09T09:25:25.901392+0000","last_active":"2023-10-09T09:28:35.557094+0000","last_peered":"2023-10-09T09:28:35.557094+0000","last_clean":"2023-10-09T09:28:35.557094+0000","last_became_active":"2023-10-09T09:25:25.901057+0000","last_became_peered":"2023-10-09T09:25:25.901057+0000","last_unstale":"2023-10-09T09:28:35.557094+0000","last_undegraded":"2023-10-09T09:28:35.557094+0000","last_fullsized":"2023-10-09T09:28:35.557094+0000","mapping_epoch":1076,"log_start":"1080'400","ondisk_log_start":"1080'400","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10010,"log_dups_size":0,"ondisk_log_size":10010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:56.342418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212336640,"num_objects":3240,"num_object_clones":0,"num_object_copies":9720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10410,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1025,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.735906+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:28:02.735906+0000","last_peered":"2023-10-09T09:28:02.735906+0000","last_clean":"2023-10-09T09:28:02.735906+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:28:02.735906+0000","last_undegraded":"2023-10-09T09:28:02.735906+0000","last_fullsized":"2023-10-09T09:28:02.735906+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1057,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.745877+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:28:02.745877+0000","last_peered":"2023-10-09T09:28:02.745877+0000","last_clean":"2023-10-09T09:28:02.745877+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:28:02.745877+0000","last_undegraded":"2023-10-09T09:28:02.745877+0000","last_fullsized":"2023-10-09T09:28:02.745877+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1092'10868","reported_seq":10894,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:34.907770+0000","last_change":"2023-10-09T09:25:25.896450+0000","last_active":"2023-10-09T09:28:34.907770+0000","last_peered":"2023-10-09T09:28:34.907770+0000","last_clean":"2023-10-09T09:28:34.907770+0000","last_became_active":"2023-10-09T09:25:25.896015+0000","last_became_peered":"2023-10-09T09:25:25.896015+0000","last_unstale":"2023-10-09T09:28:34.907770+0000","last_undegraded":"2023-10-09T09:28:34.907770+0000","last_fullsized":"2023-10-09T09:28:34.907770+0000","mapping_epoch":1076,"log_start":"1083'3500","ondisk_log_start":"1083'3500","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7368,"log_dups_size":0,"ondisk_log_size":7368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:32.630600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211550208,"num_objects":3228,"num_object_clones":0,"num_object_copies":9684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10868,"num_write_kb":451072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1043,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742718+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:28:02.742718+0000","last_peered":"2023-10-09T09:28:02.742718+0000","last_clean":"2023-10-09T09:28:02.742718+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:28:02.742718+0000","last_undegraded":"2023-10-09T09:28:02.742718+0000","last_fullsized":"2023-10-09T09:28:02.742718+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1042,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.753335+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:28:02.753335+0000","last_peered":"2023-10-09T09:28:02.753335+0000","last_clean":"2023-10-09T09:28:02.753335+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:28:02.753335+0000","last_undegraded":"2023-10-09T09:28:02.753335+0000","last_fullsized":"2023-10-09T09:28:02.753335+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1092'10739","reported_seq":10764,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:36.318314+0000","last_change":"2023-10-09T09:25:25.898838+0000","last_active":"2023-10-09T09:28:36.318314+0000","last_peered":"2023-10-09T09:28:36.318314+0000","last_clean":"2023-10-09T09:28:36.318314+0000","last_became_active":"2023-10-09T09:25:25.898426+0000","last_became_peered":"2023-10-09T09:25:25.898426+0000","last_unstale":"2023-10-09T09:28:36.318314+0000","last_undegraded":"2023-10-09T09:28:36.318314+0000","last_fullsized":"2023-10-09T09:28:36.318314+0000","mapping_epoch":1076,"log_start":"1081'700","ondisk_log_start":"1081'700","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10039,"log_dups_size":0,"ondisk_log_size":10039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204079128,"num_objects":3115,"num_object_clones":0,"num_object_copies":9345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3115,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10739,"num_write_kb":443265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":991,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:21.750506+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:28:21.750506+0000","last_peered":"2023-10-09T09:28:21.750506+0000","last_clean":"2023-10-09T09:28:21.750506+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:28:21.750506+0000","last_undegraded":"2023-10-09T09:28:21.750506+0000","last_fullsized":"2023-10-09T09:28:21.750506+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1028,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:08.735964+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:28:08.735964+0000","last_peered":"2023-10-09T09:28:08.735964+0000","last_clean":"2023-10-09T09:28:08.735964+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:28:08.735964+0000","last_undegraded":"2023-10-09T09:28:08.735964+0000","last_fullsized":"2023-10-09T09:28:08.735964+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1092'10913","reported_seq":10939,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:37.451099+0000","last_change":"2023-10-09T09:25:25.899052+0000","last_active":"2023-10-09T09:28:37.451099+0000","last_peered":"2023-10-09T09:28:37.451099+0000","last_clean":"2023-10-09T09:28:37.451099+0000","last_became_active":"2023-10-09T09:25:25.898696+0000","last_became_peered":"2023-10-09T09:25:25.898696+0000","last_unstale":"2023-10-09T09:28:37.451099+0000","last_undegraded":"2023-10-09T09:28:37.451099+0000","last_fullsized":"2023-10-09T09:28:37.451099+0000","mapping_epoch":1076,"log_start":"1081'900","ondisk_log_start":"1081'900","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10013,"log_dups_size":0,"ondisk_log_size":10013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:34.741367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202440704,"num_objects":3089,"num_object_clones":0,"num_object_copies":9267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10913,"num_write_kb":448064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1023,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.753627+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:28:02.753627+0000","last_peered":"2023-10-09T09:28:02.753627+0000","last_clean":"2023-10-09T09:28:02.753627+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:28:02.753627+0000","last_undegraded":"2023-10-09T09:28:02.753627+0000","last_fullsized":"2023-10-09T09:28:02.753627+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1059,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.746686+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:28:02.746686+0000","last_peered":"2023-10-09T09:28:02.746686+0000","last_clean":"2023-10-09T09:28:02.746686+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:28:02.746686+0000","last_undegraded":"2023-10-09T09:28:02.746686+0000","last_fullsized":"2023-10-09T09:28:02.746686+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1013,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.752806+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:28:02.752806+0000","last_peered":"2023-10-09T09:28:02.752806+0000","last_clean":"2023-10-09T09:28:02.752806+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:28:02.752806+0000","last_undegraded":"2023-10-09T09:28:02.752806+0000","last_fullsized":"2023-10-09T09:28:02.752806+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1092'10827","reported_seq":10852,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:34.878568+0000","last_change":"2023-10-09T09:25:25.901310+0000","last_active":"2023-10-09T09:28:34.878568+0000","last_peered":"2023-10-09T09:28:34.878568+0000","last_clean":"2023-10-09T09:28:34.878568+0000","last_became_active":"2023-10-09T09:25:25.900980+0000","last_became_peered":"2023-10-09T09:25:25.900980+0000","last_unstale":"2023-10-09T09:28:34.878568+0000","last_undegraded":"2023-10-09T09:28:34.878568+0000","last_fullsized":"2023-10-09T09:28:34.878568+0000","mapping_epoch":1076,"log_start":"1083'3500","ondisk_log_start":"1083'3500","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7327,"log_dups_size":0,"ondisk_log_size":7327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:10.107258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214761472,"num_objects":3277,"num_object_clones":0,"num_object_copies":9831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10827,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1048,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.737527+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:28:02.737527+0000","last_peered":"2023-10-09T09:28:02.737527+0000","last_clean":"2023-10-09T09:28:02.737527+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:28:02.737527+0000","last_undegraded":"2023-10-09T09:28:02.737527+0000","last_fullsized":"2023-10-09T09:28:02.737527+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1043,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.743035+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:28:02.743035+0000","last_peered":"2023-10-09T09:28:02.743035+0000","last_clean":"2023-10-09T09:28:02.743035+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:28:02.743035+0000","last_undegraded":"2023-10-09T09:28:02.743035+0000","last_fullsized":"2023-10-09T09:28:02.743035+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1043,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.753199+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:28:02.753199+0000","last_peered":"2023-10-09T09:28:02.753199+0000","last_clean":"2023-10-09T09:28:02.753199+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:28:02.753199+0000","last_undegraded":"2023-10-09T09:28:02.753199+0000","last_fullsized":"2023-10-09T09:28:02.753199+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"1092'10747","reported_seq":10769,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:36.316373+0000","last_change":"2023-10-09T09:25:25.903443+0000","last_active":"2023-10-09T09:28:36.316373+0000","last_peered":"2023-10-09T09:28:36.316373+0000","last_clean":"2023-10-09T09:28:36.316373+0000","last_became_active":"2023-10-09T09:25:25.903086+0000","last_became_peered":"2023-10-09T09:25:25.903086+0000","last_unstale":"2023-10-09T09:28:36.316373+0000","last_undegraded":"2023-10-09T09:28:36.316373+0000","last_fullsized":"2023-10-09T09:28:36.316373+0000","mapping_epoch":1076,"log_start":"1082'2400","ondisk_log_start":"1082'2400","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8347,"log_dups_size":0,"ondisk_log_size":8347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209125376,"num_objects":3191,"num_object_clones":0,"num_object_copies":9573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10747,"num_write_kb":446016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1010,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.744443+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:28:02.744443+0000","last_peered":"2023-10-09T09:28:02.744443+0000","last_clean":"2023-10-09T09:28:02.744443+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:28:02.744443+0000","last_undegraded":"2023-10-09T09:28:02.744443+0000","last_fullsized":"2023-10-09T09:28:02.744443+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1028,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.752807+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:28:02.752807+0000","last_peered":"2023-10-09T09:28:02.752807+0000","last_clean":"2023-10-09T09:28:02.752807+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:28:02.752807+0000","last_undegraded":"2023-10-09T09:28:02.752807+0000","last_fullsized":"2023-10-09T09:28:02.752807+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1018,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.753308+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:28:02.753308+0000","last_peered":"2023-10-09T09:28:02.753308+0000","last_clean":"2023-10-09T09:28:02.753308+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:28:02.753308+0000","last_undegraded":"2023-10-09T09:28:02.753308+0000","last_fullsized":"2023-10-09T09:28:02.753308+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1092'10499","reported_seq":10522,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:36.325653+0000","last_change":"2023-10-09T09:25:25.900270+0000","last_active":"2023-10-09T09:28:36.325653+0000","last_peered":"2023-10-09T09:28:36.325653+0000","last_clean":"2023-10-09T09:28:36.325653+0000","last_became_active":"2023-10-09T09:25:25.899813+0000","last_became_peered":"2023-10-09T09:25:25.899813+0000","last_unstale":"2023-10-09T09:28:36.325653+0000","last_undegraded":"2023-10-09T09:28:36.325653+0000","last_fullsized":"2023-10-09T09:28:36.325653+0000","mapping_epoch":1076,"log_start":"1080'400","ondisk_log_start":"1080'400","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10099,"log_dups_size":0,"ondisk_log_size":10099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202178560,"num_objects":3085,"num_object_clones":0,"num_object_copies":9255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10499,"num_write_kb":434688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1048,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.737418+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:28:02.737418+0000","last_peered":"2023-10-09T09:28:02.737418+0000","last_clean":"2023-10-09T09:28:02.737418+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:28:02.737418+0000","last_undegraded":"2023-10-09T09:28:02.737418+0000","last_fullsized":"2023-10-09T09:28:02.737418+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1256,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.747581+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:28:02.747581+0000","last_peered":"2023-10-09T09:28:02.747581+0000","last_clean":"2023-10-09T09:28:02.747581+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:28:02.747581+0000","last_undegraded":"2023-10-09T09:28:02.747581+0000","last_fullsized":"2023-10-09T09:28:02.747581+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1002,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:04.702212+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:28:04.702212+0000","last_peered":"2023-10-09T09:28:04.702212+0000","last_clean":"2023-10-09T09:28:04.702212+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:28:04.702212+0000","last_undegraded":"2023-10-09T09:28:04.702212+0000","last_fullsized":"2023-10-09T09:28:04.702212+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1009,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.734193+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:28:02.734193+0000","last_peered":"2023-10-09T09:28:02.734193+0000","last_clean":"2023-10-09T09:28:02.734193+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:28:02.734193+0000","last_undegraded":"2023-10-09T09:28:02.734193+0000","last_fullsized":"2023-10-09T09:28:02.734193+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1092'10583","reported_seq":10609,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:33.741863+0000","last_change":"2023-10-09T09:25:25.897749+0000","last_active":"2023-10-09T09:28:33.741863+0000","last_peered":"2023-10-09T09:28:33.741863+0000","last_clean":"2023-10-09T09:28:33.741863+0000","last_became_active":"2023-10-09T09:25:25.897366+0000","last_became_peered":"2023-10-09T09:25:25.897366+0000","last_unstale":"2023-10-09T09:28:33.741863+0000","last_undegraded":"2023-10-09T09:28:33.741863+0000","last_fullsized":"2023-10-09T09:28:33.741863+0000","mapping_epoch":1076,"log_start":"1082'2400","ondisk_log_start":"1082'2400","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8183,"log_dups_size":0,"ondisk_log_size":8183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:21.152296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205455360,"num_objects":3135,"num_object_clones":0,"num_object_copies":9405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10583,"num_write_kb":438976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1039,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.747655+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:28:02.747655+0000","last_peered":"2023-10-09T09:28:02.747655+0000","last_clean":"2023-10-09T09:28:02.747655+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:28:02.747655+0000","last_undegraded":"2023-10-09T09:28:02.747655+0000","last_fullsized":"2023-10-09T09:28:02.747655+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1019,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742111+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:28:02.742111+0000","last_peered":"2023-10-09T09:28:02.742111+0000","last_clean":"2023-10-09T09:28:02.742111+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:28:02.742111+0000","last_undegraded":"2023-10-09T09:28:02.742111+0000","last_fullsized":"2023-10-09T09:28:02.742111+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1030,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742919+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:28:02.742919+0000","last_peered":"2023-10-09T09:28:02.742919+0000","last_clean":"2023-10-09T09:28:02.742919+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:28:02.742919+0000","last_undegraded":"2023-10-09T09:28:02.742919+0000","last_fullsized":"2023-10-09T09:28:02.742919+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1092'10476","reported_seq":10502,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:33.778895+0000","last_change":"2023-10-09T09:25:25.895292+0000","last_active":"2023-10-09T09:28:33.778895+0000","last_peered":"2023-10-09T09:28:33.778895+0000","last_clean":"2023-10-09T09:28:33.778895+0000","last_became_active":"2023-10-09T09:25:25.894970+0000","last_became_peered":"2023-10-09T09:25:25.894970+0000","last_unstale":"2023-10-09T09:28:33.778895+0000","last_undegraded":"2023-10-09T09:28:33.778895+0000","last_fullsized":"2023-10-09T09:28:33.778895+0000","mapping_epoch":1076,"log_start":"1082'2300","ondisk_log_start":"1082'2300","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8176,"log_dups_size":0,"ondisk_log_size":8176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:21.241118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213909504,"num_objects":3264,"num_object_clones":0,"num_object_copies":9792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10476,"num_write_kb":439680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1024,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.745340+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:28:02.745340+0000","last_peered":"2023-10-09T09:28:02.745340+0000","last_clean":"2023-10-09T09:28:02.745340+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:28:02.745340+0000","last_undegraded":"2023-10-09T09:28:02.745340+0000","last_fullsized":"2023-10-09T09:28:02.745340+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1029,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.753467+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:28:02.753467+0000","last_peered":"2023-10-09T09:28:02.753467+0000","last_clean":"2023-10-09T09:28:02.753467+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:28:02.753467+0000","last_undegraded":"2023-10-09T09:28:02.753467+0000","last_fullsized":"2023-10-09T09:28:02.753467+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1003,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742852+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:28:02.742852+0000","last_peered":"2023-10-09T09:28:02.742852+0000","last_clean":"2023-10-09T09:28:02.742852+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:28:02.742852+0000","last_undegraded":"2023-10-09T09:28:02.742852+0000","last_fullsized":"2023-10-09T09:28:02.742852+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1092'10770","reported_seq":10794,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:36.331294+0000","last_change":"2023-10-09T09:25:25.903038+0000","last_active":"2023-10-09T09:28:36.331294+0000","last_peered":"2023-10-09T09:28:36.331294+0000","last_clean":"2023-10-09T09:28:36.331294+0000","last_became_active":"2023-10-09T09:25:25.902503+0000","last_became_peered":"2023-10-09T09:25:25.902503+0000","last_unstale":"2023-10-09T09:28:36.331294+0000","last_undegraded":"2023-10-09T09:28:36.331294+0000","last_fullsized":"2023-10-09T09:28:36.331294+0000","mapping_epoch":1076,"log_start":"1082'2800","ondisk_log_start":"1082'2800","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7970,"log_dups_size":0,"ondisk_log_size":7970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210239488,"num_objects":3208,"num_object_clones":0,"num_object_copies":9624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10770,"num_write_kb":447296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":1008,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:21.740696+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:28:21.740696+0000","last_peered":"2023-10-09T09:28:21.740696+0000","last_clean":"2023-10-09T09:28:21.740696+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:28:21.740696+0000","last_undegraded":"2023-10-09T09:28:21.740696+0000","last_fullsized":"2023-10-09T09:28:21.740696+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1012,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.753069+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:28:02.753069+0000","last_peered":"2023-10-09T09:28:02.753069+0000","last_clean":"2023-10-09T09:28:02.753069+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:28:02.753069+0000","last_undegraded":"2023-10-09T09:28:02.753069+0000","last_fullsized":"2023-10-09T09:28:02.753069+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1005,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.740257+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:28:02.740257+0000","last_peered":"2023-10-09T09:28:02.740257+0000","last_clean":"2023-10-09T09:28:02.740257+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:28:02.740257+0000","last_undegraded":"2023-10-09T09:28:02.740257+0000","last_fullsized":"2023-10-09T09:28:02.740257+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1092'10531","reported_seq":10555,"reported_epoch":1092,"state":"active+clean","last_fresh":"2023-10-09T09:28:36.352210+0000","last_change":"2023-10-09T09:25:25.902231+0000","last_active":"2023-10-09T09:28:36.352210+0000","last_peered":"2023-10-09T09:28:36.352210+0000","last_clean":"2023-10-09T09:28:36.352210+0000","last_became_active":"2023-10-09T09:25:25.901859+0000","last_became_peered":"2023-10-09T09:25:25.901859+0000","last_unstale":"2023-10-09T09:28:36.352210+0000","last_undegraded":"2023-10-09T09:28:36.352210+0000","last_fullsized":"2023-10-09T09:28:36.352210+0000","mapping_epoch":1076,"log_start":"1081'2100","ondisk_log_start":"1081'2100","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8431,"log_dups_size":0,"ondisk_log_size":8431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201523200,"num_objects":3075,"num_object_clones":0,"num_object_copies":9225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10531,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1028,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.734362+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:28:02.734362+0000","last_peered":"2023-10-09T09:28:02.734362+0000","last_clean":"2023-10-09T09:28:02.734362+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:28:02.734362+0000","last_undegraded":"2023-10-09T09:28:02.734362+0000","last_fullsized":"2023-10-09T09:28:02.734362+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1022,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742509+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:28:02.742509+0000","last_peered":"2023-10-09T09:28:02.742509+0000","last_clean":"2023-10-09T09:28:02.742509+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:28:02.742509+0000","last_undegraded":"2023-10-09T09:28:02.742509+0000","last_fullsized":"2023-10-09T09:28:02.742509+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1027,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742956+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:28:02.742956+0000","last_peered":"2023-10-09T09:28:02.742956+0000","last_clean":"2023-10-09T09:28:02.742956+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:28:02.742956+0000","last_undegraded":"2023-10-09T09:28:02.742956+0000","last_fullsized":"2023-10-09T09:28:02.742956+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1006,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742786+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:28:02.742786+0000","last_peered":"2023-10-09T09:28:02.742786+0000","last_clean":"2023-10-09T09:28:02.742786+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:28:02.742786+0000","last_undegraded":"2023-10-09T09:28:02.742786+0000","last_fullsized":"2023-10-09T09:28:02.742786+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1028,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.744348+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:28:02.744348+0000","last_peered":"2023-10-09T09:28:02.744348+0000","last_clean":"2023-10-09T09:28:02.744348+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:28:02.744348+0000","last_undegraded":"2023-10-09T09:28:02.744348+0000","last_fullsized":"2023-10-09T09:28:02.744348+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1046,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.750048+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:28:02.750048+0000","last_peered":"2023-10-09T09:28:02.750048+0000","last_clean":"2023-10-09T09:28:02.750048+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:28:02.750048+0000","last_undegraded":"2023-10-09T09:28:02.750048+0000","last_fullsized":"2023-10-09T09:28:02.750048+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1057,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.743609+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:28:02.743609+0000","last_peered":"2023-10-09T09:28:02.743609+0000","last_clean":"2023-10-09T09:28:02.743609+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:28:02.743609+0000","last_undegraded":"2023-10-09T09:28:02.743609+0000","last_fullsized":"2023-10-09T09:28:02.743609+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1031,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.746770+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:28:02.746770+0000","last_peered":"2023-10-09T09:28:02.746770+0000","last_clean":"2023-10-09T09:28:02.746770+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:28:02.746770+0000","last_undegraded":"2023-10-09T09:28:02.746770+0000","last_fullsized":"2023-10-09T09:28:02.746770+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1057,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.743383+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:28:02.743383+0000","last_peered":"2023-10-09T09:28:02.743383+0000","last_clean":"2023-10-09T09:28:02.743383+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:28:02.743383+0000","last_undegraded":"2023-10-09T09:28:02.743383+0000","last_fullsized":"2023-10-09T09:28:02.743383+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":1018,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.740710+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:28:02.740710+0000","last_peered":"2023-10-09T09:28:02.740710+0000","last_clean":"2023-10-09T09:28:02.740710+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:28:02.740710+0000","last_undegraded":"2023-10-09T09:28:02.740710+0000","last_fullsized":"2023-10-09T09:28:02.740710+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1030,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.739548+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:28:02.739548+0000","last_peered":"2023-10-09T09:28:02.739548+0000","last_clean":"2023-10-09T09:28:02.739548+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:28:02.739548+0000","last_undegraded":"2023-10-09T09:28:02.739548+0000","last_fullsized":"2023-10-09T09:28:02.739548+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1025,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742440+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:28:02.742440+0000","last_peered":"2023-10-09T09:28:02.742440+0000","last_clean":"2023-10-09T09:28:02.742440+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:28:02.742440+0000","last_undegraded":"2023-10-09T09:28:02.742440+0000","last_fullsized":"2023-10-09T09:28:02.742440+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1042,"reported_epoch":1091,"state":"active+clean","last_fresh":"2023-10-09T09:28:02.742769+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:28:02.742769+0000","last_peered":"2023-10-09T09:28:02.742769+0000","last_clean":"2023-10-09T09:28:02.742769+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:28:02.742769+0000","last_undegraded":"2023-10-09T09:28:02.742769+0000","last_fullsized":"2023-10-09T09:28:02.742769+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3327393816,"num_objects":50773,"num_object_clones":0,"num_object_copies":152319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50773,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170743,"num_write_kb":7088449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9993465856,"data_stored":9993453640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139843,"ondisk_log_size":139843,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397842,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1527308,"kb_used_data":1230084,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92217844,"statfs":{"total":95995035648,"available":94431072256,"internally_reserved":0,"allocated":1259606016,"data_stored":1259301135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":2.677,"5min":2.3039999999999998,"15min":1.4630000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.5459999999999994,"5min":55.296999999999997,"15min":55.296999999999997},"last":2.1840000000000002},{"interface":"front","average":{"1min":2.7669999999999999,"5min":2.4129999999999998,"15min":1.5680000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.5790000000000006,"5min":55.369999999999997,"15min":55.369999999999997},"last":2.218}]},{"osd":1,"last update":"Mon Oct 9 09:28:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9610000000000001,"5min":2.411,"15min":1.4099999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.0419999999999998,"5min":103.84699999999999,"15min":103.84699999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.1309999999999998,"5min":2.5409999999999999,"15min":1.5189999999999999},"min":{"1min":0.33300000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":9.8190000000000008,"5min":103.955,"15min":103.955},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:28:30 2023","interfaces":[{"interface":"back","average":{"1min":3.3610000000000002,"5min":1.9710000000000001,"15min":1.413},"min":{"1min":0.34200000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.0220000000000002,"5min":17.638000000000002,"15min":17.638000000000002},"last":9.1950000000000003},{"interface":"front","average":{"1min":2.125,"5min":1.6899999999999999,"15min":1.2509999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":8.2769999999999992,"5min":17.600000000000001,"15min":17.600000000000001},"last":9.2070000000000007}]},{"osd":3,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.823,"5min":3.5499999999999998,"15min":1.8520000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":12.978,"5min":107.97499999999999,"15min":107.97499999999999},"last":4.1509999999999998},{"interface":"front","average":{"1min":2.7090000000000001,"5min":3.6280000000000001,"15min":1.976},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":12.817,"5min":105.282,"15min":105.282},"last":0.54800000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.1160000000000001,"15min":1.0069999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.9500000000000002,"5min":6.1550000000000002,"15min":6.1550000000000002},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.1639999999999999,"15min":1.054},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":12.316000000000001,"5min":12.316000000000001,"15min":12.316000000000001},"last":0.45200000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":1.4730000000000001,"15min":1.2529999999999999},"min":{"1min":0.38900000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":6.4500000000000002,"5min":10.063000000000001,"15min":10.063000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.069,"5min":1.1799999999999999,"15min":1.1040000000000001},"min":{"1min":0.46100000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.0499999999999998,"5min":6.4359999999999999,"15min":6.4359999999999999},"last":0.69599999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.21,"15min":1.1279999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.5710000000000002,"5min":6.1219999999999999,"15min":6.1219999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.925,"5min":1.4530000000000001,"15min":1.2629999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":8.1980000000000004,"5min":8.1980000000000004,"15min":8.1980000000000004},"last":0.47999999999999998}]}]},{"osd":6,"up_from":43,"seq":184683594075,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1339648,"kb_used_data":1026040,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92405504,"statfs":{"total":95995035648,"available":94623236096,"internally_reserved":0,"allocated":1050664960,"data_stored":1050372221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":1.0509999999999999,"15min":0.96899999999999997},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":4.5110000000000001,"5min":4.5110000000000001,"15min":4.5110000000000001},"last":4.9249999999999998},{"interface":"front","average":{"1min":1.2250000000000001,"5min":0.96199999999999997,"15min":0.98499999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.649,"5min":5.649,"15min":5.649},"last":0.36799999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:28:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":1.415,"15min":1.1120000000000001},"min":{"1min":0.33200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.253,"5min":10.253,"15min":10.253},"last":5.6849999999999996},{"interface":"front","average":{"1min":1.679,"5min":1.1899999999999999,"15min":1.022},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.641,"5min":8.641,"15min":8.641},"last":0.53100000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.083,"15min":0.98299999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.1829999999999998,"5min":4.1829999999999998,"15min":4.1829999999999998},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.3100000000000001,"15min":1.1140000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.7629999999999999,"5min":6.0039999999999996,"15min":6.0039999999999996},"last":0.40400000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:27:53 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":1.28,"15min":1.1000000000000001},"min":{"1min":0.38600000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.04,"5min":7.04,"15min":7.04},"last":4.8869999999999996},{"interface":"front","average":{"1min":1.27,"5min":1.3680000000000001,"15min":1.1859999999999999},"min":{"1min":0.46200000000000002,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":3.4079999999999999,"5min":11.843999999999999,"15min":11.843999999999999},"last":0.49099999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:27:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.173,"15min":1.0349999999999999},"min":{"1min":0.374,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":9.1099999999999994,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.222,"5min":1.159,"15min":1.091},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.3759999999999999,"5min":3.5569999999999999,"15min":3.5569999999999999},"last":3.2480000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:27:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.103,"15min":1.0960000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":4.1280000000000001,"5min":4.1280000000000001,"15min":4.1280000000000001},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.617,"5min":1.161,"15min":1.1459999999999999},"min":{"1min":0.47699999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":8.5950000000000006,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":1.244}]},{"osd":7,"last update":"Mon Oct 9 09:28:22 2023","interfaces":[{"interface":"back","average":{"1min":2.1499999999999999,"5min":1.4710000000000001,"15min":1.2729999999999999},"min":{"1min":0.46300000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":7.1100000000000003,"5min":7.3630000000000004,"15min":7.3630000000000004},"last":1.2130000000000001},{"interface":"front","average":{"1min":2.302,"5min":1.431,"15min":1.272},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":7.6459999999999999,"5min":7.6459999999999999,"15min":7.6459999999999999},"last":4.9550000000000001}]}]},{"osd":2,"up_from":634,"seq":2723009265828,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1790304,"kb_used_data":1640920,"kb_used_omap":122,"kb_used_meta":149253,"kb_avail":91954848,"statfs":{"total":95995035648,"available":94161764352,"internally_reserved":0,"allocated":1680302080,"data_stored":1679110296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":152835659},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":1.1599999999999999,"15min":0.88800000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.9860000000000007,"5min":8.9860000000000007,"15min":8.9860000000000007},"last":1.2},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.266,"15min":0.89500000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.5709999999999997,"5min":7.71,"15min":7.71},"last":5.1420000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.0840000000000001,"15min":0.84299999999999997},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":6.6600000000000001,"5min":6.798,"15min":6.798},"last":1.6910000000000001},{"interface":"front","average":{"1min":2.27,"5min":1.369,"15min":0.98199999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":8.702,"5min":8.702,"15min":8.702},"last":5.4089999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:27:53 2023","interfaces":[{"interface":"back","average":{"1min":2.411,"5min":1.528,"15min":1.0509999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":11.597,"5min":11.597,"15min":11.597},"last":1.4510000000000001},{"interface":"front","average":{"1min":2.262,"5min":1.393,"15min":1.056},"min":{"1min":0.34599999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.0259999999999998,"5min":9.0259999999999998,"15min":9.0259999999999998},"last":2.6509999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":1.169,"15min":1.091},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":4.9630000000000001,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":1.22},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.294,"15min":1.119},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":5.1959999999999997,"5min":7.5060000000000002,"15min":7.5060000000000002},"last":13.917999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.175,"15min":1.0940000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.7619999999999996,"5min":6.7699999999999996,"15min":6.7699999999999996},"last":1.748},{"interface":"front","average":{"1min":1.552,"5min":1.2290000000000001,"15min":1.171},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.9320000000000004,"5min":4.9320000000000004,"15min":4.9320000000000004},"last":1.2490000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.224,"15min":1.1870000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.3259999999999996,"5min":5.3259999999999996,"15min":5.3259999999999996},"last":1.2350000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.353,"15min":1.1990000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.5230000000000001,"5min":7.5860000000000003,"15min":7.5860000000000003},"last":13.893000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.534,"15min":1.292},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":5.1660000000000004,"5min":7.7469999999999999,"15min":7.7469999999999999},"last":13.848000000000001},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.349,"15min":1.2250000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":6.7350000000000003,"5min":6.7350000000000003,"15min":6.7350000000000003},"last":1.726}]}]},{"osd":1,"up_from":607,"seq":2607045148841,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1172688,"kb_used_data":1055304,"kb_used_omap":107,"kb_used_meta":117268,"kb_avail":92572464,"statfs":{"total":95995035648,"available":94794203136,"internally_reserved":0,"allocated":1080631296,"data_stored":1079452665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":120083115},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":0.80100000000000005,"15min":0.71699999999999997},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":5.4569999999999999,"5min":5.4569999999999999,"15min":5.4569999999999999},"last":0.93100000000000005},{"interface":"front","average":{"1min":2.238,"5min":1.1120000000000001,"15min":0.85399999999999998},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":11.845000000000001,"5min":11.845000000000001,"15min":11.845000000000001},"last":1.1950000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:27:34 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.3220000000000001,"15min":0.93899999999999995},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.6099999999999994,"5min":9.7959999999999994,"15min":9.7959999999999994},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.929,"5min":1.1839999999999999,"15min":0.95199999999999996},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7899999999999991,"5min":9.9260000000000002,"15min":9.9260000000000002},"last":0.40500000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:27:53 2023","interfaces":[{"interface":"back","average":{"1min":8.2530000000000001,"5min":2.9500000000000002,"15min":1.5449999999999999},"min":{"1min":0.26400000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":107.426,"5min":107.426,"15min":107.426},"last":1.161},{"interface":"front","average":{"1min":8.8000000000000007,"5min":3.2050000000000001,"15min":1.677},"min":{"1min":0.32000000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":106.15900000000001,"5min":106.15900000000001,"15min":106.15900000000001},"last":0.48199999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.0189999999999999,"15min":1.0329999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.3049999999999997,"5min":7.3049999999999997,"15min":7.3049999999999997},"last":3.5529999999999999},{"interface":"front","average":{"1min":2.0489999999999999,"5min":1.266,"15min":1.121},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":11.933999999999999,"5min":11.933999999999999,"15min":11.933999999999999},"last":1.2190000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":1.1279999999999999,"15min":1.111},"min":{"1min":0.38800000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.2469999999999999,"5min":7.2469999999999999,"15min":7.2469999999999999},"last":0.439},{"interface":"front","average":{"1min":2.0059999999999998,"5min":1.2629999999999999,"15min":1.147},"min":{"1min":0.33100000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":11.907,"5min":11.907,"15min":11.907},"last":1.2410000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.385,"15min":1.1830000000000001},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.379},"max":{"1min":7.476,"5min":8.1340000000000003,"15min":8.1340000000000003},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.2390000000000001,"5min":1.179,"15min":1.1499999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":7.3970000000000002,"5min":7.3970000000000002,"15min":7.3970000000000002},"last":0.52100000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":1.5580000000000001,"15min":1.3029999999999999},"min":{"1min":0.40899999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.5119999999999996,"5min":8.1029999999999998,"15min":8.1029999999999998},"last":4.0839999999999996},{"interface":"front","average":{"1min":1.284,"5min":1.157,"15min":1.2090000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":7.3520000000000003,"5min":7.3520000000000003,"15min":7.3520000000000003},"last":2.125}]}]},{"osd":0,"up_from":581,"seq":2495375999151,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":897768,"kb_used_data":802080,"kb_used_omap":131,"kb_used_meta":95548,"kb_avail":92847384,"statfs":{"total":95995035648,"available":95075721216,"internally_reserved":0,"allocated":821329920,"data_stored":820129947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":97841753},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:28:07 2023","interfaces":[{"interface":"back","average":{"1min":1.915,"5min":1.24,"15min":0.90200000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.1230000000000002,"5min":14.44,"15min":14.44},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.385,"5min":1.349,"15min":0.98199999999999998},"min":{"1min":0.23499999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.4320000000000004,"5min":14.663,"15min":14.663},"last":0.46700000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":1.264,"15min":0.94999999999999996},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":6.976,"5min":21.093,"15min":21.093},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.841,"5min":1.143,"15min":0.89800000000000002},"min":{"1min":0.21099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.9939999999999998,"5min":19.614000000000001,"15min":19.614000000000001},"last":1.4299999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:28:00 2023","interfaces":[{"interface":"back","average":{"1min":9.0609999999999999,"5min":2.5510000000000002,"15min":1.3979999999999999},"min":{"1min":0.34399999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":122.749,"5min":122.749,"15min":122.749},"last":0.747},{"interface":"front","average":{"1min":8.6690000000000005,"5min":2.4969999999999999,"15min":1.4770000000000001},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":123.51900000000001,"5min":123.51900000000001,"15min":123.51900000000001},"last":0.64400000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:27:50 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.1870000000000001,"15min":1.0669999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":7.093,"5min":7.093,"15min":7.093},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.508,"5min":1.2230000000000001,"15min":1.095},"min":{"1min":0.35599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3310000000000004,"5min":7.3310000000000004,"15min":7.3310000000000004},"last":3.4809999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:27:50 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.173,"15min":1.1100000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.3620000000000001,"5min":7.3620000000000001,"15min":7.3620000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.9199999999999999,"5min":1.3200000000000001,"15min":1.1639999999999999},"min":{"1min":0.371,"5min":0.312,"15min":0.312},"max":{"1min":8.5609999999999999,"5min":8.5609999999999999,"15min":8.5609999999999999},"last":3.448}]},{"osd":6,"last update":"Mon Oct 9 09:27:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":1.3879999999999999,"15min":1.2130000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.202,"5min":8.202,"15min":8.202},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.8879999999999999,"5min":1.272,"15min":1.1830000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.3289999999999997,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":0.61099999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:27:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":1.4450000000000001,"15min":1.274},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":8.1769999999999996,"5min":8.1769999999999996,"15min":8.1769999999999996},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.879,"5min":1.367,"15min":1.26},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":8.2710000000000008,"5min":8.6549999999999994,"15min":8.6549999999999994},"last":3.2400000000000002}]}]},{"osd":3,"up_from":663,"seq":2847563317406,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2215428,"kb_used_data":2079228,"kb_used_omap":55,"kb_used_meta":136136,"kb_avail":91529724,"statfs":{"total":95995035648,"available":93726437376,"internally_reserved":0,"allocated":2129129472,"data_stored":2128069343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":139403874},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.2230000000000001,"15min":0.93999999999999995},"min":{"1min":0.23000000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":6.1390000000000002,"5min":7.9080000000000004,"15min":7.9080000000000004},"last":4.2729999999999997},{"interface":"front","average":{"1min":2.4620000000000002,"5min":1.3220000000000001,"15min":0.95999999999999996},"min":{"1min":0.28599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":10.938000000000001,"5min":11.507999999999999,"15min":11.507999999999999},"last":4.3579999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.401,"15min":1.04},"min":{"1min":0.36499999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":11.090999999999999,"5min":11.581,"15min":11.581},"last":4.3109999999999999},{"interface":"front","average":{"1min":2.758,"5min":1.597,"15min":1.095},"min":{"1min":0.25800000000000001,"5min":0.249,"15min":0.249},"max":{"1min":6.4660000000000002,"5min":7.6509999999999998,"15min":7.6509999999999998},"last":4.1100000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6349999999999998,"5min":1.54,"15min":1.0880000000000001},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.7370000000000001,"5min":7.867,"15min":7.867},"last":4.181},{"interface":"front","average":{"1min":2.347,"5min":1.6479999999999999,"15min":1.1419999999999999},"min":{"1min":0.248,"5min":0.223,"15min":0.223},"max":{"1min":7.3550000000000004,"5min":13.856,"15min":13.856},"last":4.5369999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":1.532,"15min":1.2050000000000001},"min":{"1min":0.42699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":12.629,"5min":12.629,"15min":12.629},"last":4.431},{"interface":"front","average":{"1min":1.9339999999999999,"5min":1.4350000000000001,"15min":1.1479999999999999},"min":{"1min":0.313,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.827,"5min":7.827,"15min":7.827},"last":4.4589999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4129999999999998,"5min":1.5940000000000001,"15min":1.2889999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":6.593,"5min":7.6929999999999996,"15min":7.6929999999999996},"last":4.7389999999999999},{"interface":"front","average":{"1min":3.0259999999999998,"5min":1.631,"15min":1.3049999999999999},"min":{"1min":0.40400000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":11.135999999999999,"5min":11.388999999999999,"15min":11.388999999999999},"last":8.1430000000000007}]},{"osd":6,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":1.54,"15min":1.242},"min":{"1min":0.33600000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.7830000000000004,"5min":7.7830000000000004,"15min":7.7830000000000004},"last":7.976},{"interface":"front","average":{"1min":2.8300000000000001,"5min":1.6739999999999999,"15min":1.3460000000000001},"min":{"1min":0.438,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":11.058999999999999,"5min":17.234999999999999,"15min":17.234999999999999},"last":7.806}]},{"osd":7,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":2.7010000000000001,"5min":1.708,"15min":1.3799999999999999},"min":{"1min":0.45900000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.5650000000000004,"5min":7.7320000000000002,"15min":7.7320000000000002},"last":7.8220000000000001},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.6439999999999999,"15min":1.3169999999999999},"min":{"1min":0.375,"5min":0.318,"15min":0.318},"max":{"1min":7.4039999999999999,"5min":7.5869999999999997,"15min":7.5869999999999997},"last":5.0800000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986540,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335172,"kb_used_data":1037948,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92409980,"statfs":{"total":95995035648,"available":94627819520,"internally_reserved":0,"allocated":1062858752,"data_stored":1062561449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":0.94299999999999995,"15min":0.90900000000000003},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.2800000000000002,"5min":4.2800000000000002,"15min":4.2800000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.698,"5min":1.0589999999999999,"15min":0.98399999999999999},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.5149999999999997,"5min":7.5149999999999997,"15min":7.5149999999999997},"last":0.27000000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:28:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":1.2370000000000001,"15min":1.0049999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.5439999999999996,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.6399999999999999,"5min":1.105,"15min":1.0329999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.5640000000000001,"5min":7.5640000000000001,"15min":7.5640000000000001},"last":0.34499999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.081,"5min":1.3440000000000001,"15min":1.1080000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.1940000000000008,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":2.0230000000000001},{"interface":"front","average":{"1min":2.0139999999999998,"5min":1.272,"15min":1.0640000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":10.773999999999999,"5min":10.773999999999999,"15min":10.773999999999999},"last":2.3010000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":3.9180000000000001,"5min":1.7130000000000001,"15min":1.2589999999999999},"min":{"1min":0.38100000000000001,"5min":0.372,"15min":0.372},"max":{"1min":17.376000000000001,"5min":17.376000000000001,"15min":17.376000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":4.2510000000000003,"5min":1.8220000000000001,"15min":1.3660000000000001},"min":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":17.277999999999999,"5min":17.277999999999999,"15min":17.277999999999999},"last":1.212}]},{"osd":5,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.2649999999999999,"15min":1.0740000000000001},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.8319999999999999,"5min":7.2999999999999998,"15min":7.2999999999999998},"last":1.976},{"interface":"front","average":{"1min":1.502,"5min":1.4410000000000001,"15min":1.2130000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.165,"5min":7.5819999999999999,"15min":7.5819999999999999},"last":0.30599999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.8,"5min":1.2030000000000001,"15min":1.1020000000000001},"min":{"1min":0.44500000000000001,"5min":0.377,"15min":0.377},"max":{"1min":7.625,"5min":7.625,"15min":7.625},"last":0.38},{"interface":"front","average":{"1min":1.9730000000000001,"5min":1.2749999999999999,"15min":1.165},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":7.1420000000000003,"5min":7.1420000000000003,"15min":7.1420000000000003},"last":2.1070000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":1.3480000000000001,"15min":1.147},"min":{"1min":0.38900000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":9.2560000000000002,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":2.0590000000000002},{"interface":"front","average":{"1min":2.5270000000000001,"5min":1.6180000000000001,"15min":1.294},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":9.1780000000000008,"5min":9.1780000000000008,"15min":9.1780000000000008},"last":0.36899999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790306,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1310968,"kb_used_data":1005552,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92434184,"statfs":{"total":95995035648,"available":94652604416,"internally_reserved":0,"allocated":1029685248,"data_stored":1029396001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:27:39 2023","interfaces":[{"interface":"back","average":{"1min":1.665,"5min":1.151,"15min":0.95699999999999996},"min":{"1min":0.309,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.5549999999999997,"5min":10.109,"15min":10.109},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.359,"5min":1.077,"15min":0.97499999999999998},"min":{"1min":0.41599999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.8630000000000004,"5min":4.8630000000000004,"15min":4.8630000000000004},"last":1.0049999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":1.1870000000000001,"15min":1.032},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.9379999999999999,"5min":1.266,"15min":1.131},"min":{"1min":0.45500000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.7210000000000001,"5min":8.7210000000000001,"15min":8.7210000000000001},"last":0.70099999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:28:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":1.6140000000000001,"15min":1.2230000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":14.019,"5min":14.019,"15min":14.019},"last":0.66200000000000003},{"interface":"front","average":{"1min":3.4670000000000001,"5min":1.6599999999999999,"15min":1.234},"min":{"1min":0.41099999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":14.037000000000001,"5min":14.037000000000001,"15min":14.037000000000001},"last":0.75600000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:27:54 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":1.3149999999999999,"15min":1.101},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.798,"5min":10.193,"15min":10.193},"last":0.628},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.1419999999999999,"15min":1.1160000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.5579999999999998,"5min":7.5579999999999998,"15min":7.5579999999999998},"last":0.64700000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.028,"15min":0.97199999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.96,"5min":10.164999999999999,"15min":10.164999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.0349999999999999,"15min":1.0580000000000001},"min":{"1min":0.373,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.5780000000000003,"5min":7.5780000000000003,"15min":7.5780000000000003},"last":0.59399999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:27:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":1.157,"15min":1.0900000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.6139999999999999,"5min":10.253,"15min":10.253},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.393,"5min":1.181,"15min":1.2250000000000001},"min":{"1min":0.46500000000000002,"5min":0.372,"15min":0.372},"max":{"1min":7.5960000000000001,"5min":7.5960000000000001,"15min":7.5960000000000001},"last":0.68799999999999994}]},{"osd":7,"last update":"Mon Oct 9 09:28:21 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.3120000000000001,"15min":1.236},"min":{"1min":0.46700000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":8.6359999999999992,"5min":8.6359999999999992,"15min":8.6359999999999992},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.351,"15min":1.2150000000000001},"min":{"1min":0.45200000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.5309999999999997,"5min":6.5229999999999997,"15min":6.5229999999999997},"last":0.85199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":810946560,"data_stored":810942488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1063784448,"data_stored":1063780376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1669136384,"data_stored":1669136384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2113798144,"data_stored":2113798144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1041170432,"data_stored":1041170432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1022758912,"data_stored":1022754840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1032519680,"data_stored":1032519680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1239351296,"data_stored":1239351296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:28:40.545 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T09:28:40.545 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:28:40.546 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T09:28:40.546 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:28:40.546 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:28:40.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T09:28:40.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:40.580 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:40.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:40.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:40.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T09:28:40.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:40.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:40.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:40.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:40.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:40 smithi181 bash[59345]: audit 2023-10-09T09:28:40.363911+0000 mon.a (mon.0) 1475 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:40.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:40.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:40.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:40.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:40 smithi163 bash[78107]: audit 2023-10-09T09:28:40.363911+0000 mon.a (mon.0) 1475 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:40.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:40 smithi163 bash[75425]: audit 2023-10-09T09:28:40.363911+0000 mon.a (mon.0) 1475 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:41.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:41.032 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:41.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:41.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:41.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:41.166 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:41.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:41.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:41.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:41.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:41.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:41.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:41.429 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:41.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:41.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:41.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:41.563 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:41.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:41.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:41.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:41.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:41.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:41.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:41.812 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:41.838 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:41.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:41.941 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:41.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:41 smithi181 bash[59345]: cluster 2023-10-09T09:28:41.657365+0000 mgr.y (mgr.34107) 671 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T09:28:41.967 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:41.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:41 smithi163 bash[78107]: cluster 2023-10-09T09:28:41.657365+0000 mgr.y (mgr.34107) 671 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T09:28:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:41 smithi163 bash[75425]: cluster 2023-10-09T09:28:41.657365+0000 mgr.y (mgr.34107) 671 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T09:28:41.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:41.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:42.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:42.105 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:42.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:42.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:42.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:42.238 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:42.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:42.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:42.343 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:42.390 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:42.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:42.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:42.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:42.638 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:42.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:42.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:42.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:42.769 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:42.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:42.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:42.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:42.896 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:42.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:42.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:43.001 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:43.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:43.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:43.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:43.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:43.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:43.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:43.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:28:43.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:28:43.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:43.289 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:43.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:43.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:43.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:43.456 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:43.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:43.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:43.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:28:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:28:43] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.33.4" 2023-10-09T09:28:43.564 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:43.592 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:43.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:43.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:43.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:43.731 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:43.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:43.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:43.836 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:43.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:43.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:43.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:43.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:44.000 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:44.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:44.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:44.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:44.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:44.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:44.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:44.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:44.269 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:44.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:44.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:44.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:44.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:44.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:44.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:44.555 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:44.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:44.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:44.660 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:44.685 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:44.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:44.714 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:44 smithi163 bash[75425]: cluster 2023-10-09T09:28:43.657874+0000 mgr.y (mgr.34107) 672 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T09:28:44.714 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:44 smithi163 bash[78107]: cluster 2023-10-09T09:28:43.657874+0000 mgr.y (mgr.34107) 672 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T09:28:44.789 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:44.816 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:44.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:44.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:44.923 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:44.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:44.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:44.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:45.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:44 smithi181 bash[59345]: cluster 2023-10-09T09:28:43.657874+0000 mgr.y (mgr.34107) 672 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T09:28:45.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:45.098 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:45.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:45.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:45.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:45.233 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:45.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:45.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:45.340 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:45.368 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:45.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:45.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:45.422 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:45.422 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:45.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:45.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:45.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:45.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:45.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:45.636 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:45.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:45.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:45.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:45.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:45.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:45.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:45.915 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:45.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:45.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:46.023 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:46.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:46.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:46.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:46.154 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:46.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:46.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:46.301 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:46.334 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:46.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:46.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:46.436 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:46.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:46.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:46.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:46.570 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:46.600 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:46.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:46.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:46.704 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:46.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:46 smithi163 bash[75425]: cluster 2023-10-09T09:28:45.658453+0000 mgr.y (mgr.34107) 673 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T09:28:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:46 smithi163 bash[78107]: cluster 2023-10-09T09:28:45.658453+0000 mgr.y (mgr.34107) 673 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T09:28:46.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:46.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:46.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:46 smithi181 bash[59345]: cluster 2023-10-09T09:28:45.658453+0000 mgr.y (mgr.34107) 673 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T09:28:46.846 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:46.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:46.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:46.983 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:47.018 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:47.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:47.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:47.122 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:47.147 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:47.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:47.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:47.251 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:47.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:47.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:47.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:47.407 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:47.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:47.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:47.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:47.464 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:47.464 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:28:47.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:47.571 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:47.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:47.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:47.676 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:47.711 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:47.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:47.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:47.819 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:47.848 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:47.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:47.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:47.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:47.976 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:47.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:47.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:48.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:48.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:48.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:48.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:48.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:48.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:48.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:48.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:48.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:48.394 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:48.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:48.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:48.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:48.534 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:48.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:48.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:48.641 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:48.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:48.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:48.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:48.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:48.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:48 smithi163 bash[78107]: cluster 2023-10-09T09:28:47.659143+0000 mgr.y (mgr.34107) 674 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T09:28:48.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:48 smithi163 bash[75425]: cluster 2023-10-09T09:28:47.659143+0000 mgr.y (mgr.34107) 674 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T09:28:48.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:48.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:48.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:48.951 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:48.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:48.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:49.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:49.062 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:48 smithi181 bash[59345]: cluster 2023-10-09T09:28:47.659143+0000 mgr.y (mgr.34107) 674 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T09:28:49.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:49.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:49.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:49.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:49.236 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:49.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:49.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:49.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:49.374 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:49.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:49.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:49.505 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:49.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:49.614 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:49.638 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:49.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:49.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:49.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:49.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:49.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:49.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:49.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:49.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:49.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:50.012 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:49 smithi163 bash[78107]: cluster 2023-10-09T09:28:49.659646+0000 mgr.y (mgr.34107) 675 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T09:28:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:49 smithi163 bash[75425]: cluster 2023-10-09T09:28:49.659646+0000 mgr.y (mgr.34107) 675 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T09:28:50.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:50.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:49 smithi181 bash[59345]: cluster 2023-10-09T09:28:49.659646+0000 mgr.y (mgr.34107) 675 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T09:28:50.053 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:50.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:50.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:50.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:50.184 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:50.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:50.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:50.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:50.324 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:50.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:50.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:50.429 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:50.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:50.458 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:50.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:50.559 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:50.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:50.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:50.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:50.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:50.721 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:50.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:50.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:50.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:50.851 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:50.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:50.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:50.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:50.980 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:50.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:51.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:51.112 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:51.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:51.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:51.215 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:51.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:51.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:51.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:51.349 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:51.383 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:51.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:51.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:51.488 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:51.521 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:51.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:51.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:51.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:51.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:51.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:51.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:51.762 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:51.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:51 smithi181 bash[59345]: cluster 2023-10-09T09:28:51.660341+0000 mgr.y (mgr.34107) 676 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-10-09T09:28:51.793 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:51.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:51.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:51.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:51 smithi163 bash[75425]: cluster 2023-10-09T09:28:51.660341+0000 mgr.y (mgr.34107) 676 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-10-09T09:28:51.910 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:51 smithi163 bash[78107]: cluster 2023-10-09T09:28:51.660341+0000 mgr.y (mgr.34107) 676 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-10-09T09:28:51.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:51.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:51.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:52.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:52.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:52.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:52.173 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:52.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:52.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:52.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:52.298 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:52.333 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:52.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:52.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:52.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:52.457 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:52.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:52.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:52.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:52.584 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:52.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:52.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:52.686 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:52.716 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:52.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:52.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:52.823 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:52.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:52.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:52.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:52.963 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:52.996 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:52.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:52.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:53.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:53.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:53.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:53.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:53.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:53.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:53.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:53.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:53.368 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:53.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:53.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:53.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:53.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:28:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:28:53] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.33.4" 2023-10-09T09:28:53.506 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:53.533 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:53.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:53.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:53.640 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:53.667 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:53.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:53.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:53.771 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:53.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:53.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:53.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:53.908 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:53.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:53.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:53.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:54.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:54.081 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:54.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:54.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:54.187 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:54.215 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:54.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:54.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:54.324 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:54.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:54.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:54.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:54.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:54.488 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:54.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:54.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:54.591 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:54.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:54.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:54.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:54.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:54.716 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:54 smithi181 bash[59345]: cluster 2023-10-09T09:28:53.660845+0000 mgr.y (mgr.34107) 677 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T09:28:54.744 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:54.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:54.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:54.771 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:54 smithi163 bash[78107]: cluster 2023-10-09T09:28:53.660845+0000 mgr.y (mgr.34107) 677 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T09:28:54.771 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:54 smithi163 bash[75425]: cluster 2023-10-09T09:28:53.660845+0000 mgr.y (mgr.34107) 677 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T09:28:54.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:54.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:54.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:54.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:54.983 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:55.007 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:55.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:55.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:55.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:55.176 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:55.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:55.278 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:55.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:55.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:55.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:55.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:55.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:55.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:55.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:55.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:55.584 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:55.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:55.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:55.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:55.719 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:55 smithi181 bash[59345]: audit 2023-10-09T09:28:55.364281+0000 mon.a (mon.0) 1476 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:55.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:55.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:55.829 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:55.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:55.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:55.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:55.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:55.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:55 smithi163 bash[75425]: audit 2023-10-09T09:28:55.364281+0000 mon.a (mon.0) 1476 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:55 smithi163 bash[78107]: audit 2023-10-09T09:28:55.364281+0000 mon.a (mon.0) 1476 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:28:56.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:56.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:56.129 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:56.158 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:56.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:56.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:56.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:56.299 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:56.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:56.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:56.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:56.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:56.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:56.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:56.535 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:56.559 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:56.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:56.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:56.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:56.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:56.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:56.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:56.800 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:56.823 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:56.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:56.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:56.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:56.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:56 smithi163 bash[75425]: cluster 2023-10-09T09:28:55.661389+0000 mgr.y (mgr.34107) 678 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T09:28:56.938 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:56 smithi163 bash[78107]: cluster 2023-10-09T09:28:55.661389+0000 mgr.y (mgr.34107) 678 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T09:28:56.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:56.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:56.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:57.061 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:57.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:56 smithi181 bash[59345]: cluster 2023-10-09T09:28:55.661389+0000 mgr.y (mgr.34107) 678 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-09T09:28:57.088 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:57.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:57.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:57.220 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:57.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:57.323 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:57.351 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:57.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:57.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:57.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:57.461 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:28:57.461 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:28:57.461 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T09:28:57.461 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T09:28:57.462 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:28:57.485 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:57.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:57.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:57.588 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:57.611 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:57.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:57.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:57.720 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:57.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:28:57.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:57.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:28:57.810 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:28:57.810 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T09:28:57.811 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:28:57.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:57.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:57.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:57.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:57.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:58.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:57 smithi163 bash[75425]: cluster 2023-10-09T09:28:57.662045+0000 mgr.y (mgr.34107) 679 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-10-09T09:28:58.008 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:57 smithi163 bash[78107]: cluster 2023-10-09T09:28:57.662045+0000 mgr.y (mgr.34107) 679 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-10-09T09:28:58.032 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:58.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:58.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:57 smithi181 bash[59345]: cluster 2023-10-09T09:28:57.662045+0000 mgr.y (mgr.34107) 679 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-10-09T09:28:58.135 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:58.156 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:58.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:58.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:58.259 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:58.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:58.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:58.393 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:58.420 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:28:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:58.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:28:58.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:58.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:28:58.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:58.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:28:58.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:58.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:58.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:58.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:58.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:58.816 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:58.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:58.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:58.923 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:58.959 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:58.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:58.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:28:58.983 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:58 smithi163 bash[75425]: audit 2023-10-09T09:28:58.917613+0000 mon.c (mon.2) 52 : audit 1 from='client.? 172.21.15.163:0/3704569548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:28:58.983 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:58 smithi163 bash[75425]: audit 2023-10-09T09:28:58.918050+0000 mon.a (mon.0) 1477 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:28:58.998 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:28:59.065 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:59.074 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:58 smithi163 bash[78107]: audit 2023-10-09T09:28:58.917613+0000 mon.c (mon.2) 52 : audit 1 from='client.? 172.21.15.163:0/3704569548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:28:59.074 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:58 smithi163 bash[78107]: audit 2023-10-09T09:28:58.918050+0000 mon.a (mon.0) 1477 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:28:59.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:28:59.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:59.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:28:59.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:59.233 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:59.237 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:58 smithi181 bash[59345]: audit 2023-10-09T09:28:58.917613+0000 mon.c (mon.2) 52 : audit 1 from='client.? 172.21.15.163:0/3704569548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:28:59.237 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:58 smithi181 bash[59345]: audit 2023-10-09T09:28:58.918050+0000 mon.a (mon.0) 1477 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:28:59.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:59.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:59.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:59.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:59.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:59.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:59.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:59.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:28:59.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:59.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:28:59.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:59.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:28:59.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:59.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:28:59.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:59.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:28:59.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:59.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:28:59.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:59.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:28:59.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:59.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:00.039 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:00.047 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:59 smithi181 bash[59345]: cluster 2023-10-09T09:28:58.971932+0000 mon.a (mon.0) 1478 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:29:00.047 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:59 smithi181 bash[59345]: cluster 2023-10-09T09:28:58.974258+0000 osd.5 (osd.5) 638 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.047 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:59 smithi181 bash[59345]: audit 2023-10-09T09:28:58.978139+0000 mon.a (mon.0) 1479 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:29:00.047 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:59 smithi181 bash[59345]: cluster 2023-10-09T09:28:58.978355+0000 mon.a (mon.0) 1480 : cluster 0 osdmap e1094: 8 total, 8 up, 8 in 2023-10-09T09:29:00.048 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:59 smithi181 bash[59345]: cluster 2023-10-09T09:28:58.982968+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.048 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:59 smithi181 bash[59345]: cluster 2023-10-09T09:28:58.997358+0000 osd.4 (osd.4) 522 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.048 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:59 smithi181 bash[59345]: cluster 2023-10-09T09:28:58.999977+0000 osd.4 (osd.4) 523 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.048 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:28:59 smithi181 bash[59345]: cluster 2023-10-09T09:28:59.662490+0000 mgr.y (mgr.34107) 680 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T09:29:00.060 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:00.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:00.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:00.164 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:00.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[75425]: cluster 2023-10-09T09:28:58.971932+0000 mon.a (mon.0) 1478 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:29:00.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[75425]: cluster 2023-10-09T09:28:58.974258+0000 osd.5 (osd.5) 638 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[75425]: audit 2023-10-09T09:28:58.978139+0000 mon.a (mon.0) 1479 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:29:00.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[75425]: cluster 2023-10-09T09:28:58.978355+0000 mon.a (mon.0) 1480 : cluster 0 osdmap e1094: 8 total, 8 up, 8 in 2023-10-09T09:29:00.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[75425]: cluster 2023-10-09T09:28:58.982968+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[75425]: cluster 2023-10-09T09:28:58.997358+0000 osd.4 (osd.4) 522 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[75425]: cluster 2023-10-09T09:28:58.999977+0000 osd.4 (osd.4) 523 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[75425]: cluster 2023-10-09T09:28:59.662490+0000 mgr.y (mgr.34107) 680 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T09:29:00.174 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[78107]: cluster 2023-10-09T09:28:58.971932+0000 mon.a (mon.0) 1478 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:29:00.174 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[78107]: cluster 2023-10-09T09:28:58.974258+0000 osd.5 (osd.5) 638 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.174 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[78107]: audit 2023-10-09T09:28:58.978139+0000 mon.a (mon.0) 1479 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:29:00.174 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[78107]: cluster 2023-10-09T09:28:58.978355+0000 mon.a (mon.0) 1480 : cluster 0 osdmap e1094: 8 total, 8 up, 8 in 2023-10-09T09:29:00.174 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[78107]: cluster 2023-10-09T09:28:58.982968+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.174 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[78107]: cluster 2023-10-09T09:28:58.997358+0000 osd.4 (osd.4) 522 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.174 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[78107]: cluster 2023-10-09T09:28:58.999977+0000 osd.4 (osd.4) 523 : cluster 3 failed to encode map e1094 with expected crc 2023-10-09T09:29:00.174 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:28:59 smithi163 bash[78107]: cluster 2023-10-09T09:28:59.662490+0000 mgr.y (mgr.34107) 680 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T09:29:00.188 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:00.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:00.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:00.198 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:29:00.245 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:29:00.288 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:29:00.292 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:00.320 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:00.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:00.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:00.332 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:29:00.365 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:29:00.398 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:29:00.428 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:00.433 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:29:00.457 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:00.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:00.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:00.478 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:29:00.482 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:29:00.482 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:29:00.482 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:29:00.482 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:29:00.482 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:29:00.483 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:29:00.483 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:29:00.483 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:29:00.560 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:00.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:00.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:00.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:00.718 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:00.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:00.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:00.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:00.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T09:29:00.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:00.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:00.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:00.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:01.007 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:01.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:01.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:01.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:01.120 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:00 smithi163 bash[75425]: audit 2023-10-09T09:29:00.145127+0000 mon.a (mon.0) 1481 : audit 0 from='client.? 172.21.15.163:0/3697526929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:01.120 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:00 smithi163 bash[78107]: audit 2023-10-09T09:29:00.145127+0000 mon.a (mon.0) 1481 : audit 0 from='client.? 172.21.15.163:0/3697526929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:01.152 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:01.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:01.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:01.262 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:01.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:01.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:01.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:01.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:00 smithi181 bash[59345]: audit 2023-10-09T09:29:00.145127+0000 mon.a (mon.0) 1481 : audit 0 from='client.? 172.21.15.163:0/3697526929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:01.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:01.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:01.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:01.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:01.529 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:01.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:01.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:01.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:01.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:01.696 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:01.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:01.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:01.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:01.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:01.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:01.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:01.934 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:01.961 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:01.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:02.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:02.108 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:02.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:02.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:02.213 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:02.254 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:02.262 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:02 smithi163 bash[75425]: cluster 2023-10-09T09:29:01.663163+0000 mgr.y (mgr.34107) 681 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-10-09T09:29:02.262 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:02 smithi163 bash[78107]: cluster 2023-10-09T09:29:01.663163+0000 mgr.y (mgr.34107) 681 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-10-09T09:29:02.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:02.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:02.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:02 smithi181 bash[59345]: cluster 2023-10-09T09:29:01.663163+0000 mgr.y (mgr.34107) 681 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 2.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-10-09T09:29:02.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:02.405 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:02.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:02.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:02.509 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:02.538 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:02.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:02.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:02.672 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:02.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:02.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:02.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:02.803 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:02.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:02.909 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:02.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:02.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:02.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:03.041 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:03.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:03.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:03.175 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:03.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:29:03.208 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:03.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:03.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:03.312 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:03.340 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:03.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:03.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:03.443 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:03.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:03.482 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:03.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:03.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:03.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:03.622 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:29:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:29:03] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.33.4" 2023-10-09T09:29:03.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:03.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:03.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:03.749 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:03.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:03.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:29:03.852 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:03.879 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:03.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:03.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:03 smithi163 bash[78107]: cluster 2023-10-09T09:29:03.663716+0000 mgr.y (mgr.34107) 682 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T09:29:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:03 smithi163 bash[75425]: cluster 2023-10-09T09:29:03.663716+0000 mgr.y (mgr.34107) 682 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T09:29:03.985 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:04.018 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:04.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:04.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:04.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:03 smithi181 bash[59345]: cluster 2023-10-09T09:29:03.663716+0000 mgr.y (mgr.34107) 682 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T09:29:04.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:04.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:04.163 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:04.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:04.264 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:04.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:04.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:04.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:04.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:04.445 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:04.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:04.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:04.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:04.576 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:04.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:04.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:04.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:04.710 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:04.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:04.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:04.818 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:04.847 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:04.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:04.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:04.952 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:04.989 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:04.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:04.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:05.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:05.125 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:05.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:05.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:05.231 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:05.271 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:05.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:05.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:05.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:05.407 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:05.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:05.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:05.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:05.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:05.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:05.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:05.667 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:05.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:05.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:05.679 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:05.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:05.807 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:05.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:05.853 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:29:05.853 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:29:05.853 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T09:29:05.853 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:29:05.853 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:29:05.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T09:29:05.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:05.943 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:05.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:05.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:06.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:06.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:06.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:06.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:06.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:06.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:06.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:06.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:06.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:06.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:06.350 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:06.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:06.451 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:06.493 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:06.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:06.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:06.603 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:06.635 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:06.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:06.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:06 smithi163 bash[75425]: cluster 2023-10-09T09:29:05.664252+0000 mgr.y (mgr.34107) 683 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-10-09T09:29:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:06 smithi163 bash[78107]: cluster 2023-10-09T09:29:05.664252+0000 mgr.y (mgr.34107) 683 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-10-09T09:29:06.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:06.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:06.766 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:06 smithi181 bash[59345]: cluster 2023-10-09T09:29:05.664252+0000 mgr.y (mgr.34107) 683 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-10-09T09:29:06.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:06.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:06.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:06.906 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:06.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:06.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:07.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:07.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:07.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:07.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:07.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:07.176 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:07.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:07.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:07.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:07.313 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:07.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:07.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:07.415 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:07.456 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:07.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:07.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:07.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:07.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:07.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:07.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:07.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:07.722 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:07.727 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:07.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:07.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:07.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:07.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:07.894 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:07.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:07.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:07.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:07 smithi163 bash[78107]: cluster 2023-10-09T09:29:07.665030+0000 mgr.y (mgr.34107) 684 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T09:29:07.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:07 smithi163 bash[75425]: cluster 2023-10-09T09:29:07.665030+0000 mgr.y (mgr.34107) 684 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T09:29:07.998 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:08.023 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:08.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:08.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:07 smithi181 bash[59345]: cluster 2023-10-09T09:29:07.665030+0000 mgr.y (mgr.34107) 684 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T09:29:08.126 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:08.152 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:08.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:08.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:08.300 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:08.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:08.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:08.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:08.440 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:08.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:08.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:08.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:08.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:08.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:08.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:08.709 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:08.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:08.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:08.812 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:08.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:08.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:08.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:08.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:08.974 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:08.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:08.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:09.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:09.120 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:09.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:09.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:09.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:09.253 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:09.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:09.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:09.358 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:09.385 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:09.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:09.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:09.490 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:09.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:09.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:09.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:09.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:09.685 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:09.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:09.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:09.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:09.816 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:09.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:09.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:09.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:09.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:09.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:10.055 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:10.091 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:10.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:10.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:10.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:10.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:10.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:10.353 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:10.378 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:10.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:10.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:10.482 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:10.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:10.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:10.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:10.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:10.651 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:29:10.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:10.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:10.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:10.760 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:10.784 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:10.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:10.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:10.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:10 smithi163 bash[78107]: cluster 2023-10-09T09:29:09.665523+0000 mgr.y (mgr.34107) 685 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T09:29:10.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:10 smithi163 bash[78107]: audit 2023-10-09T09:29:10.364677+0000 mon.a (mon.0) 1482 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:10.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:10 smithi163 bash[75425]: cluster 2023-10-09T09:29:09.665523+0000 mgr.y (mgr.34107) 685 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T09:29:10.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:10 smithi163 bash[75425]: audit 2023-10-09T09:29:10.364677+0000 mon.a (mon.0) 1482 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:10.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:10.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:10.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:11.025 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:11.048 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:11.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:11.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:10 smithi181 bash[59345]: cluster 2023-10-09T09:29:09.665523+0000 mgr.y (mgr.34107) 685 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T09:29:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:10 smithi181 bash[59345]: audit 2023-10-09T09:29:10.364677+0000 mon.a (mon.0) 1482 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:11.154 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:11.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:11.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:11.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:11.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:11.330 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:11.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:11.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:11.434 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:11.465 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:11.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:11.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:11.573 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:11.596 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:11.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:11.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:11.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:11.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:11.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:11.839 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:11 smithi181 bash[59345]: cluster 2023-10-09T09:29:11.666189+0000 mgr.y (mgr.34107) 686 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T09:29:11.860 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:11.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:11.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:11.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:11 smithi163 bash[75425]: cluster 2023-10-09T09:29:11.666189+0000 mgr.y (mgr.34107) 686 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T09:29:11.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:11 smithi163 bash[78107]: cluster 2023-10-09T09:29:11.666189+0000 mgr.y (mgr.34107) 686 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T09:29:11.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:11.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:11.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:12.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:12.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:12.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:12.225 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:12.249 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:12.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:12.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:12.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:12.380 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:12.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:12.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:12.483 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:12.512 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:12.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:12.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:12.618 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:12.646 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:12.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:12.751 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:12.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:12.775 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:29:12.775 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:29:12.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:12.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:12.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:12.900 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:12.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:12.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:13.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:13.038 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:13.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:13.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:13.171 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:13.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:13.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:13.314 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:13.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:13.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:13.418 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:13.440 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:13.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:13.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:13.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:13.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:13.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:13.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:13.661 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:13 smithi163 bash[78107]: audit 2023-10-09T09:29:13.606550+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:29:13.661 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:13 smithi163 bash[75425]: audit 2023-10-09T09:29:13.606550+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:29:13.661 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:29:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:29:13] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.33.4" 2023-10-09T09:29:13.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:13.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:13.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:13.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:13 smithi181 bash[59345]: audit 2023-10-09T09:29:13.606550+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:29:13.825 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:13.856 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:13.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:13.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:13.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:13.988 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:13.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:14.092 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:14.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:14.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:14.225 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:14.264 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:14.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:14.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:14.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:14.398 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:14.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:14.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:14.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:14.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:14.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:14.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:14.643 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T09:29:14.645 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:14.670 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:14.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:14.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:14.773 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:14.802 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:14.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:14.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:14.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:14 smithi181 bash[59345]: cluster 2023-10-09T09:29:13.671305+0000 mgr.y (mgr.34107) 687 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-10-09T09:29:14.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:14.914 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:14 smithi163 bash[78107]: cluster 2023-10-09T09:29:13.671305+0000 mgr.y (mgr.34107) 687 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-10-09T09:29:14.914 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:14 smithi163 bash[75425]: cluster 2023-10-09T09:29:13.671305+0000 mgr.y (mgr.34107) 687 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-10-09T09:29:14.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:14.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:14.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:15.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:15.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:15.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:15.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:15.174 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:15.199 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:15.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:15.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:15.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:15.339 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:15.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:15.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:15.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:15.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:15.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:15.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:15.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:15.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:15.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:15.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:15.724 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:15.750 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:15.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:15.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:15.856 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:15.896 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:15.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:15.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:16.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:16.034 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:16.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:16.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:16.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:16.166 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:16.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:16.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:16.272 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:16.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:16.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:16.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:16.407 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:16.436 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:16.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:16.457 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:29:16.512 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:29:16.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:16.551 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:16 smithi181 bash[59345]: cluster 2023-10-09T09:29:15.671817+0000 mgr.y (mgr.34107) 688 : cluster 0 pgmap v1388: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T09:29:16.552 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:16 smithi181 bash[59345]: audit 2023-10-09T09:29:16.397827+0000 mon.a (mon.0) 1484 : audit 0 from='client.? 172.21.15.163:0/1533758807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:16.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:16.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:16.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:16.569 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:29:16.615 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:29:16.658 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:29:16.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:16.677 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:16 smithi163 bash[75425]: cluster 2023-10-09T09:29:15.671817+0000 mgr.y (mgr.34107) 688 : cluster 0 pgmap v1388: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T09:29:16.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:16 smithi163 bash[75425]: audit 2023-10-09T09:29:16.397827+0000 mon.a (mon.0) 1484 : audit 0 from='client.? 172.21.15.163:0/1533758807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:16.678 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:16 smithi163 bash[78107]: cluster 2023-10-09T09:29:15.671817+0000 mgr.y (mgr.34107) 688 : cluster 0 pgmap v1388: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T09:29:16.678 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:16 smithi163 bash[78107]: audit 2023-10-09T09:29:16.397827+0000 mon.a (mon.0) 1484 : audit 0 from='client.? 172.21.15.163:0/1533758807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:16.694 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:29:16.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:16.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:16.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:16.738 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:29:16.781 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:29:16.786 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:29:16.787 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:29:16.787 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:29:16.787 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:29:16.787 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:29:16.787 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:29:16.787 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:29:16.787 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:29:16.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:16.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:16.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:16.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:16.900 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:29:16.900 INFO:teuthology.orchestra.run.smithi181.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T09:29:16.900 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:29:16.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:16.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:16.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:16.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:17.075 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:17.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:17.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:17.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:17.228 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:17.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:17.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:17.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:17.360 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:17.382 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:17.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:17.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:17.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:17 smithi181 bash[59345]: audit 2023-10-09T09:29:16.787350+0000 mon.a (mon.0) 1485 : audit 1 from='client.? 172.21.15.163:0/2620285203' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:29:17.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:17.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:17.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:17.541 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:29:17.628 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:17.655 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:17.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:17.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:17.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:17.769 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:17 smithi163 bash[75425]: audit 2023-10-09T09:29:16.787350+0000 mon.a (mon.0) 1485 : audit 1 from='client.? 172.21.15.163:0/2620285203' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:29:17.769 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:17 smithi163 bash[78107]: audit 2023-10-09T09:29:16.787350+0000 mon.a (mon.0) 1485 : audit 1 from='client.? 172.21.15.163:0/2620285203' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:29:17.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:17.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:17.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:17.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:17.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:17.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:17.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:18.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:18.075 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:18.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:18.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:18.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:18.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:18.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:18.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:18.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:18.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:18.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:18.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:18.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:18.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:18.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:18.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:18.588 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:18 smithi181 bash[59345]: cluster 2023-10-09T09:29:17.519757+0000 mon.a (mon.0) 1486 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:29:18.588 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:18 smithi181 bash[59345]: cluster 2023-10-09T09:29:17.519798+0000 mon.a (mon.0) 1487 : cluster 1 Cluster is now healthy 2023-10-09T09:29:18.588 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:18 smithi181 bash[59345]: cluster 2023-10-09T09:29:17.529199+0000 osd.5 (osd.5) 639 : cluster 3 failed to encode map e1095 with expected crc 2023-10-09T09:29:18.588 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:18 smithi181 bash[59345]: audit 2023-10-09T09:29:17.532716+0000 mon.a (mon.0) 1488 : audit 1 from='client.? 172.21.15.163:0/2620285203' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:29:18.588 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:18 smithi181 bash[59345]: cluster 2023-10-09T09:29:17.533068+0000 mon.a (mon.0) 1489 : cluster 0 osdmap e1095: 8 total, 8 up, 8 in 2023-10-09T09:29:18.589 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:18 smithi181 bash[59345]: cluster 2023-10-09T09:29:17.672246+0000 mgr.y (mgr.34107) 689 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T09:29:18.613 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:18.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:18.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:18.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:18.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:18.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:18.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:18.848 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:18.858 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[75425]: cluster 2023-10-09T09:29:17.519757+0000 mon.a (mon.0) 1486 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:29:18.858 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[75425]: cluster 2023-10-09T09:29:17.519798+0000 mon.a (mon.0) 1487 : cluster 1 Cluster is now healthy 2023-10-09T09:29:18.858 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[75425]: cluster 2023-10-09T09:29:17.529199+0000 osd.5 (osd.5) 639 : cluster 3 failed to encode map e1095 with expected crc 2023-10-09T09:29:18.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[75425]: audit 2023-10-09T09:29:17.532716+0000 mon.a (mon.0) 1488 : audit 1 from='client.? 172.21.15.163:0/2620285203' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:29:18.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[75425]: cluster 2023-10-09T09:29:17.533068+0000 mon.a (mon.0) 1489 : cluster 0 osdmap e1095: 8 total, 8 up, 8 in 2023-10-09T09:29:18.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[75425]: cluster 2023-10-09T09:29:17.672246+0000 mgr.y (mgr.34107) 689 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T09:29:18.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[78107]: cluster 2023-10-09T09:29:17.519757+0000 mon.a (mon.0) 1486 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:29:18.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[78107]: cluster 2023-10-09T09:29:17.519798+0000 mon.a (mon.0) 1487 : cluster 1 Cluster is now healthy 2023-10-09T09:29:18.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[78107]: cluster 2023-10-09T09:29:17.529199+0000 osd.5 (osd.5) 639 : cluster 3 failed to encode map e1095 with expected crc 2023-10-09T09:29:18.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[78107]: audit 2023-10-09T09:29:17.532716+0000 mon.a (mon.0) 1488 : audit 1 from='client.? 172.21.15.163:0/2620285203' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:29:18.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[78107]: cluster 2023-10-09T09:29:17.533068+0000 mon.a (mon.0) 1489 : cluster 0 osdmap e1095: 8 total, 8 up, 8 in 2023-10-09T09:29:18.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:18 smithi163 bash[78107]: cluster 2023-10-09T09:29:17.672246+0000 mgr.y (mgr.34107) 689 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T09:29:18.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:18.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:18.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:18.979 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:19.003 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:19.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:19.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:19.109 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:19.141 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:19.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:19.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:19.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:19.266 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:29:19.288 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:19.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:19.399 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:19.436 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:19.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:19.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:19.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:19.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:19.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:19.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:19.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:19.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:19.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:19.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:19.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:19.846 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:19.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:19.956 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:19.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:20.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:20.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:20.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:29:20.102 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:20.129 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:20.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:20.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:20.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:20.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:20.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:20.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:20.368 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:20.400 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:20.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:20.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:20.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:20.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:20.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:20.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:20.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:20.684 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:20.685 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:29:20.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:20.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:20.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:20.814 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:20.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:20.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:20.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:20.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:20 smithi181 bash[59345]: cluster 2023-10-09T09:29:19.673354+0000 mgr.y (mgr.34107) 690 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T09:29:20.954 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:20.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:20.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:20.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:20 smithi163 bash[75425]: cluster 2023-10-09T09:29:19.673354+0000 mgr.y (mgr.34107) 690 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T09:29:20.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:20 smithi163 bash[78107]: cluster 2023-10-09T09:29:19.673354+0000 mgr.y (mgr.34107) 690 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T09:29:21.064 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:21.090 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:21.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:21.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:21.195 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:21.225 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:21.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:21.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:21.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:21.375 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:21.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:21.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:21.478 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:21.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:21.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:21.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:21.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:21.638 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:21.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:21.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:21.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:21.777 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:21.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:21.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:21.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:21.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:21.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:22.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:22.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:21 smithi181 bash[59345]: audit 2023-10-09T09:29:20.838334+0000 mon.a (mon.0) 1490 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:21 smithi181 bash[59345]: audit 2023-10-09T09:29:20.849599+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:21 smithi181 bash[59345]: cluster 2023-10-09T09:29:21.673896+0000 mgr.y (mgr.34107) 691 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T09:29:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:21 smithi181 bash[59345]: audit 2023-10-09T09:29:21.690662+0000 mon.a (mon.0) 1492 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:21 smithi181 bash[59345]: audit 2023-10-09T09:29:21.701503+0000 mon.a (mon.0) 1493 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:22.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:22.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:22.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:22.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:21 smithi163 bash[78107]: audit 2023-10-09T09:29:20.838334+0000 mon.a (mon.0) 1490 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:21 smithi163 bash[78107]: audit 2023-10-09T09:29:20.849599+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:21 smithi163 bash[78107]: cluster 2023-10-09T09:29:21.673896+0000 mgr.y (mgr.34107) 691 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T09:29:22.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:21 smithi163 bash[78107]: audit 2023-10-09T09:29:21.690662+0000 mon.a (mon.0) 1492 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:21 smithi163 bash[78107]: audit 2023-10-09T09:29:21.701503+0000 mon.a (mon.0) 1493 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:21 smithi163 bash[75425]: audit 2023-10-09T09:29:20.838334+0000 mon.a (mon.0) 1490 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:21 smithi163 bash[75425]: audit 2023-10-09T09:29:20.849599+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.189 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:21 smithi163 bash[75425]: cluster 2023-10-09T09:29:21.673896+0000 mgr.y (mgr.34107) 691 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T09:29:22.189 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:21 smithi163 bash[75425]: audit 2023-10-09T09:29:21.690662+0000 mon.a (mon.0) 1492 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.189 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:21 smithi163 bash[75425]: audit 2023-10-09T09:29:21.701503+0000 mon.a (mon.0) 1493 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:22.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:22.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:22.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:22.317 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:22.346 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:22.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:22.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:22.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:22.486 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:22.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:22.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:22.592 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:22.616 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:22.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:22.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:22.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:22.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:22.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:22.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:22.771 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:29:22.771 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:29:22.771 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-10-09T09:29:22.771 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:29:22.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:22.898 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:22.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:22.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:22.981 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:29:22.981 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:29:22.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.981 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:203527 6.f e1095/1076 6:fc670f00:::benchmark_data_smithi163_107771_object92767:head v 1095'12531, mlcod=1095'12527)", 2023-10-09T09:29:22.981 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:22.969537+0000", 2023-10-09T09:29:22.981 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.010821434, 2023-10-09T09:29:22.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010846026999999999, 2023-10-09T09:29:22.981 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.969537+0000", 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.969537+0000", 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:29:22.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.969536+0000", 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1358e-05 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.969547+0000", 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9545999999999999e-05 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:29:22.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.969567+0000", 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5609999999999999e-06 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.969575+0000", 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002513197 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.972089+0000", 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1461999999999999e-05 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.972110+0000", 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025726249999999998 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:203539 6.2 6:460f3b86:::benchmark_data_smithi163_107771_object92779:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:22.978115+0000", 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0022434439999999998, 2023-10-09T09:29:22.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023152149999999998, 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 203539 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978115+0000", 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:29:22.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978115+0000", 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978114+0000", 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9100000000000005e-06 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978122+0000", 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3217000000000001e-05 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978135+0000", 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5669999999999997e-06 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978143+0000", 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8893e-05 2023-10-09T09:29:22.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978201+0000", 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121377 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978323+0000", 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013021500000000001 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:29:22.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978453+0000", 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00033760099999999998 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:203535 6.6 e1095/1076 6:67a8abf1:::benchmark_data_smithi163_107771_object92775:head v 1095'12972, mlcod=1095'12970)", 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:22.978376+0000", 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0019824360000000002, 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021052950000000001, 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:29:22.990 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978376+0000", 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978376+0000", 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978376+0000", 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2470000000000008e-06 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978384+0000", 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0745e-05 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:29:22.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978395+0000", 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9209999999999997e-06 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978401+0000", 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2021999999999997e-05 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978463+0000", 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7075000000000004e-05 2023-10-09T09:29:22.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:22.978530+0000", 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015315700000000001 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 3 2023-10-09T09:29:22.994 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:29:23.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:23.013 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:22 smithi181 bash[59345]: audit 2023-10-09T09:29:22.327854+0000 mon.a (mon.0) 1494 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:29:23.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:22 smithi181 bash[59345]: audit 2023-10-09T09:29:22.329358+0000 mon.a (mon.0) 1495 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:29:23.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:22 smithi181 bash[59345]: audit 2023-10-09T09:29:22.351006+0000 mon.a (mon.0) 1496 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:23.030 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:23.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:23.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:23.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:23.146 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:22 smithi163 bash[75425]: audit 2023-10-09T09:29:22.327854+0000 mon.a (mon.0) 1494 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:29:23.146 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:22 smithi163 bash[75425]: audit 2023-10-09T09:29:22.329358+0000 mon.a (mon.0) 1495 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:29:23.146 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:22 smithi163 bash[75425]: audit 2023-10-09T09:29:22.351006+0000 mon.a (mon.0) 1496 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:23.147 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:22 smithi163 bash[78107]: audit 2023-10-09T09:29:22.327854+0000 mon.a (mon.0) 1494 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:29:23.147 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:22 smithi163 bash[78107]: audit 2023-10-09T09:29:22.329358+0000 mon.a (mon.0) 1495 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:29:23.147 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:22 smithi163 bash[78107]: audit 2023-10-09T09:29:22.351006+0000 mon.a (mon.0) 1496 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:23.165 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:23.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:23.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:23.270 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:23.306 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:23.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:23.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:23.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:23.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:23.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:23.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:23.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:29:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:29:23] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.33.4" 2023-10-09T09:29:23.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:23.595 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:23.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:23.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:23.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:23.737 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:23.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:23.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:23.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T09:29:23.844 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:23.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:23 smithi163 bash[75425]: cluster 2023-10-09T09:29:23.674692+0000 mgr.y (mgr.34107) 692 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T09:29:23.906 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:23.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:23.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:24.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:24.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:23 smithi181 bash[59345]: cluster 2023-10-09T09:29:23.674692+0000 mgr.y (mgr.34107) 692 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T09:29:24.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:24.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:24.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:24.162 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:24.194 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:24.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:24.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:24.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:23 smithi163 bash[78107]: cluster 2023-10-09T09:29:23.674692+0000 mgr.y (mgr.34107) 692 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T09:29:24.266 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:24.299 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:24.335 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:24.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:24.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:24.439 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:24.467 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:24.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:24.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:24.574 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:24.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:24.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:24.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:24.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:24.744 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:24.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:24.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:24.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:24.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:24.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:24.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:24.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:25.010 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:25.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:25.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:25.113 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:25.147 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:25.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:25.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:25.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:25.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:25.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:25.283 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:25.383 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:25.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:25.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:25.519 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:25.556 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:25.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:25.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:25.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:25.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:25.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:25.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:25.796 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:25.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:25.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:25.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:25.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:25.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:25.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:25.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:26.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:26.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:26.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:26.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:26.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:26.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:26.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:26.352 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:26.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:26.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:26.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:26.485 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:26.511 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:26.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:26.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:26.617 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:26.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:26 smithi181 bash[59345]: audit 2023-10-09T09:29:25.369438+0000 mon.a (mon.0) 1497 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:26.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:26 smithi181 bash[59345]: audit 2023-10-09T09:29:25.370951+0000 mon.a (mon.0) 1498 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:26.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:26 smithi181 bash[59345]: cluster 2023-10-09T09:29:25.675272+0000 mgr.y (mgr.34107) 693 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T09:29:26.650 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:26.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:26.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:26 smithi163 bash[75425]: audit 2023-10-09T09:29:25.369438+0000 mon.a (mon.0) 1497 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:26 smithi163 bash[75425]: audit 2023-10-09T09:29:25.370951+0000 mon.a (mon.0) 1498 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:26 smithi163 bash[75425]: cluster 2023-10-09T09:29:25.675272+0000 mgr.y (mgr.34107) 693 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T09:29:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:26 smithi163 bash[78107]: audit 2023-10-09T09:29:25.369438+0000 mon.a (mon.0) 1497 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:26 smithi163 bash[78107]: audit 2023-10-09T09:29:25.370951+0000 mon.a (mon.0) 1498 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:26.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:26 smithi163 bash[78107]: cluster 2023-10-09T09:29:25.675272+0000 mgr.y (mgr.34107) 693 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T09:29:26.753 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:26.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:26.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:26.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:26.886 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:26.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:26.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:26.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:27.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:27.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:27.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:27.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:27.145 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:27.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:27.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:27.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:27.290 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:27.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:27.320 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:27.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:27.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:27.450 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:27.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:27.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:27.559 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:27.585 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:27.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:27.693 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:27.716 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:27.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:27.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:27.831 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:27 smithi181 bash[59345]: cluster 2023-10-09T09:29:27.675799+0000 mgr.y (mgr.34107) 694 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 998 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T09:29:27.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:27.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:27.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:27.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:27.976 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:27.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:28.079 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:28.088 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:27 smithi163 bash[75425]: cluster 2023-10-09T09:29:27.675799+0000 mgr.y (mgr.34107) 694 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 998 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T09:29:28.088 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:27 smithi163 bash[78107]: cluster 2023-10-09T09:29:27.675799+0000 mgr.y (mgr.34107) 694 : cluster 0 pgmap v1395: 89 pgs: 89 active+clean; 998 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T09:29:28.107 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:28.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:28.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:28.210 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:28.238 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:28.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:28.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:28.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:28.372 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:28.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:28.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:28.506 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:28.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:28.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:28.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:28.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:28.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:28.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:28.647 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:29:28.744 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:28.768 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:28.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:28.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:28.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:28.891 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:28.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:28.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:28.994 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:29.027 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:29.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:29.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:29.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:29.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:29.265 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:29.296 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:29.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:29.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:29.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:29.425 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:29.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:29.529 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:29.548 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:29.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:29.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:29.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:29.676 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:29.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:29.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:29.781 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:29.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:29.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:29.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:29.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:29.951 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:29.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:30.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:30.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:30.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:30.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:30.179 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:30.212 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:30.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:30.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:30.316 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:30.351 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:30.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:30.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:30.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:30.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:30.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:30.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:30.583 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:30.610 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:30.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:30.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:30.714 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:30.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:30.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:30.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:30.844 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:30.873 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:30.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:30.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:30.976 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:31.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:30 smithi163 bash[75425]: cluster 2023-10-09T09:29:29.676261+0000 mgr.y (mgr.34107) 695 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 971 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-10-09T09:29:31.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:30 smithi163 bash[78107]: cluster 2023-10-09T09:29:29.676261+0000 mgr.y (mgr.34107) 695 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 971 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-10-09T09:29:31.017 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:31.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:31.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:31.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:30 smithi181 bash[59345]: cluster 2023-10-09T09:29:29.676261+0000 mgr.y (mgr.34107) 695 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 971 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-10-09T09:29:31.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:31.147 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:31.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:31.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:31.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:31.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:31.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:31.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:31.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:31.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:31.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:31.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:31.509 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:31.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:31.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:31.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:31.644 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:31.669 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:31.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:31.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:31.772 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:31.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:31.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:31.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:31.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:31.938 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:31.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:31.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:32.042 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:32.078 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:32.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:32.184 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:32.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:32.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:32.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:32.318 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:32.340 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:32.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:32.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:32.445 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:32.468 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:32.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:32.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:32.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:32.610 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:32.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:32.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:32.716 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:32.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:32.743 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:32.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:32.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:32.852 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:29:32.852 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:29:32.852 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:29:32.852 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:29:32.852 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:29:32.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:32.883 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:32 smithi181 bash[59345]: cluster 2023-10-09T09:29:31.676903+0000 mgr.y (mgr.34107) 696 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 812 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T09:29:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:32.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:32.885 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:32 smithi163 bash[78107]: cluster 2023-10-09T09:29:31.676903+0000 mgr.y (mgr.34107) 696 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 812 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T09:29:32.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:32 smithi163 bash[75425]: cluster 2023-10-09T09:29:31.676903+0000 mgr.y (mgr.34107) 696 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 812 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T09:29:32.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:33.031 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:33.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:33.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:33.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:33.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:33.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:33.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:33.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:33.295 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:33.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:33.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:33.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:33.427 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:33.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:33.530 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:33.559 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:33.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:33.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:33.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:33.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:33.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:33.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:33.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T09:29:33.762 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:29:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:29:33] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.33.4" 2023-10-09T09:29:33.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:33.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:33.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:33.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:33.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:33.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:33.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:33.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:34.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:34.124 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:34.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:34.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:34.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:34.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:34.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:34.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:34.364 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:34.388 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:34.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:34.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:34.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:34.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:34.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:34.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:34.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:34.683 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:34.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:34.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:34.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:34.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:34.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:34.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:34.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:34 smithi163 bash[75425]: cluster 2023-10-09T09:29:33.679285+0000 mgr.y (mgr.34107) 697 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 787 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T09:29:34.952 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:34 smithi163 bash[78107]: cluster 2023-10-09T09:29:33.679285+0000 mgr.y (mgr.34107) 697 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 787 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T09:29:34.959 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:34.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:34.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:35.064 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:35.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:34 smithi181 bash[59345]: cluster 2023-10-09T09:29:33.679285+0000 mgr.y (mgr.34107) 697 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 787 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T09:29:35.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:35.094 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:35.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:35.195 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:35.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:35.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:35.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:35.283 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:29:35.313 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:29:35.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:35.345 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:29:35.355 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:35.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:35.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:35.401 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:29:35.440 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:29:35.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:35.489 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:29:35.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:35.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:35.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:35.530 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:29:35.567 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:29:35.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:29:35.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:29:35.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:29:35.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:29:35.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:29:35.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:29:35.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:29:35.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:29:35.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:35.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:35.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:35.718 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:35.748 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:35.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:35.854 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:35.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:35 smithi163 bash[78107]: audit 2023-10-09T09:29:35.237914+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.163:0/3437197491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:35.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:35 smithi163 bash[78107]: cluster 2023-10-09T09:29:35.679973+0000 mgr.y (mgr.34107) 698 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 718 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-09T09:29:35.920 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:35.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:35.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:36.024 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:35 smithi181 bash[59345]: audit 2023-10-09T09:29:35.237914+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.163:0/3437197491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:35 smithi181 bash[59345]: cluster 2023-10-09T09:29:35.679973+0000 mgr.y (mgr.34107) 698 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 718 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-09T09:29:36.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:36.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:36.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:36.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:36.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:35 smithi163 bash[75425]: audit 2023-10-09T09:29:35.237914+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.163:0/3437197491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:36.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:35 smithi163 bash[75425]: cluster 2023-10-09T09:29:35.679973+0000 mgr.y (mgr.34107) 698 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 718 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-09T09:29:36.199 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:36.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:36.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:36.305 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:36.338 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:36.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:36.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:36.444 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:36.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:36.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:36.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:36.582 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:36.607 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:36.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:36.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:36.710 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:36.735 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:36.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:36.752 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:29:36.753 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:36 smithi163 bash[75425]: audit 2023-10-09T09:29:36.093510+0000 mon.a (mon.0) 1499 : audit 1 from='client.? 172.21.15.163:0/30714171' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:29:36.839 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:36.864 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:36.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:36.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:36.969 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:36.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:36 smithi163 bash[78107]: audit 2023-10-09T09:29:36.093510+0000 mon.a (mon.0) 1499 : audit 1 from='client.? 172.21.15.163:0/30714171' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:29:37.000 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:37.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:37.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:37.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:36 smithi181 bash[59345]: audit 2023-10-09T09:29:36.093510+0000 mon.a (mon.0) 1499 : audit 1 from='client.? 172.21.15.163:0/30714171' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:29:37.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:37.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:37.152 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:37.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:37.253 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:37.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:37.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:37.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:37.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:37.426 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:37.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:37.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:37.532 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:37.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:37.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:37.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:37.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:37.695 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:37.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:37.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:37.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:37 smithi181 bash[59345]: cluster 2023-10-09T09:29:36.741231+0000 mon.a (mon.0) 1500 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:29:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:37 smithi181 bash[59345]: cluster 2023-10-09T09:29:36.744240+0000 osd.5 (osd.5) 640 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:37 smithi181 bash[59345]: audit 2023-10-09T09:29:36.747997+0000 mon.a (mon.0) 1501 : audit 1 from='client.? 172.21.15.163:0/30714171' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:29:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:37 smithi181 bash[59345]: cluster 2023-10-09T09:29:36.748167+0000 mon.a (mon.0) 1502 : cluster 0 osdmap e1096: 8 total, 8 up, 8 in 2023-10-09T09:29:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:37 smithi181 bash[59345]: cluster 2023-10-09T09:29:36.756175+0000 osd.4 (osd.4) 524 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:37 smithi181 bash[59345]: cluster 2023-10-09T09:29:36.769221+0000 osd.7 (osd.7) 487 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:37 smithi181 bash[59345]: cluster 2023-10-09T09:29:36.770590+0000 osd.7 (osd.7) 488 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:37 smithi181 bash[59345]: cluster 2023-10-09T09:29:37.680367+0000 mgr.y (mgr.34107) 699 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 580 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T09:29:37.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:37.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:37.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:37.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:37.961 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:37.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:37.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:38.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:38.094 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:38.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:38.198 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:38.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[75425]: cluster 2023-10-09T09:29:36.741231+0000 mon.a (mon.0) 1500 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:29:38.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[75425]: cluster 2023-10-09T09:29:36.744240+0000 osd.5 (osd.5) 640 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:38.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[75425]: audit 2023-10-09T09:29:36.747997+0000 mon.a (mon.0) 1501 : audit 1 from='client.? 172.21.15.163:0/30714171' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:29:38.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[75425]: cluster 2023-10-09T09:29:36.748167+0000 mon.a (mon.0) 1502 : cluster 0 osdmap e1096: 8 total, 8 up, 8 in 2023-10-09T09:29:38.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[75425]: cluster 2023-10-09T09:29:36.756175+0000 osd.4 (osd.4) 524 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:38.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[75425]: cluster 2023-10-09T09:29:36.769221+0000 osd.7 (osd.7) 487 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:38.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[75425]: cluster 2023-10-09T09:29:36.770590+0000 osd.7 (osd.7) 488 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:38.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[75425]: cluster 2023-10-09T09:29:37.680367+0000 mgr.y (mgr.34107) 699 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 580 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T09:29:38.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[78107]: cluster 2023-10-09T09:29:36.741231+0000 mon.a (mon.0) 1500 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:29:38.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[78107]: cluster 2023-10-09T09:29:36.744240+0000 osd.5 (osd.5) 640 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:38.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[78107]: audit 2023-10-09T09:29:36.747997+0000 mon.a (mon.0) 1501 : audit 1 from='client.? 172.21.15.163:0/30714171' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:29:38.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[78107]: cluster 2023-10-09T09:29:36.748167+0000 mon.a (mon.0) 1502 : cluster 0 osdmap e1096: 8 total, 8 up, 8 in 2023-10-09T09:29:38.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[78107]: cluster 2023-10-09T09:29:36.756175+0000 osd.4 (osd.4) 524 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:38.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[78107]: cluster 2023-10-09T09:29:36.769221+0000 osd.7 (osd.7) 487 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:38.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[78107]: cluster 2023-10-09T09:29:36.770590+0000 osd.7 (osd.7) 488 : cluster 3 failed to encode map e1096 with expected crc 2023-10-09T09:29:38.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:37 smithi163 bash[78107]: cluster 2023-10-09T09:29:37.680367+0000 mgr.y (mgr.34107) 699 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 580 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T09:29:38.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:38.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:38.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:38.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:38.354 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:38.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:38.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:38.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:38.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:38.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:38.510 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:29:38.602 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:38.633 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:38.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:38.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:38.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:38.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:38.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:38.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:38.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:38.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:38.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:38.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:39.008 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:39.049 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:39.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:39.154 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:39.179 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:39.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:39.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:39.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:39.315 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:39.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:39.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:39.420 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:39.447 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:39.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:39.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:39.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:39.586 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:39.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:39.688 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:39.715 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:39.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:39.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:39.815 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:29:39.818 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:39.822 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:29:39.846 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:39.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:39.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:39.951 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:39.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:39.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:39.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:40.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:40.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:40.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:40.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:40.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:40.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:40.258 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:29:40.352 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:40.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:40.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:40.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:40.486 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:40.514 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:40.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:40.619 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:40.676 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:40.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:40.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:40.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:40.808 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:40.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:40.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:40.913 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:40.922 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:40 smithi181 bash[59345]: cluster 2023-10-09T09:29:39.680865+0000 mgr.y (mgr.34107) 700 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 555 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T09:29:40.922 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:40 smithi181 bash[59345]: audit 2023-10-09T09:29:40.372309+0000 mon.a (mon.0) 1503 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:40.922 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:40 smithi181 bash[59345]: audit 2023-10-09T09:29:40.373770+0000 mon.a (mon.0) 1504 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:40.938 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:40.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:40.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:41.044 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:41.059 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:40 smithi163 bash[78107]: cluster 2023-10-09T09:29:39.680865+0000 mgr.y (mgr.34107) 700 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 555 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T09:29:41.059 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:40 smithi163 bash[78107]: audit 2023-10-09T09:29:40.372309+0000 mon.a (mon.0) 1503 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:41.059 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:40 smithi163 bash[78107]: audit 2023-10-09T09:29:40.373770+0000 mon.a (mon.0) 1504 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:41.060 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1402,"stamp":"2023-10-09T09:29:39.680570+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":582358968,"num_objects":8902,"num_object_clones":0,"num_object_copies":26706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8902,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":212898,"num_write_kb":7090894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139932,"ondisk_log_size":139932,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3583228,"kb_used_data":1859388,"kb_used_omap":416,"kb_used_meta":1723167,"kb_avail":746377988,"statfs":{"total":767960285184,"available":764291059712,"internally_reserved":0,"allocated":1904013312,"data_stored":1898172515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":1764523441},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,2,8,4,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-435814400,"num_objects":-6650,"num_object_clones":0,"num_object_copies":-19950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6650,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004610"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1062,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.003905+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:28:59.003905+0000","last_peered":"2023-10-09T09:28:59.003905+0000","last_clean":"2023-10-09T09:28:59.003905+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:28:59.003905+0000","last_undegraded":"2023-10-09T09:28:59.003905+0000","last_fullsized":"2023-10-09T09:28:59.003905+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1027,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.992275+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:28:58.992275+0000","last_peered":"2023-10-09T09:28:58.992275+0000","last_clean":"2023-10-09T09:28:58.992275+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:28:58.992275+0000","last_undegraded":"2023-10-09T09:28:58.992275+0000","last_fullsized":"2023-10-09T09:28:58.992275+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1032,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.550868+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:29:17.550868+0000","last_peered":"2023-10-09T09:29:17.550868+0000","last_clean":"2023-10-09T09:29:17.550868+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:29:17.550868+0000","last_undegraded":"2023-10-09T09:29:17.550868+0000","last_fullsized":"2023-10-09T09:29:17.550868+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1023,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.545180+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:29:17.545180+0000","last_peered":"2023-10-09T09:29:17.545180+0000","last_clean":"2023-10-09T09:29:17.545180+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:29:17.545180+0000","last_undegraded":"2023-10-09T09:29:17.545180+0000","last_fullsized":"2023-10-09T09:29:17.545180+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1008,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.997496+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:28:58.997496+0000","last_peered":"2023-10-09T09:28:58.997496+0000","last_clean":"2023-10-09T09:28:58.997496+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:28:58.997496+0000","last_undegraded":"2023-10-09T09:28:58.997496+0000","last_fullsized":"2023-10-09T09:28:58.997496+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1026,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.989907+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:28:58.989907+0000","last_peered":"2023-10-09T09:28:58.989907+0000","last_clean":"2023-10-09T09:28:58.989907+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:28:58.989907+0000","last_undegraded":"2023-10-09T09:28:58.989907+0000","last_fullsized":"2023-10-09T09:28:58.989907+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1023,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.546394+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:29:17.546394+0000","last_peered":"2023-10-09T09:29:17.546394+0000","last_clean":"2023-10-09T09:29:17.546394+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:29:17.546394+0000","last_undegraded":"2023-10-09T09:29:17.546394+0000","last_fullsized":"2023-10-09T09:29:17.546394+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":993,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.542537+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:29:17.542537+0000","last_peered":"2023-10-09T09:29:17.542537+0000","last_clean":"2023-10-09T09:29:17.542537+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:29:17.542537+0000","last_undegraded":"2023-10-09T09:29:17.542537+0000","last_fullsized":"2023-10-09T09:29:17.542537+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1047,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.982378+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:28:58.982378+0000","last_peered":"2023-10-09T09:28:58.982378+0000","last_clean":"2023-10-09T09:28:58.982378+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:28:58.982378+0000","last_undegraded":"2023-10-09T09:28:58.982378+0000","last_fullsized":"2023-10-09T09:28:58.982378+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1004,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.540013+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:29:17.540013+0000","last_peered":"2023-10-09T09:29:17.540013+0000","last_clean":"2023-10-09T09:29:17.540013+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:29:17.540013+0000","last_undegraded":"2023-10-09T09:29:17.540013+0000","last_fullsized":"2023-10-09T09:29:17.540013+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1026,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.996984+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:28:58.996984+0000","last_peered":"2023-10-09T09:28:58.996984+0000","last_clean":"2023-10-09T09:28:58.996984+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:28:58.996984+0000","last_undegraded":"2023-10-09T09:28:58.996984+0000","last_fullsized":"2023-10-09T09:28:58.996984+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1032,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.542749+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:29:17.542749+0000","last_peered":"2023-10-09T09:29:17.542749+0000","last_clean":"2023-10-09T09:29:17.542749+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:29:17.542749+0000","last_undegraded":"2023-10-09T09:29:17.542749+0000","last_fullsized":"2023-10-09T09:29:17.542749+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1053,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.004269+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:28:59.004269+0000","last_peered":"2023-10-09T09:28:59.004269+0000","last_clean":"2023-10-09T09:28:59.004269+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:28:59.004269+0000","last_undegraded":"2023-10-09T09:28:59.004269+0000","last_fullsized":"2023-10-09T09:28:59.004269+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1045,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.004086+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:28:59.004086+0000","last_peered":"2023-10-09T09:28:59.004086+0000","last_clean":"2023-10-09T09:28:59.004086+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:28:59.004086+0000","last_undegraded":"2023-10-09T09:28:59.004086+0000","last_fullsized":"2023-10-09T09:28:59.004086+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1025,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.996869+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:28:58.996869+0000","last_peered":"2023-10-09T09:28:58.996869+0000","last_clean":"2023-10-09T09:28:58.996869+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:28:58.996869+0000","last_undegraded":"2023-10-09T09:28:58.996869+0000","last_fullsized":"2023-10-09T09:28:58.996869+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":993,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.541985+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:29:17.541985+0000","last_peered":"2023-10-09T09:29:17.541985+0000","last_clean":"2023-10-09T09:29:17.541985+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:29:17.541985+0000","last_undegraded":"2023-10-09T09:29:17.541985+0000","last_fullsized":"2023-10-09T09:29:17.541985+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1023,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.544828+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:29:17.544828+0000","last_peered":"2023-10-09T09:29:17.544828+0000","last_clean":"2023-10-09T09:29:17.544828+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:29:17.544828+0000","last_undegraded":"2023-10-09T09:29:17.544828+0000","last_fullsized":"2023-10-09T09:29:17.544828+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1046,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.988126+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:28:58.988126+0000","last_peered":"2023-10-09T09:28:58.988126+0000","last_clean":"2023-10-09T09:28:58.988126+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:28:58.988126+0000","last_undegraded":"2023-10-09T09:28:58.988126+0000","last_fullsized":"2023-10-09T09:28:58.988126+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1008,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.999849+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:28:58.999849+0000","last_peered":"2023-10-09T09:28:58.999849+0000","last_clean":"2023-10-09T09:28:58.999849+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:28:58.999849+0000","last_undegraded":"2023-10-09T09:28:58.999849+0000","last_fullsized":"2023-10-09T09:28:58.999849+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1022,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.545099+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:29:17.545099+0000","last_peered":"2023-10-09T09:29:17.545099+0000","last_clean":"2023-10-09T09:29:17.545099+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:29:17.545099+0000","last_undegraded":"2023-10-09T09:29:17.545099+0000","last_fullsized":"2023-10-09T09:29:17.545099+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1036,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.989823+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:28:58.989823+0000","last_peered":"2023-10-09T09:28:58.989823+0000","last_clean":"2023-10-09T09:28:58.989823+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:28:58.989823+0000","last_undegraded":"2023-10-09T09:28:58.989823+0000","last_fullsized":"2023-10-09T09:28:58.989823+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1027,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.991891+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:28:58.991891+0000","last_peered":"2023-10-09T09:28:58.991891+0000","last_clean":"2023-10-09T09:28:58.991891+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:28:58.991891+0000","last_undegraded":"2023-10-09T09:28:58.991891+0000","last_fullsized":"2023-10-09T09:28:58.991891+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"1095'13516","reported_seq":13543,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:36.387960+0000","last_change":"2023-10-09T09:25:25.899629+0000","last_active":"2023-10-09T09:29:36.387960+0000","last_peered":"2023-10-09T09:29:36.387960+0000","last_clean":"2023-10-09T09:29:36.387960+0000","last_became_active":"2023-10-09T09:25:25.899245+0000","last_became_peered":"2023-10-09T09:25:25.899245+0000","last_unstale":"2023-10-09T09:29:36.387960+0000","last_undegraded":"2023-10-09T09:29:36.387960+0000","last_fullsized":"2023-10-09T09:29:36.387960+0000","mapping_epoch":1076,"log_start":"1083'3500","ondisk_log_start":"1083'3500","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10016,"log_dups_size":0,"ondisk_log_size":10016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39583744,"num_objects":604,"num_object_clones":0,"num_object_copies":1812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13516,"num_write_kb":451840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1024,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.997016+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:28:58.997016+0000","last_peered":"2023-10-09T09:28:58.997016+0000","last_clean":"2023-10-09T09:28:58.997016+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:28:58.997016+0000","last_undegraded":"2023-10-09T09:28:58.997016+0000","last_fullsized":"2023-10-09T09:28:58.997016+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1031,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.981939+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:28:58.981939+0000","last_peered":"2023-10-09T09:28:58.981939+0000","last_clean":"2023-10-09T09:28:58.981939+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:28:58.981939+0000","last_undegraded":"2023-10-09T09:28:58.981939+0000","last_fullsized":"2023-10-09T09:28:58.981939+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1095'13325","reported_seq":13354,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:35.129207+0000","last_change":"2023-10-09T09:25:25.902246+0000","last_active":"2023-10-09T09:29:35.129207+0000","last_peered":"2023-10-09T09:29:35.129207+0000","last_clean":"2023-10-09T09:29:35.129207+0000","last_became_active":"2023-10-09T09:25:25.901932+0000","last_became_peered":"2023-10-09T09:25:25.901932+0000","last_unstale":"2023-10-09T09:29:35.129207+0000","last_undegraded":"2023-10-09T09:29:35.129207+0000","last_fullsized":"2023-10-09T09:29:35.129207+0000","mapping_epoch":1076,"log_start":"1086'6000","ondisk_log_start":"1086'6000","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:10.795272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40042496,"num_objects":611,"num_object_clones":0,"num_object_copies":1833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13325,"num_write_kb":445952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1025,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.996780+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:28:58.996780+0000","last_peered":"2023-10-09T09:28:58.996780+0000","last_clean":"2023-10-09T09:28:58.996780+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:28:58.996780+0000","last_undegraded":"2023-10-09T09:28:58.996780+0000","last_fullsized":"2023-10-09T09:28:58.996780+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1060,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.003880+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:28:59.003880+0000","last_peered":"2023-10-09T09:28:59.003880+0000","last_clean":"2023-10-09T09:28:59.003880+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:28:59.003880+0000","last_undegraded":"2023-10-09T09:28:59.003880+0000","last_fullsized":"2023-10-09T09:28:59.003880+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1095'13431","reported_seq":13458,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:36.341247+0000","last_change":"2023-10-09T09:25:25.899849+0000","last_active":"2023-10-09T09:29:36.341247+0000","last_peered":"2023-10-09T09:29:36.341247+0000","last_clean":"2023-10-09T09:29:36.341247+0000","last_became_active":"2023-10-09T09:25:25.899350+0000","last_became_peered":"2023-10-09T09:25:25.899350+0000","last_unstale":"2023-10-09T09:29:36.341247+0000","last_undegraded":"2023-10-09T09:29:36.341247+0000","last_fullsized":"2023-10-09T09:29:36.341247+0000","mapping_epoch":1076,"log_start":"1085'5000","ondisk_log_start":"1085'5000","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8431,"log_dups_size":0,"ondisk_log_size":8431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36765696,"num_objects":561,"num_object_clones":0,"num_object_copies":1683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13431,"num_write_kb":447744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1009,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.997016+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:28:58.997016+0000","last_peered":"2023-10-09T09:28:58.997016+0000","last_clean":"2023-10-09T09:28:58.997016+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:28:58.997016+0000","last_undegraded":"2023-10-09T09:28:58.997016+0000","last_fullsized":"2023-10-09T09:28:58.997016+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1027,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.989852+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:28:58.989852+0000","last_peered":"2023-10-09T09:28:58.989852+0000","last_clean":"2023-10-09T09:28:58.989852+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:28:58.989852+0000","last_undegraded":"2023-10-09T09:28:58.989852+0000","last_fullsized":"2023-10-09T09:28:58.989852+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1096'12830","reported_seq":12860,"reported_epoch":1096,"state":"active+clean","last_fresh":"2023-10-09T09:29:37.180537+0000","last_change":"2023-10-09T09:25:25.899129+0000","last_active":"2023-10-09T09:29:37.180537+0000","last_peered":"2023-10-09T09:29:37.180537+0000","last_clean":"2023-10-09T09:29:37.180537+0000","last_became_active":"2023-10-09T09:25:25.898720+0000","last_became_peered":"2023-10-09T09:25:25.898720+0000","last_unstale":"2023-10-09T09:29:37.180537+0000","last_undegraded":"2023-10-09T09:29:37.180537+0000","last_fullsized":"2023-10-09T09:29:37.180537+0000","mapping_epoch":1076,"log_start":"1082'2800","ondisk_log_start":"1082'2800","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10030,"log_dups_size":0,"ondisk_log_size":10030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:06.994778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32636928,"num_objects":498,"num_object_clones":0,"num_object_copies":1494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12830,"num_write_kb":426496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1047,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.982209+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:28:58.982209+0000","last_peered":"2023-10-09T09:28:58.982209+0000","last_clean":"2023-10-09T09:28:58.982209+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:28:58.982209+0000","last_undegraded":"2023-10-09T09:28:58.982209+0000","last_fullsized":"2023-10-09T09:28:58.982209+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1022,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.545003+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:29:17.545003+0000","last_peered":"2023-10-09T09:29:17.545003+0000","last_clean":"2023-10-09T09:29:17.545003+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:29:17.545003+0000","last_undegraded":"2023-10-09T09:29:17.545003+0000","last_fullsized":"2023-10-09T09:29:17.545003+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1095'13317","reported_seq":13344,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:36.381447+0000","last_change":"2023-10-09T09:25:25.902960+0000","last_active":"2023-10-09T09:29:36.381447+0000","last_peered":"2023-10-09T09:29:36.381447+0000","last_clean":"2023-10-09T09:29:36.381447+0000","last_became_active":"2023-10-09T09:25:25.902534+0000","last_became_peered":"2023-10-09T09:25:25.902534+0000","last_unstale":"2023-10-09T09:29:36.381447+0000","last_undegraded":"2023-10-09T09:29:36.381447+0000","last_fullsized":"2023-10-09T09:29:36.381447+0000","mapping_epoch":1076,"log_start":"1085'5400","ondisk_log_start":"1085'5400","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7917,"log_dups_size":0,"ondisk_log_size":7917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36241408,"num_objects":553,"num_object_clones":0,"num_object_copies":1659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13317,"num_write_kb":443840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":1011,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:29:05.040945+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:29:05.040945+0000","last_peered":"2023-10-09T09:29:05.040945+0000","last_clean":"2023-10-09T09:29:05.040945+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:29:05.040945+0000","last_undegraded":"2023-10-09T09:29:05.040945+0000","last_fullsized":"2023-10-09T09:29:05.040945+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1045,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.004301+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:28:59.004301+0000","last_peered":"2023-10-09T09:28:59.004301+0000","last_clean":"2023-10-09T09:28:59.004301+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:28:59.004301+0000","last_undegraded":"2023-10-09T09:28:59.004301+0000","last_fullsized":"2023-10-09T09:28:59.004301+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1095'13064","reported_seq":13093,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:35.763022+0000","last_change":"2023-10-09T09:25:25.901392+0000","last_active":"2023-10-09T09:29:35.763022+0000","last_peered":"2023-10-09T09:29:35.763022+0000","last_clean":"2023-10-09T09:29:35.763022+0000","last_became_active":"2023-10-09T09:25:25.901057+0000","last_became_peered":"2023-10-09T09:25:25.901057+0000","last_unstale":"2023-10-09T09:29:35.763022+0000","last_undegraded":"2023-10-09T09:29:35.763022+0000","last_fullsized":"2023-10-09T09:29:35.763022+0000","mapping_epoch":1076,"log_start":"1083'3000","ondisk_log_start":"1083'3000","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10064,"log_dups_size":0,"ondisk_log_size":10064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:56.342418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38404096,"num_objects":586,"num_object_clones":0,"num_object_copies":1758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13064,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1028,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.990032+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:28:58.990032+0000","last_peered":"2023-10-09T09:28:58.990032+0000","last_clean":"2023-10-09T09:28:58.990032+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:28:58.990032+0000","last_undegraded":"2023-10-09T09:28:58.990032+0000","last_fullsized":"2023-10-09T09:28:58.990032+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1060,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.004399+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:28:59.004399+0000","last_peered":"2023-10-09T09:28:59.004399+0000","last_clean":"2023-10-09T09:28:59.004399+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:28:59.004399+0000","last_undegraded":"2023-10-09T09:28:59.004399+0000","last_fullsized":"2023-10-09T09:28:59.004399+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1095'13475","reported_seq":13504,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:35.075854+0000","last_change":"2023-10-09T09:25:25.896450+0000","last_active":"2023-10-09T09:29:35.075854+0000","last_peered":"2023-10-09T09:29:35.075854+0000","last_clean":"2023-10-09T09:29:35.075854+0000","last_became_active":"2023-10-09T09:25:25.896015+0000","last_became_peered":"2023-10-09T09:25:25.896015+0000","last_unstale":"2023-10-09T09:29:35.075854+0000","last_undegraded":"2023-10-09T09:29:35.075854+0000","last_fullsized":"2023-10-09T09:29:35.075854+0000","mapping_epoch":1076,"log_start":"1086'6100","ondisk_log_start":"1086'6100","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7375,"log_dups_size":0,"ondisk_log_size":7375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:32.630600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40697856,"num_objects":621,"num_object_clones":0,"num_object_copies":1863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13475,"num_write_kb":451072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1046,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.003740+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:28:59.003740+0000","last_peered":"2023-10-09T09:28:59.003740+0000","last_clean":"2023-10-09T09:28:59.003740+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:28:59.003740+0000","last_undegraded":"2023-10-09T09:28:59.003740+0000","last_fullsized":"2023-10-09T09:28:59.003740+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1045,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.009994+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:28:59.009994+0000","last_peered":"2023-10-09T09:28:59.009994+0000","last_clean":"2023-10-09T09:28:59.009994+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:28:59.009994+0000","last_undegraded":"2023-10-09T09:28:59.009994+0000","last_fullsized":"2023-10-09T09:28:59.009994+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1095'13284","reported_seq":13312,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:36.365864+0000","last_change":"2023-10-09T09:25:25.898838+0000","last_active":"2023-10-09T09:29:36.365864+0000","last_peered":"2023-10-09T09:29:36.365864+0000","last_clean":"2023-10-09T09:29:36.365864+0000","last_became_active":"2023-10-09T09:25:25.898426+0000","last_became_peered":"2023-10-09T09:25:25.898426+0000","last_unstale":"2023-10-09T09:29:36.365864+0000","last_undegraded":"2023-10-09T09:29:36.365864+0000","last_fullsized":"2023-10-09T09:29:36.365864+0000","mapping_epoch":1076,"log_start":"1083'3200","ondisk_log_start":"1083'3200","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10084,"log_dups_size":0,"ondisk_log_size":10084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37290008,"num_objects":570,"num_object_clones":0,"num_object_copies":1710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":570,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13284,"num_write_kb":443265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":994,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.997296+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:28:58.997296+0000","last_peered":"2023-10-09T09:28:58.997296+0000","last_clean":"2023-10-09T09:28:58.997296+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:28:58.997296+0000","last_undegraded":"2023-10-09T09:28:58.997296+0000","last_fullsized":"2023-10-09T09:28:58.997296+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1032,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.546860+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:29:17.546860+0000","last_peered":"2023-10-09T09:29:17.546860+0000","last_clean":"2023-10-09T09:29:17.546860+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:29:17.546860+0000","last_undegraded":"2023-10-09T09:29:17.546860+0000","last_fullsized":"2023-10-09T09:29:17.546860+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1096'13469","reported_seq":13500,"reported_epoch":1096,"state":"active+clean","last_fresh":"2023-10-09T09:29:37.194974+0000","last_change":"2023-10-09T09:25:25.899052+0000","last_active":"2023-10-09T09:29:37.194974+0000","last_peered":"2023-10-09T09:29:37.194974+0000","last_clean":"2023-10-09T09:29:37.194974+0000","last_became_active":"2023-10-09T09:25:25.898696+0000","last_became_peered":"2023-10-09T09:25:25.898696+0000","last_unstale":"2023-10-09T09:29:37.194974+0000","last_undegraded":"2023-10-09T09:29:37.194974+0000","last_fullsized":"2023-10-09T09:29:37.194974+0000","mapping_epoch":1076,"log_start":"1083'3400","ondisk_log_start":"1083'3400","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10069,"log_dups_size":0,"ondisk_log_size":10069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:34.741367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34930688,"num_objects":533,"num_object_clones":0,"num_object_copies":1599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13469,"num_write_kb":448064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1028,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.543648+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:29:17.543648+0000","last_peered":"2023-10-09T09:29:17.543648+0000","last_clean":"2023-10-09T09:29:17.543648+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:29:17.543648+0000","last_undegraded":"2023-10-09T09:29:17.543648+0000","last_fullsized":"2023-10-09T09:29:17.543648+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1062,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.988170+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:28:58.988170+0000","last_peered":"2023-10-09T09:28:58.988170+0000","last_clean":"2023-10-09T09:28:58.988170+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:28:58.988170+0000","last_undegraded":"2023-10-09T09:28:58.988170+0000","last_fullsized":"2023-10-09T09:28:58.988170+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1018,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.542643+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:29:17.542643+0000","last_peered":"2023-10-09T09:29:17.542643+0000","last_clean":"2023-10-09T09:29:17.542643+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:29:17.542643+0000","last_undegraded":"2023-10-09T09:29:17.542643+0000","last_fullsized":"2023-10-09T09:29:17.542643+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1095'13501","reported_seq":13530,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:35.130796+0000","last_change":"2023-10-09T09:25:25.901310+0000","last_active":"2023-10-09T09:29:35.130796+0000","last_peered":"2023-10-09T09:29:35.130796+0000","last_clean":"2023-10-09T09:29:35.130796+0000","last_became_active":"2023-10-09T09:25:25.900980+0000","last_became_peered":"2023-10-09T09:25:25.900980+0000","last_unstale":"2023-10-09T09:29:35.130796+0000","last_undegraded":"2023-10-09T09:29:35.130796+0000","last_fullsized":"2023-10-09T09:29:35.130796+0000","mapping_epoch":1076,"log_start":"1086'6100","ondisk_log_start":"1086'6100","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7401,"log_dups_size":0,"ondisk_log_size":7401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:10.107258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39518208,"num_objects":603,"num_object_clones":0,"num_object_copies":1809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13501,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1051,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.982715+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:28:58.982715+0000","last_peered":"2023-10-09T09:28:58.982715+0000","last_clean":"2023-10-09T09:28:58.982715+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:28:58.982715+0000","last_undegraded":"2023-10-09T09:28:58.982715+0000","last_fullsized":"2023-10-09T09:28:58.982715+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1046,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.003901+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:28:59.003901+0000","last_peered":"2023-10-09T09:28:59.003901+0000","last_clean":"2023-10-09T09:28:59.003901+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:28:59.003901+0000","last_undegraded":"2023-10-09T09:28:59.003901+0000","last_fullsized":"2023-10-09T09:28:59.003901+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1048,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.542715+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:29:17.542715+0000","last_peered":"2023-10-09T09:29:17.542715+0000","last_clean":"2023-10-09T09:29:17.542715+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:29:17.542715+0000","last_undegraded":"2023-10-09T09:29:17.542715+0000","last_fullsized":"2023-10-09T09:29:17.542715+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"1095'13378","reported_seq":13405,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:36.320213+0000","last_change":"2023-10-09T09:25:25.903443+0000","last_active":"2023-10-09T09:29:36.320213+0000","last_peered":"2023-10-09T09:29:36.320213+0000","last_clean":"2023-10-09T09:29:36.320213+0000","last_became_active":"2023-10-09T09:25:25.903086+0000","last_became_peered":"2023-10-09T09:25:25.903086+0000","last_unstale":"2023-10-09T09:29:36.320213+0000","last_undegraded":"2023-10-09T09:29:36.320213+0000","last_fullsized":"2023-10-09T09:29:36.320213+0000","mapping_epoch":1076,"log_start":"1085'5000","ondisk_log_start":"1085'5000","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8378,"log_dups_size":0,"ondisk_log_size":8378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36700160,"num_objects":560,"num_object_clones":0,"num_object_copies":1680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13378,"num_write_kb":446016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1013,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.992031+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:28:58.992031+0000","last_peered":"2023-10-09T09:28:58.992031+0000","last_clean":"2023-10-09T09:28:58.992031+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:28:58.992031+0000","last_undegraded":"2023-10-09T09:28:58.992031+0000","last_fullsized":"2023-10-09T09:28:58.992031+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1033,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.542373+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:29:17.542373+0000","last_peered":"2023-10-09T09:29:17.542373+0000","last_clean":"2023-10-09T09:29:17.542373+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:29:17.542373+0000","last_undegraded":"2023-10-09T09:29:17.542373+0000","last_fullsized":"2023-10-09T09:29:17.542373+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1022,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.546966+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:29:17.546966+0000","last_peered":"2023-10-09T09:29:17.546966+0000","last_clean":"2023-10-09T09:29:17.546966+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:29:17.546966+0000","last_undegraded":"2023-10-09T09:29:17.546966+0000","last_fullsized":"2023-10-09T09:29:17.546966+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1095'13028","reported_seq":13054,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:36.389760+0000","last_change":"2023-10-09T09:25:25.900270+0000","last_active":"2023-10-09T09:29:36.389760+0000","last_peered":"2023-10-09T09:29:36.389760+0000","last_clean":"2023-10-09T09:29:36.389760+0000","last_became_active":"2023-10-09T09:25:25.899813+0000","last_became_peered":"2023-10-09T09:25:25.899813+0000","last_unstale":"2023-10-09T09:29:36.389760+0000","last_undegraded":"2023-10-09T09:29:36.389760+0000","last_fullsized":"2023-10-09T09:29:36.389760+0000","mapping_epoch":1076,"log_start":"1083'3000","ondisk_log_start":"1083'3000","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":10028,"log_dups_size":0,"ondisk_log_size":10028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36438016,"num_objects":556,"num_object_clones":0,"num_object_copies":1668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13028,"num_write_kb":434688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1051,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.982128+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:28:58.982128+0000","last_peered":"2023-10-09T09:28:58.982128+0000","last_clean":"2023-10-09T09:28:58.982128+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:28:58.982128+0000","last_undegraded":"2023-10-09T09:28:58.982128+0000","last_fullsized":"2023-10-09T09:28:58.982128+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1259,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.996443+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:28:58.996443+0000","last_peered":"2023-10-09T09:28:58.996443+0000","last_clean":"2023-10-09T09:28:58.996443+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:28:58.996443+0000","last_undegraded":"2023-10-09T09:28:58.996443+0000","last_fullsized":"2023-10-09T09:28:58.996443+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1005,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.998584+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:28:58.998584+0000","last_peered":"2023-10-09T09:28:58.998584+0000","last_clean":"2023-10-09T09:28:58.998584+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:28:58.998584+0000","last_undegraded":"2023-10-09T09:28:58.998584+0000","last_fullsized":"2023-10-09T09:28:58.998584+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1012,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.990135+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:28:58.990135+0000","last_peered":"2023-10-09T09:28:58.990135+0000","last_clean":"2023-10-09T09:28:58.990135+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:28:58.990135+0000","last_undegraded":"2023-10-09T09:28:58.990135+0000","last_fullsized":"2023-10-09T09:28:58.990135+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1096'13291","reported_seq":13321,"reported_epoch":1096,"state":"active+clean","last_fresh":"2023-10-09T09:29:38.452497+0000","last_change":"2023-10-09T09:25:25.897749+0000","last_active":"2023-10-09T09:29:38.452497+0000","last_peered":"2023-10-09T09:29:38.452497+0000","last_clean":"2023-10-09T09:29:38.452497+0000","last_became_active":"2023-10-09T09:25:25.897366+0000","last_became_peered":"2023-10-09T09:25:25.897366+0000","last_unstale":"2023-10-09T09:29:38.452497+0000","last_undegraded":"2023-10-09T09:29:38.452497+0000","last_fullsized":"2023-10-09T09:29:38.452497+0000","mapping_epoch":1076,"log_start":"1085'5100","ondisk_log_start":"1085'5100","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8191,"log_dups_size":0,"ondisk_log_size":8191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:21.152296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27983872,"num_objects":427,"num_object_clones":0,"num_object_copies":1281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13291,"num_write_kb":438976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1042,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.996433+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:28:58.996433+0000","last_peered":"2023-10-09T09:28:58.996433+0000","last_clean":"2023-10-09T09:28:58.996433+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:28:58.996433+0000","last_undegraded":"2023-10-09T09:28:58.996433+0000","last_fullsized":"2023-10-09T09:28:58.996433+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1024,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.544653+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:29:17.544653+0000","last_peered":"2023-10-09T09:29:17.544653+0000","last_clean":"2023-10-09T09:29:17.544653+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:29:17.544653+0000","last_undegraded":"2023-10-09T09:29:17.544653+0000","last_fullsized":"2023-10-09T09:29:17.544653+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1035,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:22.186233+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:29:22.186233+0000","last_peered":"2023-10-09T09:29:22.186233+0000","last_clean":"2023-10-09T09:29:22.186233+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:29:22.186233+0000","last_undegraded":"2023-10-09T09:29:22.186233+0000","last_fullsized":"2023-10-09T09:29:22.186233+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1096'13278","reported_seq":13309,"reported_epoch":1096,"state":"active+clean","last_fresh":"2023-10-09T09:29:38.465006+0000","last_change":"2023-10-09T09:25:25.895292+0000","last_active":"2023-10-09T09:29:38.465006+0000","last_peered":"2023-10-09T09:29:38.465006+0000","last_clean":"2023-10-09T09:29:38.465006+0000","last_became_active":"2023-10-09T09:25:25.894970+0000","last_became_peered":"2023-10-09T09:25:25.894970+0000","last_unstale":"2023-10-09T09:29:38.465006+0000","last_undegraded":"2023-10-09T09:29:38.465006+0000","last_fullsized":"2023-10-09T09:29:38.465006+0000","mapping_epoch":1076,"log_start":"1085'5100","ondisk_log_start":"1085'5100","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8178,"log_dups_size":0,"ondisk_log_size":8178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:21.241118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30277632,"num_objects":462,"num_object_clones":0,"num_object_copies":1386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13278,"num_write_kb":439680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1027,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.996888+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:28:58.996888+0000","last_peered":"2023-10-09T09:28:58.996888+0000","last_clean":"2023-10-09T09:28:58.996888+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:28:58.996888+0000","last_undegraded":"2023-10-09T09:28:58.996888+0000","last_fullsized":"2023-10-09T09:28:58.996888+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1034,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.543616+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:29:17.543616+0000","last_peered":"2023-10-09T09:29:17.543616+0000","last_clean":"2023-10-09T09:29:17.543616+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:29:17.543616+0000","last_undegraded":"2023-10-09T09:29:17.543616+0000","last_fullsized":"2023-10-09T09:29:17.543616+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1008,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.545182+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:29:17.545182+0000","last_peered":"2023-10-09T09:29:17.545182+0000","last_clean":"2023-10-09T09:29:17.545182+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:29:17.545182+0000","last_undegraded":"2023-10-09T09:29:17.545182+0000","last_fullsized":"2023-10-09T09:29:17.545182+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1095'13404","reported_seq":13431,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:36.365273+0000","last_change":"2023-10-09T09:25:25.903038+0000","last_active":"2023-10-09T09:29:36.365273+0000","last_peered":"2023-10-09T09:29:36.365273+0000","last_clean":"2023-10-09T09:29:36.365273+0000","last_became_active":"2023-10-09T09:25:25.902503+0000","last_became_peered":"2023-10-09T09:25:25.902503+0000","last_unstale":"2023-10-09T09:29:36.365273+0000","last_undegraded":"2023-10-09T09:29:36.365273+0000","last_fullsized":"2023-10-09T09:29:36.365273+0000","mapping_epoch":1076,"log_start":"1085'5500","ondisk_log_start":"1085'5500","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":7904,"log_dups_size":0,"ondisk_log_size":7904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37617664,"num_objects":574,"num_object_clones":0,"num_object_copies":1722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13404,"num_write_kb":447296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":1011,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.991978+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:28:58.991978+0000","last_peered":"2023-10-09T09:28:58.991978+0000","last_clean":"2023-10-09T09:28:58.991978+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:28:58.991978+0000","last_undegraded":"2023-10-09T09:28:58.991978+0000","last_fullsized":"2023-10-09T09:28:58.991978+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1016,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.543332+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:29:17.543332+0000","last_peered":"2023-10-09T09:29:17.543332+0000","last_clean":"2023-10-09T09:29:17.543332+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:29:17.543332+0000","last_undegraded":"2023-10-09T09:29:17.543332+0000","last_fullsized":"2023-10-09T09:29:17.543332+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1009,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.541743+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:29:17.541743+0000","last_peered":"2023-10-09T09:29:17.541743+0000","last_clean":"2023-10-09T09:29:17.541743+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:29:17.541743+0000","last_undegraded":"2023-10-09T09:29:17.541743+0000","last_fullsized":"2023-10-09T09:29:17.541743+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1095'13047","reported_seq":13074,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:36.315587+0000","last_change":"2023-10-09T09:25:25.902231+0000","last_active":"2023-10-09T09:29:36.315587+0000","last_peered":"2023-10-09T09:29:36.315587+0000","last_clean":"2023-10-09T09:29:36.315587+0000","last_became_active":"2023-10-09T09:25:25.901859+0000","last_became_peered":"2023-10-09T09:25:25.901859+0000","last_unstale":"2023-10-09T09:29:36.315587+0000","last_undegraded":"2023-10-09T09:29:36.315587+0000","last_fullsized":"2023-10-09T09:29:36.315587+0000","mapping_epoch":1076,"log_start":"1085'4700","ondisk_log_start":"1085'4700","created":1076,"last_epoch_clean":1077,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:25:24.877072+0000","last_clean_scrub_stamp":"2023-10-09T09:25:24.877072+0000","objects_scrubbed":0,"log_size":8347,"log_dups_size":0,"ondisk_log_size":8347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36634624,"num_objects":559,"num_object_clones":0,"num_object_copies":1677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13047,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1031,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.989894+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:28:58.989894+0000","last_peered":"2023-10-09T09:28:58.989894+0000","last_clean":"2023-10-09T09:28:58.989894+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:28:58.989894+0000","last_undegraded":"2023-10-09T09:28:58.989894+0000","last_fullsized":"2023-10-09T09:28:58.989894+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1027,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:22.186275+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:29:22.186275+0000","last_peered":"2023-10-09T09:29:22.186275+0000","last_clean":"2023-10-09T09:29:22.186275+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:29:22.186275+0000","last_undegraded":"2023-10-09T09:29:22.186275+0000","last_fullsized":"2023-10-09T09:29:22.186275+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1030,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.003759+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:28:59.003759+0000","last_peered":"2023-10-09T09:28:59.003759+0000","last_clean":"2023-10-09T09:28:59.003759+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:28:59.003759+0000","last_undegraded":"2023-10-09T09:28:59.003759+0000","last_fullsized":"2023-10-09T09:28:59.003759+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1011,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.545100+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:29:17.545100+0000","last_peered":"2023-10-09T09:29:17.545100+0000","last_clean":"2023-10-09T09:29:17.545100+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:29:17.545100+0000","last_undegraded":"2023-10-09T09:29:17.545100+0000","last_fullsized":"2023-10-09T09:29:17.545100+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1031,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.990458+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:28:58.990458+0000","last_peered":"2023-10-09T09:28:58.990458+0000","last_clean":"2023-10-09T09:28:58.990458+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:28:58.990458+0000","last_undegraded":"2023-10-09T09:28:58.990458+0000","last_fullsized":"2023-10-09T09:28:58.990458+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1049,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.982386+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:28:58.982386+0000","last_peered":"2023-10-09T09:28:58.982386+0000","last_clean":"2023-10-09T09:28:58.982386+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:28:58.982386+0000","last_undegraded":"2023-10-09T09:28:58.982386+0000","last_fullsized":"2023-10-09T09:28:58.982386+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1060,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.004171+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:28:59.004171+0000","last_peered":"2023-10-09T09:28:59.004171+0000","last_clean":"2023-10-09T09:28:59.004171+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:28:59.004171+0000","last_undegraded":"2023-10-09T09:28:59.004171+0000","last_fullsized":"2023-10-09T09:28:59.004171+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1034,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.988193+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:28:58.988193+0000","last_peered":"2023-10-09T09:28:58.988193+0000","last_clean":"2023-10-09T09:28:58.988193+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:28:58.988193+0000","last_undegraded":"2023-10-09T09:28:58.988193+0000","last_fullsized":"2023-10-09T09:28:58.988193+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1060,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.004050+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:28:59.004050+0000","last_peered":"2023-10-09T09:28:59.004050+0000","last_clean":"2023-10-09T09:28:59.004050+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:28:59.004050+0000","last_undegraded":"2023-10-09T09:28:59.004050+0000","last_fullsized":"2023-10-09T09:28:59.004050+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":1022,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.541584+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:29:17.541584+0000","last_peered":"2023-10-09T09:29:17.541584+0000","last_clean":"2023-10-09T09:29:17.541584+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:29:17.541584+0000","last_undegraded":"2023-10-09T09:29:17.541584+0000","last_fullsized":"2023-10-09T09:29:17.541584+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1033,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:58.991455+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:28:58.991455+0000","last_peered":"2023-10-09T09:28:58.991455+0000","last_clean":"2023-10-09T09:28:58.991455+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:28:58.991455+0000","last_undegraded":"2023-10-09T09:28:58.991455+0000","last_fullsized":"2023-10-09T09:28:58.991455+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1030,"reported_epoch":1095,"state":"active+clean","last_fresh":"2023-10-09T09:29:17.545489+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:29:17.545489+0000","last_peered":"2023-10-09T09:29:17.545489+0000","last_clean":"2023-10-09T09:29:17.545489+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:29:17.545489+0000","last_undegraded":"2023-10-09T09:29:17.545489+0000","last_fullsized":"2023-10-09T09:29:17.545489+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1045,"reported_epoch":1094,"state":"active+clean","last_fresh":"2023-10-09T09:28:59.003702+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:28:59.003702+0000","last_peered":"2023-10-09T09:28:59.003702+0000","last_clean":"2023-10-09T09:28:59.003702+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:28:59.003702+0000","last_undegraded":"2023-10-09T09:28:59.003702+0000","last_fullsized":"2023-10-09T09:28:59.003702+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":581763096,"num_objects":8878,"num_object_clones":0,"num_object_copies":26634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8878,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":212638,"num_write_kb":7088449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1761095680,"data_stored":1761083464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139738,"ondisk_log_size":139738,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397854,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533924,"kb_used_data":224412,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93211228,"statfs":{"total":95995035648,"available":95448297472,"internally_reserved":0,"allocated":229797888,"data_stored":229491681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":2.677,"5min":2.3039999999999998,"15min":1.4630000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.5459999999999994,"5min":55.296999999999997,"15min":55.296999999999997},"last":0.59799999999999998},{"interface":"front","average":{"1min":2.7669999999999999,"5min":2.4129999999999998,"15min":1.5680000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.5790000000000006,"5min":55.369999999999997,"15min":55.369999999999997},"last":0.69299999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:29:15 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":2.5510000000000002,"15min":1.458},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.2449999999999992,"5min":103.84699999999999,"15min":103.84699999999999},"last":7.1159999999999997},{"interface":"front","average":{"1min":1.6819999999999999,"5min":2.6949999999999998,"15min":1.569},"min":{"1min":0.29899999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.6029999999999998,"5min":103.955,"15min":103.955},"last":3.3839999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":2.1240000000000001,"15min":1.468},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.1950000000000003,"5min":17.638000000000002,"15min":17.638000000000002},"last":1.5940000000000001},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.871,"15min":1.3100000000000001},"min":{"1min":0.39600000000000002,"5min":0.373,"15min":0.373},"max":{"1min":9.2070000000000007,"5min":17.600000000000001,"15min":17.600000000000001},"last":7.0739999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4830000000000001,"5min":3.847,"15min":1.952},"min":{"1min":0.34100000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.4989999999999997,"5min":107.97499999999999,"15min":107.97499999999999},"last":1.528},{"interface":"front","average":{"1min":2.383,"5min":3.8700000000000001,"15min":2.0609999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.3920000000000003,"5min":105.282,"15min":105.282},"last":1.4630000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.1160000000000001,"15min":1.0069999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.9500000000000002,"5min":6.1550000000000002,"15min":6.1550000000000002},"last":0.878},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.1639999999999999,"15min":1.054},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":12.316000000000001,"5min":12.316000000000001,"15min":12.316000000000001},"last":0.54900000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":1.4730000000000001,"15min":1.2529999999999999},"min":{"1min":0.38900000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":6.4500000000000002,"5min":10.063000000000001,"15min":10.063000000000001},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.069,"5min":1.1799999999999999,"15min":1.1040000000000001},"min":{"1min":0.46100000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.0499999999999998,"5min":6.4359999999999999,"15min":6.4359999999999999},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.21,"15min":1.1279999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.5710000000000002,"5min":6.1219999999999999,"15min":6.1219999999999999},"last":0.91700000000000004},{"interface":"front","average":{"1min":1.925,"5min":1.4530000000000001,"15min":1.2629999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":8.1980000000000004,"5min":8.1980000000000004,"15min":8.1980000000000004},"last":0.752}]}]},{"osd":6,"up_from":43,"seq":184683594087,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":496536,"kb_used_data":199312,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93248616,"statfs":{"total":95995035648,"available":95486582784,"internally_reserved":0,"allocated":204095488,"data_stored":203801423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":1.198,"15min":1.02},"min":{"1min":0.30399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":5.9089999999999998,"5min":5.9089999999999998,"15min":5.9089999999999998},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.081,"15min":1.0269999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.1829999999999998,"5min":7.1829999999999998,"15min":7.1829999999999998},"last":0.38900000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:29:11 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":1.857,"15min":1.258},"min":{"1min":0.30499999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":32.959000000000003,"5min":32.959000000000003,"15min":32.959000000000003},"last":0.33400000000000002},{"interface":"front","average":{"1min":3.4590000000000001,"5min":1.716,"15min":1.1950000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":34.793999999999997,"5min":34.793999999999997,"15min":34.793999999999997},"last":5.3940000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.7310000000000001,"5min":1.224,"15min":1.0389999999999999},"min":{"1min":0.32800000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.1459999999999999,"5min":7.1459999999999999,"15min":7.1459999999999999},"last":5.0330000000000004},{"interface":"front","average":{"1min":2.1579999999999999,"5min":1.5369999999999999,"15min":1.1970000000000001},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":10.409000000000001,"5min":10.409000000000001,"15min":10.409000000000001},"last":0.35199999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:28:53 2023","interfaces":[{"interface":"back","average":{"1min":3.7549999999999999,"5min":1.8140000000000001,"15min":1.286},"min":{"1min":0.38300000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":33.194000000000003,"5min":33.194000000000003,"15min":33.194000000000003},"last":0.45200000000000001},{"interface":"front","average":{"1min":3.0529999999999999,"5min":1.798,"15min":1.3200000000000001},"min":{"1min":0.42199999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":33.280000000000001,"5min":33.280000000000001,"15min":33.280000000000001},"last":5.4349999999999996}]},{"osd":4,"last update":"Mon Oct 9 09:28:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.1819999999999999,"15min":1.044},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":5.4039999999999999,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.83699999999999997,"5min":1.1259999999999999,"15min":1.085},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.2480000000000002,"5min":3.5569999999999999,"15min":3.5569999999999999},"last":5.4820000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:28:53 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":1.151,"15min":1.113},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":6.9800000000000004,"5min":6.9800000000000004,"15min":6.9800000000000004},"last":1.5680000000000001},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.137,"15min":1.1459999999999999},"min":{"1min":0.38300000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.6190000000000002,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":5.46}]},{"osd":7,"last update":"Mon Oct 9 09:29:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":1.5669999999999999,"15min":1.3100000000000001},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":6.6660000000000004,"5min":7.3630000000000004,"15min":7.3630000000000004},"last":5.5049999999999999},{"interface":"front","average":{"1min":1.5980000000000001,"5min":1.5249999999999999,"15min":1.3009999999999999},"min":{"1min":0.40600000000000003,"5min":0.371,"15min":0.371},"max":{"1min":4.9550000000000001,"5min":7.6459999999999999,"15min":7.6459999999999999},"last":4.5999999999999996}]}]},{"osd":2,"up_from":634,"seq":2723009265840,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":466684,"kb_used_data":325364,"kb_used_omap":122,"kb_used_meta":141061,"kb_avail":93278468,"statfs":{"total":95995035648,"available":95517151232,"internally_reserved":0,"allocated":333172736,"data_stored":331975861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":144447051},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:43 2023","interfaces":[{"interface":"back","average":{"1min":2.8050000000000002,"5min":1.571,"15min":1.0269999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":6.3399999999999999,"5min":8.9860000000000007,"15min":8.9860000000000007},"last":5.952},{"interface":"front","average":{"1min":2.2149999999999999,"5min":1.5609999999999999,"15min":1},"min":{"1min":0.245,"5min":0.223,"15min":0.223},"max":{"1min":8.8879999999999999,"5min":8.8879999999999999,"15min":8.8879999999999999},"last":8.2850000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:28:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.3149999999999999,"15min":0.92700000000000005},"min":{"1min":0.23300000000000001,"5min":0.184,"15min":0.184},"max":{"1min":5.899,"5min":6.798,"15min":6.798},"last":1.335},{"interface":"front","average":{"1min":3.0339999999999998,"5min":1.8200000000000001,"15min":1.1379999999999999},"min":{"1min":0.48599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":7.5149999999999997,"5min":8.702,"15min":8.702},"last":1.6539999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.736,"15min":1.1220000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.8090000000000002,"5min":11.597,"15min":11.597},"last":0.98999999999999999},{"interface":"front","average":{"1min":2.9660000000000002,"5min":1.823,"15min":1.1990000000000001},"min":{"1min":0.439,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.861000000000001,"5min":12.861000000000001,"15min":12.861000000000001},"last":0.82099999999999995}]},{"osd":4,"last update":"Mon Oct 9 09:28:43 2023","interfaces":[{"interface":"back","average":{"1min":2.786,"5min":1.5129999999999999,"15min":1.2110000000000001},"min":{"1min":0.55900000000000005,"5min":0.309,"15min":0.309},"max":{"1min":7.452,"5min":7.452,"15min":7.452},"last":0.93200000000000005},{"interface":"front","average":{"1min":1.978,"5min":1.48,"15min":1.1819999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.917999999999999,"5min":13.917999999999999,"15min":13.917999999999999},"last":3.3919999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:28:43 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":1.214,"15min":1.111},"min":{"1min":0.41799999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.8780000000000001,"5min":6.7699999999999996,"15min":6.7699999999999996},"last":1.3029999999999999},{"interface":"front","average":{"1min":2.798,"5min":1.5589999999999999,"15min":1.2849999999999999},"min":{"1min":0.53300000000000003,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":10.393000000000001,"5min":10.393000000000001,"15min":10.393000000000001},"last":0.95399999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:28:43 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.546,"15min":1.2989999999999999},"min":{"1min":0.59199999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.9470000000000001,"5min":7.9470000000000001,"15min":7.9470000000000001},"last":0.90500000000000003},{"interface":"front","average":{"1min":2.6349999999999998,"5min":1.6479999999999999,"15min":1.304},"min":{"1min":0.35499999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.893000000000001,"5min":13.893000000000001,"15min":13.893000000000001},"last":3.4220000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:28:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":1.7070000000000001,"15min":1.3560000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":13.848000000000001,"5min":13.848000000000001,"15min":13.848000000000001},"last":3.3519999999999999},{"interface":"front","average":{"1min":1.403,"5min":1.3899999999999999,"15min":1.244},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.8470000000000004,"5min":6.7350000000000003,"15min":6.7350000000000003},"last":0.96599999999999997}]}]},{"osd":1,"up_from":607,"seq":2607045148854,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":265272,"kb_used_data":164272,"kb_used_omap":107,"kb_used_meta":100884,"kb_avail":93479880,"statfs":{"total":95995035648,"available":95723397120,"internally_reserved":0,"allocated":168214528,"data_stored":167026599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":103305899},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:29:07 2023","interfaces":[{"interface":"back","average":{"1min":6.048,"5min":1.891,"15min":1.0800000000000001},"min":{"1min":0.28100000000000003,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":84.421000000000006,"5min":84.421000000000006,"15min":84.421000000000006},"last":0.371},{"interface":"front","average":{"1min":5.7990000000000004,"5min":2.153,"15min":1.2},"min":{"1min":0.223,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":84.489999999999995,"5min":84.489999999999995,"15min":84.489999999999995},"last":0.34899999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:29:35 2023","interfaces":[{"interface":"back","average":{"1min":7.1349999999999998,"5min":3.1200000000000001,"15min":1.532},"min":{"1min":0.23899999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":84.578999999999994,"5min":84.578999999999994,"15min":84.578999999999994},"last":0.42799999999999999},{"interface":"front","average":{"1min":7.931,"5min":2.742,"15min":1.466},"min":{"1min":0.33600000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":84.697000000000003,"5min":84.697000000000003,"15min":84.697000000000003},"last":0.50600000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:28:55 2023","interfaces":[{"interface":"back","average":{"1min":10.278,"5min":4.8689999999999998,"15min":2.177},"min":{"1min":0.42299999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":86.707999999999998,"5min":107.426,"15min":107.426},"last":0.83399999999999996},{"interface":"front","average":{"1min":11.542,"5min":5.3819999999999997,"15min":2.3879999999999999},"min":{"1min":0.39700000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":86.847999999999999,"5min":106.15900000000001,"15min":106.15900000000001},"last":1.8320000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:29:07 2023","interfaces":[{"interface":"back","average":{"1min":6.4000000000000004,"5min":2.1099999999999999,"15min":1.395},"min":{"1min":0.41699999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":84.875,"5min":84.875,"15min":84.875},"last":0.78000000000000003},{"interface":"front","average":{"1min":5.8959999999999999,"5min":2.2559999999999998,"15min":1.4470000000000001},"min":{"1min":0.34599999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":84.992999999999995,"5min":84.992999999999995,"15min":84.992999999999995},"last":0.60899999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:29:07 2023","interfaces":[{"interface":"back","average":{"1min":6.1639999999999997,"5min":2.1739999999999999,"15min":1.4530000000000001},"min":{"1min":0.439,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":84.805000000000007,"5min":84.805000000000007,"15min":84.805000000000007},"last":0.66500000000000004},{"interface":"front","average":{"1min":6.3419999999999996,"5min":2.347,"15min":1.5029999999999999},"min":{"1min":0.505,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":84.951999999999998,"5min":84.951999999999998,"15min":84.951999999999998},"last":0.63300000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:29:07 2023","interfaces":[{"interface":"back","average":{"1min":6.8680000000000003,"5min":2.5649999999999999,"15min":1.5720000000000001},"min":{"1min":0.40300000000000002,"5min":0.379,"15min":0.379},"max":{"1min":85.016000000000005,"5min":85.016000000000005,"15min":85.016000000000005},"last":0.68899999999999995},{"interface":"front","average":{"1min":5.9690000000000003,"5min":2.1779999999999999,"15min":1.4810000000000001},"min":{"1min":0.45300000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":84.831999999999994,"5min":84.831999999999994,"15min":84.831999999999994},"last":3.5619999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:29:07 2023","interfaces":[{"interface":"back","average":{"1min":8.266,"5min":3.008,"15min":1.778},"min":{"1min":0.48899999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":84.888000000000005,"5min":84.888000000000005,"15min":84.888000000000005},"last":0.72099999999999997},{"interface":"front","average":{"1min":6.9950000000000001,"5min":2.3479999999999999,"15min":1.5980000000000001},"min":{"1min":0.40500000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":84.742000000000004,"5min":84.742000000000004,"15min":84.742000000000004},"last":0.69899999999999995}]}]},{"osd":0,"up_from":581,"seq":2495375999163,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":258816,"kb_used_data":153272,"kb_used_omap":131,"kb_used_meta":105404,"kb_avail":93486336,"statfs":{"total":95995035648,"available":95730008064,"internally_reserved":0,"allocated":156950528,"data_stored":155749229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":107934297},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":1.4810000000000001,"15min":0.98199999999999998},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.9119999999999999,"5min":14.44,"15min":14.44},"last":3.5049999999999999},{"interface":"front","average":{"1min":1.7509999999999999,"5min":1.5640000000000001,"15min":1.0509999999999999},"min":{"1min":0.251,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.0190000000000001,"5min":14.663,"15min":14.663},"last":2.2810000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:28:38 2023","interfaces":[{"interface":"back","average":{"1min":5.585,"5min":2.2200000000000002,"15min":1.274},"min":{"1min":0.245,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":60.151000000000003,"5min":60.151000000000003,"15min":60.151000000000003},"last":7.3600000000000003},{"interface":"front","average":{"1min":4.7489999999999997,"5min":1.9350000000000001,"15min":1.167},"min":{"1min":0.311,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":59.253,"5min":59.253,"15min":59.253},"last":3.3370000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:29:03 2023","interfaces":[{"interface":"back","average":{"1min":2.3159999999999998,"5min":2.8620000000000001,"15min":1.502},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":7.7240000000000002,"5min":122.749,"15min":122.749},"last":7.4000000000000004},{"interface":"front","average":{"1min":3.0830000000000002,"5min":2.9540000000000002,"15min":1.6180000000000001},"min":{"1min":0.42399999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.608000000000001,"5min":123.51900000000001,"15min":123.51900000000001},"last":7.4240000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:28:53 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":1.296,"15min":1.107},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.312},"max":{"1min":6.5119999999999996,"5min":7.093,"15min":7.093},"last":3.4590000000000001},{"interface":"front","average":{"1min":1.849,"5min":1.4139999999999999,"15min":1.1479999999999999},"min":{"1min":0.34300000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.1909999999999998,"5min":7.3310000000000004,"15min":7.3310000000000004},"last":3.2749999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:28:53 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.296,"15min":1.1479999999999999},"min":{"1min":0.41599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0659999999999998,"5min":7.3620000000000001,"15min":7.3620000000000001},"last":3.3769999999999998},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.585,"15min":1.244},"min":{"1min":0.38600000000000001,"5min":0.312,"15min":0.312},"max":{"1min":8.1690000000000005,"5min":8.5609999999999999,"15min":8.5609999999999999},"last":2.3250000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:28:53 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.667,"15min":1.306},"min":{"1min":0.42399999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.8250000000000002,"5min":8.202,"15min":8.202},"last":7.4710000000000001},{"interface":"front","average":{"1min":1.923,"5min":1.4450000000000001,"15min":1.242},"min":{"1min":0.60399999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.6959999999999997,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":3.4279999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:28:53 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":1.6759999999999999,"15min":1.3520000000000001},"min":{"1min":0.42899999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.798,"5min":8.1769999999999996,"15min":8.1769999999999996},"last":7.4480000000000004},{"interface":"front","average":{"1min":1.897,"5min":1.5269999999999999,"15min":1.3120000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":7.532,"5min":8.6549999999999994,"15min":8.6549999999999994},"last":3.488}]}]},{"osd":3,"up_from":663,"seq":2847563317418,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":561632,"kb_used_data":399128,"kb_used_omap":55,"kb_used_meta":162376,"kb_avail":93183520,"statfs":{"total":95995035648,"available":95419924480,"internally_reserved":0,"allocated":408707072,"data_stored":407641852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":166273634},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,4,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.516,"15min":1.0329999999999999},"min":{"1min":0.21199999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.0620000000000003,"5min":7.9080000000000004,"15min":7.9080000000000004},"last":4.2949999999999999},{"interface":"front","average":{"1min":2.5910000000000002,"5min":1.698,"15min":1.0840000000000001},"min":{"1min":0.27600000000000002,"5min":0.252,"15min":0.252},"max":{"1min":9.0739999999999998,"5min":11.507999999999999,"15min":11.507999999999999},"last":4.1699999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":3.5670000000000002,"5min":1.9770000000000001,"15min":1.2230000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":14.206,"5min":14.206,"15min":14.206},"last":4.2560000000000002},{"interface":"front","average":{"1min":2.1150000000000002,"5min":1.873,"15min":1.1830000000000001},"min":{"1min":0.28699999999999998,"5min":0.249,"15min":0.249},"max":{"1min":6.1840000000000002,"5min":7.6509999999999998,"15min":7.6509999999999998},"last":6.8529999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.9750000000000001,"15min":1.226},"min":{"1min":0.42799999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.3710000000000004,"5min":7.867,"15min":7.867},"last":6.7270000000000003},{"interface":"front","average":{"1min":2.8980000000000001,"5min":2.0950000000000002,"15min":1.2829999999999999},"min":{"1min":0.32200000000000001,"5min":0.223,"15min":0.223},"max":{"1min":9.4749999999999996,"5min":13.856,"15min":13.856},"last":8.8559999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.698,"15min":1.2589999999999999},"min":{"1min":0.376,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.2850000000000001,"5min":12.629,"15min":12.629},"last":4.2069999999999999},{"interface":"front","average":{"1min":1.75,"5min":1.603,"15min":1.1990000000000001},"min":{"1min":0.318,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.7300000000000004,"5min":7.827,"15min":7.827},"last":5.8239999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":2.6309999999999998,"5min":1.9259999999999999,"15min":1.391},"min":{"1min":0.33100000000000002,"5min":0.315,"15min":0.315},"max":{"1min":11.272,"5min":11.272,"15min":11.272},"last":4.6429999999999998},{"interface":"front","average":{"1min":1.9450000000000001,"5min":1.8240000000000001,"15min":1.3640000000000001},"min":{"1min":0.39300000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.3409999999999993,"5min":11.388999999999999,"15min":11.388999999999999},"last":4.234}]},{"osd":6,"last update":"Mon Oct 9 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":1.8640000000000001,"15min":1.345},"min":{"1min":0.30499999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.2129999999999992,"5min":8.2129999999999992,"15min":8.2129999999999992},"last":3.3679999999999999},{"interface":"front","average":{"1min":2.242,"5min":1.9199999999999999,"15min":1.4199999999999999},"min":{"1min":0.442,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":8.3140000000000001,"5min":17.234999999999999,"15min":17.234999999999999},"last":4.3140000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:29:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3690000000000002,"5min":1.98,"15min":1.4590000000000001},"min":{"1min":0.44800000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.8220000000000001,"5min":7.8220000000000001,"15min":7.8220000000000001},"last":4.6779999999999999},{"interface":"front","average":{"1min":2.266,"5min":1.893,"15min":1.395},"min":{"1min":0.35599999999999998,"5min":0.318,"15min":0.318},"max":{"1min":5.3570000000000002,"5min":7.5869999999999997,"15min":7.5869999999999997},"last":0.45000000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986552,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":495644,"kb_used_data":194324,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93249508,"statfs":{"total":95995035648,"available":95487496192,"internally_reserved":0,"allocated":198987776,"data_stored":198689147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:41 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.0980000000000001,"15min":0.96199999999999997},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.4640000000000004,"5min":6.4640000000000004,"15min":6.4640000000000004},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.1160000000000001,"15min":1.002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.9710000000000001,"5min":7.5149999999999997,"15min":7.5149999999999997},"last":0.47599999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:29:12 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":1.4159999999999999,"15min":1.0720000000000001},"min":{"1min":0.32200000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.5119999999999996,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.351,"5min":1.3819999999999999,"15min":1.1220000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.6589999999999998,"5min":7.6589999999999998,"15min":7.6589999999999998},"last":0.35699999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:29:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.5820000000000001,"15min":1.175},"min":{"1min":0.41599999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.1580000000000004,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":1.5489999999999999},{"interface":"front","average":{"1min":1.5489999999999999,"5min":1.4039999999999999,"15min":1.103},"min":{"1min":0.34799999999999998,"5min":0.317,"15min":0.317},"max":{"1min":3.8769999999999998,"5min":10.773999999999999,"15min":10.773999999999999},"last":0.39600000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:29:08 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":1.992,"15min":1.3460000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.7240000000000002,"5min":17.376000000000001,"15min":17.376000000000001},"last":9.5670000000000002},{"interface":"front","average":{"1min":1.96,"5min":2.0129999999999999,"15min":1.4199999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.3849999999999998,"5min":17.277999999999999,"15min":17.277999999999999},"last":0.317}]},{"osd":5,"last update":"Mon Oct 9 09:29:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":1.5109999999999999,"15min":1.1519999999999999},"min":{"1min":0.36199999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.2959999999999994,"5min":9.2959999999999994,"15min":9.2959999999999994},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.52,"5min":1.5249999999999999,"15min":1.244},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.2000000000000002,"5min":7.5819999999999999,"15min":7.5819999999999999},"last":0.34499999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:29:08 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":1.3979999999999999,"15min":1.169},"min":{"1min":0.38,"5min":0.377,"15min":0.377},"max":{"1min":7.1639999999999997,"5min":7.625,"15min":7.625},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.9239999999999999,"5min":1.4410000000000001,"15min":1.2190000000000001},"min":{"1min":0.47499999999999998,"5min":0.371,"15min":0.371},"max":{"1min":7.1440000000000001,"5min":7.1440000000000001,"15min":7.1440000000000001},"last":0.51600000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.327,"15min":1.139},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.427,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":0.375},{"interface":"front","average":{"1min":1.405,"5min":1.6719999999999999,"15min":1.3160000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":5.7590000000000003,"5min":9.1780000000000008,"15min":9.1780000000000008},"last":0.38600000000000001}]}]},{"osd":5,"up_from":37,"seq":158913790318,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":504720,"kb_used_data":199304,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93240432,"statfs":{"total":95995035648,"available":95478202368,"internally_reserved":0,"allocated":204087296,"data_stored":203796723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:28:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.151,"15min":0.96399999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.6200000000000001,"5min":10.109,"15min":10.109},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.0939999999999999,"5min":1.323,"15min":1.056},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.5250000000000004,"5min":7.5250000000000004,"15min":7.5250000000000004},"last":0.246}]},{"osd":1,"last update":"Mon Oct 9 09:29:14 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":1.355,"15min":1.0860000000000001},"min":{"1min":0.314,"5min":0.313,"15min":0.313},"max":{"1min":7.8959999999999999,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.8959999999999999,"5min":1.4490000000000001,"15min":1.1899999999999999},"min":{"1min":0.44700000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.9119999999999999,"5min":8.7210000000000001,"15min":8.7210000000000001},"last":0.83199999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:28:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":1.6140000000000001,"15min":1.2230000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":14.019,"5min":14.019,"15min":14.019},"last":0.38200000000000001},{"interface":"front","average":{"1min":3.4670000000000001,"5min":1.6599999999999999,"15min":1.234},"min":{"1min":0.41099999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":14.037000000000001,"5min":14.037000000000001,"15min":14.037000000000001},"last":0.34100000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:28:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1269999999999998,"5min":1.571,"15min":1.1799999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.8399999999999999,"5min":10.193,"15min":10.193},"last":0.64900000000000002},{"interface":"front","average":{"1min":2.4489999999999998,"5min":1.4299999999999999,"15min":1.2090000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.4020000000000001,"5min":7.5579999999999998,"15min":7.5579999999999998},"last":0.375}]},{"osd":4,"last update":"Mon Oct 9 09:29:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.2330000000000001,"15min":1.032},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.747,"5min":10.747,"15min":10.747},"last":1.409},{"interface":"front","average":{"1min":1.9059999999999999,"5min":1.2230000000000001,"15min":1.1120000000000001},"min":{"1min":0.32200000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.1189999999999998,"5min":7.5780000000000003,"15min":7.5780000000000003},"last":0.40999999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:28:48 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":1.194,"15min":1.0980000000000001},"min":{"1min":0.433,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.7880000000000003,"5min":10.253,"15min":10.253},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.0179999999999998,"5min":1.351,"15min":1.272},"min":{"1min":0.53000000000000003,"5min":0.372,"15min":0.372},"max":{"1min":5.9130000000000003,"5min":7.5960000000000001,"15min":7.5960000000000001},"last":0.44}]},{"osd":7,"last update":"Mon Oct 9 09:29:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2480000000000002,"5min":1.5369999999999999,"15min":1.304},"min":{"1min":0.44,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.9909999999999997,"5min":8.6359999999999992,"15min":8.6359999999999992},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.3330000000000002,"5min":1.605,"15min":1.2909999999999999},"min":{"1min":0.48499999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":7.8559999999999999,"5min":7.8559999999999999,"15min":7.8559999999999999},"last":0.41899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":139268096,"data_stored":139264024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":148443136,"data_stored":148439064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":305856512,"data_stored":305856512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":397803520,"data_stored":397803520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":183238656,"data_stored":183238656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":184881152,"data_stored":184877080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":184549376,"data_stored":184549376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":217055232,"data_stored":217055232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:29:41.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-10-09T09:29:41.067 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:40 smithi163 bash[75425]: cluster 2023-10-09T09:29:39.680865+0000 mgr.y (mgr.34107) 700 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 555 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T09:29:41.067 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:40 smithi163 bash[75425]: audit 2023-10-09T09:29:40.372309+0000 mon.a (mon.0) 1503 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:29:41.067 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:40 smithi163 bash[75425]: audit 2023-10-09T09:29:40.373770+0000 mon.a (mon.0) 1504 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:41.077 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:41.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:41.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:41.198 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:29:41.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:41.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:41.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:41.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:41.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:41.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:41.489 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:41.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:41.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:41.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:41.638 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:41.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:41.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:41.747 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:41.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:41.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:41.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:41.876 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:41.911 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:41.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:42.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:42.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:41 smithi181 bash[59345]: audit 2023-10-09T09:29:39.790711+0000 mgr.y (mgr.34107) 701 : audit 0 from='client.54613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:29:42.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:42.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:42.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:42.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:42.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:42.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:42.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:42.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:41 smithi163 bash[75425]: audit 2023-10-09T09:29:39.790711+0000 mgr.y (mgr.34107) 701 : audit 0 from='client.54613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:29:42.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:41 smithi163 bash[78107]: audit 2023-10-09T09:29:39.790711+0000 mgr.y (mgr.34107) 701 : audit 0 from='client.54613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:29:42.278 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:42.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:42.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:42.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:42.415 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:42.449 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:42.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:42.546 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:29:42.546 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:29:42.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:42.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:42.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:42.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:42.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:42.736 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:42.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:42.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:42.840 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:42.848 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:42 smithi163 bash[75425]: cluster 2023-10-09T09:29:41.681568+0000 mgr.y (mgr.34107) 702 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 386 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T09:29:42.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:42 smithi163 bash[78107]: cluster 2023-10-09T09:29:41.681568+0000 mgr.y (mgr.34107) 702 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 386 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T09:29:42.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:42.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:42.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:42.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:43.020 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:43.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:43.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:42 smithi181 bash[59345]: cluster 2023-10-09T09:29:41.681568+0000 mgr.y (mgr.34107) 702 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 386 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T09:29:43.124 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:43.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:43.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:43.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:43.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:43.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:43.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:43.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:43.355 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T09:29:43.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:43.426 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:43.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:43.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:43.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:43.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:43.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:43.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:43.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:43.674 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:29:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:29:43] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.33.4" 2023-10-09T09:29:43.705 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:43.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:43.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:43.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:43.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:43 smithi181 bash[59345]: cluster 2023-10-09T09:29:43.682149+0000 mgr.y (mgr.34107) 703 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T09:29:43.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:43.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:43.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:43.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:43.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:43 smithi163 bash[75425]: cluster 2023-10-09T09:29:43.682149+0000 mgr.y (mgr.34107) 703 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T09:29:43.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:43 smithi163 bash[78107]: cluster 2023-10-09T09:29:43.682149+0000 mgr.y (mgr.34107) 703 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T09:29:43.968 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:43.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:43.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:44.077 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:44.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:44.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:44.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:44.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:44.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:44.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:44.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:44.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:44.389 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:44.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:44.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:44.530 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:44.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:44.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:44.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:44.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:44.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:44.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:44.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:44.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:44.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:44.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:44.901 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:44.926 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:44.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:45.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:45.066 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:45.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:45.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:45.086 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:45.093 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:45.098 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:29:45.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:45.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:45.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:45.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:45.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:45.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:45.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:45.439 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:45.464 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:45.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:45.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:45.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:45.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:45.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:45.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:45.701 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:45.727 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:45.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:45.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:45.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:45.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:45.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:45.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:45.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:45.999 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:46.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:46.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:46.144 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:46.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:46.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:46.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:46.272 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:46.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:46.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:46.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:46.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:46.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:46.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:46.512 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:46.537 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:46.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:46.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:46.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:46.667 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:46.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:46.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:46.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:46.782 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:46 smithi181 bash[59345]: cluster 2023-10-09T09:29:45.682747+0000 mgr.y (mgr.34107) 704 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 294 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T09:29:46.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:46.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:46.908 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:46.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:46.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:46.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:46.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:46 smithi163 bash[78107]: cluster 2023-10-09T09:29:45.682747+0000 mgr.y (mgr.34107) 704 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 294 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T09:29:46.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:46 smithi163 bash[75425]: cluster 2023-10-09T09:29:45.682747+0000 mgr.y (mgr.34107) 704 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 294 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T09:29:47.039 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:47.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:47.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:47.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:47.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:47.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:47.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:47.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:47.306 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:47.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:47.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:47.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:47.436 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:47.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:47.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:47.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:47.574 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:47.602 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:47.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:47.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:47.706 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:47.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:47.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:47.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:47.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:47.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:47.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:47.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:47.979 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:48.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:48.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:48.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:48.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:48.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:48.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:48.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:48.172 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:29:48.254 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:48.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:48.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:48.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:48.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:48.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:48.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:48.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:48.522 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:48.548 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:48.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:48.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:48.651 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:48.673 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:48.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:48.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:48.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:48.797 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:48.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:48.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:48.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:48.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:48.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:49.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:49.053 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:48 smithi163 bash[78107]: cluster 2023-10-09T09:29:47.683426+0000 mgr.y (mgr.34107) 705 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 149 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-09T09:29:49.053 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:48 smithi163 bash[75425]: cluster 2023-10-09T09:29:47.683426+0000 mgr.y (mgr.34107) 705 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 149 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-09T09:29:49.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:48 smithi181 bash[59345]: cluster 2023-10-09T09:29:47.683426+0000 mgr.y (mgr.34107) 705 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 149 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-09T09:29:49.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:49.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:49.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:49.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:49.205 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:49.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:49.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:49.263 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 110757 objects 2023-10-09T09:29:49.264 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :154.406 2023-10-09T09:29:49.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:49.341 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:49.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:49.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:49.445 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:49.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:49.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:49.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:49.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:49.596 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:49.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:49.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:49.700 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:49.732 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:49.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:49.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:49.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:49.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:49.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:49.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:49.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:49.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:49.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:50.095 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:50.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:50.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:50.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:50.219 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:50.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:50.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:50.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:50.352 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:50.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:50.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:50.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:50.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:50.511 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:50.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:50.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:50.613 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:50.634 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:50.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:50.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:50.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:50.768 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:50.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:50.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:50.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:50.896 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:50.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:50.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:50.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:51.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:50 smithi163 bash[78107]: cluster 2023-10-09T09:29:49.684190+0000 mgr.y (mgr.34107) 706 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 122 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-10-09T09:29:51.005 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:50 smithi163 bash[75425]: cluster 2023-10-09T09:29:49.684190+0000 mgr.y (mgr.34107) 706 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 122 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-10-09T09:29:51.023 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:51.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:51.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:50 smithi181 bash[59345]: cluster 2023-10-09T09:29:49.684190+0000 mgr.y (mgr.34107) 706 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 122 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-10-09T09:29:51.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:51.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:51.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:51.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:51.254 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:51.283 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:51.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:51.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:51.387 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:51.415 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:51.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:51.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:51.550 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:51.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:51.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:51.653 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:51.682 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:51.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:51.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:51.753 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:51 smithi163 bash[75425]: cluster 2023-10-09T09:29:51.685262+0000 mgr.y (mgr.34107) 707 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 17 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T09:29:51.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:51.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:51.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:51.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:51.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:51.939 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:29:51.939 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:29:51.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:51.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:51.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:51.964 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:51 smithi181 bash[59345]: cluster 2023-10-09T09:29:51.685262+0000 mgr.y (mgr.34107) 707 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 17 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T09:29:52.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:52.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:51 smithi163 bash[78107]: cluster 2023-10-09T09:29:51.685262+0000 mgr.y (mgr.34107) 707 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 17 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T09:29:52.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:52.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:52.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:52.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:52.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:52.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:52.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:52.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:52.339 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:52.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:52.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:52.462 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:52.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:52.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:52.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:52.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:52.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:52.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:52.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:52.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T09:29:52.723 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:52.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:52.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:52.827 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:52.852 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:52.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:52.956 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:52.989 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:52.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:53.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:53.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:53.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:53.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:53.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:53.255 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:53.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:53.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:53.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:53.388 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:53.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:53.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:53.520 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:53.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:53.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:53.623 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:53.652 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:53.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:53.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:29:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:29:53] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.33.4" 2023-10-09T09:29:53.756 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:53.786 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:53.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:53.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:53.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:53.914 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:53.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:53.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:53.960 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:29:53.987 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:29:54.011 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:29:54.017 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:54.037 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:29:54.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:54.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:54.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:54.064 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:29:54.088 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:29:54.115 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:29:54.140 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:29:54.142 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:29:54.143 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:29:54.143 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:29:54.143 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:29:54.143 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:29:54.143 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:29:54.143 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:29:54.143 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:29:54.152 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:54.182 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:54.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:54.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:54.286 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:54.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:54.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:54.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:54.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:54.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:54.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:54.559 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:54.594 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:54.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:54.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:54.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:54.731 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:54.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:54.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:54.753 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 6 pgp_num to 16 2023-10-09T09:29:54.753 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:29:54.836 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:54.867 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:54.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:54.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:54.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:54.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:54.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:54.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:55.024 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:54 smithi163 bash[78107]: cluster 2023-10-09T09:29:53.686091+0000 mgr.y (mgr.34107) 708 : cluster 0 pgmap v1409: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-10-09T09:29:55.024 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:54 smithi163 bash[78107]: audit 2023-10-09T09:29:53.835656+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.163:0/3305190685' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:29:55.024 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:54 smithi163 bash[78107]: audit 2023-10-09T09:29:53.836081+0000 mon.a (mon.0) 1505 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:29:55.024 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:54 smithi163 bash[78107]: audit 2023-10-09T09:29:53.934187+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.163:0/792359472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:55.024 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:54 smithi163 bash[78107]: audit 2023-10-09T09:29:54.171171+0000 mon.a (mon.0) 1506 : audit 1 from='client.? 172.21.15.163:0/3570599800' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:29:55.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:54 smithi163 bash[75425]: cluster 2023-10-09T09:29:53.686091+0000 mgr.y (mgr.34107) 708 : cluster 0 pgmap v1409: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-10-09T09:29:55.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:54 smithi163 bash[75425]: audit 2023-10-09T09:29:53.835656+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.163:0/3305190685' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:29:55.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:54 smithi163 bash[75425]: audit 2023-10-09T09:29:53.836081+0000 mon.a (mon.0) 1505 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:29:55.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:54 smithi163 bash[75425]: audit 2023-10-09T09:29:53.934187+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.163:0/792359472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:55.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:54 smithi163 bash[75425]: audit 2023-10-09T09:29:54.171171+0000 mon.a (mon.0) 1506 : audit 1 from='client.? 172.21.15.163:0/3570599800' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:29:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:54 smithi181 bash[59345]: cluster 2023-10-09T09:29:53.686091+0000 mgr.y (mgr.34107) 708 : cluster 0 pgmap v1409: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-10-09T09:29:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:54 smithi181 bash[59345]: audit 2023-10-09T09:29:53.835656+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.163:0/3305190685' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:29:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:54 smithi181 bash[59345]: audit 2023-10-09T09:29:53.836081+0000 mon.a (mon.0) 1505 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:29:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:54 smithi181 bash[59345]: audit 2023-10-09T09:29:53.934187+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.163:0/792359472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:29:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:54 smithi181 bash[59345]: audit 2023-10-09T09:29:54.171171+0000 mon.a (mon.0) 1506 : audit 1 from='client.? 172.21.15.163:0/3570599800' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:29:55.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:55.126 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:55.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:55.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:55.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:55.256 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:55.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:55.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:55.359 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:55.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:55.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:55.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:55.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:55.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:55.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:55.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:55.626 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:55.906 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[78107]: cluster 2023-10-09T09:29:54.739966+0000 mon.a (mon.0) 1507 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:29:55.906 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[78107]: cluster 2023-10-09T09:29:54.743209+0000 osd.5 (osd.5) 641 : cluster 3 failed to encode map e1097 with expected crc 2023-10-09T09:29:55.906 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[78107]: cluster 2023-10-09T09:29:54.743919+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e1097 with expected crc 2023-10-09T09:29:55.906 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[78107]: audit 2023-10-09T09:29:54.747191+0000 mon.a (mon.0) 1508 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:29:56.048 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[78107]: audit 2023-10-09T09:29:54.747324+0000 mon.a (mon.0) 1509 : audit 1 from='client.? 172.21.15.163:0/3570599800' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:29:56.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[78107]: cluster 2023-10-09T09:29:54.747409+0000 mon.a (mon.0) 1510 : cluster 0 osdmap e1097: 8 total, 8 up, 8 in 2023-10-09T09:29:56.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[78107]: audit 2023-10-09T09:29:55.366340+0000 mon.a (mon.0) 1511 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:56.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[78107]: cluster 2023-10-09T09:29:55.686802+0000 mgr.y (mgr.34107) 709 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 468 op/s 2023-10-09T09:29:56.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[75425]: cluster 2023-10-09T09:29:54.739966+0000 mon.a (mon.0) 1507 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:29:56.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[75425]: cluster 2023-10-09T09:29:54.743209+0000 osd.5 (osd.5) 641 : cluster 3 failed to encode map e1097 with expected crc 2023-10-09T09:29:56.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[75425]: cluster 2023-10-09T09:29:54.743919+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e1097 with expected crc 2023-10-09T09:29:56.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[75425]: audit 2023-10-09T09:29:54.747191+0000 mon.a (mon.0) 1508 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:29:56.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[75425]: audit 2023-10-09T09:29:54.747324+0000 mon.a (mon.0) 1509 : audit 1 from='client.? 172.21.15.163:0/3570599800' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:29:56.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[75425]: cluster 2023-10-09T09:29:54.747409+0000 mon.a (mon.0) 1510 : cluster 0 osdmap e1097: 8 total, 8 up, 8 in 2023-10-09T09:29:56.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[75425]: audit 2023-10-09T09:29:55.366340+0000 mon.a (mon.0) 1511 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:56.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:55 smithi163 bash[75425]: cluster 2023-10-09T09:29:55.686802+0000 mgr.y (mgr.34107) 709 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 468 op/s 2023-10-09T09:29:56.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:56.051 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:56.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:56.053 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T09:29:56.055 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T09:29:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:55 smithi181 bash[59345]: cluster 2023-10-09T09:29:54.739966+0000 mon.a (mon.0) 1507 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:29:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:55 smithi181 bash[59345]: cluster 2023-10-09T09:29:54.743209+0000 osd.5 (osd.5) 641 : cluster 3 failed to encode map e1097 with expected crc 2023-10-09T09:29:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:55 smithi181 bash[59345]: cluster 2023-10-09T09:29:54.743919+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e1097 with expected crc 2023-10-09T09:29:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:55 smithi181 bash[59345]: audit 2023-10-09T09:29:54.747191+0000 mon.a (mon.0) 1508 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:29:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:55 smithi181 bash[59345]: audit 2023-10-09T09:29:54.747324+0000 mon.a (mon.0) 1509 : audit 1 from='client.? 172.21.15.163:0/3570599800' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:29:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:55 smithi181 bash[59345]: cluster 2023-10-09T09:29:54.747409+0000 mon.a (mon.0) 1510 : cluster 0 osdmap e1097: 8 total, 8 up, 8 in 2023-10-09T09:29:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:55 smithi181 bash[59345]: audit 2023-10-09T09:29:55.366340+0000 mon.a (mon.0) 1511 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:29:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:55 smithi181 bash[59345]: cluster 2023-10-09T09:29:55.686802+0000 mgr.y (mgr.34107) 709 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 468 op/s 2023-10-09T09:29:56.152 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:56.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:56.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:56.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:56.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:56.318 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:56.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:56.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:56.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:56.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:56.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:56.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:56.559 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:56.591 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:56.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:29:56.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:56.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:56.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:56.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:56.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:56.864 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:56.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:56.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:56.968 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:57.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:57.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:57.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:57.026 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:29:57.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:57.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:57.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:57.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:57.239 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:57.267 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:57.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:57.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:57.371 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:57.405 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:57.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:57.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:57.506 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:29:57.508 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:57.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:57.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:57.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:57.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:57.663 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:57.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:57.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:57.767 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:57.799 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:29:57.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:57.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:29:57.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:57.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:57.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:57.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:58.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:58.056 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:29:58.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:58.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:58.174 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:58.205 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:58.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:58.309 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:58.338 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:58.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:58.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:58.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:58.470 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:58.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:58.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:58.574 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:58.602 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:29:58.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:58.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:29:58.707 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:58.738 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:58.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:58.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:58.842 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:58.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:58.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:58.972 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:58.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:58 smithi181 bash[59345]: cluster 2023-10-09T09:29:57.687837+0000 mgr.y (mgr.34107) 710 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 238 op/s 2023-10-09T09:29:59.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:59.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:59.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:59.103 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:59.135 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:59.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:59.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:59.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:58 smithi163 bash[75425]: cluster 2023-10-09T09:29:57.687837+0000 mgr.y (mgr.34107) 710 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 238 op/s 2023-10-09T09:29:59.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:58 smithi163 bash[78107]: cluster 2023-10-09T09:29:57.687837+0000 mgr.y (mgr.34107) 710 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 238 op/s 2023-10-09T09:29:59.239 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:59.272 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:29:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:59.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:29:59.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:59.404 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:29:59.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:59.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:29:59.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:59.540 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:29:59.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:59.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:29:59.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:59.671 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:29:59.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:59.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:29:59.774 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:59.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:29:59.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:59.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:29:59.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:59.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:29:59 smithi163 bash[75425]: cluster 2023-10-09T09:29:59.688622+0000 mgr.y (mgr.34107) 711 : cluster 0 pgmap v1413: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 194 op/s 2023-10-09T09:29:59.918 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:29:59 smithi163 bash[78107]: cluster 2023-10-09T09:29:59.688622+0000 mgr.y (mgr.34107) 711 : cluster 0 pgmap v1413: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 194 op/s 2023-10-09T09:29:59.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:29:59.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:29:59.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:00.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:00.054 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:29:59 smithi181 bash[59345]: cluster 2023-10-09T09:29:59.688622+0000 mgr.y (mgr.34107) 711 : cluster 0 pgmap v1413: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 194 op/s 2023-10-09T09:30:00.075 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:00.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:00.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:00.178 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:00.208 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:00.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:00.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:00.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:00.345 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:00.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:00.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:00.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:00.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:00.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:00.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:00.578 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:00.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:00.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:00.623 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:30:00.623 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:30:00.623 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:30:00.623 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:30:00.623 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:30:00.714 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:00.743 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:00.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:00.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:00.847 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:00.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:00 smithi163 bash[78107]: cluster 2023-10-09T09:30:00.000153+0000 mon.a (mon.0) 1512 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-10-09T09:30:00.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:00 smithi163 bash[78107]: cluster 2023-10-09T09:30:00.000205+0000 mon.a (mon.0) 1513 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T09:30:00.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:00 smithi163 bash[75425]: cluster 2023-10-09T09:30:00.000153+0000 mon.a (mon.0) 1512 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-10-09T09:30:00.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:00 smithi163 bash[75425]: cluster 2023-10-09T09:30:00.000205+0000 mon.a (mon.0) 1513 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T09:30:00.868 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:00.869 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:00.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:00.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:00.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:01.014 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:01.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:01.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:01.053 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:30:01.054 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:30:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:00 smithi181 bash[59345]: cluster 2023-10-09T09:30:00.000153+0000 mon.a (mon.0) 1512 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-10-09T09:30:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:00 smithi181 bash[59345]: cluster 2023-10-09T09:30:00.000205+0000 mon.a (mon.0) 1513 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T09:30:01.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:01.151 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:01.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:01.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:01.254 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:01.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:01.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:01.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:01.328 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T09:30:01.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:01.415 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:01.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:01.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:01.519 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:01.549 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:01.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:01.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:01.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:01.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:01.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:01.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:01.782 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:01.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:01.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:01.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:01.885 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:01.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:01.950 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:01.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:01.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:01 smithi163 bash[75425]: cluster 2023-10-09T09:30:01.689356+0000 mgr.y (mgr.34107) 712 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 25 op/s 2023-10-09T09:30:01.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:01 smithi163 bash[78107]: cluster 2023-10-09T09:30:01.689356+0000 mgr.y (mgr.34107) 712 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 25 op/s 2023-10-09T09:30:02.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:01 smithi181 bash[59345]: cluster 2023-10-09T09:30:01.689356+0000 mgr.y (mgr.34107) 712 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 25 op/s 2023-10-09T09:30:02.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:02.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:02.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:02.184 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:02.217 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:02.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:02.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:02.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:02.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:02.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:02.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:02.456 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:02.490 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:02.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:02.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:02.593 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:02.620 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:02.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:02.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:02.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:02.756 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:02.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:02.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:02.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:02.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:02.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:02.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:02.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:03.027 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:03.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:03.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:03.132 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:03.162 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:03.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:03.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:03.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:03.295 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:03.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:03.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:03.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:03.425 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:03.427 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:30:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:30:03] "GET /metrics HTTP/1.1" 200 36279 "" "Prometheus/2.33.4" 2023-10-09T09:30:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:03.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:03.529 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:03.561 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:03.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:03.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:03.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:03.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:03.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:03.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:03.789 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:03.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:03.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:03.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:03.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:03.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:03.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:03.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:03 smithi163 bash[75425]: cluster 2023-10-09T09:30:03.689903+0000 mgr.y (mgr.34107) 713 : cluster 0 pgmap v1415: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:03.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:03 smithi163 bash[78107]: cluster 2023-10-09T09:30:03.689903+0000 mgr.y (mgr.34107) 713 : cluster 0 pgmap v1415: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:04.062 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:04.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:03 smithi181 bash[59345]: cluster 2023-10-09T09:30:03.689903+0000 mgr.y (mgr.34107) 713 : cluster 0 pgmap v1415: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:04.095 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:04.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:04.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:04.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:04.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:04.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:04.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:04.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:04.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:04.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:04.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:04.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:04.486 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:04.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:04.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:04.621 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:04.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:04.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:04.726 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:04.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:04.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:04.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:04.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:04.879 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:04.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:04.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:04.982 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:05.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:05.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:05.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:05.121 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:05.144 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:05.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:05.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:05.247 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:05.274 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:05.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:05.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:05.377 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:05.410 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:05.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:05.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:05.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:05.549 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:05.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:05.653 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:05.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:05.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:05.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:05.876 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:05.918 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:05.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:06.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:06.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:06.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:06.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:06.156 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:30:06.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:06.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:06.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:06.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:06.312 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:06.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:06.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:06.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:06.451 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:06.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:06.488 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:06.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:06.589 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:06.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:06.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:06 smithi163 bash[78107]: cluster 2023-10-09T09:30:05.690415+0000 mgr.y (mgr.34107) 714 : cluster 0 pgmap v1416: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:06 smithi163 bash[75425]: cluster 2023-10-09T09:30:05.690415+0000 mgr.y (mgr.34107) 714 : cluster 0 pgmap v1416: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:06.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:06.729 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:06 smithi181 bash[59345]: cluster 2023-10-09T09:30:05.690415+0000 mgr.y (mgr.34107) 714 : cluster 0 pgmap v1416: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:06.748 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:06.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:06.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:06.851 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:06.884 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:06.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:06.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:06.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:07.011 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:07.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:07.116 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:07.148 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:07.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:07.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:07.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:07.283 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:07.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:07.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:07.387 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:07.416 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:07.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:07.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:07.519 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:07.548 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:07.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:07.651 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:07.681 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:07.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:07.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:07.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:07.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:07.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:07.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:07.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:07.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:07.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:08.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:08.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:07 smithi163 bash[78107]: cluster 2023-10-09T09:30:07.691467+0000 mgr.y (mgr.34107) 715 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:08.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:07 smithi163 bash[75425]: cluster 2023-10-09T09:30:07.691467+0000 mgr.y (mgr.34107) 715 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:08.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:07 smithi181 bash[59345]: cluster 2023-10-09T09:30:07.691467+0000 mgr.y (mgr.34107) 715 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:08.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:08.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:08.162 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:30:08.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:08.189 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:30:08.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:08.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:08.228 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:30:08.254 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:30:08.273 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:30:08.303 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:30:08.314 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:08.327 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:30:08.340 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:08.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:08.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:08.351 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:30:08.353 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:30:08.353 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:30:08.353 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:30:08.354 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:30:08.354 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:30:08.354 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:30:08.354 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:30:08.354 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:30:08.444 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:08.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:08.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:08.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:08.573 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:08.602 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:08.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:08.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:08.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:08.734 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:08.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:08.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:08.811 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:30:08.813 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_1' removed 2023-10-09T09:30:08.838 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:08 smithi181 bash[59345]: audit 2023-10-09T09:30:08.134704+0000 mon.a (mon.0) 1514 : audit 0 from='client.? 172.21.15.163:0/1602307859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:30:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:08 smithi181 bash[59345]: audit 2023-10-09T09:30:08.369217+0000 mon.c (mon.2) 56 : audit 1 from='client.? 172.21.15.163:0/1752197332' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:30:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:08 smithi181 bash[59345]: audit 2023-10-09T09:30:08.369625+0000 mon.a (mon.0) 1515 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:30:08.850 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:08 smithi181 bash[59345]: audit 2023-10-09T09:30:08.518813+0000 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.163:0/133541850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:30:08.875 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:08.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:08.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:08.979 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:09.006 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:09.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:09.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:09.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:09.120 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:08 smithi163 bash[78107]: audit 2023-10-09T09:30:08.134704+0000 mon.a (mon.0) 1514 : audit 0 from='client.? 172.21.15.163:0/1602307859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:30:09.121 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:08 smithi163 bash[78107]: audit 2023-10-09T09:30:08.369217+0000 mon.c (mon.2) 56 : audit 1 from='client.? 172.21.15.163:0/1752197332' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:30:09.121 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:08 smithi163 bash[78107]: audit 2023-10-09T09:30:08.369625+0000 mon.a (mon.0) 1515 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:30:09.121 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:08 smithi163 bash[78107]: audit 2023-10-09T09:30:08.518813+0000 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.163:0/133541850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:30:09.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:08 smithi163 bash[75425]: audit 2023-10-09T09:30:08.134704+0000 mon.a (mon.0) 1514 : audit 0 from='client.? 172.21.15.163:0/1602307859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:30:09.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:08 smithi163 bash[75425]: audit 2023-10-09T09:30:08.369217+0000 mon.c (mon.2) 56 : audit 1 from='client.? 172.21.15.163:0/1752197332' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:30:09.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:08 smithi163 bash[75425]: audit 2023-10-09T09:30:08.369625+0000 mon.a (mon.0) 1515 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:30:09.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:08 smithi163 bash[75425]: audit 2023-10-09T09:30:08.518813+0000 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.163:0/133541850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:30:09.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:09.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:09.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:09.224 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:30:09.224 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:30:09.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:09.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:09.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:09.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:09.371 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:09.388 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:09.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:09.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:09.513 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:09.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:09.615 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:09.643 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:09.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:09.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:09.746 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:09.775 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:09.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:09.878 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:09.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:09.903 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[78107]: cluster 2023-10-09T09:30:08.796662+0000 mon.a (mon.0) 1517 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:30:09.903 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[78107]: cluster 2023-10-09T09:30:08.805544+0000 osd.5 (osd.5) 642 : cluster 3 failed to encode map e1098 with expected crc 2023-10-09T09:30:09.903 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[78107]: cluster 2023-10-09T09:30:08.806470+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e1098 with expected crc 2023-10-09T09:30:09.904 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[78107]: audit 2023-10-09T09:30:08.807624+0000 mon.a (mon.0) 1518 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:30:09.904 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[78107]: audit 2023-10-09T09:30:08.807704+0000 mon.a (mon.0) 1519 : audit 1 from='client.? 172.21.15.163:0/133541850' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:30:09.904 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[78107]: cluster 2023-10-09T09:30:08.807758+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e1098: 8 total, 8 up, 8 in 2023-10-09T09:30:09.904 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[78107]: cluster 2023-10-09T09:30:09.691864+0000 mgr.y (mgr.34107) 716 : cluster 0 pgmap v1419: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:09.904 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[75425]: cluster 2023-10-09T09:30:08.796662+0000 mon.a (mon.0) 1517 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:30:09.904 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[75425]: cluster 2023-10-09T09:30:08.805544+0000 osd.5 (osd.5) 642 : cluster 3 failed to encode map e1098 with expected crc 2023-10-09T09:30:09.904 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[75425]: cluster 2023-10-09T09:30:08.806470+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e1098 with expected crc 2023-10-09T09:30:09.904 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[75425]: audit 2023-10-09T09:30:08.807624+0000 mon.a (mon.0) 1518 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:30:09.905 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[75425]: audit 2023-10-09T09:30:08.807704+0000 mon.a (mon.0) 1519 : audit 1 from='client.? 172.21.15.163:0/133541850' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:30:09.905 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[75425]: cluster 2023-10-09T09:30:08.807758+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e1098: 8 total, 8 up, 8 in 2023-10-09T09:30:09.905 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:09 smithi163 bash[75425]: cluster 2023-10-09T09:30:09.691864+0000 mgr.y (mgr.34107) 716 : cluster 0 pgmap v1419: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:09.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:09.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:09.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:09 smithi181 bash[59345]: cluster 2023-10-09T09:30:08.796662+0000 mon.a (mon.0) 1517 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:30:09.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:09 smithi181 bash[59345]: cluster 2023-10-09T09:30:08.805544+0000 osd.5 (osd.5) 642 : cluster 3 failed to encode map e1098 with expected crc 2023-10-09T09:30:09.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:09 smithi181 bash[59345]: cluster 2023-10-09T09:30:08.806470+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e1098 with expected crc 2023-10-09T09:30:09.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:09 smithi181 bash[59345]: audit 2023-10-09T09:30:08.807624+0000 mon.a (mon.0) 1518 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:30:09.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:09 smithi181 bash[59345]: audit 2023-10-09T09:30:08.807704+0000 mon.a (mon.0) 1519 : audit 1 from='client.? 172.21.15.163:0/133541850' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:30:09.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:09 smithi181 bash[59345]: cluster 2023-10-09T09:30:08.807758+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e1098: 8 total, 8 up, 8 in 2023-10-09T09:30:09.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:09 smithi181 bash[59345]: cluster 2023-10-09T09:30:09.691864+0000 mgr.y (mgr.34107) 716 : cluster 0 pgmap v1419: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:09.935 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T09:30:10.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:10.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:10.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:10.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:10.141 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:10.166 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:10.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:10.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:10.270 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:10.302 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:10.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:10.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:10.438 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:10.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:10.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:10.541 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:10.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:10.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:10.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:10.689 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:30:10.690 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:30:10.690 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T09:30:10.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T09:30:10.709 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:10.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:10.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:10.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:10.843 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:10.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:10.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:10.946 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:10.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:10 smithi163 bash[78107]: audit 2023-10-09T09:30:10.366708+0000 mon.a (mon.0) 1521 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:10.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:10 smithi163 bash[75425]: audit 2023-10-09T09:30:10.366708+0000 mon.a (mon.0) 1521 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:10.981 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:10.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:10.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:10 smithi181 bash[59345]: audit 2023-10-09T09:30:10.366708+0000 mon.a (mon.0) 1521 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:11.084 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:11.115 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:11.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:11.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:11.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:11.254 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:11.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:11.358 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:11.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:30:11.388 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:11.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:11.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:11.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:11.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:11.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:11.625 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:11.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:11.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:11.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:11.762 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:11.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:11.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:11.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:11.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:11.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:11.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:12.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:12.064 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:12.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:11 smithi181 bash[59345]: cluster 2023-10-09T09:30:11.692842+0000 mgr.y (mgr.34107) 717 : cluster 0 pgmap v1420: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:12.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:12.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:12.169 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:12.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:11 smithi163 bash[78107]: cluster 2023-10-09T09:30:11.692842+0000 mgr.y (mgr.34107) 717 : cluster 0 pgmap v1420: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:12.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:11 smithi163 bash[75425]: cluster 2023-10-09T09:30:11.692842+0000 mgr.y (mgr.34107) 717 : cluster 0 pgmap v1420: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:12.202 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:12.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:12.305 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:12.340 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:12.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:12.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:12.443 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:12.474 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:12.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:12.578 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:12.610 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:12.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:12.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:12.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:30:12.714 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:12.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:12.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:12.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:12.846 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:12.882 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:12.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:12.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:12.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:13.015 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:13.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:13.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:13.119 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:13.154 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:13.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:13.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:13.258 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:13.288 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:13.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:13.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:13.392 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:13.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:13.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:13.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:13.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:13.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:13.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:13.675 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:30:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:30:13] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.33.4" 2023-10-09T09:30:13.694 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:13.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:13.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:13.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:13.825 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:13.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:13.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:13.929 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:13.940 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:13 smithi163 bash[78107]: cluster 2023-10-09T09:30:13.693629+0000 mgr.y (mgr.34107) 718 : cluster 0 pgmap v1421: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:13.941 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:13 smithi163 bash[75425]: cluster 2023-10-09T09:30:13.693629+0000 mgr.y (mgr.34107) 718 : cluster 0 pgmap v1421: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:13.963 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:13.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:14.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:13 smithi181 bash[59345]: cluster 2023-10-09T09:30:13.693629+0000 mgr.y (mgr.34107) 718 : cluster 0 pgmap v1421: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:14.096 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:14.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:14.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:14.228 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:14.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:14.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:14.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:14.363 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:14.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:14.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:14.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:14.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:14.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:14.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:14.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:14.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:14.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:14.709 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:30:14.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:14.766 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:14.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:14.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:14.870 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:14.901 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:14.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:14.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:15.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:15.029 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:15.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:15.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:15.133 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:15.159 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:15.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:15.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:15.262 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:15.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:15.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:15.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:15.398 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:15.427 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:15.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:15.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:15.482 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:15.483 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:15.531 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:15.562 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:15.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:15.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:15.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:15.699 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:15.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:15.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:15.803 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:15.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:15.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:15.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:15.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:15.961 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:15.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:15.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:16.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:16.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:16.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:16.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:16.193 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:16.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:16.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:16.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:16.325 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:16.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:16.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:16.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:16.458 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:16.481 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:16.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:16.495 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:16 smithi181 bash[59345]: cluster 2023-10-09T09:30:15.694350+0000 mgr.y (mgr.34107) 719 : cluster 0 pgmap v1422: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:16.585 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:16.610 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:16.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:16.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:16.715 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:16.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:16 smithi163 bash[78107]: cluster 2023-10-09T09:30:15.694350+0000 mgr.y (mgr.34107) 719 : cluster 0 pgmap v1422: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:16.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:16 smithi163 bash[75425]: cluster 2023-10-09T09:30:15.694350+0000 mgr.y (mgr.34107) 719 : cluster 0 pgmap v1422: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:16.746 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:16.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:16.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:16.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:16.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:16.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:16.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:16.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:17.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:17.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:17.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:17.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:17.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:17.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:17.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:17.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:17.272 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:17.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:17.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:17.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:17.404 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:17.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:17.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:17.508 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:17.540 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:17.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:17.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:17.644 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:17.669 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:17.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:17.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:17.773 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:17.799 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:17.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:17.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:17.914 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:17 smithi163 bash[78107]: cluster 2023-10-09T09:30:17.695399+0000 mgr.y (mgr.34107) 720 : cluster 0 pgmap v1423: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:17.914 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:17 smithi163 bash[75425]: cluster 2023-10-09T09:30:17.695399+0000 mgr.y (mgr.34107) 720 : cluster 0 pgmap v1423: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:17.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:17.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:18.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:18.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:18.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:18.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:18.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:17 smithi181 bash[59345]: cluster 2023-10-09T09:30:17.695399+0000 mgr.y (mgr.34107) 720 : cluster 0 pgmap v1423: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:18.093 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:30:18.093 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:30:18.161 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:18.191 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:18.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:18.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:18.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:18.320 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:18.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:18.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:18.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:18.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:18.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:18.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:18.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:18.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:18.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:18.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:18.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:18.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:18.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:18.810 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:18.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T09:30:18.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:18.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:18.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:18.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:18.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:18.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:18.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:19.074 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:19.099 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:19.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:19.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:19.202 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:19.236 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:19.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:19.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:19.340 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:19.365 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:19.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:19.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:19.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:19.496 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:19.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:19.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:19.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:19.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:19.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:19.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:19.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:19.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:19.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:19.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:19.875 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:19.887 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:19 smithi181 bash[59345]: cluster 2023-10-09T09:30:19.695958+0000 mgr.y (mgr.34107) 721 : cluster 0 pgmap v1424: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:19.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:19.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:20.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:20.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:20.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:20.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:20.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:19 smithi163 bash[78107]: cluster 2023-10-09T09:30:19.695958+0000 mgr.y (mgr.34107) 721 : cluster 0 pgmap v1424: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:20.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:19 smithi163 bash[75425]: cluster 2023-10-09T09:30:19.695958+0000 mgr.y (mgr.34107) 721 : cluster 0 pgmap v1424: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:20.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:20.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:20.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:20.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:20.272 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:20.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:20.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:20.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:20.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:20.426 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:20.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:20.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:20.530 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:20.562 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:20.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:20.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:20.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:20.690 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:20.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:20.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:20.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:20.819 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:20.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:20.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:20.923 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:20.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:20.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:20.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:21.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:21.074 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:21.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:21.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:21.178 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:21.208 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:21.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:21.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:21.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:21.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:21.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:21.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:21.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:21.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:21.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:21.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:21.604 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:21.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:21.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:21.708 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:21.736 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:21.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:21.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:21.839 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:21.851 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:21 smithi181 bash[59345]: cluster 2023-10-09T09:30:21.696918+0000 mgr.y (mgr.34107) 722 : cluster 0 pgmap v1425: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:21.869 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:21.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:21.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:21 smithi163 bash[75425]: cluster 2023-10-09T09:30:21.696918+0000 mgr.y (mgr.34107) 722 : cluster 0 pgmap v1425: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:21.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:21 smithi163 bash[78107]: cluster 2023-10-09T09:30:21.696918+0000 mgr.y (mgr.34107) 722 : cluster 0 pgmap v1425: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:21.973 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:21.998 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:22.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:22.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:22.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:22.130 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:22.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:22.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:22.234 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:22.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:22.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:22.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:22.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:22.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:22.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:22.492 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:22.511 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:22.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:22.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:22.614 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:22.645 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:22.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:22.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:22.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:22.779 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:22.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:22.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:22.882 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:22.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:22 smithi181 bash[59345]: audit 2023-10-09T09:30:22.446443+0000 mon.a (mon.0) 1522 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:30:22.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:22.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:22.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:22.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:22 smithi163 bash[75425]: audit 2023-10-09T09:30:22.446443+0000 mon.a (mon.0) 1522 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:30:22.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:22 smithi163 bash[78107]: audit 2023-10-09T09:30:22.446443+0000 mon.a (mon.0) 1522 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:30:23.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:23.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:23.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:23.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:23.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:23.173 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:23.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:23.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:23.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:23.308 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:23.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:23.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:23.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:23.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:23.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:23.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:23.541 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:23.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:23.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:23.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:23.619 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:30:23.665 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:30:23.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:23.689 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:30:23.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:23.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:23.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:23.712 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:30:23.717 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:30:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:30:23] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.33.4" 2023-10-09T09:30:23.746 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:30:23.775 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:30:23.803 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:30:23.810 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:23.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:23 smithi181 bash[59345]: audit 2023-10-09T09:30:23.643399+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.163:0/76915898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:30:23.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:23 smithi181 bash[59345]: cluster 2023-10-09T09:30:23.697441+0000 mgr.y (mgr.34107) 723 : cluster 0 pgmap v1426: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:30:23.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:23.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:23.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:23.850 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:30:23.852 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:30:23.852 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:30:23.852 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:30:23.852 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:30:23.852 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:30:23.852 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:30:23.852 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:30:23.852 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:30:23.937 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:23.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:23.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:23.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:24.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:24.095 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:23 smithi163 bash[78107]: audit 2023-10-09T09:30:23.643399+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.163:0/76915898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:30:24.095 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:23 smithi163 bash[78107]: cluster 2023-10-09T09:30:23.697441+0000 mgr.y (mgr.34107) 723 : cluster 0 pgmap v1426: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:24.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:23 smithi163 bash[75425]: audit 2023-10-09T09:30:23.643399+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.163:0/76915898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:30:24.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:23 smithi163 bash[75425]: cluster 2023-10-09T09:30:23.697441+0000 mgr.y (mgr.34107) 723 : cluster 0 pgmap v1426: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:24.096 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:24.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:24.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:24.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:24.230 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:24.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:24.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:24.334 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:24.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:24.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:24.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:24.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:24.486 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:24.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:24.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:24.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:24.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:24.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:24.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:24.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:24.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:24.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:24.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:24.833 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_2' created 2023-10-09T09:30:24.835 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:30:24.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:24.884 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:24.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:24 smithi163 bash[75425]: audit 2023-10-09T09:30:24.146343+0000 mon.c (mon.2) 58 : audit 1 from='client.? 172.21.15.163:0/455806017' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:30:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:24 smithi163 bash[75425]: audit 2023-10-09T09:30:24.146722+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:30:24.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:24 smithi163 bash[75425]: audit 2023-10-09T09:30:24.591230+0000 mon.a (mon.0) 1524 : audit 1 from='client.? 172.21.15.163:0/1771045425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:30:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:24 smithi163 bash[78107]: audit 2023-10-09T09:30:24.146343+0000 mon.c (mon.2) 58 : audit 1 from='client.? 172.21.15.163:0/455806017' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:30:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:24 smithi163 bash[78107]: audit 2023-10-09T09:30:24.146722+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:30:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:24 smithi163 bash[78107]: audit 2023-10-09T09:30:24.591230+0000 mon.a (mon.0) 1524 : audit 1 from='client.? 172.21.15.163:0/1771045425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:30:24.986 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:25.015 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:25.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:25.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:24 smithi181 bash[59345]: audit 2023-10-09T09:30:24.146343+0000 mon.c (mon.2) 58 : audit 1 from='client.? 172.21.15.163:0/455806017' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:30:25.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:24 smithi181 bash[59345]: audit 2023-10-09T09:30:24.146722+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T09:30:25.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:24 smithi181 bash[59345]: audit 2023-10-09T09:30:24.591230+0000 mon.a (mon.0) 1524 : audit 1 from='client.? 172.21.15.163:0/1771045425' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:30:25.119 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:25.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:25.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:25.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:25.248 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:25.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:25.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:25.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:25.413 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:25.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:25.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:25.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:25.544 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:25.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:25.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:25.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:25.672 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:25.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:25.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:25.775 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:25.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:25.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:25.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:25.903 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:25.927 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:25.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:25.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:26.029 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: cluster 2023-10-09T09:30:24.810103+0000 mon.a (mon.0) 1525 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:30:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: cluster 2023-10-09T09:30:24.810163+0000 mon.a (mon.0) 1526 : cluster 1 Cluster is now healthy 2023-10-09T09:30:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: cluster 2023-10-09T09:30:24.820035+0000 osd.5 (osd.5) 643 : cluster 3 failed to encode map e1099 with expected crc 2023-10-09T09:30:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: audit 2023-10-09T09:30:24.823119+0000 mon.a (mon.0) 1527 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T09:30:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: audit 2023-10-09T09:30:24.823245+0000 mon.a (mon.0) 1528 : audit 1 from='client.? 172.21.15.163:0/1771045425' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:30:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: cluster 2023-10-09T09:30:24.823312+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e1099: 8 total, 8 up, 8 in 2023-10-09T09:30:26.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: cluster 2023-10-09T09:30:24.825317+0000 osd.4 (osd.4) 525 : cluster 3 failed to encode map e1099 with expected crc 2023-10-09T09:30:26.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: cluster 2023-10-09T09:30:24.831161+0000 osd.7 (osd.7) 489 : cluster 3 failed to encode map e1099 with expected crc 2023-10-09T09:30:26.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: audit 2023-10-09T09:30:25.370940+0000 mon.a (mon.0) 1530 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:26.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: audit 2023-10-09T09:30:25.372290+0000 mon.a (mon.0) 1531 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:26.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[75425]: cluster 2023-10-09T09:30:25.698115+0000 mgr.y (mgr.34107) 724 : cluster 0 pgmap v1428: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:26.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: cluster 2023-10-09T09:30:24.810103+0000 mon.a (mon.0) 1525 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:30:26.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: cluster 2023-10-09T09:30:24.810163+0000 mon.a (mon.0) 1526 : cluster 1 Cluster is now healthy 2023-10-09T09:30:26.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: cluster 2023-10-09T09:30:24.820035+0000 osd.5 (osd.5) 643 : cluster 3 failed to encode map e1099 with expected crc 2023-10-09T09:30:26.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: audit 2023-10-09T09:30:24.823119+0000 mon.a (mon.0) 1527 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T09:30:26.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: audit 2023-10-09T09:30:24.823245+0000 mon.a (mon.0) 1528 : audit 1 from='client.? 172.21.15.163:0/1771045425' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:30:26.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: cluster 2023-10-09T09:30:24.823312+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e1099: 8 total, 8 up, 8 in 2023-10-09T09:30:26.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: cluster 2023-10-09T09:30:24.825317+0000 osd.4 (osd.4) 525 : cluster 3 failed to encode map e1099 with expected crc 2023-10-09T09:30:26.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: cluster 2023-10-09T09:30:24.831161+0000 osd.7 (osd.7) 489 : cluster 3 failed to encode map e1099 with expected crc 2023-10-09T09:30:26.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: audit 2023-10-09T09:30:25.370940+0000 mon.a (mon.0) 1530 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:26.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: audit 2023-10-09T09:30:25.372290+0000 mon.a (mon.0) 1531 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:26.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:25 smithi163 bash[78107]: cluster 2023-10-09T09:30:25.698115+0000 mgr.y (mgr.34107) 724 : cluster 0 pgmap v1428: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:26.062 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:26.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:26.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: cluster 2023-10-09T09:30:24.810103+0000 mon.a (mon.0) 1525 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:30:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: cluster 2023-10-09T09:30:24.810163+0000 mon.a (mon.0) 1526 : cluster 1 Cluster is now healthy 2023-10-09T09:30:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: cluster 2023-10-09T09:30:24.820035+0000 osd.5 (osd.5) 643 : cluster 3 failed to encode map e1099 with expected crc 2023-10-09T09:30:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: audit 2023-10-09T09:30:24.823119+0000 mon.a (mon.0) 1527 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T09:30:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: audit 2023-10-09T09:30:24.823245+0000 mon.a (mon.0) 1528 : audit 1 from='client.? 172.21.15.163:0/1771045425' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:30:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: cluster 2023-10-09T09:30:24.823312+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e1099: 8 total, 8 up, 8 in 2023-10-09T09:30:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: cluster 2023-10-09T09:30:24.825317+0000 osd.4 (osd.4) 525 : cluster 3 failed to encode map e1099 with expected crc 2023-10-09T09:30:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: cluster 2023-10-09T09:30:24.831161+0000 osd.7 (osd.7) 489 : cluster 3 failed to encode map e1099 with expected crc 2023-10-09T09:30:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: audit 2023-10-09T09:30:25.370940+0000 mon.a (mon.0) 1530 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: audit 2023-10-09T09:30:25.372290+0000 mon.a (mon.0) 1531 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:26.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:25 smithi181 bash[59345]: cluster 2023-10-09T09:30:25.698115+0000 mgr.y (mgr.34107) 724 : cluster 0 pgmap v1428: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:26.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:26.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:26.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:26.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:26.301 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:26.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T09:30:26.335 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:26.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:26.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:26.439 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:26.468 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:26.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:26.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:26.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:26.601 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:26.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:26.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:26.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:30:26.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:26.731 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:26.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:26.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:26.835 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:26.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:26.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:26.879 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:30:26.880 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:30:26.880 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:30:26.880 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:30:26.880 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:30:26.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[78107]: cluster 2023-10-09T09:30:25.823729+0000 osd.5 (osd.5) 644 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[78107]: cluster 2023-10-09T09:30:25.826928+0000 mon.a (mon.0) 1532 : cluster 0 osdmap e1100: 8 total, 8 up, 8 in 2023-10-09T09:30:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[78107]: cluster 2023-10-09T09:30:25.827165+0000 osd.4 (osd.4) 526 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[78107]: cluster 2023-10-09T09:30:25.827244+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[78107]: cluster 2023-10-09T09:30:25.827459+0000 osd.7 (osd.7) 490 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[78107]: cluster 2023-10-09T09:30:25.829344+0000 osd.5 (osd.5) 645 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:26.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[75425]: cluster 2023-10-09T09:30:25.823729+0000 osd.5 (osd.5) 644 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[75425]: cluster 2023-10-09T09:30:25.826928+0000 mon.a (mon.0) 1532 : cluster 0 osdmap e1100: 8 total, 8 up, 8 in 2023-10-09T09:30:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[75425]: cluster 2023-10-09T09:30:25.827165+0000 osd.4 (osd.4) 526 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[75425]: cluster 2023-10-09T09:30:25.827244+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[75425]: cluster 2023-10-09T09:30:25.827459+0000 osd.7 (osd.7) 490 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:26 smithi163 bash[75425]: cluster 2023-10-09T09:30:25.829344+0000 osd.5 (osd.5) 645 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:26.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:26.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:26.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:27.098 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:27.133 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:27.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:27.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:27.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:27.269 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:27.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:27.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:26 smithi181 bash[59345]: cluster 2023-10-09T09:30:25.823729+0000 osd.5 (osd.5) 644 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:26 smithi181 bash[59345]: cluster 2023-10-09T09:30:25.826928+0000 mon.a (mon.0) 1532 : cluster 0 osdmap e1100: 8 total, 8 up, 8 in 2023-10-09T09:30:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:26 smithi181 bash[59345]: cluster 2023-10-09T09:30:25.827165+0000 osd.4 (osd.4) 526 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:26 smithi181 bash[59345]: cluster 2023-10-09T09:30:25.827244+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:27.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:26 smithi181 bash[59345]: cluster 2023-10-09T09:30:25.827459+0000 osd.7 (osd.7) 490 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:27.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:26 smithi181 bash[59345]: cluster 2023-10-09T09:30:25.829344+0000 osd.5 (osd.5) 645 : cluster 3 failed to encode map e1100 with expected crc 2023-10-09T09:30:27.372 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:27.398 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:27.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:27.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:27.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:27.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:27.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:27.643 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T09:30:27.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:27.695 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:27.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:27.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:27.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:27.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:27.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:27.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:27.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:27.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:27 smithi181 bash[59345]: cluster 2023-10-09T09:30:26.824509+0000 osd.5 (osd.5) 646 : cluster 3 failed to encode map e1101 with expected crc 2023-10-09T09:30:27.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:27 smithi181 bash[59345]: cluster 2023-10-09T09:30:26.824752+0000 osd.7 (osd.7) 491 : cluster 3 failed to encode map e1101 with expected crc 2023-10-09T09:30:27.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:27 smithi181 bash[59345]: cluster 2023-10-09T09:30:26.828289+0000 mon.a (mon.0) 1533 : cluster 0 osdmap e1101: 8 total, 8 up, 8 in 2023-10-09T09:30:27.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:27 smithi181 bash[59345]: cluster 2023-10-09T09:30:27.698751+0000 mgr.y (mgr.34107) 725 : cluster 0 pgmap v1431: 89 pgs: 5 creating+peering, 1 unknown, 83 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:27.967 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:27 smithi163 bash[78107]: cluster 2023-10-09T09:30:26.824509+0000 osd.5 (osd.5) 646 : cluster 3 failed to encode map e1101 with expected crc 2023-10-09T09:30:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:27 smithi163 bash[78107]: cluster 2023-10-09T09:30:26.824752+0000 osd.7 (osd.7) 491 : cluster 3 failed to encode map e1101 with expected crc 2023-10-09T09:30:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:27 smithi163 bash[78107]: cluster 2023-10-09T09:30:26.828289+0000 mon.a (mon.0) 1533 : cluster 0 osdmap e1101: 8 total, 8 up, 8 in 2023-10-09T09:30:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:27 smithi163 bash[78107]: cluster 2023-10-09T09:30:27.698751+0000 mgr.y (mgr.34107) 725 : cluster 0 pgmap v1431: 89 pgs: 5 creating+peering, 1 unknown, 83 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:27.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:27 smithi163 bash[75425]: cluster 2023-10-09T09:30:26.824509+0000 osd.5 (osd.5) 646 : cluster 3 failed to encode map e1101 with expected crc 2023-10-09T09:30:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:27 smithi163 bash[75425]: cluster 2023-10-09T09:30:26.824752+0000 osd.7 (osd.7) 491 : cluster 3 failed to encode map e1101 with expected crc 2023-10-09T09:30:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:27 smithi163 bash[75425]: cluster 2023-10-09T09:30:26.828289+0000 mon.a (mon.0) 1533 : cluster 0 osdmap e1101: 8 total, 8 up, 8 in 2023-10-09T09:30:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:27 smithi163 bash[75425]: cluster 2023-10-09T09:30:27.698751+0000 mgr.y (mgr.34107) 725 : cluster 0 pgmap v1431: 89 pgs: 5 creating+peering, 1 unknown, 83 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:27.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:28.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:28.099 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:28.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:28.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:28.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:28.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:28.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:28.317 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:30:28.336 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:28.390 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:28.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:28.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:28.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:28.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:28.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:28.623 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:28.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:28.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:28.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:28.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:28.795 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:28.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:28.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:28.899 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:28.925 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:28.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:28.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:29.029 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:29.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:29.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:29.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:29.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:29.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:29.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:29.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:29.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:29 smithi163 bash[75425]: cluster 2023-10-09T09:30:27.721815+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e1101 with expected crc 2023-10-09T09:30:29.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:29 smithi163 bash[75425]: audit 2023-10-09T09:30:28.088800+0000 mon.a (mon.0) 1534 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:29.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:29 smithi163 bash[75425]: audit 2023-10-09T09:30:28.096580+0000 mon.a (mon.0) 1535 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:29.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:29 smithi163 bash[78107]: cluster 2023-10-09T09:30:27.721815+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e1101 with expected crc 2023-10-09T09:30:29.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:29 smithi163 bash[78107]: audit 2023-10-09T09:30:28.088800+0000 mon.a (mon.0) 1534 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:29.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:29 smithi163 bash[78107]: audit 2023-10-09T09:30:28.096580+0000 mon.a (mon.0) 1535 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:29.301 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:29.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:29 smithi181 bash[59345]: cluster 2023-10-09T09:30:27.721815+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e1101 with expected crc 2023-10-09T09:30:29.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:29 smithi181 bash[59345]: audit 2023-10-09T09:30:28.088800+0000 mon.a (mon.0) 1534 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:29.314 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:29 smithi181 bash[59345]: audit 2023-10-09T09:30:28.096580+0000 mon.a (mon.0) 1535 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:29.337 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:29.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:29.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:29.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:29.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:29.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:29.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:29.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:29.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:29.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:29.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:29.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:29.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:29.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:29.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:29.869 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:29.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:29.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:30.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:30.037 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:30.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:30.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:30.098 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:30 smithi163 bash[75425]: cluster 2023-10-09T09:30:29.699326+0000 mgr.y (mgr.34107) 726 : cluster 0 pgmap v1432: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:30.098 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:30 smithi163 bash[75425]: audit 2023-10-09T09:30:30.002955+0000 mon.a (mon.0) 1536 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:30.098 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:30 smithi163 bash[75425]: audit 2023-10-09T09:30:30.010422+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:30.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:30.175 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:30.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:30.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:30.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:30.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:30.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:30.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:30 smithi181 bash[59345]: cluster 2023-10-09T09:30:29.699326+0000 mgr.y (mgr.34107) 726 : cluster 0 pgmap v1432: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:30 smithi181 bash[59345]: audit 2023-10-09T09:30:30.002955+0000 mon.a (mon.0) 1536 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:30 smithi181 bash[59345]: audit 2023-10-09T09:30:30.010422+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:30.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:30.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:30.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:30.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:30.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:30 smithi163 bash[78107]: cluster 2023-10-09T09:30:29.699326+0000 mgr.y (mgr.34107) 726 : cluster 0 pgmap v1432: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:30.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:30 smithi163 bash[78107]: audit 2023-10-09T09:30:30.002955+0000 mon.a (mon.0) 1536 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:30.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:30 smithi163 bash[78107]: audit 2023-10-09T09:30:30.010422+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:30.546 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:30.577 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:30.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:30.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:30.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:30.709 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:30.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:30.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:30.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:30.844 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:30.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:30.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:30.860 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:30.861 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:30.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:30.977 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:30.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:30.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:31.081 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:31.112 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:31.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:31.215 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:31.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:31 smithi163 bash[78107]: audit 2023-10-09T09:30:30.530470+0000 mon.a (mon.0) 1538 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:30:31.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:31 smithi163 bash[78107]: audit 2023-10-09T09:30:30.532057+0000 mon.a (mon.0) 1539 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:30:31.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:31 smithi163 bash[78107]: audit 2023-10-09T09:30:30.542495+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:31.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:31 smithi163 bash[75425]: audit 2023-10-09T09:30:30.530470+0000 mon.a (mon.0) 1538 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:30:31.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:31 smithi163 bash[75425]: audit 2023-10-09T09:30:30.532057+0000 mon.a (mon.0) 1539 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:30:31.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:31 smithi163 bash[75425]: audit 2023-10-09T09:30:30.542495+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:31.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:31.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:31.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:31.374 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:31.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:31.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:31.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:31.491 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:31 smithi181 bash[59345]: audit 2023-10-09T09:30:30.530470+0000 mon.a (mon.0) 1538 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:30:31.491 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:31 smithi181 bash[59345]: audit 2023-10-09T09:30:30.532057+0000 mon.a (mon.0) 1539 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:30:31.491 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:31 smithi181 bash[59345]: audit 2023-10-09T09:30:30.542495+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:31.510 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:31.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:31.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:31.614 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:31.645 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:31.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:31.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:31.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:31.774 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:31.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:31.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:31.878 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:31.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:31.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:31.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:32.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:32.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:32.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:32.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:32.141 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:32.170 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:32.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:32.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:32.274 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:32.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:32 smithi163 bash[75425]: cluster 2023-10-09T09:30:31.699946+0000 mgr.y (mgr.34107) 727 : cluster 0 pgmap v1433: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:32.285 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:32 smithi163 bash[78107]: cluster 2023-10-09T09:30:31.699946+0000 mgr.y (mgr.34107) 727 : cluster 0 pgmap v1433: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:32.303 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:32.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:32.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:32.408 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:32.421 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:30:32.422 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:32 smithi181 bash[59345]: cluster 2023-10-09T09:30:31.699946+0000 mgr.y (mgr.34107) 727 : cluster 0 pgmap v1433: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:32.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:32.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:32.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:32.547 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:32.572 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:32.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:32.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:32.676 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:32.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:32.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:32.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:32.805 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:32.832 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:32.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:32.888 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:32.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:32.962 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:32.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:32.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:33.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:33.094 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:33.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:33.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:33.197 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:33.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:33.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:33.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:33.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:33.363 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:33.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:33.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:33.466 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:33.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:33.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:33.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:33.598 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:33.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:33.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:33.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:33.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:33.747 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:30:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:30:33] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.33.4" 2023-10-09T09:30:33.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:33.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:33.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:33.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:33.885 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:33 smithi181 bash[59345]: cluster 2023-10-09T09:30:33.700581+0000 mgr.y (mgr.34107) 728 : cluster 0 pgmap v1434: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:33.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:33.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:34.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:34.034 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:34.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:34.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:34.092 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:33 smithi163 bash[78107]: cluster 2023-10-09T09:30:33.700581+0000 mgr.y (mgr.34107) 728 : cluster 0 pgmap v1434: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:34.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:33 smithi163 bash[75425]: cluster 2023-10-09T09:30:33.700581+0000 mgr.y (mgr.34107) 728 : cluster 0 pgmap v1434: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:34.138 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:34.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:34.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:34.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:34.268 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:34.297 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:34.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:34.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:34.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:34.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:34.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:34.536 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:34.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:34.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:34.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:34.673 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:34.703 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:34.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:34.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:34.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:34.835 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:34.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:34.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:34.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:34.960 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:34.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:34.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:35.063 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:35.093 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:35.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:35.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:35.221 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:35.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:35.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:35.350 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:35.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:35.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:35.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:35.482 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:35.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:35.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:35.585 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:35.614 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:35.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:35.618 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:30:35.619 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T09:30:35.619 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T09:30:35.619 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:30:35.619 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.619 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:148018 6.1 e1091/1076 6:824f758f:::benchmark_data_smithi163_107771_object37258:head v 1091'9124, mlcod=1091'9122)", 2023-10-09T09:30:35.619 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:03.443265+0000", 2023-10-09T09:30:35.619 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 152.16910480000001, 2023-10-09T09:30:35.619 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.028525028000000001, 2023-10-09T09:30:35.619 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.619 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:03.443265+0000", 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:03.443265+0000", 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.620 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:03.443264+0000", 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7200000000000006e-06 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:03.443271+0000", 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0101e-05 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:03.443282+0000", 2023-10-09T09:30:35.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3929999999999999e-06 2023-10-09T09:30:35.622 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.622 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.622 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:03.443287+0000", 2023-10-09T09:30:35.622 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5360999999999998e-05 2023-10-09T09:30:35.622 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.622 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.622 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:03.443372+0000", 2023-10-09T09:30:35.622 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9399e-05 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:03.443412+0000", 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.028336244999999999 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:03.471748+0000", 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1619999999999998e-05 2023-10-09T09:30:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:03.471790+0000", 2023-10-09T09:30:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.028525028000000001 2023-10-09T09:30:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:174725 6.a e1093/1076 6:5a134bde:::benchmark_data_smithi163_107771_object63965:head v 1093'11107, mlcod=1093'11105)", 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:41.714789+0000", 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 113.897580838, 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029845903, 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:41.714789+0000", 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:41.714789+0000", 2023-10-09T09:30:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:41.714787+0000", 2023-10-09T09:30:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6720000000000004e-06 2023-10-09T09:30:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:41.714795+0000", 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4278e-05 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:41.714809+0000", 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7710000000000001e-06 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:41.714816+0000", 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00033216499999999999 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:41.715148+0000", 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9783000000000001e-05 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:41.715198+0000", 2023-10-09T09:30:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029400674000000002 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:41.744599+0000", 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5623000000000001e-05 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:41.744634+0000", 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029845903 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:178277 6.a e1093/1076 6:5541cd52:::benchmark_data_smithi163_107771_object67517:head v 1093'11321, mlcod=1093'11317)", 2023-10-09T09:30:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:46.787700+0000", 2023-10-09T09:30:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 108.824669008, 2023-10-09T09:30:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.033778909000000003, 2023-10-09T09:30:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:46.787700+0000", 2023-10-09T09:30:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:46.787700+0000", 2023-10-09T09:30:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:30:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:46.787700+0000", 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9530000000000002e-06 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:46.787707+0000", 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2432000000000001e-05 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:46.787719+0000", 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6749999999999996e-06 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:46.787726+0000", 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9132999999999999e-05 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:46.787775+0000", 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1437000000000002e-05 2023-10-09T09:30:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:46.787796+0000", 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.033558193 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:46.821354+0000", 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012478599999999999 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:46.821479+0000", 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.033778909000000003 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:181557 6.a e1093/1076 6:5cf1930e:::benchmark_data_smithi163_107771_object70797:head v 1093'11560, mlcod=1093'11557)", 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:51.557776+0000", 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 104.054593692, 2023-10-09T09:30:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.036250219, 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.557776+0000", 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.557776+0000", 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.557775+0000", 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3299999999999999e-06 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.557783+0000", 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.26e-05 2023-10-09T09:30:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.557796+0000", 2023-10-09T09:30:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.618e-06 2023-10-09T09:30:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.557802+0000", 2023-10-09T09:30:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093990870000000004 2023-10-09T09:30:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.567201+0000", 2023-10-09T09:30:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6260999999999999e-05 2023-10-09T09:30:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.567218+0000", 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026733216000000001 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.593951+0000", 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4950000000000006e-05 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.594026+0000", 2023-10-09T09:30:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.036250219 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:181558 6.a e1093/1076 6:5d9e3ceb:::benchmark_data_smithi163_107771_object70798:head v 1093'11561, mlcod=1093'11557)", 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:51.567575+0000", 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 104.044794861, 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026481805000000001, 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.567575+0000", 2023-10-09T09:30:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.567575+0000", 2023-10-09T09:30:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.567574+0000", 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4469999999999995e-06 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.567583+0000", 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2281e-05 2023-10-09T09:30:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.567595+0000", 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.117e-06 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.567601+0000", 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064997600000000003 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.574101+0000", 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2105999999999999e-05 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.574153+0000", 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01987848 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.594032+0000", 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4524000000000001e-05 2023-10-09T09:30:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.594056+0000", 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026481805000000001 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:181741 6.5 e1093/1076 6:ae076df0:::benchmark_data_smithi163_107771_object70981:head v 1093'11437, mlcod=1093'11435)", 2023-10-09T09:30:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:51.811113+0000", 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 103.801256028, 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027263524000000001, 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.811113+0000", 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.811113+0000", 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.811112+0000", 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4595e-05 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.811126+0000", 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3331000000000001e-05 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.811150+0000", 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0498e-05 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.811160+0000", 2023-10-09T09:30:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000114614 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.811275+0000", 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6718000000000006e-05 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.811362+0000", 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026968981999999999 2023-10-09T09:30:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.838331+0000", 2023-10-09T09:30:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6304000000000003e-05 2023-10-09T09:30:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.838377+0000", 2023-10-09T09:30:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027263524000000001 2023-10-09T09:30:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:183441 6.5 e1093/1076 6:a2489c12:::benchmark_data_smithi163_107771_object72681:head v 1093'11560, mlcod=1093'11557)", 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:54.192743+0000", 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.41962653, 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027090828000000001, 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.192743+0000", 2023-10-09T09:30:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.192743+0000", 2023-10-09T09:30:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:30:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.192741+0000", 2023-10-09T09:30:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1191e-05 2023-10-09T09:30:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.192753+0000", 2023-10-09T09:30:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2811e-05 2023-10-09T09:30:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.192765+0000", 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7819999999999994e-06 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.192773+0000", 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0038274110000000002 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.196601+0000", 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7635999999999999e-05 2023-10-09T09:30:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.196618+0000", 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.023188231 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.219806+0000", 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7211000000000001e-05 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.219834+0000", 2023-10-09T09:30:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027090828000000001 2023-10-09T09:30:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:183833 6.a e1093/1076 6:5ffd5546:::benchmark_data_smithi163_107771_object73073:head v 1093'11686, mlcod=1093'11684)", 2023-10-09T09:30:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:54.711779+0000", 2023-10-09T09:30:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 100.900590622, 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.035993393999999998, 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.711779+0000", 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.711779+0000", 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.711778+0000", 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2760000000000006e-06 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.711787+0000", 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3320999999999999e-05 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.711810+0000", 2023-10-09T09:30:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8480000000000003e-06 2023-10-09T09:30:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.711817+0000", 2023-10-09T09:30:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8359000000000001e-05 2023-10-09T09:30:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.711876+0000", 2023-10-09T09:30:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3478000000000001e-05 2023-10-09T09:30:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.711899+0000", 2023-10-09T09:30:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.035833454000000001 2023-10-09T09:30:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.747733+0000", 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9578e-05 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.747772+0000", 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.035993393999999998 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:183835 6.a e1093/1076 6:50b7d62c:::benchmark_data_smithi163_107771_object73075:head v 1093'11687, mlcod=1093'11684)", 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:54.717564+0000", 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 100.894805013, 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.030233612, 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.717564+0000", 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.717564+0000", 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.717563+0000", 2023-10-09T09:30:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1864e-05 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.717575+0000", 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5248999999999999e-05 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.717590+0000", 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.14e-06 2023-10-09T09:30:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.717598+0000", 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001593915 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.719192+0000", 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3421000000000002e-05 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.719216+0000", 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.028564583000000001 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.747780+0000", 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7702000000000001e-05 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.747798+0000", 2023-10-09T09:30:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.030233612 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:183841 6.a e1093/1076 6:5af071d1:::benchmark_data_smithi163_107771_object73081:head v 1093'11689, mlcod=1093'11684)", 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:54.729630+0000", 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 100.882739624, 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.032235307999999997, 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.729630+0000", 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.729630+0000", 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.729629+0000", 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7530000000000007e-06 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.729638+0000", 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4154e-05 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.729652+0000", 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3319999999999992e-06 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.729660+0000", 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0069516980000000001 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.736612+0000", 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3609999999999999e-05 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.736635+0000", 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.025188413999999999 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.761824+0000", 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1261999999999999e-05 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:54.761865+0000", 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.032235307999999997 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:186992 6.3 e1094/1076 6:cf53918c:::benchmark_data_smithi163_107771_object76232:head v 1094'11494, mlcod=1094'11491)", 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:59.195334+0000", 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 96.417035572000003, 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.028978319999999998, 2023-10-09T09:30:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.195334+0000", 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.195334+0000", 2023-10-09T09:30:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.195333+0000", 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3340000000000001e-06 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.195339+0000", 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0210000000000002e-06 2023-10-09T09:30:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.195346+0000", 2023-10-09T09:30:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5750000000000002e-06 2023-10-09T09:30:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.195350+0000", 2023-10-09T09:30:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004078941 2023-10-09T09:30:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.199429+0000", 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7665000000000001e-05 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.199447+0000", 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.024827488000000002 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.224274+0000", 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7710999999999999e-05 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.224312+0000", 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.028978319999999998 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:187215 6.5 e1094/1076 6:a47da99e:::benchmark_data_smithi163_107771_object76455:head v 1094'11808, mlcod=1094'11806)", 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:59.482997+0000", 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 96.129372767999996, 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.032947160000000003, 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.482997+0000", 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.482997+0000", 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.482996+0000", 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.383e-06 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.483002+0000", 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8179999999999998e-06 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.483008+0000", 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2470000000000002e-06 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.483013+0000", 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040188400000000001 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.487031+0000", 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4590999999999999e-05 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.487056+0000", 2023-10-09T09:30:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.028867734999999999 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.515924+0000", 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0026000000000001e-05 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.515944+0000", 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.032947160000000003 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:187218 6.5 e1094/1076 6:a63be5f8:::benchmark_data_smithi163_107771_object76458:head v 1094'11809, mlcod=1094'11806)", 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:59.490310+0000", 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 96.122059113000006, 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.025648671000000001, 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.490310+0000", 2023-10-09T09:30:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.490310+0000", 2023-10-09T09:30:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:30:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.490310+0000", 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8434999999999999e-05 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.490328+0000", 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3240000000000003e-06 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.490336+0000", 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1499999999999998e-06 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.490342+0000", 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0065379399999999999 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.496880+0000", 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6109e-05 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.496906+0000", 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.019042100999999999 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.515948+0000", 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1201e-05 2023-10-09T09:30:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:59.515959+0000", 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.025648671000000001 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:201475 6.1 e1095/1076 6:8ccd2517:::benchmark_data_smithi163_107771_object90715:head v 1095'12471, mlcod=1095'12468)", 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:19.578186+0000", 2023-10-09T09:30:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 76.034183103999993, 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026916183999999999, 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:19.578186+0000", 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:19.578186+0000", 2023-10-09T09:30:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:19.578185+0000", 2023-10-09T09:30:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1049e-05 2023-10-09T09:30:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:19.578196+0000", 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6053999999999999e-05 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:19.578212+0000", 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4409999999999993e-06 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:19.578221+0000", 2023-10-09T09:30:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016712000000000001 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:19.578388+0000", 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5340999999999997e-05 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:19.578453+0000", 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026601672 2023-10-09T09:30:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:19.605055+0000", 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7587999999999999e-05 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:19.605102+0000", 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026916183999999999 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:202143 6.5 e1095/1076 6:a6216252:::benchmark_data_smithi163_107771_object91383:head v 1095'12732, mlcod=1095'12730)", 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:20.669131+0000", 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 74.943238057000002, 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.030242699000000001, 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:20.669131+0000", 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:20.669131+0000", 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:20.669129+0000", 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7744000000000002e-05 2023-10-09T09:30:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:20.669147+0000", 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4471999999999999e-05 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:20.669171+0000", 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3464e-05 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:20.669185+0000", 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014255800000000001 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:20.669327+0000", 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00078237700000000001 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:20.670110+0000", 2023-10-09T09:30:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029224344999999999 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:20.699334+0000", 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0070000000000001e-05 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:20.699374+0000", 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.030242699000000001 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:203890 6.a e1095/1076 6:56c0d08c:::benchmark_data_smithi163_107771_object93130:head v 1095'12988, mlcod=1095'12983)", 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:23.501239+0000", 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 72.111130067000005, 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027069886000000001, 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:23.501239+0000", 2023-10-09T09:30:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:23.501239+0000", 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:23.501239+0000", 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3190000000000001e-06 2023-10-09T09:30:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:23.501248+0000", 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2989999999999999e-06 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:23.501256+0000", 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6319999999999999e-06 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:23.501260+0000", 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018452638 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:23.519713+0000", 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9345000000000001e-05 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:23.519742+0000", 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085314029999999999 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:23.528274+0000", 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5602000000000002e-05 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:23.528309+0000", 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027069886000000001 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:208308 6.5 e1095/1076 6:a5b5180c:::benchmark_data_smithi163_107771_object97548:head v 1095'13119, mlcod=1095'13115)", 2023-10-09T09:30:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:29.942609+0000", 2023-10-09T09:30:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 65.669760112000006, 2023-10-09T09:30:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.031128157, 2023-10-09T09:30:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942609+0000", 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942609+0000", 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942608+0000", 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1910000000000006e-06 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942618+0000", 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2269999999999999e-05 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942630+0000", 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6939999999999998e-06 2023-10-09T09:30:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942636+0000", 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0087966520000000003 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.951433+0000", 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7081e-05 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.951460+0000", 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018663705999999999 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.970124+0000", 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00361349 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.973737+0000", 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.031128157 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:209648 6.c e1095/1076 6:380bb873:::benchmark_data_smithi163_107771_object98888:head v 1095'13345, mlcod=1095'13343)", 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:32.046851+0000", 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 63.565518212999997, 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027323077000000001, 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.046851+0000", 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.046851+0000", 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.046850+0000", 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.149e-06 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.046859+0000", 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0811000000000001e-05 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.046880+0000", 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3806e-05 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.046894+0000", 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1217000000000001e-05 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.046945+0000", 2023-10-09T09:30:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3519e-05 2023-10-09T09:30:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.046969+0000", 2023-10-09T09:30:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027063653999999999 2023-10-09T09:30:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.074032+0000", 2023-10-09T09:30:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014176999999999999 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.074174+0000", 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027323077000000001 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:209657 6.c e1095/1076 6:3f4b99d6:::benchmark_data_smithi163_107771_object98897:head v 1095'13347, mlcod=1095'13343)", 2023-10-09T09:30:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:32.058897+0000", 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 63.553472458999998, 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027463156999999998, 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.058897+0000", 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.058897+0000", 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.058896+0000", 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9500000000000007e-06 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.058905+0000", 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8289999999999999e-05 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.058923+0000", 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1019999999999997e-06 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.058931+0000", 2023-10-09T09:30:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015446603999999999 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.074378+0000", 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4829e-05 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.074393+0000", 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011861782 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.086254+0000", 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105604 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.086360+0000", 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027463156999999998 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.34791.0:210020 6.a e1095/1076 6:55bdabd7:::benchmark_data_smithi163_107771_object99260:head v 1095'13362, mlcod=1095'13359)", 2023-10-09T09:30:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:32.556429+0000", 2023-10-09T09:30:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 63.055940706000001, 2023-10-09T09:30:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026009931, 2023-10-09T09:30:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:30:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:30:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:30:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:30:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.556429+0000", 2023-10-09T09:30:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:30:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:30:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.556429+0000", 2023-10-09T09:30:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.556428+0000", 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1959999999999997e-06 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.556437+0000", 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4618e-05 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.556452+0000", 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.657e-06 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.556458+0000", 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4523000000000004e-05 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.556543+0000", 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5523e-05 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.556558+0000", 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.025763423000000001 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:30:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.582322+0000", 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116765 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:32.582439+0000", 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026009931 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:30:35.727 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:30:35.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:35.750 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:35.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:35.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:35.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:35.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:35.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:35.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:35.984 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:36.007 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:36.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:36.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:36.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:36.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:36.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:36.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:36.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:36.269 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:36.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:36.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:36.329 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T09:30:36.372 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:36.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:36.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:36.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:36.539 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:36.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:36.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:36.669 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:36.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:36.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:36.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:36.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:36 smithi163 bash[78107]: cluster 2023-10-09T09:30:35.701149+0000 mgr.y (mgr.34107) 729 : cluster 0 pgmap v1435: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:36.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:36 smithi163 bash[78107]: audit 2023-10-09T09:30:35.757404+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:36.785 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:36 smithi163 bash[75425]: cluster 2023-10-09T09:30:35.701149+0000 mgr.y (mgr.34107) 729 : cluster 0 pgmap v1435: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:36.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:36 smithi163 bash[75425]: audit 2023-10-09T09:30:35.757404+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:36.802 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:36.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:36.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:36 smithi181 bash[59345]: cluster 2023-10-09T09:30:35.701149+0000 mgr.y (mgr.34107) 729 : cluster 0 pgmap v1435: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:36 smithi181 bash[59345]: audit 2023-10-09T09:30:35.757404+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:36.906 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:36.940 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:36.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:37.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:37.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:37.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:37.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:37.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:37.201 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:37.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:37.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:37.304 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:37.336 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:37.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:37.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:37.439 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:37.463 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:37.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:37.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:37.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:37.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:37.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:37.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:37.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:37.719 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:37.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:37.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:37.821 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:37.843 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:37.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:37.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:37.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:37 smithi163 bash[78107]: cluster 2023-10-09T09:30:37.701975+0000 mgr.y (mgr.34107) 730 : cluster 0 pgmap v1436: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:37.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:37 smithi163 bash[75425]: cluster 2023-10-09T09:30:37.701975+0000 mgr.y (mgr.34107) 730 : cluster 0 pgmap v1436: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:37.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:37.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:37.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:37.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:38.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:37 smithi181 bash[59345]: cluster 2023-10-09T09:30:37.701975+0000 mgr.y (mgr.34107) 730 : cluster 0 pgmap v1436: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:38.076 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:38.106 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:38.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:38.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:38.210 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:38.240 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:38.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:38.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:38.344 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:38.370 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:38.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:38.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:38.474 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:38.501 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:38.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:38.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:38.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:38.628 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:38.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:38.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:38.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:38.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:38.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:38.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:38.861 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:38.891 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:38.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:38.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:38.956 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:30:38.982 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:30:38.994 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:39.003 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:30:39.025 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:39.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:39.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:39.032 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:30:39.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:30:39.082 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:30:39.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:39.155 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:30:39.155 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:30:39.155 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:30:39.155 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:30:39.155 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:30:39.156 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:30:39.156 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:30:39.156 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:30:39.156 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:30:39.156 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:30:39.175 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:39.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:39.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:39.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:38 smithi163 bash[78107]: audit 2023-10-09T09:30:38.929261+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.163:0/683528959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:30:39.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:38 smithi163 bash[75425]: audit 2023-10-09T09:30:38.929261+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.163:0/683528959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:30:39.278 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:39.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:39.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:39.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:38 smithi181 bash[59345]: audit 2023-10-09T09:30:38.929261+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.163:0/683528959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:30:39.403 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:39.433 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:39.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:39.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:39.537 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:39.569 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:39.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:39.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:39.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:39.692 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:39.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:39.796 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:39.822 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:39.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:39.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:39.925 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:39.952 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:39.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:39.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:39.998 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:30:39.998 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T09:30:40.055 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:40.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:40.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:40.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:40.192 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:40.206 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:39 smithi163 bash[75425]: audit 2023-10-09T09:30:39.071985+0000 mon.a (mon.0) 1542 : audit 1 from='client.? 172.21.15.163:0/582361248' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:30:40.206 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:39 smithi163 bash[75425]: audit 2023-10-09T09:30:39.609341+0000 mon.a (mon.0) 1543 : audit 1 from='client.? 172.21.15.163:0/1984106499' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:30:40.206 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:39 smithi163 bash[75425]: cluster 2023-10-09T09:30:39.702471+0000 mgr.y (mgr.34107) 731 : cluster 0 pgmap v1437: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:40.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:39 smithi163 bash[78107]: audit 2023-10-09T09:30:39.071985+0000 mon.a (mon.0) 1542 : audit 1 from='client.? 172.21.15.163:0/582361248' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:30:40.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:39 smithi163 bash[78107]: audit 2023-10-09T09:30:39.609341+0000 mon.a (mon.0) 1543 : audit 1 from='client.? 172.21.15.163:0/1984106499' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:30:40.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:39 smithi163 bash[78107]: cluster 2023-10-09T09:30:39.702471+0000 mgr.y (mgr.34107) 731 : cluster 0 pgmap v1437: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:40.224 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:40.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:40.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:40.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:39 smithi181 bash[59345]: audit 2023-10-09T09:30:39.071985+0000 mon.a (mon.0) 1542 : audit 1 from='client.? 172.21.15.163:0/582361248' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:30:40.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:39 smithi181 bash[59345]: audit 2023-10-09T09:30:39.609341+0000 mon.a (mon.0) 1543 : audit 1 from='client.? 172.21.15.163:0/1984106499' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:30:40.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:39 smithi181 bash[59345]: cluster 2023-10-09T09:30:39.702471+0000 mgr.y (mgr.34107) 731 : cluster 0 pgmap v1437: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:40.328 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:40.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:40.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:40.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:40.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:40.492 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:40.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:40.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:40.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:40.626 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:40.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:40.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:40.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:40.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:40.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:40.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:40.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:40.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:40.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:40.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:40.998 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:41.024 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:41.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:41.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:40 smithi181 bash[59345]: cluster 2023-10-09T09:30:39.981978+0000 mon.a (mon.0) 1544 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:30:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:40 smithi181 bash[59345]: cluster 2023-10-09T09:30:39.991665+0000 osd.5 (osd.5) 647 : cluster 3 failed to encode map e1102 with expected crc 2023-10-09T09:30:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:40 smithi181 bash[59345]: cluster 2023-10-09T09:30:39.992359+0000 osd.7 (osd.7) 492 : cluster 3 failed to encode map e1102 with expected crc 2023-10-09T09:30:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:40 smithi181 bash[59345]: audit 2023-10-09T09:30:39.994059+0000 mon.a (mon.0) 1545 : audit 1 from='client.? 172.21.15.163:0/582361248' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:30:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:40 smithi181 bash[59345]: audit 2023-10-09T09:30:39.994158+0000 mon.a (mon.0) 1546 : audit 1 from='client.? 172.21.15.163:0/1984106499' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:30:41.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:40 smithi181 bash[59345]: cluster 2023-10-09T09:30:39.994245+0000 mon.a (mon.0) 1547 : cluster 0 osdmap e1102: 8 total, 8 up, 8 in 2023-10-09T09:30:41.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:40 smithi181 bash[59345]: audit 2023-10-09T09:30:40.371059+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:41.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:40 smithi181 bash[59345]: audit 2023-10-09T09:30:40.372210+0000 mon.a (mon.0) 1549 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:41.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:40 smithi181 bash[59345]: cluster 2023-10-09T09:30:40.562131+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e1102 with expected crc 2023-10-09T09:30:41.110 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:30:41.127 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:41.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[78107]: cluster 2023-10-09T09:30:39.981978+0000 mon.a (mon.0) 1544 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:30:41.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[78107]: cluster 2023-10-09T09:30:39.991665+0000 osd.5 (osd.5) 647 : cluster 3 failed to encode map e1102 with expected crc 2023-10-09T09:30:41.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[78107]: cluster 2023-10-09T09:30:39.992359+0000 osd.7 (osd.7) 492 : cluster 3 failed to encode map e1102 with expected crc 2023-10-09T09:30:41.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[78107]: audit 2023-10-09T09:30:39.994059+0000 mon.a (mon.0) 1545 : audit 1 from='client.? 172.21.15.163:0/582361248' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:30:41.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[78107]: audit 2023-10-09T09:30:39.994158+0000 mon.a (mon.0) 1546 : audit 1 from='client.? 172.21.15.163:0/1984106499' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:30:41.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[78107]: cluster 2023-10-09T09:30:39.994245+0000 mon.a (mon.0) 1547 : cluster 0 osdmap e1102: 8 total, 8 up, 8 in 2023-10-09T09:30:41.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[78107]: audit 2023-10-09T09:30:40.371059+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:41.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[78107]: audit 2023-10-09T09:30:40.372210+0000 mon.a (mon.0) 1549 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:41.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[78107]: cluster 2023-10-09T09:30:40.562131+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e1102 with expected crc 2023-10-09T09:30:41.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[75425]: cluster 2023-10-09T09:30:39.981978+0000 mon.a (mon.0) 1544 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:30:41.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[75425]: cluster 2023-10-09T09:30:39.991665+0000 osd.5 (osd.5) 647 : cluster 3 failed to encode map e1102 with expected crc 2023-10-09T09:30:41.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[75425]: cluster 2023-10-09T09:30:39.992359+0000 osd.7 (osd.7) 492 : cluster 3 failed to encode map e1102 with expected crc 2023-10-09T09:30:41.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[75425]: audit 2023-10-09T09:30:39.994059+0000 mon.a (mon.0) 1545 : audit 1 from='client.? 172.21.15.163:0/582361248' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:30:41.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[75425]: audit 2023-10-09T09:30:39.994158+0000 mon.a (mon.0) 1546 : audit 1 from='client.? 172.21.15.163:0/1984106499' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:30:41.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[75425]: cluster 2023-10-09T09:30:39.994245+0000 mon.a (mon.0) 1547 : cluster 0 osdmap e1102: 8 total, 8 up, 8 in 2023-10-09T09:30:41.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[75425]: audit 2023-10-09T09:30:40.371059+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:30:41.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[75425]: audit 2023-10-09T09:30:40.372210+0000 mon.a (mon.0) 1549 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:41.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:40 smithi163 bash[75425]: cluster 2023-10-09T09:30:40.562131+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e1102 with expected crc 2023-10-09T09:30:41.152 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:41.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:41.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:41.284 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:41.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:41.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:41.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:41.416 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:41.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:41.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:41.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:41.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:41.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:41.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:41.683 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:41.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:41.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:41.786 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:41.819 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:41.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:41.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:41.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:41.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:41.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:41.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:42.034 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:30:42.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:42.063 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:41 smithi181 bash[59345]: cluster 2023-10-09T09:30:41.703425+0000 mgr.y (mgr.34107) 732 : cluster 0 pgmap v1439: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:42.091 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:42.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:42.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:42.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:42.221 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:42.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:42.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:42.324 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:42.338 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:41 smithi163 bash[75425]: cluster 2023-10-09T09:30:41.703425+0000 mgr.y (mgr.34107) 732 : cluster 0 pgmap v1439: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:42.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:41 smithi163 bash[78107]: cluster 2023-10-09T09:30:41.703425+0000 mgr.y (mgr.34107) 732 : cluster 0 pgmap v1439: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:30:42.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:42.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:42.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:42.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:42.467 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T09:30:42.467 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2023-10-09T09:30:42.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:30:42.492 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:42.494 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:30:42.494 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:30:42.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:42.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:42.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T09:30:42.539 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:30:42.539 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_130508 2023-10-09T09:30:42.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:30:42.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:30:42.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:42.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:42.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:42.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:42.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:42.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:42.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:42.859 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:42.888 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:42.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:42.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:42.992 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:43.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:43.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:43.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:43.164 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:43.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:43.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:43.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:43.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:43.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:43.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:43.437 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:43.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:43.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:43.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:30:43.539 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 2800 2784 173.959 174 0.00464303 0.00572547 2023-10-09T09:30:43.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:43.548 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:30:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:30:43] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.33.4" 2023-10-09T09:30:43.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:43.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:43.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:43.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:43.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:43.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:43.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:43.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:43.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:43.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:43.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:43.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:43.964 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:43.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:43.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:43 smithi181 bash[59345]: cluster 2023-10-09T09:30:43.704893+0000 mgr.y (mgr.34107) 733 : cluster 0 pgmap v1440: 89 pgs: 89 active+clean; 10 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1004 KiB/s wr, 15 op/s 2023-10-09T09:30:44.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:44.076 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:43 smithi163 bash[75425]: cluster 2023-10-09T09:30:43.704893+0000 mgr.y (mgr.34107) 733 : cluster 0 pgmap v1440: 89 pgs: 89 active+clean; 10 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1004 KiB/s wr, 15 op/s 2023-10-09T09:30:44.076 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:43 smithi163 bash[78107]: cluster 2023-10-09T09:30:43.704893+0000 mgr.y (mgr.34107) 733 : cluster 0 pgmap v1440: 89 pgs: 89 active+clean; 10 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1004 KiB/s wr, 15 op/s 2023-10-09T09:30:44.095 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:44.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:44.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:44.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:44.228 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:44.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:44.322 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:30:44.322 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:30:44.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:44.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:44.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:44.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:44.469 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:44.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:44.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:44.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:44.539 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 5388 5372 167.847 161.75 0.00943694 0.00593768 2023-10-09T09:30:44.607 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:44.637 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:44.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:44.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:44.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:44.769 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:44.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:44.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:44.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:44.901 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:44.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:44.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:45.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:45.027 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:45.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:45.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:45.130 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:45.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:45.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:45.260 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:45.287 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:45.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:45.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:45.420 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:45.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:45.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:45.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T09:30:45.524 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:45.539 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 7979 7963 165.872 161.938 0.00465001 0.00601536 2023-10-09T09:30:45.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:45.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:45.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:45.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:45.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:45.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:45.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:45.805 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:45.835 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:45.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:45.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:45.938 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:45.965 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:45.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:45.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:46.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:46.101 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:46.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:46.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:46.233 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:46.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:46.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:46.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:46.366 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:46.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:46.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:46.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:46.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:46.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:46.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:46.539 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 10385 10369 161.994 150.375 0.00764524 0.00616283 2023-10-09T09:30:46.600 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:46.629 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:46.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:46.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:46.734 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:46.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:46 smithi163 bash[75425]: cluster 2023-10-09T09:30:45.705554+0000 mgr.y (mgr.34107) 734 : cluster 0 pgmap v1441: 89 pgs: 89 active+clean; 116 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s wr, 184 op/s 2023-10-09T09:30:46.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:46 smithi163 bash[78107]: cluster 2023-10-09T09:30:45.705554+0000 mgr.y (mgr.34107) 734 : cluster 0 pgmap v1441: 89 pgs: 89 active+clean; 116 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s wr, 184 op/s 2023-10-09T09:30:46.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:46.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:46.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:46 smithi181 bash[59345]: cluster 2023-10-09T09:30:45.705554+0000 mgr.y (mgr.34107) 734 : cluster 0 pgmap v1441: 89 pgs: 89 active+clean; 116 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s wr, 184 op/s 2023-10-09T09:30:46.824 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:46.825 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:46.870 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:46.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:46.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:46.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:47.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:47.034 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:47.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:47.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:47.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:47.162 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:47.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:47.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:47.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:47.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:47.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:47.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:47.401 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:47.428 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:47.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:47.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:47.533 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:47.539 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 12643 12627 157.818 141.125 0.00752497 0.00632747 2023-10-09T09:30:47.562 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:47.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:47.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:47.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:47.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:47.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:47.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:47.798 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:47.807 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:47 smithi163 bash[75425]: cluster 2023-10-09T09:30:47.706302+0000 mgr.y (mgr.34107) 735 : cluster 0 pgmap v1442: 89 pgs: 89 active+clean; 543 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T09:30:47.824 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:47.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:47.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:47.846 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:30:47.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:47.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:47.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:47.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:48.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:48.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:47 smithi181 bash[59345]: cluster 2023-10-09T09:30:47.706302+0000 mgr.y (mgr.34107) 735 : cluster 0 pgmap v1442: 89 pgs: 89 active+clean; 543 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T09:30:48.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:47 smithi163 bash[78107]: cluster 2023-10-09T09:30:47.706302+0000 mgr.y (mgr.34107) 735 : cluster 0 pgmap v1442: 89 pgs: 89 active+clean; 543 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T09:30:48.094 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:48.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:48.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:48.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:48.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:48.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:48.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:48.332 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:48.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:48.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:48.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:48.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:48.490 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:48.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:48.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:48.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 14901 14885 155.033 141.125 0.0101934 0.00644218 2023-10-09T09:30:48.594 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:48.620 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:48.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:48.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:48.749 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:48.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:48.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:48.852 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:48.883 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:48.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:48.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:48.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:49.012 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:49.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:49.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:49.116 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:49.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:49.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:49.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:49.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:49.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:49.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:49.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:49.383 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:49.409 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:49.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:49.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:49.512 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:49.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:49.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 16 17105 17089 152.561 137.75 0.00594355 0.00654564 2023-10-09T09:30:49.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:49.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:49.641 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:49.671 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:49.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:49.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:49.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:49.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:49.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:49.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:49.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:49.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:49.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:50.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:50.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:49 smithi163 bash[75425]: cluster 2023-10-09T09:30:49.707189+0000 mgr.y (mgr.34107) 736 : cluster 0 pgmap v1443: 89 pgs: 89 active+clean; 726 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T09:30:50.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:49 smithi163 bash[78107]: cluster 2023-10-09T09:30:49.707189+0000 mgr.y (mgr.34107) 736 : cluster 0 pgmap v1443: 89 pgs: 89 active+clean; 726 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T09:30:50.062 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:50.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:49 smithi181 bash[59345]: cluster 2023-10-09T09:30:49.707189+0000 mgr.y (mgr.34107) 736 : cluster 0 pgmap v1443: 89 pgs: 89 active+clean; 726 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T09:30:50.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:50.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:50.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:50.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:50.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:50.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:50.301 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:50.330 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:50.333 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:30:50.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:50.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:50.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:50.468 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:50.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:50.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 19077 19061 148.896 123.25 0.00337405 0.00670844 2023-10-09T09:30:50.571 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:50.596 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:50.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:50.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:50.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:50.729 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:50.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:50.835 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:50.863 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:50.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:50.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:50.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:50.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:50.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:50.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:51.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:51.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:51.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:51.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:51.228 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:51.256 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:51.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:51.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:51.385 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:51.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:51.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:51.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:51.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:51.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:51.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 20996 20980 145.677 119.938 0.00432705 0.00685813 2023-10-09T09:30:51.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:51.636 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:51.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:51.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:51.742 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:51.767 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:51.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:51.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:51.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:51.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:51.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:51.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:51 smithi163 bash[78107]: cluster 2023-10-09T09:30:51.708054+0000 mgr.y (mgr.34107) 737 : cluster 0 pgmap v1444: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T09:30:51.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:51 smithi163 bash[75425]: cluster 2023-10-09T09:30:51.708054+0000 mgr.y (mgr.34107) 737 : cluster 0 pgmap v1444: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T09:30:52.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:52.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:52.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:52.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:52.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:51 smithi181 bash[59345]: cluster 2023-10-09T09:30:51.708054+0000 mgr.y (mgr.34107) 737 : cluster 0 pgmap v1444: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T09:30:52.130 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:52.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:52.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:52.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:52.289 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:52.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:52.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:52.392 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:52.428 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:52.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:52.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:52.531 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:52.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 22830 22814 142.571 114.625 0.00814734 0.00700498 2023-10-09T09:30:52.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:52.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:52.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:52.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:52.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:52.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:52.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:52.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:52.824 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:52.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:52.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:52.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:52.954 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:52.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:52.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:53.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:53.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:53.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:53.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:53.191 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:53.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:53.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:53.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:53.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:53.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:53.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:53.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:53.358 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:30:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:30:53] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.33.4" 2023-10-09T09:30:53.455 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:53.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:53.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:53.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:53.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 24734 24718 140.427 119 0.0455157 0.00709585 2023-10-09T09:30:53.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:53.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:53.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:53.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:53.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:53.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:53.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:53.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:53.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:53.856 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:53 smithi181 bash[59345]: cluster 2023-10-09T09:30:53.708538+0000 mgr.y (mgr.34107) 738 : cluster 0 pgmap v1445: 89 pgs: 89 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T09:30:53.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:53.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:53.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:53.974 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:54.003 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:54.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:54.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:54.107 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:54.116 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:53 smithi163 bash[75425]: cluster 2023-10-09T09:30:53.708538+0000 mgr.y (mgr.34107) 738 : cluster 0 pgmap v1445: 89 pgs: 89 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T09:30:54.116 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:53 smithi163 bash[78107]: cluster 2023-10-09T09:30:53.708538+0000 mgr.y (mgr.34107) 738 : cluster 0 pgmap v1445: 89 pgs: 89 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T09:30:54.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:54.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:54.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:54.242 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:54.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:54.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:54.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:54.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:54.413 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:54.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:54.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:54.519 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:54.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 26664 26648 138.776 120.625 0.0072776 0.0071994 2023-10-09T09:30:54.542 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:54.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:54.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:54.645 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:54.668 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:54.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:54.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:54.771 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:54.799 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:54.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:54.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:54.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:54.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:54.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:55.032 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:55.060 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:55.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:55.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:55.192 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:55.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:55.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:55.295 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:55.319 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:55.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:55.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:55.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:55.455 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:55.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:55.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:55.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 28260 28244 135.773 99.75 0.00467755 0.00736007 2023-10-09T09:30:55.560 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:55.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:55 smithi181 bash[59345]: audit 2023-10-09T09:30:55.367071+0000 mon.a (mon.0) 1550 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:55.585 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:55.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:55.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:55.689 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:55.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:55 smithi163 bash[75425]: audit 2023-10-09T09:30:55.367071+0000 mon.a (mon.0) 1550 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:55.699 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:55 smithi163 bash[78107]: audit 2023-10-09T09:30:55.367071+0000 mon.a (mon.0) 1550 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:30:55.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:55.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:55.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:55.820 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:56.045 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:56.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:56.138 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:30:56.138 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:30:56.239 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:56.265 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:56.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:56.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:56.399 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:56.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:56.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:56.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:56.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:56.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:56.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 30050 30034 134.065 111.875 0.00918353 0.00745351 2023-10-09T09:30:56.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:56.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:56.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:56.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:56.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:56 smithi181 bash[59345]: cluster 2023-10-09T09:30:55.708948+0000 mgr.y (mgr.34107) 739 : cluster 0 pgmap v1446: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-10-09T09:30:56.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:57.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:56 smithi163 bash[78107]: cluster 2023-10-09T09:30:55.708948+0000 mgr.y (mgr.34107) 739 : cluster 0 pgmap v1446: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-10-09T09:30:57.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:56 smithi163 bash[75425]: cluster 2023-10-09T09:30:55.708948+0000 mgr.y (mgr.34107) 739 : cluster 0 pgmap v1446: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-10-09T09:30:57.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:57.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:57.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:57.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T09:30:57.236 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:57.733 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 31720 31704 132.086 104.375 0.020598 0.00756107 2023-10-09T09:30:57.760 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:57.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:57.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:58.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:58.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:57 smithi163 bash[75425]: cluster 2023-10-09T09:30:57.709621+0000 mgr.y (mgr.34107) 740 : cluster 0 pgmap v1447: 89 pgs: 89 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T09:30:58.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:57 smithi163 bash[78107]: cluster 2023-10-09T09:30:57.709621+0000 mgr.y (mgr.34107) 740 : cluster 0 pgmap v1447: 89 pgs: 89 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T09:30:58.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:57 smithi181 bash[59345]: cluster 2023-10-09T09:30:57.709621+0000 mgr.y (mgr.34107) 740 : cluster 0 pgmap v1447: 89 pgs: 89 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T09:30:58.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:30:58.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:58.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:30:58.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:58.205 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:58.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:58.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:58.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:58.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:58.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:58.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:58.442 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:58.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:58.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:58.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:58.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 33359 33343 130.232 102.438 0.0243189 0.00766974 2023-10-09T09:30:58.578 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:58.601 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:58.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:58.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:58.707 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:58.822 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:58.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:58.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:58.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:58.948 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:30:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:58.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:30:59.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:59.082 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:30:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:59.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:30:59.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:59.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:59.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:59.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:59.318 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:59.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:30:59.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:59.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:30:59.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:59.483 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:30:59.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:59.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:30:59.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 34975 34959 128.512 101 0.0100623 0.00777434 2023-10-09T09:30:59.587 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:59.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:30:59.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:59.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:30:59.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:59.749 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:30:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:59.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:30:59.854 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:59.880 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:59.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:30:59.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:30:59.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:30:59 smithi163 bash[78107]: cluster 2023-10-09T09:30:59.710209+0000 mgr.y (mgr.34107) 741 : cluster 0 pgmap v1448: 89 pgs: 89 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T09:30:59.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:30:59 smithi163 bash[75425]: cluster 2023-10-09T09:30:59.710209+0000 mgr.y (mgr.34107) 741 : cluster 0 pgmap v1448: 89 pgs: 89 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T09:30:59.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:30:59.995 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:30:59 smithi181 bash[59345]: cluster 2023-10-09T09:30:59.710209+0000 mgr.y (mgr.34107) 741 : cluster 0 pgmap v1448: 89 pgs: 89 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T09:31:00.021 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:00.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:00.123 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:00.149 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:00.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:00.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:00.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:00.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:00.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:00.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:00.414 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:00.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:00.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:00.520 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:00.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 16 36539 36523 126.802 97.75 0.00832543 0.00787958 2023-10-09T09:31:00.547 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:00.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:00.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:00.651 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:00.685 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:00.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:00.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:00.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:00.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:00.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:00.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:00.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:00.955 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:00.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:00.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:01.058 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:01.085 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:01.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:01.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:01.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:01.220 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:01.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:01.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:01.323 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:01.350 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:01.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:01.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:01.488 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:01.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:01.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:01.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 38033 38017 125.042 93.375 0.00812366 0.00799143 2023-10-09T09:31:01.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:01.621 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:01.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:01.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:01.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:01.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:01.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:01.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:01.905 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:01.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:01.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:01.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:01 smithi163 bash[78107]: cluster 2023-10-09T09:31:01.710919+0000 mgr.y (mgr.34107) 742 : cluster 0 pgmap v1449: 89 pgs: 89 active+clean; 2.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T09:31:01.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:01 smithi163 bash[75425]: cluster 2023-10-09T09:31:01.710919+0000 mgr.y (mgr.34107) 742 : cluster 0 pgmap v1449: 89 pgs: 89 active+clean; 2.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T09:31:01.998 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:31:02.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:02.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:01 smithi181 bash[59345]: cluster 2023-10-09T09:31:01.710919+0000 mgr.y (mgr.34107) 742 : cluster 0 pgmap v1449: 89 pgs: 89 active+clean; 2.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T09:31:02.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:02.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:02.084 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:31:02.108 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:31:02.109 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:31:02.132 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:31:02.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:02.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:02.185 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:31:02.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:02.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:02.229 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:31:02.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:02.429 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:31:02.430 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:31:02.430 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:31:02.430 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:31:02.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:31:02.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:31:02.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:31:02.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:31:02.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:31:02.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:31:02.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:31:02.432 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:31:02.457 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:02.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:02.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:02.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:31:02.541989+0000 min lat: 0.00155566 max lat: 0.0907421 avg lat: 0.00817865 2023-10-09T09:31:02.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:31:02.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 39116 39100 122.174 67.6875 0.00905313 0.00817865 2023-10-09T09:31:02.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:02.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:02.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:02.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:02.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:02.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:02.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:02.825 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:31:02.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:02.839 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:02 smithi163 bash[75425]: audit 2023-10-09T09:31:02.024820+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.163:0/771908427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:31:02.839 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:02 smithi163 bash[75425]: audit 2023-10-09T09:31:02.085093+0000 mgr.y (mgr.34107) 743 : audit 0 from='client.44927 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:31:02.839 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:02 smithi163 bash[75425]: audit 2023-10-09T09:31:02.370421+0000 mon.a (mon.0) 1551 : audit 1 from='client.? 172.21.15.163:0/1097081262' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:31:02.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:02 smithi163 bash[78107]: audit 2023-10-09T09:31:02.024820+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.163:0/771908427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:31:02.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:02 smithi163 bash[78107]: audit 2023-10-09T09:31:02.085093+0000 mgr.y (mgr.34107) 743 : audit 0 from='client.44927 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:31:02.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:02 smithi163 bash[78107]: audit 2023-10-09T09:31:02.370421+0000 mon.a (mon.0) 1551 : audit 1 from='client.? 172.21.15.163:0/1097081262' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:31:02.864 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:02.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:02.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:02.971 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:02.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:02.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:02.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:02 smithi181 bash[59345]: audit 2023-10-09T09:31:02.024820+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.163:0/771908427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:31:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:02 smithi181 bash[59345]: audit 2023-10-09T09:31:02.085093+0000 mgr.y (mgr.34107) 743 : audit 0 from='client.44927 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:31:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:02 smithi181 bash[59345]: audit 2023-10-09T09:31:02.370421+0000 mon.a (mon.0) 1551 : audit 1 from='client.? 172.21.15.163:0/1097081262' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:31:03.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:03.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:03.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:03.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:03.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:03.257 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:03.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:03.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:03.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:03.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:03.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:03.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:03.529 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:03.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:03.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 40595 40579 120.758 92.4375 0.00675656 0.00827508 2023-10-09T09:31:03.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:03.661 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:03.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:03.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:31:03] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.33.4" 2023-10-09T09:31:03.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:03.795 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:03.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:03.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:03.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:03.920 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:03.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:04.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:04.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[75425]: cluster 2023-10-09T09:31:02.809795+0000 mon.a (mon.0) 1552 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:31:04.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[75425]: cluster 2023-10-09T09:31:02.811695+0000 osd.5 (osd.5) 648 : cluster 3 failed to encode map e1103 with expected crc 2023-10-09T09:31:04.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[75425]: cluster 2023-10-09T09:31:02.812318+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e1103 with expected crc 2023-10-09T09:31:04.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[75425]: audit 2023-10-09T09:31:02.815354+0000 mon.a (mon.0) 1553 : audit 1 from='client.? 172.21.15.163:0/1097081262' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:31:04.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[75425]: cluster 2023-10-09T09:31:02.815575+0000 mon.a (mon.0) 1554 : cluster 0 osdmap e1103: 8 total, 8 up, 8 in 2023-10-09T09:31:04.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[75425]: cluster 2023-10-09T09:31:03.711487+0000 mgr.y (mgr.34107) 744 : cluster 0 pgmap v1451: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T09:31:04.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[78107]: cluster 2023-10-09T09:31:02.809795+0000 mon.a (mon.0) 1552 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:31:04.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[78107]: cluster 2023-10-09T09:31:02.811695+0000 osd.5 (osd.5) 648 : cluster 3 failed to encode map e1103 with expected crc 2023-10-09T09:31:04.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[78107]: cluster 2023-10-09T09:31:02.812318+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e1103 with expected crc 2023-10-09T09:31:04.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[78107]: audit 2023-10-09T09:31:02.815354+0000 mon.a (mon.0) 1553 : audit 1 from='client.? 172.21.15.163:0/1097081262' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:31:04.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[78107]: cluster 2023-10-09T09:31:02.815575+0000 mon.a (mon.0) 1554 : cluster 0 osdmap e1103: 8 total, 8 up, 8 in 2023-10-09T09:31:04.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:03 smithi163 bash[78107]: cluster 2023-10-09T09:31:03.711487+0000 mgr.y (mgr.34107) 744 : cluster 0 pgmap v1451: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T09:31:04.047 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:04.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:04.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:03 smithi181 bash[59345]: cluster 2023-10-09T09:31:02.809795+0000 mon.a (mon.0) 1552 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:31:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:03 smithi181 bash[59345]: cluster 2023-10-09T09:31:02.811695+0000 osd.5 (osd.5) 648 : cluster 3 failed to encode map e1103 with expected crc 2023-10-09T09:31:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:03 smithi181 bash[59345]: cluster 2023-10-09T09:31:02.812318+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e1103 with expected crc 2023-10-09T09:31:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:03 smithi181 bash[59345]: audit 2023-10-09T09:31:02.815354+0000 mon.a (mon.0) 1553 : audit 1 from='client.? 172.21.15.163:0/1097081262' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:31:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:03 smithi181 bash[59345]: cluster 2023-10-09T09:31:02.815575+0000 mon.a (mon.0) 1554 : cluster 0 osdmap e1103: 8 total, 8 up, 8 in 2023-10-09T09:31:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:03 smithi181 bash[59345]: cluster 2023-10-09T09:31:03.711487+0000 mgr.y (mgr.34107) 744 : cluster 0 pgmap v1451: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T09:31:04.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:04.179 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:04.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:04.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:04.282 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:04.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:04.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:04.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:04.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:04.439 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:04.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:04.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 41915 41899 119.019 82.5 0.0100694 0.0083959 2023-10-09T09:31:04.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:04.568 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:04.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:04.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:04.583 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1449,"stamp":"2023-10-09T09:31:01.710444+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2298681248,"num_objects":35090,"num_object_clones":0,"num_object_copies":105270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35090,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":35326,"num_write_kb":2246669,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35260,"ondisk_log_size":35260,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8542280,"kb_used_data":6648008,"kb_used_omap":416,"kb_used_meta":1893791,"kb_avail":741418936,"statfs":{"total":767960285184,"available":759212990464,"internally_reserved":0,"allocated":6807560192,"data_stored":6801961627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":1939242417},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1537146880,"num_objects":23455,"num_object_clones":0,"num_object_copies":70365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23455,"num_write_kb":1501120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003675"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1069,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.691227+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:30:27.691227+0000","last_peered":"2023-10-09T09:30:27.691227+0000","last_clean":"2023-10-09T09:30:27.691227+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:30:27.691227+0000","last_undegraded":"2023-10-09T09:30:27.691227+0000","last_fullsized":"2023-10-09T09:30:27.691227+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1035,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:34.165232+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:30:34.165232+0000","last_peered":"2023-10-09T09:30:34.165232+0000","last_clean":"2023-10-09T09:30:34.165232+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:30:34.165232+0000","last_undegraded":"2023-10-09T09:30:34.165232+0000","last_fullsized":"2023-10-09T09:30:34.165232+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1038,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.575310+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:30:27.575310+0000","last_peered":"2023-10-09T09:30:27.575310+0000","last_clean":"2023-10-09T09:30:27.575310+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:30:27.575310+0000","last_undegraded":"2023-10-09T09:30:27.575310+0000","last_fullsized":"2023-10-09T09:30:27.575310+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1030,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.749173+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:30:27.749173+0000","last_peered":"2023-10-09T09:30:27.749173+0000","last_clean":"2023-10-09T09:30:27.749173+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:30:27.749173+0000","last_undegraded":"2023-10-09T09:30:27.749173+0000","last_fullsized":"2023-10-09T09:30:27.749173+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1016,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.064084+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:30:27.064084+0000","last_peered":"2023-10-09T09:30:27.064084+0000","last_clean":"2023-10-09T09:30:27.064084+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:30:27.064084+0000","last_undegraded":"2023-10-09T09:30:27.064084+0000","last_fullsized":"2023-10-09T09:30:27.064084+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1033,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.725488+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:30:27.725488+0000","last_peered":"2023-10-09T09:30:27.725488+0000","last_clean":"2023-10-09T09:30:27.725488+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:30:27.725488+0000","last_undegraded":"2023-10-09T09:30:27.725488+0000","last_fullsized":"2023-10-09T09:30:27.725488+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1029,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.749164+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:30:27.749164+0000","last_peered":"2023-10-09T09:30:27.749164+0000","last_clean":"2023-10-09T09:30:27.749164+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:30:27.749164+0000","last_undegraded":"2023-10-09T09:30:27.749164+0000","last_fullsized":"2023-10-09T09:30:27.749164+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1000,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.063826+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:30:27.063826+0000","last_peered":"2023-10-09T09:30:27.063826+0000","last_clean":"2023-10-09T09:30:27.063826+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:30:27.063826+0000","last_undegraded":"2023-10-09T09:30:27.063826+0000","last_fullsized":"2023-10-09T09:30:27.063826+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1054,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.834831+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:30:26.834831+0000","last_peered":"2023-10-09T09:30:26.834831+0000","last_clean":"2023-10-09T09:30:26.834831+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:30:26.834831+0000","last_undegraded":"2023-10-09T09:30:26.834831+0000","last_fullsized":"2023-10-09T09:30:26.834831+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1011,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.064033+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:30:27.064033+0000","last_peered":"2023-10-09T09:30:27.064033+0000","last_clean":"2023-10-09T09:30:27.064033+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:30:27.064033+0000","last_undegraded":"2023-10-09T09:30:27.064033+0000","last_fullsized":"2023-10-09T09:30:27.064033+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1033,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.831891+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:30:26.831891+0000","last_peered":"2023-10-09T09:30:26.831891+0000","last_clean":"2023-10-09T09:30:26.831891+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:30:26.831891+0000","last_undegraded":"2023-10-09T09:30:26.831891+0000","last_fullsized":"2023-10-09T09:30:26.831891+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1039,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:40.567208+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:30:40.567208+0000","last_peered":"2023-10-09T09:30:40.567208+0000","last_clean":"2023-10-09T09:30:40.567208+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:30:40.567208+0000","last_undegraded":"2023-10-09T09:30:40.567208+0000","last_fullsized":"2023-10-09T09:30:40.567208+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1060,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.692299+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:30:27.692299+0000","last_peered":"2023-10-09T09:30:27.692299+0000","last_clean":"2023-10-09T09:30:27.692299+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:30:27.692299+0000","last_undegraded":"2023-10-09T09:30:27.692299+0000","last_fullsized":"2023-10-09T09:30:27.692299+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1052,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.691950+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:30:27.691950+0000","last_peered":"2023-10-09T09:30:27.691950+0000","last_clean":"2023-10-09T09:30:27.691950+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:30:27.691950+0000","last_undegraded":"2023-10-09T09:30:27.691950+0000","last_fullsized":"2023-10-09T09:30:27.691950+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1032,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.831556+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:30:26.831556+0000","last_peered":"2023-10-09T09:30:26.831556+0000","last_clean":"2023-10-09T09:30:26.831556+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:30:26.831556+0000","last_undegraded":"2023-10-09T09:30:26.831556+0000","last_fullsized":"2023-10-09T09:30:26.831556+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1000,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.064340+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:30:27.064340+0000","last_peered":"2023-10-09T09:30:27.064340+0000","last_clean":"2023-10-09T09:30:27.064340+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:30:27.064340+0000","last_undegraded":"2023-10-09T09:30:27.064340+0000","last_fullsized":"2023-10-09T09:30:27.064340+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1030,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:32.233462+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:30:32.233462+0000","last_peered":"2023-10-09T09:30:32.233462+0000","last_clean":"2023-10-09T09:30:32.233462+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:30:32.233462+0000","last_undegraded":"2023-10-09T09:30:32.233462+0000","last_fullsized":"2023-10-09T09:30:32.233462+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1053,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.835019+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:30:26.835019+0000","last_peered":"2023-10-09T09:30:26.835019+0000","last_clean":"2023-10-09T09:30:26.835019+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:30:26.835019+0000","last_undegraded":"2023-10-09T09:30:26.835019+0000","last_fullsized":"2023-10-09T09:30:26.835019+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1015,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.831159+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:30:26.831159+0000","last_peered":"2023-10-09T09:30:26.831159+0000","last_clean":"2023-10-09T09:30:26.831159+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:30:26.831159+0000","last_undegraded":"2023-10-09T09:30:26.831159+0000","last_fullsized":"2023-10-09T09:30:26.831159+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1028,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.749056+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:30:27.749056+0000","last_peered":"2023-10-09T09:30:27.749056+0000","last_clean":"2023-10-09T09:30:27.749056+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:30:27.749056+0000","last_undegraded":"2023-10-09T09:30:27.749056+0000","last_fullsized":"2023-10-09T09:30:27.749056+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1043,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.725058+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:30:27.725058+0000","last_peered":"2023-10-09T09:30:27.725058+0000","last_clean":"2023-10-09T09:30:27.725058+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:30:27.725058+0000","last_undegraded":"2023-10-09T09:30:27.725058+0000","last_fullsized":"2023-10-09T09:30:27.725058+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1035,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.064700+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:30:27.064700+0000","last_peered":"2023-10-09T09:30:27.064700+0000","last_clean":"2023-10-09T09:30:27.064700+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:30:27.064700+0000","last_undegraded":"2023-10-09T09:30:27.064700+0000","last_fullsized":"2023-10-09T09:30:27.064700+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1102'2038","reported_seq":2050,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:58.468722+0000","last_change":"2023-10-09T09:30:25.841890+0000","last_active":"2023-10-09T09:30:58.468722+0000","last_peered":"2023-10-09T09:30:58.468722+0000","last_clean":"2023-10-09T09:30:58.468722+0000","last_became_active":"2023-10-09T09:30:25.841584+0000","last_became_peered":"2023-10-09T09:30:25.841584+0000","last_unstale":"2023-10-09T09:30:58.468722+0000","last_undegraded":"2023-10-09T09:30:58.468722+0000","last_fullsized":"2023-10-09T09:30:58.468722+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2038,"log_dups_size":0,"ondisk_log_size":2038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:07.145206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133562368,"num_objects":2038,"num_object_clones":0,"num_object_copies":6114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2038,"num_write_kb":130432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1031,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.832092+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:30:26.832092+0000","last_peered":"2023-10-09T09:30:26.832092+0000","last_clean":"2023-10-09T09:30:26.832092+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:30:26.832092+0000","last_undegraded":"2023-10-09T09:30:26.832092+0000","last_fullsized":"2023-10-09T09:30:26.832092+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1038,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.834646+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:30:26.834646+0000","last_peered":"2023-10-09T09:30:26.834646+0000","last_clean":"2023-10-09T09:30:26.834646+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:30:26.834646+0000","last_undegraded":"2023-10-09T09:30:26.834646+0000","last_fullsized":"2023-10-09T09:30:26.834646+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1102'2128","reported_seq":2140,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:59.156404+0000","last_change":"2023-10-09T09:30:25.842825+0000","last_active":"2023-10-09T09:30:59.156404+0000","last_peered":"2023-10-09T09:30:59.156404+0000","last_clean":"2023-10-09T09:30:59.156404+0000","last_became_active":"2023-10-09T09:30:25.842490+0000","last_became_peered":"2023-10-09T09:30:25.842490+0000","last_unstale":"2023-10-09T09:30:59.156404+0000","last_undegraded":"2023-10-09T09:30:59.156404+0000","last_fullsized":"2023-10-09T09:30:59.156404+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2128,"log_dups_size":0,"ondisk_log_size":2128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:14.954211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139460608,"num_objects":2128,"num_object_clones":0,"num_object_copies":6384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2128,"num_write_kb":136192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1032,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.831831+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:30:26.831831+0000","last_peered":"2023-10-09T09:30:26.831831+0000","last_clean":"2023-10-09T09:30:26.831831+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:30:26.831831+0000","last_undegraded":"2023-10-09T09:30:26.831831+0000","last_fullsized":"2023-10-09T09:30:26.831831+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1067,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.691544+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:30:27.691544+0000","last_peered":"2023-10-09T09:30:27.691544+0000","last_clean":"2023-10-09T09:30:27.691544+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:30:27.691544+0000","last_undegraded":"2023-10-09T09:30:27.691544+0000","last_fullsized":"2023-10-09T09:30:27.691544+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1102'1907","reported_seq":1919,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:57.212155+0000","last_change":"2023-10-09T09:30:25.838435+0000","last_active":"2023-10-09T09:30:57.212155+0000","last_peered":"2023-10-09T09:30:57.212155+0000","last_clean":"2023-10-09T09:30:57.212155+0000","last_became_active":"2023-10-09T09:30:25.838147+0000","last_became_peered":"2023-10-09T09:30:25.838147+0000","last_unstale":"2023-10-09T09:30:57.212155+0000","last_undegraded":"2023-10-09T09:30:57.212155+0000","last_fullsized":"2023-10-09T09:30:57.212155+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":1907,"log_dups_size":0,"ondisk_log_size":1907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:23.629480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124977152,"num_objects":1907,"num_object_clones":0,"num_object_copies":5721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1907,"num_write_kb":122048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1016,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.831771+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:30:26.831771+0000","last_peered":"2023-10-09T09:30:26.831771+0000","last_clean":"2023-10-09T09:30:26.831771+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:30:26.831771+0000","last_undegraded":"2023-10-09T09:30:26.831771+0000","last_fullsized":"2023-10-09T09:30:26.831771+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1034,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.725056+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:30:27.725056+0000","last_peered":"2023-10-09T09:30:27.725056+0000","last_clean":"2023-10-09T09:30:27.725056+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:30:27.725056+0000","last_undegraded":"2023-10-09T09:30:27.725056+0000","last_fullsized":"2023-10-09T09:30:27.725056+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1102'2335","reported_seq":2347,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:31:00.172001+0000","last_change":"2023-10-09T09:30:25.839289+0000","last_active":"2023-10-09T09:31:00.172001+0000","last_peered":"2023-10-09T09:31:00.172001+0000","last_clean":"2023-10-09T09:31:00.172001+0000","last_became_active":"2023-10-09T09:30:25.838794+0000","last_became_peered":"2023-10-09T09:30:25.838794+0000","last_unstale":"2023-10-09T09:31:00.172001+0000","last_undegraded":"2023-10-09T09:31:00.172001+0000","last_fullsized":"2023-10-09T09:31:00.172001+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2335,"log_dups_size":0,"ondisk_log_size":2335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.498318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153026560,"num_objects":2335,"num_object_clones":0,"num_object_copies":7005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2335,"num_write_kb":149440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1054,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.834828+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:30:26.834828+0000","last_peered":"2023-10-09T09:30:26.834828+0000","last_clean":"2023-10-09T09:30:26.834828+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:30:26.834828+0000","last_undegraded":"2023-10-09T09:30:26.834828+0000","last_fullsized":"2023-10-09T09:30:26.834828+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1029,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.748494+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:30:27.748494+0000","last_peered":"2023-10-09T09:30:27.748494+0000","last_clean":"2023-10-09T09:30:27.748494+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:30:27.748494+0000","last_undegraded":"2023-10-09T09:30:27.748494+0000","last_fullsized":"2023-10-09T09:30:27.748494+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1102'1968","reported_seq":1980,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:57.215603+0000","last_change":"2023-10-09T09:30:25.837579+0000","last_active":"2023-10-09T09:30:57.215603+0000","last_peered":"2023-10-09T09:30:57.215603+0000","last_clean":"2023-10-09T09:30:57.215603+0000","last_became_active":"2023-10-09T09:30:25.836784+0000","last_became_peered":"2023-10-09T09:30:25.836784+0000","last_unstale":"2023-10-09T09:30:57.215603+0000","last_undegraded":"2023-10-09T09:30:57.215603+0000","last_fullsized":"2023-10-09T09:30:57.215603+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":1968,"log_dups_size":0,"ondisk_log_size":1968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:55.296361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128974848,"num_objects":1968,"num_object_clones":0,"num_object_copies":5904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1968,"num_write_kb":125952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":1018,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.064281+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:30:27.064281+0000","last_peered":"2023-10-09T09:30:27.064281+0000","last_clean":"2023-10-09T09:30:27.064281+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:30:27.064281+0000","last_undegraded":"2023-10-09T09:30:27.064281+0000","last_fullsized":"2023-10-09T09:30:27.064281+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1052,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.692296+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:30:27.692296+0000","last_peered":"2023-10-09T09:30:27.692296+0000","last_clean":"2023-10-09T09:30:27.692296+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:30:27.692296+0000","last_undegraded":"2023-10-09T09:30:27.692296+0000","last_fullsized":"2023-10-09T09:30:27.692296+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1102'2244","reported_seq":2256,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:31:00.165472+0000","last_change":"2023-10-09T09:30:25.841208+0000","last_active":"2023-10-09T09:31:00.165472+0000","last_peered":"2023-10-09T09:31:00.165472+0000","last_clean":"2023-10-09T09:31:00.165472+0000","last_became_active":"2023-10-09T09:30:25.840860+0000","last_became_peered":"2023-10-09T09:30:25.840860+0000","last_unstale":"2023-10-09T09:31:00.165472+0000","last_undegraded":"2023-10-09T09:31:00.165472+0000","last_fullsized":"2023-10-09T09:31:00.165472+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2244,"log_dups_size":0,"ondisk_log_size":2244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:20.488338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147062784,"num_objects":2244,"num_object_clones":0,"num_object_copies":6732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2244,"num_write_kb":143616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1035,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.725157+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:30:27.725157+0000","last_peered":"2023-10-09T09:30:27.725157+0000","last_clean":"2023-10-09T09:30:27.725157+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:30:27.725157+0000","last_undegraded":"2023-10-09T09:30:27.725157+0000","last_fullsized":"2023-10-09T09:30:27.725157+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1067,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.692324+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:30:27.692324+0000","last_peered":"2023-10-09T09:30:27.692324+0000","last_clean":"2023-10-09T09:30:27.692324+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:30:27.692324+0000","last_undegraded":"2023-10-09T09:30:27.692324+0000","last_fullsized":"2023-10-09T09:30:27.692324+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1102'2451","reported_seq":2462,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:31:01.446868+0000","last_change":"2023-10-09T09:30:25.838078+0000","last_active":"2023-10-09T09:31:01.446868+0000","last_peered":"2023-10-09T09:31:01.446868+0000","last_clean":"2023-10-09T09:31:01.446868+0000","last_became_active":"2023-10-09T09:30:25.837594+0000","last_became_peered":"2023-10-09T09:30:25.837594+0000","last_unstale":"2023-10-09T09:31:01.446868+0000","last_undegraded":"2023-10-09T09:31:01.446868+0000","last_fullsized":"2023-10-09T09:31:01.446868+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2451,"log_dups_size":0,"ondisk_log_size":2451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160628736,"num_objects":2451,"num_object_clones":0,"num_object_copies":7353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2451,"num_write_kb":156864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1053,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.691164+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:30:27.691164+0000","last_peered":"2023-10-09T09:30:27.691164+0000","last_clean":"2023-10-09T09:30:27.691164+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:30:27.691164+0000","last_undegraded":"2023-10-09T09:30:27.691164+0000","last_fullsized":"2023-10-09T09:30:27.691164+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1052,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.692594+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:30:27.692594+0000","last_peered":"2023-10-09T09:30:27.692594+0000","last_clean":"2023-10-09T09:30:27.692594+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:30:27.692594+0000","last_undegraded":"2023-10-09T09:30:27.692594+0000","last_fullsized":"2023-10-09T09:30:27.692594+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1102'2421","reported_seq":2432,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:31:01.461922+0000","last_change":"2023-10-09T09:30:25.841044+0000","last_active":"2023-10-09T09:31:01.461922+0000","last_peered":"2023-10-09T09:31:01.461922+0000","last_clean":"2023-10-09T09:31:01.461922+0000","last_became_active":"2023-10-09T09:30:25.840627+0000","last_became_peered":"2023-10-09T09:30:25.840627+0000","last_unstale":"2023-10-09T09:31:01.461922+0000","last_undegraded":"2023-10-09T09:31:01.461922+0000","last_fullsized":"2023-10-09T09:31:01.461922+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2421,"log_dups_size":0,"ondisk_log_size":2421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158662656,"num_objects":2421,"num_object_clones":0,"num_object_copies":7263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2421,"num_write_kb":154944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1002,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:40.338750+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:30:40.338750+0000","last_peered":"2023-10-09T09:30:40.338750+0000","last_clean":"2023-10-09T09:30:40.338750+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:30:40.338750+0000","last_undegraded":"2023-10-09T09:30:40.338750+0000","last_fullsized":"2023-10-09T09:30:40.338750+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1039,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:28.496389+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:30:28.496389+0000","last_peered":"2023-10-09T09:30:28.496389+0000","last_clean":"2023-10-09T09:30:28.496389+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:30:28.496389+0000","last_undegraded":"2023-10-09T09:30:28.496389+0000","last_fullsized":"2023-10-09T09:30:28.496389+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1102'2360","reported_seq":2371,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:31:01.386611+0000","last_change":"2023-10-09T09:30:25.840930+0000","last_active":"2023-10-09T09:31:01.386611+0000","last_peered":"2023-10-09T09:31:01.386611+0000","last_clean":"2023-10-09T09:31:01.386611+0000","last_became_active":"2023-10-09T09:30:25.840518+0000","last_became_peered":"2023-10-09T09:30:25.840518+0000","last_unstale":"2023-10-09T09:31:01.386611+0000","last_undegraded":"2023-10-09T09:31:01.386611+0000","last_fullsized":"2023-10-09T09:31:01.386611+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2360,"log_dups_size":0,"ondisk_log_size":2360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154664960,"num_objects":2360,"num_object_clones":0,"num_object_copies":7080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2360,"num_write_kb":151040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1034,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.576284+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:30:27.576284+0000","last_peered":"2023-10-09T09:30:27.576284+0000","last_clean":"2023-10-09T09:30:27.576284+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:30:27.576284+0000","last_undegraded":"2023-10-09T09:30:27.576284+0000","last_fullsized":"2023-10-09T09:30:27.576284+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1069,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.835096+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:30:26.835096+0000","last_peered":"2023-10-09T09:30:26.835096+0000","last_clean":"2023-10-09T09:30:26.835096+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:30:26.835096+0000","last_undegraded":"2023-10-09T09:30:26.835096+0000","last_fullsized":"2023-10-09T09:30:26.835096+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1024,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.575851+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:30:27.575851+0000","last_peered":"2023-10-09T09:30:27.575851+0000","last_clean":"2023-10-09T09:30:27.575851+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:30:27.575851+0000","last_undegraded":"2023-10-09T09:30:27.575851+0000","last_fullsized":"2023-10-09T09:30:27.575851+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1102'2418","reported_seq":2428,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:31:01.473544+0000","last_change":"2023-10-09T09:30:25.840180+0000","last_active":"2023-10-09T09:31:01.473544+0000","last_peered":"2023-10-09T09:31:01.473544+0000","last_clean":"2023-10-09T09:31:01.473544+0000","last_became_active":"2023-10-09T09:30:25.839813+0000","last_became_peered":"2023-10-09T09:30:25.839813+0000","last_unstale":"2023-10-09T09:31:01.473544+0000","last_undegraded":"2023-10-09T09:31:01.473544+0000","last_fullsized":"2023-10-09T09:31:01.473544+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2418,"log_dups_size":0,"ondisk_log_size":2418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158466048,"num_objects":2418,"num_object_clones":0,"num_object_copies":7254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2418,"num_write_kb":154752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1058,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.834968+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:30:26.834968+0000","last_peered":"2023-10-09T09:30:26.834968+0000","last_clean":"2023-10-09T09:30:26.834968+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:30:26.834968+0000","last_undegraded":"2023-10-09T09:30:26.834968+0000","last_fullsized":"2023-10-09T09:30:26.834968+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1053,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.691883+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:30:27.691883+0000","last_peered":"2023-10-09T09:30:27.691883+0000","last_clean":"2023-10-09T09:30:27.691883+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:30:27.691883+0000","last_undegraded":"2023-10-09T09:30:27.691883+0000","last_fullsized":"2023-10-09T09:30:27.691883+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1054,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.575954+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:30:27.575954+0000","last_peered":"2023-10-09T09:30:27.575954+0000","last_clean":"2023-10-09T09:30:27.575954+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:30:27.575954+0000","last_undegraded":"2023-10-09T09:30:27.575954+0000","last_fullsized":"2023-10-09T09:30:27.575954+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1102'1906","reported_seq":1918,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:57.234503+0000","last_change":"2023-10-09T09:30:25.837152+0000","last_active":"2023-10-09T09:30:57.234503+0000","last_peered":"2023-10-09T09:30:57.234503+0000","last_clean":"2023-10-09T09:30:57.234503+0000","last_became_active":"2023-10-09T09:30:25.836769+0000","last_became_peered":"2023-10-09T09:30:25.836769+0000","last_unstale":"2023-10-09T09:30:57.234503+0000","last_undegraded":"2023-10-09T09:30:57.234503+0000","last_fullsized":"2023-10-09T09:30:57.234503+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":1906,"log_dups_size":0,"ondisk_log_size":1906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:58.080522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124911616,"num_objects":1906,"num_object_clones":0,"num_object_copies":5718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1906,"num_write_kb":121984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1021,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.255757+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:30:27.255757+0000","last_peered":"2023-10-09T09:30:27.255757+0000","last_clean":"2023-10-09T09:30:27.255757+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:30:27.255757+0000","last_undegraded":"2023-10-09T09:30:27.255757+0000","last_fullsized":"2023-10-09T09:30:27.255757+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1040,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:40.566156+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:30:40.566156+0000","last_peered":"2023-10-09T09:30:40.566156+0000","last_clean":"2023-10-09T09:30:40.566156+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:30:40.566156+0000","last_undegraded":"2023-10-09T09:30:40.566156+0000","last_fullsized":"2023-10-09T09:30:40.566156+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1029,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.576348+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:30:27.576348+0000","last_peered":"2023-10-09T09:30:27.576348+0000","last_clean":"2023-10-09T09:30:27.576348+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:30:27.576348+0000","last_undegraded":"2023-10-09T09:30:27.576348+0000","last_fullsized":"2023-10-09T09:30:27.576348+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1102'1964","reported_seq":1976,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:57.224890+0000","last_change":"2023-10-09T09:30:25.838501+0000","last_active":"2023-10-09T09:30:57.224890+0000","last_peered":"2023-10-09T09:30:57.224890+0000","last_clean":"2023-10-09T09:30:57.224890+0000","last_became_active":"2023-10-09T09:30:25.838212+0000","last_became_peered":"2023-10-09T09:30:25.838212+0000","last_unstale":"2023-10-09T09:30:57.224890+0000","last_undegraded":"2023-10-09T09:30:57.224890+0000","last_fullsized":"2023-10-09T09:30:57.224890+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":1964,"log_dups_size":0,"ondisk_log_size":1964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:49.535819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128712704,"num_objects":1964,"num_object_clones":0,"num_object_copies":5892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1964,"num_write_kb":125696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1058,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.834673+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:30:26.834673+0000","last_peered":"2023-10-09T09:30:26.834673+0000","last_clean":"2023-10-09T09:30:26.834673+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:30:26.834673+0000","last_undegraded":"2023-10-09T09:30:26.834673+0000","last_fullsized":"2023-10-09T09:30:26.834673+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1266,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.830949+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:30:26.830949+0000","last_peered":"2023-10-09T09:30:26.830949+0000","last_clean":"2023-10-09T09:30:26.830949+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:30:26.830949+0000","last_undegraded":"2023-10-09T09:30:26.830949+0000","last_fullsized":"2023-10-09T09:30:26.830949+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1013,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.064110+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:30:27.064110+0000","last_peered":"2023-10-09T09:30:27.064110+0000","last_clean":"2023-10-09T09:30:27.064110+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:30:27.064110+0000","last_undegraded":"2023-10-09T09:30:27.064110+0000","last_fullsized":"2023-10-09T09:30:27.064110+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1019,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.725831+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:30:27.725831+0000","last_peered":"2023-10-09T09:30:27.725831+0000","last_clean":"2023-10-09T09:30:27.725831+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:30:27.725831+0000","last_undegraded":"2023-10-09T09:30:27.725831+0000","last_fullsized":"2023-10-09T09:30:27.725831+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1102'2320","reported_seq":2328,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:31:01.396535+0000","last_change":"2023-10-09T09:30:25.842562+0000","last_active":"2023-10-09T09:31:01.396535+0000","last_peered":"2023-10-09T09:31:01.396535+0000","last_clean":"2023-10-09T09:31:01.396535+0000","last_became_active":"2023-10-09T09:30:25.842231+0000","last_became_peered":"2023-10-09T09:30:25.842231+0000","last_unstale":"2023-10-09T09:31:01.396535+0000","last_undegraded":"2023-10-09T09:31:01.396535+0000","last_fullsized":"2023-10-09T09:31:01.396535+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2320,"log_dups_size":0,"ondisk_log_size":2320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152043520,"num_objects":2320,"num_object_clones":0,"num_object_copies":6960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2320,"num_write_kb":148480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1049,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.830862+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:30:26.830862+0000","last_peered":"2023-10-09T09:30:26.830862+0000","last_clean":"2023-10-09T09:30:26.830862+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:30:26.830862+0000","last_undegraded":"2023-10-09T09:30:26.830862+0000","last_fullsized":"2023-10-09T09:30:26.830862+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1031,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:40.564247+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:30:40.564247+0000","last_peered":"2023-10-09T09:30:40.564247+0000","last_clean":"2023-10-09T09:30:40.564247+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:30:40.564247+0000","last_undegraded":"2023-10-09T09:30:40.564247+0000","last_fullsized":"2023-10-09T09:30:40.564247+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1041,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.747727+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:30:27.747727+0000","last_peered":"2023-10-09T09:30:27.747727+0000","last_clean":"2023-10-09T09:30:27.747727+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:30:27.747727+0000","last_undegraded":"2023-10-09T09:30:27.747727+0000","last_fullsized":"2023-10-09T09:30:27.747727+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"1102'2375","reported_seq":2386,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:31:01.404534+0000","last_change":"2023-10-09T09:30:25.842665+0000","last_active":"2023-10-09T09:31:01.404534+0000","last_peered":"2023-10-09T09:31:01.404534+0000","last_clean":"2023-10-09T09:31:01.404534+0000","last_became_active":"2023-10-09T09:30:25.842322+0000","last_became_peered":"2023-10-09T09:30:25.842322+0000","last_unstale":"2023-10-09T09:31:01.404534+0000","last_undegraded":"2023-10-09T09:31:01.404534+0000","last_fullsized":"2023-10-09T09:31:01.404534+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2375,"log_dups_size":0,"ondisk_log_size":2375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155648000,"num_objects":2375,"num_object_clones":0,"num_object_copies":7125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2375,"num_write_kb":152000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1034,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.831903+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:30:26.831903+0000","last_peered":"2023-10-09T09:30:26.831903+0000","last_clean":"2023-10-09T09:30:26.831903+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:30:26.831903+0000","last_undegraded":"2023-10-09T09:30:26.831903+0000","last_fullsized":"2023-10-09T09:30:26.831903+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1040,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.575069+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:30:27.575069+0000","last_peered":"2023-10-09T09:30:27.575069+0000","last_clean":"2023-10-09T09:30:27.575069+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:30:27.575069+0000","last_undegraded":"2023-10-09T09:30:27.575069+0000","last_fullsized":"2023-10-09T09:30:27.575069+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1015,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:40.565704+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:30:40.565704+0000","last_peered":"2023-10-09T09:30:40.565704+0000","last_clean":"2023-10-09T09:30:40.565704+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:30:40.565704+0000","last_undegraded":"2023-10-09T09:30:40.565704+0000","last_fullsized":"2023-10-09T09:30:40.565704+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1102'2147","reported_seq":2159,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:59.156142+0000","last_change":"2023-10-09T09:30:25.840862+0000","last_active":"2023-10-09T09:30:59.156142+0000","last_peered":"2023-10-09T09:30:59.156142+0000","last_clean":"2023-10-09T09:30:59.156142+0000","last_became_active":"2023-10-09T09:30:25.839943+0000","last_became_peered":"2023-10-09T09:30:25.839943+0000","last_unstale":"2023-10-09T09:30:59.156142+0000","last_undegraded":"2023-10-09T09:30:59.156142+0000","last_fullsized":"2023-10-09T09:30:59.156142+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2147,"log_dups_size":0,"ondisk_log_size":2147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:33.386856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140705792,"num_objects":2147,"num_object_clones":0,"num_object_copies":6441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2147,"num_write_kb":137408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":1019,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:40.338853+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:30:40.338853+0000","last_peered":"2023-10-09T09:30:40.338853+0000","last_clean":"2023-10-09T09:30:40.338853+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:30:40.338853+0000","last_undegraded":"2023-10-09T09:30:40.338853+0000","last_fullsized":"2023-10-09T09:30:40.338853+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1023,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.576309+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:30:27.576309+0000","last_peered":"2023-10-09T09:30:27.576309+0000","last_clean":"2023-10-09T09:30:27.576309+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:30:27.576309+0000","last_undegraded":"2023-10-09T09:30:27.576309+0000","last_fullsized":"2023-10-09T09:30:27.576309+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1016,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.063575+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:30:27.063575+0000","last_peered":"2023-10-09T09:30:27.063575+0000","last_clean":"2023-10-09T09:30:27.063575+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:30:27.063575+0000","last_undegraded":"2023-10-09T09:30:27.063575+0000","last_fullsized":"2023-10-09T09:30:27.063575+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1102'2084","reported_seq":2095,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:59.170143+0000","last_change":"2023-10-09T09:30:25.837344+0000","last_active":"2023-10-09T09:30:59.170143+0000","last_peered":"2023-10-09T09:30:59.170143+0000","last_clean":"2023-10-09T09:30:59.170143+0000","last_became_active":"2023-10-09T09:30:25.836945+0000","last_became_peered":"2023-10-09T09:30:25.836945+0000","last_unstale":"2023-10-09T09:30:59.170143+0000","last_undegraded":"2023-10-09T09:30:59.170143+0000","last_fullsized":"2023-10-09T09:30:59.170143+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":2084,"log_dups_size":0,"ondisk_log_size":2084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:52.621311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136577024,"num_objects":2084,"num_object_clones":0,"num_object_copies":6252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2084,"num_write_kb":133376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1038,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.724639+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:30:27.724639+0000","last_peered":"2023-10-09T09:30:27.724639+0000","last_clean":"2023-10-09T09:30:27.724639+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:30:27.724639+0000","last_undegraded":"2023-10-09T09:30:27.724639+0000","last_fullsized":"2023-10-09T09:30:27.724639+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1033,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.749200+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:30:27.749200+0000","last_peered":"2023-10-09T09:30:27.749200+0000","last_clean":"2023-10-09T09:30:27.749200+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:30:27.749200+0000","last_undegraded":"2023-10-09T09:30:27.749200+0000","last_fullsized":"2023-10-09T09:30:27.749200+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1037,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.691474+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:30:27.691474+0000","last_peered":"2023-10-09T09:30:27.691474+0000","last_clean":"2023-10-09T09:30:27.691474+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:30:27.691474+0000","last_undegraded":"2023-10-09T09:30:27.691474+0000","last_fullsized":"2023-10-09T09:30:27.691474+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1017,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.748769+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:30:27.748769+0000","last_peered":"2023-10-09T09:30:27.748769+0000","last_clean":"2023-10-09T09:30:27.748769+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:30:27.748769+0000","last_undegraded":"2023-10-09T09:30:27.748769+0000","last_fullsized":"2023-10-09T09:30:27.748769+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1039,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.256039+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:30:27.256039+0000","last_peered":"2023-10-09T09:30:27.256039+0000","last_clean":"2023-10-09T09:30:27.256039+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:30:27.256039+0000","last_undegraded":"2023-10-09T09:30:27.256039+0000","last_fullsized":"2023-10-09T09:30:27.256039+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1056,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.834822+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:30:26.834822+0000","last_peered":"2023-10-09T09:30:26.834822+0000","last_clean":"2023-10-09T09:30:26.834822+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:30:26.834822+0000","last_undegraded":"2023-10-09T09:30:26.834822+0000","last_fullsized":"2023-10-09T09:30:26.834822+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1067,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.692021+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:30:27.692021+0000","last_peered":"2023-10-09T09:30:27.692021+0000","last_clean":"2023-10-09T09:30:27.692021+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:30:27.692021+0000","last_undegraded":"2023-10-09T09:30:27.692021+0000","last_fullsized":"2023-10-09T09:30:27.692021+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1041,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:26.835089+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:30:26.835089+0000","last_peered":"2023-10-09T09:30:26.835089+0000","last_clean":"2023-10-09T09:30:26.835089+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:30:26.835089+0000","last_undegraded":"2023-10-09T09:30:26.835089+0000","last_fullsized":"2023-10-09T09:30:26.835089+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1067,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.691758+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:30:27.691758+0000","last_peered":"2023-10-09T09:30:27.691758+0000","last_clean":"2023-10-09T09:30:27.691758+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:30:27.691758+0000","last_undegraded":"2023-10-09T09:30:27.691758+0000","last_fullsized":"2023-10-09T09:30:27.691758+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":1029,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.064573+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:30:27.064573+0000","last_peered":"2023-10-09T09:30:27.064573+0000","last_clean":"2023-10-09T09:30:27.064573+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:30:27.064573+0000","last_undegraded":"2023-10-09T09:30:27.064573+0000","last_fullsized":"2023-10-09T09:30:27.064573+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1041,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.063805+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:30:27.063805+0000","last_peered":"2023-10-09T09:30:27.063805+0000","last_clean":"2023-10-09T09:30:27.063805+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:30:27.063805+0000","last_undegraded":"2023-10-09T09:30:27.063805+0000","last_fullsized":"2023-10-09T09:30:27.063805+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1037,"reported_epoch":1102,"state":"active+clean","last_fresh":"2023-10-09T09:30:40.567394+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:30:40.567394+0000","last_peered":"2023-10-09T09:30:40.567394+0000","last_clean":"2023-10-09T09:30:40.567394+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:30:40.567394+0000","last_undegraded":"2023-10-09T09:30:40.567394+0000","last_fullsized":"2023-10-09T09:30:40.567394+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1052,"reported_epoch":1101,"state":"active+clean","last_fresh":"2023-10-09T09:30:27.691357+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:30:27.691357+0000","last_peered":"2023-10-09T09:30:27.691357+0000","last_clean":"2023-10-09T09:30:27.691357+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:30:27.691357+0000","last_undegraded":"2023-10-09T09:30:27.691357+0000","last_fullsized":"2023-10-09T09:30:27.691357+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2298085376,"num_objects":35066,"num_object_clones":0,"num_object_copies":105198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":35066,"num_write_kb":2244224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6939934720,"data_stored":6939934720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35066,"ondisk_log_size":35066,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397871,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1055520,"kb_used_data":754200,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92689632,"statfs":{"total":95995035648,"available":94914183168,"internally_reserved":0,"allocated":772300800,"data_stored":772030360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":2.3599999999999999,"15min":1.4730000000000001},"min":{"1min":0.46400000000000002,"5min":0.249,"15min":0.249},"max":{"1min":2.286,"5min":55.296999999999997,"15min":55.296999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.1859999999999999,"5min":2.4820000000000002,"15min":1.5920000000000001},"min":{"1min":0.57499999999999996,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.3639999999999999,"5min":55.369999999999997,"15min":55.369999999999997},"last":2.915}]},{"osd":1,"last update":"Mon Oct 9 09:30:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0350000000000001,"5min":2.7690000000000001,"15min":1.534},"min":{"1min":0.30199999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.1159999999999997,"5min":103.84699999999999,"15min":103.84699999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.1699999999999999,"5min":2.7370000000000001,"15min":1.5840000000000001},"min":{"1min":0.29599999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.3839999999999999,"5min":103.955,"15min":103.955},"last":0.57799999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:30:33 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":2.129,"15min":1.4690000000000001},"min":{"1min":0.54500000000000004,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.1469999999999998,"5min":17.638000000000002,"15min":17.638000000000002},"last":0.374},{"interface":"front","average":{"1min":1.8180000000000001,"5min":2.0379999999999998,"15min":1.369},"min":{"1min":0.52200000000000002,"5min":0.373,"15min":0.373},"max":{"1min":10.836,"5min":17.600000000000001,"15min":17.600000000000001},"last":0.39400000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:30:07 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":4.1970000000000001,"15min":2.0739999999999998},"min":{"1min":0.35399999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":29.600999999999999,"5min":107.97499999999999,"15min":107.97499999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":3.1989999999999998,"5min":4.2960000000000003,"15min":2.2000000000000002},"min":{"1min":0.47899999999999998,"5min":0.317,"15min":0.317},"max":{"1min":29.329999999999998,"5min":105.282,"15min":105.282},"last":0.50700000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.399,"15min":1.105},"min":{"1min":0.47199999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.8300000000000001,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.137,"5min":1.2470000000000001,"15min":1.0720000000000001},"min":{"1min":0.53800000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.6749999999999998,"5min":12.316000000000001,"15min":12.316000000000001},"last":0.32200000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.494,"15min":1.26},"min":{"1min":0.60799999999999998,"5min":0.377,"15min":0.377},"max":{"1min":2.2559999999999998,"5min":10.063000000000001,"15min":10.063000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.5149999999999999,"5min":1.3280000000000001,"15min":1.1479999999999999},"min":{"1min":0.67900000000000005,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":8.1699999999999999,"5min":8.1699999999999999,"15min":8.1699999999999999},"last":0.41099999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":1.397,"15min":1.1919999999999999},"min":{"1min":0.68100000000000005,"5min":0.373,"15min":0.373},"max":{"1min":8.2089999999999996,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":0.497},{"interface":"front","average":{"1min":1.218,"5min":1.536,"15min":1.288},"min":{"1min":0.503,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":4.7110000000000003,"5min":8.1980000000000004,"15min":8.1980000000000004},"last":0.36099999999999999}]}]},{"osd":6,"up_from":43,"seq":184683594104,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1056276,"kb_used_data":754956,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92688876,"statfs":{"total":95995035648,"available":94913409024,"internally_reserved":0,"allocated":773074944,"data_stored":772816646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:30:44 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.427,"15min":1.0880000000000001},"min":{"1min":0.42999999999999999,"5min":0.255,"15min":0.255},"max":{"1min":16.850000000000001,"5min":16.850000000000001,"15min":16.850000000000001},"last":1.5089999999999999},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.113,"15min":1.042},"min":{"1min":0.31900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.573,"5min":7.1829999999999998,"15min":7.1829999999999998},"last":0.71499999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":1.9159999999999999,"15min":1.28},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.7320000000000002,"5min":32.959000000000003,"15min":32.959000000000003},"last":1.3240000000000001},{"interface":"front","average":{"1min":1.214,"5min":1.772,"15min":1.218},"min":{"1min":0.41699999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.3940000000000001,"5min":34.793999999999997,"15min":34.793999999999997},"last":3.4119999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.8859999999999999,"15min":1.2649999999999999},"min":{"1min":0.38600000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":30.379000000000001,"5min":30.379000000000001,"15min":30.379000000000001},"last":0.621},{"interface":"front","average":{"1min":2.2999999999999998,"5min":1.931,"15min":1.337},"min":{"1min":0.36199999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":30.332999999999998,"5min":30.332999999999998,"15min":30.332999999999998},"last":0.65600000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":3.3260000000000001,"15min":1.774},"min":{"1min":0.63900000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.9359999999999999,"5min":123.97199999999999,"15min":123.97199999999999},"last":1.2769999999999999},{"interface":"front","average":{"1min":1.161,"5min":3.3300000000000001,"15min":1.8169999999999999},"min":{"1min":0.47399999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.6580000000000004,"5min":124.059,"15min":124.059},"last":3.3799999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.125,"15min":1.026},"min":{"1min":0.39200000000000002,"5min":0.312,"15min":0.312},"max":{"1min":1.3420000000000001,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":0.745},{"interface":"front","average":{"1min":0.94599999999999995,"5min":1.306,"15min":1.1439999999999999},"min":{"1min":0.50900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.0979999999999999,"5min":6.7220000000000004,"15min":6.7220000000000004},"last":3.339}]},{"osd":5,"last update":"Mon Oct 9 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.234,"15min":1.143},"min":{"1min":0.52600000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.339,"5min":7.6479999999999997,"15min":7.6479999999999997},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.327,"15min":1.2150000000000001},"min":{"1min":0.46800000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.4900000000000002,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":3.254}]},{"osd":7,"last update":"Mon Oct 9 09:30:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.651,"15min":1.333},"min":{"1min":0.505,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":5.758,"5min":7.3630000000000004,"15min":7.3630000000000004},"last":3.2959999999999998},{"interface":"front","average":{"1min":1.359,"5min":1.5800000000000001,"15min":1.3140000000000001},"min":{"1min":0.42599999999999999,"5min":0.371,"15min":0.371},"max":{"1min":4.625,"5min":7.6459999999999999,"15min":7.6459999999999999},"last":1.4810000000000001}]}]},{"osd":2,"up_from":634,"seq":2723009265857,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1031304,"kb_used_data":858112,"kb_used_omap":122,"kb_used_meta":173061,"kb_avail":92713848,"statfs":{"total":95995035648,"available":94938980352,"internally_reserved":0,"allocated":878706688,"data_stored":877531276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":177215051},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:30:50 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.9259999999999999,"15min":1.143},"min":{"1min":0.39900000000000002,"5min":0.216,"15min":0.216},"max":{"1min":4.3419999999999996,"5min":8.9860000000000007,"15min":8.9860000000000007},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.298,"5min":2.0830000000000002,"15min":1.1879999999999999},"min":{"1min":0.374,"5min":0.221,"15min":0.221},"max":{"1min":9.5530000000000008,"5min":10.019,"15min":10.019},"last":1.742}]},{"osd":1,"last update":"Mon Oct 9 09:30:50 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":1.6220000000000001,"15min":1.0309999999999999},"min":{"1min":0.373,"5min":0.184,"15min":0.184},"max":{"1min":0.97999999999999998,"5min":9.5139999999999993,"15min":9.5139999999999993},"last":2.4590000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":2.0800000000000001,"15min":1.238},"min":{"1min":0.51600000000000001,"5min":0.216,"15min":0.216},"max":{"1min":3.6739999999999999,"5min":8.702,"15min":8.702},"last":1.8939999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:29:59 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":2.085,"15min":1.25},"min":{"1min":0.27200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":18.547999999999998,"5min":18.547999999999998,"15min":18.547999999999998},"last":2.5019999999999998},{"interface":"front","average":{"1min":3.0430000000000001,"5min":2.2429999999999999,"15min":1.3460000000000001},"min":{"1min":0.32700000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.3819999999999997,"5min":12.861000000000001,"15min":12.861000000000001},"last":0.58299999999999996}]},{"osd":4,"last update":"Mon Oct 9 09:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.722,"15min":1.2849999999999999},"min":{"1min":0.45400000000000001,"5min":0.309,"15min":0.309},"max":{"1min":2.7069999999999999,"5min":7.452,"15min":7.452},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.5920000000000001,"5min":1.843,"15min":1.3},"min":{"1min":0.45700000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.6300000000000008,"5min":13.917999999999999,"15min":13.917999999999999},"last":1.696}]},{"osd":5,"last update":"Mon Oct 9 09:30:50 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.3380000000000001,"15min":1.1519999999999999},"min":{"1min":0.41599999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.2470000000000001,"5min":6.7699999999999996,"15min":6.7699999999999996},"last":2.5430000000000001},{"interface":"front","average":{"1min":1.131,"5min":1.7150000000000001,"15min":1.333},"min":{"1min":0.50600000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.7440000000000002,"5min":10.393000000000001,"15min":10.393000000000001},"last":0.66100000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.736,"15min":1.355},"min":{"1min":0.48299999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.496,"5min":7.9470000000000001,"15min":7.9470000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.0259999999999998,"15min":1.4379999999999999},"min":{"1min":0.64100000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.064,"5min":13.893000000000001,"15min":13.893000000000001},"last":1.659}]},{"osd":7,"last update":"Mon Oct 9 09:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":2.161,"15min":1.514},"min":{"1min":0.749,"5min":0.377,"15min":0.377},"max":{"1min":9.6010000000000009,"5min":13.848000000000001,"15min":13.848000000000001},"last":1.7230000000000001},{"interface":"front","average":{"1min":0.97799999999999998,"5min":1.5980000000000001,"15min":1.3180000000000001},"min":{"1min":0.44600000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.3280000000000001,"5min":10.199,"15min":10.199},"last":2.5790000000000002}]}]},{"osd":1,"up_from":607,"seq":2607045148870,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":886392,"kb_used_data":767344,"kb_used_omap":107,"kb_used_meta":118932,"kb_avail":92858760,"statfs":{"total":95995035648,"available":95087370240,"internally_reserved":0,"allocated":785760256,"data_stored":784598631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":121787051},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":2.177,"15min":1.1799999999999999},"min":{"1min":0.30599999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":7.9630000000000001,"5min":84.421000000000006,"15min":84.421000000000006},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.5980000000000001,"5min":2.3279999999999998,"15min":1.2649999999999999},"min":{"1min":0.20899999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":7.9589999999999996,"5min":84.489999999999995,"15min":84.489999999999995},"last":0.215}]},{"osd":2,"last update":"Mon Oct 9 09:30:36 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":3.1360000000000001,"15min":1.54},"min":{"1min":0.25600000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":3.4790000000000001,"5min":84.578999999999994,"15min":84.578999999999994},"last":4.641},{"interface":"front","average":{"1min":0.64000000000000001,"5min":2.7160000000000002,"15min":1.456},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.099,"5min":84.697000000000003,"15min":84.697000000000003},"last":0.379}]},{"osd":3,"last update":"Mon Oct 9 09:29:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":5.2270000000000003,"15min":2.2919999999999998},"min":{"1min":0.32200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.093,"5min":107.426,"15min":107.426},"last":0.59199999999999997},{"interface":"front","average":{"1min":2.4020000000000001,"5min":5.6989999999999998,"15min":2.4900000000000002},"min":{"1min":0.55500000000000005,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.058999999999999,"5min":106.15900000000001,"15min":106.15900000000001},"last":4.2229999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":2.2789999999999999,"15min":1.454},"min":{"1min":0.36199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.3810000000000002,"5min":84.875,"15min":84.875},"last":0.753},{"interface":"front","average":{"1min":1.6759999999999999,"5min":2.3799999999999999,"15min":1.4910000000000001},"min":{"1min":0.31,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.0709999999999997,"5min":84.992999999999995,"15min":84.992999999999995},"last":0.31900000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":2.323,"15min":1.508},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.4260000000000002,"5min":84.805000000000007,"15min":84.805000000000007},"last":2.569},{"interface":"front","average":{"1min":2.3490000000000002,"5min":2.5899999999999999,"15min":1.5920000000000001},"min":{"1min":0.34000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.0459999999999994,"5min":84.951999999999998,"15min":84.951999999999998},"last":1.927}]},{"osd":6,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":2.6360000000000001,"15min":1.6020000000000001},"min":{"1min":0.47699999999999998,"5min":0.379,"15min":0.379},"max":{"1min":3.75,"5min":85.016000000000005,"15min":85.016000000000005},"last":4.0890000000000004},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.4380000000000002,"15min":1.579},"min":{"1min":0.47399999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":7.4050000000000002,"5min":84.831999999999994,"15min":84.831999999999994},"last":0.77500000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":3.0819999999999999,"15min":1.804},"min":{"1min":0.47599999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.5169999999999999,"5min":84.888000000000005,"15min":84.888000000000005},"last":9.7970000000000006},{"interface":"front","average":{"1min":1.907,"5min":2.488,"15min":1.647},"min":{"1min":0.51100000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":7.3460000000000001,"5min":84.742000000000004,"15min":84.742000000000004},"last":3.2909999999999999}]}]},{"osd":0,"up_from":581,"seq":2495375999179,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1107912,"kb_used_data":962176,"kb_used_omap":131,"kb_used_meta":145596,"kb_avail":92637240,"statfs":{"total":95995035648,"available":94860533760,"internally_reserved":0,"allocated":985268224,"data_stored":984092787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":149090905},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5539999999999998,"5min":1.8340000000000001,"15min":1.1080000000000001},"min":{"1min":0.26600000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.1039999999999992,"5min":14.44,"15min":14.44},"last":5.452},{"interface":"front","average":{"1min":2.6869999999999998,"5min":1.944,"15min":1.1819999999999999},"min":{"1min":0.28699999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.8870000000000005,"5min":14.663,"15min":14.663},"last":1.786}]},{"osd":2,"last update":"Mon Oct 9 09:30:43 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":2.9580000000000002,"15min":1.5189999999999999},"min":{"1min":0.55600000000000005,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":5.4770000000000003,"5min":60.151000000000003,"15min":60.151000000000003},"last":2.7109999999999999},{"interface":"front","average":{"1min":1.2390000000000001,"5min":2.2679999999999998,"15min":1.276},"min":{"1min":0.44,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.2939999999999996,"5min":59.253,"15min":59.253},"last":2.6970000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:30:07 2023","interfaces":[{"interface":"back","average":{"1min":2.5510000000000002,"5min":3.2149999999999999,"15min":1.621},"min":{"1min":0.27900000000000003,"5min":0.252,"15min":0.252},"max":{"1min":7.4000000000000004,"5min":122.749,"15min":122.749},"last":1.9990000000000001},{"interface":"front","average":{"1min":2.5699999999999998,"5min":3.2919999999999998,"15min":1.724},"min":{"1min":0.434,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.4240000000000004,"5min":123.51900000000001,"15min":123.51900000000001},"last":2.1139999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:29:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.4570000000000001,"15min":1.1679999999999999},"min":{"1min":0.66300000000000003,"5min":0.312,"15min":0.312},"max":{"1min":5.0010000000000003,"5min":7.093,"15min":7.093},"last":2.5569999999999999},{"interface":"front","average":{"1min":2.2639999999999998,"5min":1.6850000000000001,"15min":1.2290000000000001},"min":{"1min":0.35399999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.7680000000000007,"5min":8.7680000000000007,"15min":8.7680000000000007},"last":1.7230000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:29:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.4470000000000001,"15min":1.2010000000000001},"min":{"1min":0.40699999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.085,"5min":7.3620000000000001,"15min":7.3620000000000001},"last":2.629},{"interface":"front","average":{"1min":2.016,"5min":1.7909999999999999,"15min":1.3069999999999999},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.312},"max":{"1min":8.8529999999999998,"5min":8.8529999999999998,"15min":8.8529999999999998},"last":1.7649999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:29:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":1.9219999999999999,"15min":1.395},"min":{"1min":0.45300000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.4710000000000001,"5min":8.202,"15min":8.202},"last":1.8280000000000001},{"interface":"front","average":{"1min":1.8799999999999999,"5min":1.599,"15min":1.2969999999999999},"min":{"1min":0.38800000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.0540000000000003,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":2.5960000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:29:56 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":1.931,"15min":1.4390000000000001},"min":{"1min":0.47599999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.4480000000000004,"5min":8.1769999999999996,"15min":8.1769999999999996},"last":1.8460000000000001},{"interface":"front","average":{"1min":2.609,"5min":1.823,"15min":1.411},"min":{"1min":0.55000000000000004,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":10.571,"5min":10.571,"15min":10.571},"last":2.6640000000000001}]}]},{"osd":3,"up_from":663,"seq":2847563317435,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1440812,"kb_used_data":1197988,"kb_used_omap":55,"kb_used_meta":242760,"kb_avail":92304340,"statfs":{"total":95995035648,"available":94519644160,"internally_reserved":0,"allocated":1226739712,"data_stored":1225695955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":248586850},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.411,"5min":1.8480000000000001,"15min":1.1439999999999999},"min":{"1min":0.221,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.395,"5min":10.395,"15min":10.395},"last":0.219},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.911,"15min":1.1559999999999999},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":6.1349999999999998,"5min":11.507999999999999,"15min":11.507999999999999},"last":3.1859999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":2.2320000000000002,"15min":1.306},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.6579999999999995,"5min":14.206,"15min":14.206},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.4359999999999999,"5min":2.198,"15min":1.2909999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":10.433,"5min":10.433,"15min":10.433},"last":1.5680000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.2639999999999998,"5min":2.4569999999999999,"15min":1.389},"min":{"1min":0.42499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.363,"5min":10.363,"15min":10.363},"last":2.8359999999999999},{"interface":"front","average":{"1min":2.1230000000000002,"5min":2.3610000000000002,"15min":1.373},"min":{"1min":0.27400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":8.8559999999999999,"5min":13.856,"15min":13.856},"last":3.0859999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":1.845,"15min":1.3120000000000001},"min":{"1min":0.37,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.9180000000000001,"5min":12.629,"15min":12.629},"last":2.246},{"interface":"front","average":{"1min":1.7669999999999999,"5min":1.7609999999999999,"15min":1.252},"min":{"1min":0.32200000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.8239999999999998,"5min":7.827,"15min":7.827},"last":3.052}]},{"osd":5,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5430000000000001,"5min":2.218,"15min":1.488},"min":{"1min":0.57899999999999996,"5min":0.315,"15min":0.315},"max":{"1min":9.6370000000000005,"5min":11.272,"15min":11.272},"last":1.407},{"interface":"front","average":{"1min":2.0979999999999999,"5min":2.0299999999999998,"15min":1.4339999999999999},"min":{"1min":0.46600000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.6890000000000001,"5min":11.388999999999999,"15min":11.388999999999999},"last":0.51300000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":2.0209999999999999,"15min":1.3959999999999999},"min":{"1min":0.34399999999999997,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.6799999999999997,"5min":8.2129999999999992,"15min":8.2129999999999992},"last":2.3690000000000002},{"interface":"front","average":{"1min":2.2759999999999998,"5min":2.1600000000000001,"15min":1.496},"min":{"1min":0.42499999999999999,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":9.2910000000000004,"5min":17.234999999999999,"15min":17.234999999999999},"last":0.54100000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.899,"5min":2.133,"15min":1.5069999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.6779999999999999,"5min":7.8220000000000001,"15min":7.8220000000000001},"last":1.444},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.0030000000000001,"15min":1.4339999999999999},"min":{"1min":0.372,"5min":0.318,"15min":0.318},"max":{"1min":5.6470000000000002,"5min":7.5869999999999997,"15min":7.5869999999999997},"last":2.4049999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986569,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1056280,"kb_used_data":750864,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92688872,"statfs":{"total":95995035648,"available":94913404928,"internally_reserved":0,"allocated":768884736,"data_stored":768621874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:30:43 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.2669999999999999,"15min":1.014},"min":{"1min":0.30499999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.117,"5min":8.532,"15min":8.532},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.81000000000000005,"5min":1.1140000000000001,"15min":1.008},"min":{"1min":0.46700000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.3480000000000001,"5min":7.5149999999999997,"15min":7.5149999999999997},"last":1.143}]},{"osd":1,"last update":"Mon Oct 9 09:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.47,"15min":1.097},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.6959999999999997,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.385,"15min":1.1140000000000001},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.6760000000000002,"5min":7.6589999999999998,"15min":7.6589999999999998},"last":1.542}]},{"osd":2,"last update":"Mon Oct 9 09:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.726,"15min":1.2170000000000001},"min":{"1min":0.46899999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.4779999999999998,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":1.4450000000000001},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.3759999999999999,"15min":1.1000000000000001},"min":{"1min":0.39600000000000002,"5min":0.317,"15min":0.317},"max":{"1min":3.8420000000000001,"5min":10.773999999999999,"15min":10.773999999999999},"last":0.83199999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:30:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":2.23,"15min":1.4199999999999999},"min":{"1min":0.40300000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":9.5670000000000002,"5min":17.376000000000001,"15min":17.376000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.137,"5min":2.2229999999999999,"15min":1.486},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":8.657,"5min":17.277999999999999,"15min":17.277999999999999},"last":0.85999999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.5620000000000001,"15min":1.163},"min":{"1min":0.38700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.516,"5min":9.2959999999999994,"15min":9.2959999999999994},"last":1.488},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.607,"15min":1.274},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.7450000000000001,"5min":7.5819999999999999,"15min":7.5819999999999999},"last":1.167}]},{"osd":6,"last update":"Mon Oct 9 09:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.375,"15min":1.167},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":2.6219999999999999,"5min":7.625,"15min":7.625},"last":4.8319999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.413,"15min":1.208},"min":{"1min":0.42399999999999999,"5min":0.371,"15min":0.371},"max":{"1min":2.5539999999999998,"5min":7.1440000000000001,"15min":7.1440000000000001},"last":3.7730000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":1.361,"15min":1.1539999999999999},"min":{"1min":0.44900000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":7.5449999999999999,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":1.5269999999999999},{"interface":"front","average":{"1min":1.343,"5min":1.71,"15min":1.3300000000000001},"min":{"1min":0.54600000000000004,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.8719999999999999,"5min":9.1780000000000008,"15min":9.1780000000000008},"last":0.52500000000000002}]}]},{"osd":5,"up_from":37,"seq":158913790335,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907784,"kb_used_data":602368,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92837368,"statfs":{"total":95995035648,"available":95065464832,"internally_reserved":0,"allocated":616824832,"data_stored":616574098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:30:43 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":1.5429999999999999,"15min":1.1000000000000001},"min":{"1min":0.318,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.575,"5min":10.795,"15min":10.795},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.609,"15min":1.147},"min":{"1min":0.55400000000000005,"5min":0.246,"15min":0.246},"max":{"1min":5.4560000000000004,"5min":7.5250000000000004,"15min":7.5250000000000004},"last":0.375}]},{"osd":1,"last update":"Mon Oct 9 09:30:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4450000000000001,"5min":1.462,"15min":1.123},"min":{"1min":0.32200000000000001,"5min":0.313,"15min":0.313},"max":{"1min":5.8860000000000001,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.581,"5min":1.5640000000000001,"15min":1.228},"min":{"1min":0.39600000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.2350000000000003,"5min":8.7210000000000001,"15min":8.7210000000000001},"last":0.498}]},{"osd":2,"last update":"Mon Oct 9 09:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.909,"15min":1.3220000000000001},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.133,"5min":14.019,"15min":14.019},"last":0.436},{"interface":"front","average":{"1min":0.90100000000000002,"5min":2.1549999999999998,"15min":1.4159999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.0579999999999998,"5min":14.037000000000001,"15min":14.037000000000001},"last":0.54700000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:30:57 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.917,"15min":1.288},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.0790000000000002,"5min":10.747,"15min":10.747},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.583,"15min":1.2529999999999999},"min":{"1min":0.39700000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.994,"5min":7.5579999999999998,"15min":7.5579999999999998},"last":1.268}]},{"osd":4,"last update":"Mon Oct 9 09:30:17 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.335,"15min":1.0680000000000001},"min":{"1min":0.34300000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.9400000000000004,"5min":10.747,"15min":10.747},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.357,"5min":1.2929999999999999,"15min":1.1319999999999999},"min":{"1min":0.40899999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.3109999999999999,"5min":7.5780000000000003,"15min":7.5780000000000003},"last":0.58999999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.49,"15min":1.2010000000000001},"min":{"1min":0.44900000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.6020000000000001,"5min":14.518000000000001,"15min":14.518000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.5640000000000001,"15min":1.351},"min":{"1min":0.55300000000000005,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.2189999999999999,"5min":8.0280000000000005,"15min":8.0280000000000005},"last":0.625}]},{"osd":7,"last update":"Mon Oct 9 09:30:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.6339999999999999,"15min":1.3460000000000001},"min":{"1min":0.44700000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.2199999999999998,"5min":8.6359999999999992,"15min":8.6359999999999992},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.5669999999999999,"5min":1.6919999999999999,"15min":1.321},"min":{"1min":0.41899999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.6360000000000001,"5min":7.8559999999999999,"15min":7.8559999999999999},"last":0.53500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1021247488,"data_stored":1021247488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":811270144,"data_stored":811270144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":882049024,"data_stored":882049024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1267662848,"data_stored":1267662848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":772734976,"data_stored":772734976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":617414656,"data_stored":617414656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":784924672,"data_stored":784924672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":782630912,"data_stored":782630912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:31:04.589 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T09:31:04.590 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pg_num 26 2023-10-09T09:31:04.673 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:04.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:04.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:04.800 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:04.829 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:04.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:04.935 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:04.962 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:04.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:04.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:05.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:05.102 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:05.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:05.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:05.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:05.236 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:05.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:05.341 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:05.367 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:05.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:05.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:05.470 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:05.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:05.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:05.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:05.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 15 43231 43216 117.422 82.3125 0.00566815 0.00850961 2023-10-09T09:31:05.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:05.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:05.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:05.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:05.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:05.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:05.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:05.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:05.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:05.887 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:05.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:05.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:05.994 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:06.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:06.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:06.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:06.131 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:06.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:06.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:06.260 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:06.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:06.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:06.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:06.387 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:06.410 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:06.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:06.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:06.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:06.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 44676 44660 116.29 90.25 0.0049846 0.00859316 2023-10-09T09:31:06.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:06.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:06.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:06.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:06.657 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:06 smithi181 bash[59345]: cluster 2023-10-09T09:31:05.712102+0000 mgr.y (mgr.34107) 745 : cluster 0 pgmap v1452: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T09:31:06.674 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:06.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:06.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:06.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:06.807 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:06.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:06.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:06.812 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:06 smithi163 bash[75425]: cluster 2023-10-09T09:31:05.712102+0000 mgr.y (mgr.34107) 745 : cluster 0 pgmap v1452: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T09:31:06.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:06 smithi163 bash[78107]: cluster 2023-10-09T09:31:05.712102+0000 mgr.y (mgr.34107) 745 : cluster 0 pgmap v1452: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T09:31:06.913 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:06.940 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:06.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:06.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:06.997 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:31:06.997 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T09:31:06.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T09:31:06.997 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:25136 6.8 6:1613436f:::benchmark_data_smithi163_107771_object25135:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:25:56.067678+0000", 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 310.92508497300003, 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.139517896, 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 25136 2023-10-09T09:31:06.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.067678+0000", 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.067678+0000", 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:31:06.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.067677+0000", 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1538999999999995e-05 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.067739+0000", 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2396e-05 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.067751+0000", 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5839999999999997e-06 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.067759+0000", 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042766460000000003 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.072036+0000", 2023-10-09T09:31:07.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00089293399999999996 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.072928+0000", 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0090886309999999998 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.082017+0000", 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029693699999999998 2023-10-09T09:31:07.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.084986+0000", 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0051911309999999999 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.090178+0000", 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.116904173 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.207082+0000", 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5899e-05 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.207148+0000", 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8649999999999997e-05 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.207196+0000", 2023-10-09T09:31:07.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.139517896 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:25137 6.8 6:1c7ff448:::benchmark_data_smithi163_107771_object25136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:25:56.067943+0000", 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 310.92482081600002, 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.150397322, 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.005 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.006 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.006 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.006 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 25137 2023-10-09T09:31:07.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.006 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.006 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.067943+0000", 2023-10-09T09:31:07.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.067943+0000", 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.067942+0000", 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012118200000000001 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.068063+0000", 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1154e-05 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.068074+0000", 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2239999999999998e-06 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.068081+0000", 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017158692 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.085240+0000", 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5920999999999994e-05 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.085316+0000", 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4667999999999996e-05 2023-10-09T09:31:07.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.085410+0000", 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048019769999999998 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.090212+0000", 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015733430000000001 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.091786+0000", 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.126457439 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.218243+0000", 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5451000000000002e-05 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.218299+0000", 2023-10-09T09:31:07.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1362000000000001e-05 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.218340+0000", 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.150397322 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:25140 6.8 6:1201ebcd:::benchmark_data_smithi163_107771_object25139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:25:56.068621+0000", 2023-10-09T09:31:07.012 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 310.92414210700002, 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15191300699999999, 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 25140 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.068621+0000", 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.068621+0000", 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.068620+0000", 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4795999999999994e-05 2023-10-09T09:31:07.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.068685+0000", 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1576000000000001e-05 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.068696+0000", 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3749999999999997e-06 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.068704+0000", 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017682493000000001 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.086386+0000", 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4627000000000002e-05 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.086461+0000", 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7620000000000004e-05 2023-10-09T09:31:07.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.086558+0000", 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052183120000000001 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.091777+0000", 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7156e-05 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.091794+0000", 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12864194500000001 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.220436+0000", 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9481000000000003e-05 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.220485+0000", 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9101000000000003e-05 2023-10-09T09:31:07.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.220534+0000", 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15191300699999999 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:25157 6.8 6:149eb923:::benchmark_data_smithi163_107771_object25156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:25:56.081136+0000", 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 310.91162784900001, 2023-10-09T09:31:07.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.139578542, 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 25157 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.081136+0000", 2023-10-09T09:31:07.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.081136+0000", 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.081134+0000", 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2763999999999998e-05 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.081187+0000", 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1004e-05 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.081198+0000", 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2249999999999999e-06 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.081203+0000", 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0065561350000000003 2023-10-09T09:31:07.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.087759+0000", 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4487999999999995e-05 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.087844+0000", 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012827699999999999 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.089126+0000", 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00267278 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.091799+0000", 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00027705999999999998 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.092076+0000", 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12854964399999999 2023-10-09T09:31:07.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.220626+0000", 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3581999999999997e-05 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.220669+0000", 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4776e-05 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:56.220714+0000", 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.139578542 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:40100 6.8 6:1779f28a:::benchmark_data_smithi163_107771_object40099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1082)", 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:26:06.001263+0000", 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 300.99150038300002, 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.142940386, 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.026 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 40100 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001263+0000", 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001263+0000", 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999943 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001257+0000", 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4807000000000005e-05 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001322+0000", 2023-10-09T09:31:07.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6181999999999999e-05 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001338+0000", 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0139999999999998e-06 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001346+0000", 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00041174400000000002 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001758+0000", 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014808399999999999 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001906+0000", 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000162628 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.002069+0000", 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0038700150000000001 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.005939+0000", 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00047660200000000002 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006416+0000", 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13759873 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.144014+0000", 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111971 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.144126+0000", 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7186000000000003e-05 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.144203+0000", 2023-10-09T09:31:07.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.142940386 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:40101 6.8 6:1c79d4e6:::benchmark_data_smithi163_107771_object40100:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1082)", 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:26:06.001657+0000", 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 300.99110684800002, 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14957359000000001, 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.033 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 40101 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001657+0000", 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001657+0000", 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001655+0000", 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4734000000000001e-05 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001710+0000", 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.375e-05 2023-10-09T09:31:07.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001724+0000", 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6820000000000003e-06 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.001732+0000", 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001990856 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.003723+0000", 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012714599999999999 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.003850+0000", 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016189000000000001 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.004012+0000", 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00193428 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.005946+0000", 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000964984 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006911+0000", 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.144214701 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.151126+0000", 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0476000000000002e-05 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.151186+0000", 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4192e-05 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.151230+0000", 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14957359000000001 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.039 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:40111 6.5 6:a6a3883c:::benchmark_data_smithi163_107771_object40110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1082)", 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:26:06.006416+0000", 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 300.98634774099997, 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14464113300000001, 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.040 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 40111 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006416+0000", 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006416+0000", 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:31:07.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006414+0000", 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9901000000000003e-05 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006454+0000", 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3825999999999999e-05 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006468+0000", 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1729999999999998e-06 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006475+0000", 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9407000000000003e-05 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006524+0000", 2023-10-09T09:31:07.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014226899999999999 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006666+0000", 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018903600000000001 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.006855+0000", 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001624077 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.008479+0000", 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019840019999999999 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.010463+0000", 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14034442499999999 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.150808+0000", 2023-10-09T09:31:07.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019579600000000001 2023-10-09T09:31:07.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.151004+0000", 2023-10-09T09:31:07.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3112000000000002e-05 2023-10-09T09:31:07.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.151057+0000", 2023-10-09T09:31:07.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14464113300000001 2023-10-09T09:31:07.050 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.051 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.051 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.051 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:40122 6.8 6:1cbf8bde:::benchmark_data_smithi163_107771_object40121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1082)", 2023-10-09T09:31:07.051 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:26:06.012208+0000", 2023-10-09T09:31:07.051 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 300.98055521999999, 2023-10-09T09:31:07.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13921314800000001, 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 40122 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012208+0000", 2023-10-09T09:31:07.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012208+0000", 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012206+0000", 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.359e-05 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012260+0000", 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.366e-05 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012273+0000", 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7119999999999997e-06 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012279+0000", 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8461999999999998e-05 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012328+0000", 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014688900000000001 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012475+0000", 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017881599999999999 2023-10-09T09:31:07.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012653+0000", 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015607069999999999 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.014214+0000", 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13707286399999999 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.151287+0000", 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9635999999999998e-05 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.151327+0000", 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3396000000000002e-05 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.151380+0000", 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1427999999999997e-05 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.151421+0000", 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13921314800000001 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:173109 6.5 6:ab1ead70:::benchmark_data_smithi163_107771_object62349:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:39.340995+0000", 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 147.651768366, 2023-10-09T09:31:07.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.139600426, 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 173109 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.340995+0000", 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.340995+0000", 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.340993+0000", 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1769999999999998e-06 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.341001+0000", 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2585000000000001e-05 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.341014+0000", 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2458e-05 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.341026+0000", 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00034629300000000001 2023-10-09T09:31:07.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.341373+0000", 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000149059 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.341522+0000", 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000167293 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.341689+0000", 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044445750000000001 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.346134+0000", 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043310359999999999 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.350465+0000", 2023-10-09T09:31:07.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12998227000000001 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.480447+0000", 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8394999999999997e-05 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.480525+0000", 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0090000000000001e-05 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.480595+0000", 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.139600426 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:173110 6.8 6:1be05d86:::benchmark_data_smithi163_107771_object62350:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:39.343873+0000", 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 147.64889023800001, 2023-10-09T09:31:07.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.134442335, 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 173110 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.343873+0000", 2023-10-09T09:31:07.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.343873+0000", 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.343871+0000", 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0298e-05 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.343881+0000", 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9216999999999999e-05 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.343900+0000", 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1509999999999993e-06 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.343910+0000", 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3139000000000001e-05 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.343963+0000", 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013769499999999999 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.344100+0000", 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000146525 2023-10-09T09:31:07.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.344247+0000", 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039541089999999999 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.348201+0000", 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003255179 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.351456+0000", 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.126709296 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.478165+0000", 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6187000000000007e-05 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.478252+0000", 2023-10-09T09:31:07.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3845999999999998e-05 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:39.478316+0000", 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.134442335 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:181343 6.8 6:1c2c9d62:::benchmark_data_smithi163_107771_object70583:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:51.045187+0000", 2023-10-09T09:31:07.072 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.94757654, 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.189139375, 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181343 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.045187+0000", 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.045187+0000", 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.045186+0000", 2023-10-09T09:31:07.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0770000000000002e-06 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.045193+0000", 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2729e-05 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.045206+0000", 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1269999999999997e-06 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.045213+0000", 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2753e-05 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.045275+0000", 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000133528 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.045409+0000", 2023-10-09T09:31:07.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014952499999999999 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.045558+0000", 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017885183999999998 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.063444+0000", 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2441e-05 2023-10-09T09:31:07.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.063456+0000", 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.170725498 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.234182+0000", 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9068999999999998e-05 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.234261+0000", 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5612999999999997e-05 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.234326+0000", 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.189139375 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:181345 6.8 6:194f8dbf:::benchmark_data_smithi163_107771_object70585:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:51.046047+0000", 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.94671679800001, 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.192410372, 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181345 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.080 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.046047+0000", 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.046047+0000", 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.046046+0000", 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9969999999999998e-06 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.046051+0000", 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7790000000000007e-06 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.046061+0000", 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5300000000000004e-06 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.046066+0000", 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0090127009999999997 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.055079+0000", 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013211599999999999 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.055211+0000", 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015432600000000001 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.055365+0000", 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00809671 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.063462+0000", 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053671029999999998 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.068829+0000", 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16949054799999999 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.238320+0000", 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6167000000000004e-05 2023-10-09T09:31:07.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.238396+0000", 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1286999999999998e-05 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.238457+0000", 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.192410372 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:181348 6.5 6:af6b151f:::benchmark_data_smithi163_107771_object70588:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:51.054985+0000", 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.93777830499999, 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15494364599999999, 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181348 2023-10-09T09:31:07.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.054985+0000", 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.054985+0000", 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.054984+0000", 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4919999999999996e-06 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.054991+0000", 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2224e-05 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.055003+0000", 2023-10-09T09:31:07.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3353000000000002e-05 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.055027+0000", 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00042755400000000002 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.055454+0000", 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012037200000000001 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.055575+0000", 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000122223 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.055697+0000", 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062811430000000003 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.061978+0000", 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057649010000000002 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.067743+0000", 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14186332300000001 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.209606+0000", 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000173291 2023-10-09T09:31:07.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.209779+0000", 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000149369 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.209929+0000", 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15494364599999999 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:181351 6.8 6:182ef749:::benchmark_data_smithi163_107771_object70591:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:51.057318+0000", 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.935445391, 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18127935000000001, 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.094 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181351 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.057318+0000", 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.057318+0000", 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:31:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.057317+0000", 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0469999999999996e-06 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.057324+0000", 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1382999999999999e-05 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.057335+0000", 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1009999999999997e-06 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.057341+0000", 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024093830000000002 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.059751+0000", 2023-10-09T09:31:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9795000000000004e-05 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.059830+0000", 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109584 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.059940+0000", 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0078032359999999999 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.067743+0000", 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011164860000000001 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.068860+0000", 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16964601700000001 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.238506+0000", 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1335999999999999e-05 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.238547+0000", 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0353999999999999e-05 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.238597+0000", 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18127935000000001 2023-10-09T09:31:07.100 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:181360 6.5 6:ad67b2a5:::benchmark_data_smithi163_107771_object70600:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:51.070946+0000", 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.92181726699999, 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15388995999999999, 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.101 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181360 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.070946+0000", 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.070946+0000", 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.070944+0000", 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5839999999999995e-06 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.070953+0000", 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7722000000000001e-05 2023-10-09T09:31:07.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.070971+0000", 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3999999999999992e-06 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.070979+0000", 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2954999999999998e-05 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.071022+0000", 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102069 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.071124+0000", 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011063799999999999 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.071235+0000", 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039877940000000002 2023-10-09T09:31:07.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.075223+0000", 2023-10-09T09:31:07.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086514700000000003 2023-10-09T09:31:07.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.076088+0000", 2023-10-09T09:31:07.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14868035800000001 2023-10-09T09:31:07.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.224768+0000", 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8426000000000001e-05 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.224806+0000", 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9688000000000001e-05 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.224836+0000", 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15388995999999999 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:181380 6.5 6:ac21e6c3:::benchmark_data_smithi163_107771_object70620:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:51.236300+0000", 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.75646305999999, 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.134762355, 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.108 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181380 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.236300+0000", 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.236300+0000", 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.236300+0000", 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.023e-06 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.236305+0000", 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5305999999999999e-05 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.236320+0000", 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7449999999999998e-06 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.236327+0000", 2023-10-09T09:31:07.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056484789999999997 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.241975+0000", 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019559499999999999 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.242171+0000", 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000178166 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.242349+0000", 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004573939 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.246923+0000", 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0069945429999999998 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.253917+0000", 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.116935808 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.370853+0000", 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013202900000000001 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.370985+0000", 2023-10-09T09:31:07.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7453000000000005e-05 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.371063+0000", 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.134762355 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:208032 6.8 6:15956ecb:::benchmark_data_smithi163_107771_object97272:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:31:07.115 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:29.490024+0000", 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 97.502739246000004, 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.131516311, 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 208032 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.490024+0000", 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.490024+0000", 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.490022+0000", 2023-10-09T09:31:07.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8539999999999997e-06 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.490030+0000", 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0801e-05 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.490041+0000", 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2379999999999997e-06 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.490046+0000", 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9957999999999998e-05 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.490096+0000", 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000100213 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.490196+0000", 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012241800000000001 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.490319+0000", 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039036380000000001 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.494222+0000", 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0084948179999999995 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.502717+0000", 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.118574606 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.621292+0000", 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016509399999999999 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.621457+0000", 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3491000000000004e-05 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.621540+0000", 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.131516311 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.122 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:208320 6.5 6:a058bb98:::benchmark_data_smithi163_107771_object97560:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:29.957304+0000", 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 97.035459746000001, 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.133406056, 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 208320 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.957304+0000", 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.957304+0000", 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.957303+0000", 2023-10-09T09:31:07.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9639999999999999e-06 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.957306+0000", 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3249999999999998e-06 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.957312+0000", 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2160000000000002e-06 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.957315+0000", 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0065771550000000003 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.963892+0000", 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015151499999999999 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.964043+0000", 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5688999999999998e-05 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.964139+0000", 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081213569999999992 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.972260+0000", 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0099051869999999993 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.982166+0000", 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10840301500000001 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.090569+0000", 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1327000000000003e-05 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.090640+0000", 2023-10-09T09:31:07.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9766999999999998e-05 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.090710+0000", 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.133406056 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:208327 6.5 6:acc7c7b6:::benchmark_data_smithi163_107771_object97567:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:31:07.129 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:29.960340+0000", 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 97.032423823000002, 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.140395978, 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 208327 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.960340+0000", 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.960340+0000", 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.960337+0000", 2023-10-09T09:31:07.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1589999999999999e-06 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.960346+0000", 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3169e-05 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.960359+0000", 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0870000000000001e-06 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.960366+0000", 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076177969999999999 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.967984+0000", 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000104683 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.968089+0000", 2023-10-09T09:31:07.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000122683 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.968212+0000", 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041745829999999999 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.972386+0000", 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010211533 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.982598+0000", 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.118028019 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.100626+0000", 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2300999999999999e-05 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.100688+0000", 2023-10-09T09:31:07.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7518999999999997e-05 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.100736+0000", 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.140395978 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:209500 6.5 6:a00e5bb0:::benchmark_data_smithi163_107771_object98740:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:31:07.136 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:31.693704+0000", 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 95.299059337000003, 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15211796399999999, 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 209500 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.693704+0000", 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.693704+0000", 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.693702+0000", 2023-10-09T09:31:07.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1210000000000001e-06 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.693710+0000", 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5180999999999999e-05 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.693725+0000", 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6059999999999999e-06 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.693731+0000", 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7836000000000001e-05 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.693779+0000", 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7104999999999999e-05 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.693876+0000", 2023-10-09T09:31:07.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106332 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.693983+0000", 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0037480510000000001 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.697731+0000", 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001073573 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.698804+0000", 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.146904652 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.845709+0000", 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0021e-05 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.845769+0000", 2023-10-09T09:31:07.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3102000000000001e-05 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.845822+0000", 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15211796399999999 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:31:07.143 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:31:07.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:07.174 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:07.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:07.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:07.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:07.314 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:07.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:07.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:07.418 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:07.460 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:07.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:07.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:07.542 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 45989 45973 114.92 82.0625 0.00565835 0.0086949 2023-10-09T09:31:07.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:07.573 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:31:07.593 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:31:07.601 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:07.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:07.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:07.707 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:07.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:07.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:07.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:07.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:07.918 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:07.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:07.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:08.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:08.045 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:08.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:08.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:08.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:31:08.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:08.175 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:08.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:08.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:08.311 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:08.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:08.415 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:08.440 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:08.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:08.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:08.542 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 16 47142 47126 113.272 72.0625 0.0046147 0.00882143 2023-10-09T09:31:08.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:08.576 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:08.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:08.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:08.680 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:08.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:08.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:08.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:08.767 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:08 smithi181 bash[59345]: cluster 2023-10-09T09:31:07.712794+0000 mgr.y (mgr.34107) 746 : cluster 0 pgmap v1453: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-09T09:31:08.818 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:08.850 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:08.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:08.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:08.956 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:08.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:08 smithi163 bash[75425]: cluster 2023-10-09T09:31:07.712794+0000 mgr.y (mgr.34107) 746 : cluster 0 pgmap v1453: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-09T09:31:08.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:08 smithi163 bash[78107]: cluster 2023-10-09T09:31:07.712794+0000 mgr.y (mgr.34107) 746 : cluster 0 pgmap v1453: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-09T09:31:08.991 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:08.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:09.093 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:09.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:09.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:09.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:09.251 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:09.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:09.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:09.388 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:09.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:09.440 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:31:09.492 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:09.525 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:09.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:09.542 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 48397 48381 111.981 78.4375 0.0169331 0.00892351 2023-10-09T09:31:09.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:09.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:09.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:09.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:09.760 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:09.786 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:09.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:09.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:09.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:09.913 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:09.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:10.017 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:10.041 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:10.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:10.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:10.182 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:10.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:10.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:10.285 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:10.311 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:10.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:10.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:10.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:10.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:10.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:10.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:10.542 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 49685 49669 110.856 80.5 0.0120446 0.00901558 2023-10-09T09:31:10.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:10.567 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:10.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:10.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:10.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:10.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:10.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:10.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:10.807 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:10.816 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:10 smithi163 bash[75425]: cluster 2023-10-09T09:31:09.713373+0000 mgr.y (mgr.34107) 747 : cluster 0 pgmap v1454: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T09:31:10.816 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:10 smithi163 bash[75425]: audit 2023-10-09T09:31:10.368385+0000 mon.a (mon.0) 1555 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:10.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:10 smithi163 bash[78107]: cluster 2023-10-09T09:31:09.713373+0000 mgr.y (mgr.34107) 747 : cluster 0 pgmap v1454: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T09:31:10.817 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:10 smithi163 bash[78107]: audit 2023-10-09T09:31:10.368385+0000 mon.a (mon.0) 1555 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:10.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:10.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:10.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:10.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:10.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:11.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:10 smithi181 bash[59345]: cluster 2023-10-09T09:31:09.713373+0000 mgr.y (mgr.34107) 747 : cluster 0 pgmap v1454: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T09:31:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:10 smithi181 bash[59345]: audit 2023-10-09T09:31:10.368385+0000 mon.a (mon.0) 1555 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:11.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:11.094 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:11.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:11.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:11.197 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:11.230 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:11.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:11.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:11.336 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:11.374 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:11.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:11.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:11.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:11.480 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:31:11.481 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:31:11.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:11.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:11.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:11.544 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 50828 50812 109.497 71.4375 0.00993516 0.00912644 2023-10-09T09:31:11.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:11.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:11.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:11.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:11.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:11.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:11.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:11.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:11.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:11.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:11.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:11.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:11.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:12.025 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:12.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:12.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:12.132 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:12.164 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:12.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:12.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:12.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:12.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:12.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:12.399 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:12.427 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:12.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:12.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:12.495 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:31:12.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:12.542 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 52019 52003 108.328 74.4375 0.033666 0.00922468 2023-10-09T09:31:12.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:12.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:12.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:12.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:12.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:12.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:12.778 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:12 smithi163 bash[75425]: cluster 2023-10-09T09:31:11.713916+0000 mgr.y (mgr.34107) 748 : cluster 0 pgmap v1455: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T09:31:12.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:12.825 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:12.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:12.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:12.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:12.941 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:12 smithi181 bash[59345]: cluster 2023-10-09T09:31:11.713916+0000 mgr.y (mgr.34107) 748 : cluster 0 pgmap v1455: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T09:31:12.959 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:12.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:12.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:13.062 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:13.087 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:13.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:13.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:13.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:13.217 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:13.217 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:12 smithi163 bash[78107]: cluster 2023-10-09T09:31:11.713916+0000 mgr.y (mgr.34107) 748 : cluster 0 pgmap v1455: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T09:31:13.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:13.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:13.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:13.344 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:13.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:13.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:13.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:13.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:13.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:13.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:13.542 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 53347 53331 107.51 83 0.0111659 0.00929605 2023-10-09T09:31:13.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:13.611 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:13.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:13.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:13.675 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:31:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:31:13] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.33.4" 2023-10-09T09:31:13.715 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:13.740 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:13.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:13.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:13.872 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:13.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:13.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:13.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:13 smithi163 bash[75425]: cluster 2023-10-09T09:31:13.714511+0000 mgr.y (mgr.34107) 749 : cluster 0 pgmap v1456: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:31:13.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:13 smithi163 bash[78107]: cluster 2023-10-09T09:31:13.714511+0000 mgr.y (mgr.34107) 749 : cluster 0 pgmap v1456: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:31:13.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:13.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:13 smithi181 bash[59345]: cluster 2023-10-09T09:31:13.714511+0000 mgr.y (mgr.34107) 749 : cluster 0 pgmap v1456: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:31:14.007 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:14.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:14.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:14.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:14.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:14.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:14.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:14.238 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:14.264 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:14.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:14.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:14.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:14.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:14.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:14.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:14.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:14.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:14.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:14.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:14.542 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 54579 54563 106.557 77 0.00815336 0.0093788 2023-10-09T09:31:14.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:14.650 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:14.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:14.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:14.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:14.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:14.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:14.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:14.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:14.911 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:14.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:14.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:15.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:15.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:15.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:15.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:15.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:15.179 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:15.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:15.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:15.282 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:15.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:15.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:15.411 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:15.446 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:15.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:15.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:15.542 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 55688 55672 105.428 69.3125 0.00499777 0.0094788 2023-10-09T09:31:15.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:15.580 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:15.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:15.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:15.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:15.712 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:15.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:15.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:15.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:15.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:15.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:15.943 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:15.972 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:15.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:16.076 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:16.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:16.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:16.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:16.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:16.236 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:16.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:16.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:16.342 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:16.372 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:16.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:16.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:16.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:16.499 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:16.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:16.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:16.543 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 56829 56813 104.424 71.3125 0.00837914 0.00957074 2023-10-09T09:31:16.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:16.612 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:16 smithi181 bash[59345]: cluster 2023-10-09T09:31:15.716301+0000 mgr.y (mgr.34107) 750 : cluster 0 pgmap v1457: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T09:31:16.635 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:16.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:16.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:16 smithi163 bash[75425]: cluster 2023-10-09T09:31:15.716301+0000 mgr.y (mgr.34107) 750 : cluster 0 pgmap v1457: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T09:31:16.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:16 smithi163 bash[78107]: cluster 2023-10-09T09:31:15.716301+0000 mgr.y (mgr.34107) 750 : cluster 0 pgmap v1457: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T09:31:16.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:16.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:16.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:16.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:16.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:16.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:16.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:16.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:16.997 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:17.024 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:17.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:17.127 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:17.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:17.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:17.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:17.261 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:17.286 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:17.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:17.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:17.391 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:17.420 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:17.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:17.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:17.543 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 58028 58012 103.582 74.9375 0.00633513 0.00964771 2023-10-09T09:31:17.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:17.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:17.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:17.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:17.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:17.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:17.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:17.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:17.835 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:17.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:17.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:17.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:17.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:17.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:17.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:18.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:18.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:18.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:18.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:18.213 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:18.241 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:18.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:18.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:18.347 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:18.377 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:18.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:18.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:18.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:18.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:18.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:18.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:18.543 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 59188 59172 102.718 72.5 0.0263915 0.00972965 2023-10-09T09:31:18.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:18.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:18.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:18.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:18.740 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:18.775 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:18.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:18.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:18.881 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:18.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:18.914 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:18.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:19.015 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:19.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:18 smithi181 bash[59345]: cluster 2023-10-09T09:31:17.716990+0000 mgr.y (mgr.34107) 751 : cluster 0 pgmap v1458: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T09:31:19.041 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:19.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:19.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:19.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:19.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:19.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:19.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:18 smithi163 bash[78107]: cluster 2023-10-09T09:31:17.716990+0000 mgr.y (mgr.34107) 751 : cluster 0 pgmap v1458: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T09:31:19.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:18 smithi163 bash[75425]: cluster 2023-10-09T09:31:17.716990+0000 mgr.y (mgr.34107) 751 : cluster 0 pgmap v1458: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T09:31:19.275 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:19.303 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:19.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:19.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:19.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:19.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:19.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:19.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:19.539 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:19.543 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 60102 60086 101.486 57.125 0.00586346 0.00982119 2023-10-09T09:31:19.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:19.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:19.670 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:19.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:19.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:19.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:19.800 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:19.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:19.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:19.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:19.938 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:19.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:19 smithi181 bash[59345]: cluster 2023-10-09T09:31:19.717593+0000 mgr.y (mgr.34107) 752 : cluster 0 pgmap v1459: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T09:31:19.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:19.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:19.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:19.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:19 smithi163 bash[75425]: cluster 2023-10-09T09:31:19.717593+0000 mgr.y (mgr.34107) 752 : cluster 0 pgmap v1459: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T09:31:19.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:19 smithi163 bash[78107]: cluster 2023-10-09T09:31:19.717593+0000 mgr.y (mgr.34107) 752 : cluster 0 pgmap v1459: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T09:31:20.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:20.096 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:20.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:20.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:20.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:20.225 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:20.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:20.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:20.329 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:20.362 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:20.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:20.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:20.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:20.490 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:20.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:20.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:20.543 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 61257 61241 100.715 72.1875 0.00429893 0.00992251 2023-10-09T09:31:20.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:20.621 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:20.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:20.725 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:20.751 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:20.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:20.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:20.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:20.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:20.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:20.994 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:21.024 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:21.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:21.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:21.161 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:21.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:21.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:21.265 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:21.291 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:21.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:21.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:21.420 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:21.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:21.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:21.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:21.543 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 62391 62375 99.9492 70.875 0.0175231 0.00999739 2023-10-09T09:31:21.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:21.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:21.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:21.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:21.690 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:21.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:21.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:21.793 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:21.824 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:21.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:21.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:21.962 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:21.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:21.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:22.065 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:22.093 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:22.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:22.198 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:22.226 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:22.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:22.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:22.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:22.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:22.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:22.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:22.483 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:22.506 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:22.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:22.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:22.543 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:31:22.544109+0000 min lat: 0.00155566 max lat: 0.215072 avg lat: 0.0100709 2023-10-09T09:31:22.543 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:31:22.543 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 63539 63523 99.2441 71.75 0.014415 0.0100709 2023-10-09T09:31:22.614 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:22.642 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:22.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:22.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:22.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:22.767 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:22.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:22.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:22.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:22.880 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:22 smithi181 bash[59345]: cluster 2023-10-09T09:31:21.718214+0000 mgr.y (mgr.34107) 753 : cluster 0 pgmap v1460: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T09:31:22.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:22.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:22.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:23.003 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:23.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:23.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:23.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:23.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:23.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:23.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:23.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:23.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:22 smithi163 bash[78107]: cluster 2023-10-09T09:31:21.718214+0000 mgr.y (mgr.34107) 753 : cluster 0 pgmap v1460: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T09:31:23.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:22 smithi163 bash[75425]: cluster 2023-10-09T09:31:21.718214+0000 mgr.y (mgr.34107) 753 : cluster 0 pgmap v1460: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T09:31:23.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:23.327 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:23.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:23.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:23.430 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:23.459 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:23.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:23.545 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 64696 64680 98.5871 72.3125 0.00645402 0.0101366 2023-10-09T09:31:23.564 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:23.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:23.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:23.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:23.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:23.710 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:31:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:31:23] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.33.4" 2023-10-09T09:31:23.728 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:23.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:23.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:23.832 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:23.859 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:23.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:23.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:23.963 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:23.992 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:23.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:23.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:24.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:24.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:24.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:24.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:24.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:24.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:24.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:24.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:24.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:24.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:24.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:24.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:24.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:24.522 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:24.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:24.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:24.544 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 65807 65791 97.8918 69.4375 0.0144296 0.010207 2023-10-09T09:31:24.627 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:24.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:24.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:24.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:24.762 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:24.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:24.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:24.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:24.893 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:24.923 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:24.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:24.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:25.029 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:25.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:24 smithi181 bash[59345]: cluster 2023-10-09T09:31:23.718699+0000 mgr.y (mgr.34107) 754 : cluster 0 pgmap v1461: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:31:25.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:25.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:25.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:24 smithi163 bash[78107]: cluster 2023-10-09T09:31:23.718699+0000 mgr.y (mgr.34107) 754 : cluster 0 pgmap v1461: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:31:25.074 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:24 smithi163 bash[75425]: cluster 2023-10-09T09:31:23.718699+0000 mgr.y (mgr.34107) 754 : cluster 0 pgmap v1461: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:31:25.162 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:25.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:25.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:25.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:25.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:25.302 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:31:25.302 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:31:25.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.302 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:66564 7.f 7:f7bfb30b:::benchmark_data_smithi163_130508_object66563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e1103)", 2023-10-09T09:31:25.303 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:25.287823+0000", 2023-10-09T09:31:25.303 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0095631640000000007, 2023-10-09T09:31:25.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0096060739999999992, 2023-10-09T09:31:25.303 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:31:25.303 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:31:25.303 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:31:25.303 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:31:25.303 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 66564 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287823+0000", 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:31:25.304 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287823+0000", 2023-10-09T09:31:25.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:31:25.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287822+0000", 2023-10-09T09:31:25.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:31:25.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:31:25.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287848+0000", 2023-10-09T09:31:25.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6013e-05 2023-10-09T09:31:25.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:31:25.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287860+0000", 2023-10-09T09:31:25.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2106999999999999e-05 2023-10-09T09:31:25.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:31:25.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287868+0000", 2023-10-09T09:31:25.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9170000000000006e-06 2023-10-09T09:31:25.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:31:25.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287909+0000", 2023-10-09T09:31:25.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1536999999999999e-05 2023-10-09T09:31:25.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.288050+0000", 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014078100000000001 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.288215+0000", 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016484900000000001 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:66568 7.f 7:f8551a53:::benchmark_data_smithi163_130508_object66567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e1103)", 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:25.287880+0000", 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0095065099999999993, 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0096167000000000006, 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:31:25.310 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 66568 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287880+0000", 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287880+0000", 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287880+0000", 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287901+0000", 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1404e-05 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:31:25.312 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287908+0000", 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4780000000000002e-06 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.287914+0000", 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7930000000000003e-06 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.291117+0000", 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032026429999999998 2023-10-09T09:31:25.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.294759+0000", 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003641989 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.294952+0000", 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000193017 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:31:25.314 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:66566 7.7 e1103/1099 7:ec77eacb:::benchmark_data_smithi163_130508_object66565:head v 1103'4184, mlcod=1103'4182)", 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:25.288191+0000", 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0091957180000000003, 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0093966380000000006, 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:31:25.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.288191+0000", 2023-10-09T09:31:25.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:31:25.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.288191+0000", 2023-10-09T09:31:25.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:31:25.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.288190+0000", 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.288283+0000", 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2991000000000005e-05 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.288299+0000", 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6070999999999999e-05 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.288308+0000", 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4749999999999993e-06 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.288373+0000", 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5637999999999997e-05 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:31:25.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.288395+0000", 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1508000000000001e-05 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:66567 7.7 e1103/1099 7:e4ec4682:::benchmark_data_smithi163_130508_object66566:head v 1103'4185, mlcod=1103'4182)", 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:25.290627+0000", 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0067595550000000004, 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.007030311, 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:31:25.319 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:31:25.320 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:31:25.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:31:25.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.290627+0000", 2023-10-09T09:31:25.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:31:25.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.290627+0000", 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.290626+0000", 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.290737+0000", 2023-10-09T09:31:25.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011076700000000001 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.290762+0000", 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5185e-05 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.290771+0000", 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8529999999999998e-06 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.291646+0000", 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00087560000000000003 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.291668+0000", 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1557e-05 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44921.0:66564 7.f e1103/1099)", 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:25.293281+0000", 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0041055839999999998, 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0044409860000000001, 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.293281+0000", 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.293281+0000", 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.293280+0000", 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.293287+0000", 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3790000000000003e-06 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.293302+0000", 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5716999999999999e-05 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.293310+0000", 2023-10-09T09:31:25.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9370000000000004e-06 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:66573 7.d e1103/1099)", 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:25.295521+0000", 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0018652530000000001, 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022532920000000001, 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:31:25.327 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295521+0000", 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295521+0000", 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295521+0000", 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295596+0000", 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5625999999999997e-05 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295611+0000", 2023-10-09T09:31:25.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4705999999999999e-05 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295619+0000", 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9680000000000006e-06 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:66575 7.6 7:6f6cbcec:::benchmark_data_smithi163_130508_object66574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e1103)", 2023-10-09T09:31:25.330 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:25.295938+0000", 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0014482049999999999, 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019000849999999999, 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 66575 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:31:25.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295938+0000", 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295938+0000", 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295937+0000", 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:31:25.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295970+0000", 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3531999999999997e-05 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295984+0000", 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3645999999999999e-05 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.295993+0000", 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7870000000000007e-06 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.296041+0000", 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8588000000000003e-05 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.296208+0000", 2023-10-09T09:31:25.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000166727 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.296459+0000", 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00025126 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:66577 7.9 7:9059c17a:::benchmark_data_smithi163_130508_object66576:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e1103)", 2023-10-09T09:31:25.335 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:25.297448+0000", 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 4294967295.9999385, 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00047881200000000001, 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 66577 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.297448+0000", 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.297448+0000", 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.297446+0000", 2023-10-09T09:31:25.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.297473+0000", 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6843e-05 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.297489+0000", 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5707000000000001e-05 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.297498+0000", 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0569999999999999e-06 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.297554+0000", 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5883e-05 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:31:25.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.297703+0000", 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014965200000000001 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4", 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:25.297842+0000", 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000138787 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T09:31:25.340 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 8 2023-10-09T09:31:25.341 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:31:25.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:25.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:25.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:25.443 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:25.470 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:25.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:25.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:25.544 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 66790 66774 97.0439 61.4375 0.0369319 0.0102976 2023-10-09T09:31:25.577 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:25.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:25.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:25.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:25.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:25.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:25.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:25.844 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:25.853 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:25 smithi181 bash[59345]: audit 2023-10-09T09:31:25.370197+0000 mon.a (mon.0) 1556 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:25.853 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:25 smithi181 bash[59345]: cluster 2023-10-09T09:31:25.719284+0000 mgr.y (mgr.34107) 755 : cluster 0 pgmap v1462: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:31:25.873 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:25.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:25.918 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:31:25.962 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:31:25.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:25 smithi163 bash[75425]: audit 2023-10-09T09:31:25.370197+0000 mon.a (mon.0) 1556 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:25 smithi163 bash[75425]: cluster 2023-10-09T09:31:25.719284+0000 mgr.y (mgr.34107) 755 : cluster 0 pgmap v1462: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:25 smithi163 bash[78107]: audit 2023-10-09T09:31:25.370197+0000 mon.a (mon.0) 1556 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:25 smithi163 bash[78107]: cluster 2023-10-09T09:31:25.719284+0000 mgr.y (mgr.34107) 755 : cluster 0 pgmap v1462: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T09:31:25.976 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:26.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:26.009 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:31:26.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:26.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:26.045 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:31:26.082 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:31:26.120 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:26.121 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:31:26.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:26.157 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:31:26.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:26.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:26.188 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:31:26.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:31:26.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:31:26.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:31:26.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:31:26.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:31:26.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:31:26.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:31:26.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:31:26.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:26.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:26.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:26.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:26.394 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:26.419 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:26.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:26.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:26.524 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:26.544 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 67679 67663 96.101 55.5625 0.0198444 0.0104 2023-10-09T09:31:26.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:26.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:26.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:26.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:26.687 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:26.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:26.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:26.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:26.803 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 7 pg_num to 26 2023-10-09T09:31:26.808 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:31:26.815 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:26.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:26.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:26.918 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:26.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:26 smithi163 bash[75425]: audit 2023-10-09T09:31:25.839473+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.163:0/4275466740' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:31:26.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:26 smithi163 bash[75425]: audit 2023-10-09T09:31:25.839933+0000 mon.a (mon.0) 1557 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:31:26.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:26 smithi163 bash[75425]: audit 2023-10-09T09:31:25.856553+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.163:0/2804660635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:31:26.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:26 smithi163 bash[75425]: audit 2023-10-09T09:31:26.232185+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.163:0/1087760180' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:31:26.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:26 smithi163 bash[75425]: audit 2023-10-09T09:31:26.232575+0000 mon.a (mon.0) 1558 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:31:26.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:26 smithi163 bash[78107]: audit 2023-10-09T09:31:25.839473+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.163:0/4275466740' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:31:26.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:26 smithi163 bash[78107]: audit 2023-10-09T09:31:25.839933+0000 mon.a (mon.0) 1557 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:31:26.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:26 smithi163 bash[78107]: audit 2023-10-09T09:31:25.856553+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.163:0/2804660635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:31:26.929 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:26 smithi163 bash[78107]: audit 2023-10-09T09:31:26.232185+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.163:0/1087760180' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:31:26.929 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:26 smithi163 bash[78107]: audit 2023-10-09T09:31:26.232575+0000 mon.a (mon.0) 1558 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:31:26.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:26.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:26.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:27.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:26 smithi181 bash[59345]: audit 2023-10-09T09:31:25.839473+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.163:0/4275466740' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:31:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:26 smithi181 bash[59345]: audit 2023-10-09T09:31:25.839933+0000 mon.a (mon.0) 1557 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:31:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:26 smithi181 bash[59345]: audit 2023-10-09T09:31:25.856553+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.163:0/2804660635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:31:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:26 smithi181 bash[59345]: audit 2023-10-09T09:31:26.232185+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.163:0/1087760180' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:31:27.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:26 smithi181 bash[59345]: audit 2023-10-09T09:31:26.232575+0000 mon.a (mon.0) 1558 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:31:27.069 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:27.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:27.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:27.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:27.198 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:27.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:27.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:27.301 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:27.326 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:27.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:27.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:27.365 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T09:31:27.432 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:27.462 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:27.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:27.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:27.544 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 68728 68712 95.4222 65.5625 0.00478745 0.0104729 2023-10-09T09:31:27.567 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:27.593 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:27.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:27.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:27.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:27.732 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:27.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:27.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:27.837 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:27.862 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:27.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:27.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[75425]: cluster 2023-10-09T09:31:26.779702+0000 mon.a (mon.0) 1559 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:31:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[75425]: cluster 2023-10-09T09:31:26.791747+0000 osd.5 (osd.5) 649 : cluster 3 failed to encode map e1104 with expected crc 2023-10-09T09:31:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[75425]: audit 2023-10-09T09:31:26.794842+0000 mon.a (mon.0) 1560 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:31:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[75425]: audit 2023-10-09T09:31:26.795005+0000 mon.a (mon.0) 1561 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:31:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[75425]: cluster 2023-10-09T09:31:26.795780+0000 mon.a (mon.0) 1562 : cluster 0 osdmap e1104: 8 total, 8 up, 8 in 2023-10-09T09:31:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[75425]: cluster 2023-10-09T09:31:26.798629+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e1104 with expected crc 2023-10-09T09:31:27.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[75425]: cluster 2023-10-09T09:31:26.801847+0000 osd.4 (osd.4) 527 : cluster 3 failed to encode map e1104 with expected crc 2023-10-09T09:31:27.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[75425]: cluster 2023-10-09T09:31:27.719853+0000 mgr.y (mgr.34107) 756 : cluster 0 pgmap v1464: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T09:31:27.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[75425]: audit 2023-10-09T09:31:27.720882+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:31:27.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[78107]: cluster 2023-10-09T09:31:26.779702+0000 mon.a (mon.0) 1559 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:31:27.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[78107]: cluster 2023-10-09T09:31:26.791747+0000 osd.5 (osd.5) 649 : cluster 3 failed to encode map e1104 with expected crc 2023-10-09T09:31:27.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[78107]: audit 2023-10-09T09:31:26.794842+0000 mon.a (mon.0) 1560 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:31:27.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[78107]: audit 2023-10-09T09:31:26.795005+0000 mon.a (mon.0) 1561 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:31:27.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[78107]: cluster 2023-10-09T09:31:26.795780+0000 mon.a (mon.0) 1562 : cluster 0 osdmap e1104: 8 total, 8 up, 8 in 2023-10-09T09:31:27.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[78107]: cluster 2023-10-09T09:31:26.798629+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e1104 with expected crc 2023-10-09T09:31:27.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[78107]: cluster 2023-10-09T09:31:26.801847+0000 osd.4 (osd.4) 527 : cluster 3 failed to encode map e1104 with expected crc 2023-10-09T09:31:27.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[78107]: cluster 2023-10-09T09:31:27.719853+0000 mgr.y (mgr.34107) 756 : cluster 0 pgmap v1464: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T09:31:27.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:27 smithi163 bash[78107]: audit 2023-10-09T09:31:27.720882+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:31:27.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:28.002 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:28.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:28.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:27 smithi181 bash[59345]: cluster 2023-10-09T09:31:26.779702+0000 mon.a (mon.0) 1559 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:31:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:27 smithi181 bash[59345]: cluster 2023-10-09T09:31:26.791747+0000 osd.5 (osd.5) 649 : cluster 3 failed to encode map e1104 with expected crc 2023-10-09T09:31:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:27 smithi181 bash[59345]: audit 2023-10-09T09:31:26.794842+0000 mon.a (mon.0) 1560 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:31:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:27 smithi181 bash[59345]: audit 2023-10-09T09:31:26.795005+0000 mon.a (mon.0) 1561 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:31:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:27 smithi181 bash[59345]: cluster 2023-10-09T09:31:26.795780+0000 mon.a (mon.0) 1562 : cluster 0 osdmap e1104: 8 total, 8 up, 8 in 2023-10-09T09:31:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:27 smithi181 bash[59345]: cluster 2023-10-09T09:31:26.798629+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e1104 with expected crc 2023-10-09T09:31:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:27 smithi181 bash[59345]: cluster 2023-10-09T09:31:26.801847+0000 osd.4 (osd.4) 527 : cluster 3 failed to encode map e1104 with expected crc 2023-10-09T09:31:28.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:27 smithi181 bash[59345]: cluster 2023-10-09T09:31:27.719853+0000 mgr.y (mgr.34107) 756 : cluster 0 pgmap v1464: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T09:31:28.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:27 smithi181 bash[59345]: audit 2023-10-09T09:31:27.720882+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:31:28.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:28.138 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:28.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:28.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:28.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:28.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:28.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:28.385 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:28.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:28.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:28.489 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:28.521 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:28.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:28.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:28.544 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 69012 68996 93.7336 17.75 0.284015 0.0105013 2023-10-09T09:31:28.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:28.661 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:28.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:28.765 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:28.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:28.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:28.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:28.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:28.944 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:28.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:28.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:29.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:29.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:28 smithi181 bash[59345]: cluster 2023-10-09T09:31:27.805163+0000 osd.5 (osd.5) 650 : cluster 3 failed to encode map e1105 with expected crc 2023-10-09T09:31:29.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:28 smithi181 bash[59345]: audit 2023-10-09T09:31:27.805514+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:31:29.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:28 smithi181 bash[59345]: cluster 2023-10-09T09:31:27.805587+0000 mon.a (mon.0) 1565 : cluster 0 osdmap e1105: 8 total, 8 up, 8 in 2023-10-09T09:31:29.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:28 smithi181 bash[59345]: cluster 2023-10-09T09:31:27.805635+0000 osd.4 (osd.4) 528 : cluster 3 failed to encode map e1105 with expected crc 2023-10-09T09:31:29.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:28 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.008299+0000 osd.7 (osd.7) 493 : cluster 3 failed to encode map e1105 with expected crc 2023-10-09T09:31:29.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:28 smithi163 bash[78107]: cluster 2023-10-09T09:31:27.805163+0000 osd.5 (osd.5) 650 : cluster 3 failed to encode map e1105 with expected crc 2023-10-09T09:31:29.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:28 smithi163 bash[78107]: audit 2023-10-09T09:31:27.805514+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:31:29.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:28 smithi163 bash[78107]: cluster 2023-10-09T09:31:27.805587+0000 mon.a (mon.0) 1565 : cluster 0 osdmap e1105: 8 total, 8 up, 8 in 2023-10-09T09:31:29.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:28 smithi163 bash[78107]: cluster 2023-10-09T09:31:27.805635+0000 osd.4 (osd.4) 528 : cluster 3 failed to encode map e1105 with expected crc 2023-10-09T09:31:29.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:28 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.008299+0000 osd.7 (osd.7) 493 : cluster 3 failed to encode map e1105 with expected crc 2023-10-09T09:31:29.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:28 smithi163 bash[75425]: cluster 2023-10-09T09:31:27.805163+0000 osd.5 (osd.5) 650 : cluster 3 failed to encode map e1105 with expected crc 2023-10-09T09:31:29.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:28 smithi163 bash[75425]: audit 2023-10-09T09:31:27.805514+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:31:29.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:28 smithi163 bash[75425]: cluster 2023-10-09T09:31:27.805587+0000 mon.a (mon.0) 1565 : cluster 0 osdmap e1105: 8 total, 8 up, 8 in 2023-10-09T09:31:29.062 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:28 smithi163 bash[75425]: cluster 2023-10-09T09:31:27.805635+0000 osd.4 (osd.4) 528 : cluster 3 failed to encode map e1105 with expected crc 2023-10-09T09:31:29.062 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:28 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.008299+0000 osd.7 (osd.7) 493 : cluster 3 failed to encode map e1105 with expected crc 2023-10-09T09:31:29.080 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:29.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:29.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:29.186 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:29.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:29.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:29.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:29.316 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:29.346 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:29.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:29.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:29.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:29.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:29.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:29.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:29.544 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 69919 69903 92.9453 56.6875 0.0110791 0.0107533 2023-10-09T09:31:29.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:29.608 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:29.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:29.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:29.712 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:29.751 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:29.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:29.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:29.898 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:29.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:29.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:30.003 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:30.044 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:30.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.808346+0000 osd.5 (osd.5) 651 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.809142+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.810575+0000 mon.a (mon.0) 1566 : cluster 0 osdmap e1106: 8 total, 8 up, 8 in 2023-10-09T09:31:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.811389+0000 osd.5 (osd.5) 652 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.811447+0000 osd.4 (osd.4) 529 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.811482+0000 osd.7 (osd.7) 494 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.812810+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.813615+0000 osd.7 (osd.7) 495 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.814066+0000 osd.4 (osd.4) 530 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.814825+0000 osd.5 (osd.5) 653 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.815777+0000 osd.4 (osd.4) 531 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:28.999867+0000 osd.7 (osd.7) 496 : cluster 0 7.5 scrub starts 2023-10-09T09:31:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:29.037129+0000 osd.5 (osd.5) 654 : cluster 0 7.0 scrub starts 2023-10-09T09:31:30.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:29 smithi181 bash[59345]: cluster 2023-10-09T09:31:29.720463+0000 mgr.y (mgr.34107) 757 : cluster 0 pgmap v1467: 99 pgs: 8 unknown, 91 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T09:31:30.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:30.176 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:30.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:30.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:30.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.808346+0000 osd.5 (osd.5) 651 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.809142+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.810575+0000 mon.a (mon.0) 1566 : cluster 0 osdmap e1106: 8 total, 8 up, 8 in 2023-10-09T09:31:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.811389+0000 osd.5 (osd.5) 652 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.811447+0000 osd.4 (osd.4) 529 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.811482+0000 osd.7 (osd.7) 494 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.812810+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.813615+0000 osd.7 (osd.7) 495 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.814066+0000 osd.4 (osd.4) 530 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.814825+0000 osd.5 (osd.5) 653 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.815777+0000 osd.4 (osd.4) 531 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:28.999867+0000 osd.7 (osd.7) 496 : cluster 0 7.5 scrub starts 2023-10-09T09:31:30.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:29.037129+0000 osd.5 (osd.5) 654 : cluster 0 7.0 scrub starts 2023-10-09T09:31:30.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[78107]: cluster 2023-10-09T09:31:29.720463+0000 mgr.y (mgr.34107) 757 : cluster 0 pgmap v1467: 99 pgs: 8 unknown, 91 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T09:31:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.808346+0000 osd.5 (osd.5) 651 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.809142+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.810575+0000 mon.a (mon.0) 1566 : cluster 0 osdmap e1106: 8 total, 8 up, 8 in 2023-10-09T09:31:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.811389+0000 osd.5 (osd.5) 652 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.811447+0000 osd.4 (osd.4) 529 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.811482+0000 osd.7 (osd.7) 494 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.812810+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.813615+0000 osd.7 (osd.7) 495 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.814066+0000 osd.4 (osd.4) 530 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.814825+0000 osd.5 (osd.5) 653 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.815777+0000 osd.4 (osd.4) 531 : cluster 3 failed to encode map e1106 with expected crc 2023-10-09T09:31:30.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:28.999867+0000 osd.7 (osd.7) 496 : cluster 0 7.5 scrub starts 2023-10-09T09:31:30.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:29.037129+0000 osd.5 (osd.5) 654 : cluster 0 7.0 scrub starts 2023-10-09T09:31:30.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:29 smithi163 bash[75425]: cluster 2023-10-09T09:31:29.720463+0000 mgr.y (mgr.34107) 757 : cluster 0 pgmap v1467: 99 pgs: 8 unknown, 91 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T09:31:30.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:30.308 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:30.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:30.414 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:30.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:30.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:30.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:30.544 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 71294 71278 92.7992 85.9375 0.0128995 0.0107707 2023-10-09T09:31:30.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:30.577 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:30.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:30.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:30.680 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:30.716 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:30.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:30.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:30.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:30.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:30.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:30.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:30.959 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:30.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:30.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:31.062 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:31:31.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:31.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:31.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:31.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:31.224 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:31.253 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:31.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:31.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:31.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:31.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:31.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:31.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:31.500 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:31.529 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:31.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:31.544 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 72483 72467 92.4217 74.3125 0.0074071 0.0108148 2023-10-09T09:31:31.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:31.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:31.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:31.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:31 smithi163 bash[78107]: cluster 2023-10-09T09:31:29.845725+0000 osd.7 (osd.7) 497 : cluster 0 7.5 scrub ok 2023-10-09T09:31:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:31 smithi163 bash[78107]: cluster 2023-10-09T09:31:29.987086+0000 osd.5 (osd.5) 655 : cluster 0 7.0 scrub ok 2023-10-09T09:31:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:31 smithi163 bash[78107]: audit 2023-10-09T09:31:30.643100+0000 mon.a (mon.0) 1567 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:31:31.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:31 smithi163 bash[78107]: cluster 2023-10-09T09:31:31.084001+0000 osd.4 (osd.4) 532 : cluster 0 7.1 scrub starts 2023-10-09T09:31:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:31 smithi163 bash[75425]: cluster 2023-10-09T09:31:29.845725+0000 osd.7 (osd.7) 497 : cluster 0 7.5 scrub ok 2023-10-09T09:31:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:31 smithi163 bash[75425]: cluster 2023-10-09T09:31:29.987086+0000 osd.5 (osd.5) 655 : cluster 0 7.0 scrub ok 2023-10-09T09:31:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:31 smithi163 bash[75425]: audit 2023-10-09T09:31:30.643100+0000 mon.a (mon.0) 1567 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:31:31.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:31 smithi163 bash[75425]: cluster 2023-10-09T09:31:31.084001+0000 osd.4 (osd.4) 532 : cluster 0 7.1 scrub starts 2023-10-09T09:31:31.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:31.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:31 smithi181 bash[59345]: cluster 2023-10-09T09:31:29.845725+0000 osd.7 (osd.7) 497 : cluster 0 7.5 scrub ok 2023-10-09T09:31:31.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:31 smithi181 bash[59345]: cluster 2023-10-09T09:31:29.987086+0000 osd.5 (osd.5) 655 : cluster 0 7.0 scrub ok 2023-10-09T09:31:31.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:31 smithi181 bash[59345]: audit 2023-10-09T09:31:30.643100+0000 mon.a (mon.0) 1567 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:31:31.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:31 smithi181 bash[59345]: cluster 2023-10-09T09:31:31.084001+0000 osd.4 (osd.4) 532 : cluster 0 7.1 scrub starts 2023-10-09T09:31:31.790 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:31.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:31.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:31.922 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:31.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:31.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:31.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:31:32.025 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:32.058 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:32.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:32.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:32.161 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:32.188 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:32.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:32.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:32.233 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:31:32.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:32.322 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:32.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:32.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:32.426 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:32.456 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:32.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:32.545 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 73679 73663 92.0682 74.75 0.0047028 0.0108545 2023-10-09T09:31:32.559 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:32.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:32.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:32.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:32.695 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:32.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:32 smithi181 bash[59345]: cluster 2023-10-09T09:31:31.163289+0000 osd.2 (osd.2) 1 : cluster 0 7.8 scrub starts 2023-10-09T09:31:32.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:32 smithi181 bash[59345]: cluster 2023-10-09T09:31:31.721145+0000 mgr.y (mgr.34107) 758 : cluster 0 pgmap v1468: 99 pgs: 1 active+clean+scrubbing, 3 unknown, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 24 MiB/s wr, 377 op/s; 75 KiB/s, 1 objects/s recovering 2023-10-09T09:31:32.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:32 smithi181 bash[59345]: cluster 2023-10-09T09:31:32.026826+0000 osd.4 (osd.4) 533 : cluster 0 7.1 scrub ok 2023-10-09T09:31:32.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:32.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:32.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:32.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:32.854 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:32.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:32.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:32.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:32 smithi163 bash[75425]: cluster 2023-10-09T09:31:31.163289+0000 osd.2 (osd.2) 1 : cluster 0 7.8 scrub starts 2023-10-09T09:31:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:32 smithi163 bash[75425]: cluster 2023-10-09T09:31:31.721145+0000 mgr.y (mgr.34107) 758 : cluster 0 pgmap v1468: 99 pgs: 1 active+clean+scrubbing, 3 unknown, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 24 MiB/s wr, 377 op/s; 75 KiB/s, 1 objects/s recovering 2023-10-09T09:31:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:32 smithi163 bash[75425]: cluster 2023-10-09T09:31:32.026826+0000 osd.4 (osd.4) 533 : cluster 0 7.1 scrub ok 2023-10-09T09:31:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:32 smithi163 bash[78107]: cluster 2023-10-09T09:31:31.163289+0000 osd.2 (osd.2) 1 : cluster 0 7.8 scrub starts 2023-10-09T09:31:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:32 smithi163 bash[78107]: cluster 2023-10-09T09:31:31.721145+0000 mgr.y (mgr.34107) 758 : cluster 0 pgmap v1468: 99 pgs: 1 active+clean+scrubbing, 3 unknown, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 24 MiB/s wr, 377 op/s; 75 KiB/s, 1 objects/s recovering 2023-10-09T09:31:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:32 smithi163 bash[78107]: cluster 2023-10-09T09:31:32.026826+0000 osd.4 (osd.4) 533 : cluster 0 7.1 scrub ok 2023-10-09T09:31:32.987 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:32.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:32.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:33.092 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:33.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:33.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:33.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:33.223 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:33.247 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:33.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:33.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:33.353 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:33.390 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:33.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:33.500 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:33.508 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:31:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:31:33] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.33.4" 2023-10-09T09:31:33.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:33.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:33.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:33.545 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 74895 74879 91.753 76 0.00923807 0.0108934 2023-10-09T09:31:33.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:33.643 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:33 smithi181 bash[59345]: cluster 2023-10-09T09:31:32.003405+0000 osd.7 (osd.7) 498 : cluster 0 7.15 scrub starts 2023-10-09T09:31:33.661 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:33.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:33.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:33.767 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:33.775 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:33 smithi163 bash[78107]: cluster 2023-10-09T09:31:32.003405+0000 osd.7 (osd.7) 498 : cluster 0 7.15 scrub starts 2023-10-09T09:31:33.775 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:33 smithi163 bash[75425]: cluster 2023-10-09T09:31:32.003405+0000 osd.7 (osd.7) 498 : cluster 0 7.15 scrub starts 2023-10-09T09:31:33.796 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:33.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:33.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:33.900 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:33.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:33.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:33.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:34.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:34.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:34.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:34.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:34.205 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:34.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:34.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:34.311 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:34.335 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:34.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:34.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:34.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:34.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:34.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:34.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:34.545 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 76162 76146 91.5112 79.1875 0.00593491 0.0109221 2023-10-09T09:31:34.573 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:34.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:34.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:34.704 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:34.714 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[78107]: cluster 2023-10-09T09:31:32.806834+0000 osd.6 (osd.6) 372 : cluster 0 7.7 scrub starts 2023-10-09T09:31:34.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[78107]: cluster 2023-10-09T09:31:33.052694+0000 osd.7 (osd.7) 499 : cluster 0 7.15 scrub ok 2023-10-09T09:31:34.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[78107]: cluster 2023-10-09T09:31:33.120911+0000 osd.2 (osd.2) 2 : cluster 0 7.18 scrub starts 2023-10-09T09:31:34.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[78107]: cluster 2023-10-09T09:31:33.339286+0000 osd.0 (osd.0) 1 : cluster 0 7.4 scrub starts 2023-10-09T09:31:34.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[78107]: cluster 2023-10-09T09:31:33.721907+0000 mgr.y (mgr.34107) 759 : cluster 0 pgmap v1469: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 17 MiB/s wr, 277 op/s; 102 KiB/s, 1 objects/s recovering 2023-10-09T09:31:34.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[78107]: audit 2023-10-09T09:31:33.723427+0000 mon.a (mon.0) 1568 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T09:31:34.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[78107]: cluster 2023-10-09T09:31:33.829265+0000 osd.2 (osd.2) 3 : cluster 0 7.18 scrub ok 2023-10-09T09:31:34.715 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[75425]: cluster 2023-10-09T09:31:32.806834+0000 osd.6 (osd.6) 372 : cluster 0 7.7 scrub starts 2023-10-09T09:31:34.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[75425]: cluster 2023-10-09T09:31:33.052694+0000 osd.7 (osd.7) 499 : cluster 0 7.15 scrub ok 2023-10-09T09:31:34.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[75425]: cluster 2023-10-09T09:31:33.120911+0000 osd.2 (osd.2) 2 : cluster 0 7.18 scrub starts 2023-10-09T09:31:34.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[75425]: cluster 2023-10-09T09:31:33.339286+0000 osd.0 (osd.0) 1 : cluster 0 7.4 scrub starts 2023-10-09T09:31:34.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[75425]: cluster 2023-10-09T09:31:33.721907+0000 mgr.y (mgr.34107) 759 : cluster 0 pgmap v1469: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 17 MiB/s wr, 277 op/s; 102 KiB/s, 1 objects/s recovering 2023-10-09T09:31:34.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[75425]: audit 2023-10-09T09:31:33.723427+0000 mon.a (mon.0) 1568 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T09:31:34.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:34 smithi163 bash[75425]: cluster 2023-10-09T09:31:33.829265+0000 osd.2 (osd.2) 3 : cluster 0 7.18 scrub ok 2023-10-09T09:31:34.734 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:34.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:34.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:34.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:34.868 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:34.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:34.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:34.946 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:31:34.946 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:31:34.946 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T09:31:34.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:31:34.974 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:35.021 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:35.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:35.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:34 smithi181 bash[59345]: cluster 2023-10-09T09:31:32.806834+0000 osd.6 (osd.6) 372 : cluster 0 7.7 scrub starts 2023-10-09T09:31:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:34 smithi181 bash[59345]: cluster 2023-10-09T09:31:33.052694+0000 osd.7 (osd.7) 499 : cluster 0 7.15 scrub ok 2023-10-09T09:31:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:34 smithi181 bash[59345]: cluster 2023-10-09T09:31:33.120911+0000 osd.2 (osd.2) 2 : cluster 0 7.18 scrub starts 2023-10-09T09:31:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:34 smithi181 bash[59345]: cluster 2023-10-09T09:31:33.339286+0000 osd.0 (osd.0) 1 : cluster 0 7.4 scrub starts 2023-10-09T09:31:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:34 smithi181 bash[59345]: cluster 2023-10-09T09:31:33.721907+0000 mgr.y (mgr.34107) 759 : cluster 0 pgmap v1469: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 17 MiB/s wr, 277 op/s; 102 KiB/s, 1 objects/s recovering 2023-10-09T09:31:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:34 smithi181 bash[59345]: audit 2023-10-09T09:31:33.723427+0000 mon.a (mon.0) 1568 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T09:31:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:34 smithi181 bash[59345]: cluster 2023-10-09T09:31:33.829265+0000 osd.2 (osd.2) 3 : cluster 0 7.18 scrub ok 2023-10-09T09:31:35.125 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:35.152 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:35.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:35.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:35.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:35.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:35.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:35.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:35.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:35.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:35.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:35.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:35.507 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:35.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:35.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:35.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:35.545 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 76945 76929 90.7078 48.9375 0.00274488 0.0109068 2023-10-09T09:31:35.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:35.650 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:35.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:35.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:35.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:35.778 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:35.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[78107]: cluster 2023-10-09T09:31:34.589969+0000 osd.5 (osd.5) 656 : cluster 3 failed to encode map e1107 with expected crc 2023-10-09T09:31:35.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[78107]: cluster 2023-10-09T09:31:34.591767+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e1107 with expected crc 2023-10-09T09:31:35.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[78107]: audit 2023-10-09T09:31:34.596599+0000 mon.a (mon.0) 1569 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T09:31:35.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[78107]: cluster 2023-10-09T09:31:34.596705+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e1107: 8 total, 8 up, 8 in 2023-10-09T09:31:35.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[78107]: cluster 2023-10-09T09:31:35.117305+0000 osd.4 (osd.4) 534 : cluster 0 7.11 scrub starts 2023-10-09T09:31:35.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[78107]: cluster 2023-10-09T09:31:35.566135+0000 mon.a (mon.0) 1571 : cluster 0 osdmap e1108: 8 total, 8 up, 8 in 2023-10-09T09:31:35.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[75425]: cluster 2023-10-09T09:31:34.589969+0000 osd.5 (osd.5) 656 : cluster 3 failed to encode map e1107 with expected crc 2023-10-09T09:31:35.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[75425]: cluster 2023-10-09T09:31:34.591767+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e1107 with expected crc 2023-10-09T09:31:35.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[75425]: audit 2023-10-09T09:31:34.596599+0000 mon.a (mon.0) 1569 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T09:31:35.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[75425]: cluster 2023-10-09T09:31:34.596705+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e1107: 8 total, 8 up, 8 in 2023-10-09T09:31:35.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[75425]: cluster 2023-10-09T09:31:35.117305+0000 osd.4 (osd.4) 534 : cluster 0 7.11 scrub starts 2023-10-09T09:31:35.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:35 smithi163 bash[75425]: cluster 2023-10-09T09:31:35.566135+0000 mon.a (mon.0) 1571 : cluster 0 osdmap e1108: 8 total, 8 up, 8 in 2023-10-09T09:31:35.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:35.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:35.883 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:35.890 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:35 smithi181 bash[59345]: cluster 2023-10-09T09:31:34.589969+0000 osd.5 (osd.5) 656 : cluster 3 failed to encode map e1107 with expected crc 2023-10-09T09:31:35.890 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:35 smithi181 bash[59345]: cluster 2023-10-09T09:31:34.591767+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e1107 with expected crc 2023-10-09T09:31:35.890 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:35 smithi181 bash[59345]: audit 2023-10-09T09:31:34.596599+0000 mon.a (mon.0) 1569 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T09:31:35.890 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:35 smithi181 bash[59345]: cluster 2023-10-09T09:31:34.596705+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e1107: 8 total, 8 up, 8 in 2023-10-09T09:31:35.890 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:35 smithi181 bash[59345]: cluster 2023-10-09T09:31:35.117305+0000 osd.4 (osd.4) 534 : cluster 0 7.11 scrub starts 2023-10-09T09:31:35.890 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:35 smithi181 bash[59345]: cluster 2023-10-09T09:31:35.566135+0000 mon.a (mon.0) 1571 : cluster 0 osdmap e1108: 8 total, 8 up, 8 in 2023-10-09T09:31:35.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:35.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:35.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:36.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:36.038 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:36.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:36.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:36.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:36.163 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:36.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:36.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:36.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:36.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:36.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:36.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:36.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:36.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:36.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:36.502 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:31:36.503 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:31:36.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:36.545 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 76945 76929 89.028 0 - 0.0109068 2023-10-09T09:31:36.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:36.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:36.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:36.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[78107]: cluster 2023-10-09T09:31:35.562173+0000 osd.5 (osd.5) 657 : cluster 3 failed to encode map e1108 with expected crc 2023-10-09T09:31:36.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[78107]: cluster 2023-10-09T09:31:35.569536+0000 osd.4 (osd.4) 535 : cluster 3 failed to encode map e1108 with expected crc 2023-10-09T09:31:36.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[78107]: cluster 2023-10-09T09:31:35.597895+0000 osd.4 (osd.4) 536 : cluster 0 7.11 scrub ok 2023-10-09T09:31:36.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[78107]: cluster 2023-10-09T09:31:35.722347+0000 mgr.y (mgr.34107) 760 : cluster 0 pgmap v1472: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 KiB/s, 1 objects/s recovering 2023-10-09T09:31:36.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[78107]: audit 2023-10-09T09:31:35.723173+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T09:31:36.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[78107]: audit 2023-10-09T09:31:36.576844+0000 mon.a (mon.0) 1573 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T09:31:36.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[78107]: cluster 2023-10-09T09:31:36.576928+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e1109: 8 total, 8 up, 8 in 2023-10-09T09:31:36.596 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[75425]: cluster 2023-10-09T09:31:35.562173+0000 osd.5 (osd.5) 657 : cluster 3 failed to encode map e1108 with expected crc 2023-10-09T09:31:36.596 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[75425]: cluster 2023-10-09T09:31:35.569536+0000 osd.4 (osd.4) 535 : cluster 3 failed to encode map e1108 with expected crc 2023-10-09T09:31:36.596 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[75425]: cluster 2023-10-09T09:31:35.597895+0000 osd.4 (osd.4) 536 : cluster 0 7.11 scrub ok 2023-10-09T09:31:36.596 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[75425]: cluster 2023-10-09T09:31:35.722347+0000 mgr.y (mgr.34107) 760 : cluster 0 pgmap v1472: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 KiB/s, 1 objects/s recovering 2023-10-09T09:31:36.596 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[75425]: audit 2023-10-09T09:31:35.723173+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T09:31:36.596 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[75425]: audit 2023-10-09T09:31:36.576844+0000 mon.a (mon.0) 1573 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T09:31:36.596 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:36 smithi163 bash[75425]: cluster 2023-10-09T09:31:36.576928+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e1109: 8 total, 8 up, 8 in 2023-10-09T09:31:36.658 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:36.682 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:36.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:36.789 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:36.798 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:36 smithi181 bash[59345]: cluster 2023-10-09T09:31:35.562173+0000 osd.5 (osd.5) 657 : cluster 3 failed to encode map e1108 with expected crc 2023-10-09T09:31:36.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:36 smithi181 bash[59345]: cluster 2023-10-09T09:31:35.569536+0000 osd.4 (osd.4) 535 : cluster 3 failed to encode map e1108 with expected crc 2023-10-09T09:31:36.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:36 smithi181 bash[59345]: cluster 2023-10-09T09:31:35.597895+0000 osd.4 (osd.4) 536 : cluster 0 7.11 scrub ok 2023-10-09T09:31:36.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:36 smithi181 bash[59345]: cluster 2023-10-09T09:31:35.722347+0000 mgr.y (mgr.34107) 760 : cluster 0 pgmap v1472: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 KiB/s, 1 objects/s recovering 2023-10-09T09:31:36.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:36 smithi181 bash[59345]: audit 2023-10-09T09:31:35.723173+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T09:31:36.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:36 smithi181 bash[59345]: audit 2023-10-09T09:31:36.576844+0000 mon.a (mon.0) 1573 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T09:31:36.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:36 smithi181 bash[59345]: cluster 2023-10-09T09:31:36.576928+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e1109: 8 total, 8 up, 8 in 2023-10-09T09:31:36.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:36.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:36.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:36.916 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:36.961 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:36.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:36.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:37.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:37.091 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:37.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:37.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:37.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:37.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:37.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:37.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:37.359 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:37.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:37.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:37.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:37.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:37.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:37.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 77468 77452 88.0023 16.3438 0.00554439 0.011263 2023-10-09T09:31:37.595 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:37.616 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:37.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[78107]: cluster 2023-10-09T09:31:36.525043+0000 osd.3 (osd.3) 1 : cluster 0 7.2 scrub starts 2023-10-09T09:31:37.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[78107]: cluster 2023-10-09T09:31:36.573450+0000 osd.5 (osd.5) 658 : cluster 3 failed to encode map e1109 with expected crc 2023-10-09T09:31:37.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[78107]: cluster 2023-10-09T09:31:36.573578+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e1109 with expected crc 2023-10-09T09:31:37.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[78107]: cluster 2023-10-09T09:31:36.581076+0000 osd.7 (osd.7) 500 : cluster 3 failed to encode map e1109 with expected crc 2023-10-09T09:31:37.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[78107]: audit 2023-10-09T09:31:36.949039+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:37.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[78107]: audit 2023-10-09T09:31:36.960587+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:37.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[78107]: cluster 2023-10-09T09:31:37.577490+0000 mon.a (mon.0) 1577 : cluster 0 osdmap e1110: 8 total, 8 up, 8 in 2023-10-09T09:31:37.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[75425]: cluster 2023-10-09T09:31:36.525043+0000 osd.3 (osd.3) 1 : cluster 0 7.2 scrub starts 2023-10-09T09:31:37.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[75425]: cluster 2023-10-09T09:31:36.573450+0000 osd.5 (osd.5) 658 : cluster 3 failed to encode map e1109 with expected crc 2023-10-09T09:31:37.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[75425]: cluster 2023-10-09T09:31:36.573578+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e1109 with expected crc 2023-10-09T09:31:37.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[75425]: cluster 2023-10-09T09:31:36.581076+0000 osd.7 (osd.7) 500 : cluster 3 failed to encode map e1109 with expected crc 2023-10-09T09:31:37.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[75425]: audit 2023-10-09T09:31:36.949039+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:37.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[75425]: audit 2023-10-09T09:31:36.960587+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:37.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:37 smithi163 bash[75425]: cluster 2023-10-09T09:31:37.577490+0000 mon.a (mon.0) 1577 : cluster 0 osdmap e1110: 8 total, 8 up, 8 in 2023-10-09T09:31:37.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:37.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:37.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:37.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:37.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:37.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:37.842 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:37.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:37.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:37.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:37.968 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:37.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:37 smithi181 bash[59345]: cluster 2023-10-09T09:31:36.525043+0000 osd.3 (osd.3) 1 : cluster 0 7.2 scrub starts 2023-10-09T09:31:37.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:37 smithi181 bash[59345]: cluster 2023-10-09T09:31:36.573450+0000 osd.5 (osd.5) 658 : cluster 3 failed to encode map e1109 with expected crc 2023-10-09T09:31:37.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:37 smithi181 bash[59345]: cluster 2023-10-09T09:31:36.573578+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e1109 with expected crc 2023-10-09T09:31:37.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:37 smithi181 bash[59345]: cluster 2023-10-09T09:31:36.581076+0000 osd.7 (osd.7) 500 : cluster 3 failed to encode map e1109 with expected crc 2023-10-09T09:31:37.983 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:37 smithi181 bash[59345]: audit 2023-10-09T09:31:36.949039+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:37.983 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:37 smithi181 bash[59345]: audit 2023-10-09T09:31:36.960587+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:37.983 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:37 smithi181 bash[59345]: cluster 2023-10-09T09:31:37.577490+0000 mon.a (mon.0) 1577 : cluster 0 osdmap e1110: 8 total, 8 up, 8 in 2023-10-09T09:31:38.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:38.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:38.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:38.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:38.126 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:38.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:38.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:38.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:38.249 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:38.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:38.352 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:38.386 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:38.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:38.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:38.489 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:38.517 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:38.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:38.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 77472 77456 86.4354 0.25 0.00543642 0.0112627 2023-10-09T09:31:38.621 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:38.633 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: cluster 2023-10-09T09:31:36.835675+0000 osd.6 (osd.6) 375 : cluster 0 7.7 scrub starts 2023-10-09T09:31:38.633 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: cluster 2023-10-09T09:31:37.173925+0000 osd.3 (osd.3) 2 : cluster 0 7.2 scrub ok 2023-10-09T09:31:38.633 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: cluster 2023-10-09T09:31:37.296163+0000 osd.0 (osd.0) 2 : cluster 0 7.9 scrub starts 2023-10-09T09:31:38.633 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: cluster 2023-10-09T09:31:37.574824+0000 osd.5 (osd.5) 659 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.634 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: cluster 2023-10-09T09:31:37.579355+0000 osd.7 (osd.7) 501 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.634 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: cluster 2023-10-09T09:31:37.584090+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.634 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: cluster 2023-10-09T09:31:37.584183+0000 osd.4 (osd.4) 537 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.634 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: cluster 2023-10-09T09:31:37.722794+0000 mgr.y (mgr.34107) 761 : cluster 0 pgmap v1475: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail 2023-10-09T09:31:38.634 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: audit 2023-10-09T09:31:37.723749+0000 mon.a (mon.0) 1578 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T09:31:38.634 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: audit 2023-10-09T09:31:37.754455+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:38.634 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: audit 2023-10-09T09:31:37.759844+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:38.634 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: audit 2023-10-09T09:31:38.179971+0000 mon.a (mon.0) 1581 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:31:38.634 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: audit 2023-10-09T09:31:38.181351+0000 mon.a (mon.0) 1582 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:31:38.634 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: audit 2023-10-09T09:31:38.188256+0000 mon.a (mon.0) 1583 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:38.635 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: audit 2023-10-09T09:31:38.579174+0000 mon.a (mon.0) 1584 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T09:31:38.635 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:38 smithi181 bash[59345]: cluster 2023-10-09T09:31:38.579264+0000 mon.a (mon.0) 1585 : cluster 0 osdmap e1111: 8 total, 8 up, 8 in 2023-10-09T09:31:38.650 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:38.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:38.754 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:38.823 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:38.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:38.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:38.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:38.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: cluster 2023-10-09T09:31:36.835675+0000 osd.6 (osd.6) 375 : cluster 0 7.7 scrub starts 2023-10-09T09:31:38.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: cluster 2023-10-09T09:31:37.173925+0000 osd.3 (osd.3) 2 : cluster 0 7.2 scrub ok 2023-10-09T09:31:38.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: cluster 2023-10-09T09:31:37.296163+0000 osd.0 (osd.0) 2 : cluster 0 7.9 scrub starts 2023-10-09T09:31:38.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: cluster 2023-10-09T09:31:37.574824+0000 osd.5 (osd.5) 659 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: cluster 2023-10-09T09:31:37.579355+0000 osd.7 (osd.7) 501 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: cluster 2023-10-09T09:31:37.584090+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: cluster 2023-10-09T09:31:37.584183+0000 osd.4 (osd.4) 537 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: cluster 2023-10-09T09:31:37.722794+0000 mgr.y (mgr.34107) 761 : cluster 0 pgmap v1475: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: audit 2023-10-09T09:31:37.723749+0000 mon.a (mon.0) 1578 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: audit 2023-10-09T09:31:37.754455+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: audit 2023-10-09T09:31:37.759844+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: audit 2023-10-09T09:31:38.179971+0000 mon.a (mon.0) 1581 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: audit 2023-10-09T09:31:38.181351+0000 mon.a (mon.0) 1582 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: audit 2023-10-09T09:31:38.188256+0000 mon.a (mon.0) 1583 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: audit 2023-10-09T09:31:38.579174+0000 mon.a (mon.0) 1584 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T09:31:38.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[75425]: cluster 2023-10-09T09:31:38.579264+0000 mon.a (mon.0) 1585 : cluster 0 osdmap e1111: 8 total, 8 up, 8 in 2023-10-09T09:31:38.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: cluster 2023-10-09T09:31:36.835675+0000 osd.6 (osd.6) 375 : cluster 0 7.7 scrub starts 2023-10-09T09:31:38.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: cluster 2023-10-09T09:31:37.173925+0000 osd.3 (osd.3) 2 : cluster 0 7.2 scrub ok 2023-10-09T09:31:38.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: cluster 2023-10-09T09:31:37.296163+0000 osd.0 (osd.0) 2 : cluster 0 7.9 scrub starts 2023-10-09T09:31:38.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: cluster 2023-10-09T09:31:37.574824+0000 osd.5 (osd.5) 659 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: cluster 2023-10-09T09:31:37.579355+0000 osd.7 (osd.7) 501 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: cluster 2023-10-09T09:31:37.584090+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: cluster 2023-10-09T09:31:37.584183+0000 osd.4 (osd.4) 537 : cluster 3 failed to encode map e1110 with expected crc 2023-10-09T09:31:38.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: cluster 2023-10-09T09:31:37.722794+0000 mgr.y (mgr.34107) 761 : cluster 0 pgmap v1475: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail 2023-10-09T09:31:38.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: audit 2023-10-09T09:31:37.723749+0000 mon.a (mon.0) 1578 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T09:31:38.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: audit 2023-10-09T09:31:37.754455+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:38.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: audit 2023-10-09T09:31:37.759844+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:38.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: audit 2023-10-09T09:31:38.179971+0000 mon.a (mon.0) 1581 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:31:38.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: audit 2023-10-09T09:31:38.181351+0000 mon.a (mon.0) 1582 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:31:38.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: audit 2023-10-09T09:31:38.188256+0000 mon.a (mon.0) 1583 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:38.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: audit 2023-10-09T09:31:38.579174+0000 mon.a (mon.0) 1584 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T09:31:38.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:38 smithi163 bash[78107]: cluster 2023-10-09T09:31:38.579264+0000 mon.a (mon.0) 1585 : cluster 0 osdmap e1111: 8 total, 8 up, 8 in 2023-10-09T09:31:38.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:38.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:38.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:39.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:39.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:39.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:39.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:39.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:39.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:39.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:39.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:39.310 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:39.338 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:39.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:39.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:39.355 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:31:39.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:39.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:39.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:39.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:39.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 77835 77819 85.317 22.6875 0.00732168 0.0116887 2023-10-09T09:31:39.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:39.606 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:39.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:39.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:39.729 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:39.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:39.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:39.764 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:39 smithi163 bash[75425]: cluster 2023-10-09T09:31:38.576902+0000 osd.5 (osd.5) 660 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:39.764 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:39 smithi163 bash[75425]: cluster 2023-10-09T09:31:38.577593+0000 osd.4 (osd.4) 538 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:39.764 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:39 smithi163 bash[75425]: cluster 2023-10-09T09:31:38.583387+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:39.765 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:39 smithi163 bash[75425]: cluster 2023-10-09T09:31:39.583274+0000 mon.a (mon.0) 1586 : cluster 0 osdmap e1112: 8 total, 8 up, 8 in 2023-10-09T09:31:39.765 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:39 smithi163 bash[78107]: cluster 2023-10-09T09:31:38.576902+0000 osd.5 (osd.5) 660 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:39.765 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:39 smithi163 bash[78107]: cluster 2023-10-09T09:31:38.577593+0000 osd.4 (osd.4) 538 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:39.765 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:39 smithi163 bash[78107]: cluster 2023-10-09T09:31:38.583387+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:39.765 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:39 smithi163 bash[78107]: cluster 2023-10-09T09:31:39.583274+0000 mon.a (mon.0) 1586 : cluster 0 osdmap e1112: 8 total, 8 up, 8 in 2023-10-09T09:31:39.832 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:39.859 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:39.861 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:39 smithi181 bash[59345]: cluster 2023-10-09T09:31:38.576902+0000 osd.5 (osd.5) 660 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:39.861 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:39 smithi181 bash[59345]: cluster 2023-10-09T09:31:38.577593+0000 osd.4 (osd.4) 538 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:39.862 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:39 smithi181 bash[59345]: cluster 2023-10-09T09:31:38.583387+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:39.862 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:39 smithi181 bash[59345]: cluster 2023-10-09T09:31:39.583274+0000 mon.a (mon.0) 1586 : cluster 0 osdmap e1112: 8 total, 8 up, 8 in 2023-10-09T09:31:39.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:39.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:39.963 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:39.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:39.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:39.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:40.099 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:40.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:40.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:40.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:40.225 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:40.256 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:40.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:40.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:40.276 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:31:40.276 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:31:40.359 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:40.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:40.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:40.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:40.497 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:40.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:40.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:40.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:40.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 78045 78029 84.0723 13.125 0.00391581 0.0116756 2023-10-09T09:31:40.625 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:40.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:40.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:40.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:40.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:40.785 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:40.786 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[78107]: cluster 2023-10-09T09:31:38.827136+0000 osd.6 (osd.6) 378 : cluster 0 7.7 scrub starts 2023-10-09T09:31:40.786 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[78107]: cluster 2023-10-09T09:31:39.060929+0000 osd.7 (osd.7) 502 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:40.786 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[78107]: cluster 2023-10-09T09:31:39.580326+0000 osd.5 (osd.5) 661 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:40.786 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[78107]: cluster 2023-10-09T09:31:39.585885+0000 osd.4 (osd.4) 539 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:40.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[78107]: cluster 2023-10-09T09:31:39.589506+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:40.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[78107]: cluster 2023-10-09T09:31:39.596394+0000 osd.7 (osd.7) 503 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:40.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[78107]: cluster 2023-10-09T09:31:39.723426+0000 mgr.y (mgr.34107) 762 : cluster 0 pgmap v1478: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 96 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 2380/231657 objects degraded (1.027%); 2070/231657 objects misplaced (0.894%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:31:40.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[78107]: audit 2023-10-09T09:31:40.368405+0000 mon.a (mon.0) 1587 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:40.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[78107]: cluster 2023-10-09T09:31:40.586721+0000 mon.a (mon.0) 1588 : cluster 0 osdmap e1113: 8 total, 8 up, 8 in 2023-10-09T09:31:40.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[75425]: cluster 2023-10-09T09:31:38.827136+0000 osd.6 (osd.6) 378 : cluster 0 7.7 scrub starts 2023-10-09T09:31:40.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[75425]: cluster 2023-10-09T09:31:39.060929+0000 osd.7 (osd.7) 502 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:40.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[75425]: cluster 2023-10-09T09:31:39.580326+0000 osd.5 (osd.5) 661 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:40.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[75425]: cluster 2023-10-09T09:31:39.585885+0000 osd.4 (osd.4) 539 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:40.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[75425]: cluster 2023-10-09T09:31:39.589506+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:40.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[75425]: cluster 2023-10-09T09:31:39.596394+0000 osd.7 (osd.7) 503 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:40.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[75425]: cluster 2023-10-09T09:31:39.723426+0000 mgr.y (mgr.34107) 762 : cluster 0 pgmap v1478: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 96 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 2380/231657 objects degraded (1.027%); 2070/231657 objects misplaced (0.894%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:31:40.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[75425]: audit 2023-10-09T09:31:40.368405+0000 mon.a (mon.0) 1587 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:40.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:40 smithi163 bash[75425]: cluster 2023-10-09T09:31:40.586721+0000 mon.a (mon.0) 1588 : cluster 0 osdmap e1113: 8 total, 8 up, 8 in 2023-10-09T09:31:40.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:40.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:40.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:40.916 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:40.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:40.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:41.019 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:41.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:41.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:40 smithi181 bash[59345]: cluster 2023-10-09T09:31:38.827136+0000 osd.6 (osd.6) 378 : cluster 0 7.7 scrub starts 2023-10-09T09:31:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:40 smithi181 bash[59345]: cluster 2023-10-09T09:31:39.060929+0000 osd.7 (osd.7) 502 : cluster 3 failed to encode map e1111 with expected crc 2023-10-09T09:31:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:40 smithi181 bash[59345]: cluster 2023-10-09T09:31:39.580326+0000 osd.5 (osd.5) 661 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:40 smithi181 bash[59345]: cluster 2023-10-09T09:31:39.585885+0000 osd.4 (osd.4) 539 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:40 smithi181 bash[59345]: cluster 2023-10-09T09:31:39.589506+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:40 smithi181 bash[59345]: cluster 2023-10-09T09:31:39.596394+0000 osd.7 (osd.7) 503 : cluster 3 failed to encode map e1112 with expected crc 2023-10-09T09:31:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:40 smithi181 bash[59345]: cluster 2023-10-09T09:31:39.723426+0000 mgr.y (mgr.34107) 762 : cluster 0 pgmap v1478: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 96 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 2380/231657 objects degraded (1.027%); 2070/231657 objects misplaced (0.894%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:31:41.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:40 smithi181 bash[59345]: audit 2023-10-09T09:31:40.368405+0000 mon.a (mon.0) 1587 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:41.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:40 smithi181 bash[59345]: cluster 2023-10-09T09:31:40.586721+0000 mon.a (mon.0) 1588 : cluster 0 osdmap e1113: 8 total, 8 up, 8 in 2023-10-09T09:31:41.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:41.175 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:41.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:41.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:41.279 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:41.315 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:41.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:41.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:41.418 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:41.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:41.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:41.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:41.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 78320 78304 82.9387 17.1875 0.0234357 0.0120515 2023-10-09T09:31:41.548 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:41.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:41.583 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:41.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:41.684 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:41.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:41 smithi181 bash[59345]: cluster 2023-10-09T09:31:40.232406+0000 osd.0 (osd.0) 3 : cluster 0 7.6 scrub starts 2023-10-09T09:31:41.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:41 smithi181 bash[59345]: cluster 2023-10-09T09:31:40.583419+0000 osd.5 (osd.5) 662 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:41 smithi181 bash[59345]: cluster 2023-10-09T09:31:40.603000+0000 mon.a (mon.0) 1589 : cluster 3 Health check failed: Degraded data redundancy: 2380/231657 objects degraded (1.027%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:31:41.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:41 smithi181 bash[59345]: cluster 2023-10-09T09:31:40.614315+0000 osd.7 (osd.7) 504 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:41 smithi181 bash[59345]: cluster 2023-10-09T09:31:40.615681+0000 osd.7 (osd.7) 505 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:41 smithi181 bash[59345]: audit 2023-10-09T09:31:40.779398+0000 mon.a (mon.0) 1590 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:41.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:41 smithi181 bash[59345]: cluster 2023-10-09T09:31:41.305566+0000 osd.4 (osd.4) 540 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.694 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:41 smithi181 bash[59345]: cluster 2023-10-09T09:31:41.308200+0000 osd.4 (osd.4) 541 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.710 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:41.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:41.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:41.818 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:41.843 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:41.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:41.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:41.947 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[75425]: cluster 2023-10-09T09:31:40.232406+0000 osd.0 (osd.0) 3 : cluster 0 7.6 scrub starts 2023-10-09T09:31:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[75425]: cluster 2023-10-09T09:31:40.583419+0000 osd.5 (osd.5) 662 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[75425]: cluster 2023-10-09T09:31:40.603000+0000 mon.a (mon.0) 1589 : cluster 3 Health check failed: Degraded data redundancy: 2380/231657 objects degraded (1.027%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:31:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[75425]: cluster 2023-10-09T09:31:40.614315+0000 osd.7 (osd.7) 504 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[75425]: cluster 2023-10-09T09:31:40.615681+0000 osd.7 (osd.7) 505 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[75425]: audit 2023-10-09T09:31:40.779398+0000 mon.a (mon.0) 1590 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[75425]: cluster 2023-10-09T09:31:41.305566+0000 osd.4 (osd.4) 540 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[75425]: cluster 2023-10-09T09:31:41.308200+0000 osd.4 (osd.4) 541 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[78107]: cluster 2023-10-09T09:31:40.232406+0000 osd.0 (osd.0) 3 : cluster 0 7.6 scrub starts 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[78107]: cluster 2023-10-09T09:31:40.583419+0000 osd.5 (osd.5) 662 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[78107]: cluster 2023-10-09T09:31:40.603000+0000 mon.a (mon.0) 1589 : cluster 3 Health check failed: Degraded data redundancy: 2380/231657 objects degraded (1.027%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[78107]: cluster 2023-10-09T09:31:40.614315+0000 osd.7 (osd.7) 504 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[78107]: cluster 2023-10-09T09:31:40.615681+0000 osd.7 (osd.7) 505 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[78107]: audit 2023-10-09T09:31:40.779398+0000 mon.a (mon.0) 1590 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:41.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[78107]: cluster 2023-10-09T09:31:41.305566+0000 osd.4 (osd.4) 540 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:41 smithi163 bash[78107]: cluster 2023-10-09T09:31:41.308200+0000 osd.4 (osd.4) 541 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:41.973 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:41.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:42.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:42.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:42.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:42.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:42.212 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:42.233 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:42.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:42.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:42.367 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:42.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:42.473 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:42.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:42.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:42.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:42.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:31:42.547392+0000 min lat: 0.00155566 max lat: 2.40866 avg lat: 0.0120881 2023-10-09T09:31:42.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:31:42.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 79403 79387 82.6844 67.6875 0.00892991 0.0120881 2023-10-09T09:31:42.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:42.638 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:42.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:42.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:42 smithi163 bash[78107]: cluster 2023-10-09T09:31:40.849156+0000 osd.6 (osd.6) 380 : cluster 0 7.7 scrub starts 2023-10-09T09:31:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:42 smithi163 bash[78107]: cluster 2023-10-09T09:31:40.849855+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:42 smithi163 bash[78107]: cluster 2023-10-09T09:31:40.852779+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:42 smithi163 bash[78107]: cluster 2023-10-09T09:31:41.207432+0000 osd.0 (osd.0) 4 : cluster 0 7.16 scrub starts 2023-10-09T09:31:42.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:42 smithi163 bash[78107]: cluster 2023-10-09T09:31:41.724089+0000 mgr.y (mgr.34107) 763 : cluster 0 pgmap v1480: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 96 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 18 MiB/s wr, 295 op/s; 2380/233256 objects degraded (1.020%); 2070/233256 objects misplaced (0.887%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:31:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:42 smithi163 bash[75425]: cluster 2023-10-09T09:31:40.849156+0000 osd.6 (osd.6) 380 : cluster 0 7.7 scrub starts 2023-10-09T09:31:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:42 smithi163 bash[75425]: cluster 2023-10-09T09:31:40.849855+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:42 smithi163 bash[75425]: cluster 2023-10-09T09:31:40.852779+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:42 smithi163 bash[75425]: cluster 2023-10-09T09:31:41.207432+0000 osd.0 (osd.0) 4 : cluster 0 7.16 scrub starts 2023-10-09T09:31:42.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:42 smithi163 bash[75425]: cluster 2023-10-09T09:31:41.724089+0000 mgr.y (mgr.34107) 763 : cluster 0 pgmap v1480: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 96 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 18 MiB/s wr, 295 op/s; 2380/233256 objects degraded (1.020%); 2070/233256 objects misplaced (0.887%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:31:42.742 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:42.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:42.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:42.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:42.876 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:42.886 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:42 smithi181 bash[59345]: cluster 2023-10-09T09:31:40.849156+0000 osd.6 (osd.6) 380 : cluster 0 7.7 scrub starts 2023-10-09T09:31:42.886 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:42 smithi181 bash[59345]: cluster 2023-10-09T09:31:40.849855+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:42.886 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:42 smithi181 bash[59345]: cluster 2023-10-09T09:31:40.852779+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e1113 with expected crc 2023-10-09T09:31:42.886 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:42 smithi181 bash[59345]: cluster 2023-10-09T09:31:41.207432+0000 osd.0 (osd.0) 4 : cluster 0 7.16 scrub starts 2023-10-09T09:31:42.886 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:42 smithi181 bash[59345]: cluster 2023-10-09T09:31:41.724089+0000 mgr.y (mgr.34107) 763 : cluster 0 pgmap v1480: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 96 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 18 MiB/s wr, 295 op/s; 2380/233256 objects degraded (1.020%); 2070/233256 objects misplaced (0.887%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:31:42.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:42.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:42.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:43.008 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:43.035 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:43.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:43.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:43.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:43.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:43.175 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:43.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:43.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:43.304 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:43.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:43.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:43.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:43.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:43.519 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:31:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:31:43] "GET /metrics HTTP/1.1" 200 36347 "" "Prometheus/2.33.4" 2023-10-09T09:31:43.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:43.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 80552 80536 82.506 71.8125 0.00323594 0.0121145 2023-10-09T09:31:43.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:43.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:43.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:43.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:43.701 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:43.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:43.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:43.804 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:43.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:43.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:43.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:43.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:43.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:43.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:43.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:43 smithi181 bash[59345]: cluster 2023-10-09T09:31:42.932831+0000 osd.6 (osd.6) 383 : cluster 0 7.7 scrub starts 2023-10-09T09:31:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:43 smithi181 bash[59345]: cluster 2023-10-09T09:31:43.725937+0000 mgr.y (mgr.34107) 764 : cluster 0 pgmap v1481: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 21 MiB/s wr, 330 op/s; 4720/235530 objects degraded (2.004%); 4410/235530 objects misplaced (1.872%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T09:31:44.075 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:44.098 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:44.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:44.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:44.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:44.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:43 smithi163 bash[75425]: cluster 2023-10-09T09:31:42.932831+0000 osd.6 (osd.6) 383 : cluster 0 7.7 scrub starts 2023-10-09T09:31:44.211 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:43 smithi163 bash[75425]: cluster 2023-10-09T09:31:43.725937+0000 mgr.y (mgr.34107) 764 : cluster 0 pgmap v1481: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 21 MiB/s wr, 330 op/s; 4720/235530 objects degraded (2.004%); 4410/235530 objects misplaced (1.872%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T09:31:44.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:43 smithi163 bash[78107]: cluster 2023-10-09T09:31:42.932831+0000 osd.6 (osd.6) 383 : cluster 0 7.7 scrub starts 2023-10-09T09:31:44.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:43 smithi163 bash[78107]: cluster 2023-10-09T09:31:43.725937+0000 mgr.y (mgr.34107) 764 : cluster 0 pgmap v1481: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 21 MiB/s wr, 330 op/s; 4720/235530 objects degraded (2.004%); 4410/235530 objects misplaced (1.872%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T09:31:44.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:44.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:44.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:44.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:44.363 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:44.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:44.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:44.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:44.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:44.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:44.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:44.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 81518 81502 82.149 60.375 0.00522435 0.0121673 2023-10-09T09:31:44.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:44.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T09:31:44.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:44.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:44.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:44.725 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:44.749 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:44.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:44.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:44.852 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:44.875 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:44.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:44.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:45.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:45.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:45.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:45.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:45.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:45.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:45.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:45.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:45.270 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:45.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:45.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:45.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:45.403 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:45.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:45.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:45.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:45.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:45.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:45.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:45.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 82555 82539 81.8737 64.8125 0.00374279 0.0122079 2023-10-09T09:31:45.636 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:45.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:45.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:45.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:45.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:45.794 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:45.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:45.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:45.904 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:45.927 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:45.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:45.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:46.030 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:46.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:46.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:46.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:46.164 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:46.190 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:46.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:46.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:46.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:46.325 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:46.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:46.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:46.429 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:46.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:46.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:46.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 83692 83676 81.7047 71.0625 0.0138519 0.0122338 2023-10-09T09:31:46.556 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:46.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:46.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:46.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:46.691 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:46.703 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:46 smithi181 bash[59345]: cluster 2023-10-09T09:31:44.881864+0000 osd.6 (osd.6) 384 : cluster 0 7.7 scrub starts 2023-10-09T09:31:46.703 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:46 smithi181 bash[59345]: cluster 2023-10-09T09:31:45.726726+0000 mgr.y (mgr.34107) 765 : cluster 0 pgmap v1482: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 27 MiB/s wr, 434 op/s; 4720/239403 objects degraded (1.972%); 4005/239403 objects misplaced (1.673%); 6.1 MiB/s, 96 objects/s recovering 2023-10-09T09:31:46.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:46.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:46.833 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:46.842 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:46 smithi163 bash[75425]: cluster 2023-10-09T09:31:44.881864+0000 osd.6 (osd.6) 384 : cluster 0 7.7 scrub starts 2023-10-09T09:31:46.842 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:46 smithi163 bash[75425]: cluster 2023-10-09T09:31:45.726726+0000 mgr.y (mgr.34107) 765 : cluster 0 pgmap v1482: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 27 MiB/s wr, 434 op/s; 4720/239403 objects degraded (1.972%); 4005/239403 objects misplaced (1.673%); 6.1 MiB/s, 96 objects/s recovering 2023-10-09T09:31:46.842 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:46 smithi163 bash[78107]: cluster 2023-10-09T09:31:44.881864+0000 osd.6 (osd.6) 384 : cluster 0 7.7 scrub starts 2023-10-09T09:31:46.843 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:46 smithi163 bash[78107]: cluster 2023-10-09T09:31:45.726726+0000 mgr.y (mgr.34107) 765 : cluster 0 pgmap v1482: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 27 MiB/s wr, 434 op/s; 4720/239403 objects degraded (1.972%); 4005/239403 objects misplaced (1.673%); 6.1 MiB/s, 96 objects/s recovering 2023-10-09T09:31:46.866 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:46.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:46.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:46.975 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:46.998 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:47.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:47.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:47.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:47.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:47.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:47.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:47.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:47.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:47.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:47.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:47.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:47.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:47.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:47.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:47.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:47.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:47.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:47.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 84683 84667 81.4005 61.9375 0.00437075 0.0122645 2023-10-09T09:31:47.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:47.656 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:47.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:47.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:47.749 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:47 smithi163 bash[75425]: cluster 2023-10-09T09:31:45.903228+0000 osd.6 (osd.6) 385 : cluster 0 7.7 scrub starts 2023-10-09T09:31:47.758 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:47.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:47 smithi181 bash[59345]: cluster 2023-10-09T09:31:45.903228+0000 osd.6 (osd.6) 385 : cluster 0 7.7 scrub starts 2023-10-09T09:31:47.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:47 smithi181 bash[59345]: cluster 2023-10-09T09:31:46.562305+0000 mon.a (mon.0) 1591 : cluster 3 Health check update: Degraded data redundancy: 4720/239403 objects degraded (1.972%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:31:47.786 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:47.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:47.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:47.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:47.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:47.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:48.034 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:48.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:47 smithi163 bash[75425]: cluster 2023-10-09T09:31:46.562305+0000 mon.a (mon.0) 1591 : cluster 3 Health check update: Degraded data redundancy: 4720/239403 objects degraded (1.972%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:31:48.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:47 smithi163 bash[78107]: cluster 2023-10-09T09:31:45.903228+0000 osd.6 (osd.6) 385 : cluster 0 7.7 scrub starts 2023-10-09T09:31:48.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:47 smithi163 bash[78107]: cluster 2023-10-09T09:31:46.562305+0000 mon.a (mon.0) 1591 : cluster 3 Health check update: Degraded data redundancy: 4720/239403 objects degraded (1.972%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:31:48.069 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:48.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:48.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:48.174 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:48.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:48.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:48.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:48.311 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:48.332 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:48.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:48.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:48.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:48.455 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:48.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:48.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:48.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 85690 85674 81.1206 62.9375 0.0143945 0.0123222 2023-10-09T09:31:48.558 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:48.586 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:48.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:48.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:48.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:48.734 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:48.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:48.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:48.842 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:48.852 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:48 smithi181 bash[59345]: cluster 2023-10-09T09:31:47.727484+0000 mgr.y (mgr.34107) 766 : cluster 0 pgmap v1483: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 4720/248502 objects degraded (1.899%); 4005/248502 objects misplaced (1.612%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:31:48.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:48.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:48.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:48.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:48 smithi163 bash[75425]: cluster 2023-10-09T09:31:47.727484+0000 mgr.y (mgr.34107) 766 : cluster 0 pgmap v1483: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 4720/248502 objects degraded (1.899%); 4005/248502 objects misplaced (1.612%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:31:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:48 smithi163 bash[78107]: cluster 2023-10-09T09:31:47.727484+0000 mgr.y (mgr.34107) 766 : cluster 0 pgmap v1483: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 4720/248502 objects degraded (1.899%); 4005/248502 objects misplaced (1.612%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:31:48.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:49.014 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:49.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:49.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:49.117 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:49.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:49.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:49.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:49.273 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:49.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:49.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:49.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:49.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:49.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:49.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:49.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:49.540 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:49.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:49.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:49.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 86666 86650 80.8202 61 0.0178169 0.0123676 2023-10-09T09:31:49.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:49.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:49.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:49.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:49.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:49.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:49.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:49.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:49.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:49.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:50.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:50.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:49 smithi181 bash[59345]: cluster 2023-10-09T09:31:49.728188+0000 mgr.y (mgr.34107) 767 : cluster 0 pgmap v1484: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 4720/253449 objects degraded (1.862%); 3756/253449 objects misplaced (1.482%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:31:50.092 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:50.101 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:49 smithi163 bash[78107]: cluster 2023-10-09T09:31:49.728188+0000 mgr.y (mgr.34107) 767 : cluster 0 pgmap v1484: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 4720/253449 objects degraded (1.862%); 3756/253449 objects misplaced (1.482%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:31:50.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:50.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:50.102 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:49 smithi163 bash[75425]: cluster 2023-10-09T09:31:49.728188+0000 mgr.y (mgr.34107) 767 : cluster 0 pgmap v1484: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 4720/253449 objects degraded (1.862%); 3756/253449 objects misplaced (1.482%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T09:31:50.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:50.230 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:50.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:50.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:50.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:50.362 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:50.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:50.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:50.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:50.512 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:50.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:50.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:50.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 87721 87705 80.6013 65.9375 0.0294594 0.0124017 2023-10-09T09:31:50.617 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:50.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:50.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:50.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:50.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:50.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:50.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:50.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:50.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:50.907 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:50.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:50.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:51.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:51.034 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:51.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:51.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:51.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:51.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:51.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:51.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:51.267 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:51.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:51.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:51.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:51.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:51.437 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:51.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:51.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:51.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:51.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 88724 88708 80.3416 62.6875 0.00395333 0.0124379 2023-10-09T09:31:51.573 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:51.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:51.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:51.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:51.703 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:51.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:51.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:51.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:51.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:51 smithi181 bash[59345]: audit 2023-10-09T09:31:50.794977+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:51 smithi181 bash[59345]: cluster 2023-10-09T09:31:50.902708+0000 osd.6 (osd.6) 386 : cluster 0 7.7 scrub starts 2023-10-09T09:31:51.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:51 smithi181 bash[59345]: cluster 2023-10-09T09:31:51.728891+0000 mgr.y (mgr.34107) 768 : cluster 0 pgmap v1485: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 4720/260271 objects degraded (1.813%); 3756/260271 objects misplaced (1.443%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:31:51.836 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:51.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:51.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:51.942 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:51.977 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:51.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:52.080 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:52.089 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:51 smithi163 bash[75425]: audit 2023-10-09T09:31:50.794977+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:52.089 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:51 smithi163 bash[75425]: cluster 2023-10-09T09:31:50.902708+0000 osd.6 (osd.6) 386 : cluster 0 7.7 scrub starts 2023-10-09T09:31:52.089 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:51 smithi163 bash[75425]: cluster 2023-10-09T09:31:51.728891+0000 mgr.y (mgr.34107) 768 : cluster 0 pgmap v1485: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 4720/260271 objects degraded (1.813%); 3756/260271 objects misplaced (1.443%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:31:52.090 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:51 smithi163 bash[78107]: audit 2023-10-09T09:31:50.794977+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:52.090 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:51 smithi163 bash[78107]: cluster 2023-10-09T09:31:50.902708+0000 osd.6 (osd.6) 386 : cluster 0 7.7 scrub starts 2023-10-09T09:31:52.090 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:51 smithi163 bash[78107]: cluster 2023-10-09T09:31:51.728891+0000 mgr.y (mgr.34107) 768 : cluster 0 pgmap v1485: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 4720/260271 objects degraded (1.813%); 3756/260271 objects misplaced (1.443%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:31:52.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:52.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:52.219 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:52.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:52.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:52.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:52.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:52.370 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:52.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:52.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:52.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:52.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:52.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:52.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:52.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 89465 89449 79.8554 46.3125 0.0127629 0.012517 2023-10-09T09:31:52.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:52.647 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:52.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:52.684 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:31:52.753 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:52.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:52.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:52.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:52.892 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:52.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:52.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:52.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:53.019 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:53.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:52 smithi181 bash[59345]: cluster 2023-10-09T09:31:51.793491+0000 mon.a (mon.0) 1593 : cluster 3 Health check update: Degraded data redundancy: 4720/260271 objects degraded (1.813%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:31:53.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:53.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:53.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:53.151 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:53.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:52 smithi163 bash[75425]: cluster 2023-10-09T09:31:51.793491+0000 mon.a (mon.0) 1593 : cluster 3 Health check update: Degraded data redundancy: 4720/260271 objects degraded (1.813%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:31:53.161 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:52 smithi163 bash[78107]: cluster 2023-10-09T09:31:51.793491+0000 mon.a (mon.0) 1593 : cluster 3 Health check update: Degraded data redundancy: 4720/260271 objects degraded (1.813%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:31:53.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:53.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:53.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:53.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:53.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:53.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:53.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:53.432 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:31:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:31:53] "GET /metrics HTTP/1.1" 200 36347 "" "Prometheus/2.33.4" 2023-10-09T09:31:53.449 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:53.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:53.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:53.548 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 90484 90468 79.6276 63.6875 0.00430795 0.0125521 2023-10-09T09:31:53.555 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:53.579 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:53.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:53.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:53.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:53.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:53.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:53 smithi181 bash[59345]: cluster 2023-10-09T09:31:52.866426+0000 osd.6 (osd.6) 387 : cluster 0 7.7 scrub starts 2023-10-09T09:31:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:53 smithi181 bash[59345]: cluster 2023-10-09T09:31:53.729362+0000 mgr.y (mgr.34107) 769 : cluster 0 pgmap v1486: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 4720/265230 objects degraded (1.780%); 3756/265230 objects misplaced (1.416%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:31:53.820 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:53.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:53.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:53.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:53.951 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:53.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:53 smithi163 bash[78107]: cluster 2023-10-09T09:31:52.866426+0000 osd.6 (osd.6) 387 : cluster 0 7.7 scrub starts 2023-10-09T09:31:53.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:53 smithi163 bash[78107]: cluster 2023-10-09T09:31:53.729362+0000 mgr.y (mgr.34107) 769 : cluster 0 pgmap v1486: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 4720/265230 objects degraded (1.780%); 3756/265230 objects misplaced (1.416%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:31:53.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:53 smithi163 bash[75425]: cluster 2023-10-09T09:31:52.866426+0000 osd.6 (osd.6) 387 : cluster 0 7.7 scrub starts 2023-10-09T09:31:53.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:53 smithi163 bash[75425]: cluster 2023-10-09T09:31:53.729362+0000 mgr.y (mgr.34107) 769 : cluster 0 pgmap v1486: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 4720/265230 objects degraded (1.780%); 3756/265230 objects misplaced (1.416%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:31:53.979 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:53.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:53.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:54.084 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:54.122 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:54.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:54.152 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:31:54.187 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:31:54.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:54.242 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:31:54.259 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:54.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:54.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:54.303 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:31:54.335 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:31:54.362 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:54.364 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:31:54.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:54.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:54.397 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:31:54.428 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:31:54.429 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:31:54.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:31:54.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:31:54.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:31:54.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:31:54.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:31:54.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:31:54.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:31:54.497 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:54.519 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:54.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:54.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:54.548 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 91322 91306 79.249 52.375 0.105661 0.0126104 2023-10-09T09:31:54.622 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:54.648 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:54.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:54.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:54.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:54.787 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:54.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:54.891 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:54.901 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:54 smithi181 bash[59345]: cluster 2023-10-09T09:31:53.836440+0000 osd.6 (osd.6) 388 : cluster 0 7.7 scrub starts 2023-10-09T09:31:54.901 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:54 smithi181 bash[59345]: audit 2023-10-09T09:31:54.038616+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.163:0/2429819758' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:31:54.917 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:54.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:54.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:55.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:55.030 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:54 smithi163 bash[75425]: cluster 2023-10-09T09:31:53.836440+0000 osd.6 (osd.6) 388 : cluster 0 7.7 scrub starts 2023-10-09T09:31:55.030 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:54 smithi163 bash[75425]: audit 2023-10-09T09:31:54.038616+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.163:0/2429819758' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:31:55.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:54 smithi163 bash[78107]: cluster 2023-10-09T09:31:53.836440+0000 osd.6 (osd.6) 388 : cluster 0 7.7 scrub starts 2023-10-09T09:31:55.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:54 smithi163 bash[78107]: audit 2023-10-09T09:31:54.038616+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.163:0/2429819758' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:31:55.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:31:55.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:55.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:31:55.152 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:55.178 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:55.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:55.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:55.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:55.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:55.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:55.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:55.411 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:55.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:55.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:55.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:55.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:55.548 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 92353 92337 79.0461 64.4375 0.0196483 0.0126453 2023-10-09T09:31:55.580 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:55.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:55.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:55.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:55.727 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:55.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:55.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:55.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:55.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:55.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:55.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:55.957 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:31:55.960 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:31:55.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:56.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:56.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:56.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:56.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:56.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:56.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:56.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:56.273 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:56.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:56.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:56.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:56.382 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:56 smithi163 bash[78107]: cluster 2023-10-09T09:31:54.830864+0000 osd.6 (osd.6) 389 : cluster 0 7.7 scrub starts 2023-10-09T09:31:56.382 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:56 smithi163 bash[78107]: audit 2023-10-09T09:31:55.374758+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:56.382 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:56 smithi163 bash[78107]: audit 2023-10-09T09:31:55.378249+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:56.382 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:56 smithi163 bash[78107]: cluster 2023-10-09T09:31:55.729983+0000 mgr.y (mgr.34107) 770 : cluster 0 pgmap v1487: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 4720/270075 objects degraded (1.748%); 3497/270075 objects misplaced (1.295%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:31:56.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:56 smithi163 bash[75425]: cluster 2023-10-09T09:31:54.830864+0000 osd.6 (osd.6) 389 : cluster 0 7.7 scrub starts 2023-10-09T09:31:56.383 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:56 smithi163 bash[75425]: audit 2023-10-09T09:31:55.374758+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:56.383 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:56 smithi163 bash[75425]: audit 2023-10-09T09:31:55.378249+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:56.383 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:56 smithi163 bash[75425]: cluster 2023-10-09T09:31:55.729983+0000 mgr.y (mgr.34107) 770 : cluster 0 pgmap v1487: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 4720/270075 objects degraded (1.748%); 3497/270075 objects misplaced (1.295%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:31:56.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:56.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:56.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:56.495 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:56.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:56.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:56.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 93185 93169 78.6805 52 0.00360391 0.0127033 2023-10-09T09:31:56.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:56.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:56.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:56.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:56.756 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:56.767 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:56 smithi181 bash[59345]: cluster 2023-10-09T09:31:54.830864+0000 osd.6 (osd.6) 389 : cluster 0 7.7 scrub starts 2023-10-09T09:31:56.767 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:56 smithi181 bash[59345]: audit 2023-10-09T09:31:55.374758+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:31:56.767 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:56 smithi181 bash[59345]: audit 2023-10-09T09:31:55.378249+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:31:56.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:56 smithi181 bash[59345]: cluster 2023-10-09T09:31:55.729983+0000 mgr.y (mgr.34107) 770 : cluster 0 pgmap v1487: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 4720/270075 objects degraded (1.748%); 3497/270075 objects misplaced (1.295%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:31:56.782 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:56.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:56.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:56.888 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:56.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:56.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:56.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:57.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:57.039 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:57.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:57.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:57.141 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:57.166 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:57.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:57.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:57.270 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:57.296 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:57.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:57.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:57.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:57.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:57.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:57.537 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:57.548 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 94176 94160 78.4569 61.9375 0.0158377 0.0127389 2023-10-09T09:31:57.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:57.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:57.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:57.605 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:31:57.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:57.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:57.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:57 smithi181 bash[59345]: cluster 2023-10-09T09:31:55.852408+0000 osd.6 (osd.6) 390 : cluster 0 7.7 scrub starts 2023-10-09T09:31:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:57 smithi181 bash[59345]: audit 2023-10-09T09:31:55.932644+0000 mgr.y (mgr.34107) 771 : audit 0 from='client.54700 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:31:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:57 smithi181 bash[59345]: audit 2023-10-09T09:31:57.545555+0000 mon.b (mon.1) 11 : audit 1 from='client.? 172.21.15.163:0/1089468891' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:31:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:57 smithi181 bash[59345]: audit 2023-10-09T09:31:57.548336+0000 mon.a (mon.0) 1596 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:31:57.819 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:57.850 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:57.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:57.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:31:57.955 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:57.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:57 smithi163 bash[75425]: cluster 2023-10-09T09:31:55.852408+0000 osd.6 (osd.6) 390 : cluster 0 7.7 scrub starts 2023-10-09T09:31:57.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:57 smithi163 bash[75425]: audit 2023-10-09T09:31:55.932644+0000 mgr.y (mgr.34107) 771 : audit 0 from='client.54700 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:31:57.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:57 smithi163 bash[75425]: audit 2023-10-09T09:31:57.545555+0000 mon.b (mon.1) 11 : audit 1 from='client.? 172.21.15.163:0/1089468891' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:31:57.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:57 smithi163 bash[75425]: audit 2023-10-09T09:31:57.548336+0000 mon.a (mon.0) 1596 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:31:57.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:57 smithi163 bash[78107]: cluster 2023-10-09T09:31:55.852408+0000 osd.6 (osd.6) 390 : cluster 0 7.7 scrub starts 2023-10-09T09:31:57.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:57 smithi163 bash[78107]: audit 2023-10-09T09:31:55.932644+0000 mgr.y (mgr.34107) 771 : audit 0 from='client.54700 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:31:57.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:57 smithi163 bash[78107]: audit 2023-10-09T09:31:57.545555+0000 mon.b (mon.1) 11 : audit 1 from='client.? 172.21.15.163:0/1089468891' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:31:57.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:57 smithi163 bash[78107]: audit 2023-10-09T09:31:57.548336+0000 mon.a (mon.0) 1596 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:31:57.987 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:57.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:57.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:58.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:58.113 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:58.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:58.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:58.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:58.240 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:58.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:58.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:58.343 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:58.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:58.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:58.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:58.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:58.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:58.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:58.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:58.548 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 95148 95132 78.2239 60.75 0.0255173 0.0127783 2023-10-09T09:31:58.606 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:58.633 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:58.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:58.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:58.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:58.743 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:58 smithi181 bash[59345]: cluster 2023-10-09T09:31:56.825506+0000 osd.6 (osd.6) 391 : cluster 0 7.7 scrub starts 2023-10-09T09:31:58.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:58 smithi181 bash[59345]: cluster 2023-10-09T09:31:57.568618+0000 mon.a (mon.0) 1597 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:31:58.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:58 smithi181 bash[59345]: cluster 2023-10-09T09:31:57.588029+0000 osd.5 (osd.5) 663 : cluster 3 failed to encode map e1114 with expected crc 2023-10-09T09:31:58.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:58 smithi181 bash[59345]: cluster 2023-10-09T09:31:57.592154+0000 osd.7 (osd.7) 506 : cluster 3 failed to encode map e1114 with expected crc 2023-10-09T09:31:58.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:58 smithi181 bash[59345]: audit 2023-10-09T09:31:57.597326+0000 mon.a (mon.0) 1598 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:31:58.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:58 smithi181 bash[59345]: cluster 2023-10-09T09:31:57.597778+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e1114: 8 total, 8 up, 8 in 2023-10-09T09:31:58.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:58 smithi181 bash[59345]: cluster 2023-10-09T09:31:57.730519+0000 mgr.y (mgr.34107) 772 : cluster 0 pgmap v1489: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s; 4720/277773 objects degraded (1.699%); 3497/277773 objects misplaced (1.259%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T09:31:58.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:31:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:58.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:31:58.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:58.874 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[78107]: cluster 2023-10-09T09:31:56.825506+0000 osd.6 (osd.6) 391 : cluster 0 7.7 scrub starts 2023-10-09T09:31:58.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[78107]: cluster 2023-10-09T09:31:57.568618+0000 mon.a (mon.0) 1597 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:31:58.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[78107]: cluster 2023-10-09T09:31:57.588029+0000 osd.5 (osd.5) 663 : cluster 3 failed to encode map e1114 with expected crc 2023-10-09T09:31:58.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[78107]: cluster 2023-10-09T09:31:57.592154+0000 osd.7 (osd.7) 506 : cluster 3 failed to encode map e1114 with expected crc 2023-10-09T09:31:58.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[78107]: audit 2023-10-09T09:31:57.597326+0000 mon.a (mon.0) 1598 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:31:58.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[78107]: cluster 2023-10-09T09:31:57.597778+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e1114: 8 total, 8 up, 8 in 2023-10-09T09:31:58.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[78107]: cluster 2023-10-09T09:31:57.730519+0000 mgr.y (mgr.34107) 772 : cluster 0 pgmap v1489: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s; 4720/277773 objects degraded (1.699%); 3497/277773 objects misplaced (1.259%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T09:31:58.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[75425]: cluster 2023-10-09T09:31:56.825506+0000 osd.6 (osd.6) 391 : cluster 0 7.7 scrub starts 2023-10-09T09:31:58.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[75425]: cluster 2023-10-09T09:31:57.568618+0000 mon.a (mon.0) 1597 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:31:58.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[75425]: cluster 2023-10-09T09:31:57.588029+0000 osd.5 (osd.5) 663 : cluster 3 failed to encode map e1114 with expected crc 2023-10-09T09:31:58.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[75425]: cluster 2023-10-09T09:31:57.592154+0000 osd.7 (osd.7) 506 : cluster 3 failed to encode map e1114 with expected crc 2023-10-09T09:31:58.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[75425]: audit 2023-10-09T09:31:57.597326+0000 mon.a (mon.0) 1598 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:31:58.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[75425]: cluster 2023-10-09T09:31:57.597778+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e1114: 8 total, 8 up, 8 in 2023-10-09T09:31:58.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:58 smithi163 bash[75425]: cluster 2023-10-09T09:31:57.730519+0000 mgr.y (mgr.34107) 772 : cluster 0 pgmap v1489: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s; 4720/277773 objects degraded (1.699%); 3497/277773 objects misplaced (1.259%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T09:31:58.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:58.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:58.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:58.926 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1487,"stamp":"2023-10-09T09:31:55.729531+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5898868640,"num_objects":90025,"num_object_clones":0,"num_object_copies":270075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4720,"num_objects_misplaced":3497,"num_objects_unfound":0,"num_objects_dirty":90025,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":76537,"num_write_kb":4884141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1210,"num_bytes_recovered":79298560,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133251,"ondisk_log_size":133251,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19419468,"kb_used_data":17294668,"kb_used_omap":416,"kb_used_meta":2124319,"kb_avail":730541748,"statfs":{"total":767960285184,"available":748074749952,"internally_reserved":0,"allocated":17709740032,"data_stored":17704523461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":2175303089},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":754647040,"num_objects":11515,"num_object_clones":0,"num_object_copies":34545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-913,"num_objects_unfound":0,"num_objects_dirty":11515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11515,"num_write_kb":736960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":913,"num_bytes_recovered":59834368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004024"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1080,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.590849+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:31:39.590849+0000","last_peered":"2023-10-09T09:31:39.590849+0000","last_clean":"2023-10-09T09:31:39.590849+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:31:39.590849+0000","last_undegraded":"2023-10-09T09:31:39.590849+0000","last_fullsized":"2023-10-09T09:31:39.590849+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1046,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.608845+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:31:39.608845+0000","last_peered":"2023-10-09T09:31:39.608845+0000","last_clean":"2023-10-09T09:31:39.608845+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:31:39.608845+0000","last_undegraded":"2023-10-09T09:31:39.608845+0000","last_fullsized":"2023-10-09T09:31:39.608845+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1050,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.596895+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:31:39.596895+0000","last_peered":"2023-10-09T09:31:39.596895+0000","last_clean":"2023-10-09T09:31:39.596895+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:31:39.596895+0000","last_undegraded":"2023-10-09T09:31:39.596895+0000","last_fullsized":"2023-10-09T09:31:39.596895+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1041,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.595797+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:31:39.595797+0000","last_peered":"2023-10-09T09:31:39.595797+0000","last_clean":"2023-10-09T09:31:39.595797+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:31:39.595797+0000","last_undegraded":"2023-10-09T09:31:39.595797+0000","last_fullsized":"2023-10-09T09:31:39.595797+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1027,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.611181+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:31:39.611181+0000","last_peered":"2023-10-09T09:31:39.611181+0000","last_clean":"2023-10-09T09:31:39.611181+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:31:39.611181+0000","last_undegraded":"2023-10-09T09:31:39.611181+0000","last_fullsized":"2023-10-09T09:31:39.611181+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1044,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.593608+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:31:39.593608+0000","last_peered":"2023-10-09T09:31:39.593608+0000","last_clean":"2023-10-09T09:31:39.593608+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:31:39.593608+0000","last_undegraded":"2023-10-09T09:31:39.593608+0000","last_fullsized":"2023-10-09T09:31:39.593608+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"1113'5073","reported_seq":5110,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:54.184801+0000","last_change":"2023-10-09T09:31:28.896417+0000","last_active":"2023-10-09T09:31:54.184801+0000","last_peered":"2023-10-09T09:31:54.184801+0000","last_clean":"2023-10-09T09:31:54.184801+0000","last_became_active":"2023-10-09T09:31:28.870011+0000","last_became_peered":"2023-10-09T09:31:28.870011+0000","last_unstale":"2023-10-09T09:31:54.184801+0000","last_undegraded":"2023-10-09T09:31:54.184801+0000","last_fullsized":"2023-10-09T09:31:54.184801+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":5073,"log_dups_size":0,"ondisk_log_size":5073,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189005824,"num_objects":2884,"num_object_clones":0,"num_object_copies":8652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2884,"num_write_kb":184576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1041,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.596225+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:31:39.596225+0000","last_peered":"2023-10-09T09:31:39.596225+0000","last_clean":"2023-10-09T09:31:39.596225+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:31:39.596225+0000","last_undegraded":"2023-10-09T09:31:39.596225+0000","last_fullsized":"2023-10-09T09:31:39.596225+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1012,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:40.590762+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:31:40.590762+0000","last_peered":"2023-10-09T09:31:40.590762+0000","last_clean":"2023-10-09T09:31:40.590762+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:31:40.590762+0000","last_undegraded":"2023-10-09T09:31:40.590762+0000","last_fullsized":"2023-10-09T09:31:40.590762+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"1113'4848","reported_seq":4886,"reported_epoch":1113,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:31:53.404310+0000","last_change":"2023-10-09T09:31:41.277588+0000","last_active":"2023-10-09T09:31:53.404310+0000","last_peered":"2023-10-09T09:31:53.404310+0000","last_clean":"2023-10-09T09:31:37.505588+0000","last_became_active":"2023-10-09T09:31:41.273317+0000","last_became_peered":"2023-10-09T09:31:41.273317+0000","last_unstale":"2023-10-09T09:31:53.404310+0000","last_undegraded":"2023-10-09T09:31:40.658391+0000","last_fullsized":"2023-10-09T09:31:40.587106+0000","mapping_epoch":1112,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":4848,"log_dups_size":0,"ondisk_log_size":4848,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179961856,"num_objects":2746,"num_object_clones":0,"num_object_copies":8238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2340,"num_objects_misplaced":2340,"num_objects_unfound":0,"num_objects_dirty":2746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2746,"num_write_kb":175744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":406},{"shards":"1,2,3","objects":2340}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1065,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.584735+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:31:39.584735+0000","last_peered":"2023-10-09T09:31:39.584735+0000","last_clean":"2023-10-09T09:31:39.584735+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:31:39.584735+0000","last_undegraded":"2023-10-09T09:31:39.584735+0000","last_fullsized":"2023-10-09T09:31:39.584735+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1023,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:40.591804+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:31:40.591804+0000","last_peered":"2023-10-09T09:31:40.591804+0000","last_clean":"2023-10-09T09:31:40.591804+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:31:40.591804+0000","last_undegraded":"2023-10-09T09:31:40.591804+0000","last_fullsized":"2023-10-09T09:31:40.591804+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"1113'5046","reported_seq":5082,"reported_epoch":1113,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:31:54.165278+0000","last_change":"2023-10-09T09:31:39.149208+0000","last_active":"2023-10-09T09:31:54.165278+0000","last_peered":"2023-10-09T09:31:54.165278+0000","last_clean":"2023-10-09T09:31:35.416261+0000","last_became_active":"2023-10-09T09:31:39.144598+0000","last_became_peered":"2023-10-09T09:31:39.144598+0000","last_unstale":"2023-10-09T09:31:54.165278+0000","last_undegraded":"2023-10-09T09:31:38.594882+0000","last_fullsized":"2023-10-09T09:31:38.581135+0000","mapping_epoch":1110,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"1107'4587","last_scrub_stamp":"2023-10-09T09:31:35.597905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:35.597905+0000","objects_scrubbed":0,"log_size":5046,"log_dups_size":0,"ondisk_log_size":5046,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188055552,"num_objects":2870,"num_object_clones":0,"num_object_copies":8610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2380,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2870,"num_write_kb":183648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2380},{"shards":"3,6","objects":490}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1044,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.599101+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:31:39.599101+0000","last_peered":"2023-10-09T09:31:39.599101+0000","last_clean":"2023-10-09T09:31:39.599101+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:31:39.599101+0000","last_undegraded":"2023-10-09T09:31:39.599101+0000","last_fullsized":"2023-10-09T09:31:39.599101+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1050,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.599308+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:31:39.599308+0000","last_peered":"2023-10-09T09:31:39.599308+0000","last_clean":"2023-10-09T09:31:39.599308+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:31:39.599308+0000","last_undegraded":"2023-10-09T09:31:39.599308+0000","last_fullsized":"2023-10-09T09:31:39.599308+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"1113'4955","reported_seq":7395,"reported_epoch":1113,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T09:31:54.281159+0000","last_change":"2023-10-09T09:31:36.698345+0000","last_active":"2023-10-09T09:31:54.281159+0000","last_peered":"2023-10-09T09:31:54.281159+0000","last_clean":"2023-10-09T09:31:34.593892+0000","last_became_active":"2023-10-09T09:31:36.659858+0000","last_became_peered":"2023-10-09T09:31:36.659858+0000","last_unstale":"2023-10-09T09:31:54.281159+0000","last_undegraded":"2023-10-09T09:31:54.281159+0000","last_fullsized":"2023-10-09T09:31:36.579707+0000","mapping_epoch":1108,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":4955,"log_dups_size":0,"ondisk_log_size":4955,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184647680,"num_objects":2818,"num_object_clones":0,"num_object_copies":8454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1157,"num_objects_unfound":0,"num_objects_dirty":2818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2818,"num_write_kb":180320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1201,"num_bytes_recovered":78708736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":1661},{"shards":"1,3,5","objects":1157}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1071,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.591574+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:31:39.591574+0000","last_peered":"2023-10-09T09:31:39.591574+0000","last_clean":"2023-10-09T09:31:39.591574+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:31:39.591574+0000","last_undegraded":"2023-10-09T09:31:39.591574+0000","last_fullsized":"2023-10-09T09:31:39.591574+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1063,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.591231+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:31:39.591231+0000","last_peered":"2023-10-09T09:31:39.591231+0000","last_clean":"2023-10-09T09:31:39.591231+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:31:39.591231+0000","last_undegraded":"2023-10-09T09:31:39.591231+0000","last_fullsized":"2023-10-09T09:31:39.591231+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"1113'4954","reported_seq":4985,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:51.470168+0000","last_change":"2023-10-09T09:31:28.850832+0000","last_active":"2023-10-09T09:31:51.470168+0000","last_peered":"2023-10-09T09:31:51.470168+0000","last_clean":"2023-10-09T09:31:51.470168+0000","last_became_active":"2023-10-09T09:31:28.847789+0000","last_became_peered":"2023-10-09T09:31:28.847789+0000","last_unstale":"2023-10-09T09:31:51.470168+0000","last_undegraded":"2023-10-09T09:31:51.470168+0000","last_fullsized":"2023-10-09T09:31:51.470168+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":4954,"log_dups_size":0,"ondisk_log_size":4954,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182583296,"num_objects":2786,"num_object_clones":0,"num_object_copies":8358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2786,"num_write_kb":178304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1043,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.598852+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:31:39.598852+0000","last_peered":"2023-10-09T09:31:39.598852+0000","last_clean":"2023-10-09T09:31:39.598852+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:31:39.598852+0000","last_undegraded":"2023-10-09T09:31:39.598852+0000","last_fullsized":"2023-10-09T09:31:39.598852+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1011,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.608494+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:31:39.608494+0000","last_peered":"2023-10-09T09:31:39.608494+0000","last_clean":"2023-10-09T09:31:39.608494+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:31:39.608494+0000","last_undegraded":"2023-10-09T09:31:39.608494+0000","last_fullsized":"2023-10-09T09:31:39.608494+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"1113'4950","reported_seq":4984,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:52.334830+0000","last_change":"2023-10-09T09:31:41.209137+0000","last_active":"2023-10-09T09:31:52.334830+0000","last_peered":"2023-10-09T09:31:52.334830+0000","last_clean":"2023-10-09T09:31:52.334830+0000","last_became_active":"2023-10-09T09:31:28.877504+0000","last_became_peered":"2023-10-09T09:31:28.877504+0000","last_unstale":"2023-10-09T09:31:52.334830+0000","last_undegraded":"2023-10-09T09:31:52.334830+0000","last_fullsized":"2023-10-09T09:31:52.334830+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":4950,"log_dups_size":0,"ondisk_log_size":4950,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184188928,"num_objects":2811,"num_object_clones":0,"num_object_copies":8433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2811,"num_write_kb":179872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1041,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.594603+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:31:39.594603+0000","last_peered":"2023-10-09T09:31:39.594603+0000","last_clean":"2023-10-09T09:31:39.594603+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:31:39.594603+0000","last_undegraded":"2023-10-09T09:31:39.594603+0000","last_fullsized":"2023-10-09T09:31:39.594603+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1064,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.584672+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:31:39.584672+0000","last_peered":"2023-10-09T09:31:39.584672+0000","last_clean":"2023-10-09T09:31:39.584672+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:31:39.584672+0000","last_undegraded":"2023-10-09T09:31:39.584672+0000","last_fullsized":"2023-10-09T09:31:39.584672+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"1113'4948","reported_seq":4984,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:51.299536+0000","last_change":"2023-10-09T09:31:33.052832+0000","last_active":"2023-10-09T09:31:51.299536+0000","last_peered":"2023-10-09T09:31:51.299536+0000","last_clean":"2023-10-09T09:31:51.299536+0000","last_became_active":"2023-10-09T09:31:28.853222+0000","last_became_peered":"2023-10-09T09:31:28.853222+0000","last_unstale":"2023-10-09T09:31:51.299536+0000","last_undegraded":"2023-10-09T09:31:51.299536+0000","last_fullsized":"2023-10-09T09:31:51.299536+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"1106'4479","last_scrub_stamp":"2023-10-09T09:31:33.052706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:33.052706+0000","objects_scrubbed":0,"log_size":4948,"log_dups_size":0,"ondisk_log_size":4948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184418304,"num_objects":2814,"num_object_clones":0,"num_object_copies":8442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":494,"num_write_kb":31616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1026,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.599049+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:31:39.599049+0000","last_peered":"2023-10-09T09:31:39.599049+0000","last_clean":"2023-10-09T09:31:39.599049+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:31:39.599049+0000","last_undegraded":"2023-10-09T09:31:39.599049+0000","last_fullsized":"2023-10-09T09:31:39.599049+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1040,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.595708+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:31:39.595708+0000","last_peered":"2023-10-09T09:31:39.595708+0000","last_clean":"2023-10-09T09:31:39.595708+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:31:39.595708+0000","last_undegraded":"2023-10-09T09:31:39.595708+0000","last_fullsized":"2023-10-09T09:31:39.595708+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"1113'4860","reported_seq":4894,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:52.320461+0000","last_change":"2023-10-09T09:31:28.901879+0000","last_active":"2023-10-09T09:31:52.320461+0000","last_peered":"2023-10-09T09:31:52.320461+0000","last_clean":"2023-10-09T09:31:52.320461+0000","last_became_active":"2023-10-09T09:31:28.865489+0000","last_became_peered":"2023-10-09T09:31:28.865489+0000","last_unstale":"2023-10-09T09:31:52.320461+0000","last_undegraded":"2023-10-09T09:31:52.320461+0000","last_fullsized":"2023-10-09T09:31:52.320461+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":4860,"log_dups_size":0,"ondisk_log_size":4860,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179109888,"num_objects":2733,"num_object_clones":0,"num_object_copies":8199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2733,"num_write_kb":174912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1054,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.592941+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:31:39.592941+0000","last_peered":"2023-10-09T09:31:39.592941+0000","last_clean":"2023-10-09T09:31:39.592941+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:31:39.592941+0000","last_undegraded":"2023-10-09T09:31:39.592941+0000","last_fullsized":"2023-10-09T09:31:39.592941+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1046,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.609599+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:31:39.609599+0000","last_peered":"2023-10-09T09:31:39.609599+0000","last_clean":"2023-10-09T09:31:39.609599+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:31:39.609599+0000","last_undegraded":"2023-10-09T09:31:39.609599+0000","last_fullsized":"2023-10-09T09:31:39.609599+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1113'5669","reported_seq":5692,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:53.514181+0000","last_change":"2023-10-09T09:30:25.841890+0000","last_active":"2023-10-09T09:31:53.514181+0000","last_peered":"2023-10-09T09:31:53.514181+0000","last_clean":"2023-10-09T09:31:53.514181+0000","last_became_active":"2023-10-09T09:30:25.841584+0000","last_became_peered":"2023-10-09T09:30:25.841584+0000","last_unstale":"2023-10-09T09:31:53.514181+0000","last_undegraded":"2023-10-09T09:31:53.514181+0000","last_fullsized":"2023-10-09T09:31:53.514181+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":5669,"log_dups_size":0,"ondisk_log_size":5669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:07.145206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371523584,"num_objects":5669,"num_object_clones":0,"num_object_copies":17007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5669,"num_write_kb":362816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1042,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.599405+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:31:39.599405+0000","last_peered":"2023-10-09T09:31:39.599405+0000","last_clean":"2023-10-09T09:31:39.599405+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:31:39.599405+0000","last_undegraded":"2023-10-09T09:31:39.599405+0000","last_fullsized":"2023-10-09T09:31:39.599405+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1049,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.584458+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:31:39.584458+0000","last_peered":"2023-10-09T09:31:39.584458+0000","last_clean":"2023-10-09T09:31:39.584458+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:31:39.584458+0000","last_undegraded":"2023-10-09T09:31:39.584458+0000","last_fullsized":"2023-10-09T09:31:39.584458+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1113'5644","reported_seq":5667,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:54.278596+0000","last_change":"2023-10-09T09:30:25.842825+0000","last_active":"2023-10-09T09:31:54.278596+0000","last_peered":"2023-10-09T09:31:54.278596+0000","last_clean":"2023-10-09T09:31:54.278596+0000","last_became_active":"2023-10-09T09:30:25.842490+0000","last_became_peered":"2023-10-09T09:30:25.842490+0000","last_unstale":"2023-10-09T09:31:54.278596+0000","last_undegraded":"2023-10-09T09:31:54.278596+0000","last_fullsized":"2023-10-09T09:31:54.278596+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":5644,"log_dups_size":0,"ondisk_log_size":5644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:14.954211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369885184,"num_objects":5644,"num_object_clones":0,"num_object_copies":16932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5644,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1043,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.599328+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:31:39.599328+0000","last_peered":"2023-10-09T09:31:39.599328+0000","last_clean":"2023-10-09T09:31:39.599328+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:31:39.599328+0000","last_undegraded":"2023-10-09T09:31:39.599328+0000","last_fullsized":"2023-10-09T09:31:39.599328+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1078,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.591489+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:31:39.591489+0000","last_peered":"2023-10-09T09:31:39.591489+0000","last_clean":"2023-10-09T09:31:39.591489+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:31:39.591489+0000","last_undegraded":"2023-10-09T09:31:39.591489+0000","last_fullsized":"2023-10-09T09:31:39.591489+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1113'4863","reported_seq":4899,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:52.335780+0000","last_change":"2023-10-09T09:31:37.508913+0000","last_active":"2023-10-09T09:31:52.335780+0000","last_peered":"2023-10-09T09:31:52.335780+0000","last_clean":"2023-10-09T09:31:52.335780+0000","last_became_active":"2023-10-09T09:31:28.853535+0000","last_became_peered":"2023-10-09T09:31:28.853535+0000","last_unstale":"2023-10-09T09:31:52.335780+0000","last_undegraded":"2023-10-09T09:31:52.335780+0000","last_fullsized":"2023-10-09T09:31:52.335780+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":4863,"log_dups_size":0,"ondisk_log_size":4863,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179437568,"num_objects":2738,"num_object_clones":0,"num_object_copies":8214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2738,"num_write_kb":175232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1027,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.599046+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:31:39.599046+0000","last_peered":"2023-10-09T09:31:39.599046+0000","last_clean":"2023-10-09T09:31:39.599046+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:31:39.599046+0000","last_undegraded":"2023-10-09T09:31:39.599046+0000","last_fullsized":"2023-10-09T09:31:39.599046+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1045,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.593172+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:31:39.593172+0000","last_peered":"2023-10-09T09:31:39.593172+0000","last_clean":"2023-10-09T09:31:39.593172+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:31:39.593172+0000","last_undegraded":"2023-10-09T09:31:39.593172+0000","last_fullsized":"2023-10-09T09:31:39.593172+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1113'5127","reported_seq":5161,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:55.171244+0000","last_change":"2023-10-09T09:31:31.170450+0000","last_active":"2023-10-09T09:31:55.171244+0000","last_peered":"2023-10-09T09:31:55.171244+0000","last_clean":"2023-10-09T09:31:55.171244+0000","last_became_active":"2023-10-09T09:31:28.843228+0000","last_became_peered":"2023-10-09T09:31:28.843228+0000","last_unstale":"2023-10-09T09:31:55.171244+0000","last_undegraded":"2023-10-09T09:31:55.171244+0000","last_fullsized":"2023-10-09T09:31:55.171244+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":5127,"log_dups_size":0,"ondisk_log_size":5127,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190414848,"num_objects":2905,"num_object_clones":0,"num_object_copies":8715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2905,"num_write_kb":185952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1065,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.584662+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:31:39.584662+0000","last_peered":"2023-10-09T09:31:39.584662+0000","last_clean":"2023-10-09T09:31:39.584662+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:31:39.584662+0000","last_undegraded":"2023-10-09T09:31:39.584662+0000","last_fullsized":"2023-10-09T09:31:39.584662+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1040,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.594831+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:31:39.594831+0000","last_peered":"2023-10-09T09:31:39.594831+0000","last_clean":"2023-10-09T09:31:39.594831+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:31:39.594831+0000","last_undegraded":"2023-10-09T09:31:39.594831+0000","last_fullsized":"2023-10-09T09:31:39.594831+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1113'5603","reported_seq":5626,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:52.312729+0000","last_change":"2023-10-09T09:30:25.837579+0000","last_active":"2023-10-09T09:31:52.312729+0000","last_peered":"2023-10-09T09:31:52.312729+0000","last_clean":"2023-10-09T09:31:52.312729+0000","last_became_active":"2023-10-09T09:30:25.836784+0000","last_became_peered":"2023-10-09T09:30:25.836784+0000","last_unstale":"2023-10-09T09:31:52.312729+0000","last_undegraded":"2023-10-09T09:31:52.312729+0000","last_fullsized":"2023-10-09T09:31:52.312729+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":5603,"log_dups_size":0,"ondisk_log_size":5603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:55.296361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367198208,"num_objects":5603,"num_object_clones":0,"num_object_copies":16809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5603,"num_write_kb":358592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":1030,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.608685+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:31:39.608685+0000","last_peered":"2023-10-09T09:31:39.608685+0000","last_clean":"2023-10-09T09:31:39.608685+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:31:39.608685+0000","last_undegraded":"2023-10-09T09:31:39.608685+0000","last_fullsized":"2023-10-09T09:31:39.608685+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1063,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.591704+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:31:39.591704+0000","last_peered":"2023-10-09T09:31:39.591704+0000","last_clean":"2023-10-09T09:31:39.591704+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:31:39.591704+0000","last_undegraded":"2023-10-09T09:31:39.591704+0000","last_fullsized":"2023-10-09T09:31:39.591704+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1113'5772","reported_seq":5795,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:55.152260+0000","last_change":"2023-10-09T09:30:25.841208+0000","last_active":"2023-10-09T09:31:55.152260+0000","last_peered":"2023-10-09T09:31:55.152260+0000","last_clean":"2023-10-09T09:31:55.152260+0000","last_became_active":"2023-10-09T09:30:25.840860+0000","last_became_peered":"2023-10-09T09:30:25.840860+0000","last_unstale":"2023-10-09T09:31:55.152260+0000","last_undegraded":"2023-10-09T09:31:55.152260+0000","last_fullsized":"2023-10-09T09:31:55.152260+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":5772,"log_dups_size":0,"ondisk_log_size":5772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:20.488338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378273792,"num_objects":5772,"num_object_clones":0,"num_object_copies":17316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5772,"num_write_kb":369408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1046,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.593361+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:31:39.593361+0000","last_peered":"2023-10-09T09:31:39.593361+0000","last_clean":"2023-10-09T09:31:39.593361+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:31:39.593361+0000","last_undegraded":"2023-10-09T09:31:39.593361+0000","last_fullsized":"2023-10-09T09:31:39.593361+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1078,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.591219+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:31:39.591219+0000","last_peered":"2023-10-09T09:31:39.591219+0000","last_clean":"2023-10-09T09:31:39.591219+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:31:39.591219+0000","last_undegraded":"2023-10-09T09:31:39.591219+0000","last_fullsized":"2023-10-09T09:31:39.591219+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1113'5616","reported_seq":5638,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:51.488447+0000","last_change":"2023-10-09T09:30:25.838078+0000","last_active":"2023-10-09T09:31:51.488447+0000","last_peered":"2023-10-09T09:31:51.488447+0000","last_clean":"2023-10-09T09:31:51.488447+0000","last_became_active":"2023-10-09T09:30:25.837594+0000","last_became_peered":"2023-10-09T09:30:25.837594+0000","last_unstale":"2023-10-09T09:31:51.488447+0000","last_undegraded":"2023-10-09T09:31:51.488447+0000","last_fullsized":"2023-10-09T09:31:51.488447+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":5616,"log_dups_size":0,"ondisk_log_size":5616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368050176,"num_objects":5616,"num_object_clones":0,"num_object_copies":16848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5616,"num_write_kb":359424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1064,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.590843+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:31:39.590843+0000","last_peered":"2023-10-09T09:31:39.590843+0000","last_clean":"2023-10-09T09:31:39.590843+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:31:39.590843+0000","last_undegraded":"2023-10-09T09:31:39.590843+0000","last_fullsized":"2023-10-09T09:31:39.590843+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1063,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.591953+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:31:39.591953+0000","last_peered":"2023-10-09T09:31:39.591953+0000","last_clean":"2023-10-09T09:31:39.591953+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:31:39.591953+0000","last_undegraded":"2023-10-09T09:31:39.591953+0000","last_fullsized":"2023-10-09T09:31:39.591953+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1113'5584","reported_seq":5606,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:51.469847+0000","last_change":"2023-10-09T09:30:25.841044+0000","last_active":"2023-10-09T09:31:51.469847+0000","last_peered":"2023-10-09T09:31:51.469847+0000","last_clean":"2023-10-09T09:31:51.469847+0000","last_became_active":"2023-10-09T09:30:25.840627+0000","last_became_peered":"2023-10-09T09:30:25.840627+0000","last_unstale":"2023-10-09T09:31:51.469847+0000","last_undegraded":"2023-10-09T09:31:51.469847+0000","last_fullsized":"2023-10-09T09:31:51.469847+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":5584,"log_dups_size":0,"ondisk_log_size":5584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365953024,"num_objects":5584,"num_object_clones":0,"num_object_copies":16752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5584,"num_write_kb":357376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1013,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.611061+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:31:39.611061+0000","last_peered":"2023-10-09T09:31:39.611061+0000","last_clean":"2023-10-09T09:31:39.611061+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:31:39.611061+0000","last_undegraded":"2023-10-09T09:31:39.611061+0000","last_fullsized":"2023-10-09T09:31:39.611061+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1050,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.597358+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:31:39.597358+0000","last_peered":"2023-10-09T09:31:39.597358+0000","last_clean":"2023-10-09T09:31:39.597358+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:31:39.597358+0000","last_undegraded":"2023-10-09T09:31:39.597358+0000","last_fullsized":"2023-10-09T09:31:39.597358+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1113'4982","reported_seq":5018,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:51.373280+0000","last_change":"2023-10-09T09:31:32.026991+0000","last_active":"2023-10-09T09:31:51.373280+0000","last_peered":"2023-10-09T09:31:51.373280+0000","last_clean":"2023-10-09T09:31:51.373280+0000","last_became_active":"2023-10-09T09:31:28.828078+0000","last_became_peered":"2023-10-09T09:31:28.828078+0000","last_unstale":"2023-10-09T09:31:51.373280+0000","last_undegraded":"2023-10-09T09:31:51.373280+0000","last_fullsized":"2023-10-09T09:31:51.373280+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1106'4471","last_scrub_stamp":"2023-10-09T09:31:32.026835+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:32.026835+0000","objects_scrubbed":0,"log_size":4982,"log_dups_size":0,"ondisk_log_size":4982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185860096,"num_objects":2836,"num_object_clones":0,"num_object_copies":8508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":533,"num_write_kb":34112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1046,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.597555+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:31:39.597555+0000","last_peered":"2023-10-09T09:31:39.597555+0000","last_clean":"2023-10-09T09:31:39.597555+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:31:39.597555+0000","last_undegraded":"2023-10-09T09:31:39.597555+0000","last_fullsized":"2023-10-09T09:31:39.597555+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1080,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.584756+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:31:39.584756+0000","last_peered":"2023-10-09T09:31:39.584756+0000","last_clean":"2023-10-09T09:31:39.584756+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:31:39.584756+0000","last_undegraded":"2023-10-09T09:31:39.584756+0000","last_fullsized":"2023-10-09T09:31:39.584756+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1036,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.599356+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:31:39.599356+0000","last_peered":"2023-10-09T09:31:39.599356+0000","last_clean":"2023-10-09T09:31:39.599356+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:31:39.599356+0000","last_undegraded":"2023-10-09T09:31:39.599356+0000","last_fullsized":"2023-10-09T09:31:39.599356+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1113'4928","reported_seq":4976,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:51.469808+0000","last_change":"2023-10-09T09:31:50.904265+0000","last_active":"2023-10-09T09:31:51.469808+0000","last_peered":"2023-10-09T09:31:51.469808+0000","last_clean":"2023-10-09T09:31:51.469808+0000","last_became_active":"2023-10-09T09:31:28.823603+0000","last_became_peered":"2023-10-09T09:31:28.823603+0000","last_unstale":"2023-10-09T09:31:51.469808+0000","last_undegraded":"2023-10-09T09:31:51.469808+0000","last_fullsized":"2023-10-09T09:31:51.469808+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":4928,"log_dups_size":0,"ondisk_log_size":4928,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180879360,"num_objects":2760,"num_object_clones":0,"num_object_copies":8280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2760,"num_write_kb":176640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1069,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.584513+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:31:39.584513+0000","last_peered":"2023-10-09T09:31:39.584513+0000","last_clean":"2023-10-09T09:31:39.584513+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:31:39.584513+0000","last_undegraded":"2023-10-09T09:31:39.584513+0000","last_fullsized":"2023-10-09T09:31:39.584513+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1064,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.591266+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:31:39.591266+0000","last_peered":"2023-10-09T09:31:39.591266+0000","last_clean":"2023-10-09T09:31:39.591266+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:31:39.591266+0000","last_undegraded":"2023-10-09T09:31:39.591266+0000","last_fullsized":"2023-10-09T09:31:39.591266+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1066,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.596951+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:31:39.596951+0000","last_peered":"2023-10-09T09:31:39.596951+0000","last_clean":"2023-10-09T09:31:39.596951+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:31:39.596951+0000","last_undegraded":"2023-10-09T09:31:39.596951+0000","last_fullsized":"2023-10-09T09:31:39.596951+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1113'4837","reported_seq":4874,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:52.313589+0000","last_change":"2023-10-09T09:31:33.352384+0000","last_active":"2023-10-09T09:31:52.313589+0000","last_peered":"2023-10-09T09:31:52.313589+0000","last_clean":"2023-10-09T09:31:52.313589+0000","last_became_active":"2023-10-09T09:31:28.838318+0000","last_became_peered":"2023-10-09T09:31:28.838318+0000","last_unstale":"2023-10-09T09:31:52.313589+0000","last_undegraded":"2023-10-09T09:31:52.313589+0000","last_fullsized":"2023-10-09T09:31:52.313589+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":4837,"log_dups_size":0,"ondisk_log_size":4837,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177602560,"num_objects":2710,"num_object_clones":0,"num_object_copies":8130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2710,"num_write_kb":173440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1033,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:40.584429+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:31:40.584429+0000","last_peered":"2023-10-09T09:31:40.584429+0000","last_clean":"2023-10-09T09:31:40.584429+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:31:40.584429+0000","last_undegraded":"2023-10-09T09:31:40.584429+0000","last_fullsized":"2023-10-09T09:31:40.584429+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1051,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.597815+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:31:39.597815+0000","last_peered":"2023-10-09T09:31:39.597815+0000","last_clean":"2023-10-09T09:31:39.597815+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:31:39.597815+0000","last_undegraded":"2023-10-09T09:31:39.597815+0000","last_fullsized":"2023-10-09T09:31:39.597815+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1040,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.597446+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:31:39.597446+0000","last_peered":"2023-10-09T09:31:39.597446+0000","last_clean":"2023-10-09T09:31:39.597446+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:31:39.597446+0000","last_undegraded":"2023-10-09T09:31:39.597446+0000","last_fullsized":"2023-10-09T09:31:39.597446+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1113'4937","reported_seq":4971,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:52.271856+0000","last_change":"2023-10-09T09:31:41.209655+0000","last_active":"2023-10-09T09:31:52.271856+0000","last_peered":"2023-10-09T09:31:52.271856+0000","last_clean":"2023-10-09T09:31:52.271856+0000","last_became_active":"2023-10-09T09:31:28.832534+0000","last_became_peered":"2023-10-09T09:31:28.832534+0000","last_unstale":"2023-10-09T09:31:52.271856+0000","last_undegraded":"2023-10-09T09:31:52.271856+0000","last_fullsized":"2023-10-09T09:31:52.271856+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":4937,"log_dups_size":0,"ondisk_log_size":4937,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183336960,"num_objects":2797,"num_object_clones":0,"num_object_copies":8391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2797,"num_write_kb":179040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1069,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.584517+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:31:39.584517+0000","last_peered":"2023-10-09T09:31:39.584517+0000","last_clean":"2023-10-09T09:31:39.584517+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:31:39.584517+0000","last_undegraded":"2023-10-09T09:31:39.584517+0000","last_fullsized":"2023-10-09T09:31:39.584517+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1277,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.598586+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:31:39.598586+0000","last_peered":"2023-10-09T09:31:39.598586+0000","last_clean":"2023-10-09T09:31:39.598586+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:31:39.598586+0000","last_undegraded":"2023-10-09T09:31:39.598586+0000","last_fullsized":"2023-10-09T09:31:39.598586+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1024,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.609286+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:31:39.609286+0000","last_peered":"2023-10-09T09:31:39.609286+0000","last_clean":"2023-10-09T09:31:39.609286+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:31:39.609286+0000","last_undegraded":"2023-10-09T09:31:39.609286+0000","last_fullsized":"2023-10-09T09:31:39.609286+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1030,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.595324+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:31:39.595324+0000","last_peered":"2023-10-09T09:31:39.595324+0000","last_clean":"2023-10-09T09:31:39.595324+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:31:39.595324+0000","last_undegraded":"2023-10-09T09:31:39.595324+0000","last_fullsized":"2023-10-09T09:31:39.595324+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1113'4919","reported_seq":4956,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:51.323626+0000","last_change":"2023-10-09T09:31:29.845811+0000","last_active":"2023-10-09T09:31:51.323626+0000","last_peered":"2023-10-09T09:31:51.323626+0000","last_clean":"2023-10-09T09:31:51.323626+0000","last_became_active":"2023-10-09T09:31:28.832067+0000","last_became_peered":"2023-10-09T09:31:28.832067+0000","last_unstale":"2023-10-09T09:31:51.323626+0000","last_undegraded":"2023-10-09T09:31:51.323626+0000","last_fullsized":"2023-10-09T09:31:51.323626+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1106'4346","last_scrub_stamp":"2023-10-09T09:31:29.845731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:29.845731+0000","objects_scrubbed":0,"log_size":4919,"log_dups_size":0,"ondisk_log_size":4919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180224000,"num_objects":2750,"num_object_clones":0,"num_object_copies":8250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":585,"num_write_kb":37440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1060,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.598616+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:31:39.598616+0000","last_peered":"2023-10-09T09:31:39.598616+0000","last_clean":"2023-10-09T09:31:39.598616+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:31:39.598616+0000","last_undegraded":"2023-10-09T09:31:39.598616+0000","last_fullsized":"2023-10-09T09:31:39.598616+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1042,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.594356+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:31:39.594356+0000","last_peered":"2023-10-09T09:31:39.594356+0000","last_clean":"2023-10-09T09:31:39.594356+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:31:39.594356+0000","last_undegraded":"2023-10-09T09:31:39.594356+0000","last_fullsized":"2023-10-09T09:31:39.594356+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1053,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.595557+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:31:39.595557+0000","last_peered":"2023-10-09T09:31:39.595557+0000","last_clean":"2023-10-09T09:31:39.595557+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:31:39.595557+0000","last_undegraded":"2023-10-09T09:31:39.595557+0000","last_fullsized":"2023-10-09T09:31:39.595557+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"1113'4888","reported_seq":4925,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:51.396846+0000","last_change":"2023-10-09T09:31:29.987166+0000","last_active":"2023-10-09T09:31:51.396846+0000","last_peered":"2023-10-09T09:31:51.396846+0000","last_clean":"2023-10-09T09:31:51.396846+0000","last_became_active":"2023-10-09T09:31:28.836386+0000","last_became_peered":"2023-10-09T09:31:28.836386+0000","last_unstale":"2023-10-09T09:31:51.396846+0000","last_undegraded":"2023-10-09T09:31:51.396846+0000","last_fullsized":"2023-10-09T09:31:51.396846+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1106'4321","last_scrub_stamp":"2023-10-09T09:31:29.987092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:29.987092+0000","objects_scrubbed":0,"log_size":4888,"log_dups_size":0,"ondisk_log_size":4888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179699712,"num_objects":2742,"num_object_clones":0,"num_object_copies":8226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":586,"num_write_kb":37504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1045,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.598946+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:31:39.598946+0000","last_peered":"2023-10-09T09:31:39.598946+0000","last_clean":"2023-10-09T09:31:39.598946+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:31:39.598946+0000","last_undegraded":"2023-10-09T09:31:39.598946+0000","last_fullsized":"2023-10-09T09:31:39.598946+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1052,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.597083+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:31:39.597083+0000","last_peered":"2023-10-09T09:31:39.597083+0000","last_clean":"2023-10-09T09:31:39.597083+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:31:39.597083+0000","last_undegraded":"2023-10-09T09:31:39.597083+0000","last_fullsized":"2023-10-09T09:31:39.597083+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1026,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.595151+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:31:39.595151+0000","last_peered":"2023-10-09T09:31:39.595151+0000","last_clean":"2023-10-09T09:31:39.595151+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:31:39.595151+0000","last_undegraded":"2023-10-09T09:31:39.595151+0000","last_fullsized":"2023-10-09T09:31:39.595151+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1113'5084","reported_seq":5116,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:54.272985+0000","last_change":"2023-10-09T09:31:28.883097+0000","last_active":"2023-10-09T09:31:54.272985+0000","last_peered":"2023-10-09T09:31:54.272985+0000","last_clean":"2023-10-09T09:31:54.272985+0000","last_became_active":"2023-10-09T09:31:28.861343+0000","last_became_peered":"2023-10-09T09:31:28.861343+0000","last_unstale":"2023-10-09T09:31:54.272985+0000","last_undegraded":"2023-10-09T09:31:54.272985+0000","last_fullsized":"2023-10-09T09:31:54.272985+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":5084,"log_dups_size":0,"ondisk_log_size":5084,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189726720,"num_objects":2895,"num_object_clones":0,"num_object_copies":8685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2895,"num_write_kb":185280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":1030,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.609514+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:31:39.609514+0000","last_peered":"2023-10-09T09:31:39.609514+0000","last_clean":"2023-10-09T09:31:39.609514+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:31:39.609514+0000","last_undegraded":"2023-10-09T09:31:39.609514+0000","last_fullsized":"2023-10-09T09:31:39.609514+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1034,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.597355+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:31:39.597355+0000","last_peered":"2023-10-09T09:31:39.597355+0000","last_clean":"2023-10-09T09:31:39.597355+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:31:39.597355+0000","last_undegraded":"2023-10-09T09:31:39.597355+0000","last_fullsized":"2023-10-09T09:31:39.597355+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1028,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.610997+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:31:39.610997+0000","last_peered":"2023-10-09T09:31:39.610997+0000","last_clean":"2023-10-09T09:31:39.610997+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:31:39.610997+0000","last_undegraded":"2023-10-09T09:31:39.610997+0000","last_fullsized":"2023-10-09T09:31:39.610997+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1113'4918","reported_seq":5009,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:54.228603+0000","last_change":"2023-10-09T09:31:37.174228+0000","last_active":"2023-10-09T09:31:54.228603+0000","last_peered":"2023-10-09T09:31:54.228603+0000","last_clean":"2023-10-09T09:31:54.228603+0000","last_became_active":"2023-10-09T09:31:28.822827+0000","last_became_peered":"2023-10-09T09:31:28.822827+0000","last_unstale":"2023-10-09T09:31:54.228603+0000","last_undegraded":"2023-10-09T09:31:54.228603+0000","last_fullsized":"2023-10-09T09:31:54.228603+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1109'4470","last_scrub_stamp":"2023-10-09T09:31:37.173935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:37.173935+0000","objects_scrubbed":2369,"log_size":4918,"log_dups_size":0,"ondisk_log_size":4918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:32.212068+0000","scrub_duration":0.64874551999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185335808,"num_objects":2828,"num_object_clones":0,"num_object_copies":8484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":459,"num_write_kb":29376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1049,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.593605+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:31:39.593605+0000","last_peered":"2023-10-09T09:31:39.593605+0000","last_clean":"2023-10-09T09:31:39.593605+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:31:39.593605+0000","last_undegraded":"2023-10-09T09:31:39.593605+0000","last_fullsized":"2023-10-09T09:31:39.593605+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1045,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.595476+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:31:39.595476+0000","last_peered":"2023-10-09T09:31:39.595476+0000","last_clean":"2023-10-09T09:31:39.595476+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:31:39.595476+0000","last_undegraded":"2023-10-09T09:31:39.595476+0000","last_fullsized":"2023-10-09T09:31:39.595476+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1048,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.591114+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:31:39.591114+0000","last_peered":"2023-10-09T09:31:39.591114+0000","last_clean":"2023-10-09T09:31:39.591114+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:31:39.591114+0000","last_undegraded":"2023-10-09T09:31:39.591114+0000","last_fullsized":"2023-10-09T09:31:39.591114+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1029,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:41.208925+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:31:41.208925+0000","last_peered":"2023-10-09T09:31:41.208925+0000","last_clean":"2023-10-09T09:31:41.208925+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:31:41.208925+0000","last_undegraded":"2023-10-09T09:31:41.208925+0000","last_fullsized":"2023-10-09T09:31:41.208925+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1050,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.605097+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:31:39.605097+0000","last_peered":"2023-10-09T09:31:39.605097+0000","last_clean":"2023-10-09T09:31:39.605097+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:31:39.605097+0000","last_undegraded":"2023-10-09T09:31:39.605097+0000","last_fullsized":"2023-10-09T09:31:39.605097+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"1113'5167","reported_seq":5264,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:55.182061+0000","last_change":"2023-10-09T09:31:33.829441+0000","last_active":"2023-10-09T09:31:55.182061+0000","last_peered":"2023-10-09T09:31:55.182061+0000","last_clean":"2023-10-09T09:31:55.182061+0000","last_became_active":"2023-10-09T09:31:28.863946+0000","last_became_peered":"2023-10-09T09:31:28.863946+0000","last_unstale":"2023-10-09T09:31:55.182061+0000","last_undegraded":"2023-10-09T09:31:55.182061+0000","last_fullsized":"2023-10-09T09:31:55.182061+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"1106'4655","last_scrub_stamp":"2023-10-09T09:31:33.829271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:33.829271+0000","objects_scrubbed":2411,"log_size":5167,"log_dups_size":0,"ondisk_log_size":5167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:23.193885+0000","scrub_duration":0.70828463799999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192020480,"num_objects":2930,"num_object_clones":0,"num_object_copies":8790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":519,"num_write_kb":33216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1067,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.584669+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:31:39.584669+0000","last_peered":"2023-10-09T09:31:39.584669+0000","last_clean":"2023-10-09T09:31:39.584669+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:31:39.584669+0000","last_undegraded":"2023-10-09T09:31:39.584669+0000","last_fullsized":"2023-10-09T09:31:39.584669+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1078,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.591387+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:31:39.591387+0000","last_peered":"2023-10-09T09:31:39.591387+0000","last_clean":"2023-10-09T09:31:39.591387+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:31:39.591387+0000","last_undegraded":"2023-10-09T09:31:39.591387+0000","last_fullsized":"2023-10-09T09:31:39.591387+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"1113'4885","reported_seq":4917,"reported_epoch":1113,"state":"active+clean","last_fresh":"2023-10-09T09:31:52.334367+0000","last_change":"2023-10-09T09:31:28.878549+0000","last_active":"2023-10-09T09:31:52.334367+0000","last_peered":"2023-10-09T09:31:52.334367+0000","last_clean":"2023-10-09T09:31:52.334367+0000","last_became_active":"2023-10-09T09:31:28.876555+0000","last_became_peered":"2023-10-09T09:31:28.876555+0000","last_unstale":"2023-10-09T09:31:52.334367+0000","last_undegraded":"2023-10-09T09:31:52.334367+0000","last_fullsized":"2023-10-09T09:31:52.334367+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":4885,"log_dups_size":0,"ondisk_log_size":4885,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180879360,"num_objects":2760,"num_object_clones":0,"num_object_copies":8280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2760,"num_write_kb":176640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1052,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.584771+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:31:39.584771+0000","last_peered":"2023-10-09T09:31:39.584771+0000","last_clean":"2023-10-09T09:31:39.584771+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:31:39.584771+0000","last_undegraded":"2023-10-09T09:31:39.584771+0000","last_fullsized":"2023-10-09T09:31:39.584771+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1078,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.590891+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:31:39.590891+0000","last_peered":"2023-10-09T09:31:39.590891+0000","last_clean":"2023-10-09T09:31:39.590891+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:31:39.590891+0000","last_undegraded":"2023-10-09T09:31:39.590891+0000","last_fullsized":"2023-10-09T09:31:39.590891+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":1041,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.608848+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:31:39.608848+0000","last_peered":"2023-10-09T09:31:39.608848+0000","last_clean":"2023-10-09T09:31:39.608848+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:31:39.608848+0000","last_undegraded":"2023-10-09T09:31:39.608848+0000","last_fullsized":"2023-10-09T09:31:39.608848+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1052,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.611030+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:31:39.611030+0000","last_peered":"2023-10-09T09:31:39.611030+0000","last_clean":"2023-10-09T09:31:39.611030+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:31:39.611030+0000","last_undegraded":"2023-10-09T09:31:39.611030+0000","last_fullsized":"2023-10-09T09:31:39.611030+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1048,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.596880+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:31:39.596880+0000","last_peered":"2023-10-09T09:31:39.596880+0000","last_clean":"2023-10-09T09:31:39.596880+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:31:39.596880+0000","last_undegraded":"2023-10-09T09:31:39.596880+0000","last_fullsized":"2023-10-09T09:31:39.596880+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1063,"reported_epoch":1112,"state":"active+clean","last_fresh":"2023-10-09T09:31:39.590556+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:31:39.590556+0000","last_peered":"2023-10-09T09:31:39.590556+0000","last_clean":"2023-10-09T09:31:39.590556+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:31:39.590556+0000","last_undegraded":"2023-10-09T09:31:39.590556+0000","last_fullsized":"2023-10-09T09:31:39.590556+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":5898272768,"num_objects":90001,"num_object_clones":0,"num_object_copies":270003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4720,"num_objects_misplaced":3497,"num_objects_unfound":0,"num_objects_dirty":90001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":76277,"num_write_kb":4881696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1210,"num_bytes_recovered":79298560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17779720192,"data_stored":17779720192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133057,"ondisk_log_size":133057,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397881,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2103320,"kb_used_data":1806096,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91641832,"statfs":{"total":95995035648,"available":93841235968,"internally_reserved":0,"allocated":1849442304,"data_stored":1849221134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:31:41 2023","interfaces":[{"interface":"back","average":{"1min":8.3179999999999996,"5min":2.9900000000000002,"15min":1.9630000000000001},"min":{"1min":0.33300000000000002,"5min":0.249,"15min":0.249},"max":{"1min":85.811999999999998,"5min":85.811999999999998,"15min":85.811999999999998},"last":2.242},{"interface":"front","average":{"1min":8.5600000000000005,"5min":3.1389999999999998,"15min":2.0920000000000001},"min":{"1min":0.44400000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":85.762,"5min":85.762,"15min":85.762},"last":2.282}]},{"osd":1,"last update":"Mon Oct 9 09:31:18 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.5389999999999999,"15min":1.53},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.792,"5min":8.2449999999999992,"15min":103.84699999999999},"last":2.4809999999999999},{"interface":"front","average":{"1min":0.70599999999999996,"5min":1.4279999999999999,"15min":1.569},"min":{"1min":0.34300000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.137,"5min":9.8190000000000008,"15min":103.955},"last":0.33100000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:31:34 2023","interfaces":[{"interface":"back","average":{"1min":4.0860000000000003,"5min":2.4830000000000001,"15min":1.671},"min":{"1min":0.35299999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":47.023000000000003,"5min":47.023000000000003,"15min":47.023000000000003},"last":0.38700000000000001},{"interface":"front","average":{"1min":3.8650000000000002,"5min":2.3610000000000002,"15min":1.5640000000000001},"min":{"1min":0.38800000000000001,"5min":0.373,"15min":0.373},"max":{"1min":46.847000000000001,"5min":46.847000000000001,"15min":46.847000000000001},"last":0.52500000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:31:12 2023","interfaces":[{"interface":"back","average":{"1min":4.024,"5min":3.601,"15min":2.2759999999999998},"min":{"1min":0.40200000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":47.119999999999997,"5min":57.258000000000003,"15min":107.97499999999999},"last":4.1879999999999997},{"interface":"front","average":{"1min":3.891,"5min":3.6869999999999998,"15min":2.3860000000000001},"min":{"1min":0.50700000000000001,"5min":0.317,"15min":0.317},"max":{"1min":47.204999999999998,"5min":55.426000000000002,"15min":105.282},"last":4.1310000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:31:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.3300000000000001,"15min":1.117},"min":{"1min":0.38100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.1470000000000002,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.878,"5min":1.228,"15min":1.069},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.2639999999999998,"5min":12.316000000000001,"15min":12.316000000000001},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:31:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":1.597,"15min":1.2869999999999999},"min":{"1min":0.40200000000000002,"5min":0.377,"15min":0.377},"max":{"1min":4.431,"5min":10.063000000000001,"15min":10.063000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.212,"15min":1.1399999999999999},"min":{"1min":0.41099999999999998,"5min":0.38900000000000001,"15min":0.36299999999999999},"max":{"1min":2.6059999999999999,"5min":8.1699999999999999,"15min":8.1699999999999999},"last":0.59399999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:31:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.236,"15min":1.1819999999999999},"min":{"1min":0.442,"5min":0.373,"15min":0.373},"max":{"1min":2.573,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.075,"5min":1.54,"15min":1.2849999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.3050000000000002,"5min":8.1980000000000004,"15min":8.1980000000000004},"last":0.61299999999999999}]}]},{"osd":6,"up_from":43,"seq":184683594114,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2094348,"kb_used_data":1797124,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91650804,"statfs":{"total":95995035648,"available":93850423296,"internally_reserved":0,"allocated":1840254976,"data_stored":1840045948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:31:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.4930000000000001,"15min":1.1100000000000001},"min":{"1min":0.33700000000000002,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":7.2329999999999997,"5min":16.850000000000001,"15min":16.850000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.514,"5min":1.2569999999999999,"15min":1.0780000000000001},"min":{"1min":0.30099999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.2489999999999997,"5min":7.2489999999999997,"15min":7.2489999999999997},"last":0.94399999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:31:15 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.972,"15min":1.2989999999999999},"min":{"1min":0.63500000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.8919999999999999,"5min":32.959000000000003,"15min":32.959000000000003},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.2989999999999999,"5min":1.869,"15min":1.2470000000000001},"min":{"1min":0.51000000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.431,"5min":34.793999999999997,"15min":34.793999999999997},"last":1.7809999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:31:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.9379999999999999,"15min":1.2849999999999999},"min":{"1min":0.36099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.1550000000000002,"5min":30.379000000000001,"15min":30.379000000000001},"last":1.4059999999999999},{"interface":"front","average":{"1min":1.042,"5min":1.698,"15min":1.3460000000000001},"min":{"1min":0.45600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.7629999999999999,"5min":30.332999999999998,"15min":30.332999999999998},"last":1.323}]},{"osd":3,"last update":"Mon Oct 9 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":3.3260000000000001,"15min":1.774},"min":{"1min":0.63900000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.9359999999999999,"5min":123.97199999999999,"15min":123.97199999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.161,"5min":3.3300000000000001,"15min":1.8169999999999999},"min":{"1min":0.47399999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.6580000000000004,"5min":124.059,"15min":124.059},"last":0.81399999999999995}]},{"osd":4,"last update":"Mon Oct 9 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.125,"15min":1.026},"min":{"1min":0.39200000000000002,"5min":0.312,"15min":0.312},"max":{"1min":1.3420000000000001,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.94599999999999995,"5min":1.306,"15min":1.1439999999999999},"min":{"1min":0.50900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.0979999999999999,"5min":6.7220000000000004,"15min":6.7220000000000004},"last":0.85299999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.234,"15min":1.143},"min":{"1min":0.52600000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.339,"5min":7.6479999999999997,"15min":7.6479999999999997},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.327,"15min":1.2150000000000001},"min":{"1min":0.46800000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.4900000000000002,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.996}]},{"osd":7,"last update":"Mon Oct 9 09:31:27 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.655,"15min":1.3360000000000001},"min":{"1min":0.54500000000000004,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":3.2959999999999998,"5min":7.3630000000000004,"15min":7.3630000000000004},"last":1.028},{"interface":"front","average":{"1min":1.212,"5min":1.6060000000000001,"15min":1.3220000000000001},"min":{"1min":0.57399999999999995,"5min":0.371,"15min":0.371},"max":{"1min":3.1309999999999998,"5min":7.6459999999999999,"15min":7.6459999999999999},"last":0.72399999999999998}]}]},{"osd":2,"up_from":634,"seq":2723009265868,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2582040,"kb_used_data":2324176,"kb_used_omap":122,"kb_used_meta":257733,"kb_avail":91163112,"statfs":{"total":95995035648,"available":93351026688,"internally_reserved":0,"allocated":2379956224,"data_stored":2378808318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":263919179},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":1.9490000000000001,"15min":1.1899999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.5259999999999998,"5min":8.9860000000000007,"15min":8.9860000000000007},"last":2.1909999999999998},{"interface":"front","average":{"1min":1.7470000000000001,"5min":1.986,"15min":1.262},"min":{"1min":0.27600000000000002,"5min":0.221,"15min":0.221},"max":{"1min":6.351,"5min":10.019,"15min":10.019},"last":2.0550000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.534,"15min":1.0680000000000001},"min":{"1min":0.254,"5min":0.184,"15min":0.184},"max":{"1min":4.6390000000000002,"5min":9.5139999999999993,"15min":9.5139999999999993},"last":1.8420000000000001},{"interface":"front","average":{"1min":1.407,"5min":1.9139999999999999,"15min":1.286},"min":{"1min":0.313,"5min":0.216,"15min":0.216},"max":{"1min":3.7490000000000001,"5min":8.702,"15min":8.702},"last":2.3050000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":2.0310000000000001,"15min":1.254},"min":{"1min":0.38800000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.5019999999999998,"5min":18.547999999999998,"15min":18.547999999999998},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.3660000000000001,"5min":2.3149999999999999,"15min":1.3819999999999999},"min":{"1min":0.54600000000000004,"5min":0.32700000000000001,"15min":0.30199999999999999},"max":{"1min":4.383,"5min":12.861000000000001,"15min":12.861000000000001},"last":2.2770000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.6799999999999999,"15min":1.286},"min":{"1min":0.372,"5min":0.309,"15min":0.309},"max":{"1min":2.4969999999999999,"5min":7.452,"15min":7.452},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.6659999999999999,"15min":1.319},"min":{"1min":0.36799999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.194,"5min":13.917999999999999,"15min":13.917999999999999},"last":0.51600000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.296,"15min":1.167},"min":{"1min":0.39500000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.7510000000000003,"5min":6.7619999999999996,"15min":6.7699999999999996},"last":1.879},{"interface":"front","average":{"1min":1.139,"5min":1.673,"15min":1.3360000000000001},"min":{"1min":0.48299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.952,"5min":10.393000000000001,"15min":10.393000000000001},"last":2.1099999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.716,"15min":1.367},"min":{"1min":0.46300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.0880000000000001,"5min":7.9470000000000001,"15min":7.9470000000000001},"last":2.2450000000000001},{"interface":"front","average":{"1min":1.4530000000000001,"5min":1.843,"15min":1.464},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.4809999999999999,"5min":13.893000000000001,"15min":13.893000000000001},"last":1.028}]},{"osd":7,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":2.0459999999999998,"15min":1.5589999999999999},"min":{"1min":0.442,"5min":0.377,"15min":0.377},"max":{"1min":6.4359999999999999,"5min":13.848000000000001,"15min":13.848000000000001},"last":1.353},{"interface":"front","average":{"1min":1.23,"5min":1.5409999999999999,"15min":1.3260000000000001},"min":{"1min":0.44800000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.726,"5min":10.199,"15min":10.199},"last":1.909}]}]},{"osd":1,"up_from":607,"seq":2607045148881,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2310972,"kb_used_data":2143540,"kb_used_omap":107,"kb_used_meta":167316,"kb_avail":91434180,"statfs":{"total":95995035648,"available":93628600320,"internally_reserved":0,"allocated":2194984960,"data_stored":2193860197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":171332267},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":2.2090000000000001,"15min":1.1950000000000001},"min":{"1min":0.36399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":3.0310000000000001,"5min":84.421000000000006,"15min":84.421000000000006},"last":1.4550000000000001},{"interface":"front","average":{"1min":1.099,"5min":2.359,"15min":1.2969999999999999},"min":{"1min":0.215,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":7.8940000000000001,"5min":84.489999999999995,"15min":84.489999999999995},"last":1.347}]},{"osd":2,"last update":"Mon Oct 9 09:31:37 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":3.1669999999999998,"15min":1.6160000000000001},"min":{"1min":0.33500000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.5140000000000002,"5min":84.578999999999994,"15min":84.578999999999994},"last":1.3839999999999999},{"interface":"front","average":{"1min":1.3029999999999999,"5min":2.6309999999999998,"15min":1.4910000000000001},"min":{"1min":0.31,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":11.75,"5min":84.697000000000003,"15min":84.697000000000003},"last":0.92400000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:31:00 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":5.2069999999999999,"15min":2.29},"min":{"1min":0.53100000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.175,"5min":107.426,"15min":107.426},"last":1.6970000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":5.7439999999999998,"15min":2.5009999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.2229999999999999,"5min":106.15900000000001,"15min":106.15900000000001},"last":1.8049999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":2.242,"15min":1.452},"min":{"1min":0.38600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.032,"5min":84.875,"15min":84.875},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.97999999999999998,"5min":2.327,"15min":1.4890000000000001},"min":{"1min":0.31900000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.2720000000000002,"5min":84.992999999999995,"15min":84.992999999999995},"last":0.38100000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":2.274,"15min":1.4990000000000001},"min":{"1min":0.39900000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.569,"5min":84.805000000000007,"15min":84.805000000000007},"last":1.0349999999999999},{"interface":"front","average":{"1min":1.1020000000000001,"5min":2.5590000000000002,"15min":1.599},"min":{"1min":0.503,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.319,"5min":84.951999999999998,"15min":84.951999999999998},"last":0.34899999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":2.7759999999999998,"15min":1.645},"min":{"1min":0.434,"5min":0.379,"15min":0.379},"max":{"1min":5.8760000000000003,"5min":85.016000000000005,"15min":85.016000000000005},"last":1.411},{"interface":"front","average":{"1min":0.92000000000000004,"5min":2.4100000000000001,"15min":1.573},"min":{"1min":0.54500000000000004,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.5229999999999999,"5min":84.831999999999994,"15min":84.831999999999994},"last":1.0580000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":3.2589999999999999,"15min":1.859},"min":{"1min":0.33400000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.7970000000000006,"5min":84.888000000000005,"15min":84.888000000000005},"last":1.4430000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":2.4790000000000001,"15min":1.6399999999999999},"min":{"1min":0.48799999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.2909999999999999,"5min":84.742000000000004,"15min":84.742000000000004},"last":1.012}]}]},{"osd":0,"up_from":581,"seq":2495375999190,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3060872,"kb_used_data":2856896,"kb_used_omap":131,"kb_used_meta":203836,"kb_avail":90684280,"statfs":{"total":95995035648,"available":92860702720,"internally_reserved":0,"allocated":2925461504,"data_stored":2924379099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":208728665},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:31:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.946,"15min":1.141},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":5.452,"5min":14.44,"15min":14.44},"last":0.91900000000000004},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.9930000000000001,"15min":1.206},"min":{"1min":0.27300000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.3199999999999998,"5min":14.663,"15min":14.663},"last":0.94999999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:31:45 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":2.9199999999999999,"15min":1.6120000000000001},"min":{"1min":0.32700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":9.2119999999999997,"5min":60.151000000000003,"15min":60.151000000000003},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.0169999999999999,"5min":2.3410000000000002,"15min":1.3620000000000001},"min":{"1min":0.249,"5min":0.21099999999999999,"15min":0.20999999999999999},"max":{"1min":5.0259999999999998,"5min":59.253,"15min":59.253},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:31:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":3.2629999999999999,"15min":1.647},"min":{"1min":0.36599999999999999,"5min":0.252,"15min":0.252},"max":{"1min":3.7749999999999999,"5min":122.749,"15min":122.749},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.306,"5min":3.3759999999999999,"15min":1.746},"min":{"1min":0.61199999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.0549999999999997,"5min":123.51900000000001,"15min":123.51900000000001},"last":1.5369999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:30:57 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.4970000000000001,"15min":1.1830000000000001},"min":{"1min":0.503,"5min":0.312,"15min":0.312},"max":{"1min":2.5569999999999999,"5min":7.093,"15min":7.093},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.6839999999999999,"15min":1.226},"min":{"1min":0.435,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.7749999999999999,"5min":8.7680000000000007,"15min":8.7680000000000007},"last":0.65700000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:30:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.484,"15min":1.2110000000000001},"min":{"1min":0.53500000000000003,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.629,"5min":7.3620000000000001,"15min":7.3620000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.075,"5min":1.792,"15min":1.306},"min":{"1min":0.47399999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.7649999999999999,"5min":8.8529999999999998,"15min":8.8529999999999998},"last":0.60899999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:30:57 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":1.952,"15min":1.407},"min":{"1min":0.55600000000000005,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.2690000000000001,"5min":8.202,"15min":8.202},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.2669999999999999,"5min":1.6479999999999999,"15min":1.3120000000000001},"min":{"1min":0.629,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.5960000000000001,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":0.55900000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:30:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.9590000000000001,"15min":1.446},"min":{"1min":0.57099999999999995,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":2.2229999999999999,"5min":8.1769999999999996,"15min":8.1769999999999996},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.266,"5min":1.865,"15min":1.4199999999999999},"min":{"1min":0.58899999999999997,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.6640000000000001,"5min":10.571,"15min":10.571},"last":0.623}]}]},{"osd":3,"up_from":663,"seq":2847563317446,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3528572,"kb_used_data":3234228,"kb_used_omap":55,"kb_used_meta":294280,"kb_avail":90216580,"statfs":{"total":95995035648,"available":92381777920,"internally_reserved":0,"allocated":3311849472,"data_stored":3310833229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":301343330},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.8979999999999999,"15min":1.1599999999999999},"min":{"1min":0.219,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.7970000000000002,"5min":10.395,"15min":10.395},"last":3.726},{"interface":"front","average":{"1min":1.8460000000000001,"5min":2.1309999999999998,"15min":1.2310000000000001},"min":{"1min":0.47599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":9.9969999999999999,"5min":11.507999999999999,"15min":11.507999999999999},"last":7.6079999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":2.2069999999999999,"15min":1.3009999999999999},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.575,"5min":14.206,"15min":14.206},"last":6.633},{"interface":"front","average":{"1min":1.0049999999999999,"5min":2.2269999999999999,"15min":1.3049999999999999},"min":{"1min":0.42399999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.3719999999999999,"5min":10.433,"15min":10.433},"last":3.6920000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":2.6000000000000001,"15min":1.4370000000000001},"min":{"1min":0.27100000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.3789999999999996,"5min":10.363,"15min":10.363},"last":3.8769999999999998},{"interface":"front","average":{"1min":1.246,"5min":2.3050000000000002,"15min":1.4039999999999999},"min":{"1min":0.442,"5min":0.223,"15min":0.223},"max":{"1min":3.2549999999999999,"5min":13.856,"15min":13.856},"last":0.76900000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.867,"15min":1.3160000000000001},"min":{"1min":0.63200000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.246,"5min":12.629,"15min":12.629},"last":6.6840000000000002},{"interface":"front","average":{"1min":1.25,"5min":1.7150000000000001,"15min":1.2709999999999999},"min":{"1min":0.67400000000000004,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.1869999999999998,"5min":7.827,"15min":7.827},"last":0.36199999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":2.2090000000000001,"15min":1.4850000000000001},"min":{"1min":0.51100000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.4199999999999999,"5min":11.272,"15min":11.272},"last":3.8500000000000001},{"interface":"front","average":{"1min":0.92300000000000004,"5min":2.0219999999999998,"15min":1.425},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.4670000000000001,"5min":11.388999999999999,"15min":11.388999999999999},"last":6.7469999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.9830000000000001,"15min":1.4179999999999999},"min":{"1min":0.70499999999999996,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.2349999999999999,"5min":8.2129999999999992,"15min":8.2129999999999992},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.996,"5min":2.1459999999999999,"15min":1.4870000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":2.6640000000000001,"5min":17.234999999999999,"15min":17.234999999999999},"last":6.7160000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":2.1200000000000001,"15min":1.5009999999999999},"min":{"1min":0.54700000000000004,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.452,"5min":7.8220000000000001,"15min":7.8220000000000001},"last":3.8050000000000002},{"interface":"front","average":{"1min":1.3380000000000001,"5min":1.887,"15min":1.45},"min":{"1min":0.66300000000000003,"5min":0.318,"15min":0.318},"max":{"1min":3.2170000000000001,"5min":7.5869999999999997,"15min":7.5869999999999997},"last":0.39800000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986579,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2044432,"kb_used_data":1743112,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91700720,"statfs":{"total":95995035648,"available":93901537280,"internally_reserved":0,"allocated":1784946688,"data_stored":1784733096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:31:44 2023","interfaces":[{"interface":"back","average":{"1min":5.5469999999999997,"5min":2.1960000000000002,"15min":1.3280000000000001},"min":{"1min":0.25800000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":94.489999999999995,"5min":94.489999999999995,"15min":94.489999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":5.9960000000000004,"5min":2.125,"15min":1.3520000000000001},"min":{"1min":0.28299999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":94.802999999999997,"5min":94.802999999999997,"15min":94.802999999999997},"last":0.59099999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:31:16 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.4610000000000001,"15min":1.0980000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.0270000000000001,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.3919999999999999,"15min":1.109},"min":{"1min":0.30599999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.7130000000000001,"5min":7.6589999999999998,"15min":7.6589999999999998},"last":0.78200000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:31:31 2023","interfaces":[{"interface":"back","average":{"1min":5.415,"5min":2.5750000000000002,"15min":1.5109999999999999},"min":{"1min":0.33600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":99.736000000000004,"5min":99.736000000000004,"15min":99.736000000000004},"last":1.0920000000000001},{"interface":"front","average":{"1min":5.5119999999999996,"5min":2.2490000000000001,"15min":1.403},"min":{"1min":0.32500000000000001,"5min":0.317,"15min":0.317},"max":{"1min":99.212999999999994,"5min":99.212999999999994,"15min":99.212999999999994},"last":0.39100000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:31:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":2.1989999999999998,"15min":1.407},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.7509999999999999,"5min":17.376000000000001,"15min":17.376000000000001},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.0720000000000001,"5min":2.2269999999999999,"15min":1.48},"min":{"1min":0.36399999999999999,"5min":0.314,"15min":0.314},"max":{"1min":3.2850000000000001,"5min":17.277999999999999,"15min":17.277999999999999},"last":2.4609999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:31:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.506,"15min":1.149},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.0950000000000002,"5min":9.2959999999999994,"15min":9.2959999999999994},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.534,"15min":1.2669999999999999},"min":{"1min":0.40899999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.169,"5min":7.5819999999999999,"15min":7.5819999999999999},"last":0.48799999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.3999999999999999,"15min":1.1739999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.8319999999999999,"5min":7.625,"15min":7.625},"last":0.432},{"interface":"front","average":{"1min":1.042,"5min":1.4299999999999999,"15min":1.202},"min":{"1min":0.40799999999999997,"5min":0.371,"15min":0.371},"max":{"1min":3.7730000000000001,"5min":7.1440000000000001,"15min":7.1440000000000001},"last":0.40000000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:31:36 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":1.335,"15min":1.1439999999999999},"min":{"1min":0.40400000000000003,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.827,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.753,"5min":1.5549999999999999,"15min":1.3089999999999999},"min":{"1min":0.379,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.012,"5min":9.1780000000000008,"15min":9.1780000000000008},"last":0.52500000000000002}]}]},{"osd":5,"up_from":37,"seq":158913790345,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1694912,"kb_used_data":1389496,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92050240,"statfs":{"total":95995035648,"available":94259445760,"internally_reserved":0,"allocated":1422843904,"data_stored":1422642440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:31:43 2023","interfaces":[{"interface":"back","average":{"1min":4.1520000000000001,"5min":2.0800000000000001,"15min":1.3200000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":59.258000000000003,"5min":59.258000000000003,"15min":59.258000000000003},"last":0.38200000000000001},{"interface":"front","average":{"1min":4.4950000000000001,"5min":2.2200000000000002,"15min":1.389},"min":{"1min":0.30099999999999999,"5min":0.246,"15min":0.246},"max":{"1min":59.432000000000002,"5min":59.432000000000002,"15min":59.432000000000002},"last":0.47999999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:31:19 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.401,"15min":1.1200000000000001},"min":{"1min":0.45600000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.0659999999999998,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":1.7529999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.502,"15min":1.22},"min":{"1min":0.498,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":3.0920000000000001,"5min":8.7210000000000001,"15min":8.7210000000000001},"last":1.788}]},{"osd":2,"last update":"Mon Oct 9 09:31:40 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.8999999999999999,"15min":1.3220000000000001},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.024,"5min":14.019,"15min":14.019},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.375,"5min":2.2250000000000001,"15min":1.448},"min":{"1min":0.34999999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5839999999999996,"5min":14.037000000000001,"15min":14.037000000000001},"last":0.54800000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:30:57 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.917,"15min":1.288},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.0790000000000002,"5min":10.747,"15min":10.747},"last":1.5329999999999999},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.583,"15min":1.2529999999999999},"min":{"1min":0.39700000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.994,"5min":7.5579999999999998,"15min":7.5579999999999998},"last":0.67500000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:31:19 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.2909999999999999,"15min":1.05},"min":{"1min":0.34699999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.0680000000000001,"5min":10.747,"15min":10.747},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.3080000000000001,"15min":1.137},"min":{"1min":0.316,"5min":0.316,"15min":0.29799999999999999},"max":{"1min":3.9750000000000001,"5min":7.5780000000000003,"15min":7.5780000000000003},"last":0.61699999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:30:51 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.49,"15min":1.2010000000000001},"min":{"1min":0.44900000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.6020000000000001,"5min":14.518000000000001,"15min":14.518000000000001},"last":1.089},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.5640000000000001,"15min":1.351},"min":{"1min":0.55300000000000005,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.2189999999999999,"5min":8.0280000000000005,"15min":8.0280000000000005},"last":1.071}]},{"osd":7,"last update":"Mon Oct 9 09:31:23 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.671,"15min":1.3460000000000001},"min":{"1min":0.45100000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.8450000000000002,"5min":8.6359999999999992,"15min":8.6359999999999992},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.6890000000000001,"15min":1.319},"min":{"1min":0.45500000000000002,"5min":0.41699999999999998,"15min":0.41399999999999998},"max":{"1min":3.5430000000000001,"5min":7.8559999999999999,"15min":7.8559999999999999},"last":0.92200000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2946498560,"data_stored":2946498560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2203320320,"data_stored":2203320320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2378563584,"data_stored":2378563584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3335454720,"data_stored":3335454720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1789984768,"data_stored":1789984768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1421344768,"data_stored":1421344768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1851326464,"data_stored":1851326464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1853227008,"data_stored":1853227008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:31:58.933 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T09:31:58.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-10-09T09:31:59.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:59.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:59.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:59.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:59.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:59.168 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:31:59.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:59.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:31:59.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:31:59.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:59.309 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:31:59.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:59.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:31:59.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:59.459 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:59.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:59.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:59.548 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 96090 96074 77.9725 58.875 0.00684286 0.0128192 2023-10-09T09:31:59.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:59.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:31:59.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:59.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:31:59.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:59.728 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:31:59.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:59.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:31:59.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:59.840 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:31:59 smithi181 bash[59345]: cluster 2023-10-09T09:31:58.583493+0000 mon.a (mon.0) 1600 : cluster 3 Health check update: Degraded data redundancy: 4720/277773 objects degraded (1.699%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:31:59.856 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:31:59.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:59.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:31:59.959 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:59.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:31:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:31:59.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:00.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:00.101 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:31:59 smithi163 bash[75425]: cluster 2023-10-09T09:31:58.583493+0000 mon.a (mon.0) 1600 : cluster 3 Health check update: Degraded data redundancy: 4720/277773 objects degraded (1.699%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:00.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:31:59 smithi163 bash[78107]: cluster 2023-10-09T09:31:58.583493+0000 mon.a (mon.0) 1600 : cluster 3 Health check update: Degraded data redundancy: 4720/277773 objects degraded (1.699%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:00.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:00.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:00.225 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:00.260 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:00.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:00.366 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:00.399 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:00.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:00.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:00.506 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:00.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 97072 97056 77.7596 61.375 0.0206732 0.0128537 2023-10-09T09:32:00.549 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:00.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:00.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:00.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:00.664 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:00 smithi163 bash[75425]: cluster 2023-10-09T09:31:59.731235+0000 mgr.y (mgr.34107) 773 : cluster 0 pgmap v1490: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 4720/282123 objects degraded (1.673%); 3244/282123 objects misplaced (1.150%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:32:00.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:00 smithi163 bash[78107]: cluster 2023-10-09T09:31:59.731235+0000 mgr.y (mgr.34107) 773 : cluster 0 pgmap v1490: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 4720/282123 objects degraded (1.673%); 3244/282123 objects misplaced (1.150%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:32:00.692 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:00.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:00.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:00.797 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:00.819 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:00.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:00.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:00.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:00.930 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:00 smithi181 bash[59345]: cluster 2023-10-09T09:31:59.731235+0000 mgr.y (mgr.34107) 773 : cluster 0 pgmap v1490: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 4720/282123 objects degraded (1.673%); 3244/282123 objects misplaced (1.150%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:32:00.950 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:00.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:00.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:01.053 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:01.084 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:01.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:01.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:01.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:01.217 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:01.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:01.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:01.323 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:01.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:01.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:01.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:01.491 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:01.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:01.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:01.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 98088 98072 77.5791 63.5 0.00689001 0.0128847 2023-10-09T09:32:01.595 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:01.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:01.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:01.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:01.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:01 smithi163 bash[75425]: cluster 2023-10-09T09:31:59.846536+0000 osd.6 (osd.6) 392 : cluster 0 7.7 scrub starts 2023-10-09T09:32:01.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:01 smithi163 bash[78107]: cluster 2023-10-09T09:31:59.846536+0000 osd.6 (osd.6) 392 : cluster 0 7.7 scrub starts 2023-10-09T09:32:01.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:01.755 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:01.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:01.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:01.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:01.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:01.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:01.994 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:02.004 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:01 smithi181 bash[59345]: cluster 2023-10-09T09:31:59.846536+0000 osd.6 (osd.6) 392 : cluster 0 7.7 scrub starts 2023-10-09T09:32:02.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:02.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:02.128 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:02.154 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:02.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:02.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:02.259 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:02.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:02.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:02.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:02.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:02.433 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:02.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:02.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:02.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:02.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:32:02.549640+0000 min lat: 0.00155566 max lat: 2.40866 avg lat: 0.0129168 2023-10-09T09:32:02.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:32:02.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 99081 99065 77.3851 62.0625 0.0216884 0.0129168 2023-10-09T09:32:02.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:02.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:02.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:02.579 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:32:02.638 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:32:02.638 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:32:02.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:32:02.638 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:32:02.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.638 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:40102 6.0 6:0046ae12:::benchmark_data_smithi163_107771_object40101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1082)", 2023-10-09T09:32:02.638 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:26:06.002111+0000", 2023-10-09T09:32:02.638 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 356.62932322500001, 2023-10-09T09:32:02.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14278279899999999, 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 40102 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.002111+0000", 2023-10-09T09:32:02.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.002111+0000", 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.002110+0000", 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.002133+0000", 2023-10-09T09:32:02.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3102999999999999e-05 2023-10-09T09:32:02.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.002148+0000", 2023-10-09T09:32:02.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4799e-05 2023-10-09T09:32:02.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.002156+0000", 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3520000000000007e-06 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.002197+0000", 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0750000000000001e-05 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.002361+0000", 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000164267 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.002462+0000", 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010043900000000001 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.007026+0000", 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045643649999999999 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.007218+0000", 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019220399999999999 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.144601+0000", 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13738299400000001 2023-10-09T09:32:02.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.144844+0000", 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000242334 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:26:06.144894+0000", 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0445000000000002e-05 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.646 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.646 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:122973 6.0 6:049fe7c3:::benchmark_data_smithi163_107771_object12213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1087)", 2023-10-09T09:32:02.646 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.766733+0000", 2023-10-09T09:32:02.646 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 272.86470143000003, 2023-10-09T09:32:02.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.131279022, 2023-10-09T09:32:02.646 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.647 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.647 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.647 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.647 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.647 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 122973 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.766733+0000", 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.766733+0000", 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.766732+0000", 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.766737+0000", 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2159999999999997e-06 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.766748+0000", 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0506000000000001e-05 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.766754+0000", 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8210000000000004e-06 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.766824+0000", 2023-10-09T09:32:02.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9573999999999995e-05 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.766939+0000", 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115499 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.767038+0000", 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8564e-05 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.773019+0000", 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0059806440000000002 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.775784+0000", 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027655100000000001 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.897119+0000", 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.121334596 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.897571+0000", 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000452693 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.898012+0000", 2023-10-09T09:32:02.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00044086 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:122981 6.0 6:0731e430:::benchmark_data_smithi163_107771_object12221:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1087)", 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:27:29.776374+0000", 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 272.85506059699998, 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12681637400000001, 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.654 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 122981 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776374+0000", 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776374+0000", 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776373+0000", 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776379+0000", 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6359999999999999e-06 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776387+0000", 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9679999999999995e-06 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776393+0000", 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5729999999999998e-06 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776419+0000", 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5650999999999999e-05 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776501+0000", 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2140999999999998e-05 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.776572+0000", 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1555000000000002e-05 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.780782+0000", 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042091139999999999 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.786803+0000", 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006021116 2023-10-09T09:32:02.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.898507+0000", 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.111703888 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.902987+0000", 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0044808729999999998 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:27:29.903190+0000", 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020306600000000001 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:181346 6.0 6:0e5ecb67:::benchmark_data_smithi163_107771_object70586:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:28:51.047077+0000", 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 191.58435750800001, 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.17454299600000001, 2023-10-09T09:32:02.661 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181346 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.047077+0000", 2023-10-09T09:32:02.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.047077+0000", 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.047076+0000", 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.047082+0000", 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.032e-06 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.047093+0000", 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1498000000000001e-05 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.047101+0000", 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1910000000000003e-06 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.047230+0000", 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000129639 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.047546+0000", 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00031593600000000001 2023-10-09T09:32:02.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.050971+0000", 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003424374 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.060072+0000", 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091019050000000004 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062114+0000", 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002041429 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.221227+0000", 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15911357100000001 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.221573+0000", 2023-10-09T09:32:02.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00034587000000000002 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.221620+0000", 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6887e-05 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:181353 6.0 6:0e22b7a1:::benchmark_data_smithi163_107771_object70593:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:32:02.668 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:28:51.062715+0000", 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 191.56871931800001, 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.17166983, 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181353 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062715+0000", 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062715+0000", 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062714+0000", 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062719+0000", 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5720000000000002e-06 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062728+0000", 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2349999999999999e-06 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062734+0000", 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5189999999999998e-06 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062801+0000", 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7327000000000001e-05 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062985+0000", 2023-10-09T09:32:02.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000184093 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.063125+0000", 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013976700000000001 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.072985+0000", 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0098600949999999993 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.072999+0000", 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3534e-05 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.234299+0000", 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16130000799999999 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.234347+0000", 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8513000000000001e-05 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.234385+0000", 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7818999999999998e-05 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.675 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:181354 6.0 6:06bd7602:::benchmark_data_smithi163_107771_object70594:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:28:51.062851+0000", 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 191.56858377099999, 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.17617074699999999, 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181354 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062851+0000", 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062851+0000", 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062850+0000", 2023-10-09T09:32:02.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062854+0000", 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0629999999999999e-06 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062862+0000", 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3640000000000001e-06 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.062868+0000", 2023-10-09T09:32:02.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9279999999999998e-06 2023-10-09T09:32:02.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.068885+0000", 2023-10-09T09:32:02.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060169280000000004 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.069038+0000", 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000153778 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.069208+0000", 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016981900000000001 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.073074+0000", 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038660830000000002 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.075476+0000", 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00240149 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.238884+0000", 2023-10-09T09:32:02.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16340782000000001 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.238976+0000", 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2121999999999997e-05 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.239022+0000", 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6001999999999998e-05 2023-10-09T09:32:02.682 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:207580 6.0 6:01a0aa8d:::benchmark_data_smithi163_107771_object96820:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:28.788110+0000", 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.84332436, 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12981575100000001, 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.683 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207580 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.788110+0000", 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.788110+0000", 2023-10-09T09:32:02.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.788109+0000", 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.788116+0000", 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8970000000000002e-06 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.788128+0000", 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.187e-05 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.788136+0000", 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6370000000000005e-06 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.789924+0000", 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017880459999999999 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.790034+0000", 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000109774 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.790155+0000", 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120735 2023-10-09T09:32:02.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.795299+0000", 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051445240000000001 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.852515+0000", 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.057215676 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.917699+0000", 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.065183960999999999 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.917827+0000", 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000128057 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:28.917926+0000", 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8978e-05 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:208019 6.0 6:026de415:::benchmark_data_smithi163_107771_object97259:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:29.469890+0000", 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.161544803, 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14282489600000001, 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.690 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 208019 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.469890+0000", 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.469890+0000", 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.469888+0000", 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.469895+0000", 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3270000000000003e-06 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.469908+0000", 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2612000000000001e-05 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.469916+0000", 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5220000000000001e-06 2023-10-09T09:32:02.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.469952+0000", 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6310000000000003e-05 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.470083+0000", 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013155 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.470205+0000", 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121833 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.478978+0000", 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0087729729999999999 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.488223+0000", 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0092443820000000006 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.612627+0000", 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.124404657 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.612676+0000", 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8792000000000003e-05 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.612715+0000", 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8680000000000002e-05 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:208021 6.0 6:0b763fdb:::benchmark_data_smithi163_107771_object97261:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:29.474753+0000", 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.15668167300001, 2023-10-09T09:32:02.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14694073199999999, 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 208021 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.474753+0000", 2023-10-09T09:32:02.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.474753+0000", 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.474751+0000", 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.474759+0000", 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6839999999999996e-06 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.474771+0000", 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2204999999999999e-05 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.474780+0000", 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2570000000000007e-06 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.474831+0000", 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0890999999999998e-05 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.475004+0000", 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017393199999999999 2023-10-09T09:32:02.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.475148+0000", 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000143611 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.478982+0000", 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003834273 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.488242+0000", 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0092598130000000004 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.621564+0000", 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13332233700000001 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.621651+0000", 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6493e-05 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.621694+0000", 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2783e-05 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:208027 6.0 6:06306824:::benchmark_data_smithi163_107771_object97267:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:32:02.704 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:29.483924+0000", 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.147510995, 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.138403889, 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 208027 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.483924+0000", 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.483924+0000", 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.483922+0000", 2023-10-09T09:32:02.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.483929+0000", 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1469999999999998e-06 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.483936+0000", 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.542e-06 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.483941+0000", 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3249999999999998e-06 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.483974+0000", 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2629000000000002e-05 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.484096+0000", 2023-10-09T09:32:02.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012201699999999999 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.484226+0000", 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000130257 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.505018+0000", 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020791799 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.505076+0000", 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8022000000000001e-05 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.622136+0000", 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.117059863 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.622291+0000", 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000155129 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.622328+0000", 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6427000000000002e-05 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:208029 6.0 6:09f8ec72:::benchmark_data_smithi163_107771_object97269:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:32:02.711 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:29.484511+0000", 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.14692410200001, 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.138530924, 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 208029 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.484511+0000", 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.484511+0000", 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.484510+0000", 2023-10-09T09:32:02.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.484515+0000", 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.626e-06 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.484526+0000", 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0995999999999999e-05 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.484534+0000", 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2309999999999999e-06 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.488264+0000", 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037306700000000002 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.488374+0000", 2023-10-09T09:32:02.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010975 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.488444+0000", 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9975999999999995e-05 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.505058+0000", 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016614522 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.505082+0000", 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3258e-05 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.622576+0000", 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11749382999999999 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.622761+0000", 2023-10-09T09:32:02.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000185336 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:29.623042+0000", 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00028067100000000002 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:209492 6.0 6:096fcc49:::benchmark_data_smithi163_107771_object98732:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:32:02.718 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:31.688958+0000", 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 150.942476339, 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15794950499999999, 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 209492 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.719 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.688958+0000", 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.688958+0000", 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.688957+0000", 2023-10-09T09:32:02.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.688963+0000", 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7830000000000004e-06 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.688975+0000", 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1858000000000001e-05 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.688983+0000", 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1070000000000005e-06 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.689032+0000", 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9608999999999999e-05 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.689144+0000", 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111641 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.689252+0000", 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010788700000000001 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.693685+0000", 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.004432783 2023-10-09T09:32:02.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.694854+0000", 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011690750000000001 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.846713+0000", 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.151859612 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.846834+0000", 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012033599999999999 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.846908+0000", 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4128000000000006e-05 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:209495 6.9 6:9aecbb5a:::benchmark_data_smithi163_107771_object98735:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:31.690300+0000", 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 150.94113425099999, 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.171076741, 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 209495 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.726 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.690300+0000", 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.690300+0000", 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.690300+0000", 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.690305+0000", 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0810000000000001e-06 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.690314+0000", 2023-10-09T09:32:02.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5599999999999999e-06 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.690320+0000", 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7849999999999997e-06 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.690362+0000", 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1757e-05 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.690466+0000", 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000104082 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.690567+0000", 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101458 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.704042+0000", 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013474440000000001 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.707749+0000", 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003707634 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.861032+0000", 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.153282945 2023-10-09T09:32:02.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.861208+0000", 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017573999999999999 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.861377+0000", 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016912900000000001 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:209496 6.9 6:929046fc:::benchmark_data_smithi163_107771_object98736:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:31.693744+0000", 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 150.93769095100001, 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16956861100000001, 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 209496 2023-10-09T09:32:02.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.693744+0000", 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.693744+0000", 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.693742+0000", 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.693750+0000", 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4989999999999997e-06 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.693763+0000", 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3295999999999999e-05 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.693771+0000", 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7689999999999996e-06 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.695431+0000", 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001660798 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.695549+0000", 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011726399999999999 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.698962+0000", 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034127699999999999 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.707754+0000", 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0087924590000000007 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.710074+0000", 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023200970000000001 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.863163+0000", 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15308915000000001 2023-10-09T09:32:02.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.863267+0000", 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010374600000000001 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.863312+0000", 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5385e-05 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.739 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:209502 6.9 6:9cdd4d1a:::benchmark_data_smithi163_107771_object98742:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:31.701288+0000", 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 150.93014652599999, 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16218590799999999, 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 209502 2023-10-09T09:32:02.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.701288+0000", 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.701288+0000", 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.701287+0000", 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.701292+0000", 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8429999999999998e-06 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.701305+0000", 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2870999999999999e-05 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.701313+0000", 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0709999999999995e-06 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.704058+0000", 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027441340000000001 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.704152+0000", 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4747000000000006e-05 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.704234+0000", 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1601999999999996e-05 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.707917+0000", 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036826269999999999 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.710101+0000", 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002183979 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.863359+0000", 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15325867800000001 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.863439+0000", 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9852999999999996e-05 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.863474+0000", 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5037999999999999e-05 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:60048 7.4 7:211c29bc:::benchmark_data_smithi163_130508_object60047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1103)", 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:19.332436+0000", 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 43.298998791000002, 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.140365973, 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 60048 2023-10-09T09:32:02.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.332436+0000", 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.332436+0000", 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.332435+0000", 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.332455+0000", 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9488999999999999e-05 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.332462+0000", 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0690000000000004e-06 2023-10-09T09:32:02.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.332467+0000", 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3519999999999997e-06 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.340847+0000", 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0083796719999999995 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.340974+0000", 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127272 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4", 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.341108+0000", 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013391499999999999 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.345742+0000", 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046335550000000001 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.348620+0000", 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028778330000000002 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.357037+0000", 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0084175889999999996 2023-10-09T09:32:02.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.472688+0000", 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.115650925 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.472802+0000", 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011374099999999999 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.753 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:60063 7.4 7:2883d490:::benchmark_data_smithi163_130508_object60062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1103)", 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:19.348827+0000", 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 43.282607655, 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12900610800000001, 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:32:02.754 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 60063 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.348827+0000", 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.348827+0000", 2023-10-09T09:32:02.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.348826+0000", 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.348856+0000", 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0351000000000001e-05 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.348866+0000", 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8099999999999992e-06 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.348874+0000", 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4170000000000003e-06 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.349047+0000", 2023-10-09T09:32:02.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017362700000000001 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.349180+0000", 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013318800000000001 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4", 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.349319+0000", 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000139171 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.475741+0000", 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.126421335 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.477736+0000", 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019950979999999998 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.477744+0000", 2023-10-09T09:32:02.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.089e-06 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.477794+0000", 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0266000000000003e-05 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.477833+0000", 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8872999999999999e-05 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.760 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:60067 7.4 7:228ee863:::benchmark_data_smithi163_130508_object60066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1103)", 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:19.351388+0000", 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 43.280047078000003, 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12776624, 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:32:02.761 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 60067 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351388+0000", 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351388+0000", 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351386+0000", 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351410+0000", 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4134999999999998e-05 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351422+0000", 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.151e-05 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351430+0000", 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9529999999999999e-06 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.472883+0000", 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12145265299999999 2023-10-09T09:32:02.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.473033+0000", 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015042899999999999 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4", 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.473198+0000", 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016490600000000001 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.477846+0000", 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046479069999999997 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.478469+0000", 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00062305399999999995 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.479091+0000", 2023-10-09T09:32:02.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00062207099999999997 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.479128+0000", 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6927000000000001e-05 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.479154+0000", 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5976999999999999e-05 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.767 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:60070 7.6 7:698fcc54:::benchmark_data_smithi163_130508_object60069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1103)", 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:19.351694+0000", 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 43.279740425, 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12730414300000001, 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:32:02.768 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 60070 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351694+0000", 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351694+0000", 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351693+0000", 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351716+0000", 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2439e-05 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351727+0000", 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1104000000000001e-05 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351735+0000", 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3760000000000008e-06 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351779+0000", 2023-10-09T09:32:02.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3754000000000002e-05 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351897+0000", 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011793700000000001 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.352077+0000", 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018025499999999999 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.478786+0000", 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.126708298 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.478811+0000", 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5893000000000001e-05 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.478933+0000", 2023-10-09T09:32:02.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121853 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.478969+0000", 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6022000000000002e-05 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.478998+0000", 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9101000000000001e-05 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.774 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:60071 7.6 7:6fa6a402:::benchmark_data_smithi163_130508_object60070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1103)", 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:19.351752+0000", 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 43.279682637999997, 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21393225799999999, 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:32:02.775 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 60071 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351752+0000", 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351752+0000", 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:32:02.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351752+0000", 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351772+0000", 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0650999999999999e-05 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:32:02.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351779+0000", 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.922e-06 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.351784+0000", 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2209999999999996e-06 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.472829+0000", 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12104424900000001 2023-10-09T09:32:02.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.473115+0000", 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00028631000000000002 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.473500+0000", 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000384705 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.479097+0000", 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005596976 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.479116+0000", 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9723000000000001e-05 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.565546+0000", 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.086429870000000006 2023-10-09T09:32:02.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.565643+0000", 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7046000000000006e-05 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:19.565684+0000", 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1105e-05 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.781 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.782 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:32:02.782 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:32:02.782 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:32:02.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:02.783 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:02 smithi163 bash[75425]: cluster 2023-10-09T09:32:01.731900+0000 mgr.y (mgr.34107) 774 : cluster 0 pgmap v1491: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s; 4720/288657 objects degraded (1.635%); 3244/288657 objects misplaced (1.124%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:32:02.783 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:02 smithi163 bash[78107]: cluster 2023-10-09T09:32:01.731900+0000 mgr.y (mgr.34107) 774 : cluster 0 pgmap v1491: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s; 4720/288657 objects degraded (1.635%); 3244/288657 objects misplaced (1.124%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:32:02.793 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:02 smithi181 bash[59345]: cluster 2023-10-09T09:32:01.731900+0000 mgr.y (mgr.34107) 774 : cluster 0 pgmap v1491: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s; 4720/288657 objects degraded (1.635%); 3244/288657 objects misplaced (1.124%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:32:02.809 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:02.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:02.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:02.935 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:02.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:02.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:03.039 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:03.064 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:03.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:03.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:03.169 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:03.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:03.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:03.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:03.296 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:03.323 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:03.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:03.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:03.429 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:03.457 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:03.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:03.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:03.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 100000 99984 77.1387 57.4375 0.0087911 0.012958 2023-10-09T09:32:03.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:03.607 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:03.611 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:32:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:32:03] "GET /metrics HTTP/1.1" 200 36347 "" "Prometheus/2.33.4" 2023-10-09T09:32:03.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:03.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:03.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:03.741 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:03.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:03.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:03.844 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:03.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:03.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:03.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:03 smithi163 bash[75425]: cluster 2023-10-09T09:32:03.732569+0000 mgr.y (mgr.34107) 775 : cluster 0 pgmap v1492: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 4720/293580 objects degraded (1.608%); 3244/293580 objects misplaced (1.105%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:32:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:03 smithi163 bash[75425]: cluster 2023-10-09T09:32:03.783694+0000 mon.a (mon.0) 1601 : cluster 3 Health check update: Degraded data redundancy: 4720/293580 objects degraded (1.608%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:03.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:03 smithi163 bash[78107]: cluster 2023-10-09T09:32:03.732569+0000 mgr.y (mgr.34107) 775 : cluster 0 pgmap v1492: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 4720/293580 objects degraded (1.608%); 3244/293580 objects misplaced (1.105%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:32:03.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:03 smithi163 bash[78107]: cluster 2023-10-09T09:32:03.783694+0000 mon.a (mon.0) 1601 : cluster 3 Health check update: Degraded data redundancy: 4720/293580 objects degraded (1.608%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:03.979 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:04.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:04.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:04.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:03 smithi181 bash[59345]: cluster 2023-10-09T09:32:03.732569+0000 mgr.y (mgr.34107) 775 : cluster 0 pgmap v1492: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 4720/293580 objects degraded (1.608%); 3244/293580 objects misplaced (1.105%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:32:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:03 smithi181 bash[59345]: cluster 2023-10-09T09:32:03.783694+0000 mon.a (mon.0) 1601 : cluster 3 Health check update: Degraded data redundancy: 4720/293580 objects degraded (1.608%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:04.113 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:04.145 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:04.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:04.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:04.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:04.277 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:32:04.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:04.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:04.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:04.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:04.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:04.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:04.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:04.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:04.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 100981 100965 76.9456 61.3125 0.0238397 0.0129903 2023-10-09T09:32:04.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:04.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:04.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:04.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:04.688 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:04.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:04.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:04.756 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:04.763 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:04.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:04.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:04.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:04.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:04.920 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:04.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:04.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:04.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:05.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:05.080 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:05.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:05.184 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:05.236 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:05.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:05.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:05.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:05.366 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:05.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:05.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:05.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:05.501 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:05.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:05.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:05.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 101966 101950 76.7602 61.5625 0.00366589 0.013021 2023-10-09T09:32:05.606 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:05.645 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:05.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:05.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:05.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:05.775 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:05.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:05.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:05.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:05.905 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:05.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:05.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:06.016 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:06.041 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:06.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:06.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:06.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:06.174 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:06.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:06.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:06.277 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:06.306 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:06.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:06.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:06.411 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:06.454 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:06.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:06.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:06.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 102917 102901 76.5539 59.4375 0.00721421 0.0130564 2023-10-09T09:32:06.561 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:06.586 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:06.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:06.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:06.689 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:06.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:06 smithi163 bash[75425]: cluster 2023-10-09T09:32:05.733165+0000 mgr.y (mgr.34107) 776 : cluster 0 pgmap v1493: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s; 4720/298515 objects degraded (1.581%); 2942/298515 objects misplaced (0.986%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:06.699 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:06 smithi163 bash[78107]: cluster 2023-10-09T09:32:05.733165+0000 mgr.y (mgr.34107) 776 : cluster 0 pgmap v1493: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s; 4720/298515 objects degraded (1.581%); 2942/298515 objects misplaced (0.986%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:06.720 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:06.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:06.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:06.823 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:06.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:06.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:06.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:06.962 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:06.990 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:06.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:06.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:06 smithi181 bash[59345]: cluster 2023-10-09T09:32:05.733165+0000 mgr.y (mgr.34107) 776 : cluster 0 pgmap v1493: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s; 4720/298515 objects degraded (1.581%); 2942/298515 objects misplaced (0.986%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:07.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:07.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:07.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:07.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:07.255 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:07.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:07.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:07.359 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:07.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:07.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:07.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:07.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:07.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:07.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:07.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 103876 103860 76.3584 59.9375 0.00888844 0.0130907 2023-10-09T09:32:07.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:07.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:07.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:07.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:07.751 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:07 smithi163 bash[78107]: cluster 2023-10-09T09:32:05.799235+0000 osd.6 (osd.6) 393 : cluster 0 7.7 scrub starts 2023-10-09T09:32:07.751 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:07 smithi163 bash[75425]: cluster 2023-10-09T09:32:05.799235+0000 osd.6 (osd.6) 393 : cluster 0 7.7 scrub starts 2023-10-09T09:32:07.767 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:07.796 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:07.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:07.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:07.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:07 smithi181 bash[59345]: cluster 2023-10-09T09:32:05.799235+0000 osd.6 (osd.6) 393 : cluster 0 7.7 scrub starts 2023-10-09T09:32:07.817 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:07.899 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:07.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:07.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:07.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:08.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:08.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:08.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:08.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:08.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:08.196 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:08.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:08.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:08.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:08.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:08.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:08.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:08.462 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:08.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:08.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:08.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 104809 104793 76.1485 58.3125 0.0122323 0.0131264 2023-10-09T09:32:08.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:08.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:08.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:08.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:08.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:08.739 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:08.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:08.840 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:08.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:08 smithi181 bash[59345]: cluster 2023-10-09T09:32:07.733902+0000 mgr.y (mgr.34107) 777 : cluster 0 pgmap v1494: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 4720/306756 objects degraded (1.539%); 2942/306756 objects misplaced (0.959%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:32:08.869 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:08.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:08.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:08 smithi163 bash[75425]: cluster 2023-10-09T09:32:07.733902+0000 mgr.y (mgr.34107) 777 : cluster 0 pgmap v1494: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 4720/306756 objects degraded (1.539%); 2942/306756 objects misplaced (0.959%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:32:08.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:08 smithi163 bash[78107]: cluster 2023-10-09T09:32:07.733902+0000 mgr.y (mgr.34107) 777 : cluster 0 pgmap v1494: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 4720/306756 objects degraded (1.539%); 2942/306756 objects misplaced (0.959%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:32:08.972 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:09.007 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:09.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:09.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:09.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:09.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:09.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:09.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:09.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:09.274 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:09.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:09.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:09.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:09.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:09.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:09.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:09.512 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:09.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 105785 105769 75.9742 61 0.0117596 0.0131566 2023-10-09T09:32:09.550 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:09.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:09.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:09.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:09.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:09.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:09.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:09.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:09.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:09.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:09.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:09.918 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:09.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:09.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:10.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:10.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:09 smithi181 bash[59345]: cluster 2023-10-09T09:32:09.734545+0000 mgr.y (mgr.34107) 778 : cluster 0 pgmap v1495: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s; 4720/311382 objects degraded (1.516%); 2606/311382 objects misplaced (0.837%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T09:32:10.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:09 smithi181 bash[59345]: cluster 2023-10-09T09:32:09.785772+0000 mon.a (mon.0) 1602 : cluster 3 Health check update: Degraded data redundancy: 4720/311382 objects degraded (1.516%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:10.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:10.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:10.181 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:10.208 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:10.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:10.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:10.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:09 smithi163 bash[75425]: cluster 2023-10-09T09:32:09.734545+0000 mgr.y (mgr.34107) 778 : cluster 0 pgmap v1495: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s; 4720/311382 objects degraded (1.516%); 2606/311382 objects misplaced (0.837%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T09:32:10.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:09 smithi163 bash[75425]: cluster 2023-10-09T09:32:09.785772+0000 mon.a (mon.0) 1602 : cluster 3 Health check update: Degraded data redundancy: 4720/311382 objects degraded (1.516%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:10.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:09 smithi163 bash[78107]: cluster 2023-10-09T09:32:09.734545+0000 mgr.y (mgr.34107) 778 : cluster 0 pgmap v1495: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s; 4720/311382 objects degraded (1.516%); 2606/311382 objects misplaced (0.837%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T09:32:10.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:09 smithi163 bash[78107]: cluster 2023-10-09T09:32:09.785772+0000 mon.a (mon.0) 1602 : cluster 3 Health check update: Degraded data redundancy: 4720/311382 objects degraded (1.516%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:10.311 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:10.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:10.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:10.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:10.440 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:10.560 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 106778 106762 75.8161 62.0625 0.011942 0.0131844 2023-10-09T09:32:10.592 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:10.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:10.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:10.726 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:10.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:10.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:10.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:10.866 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:10.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:10.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:10.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:10.999 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:11.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:11.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:11.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:11.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:11.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:11.239 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:11.281 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:11.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:11.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:11.348 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:11.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:11.409 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:11.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:11.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:11.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:11.521 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:11 smithi163 bash[78107]: cluster 2023-10-09T09:32:09.756875+0000 osd.6 (osd.6) 394 : cluster 0 7.7 scrub starts 2023-10-09T09:32:11.521 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:11 smithi163 bash[78107]: audit 2023-10-09T09:32:10.379098+0000 mon.a (mon.0) 1603 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:11.522 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:11 smithi163 bash[78107]: audit 2023-10-09T09:32:10.385738+0000 mon.a (mon.0) 1604 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:11.522 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:11 smithi163 bash[75425]: cluster 2023-10-09T09:32:09.756875+0000 osd.6 (osd.6) 394 : cluster 0 7.7 scrub starts 2023-10-09T09:32:11.522 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:11 smithi163 bash[75425]: audit 2023-10-09T09:32:10.379098+0000 mon.a (mon.0) 1603 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:11.522 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:11 smithi163 bash[75425]: audit 2023-10-09T09:32:10.385738+0000 mon.a (mon.0) 1604 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:11.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:11.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:11.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 107753 107737 75.6489 60.9375 0.014178 0.0132143 2023-10-09T09:32:11.651 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:11.692 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:11.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:11.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:11.796 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:11 smithi181 bash[59345]: cluster 2023-10-09T09:32:09.756875+0000 osd.6 (osd.6) 394 : cluster 0 7.7 scrub starts 2023-10-09T09:32:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:11 smithi181 bash[59345]: audit 2023-10-09T09:32:10.379098+0000 mon.a (mon.0) 1603 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:11 smithi181 bash[59345]: audit 2023-10-09T09:32:10.385738+0000 mon.a (mon.0) 1604 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:11.820 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:11.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:11.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:11.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:11.955 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:11.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:12.060 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:12.084 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:12.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:12.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:12.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:12.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:12.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:12.328 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:12.361 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:12.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:12.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:12.393 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:12 smithi163 bash[75425]: cluster 2023-10-09T09:32:11.735292+0000 mgr.y (mgr.34107) 779 : cluster 0 pgmap v1496: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 4720/317532 objects degraded (1.486%); 2606/317532 objects misplaced (0.821%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T09:32:12.393 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:12 smithi163 bash[75425]: cluster 2023-10-09T09:32:11.738451+0000 osd.6 (osd.6) 395 : cluster 0 7.7 scrub starts 2023-10-09T09:32:12.466 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:12.496 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:12.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:12.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 90 7 108660 108653 75.4444 57.25 0.011864 0.0132493 2023-10-09T09:32:12.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:12 smithi181 bash[59345]: cluster 2023-10-09T09:32:11.735292+0000 mgr.y (mgr.34107) 779 : cluster 0 pgmap v1496: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 4720/317532 objects degraded (1.486%); 2606/317532 objects misplaced (0.821%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T09:32:12.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:12 smithi181 bash[59345]: cluster 2023-10-09T09:32:11.738451+0000 osd.6 (osd.6) 395 : cluster 0 7.7 scrub starts 2023-10-09T09:32:12.601 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:12.622 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:12.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:12.724 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:12.755 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:12.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:12.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:12.823 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:12 smithi163 bash[78107]: cluster 2023-10-09T09:32:11.735292+0000 mgr.y (mgr.34107) 779 : cluster 0 pgmap v1496: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 4720/317532 objects degraded (1.486%); 2606/317532 objects misplaced (0.821%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T09:32:12.823 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:12 smithi163 bash[78107]: cluster 2023-10-09T09:32:11.738451+0000 osd.6 (osd.6) 395 : cluster 0 7.7 scrub starts 2023-10-09T09:32:12.860 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:12.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:12.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:12.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:12.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:13.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:13.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:13.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:13.132 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:13.152 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:13.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:13.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:13.254 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:13.286 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:13.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:13.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:13.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:13.419 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:13.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:13.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:13.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:32:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:32:13] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.33.4" 2023-10-09T09:32:13.522 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:13.542 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:13.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:13.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:13.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.0149 2023-10-09T09:32:13.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 108660 2023-10-09T09:32:13.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T09:32:13.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T09:32:13.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 75.4458 2023-10-09T09:32:13.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 32.831 2023-10-09T09:32:13.551 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 174 2023-10-09T09:32:13.551 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 0 2023-10-09T09:32:13.551 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1207 2023-10-09T09:32:13.551 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 525.305 2023-10-09T09:32:13.551 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 2784 2023-10-09T09:32:13.551 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 0 2023-10-09T09:32:13.551 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.0132496 2023-10-09T09:32:13.551 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0427069 2023-10-09T09:32:13.551 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 2.40866 2023-10-09T09:32:13.551 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.00155566 2023-10-09T09:32:13.559 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T09:32:13.645 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:13.679 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:13.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:13.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:13.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:13.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:13.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:13.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:13.906 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:13.932 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:13.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:13.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:14.038 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:14.063 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:14.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:14.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:14.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:14.193 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:14.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:14.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:14.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:14.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:14.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:14.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:14.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:14.454 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:14.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:14.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:14.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:14 smithi181 bash[59345]: cluster 2023-10-09T09:32:13.394727+0000 osd.5 (osd.5) 664 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:14 smithi181 bash[59345]: cluster 2023-10-09T09:32:13.396570+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e1115: 8 total, 8 up, 8 in 2023-10-09T09:32:14.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:14 smithi181 bash[59345]: cluster 2023-10-09T09:32:13.398349+0000 osd.5 (osd.5) 665 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:14 smithi181 bash[59345]: cluster 2023-10-09T09:32:13.408246+0000 osd.7 (osd.7) 507 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:14 smithi181 bash[59345]: cluster 2023-10-09T09:32:13.552619+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:14 smithi181 bash[59345]: cluster 2023-10-09T09:32:13.735839+0000 mgr.y (mgr.34107) 780 : cluster 0 pgmap v1498: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 4720/322134 objects degraded (1.465%); 2606/322134 objects misplaced (0.809%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T09:32:14.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[75425]: cluster 2023-10-09T09:32:13.394727+0000 osd.5 (osd.5) 664 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[75425]: cluster 2023-10-09T09:32:13.396570+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e1115: 8 total, 8 up, 8 in 2023-10-09T09:32:14.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[75425]: cluster 2023-10-09T09:32:13.398349+0000 osd.5 (osd.5) 665 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[75425]: cluster 2023-10-09T09:32:13.408246+0000 osd.7 (osd.7) 507 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[75425]: cluster 2023-10-09T09:32:13.552619+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[75425]: cluster 2023-10-09T09:32:13.735839+0000 mgr.y (mgr.34107) 780 : cluster 0 pgmap v1498: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 4720/322134 objects degraded (1.465%); 2606/322134 objects misplaced (0.809%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T09:32:14.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[78107]: cluster 2023-10-09T09:32:13.394727+0000 osd.5 (osd.5) 664 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[78107]: cluster 2023-10-09T09:32:13.396570+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e1115: 8 total, 8 up, 8 in 2023-10-09T09:32:14.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[78107]: cluster 2023-10-09T09:32:13.398349+0000 osd.5 (osd.5) 665 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[78107]: cluster 2023-10-09T09:32:13.408246+0000 osd.7 (osd.7) 507 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[78107]: cluster 2023-10-09T09:32:13.552619+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:14.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:14 smithi163 bash[78107]: cluster 2023-10-09T09:32:13.735839+0000 mgr.y (mgr.34107) 780 : cluster 0 pgmap v1498: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 4720/322134 objects degraded (1.465%); 2606/322134 objects misplaced (0.809%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T09:32:14.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:14.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:14.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:14.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:14.726 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:14.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:14.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:14.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:14.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:14.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:14.968 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:14.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:14.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:14.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:15.099 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:15.126 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:15.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:15.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:15.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:15.260 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:15.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:15.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:15.368 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:15.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:15.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:15.427 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[78107]: cluster 2023-10-09T09:32:13.564639+0000 osd.4 (osd.4) 542 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:15.427 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[78107]: cluster 2023-10-09T09:32:14.401465+0000 osd.5 (osd.5) 666 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.427 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[78107]: cluster 2023-10-09T09:32:14.401931+0000 osd.4 (osd.4) 543 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.427 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[78107]: cluster 2023-10-09T09:32:14.404396+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e1116: 8 total, 8 up, 8 in 2023-10-09T09:32:15.427 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[78107]: cluster 2023-10-09T09:32:14.418428+0000 osd.7 (osd.7) 508 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.427 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[78107]: cluster 2023-10-09T09:32:14.421783+0000 osd.6 (osd.6) 397 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.491 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:15.499 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:15 smithi181 bash[59345]: cluster 2023-10-09T09:32:13.564639+0000 osd.4 (osd.4) 542 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:15.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:15 smithi181 bash[59345]: cluster 2023-10-09T09:32:14.401465+0000 osd.5 (osd.5) 666 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:15 smithi181 bash[59345]: cluster 2023-10-09T09:32:14.401931+0000 osd.4 (osd.4) 543 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:15 smithi181 bash[59345]: cluster 2023-10-09T09:32:14.404396+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e1116: 8 total, 8 up, 8 in 2023-10-09T09:32:15.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:15 smithi181 bash[59345]: cluster 2023-10-09T09:32:14.418428+0000 osd.7 (osd.7) 508 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:15 smithi181 bash[59345]: cluster 2023-10-09T09:32:14.421783+0000 osd.6 (osd.6) 397 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.516 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:15.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:15.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:15.619 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:15.643 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:15.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:15.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[75425]: cluster 2023-10-09T09:32:13.564639+0000 osd.4 (osd.4) 542 : cluster 3 failed to encode map e1115 with expected crc 2023-10-09T09:32:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[75425]: cluster 2023-10-09T09:32:14.401465+0000 osd.5 (osd.5) 666 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[75425]: cluster 2023-10-09T09:32:14.401931+0000 osd.4 (osd.4) 543 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[75425]: cluster 2023-10-09T09:32:14.404396+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e1116: 8 total, 8 up, 8 in 2023-10-09T09:32:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[75425]: cluster 2023-10-09T09:32:14.418428+0000 osd.7 (osd.7) 508 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:15 smithi163 bash[75425]: cluster 2023-10-09T09:32:14.421783+0000 osd.6 (osd.6) 397 : cluster 3 failed to encode map e1116 with expected crc 2023-10-09T09:32:15.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:15.773 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:15.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:15.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:15.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:15.905 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:15.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:16.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:16.045 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:16.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:16.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:16.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:16.176 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:16.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:16.282 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:16.320 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:16.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:16.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:16.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:16.434 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:16 smithi181 bash[59345]: cluster 2023-10-09T09:32:15.569236+0000 mon.a (mon.0) 1607 : cluster 3 Health check update: Degraded data redundancy: 4720/322134 objects degraded (1.465%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:16.434 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:16 smithi181 bash[59345]: cluster 2023-10-09T09:32:15.736392+0000 mgr.y (mgr.34107) 781 : cluster 0 pgmap v1500: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 794 op/s; 2340/324390 objects degraded (0.721%); 4518/324390 objects misplaced (1.393%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:32:16.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:16.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:16.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:16.559 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:16.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:16.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:16.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:16.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:16 smithi163 bash[75425]: cluster 2023-10-09T09:32:15.569236+0000 mon.a (mon.0) 1607 : cluster 3 Health check update: Degraded data redundancy: 4720/322134 objects degraded (1.465%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:16.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:16 smithi163 bash[75425]: cluster 2023-10-09T09:32:15.736392+0000 mgr.y (mgr.34107) 781 : cluster 0 pgmap v1500: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 794 op/s; 2340/324390 objects degraded (0.721%); 4518/324390 objects misplaced (1.393%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:32:16.609 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:16 smithi163 bash[78107]: cluster 2023-10-09T09:32:15.569236+0000 mon.a (mon.0) 1607 : cluster 3 Health check update: Degraded data redundancy: 4720/322134 objects degraded (1.465%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:32:16.609 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:16 smithi163 bash[78107]: cluster 2023-10-09T09:32:15.736392+0000 mgr.y (mgr.34107) 781 : cluster 0 pgmap v1500: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 794 op/s; 2340/324390 objects degraded (0.721%); 4518/324390 objects misplaced (1.393%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:32:16.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:16.715 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:16.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:16.819 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:16.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:16.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:16.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:16.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:16.987 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:16.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:17.089 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:17.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:17.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:17.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:17.251 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:17.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:17.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:17.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:17.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:17.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:17.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:17.515 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:17.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:17.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:17.620 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:17.644 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:17.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:17.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:17.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:17.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:17.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:17.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:17.898 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:17.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:17.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:17.963 INFO:teuthology.orchestra.run.smithi163.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T09:32:17.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:17 smithi163 bash[75425]: cluster 2023-10-09T09:32:17.737132+0000 mgr.y (mgr.34107) 782 : cluster 0 pgmap v1501: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 38 MiB/s wr, 841 op/s; 2340/320631 objects degraded (0.730%); 4518/320631 objects misplaced (1.409%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:32:17.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:17 smithi163 bash[78107]: cluster 2023-10-09T09:32:17.737132+0000 mgr.y (mgr.34107) 782 : cluster 0 pgmap v1501: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 38 MiB/s wr, 841 op/s; 2340/320631 objects degraded (0.730%); 4518/320631 objects misplaced (1.409%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:32:18.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:18.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:18.039 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:18.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:18.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:18.176 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:18.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:18.283 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:18.291 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:17 smithi181 bash[59345]: cluster 2023-10-09T09:32:17.737132+0000 mgr.y (mgr.34107) 782 : cluster 0 pgmap v1501: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 38 MiB/s wr, 841 op/s; 2340/320631 objects degraded (0.730%); 4518/320631 objects misplaced (1.409%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:32:18.315 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:18.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:18.419 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:18.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:18.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:18.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:18.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:18.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:18.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:18.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:18.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:18.715 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:18.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:18.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:18.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:18.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:18.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:18.952 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:18 smithi163 bash[78107]: audit 2023-10-09T09:32:17.957825+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.163:0/3165420564' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T09:32:18.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:18 smithi163 bash[78107]: audit 2023-10-09T09:32:17.960392+0000 mon.a (mon.0) 1608 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T09:32:18.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:18 smithi163 bash[75425]: audit 2023-10-09T09:32:17.957825+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.163:0/3165420564' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T09:32:18.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:18 smithi163 bash[75425]: audit 2023-10-09T09:32:17.960392+0000 mon.a (mon.0) 1608 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T09:32:18.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:18.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:18.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:19.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:19.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:19.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:19.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:19.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:19.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:19.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:19.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:19.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:18 smithi181 bash[59345]: audit 2023-10-09T09:32:17.957825+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.163:0/3165420564' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T09:32:19.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:18 smithi181 bash[59345]: audit 2023-10-09T09:32:17.960392+0000 mon.a (mon.0) 1608 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T09:32:19.368 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:19.385 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:19.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:19.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:19.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:19.513 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:19.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:19.616 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:19.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:19.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:19.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:19.746 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:19.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:19.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:19.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:19.874 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:19.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:19.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:20.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:20.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:19 smithi181 bash[59345]: cluster 2023-10-09T09:32:19.739648+0000 mgr.y (mgr.34107) 783 : cluster 0 pgmap v1502: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 22 MiB/s wr, 770 op/s; 2340/316197 objects degraded (0.740%); 4267/316197 objects misplaced (1.349%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:32:20.059 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:19 smithi163 bash[78107]: cluster 2023-10-09T09:32:19.739648+0000 mgr.y (mgr.34107) 783 : cluster 0 pgmap v1502: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 22 MiB/s wr, 770 op/s; 2340/316197 objects degraded (0.740%); 4267/316197 objects misplaced (1.349%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:32:20.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:19 smithi163 bash[75425]: cluster 2023-10-09T09:32:19.739648+0000 mgr.y (mgr.34107) 783 : cluster 0 pgmap v1502: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 22 MiB/s wr, 770 op/s; 2340/316197 objects degraded (0.740%); 4267/316197 objects misplaced (1.349%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:32:20.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:20.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:20.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:20.165 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:20.191 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:20.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:20.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:20.245 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:32:20.294 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:20.312 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:32:20.324 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:20.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:20.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:20.360 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:32:20.398 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:32:20.427 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:20.435 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:32:20.459 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:20.463 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:32:20.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:20.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:20.499 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:32:20.527 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:32:20.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:32:20.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:32:20.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:32:20.530 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:32:20.530 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:32:20.530 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:32:20.530 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:32:20.530 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:32:20.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:20.593 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:20.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:20.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:20.696 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:20.730 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:20.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:20.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:20.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:20.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:20 smithi163 bash[78107]: audit 2023-10-09T09:32:20.182516+0000 mon.a (mon.0) 1609 : audit 0 from='client.? 172.21.15.163:0/1359494987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:32:20.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:20 smithi163 bash[78107]: cluster 2023-10-09T09:32:20.570588+0000 mon.a (mon.0) 1610 : cluster 3 Health check update: Degraded data redundancy: 2340/316197 objects degraded (0.740%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:20.860 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:20 smithi163 bash[75425]: audit 2023-10-09T09:32:20.182516+0000 mon.a (mon.0) 1609 : audit 0 from='client.? 172.21.15.163:0/1359494987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:32:20.860 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:20 smithi163 bash[75425]: cluster 2023-10-09T09:32:20.570588+0000 mon.a (mon.0) 1610 : cluster 3 Health check update: Degraded data redundancy: 2340/316197 objects degraded (0.740%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:20.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:20.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:20.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:20.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:20.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:20.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:21.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:21.131 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:21.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:21.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:21.234 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:21.243 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:20 smithi181 bash[59345]: audit 2023-10-09T09:32:20.182516+0000 mon.a (mon.0) 1609 : audit 0 from='client.? 172.21.15.163:0/1359494987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:32:21.244 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:20 smithi181 bash[59345]: cluster 2023-10-09T09:32:20.570588+0000 mon.a (mon.0) 1610 : cluster 3 Health check update: Degraded data redundancy: 2340/316197 objects degraded (0.740%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:21.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:21.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:21.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:21.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:21.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:21.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:21.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:21.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:21.551 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:21.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:21.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:21.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:21.699 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:21.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:21.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:21.802 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:21.827 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:21.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:21.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:21.932 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:21.960 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:21.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:21.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:22.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:22.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:21 smithi181 bash[59345]: cluster 2023-10-09T09:32:21.740305+0000 mgr.y (mgr.34107) 784 : cluster 0 pgmap v1503: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 122 B/s rd, 12 MiB/s wr, 836 op/s; 2340/310164 objects degraded (0.754%); 4267/310164 objects misplaced (1.376%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T09:32:22.083 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:22.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:22.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:22.185 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:22.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:21 smithi163 bash[75425]: cluster 2023-10-09T09:32:21.740305+0000 mgr.y (mgr.34107) 784 : cluster 0 pgmap v1503: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 122 B/s rd, 12 MiB/s wr, 836 op/s; 2340/310164 objects degraded (0.754%); 4267/310164 objects misplaced (1.376%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T09:32:22.193 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:21 smithi163 bash[78107]: cluster 2023-10-09T09:32:21.740305+0000 mgr.y (mgr.34107) 784 : cluster 0 pgmap v1503: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 122 B/s rd, 12 MiB/s wr, 836 op/s; 2340/310164 objects degraded (0.754%); 4267/310164 objects misplaced (1.376%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T09:32:22.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:22.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:22.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:22.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:22.354 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:22.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:22.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:22.459 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T09:32:22.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:22.487 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:22.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:22.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:22.591 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:22.616 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:22.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:22.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:22.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:22.747 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:22.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:22.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:22.855 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:22.885 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:22.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:22.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:22.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:23.012 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:23.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:23.118 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:23.143 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:23.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:23.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:23.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:23.272 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:23.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:23.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:23.358 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:32:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:32:23] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.33.4" 2023-10-09T09:32:23.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:23.402 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:23.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:23.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:23.509 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:23.541 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:23.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:23.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:23.650 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:23.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:23.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:23.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:23.784 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:23.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:23.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:23.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:23.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:23.948 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:23.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:23.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:23 smithi163 bash[78107]: cluster 2023-10-09T09:32:23.743530+0000 mgr.y (mgr.34107) 785 : cluster 0 pgmap v1504: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 8.3 MiB/s wr, 817 op/s; 2340/305544 objects degraded (0.766%); 4267/305544 objects misplaced (1.397%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:32:23.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:23 smithi163 bash[75425]: cluster 2023-10-09T09:32:23.743530+0000 mgr.y (mgr.34107) 785 : cluster 0 pgmap v1504: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 8.3 MiB/s wr, 817 op/s; 2340/305544 objects degraded (0.766%); 4267/305544 objects misplaced (1.397%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:32:24.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:24.064 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:23 smithi181 bash[59345]: cluster 2023-10-09T09:32:23.743530+0000 mgr.y (mgr.34107) 785 : cluster 0 pgmap v1504: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 8.3 MiB/s wr, 817 op/s; 2340/305544 objects degraded (0.766%); 4267/305544 objects misplaced (1.397%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T09:32:24.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:24.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:24.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:24.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:24.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:24.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:24.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:24.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:24.362 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:24.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:24.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:24.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:24.492 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:24.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:24.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:24.599 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:24.626 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:24.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:24.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:24.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:24.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:24.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:24.875 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:24.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:24.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:24.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:25.007 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:25.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:25.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:25.082 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:32:25.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:25.143 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:25 smithi181 bash[59345]: audit 2023-10-09T09:32:25.002938+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.163:0/3766614414' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:32:25.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:25 smithi181 bash[59345]: audit 2023-10-09T09:32:25.007185+0000 mon.a (mon.0) 1611 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:32:25.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:25 smithi181 bash[59345]: cluster 2023-10-09T09:32:25.057705+0000 mon.a (mon.0) 1612 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:32:25.159 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:25.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:25.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:25.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:25.296 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:25.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:25.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:25.407 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:25 smithi163 bash[78107]: audit 2023-10-09T09:32:25.002938+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.163:0/3766614414' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:32:25.407 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:25 smithi163 bash[78107]: audit 2023-10-09T09:32:25.007185+0000 mon.a (mon.0) 1611 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:32:25.407 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:25 smithi163 bash[78107]: cluster 2023-10-09T09:32:25.057705+0000 mon.a (mon.0) 1612 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:32:25.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:25 smithi163 bash[75425]: audit 2023-10-09T09:32:25.002938+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.163:0/3766614414' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:32:25.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:25 smithi163 bash[75425]: audit 2023-10-09T09:32:25.007185+0000 mon.a (mon.0) 1611 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:32:25.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:25 smithi163 bash[75425]: cluster 2023-10-09T09:32:25.057705+0000 mon.a (mon.0) 1612 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:32:25.426 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:25.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:25.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:25.534 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:25.586 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:25.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:25.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:25.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:25.737 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:25.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:25.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:25.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:25.869 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:25.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:25.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:25.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:26.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:26.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:26.126 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:26.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:26.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:26.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[78107]: cluster 2023-10-09T09:32:25.066762+0000 osd.4 (osd.4) 544 : cluster 3 failed to encode map e1117 with expected crc 2023-10-09T09:32:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[78107]: cluster 2023-10-09T09:32:25.068324+0000 osd.5 (osd.5) 667 : cluster 3 failed to encode map e1117 with expected crc 2023-10-09T09:32:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[78107]: audit 2023-10-09T09:32:25.071844+0000 mon.a (mon.0) 1613 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:32:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[78107]: cluster 2023-10-09T09:32:25.071949+0000 mon.a (mon.0) 1614 : cluster 0 osdmap e1117: 8 total, 8 up, 8 in 2023-10-09T09:32:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[78107]: audit 2023-10-09T09:32:25.374487+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[78107]: audit 2023-10-09T09:32:25.375850+0000 mon.a (mon.0) 1616 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[78107]: cluster 2023-10-09T09:32:25.571784+0000 mon.a (mon.0) 1617 : cluster 3 Health check update: Degraded data redundancy: 2340/305544 objects degraded (0.766%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:26.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[78107]: cluster 2023-10-09T09:32:25.744065+0000 mgr.y (mgr.34107) 786 : cluster 0 pgmap v1506: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 2.1 MiB/s wr, 859 op/s; 2340/300591 objects degraded (0.778%); 3992/300591 objects misplaced (1.328%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T09:32:26.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[75425]: cluster 2023-10-09T09:32:25.066762+0000 osd.4 (osd.4) 544 : cluster 3 failed to encode map e1117 with expected crc 2023-10-09T09:32:26.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[75425]: cluster 2023-10-09T09:32:25.068324+0000 osd.5 (osd.5) 667 : cluster 3 failed to encode map e1117 with expected crc 2023-10-09T09:32:26.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[75425]: audit 2023-10-09T09:32:25.071844+0000 mon.a (mon.0) 1613 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:32:26.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[75425]: cluster 2023-10-09T09:32:25.071949+0000 mon.a (mon.0) 1614 : cluster 0 osdmap e1117: 8 total, 8 up, 8 in 2023-10-09T09:32:26.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[75425]: audit 2023-10-09T09:32:25.374487+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:26.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[75425]: audit 2023-10-09T09:32:25.375850+0000 mon.a (mon.0) 1616 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:26.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[75425]: cluster 2023-10-09T09:32:25.571784+0000 mon.a (mon.0) 1617 : cluster 3 Health check update: Degraded data redundancy: 2340/305544 objects degraded (0.766%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:26.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:26 smithi163 bash[75425]: cluster 2023-10-09T09:32:25.744065+0000 mgr.y (mgr.34107) 786 : cluster 0 pgmap v1506: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 2.1 MiB/s wr, 859 op/s; 2340/300591 objects degraded (0.778%); 3992/300591 objects misplaced (1.328%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T09:32:26.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:26.256 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:26.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:26.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:26.370 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:26 smithi181 bash[59345]: cluster 2023-10-09T09:32:25.066762+0000 osd.4 (osd.4) 544 : cluster 3 failed to encode map e1117 with expected crc 2023-10-09T09:32:26.370 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:26 smithi181 bash[59345]: cluster 2023-10-09T09:32:25.068324+0000 osd.5 (osd.5) 667 : cluster 3 failed to encode map e1117 with expected crc 2023-10-09T09:32:26.370 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:26 smithi181 bash[59345]: audit 2023-10-09T09:32:25.071844+0000 mon.a (mon.0) 1613 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:32:26.370 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:26 smithi181 bash[59345]: cluster 2023-10-09T09:32:25.071949+0000 mon.a (mon.0) 1614 : cluster 0 osdmap e1117: 8 total, 8 up, 8 in 2023-10-09T09:32:26.370 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:26 smithi181 bash[59345]: audit 2023-10-09T09:32:25.374487+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:26.370 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:26 smithi181 bash[59345]: audit 2023-10-09T09:32:25.375850+0000 mon.a (mon.0) 1616 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:26.371 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:26 smithi181 bash[59345]: cluster 2023-10-09T09:32:25.571784+0000 mon.a (mon.0) 1617 : cluster 3 Health check update: Degraded data redundancy: 2340/305544 objects degraded (0.766%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:26.371 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:26 smithi181 bash[59345]: cluster 2023-10-09T09:32:25.744065+0000 mgr.y (mgr.34107) 786 : cluster 0 pgmap v1506: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 2.1 MiB/s wr, 859 op/s; 2340/300591 objects degraded (0.778%); 3992/300591 objects misplaced (1.328%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T09:32:26.395 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:26.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:26.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:26.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:32:26.500 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:26.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:26.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:26.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:26.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:26.697 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:26.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:26.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:26.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:26.829 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:26.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:26.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:26.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:26.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:26.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:26.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:27.061 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:27.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:27.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:27.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:27.197 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:27.224 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:27.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:27.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:27.328 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:27.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:27.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:27.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:27.461 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:32:27.461 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:32:27.461 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T09:32:27.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:32:27.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:27.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:27.556 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:27.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:27.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:27.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:27.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:27.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:27.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:27.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:27.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:27.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:27.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:27.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:27 smithi181 bash[59345]: cluster 2023-10-09T09:32:27.744882+0000 mgr.y (mgr.34107) 787 : cluster 0 pgmap v1507: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 2340/292473 objects degraded (0.800%); 3992/292473 objects misplaced (1.365%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T09:32:27.954 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:32:27.954 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:32:27.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:27 smithi163 bash[75425]: cluster 2023-10-09T09:32:27.744882+0000 mgr.y (mgr.34107) 787 : cluster 0 pgmap v1507: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 2340/292473 objects degraded (0.800%); 3992/292473 objects misplaced (1.365%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T09:32:27.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:27 smithi163 bash[78107]: cluster 2023-10-09T09:32:27.744882+0000 mgr.y (mgr.34107) 787 : cluster 0 pgmap v1507: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 2340/292473 objects degraded (0.800%); 3992/292473 objects misplaced (1.365%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T09:32:27.972 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:27.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:28.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:28.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:28.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:28.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:28.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:28.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:28.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:28.344 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:28.377 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:28.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:28.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:28.480 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:28.509 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:28.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:28.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:28.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:28.638 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:28.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:28.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:28.745 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:28.773 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:28.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:28.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:28.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:28.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:28.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:28.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:29.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:29.031 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:29.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:29.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:29.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:29.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:29.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:29.305 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:29.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:29.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:29.408 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:29.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:29.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:29.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:29.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T09:32:29.520 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:32:29.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:29.595 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:29.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:29.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:29.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:29.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:29.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:29.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:29.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:29.881 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:29.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:29.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:29.984 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:30.041 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:30.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:30.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:30.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:30.156 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:29 smithi163 bash[78107]: cluster 2023-10-09T09:32:29.745504+0000 mgr.y (mgr.34107) 788 : cluster 0 pgmap v1508: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 2340/288123 objects degraded (0.812%); 3710/288123 objects misplaced (1.288%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:30.156 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:29 smithi163 bash[75425]: cluster 2023-10-09T09:32:29.745504+0000 mgr.y (mgr.34107) 788 : cluster 0 pgmap v1508: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 2340/288123 objects degraded (0.812%); 3710/288123 objects misplaced (1.288%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:30.179 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:30.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:30.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:29 smithi181 bash[59345]: cluster 2023-10-09T09:32:29.745504+0000 mgr.y (mgr.34107) 788 : cluster 0 pgmap v1508: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 2340/288123 objects degraded (0.812%); 3710/288123 objects misplaced (1.288%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:30.329 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:30.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:30.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:30.437 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:30.470 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:30.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:30.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:30.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:30.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:30.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:30.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:30.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:30.750 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:30.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:30.854 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:30.883 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:30.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:30.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:30.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:31.021 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:31.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:31.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:31.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:30 smithi181 bash[59345]: cluster 2023-10-09T09:32:30.573153+0000 mon.a (mon.0) 1618 : cluster 3 Health check update: Degraded data redundancy: 2340/288123 objects degraded (0.812%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:31.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:31.140 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:30 smithi163 bash[78107]: cluster 2023-10-09T09:32:30.573153+0000 mon.a (mon.0) 1618 : cluster 3 Health check update: Degraded data redundancy: 2340/288123 objects degraded (0.812%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:31.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:30 smithi163 bash[75425]: cluster 2023-10-09T09:32:30.573153+0000 mon.a (mon.0) 1618 : cluster 3 Health check update: Degraded data redundancy: 2340/288123 objects degraded (0.812%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:31.172 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:31.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:31.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:31.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:31.306 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:31.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:31.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:31.409 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:31.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:31.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:31.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:31.540 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:31.572 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:31.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:31.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:31.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:31.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:31.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:31.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:31.848 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:31.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:31.951 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:31 smithi163 bash[75425]: cluster 2023-10-09T09:32:31.746190+0000 mgr.y (mgr.34107) 789 : cluster 0 pgmap v1509: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 2340/281961 objects degraded (0.830%); 3710/281961 objects misplaced (1.316%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:31.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:31 smithi163 bash[78107]: cluster 2023-10-09T09:32:31.746190+0000 mgr.y (mgr.34107) 789 : cluster 0 pgmap v1509: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 2340/281961 objects degraded (0.830%); 3710/281961 objects misplaced (1.316%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:31.977 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:31.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:31.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:32.028 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:32.032 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:32.080 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:32.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:32.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:32.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:32.221 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:32.254 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:32.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:32.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:32.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:31 smithi181 bash[59345]: cluster 2023-10-09T09:32:31.746190+0000 mgr.y (mgr.34107) 789 : cluster 0 pgmap v1509: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 2340/281961 objects degraded (0.830%); 3710/281961 objects misplaced (1.316%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:32.360 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:32.389 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:32.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:32.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:32.523 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:32.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:32.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:32.626 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:32.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:32.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:32.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:32.757 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:32.783 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:32.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:32.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:32.889 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:32.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:32.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:32.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:33.018 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:33.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:33.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:33.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:33.153 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:33.183 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:33.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:33.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:33.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:33.313 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:33.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:33.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:33.416 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:33.445 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:33.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:33.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:33.548 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:33.584 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:33.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:33.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:33.689 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:33.713 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:33.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:33.721 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:32:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:32:33] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.33.4" 2023-10-09T09:32:33.816 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:33.840 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:33.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:33.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:33.945 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:33.975 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:33.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:33.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:34.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:33 smithi181 bash[59345]: cluster 2023-10-09T09:32:33.746784+0000 mgr.y (mgr.34107) 790 : cluster 0 pgmap v1510: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 2340/277332 objects degraded (0.844%); 3710/277332 objects misplaced (1.338%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:34.066 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:34.066 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:34.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:34.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:33 smithi163 bash[78107]: cluster 2023-10-09T09:32:33.746784+0000 mgr.y (mgr.34107) 790 : cluster 0 pgmap v1510: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 2340/277332 objects degraded (0.844%); 3710/277332 objects misplaced (1.338%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:34.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:33 smithi163 bash[75425]: cluster 2023-10-09T09:32:33.746784+0000 mgr.y (mgr.34107) 790 : cluster 0 pgmap v1510: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 2340/277332 objects degraded (0.844%); 3710/277332 objects misplaced (1.338%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:34.110 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:34.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:34.217 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:34.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:34.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:34.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:34.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:34.374 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:34.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:34.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:34.476 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:34.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:34.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:34.607 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:34.632 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:34.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:34.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:34.735 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:34.774 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:34.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:34.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:34.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:34.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:34.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:35.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:35.033 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:35.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:35.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:35.168 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:35.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:35.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:35.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:35.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:35.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:35.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:35.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:35.460 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:35.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:35.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:35.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:35.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:35.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:35.706 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:35.737 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:35.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:35.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:35.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:35.873 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:35.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:35.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:35 smithi163 bash[78107]: cluster 2023-10-09T09:32:35.574697+0000 mon.a (mon.0) 1619 : cluster 3 Health check update: Degraded data redundancy: 2340/277332 objects degraded (0.844%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:35.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:35 smithi163 bash[75425]: cluster 2023-10-09T09:32:35.574697+0000 mon.a (mon.0) 1619 : cluster 3 Health check update: Degraded data redundancy: 2340/277332 objects degraded (0.844%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:35.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:35.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:35 smithi181 bash[59345]: cluster 2023-10-09T09:32:35.574697+0000 mon.a (mon.0) 1619 : cluster 3 Health check update: Degraded data redundancy: 2340/277332 objects degraded (0.844%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:32:36.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:36.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:36.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:36.113 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:36.143 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:36.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:36.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:36.250 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:36.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:36.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:36.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:36.380 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:36.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:36.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:36.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:36.505 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:36.542 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:36.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:36.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:36.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:36 smithi163 bash[75425]: cluster 2023-10-09T09:32:35.747705+0000 mgr.y (mgr.34107) 791 : cluster 0 pgmap v1511: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 2340/272586 objects degraded (0.858%); 3404/272586 objects misplaced (1.249%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T09:32:36.646 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:36 smithi163 bash[78107]: cluster 2023-10-09T09:32:35.747705+0000 mgr.y (mgr.34107) 791 : cluster 0 pgmap v1511: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 2340/272586 objects degraded (0.858%); 3404/272586 objects misplaced (1.249%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T09:32:36.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:36.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:36.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:36.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:36.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:36.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:36.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:36.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:36.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:36.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:36.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:36.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:37.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:37.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:37.055 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:37.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:37.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:36 smithi181 bash[59345]: cluster 2023-10-09T09:32:35.747705+0000 mgr.y (mgr.34107) 791 : cluster 0 pgmap v1511: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 2340/272586 objects degraded (0.858%); 3404/272586 objects misplaced (1.249%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T09:32:37.155 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:37.188 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:37.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:37.291 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:37.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:37.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:37.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:37.420 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:37.457 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:37.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:37.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:37.563 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:37.595 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:37.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:37.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:37.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:37.737 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:37.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:37.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:37.840 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:37.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:37.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:37.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:37.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:37 smithi163 bash[78107]: cluster 2023-10-09T09:32:37.748585+0000 mgr.y (mgr.34107) 792 : cluster 0 pgmap v1512: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 2340/264828 objects degraded (0.884%); 3404/264828 objects misplaced (1.285%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T09:32:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:37 smithi163 bash[75425]: cluster 2023-10-09T09:32:37.748585+0000 mgr.y (mgr.34107) 792 : cluster 0 pgmap v1512: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 2340/264828 objects degraded (0.884%); 3404/264828 objects misplaced (1.285%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T09:32:37.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:38.004 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:38.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:38.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:38.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:38.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:38.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:38.255 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:37 smithi181 bash[59345]: cluster 2023-10-09T09:32:37.748585+0000 mgr.y (mgr.34107) 792 : cluster 0 pgmap v1512: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 2340/264828 objects degraded (0.884%); 3404/264828 objects misplaced (1.285%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T09:32:38.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:38.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:38.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:38.421 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:38.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:38.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:38.525 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:38.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:38.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:38.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:38.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:38.685 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:38.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:38.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:38.789 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:38.818 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:38.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:38.924 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:38.952 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:38.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:38.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:39.059 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:39.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:38 smithi181 bash[59345]: audit 2023-10-09T09:32:38.268691+0000 mon.a (mon.0) 1620 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:32:39.090 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:39.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:39.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:38 smithi163 bash[78107]: audit 2023-10-09T09:32:38.268691+0000 mon.a (mon.0) 1620 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:32:39.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:38 smithi163 bash[75425]: audit 2023-10-09T09:32:38.268691+0000 mon.a (mon.0) 1620 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:32:39.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:39.220 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:39.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:39.323 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:39.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:39.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:39.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:39.456 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:39.483 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:39.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:39.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:39.587 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:39.614 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:39.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:39.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:39.718 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:39.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:39.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:39.842 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:39.880 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:39.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:39.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:39.982 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:39.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:39 smithi163 bash[75425]: cluster 2023-10-09T09:32:39.749206+0000 mgr.y (mgr.34107) 793 : cluster 0 pgmap v1513: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 2340/260643 objects degraded (0.898%); 3043/260643 objects misplaced (1.167%); 4.9 MiB/s, 79 objects/s recovering 2023-10-09T09:32:39.990 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:39 smithi163 bash[78107]: cluster 2023-10-09T09:32:39.749206+0000 mgr.y (mgr.34107) 793 : cluster 0 pgmap v1513: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 2340/260643 objects degraded (0.898%); 3043/260643 objects misplaced (1.167%); 4.9 MiB/s, 79 objects/s recovering 2023-10-09T09:32:40.018 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:40.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:40.127 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:40.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:40.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:40.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:40.261 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:40.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:39 smithi181 bash[59345]: cluster 2023-10-09T09:32:39.749206+0000 mgr.y (mgr.34107) 793 : cluster 0 pgmap v1513: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 2340/260643 objects degraded (0.898%); 3043/260643 objects misplaced (1.167%); 4.9 MiB/s, 79 objects/s recovering 2023-10-09T09:32:40.294 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:40.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:40.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:40.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:40.427 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:40.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:40.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:40.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:40.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:40.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:40.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:40.718 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:40.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:40.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:40.827 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:40.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:40.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:40.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:40.865 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:40 smithi163 bash[75425]: audit 2023-10-09T09:32:40.369923+0000 mon.a (mon.0) 1621 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:40.865 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:40 smithi163 bash[75425]: cluster 2023-10-09T09:32:40.579862+0000 mon.a (mon.0) 1622 : cluster 3 Health check update: Degraded data redundancy: 2340/260643 objects degraded (0.898%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:32:40.959 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:40.982 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:40.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:41.085 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:41.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:40 smithi181 bash[59345]: audit 2023-10-09T09:32:40.369923+0000 mon.a (mon.0) 1621 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:41.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:40 smithi181 bash[59345]: cluster 2023-10-09T09:32:40.579862+0000 mon.a (mon.0) 1622 : cluster 3 Health check update: Degraded data redundancy: 2340/260643 objects degraded (0.898%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:32:41.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:41.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:41.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:41.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:40 smithi163 bash[78107]: audit 2023-10-09T09:32:40.369923+0000 mon.a (mon.0) 1621 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:40 smithi163 bash[78107]: cluster 2023-10-09T09:32:40.579862+0000 mon.a (mon.0) 1622 : cluster 3 Health check update: Degraded data redundancy: 2340/260643 objects degraded (0.898%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:32:41.241 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:41.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:41.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:41.348 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:41.378 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:41.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:41.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:41.485 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:41.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:41.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:41.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:41.611 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:41.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:41.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:41.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:41.739 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:41.764 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:41.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:41.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:41.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:41.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:41 smithi181 bash[59345]: cluster 2023-10-09T09:32:41.749974+0000 mgr.y (mgr.34107) 794 : cluster 0 pgmap v1514: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 2340/254952 objects degraded (0.918%); 3043/254952 objects misplaced (1.194%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:41.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:41.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:41.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:42.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:42.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:42.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:42.129 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:42.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:42.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:42.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:41 smithi163 bash[75425]: cluster 2023-10-09T09:32:41.749974+0000 mgr.y (mgr.34107) 794 : cluster 0 pgmap v1514: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 2340/254952 objects degraded (0.918%); 3043/254952 objects misplaced (1.194%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:42.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:41 smithi163 bash[78107]: cluster 2023-10-09T09:32:41.749974+0000 mgr.y (mgr.34107) 794 : cluster 0 pgmap v1514: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 2340/254952 objects degraded (0.918%); 3043/254952 objects misplaced (1.194%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:42.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:42.301 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:42.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:42.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:42.407 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:42.433 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:42.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:42.538 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:42.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:42.567 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:42.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:42.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:42.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:42.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:42.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:42.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:42.839 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:42.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:42.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:42.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:42.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:42.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:42.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:43.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:43.095 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:43.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:43.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:43.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:43.244 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:43.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:43.351 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:43.381 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:43.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:43.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:43.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:43.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:43.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:43.622 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:43.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:43.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:43.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:32:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:32:43] "GET /metrics HTTP/1.1" 200 36347 "" "Prometheus/2.33.4" 2023-10-09T09:32:43.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:43.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:43.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:43.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:43.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:43.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:43.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:43.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:44.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:44.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:43 smithi163 bash[75425]: cluster 2023-10-09T09:32:43.750645+0000 mgr.y (mgr.34107) 795 : cluster 0 pgmap v1515: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 2340/250494 objects degraded (0.934%); 3043/250494 objects misplaced (1.215%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:44.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:43 smithi163 bash[78107]: cluster 2023-10-09T09:32:43.750645+0000 mgr.y (mgr.34107) 795 : cluster 0 pgmap v1515: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 2340/250494 objects degraded (0.934%); 3043/250494 objects misplaced (1.215%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:43 smithi181 bash[59345]: cluster 2023-10-09T09:32:43.750645+0000 mgr.y (mgr.34107) 795 : cluster 0 pgmap v1515: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 2340/250494 objects degraded (0.934%); 3043/250494 objects misplaced (1.215%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:44.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:44.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:44.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:44.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:44.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:44.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:44.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:44.318 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:44.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:44.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:44.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:44.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:44.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:44.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:44.625 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:44.648 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:44.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:44.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:44.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:44.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:44.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:44.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:44.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:44.900 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:44.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:44.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:45.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:45.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:45.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:45.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:45.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:45.179 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:45.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:45.283 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:45.307 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:45.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:45.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:45.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:45.436 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:45.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:45.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:45.580 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:45.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:45.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:45.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:45.709 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:45.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:45.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:45.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:45.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:45.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:45.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:45.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:45.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:45 smithi163 bash[78107]: audit 2023-10-09T09:32:44.695358+0000 mon.a (mon.0) 1623 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:45.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:45 smithi163 bash[78107]: audit 2023-10-09T09:32:44.705631+0000 mon.a (mon.0) 1624 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:45.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:45 smithi163 bash[78107]: cluster 2023-10-09T09:32:45.581171+0000 mon.a (mon.0) 1625 : cluster 3 Health check update: Degraded data redundancy: 2340/250494 objects degraded (0.934%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:32:45.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:45 smithi163 bash[75425]: audit 2023-10-09T09:32:44.695358+0000 mon.a (mon.0) 1623 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:45.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:45 smithi163 bash[75425]: audit 2023-10-09T09:32:44.705631+0000 mon.a (mon.0) 1624 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:45.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:45 smithi163 bash[75425]: cluster 2023-10-09T09:32:45.581171+0000 mon.a (mon.0) 1625 : cluster 3 Health check update: Degraded data redundancy: 2340/250494 objects degraded (0.934%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:32:45.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:45.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:45.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:46.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:45 smithi181 bash[59345]: audit 2023-10-09T09:32:44.695358+0000 mon.a (mon.0) 1623 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:46.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:45 smithi181 bash[59345]: audit 2023-10-09T09:32:44.705631+0000 mon.a (mon.0) 1624 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:46.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:45 smithi181 bash[59345]: cluster 2023-10-09T09:32:45.581171+0000 mon.a (mon.0) 1625 : cluster 3 Health check update: Degraded data redundancy: 2340/250494 objects degraded (0.934%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:32:46.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:46.124 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:46.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:46.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:46.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:46.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:46.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:46.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:46.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:46.393 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:46.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:46.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:46.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:46.525 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:46.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:46.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:46.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:46.685 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:46.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:46.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:46.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:46.817 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:46.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:46.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:46.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:46.945 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:46.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:46.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:47.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:47.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:46 smithi163 bash[75425]: cluster 2023-10-09T09:32:45.751263+0000 mgr.y (mgr.34107) 796 : cluster 0 pgmap v1516: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 2340/245973 objects degraded (0.951%); 2626/245973 objects misplaced (1.068%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T09:32:47.058 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:46 smithi163 bash[78107]: cluster 2023-10-09T09:32:45.751263+0000 mgr.y (mgr.34107) 796 : cluster 0 pgmap v1516: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 2340/245973 objects degraded (0.951%); 2626/245973 objects misplaced (1.068%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T09:32:47.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:46 smithi181 bash[59345]: cluster 2023-10-09T09:32:45.751263+0000 mgr.y (mgr.34107) 796 : cluster 0 pgmap v1516: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 2340/245973 objects degraded (0.951%); 2626/245973 objects misplaced (1.068%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T09:32:47.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:47.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:47.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:47.198 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:47.249 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:47.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:47.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:47.353 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:47.385 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:47.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:47.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:47.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:47.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:47.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:47.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:47.675 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:47.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:47.779 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:47.809 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:47.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:47.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:47.935 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:47.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:47.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:48.044 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:48.070 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:48.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:48.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:48.216 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:48.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:48.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:48.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:48.352 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:48.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:48.456 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:48.482 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:48.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:48.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:48.585 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:48.594 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:32:48.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:48.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:48.642 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:32:48.688 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:32:48.718 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:48.742 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:48.746 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:32:48.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:48.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:48.754 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:32:48.793 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:32:48.827 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:32:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:32:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T09:32:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T09:32:48.828 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:32:48.837 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:32:48.852 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:48.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: cluster 2023-10-09T09:32:47.714959+0000 osd.5 (osd.5) 668 : cluster 3 failed to encode map e1118 with expected crc 2023-10-09T09:32:48.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: cluster 2023-10-09T09:32:47.719728+0000 mon.a (mon.0) 1626 : cluster 0 osdmap e1118: 8 total, 8 up, 8 in 2023-10-09T09:32:48.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: cluster 2023-10-09T09:32:47.726351+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e1118 with expected crc 2023-10-09T09:32:48.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: cluster 2023-10-09T09:32:47.729977+0000 osd.7 (osd.7) 509 : cluster 3 failed to encode map e1118 with expected crc 2023-10-09T09:32:48.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: cluster 2023-10-09T09:32:47.751750+0000 mgr.y (mgr.34107) 797 : cluster 0 pgmap v1518: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 2340/238830 objects degraded (0.980%); 2626/238830 objects misplaced (1.100%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T09:32:48.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: audit 2023-10-09T09:32:47.825853+0000 mon.a (mon.0) 1627 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:48.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: audit 2023-10-09T09:32:47.839432+0000 mon.a (mon.0) 1628 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:48.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: audit 2023-10-09T09:32:48.462720+0000 mon.a (mon.0) 1629 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:32:48.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: audit 2023-10-09T09:32:48.463899+0000 mon.a (mon.0) 1630 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:32:48.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: audit 2023-10-09T09:32:48.472483+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:48.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: audit 2023-10-09T09:32:48.537898+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.163:0/2770365080' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:32:48.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: audit 2023-10-09T09:32:48.706496+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.163:0/3245267367' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:32:48.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[75425]: audit 2023-10-09T09:32:48.707553+0000 mon.a (mon.0) 1632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:32:48.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: cluster 2023-10-09T09:32:47.714959+0000 osd.5 (osd.5) 668 : cluster 3 failed to encode map e1118 with expected crc 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: cluster 2023-10-09T09:32:47.719728+0000 mon.a (mon.0) 1626 : cluster 0 osdmap e1118: 8 total, 8 up, 8 in 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: cluster 2023-10-09T09:32:47.726351+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e1118 with expected crc 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: cluster 2023-10-09T09:32:47.729977+0000 osd.7 (osd.7) 509 : cluster 3 failed to encode map e1118 with expected crc 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: cluster 2023-10-09T09:32:47.751750+0000 mgr.y (mgr.34107) 797 : cluster 0 pgmap v1518: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 2340/238830 objects degraded (0.980%); 2626/238830 objects misplaced (1.100%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: audit 2023-10-09T09:32:47.825853+0000 mon.a (mon.0) 1627 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: audit 2023-10-09T09:32:47.839432+0000 mon.a (mon.0) 1628 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: audit 2023-10-09T09:32:48.462720+0000 mon.a (mon.0) 1629 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: audit 2023-10-09T09:32:48.463899+0000 mon.a (mon.0) 1630 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: audit 2023-10-09T09:32:48.472483+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: audit 2023-10-09T09:32:48.537898+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.163:0/2770365080' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: audit 2023-10-09T09:32:48.706496+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.163:0/3245267367' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:32:48.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:48 smithi163 bash[78107]: audit 2023-10-09T09:32:48.707553+0000 mon.a (mon.0) 1632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:32:48.875 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:48.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:48.883 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:32:48.928 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:32:48.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:32:48.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:32:48.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:32:48.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:32:48.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:32:48.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:32:48.933 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:32:48.933 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:32:48.982 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:49.010 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:49.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:49.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: cluster 2023-10-09T09:32:47.714959+0000 osd.5 (osd.5) 668 : cluster 3 failed to encode map e1118 with expected crc 2023-10-09T09:32:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: cluster 2023-10-09T09:32:47.719728+0000 mon.a (mon.0) 1626 : cluster 0 osdmap e1118: 8 total, 8 up, 8 in 2023-10-09T09:32:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: cluster 2023-10-09T09:32:47.726351+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e1118 with expected crc 2023-10-09T09:32:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: cluster 2023-10-09T09:32:47.729977+0000 osd.7 (osd.7) 509 : cluster 3 failed to encode map e1118 with expected crc 2023-10-09T09:32:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: cluster 2023-10-09T09:32:47.751750+0000 mgr.y (mgr.34107) 797 : cluster 0 pgmap v1518: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 2340/238830 objects degraded (0.980%); 2626/238830 objects misplaced (1.100%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T09:32:49.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: audit 2023-10-09T09:32:47.825853+0000 mon.a (mon.0) 1627 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:49.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: audit 2023-10-09T09:32:47.839432+0000 mon.a (mon.0) 1628 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:49.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: audit 2023-10-09T09:32:48.462720+0000 mon.a (mon.0) 1629 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:32:49.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: audit 2023-10-09T09:32:48.463899+0000 mon.a (mon.0) 1630 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:32:49.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: audit 2023-10-09T09:32:48.472483+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:32:49.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: audit 2023-10-09T09:32:48.537898+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.163:0/2770365080' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:32:49.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: audit 2023-10-09T09:32:48.706496+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.163:0/3245267367' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:32:49.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:48 smithi181 bash[59345]: audit 2023-10-09T09:32:48.707553+0000 mon.a (mon.0) 1632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:32:49.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:49.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:49.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:49.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:49.179 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:32:49.188 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:32:49.242 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:49.390 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:49.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:49.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:49.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:49.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:49.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:49.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:49.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:49.664 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:49.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:49.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:49.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:49.878 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:49.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:49.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[78107]: cluster 2023-10-09T09:32:48.714166+0000 mon.a (mon.0) 1633 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:32:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[78107]: cluster 2023-10-09T09:32:48.725412+0000 osd.5 (osd.5) 669 : cluster 3 failed to encode map e1119 with expected crc 2023-10-09T09:32:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[78107]: audit 2023-10-09T09:32:48.729758+0000 mon.a (mon.0) 1634 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:32:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[78107]: cluster 2023-10-09T09:32:48.730782+0000 mon.a (mon.0) 1635 : cluster 0 osdmap e1119: 8 total, 8 up, 8 in 2023-10-09T09:32:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[78107]: cluster 2023-10-09T09:32:48.731262+0000 osd.4 (osd.4) 545 : cluster 3 failed to encode map e1119 with expected crc 2023-10-09T09:32:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[78107]: cluster 2023-10-09T09:32:48.736190+0000 osd.7 (osd.7) 510 : cluster 3 failed to encode map e1119 with expected crc 2023-10-09T09:32:49.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[75425]: cluster 2023-10-09T09:32:48.714166+0000 mon.a (mon.0) 1633 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:32:49.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[75425]: cluster 2023-10-09T09:32:48.725412+0000 osd.5 (osd.5) 669 : cluster 3 failed to encode map e1119 with expected crc 2023-10-09T09:32:49.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[75425]: audit 2023-10-09T09:32:48.729758+0000 mon.a (mon.0) 1634 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:32:49.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[75425]: cluster 2023-10-09T09:32:48.730782+0000 mon.a (mon.0) 1635 : cluster 0 osdmap e1119: 8 total, 8 up, 8 in 2023-10-09T09:32:49.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[75425]: cluster 2023-10-09T09:32:48.731262+0000 osd.4 (osd.4) 545 : cluster 3 failed to encode map e1119 with expected crc 2023-10-09T09:32:49.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:49 smithi163 bash[75425]: cluster 2023-10-09T09:32:48.736190+0000 osd.7 (osd.7) 510 : cluster 3 failed to encode map e1119 with expected crc 2023-10-09T09:32:50.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:50.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:49 smithi181 bash[59345]: cluster 2023-10-09T09:32:48.714166+0000 mon.a (mon.0) 1633 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:32:50.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:49 smithi181 bash[59345]: cluster 2023-10-09T09:32:48.725412+0000 osd.5 (osd.5) 669 : cluster 3 failed to encode map e1119 with expected crc 2023-10-09T09:32:50.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:49 smithi181 bash[59345]: audit 2023-10-09T09:32:48.729758+0000 mon.a (mon.0) 1634 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:32:50.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:49 smithi181 bash[59345]: cluster 2023-10-09T09:32:48.730782+0000 mon.a (mon.0) 1635 : cluster 0 osdmap e1119: 8 total, 8 up, 8 in 2023-10-09T09:32:50.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:49 smithi181 bash[59345]: cluster 2023-10-09T09:32:48.731262+0000 osd.4 (osd.4) 545 : cluster 3 failed to encode map e1119 with expected crc 2023-10-09T09:32:50.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:49 smithi181 bash[59345]: cluster 2023-10-09T09:32:48.736190+0000 osd.7 (osd.7) 510 : cluster 3 failed to encode map e1119 with expected crc 2023-10-09T09:32:50.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:50.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:50.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:50.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:50.311 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:50.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:50.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:50.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:50.440 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:50.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:50.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:50.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:50.755 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:50.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:50.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:50.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:50.893 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:50.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:50.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:50.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:51.005 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:50 smithi163 bash[75425]: audit 2023-10-09T09:32:49.159008+0000 mgr.y (mgr.34107) 798 : audit 0 from='client.44987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:32:51.005 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:50 smithi163 bash[75425]: cluster 2023-10-09T09:32:49.754728+0000 mgr.y (mgr.34107) 799 : cluster 0 pgmap v1520: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 887/235863 objects degraded (0.376%); 3971/235863 objects misplaced (1.684%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:32:51.005 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:50 smithi163 bash[75425]: cluster 2023-10-09T09:32:50.582223+0000 mon.a (mon.0) 1636 : cluster 3 Health check update: Degraded data redundancy: 2340/238830 objects degraded (0.980%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:32:51.006 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:50 smithi163 bash[78107]: audit 2023-10-09T09:32:49.159008+0000 mgr.y (mgr.34107) 798 : audit 0 from='client.44987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:32:51.006 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:50 smithi163 bash[78107]: cluster 2023-10-09T09:32:49.754728+0000 mgr.y (mgr.34107) 799 : cluster 0 pgmap v1520: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 887/235863 objects degraded (0.376%); 3971/235863 objects misplaced (1.684%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:32:51.006 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:50 smithi163 bash[78107]: cluster 2023-10-09T09:32:50.582223+0000 mon.a (mon.0) 1636 : cluster 3 Health check update: Degraded data redundancy: 2340/238830 objects degraded (0.980%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:32:51.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:51.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:51.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:50 smithi181 bash[59345]: audit 2023-10-09T09:32:49.159008+0000 mgr.y (mgr.34107) 798 : audit 0 from='client.44987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:32:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:50 smithi181 bash[59345]: cluster 2023-10-09T09:32:49.754728+0000 mgr.y (mgr.34107) 799 : cluster 0 pgmap v1520: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 887/235863 objects degraded (0.376%); 3971/235863 objects misplaced (1.684%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T09:32:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:50 smithi181 bash[59345]: cluster 2023-10-09T09:32:50.582223+0000 mon.a (mon.0) 1636 : cluster 3 Health check update: Degraded data redundancy: 2340/238830 objects degraded (0.980%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:32:51.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:51.361 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:51.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:51.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:51.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:51.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:51.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:51.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:51.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:51.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:51.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:51.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:51.734 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:51.762 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:51.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:51.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:51.869 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:51.893 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:51.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:51.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:51.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:52.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:52.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:52.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:52.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:51 smithi181 bash[59345]: cluster 2023-10-09T09:32:51.755494+0000 mgr.y (mgr.34107) 800 : cluster 0 pgmap v1521: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 887/230619 objects degraded (0.385%); 3685/230619 objects misplaced (1.598%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:32:52.129 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:52.159 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:52.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:52.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:52.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:51 smithi163 bash[75425]: cluster 2023-10-09T09:32:51.755494+0000 mgr.y (mgr.34107) 800 : cluster 0 pgmap v1521: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 887/230619 objects degraded (0.385%); 3685/230619 objects misplaced (1.598%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:32:52.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:51 smithi163 bash[78107]: cluster 2023-10-09T09:32:51.755494+0000 mgr.y (mgr.34107) 800 : cluster 0 pgmap v1521: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 887/230619 objects degraded (0.385%); 3685/230619 objects misplaced (1.598%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:32:52.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:52.287 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:52.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:52.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:52.393 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:52.422 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1519,"stamp":"2023-10-09T09:32:48.726110+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5194553272,"num_objects":79279,"num_object_clones":0,"num_object_copies":237837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":887,"num_objects_misplaced":3971,"num_objects_unfound":0,"num_objects_dirty":79279,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":124641,"num_write_kb":6079534,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4515,"num_bytes_recovered":295895040,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":176555,"ondisk_log_size":176555,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17940028,"kb_used_data":15652668,"kb_used_omap":416,"kb_used_meta":2286751,"kb_avail":732021188,"statfs":{"total":767960285184,"available":749589696512,"internally_reserved":0,"allocated":16028332032,"data_stored":16023064540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":2341633457},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-498204672,"num_objects":-7602,"num_object_clones":0,"num_object_copies":-22806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1453,"num_objects_misplaced":928,"num_objects_unfound":0,"num_objects_dirty":-7602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7602,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":471,"num_bytes_recovered":30867456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.977342"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1084,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407554+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:32:14.407554+0000","last_peered":"2023-10-09T09:32:14.407554+0000","last_clean":"2023-10-09T09:32:14.407554+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:32:14.407554+0000","last_undegraded":"2023-10-09T09:32:14.407554+0000","last_fullsized":"2023-10-09T09:32:14.407554+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1050,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407220+0000","last_change":"2023-10-09T09:15:36.184235+0000","last_active":"2023-10-09T09:32:14.407220+0000","last_peered":"2023-10-09T09:32:14.407220+0000","last_clean":"2023-10-09T09:32:14.407220+0000","last_became_active":"2023-10-09T09:15:36.183538+0000","last_became_peered":"2023-10-09T09:15:36.183538+0000","last_unstale":"2023-10-09T09:32:14.407220+0000","last_undegraded":"2023-10-09T09:32:14.407220+0000","last_fullsized":"2023-10-09T09:32:14.407220+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:21.222815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1055,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.103895+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:32:25.103895+0000","last_peered":"2023-10-09T09:32:25.103895+0000","last_clean":"2023-10-09T09:32:25.103895+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:32:25.103895+0000","last_undegraded":"2023-10-09T09:32:25.103895+0000","last_fullsized":"2023-10-09T09:32:25.103895+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1046,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:17.412093+0000","last_change":"2023-10-09T09:15:36.175934+0000","last_active":"2023-10-09T09:32:17.412093+0000","last_peered":"2023-10-09T09:32:17.412093+0000","last_clean":"2023-10-09T09:32:17.412093+0000","last_became_active":"2023-10-09T09:15:36.175641+0000","last_became_peered":"2023-10-09T09:15:36.175641+0000","last_unstale":"2023-10-09T09:32:17.412093+0000","last_undegraded":"2023-10-09T09:32:17.412093+0000","last_fullsized":"2023-10-09T09:32:17.412093+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:00.977897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1032,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.088672+0000","last_change":"2023-10-09T09:15:36.173740+0000","last_active":"2023-10-09T09:32:25.088672+0000","last_peered":"2023-10-09T09:32:25.088672+0000","last_clean":"2023-10-09T09:32:25.088672+0000","last_became_active":"2023-10-09T09:15:36.173039+0000","last_became_peered":"2023-10-09T09:15:36.173039+0000","last_unstale":"2023-10-09T09:32:25.088672+0000","last_undegraded":"2023-10-09T09:32:25.088672+0000","last_fullsized":"2023-10-09T09:32:25.088672+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:34.857717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1048,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.429012+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:32:14.429012+0000","last_peered":"2023-10-09T09:32:14.429012+0000","last_clean":"2023-10-09T09:32:14.429012+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:32:14.429012+0000","last_undegraded":"2023-10-09T09:32:14.429012+0000","last_fullsized":"2023-10-09T09:32:14.429012+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"1117'6491","reported_seq":6532,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:44.412210+0000","last_change":"2023-10-09T09:31:28.896417+0000","last_active":"2023-10-09T09:32:44.412210+0000","last_peered":"2023-10-09T09:32:44.412210+0000","last_clean":"2023-10-09T09:32:44.412210+0000","last_became_active":"2023-10-09T09:31:28.870011+0000","last_became_peered":"2023-10-09T09:31:28.870011+0000","last_unstale":"2023-10-09T09:32:44.412210+0000","last_undegraded":"2023-10-09T09:32:44.412210+0000","last_fullsized":"2023-10-09T09:32:44.412210+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6491,"log_dups_size":0,"ondisk_log_size":6491,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164888576,"num_objects":2516,"num_object_clones":0,"num_object_copies":7548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4302,"num_write_kb":218176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1045,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.432030+0000","last_change":"2023-10-09T09:15:36.182669+0000","last_active":"2023-10-09T09:32:14.432030+0000","last_peered":"2023-10-09T09:32:14.432030+0000","last_clean":"2023-10-09T09:32:14.432030+0000","last_became_active":"2023-10-09T09:15:36.182359+0000","last_became_peered":"2023-10-09T09:15:36.182359+0000","last_unstale":"2023-10-09T09:32:14.432030+0000","last_undegraded":"2023-10-09T09:32:14.432030+0000","last_fullsized":"2023-10-09T09:32:14.432030+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:24.777558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1016,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406778+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:32:14.406778+0000","last_peered":"2023-10-09T09:32:14.406778+0000","last_clean":"2023-10-09T09:32:14.406778+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:32:14.406778+0000","last_undegraded":"2023-10-09T09:32:14.406778+0000","last_fullsized":"2023-10-09T09:32:14.406778+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"1118'6341","reported_seq":6547,"reported_epoch":1118,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T09:32:48.550488+0000","last_change":"2023-10-09T09:32:46.975155+0000","last_active":"2023-10-09T09:32:48.550488+0000","last_peered":"2023-10-09T09:32:48.550488+0000","last_clean":"2023-10-09T09:31:37.505588+0000","last_became_active":"2023-10-09T09:31:41.273317+0000","last_became_peered":"2023-10-09T09:31:41.273317+0000","last_unstale":"2023-10-09T09:32:48.550488+0000","last_undegraded":"2023-10-09T09:31:40.658391+0000","last_fullsized":"2023-10-09T09:31:40.587106+0000","mapping_epoch":1112,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6341,"log_dups_size":0,"ondisk_log_size":6341,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157089792,"num_objects":2397,"num_object_clones":0,"num_object_copies":7191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":887,"num_objects_misplaced":3685,"num_objects_unfound":0,"num_objects_dirty":2397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4239,"num_write_kb":212352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":3538944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":111},{"shards":"1,2,3","objects":1399}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1069,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406330+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:32:14.406330+0000","last_peered":"2023-10-09T09:32:14.406330+0000","last_clean":"2023-10-09T09:32:14.406330+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:32:14.406330+0000","last_undegraded":"2023-10-09T09:32:14.406330+0000","last_fullsized":"2023-10-09T09:32:14.406330+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1027,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.411252+0000","last_change":"2023-10-09T09:15:36.181601+0000","last_active":"2023-10-09T09:32:14.411252+0000","last_peered":"2023-10-09T09:32:14.411252+0000","last_clean":"2023-10-09T09:32:14.411252+0000","last_became_active":"2023-10-09T09:15:36.180749+0000","last_became_peered":"2023-10-09T09:15:36.180749+0000","last_unstale":"2023-10-09T09:32:14.411252+0000","last_undegraded":"2023-10-09T09:32:14.411252+0000","last_fullsized":"2023-10-09T09:32:14.411252+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:30.523607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"1117'6470","reported_seq":10699,"reported_epoch":1117,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T09:32:44.524268+0000","last_change":"2023-10-09T09:32:12.668036+0000","last_active":"2023-10-09T09:32:44.524268+0000","last_peered":"2023-10-09T09:32:44.524268+0000","last_clean":"2023-10-09T09:31:35.416261+0000","last_became_active":"2023-10-09T09:31:39.144598+0000","last_became_peered":"2023-10-09T09:31:39.144598+0000","last_unstale":"2023-10-09T09:32:44.524268+0000","last_undegraded":"2023-10-09T09:32:44.524268+0000","last_fullsized":"2023-10-09T09:31:38.581135+0000","mapping_epoch":1110,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"1107'4587","last_scrub_stamp":"2023-10-09T09:31:35.597905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:35.597905+0000","objects_scrubbed":0,"log_size":6470,"log_dups_size":0,"ondisk_log_size":6470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165642240,"num_objects":2528,"num_object_clones":0,"num_object_copies":7584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":286,"num_objects_unfound":0,"num_objects_dirty":2528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4294,"num_write_kb":218272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2094,"num_bytes_recovered":137232384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":286},{"shards":"3,6","objects":2242}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1048,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.427192+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:32:14.427192+0000","last_peered":"2023-10-09T09:32:14.427192+0000","last_clean":"2023-10-09T09:32:14.427192+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:32:14.427192+0000","last_undegraded":"2023-10-09T09:32:14.427192+0000","last_fullsized":"2023-10-09T09:32:14.427192+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1055,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.105556+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:32:25.105556+0000","last_peered":"2023-10-09T09:32:25.105556+0000","last_clean":"2023-10-09T09:32:25.105556+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:32:25.105556+0000","last_undegraded":"2023-10-09T09:32:25.105556+0000","last_fullsized":"2023-10-09T09:32:25.105556+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"1117'6416","reported_seq":11185,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:44.422879+0000","last_change":"2023-10-09T09:32:14.416857+0000","last_active":"2023-10-09T09:32:44.422879+0000","last_peered":"2023-10-09T09:32:44.422879+0000","last_clean":"2023-10-09T09:32:44.422879+0000","last_became_active":"2023-10-09T09:32:14.411081+0000","last_became_peered":"2023-10-09T09:32:14.411081+0000","last_unstale":"2023-10-09T09:32:44.422879+0000","last_undegraded":"2023-10-09T09:32:44.422879+0000","last_fullsized":"2023-10-09T09:32:44.422879+0000","mapping_epoch":1115,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6416,"log_dups_size":0,"ondisk_log_size":6416,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164003840,"num_objects":2503,"num_object_clones":0,"num_object_copies":7509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4279,"num_write_kb":216992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2358,"num_bytes_recovered":154533888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1075,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407915+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:32:14.407915+0000","last_peered":"2023-10-09T09:32:14.407915+0000","last_clean":"2023-10-09T09:32:14.407915+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:32:14.407915+0000","last_undegraded":"2023-10-09T09:32:14.407915+0000","last_fullsized":"2023-10-09T09:32:14.407915+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1067,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407993+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:32:14.407993+0000","last_peered":"2023-10-09T09:32:14.407993+0000","last_clean":"2023-10-09T09:32:14.407993+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:32:14.407993+0000","last_undegraded":"2023-10-09T09:32:14.407993+0000","last_fullsized":"2023-10-09T09:32:14.407993+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"1117'6540","reported_seq":6575,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:46.473989+0000","last_change":"2023-10-09T09:31:28.850832+0000","last_active":"2023-10-09T09:32:46.473989+0000","last_peered":"2023-10-09T09:32:46.473989+0000","last_clean":"2023-10-09T09:32:46.473989+0000","last_became_active":"2023-10-09T09:31:28.847789+0000","last_became_peered":"2023-10-09T09:31:28.847789+0000","last_unstale":"2023-10-09T09:32:46.473989+0000","last_undegraded":"2023-10-09T09:32:46.473989+0000","last_fullsized":"2023-10-09T09:32:46.473989+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163446784,"num_objects":2494,"num_object_clones":0,"num_object_copies":7482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4372,"num_write_kb":219712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1047,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.423672+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:32:14.423672+0000","last_peered":"2023-10-09T09:32:14.423672+0000","last_clean":"2023-10-09T09:32:14.423672+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:32:14.423672+0000","last_undegraded":"2023-10-09T09:32:14.423672+0000","last_fullsized":"2023-10-09T09:32:14.423672+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1016,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406788+0000","last_change":"2023-10-09T09:15:36.179286+0000","last_active":"2023-10-09T09:32:14.406788+0000","last_peered":"2023-10-09T09:32:14.406788+0000","last_clean":"2023-10-09T09:32:14.406788+0000","last_became_active":"2023-10-09T09:15:36.178840+0000","last_became_peered":"2023-10-09T09:15:36.178840+0000","last_unstale":"2023-10-09T09:32:14.406788+0000","last_undegraded":"2023-10-09T09:32:14.406788+0000","last_fullsized":"2023-10-09T09:32:14.406788+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:01.898916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"1117'6496","reported_seq":6534,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:47.428805+0000","last_change":"2023-10-09T09:31:41.209137+0000","last_active":"2023-10-09T09:32:47.428805+0000","last_peered":"2023-10-09T09:32:47.428805+0000","last_clean":"2023-10-09T09:32:47.428805+0000","last_became_active":"2023-10-09T09:31:28.877504+0000","last_became_peered":"2023-10-09T09:31:28.877504+0000","last_unstale":"2023-10-09T09:32:47.428805+0000","last_undegraded":"2023-10-09T09:32:47.428805+0000","last_fullsized":"2023-10-09T09:32:47.428805+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6496,"log_dups_size":0,"ondisk_log_size":6496,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165314560,"num_objects":2523,"num_object_clones":0,"num_object_copies":7569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4357,"num_write_kb":220128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1045,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.431529+0000","last_change":"2023-10-09T09:15:36.175564+0000","last_active":"2023-10-09T09:32:14.431529+0000","last_peered":"2023-10-09T09:32:14.431529+0000","last_clean":"2023-10-09T09:32:14.431529+0000","last_became_active":"2023-10-09T09:15:36.175242+0000","last_became_peered":"2023-10-09T09:15:36.175242+0000","last_unstale":"2023-10-09T09:32:14.431529+0000","last_undegraded":"2023-10-09T09:32:14.431529+0000","last_fullsized":"2023-10-09T09:32:14.431529+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:47.876857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1068,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406178+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:32:14.406178+0000","last_peered":"2023-10-09T09:32:14.406178+0000","last_clean":"2023-10-09T09:32:14.406178+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:32:14.406178+0000","last_undegraded":"2023-10-09T09:32:14.406178+0000","last_fullsized":"2023-10-09T09:32:14.406178+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"1117'6461","reported_seq":6501,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:46.270791+0000","last_change":"2023-10-09T09:31:33.052832+0000","last_active":"2023-10-09T09:32:46.270791+0000","last_peered":"2023-10-09T09:32:46.270791+0000","last_clean":"2023-10-09T09:32:46.270791+0000","last_became_active":"2023-10-09T09:31:28.853222+0000","last_became_peered":"2023-10-09T09:31:28.853222+0000","last_unstale":"2023-10-09T09:32:46.270791+0000","last_undegraded":"2023-10-09T09:32:46.270791+0000","last_fullsized":"2023-10-09T09:32:46.270791+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"1106'4479","last_scrub_stamp":"2023-10-09T09:31:33.052706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:33.052706+0000","objects_scrubbed":0,"log_size":6461,"log_dups_size":0,"ondisk_log_size":6461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163119104,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2007,"num_write_kb":69632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1030,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.423381+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:32:14.423381+0000","last_peered":"2023-10-09T09:32:14.423381+0000","last_clean":"2023-10-09T09:32:14.423381+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:32:14.423381+0000","last_undegraded":"2023-10-09T09:32:14.423381+0000","last_fullsized":"2023-10-09T09:32:14.423381+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1044,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.432268+0000","last_change":"2023-10-09T09:15:36.177849+0000","last_active":"2023-10-09T09:32:14.432268+0000","last_peered":"2023-10-09T09:32:14.432268+0000","last_clean":"2023-10-09T09:32:14.432268+0000","last_became_active":"2023-10-09T09:15:36.177541+0000","last_became_peered":"2023-10-09T09:15:36.177541+0000","last_unstale":"2023-10-09T09:32:14.432268+0000","last_undegraded":"2023-10-09T09:32:14.432268+0000","last_fullsized":"2023-10-09T09:32:14.432268+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:40.895264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"1117'6415","reported_seq":6453,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:47.245651+0000","last_change":"2023-10-09T09:31:28.901879+0000","last_active":"2023-10-09T09:32:47.245651+0000","last_peered":"2023-10-09T09:32:47.245651+0000","last_clean":"2023-10-09T09:32:47.245651+0000","last_became_active":"2023-10-09T09:31:28.865489+0000","last_became_peered":"2023-10-09T09:31:28.865489+0000","last_unstale":"2023-10-09T09:32:47.245651+0000","last_undegraded":"2023-10-09T09:32:47.245651+0000","last_fullsized":"2023-10-09T09:32:47.245651+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6415,"log_dups_size":0,"ondisk_log_size":6415,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154140672,"num_objects":2352,"num_object_clones":0,"num_object_copies":7056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4288,"num_write_kb":212480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1058,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.428316+0000","last_change":"2023-10-09T09:14:14.173466+0000","last_active":"2023-10-09T09:32:14.428316+0000","last_peered":"2023-10-09T09:32:14.428316+0000","last_clean":"2023-10-09T09:32:14.428316+0000","last_became_active":"2023-10-09T09:14:14.173052+0000","last_became_peered":"2023-10-09T09:14:14.173052+0000","last_unstale":"2023-10-09T09:32:14.428316+0000","last_undegraded":"2023-10-09T09:32:14.428316+0000","last_fullsized":"2023-10-09T09:32:14.428316+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1051,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.091901+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:32:25.091901+0000","last_peered":"2023-10-09T09:32:25.091901+0000","last_clean":"2023-10-09T09:32:25.091901+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:32:25.091901+0000","last_undegraded":"2023-10-09T09:32:25.091901+0000","last_fullsized":"2023-10-09T09:32:25.091901+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1118'8628","reported_seq":8657,"reported_epoch":1118,"state":"active+clean","last_fresh":"2023-10-09T09:32:48.422201+0000","last_change":"2023-10-09T09:30:25.841890+0000","last_active":"2023-10-09T09:32:48.422201+0000","last_peered":"2023-10-09T09:32:48.422201+0000","last_clean":"2023-10-09T09:32:48.422201+0000","last_became_active":"2023-10-09T09:30:25.841584+0000","last_became_peered":"2023-10-09T09:30:25.841584+0000","last_unstale":"2023-10-09T09:32:48.422201+0000","last_undegraded":"2023-10-09T09:32:48.422201+0000","last_fullsized":"2023-10-09T09:32:48.422201+0000","mapping_epoch":1099,"log_start":"1102'1400","ondisk_log_start":"1102'1400","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":7228,"log_dups_size":0,"ondisk_log_size":7228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:07.145206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320602112,"num_objects":4892,"num_object_clones":0,"num_object_copies":14676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8628,"num_write_kb":432640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1046,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.427794+0000","last_change":"2023-10-09T09:14:14.177857+0000","last_active":"2023-10-09T09:32:14.427794+0000","last_peered":"2023-10-09T09:32:14.427794+0000","last_clean":"2023-10-09T09:32:14.427794+0000","last_became_active":"2023-10-09T09:14:14.177403+0000","last_became_peered":"2023-10-09T09:14:14.177403+0000","last_unstale":"2023-10-09T09:32:14.427794+0000","last_undegraded":"2023-10-09T09:32:14.427794+0000","last_fullsized":"2023-10-09T09:32:14.427794+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1053,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.405925+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:32:14.405925+0000","last_peered":"2023-10-09T09:32:14.405925+0000","last_clean":"2023-10-09T09:32:14.405925+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:32:14.405925+0000","last_undegraded":"2023-10-09T09:32:14.405925+0000","last_fullsized":"2023-10-09T09:32:14.405925+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1117'8523","reported_seq":8551,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:44.444891+0000","last_change":"2023-10-09T09:30:25.842825+0000","last_active":"2023-10-09T09:32:44.444891+0000","last_peered":"2023-10-09T09:32:44.444891+0000","last_clean":"2023-10-09T09:32:44.444891+0000","last_became_active":"2023-10-09T09:30:25.842490+0000","last_became_peered":"2023-10-09T09:30:25.842490+0000","last_unstale":"2023-10-09T09:32:44.444891+0000","last_undegraded":"2023-10-09T09:32:44.444891+0000","last_fullsized":"2023-10-09T09:32:44.444891+0000","mapping_epoch":1099,"log_start":"1102'2000","ondisk_log_start":"1102'2000","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6523,"log_dups_size":0,"ondisk_log_size":6523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:14.954211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326434816,"num_objects":4981,"num_object_clones":0,"num_object_copies":14943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8523,"num_write_kb":432128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1047,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.426528+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:32:14.426528+0000","last_peered":"2023-10-09T09:32:14.426528+0000","last_clean":"2023-10-09T09:32:14.426528+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:32:14.426528+0000","last_undegraded":"2023-10-09T09:32:14.426528+0000","last_fullsized":"2023-10-09T09:32:14.426528+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1082,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407725+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:32:14.407725+0000","last_peered":"2023-10-09T09:32:14.407725+0000","last_clean":"2023-10-09T09:32:14.407725+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:32:14.407725+0000","last_undegraded":"2023-10-09T09:32:14.407725+0000","last_fullsized":"2023-10-09T09:32:14.407725+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1117'6451","reported_seq":6493,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:47.460972+0000","last_change":"2023-10-09T09:31:37.508913+0000","last_active":"2023-10-09T09:32:47.460972+0000","last_peered":"2023-10-09T09:32:47.460972+0000","last_clean":"2023-10-09T09:32:47.460972+0000","last_became_active":"2023-10-09T09:31:28.853535+0000","last_became_peered":"2023-10-09T09:31:28.853535+0000","last_unstale":"2023-10-09T09:32:47.460972+0000","last_undegraded":"2023-10-09T09:32:47.460972+0000","last_fullsized":"2023-10-09T09:32:47.460972+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6451,"log_dups_size":0,"ondisk_log_size":6451,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157286400,"num_objects":2400,"num_object_clones":0,"num_object_copies":7200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4326,"num_write_kb":215232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1031,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.426954+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:32:14.426954+0000","last_peered":"2023-10-09T09:32:14.426954+0000","last_clean":"2023-10-09T09:32:14.426954+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:32:14.426954+0000","last_undegraded":"2023-10-09T09:32:14.426954+0000","last_fullsized":"2023-10-09T09:32:14.426954+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1049,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.428581+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:32:14.428581+0000","last_peered":"2023-10-09T09:32:14.428581+0000","last_clean":"2023-10-09T09:32:14.428581+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:32:14.428581+0000","last_undegraded":"2023-10-09T09:32:14.428581+0000","last_fullsized":"2023-10-09T09:32:14.428581+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1117'6569","reported_seq":6608,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:45.284572+0000","last_change":"2023-10-09T09:31:31.170450+0000","last_active":"2023-10-09T09:32:45.284572+0000","last_peered":"2023-10-09T09:32:45.284572+0000","last_clean":"2023-10-09T09:32:45.284572+0000","last_became_active":"2023-10-09T09:31:28.843228+0000","last_became_peered":"2023-10-09T09:31:28.843228+0000","last_unstale":"2023-10-09T09:32:45.284572+0000","last_undegraded":"2023-10-09T09:32:45.284572+0000","last_fullsized":"2023-10-09T09:32:45.284572+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163414016,"num_objects":2493,"num_object_clones":0,"num_object_copies":7479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4347,"num_write_kb":218912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1069,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406073+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:32:14.406073+0000","last_peered":"2023-10-09T09:32:14.406073+0000","last_clean":"2023-10-09T09:32:14.406073+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:32:14.406073+0000","last_undegraded":"2023-10-09T09:32:14.406073+0000","last_fullsized":"2023-10-09T09:32:14.406073+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1045,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.431601+0000","last_change":"2023-10-09T09:14:40.487437+0000","last_active":"2023-10-09T09:32:14.431601+0000","last_peered":"2023-10-09T09:32:14.431601+0000","last_clean":"2023-10-09T09:32:14.431601+0000","last_became_active":"2023-10-09T09:14:40.487165+0000","last_became_peered":"2023-10-09T09:14:40.487165+0000","last_unstale":"2023-10-09T09:32:14.431601+0000","last_undegraded":"2023-10-09T09:32:14.431601+0000","last_fullsized":"2023-10-09T09:32:14.431601+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:08.168726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1117'8686","reported_seq":8714,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:47.419436+0000","last_change":"2023-10-09T09:30:25.837579+0000","last_active":"2023-10-09T09:32:47.419436+0000","last_peered":"2023-10-09T09:32:47.419436+0000","last_clean":"2023-10-09T09:32:47.419436+0000","last_became_active":"2023-10-09T09:30:25.836784+0000","last_became_peered":"2023-10-09T09:30:25.836784+0000","last_unstale":"2023-10-09T09:32:47.419436+0000","last_undegraded":"2023-10-09T09:32:47.419436+0000","last_fullsized":"2023-10-09T09:32:47.419436+0000","mapping_epoch":1099,"log_start":"1102'1100","ondisk_log_start":"1102'1100","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":7586,"log_dups_size":0,"ondisk_log_size":7586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:55.296361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319422464,"num_objects":4874,"num_object_clones":0,"num_object_copies":14622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8686,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":1034,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407072+0000","last_change":"2023-10-09T09:15:36.180317+0000","last_active":"2023-10-09T09:32:14.407072+0000","last_peered":"2023-10-09T09:32:14.407072+0000","last_clean":"2023-10-09T09:32:14.407072+0000","last_became_active":"2023-10-09T09:15:36.179906+0000","last_became_peered":"2023-10-09T09:15:36.179906+0000","last_unstale":"2023-10-09T09:32:14.407072+0000","last_undegraded":"2023-10-09T09:32:14.407072+0000","last_fullsized":"2023-10-09T09:32:14.407072+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:37.327102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1067,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407991+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:32:14.407991+0000","last_peered":"2023-10-09T09:32:14.407991+0000","last_clean":"2023-10-09T09:32:14.407991+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:32:14.407991+0000","last_undegraded":"2023-10-09T09:32:14.407991+0000","last_fullsized":"2023-10-09T09:32:14.407991+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1117'8616","reported_seq":8644,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:45.264848+0000","last_change":"2023-10-09T09:30:25.841208+0000","last_active":"2023-10-09T09:32:45.264848+0000","last_peered":"2023-10-09T09:32:45.264848+0000","last_clean":"2023-10-09T09:32:45.264848+0000","last_became_active":"2023-10-09T09:30:25.840860+0000","last_became_peered":"2023-10-09T09:30:25.840860+0000","last_unstale":"2023-10-09T09:32:45.264848+0000","last_undegraded":"2023-10-09T09:32:45.264848+0000","last_fullsized":"2023-10-09T09:32:45.264848+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8616,"log_dups_size":0,"ondisk_log_size":8616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:20.488338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330170368,"num_objects":5038,"num_object_clones":0,"num_object_copies":15114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8616,"num_write_kb":436928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1050,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.428767+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:32:14.428767+0000","last_peered":"2023-10-09T09:32:14.428767+0000","last_clean":"2023-10-09T09:32:14.428767+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:32:14.428767+0000","last_undegraded":"2023-10-09T09:32:14.428767+0000","last_fullsized":"2023-10-09T09:32:14.428767+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1082,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.408137+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:32:14.408137+0000","last_peered":"2023-10-09T09:32:14.408137+0000","last_clean":"2023-10-09T09:32:14.408137+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:32:14.408137+0000","last_undegraded":"2023-10-09T09:32:14.408137+0000","last_fullsized":"2023-10-09T09:32:14.408137+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1117'8789","reported_seq":8816,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:46.508894+0000","last_change":"2023-10-09T09:30:25.838078+0000","last_active":"2023-10-09T09:32:46.508894+0000","last_peered":"2023-10-09T09:32:46.508894+0000","last_clean":"2023-10-09T09:32:46.508894+0000","last_became_active":"2023-10-09T09:30:25.837594+0000","last_became_peered":"2023-10-09T09:30:25.837594+0000","last_unstale":"2023-10-09T09:32:46.508894+0000","last_undegraded":"2023-10-09T09:32:46.508894+0000","last_fullsized":"2023-10-09T09:32:46.508894+0000","mapping_epoch":1099,"log_start":"1102'200","ondisk_log_start":"1102'200","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8589,"log_dups_size":0,"ondisk_log_size":8589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326893592,"num_objects":4989,"num_object_clones":0,"num_object_copies":14967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4989,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8789,"num_write_kb":440833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1068,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407485+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:32:14.407485+0000","last_peered":"2023-10-09T09:32:14.407485+0000","last_clean":"2023-10-09T09:32:14.407485+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:32:14.407485+0000","last_undegraded":"2023-10-09T09:32:14.407485+0000","last_fullsized":"2023-10-09T09:32:14.407485+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1067,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.408073+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:32:14.408073+0000","last_peered":"2023-10-09T09:32:14.408073+0000","last_clean":"2023-10-09T09:32:14.408073+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:32:14.408073+0000","last_undegraded":"2023-10-09T09:32:14.408073+0000","last_fullsized":"2023-10-09T09:32:14.408073+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1117'8728","reported_seq":8754,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:46.552892+0000","last_change":"2023-10-09T09:30:25.841044+0000","last_active":"2023-10-09T09:32:46.552892+0000","last_peered":"2023-10-09T09:32:46.552892+0000","last_clean":"2023-10-09T09:32:46.552892+0000","last_became_active":"2023-10-09T09:30:25.840627+0000","last_became_peered":"2023-10-09T09:30:25.840627+0000","last_unstale":"2023-10-09T09:32:46.552892+0000","last_undegraded":"2023-10-09T09:32:46.552892+0000","last_fullsized":"2023-10-09T09:32:46.552892+0000","mapping_epoch":1099,"log_start":"1102'100","ondisk_log_start":"1102'100","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8628,"log_dups_size":0,"ondisk_log_size":8628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326893568,"num_objects":4988,"num_object_clones":0,"num_object_copies":14964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8728,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1017,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406682+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:32:14.406682+0000","last_peered":"2023-10-09T09:32:14.406682+0000","last_clean":"2023-10-09T09:32:14.406682+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:32:14.406682+0000","last_undegraded":"2023-10-09T09:32:14.406682+0000","last_fullsized":"2023-10-09T09:32:14.406682+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1056,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:43.572109+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:32:43.572109+0000","last_peered":"2023-10-09T09:32:43.572109+0000","last_clean":"2023-10-09T09:32:43.572109+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:32:43.572109+0000","last_undegraded":"2023-10-09T09:32:43.572109+0000","last_fullsized":"2023-10-09T09:32:43.572109+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1117'6519","reported_seq":6559,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:46.293701+0000","last_change":"2023-10-09T09:31:32.026991+0000","last_active":"2023-10-09T09:32:46.293701+0000","last_peered":"2023-10-09T09:32:46.293701+0000","last_clean":"2023-10-09T09:32:46.293701+0000","last_became_active":"2023-10-09T09:31:28.828078+0000","last_became_peered":"2023-10-09T09:31:28.828078+0000","last_unstale":"2023-10-09T09:32:46.293701+0000","last_undegraded":"2023-10-09T09:32:46.293701+0000","last_fullsized":"2023-10-09T09:32:46.293701+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1106'4471","last_scrub_stamp":"2023-10-09T09:31:32.026835+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:32.026835+0000","objects_scrubbed":0,"log_size":6519,"log_dups_size":0,"ondisk_log_size":6519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165216256,"num_objects":2521,"num_object_clones":0,"num_object_copies":7563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2070,"num_write_kb":73216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1051,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.108378+0000","last_change":"2023-10-09T09:14:40.491661+0000","last_active":"2023-10-09T09:32:25.108378+0000","last_peered":"2023-10-09T09:32:25.108378+0000","last_clean":"2023-10-09T09:32:25.108378+0000","last_became_active":"2023-10-09T09:14:40.490392+0000","last_became_peered":"2023-10-09T09:14:40.490392+0000","last_unstale":"2023-10-09T09:32:25.108378+0000","last_undegraded":"2023-10-09T09:32:25.108378+0000","last_fullsized":"2023-10-09T09:32:25.108378+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:40.491764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1084,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406273+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:32:14.406273+0000","last_peered":"2023-10-09T09:32:14.406273+0000","last_clean":"2023-10-09T09:32:14.406273+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:32:14.406273+0000","last_undegraded":"2023-10-09T09:32:14.406273+0000","last_fullsized":"2023-10-09T09:32:14.406273+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1041,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.105621+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:32:25.105621+0000","last_peered":"2023-10-09T09:32:25.105621+0000","last_clean":"2023-10-09T09:32:25.105621+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:32:25.105621+0000","last_undegraded":"2023-10-09T09:32:25.105621+0000","last_fullsized":"2023-10-09T09:32:25.105621+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1117'6468","reported_seq":6538,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:46.495971+0000","last_change":"2023-10-09T09:32:11.741204+0000","last_active":"2023-10-09T09:32:46.495971+0000","last_peered":"2023-10-09T09:32:46.495971+0000","last_clean":"2023-10-09T09:32:46.495971+0000","last_became_active":"2023-10-09T09:31:28.823603+0000","last_became_peered":"2023-10-09T09:31:28.823603+0000","last_unstale":"2023-10-09T09:32:46.495971+0000","last_undegraded":"2023-10-09T09:32:46.495971+0000","last_fullsized":"2023-10-09T09:32:46.495971+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6468,"log_dups_size":0,"ondisk_log_size":6468,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160038912,"num_objects":2442,"num_object_clones":0,"num_object_copies":7326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4300,"num_write_kb":215744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1073,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.405981+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:32:14.405981+0000","last_peered":"2023-10-09T09:32:14.405981+0000","last_clean":"2023-10-09T09:32:14.405981+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:32:14.405981+0000","last_undegraded":"2023-10-09T09:32:14.405981+0000","last_fullsized":"2023-10-09T09:32:14.405981+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1068,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407884+0000","last_change":"2023-10-09T09:14:14.171348+0000","last_active":"2023-10-09T09:32:14.407884+0000","last_peered":"2023-10-09T09:32:14.407884+0000","last_clean":"2023-10-09T09:32:14.407884+0000","last_became_active":"2023-10-09T09:14:14.170963+0000","last_became_peered":"2023-10-09T09:14:14.170963+0000","last_unstale":"2023-10-09T09:32:14.407884+0000","last_undegraded":"2023-10-09T09:32:14.407884+0000","last_fullsized":"2023-10-09T09:32:14.407884+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1071,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.104117+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:32:25.104117+0000","last_peered":"2023-10-09T09:32:25.104117+0000","last_clean":"2023-10-09T09:32:25.104117+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:32:25.104117+0000","last_undegraded":"2023-10-09T09:32:25.104117+0000","last_fullsized":"2023-10-09T09:32:25.104117+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1117'6305","reported_seq":6347,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:47.416581+0000","last_change":"2023-10-09T09:31:33.352384+0000","last_active":"2023-10-09T09:32:47.416581+0000","last_peered":"2023-10-09T09:32:47.416581+0000","last_clean":"2023-10-09T09:32:47.416581+0000","last_became_active":"2023-10-09T09:31:28.838318+0000","last_became_peered":"2023-10-09T09:31:28.838318+0000","last_unstale":"2023-10-09T09:32:47.416581+0000","last_undegraded":"2023-10-09T09:32:47.416581+0000","last_fullsized":"2023-10-09T09:32:47.416581+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6305,"log_dups_size":0,"ondisk_log_size":6305,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159907840,"num_objects":2440,"num_object_clones":0,"num_object_copies":7320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4178,"num_write_kb":211776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1037,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.408533+0000","last_change":"2023-10-09T09:15:36.166258+0000","last_active":"2023-10-09T09:32:14.408533+0000","last_peered":"2023-10-09T09:32:14.408533+0000","last_clean":"2023-10-09T09:32:14.408533+0000","last_became_active":"2023-10-09T09:15:36.165859+0000","last_became_peered":"2023-10-09T09:15:36.165859+0000","last_unstale":"2023-10-09T09:32:14.408533+0000","last_undegraded":"2023-10-09T09:32:14.408533+0000","last_fullsized":"2023-10-09T09:32:14.408533+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:24.262546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1057,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:43.571946+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:32:43.571946+0000","last_peered":"2023-10-09T09:32:43.571946+0000","last_clean":"2023-10-09T09:32:43.571946+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:32:43.571946+0000","last_undegraded":"2023-10-09T09:32:43.571946+0000","last_fullsized":"2023-10-09T09:32:43.571946+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1046,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.108198+0000","last_change":"2023-10-09T09:14:40.485205+0000","last_active":"2023-10-09T09:32:25.108198+0000","last_peered":"2023-10-09T09:32:25.108198+0000","last_clean":"2023-10-09T09:32:25.108198+0000","last_became_active":"2023-10-09T09:14:40.484702+0000","last_became_peered":"2023-10-09T09:14:40.484702+0000","last_unstale":"2023-10-09T09:32:25.108198+0000","last_undegraded":"2023-10-09T09:32:25.108198+0000","last_fullsized":"2023-10-09T09:32:25.108198+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:52.753466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1117'6519","reported_seq":6558,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:47.410174+0000","last_change":"2023-10-09T09:31:41.209655+0000","last_active":"2023-10-09T09:32:47.410174+0000","last_peered":"2023-10-09T09:32:47.410174+0000","last_clean":"2023-10-09T09:32:47.410174+0000","last_became_active":"2023-10-09T09:31:28.832534+0000","last_became_peered":"2023-10-09T09:31:28.832534+0000","last_unstale":"2023-10-09T09:32:47.410174+0000","last_undegraded":"2023-10-09T09:32:47.410174+0000","last_fullsized":"2023-10-09T09:32:47.410174+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6519,"log_dups_size":0,"ondisk_log_size":6519,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159088640,"num_objects":2427,"num_object_clones":0,"num_object_copies":7281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4379,"num_write_kb":217824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1073,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406352+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:32:14.406352+0000","last_peered":"2023-10-09T09:32:14.406352+0000","last_clean":"2023-10-09T09:32:14.406352+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:32:14.406352+0000","last_undegraded":"2023-10-09T09:32:14.406352+0000","last_fullsized":"2023-10-09T09:32:14.406352+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1281,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.423651+0000","last_change":"2023-10-09T09:14:14.176150+0000","last_active":"2023-10-09T09:32:14.423651+0000","last_peered":"2023-10-09T09:32:14.423651+0000","last_clean":"2023-10-09T09:32:14.423651+0000","last_became_active":"2023-10-09T09:14:14.174942+0000","last_became_peered":"2023-10-09T09:14:14.174942+0000","last_unstale":"2023-10-09T09:32:14.423651+0000","last_undegraded":"2023-10-09T09:32:14.423651+0000","last_fullsized":"2023-10-09T09:32:14.423651+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1029,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407326+0000","last_change":"2023-10-09T09:15:36.173154+0000","last_active":"2023-10-09T09:32:14.407326+0000","last_peered":"2023-10-09T09:32:14.407326+0000","last_clean":"2023-10-09T09:32:14.407326+0000","last_became_active":"2023-10-09T09:15:36.171641+0000","last_became_peered":"2023-10-09T09:15:36.171641+0000","last_unstale":"2023-10-09T09:32:14.407326+0000","last_undegraded":"2023-10-09T09:32:14.407326+0000","last_fullsized":"2023-10-09T09:32:14.407326+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:20.427384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1034,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.429016+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:32:14.429016+0000","last_peered":"2023-10-09T09:32:14.429016+0000","last_clean":"2023-10-09T09:32:14.429016+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:32:14.429016+0000","last_undegraded":"2023-10-09T09:32:14.429016+0000","last_fullsized":"2023-10-09T09:32:14.429016+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1117'6416","reported_seq":6457,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:46.270796+0000","last_change":"2023-10-09T09:31:29.845811+0000","last_active":"2023-10-09T09:32:46.270796+0000","last_peered":"2023-10-09T09:32:46.270796+0000","last_clean":"2023-10-09T09:32:46.270796+0000","last_became_active":"2023-10-09T09:31:28.832067+0000","last_became_peered":"2023-10-09T09:31:28.832067+0000","last_unstale":"2023-10-09T09:32:46.270796+0000","last_undegraded":"2023-10-09T09:32:46.270796+0000","last_fullsized":"2023-10-09T09:32:46.270796+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1106'4346","last_scrub_stamp":"2023-10-09T09:31:29.845731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:29.845731+0000","objects_scrubbed":0,"log_size":6416,"log_dups_size":0,"ondisk_log_size":6416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159711232,"num_objects":2437,"num_object_clones":0,"num_object_copies":7311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2082,"num_write_kb":75328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1064,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.423437+0000","last_change":"2023-10-09T09:14:40.485057+0000","last_active":"2023-10-09T09:32:14.423437+0000","last_peered":"2023-10-09T09:32:14.423437+0000","last_clean":"2023-10-09T09:32:14.423437+0000","last_became_active":"2023-10-09T09:14:40.484662+0000","last_became_peered":"2023-10-09T09:14:40.484662+0000","last_unstale":"2023-10-09T09:32:14.423437+0000","last_undegraded":"2023-10-09T09:32:14.423437+0000","last_fullsized":"2023-10-09T09:32:14.423437+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1046,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.432952+0000","last_change":"2023-10-09T09:15:36.181495+0000","last_active":"2023-10-09T09:32:14.432952+0000","last_peered":"2023-10-09T09:32:14.432952+0000","last_clean":"2023-10-09T09:32:14.432952+0000","last_became_active":"2023-10-09T09:15:36.181161+0000","last_became_peered":"2023-10-09T09:15:36.181161+0000","last_unstale":"2023-10-09T09:32:14.432952+0000","last_undegraded":"2023-10-09T09:32:14.432952+0000","last_fullsized":"2023-10-09T09:32:14.432952+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:47.309002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1057,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.432071+0000","last_change":"2023-10-09T09:15:07.212839+0000","last_active":"2023-10-09T09:32:14.432071+0000","last_peered":"2023-10-09T09:32:14.432071+0000","last_clean":"2023-10-09T09:32:14.432071+0000","last_became_active":"2023-10-09T09:15:07.212468+0000","last_became_peered":"2023-10-09T09:15:07.212468+0000","last_unstale":"2023-10-09T09:32:14.432071+0000","last_undegraded":"2023-10-09T09:32:14.432071+0000","last_fullsized":"2023-10-09T09:32:14.432071+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:26.746017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"1117'6496","reported_seq":6536,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:46.316796+0000","last_change":"2023-10-09T09:31:29.987166+0000","last_active":"2023-10-09T09:32:46.316796+0000","last_peered":"2023-10-09T09:32:46.316796+0000","last_clean":"2023-10-09T09:32:46.316796+0000","last_became_active":"2023-10-09T09:31:28.836386+0000","last_became_peered":"2023-10-09T09:31:28.836386+0000","last_unstale":"2023-10-09T09:32:46.316796+0000","last_undegraded":"2023-10-09T09:32:46.316796+0000","last_fullsized":"2023-10-09T09:32:46.316796+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1106'4321","last_scrub_stamp":"2023-10-09T09:31:29.987092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:29.987092+0000","objects_scrubbed":0,"log_size":6496,"log_dups_size":0,"ondisk_log_size":6496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160432128,"num_objects":2448,"num_object_clones":0,"num_object_copies":7344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2194,"num_write_kb":79552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1049,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.427075+0000","last_change":"2023-10-09T09:14:14.172733+0000","last_active":"2023-10-09T09:32:14.427075+0000","last_peered":"2023-10-09T09:32:14.427075+0000","last_clean":"2023-10-09T09:32:14.427075+0000","last_became_active":"2023-10-09T09:14:14.172459+0000","last_became_peered":"2023-10-09T09:14:14.172459+0000","last_unstale":"2023-10-09T09:32:14.427075+0000","last_undegraded":"2023-10-09T09:32:14.427075+0000","last_fullsized":"2023-10-09T09:32:14.427075+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1057,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.108286+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:32:25.108286+0000","last_peered":"2023-10-09T09:32:25.108286+0000","last_clean":"2023-10-09T09:32:25.108286+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:32:25.108286+0000","last_undegraded":"2023-10-09T09:32:25.108286+0000","last_fullsized":"2023-10-09T09:32:25.108286+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1030,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.431857+0000","last_change":"2023-10-09T09:14:14.173766+0000","last_active":"2023-10-09T09:32:14.431857+0000","last_peered":"2023-10-09T09:32:14.431857+0000","last_clean":"2023-10-09T09:32:14.431857+0000","last_became_active":"2023-10-09T09:14:14.172730+0000","last_became_peered":"2023-10-09T09:14:14.172730+0000","last_unstale":"2023-10-09T09:32:14.431857+0000","last_undegraded":"2023-10-09T09:32:14.431857+0000","last_fullsized":"2023-10-09T09:32:14.431857+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.693902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1117'6476","reported_seq":6514,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:44.409026+0000","last_change":"2023-10-09T09:31:28.883097+0000","last_active":"2023-10-09T09:32:44.409026+0000","last_peered":"2023-10-09T09:32:44.409026+0000","last_clean":"2023-10-09T09:32:44.409026+0000","last_became_active":"2023-10-09T09:31:28.861343+0000","last_became_peered":"2023-10-09T09:31:28.861343+0000","last_unstale":"2023-10-09T09:32:44.409026+0000","last_undegraded":"2023-10-09T09:32:44.409026+0000","last_fullsized":"2023-10-09T09:32:44.409026+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6476,"log_dups_size":0,"ondisk_log_size":6476,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168755200,"num_objects":2575,"num_object_clones":0,"num_object_copies":7725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4287,"num_write_kb":219584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":1034,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407345+0000","last_change":"2023-10-09T09:15:36.176435+0000","last_active":"2023-10-09T09:32:14.407345+0000","last_peered":"2023-10-09T09:32:14.407345+0000","last_clean":"2023-10-09T09:32:14.407345+0000","last_became_active":"2023-10-09T09:15:36.175360+0000","last_became_peered":"2023-10-09T09:15:36.175360+0000","last_unstale":"2023-10-09T09:32:14.407345+0000","last_undegraded":"2023-10-09T09:32:14.407345+0000","last_fullsized":"2023-10-09T09:32:14.407345+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1040,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.108082+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:32:25.108082+0000","last_peered":"2023-10-09T09:32:25.108082+0000","last_clean":"2023-10-09T09:32:25.108082+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:32:25.108082+0000","last_undegraded":"2023-10-09T09:32:25.108082+0000","last_fullsized":"2023-10-09T09:32:25.108082+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1032,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406585+0000","last_change":"2023-10-09T09:15:36.180495+0000","last_active":"2023-10-09T09:32:14.406585+0000","last_peered":"2023-10-09T09:32:14.406585+0000","last_clean":"2023-10-09T09:32:14.406585+0000","last_became_active":"2023-10-09T09:15:36.179861+0000","last_became_peered":"2023-10-09T09:15:36.179861+0000","last_unstale":"2023-10-09T09:32:14.406585+0000","last_undegraded":"2023-10-09T09:32:14.406585+0000","last_fullsized":"2023-10-09T09:32:14.406585+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.686677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1117'6308","reported_seq":6404,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:44.423001+0000","last_change":"2023-10-09T09:31:37.174228+0000","last_active":"2023-10-09T09:32:44.423001+0000","last_peered":"2023-10-09T09:32:44.423001+0000","last_clean":"2023-10-09T09:32:44.423001+0000","last_became_active":"2023-10-09T09:31:28.822827+0000","last_became_peered":"2023-10-09T09:31:28.822827+0000","last_unstale":"2023-10-09T09:32:44.423001+0000","last_undegraded":"2023-10-09T09:32:44.423001+0000","last_fullsized":"2023-10-09T09:32:44.423001+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1109'4470","last_scrub_stamp":"2023-10-09T09:31:37.173935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:37.173935+0000","objects_scrubbed":2369,"log_size":6308,"log_dups_size":0,"ondisk_log_size":6308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:32.212068+0000","scrub_duration":0.64874551999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840000,"num_objects":2500,"num_object_clones":0,"num_object_copies":7500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1849,"num_write_kb":63360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1053,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.428999+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:32:14.428999+0000","last_peered":"2023-10-09T09:32:14.428999+0000","last_clean":"2023-10-09T09:32:14.428999+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:32:14.428999+0000","last_undegraded":"2023-10-09T09:32:14.428999+0000","last_fullsized":"2023-10-09T09:32:14.428999+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1049,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.431990+0000","last_change":"2023-10-09T09:14:40.488505+0000","last_active":"2023-10-09T09:32:14.431990+0000","last_peered":"2023-10-09T09:32:14.431990+0000","last_clean":"2023-10-09T09:32:14.431990+0000","last_became_active":"2023-10-09T09:14:40.488140+0000","last_became_peered":"2023-10-09T09:14:40.488140+0000","last_unstale":"2023-10-09T09:32:14.431990+0000","last_undegraded":"2023-10-09T09:32:14.431990+0000","last_fullsized":"2023-10-09T09:32:14.431990+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:27.267698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1052,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407595+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:32:14.407595+0000","last_peered":"2023-10-09T09:32:14.407595+0000","last_clean":"2023-10-09T09:32:14.407595+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:32:14.407595+0000","last_undegraded":"2023-10-09T09:32:14.407595+0000","last_fullsized":"2023-10-09T09:32:14.407595+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1033,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.431768+0000","last_change":"2023-10-09T09:14:14.173309+0000","last_active":"2023-10-09T09:32:14.431768+0000","last_peered":"2023-10-09T09:32:14.431768+0000","last_clean":"2023-10-09T09:32:14.431768+0000","last_became_active":"2023-10-09T09:14:14.171919+0000","last_became_peered":"2023-10-09T09:14:14.171919+0000","last_unstale":"2023-10-09T09:32:14.431768+0000","last_undegraded":"2023-10-09T09:32:14.431768+0000","last_fullsized":"2023-10-09T09:32:14.431768+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:49.766743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1055,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.098016+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:32:25.098016+0000","last_peered":"2023-10-09T09:32:25.098016+0000","last_clean":"2023-10-09T09:32:25.098016+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:32:25.098016+0000","last_undegraded":"2023-10-09T09:32:25.098016+0000","last_fullsized":"2023-10-09T09:32:25.098016+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"1117'6641","reported_seq":6742,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:45.250999+0000","last_change":"2023-10-09T09:31:33.829441+0000","last_active":"2023-10-09T09:32:45.250999+0000","last_peered":"2023-10-09T09:32:45.250999+0000","last_clean":"2023-10-09T09:32:45.250999+0000","last_became_active":"2023-10-09T09:31:28.863946+0000","last_became_peered":"2023-10-09T09:31:28.863946+0000","last_unstale":"2023-10-09T09:32:45.250999+0000","last_undegraded":"2023-10-09T09:32:45.250999+0000","last_fullsized":"2023-10-09T09:32:45.250999+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"1106'4655","last_scrub_stamp":"2023-10-09T09:31:33.829271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:33.829271+0000","objects_scrubbed":2411,"log_size":6641,"log_dups_size":0,"ondisk_log_size":6641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:23.193885+0000","scrub_duration":0.70828463799999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164102144,"num_objects":2504,"num_object_clones":0,"num_object_copies":7512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1993,"num_write_kb":66752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1071,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406026+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:32:14.406026+0000","last_peered":"2023-10-09T09:32:14.406026+0000","last_clean":"2023-10-09T09:32:14.406026+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:32:14.406026+0000","last_undegraded":"2023-10-09T09:32:14.406026+0000","last_fullsized":"2023-10-09T09:32:14.406026+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1082,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.408187+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:32:14.408187+0000","last_peered":"2023-10-09T09:32:14.408187+0000","last_clean":"2023-10-09T09:32:14.408187+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:32:14.408187+0000","last_undegraded":"2023-10-09T09:32:14.408187+0000","last_fullsized":"2023-10-09T09:32:14.408187+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"1117'6393","reported_seq":6429,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:47.194228+0000","last_change":"2023-10-09T09:31:28.878549+0000","last_active":"2023-10-09T09:32:47.194228+0000","last_peered":"2023-10-09T09:32:47.194228+0000","last_clean":"2023-10-09T09:32:47.194228+0000","last_became_active":"2023-10-09T09:31:28.876555+0000","last_became_peered":"2023-10-09T09:31:28.876555+0000","last_unstale":"2023-10-09T09:32:47.194228+0000","last_undegraded":"2023-10-09T09:32:47.194228+0000","last_fullsized":"2023-10-09T09:32:47.194228+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6393,"log_dups_size":0,"ondisk_log_size":6393,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164102144,"num_objects":2504,"num_object_clones":0,"num_object_copies":7512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4268,"num_write_kb":216704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1056,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.406371+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:32:14.406371+0000","last_peered":"2023-10-09T09:32:14.406371+0000","last_clean":"2023-10-09T09:32:14.406371+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:32:14.406371+0000","last_undegraded":"2023-10-09T09:32:14.406371+0000","last_fullsized":"2023-10-09T09:32:14.406371+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1082,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407619+0000","last_change":"2023-10-09T09:15:36.180777+0000","last_active":"2023-10-09T09:32:14.407619+0000","last_peered":"2023-10-09T09:32:14.407619+0000","last_clean":"2023-10-09T09:32:14.407619+0000","last_became_active":"2023-10-09T09:15:36.179828+0000","last_became_peered":"2023-10-09T09:15:36.179828+0000","last_unstale":"2023-10-09T09:32:14.407619+0000","last_undegraded":"2023-10-09T09:32:14.407619+0000","last_fullsized":"2023-10-09T09:32:14.407619+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":1045,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407384+0000","last_change":"2023-10-09T09:15:36.172064+0000","last_active":"2023-10-09T09:32:14.407384+0000","last_peered":"2023-10-09T09:32:14.407384+0000","last_clean":"2023-10-09T09:32:14.407384+0000","last_became_active":"2023-10-09T09:15:36.171612+0000","last_became_peered":"2023-10-09T09:15:36.171612+0000","last_unstale":"2023-10-09T09:32:14.407384+0000","last_undegraded":"2023-10-09T09:32:14.407384+0000","last_fullsized":"2023-10-09T09:32:14.407384+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:36.980625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1057,"reported_epoch":1117,"state":"active+clean","last_fresh":"2023-10-09T09:32:25.090553+0000","last_change":"2023-10-09T09:15:36.173278+0000","last_active":"2023-10-09T09:32:25.090553+0000","last_peered":"2023-10-09T09:32:25.090553+0000","last_clean":"2023-10-09T09:32:25.090553+0000","last_became_active":"2023-10-09T09:15:36.172245+0000","last_became_peered":"2023-10-09T09:15:36.172245+0000","last_unstale":"2023-10-09T09:32:25.090553+0000","last_undegraded":"2023-10-09T09:32:25.090553+0000","last_fullsized":"2023-10-09T09:32:25.090553+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:54.806047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1052,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.432033+0000","last_change":"2023-10-09T09:15:07.213452+0000","last_active":"2023-10-09T09:32:14.432033+0000","last_peered":"2023-10-09T09:32:14.432033+0000","last_clean":"2023-10-09T09:32:14.432033+0000","last_became_active":"2023-10-09T09:15:07.213117+0000","last_became_peered":"2023-10-09T09:15:07.213117+0000","last_unstale":"2023-10-09T09:32:14.432033+0000","last_undegraded":"2023-10-09T09:32:14.432033+0000","last_fullsized":"2023-10-09T09:32:14.432033+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:45.351875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1067,"reported_epoch":1116,"state":"active+clean","last_fresh":"2023-10-09T09:32:14.407272+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:32:14.407272+0000","last_peered":"2023-10-09T09:32:14.407272+0000","last_clean":"2023-10-09T09:32:14.407272+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:32:14.407272+0000","last_undegraded":"2023-10-09T09:32:14.407272+0000","last_fullsized":"2023-10-09T09:32:14.407272+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":5193957400,"num_objects":79255,"num_object_clones":0,"num_object_copies":237765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":887,"num_objects_misplaced":3971,"num_objects_unfound":0,"num_objects_dirty":79255,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124381,"num_write_kb":6077089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4515,"num_bytes_recovered":295895040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15930961920,"data_stored":15930949704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":176361,"ondisk_log_size":176361,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397892,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2053176,"kb_used_data":1743664,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91691976,"statfs":{"total":95995035648,"available":93892583424,"internally_reserved":0,"allocated":1785511936,"data_stored":1785289259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4359999999999999,"5min":2.9369999999999998,"15min":1.988},"min":{"1min":0.35499999999999998,"5min":0.249,"15min":0.249},"max":{"1min":4.3650000000000002,"5min":85.811999999999998,"15min":85.811999999999998},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.496,"5min":3.077,"15min":2.117},"min":{"1min":0.47299999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.4199999999999999,"5min":85.762,"15min":85.762},"last":0.67500000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:32:21 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":1.5569999999999999,"15min":1.5640000000000001},"min":{"1min":0.33400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8730000000000002,"5min":8.2449999999999992,"15min":103.84699999999999},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.29,"5min":1.395,"15min":1.589},"min":{"1min":0.33100000000000002,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":5.3120000000000003,"5min":9.8190000000000008,"15min":103.955},"last":0.71699999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:32:35 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":2.3570000000000002,"15min":1.6859999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.31900000000000001},"max":{"1min":5.1959999999999997,"5min":47.023000000000003,"15min":47.023000000000003},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.3069999999999999,"5min":2.1890000000000001,"15min":1.5820000000000001},"min":{"1min":0.375,"5min":0.373,"15min":0.373},"max":{"1min":5.1689999999999996,"5min":46.847000000000001,"15min":46.847000000000001},"last":0.79700000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:32:15 2023","interfaces":[{"interface":"back","average":{"1min":5.141,"5min":3.4550000000000001,"15min":2.5539999999999998},"min":{"1min":0.46400000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":84.590999999999994,"5min":84.590999999999994,"15min":107.97499999999999},"last":1.964},{"interface":"front","average":{"1min":5.1740000000000004,"5min":3.4710000000000001,"15min":2.657},"min":{"1min":0.40899999999999997,"5min":0.317,"15min":0.317},"max":{"1min":84.713999999999999,"5min":84.713999999999999,"15min":105.282},"last":0.89600000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.385,"15min":1.153},"min":{"1min":0.35299999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.1989999999999998,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":1.077},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.1240000000000001,"15min":1.0640000000000001},"min":{"1min":0.46700000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.3399999999999999,"5min":12.316000000000001,"15min":12.316000000000001},"last":0.86699999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.4450000000000001,"15min":1.2949999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":3.9630000000000001,"5min":6.4500000000000002,"15min":10.063000000000001},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.218,"15min":1.1619999999999999},"min":{"1min":0.39800000000000002,"5min":0.38900000000000001,"15min":0.36299999999999999},"max":{"1min":3.52,"5min":8.1699999999999999,"15min":8.1699999999999999},"last":0.90600000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":1.296,"15min":1.198},"min":{"1min":0.40500000000000003,"5min":0.373,"15min":0.373},"max":{"1min":3.5590000000000002,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":1.4419999999999999},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.371,"15min":1.292},"min":{"1min":0.49299999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.8959999999999999,"5min":8.1980000000000004,"15min":8.1980000000000004},"last":0.97399999999999998}]}]},{"osd":6,"up_from":43,"seq":184683594125,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1965488,"kb_used_data":1668200,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91779664,"statfs":{"total":95995035648,"available":93982375936,"internally_reserved":0,"allocated":1708236800,"data_stored":1708022193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:31:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.4930000000000001,"15min":1.1100000000000001},"min":{"1min":0.33700000000000002,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":7.2329999999999997,"5min":16.850000000000001,"15min":16.850000000000001},"last":2.395},{"interface":"front","average":{"1min":1.514,"5min":1.2569999999999999,"15min":1.0780000000000001},"min":{"1min":0.30099999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.2489999999999997,"5min":7.2489999999999997,"15min":7.2489999999999997},"last":0.71399999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:32:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":2.0379999999999998,"15min":1.3919999999999999},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.5259999999999998,"5min":32.959000000000003,"15min":32.959000000000003},"last":2.5619999999999998},{"interface":"front","average":{"1min":1.248,"5min":1.7789999999999999,"15min":1.268},"min":{"1min":0.371,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9829999999999997,"5min":34.793999999999997,"15min":34.793999999999997},"last":0.48599999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:31:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.9379999999999999,"15min":1.2849999999999999},"min":{"1min":0.36099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.1550000000000002,"5min":30.379000000000001,"15min":30.379000000000001},"last":2.4380000000000002},{"interface":"front","average":{"1min":1.042,"5min":1.698,"15min":1.3460000000000001},"min":{"1min":0.45600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.7629999999999999,"5min":30.332999999999998,"15min":30.332999999999998},"last":0.59399999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:31:57 2023","interfaces":[{"interface":"back","average":{"1min":4.96,"5min":4.032,"15min":2.0390000000000001},"min":{"1min":0.41899999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":68.686999999999998,"5min":123.97199999999999,"15min":123.97199999999999},"last":2.4550000000000001},{"interface":"front","average":{"1min":5.8049999999999997,"5min":3.9359999999999999,"15min":2.1349999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":69.108999999999995,"5min":124.059,"15min":124.059},"last":0.51900000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:31:57 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.028,"15min":1.02},"min":{"1min":0.38500000000000001,"5min":0.312,"15min":0.312},"max":{"1min":2.7120000000000002,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.256,"15min":1.1639999999999999},"min":{"1min":0.35599999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.5599999999999996,"5min":6.7220000000000004,"15min":6.7220000000000004},"last":0.54100000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:31:57 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.1699999999999999,"15min":1.1299999999999999},"min":{"1min":0.40799999999999997,"5min":0.379,"15min":0.379},"max":{"1min":2.649,"5min":7.6479999999999997,"15min":7.6479999999999997},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.288,"5min":1.421,"15min":1.234},"min":{"1min":0.39500000000000002,"5min":0.38300000000000001,"15min":0.35999999999999999},"max":{"1min":4.1050000000000004,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.69899999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.6040000000000001,"15min":1.357},"min":{"1min":0.44800000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":7.6769999999999996,"5min":7.6769999999999996,"15min":7.6769999999999996},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.4430000000000001,"5min":1.5820000000000001,"15min":1.331},"min":{"1min":0.45400000000000001,"5min":0.371,"15min":0.371},"max":{"1min":7.6879999999999997,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":2.7989999999999999}]}]},{"osd":2,"up_from":634,"seq":2723009265878,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2365888,"kb_used_data":2085880,"kb_used_omap":122,"kb_used_meta":279877,"kb_avail":91379264,"statfs":{"total":95995035648,"available":93572366336,"internally_reserved":0,"allocated":2135941120,"data_stored":2134784214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":286594635},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":1.9490000000000001,"15min":1.1899999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.5259999999999998,"5min":8.9860000000000007,"15min":8.9860000000000007},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.7470000000000001,"5min":1.986,"15min":1.262},"min":{"1min":0.27600000000000002,"5min":0.221,"15min":0.221},"max":{"1min":6.351,"5min":10.019,"15min":10.019},"last":0.21099999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.534,"15min":1.0680000000000001},"min":{"1min":0.254,"5min":0.184,"15min":0.184},"max":{"1min":4.6390000000000002,"5min":9.5139999999999993,"15min":9.5139999999999993},"last":0.378},{"interface":"front","average":{"1min":1.407,"5min":1.9139999999999999,"15min":1.286},"min":{"1min":0.313,"5min":0.216,"15min":0.216},"max":{"1min":3.7490000000000001,"5min":8.702,"15min":8.702},"last":0.42799999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:32:03 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":2.0150000000000001,"15min":1.3580000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":6.4699999999999998,"5min":18.547999999999998,"15min":18.547999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.403,"5min":2.4079999999999999,"15min":1.4870000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":19.527999999999999,"5min":19.527999999999999,"15min":19.527999999999999},"last":1.375}]},{"osd":4,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.6799999999999999,"15min":1.286},"min":{"1min":0.372,"5min":0.309,"15min":0.309},"max":{"1min":2.4969999999999999,"5min":7.452,"15min":7.452},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.6659999999999999,"15min":1.319},"min":{"1min":0.36799999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.194,"5min":13.917999999999999,"15min":13.917999999999999},"last":5.3330000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.296,"15min":1.167},"min":{"1min":0.39500000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.7510000000000003,"5min":6.7619999999999996,"15min":6.7699999999999996},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.139,"5min":1.673,"15min":1.3360000000000001},"min":{"1min":0.48299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.952,"5min":10.393000000000001,"15min":10.393000000000001},"last":1.2829999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.716,"15min":1.367},"min":{"1min":0.46300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.0880000000000001,"5min":7.9470000000000001,"15min":7.9470000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.4530000000000001,"5min":1.843,"15min":1.464},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.4809999999999999,"5min":13.893000000000001,"15min":13.893000000000001},"last":5.7119999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":2.0459999999999998,"15min":1.5589999999999999},"min":{"1min":0.442,"5min":0.377,"15min":0.377},"max":{"1min":6.4359999999999999,"5min":13.848000000000001,"15min":13.848000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.23,"5min":1.5409999999999999,"15min":1.3260000000000001},"min":{"1min":0.44800000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.726,"5min":10.199,"15min":10.199},"last":0.57999999999999996}]}]},{"osd":1,"up_from":607,"seq":2607045148892,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2086896,"kb_used_data":1865320,"kb_used_omap":107,"kb_used_meta":221460,"kb_avail":91658256,"statfs":{"total":95995035648,"available":93858054144,"internally_reserved":0,"allocated":1910087680,"data_stored":1908949432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":226775723},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:32:14 2023","interfaces":[{"interface":"back","average":{"1min":3.0259999999999998,"5min":2.629,"15min":1.355},"min":{"1min":0.32200000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":36.817,"5min":84.421000000000006,"15min":84.421000000000006},"last":0.23699999999999999},{"interface":"front","average":{"1min":3.077,"5min":2.762,"15min":1.454},"min":{"1min":0.216,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":36.762,"5min":84.489999999999995,"15min":84.489999999999995},"last":0.20699999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:32:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":2.8559999999999999,"15min":1.6519999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.20100000000000001},"max":{"1min":6.9349999999999996,"5min":84.578999999999994,"15min":84.578999999999994},"last":69.421999999999997},{"interface":"front","average":{"1min":0.70499999999999996,"5min":2.3860000000000001,"15min":1.4710000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.726,"5min":84.697000000000003,"15min":84.697000000000003},"last":71.879000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:32:02 2023","interfaces":[{"interface":"back","average":{"1min":5.6109999999999998,"5min":5.4939999999999998,"15min":2.6110000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.23999999999999999},"max":{"1min":73.564999999999998,"5min":107.426,"15min":107.426},"last":0.27300000000000002},{"interface":"front","average":{"1min":5.8099999999999996,"5min":5.9180000000000001,"15min":2.8300000000000001},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":72.932000000000002,"5min":106.15900000000001,"15min":106.15900000000001},"last":0.27900000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":2.222,"15min":1.427},"min":{"1min":0.33800000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.9690000000000001,"5min":84.875,"15min":84.875},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.97299999999999998,"5min":2.3140000000000001,"15min":1.48},"min":{"1min":0.33600000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.9380000000000002,"5min":84.992999999999995,"15min":84.992999999999995},"last":0.42999999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":2.206,"15min":1.476},"min":{"1min":0.41699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.9930000000000001,"5min":84.805000000000007,"15min":84.805000000000007},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.026,"5min":2.5649999999999999,"15min":1.595},"min":{"1min":0.34899999999999998,"5min":0.33100000000000002,"15min":0.30399999999999999},"max":{"1min":2.9780000000000002,"5min":84.951999999999998,"15min":84.951999999999998},"last":0.56999999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":2.613,"15min":1.6559999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":4.5570000000000004,"5min":85.016000000000005,"15min":85.016000000000005},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.72699999999999998,"5min":2.266,"15min":1.54},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.2130000000000001,"5min":84.831999999999994,"15min":84.831999999999994},"last":0.57999999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":3.0099999999999998,"15min":1.8620000000000001},"min":{"1min":0.45300000000000001,"5min":0.33400000000000002,"15min":0.33100000000000002},"max":{"1min":4.5090000000000003,"5min":84.888000000000005,"15min":84.888000000000005},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.77400000000000002,"5min":2.403,"15min":1.6060000000000001},"min":{"1min":0.39600000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.2869999999999999,"5min":84.742000000000004,"15min":84.742000000000004},"last":0.63400000000000001}]}]},{"osd":0,"up_from":581,"seq":2495375999201,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2824880,"kb_used_data":2578984,"kb_used_omap":131,"kb_used_meta":245692,"kb_avail":90920272,"statfs":{"total":95995035648,"available":93102358528,"internally_reserved":0,"allocated":2640879616,"data_stored":2639787875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":251589209},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:32:19 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.841,"15min":1.202},"min":{"1min":0.307,"5min":0.223,"15min":0.223},"max":{"1min":5.0960000000000001,"5min":8.1039999999999992,"15min":14.44},"last":2.081},{"interface":"front","average":{"1min":2.23,"5min":1.8240000000000001,"15min":1.298},"min":{"1min":0.23699999999999999,"5min":0.23499999999999999,"15min":0.22700000000000001},"max":{"1min":6.9160000000000004,"5min":8.8870000000000005,"15min":14.663},"last":0.92600000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":2.9300000000000002,"15min":1.6779999999999999},"min":{"1min":0.33900000000000002,"5min":0.222,"15min":0.17799999999999999},"max":{"1min":5.5069999999999997,"5min":60.151000000000003,"15min":60.151000000000003},"last":0.91500000000000004},{"interface":"front","average":{"1min":1.49,"5min":2.2709999999999999,"15min":1.4079999999999999},"min":{"1min":0.41099999999999998,"5min":0.249,"15min":0.20999999999999999},"max":{"1min":3.5369999999999999,"5min":59.253,"15min":59.253},"last":2.1219999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:32:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":3.524,"15min":1.766},"min":{"1min":0.34000000000000002,"5min":0.252,"15min":0.252},"max":{"1min":9.7469999999999999,"5min":122.749,"15min":122.749},"last":0.439},{"interface":"front","average":{"1min":2.2450000000000001,"5min":3.5739999999999998,"15min":1.8300000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":14.185,"5min":123.51900000000001,"15min":123.51900000000001},"last":1.6859999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:31:57 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":1.399,"15min":1.2010000000000001},"min":{"1min":0.313,"5min":0.312,"15min":0.312},"max":{"1min":5.6299999999999999,"5min":7.093,"15min":7.093},"last":2.1520000000000001},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.6619999999999999,"15min":1.274},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28199999999999997},"max":{"1min":9.4169999999999998,"5min":9.4169999999999998,"15min":9.4169999999999998},"last":0.60199999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:31:57 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":1.5049999999999999,"15min":1.2230000000000001},"min":{"1min":0.33900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.6710000000000003,"5min":7.3620000000000001,"15min":7.3620000000000001},"last":4.0110000000000001},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.8180000000000001,"15min":1.3600000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.312},"max":{"1min":9.3670000000000009,"5min":9.3670000000000009,"15min":9.3670000000000009},"last":0.57699999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:31:57 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.8819999999999999,"15min":1.401},"min":{"1min":0.39300000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.4969999999999999,"5min":8.202,"15min":8.202},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.421,"5min":1.675,"15min":1.333},"min":{"1min":0.36299999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.7050000000000001,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":3.956}]},{"osd":7,"last update":"Mon Oct 9 09:31:57 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.869,"15min":1.4430000000000001},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":5.2690000000000001,"5min":8.1769999999999996,"15min":8.1769999999999996},"last":0.5},{"interface":"front","average":{"1min":1.556,"5min":1.841,"15min":1.4470000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":5.7380000000000004,"5min":10.571,"15min":10.571},"last":2.1869999999999998}]}]},{"osd":3,"up_from":663,"seq":2847563317456,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3255844,"kb_used_data":2925404,"kb_used_omap":55,"kb_used_meta":330376,"kb_avail":90489308,"statfs":{"total":95995035648,"available":92661051392,"internally_reserved":0,"allocated":2995613696,"data_stored":2994588453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":338305634},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:32:17 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":1.8440000000000001,"15min":1.2210000000000001},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":5.3899999999999997,"5min":10.395,"15min":10.395},"last":10.949},{"interface":"front","average":{"1min":2.1389999999999998,"5min":2.165,"15min":1.3260000000000001},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":7.7729999999999997,"5min":10.938000000000001,"15min":11.507999999999999},"last":0.61899999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:32:17 2023","interfaces":[{"interface":"back","average":{"1min":5.9059999999999997,"5min":3.0270000000000001,"15min":1.6399999999999999},"min":{"1min":0.33200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":91.159000000000006,"5min":91.159000000000006,"15min":91.159000000000006},"last":0.49399999999999999},{"interface":"front","average":{"1min":6.7009999999999996,"5min":3.0030000000000001,"15min":1.698},"min":{"1min":0.248,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":92.683000000000007,"5min":92.683000000000007,"15min":92.683000000000007},"last":4.1449999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:32:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":2.5059999999999998,"15min":1.528},"min":{"1min":0.35899999999999999,"5min":0.27100000000000002,"15min":0.26000000000000001},"max":{"1min":11.696999999999999,"5min":11.696999999999999,"15min":11.696999999999999},"last":7.702},{"interface":"front","average":{"1min":1.976,"5min":2.1179999999999999,"15min":1.4830000000000001},"min":{"1min":0.251,"5min":0.248,"15min":0.223},"max":{"1min":8.6639999999999997,"5min":9.4749999999999996,"15min":13.856},"last":2.8759999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:32:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.855,"15min":1.361},"min":{"1min":0.39000000000000001,"5min":0.37,"15min":0.32500000000000001},"max":{"1min":6.6840000000000002,"5min":12.629,"15min":12.629},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.377,"5min":1.615,"15min":1.298},"min":{"1min":0.36199999999999999,"5min":0.313,"15min":0.27500000000000002},"max":{"1min":7.9210000000000003,"5min":7.9210000000000003,"15min":7.9210000000000003},"last":2.9830000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:32:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4770000000000001,"5min":2.0249999999999999,"15min":1.51},"min":{"1min":0.51100000000000001,"5min":0.315,"15min":0.315},"max":{"1min":4.9429999999999996,"5min":11.272,"15min":11.272},"last":3.9060000000000001},{"interface":"front","average":{"1min":1.794,"5min":1.9570000000000001,"15min":1.474},"min":{"1min":0.36099999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.048,"5min":11.135999999999999,"15min":11.388999999999999},"last":0.57999999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:32:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":1.8620000000000001,"15min":1.4590000000000001},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":8.7040000000000006,"5min":8.7040000000000006,"15min":8.7040000000000006},"last":3.0499999999999998},{"interface":"front","average":{"1min":1.6379999999999999,"5min":1.996,"15min":1.52},"min":{"1min":0.42699999999999999,"5min":0.316,"15min":0.316},"max":{"1min":6.7160000000000002,"5min":11.058999999999999,"15min":17.234999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:32:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.929,"15min":1.5289999999999999},"min":{"1min":0.53300000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.9699999999999998,"5min":7.8220000000000001,"15min":7.8220000000000001},"last":3.8639999999999999},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.774,"15min":1.48},"min":{"1min":0.39800000000000002,"5min":0.35599999999999998,"15min":0.318},"max":{"1min":8.6210000000000004,"5min":8.6210000000000004,"15min":8.6210000000000004},"last":3.024}]}]},{"osd":4,"up_from":31,"seq":133143986590,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1889904,"kb_used_data":1584488,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91855248,"statfs":{"total":95995035648,"available":94059773952,"internally_reserved":0,"allocated":1622515712,"data_stored":1622300613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:32:44 2023","interfaces":[{"interface":"back","average":{"1min":4.2859999999999996,"5min":2.8090000000000002,"15min":1.552},"min":{"1min":0.307,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":51.237000000000002,"5min":94.489999999999995,"15min":94.489999999999995},"last":0.40300000000000002},{"interface":"front","average":{"1min":4.569,"5min":2.6989999999999998,"15min":1.5900000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":47.134,"5min":94.802999999999997,"15min":94.802999999999997},"last":0.25800000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:32:22 2023","interfaces":[{"interface":"back","average":{"1min":2.7869999999999999,"5min":1.7150000000000001,"15min":1.2190000000000001},"min":{"1min":0.34699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":38.222000000000001,"5min":38.222000000000001,"15min":38.222000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.9710000000000001,"5min":1.766,"15min":1.2390000000000001},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":38.289000000000001,"5min":38.289000000000001,"15min":38.289000000000001},"last":0.42899999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":2.4830000000000001,"15min":1.526},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.2409999999999997,"5min":99.736000000000004,"15min":99.736000000000004},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.478,"5min":2.2959999999999998,"15min":1.4390000000000001},"min":{"1min":0.375,"5min":0.317,"15min":0.317},"max":{"1min":4.5460000000000003,"5min":99.212999999999994,"15min":99.212999999999994},"last":6.2229999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:32:22 2023","interfaces":[{"interface":"back","average":{"1min":7.5010000000000003,"5min":3.3519999999999999,"15min":1.837},"min":{"1min":0.34799999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":47.32,"5min":47.32,"15min":47.32},"last":2.7269999999999999},{"interface":"front","average":{"1min":12.538,"5min":4.391,"15min":2.2389999999999999},"min":{"1min":0.505,"5min":0.314,"15min":0.314},"max":{"1min":93.843000000000004,"5min":93.843000000000004,"15min":93.843000000000004},"last":0.64300000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:32:22 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.2569999999999999,"15min":1.143},"min":{"1min":0.31900000000000001,"5min":0.312,"15min":0.312},"max":{"1min":3.7229999999999999,"5min":9.2959999999999994,"15min":9.2959999999999994},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.258,"15min":1.248},"min":{"1min":0.47199999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.7080000000000002,"5min":6.2000000000000002,"15min":7.5819999999999999},"last":0.46100000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:32:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.3819999999999999,"15min":1.1659999999999999},"min":{"1min":0.375,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.3769999999999998,"5min":7.625,"15min":7.625},"last":6.2569999999999997},{"interface":"front","average":{"1min":0.88500000000000001,"5min":1.3600000000000001,"15min":1.1839999999999999},"min":{"1min":0.40000000000000002,"5min":0.371,"15min":0.371},"max":{"1min":2.5099999999999998,"5min":7.1440000000000001,"15min":7.1440000000000001},"last":4.7409999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:32:40 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":1.2090000000000001,"15min":1.137},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.5720000000000001,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.431,"15min":1.3080000000000001},"min":{"1min":0.45400000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.0470000000000002,"5min":9.1780000000000008,"15min":9.1780000000000008},"last":0.58799999999999997}]}]},{"osd":5,"up_from":37,"seq":158913790356,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1497952,"kb_used_data":1200728,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92247200,"statfs":{"total":95995035648,"available":94461132800,"internally_reserved":0,"allocated":1229545472,"data_stored":1229342501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:32:43 2023","interfaces":[{"interface":"back","average":{"1min":4.5419999999999998,"5min":2.6549999999999998,"15min":1.5669999999999999},"min":{"1min":0.27400000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":44.036999999999999,"5min":59.258000000000003,"15min":59.258000000000003},"last":0.27900000000000003},{"interface":"front","average":{"1min":3.7610000000000001,"5min":2.7000000000000002,"15min":1.579},"min":{"1min":0.28999999999999998,"5min":0.246,"15min":0.246},"max":{"1min":44.231000000000002,"5min":59.432000000000002,"15min":59.432000000000002},"last":2.3170000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:32:23 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.4450000000000001,"15min":1.1479999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.8680000000000003,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":2.1459999999999999},{"interface":"front","average":{"1min":1.415,"5min":1.5469999999999999,"15min":1.2410000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.36399999999999999},"max":{"1min":5.9699999999999998,"5min":8.7210000000000001,"15min":8.7210000000000001},"last":2.1699999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:32:43 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":1.889,"15min":1.3819999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":12.731999999999999,"5min":14.019,"15min":14.019},"last":0.377},{"interface":"front","average":{"1min":1.8720000000000001,"5min":2.2530000000000001,"15min":1.5},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":12.717000000000001,"5min":14.037000000000001,"15min":14.037000000000001},"last":1.365}]},{"osd":3,"last update":"Mon Oct 9 09:32:01 2023","interfaces":[{"interface":"back","average":{"1min":4.407,"5min":2.3610000000000002,"15min":1.5189999999999999},"min":{"1min":0.379,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":59.652000000000001,"5min":59.652000000000001,"15min":59.652000000000001},"last":0.311},{"interface":"front","average":{"1min":4.4189999999999996,"5min":2.2080000000000002,"15min":1.478},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":61.771999999999998,"5min":61.771999999999998,"15min":61.771999999999998},"last":1.506}]},{"osd":4,"last update":"Mon Oct 9 09:32:23 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":1.147,"15min":1.0449999999999999},"min":{"1min":0.35499999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.8929999999999998,"5min":10.747,"15min":10.747},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.74299999999999999,"5min":1.2350000000000001,"15min":1.113},"min":{"1min":0.372,"5min":0.316,"15min":0.29799999999999999},"max":{"1min":2.9359999999999999,"5min":7.5780000000000003,"15min":7.5780000000000003},"last":1.4219999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:31:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.3759999999999999,"15min":1.206},"min":{"1min":0.376,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.2469999999999999,"5min":14.518000000000001,"15min":14.518000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.3899999999999999,"5min":1.6479999999999999,"15min":1.365},"min":{"1min":0.48999999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.3019999999999996,"5min":8.0280000000000005,"15min":8.0280000000000005},"last":1.544}]},{"osd":7,"last update":"Mon Oct 9 09:32:29 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.516,"15min":1.3240000000000001},"min":{"1min":0.442,"5min":0.44,"15min":0.36299999999999999},"max":{"1min":3.2770000000000001,"5min":8.6359999999999992,"15min":8.6359999999999992},"last":1.4610000000000001},{"interface":"front","average":{"1min":0.86899999999999999,"5min":1.472,"15min":1.3009999999999999},"min":{"1min":0.42999999999999999,"5min":0.41899999999999998,"15min":0.41399999999999998},"max":{"1min":3.6920000000000002,"5min":7.8559999999999999,"15min":7.8559999999999999},"last":0.45700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2632388608,"data_stored":2632384536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1901662208,"data_stored":1901658136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2124546048,"data_stored":2124546048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2963734528,"data_stored":2963734528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1614544896,"data_stored":1614544896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1222508544,"data_stored":1222508544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1691095040,"data_stored":1691090968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1780482048,"data_stored":1780482048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:32:52.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T09:32:52.431 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T09:32:52.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 21 2023-10-09T09:32:52.432 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:52.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:52.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:52.540 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:52.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:52.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:52.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:52.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:52.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:52.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:52.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:52.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:52.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:52.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:52.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:52.974 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:52.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:53.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:53.111 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:53.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:53.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:53.218 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:53.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:53.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:53.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:53.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:53.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:53.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:53.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:53.483 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:53.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:53.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:53.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:53.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:53.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:53.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:53.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:32:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:32:53] "GET /metrics HTTP/1.1" 200 36347 "" "Prometheus/2.33.4" 2023-10-09T09:32:53.745 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:53.778 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:53.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:53.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:53.884 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:53.893 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:53 smithi181 bash[59345]: cluster 2023-10-09T09:32:53.756151+0000 mgr.y (mgr.34107) 801 : cluster 0 pgmap v1522: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 887/226467 objects degraded (0.392%); 3349/226467 objects misplaced (1.479%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T09:32:53.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:53.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:53.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:54.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:54.025 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:53 smithi163 bash[78107]: cluster 2023-10-09T09:32:53.756151+0000 mgr.y (mgr.34107) 801 : cluster 0 pgmap v1522: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 887/226467 objects degraded (0.392%); 3349/226467 objects misplaced (1.479%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T09:32:54.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:53 smithi163 bash[75425]: cluster 2023-10-09T09:32:53.756151+0000 mgr.y (mgr.34107) 801 : cluster 0 pgmap v1522: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 887/226467 objects degraded (0.392%); 3349/226467 objects misplaced (1.479%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T09:32:54.041 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:54.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:54.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:54.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:54.176 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:54.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:54.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:54.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:54.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:54.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:54.382 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:32:54.420 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:54.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:32:54.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:54.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:54.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:54.573 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:54.616 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:54.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:54.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:54.718 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:54.741 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:54.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:54.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:54.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:54.871 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:54.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:54.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:54.976 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:55.011 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:55.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:55.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:55.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:55.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:55.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:55.251 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:55.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:55.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:55.380 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:55.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:55.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:55.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:55.549 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:55.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:55.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:55 smithi181 bash[59345]: audit 2023-10-09T09:32:55.370279+0000 mon.a (mon.0) 1637 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:55.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:55.662 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:55 smithi163 bash[78107]: audit 2023-10-09T09:32:55.370279+0000 mon.a (mon.0) 1637 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:55.663 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:55 smithi163 bash[75425]: audit 2023-10-09T09:32:55.370279+0000 mon.a (mon.0) 1637 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:32:55.681 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:32:55.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:55.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:32:55.788 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:55.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:55.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:55.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:55.921 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:55.944 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:55.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:56.048 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:56.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:56.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:56.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:56.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:56.215 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:56.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:56.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:56.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:56.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:56.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:56.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:56.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:56 smithi163 bash[75425]: cluster 2023-10-09T09:32:55.583484+0000 mon.a (mon.0) 1638 : cluster 3 Health check update: Degraded data redundancy: 887/226467 objects degraded (0.392%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:32:56.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:56 smithi163 bash[75425]: cluster 2023-10-09T09:32:55.756790+0000 mgr.y (mgr.34107) 802 : cluster 0 pgmap v1523: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 887/222498 objects degraded (0.399%); 3349/222498 objects misplaced (1.505%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T09:32:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:56 smithi163 bash[78107]: cluster 2023-10-09T09:32:55.583484+0000 mon.a (mon.0) 1638 : cluster 3 Health check update: Degraded data redundancy: 887/226467 objects degraded (0.392%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:32:56.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:56 smithi163 bash[78107]: cluster 2023-10-09T09:32:55.756790+0000 mgr.y (mgr.34107) 802 : cluster 0 pgmap v1523: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 887/222498 objects degraded (0.399%); 3349/222498 objects misplaced (1.505%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T09:32:56.488 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:56.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:56.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:56.591 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:56.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:56.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:56.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:56.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:56.756 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:56.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:56 smithi181 bash[59345]: cluster 2023-10-09T09:32:55.583484+0000 mon.a (mon.0) 1638 : cluster 3 Health check update: Degraded data redundancy: 887/226467 objects degraded (0.392%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:32:56.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:56 smithi181 bash[59345]: cluster 2023-10-09T09:32:55.756790+0000 mgr.y (mgr.34107) 802 : cluster 0 pgmap v1523: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 887/222498 objects degraded (0.399%); 3349/222498 objects misplaced (1.505%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T09:32:56.859 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:56.890 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:56.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:56.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:56.994 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:57.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:57.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:57.123 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:57.148 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:57.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:32:57.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:57.269 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:57.274 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:57.279 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:57.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:57.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:57.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:57.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:57.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:57.505 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:57.533 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:57.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:57.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:57.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:57.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:57.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:57.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:57.766 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:57.803 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:57.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:57.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:57.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:57.920 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:57 smithi163 bash[75425]: cluster 2023-10-09T09:32:57.757611+0000 mgr.y (mgr.34107) 803 : cluster 0 pgmap v1524: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 887/215145 objects degraded (0.412%); 3349/215145 objects misplaced (1.557%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T09:32:57.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:57 smithi163 bash[78107]: cluster 2023-10-09T09:32:57.757611+0000 mgr.y (mgr.34107) 803 : cluster 0 pgmap v1524: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 887/215145 objects degraded (0.412%); 3349/215145 objects misplaced (1.557%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T09:32:57.944 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:57.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:57.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:58.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:58.073 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:58.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:58.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:58.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:58.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:58.223 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:58.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:58.289 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:57 smithi181 bash[59345]: cluster 2023-10-09T09:32:57.757611+0000 mgr.y (mgr.34107) 803 : cluster 0 pgmap v1524: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 887/215145 objects degraded (0.412%); 3349/215145 objects misplaced (1.557%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T09:32:58.324 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:58.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:58.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:58.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:58.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:58.492 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:32:58.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:58.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:32:58.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:58.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:58.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:58.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:58.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:58.759 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:58.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:58.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:58.866 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:58.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:58.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:58.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:58.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:59.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:32:59.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:59.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:32:59.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:59.166 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:59.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:59.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:59.272 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:59.301 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:59.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:59.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:59.306 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:32:59.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:59.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:32:59.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:59.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:32:59.537 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:59.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:32:59.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:59.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:32:59.679 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:59.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:32:59.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:59.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:32:59.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:59.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:32:59.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:59.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:32:59.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:59.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:32:59 smithi181 bash[59345]: cluster 2023-10-09T09:32:59.758268+0000 mgr.y (mgr.34107) 804 : cluster 0 pgmap v1525: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 887/211755 objects degraded (0.419%); 3023/211755 objects misplaced (1.428%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:32:59.989 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:32:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:32:59.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:00.092 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:00.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:00.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:00.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:32:59 smithi163 bash[78107]: cluster 2023-10-09T09:32:59.758268+0000 mgr.y (mgr.34107) 804 : cluster 0 pgmap v1525: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 887/211755 objects degraded (0.419%); 3023/211755 objects misplaced (1.428%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:33:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:32:59 smithi163 bash[75425]: cluster 2023-10-09T09:32:59.758268+0000 mgr.y (mgr.34107) 804 : cluster 0 pgmap v1525: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 887/211755 objects degraded (0.419%); 3023/211755 objects misplaced (1.428%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:33:00.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:00.254 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:00.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:00.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:00.384 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:00.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:00.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:00.489 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:00.521 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:00.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:00.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:00.625 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:00.654 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:00.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:00.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:00.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:00.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:00.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:00.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:00.916 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:00.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:00.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:01.019 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:01.051 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:01.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:01.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:00 smithi181 bash[59345]: cluster 2023-10-09T09:33:00.585941+0000 mon.a (mon.0) 1639 : cluster 3 Health check update: Degraded data redundancy: 887/211755 objects degraded (0.419%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:01.155 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:01.181 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:01.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:01.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:00 smithi163 bash[78107]: cluster 2023-10-09T09:33:00.585941+0000 mon.a (mon.0) 1639 : cluster 3 Health check update: Degraded data redundancy: 887/211755 objects degraded (0.419%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:01.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:00 smithi163 bash[75425]: cluster 2023-10-09T09:33:00.585941+0000 mon.a (mon.0) 1639 : cluster 3 Health check update: Degraded data redundancy: 887/211755 objects degraded (0.419%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:01.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:01.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:01.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:01.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:01.420 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:01.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:01.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:01.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:01.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:01.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:01.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:01.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:01.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:01.719 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:01.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:01.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:01.824 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:01.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:01.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:01.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:01.954 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:01.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:01 smithi181 bash[59345]: cluster 2023-10-09T09:33:01.759086+0000 mgr.y (mgr.34107) 805 : cluster 0 pgmap v1526: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 887/205761 objects degraded (0.431%); 3023/205761 objects misplaced (1.469%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:33:01.980 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:01.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:01.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:02.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:02.115 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:02.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:02.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:01 smithi163 bash[78107]: cluster 2023-10-09T09:33:01.759086+0000 mgr.y (mgr.34107) 805 : cluster 0 pgmap v1526: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 887/205761 objects degraded (0.431%); 3023/205761 objects misplaced (1.469%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:33:02.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:01 smithi163 bash[75425]: cluster 2023-10-09T09:33:01.759086+0000 mgr.y (mgr.34107) 805 : cluster 0 pgmap v1526: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 887/205761 objects degraded (0.431%); 3023/205761 objects misplaced (1.469%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:33:02.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:02.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:02.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:02.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:02.356 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:02.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:02.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:02.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:02.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:02.513 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:02.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:02.616 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:02.642 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:02.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:02.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:02.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:02.773 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:02.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:02.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:02.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:02.913 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:02.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:02.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:03.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:03.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:03.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:03.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:03.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:03.179 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:03.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:03.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:03.283 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:03.309 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:03.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:03.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:03.412 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:03.440 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:03.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:03.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:03.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:03.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:03.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:03.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:03.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:03.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:03.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:03.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:33:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:33:03] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.33.4" 2023-10-09T09:33:03.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:03.843 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:03.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:03.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:03.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:03.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:03.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:03.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:04.075 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:04.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:03 smithi181 bash[59345]: cluster 2023-10-09T09:33:03.759831+0000 mgr.y (mgr.34107) 806 : cluster 0 pgmap v1527: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 887/201246 objects degraded (0.441%); 2665/201246 objects misplaced (1.324%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T09:33:04.106 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:04.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:04.212 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:04.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:03 smithi163 bash[78107]: cluster 2023-10-09T09:33:03.759831+0000 mgr.y (mgr.34107) 806 : cluster 0 pgmap v1527: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 887/201246 objects degraded (0.441%); 2665/201246 objects misplaced (1.324%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T09:33:04.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:03 smithi163 bash[75425]: cluster 2023-10-09T09:33:03.759831+0000 mgr.y (mgr.34107) 806 : cluster 0 pgmap v1527: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 887/201246 objects degraded (0.441%); 2665/201246 objects misplaced (1.324%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T09:33:04.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:04.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:04.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:04.352 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:04.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:04.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:04.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:04.480 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:04.503 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:04.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:04.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:04.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:04.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:04.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:04.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:04.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:04.763 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:04.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:04.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:04.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:04.895 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:04.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:04.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:05.025 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:05.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:05.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:05.165 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:05.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:05.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:05.268 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:05.300 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:05.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:05.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:05.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:05.431 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:05.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:05.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:05.535 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:05.561 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:05.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:05.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:05.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:05.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:05.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:05.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:05 smithi163 bash[78107]: cluster 2023-10-09T09:33:05.587409+0000 mon.a (mon.0) 1640 : cluster 3 Health check update: Degraded data redundancy: 887/201246 objects degraded (0.441%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:05.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:05 smithi163 bash[75425]: cluster 2023-10-09T09:33:05.587409+0000 mon.a (mon.0) 1640 : cluster 3 Health check update: Degraded data redundancy: 887/201246 objects degraded (0.441%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:05.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:05.821 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:05.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:05.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:05.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:05.937 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:05 smithi181 bash[59345]: cluster 2023-10-09T09:33:05.587409+0000 mon.a (mon.0) 1640 : cluster 3 Health check update: Degraded data redundancy: 887/201246 objects degraded (0.441%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:05.954 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:05.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:05.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:06.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:06.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:06.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:06.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:06.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:06.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:06.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:06.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:06.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:06.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:06.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:06.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:06.447 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:06.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:06.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:06.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:06.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:06.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:06.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:06.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:06.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:06.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:06 smithi163 bash[78107]: cluster 2023-10-09T09:33:05.760598+0000 mgr.y (mgr.34107) 807 : cluster 0 pgmap v1528: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 887/197421 objects degraded (0.449%); 2665/197421 objects misplaced (1.350%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:06.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:06 smithi163 bash[75425]: cluster 2023-10-09T09:33:05.760598+0000 mgr.y (mgr.34107) 807 : cluster 0 pgmap v1528: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 887/197421 objects degraded (0.449%); 2665/197421 objects misplaced (1.350%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:06.735 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:06.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:06.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:06.840 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:06.984 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:07.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:06 smithi181 bash[59345]: cluster 2023-10-09T09:33:05.760598+0000 mgr.y (mgr.34107) 807 : cluster 0 pgmap v1528: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 887/197421 objects degraded (0.449%); 2665/197421 objects misplaced (1.350%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:07.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:07.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:07.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:07.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:07.362 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:07.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:07.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:07.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:07.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:07.903 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:08.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:08.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:08.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:08.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:07 smithi181 bash[59345]: cluster 2023-10-09T09:33:07.761306+0000 mgr.y (mgr.34107) 808 : cluster 0 pgmap v1529: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 887/190233 objects degraded (0.466%); 2665/190233 objects misplaced (1.401%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:08.188 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:08.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:08.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:07 smithi163 bash[78107]: cluster 2023-10-09T09:33:07.761306+0000 mgr.y (mgr.34107) 808 : cluster 0 pgmap v1529: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 887/190233 objects degraded (0.466%); 2665/190233 objects misplaced (1.401%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:07 smithi163 bash[75425]: cluster 2023-10-09T09:33:07.761306+0000 mgr.y (mgr.34107) 808 : cluster 0 pgmap v1529: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 887/190233 objects degraded (0.466%); 2665/190233 objects misplaced (1.401%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:08.292 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:08.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:08.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:08.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:08.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:08.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:08.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:08.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:08.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:08.583 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:08.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:08.686 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:08.716 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:08.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:08.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:08.819 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:08.843 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:08.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:08.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:08.946 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:08.969 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:08.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:08.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:09.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:09.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:09.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:09.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:09.203 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:09.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:09.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:09.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:09.329 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:09.358 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:09.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:09.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:09.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:09.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:09.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:09.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:09.601 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:09.629 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:09.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:09.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:09.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:09.760 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:09.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:09.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:09.868 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:09.877 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:09 smithi181 bash[59345]: cluster 2023-10-09T09:33:09.761846+0000 mgr.y (mgr.34107) 809 : cluster 0 pgmap v1530: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 887/186789 objects degraded (0.475%); 2331/186789 objects misplaced (1.248%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T09:33:09.898 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:09.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:09.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:10.002 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:10.014 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:09 smithi163 bash[75425]: cluster 2023-10-09T09:33:09.761846+0000 mgr.y (mgr.34107) 809 : cluster 0 pgmap v1530: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 887/186789 objects degraded (0.475%); 2331/186789 objects misplaced (1.248%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T09:33:10.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:09 smithi163 bash[78107]: cluster 2023-10-09T09:33:09.761846+0000 mgr.y (mgr.34107) 809 : cluster 0 pgmap v1530: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 887/186789 objects degraded (0.475%); 2331/186789 objects misplaced (1.248%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T09:33:10.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:10.047 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:10.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:10.148 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:10.176 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:10.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:10.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:10.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:10.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:10.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:10.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:10.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:10.449 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:10.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:10.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:10.555 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:10.585 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:10.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:10.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:10.689 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:10.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:10.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:10.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:10.825 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:10.855 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:10.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:10.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:10.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:10.967 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:10 smithi181 bash[59345]: audit 2023-10-09T09:33:10.370583+0000 mon.a (mon.0) 1641 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:10.967 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:10 smithi181 bash[59345]: cluster 2023-10-09T09:33:10.588536+0000 mon.a (mon.0) 1642 : cluster 3 Health check update: Degraded data redundancy: 887/186789 objects degraded (0.475%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:10.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:10.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:10.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:11.089 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:11.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:11.130 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:10 smithi163 bash[75425]: audit 2023-10-09T09:33:10.370583+0000 mon.a (mon.0) 1641 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:11.130 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:10 smithi163 bash[75425]: cluster 2023-10-09T09:33:10.588536+0000 mon.a (mon.0) 1642 : cluster 3 Health check update: Degraded data redundancy: 887/186789 objects degraded (0.475%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:11.131 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:10 smithi163 bash[78107]: audit 2023-10-09T09:33:10.370583+0000 mon.a (mon.0) 1641 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:11.131 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:10 smithi163 bash[78107]: cluster 2023-10-09T09:33:10.588536+0000 mon.a (mon.0) 1642 : cluster 3 Health check update: Degraded data redundancy: 887/186789 objects degraded (0.475%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:11.131 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:11.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:11.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:11.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:11.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:11.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:11.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:11.412 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:11.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:11.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:11.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:11.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:11.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:11.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:11.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:11.673 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:11.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:11.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:11.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:11.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:11.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:11.925 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:11.954 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:11.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:12.057 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:12.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:11 smithi181 bash[59345]: cluster 2023-10-09T09:33:11.762517+0000 mgr.y (mgr.34107) 810 : cluster 0 pgmap v1531: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 887/181281 objects degraded (0.489%); 2331/181281 objects misplaced (1.286%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:12.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:12.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:12.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:12.186 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:12.204 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:11 smithi163 bash[75425]: cluster 2023-10-09T09:33:11.762517+0000 mgr.y (mgr.34107) 810 : cluster 0 pgmap v1531: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 887/181281 objects degraded (0.489%); 2331/181281 objects misplaced (1.286%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:12.205 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:11 smithi163 bash[78107]: cluster 2023-10-09T09:33:11.762517+0000 mgr.y (mgr.34107) 810 : cluster 0 pgmap v1531: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 887/181281 objects degraded (0.489%); 2331/181281 objects misplaced (1.286%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:12.224 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:12.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:12.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:12.332 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:12.362 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:12.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:12.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:12.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:12.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:12.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:12.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:12.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:12.633 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:12.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:12.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:12.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:12.768 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:12.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:12.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:12.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:12.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:12.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:13.003 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:13.031 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:13.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:13.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:13.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:13.158 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:13.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:13.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:13.264 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:13.295 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:13.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:13.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:13.397 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:13.424 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:13.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:13.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:13.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:13.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:13.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:13.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:13.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:13.664 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:33:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:33:13] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.33.4" 2023-10-09T09:33:13.695 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:13.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:13.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:13.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:13.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:13.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:13.943 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:13.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:13 smithi163 bash[75425]: cluster 2023-10-09T09:33:13.763156+0000 mgr.y (mgr.34107) 811 : cluster 0 pgmap v1532: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 887/177297 objects degraded (0.500%); 1977/177297 objects misplaced (1.115%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T09:33:13.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:13 smithi163 bash[78107]: cluster 2023-10-09T09:33:13.763156+0000 mgr.y (mgr.34107) 811 : cluster 0 pgmap v1532: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 887/177297 objects degraded (0.500%); 1977/177297 objects misplaced (1.115%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T09:33:13.976 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:13.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:13.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:14.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:14.100 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:14.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:14.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:14.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:14.212 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:13 smithi181 bash[59345]: cluster 2023-10-09T09:33:13.763156+0000 mgr.y (mgr.34107) 811 : cluster 0 pgmap v1532: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 887/177297 objects degraded (0.500%); 1977/177297 objects misplaced (1.115%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T09:33:14.237 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:14.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:14.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:14.345 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:14.371 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:14.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:14.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:14.474 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:14.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:14.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:14.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:14.628 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:14.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:14.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:14.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:14.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:14.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:14.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:14.896 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:14.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:15.003 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:15.035 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:15.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:15.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:15.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:15.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:15.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:15.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:15.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:15.296 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:15.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:15.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:15.425 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:15.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:15.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:15.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:15.556 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:15.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:15.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:15.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:15.694 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:15.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:15.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:15.803 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:15.812 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:15 smithi181 bash[59345]: cluster 2023-10-09T09:33:15.589649+0000 mon.a (mon.0) 1643 : cluster 3 Health check update: Degraded data redundancy: 887/177297 objects degraded (0.500%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:15.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:15.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:15.904 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:15 smithi163 bash[75425]: cluster 2023-10-09T09:33:15.589649+0000 mon.a (mon.0) 1643 : cluster 3 Health check update: Degraded data redundancy: 887/177297 objects degraded (0.500%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:15.904 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:15 smithi163 bash[78107]: cluster 2023-10-09T09:33:15.589649+0000 mon.a (mon.0) 1643 : cluster 3 Health check update: Degraded data redundancy: 887/177297 objects degraded (0.500%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:15.938 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:15.960 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:15.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:15.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:16.066 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:16.099 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:16.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:16.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:16.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:16.240 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:16.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:16.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:16.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:16.381 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:16.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:16.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:16.485 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:16.510 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:16.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:16.579 INFO:teuthology.orchestra.run.smithi163.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T09:33:16.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:16.631 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:33:16.631 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:33:16.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:33:16.631 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:33:16.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.631 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:173104 6.4 6:2245deec:::benchmark_data_smithi163_107771_object62344:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:33:16.631 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:28:39.336121+0000", 2023-10-09T09:33:16.631 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 277.286057863, 2023-10-09T09:33:16.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15256920099999999, 2023-10-09T09:33:16.631 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 173104 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.336121+0000", 2023-10-09T09:33:16.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.336121+0000", 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.336119+0000", 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.336126+0000", 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0960000000000003e-06 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.336139+0000", 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.307e-05 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.336145+0000", 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0730000000000001e-06 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.336214+0000", 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8459000000000003e-05 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.336341+0000", 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012720900000000001 2023-10-09T09:33:16.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.336488+0000", 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014690100000000001 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.340657+0000", 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041697080000000003 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.346186+0000", 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00552916 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.488498+0000", 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.142311241 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.488644+0000", 2023-10-09T09:33:16.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0001461 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:39.488690+0000", 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6042999999999999e-05 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.638 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:181359 6.4 6:28a23aa5:::benchmark_data_smithi163_107771_object70599:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:28:51.070259+0000", 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 265.55191959299998, 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.1614265, 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181359 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.070259+0000", 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.070259+0000", 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.070257+0000", 2023-10-09T09:33:16.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.070264+0000", 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8510000000000001e-06 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.070279+0000", 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.448e-05 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.070287+0000", 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.388e-06 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.070357+0000", 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9839999999999995e-05 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.070468+0000", 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111105 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.070579+0000", 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000110487 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.075028+0000", 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0044491319999999997 2023-10-09T09:33:16.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.076029+0000", 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001000967 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.231585+0000", 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15555633399999999 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.231644+0000", 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9194000000000003e-05 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:28:51.231685+0000", 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1254000000000002e-05 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:209488 6.4 6:2c224a2e:::benchmark_data_smithi163_107771_object98728:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:31.682345+0000", 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 224.939833099, 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15841923299999999, 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 209488 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.646 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.682345+0000", 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.682345+0000", 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.682344+0000", 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.682350+0000", 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2949999999999999e-06 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.682360+0000", 2023-10-09T09:33:16.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6320000000000003e-06 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.682366+0000", 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3589999999999996e-06 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.682414+0000", 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7689999999999999e-05 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.682528+0000", 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011453799999999999 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.683189+0000", 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00066073900000000003 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:33:16.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.688159+0000", 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049695620000000003 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.699041+0000", 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010882129000000001 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.840675+0000", 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.141634172 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.840726+0000", 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0590000000000002e-05 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.840765+0000", 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9094000000000002e-05 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.34791.0:209498 6.1 6:8e606348:::benchmark_data_smithi163_107771_object98738:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:29:31.698921+0000", 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 224.92325727599999, 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16385195699999999, 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.34791", 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 209498 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.653 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.698921+0000", 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.698921+0000", 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.698919+0000", 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.698929+0000", 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8660000000000001e-06 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.698944+0000", 2023-10-09T09:33:16.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.485e-05 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.698956+0000", 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1788e-05 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.699000+0000", 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4585000000000001e-05 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.699136+0000", 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013526599999999999 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.699252+0000", 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000116338 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.703475+0000", 2023-10-09T09:33:16.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042228129999999997 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.705395+0000", 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001919708 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.862656+0000", 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15726164100000001 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.862711+0000", 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4763999999999999e-05 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:29:31.862773+0000", 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2287999999999997e-05 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:93019 7.12 7:4d7858ba:::benchmark_data_smithi163_130508_object93018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1113)", 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:56.224970+0000", 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 80.397208672999994, 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21876399399999999, 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 93019 2023-10-09T09:33:16.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.224970+0000", 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.224970+0000", 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.224968+0000", 2023-10-09T09:33:16.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:33:16.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.224993+0000", 2023-10-09T09:33:16.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4825999999999999e-05 2023-10-09T09:33:16.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.225005+0000", 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1341e-05 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.225011+0000", 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6950000000000003e-06 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.225414+0000", 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00040239399999999998 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.225562+0000", 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014808900000000001 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2,6", 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.225716+0000", 2023-10-09T09:33:16.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000154007 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.227228+0000", 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001511994 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.234018+0000", 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067902600000000002 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.235036+0000", 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001017874 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.442059+0000", 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.207022814 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.442120+0000", 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1713999999999999e-05 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.443734+0000", 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00161341 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:93065 7.12 7:49393883:::benchmark_data_smithi163_130508_object93064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1113)", 2023-10-09T09:33:16.667 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:31:56.265209+0000", 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 80.356969508999995, 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19390959599999999, 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 93065 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.265209+0000", 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.265209+0000", 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.265208+0000", 2023-10-09T09:33:16.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.265226+0000", 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8340999999999999e-05 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.265236+0000", 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5599999999999999e-06 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.265242+0000", 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1800000000000001e-06 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.265278+0000", 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6350000000000003e-05 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.265390+0000", 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011147 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2,6", 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.265496+0000", 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000106063 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.266681+0000", 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011852799999999999 2023-10-09T09:33:16.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.269415+0000", 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027336499999999998 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.269835+0000", 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00041967699999999997 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.457707+0000", 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.18787278800000001 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.457755+0000", 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8092e-05 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:31:56.459119+0000", 2023-10-09T09:33:16.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001363178 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:151825 7.12 7:4f4afae1:::benchmark_data_smithi163_130508_object43162:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1119)", 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:03.248753+0000", 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 13.373425406000001, 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16405570999999999, 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.675 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151825 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.248753+0000", 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.248753+0000", 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.248751+0000", 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.248760+0000", 2023-10-09T09:33:16.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4069999999999993e-06 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.248771+0000", 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1450999999999999e-05 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.248779+0000", 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5349999999999999e-06 2023-10-09T09:33:16.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.250922+0000", 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002143299 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.251081+0000", 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015956499999999999 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2,6", 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.251211+0000", 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012913000000000001 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.254648+0000", 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034373509999999999 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.256602+0000", 2023-10-09T09:33:16.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019540550000000001 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.257955+0000", 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013527809999999999 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.410776+0000", 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15282104899999999 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.410857+0000", 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0950000000000003e-05 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.412809+0000", 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001952128 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:151833 7.d e1119/1099 7:b1738898:::benchmark_data_smithi163_130508_object43170:head v 1119'9673, mlcod=1119'9671)", 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:03.258684+0000", 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 13.363494535999999, 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15191260500000001, 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.258684+0000", 2023-10-09T09:33:16.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.258684+0000", 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.258684+0000", 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.258689+0000", 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0130000000000001e-06 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.258697+0000", 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9529999999999999e-06 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.258702+0000", 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4439999999999999e-06 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.258743+0000", 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1124000000000001e-05 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.258765+0000", 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.16e-05 2023-10-09T09:33:16.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.266474+0000", 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.007709591 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.410597+0000", 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14412223299999999 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:151832 7.10 e1119/1115 7:0cf0138e:::benchmark_data_smithi163_130508_object43169:head v 1119'6831, mlcod=1119'6829)", 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:03.260902+0000", 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 13.361276543000001, 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14973197099999999, 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260902+0000", 2023-10-09T09:33:16.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260902+0000", 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260901+0000", 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260909+0000", 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3900000000000004e-06 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260919+0000", 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9520000000000006e-06 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260926+0000", 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3170000000000004e-06 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260979+0000", 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2670999999999998e-05 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.261006+0000", 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7098999999999999e-05 2023-10-09T09:33:16.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.269601+0000", 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085951020000000003 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.410634+0000", 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.141033085 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.692 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:151831 7.2 e1119/1105 7:475a880f:::benchmark_data_smithi163_130508_object43168:head v 1119'6778, mlcod=1119'6776)", 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:03.260938+0000", 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 13.361240264999999, 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15196611800000001, 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260938+0000", 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260938+0000", 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260938+0000", 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T09:33:16.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260942+0000", 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4989999999999999e-06 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260948+0000", 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1530000000000002e-06 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.260952+0000", 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1150000000000004e-06 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.261028+0000", 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6390999999999994e-05 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.261048+0000", 2023-10-09T09:33:16.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9548999999999999e-05 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.412837+0000", 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15178879100000001 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.412904+0000", 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7841000000000003e-05 2023-10-09T09:33:16.697 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44921.0:151825 7.12 e1119/1112 ondisk, result = 0)", 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:03.264124+0000", 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 13.358054085999999, 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14878465299999999, 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.264124+0000", 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.264124+0000", 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.264124+0000", 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.264129+0000", 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7520000000000004e-06 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.410550+0000", 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14642112500000001 2023-10-09T09:33:16.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.410565+0000", 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5051e-05 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.410726+0000", 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016130800000000001 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.410754+0000", 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7747999999999999e-05 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:03.412909+0000", 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021552630000000001 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:155656 7.12 7:49ae2190:::benchmark_data_smithi163_130508_object46993:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1119)", 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:07.954096+0000", 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 8.6680823930000006, 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21172757, 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 155656 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.703 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.954096+0000", 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.954096+0000", 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.954095+0000", 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.954101+0000", 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9499999999999998e-06 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.954112+0000", 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1039999999999999e-05 2023-10-09T09:33:16.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.954118+0000", 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2700000000000001e-06 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.956684+0000", 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0025663019999999999 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.956841+0000", 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000157245 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2,6", 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.956979+0000", 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013759300000000001 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.959268+0000", 2023-10-09T09:33:16.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022887699999999999 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.959372+0000", 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103742 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:07.962978+0000", 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003606476 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.164230+0000", 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20125236299999999 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.164306+0000", 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5776000000000001e-05 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.165824+0000", 2023-10-09T09:33:16.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001517516 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:155869 7.2 e1119/1105 7:42977267:::benchmark_data_smithi163_130508_object47206:head v 1119'6898, mlcod=1119'6894)", 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:08.390840+0000", 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 8.2313384680000006, 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.158336688, 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.710 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390840+0000", 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390840+0000", 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390840+0000", 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390851+0000", 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.111e-05 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390859+0000", 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0350000000000001e-06 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390863+0000", 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4229999999999998e-06 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.392586+0000", 2023-10-09T09:33:16.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017228670000000001 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.392599+0000", 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2815e-05 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.395041+0000", 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024420520000000001 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.549177+0000", 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15413588 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:155876 7.d e1119/1099 7:bdfbfd20:::benchmark_data_smithi163_130508_object47213:head v 1119'9924, mlcod=1119'9920)", 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:08.390962+0000", 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 8.2312166250000001, 2023-10-09T09:33:16.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16364453700000001, 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390962+0000", 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390962+0000", 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390961+0000", 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390968+0000", 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1339999999999998e-06 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390977+0000", 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0499999999999997e-06 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.390983+0000", 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3850000000000001e-06 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.549158+0000", 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15817516700000001 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.549236+0000", 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8450999999999998e-05 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.554508+0000", 2023-10-09T09:33:16.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005271988 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.554606+0000", 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7979000000000005e-05 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.720 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44921.0:155880 7.b 7:d3dd2b58:::benchmark_data_smithi163_130508_object47217:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1119)", 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:08.391074+0000", 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 8.2311042360000002, 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16585119300000001, 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44921", 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 155880 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.391074+0000", 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.391074+0000", 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.391073+0000", 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.391078+0000", 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8969999999999999e-06 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.391086+0000", 2023-10-09T09:33:16.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8280000000000003e-06 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.391090+0000", 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3889999999999998e-06 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.391128+0000", 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8016999999999999e-05 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.391205+0000", 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6686000000000004e-05 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.391273+0000", 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8677999999999995e-05 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:33:16.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.394942+0000", 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036682820000000001 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.552709+0000", 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15776748199999999 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.556847+0000", 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041382930000000004 2023-10-09T09:33:16.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.556890+0000", 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2604000000000001e-05 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.556925+0000", 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5426000000000003e-05 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.727 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44921.0:155880 7.b e1119/1099 ondisk, result = 0)", 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:08.397447+0000", 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 8.2247312109999999, 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15526525199999999, 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.397447+0000", 2023-10-09T09:33:16.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.397447+0000", 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.397446+0000", 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.397452+0000", 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5600000000000001e-06 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.552561+0000", 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15510968 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.552576+0000", 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4319e-05 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.552693+0000", 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011748600000000001 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.552702+0000", 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6829999999999997e-06 2023-10-09T09:33:16.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:08.552713+0000", 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0746e-05 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:157633 7.2 e1119/1105 7:46e888ff:::benchmark_data_smithi163_130508_object48970:head v 1119'6957, mlcod=1119'6955)", 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:10.601725+0000", 2023-10-09T09:33:16.732 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 6.0204535359999998, 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20106769999999999, 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.601725+0000", 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.601725+0000", 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.601724+0000", 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.601732+0000", 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.61e-06 2023-10-09T09:33:16.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.601743+0000", 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1552999999999999e-05 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.601750+0000", 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1210000000000001e-06 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.601807+0000", 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6647999999999997e-05 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.601826+0000", 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8655999999999999e-05 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.802718+0000", 2023-10-09T09:33:16.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20089211300000001 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.802793+0000", 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4868000000000002e-05 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:157635 7.19 e1119/1105 7:9b4c444e:::benchmark_data_smithi163_130508_object48972:head v 1119'6995, mlcod=1119'6993)", 2023-10-09T09:33:16.737 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:10.607200+0000", 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 6.0149789809999996, 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19535666600000001, 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607200+0000", 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607200+0000", 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607199+0000", 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607207+0000", 2023-10-09T09:33:16.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6619999999999994e-06 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607219+0000", 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1929999999999999e-05 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607227+0000", 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.123e-06 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607275+0000", 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7855999999999997e-05 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607296+0000", 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0924000000000002e-05 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.802499+0000", 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19520292 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.802556+0000", 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7138000000000001e-05 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:157636 7.d e1119/1099 7:be9fa37a:::benchmark_data_smithi163_130508_object48973:head v 1119'10040, mlcod=1119'10038)", 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:10.607257+0000", 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 6.0149214110000004, 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22691145099999999, 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607257+0000", 2023-10-09T09:33:16.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607257+0000", 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607257+0000", 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607264+0000", 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7340000000000008e-06 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607272+0000", 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3459999999999996e-06 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.607277+0000", 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8810000000000004e-06 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.802582+0000", 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19530545699999999 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.802647+0000", 2023-10-09T09:33:16.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5028000000000001e-05 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.834140+0000", 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031492564000000001 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.834169+0000", 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8812000000000001e-05 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.747 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44921.0:157637 7.10 e1119/1115 7:0ac5cafb:::benchmark_data_smithi163_130508_object48974:head v 1119'7033, mlcod=1119'7031)", 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:33:10.608006+0000", 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 6.014172136, 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19949322899999999, 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:33:16.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.608006+0000", 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.608006+0000", 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.608006+0000", 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:33:16.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.608012+0000", 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5509999999999996e-06 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.802527+0000", 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.194514768 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.802553+0000", 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6378999999999999e-05 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.802615+0000", 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1468000000000002e-05 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:33:16.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.802637+0000", 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1756999999999999e-05 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.807465+0000", 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048279229999999996 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:33:10.807500+0000", 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5046000000000003e-05 2023-10-09T09:33:16.752 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.753 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.753 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.753 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:33:16.753 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:33:16.753 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:33:16.754 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:33:16.754 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:33:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:16.755 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:16.755 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:33:16.755 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:33:16.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:16.801 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:33:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:16 smithi181 bash[59345]: cluster 2023-10-09T09:33:15.763679+0000 mgr.y (mgr.34107) 812 : cluster 0 pgmap v1533: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 887/173391 objects degraded (0.512%); 1977/173391 objects misplaced (1.140%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:16 smithi181 bash[59345]: audit 2023-10-09T09:33:16.533367+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.163:0/3714410767' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:33:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:16 smithi181 bash[59345]: audit 2023-10-09T09:33:16.534352+0000 mon.a (mon.0) 1644 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:33:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:16 smithi181 bash[59345]: audit 2023-10-09T09:33:16.566670+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.163:0/3779921671' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T09:33:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:16 smithi181 bash[59345]: audit 2023-10-09T09:33:16.567583+0000 mon.a (mon.0) 1645 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T09:33:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:16 smithi181 bash[59345]: audit 2023-10-09T09:33:16.583579+0000 mon.a (mon.0) 1646 : audit 0 from='client.? 172.21.15.163:0/3715770998' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:33:16.837 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:33:16.856 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:16.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[78107]: cluster 2023-10-09T09:33:15.763679+0000 mgr.y (mgr.34107) 812 : cluster 0 pgmap v1533: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 887/173391 objects degraded (0.512%); 1977/173391 objects misplaced (1.140%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:16.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[78107]: audit 2023-10-09T09:33:16.533367+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.163:0/3714410767' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:33:16.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[78107]: audit 2023-10-09T09:33:16.534352+0000 mon.a (mon.0) 1644 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:33:16.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[78107]: audit 2023-10-09T09:33:16.566670+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.163:0/3779921671' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T09:33:16.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[78107]: audit 2023-10-09T09:33:16.567583+0000 mon.a (mon.0) 1645 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T09:33:16.865 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[78107]: audit 2023-10-09T09:33:16.583579+0000 mon.a (mon.0) 1646 : audit 0 from='client.? 172.21.15.163:0/3715770998' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:33:16.865 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[75425]: cluster 2023-10-09T09:33:15.763679+0000 mgr.y (mgr.34107) 812 : cluster 0 pgmap v1533: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 887/173391 objects degraded (0.512%); 1977/173391 objects misplaced (1.140%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:33:16.865 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[75425]: audit 2023-10-09T09:33:16.533367+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.163:0/3714410767' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:33:16.865 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[75425]: audit 2023-10-09T09:33:16.534352+0000 mon.a (mon.0) 1644 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:33:16.865 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[75425]: audit 2023-10-09T09:33:16.566670+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.163:0/3779921671' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T09:33:16.865 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[75425]: audit 2023-10-09T09:33:16.567583+0000 mon.a (mon.0) 1645 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T09:33:16.865 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:16 smithi163 bash[75425]: audit 2023-10-09T09:33:16.583579+0000 mon.a (mon.0) 1646 : audit 0 from='client.? 172.21.15.163:0/3715770998' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:33:16.867 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:33:16.885 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:16.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:16.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:16.897 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:33:16.945 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:33:16.953 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:33:16.953 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:33:16.953 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:33:16.953 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:33:16.953 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:33:16.953 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:33:16.953 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:33:16.954 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:33:16.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:17.016 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:17.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:17.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:17.123 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:17.149 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:17.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:17.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:17.278 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:17.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:17.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:17.383 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:17.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:17.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:17.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:17.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:17.541 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:17.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:17.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:17.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:17.656 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[75425]: cluster 2023-10-09T09:33:16.643407+0000 mon.a (mon.0) 1647 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:33:17.656 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[75425]: cluster 2023-10-09T09:33:16.656153+0000 osd.5 (osd.5) 670 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.656 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[75425]: audit 2023-10-09T09:33:16.662791+0000 mon.a (mon.0) 1648 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:33:17.656 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[75425]: cluster 2023-10-09T09:33:16.663040+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e1120: 8 total, 8 up, 8 in 2023-10-09T09:33:17.656 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[75425]: cluster 2023-10-09T09:33:16.679334+0000 osd.4 (osd.4) 546 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[75425]: cluster 2023-10-09T09:33:16.681564+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[75425]: cluster 2023-10-09T09:33:16.687226+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.657 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[78107]: cluster 2023-10-09T09:33:16.643407+0000 mon.a (mon.0) 1647 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:33:17.657 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[78107]: cluster 2023-10-09T09:33:16.656153+0000 osd.5 (osd.5) 670 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.657 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[78107]: audit 2023-10-09T09:33:16.662791+0000 mon.a (mon.0) 1648 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:33:17.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[78107]: cluster 2023-10-09T09:33:16.663040+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e1120: 8 total, 8 up, 8 in 2023-10-09T09:33:17.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[78107]: cluster 2023-10-09T09:33:16.679334+0000 osd.4 (osd.4) 546 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[78107]: cluster 2023-10-09T09:33:16.681564+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:17 smithi163 bash[78107]: cluster 2023-10-09T09:33:16.687226+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.668 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:17.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:17.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:17.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:17.783 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:17 smithi181 bash[59345]: cluster 2023-10-09T09:33:16.643407+0000 mon.a (mon.0) 1647 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:33:17.783 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:17 smithi181 bash[59345]: cluster 2023-10-09T09:33:16.656153+0000 osd.5 (osd.5) 670 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.783 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:17 smithi181 bash[59345]: audit 2023-10-09T09:33:16.662791+0000 mon.a (mon.0) 1648 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:33:17.783 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:17 smithi181 bash[59345]: cluster 2023-10-09T09:33:16.663040+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e1120: 8 total, 8 up, 8 in 2023-10-09T09:33:17.783 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:17 smithi181 bash[59345]: cluster 2023-10-09T09:33:16.679334+0000 osd.4 (osd.4) 546 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.783 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:17 smithi181 bash[59345]: cluster 2023-10-09T09:33:16.681564+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.783 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:17 smithi181 bash[59345]: cluster 2023-10-09T09:33:16.687226+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:17.803 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:17.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:17.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:17.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:17.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:17.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:17.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:18.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:18.066 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:18.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:18.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:18.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:18.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:18.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:18.305 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:18.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:18.337 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:18.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:18.438 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:18.496 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:18.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:18.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:18.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:18.621 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:18.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:18.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:18.726 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:18.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:18.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:18.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:18.870 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:18 smithi163 bash[75425]: cluster 2023-10-09T09:33:16.679175+0000 osd.7 (osd.7) 511 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:18.870 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:18 smithi163 bash[75425]: cluster 2023-10-09T09:33:17.115118+0000 osd.7 (osd.7) 512 : cluster 0 7.11 scrub starts 2023-10-09T09:33:18.871 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:18 smithi163 bash[75425]: cluster 2023-10-09T09:33:17.764213+0000 mgr.y (mgr.34107) 813 : cluster 0 pgmap v1535: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 887/166425 objects degraded (0.533%); 1977/166425 objects misplaced (1.188%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T09:33:18.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:18 smithi163 bash[78107]: cluster 2023-10-09T09:33:16.679175+0000 osd.7 (osd.7) 511 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:18.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:18 smithi163 bash[78107]: cluster 2023-10-09T09:33:17.115118+0000 osd.7 (osd.7) 512 : cluster 0 7.11 scrub starts 2023-10-09T09:33:18.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:18 smithi163 bash[78107]: cluster 2023-10-09T09:33:17.764213+0000 mgr.y (mgr.34107) 813 : cluster 0 pgmap v1535: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 887/166425 objects degraded (0.533%); 1977/166425 objects misplaced (1.188%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T09:33:18.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:18.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:18.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:18.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:19.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:19.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:19.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:19.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:18 smithi181 bash[59345]: cluster 2023-10-09T09:33:16.679175+0000 osd.7 (osd.7) 511 : cluster 3 failed to encode map e1120 with expected crc 2023-10-09T09:33:19.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:18 smithi181 bash[59345]: cluster 2023-10-09T09:33:17.115118+0000 osd.7 (osd.7) 512 : cluster 0 7.11 scrub starts 2023-10-09T09:33:19.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:18 smithi181 bash[59345]: cluster 2023-10-09T09:33:17.764213+0000 mgr.y (mgr.34107) 813 : cluster 0 pgmap v1535: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 887/166425 objects degraded (0.533%); 1977/166425 objects misplaced (1.188%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T09:33:19.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:19.150 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:19.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:19.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:19.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:19.281 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:19.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:19.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:19.387 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:19.415 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:19.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:19.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:19.520 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:19.548 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:19.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:19.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:19.653 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:19.688 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:19.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:19.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:19.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:19.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:19 smithi163 bash[75425]: cluster 2023-10-09T09:33:19.112205+0000 osd.7 (osd.7) 513 : cluster 0 7.11 scrub starts 2023-10-09T09:33:19.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:19 smithi163 bash[78107]: cluster 2023-10-09T09:33:19.112205+0000 osd.7 (osd.7) 513 : cluster 0 7.11 scrub starts 2023-10-09T09:33:19.823 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:19.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:19.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:19.929 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:19.952 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:19.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:19.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:20.055 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:20.063 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:19 smithi181 bash[59345]: cluster 2023-10-09T09:33:19.112205+0000 osd.7 (osd.7) 513 : cluster 0 7.11 scrub starts 2023-10-09T09:33:20.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:20.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:20.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:20.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:20.221 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:20.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:20.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:20.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:20.360 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:20.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:20.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:20.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:20.486 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:20.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:20.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:20.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:20.614 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:20.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:20.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:20.679 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:20 smithi163 bash[78107]: cluster 2023-10-09T09:33:19.683295+0000 osd.3 (osd.3) 3 : cluster 0 7.3 scrub starts 2023-10-09T09:33:20.680 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:20 smithi163 bash[78107]: cluster 2023-10-09T09:33:19.764777+0000 mgr.y (mgr.34107) 814 : cluster 0 pgmap v1536: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 887/163026 objects degraded (0.544%); 1587/163026 objects misplaced (0.973%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T09:33:20.680 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:20 smithi163 bash[78107]: cluster 2023-10-09T09:33:20.154225+0000 osd.7 (osd.7) 514 : cluster 0 7.11 scrub starts 2023-10-09T09:33:20.680 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:20 smithi163 bash[78107]: cluster 2023-10-09T09:33:20.591531+0000 mon.a (mon.0) 1650 : cluster 3 Health check update: Degraded data redundancy: 887/166425 objects degraded (0.533%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:20.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:20 smithi163 bash[75425]: cluster 2023-10-09T09:33:19.683295+0000 osd.3 (osd.3) 3 : cluster 0 7.3 scrub starts 2023-10-09T09:33:20.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:20 smithi163 bash[75425]: cluster 2023-10-09T09:33:19.764777+0000 mgr.y (mgr.34107) 814 : cluster 0 pgmap v1536: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 887/163026 objects degraded (0.544%); 1587/163026 objects misplaced (0.973%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T09:33:20.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:20 smithi163 bash[75425]: cluster 2023-10-09T09:33:20.154225+0000 osd.7 (osd.7) 514 : cluster 0 7.11 scrub starts 2023-10-09T09:33:20.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:20 smithi163 bash[75425]: cluster 2023-10-09T09:33:20.591531+0000 mon.a (mon.0) 1650 : cluster 3 Health check update: Degraded data redundancy: 887/166425 objects degraded (0.533%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:20.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:20.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:20.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:20.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:20.858 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:20 smithi181 bash[59345]: cluster 2023-10-09T09:33:19.683295+0000 osd.3 (osd.3) 3 : cluster 0 7.3 scrub starts 2023-10-09T09:33:20.858 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:20 smithi181 bash[59345]: cluster 2023-10-09T09:33:19.764777+0000 mgr.y (mgr.34107) 814 : cluster 0 pgmap v1536: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 887/163026 objects degraded (0.544%); 1587/163026 objects misplaced (0.973%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T09:33:20.858 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:20 smithi181 bash[59345]: cluster 2023-10-09T09:33:20.154225+0000 osd.7 (osd.7) 514 : cluster 0 7.11 scrub starts 2023-10-09T09:33:20.859 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:20 smithi181 bash[59345]: cluster 2023-10-09T09:33:20.591531+0000 mon.a (mon.0) 1650 : cluster 3 Health check update: Degraded data redundancy: 887/166425 objects degraded (0.533%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:20.883 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:20.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T09:33:20.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:21.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:21.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:21.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:21.128 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:21.153 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:21.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:21.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:21.182 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T09:33:21.256 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:21.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:21.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:21.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:21.418 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:21.455 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:21.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:21.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:21.583 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:21.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:21.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:21.686 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:21.710 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:21.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:21.816 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:21.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:21.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:21.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:21.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:21.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:21.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:21.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:22.084 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:22.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:21 smithi181 bash[59345]: cluster 2023-10-09T09:33:21.765453+0000 mgr.y (mgr.34107) 815 : cluster 0 pgmap v1537: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 887/157515 objects degraded (0.563%); 1587/157515 objects misplaced (1.008%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T09:33:22.112 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:22.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:22.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:22.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:22.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:21 smithi163 bash[75425]: cluster 2023-10-09T09:33:21.765453+0000 mgr.y (mgr.34107) 815 : cluster 0 pgmap v1537: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 887/157515 objects degraded (0.563%); 1587/157515 objects misplaced (1.008%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T09:33:22.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:21 smithi163 bash[78107]: cluster 2023-10-09T09:33:21.765453+0000 mgr.y (mgr.34107) 815 : cluster 0 pgmap v1537: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 887/157515 objects degraded (0.563%); 1587/157515 objects misplaced (1.008%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T09:33:22.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:22.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:22.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:22.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:22.375 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:22.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:22.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:22.481 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:22.504 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:22.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:22.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:22.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:22.643 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:22.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:22.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:22.748 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:22.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:22.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:22.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:22.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:22.890 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:22 smithi181 bash[59345]: cluster 2023-10-09T09:33:22.226939+0000 osd.7 (osd.7) 515 : cluster 0 7.11 scrub starts 2023-10-09T09:33:22.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:22.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:22.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:23.015 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:23.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:23.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:23.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:23.146 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:23.155 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:22 smithi163 bash[75425]: cluster 2023-10-09T09:33:22.226939+0000 osd.7 (osd.7) 515 : cluster 0 7.11 scrub starts 2023-10-09T09:33:23.156 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:22 smithi163 bash[78107]: cluster 2023-10-09T09:33:22.226939+0000 osd.7 (osd.7) 515 : cluster 0 7.11 scrub starts 2023-10-09T09:33:23.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:23.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:23.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:23.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:33:23.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:33:23.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:23.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:23.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:23.428 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:23.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:23.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:23.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:33:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:33:23] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.33.4" 2023-10-09T09:33:23.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:23.608 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:23.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:23.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:23.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:23.738 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:23.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:23.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:23.843 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:23.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:23.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:23.992 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:24.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:24.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:24.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:24.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:24.131 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:23 smithi181 bash[59345]: cluster 2023-10-09T09:33:22.615374+0000 osd.3 (osd.3) 4 : cluster 0 7.13 scrub starts 2023-10-09T09:33:24.131 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:23 smithi181 bash[59345]: cluster 2023-10-09T09:33:23.216225+0000 osd.7 (osd.7) 516 : cluster 0 7.11 scrub starts 2023-10-09T09:33:24.131 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:23 smithi181 bash[59345]: cluster 2023-10-09T09:33:23.766075+0000 mgr.y (mgr.34107) 816 : cluster 0 pgmap v1538: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 887/153471 objects degraded (0.578%); 1165/153471 objects misplaced (0.759%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T09:33:24.152 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:24.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:24.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:23 smithi163 bash[75425]: cluster 2023-10-09T09:33:22.615374+0000 osd.3 (osd.3) 4 : cluster 0 7.13 scrub starts 2023-10-09T09:33:24.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:23 smithi163 bash[75425]: cluster 2023-10-09T09:33:23.216225+0000 osd.7 (osd.7) 516 : cluster 0 7.11 scrub starts 2023-10-09T09:33:24.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:23 smithi163 bash[75425]: cluster 2023-10-09T09:33:23.766075+0000 mgr.y (mgr.34107) 816 : cluster 0 pgmap v1538: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 887/153471 objects degraded (0.578%); 1165/153471 objects misplaced (0.759%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T09:33:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:23 smithi163 bash[78107]: cluster 2023-10-09T09:33:22.615374+0000 osd.3 (osd.3) 4 : cluster 0 7.13 scrub starts 2023-10-09T09:33:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:23 smithi163 bash[78107]: cluster 2023-10-09T09:33:23.216225+0000 osd.7 (osd.7) 516 : cluster 0 7.11 scrub starts 2023-10-09T09:33:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:23 smithi163 bash[78107]: cluster 2023-10-09T09:33:23.766075+0000 mgr.y (mgr.34107) 816 : cluster 0 pgmap v1538: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 887/153471 objects degraded (0.578%); 1165/153471 objects misplaced (0.759%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T09:33:24.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:24.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:24.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:24.397 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:24.431 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:24.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:24.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:24.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:24.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:24.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:24.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:24.670 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:24.705 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:24.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:24.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:24.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:24.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:24.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:24.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:24.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:24.974 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:24.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:24.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:25.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:25.107 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:25.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:25.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:25.210 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:25.243 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:25.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:25.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:25.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:25.382 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:25.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:25.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:25.490 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:25.501 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:25 smithi163 bash[75425]: audit 2023-10-09T09:33:25.370982+0000 mon.a (mon.0) 1651 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:25.501 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:25 smithi163 bash[78107]: audit 2023-10-09T09:33:25.370982+0000 mon.a (mon.0) 1651 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:25.525 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:33:25.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:25.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:33:25.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:25.661 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:25.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:25.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:25.764 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:25.792 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:25.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:25.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:25 smithi181 bash[59345]: audit 2023-10-09T09:33:25.370982+0000 mon.a (mon.0) 1651 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:25.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:25.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:25.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:25.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:25.967 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:33:25.967 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:33:25.968 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:33:25.968 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T09:33:25.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T09:33:26.032 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:33:26.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:26.059 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:33:26 smithi163 systemd[1]: Stopping Ceph osd.0 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:33:26.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:26.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:26.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:26.201 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:26.224 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:26.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:26.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:26.334 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:26.367 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:26.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:26.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:26.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:26 smithi181 bash[59345]: cluster 2023-10-09T09:33:25.160427+0000 osd.7 (osd.7) 517 : cluster 0 7.11 scrub starts 2023-10-09T09:33:26.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:26 smithi181 bash[59345]: cluster 2023-10-09T09:33:25.595145+0000 mon.a (mon.0) 1652 : cluster 3 Health check update: Degraded data redundancy: 887/153471 objects degraded (0.578%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:26.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:26 smithi181 bash[59345]: cluster 2023-10-09T09:33:25.649680+0000 osd.3 (osd.3) 5 : cluster 0 7.10 scrub starts 2023-10-09T09:33:26.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:26 smithi181 bash[59345]: cluster 2023-10-09T09:33:25.766732+0000 mgr.y (mgr.34107) 817 : cluster 0 pgmap v1539: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 887/149706 objects degraded (0.592%); 1165/149706 objects misplaced (0.778%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T09:33:26.503 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:26.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:26.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:26.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:26.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:26.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[75425]: cluster 2023-10-09T09:33:25.160427+0000 osd.7 (osd.7) 517 : cluster 0 7.11 scrub starts 2023-10-09T09:33:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[75425]: cluster 2023-10-09T09:33:25.595145+0000 mon.a (mon.0) 1652 : cluster 3 Health check update: Degraded data redundancy: 887/153471 objects degraded (0.578%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[75425]: cluster 2023-10-09T09:33:25.649680+0000 osd.3 (osd.3) 5 : cluster 0 7.10 scrub starts 2023-10-09T09:33:26.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[75425]: cluster 2023-10-09T09:33:25.766732+0000 mgr.y (mgr.34107) 817 : cluster 0 pgmap v1539: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 887/149706 objects degraded (0.592%); 1165/149706 objects misplaced (0.778%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T09:33:26.719 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[84060]: debug 2023-10-09T09:33:26.629+0000 7f8023c5c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:33:26.719 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[84060]: debug 2023-10-09T09:33:26.629+0000 7f8023c5c700 -1 osd.0 1120 *** Got signal Terminated *** 2023-10-09T09:33:26.719 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[84060]: debug 2023-10-09T09:33:26.629+0000 7f8023c5c700 -1 osd.0 1120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:33:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[78107]: cluster 2023-10-09T09:33:25.160427+0000 osd.7 (osd.7) 517 : cluster 0 7.11 scrub starts 2023-10-09T09:33:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[78107]: cluster 2023-10-09T09:33:25.595145+0000 mon.a (mon.0) 1652 : cluster 3 Health check update: Degraded data redundancy: 887/153471 objects degraded (0.578%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:26.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[78107]: cluster 2023-10-09T09:33:25.649680+0000 osd.3 (osd.3) 5 : cluster 0 7.10 scrub starts 2023-10-09T09:33:26.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:26 smithi163 bash[78107]: cluster 2023-10-09T09:33:25.766732+0000 mgr.y (mgr.34107) 817 : cluster 0 pgmap v1539: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 887/149706 objects degraded (0.592%); 1165/149706 objects misplaced (0.778%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T09:33:26.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:26.769 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:26.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:26.876 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:26.903 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:26.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:26.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:27.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:27.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:27.038 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:27.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:27.139 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:27.169 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:27.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:27.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:27.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:27.303 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:27.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:27.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:27.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:27.437 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:27.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:27.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:27.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:27.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:27.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:27.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:27.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:27.703 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:27 smithi163 bash[78107]: cluster 2023-10-09T09:33:26.635248+0000 mon.a (mon.0) 1653 : cluster 1 osd.0 marked itself down and dead 2023-10-09T09:33:27.703 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:27 smithi163 bash[78107]: cluster 2023-10-09T09:33:26.653028+0000 osd.3 (osd.3) 6 : cluster 0 7.3 scrub starts 2023-10-09T09:33:27.704 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:27 smithi163 bash[75425]: cluster 2023-10-09T09:33:26.635248+0000 mon.a (mon.0) 1653 : cluster 1 osd.0 marked itself down and dead 2023-10-09T09:33:27.704 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:27 smithi163 bash[75425]: cluster 2023-10-09T09:33:26.653028+0000 osd.3 (osd.3) 6 : cluster 0 7.3 scrub starts 2023-10-09T09:33:27.706 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:27.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:27.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:27.838 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:27.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:27.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:27.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:27.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:27.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:27.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:27 smithi181 bash[59345]: cluster 2023-10-09T09:33:26.635248+0000 mon.a (mon.0) 1653 : cluster 1 osd.0 marked itself down and dead 2023-10-09T09:33:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:27 smithi181 bash[59345]: cluster 2023-10-09T09:33:26.653028+0000 osd.3 (osd.3) 6 : cluster 0 7.3 scrub starts 2023-10-09T09:33:28.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:28.094 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:28.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:28.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:28.170 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:33:28.173 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:33:28.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:28.231 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:28.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:28.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:28.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:28.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:28.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:28.469 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:28.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:28.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:28.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:28.603 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:28.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:28.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:28.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:28.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:28 smithi181 bash[59345]: cluster 2023-10-09T09:33:27.229651+0000 osd.7 (osd.7) 518 : cluster 0 7.11 scrub starts 2023-10-09T09:33:28.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:28 smithi181 bash[59345]: cluster 2023-10-09T09:33:27.603381+0000 mon.a (mon.0) 1654 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:33:28.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:28 smithi181 bash[59345]: cluster 2023-10-09T09:33:27.623286+0000 osd.5 (osd.5) 671 : cluster 3 failed to encode map e1121 with expected crc 2023-10-09T09:33:28.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:28 smithi181 bash[59345]: cluster 2023-10-09T09:33:27.624247+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e1121 with expected crc 2023-10-09T09:33:28.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:28 smithi181 bash[59345]: cluster 2023-10-09T09:33:27.627406+0000 mon.a (mon.0) 1655 : cluster 0 osdmap e1121: 8 total, 7 up, 8 in 2023-10-09T09:33:28.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:28 smithi181 bash[59345]: cluster 2023-10-09T09:33:27.767140+0000 mgr.y (mgr.34107) 818 : cluster 0 pgmap v1541: 99 pgs: 18 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 887/142629 objects degraded (0.622%); 1165/142629 objects misplaced (0.817%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T09:33:28.772 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:28.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:28.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:28.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:28.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[75425]: cluster 2023-10-09T09:33:27.229651+0000 osd.7 (osd.7) 518 : cluster 0 7.11 scrub starts 2023-10-09T09:33:28.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[75425]: cluster 2023-10-09T09:33:27.603381+0000 mon.a (mon.0) 1654 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:33:28.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[75425]: cluster 2023-10-09T09:33:27.623286+0000 osd.5 (osd.5) 671 : cluster 3 failed to encode map e1121 with expected crc 2023-10-09T09:33:28.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[75425]: cluster 2023-10-09T09:33:27.624247+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e1121 with expected crc 2023-10-09T09:33:28.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[75425]: cluster 2023-10-09T09:33:27.627406+0000 mon.a (mon.0) 1655 : cluster 0 osdmap e1121: 8 total, 7 up, 8 in 2023-10-09T09:33:28.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[75425]: cluster 2023-10-09T09:33:27.767140+0000 mgr.y (mgr.34107) 818 : cluster 0 pgmap v1541: 99 pgs: 18 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 887/142629 objects degraded (0.622%); 1165/142629 objects misplaced (0.817%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T09:33:28.886 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[78107]: cluster 2023-10-09T09:33:27.229651+0000 osd.7 (osd.7) 518 : cluster 0 7.11 scrub starts 2023-10-09T09:33:28.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[78107]: cluster 2023-10-09T09:33:27.603381+0000 mon.a (mon.0) 1654 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:33:28.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[78107]: cluster 2023-10-09T09:33:27.623286+0000 osd.5 (osd.5) 671 : cluster 3 failed to encode map e1121 with expected crc 2023-10-09T09:33:28.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[78107]: cluster 2023-10-09T09:33:27.624247+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e1121 with expected crc 2023-10-09T09:33:28.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[78107]: cluster 2023-10-09T09:33:27.627406+0000 mon.a (mon.0) 1655 : cluster 0 osdmap e1121: 8 total, 7 up, 8 in 2023-10-09T09:33:28.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:28 smithi163 bash[78107]: cluster 2023-10-09T09:33:27.767140+0000 mgr.y (mgr.34107) 818 : cluster 0 pgmap v1541: 99 pgs: 18 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 887/142629 objects degraded (0.622%); 1165/142629 objects misplaced (0.817%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T09:33:28.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:28.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:29.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:29.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:29.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:29.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:29.155 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:29.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:29.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:29.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:29.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:29.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:29.388 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:29.414 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:29.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:29.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:29.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:29.540 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:29.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:29.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:29.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:29.666 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:29.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:29.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:29.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:29.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.631590+0000 osd.5 (osd.5) 672 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.634575+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.634705+0000 osd.5 (osd.5) 673 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.634956+0000 osd.7 (osd.7) 519 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.634973+0000 mon.a (mon.0) 1656 : cluster 0 osdmap e1122: 8 total, 7 up, 8 in 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.635565+0000 osd.4 (osd.4) 547 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.636794+0000 osd.5 (osd.5) 674 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.637129+0000 osd.7 (osd.7) 520 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.638379+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.639138+0000 osd.4 (osd.4) 548 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.639276+0000 osd.5 (osd.5) 675 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.639404+0000 osd.7 (osd.7) 521 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.778 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.640637+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.779 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:28.641598+0000 osd.4 (osd.4) 549 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.779 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:29 smithi181 bash[59345]: cluster 2023-10-09T09:33:29.042660+0000 osd.2 (osd.2) 4 : cluster 0 7.8 scrub starts 2023-10-09T09:33:29.794 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:29.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:29.899 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:29.933 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:29.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:29.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.631590+0000 osd.5 (osd.5) 672 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.634575+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.634705+0000 osd.5 (osd.5) 673 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.634956+0000 osd.7 (osd.7) 519 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.634973+0000 mon.a (mon.0) 1656 : cluster 0 osdmap e1122: 8 total, 7 up, 8 in 2023-10-09T09:33:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.635565+0000 osd.4 (osd.4) 547 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.636794+0000 osd.5 (osd.5) 674 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.637129+0000 osd.7 (osd.7) 520 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.638379+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.639138+0000 osd.4 (osd.4) 548 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.639276+0000 osd.5 (osd.5) 675 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.639404+0000 osd.7 (osd.7) 521 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.640637+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:28.641598+0000 osd.4 (osd.4) 549 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[78107]: cluster 2023-10-09T09:33:29.042660+0000 osd.2 (osd.2) 4 : cluster 0 7.8 scrub starts 2023-10-09T09:33:29.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.631590+0000 osd.5 (osd.5) 672 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.634575+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.634705+0000 osd.5 (osd.5) 673 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.634956+0000 osd.7 (osd.7) 519 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.634973+0000 mon.a (mon.0) 1656 : cluster 0 osdmap e1122: 8 total, 7 up, 8 in 2023-10-09T09:33:29.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.635565+0000 osd.4 (osd.4) 547 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.636794+0000 osd.5 (osd.5) 674 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.637129+0000 osd.7 (osd.7) 520 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.638379+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.639138+0000 osd.4 (osd.4) 548 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.639276+0000 osd.5 (osd.5) 675 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.639404+0000 osd.7 (osd.7) 521 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.640637+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:28.641598+0000 osd.4 (osd.4) 549 : cluster 3 failed to encode map e1122 with expected crc 2023-10-09T09:33:29.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:29 smithi163 bash[75425]: cluster 2023-10-09T09:33:29.042660+0000 osd.2 (osd.2) 4 : cluster 0 7.8 scrub starts 2023-10-09T09:33:30.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:30.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:30.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:30.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:30.172 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:30.200 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:30.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:30.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:30.332 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:30.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:30.436 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:30.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:30.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:30.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:30.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:30.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:30.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:30.701 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:30.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[78107]: cluster 2023-10-09T09:33:29.317995+0000 osd.2 (osd.2) 5 : cluster 0 7.8 scrub ok 2023-10-09T09:33:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[78107]: cluster 2023-10-09T09:33:29.633350+0000 osd.5 (osd.5) 676 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[78107]: cluster 2023-10-09T09:33:29.634224+0000 osd.6 (osd.6) 405 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[78107]: cluster 2023-10-09T09:33:29.637854+0000 mon.a (mon.0) 1657 : cluster 0 osdmap e1123: 8 total, 7 up, 8 in 2023-10-09T09:33:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[78107]: cluster 2023-10-09T09:33:29.638883+0000 osd.6 (osd.6) 406 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[78107]: cluster 2023-10-09T09:33:29.675124+0000 osd.3 (osd.3) 7 : cluster 0 7.13 scrub starts 2023-10-09T09:33:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[78107]: cluster 2023-10-09T09:33:29.770524+0000 mgr.y (mgr.34107) 819 : cluster 0 pgmap v1544: 99 pgs: 6 active+undersized+degraded, 12 stale+active+clean, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 73 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6644/140169 objects degraded (4.740%); 1165/140169 objects misplaced (0.831%) 2023-10-09T09:33:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[78107]: cluster 2023-10-09T09:33:29.995822+0000 osd.3 (osd.3) 8 : cluster 0 7.13 scrub ok 2023-10-09T09:33:30.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[78107]: cluster 2023-10-09T09:33:30.596416+0000 mon.a (mon.0) 1658 : cluster 3 Health check update: Degraded data redundancy: 887/142629 objects degraded (0.622%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:30.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[75425]: cluster 2023-10-09T09:33:29.317995+0000 osd.2 (osd.2) 5 : cluster 0 7.8 scrub ok 2023-10-09T09:33:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[75425]: cluster 2023-10-09T09:33:29.633350+0000 osd.5 (osd.5) 676 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[75425]: cluster 2023-10-09T09:33:29.634224+0000 osd.6 (osd.6) 405 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[75425]: cluster 2023-10-09T09:33:29.637854+0000 mon.a (mon.0) 1657 : cluster 0 osdmap e1123: 8 total, 7 up, 8 in 2023-10-09T09:33:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[75425]: cluster 2023-10-09T09:33:29.638883+0000 osd.6 (osd.6) 406 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[75425]: cluster 2023-10-09T09:33:29.675124+0000 osd.3 (osd.3) 7 : cluster 0 7.13 scrub starts 2023-10-09T09:33:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[75425]: cluster 2023-10-09T09:33:29.770524+0000 mgr.y (mgr.34107) 819 : cluster 0 pgmap v1544: 99 pgs: 6 active+undersized+degraded, 12 stale+active+clean, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 73 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6644/140169 objects degraded (4.740%); 1165/140169 objects misplaced (0.831%) 2023-10-09T09:33:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[75425]: cluster 2023-10-09T09:33:29.995822+0000 osd.3 (osd.3) 8 : cluster 0 7.13 scrub ok 2023-10-09T09:33:30.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[75425]: cluster 2023-10-09T09:33:30.596416+0000 mon.a (mon.0) 1658 : cluster 3 Health check update: Degraded data redundancy: 887/142629 objects degraded (0.622%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:30.730 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:30.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:30.836 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:30.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:30.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:30.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:30.969 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:30.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:30 smithi181 bash[59345]: cluster 2023-10-09T09:33:29.317995+0000 osd.2 (osd.2) 5 : cluster 0 7.8 scrub ok 2023-10-09T09:33:30.978 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:30 smithi181 bash[59345]: cluster 2023-10-09T09:33:29.633350+0000 osd.5 (osd.5) 676 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:30.978 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:30 smithi181 bash[59345]: cluster 2023-10-09T09:33:29.634224+0000 osd.6 (osd.6) 405 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:30.978 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:30 smithi181 bash[59345]: cluster 2023-10-09T09:33:29.637854+0000 mon.a (mon.0) 1657 : cluster 0 osdmap e1123: 8 total, 7 up, 8 in 2023-10-09T09:33:30.978 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:30 smithi181 bash[59345]: cluster 2023-10-09T09:33:29.638883+0000 osd.6 (osd.6) 406 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:30.978 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:30 smithi181 bash[59345]: cluster 2023-10-09T09:33:29.675124+0000 osd.3 (osd.3) 7 : cluster 0 7.13 scrub starts 2023-10-09T09:33:30.978 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:30 smithi181 bash[59345]: cluster 2023-10-09T09:33:29.770524+0000 mgr.y (mgr.34107) 819 : cluster 0 pgmap v1544: 99 pgs: 6 active+undersized+degraded, 12 stale+active+clean, 7 active+undersized, 1 active+recovering+undersized+degraded+remapped, 73 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6644/140169 objects degraded (4.740%); 1165/140169 objects misplaced (0.831%) 2023-10-09T09:33:30.979 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:30 smithi181 bash[59345]: cluster 2023-10-09T09:33:29.995822+0000 osd.3 (osd.3) 8 : cluster 0 7.13 scrub ok 2023-10-09T09:33:30.979 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:30 smithi181 bash[59345]: cluster 2023-10-09T09:33:30.596416+0000 mon.a (mon.0) 1658 : cluster 3 Health check update: Degraded data redundancy: 887/142629 objects degraded (0.622%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:33:30.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:31.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:31.105 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:31.113 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:33:30 smithi163 bash[141244]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T09:33:31.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:31.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:31.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:31.242 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:31.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:31.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:31.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:31.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:31.408 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:31.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:31.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:31.550 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:31.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:31.651 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:31.676 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:31.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:31.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:31.783 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:31.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:31.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:31 smithi181 bash[59345]: cluster 2023-10-09T09:33:29.690239+0000 osd.7 (osd.7) 522 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:31.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:31.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:31.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:31.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:31 smithi163 bash[78107]: cluster 2023-10-09T09:33:29.690239+0000 osd.7 (osd.7) 522 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:31.938 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:33:31 smithi163 bash[141493]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T09:33:31.938 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:31 smithi163 bash[75425]: cluster 2023-10-09T09:33:29.690239+0000 osd.7 (osd.7) 522 : cluster 3 failed to encode map e1123 with expected crc 2023-10-09T09:33:31.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:31.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:31.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:32.058 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:32.085 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:32.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:32.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:32.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:32.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:32.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:32.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:32.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:32.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:32.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:32.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:32.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:32.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:32.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:32.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:32.598 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:32.624 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:32.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:32.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:32.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:32.737 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:32 smithi163 bash[75425]: cluster 2023-10-09T09:33:31.694051+0000 osd.3 (osd.3) 9 : cluster 0 7.10 scrub starts 2023-10-09T09:33:32.737 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:32 smithi163 bash[75425]: cluster 2023-10-09T09:33:31.771640+0000 mgr.y (mgr.34107) 820 : cluster 0 pgmap v1545: 99 pgs: 1 active+recovering+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 23591/134139 objects degraded (17.587%); 1.4 MiB/s, 69 objects/s recovering 2023-10-09T09:33:32.737 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:32 smithi163 bash[78107]: cluster 2023-10-09T09:33:31.694051+0000 osd.3 (osd.3) 9 : cluster 0 7.10 scrub starts 2023-10-09T09:33:32.737 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:32 smithi163 bash[78107]: cluster 2023-10-09T09:33:31.771640+0000 mgr.y (mgr.34107) 820 : cluster 0 pgmap v1545: 99 pgs: 1 active+recovering+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 23591/134139 objects degraded (17.587%); 1.4 MiB/s, 69 objects/s recovering 2023-10-09T09:33:32.754 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:32.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:32.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:32.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:32.885 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:32.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:32.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:32.989 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:32.998 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:32 smithi181 bash[59345]: cluster 2023-10-09T09:33:31.694051+0000 osd.3 (osd.3) 9 : cluster 0 7.10 scrub starts 2023-10-09T09:33:32.998 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:32 smithi181 bash[59345]: cluster 2023-10-09T09:33:31.771640+0000 mgr.y (mgr.34107) 820 : cluster 0 pgmap v1545: 99 pgs: 1 active+recovering+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 23591/134139 objects degraded (17.587%); 1.4 MiB/s, 69 objects/s recovering 2023-10-09T09:33:33.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:33.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:33.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:33.148 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:33.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:33.255 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:33.278 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:33.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:33.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:33.380 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:33.423 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:33.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:33.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:33.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:33.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:33.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:33.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:33.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:33.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:33 smithi163 bash[75425]: cluster 2023-10-09T09:33:32.287578+0000 osd.7 (osd.7) 523 : cluster 0 7.11 scrub starts 2023-10-09T09:33:33.665 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:33:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:33:33] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.33.4" 2023-10-09T09:33:33.695 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:33.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:33.800 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:33.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:33 smithi181 bash[59345]: cluster 2023-10-09T09:33:32.287578+0000 osd.7 (osd.7) 523 : cluster 0 7.11 scrub starts 2023-10-09T09:33:33.824 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:33.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:33.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:33.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:33.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:33.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:33.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:33 smithi163 bash[78107]: cluster 2023-10-09T09:33:32.287578+0000 osd.7 (osd.7) 523 : cluster 0 7.11 scrub starts 2023-10-09T09:33:34.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:34.091 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:34.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:34.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:34.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:34.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:34.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:34.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:34.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:34.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:34.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:34.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:34.463 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:34.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:34.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:34.601 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:34.639 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:34.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:34.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:34.747 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:34.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:34.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:34.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:34.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:34.926 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:34 smithi163 bash[78107]: cluster 2023-10-09T09:33:33.686798+0000 osd.3 (osd.3) 10 : cluster 0 7.3 scrub starts 2023-10-09T09:33:34.926 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:34 smithi163 bash[78107]: cluster 2023-10-09T09:33:33.772375+0000 mgr.y (mgr.34107) 821 : cluster 0 pgmap v1546: 99 pgs: 1 active+recovering+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 23279/132294 objects degraded (17.596%); 1.4 MiB/s, 67 objects/s recovering 2023-10-09T09:33:34.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:34.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:34.927 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:34 smithi163 bash[75425]: cluster 2023-10-09T09:33:33.686798+0000 osd.3 (osd.3) 10 : cluster 0 7.3 scrub starts 2023-10-09T09:33:34.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:34 smithi163 bash[75425]: cluster 2023-10-09T09:33:33.772375+0000 mgr.y (mgr.34107) 821 : cluster 0 pgmap v1546: 99 pgs: 1 active+recovering+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 23279/132294 objects degraded (17.596%); 1.4 MiB/s, 67 objects/s recovering 2023-10-09T09:33:35.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:35.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:35.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:35.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:34 smithi181 bash[59345]: cluster 2023-10-09T09:33:33.686798+0000 osd.3 (osd.3) 10 : cluster 0 7.3 scrub starts 2023-10-09T09:33:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:34 smithi181 bash[59345]: cluster 2023-10-09T09:33:33.772375+0000 mgr.y (mgr.34107) 821 : cluster 0 pgmap v1546: 99 pgs: 1 active+recovering+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 23279/132294 objects degraded (17.596%); 1.4 MiB/s, 67 objects/s recovering 2023-10-09T09:33:35.162 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:35.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:35.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:35.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:35.293 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:35.326 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:35.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:35.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:35.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:35.454 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:35.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:35.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:35.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:35.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:35.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:35.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:35.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:35.711 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:35.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:35.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:35.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:35 smithi163 bash[75425]: cluster 2023-10-09T09:33:34.231304+0000 osd.7 (osd.7) 524 : cluster 0 7.11 scrub starts 2023-10-09T09:33:35.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:35 smithi163 bash[75425]: cluster 2023-10-09T09:33:35.597572+0000 mon.a (mon.0) 1659 : cluster 3 Health check update: Degraded data redundancy: 23279/132294 objects degraded (17.596%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:35.716 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:35 smithi163 bash[78107]: cluster 2023-10-09T09:33:34.231304+0000 osd.7 (osd.7) 524 : cluster 0 7.11 scrub starts 2023-10-09T09:33:35.716 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:35 smithi163 bash[78107]: cluster 2023-10-09T09:33:35.597572+0000 mon.a (mon.0) 1659 : cluster 3 Health check update: Degraded data redundancy: 23279/132294 objects degraded (17.596%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:35.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:35.841 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:35.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:35.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:35.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:35.952 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:35 smithi181 bash[59345]: cluster 2023-10-09T09:33:34.231304+0000 osd.7 (osd.7) 524 : cluster 0 7.11 scrub starts 2023-10-09T09:33:35.952 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:35 smithi181 bash[59345]: cluster 2023-10-09T09:33:35.597572+0000 mon.a (mon.0) 1659 : cluster 3 Health check update: Degraded data redundancy: 23279/132294 objects degraded (17.596%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:35.965 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:35.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:35.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:36.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:36.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:36.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:36.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:36.189 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:36.209 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:36.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:36.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:36.312 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:36.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:36.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:36.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:36.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:36.467 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:36.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:36.571 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:36.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:36.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:36.707 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:36 smithi163 bash[78107]: cluster 2023-10-09T09:33:35.639716+0000 osd.3 (osd.3) 11 : cluster 0 7.10 scrub starts 2023-10-09T09:33:36.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:36 smithi163 bash[78107]: cluster 2023-10-09T09:33:35.773106+0000 mgr.y (mgr.34107) 822 : cluster 0 pgmap v1547: 99 pgs: 1 active+recovering+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 22643/126777 objects degraded (17.860%); 1.1 MiB/s, 51 objects/s recovering 2023-10-09T09:33:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:36 smithi163 bash[75425]: cluster 2023-10-09T09:33:35.639716+0000 osd.3 (osd.3) 11 : cluster 0 7.10 scrub starts 2023-10-09T09:33:36.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:36 smithi163 bash[75425]: cluster 2023-10-09T09:33:35.773106+0000 mgr.y (mgr.34107) 822 : cluster 0 pgmap v1547: 99 pgs: 1 active+recovering+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 22643/126777 objects degraded (17.860%); 1.1 MiB/s, 51 objects/s recovering 2023-10-09T09:33:36.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:36.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:36.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:36.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:36.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:36.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:36.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:36.980 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:36.990 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:36 smithi181 bash[59345]: cluster 2023-10-09T09:33:35.639716+0000 osd.3 (osd.3) 11 : cluster 0 7.10 scrub starts 2023-10-09T09:33:36.990 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:36 smithi181 bash[59345]: cluster 2023-10-09T09:33:35.773106+0000 mgr.y (mgr.34107) 822 : cluster 0 pgmap v1547: 99 pgs: 1 active+recovering+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 22643/126777 objects degraded (17.860%); 1.1 MiB/s, 51 objects/s recovering 2023-10-09T09:33:37.009 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:37.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:37.114 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:37.149 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:37.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:37.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:37.256 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:37.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:37.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:37.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:37.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:37.411 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:37.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:37.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:37.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:37.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:37.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:37.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:37.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:37.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:37.676 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:37 smithi163 bash[75425]: cluster 2023-10-09T09:33:36.427661+0000 osd.3 (osd.3) 12 : cluster 0 7.10 scrub ok 2023-10-09T09:33:37.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:37.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:37.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:37 smithi163 bash[78107]: cluster 2023-10-09T09:33:36.427661+0000 osd.3 (osd.3) 12 : cluster 0 7.10 scrub ok 2023-10-09T09:33:37.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:37.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:37.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:37.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:37.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:37 smithi181 bash[59345]: cluster 2023-10-09T09:33:36.427661+0000 osd.3 (osd.3) 12 : cluster 0 7.10 scrub ok 2023-10-09T09:33:37.913 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:37.938 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:37.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:37.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:38.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:38.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:38.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:38.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:38.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:38.220 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:38.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:38.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:38.323 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:38.349 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:38.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:38.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:38.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:38.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:38.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:38.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:38.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:38.606 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:38.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:38.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:38.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T09:33:38.715 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:38.762 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:38.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:38.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:38 smithi181 bash[59345]: cluster 2023-10-09T09:33:37.773850+0000 mgr.y (mgr.34107) 823 : cluster 0 pgmap v1548: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 19818/119616 objects degraded (16.568%); 1205/119616 objects misplaced (1.007%); 945 KiB/s, 111 objects/s recovering 2023-10-09T09:33:38.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:38 smithi181 bash[59345]: cluster 2023-10-09T09:33:38.127080+0000 osd.7 (osd.7) 525 : cluster 0 7.11 scrub starts 2023-10-09T09:33:38.861 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:38 smithi163 bash[75425]: cluster 2023-10-09T09:33:37.773850+0000 mgr.y (mgr.34107) 823 : cluster 0 pgmap v1548: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 19818/119616 objects degraded (16.568%); 1205/119616 objects misplaced (1.007%); 945 KiB/s, 111 objects/s recovering 2023-10-09T09:33:38.861 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:38 smithi163 bash[75425]: cluster 2023-10-09T09:33:38.127080+0000 osd.7 (osd.7) 525 : cluster 0 7.11 scrub starts 2023-10-09T09:33:38.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:38 smithi163 bash[78107]: cluster 2023-10-09T09:33:37.773850+0000 mgr.y (mgr.34107) 823 : cluster 0 pgmap v1548: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 19818/119616 objects degraded (16.568%); 1205/119616 objects misplaced (1.007%); 945 KiB/s, 111 objects/s recovering 2023-10-09T09:33:38.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:38 smithi163 bash[78107]: cluster 2023-10-09T09:33:38.127080+0000 osd.7 (osd.7) 525 : cluster 0 7.11 scrub starts 2023-10-09T09:33:38.862 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:33:38.863 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T09:33:38.870 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:38.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:38.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:38.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:39.001 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:39.029 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:39.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:39.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:39.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:39.174 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:39.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:39.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:39.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:39.305 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:39.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:39.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:39.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:39.445 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:39.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:39.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:39.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:39.588 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:39.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:39.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:39.695 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:39.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:39.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:39.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:39.832 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:39.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:39.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:39.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:39.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:39 smithi163 bash[75425]: cluster 2023-10-09T09:33:37.862083+0000 osd.6 (osd.6) 407 : cluster 0 7.12 scrub starts 2023-10-09T09:33:39.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:39 smithi163 bash[75425]: cluster 2023-10-09T09:33:38.334464+0000 osd.6 (osd.6) 408 : cluster 0 7.12 scrub ok 2023-10-09T09:33:39.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:39 smithi163 bash[75425]: cluster 2023-10-09T09:33:39.120273+0000 osd.7 (osd.7) 526 : cluster 0 7.11 scrub starts 2023-10-09T09:33:39.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:39 smithi163 bash[75425]: cluster 2023-10-09T09:33:39.578605+0000 osd.7 (osd.7) 527 : cluster 0 7.11 scrub ok 2023-10-09T09:33:39.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:39 smithi163 bash[78107]: cluster 2023-10-09T09:33:37.862083+0000 osd.6 (osd.6) 407 : cluster 0 7.12 scrub starts 2023-10-09T09:33:39.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:39 smithi163 bash[78107]: cluster 2023-10-09T09:33:38.334464+0000 osd.6 (osd.6) 408 : cluster 0 7.12 scrub ok 2023-10-09T09:33:39.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:39 smithi163 bash[78107]: cluster 2023-10-09T09:33:39.120273+0000 osd.7 (osd.7) 526 : cluster 0 7.11 scrub starts 2023-10-09T09:33:39.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:39 smithi163 bash[78107]: cluster 2023-10-09T09:33:39.578605+0000 osd.7 (osd.7) 527 : cluster 0 7.11 scrub ok 2023-10-09T09:33:39.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:39.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:39 smithi181 bash[59345]: cluster 2023-10-09T09:33:37.862083+0000 osd.6 (osd.6) 407 : cluster 0 7.12 scrub starts 2023-10-09T09:33:39.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:39 smithi181 bash[59345]: cluster 2023-10-09T09:33:38.334464+0000 osd.6 (osd.6) 408 : cluster 0 7.12 scrub ok 2023-10-09T09:33:39.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:39 smithi181 bash[59345]: cluster 2023-10-09T09:33:39.120273+0000 osd.7 (osd.7) 526 : cluster 0 7.11 scrub starts 2023-10-09T09:33:39.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:39 smithi181 bash[59345]: cluster 2023-10-09T09:33:39.578605+0000 osd.7 (osd.7) 527 : cluster 0 7.11 scrub ok 2023-10-09T09:33:40.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:40.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:40.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:40.131 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:40.162 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:40.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:40.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:40.303 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:40.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:40.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:40.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:40.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:40.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:40.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:40.535 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:40.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:40.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:40.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:40.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:40.691 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:40.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:40.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:40.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:40 smithi163 bash[75425]: cluster 2023-10-09T09:33:39.774808+0000 mgr.y (mgr.34107) 824 : cluster 0 pgmap v1549: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 19200/114003 objects degraded (16.842%); 1205/114003 objects misplaced (1.057%) 2023-10-09T09:33:40.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:40 smithi163 bash[75425]: audit 2023-10-09T09:33:40.376572+0000 mon.a (mon.0) 1660 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:40.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:40 smithi163 bash[75425]: audit 2023-10-09T09:33:40.377820+0000 mon.a (mon.0) 1661 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:40.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:40 smithi163 bash[75425]: cluster 2023-10-09T09:33:40.598908+0000 mon.a (mon.0) 1662 : cluster 3 Health check update: Degraded data redundancy: 19818/119616 objects degraded (16.568%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:40.794 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:40.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:40 smithi181 bash[59345]: cluster 2023-10-09T09:33:39.774808+0000 mgr.y (mgr.34107) 824 : cluster 0 pgmap v1549: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 19200/114003 objects degraded (16.842%); 1205/114003 objects misplaced (1.057%) 2023-10-09T09:33:40.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:40 smithi181 bash[59345]: audit 2023-10-09T09:33:40.376572+0000 mon.a (mon.0) 1660 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:40.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:40 smithi181 bash[59345]: audit 2023-10-09T09:33:40.377820+0000 mon.a (mon.0) 1661 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:40.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:40 smithi181 bash[59345]: cluster 2023-10-09T09:33:40.598908+0000 mon.a (mon.0) 1662 : cluster 3 Health check update: Degraded data redundancy: 19818/119616 objects degraded (16.568%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:40.821 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:40.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:40.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:40.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:40.947 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:40.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:40 smithi163 bash[78107]: cluster 2023-10-09T09:33:39.774808+0000 mgr.y (mgr.34107) 824 : cluster 0 pgmap v1549: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 19200/114003 objects degraded (16.842%); 1205/114003 objects misplaced (1.057%) 2023-10-09T09:33:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:40 smithi163 bash[78107]: audit 2023-10-09T09:33:40.376572+0000 mon.a (mon.0) 1660 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:40 smithi163 bash[78107]: audit 2023-10-09T09:33:40.377820+0000 mon.a (mon.0) 1661 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:40 smithi163 bash[78107]: cluster 2023-10-09T09:33:40.598908+0000 mon.a (mon.0) 1662 : cluster 3 Health check update: Degraded data redundancy: 19818/119616 objects degraded (16.568%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:41.035 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:33:41.036 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:33:41.053 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:41.081 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:41.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:41.183 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:41.213 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:41.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:41.316 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:41.333 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:33:41.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:41.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:41.376 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:33:41.412 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:33:41.442 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:41.445 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:33:41.467 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:41.472 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:33:41.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:41.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:41.516 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:33:41.551 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:33:41.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T09:33:41.551 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:33:41.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:33:41.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:33:41.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:33:41.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:33:41.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:33:41.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:33:41.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:41.613 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:41.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:41.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:41.717 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:41.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:41.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:41.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:41.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:41.878 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:41.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:41.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:41.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:41 smithi163 bash[78107]: cluster 2023-10-09T09:33:40.717574+0000 osd.3 (osd.3) 13 : cluster 0 7.3 scrub starts 2023-10-09T09:33:41.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:41 smithi163 bash[78107]: cluster 2023-10-09T09:33:41.242104+0000 osd.3 (osd.3) 14 : cluster 0 7.3 scrub ok 2023-10-09T09:33:41.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:41 smithi163 bash[78107]: audit 2023-10-09T09:33:41.295737+0000 mon.a (mon.0) 1663 : audit 0 from='client.? 172.21.15.163:0/1990249068' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:33:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:41 smithi163 bash[75425]: cluster 2023-10-09T09:33:40.717574+0000 osd.3 (osd.3) 13 : cluster 0 7.3 scrub starts 2023-10-09T09:33:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:41 smithi163 bash[75425]: cluster 2023-10-09T09:33:41.242104+0000 osd.3 (osd.3) 14 : cluster 0 7.3 scrub ok 2023-10-09T09:33:41.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:41 smithi163 bash[75425]: audit 2023-10-09T09:33:41.295737+0000 mon.a (mon.0) 1663 : audit 0 from='client.? 172.21.15.163:0/1990249068' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:33:41.983 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:41.993 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:41 smithi181 bash[59345]: cluster 2023-10-09T09:33:40.717574+0000 osd.3 (osd.3) 13 : cluster 0 7.3 scrub starts 2023-10-09T09:33:41.993 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:41 smithi181 bash[59345]: cluster 2023-10-09T09:33:41.242104+0000 osd.3 (osd.3) 14 : cluster 0 7.3 scrub ok 2023-10-09T09:33:41.993 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:41 smithi181 bash[59345]: audit 2023-10-09T09:33:41.295737+0000 mon.a (mon.0) 1663 : audit 0 from='client.? 172.21.15.163:0/1990249068' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:33:42.015 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:42.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:42.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:42.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:42.152 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:42.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:42.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:42.279 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:42.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:42.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:42.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:42.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:42.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:42.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:42.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:42.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:42.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:42.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:42.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:42.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:42.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:42.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:42.715 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:42 smithi181 bash[59345]: cluster 2023-10-09T09:33:41.775611+0000 mgr.y (mgr.34107) 825 : cluster 0 pgmap v1550: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 17224/104154 objects degraded (16.537%); 1043/104154 objects misplaced (1.001%) 2023-10-09T09:33:42.715 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:42 smithi181 bash[59345]: audit 2023-10-09T09:33:41.802137+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.163:0/3852231826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:33:42.715 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:42 smithi181 bash[59345]: audit 2023-10-09T09:33:41.802528+0000 mon.a (mon.0) 1664 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:33:42.732 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:33:42.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:42.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:42.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:42.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:42.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:42.941 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:42.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:42.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:42 smithi163 bash[75425]: cluster 2023-10-09T09:33:41.775611+0000 mgr.y (mgr.34107) 825 : cluster 0 pgmap v1550: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 17224/104154 objects degraded (16.537%); 1043/104154 objects misplaced (1.001%) 2023-10-09T09:33:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:42 smithi163 bash[75425]: audit 2023-10-09T09:33:41.802137+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.163:0/3852231826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:33:42.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:42 smithi163 bash[75425]: audit 2023-10-09T09:33:41.802528+0000 mon.a (mon.0) 1664 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:33:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:42 smithi163 bash[78107]: cluster 2023-10-09T09:33:41.775611+0000 mgr.y (mgr.34107) 825 : cluster 0 pgmap v1550: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 17224/104154 objects degraded (16.537%); 1043/104154 objects misplaced (1.001%) 2023-10-09T09:33:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:42 smithi163 bash[78107]: audit 2023-10-09T09:33:41.802137+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.163:0/3852231826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:33:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:42 smithi163 bash[78107]: audit 2023-10-09T09:33:41.802528+0000 mon.a (mon.0) 1664 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:33:43.044 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:43.078 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:43.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:43.182 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:43.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:43.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:43.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:43.323 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:43.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:43.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:43.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:43.382 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T09:33:43.451 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:43.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:43.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:43.512 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T09:33:43.513 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T09:33:43.513 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:33:43] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.33.4" 2023-10-09T09:33:43.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:43.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:43.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:43.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:43.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:43 smithi181 bash[59345]: cluster 2023-10-09T09:33:42.709591+0000 mon.a (mon.0) 1665 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:33:43.713 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:43 smithi181 bash[59345]: cluster 2023-10-09T09:33:42.722298+0000 osd.4 (osd.4) 550 : cluster 3 failed to encode map e1124 with expected crc 2023-10-09T09:33:43.713 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:43 smithi181 bash[59345]: audit 2023-10-09T09:33:42.722826+0000 mon.a (mon.0) 1666 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:33:43.713 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:43 smithi181 bash[59345]: cluster 2023-10-09T09:33:42.722963+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e1124: 8 total, 7 up, 8 in 2023-10-09T09:33:43.713 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:43 smithi181 bash[59345]: cluster 2023-10-09T09:33:42.745884+0000 osd.7 (osd.7) 528 : cluster 3 failed to encode map e1124 with expected crc 2023-10-09T09:33:43.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:43.747 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:43.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:43.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:43.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:43.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[75425]: cluster 2023-10-09T09:33:42.709591+0000 mon.a (mon.0) 1665 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:33:43.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[75425]: cluster 2023-10-09T09:33:42.722298+0000 osd.4 (osd.4) 550 : cluster 3 failed to encode map e1124 with expected crc 2023-10-09T09:33:43.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[75425]: audit 2023-10-09T09:33:42.722826+0000 mon.a (mon.0) 1666 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:33:43.860 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[75425]: cluster 2023-10-09T09:33:42.722963+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e1124: 8 total, 7 up, 8 in 2023-10-09T09:33:43.860 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[75425]: cluster 2023-10-09T09:33:42.745884+0000 osd.7 (osd.7) 528 : cluster 3 failed to encode map e1124 with expected crc 2023-10-09T09:33:43.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[78107]: cluster 2023-10-09T09:33:42.709591+0000 mon.a (mon.0) 1665 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:33:43.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[78107]: cluster 2023-10-09T09:33:42.722298+0000 osd.4 (osd.4) 550 : cluster 3 failed to encode map e1124 with expected crc 2023-10-09T09:33:43.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[78107]: audit 2023-10-09T09:33:42.722826+0000 mon.a (mon.0) 1666 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:33:43.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[78107]: cluster 2023-10-09T09:33:42.722963+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e1124: 8 total, 7 up, 8 in 2023-10-09T09:33:43.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:43 smithi163 bash[78107]: cluster 2023-10-09T09:33:42.745884+0000 osd.7 (osd.7) 528 : cluster 3 failed to encode map e1124 with expected crc 2023-10-09T09:33:43.864 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:33:43.864 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:33:43.864 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T09:33:43.864 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.220078 2023-10-09T09:33:43.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 0.22007751407470533 2023-10-09T09:33:43.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:43.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:43.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:43.984 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:44.012 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:44.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:44.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:44.120 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:44.153 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:44.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:44.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:44.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:44.287 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:44.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:44.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:44.390 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:44.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:44.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:44.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:44.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:44.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:44.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:44.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:44.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:44.685 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:44.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:44.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:44.788 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:44.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:44.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:44.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:44.922 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:44.930 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:44 smithi163 bash[75425]: cluster 2023-10-09T09:33:42.719520+0000 osd.5 (osd.5) 677 : cluster 3 failed to encode map e1124 with expected crc 2023-10-09T09:33:44.930 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:44 smithi163 bash[75425]: cluster 2023-10-09T09:33:43.776452+0000 mgr.y (mgr.34107) 826 : cluster 0 pgmap v1552: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 16914/102258 objects degraded (16.541%); 1043/102258 objects misplaced (1.020%) 2023-10-09T09:33:44.931 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:44 smithi163 bash[78107]: cluster 2023-10-09T09:33:42.719520+0000 osd.5 (osd.5) 677 : cluster 3 failed to encode map e1124 with expected crc 2023-10-09T09:33:44.931 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:44 smithi163 bash[78107]: cluster 2023-10-09T09:33:43.776452+0000 mgr.y (mgr.34107) 826 : cluster 0 pgmap v1552: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 16914/102258 objects degraded (16.541%); 1043/102258 objects misplaced (1.020%) 2023-10-09T09:33:44.959 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:44.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:44.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:45.064 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:45.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:44 smithi181 bash[59345]: cluster 2023-10-09T09:33:42.719520+0000 osd.5 (osd.5) 677 : cluster 3 failed to encode map e1124 with expected crc 2023-10-09T09:33:45.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:44 smithi181 bash[59345]: cluster 2023-10-09T09:33:43.776452+0000 mgr.y (mgr.34107) 826 : cluster 0 pgmap v1552: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 16914/102258 objects degraded (16.541%); 1043/102258 objects misplaced (1.020%) 2023-10-09T09:33:45.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:45.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:45.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:45.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:45.233 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:45.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:45.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:45.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:45.381 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:45.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:45.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:45.487 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:45.511 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:45.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:33:45.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:45.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:45.616 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:45.644 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:45.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:45.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:45.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:45.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:45.778 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:45 smithi163 bash[78107]: cluster 2023-10-09T09:33:45.600191+0000 mon.a (mon.0) 1668 : cluster 3 Health check update: Degraded data redundancy: 16914/102258 objects degraded (16.541%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:45.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:45.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:45.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:45 smithi163 bash[75425]: cluster 2023-10-09T09:33:45.600191+0000 mon.a (mon.0) 1668 : cluster 3 Health check update: Degraded data redundancy: 16914/102258 objects degraded (16.541%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:45.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:45.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:45.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:45.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:46.010 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:46.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:45 smithi181 bash[59345]: cluster 2023-10-09T09:33:45.600191+0000 mon.a (mon.0) 1668 : cluster 3 Health check update: Degraded data redundancy: 16914/102258 objects degraded (16.541%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:46.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:46.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:46.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:46.139 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:46.146 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:33:46.165 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:46.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:46.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:46.271 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:46.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:46.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:46.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:46.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:46.444 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:46.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:46.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:46.547 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:46.583 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:46.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:46.690 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:46.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:46.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:46.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:46.828 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:46.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:46.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:46.962 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:46.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:46 smithi181 bash[59345]: cluster 2023-10-09T09:33:45.779553+0000 mgr.y (mgr.34107) 827 : cluster 0 pgmap v1553: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 16295/96465 objects degraded (16.892%); 1043/96465 objects misplaced (1.081%) 2023-10-09T09:33:46.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:46.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:46.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:47.098 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:47.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:46 smithi163 bash[78107]: cluster 2023-10-09T09:33:45.779553+0000 mgr.y (mgr.34107) 827 : cluster 0 pgmap v1553: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 16295/96465 objects degraded (16.892%); 1043/96465 objects misplaced (1.081%) 2023-10-09T09:33:47.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:46 smithi163 bash[75425]: cluster 2023-10-09T09:33:45.779553+0000 mgr.y (mgr.34107) 827 : cluster 0 pgmap v1553: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 16295/96465 objects degraded (16.892%); 1043/96465 objects misplaced (1.081%) 2023-10-09T09:33:47.133 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:47.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:47.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:47.237 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:47.277 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:47.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:47.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:47.380 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:47.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:47.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:47.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:47.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:47.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:47.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:47.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:47.692 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:47.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:47.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:47.797 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:47.825 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:47.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:47.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:47.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:47.967 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:47.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:47 smithi163 bash[75425]: cluster 2023-10-09T09:33:47.780494+0000 mgr.y (mgr.34107) 828 : cluster 0 pgmap v1554: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 14736/88986 objects degraded (16.560%); 903/88986 objects misplaced (1.015%) 2023-10-09T09:33:47.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:47 smithi163 bash[78107]: cluster 2023-10-09T09:33:47.780494+0000 mgr.y (mgr.34107) 828 : cluster 0 pgmap v1554: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 14736/88986 objects degraded (16.560%); 903/88986 objects misplaced (1.015%) 2023-10-09T09:33:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:47.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:48.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:47 smithi181 bash[59345]: cluster 2023-10-09T09:33:47.780494+0000 mgr.y (mgr.34107) 828 : cluster 0 pgmap v1554: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 14736/88986 objects degraded (16.560%); 903/88986 objects misplaced (1.015%) 2023-10-09T09:33:48.071 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:48.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:48.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:48.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:48.226 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:33:48.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:48.232 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:33:48.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:48.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:48.338 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:48.382 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:48.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:48.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:48.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:49.130 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:49.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:49.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:48 smithi163 bash[78107]: audit 2023-10-09T09:33:48.554332+0000 mon.a (mon.0) 1669 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:33:49.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:49.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:48 smithi163 bash[75425]: audit 2023-10-09T09:33:48.554332+0000 mon.a (mon.0) 1669 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:33:49.235 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:49.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:48 smithi181 bash[59345]: audit 2023-10-09T09:33:48.554332+0000 mon.a (mon.0) 1669 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:33:49.261 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:49.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:49.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:49.397 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:49.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:49.501 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:49.528 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:49.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:49.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:49.632 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:49.655 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:49.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:49.759 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:49.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:49.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:49.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:49.915 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:49.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:49.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:49 smithi163 bash[78107]: cluster 2023-10-09T09:33:49.781228+0000 mgr.y (mgr.34107) 829 : cluster 0 pgmap v1555: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 14086/83181 objects degraded (16.934%); 903/83181 objects misplaced (1.086%) 2023-10-09T09:33:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:49 smithi163 bash[75425]: cluster 2023-10-09T09:33:49.781228+0000 mgr.y (mgr.34107) 829 : cluster 0 pgmap v1555: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 14086/83181 objects degraded (16.934%); 903/83181 objects misplaced (1.086%) 2023-10-09T09:33:50.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:50.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:50.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:50.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:50.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:49 smithi181 bash[59345]: cluster 2023-10-09T09:33:49.781228+0000 mgr.y (mgr.34107) 829 : cluster 0 pgmap v1555: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 14086/83181 objects degraded (16.934%); 903/83181 objects misplaced (1.086%) 2023-10-09T09:33:50.194 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:50.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:50.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:50.268 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:33:50.268 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:33:50.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:50.335 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:50.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:50.440 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:50.465 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:50.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:50.571 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:50.603 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:50.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:50.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:50.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:50.734 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:50.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:50.839 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:50.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:50.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:50.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:50.969 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:50.978 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:50 smithi181 bash[59345]: cluster 2023-10-09T09:33:50.601819+0000 mon.a (mon.0) 1670 : cluster 3 Health check update: Degraded data redundancy: 14086/83181 objects degraded (16.934%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:50.994 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:50.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:51.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:51.100 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:51.109 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:51 smithi163 bash[78107]: cluster 2023-10-09T09:33:50.601819+0000 mon.a (mon.0) 1670 : cluster 3 Health check update: Degraded data redundancy: 14086/83181 objects degraded (16.934%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:51.109 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:51 smithi163 bash[75425]: cluster 2023-10-09T09:33:50.601819+0000 mon.a (mon.0) 1670 : cluster 3 Health check update: Degraded data redundancy: 14086/83181 objects degraded (16.934%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:51.139 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:51.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:51.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:51.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:51.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:51.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:51.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:51.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:51.411 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:51.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:51.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:51.547 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:51.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:51.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:51.650 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:51.676 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:51.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:51.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:51.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:51.812 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:51.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:51.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:51.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:51.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:51 smithi163 bash[75425]: cluster 2023-10-09T09:33:51.782007+0000 mgr.y (mgr.34107) 830 : cluster 0 pgmap v1556: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 12179/73704 objects degraded (16.524%); 747/73704 objects misplaced (1.014%) 2023-10-09T09:33:51.944 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:51.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:51.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:52.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:52.077 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:52.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:52.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:52.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:52.188 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:51 smithi181 bash[59345]: cluster 2023-10-09T09:33:51.782007+0000 mgr.y (mgr.34107) 830 : cluster 0 pgmap v1556: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 12179/73704 objects degraded (16.524%); 747/73704 objects misplaced (1.014%) 2023-10-09T09:33:52.208 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:52.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:52.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:52.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:51 smithi163 bash[78107]: cluster 2023-10-09T09:33:51.782007+0000 mgr.y (mgr.34107) 830 : cluster 0 pgmap v1556: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 12179/73704 objects degraded (16.524%); 747/73704 objects misplaced (1.014%) 2023-10-09T09:33:52.313 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:52.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:52.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:52.440 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:52.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:52.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:52.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:52.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:52.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:52.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:52.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:52.725 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:52.759 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:52.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:52.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:52.863 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:52.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:52.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:52.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:52.995 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:53.024 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:53.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:53.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:53.129 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:53.149 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:53.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:53.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:53.286 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:53.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:53.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:53.360 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:33:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:33:53] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.33.4" 2023-10-09T09:33:53.392 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:53.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:53.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:53.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:53.538 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:53.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:53.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:53.674 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:53.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:53.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:53.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:53.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:53.849 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:53.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:53.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:53.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:53.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:53 smithi163 bash[75425]: cluster 2023-10-09T09:33:53.782642+0000 mgr.y (mgr.34107) 831 : cluster 0 pgmap v1557: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 12179/73704 objects degraded (16.524%); 747/73704 objects misplaced (1.014%) 2023-10-09T09:33:53.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:53 smithi163 bash[78107]: cluster 2023-10-09T09:33:53.782642+0000 mgr.y (mgr.34107) 831 : cluster 0 pgmap v1557: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 12179/73704 objects degraded (16.524%); 747/73704 objects misplaced (1.014%) 2023-10-09T09:33:53.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:53.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:54.089 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:54.140 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:54.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:54.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:54.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:54.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:53 smithi181 bash[59345]: cluster 2023-10-09T09:33:53.782642+0000 mgr.y (mgr.34107) 831 : cluster 0 pgmap v1557: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 12179/73704 objects degraded (16.524%); 747/73704 objects misplaced (1.014%) 2023-10-09T09:33:54.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:54.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:54.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:54.378 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:54.441 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:54.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:54.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:54.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:54.577 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:54.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:54.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:54.682 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:54.715 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:54.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:54.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:54.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:54.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:54.954 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:54.985 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:54.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:54.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:55.089 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:55.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:55.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:55.224 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:55.277 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:55.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:55.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:55.415 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:55.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:55.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:55.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:55.541 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:55.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:55.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:55.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:55.676 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:55.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:55.780 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:55.806 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:55.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:55.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:55.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:55.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:55.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:56.059 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:55 smithi181 bash[59345]: audit 2023-10-09T09:33:54.802115+0000 mon.a (mon.0) 1671 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:55 smithi181 bash[59345]: audit 2023-10-09T09:33:54.814128+0000 mon.a (mon.0) 1672 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:55 smithi181 bash[59345]: audit 2023-10-09T09:33:55.013821+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:55 smithi181 bash[59345]: audit 2023-10-09T09:33:55.026815+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:55 smithi181 bash[59345]: audit 2023-10-09T09:33:55.376293+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:55 smithi181 bash[59345]: audit 2023-10-09T09:33:55.377629+0000 mon.a (mon.0) 1676 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:55 smithi181 bash[59345]: audit 2023-10-09T09:33:55.558653+0000 mon.a (mon.0) 1677 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:33:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:55 smithi181 bash[59345]: audit 2023-10-09T09:33:55.559438+0000 mon.a (mon.0) 1678 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:33:56.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:55 smithi181 bash[59345]: audit 2023-10-09T09:33:55.566103+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:55 smithi181 bash[59345]: cluster 2023-10-09T09:33:55.603248+0000 mon.a (mon.0) 1680 : cluster 3 Health check update: Degraded data redundancy: 12179/73704 objects degraded (16.524%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:56.068 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[78107]: audit 2023-10-09T09:33:54.802115+0000 mon.a (mon.0) 1671 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.068 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[78107]: audit 2023-10-09T09:33:54.814128+0000 mon.a (mon.0) 1672 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.068 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[78107]: audit 2023-10-09T09:33:55.013821+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.068 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[78107]: audit 2023-10-09T09:33:55.026815+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.068 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[78107]: audit 2023-10-09T09:33:55.376293+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.068 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[78107]: audit 2023-10-09T09:33:55.377629+0000 mon.a (mon.0) 1676 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:56.068 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[78107]: audit 2023-10-09T09:33:55.558653+0000 mon.a (mon.0) 1677 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:33:56.068 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[78107]: audit 2023-10-09T09:33:55.559438+0000 mon.a (mon.0) 1678 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:33:56.068 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[78107]: audit 2023-10-09T09:33:55.566103+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[78107]: cluster 2023-10-09T09:33:55.603248+0000 mon.a (mon.0) 1680 : cluster 3 Health check update: Degraded data redundancy: 12179/73704 objects degraded (16.524%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:56.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[75425]: audit 2023-10-09T09:33:54.802115+0000 mon.a (mon.0) 1671 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[75425]: audit 2023-10-09T09:33:54.814128+0000 mon.a (mon.0) 1672 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[75425]: audit 2023-10-09T09:33:55.013821+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[75425]: audit 2023-10-09T09:33:55.026815+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[75425]: audit 2023-10-09T09:33:55.376293+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[75425]: audit 2023-10-09T09:33:55.377629+0000 mon.a (mon.0) 1676 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:33:56.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[75425]: audit 2023-10-09T09:33:55.558653+0000 mon.a (mon.0) 1677 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:33:56.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[75425]: audit 2023-10-09T09:33:55.559438+0000 mon.a (mon.0) 1678 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:33:56.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[75425]: audit 2023-10-09T09:33:55.566103+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:33:56.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:55 smithi163 bash[75425]: cluster 2023-10-09T09:33:55.603248+0000 mon.a (mon.0) 1680 : cluster 3 Health check update: Degraded data redundancy: 12179/73704 objects degraded (16.524%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:33:56.084 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:56.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:56.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:56.186 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:56.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:56.214 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:56.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:56.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:56.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:56.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:56.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:56.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:56.473 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:56.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:56.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:56.578 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:56.606 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:56.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:56.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:56.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:56.736 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:56.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:56.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:56.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:56.871 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:56.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:56.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:56.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:56 smithi163 bash[75425]: cluster 2023-10-09T09:33:55.783393+0000 mgr.y (mgr.34107) 832 : cluster 0 pgmap v1558: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.4 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 11315/66804 objects degraded (16.938%); 747/66804 objects misplaced (1.118%) 2023-10-09T09:33:56.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:56 smithi163 bash[78107]: cluster 2023-10-09T09:33:55.783393+0000 mgr.y (mgr.34107) 832 : cluster 0 pgmap v1558: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.4 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 11315/66804 objects degraded (16.938%); 747/66804 objects misplaced (1.118%) 2023-10-09T09:33:56.979 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:57.011 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:33:57.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:57.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:33:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:56 smithi181 bash[59345]: cluster 2023-10-09T09:33:55.783393+0000 mgr.y (mgr.34107) 832 : cluster 0 pgmap v1558: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.4 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 11315/66804 objects degraded (16.938%); 747/66804 objects misplaced (1.118%) 2023-10-09T09:33:57.114 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:57.146 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:57.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:57.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:57.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:57.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:57.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:57.413 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:57.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:57.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:57.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:57.576 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:57.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:57.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:57.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:57.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:57.811 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:57.850 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:57.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:57.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:57.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:57.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:57.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:57.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:58.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:58.135 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:58.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:58.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:58.244 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:58.279 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:58.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:58.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:58.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:58.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:33:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:58.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:33:58.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:58.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:58.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:58.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:58.684 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:33:58.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:58.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:33:58.788 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:58.824 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:33:58.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:58.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:33:58.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:58.935 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:33:58 smithi181 bash[59345]: cluster 2023-10-09T09:33:57.784170+0000 mgr.y (mgr.34107) 833 : cluster 0 pgmap v1559: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 9967/60297 objects degraded (16.530%); 601/60297 objects misplaced (0.997%) 2023-10-09T09:33:58.954 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:58.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:58.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:59.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:59.093 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:59.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:59.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:59.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:59.226 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:59.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:59.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:59.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:59.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:59.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:59.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:59.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:59.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:33:59 smithi163 bash[75425]: cluster 2023-10-09T09:33:57.784170+0000 mgr.y (mgr.34107) 833 : cluster 0 pgmap v1559: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 9967/60297 objects degraded (16.530%); 601/60297 objects misplaced (0.997%) 2023-10-09T09:33:59.477 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:33:59 smithi163 bash[78107]: cluster 2023-10-09T09:33:57.784170+0000 mgr.y (mgr.34107) 833 : cluster 0 pgmap v1559: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 9967/60297 objects degraded (16.530%); 601/60297 objects misplaced (0.997%) 2023-10-09T09:33:59.501 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:33:59.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:59.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:33:59.607 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:59.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:33:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:59.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:33:59.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:59.778 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:59.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:59.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:33:59.881 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:59.916 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:33:59.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:33:59.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:00.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:00.049 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:00.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:00.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:00.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:00.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:00.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:00.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:00.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:00.318 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:00.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:00.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:00.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:00.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:00.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:00.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:00.591 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:00.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:00.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:00.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:00.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:00.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:00.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:00.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:00 smithi181 bash[59345]: cluster 2023-10-09T09:33:59.785079+0000 mgr.y (mgr.34107) 834 : cluster 0 pgmap v1560: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 9687/58650 objects degraded (16.517%); 601/58650 objects misplaced (1.025%) 2023-10-09T09:34:00.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:00 smithi181 bash[59345]: cluster 2023-10-09T09:34:00.604153+0000 mon.a (mon.0) 1681 : cluster 3 Health check update: Degraded data redundancy: 9687/58650 objects degraded (16.517%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:00.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:00.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:00 smithi163 bash[75425]: cluster 2023-10-09T09:33:59.785079+0000 mgr.y (mgr.34107) 834 : cluster 0 pgmap v1560: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 9687/58650 objects degraded (16.517%); 601/58650 objects misplaced (1.025%) 2023-10-09T09:34:00.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:00 smithi163 bash[75425]: cluster 2023-10-09T09:34:00.604153+0000 mon.a (mon.0) 1681 : cluster 3 Health check update: Degraded data redundancy: 9687/58650 objects degraded (16.517%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:00.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:00 smithi163 bash[78107]: cluster 2023-10-09T09:33:59.785079+0000 mgr.y (mgr.34107) 834 : cluster 0 pgmap v1560: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 9687/58650 objects degraded (16.517%); 601/58650 objects misplaced (1.025%) 2023-10-09T09:34:00.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:00 smithi163 bash[78107]: cluster 2023-10-09T09:34:00.604153+0000 mon.a (mon.0) 1681 : cluster 3 Health check update: Degraded data redundancy: 9687/58650 objects degraded (16.517%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:00.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:00.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:00.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:00.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:00.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:00.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:01.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:01.131 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:01.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:01.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:01.236 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:01.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:01.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:01.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:01.366 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:01.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:01.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:01.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:01.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:01.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:01.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:01.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:01.639 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:01.669 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:01.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:01.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:01.773 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:01.803 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:01.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:01.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:01.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:01.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:01.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:01.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:02.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:02.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:02.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:02.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:02.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:02.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:02.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:02.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:02.299 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:02.322 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:02.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:02.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:02.436 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:34:02.437 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:34:02.437 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T09:34:02.437 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T09:34:02.437 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:34:02.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:02.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:02.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:02.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:02.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:02.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:02.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:02.712 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:02.745 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:02.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:02.763 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:34:02.831 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.3 primary-affinity to 0.220078 (8144222) 2023-10-09T09:34:02.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:02.855 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:34:02.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:02.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:02.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:02.892 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:34:02.924 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:34:02.952 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:34:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:02 smithi163 bash[75425]: cluster 2023-10-09T09:34:01.786015+0000 mgr.y (mgr.34107) 835 : cluster 0 pgmap v1561: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 957 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 7605/46002 objects degraded (16.532%); 452/46002 objects misplaced (0.983%) 2023-10-09T09:34:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:02 smithi163 bash[75425]: audit 2023-10-09T09:34:02.642446+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.163:0/3022309156' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.22007751407470533}]: dispatch 2023-10-09T09:34:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:02 smithi163 bash[75425]: audit 2023-10-09T09:34:02.643673+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.22007751407470533}]: dispatch 2023-10-09T09:34:02.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:02 smithi163 bash[75425]: audit 2023-10-09T09:34:02.700571+0000 mon.a (mon.0) 1683 : audit 0 from='client.? 172.21.15.163:0/541009427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:02 smithi163 bash[78107]: cluster 2023-10-09T09:34:01.786015+0000 mgr.y (mgr.34107) 835 : cluster 0 pgmap v1561: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 957 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 7605/46002 objects degraded (16.532%); 452/46002 objects misplaced (0.983%) 2023-10-09T09:34:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:02 smithi163 bash[78107]: audit 2023-10-09T09:34:02.642446+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.163:0/3022309156' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.22007751407470533}]: dispatch 2023-10-09T09:34:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:02 smithi163 bash[78107]: audit 2023-10-09T09:34:02.643673+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.22007751407470533}]: dispatch 2023-10-09T09:34:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:02 smithi163 bash[78107]: audit 2023-10-09T09:34:02.700571+0000 mon.a (mon.0) 1683 : audit 0 from='client.? 172.21.15.163:0/541009427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:02.973 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:34:02.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:02.998 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:34:02.998 INFO:teuthology.orchestra.run.smithi163.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T09:34:02.999 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:34:02.999 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:34:02.999 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:34:02.999 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:34:02.999 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:34:02.999 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:34:02.999 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:34:03.000 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:03.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:03.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:02 smithi181 bash[59345]: cluster 2023-10-09T09:34:01.786015+0000 mgr.y (mgr.34107) 835 : cluster 0 pgmap v1561: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 957 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 7605/46002 objects degraded (16.532%); 452/46002 objects misplaced (0.983%) 2023-10-09T09:34:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:02 smithi181 bash[59345]: audit 2023-10-09T09:34:02.642446+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.163:0/3022309156' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.22007751407470533}]: dispatch 2023-10-09T09:34:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:02 smithi181 bash[59345]: audit 2023-10-09T09:34:02.643673+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.22007751407470533}]: dispatch 2023-10-09T09:34:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:02 smithi181 bash[59345]: audit 2023-10-09T09:34:02.700571+0000 mon.a (mon.0) 1683 : audit 0 from='client.? 172.21.15.163:0/541009427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:03.104 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:03.134 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:03.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:03.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:03.238 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:03.265 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:03.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:03.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:03.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:03.404 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:03.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:03.508 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:03.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:03.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:03.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:03.645 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:03.657 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:34:03] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.33.4" 2023-10-09T09:34:03.671 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:03.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:03.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:03.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:03.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:03.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:03.853 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:34:03.908 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:03.914 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T09:34:03.914 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[78107]: cluster 2023-10-09T09:34:02.815516+0000 osd.5 (osd.5) 678 : cluster 3 failed to encode map e1125 with expected crc 2023-10-09T09:34:03.914 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[78107]: audit 2023-10-09T09:34:02.818745+0000 mon.a (mon.0) 1684 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.22007751407470533}]': finished 2023-10-09T09:34:03.915 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[78107]: cluster 2023-10-09T09:34:02.818854+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e1125: 8 total, 7 up, 8 in 2023-10-09T09:34:03.915 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[78107]: cluster 2023-10-09T09:34:02.827884+0000 osd.4 (osd.4) 551 : cluster 3 failed to encode map e1125 with expected crc 2023-10-09T09:34:03.915 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[78107]: cluster 2023-10-09T09:34:02.835504+0000 osd.4 (osd.4) 552 : cluster 3 failed to encode map e1125 with expected crc 2023-10-09T09:34:03.915 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[78107]: audit 2023-10-09T09:34:02.985684+0000 mon.a (mon.0) 1686 : audit 1 from='client.? 172.21.15.163:0/1268316797' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:34:03.915 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.786697+0000 mgr.y (mgr.34107) 836 : cluster 0 pgmap v1563: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 957 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 7605/46002 objects degraded (16.532%); 452/46002 objects misplaced (0.983%) 2023-10-09T09:34:03.915 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[75425]: cluster 2023-10-09T09:34:02.815516+0000 osd.5 (osd.5) 678 : cluster 3 failed to encode map e1125 with expected crc 2023-10-09T09:34:03.915 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[75425]: audit 2023-10-09T09:34:02.818745+0000 mon.a (mon.0) 1684 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.22007751407470533}]': finished 2023-10-09T09:34:03.915 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[75425]: cluster 2023-10-09T09:34:02.818854+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e1125: 8 total, 7 up, 8 in 2023-10-09T09:34:03.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[75425]: cluster 2023-10-09T09:34:02.827884+0000 osd.4 (osd.4) 551 : cluster 3 failed to encode map e1125 with expected crc 2023-10-09T09:34:03.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[75425]: cluster 2023-10-09T09:34:02.835504+0000 osd.4 (osd.4) 552 : cluster 3 failed to encode map e1125 with expected crc 2023-10-09T09:34:03.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[75425]: audit 2023-10-09T09:34:02.985684+0000 mon.a (mon.0) 1686 : audit 1 from='client.? 172.21.15.163:0/1268316797' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:34:03.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:03 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.786697+0000 mgr.y (mgr.34107) 836 : cluster 0 pgmap v1563: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 957 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 7605/46002 objects degraded (16.532%); 452/46002 objects misplaced (0.983%) 2023-10-09T09:34:03.932 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:03.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:03.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:04.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:03 smithi181 bash[59345]: cluster 2023-10-09T09:34:02.815516+0000 osd.5 (osd.5) 678 : cluster 3 failed to encode map e1125 with expected crc 2023-10-09T09:34:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:03 smithi181 bash[59345]: audit 2023-10-09T09:34:02.818745+0000 mon.a (mon.0) 1684 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.22007751407470533}]': finished 2023-10-09T09:34:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:03 smithi181 bash[59345]: cluster 2023-10-09T09:34:02.818854+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e1125: 8 total, 7 up, 8 in 2023-10-09T09:34:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:03 smithi181 bash[59345]: cluster 2023-10-09T09:34:02.827884+0000 osd.4 (osd.4) 551 : cluster 3 failed to encode map e1125 with expected crc 2023-10-09T09:34:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:03 smithi181 bash[59345]: cluster 2023-10-09T09:34:02.835504+0000 osd.4 (osd.4) 552 : cluster 3 failed to encode map e1125 with expected crc 2023-10-09T09:34:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:03 smithi181 bash[59345]: audit 2023-10-09T09:34:02.985684+0000 mon.a (mon.0) 1686 : audit 1 from='client.? 172.21.15.163:0/1268316797' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:34:04.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:03 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.786697+0000 mgr.y (mgr.34107) 836 : cluster 0 pgmap v1563: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 957 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 7605/46002 objects degraded (16.532%); 452/46002 objects misplaced (0.983%) 2023-10-09T09:34:04.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:04.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:04.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:04.174 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:04.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:04.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:04.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:04.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:04.335 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:04.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:04.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:04.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:04.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:04.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:04.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:04.597 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:04.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:04.700 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:04.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:04.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:04.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:04.855 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:04.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:04.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:04.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:04.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.824106+0000 mon.a (mon.0) 1687 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:34:04.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.832043+0000 osd.5 (osd.5) 679 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:04.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.836015+0000 osd.5 (osd.5) 680 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:04.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: audit 2023-10-09T09:34:03.836257+0000 mon.a (mon.0) 1688 : audit 1 from='client.? 172.21.15.163:0/1268316797' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:34:04.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.836315+0000 mon.a (mon.0) 1689 : cluster 0 osdmap e1126: 8 total, 7 up, 8 in 2023-10-09T09:34:04.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.837396+0000 osd.6 (osd.6) 409 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:04.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.837432+0000 osd.7 (osd.7) 529 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:04.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.838023+0000 osd.4 (osd.4) 553 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:04.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.840245+0000 osd.6 (osd.6) 410 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:04.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.841261+0000 osd.7 (osd.7) 530 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:04.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.841875+0000 osd.4 (osd.4) 554 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:04.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:04 smithi181 bash[59345]: cluster 2023-10-09T09:34:03.845103+0000 osd.4 (osd.4) 555 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:04.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:04.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:04.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:05.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:05.106 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.824106+0000 mon.a (mon.0) 1687 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:34:05.106 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.832043+0000 osd.5 (osd.5) 679 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.836015+0000 osd.5 (osd.5) 680 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: audit 2023-10-09T09:34:03.836257+0000 mon.a (mon.0) 1688 : audit 1 from='client.? 172.21.15.163:0/1268316797' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:34:05.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.836315+0000 mon.a (mon.0) 1689 : cluster 0 osdmap e1126: 8 total, 7 up, 8 in 2023-10-09T09:34:05.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.837396+0000 osd.6 (osd.6) 409 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.837432+0000 osd.7 (osd.7) 529 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.838023+0000 osd.4 (osd.4) 553 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.840245+0000 osd.6 (osd.6) 410 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.841261+0000 osd.7 (osd.7) 530 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.841875+0000 osd.4 (osd.4) 554 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[78107]: cluster 2023-10-09T09:34:03.845103+0000 osd.4 (osd.4) 555 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.824106+0000 mon.a (mon.0) 1687 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:34:05.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.832043+0000 osd.5 (osd.5) 679 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.836015+0000 osd.5 (osd.5) 680 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: audit 2023-10-09T09:34:03.836257+0000 mon.a (mon.0) 1688 : audit 1 from='client.? 172.21.15.163:0/1268316797' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:34:05.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.836315+0000 mon.a (mon.0) 1689 : cluster 0 osdmap e1126: 8 total, 7 up, 8 in 2023-10-09T09:34:05.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.837396+0000 osd.6 (osd.6) 409 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.837432+0000 osd.7 (osd.7) 529 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.109 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.838023+0000 osd.4 (osd.4) 553 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.109 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.840245+0000 osd.6 (osd.6) 410 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.109 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.841261+0000 osd.7 (osd.7) 530 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.109 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.841875+0000 osd.4 (osd.4) 554 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.109 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:04 smithi163 bash[75425]: cluster 2023-10-09T09:34:03.845103+0000 osd.4 (osd.4) 555 : cluster 3 failed to encode map e1126 with expected crc 2023-10-09T09:34:05.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:05.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:05.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:05.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:05.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:05.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:05.368 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:05.400 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:05.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:05.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:05.509 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:05.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:05.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:05.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:05.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:05.669 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:05.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:05.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:05.774 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:05.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:05.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:05.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:05.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:05.959 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:05.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:06.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:05 smithi163 bash[75425]: cluster 2023-10-09T09:34:05.605468+0000 mon.a (mon.0) 1690 : cluster 3 Health check update: Degraded data redundancy: 7605/46002 objects degraded (16.532%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:06.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:05 smithi163 bash[75425]: cluster 2023-10-09T09:34:05.787324+0000 mgr.y (mgr.34107) 837 : cluster 0 pgmap v1565: 99 pgs: 1 active+clean+remapped, 3 peering, 20 active+undersized+degraded, 18 active+undersized, 57 active+clean; 832 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 6844/39963 objects degraded (17.126%); 452/39963 objects misplaced (1.131%) 2023-10-09T09:34:06.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:05 smithi163 bash[78107]: cluster 2023-10-09T09:34:05.605468+0000 mon.a (mon.0) 1690 : cluster 3 Health check update: Degraded data redundancy: 7605/46002 objects degraded (16.532%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:06.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:05 smithi163 bash[78107]: cluster 2023-10-09T09:34:05.787324+0000 mgr.y (mgr.34107) 837 : cluster 0 pgmap v1565: 99 pgs: 1 active+clean+remapped, 3 peering, 20 active+undersized+degraded, 18 active+undersized, 57 active+clean; 832 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 6844/39963 objects degraded (17.126%); 452/39963 objects misplaced (1.131%) 2023-10-09T09:34:06.064 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:06.096 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:06.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:06.173 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T09:34:06.174 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T09:34:06.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:06.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:05 smithi181 bash[59345]: cluster 2023-10-09T09:34:05.605468+0000 mon.a (mon.0) 1690 : cluster 3 Health check update: Degraded data redundancy: 7605/46002 objects degraded (16.532%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:06.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:05 smithi181 bash[59345]: cluster 2023-10-09T09:34:05.787324+0000 mgr.y (mgr.34107) 837 : cluster 0 pgmap v1565: 99 pgs: 1 active+clean+remapped, 3 peering, 20 active+undersized+degraded, 18 active+undersized, 57 active+clean; 832 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 6844/39963 objects degraded (17.126%); 452/39963 objects misplaced (1.131%) 2023-10-09T09:34:06.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:06.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:06.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:06.344 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:06.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:06.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:06.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:06.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:06.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:06.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:06.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:06.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:06.647 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:06.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:06.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:06.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:06.783 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:06.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:06.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:06.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:06.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:06.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:06.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:07.029 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:07.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:06 smithi181 bash[59345]: cluster 2023-10-09T09:34:06.608475+0000 mon.a (mon.0) 1691 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:34:07.055 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:07.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:07.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:07.160 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:07.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:06 smithi163 bash[75425]: cluster 2023-10-09T09:34:06.608475+0000 mon.a (mon.0) 1691 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:34:07.170 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:06 smithi163 bash[78107]: cluster 2023-10-09T09:34:06.608475+0000 mon.a (mon.0) 1691 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:34:07.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:07.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:07.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:07.302 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:07.338 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:07.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:07.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:07.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:07.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:07.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:07.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:07.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:07.599 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:07.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:07.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:07.707 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:07.735 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:07.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:07.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:07.839 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:07.865 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:07.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:07.968 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:07.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:07 smithi181 bash[59345]: cluster 2023-10-09T09:34:07.788230+0000 mgr.y (mgr.34107) 838 : cluster 0 pgmap v1566: 99 pgs: 1 active+clean+remapped, 3 peering, 20 active+undersized+degraded, 18 active+undersized, 57 active+clean; 717 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5659/34452 objects degraded (16.426%); 336/34452 objects misplaced (0.975%) 2023-10-09T09:34:07.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:07.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:07.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:08.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:08.135 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:08.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:08.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:08.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:34:08.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:07 smithi163 bash[75425]: cluster 2023-10-09T09:34:07.788230+0000 mgr.y (mgr.34107) 838 : cluster 0 pgmap v1566: 99 pgs: 1 active+clean+remapped, 3 peering, 20 active+undersized+degraded, 18 active+undersized, 57 active+clean; 717 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5659/34452 objects degraded (16.426%); 336/34452 objects misplaced (0.975%) 2023-10-09T09:34:08.184 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:07 smithi163 bash[78107]: cluster 2023-10-09T09:34:07.788230+0000 mgr.y (mgr.34107) 838 : cluster 0 pgmap v1566: 99 pgs: 1 active+clean+remapped, 3 peering, 20 active+undersized+degraded, 18 active+undersized, 57 active+clean; 717 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5659/34452 objects degraded (16.426%); 336/34452 objects misplaced (0.975%) 2023-10-09T09:34:08.190 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:34:08.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:08.295 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:08.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:08.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:08.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:08.441 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:08.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:08.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:08.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:08.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:08.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:08.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:08.704 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:08.727 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:08.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:08.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:08.772 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:34:08.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:08.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:08.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:08.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:08.979 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:09.002 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:09.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:09.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:09.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:09.133 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:09.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:09.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:09.238 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:09.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:09.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:09.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:09.366 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:09.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:09.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:09.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:09.500 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:09.535 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:09.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:09.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:09.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:09.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:09.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:09.784 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:09.809 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:09.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:09.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:09.911 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:09.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:09 smithi181 bash[59345]: cluster 2023-10-09T09:34:09.788970+0000 mgr.y (mgr.34107) 839 : cluster 0 pgmap v1567: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 675 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 5410/32460 objects degraded (16.667%); 336/32460 objects misplaced (1.035%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:09.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:09 smithi181 bash[59345]: cluster 2023-10-09T09:34:09.840220+0000 mon.a (mon.0) 1692 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:34:09.945 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:09.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:10.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:10.074 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:10.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:10.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:10.178 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:10.206 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:10.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:10.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:10.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:09 smithi163 bash[75425]: cluster 2023-10-09T09:34:09.788970+0000 mgr.y (mgr.34107) 839 : cluster 0 pgmap v1567: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 675 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 5410/32460 objects degraded (16.667%); 336/32460 objects misplaced (1.035%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:10.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:09 smithi163 bash[75425]: cluster 2023-10-09T09:34:09.840220+0000 mon.a (mon.0) 1692 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:34:10.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:09 smithi163 bash[78107]: cluster 2023-10-09T09:34:09.788970+0000 mgr.y (mgr.34107) 839 : cluster 0 pgmap v1567: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 675 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 5410/32460 objects degraded (16.667%); 336/32460 objects misplaced (1.035%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:10.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:09 smithi163 bash[78107]: cluster 2023-10-09T09:34:09.840220+0000 mon.a (mon.0) 1692 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:34:10.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:10.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:10.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:10.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:10.446 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:10.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:10.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:10.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:10.599 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:10.629 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:10.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:10.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:10.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:10.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:10.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:10.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:10.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:10.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:10.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:10.998 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:11.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:11.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:11.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:11.062 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:34:11.062 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:34:11.062 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2023-10-09T09:34:11.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:34:11.132 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:11.165 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:11.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:11.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:11.272 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:11.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:11.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:11.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:11.409 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:11.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:11 smithi181 bash[59345]: audit 2023-10-09T09:34:10.382603+0000 mon.a (mon.0) 1693 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:34:11.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:11 smithi181 bash[59345]: audit 2023-10-09T09:34:10.383899+0000 mon.a (mon.0) 1694 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:11.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:11 smithi181 bash[59345]: cluster 2023-10-09T09:34:10.607144+0000 mon.a (mon.0) 1695 : cluster 3 Health check update: Degraded data redundancy: 5410/32460 objects degraded (16.667%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:11.439 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:11.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:11.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:11.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:11.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:11.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:11.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:11.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:11.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:11 smithi163 bash[75425]: audit 2023-10-09T09:34:10.382603+0000 mon.a (mon.0) 1693 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:34:11.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:11 smithi163 bash[75425]: audit 2023-10-09T09:34:10.383899+0000 mon.a (mon.0) 1694 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:11.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:11 smithi163 bash[75425]: cluster 2023-10-09T09:34:10.607144+0000 mon.a (mon.0) 1695 : cluster 3 Health check update: Degraded data redundancy: 5410/32460 objects degraded (16.667%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:11.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:11 smithi163 bash[78107]: audit 2023-10-09T09:34:10.382603+0000 mon.a (mon.0) 1693 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:34:11.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:11 smithi163 bash[78107]: audit 2023-10-09T09:34:10.383899+0000 mon.a (mon.0) 1694 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:11.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:11 smithi163 bash[78107]: cluster 2023-10-09T09:34:10.607144+0000 mon.a (mon.0) 1695 : cluster 3 Health check update: Degraded data redundancy: 5410/32460 objects degraded (16.667%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:11.706 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:11.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:11.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:11.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:11.831 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:11.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:11.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:11.938 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:11.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:11.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:12.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:12.105 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:12.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:12.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:12.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:12.236 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:12.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:12.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:12.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:12.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:12.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:12.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:12 smithi163 bash[75425]: cluster 2023-10-09T09:34:11.789857+0000 mgr.y (mgr.34107) 840 : cluster 0 pgmap v1568: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 425 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 3321/20445 objects degraded (16.244%); 197/20445 objects misplaced (0.964%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:12.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:12 smithi163 bash[78107]: cluster 2023-10-09T09:34:11.789857+0000 mgr.y (mgr.34107) 840 : cluster 0 pgmap v1568: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 425 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 3321/20445 objects degraded (16.244%); 197/20445 objects misplaced (0.964%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:12.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:12.494 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:12.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:12.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:12.597 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:12.622 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:12.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:12.727 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:12.762 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:12.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:12.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:12 smithi181 bash[59345]: cluster 2023-10-09T09:34:11.789857+0000 mgr.y (mgr.34107) 840 : cluster 0 pgmap v1568: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 425 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 3321/20445 objects degraded (16.244%); 197/20445 objects misplaced (0.964%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:12.838 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:34:12.838 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:34:12.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:12.907 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:12.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:12.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:13.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:13.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:13.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:13.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:13.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:13.171 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:13.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:13.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:13.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:13.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:13.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:13.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:13.403 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:13.429 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:13.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:13.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:13.432 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:34:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:34:13] "GET /metrics HTTP/1.1" 200 36343 "" "Prometheus/2.33.4" 2023-10-09T09:34:13.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:13.564 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:13.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:13.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:13.671 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:13.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:13.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:13.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:13.734 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:34:13.735 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:34:13.806 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:13.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:13.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:13.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:13.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:13.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:13 smithi181 bash[59345]: cluster 2023-10-09T09:34:13.790443+0000 mgr.y (mgr.34107) 841 : cluster 0 pgmap v1569: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 425 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 3321/20445 objects degraded (16.244%); 197/20445 objects misplaced (0.964%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:13.971 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:13.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:13.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:14.076 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:14.108 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:14.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:14.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:14.212 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:14.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:13 smithi163 bash[75425]: cluster 2023-10-09T09:34:13.790443+0000 mgr.y (mgr.34107) 841 : cluster 0 pgmap v1569: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 425 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 3321/20445 objects degraded (16.244%); 197/20445 objects misplaced (0.964%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:14.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:13 smithi163 bash[78107]: cluster 2023-10-09T09:34:13.790443+0000 mgr.y (mgr.34107) 841 : cluster 0 pgmap v1569: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 425 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 3321/20445 objects degraded (16.244%); 197/20445 objects misplaced (0.964%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:14.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:14.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:14.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:14.342 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:14.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:14.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:14.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:14.475 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:14.506 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:14.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:14.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:14.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:14.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:14.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:14.769 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:14.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:14.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:14.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:14.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:14.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:15.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:15.034 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:15.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:15.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:15.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:15.170 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:15.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:15.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:15.274 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:15.305 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:15.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:15.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:15.408 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:15.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:15.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:15.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:15.537 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:15.564 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:15.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:15.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:15.663 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:15 smithi163 bash[75425]: cluster 2023-10-09T09:34:15.609761+0000 mon.a (mon.0) 1696 : cluster 3 Health check update: Degraded data redundancy: 3321/20445 objects degraded (16.244%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:15.668 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:15.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:15 smithi181 bash[59345]: cluster 2023-10-09T09:34:15.609761+0000 mon.a (mon.0) 1696 : cluster 3 Health check update: Degraded data redundancy: 3321/20445 objects degraded (16.244%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:15.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:15.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:15.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:15.805 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:15.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:15.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:15.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:15.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:15 smithi163 bash[78107]: cluster 2023-10-09T09:34:15.609761+0000 mon.a (mon.0) 1696 : cluster 3 Health check update: Degraded data redundancy: 3321/20445 objects degraded (16.244%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:15.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:15.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:15.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:16.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:16.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:16.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:16.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:16.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:16.235 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:16.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:16.341 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:16.368 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:16.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:16.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:16.475 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:16.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:16.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:16.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:16.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T09:34:16.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:16.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:16.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:16.769 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:16.792 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:16.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:16.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:16.888 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:16 smithi163 bash[75425]: cluster 2023-10-09T09:34:15.791261+0000 mgr.y (mgr.34107) 842 : cluster 0 pgmap v1570: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 271 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 2360/13038 objects degraded (18.101%); 197/13038 objects misplaced (1.511%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:16.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:16 smithi163 bash[78107]: cluster 2023-10-09T09:34:15.791261+0000 mgr.y (mgr.34107) 842 : cluster 0 pgmap v1570: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 271 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 2360/13038 objects degraded (18.101%); 197/13038 objects misplaced (1.511%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:16.897 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:16.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:16.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:16.932 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:16 smithi181 bash[59345]: cluster 2023-10-09T09:34:15.791261+0000 mgr.y (mgr.34107) 842 : cluster 0 pgmap v1570: 99 pgs: 1 active+clean+remapped, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 271 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 2360/13038 objects degraded (18.101%); 197/13038 objects misplaced (1.511%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:17.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:17.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:17.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:17.120 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:34:17.182 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:17.209 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:17.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:17.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:17.312 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:17.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:17.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:17.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:17.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:17.490 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:17.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:17.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:17.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:17.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:17.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:17.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:17.679 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 108660 objects 2023-10-09T09:34:17.679 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :124.12 2023-10-09T09:34:17.726 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:17.754 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:17.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:17.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:17.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:17.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:17.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:17.991 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:18.022 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:18.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:18.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:18.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:18.138 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:17 smithi181 bash[59345]: cluster 2023-10-09T09:34:17.792019+0000 mgr.y (mgr.34107) 843 : cluster 0 pgmap v1571: 99 pgs: 1 active+clean+remapped, 18 active+undersized+degraded, 21 active+undersized, 59 active+clean; 118 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 870/5721 objects degraded (15.207%); 29/5721 objects misplaced (0.507%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:18.153 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:18.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:18.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:18.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:17 smithi163 bash[75425]: cluster 2023-10-09T09:34:17.792019+0000 mgr.y (mgr.34107) 843 : cluster 0 pgmap v1571: 99 pgs: 1 active+clean+remapped, 18 active+undersized+degraded, 21 active+undersized, 59 active+clean; 118 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 870/5721 objects degraded (15.207%); 29/5721 objects misplaced (0.507%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:18.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:17 smithi163 bash[78107]: cluster 2023-10-09T09:34:17.792019+0000 mgr.y (mgr.34107) 843 : cluster 0 pgmap v1571: 99 pgs: 1 active+clean+remapped, 18 active+undersized+degraded, 21 active+undersized, 59 active+clean; 118 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 870/5721 objects degraded (15.207%); 29/5721 objects misplaced (0.507%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:18.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:18.283 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:18.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:18.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:18.387 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:18.417 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:18.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:18.520 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:18.551 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:18.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:18.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:18.654 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:18.675 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:18.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:18.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:18.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:18.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:18.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:18.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:18.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:18.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:18.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:18.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:19.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:19.063 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:19.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:19.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:19.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:19.200 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:19.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:19.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:19.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:19.337 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:19.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:19.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:19.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:19.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:19.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:19.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:19.595 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:19.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:19.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:19.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:19.734 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:19.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:19.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:19.859 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:19.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:19.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:19.962 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:19.987 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:19.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:19.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:20.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:19 smithi181 bash[59345]: cluster 2023-10-09T09:34:19.792606+0000 mgr.y (mgr.34107) 844 : cluster 0 pgmap v1572: 99 pgs: 1 active+clean+remapped, 17 active+undersized+degraded, 22 active+undersized, 59 active+clean; 84 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 657/4086 objects degraded (16.079%); 29/4086 objects misplaced (0.710%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:20.090 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:20.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:20.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:20.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:20.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:19 smithi163 bash[75425]: cluster 2023-10-09T09:34:19.792606+0000 mgr.y (mgr.34107) 844 : cluster 0 pgmap v1572: 99 pgs: 1 active+clean+remapped, 17 active+undersized+degraded, 22 active+undersized, 59 active+clean; 84 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 657/4086 objects degraded (16.079%); 29/4086 objects misplaced (0.710%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:20.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:19 smithi163 bash[78107]: cluster 2023-10-09T09:34:19.792606+0000 mgr.y (mgr.34107) 844 : cluster 0 pgmap v1572: 99 pgs: 1 active+clean+remapped, 17 active+undersized+degraded, 22 active+undersized, 59 active+clean; 84 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 657/4086 objects degraded (16.079%); 29/4086 objects misplaced (0.710%); 0 B/s, 0 objects/s recovering 2023-10-09T09:34:20.225 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:20.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:20.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:20.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:20.360 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:20.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:20.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:20.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:20.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:20.528 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:20.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:20.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:20.661 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:20.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:20.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:20.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:20.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:20.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:20.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:20.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:20.909 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:20 smithi181 bash[59345]: cluster 2023-10-09T09:34:20.611004+0000 mon.a (mon.0) 1697 : cluster 3 Health check update: Degraded data redundancy: 657/4086 objects degraded (16.079%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:20.925 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:20.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:20.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:21.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:21.056 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:21.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:21.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:21.160 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:21.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:20 smithi163 bash[75425]: cluster 2023-10-09T09:34:20.611004+0000 mon.a (mon.0) 1697 : cluster 3 Health check update: Degraded data redundancy: 657/4086 objects degraded (16.079%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:21.172 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:20 smithi163 bash[78107]: cluster 2023-10-09T09:34:20.611004+0000 mon.a (mon.0) 1697 : cluster 3 Health check update: Degraded data redundancy: 657/4086 objects degraded (16.079%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:21.191 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:21.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:21.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:21.322 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:21.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:21.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:21.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:21.453 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:21.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:21.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:21.482 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:34:21.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:21.584 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:21.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:21.687 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:21.714 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:21.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:21.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:21.839 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:21.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:21.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:21.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:21.968 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:21.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:21.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:21 smithi181 bash[59345]: cluster 2023-10-09T09:34:21.793457+0000 mgr.y (mgr.34107) 845 : cluster 0 pgmap v1573: 99 pgs: 1 active+clean+remapped, 16 active+undersized+degraded, 23 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:22.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:22.101 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:22.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:22.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:22.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:22.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:21 smithi163 bash[78107]: cluster 2023-10-09T09:34:21.793457+0000 mgr.y (mgr.34107) 845 : cluster 0 pgmap v1573: 99 pgs: 1 active+clean+remapped, 16 active+undersized+degraded, 23 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:22.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:21 smithi163 bash[75425]: cluster 2023-10-09T09:34:21.793457+0000 mgr.y (mgr.34107) 845 : cluster 0 pgmap v1573: 99 pgs: 1 active+clean+remapped, 16 active+undersized+degraded, 23 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:22.230 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:22.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:22.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:22.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:22.362 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:22.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:22.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:22.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:22.491 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:22.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:22.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:22.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:22.620 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:22.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:22.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:22.634 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:34:22.654 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:34:22.673 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:34:22.694 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:34:22.716 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:34:22.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:22.741 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:34:22.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:22.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:22.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:22.767 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:34:22.767 INFO:teuthology.orchestra.run.smithi163.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T09:34:22.767 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:34:22.767 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:34:22.767 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:34:22.767 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:34:22.767 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:34:22.767 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:34:22.768 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:34:22.848 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:22.879 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:22.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:22.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:22.902 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:34:22.982 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:23.013 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:23.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:23.092 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:22 smithi181 bash[59345]: audit 2023-10-09T09:34:22.611213+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.163:0/1630316602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:23.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:22 smithi181 bash[59345]: audit 2023-10-09T09:34:22.735574+0000 mon.a (mon.0) 1698 : audit 1 from='client.? 172.21.15.163:0/2191778410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:34:23.116 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:23.144 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:23.145 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:22 smithi163 bash[75425]: audit 2023-10-09T09:34:22.611213+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.163:0/1630316602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:23.145 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:22 smithi163 bash[75425]: audit 2023-10-09T09:34:22.735574+0000 mon.a (mon.0) 1698 : audit 1 from='client.? 172.21.15.163:0/2191778410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:34:23.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:22 smithi163 bash[78107]: audit 2023-10-09T09:34:22.611213+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.163:0/1630316602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:23.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:22 smithi163 bash[78107]: audit 2023-10-09T09:34:22.735574+0000 mon.a (mon.0) 1698 : audit 1 from='client.? 172.21.15.163:0/2191778410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:34:23.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:23.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:23.247 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:23.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:23.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:23.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:23.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:23.407 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:23.408 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:34:23] "GET /metrics HTTP/1.1" 200 36343 "" "Prometheus/2.33.4" 2023-10-09T09:34:23.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:23.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:23.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:23.536 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:23.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:23.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:23.639 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:23.661 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:23.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:23.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:23.795 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:23.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:23.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:23.898 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:23.923 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:23.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:23.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:24.026 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:24.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[75425]: cluster 2023-10-09T09:34:22.886481+0000 mon.a (mon.0) 1699 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:34:24.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[75425]: cluster 2023-10-09T09:34:22.896097+0000 osd.5 (osd.5) 681 : cluster 3 failed to encode map e1127 with expected crc 2023-10-09T09:34:24.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[75425]: cluster 2023-10-09T09:34:22.896265+0000 osd.6 (osd.6) 411 : cluster 3 failed to encode map e1127 with expected crc 2023-10-09T09:34:24.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[75425]: audit 2023-10-09T09:34:22.898099+0000 mon.a (mon.0) 1700 : audit 1 from='client.? 172.21.15.163:0/2191778410' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:34:24.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[75425]: cluster 2023-10-09T09:34:22.898162+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e1127: 8 total, 7 up, 8 in 2023-10-09T09:34:24.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[75425]: cluster 2023-10-09T09:34:23.794153+0000 mgr.y (mgr.34107) 846 : cluster 0 pgmap v1575: 99 pgs: 1 active+clean+remapped, 16 active+undersized+degraded, 23 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:24.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[78107]: cluster 2023-10-09T09:34:22.886481+0000 mon.a (mon.0) 1699 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:34:24.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[78107]: cluster 2023-10-09T09:34:22.896097+0000 osd.5 (osd.5) 681 : cluster 3 failed to encode map e1127 with expected crc 2023-10-09T09:34:24.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[78107]: cluster 2023-10-09T09:34:22.896265+0000 osd.6 (osd.6) 411 : cluster 3 failed to encode map e1127 with expected crc 2023-10-09T09:34:24.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[78107]: audit 2023-10-09T09:34:22.898099+0000 mon.a (mon.0) 1700 : audit 1 from='client.? 172.21.15.163:0/2191778410' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:34:24.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[78107]: cluster 2023-10-09T09:34:22.898162+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e1127: 8 total, 7 up, 8 in 2023-10-09T09:34:24.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:23 smithi163 bash[78107]: cluster 2023-10-09T09:34:23.794153+0000 mgr.y (mgr.34107) 846 : cluster 0 pgmap v1575: 99 pgs: 1 active+clean+remapped, 16 active+undersized+degraded, 23 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:24.054 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:24.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:24.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:23 smithi181 bash[59345]: cluster 2023-10-09T09:34:22.886481+0000 mon.a (mon.0) 1699 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:34:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:23 smithi181 bash[59345]: cluster 2023-10-09T09:34:22.896097+0000 osd.5 (osd.5) 681 : cluster 3 failed to encode map e1127 with expected crc 2023-10-09T09:34:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:23 smithi181 bash[59345]: cluster 2023-10-09T09:34:22.896265+0000 osd.6 (osd.6) 411 : cluster 3 failed to encode map e1127 with expected crc 2023-10-09T09:34:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:23 smithi181 bash[59345]: audit 2023-10-09T09:34:22.898099+0000 mon.a (mon.0) 1700 : audit 1 from='client.? 172.21.15.163:0/2191778410' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:34:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:23 smithi181 bash[59345]: cluster 2023-10-09T09:34:22.898162+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e1127: 8 total, 7 up, 8 in 2023-10-09T09:34:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:23 smithi181 bash[59345]: cluster 2023-10-09T09:34:23.794153+0000 mgr.y (mgr.34107) 846 : cluster 0 pgmap v1575: 99 pgs: 1 active+clean+remapped, 16 active+undersized+degraded, 23 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:24.158 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:24.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:24.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:24.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:24.314 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:24.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:24.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:24.418 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:24.447 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:24.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:24.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:24.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:24.579 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:24.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:24.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:24.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:24.706 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:24.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:24.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:24.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:24.836 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:24.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:24.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:24.969 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:24.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:25.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:25.099 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:25.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:25.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:25.124 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:34:25.124 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:34:25.124 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T09:34:25.124 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:34:25.203 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:25.229 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:25.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:25.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:25.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:25.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:25.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:25.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:25.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:25.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:25.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:25.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:25.611 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:25 smithi181 bash[59345]: audit 2023-10-09T09:34:25.372916+0000 mon.a (mon.0) 1702 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:25.630 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:25.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:25.686 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:25 smithi163 bash[78107]: audit 2023-10-09T09:34:25.372916+0000 mon.a (mon.0) 1702 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:25.686 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:25 smithi163 bash[75425]: audit 2023-10-09T09:34:25.372916+0000 mon.a (mon.0) 1702 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T09:34:25.729 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T09:34:25.733 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:25.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:25.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:25.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:25.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:25.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:25.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:25.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T09:34:26.001 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:26.044 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:26.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:26.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:26.074 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:34:26.077 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:34:26.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:26.171 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:26.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:26.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:26.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:26.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:26.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:26.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:26.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:26.437 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:26.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:26.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:26.541 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:26.574 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:26.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:26.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:26.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:26.689 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:26 smithi163 bash[75425]: cluster 2023-10-09T09:34:25.612278+0000 mon.a (mon.0) 1703 : cluster 3 Health check update: Degraded data redundancy: 118/390 objects degraded (30.256%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:26.689 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:26 smithi163 bash[75425]: cluster 2023-10-09T09:34:25.795245+0000 mgr.y (mgr.34107) 847 : cluster 0 pgmap v1576: 99 pgs: 1 active+clean+remapped, 14 active+undersized+degraded, 25 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:26.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:26 smithi163 bash[78107]: cluster 2023-10-09T09:34:25.612278+0000 mon.a (mon.0) 1703 : cluster 3 Health check update: Degraded data redundancy: 118/390 objects degraded (30.256%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:26.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:26 smithi163 bash[78107]: cluster 2023-10-09T09:34:25.795245+0000 mgr.y (mgr.34107) 847 : cluster 0 pgmap v1576: 99 pgs: 1 active+clean+remapped, 14 active+undersized+degraded, 25 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:26.705 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:26.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:26.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:26 smithi181 bash[59345]: cluster 2023-10-09T09:34:25.612278+0000 mon.a (mon.0) 1703 : cluster 3 Health check update: Degraded data redundancy: 118/390 objects degraded (30.256%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T09:34:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:26 smithi181 bash[59345]: cluster 2023-10-09T09:34:25.795245+0000 mgr.y (mgr.34107) 847 : cluster 0 pgmap v1576: 99 pgs: 1 active+clean+remapped, 14 active+undersized+degraded, 25 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:26.839 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:26.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:26.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:26.942 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:26.974 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:26.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:26.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:27.065 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1576,"stamp":"2023-10-09T09:34:25.794391+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7509920,"num_objects":130,"num_object_clones":0,"num_object_copies":390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":118,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":130,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":170517,"num_write_kb":4775246,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":227303,"ondisk_log_size":227303,"up":257,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4171468,"kb_used_data":1692812,"kb_used_omap":416,"kb_used_meta":2478175,"kb_avail":745789748,"statfs":{"total":767960285184,"available":763688701952,"internally_reserved":0,"allocated":1733439488,"data_stored":1728100556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":426575,"internal_metadata":2537651633},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-276267032,"num_objects":-4216,"num_object_clones":0,"num_object_copies":-12648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2242,"num_objects_misplaced":-197,"num_objects_unfound":0,"num_objects_dirty":-4216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4322,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003704"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1093,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.837252+0000","last_change":"2023-10-09T09:15:07.216468+0000","last_active":"2023-10-09T09:34:02.837252+0000","last_peered":"2023-10-09T09:34:02.837252+0000","last_clean":"2023-10-09T09:34:02.837252+0000","last_became_active":"2023-10-09T09:15:07.215908+0000","last_became_peered":"2023-10-09T09:15:07.215908+0000","last_unstale":"2023-10-09T09:34:02.837252+0000","last_undegraded":"2023-10-09T09:34:02.837252+0000","last_fullsized":"2023-10-09T09:34:02.837252+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":616,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:05.430102+0000","last_change":"2023-10-09T09:34:03.851672+0000","last_active":"2023-10-09T09:34:05.430102+0000","last_peered":"2023-10-09T09:34:05.430102+0000","last_clean":"2023-10-09T09:34:05.430102+0000","last_became_active":"2023-10-09T09:34:03.850986+0000","last_became_peered":"2023-10-09T09:34:03.850986+0000","last_unstale":"2023-10-09T09:34:05.430102+0000","last_undegraded":"2023-10-09T09:34:05.430102+0000","last_fullsized":"2023-10-09T09:34:05.430102+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:46.208257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1065,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.834726+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:34:03.834726+0000","last_peered":"2023-10-09T09:34:03.834726+0000","last_clean":"2023-10-09T09:34:03.834726+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:34:03.834726+0000","last_undegraded":"2023-10-09T09:34:03.834726+0000","last_fullsized":"2023-10-09T09:34:03.834726+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":605,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.831996+0000","last_change":"2023-10-09T09:33:28.650119+0000","last_active":"2023-10-09T09:34:02.831996+0000","last_peered":"2023-10-09T09:34:02.831996+0000","last_clean":"2023-10-09T09:15:20.154467+0000","last_became_active":"2023-10-09T09:33:28.650119+0000","last_became_peered":"2023-10-09T09:33:28.650119+0000","last_unstale":"2023-10-09T09:34:02.831996+0000","last_undegraded":"2023-10-09T09:34:02.831996+0000","last_fullsized":"2023-10-09T09:33:28.644777+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1047,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:03.845009+0000","last_change":"2023-10-09T09:34:03.843738+0000","last_active":"2023-10-09T09:34:03.845009+0000","last_peered":"2023-10-09T09:34:03.845009+0000","last_clean":"2023-10-09T09:33:16.681413+0000","last_became_active":"2023-10-09T09:34:03.843738+0000","last_became_peered":"2023-10-09T09:34:03.843738+0000","last_unstale":"2023-10-09T09:34:03.845009+0000","last_undegraded":"2023-10-09T09:34:03.845009+0000","last_fullsized":"2023-10-09T09:34:03.834611+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:53.606627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1057,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.827038+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:34:02.827038+0000","last_peered":"2023-10-09T09:34:02.827038+0000","last_clean":"2023-10-09T09:34:02.827038+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:34:02.827038+0000","last_undegraded":"2023-10-09T09:34:02.827038+0000","last_fullsized":"2023-10-09T09:34:02.827038+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"1126'9066","reported_seq":9167,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.662009+0000","last_change":"2023-10-09T09:34:03.887618+0000","last_active":"2023-10-09T09:34:17.662009+0000","last_peered":"2023-10-09T09:34:17.662009+0000","last_clean":"2023-10-09T09:34:17.662009+0000","last_became_active":"2023-10-09T09:34:03.856093+0000","last_became_peered":"2023-10-09T09:34:03.856093+0000","last_unstale":"2023-10-09T09:34:17.662009+0000","last_undegraded":"2023-10-09T09:34:17.662009+0000","last_fullsized":"2023-10-09T09:34:17.662009+0000","mapping_epoch":1125,"log_start":"1102'1","ondisk_log_start":"1102'1","created":1105,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":5,"last_scrub":"1123'7668","last_scrub_stamp":"2023-10-09T09:33:29.995827+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:33:29.995827+0000","objects_scrubbed":1404,"log_size":9065,"log_dups_size":0,"ondisk_log_size":9065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:01.575765+0000","scrub_duration":0.32018834299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1404,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":606,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.823944+0000","last_change":"2023-10-09T09:33:28.694893+0000","last_active":"2023-10-09T09:34:02.823944+0000","last_peered":"2023-10-09T09:34:02.823944+0000","last_clean":"2023-10-09T09:15:20.155939+0000","last_became_active":"2023-10-09T09:33:28.694893+0000","last_became_peered":"2023-10-09T09:33:28.694893+0000","last_unstale":"2023-10-09T09:34:02.823944+0000","last_undegraded":"2023-10-09T09:34:02.823944+0000","last_fullsized":"2023-10-09T09:33:28.645658+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1027,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:14.829740+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:34:14.829740+0000","last_peered":"2023-10-09T09:34:14.829740+0000","last_clean":"2023-10-09T09:34:14.829740+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:34:14.829740+0000","last_undegraded":"2023-10-09T09:34:14.829740+0000","last_fullsized":"2023-10-09T09:34:14.829740+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"1126'8726","reported_seq":14968,"reported_epoch":1126,"state":"active+clean+remapped","last_fresh":"2023-10-09T09:34:17.640995+0000","last_change":"2023-10-09T09:33:38.334673+0000","last_active":"2023-10-09T09:34:17.640995+0000","last_peered":"2023-10-09T09:34:17.640995+0000","last_clean":"2023-10-09T09:34:17.640995+0000","last_became_active":"2023-10-09T09:33:29.656619+0000","last_became_peered":"2023-10-09T09:33:29.656619+0000","last_unstale":"2023-10-09T09:34:17.640995+0000","last_undegraded":"2023-10-09T09:34:17.640995+0000","last_fullsized":"2023-10-09T09:34:17.640995+0000","mapping_epoch":1122,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1123,"parent":"0.0","parent_split_bits":5,"last_scrub":"1123'7586","last_scrub_stamp":"2023-10-09T09:33:38.334475+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:33:38.334475+0000","objects_scrubbed":0,"log_size":8726,"log_dups_size":0,"ondisk_log_size":8726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1145,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1078,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.824307+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:34:02.824307+0000","last_peered":"2023-10-09T09:34:02.824307+0000","last_clean":"2023-10-09T09:34:02.824307+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:34:02.824307+0000","last_undegraded":"2023-10-09T09:34:02.824307+0000","last_fullsized":"2023-10-09T09:34:02.824307+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":586,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.852834+0000","last_change":"2023-10-09T09:34:03.852834+0000","last_active":"2023-10-09T09:34:03.852834+0000","last_peered":"2023-10-09T09:34:03.852834+0000","last_clean":"2023-10-09T09:34:03.852834+0000","last_became_active":"2023-10-09T09:34:03.852517+0000","last_became_peered":"2023-10-09T09:34:03.852517+0000","last_unstale":"2023-10-09T09:34:03.852834+0000","last_undegraded":"2023-10-09T09:34:03.852834+0000","last_fullsized":"2023-10-09T09:34:03.852834+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.11","version":"1126'8967","reported_seq":13813,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.677402+0000","last_change":"2023-10-09T09:33:39.578704+0000","last_active":"2023-10-09T09:34:17.677402+0000","last_peered":"2023-10-09T09:34:17.677402+0000","last_clean":"2023-10-09T09:34:17.677402+0000","last_became_active":"2023-10-09T09:32:48.751363+0000","last_became_peered":"2023-10-09T09:32:48.751363+0000","last_unstale":"2023-10-09T09:34:17.677402+0000","last_undegraded":"2023-10-09T09:34:17.677402+0000","last_fullsized":"2023-10-09T09:34:17.677402+0000","mapping_epoch":1118,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1119,"parent":"0.0","parent_split_bits":5,"last_scrub":"1123'7858","last_scrub_stamp":"2023-10-09T09:33:39.578613+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:33:39.578613+0000","objects_scrubbed":0,"log_size":8967,"log_dups_size":0,"ondisk_log_size":8967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1114,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1057,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.830427+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:34:02.830427+0000","last_peered":"2023-10-09T09:34:02.830427+0000","last_clean":"2023-10-09T09:34:02.830427+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:34:02.830427+0000","last_undegraded":"2023-10-09T09:34:02.830427+0000","last_fullsized":"2023-10-09T09:34:02.830427+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1065,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.843061+0000","last_change":"2023-10-09T09:15:07.204630+0000","last_active":"2023-10-09T09:34:03.843061+0000","last_peered":"2023-10-09T09:34:03.843061+0000","last_clean":"2023-10-09T09:34:03.843061+0000","last_became_active":"2023-10-09T09:15:07.201880+0000","last_became_peered":"2023-10-09T09:15:07.201880+0000","last_unstale":"2023-10-09T09:34:03.843061+0000","last_undegraded":"2023-10-09T09:34:03.843061+0000","last_fullsized":"2023-10-09T09:34:03.843061+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:46.471940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"1126'8927","reported_seq":13754,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.680057+0000","last_change":"2023-10-09T09:34:03.878472+0000","last_active":"2023-10-09T09:34:17.680057+0000","last_peered":"2023-10-09T09:34:17.680057+0000","last_clean":"2023-10-09T09:34:17.680057+0000","last_became_active":"2023-10-09T09:34:03.844414+0000","last_became_peered":"2023-10-09T09:34:03.844414+0000","last_unstale":"2023-10-09T09:34:17.680057+0000","last_undegraded":"2023-10-09T09:34:17.680057+0000","last_fullsized":"2023-10-09T09:34:17.680057+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":5,"last_scrub":"1123'7700","last_scrub_stamp":"2023-10-09T09:33:36.427666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:33:36.427666+0000","objects_scrubbed":1240,"log_size":8927,"log_dups_size":0,"ondisk_log_size":8927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:43.571453+0000","scrub_duration":0.78788483799999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1240,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1084,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.837867+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:34:02.837867+0000","last_peered":"2023-10-09T09:34:02.837867+0000","last_clean":"2023-10-09T09:34:02.837867+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:34:02.837867+0000","last_undegraded":"2023-10-09T09:34:02.837867+0000","last_fullsized":"2023-10-09T09:34:02.837867+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1076,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.838345+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:34:02.838345+0000","last_peered":"2023-10-09T09:34:02.838345+0000","last_clean":"2023-10-09T09:34:02.838345+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:34:02.838345+0000","last_undegraded":"2023-10-09T09:34:02.838345+0000","last_fullsized":"2023-10-09T09:34:02.838345+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"1126'9024","reported_seq":9074,"reported_epoch":1126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:17.665344+0000","last_change":"2023-10-09T09:33:28.656831+0000","last_active":"2023-10-09T09:34:17.665344+0000","last_peered":"2023-10-09T09:34:17.665344+0000","last_clean":"2023-10-09T09:33:27.576977+0000","last_became_active":"2023-10-09T09:33:28.656831+0000","last_became_peered":"2023-10-09T09:33:28.656831+0000","last_unstale":"2023-10-09T09:34:17.665344+0000","last_undegraded":"2023-10-09T09:33:28.649237+0000","last_fullsized":"2023-10-09T09:33:28.649083+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":9024,"log_dups_size":0,"ondisk_log_size":9024,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":655360,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6856,"num_write_kb":219712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":10}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1056,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.828764+0000","last_change":"2023-10-09T09:15:07.200997+0000","last_active":"2023-10-09T09:34:02.828764+0000","last_peered":"2023-10-09T09:34:02.828764+0000","last_clean":"2023-10-09T09:34:02.828764+0000","last_became_active":"2023-10-09T09:15:07.200564+0000","last_became_peered":"2023-10-09T09:15:07.200564+0000","last_unstale":"2023-10-09T09:34:02.828764+0000","last_undegraded":"2023-10-09T09:34:02.828764+0000","last_fullsized":"2023-10-09T09:34:02.828764+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":583,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.849331+0000","last_change":"2023-10-09T09:34:03.848913+0000","last_active":"2023-10-09T09:34:03.849331+0000","last_peered":"2023-10-09T09:34:03.849331+0000","last_clean":"2023-10-09T09:34:03.849331+0000","last_became_active":"2023-10-09T09:34:03.848474+0000","last_became_peered":"2023-10-09T09:34:03.848474+0000","last_unstale":"2023-10-09T09:34:03.849331+0000","last_undegraded":"2023-10-09T09:34:03.849331+0000","last_fullsized":"2023-10-09T09:34:03.849331+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:59.618935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"1126'8972","reported_seq":9023,"reported_epoch":1126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:17.605914+0000","last_change":"2023-10-09T09:33:28.654097+0000","last_active":"2023-10-09T09:34:17.605914+0000","last_peered":"2023-10-09T09:34:17.605914+0000","last_clean":"2023-10-09T09:33:27.467767+0000","last_became_active":"2023-10-09T09:33:28.654097+0000","last_became_peered":"2023-10-09T09:33:28.654097+0000","last_unstale":"2023-10-09T09:34:17.605914+0000","last_undegraded":"2023-10-09T09:33:28.646065+0000","last_fullsized":"2023-10-09T09:33:28.645913+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8972,"log_dups_size":0,"ondisk_log_size":8972,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3047424,"num_objects":47,"num_object_clones":0,"num_object_copies":141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6833,"num_write_kb":220128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":47}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":606,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.828933+0000","last_change":"2023-10-09T09:33:28.645259+0000","last_active":"2023-10-09T09:34:02.828933+0000","last_peered":"2023-10-09T09:34:02.828933+0000","last_clean":"2023-10-09T09:15:20.154024+0000","last_became_active":"2023-10-09T09:33:28.645259+0000","last_became_peered":"2023-10-09T09:33:28.645259+0000","last_unstale":"2023-10-09T09:34:02.828933+0000","last_undegraded":"2023-10-09T09:34:02.828933+0000","last_fullsized":"2023-10-09T09:33:28.642887+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1077,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.823991+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:34:02.823991+0000","last_peered":"2023-10-09T09:34:02.823991+0000","last_clean":"2023-10-09T09:34:02.823991+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:34:02.823991+0000","last_undegraded":"2023-10-09T09:34:02.823991+0000","last_fullsized":"2023-10-09T09:34:02.823991+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"1126'8950","reported_seq":9005,"reported_epoch":1126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:17.592918+0000","last_change":"2023-10-09T09:33:28.644926+0000","last_active":"2023-10-09T09:34:17.592918+0000","last_peered":"2023-10-09T09:34:17.592918+0000","last_clean":"2023-10-09T09:33:27.608675+0000","last_became_active":"2023-10-09T09:33:28.644926+0000","last_became_peered":"2023-10-09T09:33:28.644926+0000","last_unstale":"2023-10-09T09:34:17.592918+0000","last_undegraded":"2023-10-09T09:33:28.643025+0000","last_fullsized":"2023-10-09T09:33:28.642873+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"1106'4479","last_scrub_stamp":"2023-10-09T09:31:33.052706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:33.052706+0000","objects_scrubbed":0,"log_size":8950,"log_dups_size":0,"ondisk_log_size":8950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4496,"num_write_kb":69632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1039,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.829002+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:34:02.829002+0000","last_peered":"2023-10-09T09:34:02.829002+0000","last_clean":"2023-10-09T09:34:02.829002+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:34:02.829002+0000","last_undegraded":"2023-10-09T09:34:02.829002+0000","last_fullsized":"2023-10-09T09:34:02.829002+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":603,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:03.850574+0000","last_change":"2023-10-09T09:34:03.850502+0000","last_active":"2023-10-09T09:34:03.850574+0000","last_peered":"2023-10-09T09:34:03.850574+0000","last_clean":"2023-10-09T09:15:20.154364+0000","last_became_active":"2023-10-09T09:34:03.850502+0000","last_became_peered":"2023-10-09T09:34:03.850502+0000","last_unstale":"2023-10-09T09:34:03.850574+0000","last_undegraded":"2023-10-09T09:34:03.850574+0000","last_fullsized":"2023-10-09T09:34:03.843862+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.14","version":"1126'8792","reported_seq":8844,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:17.623407+0000","last_change":"2023-10-09T09:34:16.502892+0000","last_active":"2023-10-09T09:34:17.623407+0000","last_peered":"2023-10-09T09:34:17.623407+0000","last_clean":"2023-10-09T09:33:27.615140+0000","last_became_active":"2023-10-09T09:33:28.660747+0000","last_became_peered":"2023-10-09T09:33:28.660747+0000","last_unstale":"2023-10-09T09:34:17.623407+0000","last_undegraded":"2023-10-09T09:34:17.623407+0000","last_fullsized":"2023-10-09T09:33:28.655992+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8792,"log_dups_size":0,"ondisk_log_size":8792,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6665,"num_write_kb":212480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1073,"reported_epoch":1125,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:02.826153+0000","last_change":"2023-10-09T09:33:28.659671+0000","last_active":"2023-10-09T09:34:02.826153+0000","last_peered":"2023-10-09T09:34:02.826153+0000","last_clean":"2023-10-09T09:33:16.690243+0000","last_became_active":"2023-10-09T09:33:28.659671+0000","last_became_peered":"2023-10-09T09:33:28.659671+0000","last_unstale":"2023-10-09T09:34:02.826153+0000","last_undegraded":"2023-10-09T09:33:28.654346+0000","last_fullsized":"2023-10-09T09:33:28.654211+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1061,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.842565+0000","last_change":"2023-10-09T09:15:36.176004+0000","last_active":"2023-10-09T09:34:03.842565+0000","last_peered":"2023-10-09T09:34:03.842565+0000","last_clean":"2023-10-09T09:34:03.842565+0000","last_became_active":"2023-10-09T09:15:36.174644+0000","last_became_peered":"2023-10-09T09:15:36.174644+0000","last_unstale":"2023-10-09T09:34:03.842565+0000","last_undegraded":"2023-10-09T09:34:03.842565+0000","last_fullsized":"2023-10-09T09:34:03.842565+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:35.283525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1126'13520","reported_seq":13558,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.658112+0000","last_change":"2023-10-09T09:30:25.841890+0000","last_active":"2023-10-09T09:34:17.658112+0000","last_peered":"2023-10-09T09:34:17.658112+0000","last_clean":"2023-10-09T09:34:17.658112+0000","last_became_active":"2023-10-09T09:30:25.841584+0000","last_became_peered":"2023-10-09T09:30:25.841584+0000","last_unstale":"2023-10-09T09:34:17.658112+0000","last_undegraded":"2023-10-09T09:34:17.658112+0000","last_fullsized":"2023-10-09T09:34:17.658112+0000","mapping_epoch":1099,"log_start":"1114'6300","ondisk_log_start":"1114'6300","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":7220,"log_dups_size":0,"ondisk_log_size":7220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:07.145206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13520,"num_write_kb":432640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1061,"reported_epoch":1125,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:02.831496+0000","last_change":"2023-10-09T09:33:28.661640+0000","last_active":"2023-10-09T09:34:02.831496+0000","last_peered":"2023-10-09T09:34:02.831496+0000","last_clean":"2023-10-09T09:33:16.685131+0000","last_became_active":"2023-10-09T09:33:28.661640+0000","last_became_peered":"2023-10-09T09:33:28.661640+0000","last_unstale":"2023-10-09T09:34:02.831496+0000","last_undegraded":"2023-10-09T09:33:28.647562+0000","last_fullsized":"2023-10-09T09:33:28.647482+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1062,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.823628+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:34:02.823628+0000","last_peered":"2023-10-09T09:34:02.823628+0000","last_clean":"2023-10-09T09:34:02.823628+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:34:02.823628+0000","last_undegraded":"2023-10-09T09:34:02.823628+0000","last_fullsized":"2023-10-09T09:34:02.823628+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1126'13504","reported_seq":13548,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.675076+0000","last_change":"2023-10-09T09:34:03.888718+0000","last_active":"2023-10-09T09:34:17.675076+0000","last_peered":"2023-10-09T09:34:17.675076+0000","last_clean":"2023-10-09T09:34:17.675076+0000","last_became_active":"2023-10-09T09:34:03.844454+0000","last_became_peered":"2023-10-09T09:34:03.844454+0000","last_unstale":"2023-10-09T09:34:17.675076+0000","last_undegraded":"2023-10-09T09:34:17.675076+0000","last_fullsized":"2023-10-09T09:34:17.675076+0000","mapping_epoch":1125,"log_start":"1114'6100","ondisk_log_start":"1114'6100","created":1099,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:34.321355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13504,"num_write_kb":432128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1056,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.832821+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:34:02.832821+0000","last_peered":"2023-10-09T09:34:02.832821+0000","last_clean":"2023-10-09T09:34:02.832821+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:34:02.832821+0000","last_undegraded":"2023-10-09T09:34:02.832821+0000","last_fullsized":"2023-10-09T09:34:02.832821+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1091,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.839448+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:34:02.839448+0000","last_peered":"2023-10-09T09:34:02.839448+0000","last_clean":"2023-10-09T09:34:02.839448+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:34:02.839448+0000","last_undegraded":"2023-10-09T09:34:02.839448+0000","last_fullsized":"2023-10-09T09:34:02.839448+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1126'8875","reported_seq":8932,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:17.666203+0000","last_change":"2023-10-09T09:34:16.832955+0000","last_active":"2023-10-09T09:34:17.666203+0000","last_peered":"2023-10-09T09:34:17.666203+0000","last_clean":"2023-10-09T09:33:27.572382+0000","last_became_active":"2023-10-09T09:33:28.667172+0000","last_became_peered":"2023-10-09T09:33:28.667172+0000","last_unstale":"2023-10-09T09:34:17.666203+0000","last_undegraded":"2023-10-09T09:34:17.666203+0000","last_fullsized":"2023-10-09T09:33:28.639530+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8875,"log_dups_size":0,"ondisk_log_size":8875,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6750,"num_write_kb":215232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1040,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.830199+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:34:02.830199+0000","last_peered":"2023-10-09T09:34:02.830199+0000","last_clean":"2023-10-09T09:34:02.830199+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:34:02.830199+0000","last_undegraded":"2023-10-09T09:34:02.830199+0000","last_fullsized":"2023-10-09T09:34:02.830199+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1058,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.827011+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:34:02.827011+0000","last_peered":"2023-10-09T09:34:02.827011+0000","last_clean":"2023-10-09T09:34:02.827011+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:34:02.827011+0000","last_undegraded":"2023-10-09T09:34:02.827011+0000","last_fullsized":"2023-10-09T09:34:02.827011+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1126'9078","reported_seq":9165,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.670498+0000","last_change":"2023-10-09T09:33:29.318189+0000","last_active":"2023-10-09T09:34:17.670498+0000","last_peered":"2023-10-09T09:34:17.670498+0000","last_clean":"2023-10-09T09:34:17.670498+0000","last_became_active":"2023-10-09T09:31:28.843228+0000","last_became_peered":"2023-10-09T09:31:28.843228+0000","last_unstale":"2023-10-09T09:34:17.670498+0000","last_undegraded":"2023-10-09T09:34:17.670498+0000","last_fullsized":"2023-10-09T09:34:17.670498+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1122'7688","last_scrub_stamp":"2023-10-09T09:33:29.318000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:33:29.318000+0000","objects_scrubbed":1392,"log_size":9078,"log_dups_size":0,"ondisk_log_size":9078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:09.439803+0000","scrub_duration":0.275267498,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1392,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1078,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.823771+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:34:02.823771+0000","last_peered":"2023-10-09T09:34:02.823771+0000","last_clean":"2023-10-09T09:34:02.823771+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:34:02.823771+0000","last_undegraded":"2023-10-09T09:34:02.823771+0000","last_fullsized":"2023-10-09T09:34:02.823771+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":552,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:03.835119+0000","last_change":"2023-10-09T09:33:28.665153+0000","last_active":"2023-10-09T09:34:03.835119+0000","last_peered":"2023-10-09T09:34:03.835119+0000","last_clean":"2023-10-09T09:14:26.104810+0000","last_became_active":"2023-10-09T09:33:28.665153+0000","last_became_peered":"2023-10-09T09:33:28.665153+0000","last_unstale":"2023-10-09T09:34:03.835119+0000","last_undegraded":"2023-10-09T09:34:03.835119+0000","last_fullsized":"2023-10-09T09:33:28.639817+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:53.001511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"1126'13560","reported_seq":13603,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:17.674221+0000","last_change":"2023-10-09T09:34:23.027384+0000","last_active":"2023-10-09T09:34:17.674221+0000","last_peered":"2023-10-09T09:34:17.674221+0000","last_clean":"2023-10-09T09:33:27.559322+0000","last_became_active":"2023-10-09T09:33:28.650961+0000","last_became_peered":"2023-10-09T09:33:28.650961+0000","last_unstale":"2023-10-09T09:34:17.674221+0000","last_undegraded":"2023-10-09T09:33:28.637613+0000","last_fullsized":"2023-10-09T09:33:28.637491+0000","mapping_epoch":1121,"log_start":"1116'6987","ondisk_log_start":"1116'6987","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:35.375312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13560,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":20,"reported_epoch":1126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:03.854895+0000","last_change":"2023-10-09T09:34:03.854790+0000","last_active":"2023-10-09T09:34:03.854895+0000","last_peered":"2023-10-09T09:34:03.854895+0000","last_clean":"2023-10-09T09:03:26.326844+0000","last_became_active":"2023-10-09T09:34:03.854790+0000","last_became_peered":"2023-10-09T09:34:03.854790+0000","last_unstale":"2023-10-09T09:34:03.854895+0000","last_undegraded":"2023-10-09T09:34:03.848228+0000","last_fullsized":"2023-10-09T09:34:03.848098+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1076,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.837870+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:34:02.837870+0000","last_peered":"2023-10-09T09:34:02.837870+0000","last_clean":"2023-10-09T09:34:02.837870+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:34:02.837870+0000","last_undegraded":"2023-10-09T09:34:02.837870+0000","last_fullsized":"2023-10-09T09:34:02.837870+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1126'13654","reported_seq":13698,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:17.675196+0000","last_change":"2023-10-09T09:34:23.030005+0000","last_active":"2023-10-09T09:34:17.675196+0000","last_peered":"2023-10-09T09:34:17.675196+0000","last_clean":"2023-10-09T09:33:27.605146+0000","last_became_active":"2023-10-09T09:33:28.670244+0000","last_became_peered":"2023-10-09T09:33:28.670244+0000","last_unstale":"2023-10-09T09:34:17.675196+0000","last_undegraded":"2023-10-09T09:33:28.641517+0000","last_fullsized":"2023-10-09T09:33:28.633720+0000","mapping_epoch":1121,"log_start":"1103'3600","ondisk_log_start":"1103'3600","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":10054,"log_dups_size":0,"ondisk_log_size":10054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:54:00.889936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13654,"num_write_kb":436928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1059,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.827183+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:34:02.827183+0000","last_peered":"2023-10-09T09:34:02.827183+0000","last_clean":"2023-10-09T09:34:02.827183+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:34:02.827183+0000","last_undegraded":"2023-10-09T09:34:02.827183+0000","last_fullsized":"2023-10-09T09:34:02.827183+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1091,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.837560+0000","last_change":"2023-10-09T09:15:07.216161+0000","last_active":"2023-10-09T09:34:02.837560+0000","last_peered":"2023-10-09T09:34:02.837560+0000","last_clean":"2023-10-09T09:34:02.837560+0000","last_became_active":"2023-10-09T09:15:07.215802+0000","last_became_peered":"2023-10-09T09:15:07.215802+0000","last_unstale":"2023-10-09T09:34:02.837560+0000","last_undegraded":"2023-10-09T09:34:02.837560+0000","last_fullsized":"2023-10-09T09:34:02.837560+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1126'13778","reported_seq":13820,"reported_epoch":1126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:17.636155+0000","last_change":"2023-10-09T09:33:28.648333+0000","last_active":"2023-10-09T09:34:17.636155+0000","last_peered":"2023-10-09T09:34:17.636155+0000","last_clean":"2023-10-09T09:33:27.623386+0000","last_became_active":"2023-10-09T09:33:28.648333+0000","last_became_peered":"2023-10-09T09:33:28.648333+0000","last_unstale":"2023-10-09T09:34:17.636155+0000","last_undegraded":"2023-10-09T09:33:28.646938+0000","last_fullsized":"2023-10-09T09:33:28.646780+0000","mapping_epoch":1121,"log_start":"1113'5200","ondisk_log_start":"1113'5200","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8578,"log_dups_size":0,"ondisk_log_size":8578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13778,"num_write_kb":440833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1077,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.837236+0000","last_change":"2023-10-09T09:15:07.216972+0000","last_active":"2023-10-09T09:34:02.837236+0000","last_peered":"2023-10-09T09:34:02.837236+0000","last_clean":"2023-10-09T09:34:02.837236+0000","last_became_active":"2023-10-09T09:15:07.216317+0000","last_became_peered":"2023-10-09T09:15:07.216317+0000","last_unstale":"2023-10-09T09:34:02.837236+0000","last_undegraded":"2023-10-09T09:34:02.837236+0000","last_fullsized":"2023-10-09T09:34:02.837236+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1076,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.837933+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:34:02.837933+0000","last_peered":"2023-10-09T09:34:02.837933+0000","last_clean":"2023-10-09T09:34:02.837933+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:34:02.837933+0000","last_undegraded":"2023-10-09T09:34:02.837933+0000","last_fullsized":"2023-10-09T09:34:02.837933+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1126'13716","reported_seq":13751,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.674495+0000","last_change":"2023-10-09T09:30:25.841044+0000","last_active":"2023-10-09T09:34:17.674495+0000","last_peered":"2023-10-09T09:34:17.674495+0000","last_clean":"2023-10-09T09:34:17.674495+0000","last_became_active":"2023-10-09T09:30:25.840627+0000","last_became_peered":"2023-10-09T09:30:25.840627+0000","last_unstale":"2023-10-09T09:34:17.674495+0000","last_undegraded":"2023-10-09T09:34:17.674495+0000","last_fullsized":"2023-10-09T09:34:17.674495+0000","mapping_epoch":1099,"log_start":"1113'5100","ondisk_log_start":"1113'5100","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8616,"log_dups_size":0,"ondisk_log_size":8616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13716,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1028,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.841068+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:34:03.841068+0000","last_peered":"2023-10-09T09:34:03.841068+0000","last_clean":"2023-10-09T09:34:03.841068+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:34:03.841068+0000","last_undegraded":"2023-10-09T09:34:03.841068+0000","last_fullsized":"2023-10-09T09:34:03.841068+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1065,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.835166+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:34:03.835166+0000","last_peered":"2023-10-09T09:34:03.835166+0000","last_clean":"2023-10-09T09:34:03.835166+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:34:03.835166+0000","last_undegraded":"2023-10-09T09:34:03.835166+0000","last_fullsized":"2023-10-09T09:34:03.835166+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1126'9040","reported_seq":9089,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.648850+0000","last_change":"2023-10-09T09:31:32.026991+0000","last_active":"2023-10-09T09:34:17.648850+0000","last_peered":"2023-10-09T09:34:17.648850+0000","last_clean":"2023-10-09T09:34:17.648850+0000","last_became_active":"2023-10-09T09:31:28.828078+0000","last_became_peered":"2023-10-09T09:31:28.828078+0000","last_unstale":"2023-10-09T09:34:17.648850+0000","last_undegraded":"2023-10-09T09:34:17.648850+0000","last_fullsized":"2023-10-09T09:34:17.648850+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1106'4471","last_scrub_stamp":"2023-10-09T09:31:32.026835+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:32.026835+0000","objects_scrubbed":0,"log_size":9040,"log_dups_size":0,"ondisk_log_size":9040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4591,"num_write_kb":73216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1067,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:03.841437+0000","last_change":"2023-10-09T09:33:28.657795+0000","last_active":"2023-10-09T09:34:03.841437+0000","last_peered":"2023-10-09T09:34:03.841437+0000","last_clean":"2023-10-09T09:33:23.699474+0000","last_became_active":"2023-10-09T09:33:28.657795+0000","last_became_peered":"2023-10-09T09:33:28.657795+0000","last_unstale":"2023-10-09T09:34:03.841437+0000","last_undegraded":"2023-10-09T09:34:03.841437+0000","last_fullsized":"2023-10-09T09:33:28.640897+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:08.053644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1093,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.823976+0000","last_change":"2023-10-09T09:15:36.171699+0000","last_active":"2023-10-09T09:34:02.823976+0000","last_peered":"2023-10-09T09:34:02.823976+0000","last_clean":"2023-10-09T09:34:02.823976+0000","last_became_active":"2023-10-09T09:15:36.171198+0000","last_became_peered":"2023-10-09T09:15:36.171198+0000","last_unstale":"2023-10-09T09:34:02.823976+0000","last_undegraded":"2023-10-09T09:34:02.823976+0000","last_fullsized":"2023-10-09T09:34:02.823976+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1051,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.843146+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:34:03.843146+0000","last_peered":"2023-10-09T09:34:03.843146+0000","last_clean":"2023-10-09T09:34:03.843146+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:34:03.843146+0000","last_undegraded":"2023-10-09T09:34:03.843146+0000","last_fullsized":"2023-10-09T09:34:03.843146+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1126'8920","reported_seq":9005,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:17.601808+0000","last_change":"2023-10-09T09:34:17.298568+0000","last_active":"2023-10-09T09:34:17.601808+0000","last_peered":"2023-10-09T09:34:17.601808+0000","last_clean":"2023-10-09T09:33:27.629082+0000","last_became_active":"2023-10-09T09:33:28.649891+0000","last_became_peered":"2023-10-09T09:33:28.649891+0000","last_unstale":"2023-10-09T09:34:17.601808+0000","last_undegraded":"2023-10-09T09:34:17.601808+0000","last_fullsized":"2023-10-09T09:33:28.648596+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8920,"log_dups_size":0,"ondisk_log_size":8920,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6752,"num_write_kb":215744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1082,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.824105+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:34:02.824105+0000","last_peered":"2023-10-09T09:34:02.824105+0000","last_clean":"2023-10-09T09:34:02.824105+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:34:02.824105+0000","last_undegraded":"2023-10-09T09:34:02.824105+0000","last_fullsized":"2023-10-09T09:34:02.824105+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1083,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.838193+0000","last_change":"2023-10-09T09:33:28.664467+0000","last_active":"2023-10-09T09:34:02.838193+0000","last_peered":"2023-10-09T09:34:02.838193+0000","last_clean":"2023-10-09T09:33:16.682701+0000","last_became_active":"2023-10-09T09:33:28.664467+0000","last_became_peered":"2023-10-09T09:33:28.664467+0000","last_unstale":"2023-10-09T09:34:02.838193+0000","last_undegraded":"2023-10-09T09:34:02.838193+0000","last_fullsized":"2023-10-09T09:33:28.656612+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1081,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.840482+0000","last_change":"2023-10-09T09:15:36.173032+0000","last_active":"2023-10-09T09:34:03.840482+0000","last_peered":"2023-10-09T09:34:03.840482+0000","last_clean":"2023-10-09T09:34:03.840482+0000","last_became_active":"2023-10-09T09:15:36.172666+0000","last_became_peered":"2023-10-09T09:15:36.172666+0000","last_unstale":"2023-10-09T09:34:03.840482+0000","last_undegraded":"2023-10-09T09:34:03.840482+0000","last_fullsized":"2023-10-09T09:34:03.840482+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.570927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1126'8720","reported_seq":8776,"reported_epoch":1126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:17.648613+0000","last_change":"2023-10-09T09:33:28.659686+0000","last_active":"2023-10-09T09:34:17.648613+0000","last_peered":"2023-10-09T09:34:17.648613+0000","last_clean":"2023-10-09T09:33:27.572516+0000","last_became_active":"2023-10-09T09:33:28.659686+0000","last_became_peered":"2023-10-09T09:33:28.659686+0000","last_unstale":"2023-10-09T09:34:17.648613+0000","last_undegraded":"2023-10-09T09:33:28.655700+0000","last_fullsized":"2023-10-09T09:33:28.655561+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8720,"log_dups_size":0,"ondisk_log_size":8720,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1638400,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6593,"num_write_kb":211776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":25}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1054,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:03.836898+0000","last_change":"2023-10-09T09:33:28.637739+0000","last_active":"2023-10-09T09:34:03.836898+0000","last_peered":"2023-10-09T09:34:03.836898+0000","last_clean":"2023-10-09T09:33:16.667327+0000","last_became_active":"2023-10-09T09:33:28.637739+0000","last_became_peered":"2023-10-09T09:33:28.637739+0000","last_unstale":"2023-10-09T09:34:03.836898+0000","last_undegraded":"2023-10-09T09:34:03.836898+0000","last_fullsized":"2023-10-09T09:33:28.632965+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:45.063932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1066,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.834818+0000","last_change":"2023-10-09T09:15:07.206410+0000","last_active":"2023-10-09T09:34:03.834818+0000","last_peered":"2023-10-09T09:34:03.834818+0000","last_clean":"2023-10-09T09:34:03.834818+0000","last_became_active":"2023-10-09T09:15:07.205581+0000","last_became_peered":"2023-10-09T09:15:07.205581+0000","last_unstale":"2023-10-09T09:34:03.834818+0000","last_undegraded":"2023-10-09T09:34:03.834818+0000","last_fullsized":"2023-10-09T09:34:03.834818+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:25.053059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1062,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:08.820351+0000","last_change":"2023-10-09T09:33:28.655627+0000","last_active":"2023-10-09T09:34:08.820351+0000","last_peered":"2023-10-09T09:34:08.820351+0000","last_clean":"2023-10-09T09:33:16.688193+0000","last_became_active":"2023-10-09T09:33:28.655627+0000","last_became_peered":"2023-10-09T09:33:28.655627+0000","last_unstale":"2023-10-09T09:34:08.820351+0000","last_undegraded":"2023-10-09T09:34:08.820351+0000","last_fullsized":"2023-10-09T09:33:28.640252+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:29.860348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1126'8993","reported_seq":9046,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:17.666692+0000","last_change":"2023-10-09T09:34:15.765229+0000","last_active":"2023-10-09T09:34:17.666692+0000","last_peered":"2023-10-09T09:34:17.666692+0000","last_clean":"2023-10-09T09:33:27.539684+0000","last_became_active":"2023-10-09T09:33:28.650528+0000","last_became_peered":"2023-10-09T09:33:28.650528+0000","last_unstale":"2023-10-09T09:34:17.666692+0000","last_undegraded":"2023-10-09T09:34:17.666692+0000","last_fullsized":"2023-10-09T09:33:28.645431+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8993,"log_dups_size":0,"ondisk_log_size":8993,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6853,"num_write_kb":217824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1082,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.824330+0000","last_change":"2023-10-09T09:15:07.202257+0000","last_active":"2023-10-09T09:34:02.824330+0000","last_peered":"2023-10-09T09:34:02.824330+0000","last_clean":"2023-10-09T09:34:02.824330+0000","last_became_active":"2023-10-09T09:15:07.201896+0000","last_became_peered":"2023-10-09T09:15:07.201896+0000","last_unstale":"2023-10-09T09:34:02.824330+0000","last_undegraded":"2023-10-09T09:34:02.824330+0000","last_fullsized":"2023-10-09T09:34:02.824330+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1296,"reported_epoch":1125,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:02.828292+0000","last_change":"2023-10-09T09:33:28.660758+0000","last_active":"2023-10-09T09:34:02.828292+0000","last_peered":"2023-10-09T09:34:02.828292+0000","last_clean":"2023-10-09T09:33:16.684756+0000","last_became_active":"2023-10-09T09:33:28.660758+0000","last_became_peered":"2023-10-09T09:33:28.660758+0000","last_unstale":"2023-10-09T09:34:02.828292+0000","last_undegraded":"2023-10-09T09:33:28.648649+0000","last_fullsized":"2023-10-09T09:33:28.648536+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":587,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.850227+0000","last_change":"2023-10-09T09:34:03.850227+0000","last_active":"2023-10-09T09:34:03.850227+0000","last_peered":"2023-10-09T09:34:03.850227+0000","last_clean":"2023-10-09T09:34:03.850227+0000","last_became_active":"2023-10-09T09:34:03.849667+0000","last_became_peered":"2023-10-09T09:34:03.849667+0000","last_unstale":"2023-10-09T09:34:03.850227+0000","last_undegraded":"2023-10-09T09:34:03.850227+0000","last_fullsized":"2023-10-09T09:34:03.850227+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1043,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.829251+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:34:02.829251+0000","last_peered":"2023-10-09T09:34:02.829251+0000","last_clean":"2023-10-09T09:34:02.829251+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:34:02.829251+0000","last_undegraded":"2023-10-09T09:34:02.829251+0000","last_fullsized":"2023-10-09T09:34:02.829251+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1126'8853","reported_seq":8908,"reported_epoch":1126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:17.635614+0000","last_change":"2023-10-09T09:33:28.645138+0000","last_active":"2023-10-09T09:34:17.635614+0000","last_peered":"2023-10-09T09:34:17.635614+0000","last_clean":"2023-10-09T09:33:27.590725+0000","last_became_active":"2023-10-09T09:33:28.645138+0000","last_became_peered":"2023-10-09T09:33:28.645138+0000","last_unstale":"2023-10-09T09:34:17.635614+0000","last_undegraded":"2023-10-09T09:33:28.643548+0000","last_fullsized":"2023-10-09T09:33:28.643402+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1106'4346","last_scrub_stamp":"2023-10-09T09:31:29.845731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:29.845731+0000","objects_scrubbed":0,"log_size":8853,"log_dups_size":0,"ondisk_log_size":8853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4519,"num_write_kb":75328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1079,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.828096+0000","last_change":"2023-10-09T09:33:28.656505+0000","last_active":"2023-10-09T09:34:02.828096+0000","last_peered":"2023-10-09T09:34:02.828096+0000","last_clean":"2023-10-09T09:33:16.684645+0000","last_became_active":"2023-10-09T09:33:28.656505+0000","last_became_peered":"2023-10-09T09:33:28.656505+0000","last_unstale":"2023-10-09T09:34:02.828096+0000","last_undegraded":"2023-10-09T09:34:02.828096+0000","last_fullsized":"2023-10-09T09:33:28.648069+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":606,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.839651+0000","last_change":"2023-10-09T09:33:28.665626+0000","last_active":"2023-10-09T09:34:02.839651+0000","last_peered":"2023-10-09T09:34:02.839651+0000","last_clean":"2023-10-09T09:15:20.153276+0000","last_became_active":"2023-10-09T09:33:28.665626+0000","last_became_peered":"2023-10-09T09:33:28.665626+0000","last_unstale":"2023-10-09T09:34:02.839651+0000","last_undegraded":"2023-10-09T09:34:02.839651+0000","last_fullsized":"2023-10-09T09:33:28.656925+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":589,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.824553+0000","last_change":"2023-10-09T09:33:28.656555+0000","last_active":"2023-10-09T09:34:02.824553+0000","last_peered":"2023-10-09T09:34:02.824553+0000","last_clean":"2023-10-09T09:14:52.134630+0000","last_became_active":"2023-10-09T09:33:28.656555+0000","last_became_peered":"2023-10-09T09:33:28.656555+0000","last_unstale":"2023-10-09T09:34:02.824553+0000","last_undegraded":"2023-10-09T09:34:02.824553+0000","last_fullsized":"2023-10-09T09:33:28.642793+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.0","version":"1126'8944","reported_seq":8994,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.634097+0000","last_change":"2023-10-09T09:31:29.987166+0000","last_active":"2023-10-09T09:34:17.634097+0000","last_peered":"2023-10-09T09:34:17.634097+0000","last_clean":"2023-10-09T09:34:17.634097+0000","last_became_active":"2023-10-09T09:31:28.836386+0000","last_became_peered":"2023-10-09T09:31:28.836386+0000","last_unstale":"2023-10-09T09:34:17.634097+0000","last_undegraded":"2023-10-09T09:34:17.634097+0000","last_fullsized":"2023-10-09T09:34:17.634097+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"1106'4321","last_scrub_stamp":"2023-10-09T09:31:29.987092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:29.987092+0000","objects_scrubbed":0,"log_size":8944,"log_dups_size":0,"ondisk_log_size":8944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4642,"num_write_kb":79552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1064,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.830294+0000","last_change":"2023-10-09T09:33:28.686360+0000","last_active":"2023-10-09T09:34:02.830294+0000","last_peered":"2023-10-09T09:34:02.830294+0000","last_clean":"2023-10-09T09:33:16.685978+0000","last_became_active":"2023-10-09T09:33:28.686360+0000","last_became_peered":"2023-10-09T09:33:28.686360+0000","last_unstale":"2023-10-09T09:34:02.830294+0000","last_undegraded":"2023-10-09T09:34:02.830294+0000","last_fullsized":"2023-10-09T09:33:28.644174+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1067,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.841238+0000","last_change":"2023-10-09T09:15:07.207325+0000","last_active":"2023-10-09T09:34:03.841238+0000","last_peered":"2023-10-09T09:34:03.841238+0000","last_clean":"2023-10-09T09:34:03.841238+0000","last_became_active":"2023-10-09T09:15:07.207095+0000","last_became_peered":"2023-10-09T09:15:07.207095+0000","last_unstale":"2023-10-09T09:34:03.841238+0000","last_undegraded":"2023-10-09T09:34:03.841238+0000","last_fullsized":"2023-10-09T09:34:03.841238+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:09.357080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":512,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.824461+0000","last_change":"2023-10-09T09:33:28.655517+0000","last_active":"2023-10-09T09:34:02.824461+0000","last_peered":"2023-10-09T09:34:02.824461+0000","last_clean":"2023-10-09T09:03:24.275886+0000","last_became_active":"2023-10-09T09:33:28.655517+0000","last_became_peered":"2023-10-09T09:33:28.655517+0000","last_unstale":"2023-10-09T09:34:02.824461+0000","last_undegraded":"2023-10-09T09:34:02.824461+0000","last_fullsized":"2023-10-09T09:33:28.644187+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"1126'8992","reported_seq":9087,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.670625+0000","last_change":"2023-10-09T09:34:03.887613+0000","last_active":"2023-10-09T09:34:17.670625+0000","last_peered":"2023-10-09T09:34:17.670625+0000","last_clean":"2023-10-09T09:34:17.670625+0000","last_became_active":"2023-10-09T09:34:03.856020+0000","last_became_peered":"2023-10-09T09:34:03.856020+0000","last_unstale":"2023-10-09T09:34:17.670625+0000","last_undegraded":"2023-10-09T09:34:17.670625+0000","last_fullsized":"2023-10-09T09:34:17.670625+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"1123'7919","last_scrub_stamp":"2023-10-09T09:33:41.242111+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:33:41.242111+0000","objects_scrubbed":1086,"log_size":8992,"log_dups_size":0,"ondisk_log_size":8992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:41.770867+0000","scrub_duration":0.52443578700000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1086,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":39,"reported_epoch":1126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:03.851679+0000","last_change":"2023-10-09T09:34:03.851556+0000","last_active":"2023-10-09T09:34:03.851679+0000","last_peered":"2023-10-09T09:34:03.851679+0000","last_clean":"2023-10-09T09:03:26.327447+0000","last_became_active":"2023-10-09T09:34:03.851556+0000","last_became_peered":"2023-10-09T09:34:03.851556+0000","last_unstale":"2023-10-09T09:34:03.851679+0000","last_undegraded":"2023-10-09T09:34:03.846210+0000","last_fullsized":"2023-10-09T09:34:03.846089+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1050,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:08.820306+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:34:08.820306+0000","last_peered":"2023-10-09T09:34:08.820306+0000","last_clean":"2023-10-09T09:34:08.820306+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:34:08.820306+0000","last_undegraded":"2023-10-09T09:34:08.820306+0000","last_fullsized":"2023-10-09T09:34:08.820306+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1046,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:03.850838+0000","last_change":"2023-10-09T09:34:03.850716+0000","last_active":"2023-10-09T09:34:03.850838+0000","last_peered":"2023-10-09T09:34:03.850838+0000","last_clean":"2023-10-09T09:33:16.680955+0000","last_became_active":"2023-10-09T09:34:03.850716+0000","last_became_peered":"2023-10-09T09:34:03.850716+0000","last_unstale":"2023-10-09T09:34:03.850838+0000","last_undegraded":"2023-10-09T09:34:03.850838+0000","last_fullsized":"2023-10-09T09:34:03.845663+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"1126'8808","reported_seq":8927,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.522999+0000","last_change":"2023-10-09T09:34:03.974931+0000","last_active":"2023-10-09T09:34:17.522999+0000","last_peered":"2023-10-09T09:34:17.522999+0000","last_clean":"2023-10-09T09:34:17.522999+0000","last_became_active":"2023-10-09T09:34:03.905612+0000","last_became_peered":"2023-10-09T09:34:03.905612+0000","last_unstale":"2023-10-09T09:34:17.522999+0000","last_undegraded":"2023-10-09T09:34:17.522999+0000","last_fullsized":"2023-10-09T09:34:17.522999+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"1109'4470","last_scrub_stamp":"2023-10-09T09:31:37.173935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:37.173935+0000","objects_scrubbed":2369,"log_size":8808,"log_dups_size":0,"ondisk_log_size":8808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:56.858013+0000","scrub_duration":0.64874551999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4349,"num_write_kb":63360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1062,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.826992+0000","last_change":"2023-10-09T09:15:07.195197+0000","last_active":"2023-10-09T09:34:02.826992+0000","last_peered":"2023-10-09T09:34:02.826992+0000","last_clean":"2023-10-09T09:34:02.826992+0000","last_became_active":"2023-10-09T09:15:07.194753+0000","last_became_peered":"2023-10-09T09:15:07.194753+0000","last_unstale":"2023-10-09T09:34:02.826992+0000","last_undegraded":"2023-10-09T09:34:02.826992+0000","last_fullsized":"2023-10-09T09:34:02.826992+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":44,"reported_epoch":1126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:03.835381+0000","last_change":"2023-10-09T09:33:28.659097+0000","last_active":"2023-10-09T09:34:03.835381+0000","last_peered":"2023-10-09T09:34:03.835381+0000","last_clean":"2023-10-09T09:03:26.323453+0000","last_became_active":"2023-10-09T09:33:28.659097+0000","last_became_peered":"2023-10-09T09:33:28.659097+0000","last_unstale":"2023-10-09T09:34:03.835381+0000","last_undegraded":"2023-10-09T09:33:28.640037+0000","last_fullsized":"2023-10-09T09:33:28.639909+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:29.940228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1061,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.839228+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:34:02.839228+0000","last_peered":"2023-10-09T09:34:02.839228+0000","last_clean":"2023-10-09T09:34:02.839228+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:34:02.839228+0000","last_undegraded":"2023-10-09T09:34:02.839228+0000","last_fullsized":"2023-10-09T09:34:02.839228+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":32,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.838564+0000","last_change":"2023-10-09T09:33:28.666420+0000","last_active":"2023-10-09T09:34:02.838564+0000","last_peered":"2023-10-09T09:34:02.838564+0000","last_clean":"2023-10-09T09:03:23.262803+0000","last_became_active":"2023-10-09T09:33:28.666420+0000","last_became_peered":"2023-10-09T09:33:28.666420+0000","last_unstale":"2023-10-09T09:34:02.838564+0000","last_undegraded":"2023-10-09T09:34:02.838564+0000","last_fullsized":"2023-10-09T09:33:28.656124+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1065,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.837023+0000","last_change":"2023-10-09T09:15:36.161773+0000","last_active":"2023-10-09T09:34:03.837023+0000","last_peered":"2023-10-09T09:34:03.837023+0000","last_clean":"2023-10-09T09:34:03.837023+0000","last_became_active":"2023-10-09T09:15:36.161436+0000","last_became_peered":"2023-10-09T09:15:36.161436+0000","last_unstale":"2023-10-09T09:34:03.837023+0000","last_undegraded":"2023-10-09T09:34:03.837023+0000","last_fullsized":"2023-10-09T09:34:03.837023+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:32.531105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"1126'9145","reported_seq":9257,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:17.611017+0000","last_change":"2023-10-09T09:31:33.829441+0000","last_active":"2023-10-09T09:34:17.611017+0000","last_peered":"2023-10-09T09:34:17.611017+0000","last_clean":"2023-10-09T09:34:17.611017+0000","last_became_active":"2023-10-09T09:31:28.863946+0000","last_became_peered":"2023-10-09T09:31:28.863946+0000","last_unstale":"2023-10-09T09:34:17.611017+0000","last_undegraded":"2023-10-09T09:34:17.611017+0000","last_fullsized":"2023-10-09T09:34:17.611017+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"1106'4655","last_scrub_stamp":"2023-10-09T09:31:33.829271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:31:33.829271+0000","objects_scrubbed":2411,"log_size":9145,"log_dups_size":0,"ondisk_log_size":9145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:23.193885+0000","scrub_duration":0.70828463799999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4497,"num_write_kb":66752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1080,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.823928+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:34:02.823928+0000","last_peered":"2023-10-09T09:34:02.823928+0000","last_clean":"2023-10-09T09:34:02.823928+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:34:02.823928+0000","last_undegraded":"2023-10-09T09:34:02.823928+0000","last_fullsized":"2023-10-09T09:34:02.823928+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1091,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.838512+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:34:02.838512+0000","last_peered":"2023-10-09T09:34:02.838512+0000","last_clean":"2023-10-09T09:34:02.838512+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:34:02.838512+0000","last_undegraded":"2023-10-09T09:34:02.838512+0000","last_fullsized":"2023-10-09T09:34:02.838512+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"1126'8873","reported_seq":8924,"reported_epoch":1126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:17.670125+0000","last_change":"2023-10-09T09:33:28.667062+0000","last_active":"2023-10-09T09:34:17.670125+0000","last_peered":"2023-10-09T09:34:17.670125+0000","last_clean":"2023-10-09T09:33:27.604123+0000","last_became_active":"2023-10-09T09:33:28.667062+0000","last_became_peered":"2023-10-09T09:33:28.667062+0000","last_unstale":"2023-10-09T09:34:17.670125+0000","last_undegraded":"2023-10-09T09:33:28.638994+0000","last_fullsized":"2023-10-09T09:33:28.638845+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1105,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:30:24.809353+0000","last_clean_scrub_stamp":"2023-10-09T09:30:24.809353+0000","objects_scrubbed":0,"log_size":8873,"log_dups_size":0,"ondisk_log_size":8873,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1572864,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6748,"num_write_kb":216704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":24}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1065,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.824117+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:34:02.824117+0000","last_peered":"2023-10-09T09:34:02.824117+0000","last_clean":"2023-10-09T09:34:02.824117+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:34:02.824117+0000","last_undegraded":"2023-10-09T09:34:02.824117+0000","last_fullsized":"2023-10-09T09:34:02.824117+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1097,"reported_epoch":1125,"state":"active+undersized","last_fresh":"2023-10-09T09:34:02.837391+0000","last_change":"2023-10-09T09:33:28.660658+0000","last_active":"2023-10-09T09:34:02.837391+0000","last_peered":"2023-10-09T09:34:02.837391+0000","last_clean":"2023-10-09T09:33:16.682402+0000","last_became_active":"2023-10-09T09:33:28.660658+0000","last_became_peered":"2023-10-09T09:33:28.660658+0000","last_unstale":"2023-10-09T09:34:02.837391+0000","last_undegraded":"2023-10-09T09:34:02.837391+0000","last_fullsized":"2023-10-09T09:33:28.656667+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":32,"reported_epoch":1126,"state":"active+clean","last_fresh":"2023-10-09T09:34:03.852043+0000","last_change":"2023-10-09T09:34:03.852043+0000","last_active":"2023-10-09T09:34:03.852043+0000","last_peered":"2023-10-09T09:34:03.852043+0000","last_clean":"2023-10-09T09:34:03.852043+0000","last_became_active":"2023-10-09T09:34:03.851598+0000","last_became_peered":"2023-10-09T09:34:03.851598+0000","last_unstale":"2023-10-09T09:34:03.852043+0000","last_undegraded":"2023-10-09T09:34:03.852043+0000","last_fullsized":"2023-10-09T09:34:03.852043+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1072,"reported_epoch":1126,"state":"active+undersized","last_fresh":"2023-10-09T09:34:05.429987+0000","last_change":"2023-10-09T09:34:03.851280+0000","last_active":"2023-10-09T09:34:05.429987+0000","last_peered":"2023-10-09T09:34:05.429987+0000","last_clean":"2023-10-09T09:33:16.682496+0000","last_became_active":"2023-10-09T09:34:03.851280+0000","last_became_peered":"2023-10-09T09:34:03.851280+0000","last_unstale":"2023-10-09T09:34:05.429987+0000","last_undegraded":"2023-10-09T09:34:05.429987+0000","last_fullsized":"2023-10-09T09:34:03.836898+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:21.588575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":43,"reported_epoch":1125,"state":"active+undersized+degraded","last_fresh":"2023-10-09T09:34:02.824114+0000","last_change":"2023-10-09T09:33:28.655442+0000","last_active":"2023-10-09T09:34:02.824114+0000","last_peered":"2023-10-09T09:34:02.824114+0000","last_clean":"2023-10-09T09:03:26.323629+0000","last_became_active":"2023-10-09T09:33:28.655442+0000","last_became_peered":"2023-10-09T09:33:28.655442+0000","last_unstale":"2023-10-09T09:34:02.824114+0000","last_undegraded":"2023-10-09T09:33:28.645345+0000","last_fullsized":"2023-10-09T09:33:28.645200+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1076,"reported_epoch":1125,"state":"active+clean","last_fresh":"2023-10-09T09:34:02.837117+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:34:02.837117+0000","last_peered":"2023-10-09T09:34:02.837117+0000","last_clean":"2023-10-09T09:34:02.837117+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:34:02.837117+0000","last_undegraded":"2023-10-09T09:34:02.837117+0000","last_fullsized":"2023-10-09T09:34:02.837117+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":6914048,"num_objects":106,"num_object_clones":0,"num_object_copies":318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":106,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170257,"num_write_kb":4772801,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1664356352,"data_stored":1664352280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":227109,"ondisk_log_size":227109,"up":64,"acting":65,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":87,"acting":87,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":84,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":2,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397911,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302080,"kb_used_data":4856,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93443072,"statfs":{"total":95995035648,"available":95685705728,"internally_reserved":0,"allocated":4972544,"data_stored":4746234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4359999999999999,"5min":2.9369999999999998,"15min":1.988},"min":{"1min":0.35499999999999998,"5min":0.249,"15min":0.249},"max":{"1min":4.3650000000000002,"5min":85.811999999999998,"15min":85.811999999999998},"last":4.9729999999999999},{"interface":"front","average":{"1min":1.496,"5min":3.077,"15min":2.117},"min":{"1min":0.47299999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.4199999999999999,"5min":85.762,"15min":85.762},"last":5.0190000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:33:21 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.3819999999999999,"15min":1.579},"min":{"1min":0.29299999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.2830000000000004,"5min":8.2449999999999992,"15min":103.84699999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.272,"15min":1.627},"min":{"1min":0.34999999999999998,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":4.8559999999999999,"5min":6.6029999999999998,"15min":103.955},"last":1.0640000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:33:35 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":1.9990000000000001,"15min":1.722},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.2489999999999997,"5min":47.023000000000003,"15min":47.023000000000003},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.837,"5min":2.1309999999999998,"15min":1.633},"min":{"1min":0.42799999999999999,"5min":0.375,"15min":0.373},"max":{"1min":6.2590000000000003,"5min":46.847000000000001,"15min":46.847000000000001},"last":0.88500000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:34:20 2023","interfaces":[{"interface":"back","average":{"1min":3.839,"5min":3.472,"15min":2.7770000000000001},"min":{"1min":0.377,"5min":0.35399999999999998,"15min":0.29999999999999999},"max":{"1min":51.332000000000001,"5min":84.590999999999994,"15min":107.97499999999999},"last":0.80700000000000005},{"interface":"front","average":{"1min":4.0659999999999998,"5min":3.5990000000000002,"15min":2.883},"min":{"1min":0.377,"5min":0.377,"15min":0.317},"max":{"1min":53.774999999999999,"5min":84.713999999999999,"15min":105.282},"last":0.90200000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:33:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":1.5529999999999999,"15min":1.212},"min":{"1min":0.38,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.7789999999999999,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0269999999999999,"15min":1.0640000000000001},"min":{"1min":0.39100000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.6210000000000004,"5min":6.6210000000000004,"15min":12.316000000000001},"last":0.74199999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:33:47 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.2170000000000001,"15min":1.2929999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.0839999999999996,"5min":4.431,"15min":10.063000000000001},"last":1.022},{"interface":"front","average":{"1min":2.0329999999999999,"5min":1.411,"15min":1.2230000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":5.8099999999999996,"5min":8.1699999999999999,"15min":8.1699999999999999},"last":0.77600000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:33:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":1.3979999999999999,"15min":1.234},"min":{"1min":0.52200000000000002,"5min":0.38600000000000001,"15min":0.373},"max":{"1min":4.8140000000000001,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.2629999999999999,"15min":1.304},"min":{"1min":0.40799999999999997,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.4699999999999998,"5min":6.4699999999999998,"15min":8.1980000000000004},"last":0.80000000000000004}]}]},{"osd":6,"up_from":43,"seq":184683594144,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306164,"kb_used_data":4844,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93438988,"statfs":{"total":95995035648,"available":95681523712,"internally_reserved":0,"allocated":4960256,"data_stored":4746088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.5389999999999999,"15min":1.1699999999999999},"min":{"1min":0.33200000000000002,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":7.1100000000000003,"5min":16.850000000000001,"15min":16.850000000000001},"last":2.508},{"interface":"front","average":{"1min":1.5680000000000001,"5min":1.3260000000000001,"15min":1.113},"min":{"1min":0.34999999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.234,"5min":8.234,"15min":8.234},"last":1.905}]},{"osd":1,"last update":"Mon Oct 9 09:34:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4620000000000002,"5min":3.3119999999999998,"15min":2.0339999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.631,"5min":103.259,"15min":103.259},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.2529999999999999,"5min":2.9510000000000001,"15min":1.863},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29799999999999999},"max":{"1min":7.1920000000000002,"5min":102.005,"15min":102.005},"last":0.70799999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:33:50 2023","interfaces":[{"interface":"back","average":{"1min":3.5209999999999999,"5min":2.2839999999999998,"15min":1.4930000000000001},"min":{"1min":0.32400000000000001,"5min":0.314,"15min":0.314},"max":{"1min":33.210000000000001,"5min":33.210000000000001,"15min":33.210000000000001},"last":0.998},{"interface":"front","average":{"1min":3.7069999999999999,"5min":2.0800000000000001,"15min":1.5669999999999999},"min":{"1min":0.43099999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":37.246000000000002,"5min":37.246000000000002,"15min":37.246000000000002},"last":1.0620000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:34:02 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":3.6549999999999998,"15min":2.157},"min":{"1min":0.35299999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.497,"5min":123.97199999999999,"15min":123.97199999999999},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.996,"5min":3.6190000000000002,"15min":2.1560000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.8630000000000004,"5min":124.059,"15min":124.059},"last":0.874}]},{"osd":4,"last update":"Mon Oct 9 09:34:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":0.94999999999999996,"15min":1.0369999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.312},"max":{"1min":8.6890000000000001,"5min":8.6890000000000001,"15min":9.1099999999999994},"last":0.75800000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.4450000000000001,"15min":1.206},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":5.508,"5min":9.7919999999999998,"15min":9.7919999999999998},"last":0.79800000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:34:02 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.1120000000000001,"15min":1.125},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":8.5649999999999995,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.89500000000000002},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.4079999999999999,"15min":1.232},"min":{"1min":0.37,"5min":0.37,"15min":0.35999999999999999},"max":{"1min":5.5730000000000004,"5min":5.8869999999999996,"15min":8.5950000000000006},"last":0.93200000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:33:32 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.3839999999999999,"15min":1.3460000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":4.1779999999999999,"5min":7.6769999999999996,"15min":7.6769999999999996},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.71,"5min":1.464,"15min":1.3620000000000001},"min":{"1min":0.58599999999999997,"5min":0.40600000000000003,"15min":0.371},"max":{"1min":5.6699999999999999,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.96799999999999997}]}]},{"osd":2,"up_from":634,"seq":2723009265898,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":321504,"kb_used_data":5400,"kb_used_omap":122,"kb_used_meta":315973,"kb_avail":93423648,"statfs":{"total":95995035648,"available":95665815552,"internally_reserved":0,"allocated":5529600,"data_stored":4353790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":323556939},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:32:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5310000000000001,"5min":1.98,"15min":1.3049999999999999},"min":{"1min":0.216,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":19.588000000000001,"5min":19.588000000000001,"15min":19.588000000000001},"last":0.873},{"interface":"front","average":{"1min":2.5979999999999999,"5min":2.129,"15min":1.385},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":18.785,"5min":18.785,"15min":18.785},"last":3.1160000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:33:56 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.6599999999999999,"15min":1.2430000000000001},"min":{"1min":0.26100000000000001,"5min":0.20200000000000001,"15min":0.184},"max":{"1min":9.6389999999999993,"5min":9.6389999999999993,"15min":9.6389999999999993},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.9390000000000001,"5min":1.7430000000000001,"15min":1.4690000000000001},"min":{"1min":0.26800000000000002,"5min":0.26300000000000001,"15min":0.216},"max":{"1min":7.5910000000000002,"5min":8.1329999999999991,"15min":8.702},"last":0.34000000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:34:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":2.165,"15min":1.577},"min":{"1min":0.255,"5min":0.252,"15min":0.252},"max":{"1min":9.6140000000000008,"5min":18.547999999999998,"15min":18.547999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.3380000000000001,"5min":2.262,"15min":1.6659999999999999},"min":{"1min":0.40000000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":13.23,"5min":19.527999999999999,"15min":19.527999999999999},"last":0.38100000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:33:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":1.4319999999999999,"15min":1.3460000000000001},"min":{"1min":0.44900000000000001,"5min":0.372,"15min":0.309},"max":{"1min":7.6280000000000001,"5min":8.0370000000000008,"15min":8.0370000000000008},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.458,"5min":1.659,"15min":1.3959999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32200000000000001},"max":{"1min":6.7709999999999999,"5min":9.6300000000000008,"15min":13.917999999999999},"last":0.504}]},{"osd":5,"last update":"Mon Oct 9 09:33:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":1.524,"15min":1.276},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.5310000000000006,"5min":9.5310000000000006,"15min":9.5310000000000006},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.5009999999999999,"15min":1.411},"min":{"1min":0.37,"5min":0.37,"15min":0.36499999999999999},"max":{"1min":7.7329999999999997,"5min":8.1080000000000005,"15min":10.393000000000001},"last":0.70599999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:33:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":1.6739999999999999,"15min":1.4770000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33100000000000002},"max":{"1min":7.7009999999999996,"5min":8.0809999999999995,"15min":8.0809999999999995},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.919,"5min":1.9590000000000001,"15min":1.5940000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.8259999999999996,"5min":10.064,"15min":13.893000000000001},"last":0.69299999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:33:56 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":2.1299999999999999,"15min":1.6859999999999999},"min":{"1min":0.45600000000000002,"5min":0.41099999999999998,"15min":0.377},"max":{"1min":6.7370000000000001,"5min":10.18,"15min":13.848000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.7349999999999999,"5min":1.9330000000000001,"15min":1.508},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.34499999999999997},"max":{"1min":9.5790000000000006,"5min":10.199,"15min":10.199},"last":0.56799999999999995}]}]},{"osd":1,"up_from":607,"seq":2607045148911,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":273984,"kb_used_data":5368,"kb_used_omap":107,"kb_used_meta":268500,"kb_avail":93471168,"statfs":{"total":95995035648,"available":95714476032,"internally_reserved":0,"allocated":5496832,"data_stored":4347909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":274944683},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:33:15 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":2.734,"15min":1.415},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":8.4730000000000008,"5min":84.421000000000006,"15min":84.421000000000006},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.3380000000000001,"5min":2.5819999999999999,"15min":1.4910000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.8620000000000001,"5min":84.489999999999995,"15min":84.489999999999995},"last":0.45400000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:33:42 2023","interfaces":[{"interface":"back","average":{"1min":6.4000000000000004,"5min":3.508,"15min":2.0310000000000001},"min":{"1min":0.23899999999999999,"5min":0.224,"15min":0.20100000000000001},"max":{"1min":69.421999999999997,"5min":84.578999999999994,"15min":84.578999999999994},"last":0.41599999999999998},{"interface":"front","average":{"1min":5.3840000000000003,"5min":3.1920000000000002,"15min":1.77},"min":{"1min":0.32800000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":71.879000000000005,"5min":84.697000000000003,"15min":84.697000000000003},"last":0.47399999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:34:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":2.6840000000000002,"15min":2.7949999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":13.004,"5min":73.564999999999998,"15min":107.426},"last":0.45700000000000002},{"interface":"front","average":{"1min":3.641,"5min":3.0840000000000001,"15min":3.1110000000000002},"min":{"1min":0.309,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":17.643999999999998,"5min":72.932000000000002,"15min":106.15900000000001},"last":0.504}]},{"osd":4,"last update":"Mon Oct 9 09:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":1.1359999999999999,"15min":1.4259999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26500000000000001},"max":{"1min":4.585,"5min":7.3810000000000002,"15min":84.875},"last":0.68500000000000005},{"interface":"front","average":{"1min":3.1859999999999999,"5min":1.583,"15min":1.617},"min":{"1min":0.33100000000000002,"5min":0.31,"15min":0.29499999999999998},"max":{"1min":32.151000000000003,"5min":32.151000000000003,"15min":84.992999999999995},"last":0.60799999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.1890000000000001,"15min":1.4790000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.5389999999999997,"5min":7.4260000000000002,"15min":84.805000000000007},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.351,"15min":1.5920000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":4.266,"5min":8.0459999999999994,"15min":84.951999999999998},"last":0.77300000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:34:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5659999999999998,"5min":1.7290000000000001,"15min":1.7849999999999999},"min":{"1min":0.47799999999999998,"5min":0.373,"15min":0.373},"max":{"1min":7.6509999999999998,"5min":7.6509999999999998,"15min":85.016000000000005},"last":0.628},{"interface":"front","average":{"1min":1.861,"5min":1.4179999999999999,"15min":1.5780000000000001},"min":{"1min":0.378,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.6609999999999996,"5min":7.4050000000000002,"15min":84.831999999999994},"last":0.82999999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:34:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":1.7490000000000001,"15min":1.9650000000000001},"min":{"1min":0.438,"5min":0.33400000000000002,"15min":0.33100000000000002},"max":{"1min":5.8019999999999996,"5min":9.7970000000000006,"15min":84.888000000000005},"last":0.75600000000000001},{"interface":"front","average":{"1min":3.8450000000000002,"5min":1.784,"15min":1.7809999999999999},"min":{"1min":0.44500000000000001,"5min":0.39600000000000002,"15min":0.36699999999999999},"max":{"1min":32.078000000000003,"5min":32.078000000000003,"15min":84.742000000000004},"last":0.79500000000000004}]}]},{"osd":0,"up_from":581,"seq":2495375999209,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1920848,"kb_used_data":1602824,"kb_used_omap":131,"kb_used_meta":317884,"kb_avail":91824304,"statfs":{"total":95995035648,"available":94028087296,"internally_reserved":0,"allocated":1641291776,"data_stored":1640191295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134567,"internal_metadata":325513817},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:33:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2999999999999998,"5min":1.9179999999999999,"15min":1.302},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.3369999999999997,"5min":8.3369999999999997,"15min":14.44},"last":7.4859999999999998},{"interface":"front","average":{"1min":1.883,"5min":1.923,"15min":1.367},"min":{"1min":0.34899999999999998,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":4.9550000000000001,"5min":8.8870000000000005,"15min":14.663},"last":3.2959999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:32:46 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":2.9300000000000002,"15min":1.6779999999999999},"min":{"1min":0.33900000000000002,"5min":0.222,"15min":0.17799999999999999},"max":{"1min":5.5069999999999997,"5min":60.151000000000003,"15min":60.151000000000003},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.49,"5min":2.2709999999999999,"15min":1.4079999999999999},"min":{"1min":0.41099999999999998,"5min":0.249,"15min":0.20999999999999999},"max":{"1min":3.5369999999999999,"5min":59.253,"15min":59.253},"last":7.5279999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:33:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":2.2410000000000001,"15min":1.883},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.3239999999999998,"5min":9.7469999999999999,"15min":122.749},"last":7.6989999999999998},{"interface":"front","average":{"1min":3.419,"5min":2.524,"15min":1.994},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":13.24,"5min":14.185,"15min":123.51900000000001},"last":0.73499999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":1.407,"15min":1.216},"min":{"1min":0.35299999999999998,"5min":0.313,"15min":0.312},"max":{"1min":4.6929999999999996,"5min":6.5119999999999996,"15min":7.093},"last":7.3970000000000002},{"interface":"front","average":{"1min":1.498,"5min":1.6599999999999999,"15min":1.3029999999999999},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.28199999999999997},"max":{"1min":6.7859999999999996,"5min":9.4169999999999998,"15min":9.4169999999999998},"last":0.39300000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.4570000000000001,"15min":1.2470000000000001},"min":{"1min":0.42199999999999999,"5min":0.33900000000000002,"15min":0.29299999999999998},"max":{"1min":4.7759999999999998,"5min":5.6710000000000003,"15min":7.3620000000000001},"last":7.5869999999999997},{"interface":"front","average":{"1min":1.6120000000000001,"5min":1.756,"15min":1.393},"min":{"1min":0.41799999999999998,"5min":0.32400000000000001,"15min":0.312},"max":{"1min":6.8330000000000002,"5min":9.3670000000000009,"15min":9.3670000000000009},"last":0.35899999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":1.726,"15min":1.4299999999999999},"min":{"1min":0.40200000000000002,"5min":0.39300000000000002,"15min":0.32800000000000001},"max":{"1min":5.3300000000000001,"5min":7.8250000000000002,"15min":8.202},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.538,"5min":1.605,"15min":1.353},"min":{"1min":0.41399999999999998,"5min":0.36299999999999999,"15min":0.34599999999999997},"max":{"1min":4.7439999999999998,"5min":6.6959999999999997,"15min":7.3289999999999997},"last":7.6369999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.7170000000000001,"15min":1.4690000000000001},"min":{"1min":0.441,"5min":0.372,"15min":0.372},"max":{"1min":5.3029999999999999,"5min":7.798,"15min":8.1769999999999996},"last":0.878},{"interface":"front","average":{"1min":1.522,"5min":1.77,"15min":1.4630000000000001},"min":{"1min":0.47299999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.835,"5min":10.571,"15min":10.571},"last":7.6139999999999999}]}]},{"osd":3,"up_from":663,"seq":2847563317476,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":427528,"kb_used_data":60992,"kb_used_omap":55,"kb_used_meta":366472,"kb_avail":93317624,"statfs":{"total":95995035648,"available":95557246976,"internally_reserved":0,"allocated":62455808,"data_stored":61407904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":375267938},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:33:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1139999999999999,"5min":2.0600000000000001,"15min":1.375},"min":{"1min":0.23100000000000001,"5min":0.191,"15min":0.191},"max":{"1min":20.469999999999999,"5min":20.469999999999999,"15min":20.469999999999999},"last":0.93100000000000005},{"interface":"front","average":{"1min":3.847,"5min":2.4420000000000002,"15min":1.5269999999999999},"min":{"1min":0.39800000000000002,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":23.437999999999999,"5min":23.437999999999999,"15min":23.437999999999999},"last":1.214}]},{"osd":1,"last update":"Mon Oct 9 09:34:21 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":2.4119999999999999,"15min":1.7490000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":4.5030000000000001,"5min":91.159000000000006,"15min":91.159000000000006},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.1579999999999999,"5min":2.778,"15min":1.8280000000000001},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.7380000000000004,"5min":92.683000000000007,"15min":92.683000000000007},"last":0.46200000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:34:21 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":3.2330000000000001,"15min":2.0209999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.4569999999999999,"5min":106.122,"15min":106.122},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.587,"5min":2.6779999999999999,"15min":1.905},"min":{"1min":0.25,"5min":0.25,"15min":0.223},"max":{"1min":7.391,"5min":104.102,"15min":104.102},"last":0.52100000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:34:21 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.4750000000000001,"15min":1.417},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.7690000000000001,"5min":6.6840000000000002,"15min":12.629},"last":1.7150000000000001},{"interface":"front","average":{"1min":1.6559999999999999,"5min":1.4630000000000001,"15min":1.3520000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.27500000000000002},"max":{"1min":6.9569999999999999,"5min":7.9210000000000003,"15min":7.9210000000000003},"last":1.304}]},{"osd":5,"last update":"Mon Oct 9 09:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":1.7989999999999999,"15min":1.6140000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.315},"max":{"1min":7.8200000000000003,"5min":9.6370000000000005,"15min":11.272},"last":1.0009999999999999},{"interface":"front","average":{"1min":1.625,"5min":1.6519999999999999,"15min":1.5449999999999999},"min":{"1min":0.36499999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.1559999999999997,"5min":9.6890000000000001,"15min":11.388999999999999},"last":1.694}]},{"osd":6,"last update":"Mon Oct 9 09:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3159999999999998,"5min":1.7010000000000001,"15min":1.5529999999999999},"min":{"1min":0.38300000000000001,"5min":0.34399999999999997,"15min":0.30099999999999999},"max":{"1min":6.9240000000000004,"5min":8.7040000000000006,"15min":8.7040000000000006},"last":1.2629999999999999},{"interface":"front","average":{"1min":1.601,"5min":1.635,"15min":1.5780000000000001},"min":{"1min":0.42399999999999999,"5min":0.316,"15min":0.316},"max":{"1min":4.181,"5min":9.2910000000000004,"15min":17.234999999999999},"last":1.6559999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8050000000000002,"5min":1.792,"15min":1.653},"min":{"1min":0.36599999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.029999999999999,"5min":10.029999999999999,"15min":10.029999999999999},"last":0.90900000000000003},{"interface":"front","average":{"1min":2.1760000000000002,"5min":1.7090000000000001,"15min":1.5820000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":7.0999999999999996,"5min":8.6210000000000004,"15min":8.6210000000000004},"last":1.208}]}]},{"osd":4,"up_from":31,"seq":133143986609,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313784,"kb_used_data":4272,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93431368,"statfs":{"total":95995035648,"available":95673720832,"internally_reserved":0,"allocated":4374528,"data_stored":4155796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:32:44 2023","interfaces":[{"interface":"back","average":{"1min":4.2859999999999996,"5min":2.8090000000000002,"15min":1.552},"min":{"1min":0.307,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":51.237000000000002,"5min":94.489999999999995,"15min":94.489999999999995},"last":0.65300000000000002},{"interface":"front","average":{"1min":4.569,"5min":2.6989999999999998,"15min":1.5900000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":47.134,"5min":94.802999999999997,"15min":94.802999999999997},"last":0.432}]},{"osd":1,"last update":"Mon Oct 9 09:33:23 2023","interfaces":[{"interface":"back","average":{"1min":6.5309999999999997,"5min":2.633,"15min":1.5960000000000001},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":85.186999999999998,"5min":85.186999999999998,"15min":85.186999999999998},"last":0.94999999999999996},{"interface":"front","average":{"1min":7.0960000000000001,"5min":2.8570000000000002,"15min":1.6499999999999999},"min":{"1min":0.37,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":85.159000000000006,"5min":85.159000000000006,"15min":85.159000000000006},"last":0.81799999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:33:35 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":2.4159999999999999,"15min":1.579},"min":{"1min":0.39300000000000002,"5min":0.33600000000000002,"15min":0.31900000000000001},"max":{"1min":6.21,"5min":99.736000000000004,"15min":99.736000000000004},"last":0.86399999999999999},{"interface":"front","average":{"1min":2.2490000000000001,"5min":2.343,"15min":1.526},"min":{"1min":0.46899999999999997,"5min":0.32500000000000001,"15min":0.317},"max":{"1min":6.2229999999999999,"5min":99.212999999999994,"15min":99.212999999999994},"last":0.92300000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:33:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":2.9750000000000001,"15min":1.9039999999999999},"min":{"1min":0.40699999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.8369999999999997,"5min":47.32,"15min":47.32},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.4079999999999999,"5min":3.823,"15min":2.2639999999999998},"min":{"1min":0.40600000000000003,"5min":0.314,"15min":0.314},"max":{"1min":2.9300000000000002,"5min":93.843000000000004,"15min":93.843000000000004},"last":1.1539999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:33:23 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.2350000000000001,"15min":1.151},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.7719999999999998,"5min":9.2959999999999994,"15min":9.2959999999999994},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.1539999999999999,"15min":1.2410000000000001},"min":{"1min":0.38900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.806,"5min":6.2000000000000002,"15min":7.5819999999999999},"last":1.0429999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:33:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.262,"15min":1.1819999999999999},"min":{"1min":0.374,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.2569999999999997,"5min":7.1639999999999997,"15min":7.625},"last":1.0860000000000001},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.1870000000000001,"15min":1.1890000000000001},"min":{"1min":0.40500000000000003,"5min":0.40000000000000002,"15min":0.371},"max":{"1min":4.7409999999999997,"5min":7.1440000000000001,"15min":7.1440000000000001},"last":1.0589999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:33:43 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":1.0369999999999999,"15min":1.149},"min":{"1min":0.42399999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.032,"5min":7.5449999999999999,"15min":9.2560000000000002},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.171,"15min":1.3149999999999999},"min":{"1min":0.41099999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":6.3239999999999998,"5min":6.3239999999999998,"15min":9.1780000000000008},"last":1.111}]}]},{"osd":5,"up_from":37,"seq":158913790375,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305576,"kb_used_data":4256,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93439576,"statfs":{"total":95995035648,"available":95682125824,"internally_reserved":0,"allocated":4358144,"data_stored":4151540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:32:43 2023","interfaces":[{"interface":"back","average":{"1min":4.5419999999999998,"5min":2.6549999999999998,"15min":1.5669999999999999},"min":{"1min":0.27400000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":44.036999999999999,"5min":59.258000000000003,"15min":59.258000000000003},"last":1.0569999999999999},{"interface":"front","average":{"1min":3.7610000000000001,"5min":2.7000000000000002,"15min":1.579},"min":{"1min":0.28999999999999998,"5min":0.246,"15min":0.246},"max":{"1min":44.231000000000002,"5min":59.432000000000002,"15min":59.432000000000002},"last":0.33600000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:33:25 2023","interfaces":[{"interface":"back","average":{"1min":8.5839999999999996,"5min":2.7839999999999998,"15min":1.653},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":114.95999999999999,"5min":114.95999999999999,"15min":114.95999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":8.6449999999999996,"5min":2.8889999999999998,"15min":1.746},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":115.087,"5min":115.087,"15min":115.087},"last":0.85399999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:33:45 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.645,"15min":1.4330000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.4100000000000001,"5min":12.731999999999999,"15min":14.019},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.8779999999999999,"5min":1.9350000000000001,"15min":1.552},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.3970000000000002,"5min":12.717000000000001,"15min":14.037000000000001},"last":0.79400000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":2.7400000000000002,"15min":1.7609999999999999},"min":{"1min":0.32600000000000001,"5min":0.311,"15min":0.311},"max":{"1min":4.0960000000000001,"5min":59.652000000000001,"15min":59.652000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.476,"5min":2.778,"15min":1.7749999999999999},"min":{"1min":0.375,"5min":0.35399999999999998,"15min":0.34899999999999998},"max":{"1min":4.7080000000000002,"5min":61.771999999999998,"15min":61.771999999999998},"last":0.89200000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:33:25 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":1.125,"15min":1.0369999999999999},"min":{"1min":0.317,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.0720000000000001,"5min":10.747,"15min":10.747},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.226,"15min":1.1100000000000001},"min":{"1min":0.38900000000000001,"5min":0.316,"15min":0.29799999999999999},"max":{"1min":4.3899999999999997,"5min":7.3109999999999999,"15min":7.5780000000000003},"last":0.96999999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:33:56 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.4359999999999999,"15min":1.232},"min":{"1min":0.34499999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.1779999999999999,"5min":14.518000000000001,"15min":14.518000000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":1.835,"5min":1.5900000000000001,"15min":1.397},"min":{"1min":0.42399999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.718,"5min":8.0280000000000005,"15min":8.0280000000000005},"last":1.044}]},{"osd":7,"last update":"Mon Oct 9 09:33:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3180000000000001,"5min":1.4510000000000001,"15min":1.3260000000000001},"min":{"1min":0.45000000000000001,"5min":0.44,"15min":0.36299999999999999},"max":{"1min":6.1020000000000003,"5min":7.9909999999999997,"15min":8.6359999999999992},"last":1.129},{"interface":"front","average":{"1min":1.232,"5min":1.407,"15min":1.306},"min":{"1min":0.442,"5min":0.41899999999999998,"15min":0.41399999999999998},"max":{"1min":8.4619999999999997,"5min":8.4619999999999997,"15min":8.4619999999999997},"last":1.085}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1607274496,"data_stored":1607270424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":57081856,"data_stored":57081856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:34:27.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pgp_num 26 2023-10-09T09:34:27.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:27.146 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:27.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:27.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:27.250 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:27.276 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:27.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:27.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:27.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:27.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:27.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:27.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:27.512 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:27.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:27.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:27.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:27.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:27.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:27.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:27 smithi163 bash[75425]: audit 2023-10-09T09:34:26.057962+0000 mgr.y (mgr.34107) 848 : audit 0 from='client.35121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:34:27.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:27 smithi163 bash[78107]: audit 2023-10-09T09:34:26.057962+0000 mgr.y (mgr.34107) 848 : audit 0 from='client.35121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:34:27.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:34:27.746 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:34:27.781 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:27.795 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:27 smithi181 bash[59345]: audit 2023-10-09T09:34:26.057962+0000 mgr.y (mgr.34107) 848 : audit 0 from='client.35121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:34:27.814 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:27.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:27.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:27.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:27.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:27.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:27.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:28.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:28.083 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:28.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:28.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:28.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:28.215 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:28.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:28.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:28.318 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:28.350 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:28.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:28.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:28.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:28.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:28.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:28.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:28.654 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:28.688 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:28.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:28.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:28.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:28 smithi163 bash[75425]: cluster 2023-10-09T09:34:27.796410+0000 mgr.y (mgr.34107) 849 : cluster 0 pgmap v1577: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 186 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:28 smithi163 bash[78107]: cluster 2023-10-09T09:34:27.796410+0000 mgr.y (mgr.34107) 849 : cluster 0 pgmap v1577: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 186 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:28.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:28.823 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:28.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:28.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:28.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:28.938 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:28 smithi181 bash[59345]: cluster 2023-10-09T09:34:27.796410+0000 mgr.y (mgr.34107) 849 : cluster 0 pgmap v1577: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 186 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:28.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:28.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:29.062 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:29.096 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:29.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:29.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:29.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:29.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:29.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:29.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:29.337 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:29.378 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:29.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:29.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:29.483 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:29.515 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:29.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:29.619 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:29.650 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:29.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:29.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:29.754 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:29.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:29.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:29.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:29.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:29.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:29 smithi163 bash[78107]: cluster 2023-10-09T09:34:29.797812+0000 mgr.y (mgr.34107) 850 : cluster 0 pgmap v1578: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:29.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:29 smithi163 bash[75425]: cluster 2023-10-09T09:34:29.797812+0000 mgr.y (mgr.34107) 850 : cluster 0 pgmap v1578: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:29.916 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:29.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:29.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:30.019 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:30.052 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:30.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:30.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:29 smithi181 bash[59345]: cluster 2023-10-09T09:34:29.797812+0000 mgr.y (mgr.34107) 850 : cluster 0 pgmap v1578: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 118/390 objects degraded (30.256%) 2023-10-09T09:34:30.157 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:30.191 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:30.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:30.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:30.295 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:30.330 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:30.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:30.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:30.433 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:30.465 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:30.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:30.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:30.568 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:30.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:30.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:30.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:30.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:30.734 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:30.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:30.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:30.739 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:34:30.838 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:30.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:30.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:30.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:30.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:30 smithi163 bash[75425]: cluster 2023-10-09T09:34:30.613503+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 118/390 objects degraded (30.256%), 11 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-10-09T09:34:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:30 smithi163 bash[78107]: cluster 2023-10-09T09:34:30.613503+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 118/390 objects degraded (30.256%), 11 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-10-09T09:34:30.972 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:31.003 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:31.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:31.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:30 smithi181 bash[59345]: cluster 2023-10-09T09:34:30.613503+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 118/390 objects degraded (30.256%), 11 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-10-09T09:34:31.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:31.138 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:31.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:31.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:31.242 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:31.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:31.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:31.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:31.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:31.413 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:31.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:31.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:31.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:31.544 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:31.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:31.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:31.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:31.677 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:31.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:31.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:31.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:31.811 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:31.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:31.851 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:34:31.852 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:34:31.852 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:34:31.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:31.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:31.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:31.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:31.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:31 smithi163 bash[78107]: cluster 2023-10-09T09:34:31.799643+0000 mgr.y (mgr.34107) 851 : cluster 0 pgmap v1579: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:31.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:31 smithi163 bash[75425]: cluster 2023-10-09T09:34:31.799643+0000 mgr.y (mgr.34107) 851 : cluster 0 pgmap v1579: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:32.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:32.058 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:31 smithi181 bash[59345]: cluster 2023-10-09T09:34:31.799643+0000 mgr.y (mgr.34107) 851 : cluster 0 pgmap v1579: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:32.077 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:32.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:32.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:32.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:32.201 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:32.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:32.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:32.304 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:32.332 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:32.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:32.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:32.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:32.463 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:32.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:32.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:32.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:32.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:32.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:32.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:32.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:32.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:32.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:32.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:32.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:32.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:32.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:32.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:32.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:32.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:33.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:33.127 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:33.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:33.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:33.231 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:33.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:33.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:33.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:33.366 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:33.395 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:33.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:33.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:33.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:33.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:33.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:33.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:33.605 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:34:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:34:33] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.33.4" 2023-10-09T09:34:33.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:33.655 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:33.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:33.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:33.758 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:33.784 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:33.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:33.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:33.887 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:33.890 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:34:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:34:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:34:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:34:33.891 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:34:33.915 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:33.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:33.936 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:33 smithi181 bash[59345]: cluster 2023-10-09T09:34:33.800356+0000 mgr.y (mgr.34107) 852 : cluster 0 pgmap v1580: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:34.019 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:34.055 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:34.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:34.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:34.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:34.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:33 smithi163 bash[75425]: cluster 2023-10-09T09:34:33.800356+0000 mgr.y (mgr.34107) 852 : cluster 0 pgmap v1580: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:34.171 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:33 smithi163 bash[78107]: cluster 2023-10-09T09:34:33.800356+0000 mgr.y (mgr.34107) 852 : cluster 0 pgmap v1580: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:34.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:34.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:34.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:34.288 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:34.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:34.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:34.420 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:34.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:34.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:34.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:34.586 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:34.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:34.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:34.665 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T09:34:34.689 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:34.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:34.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:34.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:34.843 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:34.874 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:34.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:34.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:34.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:35.011 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:35.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:35.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:35.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:35.144 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:35.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:35.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:35.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:35.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:35.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:35.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:35.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:35.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:35.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:35.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:35.534 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:35.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:35.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:35.636 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:35.666 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:35.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:35.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:35.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:35.796 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:35.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:35.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:35.899 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:35.927 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:35.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:35.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:36.030 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:36.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:36.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:36.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:36.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:36.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:36.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:36.301 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:36.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:36.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:36.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:36.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:36.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:36.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:36.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:36.578 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:36.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:36.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:36.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:36.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:36.743 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:36.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:36.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:36.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:36.858 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:36 smithi181 bash[59345]: cluster 2023-10-09T09:34:35.801180+0000 mgr.y (mgr.34107) 853 : cluster 0 pgmap v1581: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:36.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:36.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:36.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:36.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:36 smithi163 bash[78107]: cluster 2023-10-09T09:34:35.801180+0000 mgr.y (mgr.34107) 853 : cluster 0 pgmap v1581: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:36 smithi163 bash[75425]: cluster 2023-10-09T09:34:35.801180+0000 mgr.y (mgr.34107) 853 : cluster 0 pgmap v1581: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:36.980 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:37.010 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:37.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:37.114 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:37.144 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:37.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:37.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:37.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:37.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:37.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:37.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:37.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:37.419 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:37.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:37.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:37.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:37.551 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:37.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:37.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:37.613 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:34:37.644 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:34:37.645 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:34:37.654 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:37.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:37 smithi181 bash[59345]: audit 2023-10-09T09:34:37.504765+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.163:0/572332397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:34:37.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:37 smithi181 bash[59345]: audit 2023-10-09T09:34:37.505125+0000 mon.a (mon.0) 1705 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:34:37.664 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:37 smithi181 bash[59345]: audit 2023-10-09T09:34:37.587280+0000 mon.a (mon.0) 1706 : audit 0 from='client.? 172.21.15.163:0/3846701464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:37.668 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:34:37.672 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:37 smithi163 bash[78107]: audit 2023-10-09T09:34:37.504765+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.163:0/572332397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:34:37.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:37 smithi163 bash[78107]: audit 2023-10-09T09:34:37.505125+0000 mon.a (mon.0) 1705 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:34:37.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:37 smithi163 bash[78107]: audit 2023-10-09T09:34:37.587280+0000 mon.a (mon.0) 1706 : audit 0 from='client.? 172.21.15.163:0/3846701464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:37.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:37 smithi163 bash[75425]: audit 2023-10-09T09:34:37.504765+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.163:0/572332397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:34:37.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:37 smithi163 bash[75425]: audit 2023-10-09T09:34:37.505125+0000 mon.a (mon.0) 1705 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:34:37.673 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:37 smithi163 bash[75425]: audit 2023-10-09T09:34:37.587280+0000 mon.a (mon.0) 1706 : audit 0 from='client.? 172.21.15.163:0/3846701464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:37.683 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:37.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:37.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:37.697 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:34:37.724 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:34:37.752 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:34:37.778 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:34:37.778 INFO:teuthology.orchestra.run.smithi163.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T09:34:37.778 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:34:37.778 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:34:37.778 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:34:37.778 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:34:37.778 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:34:37.778 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:34:37.778 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:34:37.787 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:37.809 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:37.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:37.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:37.912 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:37.939 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:37.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:37.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:38.043 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:38.074 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:38.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:38.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:38.178 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:38.207 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:38.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:38.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:38.310 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:38.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:38.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:38.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:38.446 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:38.471 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:38.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:38.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:38.574 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:38.599 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:38.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:38.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:38.648 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 7 pgp_num to 26 2023-10-09T09:34:38.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:38.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:38.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:38.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:38.827 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:38.856 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:38.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:38.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:38.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:38 smithi181 bash[59345]: cluster 2023-10-09T09:34:37.627275+0000 mon.a (mon.0) 1707 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:34:38.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:38 smithi181 bash[59345]: cluster 2023-10-09T09:34:37.635853+0000 osd.5 (osd.5) 682 : cluster 3 failed to encode map e1128 with expected crc 2023-10-09T09:34:38.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:38 smithi181 bash[59345]: audit 2023-10-09T09:34:37.638967+0000 mon.a (mon.0) 1708 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:34:38.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:38 smithi181 bash[59345]: cluster 2023-10-09T09:34:37.639046+0000 mon.a (mon.0) 1709 : cluster 0 osdmap e1128: 8 total, 7 up, 8 in 2023-10-09T09:34:38.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:38 smithi181 bash[59345]: cluster 2023-10-09T09:34:37.802044+0000 mgr.y (mgr.34107) 854 : cluster 0 pgmap v1583: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:38.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:38 smithi181 bash[59345]: audit 2023-10-09T09:34:37.843975+0000 mon.a (mon.0) 1710 : audit 1 from='client.? 172.21.15.163:0/2580795432' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T09:34:38.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:38.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:39.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[78107]: cluster 2023-10-09T09:34:37.627275+0000 mon.a (mon.0) 1707 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:34:39.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[78107]: cluster 2023-10-09T09:34:37.635853+0000 osd.5 (osd.5) 682 : cluster 3 failed to encode map e1128 with expected crc 2023-10-09T09:34:39.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[78107]: audit 2023-10-09T09:34:37.638967+0000 mon.a (mon.0) 1708 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:34:39.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[78107]: cluster 2023-10-09T09:34:37.639046+0000 mon.a (mon.0) 1709 : cluster 0 osdmap e1128: 8 total, 7 up, 8 in 2023-10-09T09:34:39.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[78107]: cluster 2023-10-09T09:34:37.802044+0000 mgr.y (mgr.34107) 854 : cluster 0 pgmap v1583: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:39.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[78107]: audit 2023-10-09T09:34:37.843975+0000 mon.a (mon.0) 1710 : audit 1 from='client.? 172.21.15.163:0/2580795432' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T09:34:39.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[75425]: cluster 2023-10-09T09:34:37.627275+0000 mon.a (mon.0) 1707 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:34:39.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[75425]: cluster 2023-10-09T09:34:37.635853+0000 osd.5 (osd.5) 682 : cluster 3 failed to encode map e1128 with expected crc 2023-10-09T09:34:39.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[75425]: audit 2023-10-09T09:34:37.638967+0000 mon.a (mon.0) 1708 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:34:39.014 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[75425]: cluster 2023-10-09T09:34:37.639046+0000 mon.a (mon.0) 1709 : cluster 0 osdmap e1128: 8 total, 7 up, 8 in 2023-10-09T09:34:39.014 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[75425]: cluster 2023-10-09T09:34:37.802044+0000 mgr.y (mgr.34107) 854 : cluster 0 pgmap v1583: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:39.014 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:38 smithi163 bash[75425]: audit 2023-10-09T09:34:37.843975+0000 mon.a (mon.0) 1710 : audit 1 from='client.? 172.21.15.163:0/2580795432' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T09:34:39.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:39.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:39.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:39.225 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:39.258 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:39.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:39.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:39.362 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:39.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:39.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:39.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:39.435 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:34:39.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:39.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:39.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:39.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:39.619 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:39.645 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:39.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:39.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:39.749 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:39.761 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:39 smithi181 bash[59345]: cluster 2023-10-09T09:34:38.023560+0000 osd.7 (osd.7) 531 : cluster 3 failed to encode map e1128 with expected crc 2023-10-09T09:34:39.761 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:39 smithi181 bash[59345]: cluster 2023-10-09T09:34:38.641088+0000 osd.5 (osd.5) 683 : cluster 3 failed to encode map e1129 with expected crc 2023-10-09T09:34:39.761 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:39 smithi181 bash[59345]: audit 2023-10-09T09:34:38.643645+0000 mon.a (mon.0) 1711 : audit 1 from='client.? 172.21.15.163:0/2580795432' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T09:34:39.761 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:39 smithi181 bash[59345]: cluster 2023-10-09T09:34:38.643718+0000 mon.a (mon.0) 1712 : cluster 0 osdmap e1129: 8 total, 7 up, 8 in 2023-10-09T09:34:39.781 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:39.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:39.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:39.885 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:39.920 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:39.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:39.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:39.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:39 smithi163 bash[75425]: cluster 2023-10-09T09:34:38.023560+0000 osd.7 (osd.7) 531 : cluster 3 failed to encode map e1128 with expected crc 2023-10-09T09:34:39.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:39 smithi163 bash[75425]: cluster 2023-10-09T09:34:38.641088+0000 osd.5 (osd.5) 683 : cluster 3 failed to encode map e1129 with expected crc 2023-10-09T09:34:39.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:39 smithi163 bash[75425]: audit 2023-10-09T09:34:38.643645+0000 mon.a (mon.0) 1711 : audit 1 from='client.? 172.21.15.163:0/2580795432' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T09:34:39.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:39 smithi163 bash[75425]: cluster 2023-10-09T09:34:38.643718+0000 mon.a (mon.0) 1712 : cluster 0 osdmap e1129: 8 total, 7 up, 8 in 2023-10-09T09:34:39.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:39 smithi163 bash[78107]: cluster 2023-10-09T09:34:38.023560+0000 osd.7 (osd.7) 531 : cluster 3 failed to encode map e1128 with expected crc 2023-10-09T09:34:39.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:39 smithi163 bash[78107]: cluster 2023-10-09T09:34:38.641088+0000 osd.5 (osd.5) 683 : cluster 3 failed to encode map e1129 with expected crc 2023-10-09T09:34:39.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:39 smithi163 bash[78107]: audit 2023-10-09T09:34:38.643645+0000 mon.a (mon.0) 1711 : audit 1 from='client.? 172.21.15.163:0/2580795432' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T09:34:39.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:39 smithi163 bash[78107]: cluster 2023-10-09T09:34:38.643718+0000 mon.a (mon.0) 1712 : cluster 0 osdmap e1129: 8 total, 7 up, 8 in 2023-10-09T09:34:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T09:34:39.934 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T09:34:40.023 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:40.055 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:40.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:40.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:40.158 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:40.191 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:40.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:40.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:40.295 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:40.322 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:40.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:40.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:40.427 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:40.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:40.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:40.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:40.460 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T09:34:40.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T09:34:40.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:40.592 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:40.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:40.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:40.696 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:40.707 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:40 smithi181 bash[59345]: cluster 2023-10-09T09:34:39.802864+0000 mgr.y (mgr.34107) 855 : cluster 0 pgmap v1585: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:40.707 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:40 smithi181 bash[59345]: audit 2023-10-09T09:34:40.373319+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:40 smithi163 bash[75425]: cluster 2023-10-09T09:34:39.802864+0000 mgr.y (mgr.34107) 855 : cluster 0 pgmap v1585: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:40.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:40 smithi163 bash[75425]: audit 2023-10-09T09:34:40.373319+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:40 smithi163 bash[78107]: cluster 2023-10-09T09:34:39.802864+0000 mgr.y (mgr.34107) 855 : cluster 0 pgmap v1585: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:40.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:40 smithi163 bash[78107]: audit 2023-10-09T09:34:40.373319+0000 mon.a (mon.0) 1713 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:40.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:40.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:40.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:40.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:40.856 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:40.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:40.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:40.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:40.991 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:40.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:40.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:41.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:41.122 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:41.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:41.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:41.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:41.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:41.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:41.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:41.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:41.395 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:41.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:41.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:41.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:41.527 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:34:41.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:41.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:41.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:41.660 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:41.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:41.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:41.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:41.790 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:41.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:41.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:41.894 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:41.919 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:41.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:41.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:34:42.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:42.052 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:42.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:42.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:41 smithi181 bash[59345]: cluster 2023-10-09T09:34:41.804558+0000 mgr.y (mgr.34107) 856 : cluster 0 pgmap v1586: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:42.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:42.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:42.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:42.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:41 smithi163 bash[75425]: cluster 2023-10-09T09:34:41.804558+0000 mgr.y (mgr.34107) 856 : cluster 0 pgmap v1586: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:42.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:41 smithi163 bash[78107]: cluster 2023-10-09T09:34:41.804558+0000 mgr.y (mgr.34107) 856 : cluster 0 pgmap v1586: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:42.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:42.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:42.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:42.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:42.434 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:42.465 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:42.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:42.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:42.563 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:34:42.563 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:34:42.569 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:42.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:42.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:42.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:42.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:42.735 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:42.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:42.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:42.839 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:42.867 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:42.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:42.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:42.970 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:42.999 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:43.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:43.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:43.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:43.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:43.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:43.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:43.239 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:43.268 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:43.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:43.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:43.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:43.378 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T09:34:43.400 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:43.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:43.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:43.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:43.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:43.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:43.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:43.636 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:43.668 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:43.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:34:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:34:43] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.33.4" 2023-10-09T09:34:43.773 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:43.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:43.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:43.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:43.908 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:43.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:43.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:44.046 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:44.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:43 smithi163 bash[78107]: cluster 2023-10-09T09:34:43.805292+0000 mgr.y (mgr.34107) 857 : cluster 0 pgmap v1587: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:44.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:43 smithi163 bash[75425]: cluster 2023-10-09T09:34:43.805292+0000 mgr.y (mgr.34107) 857 : cluster 0 pgmap v1587: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:44.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:44.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:44.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:44.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:44.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:43 smithi181 bash[59345]: cluster 2023-10-09T09:34:43.805292+0000 mgr.y (mgr.34107) 857 : cluster 0 pgmap v1587: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:44.215 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:44.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:44.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:44.356 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:44.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:44.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:44.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:44.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:44.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:44.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:44.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:44.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:44.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:44.726 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:44.758 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:44.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:44.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:44.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:44.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:44.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:44.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:44.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:45.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:45.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:45.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:45.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:45.159 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:45.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:45.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:45.230 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:34:45.263 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:45.287 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:45.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:45.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:45.390 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:45.424 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:45.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:45.461 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:34:45.461 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:34:45.461 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T09:34:45.461 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:34:45.461 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:34:45.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T09:34:45.528 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:45.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:45.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:45.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:45.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:45.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:45.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:45.824 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:45.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:45.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:45.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:45.961 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:45.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:45.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:46.064 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:46.093 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:46.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:46.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:46.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:46.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:46.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:46.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:46.245 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:34:46.246 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:34:46.334 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:46.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:46.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:46.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:46.470 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:46.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:46.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:46.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:46.603 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:46.635 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:46.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:46.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:46.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:46.751 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:46 smithi181 bash[59345]: cluster 2023-10-09T09:34:45.806690+0000 mgr.y (mgr.34107) 858 : cluster 0 pgmap v1588: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:46.769 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:46.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:46.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:46.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:46.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:46.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:46.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:46.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:46 smithi163 bash[75425]: cluster 2023-10-09T09:34:45.806690+0000 mgr.y (mgr.34107) 858 : cluster 0 pgmap v1588: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:46.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:46 smithi163 bash[78107]: cluster 2023-10-09T09:34:45.806690+0000 mgr.y (mgr.34107) 858 : cluster 0 pgmap v1588: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:47.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:47.039 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:47.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:47.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:47.142 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:47.171 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:47.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:47.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:47.274 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:47.308 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:47.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:47.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:47.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:47.444 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:47.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:47.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:47.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:47.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:47.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:47.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:47.682 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:47.715 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:47.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:47.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:47.819 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:47.851 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:47.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:47.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:47.954 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:47.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:47.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:47.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:48.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:48.098 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:47 smithi163 bash[75425]: cluster 2023-10-09T09:34:47.808287+0000 mgr.y (mgr.34107) 859 : cluster 0 pgmap v1589: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:48.098 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:47 smithi163 bash[78107]: cluster 2023-10-09T09:34:47.808287+0000 mgr.y (mgr.34107) 859 : cluster 0 pgmap v1589: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:48.112 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:48.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:48.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:48.164 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:34:48.189 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:47 smithi181 bash[59345]: cluster 2023-10-09T09:34:47.808287+0000 mgr.y (mgr.34107) 859 : cluster 0 pgmap v1589: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:48.215 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:48.240 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:48.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:48.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:48.344 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:48.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:48.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:48.478 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:48.505 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:48.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:48.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:48.609 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:48.637 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:48.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:48.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:48.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:48.774 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:48.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:48.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:48.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:48.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:48.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:48.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:49.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:49.035 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:49.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:49.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:49.139 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:49.166 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:49.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:49.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:49.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:49.299 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:49.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:49.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:49.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:49.428 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:49.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:49.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:49.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:49.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:49.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:49.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:49.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:49.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:49.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:49.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:49.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:49.819 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:49.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:49.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:49.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:49.954 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:49.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:49.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:49 smithi163 bash[75425]: cluster 2023-10-09T09:34:49.809628+0000 mgr.y (mgr.34107) 860 : cluster 0 pgmap v1590: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:49 smithi163 bash[78107]: cluster 2023-10-09T09:34:49.809628+0000 mgr.y (mgr.34107) 860 : cluster 0 pgmap v1590: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:50.059 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:50.083 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:50.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:50.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:50.186 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:50.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:49 smithi181 bash[59345]: cluster 2023-10-09T09:34:49.809628+0000 mgr.y (mgr.34107) 860 : cluster 0 pgmap v1590: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:50.217 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:50.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:50.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:50.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:50.351 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:50.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:50.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:50.455 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:50.482 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:50.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:50.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:50.571 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:34:50.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:50.617 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:50.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:50.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:50.720 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:50.746 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:50.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:50.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:50.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:50.879 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:50.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:50.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:50.983 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:51.012 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:51.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:51.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:51.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:51.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:51.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:51.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:51.238 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:51.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:51.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:51.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:51.370 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:51.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:51.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:51.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:51.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:51.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:51.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:51.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:51.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:51.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:51.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:51.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:51.797 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:51.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:51.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:51.927 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:51.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:51.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:52.030 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:52.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:51 smithi181 bash[59345]: cluster 2023-10-09T09:34:51.811588+0000 mgr.y (mgr.34107) 861 : cluster 0 pgmap v1591: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:52.055 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:52.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:52.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:52.133 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:34:52.134 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:34:52.159 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:52.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:52.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:52.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:52.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:51 smithi163 bash[75425]: cluster 2023-10-09T09:34:51.811588+0000 mgr.y (mgr.34107) 861 : cluster 0 pgmap v1591: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:52.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:51 smithi163 bash[78107]: cluster 2023-10-09T09:34:51.811588+0000 mgr.y (mgr.34107) 861 : cluster 0 pgmap v1591: 99 pgs: 1 active+clean+remapped, 11 active+undersized+degraded, 28 active+undersized, 59 active+clean; 7.2 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118/390 objects degraded (30.256%) 2023-10-09T09:34:52.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:52.315 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:52.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:52.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:52.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:52.440 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:52.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:52.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:52.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:52.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:52.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:52.674 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:52.697 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:52.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:52.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:52.800 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:52.822 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:52.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:52.925 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:52.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:52.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:52.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:53.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:53.083 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:53.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:53.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:53.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:53.221 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:53.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:53.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:53.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:53.350 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:34:53.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:53.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:34:53.414 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T09:34:53.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:53.482 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:53.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:53.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:53.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:53.607 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:53.608 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:34:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:34:53] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.33.4" 2023-10-09T09:34:53.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:53.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:53.711 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:53.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:53 smithi181 bash[59345]: audit 2023-10-09T09:34:53.645502+0000 mon.b (mon.1) 15 : audit 1 from='client.? 172.21.15.163:0/12210920' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:34:53.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:53 smithi181 bash[59345]: audit 2023-10-09T09:34:53.646816+0000 mon.a (mon.0) 1714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:34:53.715 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_2' removed 2023-10-09T09:34:53.740 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:53.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:53.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:53.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:53.864 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:53.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:53.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:53.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:53 smithi163 bash[75425]: audit 2023-10-09T09:34:53.645502+0000 mon.b (mon.1) 15 : audit 1 from='client.? 172.21.15.163:0/12210920' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:34:53.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:53 smithi163 bash[75425]: audit 2023-10-09T09:34:53.646816+0000 mon.a (mon.0) 1714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:34:53.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:53 smithi163 bash[78107]: audit 2023-10-09T09:34:53.645502+0000 mon.b (mon.1) 15 : audit 1 from='client.? 172.21.15.163:0/12210920' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:34:53.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:53 smithi163 bash[78107]: audit 2023-10-09T09:34:53.646816+0000 mon.a (mon.0) 1714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:34:53.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:53.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:53.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:53.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:54.089 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:54.108 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:54.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:54.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:54.212 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:54.234 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:54.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:54.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:54.363 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:54.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:54.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:54.466 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:54.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:54.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:54.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:54.597 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:54.626 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:54.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:54.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:54.711 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:34:54.782 INFO:teuthology.orchestra.run.smithi181.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T09:34:54.782 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:34:54.783 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:54.814 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:54.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:54.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:54.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:54 smithi163 bash[78107]: cluster 2023-10-09T09:34:53.700483+0000 osd.5 (osd.5) 684 : cluster 3 failed to encode map e1130 with expected crc 2023-10-09T09:34:54.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:54 smithi163 bash[78107]: audit 2023-10-09T09:34:53.709139+0000 mon.a (mon.0) 1715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:34:54.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:54 smithi163 bash[78107]: cluster 2023-10-09T09:34:53.709218+0000 mon.a (mon.0) 1716 : cluster 0 osdmap e1130: 8 total, 7 up, 8 in 2023-10-09T09:34:54.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:54 smithi163 bash[78107]: cluster 2023-10-09T09:34:53.812301+0000 mgr.y (mgr.34107) 862 : cluster 0 pgmap v1593: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:54.929 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:54 smithi163 bash[78107]: cluster 2023-10-09T09:34:53.855082+0000 osd.7 (osd.7) 532 : cluster 3 failed to encode map e1130 with expected crc 2023-10-09T09:34:54.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:54 smithi163 bash[75425]: cluster 2023-10-09T09:34:53.700483+0000 osd.5 (osd.5) 684 : cluster 3 failed to encode map e1130 with expected crc 2023-10-09T09:34:54.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:54 smithi163 bash[75425]: audit 2023-10-09T09:34:53.709139+0000 mon.a (mon.0) 1715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:34:54.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:54 smithi163 bash[75425]: cluster 2023-10-09T09:34:53.709218+0000 mon.a (mon.0) 1716 : cluster 0 osdmap e1130: 8 total, 7 up, 8 in 2023-10-09T09:34:54.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:54 smithi163 bash[75425]: cluster 2023-10-09T09:34:53.812301+0000 mgr.y (mgr.34107) 862 : cluster 0 pgmap v1593: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:54.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:54 smithi163 bash[75425]: cluster 2023-10-09T09:34:53.855082+0000 osd.7 (osd.7) 532 : cluster 3 failed to encode map e1130 with expected crc 2023-10-09T09:34:54.941 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:54.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:54.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:55.045 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:55.055 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:54 smithi181 bash[59345]: cluster 2023-10-09T09:34:53.700483+0000 osd.5 (osd.5) 684 : cluster 3 failed to encode map e1130 with expected crc 2023-10-09T09:34:55.055 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:54 smithi181 bash[59345]: audit 2023-10-09T09:34:53.709139+0000 mon.a (mon.0) 1715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:34:55.055 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:54 smithi181 bash[59345]: cluster 2023-10-09T09:34:53.709218+0000 mon.a (mon.0) 1716 : cluster 0 osdmap e1130: 8 total, 7 up, 8 in 2023-10-09T09:34:55.055 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:54 smithi181 bash[59345]: cluster 2023-10-09T09:34:53.812301+0000 mgr.y (mgr.34107) 862 : cluster 0 pgmap v1593: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:55.055 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:54 smithi181 bash[59345]: cluster 2023-10-09T09:34:53.855082+0000 osd.7 (osd.7) 532 : cluster 3 failed to encode map e1130 with expected crc 2023-10-09T09:34:55.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:55.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:55.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:55.178 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:55.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:55.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:55.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:55.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:55.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:55.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:55.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:55.446 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:55.478 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:55.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:55.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:55.583 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:55.613 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:55.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:55.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:55.718 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:55.735 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:34:55.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:55.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:55.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:55.748 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:34:55.749 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:34:55.749 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T09:34:55.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T09:34:55.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:55.861 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:55 smithi181 bash[59345]: cluster 2023-10-09T09:34:53.930148+0000 osd.6 (osd.6) 412 : cluster 3 failed to encode map e1130 with expected crc 2023-10-09T09:34:55.861 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:55 smithi181 bash[59345]: cluster 2023-10-09T09:34:54.711638+0000 mon.a (mon.0) 1717 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T09:34:55.861 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:55 smithi181 bash[59345]: audit 2023-10-09T09:34:55.061554+0000 mon.a (mon.0) 1718 : audit 1 from='client.? 172.21.15.163:0/552351710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:34:55.861 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:55 smithi181 bash[59345]: audit 2023-10-09T09:34:55.372674+0000 mon.a (mon.0) 1719 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:55.862 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:55 smithi181 bash[59345]: audit 2023-10-09T09:34:55.626937+0000 mon.a (mon.0) 1720 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:34:55.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:55.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:55.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:55.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:55 smithi163 bash[78107]: cluster 2023-10-09T09:34:53.930148+0000 osd.6 (osd.6) 412 : cluster 3 failed to encode map e1130 with expected crc 2023-10-09T09:34:55.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:55 smithi163 bash[78107]: cluster 2023-10-09T09:34:54.711638+0000 mon.a (mon.0) 1717 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T09:34:55.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:55 smithi163 bash[78107]: audit 2023-10-09T09:34:55.061554+0000 mon.a (mon.0) 1718 : audit 1 from='client.? 172.21.15.163:0/552351710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:34:55.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:55 smithi163 bash[78107]: audit 2023-10-09T09:34:55.372674+0000 mon.a (mon.0) 1719 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:55.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:55 smithi163 bash[78107]: audit 2023-10-09T09:34:55.626937+0000 mon.a (mon.0) 1720 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:34:55.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:55 smithi163 bash[75425]: cluster 2023-10-09T09:34:53.930148+0000 osd.6 (osd.6) 412 : cluster 3 failed to encode map e1130 with expected crc 2023-10-09T09:34:55.979 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:55 smithi163 bash[75425]: cluster 2023-10-09T09:34:54.711638+0000 mon.a (mon.0) 1717 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T09:34:55.979 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:55 smithi163 bash[75425]: audit 2023-10-09T09:34:55.061554+0000 mon.a (mon.0) 1718 : audit 1 from='client.? 172.21.15.163:0/552351710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:34:55.979 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:55 smithi163 bash[75425]: audit 2023-10-09T09:34:55.372674+0000 mon.a (mon.0) 1719 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:34:55.979 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:55 smithi163 bash[75425]: audit 2023-10-09T09:34:55.626937+0000 mon.a (mon.0) 1720 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:34:55.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:56.013 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:56.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:56.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:56.050 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:34:56.074 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:34:56.099 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:34:56.117 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:56.127 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:34:56.153 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:56.155 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:34:56.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:56.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:56.180 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:34:56.209 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:34:56.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T09:34:56.209 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:34:56.209 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:34:56.209 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:34:56.209 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:34:56.209 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:34:56.210 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:34:56.210 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:34:56.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:56.285 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:56.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:56.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:56.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:56.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:56.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:56.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:56.520 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:56.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:56.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:56.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:56.656 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:56.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:56.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:56.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:56.793 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:56.826 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:56.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:56.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:56.930 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:56.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:56 smithi181 bash[59345]: cluster 2023-10-09T09:34:55.718968+0000 mon.a (mon.0) 1721 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:34:56.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:56 smithi181 bash[59345]: cluster 2023-10-09T09:34:55.726971+0000 osd.5 (osd.5) 685 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:56.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:56 smithi181 bash[59345]: audit 2023-10-09T09:34:55.731162+0000 mon.a (mon.0) 1722 : audit 1 from='client.? 172.21.15.163:0/552351710' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:34:56.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:56 smithi181 bash[59345]: cluster 2023-10-09T09:34:55.731225+0000 mon.a (mon.0) 1723 : cluster 0 osdmap e1131: 8 total, 7 up, 8 in 2023-10-09T09:34:56.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:56 smithi181 bash[59345]: cluster 2023-10-09T09:34:55.813194+0000 mgr.y (mgr.34107) 863 : cluster 0 pgmap v1595: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:56.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:56 smithi181 bash[59345]: audit 2023-10-09T09:34:56.028428+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.163:0/3080228452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:56.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:56 smithi181 bash[59345]: cluster 2023-10-09T09:34:56.250331+0000 osd.4 (osd.4) 556 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:56.961 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:56.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:56.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:57.020 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[75425]: cluster 2023-10-09T09:34:55.718968+0000 mon.a (mon.0) 1721 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:34:57.020 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[75425]: cluster 2023-10-09T09:34:55.726971+0000 osd.5 (osd.5) 685 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:57.020 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[75425]: audit 2023-10-09T09:34:55.731162+0000 mon.a (mon.0) 1722 : audit 1 from='client.? 172.21.15.163:0/552351710' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:34:57.020 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[75425]: cluster 2023-10-09T09:34:55.731225+0000 mon.a (mon.0) 1723 : cluster 0 osdmap e1131: 8 total, 7 up, 8 in 2023-10-09T09:34:57.020 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[75425]: cluster 2023-10-09T09:34:55.813194+0000 mgr.y (mgr.34107) 863 : cluster 0 pgmap v1595: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:57.020 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[75425]: audit 2023-10-09T09:34:56.028428+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.163:0/3080228452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:57.020 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[75425]: cluster 2023-10-09T09:34:56.250331+0000 osd.4 (osd.4) 556 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:57.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[78107]: cluster 2023-10-09T09:34:55.718968+0000 mon.a (mon.0) 1721 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:34:57.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[78107]: cluster 2023-10-09T09:34:55.726971+0000 osd.5 (osd.5) 685 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:57.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[78107]: audit 2023-10-09T09:34:55.731162+0000 mon.a (mon.0) 1722 : audit 1 from='client.? 172.21.15.163:0/552351710' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:34:57.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[78107]: cluster 2023-10-09T09:34:55.731225+0000 mon.a (mon.0) 1723 : cluster 0 osdmap e1131: 8 total, 7 up, 8 in 2023-10-09T09:34:57.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[78107]: cluster 2023-10-09T09:34:55.813194+0000 mgr.y (mgr.34107) 863 : cluster 0 pgmap v1595: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:57.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[78107]: audit 2023-10-09T09:34:56.028428+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.163:0/3080228452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:34:57.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:56 smithi163 bash[78107]: cluster 2023-10-09T09:34:56.250331+0000 osd.4 (osd.4) 556 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:57.064 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:57.092 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:57.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:57.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:57.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:57.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:57.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:57.331 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:57.364 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:57.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:57.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:57.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:57.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:57.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:57.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:57.630 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T09:34:57.631 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T09:34:57.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:57.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:57.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:57.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:57.765 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:57.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:57.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:57.869 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:57.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:57 smithi163 bash[78107]: cluster 2023-10-09T09:34:56.827128+0000 osd.7 (osd.7) 533 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:57.884 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:57 smithi163 bash[75425]: cluster 2023-10-09T09:34:56.827128+0000 osd.7 (osd.7) 533 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:57.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:57.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:58.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:58.040 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:58.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:58.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:58.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:57 smithi181 bash[59345]: cluster 2023-10-09T09:34:56.827128+0000 osd.7 (osd.7) 533 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:58.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:58.175 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:34:58.178 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:58.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:58.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:58.283 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:58.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:34:58.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:58.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:34:58.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:58.450 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:58.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:58.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:58.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:58.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:58.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:58.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:58.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:58.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:58.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:58.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:58.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:58.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:58.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:58.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:58.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:58.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:58 smithi163 bash[78107]: cluster 2023-10-09T09:34:56.936521+0000 osd.6 (osd.6) 413 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:58.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:58 smithi163 bash[78107]: cluster 2023-10-09T09:34:57.814525+0000 mgr.y (mgr.34107) 864 : cluster 0 pgmap v1596: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:58.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:58 smithi163 bash[75425]: cluster 2023-10-09T09:34:56.936521+0000 osd.6 (osd.6) 413 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:58.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:58 smithi163 bash[75425]: cluster 2023-10-09T09:34:57.814525+0000 mgr.y (mgr.34107) 864 : cluster 0 pgmap v1596: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:58.996 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:58.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:58.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:34:59.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:34:59.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:58 smithi181 bash[59345]: cluster 2023-10-09T09:34:56.936521+0000 osd.6 (osd.6) 413 : cluster 3 failed to encode map e1131 with expected crc 2023-10-09T09:34:59.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:58 smithi181 bash[59345]: cluster 2023-10-09T09:34:57.814525+0000 mgr.y (mgr.34107) 864 : cluster 0 pgmap v1596: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:59.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:59.130 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:59.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:59.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:59.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:59.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:59.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:59.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:59.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:59.397 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:34:59.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:59.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:34:59.501 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:59.533 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:34:59.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:59.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:34:59.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:34:59.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:59.663 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:34:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:59.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:34:59.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:59.794 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:34:59.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:59.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:34:59.898 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:59.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:34:59 smithi163 bash[78107]: cluster 2023-10-09T09:34:59.815299+0000 mgr.y (mgr.34107) 865 : cluster 0 pgmap v1597: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:59.911 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:34:59 smithi163 bash[75425]: cluster 2023-10-09T09:34:59.815299+0000 mgr.y (mgr.34107) 865 : cluster 0 pgmap v1597: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:34:59.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:34:59.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:34:59.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:00.032 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:00.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:34:59 smithi181 bash[59345]: cluster 2023-10-09T09:34:59.815299+0000 mgr.y (mgr.34107) 865 : cluster 0 pgmap v1597: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:00.058 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:00.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:00.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:00.161 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:00.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:00.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:00.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:00.301 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:00.335 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:00.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:00.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:00.410 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:35:00.410 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:35:00.410 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2023-10-09T09:35:00.411 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2023-10-09T09:35:00.411 DEBUG:teuthology.orchestra.run.smithi163:osd.0> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T09:35:00.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl start ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T09:35:00.431 INFO:journalctl@ceph.osd.0.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T09:35:00.439 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:00.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:35:00.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:00.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:00.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:00.621 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:00.650 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:00.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:00.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:00.718 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:00 smithi163 systemd[1]: Started Ceph osd.0 for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T09:35:00.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:00.774 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:00.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:00.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:00.901 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:00.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:00.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:01.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:01.050 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:01.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:01.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:01.153 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:01.161 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:35:01.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:01.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:01.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:01.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:01.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:01.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:01.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:01.363 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:35:01.363 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:35:01.363 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:35:01.363 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:35:01.364 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:35:01.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:01.435 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:01.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:01.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:01.560 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:01.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:01.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:01.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:01.688 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:01.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:01.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:01.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:01.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:01.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:01.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:01.915 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:01.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:01.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:01.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:02.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:02.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:02.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:02.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:01 smithi181 bash[59345]: cluster 2023-10-09T09:35:01.816158+0000 mgr.y (mgr.34107) 866 : cluster 0 pgmap v1598: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:01 smithi181 bash[59345]: audit 2023-10-09T09:35:01.865793+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:01 smithi181 bash[59345]: audit 2023-10-09T09:35:01.877231+0000 mon.a (mon.0) 1725 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:02.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T09:35:02.160 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:02.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:02.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:01 smithi163 bash[75425]: cluster 2023-10-09T09:35:01.816158+0000 mgr.y (mgr.34107) 866 : cluster 0 pgmap v1598: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:02.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:01 smithi163 bash[75425]: audit 2023-10-09T09:35:01.865793+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:02.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:01 smithi163 bash[75425]: audit 2023-10-09T09:35:01.877231+0000 mon.a (mon.0) 1725 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:01 smithi163 bash[78107]: cluster 2023-10-09T09:35:01.816158+0000 mgr.y (mgr.34107) 866 : cluster 0 pgmap v1598: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:01 smithi163 bash[78107]: audit 2023-10-09T09:35:01.865793+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:02.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:01 smithi163 bash[78107]: audit 2023-10-09T09:35:01.877231+0000 mon.a (mon.0) 1725 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:02.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:02.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:02.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:02.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:02.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:02.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:02.451 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:02.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:02.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:02.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:02.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:02.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:02.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:02.756 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:02.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:02.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:02.860 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:02.894 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:02.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:02.998 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:03.023 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:03.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:03.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:03.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:03.160 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:03.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:03.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:03.183 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:35:03.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:03.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:03.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:03.355 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[148063]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T09:35:03.355 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[148063]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T09:35:03.355 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[148063]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T09:35:03.355 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[148063]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T09:35:03.355 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[148063]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T09:35:03.355 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[148063]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T09:35:03.356 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[148063]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T09:35:03.356 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[148063]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T09:35:03.403 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:03.415 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:03 smithi181 bash[59345]: audit 2023-10-09T09:35:02.219235+0000 mon.a (mon.0) 1726 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:03.415 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:03 smithi181 bash[59345]: audit 2023-10-09T09:35:02.226222+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:03.415 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:03 smithi181 bash[59345]: audit 2023-10-09T09:35:02.758882+0000 mon.a (mon.0) 1728 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:35:03.416 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:03 smithi181 bash[59345]: audit 2023-10-09T09:35:02.760302+0000 mon.a (mon.0) 1729 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:35:03.416 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:03 smithi181 bash[59345]: audit 2023-10-09T09:35:02.768938+0000 mon.a (mon.0) 1730 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:03.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:03.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:03.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:03.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:35:03] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.33.4" 2023-10-09T09:35:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[75425]: audit 2023-10-09T09:35:02.219235+0000 mon.a (mon.0) 1726 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[75425]: audit 2023-10-09T09:35:02.226222+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:03.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[75425]: audit 2023-10-09T09:35:02.758882+0000 mon.a (mon.0) 1728 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:35:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[75425]: audit 2023-10-09T09:35:02.760302+0000 mon.a (mon.0) 1729 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:35:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[75425]: audit 2023-10-09T09:35:02.768938+0000 mon.a (mon.0) 1730 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:03.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[78107]: audit 2023-10-09T09:35:02.219235+0000 mon.a (mon.0) 1726 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:03.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[78107]: audit 2023-10-09T09:35:02.226222+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:03.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[78107]: audit 2023-10-09T09:35:02.758882+0000 mon.a (mon.0) 1728 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:35:03.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[78107]: audit 2023-10-09T09:35:02.760302+0000 mon.a (mon.0) 1729 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:35:03.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:03 smithi163 bash[78107]: audit 2023-10-09T09:35:02.768938+0000 mon.a (mon.0) 1730 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:03.539 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:03.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:03.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:03.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:03.673 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:03.708 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:03.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:03.811 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:03.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:03.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:03.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:03.937 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:03.960 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:03.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:04.063 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:04.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:04.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:04.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:04.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:04.233 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:04.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:04.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:04.338 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:04.360 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:04.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:04.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:04.430 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:04 smithi163 bash[78107]: cluster 2023-10-09T09:35:03.817123+0000 mgr.y (mgr.34107) 867 : cluster 0 pgmap v1599: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:04.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:04 smithi163 bash[75425]: cluster 2023-10-09T09:35:03.817123+0000 mgr.y (mgr.34107) 867 : cluster 0 pgmap v1599: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:04.463 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:04.489 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:04.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:04.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:04 smithi181 bash[59345]: cluster 2023-10-09T09:35:03.817123+0000 mgr.y (mgr.34107) 867 : cluster 0 pgmap v1599: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:04.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:04.624 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:04.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:04.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:04.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:04.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:04.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:04.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:04.860 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:04.888 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:04.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:04.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:04.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:05.016 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:05.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:05.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:05.150 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:05.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:05.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:05.254 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:05.285 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:05.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:05.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:05.412 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:05.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:05.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:05.543 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:05.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:05.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:05.646 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:05.672 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:05.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:05.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:05.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:05.803 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:05.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:05.906 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:05.937 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:05.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:05.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:06.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:06.073 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:06.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:06.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:06.177 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:06.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:06.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:06.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:06.346 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:06.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:06.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:06.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:06.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:06.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:06.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:06.583 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:06.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:06.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:06.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:06.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:06.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:06 smithi181 bash[59345]: cluster 2023-10-09T09:35:05.818347+0000 mgr.y (mgr.34107) 868 : cluster 0 pgmap v1600: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:06.755 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:06.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:06.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:06.834 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:35:06.859 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:06.890 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:06.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:06.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:06.911 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:35:06.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:06 smithi163 bash[75425]: cluster 2023-10-09T09:35:05.818347+0000 mgr.y (mgr.34107) 868 : cluster 0 pgmap v1600: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:06.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:06 smithi163 bash[78107]: cluster 2023-10-09T09:35:05.818347+0000 mgr.y (mgr.34107) 868 : cluster 0 pgmap v1600: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:06.994 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:07.015 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:07.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:07.118 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:07.145 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:07.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:07.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:07.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:07.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:07.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:07.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:07.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:07.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:07.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:07.507 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:07.538 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:07.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:07.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:07.641 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:07.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:07 smithi163 bash[78107]: cluster 2023-10-09T09:35:06.627278+0000 mon.a (mon.0) 1731 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T09:35:07.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:07 smithi163 bash[75425]: cluster 2023-10-09T09:35:06.627278+0000 mon.a (mon.0) 1731 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T09:35:07.665 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:07.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:07.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:07.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:07.779 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:07 smithi181 bash[59345]: cluster 2023-10-09T09:35:06.627278+0000 mon.a (mon.0) 1731 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T09:35:07.794 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:07.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:07.847 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:35:07.897 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:07.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:07.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:08.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:08.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:08.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:08.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:08.161 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:08.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:08.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:08.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:08.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:08.314 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:08.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:08.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:08.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:08.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:08.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:08.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:08.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:08.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:08.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:08.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:08.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:08.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:08.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:08.814 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:08 smithi163 bash[78107]: cluster 2023-10-09T09:35:07.819461+0000 mgr.y (mgr.34107) 869 : cluster 0 pgmap v1601: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:08.814 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:08 smithi163 bash[75425]: cluster 2023-10-09T09:35:07.819461+0000 mgr.y (mgr.34107) 869 : cluster 0 pgmap v1601: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:08.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:08.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:08.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:08.904 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:08 smithi181 bash[59345]: cluster 2023-10-09T09:35:07.819461+0000 mgr.y (mgr.34107) 869 : cluster 0 pgmap v1601: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:08.934 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:08.961 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:08.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:08.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:09.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:09.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:09.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:09.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:09.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:09.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:09.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:09.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:09.353 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:09.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:09.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:09.456 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:09.492 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:09.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:09.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:09.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:09.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:09.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:09.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:09.734 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:09.765 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:09.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:09.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:09.869 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:09.895 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:09.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:09.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:09.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:09 smithi163 bash[75425]: cluster 2023-10-09T09:35:09.820103+0000 mgr.y (mgr.34107) 870 : cluster 0 pgmap v1602: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:09.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:09 smithi163 bash[78107]: cluster 2023-10-09T09:35:09.820103+0000 mgr.y (mgr.34107) 870 : cluster 0 pgmap v1602: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:09.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:10.023 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:10.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:10.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:10.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:09 smithi181 bash[59345]: cluster 2023-10-09T09:35:09.820103+0000 mgr.y (mgr.34107) 870 : cluster 0 pgmap v1602: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:10.091 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:35:10.091 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T09:35:10.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T09:35:10.091 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:35:10.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.092 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:27812 6.7 6:ea29e4bc:::benchmark_data_smithi163_107771_object27811:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1081)", 2023-10-09T09:35:10.092 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:25:57.816517+0000", 2023-10-09T09:35:10.092 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 552.27085562499997, 2023-10-09T09:35:10.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12622804900000001, 2023-10-09T09:35:10.092 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.092 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.092 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.092 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.092 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.092 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 27812 2023-10-09T09:35:10.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.093 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.816517+0000", 2023-10-09T09:35:10.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.816517+0000", 2023-10-09T09:35:10.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:35:10.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.816515+0000", 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0555000000000004e-05 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.816595+0000", 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3302000000000001e-05 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.816609+0000", 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1799999999999999e-06 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.816616+0000", 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00084588600000000001 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.817462+0000", 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103547 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.817565+0000", 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000101858 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.817667+0000", 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039159379999999999 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.821583+0000", 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000149495 2023-10-09T09:35:10.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.821733+0000", 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.120886008 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.942619+0000", 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6233e-05 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.942695+0000", 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9929999999999998e-05 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:25:57.942745+0000", 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12622804900000001 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:40105 6.7 6:e9bee6c8:::benchmark_data_smithi163_107771_object40104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1082)", 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:26:06.002201+0000", 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 544.08517123599995, 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15079642600000001, 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 40105 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.100 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.002201+0000", 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.002201+0000", 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.002200+0000", 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8108000000000005e-05 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.002278+0000", 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0012000000000001e-05 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.002298+0000", 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3479999999999993e-06 2023-10-09T09:35:10.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.002307+0000", 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0428999999999999e-05 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.002378+0000", 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017522699999999999 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.002553+0000", 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026497599999999999 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.002818+0000", 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001605918 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.004424+0000", 2023-10-09T09:35:10.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0031696760000000002 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.007594+0000", 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14530834300000001 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.152902+0000", 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6057000000000002e-05 2023-10-09T09:35:10.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.152958+0000", 2023-10-09T09:35:10.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9666999999999998e-05 2023-10-09T09:35:10.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.152998+0000", 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15079642600000001 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:40108 6.7 6:e53c0d10:::benchmark_data_smithi163_107771_object40107:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1082)", 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:26:06.004820+0000", 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 544.08255198799998, 2023-10-09T09:35:10.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14829341400000001, 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 40108 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.004820+0000", 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.004820+0000", 2023-10-09T09:35:10.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:35:10.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.004818+0000", 2023-10-09T09:35:10.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2524000000000001e-05 2023-10-09T09:35:10.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.004861+0000", 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2703e-05 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.004874+0000", 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4520000000000002e-06 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.004880+0000", 2023-10-09T09:35:10.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1224999999999999e-05 2023-10-09T09:35:10.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.004921+0000", 2023-10-09T09:35:10.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120399 2023-10-09T09:35:10.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.005042+0000", 2023-10-09T09:35:10.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121006 2023-10-09T09:35:10.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.005163+0000", 2023-10-09T09:35:10.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024479369999999999 2023-10-09T09:35:10.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.007611+0000", 2023-10-09T09:35:10.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021675689999999998 2023-10-09T09:35:10.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.009778+0000", 2023-10-09T09:35:10.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14326038199999999 2023-10-09T09:35:10.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.153039+0000", 2023-10-09T09:35:10.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0896999999999999e-05 2023-10-09T09:35:10.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.153069+0000", 2023-10-09T09:35:10.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4388999999999998e-05 2023-10-09T09:35:10.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.153114+0000", 2023-10-09T09:35:10.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14829341400000001 2023-10-09T09:35:10.120 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.120 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.120 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.120 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:40123 6.7 6:efa31de0:::benchmark_data_smithi163_107771_object40122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1082)", 2023-10-09T09:35:10.121 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:26:06.012436+0000", 2023-10-09T09:35:10.121 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 544.07493640600001, 2023-10-09T09:35:10.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14078888000000001, 2023-10-09T09:35:10.121 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.121 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.121 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.121 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.122 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.122 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 40123 2023-10-09T09:35:10.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.122 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012436+0000", 2023-10-09T09:35:10.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012436+0000", 2023-10-09T09:35:10.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:35:10.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012434+0000", 2023-10-09T09:35:10.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4212000000000003e-05 2023-10-09T09:35:10.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012478+0000", 2023-10-09T09:35:10.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.912e-05 2023-10-09T09:35:10.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012497+0000", 2023-10-09T09:35:10.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1849999999999998e-06 2023-10-09T09:35:10.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012504+0000", 2023-10-09T09:35:10.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8498000000000002e-05 2023-10-09T09:35:10.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012553+0000", 2023-10-09T09:35:10.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000129844 2023-10-09T09:35:10.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012683+0000", 2023-10-09T09:35:10.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016849500000000001 2023-10-09T09:35:10.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.012851+0000", 2023-10-09T09:35:10.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015449579999999999 2023-10-09T09:35:10.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.014396+0000", 2023-10-09T09:35:10.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000132096 2023-10-09T09:35:10.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.014528+0000", 2023-10-09T09:35:10.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.138627308 2023-10-09T09:35:10.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.153156+0000", 2023-10-09T09:35:10.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2577000000000003e-05 2023-10-09T09:35:10.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.153188+0000", 2023-10-09T09:35:10.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6643000000000001e-05 2023-10-09T09:35:10.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:26:06.153225+0000", 2023-10-09T09:35:10.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14078888000000001 2023-10-09T09:35:10.134 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.134 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.135 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.136 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:153515 6.7 6:ea0709c3:::benchmark_data_smithi163_107771_object42755:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1091)", 2023-10-09T09:35:10.136 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:10.944334+0000", 2023-10-09T09:35:10.136 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 419.14303824299998, 2023-10-09T09:35:10.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.122766212, 2023-10-09T09:35:10.136 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.137 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.137 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.137 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.137 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.137 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 153515 2023-10-09T09:35:10.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.138 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944334+0000", 2023-10-09T09:35:10.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944334+0000", 2023-10-09T09:35:10.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999876 2023-10-09T09:35:10.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944322+0000", 2023-10-09T09:35:10.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9488e-05 2023-10-09T09:35:10.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944341+0000", 2023-10-09T09:35:10.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3935e-05 2023-10-09T09:35:10.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944355+0000", 2023-10-09T09:35:10.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4540000000000004e-06 2023-10-09T09:35:10.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944362+0000", 2023-10-09T09:35:10.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4743000000000002e-05 2023-10-09T09:35:10.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944406+0000", 2023-10-09T09:35:10.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000159524 2023-10-09T09:35:10.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944566+0000", 2023-10-09T09:35:10.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8062999999999999e-05 2023-10-09T09:35:10.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944664+0000", 2023-10-09T09:35:10.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012622991 2023-10-09T09:35:10.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.957287+0000", 2023-10-09T09:35:10.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6511999999999997e-05 2023-10-09T09:35:10.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.957343+0000", 2023-10-09T09:35:10.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.109661482 2023-10-09T09:35:10.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:11.067005+0000", 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2287000000000003e-05 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:11.067057+0000", 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3259e-05 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:11.067100+0000", 2023-10-09T09:35:10.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.122766212 2023-10-09T09:35:10.154 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.154 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.154 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.154 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:153516 6.7 6:e12581d0:::benchmark_data_smithi163_107771_object42756:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1091)", 2023-10-09T09:35:10.154 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:10.944373+0000", 2023-10-09T09:35:10.155 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 419.142999223, 2023-10-09T09:35:10.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.122835634, 2023-10-09T09:35:10.155 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.155 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.155 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.155 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.155 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.155 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 153516 2023-10-09T09:35:10.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.155 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944373+0000", 2023-10-09T09:35:10.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944373+0000", 2023-10-09T09:35:10.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:35:10.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944373+0000", 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5600000000000001e-06 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944375+0000", 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5709999999999999e-06 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944380+0000", 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0019999999999998e-06 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.944383+0000", 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046390759999999998 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.949022+0000", 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012649199999999999 2023-10-09T09:35:10.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.949148+0000", 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118928 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.949267+0000", 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0080313580000000006 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.957299+0000", 2023-10-09T09:35:10.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000387363 2023-10-09T09:35:10.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:10.957686+0000", 2023-10-09T09:35:10.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10944991900000001 2023-10-09T09:35:10.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:11.067136+0000", 2023-10-09T09:35:10.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9236999999999997e-05 2023-10-09T09:35:10.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:11.067175+0000", 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3574000000000001e-05 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:11.067209+0000", 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.122835634 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.163 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:181361 6.7 6:eb2a43c8:::benchmark_data_smithi163_107771_object70601:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1093)", 2023-10-09T09:35:10.163 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:28:51.074001+0000", 2023-10-09T09:35:10.163 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 379.01337105499999, 2023-10-09T09:35:10.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.141165544, 2023-10-09T09:35:10.163 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.163 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.164 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.164 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.164 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.164 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181361 2023-10-09T09:35:10.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.164 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.074001+0000", 2023-10-09T09:35:10.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.074001+0000", 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.074000+0000", 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1960000000000008e-06 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.074008+0000", 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5829e-05 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.074024+0000", 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1710000000000002e-06 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.074032+0000", 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3322000000000003e-05 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.074075+0000", 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2134999999999998e-05 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.074167+0000", 2023-10-09T09:35:10.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4621000000000001e-05 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.074262+0000", 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043312740000000004 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.078593+0000", 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00070158799999999995 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.079295+0000", 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.135629741 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.214925+0000", 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014023400000000001 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.215065+0000", 2023-10-09T09:35:10.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102116 2023-10-09T09:35:10.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:28:51.215167+0000", 2023-10-09T09:35:10.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.141165544 2023-10-09T09:35:10.170 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.170 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.171 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.171 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:208313 6.7 6:e25da8a2:::benchmark_data_smithi163_107771_object97553:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:35:10.171 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:29.942013+0000", 2023-10-09T09:35:10.171 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 340.14535908200003, 2023-10-09T09:35:10.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14806444699999999, 2023-10-09T09:35:10.171 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.171 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.172 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.172 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.172 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.172 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 208313 2023-10-09T09:35:10.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.172 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942013+0000", 2023-10-09T09:35:10.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942013+0000", 2023-10-09T09:35:10.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:35:10.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942012+0000", 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7680000000000003e-06 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942019+0000", 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.747e-06 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942029+0000", 2023-10-09T09:35:10.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7389999999999996e-06 2023-10-09T09:35:10.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.942034+0000", 2023-10-09T09:35:10.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018005574 2023-10-09T09:35:10.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.960040+0000", 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012454299999999999 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.960165+0000", 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013029099999999999 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.960295+0000", 2023-10-09T09:35:10.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014071159999999999 2023-10-09T09:35:10.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.974366+0000", 2023-10-09T09:35:10.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024670809999999999 2023-10-09T09:35:10.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.976833+0000", 2023-10-09T09:35:10.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.11307982900000001 2023-10-09T09:35:10.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.089913+0000", 2023-10-09T09:35:10.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8105999999999999e-05 2023-10-09T09:35:10.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.090011+0000", 2023-10-09T09:35:10.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6769999999999999e-05 2023-10-09T09:35:10.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.090078+0000", 2023-10-09T09:35:10.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14806444699999999 2023-10-09T09:35:10.180 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.180 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.180 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.181 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:208317 6.7 6:e117d1ec:::benchmark_data_smithi163_107771_object97557:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:35:10.181 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:29.951724+0000", 2023-10-09T09:35:10.181 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 340.13564824100001, 2023-10-09T09:35:10.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.167366023, 2023-10-09T09:35:10.181 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.181 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.181 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.182 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.182 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.182 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 208317 2023-10-09T09:35:10.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.182 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.951724+0000", 2023-10-09T09:35:10.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.951724+0000", 2023-10-09T09:35:10.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.951723+0000", 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8730000000000007e-06 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.951730+0000", 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3755e-05 2023-10-09T09:35:10.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.951744+0000", 2023-10-09T09:35:10.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9999999999999999e-06 2023-10-09T09:35:10.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.951751+0000", 2023-10-09T09:35:10.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016911157 2023-10-09T09:35:10.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.968662+0000", 2023-10-09T09:35:10.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112496 2023-10-09T09:35:10.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.968775+0000", 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000137286 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.968912+0000", 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082825260000000001 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.977195+0000", 2023-10-09T09:35:10.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014545000000000001 2023-10-09T09:35:10.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.189 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:10.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.977340+0000", 2023-10-09T09:35:10.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14161644000000001 2023-10-09T09:35:10.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.118957+0000", 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2297000000000003e-05 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.119029+0000", 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1365000000000006e-05 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.119090+0000", 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.167366023 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:208339 6.7 6:e650960f:::benchmark_data_smithi163_107771_object97579:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:29.983037+0000", 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 340.10433501599999, 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.136234615, 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.192 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 208339 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.983037+0000", 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.983037+0000", 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.983035+0000", 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7800000000000006e-06 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.983044+0000", 2023-10-09T09:35:10.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8437e-05 2023-10-09T09:35:10.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.983063+0000", 2023-10-09T09:35:10.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9809999999999997e-06 2023-10-09T09:35:10.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.983070+0000", 2023-10-09T09:35:10.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4719000000000001e-05 2023-10-09T09:35:10.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.983144+0000", 2023-10-09T09:35:10.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016421900000000001 2023-10-09T09:35:10.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.983308+0000", 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000195997 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.983504+0000", 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010036916 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.993541+0000", 2023-10-09T09:35:10.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00042001799999999998 2023-10-09T09:35:10.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.993961+0000", 2023-10-09T09:35:10.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.125207507 2023-10-09T09:35:10.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.119169+0000", 2023-10-09T09:35:10.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6054000000000002e-05 2023-10-09T09:35:10.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.119225+0000", 2023-10-09T09:35:10.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.702e-05 2023-10-09T09:35:10.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.119272+0000", 2023-10-09T09:35:10.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.136234615 2023-10-09T09:35:10.200 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.200 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:208342 6.7 6:ebb267cc:::benchmark_data_smithi163_107771_object97582:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:29.988638+0000", 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 340.09873431699998, 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13079057399999999, 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.201 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 208342 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.988638+0000", 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.988638+0000", 2023-10-09T09:35:10.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.988636+0000", 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9910000000000008e-06 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.988645+0000", 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4819e-05 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.988660+0000", 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5140000000000004e-06 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.988667+0000", 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1890000000000005e-05 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.988729+0000", 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011767 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.988847+0000", 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000144757 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.988992+0000", 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0047765810000000002 2023-10-09T09:35:10.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.206 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.206 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.993768+0000", 2023-10-09T09:35:10.206 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00027557699999999999 2023-10-09T09:35:10.206 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.206 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:29.994044+0000", 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12528296799999999 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.119327+0000", 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5686000000000002e-05 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.119382+0000", 2023-10-09T09:35:10.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6159000000000003e-05 2023-10-09T09:35:10.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.208 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.208 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:30.119429+0000", 2023-10-09T09:35:10.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13079057399999999 2023-10-09T09:35:10.208 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.208 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.208 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.209 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.34791.0:209482 6.7 6:e93e803d:::benchmark_data_smithi163_107771_object98722:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1095)", 2023-10-09T09:35:10.209 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:29:31.672967+0000", 2023-10-09T09:35:10.209 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 338.41440528599998, 2023-10-09T09:35:10.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16171888500000001, 2023-10-09T09:35:10.209 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.209 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.209 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.209 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.34791", 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2292012769", 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 209482 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.672967+0000", 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.672967+0000", 2023-10-09T09:35:10.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:35:10.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.672965+0000", 2023-10-09T09:35:10.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1545999999999999e-05 2023-10-09T09:35:10.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.672976+0000", 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0625999999999999e-05 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.672997+0000", 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5920000000000001e-06 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.673006+0000", 2023-10-09T09:35:10.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00168933 2023-10-09T09:35:10.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.674695+0000", 2023-10-09T09:35:10.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013314499999999999 2023-10-09T09:35:10.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.674828+0000", 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000157844 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.674986+0000", 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0087915560000000007 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.683778+0000", 2023-10-09T09:35:10.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016026598999999999 2023-10-09T09:35:10.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.699804+0000", 2023-10-09T09:35:10.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13480119400000001 2023-10-09T09:35:10.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.834605+0000", 2023-10-09T09:35:10.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5652000000000003e-05 2023-10-09T09:35:10.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.834651+0000", 2023-10-09T09:35:10.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5046999999999998e-05 2023-10-09T09:35:10.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.217 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:29:31.834686+0000", 2023-10-09T09:35:10.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16171888500000001 2023-10-09T09:35:10.217 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.217 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.217 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.218 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:157661 7.0 7:03490724:::benchmark_data_smithi163_130508_object48998:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1119)", 2023-10-09T09:35:10.218 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:10.628452+0000", 2023-10-09T09:35:10.218 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 119.45892007099999, 2023-10-09T09:35:10.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18390770000000001, 2023-10-09T09:35:10.218 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.218 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.219 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.219 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:35:10.219 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:35:10.219 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 157661 2023-10-09T09:35:10.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.219 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628452+0000", 2023-10-09T09:35:10.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628452+0000", 2023-10-09T09:35:10.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:35:10.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628450+0000", 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2649e-05 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628462+0000", 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1912e-05 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628484+0000", 2023-10-09T09:35:10.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8519999999999999e-06 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628494+0000", 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4939999999999998e-05 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628559+0000", 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124454 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628684+0000", 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4221000000000006e-05 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628768+0000", 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025072129999999999 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.631275+0000", 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000171702 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.631447+0000", 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.180681112 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.812128+0000", 2023-10-09T09:35:10.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012908799999999999 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.812257+0000", 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103148 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.812360+0000", 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18390770000000001 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.225 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:193714 7.0 7:0582e4cd:::benchmark_data_smithi163_130508_object85051:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:51.443643+0000", 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 78.643729327000003, 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.135416231, 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:35:10.226 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 193714 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.443643+0000", 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.443643+0000", 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:35:10.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.443641+0000", 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3259999999999996e-06 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.443650+0000", 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7391e-05 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.443667+0000", 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0409999999999998e-06 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.443675+0000", 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1437000000000002e-05 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.443727+0000", 2023-10-09T09:35:10.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011914299999999999 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.443846+0000", 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012461699999999999 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.443970+0000", 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0035845120000000002 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.447555+0000", 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00042792399999999999 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.447983+0000", 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13080636500000001 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.578789+0000", 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138384 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.578927+0000", 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013181099999999999 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:51.579059+0000", 2023-10-09T09:35:10.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.135416231 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:194883 7.0 7:02575de7:::benchmark_data_smithi163_130508_object86220:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:52.660179+0000", 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 77.427193470999995, 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.123099952, 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.233 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 194883 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660179+0000", 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660179+0000", 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660177+0000", 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1034e-05 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660188+0000", 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3669000000000001e-05 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660202+0000", 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0759999999999999e-06 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660208+0000", 2023-10-09T09:35:10.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2210000000000002e-05 2023-10-09T09:35:10.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660280+0000", 2023-10-09T09:35:10.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000205307 2023-10-09T09:35:10.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660485+0000", 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018126000000000001 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660667+0000", 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066337039999999998 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.667300+0000", 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010890100000000001 2023-10-09T09:35:10.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.667409+0000", 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.115775882 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.783185+0000", 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3662999999999998e-05 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.783239+0000", 2023-10-09T09:35:10.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0164999999999999e-05 2023-10-09T09:35:10.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.783279+0000", 2023-10-09T09:35:10.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.123099952 2023-10-09T09:35:10.240 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.240 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:194887 7.0 7:06debc10:::benchmark_data_smithi163_130508_object86224:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:52.660604+0000", 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 77.426768379999999, 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.124012334, 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:35:10.241 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:35:10.242 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 194887 2023-10-09T09:35:10.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.242 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660604+0000", 2023-10-09T09:35:10.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660604+0000", 2023-10-09T09:35:10.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:35:10.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660603+0000", 2023-10-09T09:35:10.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9820000000000008e-06 2023-10-09T09:35:10.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660611+0000", 2023-10-09T09:35:10.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2388000000000001e-05 2023-10-09T09:35:10.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660623+0000", 2023-10-09T09:35:10.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2640000000000002e-06 2023-10-09T09:35:10.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.660630+0000", 2023-10-09T09:35:10.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002987754 2023-10-09T09:35:10.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.663618+0000", 2023-10-09T09:35:10.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000143984 2023-10-09T09:35:10.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.663762+0000", 2023-10-09T09:35:10.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000158214 2023-10-09T09:35:10.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:35:10.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.663920+0000", 2023-10-09T09:35:10.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003584491 2023-10-09T09:35:10.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.667505+0000", 2023-10-09T09:35:10.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019390290000000001 2023-10-09T09:35:10.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.669444+0000", 2023-10-09T09:35:10.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.115056809 2023-10-09T09:35:10.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.784501+0000", 2023-10-09T09:35:10.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4636999999999997e-05 2023-10-09T09:35:10.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.784575+0000", 2023-10-09T09:35:10.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0954000000000001e-05 2023-10-09T09:35:10.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.784616+0000", 2023-10-09T09:35:10.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.124012334 2023-10-09T09:35:10.250 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.250 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.250 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.250 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:194916 7.0 7:068432dc:::benchmark_data_smithi163_130508_object86253:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:35:10.250 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:52.793913+0000", 2023-10-09T09:35:10.251 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 77.293459647999995, 2023-10-09T09:35:10.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.125087854, 2023-10-09T09:35:10.251 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.251 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.251 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.251 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:35:10.251 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:35:10.251 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 194916 2023-10-09T09:35:10.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.252 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.252 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.252 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.793913+0000", 2023-10-09T09:35:10.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.793913+0000", 2023-10-09T09:35:10.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:35:10.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.793911+0000", 2023-10-09T09:35:10.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1447000000000001e-05 2023-10-09T09:35:10.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.793922+0000", 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1478999999999999e-05 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.793943+0000", 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0081999999999999e-05 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.793954+0000", 2023-10-09T09:35:10.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5098000000000006e-05 2023-10-09T09:35:10.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.794019+0000", 2023-10-09T09:35:10.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015210399999999999 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.794171+0000", 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117487 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.794288+0000", 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039064859999999998 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.798195+0000", 2023-10-09T09:35:10.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00096527199999999998 2023-10-09T09:35:10.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.799160+0000", 2023-10-09T09:35:10.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.119605234 2023-10-09T09:35:10.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.918765+0000", 2023-10-09T09:35:10.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012239400000000001 2023-10-09T09:35:10.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.918888+0000", 2023-10-09T09:35:10.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113021 2023-10-09T09:35:10.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.259 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.919001+0000", 2023-10-09T09:35:10.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.125087854 2023-10-09T09:35:10.259 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.259 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.259 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.259 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.259 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.259 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:194927 7.0 7:077d0ded:::benchmark_data_smithi163_130508_object86264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:35:10.259 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:52.807663+0000", 2023-10-09T09:35:10.260 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 77.279709883999999, 2023-10-09T09:35:10.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12199666000000001, 2023-10-09T09:35:10.260 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.260 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.260 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.260 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:35:10.260 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:35:10.260 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 194927 2023-10-09T09:35:10.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.260 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.807663+0000", 2023-10-09T09:35:10.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.807663+0000", 2023-10-09T09:35:10.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999952 2023-10-09T09:35:10.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.807658+0000", 2023-10-09T09:35:10.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2463000000000001e-05 2023-10-09T09:35:10.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.807670+0000", 2023-10-09T09:35:10.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4452e-05 2023-10-09T09:35:10.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.807685+0000", 2023-10-09T09:35:10.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7250000000000002e-06 2023-10-09T09:35:10.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.807690+0000", 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4555000000000003e-05 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.807735+0000", 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010163200000000001 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.807836+0000", 2023-10-09T09:35:10.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1694999999999995e-05 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.807928+0000", 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052769410000000003 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.813205+0000", 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7462999999999997e-05 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.813243+0000", 2023-10-09T09:35:10.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.11630760499999999 2023-10-09T09:35:10.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.929550+0000", 2023-10-09T09:35:10.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1607999999999997e-05 2023-10-09T09:35:10.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.929612+0000", 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7382000000000001e-05 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:52.929659+0000", 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12199666000000001 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.267 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:198422 7.0 7:048bb4e3:::benchmark_data_smithi163_130508_object89759:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:56.896650+0000", 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 73.190722235999999, 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.134155513, 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:35:10.268 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 198422 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.896650+0000", 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.896650+0000", 2023-10-09T09:35:10.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999971 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.896648+0000", 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1576999999999999e-05 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.896659+0000", 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0159000000000001e-05 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.896679+0000", 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8439999999999993e-06 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.896689+0000", 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9812000000000003e-05 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.896749+0000", 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000110901 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.896860+0000", 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9465999999999999e-05 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.896959+0000", 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003124117 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.900083+0000", 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00054505700000000005 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.900628+0000", 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13004434400000001 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:57.030673+0000", 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0434999999999998e-05 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:57.030753+0000", 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2438999999999997e-05 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:57.030806+0000", 2023-10-09T09:35:10.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.134155513 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:198427 7.0 7:030d17c6:::benchmark_data_smithi163_130508_object89764:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:56.905550+0000", 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 73.181822675999996, 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12539276199999999, 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:10.275 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:35:10.276 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:35:10.276 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:35:10.276 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:35:10.276 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 198427 2023-10-09T09:35:10.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.276 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:10.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:10.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.905550+0000", 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.905550+0000", 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.905547+0000", 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1182000000000001e-05 2023-10-09T09:35:10.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:10.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.905558+0000", 2023-10-09T09:35:10.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0962999999999999e-05 2023-10-09T09:35:10.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:10.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.905579+0000", 2023-10-09T09:35:10.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0338e-05 2023-10-09T09:35:10.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:10.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.905590+0000", 2023-10-09T09:35:10.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8177000000000002e-05 2023-10-09T09:35:10.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:10.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.905648+0000", 2023-10-09T09:35:10.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015084199999999999 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.905799+0000", 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011903100000000001 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.905918+0000", 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027697059999999998 2023-10-09T09:35:10.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:35:10.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.908688+0000", 2023-10-09T09:35:10.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00056619400000000001 2023-10-09T09:35:10.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:56.909254+0000", 2023-10-09T09:35:10.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12159708900000001 2023-10-09T09:35:10.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:35:10.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:57.030851+0000", 2023-10-09T09:35:10.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8338999999999999e-05 2023-10-09T09:35:10.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:35:10.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:57.030899+0000", 2023-10-09T09:35:10.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3331999999999998e-05 2023-10-09T09:35:10.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:10.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:10.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:35:10.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:57.030942+0000", 2023-10-09T09:35:10.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12539276199999999 2023-10-09T09:35:10.283 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.283 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.283 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.283 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:10.284 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:10.284 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:35:10.311 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:10.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:10.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:10.415 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:10.445 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:10.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:10.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:10.548 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:10.561 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:10 smithi163 bash[149053]: debug 2023-10-09T09:35:10.419+0000 7f3332ad4700 1 -- 172.21.15.163:0/3996404871 <== mon.2 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564a53f284e0 con 0x564a53f83000 2023-10-09T09:35:10.571 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:10.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:10.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:10.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:10.698 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:10.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:10.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:10.803 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:10.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:10.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:10.904 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_3' created 2023-10-09T09:35:10.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:10.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:10 smithi181 bash[59345]: audit 2023-10-09T09:35:10.079905+0000 mon.a (mon.0) 1732 : audit 1 from='client.? 172.21.15.163:0/3391826661' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T09:35:10.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:10 smithi181 bash[59345]: audit 2023-10-09T09:35:10.507407+0000 mon.a (mon.0) 1733 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:10.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:10 smithi181 bash[59345]: audit 2023-10-09T09:35:10.508321+0000 mon.a (mon.0) 1734 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:10.962 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:10.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:10.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:10.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T09:35:11.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:11.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:11.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:11.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:11.218 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:10 smithi163 bash[149053]: debug 2023-10-09T09:35:10.974+0000 7f333bc5a640 -1 Falling back to public interface 2023-10-09T09:35:11.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:10 smithi163 bash[78107]: audit 2023-10-09T09:35:10.079905+0000 mon.a (mon.0) 1732 : audit 1 from='client.? 172.21.15.163:0/3391826661' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T09:35:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:10 smithi163 bash[78107]: audit 2023-10-09T09:35:10.507407+0000 mon.a (mon.0) 1733 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:10 smithi163 bash[78107]: audit 2023-10-09T09:35:10.508321+0000 mon.a (mon.0) 1734 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:10 smithi163 bash[75425]: audit 2023-10-09T09:35:10.079905+0000 mon.a (mon.0) 1732 : audit 1 from='client.? 172.21.15.163:0/3391826661' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T09:35:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:10 smithi163 bash[75425]: audit 2023-10-09T09:35:10.507407+0000 mon.a (mon.0) 1733 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:10 smithi163 bash[75425]: audit 2023-10-09T09:35:10.508321+0000 mon.a (mon.0) 1734 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:11.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:11.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:11.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:11.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:11.376 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:11.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:11.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:11.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:11.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:11.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:11.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:11.638 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:11.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:11.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:11.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:11.948 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:11.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:11.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:12.051 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:12.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:12.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:12.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:12.186 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:12.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:11 smithi163 bash[78107]: cluster 2023-10-09T09:35:10.893841+0000 osd.5 (osd.5) 686 : cluster 3 failed to encode map e1132 with expected crc 2023-10-09T09:35:12.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:11 smithi163 bash[78107]: cluster 2023-10-09T09:35:10.894109+0000 osd.7 (osd.7) 534 : cluster 3 failed to encode map e1132 with expected crc 2023-10-09T09:35:12.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:11 smithi163 bash[78107]: audit 2023-10-09T09:35:10.896148+0000 mon.a (mon.0) 1735 : audit 1 from='client.? 172.21.15.163:0/3391826661' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T09:35:12.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:11 smithi163 bash[78107]: cluster 2023-10-09T09:35:10.896227+0000 mon.a (mon.0) 1736 : cluster 0 osdmap e1132: 8 total, 7 up, 8 in 2023-10-09T09:35:12.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:11 smithi163 bash[78107]: cluster 2023-10-09T09:35:11.821520+0000 mgr.y (mgr.34107) 871 : cluster 0 pgmap v1604: 89 pgs: 10 creating+peering, 6 unknown, 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:12.196 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:11 smithi163 bash[75425]: cluster 2023-10-09T09:35:10.893841+0000 osd.5 (osd.5) 686 : cluster 3 failed to encode map e1132 with expected crc 2023-10-09T09:35:12.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:11 smithi163 bash[75425]: cluster 2023-10-09T09:35:10.894109+0000 osd.7 (osd.7) 534 : cluster 3 failed to encode map e1132 with expected crc 2023-10-09T09:35:12.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:11 smithi163 bash[75425]: audit 2023-10-09T09:35:10.896148+0000 mon.a (mon.0) 1735 : audit 1 from='client.? 172.21.15.163:0/3391826661' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T09:35:12.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:11 smithi163 bash[75425]: cluster 2023-10-09T09:35:10.896227+0000 mon.a (mon.0) 1736 : cluster 0 osdmap e1132: 8 total, 7 up, 8 in 2023-10-09T09:35:12.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:11 smithi163 bash[75425]: cluster 2023-10-09T09:35:11.821520+0000 mgr.y (mgr.34107) 871 : cluster 0 pgmap v1604: 89 pgs: 10 creating+peering, 6 unknown, 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:12.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:12.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:12.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:12.313 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:11 smithi181 bash[59345]: cluster 2023-10-09T09:35:10.893841+0000 osd.5 (osd.5) 686 : cluster 3 failed to encode map e1132 with expected crc 2023-10-09T09:35:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:11 smithi181 bash[59345]: cluster 2023-10-09T09:35:10.894109+0000 osd.7 (osd.7) 534 : cluster 3 failed to encode map e1132 with expected crc 2023-10-09T09:35:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:11 smithi181 bash[59345]: audit 2023-10-09T09:35:10.896148+0000 mon.a (mon.0) 1735 : audit 1 from='client.? 172.21.15.163:0/3391826661' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T09:35:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:11 smithi181 bash[59345]: cluster 2023-10-09T09:35:10.896227+0000 mon.a (mon.0) 1736 : cluster 0 osdmap e1132: 8 total, 7 up, 8 in 2023-10-09T09:35:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:11 smithi181 bash[59345]: cluster 2023-10-09T09:35:11.821520+0000 mgr.y (mgr.34107) 871 : cluster 0 pgmap v1604: 89 pgs: 10 creating+peering, 6 unknown, 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:12.334 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:12.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:12.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:12.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:12.621 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:12.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:12.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:12.724 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:12.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:12.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:12.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:12.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:12.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:12.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:12.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:12.991 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:13.131 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:12 smithi181 bash[59345]: cluster 2023-10-09T09:35:11.902725+0000 osd.5 (osd.5) 687 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.135 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:12 smithi181 bash[59345]: cluster 2023-10-09T09:35:11.904761+0000 mon.a (mon.0) 1737 : cluster 0 osdmap e1133: 8 total, 7 up, 8 in 2023-10-09T09:35:13.135 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:12 smithi181 bash[59345]: cluster 2023-10-09T09:35:11.905891+0000 osd.5 (osd.5) 688 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.135 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:12 smithi181 bash[59345]: cluster 2023-10-09T09:35:11.905941+0000 osd.7 (osd.7) 535 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.135 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:12 smithi181 bash[59345]: cluster 2023-10-09T09:35:11.907976+0000 osd.6 (osd.6) 414 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.136 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:12 smithi181 bash[59345]: cluster 2023-10-09T09:35:11.908041+0000 osd.4 (osd.4) 557 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.136 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:12 smithi181 bash[59345]: cluster 2023-10-09T09:35:11.909381+0000 osd.7 (osd.7) 536 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:13.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:13.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[75425]: cluster 2023-10-09T09:35:11.902725+0000 osd.5 (osd.5) 687 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[75425]: cluster 2023-10-09T09:35:11.904761+0000 mon.a (mon.0) 1737 : cluster 0 osdmap e1133: 8 total, 7 up, 8 in 2023-10-09T09:35:13.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[75425]: cluster 2023-10-09T09:35:11.905891+0000 osd.5 (osd.5) 688 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[75425]: cluster 2023-10-09T09:35:11.905941+0000 osd.7 (osd.7) 535 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[75425]: cluster 2023-10-09T09:35:11.907976+0000 osd.6 (osd.6) 414 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[75425]: cluster 2023-10-09T09:35:11.908041+0000 osd.4 (osd.4) 557 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[75425]: cluster 2023-10-09T09:35:11.909381+0000 osd.7 (osd.7) 536 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[78107]: cluster 2023-10-09T09:35:11.902725+0000 osd.5 (osd.5) 687 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[78107]: cluster 2023-10-09T09:35:11.904761+0000 mon.a (mon.0) 1737 : cluster 0 osdmap e1133: 8 total, 7 up, 8 in 2023-10-09T09:35:13.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[78107]: cluster 2023-10-09T09:35:11.905891+0000 osd.5 (osd.5) 688 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[78107]: cluster 2023-10-09T09:35:11.905941+0000 osd.7 (osd.7) 535 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[78107]: cluster 2023-10-09T09:35:11.907976+0000 osd.6 (osd.6) 414 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[78107]: cluster 2023-10-09T09:35:11.908041+0000 osd.4 (osd.4) 557 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:12 smithi163 bash[78107]: cluster 2023-10-09T09:35:11.909381+0000 osd.7 (osd.7) 536 : cluster 3 failed to encode map e1133 with expected crc 2023-10-09T09:35:13.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:13.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:13.412 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:13.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:13.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:13.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:13.549 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:13.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:13.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:13.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:13.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:13.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:35:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:35:13] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.33.4" 2023-10-09T09:35:13.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:13.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:13.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:13.924 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:13.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:13.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:13.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:14.044 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:14.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:14.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:14.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:14.109 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:13 smithi163 bash[75425]: cluster 2023-10-09T09:35:12.911648+0000 osd.5 (osd.5) 689 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:14.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:13 smithi163 bash[75425]: cluster 2023-10-09T09:35:12.914279+0000 mon.a (mon.0) 1738 : cluster 0 osdmap e1134: 8 total, 7 up, 8 in 2023-10-09T09:35:14.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:13 smithi163 bash[75425]: cluster 2023-10-09T09:35:13.822145+0000 mgr.y (mgr.34107) 872 : cluster 0 pgmap v1607: 89 pgs: 10 creating+peering, 6 unknown, 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:14.285 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:13 smithi163 bash[78107]: cluster 2023-10-09T09:35:12.911648+0000 osd.5 (osd.5) 689 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:14.285 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:13 smithi163 bash[78107]: cluster 2023-10-09T09:35:12.914279+0000 mon.a (mon.0) 1738 : cluster 0 osdmap e1134: 8 total, 7 up, 8 in 2023-10-09T09:35:14.285 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:13 smithi163 bash[78107]: cluster 2023-10-09T09:35:13.822145+0000 mgr.y (mgr.34107) 872 : cluster 0 pgmap v1607: 89 pgs: 10 creating+peering, 6 unknown, 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:14.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T09:35:14.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:14.312 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:14.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:14.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:14.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:13 smithi181 bash[59345]: cluster 2023-10-09T09:35:12.911648+0000 osd.5 (osd.5) 689 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:14.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:13 smithi181 bash[59345]: cluster 2023-10-09T09:35:12.914279+0000 mon.a (mon.0) 1738 : cluster 0 osdmap e1134: 8 total, 7 up, 8 in 2023-10-09T09:35:14.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:13 smithi181 bash[59345]: cluster 2023-10-09T09:35:13.822145+0000 mgr.y (mgr.34107) 872 : cluster 0 pgmap v1607: 89 pgs: 10 creating+peering, 6 unknown, 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:14.415 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:14.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:14.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:14.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:14.550 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:14.568 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:14.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:14.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:14.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:14.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:14.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:14.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:14.809 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:14.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:14.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:14.947 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:14.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:14.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:15.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:15.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[78107]: cluster 2023-10-09T09:35:14.433888+0000 osd.6 (osd.6) 415 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[78107]: cluster 2023-10-09T09:35:14.435011+0000 osd.6 (osd.6) 416 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[78107]: cluster 2023-10-09T09:35:14.541123+0000 osd.4 (osd.4) 558 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[78107]: cluster 2023-10-09T09:35:14.544086+0000 osd.4 (osd.4) 559 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[78107]: cluster 2023-10-09T09:35:14.660234+0000 osd.7 (osd.7) 537 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[78107]: cluster 2023-10-09T09:35:14.663117+0000 osd.7 (osd.7) 538 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[75425]: cluster 2023-10-09T09:35:14.433888+0000 osd.6 (osd.6) 415 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[75425]: cluster 2023-10-09T09:35:14.435011+0000 osd.6 (osd.6) 416 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[75425]: cluster 2023-10-09T09:35:14.541123+0000 osd.4 (osd.4) 558 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[75425]: cluster 2023-10-09T09:35:14.544086+0000 osd.4 (osd.4) 559 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[75425]: cluster 2023-10-09T09:35:14.660234+0000 osd.7 (osd.7) 537 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:14 smithi163 bash[75425]: cluster 2023-10-09T09:35:14.663117+0000 osd.7 (osd.7) 538 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:15.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:15.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:15.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:14 smithi181 bash[59345]: cluster 2023-10-09T09:35:14.433888+0000 osd.6 (osd.6) 415 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:14 smithi181 bash[59345]: cluster 2023-10-09T09:35:14.435011+0000 osd.6 (osd.6) 416 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:14 smithi181 bash[59345]: cluster 2023-10-09T09:35:14.541123+0000 osd.4 (osd.4) 558 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.198 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:14 smithi181 bash[59345]: cluster 2023-10-09T09:35:14.544086+0000 osd.4 (osd.4) 559 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.198 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:14 smithi181 bash[59345]: cluster 2023-10-09T09:35:14.660234+0000 osd.7 (osd.7) 537 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.198 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:14 smithi181 bash[59345]: cluster 2023-10-09T09:35:14.663117+0000 osd.7 (osd.7) 538 : cluster 3 failed to encode map e1134 with expected crc 2023-10-09T09:35:15.215 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:15.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:15.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:15.320 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:15.346 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:15.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:15.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:15.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:15.476 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:15.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:15.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:15.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:15.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:15.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:15.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:15.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:15.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:15.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:15.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:15.817 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:35:15.848 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:15.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:15.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:15.980 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:16.006 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:16.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:16.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:16.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:16.137 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:16.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:16.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:16.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:16.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:16.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:16.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:16.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:16.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:16.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:16.468 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:16 smithi163 bash[149053]: debug 2023-10-09T09:35:16.242+0000 7f333bc5a640 -1 osd.0 1120 log_to_monitors true 2023-10-09T09:35:16.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:16.534 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:16.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:16.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:16.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:16.638 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:35:16.639 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:16 smithi163 bash[75425]: cluster 2023-10-09T09:35:15.823377+0000 mgr.y (mgr.34107) 873 : cluster 0 pgmap v1608: 89 pgs: 10 creating+peering, 7 active+undersized+degraded, 21 active+undersized, 51 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:16.639 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:16 smithi163 bash[75425]: audit 2023-10-09T09:35:16.250750+0000 mon.c (mon.2) 72 : audit 1 from='osd.0 [v2:172.21.15.163:6802/3998822132,v1:172.21.15.163:6803/3998822132]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:35:16.639 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:16 smithi163 bash[75425]: audit 2023-10-09T09:35:16.251133+0000 mon.a (mon.0) 1739 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:35:16.639 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:16 smithi163 bash[75425]: audit 2023-10-09T09:35:16.601692+0000 mon.a (mon.0) 1740 : audit 1 from='client.? 172.21.15.163:0/1728243035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:35:16.663 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:16.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:16.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:16.765 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:16.776 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:16 smithi181 bash[59345]: cluster 2023-10-09T09:35:15.823377+0000 mgr.y (mgr.34107) 873 : cluster 0 pgmap v1608: 89 pgs: 10 creating+peering, 7 active+undersized+degraded, 21 active+undersized, 51 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:16.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:16 smithi181 bash[59345]: audit 2023-10-09T09:35:16.250750+0000 mon.c (mon.2) 72 : audit 1 from='osd.0 [v2:172.21.15.163:6802/3998822132,v1:172.21.15.163:6803/3998822132]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:35:16.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:16 smithi181 bash[59345]: audit 2023-10-09T09:35:16.251133+0000 mon.a (mon.0) 1739 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:35:16.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:16 smithi181 bash[59345]: audit 2023-10-09T09:35:16.601692+0000 mon.a (mon.0) 1740 : audit 1 from='client.? 172.21.15.163:0/1728243035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:35:16.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:16.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:16.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:16.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:16.926 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:16.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:16.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:16.968 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 09:35:16 smithi163 bash[149053]: debug 2023-10-09T09:35:16.690+0000 7f332a8c1700 -1 osd.0 1120 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:35:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:16 smithi163 bash[78107]: cluster 2023-10-09T09:35:15.823377+0000 mgr.y (mgr.34107) 873 : cluster 0 pgmap v1608: 89 pgs: 10 creating+peering, 7 active+undersized+degraded, 21 active+undersized, 51 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:16.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:16 smithi163 bash[78107]: audit 2023-10-09T09:35:16.250750+0000 mon.c (mon.2) 72 : audit 1 from='osd.0 [v2:172.21.15.163:6802/3998822132,v1:172.21.15.163:6803/3998822132]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:35:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:16 smithi163 bash[78107]: audit 2023-10-09T09:35:16.251133+0000 mon.a (mon.0) 1739 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:35:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:16 smithi163 bash[78107]: audit 2023-10-09T09:35:16.601692+0000 mon.a (mon.0) 1740 : audit 1 from='client.? 172.21.15.163:0/1728243035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:35:17.030 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:17.066 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:17.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:17.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:17.169 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:17.199 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:17.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:17.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:17.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:17.327 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:17.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:17.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:17.429 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:17.458 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:17.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:17.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:17.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:17.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:17.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:17.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:17.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:17.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:18.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:18.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[75425]: cluster 2023-10-09T09:35:16.629843+0000 mon.a (mon.0) 1741 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:35:18.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[75425]: cluster 2023-10-09T09:35:16.631643+0000 osd.5 (osd.5) 690 : cluster 3 failed to encode map e1135 with expected crc 2023-10-09T09:35:18.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[75425]: audit 2023-10-09T09:35:16.634039+0000 mon.a (mon.0) 1742 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:35:18.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[75425]: audit 2023-10-09T09:35:16.634102+0000 mon.a (mon.0) 1743 : audit 1 from='client.? 172.21.15.163:0/1728243035' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:35:18.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[75425]: cluster 2023-10-09T09:35:16.634271+0000 mon.a (mon.0) 1744 : cluster 0 osdmap e1135: 8 total, 7 up, 8 in 2023-10-09T09:35:18.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[75425]: audit 2023-10-09T09:35:16.637662+0000 mon.c (mon.2) 73 : audit 1 from='osd.0 [v2:172.21.15.163:6802/3998822132,v1:172.21.15.163:6803/3998822132]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:35:18.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[75425]: audit 2023-10-09T09:35:16.642171+0000 mon.a (mon.0) 1745 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:35:18.034 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[78107]: cluster 2023-10-09T09:35:16.629843+0000 mon.a (mon.0) 1741 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:35:18.034 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[78107]: cluster 2023-10-09T09:35:16.631643+0000 osd.5 (osd.5) 690 : cluster 3 failed to encode map e1135 with expected crc 2023-10-09T09:35:18.034 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[78107]: audit 2023-10-09T09:35:16.634039+0000 mon.a (mon.0) 1742 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:35:18.034 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[78107]: audit 2023-10-09T09:35:16.634102+0000 mon.a (mon.0) 1743 : audit 1 from='client.? 172.21.15.163:0/1728243035' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:35:18.034 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[78107]: cluster 2023-10-09T09:35:16.634271+0000 mon.a (mon.0) 1744 : cluster 0 osdmap e1135: 8 total, 7 up, 8 in 2023-10-09T09:35:18.034 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[78107]: audit 2023-10-09T09:35:16.637662+0000 mon.c (mon.2) 73 : audit 1 from='osd.0 [v2:172.21.15.163:6802/3998822132,v1:172.21.15.163:6803/3998822132]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:35:18.034 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:17 smithi163 bash[78107]: audit 2023-10-09T09:35:16.642171+0000 mon.a (mon.0) 1745 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:35:18.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:17 smithi181 bash[59345]: cluster 2023-10-09T09:35:16.629843+0000 mon.a (mon.0) 1741 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:35:18.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:17 smithi181 bash[59345]: cluster 2023-10-09T09:35:16.631643+0000 osd.5 (osd.5) 690 : cluster 3 failed to encode map e1135 with expected crc 2023-10-09T09:35:18.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:17 smithi181 bash[59345]: audit 2023-10-09T09:35:16.634039+0000 mon.a (mon.0) 1742 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:35:18.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:17 smithi181 bash[59345]: audit 2023-10-09T09:35:16.634102+0000 mon.a (mon.0) 1743 : audit 1 from='client.? 172.21.15.163:0/1728243035' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:35:18.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:17 smithi181 bash[59345]: cluster 2023-10-09T09:35:16.634271+0000 mon.a (mon.0) 1744 : cluster 0 osdmap e1135: 8 total, 7 up, 8 in 2023-10-09T09:35:18.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:17 smithi181 bash[59345]: audit 2023-10-09T09:35:16.637662+0000 mon.c (mon.2) 73 : audit 1 from='osd.0 [v2:172.21.15.163:6802/3998822132,v1:172.21.15.163:6803/3998822132]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:35:18.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:17 smithi181 bash[59345]: audit 2023-10-09T09:35:16.642171+0000 mon.a (mon.0) 1745 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:35:18.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:18.165 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:18.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:18.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:18.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:18.297 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:18.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:18.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:18.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:18.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:18.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:18.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:18.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:18.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:18.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:18.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:18.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:18.698 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:18.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:18.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:18.801 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:18.831 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:18.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:18.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:18.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[78107]: cluster 2023-10-09T09:35:17.638679+0000 mon.a (mon.0) 1746 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:35:18.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[78107]: cluster 2023-10-09T09:35:17.643602+0000 osd.5 (osd.5) 691 : cluster 3 failed to encode map e1136 with expected crc 2023-10-09T09:35:18.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[78107]: cluster 2023-10-09T09:35:17.648039+0000 mon.a (mon.0) 1747 : cluster 1 osd.0 [v2:172.21.15.163:6802/3998822132,v1:172.21.15.163:6803/3998822132] boot 2023-10-09T09:35:18.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[78107]: cluster 2023-10-09T09:35:17.648190+0000 mon.a (mon.0) 1748 : cluster 0 osdmap e1136: 8 total, 8 up, 8 in 2023-10-09T09:35:18.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[78107]: cluster 2023-10-09T09:35:17.650881+0000 osd.7 (osd.7) 539 : cluster 3 failed to encode map e1136 with expected crc 2023-10-09T09:35:18.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[78107]: cluster 2023-10-09T09:35:17.655408+0000 osd.4 (osd.4) 560 : cluster 3 failed to encode map e1136 with expected crc 2023-10-09T09:35:18.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[78107]: audit 2023-10-09T09:35:17.656767+0000 mon.a (mon.0) 1749 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:35:18.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[78107]: cluster 2023-10-09T09:35:17.824082+0000 mgr.y (mgr.34107) 874 : cluster 0 pgmap v1611: 89 pgs: 7 active+undersized+degraded, 25 active+undersized, 57 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:18.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[78107]: cluster 2023-10-09T09:35:18.650590+0000 mon.a (mon.0) 1750 : cluster 0 osdmap e1137: 8 total, 8 up, 8 in 2023-10-09T09:35:18.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[75425]: cluster 2023-10-09T09:35:17.638679+0000 mon.a (mon.0) 1746 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:35:18.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[75425]: cluster 2023-10-09T09:35:17.643602+0000 osd.5 (osd.5) 691 : cluster 3 failed to encode map e1136 with expected crc 2023-10-09T09:35:18.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[75425]: cluster 2023-10-09T09:35:17.648039+0000 mon.a (mon.0) 1747 : cluster 1 osd.0 [v2:172.21.15.163:6802/3998822132,v1:172.21.15.163:6803/3998822132] boot 2023-10-09T09:35:18.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[75425]: cluster 2023-10-09T09:35:17.648190+0000 mon.a (mon.0) 1748 : cluster 0 osdmap e1136: 8 total, 8 up, 8 in 2023-10-09T09:35:18.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[75425]: cluster 2023-10-09T09:35:17.650881+0000 osd.7 (osd.7) 539 : cluster 3 failed to encode map e1136 with expected crc 2023-10-09T09:35:18.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[75425]: cluster 2023-10-09T09:35:17.655408+0000 osd.4 (osd.4) 560 : cluster 3 failed to encode map e1136 with expected crc 2023-10-09T09:35:18.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[75425]: audit 2023-10-09T09:35:17.656767+0000 mon.a (mon.0) 1749 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:35:18.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[75425]: cluster 2023-10-09T09:35:17.824082+0000 mgr.y (mgr.34107) 874 : cluster 0 pgmap v1611: 89 pgs: 7 active+undersized+degraded, 25 active+undersized, 57 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:18.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:18 smithi163 bash[75425]: cluster 2023-10-09T09:35:18.650590+0000 mon.a (mon.0) 1750 : cluster 0 osdmap e1137: 8 total, 8 up, 8 in 2023-10-09T09:35:18.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:18 smithi181 bash[59345]: cluster 2023-10-09T09:35:17.638679+0000 mon.a (mon.0) 1746 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:35:18.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:18 smithi181 bash[59345]: cluster 2023-10-09T09:35:17.643602+0000 osd.5 (osd.5) 691 : cluster 3 failed to encode map e1136 with expected crc 2023-10-09T09:35:18.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:18 smithi181 bash[59345]: cluster 2023-10-09T09:35:17.648039+0000 mon.a (mon.0) 1747 : cluster 1 osd.0 [v2:172.21.15.163:6802/3998822132,v1:172.21.15.163:6803/3998822132] boot 2023-10-09T09:35:18.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:18 smithi181 bash[59345]: cluster 2023-10-09T09:35:17.648190+0000 mon.a (mon.0) 1748 : cluster 0 osdmap e1136: 8 total, 8 up, 8 in 2023-10-09T09:35:18.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:18 smithi181 bash[59345]: cluster 2023-10-09T09:35:17.650881+0000 osd.7 (osd.7) 539 : cluster 3 failed to encode map e1136 with expected crc 2023-10-09T09:35:18.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:18 smithi181 bash[59345]: cluster 2023-10-09T09:35:17.655408+0000 osd.4 (osd.4) 560 : cluster 3 failed to encode map e1136 with expected crc 2023-10-09T09:35:18.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:18 smithi181 bash[59345]: audit 2023-10-09T09:35:17.656767+0000 mon.a (mon.0) 1749 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:35:18.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:18 smithi181 bash[59345]: cluster 2023-10-09T09:35:17.824082+0000 mgr.y (mgr.34107) 874 : cluster 0 pgmap v1611: 89 pgs: 7 active+undersized+degraded, 25 active+undersized, 57 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:18.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:18 smithi181 bash[59345]: cluster 2023-10-09T09:35:18.650590+0000 mon.a (mon.0) 1750 : cluster 0 osdmap e1137: 8 total, 8 up, 8 in 2023-10-09T09:35:18.964 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:18.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:19.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:19.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:19.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:19.113 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:35:19.113 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:35:19.113 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T09:35:19.113 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:35:19.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:19.235 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:19.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:19.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:19.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:19.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:19.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:19.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:19.439 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:35:19.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:19.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:19.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:19.549 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:35:19.549 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:35:19.549 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T09:35:19.549 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:35:19.551 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:35:19.573 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:35:19.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:19.597 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:35:19.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:19.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:19.621 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:35:19.649 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:35:19.679 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:35:19.702 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:35:19.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:19 smithi163 bash[75425]: cluster 2023-10-09T09:35:18.645313+0000 osd.7 (osd.7) 540 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:19 smithi163 bash[75425]: cluster 2023-10-09T09:35:18.648030+0000 osd.5 (osd.5) 692 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:19 smithi163 bash[75425]: cluster 2023-10-09T09:35:18.651125+0000 osd.6 (osd.6) 417 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:19 smithi163 bash[75425]: cluster 2023-10-09T09:35:18.651248+0000 osd.4 (osd.4) 561 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:19 smithi163 bash[75425]: audit 2023-10-09T09:35:19.529805+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.163:0/3563697695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:35:19.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:19 smithi163 bash[78107]: cluster 2023-10-09T09:35:18.645313+0000 osd.7 (osd.7) 540 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:19 smithi163 bash[78107]: cluster 2023-10-09T09:35:18.648030+0000 osd.5 (osd.5) 692 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:19 smithi163 bash[78107]: cluster 2023-10-09T09:35:18.651125+0000 osd.6 (osd.6) 417 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:19 smithi163 bash[78107]: cluster 2023-10-09T09:35:18.651248+0000 osd.4 (osd.4) 561 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:19 smithi163 bash[78107]: audit 2023-10-09T09:35:19.529805+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.163:0/3563697695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:35:19.720 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:19.724 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:35:19.726 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:35:19.726 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:35:19.726 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:35:19.726 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:35:19.726 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:35:19.726 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:35:19.726 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:35:19.726 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:35:19.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:19.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:19.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:19.846 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:19.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:19.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:19.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:19.882 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T09:35:19.935 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:19 smithi181 bash[59345]: cluster 2023-10-09T09:35:18.645313+0000 osd.7 (osd.7) 540 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.935 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:19 smithi181 bash[59345]: cluster 2023-10-09T09:35:18.648030+0000 osd.5 (osd.5) 692 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.936 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:19 smithi181 bash[59345]: cluster 2023-10-09T09:35:18.651125+0000 osd.6 (osd.6) 417 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.936 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:19 smithi181 bash[59345]: cluster 2023-10-09T09:35:18.651248+0000 osd.4 (osd.4) 561 : cluster 3 failed to encode map e1137 with expected crc 2023-10-09T09:35:19.936 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:19 smithi181 bash[59345]: audit 2023-10-09T09:35:19.529805+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.163:0/3563697695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:35:19.979 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:20.004 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:20.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:20.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:20.107 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:20.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:20.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:20.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:20.238 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:20.270 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:20.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:20.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:20.372 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:20.403 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:20.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:20.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:20.507 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:20.548 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:20.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:20.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:20.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:20.682 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:20.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:20.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:20.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:20.816 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:20.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:20.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:20.920 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:20.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:20 smithi163 bash[75425]: cluster 2023-10-09T09:35:19.824807+0000 mgr.y (mgr.34107) 875 : cluster 0 pgmap v1613: 89 pgs: 7 active+undersized+degraded, 22 active+undersized, 60 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:20.934 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:20 smithi163 bash[78107]: cluster 2023-10-09T09:35:19.824807+0000 mgr.y (mgr.34107) 875 : cluster 0 pgmap v1613: 89 pgs: 7 active+undersized+degraded, 22 active+undersized, 60 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:20.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:20.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:20.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:20.973 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:35:21.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:20 smithi181 bash[59345]: cluster 2023-10-09T09:35:19.824807+0000 mgr.y (mgr.34107) 875 : cluster 0 pgmap v1613: 89 pgs: 7 active+undersized+degraded, 22 active+undersized, 60 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:35:21.088 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:21.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:21.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:21.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:21.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:21.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:21.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:21.329 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:21.363 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:21.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:21.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:21.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:21.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:21.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:21.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:21.601 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:21.627 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:21.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:21.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:21.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:21.760 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:21.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:21.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:21.863 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:21.898 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:21.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:21.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:21 smithi163 bash[75425]: cluster 2023-10-09T09:35:20.675012+0000 mon.a (mon.0) 1751 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T09:35:21.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:21 smithi163 bash[78107]: cluster 2023-10-09T09:35:20.675012+0000 mon.a (mon.0) 1751 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T09:35:22.002 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:21 smithi181 bash[59345]: cluster 2023-10-09T09:35:20.675012+0000 mon.a (mon.0) 1751 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T09:35:22.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:22.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:22.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:22.175 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:22.204 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:22.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:22.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:22.307 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:22.341 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:22.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:22.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:22.446 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:22.478 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:22.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:22.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:22.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:22.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:22.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:22.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:22.718 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:22.748 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:22.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:22.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:22.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:22 smithi181 bash[59345]: cluster 2023-10-09T09:35:21.825738+0000 mgr.y (mgr.34107) 876 : cluster 0 pgmap v1614: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:22.851 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:22.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:22.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:22.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:22.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:22 smithi163 bash[78107]: cluster 2023-10-09T09:35:21.825738+0000 mgr.y (mgr.34107) 876 : cluster 0 pgmap v1614: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:22.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:22 smithi163 bash[75425]: cluster 2023-10-09T09:35:21.825738+0000 mgr.y (mgr.34107) 876 : cluster 0 pgmap v1614: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:22.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:23.018 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:23.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:23.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:23.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:23.153 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:23.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:23.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:23.256 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:23.286 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:23.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:23.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:23.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:23.403 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:35:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:35:23] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.33.4" 2023-10-09T09:35:23.417 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:23.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:23.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:23.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:23.554 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:23.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:23.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:23.657 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:23.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:23.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:23.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:23.800 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:23.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:23 smithi181 bash[59345]: cluster 2023-10-09T09:35:22.683064+0000 mon.a (mon.0) 1752 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 24 pgs undersized) 2023-10-09T09:35:23.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:23.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:23.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:23.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:23.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:23 smithi163 bash[75425]: cluster 2023-10-09T09:35:22.683064+0000 mon.a (mon.0) 1752 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 24 pgs undersized) 2023-10-09T09:35:23.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:23 smithi163 bash[78107]: cluster 2023-10-09T09:35:22.683064+0000 mon.a (mon.0) 1752 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 24 pgs undersized) 2023-10-09T09:35:23.969 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:23.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:23.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:24.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:35:24.072 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:24.105 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:24.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:24.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:24.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:24.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:24.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:24.347 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:24.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:24.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:24.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:24.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:24.509 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:24.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:24.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:24.613 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:24.642 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:24.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:24.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:24.703 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:35:24.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:24.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:24.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:24.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:24.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:24.892 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:24 smithi163 bash[75425]: cluster 2023-10-09T09:35:23.826145+0000 mgr.y (mgr.34107) 877 : cluster 0 pgmap v1615: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:24.892 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:24 smithi163 bash[78107]: cluster 2023-10-09T09:35:23.826145+0000 mgr.y (mgr.34107) 877 : cluster 0 pgmap v1615: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:24.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:24.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:24.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:25.015 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:25.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:24 smithi181 bash[59345]: cluster 2023-10-09T09:35:23.826145+0000 mgr.y (mgr.34107) 877 : cluster 0 pgmap v1615: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:25.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:25.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:25.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:25.143 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:25.174 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:25.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:25.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:25.278 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:25.312 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:25.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:25.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:25.416 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:25.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:25.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:25.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:25.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:25.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:25.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:25.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:25.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:25.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:25.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:25.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:25.726 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T09:35:25.805 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:25.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:25 smithi181 bash[59345]: audit 2023-10-09T09:35:24.734073+0000 mon.a (mon.0) 1753 : audit 1 from='client.? 172.21.15.163:0/2553539070' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:35:25.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:25 smithi181 bash[59345]: audit 2023-10-09T09:35:25.380062+0000 mon.a (mon.0) 1754 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:25.814 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:25 smithi181 bash[59345]: audit 2023-10-09T09:35:25.381394+0000 mon.a (mon.0) 1755 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:25.829 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:25.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:25.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:25.933 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:25.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:25 smithi163 bash[75425]: audit 2023-10-09T09:35:24.734073+0000 mon.a (mon.0) 1753 : audit 1 from='client.? 172.21.15.163:0/2553539070' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:35:25.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:25 smithi163 bash[75425]: audit 2023-10-09T09:35:25.380062+0000 mon.a (mon.0) 1754 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:25.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:25 smithi163 bash[75425]: audit 2023-10-09T09:35:25.381394+0000 mon.a (mon.0) 1755 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:25.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:25 smithi163 bash[78107]: audit 2023-10-09T09:35:24.734073+0000 mon.a (mon.0) 1753 : audit 1 from='client.? 172.21.15.163:0/2553539070' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:35:25.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:25 smithi163 bash[78107]: audit 2023-10-09T09:35:25.380062+0000 mon.a (mon.0) 1754 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:25.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:25 smithi163 bash[78107]: audit 2023-10-09T09:35:25.381394+0000 mon.a (mon.0) 1755 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:25.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:25.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:26.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:26.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:26.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:26.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:26.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:26.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:26.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:26.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:26.331 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:26.367 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:26.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:26.470 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:26.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:26.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:26.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:26.594 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:26.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:26.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:26.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:26.726 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:26.752 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:26.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:26.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:26 smithi181 bash[59345]: cluster 2023-10-09T09:35:25.717142+0000 osd.7 (osd.7) 541 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:26 smithi181 bash[59345]: cluster 2023-10-09T09:35:25.717160+0000 osd.5 (osd.5) 693 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:26 smithi181 bash[59345]: audit 2023-10-09T09:35:25.720912+0000 mon.a (mon.0) 1756 : audit 1 from='client.? 172.21.15.163:0/2553539070' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:35:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:26 smithi181 bash[59345]: cluster 2023-10-09T09:35:25.720982+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e1138: 8 total, 8 up, 8 in 2023-10-09T09:35:26.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:26 smithi181 bash[59345]: cluster 2023-10-09T09:35:25.826807+0000 mgr.y (mgr.34107) 878 : cluster 0 pgmap v1617: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:26.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:26.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:26.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:26 smithi163 bash[78107]: cluster 2023-10-09T09:35:25.717142+0000 osd.7 (osd.7) 541 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:26.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:26 smithi163 bash[78107]: cluster 2023-10-09T09:35:25.717160+0000 osd.5 (osd.5) 693 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:26 smithi163 bash[78107]: audit 2023-10-09T09:35:25.720912+0000 mon.a (mon.0) 1756 : audit 1 from='client.? 172.21.15.163:0/2553539070' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:35:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:26 smithi163 bash[78107]: cluster 2023-10-09T09:35:25.720982+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e1138: 8 total, 8 up, 8 in 2023-10-09T09:35:26.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:26 smithi163 bash[78107]: cluster 2023-10-09T09:35:25.826807+0000 mgr.y (mgr.34107) 878 : cluster 0 pgmap v1617: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:26 smithi163 bash[75425]: cluster 2023-10-09T09:35:25.717142+0000 osd.7 (osd.7) 541 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:26 smithi163 bash[75425]: cluster 2023-10-09T09:35:25.717160+0000 osd.5 (osd.5) 693 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:26.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:26 smithi163 bash[75425]: audit 2023-10-09T09:35:25.720912+0000 mon.a (mon.0) 1756 : audit 1 from='client.? 172.21.15.163:0/2553539070' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:35:26.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:26 smithi163 bash[75425]: cluster 2023-10-09T09:35:25.720982+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e1138: 8 total, 8 up, 8 in 2023-10-09T09:35:26.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:26 smithi163 bash[75425]: cluster 2023-10-09T09:35:25.826807+0000 mgr.y (mgr.34107) 878 : cluster 0 pgmap v1617: 89 pgs: 89 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:26.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:27.023 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:27.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:27.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:27.049 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:35:27.049 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:35:27.049 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:35:27.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T09:35:27.095 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:35:27.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:27.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:27.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:27.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:27.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:27.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:27.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:27.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:27.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:27.424 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:27.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:27.527 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:27.556 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:27.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:27.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:27.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:27.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:27.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:27.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:27.698 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T09:35:27.736 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:35:27.736 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:35:27.777 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T09:35:27.778 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:35:27.778 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_151167 2023-10-09T09:35:27.778 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:35:27.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:35:27.789 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:27.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:27.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:27.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:27.935 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:27.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:27 smithi163 bash[78107]: cluster 2023-10-09T09:35:26.080433+0000 osd.6 (osd.6) 418 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:27 smithi163 bash[78107]: cluster 2023-10-09T09:35:26.785405+0000 osd.4 (osd.4) 562 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:27.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:27 smithi163 bash[75425]: cluster 2023-10-09T09:35:26.080433+0000 osd.6 (osd.6) 418 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:27.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:27 smithi163 bash[75425]: cluster 2023-10-09T09:35:26.785405+0000 osd.4 (osd.4) 562 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:27.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:27 smithi181 bash[59345]: cluster 2023-10-09T09:35:26.080433+0000 osd.6 (osd.6) 418 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:27 smithi181 bash[59345]: cluster 2023-10-09T09:35:26.785405+0000 osd.4 (osd.4) 562 : cluster 3 failed to encode map e1138 with expected crc 2023-10-09T09:35:28.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:28.089 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:35:28.089 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:35:28.096 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:28.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:28.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:28.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:28.234 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:28.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:28.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:28.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:28.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:28.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:28.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:28.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:28.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:28.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:28.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:28.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:28.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:28.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:28.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:28.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:28.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:28.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:28.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:28.778 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 2932 2916 182.24 182.25 0.00330863 0.00546668 2023-10-09T09:35:28.870 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:28.897 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:28.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:28.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:29.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:29.012 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:28 smithi163 bash[78107]: cluster 2023-10-09T09:35:27.827679+0000 mgr.y (mgr.34107) 879 : cluster 0 pgmap v1618: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:29.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:28 smithi163 bash[75425]: cluster 2023-10-09T09:35:27.827679+0000 mgr.y (mgr.34107) 879 : cluster 0 pgmap v1618: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:29.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:29.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:29.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:28 smithi181 bash[59345]: cluster 2023-10-09T09:35:27.827679+0000 mgr.y (mgr.34107) 879 : cluster 0 pgmap v1618: 89 pgs: 89 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:35:29.132 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:29.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:29.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:29.224 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T09:35:29.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:29.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:29.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:29.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:29.392 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:29.422 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:29.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:29.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:29.525 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:29.558 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:29.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:29.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:29.695 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:29.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:29.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:29.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 5813 5797 181.143 180.062 0.00654269 0.00550727 2023-10-09T09:35:29.800 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:29.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:29.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:29.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:29.930 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:29.960 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:29.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:29 smithi163 bash[75425]: cluster 2023-10-09T09:35:29.828316+0000 mgr.y (mgr.34107) 880 : cluster 0 pgmap v1619: 89 pgs: 89 active+clean; 51 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 80 op/s 2023-10-09T09:35:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:29 smithi163 bash[78107]: cluster 2023-10-09T09:35:29.828316+0000 mgr.y (mgr.34107) 880 : cluster 0 pgmap v1619: 89 pgs: 89 active+clean; 51 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 80 op/s 2023-10-09T09:35:30.064 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:30.094 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:30.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:30.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:30.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:30.209 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:29 smithi181 bash[59345]: cluster 2023-10-09T09:35:29.828316+0000 mgr.y (mgr.34107) 880 : cluster 0 pgmap v1619: 89 pgs: 89 active+clean; 51 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 80 op/s 2023-10-09T09:35:30.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:30.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:30.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:30.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:30.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:30.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:30.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:30.466 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:30.492 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:30.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:30.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:30.595 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:30.626 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:30.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:30.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:30.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:30.759 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:30.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:30.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:30.778 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 8212 8196 170.736 149.938 0.00365762 0.00584576 2023-10-09T09:35:30.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:30.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:30.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:30.991 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:31.103 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:35:31.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:31.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:31.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:31.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:31.281 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:31.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:31.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:31.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:31.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:31.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:31.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:31.519 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:31.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:31.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:31.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:31.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:31.676 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:31.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:31.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:31.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:31.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 10774 10758 168.079 160.125 0.00349816 0.00593905 2023-10-09T09:35:31.803 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:31.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:31.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:31.861 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:31 smithi163 bash[75425]: audit 2023-10-09T09:35:30.855181+0000 mon.a (mon.0) 1758 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:31.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:31 smithi163 bash[75425]: cluster 2023-10-09T09:35:31.829435+0000 mgr.y (mgr.34107) 881 : cluster 0 pgmap v1620: 89 pgs: 89 active+clean; 521 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T09:35:31.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:31.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:31.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:31.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:32.037 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:32.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:31 smithi181 bash[59345]: audit 2023-10-09T09:35:30.855181+0000 mon.a (mon.0) 1758 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:32.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:31 smithi181 bash[59345]: cluster 2023-10-09T09:35:31.829435+0000 mgr.y (mgr.34107) 881 : cluster 0 pgmap v1620: 89 pgs: 89 active+clean; 521 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T09:35:32.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:32.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:32.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:32.186 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:31 smithi163 bash[78107]: audit 2023-10-09T09:35:30.855181+0000 mon.a (mon.0) 1758 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:32.186 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:31 smithi163 bash[78107]: cluster 2023-10-09T09:35:31.829435+0000 mgr.y (mgr.34107) 881 : cluster 0 pgmap v1620: 89 pgs: 89 active+clean; 521 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T09:35:32.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:32.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:32.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:32.302 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:32.326 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:32.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:32.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:32.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:32.463 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:32.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:32.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:32.567 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:32.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:32.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:32.702 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:32.734 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:32.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:32.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:32.778 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 13113 13097 163.698 146.188 0.00261336 0.00606968 2023-10-09T09:35:32.837 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:32.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:32.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:32.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:32.974 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:33.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:33.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:33.111 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:33.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:33.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:33.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:33.242 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:33.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:33.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:33.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:33.360 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:35:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:35:33] "GET /metrics HTTP/1.1" 200 36276 "" "Prometheus/2.33.4" 2023-10-09T09:35:33.373 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:33.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:33.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:33.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:33.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:33.540 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:33.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:33.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:33.606 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:35:33.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:33.678 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:33.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:33.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 15193 15177 158.08 130 0.00407682 0.00631886 2023-10-09T09:35:33.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:33.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:33.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:33.888 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:33 smithi163 bash[75425]: cluster 2023-10-09T09:35:33.829969+0000 mgr.y (mgr.34107) 882 : cluster 0 pgmap v1621: 89 pgs: 89 active+clean; 521 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T09:35:33.915 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:33.924 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:33 smithi181 bash[59345]: cluster 2023-10-09T09:35:33.829969+0000 mgr.y (mgr.34107) 882 : cluster 0 pgmap v1621: 89 pgs: 89 active+clean; 521 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T09:35:33.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:33.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:34.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:34.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:34.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:34.096 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:35:34.175 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:34.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:34.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:34.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:34.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:33 smithi163 bash[78107]: cluster 2023-10-09T09:35:33.829969+0000 mgr.y (mgr.34107) 882 : cluster 0 pgmap v1621: 89 pgs: 89 active+clean; 521 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T09:35:34.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:34.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:34.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:34.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:34.437 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:34.463 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:34.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:34.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:34.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:34.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:34.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:34.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:34.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:34.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:34.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:34.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:34.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 16 17248 17232 153.843 128.438 0.00647235 0.00649318 2023-10-09T09:35:34.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:34.855 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:34.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:34.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:34.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:34.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:34.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:34.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:35.096 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:35.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:35.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:35.225 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:35.253 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:35.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:35.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:35.357 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:35.381 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:35.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:35.484 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:35.505 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:35.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:35.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:35.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:35.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:35.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:35.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:35.739 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:35.767 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:35.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:35.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:35.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 19429 19413 151.651 136.312 0.00251277 0.00658602 2023-10-09T09:35:35.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:35.896 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:35.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:35.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:35.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:36.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:36.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:36.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:36.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:36.149 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:36.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:36.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:36.259 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:36.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:36.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:36.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:36.388 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:36.425 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:36.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:36.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:36.530 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:36.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:36.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:36.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:36.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:36 smithi163 bash[75425]: cluster 2023-10-09T09:35:35.830534+0000 mgr.y (mgr.34107) 883 : cluster 0 pgmap v1622: 89 pgs: 89 active+clean; 756 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T09:35:36.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:36.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:36.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:36.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:36.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 21105 21089 146.438 104.75 0.00672163 0.00682054 2023-10-09T09:35:36.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:36.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:36 smithi181 bash[59345]: cluster 2023-10-09T09:35:35.830534+0000 mgr.y (mgr.34107) 883 : cluster 0 pgmap v1622: 89 pgs: 89 active+clean; 756 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T09:35:36.832 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:36.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:36.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:36.900 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:36 smithi163 bash[78107]: cluster 2023-10-09T09:35:35.830534+0000 mgr.y (mgr.34107) 883 : cluster 0 pgmap v1622: 89 pgs: 89 active+clean; 756 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T09:35:36.937 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:36.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:36.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:36.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:37.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:37.099 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:37.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:37.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:37.231 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:37.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:37.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:37.363 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:37.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:37.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:37.466 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:37.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:37.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:37.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:37.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:37.630 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:37.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:37.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:37.668 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:35:37.734 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:37.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:37.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:37.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 22338 22322 139.5 77.0625 0.00375032 0.00716126 2023-10-09T09:35:37.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:37 smithi181 bash[59345]: audit 2023-10-09T09:35:37.157851+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.163:0/2316785438' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:35:37.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:37 smithi181 bash[59345]: audit 2023-10-09T09:35:37.158421+0000 mon.a (mon.0) 1759 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:35:37.863 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:37.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:37.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:37.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:37 smithi163 bash[75425]: audit 2023-10-09T09:35:37.157851+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.163:0/2316785438' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:35:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:37 smithi163 bash[75425]: audit 2023-10-09T09:35:37.158421+0000 mon.a (mon.0) 1759 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:35:37.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:37 smithi163 bash[78107]: audit 2023-10-09T09:35:37.157851+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.163:0/2316785438' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:35:37.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:37 smithi163 bash[78107]: audit 2023-10-09T09:35:37.158421+0000 mon.a (mon.0) 1759 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:35:37.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:38.025 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:38.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:38.129 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:38.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:38.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:38.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:38.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:38.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:38.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:38.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:38.423 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:38.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:38.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:38.529 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:38.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:38.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:38.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:38.690 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:38.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:38.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:38.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 24263 24247 137.754 120.312 0.00931303 0.00725257 2023-10-09T09:35:38.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:38.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:38 smithi181 bash[59345]: cluster 2023-10-09T09:35:37.648904+0000 mon.a (mon.0) 1760 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:35:38.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:38 smithi181 bash[59345]: cluster 2023-10-09T09:35:37.658100+0000 osd.5 (osd.5) 694 : cluster 3 failed to encode map e1139 with expected crc 2023-10-09T09:35:38.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:38 smithi181 bash[59345]: audit 2023-10-09T09:35:37.660115+0000 mon.a (mon.0) 1761 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:35:38.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:38 smithi181 bash[59345]: cluster 2023-10-09T09:35:37.660182+0000 mon.a (mon.0) 1762 : cluster 0 osdmap e1139: 8 total, 8 up, 8 in 2023-10-09T09:35:38.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:38 smithi181 bash[59345]: cluster 2023-10-09T09:35:37.660619+0000 osd.6 (osd.6) 419 : cluster 3 failed to encode map e1139 with expected crc 2023-10-09T09:35:38.803 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:38 smithi181 bash[59345]: cluster 2023-10-09T09:35:37.830926+0000 mgr.y (mgr.34107) 884 : cluster 0 pgmap v1624: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-10-09T09:35:38.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:38.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:38.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:38.932 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:38.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[78107]: cluster 2023-10-09T09:35:37.648904+0000 mon.a (mon.0) 1760 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:35:38.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[78107]: cluster 2023-10-09T09:35:37.658100+0000 osd.5 (osd.5) 694 : cluster 3 failed to encode map e1139 with expected crc 2023-10-09T09:35:38.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[78107]: audit 2023-10-09T09:35:37.660115+0000 mon.a (mon.0) 1761 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:35:38.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[78107]: cluster 2023-10-09T09:35:37.660182+0000 mon.a (mon.0) 1762 : cluster 0 osdmap e1139: 8 total, 8 up, 8 in 2023-10-09T09:35:38.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[78107]: cluster 2023-10-09T09:35:37.660619+0000 osd.6 (osd.6) 419 : cluster 3 failed to encode map e1139 with expected crc 2023-10-09T09:35:38.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[78107]: cluster 2023-10-09T09:35:37.830926+0000 mgr.y (mgr.34107) 884 : cluster 0 pgmap v1624: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-10-09T09:35:38.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[75425]: cluster 2023-10-09T09:35:37.648904+0000 mon.a (mon.0) 1760 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:35:38.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[75425]: cluster 2023-10-09T09:35:37.658100+0000 osd.5 (osd.5) 694 : cluster 3 failed to encode map e1139 with expected crc 2023-10-09T09:35:38.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[75425]: audit 2023-10-09T09:35:37.660115+0000 mon.a (mon.0) 1761 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:35:38.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[75425]: cluster 2023-10-09T09:35:37.660182+0000 mon.a (mon.0) 1762 : cluster 0 osdmap e1139: 8 total, 8 up, 8 in 2023-10-09T09:35:38.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[75425]: cluster 2023-10-09T09:35:37.660619+0000 osd.6 (osd.6) 419 : cluster 3 failed to encode map e1139 with expected crc 2023-10-09T09:35:38.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:38 smithi163 bash[75425]: cluster 2023-10-09T09:35:37.830926+0000 mgr.y (mgr.34107) 884 : cluster 0 pgmap v1624: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-10-09T09:35:38.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:38.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:38.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:39.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:39.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:39.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:39.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:39.198 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:39.223 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:39.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:39.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:39.327 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:39.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:39.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:39.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:39.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:39.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:39.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:39.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:39.580 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:39.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:39.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:39.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:39.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:39.740 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:39.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:39.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:39.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 26083 26067 135.753 113.75 0.0080677 0.00735849 2023-10-09T09:35:39.846 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:39.872 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:39.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:39.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:39.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:39 smithi163 bash[75425]: cluster 2023-10-09T09:35:39.831545+0000 mgr.y (mgr.34107) 885 : cluster 0 pgmap v1625: 89 pgs: 89 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-10-09T09:35:39.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:39 smithi163 bash[78107]: cluster 2023-10-09T09:35:39.831545+0000 mgr.y (mgr.34107) 885 : cluster 0 pgmap v1625: 89 pgs: 89 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-10-09T09:35:39.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:40.009 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:40.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:40.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:40.138 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:40.140 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:39 smithi181 bash[59345]: cluster 2023-10-09T09:35:39.831545+0000 mgr.y (mgr.34107) 885 : cluster 0 pgmap v1625: 89 pgs: 89 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-10-09T09:35:40.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:40.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:40.242 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:40.269 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:40.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:40.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:40.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:40.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:40.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:40.492 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:40.533 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:40.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:40.638 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:40.667 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:40.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:40.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:40.697 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:35:40.697 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:35:40.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:40.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 27808 27792 133.603 107.812 0.00847663 0.00747823 2023-10-09T09:35:40.806 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:40.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:40.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:40.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:40.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:40.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:40.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:41.048 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:41.058 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:40 smithi181 bash[59345]: audit 2023-10-09T09:35:40.373674+0000 mon.a (mon.0) 1763 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:41.073 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:41.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:41.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:41.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:41.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:40 smithi163 bash[75425]: audit 2023-10-09T09:35:40.373674+0000 mon.a (mon.0) 1763 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:41.185 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:40 smithi163 bash[78107]: audit 2023-10-09T09:35:40.373674+0000 mon.a (mon.0) 1763 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:41.201 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:41.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:41.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:41.305 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:41.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:41.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:41.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:41.440 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:41.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:41.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:41.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:41.577 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:41.603 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:41.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:41.706 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:41.734 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:41.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:41.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:41.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 29588 29572 132.006 111.25 0.0117497 0.00756888 2023-10-09T09:35:41.796 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T09:35:41.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:41.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:41.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:41.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:41.969 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:41.996 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:42.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:42.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:42.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:41 smithi181 bash[59345]: cluster 2023-10-09T09:35:41.832317+0000 mgr.y (mgr.34107) 886 : cluster 0 pgmap v1626: 89 pgs: 89 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T09:35:42.102 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:42.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:42.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:42.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:42.141 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:41 smithi163 bash[78107]: cluster 2023-10-09T09:35:41.832317+0000 mgr.y (mgr.34107) 886 : cluster 0 pgmap v1626: 89 pgs: 89 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T09:35:42.145 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:41 smithi163 bash[75425]: cluster 2023-10-09T09:35:41.832317+0000 mgr.y (mgr.34107) 886 : cluster 0 pgmap v1626: 89 pgs: 89 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T09:35:42.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:42.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:42.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:42.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:42.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:42.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:42.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:42.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:42.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:42.519 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:42.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:42.622 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:42.653 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:42.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:42.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:42.756 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:42.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 31390 31374 130.713 112.625 0.015675 0.00764305 2023-10-09T09:35:42.784 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:42.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:42.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:42.887 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:42.920 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:42.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:43.024 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:43.053 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:43.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:43.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:43.157 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:43.181 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:43.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:43.283 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:43.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:43.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:43.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:43.359 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:35:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:35:43] "GET /metrics HTTP/1.1" 200 36331 "" "Prometheus/2.33.4" 2023-10-09T09:35:43.420 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:43.459 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:43.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:43.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:43.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:43.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:43.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:43.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:43.693 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:43.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:43.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:43.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:43.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 15 32811 32796 128.098 88.875 0.00993252 0.00780083 2023-10-09T09:35:43.827 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:43.854 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:43.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:43.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:43.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:43.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:43 smithi163 bash[75425]: cluster 2023-10-09T09:35:43.833141+0000 mgr.y (mgr.34107) 887 : cluster 0 pgmap v1627: 89 pgs: 89 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T09:35:43.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:43 smithi163 bash[78107]: cluster 2023-10-09T09:35:43.833141+0000 mgr.y (mgr.34107) 887 : cluster 0 pgmap v1627: 89 pgs: 89 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T09:35:43.982 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:43.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:43.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:44.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:44.110 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:44.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:44.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:44.214 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:44.223 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:43 smithi181 bash[59345]: cluster 2023-10-09T09:35:43.833141+0000 mgr.y (mgr.34107) 887 : cluster 0 pgmap v1627: 89 pgs: 89 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T09:35:44.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:44.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:44.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:44.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:44.376 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:44.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:44.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:44.480 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:44.506 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:44.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:44.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:44.611 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:44.646 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:35:44.656 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:44.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:44.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:44.685 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:35:44.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:35:44.721 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:35:44.760 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:35:44.761 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:44.779 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 34426 34410 126.496 100.875 0.0148339 0.00789869 2023-10-09T09:35:44.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:44.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:44.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:44.801 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:35:44.829 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:35:44.852 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:35:44.873 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:35:44.874 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:35:44.875 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:35:44.875 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:35:44.875 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:35:44.875 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:35:44.875 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:35:44.875 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:35:44.875 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:35:44.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:44.917 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:44.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:44.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:44 smithi163 bash[78107]: audit 2023-10-09T09:35:44.604802+0000 mon.a (mon.0) 1764 : audit 0 from='client.? 172.21.15.163:0/1714341573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:35:44.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:44 smithi163 bash[75425]: audit 2023-10-09T09:35:44.604802+0000 mon.a (mon.0) 1764 : audit 0 from='client.? 172.21.15.163:0/1714341573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:35:45.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:45.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:45.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:45.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:45.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:44 smithi181 bash[59345]: audit 2023-10-09T09:35:44.604802+0000 mon.a (mon.0) 1764 : audit 0 from='client.? 172.21.15.163:0/1714341573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:35:45.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:45.178 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:45.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:45.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:45.309 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:45.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:45.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:45.412 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:45.441 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:45.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:45.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:45.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:45.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:45.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:45.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:45.711 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:45.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:45.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:45.780 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 16 35887 35871 124.541 91.3125 0.00426743 0.00802302 2023-10-09T09:35:45.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:45.844 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:45.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:45.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:45.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:45.977 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:45.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:45.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:46.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:46.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:46.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:46.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:46.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:46.236 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:46.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:46.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:46.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:46.372 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:46.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:46.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:46.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:46.506 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:46.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:46.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:46.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:46.646 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:46.648 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:35:46.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:46.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:46.753 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:46.780 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 37412 37396 123.002 95.3125 0.0109292 0.00812506 2023-10-09T09:35:46.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:46.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:46.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:46.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:46.900 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:46 smithi163 bash[78107]: cluster 2023-10-09T09:35:45.833695+0000 mgr.y (mgr.34107) 888 : cluster 0 pgmap v1628: 89 pgs: 89 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T09:35:46.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:46 smithi163 bash[75425]: cluster 2023-10-09T09:35:45.833695+0000 mgr.y (mgr.34107) 888 : cluster 0 pgmap v1628: 89 pgs: 89 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T09:35:46.933 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:46.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:46.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:47.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:47.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:46 smithi181 bash[59345]: cluster 2023-10-09T09:35:45.833695+0000 mgr.y (mgr.34107) 888 : cluster 0 pgmap v1628: 89 pgs: 89 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T09:35:47.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:47.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:47.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:47.207 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:47.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:47.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:47.311 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:47.352 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:47.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:47.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:47.456 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:47.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:47 smithi163 bash[75425]: debug 2023-10-09T09:35:47.214+0000 7f22f321a700 -1 mon.a@0(leader).osd e1139 definitely_dead 0 2023-10-09T09:35:47.492 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:47.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:47.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:47.599 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:47.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:47.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:47.687 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.2. 2023-10-09T09:35:47.734 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:47.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:47 smithi163 bash[78107]: audit 2023-10-09T09:35:47.218082+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.163:0/1463734820' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:35:47.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:47 smithi163 bash[78107]: audit 2023-10-09T09:35:47.218534+0000 mon.a (mon.0) 1765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:35:47.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:47 smithi163 bash[78107]: cluster 2023-10-09T09:35:47.218680+0000 mon.a (mon.0) 1766 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T09:35:47.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:47 smithi163 bash[75425]: audit 2023-10-09T09:35:47.218082+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.163:0/1463734820' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:35:47.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:47 smithi163 bash[75425]: audit 2023-10-09T09:35:47.218534+0000 mon.a (mon.0) 1765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:35:47.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:47 smithi163 bash[75425]: cluster 2023-10-09T09:35:47.218680+0000 mon.a (mon.0) 1766 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T09:35:47.762 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:47.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:47.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:47.780 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:35:47.780715+0000 min lat: 0.00132712 max lat: 0.279862 avg lat: 0.00821711 2023-10-09T09:35:47.780 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:35:47.780 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 38794 38778 121.17 86.375 0.00243979 0.00821711 2023-10-09T09:35:47.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:47.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:47.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:47.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:47.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:48.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:48.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:48.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:48.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:47 smithi181 bash[59345]: audit 2023-10-09T09:35:47.218082+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.163:0/1463734820' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:35:48.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:47 smithi181 bash[59345]: audit 2023-10-09T09:35:47.218534+0000 mon.a (mon.0) 1765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T09:35:48.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:47 smithi181 bash[59345]: cluster 2023-10-09T09:35:47.218680+0000 mon.a (mon.0) 1766 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T09:35:48.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:48.161 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:48.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:48.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:48.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:48.295 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:48.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:48.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:48.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:48.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:48.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:48.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:48.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:48.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:48.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:48.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:48.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:48.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:48.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:48.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:48.780 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 38794 38778 115.399 0 - 0.00821711 2023-10-09T09:35:48.814 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:48.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:48 smithi181 bash[59345]: cluster 2023-10-09T09:35:47.668483+0000 osd.4 (osd.4) 563 : cluster 3 failed to encode map e1140 with expected crc 2023-10-09T09:35:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:48 smithi181 bash[59345]: cluster 2023-10-09T09:35:47.673780+0000 osd.5 (osd.5) 695 : cluster 3 failed to encode map e1140 with expected crc 2023-10-09T09:35:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:48 smithi181 bash[59345]: audit 2023-10-09T09:35:47.677251+0000 mon.a (mon.0) 1767 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T09:35:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:48 smithi181 bash[59345]: cluster 2023-10-09T09:35:47.677435+0000 mon.a (mon.0) 1768 : cluster 0 osdmap e1140: 8 total, 8 up, 7 in 2023-10-09T09:35:48.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:48 smithi181 bash[59345]: cluster 2023-10-09T09:35:47.834355+0000 mgr.y (mgr.34107) 889 : cluster 0 pgmap v1630: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-10-09T09:35:48.840 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:48.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:48.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:48.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:48.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:48 smithi163 bash[78107]: cluster 2023-10-09T09:35:47.668483+0000 osd.4 (osd.4) 563 : cluster 3 failed to encode map e1140 with expected crc 2023-10-09T09:35:48.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:48 smithi163 bash[78107]: cluster 2023-10-09T09:35:47.673780+0000 osd.5 (osd.5) 695 : cluster 3 failed to encode map e1140 with expected crc 2023-10-09T09:35:48.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:48 smithi163 bash[78107]: audit 2023-10-09T09:35:47.677251+0000 mon.a (mon.0) 1767 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T09:35:48.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:48 smithi163 bash[78107]: cluster 2023-10-09T09:35:47.677435+0000 mon.a (mon.0) 1768 : cluster 0 osdmap e1140: 8 total, 8 up, 7 in 2023-10-09T09:35:48.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:48 smithi163 bash[78107]: cluster 2023-10-09T09:35:47.834355+0000 mgr.y (mgr.34107) 889 : cluster 0 pgmap v1630: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-10-09T09:35:48.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:48 smithi163 bash[75425]: cluster 2023-10-09T09:35:47.668483+0000 osd.4 (osd.4) 563 : cluster 3 failed to encode map e1140 with expected crc 2023-10-09T09:35:48.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:48 smithi163 bash[75425]: cluster 2023-10-09T09:35:47.673780+0000 osd.5 (osd.5) 695 : cluster 3 failed to encode map e1140 with expected crc 2023-10-09T09:35:48.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:48 smithi163 bash[75425]: audit 2023-10-09T09:35:47.677251+0000 mon.a (mon.0) 1767 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T09:35:48.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:48 smithi163 bash[75425]: cluster 2023-10-09T09:35:47.677435+0000 mon.a (mon.0) 1768 : cluster 0 osdmap e1140: 8 total, 8 up, 7 in 2023-10-09T09:35:48.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:48 smithi163 bash[75425]: cluster 2023-10-09T09:35:47.834355+0000 mgr.y (mgr.34107) 889 : cluster 0 pgmap v1630: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-10-09T09:35:48.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:48.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:48.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:49.066 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:49.092 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:49.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:49.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:49.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:49.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:35:49.230 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:49.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:49.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:49.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:49.361 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:49.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:49.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:49.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:49.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:49.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:49.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:49.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:49.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:49.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:49.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:49.745 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.681339+0000 osd.5 (osd.5) 696 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.745 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.685267+0000 osd.5 (osd.5) 697 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.745 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.685518+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e1141: 8 total, 8 up, 7 in 2023-10-09T09:35:49.746 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.686146+0000 osd.4 (osd.4) 564 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.746 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.686634+0000 osd.7 (osd.7) 542 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.746 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.686776+0000 osd.6 (osd.6) 420 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.746 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.687519+0000 osd.4 (osd.4) 565 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.746 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.687519+0000 osd.5 (osd.5) 698 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.746 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.690988+0000 osd.6 (osd.6) 421 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.746 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.691165+0000 osd.7 (osd.7) 543 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.747 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.691249+0000 osd.5 (osd.5) 699 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.747 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.693363+0000 osd.5 (osd.5) 700 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.747 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:49 smithi181 bash[59345]: cluster 2023-10-09T09:35:48.695288+0000 osd.5 (osd.5) 701 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.760 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:49.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:49.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:49.780 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 38794 38778 110.154 0 - 0.00821711 2023-10-09T09:35:49.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:49.886 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:49.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:49.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.681339+0000 osd.5 (osd.5) 696 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.685267+0000 osd.5 (osd.5) 697 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.685518+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e1141: 8 total, 8 up, 7 in 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.686146+0000 osd.4 (osd.4) 564 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.686634+0000 osd.7 (osd.7) 542 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.686776+0000 osd.6 (osd.6) 420 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.687519+0000 osd.4 (osd.4) 565 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.687519+0000 osd.5 (osd.5) 698 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.690988+0000 osd.6 (osd.6) 421 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.691165+0000 osd.7 (osd.7) 543 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.691249+0000 osd.5 (osd.5) 699 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.693363+0000 osd.5 (osd.5) 700 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[78107]: cluster 2023-10-09T09:35:48.695288+0000 osd.5 (osd.5) 701 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.681339+0000 osd.5 (osd.5) 696 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.685267+0000 osd.5 (osd.5) 697 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.685518+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e1141: 8 total, 8 up, 7 in 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.686146+0000 osd.4 (osd.4) 564 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.686634+0000 osd.7 (osd.7) 542 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.686776+0000 osd.6 (osd.6) 420 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.687519+0000 osd.4 (osd.4) 565 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.687519+0000 osd.5 (osd.5) 698 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.690988+0000 osd.6 (osd.6) 421 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.691165+0000 osd.7 (osd.7) 543 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.691249+0000 osd.5 (osd.5) 699 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.693363+0000 osd.5 (osd.5) 700 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:49 smithi163 bash[75425]: cluster 2023-10-09T09:35:48.695288+0000 osd.5 (osd.5) 701 : cluster 3 failed to encode map e1141 with expected crc 2023-10-09T09:35:49.991 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:50.030 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:50.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:50.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:50.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:50.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:50.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:50.274 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:50.311 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:50.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:50.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:50.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:50.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:50.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:50.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:50.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:50.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:50.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:50.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:50.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:50.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:50.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:50.780 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 39781 39765 108.046 20.5625 0.0207193 0.00924782 2023-10-09T09:35:50.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:50.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:50.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:50.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:50.935 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:50.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:50 smithi181 bash[59345]: cluster 2023-10-09T09:35:49.679056+0000 osd.5 (osd.5) 702 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:50 smithi181 bash[59345]: cluster 2023-10-09T09:35:49.682842+0000 osd.5 (osd.5) 703 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:50 smithi181 bash[59345]: cluster 2023-10-09T09:35:49.684012+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e1142: 8 total, 8 up, 7 in 2023-10-09T09:35:50.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:50 smithi181 bash[59345]: cluster 2023-10-09T09:35:49.685422+0000 osd.5 (osd.5) 704 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.945 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:50 smithi181 bash[59345]: cluster 2023-10-09T09:35:49.702647+0000 osd.4 (osd.4) 566 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.945 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:50 smithi181 bash[59345]: cluster 2023-10-09T09:35:49.796970+0000 osd.6 (osd.6) 422 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.945 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:50 smithi181 bash[59345]: cluster 2023-10-09T09:35:49.834831+0000 mgr.y (mgr.34107) 890 : cluster 0 pgmap v1633: 89 pgs: 3 remapped+peering, 86 active+clean; 2.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T09:35:50.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:50.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:50.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[78107]: cluster 2023-10-09T09:35:49.679056+0000 osd.5 (osd.5) 702 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[78107]: cluster 2023-10-09T09:35:49.682842+0000 osd.5 (osd.5) 703 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[78107]: cluster 2023-10-09T09:35:49.684012+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e1142: 8 total, 8 up, 7 in 2023-10-09T09:35:50.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[78107]: cluster 2023-10-09T09:35:49.685422+0000 osd.5 (osd.5) 704 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[78107]: cluster 2023-10-09T09:35:49.702647+0000 osd.4 (osd.4) 566 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[78107]: cluster 2023-10-09T09:35:49.796970+0000 osd.6 (osd.6) 422 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[78107]: cluster 2023-10-09T09:35:49.834831+0000 mgr.y (mgr.34107) 890 : cluster 0 pgmap v1633: 89 pgs: 3 remapped+peering, 86 active+clean; 2.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T09:35:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[75425]: cluster 2023-10-09T09:35:49.679056+0000 osd.5 (osd.5) 702 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[75425]: cluster 2023-10-09T09:35:49.682842+0000 osd.5 (osd.5) 703 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[75425]: cluster 2023-10-09T09:35:49.684012+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e1142: 8 total, 8 up, 7 in 2023-10-09T09:35:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[75425]: cluster 2023-10-09T09:35:49.685422+0000 osd.5 (osd.5) 704 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[75425]: cluster 2023-10-09T09:35:49.702647+0000 osd.4 (osd.4) 566 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[75425]: cluster 2023-10-09T09:35:49.796970+0000 osd.6 (osd.6) 422 : cluster 3 failed to encode map e1142 with expected crc 2023-10-09T09:35:50.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:50 smithi163 bash[75425]: cluster 2023-10-09T09:35:49.834831+0000 mgr.y (mgr.34107) 890 : cluster 0 pgmap v1633: 89 pgs: 3 remapped+peering, 86 active+clean; 2.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T09:35:51.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:51.098 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:51.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:51.202 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:51.225 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:51.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:51.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:51.329 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:51.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:51.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:51.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:51.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:51.489 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:51.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:51.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:51.592 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:51.606 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:35:51.630 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:51.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:51.733 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:51.768 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:51.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:51.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:51.780 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 40911 40895 106.487 70.625 0.00657374 0.00938156 2023-10-09T09:35:51.861 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:35:51.861 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:35:51.861 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.861 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.45122.0:40998 8.8 e1142/1132 8:195d8d6a:::benchmark_data_smithi163_151167_object40997:head v 1142'2556, mlcod=1142'2554)", 2023-10-09T09:35:51.861 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:51.859638+0000", 2023-10-09T09:35:51.861 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.00052794500000000004, 2023-10-09T09:35:51.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00055088299999999995, 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859638+0000", 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859638+0000", 2023-10-09T09:35:51.862 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:35:51.865 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.865 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.865 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:51.865 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859637+0000", 2023-10-09T09:35:51.865 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7033e-05 2023-10-09T09:35:51.865 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.865 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.865 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:51.865 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859664+0000", 2023-10-09T09:35:51.865 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2255e-05 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859676+0000", 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4589999999999997e-06 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859682+0000", 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7884000000000002e-05 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.866 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859760+0000", 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6315000000000001e-05 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859786+0000", 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000148412 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.45122.0:40997 8.c e1142/1136 8:300b4eca:::benchmark_data_smithi163_151167_object40996:head v 1142'2550, mlcod=1142'2548)", 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:51.858418+0000", 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0017479419999999999, 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018166759999999999, 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.858418+0000", 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:51.868 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.858418+0000", 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.858416+0000", 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8992000000000002e-05 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.858455+0000", 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8153000000000001e-05 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.858474+0000", 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1049999999999997e-06 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:51.870 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.858481+0000", 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0506000000000005e-05 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.858561+0000", 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7492999999999999e-05 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.858589+0000", 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017110899999999999 2023-10-09T09:35:51.871 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.45122.0:40986 8.d e1142/1141)", 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:51.859548+0000", 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.000617301, 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00072214200000000005, 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.872 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859548+0000", 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859548+0000", 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859547+0000", 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0351e-05 2023-10-09T09:35:51.873 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859597+0000", 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7663e-05 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859615+0000", 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.764e-06 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:35:51.874 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:35:51.875 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:51.859622+0000", 2023-10-09T09:35:51.875 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3532e-05 2023-10-09T09:35:51.875 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:51.875 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:35:51.875 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:51.875 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:35:51.875 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T09:35:51.875 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 3 2023-10-09T09:35:51.875 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:35:51.876 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:51.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:51.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:51.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:52.007 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:52.017 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:51 smithi181 bash[59345]: cluster 2023-10-09T09:35:51.890798+0000 mon.a (mon.0) 1771 : cluster 3 Health check failed: Degraded data redundancy: 4656/120327 objects degraded (3.869%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:35:52.039 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:52.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:52.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:52.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:52.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:52.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:52.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:51 smithi163 bash[78107]: cluster 2023-10-09T09:35:51.890798+0000 mon.a (mon.0) 1771 : cluster 3 Health check failed: Degraded data redundancy: 4656/120327 objects degraded (3.869%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:35:52.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:51 smithi163 bash[75425]: cluster 2023-10-09T09:35:51.890798+0000 mon.a (mon.0) 1771 : cluster 3 Health check failed: Degraded data redundancy: 4656/120327 objects degraded (3.869%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:35:52.283 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:52.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:52.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:52.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:52.417 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:52.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:52.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:52.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:52.553 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:52.577 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:52.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:52.679 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:52.708 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:52.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:52.781 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 42029 42013 105.022 69.875 0.0270913 0.00951614 2023-10-09T09:35:52.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:52.844 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:52.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:52.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:52.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:52.978 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:52.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:53.064 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T09:35:53.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:53.105 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:53.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:53.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:53.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:52 smithi163 bash[75425]: cluster 2023-10-09T09:35:51.839617+0000 mgr.y (mgr.34107) 891 : cluster 0 pgmap v1634: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s; 4656/120327 objects degraded (3.869%); 2373/120327 objects misplaced (1.972%); 1.0 MiB/s, 1 keys/s, 16 objects/s recovering 2023-10-09T09:35:53.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:52 smithi163 bash[78107]: cluster 2023-10-09T09:35:51.839617+0000 mgr.y (mgr.34107) 891 : cluster 0 pgmap v1634: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s; 4656/120327 objects degraded (3.869%); 2373/120327 objects misplaced (1.972%); 1.0 MiB/s, 1 keys/s, 16 objects/s recovering 2023-10-09T09:35:53.219 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:52 smithi181 bash[59345]: cluster 2023-10-09T09:35:51.839617+0000 mgr.y (mgr.34107) 891 : cluster 0 pgmap v1634: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s; 4656/120327 objects degraded (3.869%); 2373/120327 objects misplaced (1.972%); 1.0 MiB/s, 1 keys/s, 16 objects/s recovering 2023-10-09T09:35:53.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:53.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:53.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:53.348 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:53.381 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:53.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:53.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:53.487 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:53.517 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:53.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:53.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:53.621 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:53.638 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:35:53.650 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:53.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:53.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:53.666 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:35:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:35:53] "GET /metrics HTTP/1.1" 200 36331 "" "Prometheus/2.33.4" 2023-10-09T09:35:53.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:53.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:53.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:53.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:53.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 16 43099 43083 103.553 66.875 0.0309832 0.00964699 2023-10-09T09:35:53.882 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:53.909 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:53.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:53.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:54.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:54.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:54.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:54.092 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:35:54.092 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:35:54.093 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T09:35:54.093 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:35:54.093 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:35:54.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T09:35:54.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:54.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:54.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:54.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:54.277 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:54.302 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:54.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:54.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:54.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:54.433 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:54.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:54.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:54.537 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:54.559 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:54.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:54.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:54.701 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:54.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:54.781 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 15 43968 43953 101.732 54.375 0.0093421 0.00982369 2023-10-09T09:35:54.810 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:54.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:54.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:54.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:54.940 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:54.956 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:54 smithi181 bash[59345]: cluster 2023-10-09T09:35:53.840042+0000 mgr.y (mgr.34107) 892 : cluster 0 pgmap v1635: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 42 MiB/s wr, 669 op/s; 4656/120327 objects degraded (3.869%); 2373/120327 objects misplaced (1.972%); 1017 KiB/s, 1 keys/s, 16 objects/s recovering 2023-10-09T09:35:54.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:54.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:54.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:55.072 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:55.105 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:55.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:55.210 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:55.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:54 smithi163 bash[78107]: cluster 2023-10-09T09:35:53.840042+0000 mgr.y (mgr.34107) 892 : cluster 0 pgmap v1635: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 42 MiB/s wr, 669 op/s; 4656/120327 objects degraded (3.869%); 2373/120327 objects misplaced (1.972%); 1017 KiB/s, 1 keys/s, 16 objects/s recovering 2023-10-09T09:35:55.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:54 smithi163 bash[75425]: cluster 2023-10-09T09:35:53.840042+0000 mgr.y (mgr.34107) 892 : cluster 0 pgmap v1635: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 42 MiB/s wr, 669 op/s; 4656/120327 objects degraded (3.869%); 2373/120327 objects misplaced (1.972%); 1017 KiB/s, 1 keys/s, 16 objects/s recovering 2023-10-09T09:35:55.240 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:55.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:55.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:55.380 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:55.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:55.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:55.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:55.509 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:55.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:55.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:55.613 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:55.642 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:55.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:55.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:55.748 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:55.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:55.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:55.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:55.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 45076 45060 100.569 69.1875 0.0133354 0.00993728 2023-10-09T09:35:55.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:55.905 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:55.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:56.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:56.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:56.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:56.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:56.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:56.163 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:56.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:56.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:56.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:56.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:56.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:56.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:56.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:56.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:56 smithi181 bash[59345]: audit 2023-10-09T09:35:55.380837+0000 mon.a (mon.0) 1772 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:56.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:56 smithi181 bash[59345]: audit 2023-10-09T09:35:55.384254+0000 mon.a (mon.0) 1773 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:56.423 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:56.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:56.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:56.527 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:56.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:56.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:56.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:56.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:56 smithi163 bash[75425]: audit 2023-10-09T09:35:55.380837+0000 mon.a (mon.0) 1772 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:56.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:56 smithi163 bash[75425]: audit 2023-10-09T09:35:55.384254+0000 mon.a (mon.0) 1773 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:56.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:56 smithi163 bash[78107]: audit 2023-10-09T09:35:55.380837+0000 mon.a (mon.0) 1772 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:35:56.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:56 smithi163 bash[78107]: audit 2023-10-09T09:35:55.384254+0000 mon.a (mon.0) 1773 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:35:56.657 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:56.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:56.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:56.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:56.781 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 46038 46022 99.1744 60.125 0.0121112 0.010076 2023-10-09T09:35:56.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:56.809 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:56.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:56.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:56.912 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:56.938 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:56.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:56.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:57.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:57.067 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:57.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:57.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:57.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:57.205 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:57.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:57.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:57.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:57.333 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:35:57.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:57.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:35:57.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:57.462 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:57.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:57.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:57.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:57.599 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:57.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:57.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:57.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:57.712 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:57 smithi163 bash[78107]: cluster 2023-10-09T09:35:55.840635+0000 mgr.y (mgr.34107) 893 : cluster 0 pgmap v1636: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s; 9486/124767 objects degraded (7.603%); 4458/124767 objects misplaced (3.573%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2023-10-09T09:35:57.713 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:57 smithi163 bash[75425]: cluster 2023-10-09T09:35:55.840635+0000 mgr.y (mgr.34107) 893 : cluster 0 pgmap v1636: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s; 9486/124767 objects degraded (7.603%); 4458/124767 objects misplaced (3.573%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2023-10-09T09:35:57.730 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:35:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:57.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:35:57.781 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 47110 47094 98.1017 67 0.0319284 0.0101858 2023-10-09T09:35:57.839 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:57.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:57.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:57.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:57.919 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:35:57.969 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:57.984 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:57 smithi181 bash[59345]: cluster 2023-10-09T09:35:55.840635+0000 mgr.y (mgr.34107) 893 : cluster 0 pgmap v1636: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s; 9486/124767 objects degraded (7.603%); 4458/124767 objects misplaced (3.573%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2023-10-09T09:35:58.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:58.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:58.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:58.109 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:58.144 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:58.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:58.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:58.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:58.278 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:58.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:58.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:58.382 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:58.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:58.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:58.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:58.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:58.544 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:58.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:58.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:58.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:58.691 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:58.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:58.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:58.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 48166 48150 97.066 66 0.0298326 0.0102949 2023-10-09T09:35:58.797 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:58.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:58 smithi181 bash[59345]: cluster 2023-10-09T09:35:58.644838+0000 mon.a (mon.0) 1774 : cluster 3 Health check update: Degraded data redundancy: 9486/135036 objects degraded (7.025%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:35:58.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:58.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:58.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:58.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:58.961 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:35:58.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:58.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:35:58.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:58 smithi163 bash[78107]: cluster 2023-10-09T09:35:58.644838+0000 mon.a (mon.0) 1774 : cluster 3 Health check update: Degraded data redundancy: 9486/135036 objects degraded (7.025%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:35:58.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:58 smithi163 bash[75425]: cluster 2023-10-09T09:35:58.644838+0000 mon.a (mon.0) 1774 : cluster 3 Health check update: Degraded data redundancy: 9486/135036 objects degraded (7.025%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:35:59.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:59.090 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:35:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:59.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:35:59.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:59.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:59.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:59.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:59.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:59.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:35:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:59.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:35:59.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:59.472 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:59.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:59.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:59.578 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:59.607 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:35:59.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:59.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:35:59.714 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:59.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:35:59 smithi163 bash[75425]: cluster 2023-10-09T09:35:57.841328+0000 mgr.y (mgr.34107) 894 : cluster 0 pgmap v1637: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 62 MiB/s wr, 986 op/s; 9486/135036 objects degraded (7.025%); 4223/135036 objects misplaced (3.127%); 3.9 MiB/s, 1 keys/s, 62 objects/s recovering 2023-10-09T09:35:59.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:35:59 smithi163 bash[78107]: cluster 2023-10-09T09:35:57.841328+0000 mgr.y (mgr.34107) 894 : cluster 0 pgmap v1637: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 62 MiB/s wr, 986 op/s; 9486/135036 objects degraded (7.025%); 4223/135036 objects misplaced (3.127%); 3.9 MiB/s, 1 keys/s, 62 objects/s recovering 2023-10-09T09:35:59.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:35:59.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:59.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:35:59.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 49277 49261 96.2024 69.4375 0.012555 0.0103895 2023-10-09T09:35:59.846 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:59.856 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:35:59 smithi181 bash[59345]: cluster 2023-10-09T09:35:57.841328+0000 mgr.y (mgr.34107) 894 : cluster 0 pgmap v1637: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 62 MiB/s wr, 986 op/s; 9486/135036 objects degraded (7.025%); 4223/135036 objects misplaced (3.127%); 3.9 MiB/s, 1 keys/s, 62 objects/s recovering 2023-10-09T09:35:59.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:35:59.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:35:59.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:35:59.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:00.023 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:00.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:00.127 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:00.159 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:00.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:00.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:00.295 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:00.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:00.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:00.394 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:36:00.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:00.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:00.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:00.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:00.534 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:00.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:00.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:00.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:00.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:00.698 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:00.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:00.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:00.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 50275 50259 95.1772 62.375 0.010228 0.0105003 2023-10-09T09:36:00.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:00.829 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:00.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:00.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:00.932 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:00.959 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:00.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:00.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:01.061 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:01.094 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:01.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:01.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:01.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:01.240 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:01.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:01.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:01.344 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:01.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:01.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:01.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:01.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:01.505 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:01.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:01.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:01.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:01.635 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:01.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:01.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:01.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:01.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:01 smithi163 bash[75425]: cluster 2023-10-09T09:35:59.841901+0000 mgr.y (mgr.34107) 895 : cluster 0 pgmap v1638: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 9486/139821 objects degraded (6.784%); 3869/139821 objects misplaced (2.767%); 5.7 MiB/s, 0 keys/s, 91 objects/s recovering 2023-10-09T09:36:01.753 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:01 smithi163 bash[75425]: audit 2023-10-09T09:36:00.875033+0000 mon.a (mon.0) 1775 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:01.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:01 smithi163 bash[78107]: cluster 2023-10-09T09:35:59.841901+0000 mgr.y (mgr.34107) 895 : cluster 0 pgmap v1638: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 9486/139821 objects degraded (6.784%); 3869/139821 objects misplaced (2.767%); 5.7 MiB/s, 0 keys/s, 91 objects/s recovering 2023-10-09T09:36:01.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:01 smithi163 bash[78107]: audit 2023-10-09T09:36:00.875033+0000 mon.a (mon.0) 1775 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:01.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:01.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:01.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:01.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 51218 51202 94.1112 58.9375 0.0035215 0.0106185 2023-10-09T09:36:01.868 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:01.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:01.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:01.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:02.002 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:02.011 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:01 smithi181 bash[59345]: cluster 2023-10-09T09:35:59.841901+0000 mgr.y (mgr.34107) 895 : cluster 0 pgmap v1638: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s; 9486/139821 objects degraded (6.784%); 3869/139821 objects misplaced (2.767%); 5.7 MiB/s, 0 keys/s, 91 objects/s recovering 2023-10-09T09:36:02.011 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:01 smithi181 bash[59345]: audit 2023-10-09T09:36:00.875033+0000 mon.a (mon.0) 1775 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:02.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:02.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:02.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:02.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:02.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:02.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:02.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:02.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:02.393 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:02.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:02.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:02.521 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:02.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:02.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:02.624 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:02.649 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:02.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:02.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:02.752 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:02.782 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:02.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 52232 52216 93.2328 63.375 0.0582395 0.0107105 2023-10-09T09:36:02.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:02.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:02.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:02.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:02.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:02.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:03.026 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:03.054 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:03.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:03.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:03.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:03.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:03.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:03.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:03.292 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:03.315 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:03.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:03.381 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:36:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:36:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:36:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:36:03.381 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:36:03.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:03.427 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:36:03] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.33.4" 2023-10-09T09:36:03.442 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:03.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:03.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:03.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:03.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:03.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:03.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:03.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:03.710 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:03 smithi181 bash[59345]: cluster 2023-10-09T09:36:01.842923+0000 mgr.y (mgr.34107) 896 : cluster 0 pgmap v1639: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 9486/150612 objects degraded (6.298%); 3605/150612 objects misplaced (2.394%); 6.2 MiB/s, 0 keys/s, 99 objects/s recovering 2023-10-09T09:36:03.710 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:03 smithi181 bash[59345]: audit 2023-10-09T09:36:02.844640+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:36:03.710 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:03 smithi181 bash[59345]: audit 2023-10-09T09:36:03.322097+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.710 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:03 smithi181 bash[59345]: audit 2023-10-09T09:36:03.328216+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.710 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:03 smithi181 bash[59345]: audit 2023-10-09T09:36:03.373999+0000 mon.a (mon.0) 1779 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.710 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:03 smithi181 bash[59345]: audit 2023-10-09T09:36:03.386426+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.725 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:03.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:03.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:03.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 53103 53087 92.1551 54.4375 0.00436888 0.0108429 2023-10-09T09:36:03.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:03.856 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:03.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:03.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:03.959 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:03.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[78107]: cluster 2023-10-09T09:36:01.842923+0000 mgr.y (mgr.34107) 896 : cluster 0 pgmap v1639: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 9486/150612 objects degraded (6.298%); 3605/150612 objects misplaced (2.394%); 6.2 MiB/s, 0 keys/s, 99 objects/s recovering 2023-10-09T09:36:03.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[78107]: audit 2023-10-09T09:36:02.844640+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:36:03.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[78107]: audit 2023-10-09T09:36:03.322097+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[78107]: audit 2023-10-09T09:36:03.328216+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[78107]: audit 2023-10-09T09:36:03.373999+0000 mon.a (mon.0) 1779 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[78107]: audit 2023-10-09T09:36:03.386426+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[75425]: cluster 2023-10-09T09:36:01.842923+0000 mgr.y (mgr.34107) 896 : cluster 0 pgmap v1639: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 9486/150612 objects degraded (6.298%); 3605/150612 objects misplaced (2.394%); 6.2 MiB/s, 0 keys/s, 99 objects/s recovering 2023-10-09T09:36:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[75425]: audit 2023-10-09T09:36:02.844640+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:36:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[75425]: audit 2023-10-09T09:36:03.322097+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[75425]: audit 2023-10-09T09:36:03.328216+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[75425]: audit 2023-10-09T09:36:03.373999+0000 mon.a (mon.0) 1779 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:03 smithi163 bash[75425]: audit 2023-10-09T09:36:03.386426+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:03.982 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:03.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:03.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:04.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:04.111 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:04.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:04.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:04.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:04.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:04.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:04.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:04.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:04.381 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:04.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:04.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:04.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T09:36:04.487 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:04.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:04.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:04.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:04.613 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:04.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:04.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:04.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:04.733 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:04.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:04.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:04.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:04.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 53957 53941 91.1068 53.375 0.012958 0.0109696 2023-10-09T09:36:04.868 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:04.901 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:04.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:04.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:05.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:05.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:05.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:05.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:05.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:05.177 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:05.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:05.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:05.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:05.306 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:05.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:05.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:05.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:05.435 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:05.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:05.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:05.540 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:05.576 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:05.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:05.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:05.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:05.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:05.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:05.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 54938 54922 90.3227 61.3125 0.00687582 0.0110594 2023-10-09T09:36:05.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:05.845 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:05.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:05.918 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:36:05.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:05.958 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:36:05.962 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:05 smithi181 bash[59345]: cluster 2023-10-09T09:36:03.843459+0000 mgr.y (mgr.34107) 897 : cluster 0 pgmap v1640: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s; 9486/150612 objects degraded (6.298%); 3605/150612 objects misplaced (2.394%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T09:36:05.962 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:05 smithi181 bash[59345]: cluster 2023-10-09T09:36:05.641272+0000 mon.a (mon.0) 1781 : cluster 3 Health check update: Degraded data redundancy: 9486/150612 objects degraded (6.298%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:05.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:05 smithi163 bash[75425]: cluster 2023-10-09T09:36:03.843459+0000 mgr.y (mgr.34107) 897 : cluster 0 pgmap v1640: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s; 9486/150612 objects degraded (6.298%); 3605/150612 objects misplaced (2.394%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T09:36:05.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:05 smithi163 bash[75425]: cluster 2023-10-09T09:36:05.641272+0000 mon.a (mon.0) 1781 : cluster 3 Health check update: Degraded data redundancy: 9486/150612 objects degraded (6.298%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:05 smithi163 bash[78107]: cluster 2023-10-09T09:36:03.843459+0000 mgr.y (mgr.34107) 897 : cluster 0 pgmap v1640: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s; 9486/150612 objects degraded (6.298%); 3605/150612 objects misplaced (2.394%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T09:36:05.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:05 smithi163 bash[78107]: cluster 2023-10-09T09:36:05.641272+0000 mon.a (mon.0) 1781 : cluster 3 Health check update: Degraded data redundancy: 9486/150612 objects degraded (6.298%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:05.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:05.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:06.009 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:36:06.064 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:36:06.084 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:36:06.091 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:06.113 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:36:06.115 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:06.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:06.151 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:36:06.189 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:36:06.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:36:06.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:36:06.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:36:06.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:36:06.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:36:06.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:36:06.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:36:06.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:36:06.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:06.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:06.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:06.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:06.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:06.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:06.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:06.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:06.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:06.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:06.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:06.610 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:06.635 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:06.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:06.738 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:06.747 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:36:06.765 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:06.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:06.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 55870 55854 89.5002 58.25 0.0192607 0.0111666 2023-10-09T09:36:06.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:06.877 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:06 smithi163 bash[75425]: audit 2023-10-09T09:36:05.868874+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.163:0/74621466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:36:06.877 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:06 smithi163 bash[75425]: audit 2023-10-09T09:36:06.291108+0000 mon.a (mon.0) 1782 : audit 1 from='client.? 172.21.15.163:0/3540687941' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:36:06.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:06 smithi163 bash[78107]: audit 2023-10-09T09:36:05.868874+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.163:0/74621466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:36:06.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:06 smithi163 bash[78107]: audit 2023-10-09T09:36:06.291108+0000 mon.a (mon.0) 1782 : audit 1 from='client.? 172.21.15.163:0/3540687941' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:36:06.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:06.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:06.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:07.001 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:07.010 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:06 smithi181 bash[59345]: audit 2023-10-09T09:36:05.868874+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.163:0/74621466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:36:07.011 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:06 smithi181 bash[59345]: audit 2023-10-09T09:36:06.291108+0000 mon.a (mon.0) 1782 : audit 1 from='client.? 172.21.15.163:0/3540687941' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:36:07.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:07.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:07.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:07.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:07.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:07.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:07.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:07.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:07.300 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:07.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:07.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:07.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:07.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:07.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:07.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:07.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:07.559 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:07.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:07.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:07.662 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:07.694 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:07.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:07.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:07.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:36:07.782989+0000 min lat: 0.00132712 max lat: 2.21788 avg lat: 0.0112482 2023-10-09T09:36:07.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:36:07.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 56881 56865 88.8422 63.1875 0.0185338 0.0112482 2023-10-09T09:36:07.799 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:07.808 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:07 smithi181 bash[59345]: cluster 2023-10-09T09:36:05.844039+0000 mgr.y (mgr.34107) 898 : cluster 0 pgmap v1641: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s; 9486/155346 objects degraded (6.106%); 3220/155346 objects misplaced (2.073%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T09:36:07.808 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:07 smithi181 bash[59345]: cluster 2023-10-09T09:36:06.714005+0000 mon.a (mon.0) 1783 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:36:07.809 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:07 smithi181 bash[59345]: cluster 2023-10-09T09:36:06.728455+0000 osd.5 (osd.5) 705 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.809 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:07 smithi181 bash[59345]: cluster 2023-10-09T09:36:06.731519+0000 osd.5 (osd.5) 706 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.809 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:07 smithi181 bash[59345]: cluster 2023-10-09T09:36:06.733103+0000 osd.6 (osd.6) 423 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.809 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:07 smithi181 bash[59345]: cluster 2023-10-09T09:36:06.737971+0000 osd.4 (osd.4) 567 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.809 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:07 smithi181 bash[59345]: audit 2023-10-09T09:36:06.739600+0000 mon.a (mon.0) 1784 : audit 1 from='client.? 172.21.15.163:0/3540687941' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:36:07.809 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:07 smithi181 bash[59345]: cluster 2023-10-09T09:36:06.739804+0000 mon.a (mon.0) 1785 : cluster 0 osdmap e1143: 8 total, 8 up, 7 in 2023-10-09T09:36:07.830 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:07.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:07.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:07.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[75425]: cluster 2023-10-09T09:36:05.844039+0000 mgr.y (mgr.34107) 898 : cluster 0 pgmap v1641: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s; 9486/155346 objects degraded (6.106%); 3220/155346 objects misplaced (2.073%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T09:36:07.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[75425]: cluster 2023-10-09T09:36:06.714005+0000 mon.a (mon.0) 1783 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:36:07.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[75425]: cluster 2023-10-09T09:36:06.728455+0000 osd.5 (osd.5) 705 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[75425]: cluster 2023-10-09T09:36:06.731519+0000 osd.5 (osd.5) 706 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[75425]: cluster 2023-10-09T09:36:06.733103+0000 osd.6 (osd.6) 423 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[75425]: cluster 2023-10-09T09:36:06.737971+0000 osd.4 (osd.4) 567 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[75425]: audit 2023-10-09T09:36:06.739600+0000 mon.a (mon.0) 1784 : audit 1 from='client.? 172.21.15.163:0/3540687941' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:36:07.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[75425]: cluster 2023-10-09T09:36:06.739804+0000 mon.a (mon.0) 1785 : cluster 0 osdmap e1143: 8 total, 8 up, 7 in 2023-10-09T09:36:07.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[78107]: cluster 2023-10-09T09:36:05.844039+0000 mgr.y (mgr.34107) 898 : cluster 0 pgmap v1641: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s; 9486/155346 objects degraded (6.106%); 3220/155346 objects misplaced (2.073%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T09:36:07.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[78107]: cluster 2023-10-09T09:36:06.714005+0000 mon.a (mon.0) 1783 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:36:07.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[78107]: cluster 2023-10-09T09:36:06.728455+0000 osd.5 (osd.5) 705 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[78107]: cluster 2023-10-09T09:36:06.731519+0000 osd.5 (osd.5) 706 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[78107]: cluster 2023-10-09T09:36:06.733103+0000 osd.6 (osd.6) 423 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[78107]: cluster 2023-10-09T09:36:06.737971+0000 osd.4 (osd.4) 567 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:07.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[78107]: audit 2023-10-09T09:36:06.739600+0000 mon.a (mon.0) 1784 : audit 1 from='client.? 172.21.15.163:0/3540687941' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:36:07.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:07 smithi163 bash[78107]: cluster 2023-10-09T09:36:06.739804+0000 mon.a (mon.0) 1785 : cluster 0 osdmap e1143: 8 total, 8 up, 7 in 2023-10-09T09:36:07.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:07.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:08.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:08.108 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:08.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:08.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:08.212 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:08.240 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:08.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:08.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:08.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:08.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:08.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:08.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:08.397 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:36:08.474 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:08.513 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:08.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:08.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:08.618 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:08.647 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:08.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:08.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:08.723 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:08 smithi181 bash[59345]: cluster 2023-10-09T09:36:06.738929+0000 osd.7 (osd.7) 544 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:08.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:08.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 57817 57801 88.102 58.5 0.0289845 0.0113425 2023-10-09T09:36:08.794 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:08.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:08.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:08.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:08.909 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:08 smithi163 bash[75425]: cluster 2023-10-09T09:36:06.738929+0000 osd.7 (osd.7) 544 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:08.910 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:08 smithi163 bash[78107]: cluster 2023-10-09T09:36:06.738929+0000 osd.7 (osd.7) 544 : cluster 3 failed to encode map e1143 with expected crc 2023-10-09T09:36:08.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:08.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:08.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:09.032 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:09.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:09.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:09.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:09.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:09.202 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:09.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:09.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:09.339 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:09.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:09.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:09.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:09.479 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:09.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:09.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:09.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:09.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:09.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:09.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:09.720 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:09.749 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:09.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:09.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:09.782 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 58718 58702 87.345 56.3125 0.027906 0.0114424 2023-10-09T09:36:09.852 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:09.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:09.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:09.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:09.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:09.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:09 smithi181 bash[59345]: cluster 2023-10-09T09:36:07.844555+0000 mgr.y (mgr.34107) 899 : cluster 0 pgmap v1643: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s; 9486/164763 objects degraded (5.757%); 2958/164763 objects misplaced (1.795%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T09:36:10.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:10.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:10.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:10.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:10.140 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:09 smithi163 bash[78107]: cluster 2023-10-09T09:36:07.844555+0000 mgr.y (mgr.34107) 899 : cluster 0 pgmap v1643: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s; 9486/164763 objects degraded (5.757%); 2958/164763 objects misplaced (1.795%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T09:36:10.141 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:09 smithi163 bash[75425]: cluster 2023-10-09T09:36:07.844555+0000 mgr.y (mgr.34107) 899 : cluster 0 pgmap v1643: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s; 9486/164763 objects degraded (5.757%); 2958/164763 objects misplaced (1.795%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T09:36:10.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:10.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:10.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:10.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:10.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:10.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:10.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:10.391 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:10.421 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:10.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:10.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:10.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:10.556 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:10.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:10.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:10.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:10.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:10.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:10.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 59598 59582 86.5927 55 0.00905354 0.0115406 2023-10-09T09:36:10.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:10.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:10.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:10.937 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:10.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:10.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:10.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:11.071 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:11.099 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:11.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:11.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:11.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:11 smithi163 bash[78107]: cluster 2023-10-09T09:36:09.845059+0000 mgr.y (mgr.34107) 900 : cluster 0 pgmap v1644: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 61 MiB/s wr, 980 op/s; 9486/169251 objects degraded (5.605%); 2555/169251 objects misplaced (1.510%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T09:36:11.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:11 smithi163 bash[78107]: audit 2023-10-09T09:36:10.175500+0000 mon.a (mon.0) 1786 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:11.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:11 smithi163 bash[78107]: audit 2023-10-09T09:36:10.184520+0000 mon.a (mon.0) 1787 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:11.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:11 smithi163 bash[78107]: audit 2023-10-09T09:36:10.375126+0000 mon.a (mon.0) 1788 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:11.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:11 smithi163 bash[78107]: cluster 2023-10-09T09:36:10.642498+0000 mon.a (mon.0) 1789 : cluster 3 Health check update: Degraded data redundancy: 9486/164763 objects degraded (5.757%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:11.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:11.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:11.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:11.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:11.342 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:11 smithi181 bash[59345]: cluster 2023-10-09T09:36:09.845059+0000 mgr.y (mgr.34107) 900 : cluster 0 pgmap v1644: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 61 MiB/s wr, 980 op/s; 9486/169251 objects degraded (5.605%); 2555/169251 objects misplaced (1.510%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T09:36:11.342 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:11 smithi181 bash[59345]: audit 2023-10-09T09:36:10.175500+0000 mon.a (mon.0) 1786 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:11.342 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:11 smithi181 bash[59345]: audit 2023-10-09T09:36:10.184520+0000 mon.a (mon.0) 1787 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:11.342 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:11 smithi181 bash[59345]: audit 2023-10-09T09:36:10.375126+0000 mon.a (mon.0) 1788 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:11.342 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:11 smithi181 bash[59345]: cluster 2023-10-09T09:36:10.642498+0000 mon.a (mon.0) 1789 : cluster 3 Health check update: Degraded data redundancy: 9486/164763 objects degraded (5.757%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:11.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:11.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:11.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:11.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:11 smithi163 bash[75425]: cluster 2023-10-09T09:36:09.845059+0000 mgr.y (mgr.34107) 900 : cluster 0 pgmap v1644: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 61 MiB/s wr, 980 op/s; 9486/169251 objects degraded (5.605%); 2555/169251 objects misplaced (1.510%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T09:36:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:11 smithi163 bash[75425]: audit 2023-10-09T09:36:10.175500+0000 mon.a (mon.0) 1786 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:11 smithi163 bash[75425]: audit 2023-10-09T09:36:10.184520+0000 mon.a (mon.0) 1787 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:11 smithi163 bash[75425]: audit 2023-10-09T09:36:10.375126+0000 mon.a (mon.0) 1788 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:11.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:11 smithi163 bash[75425]: cluster 2023-10-09T09:36:10.642498+0000 mon.a (mon.0) 1789 : cluster 3 Health check update: Degraded data redundancy: 9486/164763 objects degraded (5.757%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:11.511 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:11.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:11.618 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:11.645 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:11.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:11.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:11.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:11.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:11.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:11.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 60541 60525 85.964 58.9375 0.00335986 0.0116196 2023-10-09T09:36:11.874 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:11.909 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:11.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:12.014 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:12.045 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:12.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:12.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:12.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:12.177 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:12.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:12.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:12.283 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:12.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:12.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:12.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:12.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:12.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:12.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:12.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:12.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:12.577 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:12.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:12.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:12.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:12.718 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:12.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:12.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:12.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 61325 61309 85.1425 49 0.0143816 0.0117386 2023-10-09T09:36:12.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:12.848 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:12.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:12.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:12.899 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:36:12.899 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:36:12.950 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:12.977 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:12.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:12.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:13.085 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:13.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:13.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:13.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:13.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:13 smithi163 bash[75425]: cluster 2023-10-09T09:36:11.845662+0000 mgr.y (mgr.34107) 901 : cluster 0 pgmap v1645: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 53 MiB/s wr, 855 op/s; 9486/176283 objects degraded (5.381%); 2555/176283 objects misplaced (1.449%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T09:36:13.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:13.248 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:36:13.248 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T09:36:13.248 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:36:13.271 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:13.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:13 smithi163 bash[78107]: cluster 2023-10-09T09:36:11.845662+0000 mgr.y (mgr.34107) 901 : cluster 0 pgmap v1645: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 53 MiB/s wr, 855 op/s; 9486/176283 objects degraded (5.381%); 2555/176283 objects misplaced (1.449%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T09:36:13.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:13.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:13.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:13 smithi181 bash[59345]: cluster 2023-10-09T09:36:11.845662+0000 mgr.y (mgr.34107) 901 : cluster 0 pgmap v1645: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 53 MiB/s wr, 855 op/s; 9486/176283 objects degraded (5.381%); 2555/176283 objects misplaced (1.449%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T09:36:13.377 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:13.413 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:13.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:13.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:13.435 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:36:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:36:13] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.33.4" 2023-10-09T09:36:13.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:13.543 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:13.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:13.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:13.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:13.675 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:13.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:13.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:13.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 62211 62195 84.4953 55.375 0.0249372 0.0118282 2023-10-09T09:36:13.821 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:13.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:13.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:13.853 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:36:13.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:13.957 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:13.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:14.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:36:14.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:14.172 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:14.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:14.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:14.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:14.311 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:14.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:14.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:14.418 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:14.456 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:14.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:14.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:14.560 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:14.584 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:14.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:14.688 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:14.714 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:14.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:14.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:36:14.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 63050 63034 83.8131 52.4375 0.0198984 0.0119245 2023-10-09T09:36:14.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:14.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:14.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:14.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:14.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:14.998 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:15.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:15.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:15.104 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:15.132 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:15.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:15.240 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:15.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:15.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:15.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:15.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:15.412 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:15.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:15.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:15.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:15.548 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:15.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:15.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:15.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:15.682 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:15.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:15.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 63975 63959 83.2713 57.8125 0.0110441 0.0120029 2023-10-09T09:36:15.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:15.809 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:15.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:15.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:15.913 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:15.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:15 smithi181 bash[59345]: cluster 2023-10-09T09:36:13.850449+0000 mgr.y (mgr.34107) 902 : cluster 0 pgmap v1646: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s; 9486/178872 objects degraded (5.303%); 2288/178872 objects misplaced (1.279%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T09:36:15.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:15 smithi181 bash[59345]: audit 2023-10-09T09:36:14.705536+0000 mon.a (mon.0) 1790 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:15.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:15 smithi181 bash[59345]: audit 2023-10-09T09:36:14.715263+0000 mon.a (mon.0) 1791 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:15.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:15 smithi181 bash[59345]: audit 2023-10-09T09:36:15.344086+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:36:15.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:15 smithi181 bash[59345]: audit 2023-10-09T09:36:15.345335+0000 mon.a (mon.0) 1793 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:36:15.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:15 smithi181 bash[59345]: audit 2023-10-09T09:36:15.356009+0000 mon.a (mon.0) 1794 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:15.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:15 smithi181 bash[59345]: cluster 2023-10-09T09:36:15.644042+0000 mon.a (mon.0) 1795 : cluster 3 Health check update: Degraded data redundancy: 9486/178872 objects degraded (5.303%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:15.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:15.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:15.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[78107]: cluster 2023-10-09T09:36:13.850449+0000 mgr.y (mgr.34107) 902 : cluster 0 pgmap v1646: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s; 9486/178872 objects degraded (5.303%); 2288/178872 objects misplaced (1.279%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T09:36:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[78107]: audit 2023-10-09T09:36:14.705536+0000 mon.a (mon.0) 1790 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[78107]: audit 2023-10-09T09:36:14.715263+0000 mon.a (mon.0) 1791 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[78107]: audit 2023-10-09T09:36:15.344086+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:36:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[78107]: audit 2023-10-09T09:36:15.345335+0000 mon.a (mon.0) 1793 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:36:15.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[78107]: audit 2023-10-09T09:36:15.356009+0000 mon.a (mon.0) 1794 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[78107]: cluster 2023-10-09T09:36:15.644042+0000 mon.a (mon.0) 1795 : cluster 3 Health check update: Degraded data redundancy: 9486/178872 objects degraded (5.303%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[75425]: cluster 2023-10-09T09:36:13.850449+0000 mgr.y (mgr.34107) 902 : cluster 0 pgmap v1646: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s; 9486/178872 objects degraded (5.303%); 2288/178872 objects misplaced (1.279%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T09:36:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[75425]: audit 2023-10-09T09:36:14.705536+0000 mon.a (mon.0) 1790 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[75425]: audit 2023-10-09T09:36:14.715263+0000 mon.a (mon.0) 1791 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[75425]: audit 2023-10-09T09:36:15.344086+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:36:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[75425]: audit 2023-10-09T09:36:15.345335+0000 mon.a (mon.0) 1793 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:36:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[75425]: audit 2023-10-09T09:36:15.356009+0000 mon.a (mon.0) 1794 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:15.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:15 smithi163 bash[75425]: cluster 2023-10-09T09:36:15.644042+0000 mon.a (mon.0) 1795 : cluster 3 Health check update: Degraded data redundancy: 9486/178872 objects degraded (5.303%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:16.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:16.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:16.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:16.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:16.189 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:16.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:16.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:16.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:16.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:16.368 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:16.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:16.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:16.496 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:16.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:16.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:16.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:16.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:16.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:16.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:16.739 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:16.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:16.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:16.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:16.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 64900 64884 82.7516 57.8125 0.0217181 0.0120781 2023-10-09T09:36:16.870 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:16.909 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:16.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:16.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:17.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:17.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:17.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:17.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:17.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:17.176 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:17.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:17.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:17.280 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:17.311 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:17.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:17.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:17.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:17.445 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:17.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:17.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:17.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:17.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:17.585 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:17.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:17.686 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:17.721 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:17.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:17.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:17.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 65840 65824 82.2715 58.75 0.0065142 0.0121483 2023-10-09T09:36:17.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:17 smithi181 bash[59345]: cluster 2023-10-09T09:36:15.851097+0000 mgr.y (mgr.34107) 903 : cluster 0 pgmap v1647: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 921 op/s; 9486/183012 objects degraded (5.183%); 1783/183012 objects misplaced (0.974%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T09:36:17.827 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:17.863 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:17.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:17.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:17.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:17 smithi163 bash[78107]: cluster 2023-10-09T09:36:15.851097+0000 mgr.y (mgr.34107) 903 : cluster 0 pgmap v1647: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 921 op/s; 9486/183012 objects degraded (5.183%); 1783/183012 objects misplaced (0.974%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T09:36:17.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:17 smithi163 bash[75425]: cluster 2023-10-09T09:36:15.851097+0000 mgr.y (mgr.34107) 903 : cluster 0 pgmap v1647: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 921 op/s; 9486/183012 objects degraded (5.183%); 1783/183012 objects misplaced (0.974%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T09:36:18.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:18.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:18.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:18.113 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:18.158 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:18.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:18.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:18.263 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:18.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:18.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:18.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:18.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:18.425 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:18.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:18.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:18.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:18.558 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:18.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:18.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:18.663 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:18.694 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:18.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:18.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:18.710 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:36:18.714 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:36:18.714 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:36:18.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 66784 66768 81.8151 59 0.0102707 0.0122156 2023-10-09T09:36:18.797 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:18.821 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:18.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:18.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:18.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:18.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:18.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:19.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:19.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:19.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:19.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:19.192 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:19.216 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:19.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:19.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:19.320 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:19.349 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:19.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:19.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:19.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:19.483 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:19.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:19.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:19.588 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:19.620 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:19.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:19.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:19.727 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:19.754 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:19.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:19.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:19.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 67098 67082 80.6191 19.625 0.00832145 0.0123023 2023-10-09T09:36:19.862 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:19.890 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:19.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:19.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:19.905 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:36:19.905 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:36:19.906 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T09:36:19.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T09:36:19.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:20.006 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:19 smithi181 bash[59345]: cluster 2023-10-09T09:36:17.852001+0000 mgr.y (mgr.34107) 904 : cluster 0 pgmap v1648: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 896 op/s; 9486/192063 objects degraded (4.939%); 1494/192063 objects misplaced (0.778%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T09:36:20.006 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:19 smithi181 bash[59345]: cluster 2023-10-09T09:36:18.735919+0000 osd.5 (osd.5) 707 : cluster 3 failed to encode map e1144 with expected crc 2023-10-09T09:36:20.006 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:19 smithi181 bash[59345]: cluster 2023-10-09T09:36:18.740812+0000 mon.a (mon.0) 1796 : cluster 0 osdmap e1144: 8 total, 8 up, 7 in 2023-10-09T09:36:20.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:20.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:20.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:20.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:20.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:19 smithi163 bash[78107]: cluster 2023-10-09T09:36:17.852001+0000 mgr.y (mgr.34107) 904 : cluster 0 pgmap v1648: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 896 op/s; 9486/192063 objects degraded (4.939%); 1494/192063 objects misplaced (0.778%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T09:36:20.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:19 smithi163 bash[78107]: cluster 2023-10-09T09:36:18.735919+0000 osd.5 (osd.5) 707 : cluster 3 failed to encode map e1144 with expected crc 2023-10-09T09:36:20.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:19 smithi163 bash[78107]: cluster 2023-10-09T09:36:18.740812+0000 mon.a (mon.0) 1796 : cluster 0 osdmap e1144: 8 total, 8 up, 7 in 2023-10-09T09:36:20.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:19 smithi163 bash[75425]: cluster 2023-10-09T09:36:17.852001+0000 mgr.y (mgr.34107) 904 : cluster 0 pgmap v1648: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 896 op/s; 9486/192063 objects degraded (4.939%); 1494/192063 objects misplaced (0.778%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T09:36:20.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:19 smithi163 bash[75425]: cluster 2023-10-09T09:36:18.735919+0000 osd.5 (osd.5) 707 : cluster 3 failed to encode map e1144 with expected crc 2023-10-09T09:36:20.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:19 smithi163 bash[75425]: cluster 2023-10-09T09:36:18.740812+0000 mon.a (mon.0) 1796 : cluster 0 osdmap e1144: 8 total, 8 up, 7 in 2023-10-09T09:36:20.165 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:20.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:20.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:20.268 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:20.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:20.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:20.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:20.402 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:20.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:20.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:20.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:20.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:20.573 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:20.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:20.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:20.680 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:20.706 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:20.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:20.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:20.783 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 68024 68008 80.1899 57.875 0.00522344 0.0124605 2023-10-09T09:36:20.810 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:20 smithi181 bash[59345]: cluster 2023-10-09T09:36:19.740442+0000 osd.5 (osd.5) 708 : cluster 3 failed to encode map e1145 with expected crc 2023-10-09T09:36:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:20 smithi181 bash[59345]: cluster 2023-10-09T09:36:19.743976+0000 mon.a (mon.0) 1797 : cluster 0 osdmap e1145: 8 total, 8 up, 7 in 2023-10-09T09:36:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:20 smithi181 bash[59345]: cluster 2023-10-09T09:36:19.760195+0000 osd.6 (osd.6) 424 : cluster 3 failed to encode map e1145 with expected crc 2023-10-09T09:36:20.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:20 smithi181 bash[59345]: cluster 2023-10-09T09:36:20.645091+0000 mon.a (mon.0) 1798 : cluster 3 Health check update: Degraded data redundancy: 9486/192063 objects degraded (4.939%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:20.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:20.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:20.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:20.933 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:20.943 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:20 smithi163 bash[78107]: cluster 2023-10-09T09:36:19.740442+0000 osd.5 (osd.5) 708 : cluster 3 failed to encode map e1145 with expected crc 2023-10-09T09:36:20.943 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:20 smithi163 bash[78107]: cluster 2023-10-09T09:36:19.743976+0000 mon.a (mon.0) 1797 : cluster 0 osdmap e1145: 8 total, 8 up, 7 in 2023-10-09T09:36:20.943 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:20 smithi163 bash[78107]: cluster 2023-10-09T09:36:19.760195+0000 osd.6 (osd.6) 424 : cluster 3 failed to encode map e1145 with expected crc 2023-10-09T09:36:20.943 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:20 smithi163 bash[78107]: cluster 2023-10-09T09:36:20.645091+0000 mon.a (mon.0) 1798 : cluster 3 Health check update: Degraded data redundancy: 9486/192063 objects degraded (4.939%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:20.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:20 smithi163 bash[75425]: cluster 2023-10-09T09:36:19.740442+0000 osd.5 (osd.5) 708 : cluster 3 failed to encode map e1145 with expected crc 2023-10-09T09:36:20.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:20 smithi163 bash[75425]: cluster 2023-10-09T09:36:19.743976+0000 mon.a (mon.0) 1797 : cluster 0 osdmap e1145: 8 total, 8 up, 7 in 2023-10-09T09:36:20.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:20 smithi163 bash[75425]: cluster 2023-10-09T09:36:19.760195+0000 osd.6 (osd.6) 424 : cluster 3 failed to encode map e1145 with expected crc 2023-10-09T09:36:20.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:20 smithi163 bash[75425]: cluster 2023-10-09T09:36:20.645091+0000 mon.a (mon.0) 1798 : cluster 3 Health check update: Degraded data redundancy: 9486/192063 objects degraded (4.939%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:20.957 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:20.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:20.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:21.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:21.093 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:21.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:21.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:21.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:21.224 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:21.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:21.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:21.332 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:21.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:21.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:21.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:21.459 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:21.486 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:21.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:21.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:21.591 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:21.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:21.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:21.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:21.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:21.753 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:21 smithi163 bash[75425]: cluster 2023-10-09T09:36:19.852487+0000 mgr.y (mgr.34107) 905 : cluster 0 pgmap v1651: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 52 MiB/s wr, 836 op/s; 9486/196371 objects degraded (4.831%); 1056/196371 objects misplaced (0.538%); 12 MiB/s, 187 objects/s recovering 2023-10-09T09:36:21.759 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:21.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:21.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:21.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 68961 68945 79.7893 58.5625 0.00362618 0.0125238 2023-10-09T09:36:21.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:21.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:21.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:21.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:21.992 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:22.020 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:22.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:22.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:21 smithi181 bash[59345]: cluster 2023-10-09T09:36:19.852487+0000 mgr.y (mgr.34107) 905 : cluster 0 pgmap v1651: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 52 MiB/s wr, 836 op/s; 9486/196371 objects degraded (4.831%); 1056/196371 objects misplaced (0.538%); 12 MiB/s, 187 objects/s recovering 2023-10-09T09:36:22.124 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:22.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:22.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:22.193 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:21 smithi163 bash[78107]: cluster 2023-10-09T09:36:19.852487+0000 mgr.y (mgr.34107) 905 : cluster 0 pgmap v1651: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 52 MiB/s wr, 836 op/s; 9486/196371 objects degraded (4.831%); 1056/196371 objects misplaced (0.538%); 12 MiB/s, 187 objects/s recovering 2023-10-09T09:36:22.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:22.314 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:22.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:22.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:22.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:22.443 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:22.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:22.547 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:22.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:22.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:22.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:22.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:22.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:22.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:22.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 69874 69858 79.376 57.0625 0.0158076 0.0125925 2023-10-09T09:36:22.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:22.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:22.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:22.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:22.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:22.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:22.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:23.100 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:23.133 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:23.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:23.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:23.236 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:23.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:23.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:23.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:23.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:23.389 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:36:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:36:23] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.33.4" 2023-10-09T09:36:23.405 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:23.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:23.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:23.510 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:23.536 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:23.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:23.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:23.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:23.677 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:23.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:23.781 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:23.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 70809 70793 79.002 58.4375 0.0275936 0.0126513 2023-10-09T09:36:23.805 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:23.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:23.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:23.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:23.954 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:23.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:23 smithi163 bash[75425]: cluster 2023-10-09T09:36:21.853183+0000 mgr.y (mgr.34107) 906 : cluster 0 pgmap v1652: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 982 op/s; 9486/202467 objects degraded (4.685%); 1056/202467 objects misplaced (0.522%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T09:36:23.955 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:23 smithi163 bash[78107]: cluster 2023-10-09T09:36:21.853183+0000 mgr.y (mgr.34107) 906 : cluster 0 pgmap v1652: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 982 op/s; 9486/202467 objects degraded (4.685%); 1056/202467 objects misplaced (0.522%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T09:36:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:23.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:24.059 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:23 smithi181 bash[59345]: cluster 2023-10-09T09:36:21.853183+0000 mgr.y (mgr.34107) 906 : cluster 0 pgmap v1652: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 982 op/s; 9486/202467 objects degraded (4.685%); 1056/202467 objects misplaced (0.522%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T09:36:24.083 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:24.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:24.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:24.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:24.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:24.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:24.324 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:24.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:24.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:24.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:24.455 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:24.478 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:24.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:24.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:24.608 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:24.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:24.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:24.711 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:24.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:24.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:24.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:24.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 71699 71683 78.5918 55.625 0.00499918 0.0127176 2023-10-09T09:36:24.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:24.878 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:24.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:24.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:24.982 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:25.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:25.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:25.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:25.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:25.160 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:25.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:25.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:25.264 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:25.293 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:25.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:25.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:25.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:25.420 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:25.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:25.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:25.525 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:25.554 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:25.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:25.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:25.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:25.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:25.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:25.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:25.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 72597 72581 78.2043 56.125 0.0145382 0.0127788 2023-10-09T09:36:25.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:25.801 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:25 smithi181 bash[59345]: cluster 2023-10-09T09:36:23.853721+0000 mgr.y (mgr.34107) 907 : cluster 0 pgmap v1653: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s; 9486/204771 objects degraded (4.632%); 652/204771 objects misplaced (0.318%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T09:36:25.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:25 smithi181 bash[59345]: audit 2023-10-09T09:36:25.374630+0000 mon.a (mon.0) 1799 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:25.802 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:25 smithi181 bash[59345]: cluster 2023-10-09T09:36:25.646402+0000 mon.a (mon.0) 1800 : cluster 3 Health check update: Degraded data redundancy: 9486/204771 objects degraded (4.632%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:25.826 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:25.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:25.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:25.933 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:25.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:25 smithi163 bash[75425]: cluster 2023-10-09T09:36:23.853721+0000 mgr.y (mgr.34107) 907 : cluster 0 pgmap v1653: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s; 9486/204771 objects degraded (4.632%); 652/204771 objects misplaced (0.318%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T09:36:25.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:25 smithi163 bash[75425]: audit 2023-10-09T09:36:25.374630+0000 mon.a (mon.0) 1799 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:25 smithi163 bash[75425]: cluster 2023-10-09T09:36:25.646402+0000 mon.a (mon.0) 1800 : cluster 3 Health check update: Degraded data redundancy: 9486/204771 objects degraded (4.632%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:25 smithi163 bash[78107]: cluster 2023-10-09T09:36:23.853721+0000 mgr.y (mgr.34107) 907 : cluster 0 pgmap v1653: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s; 9486/204771 objects degraded (4.632%); 652/204771 objects misplaced (0.318%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T09:36:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:25 smithi163 bash[78107]: audit 2023-10-09T09:36:25.374630+0000 mon.a (mon.0) 1799 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:25 smithi163 bash[78107]: cluster 2023-10-09T09:36:25.646402+0000 mon.a (mon.0) 1800 : cluster 3 Health check update: Degraded data redundancy: 9486/204771 objects degraded (4.632%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:25.978 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:25.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:26.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:26.108 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:26.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:26.212 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:26.235 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:26.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:26.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:26.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:26.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:26.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:26.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:26.474 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:26.502 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:26.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:26.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:26.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:26.634 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:26.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:26.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:26.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:26.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:26.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 73485 73469 77.8194 55.5 0.00736606 0.0128439 2023-10-09T09:36:26.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:26.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:26.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:26.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:27.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:27.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:27.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:27.150 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:27.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:27.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:27.254 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:27.288 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:27.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:27.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:27.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:27.421 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:27.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:27.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:27.559 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:27.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:27.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:27.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:27.691 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:27.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:27.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:27.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:36:27.784915+0000 min lat: 0.00132712 max lat: 2.21788 avg lat: 0.0129033 2023-10-09T09:36:27.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:36:27.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 74376 74360 77.4504 55.6875 0.00894913 0.0129033 2023-10-09T09:36:27.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:27.806 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:27 smithi181 bash[59345]: cluster 2023-10-09T09:36:25.854601+0000 mgr.y (mgr.34107) 908 : cluster 0 pgmap v1654: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s; 9486/208569 objects degraded (4.548%); 652/208569 objects misplaced (0.313%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T09:36:27.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:27.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:27.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:27.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:27 smithi163 bash[78107]: cluster 2023-10-09T09:36:25.854601+0000 mgr.y (mgr.34107) 908 : cluster 0 pgmap v1654: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s; 9486/208569 objects degraded (4.548%); 652/208569 objects misplaced (0.313%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T09:36:27.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:27 smithi163 bash[75425]: cluster 2023-10-09T09:36:25.854601+0000 mgr.y (mgr.34107) 908 : cluster 0 pgmap v1654: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s; 9486/208569 objects degraded (4.548%); 652/208569 objects misplaced (0.313%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T09:36:27.975 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:27.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:28.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:28.112 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:28.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:28.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:28.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:28.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:28.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:28.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:28.357 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:28.382 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:28.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:28.944 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 75266 75250 77.0925 55.625 0.0645754 0.0129633 2023-10-09T09:36:29.046 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:29.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:29.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:29.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:29.356 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:29.390 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:29.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:29.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:29.507 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:29.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:29.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:29.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:29.667 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:29.833 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:29 smithi181 bash[59345]: cluster 2023-10-09T09:36:27.855780+0000 mgr.y (mgr.34107) 909 : cluster 0 pgmap v1655: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 946 op/s; 9486/217944 objects degraded (4.352%); 300/217944 objects misplaced (0.138%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T09:36:29.834 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 76107 76091 76.6968 52.5625 0.0233142 0.0130315 2023-10-09T09:36:29.862 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:29.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:29.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:29.903 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:36:29.975 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:30.002 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:30.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:30.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:30.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:29 smithi163 bash[78107]: cluster 2023-10-09T09:36:27.855780+0000 mgr.y (mgr.34107) 909 : cluster 0 pgmap v1655: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 946 op/s; 9486/217944 objects degraded (4.352%); 300/217944 objects misplaced (0.138%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T09:36:30.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:29 smithi163 bash[75425]: cluster 2023-10-09T09:36:27.855780+0000 mgr.y (mgr.34107) 909 : cluster 0 pgmap v1655: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 946 op/s; 9486/217944 objects degraded (4.352%); 300/217944 objects misplaced (0.138%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T09:36:30.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:30.253 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:30.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:30.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:30.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:30.398 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:30.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:30.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:30.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:30.526 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:30.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:30.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:30.630 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:30.672 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:30.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:30.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:30.775 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:30.784 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 76902 76886 76.268 49.6875 0.00478934 0.0130958 2023-10-09T09:36:30.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:30.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:30.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:30.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:30.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:30 smithi181 bash[59345]: cluster 2023-10-09T09:36:30.647339+0000 mon.a (mon.0) 1801 : cluster 3 Health check update: Degraded data redundancy: 9486/217944 objects degraded (4.352%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:30.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:30 smithi181 bash[59345]: cluster 2023-10-09T09:36:30.651597+0000 mon.a (mon.0) 1802 : cluster 0 osdmap e1146: 8 total, 8 up, 7 in 2023-10-09T09:36:30.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:30 smithi181 bash[59345]: cluster 2023-10-09T09:36:30.658057+0000 osd.7 (osd.7) 545 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:30.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:30.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:30.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:30 smithi163 bash[78107]: cluster 2023-10-09T09:36:30.647339+0000 mon.a (mon.0) 1801 : cluster 3 Health check update: Degraded data redundancy: 9486/217944 objects degraded (4.352%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:30 smithi163 bash[78107]: cluster 2023-10-09T09:36:30.651597+0000 mon.a (mon.0) 1802 : cluster 0 osdmap e1146: 8 total, 8 up, 7 in 2023-10-09T09:36:30.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:30 smithi163 bash[78107]: cluster 2023-10-09T09:36:30.658057+0000 osd.7 (osd.7) 545 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:30.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:30 smithi163 bash[75425]: cluster 2023-10-09T09:36:30.647339+0000 mon.a (mon.0) 1801 : cluster 3 Health check update: Degraded data redundancy: 9486/217944 objects degraded (4.352%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:30 smithi163 bash[75425]: cluster 2023-10-09T09:36:30.651597+0000 mon.a (mon.0) 1802 : cluster 0 osdmap e1146: 8 total, 8 up, 7 in 2023-10-09T09:36:30.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:30 smithi163 bash[75425]: cluster 2023-10-09T09:36:30.658057+0000 osd.7 (osd.7) 545 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:31.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:31.070 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:31.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:31.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:31.173 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:31.198 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:31.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:31.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:31.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:31.331 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:31.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:31.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:31.435 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:31.463 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:31.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:31.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:31.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:31.587 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:31.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:31.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:31.715 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:31.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:31.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:31.785 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 77163 77147 75.3312 16.3125 0.0174304 0.0132682 2023-10-09T09:36:31.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:31 smithi181 bash[59345]: cluster 2023-10-09T09:36:29.856473+0000 mgr.y (mgr.34107) 910 : cluster 0 pgmap v1656: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s; 9486/222240 objects degraded (4.268%); 300/222240 objects misplaced (0.135%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T09:36:31.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:31 smithi181 bash[59345]: cluster 2023-10-09T09:36:30.648939+0000 osd.5 (osd.5) 709 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:31.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:31 smithi181 bash[59345]: cluster 2023-10-09T09:36:30.653995+0000 osd.4 (osd.4) 568 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:31.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:31 smithi181 bash[59345]: cluster 2023-10-09T09:36:30.657847+0000 osd.4 (osd.4) 569 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:31.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:31 smithi181 bash[59345]: cluster 2023-10-09T09:36:31.660080+0000 osd.7 (osd.7) 546 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:31.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:31 smithi181 bash[59345]: cluster 2023-10-09T09:36:31.662240+0000 mon.a (mon.0) 1803 : cluster 0 osdmap e1147: 8 total, 8 up, 7 in 2023-10-09T09:36:31.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:31.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[78107]: cluster 2023-10-09T09:36:29.856473+0000 mgr.y (mgr.34107) 910 : cluster 0 pgmap v1656: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s; 9486/222240 objects degraded (4.268%); 300/222240 objects misplaced (0.135%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T09:36:31.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[78107]: cluster 2023-10-09T09:36:30.648939+0000 osd.5 (osd.5) 709 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:31.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[78107]: cluster 2023-10-09T09:36:30.653995+0000 osd.4 (osd.4) 568 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:31.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[78107]: cluster 2023-10-09T09:36:30.657847+0000 osd.4 (osd.4) 569 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:31.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[78107]: cluster 2023-10-09T09:36:31.660080+0000 osd.7 (osd.7) 546 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:31.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[78107]: cluster 2023-10-09T09:36:31.662240+0000 mon.a (mon.0) 1803 : cluster 0 osdmap e1147: 8 total, 8 up, 7 in 2023-10-09T09:36:31.827 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[75425]: cluster 2023-10-09T09:36:29.856473+0000 mgr.y (mgr.34107) 910 : cluster 0 pgmap v1656: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 852 op/s; 9486/222240 objects degraded (4.268%); 300/222240 objects misplaced (0.135%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T09:36:31.827 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[75425]: cluster 2023-10-09T09:36:30.648939+0000 osd.5 (osd.5) 709 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:31.827 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[75425]: cluster 2023-10-09T09:36:30.653995+0000 osd.4 (osd.4) 568 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:31.827 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[75425]: cluster 2023-10-09T09:36:30.657847+0000 osd.4 (osd.4) 569 : cluster 3 failed to encode map e1146 with expected crc 2023-10-09T09:36:31.827 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[75425]: cluster 2023-10-09T09:36:31.660080+0000 osd.7 (osd.7) 546 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:31.827 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:31 smithi163 bash[75425]: cluster 2023-10-09T09:36:31.662240+0000 mon.a (mon.0) 1803 : cluster 0 osdmap e1147: 8 total, 8 up, 7 in 2023-10-09T09:36:31.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:31.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:31.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:31.952 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:31.984 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:31.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:31.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:32.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:32.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:32.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:32.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:32.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:32.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:32.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:32.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:32.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:32.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:32.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:32.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:32.492 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:32.525 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:32.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:32.627 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:32.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:32.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:32.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:32.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:32.785 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 77991 77975 74.9683 51.75 0.00780527 0.0133321 2023-10-09T09:36:32.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:32.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:32.803 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:36:32.897 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:32.921 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:32.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:32.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:33.025 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:33.035 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:32 smithi181 bash[59345]: cluster 2023-10-09T09:36:31.659503+0000 osd.5 (osd.5) 710 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:33.035 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:32 smithi181 bash[59345]: cluster 2023-10-09T09:36:31.666523+0000 osd.4 (osd.4) 570 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:33.035 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:32 smithi181 bash[59345]: cluster 2023-10-09T09:36:31.683365+0000 osd.6 (osd.6) 425 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:33.035 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:32 smithi181 bash[59345]: audit 2023-10-09T09:36:32.782139+0000 mon.a (mon.0) 1804 : audit 1 from='client.? 172.21.15.163:0/2632972427' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:36:33.049 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:33.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:33.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:33.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:32 smithi163 bash[78107]: cluster 2023-10-09T09:36:31.659503+0000 osd.5 (osd.5) 710 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:33.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:32 smithi163 bash[78107]: cluster 2023-10-09T09:36:31.666523+0000 osd.4 (osd.4) 570 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:33.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:32 smithi163 bash[78107]: cluster 2023-10-09T09:36:31.683365+0000 osd.6 (osd.6) 425 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:33.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:32 smithi163 bash[78107]: audit 2023-10-09T09:36:32.782139+0000 mon.a (mon.0) 1804 : audit 1 from='client.? 172.21.15.163:0/2632972427' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:36:33.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:32 smithi163 bash[75425]: cluster 2023-10-09T09:36:31.659503+0000 osd.5 (osd.5) 710 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:33.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:32 smithi163 bash[75425]: cluster 2023-10-09T09:36:31.666523+0000 osd.4 (osd.4) 570 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:33.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:32 smithi163 bash[75425]: cluster 2023-10-09T09:36:31.683365+0000 osd.6 (osd.6) 425 : cluster 3 failed to encode map e1147 with expected crc 2023-10-09T09:36:33.061 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:32 smithi163 bash[75425]: audit 2023-10-09T09:36:32.782139+0000 mon.a (mon.0) 1804 : audit 1 from='client.? 172.21.15.163:0/2632972427' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:36:33.155 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:33.182 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:33.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:33.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:33.295 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:33.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:33.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:33.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:33.434 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:33.461 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:33.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:33.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:33.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:33.591 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:33.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:33.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:33.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:33.703 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:36:33] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.33.4" 2023-10-09T09:36:33.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:33.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:33.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:33.785 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 78803 78787 74.6013 50.75 0.0180196 0.013398 2023-10-09T09:36:33.824 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:33.846 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:33.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:33.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:33.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[75425]: cluster 2023-10-09T09:36:31.857095+0000 mgr.y (mgr.34107) 911 : cluster 0 pgmap v1659: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s; 7198/227799 objects degraded (3.160%); 2565/227799 objects misplaced (1.126%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T09:36:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[75425]: cluster 2023-10-09T09:36:32.787970+0000 mon.a (mon.0) 1805 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:36:33.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[75425]: cluster 2023-10-09T09:36:32.794439+0000 osd.6 (osd.6) 426 : cluster 3 failed to encode map e1148 with expected crc 2023-10-09T09:36:33.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[75425]: cluster 2023-10-09T09:36:32.794455+0000 osd.5 (osd.5) 711 : cluster 3 failed to encode map e1148 with expected crc 2023-10-09T09:36:33.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[75425]: audit 2023-10-09T09:36:32.798588+0000 mon.a (mon.0) 1806 : audit 1 from='client.? 172.21.15.163:0/2632972427' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:36:33.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[75425]: cluster 2023-10-09T09:36:32.798682+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e1148: 8 total, 8 up, 7 in 2023-10-09T09:36:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[78107]: cluster 2023-10-09T09:36:31.857095+0000 mgr.y (mgr.34107) 911 : cluster 0 pgmap v1659: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s; 7198/227799 objects degraded (3.160%); 2565/227799 objects misplaced (1.126%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T09:36:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[78107]: cluster 2023-10-09T09:36:32.787970+0000 mon.a (mon.0) 1805 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:36:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[78107]: cluster 2023-10-09T09:36:32.794439+0000 osd.6 (osd.6) 426 : cluster 3 failed to encode map e1148 with expected crc 2023-10-09T09:36:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[78107]: cluster 2023-10-09T09:36:32.794455+0000 osd.5 (osd.5) 711 : cluster 3 failed to encode map e1148 with expected crc 2023-10-09T09:36:33.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[78107]: audit 2023-10-09T09:36:32.798588+0000 mon.a (mon.0) 1806 : audit 1 from='client.? 172.21.15.163:0/2632972427' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:36:33.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:33 smithi163 bash[78107]: cluster 2023-10-09T09:36:32.798682+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e1148: 8 total, 8 up, 7 in 2023-10-09T09:36:33.987 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:33.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:33.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:34.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:33 smithi181 bash[59345]: cluster 2023-10-09T09:36:31.857095+0000 mgr.y (mgr.34107) 911 : cluster 0 pgmap v1659: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s; 7198/227799 objects degraded (3.160%); 2565/227799 objects misplaced (1.126%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T09:36:34.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:33 smithi181 bash[59345]: cluster 2023-10-09T09:36:32.787970+0000 mon.a (mon.0) 1805 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:36:34.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:33 smithi181 bash[59345]: cluster 2023-10-09T09:36:32.794439+0000 osd.6 (osd.6) 426 : cluster 3 failed to encode map e1148 with expected crc 2023-10-09T09:36:34.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:33 smithi181 bash[59345]: cluster 2023-10-09T09:36:32.794455+0000 osd.5 (osd.5) 711 : cluster 3 failed to encode map e1148 with expected crc 2023-10-09T09:36:34.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:33 smithi181 bash[59345]: audit 2023-10-09T09:36:32.798588+0000 mon.a (mon.0) 1806 : audit 1 from='client.? 172.21.15.163:0/2632972427' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:36:34.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:33 smithi181 bash[59345]: cluster 2023-10-09T09:36:32.798682+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e1148: 8 total, 8 up, 7 in 2023-10-09T09:36:34.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:34.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:34.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:34.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:34.221 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:34.245 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:34.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:34.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:34.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:34.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:34.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:34.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:34.493 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:36:34.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:34.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:34.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:34.521 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:36:34.549 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:36:34.576 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:36:34.603 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:36:34.614 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:34.626 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:36:34.644 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:34.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:34.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:34.656 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:36:34.686 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:36:34.690 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:36:34.690 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:36:34.690 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:36:34.691 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:36:34.691 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:36:34.691 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:36:34.691 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:36:34.691 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:36:34.751 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:34.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:34.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:34.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:34.785 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 79561 79545 74.1949 47.375 0.026535 0.0134695 2023-10-09T09:36:34.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:34.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:34.905 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:34 smithi163 bash[75425]: audit 2023-10-09T09:36:34.460860+0000 mon.a (mon.0) 1808 : audit 0 from='client.? 172.21.15.163:0/1991244957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:36:34.906 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:34 smithi163 bash[78107]: audit 2023-10-09T09:36:34.460860+0000 mon.a (mon.0) 1808 : audit 0 from='client.? 172.21.15.163:0/1991244957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:36:34.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:34.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:35.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:35.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:35.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:35.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:34 smithi181 bash[59345]: audit 2023-10-09T09:36:34.460860+0000 mon.a (mon.0) 1808 : audit 0 from='client.? 172.21.15.163:0/1991244957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:36:35.141 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:35.164 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:35.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:35.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:35.271 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:35.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:35.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:35.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:35.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:35.435 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:35.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:35.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:35.540 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:35.569 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:35.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:35.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:35.714 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:35.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:35.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:35.785 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 80389 80373 73.8648 51.75 0.0258743 0.0135323 2023-10-09T09:36:35.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:35.852 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:35.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:35.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:35.956 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:35.992 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:35.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:35.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:35 smithi181 bash[59345]: cluster 2023-10-09T09:36:33.857574+0000 mgr.y (mgr.34107) 912 : cluster 0 pgmap v1661: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 661 op/s; 7198/229851 objects degraded (3.132%); 2265/229851 objects misplaced (0.985%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T09:36:36.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:35 smithi181 bash[59345]: cluster 2023-10-09T09:36:35.648497+0000 mon.a (mon.0) 1809 : cluster 3 Health check update: Degraded data redundancy: 7198/229851 objects degraded (3.132%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:36.096 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:36.125 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:36.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:36.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:36.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:35 smithi163 bash[78107]: cluster 2023-10-09T09:36:33.857574+0000 mgr.y (mgr.34107) 912 : cluster 0 pgmap v1661: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 661 op/s; 7198/229851 objects degraded (3.132%); 2265/229851 objects misplaced (0.985%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T09:36:36.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:35 smithi163 bash[78107]: cluster 2023-10-09T09:36:35.648497+0000 mon.a (mon.0) 1809 : cluster 3 Health check update: Degraded data redundancy: 7198/229851 objects degraded (3.132%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:36.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:35 smithi163 bash[75425]: cluster 2023-10-09T09:36:33.857574+0000 mgr.y (mgr.34107) 912 : cluster 0 pgmap v1661: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 661 op/s; 7198/229851 objects degraded (3.132%); 2265/229851 objects misplaced (0.985%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T09:36:36.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:35 smithi163 bash[75425]: cluster 2023-10-09T09:36:35.648497+0000 mon.a (mon.0) 1809 : cluster 3 Health check update: Degraded data redundancy: 7198/229851 objects degraded (3.132%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:36.228 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:36:36.228 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:36:36.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.228 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:80727 8.c 8:35f62d5b:::benchmark_data_smithi163_151167_object80726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:36:36.228 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:36.218979+0000", 2023-10-09T09:36:36.228 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0086646099999999997, 2023-10-09T09:36:36.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0086972560000000004, 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80727 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.218979+0000", 2023-10-09T09:36:36.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.218979+0000", 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.218978+0000", 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:36:36.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:36:36.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.219000+0000", 2023-10-09T09:36:36.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1991999999999999e-05 2023-10-09T09:36:36.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:36:36.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.219011+0000", 2023-10-09T09:36:36.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0937e-05 2023-10-09T09:36:36.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:36:36.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.219017+0000", 2023-10-09T09:36:36.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.694e-06 2023-10-09T09:36:36.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:36:36.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.219077+0000", 2023-10-09T09:36:36.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.029e-05 2023-10-09T09:36:36.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:36:36.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.219253+0000", 2023-10-09T09:36:36.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000175868 2023-10-09T09:36:36.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:36:36.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.219469+0000", 2023-10-09T09:36:36.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000216395 2023-10-09T09:36:36.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:36:36.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.222958+0000", 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034883010000000001 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.223583+0000", 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00062577800000000001 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.235 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:80735 8.c 8:303a8d77:::benchmark_data_smithi163_151167_object80734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:36.224501+0000", 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.003142504, 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003245431, 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80735 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:36:36.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.224501+0000", 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.224501+0000", 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:36:36.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.224499+0000", 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.224533+0000", 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4057999999999998e-05 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.224554+0000", 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0560999999999998e-05 2023-10-09T09:36:36.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.224565+0000", 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1090999999999999e-05 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.226099+0000", 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015341140000000001 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.226258+0000", 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015920100000000001 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.226429+0000", 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017110199999999999 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.241 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.45122.0:80727 8.c e1148/1136)", 2023-10-09T09:36:36.241 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:36.226214+0000", 2023-10-09T09:36:36.241 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0014301749999999999, 2023-10-09T09:36:36.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001583349, 2023-10-09T09:36:36.241 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:36:36.241 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:36:36.241 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:36:36.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:36:36.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.226214+0000", 2023-10-09T09:36:36.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:36:36.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:36:36.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.226214+0000", 2023-10-09T09:36:36.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:36:36.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:36:36.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.226213+0000", 2023-10-09T09:36:36.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:36:36.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:36:36.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.226221+0000", 2023-10-09T09:36:36.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3019999999999995e-06 2023-10-09T09:36:36.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:36:36.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.226237+0000", 2023-10-09T09:36:36.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6126000000000001e-05 2023-10-09T09:36:36.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:36:36.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:36:36.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:36:36.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:36.226248+0000", 2023-10-09T09:36:36.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0558e-05 2023-10-09T09:36:36.250 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:36:36.250 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:36:36.250 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:36:36.250 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:36:36.250 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T09:36:36.250 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 3 2023-10-09T09:36:36.250 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:36:36.251 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:36.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:36.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:36.387 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:36.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:36.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:36.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:36.524 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:36.556 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:36.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:36.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:36.659 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:36.677 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:36.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:36.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:36.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:36.785 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 81163 81147 73.4953 48.375 0.0128798 0.0135992 2023-10-09T09:36:36.806 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:36.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:36.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:36.909 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:36.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:36.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:36.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:37.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:37.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:37.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:37.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:37.175 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:37.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:37.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:37.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:37.337 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:37.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:37.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:37.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:37.463 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:37.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:37.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:37.567 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:37.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:37.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:37.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:37.700 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:37.728 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:37.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:37.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:37.785 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 81930 81914 73.1301 47.9375 0.042803 0.0136673 2023-10-09T09:36:37.835 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:37.844 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:37 smithi181 bash[59345]: cluster 2023-10-09T09:36:35.858181+0000 mgr.y (mgr.34107) 913 : cluster 0 pgmap v1662: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 603 op/s; 4825/233109 objects degraded (2.070%); 4456/233109 objects misplaced (1.912%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T09:36:37.865 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:37.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:37.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:37.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:37.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:37.995 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:37 smithi163 bash[78107]: cluster 2023-10-09T09:36:35.858181+0000 mgr.y (mgr.34107) 913 : cluster 0 pgmap v1662: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 603 op/s; 4825/233109 objects degraded (2.070%); 4456/233109 objects misplaced (1.912%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T09:36:37.995 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:37 smithi163 bash[75425]: cluster 2023-10-09T09:36:35.858181+0000 mgr.y (mgr.34107) 913 : cluster 0 pgmap v1662: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 603 op/s; 4825/233109 objects degraded (2.070%); 4456/233109 objects misplaced (1.912%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T09:36:37.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:37.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:38.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:38.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:38.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:38.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:38.235 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:38.264 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:38.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:38.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:38.372 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:38.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:38.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:38.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:38.505 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:38.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:38.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:38.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:38.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:38.679 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:38.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:38.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:38.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:36:38.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:38.785 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 82729 82713 72.8034 49.9375 0.0225254 0.0137299 2023-10-09T09:36:38.813 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:38.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:38.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:38.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:38.957 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:38.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:38.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:39.061 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:39.087 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:39.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:39.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:39.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:39.224 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:39.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:39.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:39.329 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:39.352 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:39.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:39.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:39.455 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:39.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:39.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:39.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:39.588 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:39.627 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:39.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:39.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:39.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:39.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:39.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:39.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:39.785 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 83548 83532 72.5031 51.1875 0.0208741 0.0137868 2023-10-09T09:36:39.842 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T09:36:39.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:39.886 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:39 smithi181 bash[59345]: cluster 2023-10-09T09:36:37.859153+0000 mgr.y (mgr.34107) 914 : cluster 0 pgmap v1663: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 884 op/s; 4825/241365 objects degraded (1.999%); 4350/241365 objects misplaced (1.802%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T09:36:39.913 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:39.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:39.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:40.017 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:40.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:40.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:40.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:40.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:40.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:40.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:40.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:40.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:39 smithi163 bash[75425]: cluster 2023-10-09T09:36:37.859153+0000 mgr.y (mgr.34107) 914 : cluster 0 pgmap v1663: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 884 op/s; 4825/241365 objects degraded (1.999%); 4350/241365 objects misplaced (1.802%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T09:36:40.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:39 smithi163 bash[78107]: cluster 2023-10-09T09:36:37.859153+0000 mgr.y (mgr.34107) 914 : cluster 0 pgmap v1663: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 884 op/s; 4825/241365 objects degraded (1.999%); 4350/241365 objects misplaced (1.802%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T09:36:40.246 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:36:40.290 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:40.319 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:40.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:40.423 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:40.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:40.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:40.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:40.560 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:40.583 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:40.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:40.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:40.688 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:40.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:40.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:40.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:40.785 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 84164 84148 72.0372 38.5 0.0167248 0.0138742 2023-10-09T09:36:40.829 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:40.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:40.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:40.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:40.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:41.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:41.002 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:41.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:41.103 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:41.135 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:41.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:41.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:41.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:41.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:41.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:41.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:41.389 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:41.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:41.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:41.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:41.506 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:41 smithi163 bash[78107]: cluster 2023-10-09T09:36:39.859757+0000 mgr.y (mgr.34107) 915 : cluster 0 pgmap v1664: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 758 op/s; 4825/245022 objects degraded (1.969%); 4145/245022 objects misplaced (1.692%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T09:36:41.506 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:41 smithi163 bash[78107]: audit 2023-10-09T09:36:40.378551+0000 mon.a (mon.0) 1810 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:41.506 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:41 smithi163 bash[78107]: audit 2023-10-09T09:36:40.379667+0000 mon.a (mon.0) 1811 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:41.506 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:41 smithi163 bash[78107]: cluster 2023-10-09T09:36:40.649770+0000 mon.a (mon.0) 1812 : cluster 3 Health check update: Degraded data redundancy: 4825/241365 objects degraded (1.999%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:41.507 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:41 smithi163 bash[75425]: cluster 2023-10-09T09:36:39.859757+0000 mgr.y (mgr.34107) 915 : cluster 0 pgmap v1664: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 758 op/s; 4825/245022 objects degraded (1.969%); 4145/245022 objects misplaced (1.692%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T09:36:41.507 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:41 smithi163 bash[75425]: audit 2023-10-09T09:36:40.378551+0000 mon.a (mon.0) 1810 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:41.507 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:41 smithi163 bash[75425]: audit 2023-10-09T09:36:40.379667+0000 mon.a (mon.0) 1811 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:41.507 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:41 smithi163 bash[75425]: cluster 2023-10-09T09:36:40.649770+0000 mon.a (mon.0) 1812 : cluster 3 Health check update: Degraded data redundancy: 4825/241365 objects degraded (1.999%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:41.537 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:41.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:41.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:41.640 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:41.649 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:41 smithi181 bash[59345]: cluster 2023-10-09T09:36:39.859757+0000 mgr.y (mgr.34107) 915 : cluster 0 pgmap v1664: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 758 op/s; 4825/245022 objects degraded (1.969%); 4145/245022 objects misplaced (1.692%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T09:36:41.650 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:41 smithi181 bash[59345]: audit 2023-10-09T09:36:40.378551+0000 mon.a (mon.0) 1810 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:41.650 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:41 smithi181 bash[59345]: audit 2023-10-09T09:36:40.379667+0000 mon.a (mon.0) 1811 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:41.650 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:41 smithi181 bash[59345]: cluster 2023-10-09T09:36:40.649770+0000 mon.a (mon.0) 1812 : cluster 3 Health check update: Degraded data redundancy: 4825/241365 objects degraded (1.999%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:41.682 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:41.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:41.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:41.786 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 84951 84935 71.7284 49.1875 0.0124669 0.013935 2023-10-09T09:36:41.788 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:41.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:41.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:41.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:41.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:41.978 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:41.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:41.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:42.081 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:42.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:42.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:42.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:42.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:42 smithi163 bash[75425]: debug 2023-10-09T09:36:42.192+0000 7f22f321a700 -1 mon.a@0(leader).osd e1148 definitely_dead 0 2023-10-09T09:36:42.224 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:42.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:42.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:42.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:42.368 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:42.397 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:42.399 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.2. 2023-10-09T09:36:42.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:42.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:42.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:42.531 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:42.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:42.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:42 smithi181 bash[59345]: audit 2023-10-09T09:36:42.190613+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.163:0/3675520760' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:36:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:42 smithi181 bash[59345]: audit 2023-10-09T09:36:42.195069+0000 mon.a (mon.0) 1813 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:36:42.635 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:42.646 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:42 smithi163 bash[78107]: audit 2023-10-09T09:36:42.190613+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.163:0/3675520760' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:36:42.646 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:42 smithi163 bash[78107]: audit 2023-10-09T09:36:42.195069+0000 mon.a (mon.0) 1813 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:36:42.646 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:42 smithi163 bash[75425]: audit 2023-10-09T09:36:42.190613+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.163:0/3675520760' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:36:42.646 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:42 smithi163 bash[75425]: audit 2023-10-09T09:36:42.195069+0000 mon.a (mon.0) 1813 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T09:36:42.667 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:42.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:42.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:42.770 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:42.786 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 85442 85426 71.1811 30.6875 0.00436174 0.0139738 2023-10-09T09:36:42.805 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:42.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:42.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:42.910 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:42.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:42.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:42.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:43.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:43.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:43.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:43.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:43.164 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:43.188 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:43.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:43.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:43.292 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:43.315 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:43.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:43.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:43.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:43.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:43.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:43.551 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:43.585 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:43.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:43.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:43.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:43.700 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:36:43] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.33.4" 2023-10-09T09:36:43.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[78107]: cluster 2023-10-09T09:36:41.860505+0000 mgr.y (mgr.34107) 916 : cluster 0 pgmap v1665: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 764 op/s; 4825/250731 objects degraded (1.924%); 4041/250731 objects misplaced (1.612%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:36:43.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[78107]: cluster 2023-10-09T09:36:42.388443+0000 osd.6 (osd.6) 427 : cluster 3 failed to encode map e1149 with expected crc 2023-10-09T09:36:43.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[78107]: audit 2023-10-09T09:36:42.391766+0000 mon.a (mon.0) 1814 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T09:36:43.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[78107]: cluster 2023-10-09T09:36:42.391877+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e1149: 8 total, 8 up, 8 in 2023-10-09T09:36:43.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[78107]: cluster 2023-10-09T09:36:42.394227+0000 osd.5 (osd.5) 712 : cluster 3 failed to encode map e1149 with expected crc 2023-10-09T09:36:43.701 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[75425]: cluster 2023-10-09T09:36:41.860505+0000 mgr.y (mgr.34107) 916 : cluster 0 pgmap v1665: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 764 op/s; 4825/250731 objects degraded (1.924%); 4041/250731 objects misplaced (1.612%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:36:43.701 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[75425]: cluster 2023-10-09T09:36:42.388443+0000 osd.6 (osd.6) 427 : cluster 3 failed to encode map e1149 with expected crc 2023-10-09T09:36:43.701 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[75425]: audit 2023-10-09T09:36:42.391766+0000 mon.a (mon.0) 1814 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T09:36:43.701 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[75425]: cluster 2023-10-09T09:36:42.391877+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e1149: 8 total, 8 up, 8 in 2023-10-09T09:36:43.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:43 smithi163 bash[75425]: cluster 2023-10-09T09:36:42.394227+0000 osd.5 (osd.5) 712 : cluster 3 failed to encode map e1149 with expected crc 2023-10-09T09:36:43.718 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:43.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:43.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:43.784 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T09:36:43.786 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 85541 85525 70.3259 6.1875 0.00497641 0.0141362 2023-10-09T09:36:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:43 smithi181 bash[59345]: cluster 2023-10-09T09:36:41.860505+0000 mgr.y (mgr.34107) 916 : cluster 0 pgmap v1665: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 764 op/s; 4825/250731 objects degraded (1.924%); 4041/250731 objects misplaced (1.612%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T09:36:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:43 smithi181 bash[59345]: cluster 2023-10-09T09:36:42.388443+0000 osd.6 (osd.6) 427 : cluster 3 failed to encode map e1149 with expected crc 2023-10-09T09:36:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:43 smithi181 bash[59345]: audit 2023-10-09T09:36:42.391766+0000 mon.a (mon.0) 1814 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T09:36:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:43 smithi181 bash[59345]: cluster 2023-10-09T09:36:42.391877+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e1149: 8 total, 8 up, 8 in 2023-10-09T09:36:43.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:43 smithi181 bash[59345]: cluster 2023-10-09T09:36:42.394227+0000 osd.5 (osd.5) 712 : cluster 3 failed to encode map e1149 with expected crc 2023-10-09T09:36:43.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:43.854 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:43.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:43.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:43.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:43.990 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:43.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:43.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:44.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:44.125 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:44.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:44.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:44.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:44.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:44.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:44.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:44.265 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:36:44.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:44.399 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:44.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:44.502 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:44.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.392529+0000 osd.5 (osd.5) 713 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.395898+0000 osd.5 (osd.5) 714 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.395977+0000 mon.a (mon.0) 1816 : cluster 0 osdmap e1150: 8 total, 8 up, 8 in 2023-10-09T09:36:44.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.396463+0000 osd.4 (osd.4) 571 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.397334+0000 osd.5 (osd.5) 715 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.398438+0000 osd.6 (osd.6) 428 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.398552+0000 osd.7 (osd.7) 547 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.399032+0000 osd.5 (osd.5) 716 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.401121+0000 osd.6 (osd.6) 429 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.402247+0000 osd.4 (osd.4) 572 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.402392+0000 osd.7 (osd.7) 548 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.405359+0000 osd.6 (osd.6) 430 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.407932+0000 osd.6 (osd.6) 431 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.409893+0000 osd.4 (osd.4) 573 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.514 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.392529+0000 osd.5 (osd.5) 713 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.514 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.395898+0000 osd.5 (osd.5) 714 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.514 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.395977+0000 mon.a (mon.0) 1816 : cluster 0 osdmap e1150: 8 total, 8 up, 8 in 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.396463+0000 osd.4 (osd.4) 571 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.397334+0000 osd.5 (osd.5) 715 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.398438+0000 osd.6 (osd.6) 428 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.398552+0000 osd.7 (osd.7) 547 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.399032+0000 osd.5 (osd.5) 716 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.401121+0000 osd.6 (osd.6) 429 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.402247+0000 osd.4 (osd.4) 572 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.402392+0000 osd.7 (osd.7) 548 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.405359+0000 osd.6 (osd.6) 430 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.407932+0000 osd.6 (osd.6) 431 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:44 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.409893+0000 osd.4 (osd.4) 573 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.529 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:44.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:44.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:44.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:44.655 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:44.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:44.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:44.757 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:44.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:44.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:44.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:44.786 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 85543 85527 69.4142 0.125 0.00481735 0.0141361 2023-10-09T09:36:44.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.392529+0000 osd.5 (osd.5) 713 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.395898+0000 osd.5 (osd.5) 714 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.395977+0000 mon.a (mon.0) 1816 : cluster 0 osdmap e1150: 8 total, 8 up, 8 in 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.396463+0000 osd.4 (osd.4) 571 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.397334+0000 osd.5 (osd.5) 715 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.398438+0000 osd.6 (osd.6) 428 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.398552+0000 osd.7 (osd.7) 547 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.399032+0000 osd.5 (osd.5) 716 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.401121+0000 osd.6 (osd.6) 429 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.402247+0000 osd.4 (osd.4) 572 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.402392+0000 osd.7 (osd.7) 548 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.818 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.405359+0000 osd.6 (osd.6) 430 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.407932+0000 osd.6 (osd.6) 431 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.819 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:44 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.409893+0000 osd.4 (osd.4) 573 : cluster 3 failed to encode map e1150 with expected crc 2023-10-09T09:36:44.884 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:44.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:44.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:44.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:45.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:45.040 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:45.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:45.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:45.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:45.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:45.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:45.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:45.277 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:45.301 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:45.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:45.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:45.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:45.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:45 smithi181 bash[59345]: cluster 2023-10-09T09:36:43.861174+0000 mgr.y (mgr.34107) 917 : cluster 0 pgmap v1668: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s; 4825/252837 objects degraded (1.908%); 4041/252837 objects misplaced (1.598%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:36:45.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:45 smithi181 bash[59345]: cluster 2023-10-09T09:36:44.410332+0000 osd.5 (osd.5) 717 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:45 smithi181 bash[59345]: cluster 2023-10-09T09:36:44.411111+0000 osd.7 (osd.7) 549 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:45 smithi181 bash[59345]: cluster 2023-10-09T09:36:44.416104+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e1151: 8 total, 8 up, 8 in 2023-10-09T09:36:45.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:45 smithi181 bash[59345]: cluster 2023-10-09T09:36:44.418412+0000 osd.1 (osd.1) 1 : cluster 0 8.a starting backfill to osd.2 from (0'0,1139'2322] MIN to 1148'5226 2023-10-09T09:36:45.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:45 smithi181 bash[59345]: cluster 2023-10-09T09:36:44.418424+0000 osd.0 (osd.0) 1 : cluster 0 8.e starting backfill to osd.2 from (0'0,1139'2471] MIN to 1148'5430 2023-10-09T09:36:45.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:45 smithi181 bash[59345]: cluster 2023-10-09T09:36:44.431045+0000 osd.4 (osd.4) 574 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:45 smithi181 bash[59345]: cluster 2023-10-09T09:36:44.833825+0000 osd.6 (osd.6) 432 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.459 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:45.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:45.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:45.563 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:45.597 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:45.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:45.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:45.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:45.709 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[78107]: cluster 2023-10-09T09:36:43.861174+0000 mgr.y (mgr.34107) 917 : cluster 0 pgmap v1668: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s; 4825/252837 objects degraded (1.908%); 4041/252837 objects misplaced (1.598%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:36:45.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[78107]: cluster 2023-10-09T09:36:44.410332+0000 osd.5 (osd.5) 717 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[78107]: cluster 2023-10-09T09:36:44.411111+0000 osd.7 (osd.7) 549 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[78107]: cluster 2023-10-09T09:36:44.416104+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e1151: 8 total, 8 up, 8 in 2023-10-09T09:36:45.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[78107]: cluster 2023-10-09T09:36:44.418412+0000 osd.1 (osd.1) 1 : cluster 0 8.a starting backfill to osd.2 from (0'0,1139'2322] MIN to 1148'5226 2023-10-09T09:36:45.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[78107]: cluster 2023-10-09T09:36:44.418424+0000 osd.0 (osd.0) 1 : cluster 0 8.e starting backfill to osd.2 from (0'0,1139'2471] MIN to 1148'5430 2023-10-09T09:36:45.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[78107]: cluster 2023-10-09T09:36:44.431045+0000 osd.4 (osd.4) 574 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[78107]: cluster 2023-10-09T09:36:44.833825+0000 osd.6 (osd.6) 432 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[75425]: cluster 2023-10-09T09:36:43.861174+0000 mgr.y (mgr.34107) 917 : cluster 0 pgmap v1668: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s; 4825/252837 objects degraded (1.908%); 4041/252837 objects misplaced (1.598%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T09:36:45.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[75425]: cluster 2023-10-09T09:36:44.410332+0000 osd.5 (osd.5) 717 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.711 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[75425]: cluster 2023-10-09T09:36:44.411111+0000 osd.7 (osd.7) 549 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.711 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[75425]: cluster 2023-10-09T09:36:44.416104+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e1151: 8 total, 8 up, 8 in 2023-10-09T09:36:45.711 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[75425]: cluster 2023-10-09T09:36:44.418412+0000 osd.1 (osd.1) 1 : cluster 0 8.a starting backfill to osd.2 from (0'0,1139'2322] MIN to 1148'5226 2023-10-09T09:36:45.711 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[75425]: cluster 2023-10-09T09:36:44.418424+0000 osd.0 (osd.0) 1 : cluster 0 8.e starting backfill to osd.2 from (0'0,1139'2471] MIN to 1148'5430 2023-10-09T09:36:45.711 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[75425]: cluster 2023-10-09T09:36:44.431045+0000 osd.4 (osd.4) 574 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.711 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:45 smithi163 bash[75425]: cluster 2023-10-09T09:36:44.833825+0000 osd.6 (osd.6) 432 : cluster 3 failed to encode map e1151 with expected crc 2023-10-09T09:36:45.729 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:45.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:45.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:45.786 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 86278 86262 69.1131 45.9375 0.0130991 0.0144619 2023-10-09T09:36:45.835 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:45.866 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:45.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:45.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:45.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:46.002 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:46.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:46.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:46.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:46.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:46.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:46.240 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:46.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:46.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:46.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:46.302 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:36:46.302 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:36:46.371 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:46.402 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:46.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:46.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:46.506 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:46.542 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:46.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:46.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:46.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:46.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:46 smithi163 bash[78107]: cluster 2023-10-09T09:36:45.650955+0000 mon.a (mon.0) 1818 : cluster 3 Health check update: Degraded data redundancy: 4825/252837 objects degraded (1.908%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:46.658 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:46 smithi163 bash[75425]: cluster 2023-10-09T09:36:45.650955+0000 mon.a (mon.0) 1818 : cluster 3 Health check update: Degraded data redundancy: 4825/252837 objects degraded (1.908%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:46.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:46.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:46.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:46.782 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:46.786 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 87048 87032 68.8474 48.125 0.0131346 0.0145186 2023-10-09T09:36:46.806 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:46.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:46.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:46 smithi181 bash[59345]: cluster 2023-10-09T09:36:45.650955+0000 mon.a (mon.0) 1818 : cluster 3 Health check update: Degraded data redundancy: 4825/252837 objects degraded (1.908%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:46.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:46.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:46.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:46.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:47.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:47.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:47.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:47.172 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:47.206 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:47.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:47.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:47.316 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:47.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:47.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:47.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:47.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:47.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:47.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:47.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:47.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:47.613 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:47.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:47.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:47.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:47 smithi163 bash[75425]: cluster 2023-10-09T09:36:45.863431+0000 mgr.y (mgr.34107) 918 : cluster 0 pgmap v1670: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 570 op/s; 5382/255291 objects degraded (2.108%); 4990/255291 objects misplaced (1.955%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T09:36:47.717 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:47.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:47.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:47.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:47.786 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:36:47.786982+0000 min lat: 0.00132712 max lat: 2.45143 avg lat: 0.0145724 2023-10-09T09:36:47.786 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:36:47.787 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 87818 87802 68.5883 48.125 0.0117624 0.0145724 2023-10-09T09:36:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:47 smithi181 bash[59345]: cluster 2023-10-09T09:36:45.863431+0000 mgr.y (mgr.34107) 918 : cluster 0 pgmap v1670: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 570 op/s; 5382/255291 objects degraded (2.108%); 4990/255291 objects misplaced (1.955%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T09:36:47.861 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:47.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:47.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:47.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:47.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:47 smithi163 bash[78107]: cluster 2023-10-09T09:36:45.863431+0000 mgr.y (mgr.34107) 918 : cluster 0 pgmap v1670: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 570 op/s; 5382/255291 objects degraded (2.108%); 4990/255291 objects misplaced (1.955%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T09:36:47.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:48.023 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:48.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:48.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:48.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:48.154 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:48.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:48.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:48.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:48.286 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:48.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:48.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:48.391 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:48.427 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:48.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:48.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:48.534 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:48.563 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:48.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:48.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:48.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:48.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:48.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:48.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:48.784 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:36:48.784 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:36:48.785 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T09:36:48.785 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:36:48.785 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:36:48.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T09:36:48.786 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 88635 88619 68.3719 51.0625 0.0107138 0.0146192 2023-10-09T09:36:48.803 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:48.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:48.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:48.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:48.935 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:48.973 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:48.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:48.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:49.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:49.120 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:49.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:49.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:49.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:49.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:49.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:49.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:49.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:49.387 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:49.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:49.489 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:49.517 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:49.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:49.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:49.620 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:49.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:49.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:49.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:49.753 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:49.761 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:49 smithi181 bash[59345]: cluster 2023-10-09T09:36:47.864214+0000 mgr.y (mgr.34107) 919 : cluster 0 pgmap v1671: 89 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 31 MiB/s wr, 496 op/s; 8935/259677 objects degraded (3.441%); 8472/259677 objects misplaced (3.263%); 1.9 MiB/s, 1 keys/s, 31 objects/s recovering 2023-10-09T09:36:49.786 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 89411 89395 68.1295 48.5 0.0122444 0.0146717 2023-10-09T09:36:49.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:49.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:49.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:49.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:49.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:49.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:49 smithi163 bash[78107]: cluster 2023-10-09T09:36:47.864214+0000 mgr.y (mgr.34107) 919 : cluster 0 pgmap v1671: 89 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 31 MiB/s wr, 496 op/s; 8935/259677 objects degraded (3.441%); 8472/259677 objects misplaced (3.263%); 1.9 MiB/s, 1 keys/s, 31 objects/s recovering 2023-10-09T09:36:49.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:49 smithi163 bash[75425]: cluster 2023-10-09T09:36:47.864214+0000 mgr.y (mgr.34107) 919 : cluster 0 pgmap v1671: 89 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 31 MiB/s wr, 496 op/s; 8935/259677 objects degraded (3.441%); 8472/259677 objects misplaced (3.263%); 1.9 MiB/s, 1 keys/s, 31 objects/s recovering 2023-10-09T09:36:50.034 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:50.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:50.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:50.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:50.172 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:50.200 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:50.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:50.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:50.306 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:50.343 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:50.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:50.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:50.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:50.478 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:50.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:50.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:50.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:50.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:50.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:50.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:50.712 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:50.735 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:50.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:50.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:50.787 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 90227 90211 67.923 51 0.0381488 0.0147167 2023-10-09T09:36:50.838 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:50.866 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:50.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:50.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:50.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:50 smithi163 bash[75425]: cluster 2023-10-09T09:36:50.652235+0000 mon.a (mon.0) 1819 : cluster 3 Health check update: Degraded data redundancy: 8935/259677 objects degraded (3.441%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:50.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:50 smithi163 bash[78107]: cluster 2023-10-09T09:36:50.652235+0000 mon.a (mon.0) 1819 : cluster 3 Health check update: Degraded data redundancy: 8935/259677 objects degraded (3.441%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:50.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:50.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:50.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:50.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:50 smithi181 bash[59345]: cluster 2023-10-09T09:36:50.652235+0000 mon.a (mon.0) 1819 : cluster 3 Health check update: Degraded data redundancy: 8935/259677 objects degraded (3.441%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:51.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:51.126 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:51.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:51.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:51.257 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:51.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:51.365 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:51.400 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:51.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:51.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:51.507 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:51.529 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:51.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:51.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:51.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:51.658 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:51.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:51.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:51.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:51.787 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 90965 90949 67.6635 46.125 0.0143753 0.0147724 2023-10-09T09:36:51.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:51.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:51.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:51.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:51.920 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:51 smithi163 bash[75425]: cluster 2023-10-09T09:36:49.864779+0000 mgr.y (mgr.34107) 920 : cluster 0 pgmap v1672: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 27 MiB/s wr, 439 op/s; 8935/262695 objects degraded (3.401%); 13734/262695 objects misplaced (5.228%); 2.8 MiB/s, 1 keys/s, 46 objects/s recovering 2023-10-09T09:36:51.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:51 smithi163 bash[78107]: cluster 2023-10-09T09:36:49.864779+0000 mgr.y (mgr.34107) 920 : cluster 0 pgmap v1672: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 27 MiB/s wr, 439 op/s; 8935/262695 objects degraded (3.401%); 13734/262695 objects misplaced (5.228%); 2.8 MiB/s, 1 keys/s, 46 objects/s recovering 2023-10-09T09:36:51.939 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:51.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:51.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:52.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:52.055 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:51 smithi181 bash[59345]: cluster 2023-10-09T09:36:49.864779+0000 mgr.y (mgr.34107) 920 : cluster 0 pgmap v1672: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 27 MiB/s wr, 439 op/s; 8935/262695 objects degraded (3.401%); 13734/262695 objects misplaced (5.228%); 2.8 MiB/s, 1 keys/s, 46 objects/s recovering 2023-10-09T09:36:52.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:52.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:52.173 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:52.209 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:52.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:52.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:52.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:52.348 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:52.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:52.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:52.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:52.482 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:52.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:52.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:52.611 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:52.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:52.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:52.715 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:52.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:52.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:52.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:52.787 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 91659 91643 67.3777 43.375 0.00497079 0.014835 2023-10-09T09:36:52.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:52.878 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:52.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:52.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:52.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:53.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:53.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:53.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:53.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:53.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:53.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:53.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:53.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:53.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:53.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:53.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:53.359 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:36:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:36:53] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.33.4" 2023-10-09T09:36:53.383 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:53.422 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:53.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:53.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:53.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:53.556 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:53.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:53.604 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:36:53.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:53.701 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:53.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:53.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:53.707 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:36:53.787 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 92409 92393 67.1392 46.875 0.034049 0.0148872 2023-10-09T09:36:53.805 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:53 smithi181 bash[59345]: cluster 2023-10-09T09:36:51.865388+0000 mgr.y (mgr.34107) 921 : cluster 0 pgmap v1673: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 8935/268584 objects degraded (3.327%); 13734/268584 objects misplaced (5.113%); 2.5 MiB/s, 1 keys/s, 40 objects/s recovering 2023-10-09T09:36:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:53 smithi181 bash[59345]: audit 2023-10-09T09:36:53.509151+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.163:0/644011008' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:36:53.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:53 smithi181 bash[59345]: audit 2023-10-09T09:36:53.509861+0000 mon.a (mon.0) 1820 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:36:53.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:53.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:53.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:53.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:53.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:53 smithi163 bash[78107]: cluster 2023-10-09T09:36:51.865388+0000 mgr.y (mgr.34107) 921 : cluster 0 pgmap v1673: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 8935/268584 objects degraded (3.327%); 13734/268584 objects misplaced (5.113%); 2.5 MiB/s, 1 keys/s, 40 objects/s recovering 2023-10-09T09:36:53.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:53 smithi163 bash[78107]: audit 2023-10-09T09:36:53.509151+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.163:0/644011008' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:36:53.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:53 smithi163 bash[78107]: audit 2023-10-09T09:36:53.509861+0000 mon.a (mon.0) 1820 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:36:53.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:53 smithi163 bash[75425]: cluster 2023-10-09T09:36:51.865388+0000 mgr.y (mgr.34107) 921 : cluster 0 pgmap v1673: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 8935/268584 objects degraded (3.327%); 13734/268584 objects misplaced (5.113%); 2.5 MiB/s, 1 keys/s, 40 objects/s recovering 2023-10-09T09:36:53.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:53 smithi163 bash[75425]: audit 2023-10-09T09:36:53.509151+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.163:0/644011008' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:36:53.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:53 smithi163 bash[75425]: audit 2023-10-09T09:36:53.509861+0000 mon.a (mon.0) 1820 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:36:53.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:53.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:54.089 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:54.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:54.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:54.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:54.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:54.244 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:54.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:54.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:54.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:54.374 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:54.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:54.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:54.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:54.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:54.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:54.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:54.619 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:54.655 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:54.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:54.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:54.758 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:54.766 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:54 smithi163 bash[75425]: cluster 2023-10-09T09:36:53.688405+0000 mon.a (mon.0) 1821 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:36:54.766 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:54 smithi163 bash[75425]: cluster 2023-10-09T09:36:53.691155+0000 osd.5 (osd.5) 718 : cluster 3 failed to encode map e1152 with expected crc 2023-10-09T09:36:54.766 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:54 smithi163 bash[75425]: audit 2023-10-09T09:36:53.698609+0000 mon.a (mon.0) 1822 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:36:54.766 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:54 smithi163 bash[75425]: cluster 2023-10-09T09:36:53.698748+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e1152: 8 total, 8 up, 8 in 2023-10-09T09:36:54.766 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:54 smithi163 bash[75425]: cluster 2023-10-09T09:36:53.715734+0000 osd.4 (osd.4) 575 : cluster 3 failed to encode map e1152 with expected crc 2023-10-09T09:36:54.767 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:54 smithi163 bash[78107]: cluster 2023-10-09T09:36:53.688405+0000 mon.a (mon.0) 1821 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:36:54.767 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:54 smithi163 bash[78107]: cluster 2023-10-09T09:36:53.691155+0000 osd.5 (osd.5) 718 : cluster 3 failed to encode map e1152 with expected crc 2023-10-09T09:36:54.767 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:54 smithi163 bash[78107]: audit 2023-10-09T09:36:53.698609+0000 mon.a (mon.0) 1822 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:36:54.767 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:54 smithi163 bash[78107]: cluster 2023-10-09T09:36:53.698748+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e1152: 8 total, 8 up, 8 in 2023-10-09T09:36:54.767 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:54 smithi163 bash[78107]: cluster 2023-10-09T09:36:53.715734+0000 osd.4 (osd.4) 575 : cluster 3 failed to encode map e1152 with expected crc 2023-10-09T09:36:54.787 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 93160 93144 66.907 46.9375 0.0101103 0.0149397 2023-10-09T09:36:54.789 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:54.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:54.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:54.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:54.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:54.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:54.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:55.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:55.063 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:55.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:55.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:54 smithi181 bash[59345]: cluster 2023-10-09T09:36:53.688405+0000 mon.a (mon.0) 1821 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:36:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:54 smithi181 bash[59345]: cluster 2023-10-09T09:36:53.691155+0000 osd.5 (osd.5) 718 : cluster 3 failed to encode map e1152 with expected crc 2023-10-09T09:36:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:54 smithi181 bash[59345]: audit 2023-10-09T09:36:53.698609+0000 mon.a (mon.0) 1822 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:36:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:54 smithi181 bash[59345]: cluster 2023-10-09T09:36:53.698748+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e1152: 8 total, 8 up, 8 in 2023-10-09T09:36:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:54 smithi181 bash[59345]: cluster 2023-10-09T09:36:53.715734+0000 osd.4 (osd.4) 575 : cluster 3 failed to encode map e1152 with expected crc 2023-10-09T09:36:55.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:55.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:55.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:55.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:55.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:55.333 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:55.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:55.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:36:55.438 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:55.460 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:55.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:55.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:55.564 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:55.593 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:55.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:55.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:55.696 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:55.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:55.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:55.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:55.787 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 93674 93658 66.5117 32.125 0.217197 0.0150026 2023-10-09T09:36:55.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:55.855 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:55.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:55.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:55.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:55 smithi163 bash[78107]: cluster 2023-10-09T09:36:53.865875+0000 mgr.y (mgr.34107) 922 : cluster 0 pgmap v1675: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 545 op/s; 8935/270753 objects degraded (3.300%); 13992/270753 objects misplaced (5.168%); 1.1 MiB/s, 1 keys/s, 18 objects/s recovering 2023-10-09T09:36:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:55 smithi163 bash[78107]: audit 2023-10-09T09:36:55.383622+0000 mon.a (mon.0) 1824 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:55 smithi163 bash[78107]: audit 2023-10-09T09:36:55.389303+0000 mon.a (mon.0) 1825 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:55 smithi163 bash[78107]: cluster 2023-10-09T09:36:55.653702+0000 mon.a (mon.0) 1826 : cluster 3 Health check update: Degraded data redundancy: 8935/270753 objects degraded (3.300%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:55 smithi163 bash[75425]: cluster 2023-10-09T09:36:53.865875+0000 mgr.y (mgr.34107) 922 : cluster 0 pgmap v1675: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 545 op/s; 8935/270753 objects degraded (3.300%); 13992/270753 objects misplaced (5.168%); 1.1 MiB/s, 1 keys/s, 18 objects/s recovering 2023-10-09T09:36:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:55 smithi163 bash[75425]: audit 2023-10-09T09:36:55.383622+0000 mon.a (mon.0) 1824 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:55 smithi163 bash[75425]: audit 2023-10-09T09:36:55.389303+0000 mon.a (mon.0) 1825 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:55 smithi163 bash[75425]: cluster 2023-10-09T09:36:55.653702+0000 mon.a (mon.0) 1826 : cluster 3 Health check update: Degraded data redundancy: 8935/270753 objects degraded (3.300%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:55.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:55.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:55.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:55 smithi181 bash[59345]: cluster 2023-10-09T09:36:53.865875+0000 mgr.y (mgr.34107) 922 : cluster 0 pgmap v1675: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 545 op/s; 8935/270753 objects degraded (3.300%); 13992/270753 objects misplaced (5.168%); 1.1 MiB/s, 1 keys/s, 18 objects/s recovering 2023-10-09T09:36:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:55 smithi181 bash[59345]: audit 2023-10-09T09:36:55.383622+0000 mon.a (mon.0) 1824 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:36:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:55 smithi181 bash[59345]: audit 2023-10-09T09:36:55.389303+0000 mon.a (mon.0) 1825 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:36:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:55 smithi181 bash[59345]: cluster 2023-10-09T09:36:55.653702+0000 mon.a (mon.0) 1826 : cluster 3 Health check update: Degraded data redundancy: 8935/270753 objects degraded (3.300%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:36:56.099 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:56.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:56.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:56.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:56.258 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:56.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:56.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:56.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:56.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:56.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:56.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:56.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:56.522 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:56.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:56.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:56.654 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:56.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:56.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:56.760 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:56.787 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 94468 94452 66.3219 49.625 0.00739367 0.0150723 2023-10-09T09:36:56.795 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:56.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:56.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:56.901 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:56.930 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:56.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:56.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:57.037 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:57.062 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:57.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:57.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:57.189 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:57.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:57.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:57.293 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:57.322 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:57.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:57.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:57.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:57.453 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:57.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:57.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:57.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:57.582 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:57.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:57.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:57.687 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:57.712 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:57.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:57.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:57.716 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:57 smithi163 bash[78107]: cluster 2023-10-09T09:36:55.866613+0000 mgr.y (mgr.34107) 923 : cluster 0 pgmap v1676: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s; 8935/274329 objects degraded (3.257%); 14120/274329 objects misplaced (5.147%); 1.7 MiB/s, 0 keys/s, 27 objects/s recovering 2023-10-09T09:36:57.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:57 smithi163 bash[75425]: cluster 2023-10-09T09:36:55.866613+0000 mgr.y (mgr.34107) 923 : cluster 0 pgmap v1676: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s; 8935/274329 objects degraded (3.257%); 14120/274329 objects misplaced (5.147%); 1.7 MiB/s, 0 keys/s, 27 objects/s recovering 2023-10-09T09:36:57.787 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 90 10 95230 95220 66.1183 48 0.0133448 0.0151177 2023-10-09T09:36:57.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:57.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:57 smithi181 bash[59345]: cluster 2023-10-09T09:36:55.866613+0000 mgr.y (mgr.34107) 923 : cluster 0 pgmap v1676: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s; 8935/274329 objects degraded (3.257%); 14120/274329 objects misplaced (5.147%); 1.7 MiB/s, 0 keys/s, 27 objects/s recovering 2023-10-09T09:36:57.839 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:57.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:57.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:57.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:57.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:57.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:57.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:58.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:58.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:58.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:58.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:58.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:58.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:58.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:58.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:58.342 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:58.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:58.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:58.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:58.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:58.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:58.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:58.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:58.607 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:58.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:36:58.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:58.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:36:58.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:58.756 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:58.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:58.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:58.788 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.0129 2023-10-09T09:36:58.788 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 95230 2023-10-09T09:36:58.788 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T09:36:58.788 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T09:36:58.788 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 66.1225 2023-10-09T09:36:58.788 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 35.8204 2023-10-09T09:36:58.788 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 182.25 2023-10-09T09:36:58.788 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 0 2023-10-09T09:36:58.788 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1057 2023-10-09T09:36:58.789 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 573.127 2023-10-09T09:36:58.789 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 2916 2023-10-09T09:36:58.789 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 0 2023-10-09T09:36:58.789 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.0151186 2023-10-09T09:36:58.789 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0416738 2023-10-09T09:36:58.789 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 2.45143 2023-10-09T09:36:58.789 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.00132712 2023-10-09T09:36:58.802 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T09:36:58.860 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:58.889 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:58.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:58.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:58.995 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:59.033 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:36:59.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:59.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:36:59.137 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:59.171 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:59.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:59.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:59.277 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:59.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:36:59.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:59.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:36:59.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:59.451 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:36:59.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:59.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:36:59.554 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:59.577 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:36:59.577 INFO:teuthology.orchestra.run.smithi181.stdout: "success": "" 2023-10-09T09:36:59.577 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:36:59.579 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:36:59.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:59.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:36:59.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:36:59.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:59.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:36:59.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:59.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:36:59.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:59.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:59.844 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:36:59.845 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:36:59.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:36:59.922 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:36:59.946 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:59.955 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:36:59 smithi181 bash[59345]: cluster 2023-10-09T09:36:57.867351+0000 mgr.y (mgr.34107) 924 : cluster 0 pgmap v1677: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 8935/281730 objects degraded (3.171%); 14351/281730 objects misplaced (5.094%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T09:36:59.962 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:36:59.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:36:59.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:36:59.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:00.003 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:37:00.019 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:37:00.020 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:37:00.053 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:36:59 smithi163 bash[75425]: cluster 2023-10-09T09:36:57.867351+0000 mgr.y (mgr.34107) 924 : cluster 0 pgmap v1677: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 8935/281730 objects degraded (3.171%); 14351/281730 objects misplaced (5.094%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T09:37:00.053 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:36:59 smithi163 bash[78107]: cluster 2023-10-09T09:36:57.867351+0000 mgr.y (mgr.34107) 924 : cluster 0 pgmap v1677: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 8935/281730 objects degraded (3.171%); 14351/281730 objects misplaced (5.094%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T09:37:00.054 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:37:00.087 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:37:00.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:00.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:00.122 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:37:00.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:00.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:00.160 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:37:00.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:37:00.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:37:00.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:37:00.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:37:00.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:37:00.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:37:00.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:37:00.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:37:00.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:00.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:00.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:00.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:00.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:00.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:00.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:00.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:00.526 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:00.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:00.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:00.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:00.681 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:00.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:00.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:00.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:00.819 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:00.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:00.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:00.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:00.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:00 smithi181 bash[59345]: audit 2023-10-09T09:36:59.785181+0000 mon.a (mon.0) 1827 : audit 0 from='client.? 172.21.15.163:0/188732487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:37:00.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:00 smithi181 bash[59345]: cluster 2023-10-09T09:37:00.655457+0000 mon.a (mon.0) 1828 : cluster 3 Health check update: Degraded data redundancy: 8935/281730 objects degraded (3.171%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:00.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:00.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:00.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:01.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:01.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:01.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:01.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:01.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:01.230 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:01.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:01.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:01.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:00 smithi163 bash[78107]: audit 2023-10-09T09:36:59.785181+0000 mon.a (mon.0) 1827 : audit 0 from='client.? 172.21.15.163:0/188732487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:37:01.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:00 smithi163 bash[78107]: cluster 2023-10-09T09:37:00.655457+0000 mon.a (mon.0) 1828 : cluster 3 Health check update: Degraded data redundancy: 8935/281730 objects degraded (3.171%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:01.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:00 smithi163 bash[75425]: audit 2023-10-09T09:36:59.785181+0000 mon.a (mon.0) 1827 : audit 0 from='client.? 172.21.15.163:0/188732487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:37:01.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:00 smithi163 bash[75425]: cluster 2023-10-09T09:37:00.655457+0000 mon.a (mon.0) 1828 : cluster 3 Health check update: Degraded data redundancy: 8935/281730 objects degraded (3.171%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:01.337 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:01.364 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:01.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:01.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:01.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:01.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:01.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:01.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:01.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:01.627 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:01.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:01.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:01.731 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:01.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:01.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:01.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:01.862 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:01.870 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:01 smithi163 bash[78107]: cluster 2023-10-09T09:36:59.867989+0000 mgr.y (mgr.34107) 925 : cluster 0 pgmap v1678: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 8935/283980 objects degraded (3.146%); 14314/283980 objects misplaced (5.040%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:37:01.870 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:01 smithi163 bash[75425]: cluster 2023-10-09T09:36:59.867989+0000 mgr.y (mgr.34107) 925 : cluster 0 pgmap v1678: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 8935/283980 objects degraded (3.146%); 14314/283980 objects misplaced (5.040%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:37:01.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:01.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:01.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:01.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:02.007 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:01 smithi181 bash[59345]: cluster 2023-10-09T09:36:59.867989+0000 mgr.y (mgr.34107) 925 : cluster 0 pgmap v1678: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 8935/283980 objects degraded (3.146%); 14314/283980 objects misplaced (5.040%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:37:02.022 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:02.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:02.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:02.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T09:37:02.128 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:02.158 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:02.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:02.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:02.263 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:02.293 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:02.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:02.396 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:02.422 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:02.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:02.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:02.527 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:02.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:02.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:02.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:02.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:02.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:02.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:02.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:02.790 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:02.933 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:02.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:03.036 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:03.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:03.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:03.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:03.177 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:03.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:03.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:03.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:03.408 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:03.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:03.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:03.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:03.543 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:03.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:03.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:03.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:03.681 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:03.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:03.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:37:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:37:03] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.33.4" 2023-10-09T09:37:03.788 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:03.810 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:03.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:03.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:03.913 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:03.941 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:03.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:03.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:04.044 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:04.053 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:03 smithi163 bash[75425]: cluster 2023-10-09T09:37:01.868907+0000 mgr.y (mgr.34107) 926 : cluster 0 pgmap v1679: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 670 op/s; 8935/282048 objects degraded (3.168%); 14314/282048 objects misplaced (5.075%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:37:04.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:03 smithi163 bash[78107]: cluster 2023-10-09T09:37:01.868907+0000 mgr.y (mgr.34107) 926 : cluster 0 pgmap v1679: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 670 op/s; 8935/282048 objects degraded (3.168%); 14314/282048 objects misplaced (5.075%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:37:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:03 smithi181 bash[59345]: cluster 2023-10-09T09:37:01.868907+0000 mgr.y (mgr.34107) 926 : cluster 0 pgmap v1679: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 670 op/s; 8935/282048 objects degraded (3.168%); 14314/282048 objects misplaced (5.075%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:37:04.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:04.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:04.114 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:37:04.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:04.208 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:04.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:04.313 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:04.358 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:04.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:04.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:04.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:04.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:04.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:04.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:04.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:04.642 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:04.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:04.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:04.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:04.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:04.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:04.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:04.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:04.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:04.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:04.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:05.007 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:05.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:05.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:05.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:05.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:05.174 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:05.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:05.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:05.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:05.313 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:05.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:05.418 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:05.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:05.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:05.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:05.465 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:37:05.466 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:37:05.466 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2023-10-09T09:37:05.466 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:37:05.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:05.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:05.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:05.646 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:05.709 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:05.741 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:05.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:05.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:05.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:05.868 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:05.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:05.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:05.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:06.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:06.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:06.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:06.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:05 smithi181 bash[59345]: cluster 2023-10-09T09:37:03.869565+0000 mgr.y (mgr.34107) 927 : cluster 0 pgmap v1680: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 100 B/s rd, 31 MiB/s wr, 646 op/s; 8935/281046 objects degraded (3.179%); 14197/281046 objects misplaced (5.051%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:37:06.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:05 smithi181 bash[59345]: cluster 2023-10-09T09:37:05.659163+0000 mon.a (mon.0) 1829 : cluster 3 Health check update: Degraded data redundancy: 8935/281046 objects degraded (3.179%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:06.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:06.140 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:06.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:06.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:06.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:05 smithi163 bash[75425]: cluster 2023-10-09T09:37:03.869565+0000 mgr.y (mgr.34107) 927 : cluster 0 pgmap v1680: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 100 B/s rd, 31 MiB/s wr, 646 op/s; 8935/281046 objects degraded (3.179%); 14197/281046 objects misplaced (5.051%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:37:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:05 smithi163 bash[75425]: cluster 2023-10-09T09:37:05.659163+0000 mon.a (mon.0) 1829 : cluster 3 Health check update: Degraded data redundancy: 8935/281046 objects degraded (3.179%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:05 smithi163 bash[78107]: cluster 2023-10-09T09:37:03.869565+0000 mgr.y (mgr.34107) 927 : cluster 0 pgmap v1680: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 100 B/s rd, 31 MiB/s wr, 646 op/s; 8935/281046 objects degraded (3.179%); 14197/281046 objects misplaced (5.051%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T09:37:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:05 smithi163 bash[78107]: cluster 2023-10-09T09:37:05.659163+0000 mon.a (mon.0) 1829 : cluster 3 Health check update: Degraded data redundancy: 8935/281046 objects degraded (3.179%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:06.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:06.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:06.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:06.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:06.371 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:06.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:06.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:06.508 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:06.541 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:06.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:06.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:06.643 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:06.666 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:06.666 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:06.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:06.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:06.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:06.796 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:06.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:06.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:06.902 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:06.927 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:06.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:06.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:07.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:07.062 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:07.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:07.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:07.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:07.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:07.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:07.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:07.312 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:07.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:07.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:07.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:07.448 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:07.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:07.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:07.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:07.585 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:07.618 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:07.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:07.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:07.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:07.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:07.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:07.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:07.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:07.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:08.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:08.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:07 smithi181 bash[59345]: cluster 2023-10-09T09:37:05.870209+0000 mgr.y (mgr.34107) 928 : cluster 0 pgmap v1681: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 650 op/s; 8935/277347 objects degraded (3.222%); 13835/277347 objects misplaced (4.988%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T09:37:08.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:08.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:08.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:08.137 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:08.145 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:07 smithi163 bash[78107]: cluster 2023-10-09T09:37:05.870209+0000 mgr.y (mgr.34107) 928 : cluster 0 pgmap v1681: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 650 op/s; 8935/277347 objects degraded (3.222%); 13835/277347 objects misplaced (4.988%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T09:37:08.145 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:07 smithi163 bash[75425]: cluster 2023-10-09T09:37:05.870209+0000 mgr.y (mgr.34107) 928 : cluster 0 pgmap v1681: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 650 op/s; 8935/277347 objects degraded (3.222%); 13835/277347 objects misplaced (4.988%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T09:37:08.168 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:08.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:08.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:08.274 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:08.303 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:08.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:08.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:08.411 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:08.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:08.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:08.559 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:08.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:08.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:08.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:08.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:08.704 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:08.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:08.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:08.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:08.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:08.855 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:08.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:08.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:08.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:08.974 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:08.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:08.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:09.076 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:09.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:09.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:09.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:09.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:09.243 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:09.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:09.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:09.373 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:09.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:09.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:09.478 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:09.504 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:09.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:09.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:09.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:09.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:09.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:09.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:09.722 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:09.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:09.763 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:09 smithi163 bash[75425]: cluster 2023-10-09T09:37:07.870974+0000 mgr.y (mgr.34107) 929 : cluster 0 pgmap v1682: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 773 op/s; 8935/269364 objects degraded (3.317%); 13588/269364 objects misplaced (5.044%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:37:09.767 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:09.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:09.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:09.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:09.881 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:09 smithi181 bash[59345]: cluster 2023-10-09T09:37:07.870974+0000 mgr.y (mgr.34107) 929 : cluster 0 pgmap v1682: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 773 op/s; 8935/269364 objects degraded (3.317%); 13588/269364 objects misplaced (5.044%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:37:09.905 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:09.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:09.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:10.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:10.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:10.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:10.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:10.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:10.158 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:09 smithi163 bash[78107]: cluster 2023-10-09T09:37:07.870974+0000 mgr.y (mgr.34107) 929 : cluster 0 pgmap v1682: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 773 op/s; 8935/269364 objects degraded (3.317%); 13588/269364 objects misplaced (5.044%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:37:10.173 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:10.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:10.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:10.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:10.308 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:10.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:10.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:10.454 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:10.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:10.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:10.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:10.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:10.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:10.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:10.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:10.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:10.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:10.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:10.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:10.836 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:10.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:10.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:10.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:10 smithi181 bash[59345]: audit 2023-10-09T09:37:10.375641+0000 mon.a (mon.0) 1830 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:10.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:10 smithi181 bash[59345]: cluster 2023-10-09T09:37:10.664490+0000 mon.a (mon.0) 1831 : cluster 3 Health check update: Degraded data redundancy: 8935/269364 objects degraded (3.317%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:10.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:10.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:11.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:11.077 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:10 smithi163 bash[75425]: audit 2023-10-09T09:37:10.375641+0000 mon.a (mon.0) 1830 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:11.077 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:10 smithi163 bash[75425]: cluster 2023-10-09T09:37:10.664490+0000 mon.a (mon.0) 1831 : cluster 3 Health check update: Degraded data redundancy: 8935/269364 objects degraded (3.317%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:11.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:10 smithi163 bash[78107]: audit 2023-10-09T09:37:10.375641+0000 mon.a (mon.0) 1830 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:11.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:10 smithi163 bash[78107]: cluster 2023-10-09T09:37:10.664490+0000 mon.a (mon.0) 1831 : cluster 3 Health check update: Degraded data redundancy: 8935/269364 objects degraded (3.317%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:11.098 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:11.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:11.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:11.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:11.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:11.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:11.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:11.361 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:11.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:11.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:11.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:11.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:11.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:11.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:11.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:11.640 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:11.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:11.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:11.771 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:11.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:11.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:11.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:11.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:11.904 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:11 smithi163 bash[78107]: cluster 2023-10-09T09:37:09.872300+0000 mgr.y (mgr.34107) 930 : cluster 0 pgmap v1683: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 7.0 MiB/s wr, 660 op/s; 8935/266001 objects degraded (3.359%); 13247/266001 objects misplaced (4.980%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:37:11.904 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:11 smithi163 bash[75425]: cluster 2023-10-09T09:37:09.872300+0000 mgr.y (mgr.34107) 930 : cluster 0 pgmap v1683: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 7.0 MiB/s wr, 660 op/s; 8935/266001 objects degraded (3.359%); 13247/266001 objects misplaced (4.980%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:37:11.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:11.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:12.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:12.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:11 smithi181 bash[59345]: cluster 2023-10-09T09:37:09.872300+0000 mgr.y (mgr.34107) 930 : cluster 0 pgmap v1683: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 7.0 MiB/s wr, 660 op/s; 8935/266001 objects degraded (3.359%); 13247/266001 objects misplaced (4.980%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:37:12.031 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:12.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:12.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:12.139 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:12.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:12.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:12.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:12.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:12.290 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:12.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:12.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:12.393 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:12.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:12.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:12.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:12.532 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:12.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:12.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:12.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:12.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:12.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:12.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:12.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:12.806 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:12.839 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:12.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:12.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:12.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:12.966 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:12.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:12.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:13.073 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:13.092 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:13.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:13.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:13.194 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:13.231 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:13.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:13.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:13.337 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:13.364 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:13.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:13.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:13.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:13.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:13.500 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:37:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:37:13] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.33.4" 2023-10-09T09:37:13.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:13.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:13.601 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:13.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:13.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:13.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:13.738 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:13.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:13.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:13.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:13.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:13.887 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:13 smithi181 bash[59345]: cluster 2023-10-09T09:37:11.873027+0000 mgr.y (mgr.34107) 931 : cluster 0 pgmap v1684: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 2.8 MiB/s wr, 738 op/s; 8935/260667 objects degraded (3.428%); 13247/260667 objects misplaced (5.082%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:37:13.906 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:13.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:13.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:14.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:14.036 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:14.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:13 smithi163 bash[75425]: cluster 2023-10-09T09:37:11.873027+0000 mgr.y (mgr.34107) 931 : cluster 0 pgmap v1684: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 2.8 MiB/s wr, 738 op/s; 8935/260667 objects degraded (3.428%); 13247/260667 objects misplaced (5.082%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:37:14.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:13 smithi163 bash[78107]: cluster 2023-10-09T09:37:11.873027+0000 mgr.y (mgr.34107) 931 : cluster 0 pgmap v1684: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 2.8 MiB/s wr, 738 op/s; 8935/260667 objects degraded (3.428%); 13247/260667 objects misplaced (5.082%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:37:14.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:14.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:14.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:14.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:14.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:14.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:14.275 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:14.301 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:14.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:14.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:14.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:14.439 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:14.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:14.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:14.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:14.571 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:14.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:14.680 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:14.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:14.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:14.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:14.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:14.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:14.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:14.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:14.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:14.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:15.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:15.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:15.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:15.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:15.232 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:15.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:15.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:15.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:15.405 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:15.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:15.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:15.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:15.540 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:15.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:15.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:15.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:15.681 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:15.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:15.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:15.787 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:15.820 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:15.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:15.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:15.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:15.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:15 smithi181 bash[59345]: cluster 2023-10-09T09:37:13.873611+0000 mgr.y (mgr.34107) 932 : cluster 0 pgmap v1685: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 682 KiB/s wr, 673 op/s; 8935/258555 objects degraded (3.456%); 12992/258555 objects misplaced (5.025%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:37:15.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:15 smithi181 bash[59345]: audit 2023-10-09T09:37:15.435074+0000 mon.a (mon.0) 1832 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:37:15.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:15 smithi181 bash[59345]: cluster 2023-10-09T09:37:15.665741+0000 mon.a (mon.0) 1833 : cluster 3 Health check update: Degraded data redundancy: 8935/258555 objects degraded (3.456%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:15.947 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:15.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:15.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:16.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:16.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:15 smithi163 bash[75425]: cluster 2023-10-09T09:37:13.873611+0000 mgr.y (mgr.34107) 932 : cluster 0 pgmap v1685: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 682 KiB/s wr, 673 op/s; 8935/258555 objects degraded (3.456%); 12992/258555 objects misplaced (5.025%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:37:16.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:15 smithi163 bash[75425]: audit 2023-10-09T09:37:15.435074+0000 mon.a (mon.0) 1832 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:37:16.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:15 smithi163 bash[75425]: cluster 2023-10-09T09:37:15.665741+0000 mon.a (mon.0) 1833 : cluster 3 Health check update: Degraded data redundancy: 8935/258555 objects degraded (3.456%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:16.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:15 smithi163 bash[78107]: cluster 2023-10-09T09:37:13.873611+0000 mgr.y (mgr.34107) 932 : cluster 0 pgmap v1685: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 682 KiB/s wr, 673 op/s; 8935/258555 objects degraded (3.456%); 12992/258555 objects misplaced (5.025%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:37:16.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:15 smithi163 bash[78107]: audit 2023-10-09T09:37:15.435074+0000 mon.a (mon.0) 1832 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:37:16.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:15 smithi163 bash[78107]: cluster 2023-10-09T09:37:15.665741+0000 mon.a (mon.0) 1833 : cluster 3 Health check update: Degraded data redundancy: 8935/258555 objects degraded (3.456%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:16.087 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:16.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:16.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:16.191 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:16.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:16.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:16.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:16.329 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:16.363 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:16.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:16.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:16.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:16.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:16.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:16.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:16.597 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:16.634 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:16.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:16.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:16.739 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:16.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:16.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:16.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:16.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:16.910 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:16.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:16.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:17.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:17.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:17.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:17.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:17.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:17.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:17.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:17.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:17.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:17.324 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:17.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:17.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:17.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:17.459 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:17.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:17.564 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:17.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:17.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:17.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:17.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:17.719 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:17.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:17.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:17.826 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:17.849 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:17.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:17.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:17.953 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:17.962 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:17 smithi163 bash[78107]: cluster 2023-10-09T09:37:15.874211+0000 mgr.y (mgr.34107) 933 : cluster 0 pgmap v1686: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8935/255141 objects degraded (3.502%); 12621/255141 objects misplaced (4.947%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T09:37:17.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:17 smithi163 bash[75425]: cluster 2023-10-09T09:37:15.874211+0000 mgr.y (mgr.34107) 933 : cluster 0 pgmap v1686: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8935/255141 objects degraded (3.502%); 12621/255141 objects misplaced (4.947%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T09:37:17.990 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:17.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:17.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:18.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:17 smithi181 bash[59345]: cluster 2023-10-09T09:37:15.874211+0000 mgr.y (mgr.34107) 933 : cluster 0 pgmap v1686: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8935/255141 objects degraded (3.502%); 12621/255141 objects misplaced (4.947%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T09:37:18.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:18.123 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:18.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:18.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:18.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:18.250 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:18.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:18.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:18.377 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:18.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:18.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:18.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:18.515 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:18.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:18.618 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:18.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:18.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:18.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:18.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:18.778 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:18.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:18.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:18.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:18.919 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:18.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:18.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:19.024 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:19.044 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:19.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:19.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:19.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:19.174 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:19.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:19.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:19.279 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:19.318 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:19.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:19.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:19.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:19.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:19.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:19.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:19.553 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:19.594 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:19.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:19.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:19.701 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:19.740 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:19.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:19 smithi181 bash[59345]: cluster 2023-10-09T09:37:17.875029+0000 mgr.y (mgr.34107) 934 : cluster 0 pgmap v1687: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 8935/247479 objects degraded (3.610%); 12429/247479 objects misplaced (5.022%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:19.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:19.875 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:20.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:20.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:20.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:20.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:19 smithi163 bash[75425]: cluster 2023-10-09T09:37:17.875029+0000 mgr.y (mgr.34107) 934 : cluster 0 pgmap v1687: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 8935/247479 objects degraded (3.610%); 12429/247479 objects misplaced (5.022%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:20.144 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:19 smithi163 bash[78107]: cluster 2023-10-09T09:37:17.875029+0000 mgr.y (mgr.34107) 934 : cluster 0 pgmap v1687: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 8935/247479 objects degraded (3.610%); 12429/247479 objects misplaced (5.022%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:20.171 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:20.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:20.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:20.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:20.315 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:20.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:20.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:20.420 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:20.449 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:20.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:20.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:20.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:20.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:20.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:20.858 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:20.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:20.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:20.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:20.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:20 smithi163 bash[75425]: cluster 2023-10-09T09:37:20.667792+0000 mon.a (mon.0) 1834 : cluster 3 Health check update: Degraded data redundancy: 8935/247479 objects degraded (3.610%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:20.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:20 smithi163 bash[78107]: cluster 2023-10-09T09:37:20.667792+0000 mon.a (mon.0) 1834 : cluster 3 Health check update: Degraded data redundancy: 8935/247479 objects degraded (3.610%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:21.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:21.049 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:21.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:21.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:21.052 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:20 smithi181 bash[59345]: cluster 2023-10-09T09:37:20.667792+0000 mon.a (mon.0) 1834 : cluster 3 Health check update: Degraded data redundancy: 8935/247479 objects degraded (3.610%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:21.152 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:21.184 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:21.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:21.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:21.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:21.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:21.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:21.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:21.446 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:21.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:21.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:21.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:21.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:21.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:21.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:21.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:21.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:21.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:21.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:21.919 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:21.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:22.024 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:22.054 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:22.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:21 smithi163 bash[75425]: cluster 2023-10-09T09:37:19.875579+0000 mgr.y (mgr.34107) 935 : cluster 0 pgmap v1688: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 8935/243891 objects degraded (3.664%); 12067/243891 objects misplaced (4.948%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T09:37:22.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:21 smithi163 bash[78107]: cluster 2023-10-09T09:37:19.875579+0000 mgr.y (mgr.34107) 935 : cluster 0 pgmap v1688: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 8935/243891 objects degraded (3.664%); 12067/243891 objects misplaced (4.948%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T09:37:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:22.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:21 smithi181 bash[59345]: cluster 2023-10-09T09:37:19.875579+0000 mgr.y (mgr.34107) 935 : cluster 0 pgmap v1688: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 8935/243891 objects degraded (3.664%); 12067/243891 objects misplaced (4.948%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T09:37:22.158 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:22.193 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:22.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:22.299 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:22.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:22.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:22.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:22.419 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:22.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:22.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:22.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:22.559 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:22.582 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:22.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:22.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:22.685 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:22.708 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:22.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:22.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:22.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:22.837 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:22.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:22.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:22.943 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:22.990 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:22.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:22.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:23.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:23.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:22 smithi181 bash[59345]: cluster 2023-10-09T09:37:21.876133+0000 mgr.y (mgr.34107) 936 : cluster 0 pgmap v1689: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 8935/238422 objects degraded (3.748%); 12067/238422 objects misplaced (5.061%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:23.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:22 smithi181 bash[59345]: audit 2023-10-09T09:37:21.991509+0000 mon.a (mon.0) 1835 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:23.106 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:22 smithi181 bash[59345]: audit 2023-10-09T09:37:21.999806+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:23.122 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:23.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:23.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:23.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:23.249 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:23.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:23.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:23.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:23.372 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:23.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:23.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:23.407 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:22 smithi163 bash[78107]: cluster 2023-10-09T09:37:21.876133+0000 mgr.y (mgr.34107) 936 : cluster 0 pgmap v1689: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 8935/238422 objects degraded (3.748%); 12067/238422 objects misplaced (5.061%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:23.408 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:22 smithi163 bash[78107]: audit 2023-10-09T09:37:21.991509+0000 mon.a (mon.0) 1835 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:23.408 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:22 smithi163 bash[78107]: audit 2023-10-09T09:37:21.999806+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:23.408 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:37:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:37:23] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.33.4" 2023-10-09T09:37:23.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:22 smithi163 bash[75425]: cluster 2023-10-09T09:37:21.876133+0000 mgr.y (mgr.34107) 936 : cluster 0 pgmap v1689: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 8935/238422 objects degraded (3.748%); 12067/238422 objects misplaced (5.061%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:23.409 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:22 smithi163 bash[75425]: audit 2023-10-09T09:37:21.991509+0000 mon.a (mon.0) 1835 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:23.409 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:22 smithi163 bash[75425]: audit 2023-10-09T09:37:21.999806+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:23.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:23.511 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:23.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:23.614 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:23.657 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:23.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:23.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:23.762 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:23.788 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:23.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:23.897 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:23.927 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:23.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:23.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:24.030 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:24.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:24.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:24.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:24.165 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:24.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:24.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:24.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:24.299 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:24.330 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:24.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:24.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:24.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:24.467 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:24.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:24.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:24.499 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:37:24.500 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:37:24.574 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:24.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:24.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:24.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:24.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:24.735 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:24.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:24.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:24.838 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:24.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:24.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:24.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:24.980 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:25.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:25.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:25.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:25.022 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:37:25.080 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:37:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:25 smithi163 bash[75425]: cluster 2023-10-09T09:37:23.876732+0000 mgr.y (mgr.34107) 937 : cluster 0 pgmap v1690: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8935/236454 objects degraded (3.779%); 11859/236454 objects misplaced (5.015%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:25 smithi163 bash[75425]: audit 2023-10-09T09:37:24.474310+0000 mon.b (mon.1) 17 : audit 1 from='client.? 172.21.15.163:0/4159822192' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:37:25.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:25 smithi163 bash[75425]: audit 2023-10-09T09:37:24.475571+0000 mon.a (mon.0) 1837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:37:25.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:25 smithi163 bash[78107]: cluster 2023-10-09T09:37:23.876732+0000 mgr.y (mgr.34107) 937 : cluster 0 pgmap v1690: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8935/236454 objects degraded (3.779%); 11859/236454 objects misplaced (5.015%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:25.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:25 smithi163 bash[78107]: audit 2023-10-09T09:37:24.474310+0000 mon.b (mon.1) 17 : audit 1 from='client.? 172.21.15.163:0/4159822192' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:37:25.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:25 smithi163 bash[78107]: audit 2023-10-09T09:37:24.475571+0000 mon.a (mon.0) 1837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:37:25.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:25.133 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:37:25.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:25.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:25.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:25.190 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:37:25.233 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:37:25.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:25.273 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:37:25.299 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:25.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:25.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:25 smithi181 bash[59345]: cluster 2023-10-09T09:37:23.876732+0000 mgr.y (mgr.34107) 937 : cluster 0 pgmap v1690: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8935/236454 objects degraded (3.779%); 11859/236454 objects misplaced (5.015%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:25 smithi181 bash[59345]: audit 2023-10-09T09:37:24.474310+0000 mon.b (mon.1) 17 : audit 1 from='client.? 172.21.15.163:0/4159822192' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:37:25.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:25 smithi181 bash[59345]: audit 2023-10-09T09:37:24.475571+0000 mon.a (mon.0) 1837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:37:25.325 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:37:25.370 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:37:25.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:25.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:25.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:25.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:25.435 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:37:25.445 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:37:25.445 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:37:25.445 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:37:25.446 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:37:25.446 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:37:25.446 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:37:25.446 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:37:25.446 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:37:25.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:25.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:25.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:25.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:25.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:25.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:25.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:25.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:25.823 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:25.837 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:37:25.839 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:37:25.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:25.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:25.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:25.951 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:25.983 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:25.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:25.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:26.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:26.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:26.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:26.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:26.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: cluster 2023-10-09T09:37:24.990970+0000 mon.a (mon.0) 1838 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:37:26.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: cluster 2023-10-09T09:37:24.996255+0000 osd.5 (osd.5) 719 : cluster 3 failed to encode map e1153 with expected crc 2023-10-09T09:37:26.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: cluster 2023-10-09T09:37:24.999175+0000 osd.6 (osd.6) 433 : cluster 3 failed to encode map e1153 with expected crc 2023-10-09T09:37:26.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: audit 2023-10-09T09:37:25.000464+0000 mon.a (mon.0) 1839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:37:26.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: cluster 2023-10-09T09:37:25.000664+0000 mon.a (mon.0) 1840 : cluster 0 osdmap e1153: 8 total, 8 up, 8 in 2023-10-09T09:37:26.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: audit 2023-10-09T09:37:25.010829+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.163:0/2915110476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:37:26.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: audit 2023-10-09T09:37:25.376372+0000 mon.a (mon.0) 1841 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:26.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: audit 2023-10-09T09:37:25.622316+0000 mon.a (mon.0) 1842 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:26.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: audit 2023-10-09T09:37:25.637346+0000 mon.a (mon.0) 1843 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:26.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: cluster 2023-10-09T09:37:25.668967+0000 mon.a (mon.0) 1844 : cluster 3 Health check update: Degraded data redundancy: 8935/236454 objects degraded (3.779%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:26.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:26 smithi181 bash[59345]: audit 2023-10-09T09:37:25.815190+0000 mgr.y (mgr.34107) 938 : audit 0 from='client.45191 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:37:26.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:26.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:26.348 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:26.372 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:26.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:26.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:26.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: cluster 2023-10-09T09:37:24.990970+0000 mon.a (mon.0) 1838 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:37:26.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: cluster 2023-10-09T09:37:24.996255+0000 osd.5 (osd.5) 719 : cluster 3 failed to encode map e1153 with expected crc 2023-10-09T09:37:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: cluster 2023-10-09T09:37:24.999175+0000 osd.6 (osd.6) 433 : cluster 3 failed to encode map e1153 with expected crc 2023-10-09T09:37:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: audit 2023-10-09T09:37:25.000464+0000 mon.a (mon.0) 1839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:37:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: cluster 2023-10-09T09:37:25.000664+0000 mon.a (mon.0) 1840 : cluster 0 osdmap e1153: 8 total, 8 up, 8 in 2023-10-09T09:37:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: audit 2023-10-09T09:37:25.010829+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.163:0/2915110476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:37:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: audit 2023-10-09T09:37:25.376372+0000 mon.a (mon.0) 1841 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: audit 2023-10-09T09:37:25.622316+0000 mon.a (mon.0) 1842 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: audit 2023-10-09T09:37:25.637346+0000 mon.a (mon.0) 1843 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: cluster 2023-10-09T09:37:25.668967+0000 mon.a (mon.0) 1844 : cluster 3 Health check update: Degraded data redundancy: 8935/236454 objects degraded (3.779%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[78107]: audit 2023-10-09T09:37:25.815190+0000 mgr.y (mgr.34107) 938 : audit 0 from='client.45191 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:37:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: cluster 2023-10-09T09:37:24.990970+0000 mon.a (mon.0) 1838 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:37:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: cluster 2023-10-09T09:37:24.996255+0000 osd.5 (osd.5) 719 : cluster 3 failed to encode map e1153 with expected crc 2023-10-09T09:37:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: cluster 2023-10-09T09:37:24.999175+0000 osd.6 (osd.6) 433 : cluster 3 failed to encode map e1153 with expected crc 2023-10-09T09:37:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: audit 2023-10-09T09:37:25.000464+0000 mon.a (mon.0) 1839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:37:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: cluster 2023-10-09T09:37:25.000664+0000 mon.a (mon.0) 1840 : cluster 0 osdmap e1153: 8 total, 8 up, 8 in 2023-10-09T09:37:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: audit 2023-10-09T09:37:25.010829+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.163:0/2915110476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:37:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: audit 2023-10-09T09:37:25.376372+0000 mon.a (mon.0) 1841 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: audit 2023-10-09T09:37:25.622316+0000 mon.a (mon.0) 1842 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: audit 2023-10-09T09:37:25.637346+0000 mon.a (mon.0) 1843 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: cluster 2023-10-09T09:37:25.668967+0000 mon.a (mon.0) 1844 : cluster 3 Health check update: Degraded data redundancy: 8935/236454 objects degraded (3.779%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:26 smithi163 bash[75425]: audit 2023-10-09T09:37:25.815190+0000 mgr.y (mgr.34107) 938 : audit 0 from='client.45191 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:37:26.480 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:26.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:26.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:26.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:26.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:26.656 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:26.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:26.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:26.797 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:26.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:26.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:26.902 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:26.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:26.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:26.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:27.034 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:27.065 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:27.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:27.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:27.169 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:27.196 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:27.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:27.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:27 smithi163 bash[75425]: cluster 2023-10-09T09:37:25.877161+0000 mgr.y (mgr.34107) 939 : cluster 0 pgmap v1692: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8935/233244 objects degraded (3.831%); 11541/233244 objects misplaced (4.948%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T09:37:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:27 smithi163 bash[75425]: audit 2023-10-09T09:37:26.207310+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:37:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:27 smithi163 bash[75425]: audit 2023-10-09T09:37:26.208008+0000 mon.a (mon.0) 1846 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:37:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:27 smithi163 bash[75425]: audit 2023-10-09T09:37:26.212475+0000 mon.a (mon.0) 1847 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:27.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:27 smithi163 bash[78107]: cluster 2023-10-09T09:37:25.877161+0000 mgr.y (mgr.34107) 939 : cluster 0 pgmap v1692: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8935/233244 objects degraded (3.831%); 11541/233244 objects misplaced (4.948%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T09:37:27.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:27 smithi163 bash[78107]: audit 2023-10-09T09:37:26.207310+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:37:27.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:27 smithi163 bash[78107]: audit 2023-10-09T09:37:26.208008+0000 mon.a (mon.0) 1846 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:37:27.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:27 smithi163 bash[78107]: audit 2023-10-09T09:37:26.212475+0000 mon.a (mon.0) 1847 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:27.299 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:27 smithi181 bash[59345]: cluster 2023-10-09T09:37:25.877161+0000 mgr.y (mgr.34107) 939 : cluster 0 pgmap v1692: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8935/233244 objects degraded (3.831%); 11541/233244 objects misplaced (4.948%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T09:37:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:27 smithi181 bash[59345]: audit 2023-10-09T09:37:26.207310+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:37:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:27 smithi181 bash[59345]: audit 2023-10-09T09:37:26.208008+0000 mon.a (mon.0) 1846 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:37:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:27 smithi181 bash[59345]: audit 2023-10-09T09:37:26.212475+0000 mon.a (mon.0) 1847 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:37:27.332 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:27.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:27.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:27.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:27.462 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:27.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:27.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:27.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:37:27.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:27.595 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:27.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:27.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:27.700 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:27.738 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:27.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:27.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:27.872 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:27.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:27.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:27.979 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:28.007 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:28.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:28.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:28.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:28.143 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:28.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:28.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:28.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:28.290 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:28.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:28.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:28.395 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:28.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:28.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:28.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:28.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:28.564 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:28.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:28.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:28.667 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:28.685 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:28.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:28.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:28.789 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:28.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:28.832 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:28.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:28.933 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:28.959 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:28.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:28.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:29.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:29.091 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:29.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:29.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:29.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:29.207 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:29 smithi181 bash[59345]: cluster 2023-10-09T09:37:27.877866+0000 mgr.y (mgr.34107) 940 : cluster 0 pgmap v1693: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8935/226875 objects degraded (3.938%); 11339/226875 objects misplaced (4.998%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T09:37:29.231 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:29.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:29.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:29.259 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1691,"stamp":"2023-10-09T09:37:24.998057+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5094250424,"num_objects":77748,"num_object_clones":0,"num_object_copies":233244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8935,"num_objects_misplaced":11541,"num_objects_unfound":0,"num_objects_dirty":77748,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":112998,"num_write_kb":6097166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6678,"num_bytes_recovered":437126296,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112932,"ondisk_log_size":112932,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17993372,"kb_used_data":15407324,"kb_used_omap":565,"kb_used_meta":2585098,"kb_avail":731967844,"statfs":{"total":767960285184,"available":749535072256,"internally_reserved":0,"allocated":15777099776,"data_stored":15771394292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":579342,"internal_metadata":2647140594},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-552927232,"num_objects":-8437,"num_object_clones":0,"num_object_copies":-25311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1451,"num_objects_unfound":0,"num_objects_dirty":-8437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8437,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":387,"num_bytes_recovered":25362432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.124835"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1137,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.434951+0000","last_change":"2023-10-09T09:36:43.418759+0000","last_active":"2023-10-09T09:36:44.434951+0000","last_peered":"2023-10-09T09:36:44.434951+0000","last_clean":"2023-10-09T09:36:44.434951+0000","last_became_active":"2023-10-09T09:36:43.418371+0000","last_became_peered":"2023-10-09T09:36:43.418371+0000","last_unstale":"2023-10-09T09:36:44.434951+0000","last_undegraded":"2023-10-09T09:36:44.434951+0000","last_fullsized":"2023-10-09T09:36:44.434951+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":22,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.420095+0000","last_change":"2023-10-09T09:35:48.698707+0000","last_active":"2023-10-09T09:36:44.420095+0000","last_peered":"2023-10-09T09:36:44.420095+0000","last_clean":"2023-10-09T09:36:44.420095+0000","last_became_active":"2023-10-09T09:35:48.698382+0000","last_became_peered":"2023-10-09T09:35:48.698382+0000","last_unstale":"2023-10-09T09:36:44.420095+0000","last_undegraded":"2023-10-09T09:36:44.420095+0000","last_fullsized":"2023-10-09T09:36:44.420095+0000","mapping_epoch":1140,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:01.722355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1091,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419304+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:36:44.419304+0000","last_peered":"2023-10-09T09:36:44.419304+0000","last_clean":"2023-10-09T09:36:44.419304+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:36:44.419304+0000","last_undegraded":"2023-10-09T09:36:44.419304+0000","last_fullsized":"2023-10-09T09:36:44.419304+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1073,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:36:53.724452+0000","last_change":"2023-10-09T09:35:18.684800+0000","last_active":"2023-10-09T09:36:53.724452+0000","last_peered":"2023-10-09T09:36:53.724452+0000","last_clean":"2023-10-09T09:36:53.724452+0000","last_became_active":"2023-10-09T09:35:18.681331+0000","last_became_peered":"2023-10-09T09:35:18.681331+0000","last_unstale":"2023-10-09T09:36:53.724452+0000","last_undegraded":"2023-10-09T09:36:53.724452+0000","last_fullsized":"2023-10-09T09:36:53.724452+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:38.385662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":615,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.417833+0000","last_change":"2023-10-09T09:35:18.670124+0000","last_active":"2023-10-09T09:36:44.417833+0000","last_peered":"2023-10-09T09:36:44.417833+0000","last_clean":"2023-10-09T09:36:44.417833+0000","last_became_active":"2023-10-09T09:35:18.668954+0000","last_became_peered":"2023-10-09T09:35:18.668954+0000","last_unstale":"2023-10-09T09:36:44.417833+0000","last_undegraded":"2023-10-09T09:36:44.417833+0000","last_fullsized":"2023-10-09T09:36:44.417833+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:42.439629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1083,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.839716+0000","last_change":"2023-10-09T09:14:40.495522+0000","last_active":"2023-10-09T09:36:44.839716+0000","last_peered":"2023-10-09T09:36:44.839716+0000","last_clean":"2023-10-09T09:36:44.839716+0000","last_became_active":"2023-10-09T09:14:40.495145+0000","last_became_peered":"2023-10-09T09:14:40.495145+0000","last_unstale":"2023-10-09T09:36:44.839716+0000","last_undegraded":"2023-10-09T09:36:44.839716+0000","last_fullsized":"2023-10-09T09:36:44.839716+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1072,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418583+0000","last_change":"2023-10-09T09:35:18.672962+0000","last_active":"2023-10-09T09:36:44.418583+0000","last_peered":"2023-10-09T09:36:44.418583+0000","last_clean":"2023-10-09T09:36:44.418583+0000","last_became_active":"2023-10-09T09:35:18.672134+0000","last_became_peered":"2023-10-09T09:35:18.672134+0000","last_unstale":"2023-10-09T09:36:44.418583+0000","last_undegraded":"2023-10-09T09:36:44.418583+0000","last_fullsized":"2023-10-09T09:36:44.418583+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:45.373989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1053,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.435400+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:36:44.435400+0000","last_peered":"2023-10-09T09:36:44.435400+0000","last_clean":"2023-10-09T09:36:44.435400+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:36:44.435400+0000","last_undegraded":"2023-10-09T09:36:44.435400+0000","last_fullsized":"2023-10-09T09:36:44.435400+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1104,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419390+0000","last_change":"2023-10-09T09:14:40.486667+0000","last_active":"2023-10-09T09:36:44.419390+0000","last_peered":"2023-10-09T09:36:44.419390+0000","last_clean":"2023-10-09T09:36:44.419390+0000","last_became_active":"2023-10-09T09:14:40.486316+0000","last_became_peered":"2023-10-09T09:14:40.486316+0000","last_unstale":"2023-10-09T09:36:44.419390+0000","last_undegraded":"2023-10-09T09:36:44.419390+0000","last_fullsized":"2023-10-09T09:36:44.419390+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":612,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.417750+0000","last_change":"2023-10-09T09:34:03.852834+0000","last_active":"2023-10-09T09:36:44.417750+0000","last_peered":"2023-10-09T09:36:44.417750+0000","last_clean":"2023-10-09T09:36:44.417750+0000","last_became_active":"2023-10-09T09:34:03.852517+0000","last_became_peered":"2023-10-09T09:34:03.852517+0000","last_unstale":"2023-10-09T09:36:44.417750+0000","last_undegraded":"2023-10-09T09:36:44.417750+0000","last_fullsized":"2023-10-09T09:36:44.417750+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1083,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419247+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:36:44.419247+0000","last_peered":"2023-10-09T09:36:44.419247+0000","last_clean":"2023-10-09T09:36:44.419247+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:36:44.419247+0000","last_undegraded":"2023-10-09T09:36:44.419247+0000","last_fullsized":"2023-10-09T09:36:44.419247+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1109,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418837+0000","last_change":"2023-10-09T09:36:43.421750+0000","last_active":"2023-10-09T09:36:44.418837+0000","last_peered":"2023-10-09T09:36:44.418837+0000","last_clean":"2023-10-09T09:36:44.418837+0000","last_became_active":"2023-10-09T09:36:43.421389+0000","last_became_peered":"2023-10-09T09:36:43.421389+0000","last_unstale":"2023-10-09T09:36:44.418837+0000","last_undegraded":"2023-10-09T09:36:44.418837+0000","last_fullsized":"2023-10-09T09:36:44.418837+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:15:05.597010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1110,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.435392+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:36:44.435392+0000","last_peered":"2023-10-09T09:36:44.435392+0000","last_clean":"2023-10-09T09:36:44.435392+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:36:44.435392+0000","last_undegraded":"2023-10-09T09:36:44.435392+0000","last_fullsized":"2023-10-09T09:36:44.435392+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1102,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.435755+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:36:44.435755+0000","last_peered":"2023-10-09T09:36:44.435755+0000","last_clean":"2023-10-09T09:36:44.435755+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:36:44.435755+0000","last_undegraded":"2023-10-09T09:36:44.435755+0000","last_fullsized":"2023-10-09T09:36:44.435755+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1112,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418555+0000","last_change":"2023-10-09T09:36:43.512444+0000","last_active":"2023-10-09T09:36:44.418555+0000","last_peered":"2023-10-09T09:36:44.418555+0000","last_clean":"2023-10-09T09:36:44.418555+0000","last_became_active":"2023-10-09T09:36:43.421525+0000","last_became_peered":"2023-10-09T09:36:43.421525+0000","last_unstale":"2023-10-09T09:36:44.418555+0000","last_undegraded":"2023-10-09T09:36:44.418555+0000","last_fullsized":"2023-10-09T09:36:44.418555+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":609,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418118+0000","last_change":"2023-10-09T09:34:03.848913+0000","last_active":"2023-10-09T09:36:44.418118+0000","last_peered":"2023-10-09T09:36:44.418118+0000","last_clean":"2023-10-09T09:36:44.418118+0000","last_became_active":"2023-10-09T09:34:03.848474+0000","last_became_peered":"2023-10-09T09:34:03.848474+0000","last_unstale":"2023-10-09T09:36:44.418118+0000","last_undegraded":"2023-10-09T09:36:44.418118+0000","last_fullsized":"2023-10-09T09:36:44.418118+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:59.618935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1072,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.417960+0000","last_change":"2023-10-09T09:35:18.670309+0000","last_active":"2023-10-09T09:36:44.417960+0000","last_peered":"2023-10-09T09:36:44.417960+0000","last_clean":"2023-10-09T09:36:44.417960+0000","last_became_active":"2023-10-09T09:35:18.669706+0000","last_became_peered":"2023-10-09T09:35:18.669706+0000","last_unstale":"2023-10-09T09:36:44.417960+0000","last_undegraded":"2023-10-09T09:36:44.417960+0000","last_fullsized":"2023-10-09T09:36:44.417960+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:39.673273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1103,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.424691+0000","last_change":"2023-10-09T09:15:36.175698+0000","last_active":"2023-10-09T09:36:44.424691+0000","last_peered":"2023-10-09T09:36:44.424691+0000","last_clean":"2023-10-09T09:36:44.424691+0000","last_became_active":"2023-10-09T09:15:36.175228+0000","last_became_peered":"2023-10-09T09:15:36.175228+0000","last_unstale":"2023-10-09T09:36:44.424691+0000","last_undegraded":"2023-10-09T09:36:44.424691+0000","last_fullsized":"2023-10-09T09:36:44.424691+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1065,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.417868+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:36:44.417868+0000","last_peered":"2023-10-09T09:36:44.417868+0000","last_clean":"2023-10-09T09:36:44.417868+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:36:44.417868+0000","last_undegraded":"2023-10-09T09:36:44.417868+0000","last_fullsized":"2023-10-09T09:36:44.417868+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1071,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419163+0000","last_change":"2023-10-09T09:35:18.685281+0000","last_active":"2023-10-09T09:36:44.419163+0000","last_peered":"2023-10-09T09:36:44.419163+0000","last_clean":"2023-10-09T09:36:44.419163+0000","last_became_active":"2023-10-09T09:35:18.684094+0000","last_became_peered":"2023-10-09T09:35:18.684094+0000","last_unstale":"2023-10-09T09:36:44.419163+0000","last_undegraded":"2023-10-09T09:36:44.419163+0000","last_fullsized":"2023-10-09T09:36:44.419163+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:13.291210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1108,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.838499+0000","last_change":"2023-10-09T09:35:18.668815+0000","last_active":"2023-10-09T09:36:44.838499+0000","last_peered":"2023-10-09T09:36:44.838499+0000","last_clean":"2023-10-09T09:36:44.838499+0000","last_became_active":"2023-10-09T09:35:18.668522+0000","last_became_peered":"2023-10-09T09:35:18.668522+0000","last_unstale":"2023-10-09T09:36:44.838499+0000","last_undegraded":"2023-10-09T09:36:44.838499+0000","last_fullsized":"2023-10-09T09:36:44.838499+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1097,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:36:53.715197+0000","last_change":"2023-10-09T09:35:48.703715+0000","last_active":"2023-10-09T09:36:53.715197+0000","last_peered":"2023-10-09T09:36:53.715197+0000","last_clean":"2023-10-09T09:36:53.715197+0000","last_became_active":"2023-10-09T09:35:48.703081+0000","last_became_peered":"2023-10-09T09:35:48.703081+0000","last_unstale":"2023-10-09T09:36:53.715197+0000","last_undegraded":"2023-10-09T09:36:53.715197+0000","last_fullsized":"2023-10-09T09:36:53.715197+0000","mapping_epoch":1140,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:41:38.594618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"1152'7024","reported_seq":7068,"reported_epoch":1152,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:37:21.573122+0000","last_change":"2023-10-09T09:36:43.598552+0000","last_active":"2023-10-09T09:37:21.573122+0000","last_peered":"2023-10-09T09:37:21.573122+0000","last_clean":"2023-10-09T09:35:47.668893+0000","last_became_active":"2023-10-09T09:36:43.575662+0000","last_became_peered":"2023-10-09T09:36:43.575662+0000","last_unstale":"2023-10-09T09:37:21.573122+0000","last_undegraded":"2023-10-09T09:36:43.434940+0000","last_fullsized":"2023-10-09T09:36:43.401787+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7024,"log_dups_size":0,"ondisk_log_size":7024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321126400,"num_objects":4900,"num_object_clones":0,"num_object_copies":14700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2956,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7024,"num_write_kb":381568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1944},{"shards":"4,5,6","objects":2956}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1096,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418990+0000","last_change":"2023-10-09T09:35:18.685969+0000","last_active":"2023-10-09T09:36:44.418990+0000","last_peered":"2023-10-09T09:36:44.418990+0000","last_clean":"2023-10-09T09:36:44.418990+0000","last_became_active":"2023-10-09T09:35:18.684984+0000","last_became_peered":"2023-10-09T09:35:18.684984+0000","last_unstale":"2023-10-09T09:36:44.418990+0000","last_undegraded":"2023-10-09T09:36:44.418990+0000","last_fullsized":"2023-10-09T09:36:44.418990+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1088,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418454+0000","last_change":"2023-10-09T09:03:25.754443+0000","last_active":"2023-10-09T09:36:44.418454+0000","last_peered":"2023-10-09T09:36:44.418454+0000","last_clean":"2023-10-09T09:36:44.418454+0000","last_became_active":"2023-10-09T09:03:25.753964+0000","last_became_peered":"2023-10-09T09:03:25.753964+0000","last_unstale":"2023-10-09T09:36:44.418454+0000","last_undegraded":"2023-10-09T09:36:44.418454+0000","last_fullsized":"2023-10-09T09:36:44.418454+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"1152'7032","reported_seq":7060,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:37:21.587160+0000","last_change":"2023-10-09T09:35:11.923645+0000","last_active":"2023-10-09T09:37:21.587160+0000","last_peered":"2023-10-09T09:37:21.587160+0000","last_clean":"2023-10-09T09:37:21.587160+0000","last_became_active":"2023-10-09T09:35:11.923211+0000","last_became_peered":"2023-10-09T09:35:11.923211+0000","last_unstale":"2023-10-09T09:37:21.587160+0000","last_undegraded":"2023-10-09T09:37:21.587160+0000","last_fullsized":"2023-10-09T09:37:21.587160+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320077824,"num_objects":4884,"num_object_clones":0,"num_object_copies":14652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7032,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1082,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418919+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:36:44.418919+0000","last_peered":"2023-10-09T09:36:44.418919+0000","last_clean":"2023-10-09T09:36:44.418919+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:36:44.418919+0000","last_undegraded":"2023-10-09T09:36:44.418919+0000","last_fullsized":"2023-10-09T09:36:44.418919+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1117,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.435015+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:36:44.435015+0000","last_peered":"2023-10-09T09:36:44.435015+0000","last_clean":"2023-10-09T09:36:44.435015+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:36:44.435015+0000","last_undegraded":"2023-10-09T09:36:44.435015+0000","last_fullsized":"2023-10-09T09:36:44.435015+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"1152'6942","reported_seq":6968,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:37:21.893729+0000","last_change":"2023-10-09T09:35:18.669719+0000","last_active":"2023-10-09T09:37:21.893729+0000","last_peered":"2023-10-09T09:37:21.893729+0000","last_clean":"2023-10-09T09:37:21.893729+0000","last_became_active":"2023-10-09T09:35:18.669369+0000","last_became_peered":"2023-10-09T09:35:18.669369+0000","last_unstale":"2023-10-09T09:37:21.893729+0000","last_undegraded":"2023-10-09T09:37:21.893729+0000","last_fullsized":"2023-10-09T09:37:21.893729+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":6942,"log_dups_size":0,"ondisk_log_size":6942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:18.617658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311689216,"num_objects":4756,"num_object_clones":0,"num_object_copies":14268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6942,"num_write_kb":374336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1066,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418961+0000","last_change":"2023-10-09T09:03:24.312247+0000","last_active":"2023-10-09T09:36:44.418961+0000","last_peered":"2023-10-09T09:36:44.418961+0000","last_clean":"2023-10-09T09:36:44.418961+0000","last_became_active":"2023-10-09T09:03:24.311811+0000","last_became_peered":"2023-10-09T09:03:24.311811+0000","last_unstale":"2023-10-09T09:36:44.418961+0000","last_undegraded":"2023-10-09T09:36:44.418961+0000","last_fullsized":"2023-10-09T09:36:44.418961+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1084,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.838755+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:36:44.838755+0000","last_peered":"2023-10-09T09:36:44.838755+0000","last_clean":"2023-10-09T09:36:44.838755+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:36:44.838755+0000","last_undegraded":"2023-10-09T09:36:44.838755+0000","last_fullsized":"2023-10-09T09:36:44.838755+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1152'7111","reported_seq":7141,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:37:24.116317+0000","last_change":"2023-10-09T09:35:12.107440+0000","last_active":"2023-10-09T09:37:24.116317+0000","last_peered":"2023-10-09T09:37:24.116317+0000","last_clean":"2023-10-09T09:37:24.116317+0000","last_became_active":"2023-10-09T09:35:12.106671+0000","last_became_peered":"2023-10-09T09:35:12.106671+0000","last_unstale":"2023-10-09T09:37:24.116317+0000","last_undegraded":"2023-10-09T09:37:24.116317+0000","last_fullsized":"2023-10-09T09:37:24.116317+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7111,"log_dups_size":0,"ondisk_log_size":7111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:21.342322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317784064,"num_objects":4849,"num_object_clones":0,"num_object_copies":14547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7111,"num_write_kb":382720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1104,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419966+0000","last_change":"2023-10-09T09:15:36.171634+0000","last_active":"2023-10-09T09:36:44.419966+0000","last_peered":"2023-10-09T09:36:44.419966+0000","last_clean":"2023-10-09T09:36:44.419966+0000","last_became_active":"2023-10-09T09:15:36.171249+0000","last_became_peered":"2023-10-09T09:15:36.171249+0000","last_unstale":"2023-10-09T09:36:44.419966+0000","last_undegraded":"2023-10-09T09:36:44.419966+0000","last_fullsized":"2023-10-09T09:36:44.419966+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1056,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418021+0000","last_change":"2023-10-09T09:35:18.681438+0000","last_active":"2023-10-09T09:36:44.418021+0000","last_peered":"2023-10-09T09:36:44.418021+0000","last_clean":"2023-10-09T09:36:44.418021+0000","last_became_active":"2023-10-09T09:35:18.680806+0000","last_became_peered":"2023-10-09T09:35:18.680806+0000","last_unstale":"2023-10-09T09:36:44.418021+0000","last_undegraded":"2023-10-09T09:36:44.418021+0000","last_fullsized":"2023-10-09T09:36:44.418021+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:30.669832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"1152'7013","reported_seq":7041,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:37:21.703140+0000","last_change":"2023-10-09T09:35:12.108597+0000","last_active":"2023-10-09T09:37:21.703140+0000","last_peered":"2023-10-09T09:37:21.703140+0000","last_clean":"2023-10-09T09:37:21.703140+0000","last_became_active":"2023-10-09T09:35:12.108156+0000","last_became_peered":"2023-10-09T09:35:12.108156+0000","last_unstale":"2023-10-09T09:37:21.703140+0000","last_undegraded":"2023-10-09T09:37:21.703140+0000","last_fullsized":"2023-10-09T09:37:21.703140+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317915136,"num_objects":4851,"num_object_clones":0,"num_object_copies":14553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7013,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":60,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418385+0000","last_change":"2023-10-09T09:35:18.683909+0000","last_active":"2023-10-09T09:36:44.418385+0000","last_peered":"2023-10-09T09:36:44.418385+0000","last_clean":"2023-10-09T09:36:44.418385+0000","last_became_active":"2023-10-09T09:35:18.678833+0000","last_became_peered":"2023-10-09T09:35:18.678833+0000","last_unstale":"2023-10-09T09:36:44.418385+0000","last_undegraded":"2023-10-09T09:36:44.418385+0000","last_fullsized":"2023-10-09T09:36:44.418385+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:47.275731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1102,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.435401+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:36:44.435401+0000","last_peered":"2023-10-09T09:36:44.435401+0000","last_clean":"2023-10-09T09:36:44.435401+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:36:44.435401+0000","last_undegraded":"2023-10-09T09:36:44.435401+0000","last_fullsized":"2023-10-09T09:36:44.435401+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"1152'7262","reported_seq":7313,"reported_epoch":1152,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:37:24.127032+0000","last_change":"2023-10-09T09:36:43.596548+0000","last_active":"2023-10-09T09:37:24.127032+0000","last_peered":"2023-10-09T09:37:24.127032+0000","last_clean":"2023-10-09T09:35:47.682702+0000","last_became_active":"2023-10-09T09:36:43.583902+0000","last_became_peered":"2023-10-09T09:36:43.583902+0000","last_unstale":"2023-10-09T09:37:24.127032+0000","last_undegraded":"2023-10-09T09:36:43.439966+0000","last_fullsized":"2023-10-09T09:36:43.400069+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7262,"log_dups_size":0,"ondisk_log_size":7262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:16.512338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322437120,"num_objects":4920,"num_object_clones":0,"num_object_copies":14760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3014,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7262,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1906},{"shards":"0,1,5","objects":3014}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1085,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.838875+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:36:44.838875+0000","last_peered":"2023-10-09T09:36:44.838875+0000","last_clean":"2023-10-09T09:36:44.838875+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:36:44.838875+0000","last_undegraded":"2023-10-09T09:36:44.838875+0000","last_fullsized":"2023-10-09T09:36:44.838875+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1135,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.435295+0000","last_change":"2023-10-09T09:36:43.419348+0000","last_active":"2023-10-09T09:36:44.435295+0000","last_peered":"2023-10-09T09:36:44.435295+0000","last_clean":"2023-10-09T09:36:44.435295+0000","last_became_active":"2023-10-09T09:36:43.418525+0000","last_became_peered":"2023-10-09T09:36:43.418525+0000","last_unstale":"2023-10-09T09:36:44.435295+0000","last_undegraded":"2023-10-09T09:36:44.435295+0000","last_fullsized":"2023-10-09T09:36:44.435295+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"1152'6957","reported_seq":6991,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:37:21.580993+0000","last_change":"2023-10-09T09:35:18.685439+0000","last_active":"2023-10-09T09:37:21.580993+0000","last_peered":"2023-10-09T09:37:21.580993+0000","last_clean":"2023-10-09T09:37:21.580993+0000","last_became_active":"2023-10-09T09:35:18.685078+0000","last_became_peered":"2023-10-09T09:35:18.685078+0000","last_unstale":"2023-10-09T09:37:21.580993+0000","last_undegraded":"2023-10-09T09:37:21.580993+0000","last_fullsized":"2023-10-09T09:37:21.580993+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":6957,"log_dups_size":0,"ondisk_log_size":6957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318046208,"num_objects":4853,"num_object_clones":0,"num_object_copies":14559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6957,"num_write_kb":377920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1121,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.434743+0000","last_change":"2023-10-09T09:36:43.418845+0000","last_active":"2023-10-09T09:36:44.434743+0000","last_peered":"2023-10-09T09:36:44.434743+0000","last_clean":"2023-10-09T09:36:44.434743+0000","last_became_active":"2023-10-09T09:36:43.418053+0000","last_became_peered":"2023-10-09T09:36:43.418053+0000","last_unstale":"2023-10-09T09:36:44.434743+0000","last_undegraded":"2023-10-09T09:36:44.434743+0000","last_fullsized":"2023-10-09T09:36:44.434743+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1102,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.435807+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:36:44.435807+0000","last_peered":"2023-10-09T09:36:44.435807+0000","last_clean":"2023-10-09T09:36:44.435807+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:36:44.435807+0000","last_undegraded":"2023-10-09T09:36:44.435807+0000","last_fullsized":"2023-10-09T09:36:44.435807+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"1152'7282","reported_seq":7312,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:37:24.147348+0000","last_change":"2023-10-09T09:35:11.918627+0000","last_active":"2023-10-09T09:37:24.147348+0000","last_peered":"2023-10-09T09:37:24.147348+0000","last_clean":"2023-10-09T09:37:24.147348+0000","last_became_active":"2023-10-09T09:35:11.918265+0000","last_became_peered":"2023-10-09T09:35:11.918265+0000","last_unstale":"2023-10-09T09:37:24.147348+0000","last_undegraded":"2023-10-09T09:37:24.147348+0000","last_fullsized":"2023-10-09T09:37:24.147348+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7282,"log_dups_size":0,"ondisk_log_size":7282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:48.237413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317587456,"num_objects":4846,"num_object_clones":0,"num_object_copies":14538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7282,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1055,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:36:53.703504+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:36:53.703504+0000","last_peered":"2023-10-09T09:36:53.703504+0000","last_clean":"2023-10-09T09:36:53.703504+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:36:53.703504+0000","last_undegraded":"2023-10-09T09:36:53.703504+0000","last_fullsized":"2023-10-09T09:36:53.703504+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1092,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:36:53.722832+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:36:53.722832+0000","last_peered":"2023-10-09T09:36:53.722832+0000","last_clean":"2023-10-09T09:36:53.722832+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:36:53.722832+0000","last_undegraded":"2023-10-09T09:36:53.722832+0000","last_fullsized":"2023-10-09T09:36:53.722832+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"1152'7133","reported_seq":12128,"reported_epoch":1152,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:37:21.932889+0000","last_change":"2023-10-09T09:36:44.841431+0000","last_active":"2023-10-09T09:37:21.932889+0000","last_peered":"2023-10-09T09:37:21.932889+0000","last_clean":"2023-10-09T09:36:42.389221+0000","last_became_active":"2023-10-09T09:36:44.832255+0000","last_became_peered":"2023-10-09T09:36:44.832255+0000","last_unstale":"2023-10-09T09:37:21.932889+0000","last_undegraded":"2023-10-09T09:37:21.932889+0000","last_fullsized":"2023-10-09T09:37:21.932889+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7133,"log_dups_size":0,"ondisk_log_size":7133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:51.849587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328138752,"num_objects":5007,"num_object_clones":0,"num_object_copies":15021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5007,"num_objects_unfound":0,"num_objects_dirty":5007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7133,"num_write_kb":388480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2471,"num_bytes_recovered":161939456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":5007}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1102,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419841+0000","last_change":"2023-10-09T09:35:18.664940+0000","last_active":"2023-10-09T09:36:44.419841+0000","last_peered":"2023-10-09T09:36:44.419841+0000","last_clean":"2023-10-09T09:36:44.419841+0000","last_became_active":"2023-10-09T09:35:18.664570+0000","last_became_peered":"2023-10-09T09:35:18.664570+0000","last_unstale":"2023-10-09T09:36:44.419841+0000","last_undegraded":"2023-10-09T09:36:44.419841+0000","last_fullsized":"2023-10-09T09:36:44.419841+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:33.263100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1137,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.424761+0000","last_change":"2023-10-09T09:36:43.433097+0000","last_active":"2023-10-09T09:36:44.424761+0000","last_peered":"2023-10-09T09:36:44.424761+0000","last_clean":"2023-10-09T09:36:44.424761+0000","last_became_active":"2023-10-09T09:36:43.432738+0000","last_became_peered":"2023-10-09T09:36:43.432738+0000","last_unstale":"2023-10-09T09:36:44.424761+0000","last_undegraded":"2023-10-09T09:36:44.424761+0000","last_fullsized":"2023-10-09T09:36:44.424761+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1077,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.422086+0000","last_change":"2023-10-09T09:14:40.491193+0000","last_active":"2023-10-09T09:36:44.422086+0000","last_peered":"2023-10-09T09:36:44.422086+0000","last_clean":"2023-10-09T09:36:44.422086+0000","last_became_active":"2023-10-09T09:14:40.490777+0000","last_became_peered":"2023-10-09T09:14:40.490777+0000","last_unstale":"2023-10-09T09:36:44.422086+0000","last_undegraded":"2023-10-09T09:36:44.422086+0000","last_fullsized":"2023-10-09T09:36:44.422086+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:18.132080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"1152'6982","reported_seq":7010,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:37:21.572298+0000","last_change":"2023-10-09T09:35:12.109048+0000","last_active":"2023-10-09T09:37:21.572298+0000","last_peered":"2023-10-09T09:37:21.572298+0000","last_clean":"2023-10-09T09:37:21.572298+0000","last_became_active":"2023-10-09T09:35:12.108164+0000","last_became_peered":"2023-10-09T09:35:12.108164+0000","last_unstale":"2023-10-09T09:37:21.572298+0000","last_undegraded":"2023-10-09T09:37:21.572298+0000","last_fullsized":"2023-10-09T09:37:21.572298+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320864256,"num_objects":4896,"num_object_clones":0,"num_object_copies":14688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6982,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1108,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419604+0000","last_change":"2023-10-09T09:14:40.485566+0000","last_active":"2023-10-09T09:36:44.419604+0000","last_peered":"2023-10-09T09:36:44.419604+0000","last_clean":"2023-10-09T09:36:44.419604+0000","last_became_active":"2023-10-09T09:14:40.485155+0000","last_became_peered":"2023-10-09T09:14:40.485155+0000","last_unstale":"2023-10-09T09:36:44.419604+0000","last_undegraded":"2023-10-09T09:36:44.419604+0000","last_fullsized":"2023-10-09T09:36:44.419604+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1118,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.435552+0000","last_change":"2023-10-09T09:35:18.685554+0000","last_active":"2023-10-09T09:36:44.435552+0000","last_peered":"2023-10-09T09:36:44.435552+0000","last_clean":"2023-10-09T09:36:44.435552+0000","last_became_active":"2023-10-09T09:35:18.685119+0000","last_became_peered":"2023-10-09T09:35:18.685119+0000","last_unstale":"2023-10-09T09:36:44.435552+0000","last_undegraded":"2023-10-09T09:36:44.435552+0000","last_fullsized":"2023-10-09T09:36:44.435552+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1125,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418014+0000","last_change":"2023-10-09T09:36:43.406338+0000","last_active":"2023-10-09T09:36:44.418014+0000","last_peered":"2023-10-09T09:36:44.418014+0000","last_clean":"2023-10-09T09:36:44.418014+0000","last_became_active":"2023-10-09T09:36:43.405976+0000","last_became_peered":"2023-10-09T09:36:43.405976+0000","last_unstale":"2023-10-09T09:36:44.418014+0000","last_undegraded":"2023-10-09T09:36:44.418014+0000","last_fullsized":"2023-10-09T09:36:44.418014+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:22.618067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"1152'7051","reported_seq":7079,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:37:21.511323+0000","last_change":"2023-10-09T09:35:12.106534+0000","last_active":"2023-10-09T09:37:21.511323+0000","last_peered":"2023-10-09T09:37:21.511323+0000","last_clean":"2023-10-09T09:37:21.511323+0000","last_became_active":"2023-10-09T09:35:12.105788+0000","last_became_peered":"2023-10-09T09:35:12.105788+0000","last_unstale":"2023-10-09T09:37:21.511323+0000","last_undegraded":"2023-10-09T09:37:21.511323+0000","last_fullsized":"2023-10-09T09:37:21.511323+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7051,"log_dups_size":0,"ondisk_log_size":7051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318046208,"num_objects":4853,"num_object_clones":0,"num_object_copies":14559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7051,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":13,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:45.466075+0000","last_change":"2023-10-09T09:36:43.412227+0000","last_active":"2023-10-09T09:36:45.466075+0000","last_peered":"2023-10-09T09:36:45.466075+0000","last_clean":"2023-10-09T09:36:45.466075+0000","last_became_active":"2023-10-09T09:36:43.411884+0000","last_became_peered":"2023-10-09T09:36:43.411884+0000","last_unstale":"2023-10-09T09:36:45.466075+0000","last_undegraded":"2023-10-09T09:36:45.466075+0000","last_fullsized":"2023-10-09T09:36:45.466075+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:09.042244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1111,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418354+0000","last_change":"2023-10-09T09:36:43.421868+0000","last_active":"2023-10-09T09:36:44.418354+0000","last_peered":"2023-10-09T09:36:44.418354+0000","last_clean":"2023-10-09T09:36:44.418354+0000","last_became_active":"2023-10-09T09:36:43.421013+0000","last_became_peered":"2023-10-09T09:36:43.421013+0000","last_unstale":"2023-10-09T09:36:44.418354+0000","last_undegraded":"2023-10-09T09:36:44.418354+0000","last_fullsized":"2023-10-09T09:36:44.418354+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:43.353667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1097,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.421811+0000","last_change":"2023-10-09T09:35:18.673011+0000","last_active":"2023-10-09T09:36:44.421811+0000","last_peered":"2023-10-09T09:36:44.421811+0000","last_clean":"2023-10-09T09:36:44.421811+0000","last_became_active":"2023-10-09T09:35:18.672567+0000","last_became_peered":"2023-10-09T09:35:18.672567+0000","last_unstale":"2023-10-09T09:36:44.421811+0000","last_undegraded":"2023-10-09T09:36:44.421811+0000","last_fullsized":"2023-10-09T09:36:44.421811+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:23.602764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"1152'7012","reported_seq":7046,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:37:21.736222+0000","last_change":"2023-10-09T09:35:18.676647+0000","last_active":"2023-10-09T09:37:21.736222+0000","last_peered":"2023-10-09T09:37:21.736222+0000","last_clean":"2023-10-09T09:37:21.736222+0000","last_became_active":"2023-10-09T09:35:18.676257+0000","last_became_peered":"2023-10-09T09:35:18.676257+0000","last_unstale":"2023-10-09T09:37:21.736222+0000","last_undegraded":"2023-10-09T09:37:21.736222+0000","last_fullsized":"2023-10-09T09:37:21.736222+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7012,"log_dups_size":0,"ondisk_log_size":7012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321126400,"num_objects":4900,"num_object_clones":0,"num_object_copies":14700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7012,"num_write_kb":381184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1126,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419235+0000","last_change":"2023-10-09T09:36:43.421730+0000","last_active":"2023-10-09T09:36:44.419235+0000","last_peered":"2023-10-09T09:36:44.419235+0000","last_clean":"2023-10-09T09:36:44.419235+0000","last_became_active":"2023-10-09T09:36:43.421343+0000","last_became_peered":"2023-10-09T09:36:43.421343+0000","last_unstale":"2023-10-09T09:36:44.419235+0000","last_undegraded":"2023-10-09T09:36:44.419235+0000","last_fullsized":"2023-10-09T09:36:44.419235+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1331,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.417649+0000","last_change":"2023-10-09T09:35:18.660120+0000","last_active":"2023-10-09T09:36:44.417649+0000","last_peered":"2023-10-09T09:36:44.417649+0000","last_clean":"2023-10-09T09:36:44.417649+0000","last_became_active":"2023-10-09T09:35:18.659460+0000","last_became_peered":"2023-10-09T09:35:18.659460+0000","last_unstale":"2023-10-09T09:36:44.417649+0000","last_undegraded":"2023-10-09T09:36:44.417649+0000","last_fullsized":"2023-10-09T09:36:44.417649+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":613,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419725+0000","last_change":"2023-10-09T09:34:03.850227+0000","last_active":"2023-10-09T09:36:44.419725+0000","last_peered":"2023-10-09T09:36:44.419725+0000","last_clean":"2023-10-09T09:36:44.419725+0000","last_became_active":"2023-10-09T09:34:03.849667+0000","last_became_peered":"2023-10-09T09:34:03.849667+0000","last_unstale":"2023-10-09T09:36:44.419725+0000","last_undegraded":"2023-10-09T09:36:44.419725+0000","last_fullsized":"2023-10-09T09:36:44.419725+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1069,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.839398+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:36:44.839398+0000","last_peered":"2023-10-09T09:36:44.839398+0000","last_clean":"2023-10-09T09:36:44.839398+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:36:44.839398+0000","last_undegraded":"2023-10-09T09:36:44.839398+0000","last_fullsized":"2023-10-09T09:36:44.839398+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1152'6951","reported_seq":11651,"reported_epoch":1152,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:37:24.148909+0000","last_change":"2023-10-09T09:36:44.832844+0000","last_active":"2023-10-09T09:37:24.148909+0000","last_peered":"2023-10-09T09:37:24.148909+0000","last_clean":"2023-10-09T09:36:42.376092+0000","last_became_active":"2023-10-09T09:36:44.828215+0000","last_became_peered":"2023-10-09T09:36:44.828215+0000","last_unstale":"2023-10-09T09:37:24.148909+0000","last_undegraded":"2023-10-09T09:37:24.148909+0000","last_fullsized":"2023-10-09T09:37:24.148909+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":6951,"log_dups_size":0,"ondisk_log_size":6951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:41.085815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306249728,"num_objects":4673,"num_object_clones":0,"num_object_copies":14019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4673,"num_objects_unfound":0,"num_objects_dirty":4673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6951,"num_write_kb":371968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2322,"num_bytes_recovered":152174592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,5,3],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":4673}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1114,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.417760+0000","last_change":"2023-10-09T09:35:18.659469+0000","last_active":"2023-10-09T09:36:44.417760+0000","last_peered":"2023-10-09T09:36:44.417760+0000","last_clean":"2023-10-09T09:36:44.417760+0000","last_became_active":"2023-10-09T09:35:18.659007+0000","last_became_peered":"2023-10-09T09:35:18.659007+0000","last_unstale":"2023-10-09T09:36:44.417760+0000","last_undegraded":"2023-10-09T09:36:44.417760+0000","last_fullsized":"2023-10-09T09:36:44.417760+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1074,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:36:53.724420+0000","last_change":"2023-10-09T09:35:18.684437+0000","last_active":"2023-10-09T09:36:53.724420+0000","last_peered":"2023-10-09T09:36:53.724420+0000","last_clean":"2023-10-09T09:36:53.724420+0000","last_became_active":"2023-10-09T09:35:18.678845+0000","last_became_peered":"2023-10-09T09:35:18.678845+0000","last_unstale":"2023-10-09T09:36:53.724420+0000","last_undegraded":"2023-10-09T09:36:53.724420+0000","last_fullsized":"2023-10-09T09:36:53.724420+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:44:13.569461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1070,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419272+0000","last_change":"2023-10-09T09:36:43.414085+0000","last_active":"2023-10-09T09:36:44.419272+0000","last_peered":"2023-10-09T09:36:44.419272+0000","last_clean":"2023-10-09T09:36:44.419272+0000","last_became_active":"2023-10-09T09:36:43.410306+0000","last_became_peered":"2023-10-09T09:36:43.410306+0000","last_unstale":"2023-10-09T09:36:44.419272+0000","last_undegraded":"2023-10-09T09:36:44.419272+0000","last_fullsized":"2023-10-09T09:36:44.419272+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:09.154474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"1152'7094","reported_seq":10396,"reported_epoch":1152,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T09:37:24.159005+0000","last_change":"2023-10-09T09:36:43.583905+0000","last_active":"2023-10-09T09:37:24.159005+0000","last_peered":"2023-10-09T09:37:24.159005+0000","last_clean":"2023-10-09T09:35:47.669225+0000","last_became_active":"2023-10-09T09:36:43.559820+0000","last_became_peered":"2023-10-09T09:36:43.559820+0000","last_unstale":"2023-10-09T09:37:24.159005+0000","last_undegraded":"2023-10-09T09:37:24.159005+0000","last_fullsized":"2023-10-09T09:36:43.400511+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7094,"log_dups_size":0,"ondisk_log_size":7094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:52:21.921017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314441728,"num_objects":4798,"num_object_clones":0,"num_object_copies":14394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1861,"num_objects_unfound":0,"num_objects_dirty":4798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7094,"num_write_kb":380544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1630,"num_bytes_recovered":106823680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":2937},{"shards":"1,3,4","objects":1861}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1099,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419102+0000","last_change":"2023-10-09T09:35:18.685388+0000","last_active":"2023-10-09T09:36:44.419102+0000","last_peered":"2023-10-09T09:36:44.419102+0000","last_clean":"2023-10-09T09:36:44.419102+0000","last_became_active":"2023-10-09T09:35:18.684983+0000","last_became_peered":"2023-10-09T09:35:18.684983+0000","last_unstale":"2023-10-09T09:36:44.419102+0000","last_undegraded":"2023-10-09T09:36:44.419102+0000","last_fullsized":"2023-10-09T09:36:44.419102+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1111,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419618+0000","last_change":"2023-10-09T09:36:43.443586+0000","last_active":"2023-10-09T09:36:44.419618+0000","last_peered":"2023-10-09T09:36:44.419618+0000","last_clean":"2023-10-09T09:36:44.419618+0000","last_became_active":"2023-10-09T09:36:43.443235+0000","last_became_peered":"2023-10-09T09:36:43.443235+0000","last_unstale":"2023-10-09T09:36:44.419618+0000","last_undegraded":"2023-10-09T09:36:44.419618+0000","last_fullsized":"2023-10-09T09:36:44.419618+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:38:52.676452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1058,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:36:53.723914+0000","last_change":"2023-10-09T09:35:18.662791+0000","last_active":"2023-10-09T09:36:53.723914+0000","last_peered":"2023-10-09T09:36:53.723914+0000","last_clean":"2023-10-09T09:36:53.723914+0000","last_became_active":"2023-10-09T09:35:18.662390+0000","last_became_peered":"2023-10-09T09:35:18.662390+0000","last_unstale":"2023-10-09T09:36:53.723914+0000","last_undegraded":"2023-10-09T09:36:53.723914+0000","last_fullsized":"2023-10-09T09:36:53.723914+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:38.393518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"1152'7017","reported_seq":7043,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:37:21.893968+0000","last_change":"2023-10-09T09:35:18.673315+0000","last_active":"2023-10-09T09:37:21.893968+0000","last_peered":"2023-10-09T09:37:21.893968+0000","last_clean":"2023-10-09T09:37:21.893968+0000","last_became_active":"2023-10-09T09:35:18.672993+0000","last_became_peered":"2023-10-09T09:35:18.672993+0000","last_unstale":"2023-10-09T09:37:21.893968+0000","last_undegraded":"2023-10-09T09:37:21.893968+0000","last_fullsized":"2023-10-09T09:37:21.893968+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7017,"log_dups_size":0,"ondisk_log_size":7017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:19.360755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320012288,"num_objects":4883,"num_object_clones":0,"num_object_copies":14649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7017,"num_write_kb":380800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":74,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.418645+0000","last_change":"2023-10-09T09:35:18.676960+0000","last_active":"2023-10-09T09:36:44.418645+0000","last_peered":"2023-10-09T09:36:44.418645+0000","last_clean":"2023-10-09T09:36:44.418645+0000","last_became_active":"2023-10-09T09:35:18.676357+0000","last_became_peered":"2023-10-09T09:35:18.676357+0000","last_unstale":"2023-10-09T09:36:44.418645+0000","last_undegraded":"2023-10-09T09:36:44.418645+0000","last_fullsized":"2023-10-09T09:36:44.418645+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1076,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.420138+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:36:44.420138+0000","last_peered":"2023-10-09T09:36:44.420138+0000","last_clean":"2023-10-09T09:36:44.420138+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:36:44.420138+0000","last_undegraded":"2023-10-09T09:36:44.420138+0000","last_fullsized":"2023-10-09T09:36:44.420138+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1081,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.417859+0000","last_change":"2023-10-09T09:35:18.663646+0000","last_active":"2023-10-09T09:36:44.417859+0000","last_peered":"2023-10-09T09:36:44.417859+0000","last_clean":"2023-10-09T09:36:44.417859+0000","last_became_active":"2023-10-09T09:35:18.663357+0000","last_became_peered":"2023-10-09T09:35:18.663357+0000","last_unstale":"2023-10-09T09:36:44.417859+0000","last_undegraded":"2023-10-09T09:36:44.417859+0000","last_fullsized":"2023-10-09T09:36:44.417859+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"1152'6875","reported_seq":7415,"reported_epoch":1152,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:37:21.577719+0000","last_change":"2023-10-09T09:36:43.591295+0000","last_active":"2023-10-09T09:37:21.577719+0000","last_peered":"2023-10-09T09:37:21.577719+0000","last_clean":"2023-10-09T09:35:47.668825+0000","last_became_active":"2023-10-09T09:36:43.568479+0000","last_became_peered":"2023-10-09T09:36:43.568479+0000","last_unstale":"2023-10-09T09:37:21.577719+0000","last_undegraded":"2023-10-09T09:36:43.430806+0000","last_fullsized":"2023-10-09T09:36:43.404257+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":6875,"log_dups_size":0,"ondisk_log_size":6875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318111768,"num_objects":4855,"num_object_clones":0,"num_object_copies":14565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4855,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6875,"num_write_kb":375297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":247,"num_bytes_recovered":16187392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,6,7","objects":2965},{"shards":"5,7","objects":1890}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1106,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.839584+0000","last_change":"2023-10-09T09:36:43.423809+0000","last_active":"2023-10-09T09:36:44.839584+0000","last_peered":"2023-10-09T09:36:44.839584+0000","last_clean":"2023-10-09T09:36:44.839584+0000","last_became_active":"2023-10-09T09:36:43.423506+0000","last_became_peered":"2023-10-09T09:36:43.423506+0000","last_unstale":"2023-10-09T09:36:44.839584+0000","last_undegraded":"2023-10-09T09:36:44.839584+0000","last_fullsized":"2023-10-09T09:36:44.839584+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1060,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.419486+0000","last_change":"2023-10-09T09:35:18.685030+0000","last_active":"2023-10-09T09:36:44.419486+0000","last_peered":"2023-10-09T09:36:44.419486+0000","last_clean":"2023-10-09T09:36:44.419486+0000","last_became_active":"2023-10-09T09:35:18.675810+0000","last_became_peered":"2023-10-09T09:35:18.675810+0000","last_unstale":"2023-10-09T09:36:44.419486+0000","last_undegraded":"2023-10-09T09:36:44.419486+0000","last_fullsized":"2023-10-09T09:36:44.419486+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:52.793405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1087,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.434848+0000","last_change":"2023-10-09T09:03:25.312339+0000","last_active":"2023-10-09T09:36:44.434848+0000","last_peered":"2023-10-09T09:36:44.434848+0000","last_clean":"2023-10-09T09:36:44.434848+0000","last_became_active":"2023-10-09T09:03:25.312038+0000","last_became_peered":"2023-10-09T09:03:25.312038+0000","last_unstale":"2023-10-09T09:36:44.434848+0000","last_undegraded":"2023-10-09T09:36:44.434848+0000","last_fullsized":"2023-10-09T09:36:44.434848+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1060,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.417865+0000","last_change":"2023-10-09T09:35:18.680525+0000","last_active":"2023-10-09T09:36:44.417865+0000","last_peered":"2023-10-09T09:36:44.417865+0000","last_clean":"2023-10-09T09:36:44.417865+0000","last_became_active":"2023-10-09T09:35:18.675584+0000","last_became_peered":"2023-10-09T09:35:18.675584+0000","last_unstale":"2023-10-09T09:36:44.417865+0000","last_undegraded":"2023-10-09T09:36:44.417865+0000","last_fullsized":"2023-10-09T09:36:44.417865+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:09.246045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:45.466092+0000","last_change":"2023-10-09T09:36:43.406032+0000","last_active":"2023-10-09T09:36:45.466092+0000","last_peered":"2023-10-09T09:36:45.466092+0000","last_clean":"2023-10-09T09:36:45.466092+0000","last_became_active":"2023-10-09T09:36:43.405092+0000","last_became_peered":"2023-10-09T09:36:43.405092+0000","last_unstale":"2023-10-09T09:36:45.466092+0000","last_undegraded":"2023-10-09T09:36:45.466092+0000","last_fullsized":"2023-10-09T09:36:45.466092+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:20.087534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":1106,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.420093+0000","last_change":"2023-10-09T09:14:40.485527+0000","last_active":"2023-10-09T09:36:44.420093+0000","last_peered":"2023-10-09T09:36:44.420093+0000","last_clean":"2023-10-09T09:36:44.420093+0000","last_became_active":"2023-10-09T09:14:40.485111+0000","last_became_peered":"2023-10-09T09:14:40.485111+0000","last_unstale":"2023-10-09T09:36:44.420093+0000","last_undegraded":"2023-10-09T09:36:44.420093+0000","last_fullsized":"2023-10-09T09:36:44.420093+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1117,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.435895+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:36:44.435895+0000","last_peered":"2023-10-09T09:36:44.435895+0000","last_clean":"2023-10-09T09:36:44.435895+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:36:44.435895+0000","last_undegraded":"2023-10-09T09:36:44.435895+0000","last_fullsized":"2023-10-09T09:36:44.435895+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":1091,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.424831+0000","last_change":"2023-10-09T09:03:24.300396+0000","last_active":"2023-10-09T09:36:44.424831+0000","last_peered":"2023-10-09T09:36:44.424831+0000","last_clean":"2023-10-09T09:36:44.424831+0000","last_became_active":"2023-10-09T09:03:24.300012+0000","last_became_peered":"2023-10-09T09:03:24.300012+0000","last_unstale":"2023-10-09T09:36:44.424831+0000","last_undegraded":"2023-10-09T09:36:44.424831+0000","last_fullsized":"2023-10-09T09:36:44.424831+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1132,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.435040+0000","last_change":"2023-10-09T09:35:18.686704+0000","last_active":"2023-10-09T09:36:44.435040+0000","last_peered":"2023-10-09T09:36:44.435040+0000","last_clean":"2023-10-09T09:36:44.435040+0000","last_became_active":"2023-10-09T09:35:18.686421+0000","last_became_peered":"2023-10-09T09:35:18.686421+0000","last_unstale":"2023-10-09T09:36:44.435040+0000","last_undegraded":"2023-10-09T09:36:44.435040+0000","last_fullsized":"2023-10-09T09:36:44.435040+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":76,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.839583+0000","last_change":"2023-10-09T09:36:43.424942+0000","last_active":"2023-10-09T09:36:44.839583+0000","last_peered":"2023-10-09T09:36:44.839583+0000","last_clean":"2023-10-09T09:36:44.839583+0000","last_became_active":"2023-10-09T09:36:43.424555+0000","last_became_peered":"2023-10-09T09:36:43.424555+0000","last_unstale":"2023-10-09T09:36:44.839583+0000","last_undegraded":"2023-10-09T09:36:44.839583+0000","last_fullsized":"2023-10-09T09:36:44.839583+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:45.466053+0000","last_change":"2023-10-09T09:36:43.412400+0000","last_active":"2023-10-09T09:36:45.466053+0000","last_peered":"2023-10-09T09:36:45.466053+0000","last_clean":"2023-10-09T09:36:45.466053+0000","last_became_active":"2023-10-09T09:36:43.412039+0000","last_became_peered":"2023-10-09T09:36:43.412039+0000","last_unstale":"2023-10-09T09:36:45.466053+0000","last_undegraded":"2023-10-09T09:36:45.466053+0000","last_fullsized":"2023-10-09T09:36:45.466053+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:40:22.872096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1079,"reported_epoch":1152,"state":"active+clean","last_fresh":"2023-10-09T09:36:53.724316+0000","last_change":"2023-10-09T09:36:43.416743+0000","last_active":"2023-10-09T09:36:53.724316+0000","last_peered":"2023-10-09T09:36:53.724316+0000","last_clean":"2023-10-09T09:36:53.724316+0000","last_became_active":"2023-10-09T09:36:43.408799+0000","last_became_peered":"2023-10-09T09:36:43.408799+0000","last_unstale":"2023-10-09T09:36:53.724316+0000","last_undegraded":"2023-10-09T09:36:53.724316+0000","last_fullsized":"2023-10-09T09:36:53.724316+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:12.436053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1102,"reported_epoch":1151,"state":"active+clean","last_fresh":"2023-10-09T09:36:44.434633+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:36:44.434633+0000","last_peered":"2023-10-09T09:36:44.434633+0000","last_clean":"2023-10-09T09:36:44.434633+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:36:44.434633+0000","last_undegraded":"2023-10-09T09:36:44.434633+0000","last_fullsized":"2023-10-09T09:36:44.434633+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":1176,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5093654552,"num_objects":77724,"num_object_clones":0,"num_object_copies":233172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8935,"num_objects_misplaced":11541,"num_objects_unfound":0,"num_objects_dirty":77724,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112738,"num_write_kb":6094721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6670,"num_bytes_recovered":437125120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15652433920,"data_stored":15652421704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112738,"ondisk_log_size":112738,"up":48,"acting":44,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397947,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2194008,"kb_used_data":1892560,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91551144,"statfs":{"total":95995035648,"available":93748371456,"internally_reserved":0,"allocated":1937981440,"data_stored":1937737046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.1660000000000004,"5min":7.1660000000000004,"15min":7.1660000000000004},"last":3.9510000000000001},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.3860000000000001,"5min":5.3860000000000001,"15min":5.3860000000000001},"last":3.7080000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:36:32 2023","interfaces":[{"interface":"back","average":{"1min":2.8610000000000002,"5min":2.0990000000000002,"15min":1.923},"min":{"1min":0.30299999999999999,"5min":0.29299999999999998,"15min":0.26300000000000001},"max":{"1min":16.785,"5min":58.911000000000001,"15min":103.84699999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":2.6949999999999998,"5min":2.056,"15min":1.905},"min":{"1min":0.314,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":17.062999999999999,"5min":50.378,"15min":103.955},"last":5.0730000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:36:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":1.75,"15min":1.861},"min":{"1min":0.34799999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.5679999999999996,"5min":50.756,"15min":50.756},"last":1.875},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.8260000000000001,"15min":1.8080000000000001},"min":{"1min":0.45400000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.4699999999999998,"5min":50.82,"15min":50.82},"last":1.796}]},{"osd":3,"last update":"Mon Oct 9 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":2.5659999999999998,"15min":2.798},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.29999999999999999},"max":{"1min":3.4870000000000001,"5min":84.590999999999994,"15min":107.97499999999999},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.1639999999999999,"5min":2.6200000000000001,"15min":2.8700000000000001},"min":{"1min":0.38400000000000001,"5min":0.377,"15min":0.317},"max":{"1min":3.4399999999999999,"5min":84.713999999999999,"15min":105.282},"last":0.76700000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:36:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6949999999999998,"5min":1.6160000000000001,"15min":1.329},"min":{"1min":0.34399999999999997,"5min":0.29499999999999998,"15min":0.26500000000000001},"max":{"1min":8.3789999999999996,"5min":8.3789999999999996,"15min":9.8300000000000001},"last":3.9129999999999998},{"interface":"front","average":{"1min":1.335,"5min":1.1080000000000001,"15min":1.1060000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.4119999999999999,"5min":6.6210000000000004,"15min":12.316000000000001},"last":1.0649999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:36:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":1.4710000000000001,"15min":1.3919999999999999},"min":{"1min":0.372,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.4250000000000007,"5min":9.4250000000000007,"15min":10.063000000000001},"last":0.80000000000000004},{"interface":"front","average":{"1min":3.5390000000000001,"5min":1.7969999999999999,"15min":1.3839999999999999},"min":{"1min":0.36899999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.038,"5min":10.038,"15min":10.038},"last":1.8149999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:36:54 2023","interfaces":[{"interface":"back","average":{"1min":2.7909999999999999,"5min":1.667,"15min":1.3740000000000001},"min":{"1min":0.41499999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.4079999999999995,"5min":8.4079999999999995,"15min":8.4079999999999995},"last":1.3919999999999999},{"interface":"front","average":{"1min":2.0489999999999999,"5min":1.4379999999999999,"15min":1.371},"min":{"1min":0.55600000000000005,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.3559999999999999,"5min":6.4699999999999998,"15min":8.1980000000000004},"last":0.69599999999999995}]}]},{"osd":6,"up_from":43,"seq":184683594180,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2272456,"kb_used_data":1975232,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91472696,"statfs":{"total":95995035648,"available":93668040704,"internally_reserved":0,"allocated":2022637568,"data_stored":2022409388,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:37:20 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.2250000000000001,"15min":1.177},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.7910000000000004,"5min":6.7910000000000004,"15min":6.7910000000000004},"last":4.3630000000000004},{"interface":"front","average":{"1min":1.589,"5min":1.306,"15min":1.2589999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.4020000000000001,"5min":6.4020000000000001,"15min":6.4020000000000001},"last":4.29}]},{"osd":1,"last update":"Mon Oct 9 09:37:20 2023","interfaces":[{"interface":"back","average":{"1min":4.2670000000000003,"5min":3.7490000000000001,"15min":2.3079999999999998},"min":{"1min":0.39100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":38.709000000000003,"5min":103.259,"15min":103.259},"last":4.71},{"interface":"front","average":{"1min":3.5739999999999998,"5min":3.2559999999999998,"15min":2.0259999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":35.820999999999998,"5min":102.005,"15min":102.005},"last":4.085}]},{"osd":2,"last update":"Mon Oct 9 09:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.6890000000000001,"15min":1.5269999999999999},"min":{"1min":0.35399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.0629999999999997,"5min":33.210000000000001,"15min":33.210000000000001},"last":3.9060000000000001},{"interface":"front","average":{"1min":1.413,"5min":1.758,"15min":1.575},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.343,"5min":37.246000000000002,"15min":37.246000000000002},"last":6.3159999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:37:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.5660000000000001,"15min":2.2189999999999999},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.859,"5min":10.497,"15min":123.97199999999999},"last":0.78800000000000003},{"interface":"front","average":{"1min":1.494,"5min":1.1699999999999999,"15min":2.1480000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.0140000000000002,"5min":5.8630000000000004,"15min":124.059},"last":0.55500000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:37:08 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.101,"15min":1.0580000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.0659999999999998,"5min":8.6890000000000001,"15min":9.1099999999999994},"last":0.76500000000000001},{"interface":"front","average":{"1min":1.595,"5min":1.349,"15min":1.25},"min":{"1min":0.34599999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.4089999999999998,"5min":9.7919999999999998,"15min":9.7919999999999998},"last":0.59499999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:37:08 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.4199999999999999,"15min":1.2330000000000001},"min":{"1min":0.40600000000000003,"5min":0.38,"15min":0.379},"max":{"1min":7.4870000000000001,"5min":9.0559999999999992,"15min":9.0559999999999992},"last":0.68300000000000005},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.52,"15min":1.3440000000000001},"min":{"1min":0.36099999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.7889999999999997,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.81599999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:36:41 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.2450000000000001,"15min":1.347},"min":{"1min":0.40400000000000003,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.6589999999999998,"5min":7.6769999999999996,"15min":7.6769999999999996},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.879,"5min":1.413,"15min":1.3879999999999999},"min":{"1min":0.432,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.8840000000000003,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":1.042}]}]},{"osd":2,"up_from":634,"seq":2723009265933,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1202212,"kb_used_data":827868,"kb_used_omap":122,"kb_used_meta":374213,"kb_avail":92542940,"statfs":{"total":95995035648,"available":94763970560,"internally_reserved":0,"allocated":847736832,"data_stored":846496235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":383194699},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:36:18 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":2.617,"15min":2.617},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":17.927,"5min":17.927,"15min":17.927},"last":0.252},{"interface":"front","average":{"1min":3.032,"5min":3.032,"15min":3.032},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":21.431000000000001,"5min":21.431000000000001,"15min":21.431000000000001},"last":1.6379999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3410000000000002,"5min":2.7010000000000001,"15min":1.718},"min":{"1min":0.34399999999999997,"5min":0.20200000000000001,"15min":0.184},"max":{"1min":18.427,"5min":81.662999999999997,"15min":81.662999999999997},"last":3.3159999999999998},{"interface":"front","average":{"1min":2.6309999999999998,"5min":2.7360000000000002,"15min":1.9119999999999999},"min":{"1min":0.30399999999999999,"5min":0.222,"15min":0.216},"max":{"1min":19.815000000000001,"5min":81.712999999999994,"15min":81.712999999999994},"last":1.9610000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:37:12 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.6579999999999999,"15min":1.629},"min":{"1min":0.312,"5min":0.254,"15min":0.252},"max":{"1min":5.1840000000000002,"5min":9.6140000000000008,"15min":18.547999999999998},"last":3.0699999999999998},{"interface":"front","average":{"1min":0.78300000000000003,"5min":1.4299999999999999,"15min":1.659},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26200000000000001},"max":{"1min":3.2069999999999999,"5min":13.23,"15min":19.527999999999999},"last":1.843}]},{"osd":4,"last update":"Mon Oct 9 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":1.286,"15min":1.353},"min":{"1min":0.35799999999999998,"5min":0.34599999999999997,"15min":0.309},"max":{"1min":5.7130000000000001,"5min":8.0370000000000008,"15min":8.0370000000000008},"last":1.883},{"interface":"front","average":{"1min":1.712,"5min":1.4530000000000001,"15min":1.4610000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.5969999999999995,"5min":9.5969999999999995,"15min":13.917999999999999},"last":0.57199999999999995}]},{"osd":5,"last update":"Mon Oct 9 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.4530000000000001,"15min":1.292},"min":{"1min":0.44500000000000001,"5min":0.313,"15min":0.313},"max":{"1min":3.4399999999999999,"5min":9.7599999999999998,"15min":9.7599999999999998},"last":3.1419999999999999},{"interface":"front","average":{"1min":1.446,"5min":1.4910000000000001,"15min":1.4419999999999999},"min":{"1min":0.42199999999999999,"5min":0.37,"15min":0.36499999999999999},"max":{"1min":6.101,"5min":8.1080000000000005,"15min":10.393000000000001},"last":1.9299999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.681,"15min":1.5269999999999999},"min":{"1min":0.48699999999999999,"5min":0.34699999999999998,"15min":0.33100000000000002},"max":{"1min":4.6120000000000001,"5min":8.0809999999999995,"15min":8.0809999999999995},"last":1.994},{"interface":"front","average":{"1min":1.157,"5min":1.5640000000000001,"15min":1.6020000000000001},"min":{"1min":0.39000000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.8490000000000002,"5min":7.8259999999999996,"15min":13.893000000000001},"last":0.64700000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":1.649,"15min":1.704},"min":{"1min":0.55700000000000005,"5min":0.40400000000000003,"15min":0.377},"max":{"1min":4.9749999999999996,"5min":7.3849999999999998,"15min":13.848000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.252,"5min":1.7370000000000001,"15min":1.506},"min":{"1min":0.56200000000000006,"5min":0.40400000000000003,"15min":0.34499999999999997},"max":{"1min":4.923,"5min":9.5790000000000006,"15min":10.199},"last":3.1110000000000002}]}]},{"osd":1,"up_from":607,"seq":2607045148947,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2513744,"kb_used_data":2164744,"kb_used_omap":107,"kb_used_meta":348884,"kb_avail":91231408,"statfs":{"total":95995035648,"available":93420961792,"internally_reserved":0,"allocated":2216697856,"data_stored":2215474436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":357257899},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:37:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.3220000000000001,"15min":1.3740000000000001},"min":{"1min":0.33700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":3.7810000000000001,"5min":6.907,"15min":6.907},"last":2.077},{"interface":"front","average":{"1min":1.5660000000000001,"5min":2.3250000000000002,"15min":2.4510000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":5.6139999999999999,"5min":11.108000000000001,"15min":11.108000000000001},"last":1.0309999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:36:46 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":2.7160000000000002,"15min":2.2589999999999999},"min":{"1min":0.27500000000000002,"5min":0.224,"15min":0.20100000000000001},"max":{"1min":11.164,"5min":69.421999999999997,"15min":84.578999999999994},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.84599999999999997,"5min":2.0630000000000002,"15min":1.8879999999999999},"min":{"1min":0.27100000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":2.7919999999999998,"5min":71.879000000000005,"15min":84.697000000000003},"last":0.41299999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":1.47,"15min":2.8109999999999999},"min":{"1min":0.34799999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.8239999999999998,"5min":13.004,"15min":107.426},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.0470000000000002,"5min":2.1819999999999999,"15min":3.2410000000000001},"min":{"1min":0.372,"5min":0.27900000000000003,"15min":0.23200000000000001},"max":{"1min":6.0839999999999996,"5min":17.643999999999998,"15min":106.15900000000001},"last":0.47699999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:37:23 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.129,"15min":1.4470000000000001},"min":{"1min":0.36799999999999999,"5min":0.28299999999999997,"15min":0.26500000000000001},"max":{"1min":4.1820000000000004,"5min":7.2759999999999998,"15min":84.875},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.355,"5min":1.5269999999999999,"15min":1.635},"min":{"1min":0.373,"5min":0.33100000000000002,"15min":0.29499999999999998},"max":{"1min":5.3550000000000004,"5min":32.151000000000003,"15min":84.992999999999995},"last":0.54500000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:37:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.25,"15min":1.5129999999999999},"min":{"1min":0.34699999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.2130000000000001,"5min":7.2380000000000004,"15min":84.805000000000007},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.6970000000000001,"5min":1.3,"15min":1.651},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":5.9649999999999999,"5min":6.2859999999999996,"15min":84.951999999999998},"last":2.0430000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:37:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":1.877,"15min":1.907},"min":{"1min":0.66500000000000004,"5min":0.41599999999999998,"15min":0.373},"max":{"1min":7.3940000000000001,"5min":11.343999999999999,"15min":85.016000000000005},"last":1.006},{"interface":"front","average":{"1min":1.8440000000000001,"5min":1.4790000000000001,"15min":1.6359999999999999},"min":{"1min":0.51700000000000002,"5min":0.378,"15min":0.36799999999999999},"max":{"1min":6.593,"5min":7.3209999999999997,"15min":84.831999999999994},"last":0.70199999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:37:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":1.915,"15min":2.0990000000000002},"min":{"1min":0.40999999999999998,"5min":0.40899999999999997,"15min":0.33100000000000002},"max":{"1min":7.6180000000000003,"5min":11.382999999999999,"15min":84.888000000000005},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.8660000000000001,"15min":1.8049999999999999},"min":{"1min":0.56599999999999995,"5min":0.42799999999999999,"15min":0.36699999999999999},"max":{"1min":4.2770000000000001,"5min":32.078000000000003,"15min":84.742000000000004},"last":0.72899999999999998}]}]},{"osd":0,"up_from":1136,"seq":4879082848282,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2067248,"kb_used_data":1899816,"kb_used_omap":280,"kb_used_meta":167079,"kb_avail":91677904,"statfs":{"total":95995035648,"available":93878173696,"internally_reserved":0,"allocated":1945411584,"data_stored":1944212594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":171089306},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":2.452,"5min":2.452,"15min":2.452},"last":1.5920000000000001},{"interface":"front","average":{"1min":1.04,"5min":1.04,"15min":1.04},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":6.8410000000000002,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":0.36599999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.878,"15min":0.878},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.4260000000000002,"5min":2.4260000000000002,"15min":2.4260000000000002},"last":5.149},{"interface":"front","average":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.6159999999999997,"5min":5.6159999999999997,"15min":5.6159999999999997},"last":1.627}]},{"osd":3,"last update":"Mon Oct 9 09:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.6210000000000004,"5min":5.6210000000000004,"15min":5.6210000000000004},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.0059999999999998,"5min":3.0059999999999998,"15min":3.0059999999999998},"last":4.2640000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.4159999999999999,"5min":2.4159999999999999,"15min":2.4159999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.242,"5min":1.242,"15min":1.242},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":2.798,"5min":2.798,"15min":2.798},"last":0.749}]},{"osd":5,"last update":"Mon Oct 9 09:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.6190000000000002,"5min":3.6190000000000002,"15min":3.6190000000000002},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.619,"5min":1.619,"15min":1.619},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":7.7149999999999999,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":1.6499999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":3.778,"5min":3.778,"15min":3.778},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.8140000000000001,"5min":1.8140000000000001,"15min":1.8140000000000001},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":5.0789999999999997,"5min":5.0789999999999997,"15min":5.0789999999999997},"last":0.78300000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":1.179,"15min":1.179},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":3.6299999999999999,"5min":3.6299999999999999,"15min":3.6299999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.605,"5min":1.605,"15min":1.605},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":5.6529999999999996,"5min":5.6529999999999996,"15min":5.6529999999999996},"last":0.71299999999999997}]}]},{"osd":3,"up_from":663,"seq":2847563317512,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2631308,"kb_used_data":2133380,"kb_used_omap":55,"kb_used_meta":497864,"kb_avail":91113844,"statfs":{"total":95995035648,"available":93300576256,"internally_reserved":0,"allocated":2184581120,"data_stored":2183468649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":509813346},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":0.89100000000000001,"15min":0.875},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":4.2140000000000004,"5min":4.2649999999999997,"15min":4.2649999999999997},"last":0.185},{"interface":"front","average":{"1min":1.536,"5min":1.026,"15min":0.94099999999999995},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.657,"5min":5.657,"15min":5.657},"last":1.379}]},{"osd":1,"last update":"Mon Oct 9 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":2.1709999999999998,"15min":1.726},"min":{"1min":0.313,"5min":0.245,"15min":0.245},"max":{"1min":2.5110000000000001,"5min":91.159000000000006,"15min":91.159000000000006},"last":0.94899999999999995},{"interface":"front","average":{"1min":2.3210000000000002,"5min":2.669,"15min":1.9139999999999999},"min":{"1min":0.39900000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.8380000000000001,"5min":92.683000000000007,"15min":92.683000000000007},"last":2.0299999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.5750000000000002,"15min":2.008},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":2.4209999999999998,"5min":106.122,"15min":106.122},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.81999999999999995,"5min":2.2890000000000001,"15min":1.853},"min":{"1min":0.41799999999999998,"5min":0.25,"15min":0.223},"max":{"1min":1.833,"5min":104.102,"15min":104.102},"last":0.80800000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.321,"15min":1.395},"min":{"1min":0.32900000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.986,"5min":6.6840000000000002,"15min":12.629},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.252,"15min":1.3520000000000001},"min":{"1min":0.45300000000000001,"5min":0.307,"15min":0.27500000000000002},"max":{"1min":2.3799999999999999,"5min":7.9210000000000003,"15min":7.9210000000000003},"last":0.34200000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":1.4770000000000001,"15min":1.599},"min":{"1min":0.372,"5min":0.34100000000000003,"15min":0.315},"max":{"1min":2.726,"5min":7.8200000000000003,"15min":11.272},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.292,"5min":1.4850000000000001,"15min":1.526},"min":{"1min":0.47499999999999998,"5min":0.36099999999999999,"15min":0.34100000000000003},"max":{"1min":3.9319999999999999,"5min":7.048,"15min":11.388999999999999},"last":0.73699999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.4570000000000001,"15min":1.54},"min":{"1min":0.47099999999999997,"5min":0.35299999999999998,"15min":0.30099999999999999},"max":{"1min":4.0919999999999996,"5min":8.7040000000000006,"15min":8.7040000000000006},"last":0.373},{"interface":"front","average":{"1min":1.4930000000000001,"5min":1.462,"15min":1.5740000000000001},"min":{"1min":0.52700000000000002,"5min":0.42399999999999999,"15min":0.316},"max":{"1min":3.9820000000000002,"5min":6.7160000000000002,"15min":17.234999999999999},"last":0.78600000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:36:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4359999999999999,"5min":1.6639999999999999,"15min":1.653},"min":{"1min":0.49399999999999999,"5min":0.36599999999999999,"15min":0.34999999999999998},"max":{"1min":5.8849999999999998,"5min":10.029999999999999,"15min":10.029999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.147,"5min":1.516,"15min":1.5720000000000001},"min":{"1min":0.46100000000000002,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":5.7830000000000004,"5min":8.6210000000000004,"15min":8.6210000000000004},"last":0.40000000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986645,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2110412,"kb_used_data":1808964,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91634740,"statfs":{"total":95995035648,"available":93833973760,"internally_reserved":0,"allocated":1852379136,"data_stored":1852146392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:37:18 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":1.2649999999999999,"15min":1.0589999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":16.408999999999999,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":2.573,"5min":1.4370000000000001,"15min":1.248},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":16.387,"5min":16.387,"15min":16.387},"last":0.84099999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:36:30 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":2.6379999999999999,"15min":1.6719999999999999},"min":{"1min":0.315,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":16.449999999999999,"5min":85.186999999999998,"15min":85.186999999999998},"last":0.432},{"interface":"front","average":{"1min":2.2690000000000001,"5min":3.0059999999999998,"15min":1.7849999999999999},"min":{"1min":0.31,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":16.460000000000001,"5min":85.159000000000006,"15min":85.159000000000006},"last":0.308}]},{"osd":2,"last update":"Mon Oct 9 09:36:37 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.1830000000000001,"15min":1.5820000000000001},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.79,"5min":6.21,"15min":99.736000000000004},"last":0.625},{"interface":"front","average":{"1min":1.399,"5min":1.3999999999999999,"15min":1.5469999999999999},"min":{"1min":0.38700000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.1399999999999997,"5min":6.2229999999999999,"15min":99.212999999999994},"last":0.34599999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:36:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.5590000000000002,"15min":1.9179999999999999},"min":{"1min":0.31900000000000001,"5min":0.27600000000000002,"15min":0.26500000000000001},"max":{"1min":6.7549999999999999,"5min":47.32,"15min":47.32},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.478,"5min":3.6400000000000001,"15min":2.3250000000000002},"min":{"1min":0.376,"5min":0.34999999999999998,"15min":0.314},"max":{"1min":5.1719999999999997,"5min":93.843000000000004,"15min":93.843000000000004},"last":0.374}]},{"osd":5,"last update":"Mon Oct 9 09:36:30 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":1.1739999999999999,"15min":1.2170000000000001},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.5949999999999998,"5min":6.5949999999999998,"15min":9.2959999999999994},"last":0.68200000000000005},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.202,"15min":1.296},"min":{"1min":0.34899999999999998,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":6.7969999999999997,"5min":6.7969999999999997,"15min":7.5819999999999999},"last":0.60299999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:36:30 2023","interfaces":[{"interface":"back","average":{"1min":1.415,"5min":1.1779999999999999,"15min":1.214},"min":{"1min":0.39700000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.1909999999999998,"5min":6.2569999999999997,"15min":7.625},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.444,"5min":1.2,"15min":1.23},"min":{"1min":0.375,"5min":0.375,"15min":0.371},"max":{"1min":9.3369999999999997,"5min":9.3369999999999997,"15min":9.3369999999999997},"last":0.65600000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":1.339,"15min":1.246},"min":{"1min":0.40100000000000002,"5min":0.38,"15min":0.33400000000000002},"max":{"1min":9.2889999999999997,"5min":9.2889999999999997,"15min":9.2889999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.839,"5min":1.4390000000000001,"15min":1.4079999999999999},"min":{"1min":0.38300000000000001,"5min":0.374,"15min":0.36899999999999999},"max":{"1min":7.8579999999999997,"5min":12.34,"15min":12.34},"last":0.53100000000000003}]}]},{"osd":5,"up_from":37,"seq":158913790411,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3001984,"kb_used_data":2704760,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":90743168,"statfs":{"total":95995035648,"available":92921004032,"internally_reserved":0,"allocated":2769674240,"data_stored":2769449552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:37:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":5.9850000000000003,"15min":6.5209999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":12.217000000000001,"5min":85.363,"15min":85.363},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.1990000000000001,"5min":4.9660000000000002,"15min":5.5940000000000003},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":6.8970000000000002,"5min":85.334999999999994,"15min":85.334999999999994},"last":0.61699999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:36:31 2023","interfaces":[{"interface":"back","average":{"1min":5.3109999999999999,"5min":3.5379999999999998,"15min":1.9650000000000001},"min":{"1min":0.29699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":85.209999999999994,"5min":114.95999999999999,"15min":114.95999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":5.3570000000000002,"5min":3.6150000000000002,"15min":2.0499999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":85.238,"5min":115.087,"15min":115.087},"last":0.76200000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:36:54 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":1.3320000000000001,"15min":1.411},"min":{"1min":0.34000000000000002,"5min":0.31,"15min":0.31},"max":{"1min":3.214,"5min":12.731999999999999,"15min":14.019},"last":0.70499999999999996},{"interface":"front","average":{"1min":3.077,"5min":1.756,"15min":1.6759999999999999},"min":{"1min":0.42299999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":12.250999999999999,"5min":12.717000000000001,"15min":14.037000000000001},"last":0.77700000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:37:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":1.845,"15min":1.7869999999999999},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.311},"max":{"1min":8.968,"5min":45.158999999999999,"15min":59.652000000000001},"last":1.45},{"interface":"front","average":{"1min":2.9910000000000001,"5min":2.448,"15min":1.909},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":14.087,"5min":46.085999999999999,"15min":61.771999999999998},"last":0.69799999999999995}]},{"osd":4,"last update":"Mon Oct 9 09:36:31 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.90100000000000002,"15min":1.0309999999999999},"min":{"1min":0.32400000000000001,"5min":0.30299999999999999,"15min":0.28799999999999998},"max":{"1min":3.835,"5min":3.8929999999999998,"15min":10.747},"last":0.84399999999999997},{"interface":"front","average":{"1min":3.0129999999999999,"5min":1.3560000000000001,"15min":1.218},"min":{"1min":0.47899999999999998,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":12.151,"5min":12.151,"15min":12.151},"last":1.5840000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":1.3540000000000001,"15min":1.2909999999999999},"min":{"1min":0.35499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.1989999999999998,"5min":8.1989999999999998,"15min":14.518000000000001},"last":1.486},{"interface":"front","average":{"1min":4.2729999999999997,"5min":2.113,"15min":1.6379999999999999},"min":{"1min":0.45100000000000001,"5min":0.38,"15min":0.32900000000000001},"max":{"1min":18.638999999999999,"5min":18.638999999999999,"15min":18.638999999999999},"last":1.228}]},{"osd":7,"last update":"Mon Oct 9 09:36:38 2023","interfaces":[{"interface":"back","average":{"1min":4.0700000000000003,"5min":1.6259999999999999,"15min":1.474},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":18.683,"5min":18.683,"15min":18.683},"last":3.8250000000000002},{"interface":"front","average":{"1min":1.5980000000000001,"5min":1.282,"15min":1.357},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.5140000000000002,"5min":8.4619999999999997,"15min":8.4619999999999997},"last":1.355}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1925971968,"data_stored":1925971968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2201026560,"data_stored":2201026560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":835391488,"data_stored":835387416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2176188416,"data_stored":2176188416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1835008000,"data_stored":1835008000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2750877696,"data_stored":2750873624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2006515712,"data_stored":2006515712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1921454080,"data_stored":1921450008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:37:29.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pgp_num 16 2023-10-09T09:37:29.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:29 smithi163 bash[75425]: cluster 2023-10-09T09:37:27.877866+0000 mgr.y (mgr.34107) 940 : cluster 0 pgmap v1693: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8935/226875 objects degraded (3.938%); 11339/226875 objects misplaced (4.998%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T09:37:29.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:29 smithi163 bash[78107]: cluster 2023-10-09T09:37:27.877866+0000 mgr.y (mgr.34107) 940 : cluster 0 pgmap v1693: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8935/226875 objects degraded (3.938%); 11339/226875 objects misplaced (4.998%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T09:37:29.338 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:29.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:29.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:29.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:29.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:29.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:29.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:29.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:29.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:29.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:29.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:29.770 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:29.816 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:29.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:29.920 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:29.961 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:29.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:29.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:30.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:30.092 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:30.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:30.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:30.197 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:30.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:30.225 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:30.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:30.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:37:30.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:30.369 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:30.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:30.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:30.474 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:30.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:30.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:30.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:30.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:30.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:30.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:30.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:30.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:30.763 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:30.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:30.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:30.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:30.896 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:30.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:30.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:31.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:31.029 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:31.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:31.133 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:31.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:31.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:31.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:31.282 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:31.291 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:37:31.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:31 smithi163 bash[78107]: cluster 2023-10-09T09:37:29.878516+0000 mgr.y (mgr.34107) 941 : cluster 0 pgmap v1694: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 8935/223929 objects degraded (3.990%); 11038/223929 objects misplaced (4.929%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:31.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:31 smithi163 bash[78107]: cluster 2023-10-09T09:37:30.678538+0000 mon.a (mon.0) 1848 : cluster 3 Health check update: Degraded data redundancy: 8935/223929 objects degraded (3.990%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:31.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:31 smithi163 bash[75425]: cluster 2023-10-09T09:37:29.878516+0000 mgr.y (mgr.34107) 941 : cluster 0 pgmap v1694: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 8935/223929 objects degraded (3.990%); 11038/223929 objects misplaced (4.929%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:31.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:31 smithi163 bash[75425]: cluster 2023-10-09T09:37:30.678538+0000 mon.a (mon.0) 1848 : cluster 3 Health check update: Degraded data redundancy: 8935/223929 objects degraded (3.990%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:31.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:31 smithi181 bash[59345]: cluster 2023-10-09T09:37:29.878516+0000 mgr.y (mgr.34107) 941 : cluster 0 pgmap v1694: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 8935/223929 objects degraded (3.990%); 11038/223929 objects misplaced (4.929%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:31.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:31 smithi181 bash[59345]: cluster 2023-10-09T09:37:30.678538+0000 mon.a (mon.0) 1848 : cluster 3 Health check update: Degraded data redundancy: 8935/223929 objects degraded (3.990%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:31.325 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:31.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:31.429 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:31.459 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:31.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:31.566 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:31.591 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:31.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:31.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:31.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:31.725 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:31.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:31.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:31.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:31.872 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:31.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:31.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:31.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:32.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:32.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:32.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:32.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:32.138 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:32.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:32.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:32.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:32.271 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:32.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:32.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:32.377 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:32.408 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:32.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:32.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:32.448 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:32.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:32.558 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:32.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:32.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:32.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:32.697 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:32.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:32.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:32.803 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:32.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:32.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:32.937 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:32.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:32.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:32.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:33.075 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:33.100 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:33.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:33.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:33.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:33.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:33.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:33.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:33.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:33 smithi181 bash[59345]: cluster 2023-10-09T09:37:31.879260+0000 mgr.y (mgr.34107) 942 : cluster 0 pgmap v1695: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8935/218931 objects degraded (4.081%); 11038/218931 objects misplaced (5.042%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:33.334 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:33.359 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:33.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:33.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:33.458 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:33 smithi163 bash[78107]: cluster 2023-10-09T09:37:31.879260+0000 mgr.y (mgr.34107) 942 : cluster 0 pgmap v1695: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8935/218931 objects degraded (4.081%); 11038/218931 objects misplaced (5.042%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:33.459 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:37:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:37:33] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.33.4" 2023-10-09T09:37:33.459 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:33 smithi163 bash[75425]: cluster 2023-10-09T09:37:31.879260+0000 mgr.y (mgr.34107) 942 : cluster 0 pgmap v1695: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8935/218931 objects degraded (4.081%); 11038/218931 objects misplaced (5.042%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:33.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:33.474 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:33.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:33.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:33.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:33.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:33.645 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:33.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:33.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:33.749 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:33.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:33.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:33.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:33.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:33.910 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:33.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:34.013 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:34.048 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:34.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:34.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:34.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:34.177 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:34.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:34.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:34.280 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:34.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:34.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:34.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:34.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:34.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:34.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:34.488 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:34.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:34.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:34.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:34.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:34.762 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:34.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:34.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:34.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:34.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:34.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:35.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:35.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:35.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:35.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:35.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:35.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:35.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:35.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:35.262 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:35.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:35.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:35.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:35.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:35 smithi181 bash[59345]: cluster 2023-10-09T09:37:33.879886+0000 mgr.y (mgr.34107) 943 : cluster 0 pgmap v1696: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8935/216963 objects degraded (4.118%); 10826/216963 objects misplaced (4.990%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:35.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:35.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:35.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:35.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:35.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:35 smithi163 bash[78107]: cluster 2023-10-09T09:37:33.879886+0000 mgr.y (mgr.34107) 943 : cluster 0 pgmap v1696: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8935/216963 objects degraded (4.118%); 10826/216963 objects misplaced (4.990%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:35.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:35 smithi163 bash[75425]: cluster 2023-10-09T09:37:33.879886+0000 mgr.y (mgr.34107) 943 : cluster 0 pgmap v1696: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8935/216963 objects degraded (4.118%); 10826/216963 objects misplaced (4.990%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:35.516 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:35.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:35.573 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:35.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:35.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:35.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:35.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:35.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:35.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:35.839 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:35.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:35.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:35.945 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:35.977 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:35.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:35.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:36.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:36.114 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:36.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:36.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:36.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:36.247 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:36.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:36.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:36 smithi181 bash[59345]: cluster 2023-10-09T09:37:35.680074+0000 mon.a (mon.0) 1849 : cluster 3 Health check update: Degraded data redundancy: 8935/216963 objects degraded (4.118%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:36.352 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:36.388 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:36.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:36.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:36.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:36.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:36.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:36.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:36.562 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:36 smithi163 bash[75425]: cluster 2023-10-09T09:37:35.680074+0000 mon.a (mon.0) 1849 : cluster 3 Health check update: Degraded data redundancy: 8935/216963 objects degraded (4.118%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:36.562 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:36 smithi163 bash[78107]: cluster 2023-10-09T09:37:35.680074+0000 mon.a (mon.0) 1849 : cluster 3 Health check update: Degraded data redundancy: 8935/216963 objects degraded (4.118%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:36.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:36.662 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:36.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:36.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:36.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:36.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:36.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:36.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:36.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:36.925 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:36.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:36.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:37.029 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:37.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:37.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:37.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:37.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:37.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:37.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:37.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:37.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:37.330 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:37.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:37.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:37.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:37.468 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:37.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:37.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:37.574 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:37.607 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:37.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:37.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:37.733 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:37.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:37.758 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:37 smithi163 bash[75425]: cluster 2023-10-09T09:37:35.880393+0000 mgr.y (mgr.34107) 944 : cluster 0 pgmap v1697: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 8935/213774 objects degraded (4.180%); 10463/213774 objects misplaced (4.894%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:37:37.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:37.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:37 smithi163 bash[78107]: cluster 2023-10-09T09:37:35.880393+0000 mgr.y (mgr.34107) 944 : cluster 0 pgmap v1697: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 8935/213774 objects degraded (4.180%); 10463/213774 objects misplaced (4.894%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:37:37.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:37.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:37.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:37.973 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:37.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:37 smithi181 bash[59345]: cluster 2023-10-09T09:37:35.880393+0000 mgr.y (mgr.34107) 944 : cluster 0 pgmap v1697: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 8935/213774 objects degraded (4.180%); 10463/213774 objects misplaced (4.894%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:37:37.997 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:37.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:37.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:38.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:38.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:38.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:38.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:38.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:38.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:38.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:38.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:38.368 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:38.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:38.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:38.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:38.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:38.513 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:38.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:38.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:38.615 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:38.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:38.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:38.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:38.760 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:38.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:38.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:38.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:38.888 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:38.915 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:38.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:38.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:39.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:39.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:39.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:39.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:39.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:39.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:39.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:39.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:39.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:39.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:39.416 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:39.449 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:39.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:39.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:39.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:39.585 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:39.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:39.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:39.688 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:39.720 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:39.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:39.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:39.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:39 smithi181 bash[59345]: cluster 2023-10-09T09:37:37.881017+0000 mgr.y (mgr.34107) 945 : cluster 0 pgmap v1698: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 8935/206709 objects degraded (4.323%); 10244/206709 objects misplaced (4.956%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:39.824 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:39.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:39.855 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:39.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:39.956 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:39.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:39 smithi163 bash[75425]: cluster 2023-10-09T09:37:37.881017+0000 mgr.y (mgr.34107) 945 : cluster 0 pgmap v1698: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 8935/206709 objects degraded (4.323%); 10244/206709 objects misplaced (4.956%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:39.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:39 smithi163 bash[78107]: cluster 2023-10-09T09:37:37.881017+0000 mgr.y (mgr.34107) 945 : cluster 0 pgmap v1698: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 8935/206709 objects degraded (4.323%); 10244/206709 objects misplaced (4.956%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:37:39.981 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:39.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:40.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:40.111 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:40.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:40.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:40.252 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:40.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:40.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:40.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:40.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:40.483 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:40.507 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:40.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:40.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:40.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:40.649 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:40.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:40.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:40.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:40 smithi163 bash[75425]: audit 2023-10-09T09:37:40.376621+0000 mon.a (mon.0) 1850 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:40.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:40.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:40.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:40.888 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:40.896 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:40 smithi181 bash[59345]: audit 2023-10-09T09:37:40.376621+0000 mon.a (mon.0) 1850 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:40.896 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:40 smithi181 bash[59345]: cluster 2023-10-09T09:37:40.681492+0000 mon.a (mon.0) 1851 : cluster 3 Health check update: Degraded data redundancy: 8935/206709 objects degraded (4.323%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:40.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:40.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:40.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:40.985 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:40 smithi163 bash[78107]: audit 2023-10-09T09:37:40.376621+0000 mon.a (mon.0) 1850 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:40.985 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:40 smithi163 bash[78107]: cluster 2023-10-09T09:37:40.681492+0000 mon.a (mon.0) 1851 : cluster 3 Health check update: Degraded data redundancy: 8935/206709 objects degraded (4.323%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:40.985 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:40 smithi163 bash[75425]: cluster 2023-10-09T09:37:40.681492+0000 mon.a (mon.0) 1851 : cluster 3 Health check update: Degraded data redundancy: 8935/206709 objects degraded (4.323%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:37:41.018 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:41.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:41.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:41.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:41.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:41.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:41.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:41.275 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:41.303 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:41.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:41.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:41.408 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:41.440 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:41.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:41.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:41.543 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:41.567 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:41.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:41.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:41.671 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:41.703 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:41.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:41.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:41.807 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:41.835 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:41.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:41.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:41.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:41.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:41 smithi163 bash[75425]: cluster 2023-10-09T09:37:39.881568+0000 mgr.y (mgr.34107) 946 : cluster 0 pgmap v1699: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 8935/203319 objects degraded (4.395%); 9882/203319 objects misplaced (4.860%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:37:41.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:41 smithi163 bash[78107]: cluster 2023-10-09T09:37:39.881568+0000 mgr.y (mgr.34107) 946 : cluster 0 pgmap v1699: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 8935/203319 objects degraded (4.395%); 9882/203319 objects misplaced (4.860%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:37:41.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:41.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:42.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:41 smithi181 bash[59345]: cluster 2023-10-09T09:37:39.881568+0000 mgr.y (mgr.34107) 946 : cluster 0 pgmap v1699: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 8935/203319 objects degraded (4.395%); 9882/203319 objects misplaced (4.860%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:37:42.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:42.088 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:42.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:42.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:42.190 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:42.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:42.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:42.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:42.334 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:42.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:42.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:42.474 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:42.504 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:42.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:42.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:42.611 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:42.644 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:42.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:42.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:42.748 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:42.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:42.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:42.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:42.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:42.918 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:42.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:42.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:43.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:43.049 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:43.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:43.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:43.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:43.175 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:43.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:43.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:43.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:43.302 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:43.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:43.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:43.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:43.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:43.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:43.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:43.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:43.551 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:37:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:37:43] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.33.4" 2023-10-09T09:37:43.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:43.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:43.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:43.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:43.698 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:43.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:43.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:43.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:43.810 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:43 smithi163 bash[75425]: cluster 2023-10-09T09:37:41.882317+0000 mgr.y (mgr.34107) 947 : cluster 0 pgmap v1700: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 8935/197973 objects degraded (4.513%); 9882/197973 objects misplaced (4.992%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:43.811 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:43 smithi163 bash[78107]: cluster 2023-10-09T09:37:41.882317+0000 mgr.y (mgr.34107) 947 : cluster 0 pgmap v1700: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 8935/197973 objects degraded (4.513%); 9882/197973 objects misplaced (4.992%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:43.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:43.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:43.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:43.937 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:43.974 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:43.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:43.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:43 smithi181 bash[59345]: cluster 2023-10-09T09:37:41.882317+0000 mgr.y (mgr.34107) 947 : cluster 0 pgmap v1700: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 8935/197973 objects degraded (4.513%); 9882/197973 objects misplaced (4.992%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:44.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:44.102 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:44.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:44.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:44.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:44.250 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:44.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:44.351 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:44.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:44.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:44.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:44.483 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:44.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:44.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:44.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:44.610 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:44.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:44.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:44.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:44.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:44.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:44.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:44.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:44.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:44.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:44.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:44.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:44.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:45.031 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:45.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:45.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:45.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:45.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:45.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:45.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:45.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:45.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:45.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:45.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:45.415 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:45.442 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:45.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:45.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:45.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:45.594 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:45.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:45.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:45.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:45.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:45.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:45.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:45.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:45 smithi163 bash[78107]: cluster 2023-10-09T09:37:43.882923+0000 mgr.y (mgr.34107) 948 : cluster 0 pgmap v1701: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 8935/195885 objects degraded (4.561%); 9650/195885 objects misplaced (4.926%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:45.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:45 smithi163 bash[78107]: cluster 2023-10-09T09:37:45.682434+0000 mon.a (mon.0) 1852 : cluster 3 Health check update: Degraded data redundancy: 8935/195885 objects degraded (4.561%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:37:45.726 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:45 smithi163 bash[75425]: cluster 2023-10-09T09:37:43.882923+0000 mgr.y (mgr.34107) 948 : cluster 0 pgmap v1701: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 8935/195885 objects degraded (4.561%); 9650/195885 objects misplaced (4.926%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:45.726 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:45 smithi163 bash[75425]: cluster 2023-10-09T09:37:45.682434+0000 mon.a (mon.0) 1852 : cluster 3 Health check update: Degraded data redundancy: 8935/195885 objects degraded (4.561%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:37:45.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:45.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:45.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:45.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:45.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:45.998 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:46.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:46.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:46.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:45 smithi181 bash[59345]: cluster 2023-10-09T09:37:43.882923+0000 mgr.y (mgr.34107) 948 : cluster 0 pgmap v1701: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 8935/195885 objects degraded (4.561%); 9650/195885 objects misplaced (4.926%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:37:46.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:45 smithi181 bash[59345]: cluster 2023-10-09T09:37:45.682434+0000 mon.a (mon.0) 1852 : cluster 3 Health check update: Degraded data redundancy: 8935/195885 objects degraded (4.561%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:37:46.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:46.134 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:46.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:46.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:46.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:46.275 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:46.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:46.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:46.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:46.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:46.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:46.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:46.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:46.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:46.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:46.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:46.679 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:46.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:46.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:46.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:46.809 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:46.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:46.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:46.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:46.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:46.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:46.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:47.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:47.088 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:47.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:47.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:47.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:47.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:47.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:47.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:47.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:47.358 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:47.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:47.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:47.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:47.502 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:47.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:47.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:47.609 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:47.641 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:47.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:47.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:47.745 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:47.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:47.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:47.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:47 smithi181 bash[59345]: cluster 2023-10-09T09:37:45.883553+0000 mgr.y (mgr.34107) 949 : cluster 0 pgmap v1702: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 8935/192672 objects degraded (4.637%); 9288/192672 objects misplaced (4.821%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T09:37:47.867 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:47 smithi163 bash[75425]: cluster 2023-10-09T09:37:45.883553+0000 mgr.y (mgr.34107) 949 : cluster 0 pgmap v1702: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 8935/192672 objects degraded (4.637%); 9288/192672 objects misplaced (4.821%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T09:37:47.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:47 smithi163 bash[78107]: cluster 2023-10-09T09:37:45.883553+0000 mgr.y (mgr.34107) 949 : cluster 0 pgmap v1702: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 8935/192672 objects degraded (4.637%); 9288/192672 objects misplaced (4.821%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T09:37:47.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:47.907 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:47.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:47.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:48.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:48.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:48.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:48.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:48.062 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:37:48.062 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:37:48.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.063 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:129941 8.7 8:e9d33c25:::benchmark_data_smithi163_151167_object34708:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1153)", 2023-10-09T09:37:48.063 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:37:48.044952+0000", 2023-10-09T09:37:48.063 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.016838609000000001, 2023-10-09T09:37:48.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016876156999999999, 2023-10-09T09:37:48.063 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:37:48.063 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:37:48.063 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:37:48.064 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:37:48.064 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:37:48.064 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 129941 2023-10-09T09:37:48.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.064 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:37:48.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:37:48.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.044952+0000", 2023-10-09T09:37:48.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:37:48.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:37:48.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.044952+0000", 2023-10-09T09:37:48.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:37:48.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:37:48.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.044950+0000", 2023-10-09T09:37:48.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:37:48.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:37:48.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.044957+0000", 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4239999999999998e-06 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.044967+0000", 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0247e-05 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.044973+0000", 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6119999999999998e-06 2023-10-09T09:37:48.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.048582+0000", 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036093940000000001 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.048703+0000", 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012084299999999999 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,6", 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.048821+0000", 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000118051 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.058556+0000", 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0097353219999999994 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:37:48.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.059046+0000", 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00048995899999999999 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:129946 8.1 8:8119acd9:::benchmark_data_smithi163_151167_object34713:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1153)", 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:37:48.054483+0000", 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0073075459999999998, 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0074394109999999999, 2023-10-09T09:37:48.070 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 129946 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.054483+0000", 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:37:48.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.054483+0000", 2023-10-09T09:37:48.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:37:48.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:37:48.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.054481+0000", 2023-10-09T09:37:48.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:37:48.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:37:48.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.054489+0000", 2023-10-09T09:37:48.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8909999999999995e-06 2023-10-09T09:37:48.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:37:48.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.054510+0000", 2023-10-09T09:37:48.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0798e-05 2023-10-09T09:37:48.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:37:48.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.054519+0000", 2023-10-09T09:37:48.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9330000000000007e-06 2023-10-09T09:37:48.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:37:48.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.058937+0000", 2023-10-09T09:37:48.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0044185600000000002 2023-10-09T09:37:48.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:37:48.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.059144+0000", 2023-10-09T09:37:48.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000206479 2023-10-09T09:37:48.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T09:37:48.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.059296+0000", 2023-10-09T09:37:48.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015204699999999999 2023-10-09T09:37:48.077 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:37:48.077 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:37:48.077 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:37:48.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.077 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:129948 8.f 8:f8fe3cae:::benchmark_data_smithi163_151167_object34715:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1153)", 2023-10-09T09:37:48.078 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:37:48.057084+0000", 2023-10-09T09:37:48.078 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0047065329999999997, 2023-10-09T09:37:48.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049124240000000003, 2023-10-09T09:37:48.078 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:37:48.078 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:37:48.078 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:37:48.078 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:37:48.078 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:37:48.079 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 129948 2023-10-09T09:37:48.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.079 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:37:48.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:37:48.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.057084+0000", 2023-10-09T09:37:48.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:37:48.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:37:48.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.057084+0000", 2023-10-09T09:37:48.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:37:48.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:37:48.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.057082+0000", 2023-10-09T09:37:48.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:37:48.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:37:48.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.057088+0000", 2023-10-09T09:37:48.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3229999999999997e-06 2023-10-09T09:37:48.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:37:48.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.057097+0000", 2023-10-09T09:37:48.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0650000000000005e-06 2023-10-09T09:37:48.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:37:48.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.057102+0000", 2023-10-09T09:37:48.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3850000000000001e-06 2023-10-09T09:37:48.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:37:48.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.058582+0000", 2023-10-09T09:37:48.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014799220000000001 2023-10-09T09:37:48.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:37:48.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.058742+0000", 2023-10-09T09:37:48.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016016000000000001 2023-10-09T09:37:48.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:37:48.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.058937+0000", 2023-10-09T09:37:48.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019514599999999999 2023-10-09T09:37:48.085 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:37:48.085 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:37:48.085 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:37:48.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.086 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:129928 8.f 8:f49c6e17:::benchmark_data_smithi163_151167_object34695:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1153)", 2023-10-09T09:37:48.086 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:37:48.027329+0000", 2023-10-09T09:37:48.086 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.034461299000000001, 2023-10-09T09:37:48.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.034740820999999998, 2023-10-09T09:37:48.086 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:37:48.086 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:37:48.086 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:37:48.086 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:37:48.087 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:37:48.087 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 129928 2023-10-09T09:37:48.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.087 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:37:48.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:37:48.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.027329+0000", 2023-10-09T09:37:48.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:37:48.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:37:48.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.027329+0000", 2023-10-09T09:37:48.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:37:48.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:37:48.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.027328+0000", 2023-10-09T09:37:48.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:37:48.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:37:48.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.027334+0000", 2023-10-09T09:37:48.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.857e-06 2023-10-09T09:37:48.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:37:48.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.027347+0000", 2023-10-09T09:37:48.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2904e-05 2023-10-09T09:37:48.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:37:48.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.027355+0000", 2023-10-09T09:37:48.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2109999999999998e-06 2023-10-09T09:37:48.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:37:48.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.046921+0000", 2023-10-09T09:37:48.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019565803999999999 2023-10-09T09:37:48.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:37:48.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.047017+0000", 2023-10-09T09:37:48.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6144999999999995e-05 2023-10-09T09:37:48.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:37:48.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.047102+0000", 2023-10-09T09:37:48.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5385999999999999e-05 2023-10-09T09:37:48.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.093 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:37:48.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.058439+0000", 2023-10-09T09:37:48.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011337053999999999 2023-10-09T09:37:48.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:37:48.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:37:48.094 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:37:48.094 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:37:48.058550+0000", 2023-10-09T09:37:48.094 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000110964 2023-10-09T09:37:48.094 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:37:48.094 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:37:48.094 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:37:48.094 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:37:48.094 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T09:37:48.094 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 4 2023-10-09T09:37:48.095 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:37:48.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:48.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:48.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:48.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:48.282 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:48.307 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:48.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:48.414 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:48.439 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:48.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:48.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:48.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:48.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:48.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:48.699 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:48.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:48.804 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:48.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:48.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:48.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:48.938 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:48.981 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:48.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:48.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:49.085 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:49.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:49.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:49.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:49.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:49.256 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:49.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:49.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:49.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:49.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:49.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:49.499 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:49.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:49.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:49.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:49.642 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:37:49.653 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:49.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:49.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:49.713 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:37:49.731 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:37:49.732 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 8 pgp_num to 16 2023-10-09T09:37:49.749 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:37:49.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:49.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:49 smithi181 bash[59345]: cluster 2023-10-09T09:37:47.887695+0000 mgr.y (mgr.34107) 950 : cluster 0 pgmap v1703: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 8935/185784 objects degraded (4.809%); 9061/185784 objects misplaced (4.877%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:37:49.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:49 smithi181 bash[59345]: audit 2023-10-09T09:37:48.966429+0000 mon.a (mon.0) 1853 : audit 1 from='client.? 172.21.15.163:0/3973450332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:37:49.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:49 smithi181 bash[59345]: audit 2023-10-09T09:37:49.424025+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.163:0/1979904365' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:37:49.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:49 smithi181 bash[59345]: audit 2023-10-09T09:37:49.425730+0000 mon.a (mon.0) 1854 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:37:49.765 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:49 smithi181 bash[59345]: audit 2023-10-09T09:37:49.599365+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.163:0/1765276384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:37:49.782 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:49.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:49.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:49.803 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:37:49.843 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:37:49.873 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:37:49.888 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:49.904 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:37:49.917 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:49.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:49.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:49.958 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:37:49.963 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:37:49.963 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:37:49.963 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:37:49.964 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:37:49.964 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:37:49.964 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:37:49.964 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:37:49.964 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:37:50.019 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:50.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:49 smithi163 bash[75425]: cluster 2023-10-09T09:37:47.887695+0000 mgr.y (mgr.34107) 950 : cluster 0 pgmap v1703: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 8935/185784 objects degraded (4.809%); 9061/185784 objects misplaced (4.877%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:37:50.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:49 smithi163 bash[75425]: audit 2023-10-09T09:37:48.966429+0000 mon.a (mon.0) 1853 : audit 1 from='client.? 172.21.15.163:0/3973450332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:37:50.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:49 smithi163 bash[75425]: audit 2023-10-09T09:37:49.424025+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.163:0/1979904365' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:37:50.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:49 smithi163 bash[75425]: audit 2023-10-09T09:37:49.425730+0000 mon.a (mon.0) 1854 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:37:50.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:49 smithi163 bash[75425]: audit 2023-10-09T09:37:49.599365+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.163:0/1765276384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:37:50.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:49 smithi163 bash[78107]: cluster 2023-10-09T09:37:47.887695+0000 mgr.y (mgr.34107) 950 : cluster 0 pgmap v1703: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 8935/185784 objects degraded (4.809%); 9061/185784 objects misplaced (4.877%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:37:50.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:49 smithi163 bash[78107]: audit 2023-10-09T09:37:48.966429+0000 mon.a (mon.0) 1853 : audit 1 from='client.? 172.21.15.163:0/3973450332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:37:50.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:49 smithi163 bash[78107]: audit 2023-10-09T09:37:49.424025+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.163:0/1979904365' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:37:50.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:49 smithi163 bash[78107]: audit 2023-10-09T09:37:49.425730+0000 mon.a (mon.0) 1854 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T09:37:50.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:49 smithi163 bash[78107]: audit 2023-10-09T09:37:49.599365+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.163:0/1765276384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:37:50.066 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:50.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:50.173 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:50.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:50.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:50.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:50.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T09:37:50.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:50.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:50.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:50.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:50.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:50.479 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:50.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:50.582 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:50.608 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:50.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:50.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:50.710 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:50.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:50.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:50.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:50.857 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:50.867 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:50 smithi181 bash[59345]: cluster 2023-10-09T09:37:49.710054+0000 mon.a (mon.0) 1855 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:37:50.867 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:50 smithi181 bash[59345]: cluster 2023-10-09T09:37:49.715824+0000 osd.5 (osd.5) 720 : cluster 3 failed to encode map e1154 with expected crc 2023-10-09T09:37:50.867 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:50 smithi181 bash[59345]: cluster 2023-10-09T09:37:49.716021+0000 osd.6 (osd.6) 434 : cluster 3 failed to encode map e1154 with expected crc 2023-10-09T09:37:50.867 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:50 smithi181 bash[59345]: audit 2023-10-09T09:37:49.720944+0000 mon.a (mon.0) 1856 : audit 1 from='client.? 172.21.15.163:0/3973450332' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:37:50.867 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:50 smithi181 bash[59345]: audit 2023-10-09T09:37:49.721233+0000 mon.a (mon.0) 1857 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:37:50.869 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:50 smithi181 bash[59345]: cluster 2023-10-09T09:37:49.721545+0000 mon.a (mon.0) 1858 : cluster 0 osdmap e1154: 8 total, 8 up, 8 in 2023-10-09T09:37:50.869 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:50 smithi181 bash[59345]: cluster 2023-10-09T09:37:50.683648+0000 mon.a (mon.0) 1859 : cluster 3 Health check update: Degraded data redundancy: 8935/185784 objects degraded (4.809%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:37:50.891 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:50.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:50.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:51.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:51.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[78107]: cluster 2023-10-09T09:37:49.710054+0000 mon.a (mon.0) 1855 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:37:51.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[78107]: cluster 2023-10-09T09:37:49.715824+0000 osd.5 (osd.5) 720 : cluster 3 failed to encode map e1154 with expected crc 2023-10-09T09:37:51.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[78107]: cluster 2023-10-09T09:37:49.716021+0000 osd.6 (osd.6) 434 : cluster 3 failed to encode map e1154 with expected crc 2023-10-09T09:37:51.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[78107]: audit 2023-10-09T09:37:49.720944+0000 mon.a (mon.0) 1856 : audit 1 from='client.? 172.21.15.163:0/3973450332' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:37:51.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[78107]: audit 2023-10-09T09:37:49.721233+0000 mon.a (mon.0) 1857 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:37:51.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[78107]: cluster 2023-10-09T09:37:49.721545+0000 mon.a (mon.0) 1858 : cluster 0 osdmap e1154: 8 total, 8 up, 8 in 2023-10-09T09:37:51.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[78107]: cluster 2023-10-09T09:37:50.683648+0000 mon.a (mon.0) 1859 : cluster 3 Health check update: Degraded data redundancy: 8935/185784 objects degraded (4.809%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:37:51.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[75425]: cluster 2023-10-09T09:37:49.710054+0000 mon.a (mon.0) 1855 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:37:51.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[75425]: cluster 2023-10-09T09:37:49.715824+0000 osd.5 (osd.5) 720 : cluster 3 failed to encode map e1154 with expected crc 2023-10-09T09:37:51.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[75425]: cluster 2023-10-09T09:37:49.716021+0000 osd.6 (osd.6) 434 : cluster 3 failed to encode map e1154 with expected crc 2023-10-09T09:37:51.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[75425]: audit 2023-10-09T09:37:49.720944+0000 mon.a (mon.0) 1856 : audit 1 from='client.? 172.21.15.163:0/3973450332' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:37:51.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[75425]: audit 2023-10-09T09:37:49.721233+0000 mon.a (mon.0) 1857 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T09:37:51.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[75425]: cluster 2023-10-09T09:37:49.721545+0000 mon.a (mon.0) 1858 : cluster 0 osdmap e1154: 8 total, 8 up, 8 in 2023-10-09T09:37:51.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:50 smithi163 bash[75425]: cluster 2023-10-09T09:37:50.683648+0000 mon.a (mon.0) 1859 : cluster 3 Health check update: Degraded data redundancy: 8935/185784 objects degraded (4.809%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:37:51.023 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:51.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:51.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:51.129 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:51.155 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:51.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:51.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:51.263 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:51.292 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:51.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:51.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:51.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:51.431 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:51.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:51.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:51.534 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:51.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:51.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:51.654 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:51.685 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:51.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:51.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:51.790 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:51.825 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:51.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:51.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:51.937 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:51 smithi181 bash[59345]: cluster 2023-10-09T09:37:49.888198+0000 mgr.y (mgr.34107) 951 : cluster 0 pgmap v1705: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 8935/182691 objects degraded (4.891%); 8726/182691 objects misplaced (4.776%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:51.957 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:51.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:51.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:51.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:51 smithi163 bash[78107]: cluster 2023-10-09T09:37:49.888198+0000 mgr.y (mgr.34107) 951 : cluster 0 pgmap v1705: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 8935/182691 objects degraded (4.891%); 8726/182691 objects misplaced (4.776%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:51.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:51 smithi163 bash[75425]: cluster 2023-10-09T09:37:49.888198+0000 mgr.y (mgr.34107) 951 : cluster 0 pgmap v1705: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 8935/182691 objects degraded (4.891%); 8726/182691 objects misplaced (4.776%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:52.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:52.088 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:52.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:52.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:52.192 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:52.215 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:52.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:52.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:52.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:52.356 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:52.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:52.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:52.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:52.488 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:52.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:52.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:52.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:52.630 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:52.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:52.734 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:52.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:52.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:52.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:52.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:52.910 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:52.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:52.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:53.017 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:53.050 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:53.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:53.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:53.157 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:53.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:53.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:53.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:53.304 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:53.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:53.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:53.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:53.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:53.441 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:37:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:37:53] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.33.4" 2023-10-09T09:37:53.459 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:53.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:53.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:53.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:53.594 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:53.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:53.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:53.699 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:53.737 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:53.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:53.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:53.840 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:53.849 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:53 smithi181 bash[59345]: cluster 2023-10-09T09:37:51.889467+0000 mgr.y (mgr.34107) 952 : cluster 0 pgmap v1706: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 8935/177876 objects degraded (5.023%); 8726/177876 objects misplaced (4.906%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:53.869 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:53.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:53.975 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:54.002 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:54.003 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:53 smithi163 bash[78107]: cluster 2023-10-09T09:37:51.889467+0000 mgr.y (mgr.34107) 952 : cluster 0 pgmap v1706: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 8935/177876 objects degraded (5.023%); 8726/177876 objects misplaced (4.906%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:54.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:53 smithi163 bash[75425]: cluster 2023-10-09T09:37:51.889467+0000 mgr.y (mgr.34107) 952 : cluster 0 pgmap v1706: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 8935/177876 objects degraded (5.023%); 8726/177876 objects misplaced (4.906%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:54.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:54.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:54.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:54.134 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:54.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:54.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:54.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:54.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:54.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:54.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:54.372 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:54.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:54.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:54.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:54.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:54.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:54.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:54.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:54.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:54.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:54.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:54.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:54.787 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:54.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:54.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:54.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:54.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:54.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:55.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:55.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:55.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:55.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:55.172 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:55.200 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:55.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:55.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:55.304 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:55.355 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:55.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:55.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:55.463 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:55.489 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:55.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:55.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:55.592 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:55.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:55.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:55.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:55.679 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:37:55.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:37:55.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:55.746 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:55.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:55.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:55.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:55.874 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:55 smithi163 bash[78107]: cluster 2023-10-09T09:37:53.890203+0000 mgr.y (mgr.34107) 953 : cluster 0 pgmap v1707: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 8935/175983 objects degraded (5.077%); 8506/175983 objects misplaced (4.833%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:55.874 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:55 smithi163 bash[78107]: audit 2023-10-09T09:37:55.377529+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:55.874 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:55 smithi163 bash[78107]: cluster 2023-10-09T09:37:55.686976+0000 mon.a (mon.0) 1861 : cluster 3 Health check update: Degraded data redundancy: 8935/175983 objects degraded (5.077%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:37:55.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:55 smithi163 bash[75425]: cluster 2023-10-09T09:37:53.890203+0000 mgr.y (mgr.34107) 953 : cluster 0 pgmap v1707: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 8935/175983 objects degraded (5.077%); 8506/175983 objects misplaced (4.833%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:55.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:55 smithi163 bash[75425]: audit 2023-10-09T09:37:55.377529+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:55.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:55 smithi163 bash[75425]: cluster 2023-10-09T09:37:55.686976+0000 mon.a (mon.0) 1861 : cluster 3 Health check update: Degraded data redundancy: 8935/175983 objects degraded (5.077%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:37:55.896 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:55.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:55.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:56.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:56.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:55 smithi181 bash[59345]: cluster 2023-10-09T09:37:53.890203+0000 mgr.y (mgr.34107) 953 : cluster 0 pgmap v1707: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 8935/175983 objects degraded (5.077%); 8506/175983 objects misplaced (4.833%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:56.016 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:55 smithi181 bash[59345]: audit 2023-10-09T09:37:55.377529+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:37:56.016 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:55 smithi181 bash[59345]: cluster 2023-10-09T09:37:55.686976+0000 mon.a (mon.0) 1861 : cluster 3 Health check update: Degraded data redundancy: 8935/175983 objects degraded (5.077%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:37:56.037 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:56.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:56.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:56.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:56.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:56.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:56.279 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:56.306 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:56.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:56.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:56.442 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:56.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:56.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:56.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:56.577 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:56.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:56.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:56.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:56.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:56.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:56.749 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:56.820 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:56.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:56.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:56.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:56.958 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:57.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:57.009 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:57.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:57.110 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:57.145 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:37:57.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:57.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:37:57.253 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:57.279 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:57.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:57.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:57.385 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:57.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:57.412 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:57.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:57.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:57.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:57.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:57.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:57.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:57.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:57.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:57.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:57.701 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:37:57.701 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:37:57.701 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T09:37:57.701 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:37:57.701 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:37:57.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T09:37:57.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:57.817 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:57.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:57.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:57.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:57 smithi163 bash[75425]: cluster 2023-10-09T09:37:55.897416+0000 mgr.y (mgr.34107) 954 : cluster 0 pgmap v1708: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 8935/172863 objects degraded (5.169%); 8147/172863 objects misplaced (4.713%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:57.920 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:57.954 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:57.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:57.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:58.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:58.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:57 smithi181 bash[59345]: cluster 2023-10-09T09:37:55.897416+0000 mgr.y (mgr.34107) 954 : cluster 0 pgmap v1708: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 8935/172863 objects degraded (5.169%); 8147/172863 objects misplaced (4.713%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:58.084 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:37:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:58.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:37:58.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:58.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:57 smithi163 bash[78107]: cluster 2023-10-09T09:37:55.897416+0000 mgr.y (mgr.34107) 954 : cluster 0 pgmap v1708: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 8935/172863 objects degraded (5.169%); 8147/172863 objects misplaced (4.713%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:58.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:58.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:58.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:58.353 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:58.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:58.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:58.456 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:58.486 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:58.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:58.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:58.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:58.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:37:58.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:58.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:37:58.722 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:58.748 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:58.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:58.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:58.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:58.889 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:37:58.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:58.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:37:58.991 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:59.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:59.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:59.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:59.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:59.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:59.268 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:59.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:59.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:59.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:59.444 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:37:59.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:59.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:37:59.547 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:59.572 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:37:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:59.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:37:59.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:59.710 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:59.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:37:59.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:59.814 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:59.814 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:37:59.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:37:59 smithi181 bash[59345]: cluster 2023-10-09T09:37:57.898228+0000 mgr.y (mgr.34107) 955 : cluster 0 pgmap v1709: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 8935/165900 objects degraded (5.386%); 7946/165900 objects misplaced (4.790%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:59.836 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:37:59.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:59.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:37:59.850 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:37:59 smithi163 bash[75425]: cluster 2023-10-09T09:37:57.898228+0000 mgr.y (mgr.34107) 955 : cluster 0 pgmap v1709: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 8935/165900 objects degraded (5.386%); 7946/165900 objects misplaced (4.790%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:59.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:37:59 smithi163 bash[78107]: cluster 2023-10-09T09:37:57.898228+0000 mgr.y (mgr.34107) 955 : cluster 0 pgmap v1709: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 8935/165900 objects degraded (5.386%); 7946/165900 objects misplaced (4.790%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:37:59.940 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:59.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:37:59.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:37:59.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:00.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:00.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:00.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:00.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:00.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:00.238 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:00.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:00.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:00.341 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:00.380 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:00.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:00.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:00.484 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:00.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:00.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:00.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:00.621 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:00.648 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:00.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:00.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:00.757 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:00.788 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:00.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:00.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:00.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:00.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:00.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:01.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:01.035 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:00 smithi181 bash[59345]: cluster 2023-10-09T09:38:00.695177+0000 mon.a (mon.0) 1862 : cluster 3 Health check update: Degraded data redundancy: 8935/165900 objects degraded (5.386%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:01.051 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:01.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:01.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:01.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:01.179 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:01.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:01.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:01.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:00 smithi163 bash[78107]: cluster 2023-10-09T09:38:00.695177+0000 mon.a (mon.0) 1862 : cluster 3 Health check update: Degraded data redundancy: 8935/165900 objects degraded (5.386%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:01.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:00 smithi163 bash[75425]: cluster 2023-10-09T09:38:00.695177+0000 mon.a (mon.0) 1862 : cluster 3 Health check update: Degraded data redundancy: 8935/165900 objects degraded (5.386%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:01.282 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:01.312 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:01.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:01.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:01.415 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:01.450 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:01.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:01.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:01.554 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:01.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:01.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:01.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:01.701 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:01.726 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:01.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:01.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:01.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:01 smithi181 bash[59345]: cluster 2023-10-09T09:37:59.898889+0000 mgr.y (mgr.34107) 956 : cluster 0 pgmap v1710: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 8935/162555 objects degraded (5.497%); 7558/162555 objects misplaced (4.650%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:38:01.829 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:01 smithi163 bash[75425]: cluster 2023-10-09T09:37:59.898889+0000 mgr.y (mgr.34107) 956 : cluster 0 pgmap v1710: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 8935/162555 objects degraded (5.497%); 7558/162555 objects misplaced (4.650%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:38:01.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:01.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:01.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:01.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:01.969 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:01.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:01.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:01.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:02.006 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:38:02.096 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:02.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:01 smithi163 bash[78107]: cluster 2023-10-09T09:37:59.898889+0000 mgr.y (mgr.34107) 956 : cluster 0 pgmap v1710: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 8935/162555 objects degraded (5.497%); 7558/162555 objects misplaced (4.650%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:38:02.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:02.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:02.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:02.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:02.263 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:02.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:02.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:02.371 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:02.407 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:02.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:02.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:02.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:02.554 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:02.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:02.664 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:02.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:02.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:02.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:02.793 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:02.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:02.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:02.930 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:02.947 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:02.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:03.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:03.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:03.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:03.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:03.182 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:03.207 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:03.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:03.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:03.313 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:03.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:03.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:03.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:03.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:03.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:03.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:03.491 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:38:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:38:03] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.33.4" 2023-10-09T09:38:03.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:03.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:03.626 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:03.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:03.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:03.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:03.760 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:03.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:03.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:03.786 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:03 smithi163 bash[78107]: cluster 2023-10-09T09:38:01.899707+0000 mgr.y (mgr.34107) 957 : cluster 0 pgmap v1711: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 8935/157557 objects degraded (5.671%); 7558/157557 objects misplaced (4.797%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:38:03.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:03 smithi163 bash[75425]: cluster 2023-10-09T09:38:01.899707+0000 mgr.y (mgr.34107) 957 : cluster 0 pgmap v1711: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 8935/157557 objects degraded (5.671%); 7558/157557 objects misplaced (4.797%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:38:03.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:03.880 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:03 smithi181 bash[59345]: cluster 2023-10-09T09:38:01.899707+0000 mgr.y (mgr.34107) 957 : cluster 0 pgmap v1711: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 8935/157557 objects degraded (5.671%); 7558/157557 objects misplaced (4.797%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:38:03.898 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:03.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:03.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:04.001 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:04.031 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:04.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:04.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:04.139 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:04.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:04.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:04.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:04.265 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:04.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:04.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:04.408 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:04.434 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:04.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:04.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:04.590 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:04.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:04.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:04.696 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:04.727 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:04.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:04.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:04.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:04.855 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:04.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:04.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:04.961 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:04.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:04.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:05.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:05.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:05.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:05.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:05.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:05.272 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:05.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:05.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:05.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:05.415 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:05.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:05.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:05.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:05.561 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:05.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:05.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:05.667 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:05.705 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:05.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:05.791 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:05 smithi181 bash[59345]: cluster 2023-10-09T09:38:03.900513+0000 mgr.y (mgr.34107) 958 : cluster 0 pgmap v1712: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 8935/155796 objects degraded (5.735%); 7358/155796 objects misplaced (4.723%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:38:05.791 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:05 smithi181 bash[59345]: cluster 2023-10-09T09:38:05.696196+0000 mon.a (mon.0) 1863 : cluster 3 Health check update: Degraded data redundancy: 8935/155796 objects degraded (5.735%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:05.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:05.835 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:05.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:05.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:05.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:05.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:05 smithi163 bash[78107]: cluster 2023-10-09T09:38:03.900513+0000 mgr.y (mgr.34107) 958 : cluster 0 pgmap v1712: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 8935/155796 objects degraded (5.735%); 7358/155796 objects misplaced (4.723%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:38:05.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:05 smithi163 bash[78107]: cluster 2023-10-09T09:38:05.696196+0000 mon.a (mon.0) 1863 : cluster 3 Health check update: Degraded data redundancy: 8935/155796 objects degraded (5.735%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:05.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:05 smithi163 bash[75425]: cluster 2023-10-09T09:38:03.900513+0000 mgr.y (mgr.34107) 958 : cluster 0 pgmap v1712: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 8935/155796 objects degraded (5.735%); 7358/155796 objects misplaced (4.723%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:38:05.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:05 smithi163 bash[75425]: cluster 2023-10-09T09:38:05.696196+0000 mon.a (mon.0) 1863 : cluster 3 Health check update: Degraded data redundancy: 8935/155796 objects degraded (5.735%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:05.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:05.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:06.096 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:06.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:06.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:06.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:06.235 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:06.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:06.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:06.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:06.370 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:06.405 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:06.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:06.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:06.535 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:06.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:06.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:06.641 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:06.679 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:06.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:06.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:06.783 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:06.807 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:06.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:06.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:06.854 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:38:06.855 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:38:06.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:06.945 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:06.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:06.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:07.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:07.074 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:07.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:07.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:07.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:07.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:07.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:07.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:07.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:07.338 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:07.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:07.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:07.443 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:07.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:07.473 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:07.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:07.574 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:07.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:07.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:07.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:07.720 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:07.745 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:07.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:07.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:07.850 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:07.859 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:07 smithi181 bash[59345]: cluster 2023-10-09T09:38:05.901093+0000 mgr.y (mgr.34107) 959 : cluster 0 pgmap v1713: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 8935/152523 objects degraded (5.858%); 6960/152523 objects misplaced (4.563%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T09:38:07.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:07.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:07.980 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:08.017 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:08.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:08.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:08.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:08.131 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:07 smithi163 bash[78107]: cluster 2023-10-09T09:38:05.901093+0000 mgr.y (mgr.34107) 959 : cluster 0 pgmap v1713: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 8935/152523 objects degraded (5.858%); 6960/152523 objects misplaced (4.563%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T09:38:08.131 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:07 smithi163 bash[75425]: cluster 2023-10-09T09:38:05.901093+0000 mgr.y (mgr.34107) 959 : cluster 0 pgmap v1713: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 8935/152523 objects degraded (5.858%); 6960/152523 objects misplaced (4.563%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T09:38:08.149 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:08.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:08.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:08.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:08.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:08.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:08.387 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:08.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:08.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:08.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:08.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:08.533 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:08.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:08.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:08.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:08.669 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:08.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:08.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:08.777 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:08.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:08.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:08.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:08.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:08.948 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:08.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:09.053 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:09.083 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:09.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:09.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:09.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:09.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:09.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:09.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:09.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:09.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:09.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:09.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:09.442 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:09.471 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:09.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:09.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:09.610 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:09.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:09.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:09.714 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:09.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:09.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:09.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:09.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:09.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:09.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:09.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:09.972 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:09.980 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:09 smithi181 bash[59345]: cluster 2023-10-09T09:38:07.901846+0000 mgr.y (mgr.34107) 960 : cluster 0 pgmap v1714: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 8935/145929 objects degraded (6.123%); 6741/145929 objects misplaced (4.619%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:38:09.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:10.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:10.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:10.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:10.109 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:09 smithi163 bash[78107]: cluster 2023-10-09T09:38:07.901846+0000 mgr.y (mgr.34107) 960 : cluster 0 pgmap v1714: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 8935/145929 objects degraded (6.123%); 6741/145929 objects misplaced (4.619%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:38:10.109 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:09 smithi163 bash[75425]: cluster 2023-10-09T09:38:07.901846+0000 mgr.y (mgr.34107) 960 : cluster 0 pgmap v1714: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 8935/145929 objects degraded (6.123%); 6741/145929 objects misplaced (4.619%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:38:10.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:10.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:10.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:10.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:10.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:10.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:10.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:10.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T09:38:10.395 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:10.427 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:10.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:10.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:10.532 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:10.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:10.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:10.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:10.668 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:10.701 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:10.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:10.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:10.805 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:10.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:10.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:10.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:10.961 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:10.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:10 smithi181 bash[59345]: audit 2023-10-09T09:38:10.380218+0000 mon.a (mon.0) 1864 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:11.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:10 smithi181 bash[59345]: cluster 2023-10-09T09:38:10.697342+0000 mon.a (mon.0) 1865 : cluster 3 Health check update: Degraded data redundancy: 8935/145929 objects degraded (6.123%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:11.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:11.098 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:11.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:11.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:11.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:11.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:10 smithi163 bash[75425]: audit 2023-10-09T09:38:10.380218+0000 mon.a (mon.0) 1864 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:11.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:10 smithi163 bash[75425]: cluster 2023-10-09T09:38:10.697342+0000 mon.a (mon.0) 1865 : cluster 3 Health check update: Degraded data redundancy: 8935/145929 objects degraded (6.123%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:11.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:10 smithi163 bash[78107]: audit 2023-10-09T09:38:10.380218+0000 mon.a (mon.0) 1864 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:11.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:10 smithi163 bash[78107]: cluster 2023-10-09T09:38:10.697342+0000 mon.a (mon.0) 1865 : cluster 3 Health check update: Degraded data redundancy: 8935/145929 objects degraded (6.123%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:11.237 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:11.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:11.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:11.344 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:11.370 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:11.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:11.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:11.475 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:11.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:11.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:11.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:11.600 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:11.626 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:11.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:11.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:11.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:11.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:11.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:11.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:11.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:11.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:11.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:11.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:11.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:12.002 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:11 smithi163 bash[78107]: cluster 2023-10-09T09:38:09.902469+0000 mgr.y (mgr.34107) 961 : cluster 0 pgmap v1715: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 8935/142815 objects degraded (6.256%); 6313/142815 objects misplaced (4.420%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T09:38:12.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:11 smithi163 bash[75425]: cluster 2023-10-09T09:38:09.902469+0000 mgr.y (mgr.34107) 961 : cluster 0 pgmap v1715: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 8935/142815 objects degraded (6.256%); 6313/142815 objects misplaced (4.420%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T09:38:12.017 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:12.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:12.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:12.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:11 smithi181 bash[59345]: cluster 2023-10-09T09:38:09.902469+0000 mgr.y (mgr.34107) 961 : cluster 0 pgmap v1715: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 8935/142815 objects degraded (6.256%); 6313/142815 objects misplaced (4.420%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T09:38:12.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:12.149 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:12.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:12.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:12.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:12.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:12.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:12.391 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:12.415 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:12.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:12.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:12.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:12.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:12.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:12.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:12.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:12.682 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:12.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:12.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:12.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:12.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:12.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:12.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:12.966 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:12.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:12.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:13.073 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:13.101 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:13.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:13.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:13.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:13.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:13.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:13.347 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:13.378 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:13.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:13.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:13.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:13.521 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:13.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:13.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:13.624 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:13.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:13.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:13.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:13.716 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:38:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:38:13] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.33.4" 2023-10-09T09:38:13.764 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:13.771 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:38:13.797 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:13.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:13.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:13.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:13 smithi181 bash[59345]: cluster 2023-10-09T09:38:11.903206+0000 mgr.y (mgr.34107) 962 : cluster 0 pgmap v1716: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8935/137835 objects degraded (6.482%); 6313/137835 objects misplaced (4.580%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T09:38:13.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:13 smithi181 bash[59345]: audit 2023-10-09T09:38:13.718622+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.163:0/3977112973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:38:13.850 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:38:13.903 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:13.919 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:38:13.939 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:13.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:13.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:13.956 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:38:13.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:13 smithi163 bash[78107]: cluster 2023-10-09T09:38:11.903206+0000 mgr.y (mgr.34107) 962 : cluster 0 pgmap v1716: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8935/137835 objects degraded (6.482%); 6313/137835 objects misplaced (4.580%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T09:38:13.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:13 smithi163 bash[78107]: audit 2023-10-09T09:38:13.718622+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.163:0/3977112973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:38:13.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:13 smithi163 bash[75425]: cluster 2023-10-09T09:38:11.903206+0000 mgr.y (mgr.34107) 962 : cluster 0 pgmap v1716: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8935/137835 objects degraded (6.482%); 6313/137835 objects misplaced (4.580%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T09:38:13.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:13 smithi163 bash[75425]: audit 2023-10-09T09:38:13.718622+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.163:0/3977112973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:38:14.001 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:38:14.038 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:38:14.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:14.054 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:38:14.054 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "" 2023-10-09T09:38:14.054 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:38:14.074 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:14.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:14.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:14.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:38:14.150 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:38:14.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:38:14.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:38:14.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:38:14.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:38:14.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:38:14.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:38:14.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:38:14.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:38:14.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:14.215 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:14.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:14.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:14.320 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:14.365 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:14.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:14.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:14.470 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:14.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:14.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:14.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:14.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:14.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:14.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:14.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:14.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:14.760 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:14.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:14.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:14.841 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:38:14.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:14.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:14.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:14.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:15.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:15.050 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:15.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:15.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:15.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:14 smithi181 bash[59345]: audit 2023-10-09T09:38:14.415661+0000 mon.a (mon.0) 1866 : audit 1 from='client.? 172.21.15.163:0/3844938146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:38:15.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:15.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:14 smithi163 bash[75425]: audit 2023-10-09T09:38:14.415661+0000 mon.a (mon.0) 1866 : audit 1 from='client.? 172.21.15.163:0/3844938146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:38:15.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:14 smithi163 bash[78107]: audit 2023-10-09T09:38:14.415661+0000 mon.a (mon.0) 1866 : audit 1 from='client.? 172.21.15.163:0/3844938146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:38:15.181 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:15.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:15.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:15.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:15.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:15.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:15.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:15.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:15.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:15.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:15.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:15.543 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:15.569 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:15.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:15.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:15.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:15.701 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:15.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:15.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:15.804 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:15.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:15.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:15.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:15.950 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:15.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:15 smithi181 bash[59345]: cluster 2023-10-09T09:38:13.903843+0000 mgr.y (mgr.34107) 963 : cluster 0 pgmap v1717: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 8935/135990 objects degraded (6.570%); 6090/135990 objects misplaced (4.478%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T09:38:15.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:15 smithi181 bash[59345]: cluster 2023-10-09T09:38:14.812728+0000 mon.a (mon.0) 1867 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:38:15.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:15 smithi181 bash[59345]: cluster 2023-10-09T09:38:14.827488+0000 osd.5 (osd.5) 721 : cluster 3 failed to encode map e1155 with expected crc 2023-10-09T09:38:15.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:15 smithi181 bash[59345]: audit 2023-10-09T09:38:14.830003+0000 mon.a (mon.0) 1868 : audit 1 from='client.? 172.21.15.163:0/3844938146' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:38:15.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:15 smithi181 bash[59345]: cluster 2023-10-09T09:38:14.830086+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e1155: 8 total, 8 up, 8 in 2023-10-09T09:38:15.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:15 smithi181 bash[59345]: cluster 2023-10-09T09:38:14.836343+0000 osd.6 (osd.6) 435 : cluster 3 failed to encode map e1155 with expected crc 2023-10-09T09:38:15.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:15 smithi181 bash[59345]: cluster 2023-10-09T09:38:14.838470+0000 osd.7 (osd.7) 550 : cluster 3 failed to encode map e1155 with expected crc 2023-10-09T09:38:15.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:15 smithi181 bash[59345]: cluster 2023-10-09T09:38:15.698566+0000 mon.a (mon.0) 1870 : cluster 3 Health check update: Degraded data redundancy: 8935/135990 objects degraded (6.570%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:15.978 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:15.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:15.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:16.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:16.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:16.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:16.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:16.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:16.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[75425]: cluster 2023-10-09T09:38:13.903843+0000 mgr.y (mgr.34107) 963 : cluster 0 pgmap v1717: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 8935/135990 objects degraded (6.570%); 6090/135990 objects misplaced (4.478%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T09:38:16.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[75425]: cluster 2023-10-09T09:38:14.812728+0000 mon.a (mon.0) 1867 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:38:16.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[75425]: cluster 2023-10-09T09:38:14.827488+0000 osd.5 (osd.5) 721 : cluster 3 failed to encode map e1155 with expected crc 2023-10-09T09:38:16.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[75425]: audit 2023-10-09T09:38:14.830003+0000 mon.a (mon.0) 1868 : audit 1 from='client.? 172.21.15.163:0/3844938146' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:38:16.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[75425]: cluster 2023-10-09T09:38:14.830086+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e1155: 8 total, 8 up, 8 in 2023-10-09T09:38:16.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[75425]: cluster 2023-10-09T09:38:14.836343+0000 osd.6 (osd.6) 435 : cluster 3 failed to encode map e1155 with expected crc 2023-10-09T09:38:16.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[75425]: cluster 2023-10-09T09:38:14.838470+0000 osd.7 (osd.7) 550 : cluster 3 failed to encode map e1155 with expected crc 2023-10-09T09:38:16.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[75425]: cluster 2023-10-09T09:38:15.698566+0000 mon.a (mon.0) 1870 : cluster 3 Health check update: Degraded data redundancy: 8935/135990 objects degraded (6.570%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:16.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[78107]: cluster 2023-10-09T09:38:13.903843+0000 mgr.y (mgr.34107) 963 : cluster 0 pgmap v1717: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 8935/135990 objects degraded (6.570%); 6090/135990 objects misplaced (4.478%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T09:38:16.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[78107]: cluster 2023-10-09T09:38:14.812728+0000 mon.a (mon.0) 1867 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:38:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[78107]: cluster 2023-10-09T09:38:14.827488+0000 osd.5 (osd.5) 721 : cluster 3 failed to encode map e1155 with expected crc 2023-10-09T09:38:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[78107]: audit 2023-10-09T09:38:14.830003+0000 mon.a (mon.0) 1868 : audit 1 from='client.? 172.21.15.163:0/3844938146' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:38:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[78107]: cluster 2023-10-09T09:38:14.830086+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e1155: 8 total, 8 up, 8 in 2023-10-09T09:38:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[78107]: cluster 2023-10-09T09:38:14.836343+0000 osd.6 (osd.6) 435 : cluster 3 failed to encode map e1155 with expected crc 2023-10-09T09:38:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[78107]: cluster 2023-10-09T09:38:14.838470+0000 osd.7 (osd.7) 550 : cluster 3 failed to encode map e1155 with expected crc 2023-10-09T09:38:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:15 smithi163 bash[78107]: cluster 2023-10-09T09:38:15.698566+0000 mon.a (mon.0) 1870 : cluster 3 Health check update: Degraded data redundancy: 8935/135990 objects degraded (6.570%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:16.240 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:16.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:16.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:16.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:16.384 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:16.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:16.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:16.487 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:16.516 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:16.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:16.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:16.619 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:16.652 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:16.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:16.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:16.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:16.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:16.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:16.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:16.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:16.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:16.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:17.062 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:17.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:17.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:17.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:17.193 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:17.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:17.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:17.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:17.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:17.351 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:38:17.380 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:17.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:17.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:17.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:17.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:17.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:17.632 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:17.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:17.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:17.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:17.780 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:17.805 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:17.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:17.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:17.908 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:17.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:17 smithi163 bash[75425]: cluster 2023-10-09T09:38:15.904474+0000 mgr.y (mgr.34107) 964 : cluster 0 pgmap v1719: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 8935/132879 objects degraded (6.724%); 5675/132879 objects misplaced (4.271%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T09:38:17.918 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:17 smithi163 bash[78107]: cluster 2023-10-09T09:38:15.904474+0000 mgr.y (mgr.34107) 964 : cluster 0 pgmap v1719: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 8935/132879 objects degraded (6.724%); 5675/132879 objects misplaced (4.271%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T09:38:17.933 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:17.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:17.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:18.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:18.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:17 smithi181 bash[59345]: cluster 2023-10-09T09:38:15.904474+0000 mgr.y (mgr.34107) 964 : cluster 0 pgmap v1719: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 8935/132879 objects degraded (6.724%); 5675/132879 objects misplaced (4.271%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T09:38:18.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:18.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:18.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:18.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:18.212 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:18.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:18.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:18.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:18.349 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:18.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:18.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:18.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:18.479 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:18.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:18.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:18.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:18.611 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:18.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:18.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:18.714 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:18.744 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:18.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:18.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:18.847 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:18.855 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:38:18.877 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:18.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:18.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:18.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:38:18.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:18.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:18 smithi163 bash[75425]: cluster 2023-10-09T09:38:17.843966+0000 osd.5 (osd.5) 722 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:18.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:18 smithi163 bash[75425]: cluster 2023-10-09T09:38:17.852256+0000 mon.a (mon.0) 1871 : cluster 0 osdmap e1156: 8 total, 8 up, 8 in 2023-10-09T09:38:18.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:18 smithi163 bash[75425]: cluster 2023-10-09T09:38:17.869356+0000 osd.7 (osd.7) 551 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:18.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:18 smithi163 bash[75425]: cluster 2023-10-09T09:38:17.869645+0000 osd.4 (osd.4) 576 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:18.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:18 smithi163 bash[75425]: cluster 2023-10-09T09:38:17.871348+0000 osd.4 (osd.4) 577 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:18.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:18 smithi163 bash[78107]: cluster 2023-10-09T09:38:17.843966+0000 osd.5 (osd.5) 722 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:18.993 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:18 smithi163 bash[78107]: cluster 2023-10-09T09:38:17.852256+0000 mon.a (mon.0) 1871 : cluster 0 osdmap e1156: 8 total, 8 up, 8 in 2023-10-09T09:38:18.993 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:18 smithi163 bash[78107]: cluster 2023-10-09T09:38:17.869356+0000 osd.7 (osd.7) 551 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:18.993 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:18 smithi163 bash[78107]: cluster 2023-10-09T09:38:17.869645+0000 osd.4 (osd.4) 576 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:18.993 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:18 smithi163 bash[78107]: cluster 2023-10-09T09:38:17.871348+0000 osd.4 (osd.4) 577 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:19.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:19.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:19.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:19.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:19.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:18 smithi181 bash[59345]: cluster 2023-10-09T09:38:17.843966+0000 osd.5 (osd.5) 722 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:19.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:18 smithi181 bash[59345]: cluster 2023-10-09T09:38:17.852256+0000 mon.a (mon.0) 1871 : cluster 0 osdmap e1156: 8 total, 8 up, 8 in 2023-10-09T09:38:19.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:18 smithi181 bash[59345]: cluster 2023-10-09T09:38:17.869356+0000 osd.7 (osd.7) 551 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:19.138 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:18 smithi181 bash[59345]: cluster 2023-10-09T09:38:17.869645+0000 osd.4 (osd.4) 576 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:19.138 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:18 smithi181 bash[59345]: cluster 2023-10-09T09:38:17.871348+0000 osd.4 (osd.4) 577 : cluster 3 failed to encode map e1156 with expected crc 2023-10-09T09:38:19.148 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:19.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:19.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:19.255 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:19.285 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:19.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:19.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:19.391 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:19.424 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:19.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:19.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:19.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:19.554 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:19.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:19.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:19.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:19.686 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:19.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:19.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:19.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:19.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:19.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:19.925 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:19.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:19 smithi181 bash[59345]: cluster 2023-10-09T09:38:17.905036+0000 mgr.y (mgr.34107) 965 : cluster 0 pgmap v1721: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8935/126330 objects degraded (7.073%); 5482/126330 objects misplaced (4.339%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:38:19.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:19 smithi181 bash[59345]: cluster 2023-10-09T09:38:18.855545+0000 osd.6 (osd.6) 436 : cluster 3 failed to encode map e1157 with expected crc 2023-10-09T09:38:19.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:19 smithi181 bash[59345]: cluster 2023-10-09T09:38:18.855562+0000 osd.5 (osd.5) 723 : cluster 3 failed to encode map e1157 with expected crc 2023-10-09T09:38:19.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:19 smithi181 bash[59345]: cluster 2023-10-09T09:38:18.860660+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e1157: 8 total, 8 up, 8 in 2023-10-09T09:38:19.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:19 smithi181 bash[59345]: cluster 2023-10-09T09:38:18.895248+0000 osd.4 (osd.4) 578 : cluster 3 failed to encode map e1157 with expected crc 2023-10-09T09:38:19.953 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:19.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:19.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:20.058 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:20.093 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:20.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:20.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:20.200 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:20.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:19 smithi163 bash[75425]: cluster 2023-10-09T09:38:17.905036+0000 mgr.y (mgr.34107) 965 : cluster 0 pgmap v1721: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8935/126330 objects degraded (7.073%); 5482/126330 objects misplaced (4.339%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:38:20.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:19 smithi163 bash[75425]: cluster 2023-10-09T09:38:18.855545+0000 osd.6 (osd.6) 436 : cluster 3 failed to encode map e1157 with expected crc 2023-10-09T09:38:20.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:19 smithi163 bash[75425]: cluster 2023-10-09T09:38:18.855562+0000 osd.5 (osd.5) 723 : cluster 3 failed to encode map e1157 with expected crc 2023-10-09T09:38:20.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:19 smithi163 bash[75425]: cluster 2023-10-09T09:38:18.860660+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e1157: 8 total, 8 up, 8 in 2023-10-09T09:38:20.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:19 smithi163 bash[75425]: cluster 2023-10-09T09:38:18.895248+0000 osd.4 (osd.4) 578 : cluster 3 failed to encode map e1157 with expected crc 2023-10-09T09:38:20.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:19 smithi163 bash[78107]: cluster 2023-10-09T09:38:17.905036+0000 mgr.y (mgr.34107) 965 : cluster 0 pgmap v1721: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8935/126330 objects degraded (7.073%); 5482/126330 objects misplaced (4.339%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T09:38:20.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:19 smithi163 bash[78107]: cluster 2023-10-09T09:38:18.855545+0000 osd.6 (osd.6) 436 : cluster 3 failed to encode map e1157 with expected crc 2023-10-09T09:38:20.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:19 smithi163 bash[78107]: cluster 2023-10-09T09:38:18.855562+0000 osd.5 (osd.5) 723 : cluster 3 failed to encode map e1157 with expected crc 2023-10-09T09:38:20.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:19 smithi163 bash[78107]: cluster 2023-10-09T09:38:18.860660+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e1157: 8 total, 8 up, 8 in 2023-10-09T09:38:20.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:19 smithi163 bash[78107]: cluster 2023-10-09T09:38:18.895248+0000 osd.4 (osd.4) 578 : cluster 3 failed to encode map e1157 with expected crc 2023-10-09T09:38:20.227 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:20.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:20.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:20.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:20.366 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:20.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:20.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:20.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:20.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:20.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:20.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:20.630 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:20.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:20.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:20.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:20.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:20.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:20.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:20.869 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:20 smithi181 bash[59345]: cluster 2023-10-09T09:38:20.699843+0000 mon.a (mon.0) 1873 : cluster 3 Health check update: Degraded data redundancy: 8935/126330 objects degraded (7.073%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:20.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:20.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:20.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:20.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:21.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:21.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:21.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:21.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:21.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:21.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:21.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:21.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:21.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:20 smithi163 bash[75425]: cluster 2023-10-09T09:38:20.699843+0000 mon.a (mon.0) 1873 : cluster 3 Health check update: Degraded data redundancy: 8935/126330 objects degraded (7.073%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:21.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:20 smithi163 bash[78107]: cluster 2023-10-09T09:38:20.699843+0000 mon.a (mon.0) 1873 : cluster 3 Health check update: Degraded data redundancy: 8935/126330 objects degraded (7.073%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:21.285 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:21.334 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:21.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:21.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:21.481 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:21.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:21.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:21.538 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38750 8.1 8:81be6a0d:::benchmark_data_smithi163_151167_object38749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1139)", 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.671072+0000", 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.86215264099999, 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1217603409999999, 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:38:21.539 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38750 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.671072+0000", 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.540 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.671072+0000", 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.671072+0000", 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.541 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.671129+0000", 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7200000000000001e-05 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.671138+0000", 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3840000000000003e-06 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.671146+0000", 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.729e-06 2023-10-09T09:38:21.542 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.674492+0000", 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003345649 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.674704+0000", 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021240199999999999 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.543 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.674852+0000", 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014833700000000001 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.682449+0000", 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.007596528 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.544 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.682628+0000", 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017955599999999999 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.698343+0000", 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015714466999999999 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.698355+0000", 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2649e-05 2023-10-09T09:38:21.545 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.705937+0000", 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0075818830000001 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.705954+0000", 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6381000000000001e-05 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.546 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.792689+0000", 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0867352880000001 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.792833+0000", 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014358400000000001 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.547 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38760 8.a 8:5de1a095:::benchmark_data_smithi163_151167_object38759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1139)", 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.678757+0000", 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.854468376, 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1044338119999999, 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.548 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38760 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.678757+0000", 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.678757+0000", 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.549 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.678756+0000", 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.678778+0000", 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2024000000000001e-05 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.550 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.678787+0000", 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3850000000000008e-06 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.678794+0000", 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5039999999999999e-06 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.678915+0000", 2023-10-09T09:38:21.551 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012123199999999999 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.679085+0000", 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017028400000000001 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,5", 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.679355+0000", 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00027001800000000002 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.552 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.553 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.682091+0000", 2023-10-09T09:38:21.553 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002735243 2023-10-09T09:38:21.553 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.553 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.553 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.553 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.697989+0000", 2023-10-09T09:38:21.553 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015898874 2023-10-09T09:38:21.553 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.698001+0000", 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1311000000000001e-05 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.693523+0000", 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99552225299999997 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.554 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.693541+0000", 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7592e-05 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.782954+0000", 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0894136590000001 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.783190+0000", 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023613700000000001 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38774 8.a 8:5622e034:::benchmark_data_smithi163_151167_object38773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.703776+0000", 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.829449267, 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0933209320000001, 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.556 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38774 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.703776+0000", 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.557 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.703776+0000", 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.703774+0000", 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.703803+0000", 2023-10-09T09:38:21.558 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8623e-05 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.703811+0000", 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9889999999999992e-06 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.703819+0000", 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2290000000000003e-06 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.559 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704571+0000", 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00075216400000000004 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704581+0000", 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4050000000000006e-06 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.560 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.693572+0000", 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.988990971 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.693581+0000", 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5000000000000005e-06 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.785127+0000", 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.091545805 2023-10-09T09:38:21.561 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.785328+0000", 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020119100000000001 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.785565+0000", 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023715699999999999 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.790162+0000", 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045968329999999998 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.792271+0000", 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002108562 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.795241+0000", 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029701419999999998 2023-10-09T09:38:21.563 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.795304+0000", 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3317000000000004e-05 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.797097+0000", 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017923069999999999 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.564 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38776 8.a 8:57e7f7a8:::benchmark_data_smithi163_151167_object38775:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.704710+0000", 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.82851536000001, 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0952255420000001, 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.565 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38776 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704710+0000", 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704710+0000", 2023-10-09T09:38:21.566 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704709+0000", 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704731+0000", 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2552999999999999e-05 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.567 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704740+0000", 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0000000000000002e-06 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704748+0000", 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1969999999999996e-06 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704904+0000", 2023-10-09T09:38:21.568 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015584400000000001 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704913+0000", 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5459999999999992e-06 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.693608+0000", 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98869478600000005 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.569 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.693616+0000", 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5169999999999999e-06 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.787968+0000", 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.094351442 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.570 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.788214+0000", 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000246047 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.788422+0000", 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020810300000000001 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.790385+0000", 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001963247 2023-10-09T09:38:21.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.792291+0000", 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00190566 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.797154+0000", 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048634469999999999 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.572 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.797216+0000", 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1482999999999994e-05 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.799935+0000", 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027195259999999999 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.573 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38778 8.1 8:807a0b8c:::benchmark_data_smithi163_151167_object38777:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.704759+0000", 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.828465833, 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1027932140000001, 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38778 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.574 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704759+0000", 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704759+0000", 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704759+0000", 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704780+0000", 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1874000000000001e-05 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704787+0000", 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7449999999999998e-06 2023-10-09T09:38:21.576 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704792+0000", 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1510000000000003e-06 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704937+0000", 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014483899999999999 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.577 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.704947+0000", 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7659999999999993e-06 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.705977+0000", 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0010298660000001 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.705986+0000", 2023-10-09T09:38:21.578 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2709999999999998e-06 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.794761+0000", 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0887748989999999 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.794942+0000", 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018115300000000001 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.579 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.795143+0000", 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020113600000000001 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.800563+0000", 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054197230000000004 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.580 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.802627+0000", 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020637469999999999 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.805163+0000", 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0025360270000000002 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.805229+0000", 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6118000000000006e-05 2023-10-09T09:38:21.581 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.807552+0000", 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023234509999999998 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38781 8.1 8:8f52d414:::benchmark_data_smithi163_151167_object38780:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.708202+0000", 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.825023214, 2023-10-09T09:38:21.582 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1015419949999998, 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38781 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.583 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708202+0000", 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708202+0000", 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708200+0000", 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.584 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708226+0000", 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.65e-05 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708242+0000", 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5214999999999999e-05 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708249+0000", 2023-10-09T09:38:21.585 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4699999999999996e-06 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708325+0000", 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5729999999999995e-05 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708338+0000", 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3040999999999999e-05 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.706079+0000", 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99774136899999999 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.706090+0000", 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1249e-05 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.587 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.796743+0000", 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0906524790000001 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.796872+0000", 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012909999999999999 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.797034+0000", 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016227900000000001 2023-10-09T09:38:21.588 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.802592+0000", 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055572369999999996 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.807600+0000", 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050087400000000002 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.589 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.807909+0000", 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00030867699999999999 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.807973+0000", 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4073999999999997e-05 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.809744+0000", 2023-10-09T09:38:21.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001770621 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38788 8.1 8:8cdf30ba:::benchmark_data_smithi163_151167_object38787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.711143+0000", 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.82208186599999, 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1005076599999999, 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.591 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38788 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711143+0000", 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.592 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711143+0000", 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711141+0000", 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.593 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711169+0000", 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8022e-05 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711186+0000", 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6665999999999999e-05 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711208+0000", 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2251000000000001e-05 2023-10-09T09:38:21.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711562+0000", 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00035371900000000001 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711572+0000", 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9059999999999997e-06 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.595 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.706112+0000", 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99454023499999999 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.706119+0000", 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9380000000000003e-06 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.798671+0000", 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.092552097 2023-10-09T09:38:21.596 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.798823+0000", 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015225500000000001 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.798990+0000", 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016700499999999999 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.597 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.802635+0000", 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036444910000000001 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.807639+0000", 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050040249999999996 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.809815+0000", 2023-10-09T09:38:21.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021762209999999999 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.809870+0000", 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4502000000000001e-05 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.811651+0000", 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001780982 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.599 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38789 8.a 8:52069eb1:::benchmark_data_smithi163_151167_object38788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.711222+0000", 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.822002446, 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0909839130000001, 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.600 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38789 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711222+0000", 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711222+0000", 2023-10-09T09:38:21.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711222+0000", 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711244+0000", 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2526000000000001e-05 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711254+0000", 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8479999999999996e-06 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711261+0000", 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5459999999999997e-06 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711333+0000", 2023-10-09T09:38:21.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2502999999999993e-05 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.711346+0000", 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2877999999999999e-05 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.693641+0000", 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98229500599999997 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.693649+0000", 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9810000000000003e-06 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.790435+0000", 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0967855440000001 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.790590+0000", 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000154797 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.790854+0000", 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026469200000000002 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.792396+0000", 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015420939999999999 2023-10-09T09:38:21.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.792599+0000", 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020254 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.800035+0000", 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0074359250000000003 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.607 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.800239+0000", 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020424700000000001 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.802206+0000", 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019673690000000001 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.608 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38790 8.f 8:fa0c26af:::benchmark_data_smithi163_151167_object38789:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.712456+0000", 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.82076859399999, 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0909704900000001, 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38790 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712456+0000", 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712456+0000", 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712455+0000", 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712491+0000", 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5667000000000003e-05 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712503+0000", 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.198e-05 2023-10-09T09:38:21.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712509+0000", 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8719999999999999e-06 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712901+0000", 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000392679 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.713002+0000", 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010058800000000001 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.694600+0000", 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98159855200000001 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.694612+0000", 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1783000000000001e-05 2023-10-09T09:38:21.613 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.789440+0000", 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.094827467 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.790453+0000", 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001013105 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:38:21.614 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.790619+0000", 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000166176 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.796669+0000", 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060496550000000001 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.799501+0000", 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028318420000000002 2023-10-09T09:38:21.615 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.801555+0000", 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020543240000000002 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.801596+0000", 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0672e-05 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.616 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.803427+0000", 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001831278 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38794 8.f 8:f2d2dc12:::benchmark_data_smithi163_151167_object38793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.717875+0000", 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.81535039900001, 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0929777719999998, 2023-10-09T09:38:21.617 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38794 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.717875+0000", 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.717875+0000", 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.717873+0000", 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.717896+0000", 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2918999999999999e-05 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.717909+0000", 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2784e-05 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.717915+0000", 2023-10-09T09:38:21.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5320000000000003e-06 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.717964+0000", 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.833e-05 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.717975+0000", 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1049e-05 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.694823+0000", 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97684795199999996 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.694835+0000", 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2561999999999999e-05 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.794578+0000", 2023-10-09T09:38:21.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.099742947 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.794720+0000", 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014192799999999999 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.794936+0000", 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021590199999999999 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.623 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.801524+0000", 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065883089999999997 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.803820+0000", 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022953610000000001 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.624 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.809090+0000", 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052704659999999997 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.809127+0000", 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7017999999999997e-05 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.810852+0000", 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001725201 2023-10-09T09:38:21.625 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85402 8.f 8:f616f4b2:::benchmark_data_smithi163_151167_object85401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:42.368041+0000", 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 99.165183972999998, 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.202803992, 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.626 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85402 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.368041+0000", 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.368041+0000", 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.368039+0000", 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.368067+0000", 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7532999999999999e-05 2023-10-09T09:38:21.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.368493+0000", 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000426379 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.368505+0000", 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.187e-05 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.369549+0000", 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010436519999999999 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.369682+0000", 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000133549 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.370029+0000", 2023-10-09T09:38:21.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00034620899999999999 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.375759+0000", 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0057299860000000003 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.375930+0000", 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017139600000000001 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.407435+0000", 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031505182 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.407448+0000", 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2864000000000001e-05 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.570662+0000", 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1632143189999999 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.570845+0000", 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018249999999999999 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.633 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85412 8.f 8:fdf3320b:::benchmark_data_smithi163_151167_object85411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:42.377864+0000", 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 99.155360836, 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2002023129999999, 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85412 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.634 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.377864+0000", 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.377864+0000", 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.377863+0000", 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.377898+0000", 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4675000000000003e-05 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.377909+0000", 2023-10-09T09:38:21.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0728e-05 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.377915+0000", 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2179999999999996e-06 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.380255+0000", 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023404770000000001 2023-10-09T09:38:21.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.380436+0000", 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000180602 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.380632+0000", 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019615299999999999 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.387478+0000", 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0068462369999999998 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.387528+0000", 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9935999999999998e-05 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.407652+0000", 2023-10-09T09:38:21.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020123657 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.407663+0000", 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1365e-05 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.574224+0000", 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.16656114 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.578066+0000", 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038420389999999998 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85414 8.a 8:5bdbac66:::benchmark_data_smithi163_151167_object85413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:42.384542+0000", 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 99.148682401000002, 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4481604219999999, 2023-10-09T09:38:21.641 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85414 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.384542+0000", 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.384542+0000", 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.384541+0000", 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.384568+0000", 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6967000000000001e-05 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.384581+0000", 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3009e-05 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.384588+0000", 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8889999999999996e-06 2023-10-09T09:38:21.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.384644+0000", 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6073999999999999e-05 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.384803+0000", 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015891700000000001 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.384961+0000", 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000158698 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.390361+0000", 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053998839999999998 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.390505+0000", 2023-10-09T09:38:21.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014385299999999999 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.406066+0000", 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015561264999999999 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.406099+0000", 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3209e-05 2023-10-09T09:38:21.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.401841+0000", 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.995741074 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.401859+0000", 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8525e-05 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.832643+0000", 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.430783814 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.832703+0000", 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0016999999999998e-05 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.649 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85425 8.a 8:5f5d42b6:::benchmark_data_smithi163_151167_object85424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1149)", 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:42.410530+0000", 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 99.122694417999995, 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4322813500000002, 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85425 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.410530+0000", 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.410530+0000", 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.410529+0000", 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.410559+0000", 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9949000000000001e-05 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.410574+0000", 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5223e-05 2023-10-09T09:38:21.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.410582+0000", 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6309999999999992e-06 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.410630+0000", 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7135000000000002e-05 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.410643+0000", 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2924e-05 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.401986+0000", 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99134339800000004 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.401996+0000", 2023-10-09T09:38:21.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0403e-05 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.832954+0000", 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.430958092 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.833119+0000", 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016475300000000001 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.833241+0000", 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121582 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.842526+0000", 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0092857459999999992 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.842589+0000", 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2209e-05 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.842642+0000", 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2862000000000003e-05 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.842651+0000", 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1079999999999999e-06 2023-10-09T09:38:21.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.842704+0000", 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3043e-05 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.842812+0000", 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000108117 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.658 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85432 8.a 8:5d177c1e:::benchmark_data_smithi163_151167_object85431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1149)", 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:42.420430+0000", 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 99.112795057, 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4512769680000002, 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.659 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85432 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.420430+0000", 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.420430+0000", 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.420428+0000", 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.420460+0000", 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1948e-05 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.420475+0000", 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5056e-05 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.420486+0000", 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0791000000000001e-05 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.420541+0000", 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5532999999999998e-05 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.420554+0000", 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2875e-05 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.402023+0000", 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98146892500000005 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.402032+0000", 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7280000000000001e-06 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.835952+0000", 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4339199899999999 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.836126+0000", 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017384200000000001 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.836289+0000", 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016292100000000001 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.842828+0000", 2023-10-09T09:38:21.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065398189999999997 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.849910+0000", 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0070815089999999997 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856812+0000", 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069019119999999996 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.871461+0000", 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014649294 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.871579+0000", 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011762 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.871707+0000", 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012808700000000001 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85442 8.a 8:5add29a8:::benchmark_data_smithi163_151167_object85441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1149)", 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:42.450686+0000", 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 99.082538626000002, 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4212817859999998, 2023-10-09T09:38:21.668 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85442 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.450686+0000", 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.450686+0000", 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.450685+0000", 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.450713+0000", 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8124e-05 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.450729+0000", 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6365e-05 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.450738+0000", 2023-10-09T09:38:21.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.568e-06 2023-10-09T09:38:21.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.450790+0000", 2023-10-09T09:38:21.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2094e-05 2023-10-09T09:38:21.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:42.450801+0000", 2023-10-09T09:38:21.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:21.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1600000000000001e-05 2023-10-09T09:38:21.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.402219+0000", 2023-10-09T09:38:21.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95141798099999997 2023-10-09T09:38:21.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.402231+0000", 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1296e-05 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.839141+0000", 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4369099620000001 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.839270+0000", 2023-10-09T09:38:21.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012941 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.839392+0000", 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121817 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.842835+0000", 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00344298 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856866+0000", 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014031709999999999 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856927+0000", 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0918999999999997e-05 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.871821+0000", 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014893254999999999 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.871873+0000", 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1940000000000001e-05 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.871968+0000", 2023-10-09T09:38:21.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5449e-05 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85477 8.a 8:5072507f:::benchmark_data_smithi163_151167_object85476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1150)", 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:43.636474+0000", 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 97.896750705000002, 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2357369659999999, 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.679 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85477 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.636474+0000", 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.636474+0000", 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.636464+0000", 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999895 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.636505+0000", 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1006e-05 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.636540+0000", 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4918e-05 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.636550+0000", 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0929e-05 2023-10-09T09:38:21.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.636729+0000", 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017868300000000001 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.636746+0000", 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7093999999999999e-05 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.842863+0000", 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2061162539999999 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.843019+0000", 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015649699999999999 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.843178+0000", 2023-10-09T09:38:21.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015942199999999999 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856876+0000", 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013697106000000001 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856966+0000", 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0488000000000005e-05 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.871419+0000", 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014453086 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872062+0000", 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00064253200000000002 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872108+0000", 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6285999999999997e-05 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872211+0000", 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010323300000000001 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85486 8.a 8:50a8565e:::benchmark_data_smithi163_151167_object85485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1150)", 2023-10-09T09:38:21.687 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:43.646584+0000", 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 97.886640936999996, 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2258526510000001, 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85486 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646584+0000", 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646584+0000", 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646583+0000", 2023-10-09T09:38:21.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646610+0000", 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6503999999999999e-05 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646618+0000", 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8900000000000007e-06 2023-10-09T09:38:21.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646624+0000", 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7490000000000004e-06 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646680+0000", 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6817000000000002e-05 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646692+0000", 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1511e-05 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.846222+0000", 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.199530467 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.846392+0000", 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017011000000000001 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.846588+0000", 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019517499999999999 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856883+0000", 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010294862 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.857014+0000", 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013128400000000001 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.871787+0000", 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014772763 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872286+0000", 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00049989499999999998 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872334+0000", 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7485999999999999e-05 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872437+0000", 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010268300000000001 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85487 8.a 8:505562ae:::benchmark_data_smithi163_151167_object85486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1150)", 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:43.646821+0000", 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 97.886404275000004, 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2258836200000001, 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.696 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85487 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646821+0000", 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646821+0000", 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646820+0000", 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646862+0000", 2023-10-09T09:38:21.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1664e-05 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646879+0000", 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7532000000000001e-05 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646887+0000", 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9410000000000007e-06 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646928+0000", 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0914000000000002e-05 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.646939+0000", 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1316e-05 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.849941+0000", 2023-10-09T09:38:21.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2030015890000001 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.850101+0000", 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016027000000000001 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.850291+0000", 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000189447 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856889+0000", 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065984829999999996 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.857060+0000", 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000170767 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872033+0000", 2023-10-09T09:38:21.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014973202 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872488+0000", 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000455227 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872534+0000", 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5590000000000002e-05 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872704+0000", 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000170452 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85501 8.a 8:5fcdb291:::benchmark_data_smithi163_151167_object85500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1150)", 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:43.669337+0000", 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 97.863887621000003, 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2035748049999999, 2023-10-09T09:38:21.704 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85501 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.669337+0000", 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.669337+0000", 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.669337+0000", 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.669360+0000", 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3740999999999999e-05 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.669374+0000", 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3423e-05 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.669381+0000", 2023-10-09T09:38:21.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6850000000000001e-06 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.669451+0000", 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9845999999999995e-05 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.669463+0000", 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2211999999999999e-05 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.853315+0000", 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1838520340000001 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.853450+0000", 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013487699999999999 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.853599+0000", 2023-10-09T09:38:21.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014871 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.863879+0000", 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010280253 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.863927+0000", 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7784999999999997e-05 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872253+0000", 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0083260160000000003 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872746+0000", 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000493368 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872802+0000", 2023-10-09T09:38:21.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5788999999999997e-05 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.872912+0000", 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000109826 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:38:21.712 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:38:21.763 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:21.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:21.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:21.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:21.873 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:38:21.874 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:38:21.874 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T09:38:21.874 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.401892 2023-10-09T09:38:21.874 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0.40189232106549444 2023-10-09T09:38:21.878 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:21 smithi163 bash[75425]: cluster 2023-10-09T09:38:19.907554+0000 mgr.y (mgr.34107) 966 : cluster 0 pgmap v1723: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 8935/124014 objects degraded (7.205%); 5286/124014 objects misplaced (4.262%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:38:21.878 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:21 smithi163 bash[78107]: cluster 2023-10-09T09:38:19.907554+0000 mgr.y (mgr.34107) 966 : cluster 0 pgmap v1723: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 8935/124014 objects degraded (7.205%); 5286/124014 objects misplaced (4.262%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:38:21.910 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:21.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:21.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:22.016 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:22.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:22.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:22.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:22.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:22.219 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:22.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:22.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:21 smithi181 bash[59345]: cluster 2023-10-09T09:38:19.907554+0000 mgr.y (mgr.34107) 966 : cluster 0 pgmap v1723: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 8935/124014 objects degraded (7.205%); 5286/124014 objects misplaced (4.262%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:38:22.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:22.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:22.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:22.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:22.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:22.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:22.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:22.590 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:22.617 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:22.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:22.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:22.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:22.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:22.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:22.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:22.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T09:38:22.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:22.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:22.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:22.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:23.039 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:23.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:23.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:23.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:23.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:23.205 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:23.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:23.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:23.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:23.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:23.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:23.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:23.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:23.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:23.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:23.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:23.582 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:23.628 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:23.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:23.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:23.635 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:38:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:38:23] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.33.4" 2023-10-09T09:38:23.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:23.756 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:38:23.757 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:38:23.772 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:23.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:23.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:23.876 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:23.890 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:23 smithi163 bash[75425]: cluster 2023-10-09T09:38:21.908303+0000 mgr.y (mgr.34107) 967 : cluster 0 pgmap v1724: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7026/119178 objects degraded (5.895%); 6935/119178 objects misplaced (5.819%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:38:23.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:23 smithi163 bash[78107]: cluster 2023-10-09T09:38:21.908303+0000 mgr.y (mgr.34107) 967 : cluster 0 pgmap v1724: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7026/119178 objects degraded (5.895%); 6935/119178 objects misplaced (5.819%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:38:23.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:23.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:23.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:24.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:24.055 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:24.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:24.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:23 smithi181 bash[59345]: cluster 2023-10-09T09:38:21.908303+0000 mgr.y (mgr.34107) 967 : cluster 0 pgmap v1724: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7026/119178 objects degraded (5.895%); 6935/119178 objects misplaced (5.819%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:38:24.161 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:24.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:24.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:24.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:24.311 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:24.340 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:24.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:24.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:24.443 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:24.480 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:24.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:24.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:24.587 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:24.628 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:24.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:24.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:24.735 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:24.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:24.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:24.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:24.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:24.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:24.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:25.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:25.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:25.038 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:25.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:25.139 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:25.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:25.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:25.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:25.274 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:25.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:25.311 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:25.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:25.412 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:25.432 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:25.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:25.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:25.537 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:25.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:25.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:25.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:25.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:25.720 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:25.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:25.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:25.788 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:38:25.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:25.871 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:25.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:25.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:25.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:25 smithi163 bash[75425]: cluster 2023-10-09T09:38:23.908895+0000 mgr.y (mgr.34107) 968 : cluster 0 pgmap v1725: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 7026/117657 objects degraded (5.972%); 6777/117657 objects misplaced (5.760%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:38:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:25 smithi163 bash[75425]: audit 2023-10-09T09:38:25.385210+0000 mon.a (mon.0) 1874 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:25 smithi163 bash[75425]: audit 2023-10-09T09:38:25.386468+0000 mon.a (mon.0) 1875 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:25 smithi163 bash[75425]: cluster 2023-10-09T09:38:25.700838+0000 mon.a (mon.0) 1876 : cluster 3 Health check update: Degraded data redundancy: 7026/117657 objects degraded (5.972%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:25 smithi163 bash[78107]: cluster 2023-10-09T09:38:23.908895+0000 mgr.y (mgr.34107) 968 : cluster 0 pgmap v1725: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 7026/117657 objects degraded (5.972%); 6777/117657 objects misplaced (5.760%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:38:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:25 smithi163 bash[78107]: audit 2023-10-09T09:38:25.385210+0000 mon.a (mon.0) 1874 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:25 smithi163 bash[78107]: audit 2023-10-09T09:38:25.386468+0000 mon.a (mon.0) 1875 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:25.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:25 smithi163 bash[78107]: cluster 2023-10-09T09:38:25.700838+0000 mon.a (mon.0) 1876 : cluster 3 Health check update: Degraded data redundancy: 7026/117657 objects degraded (5.972%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:25.974 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:25.999 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:26.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:26.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:26.102 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:26.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:25 smithi181 bash[59345]: cluster 2023-10-09T09:38:23.908895+0000 mgr.y (mgr.34107) 968 : cluster 0 pgmap v1725: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 7026/117657 objects degraded (5.972%); 6777/117657 objects misplaced (5.760%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:38:26.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:25 smithi181 bash[59345]: audit 2023-10-09T09:38:25.385210+0000 mon.a (mon.0) 1874 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:26.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:25 smithi181 bash[59345]: audit 2023-10-09T09:38:25.386468+0000 mon.a (mon.0) 1875 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:26.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:25 smithi181 bash[59345]: cluster 2023-10-09T09:38:25.700838+0000 mon.a (mon.0) 1876 : cluster 3 Health check update: Degraded data redundancy: 7026/117657 objects degraded (5.972%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:26.129 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:26.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:26.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:26.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:26.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:26.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:26.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:26.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:26.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:26.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:26.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:26.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:26.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:26.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:26.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:26.670 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:26.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:26.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:26.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:26.804 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:38:26.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:26.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:26.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:26.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:26.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:26.975 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:26.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:26.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:27.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:27.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:27.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:27.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:27.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:27.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:26 smithi163 bash[78107]: audit 2023-10-09T09:38:26.253283+0000 mon.a (mon.0) 1877 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:38:27.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:26 smithi163 bash[75425]: audit 2023-10-09T09:38:26.253283+0000 mon.a (mon.0) 1877 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:38:27.243 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:27.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:27.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:26 smithi181 bash[59345]: audit 2023-10-09T09:38:26.253283+0000 mon.a (mon.0) 1877 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:38:27.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:27.386 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:27.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:27.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:27.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:27.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:27.525 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:27.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:27.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:27.682 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:27.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:27.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:27.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:27.819 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:27.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:27.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:27.892 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:27 smithi181 bash[59345]: cluster 2023-10-09T09:38:25.909491+0000 mgr.y (mgr.34107) 969 : cluster 0 pgmap v1726: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 7026/114561 objects degraded (6.133%); 6550/114561 objects misplaced (5.717%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:38:27.893 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:27 smithi163 bash[75425]: cluster 2023-10-09T09:38:25.909491+0000 mgr.y (mgr.34107) 969 : cluster 0 pgmap v1726: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 7026/114561 objects degraded (6.133%); 6550/114561 objects misplaced (5.717%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:38:27.893 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:27 smithi163 bash[78107]: cluster 2023-10-09T09:38:25.909491+0000 mgr.y (mgr.34107) 969 : cluster 0 pgmap v1726: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 7026/114561 objects degraded (6.133%); 6550/114561 objects misplaced (5.717%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T09:38:27.925 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:27.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:27.953 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:27.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:28.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:28.080 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:28.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:28.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:28.224 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:28.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:28.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:28.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:28.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:28.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:28.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:28.458 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:28.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:28.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:28.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:28.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:28.622 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:28.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:28.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:28.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:28.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:28.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:28.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:28.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:28.913 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:28.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:28.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:29.017 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:29.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:29.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:29.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:29.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:29.181 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:29.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:29.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:29.287 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:29.325 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:29.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:29.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:29.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:29.460 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:29.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:29.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:29.566 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:29.591 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:29.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:29.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:29.695 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:29.745 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:29.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:29.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:29.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:29.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:29.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:29.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:29.976 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:30.012 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:30.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:30.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:30.120 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:30.141 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:30.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:30.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:30.149 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:29 smithi163 bash[75425]: cluster 2023-10-09T09:38:27.910219+0000 mgr.y (mgr.34107) 970 : cluster 0 pgmap v1727: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 7102/107958 objects degraded (6.578%); 6173/107958 objects misplaced (5.718%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:38:30.149 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:29 smithi163 bash[78107]: cluster 2023-10-09T09:38:27.910219+0000 mgr.y (mgr.34107) 970 : cluster 0 pgmap v1727: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 7102/107958 objects degraded (6.578%); 6173/107958 objects misplaced (5.718%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:38:30.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:30.252 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:29 smithi181 bash[59345]: cluster 2023-10-09T09:38:27.910219+0000 mgr.y (mgr.34107) 970 : cluster 0 pgmap v1727: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 7102/107958 objects degraded (6.578%); 6173/107958 objects misplaced (5.718%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T09:38:30.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:30.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:30.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:30.380 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:30.405 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:30.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:30.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:30.538 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:30.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:30.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:30.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:30.686 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:30.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:30.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:30.824 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:30.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:30.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:30.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:30.938 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:30 smithi181 bash[59345]: cluster 2023-10-09T09:38:30.701953+0000 mon.a (mon.0) 1878 : cluster 3 Health check update: Degraded data redundancy: 7102/107958 objects degraded (6.578%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:30.957 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:30.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:30.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:31.062 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:31.102 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:31.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:31.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:31.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:31.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:30 smithi163 bash[75425]: cluster 2023-10-09T09:38:30.701953+0000 mon.a (mon.0) 1878 : cluster 3 Health check update: Degraded data redundancy: 7102/107958 objects degraded (6.578%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:31.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:30 smithi163 bash[78107]: cluster 2023-10-09T09:38:30.701953+0000 mon.a (mon.0) 1878 : cluster 3 Health check update: Degraded data redundancy: 7102/107958 objects degraded (6.578%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:31.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:31.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:31.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:31.352 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:31.402 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:31.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:31.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:31.511 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:31.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:31.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:31.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:31.646 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:31.673 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:31.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:31.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:31.802 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:31.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:31.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:31.909 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:31.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:31.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:31 smithi163 bash[75425]: cluster 2023-10-09T09:38:29.911272+0000 mgr.y (mgr.34107) 971 : cluster 0 pgmap v1728: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 7102/105600 objects degraded (6.725%); 5988/105600 objects misplaced (5.670%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:38:31.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:31 smithi163 bash[78107]: cluster 2023-10-09T09:38:29.911272+0000 mgr.y (mgr.34107) 971 : cluster 0 pgmap v1728: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 7102/105600 objects degraded (6.725%); 5988/105600 objects misplaced (5.670%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:38:31.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:31.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:32.053 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:32.096 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:32.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:32.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:32.206 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:32.216 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:31 smithi181 bash[59345]: cluster 2023-10-09T09:38:29.911272+0000 mgr.y (mgr.34107) 971 : cluster 0 pgmap v1728: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 7102/105600 objects degraded (6.725%); 5988/105600 objects misplaced (5.670%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:38:32.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:32.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:32.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:32.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:32.374 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:32.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:32.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:32.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:32.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:32.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:32.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:32.607 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:32.652 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:32.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:32.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:32.784 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:32.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:32.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:32.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:32.919 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:32 smithi163 bash[75425]: cluster 2023-10-09T09:38:31.911990+0000 mgr.y (mgr.34107) 972 : cluster 0 pgmap v1729: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 7193/100146 objects degraded (7.183%); 5790/100146 objects misplaced (5.782%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T09:38:32.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:32.960 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:32.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:32.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:33.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:33.094 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:33.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:33.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:33.201 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:33.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:32 smithi163 bash[78107]: cluster 2023-10-09T09:38:31.911990+0000 mgr.y (mgr.34107) 972 : cluster 0 pgmap v1729: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 7193/100146 objects degraded (7.183%); 5790/100146 objects misplaced (5.782%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T09:38:33.221 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:32 smithi181 bash[59345]: cluster 2023-10-09T09:38:31.911990+0000 mgr.y (mgr.34107) 972 : cluster 0 pgmap v1729: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 7193/100146 objects degraded (7.183%); 5790/100146 objects misplaced (5.782%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T09:38:33.252 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:33.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:33.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:33.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:33.387 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:33.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:33.493 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:33.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:33.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:33.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:33.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:33.655 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:38:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:38:33] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.33.4" 2023-10-09T09:38:33.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:33.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:33.783 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:33.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:33.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:33.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:33.919 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:33.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:33 smithi163 bash[75425]: audit 2023-10-09T09:38:32.922502+0000 mon.a (mon.0) 1879 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:33.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:33 smithi163 bash[75425]: audit 2023-10-09T09:38:32.932762+0000 mon.a (mon.0) 1880 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:33.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:33 smithi163 bash[78107]: audit 2023-10-09T09:38:32.922502+0000 mon.a (mon.0) 1879 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:33.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:33 smithi163 bash[78107]: audit 2023-10-09T09:38:32.932762+0000 mon.a (mon.0) 1880 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:33.950 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:33.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:34.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:34.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:34.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:34.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:34.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:34.214 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:33 smithi181 bash[59345]: audit 2023-10-09T09:38:32.922502+0000 mon.a (mon.0) 1879 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:34.214 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:33 smithi181 bash[59345]: audit 2023-10-09T09:38:32.932762+0000 mon.a (mon.0) 1880 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:34.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:34.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:34.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:34.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:34.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:34.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:34.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:34.475 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:34.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:34.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:34.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:34.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:34.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:34.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:34.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:34.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:34.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:34.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:34.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:34.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:34.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:34.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:34.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:35.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:35.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:35.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:35.151 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:35.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:35.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:35.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:35.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:34 smithi181 bash[59345]: cluster 2023-10-09T09:38:33.912580+0000 mgr.y (mgr.34107) 973 : cluster 0 pgmap v1730: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 7193/98385 objects degraded (7.311%); 5577/98385 objects misplaced (5.669%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:38:35.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:34 smithi163 bash[75425]: cluster 2023-10-09T09:38:33.912580+0000 mgr.y (mgr.34107) 973 : cluster 0 pgmap v1730: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 7193/98385 objects degraded (7.311%); 5577/98385 objects misplaced (5.669%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:38:35.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:34 smithi163 bash[78107]: cluster 2023-10-09T09:38:33.912580+0000 mgr.y (mgr.34107) 973 : cluster 0 pgmap v1730: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 7193/98385 objects degraded (7.311%); 5577/98385 objects misplaced (5.669%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:38:35.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:35.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:35.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:35.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:35.434 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:35.462 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:35.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:35.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:35.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:35.592 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:35.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:35.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:35.741 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:35.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:35.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:35.848 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:35.940 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:35 smithi181 bash[59345]: cluster 2023-10-09T09:38:35.702907+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: Degraded data redundancy: 7193/98385 objects degraded (7.311%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:36.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:35 smithi163 bash[75425]: cluster 2023-10-09T09:38:35.702907+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: Degraded data redundancy: 7193/98385 objects degraded (7.311%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:36.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:36 smithi163 bash[78107]: cluster 2023-10-09T09:38:35.702907+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: Degraded data redundancy: 7193/98385 objects degraded (7.311%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:36.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:36.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:36.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:36.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:36.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:36.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:36.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:36.601 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:36.640 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:36.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:36.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:36.743 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:36.785 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:36.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:36.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:36.890 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:36.923 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:36.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:36.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:37.026 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:37.054 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:37.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:37.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:37.157 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:37.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:37.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:37.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:37.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:37.409 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:37.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:37.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:37.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:37.541 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:37.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:37.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:37.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:37.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:37.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:37.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:37.723 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:37 smithi181 bash[59345]: cluster 2023-10-09T09:38:35.913235+0000 mgr.y (mgr.34107) 974 : cluster 0 pgmap v1731: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 7193/95388 objects degraded (7.541%); 5370/95388 objects misplaced (5.630%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:38:37.723 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:37 smithi181 bash[59345]: audit 2023-10-09T09:38:37.346199+0000 mon.a (mon.0) 1882 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:37.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:37 smithi181 bash[59345]: audit 2023-10-09T09:38:37.353326+0000 mon.a (mon.0) 1883 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:37.783 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:37.826 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:37.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:37.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:37.930 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:37.969 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:37.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:37.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:37.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:37 smithi163 bash[75425]: cluster 2023-10-09T09:38:35.913235+0000 mgr.y (mgr.34107) 974 : cluster 0 pgmap v1731: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 7193/95388 objects degraded (7.541%); 5370/95388 objects misplaced (5.630%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:38:37.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:37 smithi163 bash[75425]: audit 2023-10-09T09:38:37.346199+0000 mon.a (mon.0) 1882 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:37.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:37 smithi163 bash[75425]: audit 2023-10-09T09:38:37.353326+0000 mon.a (mon.0) 1883 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:37.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:37 smithi163 bash[78107]: cluster 2023-10-09T09:38:35.913235+0000 mgr.y (mgr.34107) 974 : cluster 0 pgmap v1731: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 7193/95388 objects degraded (7.541%); 5370/95388 objects misplaced (5.630%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:38:37.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:37 smithi163 bash[78107]: audit 2023-10-09T09:38:37.346199+0000 mon.a (mon.0) 1882 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:37.989 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:37 smithi163 bash[78107]: audit 2023-10-09T09:38:37.353326+0000 mon.a (mon.0) 1883 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:38.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:38.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:38.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:38.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:38.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:38.234 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:38.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:38.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:38.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:38.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:38.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:38.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:38.474 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:38.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:38.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:38.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:38.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:38.643 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:38.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:38.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:38.748 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:38.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:38.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:38.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:38.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:38.910 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:38.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:38.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:39.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:38 smithi163 bash[75425]: audit 2023-10-09T09:38:38.033955+0000 mon.a (mon.0) 1884 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:38:39.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:38 smithi163 bash[75425]: audit 2023-10-09T09:38:38.035059+0000 mon.a (mon.0) 1885 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:38:39.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:38 smithi163 bash[75425]: audit 2023-10-09T09:38:38.043743+0000 mon.a (mon.0) 1886 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:39.006 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:38 smithi163 bash[78107]: audit 2023-10-09T09:38:38.033955+0000 mon.a (mon.0) 1884 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:38:39.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:38 smithi163 bash[78107]: audit 2023-10-09T09:38:38.035059+0000 mon.a (mon.0) 1885 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:38:39.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:38 smithi163 bash[78107]: audit 2023-10-09T09:38:38.043743+0000 mon.a (mon.0) 1886 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:39.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:39.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:38 smithi181 bash[59345]: audit 2023-10-09T09:38:38.033955+0000 mon.a (mon.0) 1884 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:38:39.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:38 smithi181 bash[59345]: audit 2023-10-09T09:38:38.035059+0000 mon.a (mon.0) 1885 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:38:39.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:38 smithi181 bash[59345]: audit 2023-10-09T09:38:38.043743+0000 mon.a (mon.0) 1886 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:39.047 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:39.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:39.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:39.065 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:38:39.119 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:38:39.152 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:39.174 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:38:39.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:39.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:39.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:39.224 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:38:39.276 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:38:39.292 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:39.325 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:39.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:39.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:39.343 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:38:39.375 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:38:39.416 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:38:39.421 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:38:39.421 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:38:39.421 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:38:39.421 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:38:39.421 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:38:39.421 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:38:39.421 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:38:39.421 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:38:39.429 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:39.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:39.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:39.560 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:39.627 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:39.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:39.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:39.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:39.740 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:39 smithi181 bash[59345]: cluster 2023-10-09T09:38:37.914023+0000 mgr.y (mgr.34107) 975 : cluster 0 pgmap v1732: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 7290/89148 objects degraded (8.177%); 4975/89148 objects misplaced (5.581%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:38:39.740 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:39 smithi181 bash[59345]: audit 2023-10-09T09:38:39.029813+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.163:0/4170582651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:38:39.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:39.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:39.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:39.790 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:38:39.790 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:38:39.866 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:39.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:39.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:39.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:39.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:39 smithi163 bash[75425]: cluster 2023-10-09T09:38:37.914023+0000 mgr.y (mgr.34107) 975 : cluster 0 pgmap v1732: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 7290/89148 objects degraded (8.177%); 4975/89148 objects misplaced (5.581%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:38:39.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:39 smithi163 bash[75425]: audit 2023-10-09T09:38:39.029813+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.163:0/4170582651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:38:39.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:39 smithi163 bash[78107]: cluster 2023-10-09T09:38:37.914023+0000 mgr.y (mgr.34107) 975 : cluster 0 pgmap v1732: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 7290/89148 objects degraded (8.177%); 4975/89148 objects misplaced (5.581%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T09:38:39.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:39 smithi163 bash[78107]: audit 2023-10-09T09:38:39.029813+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.163:0/4170582651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:38:40.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:40.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:40.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:40.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:40.141 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:40.172 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:40.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:40.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:40.275 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:40.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:40.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:40.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:40.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:40.456 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:40.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:40.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:40.560 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:40.742 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[75425]: audit 2023-10-09T09:38:39.837949+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.163:0/2073125613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:38:40.742 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[75425]: audit 2023-10-09T09:38:39.838617+0000 mon.a (mon.0) 1887 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:38:40.742 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[75425]: audit 2023-10-09T09:38:40.065186+0000 mon.b (mon.1) 20 : audit 1 from='client.? 172.21.15.163:0/3856627726' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.40189232106549444}]: dispatch 2023-10-09T09:38:40.742 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[75425]: audit 2023-10-09T09:38:40.066080+0000 mon.a (mon.0) 1888 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.40189232106549444}]: dispatch 2023-10-09T09:38:40.742 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[75425]: audit 2023-10-09T09:38:40.380651+0000 mon.a (mon.0) 1889 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:40.742 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[75425]: cluster 2023-10-09T09:38:40.706266+0000 mon.a (mon.0) 1890 : cluster 3 Health check update: Degraded data redundancy: 7290/89148 objects degraded (8.177%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:40.758 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.5 primary-affinity to 0.401892 (8263382) 2023-10-09T09:38:40.770 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:38:40.868 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:40.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:40.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:40.970 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:40.990 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:40.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:40.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:41.012 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[78107]: audit 2023-10-09T09:38:39.837949+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.163:0/2073125613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:38:41.012 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[78107]: audit 2023-10-09T09:38:39.838617+0000 mon.a (mon.0) 1887 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:38:41.012 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[78107]: audit 2023-10-09T09:38:40.065186+0000 mon.b (mon.1) 20 : audit 1 from='client.? 172.21.15.163:0/3856627726' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.40189232106549444}]: dispatch 2023-10-09T09:38:41.012 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[78107]: audit 2023-10-09T09:38:40.066080+0000 mon.a (mon.0) 1888 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.40189232106549444}]: dispatch 2023-10-09T09:38:41.012 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[78107]: audit 2023-10-09T09:38:40.380651+0000 mon.a (mon.0) 1889 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:41.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:40 smithi163 bash[78107]: cluster 2023-10-09T09:38:40.706266+0000 mon.a (mon.0) 1890 : cluster 3 Health check update: Degraded data redundancy: 7290/89148 objects degraded (8.177%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:40 smithi181 bash[59345]: audit 2023-10-09T09:38:39.837949+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.163:0/2073125613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:38:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:40 smithi181 bash[59345]: audit 2023-10-09T09:38:39.838617+0000 mon.a (mon.0) 1887 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:38:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:40 smithi181 bash[59345]: audit 2023-10-09T09:38:40.065186+0000 mon.b (mon.1) 20 : audit 1 from='client.? 172.21.15.163:0/3856627726' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.40189232106549444}]: dispatch 2023-10-09T09:38:41.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:40 smithi181 bash[59345]: audit 2023-10-09T09:38:40.066080+0000 mon.a (mon.0) 1888 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.40189232106549444}]: dispatch 2023-10-09T09:38:41.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:40 smithi181 bash[59345]: audit 2023-10-09T09:38:40.380651+0000 mon.a (mon.0) 1889 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:41.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:40 smithi181 bash[59345]: cluster 2023-10-09T09:38:40.706266+0000 mon.a (mon.0) 1890 : cluster 3 Health check update: Degraded data redundancy: 7290/89148 objects degraded (8.177%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:41.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:41.113 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:41.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:41.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:41.216 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:41.235 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:41.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:41.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:41.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:41.371 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:41.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:41.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:41.475 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:41.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:41.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:41.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:41.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:41.647 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:41.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:41.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:41.751 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:41.773 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:41.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:41.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:41.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:41.907 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:41.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:42.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:42.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:41 smithi181 bash[59345]: cluster 2023-10-09T09:38:39.914631+0000 mgr.y (mgr.34107) 976 : cluster 0 pgmap v1733: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 7290/86964 objects degraded (8.383%); 4808/86964 objects misplaced (5.529%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:38:42.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:41 smithi181 bash[59345]: cluster 2023-10-09T09:38:40.739376+0000 mon.a (mon.0) 1891 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:38:42.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:41 smithi181 bash[59345]: cluster 2023-10-09T09:38:40.748910+0000 osd.5 (osd.5) 724 : cluster 3 failed to encode map e1158 with expected crc 2023-10-09T09:38:42.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:41 smithi181 bash[59345]: cluster 2023-10-09T09:38:40.749274+0000 osd.6 (osd.6) 437 : cluster 3 failed to encode map e1158 with expected crc 2023-10-09T09:38:42.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:41 smithi181 bash[59345]: audit 2023-10-09T09:38:40.753019+0000 mon.a (mon.0) 1892 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:38:42.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:41 smithi181 bash[59345]: audit 2023-10-09T09:38:40.753172+0000 mon.a (mon.0) 1893 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.40189232106549444}]': finished 2023-10-09T09:38:42.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:41 smithi181 bash[59345]: cluster 2023-10-09T09:38:40.753335+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e1158: 8 total, 8 up, 8 in 2023-10-09T09:38:42.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:42.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:42.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:42.146 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:42.168 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:42.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:42.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:42.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[78107]: cluster 2023-10-09T09:38:39.914631+0000 mgr.y (mgr.34107) 976 : cluster 0 pgmap v1733: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 7290/86964 objects degraded (8.383%); 4808/86964 objects misplaced (5.529%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:38:42.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[78107]: cluster 2023-10-09T09:38:40.739376+0000 mon.a (mon.0) 1891 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:38:42.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[78107]: cluster 2023-10-09T09:38:40.748910+0000 osd.5 (osd.5) 724 : cluster 3 failed to encode map e1158 with expected crc 2023-10-09T09:38:42.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[78107]: cluster 2023-10-09T09:38:40.749274+0000 osd.6 (osd.6) 437 : cluster 3 failed to encode map e1158 with expected crc 2023-10-09T09:38:42.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[78107]: audit 2023-10-09T09:38:40.753019+0000 mon.a (mon.0) 1892 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:38:42.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[78107]: audit 2023-10-09T09:38:40.753172+0000 mon.a (mon.0) 1893 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.40189232106549444}]': finished 2023-10-09T09:38:42.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[78107]: cluster 2023-10-09T09:38:40.753335+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e1158: 8 total, 8 up, 8 in 2023-10-09T09:38:42.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[75425]: cluster 2023-10-09T09:38:39.914631+0000 mgr.y (mgr.34107) 976 : cluster 0 pgmap v1733: 89 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 7290/86964 objects degraded (8.383%); 4808/86964 objects misplaced (5.529%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:38:42.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[75425]: cluster 2023-10-09T09:38:40.739376+0000 mon.a (mon.0) 1891 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:38:42.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[75425]: cluster 2023-10-09T09:38:40.748910+0000 osd.5 (osd.5) 724 : cluster 3 failed to encode map e1158 with expected crc 2023-10-09T09:38:42.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[75425]: cluster 2023-10-09T09:38:40.749274+0000 osd.6 (osd.6) 437 : cluster 3 failed to encode map e1158 with expected crc 2023-10-09T09:38:42.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[75425]: audit 2023-10-09T09:38:40.753019+0000 mon.a (mon.0) 1892 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:38:42.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[75425]: audit 2023-10-09T09:38:40.753172+0000 mon.a (mon.0) 1893 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.40189232106549444}]': finished 2023-10-09T09:38:42.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:41 smithi163 bash[75425]: cluster 2023-10-09T09:38:40.753335+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e1158: 8 total, 8 up, 8 in 2023-10-09T09:38:42.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:42.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:42.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:42.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:42.407 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:42.437 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:42.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:42.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:42.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:42.563 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:42.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:42.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:42.666 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:42.695 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:42.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:42.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:42.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:42.806 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[75425]: cluster 2023-10-09T09:38:41.749495+0000 osd.5 (osd.5) 725 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.806 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[75425]: cluster 2023-10-09T09:38:41.751402+0000 osd.4 (osd.4) 579 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.807 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[75425]: cluster 2023-10-09T09:38:41.752010+0000 mon.a (mon.0) 1895 : cluster 0 osdmap e1159: 8 total, 8 up, 8 in 2023-10-09T09:38:42.807 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[75425]: cluster 2023-10-09T09:38:41.752091+0000 osd.5 (osd.5) 726 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.807 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[75425]: cluster 2023-10-09T09:38:41.752732+0000 osd.7 (osd.7) 552 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.807 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[75425]: cluster 2023-10-09T09:38:41.752732+0000 osd.6 (osd.6) 438 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.807 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[75425]: cluster 2023-10-09T09:38:41.754150+0000 osd.4 (osd.4) 580 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.807 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[75425]: cluster 2023-10-09T09:38:41.755763+0000 osd.7 (osd.7) 553 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.807 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[75425]: cluster 2023-10-09T09:38:41.757439+0000 osd.4 (osd.4) 581 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.807 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[78107]: cluster 2023-10-09T09:38:41.749495+0000 osd.5 (osd.5) 725 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.807 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[78107]: cluster 2023-10-09T09:38:41.751402+0000 osd.4 (osd.4) 579 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.807 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[78107]: cluster 2023-10-09T09:38:41.752010+0000 mon.a (mon.0) 1895 : cluster 0 osdmap e1159: 8 total, 8 up, 8 in 2023-10-09T09:38:42.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[78107]: cluster 2023-10-09T09:38:41.752091+0000 osd.5 (osd.5) 726 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[78107]: cluster 2023-10-09T09:38:41.752732+0000 osd.7 (osd.7) 552 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[78107]: cluster 2023-10-09T09:38:41.752732+0000 osd.6 (osd.6) 438 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[78107]: cluster 2023-10-09T09:38:41.754150+0000 osd.4 (osd.4) 580 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[78107]: cluster 2023-10-09T09:38:41.755763+0000 osd.7 (osd.7) 553 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:42.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:42 smithi163 bash[78107]: cluster 2023-10-09T09:38:41.757439+0000 osd.4 (osd.4) 581 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:42 smithi181 bash[59345]: cluster 2023-10-09T09:38:41.749495+0000 osd.5 (osd.5) 725 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:42 smithi181 bash[59345]: cluster 2023-10-09T09:38:41.751402+0000 osd.4 (osd.4) 579 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:42 smithi181 bash[59345]: cluster 2023-10-09T09:38:41.752010+0000 mon.a (mon.0) 1895 : cluster 0 osdmap e1159: 8 total, 8 up, 8 in 2023-10-09T09:38:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:42 smithi181 bash[59345]: cluster 2023-10-09T09:38:41.752091+0000 osd.5 (osd.5) 726 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:42 smithi181 bash[59345]: cluster 2023-10-09T09:38:41.752732+0000 osd.7 (osd.7) 552 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:42 smithi181 bash[59345]: cluster 2023-10-09T09:38:41.752732+0000 osd.6 (osd.6) 438 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:42 smithi181 bash[59345]: cluster 2023-10-09T09:38:41.754150+0000 osd.4 (osd.4) 580 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:43.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:42 smithi181 bash[59345]: cluster 2023-10-09T09:38:41.755763+0000 osd.7 (osd.7) 553 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:43.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:42 smithi181 bash[59345]: cluster 2023-10-09T09:38:41.757439+0000 osd.4 (osd.4) 581 : cluster 3 failed to encode map e1159 with expected crc 2023-10-09T09:38:43.143 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:43.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:43.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:43.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:43.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:43.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:43.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:43.418 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:43.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:43.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:43.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:38:43] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.33.4" 2023-10-09T09:38:43.525 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:43.567 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:43.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:43.670 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:43.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:43.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:43.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:43.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:43.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[78107]: cluster 2023-10-09T09:38:41.915389+0000 mgr.y (mgr.34107) 977 : cluster 0 pgmap v1736: 89 pgs: 5 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 7290/83568 objects degraded (8.723%); 4808/83568 objects misplaced (5.753%); 824 KiB/s, 12 objects/s recovering 2023-10-09T09:38:43.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[78107]: cluster 2023-10-09T09:38:42.749054+0000 mon.a (mon.0) 1896 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T09:38:43.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[78107]: cluster 2023-10-09T09:38:42.759452+0000 osd.5 (osd.5) 727 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:43.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[78107]: cluster 2023-10-09T09:38:42.759470+0000 osd.6 (osd.6) 439 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:43.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[78107]: cluster 2023-10-09T09:38:42.762513+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e1160: 8 total, 8 up, 8 in 2023-10-09T09:38:43.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[78107]: cluster 2023-10-09T09:38:42.763106+0000 osd.7 (osd.7) 554 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:43.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[78107]: cluster 2023-10-09T09:38:42.763326+0000 osd.4 (osd.4) 582 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:43.816 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[75425]: cluster 2023-10-09T09:38:41.915389+0000 mgr.y (mgr.34107) 977 : cluster 0 pgmap v1736: 89 pgs: 5 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 7290/83568 objects degraded (8.723%); 4808/83568 objects misplaced (5.753%); 824 KiB/s, 12 objects/s recovering 2023-10-09T09:38:43.816 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[75425]: cluster 2023-10-09T09:38:42.749054+0000 mon.a (mon.0) 1896 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T09:38:43.817 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[75425]: cluster 2023-10-09T09:38:42.759452+0000 osd.5 (osd.5) 727 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:43.817 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[75425]: cluster 2023-10-09T09:38:42.759470+0000 osd.6 (osd.6) 439 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:43.817 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[75425]: cluster 2023-10-09T09:38:42.762513+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e1160: 8 total, 8 up, 8 in 2023-10-09T09:38:43.817 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[75425]: cluster 2023-10-09T09:38:42.763106+0000 osd.7 (osd.7) 554 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:43.817 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:43 smithi163 bash[75425]: cluster 2023-10-09T09:38:42.763326+0000 osd.4 (osd.4) 582 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:43.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:43.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:43.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:43.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:43.958 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:43.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:43.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:44.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:43 smithi181 bash[59345]: cluster 2023-10-09T09:38:41.915389+0000 mgr.y (mgr.34107) 977 : cluster 0 pgmap v1736: 89 pgs: 5 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 7290/83568 objects degraded (8.723%); 4808/83568 objects misplaced (5.753%); 824 KiB/s, 12 objects/s recovering 2023-10-09T09:38:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:43 smithi181 bash[59345]: cluster 2023-10-09T09:38:42.749054+0000 mon.a (mon.0) 1896 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T09:38:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:43 smithi181 bash[59345]: cluster 2023-10-09T09:38:42.759452+0000 osd.5 (osd.5) 727 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:43 smithi181 bash[59345]: cluster 2023-10-09T09:38:42.759470+0000 osd.6 (osd.6) 439 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:43 smithi181 bash[59345]: cluster 2023-10-09T09:38:42.762513+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e1160: 8 total, 8 up, 8 in 2023-10-09T09:38:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:43 smithi181 bash[59345]: cluster 2023-10-09T09:38:42.763106+0000 osd.7 (osd.7) 554 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:43 smithi181 bash[59345]: cluster 2023-10-09T09:38:42.763326+0000 osd.4 (osd.4) 582 : cluster 3 failed to encode map e1160 with expected crc 2023-10-09T09:38:44.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:44.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:44.188 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:44.219 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:44.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:44.358 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:44.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:44.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:44.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:44.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:44.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:44.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:44.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:38:45.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T09:38:45.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:45.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:45.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:45.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:45.189 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:45.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:45.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:45.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:45.316 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:45.338 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:45.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:45.445 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:45.470 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:45.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:45.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:45.574 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:45.599 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:45.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:45.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:45.708 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:45.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:45.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:45.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:45.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:45.858 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:45 smithi163 bash[75425]: cluster 2023-10-09T09:38:43.915941+0000 mgr.y (mgr.34107) 978 : cluster 0 pgmap v1738: 89 pgs: 5 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 7290/82185 objects degraded (8.870%); 4635/82185 objects misplaced (5.640%) 2023-10-09T09:38:45.858 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:45 smithi163 bash[75425]: cluster 2023-10-09T09:38:45.707630+0000 mon.a (mon.0) 1898 : cluster 3 Health check update: Degraded data redundancy: 7290/82185 objects degraded (8.870%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:45.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:45 smithi163 bash[78107]: cluster 2023-10-09T09:38:43.915941+0000 mgr.y (mgr.34107) 978 : cluster 0 pgmap v1738: 89 pgs: 5 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 7290/82185 objects degraded (8.870%); 4635/82185 objects misplaced (5.640%) 2023-10-09T09:38:45.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:45 smithi163 bash[78107]: cluster 2023-10-09T09:38:45.707630+0000 mon.a (mon.0) 1898 : cluster 3 Health check update: Degraded data redundancy: 7290/82185 objects degraded (8.870%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:45.880 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:45.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:45.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:45.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:45.994 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:45 smithi181 bash[59345]: cluster 2023-10-09T09:38:43.915941+0000 mgr.y (mgr.34107) 978 : cluster 0 pgmap v1738: 89 pgs: 5 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 7290/82185 objects degraded (8.870%); 4635/82185 objects misplaced (5.640%) 2023-10-09T09:38:45.994 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:45 smithi181 bash[59345]: cluster 2023-10-09T09:38:45.707630+0000 mon.a (mon.0) 1898 : cluster 3 Health check update: Degraded data redundancy: 7290/82185 objects degraded (8.870%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:38:46.012 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:46.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:46.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:46.118 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:46.143 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:46.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:46.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:46.246 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:46.271 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:46.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:46.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:46.377 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:46.400 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:46.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:46.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:46.503 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:46.544 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:46.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:46.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:46.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:46.686 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:46.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:46.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:46.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:46.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:46.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:46.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:46.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:47.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:47.081 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:47.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:47.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:47.184 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:47.220 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:47.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:47.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:47.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:47.353 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:47.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:47.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:47.458 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:47.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:47.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:47.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:47.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:47.624 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:47.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:47.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:47.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:38:47.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:47.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:47.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:47.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:47.793 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:47 smithi181 bash[59345]: cluster 2023-10-09T09:38:45.916576+0000 mgr.y (mgr.34107) 979 : cluster 0 pgmap v1739: 89 pgs: 4 peering, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 365 op/s; 6171/80385 objects degraded (7.677%); 5373/80385 objects misplaced (6.684%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T09:38:47.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:47.905 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:47.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:47.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:47.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:47 smithi163 bash[78107]: cluster 2023-10-09T09:38:45.916576+0000 mgr.y (mgr.34107) 979 : cluster 0 pgmap v1739: 89 pgs: 4 peering, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 365 op/s; 6171/80385 objects degraded (7.677%); 5373/80385 objects misplaced (6.684%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T09:38:47.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:47 smithi163 bash[75425]: cluster 2023-10-09T09:38:45.916576+0000 mgr.y (mgr.34107) 979 : cluster 0 pgmap v1739: 89 pgs: 4 peering, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 365 op/s; 6171/80385 objects degraded (7.677%); 5373/80385 objects misplaced (6.684%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T09:38:48.010 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:48.039 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:48.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:48.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:48.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:48.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:48.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:48.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:48.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:48.341 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:48.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:48.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:48.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:48.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:48.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:48.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:48.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:48.620 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:48.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:48.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:48.749 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:48.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:48.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:48.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:48.886 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:48.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:48.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:48.992 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:49.001 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:48 smithi181 bash[59345]: cluster 2023-10-09T09:38:48.791543+0000 mon.a (mon.0) 1899 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-10-09T09:38:49.015 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:49.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:49.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:49.127 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:48 smithi163 bash[78107]: cluster 2023-10-09T09:38:48.791543+0000 mon.a (mon.0) 1899 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-10-09T09:38:49.128 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:48 smithi163 bash[75425]: cluster 2023-10-09T09:38:48.791543+0000 mon.a (mon.0) 1899 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-10-09T09:38:49.155 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:49.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:49.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:49.260 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:49.290 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:49.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:49.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:49.393 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:49.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:49.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:49.540 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:49.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:49.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:49.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:49.679 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:49.710 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:49.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:49.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:49.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:49.845 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:49.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:49.950 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:49.959 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:49 smithi163 bash[75425]: cluster 2023-10-09T09:38:47.919187+0000 mgr.y (mgr.34107) 980 : cluster 0 pgmap v1740: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 7155/74616 objects degraded (9.589%); 3985/74616 objects misplaced (5.341%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:38:49.959 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:49 smithi163 bash[78107]: cluster 2023-10-09T09:38:47.919187+0000 mgr.y (mgr.34107) 980 : cluster 0 pgmap v1740: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 7155/74616 objects degraded (9.589%); 3985/74616 objects misplaced (5.341%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:38:49.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:50.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:50.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:50.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:50.163 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:50.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:50.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:50.267 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:50.274 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:49 smithi181 bash[59345]: cluster 2023-10-09T09:38:47.919187+0000 mgr.y (mgr.34107) 980 : cluster 0 pgmap v1740: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 7155/74616 objects degraded (9.589%); 3985/74616 objects misplaced (5.341%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:38:50.290 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:50.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:50.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:50.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:50.420 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:38:50.420 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:38:50.431 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:50.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:50.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:50.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:50.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:50.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:50.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:50.581 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:38:50.581 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:38:50.581 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T09:38:50.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-09T09:38:50.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:50.728 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:50.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:50.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:50.836 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:50.867 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:50.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:50.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:50.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:50 smithi163 bash[75425]: cluster 2023-10-09T09:38:50.708748+0000 mon.a (mon.0) 1900 : cluster 3 Health check update: Degraded data redundancy: 7155/74616 objects degraded (9.589%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:38:50.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:50 smithi163 bash[78107]: cluster 2023-10-09T09:38:50.708748+0000 mon.a (mon.0) 1900 : cluster 3 Health check update: Degraded data redundancy: 7155/74616 objects degraded (9.589%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:38:50.970 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:51.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:51.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:51.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:51.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:51.297 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:50 smithi181 bash[59345]: cluster 2023-10-09T09:38:50.708748+0000 mon.a (mon.0) 1900 : cluster 3 Health check update: Degraded data redundancy: 7155/74616 objects degraded (9.589%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:38:51.323 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:51.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:51.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:51.428 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:51.460 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:51.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:51.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:51.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:51.601 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:51.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:51.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:51.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:51.734 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:51.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:51.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:51.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:51.893 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:51.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:51.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:51 smithi163 bash[78107]: cluster 2023-10-09T09:38:49.919738+0000 mgr.y (mgr.34107) 981 : cluster 0 pgmap v1741: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 1.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 7155/72324 objects degraded (9.893%); 3670/72324 objects misplaced (5.074%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T09:38:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:51 smithi163 bash[75425]: cluster 2023-10-09T09:38:49.919738+0000 mgr.y (mgr.34107) 981 : cluster 0 pgmap v1741: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 1.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 7155/72324 objects degraded (9.893%); 3670/72324 objects misplaced (5.074%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T09:38:52.001 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:52.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:52.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:52.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:52.135 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:52.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:52.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:52.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:52.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:52.290 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:52.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:52.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:51 smithi181 bash[59345]: cluster 2023-10-09T09:38:49.919738+0000 mgr.y (mgr.34107) 981 : cluster 0 pgmap v1741: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 1.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 7155/72324 objects degraded (9.893%); 3670/72324 objects misplaced (5.074%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T09:38:52.393 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:52.747 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:38:52.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:52.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:52.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:53.042 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:53.087 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:53.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:53.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:53.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:53.224 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:53.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:53.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:53.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:53.365 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:53.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:53.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:53.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:53.637 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:38:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:38:53] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.33.4" 2023-10-09T09:38:53.650 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:53.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:53.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:53.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:53.788 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:53.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:53.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:53.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:53.899 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:53 smithi163 bash[78107]: cluster 2023-10-09T09:38:51.920571+0000 mgr.y (mgr.34107) 982 : cluster 0 pgmap v1742: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 6966/67107 objects degraded (10.380%); 3670/67107 objects misplaced (5.469%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:38:53.899 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:53 smithi163 bash[75425]: cluster 2023-10-09T09:38:51.920571+0000 mgr.y (mgr.34107) 982 : cluster 0 pgmap v1742: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 6966/67107 objects degraded (10.380%); 3670/67107 objects misplaced (5.469%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:38:53.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:53.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:53.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:54.030 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:54.065 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:54.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:53 smithi181 bash[59345]: cluster 2023-10-09T09:38:51.920571+0000 mgr.y (mgr.34107) 982 : cluster 0 pgmap v1742: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 6966/67107 objects degraded (10.380%); 3670/67107 objects misplaced (5.469%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T09:38:54.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:54.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:54.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:54.195 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:54.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:54.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:54.304 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:54.330 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:54.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:54.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:54.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:54.543 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:54.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:54.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:54.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:54.676 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:54.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:54.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:54.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:54.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:54.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:54.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:54.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:54.947 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:54.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:54.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:55.052 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:55.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:55.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:55.187 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:55.218 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:55.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:55.324 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:55.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:55.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:55.558 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:55.577 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:38:55.582 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:55.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:55.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:55.687 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:55.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:55.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:55.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:55.827 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:55.854 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:55.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:55.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:55.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:55.987 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:55.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:55.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:56.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:56.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:56.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:56.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:55 smithi163 bash[75425]: cluster 2023-10-09T09:38:53.924329+0000 mgr.y (mgr.34107) 983 : cluster 0 pgmap v1743: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 6966/65367 objects degraded (10.657%); 3471/65367 objects misplaced (5.310%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:38:56.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:55 smithi163 bash[75425]: audit 2023-10-09T09:38:55.383783+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:56.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:55 smithi163 bash[75425]: audit 2023-10-09T09:38:55.384614+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:56.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:55 smithi163 bash[75425]: cluster 2023-10-09T09:38:55.710029+0000 mon.a (mon.0) 1903 : cluster 3 Health check update: Degraded data redundancy: 6966/65367 objects degraded (10.657%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:38:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:55 smithi163 bash[78107]: cluster 2023-10-09T09:38:53.924329+0000 mgr.y (mgr.34107) 983 : cluster 0 pgmap v1743: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 6966/65367 objects degraded (10.657%); 3471/65367 objects misplaced (5.310%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:38:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:55 smithi163 bash[78107]: audit 2023-10-09T09:38:55.383783+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:55 smithi163 bash[78107]: audit 2023-10-09T09:38:55.384614+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:55 smithi163 bash[78107]: cluster 2023-10-09T09:38:55.710029+0000 mon.a (mon.0) 1903 : cluster 3 Health check update: Degraded data redundancy: 6966/65367 objects degraded (10.657%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:38:56.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:56.274 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:56.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:56.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:55 smithi181 bash[59345]: cluster 2023-10-09T09:38:53.924329+0000 mgr.y (mgr.34107) 983 : cluster 0 pgmap v1743: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 6966/65367 objects degraded (10.657%); 3471/65367 objects misplaced (5.310%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:38:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:55 smithi181 bash[59345]: audit 2023-10-09T09:38:55.383783+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:38:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:55 smithi181 bash[59345]: audit 2023-10-09T09:38:55.384614+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:38:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:55 smithi181 bash[59345]: cluster 2023-10-09T09:38:55.710029+0000 mon.a (mon.0) 1903 : cluster 3 Health check update: Degraded data redundancy: 6966/65367 objects degraded (10.657%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:38:56.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:56.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:56.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:56.510 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:56.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:56.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:56.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:56.643 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:56.680 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:56.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:56.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:56.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:56.814 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:38:56.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:56.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:38:56.920 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:56.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:56.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:56.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:57.052 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:57.087 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:57.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:57.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:57.193 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:57.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:57.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:57.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:57.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:57.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:38:57.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:57.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:38:57.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:57.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:57.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:57.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:57.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:57.760 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:57.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:57.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:57.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:57.889 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:38:57.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:57.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:38:57.998 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:58.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:57 smithi163 bash[78107]: cluster 2023-10-09T09:38:55.924944+0000 mgr.y (mgr.34107) 984 : cluster 0 pgmap v1744: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 6966/62301 objects degraded (11.181%); 3123/62301 objects misplaced (5.013%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T09:38:58.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:57 smithi163 bash[75425]: cluster 2023-10-09T09:38:55.924944+0000 mgr.y (mgr.34107) 984 : cluster 0 pgmap v1744: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 6966/62301 objects degraded (11.181%); 3123/62301 objects misplaced (5.013%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T09:38:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:57 smithi181 bash[59345]: cluster 2023-10-09T09:38:55.924944+0000 mgr.y (mgr.34107) 984 : cluster 0 pgmap v1744: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 6966/62301 objects degraded (11.181%); 3123/62301 objects misplaced (5.013%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T09:38:58.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:58.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:58.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:58.426 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:58.455 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:38:58.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:38:58.556 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:58.587 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:58.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:58.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:58.693 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:58.737 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:58.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:58.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:58.840 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:58.863 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:58.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:58.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:58.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:59.004 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:38:59.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:59.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:38:59.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:59.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:59.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:59.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:59.437 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:38:59.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:59.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:38:59.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:59.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:59.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:59.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:59.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:59.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:38:59.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:59.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:38:59.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:59.838 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:59.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:59.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:38:59.945 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:59.969 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:38:59.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:38:59.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:00.079 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:00.115 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:00.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:00.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:00.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:00.247 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:00.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:00.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:38:59 smithi181 bash[59345]: cluster 2023-10-09T09:38:57.926351+0000 mgr.y (mgr.34107) 985 : cluster 0 pgmap v1745: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 6784/56028 objects degraded (12.108%); 2926/56028 objects misplaced (5.222%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T09:39:00.347 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:38:59 smithi163 bash[75425]: cluster 2023-10-09T09:38:57.926351+0000 mgr.y (mgr.34107) 985 : cluster 0 pgmap v1745: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 6784/56028 objects degraded (12.108%); 2926/56028 objects misplaced (5.222%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T09:39:00.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:38:59 smithi163 bash[78107]: cluster 2023-10-09T09:38:57.926351+0000 mgr.y (mgr.34107) 985 : cluster 0 pgmap v1745: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 6784/56028 objects degraded (12.108%); 2926/56028 objects misplaced (5.222%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T09:39:00.348 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:00.381 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:00.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:00.487 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:00.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:00.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:00.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:00.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:00.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:00.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:00.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:00.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:00.829 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:00.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:00.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:00.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:00.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:00.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:00.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:00.711172+0000 mon.a (mon.0) 1904 : cluster 3 Health check update: Degraded data redundancy: 6784/56028 objects degraded (12.108%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:01.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:01.098 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:01.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:01.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:01.202 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:01.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:00.711172+0000 mon.a (mon.0) 1904 : cluster 3 Health check update: Degraded data redundancy: 6784/56028 objects degraded (12.108%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:01.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:00.711172+0000 mon.a (mon.0) 1904 : cluster 3 Health check update: Degraded data redundancy: 6784/56028 objects degraded (12.108%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:01.244 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:01.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:01.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:01.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:01.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:01.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:01.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:01.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:01.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:01.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:01.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:01.644 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:01.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:01.749 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:01.781 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:01.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:01.884 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:01.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:01.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:01.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:02.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:02.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:02.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:02.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:01 smithi181 bash[59345]: cluster 2023-10-09T09:38:59.926957+0000 mgr.y (mgr.34107) 986 : cluster 0 pgmap v1746: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 6784/53730 objects degraded (12.626%); 2567/53730 objects misplaced (4.778%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:39:02.153 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:02.184 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:02.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:02.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:02.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:01 smithi163 bash[75425]: cluster 2023-10-09T09:38:59.926957+0000 mgr.y (mgr.34107) 986 : cluster 0 pgmap v1746: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 6784/53730 objects degraded (12.626%); 2567/53730 objects misplaced (4.778%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:39:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:01 smithi163 bash[78107]: cluster 2023-10-09T09:38:59.926957+0000 mgr.y (mgr.34107) 986 : cluster 0 pgmap v1746: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 6784/53730 objects degraded (12.626%); 2567/53730 objects misplaced (4.778%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T09:39:02.287 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:02.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:02.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:02.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:02.423 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:02.452 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:02.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:02.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:02.555 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:02.582 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:02.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:02.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:02.685 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:02.709 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:02.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:02.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:02.811 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:02.848 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:02.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:02.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:02.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:02.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:02.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:02.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:03.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:03.115 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:03.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:03.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:03.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:03.253 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:03.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:03.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:03.358 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:03.388 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:03.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:03.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:03.397 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:39:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:39:03] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.33.4" 2023-10-09T09:39:03.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:03.521 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:03.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:03.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:03.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:03.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:03.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:03.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:03.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:03.805 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:03.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:03.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:03.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:03.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:03.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:03.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:04.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:03 smithi181 bash[59345]: cluster 2023-10-09T09:39:01.927702+0000 mgr.y (mgr.34107) 987 : cluster 0 pgmap v1747: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1005 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 6596/48288 objects degraded (13.660%); 2567/48288 objects misplaced (5.316%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:39:04.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:04.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:04.181 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:04.204 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:04.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:04.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:04.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:03 smithi163 bash[75425]: cluster 2023-10-09T09:39:01.927702+0000 mgr.y (mgr.34107) 987 : cluster 0 pgmap v1747: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1005 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 6596/48288 objects degraded (13.660%); 2567/48288 objects misplaced (5.316%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:39:04.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:03 smithi163 bash[78107]: cluster 2023-10-09T09:39:01.927702+0000 mgr.y (mgr.34107) 987 : cluster 0 pgmap v1747: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1005 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 6596/48288 objects degraded (13.660%); 2567/48288 objects misplaced (5.316%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:39:04.314 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:04.345 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:04.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:04.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:04.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:04.484 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:04.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:04.591 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:04.671 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:04.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:04.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:04.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:04.808 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:04.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:04.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:04.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:04.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:04.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:04.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:05.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:05.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:05.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:05.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:05.197 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:05.231 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:05.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:05.338 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:05.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:05.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:05.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:05.640 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:05.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:05.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:05.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:05.790 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:05.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:05.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:05.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:05.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:05.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:05 smithi181 bash[59345]: cluster 2023-10-09T09:39:03.928308+0000 mgr.y (mgr.34107) 988 : cluster 0 pgmap v1748: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 969 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 6596/46551 objects degraded (14.169%); 2370/46551 objects misplaced (5.091%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T09:39:05.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:05 smithi181 bash[59345]: cluster 2023-10-09T09:39:05.712284+0000 mon.a (mon.0) 1905 : cluster 3 Health check update: Degraded data redundancy: 6596/46551 objects degraded (14.169%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:05.950 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:05.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:05.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:06.060 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:06.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:06.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:06.196 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:05 smithi163 bash[75425]: cluster 2023-10-09T09:39:03.928308+0000 mgr.y (mgr.34107) 988 : cluster 0 pgmap v1748: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 969 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 6596/46551 objects degraded (14.169%); 2370/46551 objects misplaced (5.091%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T09:39:06.196 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:05 smithi163 bash[75425]: cluster 2023-10-09T09:39:05.712284+0000 mon.a (mon.0) 1905 : cluster 3 Health check update: Degraded data redundancy: 6596/46551 objects degraded (14.169%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:06.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:05 smithi163 bash[78107]: cluster 2023-10-09T09:39:03.928308+0000 mgr.y (mgr.34107) 988 : cluster 0 pgmap v1748: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 969 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 6596/46551 objects degraded (14.169%); 2370/46551 objects misplaced (5.091%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T09:39:06.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:05 smithi163 bash[78107]: cluster 2023-10-09T09:39:05.712284+0000 mon.a (mon.0) 1905 : cluster 3 Health check update: Degraded data redundancy: 6596/46551 objects degraded (14.169%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:06.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:06.290 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:06.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:06.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:06.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:06.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:06.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:06.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:06.540 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:06.569 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:06.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:06.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:06.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:06.695 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:39:06.695 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:39:06.705 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:06.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:06.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:06.810 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:06.842 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:06.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:06.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:06.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:06.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:06.979 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:06.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:07.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:07.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:07.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:07.215 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:07.236 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:07.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:07.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:07.367 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:07.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:07.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:07.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:07.519 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:07.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:07.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:07.625 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:07.652 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:07.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:07.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:07.787 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:07.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:07.894 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:07.921 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:39:07.921 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:07.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:07.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:08.026 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:08.035 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:07 smithi181 bash[59345]: cluster 2023-10-09T09:39:05.929279+0000 mgr.y (mgr.34107) 989 : cluster 0 pgmap v1749: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 908 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 6596/43638 objects degraded (15.115%); 2002/43638 objects misplaced (4.588%); 4.6 MiB/s, 72 objects/s recovering 2023-10-09T09:39:08.035 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:07 smithi181 bash[59345]: audit 2023-10-09T09:39:07.218754+0000 mon.a (mon.0) 1906 : audit 1 from='client.? 172.21.15.163:0/2562033002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:39:08.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:08.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:08.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:08.164 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:08.173 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:07 smithi163 bash[78107]: cluster 2023-10-09T09:39:05.929279+0000 mgr.y (mgr.34107) 989 : cluster 0 pgmap v1749: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 908 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 6596/43638 objects degraded (15.115%); 2002/43638 objects misplaced (4.588%); 4.6 MiB/s, 72 objects/s recovering 2023-10-09T09:39:08.174 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:07 smithi163 bash[78107]: audit 2023-10-09T09:39:07.218754+0000 mon.a (mon.0) 1906 : audit 1 from='client.? 172.21.15.163:0/2562033002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:39:08.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:07 smithi163 bash[75425]: cluster 2023-10-09T09:39:05.929279+0000 mgr.y (mgr.34107) 989 : cluster 0 pgmap v1749: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 908 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 6596/43638 objects degraded (15.115%); 2002/43638 objects misplaced (4.588%); 4.6 MiB/s, 72 objects/s recovering 2023-10-09T09:39:08.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:07 smithi163 bash[75425]: audit 2023-10-09T09:39:07.218754+0000 mon.a (mon.0) 1906 : audit 1 from='client.? 172.21.15.163:0/2562033002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:39:08.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:08.201 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:08.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:08.255 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:39:08.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:08.304 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:39:08.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:08.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:08.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:08.352 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:39:08.394 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:39:08.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:08.437 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:39:08.460 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:08.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:08.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:08.472 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:39:08.510 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:39:08.550 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:39:08.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:39:08.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:39:08.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:39:08.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:39:08.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:39:08.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:39:08.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:39:08.552 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:39:08.566 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:08.599 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:08.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:08.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:08.703 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:08.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:08.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:08.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:08.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:08.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:08.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:08.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:08.901 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[75425]: cluster 2023-10-09T09:39:07.895781+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:39:08.901 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[75425]: cluster 2023-10-09T09:39:07.909982+0000 osd.5 (osd.5) 728 : cluster 3 failed to encode map e1161 with expected crc 2023-10-09T09:39:08.901 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[75425]: audit 2023-10-09T09:39:07.912431+0000 mon.a (mon.0) 1908 : audit 1 from='client.? 172.21.15.163:0/2562033002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:39:08.901 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[75425]: cluster 2023-10-09T09:39:07.914961+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e1161: 8 total, 8 up, 8 in 2023-10-09T09:39:08.901 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[75425]: cluster 2023-10-09T09:39:07.916763+0000 osd.6 (osd.6) 440 : cluster 3 failed to encode map e1161 with expected crc 2023-10-09T09:39:08.901 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[75425]: audit 2023-10-09T09:39:08.211687+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.163:0/99643856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:39:09.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:09.047 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:09.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:09.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:09.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:09.161 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[78107]: cluster 2023-10-09T09:39:07.895781+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:39:09.161 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[78107]: cluster 2023-10-09T09:39:07.909982+0000 osd.5 (osd.5) 728 : cluster 3 failed to encode map e1161 with expected crc 2023-10-09T09:39:09.162 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[78107]: audit 2023-10-09T09:39:07.912431+0000 mon.a (mon.0) 1908 : audit 1 from='client.? 172.21.15.163:0/2562033002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:39:09.162 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[78107]: cluster 2023-10-09T09:39:07.914961+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e1161: 8 total, 8 up, 8 in 2023-10-09T09:39:09.162 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[78107]: cluster 2023-10-09T09:39:07.916763+0000 osd.6 (osd.6) 440 : cluster 3 failed to encode map e1161 with expected crc 2023-10-09T09:39:09.162 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:08 smithi163 bash[78107]: audit 2023-10-09T09:39:08.211687+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.163:0/99643856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:39:09.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:09.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:09.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:09.290 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:09.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:08 smithi181 bash[59345]: cluster 2023-10-09T09:39:07.895781+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:39:09.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:08 smithi181 bash[59345]: cluster 2023-10-09T09:39:07.909982+0000 osd.5 (osd.5) 728 : cluster 3 failed to encode map e1161 with expected crc 2023-10-09T09:39:09.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:08 smithi181 bash[59345]: audit 2023-10-09T09:39:07.912431+0000 mon.a (mon.0) 1908 : audit 1 from='client.? 172.21.15.163:0/2562033002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:39:09.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:08 smithi181 bash[59345]: cluster 2023-10-09T09:39:07.914961+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e1161: 8 total, 8 up, 8 in 2023-10-09T09:39:09.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:08 smithi181 bash[59345]: cluster 2023-10-09T09:39:07.916763+0000 osd.6 (osd.6) 440 : cluster 3 failed to encode map e1161 with expected crc 2023-10-09T09:39:09.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:08 smithi181 bash[59345]: audit 2023-10-09T09:39:08.211687+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.163:0/99643856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:39:09.318 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:09.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:09.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:09.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:09.449 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:09.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:09.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:09.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:09.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:09.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:09.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:09.691 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:09.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:09.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:09.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:09.827 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:09.872 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:09.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:09.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:09.905 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:09 smithi163 bash[75425]: cluster 2023-10-09T09:39:07.929837+0000 mgr.y (mgr.34107) 990 : cluster 0 pgmap v1751: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 773 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 6367/37128 objects degraded (17.149%); 1799/37128 objects misplaced (4.845%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:39:09.905 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:09 smithi163 bash[78107]: cluster 2023-10-09T09:39:07.929837+0000 mgr.y (mgr.34107) 990 : cluster 0 pgmap v1751: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 773 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 6367/37128 objects degraded (17.149%); 1799/37128 objects misplaced (4.845%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:39:09.976 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:09.985 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:09 smithi181 bash[59345]: cluster 2023-10-09T09:39:07.929837+0000 mgr.y (mgr.34107) 990 : cluster 0 pgmap v1751: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 773 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 6367/37128 objects degraded (17.149%); 1799/37128 objects misplaced (4.845%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T09:39:09.999 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:10.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:10.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:10.102 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:10.125 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:10.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:10.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:10.268 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:10.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:10.374 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:10.402 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:10.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:10.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:10.505 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:10.529 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:10.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:10.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:10.595 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T09:39:10.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:10.661 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:10.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:10.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:10.764 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:10.794 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:10.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:10.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:10.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:10.917 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:10 smithi163 bash[75425]: audit 2023-10-09T09:39:10.381306+0000 mon.a (mon.0) 1910 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:10.917 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:10 smithi163 bash[75425]: cluster 2023-10-09T09:39:10.713364+0000 mon.a (mon.0) 1911 : cluster 3 Health check update: Degraded data redundancy: 6367/37128 objects degraded (17.149%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:10.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:10.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:11.039 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:11.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:11.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:11.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:11.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:10 smithi181 bash[59345]: audit 2023-10-09T09:39:10.381306+0000 mon.a (mon.0) 1910 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:10 smithi181 bash[59345]: cluster 2023-10-09T09:39:10.713364+0000 mon.a (mon.0) 1911 : cluster 3 Health check update: Degraded data redundancy: 6367/37128 objects degraded (17.149%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:11.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:11.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:11.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:11.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:10 smithi163 bash[78107]: audit 2023-10-09T09:39:10.381306+0000 mon.a (mon.0) 1910 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:11.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:10 smithi163 bash[78107]: cluster 2023-10-09T09:39:10.713364+0000 mon.a (mon.0) 1911 : cluster 3 Health check update: Degraded data redundancy: 6367/37128 objects degraded (17.149%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:11.301 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:11.337 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:11.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:11.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:11.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:11.481 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:11.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:11.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:11.585 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:11.612 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:11.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:11.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:11.718 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:11 smithi163 bash[75425]: debug 2023-10-09T09:39:11.509+0000 7f22f321a700 -1 mon.a@0(leader).osd e1161 definitely_dead 0 2023-10-09T09:39:11.755 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:11.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:11.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:11.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:11.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:11.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:11.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:11.945 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.5. 2023-10-09T09:39:11.994 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:12.006 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:11 smithi181 bash[59345]: cluster 2023-10-09T09:39:09.930363+0000 mgr.y (mgr.34107) 991 : cluster 0 pgmap v1752: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 723 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 6385/34749 objects degraded (18.375%); 1402/34749 objects misplaced (4.035%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T09:39:12.007 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:11 smithi181 bash[59345]: audit 2023-10-09T09:39:11.513802+0000 mon.a (mon.0) 1912 : audit 1 from='client.? 172.21.15.163:0/3423916151' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T09:39:12.007 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:11 smithi181 bash[59345]: cluster 2023-10-09T09:39:11.514186+0000 mon.a (mon.0) 1913 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T09:39:12.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:12.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:12.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:12.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:12.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:11 smithi163 bash[78107]: cluster 2023-10-09T09:39:09.930363+0000 mgr.y (mgr.34107) 991 : cluster 0 pgmap v1752: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 723 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 6385/34749 objects degraded (18.375%); 1402/34749 objects misplaced (4.035%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T09:39:12.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:11 smithi163 bash[78107]: audit 2023-10-09T09:39:11.513802+0000 mon.a (mon.0) 1912 : audit 1 from='client.? 172.21.15.163:0/3423916151' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T09:39:12.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:11 smithi163 bash[78107]: cluster 2023-10-09T09:39:11.514186+0000 mon.a (mon.0) 1913 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T09:39:12.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:11 smithi163 bash[75425]: cluster 2023-10-09T09:39:09.930363+0000 mgr.y (mgr.34107) 991 : cluster 0 pgmap v1752: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 84 active+clean; 723 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 6385/34749 objects degraded (18.375%); 1402/34749 objects misplaced (4.035%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T09:39:12.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:11 smithi163 bash[75425]: audit 2023-10-09T09:39:11.513802+0000 mon.a (mon.0) 1912 : audit 1 from='client.? 172.21.15.163:0/3423916151' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T09:39:12.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:11 smithi163 bash[75425]: cluster 2023-10-09T09:39:11.514186+0000 mon.a (mon.0) 1913 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T09:39:12.150 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:12.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:12.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:12.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:12.279 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:12.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:12.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:12.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:12.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:12.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:12.505 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:12.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:12.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:12.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:12.641 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:12.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:39:12.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:39:12.673 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:12.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:12.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:12.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:12.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:12 smithi163 bash[78107]: cluster 2023-10-09T09:39:11.928817+0000 osd.6 (osd.6) 441 : cluster 3 failed to encode map e1162 with expected crc 2023-10-09T09:39:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:12 smithi163 bash[78107]: cluster 2023-10-09T09:39:11.930819+0000 mgr.y (mgr.34107) 992 : cluster 0 pgmap v1754: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 625 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 6226/30060 objects degraded (20.712%); 1402/30060 objects misplaced (4.664%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:39:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:12 smithi163 bash[78107]: audit 2023-10-09T09:39:11.931062+0000 mon.a (mon.0) 1914 : audit 1 from='client.? 172.21.15.163:0/3423916151' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T09:39:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:12 smithi163 bash[78107]: cluster 2023-10-09T09:39:11.931145+0000 mon.a (mon.0) 1915 : cluster 0 osdmap e1162: 8 total, 8 up, 7 in 2023-10-09T09:39:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:12 smithi163 bash[75425]: cluster 2023-10-09T09:39:11.928817+0000 osd.6 (osd.6) 441 : cluster 3 failed to encode map e1162 with expected crc 2023-10-09T09:39:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:12 smithi163 bash[75425]: cluster 2023-10-09T09:39:11.930819+0000 mgr.y (mgr.34107) 992 : cluster 0 pgmap v1754: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 625 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 6226/30060 objects degraded (20.712%); 1402/30060 objects misplaced (4.664%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:39:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:12 smithi163 bash[75425]: audit 2023-10-09T09:39:11.931062+0000 mon.a (mon.0) 1914 : audit 1 from='client.? 172.21.15.163:0/3423916151' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T09:39:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:12 smithi163 bash[75425]: cluster 2023-10-09T09:39:11.931145+0000 mon.a (mon.0) 1915 : cluster 0 osdmap e1162: 8 total, 8 up, 7 in 2023-10-09T09:39:13.013 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:13.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:13.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:13.116 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:13.145 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:13.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:13.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:13.248 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:13.271 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:13.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:13.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:13.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:12 smithi181 bash[59345]: cluster 2023-10-09T09:39:11.928817+0000 osd.6 (osd.6) 441 : cluster 3 failed to encode map e1162 with expected crc 2023-10-09T09:39:13.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:12 smithi181 bash[59345]: cluster 2023-10-09T09:39:11.930819+0000 mgr.y (mgr.34107) 992 : cluster 0 pgmap v1754: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 625 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 6226/30060 objects degraded (20.712%); 1402/30060 objects misplaced (4.664%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:39:13.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:12 smithi181 bash[59345]: audit 2023-10-09T09:39:11.931062+0000 mon.a (mon.0) 1914 : audit 1 from='client.? 172.21.15.163:0/3423916151' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T09:39:13.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:12 smithi181 bash[59345]: cluster 2023-10-09T09:39:11.931145+0000 mon.a (mon.0) 1915 : cluster 0 osdmap e1162: 8 total, 8 up, 7 in 2023-10-09T09:39:13.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:13.407 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:13.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:13.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:13.511 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:13.543 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:13.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:13.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:13.646 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:13.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:13.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:13.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:39:13] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.33.4" 2023-10-09T09:39:13.779 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:13.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:13.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:13.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:13.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:13.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:13.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:13.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:14.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:14.100 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:14.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:14.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[75425]: cluster 2023-10-09T09:39:11.926354+0000 osd.5 (osd.5) 729 : cluster 3 failed to encode map e1162 with expected crc 2023-10-09T09:39:14.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[75425]: cluster 2023-10-09T09:39:12.932824+0000 osd.5 (osd.5) 730 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[75425]: cluster 2023-10-09T09:39:12.933879+0000 osd.6 (osd.6) 442 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[75425]: cluster 2023-10-09T09:39:12.935400+0000 mon.a (mon.0) 1916 : cluster 0 osdmap e1163: 8 total, 8 up, 7 in 2023-10-09T09:39:14.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[75425]: cluster 2023-10-09T09:39:12.936913+0000 osd.7 (osd.7) 555 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[75425]: cluster 2023-10-09T09:39:12.937037+0000 osd.4 (osd.4) 583 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[75425]: cluster 2023-10-09T09:39:12.937135+0000 osd.6 (osd.6) 443 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[75425]: cluster 2023-10-09T09:39:12.941117+0000 osd.6 (osd.6) 444 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[75425]: cluster 2023-10-09T09:39:12.942049+0000 osd.4 (osd.4) 584 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[75425]: cluster 2023-10-09T09:39:12.943234+0000 osd.7 (osd.7) 556 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[78107]: cluster 2023-10-09T09:39:11.926354+0000 osd.5 (osd.5) 729 : cluster 3 failed to encode map e1162 with expected crc 2023-10-09T09:39:14.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[78107]: cluster 2023-10-09T09:39:12.932824+0000 osd.5 (osd.5) 730 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[78107]: cluster 2023-10-09T09:39:12.933879+0000 osd.6 (osd.6) 442 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[78107]: cluster 2023-10-09T09:39:12.935400+0000 mon.a (mon.0) 1916 : cluster 0 osdmap e1163: 8 total, 8 up, 7 in 2023-10-09T09:39:14.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[78107]: cluster 2023-10-09T09:39:12.936913+0000 osd.7 (osd.7) 555 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[78107]: cluster 2023-10-09T09:39:12.937037+0000 osd.4 (osd.4) 583 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[78107]: cluster 2023-10-09T09:39:12.937135+0000 osd.6 (osd.6) 443 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[78107]: cluster 2023-10-09T09:39:12.941117+0000 osd.6 (osd.6) 444 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[78107]: cluster 2023-10-09T09:39:12.942049+0000 osd.4 (osd.4) 584 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.109 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:13 smithi163 bash[78107]: cluster 2023-10-09T09:39:12.943234+0000 osd.7 (osd.7) 556 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:14.210 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:14.219 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:13 smithi181 bash[59345]: cluster 2023-10-09T09:39:11.926354+0000 osd.5 (osd.5) 729 : cluster 3 failed to encode map e1162 with expected crc 2023-10-09T09:39:14.219 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:13 smithi181 bash[59345]: cluster 2023-10-09T09:39:12.932824+0000 osd.5 (osd.5) 730 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:13 smithi181 bash[59345]: cluster 2023-10-09T09:39:12.933879+0000 osd.6 (osd.6) 442 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:13 smithi181 bash[59345]: cluster 2023-10-09T09:39:12.935400+0000 mon.a (mon.0) 1916 : cluster 0 osdmap e1163: 8 total, 8 up, 7 in 2023-10-09T09:39:14.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:13 smithi181 bash[59345]: cluster 2023-10-09T09:39:12.936913+0000 osd.7 (osd.7) 555 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:13 smithi181 bash[59345]: cluster 2023-10-09T09:39:12.937037+0000 osd.4 (osd.4) 583 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:13 smithi181 bash[59345]: cluster 2023-10-09T09:39:12.937135+0000 osd.6 (osd.6) 443 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:13 smithi181 bash[59345]: cluster 2023-10-09T09:39:12.941117+0000 osd.6 (osd.6) 444 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:13 smithi181 bash[59345]: cluster 2023-10-09T09:39:12.942049+0000 osd.4 (osd.4) 584 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:13 smithi181 bash[59345]: cluster 2023-10-09T09:39:12.943234+0000 osd.7 (osd.7) 556 : cluster 3 failed to encode map e1163 with expected crc 2023-10-09T09:39:14.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:14.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:14.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:14.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:14.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:14.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:14.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:14.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:14.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:14.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:14.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:14.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:14.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:14.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:14.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:14.797 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:14.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:14.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:14.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:14.925 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:14.955 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:14.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:14.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:15.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:15.095 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:15.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:15.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:15.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:15.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.931290+0000 mgr.y (mgr.34107) 993 : cluster 0 pgmap v1756: 89 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 588 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 6226/28275 objects degraded (22.019%); 1203/28275 objects misplaced (4.255%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T09:39:15.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.935751+0000 osd.5 (osd.5) 731 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.935977+0000 osd.6 (osd.6) 445 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.938193+0000 mon.a (mon.0) 1917 : cluster 0 osdmap e1164: 8 total, 8 up, 7 in 2023-10-09T09:39:15.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.939728+0000 osd.7 (osd.7) 557 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.939810+0000 osd.4 (osd.4) 585 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.940956+0000 osd.1 (osd.1) 2 : cluster 0 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1161'11306 2023-10-09T09:39:15.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.940986+0000 osd.1 (osd.1) 3 : cluster 1 8.a continuing backfill to osd.2 from (1143'3700,1161'11105] MIN to 1161'11105 2023-10-09T09:39:15.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.941594+0000 osd.6 (osd.6) 446 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.941673+0000 osd.1 (osd.1) 4 : cluster 0 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 1161'11562 2023-10-09T09:39:15.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.944889+0000 osd.6 (osd.6) 447 : cluster 0 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 1161'11339 2023-10-09T09:39:15.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.945417+0000 osd.7 (osd.7) 558 : cluster 0 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 1161'11346 2023-10-09T09:39:15.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[75425]: cluster 2023-10-09T09:39:13.947117+0000 osd.4 (osd.4) 586 : cluster 0 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 1161'11279 2023-10-09T09:39:15.215 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.931290+0000 mgr.y (mgr.34107) 993 : cluster 0 pgmap v1756: 89 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 588 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 6226/28275 objects degraded (22.019%); 1203/28275 objects misplaced (4.255%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T09:39:15.215 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.935751+0000 osd.5 (osd.5) 731 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.935977+0000 osd.6 (osd.6) 445 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.938193+0000 mon.a (mon.0) 1917 : cluster 0 osdmap e1164: 8 total, 8 up, 7 in 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.939728+0000 osd.7 (osd.7) 557 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.939810+0000 osd.4 (osd.4) 585 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.940956+0000 osd.1 (osd.1) 2 : cluster 0 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1161'11306 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.940986+0000 osd.1 (osd.1) 3 : cluster 1 8.a continuing backfill to osd.2 from (1143'3700,1161'11105] MIN to 1161'11105 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.941594+0000 osd.6 (osd.6) 446 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.941673+0000 osd.1 (osd.1) 4 : cluster 0 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 1161'11562 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.944889+0000 osd.6 (osd.6) 447 : cluster 0 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 1161'11339 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.945417+0000 osd.7 (osd.7) 558 : cluster 0 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 1161'11346 2023-10-09T09:39:15.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:14 smithi163 bash[78107]: cluster 2023-10-09T09:39:13.947117+0000 osd.4 (osd.4) 586 : cluster 0 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 1161'11279 2023-10-09T09:39:15.255 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:15.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:15.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:15.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.931290+0000 mgr.y (mgr.34107) 993 : cluster 0 pgmap v1756: 89 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 588 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 6226/28275 objects degraded (22.019%); 1203/28275 objects misplaced (4.255%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T09:39:15.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.935751+0000 osd.5 (osd.5) 731 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.935977+0000 osd.6 (osd.6) 445 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.938193+0000 mon.a (mon.0) 1917 : cluster 0 osdmap e1164: 8 total, 8 up, 7 in 2023-10-09T09:39:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.939728+0000 osd.7 (osd.7) 557 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.939810+0000 osd.4 (osd.4) 585 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.940956+0000 osd.1 (osd.1) 2 : cluster 0 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1161'11306 2023-10-09T09:39:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.940986+0000 osd.1 (osd.1) 3 : cluster 1 8.a continuing backfill to osd.2 from (1143'3700,1161'11105] MIN to 1161'11105 2023-10-09T09:39:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.941594+0000 osd.6 (osd.6) 446 : cluster 3 failed to encode map e1164 with expected crc 2023-10-09T09:39:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.941673+0000 osd.1 (osd.1) 4 : cluster 0 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 1161'11562 2023-10-09T09:39:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.944889+0000 osd.6 (osd.6) 447 : cluster 0 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 1161'11339 2023-10-09T09:39:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.945417+0000 osd.7 (osd.7) 558 : cluster 0 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 1161'11346 2023-10-09T09:39:15.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:14 smithi181 bash[59345]: cluster 2023-10-09T09:39:13.947117+0000 osd.4 (osd.4) 586 : cluster 0 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 1161'11279 2023-10-09T09:39:15.363 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:15.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:15.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:15.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:15.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:15.538 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:15.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:15.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:15.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:15.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:15.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:15.784 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:15.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:15.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:15.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:15.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:15.941 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:15.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:15.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:15.951 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:15 smithi181 bash[59345]: cluster 2023-10-09T09:39:15.714590+0000 mon.a (mon.0) 1918 : cluster 3 Health check update: Degraded data redundancy: 6226/28275 objects degraded (22.019%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:16.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:16.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:16.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:16.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:16.179 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:16.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:16.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:16.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:16.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:15 smithi163 bash[75425]: cluster 2023-10-09T09:39:15.714590+0000 mon.a (mon.0) 1918 : cluster 3 Health check update: Degraded data redundancy: 6226/28275 objects degraded (22.019%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:15 smithi163 bash[78107]: cluster 2023-10-09T09:39:15.714590+0000 mon.a (mon.0) 1918 : cluster 3 Health check update: Degraded data redundancy: 6226/28275 objects degraded (22.019%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:16.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:16.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:16.348 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:39:16.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:16.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:16.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:16.494 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:16.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:16.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:16.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:16.625 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:16.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:16.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:16.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:16.759 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:16.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:16.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:16.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:16.905 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:16.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:16.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:17.010 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:17.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:17.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:17.145 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:17.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:17.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:17.313 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:17.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:17.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:17.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:17.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:17.490 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:17.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:17.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:17.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:17.622 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:17.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:17.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:17.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:17.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:17.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:17.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:17.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:17.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:17 smithi163 bash[78107]: cluster 2023-10-09T09:39:15.931940+0000 mgr.y (mgr.34107) 994 : cluster 0 pgmap v1758: 89 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 543 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 5814/26130 objects degraded (22.250%); 2186/26130 objects misplaced (8.366%); 1.8 MiB/s, 97 objects/s recovering 2023-10-09T09:39:17.877 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:17 smithi163 bash[75425]: cluster 2023-10-09T09:39:15.931940+0000 mgr.y (mgr.34107) 994 : cluster 0 pgmap v1758: 89 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 543 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 5814/26130 objects degraded (22.250%); 2186/26130 objects misplaced (8.366%); 1.8 MiB/s, 97 objects/s recovering 2023-10-09T09:39:17.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:17.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:17.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:18.013 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:18.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:18.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:18.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:18.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:17 smithi181 bash[59345]: cluster 2023-10-09T09:39:15.931940+0000 mgr.y (mgr.34107) 994 : cluster 0 pgmap v1758: 89 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 543 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 5814/26130 objects degraded (22.250%); 2186/26130 objects misplaced (8.366%); 1.8 MiB/s, 97 objects/s recovering 2023-10-09T09:39:18.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:18.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:18.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:18.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:18.301 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:18.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:18.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:18.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:18.387 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:39:18.391 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:39:18.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:18.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:18.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:18.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:18.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:18.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:18.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:18.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:18.717 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:18.748 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:18.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:18.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:18.831 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 6, 7, 2] out_osds: [5] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:39:18.831 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:39:18.831 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-10-09T09:39:18.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:39:18.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:18.885 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:18.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:18.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:18.989 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:19.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:19.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:19.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:19.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:19.145 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:19.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:19.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:19.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:19.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:19.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:19.387 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:19.424 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:19.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:19.527 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:19.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:19.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:19.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:19.665 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:19.701 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:19.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:19.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:19.806 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:19.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:19 smithi181 bash[59345]: cluster 2023-10-09T09:39:17.932686+0000 mgr.y (mgr.34107) 995 : cluster 0 pgmap v1759: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 479 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 17638/23031 objects degraded (76.584%); 3211/23031 objects misplaced (13.942%); 2.9 MiB/s, 116 objects/s recovering 2023-10-09T09:39:19.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:19.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:19.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:19.938 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:19.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:19.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:19.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:20.075 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:20.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:20.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:20.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:20.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:20.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:19 smithi163 bash[78107]: cluster 2023-10-09T09:39:17.932686+0000 mgr.y (mgr.34107) 995 : cluster 0 pgmap v1759: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 479 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 17638/23031 objects degraded (76.584%); 3211/23031 objects misplaced (13.942%); 2.9 MiB/s, 116 objects/s recovering 2023-10-09T09:39:20.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:19 smithi163 bash[75425]: cluster 2023-10-09T09:39:17.932686+0000 mgr.y (mgr.34107) 995 : cluster 0 pgmap v1759: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 479 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 390 op/s; 17638/23031 objects degraded (76.584%); 3211/23031 objects misplaced (13.942%); 2.9 MiB/s, 116 objects/s recovering 2023-10-09T09:39:20.221 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:20.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:20.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:20.324 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:20.346 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:20.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:20.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:20.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:20.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:20.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:20.572 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:20.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:20.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:20.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:20.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:20.727 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:20.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:20.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:20.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:20.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:20.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:20.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:20.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:21.001 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:21.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:21.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:21.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:20 smithi163 bash[75425]: cluster 2023-10-09T09:39:19.727559+0000 osd.6 (osd.6) 448 : cluster 3 failed to encode map e1165 with expected crc 2023-10-09T09:39:21.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:20 smithi163 bash[75425]: cluster 2023-10-09T09:39:19.727665+0000 osd.5 (osd.5) 732 : cluster 3 failed to encode map e1165 with expected crc 2023-10-09T09:39:21.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:20 smithi163 bash[75425]: cluster 2023-10-09T09:39:19.730065+0000 mon.a (mon.0) 1919 : cluster 0 osdmap e1165: 8 total, 8 up, 7 in 2023-10-09T09:39:21.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:20 smithi163 bash[75425]: cluster 2023-10-09T09:39:19.732738+0000 osd.6 (osd.6) 449 : cluster 3 failed to encode map e1165 with expected crc 2023-10-09T09:39:21.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:20 smithi163 bash[75425]: cluster 2023-10-09T09:39:20.715887+0000 mon.a (mon.0) 1920 : cluster 3 Health check update: Degraded data redundancy: 17638/23031 objects degraded (76.584%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:21.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:20 smithi163 bash[78107]: cluster 2023-10-09T09:39:19.727559+0000 osd.6 (osd.6) 448 : cluster 3 failed to encode map e1165 with expected crc 2023-10-09T09:39:21.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:20 smithi163 bash[78107]: cluster 2023-10-09T09:39:19.727665+0000 osd.5 (osd.5) 732 : cluster 3 failed to encode map e1165 with expected crc 2023-10-09T09:39:21.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:20 smithi163 bash[78107]: cluster 2023-10-09T09:39:19.730065+0000 mon.a (mon.0) 1919 : cluster 0 osdmap e1165: 8 total, 8 up, 7 in 2023-10-09T09:39:21.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:20 smithi163 bash[78107]: cluster 2023-10-09T09:39:19.732738+0000 osd.6 (osd.6) 449 : cluster 3 failed to encode map e1165 with expected crc 2023-10-09T09:39:21.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:20 smithi163 bash[78107]: cluster 2023-10-09T09:39:20.715887+0000 mon.a (mon.0) 1920 : cluster 3 Health check update: Degraded data redundancy: 17638/23031 objects degraded (76.584%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:20 smithi181 bash[59345]: cluster 2023-10-09T09:39:19.727559+0000 osd.6 (osd.6) 448 : cluster 3 failed to encode map e1165 with expected crc 2023-10-09T09:39:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:20 smithi181 bash[59345]: cluster 2023-10-09T09:39:19.727665+0000 osd.5 (osd.5) 732 : cluster 3 failed to encode map e1165 with expected crc 2023-10-09T09:39:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:20 smithi181 bash[59345]: cluster 2023-10-09T09:39:19.730065+0000 mon.a (mon.0) 1919 : cluster 0 osdmap e1165: 8 total, 8 up, 7 in 2023-10-09T09:39:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:20 smithi181 bash[59345]: cluster 2023-10-09T09:39:19.732738+0000 osd.6 (osd.6) 449 : cluster 3 failed to encode map e1165 with expected crc 2023-10-09T09:39:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:20 smithi181 bash[59345]: cluster 2023-10-09T09:39:20.715887+0000 mon.a (mon.0) 1920 : cluster 3 Health check update: Degraded data redundancy: 17638/23031 objects degraded (76.584%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:21.108 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:21.140 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:21.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:21.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:21.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:21.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:21.379 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:21.409 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:21.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:21.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:21.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:21.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:21.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:21.663 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:21.693 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:21.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:21.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:21.797 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:21.830 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:21.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:21.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:21.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:21.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:21 smithi163 bash[78107]: cluster 2023-10-09T09:39:19.933186+0000 mgr.y (mgr.34107) 996 : cluster 0 pgmap v1761: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 417 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 17427/20085 objects degraded (86.766%); 2624/20085 objects misplaced (13.064%); 2.4 MiB/s, 129 objects/s recovering 2023-10-09T09:39:21.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:21 smithi163 bash[78107]: cluster 2023-10-09T09:39:20.747704+0000 osd.5 (osd.5) 733 : cluster 3 failed to encode map e1166 with expected crc 2023-10-09T09:39:21.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:21 smithi163 bash[78107]: cluster 2023-10-09T09:39:20.747839+0000 osd.6 (osd.6) 450 : cluster 3 failed to encode map e1166 with expected crc 2023-10-09T09:39:21.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:21 smithi163 bash[78107]: cluster 2023-10-09T09:39:20.751244+0000 mon.a (mon.0) 1921 : cluster 0 osdmap e1166: 8 total, 8 up, 7 in 2023-10-09T09:39:21.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:21 smithi163 bash[78107]: cluster 2023-10-09T09:39:20.792374+0000 osd.4 (osd.4) 587 : cluster 3 failed to encode map e1166 with expected crc 2023-10-09T09:39:21.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:21 smithi163 bash[75425]: cluster 2023-10-09T09:39:19.933186+0000 mgr.y (mgr.34107) 996 : cluster 0 pgmap v1761: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 417 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 17427/20085 objects degraded (86.766%); 2624/20085 objects misplaced (13.064%); 2.4 MiB/s, 129 objects/s recovering 2023-10-09T09:39:21.946 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:21 smithi163 bash[75425]: cluster 2023-10-09T09:39:20.747704+0000 osd.5 (osd.5) 733 : cluster 3 failed to encode map e1166 with expected crc 2023-10-09T09:39:21.946 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:21 smithi163 bash[75425]: cluster 2023-10-09T09:39:20.747839+0000 osd.6 (osd.6) 450 : cluster 3 failed to encode map e1166 with expected crc 2023-10-09T09:39:21.946 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:21 smithi163 bash[75425]: cluster 2023-10-09T09:39:20.751244+0000 mon.a (mon.0) 1921 : cluster 0 osdmap e1166: 8 total, 8 up, 7 in 2023-10-09T09:39:21.946 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:21 smithi163 bash[75425]: cluster 2023-10-09T09:39:20.792374+0000 osd.4 (osd.4) 587 : cluster 3 failed to encode map e1166 with expected crc 2023-10-09T09:39:21.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:22.146 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:21 smithi181 bash[59345]: cluster 2023-10-09T09:39:19.933186+0000 mgr.y (mgr.34107) 996 : cluster 0 pgmap v1761: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 417 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 17427/20085 objects degraded (86.766%); 2624/20085 objects misplaced (13.064%); 2.4 MiB/s, 129 objects/s recovering 2023-10-09T09:39:22.146 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:21 smithi181 bash[59345]: cluster 2023-10-09T09:39:20.747704+0000 osd.5 (osd.5) 733 : cluster 3 failed to encode map e1166 with expected crc 2023-10-09T09:39:22.146 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:21 smithi181 bash[59345]: cluster 2023-10-09T09:39:20.747839+0000 osd.6 (osd.6) 450 : cluster 3 failed to encode map e1166 with expected crc 2023-10-09T09:39:22.146 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:21 smithi181 bash[59345]: cluster 2023-10-09T09:39:20.751244+0000 mon.a (mon.0) 1921 : cluster 0 osdmap e1166: 8 total, 8 up, 7 in 2023-10-09T09:39:22.146 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:21 smithi181 bash[59345]: cluster 2023-10-09T09:39:20.792374+0000 osd.4 (osd.4) 587 : cluster 3 failed to encode map e1166 with expected crc 2023-10-09T09:39:22.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:22.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:22.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:22.301 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:22.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:22.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:22.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:22.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:22.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:22.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:22.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:22.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:22.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:22.869 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:22.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:22.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:22.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:23.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:23.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:23.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:23.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:23.132 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:23.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:23.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:23.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:23.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:23.369 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:23.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:23.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:23.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:23.503 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:23.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:23.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:23.607 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:39:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:39:23] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.33.4" 2023-10-09T09:39:23.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:23.638 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:23.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:23.745 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:23.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:23 smithi181 bash[59345]: cluster 2023-10-09T09:39:21.934063+0000 mgr.y (mgr.34107) 997 : cluster 0 pgmap v1763: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 332 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 17427/15999 objects degraded (108.926%); 2057/15999 objects misplaced (12.857%); 3.8 MiB/s, 140 objects/s recovering 2023-10-09T09:39:23.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:23.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:23.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:23.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:23.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:23 smithi163 bash[75425]: cluster 2023-10-09T09:39:21.934063+0000 mgr.y (mgr.34107) 997 : cluster 0 pgmap v1763: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 332 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 17427/15999 objects degraded (108.926%); 2057/15999 objects misplaced (12.857%); 3.8 MiB/s, 140 objects/s recovering 2023-10-09T09:39:23.898 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:23 smithi163 bash[78107]: cluster 2023-10-09T09:39:21.934063+0000 mgr.y (mgr.34107) 997 : cluster 0 pgmap v1763: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 332 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 17427/15999 objects degraded (108.926%); 2057/15999 objects misplaced (12.857%); 3.8 MiB/s, 140 objects/s recovering 2023-10-09T09:39:23.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:23.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:23.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:24.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:24.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:24.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:24.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:24.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:24.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:24.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:24.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:24.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:24.505 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:24.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:24.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:24.607 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:24.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:24.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:24.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:24.733 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:24.773 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:24.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:24.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:24.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:24.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:24.910 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:24.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:25.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:25.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:25.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:25.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:25.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:25.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:25.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:25.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:25.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:25.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:25.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:25.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:25.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:25.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:25.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:25.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:25.553 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:25.593 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:25.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:25.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:25.699 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:25.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:25.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:25.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:25 smithi181 bash[59345]: cluster 2023-10-09T09:39:23.934656+0000 mgr.y (mgr.34107) 998 : cluster 0 pgmap v1764: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 301 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 484 op/s; 17427/14493 objects degraded (120.244%); 1875/14493 objects misplaced (12.937%); 3.7 MiB/s, 87 objects/s recovering 2023-10-09T09:39:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:25 smithi181 bash[59345]: audit 2023-10-09T09:39:25.382226+0000 mon.a (mon.0) 1922 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:25.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:25 smithi181 bash[59345]: cluster 2023-10-09T09:39:25.717343+0000 mon.a (mon.0) 1923 : cluster 3 Health check update: Degraded data redundancy: 17427/14493 objects degraded (120.244%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:25.824 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:25.853 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:25.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:25.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:25.958 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:25.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:25.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:25.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:26.091 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:26.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:26.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:26.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:25 smithi163 bash[75425]: cluster 2023-10-09T09:39:23.934656+0000 mgr.y (mgr.34107) 998 : cluster 0 pgmap v1764: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 301 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 484 op/s; 17427/14493 objects degraded (120.244%); 1875/14493 objects misplaced (12.937%); 3.7 MiB/s, 87 objects/s recovering 2023-10-09T09:39:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:25 smithi163 bash[75425]: audit 2023-10-09T09:39:25.382226+0000 mon.a (mon.0) 1922 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:25 smithi163 bash[75425]: cluster 2023-10-09T09:39:25.717343+0000 mon.a (mon.0) 1923 : cluster 3 Health check update: Degraded data redundancy: 17427/14493 objects degraded (120.244%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:25 smithi163 bash[78107]: cluster 2023-10-09T09:39:23.934656+0000 mgr.y (mgr.34107) 998 : cluster 0 pgmap v1764: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 301 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 484 op/s; 17427/14493 objects degraded (120.244%); 1875/14493 objects misplaced (12.937%); 3.7 MiB/s, 87 objects/s recovering 2023-10-09T09:39:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:25 smithi163 bash[78107]: audit 2023-10-09T09:39:25.382226+0000 mon.a (mon.0) 1922 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:25 smithi163 bash[78107]: cluster 2023-10-09T09:39:25.717343+0000 mon.a (mon.0) 1923 : cluster 3 Health check update: Degraded data redundancy: 17427/14493 objects degraded (120.244%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:26.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:26.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:26.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:26.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:26.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:26.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:26.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:26.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:26.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:26.576 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:26.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:26.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:26.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:26.709 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:26.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:26.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:26.815 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:26.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:26.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:26.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:26.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:26.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:26.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:26.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:27.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:27.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:27.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:27.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:27.211 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:27.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:27.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:27.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:27.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:27.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:27.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:27.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:27.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:27.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:27.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:27.632 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:27.666 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:27.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:27.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:27.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:27.810 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:27.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:27.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:27 smithi181 bash[59345]: cluster 2023-10-09T09:39:25.935397+0000 mgr.y (mgr.34107) 999 : cluster 0 pgmap v1765: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 232 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 17047/11166 objects degraded (152.669%); 1324/11166 objects misplaced (11.857%); 1.7 MiB/s, 100 objects/s recovering 2023-10-09T09:39:28.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:28.078 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:27 smithi163 bash[75425]: cluster 2023-10-09T09:39:25.935397+0000 mgr.y (mgr.34107) 999 : cluster 0 pgmap v1765: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 232 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 17047/11166 objects degraded (152.669%); 1324/11166 objects misplaced (11.857%); 1.7 MiB/s, 100 objects/s recovering 2023-10-09T09:39:28.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:27 smithi163 bash[78107]: cluster 2023-10-09T09:39:25.935397+0000 mgr.y (mgr.34107) 999 : cluster 0 pgmap v1765: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 232 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 17047/11166 objects degraded (152.669%); 1324/11166 objects misplaced (11.857%); 1.7 MiB/s, 100 objects/s recovering 2023-10-09T09:39:28.099 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:28.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:28.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:28.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:28.238 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:28.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:28.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:28.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:28.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:28.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:28.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:28.473 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:28.512 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:28.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:28.616 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:28.647 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:28.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:28.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:28.792 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:28.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:28.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:28.862 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 95230 objects 2023-10-09T09:39:28.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :150.06 2023-10-09T09:39:28.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:28.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:28.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:28.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:29.025 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:29.045 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:29.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:29.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:29.148 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:29.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:29.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:29.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:29.283 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:29.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:29.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:29.420 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:29.452 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:29.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:29.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:29.556 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:29.577 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:29.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:29.658 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:39:29.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:29.712 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:29.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:29.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:29.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:29.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:29 smithi163 bash[75425]: cluster 2023-10-09T09:39:27.936179+0000 mgr.y (mgr.34107) 1000 : cluster 0 pgmap v1766: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 111 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 17047/5355 objects degraded (318.338%); 732/5355 objects misplaced (13.669%); 1.6 MiB/s, 72 objects/s recovering 2023-10-09T09:39:29.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:29 smithi163 bash[78107]: cluster 2023-10-09T09:39:27.936179+0000 mgr.y (mgr.34107) 1000 : cluster 0 pgmap v1766: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 111 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 17047/5355 objects degraded (318.338%); 732/5355 objects misplaced (13.669%); 1.6 MiB/s, 72 objects/s recovering 2023-10-09T09:39:29.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:29.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:29.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:29.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:30.018 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:30.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:30.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:29 smithi181 bash[59345]: cluster 2023-10-09T09:39:27.936179+0000 mgr.y (mgr.34107) 1000 : cluster 0 pgmap v1766: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 111 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 17047/5355 objects degraded (318.338%); 732/5355 objects misplaced (13.669%); 1.6 MiB/s, 72 objects/s recovering 2023-10-09T09:39:30.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:30.153 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:30.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:30.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:30.257 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:30.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:30.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:30.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:30.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:30.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:30.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:30.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:30.534 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:30.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:30.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:30.638 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:30.666 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:30.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:30.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:30.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:30.790 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:30.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:30.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:30.893 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:30.915 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:30.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:30.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:31.018 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:31.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:31.038 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:30 smithi163 bash[78107]: cluster 2023-10-09T09:39:30.718191+0000 mon.a (mon.0) 1924 : cluster 3 Health check update: Degraded data redundancy: 17047/5355 objects degraded (318.338%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:31.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:30 smithi163 bash[75425]: cluster 2023-10-09T09:39:30.718191+0000 mon.a (mon.0) 1924 : cluster 3 Health check update: Degraded data redundancy: 17047/5355 objects degraded (318.338%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:31.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:31.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:31.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:30 smithi181 bash[59345]: cluster 2023-10-09T09:39:30.718191+0000 mon.a (mon.0) 1924 : cluster 3 Health check update: Degraded data redundancy: 17047/5355 objects degraded (318.338%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:31.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:31.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:31.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:31.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:31.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:31.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:31.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:31.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:31.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:31.420 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:31.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:31.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:31.523 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:31.544 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:31.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:31.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:31.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:31.668 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:31.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:31.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:31.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:31.799 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:31.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:31.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:31.856 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:31 smithi163 bash[75425]: cluster 2023-10-09T09:39:29.936971+0000 mgr.y (mgr.34107) 1001 : cluster 0 pgmap v1767: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 62 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 16578/3030 objects degraded (547.129%); 254/3030 objects misplaced (8.383%); 1.3 MiB/s, 106 objects/s recovering 2023-10-09T09:39:31.902 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:31.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:31.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:31.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:32.034 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:32.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:32.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:32.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:31 smithi181 bash[59345]: cluster 2023-10-09T09:39:29.936971+0000 mgr.y (mgr.34107) 1001 : cluster 0 pgmap v1767: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 62 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 16578/3030 objects degraded (547.129%); 254/3030 objects misplaced (8.383%); 1.3 MiB/s, 106 objects/s recovering 2023-10-09T09:39:32.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:32.167 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:31 smithi163 bash[78107]: cluster 2023-10-09T09:39:29.936971+0000 mgr.y (mgr.34107) 1001 : cluster 0 pgmap v1767: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 62 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 16578/3030 objects degraded (547.129%); 254/3030 objects misplaced (8.383%); 1.3 MiB/s, 106 objects/s recovering 2023-10-09T09:39:32.180 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:32.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:32.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:32.282 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:32.314 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:32.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:32.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:32.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:32.445 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:32.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:32.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:32.549 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:32.577 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:32.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:32.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:32.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:32.708 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:32.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:32.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:32.812 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:32.841 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:32.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:32.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:32.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:32 smithi163 bash[75425]: cluster 2023-10-09T09:39:31.785063+0000 osd.5 (osd.5) 734 : cluster 3 failed to encode map e1167 with expected crc 2023-10-09T09:39:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:32 smithi163 bash[75425]: cluster 2023-10-09T09:39:31.786926+0000 mon.a (mon.0) 1925 : cluster 0 osdmap e1167: 8 total, 8 up, 7 in 2023-10-09T09:39:32.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:32 smithi163 bash[75425]: cluster 2023-10-09T09:39:31.886237+0000 osd.4 (osd.4) 588 : cluster 3 failed to encode map e1167 with expected crc 2023-10-09T09:39:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:32 smithi163 bash[78107]: cluster 2023-10-09T09:39:31.785063+0000 osd.5 (osd.5) 734 : cluster 3 failed to encode map e1167 with expected crc 2023-10-09T09:39:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:32 smithi163 bash[78107]: cluster 2023-10-09T09:39:31.786926+0000 mon.a (mon.0) 1925 : cluster 0 osdmap e1167: 8 total, 8 up, 7 in 2023-10-09T09:39:32.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:32 smithi163 bash[78107]: cluster 2023-10-09T09:39:31.886237+0000 osd.4 (osd.4) 588 : cluster 3 failed to encode map e1167 with expected crc 2023-10-09T09:39:32.974 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:32.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:32.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:33.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:32 smithi181 bash[59345]: cluster 2023-10-09T09:39:31.785063+0000 osd.5 (osd.5) 734 : cluster 3 failed to encode map e1167 with expected crc 2023-10-09T09:39:33.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:32 smithi181 bash[59345]: cluster 2023-10-09T09:39:31.786926+0000 mon.a (mon.0) 1925 : cluster 0 osdmap e1167: 8 total, 8 up, 7 in 2023-10-09T09:39:33.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:32 smithi181 bash[59345]: cluster 2023-10-09T09:39:31.886237+0000 osd.4 (osd.4) 588 : cluster 3 failed to encode map e1167 with expected crc 2023-10-09T09:39:33.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:33.098 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:33.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:33.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:33.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:33.229 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:33.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:33.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:33.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:33.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:33.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:33.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:33.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:33.489 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:33.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:33.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:33.591 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:33.613 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:33.614 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:39:33] "GET /metrics HTTP/1.1" 200 36381 "" "Prometheus/2.33.4" 2023-10-09T09:39:33.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:33.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:33.716 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:33.746 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:33.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:33.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:33.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:33.884 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:33.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:33.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:33.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[75425]: cluster 2023-10-09T09:39:31.937680+0000 mgr.y (mgr.34107) 1002 : cluster 0 pgmap v1769: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 80 active+clean; 13 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 9330/681 objects degraded (1370.044%); 67/681 objects misplaced (9.838%); 0 B/s, 85 objects/s recovering 2023-10-09T09:39:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[75425]: cluster 2023-10-09T09:39:32.793989+0000 osd.5 (osd.5) 735 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[75425]: cluster 2023-10-09T09:39:32.799451+0000 osd.6 (osd.6) 451 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[75425]: cluster 2023-10-09T09:39:32.802321+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e1168: 8 total, 8 up, 7 in 2023-10-09T09:39:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[75425]: cluster 2023-10-09T09:39:32.802809+0000 osd.7 (osd.7) 559 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[75425]: cluster 2023-10-09T09:39:32.833975+0000 osd.4 (osd.4) 589 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[78107]: cluster 2023-10-09T09:39:31.937680+0000 mgr.y (mgr.34107) 1002 : cluster 0 pgmap v1769: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 80 active+clean; 13 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 9330/681 objects degraded (1370.044%); 67/681 objects misplaced (9.838%); 0 B/s, 85 objects/s recovering 2023-10-09T09:39:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[78107]: cluster 2023-10-09T09:39:32.793989+0000 osd.5 (osd.5) 735 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[78107]: cluster 2023-10-09T09:39:32.799451+0000 osd.6 (osd.6) 451 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[78107]: cluster 2023-10-09T09:39:32.802321+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e1168: 8 total, 8 up, 7 in 2023-10-09T09:39:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[78107]: cluster 2023-10-09T09:39:32.802809+0000 osd.7 (osd.7) 559 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:33 smithi163 bash[78107]: cluster 2023-10-09T09:39:32.833975+0000 osd.4 (osd.4) 589 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:33.987 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:34.001 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:33 smithi181 bash[59345]: cluster 2023-10-09T09:39:31.937680+0000 mgr.y (mgr.34107) 1002 : cluster 0 pgmap v1769: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 80 active+clean; 13 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 9330/681 objects degraded (1370.044%); 67/681 objects misplaced (9.838%); 0 B/s, 85 objects/s recovering 2023-10-09T09:39:34.001 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:33 smithi181 bash[59345]: cluster 2023-10-09T09:39:32.793989+0000 osd.5 (osd.5) 735 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:34.001 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:33 smithi181 bash[59345]: cluster 2023-10-09T09:39:32.799451+0000 osd.6 (osd.6) 451 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:34.001 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:33 smithi181 bash[59345]: cluster 2023-10-09T09:39:32.802321+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e1168: 8 total, 8 up, 7 in 2023-10-09T09:39:34.001 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:33 smithi181 bash[59345]: cluster 2023-10-09T09:39:32.802809+0000 osd.7 (osd.7) 559 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:34.001 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:33 smithi181 bash[59345]: cluster 2023-10-09T09:39:32.833975+0000 osd.4 (osd.4) 589 : cluster 3 failed to encode map e1168 with expected crc 2023-10-09T09:39:34.015 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:34.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:34.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:34.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:34.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:34.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:34.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:34.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:34.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:34.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:34.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:34.399 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:34.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:34.491 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:39:34.505 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:39:34.505 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:39:34.505 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:39:34.506 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:39:34.506 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.506 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149568 8.f 8:f5b59462:::benchmark_data_smithi163_151167_object54335:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e1156)", 2023-10-09T09:39:34.506 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:17.862920+0000", 2023-10-09T09:39:34.506 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.636570519000003, 2023-10-09T09:39:34.506 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0313238870000001, 2023-10-09T09:39:34.506 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.506 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:39:34.506 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.507 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.507 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.507 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149568 2023-10-09T09:39:34.507 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.507 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.507 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.507 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.507 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862920+0000", 2023-10-09T09:39:34.507 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.507 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862920+0000", 2023-10-09T09:39:34.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862917+0000", 2023-10-09T09:39:34.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:39:34.509 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.509 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.509 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.509 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862928+0000", 2023-10-09T09:39:34.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0443e-05 2023-10-09T09:39:34.509 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.509 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.509 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.509 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862943+0000", 2023-10-09T09:39:34.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5537e-05 2023-10-09T09:39:34.510 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.510 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.510 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.510 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862954+0000", 2023-10-09T09:39:34.510 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0417999999999999e-05 2023-10-09T09:39:34.510 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.510 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.510 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.510 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.864772+0000", 2023-10-09T09:39:34.510 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0018178650000000001 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op must wait for map", 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.864797+0000", 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5316e-05 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.869967+0000", 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051704309999999996 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.511 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.869978+0000", 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0801e-05 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.894180+0000", 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0242017859999999 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.894243+0000", 2023-10-09T09:39:34.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3491000000000006e-05 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149574 8.f 8:f702f824:::benchmark_data_smithi163_151167_object54341:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e1156)", 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:17.862969+0000", 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.636520887000003, 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0628312559999999, 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.513 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149574 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862969+0000", 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862969+0000", 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862969+0000", 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862972+0000", 2023-10-09T09:39:34.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.303e-06 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862979+0000", 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6980000000000001e-06 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862984+0000", 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9919999999999998e-06 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.865070+0000", 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020865319999999999 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.865091+0000", 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1090999999999998e-05 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.870002+0000", 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049112829999999998 2023-10-09T09:39:34.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.870011+0000", 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4940000000000003e-06 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.894829+0000", 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0248184119999999 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.895067+0000", 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023758199999999999 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.895164+0000", 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7553999999999995e-05 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911292+0000", 2023-10-09T09:39:34.519 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016127418000000001 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911332+0000", 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9999000000000001e-05 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925713+0000", 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014380828 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925761+0000", 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8671999999999997e-05 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925800+0000", 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9050000000000001e-05 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.521 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149602 8.f 8:f9b9e31f:::benchmark_data_smithi163_151167_object54369:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:17.909905+0000", 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.589584841999994, 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.015980871, 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149602 2023-10-09T09:39:34.522 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909905+0000", 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909905+0000", 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909904+0000", 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909912+0000", 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8910000000000001e-06 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909927+0000", 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4151000000000001e-05 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909936+0000", 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4939999999999992e-06 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.910014+0000", 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7976e-05 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.910044+0000", 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9462999999999999e-05 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.898508+0000", 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98846395499999995 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.526 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.898613+0000", 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000105225 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.898705+0000", 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2535999999999996e-05 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911362+0000", 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012656970999999999 2023-10-09T09:39:34.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911394+0000", 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1652999999999997e-05 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925810+0000", 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014416528 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925851+0000", 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0191999999999997e-05 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925886+0000", 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5478999999999997e-05 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149645 8.f 8:f2fa5649:::benchmark_data_smithi163_151167_object54412:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:17.985066+0000", 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.514424097000003, 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95494577199999997, 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149645 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.530 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985066+0000", 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985066+0000", 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.531 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985064+0000", 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985075+0000", 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0907000000000001e-05 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985091+0000", 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6419999999999999e-05 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985103+0000", 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1884e-05 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985505+0000", 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00040182999999999999 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985521+0000", 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6116e-05 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.906317+0000", 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.92079660500000005 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.906486+0000", 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000168765 2023-10-09T09:39:34.534 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.906654+0000", 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016745799999999999 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925894+0000", 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019240489999999999 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.535 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931545+0000", 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056503940000000004 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.939918+0000", 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0083738700000000003 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.939967+0000", 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9047e-05 2023-10-09T09:39:34.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940012+0000", 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4277999999999999e-05 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149674 8.f 8:f3b05e3f:::benchmark_data_smithi163_151167_object54441:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.032157+0000", 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.467333030999995, 2023-10-09T09:39:34.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.90795880600000001, 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149674 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032157+0000", 2023-10-09T09:39:34.538 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032157+0000", 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032155+0000", 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.539 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032164+0000", 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4780000000000008e-06 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032180+0000", 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6359e-05 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032188+0000", 2023-10-09T09:39:34.540 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2539999999999992e-06 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032235+0000", 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6653000000000002e-05 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032244+0000", 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0669999999999998e-06 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.541 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911421+0000", 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.87917706900000003 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911522+0000", 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000100942 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.542 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911614+0000", 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1802e-05 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925900+0000", 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014285613000000001 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931626+0000", 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0057267189999999999 2023-10-09T09:39:34.543 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940048+0000", 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0084219800000000008 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940084+0000", 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6093999999999997e-05 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.544 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940116+0000", 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1548999999999999e-05 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149676 8.f 8:f59131ee:::benchmark_data_smithi163_151167_object54443:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.035104+0000", 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.464385758999995, 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.90513551400000003, 2023-10-09T09:39:34.545 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149676 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035104+0000", 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.546 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035104+0000", 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035103+0000", 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.547 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035110+0000", 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0140000000000001e-06 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035125+0000", 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5302e-05 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035133+0000", 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8150000000000006e-06 2023-10-09T09:39:34.548 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035180+0000", 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6975999999999999e-05 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035189+0000", 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0259999999999996e-06 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.549 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.915840+0000", 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.88065113800000006 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.916025+0000", 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018522600000000001 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.916203+0000", 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000177276 2023-10-09T09:39:34.550 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931018+0000", 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014815399 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940147+0000", 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091291749999999998 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940174+0000", 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6531999999999999e-05 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940209+0000", 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5293999999999998e-05 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940240+0000", 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0859999999999999e-05 2023-10-09T09:39:34.552 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149684 8.f 8:f37ab3bd:::benchmark_data_smithi163_151167_object54451:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.051053+0000", 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.448436998999995, 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.91798173000000005, 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.553 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149684 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051053+0000", 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.554 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051053+0000", 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051051+0000", 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051062+0000", 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1054e-05 2023-10-09T09:39:34.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051080+0000", 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7592e-05 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051090+0000", 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0839000000000001e-05 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.556 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051167+0000", 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6117000000000003e-05 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051183+0000", 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6262000000000001e-05 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.921010+0000", 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.86982745500000003 2023-10-09T09:39:34.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.921180+0000", 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000169316 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.921331+0000", 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015102 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.558 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931026+0000", 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0096958809999999999 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940271+0000", 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0092446690000000005 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.968932+0000", 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028660354999999998 2023-10-09T09:39:34.559 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.968989+0000", 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7973000000000002e-05 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969035+0000", 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5339000000000001e-05 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.560 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149724 8.f 8:f2f8c3fa:::benchmark_data_smithi163_151167_object54491:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.114549+0000", 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.384940826999994, 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.85463660900000005, 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149724 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.561 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114549+0000", 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114549+0000", 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114547+0000", 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114556+0000", 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0010000000000007e-06 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114574+0000", 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7626999999999999e-05 2023-10-09T09:39:34.563 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114584+0000", 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.713e-06 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114664+0000", 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0255999999999999e-05 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.564 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114679+0000", 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4728e-05 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925931+0000", 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.81125247599999994 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.926033+0000", 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000102045 2023-10-09T09:39:34.565 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.926151+0000", 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000117413 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.944896+0000", 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018745458 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.566 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969082+0000", 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.02418582 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969113+0000", 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0744000000000002e-05 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969152+0000", 2023-10-09T09:39:34.567 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9651999999999999e-05 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969186+0000", 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3639000000000001e-05 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149746 8.f 8:ff8f4271:::benchmark_data_smithi163_151167_object54513:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.568 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.159624+0000", 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.339865810000006, 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.80969376900000001, 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149746 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.569 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159624+0000", 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159624+0000", 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159623+0000", 2023-10-09T09:39:34.570 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159632+0000", 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0610000000000002e-06 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159647+0000", 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5207e-05 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.571 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159655+0000", 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0669999999999992e-06 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159703+0000", 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8127999999999997e-05 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159712+0000", 2023-10-09T09:39:34.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8100000000000004e-06 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931662+0000", 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.77195008099999995 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931797+0000", 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013457599999999999 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.573 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931901+0000", 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010447599999999999 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.944903+0000", 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013002231 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969220+0000", 2023-10-09T09:39:34.574 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024316873999999999 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969249+0000", 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8609e-05 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969287+0000", 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8393999999999999e-05 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.575 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969318+0000", 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0932000000000001e-05 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149777 8.f 8:f1a53e60:::benchmark_data_smithi163_151167_object54544:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.216787+0000", 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.282702661000002, 2023-10-09T09:39:34.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.75266294300000003, 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149777 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216787+0000", 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.577 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216787+0000", 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216785+0000", 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999971 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.578 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216796+0000", 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0947e-05 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216818+0000", 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2759e-05 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216830+0000", 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1379000000000001e-05 2023-10-09T09:39:34.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216904+0000", 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4221999999999995e-05 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216918+0000", 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3568e-05 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.580 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.936247+0000", 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.719328996 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.936397+0000", 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000150385 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.936526+0000", 2023-10-09T09:39:34.581 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012890500000000001 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.964436+0000", 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.027910213999999999 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969351+0000", 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049151129999999996 2023-10-09T09:39:34.582 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969380+0000", 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8578999999999999e-05 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969418+0000", 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8563999999999999e-05 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.583 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969450+0000", 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2038000000000001e-05 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149811 8.f 8:fd4d8c61:::benchmark_data_smithi163_151167_object54578:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.278487+0000", 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.221003113999998, 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.691092178, 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.584 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149811 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278487+0000", 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.585 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278487+0000", 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278485+0000", 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278495+0000", 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0244000000000001e-05 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278515+0000", 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9905e-05 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278527+0000", 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2481e-05 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.587 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278600+0000", 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2135e-05 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278613+0000", 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3648e-05 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.588 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940291+0000", 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.661677403 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940420+0000", 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000129636 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940539+0000", 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000118744 2023-10-09T09:39:34.589 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.964447+0000", 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023907726000000001 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969485+0000", 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050383040000000004 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969514+0000", 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8507999999999999e-05 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969549+0000", 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5074999999999999e-05 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969579+0000", 2023-10-09T09:39:34.591 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0528999999999999e-05 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149812 8.f 8:f90f3522:::benchmark_data_smithi163_151167_object54579:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.282166+0000", 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.217323977999996, 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.70568355299999996, 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.592 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149812 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282166+0000", 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.593 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282166+0000", 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282164+0000", 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282175+0000", 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0552e-05 2023-10-09T09:39:34.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282194+0000", 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9919e-05 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282213+0000", 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8349999999999999e-05 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282279+0000", 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6113999999999998e-05 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282300+0000", 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1330000000000001e-05 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.944929+0000", 2023-10-09T09:39:34.596 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.66262830900000003 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.945067+0000", 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000138114 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.945189+0000", 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012192399999999999 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.597 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969612+0000", 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024423112 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969640+0000", 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8708e-05 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987752+0000", 2023-10-09T09:39:34.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018111795 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987804+0000", 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1712999999999998e-05 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987850+0000", 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5710000000000001e-05 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.599 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149816 8.f 8:fc3bc3c1:::benchmark_data_smithi163_151167_object54583:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.288122+0000", 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.211368453000006, 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.69983973899999996, 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.600 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149816 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288122+0000", 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288122+0000", 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288120+0000", 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288129+0000", 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9279999999999999e-06 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288146+0000", 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7521999999999999e-05 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288157+0000", 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0482999999999999e-05 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288221+0000", 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4632000000000001e-05 2023-10-09T09:39:34.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288232+0000", 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0434999999999999e-05 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.949547+0000", 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.66131544900000006 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.604 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.949764+0000", 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000216572 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.949895+0000", 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000131421 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969670+0000", 2023-10-09T09:39:34.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019774131 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969700+0000", 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0645000000000002e-05 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987896+0000", 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018195847000000001 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987930+0000", 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3992999999999998e-05 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987961+0000", 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1390999999999998e-05 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.607 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149861 8.f 8:f3f94ebf:::benchmark_data_smithi163_151167_object54628:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.407055+0000", 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 76.092435464000005, 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.60205851899999996, 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149861 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.608 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407055+0000", 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407055+0000", 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407053+0000", 2023-10-09T09:39:34.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407064+0000", 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1618e-05 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407091+0000", 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6477000000000001e-05 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407104+0000", 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3875e-05 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407229+0000", 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012451699999999999 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407257+0000", 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7826000000000001e-05 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.964488+0000", 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.55723115599999995 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.964665+0000", 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017701699999999999 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.964792+0000", 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000126797 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987889+0000", 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023097166999999998 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987967+0000", 2023-10-09T09:39:34.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8273000000000001e-05 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009012+0000", 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021045257000000001 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009071+0000", 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9008000000000002e-05 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.614 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009113+0000", 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1649000000000001e-05 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149888 8.f 8:f6339202:::benchmark_data_smithi163_151167_object54655:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.507613+0000", 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 75.991876731000005, 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.50201627400000004, 2023-10-09T09:39:34.615 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149888 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507613+0000", 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.616 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507613+0000", 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507611+0000", 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507621+0000", 2023-10-09T09:39:34.617 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.363e-06 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507638+0000", 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7513e-05 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507654+0000", 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5265e-05 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507713+0000", 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9049999999999999e-05 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507723+0000", 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0159999999999999e-05 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969722+0000", 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.46199886200000001 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969849+0000", 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012760500000000001 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969972+0000", 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012303800000000001 2023-10-09T09:39:34.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987972+0000", 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017999505999999998 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.988007+0000", 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5256999999999998e-05 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009540+0000", 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021533251999999999 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009588+0000", 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7685999999999997e-05 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009630+0000", 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1603000000000001e-05 2023-10-09T09:39:34.622 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149908 8.f 8:fcc39bf8:::benchmark_data_smithi163_151167_object54675:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.649138+0000", 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 75.850352380000004, 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.36085224100000002, 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:39:34.623 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149908 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649138+0000", 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649138+0000", 2023-10-09T09:39:34.624 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649135+0000", 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649148+0000", 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2284e-05 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.625 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649172+0000", 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4083e-05 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649188+0000", 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6019999999999999e-05 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649270+0000", 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2602999999999995e-05 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649285+0000", 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4171999999999999e-05 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.973652+0000", 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32436722299999998 2023-10-09T09:39:34.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.973763+0000", 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011120600000000001 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.973867+0000", 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0001041 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987977+0000", 2023-10-09T09:39:34.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01411041 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.988035+0000", 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7345000000000001e-05 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009683+0000", 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021648356000000001 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009934+0000", 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000250726 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009990+0000", 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6064999999999999e-05 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.630 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45122.0:189420 8.4 e1166/1163 8:23866f45:::benchmark_data_smithi163_151167_object94187:head v 1166'11857, mlcod=1151'5395)", 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:39:26.962808+0000", 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 7.5366824189999999, 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.30114892100000001, 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962808+0000", 2023-10-09T09:39:34.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962808+0000", 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962807+0000", 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962816+0000", 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3499999999999997e-06 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962822+0000", 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6170000000000003e-06 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962827+0000", 2023-10-09T09:39:34.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8930000000000004e-06 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.968913+0000", 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060862099999999999 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.968943+0000", 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9428999999999999e-05 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.263876+0000", 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.294933628 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.263957+0000", 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0199999999999998e-05 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45122.0:189423 8.a e1166/1163 8:5d6d4aa6:::benchmark_data_smithi163_151167_object94190:head v 1166'11568, mlcod=1166'11565)", 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:39:26.970683+0000", 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 7.5288072760000002, 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.267407742, 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970683+0000", 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970683+0000", 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970682+0000", 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970688+0000", 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8869999999999997e-06 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970700+0000", 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2157000000000001e-05 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970707+0000", 2023-10-09T09:39:34.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7039999999999994e-06 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970764+0000", 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6124e-05 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970789+0000", 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5814000000000001e-05 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.978766+0000", 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079765459999999993 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.238091+0000", 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.259324634 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45122.0:189421 8.e e1166/1150 8:7b02ee27:::benchmark_data_smithi163_151167_object94188:head v 1166'12071, mlcod=1166'12066)", 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:39:26.970736+0000", 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 7.5287545790000001, 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28797570300000003, 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970736+0000", 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970736+0000", 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970735+0000", 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970740+0000", 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7469999999999996e-06 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970747+0000", 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2110000000000001e-06 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970752+0000", 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4890000000000003e-06 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970804+0000", 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1493999999999999e-05 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970823+0000", 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8776000000000001e-05 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.258673+0000", 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28785032500000002 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.258711+0000", 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8142000000000002e-05 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45122.0:189429 8.e e1166/1150 8:7a233770:::benchmark_data_smithi163_151167_object94196:head v 1166'12072, mlcod=1166'12068)", 2023-10-09T09:39:34.645 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:39:26.975993+0000", 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 7.5234975229999996, 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28274761199999998, 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.975993+0000", 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.646 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.975993+0000", 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.975991+0000", 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.975998+0000", 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4729999999999997e-06 2023-10-09T09:39:34.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.238131+0000", 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26213298400000001 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.238157+0000", 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6132999999999999e-05 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.238276+0000", 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011887699999999999 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.238347+0000", 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0772000000000006e-05 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.258717+0000", 2023-10-09T09:39:34.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020370558 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.258740+0000", 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.304e-05 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:39:34.650 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:39:34.651 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:34.652 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:39:34.652 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:39:34.652 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:39:34.652 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:39:34.652 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:39:34.652 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:39:34.672 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:39:34.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:34.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:39:34.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:39:34.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:39:34.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:39:34.675 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:39:34.675 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:39:34.675 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:39:34.675 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:39:34.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:34.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:34.777 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:34.808 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:39:34.811 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:34.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:34.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:34.914 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:34.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:34.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:34.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:35.046 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:35.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:34 smithi163 bash[75425]: audit 2023-10-09T09:39:34.465833+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.163:0/3878721669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:39:35.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:34 smithi163 bash[75425]: audit 2023-10-09T09:39:34.734666+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.163:0/1992368052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:39:35.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:34 smithi163 bash[75425]: audit 2023-10-09T09:39:34.735204+0000 mon.a (mon.0) 1927 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:39:35.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:34 smithi163 bash[78107]: audit 2023-10-09T09:39:34.465833+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.163:0/3878721669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:39:35.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:34 smithi163 bash[78107]: audit 2023-10-09T09:39:34.734666+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.163:0/1992368052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:39:35.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:34 smithi163 bash[78107]: audit 2023-10-09T09:39:34.735204+0000 mon.a (mon.0) 1927 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:39:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:34 smithi181 bash[59345]: audit 2023-10-09T09:39:34.465833+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.163:0/3878721669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:39:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:34 smithi181 bash[59345]: audit 2023-10-09T09:39:34.734666+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.163:0/1992368052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:39:35.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:34 smithi181 bash[59345]: audit 2023-10-09T09:39:34.735204+0000 mon.a (mon.0) 1927 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:39:35.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:35.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:35.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:35.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:35.204 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:35.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:35.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:35.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:35.335 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:35.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:35.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:35.438 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:35.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:35.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:35.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:35.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:35.597 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:35.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:35.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:35.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:35.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:35.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:35.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:35.825 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:35.854 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:35.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:35.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:35.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:35.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[78107]: cluster 2023-10-09T09:39:33.938327+0000 mgr.y (mgr.34107) 1003 : cluster 0 pgmap v1771: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 80 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 9330/72 objects degraded (12958.333%); 0 B/s, 59 objects/s recovering 2023-10-09T09:39:35.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[78107]: cluster 2023-10-09T09:39:34.794754+0000 mon.a (mon.0) 1928 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:39:35.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[78107]: cluster 2023-10-09T09:39:34.797857+0000 osd.5 (osd.5) 736 : cluster 3 failed to encode map e1169 with expected crc 2023-10-09T09:39:35.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[78107]: cluster 2023-10-09T09:39:34.798376+0000 osd.6 (osd.6) 452 : cluster 3 failed to encode map e1169 with expected crc 2023-10-09T09:39:35.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[78107]: audit 2023-10-09T09:39:34.801962+0000 mon.a (mon.0) 1929 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:39:35.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[78107]: cluster 2023-10-09T09:39:34.802050+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e1169: 8 total, 8 up, 7 in 2023-10-09T09:39:35.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[78107]: cluster 2023-10-09T09:39:35.719603+0000 mon.a (mon.0) 1931 : cluster 3 Health check update: Degraded data redundancy: 9330/72 objects degraded (12958.333%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:35.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[75425]: cluster 2023-10-09T09:39:33.938327+0000 mgr.y (mgr.34107) 1003 : cluster 0 pgmap v1771: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 80 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 9330/72 objects degraded (12958.333%); 0 B/s, 59 objects/s recovering 2023-10-09T09:39:35.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[75425]: cluster 2023-10-09T09:39:34.794754+0000 mon.a (mon.0) 1928 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:39:35.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[75425]: cluster 2023-10-09T09:39:34.797857+0000 osd.5 (osd.5) 736 : cluster 3 failed to encode map e1169 with expected crc 2023-10-09T09:39:35.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[75425]: cluster 2023-10-09T09:39:34.798376+0000 osd.6 (osd.6) 452 : cluster 3 failed to encode map e1169 with expected crc 2023-10-09T09:39:35.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[75425]: audit 2023-10-09T09:39:34.801962+0000 mon.a (mon.0) 1929 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:39:35.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[75425]: cluster 2023-10-09T09:39:34.802050+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e1169: 8 total, 8 up, 7 in 2023-10-09T09:39:35.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:35 smithi163 bash[75425]: cluster 2023-10-09T09:39:35.719603+0000 mon.a (mon.0) 1931 : cluster 3 Health check update: Degraded data redundancy: 9330/72 objects degraded (12958.333%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:35.988 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:35.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:35.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:35 smithi181 bash[59345]: cluster 2023-10-09T09:39:33.938327+0000 mgr.y (mgr.34107) 1003 : cluster 0 pgmap v1771: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 80 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 9330/72 objects degraded (12958.333%); 0 B/s, 59 objects/s recovering 2023-10-09T09:39:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:35 smithi181 bash[59345]: cluster 2023-10-09T09:39:34.794754+0000 mon.a (mon.0) 1928 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:39:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:35 smithi181 bash[59345]: cluster 2023-10-09T09:39:34.797857+0000 osd.5 (osd.5) 736 : cluster 3 failed to encode map e1169 with expected crc 2023-10-09T09:39:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:35 smithi181 bash[59345]: cluster 2023-10-09T09:39:34.798376+0000 osd.6 (osd.6) 452 : cluster 3 failed to encode map e1169 with expected crc 2023-10-09T09:39:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:35 smithi181 bash[59345]: audit 2023-10-09T09:39:34.801962+0000 mon.a (mon.0) 1929 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:39:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:35 smithi181 bash[59345]: cluster 2023-10-09T09:39:34.802050+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e1169: 8 total, 8 up, 7 in 2023-10-09T09:39:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:35 smithi181 bash[59345]: cluster 2023-10-09T09:39:35.719603+0000 mon.a (mon.0) 1931 : cluster 3 Health check update: Degraded data redundancy: 9330/72 objects degraded (12958.333%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:39:36.092 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:36.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:36.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:36.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:36.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:36.244 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:36.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:36.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:36.347 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:36.373 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:36.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:36.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:36.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:36.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:36.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:36.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:36.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:36.630 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:36.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:36.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:36.733 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:36.764 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:36.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:36.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:36.868 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:36.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:36.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:36.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:37.001 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:37.030 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:37.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:37.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:37.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:37.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:37.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:37.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:37.262 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:37.292 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:37.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:37.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:37.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:37.422 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:37.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:37.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:37.526 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:37.550 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:37.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:37.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:37.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:37.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:37.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:37.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:37.781 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:37.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:37.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:37.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:37.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:37.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:37.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:37.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:37.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:37 smithi163 bash[78107]: cluster 2023-10-09T09:39:35.939286+0000 mgr.y (mgr.34107) 1004 : cluster 0 pgmap v1773: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 8889/72 objects degraded (12345.833%); 0 B/s, 74 objects/s recovering 2023-10-09T09:39:37.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:37 smithi163 bash[75425]: cluster 2023-10-09T09:39:35.939286+0000 mgr.y (mgr.34107) 1004 : cluster 0 pgmap v1773: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 8889/72 objects degraded (12345.833%); 0 B/s, 74 objects/s recovering 2023-10-09T09:39:38.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:38.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:37 smithi181 bash[59345]: cluster 2023-10-09T09:39:35.939286+0000 mgr.y (mgr.34107) 1004 : cluster 0 pgmap v1773: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 8889/72 objects degraded (12345.833%); 0 B/s, 74 objects/s recovering 2023-10-09T09:39:38.073 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:38.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:38.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:38.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T09:39:38.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:38.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:38.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:38.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:38.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:38.333 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:38.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:38.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:38.464 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:38.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:38.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:38.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:38.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:38.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:38.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:38.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:38.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:38.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:38.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:38.825 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:38.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:38.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:38.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:38.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:38.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:38 smithi163 bash[75425]: audit 2023-10-09T09:39:38.208971+0000 mon.a (mon.0) 1932 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:39:38.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:38 smithi163 bash[78107]: audit 2023-10-09T09:39:38.208971+0000 mon.a (mon.0) 1932 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:39:38.988 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:38.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:38.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:39.091 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:39.133 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:39.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:39.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:39.236 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:39.247 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:38 smithi181 bash[59345]: audit 2023-10-09T09:39:38.208971+0000 mon.a (mon.0) 1932 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:39:39.266 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:39.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:39.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:39.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:39.397 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:39.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:39.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:39.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:39.529 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:39.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:39.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:39.662 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:39.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:39.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:39.766 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:39.793 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:39:39.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:39.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:39.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:39.797 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:39:39.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:39.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:39.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:39.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:40.026 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:40.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:39 smithi181 bash[59345]: cluster 2023-10-09T09:39:37.940175+0000 mgr.y (mgr.34107) 1005 : cluster 0 pgmap v1774: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 32 op/s; 8889/72 objects degraded (12345.833%); 0 B/s, 90 objects/s recovering 2023-10-09T09:39:40.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:40.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:40.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:40.154 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:40.183 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:40.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:40.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:40.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:39 smithi163 bash[75425]: cluster 2023-10-09T09:39:37.940175+0000 mgr.y (mgr.34107) 1005 : cluster 0 pgmap v1774: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 32 op/s; 8889/72 objects degraded (12345.833%); 0 B/s, 90 objects/s recovering 2023-10-09T09:39:40.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:39 smithi163 bash[78107]: cluster 2023-10-09T09:39:37.940175+0000 mgr.y (mgr.34107) 1005 : cluster 0 pgmap v1774: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 32 op/s; 8889/72 objects degraded (12345.833%); 0 B/s, 90 objects/s recovering 2023-10-09T09:39:40.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:40.313 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:40.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:40.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:40.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:40.440 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:40.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:40.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:40.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:40.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:40.679 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:40.703 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:40.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:40.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:40.824 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1774,"stamp":"2023-10-09T09:39:37.939650+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8889,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":190722,"num_write_kb":6097166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13767,"num_bytes_recovered":767690504,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120916,"ondisk_log_size":120916,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3236056,"kb_used_data":743904,"kb_used_omap":565,"kb_used_meta":2491530,"kb_avail":652980008,"statfs":{"total":671965249536,"available":668651528192,"internally_reserved":0,"allocated":761757696,"data_stored":756158149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":579342,"internal_metadata":2551326962},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-13303808,"num_objects":-203,"num_object_clones":0,"num_object_copies":-609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-441,"num_objects_misplaced":-67,"num_objects_unfound":0,"num_objects_dirty":-203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":203,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":557,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.154624"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1154,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.837487+0000","last_change":"2023-10-09T09:36:43.418759+0000","last_active":"2023-10-09T09:39:32.837487+0000","last_peered":"2023-10-09T09:39:32.837487+0000","last_clean":"2023-10-09T09:39:32.837487+0000","last_became_active":"2023-10-09T09:36:43.418371+0000","last_became_peered":"2023-10-09T09:36:43.418371+0000","last_unstale":"2023-10-09T09:39:32.837487+0000","last_undegraded":"2023-10-09T09:39:32.837487+0000","last_fullsized":"2023-10-09T09:39:32.837487+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":39,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.856671+0000","last_change":"2023-10-09T09:35:48.698707+0000","last_active":"2023-10-09T09:39:31.856671+0000","last_peered":"2023-10-09T09:39:31.856671+0000","last_clean":"2023-10-09T09:39:31.856671+0000","last_became_active":"2023-10-09T09:35:48.698382+0000","last_became_peered":"2023-10-09T09:35:48.698382+0000","last_unstale":"2023-10-09T09:39:31.856671+0000","last_undegraded":"2023-10-09T09:39:31.856671+0000","last_fullsized":"2023-10-09T09:39:31.856671+0000","mapping_epoch":1140,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:01.722355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1108,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.788757+0000","last_change":"2023-10-09T09:15:36.181688+0000","last_active":"2023-10-09T09:39:31.788757+0000","last_peered":"2023-10-09T09:39:31.788757+0000","last_clean":"2023-10-09T09:39:31.788757+0000","last_became_active":"2023-10-09T09:15:36.181167+0000","last_became_peered":"2023-10-09T09:15:36.181167+0000","last_unstale":"2023-10-09T09:39:31.788757+0000","last_undegraded":"2023-10-09T09:39:31.788757+0000","last_fullsized":"2023-10-09T09:39:31.788757+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.613026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1091,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807292+0000","last_change":"2023-10-09T09:35:18.684800+0000","last_active":"2023-10-09T09:39:32.807292+0000","last_peered":"2023-10-09T09:39:32.807292+0000","last_clean":"2023-10-09T09:39:32.807292+0000","last_became_active":"2023-10-09T09:35:18.681331+0000","last_became_peered":"2023-10-09T09:35:18.681331+0000","last_unstale":"2023-10-09T09:39:32.807292+0000","last_undegraded":"2023-10-09T09:39:32.807292+0000","last_fullsized":"2023-10-09T09:39:32.807292+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:38.385662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":633,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807178+0000","last_change":"2023-10-09T09:35:18.670124+0000","last_active":"2023-10-09T09:39:32.807178+0000","last_peered":"2023-10-09T09:39:32.807178+0000","last_clean":"2023-10-09T09:39:32.807178+0000","last_became_active":"2023-10-09T09:35:18.668954+0000","last_became_peered":"2023-10-09T09:35:18.668954+0000","last_unstale":"2023-10-09T09:39:32.807178+0000","last_undegraded":"2023-10-09T09:39:32.807178+0000","last_fullsized":"2023-10-09T09:39:32.807178+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:42.439629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1109,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.805242+0000","last_change":"2023-10-09T09:39:12.967452+0000","last_active":"2023-10-09T09:39:32.805242+0000","last_peered":"2023-10-09T09:39:32.805242+0000","last_clean":"2023-10-09T09:39:32.805242+0000","last_became_active":"2023-10-09T09:39:12.966943+0000","last_became_peered":"2023-10-09T09:39:12.966943+0000","last_unstale":"2023-10-09T09:39:32.805242+0000","last_undegraded":"2023-10-09T09:39:32.805242+0000","last_fullsized":"2023-10-09T09:39:32.805242+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1099,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807352+0000","last_change":"2023-10-09T09:39:12.968639+0000","last_active":"2023-10-09T09:39:32.807352+0000","last_peered":"2023-10-09T09:39:32.807352+0000","last_clean":"2023-10-09T09:39:32.807352+0000","last_became_active":"2023-10-09T09:39:12.968211+0000","last_became_peered":"2023-10-09T09:39:12.968211+0000","last_unstale":"2023-10-09T09:39:32.807352+0000","last_undegraded":"2023-10-09T09:39:32.807352+0000","last_fullsized":"2023-10-09T09:39:32.807352+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:27:48.289591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1070,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.794867+0000","last_change":"2023-10-09T09:15:36.172938+0000","last_active":"2023-10-09T09:39:31.794867+0000","last_peered":"2023-10-09T09:39:31.794867+0000","last_clean":"2023-10-09T09:39:31.794867+0000","last_became_active":"2023-10-09T09:15:36.172649+0000","last_became_peered":"2023-10-09T09:15:36.172649+0000","last_unstale":"2023-10-09T09:39:31.794867+0000","last_undegraded":"2023-10-09T09:39:31.794867+0000","last_fullsized":"2023-10-09T09:39:31.794867+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:29.151028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":570,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807241+0000","last_change":"2023-10-09T09:39:12.956017+0000","last_active":"2023-10-09T09:39:32.807241+0000","last_peered":"2023-10-09T09:39:32.807241+0000","last_clean":"2023-10-09T09:39:32.807241+0000","last_became_active":"2023-10-09T09:39:12.955588+0000","last_became_peered":"2023-10-09T09:39:12.955588+0000","last_unstale":"2023-10-09T09:39:32.807241+0000","last_undegraded":"2023-10-09T09:39:32.807241+0000","last_fullsized":"2023-10-09T09:39:32.807241+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":629,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806359+0000","last_change":"2023-10-09T09:34:03.852834+0000","last_active":"2023-10-09T09:39:32.806359+0000","last_peered":"2023-10-09T09:39:32.806359+0000","last_clean":"2023-10-09T09:39:32.806359+0000","last_became_active":"2023-10-09T09:34:03.852517+0000","last_became_peered":"2023-10-09T09:34:03.852517+0000","last_unstale":"2023-10-09T09:39:32.806359+0000","last_undegraded":"2023-10-09T09:39:32.806359+0000","last_fullsized":"2023-10-09T09:39:32.806359+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":1100,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807215+0000","last_change":"2023-10-09T09:15:36.178932+0000","last_active":"2023-10-09T09:39:32.807215+0000","last_peered":"2023-10-09T09:39:32.807215+0000","last_clean":"2023-10-09T09:39:32.807215+0000","last_became_active":"2023-10-09T09:15:36.178428+0000","last_became_peered":"2023-10-09T09:15:36.178428+0000","last_unstale":"2023-10-09T09:39:32.807215+0000","last_undegraded":"2023-10-09T09:39:32.807215+0000","last_fullsized":"2023-10-09T09:39:32.807215+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1126,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.789245+0000","last_change":"2023-10-09T09:36:43.421750+0000","last_active":"2023-10-09T09:39:31.789245+0000","last_peered":"2023-10-09T09:39:31.789245+0000","last_clean":"2023-10-09T09:39:31.789245+0000","last_became_active":"2023-10-09T09:36:43.421389+0000","last_became_peered":"2023-10-09T09:36:43.421389+0000","last_unstale":"2023-10-09T09:39:31.789245+0000","last_undegraded":"2023-10-09T09:39:31.789245+0000","last_fullsized":"2023-10-09T09:39:31.789245+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:15:05.597010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1127,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.838182+0000","last_change":"2023-10-09T09:03:24.306257+0000","last_active":"2023-10-09T09:39:32.838182+0000","last_peered":"2023-10-09T09:39:32.838182+0000","last_clean":"2023-10-09T09:39:32.838182+0000","last_became_active":"2023-10-09T09:03:24.305824+0000","last_became_peered":"2023-10-09T09:03:24.305824+0000","last_unstale":"2023-10-09T09:39:32.838182+0000","last_undegraded":"2023-10-09T09:39:32.838182+0000","last_fullsized":"2023-10-09T09:39:32.838182+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1119,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.838021+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:39:32.838021+0000","last_peered":"2023-10-09T09:39:32.838021+0000","last_clean":"2023-10-09T09:39:32.838021+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:39:32.838021+0000","last_undegraded":"2023-10-09T09:39:32.838021+0000","last_fullsized":"2023-10-09T09:39:32.838021+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1129,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806720+0000","last_change":"2023-10-09T09:36:43.512444+0000","last_active":"2023-10-09T09:39:32.806720+0000","last_peered":"2023-10-09T09:39:32.806720+0000","last_clean":"2023-10-09T09:39:32.806720+0000","last_became_active":"2023-10-09T09:36:43.421525+0000","last_became_peered":"2023-10-09T09:36:43.421525+0000","last_unstale":"2023-10-09T09:39:32.806720+0000","last_undegraded":"2023-10-09T09:39:32.806720+0000","last_fullsized":"2023-10-09T09:39:32.806720+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":626,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.789176+0000","last_change":"2023-10-09T09:34:03.848913+0000","last_active":"2023-10-09T09:39:31.789176+0000","last_peered":"2023-10-09T09:39:31.789176+0000","last_clean":"2023-10-09T09:39:31.789176+0000","last_became_active":"2023-10-09T09:34:03.848474+0000","last_became_peered":"2023-10-09T09:34:03.848474+0000","last_unstale":"2023-10-09T09:39:31.789176+0000","last_undegraded":"2023-10-09T09:39:31.789176+0000","last_fullsized":"2023-10-09T09:39:31.789176+0000","mapping_epoch":1125,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:59.618935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1090,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806700+0000","last_change":"2023-10-09T09:35:18.670309+0000","last_active":"2023-10-09T09:39:32.806700+0000","last_peered":"2023-10-09T09:39:32.806700+0000","last_clean":"2023-10-09T09:39:32.806700+0000","last_became_active":"2023-10-09T09:35:18.669706+0000","last_became_peered":"2023-10-09T09:35:18.669706+0000","last_unstale":"2023-10-09T09:39:32.806700+0000","last_undegraded":"2023-10-09T09:39:32.806700+0000","last_fullsized":"2023-10-09T09:39:32.806700+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:39.673273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":637,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807530+0000","last_change":"2023-10-09T09:39:12.959569+0000","last_active":"2023-10-09T09:39:32.807530+0000","last_peered":"2023-10-09T09:39:32.807530+0000","last_clean":"2023-10-09T09:39:32.807530+0000","last_became_active":"2023-10-09T09:39:12.959233+0000","last_became_peered":"2023-10-09T09:39:12.959233+0000","last_unstale":"2023-10-09T09:39:32.807530+0000","last_undegraded":"2023-10-09T09:39:32.807530+0000","last_fullsized":"2023-10-09T09:39:32.807530+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1082,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806386+0000","last_change":"2023-10-09T09:03:24.303864+0000","last_active":"2023-10-09T09:39:32.806386+0000","last_peered":"2023-10-09T09:39:32.806386+0000","last_clean":"2023-10-09T09:39:32.806386+0000","last_became_active":"2023-10-09T09:03:24.303485+0000","last_became_peered":"2023-10-09T09:03:24.303485+0000","last_unstale":"2023-10-09T09:39:32.806386+0000","last_undegraded":"2023-10-09T09:39:32.806386+0000","last_fullsized":"2023-10-09T09:39:32.806386+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1089,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807124+0000","last_change":"2023-10-09T09:35:18.685281+0000","last_active":"2023-10-09T09:39:32.807124+0000","last_peered":"2023-10-09T09:39:32.807124+0000","last_clean":"2023-10-09T09:39:32.807124+0000","last_became_active":"2023-10-09T09:35:18.684094+0000","last_became_peered":"2023-10-09T09:35:18.684094+0000","last_unstale":"2023-10-09T09:39:32.807124+0000","last_undegraded":"2023-10-09T09:39:32.807124+0000","last_fullsized":"2023-10-09T09:39:32.807124+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:13.291210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1144,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.804905+0000","last_change":"2023-10-09T09:39:12.978373+0000","last_active":"2023-10-09T09:39:32.804905+0000","last_peered":"2023-10-09T09:39:32.804905+0000","last_clean":"2023-10-09T09:39:32.804905+0000","last_became_active":"2023-10-09T09:39:12.957741+0000","last_became_peered":"2023-10-09T09:39:12.957741+0000","last_unstale":"2023-10-09T09:39:32.804905+0000","last_undegraded":"2023-10-09T09:39:32.804905+0000","last_fullsized":"2023-10-09T09:39:32.804905+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1113,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.794891+0000","last_change":"2023-10-09T09:35:48.703715+0000","last_active":"2023-10-09T09:39:31.794891+0000","last_peered":"2023-10-09T09:39:31.794891+0000","last_clean":"2023-10-09T09:39:31.794891+0000","last_became_active":"2023-10-09T09:35:48.703081+0000","last_became_peered":"2023-10-09T09:35:48.703081+0000","last_unstale":"2023-10-09T09:39:31.794891+0000","last_undegraded":"2023-10-09T09:39:31.794891+0000","last_fullsized":"2023-10-09T09:39:31.794891+0000","mapping_epoch":1140,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:41:38.594618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"1166'11924","reported_seq":14565,"reported_epoch":1169,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T09:39:36.188045+0000","last_change":"2023-10-09T09:39:31.058672+0000","last_active":"2023-10-09T09:39:36.188045+0000","last_peered":"2023-10-09T09:39:36.188045+0000","last_clean":"2023-10-09T09:35:47.668893+0000","last_became_active":"2023-10-09T09:39:14.320930+0000","last_became_peered":"2023-10-09T09:39:14.320930+0000","last_unstale":"2023-10-09T09:39:36.188045+0000","last_undegraded":"2023-10-09T09:39:14.222345+0000","last_fullsized":"2023-10-09T09:39:13.951168+0000","mapping_epoch":1163,"log_start":"1145'4715","ondisk_log_start":"1145'4715","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7209,"log_dups_size":0,"ondisk_log_size":7209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":567,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11924,"num_write_kb":381568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1223,"num_bytes_recovered":72220672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"2,4","objects":-567}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":1113,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807350+0000","last_change":"2023-10-09T09:35:18.685969+0000","last_active":"2023-10-09T09:39:32.807350+0000","last_peered":"2023-10-09T09:39:32.807350+0000","last_clean":"2023-10-09T09:39:32.807350+0000","last_became_active":"2023-10-09T09:35:18.684984+0000","last_became_peered":"2023-10-09T09:35:18.684984+0000","last_unstale":"2023-10-09T09:39:32.807350+0000","last_undegraded":"2023-10-09T09:39:32.807350+0000","last_fullsized":"2023-10-09T09:39:32.807350+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":19,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.808111+0000","last_change":"2023-10-09T09:39:12.962660+0000","last_active":"2023-10-09T09:39:32.808111+0000","last_peered":"2023-10-09T09:39:32.808111+0000","last_clean":"2023-10-09T09:39:32.808111+0000","last_became_active":"2023-10-09T09:39:12.962233+0000","last_became_peered":"2023-10-09T09:39:12.962233+0000","last_unstale":"2023-10-09T09:39:32.808111+0000","last_undegraded":"2023-10-09T09:39:32.808111+0000","last_fullsized":"2023-10-09T09:39:32.808111+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"1166'11916","reported_seq":11961,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.805103+0000","last_change":"2023-10-09T09:35:11.923645+0000","last_active":"2023-10-09T09:39:32.805103+0000","last_peered":"2023-10-09T09:39:32.805103+0000","last_clean":"2023-10-09T09:39:32.805103+0000","last_became_active":"2023-10-09T09:35:11.923211+0000","last_became_peered":"2023-10-09T09:35:11.923211+0000","last_unstale":"2023-10-09T09:39:32.805103+0000","last_undegraded":"2023-10-09T09:39:32.805103+0000","last_fullsized":"2023-10-09T09:39:32.805103+0000","mapping_epoch":1132,"log_start":"1144'4171","ondisk_log_start":"1144'4171","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7745,"log_dups_size":0,"ondisk_log_size":7745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11916,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":1099,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807597+0000","last_change":"2023-10-09T09:14:40.485065+0000","last_active":"2023-10-09T09:39:32.807597+0000","last_peered":"2023-10-09T09:39:32.807597+0000","last_clean":"2023-10-09T09:39:32.807597+0000","last_became_active":"2023-10-09T09:14:40.484656+0000","last_became_peered":"2023-10-09T09:14:40.484656+0000","last_unstale":"2023-10-09T09:39:32.807597+0000","last_undegraded":"2023-10-09T09:39:32.807597+0000","last_fullsized":"2023-10-09T09:39:32.807597+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1134,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.837116+0000","last_change":"2023-10-09T09:15:36.183656+0000","last_active":"2023-10-09T09:39:32.837116+0000","last_peered":"2023-10-09T09:39:32.837116+0000","last_clean":"2023-10-09T09:39:32.837116+0000","last_became_active":"2023-10-09T09:15:36.183289+0000","last_became_peered":"2023-10-09T09:15:36.183289+0000","last_unstale":"2023-10-09T09:39:32.837116+0000","last_undegraded":"2023-10-09T09:39:32.837116+0000","last_fullsized":"2023-10-09T09:39:32.837116+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"1166'11698","reported_seq":11742,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806979+0000","last_change":"2023-10-09T09:35:18.669719+0000","last_active":"2023-10-09T09:39:32.806979+0000","last_peered":"2023-10-09T09:39:32.806979+0000","last_clean":"2023-10-09T09:39:32.806979+0000","last_became_active":"2023-10-09T09:35:18.669369+0000","last_became_peered":"2023-10-09T09:35:18.669369+0000","last_unstale":"2023-10-09T09:39:32.806979+0000","last_undegraded":"2023-10-09T09:39:32.806979+0000","last_fullsized":"2023-10-09T09:39:32.806979+0000","mapping_epoch":1136,"log_start":"1143'3773","ondisk_log_start":"1143'3773","created":1132,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7925,"log_dups_size":0,"ondisk_log_size":7925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:18.617658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11698,"num_write_kb":374336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1096,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806817+0000","last_change":"2023-10-09T09:39:13.019253+0000","last_active":"2023-10-09T09:39:32.806817+0000","last_peered":"2023-10-09T09:39:32.806817+0000","last_clean":"2023-10-09T09:39:32.806817+0000","last_became_active":"2023-10-09T09:39:12.978298+0000","last_became_peered":"2023-10-09T09:39:12.978298+0000","last_unstale":"2023-10-09T09:39:32.806817+0000","last_undegraded":"2023-10-09T09:39:32.806817+0000","last_fullsized":"2023-10-09T09:39:32.806817+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1101,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.804647+0000","last_change":"2023-10-09T09:14:40.494559+0000","last_active":"2023-10-09T09:39:32.804647+0000","last_peered":"2023-10-09T09:39:32.804647+0000","last_clean":"2023-10-09T09:39:32.804647+0000","last_became_active":"2023-10-09T09:14:40.494237+0000","last_became_peered":"2023-10-09T09:14:40.494237+0000","last_unstale":"2023-10-09T09:39:32.804647+0000","last_undegraded":"2023-10-09T09:39:32.804647+0000","last_fullsized":"2023-10-09T09:39:32.804647+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1166'11960","reported_seq":12007,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.805641+0000","last_change":"2023-10-09T09:35:12.107440+0000","last_active":"2023-10-09T09:39:32.805641+0000","last_peered":"2023-10-09T09:39:32.805641+0000","last_clean":"2023-10-09T09:39:32.805641+0000","last_became_active":"2023-10-09T09:35:12.106671+0000","last_became_peered":"2023-10-09T09:35:12.106671+0000","last_unstale":"2023-10-09T09:39:32.805641+0000","last_undegraded":"2023-10-09T09:39:32.805641+0000","last_fullsized":"2023-10-09T09:39:32.805641+0000","mapping_epoch":1132,"log_start":"1148'5272","ondisk_log_start":"1148'5272","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":6688,"log_dups_size":0,"ondisk_log_size":6688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:21.342322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11960,"num_write_kb":382720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":638,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.805387+0000","last_change":"2023-10-09T09:39:12.957499+0000","last_active":"2023-10-09T09:39:32.805387+0000","last_peered":"2023-10-09T09:39:32.805387+0000","last_clean":"2023-10-09T09:39:32.805387+0000","last_became_active":"2023-10-09T09:39:12.955785+0000","last_became_peered":"2023-10-09T09:39:12.955785+0000","last_unstale":"2023-10-09T09:39:32.805387+0000","last_undegraded":"2023-10-09T09:39:32.805387+0000","last_fullsized":"2023-10-09T09:39:32.805387+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1074,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806737+0000","last_change":"2023-10-09T09:35:18.681438+0000","last_active":"2023-10-09T09:39:32.806737+0000","last_peered":"2023-10-09T09:39:32.806737+0000","last_clean":"2023-10-09T09:39:32.806737+0000","last_became_active":"2023-10-09T09:35:18.680806+0000","last_became_peered":"2023-10-09T09:35:18.680806+0000","last_unstale":"2023-10-09T09:39:32.806737+0000","last_undegraded":"2023-10-09T09:39:32.806737+0000","last_fullsized":"2023-10-09T09:39:32.806737+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:30.669832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"1166'11864","reported_seq":11916,"reported_epoch":1167,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:39:31.788644+0000","last_change":"2023-10-09T09:39:14.227009+0000","last_active":"2023-10-09T09:39:31.788644+0000","last_peered":"2023-10-09T09:39:31.788644+0000","last_clean":"2023-10-09T09:39:11.928065+0000","last_became_active":"2023-10-09T09:39:14.190093+0000","last_became_peered":"2023-10-09T09:39:14.190093+0000","last_unstale":"2023-10-09T09:39:31.788644+0000","last_undegraded":"2023-10-09T09:39:31.788644+0000","last_fullsized":"2023-10-09T09:39:31.788644+0000","mapping_epoch":1163,"log_start":"1148'5140","ondisk_log_start":"1148'5140","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":6724,"log_dups_size":0,"ondisk_log_size":6724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:30.958424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11864,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,5],"avail_no_missing":["1","3","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":78,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806837+0000","last_change":"2023-10-09T09:35:18.683909+0000","last_active":"2023-10-09T09:39:32.806837+0000","last_peered":"2023-10-09T09:39:32.806837+0000","last_clean":"2023-10-09T09:39:32.806837+0000","last_became_active":"2023-10-09T09:35:18.678833+0000","last_became_peered":"2023-10-09T09:35:18.678833+0000","last_unstale":"2023-10-09T09:39:32.806837+0000","last_undegraded":"2023-10-09T09:39:32.806837+0000","last_fullsized":"2023-10-09T09:39:32.806837+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:47.275731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1119,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.838270+0000","last_change":"2023-10-09T09:15:36.180442+0000","last_active":"2023-10-09T09:39:32.838270+0000","last_peered":"2023-10-09T09:39:32.838270+0000","last_clean":"2023-10-09T09:39:32.838270+0000","last_became_active":"2023-10-09T09:15:36.179962+0000","last_became_peered":"2023-10-09T09:15:36.179962+0000","last_unstale":"2023-10-09T09:39:32.838270+0000","last_undegraded":"2023-10-09T09:39:32.838270+0000","last_fullsized":"2023-10-09T09:39:32.838270+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"1166'12182","reported_seq":18292,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.841894+0000","last_change":"2023-10-09T09:39:32.841894+0000","last_active":"2023-10-09T09:39:32.841894+0000","last_peered":"2023-10-09T09:39:32.841894+0000","last_clean":"2023-10-09T09:39:32.841894+0000","last_became_active":"2023-10-09T09:39:32.811444+0000","last_became_peered":"2023-10-09T09:39:32.811444+0000","last_unstale":"2023-10-09T09:39:32.841894+0000","last_undegraded":"2023-10-09T09:39:32.841894+0000","last_fullsized":"2023-10-09T09:39:32.841894+0000","mapping_epoch":1167,"log_start":"1148'5468","ondisk_log_start":"1148'5468","created":1132,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":6714,"log_dups_size":0,"ondisk_log_size":6714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:42.283190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12182,"num_write_kb":389824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3014,"num_bytes_recovered":72155136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1102,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.804906+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:39:32.804906+0000","last_peered":"2023-10-09T09:39:32.804906+0000","last_clean":"2023-10-09T09:39:32.804906+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:39:32.804906+0000","last_undegraded":"2023-10-09T09:39:32.804906+0000","last_fullsized":"2023-10-09T09:39:32.804906+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1152,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.838084+0000","last_change":"2023-10-09T09:36:43.419348+0000","last_active":"2023-10-09T09:39:32.838084+0000","last_peered":"2023-10-09T09:39:32.838084+0000","last_clean":"2023-10-09T09:39:32.838084+0000","last_became_active":"2023-10-09T09:36:43.418525+0000","last_became_peered":"2023-10-09T09:36:43.418525+0000","last_unstale":"2023-10-09T09:39:32.838084+0000","last_undegraded":"2023-10-09T09:39:32.838084+0000","last_fullsized":"2023-10-09T09:39:32.838084+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"1166'11810","reported_seq":11880,"reported_epoch":1168,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:39:32.837414+0000","last_change":"2023-10-09T09:39:14.395908+0000","last_active":"2023-10-09T09:39:32.837414+0000","last_peered":"2023-10-09T09:39:32.837414+0000","last_clean":"2023-10-09T09:39:11.921962+0000","last_became_active":"2023-10-09T09:39:14.144198+0000","last_became_peered":"2023-10-09T09:39:14.144198+0000","last_unstale":"2023-10-09T09:39:32.837414+0000","last_undegraded":"2023-10-09T09:39:32.837414+0000","last_fullsized":"2023-10-09T09:39:32.837414+0000","mapping_epoch":1163,"log_start":"1145'4637","ondisk_log_start":"1145'4637","created":1132,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7173,"log_dups_size":0,"ondisk_log_size":7173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11810,"num_write_kb":377920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,5],"avail_no_missing":["4","0","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1138,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.836847+0000","last_change":"2023-10-09T09:36:43.418845+0000","last_active":"2023-10-09T09:39:32.836847+0000","last_peered":"2023-10-09T09:39:32.836847+0000","last_clean":"2023-10-09T09:39:32.836847+0000","last_became_active":"2023-10-09T09:36:43.418053+0000","last_became_peered":"2023-10-09T09:36:43.418053+0000","last_unstale":"2023-10-09T09:39:32.836847+0000","last_undegraded":"2023-10-09T09:39:32.836847+0000","last_fullsized":"2023-10-09T09:39:32.836847+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1119,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.838548+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:39:32.838548+0000","last_peered":"2023-10-09T09:39:32.838548+0000","last_clean":"2023-10-09T09:39:32.838548+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:39:32.838548+0000","last_undegraded":"2023-10-09T09:39:32.838548+0000","last_fullsized":"2023-10-09T09:39:32.838548+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"1166'12128","reported_seq":12184,"reported_epoch":1167,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:39:31.789107+0000","last_change":"2023-10-09T09:39:14.165593+0000","last_active":"2023-10-09T09:39:31.789107+0000","last_peered":"2023-10-09T09:39:31.789107+0000","last_clean":"2023-10-09T09:39:11.807633+0000","last_became_active":"2023-10-09T09:39:14.118289+0000","last_became_peered":"2023-10-09T09:39:14.118289+0000","last_unstale":"2023-10-09T09:39:31.789107+0000","last_undegraded":"2023-10-09T09:39:31.789107+0000","last_fullsized":"2023-10-09T09:39:31.789107+0000","mapping_epoch":1163,"log_start":"1148'5396","ondisk_log_start":"1148'5396","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":6732,"log_dups_size":0,"ondisk_log_size":6732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:36.348588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12128,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,5],"avail_no_missing":["1","2","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1072,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.794943+0000","last_change":"2023-10-09T09:15:36.180113+0000","last_active":"2023-10-09T09:39:31.794943+0000","last_peered":"2023-10-09T09:39:31.794943+0000","last_clean":"2023-10-09T09:39:31.794943+0000","last_became_active":"2023-10-09T09:15:36.179736+0000","last_became_peered":"2023-10-09T09:15:36.179736+0000","last_unstale":"2023-10-09T09:39:31.794943+0000","last_undegraded":"2023-10-09T09:39:31.794943+0000","last_fullsized":"2023-10-09T09:39:31.794943+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:09.529329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1109,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.856601+0000","last_change":"2023-10-09T09:15:36.180068+0000","last_active":"2023-10-09T09:39:31.856601+0000","last_peered":"2023-10-09T09:39:31.856601+0000","last_clean":"2023-10-09T09:39:31.856601+0000","last_became_active":"2023-10-09T09:15:36.179711+0000","last_became_peered":"2023-10-09T09:15:36.179711+0000","last_unstale":"2023-10-09T09:39:31.856601+0000","last_undegraded":"2023-10-09T09:39:31.856601+0000","last_fullsized":"2023-10-09T09:39:31.856601+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:21.820306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"1166'12140","reported_seq":17158,"reported_epoch":1168,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:39:32.807076+0000","last_change":"2023-10-09T09:39:12.975615+0000","last_active":"2023-10-09T09:39:32.807076+0000","last_peered":"2023-10-09T09:39:32.807076+0000","last_clean":"2023-10-09T09:36:42.389221+0000","last_became_active":"2023-10-09T09:36:44.832255+0000","last_became_peered":"2023-10-09T09:36:44.832255+0000","last_unstale":"2023-10-09T09:39:32.807076+0000","last_undegraded":"2023-10-09T09:39:32.807076+0000","last_fullsized":"2023-10-09T09:39:32.807076+0000","mapping_epoch":1150,"log_start":"1143'4176","ondisk_log_start":"1143'4176","created":1132,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7964,"log_dups_size":0,"ondisk_log_size":7964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:51.849587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12140,"num_write_kb":388480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2471,"num_bytes_recovered":161939456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,4,7],"avail_no_missing":["0","2","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1119,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.856765+0000","last_change":"2023-10-09T09:35:18.664940+0000","last_active":"2023-10-09T09:39:31.856765+0000","last_peered":"2023-10-09T09:39:31.856765+0000","last_clean":"2023-10-09T09:39:31.856765+0000","last_became_active":"2023-10-09T09:35:18.664570+0000","last_became_peered":"2023-10-09T09:35:18.664570+0000","last_unstale":"2023-10-09T09:39:31.856765+0000","last_undegraded":"2023-10-09T09:39:31.856765+0000","last_fullsized":"2023-10-09T09:39:31.856765+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:33.263100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1144,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.809860+0000","last_change":"2023-10-09T09:39:12.967382+0000","last_active":"2023-10-09T09:39:32.809860+0000","last_peered":"2023-10-09T09:39:32.809860+0000","last_clean":"2023-10-09T09:39:32.809860+0000","last_became_active":"2023-10-09T09:39:12.966969+0000","last_became_peered":"2023-10-09T09:39:12.966969+0000","last_unstale":"2023-10-09T09:39:32.809860+0000","last_undegraded":"2023-10-09T09:39:32.809860+0000","last_fullsized":"2023-10-09T09:39:32.809860+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:09.698196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1103,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.789315+0000","last_change":"2023-10-09T09:39:12.953533+0000","last_active":"2023-10-09T09:39:31.789315+0000","last_peered":"2023-10-09T09:39:31.789315+0000","last_clean":"2023-10-09T09:39:31.789315+0000","last_became_active":"2023-10-09T09:39:12.953134+0000","last_became_peered":"2023-10-09T09:39:12.953134+0000","last_unstale":"2023-10-09T09:39:31.789315+0000","last_undegraded":"2023-10-09T09:39:31.789315+0000","last_fullsized":"2023-10-09T09:39:31.789315+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:57.994467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"1166'11878","reported_seq":11952,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.851355+0000","last_change":"2023-10-09T09:39:32.851085+0000","last_active":"2023-10-09T09:39:32.851355+0000","last_peered":"2023-10-09T09:39:32.851355+0000","last_clean":"2023-10-09T09:39:32.851355+0000","last_became_active":"2023-10-09T09:39:32.810229+0000","last_became_peered":"2023-10-09T09:39:32.810229+0000","last_unstale":"2023-10-09T09:39:32.851355+0000","last_undegraded":"2023-10-09T09:39:32.851355+0000","last_fullsized":"2023-10-09T09:39:32.851355+0000","mapping_epoch":1167,"log_start":"1143'3738","ondisk_log_start":"1143'3738","created":1132,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":8140,"log_dups_size":0,"ondisk_log_size":8140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11878,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":586,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.789048+0000","last_change":"2023-10-09T09:39:12.957426+0000","last_active":"2023-10-09T09:39:31.789048+0000","last_peered":"2023-10-09T09:39:31.789048+0000","last_clean":"2023-10-09T09:39:31.789048+0000","last_became_active":"2023-10-09T09:39:12.957061+0000","last_became_peered":"2023-10-09T09:39:12.957061+0000","last_unstale":"2023-10-09T09:39:31.789048+0000","last_undegraded":"2023-10-09T09:39:31.789048+0000","last_fullsized":"2023-10-09T09:39:31.789048+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:39.324710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1135,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.837918+0000","last_change":"2023-10-09T09:35:18.685554+0000","last_active":"2023-10-09T09:39:32.837918+0000","last_peered":"2023-10-09T09:39:32.837918+0000","last_clean":"2023-10-09T09:39:32.837918+0000","last_became_active":"2023-10-09T09:35:18.685119+0000","last_became_peered":"2023-10-09T09:35:18.685119+0000","last_unstale":"2023-10-09T09:39:32.837918+0000","last_undegraded":"2023-10-09T09:39:32.837918+0000","last_fullsized":"2023-10-09T09:39:32.837918+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1142,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.789059+0000","last_change":"2023-10-09T09:36:43.406338+0000","last_active":"2023-10-09T09:39:31.789059+0000","last_peered":"2023-10-09T09:39:31.789059+0000","last_clean":"2023-10-09T09:39:31.789059+0000","last_became_active":"2023-10-09T09:36:43.405976+0000","last_became_peered":"2023-10-09T09:36:43.405976+0000","last_unstale":"2023-10-09T09:39:31.789059+0000","last_undegraded":"2023-10-09T09:39:31.789059+0000","last_fullsized":"2023-10-09T09:39:31.789059+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:22.618067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"1166'11904","reported_seq":11949,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.837804+0000","last_change":"2023-10-09T09:35:12.106534+0000","last_active":"2023-10-09T09:39:32.837804+0000","last_peered":"2023-10-09T09:39:32.837804+0000","last_clean":"2023-10-09T09:39:32.837804+0000","last_became_active":"2023-10-09T09:35:12.105788+0000","last_became_peered":"2023-10-09T09:35:12.105788+0000","last_unstale":"2023-10-09T09:39:32.837804+0000","last_undegraded":"2023-10-09T09:39:32.837804+0000","last_fullsized":"2023-10-09T09:39:32.837804+0000","mapping_epoch":1132,"log_start":"1145'4753","ondisk_log_start":"1145'4753","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7151,"log_dups_size":0,"ondisk_log_size":7151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11904,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":31,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.809014+0000","last_change":"2023-10-09T09:36:43.412227+0000","last_active":"2023-10-09T09:39:32.809014+0000","last_peered":"2023-10-09T09:39:32.809014+0000","last_clean":"2023-10-09T09:39:32.809014+0000","last_became_active":"2023-10-09T09:36:43.411884+0000","last_became_peered":"2023-10-09T09:36:43.411884+0000","last_unstale":"2023-10-09T09:39:32.809014+0000","last_undegraded":"2023-10-09T09:39:32.809014+0000","last_fullsized":"2023-10-09T09:39:32.809014+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:09.042244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1128,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.788994+0000","last_change":"2023-10-09T09:36:43.421868+0000","last_active":"2023-10-09T09:39:31.788994+0000","last_peered":"2023-10-09T09:39:31.788994+0000","last_clean":"2023-10-09T09:39:31.788994+0000","last_became_active":"2023-10-09T09:36:43.421013+0000","last_became_peered":"2023-10-09T09:36:43.421013+0000","last_unstale":"2023-10-09T09:39:31.788994+0000","last_undegraded":"2023-10-09T09:39:31.788994+0000","last_fullsized":"2023-10-09T09:39:31.788994+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:43.353667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1123,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.856711+0000","last_change":"2023-10-09T09:39:12.952454+0000","last_active":"2023-10-09T09:39:31.856711+0000","last_peered":"2023-10-09T09:39:31.856711+0000","last_clean":"2023-10-09T09:39:31.856711+0000","last_became_active":"2023-10-09T09:39:12.951861+0000","last_became_peered":"2023-10-09T09:39:12.951861+0000","last_unstale":"2023-10-09T09:39:31.856711+0000","last_undegraded":"2023-10-09T09:39:31.856711+0000","last_fullsized":"2023-10-09T09:39:31.856711+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:50.740128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"1166'11912","reported_seq":12928,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.803885+0000","last_change":"2023-10-09T09:39:20.791256+0000","last_active":"2023-10-09T09:39:32.803885+0000","last_peered":"2023-10-09T09:39:32.803885+0000","last_clean":"2023-10-09T09:39:32.803885+0000","last_became_active":"2023-10-09T09:39:20.756056+0000","last_became_peered":"2023-10-09T09:39:20.756056+0000","last_unstale":"2023-10-09T09:39:32.803885+0000","last_undegraded":"2023-10-09T09:39:32.803885+0000","last_fullsized":"2023-10-09T09:39:32.803885+0000","mapping_epoch":1165,"log_start":"1144'4147","ondisk_log_start":"1144'4147","created":1132,"last_epoch_clean":1166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7765,"log_dups_size":0,"ondisk_log_size":7765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11912,"num_write_kb":381184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":473,"num_bytes_recovered":30998528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1146,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.808893+0000","last_change":"2023-10-09T09:39:12.957801+0000","last_active":"2023-10-09T09:39:32.808893+0000","last_peered":"2023-10-09T09:39:32.808893+0000","last_clean":"2023-10-09T09:39:32.808893+0000","last_became_active":"2023-10-09T09:39:12.955560+0000","last_became_peered":"2023-10-09T09:39:12.955560+0000","last_unstale":"2023-10-09T09:39:32.808893+0000","last_undegraded":"2023-10-09T09:39:32.808893+0000","last_fullsized":"2023-10-09T09:39:32.808893+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:25.531467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":1348,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807818+0000","last_change":"2023-10-09T09:35:18.660120+0000","last_active":"2023-10-09T09:39:32.807818+0000","last_peered":"2023-10-09T09:39:32.807818+0000","last_clean":"2023-10-09T09:39:32.807818+0000","last_became_active":"2023-10-09T09:35:18.659460+0000","last_became_peered":"2023-10-09T09:35:18.659460+0000","last_unstale":"2023-10-09T09:39:32.807818+0000","last_undegraded":"2023-10-09T09:39:32.807818+0000","last_fullsized":"2023-10-09T09:39:32.807818+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":596,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.804494+0000","last_change":"2023-10-09T09:39:12.953366+0000","last_active":"2023-10-09T09:39:32.804494+0000","last_peered":"2023-10-09T09:39:32.804494+0000","last_clean":"2023-10-09T09:39:32.804494+0000","last_became_active":"2023-10-09T09:39:12.952903+0000","last_became_peered":"2023-10-09T09:39:12.952903+0000","last_unstale":"2023-10-09T09:39:32.804494+0000","last_undegraded":"2023-10-09T09:39:32.804494+0000","last_fullsized":"2023-10-09T09:39:32.804494+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1086,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.804696+0000","last_change":"2023-10-09T09:03:25.755367+0000","last_active":"2023-10-09T09:39:32.804696+0000","last_peered":"2023-10-09T09:39:32.804696+0000","last_clean":"2023-10-09T09:39:32.804696+0000","last_became_active":"2023-10-09T09:03:25.754459+0000","last_became_peered":"2023-10-09T09:03:25.754459+0000","last_unstale":"2023-10-09T09:39:32.804696+0000","last_undegraded":"2023-10-09T09:39:32.804696+0000","last_fullsized":"2023-10-09T09:39:32.804696+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1166'11624","reported_seq":16350,"reported_epoch":1168,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T09:39:32.804411+0000","last_change":"2023-10-09T09:39:14.004794+0000","last_active":"2023-10-09T09:39:32.804411+0000","last_peered":"2023-10-09T09:39:32.804411+0000","last_clean":"2023-10-09T09:36:42.376092+0000","last_became_active":"2023-10-09T09:39:13.951626+0000","last_became_peered":"2023-10-09T09:39:13.951626+0000","last_unstale":"2023-10-09T09:39:32.804411+0000","last_undegraded":"2023-10-09T09:39:32.804411+0000","last_fullsized":"2023-10-09T09:39:32.804411+0000","mapping_epoch":1163,"log_start":"1148'4939","ondisk_log_start":"1148'4939","created":1132,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:09.762080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11624,"num_write_kb":371968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2322,"num_bytes_recovered":152174592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,3,5],"avail_no_missing":["1","2","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1131,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807688+0000","last_change":"2023-10-09T09:35:18.659469+0000","last_active":"2023-10-09T09:39:32.807688+0000","last_peered":"2023-10-09T09:39:32.807688+0000","last_clean":"2023-10-09T09:39:32.807688+0000","last_became_active":"2023-10-09T09:35:18.659007+0000","last_became_peered":"2023-10-09T09:35:18.659007+0000","last_unstale":"2023-10-09T09:39:32.807688+0000","last_undegraded":"2023-10-09T09:39:32.807688+0000","last_fullsized":"2023-10-09T09:39:32.807688+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1092,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806637+0000","last_change":"2023-10-09T09:35:18.684437+0000","last_active":"2023-10-09T09:39:32.806637+0000","last_peered":"2023-10-09T09:39:32.806637+0000","last_clean":"2023-10-09T09:39:32.806637+0000","last_became_active":"2023-10-09T09:35:18.678845+0000","last_became_peered":"2023-10-09T09:35:18.678845+0000","last_unstale":"2023-10-09T09:39:32.806637+0000","last_undegraded":"2023-10-09T09:39:32.806637+0000","last_fullsized":"2023-10-09T09:39:32.806637+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:44:13.569461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1098,"reported_epoch":1169,"state":"active+clean","last_fresh":"2023-10-09T09:39:35.694443+0000","last_change":"2023-10-09T09:39:12.949268+0000","last_active":"2023-10-09T09:39:35.694443+0000","last_peered":"2023-10-09T09:39:35.694443+0000","last_clean":"2023-10-09T09:39:35.694443+0000","last_became_active":"2023-10-09T09:39:12.948924+0000","last_became_peered":"2023-10-09T09:39:12.948924+0000","last_unstale":"2023-10-09T09:39:35.694443+0000","last_undegraded":"2023-10-09T09:39:35.694443+0000","last_fullsized":"2023-10-09T09:39:35.694443+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:31.368540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"1166'11892","reported_seq":18944,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.810964+0000","last_change":"2023-10-09T09:38:18.894616+0000","last_active":"2023-10-09T09:39:32.810964+0000","last_peered":"2023-10-09T09:39:32.810964+0000","last_clean":"2023-10-09T09:39:32.810964+0000","last_became_active":"2023-10-09T09:38:18.869885+0000","last_became_peered":"2023-10-09T09:38:18.869885+0000","last_unstale":"2023-10-09T09:39:32.810964+0000","last_undegraded":"2023-10-09T09:39:32.810964+0000","last_fullsized":"2023-10-09T09:39:32.810964+0000","mapping_epoch":1156,"log_start":"1139'1900","ondisk_log_start":"1139'1900","created":1132,"last_epoch_clean":1157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":9992,"log_dups_size":0,"ondisk_log_size":9992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:08.557236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11892,"num_write_kb":380544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3491,"num_bytes_recovered":228786176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1116,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806992+0000","last_change":"2023-10-09T09:35:18.685388+0000","last_active":"2023-10-09T09:39:32.806992+0000","last_peered":"2023-10-09T09:39:32.806992+0000","last_clean":"2023-10-09T09:39:32.806992+0000","last_became_active":"2023-10-09T09:35:18.684983+0000","last_became_peered":"2023-10-09T09:35:18.684983+0000","last_unstale":"2023-10-09T09:39:32.806992+0000","last_undegraded":"2023-10-09T09:39:32.806992+0000","last_fullsized":"2023-10-09T09:39:32.806992+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1137,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.789335+0000","last_change":"2023-10-09T09:39:12.951046+0000","last_active":"2023-10-09T09:39:31.789335+0000","last_peered":"2023-10-09T09:39:31.789335+0000","last_clean":"2023-10-09T09:39:31.789335+0000","last_became_active":"2023-10-09T09:39:12.950507+0000","last_became_peered":"2023-10-09T09:39:12.950507+0000","last_unstale":"2023-10-09T09:39:31.789335+0000","last_undegraded":"2023-10-09T09:39:31.789335+0000","last_fullsized":"2023-10-09T09:39:31.789335+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:10.825434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1085,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806951+0000","last_change":"2023-10-09T09:39:12.952113+0000","last_active":"2023-10-09T09:39:32.806951+0000","last_peered":"2023-10-09T09:39:32.806951+0000","last_clean":"2023-10-09T09:39:32.806951+0000","last_became_active":"2023-10-09T09:39:12.951745+0000","last_became_peered":"2023-10-09T09:39:12.951745+0000","last_unstale":"2023-10-09T09:39:32.806951+0000","last_undegraded":"2023-10-09T09:39:32.806951+0000","last_fullsized":"2023-10-09T09:39:32.806951+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:52.519183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"1166'11900","reported_seq":11944,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807347+0000","last_change":"2023-10-09T09:35:18.673315+0000","last_active":"2023-10-09T09:39:32.807347+0000","last_peered":"2023-10-09T09:39:32.807347+0000","last_clean":"2023-10-09T09:39:32.807347+0000","last_became_active":"2023-10-09T09:35:18.672993+0000","last_became_peered":"2023-10-09T09:35:18.672993+0000","last_unstale":"2023-10-09T09:39:32.807347+0000","last_undegraded":"2023-10-09T09:39:32.807347+0000","last_fullsized":"2023-10-09T09:39:32.807347+0000","mapping_epoch":1136,"log_start":"1143'3942","ondisk_log_start":"1143'3942","created":1132,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":7958,"log_dups_size":0,"ondisk_log_size":7958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:19.360755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11900,"num_write_kb":380800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":91,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807128+0000","last_change":"2023-10-09T09:35:18.676960+0000","last_active":"2023-10-09T09:39:32.807128+0000","last_peered":"2023-10-09T09:39:32.807128+0000","last_clean":"2023-10-09T09:39:32.807128+0000","last_became_active":"2023-10-09T09:35:18.676357+0000","last_became_peered":"2023-10-09T09:35:18.676357+0000","last_unstale":"2023-10-09T09:39:32.807128+0000","last_undegraded":"2023-10-09T09:39:32.807128+0000","last_fullsized":"2023-10-09T09:39:32.807128+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1093,"reported_epoch":1167,"state":"active+clean","last_fresh":"2023-10-09T09:39:31.856804+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:39:31.856804+0000","last_peered":"2023-10-09T09:39:31.856804+0000","last_clean":"2023-10-09T09:39:31.856804+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:39:31.856804+0000","last_undegraded":"2023-10-09T09:39:31.856804+0000","last_fullsized":"2023-10-09T09:39:31.856804+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1098,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807975+0000","last_change":"2023-10-09T09:35:18.663646+0000","last_active":"2023-10-09T09:39:32.807975+0000","last_peered":"2023-10-09T09:39:32.807975+0000","last_clean":"2023-10-09T09:39:32.807975+0000","last_became_active":"2023-10-09T09:35:18.663357+0000","last_became_peered":"2023-10-09T09:35:18.663357+0000","last_unstale":"2023-10-09T09:39:32.807975+0000","last_undegraded":"2023-10-09T09:39:32.807975+0000","last_fullsized":"2023-10-09T09:39:32.807975+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"1166'11730","reported_seq":13312,"reported_epoch":1168,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:39:32.807476+0000","last_change":"2023-10-09T09:39:14.396608+0000","last_active":"2023-10-09T09:39:32.807476+0000","last_peered":"2023-10-09T09:39:32.807476+0000","last_clean":"2023-10-09T09:35:47.668825+0000","last_became_active":"2023-10-09T09:39:14.339331+0000","last_became_peered":"2023-10-09T09:39:14.339331+0000","last_unstale":"2023-10-09T09:39:32.807476+0000","last_undegraded":"2023-10-09T09:39:14.185324+0000","last_fullsized":"2023-10-09T09:39:13.944988+0000","mapping_epoch":1163,"log_start":"1143'3573","ondisk_log_start":"1143'3573","created":1132,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:35:10.884095+0000","last_clean_scrub_stamp":"2023-10-09T09:35:10.884095+0000","objects_scrubbed":0,"log_size":8157,"log_dups_size":0,"ondisk_log_size":8157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8322,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11730,"num_write_kb":375297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":753,"num_bytes_recovered":49217536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"2,7","objects":-549}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1123,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.805138+0000","last_change":"2023-10-09T09:36:43.423809+0000","last_active":"2023-10-09T09:39:32.805138+0000","last_peered":"2023-10-09T09:39:32.805138+0000","last_clean":"2023-10-09T09:39:32.805138+0000","last_became_active":"2023-10-09T09:36:43.423506+0000","last_became_peered":"2023-10-09T09:36:43.423506+0000","last_unstale":"2023-10-09T09:39:32.805138+0000","last_undegraded":"2023-10-09T09:39:32.805138+0000","last_fullsized":"2023-10-09T09:39:32.805138+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1079,"reported_epoch":1169,"state":"active+clean","last_fresh":"2023-10-09T09:39:35.694632+0000","last_change":"2023-10-09T09:35:18.685030+0000","last_active":"2023-10-09T09:39:35.694632+0000","last_peered":"2023-10-09T09:39:35.694632+0000","last_clean":"2023-10-09T09:39:35.694632+0000","last_became_active":"2023-10-09T09:35:18.675810+0000","last_became_peered":"2023-10-09T09:35:18.675810+0000","last_unstale":"2023-10-09T09:39:35.694632+0000","last_undegraded":"2023-10-09T09:39:35.694632+0000","last_fullsized":"2023-10-09T09:39:35.694632+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:52.793405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1113,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.836956+0000","last_change":"2023-10-09T09:39:12.958984+0000","last_active":"2023-10-09T09:39:32.836956+0000","last_peered":"2023-10-09T09:39:32.836956+0000","last_clean":"2023-10-09T09:39:32.836956+0000","last_became_active":"2023-10-09T09:39:12.958519+0000","last_became_peered":"2023-10-09T09:39:12.958519+0000","last_unstale":"2023-10-09T09:39:32.836956+0000","last_undegraded":"2023-10-09T09:39:32.836956+0000","last_fullsized":"2023-10-09T09:39:32.836956+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1078,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.806617+0000","last_change":"2023-10-09T09:35:18.680525+0000","last_active":"2023-10-09T09:39:32.806617+0000","last_peered":"2023-10-09T09:39:32.806617+0000","last_clean":"2023-10-09T09:39:32.806617+0000","last_became_active":"2023-10-09T09:35:18.675584+0000","last_became_peered":"2023-10-09T09:35:18.675584+0000","last_unstale":"2023-10-09T09:39:32.806617+0000","last_undegraded":"2023-10-09T09:39:32.806617+0000","last_fullsized":"2023-10-09T09:39:32.806617+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:09.246045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":31,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.809013+0000","last_change":"2023-10-09T09:36:43.406032+0000","last_active":"2023-10-09T09:39:32.809013+0000","last_peered":"2023-10-09T09:39:32.809013+0000","last_clean":"2023-10-09T09:39:32.809013+0000","last_became_active":"2023-10-09T09:36:43.405092+0000","last_became_peered":"2023-10-09T09:36:43.405092+0000","last_unstale":"2023-10-09T09:39:32.809013+0000","last_undegraded":"2023-10-09T09:39:32.809013+0000","last_fullsized":"2023-10-09T09:39:32.809013+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:20.087534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":46,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.838079+0000","last_change":"2023-10-09T09:39:13.001634+0000","last_active":"2023-10-09T09:39:32.838079+0000","last_peered":"2023-10-09T09:39:32.838079+0000","last_clean":"2023-10-09T09:39:32.838079+0000","last_became_active":"2023-10-09T09:39:12.953091+0000","last_became_peered":"2023-10-09T09:39:12.953091+0000","last_unstale":"2023-10-09T09:39:32.838079+0000","last_undegraded":"2023-10-09T09:39:32.838079+0000","last_fullsized":"2023-10-09T09:39:32.838079+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1134,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.838211+0000","last_change":"2023-10-09T09:15:36.182683+0000","last_active":"2023-10-09T09:39:32.838211+0000","last_peered":"2023-10-09T09:39:32.838211+0000","last_clean":"2023-10-09T09:39:32.838211+0000","last_became_active":"2023-10-09T09:15:36.182268+0000","last_became_peered":"2023-10-09T09:15:36.182268+0000","last_unstale":"2023-10-09T09:39:32.838211+0000","last_undegraded":"2023-10-09T09:39:32.838211+0000","last_fullsized":"2023-10-09T09:39:32.838211+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":46,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.838404+0000","last_change":"2023-10-09T09:39:13.063164+0000","last_active":"2023-10-09T09:39:32.838404+0000","last_peered":"2023-10-09T09:39:32.838404+0000","last_clean":"2023-10-09T09:39:32.838404+0000","last_became_active":"2023-10-09T09:39:12.955031+0000","last_became_peered":"2023-10-09T09:39:12.955031+0000","last_unstale":"2023-10-09T09:39:32.838404+0000","last_undegraded":"2023-10-09T09:39:32.838404+0000","last_fullsized":"2023-10-09T09:39:32.838404+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1149,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.837573+0000","last_change":"2023-10-09T09:35:18.686704+0000","last_active":"2023-10-09T09:39:32.837573+0000","last_peered":"2023-10-09T09:39:32.837573+0000","last_clean":"2023-10-09T09:39:32.837573+0000","last_became_active":"2023-10-09T09:35:18.686421+0000","last_became_peered":"2023-10-09T09:35:18.686421+0000","last_unstale":"2023-10-09T09:39:32.837573+0000","last_undegraded":"2023-10-09T09:39:32.837573+0000","last_fullsized":"2023-10-09T09:39:32.837573+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":93,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.804916+0000","last_change":"2023-10-09T09:36:43.424942+0000","last_active":"2023-10-09T09:39:32.804916+0000","last_peered":"2023-10-09T09:39:32.804916+0000","last_clean":"2023-10-09T09:39:32.804916+0000","last_became_active":"2023-10-09T09:36:43.424555+0000","last_became_peered":"2023-10-09T09:36:43.424555+0000","last_unstale":"2023-10-09T09:39:32.804916+0000","last_undegraded":"2023-10-09T09:39:32.804916+0000","last_fullsized":"2023-10-09T09:39:32.804916+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":31,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.808878+0000","last_change":"2023-10-09T09:36:43.412400+0000","last_active":"2023-10-09T09:39:32.808878+0000","last_peered":"2023-10-09T09:39:32.808878+0000","last_clean":"2023-10-09T09:39:32.808878+0000","last_became_active":"2023-10-09T09:36:43.412039+0000","last_became_peered":"2023-10-09T09:36:43.412039+0000","last_unstale":"2023-10-09T09:39:32.808878+0000","last_undegraded":"2023-10-09T09:39:32.808878+0000","last_fullsized":"2023-10-09T09:39:32.808878+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:40:22.872096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1112,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.807292+0000","last_change":"2023-10-09T09:39:13.008397+0000","last_active":"2023-10-09T09:39:32.807292+0000","last_peered":"2023-10-09T09:39:32.807292+0000","last_clean":"2023-10-09T09:39:32.807292+0000","last_became_active":"2023-10-09T09:39:12.968234+0000","last_became_peered":"2023-10-09T09:39:12.968234+0000","last_unstale":"2023-10-09T09:39:32.807292+0000","last_undegraded":"2023-10-09T09:39:32.807292+0000","last_fullsized":"2023-10-09T09:39:32.807292+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:59.189505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1119,"reported_epoch":1168,"state":"active+clean","last_fresh":"2023-10-09T09:39:32.836889+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:39:32.836889+0000","last_peered":"2023-10-09T09:39:32.836889+0000","last_clean":"2023-10-09T09:39:32.836889+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:39:32.836889+0000","last_undegraded":"2023-10-09T09:39:32.836889+0000","last_fullsized":"2023-10-09T09:39:32.836889+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1800,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8889,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":190462,"num_write_kb":6094721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13750,"num_bytes_recovered":767688704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":982061056,"data_stored":982056984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120722,"ondisk_log_size":120722,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397974,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302420,"kb_used_data":5196,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93442732,"statfs":{"total":95995035648,"available":95685357568,"internally_reserved":0,"allocated":5320704,"data_stored":5075844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:39:23 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.9990000000000001,"15min":1.5649999999999999},"min":{"1min":0.29299999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.2649999999999997,"5min":35.549999999999997,"15min":35.549999999999997},"last":0.96099999999999997},{"interface":"front","average":{"1min":2.4510000000000001,"5min":2.3199999999999998,"15min":1.8620000000000001},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.8550000000000004,"5min":35.468000000000004,"15min":35.468000000000004},"last":0.80500000000000005}]},{"osd":1,"last update":"Mon Oct 9 09:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":1.8879999999999999,"15min":2.1190000000000002},"min":{"1min":0.36699999999999999,"5min":0.30299999999999999,"15min":0.26300000000000001},"max":{"1min":9.6649999999999991,"5min":16.785,"15min":103.84699999999999},"last":0.86099999999999999},{"interface":"front","average":{"1min":2.1600000000000001,"5min":2.3809999999999998,"15min":2.2599999999999998},"min":{"1min":0.48899999999999999,"5min":0.314,"15min":0.28199999999999997},"max":{"1min":6.9480000000000004,"5min":17.062999999999999,"15min":103.955},"last":0.88700000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:38:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":2.1909999999999998,"15min":2.0539999999999998},"min":{"1min":0.434,"5min":0.34799999999999998,"15min":0.29399999999999998},"max":{"1min":8.6010000000000009,"5min":50.756,"15min":50.756},"last":1.097},{"interface":"front","average":{"1min":2.2799999999999998,"5min":2.081,"15min":1.968},"min":{"1min":0.33200000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.1470000000000002,"5min":50.82,"15min":50.82},"last":0.98599999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:39:33 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":1.962,"15min":3.0939999999999999},"min":{"1min":0.39800000000000002,"5min":0.34300000000000003,"15min":0.29999999999999999},"max":{"1min":12.710000000000001,"5min":12.710000000000001,"15min":107.97499999999999},"last":1.214},{"interface":"front","average":{"1min":3.0409999999999999,"5min":1.931,"15min":3.133},"min":{"1min":0.39900000000000002,"5min":0.38400000000000001,"15min":0.317},"max":{"1min":12.545,"5min":12.545,"15min":105.282},"last":1.147}]},{"osd":4,"last update":"Mon Oct 9 09:38:56 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":1.494,"15min":1.387},"min":{"1min":0.35999999999999999,"5min":0.29499999999999998,"15min":0.26500000000000001},"max":{"1min":4.7969999999999997,"5min":8.3789999999999996,"15min":9.8300000000000001},"last":0.73599999999999999},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.4590000000000001,"15min":1.2170000000000001},"min":{"1min":0.44500000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.0090000000000003,"5min":6.5629999999999997,"15min":12.316000000000001},"last":0.93200000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:38:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":1.927,"15min":1.5389999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33300000000000002},"max":{"1min":12.667,"5min":12.667,"15min":12.667},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.831,"15min":1.474},"min":{"1min":0.41899999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.2549999999999999,"5min":10.038,"15min":10.038},"last":1.165}]},{"osd":6,"last update":"Mon Oct 9 09:38:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":1.6870000000000001,"15min":1.431},"min":{"1min":0.45200000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.3079999999999998,"5min":8.4079999999999995,"15min":8.4079999999999995},"last":1.071},{"interface":"front","average":{"1min":2.3090000000000002,"5min":1.6839999999999999,"15min":1.4670000000000001},"min":{"1min":0.47699999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.9050000000000002,"5min":6.194,"15min":8.1980000000000004},"last":1.038}]}]},{"osd":6,"up_from":43,"seq":184683594207,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":690388,"kb_used_data":393164,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93054764,"statfs":{"total":95995035648,"available":95288078336,"internally_reserved":0,"allocated":402599936,"data_stored":402363402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:39:23 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.55,"15min":1.286},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.9530000000000003,"5min":11.500999999999999,"15min":11.500999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":2.7149999999999999,"5min":1.843,"15min":1.4379999999999999},"min":{"1min":0.34599999999999997,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.791,"5min":10.791,"15min":10.791},"last":0.85599999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:39:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6949999999999998,"5min":2.4620000000000002,"15min":2.544},"min":{"1min":0.316,"5min":0.27200000000000002,"15min":0.22900000000000001},"max":{"1min":10.342000000000001,"5min":38.709000000000003,"15min":103.259},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.754,"5min":2.246,"15min":2.3039999999999998},"min":{"1min":0.38600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.5020000000000007,"5min":35.820999999999998,"15min":102.005},"last":0.70899999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":1.589,"15min":1.6990000000000001},"min":{"1min":0.30599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.359,"5min":12.657,"15min":33.210000000000001},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.7390000000000001,"5min":1.742,"15min":1.786},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.6230000000000002,"5min":13.101000000000001,"15min":37.246000000000002},"last":0.67400000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:39:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9139999999999999,"5min":1.7350000000000001,"15min":2.4009999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":11.547000000000001,"5min":11.547000000000001,"15min":123.97199999999999},"last":0.753},{"interface":"front","average":{"1min":2.0579999999999998,"5min":1.3779999999999999,"15min":2.2650000000000001},"min":{"1min":0.32900000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.4550000000000001,"5min":7.7610000000000001,"15min":124.059},"last":0.80800000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:39:13 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":1.286,"15min":1.139},"min":{"1min":0.32300000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.0800000000000001,"5min":8.0800000000000001,"15min":9.1099999999999994},"last":0.78300000000000003},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.425,"15min":1.3320000000000001},"min":{"1min":0.35299999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.0730000000000004,"5min":5.9470000000000001,"15min":9.7919999999999998},"last":0.96299999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:39:13 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":1.579,"15min":1.2809999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":8.1400000000000006,"5min":9.0559999999999992,"15min":9.0559999999999992},"last":0.91000000000000003},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.7450000000000001,"15min":1.4299999999999999},"min":{"1min":0.42799999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.9779999999999998,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.58899999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:38:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.2989999999999999,"15min":1.385},"min":{"1min":0.41499999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.1390000000000002,"5min":5.609,"15min":7.6769999999999996},"last":0.88900000000000001},{"interface":"front","average":{"1min":1.5349999999999999,"5min":1.2909999999999999,"15min":1.395},"min":{"1min":0.35599999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.835,"5min":6.8840000000000003,"15min":7.6879999999999997},"last":0.878}]}]},{"osd":2,"up_from":634,"seq":2723009265960,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":685128,"kb_used_data":247424,"kb_used_omap":122,"kb_used_meta":437573,"kb_avail":93060024,"statfs":{"total":95995035648,"available":95293464576,"internally_reserved":0,"allocated":253362176,"data_stored":252090684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":448075339},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:39:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6059999999999999,"5min":3.4100000000000001,"15min":2.8809999999999998},"min":{"1min":0.26400000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.6699999999999999,"5min":86.221999999999994,"15min":86.221999999999994},"last":0.314},{"interface":"front","average":{"1min":2.4670000000000001,"5min":3.5459999999999998,"15min":3.2029999999999998},"min":{"1min":0.25700000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.4340000000000002,"5min":82.183000000000007,"15min":82.183000000000007},"last":0.69599999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:39:11 2023","interfaces":[{"interface":"back","average":{"1min":3.6429999999999998,"5min":4.2080000000000002,"15min":2.3940000000000001},"min":{"1min":0.25600000000000001,"5min":0.20799999999999999,"15min":0.184},"max":{"1min":15.016,"5min":81.662999999999997,"15min":81.662999999999997},"last":0.33700000000000002},{"interface":"front","average":{"1min":3.1739999999999999,"5min":4.0999999999999996,"15min":2.5550000000000002},"min":{"1min":0.45300000000000001,"5min":0.222,"15min":0.216},"max":{"1min":10.907,"5min":81.712999999999994,"15min":81.712999999999994},"last":0.47499999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:39:22 2023","interfaces":[{"interface":"back","average":{"1min":3.6970000000000001,"5min":1.7270000000000001,"15min":1.8759999999999999},"min":{"1min":0.44900000000000001,"5min":0.254,"15min":0.252},"max":{"1min":13.526,"5min":13.526,"15min":18.547999999999998},"last":0.38400000000000001},{"interface":"front","average":{"1min":3.2229999999999999,"5min":1.7230000000000001,"15min":1.9359999999999999},"min":{"1min":0.40200000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.6359999999999992,"5min":11.760999999999999,"15min":19.527999999999999},"last":0.501}]},{"osd":4,"last update":"Mon Oct 9 09:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.819,"5min":1.782,"15min":1.5760000000000001},"min":{"1min":0.44900000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.5690000000000008,"5min":11.622999999999999,"15min":11.622999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":3.2160000000000002,"5min":1.7490000000000001,"15min":1.629},"min":{"1min":0.57999999999999996,"5min":0.313,"15min":0.313},"max":{"1min":10.394,"5min":10.394,"15min":13.917999999999999},"last":0.59499999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.4279999999999999,"5min":1.6399999999999999,"15min":1.4590000000000001},"min":{"1min":0.45600000000000002,"5min":0.314,"15min":0.313},"max":{"1min":9.7379999999999995,"5min":9.7599999999999998,"15min":9.7599999999999998},"last":0.629},{"interface":"front","average":{"1min":2.956,"5min":1.9079999999999999,"15min":1.6559999999999999},"min":{"1min":0.58999999999999997,"5min":0.38300000000000001,"15min":0.36499999999999999},"max":{"1min":9.7140000000000004,"5min":11.718999999999999,"15min":11.718999999999999},"last":0.61499999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:39:11 2023","interfaces":[{"interface":"back","average":{"1min":3.2730000000000001,"5min":2.0950000000000002,"15min":1.7709999999999999},"min":{"1min":0.72599999999999998,"5min":0.40300000000000002,"15min":0.33100000000000002},"max":{"1min":9.8219999999999992,"5min":11.648,"15min":11.648},"last":0.76100000000000001},{"interface":"front","average":{"1min":2.8439999999999999,"5min":1.621,"15min":1.7430000000000001},"min":{"1min":0.45800000000000002,"5min":0.35599999999999998,"15min":0.33300000000000002},"max":{"1min":10.436999999999999,"5min":10.436999999999999,"15min":13.893000000000001},"last":0.64800000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:39:11 2023","interfaces":[{"interface":"back","average":{"1min":3.2610000000000001,"5min":1.6950000000000001,"15min":1.8440000000000001},"min":{"1min":0.51200000000000001,"5min":0.39400000000000002,"15min":0.377},"max":{"1min":10.462999999999999,"5min":10.462999999999999,"15min":13.848000000000001},"last":0.72699999999999998},{"interface":"front","average":{"1min":2.6219999999999999,"5min":1.665,"15min":1.6619999999999999},"min":{"1min":0.51500000000000001,"5min":0.42899999999999999,"15min":0.34499999999999997},"max":{"1min":9.7759999999999998,"5min":9.7759999999999998,"15min":10.199},"last":0.77200000000000002}]}]},{"osd":1,"up_from":607,"seq":2607045148973,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":400748,"kb_used_data":5796,"kb_used_omap":107,"kb_used_meta":394836,"kb_avail":93344404,"statfs":{"total":95995035648,"available":95584669696,"internally_reserved":0,"allocated":5935104,"data_stored":4680814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":404312747},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:39:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":1.8009999999999999,"15min":1.534},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.7489999999999997,"5min":11.458,"15min":11.458},"last":0.89200000000000002},{"interface":"front","average":{"1min":2.6739999999999999,"5min":2.5979999999999999,"15min":2.5419999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.1080000000000005,"5min":15.603,"15min":15.603},"last":0.83099999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:38:49 2023","interfaces":[{"interface":"back","average":{"1min":2.734,"5min":2.2650000000000001,"15min":2.5299999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20100000000000001},"max":{"1min":10.201000000000001,"5min":32.345999999999997,"15min":84.578999999999994},"last":1.0349999999999999},{"interface":"front","average":{"1min":2.423,"5min":1.6759999999999999,"15min":2.0569999999999999},"min":{"1min":0.22900000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.2770000000000001,"5min":32.131999999999998,"15min":84.697000000000003},"last":0.98799999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:39:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":1.645,"15min":3.0659999999999998},"min":{"1min":0.26000000000000001,"5min":0.23899999999999999,"15min":0.23599999999999999},"max":{"1min":12.215999999999999,"5min":12.589,"15min":107.426},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.742,"5min":2.0720000000000001,"15min":3.5129999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.23200000000000001},"max":{"1min":8.3010000000000002,"5min":14.27,"15min":106.15900000000001},"last":0.627}]},{"osd":4,"last update":"Mon Oct 9 09:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":1.5069999999999999,"15min":1.585},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.26500000000000001},"max":{"1min":6.5339999999999998,"5min":7.2759999999999998,"15min":84.875},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.742,"5min":1.5049999999999999,"15min":1.7809999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.29499999999999998},"max":{"1min":7.7560000000000002,"5min":7.7560000000000002,"15min":84.992999999999995},"last":0.70599999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.6160000000000001,"15min":1.6599999999999999},"min":{"1min":0.38300000000000001,"5min":0.32600000000000001,"15min":0.32500000000000001},"max":{"1min":6.4509999999999996,"5min":8.6869999999999994,"15min":84.805000000000007},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.292,"5min":1.5660000000000001,"15min":1.7549999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.30399999999999999},"max":{"1min":5.9859999999999998,"5min":6.2859999999999996,"15min":84.951999999999998},"last":0.77900000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:39:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8769999999999998,"5min":2.109,"15min":2.1339999999999999},"min":{"1min":0.66100000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":9.1029999999999998,"5min":14.335000000000001,"15min":85.016000000000005},"last":1.085},{"interface":"front","average":{"1min":2.3159999999999998,"5min":1.833,"15min":1.8089999999999999},"min":{"1min":0.435,"5min":0.371,"15min":0.36799999999999999},"max":{"1min":6.5650000000000004,"5min":7.3209999999999997,"15min":84.831999999999994},"last":0.80100000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:39:28 2023","interfaces":[{"interface":"back","average":{"1min":2.4780000000000002,"5min":2.1760000000000002,"15min":2.3109999999999999},"min":{"1min":0.55000000000000004,"5min":0.36199999999999999,"15min":0.33100000000000002},"max":{"1min":8.1959999999999997,"5min":14.364000000000001,"15min":84.888000000000005},"last":0.94399999999999995},{"interface":"front","average":{"1min":2.6829999999999998,"5min":1.8380000000000001,"15min":1.99},"min":{"1min":0.51900000000000002,"5min":0.441,"15min":0.36699999999999999},"max":{"1min":7.2549999999999999,"5min":8.9540000000000006,"15min":84.742000000000004},"last":0.871}]}]},{"osd":0,"up_from":1136,"seq":4879082848309,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":187656,"kb_used_data":6336,"kb_used_omap":280,"kb_used_meta":181031,"kb_avail":93557496,"statfs":{"total":95995035648,"available":95802875904,"internally_reserved":0,"allocated":6488064,"data_stored":5269261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":185376154},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:39:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":2.1760000000000002,"15min":1.2549999999999999},"min":{"1min":0.245,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":9.3140000000000001,"5min":33.634,"15min":33.634},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.3140000000000001,"15min":1.464},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":12.073,"5min":33.661000000000001,"15min":33.661000000000001},"last":0.53000000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:39:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8540000000000001,"5min":1.5089999999999999,"15min":1.0880000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.0709999999999997,"5min":8.9640000000000004,"15min":8.9640000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.9510000000000001,"5min":1.7390000000000001,"15min":1.413},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.7560000000000002,"5min":14.382999999999999,"15min":14.382999999999999},"last":0.79100000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:39:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":1.5720000000000001,"15min":1.339},"min":{"1min":0.30299999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.29,"5min":7.9219999999999997,"15min":7.9219999999999997},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.5940000000000001,"15min":1.103},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":9.2400000000000002,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.66400000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:39:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":1.528,"15min":1.2789999999999999},"min":{"1min":0.29999999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.3780000000000001,"5min":6.8390000000000004,"15min":6.8390000000000004},"last":0.85199999999999998},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.6539999999999999,"15min":1.379},"min":{"1min":0.36199999999999999,"5min":0.313,"15min":0.313},"max":{"1min":6.9009999999999998,"5min":6.9009999999999998,"15min":6.9009999999999998},"last":0.96899999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:39:25 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.5169999999999999,"15min":1.347},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.694,"5min":6.694,"15min":6.694},"last":1.1659999999999999},{"interface":"front","average":{"1min":1.6200000000000001,"5min":1.748,"15min":1.6619999999999999},"min":{"1min":0.35699999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.8559999999999999,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":1.024}]},{"osd":6,"last update":"Mon Oct 9 09:39:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7689999999999999,"5min":1.55,"15min":1.2310000000000001},"min":{"1min":0.34300000000000003,"5min":0.315,"15min":0.315},"max":{"1min":6.2830000000000004,"5min":7.6909999999999998,"15min":7.6909999999999998},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.837,"5min":2.0129999999999999,"15min":1.8799999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":5.4180000000000001,"5min":8.625,"15min":8.625},"last":1.0069999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:39:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":1.833,"15min":1.397},"min":{"1min":0.502,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":9.2159999999999993,"5min":9.2159999999999993,"15min":9.2159999999999993},"last":1.1379999999999999},{"interface":"front","average":{"1min":1.5820000000000001,"5min":1.7949999999999999,"15min":1.6679999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":4.2370000000000001,"5min":8.1319999999999997,"15min":8.1319999999999997},"last":1.1240000000000001}]}]},{"osd":3,"up_from":663,"seq":2847563317538,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588064,"kb_used_data":5656,"kb_used_omap":55,"kb_used_meta":582344,"kb_avail":93157088,"statfs":{"total":95995035648,"available":95392858112,"internally_reserved":0,"allocated":5791744,"data_stored":4655732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":596320866},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:39:26 2023","interfaces":[{"interface":"back","average":{"1min":2.395,"5min":1.609,"15min":1.115},"min":{"1min":0.30099999999999999,"5min":0.185,"15min":0.185},"max":{"1min":9.1669999999999998,"5min":28.391999999999999,"15min":28.391999999999999},"last":0.745},{"interface":"front","average":{"1min":1.6060000000000001,"5min":1.4359999999999999,"15min":1.0780000000000001},"min":{"1min":0.221,"5min":0.214,"15min":0.214},"max":{"1min":6.7359999999999998,"5min":6.7359999999999998,"15min":6.7359999999999998},"last":0.55800000000000005}]},{"osd":1,"last update":"Mon Oct 9 09:39:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":2.4830000000000001,"15min":2.2909999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.245},"max":{"1min":8.3949999999999996,"5min":80.864000000000004,"15min":91.159000000000006},"last":0.69199999999999995},{"interface":"front","average":{"1min":3.089,"5min":3.4580000000000002,"15min":2.7029999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.22900000000000001},"max":{"1min":9.5570000000000004,"5min":81.924000000000007,"15min":92.683000000000007},"last":0.65300000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:39:26 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":1.8120000000000001,"15min":2.3399999999999999},"min":{"1min":0.29499999999999998,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":7.452,"5min":8.8290000000000006,"15min":106.122},"last":0.72199999999999998},{"interface":"front","average":{"1min":3.1749999999999998,"5min":1.7689999999999999,"15min":2.181},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.223},"max":{"1min":9.2140000000000004,"5min":28.417000000000002,"15min":104.102},"last":0.60299999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:39:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":1.323,"15min":1.498},"min":{"1min":0.34200000000000003,"5min":0.32900000000000001,"15min":0.32200000000000001},"max":{"1min":11.32,"5min":11.32,"15min":12.629},"last":0.874},{"interface":"front","average":{"1min":1.79,"5min":1.581,"15min":1.5489999999999999},"min":{"1min":0.53100000000000003,"5min":0.34200000000000003,"15min":0.27500000000000002},"max":{"1min":6.2009999999999996,"5min":28.292999999999999,"15min":28.292999999999999},"last":0.93100000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:39:26 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":1.569,"15min":1.764},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":7.5030000000000001,"5min":8.2289999999999992,"15min":11.272},"last":0.88900000000000001},{"interface":"front","average":{"1min":1.395,"5min":1.4690000000000001,"15min":1.6479999999999999},"min":{"1min":0.35699999999999998,"5min":0.34899999999999998,"15min":0.34100000000000003},"max":{"1min":6.0529999999999999,"5min":7.1120000000000001,"15min":11.388999999999999},"last":0.91200000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:39:26 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":1.5840000000000001,"15min":1.716},"min":{"1min":0.52600000000000002,"5min":0.373,"15min":0.30099999999999999},"max":{"1min":6.226,"5min":28.341000000000001,"15min":28.341000000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":1.6890000000000001,"5min":1.5720000000000001,"15min":1.7090000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.316},"max":{"1min":5.0839999999999996,"5min":8.1150000000000002,"15min":17.234999999999999},"last":0.77400000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:39:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4329999999999998,"5min":1.8680000000000001,"15min":1.8799999999999999},"min":{"1min":0.54200000000000004,"5min":0.38300000000000001,"15min":0.34999999999999998},"max":{"1min":5.6870000000000003,"5min":8.6310000000000002,"15min":10.029999999999999},"last":0.80000000000000004},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.694,"15min":1.7649999999999999},"min":{"1min":0.55400000000000005,"5min":0.40000000000000002,"15min":0.318},"max":{"1min":9.2349999999999994,"5min":28.367999999999999,"15min":28.367999999999999},"last":0.98599999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986672,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":381652,"kb_used_data":80332,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93363500,"statfs":{"total":95995035648,"available":95604224000,"internally_reserved":0,"allocated":82259968,"data_stored":82022412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:39:20 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.6080000000000001,"15min":1.173},"min":{"1min":0.309,"5min":0.253,"15min":0.253},"max":{"1min":6.5590000000000002,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.78300000000000003},{"interface":"front","average":{"1min":2.4169999999999998,"5min":1.863,"15min":1.3899999999999999},"min":{"1min":0.30499999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":11.627000000000001,"5min":16.387,"15min":16.387},"last":0.81200000000000006}]},{"osd":1,"last update":"Mon Oct 9 09:39:33 2023","interfaces":[{"interface":"back","average":{"1min":1.903,"5min":2.6520000000000001,"15min":2.1709999999999998},"min":{"1min":0.32300000000000001,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":11.678000000000001,"5min":64.671999999999997,"15min":85.186999999999998},"last":0.65400000000000003},{"interface":"front","average":{"1min":2.387,"5min":3.2090000000000001,"15min":2.4500000000000002},"min":{"1min":0.32600000000000001,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":13.612,"5min":64.576999999999998,"15min":85.159000000000006},"last":0.84599999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:38:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.226,"15min":1.6619999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.8789999999999996,"5min":9.4619999999999997,"15min":99.736000000000004},"last":0.625},{"interface":"front","average":{"1min":2.4649999999999999,"5min":1.399,"15min":1.671},"min":{"1min":0.32100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.7219999999999995,"5min":8.7219999999999995,"15min":99.212999999999994},"last":0.68000000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:39:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.367,"15min":2.028},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.26500000000000001},"max":{"1min":5.5789999999999997,"5min":6.7549999999999999,"15min":47.32},"last":0.73399999999999999},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.6859999999999999,"15min":2.504},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.314},"max":{"1min":16.507000000000001,"5min":16.507000000000001,"15min":93.843000000000004},"last":0.75800000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:39:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.633,"15min":1.407},"min":{"1min":0.38,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":8.1850000000000005,"5min":17.212,"15min":17.212},"last":0.83699999999999997},{"interface":"front","average":{"1min":2.8050000000000002,"5min":1.9259999999999999,"15min":1.514},"min":{"1min":0.437,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":11.566000000000001,"5min":17.268999999999998,"15min":17.268999999999998},"last":0.92700000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:39:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":1.5569999999999999,"15min":1.3600000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":13.654,"5min":13.654,"15min":13.654},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.47,"5min":1.5549999999999999,"15min":1.369},"min":{"1min":0.41599999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.9829999999999997,"5min":9.3369999999999997,"15min":9.3369999999999997},"last":0.86799999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.4119999999999999,"15min":1.2649999999999999},"min":{"1min":0.45600000000000002,"5min":0.40100000000000002,"15min":0.33400000000000002},"max":{"1min":5.0880000000000001,"5min":9.2889999999999997,"15min":9.2889999999999997},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.83299999999999996,"5min":1.429,"15min":1.4059999999999999},"min":{"1min":0.39800000000000002,"5min":0.374,"15min":0.36899999999999999},"max":{"1min":2.6640000000000001,"5min":12.34,"15min":12.34},"last":0.88600000000000001}]}]},{"osd":5,"up_from":0,"seq":158913790438,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":247398400,"data_stored":247398400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":75563008,"data_stored":75563008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":262803456,"data_stored":262799384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":396296192,"data_stored":396296192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:39:40.830 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T09:39:40.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 26 2023-10-09T09:39:40.836 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:40.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:40.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:40.940 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:40.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:40.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:40.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:41.039 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:39:41.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:39:41.073 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:41.099 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:40 smithi163 bash[78107]: audit 2023-10-09T09:39:39.774024+0000 mgr.y (mgr.34107) 1006 : audit 0 from='client.35319 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:39:41.099 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:40 smithi163 bash[78107]: audit 2023-10-09T09:39:40.383342+0000 mon.a (mon.0) 1933 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:41.099 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:40 smithi163 bash[78107]: cluster 2023-10-09T09:39:40.721078+0000 mon.a (mon.0) 1934 : cluster 3 Health check update: Degraded data redundancy: 8889/72 objects degraded (12345.833%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:39:41.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:40 smithi163 bash[75425]: audit 2023-10-09T09:39:39.774024+0000 mgr.y (mgr.34107) 1006 : audit 0 from='client.35319 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:39:41.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:40 smithi163 bash[75425]: audit 2023-10-09T09:39:40.383342+0000 mon.a (mon.0) 1933 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:41.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:40 smithi163 bash[75425]: cluster 2023-10-09T09:39:40.721078+0000 mon.a (mon.0) 1934 : cluster 3 Health check update: Degraded data redundancy: 8889/72 objects degraded (12345.833%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:39:41.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:41.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:41.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:41.230 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:40 smithi181 bash[59345]: audit 2023-10-09T09:39:39.774024+0000 mgr.y (mgr.34107) 1006 : audit 0 from='client.35319 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:39:41.230 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:40 smithi181 bash[59345]: audit 2023-10-09T09:39:40.383342+0000 mon.a (mon.0) 1933 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:41.231 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:40 smithi181 bash[59345]: cluster 2023-10-09T09:39:40.721078+0000 mon.a (mon.0) 1934 : cluster 3 Health check update: Degraded data redundancy: 8889/72 objects degraded (12345.833%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:39:41.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:41.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:41.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:41.350 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:41.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:41.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:41.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:41.486 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:41.513 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:41.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:41.617 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:41.650 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:41.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:41.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:41.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:41.785 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:41.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:41.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:41.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:41.923 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:41.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:41.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:42.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:42.038 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:41 smithi163 bash[78107]: cluster 2023-10-09T09:39:39.941123+0000 mgr.y (mgr.34107) 1007 : cluster 0 pgmap v1775: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 25 op/s; 8889/72 objects degraded (12345.833%); 0 B/s, 69 objects/s recovering 2023-10-09T09:39:42.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:41 smithi163 bash[75425]: cluster 2023-10-09T09:39:39.941123+0000 mgr.y (mgr.34107) 1007 : cluster 0 pgmap v1775: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 25 op/s; 8889/72 objects degraded (12345.833%); 0 B/s, 69 objects/s recovering 2023-10-09T09:39:42.058 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:42.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:42.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:42.162 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:42.172 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:41 smithi181 bash[59345]: cluster 2023-10-09T09:39:39.941123+0000 mgr.y (mgr.34107) 1007 : cluster 0 pgmap v1775: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 25 op/s; 8889/72 objects degraded (12345.833%); 0 B/s, 69 objects/s recovering 2023-10-09T09:39:42.188 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:42.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:42.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:42.291 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:42.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:42.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:42.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:42.420 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:42.451 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:42.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:42.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:42.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:42.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:42.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:42.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:42.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:42.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:42.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:42.827 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:42.853 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:42.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:42.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:42.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:42.991 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:42.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:42.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:43.095 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:43.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:43.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:43.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:43.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:43.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:43.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:43.341 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:39:43.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:43.359 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:39:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:39:43] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.33.4" 2023-10-09T09:39:43.374 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:43.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:43.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:43.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:43.503 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:43.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:43.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:43.607 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:43.629 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:43.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:43.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:43.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:43.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:43.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:43.859 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:43.886 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:43.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:43.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:43.990 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:43.997 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:43 smithi163 bash[78107]: cluster 2023-10-09T09:39:41.942500+0000 mgr.y (mgr.34107) 1008 : cluster 0 pgmap v1776: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 73 objects/s recovering 2023-10-09T09:39:43.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:43 smithi163 bash[75425]: cluster 2023-10-09T09:39:41.942500+0000 mgr.y (mgr.34107) 1008 : cluster 0 pgmap v1776: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 73 objects/s recovering 2023-10-09T09:39:44.012 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:44.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:44.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:44.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:44.141 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:44.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:44.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:44.244 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:44.273 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:44.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:44.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:44.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:43 smithi181 bash[59345]: cluster 2023-10-09T09:39:41.942500+0000 mgr.y (mgr.34107) 1008 : cluster 0 pgmap v1776: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 73 objects/s recovering 2023-10-09T09:39:44.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:44.421 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:44.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:44.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:44.525 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:44.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:44.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:44.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:44.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:44.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:44.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:44.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:44.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:44.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:44.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:44.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:44.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:44.958 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:44.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:44.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:45.062 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:45.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:45.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:45.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:45.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:45.220 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:45.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:45.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:45.324 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:45.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:45 smithi163 bash[78107]: cluster 2023-10-09T09:39:43.943122+0000 mgr.y (mgr.34107) 1009 : cluster 0 pgmap v1777: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 67 objects/s recovering 2023-10-09T09:39:45.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:45 smithi163 bash[78107]: audit 2023-10-09T09:39:44.250561+0000 mon.a (mon.0) 1935 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:45 smithi163 bash[78107]: audit 2023-10-09T09:39:44.259115+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:45 smithi163 bash[78107]: audit 2023-10-09T09:39:45.111227+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:45 smithi163 bash[78107]: audit 2023-10-09T09:39:45.117301+0000 mon.a (mon.0) 1938 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:45 smithi163 bash[75425]: cluster 2023-10-09T09:39:43.943122+0000 mgr.y (mgr.34107) 1009 : cluster 0 pgmap v1777: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 67 objects/s recovering 2023-10-09T09:39:45.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:45 smithi163 bash[75425]: audit 2023-10-09T09:39:44.250561+0000 mon.a (mon.0) 1935 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:45 smithi163 bash[75425]: audit 2023-10-09T09:39:44.259115+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:45 smithi163 bash[75425]: audit 2023-10-09T09:39:45.111227+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:45 smithi163 bash[75425]: audit 2023-10-09T09:39:45.117301+0000 mon.a (mon.0) 1938 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.348 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:45.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:45.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:45.451 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:45.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:45.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:45 smithi181 bash[59345]: cluster 2023-10-09T09:39:43.943122+0000 mgr.y (mgr.34107) 1009 : cluster 0 pgmap v1777: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 67 objects/s recovering 2023-10-09T09:39:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:45 smithi181 bash[59345]: audit 2023-10-09T09:39:44.250561+0000 mon.a (mon.0) 1935 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:45 smithi181 bash[59345]: audit 2023-10-09T09:39:44.259115+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:45 smithi181 bash[59345]: audit 2023-10-09T09:39:45.111227+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:45 smithi181 bash[59345]: audit 2023-10-09T09:39:45.117301+0000 mon.a (mon.0) 1938 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:45.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:45.608 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:45.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:45.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:45.712 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:45.738 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:45.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:45.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:45.842 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:45.866 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:45.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:45.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:45.991 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:45.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:45.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:46.095 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:46.120 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:46.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:46.223 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:46.254 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:46.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:46.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:46.358 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:46.378 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:46.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:46.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:46.403 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:39:46.403 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:39:46.404 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:39:46.480 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:46.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:46 smithi163 bash[78107]: audit 2023-10-09T09:39:45.685873+0000 mon.a (mon.0) 1939 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:39:46.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:46 smithi163 bash[78107]: audit 2023-10-09T09:39:45.687119+0000 mon.a (mon.0) 1940 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:39:46.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:46 smithi163 bash[78107]: audit 2023-10-09T09:39:45.694807+0000 mon.a (mon.0) 1941 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:46.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:46 smithi163 bash[75425]: audit 2023-10-09T09:39:45.685873+0000 mon.a (mon.0) 1939 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:39:46.494 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:46 smithi163 bash[75425]: audit 2023-10-09T09:39:45.687119+0000 mon.a (mon.0) 1940 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:39:46.494 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:46 smithi163 bash[75425]: audit 2023-10-09T09:39:45.694807+0000 mon.a (mon.0) 1941 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:46.507 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:46.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:46.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:46 smithi181 bash[59345]: audit 2023-10-09T09:39:45.685873+0000 mon.a (mon.0) 1939 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:39:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:46 smithi181 bash[59345]: audit 2023-10-09T09:39:45.687119+0000 mon.a (mon.0) 1940 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:39:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:46 smithi181 bash[59345]: audit 2023-10-09T09:39:45.694807+0000 mon.a (mon.0) 1941 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:39:46.611 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:46.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:46.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:46.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:46.744 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:46.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:46.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:46.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:46.903 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:46.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:46.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:47.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:47.034 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:47.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:47.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:47.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:47.163 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:47.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:47.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:47.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:47.296 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:47.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:47.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:47.424 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:47.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:47.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:47.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:47.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:47.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:47.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:47.688 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:47.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:47.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:47.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:47.798 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:47 smithi181 bash[59345]: cluster 2023-10-09T09:39:45.943803+0000 mgr.y (mgr.34107) 1010 : cluster 0 pgmap v1778: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 20 objects/s recovering 2023-10-09T09:39:47.816 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:47.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:47.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:47.919 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:47.945 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:47.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:47.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:48.048 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:48.075 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:48.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:48.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:48.178 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:48.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:48.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:48.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:48.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:47 smithi163 bash[75425]: cluster 2023-10-09T09:39:45.943803+0000 mgr.y (mgr.34107) 1010 : cluster 0 pgmap v1778: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 20 objects/s recovering 2023-10-09T09:39:48.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:47 smithi163 bash[78107]: cluster 2023-10-09T09:39:45.943803+0000 mgr.y (mgr.34107) 1010 : cluster 0 pgmap v1778: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 20 objects/s recovering 2023-10-09T09:39:48.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:48.340 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:48.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:48.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:48.444 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:48.474 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:48.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:48.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:48.578 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:48.606 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:48.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:48.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:48.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:48.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:48.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:48.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:48.745 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:39:48.745 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:39:48.842 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:48.867 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:48.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:48.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:48.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:49.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:49.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:49.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:49.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:49.134 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:49.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:49.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:49.237 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:49.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:49.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:49.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:49.366 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:49.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:49.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:49.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:49.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:49.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:49.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:49.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:49.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:49.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:49.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:49.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:49.761 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:49.775 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:49 smithi181 bash[59345]: cluster 2023-10-09T09:39:47.944997+0000 mgr.y (mgr.34107) 1011 : cluster 0 pgmap v1779: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 29 objects/s recovering 2023-10-09T09:39:49.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:49.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:49.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:49.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:49.904 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:49 smithi163 bash[78107]: cluster 2023-10-09T09:39:47.944997+0000 mgr.y (mgr.34107) 1011 : cluster 0 pgmap v1779: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 29 objects/s recovering 2023-10-09T09:39:49.904 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:49 smithi163 bash[75425]: cluster 2023-10-09T09:39:47.944997+0000 mgr.y (mgr.34107) 1011 : cluster 0 pgmap v1779: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 29 objects/s recovering 2023-10-09T09:39:49.918 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:49.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:49.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:50.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:50.044 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:50.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:50.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:50.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:50.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:50.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:50.277 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:50.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:50.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:50.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:50.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:50.429 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:50.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:50.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:50.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T09:39:50.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:50.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:50.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:50.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:50.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:50.679 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:50.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:50.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:50.783 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:50.809 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:50.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:50.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:50.912 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:50.943 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:50.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:50.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:51.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:51.075 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:51.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:51.178 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:51.206 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:51.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:51.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:51.335 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:51.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:51.438 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:51.469 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:51.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:51.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:51.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:51.600 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:51.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:51.703 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:51.732 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:51.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:51.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:51.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:51.868 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:51.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:51.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:51.973 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:51.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:51.996 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:51 smithi163 bash[78107]: cluster 2023-10-09T09:39:49.945659+0000 mgr.y (mgr.34107) 1012 : cluster 0 pgmap v1780: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 19 objects/s recovering 2023-10-09T09:39:51.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:51 smithi163 bash[75425]: cluster 2023-10-09T09:39:49.945659+0000 mgr.y (mgr.34107) 1012 : cluster 0 pgmap v1780: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 19 objects/s recovering 2023-10-09T09:39:51.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:51.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:52.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:51 smithi181 bash[59345]: cluster 2023-10-09T09:39:49.945659+0000 mgr.y (mgr.34107) 1012 : cluster 0 pgmap v1780: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 19 objects/s recovering 2023-10-09T09:39:52.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:52.130 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:52.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:52.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:52.234 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:52.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:52.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:52.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:52.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:52.400 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:52.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:52.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:52.502 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:52.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:52.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:52.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:52.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:52.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:52.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:52.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:52.791 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:52.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:52.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:52.893 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:52.923 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:52.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:52.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:53.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:53.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:53.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:53.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:53.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:53.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:53.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:53.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:53.318 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:53.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:53.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:53.422 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:53.446 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:53.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:53.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:53.549 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:53.561 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:39:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:39:53] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.33.4" 2023-10-09T09:39:53.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:53.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:53.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:53.711 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:53.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:53.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:53.815 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:53.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:53.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:53.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:53.948 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:53.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:53 smithi163 bash[75425]: cluster 2023-10-09T09:39:51.946404+0000 mgr.y (mgr.34107) 1013 : cluster 0 pgmap v1781: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 29 objects/s recovering 2023-10-09T09:39:53.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:53 smithi163 bash[78107]: cluster 2023-10-09T09:39:51.946404+0000 mgr.y (mgr.34107) 1013 : cluster 0 pgmap v1781: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 29 objects/s recovering 2023-10-09T09:39:53.976 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:53.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:53.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:54.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:53 smithi181 bash[59345]: cluster 2023-10-09T09:39:51.946404+0000 mgr.y (mgr.34107) 1013 : cluster 0 pgmap v1781: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 29 objects/s recovering 2023-10-09T09:39:54.080 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:54.104 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:54.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:54.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:54.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:54.238 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:54.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:54.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:54.341 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:54.374 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:54.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:54.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:54.502 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:54.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:54.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:54.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:54.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:54.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:54.734 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:54.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:54.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:54.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:54.862 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:54.874 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:39:54.887 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:54.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:54.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:54.897 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:39:54.926 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:39:54.956 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:39:54.984 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:39:54.990 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:55.005 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:39:55.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:55.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:55.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:55.026 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:39:55.052 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:39:55.054 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:39:55.054 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:39:55.054 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:39:55.054 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:39:55.054 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:39:55.054 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:39:55.054 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:39:55.054 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:39:55.123 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:55.151 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:55.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:55.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:55.228 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:39:55.255 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:55.290 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:55.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:55.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:55.421 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:55.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:55.525 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:55.546 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:55.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:55.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:55.676 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:55.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:55.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:55.760 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 8 pg_num to 26 2023-10-09T09:39:55.760 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:39:55.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:55.802 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:55.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:55.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:55.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:55.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:55.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:55.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:56.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:56.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[75425]: cluster 2023-10-09T09:39:53.947023+0000 mgr.y (mgr.34107) 1014 : cluster 0 pgmap v1782: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 19 objects/s recovering 2023-10-09T09:39:56.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[75425]: audit 2023-10-09T09:39:54.853199+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.163:0/13312459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:39:56.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[75425]: audit 2023-10-09T09:39:55.378648+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.163:0/160570682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:39:56.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[75425]: audit 2023-10-09T09:39:55.379087+0000 mon.a (mon.0) 1942 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:39:56.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[75425]: audit 2023-10-09T09:39:55.382598+0000 mon.a (mon.0) 1943 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:56.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[75425]: audit 2023-10-09T09:39:55.390478+0000 mon.a (mon.0) 1944 : audit 1 from='client.? 172.21.15.163:0/877900038' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:39:56.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[78107]: cluster 2023-10-09T09:39:53.947023+0000 mgr.y (mgr.34107) 1014 : cluster 0 pgmap v1782: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 19 objects/s recovering 2023-10-09T09:39:56.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[78107]: audit 2023-10-09T09:39:54.853199+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.163:0/13312459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:39:56.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[78107]: audit 2023-10-09T09:39:55.378648+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.163:0/160570682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:39:56.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[78107]: audit 2023-10-09T09:39:55.379087+0000 mon.a (mon.0) 1942 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:39:56.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[78107]: audit 2023-10-09T09:39:55.382598+0000 mon.a (mon.0) 1943 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:56.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:55 smithi163 bash[78107]: audit 2023-10-09T09:39:55.390478+0000 mon.a (mon.0) 1944 : audit 1 from='client.? 172.21.15.163:0/877900038' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:39:56.053 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:56.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:56.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:55 smithi181 bash[59345]: cluster 2023-10-09T09:39:53.947023+0000 mgr.y (mgr.34107) 1014 : cluster 0 pgmap v1782: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 19 objects/s recovering 2023-10-09T09:39:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:55 smithi181 bash[59345]: audit 2023-10-09T09:39:54.853199+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.163:0/13312459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:39:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:55 smithi181 bash[59345]: audit 2023-10-09T09:39:55.378648+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.163:0/160570682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:39:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:55 smithi181 bash[59345]: audit 2023-10-09T09:39:55.379087+0000 mon.a (mon.0) 1942 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:39:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:55 smithi181 bash[59345]: audit 2023-10-09T09:39:55.382598+0000 mon.a (mon.0) 1943 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:39:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:55 smithi181 bash[59345]: audit 2023-10-09T09:39:55.390478+0000 mon.a (mon.0) 1944 : audit 1 from='client.? 172.21.15.163:0/877900038' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:39:56.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:56.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:56.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:56.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:56.287 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:56.315 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:56.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:56.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:56.418 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:56.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:56.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:56.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:56.546 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:56.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:56.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:56.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:56.679 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:56.701 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:56.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:56.805 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:56.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:56.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:56.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:56.957 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[75425]: cluster 2023-10-09T09:39:55.740312+0000 mon.a (mon.0) 1945 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:39:56.957 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[75425]: cluster 2023-10-09T09:39:55.751448+0000 osd.5 (osd.5) 737 : cluster 3 failed to encode map e1170 with expected crc 2023-10-09T09:39:56.957 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[75425]: cluster 2023-10-09T09:39:55.751869+0000 osd.7 (osd.7) 560 : cluster 3 failed to encode map e1170 with expected crc 2023-10-09T09:39:56.957 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[75425]: audit 2023-10-09T09:39:55.755086+0000 mon.a (mon.0) 1946 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:39:56.958 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[75425]: audit 2023-10-09T09:39:55.755221+0000 mon.a (mon.0) 1947 : audit 1 from='client.? 172.21.15.163:0/877900038' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:39:56.958 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[75425]: cluster 2023-10-09T09:39:55.755324+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e1170: 8 total, 8 up, 7 in 2023-10-09T09:39:56.958 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[75425]: audit 2023-10-09T09:39:55.949106+0000 mon.a (mon.0) 1949 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:39:56.958 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[78107]: cluster 2023-10-09T09:39:55.740312+0000 mon.a (mon.0) 1945 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:39:56.958 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[78107]: cluster 2023-10-09T09:39:55.751448+0000 osd.5 (osd.5) 737 : cluster 3 failed to encode map e1170 with expected crc 2023-10-09T09:39:56.958 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[78107]: cluster 2023-10-09T09:39:55.751869+0000 osd.7 (osd.7) 560 : cluster 3 failed to encode map e1170 with expected crc 2023-10-09T09:39:56.958 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[78107]: audit 2023-10-09T09:39:55.755086+0000 mon.a (mon.0) 1946 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:39:56.958 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[78107]: audit 2023-10-09T09:39:55.755221+0000 mon.a (mon.0) 1947 : audit 1 from='client.? 172.21.15.163:0/877900038' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:39:56.959 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[78107]: cluster 2023-10-09T09:39:55.755324+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e1170: 8 total, 8 up, 7 in 2023-10-09T09:39:56.959 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:56 smithi163 bash[78107]: audit 2023-10-09T09:39:55.949106+0000 mon.a (mon.0) 1949 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:39:56.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:56.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:56.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:56 smithi181 bash[59345]: cluster 2023-10-09T09:39:55.740312+0000 mon.a (mon.0) 1945 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:39:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:56 smithi181 bash[59345]: cluster 2023-10-09T09:39:55.751448+0000 osd.5 (osd.5) 737 : cluster 3 failed to encode map e1170 with expected crc 2023-10-09T09:39:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:56 smithi181 bash[59345]: cluster 2023-10-09T09:39:55.751869+0000 osd.7 (osd.7) 560 : cluster 3 failed to encode map e1170 with expected crc 2023-10-09T09:39:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:56 smithi181 bash[59345]: audit 2023-10-09T09:39:55.755086+0000 mon.a (mon.0) 1946 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:39:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:56 smithi181 bash[59345]: audit 2023-10-09T09:39:55.755221+0000 mon.a (mon.0) 1947 : audit 1 from='client.? 172.21.15.163:0/877900038' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:39:57.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:56 smithi181 bash[59345]: cluster 2023-10-09T09:39:55.755324+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e1170: 8 total, 8 up, 7 in 2023-10-09T09:39:57.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:56 smithi181 bash[59345]: audit 2023-10-09T09:39:55.949106+0000 mon.a (mon.0) 1949 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:39:57.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:57.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:57.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:57.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:57.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:57.240 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:57.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:57.344 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:57.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:57.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:57.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:57.473 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:57.500 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:57.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:57.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:57.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:57.625 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:57.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:57.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:57.727 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:57.748 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:57.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:57.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:57.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:57.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:57 smithi163 bash[75425]: cluster 2023-10-09T09:39:55.947539+0000 mgr.y (mgr.34107) 1015 : cluster 0 pgmap v1784: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 23 objects/s recovering 2023-10-09T09:39:57.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:57 smithi163 bash[75425]: cluster 2023-10-09T09:39:56.758516+0000 osd.5 (osd.5) 738 : cluster 3 failed to encode map e1171 with expected crc 2023-10-09T09:39:57.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:57 smithi163 bash[75425]: audit 2023-10-09T09:39:56.761200+0000 mon.a (mon.0) 1950 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:39:57.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:57 smithi163 bash[75425]: cluster 2023-10-09T09:39:56.761330+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e1171: 8 total, 8 up, 7 in 2023-10-09T09:39:57.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:57 smithi163 bash[78107]: cluster 2023-10-09T09:39:55.947539+0000 mgr.y (mgr.34107) 1015 : cluster 0 pgmap v1784: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 23 objects/s recovering 2023-10-09T09:39:57.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:57 smithi163 bash[78107]: cluster 2023-10-09T09:39:56.758516+0000 osd.5 (osd.5) 738 : cluster 3 failed to encode map e1171 with expected crc 2023-10-09T09:39:57.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:57 smithi163 bash[78107]: audit 2023-10-09T09:39:56.761200+0000 mon.a (mon.0) 1950 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:39:57.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:57 smithi163 bash[78107]: cluster 2023-10-09T09:39:56.761330+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e1171: 8 total, 8 up, 7 in 2023-10-09T09:39:57.874 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:57.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:57.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:57.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:57.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:57.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:57.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:58.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:57 smithi181 bash[59345]: cluster 2023-10-09T09:39:55.947539+0000 mgr.y (mgr.34107) 1015 : cluster 0 pgmap v1784: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 23 objects/s recovering 2023-10-09T09:39:58.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:57 smithi181 bash[59345]: cluster 2023-10-09T09:39:56.758516+0000 osd.5 (osd.5) 738 : cluster 3 failed to encode map e1171 with expected crc 2023-10-09T09:39:58.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:57 smithi181 bash[59345]: audit 2023-10-09T09:39:56.761200+0000 mon.a (mon.0) 1950 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:39:58.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:57 smithi181 bash[59345]: cluster 2023-10-09T09:39:56.761330+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e1171: 8 total, 8 up, 7 in 2023-10-09T09:39:58.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:58.130 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:58.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:58.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:58.234 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:58.259 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:58.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:58.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:58.363 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:58.387 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:58.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:58.490 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:58.520 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:39:58.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:58.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:39:58.623 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:58.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:58.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:58.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:58.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:58.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:39:58.791 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:39:58.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:58.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:39:58.894 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:58.904 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.760391+0000 osd.5 (osd.5) 739 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.904 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.760409+0000 osd.6 (osd.6) 453 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.904 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.761643+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e1172: 8 total, 8 up, 7 in 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.762125+0000 osd.6 (osd.6) 454 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.762251+0000 osd.7 (osd.7) 561 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.762310+0000 osd.4 (osd.4) 590 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.763864+0000 osd.6 (osd.6) 455 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.764405+0000 osd.1 (osd.1) 5 : cluster 1 8.3 continuing backfill to osd.2 from (1148'5396,1166'12128] MIN to 1166'12128 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.764893+0000 osd.1 (osd.1) 6 : cluster 1 8.0 continuing backfill to osd.4 from (1148'5140,1166'11864] MIN to 1166'11864 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.765302+0000 osd.5 (osd.5) 740 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.765488+0000 osd.7 (osd.7) 562 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.766245+0000 osd.4 (osd.4) 591 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.767584+0000 osd.6 (osd.6) 456 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.768937+0000 osd.4 (osd.4) 592 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.771203+0000 osd.4 (osd.4) 593 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:58.906 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:58 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.774383+0000 osd.4 (osd.4) 594 : cluster 1 8.2 continuing backfill to osd.3 from (1145'4637,1166'11810] MIN to 1166'11810 2023-10-09T09:39:58.918 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:58.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:58.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:39:59.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:59.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.760391+0000 osd.5 (osd.5) 739 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.760409+0000 osd.6 (osd.6) 453 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.761643+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e1172: 8 total, 8 up, 7 in 2023-10-09T09:39:59.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.762125+0000 osd.6 (osd.6) 454 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.762251+0000 osd.7 (osd.7) 561 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.762310+0000 osd.4 (osd.4) 590 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.763864+0000 osd.6 (osd.6) 455 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.764405+0000 osd.1 (osd.1) 5 : cluster 1 8.3 continuing backfill to osd.2 from (1148'5396,1166'12128] MIN to 1166'12128 2023-10-09T09:39:59.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.764893+0000 osd.1 (osd.1) 6 : cluster 1 8.0 continuing backfill to osd.4 from (1148'5140,1166'11864] MIN to 1166'11864 2023-10-09T09:39:59.029 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.765302+0000 osd.5 (osd.5) 740 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.029 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.765488+0000 osd.7 (osd.7) 562 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.029 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.766245+0000 osd.4 (osd.4) 591 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.029 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.767584+0000 osd.6 (osd.6) 456 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.029 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.768937+0000 osd.4 (osd.4) 592 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.029 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.771203+0000 osd.4 (osd.4) 593 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.030 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.774383+0000 osd.4 (osd.4) 594 : cluster 1 8.2 continuing backfill to osd.3 from (1145'4637,1166'11810] MIN to 1166'11810 2023-10-09T09:39:59.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.760391+0000 osd.5 (osd.5) 739 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.760409+0000 osd.6 (osd.6) 453 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.761643+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e1172: 8 total, 8 up, 7 in 2023-10-09T09:39:59.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.762125+0000 osd.6 (osd.6) 454 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.762251+0000 osd.7 (osd.7) 561 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.762310+0000 osd.4 (osd.4) 590 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.763864+0000 osd.6 (osd.6) 455 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.764405+0000 osd.1 (osd.1) 5 : cluster 1 8.3 continuing backfill to osd.2 from (1148'5396,1166'12128] MIN to 1166'12128 2023-10-09T09:39:59.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.764893+0000 osd.1 (osd.1) 6 : cluster 1 8.0 continuing backfill to osd.4 from (1148'5140,1166'11864] MIN to 1166'11864 2023-10-09T09:39:59.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.765302+0000 osd.5 (osd.5) 740 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.765488+0000 osd.7 (osd.7) 562 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.766245+0000 osd.4 (osd.4) 591 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.767584+0000 osd.6 (osd.6) 456 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.768937+0000 osd.4 (osd.4) 592 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.771203+0000 osd.4 (osd.4) 593 : cluster 3 failed to encode map e1172 with expected crc 2023-10-09T09:39:59.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:58 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.774383+0000 osd.4 (osd.4) 594 : cluster 1 8.2 continuing backfill to osd.3 from (1145'4637,1166'11810] MIN to 1166'11810 2023-10-09T09:39:59.033 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T09:39:59.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T09:39:59.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:39:59.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:59.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:39:59.155 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:59.179 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:59.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:59.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:59.283 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:59.303 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:39:59.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:59.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:39:59.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:59.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:39:59.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:59.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:39:59.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:59.564 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:39:59.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:59.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:39:59.667 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:59.691 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:59.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:59.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:59.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:59.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[75425]: cluster 2023-10-09T09:39:57.948317+0000 mgr.y (mgr.34107) 1016 : cluster 0 pgmap v1787: 99 pgs: 1 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 19 objects/s recovering 2023-10-09T09:39:59.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[75425]: cluster 2023-10-09T09:39:58.775317+0000 osd.5 (osd.5) 741 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[75425]: cluster 2023-10-09T09:39:58.778193+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e1173: 8 total, 8 up, 7 in 2023-10-09T09:39:59.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[75425]: cluster 2023-10-09T09:39:58.779138+0000 osd.4 (osd.4) 595 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[75425]: cluster 2023-10-09T09:39:58.779913+0000 osd.6 (osd.6) 457 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[75425]: cluster 2023-10-09T09:39:58.782454+0000 osd.7 (osd.7) 563 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[75425]: cluster 2023-10-09T09:39:58.783438+0000 osd.4 (osd.4) 596 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[75425]: cluster 2023-10-09T09:39:58.785223+0000 osd.4 (osd.4) 597 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[78107]: cluster 2023-10-09T09:39:57.948317+0000 mgr.y (mgr.34107) 1016 : cluster 0 pgmap v1787: 99 pgs: 1 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 19 objects/s recovering 2023-10-09T09:39:59.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[78107]: cluster 2023-10-09T09:39:58.775317+0000 osd.5 (osd.5) 741 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[78107]: cluster 2023-10-09T09:39:58.778193+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e1173: 8 total, 8 up, 7 in 2023-10-09T09:39:59.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[78107]: cluster 2023-10-09T09:39:58.779138+0000 osd.4 (osd.4) 595 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[78107]: cluster 2023-10-09T09:39:58.779913+0000 osd.6 (osd.6) 457 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[78107]: cluster 2023-10-09T09:39:58.782454+0000 osd.7 (osd.7) 563 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[78107]: cluster 2023-10-09T09:39:58.783438+0000 osd.4 (osd.4) 596 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:39:59 smithi163 bash[78107]: cluster 2023-10-09T09:39:58.785223+0000 osd.4 (osd.4) 597 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:39:59.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:39:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:59.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:39:59.920 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:59.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:39:59.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:39:59.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:00.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:59 smithi181 bash[59345]: cluster 2023-10-09T09:39:57.948317+0000 mgr.y (mgr.34107) 1016 : cluster 0 pgmap v1787: 99 pgs: 1 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 19 objects/s recovering 2023-10-09T09:40:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:59 smithi181 bash[59345]: cluster 2023-10-09T09:39:58.775317+0000 osd.5 (osd.5) 741 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:40:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:59 smithi181 bash[59345]: cluster 2023-10-09T09:39:58.778193+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e1173: 8 total, 8 up, 7 in 2023-10-09T09:40:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:59 smithi181 bash[59345]: cluster 2023-10-09T09:39:58.779138+0000 osd.4 (osd.4) 595 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:40:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:59 smithi181 bash[59345]: cluster 2023-10-09T09:39:58.779913+0000 osd.6 (osd.6) 457 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:40:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:59 smithi181 bash[59345]: cluster 2023-10-09T09:39:58.782454+0000 osd.7 (osd.7) 563 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:40:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:59 smithi181 bash[59345]: cluster 2023-10-09T09:39:58.783438+0000 osd.4 (osd.4) 596 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:40:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:39:59 smithi181 bash[59345]: cluster 2023-10-09T09:39:58.785223+0000 osd.4 (osd.4) 597 : cluster 3 failed to encode map e1173 with expected crc 2023-10-09T09:40:00.069 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:00.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:00.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:00.174 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:00.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:00.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:00.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:00.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:00.335 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:00.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:00.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:00.439 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:00.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:00.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:00.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:00.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:00.600 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:00.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:00.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:00.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:00.733 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:00.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:00.837 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:00.868 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:00.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:00.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.779642+0000 osd.5 (osd.5) 742 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.782943+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e1174: 8 total, 8 up, 7 in 2023-10-09T09:40:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.783266+0000 osd.4 (osd.4) 598 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.785124+0000 osd.1 (osd.1) 7 : cluster 1 8.10 continuing backfill to osd.4 from (1148'5140,1166'11864] MIN to 1166'11864 2023-10-09T09:40:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.785488+0000 osd.4 (osd.4) 599 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.787010+0000 osd.1 (osd.1) 8 : cluster 1 8.13 continuing backfill to osd.2 from (1148'5396,1166'12128] MIN to 1166'12128 2023-10-09T09:40:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.787963+0000 osd.4 (osd.4) 600 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.788088+0000 osd.6 (osd.6) 458 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.792420+0000 osd.7 (osd.7) 564 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.793554+0000 osd.4 (osd.4) 601 : cluster 1 8.12 continuing backfill to osd.3 from (1145'4637,1166'11810] MIN to 1166'11810 2023-10-09T09:40:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:40:00.000122+0000 mon.a (mon.0) 1955 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 8889/72 objects degraded (12345.833%), 2 pgs degraded 2023-10-09T09:40:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:40:00.000147+0000 mon.a (mon.0) 1956 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 8889/72 objects degraded (12345.833%), 2 pgs degraded 2023-10-09T09:40:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:40:00.000155+0000 mon.a (mon.0) 1957 : cluster 3 pg 8.4 is active+recovering+undersized+degraded+remapped, acting [4,2] 2023-10-09T09:40:00.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[75425]: cluster 2023-10-09T09:40:00.000164+0000 mon.a (mon.0) 1958 : cluster 3 pg 8.d is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-10-09T09:40:00.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.779642+0000 osd.5 (osd.5) 742 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.782943+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e1174: 8 total, 8 up, 7 in 2023-10-09T09:40:00.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.783266+0000 osd.4 (osd.4) 598 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.785124+0000 osd.1 (osd.1) 7 : cluster 1 8.10 continuing backfill to osd.4 from (1148'5140,1166'11864] MIN to 1166'11864 2023-10-09T09:40:00.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.785488+0000 osd.4 (osd.4) 599 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.787010+0000 osd.1 (osd.1) 8 : cluster 1 8.13 continuing backfill to osd.2 from (1148'5396,1166'12128] MIN to 1166'12128 2023-10-09T09:40:00.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.787963+0000 osd.4 (osd.4) 600 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.788088+0000 osd.6 (osd.6) 458 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.792420+0000 osd.7 (osd.7) 564 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:00.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.793554+0000 osd.4 (osd.4) 601 : cluster 1 8.12 continuing backfill to osd.3 from (1145'4637,1166'11810] MIN to 1166'11810 2023-10-09T09:40:00.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:40:00.000122+0000 mon.a (mon.0) 1955 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 8889/72 objects degraded (12345.833%), 2 pgs degraded 2023-10-09T09:40:00.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:40:00.000147+0000 mon.a (mon.0) 1956 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 8889/72 objects degraded (12345.833%), 2 pgs degraded 2023-10-09T09:40:00.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:40:00.000155+0000 mon.a (mon.0) 1957 : cluster 3 pg 8.4 is active+recovering+undersized+degraded+remapped, acting [4,2] 2023-10-09T09:40:00.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:00 smithi163 bash[78107]: cluster 2023-10-09T09:40:00.000164+0000 mon.a (mon.0) 1958 : cluster 3 pg 8.d is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-10-09T09:40:00.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:00.997 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:00.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:01.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:01.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:40:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.779642+0000 osd.5 (osd.5) 742 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.782943+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e1174: 8 total, 8 up, 7 in 2023-10-09T09:40:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.783266+0000 osd.4 (osd.4) 598 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.785124+0000 osd.1 (osd.1) 7 : cluster 1 8.10 continuing backfill to osd.4 from (1148'5140,1166'11864] MIN to 1166'11864 2023-10-09T09:40:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.785488+0000 osd.4 (osd.4) 599 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.787010+0000 osd.1 (osd.1) 8 : cluster 1 8.13 continuing backfill to osd.2 from (1148'5396,1166'12128] MIN to 1166'12128 2023-10-09T09:40:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.787963+0000 osd.4 (osd.4) 600 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.788088+0000 osd.6 (osd.6) 458 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.792420+0000 osd.7 (osd.7) 564 : cluster 3 failed to encode map e1174 with expected crc 2023-10-09T09:40:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.793554+0000 osd.4 (osd.4) 601 : cluster 1 8.12 continuing backfill to osd.3 from (1145'4637,1166'11810] MIN to 1166'11810 2023-10-09T09:40:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:40:00.000122+0000 mon.a (mon.0) 1955 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 8889/72 objects degraded (12345.833%), 2 pgs degraded 2023-10-09T09:40:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:40:00.000147+0000 mon.a (mon.0) 1956 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 8889/72 objects degraded (12345.833%), 2 pgs degraded 2023-10-09T09:40:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:40:00.000155+0000 mon.a (mon.0) 1957 : cluster 3 pg 8.4 is active+recovering+undersized+degraded+remapped, acting [4,2] 2023-10-09T09:40:01.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:00 smithi181 bash[59345]: cluster 2023-10-09T09:40:00.000164+0000 mon.a (mon.0) 1958 : cluster 3 pg 8.d is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-10-09T09:40:01.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:01.129 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:01.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:01.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:01.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:01.256 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:01.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:01.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:01.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:01.390 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:01.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:01.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:01.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:01.525 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:01.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:01.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:01.628 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:01.660 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:01.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:01.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:01.764 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:01.791 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:01.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:01.894 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:01.920 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:01.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:01.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:01.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:01 smithi163 bash[75425]: cluster 2023-10-09T09:39:59.949071+0000 mgr.y (mgr.34107) 1017 : cluster 0 pgmap v1790: 99 pgs: 2 remapped+peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 1 peering; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 29 objects/s recovering 2023-10-09T09:40:01.945 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:01 smithi163 bash[78107]: cluster 2023-10-09T09:39:59.949071+0000 mgr.y (mgr.34107) 1017 : cluster 0 pgmap v1790: 99 pgs: 2 remapped+peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 1 peering; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 29 objects/s recovering 2023-10-09T09:40:02.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:02.051 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:02.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:02.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:01 smithi181 bash[59345]: cluster 2023-10-09T09:39:59.949071+0000 mgr.y (mgr.34107) 1017 : cluster 0 pgmap v1790: 99 pgs: 2 remapped+peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 1 peering; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 8889/72 objects degraded (12345.833%); 0 B/s, 29 objects/s recovering 2023-10-09T09:40:02.154 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:02.180 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:02.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:02.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:02.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:02.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:02.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:02.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:02.361 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:40:02.361 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:40:02.361 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T09:40:02.361 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T09:40:02.361 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:40:02.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:02.442 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:02.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:02.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:02.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:02.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:02.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:02.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:02.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:02.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:02.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:02.805 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:02.835 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:02.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:02.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:02.938 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:02.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:02.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:02.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:03.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:03.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:02 smithi163 bash[75425]: cluster 2023-10-09T09:40:02.796967+0000 mon.a (mon.0) 1959 : cluster 3 Health check update: Degraded data redundancy: 7169/72 objects degraded (9956.944%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:40:03.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:02 smithi163 bash[78107]: cluster 2023-10-09T09:40:02.796967+0000 mon.a (mon.0) 1959 : cluster 3 Health check update: Degraded data redundancy: 7169/72 objects degraded (9956.944%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:40:03.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:03.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:03.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:03.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:03.211 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:02 smithi181 bash[59345]: cluster 2023-10-09T09:40:02.796967+0000 mon.a (mon.0) 1959 : cluster 3 Health check update: Degraded data redundancy: 7169/72 objects degraded (9956.944%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:40:03.230 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:03.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:03.334 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:03.363 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:03.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:03.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:03.453 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T09:40:03.466 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:03.494 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:03.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:03.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:03.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:03.621 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:03.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:03.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:40:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:40:03] "GET /metrics HTTP/1.1" 200 36333 "" "Prometheus/2.33.4" 2023-10-09T09:40:03.724 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:03.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:03.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:03.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:03.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:03.884 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:03.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:03.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:03.989 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:04.000 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:03 smithi181 bash[59345]: cluster 2023-10-09T09:40:01.949943+0000 mgr.y (mgr.34107) 1018 : cluster 0 pgmap v1791: 99 pgs: 2 remapped+peering, 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 unknown, 1 peering; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 53 objects/s recovering 2023-10-09T09:40:04.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:04.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:04.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:04.034 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 6, 7, 2] out_osds: [5] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:40:04.035 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:40:04.035 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-09T09:40:04.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-09T09:40:04.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:04.129 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:03 smithi163 bash[78107]: cluster 2023-10-09T09:40:01.949943+0000 mgr.y (mgr.34107) 1018 : cluster 0 pgmap v1791: 99 pgs: 2 remapped+peering, 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 unknown, 1 peering; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 53 objects/s recovering 2023-10-09T09:40:04.130 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:03 smithi163 bash[75425]: cluster 2023-10-09T09:40:01.949943+0000 mgr.y (mgr.34107) 1018 : cluster 0 pgmap v1791: 99 pgs: 2 remapped+peering, 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 unknown, 1 peering; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 53 objects/s recovering 2023-10-09T09:40:04.145 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:04.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:04.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:04.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:04.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:04.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:04.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:04.383 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:04.410 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:04.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:04.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:04.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:04.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:04.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:04.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:04.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:04.675 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:04.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:04.779 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:04.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:04.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:04.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:04.913 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:04.943 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:04.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:04.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:04.950 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:04.953 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:05.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:05.074 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:05.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:05.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:05.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:05.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:05.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:05.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:05.307 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:05.334 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:05.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:05.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:05.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:05.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:05.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:05.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:05.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:05.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:05.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:05.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:05.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:05.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:05.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:05.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:05.825 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:05.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:05.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:05.953 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:05.969 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:05.980 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:05.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:05.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:06.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:05 smithi181 bash[59345]: cluster 2023-10-09T09:40:03.950589+0000 mgr.y (mgr.34107) 1019 : cluster 0 pgmap v1792: 99 pgs: 2 remapped+peering, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 44 objects/s recovering 2023-10-09T09:40:06.084 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:06.113 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:06.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:06.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:06.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:06.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:05 smithi163 bash[78107]: cluster 2023-10-09T09:40:03.950589+0000 mgr.y (mgr.34107) 1019 : cluster 0 pgmap v1792: 99 pgs: 2 remapped+peering, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 44 objects/s recovering 2023-10-09T09:40:06.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:05 smithi163 bash[75425]: cluster 2023-10-09T09:40:03.950589+0000 mgr.y (mgr.34107) 1019 : cluster 0 pgmap v1792: 99 pgs: 2 remapped+peering, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 44 objects/s recovering 2023-10-09T09:40:06.243 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:06.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:06.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:06.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:06.374 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:06.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:06.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:06.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:06.504 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:06.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:06.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:06.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:06.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:06.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:06.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:06.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:06.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:06.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:06.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:06.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:06.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:06.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:07.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:07.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:07.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:07.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:07.146 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:07.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:07.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:07.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:07.272 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:07.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:07.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:07.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:07.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:07.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:07.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:07.510 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:07.536 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:07.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:07.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:07.640 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:07.667 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:07.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:07.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:07.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:07.803 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:07.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:07.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:07.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:07.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:07.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:07.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:07.997 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:07.999 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:08.034 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:08.045 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:07 smithi181 bash[59345]: cluster 2023-10-09T09:40:05.951429+0000 mgr.y (mgr.34107) 1020 : cluster 0 pgmap v1793: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 34 objects/s recovering 2023-10-09T09:40:08.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:08.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:08.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:08.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:08.193 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:08.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:08.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:07 smithi163 bash[78107]: cluster 2023-10-09T09:40:05.951429+0000 mgr.y (mgr.34107) 1020 : cluster 0 pgmap v1793: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 34 objects/s recovering 2023-10-09T09:40:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:07 smithi163 bash[75425]: cluster 2023-10-09T09:40:05.951429+0000 mgr.y (mgr.34107) 1020 : cluster 0 pgmap v1793: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 34 objects/s recovering 2023-10-09T09:40:08.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:08.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:08.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:08.427 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:08.452 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:08.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:08.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:08.581 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:08.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:08.684 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:08.713 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:08.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:08.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:08.816 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:08.840 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:08.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:08.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:08.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:08.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:08.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:08.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:09.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:08 smithi181 bash[59345]: cluster 2023-10-09T09:40:08.857447+0000 mon.a (mon.0) 1960 : cluster 3 Health check update: Degraded data redundancy: 7167/72 objects degraded (9954.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:40:09.074 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:09.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:09.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:09.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:09.130 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:08 smithi163 bash[78107]: cluster 2023-10-09T09:40:08.857447+0000 mon.a (mon.0) 1960 : cluster 3 Health check update: Degraded data redundancy: 7167/72 objects degraded (9954.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:40:09.130 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:08 smithi163 bash[75425]: cluster 2023-10-09T09:40:08.857447+0000 mon.a (mon.0) 1960 : cluster 3 Health check update: Degraded data redundancy: 7167/72 objects degraded (9954.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:40:09.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:09.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:09.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:09.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:09.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:09.367 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:09.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:09.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:09.502 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:09.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:09.606 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:09.628 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:09.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:09.731 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:09.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:09.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:09.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:09.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:09.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:09 smithi163 bash[75425]: cluster 2023-10-09T09:40:07.952355+0000 mgr.y (mgr.34107) 1021 : cluster 0 pgmap v1794: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 77 objects/s recovering 2023-10-09T09:40:09.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:09 smithi163 bash[78107]: cluster 2023-10-09T09:40:07.952355+0000 mgr.y (mgr.34107) 1021 : cluster 0 pgmap v1794: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 77 objects/s recovering 2023-10-09T09:40:09.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:09.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:09.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:10.096 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:10.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:10.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:10.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:10.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:10.247 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:10.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:10.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:09 smithi181 bash[59345]: cluster 2023-10-09T09:40:07.952355+0000 mgr.y (mgr.34107) 1021 : cluster 0 pgmap v1794: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 77 objects/s recovering 2023-10-09T09:40:10.351 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:10.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:10.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:10.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:10.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:10.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:10.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:10.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:10.600 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:10.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:10.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:10.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:10.755 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:10.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:10.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:10.858 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:10.887 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:10.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:10.991 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:11.011 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:11.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:11.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:11.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:11.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:11.140 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:11.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:11.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:11.272 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:11.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:11.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:11.375 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:11.388 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:11 smithi163 bash[75425]: cluster 2023-10-09T09:40:09.953210+0000 mgr.y (mgr.34107) 1022 : cluster 0 pgmap v1795: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 69 objects/s recovering 2023-10-09T09:40:11.389 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:11 smithi163 bash[75425]: audit 2023-10-09T09:40:10.388193+0000 mon.a (mon.0) 1961 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:11.389 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:11 smithi163 bash[75425]: audit 2023-10-09T09:40:10.389229+0000 mon.a (mon.0) 1962 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:11.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:11.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:11.505 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:11.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:11.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:11.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:11 smithi181 bash[59345]: cluster 2023-10-09T09:40:09.953210+0000 mgr.y (mgr.34107) 1022 : cluster 0 pgmap v1795: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 69 objects/s recovering 2023-10-09T09:40:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:11 smithi181 bash[59345]: audit 2023-10-09T09:40:10.388193+0000 mon.a (mon.0) 1961 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:11 smithi181 bash[59345]: audit 2023-10-09T09:40:10.389229+0000 mon.a (mon.0) 1962 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:11.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:11.655 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:11.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:11.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:11.676 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:11 smithi163 bash[78107]: cluster 2023-10-09T09:40:09.953210+0000 mgr.y (mgr.34107) 1022 : cluster 0 pgmap v1795: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 69 objects/s recovering 2023-10-09T09:40:11.676 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:11 smithi163 bash[78107]: audit 2023-10-09T09:40:10.388193+0000 mon.a (mon.0) 1961 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:11.676 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:11 smithi163 bash[78107]: audit 2023-10-09T09:40:10.389229+0000 mon.a (mon.0) 1962 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:11.759 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:11.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:11.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:11.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:11.884 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:11.910 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:11.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:11.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:12.014 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:12.038 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:12.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:12.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:12.141 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:12.170 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:12.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:12.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:12.272 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:12.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:12.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:12.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:12.402 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:12.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:12.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:12.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:12.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:12.554 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:12.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:12.657 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:12.687 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:12.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:12.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:12.816 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:12.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:12.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:12.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:12.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:12.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:13.043 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:13.065 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:13.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:13.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:13.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:13.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:13.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:13.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:13.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:13.328 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:13.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:13.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:13.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:13.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:13.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:13.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:13.566 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:40:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:40:13] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.33.4" 2023-10-09T09:40:13.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:13 smithi163 bash[78107]: cluster 2023-10-09T09:40:11.953923+0000 mgr.y (mgr.34107) 1023 : cluster 0 pgmap v1796: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 99 objects/s recovering 2023-10-09T09:40:13.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:13 smithi163 bash[75425]: cluster 2023-10-09T09:40:11.953923+0000 mgr.y (mgr.34107) 1023 : cluster 0 pgmap v1796: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 99 objects/s recovering 2023-10-09T09:40:13.580 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:13.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:13.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:13.707 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:13.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:13.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:13.810 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:13 smithi181 bash[59345]: cluster 2023-10-09T09:40:11.953923+0000 mgr.y (mgr.34107) 1023 : cluster 0 pgmap v1796: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 99 objects/s recovering 2023-10-09T09:40:13.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:13.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:13.931 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:13.959 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:13.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:13.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:14.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:14.082 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:14.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:14.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:14.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:14.207 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:14.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:14.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:14.310 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:14.340 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:14.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:14.444 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:14.465 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:14.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:14.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:14.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:14 smithi181 bash[59345]: cluster 2023-10-09T09:40:14.389276+0000 mon.a (mon.0) 1963 : cluster 3 Health check update: Degraded data redundancy: 7167/72 objects degraded (9954.167%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:40:14.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:14.595 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:14.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:14.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:14.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:14.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:14 smithi163 bash[78107]: cluster 2023-10-09T09:40:14.389276+0000 mon.a (mon.0) 1963 : cluster 3 Health check update: Degraded data redundancy: 7167/72 objects degraded (9954.167%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:40:14.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:14 smithi163 bash[75425]: cluster 2023-10-09T09:40:14.389276+0000 mon.a (mon.0) 1963 : cluster 3 Health check update: Degraded data redundancy: 7167/72 objects degraded (9954.167%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:40:14.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:14.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:14.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:14.825 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:14.850 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:14.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:14.954 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:14.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:14.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:14.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:15.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:15.112 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:15.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:15.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:15.215 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:15.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:15.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:15.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:15.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:15.369 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:15.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:15.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:15.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:15.493 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:15.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:15.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:15.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:15 smithi181 bash[59345]: cluster 2023-10-09T09:40:13.954391+0000 mgr.y (mgr.34107) 1024 : cluster 0 pgmap v1797: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 76 objects/s recovering 2023-10-09T09:40:15.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:15.621 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:15.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:15.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:15.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:15.746 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:15.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:15.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:15.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:15 smithi163 bash[75425]: cluster 2023-10-09T09:40:13.954391+0000 mgr.y (mgr.34107) 1024 : cluster 0 pgmap v1797: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 76 objects/s recovering 2023-10-09T09:40:15.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:15 smithi163 bash[78107]: cluster 2023-10-09T09:40:13.954391+0000 mgr.y (mgr.34107) 1024 : cluster 0 pgmap v1797: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 76 objects/s recovering 2023-10-09T09:40:15.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:15.875 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:15.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:15.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:15.978 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:16.007 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:16.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:16.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:16.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:16.134 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:16.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:16.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:16.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:16.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:16.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:16.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:16.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:16.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:16.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:16.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:16.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:16.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:16.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:16.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:16.629 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:16.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:16.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:16.731 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:16.750 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:16.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:16.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:16.854 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:16.879 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:16.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:16.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:16.982 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:17.004 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:17.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:17.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:17.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:17.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:17.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:17.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:17.240 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:17.268 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:17.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:17.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:17.371 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:17.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:17.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:17.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:17.517 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:17.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:17.620 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:17.641 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:17.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:17.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:17.745 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:17.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:17.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:17.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:17.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:17.898 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:17.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:17.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:18.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:18.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:18.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:18.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:18.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:17 smithi181 bash[59345]: cluster 2023-10-09T09:40:15.955139+0000 mgr.y (mgr.34107) 1025 : cluster 0 pgmap v1798: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 76 objects/s recovering 2023-10-09T09:40:18.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:17 smithi163 bash[78107]: cluster 2023-10-09T09:40:15.955139+0000 mgr.y (mgr.34107) 1025 : cluster 0 pgmap v1798: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 76 objects/s recovering 2023-10-09T09:40:18.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:17 smithi163 bash[75425]: cluster 2023-10-09T09:40:15.955139+0000 mgr.y (mgr.34107) 1025 : cluster 0 pgmap v1798: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 76 objects/s recovering 2023-10-09T09:40:18.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:18.155 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:18.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:18.258 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:18.283 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:18.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:18.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:18.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:18.410 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:18.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:18.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:18.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:18.535 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:18.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:18.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:18.638 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:18.664 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:18.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:18.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:18.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:18.792 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:18.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:18.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:18.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:18.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:18.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:18.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:19.025 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:19.049 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:19.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:19.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:19.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:19.177 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:19.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:19.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:19.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:19.304 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:19.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:19.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:19.408 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:19.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:19.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:19.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:19.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:19.556 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:19.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:19.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:19.659 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:19.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:19.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:19.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:19.789 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:19.811 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:19.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:19.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:19.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:19.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:19.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:19.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:19.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:19 smithi163 bash[78107]: cluster 2023-10-09T09:40:17.955815+0000 mgr.y (mgr.34107) 1026 : cluster 0 pgmap v1799: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 125 objects/s recovering 2023-10-09T09:40:19.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:19 smithi163 bash[75425]: cluster 2023-10-09T09:40:17.955815+0000 mgr.y (mgr.34107) 1026 : cluster 0 pgmap v1799: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 125 objects/s recovering 2023-10-09T09:40:20.043 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:20.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:19 smithi181 bash[59345]: cluster 2023-10-09T09:40:17.955815+0000 mgr.y (mgr.34107) 1026 : cluster 0 pgmap v1799: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 125 objects/s recovering 2023-10-09T09:40:20.070 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:20.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:20.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:20.173 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:20.196 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:20.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:20.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:20.299 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:20.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:20.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:20.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:20.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:20.447 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:20.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:20.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:20.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:21.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:20 smithi181 bash[59345]: cluster 2023-10-09T09:40:20.731696+0000 mon.a (mon.0) 1964 : cluster 3 Health check update: Degraded data redundancy: 7169/72 objects degraded (9956.944%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:40:21.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:20 smithi163 bash[75425]: cluster 2023-10-09T09:40:20.731696+0000 mon.a (mon.0) 1964 : cluster 3 Health check update: Degraded data redundancy: 7169/72 objects degraded (9956.944%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:40:21.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:20 smithi163 bash[78107]: cluster 2023-10-09T09:40:20.731696+0000 mon.a (mon.0) 1964 : cluster 3 Health check update: Degraded data redundancy: 7169/72 objects degraded (9956.944%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:40:21.458 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:21.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:21.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:21.714 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:21.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:21.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:21.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:21.846 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:21.909 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:40:21.929 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:40:21.959 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:40:21.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:21 smithi163 bash[78107]: cluster 2023-10-09T09:40:19.956567+0000 mgr.y (mgr.34107) 1027 : cluster 0 pgmap v1800: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 88 objects/s recovering 2023-10-09T09:40:21.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:21 smithi163 bash[75425]: cluster 2023-10-09T09:40:19.956567+0000 mgr.y (mgr.34107) 1027 : cluster 0 pgmap v1800: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 88 objects/s recovering 2023-10-09T09:40:21.981 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:40:22.010 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:40:22.030 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:40:22.052 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:40:22.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:21 smithi181 bash[59345]: cluster 2023-10-09T09:40:19.956567+0000 mgr.y (mgr.34107) 1027 : cluster 0 pgmap v1800: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 7169/72 objects degraded (9956.944%); 0 B/s, 88 objects/s recovering 2023-10-09T09:40:22.075 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:40:22.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:40:22.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:40:22.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:40:22.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:40:22.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:40:22.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:40:22.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:40:22.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:40:22.773 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:40:22.787 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_3' removed 2023-10-09T09:40:23.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:22 smithi181 bash[59345]: audit 2023-10-09T09:40:21.884057+0000 mon.a (mon.0) 1965 : audit 0 from='client.? 172.21.15.163:0/1702279672' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:40:23.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:22 smithi181 bash[59345]: audit 2023-10-09T09:40:22.481248+0000 mon.a (mon.0) 1966 : audit 1 from='client.? 172.21.15.163:0/1749662867' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:40:23.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:22 smithi181 bash[59345]: audit 2023-10-09T09:40:22.492712+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.163:0/1746594936' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:40:23.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:22 smithi181 bash[59345]: audit 2023-10-09T09:40:22.493266+0000 mon.a (mon.0) 1967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:40:23.188 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:23.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:23.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:23.291 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:23.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:23.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:23.328 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:22 smithi163 bash[75425]: audit 2023-10-09T09:40:21.884057+0000 mon.a (mon.0) 1965 : audit 0 from='client.? 172.21.15.163:0/1702279672' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:40:23.328 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:22 smithi163 bash[75425]: audit 2023-10-09T09:40:22.481248+0000 mon.a (mon.0) 1966 : audit 1 from='client.? 172.21.15.163:0/1749662867' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:40:23.328 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:22 smithi163 bash[75425]: audit 2023-10-09T09:40:22.492712+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.163:0/1746594936' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:40:23.328 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:22 smithi163 bash[75425]: audit 2023-10-09T09:40:22.493266+0000 mon.a (mon.0) 1967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:40:23.328 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:22 smithi163 bash[78107]: audit 2023-10-09T09:40:21.884057+0000 mon.a (mon.0) 1965 : audit 0 from='client.? 172.21.15.163:0/1702279672' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:40:23.329 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:22 smithi163 bash[78107]: audit 2023-10-09T09:40:22.481248+0000 mon.a (mon.0) 1966 : audit 1 from='client.? 172.21.15.163:0/1749662867' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:40:23.329 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:22 smithi163 bash[78107]: audit 2023-10-09T09:40:22.492712+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.163:0/1746594936' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:40:23.329 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:22 smithi163 bash[78107]: audit 2023-10-09T09:40:22.493266+0000 mon.a (mon.0) 1967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:40:23.420 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:23.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:23.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:23.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:23.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:23.706 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:40:23] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.33.4" 2023-10-09T09:40:23.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[78107]: cluster 2023-10-09T09:40:21.957389+0000 mgr.y (mgr.34107) 1028 : cluster 0 pgmap v1801: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 153 objects/s recovering 2023-10-09T09:40:23.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[78107]: cluster 2023-10-09T09:40:22.747425+0000 mon.a (mon.0) 1968 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:40:23.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[78107]: cluster 2023-10-09T09:40:22.758879+0000 osd.5 (osd.5) 743 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:23.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[78107]: audit 2023-10-09T09:40:22.764308+0000 mon.a (mon.0) 1969 : audit 1 from='client.? 172.21.15.163:0/1749662867' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:40:23.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[78107]: audit 2023-10-09T09:40:22.764468+0000 mon.a (mon.0) 1970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:40:23.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[78107]: cluster 2023-10-09T09:40:22.764553+0000 mon.a (mon.0) 1971 : cluster 0 osdmap e1175: 8 total, 8 up, 7 in 2023-10-09T09:40:23.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[78107]: cluster 2023-10-09T09:40:23.033893+0000 osd.4 (osd.4) 602 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[75425]: cluster 2023-10-09T09:40:21.957389+0000 mgr.y (mgr.34107) 1028 : cluster 0 pgmap v1801: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 153 objects/s recovering 2023-10-09T09:40:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[75425]: cluster 2023-10-09T09:40:22.747425+0000 mon.a (mon.0) 1968 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:40:23.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[75425]: cluster 2023-10-09T09:40:22.758879+0000 osd.5 (osd.5) 743 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:23.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[75425]: audit 2023-10-09T09:40:22.764308+0000 mon.a (mon.0) 1969 : audit 1 from='client.? 172.21.15.163:0/1749662867' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:40:23.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[75425]: audit 2023-10-09T09:40:22.764468+0000 mon.a (mon.0) 1970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:40:23.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[75425]: cluster 2023-10-09T09:40:22.764553+0000 mon.a (mon.0) 1971 : cluster 0 osdmap e1175: 8 total, 8 up, 7 in 2023-10-09T09:40:23.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:23 smithi163 bash[75425]: cluster 2023-10-09T09:40:23.033893+0000 osd.4 (osd.4) 602 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:23.989 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:23.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:23.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:23 smithi181 bash[59345]: cluster 2023-10-09T09:40:21.957389+0000 mgr.y (mgr.34107) 1028 : cluster 0 pgmap v1801: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 7167/72 objects degraded (9954.167%); 0 B/s, 153 objects/s recovering 2023-10-09T09:40:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:23 smithi181 bash[59345]: cluster 2023-10-09T09:40:22.747425+0000 mon.a (mon.0) 1968 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:40:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:23 smithi181 bash[59345]: cluster 2023-10-09T09:40:22.758879+0000 osd.5 (osd.5) 743 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:23 smithi181 bash[59345]: audit 2023-10-09T09:40:22.764308+0000 mon.a (mon.0) 1969 : audit 1 from='client.? 172.21.15.163:0/1749662867' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:40:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:23 smithi181 bash[59345]: audit 2023-10-09T09:40:22.764468+0000 mon.a (mon.0) 1970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:40:24.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:23 smithi181 bash[59345]: cluster 2023-10-09T09:40:22.764553+0000 mon.a (mon.0) 1971 : cluster 0 osdmap e1175: 8 total, 8 up, 7 in 2023-10-09T09:40:24.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:23 smithi181 bash[59345]: cluster 2023-10-09T09:40:23.033893+0000 osd.4 (osd.4) 602 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:24.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:24.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:24.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:24.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:24.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:24.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:24.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:24.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:24.329 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:40:24.330 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:40:24.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:24.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:24.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:24.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:24.482 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:24.509 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:24.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:24.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:24.613 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:24.636 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:24.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:24.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:24.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:24.778 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.5. 2023-10-09T09:40:24.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:24.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:24.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:24.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:24 smithi163 bash[78107]: cluster 2023-10-09T09:40:23.034538+0000 osd.6 (osd.6) 459 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:24.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:24 smithi163 bash[78107]: cluster 2023-10-09T09:40:23.034558+0000 osd.7 (osd.7) 565 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:24.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:24 smithi163 bash[78107]: audit 2023-10-09T09:40:24.751136+0000 mon.a (mon.0) 1972 : audit 0 from='client.? 172.21.15.163:0/2642852738' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T09:40:24.857 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:24 smithi163 bash[75425]: debug 2023-10-09T09:40:24.747+0000 7f22f321a700 -1 mon.a@0(leader).osd e1175 definitely_dead 0 2023-10-09T09:40:24.858 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:24 smithi163 bash[75425]: cluster 2023-10-09T09:40:23.034538+0000 osd.6 (osd.6) 459 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:24.858 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:24 smithi163 bash[75425]: cluster 2023-10-09T09:40:23.034558+0000 osd.7 (osd.7) 565 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:24.858 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:24 smithi163 bash[75425]: audit 2023-10-09T09:40:24.751136+0000 mon.a (mon.0) 1972 : audit 0 from='client.? 172.21.15.163:0/2642852738' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T09:40:24.888 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:24.919 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:24.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:24.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:25.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:25.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:25.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:25.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:25.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:24 smithi181 bash[59345]: cluster 2023-10-09T09:40:23.034538+0000 osd.6 (osd.6) 459 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:25.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:24 smithi181 bash[59345]: cluster 2023-10-09T09:40:23.034558+0000 osd.7 (osd.7) 565 : cluster 3 failed to encode map e1175 with expected crc 2023-10-09T09:40:25.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:24 smithi181 bash[59345]: audit 2023-10-09T09:40:24.751136+0000 mon.a (mon.0) 1972 : audit 0 from='client.? 172.21.15.163:0/2642852738' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T09:40:25.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:25.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:25.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:25.265 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:25.297 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:25.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:25.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:25.401 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:25.420 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:25.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:25.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:25.524 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:25.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:25.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:25.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:25.687 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:25.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:25.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:25.790 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:25.800 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: cluster 2023-10-09T09:40:23.957780+0000 mgr.y (mgr.34107) 1029 : cluster 0 pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:40:25.800 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: cluster 2023-10-09T09:40:24.758170+0000 mon.a (mon.0) 1973 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7167/72 objects degraded (9954.167%), 3 pgs degraded, 1 pg undersized) 2023-10-09T09:40:25.800 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: cluster 2023-10-09T09:40:24.766391+0000 osd.5 (osd.5) 744 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.800 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: cluster 2023-10-09T09:40:24.769189+0000 osd.5 (osd.5) 745 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.800 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: cluster 2023-10-09T09:40:24.769540+0000 osd.7 (osd.7) 566 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.800 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: audit 2023-10-09T09:40:24.769592+0000 mon.a (mon.0) 1974 : audit 1 from='client.? 172.21.15.163:0/2642852738' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T09:40:25.800 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: cluster 2023-10-09T09:40:24.769652+0000 mon.a (mon.0) 1975 : cluster 0 osdmap e1176: 8 total, 8 up, 8 in 2023-10-09T09:40:25.800 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: cluster 2023-10-09T09:40:24.771847+0000 osd.4 (osd.4) 603 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.801 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: cluster 2023-10-09T09:40:24.772070+0000 osd.7 (osd.7) 567 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.801 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: cluster 2023-10-09T09:40:24.772700+0000 osd.6 (osd.6) 460 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.801 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: cluster 2023-10-09T09:40:24.774586+0000 osd.6 (osd.6) 461 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.801 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: audit 2023-10-09T09:40:25.384600+0000 mon.a (mon.0) 1976 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-09T09:40:25.801 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: audit 2023-10-09T09:40:25.384958+0000 mon.a (mon.0) 1977 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T09:40:25.801 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: audit 2023-10-09T09:40:25.385305+0000 mon.a (mon.0) 1978 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:40:25.801 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: audit 2023-10-09T09:40:25.388295+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:25.801 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:25 smithi181 bash[59345]: audit 2023-10-09T09:40:25.389367+0000 mon.a (mon.0) 1980 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:25.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:25.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:25.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:25.941 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:25.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:25.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: cluster 2023-10-09T09:40:23.957780+0000 mgr.y (mgr.34107) 1029 : cluster 0 pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:40:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: cluster 2023-10-09T09:40:24.758170+0000 mon.a (mon.0) 1973 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7167/72 objects degraded (9954.167%), 3 pgs degraded, 1 pg undersized) 2023-10-09T09:40:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: cluster 2023-10-09T09:40:24.766391+0000 osd.5 (osd.5) 744 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: cluster 2023-10-09T09:40:24.769189+0000 osd.5 (osd.5) 745 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: cluster 2023-10-09T09:40:24.769540+0000 osd.7 (osd.7) 566 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: audit 2023-10-09T09:40:24.769592+0000 mon.a (mon.0) 1974 : audit 1 from='client.? 172.21.15.163:0/2642852738' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T09:40:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: cluster 2023-10-09T09:40:24.769652+0000 mon.a (mon.0) 1975 : cluster 0 osdmap e1176: 8 total, 8 up, 8 in 2023-10-09T09:40:25.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: cluster 2023-10-09T09:40:24.771847+0000 osd.4 (osd.4) 603 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: cluster 2023-10-09T09:40:24.772070+0000 osd.7 (osd.7) 567 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: cluster 2023-10-09T09:40:24.772700+0000 osd.6 (osd.6) 460 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: cluster 2023-10-09T09:40:24.774586+0000 osd.6 (osd.6) 461 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: audit 2023-10-09T09:40:25.384600+0000 mon.a (mon.0) 1976 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-09T09:40:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: audit 2023-10-09T09:40:25.384958+0000 mon.a (mon.0) 1977 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T09:40:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: audit 2023-10-09T09:40:25.385305+0000 mon.a (mon.0) 1978 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:40:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: audit 2023-10-09T09:40:25.388295+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:25.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[75425]: audit 2023-10-09T09:40:25.389367+0000 mon.a (mon.0) 1980 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:25.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: cluster 2023-10-09T09:40:23.957780+0000 mgr.y (mgr.34107) 1029 : cluster 0 pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:40:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: cluster 2023-10-09T09:40:24.758170+0000 mon.a (mon.0) 1973 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7167/72 objects degraded (9954.167%), 3 pgs degraded, 1 pg undersized) 2023-10-09T09:40:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: cluster 2023-10-09T09:40:24.766391+0000 osd.5 (osd.5) 744 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: cluster 2023-10-09T09:40:24.769189+0000 osd.5 (osd.5) 745 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: cluster 2023-10-09T09:40:24.769540+0000 osd.7 (osd.7) 566 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: audit 2023-10-09T09:40:24.769592+0000 mon.a (mon.0) 1974 : audit 1 from='client.? 172.21.15.163:0/2642852738' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T09:40:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: cluster 2023-10-09T09:40:24.769652+0000 mon.a (mon.0) 1975 : cluster 0 osdmap e1176: 8 total, 8 up, 8 in 2023-10-09T09:40:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: cluster 2023-10-09T09:40:24.771847+0000 osd.4 (osd.4) 603 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: cluster 2023-10-09T09:40:24.772070+0000 osd.7 (osd.7) 567 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: cluster 2023-10-09T09:40:24.772700+0000 osd.6 (osd.6) 460 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: cluster 2023-10-09T09:40:24.774586+0000 osd.6 (osd.6) 461 : cluster 3 failed to encode map e1176 with expected crc 2023-10-09T09:40:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: audit 2023-10-09T09:40:25.384600+0000 mon.a (mon.0) 1976 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-09T09:40:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: audit 2023-10-09T09:40:25.384958+0000 mon.a (mon.0) 1977 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T09:40:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: audit 2023-10-09T09:40:25.385305+0000 mon.a (mon.0) 1978 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T09:40:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: audit 2023-10-09T09:40:25.388295+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:25.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:25 smithi163 bash[78107]: audit 2023-10-09T09:40:25.389367+0000 mon.a (mon.0) 1980 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:26.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:40:26.044 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:26.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:26.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:26.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:26.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:26.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:26.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:26.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:26.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:26.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:26.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:26.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:26.474 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:26.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:26.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:26.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:26.608 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:26.613 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:40:26.613 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:40:26.614 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T09:40:26.614 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T09:40:26.632 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:26.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:26.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:26.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:26.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:26.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:26.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:26.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:26.873 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: cluster 2023-10-09T09:40:25.776729+0000 osd.5 (osd.5) 746 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: audit 2023-10-09T09:40:25.778841+0000 mon.a (mon.0) 1981 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-09T09:40:26.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: cluster 2023-10-09T09:40:25.778879+0000 osd.5 (osd.5) 747 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: audit 2023-10-09T09:40:25.778944+0000 mon.a (mon.0) 1982 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T09:40:26.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: audit 2023-10-09T09:40:25.778995+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:40:26.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: cluster 2023-10-09T09:40:25.779066+0000 mon.a (mon.0) 1984 : cluster 0 osdmap e1177: 8 total, 8 up, 8 in 2023-10-09T09:40:26.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: cluster 2023-10-09T09:40:25.779377+0000 osd.7 (osd.7) 568 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: cluster 2023-10-09T09:40:25.781942+0000 osd.7 (osd.7) 569 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: cluster 2023-10-09T09:40:25.782013+0000 osd.4 (osd.4) 604 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: cluster 2023-10-09T09:40:25.782013+0000 osd.6 (osd.6) 462 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: cluster 2023-10-09T09:40:25.784111+0000 osd.6 (osd.6) 463 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[75425]: cluster 2023-10-09T09:40:25.790612+0000 osd.4 (osd.4) 605 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: cluster 2023-10-09T09:40:25.776729+0000 osd.5 (osd.5) 746 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: audit 2023-10-09T09:40:25.778841+0000 mon.a (mon.0) 1981 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-09T09:40:26.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: cluster 2023-10-09T09:40:25.778879+0000 osd.5 (osd.5) 747 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: audit 2023-10-09T09:40:25.778944+0000 mon.a (mon.0) 1982 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T09:40:26.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: audit 2023-10-09T09:40:25.778995+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:40:26.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: cluster 2023-10-09T09:40:25.779066+0000 mon.a (mon.0) 1984 : cluster 0 osdmap e1177: 8 total, 8 up, 8 in 2023-10-09T09:40:26.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: cluster 2023-10-09T09:40:25.779377+0000 osd.7 (osd.7) 568 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: cluster 2023-10-09T09:40:25.781942+0000 osd.7 (osd.7) 569 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: cluster 2023-10-09T09:40:25.782013+0000 osd.4 (osd.4) 604 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: cluster 2023-10-09T09:40:25.782013+0000 osd.6 (osd.6) 462 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: cluster 2023-10-09T09:40:25.784111+0000 osd.6 (osd.6) 463 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:26 smithi163 bash[78107]: cluster 2023-10-09T09:40:25.790612+0000 osd.4 (osd.4) 605 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:26.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:26.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:26.989 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:27.022 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:27.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:27.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:27.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: cluster 2023-10-09T09:40:25.776729+0000 osd.5 (osd.5) 746 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: audit 2023-10-09T09:40:25.778841+0000 mon.a (mon.0) 1981 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-09T09:40:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: cluster 2023-10-09T09:40:25.778879+0000 osd.5 (osd.5) 747 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: audit 2023-10-09T09:40:25.778944+0000 mon.a (mon.0) 1982 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T09:40:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: audit 2023-10-09T09:40:25.778995+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T09:40:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: cluster 2023-10-09T09:40:25.779066+0000 mon.a (mon.0) 1984 : cluster 0 osdmap e1177: 8 total, 8 up, 8 in 2023-10-09T09:40:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: cluster 2023-10-09T09:40:25.779377+0000 osd.7 (osd.7) 568 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: cluster 2023-10-09T09:40:25.781942+0000 osd.7 (osd.7) 569 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: cluster 2023-10-09T09:40:25.782013+0000 osd.4 (osd.4) 604 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: cluster 2023-10-09T09:40:25.782013+0000 osd.6 (osd.6) 462 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:27.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: cluster 2023-10-09T09:40:25.784111+0000 osd.6 (osd.6) 463 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:27.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:26 smithi181 bash[59345]: cluster 2023-10-09T09:40:25.790612+0000 osd.4 (osd.4) 605 : cluster 3 failed to encode map e1177 with expected crc 2023-10-09T09:40:27.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:27.162 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:27.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:27.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:27.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:27.299 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:27.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:27.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:27.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:27.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:27.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:27.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:27.532 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:27.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:27.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:27.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:27.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:27.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:27.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:27.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:27.805 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:27 smithi181 bash[59345]: cluster 2023-10-09T09:40:25.958163+0000 mgr.y (mgr.34107) 1030 : cluster 0 pgmap v1806: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:40:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:27 smithi181 bash[59345]: cluster 2023-10-09T09:40:26.784130+0000 osd.5 (osd.5) 748 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:27 smithi181 bash[59345]: cluster 2023-10-09T09:40:26.786991+0000 mon.a (mon.0) 1985 : cluster 0 osdmap e1178: 8 total, 8 up, 8 in 2023-10-09T09:40:27.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:27 smithi181 bash[59345]: cluster 2023-10-09T09:40:26.787406+0000 osd.5 (osd.5) 749 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:27.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:27.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:27.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:27.854 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T09:40:27.870 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-09T09:40:27.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:27.948 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:27 smithi163 bash[78107]: cluster 2023-10-09T09:40:25.958163+0000 mgr.y (mgr.34107) 1030 : cluster 0 pgmap v1806: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:40:27.948 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:27 smithi163 bash[78107]: cluster 2023-10-09T09:40:26.784130+0000 osd.5 (osd.5) 748 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:27.948 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:27 smithi163 bash[78107]: cluster 2023-10-09T09:40:26.786991+0000 mon.a (mon.0) 1985 : cluster 0 osdmap e1178: 8 total, 8 up, 8 in 2023-10-09T09:40:27.948 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:27 smithi163 bash[78107]: cluster 2023-10-09T09:40:26.787406+0000 osd.5 (osd.5) 749 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:27.948 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:27 smithi163 bash[75425]: cluster 2023-10-09T09:40:25.958163+0000 mgr.y (mgr.34107) 1030 : cluster 0 pgmap v1806: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:40:27.948 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:27 smithi163 bash[75425]: cluster 2023-10-09T09:40:26.784130+0000 osd.5 (osd.5) 748 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:27.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:27 smithi163 bash[75425]: cluster 2023-10-09T09:40:26.786991+0000 mon.a (mon.0) 1985 : cluster 0 osdmap e1178: 8 total, 8 up, 8 in 2023-10-09T09:40:27.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:27 smithi163 bash[75425]: cluster 2023-10-09T09:40:26.787406+0000 osd.5 (osd.5) 749 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:27.968 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:27.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:27.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:28.072 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:28.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:28.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:28.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:28.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:40:28.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:28.233 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:28.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:28.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:28.337 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:28.369 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:28.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:28.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:28.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:28.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:28.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:28.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:28.606 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:28.642 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:28.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:28.746 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:28.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:28.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:28.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:28.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:28.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:28 smithi163 bash[78107]: cluster 2023-10-09T09:40:27.018069+0000 osd.6 (osd.6) 464 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:28.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:28 smithi163 bash[78107]: cluster 2023-10-09T09:40:27.021381+0000 osd.6 (osd.6) 465 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:28.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:28 smithi163 bash[75425]: cluster 2023-10-09T09:40:27.018069+0000 osd.6 (osd.6) 464 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:28.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:28 smithi163 bash[75425]: cluster 2023-10-09T09:40:27.021381+0000 osd.6 (osd.6) 465 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:28.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:28.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:28.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:29.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:29.047 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:29.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:29.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:29.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:28 smithi181 bash[59345]: cluster 2023-10-09T09:40:27.018069+0000 osd.6 (osd.6) 464 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:29.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:28 smithi181 bash[59345]: cluster 2023-10-09T09:40:27.021381+0000 osd.6 (osd.6) 465 : cluster 3 failed to encode map e1178 with expected crc 2023-10-09T09:40:29.150 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:29.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:29.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:29.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:29.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:29.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:29.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:29.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:29.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:29.456 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:29.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:29.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:29.560 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:29.591 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:29.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:29.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:29.695 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:29.727 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:29.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:29.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:29.863 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:29.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:29.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:29.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:29.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:29 smithi181 bash[59345]: cluster 2023-10-09T09:40:27.958892+0000 mgr.y (mgr.34107) 1031 : cluster 0 pgmap v1808: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 120 B/s, 1 objects/s recovering 2023-10-09T09:40:29.999 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:30.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:30.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:30.102 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:30.137 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:30.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:30.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:29 smithi163 bash[75425]: cluster 2023-10-09T09:40:27.958892+0000 mgr.y (mgr.34107) 1031 : cluster 0 pgmap v1808: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 120 B/s, 1 objects/s recovering 2023-10-09T09:40:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:29 smithi163 bash[78107]: cluster 2023-10-09T09:40:27.958892+0000 mgr.y (mgr.34107) 1031 : cluster 0 pgmap v1808: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 120 B/s, 1 objects/s recovering 2023-10-09T09:40:30.239 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:30.272 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:30.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:30.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:30.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:30.410 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:30.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:30.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:30.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:30.546 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:30.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:30.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:30.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:30.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:30.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:30.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:30.779 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:30.780 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:30.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:30.815 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:30.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:30.920 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:30.950 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:30.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:30.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:31.053 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:31.084 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:31.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:31.187 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:31.218 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:31.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:31.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:31.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:31.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:31.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:31.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:31.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:31.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:31.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:31.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:31.616 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:31.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:31.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:31.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:31.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:31.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:31.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:31.855 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:31.890 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:31.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:31.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:31.994 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:32.021 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:32.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:32.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:31 smithi181 bash[59345]: cluster 2023-10-09T09:40:29.959607+0000 mgr.y (mgr.34107) 1032 : cluster 0 pgmap v1809: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 103 B/s, 1 objects/s recovering 2023-10-09T09:40:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:31 smithi181 bash[59345]: audit 2023-10-09T09:40:30.946584+0000 mon.a (mon.0) 1986 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:32.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:32.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:32.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:32.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:32.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:31 smithi163 bash[78107]: cluster 2023-10-09T09:40:29.959607+0000 mgr.y (mgr.34107) 1032 : cluster 0 pgmap v1809: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 103 B/s, 1 objects/s recovering 2023-10-09T09:40:32.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:31 smithi163 bash[78107]: audit 2023-10-09T09:40:30.946584+0000 mon.a (mon.0) 1986 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:31 smithi163 bash[75425]: cluster 2023-10-09T09:40:29.959607+0000 mgr.y (mgr.34107) 1032 : cluster 0 pgmap v1809: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 103 B/s, 1 objects/s recovering 2023-10-09T09:40:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:31 smithi163 bash[75425]: audit 2023-10-09T09:40:30.946584+0000 mon.a (mon.0) 1986 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:32.260 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:32.295 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:32.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:32.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:32.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:32.422 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:32.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:32.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:32.525 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:32.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:32.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:32.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:32.654 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:32.677 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:40:32.683 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:32.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:32.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:32.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:32.810 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:32.813 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:32.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:32.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:32.870 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 6, 7, 2, 5] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:40:32.871 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:40:32.871 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 3, 4, 6, 7, 2, 5] 2023-10-09T09:40:32.871 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T09:40:32.916 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:32.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:32.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:32.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:33.043 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:33.073 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:33.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:33.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:33.177 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:33.209 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:33.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:33.313 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:33.343 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:33.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:33.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:33.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:33.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:33.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:33.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:33.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:33.594 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:33.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:33.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:33.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:40:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:40:33] "GET /metrics HTTP/1.1" 200 33985 "" "Prometheus/2.33.4" 2023-10-09T09:40:33.727 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:33.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:33.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:33.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:33.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:33.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:33.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:33.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:33.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:33.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:33.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:34.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:33 smithi181 bash[59345]: cluster 2023-10-09T09:40:31.960815+0000 mgr.y (mgr.34107) 1033 : cluster 0 pgmap v1810: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 86 B/s, 1 objects/s recovering 2023-10-09T09:40:34.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:34.096 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:33 smithi163 bash[78107]: cluster 2023-10-09T09:40:31.960815+0000 mgr.y (mgr.34107) 1033 : cluster 0 pgmap v1810: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 86 B/s, 1 objects/s recovering 2023-10-09T09:40:34.096 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:33 smithi163 bash[75425]: cluster 2023-10-09T09:40:31.960815+0000 mgr.y (mgr.34107) 1033 : cluster 0 pgmap v1810: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 86 B/s, 1 objects/s recovering 2023-10-09T09:40:34.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:34.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:34.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:34.213 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:34.238 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:34.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:34.341 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:34.369 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:34.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:34.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:34.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:34.496 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:34.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:34.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:34.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:34.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:34.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:34.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:34.727 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:34.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:34.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:34.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:34.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:34.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:34.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:34.982 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:35.009 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:35.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:35.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:35.114 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:35.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:35.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:35.239 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:35.265 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:35.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:35.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:35.368 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:35.397 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:35.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:35.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:35.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:35.531 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:35.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:35.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:35.666 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:35.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:35.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:35.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:35.799 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:35.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:35.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:35.902 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:35.931 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:35.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:35 smithi181 bash[59345]: cluster 2023-10-09T09:40:33.961616+0000 mgr.y (mgr.34107) 1034 : cluster 0 pgmap v1811: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 76 B/s, 1 objects/s recovering 2023-10-09T09:40:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:35.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:35.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:35 smithi163 bash[75425]: cluster 2023-10-09T09:40:33.961616+0000 mgr.y (mgr.34107) 1034 : cluster 0 pgmap v1811: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 76 B/s, 1 objects/s recovering 2023-10-09T09:40:35.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:35 smithi163 bash[78107]: cluster 2023-10-09T09:40:33.961616+0000 mgr.y (mgr.34107) 1034 : cluster 0 pgmap v1811: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 76 B/s, 1 objects/s recovering 2023-10-09T09:40:36.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:36.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:36.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:36.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:36.160 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:36.189 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:36.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:36.293 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:36.319 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:36.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:36.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:36.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:36.444 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:36.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:36.547 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:36.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:36.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:36.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:36.676 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:36.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:36.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:36.810 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:36.841 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:36.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:36.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:36.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:36.974 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:36.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:36.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:37.032 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:40:37.033 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:40:37.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:37.109 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:37.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:37.212 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:37.234 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:37.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:37.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:37.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:37.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:37.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:37.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:37.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:37.493 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:37.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:37.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:37.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:37.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:37.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:37.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:37.734 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:37.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:37.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:37.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:37.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T09:40:37.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:37.886 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:37.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:37.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:37.989 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:38.000 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:37 smithi181 bash[59345]: cluster 2023-10-09T09:40:35.962446+0000 mgr.y (mgr.34107) 1035 : cluster 0 pgmap v1812: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 62 B/s, 0 objects/s recovering 2023-10-09T09:40:38.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:38.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:38.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:38.121 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:38.144 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:38.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:38.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:38.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:37 smithi163 bash[78107]: cluster 2023-10-09T09:40:35.962446+0000 mgr.y (mgr.34107) 1035 : cluster 0 pgmap v1812: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 62 B/s, 0 objects/s recovering 2023-10-09T09:40:38.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:37 smithi163 bash[75425]: cluster 2023-10-09T09:40:35.962446+0000 mgr.y (mgr.34107) 1035 : cluster 0 pgmap v1812: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 62 B/s, 0 objects/s recovering 2023-10-09T09:40:38.247 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:38.272 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:38.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:38.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:38.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:38.409 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:38.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:38.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:38.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:38.544 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:38.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:38.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:38.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:38.679 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:38.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:38.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:38.781 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:38.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:38.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:38.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:38.918 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:38.939 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:38.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:39.042 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:39.073 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:39.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:39.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:39.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:39.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:39.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:39.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:39.256 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:39.307 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:39.337 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:39.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:39.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:39.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:39.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:39.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:39.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:39.577 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:39.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:39.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:39.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:39.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:39.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:39.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:39.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:39.867 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:39.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:39.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:39.970 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:40.006 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:40.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:40.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:40.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:39 smithi181 bash[59345]: cluster 2023-10-09T09:40:37.963500+0000 mgr.y (mgr.34107) 1036 : cluster 0 pgmap v1813: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 49 B/s, 0 objects/s recovering 2023-10-09T09:40:40.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:40.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:40.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:40.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:39 smithi163 bash[75425]: cluster 2023-10-09T09:40:37.963500+0000 mgr.y (mgr.34107) 1036 : cluster 0 pgmap v1813: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 49 B/s, 0 objects/s recovering 2023-10-09T09:40:40.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:39 smithi163 bash[78107]: cluster 2023-10-09T09:40:37.963500+0000 mgr.y (mgr.34107) 1036 : cluster 0 pgmap v1813: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 49 B/s, 0 objects/s recovering 2023-10-09T09:40:40.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:40.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:40.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:40.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:40.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:40.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:40.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:40.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:40.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:40.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:40.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:40.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:40.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:40.670 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:40.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:40.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:40.775 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:40.806 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:40.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:40.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:40.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:40.921 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:40 smithi181 bash[59345]: audit 2023-10-09T09:40:40.384800+0000 mon.a (mon.0) 1987 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:40.939 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:40.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:40.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:41.042 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:41.075 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:41.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:41.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:41.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:41.207 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:41.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:41.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:41.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:40 smithi163 bash[78107]: audit 2023-10-09T09:40:40.384800+0000 mon.a (mon.0) 1987 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:41.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:40 smithi163 bash[75425]: audit 2023-10-09T09:40:40.384800+0000 mon.a (mon.0) 1987 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:41.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:41.335 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:41.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:41.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:41.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:41.465 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:41.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:41.568 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:41.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:41.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:41.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:41.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:41.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:41.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:41.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:41.829 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:41.840 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:41 smithi181 bash[59345]: cluster 2023-10-09T09:40:39.964272+0000 mgr.y (mgr.34107) 1037 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:41.841 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:41 smithi181 bash[59345]: audit 2023-10-09T09:40:40.964396+0000 mon.a (mon.0) 1988 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:41.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:41.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:41.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:41.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:41.989 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:40:41.997 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:41.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:42.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:42.013 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:40:42.034 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:40:42.058 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:40:42.084 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:40:42.100 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:42.113 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:40:42.113 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:41 smithi163 bash[78107]: cluster 2023-10-09T09:40:39.964272+0000 mgr.y (mgr.34107) 1037 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:42.114 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:41 smithi163 bash[78107]: audit 2023-10-09T09:40:40.964396+0000 mon.a (mon.0) 1988 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:42.114 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:41 smithi163 bash[75425]: cluster 2023-10-09T09:40:39.964272+0000 mgr.y (mgr.34107) 1037 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:42.114 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:41 smithi163 bash[75425]: audit 2023-10-09T09:40:40.964396+0000 mon.a (mon.0) 1988 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:42.131 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:42.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:42.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:42.138 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:40:42.165 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:40:42.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:40:42.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:40:42.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:40:42.167 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:40:42.168 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:40:42.168 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:40:42.168 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:40:42.168 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:40:42.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:42.255 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:42.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:42.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:42.358 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:42.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:42.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:42.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:42.495 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:42.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:42.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:42.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:42.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:42.634 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:40:42.649 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:42.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:42.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:42.752 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:42.782 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:42.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:42.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:42.886 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:42.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:42.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:42.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:43.018 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:43.040 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:43.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:43.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:43.143 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:43.160 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:43.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:43.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:43.263 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:43.279 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:43.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:43.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:43.307 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_4' created 2023-10-09T09:40:43.383 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:43.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:43.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:43.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:43.480 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:40:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:40:43] "GET /metrics HTTP/1.1" 200 33975 "" "Prometheus/2.33.4" 2023-10-09T09:40:43.921 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:43 smithi163 bash[78107]: audit 2023-10-09T09:40:41.962964+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.163:0/1743568385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:40:43.922 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:43 smithi163 bash[75425]: audit 2023-10-09T09:40:41.962964+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.163:0/1743568385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:40:43.922 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:43.922 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:43 smithi181 bash[59345]: audit 2023-10-09T09:40:41.962964+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.163:0/1743568385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:40:43.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:43.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:43.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:44.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:44.082 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:44.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:44.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:44.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:44.212 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:44.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:44.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:44.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:44.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:44 smithi181 bash[59345]: cluster 2023-10-09T09:40:41.964782+0000 mgr.y (mgr.34107) 1038 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:44.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:44 smithi181 bash[59345]: audit 2023-10-09T09:40:42.794397+0000 mon.a (mon.0) 1989 : audit 1 from='client.? 172.21.15.163:0/2380306385' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T09:40:44.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:44 smithi181 bash[59345]: cluster 2023-10-09T09:40:43.296134+0000 osd.5 (osd.5) 750 : cluster 3 failed to encode map e1179 with expected crc 2023-10-09T09:40:44.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:44 smithi181 bash[59345]: cluster 2023-10-09T09:40:43.296241+0000 osd.7 (osd.7) 570 : cluster 3 failed to encode map e1179 with expected crc 2023-10-09T09:40:44.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:44 smithi181 bash[59345]: audit 2023-10-09T09:40:43.299297+0000 mon.a (mon.0) 1990 : audit 1 from='client.? 172.21.15.163:0/2380306385' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T09:40:44.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:44 smithi181 bash[59345]: cluster 2023-10-09T09:40:43.299423+0000 mon.a (mon.0) 1991 : cluster 0 osdmap e1179: 8 total, 8 up, 8 in 2023-10-09T09:40:44.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:44 smithi181 bash[59345]: audit 2023-10-09T09:40:43.317984+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.163:0/1947248746' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:40:44.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:44 smithi181 bash[59345]: audit 2023-10-09T09:40:43.318372+0000 mon.a (mon.0) 1992 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:40:44.317 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:40:44.337 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:44.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:44.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:44.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:44.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:44.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:44.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:44.573 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:44.602 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:44.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:44.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:44.706 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:44.735 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:44.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:44.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:44.825 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[75425]: cluster 2023-10-09T09:40:41.964782+0000 mgr.y (mgr.34107) 1038 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:44.825 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[75425]: audit 2023-10-09T09:40:42.794397+0000 mon.a (mon.0) 1989 : audit 1 from='client.? 172.21.15.163:0/2380306385' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T09:40:44.826 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[75425]: cluster 2023-10-09T09:40:43.296134+0000 osd.5 (osd.5) 750 : cluster 3 failed to encode map e1179 with expected crc 2023-10-09T09:40:44.826 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[75425]: cluster 2023-10-09T09:40:43.296241+0000 osd.7 (osd.7) 570 : cluster 3 failed to encode map e1179 with expected crc 2023-10-09T09:40:44.826 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[75425]: audit 2023-10-09T09:40:43.299297+0000 mon.a (mon.0) 1990 : audit 1 from='client.? 172.21.15.163:0/2380306385' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T09:40:44.826 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[75425]: cluster 2023-10-09T09:40:43.299423+0000 mon.a (mon.0) 1991 : cluster 0 osdmap e1179: 8 total, 8 up, 8 in 2023-10-09T09:40:44.826 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[75425]: audit 2023-10-09T09:40:43.317984+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.163:0/1947248746' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:40:44.826 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[75425]: audit 2023-10-09T09:40:43.318372+0000 mon.a (mon.0) 1992 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:40:44.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[78107]: cluster 2023-10-09T09:40:41.964782+0000 mgr.y (mgr.34107) 1038 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:44.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[78107]: audit 2023-10-09T09:40:42.794397+0000 mon.a (mon.0) 1989 : audit 1 from='client.? 172.21.15.163:0/2380306385' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T09:40:44.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[78107]: cluster 2023-10-09T09:40:43.296134+0000 osd.5 (osd.5) 750 : cluster 3 failed to encode map e1179 with expected crc 2023-10-09T09:40:44.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[78107]: cluster 2023-10-09T09:40:43.296241+0000 osd.7 (osd.7) 570 : cluster 3 failed to encode map e1179 with expected crc 2023-10-09T09:40:44.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[78107]: audit 2023-10-09T09:40:43.299297+0000 mon.a (mon.0) 1990 : audit 1 from='client.? 172.21.15.163:0/2380306385' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T09:40:44.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[78107]: cluster 2023-10-09T09:40:43.299423+0000 mon.a (mon.0) 1991 : cluster 0 osdmap e1179: 8 total, 8 up, 8 in 2023-10-09T09:40:44.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[78107]: audit 2023-10-09T09:40:43.317984+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.163:0/1947248746' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:40:44.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:44 smithi163 bash[78107]: audit 2023-10-09T09:40:43.318372+0000 mon.a (mon.0) 1992 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:40:44.838 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:44.863 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:44.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:44.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:44.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:44.992 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:44.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:44.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:45.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:45.119 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:45.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:45.221 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:45.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:45.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:45.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:45.345 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:45.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:45.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:45.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:45.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[75425]: cluster 2023-10-09T09:40:43.965412+0000 mgr.y (mgr.34107) 1039 : cluster 0 pgmap v1817: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:45.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[75425]: cluster 2023-10-09T09:40:44.295608+0000 mon.a (mon.0) 1993 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:40:45.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[75425]: cluster 2023-10-09T09:40:44.303670+0000 osd.5 (osd.5) 751 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[75425]: cluster 2023-10-09T09:40:44.303983+0000 osd.6 (osd.6) 466 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[75425]: audit 2023-10-09T09:40:44.307749+0000 mon.a (mon.0) 1994 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:40:45.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[75425]: cluster 2023-10-09T09:40:44.307823+0000 osd.5 (osd.5) 752 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[75425]: cluster 2023-10-09T09:40:44.307874+0000 mon.a (mon.0) 1995 : cluster 0 osdmap e1180: 8 total, 8 up, 8 in 2023-10-09T09:40:45.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[75425]: cluster 2023-10-09T09:40:44.308139+0000 osd.7 (osd.7) 571 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[75425]: cluster 2023-10-09T09:40:44.308193+0000 osd.4 (osd.4) 606 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[78107]: cluster 2023-10-09T09:40:43.965412+0000 mgr.y (mgr.34107) 1039 : cluster 0 pgmap v1817: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[78107]: cluster 2023-10-09T09:40:44.295608+0000 mon.a (mon.0) 1993 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:40:45.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[78107]: cluster 2023-10-09T09:40:44.303670+0000 osd.5 (osd.5) 751 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[78107]: cluster 2023-10-09T09:40:44.303983+0000 osd.6 (osd.6) 466 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[78107]: audit 2023-10-09T09:40:44.307749+0000 mon.a (mon.0) 1994 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:40:45.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[78107]: cluster 2023-10-09T09:40:44.307823+0000 osd.5 (osd.5) 752 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[78107]: cluster 2023-10-09T09:40:44.307874+0000 mon.a (mon.0) 1995 : cluster 0 osdmap e1180: 8 total, 8 up, 8 in 2023-10-09T09:40:45.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[78107]: cluster 2023-10-09T09:40:44.308139+0000 osd.7 (osd.7) 571 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:45 smithi163 bash[78107]: cluster 2023-10-09T09:40:44.308193+0000 osd.4 (osd.4) 606 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:45.506 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:45.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:45.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:45 smithi181 bash[59345]: cluster 2023-10-09T09:40:43.965412+0000 mgr.y (mgr.34107) 1039 : cluster 0 pgmap v1817: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:45 smithi181 bash[59345]: cluster 2023-10-09T09:40:44.295608+0000 mon.a (mon.0) 1993 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:40:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:45 smithi181 bash[59345]: cluster 2023-10-09T09:40:44.303670+0000 osd.5 (osd.5) 751 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:45 smithi181 bash[59345]: cluster 2023-10-09T09:40:44.303983+0000 osd.6 (osd.6) 466 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:45 smithi181 bash[59345]: audit 2023-10-09T09:40:44.307749+0000 mon.a (mon.0) 1994 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:40:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:45 smithi181 bash[59345]: cluster 2023-10-09T09:40:44.307823+0000 osd.5 (osd.5) 752 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:45 smithi181 bash[59345]: cluster 2023-10-09T09:40:44.307874+0000 mon.a (mon.0) 1995 : cluster 0 osdmap e1180: 8 total, 8 up, 8 in 2023-10-09T09:40:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:45 smithi181 bash[59345]: cluster 2023-10-09T09:40:44.308139+0000 osd.7 (osd.7) 571 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:45 smithi181 bash[59345]: cluster 2023-10-09T09:40:44.308193+0000 osd.4 (osd.4) 606 : cluster 3 failed to encode map e1180 with expected crc 2023-10-09T09:40:45.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:45.639 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:45.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:45.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:45.743 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:45.772 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:45.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:45.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:45.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:45.900 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:45.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:45.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:46.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:46.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:46.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:46.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:46.076 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:40:46.076 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:40:46.076 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:40:46.076 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:40:46.076 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:40:46.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:40:46.130 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:46.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:46.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:46.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:46.264 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:46.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T09:40:46.287 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:46.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:46.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:46.390 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:46.402 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:46 smithi181 bash[59345]: cluster 2023-10-09T09:40:45.305939+0000 osd.5 (osd.5) 753 : cluster 3 failed to encode map e1181 with expected crc 2023-10-09T09:40:46.402 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:46 smithi181 bash[59345]: cluster 2023-10-09T09:40:45.306572+0000 osd.4 (osd.4) 607 : cluster 3 failed to encode map e1181 with expected crc 2023-10-09T09:40:46.402 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:46 smithi181 bash[59345]: cluster 2023-10-09T09:40:45.309189+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e1181: 8 total, 8 up, 8 in 2023-10-09T09:40:46.402 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:46 smithi181 bash[59345]: audit 2023-10-09T09:40:45.907086+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:40:46.418 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:46.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:46.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:46.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:46 smithi163 bash[78107]: cluster 2023-10-09T09:40:45.305939+0000 osd.5 (osd.5) 753 : cluster 3 failed to encode map e1181 with expected crc 2023-10-09T09:40:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:46 smithi163 bash[78107]: cluster 2023-10-09T09:40:45.306572+0000 osd.4 (osd.4) 607 : cluster 3 failed to encode map e1181 with expected crc 2023-10-09T09:40:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:46 smithi163 bash[78107]: cluster 2023-10-09T09:40:45.309189+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e1181: 8 total, 8 up, 8 in 2023-10-09T09:40:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:46 smithi163 bash[78107]: audit 2023-10-09T09:40:45.907086+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:40:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:46 smithi163 bash[75425]: cluster 2023-10-09T09:40:45.305939+0000 osd.5 (osd.5) 753 : cluster 3 failed to encode map e1181 with expected crc 2023-10-09T09:40:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:46 smithi163 bash[75425]: cluster 2023-10-09T09:40:45.306572+0000 osd.4 (osd.4) 607 : cluster 3 failed to encode map e1181 with expected crc 2023-10-09T09:40:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:46 smithi163 bash[75425]: cluster 2023-10-09T09:40:45.309189+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e1181: 8 total, 8 up, 8 in 2023-10-09T09:40:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:46 smithi163 bash[75425]: audit 2023-10-09T09:40:45.907086+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:40:46.522 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:46.547 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:46.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:46.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:46.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:46.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:46.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:46.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:46.783 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:46.805 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:46.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:46.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:46.908 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:46.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:46.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:46.984 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T09:40:47.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:47.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:47.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:47.182 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:47.206 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:47.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:47.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:47.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:47.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:47.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:47.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:47.446 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:47.471 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:47.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:47.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:47.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:47.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:47.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:47.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:47.709 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:47.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:47.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:47.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:47.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:47.875 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:47.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:47.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:47.979 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:48.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:48.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:47 smithi163 bash[78107]: cluster 2023-10-09T09:40:45.966195+0000 mgr.y (mgr.34107) 1040 : cluster 0 pgmap v1820: 89 pgs: 12 unknown, 77 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:48.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:47 smithi163 bash[75425]: cluster 2023-10-09T09:40:45.966195+0000 mgr.y (mgr.34107) 1040 : cluster 0 pgmap v1820: 89 pgs: 12 unknown, 77 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:48.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:48.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:48.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:47 smithi181 bash[59345]: cluster 2023-10-09T09:40:45.966195+0000 mgr.y (mgr.34107) 1040 : cluster 0 pgmap v1820: 89 pgs: 12 unknown, 77 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:48.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:48.140 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:48.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:48.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:48.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:48.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:48.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:48.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:48.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:48.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:48.512 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:48.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:48.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:48.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:48.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:48.661 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:48 smithi163 bash[75425]: debug 2023-10-09T09:40:48.451+0000 7f22f321a700 -1 mon.a@0(leader).osd e1181 definitely_dead 0 2023-10-09T09:40:48.684 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:48.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:48.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:48.784 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.7. 2023-10-09T09:40:48.788 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:48.797 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:48 smithi181 bash[59345]: audit 2023-10-09T09:40:48.457551+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.163:0/1893132547' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T09:40:48.797 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:48 smithi181 bash[59345]: audit 2023-10-09T09:40:48.458071+0000 mon.a (mon.0) 1998 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T09:40:48.797 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:48 smithi181 bash[59345]: cluster 2023-10-09T09:40:48.458228+0000 mon.a (mon.0) 1999 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T09:40:48.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:48.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:48.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:48.950 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:48.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:48.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:48 smithi163 bash[78107]: audit 2023-10-09T09:40:48.457551+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.163:0/1893132547' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T09:40:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:48 smithi163 bash[78107]: audit 2023-10-09T09:40:48.458071+0000 mon.a (mon.0) 1998 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T09:40:48.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:48 smithi163 bash[78107]: cluster 2023-10-09T09:40:48.458228+0000 mon.a (mon.0) 1999 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T09:40:48.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:48 smithi163 bash[75425]: audit 2023-10-09T09:40:48.457551+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.163:0/1893132547' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T09:40:48.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:48 smithi163 bash[75425]: audit 2023-10-09T09:40:48.458071+0000 mon.a (mon.0) 1998 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T09:40:48.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:48 smithi163 bash[75425]: cluster 2023-10-09T09:40:48.458228+0000 mon.a (mon.0) 1999 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T09:40:49.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:49.087 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:49.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:49.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:49.191 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:49.216 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:49.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:49.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:49.348 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:49.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:49.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:49.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:49.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:40:49.487 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:49.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:49.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:49.591 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:49.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:49.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:49.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:49.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:49.747 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:49.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:49.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:49.850 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:49.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:49.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:49.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:49 smithi163 bash[78107]: cluster 2023-10-09T09:40:47.966966+0000 mgr.y (mgr.34107) 1041 : cluster 0 pgmap v1821: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:49 smithi163 bash[78107]: cluster 2023-10-09T09:40:48.761760+0000 osd.6 (osd.6) 467 : cluster 3 failed to encode map e1182 with expected crc 2023-10-09T09:40:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:49 smithi163 bash[78107]: cluster 2023-10-09T09:40:48.762019+0000 osd.5 (osd.5) 754 : cluster 3 failed to encode map e1182 with expected crc 2023-10-09T09:40:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:49 smithi163 bash[78107]: audit 2023-10-09T09:40:48.765835+0000 mon.a (mon.0) 2000 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T09:40:49.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:49 smithi163 bash[78107]: cluster 2023-10-09T09:40:48.765991+0000 mon.a (mon.0) 2001 : cluster 0 osdmap e1182: 8 total, 8 up, 7 in 2023-10-09T09:40:49.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:49 smithi163 bash[75425]: cluster 2023-10-09T09:40:47.966966+0000 mgr.y (mgr.34107) 1041 : cluster 0 pgmap v1821: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:49 smithi163 bash[75425]: cluster 2023-10-09T09:40:48.761760+0000 osd.6 (osd.6) 467 : cluster 3 failed to encode map e1182 with expected crc 2023-10-09T09:40:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:49 smithi163 bash[75425]: cluster 2023-10-09T09:40:48.762019+0000 osd.5 (osd.5) 754 : cluster 3 failed to encode map e1182 with expected crc 2023-10-09T09:40:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:49 smithi163 bash[75425]: audit 2023-10-09T09:40:48.765835+0000 mon.a (mon.0) 2000 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T09:40:49.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:49 smithi163 bash[75425]: cluster 2023-10-09T09:40:48.765991+0000 mon.a (mon.0) 2001 : cluster 0 osdmap e1182: 8 total, 8 up, 7 in 2023-10-09T09:40:49.974 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:50.006 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:50.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:50.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:49 smithi181 bash[59345]: cluster 2023-10-09T09:40:47.966966+0000 mgr.y (mgr.34107) 1041 : cluster 0 pgmap v1821: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T09:40:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:49 smithi181 bash[59345]: cluster 2023-10-09T09:40:48.761760+0000 osd.6 (osd.6) 467 : cluster 3 failed to encode map e1182 with expected crc 2023-10-09T09:40:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:49 smithi181 bash[59345]: cluster 2023-10-09T09:40:48.762019+0000 osd.5 (osd.5) 754 : cluster 3 failed to encode map e1182 with expected crc 2023-10-09T09:40:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:49 smithi181 bash[59345]: audit 2023-10-09T09:40:48.765835+0000 mon.a (mon.0) 2000 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T09:40:50.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:49 smithi181 bash[59345]: cluster 2023-10-09T09:40:48.765991+0000 mon.a (mon.0) 2001 : cluster 0 osdmap e1182: 8 total, 8 up, 7 in 2023-10-09T09:40:50.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:50.140 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:50.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:50.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:50.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:50.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:50.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:50.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:50.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:50.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:50.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:50.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:50.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:50.544 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:50.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:50.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:50.681 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:50.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:50.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:50.780 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:40:50.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:50.813 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:50.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:50.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:50.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:50.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:50.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:50.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[75425]: cluster 2023-10-09T09:40:49.769043+0000 osd.5 (osd.5) 755 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[75425]: cluster 2023-10-09T09:40:49.770800+0000 mon.a (mon.0) 2002 : cluster 0 osdmap e1183: 8 total, 8 up, 7 in 2023-10-09T09:40:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[75425]: cluster 2023-10-09T09:40:49.771821+0000 osd.5 (osd.5) 756 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[75425]: cluster 2023-10-09T09:40:49.772038+0000 osd.4 (osd.4) 608 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[75425]: cluster 2023-10-09T09:40:49.772862+0000 osd.6 (osd.6) 468 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[75425]: cluster 2023-10-09T09:40:49.774635+0000 osd.5 (osd.5) 757 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[75425]: cluster 2023-10-09T09:40:49.774695+0000 osd.6 (osd.6) 469 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[75425]: cluster 2023-10-09T09:40:49.774709+0000 osd.4 (osd.4) 609 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[78107]: cluster 2023-10-09T09:40:49.769043+0000 osd.5 (osd.5) 755 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[78107]: cluster 2023-10-09T09:40:49.770800+0000 mon.a (mon.0) 2002 : cluster 0 osdmap e1183: 8 total, 8 up, 7 in 2023-10-09T09:40:50.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[78107]: cluster 2023-10-09T09:40:49.771821+0000 osd.5 (osd.5) 756 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[78107]: cluster 2023-10-09T09:40:49.772038+0000 osd.4 (osd.4) 608 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[78107]: cluster 2023-10-09T09:40:49.772862+0000 osd.6 (osd.6) 468 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[78107]: cluster 2023-10-09T09:40:49.774635+0000 osd.5 (osd.5) 757 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[78107]: cluster 2023-10-09T09:40:49.774695+0000 osd.6 (osd.6) 469 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:50.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:50 smithi163 bash[78107]: cluster 2023-10-09T09:40:49.774709+0000 osd.4 (osd.4) 609 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:51.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:50 smithi181 bash[59345]: cluster 2023-10-09T09:40:49.769043+0000 osd.5 (osd.5) 755 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:50 smithi181 bash[59345]: cluster 2023-10-09T09:40:49.770800+0000 mon.a (mon.0) 2002 : cluster 0 osdmap e1183: 8 total, 8 up, 7 in 2023-10-09T09:40:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:50 smithi181 bash[59345]: cluster 2023-10-09T09:40:49.771821+0000 osd.5 (osd.5) 756 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:50 smithi181 bash[59345]: cluster 2023-10-09T09:40:49.772038+0000 osd.4 (osd.4) 608 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:50 smithi181 bash[59345]: cluster 2023-10-09T09:40:49.772862+0000 osd.6 (osd.6) 468 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:50 smithi181 bash[59345]: cluster 2023-10-09T09:40:49.774635+0000 osd.5 (osd.5) 757 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:50 smithi181 bash[59345]: cluster 2023-10-09T09:40:49.774695+0000 osd.6 (osd.6) 469 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:51.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:50 smithi181 bash[59345]: cluster 2023-10-09T09:40:49.774709+0000 osd.4 (osd.4) 609 : cluster 3 failed to encode map e1183 with expected crc 2023-10-09T09:40:51.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:51.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:51.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:51.183 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:51.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:51.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:51.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:51.313 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:51.345 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:51.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:51.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:51.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:51.469 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:51.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:51.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:51.574 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:51.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:51.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:51.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:51.709 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:51.733 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:51.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:51.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:51.836 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:51.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:51.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:51.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[78107]: cluster 2023-10-09T09:40:49.967597+0000 mgr.y (mgr.34107) 1042 : cluster 0 pgmap v1824: 89 pgs: 5 peering, 84 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:40:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[78107]: cluster 2023-10-09T09:40:50.769293+0000 mon.a (mon.0) 2003 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:40:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[78107]: cluster 2023-10-09T09:40:50.779369+0000 osd.5 (osd.5) 758 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[78107]: cluster 2023-10-09T09:40:50.781929+0000 mon.a (mon.0) 2004 : cluster 0 osdmap e1184: 8 total, 8 up, 7 in 2023-10-09T09:40:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[78107]: cluster 2023-10-09T09:40:50.782766+0000 osd.5 (osd.5) 759 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[78107]: cluster 2023-10-09T09:40:50.786158+0000 osd.6 (osd.6) 470 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[78107]: cluster 2023-10-09T09:40:50.786376+0000 osd.7 (osd.7) 572 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[78107]: cluster 2023-10-09T09:40:51.440988+0000 osd.4 (osd.4) 610 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[75425]: cluster 2023-10-09T09:40:49.967597+0000 mgr.y (mgr.34107) 1042 : cluster 0 pgmap v1824: 89 pgs: 5 peering, 84 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:40:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[75425]: cluster 2023-10-09T09:40:50.769293+0000 mon.a (mon.0) 2003 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:40:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[75425]: cluster 2023-10-09T09:40:50.779369+0000 osd.5 (osd.5) 758 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[75425]: cluster 2023-10-09T09:40:50.781929+0000 mon.a (mon.0) 2004 : cluster 0 osdmap e1184: 8 total, 8 up, 7 in 2023-10-09T09:40:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[75425]: cluster 2023-10-09T09:40:50.782766+0000 osd.5 (osd.5) 759 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[75425]: cluster 2023-10-09T09:40:50.786158+0000 osd.6 (osd.6) 470 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[75425]: cluster 2023-10-09T09:40:50.786376+0000 osd.7 (osd.7) 572 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:51 smithi163 bash[75425]: cluster 2023-10-09T09:40:51.440988+0000 osd.4 (osd.4) 610 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:51.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:51 smithi181 bash[59345]: cluster 2023-10-09T09:40:49.967597+0000 mgr.y (mgr.34107) 1042 : cluster 0 pgmap v1824: 89 pgs: 5 peering, 84 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:40:51.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:51 smithi181 bash[59345]: cluster 2023-10-09T09:40:50.769293+0000 mon.a (mon.0) 2003 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:40:51.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:51 smithi181 bash[59345]: cluster 2023-10-09T09:40:50.779369+0000 osd.5 (osd.5) 758 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:51 smithi181 bash[59345]: cluster 2023-10-09T09:40:50.781929+0000 mon.a (mon.0) 2004 : cluster 0 osdmap e1184: 8 total, 8 up, 7 in 2023-10-09T09:40:51.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:51 smithi181 bash[59345]: cluster 2023-10-09T09:40:50.782766+0000 osd.5 (osd.5) 759 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:51 smithi181 bash[59345]: cluster 2023-10-09T09:40:50.786158+0000 osd.6 (osd.6) 470 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.990 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:51 smithi181 bash[59345]: cluster 2023-10-09T09:40:50.786376+0000 osd.7 (osd.7) 572 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:51.990 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:51 smithi181 bash[59345]: cluster 2023-10-09T09:40:51.440988+0000 osd.4 (osd.4) 610 : cluster 3 failed to encode map e1184 with expected crc 2023-10-09T09:40:52.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:52.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:52.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:52.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:52.137 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:52.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:52.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:52.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:52.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:52.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:52.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:52.341 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:52.342 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:52.370 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:52.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:52.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:52.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:52.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:52.524 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:52.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:52.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:52.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:52.654 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:52.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:52.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:52.757 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:52.783 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:52.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:52.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:52.887 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:52.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:52.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:52.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[78107]: cluster 2023-10-09T09:40:51.780599+0000 osd.5 (osd.5) 760 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[78107]: cluster 2023-10-09T09:40:51.783749+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e1185: 8 total, 8 up, 7 in 2023-10-09T09:40:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[78107]: cluster 2023-10-09T09:40:51.787111+0000 osd.6 (osd.6) 471 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[78107]: cluster 2023-10-09T09:40:51.787217+0000 osd.7 (osd.7) 573 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[78107]: audit 2023-10-09T09:40:52.396135+0000 mon.a (mon.0) 2006 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[78107]: audit 2023-10-09T09:40:52.403372+0000 mon.a (mon.0) 2007 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[78107]: cluster 2023-10-09T09:40:52.590355+0000 osd.4 (osd.4) 611 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:52.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[75425]: cluster 2023-10-09T09:40:51.780599+0000 osd.5 (osd.5) 760 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:52.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[75425]: cluster 2023-10-09T09:40:51.783749+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e1185: 8 total, 8 up, 7 in 2023-10-09T09:40:52.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[75425]: cluster 2023-10-09T09:40:51.787111+0000 osd.6 (osd.6) 471 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:52.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[75425]: cluster 2023-10-09T09:40:51.787217+0000 osd.7 (osd.7) 573 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:52.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[75425]: audit 2023-10-09T09:40:52.396135+0000 mon.a (mon.0) 2006 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:52.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[75425]: audit 2023-10-09T09:40:52.403372+0000 mon.a (mon.0) 2007 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:52.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:52 smithi163 bash[75425]: cluster 2023-10-09T09:40:52.590355+0000 osd.4 (osd.4) 611 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:53.014 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:53.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:53.044 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:52 smithi181 bash[59345]: cluster 2023-10-09T09:40:51.780599+0000 osd.5 (osd.5) 760 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:53.045 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:52 smithi181 bash[59345]: cluster 2023-10-09T09:40:51.783749+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e1185: 8 total, 8 up, 7 in 2023-10-09T09:40:53.045 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:52 smithi181 bash[59345]: cluster 2023-10-09T09:40:51.787111+0000 osd.6 (osd.6) 471 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:53.045 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:52 smithi181 bash[59345]: cluster 2023-10-09T09:40:51.787217+0000 osd.7 (osd.7) 573 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:53.045 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:52 smithi181 bash[59345]: audit 2023-10-09T09:40:52.396135+0000 mon.a (mon.0) 2006 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:53.045 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:52 smithi181 bash[59345]: audit 2023-10-09T09:40:52.403372+0000 mon.a (mon.0) 2007 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:53.045 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:52 smithi181 bash[59345]: cluster 2023-10-09T09:40:52.590355+0000 osd.4 (osd.4) 611 : cluster 3 failed to encode map e1185 with expected crc 2023-10-09T09:40:53.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:53.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:53.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:53.172 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:53.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:53.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:53.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:53.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:53.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:53.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:53.401 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:53.408 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:40:53] "GET /metrics HTTP/1.1" 200 33975 "" "Prometheus/2.33.4" 2023-10-09T09:40:53.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:53.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:53.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:53.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:53.555 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:53.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:53.660 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:53.695 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:53.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:53.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:53.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:53.819 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:53.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:53.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:53.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:53.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:53 smithi181 bash[59345]: cluster 2023-10-09T09:40:51.968279+0000 mgr.y (mgr.34107) 1043 : cluster 0 pgmap v1827: 89 pgs: 5 peering, 84 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 1.2 KiB/s, 1 objects/s recovering 2023-10-09T09:40:53.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:53 smithi181 bash[59345]: cluster 2023-10-09T09:40:52.788494+0000 osd.5 (osd.5) 761 : cluster 3 failed to encode map e1186 with expected crc 2023-10-09T09:40:53.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:53 smithi181 bash[59345]: cluster 2023-10-09T09:40:52.793447+0000 osd.6 (osd.6) 472 : cluster 3 failed to encode map e1186 with expected crc 2023-10-09T09:40:53.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:53 smithi181 bash[59345]: cluster 2023-10-09T09:40:52.794378+0000 mon.a (mon.0) 2008 : cluster 0 osdmap e1186: 8 total, 8 up, 7 in 2023-10-09T09:40:53.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:53 smithi181 bash[59345]: audit 2023-10-09T09:40:53.463206+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:53.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:53 smithi181 bash[59345]: audit 2023-10-09T09:40:53.470387+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:53.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:53.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:54.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:54.062 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[78107]: cluster 2023-10-09T09:40:51.968279+0000 mgr.y (mgr.34107) 1043 : cluster 0 pgmap v1827: 89 pgs: 5 peering, 84 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 1.2 KiB/s, 1 objects/s recovering 2023-10-09T09:40:54.062 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[78107]: cluster 2023-10-09T09:40:52.788494+0000 osd.5 (osd.5) 761 : cluster 3 failed to encode map e1186 with expected crc 2023-10-09T09:40:54.062 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[78107]: cluster 2023-10-09T09:40:52.793447+0000 osd.6 (osd.6) 472 : cluster 3 failed to encode map e1186 with expected crc 2023-10-09T09:40:54.062 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[78107]: cluster 2023-10-09T09:40:52.794378+0000 mon.a (mon.0) 2008 : cluster 0 osdmap e1186: 8 total, 8 up, 7 in 2023-10-09T09:40:54.062 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[78107]: audit 2023-10-09T09:40:53.463206+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:54.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[78107]: audit 2023-10-09T09:40:53.470387+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:54.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[75425]: cluster 2023-10-09T09:40:51.968279+0000 mgr.y (mgr.34107) 1043 : cluster 0 pgmap v1827: 89 pgs: 5 peering, 84 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 1.2 KiB/s, 1 objects/s recovering 2023-10-09T09:40:54.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[75425]: cluster 2023-10-09T09:40:52.788494+0000 osd.5 (osd.5) 761 : cluster 3 failed to encode map e1186 with expected crc 2023-10-09T09:40:54.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[75425]: cluster 2023-10-09T09:40:52.793447+0000 osd.6 (osd.6) 472 : cluster 3 failed to encode map e1186 with expected crc 2023-10-09T09:40:54.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[75425]: cluster 2023-10-09T09:40:52.794378+0000 mon.a (mon.0) 2008 : cluster 0 osdmap e1186: 8 total, 8 up, 7 in 2023-10-09T09:40:54.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[75425]: audit 2023-10-09T09:40:53.463206+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:54.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:53 smithi163 bash[75425]: audit 2023-10-09T09:40:53.470387+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:54.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:54.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:54.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:54.179 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:54.207 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:54.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:54.311 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:54.339 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:54.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:54.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:54.367 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:54.378 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:40:54.378 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T09:40:54.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T09:40:54.378 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:40:54.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.378 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:43816 8.2 8:4abe0272:::benchmark_data_smithi163_151167_object43815:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1142)", 2023-10-09T09:40:54.379 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:54.460043+0000", 2023-10-09T09:40:54.379 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 299.91082687400001, 2023-10-09T09:40:54.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23700337299999999, 2023-10-09T09:40:54.379 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.379 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.379 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.379 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.380 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.380 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43816 2023-10-09T09:40:54.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.380 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.460043+0000", 2023-10-09T09:40:54.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.460043+0000", 2023-10-09T09:40:54.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:40:54.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.460041+0000", 2023-10-09T09:40:54.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7731000000000002e-05 2023-10-09T09:40:54.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.460078+0000", 2023-10-09T09:40:54.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7487e-05 2023-10-09T09:40:54.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.460096+0000", 2023-10-09T09:40:54.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0790000000000007e-06 2023-10-09T09:40:54.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.460105+0000", 2023-10-09T09:40:54.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011296209999999999 2023-10-09T09:40:54.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.461235+0000", 2023-10-09T09:40:54.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000147856 2023-10-09T09:40:54.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.461382+0000", 2023-10-09T09:40:54.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015757100000000001 2023-10-09T09:40:54.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:40:54.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.461540+0000", 2023-10-09T09:40:54.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004659949 2023-10-09T09:40:54.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.466200+0000", 2023-10-09T09:40:54.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3777e-05 2023-10-09T09:40:54.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.466214+0000", 2023-10-09T09:40:54.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23068002400000001 2023-10-09T09:40:54.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.696894+0000", 2023-10-09T09:40:54.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9190000000000004e-05 2023-10-09T09:40:54.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.696993+0000", 2023-10-09T09:40:54.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3007000000000003e-05 2023-10-09T09:40:54.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697046+0000", 2023-10-09T09:40:54.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23700337299999999 2023-10-09T09:40:54.389 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.389 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.390 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.390 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:43826 8.2 8:462917b7:::benchmark_data_smithi163_151167_object43825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1142)", 2023-10-09T09:40:54.390 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:54.467029+0000", 2023-10-09T09:40:54.390 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 299.90384023199999, 2023-10-09T09:40:54.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.230201985, 2023-10-09T09:40:54.390 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.391 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.391 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.391 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.391 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.391 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43826 2023-10-09T09:40:54.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.391 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.467029+0000", 2023-10-09T09:40:54.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.467029+0000", 2023-10-09T09:40:54.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:40:54.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.467027+0000", 2023-10-09T09:40:54.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1617000000000002e-05 2023-10-09T09:40:54.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.467099+0000", 2023-10-09T09:40:54.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2252999999999999e-05 2023-10-09T09:40:54.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.467111+0000", 2023-10-09T09:40:54.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9e-06 2023-10-09T09:40:54.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.467118+0000", 2023-10-09T09:40:54.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2633000000000003e-05 2023-10-09T09:40:54.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.467161+0000", 2023-10-09T09:40:54.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000146608 2023-10-09T09:40:54.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.467307+0000", 2023-10-09T09:40:54.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014265400000000001 2023-10-09T09:40:54.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:40:54.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.467450+0000", 2023-10-09T09:40:54.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0060713399999999997 2023-10-09T09:40:54.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.473521+0000", 2023-10-09T09:40:54.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003227153 2023-10-09T09:40:54.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.476748+0000", 2023-10-09T09:40:54.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22036256300000001 2023-10-09T09:40:54.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697111+0000", 2023-10-09T09:40:54.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4337e-05 2023-10-09T09:40:54.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697145+0000", 2023-10-09T09:40:54.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5776e-05 2023-10-09T09:40:54.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697231+0000", 2023-10-09T09:40:54.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.230201985 2023-10-09T09:40:54.400 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.400 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.400 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.401 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:43827 8.2 8:40e16023:::benchmark_data_smithi163_151167_object43826:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1142)", 2023-10-09T09:40:54.401 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:54.468699+0000", 2023-10-09T09:40:54.401 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 299.90217018300001, 2023-10-09T09:40:54.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.228807125, 2023-10-09T09:40:54.401 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.401 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.401 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.402 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.402 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.402 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43827 2023-10-09T09:40:54.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.402 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.468699+0000", 2023-10-09T09:40:54.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.468699+0000", 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.468698+0000", 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5813999999999999e-05 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.468743+0000", 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3200000000000001e-05 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.468757+0000", 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2509999999999997e-06 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.468764+0000", 2023-10-09T09:40:54.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00057592600000000004 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.469340+0000", 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000132881 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.469473+0000", 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015637299999999999 2023-10-09T09:40:54.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.469629+0000", 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0071257129999999997 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.476755+0000", 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001637002 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.478392+0000", 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21896495799999999 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697357+0000", 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010520500000000001 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697462+0000", 2023-10-09T09:40:54.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4376000000000003e-05 2023-10-09T09:40:54.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697506+0000", 2023-10-09T09:40:54.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.228807125 2023-10-09T09:40:54.408 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.408 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.408 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.409 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:43828 8.2 8:42c0a02a:::benchmark_data_smithi163_151167_object43827:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1142)", 2023-10-09T09:40:54.409 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:54.470198+0000", 2023-10-09T09:40:54.409 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 299.90067144699998, 2023-10-09T09:40:54.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22745953299999999, 2023-10-09T09:40:54.409 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.409 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.409 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.410 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.410 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.410 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43828 2023-10-09T09:40:54.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.410 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470198+0000", 2023-10-09T09:40:54.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470198+0000", 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999895 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470187+0000", 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0448000000000002e-05 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470238+0000", 2023-10-09T09:40:54.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8987e-05 2023-10-09T09:40:54.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470257+0000", 2023-10-09T09:40:54.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7079999999999997e-06 2023-10-09T09:40:54.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470264+0000", 2023-10-09T09:40:54.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011826 2023-10-09T09:40:54.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.471447+0000", 2023-10-09T09:40:54.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013094300000000001 2023-10-09T09:40:54.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.471578+0000", 2023-10-09T09:40:54.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016092799999999999 2023-10-09T09:40:54.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:40:54.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.471739+0000", 2023-10-09T09:40:54.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050203080000000002 2023-10-09T09:40:54.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.476759+0000", 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017041789999999999 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.478463+0000", 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21909854200000001 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697562+0000", 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4818000000000003e-05 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697617+0000", 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0745999999999999e-05 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697657+0000", 2023-10-09T09:40:54.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22745953299999999 2023-10-09T09:40:54.419 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.419 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.419 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.419 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:43831 8.2 8:41273caa:::benchmark_data_smithi163_151167_object43830:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1142)", 2023-10-09T09:40:54.419 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:54.470454+0000", 2023-10-09T09:40:54.419 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 299.90041568599997, 2023-10-09T09:40:54.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.227411378, 2023-10-09T09:40:54.420 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.420 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.420 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.420 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.420 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.420 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43831 2023-10-09T09:40:54.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.420 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470454+0000", 2023-10-09T09:40:54.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470454+0000", 2023-10-09T09:40:54.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:40:54.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470453+0000", 2023-10-09T09:40:54.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4079999999999999e-05 2023-10-09T09:40:54.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470487+0000", 2023-10-09T09:40:54.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2104e-05 2023-10-09T09:40:54.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470499+0000", 2023-10-09T09:40:54.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9489999999999996e-06 2023-10-09T09:40:54.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.470507+0000", 2023-10-09T09:40:54.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030364720000000001 2023-10-09T09:40:54.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.473543+0000", 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000148644 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.473692+0000", 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016306299999999999 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:40:54.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.473855+0000", 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029089179999999999 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.476764+0000", 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019781989999999999 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.478742+0000", 2023-10-09T09:40:54.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.218979964 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697722+0000", 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8793000000000001e-05 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697821+0000", 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4249000000000003e-05 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:54.697865+0000", 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.227411378 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:83822 8.b 8:d11a68ca:::benchmark_data_smithi163_151167_object83821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:40.138993+0000", 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 254.231876379, 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26774009799999998, 2023-10-09T09:40:54.429 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.430 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.430 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.430 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.430 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.430 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 83822 2023-10-09T09:40:54.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.430 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.138993+0000", 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.138993+0000", 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.138991+0000", 2023-10-09T09:40:54.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5488999999999998e-05 2023-10-09T09:40:54.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.139037+0000", 2023-10-09T09:40:54.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1384e-05 2023-10-09T09:40:54.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.139048+0000", 2023-10-09T09:40:54.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3260000000000001e-06 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.139054+0000", 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6511999999999998e-05 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.139101+0000", 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013518699999999999 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.139236+0000", 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015404399999999999 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.139390+0000", 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066156740000000002 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.146006+0000", 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036642089999999999 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.149670+0000", 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25694951100000002 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.406620+0000", 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4447000000000001e-05 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.406684+0000", 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9175000000000003e-05 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.406733+0000", 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26774009799999998 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:83824 8.b 8:d8bef780:::benchmark_data_smithi163_151167_object83823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:40.143406+0000", 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 254.22746354899999, 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.270288686, 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 83824 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.143406+0000", 2023-10-09T09:40:54.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.143406+0000", 2023-10-09T09:40:54.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:40:54.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.143404+0000", 2023-10-09T09:40:54.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1508000000000003e-05 2023-10-09T09:40:54.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.143445+0000", 2023-10-09T09:40:54.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0628e-05 2023-10-09T09:40:54.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.143466+0000", 2023-10-09T09:40:54.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8040000000000007e-06 2023-10-09T09:40:54.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.143475+0000", 2023-10-09T09:40:54.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025546779999999999 2023-10-09T09:40:54.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.146030+0000", 2023-10-09T09:40:54.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012751700000000001 2023-10-09T09:40:54.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.146157+0000", 2023-10-09T09:40:54.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015793599999999999 2023-10-09T09:40:54.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:40:54.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.146315+0000", 2023-10-09T09:40:54.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034996630000000001 2023-10-09T09:40:54.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.149815+0000", 2023-10-09T09:40:54.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001204852 2023-10-09T09:40:54.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.151019+0000", 2023-10-09T09:40:54.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26238928900000003 2023-10-09T09:40:54.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.413409+0000", 2023-10-09T09:40:54.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022098199999999999 2023-10-09T09:40:54.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.413630+0000", 2023-10-09T09:40:54.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4783000000000006e-05 2023-10-09T09:40:54.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.413695+0000", 2023-10-09T09:40:54.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.270288686 2023-10-09T09:40:54.447 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.448 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.448 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.448 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:83846 8.b 8:d46552c9:::benchmark_data_smithi163_151167_object83845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:40:54.448 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:40.175847+0000", 2023-10-09T09:40:54.448 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 254.195022502, 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23889233900000001, 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 83846 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.175847+0000", 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.175847+0000", 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.175845+0000", 2023-10-09T09:40:54.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9014000000000002e-05 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.175894+0000", 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7699000000000001e-05 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.175912+0000", 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.568e-06 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.175920+0000", 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2613999999999994e-05 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.175983+0000", 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017269400000000001 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.176156+0000", 2023-10-09T09:40:54.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000189707 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.176345+0000", 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002897864 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.179243+0000", 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8780000000000003e-06 2023-10-09T09:40:54.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.179249+0000", 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23538784300000001 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.414637+0000", 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3163000000000005e-05 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.414700+0000", 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9280999999999999e-05 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.414739+0000", 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23889233900000001 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:93654 8.b 8:d380bc3e:::benchmark_data_smithi163_151167_object93653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1152)", 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:55.581570+0000", 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 238.789299548, 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21321504499999999, 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93654 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.581570+0000", 2023-10-09T09:40:54.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.581570+0000", 2023-10-09T09:40:54.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:40:54.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.581568+0000", 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9201e-05 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.581607+0000", 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0098e-05 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.581627+0000", 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9170000000000006e-06 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.581635+0000", 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4131999999999999e-05 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.581730+0000", 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020143900000000001 2023-10-09T09:40:54.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.581931+0000", 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000209574 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.582141+0000", 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002811699 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.584952+0000", 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010958262 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.595910+0000", 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.19876107500000001 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.794672+0000", 2023-10-09T09:40:54.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3744999999999994e-05 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.794735+0000", 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9601999999999997e-05 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.794785+0000", 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21321504499999999 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.462 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:93663 8.b 8:d34883e1:::benchmark_data_smithi163_151167_object93662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1152)", 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:55.587500+0000", 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 238.78336931699999, 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.213104871, 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.463 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93663 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.587500+0000", 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.587500+0000", 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:40:54.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.587498+0000", 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1298000000000003e-05 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.587539+0000", 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4611e-05 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.587554+0000", 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3550000000000001e-06 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.587560+0000", 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1724999999999997e-05 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.587612+0000", 2023-10-09T09:40:54.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0001707 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.587783+0000", 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019578900000000001 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.587979+0000", 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039698579999999997 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.591948+0000", 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003996416 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.595945+0000", 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20454019600000001 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.800485+0000", 2023-10-09T09:40:54.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1371000000000005e-05 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.800556+0000", 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8591000000000003e-05 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.800605+0000", 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.213104871 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.469 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181706 8.4 8:2ecb8db1:::benchmark_data_smithi163_151167_object86473:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1161)", 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.909264+0000", 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 102.461605335, 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5310189379999999, 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.470 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181706 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909264+0000", 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909264+0000", 2023-10-09T09:40:54.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909264+0000", 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0439999999999997e-06 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909268+0000", 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2739999999999996e-06 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.472 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909277+0000", 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4819999999999996e-06 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909281+0000", 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6717e-05 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.473 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909328+0000", 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000100179 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909428+0000", 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012467000000000001 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909553+0000", 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062400060000000002 2023-10-09T09:40:54.474 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.915793+0000", 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003638287 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.919431+0000", 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022810609999999999 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.475 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.942242+0000", 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2697000000000001e-05 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.942254+0000", 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.006853617 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.949108+0000", 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3545e-05 2023-10-09T09:40:54.476 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.949122+0000", 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4350656939999999 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.384187+0000", 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016446599999999999 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.477 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for missing object", 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.384352+0000", 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.055816106999999997 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.440168+0000", 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115098 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.440283+0000", 2023-10-09T09:40:54.478 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5310189379999999 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181726 8.2 8:49011ff1:::benchmark_data_smithi163_151167_object86493:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1162)", 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.961439+0000", 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 102.40943004899999, 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2811808660000001, 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.479 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181726 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961439+0000", 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961439+0000", 2023-10-09T09:40:54.480 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961438+0000", 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5080000000000005e-06 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961446+0000", 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3454e-05 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.481 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961460+0000", 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.241e-06 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961466+0000", 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2775999999999997e-05 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.482 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961529+0000", 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3730000000000001e-05 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961542+0000", 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98586582300000003 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.947408+0000", 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0344999999999999e-05 2023-10-09T09:40:54.483 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.947428+0000", 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2693667559999999 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.216795+0000", 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2419999999999999e-05 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.484 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.216888+0000", 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00064060199999999999 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.217528+0000", 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.024105868999999999 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.241634+0000", 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029292499999999998 2023-10-09T09:40:54.485 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.241927+0000", 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9008000000000003e-05 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.241966+0000", 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00044008900000000001 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.486 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.242406+0000", 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6975000000000002e-05 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.242473+0000", 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014713700000000001 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.242620+0000", 2023-10-09T09:40:54.487 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2811808660000001 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181735 8.2 8:495c0466:::benchmark_data_smithi163_151167_object86502:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1162)", 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.984259+0000", 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 102.38661036400001, 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2625273049999999, 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.488 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181735 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984259+0000", 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.489 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984259+0000", 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984257+0000", 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1745e-05 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984269+0000", 2023-10-09T09:40:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4559e-05 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984293+0000", 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7567e-05 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984311+0000", 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1023000000000001e-05 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984382+0000", 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4260999999999999e-05 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984396+0000", 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96307082399999999 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.947467+0000", 2023-10-09T09:40:54.492 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0461e-05 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.947477+0000", 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2808528299999999 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.228330+0000", 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013772900000000001 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.493 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.228468+0000", 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000164411 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.228632+0000", 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013010261 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.241642+0000", 2023-10-09T09:40:54.494 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00036442499999999998 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.242007+0000", 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000356187 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.242363+0000", 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041758339999999998 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.495 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.246539+0000", 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010223 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.246641+0000", 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014521 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.246786+0000", 2023-10-09T09:40:54.496 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2625273049999999 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181747 8.4 8:2fbe244d:::benchmark_data_smithi163_151167_object86514:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1164)", 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:14.226835+0000", 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 100.144034198, 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23001891499999999, 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.497 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181747 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.226835+0000", 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.226835+0000", 2023-10-09T09:40:54.498 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.226833+0000", 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1009999999999998e-06 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.226842+0000", 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2028000000000001e-05 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.499 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.226854+0000", 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6280000000000002e-06 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.226860+0000", 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010437210000000001 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.227903+0000", 2023-10-09T09:40:54.500 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0823e-05 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.227914+0000", 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15647513199999999 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.384389+0000", 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016607899999999999 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.501 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.384555+0000", 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00070914999999999997 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.385265+0000", 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013953886 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.502 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.399219+0000", 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0065147299999999998 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.405733+0000", 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050149830999999999 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.455883+0000", 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8782000000000001e-05 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.503 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.455942+0000", 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00091227799999999998 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.456854+0000", 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23001891499999999 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.504 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189411 8.4 8:2d2046c1:::benchmark_data_smithi163_151167_object94178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.945426+0000", 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 87.425443620999999, 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29429408800000001, 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189411 2023-10-09T09:40:54.505 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945426+0000", 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945426+0000", 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:40:54.506 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945425+0000", 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5380000000000002e-06 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945430+0000", 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9080000000000007e-06 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.507 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945440+0000", 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3459999999999999e-06 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945446+0000", 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0451000000000001e-05 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945507+0000", 2023-10-09T09:40:54.508 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021344100000000001 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945720+0000", 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022876 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945949+0000", 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034423269999999998 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.509 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.949391+0000", 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015354016999999999 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.964745+0000", 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27384020599999997 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.238586+0000", 2023-10-09T09:40:54.510 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5902000000000005e-05 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.238671+0000", 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010484439999999999 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.239720+0000", 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29429408800000001 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.511 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189414 8.4 8:2f4d7c9e:::benchmark_data_smithi163_151167_object94181:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.951055+0000", 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 87.419814888999994, 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.28962749399999999, 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.512 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189414 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951055+0000", 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951055+0000", 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.513 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951053+0000", 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6830000000000008e-06 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951061+0000", 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4021e-05 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.514 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951075+0000", 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4969999999999998e-06 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951081+0000", 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019693900000000001 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951278+0000", 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000190495 2023-10-09T09:40:54.515 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951469+0000", 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000215503 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951684+0000", 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0084840320000000007 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.516 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.960168+0000", 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0047018959999999997 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.964870+0000", 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.274971242 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.239841+0000", 2023-10-09T09:40:54.517 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6891000000000005e-05 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.239938+0000", 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00074391400000000003 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.240682+0000", 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.28962749399999999 2023-10-09T09:40:54.518 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189420 8.4 8:23866f45:::benchmark_data_smithi163_151167_object94187:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.955287+0000", 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 87.415582404000006, 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31199607499999998, 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.519 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189420 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955287+0000", 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955287+0000", 2023-10-09T09:40:54.520 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955285+0000", 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9729999999999997e-06 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955293+0000", 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3506000000000001e-05 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.521 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955307+0000", 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3180000000000001e-06 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955314+0000", 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050446090000000002 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.960359+0000", 2023-10-09T09:40:54.522 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014191900000000001 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.960501+0000", 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000132738 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.960633+0000", 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015849736 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.523 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.976483+0000", 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026229599999999999 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.002713+0000", 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26364709800000002 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.524 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.266360+0000", 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3532000000000003e-05 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.266403+0000", 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00087977300000000004 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.267283+0000", 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31199607499999998 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.525 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189425 8.4 8:24365966:::benchmark_data_smithi163_151167_object94192:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.967562+0000", 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 87.403307343999998, 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31396539400000001, 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.526 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189425 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967562+0000", 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967562+0000", 2023-10-09T09:40:54.527 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967560+0000", 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7849999999999997e-06 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967569+0000", 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6466000000000001e-05 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.528 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967586+0000", 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4059999999999999e-06 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967593+0000", 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002803027 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.970396+0000", 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020983500000000001 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.529 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.970606+0000", 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00075611799999999996 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.971362+0000", 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0063786060000000002 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.530 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.977741+0000", 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.024996424999999999 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.002737+0000", 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27793337200000001 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.280670+0000", 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9372e-05 2023-10-09T09:40:54.531 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.280730+0000", 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00079766199999999998 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.281527+0000", 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31396539400000001 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.532 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189435 8.4 8:29678ca8:::benchmark_data_smithi163_151167_object94202:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.979108+0000", 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 87.391761858999999, 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30305321699999999, 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189435 2023-10-09T09:40:54.533 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979108+0000", 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979108+0000", 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:40:54.534 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979107+0000", 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0119999999999999e-06 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979111+0000", 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1892000000000001e-05 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979123+0000", 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5139999999999998e-06 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979129+0000", 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1113999999999999e-05 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979180+0000", 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014542900000000001 2023-10-09T09:40:54.536 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979326+0000", 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017861299999999999 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979504+0000", 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016815219999999999 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.537 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.996320+0000", 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064391500000000003 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.002759+0000", 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27884090299999997 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.281600+0000", 2023-10-09T09:40:54.538 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4296999999999998e-05 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.281664+0000", 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00049690200000000002 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.282161+0000", 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30305321699999999 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.539 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189436 8.4 8:29159f79:::benchmark_data_smithi163_151167_object94203:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.984927+0000", 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 87.385942354999997, 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.297717699, 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:40:54.540 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189436 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984927+0000", 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984927+0000", 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:40:54.541 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984926+0000", 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7389999999999999e-06 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984933+0000", 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3361e-05 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.542 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984947+0000", 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1429999999999998e-06 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984955+0000", 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043666160000000002 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:40:54.543 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989322+0000", 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000151569 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989473+0000", 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000155713 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989629+0000", 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096470640000000003 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.544 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.999276+0000", 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0035051119999999999 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.002781+0000", 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27945128699999999 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:40:54.545 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.282232+0000", 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6711000000000001e-05 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.282279+0000", 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00036571900000000003 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.282645+0000", 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.297717699 2023-10-09T09:40:54.546 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.547 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.547 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.547 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:40:54.547 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:40:54.547 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:40:54.547 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:54.574 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:54.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:54.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:54.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:54.711 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:54.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:54.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:54.815 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:54 smithi181 bash[59345]: audit 2023-10-09T09:40:53.940586+0000 mon.a (mon.0) 2011 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:40:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:54 smithi181 bash[59345]: audit 2023-10-09T09:40:53.941714+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:40:54.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:54 smithi181 bash[59345]: audit 2023-10-09T09:40:53.947595+0000 mon.a (mon.0) 2013 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:54.830 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 6, 2, 5] out_osds: [7] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:40:54.830 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:40:54.830 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 3, 4, 6, 2, 5] 2023-10-09T09:40:54.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd out 3 2023-10-09T09:40:54.839 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:54.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:54.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:54 smithi163 bash[78107]: audit 2023-10-09T09:40:53.940586+0000 mon.a (mon.0) 2011 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:40:54.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:54 smithi163 bash[78107]: audit 2023-10-09T09:40:53.941714+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:40:54.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:54 smithi163 bash[78107]: audit 2023-10-09T09:40:53.947595+0000 mon.a (mon.0) 2013 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:54.846 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:54 smithi163 bash[75425]: audit 2023-10-09T09:40:53.940586+0000 mon.a (mon.0) 2011 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:40:54.846 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:54 smithi163 bash[75425]: audit 2023-10-09T09:40:53.941714+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:40:54.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:54 smithi163 bash[75425]: audit 2023-10-09T09:40:53.947595+0000 mon.a (mon.0) 2013 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:54.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:54.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:54.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:54.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:55.065 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:55.094 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:55.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:55.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:55.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:55.224 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:55.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:55.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:55.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T09:40:55.327 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:55.361 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:55.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:55.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:55.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:55.496 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:55.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:55.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:55.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:55.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:55.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:55.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:55.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:55.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:55.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:55.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:55.870 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:55.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:55.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:55.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:56.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:56.038 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:56.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:55 smithi181 bash[59345]: cluster 2023-10-09T09:40:53.968720+0000 mgr.y (mgr.34107) 1044 : cluster 0 pgmap v1829: 89 pgs: 6 peering, 83 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 138 KiB/s, 2 objects/s recovering 2023-10-09T09:40:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:55 smithi181 bash[59345]: audit 2023-10-09T09:40:55.394197+0000 mon.a (mon.0) 2014 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:55 smithi181 bash[59345]: audit 2023-10-09T09:40:55.395617+0000 mon.a (mon.0) 2015 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:56.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:56.174 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:56.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:56.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:56.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:55 smithi163 bash[75425]: cluster 2023-10-09T09:40:53.968720+0000 mgr.y (mgr.34107) 1044 : cluster 0 pgmap v1829: 89 pgs: 6 peering, 83 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 138 KiB/s, 2 objects/s recovering 2023-10-09T09:40:56.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:55 smithi163 bash[75425]: audit 2023-10-09T09:40:55.394197+0000 mon.a (mon.0) 2014 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:56.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:55 smithi163 bash[75425]: audit 2023-10-09T09:40:55.395617+0000 mon.a (mon.0) 2015 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:55 smithi163 bash[78107]: cluster 2023-10-09T09:40:53.968720+0000 mgr.y (mgr.34107) 1044 : cluster 0 pgmap v1829: 89 pgs: 6 peering, 83 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 138 KiB/s, 2 objects/s recovering 2023-10-09T09:40:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:55 smithi163 bash[78107]: audit 2023-10-09T09:40:55.394197+0000 mon.a (mon.0) 2014 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:55 smithi163 bash[78107]: audit 2023-10-09T09:40:55.395617+0000 mon.a (mon.0) 2015 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:40:56.278 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:56.307 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:56.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:56.411 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:56.446 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:56.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:56.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:56.550 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:56.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:56.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:56.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:56.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:56.718 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:56.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:56.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:56.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:56.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:56.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:56.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:56.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:56.983 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:40:56.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:56.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:40:57.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:57.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:57.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:57.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:56 smithi163 bash[75425]: cluster 2023-10-09T09:40:55.969606+0000 mgr.y (mgr.34107) 1045 : cluster 0 pgmap v1830: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 97 KiB/s, 1 objects/s recovering 2023-10-09T09:40:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:56 smithi163 bash[75425]: audit 2023-10-09T09:40:55.983815+0000 mon.a (mon.0) 2016 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:56 smithi163 bash[75425]: cluster 2023-10-09T09:40:56.794151+0000 mon.a (mon.0) 2017 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:40:57.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:56 smithi163 bash[78107]: cluster 2023-10-09T09:40:55.969606+0000 mgr.y (mgr.34107) 1045 : cluster 0 pgmap v1830: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 97 KiB/s, 1 objects/s recovering 2023-10-09T09:40:57.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:56 smithi163 bash[78107]: audit 2023-10-09T09:40:55.983815+0000 mon.a (mon.0) 2016 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:57.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:56 smithi163 bash[78107]: cluster 2023-10-09T09:40:56.794151+0000 mon.a (mon.0) 2017 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:40:57.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:57.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:57.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:56 smithi181 bash[59345]: cluster 2023-10-09T09:40:55.969606+0000 mgr.y (mgr.34107) 1045 : cluster 0 pgmap v1830: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 97 KiB/s, 1 objects/s recovering 2023-10-09T09:40:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:56 smithi181 bash[59345]: audit 2023-10-09T09:40:55.983815+0000 mon.a (mon.0) 2016 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:40:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:56 smithi181 bash[59345]: cluster 2023-10-09T09:40:56.794151+0000 mon.a (mon.0) 2017 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T09:40:57.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:57.387 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:57.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:57.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:57.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:57.515 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:57.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:57.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:57.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:57.654 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:57.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:57.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:57.758 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:57.782 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:57.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:57.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:57.886 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:57.915 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:57.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:57.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:58.019 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:58.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:58.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:58.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:58.145 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:58.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:58.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:58.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:58.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:58.302 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:58.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:58.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:58.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:58.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:58.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:58.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:58.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:58.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:58.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:58.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:58.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:58.691 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:58.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:58.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:58.822 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:40:58.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:58.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:40:58.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:58.960 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:40:58.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:58.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:40:59.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:59.098 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:59.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:59.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:59.168 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:59 smithi163 bash[78107]: cluster 2023-10-09T09:40:57.970708+0000 mgr.y (mgr.34107) 1046 : cluster 0 pgmap v1831: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-10-09T09:40:59.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:58 smithi163 bash[75425]: cluster 2023-10-09T09:40:57.970708+0000 mgr.y (mgr.34107) 1046 : cluster 0 pgmap v1831: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-10-09T09:40:59.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:59.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:59.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:59.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:59.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:58 smithi181 bash[59345]: cluster 2023-10-09T09:40:57.970708+0000 mgr.y (mgr.34107) 1046 : cluster 0 pgmap v1831: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-10-09T09:40:59.331 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:59.358 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:59.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:59.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:40:59.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:59.494 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:40:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:59.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:40:59.585 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:40:59.600 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:59.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:40:59.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:59.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:40:59.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:59.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:40:59.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:59.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:40:59.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:59.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:40:59.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:40:59.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:40:59.936 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:40:59.958 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:40:59.978 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:40:59.978 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:40:59.999 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:00.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:00.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:00.005 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:41:00.006 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T09:41:00.018 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:41:00.032 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:41:00.057 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:41:00.076 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:41:00.098 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:41:00.100 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:41:00.100 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:41:00.100 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:41:00.100 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:41:00.100 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:41:00.100 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:41:00.100 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:41:00.100 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:41:00.103 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:00.129 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:00.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:00.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:00.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:59 smithi163 bash[75425]: audit 2023-10-09T09:40:59.792516+0000 mon.c (mon.2) 92 : audit 1 from='client.? 172.21.15.163:0/3186705805' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:41:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:59 smithi163 bash[75425]: audit 2023-10-09T09:40:59.793123+0000 mon.a (mon.0) 2018 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:41:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:40:59 smithi163 bash[75425]: audit 2023-10-09T09:40:59.916364+0000 mon.a (mon.0) 2019 : audit 0 from='client.? 172.21.15.163:0/3203704515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:41:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:59 smithi163 bash[78107]: audit 2023-10-09T09:40:59.792516+0000 mon.c (mon.2) 92 : audit 1 from='client.? 172.21.15.163:0/3186705805' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:41:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:59 smithi163 bash[78107]: audit 2023-10-09T09:40:59.793123+0000 mon.a (mon.0) 2018 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:41:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:40:59 smithi163 bash[78107]: audit 2023-10-09T09:40:59.916364+0000 mon.a (mon.0) 2019 : audit 0 from='client.? 172.21.15.163:0/3203704515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:41:00.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:00.257 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:00.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:00.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:59 smithi181 bash[59345]: audit 2023-10-09T09:40:59.792516+0000 mon.c (mon.2) 92 : audit 1 from='client.? 172.21.15.163:0/3186705805' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:41:00.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:59 smithi181 bash[59345]: audit 2023-10-09T09:40:59.793123+0000 mon.a (mon.0) 2018 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:41:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:40:59 smithi181 bash[59345]: audit 2023-10-09T09:40:59.916364+0000 mon.a (mon.0) 2019 : audit 0 from='client.? 172.21.15.163:0/3203704515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:41:00.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:00.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:00.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:00.495 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:00.525 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:00.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:00.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:00.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:00.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:00.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:00.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:00.756 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:00.787 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:00.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:00.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:00.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:00.920 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:00.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:00.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:01.024 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:01.049 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:01.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:01.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:01.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:01.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[75425]: cluster 2023-10-09T09:40:59.971199+0000 mgr.y (mgr.34107) 1047 : cluster 0 pgmap v1832: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-10-09T09:41:01.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[75425]: cluster 2023-10-09T09:40:59.997569+0000 osd.5 (osd.5) 762 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:01.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[75425]: audit 2023-10-09T09:41:00.000754+0000 mon.a (mon.0) 2020 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:41:01.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[75425]: cluster 2023-10-09T09:41:00.000845+0000 mon.a (mon.0) 2021 : cluster 0 osdmap e1187: 8 total, 8 up, 7 in 2023-10-09T09:41:01.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[75425]: cluster 2023-10-09T09:41:00.543925+0000 osd.7 (osd.7) 574 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:01.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[75425]: cluster 2023-10-09T09:41:00.778250+0000 osd.6 (osd.6) 473 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:01.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[78107]: cluster 2023-10-09T09:40:59.971199+0000 mgr.y (mgr.34107) 1047 : cluster 0 pgmap v1832: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-10-09T09:41:01.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[78107]: cluster 2023-10-09T09:40:59.997569+0000 osd.5 (osd.5) 762 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:01.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[78107]: audit 2023-10-09T09:41:00.000754+0000 mon.a (mon.0) 2020 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:41:01.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[78107]: cluster 2023-10-09T09:41:00.000845+0000 mon.a (mon.0) 2021 : cluster 0 osdmap e1187: 8 total, 8 up, 7 in 2023-10-09T09:41:01.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[78107]: cluster 2023-10-09T09:41:00.543925+0000 osd.7 (osd.7) 574 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:01.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:01 smithi163 bash[78107]: cluster 2023-10-09T09:41:00.778250+0000 osd.6 (osd.6) 473 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:01.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:01.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:01.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:01.311 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:01.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:01.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:00 smithi181 bash[59345]: cluster 2023-10-09T09:40:59.971199+0000 mgr.y (mgr.34107) 1047 : cluster 0 pgmap v1832: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-10-09T09:41:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:00 smithi181 bash[59345]: cluster 2023-10-09T09:40:59.997569+0000 osd.5 (osd.5) 762 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:00 smithi181 bash[59345]: audit 2023-10-09T09:41:00.000754+0000 mon.a (mon.0) 2020 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:41:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:00 smithi181 bash[59345]: cluster 2023-10-09T09:41:00.000845+0000 mon.a (mon.0) 2021 : cluster 0 osdmap e1187: 8 total, 8 up, 7 in 2023-10-09T09:41:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:01 smithi181 bash[59345]: cluster 2023-10-09T09:41:00.543925+0000 osd.7 (osd.7) 574 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:01 smithi181 bash[59345]: cluster 2023-10-09T09:41:00.778250+0000 osd.6 (osd.6) 473 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:01.414 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:01.447 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:01.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:01.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:01.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:01.577 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:01.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:01.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:01.712 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:01.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:01.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:01.815 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:01.842 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:01.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:01.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:01.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:01.974 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:01.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:01.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:02.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:02.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:02.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:02.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:02 smithi163 bash[75425]: cluster 2023-10-09T09:41:00.778084+0000 osd.4 (osd.4) 612 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:02.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:02 smithi163 bash[78107]: cluster 2023-10-09T09:41:00.778084+0000 osd.4 (osd.4) 612 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:02.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:02.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:02 smithi181 bash[59345]: cluster 2023-10-09T09:41:00.778084+0000 osd.4 (osd.4) 612 : cluster 3 failed to encode map e1187 with expected crc 2023-10-09T09:41:02.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:02.411 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:02.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:02.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:02.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:02.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:02.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:02.646 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:02.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:02.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:02.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:02.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:02.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:02.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:02.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:02.908 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:02.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:02.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:02.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:03.040 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:03.065 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:03.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:03.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:03.169 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:03.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:03 smithi163 bash[78107]: cluster 2023-10-09T09:41:01.971931+0000 mgr.y (mgr.34107) 1048 : cluster 0 pgmap v1834: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:41:03.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:03 smithi163 bash[75425]: cluster 2023-10-09T09:41:01.971931+0000 mgr.y (mgr.34107) 1048 : cluster 0 pgmap v1834: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:41:03.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:03.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:03.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:03.296 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:03.307 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:03 smithi181 bash[59345]: cluster 2023-10-09T09:41:01.971931+0000 mgr.y (mgr.34107) 1048 : cluster 0 pgmap v1834: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:41:03.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:03.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:03.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:03.338 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:41:03.338 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:41:03.432 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:03.457 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:03.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:03.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:03.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:41:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:41:03] "GET /metrics HTTP/1.1" 200 36282 "" "Prometheus/2.33.4" 2023-10-09T09:41:03.561 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:03.588 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:03.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:03.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:03.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:03.723 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:03.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:03.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:03.827 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:03.853 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:03.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:03.956 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:03.989 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:03.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:03.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:04.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:04.119 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:04.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:04.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:04.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T09:41:04.223 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:04.247 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:04.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:04.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:04.351 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:04.377 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:04.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:04.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:04.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:04.512 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:04.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:04.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:04.615 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:04.639 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:04.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:04.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:04.742 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:04.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:04.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:04.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:04.870 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:04.896 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:04.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:04.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:04.935 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T09:41:04.944 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:41:04.944 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:41:05.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:05.024 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:05.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:05.030 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:41:05.054 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T09:41:05.054 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:41:05.054 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_175771 2023-10-09T09:41:05.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:41:05.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:41:05.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:05.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:05 smithi163 bash[75425]: cluster 2023-10-09T09:41:03.972474+0000 mgr.y (mgr.34107) 1049 : cluster 0 pgmap v1835: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:41:05.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:05 smithi163 bash[75425]: audit 2023-10-09T09:41:04.917607+0000 mon.c (mon.2) 93 : audit 1 from='client.? 172.21.15.163:0/1866420565' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:41:05.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:05 smithi163 bash[75425]: audit 2023-10-09T09:41:04.918055+0000 mon.a (mon.0) 2022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:41:05.141 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:05 smithi163 bash[78107]: cluster 2023-10-09T09:41:03.972474+0000 mgr.y (mgr.34107) 1049 : cluster 0 pgmap v1835: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:41:05.141 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:05 smithi163 bash[78107]: audit 2023-10-09T09:41:04.917607+0000 mon.c (mon.2) 93 : audit 1 from='client.? 172.21.15.163:0/1866420565' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:41:05.142 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:05 smithi163 bash[78107]: audit 2023-10-09T09:41:04.918055+0000 mon.a (mon.0) 2022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:41:05.164 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:05.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:05.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:05.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:05.281 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:05 smithi181 bash[59345]: cluster 2023-10-09T09:41:03.972474+0000 mgr.y (mgr.34107) 1049 : cluster 0 pgmap v1835: 89 pgs: 89 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T09:41:05.282 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:05 smithi181 bash[59345]: audit 2023-10-09T09:41:04.917607+0000 mon.c (mon.2) 93 : audit 1 from='client.? 172.21.15.163:0/1866420565' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:41:05.282 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:05 smithi181 bash[59345]: audit 2023-10-09T09:41:04.918055+0000 mon.a (mon.0) 2022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:41:05.300 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:05.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:05.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:05.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:05.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:05.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:05.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:05.535 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:05.561 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:05.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:05.665 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:05.694 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:05.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:05.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:05.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:05.827 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:05.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:05.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:05.932 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:05.963 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:05.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:05.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:05.976 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:41:06.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 2496 2480 154.992 155 0.00177842 0.00633915 2023-10-09T09:41:06.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:06.096 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:06.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:06.200 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:06.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[75425]: cluster 2023-10-09T09:41:05.013297+0000 mon.a (mon.0) 2023 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:41:06.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[75425]: cluster 2023-10-09T09:41:05.022048+0000 osd.5 (osd.5) 763 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[75425]: audit 2023-10-09T09:41:05.026152+0000 mon.a (mon.0) 2024 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:41:06.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[75425]: cluster 2023-10-09T09:41:05.026272+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e1188: 8 total, 8 up, 7 in 2023-10-09T09:41:06.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[75425]: cluster 2023-10-09T09:41:05.057736+0000 osd.4 (osd.4) 613 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[75425]: cluster 2023-10-09T09:41:05.059629+0000 osd.6 (osd.6) 474 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[75425]: cluster 2023-10-09T09:41:05.062449+0000 osd.4 (osd.4) 614 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[75425]: cluster 2023-10-09T09:41:05.592127+0000 osd.7 (osd.7) 575 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[75425]: audit 2023-10-09T09:41:05.998273+0000 mon.a (mon.0) 2026 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:06.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[78107]: cluster 2023-10-09T09:41:05.013297+0000 mon.a (mon.0) 2023 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:41:06.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[78107]: cluster 2023-10-09T09:41:05.022048+0000 osd.5 (osd.5) 763 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[78107]: audit 2023-10-09T09:41:05.026152+0000 mon.a (mon.0) 2024 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:41:06.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[78107]: cluster 2023-10-09T09:41:05.026272+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e1188: 8 total, 8 up, 7 in 2023-10-09T09:41:06.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[78107]: cluster 2023-10-09T09:41:05.057736+0000 osd.4 (osd.4) 613 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[78107]: cluster 2023-10-09T09:41:05.059629+0000 osd.6 (osd.6) 474 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[78107]: cluster 2023-10-09T09:41:05.062449+0000 osd.4 (osd.4) 614 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[78107]: cluster 2023-10-09T09:41:05.592127+0000 osd.7 (osd.7) 575 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:06 smithi163 bash[78107]: audit 2023-10-09T09:41:05.998273+0000 mon.a (mon.0) 2026 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:06.218 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:06.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:06.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:06 smithi181 bash[59345]: cluster 2023-10-09T09:41:05.013297+0000 mon.a (mon.0) 2023 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:41:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:06 smithi181 bash[59345]: cluster 2023-10-09T09:41:05.022048+0000 osd.5 (osd.5) 763 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:06 smithi181 bash[59345]: audit 2023-10-09T09:41:05.026152+0000 mon.a (mon.0) 2024 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:41:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:06 smithi181 bash[59345]: cluster 2023-10-09T09:41:05.026272+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e1188: 8 total, 8 up, 7 in 2023-10-09T09:41:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:06 smithi181 bash[59345]: cluster 2023-10-09T09:41:05.057736+0000 osd.4 (osd.4) 613 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:06 smithi181 bash[59345]: cluster 2023-10-09T09:41:05.059629+0000 osd.6 (osd.6) 474 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:06 smithi181 bash[59345]: cluster 2023-10-09T09:41:05.062449+0000 osd.4 (osd.4) 614 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:06 smithi181 bash[59345]: cluster 2023-10-09T09:41:05.592127+0000 osd.7 (osd.7) 575 : cluster 3 failed to encode map e1188 with expected crc 2023-10-09T09:41:06.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:06 smithi181 bash[59345]: audit 2023-10-09T09:41:05.998273+0000 mon.a (mon.0) 2026 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:06.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:06.342 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:06.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:06.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:06.445 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:06.478 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:06.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:06.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:06.582 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:06.607 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:06.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:06.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:06.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:06.767 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:06.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:06.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:06.870 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:06.906 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:06.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:06.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:07.010 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:07.038 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:07.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:07.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:07.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 4019 4003 125.084 95.1875 0.00378586 0.00797562 2023-10-09T09:41:07.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:07.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:07.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:07.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:07.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:07.313 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:07.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:07.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:07.442 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:07.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:07.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:07.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:07.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:07.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:07.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:07.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:07.708 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:07.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:07.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:07.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:07.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:07.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:07.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:07.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:07.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:07 smithi163 bash[78107]: cluster 2023-10-09T09:41:05.972905+0000 mgr.y (mgr.34107) 1050 : cluster 0 pgmap v1837: 89 pgs: 89 active+clean; 9.3 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 1.1 MiB/s wr, 17 op/s 2023-10-09T09:41:07.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:07 smithi163 bash[75425]: cluster 2023-10-09T09:41:05.972905+0000 mgr.y (mgr.34107) 1050 : cluster 0 pgmap v1837: 89 pgs: 89 active+clean; 9.3 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 1.1 MiB/s wr, 17 op/s 2023-10-09T09:41:07.983 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:07.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:07.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:08.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 6327 6311 131.469 144.25 0.00254822 0.00759447 2023-10-09T09:41:08.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:07 smithi181 bash[59345]: cluster 2023-10-09T09:41:05.972905+0000 mgr.y (mgr.34107) 1050 : cluster 0 pgmap v1837: 89 pgs: 89 active+clean; 9.3 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 1.1 MiB/s wr, 17 op/s 2023-10-09T09:41:08.086 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:08.115 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:08.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:08.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:08.218 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:08.266 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:08.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:08.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:08.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:08.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:08.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:08.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:08.507 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:08.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:08.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:08.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:08.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:08.658 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:08.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:08.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:08.761 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:08.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:08.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:08.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:08.897 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:08.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:08.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:08.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:08.942 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:41:09.026 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:09.053 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:09.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 8651 8635 134.911 145.25 0.00723977 0.00739792 2023-10-09T09:41:09.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:09.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:09.157 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:09.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:09.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:09.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:09.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:09.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:09.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:09.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:09.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:09.707 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:09.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:09.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:09.811 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:09.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:09 smithi181 bash[59345]: cluster 2023-10-09T09:41:07.973573+0000 mgr.y (mgr.34107) 1051 : cluster 0 pgmap v1838: 89 pgs: 89 active+clean; 131 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 16 MiB/s wr, 260 op/s 2023-10-09T09:41:09.955 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:09 smithi163 bash[78107]: cluster 2023-10-09T09:41:07.973573+0000 mgr.y (mgr.34107) 1051 : cluster 0 pgmap v1838: 89 pgs: 89 active+clean; 131 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 16 MiB/s wr, 260 op/s 2023-10-09T09:41:09.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:09 smithi163 bash[75425]: cluster 2023-10-09T09:41:07.973573+0000 mgr.y (mgr.34107) 1051 : cluster 0 pgmap v1838: 89 pgs: 89 active+clean; 131 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 16 MiB/s wr, 260 op/s 2023-10-09T09:41:09.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:09 smithi163 bash[75425]: debug 2023-10-09T09:41:09.950+0000 7f22f321a700 -1 mon.a@0(leader).osd e1188 definitely_dead 0 2023-10-09T09:41:09.965 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:09.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:09.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:10.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:41:10.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 10703 10687 133.576 128.25 0.00580461 0.00747237 2023-10-09T09:41:10.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:10.106 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:10.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:10.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:10.240 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:10.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:10.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:10.343 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:10.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:10.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:10.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:10.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:10.515 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:10.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:10.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:10.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:10.646 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:10.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:10.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:10.753 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:10.779 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:10.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:10.788 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.3. 2023-10-09T09:41:10.883 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:10.896 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:10 smithi181 bash[59345]: audit 2023-10-09T09:41:09.953834+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.163:0/3995485989' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T09:41:10.896 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:10 smithi181 bash[59345]: audit 2023-10-09T09:41:09.954250+0000 mon.a (mon.0) 2027 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T09:41:10.896 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:10 smithi181 bash[59345]: cluster 2023-10-09T09:41:09.954327+0000 mon.a (mon.0) 2028 : cluster 1 Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T09:41:10.897 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:10 smithi181 bash[59345]: audit 2023-10-09T09:41:10.391823+0000 mon.a (mon.0) 2029 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:10.897 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:10 smithi181 bash[59345]: audit 2023-10-09T09:41:10.393284+0000 mon.a (mon.0) 2030 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:10.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:10.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:10.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:11.014 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:11.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:10 smithi163 bash[75425]: audit 2023-10-09T09:41:09.953834+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.163:0/3995485989' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T09:41:11.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:10 smithi163 bash[75425]: audit 2023-10-09T09:41:09.954250+0000 mon.a (mon.0) 2027 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T09:41:11.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:10 smithi163 bash[75425]: cluster 2023-10-09T09:41:09.954327+0000 mon.a (mon.0) 2028 : cluster 1 Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T09:41:11.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:10 smithi163 bash[75425]: audit 2023-10-09T09:41:10.391823+0000 mon.a (mon.0) 2029 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:11.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:10 smithi163 bash[75425]: audit 2023-10-09T09:41:10.393284+0000 mon.a (mon.0) 2030 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:11.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:10 smithi163 bash[78107]: audit 2023-10-09T09:41:09.953834+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.163:0/3995485989' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T09:41:11.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:10 smithi163 bash[78107]: audit 2023-10-09T09:41:09.954250+0000 mon.a (mon.0) 2027 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T09:41:11.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:10 smithi163 bash[78107]: cluster 2023-10-09T09:41:09.954327+0000 mon.a (mon.0) 2028 : cluster 1 Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T09:41:11.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:10 smithi163 bash[78107]: audit 2023-10-09T09:41:10.391823+0000 mon.a (mon.0) 2029 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:11.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:10 smithi163 bash[78107]: audit 2023-10-09T09:41:10.393284+0000 mon.a (mon.0) 2030 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:11.044 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:11.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:11.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 12176 12160 126.655 92.0625 0.00253465 0.00755168 2023-10-09T09:41:11.148 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:11.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:11.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:11.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:11.275 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:11.309 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:11.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:11.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:11.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:11.441 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:11.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:11.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:11.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:11.574 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:11.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:11.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:11.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:11.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:11.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:11.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:11.810 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:11.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:11.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:11.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:11.966 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:11.976 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:11 smithi181 bash[59345]: cluster 2023-10-09T09:41:09.974140+0000 mgr.y (mgr.34107) 1052 : cluster 0 pgmap v1839: 89 pgs: 89 active+clean; 325 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 33 MiB/s wr, 520 op/s 2023-10-09T09:41:11.976 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:11 smithi181 bash[59345]: cluster 2023-10-09T09:41:10.770394+0000 osd.7 (osd.7) 576 : cluster 3 failed to encode map e1189 with expected crc 2023-10-09T09:41:11.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:11 smithi181 bash[59345]: cluster 2023-10-09T09:41:10.770703+0000 osd.5 (osd.5) 764 : cluster 3 failed to encode map e1189 with expected crc 2023-10-09T09:41:11.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:11 smithi181 bash[59345]: audit 2023-10-09T09:41:10.781071+0000 mon.a (mon.0) 2031 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T09:41:11.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:11 smithi181 bash[59345]: cluster 2023-10-09T09:41:10.781277+0000 mon.a (mon.0) 2032 : cluster 0 osdmap e1189: 8 total, 8 up, 6 in 2023-10-09T09:41:11.990 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:11.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:11.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:12.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 16 12176 12160 108.561 0 - 0.00755168 2023-10-09T09:41:12.093 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:12.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:12.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:12.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:12.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:11 smithi163 bash[78107]: cluster 2023-10-09T09:41:09.974140+0000 mgr.y (mgr.34107) 1052 : cluster 0 pgmap v1839: 89 pgs: 89 active+clean; 325 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 33 MiB/s wr, 520 op/s 2023-10-09T09:41:12.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:11 smithi163 bash[78107]: cluster 2023-10-09T09:41:10.770394+0000 osd.7 (osd.7) 576 : cluster 3 failed to encode map e1189 with expected crc 2023-10-09T09:41:12.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:11 smithi163 bash[78107]: cluster 2023-10-09T09:41:10.770703+0000 osd.5 (osd.5) 764 : cluster 3 failed to encode map e1189 with expected crc 2023-10-09T09:41:12.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:11 smithi163 bash[78107]: audit 2023-10-09T09:41:10.781071+0000 mon.a (mon.0) 2031 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T09:41:12.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:11 smithi163 bash[78107]: cluster 2023-10-09T09:41:10.781277+0000 mon.a (mon.0) 2032 : cluster 0 osdmap e1189: 8 total, 8 up, 6 in 2023-10-09T09:41:12.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:11 smithi163 bash[75425]: cluster 2023-10-09T09:41:09.974140+0000 mgr.y (mgr.34107) 1052 : cluster 0 pgmap v1839: 89 pgs: 89 active+clean; 325 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 33 MiB/s wr, 520 op/s 2023-10-09T09:41:12.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:11 smithi163 bash[75425]: cluster 2023-10-09T09:41:10.770394+0000 osd.7 (osd.7) 576 : cluster 3 failed to encode map e1189 with expected crc 2023-10-09T09:41:12.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:11 smithi163 bash[75425]: cluster 2023-10-09T09:41:10.770703+0000 osd.5 (osd.5) 764 : cluster 3 failed to encode map e1189 with expected crc 2023-10-09T09:41:12.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:11 smithi163 bash[75425]: audit 2023-10-09T09:41:10.781071+0000 mon.a (mon.0) 2031 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T09:41:12.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:11 smithi163 bash[75425]: cluster 2023-10-09T09:41:10.781277+0000 mon.a (mon.0) 2032 : cluster 0 osdmap e1189: 8 total, 8 up, 6 in 2023-10-09T09:41:12.222 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:12.253 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:12.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:12.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:12.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:12.387 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:12.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:12.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:12.490 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:12.513 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:12.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:12.616 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:12.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:12.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:12.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:12.743 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:12.767 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:12.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:12.869 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:12.925 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:12.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:12.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:13.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[75425]: cluster 2023-10-09T09:41:11.774178+0000 osd.5 (osd.5) 765 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[75425]: cluster 2023-10-09T09:41:11.775076+0000 osd.6 (osd.6) 475 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[75425]: cluster 2023-10-09T09:41:11.776732+0000 mon.a (mon.0) 2033 : cluster 0 osdmap e1190: 8 total, 8 up, 6 in 2023-10-09T09:41:13.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[75425]: cluster 2023-10-09T09:41:11.777066+0000 osd.5 (osd.5) 766 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[75425]: cluster 2023-10-09T09:41:11.777625+0000 osd.6 (osd.6) 476 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[75425]: cluster 2023-10-09T09:41:11.778000+0000 osd.4 (osd.4) 615 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[75425]: cluster 2023-10-09T09:41:11.781890+0000 osd.4 (osd.4) 616 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[78107]: cluster 2023-10-09T09:41:11.774178+0000 osd.5 (osd.5) 765 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[78107]: cluster 2023-10-09T09:41:11.775076+0000 osd.6 (osd.6) 475 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[78107]: cluster 2023-10-09T09:41:11.776732+0000 mon.a (mon.0) 2033 : cluster 0 osdmap e1190: 8 total, 8 up, 6 in 2023-10-09T09:41:13.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[78107]: cluster 2023-10-09T09:41:11.777066+0000 osd.5 (osd.5) 766 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.028 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[78107]: cluster 2023-10-09T09:41:11.777625+0000 osd.6 (osd.6) 476 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.028 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[78107]: cluster 2023-10-09T09:41:11.778000+0000 osd.4 (osd.4) 615 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.028 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:12 smithi163 bash[78107]: cluster 2023-10-09T09:41:11.781890+0000 osd.4 (osd.4) 616 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.029 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:13.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:12 smithi181 bash[59345]: cluster 2023-10-09T09:41:11.774178+0000 osd.5 (osd.5) 765 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:12 smithi181 bash[59345]: cluster 2023-10-09T09:41:11.775076+0000 osd.6 (osd.6) 475 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:12 smithi181 bash[59345]: cluster 2023-10-09T09:41:11.776732+0000 mon.a (mon.0) 2033 : cluster 0 osdmap e1190: 8 total, 8 up, 6 in 2023-10-09T09:41:13.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:12 smithi181 bash[59345]: cluster 2023-10-09T09:41:11.777066+0000 osd.5 (osd.5) 766 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:12 smithi181 bash[59345]: cluster 2023-10-09T09:41:11.777625+0000 osd.6 (osd.6) 476 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:12 smithi181 bash[59345]: cluster 2023-10-09T09:41:11.778000+0000 osd.4 (osd.4) 615 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:12 smithi181 bash[59345]: cluster 2023-10-09T09:41:11.781890+0000 osd.4 (osd.4) 616 : cluster 3 failed to encode map e1190 with expected crc 2023-10-09T09:41:13.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 12535 12519 97.7959 11.2188 0.0103185 0.0102127 2023-10-09T09:41:13.062 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:13.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:13.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:13.165 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:13.193 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:13.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:13.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:13.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:13.320 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:13.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:13.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:13.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:13.446 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:41:13.449 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:13.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:13.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:13.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:13.562 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:41:13] "GET /metrics HTTP/1.1" 200 36325 "" "Prometheus/2.33.4" 2023-10-09T09:41:13.581 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:13.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:13.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:13.686 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:13.714 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:13.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:13.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:13.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:13.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[78107]: cluster 2023-10-09T09:41:11.974585+0000 mgr.y (mgr.34107) 1053 : cluster 0 pgmap v1842: 89 pgs: 1 unknown, 23 peering, 65 active+clean; 594 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T09:41:13.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[78107]: cluster 2023-10-09T09:41:12.772916+0000 mon.a (mon.0) 2034 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:41:13.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[78107]: cluster 2023-10-09T09:41:12.784378+0000 osd.5 (osd.5) 767 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:13.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[78107]: cluster 2023-10-09T09:41:12.789469+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e1191: 8 total, 8 up, 6 in 2023-10-09T09:41:13.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[78107]: cluster 2023-10-09T09:41:12.789583+0000 osd.5 (osd.5) 768 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:13.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[78107]: cluster 2023-10-09T09:41:12.791213+0000 osd.4 (osd.4) 617 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:13.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[78107]: cluster 2023-10-09T09:41:12.791363+0000 osd.6 (osd.6) 477 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:13.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[78107]: cluster 2023-10-09T09:41:12.795369+0000 osd.4 (osd.4) 618 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:13.827 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[75425]: cluster 2023-10-09T09:41:11.974585+0000 mgr.y (mgr.34107) 1053 : cluster 0 pgmap v1842: 89 pgs: 1 unknown, 23 peering, 65 active+clean; 594 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T09:41:13.827 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[75425]: cluster 2023-10-09T09:41:12.772916+0000 mon.a (mon.0) 2034 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:41:13.827 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[75425]: cluster 2023-10-09T09:41:12.784378+0000 osd.5 (osd.5) 767 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:13.828 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[75425]: cluster 2023-10-09T09:41:12.789469+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e1191: 8 total, 8 up, 6 in 2023-10-09T09:41:13.828 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[75425]: cluster 2023-10-09T09:41:12.789583+0000 osd.5 (osd.5) 768 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:13.828 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[75425]: cluster 2023-10-09T09:41:12.791213+0000 osd.4 (osd.4) 617 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:13.828 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[75425]: cluster 2023-10-09T09:41:12.791363+0000 osd.6 (osd.6) 477 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:13.828 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:13 smithi163 bash[75425]: cluster 2023-10-09T09:41:12.795369+0000 osd.4 (osd.4) 618 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:13.852 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:13.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:13.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:13.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:13.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:13.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:13.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:14.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 14198 14182 98.4772 103.938 0.00758324 0.0101463 2023-10-09T09:41:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:13 smithi181 bash[59345]: cluster 2023-10-09T09:41:11.974585+0000 mgr.y (mgr.34107) 1053 : cluster 0 pgmap v1842: 89 pgs: 1 unknown, 23 peering, 65 active+clean; 594 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T09:41:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:13 smithi181 bash[59345]: cluster 2023-10-09T09:41:12.772916+0000 mon.a (mon.0) 2034 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T09:41:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:13 smithi181 bash[59345]: cluster 2023-10-09T09:41:12.784378+0000 osd.5 (osd.5) 767 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:13 smithi181 bash[59345]: cluster 2023-10-09T09:41:12.789469+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e1191: 8 total, 8 up, 6 in 2023-10-09T09:41:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:13 smithi181 bash[59345]: cluster 2023-10-09T09:41:12.789583+0000 osd.5 (osd.5) 768 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:13 smithi181 bash[59345]: cluster 2023-10-09T09:41:12.791213+0000 osd.4 (osd.4) 617 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:14.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:13 smithi181 bash[59345]: cluster 2023-10-09T09:41:12.791363+0000 osd.6 (osd.6) 477 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:14.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:13 smithi181 bash[59345]: cluster 2023-10-09T09:41:12.795369+0000 osd.4 (osd.4) 618 : cluster 3 failed to encode map e1191 with expected crc 2023-10-09T09:41:14.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:14.114 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:14.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:14.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:14.218 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:14.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:14.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:14.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:14.349 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:14.375 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:14.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:14.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:14.478 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:14.505 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:14.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:14.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:14.609 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:14.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:14.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:14.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:14.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:14.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:14.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:14.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:14.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:14.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:14.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:14.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:15.010 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:15.036 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:15.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:15.055 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 15975 15959 99.7347 111.062 0.00694491 0.0100178 2023-10-09T09:41:15.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:15.168 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:15.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:15.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:15.260 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:41:15.260 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:41:15.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:15.300 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:15.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:15.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:15.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:15.435 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:15.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:15.537 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:15.567 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:15.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:15.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:15.671 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:15.699 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:15.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:15.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:15.802 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:15.836 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:15.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:15.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:15.943 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:15.952 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:15 smithi181 bash[59345]: cluster 2023-10-09T09:41:13.975099+0000 mgr.y (mgr.34107) 1054 : cluster 0 pgmap v1844: 89 pgs: 2 remapped+peering, 1 unknown, 23 peering, 63 active+clean; 716 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T09:41:15.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:15 smithi163 bash[75425]: cluster 2023-10-09T09:41:13.975099+0000 mgr.y (mgr.34107) 1054 : cluster 0 pgmap v1844: 89 pgs: 2 remapped+peering, 1 unknown, 23 peering, 63 active+clean; 716 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T09:41:15.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:15 smithi163 bash[78107]: cluster 2023-10-09T09:41:13.975099+0000 mgr.y (mgr.34107) 1054 : cluster 0 pgmap v1844: 89 pgs: 2 remapped+peering, 1 unknown, 23 peering, 63 active+clean; 716 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T09:41:15.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:15.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:15.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:16.056 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 17700 17684 100.468 107.812 0.00976057 0.00994252 2023-10-09T09:41:16.075 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:16.108 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:16.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:16.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:16.211 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:16.322 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:16.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:16.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:16.426 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:16.453 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:16.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:16.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:16.563 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:41:16.600 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:16.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:16.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:16.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:16.736 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:16.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:16.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:16.801 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T09:41:16.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:16.878 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:16.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:16.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:16 smithi163 bash[75425]: cluster 2023-10-09T09:41:16.803643+0000 mon.a (mon.0) 2036 : cluster 3 Health check failed: Degraded data redundancy: 3041/41715 objects degraded (7.290%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:16 smithi163 bash[78107]: cluster 2023-10-09T09:41:16.803643+0000 mon.a (mon.0) 2036 : cluster 3 Health check failed: Degraded data redundancy: 3041/41715 objects degraded (7.290%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:16.985 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:17.017 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:17.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:17.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:17.056 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 19296 19280 100.407 99.75 0.00968837 0.00995086 2023-10-09T09:41:17.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:16 smithi181 bash[59345]: cluster 2023-10-09T09:41:16.803643+0000 mon.a (mon.0) 2036 : cluster 3 Health check failed: Degraded data redundancy: 3041/41715 objects degraded (7.290%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:17.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:17.153 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:17.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:17.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:17.258 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:17.295 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:17.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:17.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:17.431 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:17.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:17.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:17.536 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:17.562 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:17.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:17.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:17.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:17.688 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:17.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:17.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:17.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:17.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:17.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:17.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:17.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:17.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:17.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:18.027 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 6, 2, 5] out_osds: [7, 3] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:41:18.027 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:41:18.028 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-10-09T09:41:18.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:41:18.037 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:17 smithi163 bash[78107]: cluster 2023-10-09T09:41:15.975754+0000 mgr.y (mgr.34107) 1055 : cluster 0 pgmap v1845: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 18 peering, 65 active+clean; 868 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 90 MiB/s wr, 1.45k op/s; 3041/41715 objects degraded (7.290%); 43 KiB/s, 0 objects/s recovering 2023-10-09T09:41:18.037 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:17 smithi163 bash[75425]: cluster 2023-10-09T09:41:15.975754+0000 mgr.y (mgr.34107) 1055 : cluster 0 pgmap v1845: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 18 peering, 65 active+clean; 868 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 90 MiB/s wr, 1.45k op/s; 3041/41715 objects degraded (7.290%); 43 KiB/s, 0 objects/s recovering 2023-10-09T09:41:18.055 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:18.056 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 20779 20763 99.8129 92.6875 0.0121862 0.0100123 2023-10-09T09:41:18.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:18.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:18.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:18.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:17 smithi181 bash[59345]: cluster 2023-10-09T09:41:15.975754+0000 mgr.y (mgr.34107) 1055 : cluster 0 pgmap v1845: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 18 peering, 65 active+clean; 868 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 90 MiB/s wr, 1.45k op/s; 3041/41715 objects degraded (7.290%); 43 KiB/s, 0 objects/s recovering 2023-10-09T09:41:18.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:18.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:18.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:18.318 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:18.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:18.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:18.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:18.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:18.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:18.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:18.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:18.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:18.614 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:18.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:18.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:18.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:18.750 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:18.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:18.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:18.854 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:18.882 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:18.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:18.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:18.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:18 smithi163 bash[75425]: cluster 2023-10-09T09:41:18.861978+0000 mon.a (mon.0) 2037 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T09:41:18.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:19.016 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:19.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:19.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:19.056 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 22437 22421 100.084 103.625 0.0054516 0.00994862 2023-10-09T09:41:19.119 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:19.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:18 smithi181 bash[59345]: cluster 2023-10-09T09:41:18.861978+0000 mon.a (mon.0) 2037 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T09:41:19.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:18 smithi181 bash[59345]: cluster 2023-10-09T09:41:18.879974+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e1192: 8 total, 8 up, 6 in 2023-10-09T09:41:19.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:19.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:19.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:19.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:18 smithi163 bash[75425]: cluster 2023-10-09T09:41:18.879974+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e1192: 8 total, 8 up, 6 in 2023-10-09T09:41:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:18 smithi163 bash[78107]: cluster 2023-10-09T09:41:18.861978+0000 mon.a (mon.0) 2037 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T09:41:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:18 smithi163 bash[78107]: cluster 2023-10-09T09:41:18.879974+0000 mon.a (mon.0) 2038 : cluster 0 osdmap e1192: 8 total, 8 up, 6 in 2023-10-09T09:41:19.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:19.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:19.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:19.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:19.380 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:19.399 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:19.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:19.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:19.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:19.533 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:19.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:19.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:19.636 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:19.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:19.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:19.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:19.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:19.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:19.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:19.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:19.928 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:19.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:20.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:20.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 22786 22770 94.8661 21.8125 0.00573665 0.010497 2023-10-09T09:41:20.058 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:20.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:20.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:19 smithi181 bash[59345]: cluster 2023-10-09T09:41:17.976539+0000 mgr.y (mgr.34107) 1056 : cluster 0 pgmap v1846: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.1 GiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 101 MiB/s wr, 1.62k op/s; 6028/52710 objects degraded (11.436%); 598/52710 objects misplaced (1.135%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T09:41:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:19 smithi181 bash[59345]: cluster 2023-10-09T09:41:18.877588+0000 osd.5 (osd.5) 769 : cluster 3 failed to encode map e1192 with expected crc 2023-10-09T09:41:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:19 smithi181 bash[59345]: cluster 2023-10-09T09:41:18.882330+0000 osd.6 (osd.6) 478 : cluster 3 failed to encode map e1192 with expected crc 2023-10-09T09:41:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:19 smithi181 bash[59345]: cluster 2023-10-09T09:41:18.886282+0000 osd.6 (osd.6) 479 : cluster 3 failed to encode map e1192 with expected crc 2023-10-09T09:41:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:19 smithi181 bash[59345]: cluster 2023-10-09T09:41:19.882481+0000 mon.a (mon.0) 2039 : cluster 0 osdmap e1193: 8 total, 8 up, 6 in 2023-10-09T09:41:20.162 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:20.193 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:20.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:20.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:19 smithi163 bash[78107]: cluster 2023-10-09T09:41:17.976539+0000 mgr.y (mgr.34107) 1056 : cluster 0 pgmap v1846: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.1 GiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 101 MiB/s wr, 1.62k op/s; 6028/52710 objects degraded (11.436%); 598/52710 objects misplaced (1.135%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T09:41:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:19 smithi163 bash[78107]: cluster 2023-10-09T09:41:18.877588+0000 osd.5 (osd.5) 769 : cluster 3 failed to encode map e1192 with expected crc 2023-10-09T09:41:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:19 smithi163 bash[78107]: cluster 2023-10-09T09:41:18.882330+0000 osd.6 (osd.6) 478 : cluster 3 failed to encode map e1192 with expected crc 2023-10-09T09:41:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:19 smithi163 bash[78107]: cluster 2023-10-09T09:41:18.886282+0000 osd.6 (osd.6) 479 : cluster 3 failed to encode map e1192 with expected crc 2023-10-09T09:41:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:19 smithi163 bash[78107]: cluster 2023-10-09T09:41:19.882481+0000 mon.a (mon.0) 2039 : cluster 0 osdmap e1193: 8 total, 8 up, 6 in 2023-10-09T09:41:20.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:19 smithi163 bash[75425]: cluster 2023-10-09T09:41:17.976539+0000 mgr.y (mgr.34107) 1056 : cluster 0 pgmap v1846: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.1 GiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 101 MiB/s wr, 1.62k op/s; 6028/52710 objects degraded (11.436%); 598/52710 objects misplaced (1.135%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T09:41:20.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:19 smithi163 bash[75425]: cluster 2023-10-09T09:41:18.877588+0000 osd.5 (osd.5) 769 : cluster 3 failed to encode map e1192 with expected crc 2023-10-09T09:41:20.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:19 smithi163 bash[75425]: cluster 2023-10-09T09:41:18.882330+0000 osd.6 (osd.6) 478 : cluster 3 failed to encode map e1192 with expected crc 2023-10-09T09:41:20.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:19 smithi163 bash[75425]: cluster 2023-10-09T09:41:18.886282+0000 osd.6 (osd.6) 479 : cluster 3 failed to encode map e1192 with expected crc 2023-10-09T09:41:20.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:19 smithi163 bash[75425]: cluster 2023-10-09T09:41:19.882481+0000 mon.a (mon.0) 2039 : cluster 0 osdmap e1193: 8 total, 8 up, 6 in 2023-10-09T09:41:20.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:20.329 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:20.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:20.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:20.433 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:20.465 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:20.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:20.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:20.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:20.597 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:20.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:20.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:20.700 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:20.732 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:20.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:20.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:20.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:20.863 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:20.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:20.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:20.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:20.976 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:20 smithi181 bash[59345]: cluster 2023-10-09T09:41:19.877504+0000 osd.5 (osd.5) 770 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:20.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:20 smithi181 bash[59345]: cluster 2023-10-09T09:41:19.880042+0000 osd.6 (osd.6) 480 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:20.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:20 smithi181 bash[59345]: cluster 2023-10-09T09:41:19.883244+0000 osd.6 (osd.6) 481 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:20.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:20 smithi181 bash[59345]: cluster 2023-10-09T09:41:19.885993+0000 osd.6 (osd.6) 482 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:20.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:20 smithi181 bash[59345]: cluster 2023-10-09T09:41:20.885013+0000 mon.a (mon.0) 2040 : cluster 0 osdmap e1194: 8 total, 8 up, 6 in 2023-10-09T09:41:20.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:20.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:20.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:21.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 23072 23056 90.054 17.875 0.00926795 0.0110984 2023-10-09T09:41:21.096 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:21.105 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:20 smithi163 bash[75425]: cluster 2023-10-09T09:41:19.877504+0000 osd.5 (osd.5) 770 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:21.106 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:20 smithi163 bash[75425]: cluster 2023-10-09T09:41:19.880042+0000 osd.6 (osd.6) 480 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:21.106 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:20 smithi163 bash[75425]: cluster 2023-10-09T09:41:19.883244+0000 osd.6 (osd.6) 481 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:21.106 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:20 smithi163 bash[75425]: cluster 2023-10-09T09:41:19.885993+0000 osd.6 (osd.6) 482 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:21.106 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:20 smithi163 bash[75425]: cluster 2023-10-09T09:41:20.885013+0000 mon.a (mon.0) 2040 : cluster 0 osdmap e1194: 8 total, 8 up, 6 in 2023-10-09T09:41:21.106 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:20 smithi163 bash[78107]: cluster 2023-10-09T09:41:19.877504+0000 osd.5 (osd.5) 770 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:21.106 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:20 smithi163 bash[78107]: cluster 2023-10-09T09:41:19.880042+0000 osd.6 (osd.6) 480 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:21.106 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:20 smithi163 bash[78107]: cluster 2023-10-09T09:41:19.883244+0000 osd.6 (osd.6) 481 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:21.106 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:20 smithi163 bash[78107]: cluster 2023-10-09T09:41:19.885993+0000 osd.6 (osd.6) 482 : cluster 3 failed to encode map e1193 with expected crc 2023-10-09T09:41:21.106 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:20 smithi163 bash[78107]: cluster 2023-10-09T09:41:20.885013+0000 mon.a (mon.0) 2040 : cluster 0 osdmap e1194: 8 total, 8 up, 6 in 2023-10-09T09:41:21.130 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:21.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:21.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:21.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:21.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:21.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:21.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:21.371 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:21.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:21.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:21.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:21.506 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:21.530 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:21.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:21.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:21.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:21.651 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:41:21.657 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:21.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:21.760 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:21.787 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:21.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:21.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:21.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:21.913 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:21.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:21.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:22.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:22.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:21 smithi163 bash[75425]: cluster 2023-10-09T09:41:19.977575+0000 mgr.y (mgr.34107) 1057 : cluster 0 pgmap v1849: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.2 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s; 6028/59430 objects degraded (10.143%); 598/59430 objects misplaced (1.006%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T09:41:22.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:21 smithi163 bash[75425]: cluster 2023-10-09T09:41:20.882276+0000 osd.6 (osd.6) 483 : cluster 3 failed to encode map e1194 with expected crc 2023-10-09T09:41:22.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:21 smithi163 bash[75425]: cluster 2023-10-09T09:41:20.882309+0000 osd.5 (osd.5) 771 : cluster 3 failed to encode map e1194 with expected crc 2023-10-09T09:41:22.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:21 smithi163 bash[78107]: cluster 2023-10-09T09:41:19.977575+0000 mgr.y (mgr.34107) 1057 : cluster 0 pgmap v1849: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.2 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s; 6028/59430 objects degraded (10.143%); 598/59430 objects misplaced (1.006%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T09:41:22.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:21 smithi163 bash[78107]: cluster 2023-10-09T09:41:20.882276+0000 osd.6 (osd.6) 483 : cluster 3 failed to encode map e1194 with expected crc 2023-10-09T09:41:22.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:21 smithi163 bash[78107]: cluster 2023-10-09T09:41:20.882309+0000 osd.5 (osd.5) 771 : cluster 3 failed to encode map e1194 with expected crc 2023-10-09T09:41:22.056 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 24461 24445 89.8628 86.8125 0.00900208 0.0111186 2023-10-09T09:41:22.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:22.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:22.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:21 smithi181 bash[59345]: cluster 2023-10-09T09:41:19.977575+0000 mgr.y (mgr.34107) 1057 : cluster 0 pgmap v1849: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.2 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s; 6028/59430 objects degraded (10.143%); 598/59430 objects misplaced (1.006%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T09:41:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:21 smithi181 bash[59345]: cluster 2023-10-09T09:41:20.882276+0000 osd.6 (osd.6) 483 : cluster 3 failed to encode map e1194 with expected crc 2023-10-09T09:41:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:21 smithi181 bash[59345]: cluster 2023-10-09T09:41:20.882309+0000 osd.5 (osd.5) 771 : cluster 3 failed to encode map e1194 with expected crc 2023-10-09T09:41:22.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:22.187 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:22.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:22.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:22.290 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:22.319 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:22.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:22.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:22.422 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:22.450 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:22.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:22.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:22.554 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:22.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:22.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:22.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:22.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:22.712 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:22.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:22.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:22.816 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:22.843 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:22.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:22.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:22.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:23.059 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:41:23.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 16 25963 25947 90.0853 93.875 0.0038424 0.0110925 2023-10-09T09:41:23.094 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:23.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:23.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:23.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:22 smithi163 bash[75425]: cluster 2023-10-09T09:41:21.978360+0000 mgr.y (mgr.34107) 1058 : cluster 0 pgmap v1851: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 86 MiB/s wr, 1.37k op/s; 5279/66471 objects degraded (7.942%); 1031/66471 objects misplaced (1.551%); 13 MiB/s, 204 objects/s recovering 2023-10-09T09:41:23.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:22 smithi163 bash[75425]: cluster 2023-10-09T09:41:22.934019+0000 mon.a (mon.0) 2041 : cluster 3 Health check update: Degraded data redundancy: 5279/66471 objects degraded (7.942%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:23.229 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:22 smithi163 bash[78107]: cluster 2023-10-09T09:41:21.978360+0000 mgr.y (mgr.34107) 1058 : cluster 0 pgmap v1851: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 86 MiB/s wr, 1.37k op/s; 5279/66471 objects degraded (7.942%); 1031/66471 objects misplaced (1.551%); 13 MiB/s, 204 objects/s recovering 2023-10-09T09:41:23.229 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:22 smithi163 bash[78107]: cluster 2023-10-09T09:41:22.934019+0000 mon.a (mon.0) 2041 : cluster 3 Health check update: Degraded data redundancy: 5279/66471 objects degraded (7.942%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:23.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:22 smithi181 bash[59345]: cluster 2023-10-09T09:41:21.978360+0000 mgr.y (mgr.34107) 1058 : cluster 0 pgmap v1851: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 86 MiB/s wr, 1.37k op/s; 5279/66471 objects degraded (7.942%); 1031/66471 objects misplaced (1.551%); 13 MiB/s, 204 objects/s recovering 2023-10-09T09:41:23.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:22 smithi181 bash[59345]: cluster 2023-10-09T09:41:22.934019+0000 mon.a (mon.0) 2041 : cluster 3 Health check update: Degraded data redundancy: 5279/66471 objects degraded (7.942%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:23.329 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:23.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:23.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:23.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:23.490 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:23.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:23.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:23.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:23.625 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:23.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:23.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:41:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:41:23] "GET /metrics HTTP/1.1" 200 36325 "" "Prometheus/2.33.4" 2023-10-09T09:41:23.869 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:23.877 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:41:23.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:23.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:23.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:23.929 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:41:23.965 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:41:23.994 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:23 smithi181 bash[59345]: audit 2023-10-09T09:41:23.563064+0000 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.163:0/1940123787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:41:24.003 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:41:24.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:24.033 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:24.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:24.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:24.040 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:41:24.056 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 27096 27080 89.0706 70.8125 0.00539708 0.0112198 2023-10-09T09:41:24.081 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:41:24.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:41:24.138 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:41:24.139 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:24.140 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:41:24.140 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:41:24.140 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:41:24.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:41:24.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:41:24.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:41:24.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:41:24.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:41:24.149 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:23 smithi163 bash[75425]: audit 2023-10-09T09:41:23.563064+0000 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.163:0/1940123787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:41:24.150 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:23 smithi163 bash[78107]: audit 2023-10-09T09:41:23.563064+0000 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.163:0/1940123787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:41:24.171 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:24.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:24.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:24.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:24.312 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:24.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:24.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:24.440 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:24.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:24.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:24.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:24.567 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:24.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:24.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:24.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:24.697 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:24.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:24.802 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:24.829 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:24.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:24.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:24.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:24.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:24.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:24.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:25.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:41:25.057001+0000 min lat: 0.00132859 max lat: 2.09983 avg lat: 0.0111523 2023-10-09T09:41:25.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:41:25.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 28695 28679 89.6135 99.9375 0.00747508 0.0111523 2023-10-09T09:41:25.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:25.096 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:25.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:25.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:25.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:25.224 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:25.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:25.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:24 smithi181 bash[59345]: cluster 2023-10-09T09:41:23.978862+0000 mgr.y (mgr.34107) 1059 : cluster 0 pgmap v1852: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 61 MiB/s wr, 980 op/s; 5279/70362 objects degraded (7.503%); 705/70362 objects misplaced (1.002%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T09:41:25.328 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:25.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:24 smithi163 bash[75425]: cluster 2023-10-09T09:41:23.978862+0000 mgr.y (mgr.34107) 1059 : cluster 0 pgmap v1852: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 61 MiB/s wr, 980 op/s; 5279/70362 objects degraded (7.503%); 705/70362 objects misplaced (1.002%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T09:41:25.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:25 smithi163 bash[78107]: cluster 2023-10-09T09:41:23.978862+0000 mgr.y (mgr.34107) 1059 : cluster 0 pgmap v1852: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 61 MiB/s wr, 980 op/s; 5279/70362 objects degraded (7.503%); 705/70362 objects misplaced (1.002%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T09:41:25.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:25.482 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:25.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:25.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:25.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:25.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:25.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:25.760 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:25.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:25.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:25.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:25.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:25.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:25.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:25.994 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:26.013 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:26.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:26.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:26.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 30019 30003 89.2859 82.75 0.0029677 0.0111426 2023-10-09T09:41:26.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:26.145 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:26.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:26.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:26.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:26.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:26.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:26.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:26.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:26.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:26.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:26 smithi181 bash[59345]: audit 2023-10-09T09:41:25.391111+0000 mon.a (mon.0) 2042 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:26 smithi181 bash[59345]: audit 2023-10-09T09:41:25.392944+0000 mon.a (mon.0) 2043 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:26 smithi181 bash[59345]: audit 2023-10-09T09:41:26.011459+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:26.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:26.599 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:26.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:26.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:26.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:26 smithi163 bash[75425]: audit 2023-10-09T09:41:25.391111+0000 mon.a (mon.0) 2042 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:26.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:26 smithi163 bash[75425]: audit 2023-10-09T09:41:25.392944+0000 mon.a (mon.0) 2043 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:26.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:26 smithi163 bash[75425]: audit 2023-10-09T09:41:26.011459+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:26.605 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:26 smithi163 bash[78107]: audit 2023-10-09T09:41:25.391111+0000 mon.a (mon.0) 2042 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:26.605 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:26 smithi163 bash[78107]: audit 2023-10-09T09:41:25.392944+0000 mon.a (mon.0) 2043 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:26.605 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:26 smithi163 bash[78107]: audit 2023-10-09T09:41:26.011459+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:26.705 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:26.732 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:26.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:26.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:26.837 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:26.863 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:26.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:26.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:26.966 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:26.996 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:26.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:26.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:27.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 31569 31553 89.6305 96.875 0.0111966 0.0111503 2023-10-09T09:41:27.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:27.126 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:27.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:27.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:27.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:27.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:27.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:27.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:27.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:27.393 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:41:27.393 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T09:41:27.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T09:41:27.393 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:41:27.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:157661 7.0 7:03490724:::benchmark_data_smithi163_130508_object48998:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1119)", 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:10.628452+0000", 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 496.75991304000001, 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18390770000000001, 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 157661 2023-10-09T09:41:27.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628452+0000", 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628452+0000", 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628450+0000", 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2649e-05 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628462+0000", 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1912e-05 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628484+0000", 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8519999999999999e-06 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628494+0000", 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4939999999999998e-05 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628559+0000", 2023-10-09T09:41:27.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124454 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628684+0000", 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4221000000000006e-05 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.628768+0000", 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025072129999999999 2023-10-09T09:41:27.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.631275+0000", 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000171702 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.631447+0000", 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.180681112 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.812128+0000", 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012908799999999999 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.812257+0000", 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103148 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:10.812360+0000", 2023-10-09T09:41:27.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18390770000000001 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:38773 8.d 8:bd34a6dd:::benchmark_data_smithi163_151167_object38772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:47.700755+0000", 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 339.68761050900002, 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.221105761, 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.401 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38773 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.700755+0000", 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.700755+0000", 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.700754+0000", 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0431e-05 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.700794+0000", 2023-10-09T09:41:27.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.276e-06 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.700803+0000", 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3220000000000003e-06 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.700807+0000", 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7221999999999997e-05 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.700844+0000", 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4500000000000003e-06 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.700850+0000", 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.003426368 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:48.704276+0000", 2023-10-09T09:41:27.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9482000000000001e-05 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:48.704295+0000", 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.194313234 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.898609+0000", 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000140333 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.898749+0000", 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013512600000000001 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 6,7", 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.898884+0000", 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011035727 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.909920+0000", 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0045330839999999997 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.914453+0000", 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112027 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.914565+0000", 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5166999999999995e-05 2023-10-09T09:41:27.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.914630+0000", 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0072304279999999997 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.921861+0000", 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.221105761 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.409 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:38784 8.d 8:be0d51b1:::benchmark_data_smithi163_151167_object38783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:47.709235+0000", 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 339.67913001900001, 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2148319810000001, 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.410 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38784 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709235+0000", 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709235+0000", 2023-10-09T09:41:27.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709234+0000", 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0179999999999995e-05 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709315+0000", 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0852e-05 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709325+0000", 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4670000000000001e-06 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709332+0000", 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5258000000000002e-05 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709427+0000", 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2449999999999999e-05 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709440+0000", 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99488975199999996 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:48.704329+0000", 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8190000000000003e-06 2023-10-09T09:41:27.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:48.704339+0000", 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.196243889 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.900583+0000", 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010204600000000001 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.900685+0000", 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000145198 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 6,7", 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.900830+0000", 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0090993340000000006 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.909930+0000", 2023-10-09T09:41:27.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0045614660000000001 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.914491+0000", 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0074701699999999999 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.921961+0000", 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2954999999999996e-05 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.922024+0000", 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020430689999999998 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.924067+0000", 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2148319810000001 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.418 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.419 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:38785 8.4 8:2ec8ac35:::benchmark_data_smithi163_151167_object38784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:41:27.419 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:35:47.709346+0000", 2023-10-09T09:41:27.419 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 339.67901921999999, 2023-10-09T09:41:27.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2195565159999999, 2023-10-09T09:41:27.419 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.419 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.419 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38785 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709346+0000", 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709346+0000", 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709345+0000", 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8742999999999997e-05 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709394+0000", 2023-10-09T09:41:27.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7549999999999999e-05 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709422+0000", 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5540000000000003e-06 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709428+0000", 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2021000000000005e-05 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709520+0000", 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1867e-05 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:47.709532+0000", 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99477007799999995 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:48.704302+0000", 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1663e-05 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:48.704314+0000", 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.204362028 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.908676+0000", 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000126427 2023-10-09T09:41:27.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.908802+0000", 2023-10-09T09:41:27.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017271100000000001 2023-10-09T09:41:27.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 4,6", 2023-10-09T09:41:27.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.908975+0000", 2023-10-09T09:41:27.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014099527000000001 2023-10-09T09:41:27.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.923075+0000", 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4170000000000004e-05 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.923139+0000", 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036871450000000002 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.926826+0000", 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010327000000000001 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.926929+0000", 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019734940000000001 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:35:49.928903+0000", 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2195565159999999 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:83831 8.0 8:0b913c1d:::benchmark_data_smithi163_151167_object83830:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:40.156381+0000", 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 287.23198430000002, 2023-10-09T09:41:27.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24537260799999999, 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 83831 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.156381+0000", 2023-10-09T09:41:27.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.156381+0000", 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.156379+0000", 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1492999999999997e-05 2023-10-09T09:41:27.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.156421+0000", 2023-10-09T09:41:27.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5602000000000001e-05 2023-10-09T09:41:27.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.156436+0000", 2023-10-09T09:41:27.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8229999999999997e-06 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.156443+0000", 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8411000000000001e-05 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.156491+0000", 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112531 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.156604+0000", 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020454800000000001 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.156808+0000", 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029822970000000001 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.159791+0000", 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003675317 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.163466+0000", 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23815233899999999 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.401618+0000", 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2313999999999998e-05 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.401701+0000", 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2995999999999999e-05 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.401754+0000", 2023-10-09T09:41:27.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24537260799999999 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:83838 8.0 8:0b8e0bc7:::benchmark_data_smithi163_151167_object83837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:40.164166+0000", 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 287.22419900199998, 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23792817099999999, 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.436 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 83838 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.164166+0000", 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.164166+0000", 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.164164+0000", 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7417999999999995e-05 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.164262+0000", 2023-10-09T09:41:27.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3020000000000001e-05 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.164275+0000", 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8310000000000003e-06 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.164282+0000", 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00523983 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.169522+0000", 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003695771 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.173217+0000", 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012517600000000001 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:41:27.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.173343+0000", 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.006949635 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.180292+0000", 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003684639 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.183977+0000", 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21788396500000001 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.401861+0000", 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118338 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.401979+0000", 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115433 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.402095+0000", 2023-10-09T09:41:27.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23792817099999999 2023-10-09T09:41:27.443 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.443 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.443 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.443 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:83850 8.0 8:0a56e718:::benchmark_data_smithi163_151167_object83849:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1148)", 2023-10-09T09:41:27.444 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:40.179842+0000", 2023-10-09T09:41:27.444 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 287.20852353100003, 2023-10-09T09:41:27.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22238054600000001, 2023-10-09T09:41:27.444 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.444 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.444 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.444 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.444 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.444 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 83850 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.179842+0000", 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.179842+0000", 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.179840+0000", 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5163999999999997e-05 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.179875+0000", 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5197e-05 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.179890+0000", 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6539999999999999e-06 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.179898+0000", 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000423934 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.180321+0000", 2023-10-09T09:41:27.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000143039 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.180465+0000", 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016809999999999999 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.180633+0000", 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034603440000000002 2023-10-09T09:41:27.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.184093+0000", 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002901617 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.186995+0000", 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21514672800000001 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.402141+0000", 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6100999999999999e-05 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.402177+0000", 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4960000000000003e-05 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:40.402222+0000", 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22238054600000001 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:85427 8.d 8:b3d9446c:::benchmark_data_smithi163_151167_object85426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1149)", 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:42.410618+0000", 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 284.977747862, 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1847084190000001, 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.451 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 85427 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.410618+0000", 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.410618+0000", 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.410616+0000", 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4461000000000001e-05 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.410650+0000", 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2631e-05 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.410663+0000", 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2529999999999998e-06 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.410670+0000", 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003248435 2023-10-09T09:41:27.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413919+0000", 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0319999999999999e-05 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413929+0000", 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.166435661 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.580365+0000", 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013949200000000001 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.580504+0000", 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000216424 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.580721+0000", 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012640508 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.593361+0000", 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3617e-05 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.593375+0000", 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6994000000000003e-05 2023-10-09T09:41:27.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.593442+0000", 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2132999999999997e-05 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.593494+0000", 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018320610000000001 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.595326+0000", 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1847084190000001 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:85428 8.4 8:26a097db:::benchmark_data_smithi163_151167_object85427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1149)", 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:42.413733+0000", 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 284.97463199600003, 2023-10-09T09:41:27.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2067080210000001, 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 85428 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413733+0000", 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413733+0000", 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413732+0000", 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4567000000000001e-05 2023-10-09T09:41:27.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413787+0000", 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3944000000000001e-05 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413801+0000", 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5469999999999999e-06 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413808+0000", 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5587999999999998e-05 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413854+0000", 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6170000000000001e-06 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413862+0000", 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.170912052 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.584774+0000", 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000130603 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.584904+0000", 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017824799999999999 2023-10-09T09:41:27.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.585083+0000", 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010294875 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.595377+0000", 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0049295149999999998 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.600307+0000", 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017862908 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.618170+0000", 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3088000000000001e-05 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.618253+0000", 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021885089999999999 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.620441+0000", 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2067080210000001 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:85429 8.d 8:b3d4db3d:::benchmark_data_smithi163_151167_object85428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1149)", 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:42.413822+0000", 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 284.97454322999999, 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1991830029999999, 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 85429 2023-10-09T09:41:27.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413822+0000", 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413822+0000", 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413821+0000", 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9002999999999999e-05 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413860+0000", 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3449999999999995e-06 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413870+0000", 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6550000000000001e-06 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.413875+0000", 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013012500000000001 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.414005+0000", 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7310000000000005e-06 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.414015+0000", 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1699432299999999 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.583958+0000", 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021144499999999999 2023-10-09T09:41:27.472 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.584170+0000", 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024326299999999999 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.584413+0000", 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089699189999999998 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.593383+0000", 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029389049999999999 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.596322+0000", 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014135194 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.474 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.610457+0000", 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5506999999999994e-05 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.610533+0000", 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024725609999999999 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.613005+0000", 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1991830029999999 2023-10-09T09:41:27.475 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:85433 8.d 8:ba8b4588:::benchmark_data_smithi163_151167_object85432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1149)", 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:42.420299+0000", 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 284.96806627500001, 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1951281140000001, 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.476 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 85433 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.420299+0000", 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.477 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.420299+0000", 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.420297+0000", 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5433999999999999e-05 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.478 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.420343+0000", 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4949999999999999e-05 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.420358+0000", 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3390000000000004e-06 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.420365+0000", 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6708000000000001e-05 2023-10-09T09:41:27.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.420412+0000", 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2740000000000007e-06 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.420420+0000", 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1673712620000001 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.480 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.587791+0000", 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000161464 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.587953+0000", 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020857199999999999 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.588161+0000", 2023-10-09T09:41:27.481 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0083721100000000003 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.596534+0000", 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012995500000000001 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.596663+0000", 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016441832999999999 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.482 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.613105+0000", 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4569999999999998e-05 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.613190+0000", 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022373100000000002 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.483 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.615427+0000", 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1951281140000001 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:85436 8.4 8:214c10ca:::benchmark_data_smithi163_151167_object85435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1149)", 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:42.426905+0000", 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 284.96146085599997, 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1951563890000001, 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.484 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 85436 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.426905+0000", 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.485 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.426905+0000", 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.426904+0000", 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3779999999999998e-05 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.486 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.426947+0000", 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9829999999999996e-06 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.426955+0000", 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9180000000000002e-06 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.487 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.426960+0000", 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9076000000000002e-05 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.427009+0000", 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1910000000000006e-06 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.427019+0000", 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.161269973 2023-10-09T09:41:27.488 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.588289+0000", 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016831100000000001 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.588457+0000", 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019033400000000001 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.489 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:41:27.490 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.588647+0000", 2023-10-09T09:41:27.490 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011599293 2023-10-09T09:41:27.490 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.490 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.490 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.490 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.600246+0000", 2023-10-09T09:41:27.490 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013218999999999999 2023-10-09T09:41:27.490 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.490 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.490 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.600379+0000", 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.020164646000000001 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.620543+0000", 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3820999999999996e-05 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.491 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.620627+0000", 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014337659999999999 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.622061+0000", 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1951563890000001 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.492 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:85440 8.4 8:28e2d7ff:::benchmark_data_smithi163_151167_object85439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1149)", 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:42.440118+0000", 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 284.948247224, 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1836288429999999, 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 85440 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.493 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.440118+0000", 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.440118+0000", 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.494 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.440117+0000", 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1989000000000006e-05 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.440179+0000", 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5503999999999999e-05 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.440194+0000", 2023-10-09T09:41:27.495 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0026e-05 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.440204+0000", 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027494529999999998 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.442954+0000", 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1939e-05 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.496 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:42.442965+0000", 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1488326899999999 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.591798+0000", 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000158391 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.497 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.591957+0000", 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021556900000000001 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.592172+0000", 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082218810000000003 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.600394+0000", 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7203000000000002e-05 2023-10-09T09:41:27.498 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.600441+0000", 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022100949000000002 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.622542+0000", 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3622000000000007e-05 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.499 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.622626+0000", 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011211960000000001 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:43.623747+0000", 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1836288429999999 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.500 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:93671 8.0 8:093e9202:::benchmark_data_smithi163_151167_object93670:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1152)", 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:36:55.612975+0000", 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 271.77539083900001, 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17748765499999999, 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93671 2023-10-09T09:41:27.501 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.612975+0000", 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.612975+0000", 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:41:27.502 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.612973+0000", 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8301999999999999e-05 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.613021+0000", 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6402999999999999e-05 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.503 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.613038+0000", 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8169999999999994e-06 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.613048+0000", 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4265999999999998e-05 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.613102+0000", 2023-10-09T09:41:27.504 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014170799999999999 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.613244+0000", 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015002200000000001 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.613394+0000", 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029396560000000001 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.505 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.616333+0000", 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023755799999999999 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.616571+0000", 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.173792107 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.506 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.790363+0000", 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8501999999999997e-05 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.790411+0000", 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0797000000000002e-05 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.507 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:36:55.790462+0000", 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17748765499999999 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:116604 8.4 8:23386f06:::benchmark_data_smithi163_151167_object21371:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1153)", 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:37:28.629539+0000", 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 238.758826271, 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17966183199999999, 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.508 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116604 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.629539+0000", 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.509 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.629539+0000", 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.629537+0000", 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1015e-05 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.510 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.629548+0000", 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2388999999999999e-05 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.629571+0000", 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0363000000000001e-05 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.629581+0000", 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8360000000000003e-05 2023-10-09T09:41:27.511 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.629649+0000", 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015951600000000001 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.629809+0000", 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036710250000000001 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.512 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.633480+0000", 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0092232249999999998 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.642703+0000", 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6793999999999998e-05 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.642770+0000", 2023-10-09T09:41:27.513 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16353502 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.806305+0000", 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9938000000000004e-05 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.806405+0000", 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027959999999999999 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.514 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.809201+0000", 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17966183199999999 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:116607 8.d 8:b73e4320:::benchmark_data_smithi163_151167_object21374:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1153)", 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:37:28.632517+0000", 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 238.75584832800001, 2023-10-09T09:41:27.515 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20469971100000001, 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116607 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.632517+0000", 2023-10-09T09:41:27.516 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.632517+0000", 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.632515+0000", 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3449999999999995e-06 2023-10-09T09:41:27.517 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.632525+0000", 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4568e-05 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.632539+0000", 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.106e-06 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.518 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.632547+0000", 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0640999999999998e-05 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.632608+0000", 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013177800000000001 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.632740+0000", 2023-10-09T09:41:27.519 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014503399999999999 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.632885+0000", 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089747669999999998 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.641860+0000", 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000440498 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.520 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.642300+0000", 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18787738400000001 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.830177+0000", 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0562000000000006e-05 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.521 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.830258+0000", 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0069587069999999997 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.837217+0000", 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20469971100000001 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.522 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:116610 8.4 8:2ac02ceb:::benchmark_data_smithi163_151167_object21377:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1153)", 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:37:28.638153+0000", 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 238.75021235899999, 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18146253300000001, 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116610 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.523 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.638153+0000", 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.638153+0000", 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.524 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.638151+0000", 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8609999999999993e-06 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.638161+0000", 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6249e-05 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.525 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.638177+0000", 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5220000000000007e-06 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.638186+0000", 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2442999999999998e-05 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.638248+0000", 2023-10-09T09:41:27.526 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015547400000000001 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.638404+0000", 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016025100000000001 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,4", 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.638564+0000", 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043322380000000004 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.527 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.642896+0000", 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027214209999999999 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.645617+0000", 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.171622108 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.817240+0000", 2023-10-09T09:41:27.528 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106292 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.817346+0000", 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002269699 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.819616+0000", 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18146253300000001 2023-10-09T09:41:27.529 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.530 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.530 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.530 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.530 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.530 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:116622 8.d 8:b373e327:::benchmark_data_smithi163_151167_object21389:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1153)", 2023-10-09T09:41:27.530 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:37:28.656608+0000", 2023-10-09T09:41:27.530 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 238.731757652, 2023-10-09T09:41:27.530 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.192075254, 2023-10-09T09:41:27.530 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.530 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116622 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.656608+0000", 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.531 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.656608+0000", 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.656606+0000", 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0289999999999994e-06 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.532 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.656615+0000", 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5866e-05 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.656631+0000", 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7829999999999999e-06 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.656638+0000", 2023-10-09T09:41:27.533 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5154000000000007e-05 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.656704+0000", 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017100700000000001 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.656875+0000", 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021274600000000001 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.534 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.657087+0000", 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0047734140000000001 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.661861+0000", 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027846749999999999 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.664645+0000", 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.181008692 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.845654+0000", 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013557599999999999 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.845790+0000", 2023-10-09T09:41:27.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028933349999999999 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.848683+0000", 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.192075254 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:116632 8.d 8:b08a4b39:::benchmark_data_smithi163_151167_object21399:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1153)", 2023-10-09T09:41:27.537 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:37:28.684995+0000", 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 238.70337062499999, 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16636761999999999, 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116632 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.538 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.539 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.684995+0000", 2023-10-09T09:41:27.539 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.539 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.539 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.539 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.539 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.684995+0000", 2023-10-09T09:41:27.539 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:41:27.539 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.539 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.539 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.684993+0000", 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4479999999999997e-06 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.685000+0000", 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2162e-05 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.685013+0000", 2023-10-09T09:41:27.540 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3589999999999999e-06 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.685018+0000", 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013117099999999999 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.685149+0000", 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024676199999999998 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.541 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.685396+0000", 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00025650400000000001 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.685652+0000", 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0051487089999999996 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.542 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.690801+0000", 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7928000000000003e-05 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.690879+0000", 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15787626699999999 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.848755+0000", 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6816000000000005e-05 2023-10-09T09:41:27.543 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.848822+0000", 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025402049999999998 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:37:28.851362+0000", 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16636761999999999 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.544 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:29981 9.f 9:fa336c95:::benchmark_data_smithi163_175771_object29980:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1194)", 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:41:25.943826+0000", 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 1.4445391779999999, 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13575198499999999, 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:41:27.545 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 29981 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.943826+0000", 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.943826+0000", 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:41:27.546 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.547 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.547 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:27.547 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.943826+0000", 2023-10-09T09:41:27.547 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4026000000000003e-05 2023-10-09T09:41:27.547 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.547 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.547 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:27.547 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.943860+0000", 2023-10-09T09:41:27.548 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7649999999999993e-06 2023-10-09T09:41:27.548 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.548 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.548 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:27.548 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.943867+0000", 2023-10-09T09:41:27.548 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6110000000000004e-06 2023-10-09T09:41:27.548 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.548 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.548 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:27.548 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.943872+0000", 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015799270000000001 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.945452+0000", 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000171182 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.945623+0000", 2023-10-09T09:41:27.549 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00037605599999999999 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.945999+0000", 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021476849999999999 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.948147+0000", 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0340999999999999e-05 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:25.948187+0000", 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.130625455 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:26.078813+0000", 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2020000000000001e-05 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:27.551 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:26.078855+0000", 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00072359900000000001 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:26.079578+0000", 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13575198499999999 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:27.552 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:27.553 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:41:27.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:27.554 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:27.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:27.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:27.692 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:27.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:27.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:27.796 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:27.824 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:27.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:27.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:27.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:27.935 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:27 smithi181 bash[59345]: cluster 2023-10-09T09:41:25.979462+0000 mgr.y (mgr.34107) 1060 : cluster 0 pgmap v1853: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s; 5279/78015 objects degraded (6.767%); 705/78015 objects misplaced (0.904%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T09:41:27.954 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:27.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:27.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:27.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:27 smithi163 bash[78107]: cluster 2023-10-09T09:41:25.979462+0000 mgr.y (mgr.34107) 1060 : cluster 0 pgmap v1853: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s; 5279/78015 objects degraded (6.767%); 705/78015 objects misplaced (0.904%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T09:41:27.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:27 smithi163 bash[75425]: cluster 2023-10-09T09:41:25.979462+0000 mgr.y (mgr.34107) 1060 : cluster 0 pgmap v1853: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s; 5279/78015 objects degraded (6.767%); 705/78015 objects misplaced (0.904%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T09:41:28.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 32887 32871 89.3147 82.375 0.00909684 0.011189 2023-10-09T09:41:28.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:28.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:28.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:28.184 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:28.222 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:28.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:28.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:28.324 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:28.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:28.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:28.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:28.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:28.474 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:28.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:28.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:28.569 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T09:41:28.579 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:28.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:28.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:28.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:28.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:28.729 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:28.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:28.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:28.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:28.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:28.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:28.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:28.969 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:28.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:28 smithi163 bash[75425]: cluster 2023-10-09T09:41:28.759140+0000 mon.a (mon.0) 2045 : cluster 3 Health check update: Degraded data redundancy: 5279/90066 objects degraded (5.861%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:28.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:28 smithi163 bash[78107]: cluster 2023-10-09T09:41:28.759140+0000 mon.a (mon.0) 2045 : cluster 3 Health check update: Degraded data redundancy: 5279/90066 objects degraded (5.861%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:29.004 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:29.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:29.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:29.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 33954 33938 88.3716 66.6875 0.00500598 0.0113084 2023-10-09T09:41:29.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:28 smithi181 bash[59345]: cluster 2023-10-09T09:41:28.759140+0000 mon.a (mon.0) 2045 : cluster 3 Health check update: Degraded data redundancy: 5279/90066 objects degraded (5.861%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:29.110 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:29.139 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:29.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:29.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:29.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:29.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:29.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:29.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:29.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:29.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:29.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:29.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:29.535 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:29.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:29.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:29.641 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:29.670 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:29.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:29.774 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:29.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:29.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:29.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:29.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:29.948 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:29.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:29.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:29 smithi163 bash[75425]: cluster 2023-10-09T09:41:27.980167+0000 mgr.y (mgr.34107) 1061 : cluster 0 pgmap v1854: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 5279/90066 objects degraded (5.861%); 589/90066 objects misplaced (0.654%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T09:41:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:29 smithi163 bash[78107]: cluster 2023-10-09T09:41:27.980167+0000 mgr.y (mgr.34107) 1061 : cluster 0 pgmap v1854: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 5279/90066 objects degraded (5.861%); 589/90066 objects misplaced (0.654%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T09:41:30.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:30.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 35286 35270 88.1664 83.25 0.0118848 0.0113354 2023-10-09T09:41:30.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:29 smithi181 bash[59345]: cluster 2023-10-09T09:41:27.980167+0000 mgr.y (mgr.34107) 1061 : cluster 0 pgmap v1854: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 5279/90066 objects degraded (5.861%); 589/90066 objects misplaced (0.654%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T09:41:30.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:30.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:30.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:30.193 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:30.219 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:30.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:30.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:30.323 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:30.350 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:30.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:30.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:30.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:30.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:30.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:30.588 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:30.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:30.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:30.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:30.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:30.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:30.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:30.850 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:41:30.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:30.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:30.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:30.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:30.994 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:31.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:31.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:31.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:31.046 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:41:31.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 16 36654 36638 88.0635 85.5 0.013191 0.0113463 2023-10-09T09:41:31.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:30 smithi181 bash[59345]: audit 2023-10-09T09:41:30.233726+0000 mon.c (mon.2) 95 : audit 1 from='client.? 172.21.15.163:0/2014880966' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:41:31.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:30 smithi181 bash[59345]: audit 2023-10-09T09:41:30.234441+0000 mon.a (mon.0) 2046 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:41:31.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:31.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:30 smithi163 bash[75425]: audit 2023-10-09T09:41:30.233726+0000 mon.c (mon.2) 95 : audit 1 from='client.? 172.21.15.163:0/2014880966' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:41:31.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:30 smithi163 bash[75425]: audit 2023-10-09T09:41:30.234441+0000 mon.a (mon.0) 2046 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:41:31.137 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:30 smithi163 bash[78107]: audit 2023-10-09T09:41:30.233726+0000 mon.c (mon.2) 95 : audit 1 from='client.? 172.21.15.163:0/2014880966' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:41:31.137 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:30 smithi163 bash[78107]: audit 2023-10-09T09:41:30.234441+0000 mon.a (mon.0) 2046 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:41:31.166 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:31.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:31.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:31.307 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:31.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:31.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:31.411 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:31.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:31.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:31.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:31.554 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:31.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:31.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:31.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:31.700 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:31.814 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:31.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:31.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:31.920 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:31.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:31 smithi163 bash[75425]: cluster 2023-10-09T09:41:29.980912+0000 mgr.y (mgr.34107) 1062 : cluster 0 pgmap v1855: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s; 5279/96279 objects degraded (5.483%); 589/96279 objects misplaced (0.612%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T09:41:31.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:31 smithi163 bash[75425]: cluster 2023-10-09T09:41:30.818367+0000 mon.a (mon.0) 2047 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:41:31.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:31 smithi163 bash[75425]: cluster 2023-10-09T09:41:30.829928+0000 osd.5 (osd.5) 772 : cluster 3 failed to encode map e1195 with expected crc 2023-10-09T09:41:31.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:31 smithi163 bash[75425]: audit 2023-10-09T09:41:30.835096+0000 mon.a (mon.0) 2048 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:41:31.930 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:31 smithi163 bash[75425]: cluster 2023-10-09T09:41:30.835558+0000 mon.a (mon.0) 2049 : cluster 0 osdmap e1195: 8 total, 8 up, 6 in 2023-10-09T09:41:31.930 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:31 smithi163 bash[78107]: cluster 2023-10-09T09:41:29.980912+0000 mgr.y (mgr.34107) 1062 : cluster 0 pgmap v1855: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s; 5279/96279 objects degraded (5.483%); 589/96279 objects misplaced (0.612%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T09:41:31.930 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:31 smithi163 bash[78107]: cluster 2023-10-09T09:41:30.818367+0000 mon.a (mon.0) 2047 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:41:31.930 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:31 smithi163 bash[78107]: cluster 2023-10-09T09:41:30.829928+0000 osd.5 (osd.5) 772 : cluster 3 failed to encode map e1195 with expected crc 2023-10-09T09:41:31.930 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:31 smithi163 bash[78107]: audit 2023-10-09T09:41:30.835096+0000 mon.a (mon.0) 2048 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:41:31.930 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:31 smithi163 bash[78107]: cluster 2023-10-09T09:41:30.835558+0000 mon.a (mon.0) 2049 : cluster 0 osdmap e1195: 8 total, 8 up, 6 in 2023-10-09T09:41:31.952 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:31.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:31.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:32.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 37989 37973 87.8919 83.4375 0.0134227 0.0113705 2023-10-09T09:41:32.062 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:32.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:32.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:32.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:32.192 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:32.226 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:32.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:32.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:31 smithi181 bash[59345]: cluster 2023-10-09T09:41:29.980912+0000 mgr.y (mgr.34107) 1062 : cluster 0 pgmap v1855: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s; 5279/96279 objects degraded (5.483%); 589/96279 objects misplaced (0.612%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T09:41:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:31 smithi181 bash[59345]: cluster 2023-10-09T09:41:30.818367+0000 mon.a (mon.0) 2047 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:41:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:31 smithi181 bash[59345]: cluster 2023-10-09T09:41:30.829928+0000 osd.5 (osd.5) 772 : cluster 3 failed to encode map e1195 with expected crc 2023-10-09T09:41:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:31 smithi181 bash[59345]: audit 2023-10-09T09:41:30.835096+0000 mon.a (mon.0) 2048 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:41:32.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:31 smithi181 bash[59345]: cluster 2023-10-09T09:41:30.835558+0000 mon.a (mon.0) 2049 : cluster 0 osdmap e1195: 8 total, 8 up, 6 in 2023-10-09T09:41:32.329 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:32.354 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:32.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:32.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:32.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:32.487 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:32.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:32.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:32.591 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:32.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:32.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:32.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:32.725 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:32.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:32.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:32.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:32.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:32.896 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:32.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:32.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:33.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:33.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:33.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:33.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 39384 39368 87.8665 87.1875 0.00685527 0.0113751 2023-10-09T09:41:33.130 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:33.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:33.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:33.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:33.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:33.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:33.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:33.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:33.405 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:41:33.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:33.445 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:33.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:33.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:33.548 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:33.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:33.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:33.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:33.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:33.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:33.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:33.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:41:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:41:33] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.33.4" 2023-10-09T09:41:33.812 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:33.838 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:33.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:33.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:33.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:33.980 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:33.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:33.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:34.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 40543 40527 87.3342 72.4375 0.00674601 0.011443 2023-10-09T09:41:34.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:33 smithi181 bash[59345]: cluster 2023-10-09T09:41:31.981665+0000 mgr.y (mgr.34107) 1063 : cluster 0 pgmap v1857: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s; 5279/104829 objects degraded (5.036%); 473/104829 objects misplaced (0.451%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:41:34.086 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:34.095 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:33 smithi163 bash[78107]: cluster 2023-10-09T09:41:31.981665+0000 mgr.y (mgr.34107) 1063 : cluster 0 pgmap v1857: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s; 5279/104829 objects degraded (5.036%); 473/104829 objects misplaced (0.451%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:41:34.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:33 smithi163 bash[75425]: cluster 2023-10-09T09:41:31.981665+0000 mgr.y (mgr.34107) 1063 : cluster 0 pgmap v1857: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s; 5279/104829 objects degraded (5.036%); 473/104829 objects misplaced (0.451%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T09:41:34.114 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:34.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:34.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:34.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:34.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:34.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:34.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:34.351 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:34.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:34.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:34.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:34.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:34.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:34.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:34.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:34.662 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:34.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:34.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:34.766 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:34.784 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:34.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:34.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:34.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:34.896 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:34 smithi163 bash[75425]: cluster 2023-10-09T09:41:34.826967+0000 mon.a (mon.0) 2050 : cluster 3 Health check update: Degraded data redundancy: 5279/109671 objects degraded (4.813%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:34.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:34 smithi163 bash[78107]: cluster 2023-10-09T09:41:34.826967+0000 mon.a (mon.0) 2050 : cluster 3 Health check update: Degraded data redundancy: 5279/109671 objects degraded (4.813%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:34.918 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:34.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:34.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:35.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:35.053 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:35.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:35.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:35.057 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 41406 41390 86.2208 53.9375 0.022842 0.0115903 2023-10-09T09:41:35.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:34 smithi181 bash[59345]: cluster 2023-10-09T09:41:34.826967+0000 mon.a (mon.0) 2050 : cluster 3 Health check update: Degraded data redundancy: 5279/109671 objects degraded (4.813%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:35.157 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:35.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:35.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:35.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:41:35.242 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:41:35.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:35.315 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:35.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:35.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:35.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:35.459 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:35.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:35.563 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:35.590 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:35.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:35.695 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:35.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:35.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:35.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:35.827 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:35.855 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:35.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:35.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:35.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:35.967 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:35 smithi181 bash[59345]: cluster 2023-10-09T09:41:33.982298+0000 mgr.y (mgr.34107) 1064 : cluster 0 pgmap v1858: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s; 5279/109671 objects degraded (4.813%); 473/109671 objects misplaced (0.431%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:41:35.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:35.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:35.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:36.058 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 42678 42662 86.0038 79.5 0.015017 0.0116205 2023-10-09T09:41:36.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:41:36.086 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:36.160 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:36.164 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:35 smithi163 bash[78107]: cluster 2023-10-09T09:41:33.982298+0000 mgr.y (mgr.34107) 1064 : cluster 0 pgmap v1858: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s; 5279/109671 objects degraded (4.813%); 473/109671 objects misplaced (0.431%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:41:36.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:36.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:36.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:35 smithi163 bash[75425]: cluster 2023-10-09T09:41:33.982298+0000 mgr.y (mgr.34107) 1064 : cluster 0 pgmap v1858: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s; 5279/109671 objects degraded (4.813%); 473/109671 objects misplaced (0.431%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:41:36.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:36.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:36.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:36.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:36.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:36.443 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:36.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:36.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:36.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:36.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:36.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:36.715 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:36.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:36.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:36.820 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:36.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:36.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:36.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:36.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:36 smithi163 bash[75425]: audit 2023-10-09T09:41:35.211825+0000 mgr.y (mgr.34107) 1065 : audit 0 from='client.35436 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:41:36.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:36 smithi163 bash[78107]: audit 2023-10-09T09:41:35.211825+0000 mgr.y (mgr.34107) 1065 : audit 0 from='client.35436 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:41:37.005 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:37.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:37.019 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1858,"stamp":"2023-10-09T09:41:33.981883+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2394822560,"num_objects":36557,"num_object_clones":0,"num_object_copies":109671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5279,"num_objects_misplaced":473,"num_objects_unfound":0,"num_objects_dirty":36557,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":36793,"num_write_kb":2340557,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1827,"num_bytes_recovered":117841578,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":36727,"ondisk_log_size":36727,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8957676,"kb_used_data":6576636,"kb_used_omap":510,"kb_used_meta":2380481,"kb_avail":553513236,"statfs":{"total":575970213888,"available":566797553664,"internally_reserved":0,"allocated":6734475264,"data_stored":6729735518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":522608,"internal_metadata":2437613200},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,8,5,4,8],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":858718208,"num_objects":13103,"num_object_clones":0,"num_object_copies":39309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-232,"num_objects_unfound":0,"num_objects_dirty":13103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13103,"num_write_kb":838592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":15204352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003349"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1180,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.891928+0000","last_change":"2023-10-09T09:36:43.418759+0000","last_active":"2023-10-09T09:41:20.891928+0000","last_peered":"2023-10-09T09:41:20.891928+0000","last_clean":"2023-10-09T09:41:20.891928+0000","last_became_active":"2023-10-09T09:36:43.418371+0000","last_became_peered":"2023-10-09T09:36:43.418371+0000","last_unstale":"2023-10-09T09:41:20.891928+0000","last_undegraded":"2023-10-09T09:41:20.891928+0000","last_fullsized":"2023-10-09T09:41:20.891928+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":75,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.903100+0000","last_change":"2023-10-09T09:41:11.811997+0000","last_active":"2023-10-09T09:41:19.903100+0000","last_peered":"2023-10-09T09:41:19.903100+0000","last_clean":"2023-10-09T09:41:19.903100+0000","last_became_active":"2023-10-09T09:41:11.811300+0000","last_became_peered":"2023-10-09T09:41:11.811300+0000","last_unstale":"2023-10-09T09:41:19.903100+0000","last_undegraded":"2023-10-09T09:41:19.903100+0000","last_fullsized":"2023-10-09T09:41:19.903100+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.346300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1144,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.901561+0000","last_change":"2023-10-09T09:41:11.811870+0000","last_active":"2023-10-09T09:41:19.901561+0000","last_peered":"2023-10-09T09:41:19.901561+0000","last_clean":"2023-10-09T09:41:19.901561+0000","last_became_active":"2023-10-09T09:41:11.811570+0000","last_became_peered":"2023-10-09T09:41:11.811570+0000","last_unstale":"2023-10-09T09:41:19.901561+0000","last_undegraded":"2023-10-09T09:41:19.901561+0000","last_fullsized":"2023-10-09T09:41:19.901561+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:57.923729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1135,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887704+0000","last_change":"2023-10-09T09:41:11.800661+0000","last_active":"2023-10-09T09:41:20.887704+0000","last_peered":"2023-10-09T09:41:20.887704+0000","last_clean":"2023-10-09T09:41:20.887704+0000","last_became_active":"2023-10-09T09:41:11.800279+0000","last_became_peered":"2023-10-09T09:41:11.800279+0000","last_unstale":"2023-10-09T09:41:20.887704+0000","last_undegraded":"2023-10-09T09:41:20.887704+0000","last_fullsized":"2023-10-09T09:41:20.887704+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:14:19.131995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":669,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.886694+0000","last_change":"2023-10-09T09:41:11.801649+0000","last_active":"2023-10-09T09:41:20.886694+0000","last_peered":"2023-10-09T09:41:20.886694+0000","last_clean":"2023-10-09T09:41:20.886694+0000","last_became_active":"2023-10-09T09:41:11.798585+0000","last_became_peered":"2023-10-09T09:41:11.798585+0000","last_unstale":"2023-10-09T09:41:20.886694+0000","last_undegraded":"2023-10-09T09:41:20.886694+0000","last_fullsized":"2023-10-09T09:41:20.886694+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:11.450331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1144,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887426+0000","last_change":"2023-10-09T09:40:25.799705+0000","last_active":"2023-10-09T09:41:20.887426+0000","last_peered":"2023-10-09T09:41:20.887426+0000","last_clean":"2023-10-09T09:41:20.887426+0000","last_became_active":"2023-10-09T09:40:25.799341+0000","last_became_peered":"2023-10-09T09:40:25.799341+0000","last_unstale":"2023-10-09T09:41:20.887426+0000","last_undegraded":"2023-10-09T09:41:20.887426+0000","last_fullsized":"2023-10-09T09:41:20.887426+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1143,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887579+0000","last_change":"2023-10-09T09:41:11.801830+0000","last_active":"2023-10-09T09:41:20.887579+0000","last_peered":"2023-10-09T09:41:20.887579+0000","last_clean":"2023-10-09T09:41:20.887579+0000","last_became_active":"2023-10-09T09:41:11.799923+0000","last_became_peered":"2023-10-09T09:41:11.799923+0000","last_unstale":"2023-10-09T09:41:20.887579+0000","last_undegraded":"2023-10-09T09:41:20.887579+0000","last_fullsized":"2023-10-09T09:41:20.887579+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:55.450110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":586,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.900653+0000","last_change":"2023-10-09T09:41:11.814521+0000","last_active":"2023-10-09T09:41:19.900653+0000","last_peered":"2023-10-09T09:41:19.900653+0000","last_clean":"2023-10-09T09:41:19.900653+0000","last_became_active":"2023-10-09T09:41:11.814148+0000","last_became_peered":"2023-10-09T09:41:11.814148+0000","last_unstale":"2023-10-09T09:41:19.900653+0000","last_undegraded":"2023-10-09T09:41:19.900653+0000","last_fullsized":"2023-10-09T09:41:19.900653+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:45.497593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":46,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.889088+0000","last_change":"2023-10-09T09:41:11.833334+0000","last_active":"2023-10-09T09:41:20.889088+0000","last_peered":"2023-10-09T09:41:20.889088+0000","last_clean":"2023-10-09T09:41:20.889088+0000","last_became_active":"2023-10-09T09:41:11.832509+0000","last_became_peered":"2023-10-09T09:41:11.832509+0000","last_unstale":"2023-10-09T09:41:20.889088+0000","last_undegraded":"2023-10-09T09:41:20.889088+0000","last_fullsized":"2023-10-09T09:41:20.889088+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":610,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.888203+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:41:20.888203+0000","last_peered":"2023-10-09T09:41:20.888203+0000","last_clean":"2023-10-09T09:41:20.888203+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:41:20.888203+0000","last_undegraded":"2023-10-09T09:41:20.888203+0000","last_fullsized":"2023-10-09T09:41:20.888203+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":34,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892281+0000","last_change":"2023-10-09T09:41:11.929105+0000","last_active":"2023-10-09T09:41:20.892281+0000","last_peered":"2023-10-09T09:41:20.892281+0000","last_clean":"2023-10-09T09:41:20.892281+0000","last_became_active":"2023-10-09T09:41:11.799443+0000","last_became_peered":"2023-10-09T09:41:11.799443+0000","last_unstale":"2023-10-09T09:41:20.892281+0000","last_undegraded":"2023-10-09T09:41:20.892281+0000","last_fullsized":"2023-10-09T09:41:20.892281+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":27,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.888447+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:41:20.888447+0000","last_peered":"2023-10-09T09:41:20.888447+0000","last_clean":"2023-10-09T09:41:20.888447+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:41:20.888447+0000","last_undegraded":"2023-10-09T09:41:20.888447+0000","last_fullsized":"2023-10-09T09:41:20.888447+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1172,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892334+0000","last_change":"2023-10-09T09:40:50.110970+0000","last_active":"2023-10-09T09:41:20.892334+0000","last_peered":"2023-10-09T09:41:20.892334+0000","last_clean":"2023-10-09T09:41:20.892334+0000","last_became_active":"2023-10-09T09:40:49.792764+0000","last_became_peered":"2023-10-09T09:40:49.792764+0000","last_unstale":"2023-10-09T09:41:20.892334+0000","last_undegraded":"2023-10-09T09:41:20.892334+0000","last_fullsized":"2023-10-09T09:41:20.892334+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1145,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.891986+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:41:20.891986+0000","last_peered":"2023-10-09T09:41:20.891986+0000","last_clean":"2023-10-09T09:41:20.891986+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:41:20.891986+0000","last_undegraded":"2023-10-09T09:41:20.891986+0000","last_fullsized":"2023-10-09T09:41:20.891986+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":41,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892059+0000","last_change":"2023-10-09T09:40:49.937490+0000","last_active":"2023-10-09T09:41:20.892059+0000","last_peered":"2023-10-09T09:41:20.892059+0000","last_clean":"2023-10-09T09:41:20.892059+0000","last_became_active":"2023-10-09T09:40:49.792044+0000","last_became_peered":"2023-10-09T09:40:49.792044+0000","last_unstale":"2023-10-09T09:41:20.892059+0000","last_undegraded":"2023-10-09T09:41:20.892059+0000","last_fullsized":"2023-10-09T09:41:20.892059+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":671,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.905543+0000","last_change":"2023-10-09T09:41:11.826293+0000","last_active":"2023-10-09T09:41:19.905543+0000","last_peered":"2023-10-09T09:41:19.905543+0000","last_clean":"2023-10-09T09:41:19.905543+0000","last_became_active":"2023-10-09T09:41:11.825808+0000","last_became_peered":"2023-10-09T09:41:11.825808+0000","last_unstale":"2023-10-09T09:41:19.905543+0000","last_undegraded":"2023-10-09T09:41:19.905543+0000","last_fullsized":"2023-10-09T09:41:19.905543+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:33:14.204593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1134,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.886935+0000","last_change":"2023-10-09T09:41:11.795047+0000","last_active":"2023-10-09T09:41:20.886935+0000","last_peered":"2023-10-09T09:41:20.886935+0000","last_clean":"2023-10-09T09:41:20.886935+0000","last_became_active":"2023-10-09T09:41:11.794662+0000","last_became_peered":"2023-10-09T09:41:11.794662+0000","last_unstale":"2023-10-09T09:41:20.886935+0000","last_undegraded":"2023-10-09T09:41:20.886935+0000","last_fullsized":"2023-10-09T09:41:20.886935+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:12.266267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":31,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887098+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:41:20.887098+0000","last_peered":"2023-10-09T09:41:20.887098+0000","last_clean":"2023-10-09T09:41:20.887098+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:41:20.887098+0000","last_undegraded":"2023-10-09T09:41:20.887098+0000","last_fullsized":"2023-10-09T09:41:20.887098+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":24,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.891935+0000","last_change":"2023-10-09T09:40:49.797133+0000","last_active":"2023-10-09T09:41:20.891935+0000","last_peered":"2023-10-09T09:41:20.891935+0000","last_clean":"2023-10-09T09:41:20.891935+0000","last_became_active":"2023-10-09T09:40:49.796773+0000","last_became_peered":"2023-10-09T09:40:49.796773+0000","last_unstale":"2023-10-09T09:41:20.891935+0000","last_undegraded":"2023-10-09T09:41:20.891935+0000","last_fullsized":"2023-10-09T09:41:20.891935+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1133,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887514+0000","last_change":"2023-10-09T09:41:11.791237+0000","last_active":"2023-10-09T09:41:20.887514+0000","last_peered":"2023-10-09T09:41:20.887514+0000","last_clean":"2023-10-09T09:41:20.887514+0000","last_became_active":"2023-10-09T09:41:11.790857+0000","last_became_peered":"2023-10-09T09:41:11.790857+0000","last_unstale":"2023-10-09T09:41:20.887514+0000","last_undegraded":"2023-10-09T09:41:20.887514+0000","last_fullsized":"2023-10-09T09:41:20.887514+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:17.466387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1189,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.886510+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:41:20.886510+0000","last_peered":"2023-10-09T09:41:20.886510+0000","last_clean":"2023-10-09T09:41:20.886510+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:41:20.886510+0000","last_undegraded":"2023-10-09T09:41:20.886510+0000","last_fullsized":"2023-10-09T09:41:20.886510+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1154,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.902711+0000","last_change":"2023-10-09T09:41:11.805335+0000","last_active":"2023-10-09T09:41:19.902711+0000","last_peered":"2023-10-09T09:41:19.902711+0000","last_clean":"2023-10-09T09:41:19.902711+0000","last_became_active":"2023-10-09T09:41:11.804963+0000","last_became_peered":"2023-10-09T09:41:11.804963+0000","last_unstale":"2023-10-09T09:41:19.902711+0000","last_undegraded":"2023-10-09T09:41:19.902711+0000","last_fullsized":"2023-10-09T09:41:19.902711+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:09.265362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"1195'2407","reported_seq":2441,"reported_epoch":1195,"state":"active+clean","last_fresh":"2023-10-09T09:41:32.297565+0000","last_change":"2023-10-09T09:40:49.793913+0000","last_active":"2023-10-09T09:41:32.297565+0000","last_peered":"2023-10-09T09:41:32.297565+0000","last_clean":"2023-10-09T09:41:32.297565+0000","last_became_active":"2023-10-09T09:40:49.793553+0000","last_became_peered":"2023-10-09T09:40:49.793553+0000","last_unstale":"2023-10-09T09:41:32.297565+0000","last_undegraded":"2023-10-09T09:41:32.297565+0000","last_fullsized":"2023-10-09T09:41:32.297565+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2407,"log_dups_size":0,"ondisk_log_size":2407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:08:21.437081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157745152,"num_objects":2407,"num_object_clones":0,"num_object_copies":7221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2407,"num_write_kb":154048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":39,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892710+0000","last_change":"2023-10-09T09:40:49.847626+0000","last_active":"2023-10-09T09:41:20.892710+0000","last_peered":"2023-10-09T09:41:20.892710+0000","last_clean":"2023-10-09T09:41:20.892710+0000","last_became_active":"2023-10-09T09:40:49.790030+0000","last_became_peered":"2023-10-09T09:40:49.790030+0000","last_unstale":"2023-10-09T09:41:20.892710+0000","last_undegraded":"2023-10-09T09:41:20.892710+0000","last_fullsized":"2023-10-09T09:41:20.892710+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":37,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.886813+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:41:20.886813+0000","last_peered":"2023-10-09T09:41:20.886813+0000","last_clean":"2023-10-09T09:41:20.886813+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:41:20.886813+0000","last_undegraded":"2023-10-09T09:41:20.886813+0000","last_fullsized":"2023-10-09T09:41:20.886813+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1194'2178","reported_seq":2215,"reported_epoch":1194,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:41:29.715028+0000","last_change":"2023-10-09T09:41:12.866117+0000","last_active":"2023-10-09T09:41:29.715028+0000","last_peered":"2023-10-09T09:41:29.715028+0000","last_clean":"2023-10-09T09:41:10.777223+0000","last_became_active":"2023-10-09T09:41:12.852460+0000","last_became_peered":"2023-10-09T09:41:12.852460+0000","last_unstale":"2023-10-09T09:41:29.715028+0000","last_undegraded":"2023-10-09T09:41:12.849103+0000","last_fullsized":"2023-10-09T09:41:12.796086+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2178,"log_dups_size":0,"ondisk_log_size":2178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:27.982907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142737408,"num_objects":2178,"num_object_clones":0,"num_object_copies":6534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":769,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2178,"num_write_kb":139392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":769},{"shards":"1,4","objects":1409}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":39,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892515+0000","last_change":"2023-10-09T09:40:50.158191+0000","last_active":"2023-10-09T09:41:20.892515+0000","last_peered":"2023-10-09T09:41:20.892515+0000","last_clean":"2023-10-09T09:41:20.892515+0000","last_became_active":"2023-10-09T09:40:49.797431+0000","last_became_peered":"2023-10-09T09:40:49.797431+0000","last_unstale":"2023-10-09T09:41:20.892515+0000","last_undegraded":"2023-10-09T09:41:20.892515+0000","last_fullsized":"2023-10-09T09:41:20.892515+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1169,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.891822+0000","last_change":"2023-10-09T09:41:11.797835+0000","last_active":"2023-10-09T09:41:20.891822+0000","last_peered":"2023-10-09T09:41:20.891822+0000","last_clean":"2023-10-09T09:41:20.891822+0000","last_became_active":"2023-10-09T09:41:11.797437+0000","last_became_peered":"2023-10-09T09:41:11.797437+0000","last_unstale":"2023-10-09T09:41:20.891822+0000","last_undegraded":"2023-10-09T09:41:20.891822+0000","last_fullsized":"2023-10-09T09:41:20.891822+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1195'2261","reported_seq":2290,"reported_epoch":1195,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:41:30.843301+0000","last_change":"2023-10-09T09:41:12.863852+0000","last_active":"2023-10-09T09:41:30.843301+0000","last_peered":"2023-10-09T09:41:30.843301+0000","last_clean":"2023-10-09T09:41:10.748257+0000","last_became_active":"2023-10-09T09:41:12.862147+0000","last_became_peered":"2023-10-09T09:41:12.862147+0000","last_unstale":"2023-10-09T09:41:30.843301+0000","last_undegraded":"2023-10-09T09:41:12.799855+0000","last_fullsized":"2023-10-09T09:41:12.795476+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2261,"log_dups_size":0,"ondisk_log_size":2261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:01.747681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148176896,"num_objects":2261,"num_object_clones":0,"num_object_copies":6783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":761,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2261,"num_write_kb":144704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":761},{"shards":"2,5","objects":1500}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1145,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.888847+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:41:20.888847+0000","last_peered":"2023-10-09T09:41:20.888847+0000","last_clean":"2023-10-09T09:41:20.888847+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:41:20.888847+0000","last_undegraded":"2023-10-09T09:41:20.888847+0000","last_fullsized":"2023-10-09T09:41:20.888847+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1136,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887314+0000","last_change":"2023-10-09T09:40:49.793789+0000","last_active":"2023-10-09T09:41:20.887314+0000","last_peered":"2023-10-09T09:41:20.887314+0000","last_clean":"2023-10-09T09:41:20.887314+0000","last_became_active":"2023-10-09T09:40:49.793374+0000","last_became_peered":"2023-10-09T09:40:49.793374+0000","last_unstale":"2023-10-09T09:41:20.887314+0000","last_undegraded":"2023-10-09T09:41:20.887314+0000","last_fullsized":"2023-10-09T09:41:20.887314+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1195'2297","reported_seq":2321,"reported_epoch":1195,"state":"active+clean","last_fresh":"2023-10-09T09:41:30.889041+0000","last_change":"2023-10-09T09:40:44.315598+0000","last_active":"2023-10-09T09:41:30.889041+0000","last_peered":"2023-10-09T09:41:30.889041+0000","last_clean":"2023-10-09T09:41:30.889041+0000","last_became_active":"2023-10-09T09:40:44.315323+0000","last_became_peered":"2023-10-09T09:40:44.315323+0000","last_unstale":"2023-10-09T09:41:30.889041+0000","last_undegraded":"2023-10-09T09:41:30.889041+0000","last_fullsized":"2023-10-09T09:41:30.889041+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2297,"log_dups_size":0,"ondisk_log_size":2297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:25.508761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150536192,"num_objects":2297,"num_object_clones":0,"num_object_copies":6891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2297,"num_write_kb":147008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":682,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.888064+0000","last_change":"2023-10-09T09:41:11.815662+0000","last_active":"2023-10-09T09:41:20.888064+0000","last_peered":"2023-10-09T09:41:20.888064+0000","last_clean":"2023-10-09T09:41:20.888064+0000","last_became_active":"2023-10-09T09:41:11.815203+0000","last_became_peered":"2023-10-09T09:41:11.815203+0000","last_unstale":"2023-10-09T09:41:20.888064+0000","last_undegraded":"2023-10-09T09:41:20.888064+0000","last_fullsized":"2023-10-09T09:41:20.888064+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1100,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887194+0000","last_change":"2023-10-09T09:35:18.681438+0000","last_active":"2023-10-09T09:41:20.887194+0000","last_peered":"2023-10-09T09:41:20.887194+0000","last_clean":"2023-10-09T09:41:20.887194+0000","last_became_active":"2023-10-09T09:35:18.680806+0000","last_became_peered":"2023-10-09T09:35:18.680806+0000","last_unstale":"2023-10-09T09:41:20.887194+0000","last_undegraded":"2023-10-09T09:41:20.887194+0000","last_fullsized":"2023-10-09T09:41:20.887194+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:30.669832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"1195'2264","reported_seq":3772,"reported_epoch":1195,"state":"active+clean","last_fresh":"2023-10-09T09:41:31.247555+0000","last_change":"2023-10-09T09:41:19.895176+0000","last_active":"2023-10-09T09:41:31.247555+0000","last_peered":"2023-10-09T09:41:31.247555+0000","last_clean":"2023-10-09T09:41:31.247555+0000","last_became_active":"2023-10-09T09:41:19.892209+0000","last_became_peered":"2023-10-09T09:41:19.892209+0000","last_unstale":"2023-10-09T09:41:31.247555+0000","last_undegraded":"2023-10-09T09:41:31.247555+0000","last_fullsized":"2023-10-09T09:41:31.247555+0000","mapping_epoch":1192,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2264,"log_dups_size":0,"ondisk_log_size":2264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148373504,"num_objects":2264,"num_object_clones":0,"num_object_copies":6792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2264,"num_write_kb":144896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":728,"num_bytes_recovered":47710208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":117,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887317+0000","last_change":"2023-10-09T09:41:11.847764+0000","last_active":"2023-10-09T09:41:20.887317+0000","last_peered":"2023-10-09T09:41:20.887317+0000","last_clean":"2023-10-09T09:41:20.887317+0000","last_became_active":"2023-10-09T09:41:11.800352+0000","last_became_peered":"2023-10-09T09:41:11.800352+0000","last_unstale":"2023-10-09T09:41:20.887317+0000","last_undegraded":"2023-10-09T09:41:20.887317+0000","last_fullsized":"2023-10-09T09:41:20.887317+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:56.392044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1154,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892410+0000","last_change":"2023-10-09T09:41:11.798393+0000","last_active":"2023-10-09T09:41:20.892410+0000","last_peered":"2023-10-09T09:41:20.892410+0000","last_clean":"2023-10-09T09:41:20.892410+0000","last_became_active":"2023-10-09T09:41:11.798022+0000","last_became_peered":"2023-10-09T09:41:11.798022+0000","last_unstale":"2023-10-09T09:41:20.892410+0000","last_undegraded":"2023-10-09T09:41:20.892410+0000","last_fullsized":"2023-10-09T09:41:20.892410+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1194'2169","reported_seq":2193,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:29.623410+0000","last_change":"2023-10-09T09:40:44.313621+0000","last_active":"2023-10-09T09:41:29.623410+0000","last_peered":"2023-10-09T09:41:29.623410+0000","last_clean":"2023-10-09T09:41:29.623410+0000","last_became_active":"2023-10-09T09:40:44.313290+0000","last_became_peered":"2023-10-09T09:40:44.313290+0000","last_unstale":"2023-10-09T09:41:29.623410+0000","last_undegraded":"2023-10-09T09:41:29.623410+0000","last_fullsized":"2023-10-09T09:41:29.623410+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2169,"log_dups_size":0,"ondisk_log_size":2169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:25.184192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142147584,"num_objects":2169,"num_object_clones":0,"num_object_copies":6507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2169,"num_write_kb":138816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1128,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887106+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:41:20.887106+0000","last_peered":"2023-10-09T09:41:20.887106+0000","last_clean":"2023-10-09T09:41:20.887106+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:41:20.887106+0000","last_undegraded":"2023-10-09T09:41:20.887106+0000","last_fullsized":"2023-10-09T09:41:20.887106+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1178,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892159+0000","last_change":"2023-10-09T09:36:43.419348+0000","last_active":"2023-10-09T09:41:20.892159+0000","last_peered":"2023-10-09T09:41:20.892159+0000","last_clean":"2023-10-09T09:41:20.892159+0000","last_became_active":"2023-10-09T09:36:43.418525+0000","last_became_peered":"2023-10-09T09:36:43.418525+0000","last_unstale":"2023-10-09T09:41:20.892159+0000","last_undegraded":"2023-10-09T09:41:20.892159+0000","last_fullsized":"2023-10-09T09:41:20.892159+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1194'2187","reported_seq":2218,"reported_epoch":1194,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:41:29.670272+0000","last_change":"2023-10-09T09:41:12.845441+0000","last_active":"2023-10-09T09:41:29.670272+0000","last_peered":"2023-10-09T09:41:29.670272+0000","last_clean":"2023-10-09T09:41:10.771882+0000","last_became_active":"2023-10-09T09:41:12.842478+0000","last_became_peered":"2023-10-09T09:41:12.842478+0000","last_unstale":"2023-10-09T09:41:29.670272+0000","last_undegraded":"2023-10-09T09:41:12.814568+0000","last_fullsized":"2023-10-09T09:41:12.795888+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2187,"log_dups_size":0,"ondisk_log_size":2187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:24.505859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327232,"num_objects":2187,"num_object_clones":0,"num_object_copies":6561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":748,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2187,"num_write_kb":139968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":748},{"shards":"1,5","objects":1439}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1173,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.891708+0000","last_change":"2023-10-09T09:40:49.785820+0000","last_active":"2023-10-09T09:41:20.891708+0000","last_peered":"2023-10-09T09:41:20.891708+0000","last_clean":"2023-10-09T09:41:20.891708+0000","last_became_active":"2023-10-09T09:40:49.785373+0000","last_became_peered":"2023-10-09T09:40:49.785373+0000","last_unstale":"2023-10-09T09:41:20.891708+0000","last_undegraded":"2023-10-09T09:41:20.891708+0000","last_fullsized":"2023-10-09T09:41:20.891708+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1145,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892615+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:41:20.892615+0000","last_peered":"2023-10-09T09:41:20.892615+0000","last_clean":"2023-10-09T09:41:20.892615+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:41:20.892615+0000","last_undegraded":"2023-10-09T09:41:20.892615+0000","last_fullsized":"2023-10-09T09:41:20.892615+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1194'2125","reported_seq":2159,"reported_epoch":1194,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:41:29.693987+0000","last_change":"2023-10-09T09:41:12.861349+0000","last_active":"2023-10-09T09:41:29.693987+0000","last_peered":"2023-10-09T09:41:29.693987+0000","last_clean":"2023-10-09T09:41:10.776706+0000","last_became_active":"2023-10-09T09:41:12.858736+0000","last_became_peered":"2023-10-09T09:41:12.858736+0000","last_unstale":"2023-10-09T09:41:29.693987+0000","last_undegraded":"2023-10-09T09:41:12.845180+0000","last_fullsized":"2023-10-09T09:41:12.793493+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2125,"log_dups_size":0,"ondisk_log_size":2125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:47.637941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139264000,"num_objects":2125,"num_object_clones":0,"num_object_copies":6375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":763,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2125,"num_write_kb":136000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":1362},{"shards":"1,2,3","objects":763}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1103,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.900777+0000","last_change":"2023-10-09T09:41:11.814910+0000","last_active":"2023-10-09T09:41:19.900777+0000","last_peered":"2023-10-09T09:41:19.900777+0000","last_clean":"2023-10-09T09:41:19.900777+0000","last_became_active":"2023-10-09T09:41:11.814561+0000","last_became_peered":"2023-10-09T09:41:11.814561+0000","last_unstale":"2023-10-09T09:41:19.900777+0000","last_undegraded":"2023-10-09T09:41:19.900777+0000","last_fullsized":"2023-10-09T09:41:19.900777+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:25:51.140694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":36,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.898145+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:41:20.898145+0000","last_peered":"2023-10-09T09:41:20.898145+0000","last_clean":"2023-10-09T09:41:20.898145+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:41:20.898145+0000","last_undegraded":"2023-10-09T09:41:20.898145+0000","last_fullsized":"2023-10-09T09:41:20.898145+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"1195'2274","reported_seq":2304,"reported_epoch":1195,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:41:31.205926+0000","last_change":"2023-10-09T09:41:12.863984+0000","last_active":"2023-10-09T09:41:31.205926+0000","last_peered":"2023-10-09T09:41:31.205926+0000","last_clean":"2023-10-09T09:41:10.776940+0000","last_became_active":"2023-10-09T09:41:12.862392+0000","last_became_peered":"2023-10-09T09:41:12.862392+0000","last_unstale":"2023-10-09T09:41:31.205926+0000","last_undegraded":"2023-10-09T09:41:12.801069+0000","last_fullsized":"2023-10-09T09:41:12.794241+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2274,"log_dups_size":0,"ondisk_log_size":2274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149028864,"num_objects":2274,"num_object_clones":0,"num_object_copies":6822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":726,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2274,"num_write_kb":145536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":726},{"shards":"1,5","objects":1548}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1155,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.905916+0000","last_change":"2023-10-09T09:40:49.781045+0000","last_active":"2023-10-09T09:41:19.905916+0000","last_peered":"2023-10-09T09:41:19.905916+0000","last_clean":"2023-10-09T09:41:19.905916+0000","last_became_active":"2023-10-09T09:40:49.780425+0000","last_became_peered":"2023-10-09T09:40:49.780425+0000","last_unstale":"2023-10-09T09:41:19.905916+0000","last_undegraded":"2023-10-09T09:41:19.905916+0000","last_fullsized":"2023-10-09T09:41:19.905916+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:56.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":37,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.888938+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:41:20.888938+0000","last_peered":"2023-10-09T09:41:20.888938+0000","last_clean":"2023-10-09T09:41:20.888938+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:41:20.888938+0000","last_undegraded":"2023-10-09T09:41:20.888938+0000","last_fullsized":"2023-10-09T09:41:20.888938+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1139,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.907217+0000","last_change":"2023-10-09T09:40:25.799081+0000","last_active":"2023-10-09T09:41:19.907217+0000","last_peered":"2023-10-09T09:41:19.907217+0000","last_clean":"2023-10-09T09:41:19.907217+0000","last_became_active":"2023-10-09T09:40:25.798693+0000","last_became_peered":"2023-10-09T09:40:25.798693+0000","last_unstale":"2023-10-09T09:41:19.907217+0000","last_undegraded":"2023-10-09T09:41:19.907217+0000","last_fullsized":"2023-10-09T09:41:19.907217+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:35.527067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"1195'2391","reported_seq":4006,"reported_epoch":1195,"state":"active+clean","last_fresh":"2023-10-09T09:41:32.297337+0000","last_change":"2023-10-09T09:41:20.893879+0000","last_active":"2023-10-09T09:41:32.297337+0000","last_peered":"2023-10-09T09:41:32.297337+0000","last_clean":"2023-10-09T09:41:32.297337+0000","last_became_active":"2023-10-09T09:41:20.890059+0000","last_became_peered":"2023-10-09T09:41:20.890059+0000","last_unstale":"2023-10-09T09:41:32.297337+0000","last_undegraded":"2023-10-09T09:41:32.297337+0000","last_fullsized":"2023-10-09T09:41:32.297337+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2391,"log_dups_size":0,"ondisk_log_size":2391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:34.115475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156696576,"num_objects":2391,"num_object_clones":0,"num_object_copies":7173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2391,"num_write_kb":153024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":781,"num_bytes_recovered":51183616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":622,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.907121+0000","last_change":"2023-10-09T09:40:25.795910+0000","last_active":"2023-10-09T09:41:19.907121+0000","last_peered":"2023-10-09T09:41:19.907121+0000","last_clean":"2023-10-09T09:41:19.907121+0000","last_became_active":"2023-10-09T09:40:25.795327+0000","last_became_peered":"2023-10-09T09:40:25.795327+0000","last_unstale":"2023-10-09T09:41:19.907121+0000","last_undegraded":"2023-10-09T09:41:19.907121+0000","last_fullsized":"2023-10-09T09:41:19.907121+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:58.343181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1161,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892413+0000","last_change":"2023-10-09T09:35:18.685554+0000","last_active":"2023-10-09T09:41:20.892413+0000","last_peered":"2023-10-09T09:41:20.892413+0000","last_clean":"2023-10-09T09:41:20.892413+0000","last_became_active":"2023-10-09T09:35:18.685119+0000","last_became_peered":"2023-10-09T09:35:18.685119+0000","last_unstale":"2023-10-09T09:41:20.892413+0000","last_undegraded":"2023-10-09T09:41:20.892413+0000","last_fullsized":"2023-10-09T09:41:20.892413+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1178,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.905403+0000","last_change":"2023-10-09T09:41:11.811782+0000","last_active":"2023-10-09T09:41:19.905403+0000","last_peered":"2023-10-09T09:41:19.905403+0000","last_clean":"2023-10-09T09:41:19.905403+0000","last_became_active":"2023-10-09T09:41:11.811431+0000","last_became_peered":"2023-10-09T09:41:11.811431+0000","last_unstale":"2023-10-09T09:41:19.905403+0000","last_undegraded":"2023-10-09T09:41:19.905403+0000","last_fullsized":"2023-10-09T09:41:19.905403+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:05.197258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"1195'2331","reported_seq":2371,"reported_epoch":1195,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:41:31.241758+0000","last_change":"2023-10-09T09:41:12.855472+0000","last_active":"2023-10-09T09:41:31.241758+0000","last_peered":"2023-10-09T09:41:31.241758+0000","last_clean":"2023-10-09T09:41:10.774573+0000","last_became_active":"2023-10-09T09:41:12.853578+0000","last_became_peered":"2023-10-09T09:41:12.853578+0000","last_unstale":"2023-10-09T09:41:31.241758+0000","last_undegraded":"2023-10-09T09:41:12.803496+0000","last_fullsized":"2023-10-09T09:41:12.793003+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2331,"log_dups_size":0,"ondisk_log_size":2331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152764416,"num_objects":2331,"num_object_clones":0,"num_object_copies":6993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":723,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2331,"num_write_kb":149184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":723},{"shards":"1,5","objects":1608}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":66,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.897459+0000","last_change":"2023-10-09T09:41:11.801228+0000","last_active":"2023-10-09T09:41:20.897459+0000","last_peered":"2023-10-09T09:41:20.897459+0000","last_clean":"2023-10-09T09:41:20.897459+0000","last_became_active":"2023-10-09T09:41:11.800068+0000","last_became_peered":"2023-10-09T09:41:11.800068+0000","last_unstale":"2023-10-09T09:41:20.897459+0000","last_undegraded":"2023-10-09T09:41:20.897459+0000","last_fullsized":"2023-10-09T09:41:20.897459+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:33.425580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1155,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.900742+0000","last_change":"2023-10-09T09:36:43.421868+0000","last_active":"2023-10-09T09:41:19.900742+0000","last_peered":"2023-10-09T09:41:19.900742+0000","last_clean":"2023-10-09T09:41:19.900742+0000","last_became_active":"2023-10-09T09:36:43.421013+0000","last_became_peered":"2023-10-09T09:36:43.421013+0000","last_unstale":"2023-10-09T09:41:19.900742+0000","last_undegraded":"2023-10-09T09:41:19.900742+0000","last_fullsized":"2023-10-09T09:41:19.900742+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:43.353667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1159,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.903037+0000","last_change":"2023-10-09T09:40:25.799440+0000","last_active":"2023-10-09T09:41:19.903037+0000","last_peered":"2023-10-09T09:41:19.903037+0000","last_clean":"2023-10-09T09:41:19.903037+0000","last_became_active":"2023-10-09T09:40:25.797967+0000","last_became_peered":"2023-10-09T09:40:25.797967+0000","last_unstale":"2023-10-09T09:41:19.903037+0000","last_undegraded":"2023-10-09T09:41:19.903037+0000","last_fullsized":"2023-10-09T09:41:19.903037+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:51.038572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1195'2417","reported_seq":2450,"reported_epoch":1195,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:41:32.279916+0000","last_change":"2023-10-09T09:41:12.859162+0000","last_active":"2023-10-09T09:41:32.279916+0000","last_peered":"2023-10-09T09:41:32.279916+0000","last_clean":"2023-10-09T09:41:10.762192+0000","last_became_active":"2023-10-09T09:41:12.856058+0000","last_became_peered":"2023-10-09T09:41:12.856058+0000","last_unstale":"2023-10-09T09:41:32.279916+0000","last_undegraded":"2023-10-09T09:41:12.800139+0000","last_fullsized":"2023-10-09T09:41:12.790413+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2417,"log_dups_size":0,"ondisk_log_size":2417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:17.132594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158400512,"num_objects":2417,"num_object_clones":0,"num_object_copies":7251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":789,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2417,"num_write_kb":154688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":789},{"shards":"0,5","objects":1628}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1199,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.897731+0000","last_change":"2023-10-09T09:41:11.795802+0000","last_active":"2023-10-09T09:41:20.897731+0000","last_peered":"2023-10-09T09:41:20.897731+0000","last_clean":"2023-10-09T09:41:20.897731+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:41:20.897731+0000","last_undegraded":"2023-10-09T09:41:20.897731+0000","last_fullsized":"2023-10-09T09:41:20.897731+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":517,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.886807+0000","last_change":"2023-10-09T09:40:52.800168+0000","last_active":"2023-10-09T09:41:20.886807+0000","last_peered":"2023-10-09T09:41:20.886807+0000","last_clean":"2023-10-09T09:41:20.886807+0000","last_became_active":"2023-10-09T09:40:52.799543+0000","last_became_peered":"2023-10-09T09:40:52.799543+0000","last_unstale":"2023-10-09T09:41:20.886807+0000","last_undegraded":"2023-10-09T09:41:20.886807+0000","last_fullsized":"2023-10-09T09:41:20.886807+0000","mapping_epoch":1185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:44.481003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":37,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.888557+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:41:20.888557+0000","last_peered":"2023-10-09T09:41:20.888557+0000","last_clean":"2023-10-09T09:41:20.888557+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:41:20.888557+0000","last_undegraded":"2023-10-09T09:41:20.888557+0000","last_fullsized":"2023-10-09T09:41:20.888557+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1121,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887184+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:41:20.887184+0000","last_peered":"2023-10-09T09:41:20.887184+0000","last_clean":"2023-10-09T09:41:20.887184+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:41:20.887184+0000","last_undegraded":"2023-10-09T09:41:20.887184+0000","last_fullsized":"2023-10-09T09:41:20.887184+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1195'2426","reported_seq":2447,"reported_epoch":1195,"state":"active+clean","last_fresh":"2023-10-09T09:41:32.297661+0000","last_change":"2023-10-09T09:40:44.315824+0000","last_active":"2023-10-09T09:41:32.297661+0000","last_peered":"2023-10-09T09:41:32.297661+0000","last_clean":"2023-10-09T09:41:32.297661+0000","last_became_active":"2023-10-09T09:40:44.315401+0000","last_became_peered":"2023-10-09T09:40:44.315401+0000","last_unstale":"2023-10-09T09:41:32.297661+0000","last_undegraded":"2023-10-09T09:41:32.297661+0000","last_fullsized":"2023-10-09T09:41:32.297661+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2426,"log_dups_size":0,"ondisk_log_size":2426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:51.462075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158990336,"num_objects":2426,"num_object_clones":0,"num_object_copies":7278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2426,"num_write_kb":155264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1122,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.900521+0000","last_change":"2023-10-09T09:41:11.815645+0000","last_active":"2023-10-09T09:41:19.900521+0000","last_peered":"2023-10-09T09:41:19.900521+0000","last_clean":"2023-10-09T09:41:19.900521+0000","last_became_active":"2023-10-09T09:41:11.815086+0000","last_became_peered":"2023-10-09T09:41:11.815086+0000","last_unstale":"2023-10-09T09:41:19.900521+0000","last_undegraded":"2023-10-09T09:41:19.900521+0000","last_fullsized":"2023-10-09T09:41:19.900521+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:29.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1127,"reported_epoch":1195,"state":"active+clean","last_fresh":"2023-10-09T09:41:30.846755+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:41:30.846755+0000","last_peered":"2023-10-09T09:41:30.846755+0000","last_clean":"2023-10-09T09:41:30.846755+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:41:30.846755+0000","last_undegraded":"2023-10-09T09:41:30.846755+0000","last_fullsized":"2023-10-09T09:41:30.846755+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1123,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887610+0000","last_change":"2023-10-09T09:39:12.949268+0000","last_active":"2023-10-09T09:41:20.887610+0000","last_peered":"2023-10-09T09:41:20.887610+0000","last_clean":"2023-10-09T09:41:20.887610+0000","last_became_active":"2023-10-09T09:39:12.948924+0000","last_became_peered":"2023-10-09T09:39:12.948924+0000","last_unstale":"2023-10-09T09:41:20.887610+0000","last_undegraded":"2023-10-09T09:41:20.887610+0000","last_fullsized":"2023-10-09T09:41:20.887610+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:31.368540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"1195'2322","reported_seq":2346,"reported_epoch":1195,"state":"active+clean","last_fresh":"2023-10-09T09:41:31.204354+0000","last_change":"2023-10-09T09:40:49.803104+0000","last_active":"2023-10-09T09:41:31.204354+0000","last_peered":"2023-10-09T09:41:31.204354+0000","last_clean":"2023-10-09T09:41:31.204354+0000","last_became_active":"2023-10-09T09:40:49.802743+0000","last_became_peered":"2023-10-09T09:40:49.802743+0000","last_unstale":"2023-10-09T09:41:31.204354+0000","last_undegraded":"2023-10-09T09:41:31.204354+0000","last_fullsized":"2023-10-09T09:41:31.204354+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2322,"log_dups_size":0,"ondisk_log_size":2322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152174592,"num_objects":2322,"num_object_clones":0,"num_object_copies":6966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2322,"num_write_kb":148608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1099,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887948+0000","last_change":"2023-10-09T09:40:49.787387+0000","last_active":"2023-10-09T09:41:20.887948+0000","last_peered":"2023-10-09T09:41:20.887948+0000","last_clean":"2023-10-09T09:41:20.887948+0000","last_became_active":"2023-10-09T09:40:49.786954+0000","last_became_peered":"2023-10-09T09:40:49.786954+0000","last_unstale":"2023-10-09T09:41:20.887948+0000","last_undegraded":"2023-10-09T09:41:20.887948+0000","last_fullsized":"2023-10-09T09:41:20.887948+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:51.055735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1173,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.905776+0000","last_change":"2023-10-09T09:40:25.797470+0000","last_active":"2023-10-09T09:41:19.905776+0000","last_peered":"2023-10-09T09:41:19.905776+0000","last_clean":"2023-10-09T09:41:19.905776+0000","last_became_active":"2023-10-09T09:40:25.796966+0000","last_became_peered":"2023-10-09T09:40:25.796966+0000","last_unstale":"2023-10-09T09:41:19.905776+0000","last_undegraded":"2023-10-09T09:41:19.905776+0000","last_fullsized":"2023-10-09T09:41:19.905776+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:24.526350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1129,"reported_epoch":1195,"state":"active+clean","last_fresh":"2023-10-09T09:41:30.847607+0000","last_change":"2023-10-09T09:40:49.792931+0000","last_active":"2023-10-09T09:41:30.847607+0000","last_peered":"2023-10-09T09:41:30.847607+0000","last_clean":"2023-10-09T09:41:30.847607+0000","last_became_active":"2023-10-09T09:40:49.792543+0000","last_became_peered":"2023-10-09T09:40:49.792543+0000","last_unstale":"2023-10-09T09:41:30.847607+0000","last_undegraded":"2023-10-09T09:41:30.847607+0000","last_fullsized":"2023-10-09T09:41:30.847607+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:00.778249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"1194'2190","reported_seq":2213,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:29.696675+0000","last_change":"2023-10-09T09:40:49.794927+0000","last_active":"2023-10-09T09:41:29.696675+0000","last_peered":"2023-10-09T09:41:29.696675+0000","last_clean":"2023-10-09T09:41:29.696675+0000","last_became_active":"2023-10-09T09:40:49.794550+0000","last_became_peered":"2023-10-09T09:40:49.794550+0000","last_unstale":"2023-10-09T09:41:29.696675+0000","last_undegraded":"2023-10-09T09:41:29.696675+0000","last_fullsized":"2023-10-09T09:41:29.696675+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2190,"log_dups_size":0,"ondisk_log_size":2190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:34.821565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143523840,"num_objects":2190,"num_object_clones":0,"num_object_copies":6570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2190,"num_write_kb":140160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":52,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887936+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:41:20.887936+0000","last_peered":"2023-10-09T09:41:20.887936+0000","last_clean":"2023-10-09T09:41:20.887936+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:41:20.887936+0000","last_undegraded":"2023-10-09T09:41:20.887936+0000","last_fullsized":"2023-10-09T09:41:20.887936+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1120,"reported_epoch":1193,"state":"active+clean","last_fresh":"2023-10-09T09:41:19.903388+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:41:19.903388+0000","last_peered":"2023-10-09T09:41:19.903388+0000","last_clean":"2023-10-09T09:41:19.903388+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:41:19.903388+0000","last_undegraded":"2023-10-09T09:41:19.903388+0000","last_fullsized":"2023-10-09T09:41:19.903388+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1090,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887128+0000","last_change":"2023-10-09T09:41:11.801414+0000","last_active":"2023-10-09T09:41:20.887128+0000","last_peered":"2023-10-09T09:41:20.887128+0000","last_clean":"2023-10-09T09:41:20.887128+0000","last_became_active":"2023-10-09T09:41:11.800211+0000","last_became_peered":"2023-10-09T09:41:11.800211+0000","last_unstale":"2023-10-09T09:41:20.887128+0000","last_undegraded":"2023-10-09T09:41:20.887128+0000","last_fullsized":"2023-10-09T09:41:20.887128+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:44.155906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"1195'2294","reported_seq":2888,"reported_epoch":1195,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T09:41:31.235897+0000","last_change":"2023-10-09T09:41:19.388114+0000","last_active":"2023-10-09T09:41:31.235897+0000","last_peered":"2023-10-09T09:41:31.235897+0000","last_clean":"2023-10-09T09:41:10.786762+0000","last_became_active":"2023-10-09T09:41:12.839816+0000","last_became_peered":"2023-10-09T09:41:12.839816+0000","last_unstale":"2023-10-09T09:41:31.235897+0000","last_undegraded":"2023-10-09T09:41:31.235897+0000","last_fullsized":"2023-10-09T09:41:12.799266+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":2294,"log_dups_size":0,"ondisk_log_size":2294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150339584,"num_objects":2294,"num_object_clones":0,"num_object_copies":6882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":473,"num_objects_unfound":0,"num_objects_dirty":2294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2294,"num_write_kb":146816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":18087936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":473},{"shards":"0,4","objects":1821}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1167,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887193+0000","last_change":"2023-10-09T09:41:11.815721+0000","last_active":"2023-10-09T09:41:20.887193+0000","last_peered":"2023-10-09T09:41:20.887193+0000","last_clean":"2023-10-09T09:41:20.887193+0000","last_became_active":"2023-10-09T09:41:11.815237+0000","last_became_peered":"2023-10-09T09:41:11.815237+0000","last_unstale":"2023-10-09T09:41:20.887193+0000","last_undegraded":"2023-10-09T09:41:20.887193+0000","last_fullsized":"2023-10-09T09:41:20.887193+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1104,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887812+0000","last_change":"2023-10-09T09:35:18.685030+0000","last_active":"2023-10-09T09:41:20.887812+0000","last_peered":"2023-10-09T09:41:20.887812+0000","last_clean":"2023-10-09T09:41:20.887812+0000","last_became_active":"2023-10-09T09:35:18.675810+0000","last_became_peered":"2023-10-09T09:35:18.675810+0000","last_unstale":"2023-10-09T09:41:20.887812+0000","last_undegraded":"2023-10-09T09:41:20.887812+0000","last_fullsized":"2023-10-09T09:41:20.887812+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:52.793405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1148,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.891733+0000","last_change":"2023-10-09T09:40:25.804139+0000","last_active":"2023-10-09T09:41:20.891733+0000","last_peered":"2023-10-09T09:41:20.891733+0000","last_clean":"2023-10-09T09:41:20.891733+0000","last_became_active":"2023-10-09T09:40:25.803753+0000","last_became_peered":"2023-10-09T09:40:25.803753+0000","last_unstale":"2023-10-09T09:41:20.891733+0000","last_undegraded":"2023-10-09T09:41:20.891733+0000","last_fullsized":"2023-10-09T09:41:20.891733+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1113,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887119+0000","last_change":"2023-10-09T09:40:49.788328+0000","last_active":"2023-10-09T09:41:20.887119+0000","last_peered":"2023-10-09T09:41:20.887119+0000","last_clean":"2023-10-09T09:41:20.887119+0000","last_became_active":"2023-10-09T09:40:49.787927+0000","last_became_peered":"2023-10-09T09:40:49.787927+0000","last_unstale":"2023-10-09T09:41:20.887119+0000","last_undegraded":"2023-10-09T09:41:20.887119+0000","last_fullsized":"2023-10-09T09:41:20.887119+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:32.616104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":65,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.898079+0000","last_change":"2023-10-09T09:41:11.800443+0000","last_active":"2023-10-09T09:41:20.898079+0000","last_peered":"2023-10-09T09:41:20.898079+0000","last_clean":"2023-10-09T09:41:20.898079+0000","last_became_active":"2023-10-09T09:41:11.799864+0000","last_became_peered":"2023-10-09T09:41:11.799864+0000","last_unstale":"2023-10-09T09:41:20.898079+0000","last_undegraded":"2023-10-09T09:41:20.898079+0000","last_fullsized":"2023-10-09T09:41:20.898079+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:48:22.481162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":85,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892270+0000","last_change":"2023-10-09T09:40:25.856640+0000","last_active":"2023-10-09T09:41:20.892270+0000","last_peered":"2023-10-09T09:41:20.892270+0000","last_clean":"2023-10-09T09:41:20.892270+0000","last_became_active":"2023-10-09T09:40:25.802924+0000","last_became_peered":"2023-10-09T09:40:25.802924+0000","last_unstale":"2023-10-09T09:41:20.892270+0000","last_undegraded":"2023-10-09T09:41:20.892270+0000","last_fullsized":"2023-10-09T09:41:20.892270+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1169,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892119+0000","last_change":"2023-10-09T09:41:11.801626+0000","last_active":"2023-10-09T09:41:20.892119+0000","last_peered":"2023-10-09T09:41:20.892119+0000","last_clean":"2023-10-09T09:41:20.892119+0000","last_became_active":"2023-10-09T09:41:11.801169+0000","last_became_peered":"2023-10-09T09:41:11.801169+0000","last_unstale":"2023-10-09T09:41:20.892119+0000","last_undegraded":"2023-10-09T09:41:20.892119+0000","last_fullsized":"2023-10-09T09:41:20.892119+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":85,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892217+0000","last_change":"2023-10-09T09:40:25.901197+0000","last_active":"2023-10-09T09:41:20.892217+0000","last_peered":"2023-10-09T09:41:20.892217+0000","last_clean":"2023-10-09T09:41:20.892217+0000","last_became_active":"2023-10-09T09:40:25.804595+0000","last_became_peered":"2023-10-09T09:40:25.804595+0000","last_unstale":"2023-10-09T09:41:20.892217+0000","last_undegraded":"2023-10-09T09:41:20.892217+0000","last_fullsized":"2023-10-09T09:41:20.892217+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1184,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.892087+0000","last_change":"2023-10-09T09:41:11.801341+0000","last_active":"2023-10-09T09:41:20.892087+0000","last_peered":"2023-10-09T09:41:20.892087+0000","last_clean":"2023-10-09T09:41:20.892087+0000","last_became_active":"2023-10-09T09:41:11.800954+0000","last_became_peered":"2023-10-09T09:41:11.800954+0000","last_unstale":"2023-10-09T09:41:20.892087+0000","last_undegraded":"2023-10-09T09:41:20.892087+0000","last_fullsized":"2023-10-09T09:41:20.892087+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":128,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.887289+0000","last_change":"2023-10-09T09:41:11.816169+0000","last_active":"2023-10-09T09:41:20.887289+0000","last_peered":"2023-10-09T09:41:20.887289+0000","last_clean":"2023-10-09T09:41:20.887289+0000","last_became_active":"2023-10-09T09:41:11.815426+0000","last_became_peered":"2023-10-09T09:41:11.815426+0000","last_unstale":"2023-10-09T09:41:20.887289+0000","last_undegraded":"2023-10-09T09:41:20.887289+0000","last_fullsized":"2023-10-09T09:41:20.887289+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":65,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.897355+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:41:20.897355+0000","last_peered":"2023-10-09T09:41:20.897355+0000","last_clean":"2023-10-09T09:41:20.897355+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:41:20.897355+0000","last_undegraded":"2023-10-09T09:41:20.897355+0000","last_fullsized":"2023-10-09T09:41:20.897355+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1153,"reported_epoch":1195,"state":"active+clean","last_fresh":"2023-10-09T09:41:30.846887+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:41:30.846887+0000","last_peered":"2023-10-09T09:41:30.846887+0000","last_clean":"2023-10-09T09:41:30.846887+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:41:30.846887+0000","last_undegraded":"2023-10-09T09:41:30.846887+0000","last_fullsized":"2023-10-09T09:41:30.846887+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1145,"reported_epoch":1194,"state":"active+clean","last_fresh":"2023-10-09T09:41:20.891714+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:41:20.891714+0000","last_peered":"2023-10-09T09:41:20.891714+0000","last_clean":"2023-10-09T09:41:20.891714+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:41:20.891714+0000","last_undegraded":"2023-10-09T09:41:20.891714+0000","last_fullsized":"2023-10-09T09:41:20.891714+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":7306,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2394226688,"num_objects":36533,"num_object_clones":0,"num_object_copies":109599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5279,"num_objects_misplaced":473,"num_objects_unfound":0,"num_objects_dirty":36533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":36533,"num_write_kb":2338112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1789,"num_bytes_recovered":117243904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7258898432,"data_stored":7258898432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":36533,"ondisk_log_size":36533,"up":48,"acting":40,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453397997,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594230,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1270180,"kb_used_data":972956,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92474972,"statfs":{"total":95995035648,"available":94694371328,"internally_reserved":0,"allocated":996306944,"data_stored":996052426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:41:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7029999999999998,"5min":1.77,"15min":1.359},"min":{"1min":0.46700000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":22.742000000000001,"5min":22.742000000000001,"15min":22.742000000000001},"last":0.995},{"interface":"front","average":{"1min":2.7639999999999998,"5min":2.0459999999999998,"15min":1.506},"min":{"1min":0.503,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":22.766999999999999,"5min":22.766999999999999,"15min":22.766999999999999},"last":1.399}]},{"osd":1,"last update":"Mon Oct 9 09:41:28 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":2.2309999999999999,"15min":2.5249999999999999},"min":{"1min":0.42099999999999999,"5min":0.27200000000000002,"15min":0.22900000000000001},"max":{"1min":2.4889999999999999,"5min":38.709000000000003,"15min":103.259},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.93600000000000005,"5min":2.218,"15min":2.2930000000000001},"min":{"1min":0.40400000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.899,"5min":35.820999999999998,"15min":102.005},"last":0.68000000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.948,"15min":1.843},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":1.2090000000000001,"5min":32.661999999999999,"15min":33.210000000000001},"last":1.3340000000000001},{"interface":"front","average":{"1min":0.80200000000000005,"5min":2.198,"15min":1.9379999999999999},"min":{"1min":0.41299999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.427,"5min":34.030999999999999,"15min":37.246000000000002},"last":1.379}]},{"osd":3,"last update":"Mon Oct 9 09:41:19 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.76,"15min":2.4390000000000001},"min":{"1min":0.56899999999999995,"5min":0.309,"15min":0.309},"max":{"1min":1.0409999999999999,"5min":11.547000000000001,"15min":123.97199999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.80300000000000005,"5min":1.571,"15min":2.3109999999999999},"min":{"1min":0.47499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.536,"5min":9.5679999999999996,"15min":124.059},"last":0.64800000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:41:19 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":1.2210000000000001,"15min":1.1160000000000001},"min":{"1min":0.28399999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.0389999999999999,"5min":8.0800000000000001,"15min":9.1099999999999994},"last":1.2969999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":1.401,"15min":1.3320000000000001},"min":{"1min":0.32400000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.2010000000000001,"5min":5.9470000000000001,"15min":9.7919999999999998},"last":0.56200000000000006}]},{"osd":5,"last update":"Mon Oct 9 09:41:19 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.3169999999999999,"15min":1.26},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.167,"5min":8.1400000000000006,"15min":9.0559999999999992},"last":1.359},{"interface":"front","average":{"1min":0.84299999999999997,"5min":1.587,"15min":1.4039999999999999},"min":{"1min":0.35499999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.2230000000000001,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.59999999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.361,"15min":1.3979999999999999},"min":{"1min":0.52100000000000002,"5min":0.39600000000000002,"15min":0.35599999999999998},"max":{"1min":1.274,"5min":5.1390000000000002,"15min":7.6769999999999996},"last":0.625},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.3740000000000001,"15min":1.411},"min":{"1min":0.60299999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.532,"5min":6.931,"15min":7.6879999999999997},"last":0.51300000000000001}]}]},{"osd":2,"up_from":634,"seq":2723009265983,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1312480,"kb_used_data":830488,"kb_used_omap":122,"kb_used_meta":481861,"kb_avail":92432672,"statfs":{"total":95995035648,"available":94651056128,"internally_reserved":0,"allocated":850419712,"data_stored":849068666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":493426251},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:41:29 2023","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":2.6400000000000001,"15min":2.6240000000000001},"min":{"1min":0.37,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.415,"5min":86.221999999999994,"15min":86.221999999999994},"last":3.415},{"interface":"front","average":{"1min":0.97499999999999998,"5min":2.6349999999999998,"15min":2.899},"min":{"1min":0.39000000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.0910000000000002,"5min":82.183000000000007,"15min":82.183000000000007},"last":2.2690000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:41:14 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":3.2250000000000001,"15min":2.4420000000000002},"min":{"1min":0.35299999999999998,"5min":0.25600000000000001,"15min":0.184},"max":{"1min":1.9610000000000001,"5min":80.655000000000001,"15min":81.662999999999997},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.755,"5min":3.1669999999999998,"15min":2.5790000000000002},"min":{"1min":0.42199999999999999,"5min":0.29199999999999998,"15min":0.216},"max":{"1min":2.202,"5min":80.381,"15min":81.712999999999994},"last":0.80400000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":14.214,"5min":4.415,"15min":2.7480000000000002},"min":{"1min":0.41099999999999998,"5min":0.29699999999999999,"15min":0.252},"max":{"1min":230.476,"5min":230.476,"15min":230.476},"last":0.67100000000000004},{"interface":"front","average":{"1min":14.17,"5min":4.3200000000000003,"15min":2.7959999999999998},"min":{"1min":0.55500000000000005,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":228.893,"5min":228.893,"15min":228.893},"last":0.84099999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:41:14 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.732,"15min":1.5580000000000001},"min":{"1min":0.55400000000000005,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.3109999999999999,"5min":11.622999999999999,"15min":11.622999999999999},"last":1.984},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.6060000000000001,"15min":1.6080000000000001},"min":{"1min":0.52600000000000002,"5min":0.313,"15min":0.313},"max":{"1min":1.272,"5min":10.394,"15min":13.917999999999999},"last":1.762}]},{"osd":5,"last update":"Mon Oct 9 09:41:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":1.5600000000000001,"15min":1.4650000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.313},"max":{"1min":1.2809999999999999,"5min":9.7379999999999995,"15min":9.7599999999999998},"last":3.6179999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.788,"15min":1.6439999999999999},"min":{"1min":0.58799999999999997,"5min":0.38300000000000001,"15min":0.36499999999999999},"max":{"1min":1.327,"5min":11.718999999999999,"15min":11.718999999999999},"last":2.024}]},{"osd":6,"last update":"Mon Oct 9 09:41:14 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":2.133,"15min":1.8160000000000001},"min":{"1min":0.53700000000000003,"5min":0.40300000000000002,"15min":0.33100000000000002},"max":{"1min":1.2869999999999999,"5min":11.648,"15min":11.648},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.5209999999999999,"15min":1.7230000000000001},"min":{"1min":0.65000000000000002,"5min":0.35599999999999998,"15min":0.33300000000000002},"max":{"1min":1.1799999999999999,"5min":10.436999999999999,"15min":13.893000000000001},"last":1.7989999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:41:14 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.6120000000000001,"15min":1.8340000000000001},"min":{"1min":0.68999999999999995,"5min":0.39400000000000002,"15min":0.377},"max":{"1min":3.6080000000000001,"5min":10.462999999999999,"15min":13.848000000000001},"last":1.875},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.653,"15min":1.661},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.34499999999999997},"max":{"1min":3.6720000000000002,"5min":9.7759999999999998,"15min":10.199},"last":0.82999999999999996}]}]},{"osd":1,"up_from":607,"seq":2607045148996,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2026256,"kb_used_data":1567304,"kb_used_omap":107,"kb_used_meta":458836,"kb_avail":91718896,"statfs":{"total":95995035648,"available":93920149504,"internally_reserved":0,"allocated":1604919296,"data_stored":1603577626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":469848747},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,3,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:41:28 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":1.49,"15min":1.4299999999999999},"min":{"1min":0.32500000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.611,"5min":11.458,"15min":11.458},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.9259999999999999,"15min":2.3180000000000001},"min":{"1min":0.46100000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.2349999999999999,"5min":15.603,"15min":15.603},"last":2.2349999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.9830000000000001,"15min":2.613},"min":{"1min":0.377,"5min":0.23300000000000001,"15min":0.20100000000000001},"max":{"1min":1.2330000000000001,"5min":14.225,"15min":84.578999999999994},"last":2.2829999999999999},{"interface":"front","average":{"1min":0.68000000000000005,"5min":1.55,"15min":2.1400000000000001},"min":{"1min":0.375,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":1.323,"5min":8.3539999999999992,"15min":84.697000000000003},"last":1.5580000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:41:28 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.6839999999999999,"15min":3.073},"min":{"1min":0.44500000000000001,"5min":0.26000000000000001,"15min":0.23599999999999999},"max":{"1min":2.964,"5min":12.589,"15min":107.426},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.9470000000000001,"15min":3.5419999999999998},"min":{"1min":0.58899999999999997,"5min":0.29099999999999998,"15min":0.23200000000000001},"max":{"1min":2.706,"5min":14.27,"15min":106.15900000000001},"last":2.3090000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:40:29 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.494,"15min":1.5680000000000001},"min":{"1min":0.45200000000000001,"5min":0.32100000000000001,"15min":0.26500000000000001},"max":{"1min":1.091,"5min":7.2759999999999998,"15min":84.875},"last":4.1500000000000004},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.5009999999999999,"15min":1.764},"min":{"1min":0.59799999999999998,"5min":0.33200000000000002,"15min":0.29499999999999998},"max":{"1min":0.98899999999999999,"5min":7.7560000000000002,"15min":84.992999999999995},"last":0.83999999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:40:29 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.6080000000000001,"15min":1.6379999999999999},"min":{"1min":0.498,"5min":0.32600000000000001,"15min":0.32500000000000001},"max":{"1min":1.21,"5min":8.6869999999999994,"15min":84.805000000000007},"last":1.5860000000000001},{"interface":"front","average":{"1min":0.84299999999999997,"5min":1.5149999999999999,"15min":1.7350000000000001},"min":{"1min":0.55500000000000005,"5min":0.35899999999999999,"15min":0.30399999999999999},"max":{"1min":1.1879999999999999,"5min":6.2569999999999997,"15min":84.951999999999998},"last":3.0449999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:40:29 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":2.1080000000000001,"15min":2.121},"min":{"1min":0.47699999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.448,"5min":14.335000000000001,"15min":85.016000000000005},"last":2.4159999999999999},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.8200000000000001,"15min":1.7869999999999999},"min":{"1min":0.47099999999999997,"5min":0.371,"15min":0.36799999999999999},"max":{"1min":1.2829999999999999,"5min":7.3209999999999997,"15min":84.831999999999994},"last":1.647}]},{"osd":7,"last update":"Mon Oct 9 09:40:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":2.1869999999999998,"15min":2.3029999999999999},"min":{"1min":0.67500000000000004,"5min":0.36199999999999999,"15min":0.33100000000000002},"max":{"1min":2.0230000000000001,"5min":14.364000000000001,"15min":84.888000000000005},"last":2.5659999999999998},{"interface":"front","average":{"1min":1,"5min":1.821,"15min":1.976},"min":{"1min":0.63600000000000001,"5min":0.441,"15min":0.36699999999999999},"max":{"1min":2.012,"5min":8.9540000000000006,"15min":84.742000000000004},"last":1.671}]}]},{"osd":0,"up_from":1136,"seq":4879082848332,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1255140,"kb_used_data":1015580,"kb_used_omap":280,"kb_used_meta":239271,"kb_avail":92490012,"statfs":{"total":95995035648,"available":94709772288,"internally_reserved":0,"allocated":1039953920,"data_stored":1038655334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":245013914},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":2.3149999999999999,"15min":1.3009999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":8.3569999999999993,"5min":33.634,"15min":33.634},"last":0.89700000000000002},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.3260000000000001,"15min":1.468},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.4030000000000005,"5min":33.661000000000001,"15min":33.661000000000001},"last":0.55200000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.5700000000000001,"15min":1.1080000000000001},"min":{"1min":0.29099999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.573,"5min":8.9640000000000004,"15min":8.9640000000000004},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.5920000000000001,"15min":1.3640000000000001},"min":{"1min":0.317,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.9569999999999999,"5min":14.382999999999999,"15min":14.382999999999999},"last":4.2770000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.4690000000000001,"15min":1.3049999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":3.6579999999999999,"5min":7.9219999999999997,"15min":7.9219999999999997},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.5940000000000001,"15min":1.103},"min":{"1min":0.25800000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.0129999999999999,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.57799999999999996}]},{"osd":4,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.4079999999999999,"15min":1.2390000000000001},"min":{"1min":0.46300000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.8160000000000001,"5min":6.8390000000000004,"15min":6.8390000000000004},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.383,"5min":1.667,"15min":1.383},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.8079999999999998,"5min":6.9009999999999998,"15min":6.9009999999999998},"last":0.374}]},{"osd":5,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.482,"15min":1.3360000000000001},"min":{"1min":0.41999999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.7000000000000002,"5min":6.915,"15min":6.915},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.2729999999999999,"5min":1.5409999999999999,"15min":1.593},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0499999999999998,"5min":6.242,"15min":7.7149999999999999},"last":1.0549999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.5069999999999999,"15min":1.2170000000000001},"min":{"1min":0.44700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.762,"5min":7.6909999999999998,"15min":7.6909999999999998},"last":1.036},{"interface":"front","average":{"1min":1.3089999999999999,"5min":1.794,"15min":1.8069999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.718,"5min":8.625,"15min":8.625},"last":0.93300000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.7410000000000001,"15min":1.3660000000000001},"min":{"1min":0.53500000000000003,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.7549999999999999,"5min":9.2159999999999993,"15min":9.2159999999999993},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.724,"15min":1.6439999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":3.7360000000000002,"5min":14.361000000000001,"15min":14.361000000000001},"last":0.88200000000000001}]}]},{"osd":3,"up_from":0,"seq":2847563317561,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986695,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1037156,"kb_used_data":727644,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92707996,"statfs":{"total":95995035648,"available":94932987904,"internally_reserved":0,"allocated":745107456,"data_stored":744852416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:41:26 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.5169999999999999,"15min":1.143},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":1.216,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.75,"15min":1.3520000000000001},"min":{"1min":0.37,"5min":0.30099999999999999,"15min":0.29899999999999999},"max":{"1min":4.0069999999999997,"5min":16.387,"15min":16.387},"last":0.32600000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":2.6259999999999999,"15min":2.1560000000000001},"min":{"1min":0.43099999999999999,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":1.143,"5min":64.671999999999997,"15min":85.186999999999998},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.745,"5min":3.1920000000000002,"15min":2.4399999999999999},"min":{"1min":0.40799999999999997,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":1.0640000000000001,"5min":64.576999999999998,"15min":85.159000000000006},"last":0.50600000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:40:45 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":1.625,"15min":1.804},"min":{"1min":0.39600000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.21,"5min":33.743000000000002,"15min":99.736000000000004},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.79700000000000004,"5min":1.887,"15min":1.829},"min":{"1min":0.46500000000000002,"5min":0.32100000000000001,"15min":0.30099999999999999},"max":{"1min":1.1759999999999999,"5min":33.805999999999997,"15min":99.212999999999994},"last":0.748}]},{"osd":3,"last update":"Mon Oct 9 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.3620000000000001,"15min":2.0190000000000001},"min":{"1min":0.48799999999999999,"5min":0.30199999999999999,"15min":0.26500000000000001},"max":{"1min":1.228,"5min":6.7549999999999999,"15min":47.32},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.78000000000000003,"5min":1.671,"15min":2.484},"min":{"1min":0.49099999999999999,"5min":0.36599999999999999,"15min":0.314},"max":{"1min":1.0800000000000001,"5min":16.507000000000001,"15min":93.843000000000004},"last":0.68999999999999995}]},{"osd":5,"last update":"Mon Oct 9 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":1.6259999999999999,"15min":1.395},"min":{"1min":0.40200000000000002,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":1.228,"5min":17.212,"15min":17.212},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.91200000000000003,"5min":1.919,"15min":1.502},"min":{"1min":0.53400000000000003,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":1.306,"5min":17.268999999999998,"15min":17.268999999999998},"last":0.55500000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":1.5449999999999999,"15min":1.345},"min":{"1min":0.46500000000000002,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":1.51,"5min":13.654,"15min":13.654},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.5369999999999999,"15min":1.357},"min":{"1min":0.52500000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.637,"5min":9.3369999999999997,"15min":9.3369999999999997},"last":0.75900000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:40:55 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.321,"15min":1.2669999999999999},"min":{"1min":0.54400000000000004,"5min":0.40100000000000002,"15min":0.33400000000000002},"max":{"1min":1.3480000000000001,"5min":9.2889999999999997,"15min":9.2889999999999997},"last":0.80400000000000005},{"interface":"front","average":{"1min":1.137,"5min":1.3340000000000001,"15min":1.4219999999999999},"min":{"1min":0.48499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.9510000000000001,"5min":7.8579999999999997,"15min":12.34},"last":3.742}]}]},{"osd":5,"up_from":37,"seq":158913790461,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2056464,"kb_used_data":1462664,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91688688,"statfs":{"total":95995035648,"available":93889216512,"internally_reserved":0,"allocated":1497767936,"data_stored":1497529050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:41:29 2023","interfaces":[{"interface":"back","average":{"1min":6.9279999999999999,"5min":3.6989999999999998,"15min":5.7590000000000003},"min":{"1min":0.49099999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":119.724,"5min":119.724,"15min":119.724},"last":1.603},{"interface":"front","average":{"1min":6.9889999999999999,"5min":3.5529999999999999,"15min":5.1230000000000002},"min":{"1min":0.40999999999999998,"5min":0.245,"15min":0.245},"max":{"1min":119.872,"5min":119.872,"15min":119.872},"last":2.5659999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":6.7240000000000002,"15min":3.6099999999999999},"min":{"1min":0.42099999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.7210000000000001,"5min":142.67500000000001,"15min":142.67500000000001},"last":7.4640000000000004},{"interface":"front","average":{"1min":0.871,"5min":6.7839999999999998,"15min":3.6909999999999998},"min":{"1min":0.55800000000000005,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.5700000000000001,"5min":142.56399999999999,"15min":142.56399999999999},"last":7.4889999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:41:02 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.347,"15min":1.544},"min":{"1min":0.503,"5min":0.312,"15min":0.31},"max":{"1min":1.488,"5min":7.7110000000000003,"15min":14.019},"last":2.641},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.9350000000000001,"15min":1.835},"min":{"1min":0.43099999999999999,"5min":0.38100000000000001,"15min":0.28000000000000003},"max":{"1min":1.4770000000000001,"5min":13.356,"15min":14.037000000000001},"last":1.6699999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":1.452,"15min":1.9419999999999999},"min":{"1min":0.41299999999999998,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":1.056,"5min":10.125999999999999,"15min":59.652000000000001},"last":2.4140000000000001},{"interface":"front","average":{"1min":0.91200000000000003,"5min":1.6240000000000001,"15min":1.9810000000000001},"min":{"1min":0.47799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.8050000000000002,"5min":14.087,"15min":61.771999999999998},"last":1.641}]},{"osd":4,"last update":"Mon Oct 9 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.1819999999999999,"15min":1.1200000000000001},"min":{"1min":0.34599999999999997,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":1.2430000000000001,"5min":8.4019999999999992,"15min":10.747},"last":2.2509999999999999},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.8700000000000001,"15min":1.3720000000000001},"min":{"1min":0.371,"5min":0.33300000000000002,"15min":0.29799999999999999},"max":{"1min":1.3799999999999999,"5min":12.151,"15min":12.151},"last":2.9199999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:41:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.4330000000000001,"15min":1.3819999999999999},"min":{"1min":0.442,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":1.341,"5min":8.1989999999999998,"15min":14.518000000000001},"last":2.1949999999999998},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.98,"15min":1.694},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.32900000000000001},"max":{"1min":1.6279999999999999,"5min":18.638999999999999,"15min":18.638999999999999},"last":1.623}]},{"osd":7,"last update":"Mon Oct 9 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":2.1000000000000001,"15min":1.5900000000000001},"min":{"1min":0.39700000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.407,"5min":18.683,"15min":18.683},"last":1.573},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.613,"15min":1.4910000000000001},"min":{"1min":0.49199999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.369,"5min":10.192,"15min":10.192},"last":2.6040000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1052114944,"data_stored":1052114944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1612316672,"data_stored":1612316672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":839319552,"data_stored":839319552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":455016448,"data_stored":455016448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":755695616,"data_stored":755695616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1516830720,"data_stored":1516830720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1027604480,"data_stored":1027604480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:41:37.027 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T09:41:37.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-10-09T09:41:37.058 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 43975 43959 85.8491 81.0625 0.0111205 0.0116423 2023-10-09T09:41:37.109 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:37.140 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:37.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:37.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:37.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:37.256 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:36 smithi181 bash[59345]: audit 2023-10-09T09:41:35.211825+0000 mgr.y (mgr.34107) 1065 : audit 0 from='client.35436 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:41:37.271 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:37.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:37.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:37.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:37.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:37.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:37.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:37.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:37.538 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:37.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:37.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:37.641 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:37.672 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:37.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:37.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:37.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:37.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:37.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:37.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:37.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:37.919 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:37 smithi163 bash[75425]: cluster 2023-10-09T09:41:35.983015+0000 mgr.y (mgr.34107) 1066 : cluster 0 pgmap v1859: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s; 5279/117966 objects degraded (4.475%); 473/117966 objects misplaced (0.401%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:41:37.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:37 smithi163 bash[78107]: cluster 2023-10-09T09:41:35.983015+0000 mgr.y (mgr.34107) 1066 : cluster 0 pgmap v1859: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s; 5279/117966 objects degraded (4.475%); 473/117966 objects misplaced (0.401%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:41:37.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:37.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:38.039 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:38.058 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 45273 45257 85.7057 81.125 0.0123574 0.0116607 2023-10-09T09:41:38.063 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:38.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:37 smithi181 bash[59345]: cluster 2023-10-09T09:41:35.983015+0000 mgr.y (mgr.34107) 1066 : cluster 0 pgmap v1859: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s; 5279/117966 objects degraded (4.475%); 473/117966 objects misplaced (0.401%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:41:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:38.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:38.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:38.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:38.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:38.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:38.301 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:38.329 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:38.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:38.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:38.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:38.463 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:38.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:38.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:38.504 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:41:38.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:38.600 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:38.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:38.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:38.703 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:38.748 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:38.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:38.851 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:38.889 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:38.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:38.958 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:41:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:41:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35412.0:46379 9.b e1195/1179 9:df76fcca:::benchmark_data_smithi163_175771_object46378:head v 1195'2913, mlcod=1195'2911)", 2023-10-09T09:41:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:41:38.949970+0000", 2023-10-09T09:41:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0074608959999999998, 2023-10-09T09:41:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0074841150000000004, 2023-10-09T09:41:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:41:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.949970+0000", 2023-10-09T09:41:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.949970+0000", 2023-10-09T09:41:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:41:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.949969+0000", 2023-10-09T09:41:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8734999999999997e-05 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950048+0000", 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2734e-05 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950061+0000", 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.93e-06 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950066+0000", 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1859000000000005e-05 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950128+0000", 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0142000000000001e-05 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950148+0000", 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0072802520000000001 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.957429+0000", 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0074589720000000003 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35412.0:46377 9.2 e1195/1190 9:4ae267e7:::benchmark_data_smithi163_175771_object46376:head v 1195'2818, mlcod=0'0)", 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:41:38.950857+0000", 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0065738899999999998, 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066702630000000001, 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950857+0000", 2023-10-09T09:41:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950857+0000", 2023-10-09T09:41:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:41:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950856+0000", 2023-10-09T09:41:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3168000000000001e-05 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950929+0000", 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1046999999999999e-05 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950940+0000", 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0270000000000001e-06 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.950946+0000", 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028476980000000001 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.953794+0000", 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7078999999999999e-05 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.953821+0000", 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029642520000000001 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35412.0:46381 9.3 e1195/1190 9:c35c018d:::benchmark_data_smithi163_175771_object46380:head v 1195'2918, mlcod=0'0)", 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:41:38.954519+0000", 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0029119900000000002, 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030490719999999999, 2023-10-09T09:41:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.954519+0000", 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.954519+0000", 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:41:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.954518+0000", 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1791000000000003e-05 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.954610+0000", 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1197e-05 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.954621+0000", 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8370000000000003e-06 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.954630+0000", 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2524e-05 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.954692+0000", 2023-10-09T09:41:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8915999999999999e-05 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.954721+0000", 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000202547 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35412.0:46382 9.0 e1195/1179 9:05375716:::benchmark_data_smithi163_175771_object46381:head v 1195'2892, mlcod=1195'2890)", 2023-10-09T09:41:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:41:38.955450+0000", 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0019807650000000002, 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021577129999999999, 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.955450+0000", 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.955450+0000", 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.955449+0000", 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010324500000000001 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:41:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.955552+0000", 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0257e-05 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.955563+0000", 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1589999999999998e-06 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.955569+0000", 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2614000000000001e-05 2023-10-09T09:41:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.955621+0000", 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0435999999999999e-05 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:41:38.955642+0000", 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019200599999999999 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:41:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:41:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T09:41:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 4 2023-10-09T09:41:38.978 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:41:38.997 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:39.030 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:39.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:39.058 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 46503 46487 85.4458 76.875 0.0131834 0.0116978 2023-10-09T09:41:39.135 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:39.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:39.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:39.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:39.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:39.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:39.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:39.407 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:39.434 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:39.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:39.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:39.538 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:39.564 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:39.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:39.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:39.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:39.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:39.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:39.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:39.803 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:39.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:39.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:39.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:39.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:39.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:39.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:40.058 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 47700 47684 85.1418 74.8125 0.00735254 0.0117386 2023-10-09T09:41:40.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:40.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:39 smithi163 bash[75425]: cluster 2023-10-09T09:41:37.983892+0000 mgr.y (mgr.34107) 1067 : cluster 0 pgmap v1860: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 5279/127950 objects degraded (4.126%); 356/127950 objects misplaced (0.278%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:41:40.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:39 smithi163 bash[78107]: cluster 2023-10-09T09:41:37.983892+0000 mgr.y (mgr.34107) 1067 : cluster 0 pgmap v1860: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 5279/127950 objects degraded (4.126%); 356/127950 objects misplaced (0.278%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:41:40.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:40.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:40.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:40.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:40.209 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:39 smithi181 bash[59345]: cluster 2023-10-09T09:41:37.983892+0000 mgr.y (mgr.34107) 1067 : cluster 0 pgmap v1860: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 5279/127950 objects degraded (4.126%); 356/127950 objects misplaced (0.278%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:41:40.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:40.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:40.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:40.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T09:41:40.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:40.375 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:40.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:40.481 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:40.505 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:40.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:40.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:40.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:40.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:40.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:40.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:40.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:40.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:40.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:40.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:40.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:40.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:41.029 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:41.058 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 48677 48661 84.4728 61.0625 0.0109241 0.0118318 2023-10-09T09:41:41.065 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:41.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:41.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:41.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:41.199 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:41.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:41.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:41.305 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:41.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:41.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:41.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:41.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:41.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:41.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:41.556 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:41:41.557 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:41:41.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:41 smithi163 bash[78107]: cluster 2023-10-09T09:41:39.984705+0000 mgr.y (mgr.34107) 1068 : cluster 0 pgmap v1861: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 5279/133746 objects degraded (3.947%); 356/133746 objects misplaced (0.266%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:41:41.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:41 smithi163 bash[78107]: audit 2023-10-09T09:41:40.391311+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:41.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:41 smithi163 bash[78107]: audit 2023-10-09T09:41:40.395117+0000 mon.a (mon.0) 2052 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:41.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:41 smithi163 bash[78107]: cluster 2023-10-09T09:41:40.755424+0000 mon.a (mon.0) 2053 : cluster 3 Health check update: Degraded data redundancy: 5279/127950 objects degraded (4.126%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:41.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:41 smithi163 bash[75425]: cluster 2023-10-09T09:41:39.984705+0000 mgr.y (mgr.34107) 1068 : cluster 0 pgmap v1861: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 5279/133746 objects degraded (3.947%); 356/133746 objects misplaced (0.266%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:41:41.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:41 smithi163 bash[75425]: audit 2023-10-09T09:41:40.391311+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:41.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:41 smithi163 bash[75425]: audit 2023-10-09T09:41:40.395117+0000 mon.a (mon.0) 2052 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:41.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:41 smithi163 bash[75425]: cluster 2023-10-09T09:41:40.755424+0000 mon.a (mon.0) 2053 : cluster 3 Health check update: Degraded data redundancy: 5279/127950 objects degraded (4.126%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:41.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:41.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:41.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:41.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:41.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:41.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:41.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:41.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:41 smithi181 bash[59345]: cluster 2023-10-09T09:41:39.984705+0000 mgr.y (mgr.34107) 1068 : cluster 0 pgmap v1861: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 5279/133746 objects degraded (3.947%); 356/133746 objects misplaced (0.266%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:41:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:41 smithi181 bash[59345]: audit 2023-10-09T09:41:40.391311+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:41:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:41 smithi181 bash[59345]: audit 2023-10-09T09:41:40.395117+0000 mon.a (mon.0) 2052 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:41 smithi181 bash[59345]: cluster 2023-10-09T09:41:40.755424+0000 mon.a (mon.0) 2053 : cluster 3 Health check update: Degraded data redundancy: 5279/127950 objects degraded (4.126%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:41.844 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:41.872 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:41.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:41.976 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:42.012 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:42.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:42.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:42.058 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 49839 49823 84.1524 72.625 0.00741988 0.0118771 2023-10-09T09:41:42.117 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:42.148 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:42.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:42.255 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:42.281 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:42.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:42.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:42.410 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:42.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:42.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:42.543 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:42.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:42.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:42.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:42.670 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:42.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:42.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:42.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:42.803 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:42.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:42.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:42.909 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:42.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:42.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:42.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:43.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:43.058 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 51003 50987 83.8521 72.75 0.0150864 0.0119195 2023-10-09T09:41:43.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:43.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:43.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:43.184 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:43.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:43.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:43.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:43.316 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:43.341 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:43.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:43.444 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:43.473 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:43.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:43.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:43.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:43.588 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:43 smithi181 bash[59345]: cluster 2023-10-09T09:41:41.985603+0000 mgr.y (mgr.34107) 1069 : cluster 0 pgmap v1862: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s; 5279/141660 objects degraded (3.727%); 240/141660 objects misplaced (0.169%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T09:41:43.605 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:43.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:43.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:43.710 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:41:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:41:43] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.33.4" 2023-10-09T09:41:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:43 smithi163 bash[78107]: cluster 2023-10-09T09:41:41.985603+0000 mgr.y (mgr.34107) 1069 : cluster 0 pgmap v1862: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s; 5279/141660 objects degraded (3.727%); 240/141660 objects misplaced (0.169%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T09:41:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:43 smithi163 bash[75425]: cluster 2023-10-09T09:41:41.985603+0000 mgr.y (mgr.34107) 1069 : cluster 0 pgmap v1862: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s; 5279/141660 objects degraded (3.727%); 240/141660 objects misplaced (0.169%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T09:41:43.738 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:43.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:43.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:43.842 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:43.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:43.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:43.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:43.974 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:44.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:44.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:44.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:44.058 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 52187 52171 83.5993 74 0.0130307 0.0119539 2023-10-09T09:41:44.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:44.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:44.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:44.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:44.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:44.268 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:44.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:44.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:44.373 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:44.397 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:44.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:44.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:44.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:44.529 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:44.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:44.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:44.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:44.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:44.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:44.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:44.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:44.791 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:44.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:44.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:44.897 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:44.920 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:44.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:44.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:45.026 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:45.058 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:41:45.058985+0000 min lat: 0.00132859 max lat: 2.09983 avg lat: 0.0119841 2023-10-09T09:41:45.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:41:45.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 53395 53379 83.3966 75.5 0.0104368 0.0119841 2023-10-09T09:41:45.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:45.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:45.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:45.123 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:41:45.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:45.194 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:45.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:45.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:45.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:45.326 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:45.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:45.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:45.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:45.472 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:45.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:45.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:45 smithi181 bash[59345]: cluster 2023-10-09T09:41:43.986242+0000 mgr.y (mgr.34107) 1070 : cluster 0 pgmap v1863: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s; 5279/145947 objects degraded (3.617%); 240/145947 objects misplaced (0.164%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:45.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:45.607 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:45.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:45.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:45.715 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:45.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:45 smithi163 bash[75425]: cluster 2023-10-09T09:41:43.986242+0000 mgr.y (mgr.34107) 1070 : cluster 0 pgmap v1863: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s; 5279/145947 objects degraded (3.617%); 240/145947 objects misplaced (0.164%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:45.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:45 smithi163 bash[78107]: cluster 2023-10-09T09:41:43.986242+0000 mgr.y (mgr.34107) 1070 : cluster 0 pgmap v1863: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s; 5279/145947 objects degraded (3.617%); 240/145947 objects misplaced (0.164%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:45.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:45.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:45.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:45.846 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:45.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:45.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:45.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:45.980 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:46.011 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:46.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:46.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:46.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 54492 54476 83.0347 68.5625 0.0199373 0.0120369 2023-10-09T09:41:46.117 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:46.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:46 smithi163 bash[75425]: cluster 2023-10-09T09:41:45.756564+0000 mon.a (mon.0) 2054 : cluster 3 Health check update: Degraded data redundancy: 5279/145947 objects degraded (3.617%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:46.485 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:46.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:46.590 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:46.625 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:46.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:46.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:46.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:46 smithi163 bash[78107]: cluster 2023-10-09T09:41:45.756564+0000 mon.a (mon.0) 2054 : cluster 3 Health check update: Degraded data redundancy: 5279/145947 objects degraded (3.617%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:46.730 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:46.738 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:46 smithi181 bash[59345]: cluster 2023-10-09T09:41:45.756564+0000 mon.a (mon.0) 2054 : cluster 3 Health check update: Degraded data redundancy: 5279/145947 objects degraded (3.617%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:46.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:46.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:46.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:46.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:46.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:46.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:46.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:47.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:47.030 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:47.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:47.060 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 55585 55569 82.684 68.3125 0.004823 0.0120603 2023-10-09T09:41:47.133 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:47.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:47.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:47.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:47.265 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:47.295 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:47.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:47.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:47.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:47.422 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:47.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:47.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:47.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:47.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:47.561 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:47.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:47.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:47.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:47.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:47.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:47.801 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:47.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:47.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:47.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:47.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:47.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:47 smithi181 bash[59345]: cluster 2023-10-09T09:41:45.986938+0000 mgr.y (mgr.34107) 1071 : cluster 0 pgmap v1864: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 5279/153510 objects degraded (3.439%); 240/153510 objects misplaced (0.156%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:47.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:47.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:48.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:47 smithi163 bash[78107]: cluster 2023-10-09T09:41:45.986938+0000 mgr.y (mgr.34107) 1071 : cluster 0 pgmap v1864: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 5279/153510 objects degraded (3.439%); 240/153510 objects misplaced (0.156%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:48.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:47 smithi163 bash[75425]: cluster 2023-10-09T09:41:45.986938+0000 mgr.y (mgr.34107) 1071 : cluster 0 pgmap v1864: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 5279/153510 objects degraded (3.439%); 240/153510 objects misplaced (0.156%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:48.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 56527 56511 82.1302 58.875 0.0183494 0.0121696 2023-10-09T09:41:48.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:48.102 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:48.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:48.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:48.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:48.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:48.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:48.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:48.334 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:48.364 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:48.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:48.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:48.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:48.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:48.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:48.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:48.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:48.634 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:48.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:48.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:48.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:48.777 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:48.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:48.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:48.886 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:48.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:48.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:48.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:49.017 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:49.054 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:49.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:49.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:49.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 57674 57658 81.8927 71.6875 0.0138547 0.0122043 2023-10-09T09:41:49.160 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:49.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:49.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:49.302 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:49.333 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:49.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:49.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:49.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:49.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:49.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:49.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:49.566 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:49.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:49.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:49.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:49.702 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:49.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:49.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:49.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:49.838 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:49.859 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:49.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:49.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:49.895 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:41:49.938 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:41:49.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:49.981 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:41:49.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:49.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:49.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:49.997 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:49 smithi181 bash[59345]: cluster 2023-10-09T09:41:47.987752+0000 mgr.y (mgr.34107) 1072 : cluster 0 pgmap v1865: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 5279/163113 objects degraded (3.236%); 123/163113 objects misplaced (0.075%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:41:50.017 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:41:50.052 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:41:50.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 58819 58803 81.6629 71.5625 0.0147612 0.0122381 2023-10-09T09:41:50.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:50.100 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:41:50.106 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:49 smithi163 bash[78107]: cluster 2023-10-09T09:41:47.987752+0000 mgr.y (mgr.34107) 1072 : cluster 0 pgmap v1865: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 5279/163113 objects degraded (3.236%); 123/163113 objects misplaced (0.075%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:41:50.106 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:49 smithi163 bash[75425]: cluster 2023-10-09T09:41:47.987752+0000 mgr.y (mgr.34107) 1072 : cluster 0 pgmap v1865: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 5279/163113 objects degraded (3.236%); 123/163113 objects misplaced (0.075%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:41:50.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:50.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:50.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:50.139 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:41:50.175 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:41:50.179 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:41:50.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:41:50.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:41:50.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:41:50.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:41:50.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:41:50.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:41:50.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:41:50.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:50.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:50.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:50.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:50.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:50.419 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:50.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:50.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:50.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:50.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:50.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:50.574 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:41:50.574 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:41:50.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:50.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:50.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:50.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:50.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:50 smithi181 bash[59345]: audit 2023-10-09T09:41:49.844167+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.163:0/4207131807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:41:50.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:50 smithi181 bash[59345]: cluster 2023-10-09T09:41:50.758763+0000 mon.a (mon.0) 2055 : cluster 3 Health check update: Degraded data redundancy: 5279/163113 objects degraded (3.236%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:50.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:50.836 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:50.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:50.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:50.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:50.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:50.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:50.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:51.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 59914 59898 81.3753 68.4375 0.0098291 0.0122827 2023-10-09T09:41:51.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:51.092 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:51.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:51.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:51.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:51.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:50 smithi163 bash[75425]: audit 2023-10-09T09:41:49.844167+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.163:0/4207131807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:41:51.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:50 smithi163 bash[75425]: cluster 2023-10-09T09:41:50.758763+0000 mon.a (mon.0) 2055 : cluster 3 Health check update: Degraded data redundancy: 5279/163113 objects degraded (3.236%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:50 smithi163 bash[78107]: audit 2023-10-09T09:41:49.844167+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.163:0/4207131807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:41:51.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:50 smithi163 bash[78107]: cluster 2023-10-09T09:41:50.758763+0000 mon.a (mon.0) 2055 : cluster 3 Health check update: Degraded data redundancy: 5279/163113 objects degraded (3.236%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:51.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:51.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:51.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:51.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:51.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:51.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:51.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:51.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:51.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:51.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:51.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:51.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:51.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:51.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:51.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:51.768 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:51.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:51.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:51.788 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:51 smithi181 bash[59345]: cluster 2023-10-09T09:41:49.988383+0000 mgr.y (mgr.34107) 1073 : cluster 0 pgmap v1866: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s; 5279/168417 objects degraded (3.134%); 123/168417 objects misplaced (0.073%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:51.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:51.897 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T09:41:51.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:51.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:52.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:52.017 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:51 smithi163 bash[78107]: cluster 2023-10-09T09:41:49.988383+0000 mgr.y (mgr.34107) 1073 : cluster 0 pgmap v1866: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s; 5279/168417 objects degraded (3.134%); 123/168417 objects misplaced (0.073%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:52.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:51 smithi163 bash[75425]: cluster 2023-10-09T09:41:49.988383+0000 mgr.y (mgr.34107) 1073 : cluster 0 pgmap v1866: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s; 5279/168417 objects degraded (3.134%); 123/168417 objects misplaced (0.073%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:52.031 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:52.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:52.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:52.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 60738 60722 80.7396 51.5 0.00382095 0.0123337 2023-10-09T09:41:52.135 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:52.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:52.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:52.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:52.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:52.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:52.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:52.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:52.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:52.428 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:52.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:52.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:52.564 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:52.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:52.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:52.667 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:52.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:52.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:52.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:52.800 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:52.827 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:52.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:52.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:52.930 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:52.961 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:52.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:52.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:53.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 60995 60979 79.3921 16.0625 0.0197235 0.0125893 2023-10-09T09:41:53.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:52 smithi181 bash[59345]: cluster 2023-10-09T09:41:51.794473+0000 osd.5 (osd.5) 773 : cluster 3 failed to encode map e1196 with expected crc 2023-10-09T09:41:53.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:52 smithi181 bash[59345]: cluster 2023-10-09T09:41:51.797229+0000 mon.a (mon.0) 2056 : cluster 0 osdmap e1196: 8 total, 8 up, 6 in 2023-10-09T09:41:53.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:52 smithi181 bash[59345]: cluster 2023-10-09T09:41:51.801244+0000 osd.4 (osd.4) 619 : cluster 3 failed to encode map e1196 with expected crc 2023-10-09T09:41:53.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:53.098 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:53.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:53.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:53.203 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:53.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:52 smithi163 bash[75425]: cluster 2023-10-09T09:41:51.794473+0000 osd.5 (osd.5) 773 : cluster 3 failed to encode map e1196 with expected crc 2023-10-09T09:41:53.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:52 smithi163 bash[75425]: cluster 2023-10-09T09:41:51.797229+0000 mon.a (mon.0) 2056 : cluster 0 osdmap e1196: 8 total, 8 up, 6 in 2023-10-09T09:41:53.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:52 smithi163 bash[75425]: cluster 2023-10-09T09:41:51.801244+0000 osd.4 (osd.4) 619 : cluster 3 failed to encode map e1196 with expected crc 2023-10-09T09:41:53.215 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:52 smithi163 bash[78107]: cluster 2023-10-09T09:41:51.794473+0000 osd.5 (osd.5) 773 : cluster 3 failed to encode map e1196 with expected crc 2023-10-09T09:41:53.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:52 smithi163 bash[78107]: cluster 2023-10-09T09:41:51.797229+0000 mon.a (mon.0) 2056 : cluster 0 osdmap e1196: 8 total, 8 up, 6 in 2023-10-09T09:41:53.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:52 smithi163 bash[78107]: cluster 2023-10-09T09:41:51.801244+0000 osd.4 (osd.4) 619 : cluster 3 failed to encode map e1196 with expected crc 2023-10-09T09:41:53.237 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:53.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:53.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:53.342 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:53.361 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:53.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:53.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:53.463 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:53.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:41:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:41:53] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.33.4" 2023-10-09T09:41:53.490 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:53.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:53.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:53.595 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:53.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:53.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:53.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:53.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:53.775 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:53.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:53.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:53.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:53 smithi163 bash[75425]: cluster 2023-10-09T09:41:51.988901+0000 mgr.y (mgr.34107) 1074 : cluster 0 pgmap v1868: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 70 MiB/s wr, 1.13k op/s; 5279/175476 objects degraded (3.008%); 7/175476 objects misplaced (0.004%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:41:53.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:53.906 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:53.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:53.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:54.010 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:54.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:54.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:54.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:54.059 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 61990 61974 79.0409 62.1875 0.00949948 0.0126452 2023-10-09T09:41:54.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:53 smithi181 bash[59345]: cluster 2023-10-09T09:41:51.988901+0000 mgr.y (mgr.34107) 1074 : cluster 0 pgmap v1868: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 70 MiB/s wr, 1.13k op/s; 5279/175476 objects degraded (3.008%); 7/175476 objects misplaced (0.004%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:41:54.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:53 smithi181 bash[59345]: cluster 2023-10-09T09:41:52.801880+0000 osd.5 (osd.5) 774 : cluster 3 failed to encode map e1197 with expected crc 2023-10-09T09:41:54.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:53 smithi181 bash[59345]: cluster 2023-10-09T09:41:52.801978+0000 osd.4 (osd.4) 620 : cluster 3 failed to encode map e1197 with expected crc 2023-10-09T09:41:54.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:53 smithi181 bash[59345]: cluster 2023-10-09T09:41:52.804778+0000 mon.a (mon.0) 2057 : cluster 0 osdmap e1197: 8 total, 8 up, 6 in 2023-10-09T09:41:54.116 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:53 smithi163 bash[75425]: cluster 2023-10-09T09:41:52.801880+0000 osd.5 (osd.5) 774 : cluster 3 failed to encode map e1197 with expected crc 2023-10-09T09:41:54.117 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:53 smithi163 bash[75425]: cluster 2023-10-09T09:41:52.801978+0000 osd.4 (osd.4) 620 : cluster 3 failed to encode map e1197 with expected crc 2023-10-09T09:41:54.117 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:53 smithi163 bash[75425]: cluster 2023-10-09T09:41:52.804778+0000 mon.a (mon.0) 2057 : cluster 0 osdmap e1197: 8 total, 8 up, 6 in 2023-10-09T09:41:54.117 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:53 smithi163 bash[78107]: cluster 2023-10-09T09:41:51.988901+0000 mgr.y (mgr.34107) 1074 : cluster 0 pgmap v1868: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 70 MiB/s wr, 1.13k op/s; 5279/175476 objects degraded (3.008%); 7/175476 objects misplaced (0.004%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:41:54.117 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:53 smithi163 bash[78107]: cluster 2023-10-09T09:41:52.801880+0000 osd.5 (osd.5) 774 : cluster 3 failed to encode map e1197 with expected crc 2023-10-09T09:41:54.117 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:53 smithi163 bash[78107]: cluster 2023-10-09T09:41:52.801978+0000 osd.4 (osd.4) 620 : cluster 3 failed to encode map e1197 with expected crc 2023-10-09T09:41:54.117 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:53 smithi163 bash[78107]: cluster 2023-10-09T09:41:52.804778+0000 mon.a (mon.0) 2057 : cluster 0 osdmap e1197: 8 total, 8 up, 6 in 2023-10-09T09:41:54.145 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:54.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:54.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:54.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:54.298 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:54.330 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:54.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:54.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:54.434 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:54.462 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:54.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:54.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:54.566 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:54.593 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:54.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:54.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:54.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:54.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:54.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:54.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:54.830 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:54.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:54 smithi181 bash[59345]: audit 2023-10-09T09:41:54.059207+0000 mon.a (mon.0) 2058 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:41:54.862 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:54.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:54.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:54.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:54 smithi163 bash[78107]: audit 2023-10-09T09:41:54.059207+0000 mon.a (mon.0) 2058 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:41:54.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:54 smithi163 bash[75425]: audit 2023-10-09T09:41:54.059207+0000 mon.a (mon.0) 2058 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:41:54.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:54.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:54.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:55.060 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 62970 62954 78.6849 61.25 0.0172258 0.0127027 2023-10-09T09:41:55.096 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:55.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:55.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:55.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:55.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:55.256 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:55.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:55.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:55.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:55.388 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:55.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:55.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:55.493 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:55.525 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:55.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:55.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:55.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:55.664 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:41:55.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:55.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:41:55.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:41:55.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:55.797 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:55.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:55.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:55.839 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 9 pg_num to 26 2023-10-09T09:41:55.850 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:41:55.896 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[78107]: cluster 2023-10-09T09:41:53.989609+0000 mgr.y (mgr.34107) 1075 : cluster 0 pgmap v1870: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s; 5279/179295 objects degraded (2.944%); 7/179295 objects misplaced (0.004%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:41:55.896 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[78107]: audit 2023-10-09T09:41:55.386461+0000 mon.a (mon.0) 2059 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:55.896 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[78107]: audit 2023-10-09T09:41:55.405012+0000 mon.a (mon.0) 2060 : audit 1 from='client.? 172.21.15.163:0/9601076' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:41:55.896 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[78107]: audit 2023-10-09T09:41:55.477697+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.163:0/1401498281' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:41:55.896 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[78107]: audit 2023-10-09T09:41:55.479416+0000 mon.a (mon.0) 2061 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:41:55.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[78107]: cluster 2023-10-09T09:41:55.759948+0000 mon.a (mon.0) 2062 : cluster 3 Health check update: Degraded data redundancy: 5279/179295 objects degraded (2.944%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:55.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[75425]: cluster 2023-10-09T09:41:53.989609+0000 mgr.y (mgr.34107) 1075 : cluster 0 pgmap v1870: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s; 5279/179295 objects degraded (2.944%); 7/179295 objects misplaced (0.004%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:41:55.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[75425]: audit 2023-10-09T09:41:55.386461+0000 mon.a (mon.0) 2059 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:55.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[75425]: audit 2023-10-09T09:41:55.405012+0000 mon.a (mon.0) 2060 : audit 1 from='client.? 172.21.15.163:0/9601076' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:41:55.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[75425]: audit 2023-10-09T09:41:55.477697+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.163:0/1401498281' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:41:55.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[75425]: audit 2023-10-09T09:41:55.479416+0000 mon.a (mon.0) 2061 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:41:55.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:55 smithi163 bash[75425]: cluster 2023-10-09T09:41:55.759948+0000 mon.a (mon.0) 2062 : cluster 3 Health check update: Degraded data redundancy: 5279/179295 objects degraded (2.944%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:55.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:55.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:55.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:55.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:56.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:56.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 63950 63934 78.3429 61.25 0.00906543 0.0127586 2023-10-09T09:41:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:55 smithi181 bash[59345]: cluster 2023-10-09T09:41:53.989609+0000 mgr.y (mgr.34107) 1075 : cluster 0 pgmap v1870: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s; 5279/179295 objects degraded (2.944%); 7/179295 objects misplaced (0.004%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:41:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:55 smithi181 bash[59345]: audit 2023-10-09T09:41:55.386461+0000 mon.a (mon.0) 2059 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:41:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:55 smithi181 bash[59345]: audit 2023-10-09T09:41:55.405012+0000 mon.a (mon.0) 2060 : audit 1 from='client.? 172.21.15.163:0/9601076' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T09:41:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:55 smithi181 bash[59345]: audit 2023-10-09T09:41:55.477697+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.163:0/1401498281' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:41:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:55 smithi181 bash[59345]: audit 2023-10-09T09:41:55.479416+0000 mon.a (mon.0) 2061 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:41:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:55 smithi181 bash[59345]: cluster 2023-10-09T09:41:55.759948+0000 mon.a (mon.0) 2062 : cluster 3 Health check update: Degraded data redundancy: 5279/179295 objects degraded (2.944%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:41:56.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:56.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:56.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:56.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:56.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:56.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:56.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:56.324 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:56.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:56.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:56.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:56.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:56.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:56.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:56.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:56.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:56.630 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:56.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:56.712 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:41:56.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:56.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:56.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:56.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:56.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:56.920 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:56.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:56.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:57.030 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:57.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[75425]: cluster 2023-10-09T09:41:55.811624+0000 mon.a (mon.0) 2063 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:41:57.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[75425]: cluster 2023-10-09T09:41:55.825255+0000 osd.5 (osd.5) 775 : cluster 3 failed to encode map e1198 with expected crc 2023-10-09T09:41:57.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[75425]: audit 2023-10-09T09:41:55.829264+0000 mon.a (mon.0) 2064 : audit 1 from='client.? 172.21.15.163:0/9601076' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:41:57.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[75425]: audit 2023-10-09T09:41:55.830929+0000 mon.a (mon.0) 2065 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:41:57.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[75425]: cluster 2023-10-09T09:41:55.831039+0000 mon.a (mon.0) 2066 : cluster 0 osdmap e1198: 8 total, 8 up, 6 in 2023-10-09T09:41:57.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[75425]: audit 2023-10-09T09:41:55.991350+0000 mon.a (mon.0) 2067 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:41:57.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[78107]: cluster 2023-10-09T09:41:55.811624+0000 mon.a (mon.0) 2063 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:41:57.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[78107]: cluster 2023-10-09T09:41:55.825255+0000 osd.5 (osd.5) 775 : cluster 3 failed to encode map e1198 with expected crc 2023-10-09T09:41:57.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[78107]: audit 2023-10-09T09:41:55.829264+0000 mon.a (mon.0) 2064 : audit 1 from='client.? 172.21.15.163:0/9601076' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:41:57.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[78107]: audit 2023-10-09T09:41:55.830929+0000 mon.a (mon.0) 2065 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:41:57.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[78107]: cluster 2023-10-09T09:41:55.831039+0000 mon.a (mon.0) 2066 : cluster 0 osdmap e1198: 8 total, 8 up, 6 in 2023-10-09T09:41:57.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:56 smithi163 bash[78107]: audit 2023-10-09T09:41:55.991350+0000 mon.a (mon.0) 2067 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:41:57.062 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 64787 64771 77.8381 52.3125 0.0895353 0.0127939 2023-10-09T09:41:57.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:57.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:57.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:57.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:57.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:57.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:57.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:56 smithi181 bash[59345]: cluster 2023-10-09T09:41:55.811624+0000 mon.a (mon.0) 2063 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:41:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:56 smithi181 bash[59345]: cluster 2023-10-09T09:41:55.825255+0000 osd.5 (osd.5) 775 : cluster 3 failed to encode map e1198 with expected crc 2023-10-09T09:41:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:56 smithi181 bash[59345]: audit 2023-10-09T09:41:55.829264+0000 mon.a (mon.0) 2064 : audit 1 from='client.? 172.21.15.163:0/9601076' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T09:41:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:56 smithi181 bash[59345]: audit 2023-10-09T09:41:55.830929+0000 mon.a (mon.0) 2065 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:41:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:56 smithi181 bash[59345]: cluster 2023-10-09T09:41:55.831039+0000 mon.a (mon.0) 2066 : cluster 0 osdmap e1198: 8 total, 8 up, 6 in 2023-10-09T09:41:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:56 smithi181 bash[59345]: audit 2023-10-09T09:41:55.991350+0000 mon.a (mon.0) 2067 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T09:41:57.322 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:57.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:57.424 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:57.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:57.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:57.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:57.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:57.672 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:41:57.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:57.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:41:57.775 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:57.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:57.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:57.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:57.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:57.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:41:57.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:57.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:41:58.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:57 smithi163 bash[78107]: cluster 2023-10-09T09:41:55.990120+0000 mgr.y (mgr.34107) 1076 : cluster 0 pgmap v1872: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 907 op/s; 5279/184752 objects degraded (2.857%); 7/184752 objects misplaced (0.004%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:57 smithi163 bash[78107]: cluster 2023-10-09T09:41:56.831602+0000 osd.5 (osd.5) 776 : cluster 3 failed to encode map e1199 with expected crc 2023-10-09T09:41:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:57 smithi163 bash[78107]: audit 2023-10-09T09:41:56.834893+0000 mon.a (mon.0) 2068 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:41:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:57 smithi163 bash[78107]: cluster 2023-10-09T09:41:56.834980+0000 mon.a (mon.0) 2069 : cluster 0 osdmap e1199: 8 total, 8 up, 6 in 2023-10-09T09:41:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:57 smithi163 bash[78107]: cluster 2023-10-09T09:41:56.835960+0000 osd.7 (osd.7) 577 : cluster 3 failed to encode map e1199 with expected crc 2023-10-09T09:41:58.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:57 smithi163 bash[75425]: cluster 2023-10-09T09:41:55.990120+0000 mgr.y (mgr.34107) 1076 : cluster 0 pgmap v1872: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 907 op/s; 5279/184752 objects degraded (2.857%); 7/184752 objects misplaced (0.004%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:58.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:57 smithi163 bash[75425]: cluster 2023-10-09T09:41:56.831602+0000 osd.5 (osd.5) 776 : cluster 3 failed to encode map e1199 with expected crc 2023-10-09T09:41:58.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:57 smithi163 bash[75425]: audit 2023-10-09T09:41:56.834893+0000 mon.a (mon.0) 2068 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:41:58.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:57 smithi163 bash[75425]: cluster 2023-10-09T09:41:56.834980+0000 mon.a (mon.0) 2069 : cluster 0 osdmap e1199: 8 total, 8 up, 6 in 2023-10-09T09:41:58.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:57 smithi163 bash[75425]: cluster 2023-10-09T09:41:56.835960+0000 osd.7 (osd.7) 577 : cluster 3 failed to encode map e1199 with expected crc 2023-10-09T09:41:58.059 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:58.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:58.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:58.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 64856 64840 76.4509 4.3125 0.00315448 0.0130022 2023-10-09T09:41:58.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:58.185 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:58.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:58.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:58.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:57 smithi181 bash[59345]: cluster 2023-10-09T09:41:55.990120+0000 mgr.y (mgr.34107) 1076 : cluster 0 pgmap v1872: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 907 op/s; 5279/184752 objects degraded (2.857%); 7/184752 objects misplaced (0.004%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:41:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:57 smithi181 bash[59345]: cluster 2023-10-09T09:41:56.831602+0000 osd.5 (osd.5) 776 : cluster 3 failed to encode map e1199 with expected crc 2023-10-09T09:41:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:57 smithi181 bash[59345]: audit 2023-10-09T09:41:56.834893+0000 mon.a (mon.0) 2068 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T09:41:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:57 smithi181 bash[59345]: cluster 2023-10-09T09:41:56.834980+0000 mon.a (mon.0) 2069 : cluster 0 osdmap e1199: 8 total, 8 up, 6 in 2023-10-09T09:41:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:57 smithi181 bash[59345]: cluster 2023-10-09T09:41:56.835960+0000 osd.7 (osd.7) 577 : cluster 3 failed to encode map e1199 with expected crc 2023-10-09T09:41:58.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:58.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:58.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:58.423 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:58.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:58.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:58.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:58.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:58.581 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:41:58.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:58.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:41:58.685 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:58.711 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:58.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:58.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:58.815 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:58.842 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:58.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:58.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:58.946 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:58.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:58 smithi181 bash[59345]: cluster 2023-10-09T09:41:57.839173+0000 osd.5 (osd.5) 777 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:58.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:58 smithi181 bash[59345]: cluster 2023-10-09T09:41:57.841281+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e1200: 8 total, 8 up, 6 in 2023-10-09T09:41:58.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:58 smithi181 bash[59345]: cluster 2023-10-09T09:41:57.841445+0000 osd.6 (osd.6) 484 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:58.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:58 smithi181 bash[59345]: cluster 2023-10-09T09:41:57.842820+0000 osd.5 (osd.5) 778 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:58.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:58 smithi181 bash[59345]: cluster 2023-10-09T09:41:57.843792+0000 osd.6 (osd.6) 485 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:58.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:58.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:58.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:59.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 65077 65061 75.291 13.8125 0.0114492 0.0132757 2023-10-09T09:41:59.091 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:59.101 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:58 smithi163 bash[75425]: cluster 2023-10-09T09:41:57.839173+0000 osd.5 (osd.5) 777 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:59.101 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:58 smithi163 bash[75425]: cluster 2023-10-09T09:41:57.841281+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e1200: 8 total, 8 up, 6 in 2023-10-09T09:41:59.101 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:58 smithi163 bash[75425]: cluster 2023-10-09T09:41:57.841445+0000 osd.6 (osd.6) 484 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:59.101 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:58 smithi163 bash[75425]: cluster 2023-10-09T09:41:57.842820+0000 osd.5 (osd.5) 778 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:59.101 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:58 smithi163 bash[75425]: cluster 2023-10-09T09:41:57.843792+0000 osd.6 (osd.6) 485 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:59.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:58 smithi163 bash[78107]: cluster 2023-10-09T09:41:57.839173+0000 osd.5 (osd.5) 777 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:59.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:58 smithi163 bash[78107]: cluster 2023-10-09T09:41:57.841281+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e1200: 8 total, 8 up, 6 in 2023-10-09T09:41:59.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:58 smithi163 bash[78107]: cluster 2023-10-09T09:41:57.841445+0000 osd.6 (osd.6) 484 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:59.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:58 smithi163 bash[78107]: cluster 2023-10-09T09:41:57.842820+0000 osd.5 (osd.5) 778 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:59.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:58 smithi163 bash[78107]: cluster 2023-10-09T09:41:57.843792+0000 osd.6 (osd.6) 485 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:41:59.120 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:41:59.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:59.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:41:59.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:59.249 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:59.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:59.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:59.357 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:59.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:59.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:59.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:59.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:59.534 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:59.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:59.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:41:59.639 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:59.668 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:41:59.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:59.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:41:59.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:59.798 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:41:59.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:59.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:41:59.903 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:59.933 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:41:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:41:59.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:00.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:00.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:00.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:00.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:00.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 66360 66344 75.3799 80.1875 0.0115218 0.0132607 2023-10-09T09:42:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:59 smithi181 bash[59345]: cluster 2023-10-09T09:41:57.990702+0000 mgr.y (mgr.34107) 1077 : cluster 0 pgmap v1875: 99 pgs: 1 remapped+peering, 2 peering, 10 unknown, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 49 MiB/s wr, 787 op/s; 3767/191553 objects degraded (1.967%); 613/191553 objects misplaced (0.320%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:42:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:59 smithi181 bash[59345]: cluster 2023-10-09T09:41:58.752557+0000 osd.7 (osd.7) 578 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:42:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:59 smithi181 bash[59345]: cluster 2023-10-09T09:41:58.846232+0000 osd.5 (osd.5) 779 : cluster 3 failed to encode map e1201 with expected crc 2023-10-09T09:42:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:59 smithi181 bash[59345]: cluster 2023-10-09T09:41:58.855048+0000 mon.a (mon.0) 2071 : cluster 0 osdmap e1201: 8 total, 8 up, 6 in 2023-10-09T09:42:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:59 smithi181 bash[59345]: cluster 2023-10-09T09:41:58.860355+0000 osd.6 (osd.6) 486 : cluster 3 failed to encode map e1201 with expected crc 2023-10-09T09:42:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:41:59 smithi181 bash[59345]: cluster 2023-10-09T09:41:58.862059+0000 osd.4 (osd.4) 621 : cluster 3 failed to encode map e1201 with expected crc 2023-10-09T09:42:00.155 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:42:00.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:00.164 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[75425]: cluster 2023-10-09T09:41:57.990702+0000 mgr.y (mgr.34107) 1077 : cluster 0 pgmap v1875: 99 pgs: 1 remapped+peering, 2 peering, 10 unknown, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 49 MiB/s wr, 787 op/s; 3767/191553 objects degraded (1.967%); 613/191553 objects misplaced (0.320%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:42:00.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[75425]: cluster 2023-10-09T09:41:58.752557+0000 osd.7 (osd.7) 578 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:42:00.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[75425]: cluster 2023-10-09T09:41:58.846232+0000 osd.5 (osd.5) 779 : cluster 3 failed to encode map e1201 with expected crc 2023-10-09T09:42:00.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[75425]: cluster 2023-10-09T09:41:58.855048+0000 mon.a (mon.0) 2071 : cluster 0 osdmap e1201: 8 total, 8 up, 6 in 2023-10-09T09:42:00.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[75425]: cluster 2023-10-09T09:41:58.860355+0000 osd.6 (osd.6) 486 : cluster 3 failed to encode map e1201 with expected crc 2023-10-09T09:42:00.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[75425]: cluster 2023-10-09T09:41:58.862059+0000 osd.4 (osd.4) 621 : cluster 3 failed to encode map e1201 with expected crc 2023-10-09T09:42:00.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[78107]: cluster 2023-10-09T09:41:57.990702+0000 mgr.y (mgr.34107) 1077 : cluster 0 pgmap v1875: 99 pgs: 1 remapped+peering, 2 peering, 10 unknown, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 49 MiB/s wr, 787 op/s; 3767/191553 objects degraded (1.967%); 613/191553 objects misplaced (0.320%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:42:00.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[78107]: cluster 2023-10-09T09:41:58.752557+0000 osd.7 (osd.7) 578 : cluster 3 failed to encode map e1200 with expected crc 2023-10-09T09:42:00.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[78107]: cluster 2023-10-09T09:41:58.846232+0000 osd.5 (osd.5) 779 : cluster 3 failed to encode map e1201 with expected crc 2023-10-09T09:42:00.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[78107]: cluster 2023-10-09T09:41:58.855048+0000 mon.a (mon.0) 2071 : cluster 0 osdmap e1201: 8 total, 8 up, 6 in 2023-10-09T09:42:00.166 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[78107]: cluster 2023-10-09T09:41:58.860355+0000 osd.6 (osd.6) 486 : cluster 3 failed to encode map e1201 with expected crc 2023-10-09T09:42:00.166 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:41:59 smithi163 bash[78107]: cluster 2023-10-09T09:41:58.862059+0000 osd.4 (osd.4) 621 : cluster 3 failed to encode map e1201 with expected crc 2023-10-09T09:42:00.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:00.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:00.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:00.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:00.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:00.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:00.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:00.444 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:00.472 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:00.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:00.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:00.579 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:00.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:00.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:00.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:00.712 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:00.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:00.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:00.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:00.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:00.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:00.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:00.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:00.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:01.009 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:01.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:01.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 67519 67503 75.3272 72.4375 0.0161737 0.0132693 2023-10-09T09:42:01.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:01.122 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:00 smithi163 bash[75425]: cluster 2023-10-09T09:42:00.760962+0000 mon.a (mon.0) 2072 : cluster 3 Health check update: Degraded data redundancy: 3767/191553 objects degraded (1.967%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:42:01.122 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:00 smithi163 bash[78107]: cluster 2023-10-09T09:42:00.760962+0000 mon.a (mon.0) 2072 : cluster 3 Health check update: Degraded data redundancy: 3767/191553 objects degraded (1.967%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:42:01.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:01.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:01.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:01.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:01.257 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:00 smithi181 bash[59345]: cluster 2023-10-09T09:42:00.760962+0000 mon.a (mon.0) 2072 : cluster 3 Health check update: Degraded data redundancy: 3767/191553 objects degraded (1.967%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:42:01.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:01.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:01.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:01.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:01.410 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:01.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:01.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:01.516 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:01.548 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:01.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:01.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:01.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:01.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:01.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:01.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:01.713 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:01.759 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:42:01.759 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:42:01.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:01.829 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:01.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:01.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:01.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:01.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:01.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:02.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 68556 68540 75.1426 64.8125 0.0257459 0.0133016 2023-10-09T09:42:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:01 smithi181 bash[59345]: cluster 2023-10-09T09:41:59.991412+0000 mgr.y (mgr.34107) 1078 : cluster 0 pgmap v1877: 99 pgs: 5 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 6 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s; 3397/194532 objects degraded (1.746%); 613/194532 objects misplaced (0.315%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T09:42:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:01 smithi181 bash[59345]: audit 2023-10-09T09:42:01.418086+0000 mon.a (mon.0) 2073 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:01 smithi181 bash[59345]: audit 2023-10-09T09:42:01.428189+0000 mon.a (mon.0) 2074 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:02.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:02.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:02.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:02.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:02.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:01 smithi163 bash[75425]: cluster 2023-10-09T09:41:59.991412+0000 mgr.y (mgr.34107) 1078 : cluster 0 pgmap v1877: 99 pgs: 5 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 6 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s; 3397/194532 objects degraded (1.746%); 613/194532 objects misplaced (0.315%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T09:42:02.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:01 smithi163 bash[75425]: audit 2023-10-09T09:42:01.418086+0000 mon.a (mon.0) 2073 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:02.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:01 smithi163 bash[75425]: audit 2023-10-09T09:42:01.428189+0000 mon.a (mon.0) 2074 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:02.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:01 smithi163 bash[78107]: cluster 2023-10-09T09:41:59.991412+0000 mgr.y (mgr.34107) 1078 : cluster 0 pgmap v1877: 99 pgs: 5 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 6 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s; 3397/194532 objects degraded (1.746%); 613/194532 objects misplaced (0.315%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T09:42:02.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:01 smithi163 bash[78107]: audit 2023-10-09T09:42:01.418086+0000 mon.a (mon.0) 2073 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:02.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:01 smithi163 bash[78107]: audit 2023-10-09T09:42:01.428189+0000 mon.a (mon.0) 2074 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:02.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:02.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:02.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:02.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:02.363 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:02.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:02.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:02.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:02.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:02.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:02.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:02.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:02.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:02.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:02.792 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:02.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:02.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:02.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:02.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:02.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:02.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:02.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:02.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T09:42:03.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:03.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 69689 69673 75.0679 70.8125 0.0120432 0.0133149 2023-10-09T09:42:03.095 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:03.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:03.168 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 6, 2, 5] out_osds: [7, 3] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:42:03.168 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:42:03.169 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T09:42:03.169 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:42:03.201 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:03 smithi181 bash[59345]: cluster 2023-10-09T09:42:01.992245+0000 mgr.y (mgr.34107) 1079 : cluster 0 pgmap v1878: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 3 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 83 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 50 MiB/s wr, 794 op/s; 3397/199050 objects degraded (1.707%); 497/199050 objects misplaced (0.250%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:42:03.201 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:03 smithi181 bash[59345]: audit 2023-10-09T09:42:02.199788+0000 mon.a (mon.0) 2075 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:03.201 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:03 smithi181 bash[59345]: audit 2023-10-09T09:42:02.209195+0000 mon.a (mon.0) 2076 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:03.202 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:03 smithi181 bash[59345]: audit 2023-10-09T09:42:02.794514+0000 mon.a (mon.0) 2077 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:42:03.202 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:03 smithi181 bash[59345]: audit 2023-10-09T09:42:02.795387+0000 mon.a (mon.0) 2078 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:42:03.202 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:03 smithi181 bash[59345]: audit 2023-10-09T09:42:02.802173+0000 mon.a (mon.0) 2079 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:03.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:03.241 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:03.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:03.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:03.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:03.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:03.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:03.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[75425]: cluster 2023-10-09T09:42:01.992245+0000 mgr.y (mgr.34107) 1079 : cluster 0 pgmap v1878: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 3 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 83 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 50 MiB/s wr, 794 op/s; 3397/199050 objects degraded (1.707%); 497/199050 objects misplaced (0.250%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:42:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[75425]: audit 2023-10-09T09:42:02.199788+0000 mon.a (mon.0) 2075 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[75425]: audit 2023-10-09T09:42:02.209195+0000 mon.a (mon.0) 2076 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[75425]: audit 2023-10-09T09:42:02.794514+0000 mon.a (mon.0) 2077 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:42:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[75425]: audit 2023-10-09T09:42:02.795387+0000 mon.a (mon.0) 2078 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:42:03.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[75425]: audit 2023-10-09T09:42:02.802173+0000 mon.a (mon.0) 2079 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[78107]: cluster 2023-10-09T09:42:01.992245+0000 mgr.y (mgr.34107) 1079 : cluster 0 pgmap v1878: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 3 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 83 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 50 MiB/s wr, 794 op/s; 3397/199050 objects degraded (1.707%); 497/199050 objects misplaced (0.250%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T09:42:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[78107]: audit 2023-10-09T09:42:02.199788+0000 mon.a (mon.0) 2075 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[78107]: audit 2023-10-09T09:42:02.209195+0000 mon.a (mon.0) 2076 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[78107]: audit 2023-10-09T09:42:02.794514+0000 mon.a (mon.0) 2077 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:42:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[78107]: audit 2023-10-09T09:42:02.795387+0000 mon.a (mon.0) 2078 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:42:03.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[78107]: audit 2023-10-09T09:42:02.802173+0000 mon.a (mon.0) 2079 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:03.470 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:42:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:42:03] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.33.4" 2023-10-09T09:42:03.491 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:03.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:03.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:03.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:03.626 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:03.655 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:03.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:03.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:03.761 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:03.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:03.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:03.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:03.886 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:03.917 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:03.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:03.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:04.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:04.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 70954 70938 75.1355 79.0625 0.0151075 0.0133037 2023-10-09T09:42:04.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:04.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:04.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:04.169 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:04.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:04.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:04.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:04.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:04.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:04.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:04.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:04.445 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:04.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:04.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:04.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:04.582 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:04.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:04.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:04.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:04.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:04.735 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:04.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:04.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:04.766 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:04.839 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:04.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:04.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:04.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:04.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:05.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:05.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:05.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:05.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:42:05.063648+0000 min lat: 0.00132859 max lat: 2.09983 avg lat: 0.0132829 2023-10-09T09:42:05.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:42:05.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 72271 72255 75.2549 82.3125 0.00656355 0.0132829 2023-10-09T09:42:05.109 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:05.137 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:05.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:05.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:05.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:05.263 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:05.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:05.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:05.368 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:05.378 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:05 smithi163 bash[78107]: cluster 2023-10-09T09:42:03.992953+0000 mgr.y (mgr.34107) 1080 : cluster 0 pgmap v1879: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 40 MiB/s wr, 632 op/s; 4186/202233 objects degraded (2.070%); 497/202233 objects misplaced (0.246%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T09:42:05.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:05 smithi163 bash[75425]: cluster 2023-10-09T09:42:03.992953+0000 mgr.y (mgr.34107) 1080 : cluster 0 pgmap v1879: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 40 MiB/s wr, 632 op/s; 4186/202233 objects degraded (2.070%); 497/202233 objects misplaced (0.246%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T09:42:05.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:05.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:05.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:05.509 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:05.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:05 smithi181 bash[59345]: cluster 2023-10-09T09:42:03.992953+0000 mgr.y (mgr.34107) 1080 : cluster 0 pgmap v1879: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 40 MiB/s wr, 632 op/s; 4186/202233 objects degraded (2.070%); 497/202233 objects misplaced (0.246%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T09:42:05.549 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:05.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:05.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:05.676 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:05.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:05.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:05.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:05.803 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:05.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:05.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:05.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:05.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:05.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:06.040 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:06.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 73467 73451 75.2465 74.75 0.0174872 0.0132839 2023-10-09T09:42:06.067 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:06.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:06.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:06.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:06.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:06.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:06.310 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:06 smithi181 bash[59345]: cluster 2023-10-09T09:42:05.762437+0000 mon.a (mon.0) 2080 : cluster 3 Health check update: Degraded data redundancy: 4186/202233 objects degraded (2.070%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T09:42:06.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:06.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:06.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:06.446 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:06.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:06 smithi163 bash[75425]: cluster 2023-10-09T09:42:05.762437+0000 mon.a (mon.0) 2080 : cluster 3 Health check update: Degraded data redundancy: 4186/202233 objects degraded (2.070%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T09:42:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:06 smithi163 bash[78107]: cluster 2023-10-09T09:42:05.762437+0000 mon.a (mon.0) 2080 : cluster 3 Health check update: Degraded data redundancy: 4186/202233 objects degraded (2.070%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T09:42:06.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:06.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:06.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:06.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:06.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:06.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:06.720 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:06.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:06.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:06.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:06.854 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:06.883 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:06.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:06.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:06.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:07.016 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:07.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:07.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:07.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 74713 74697 75.2888 77.875 0.00893812 0.0132766 2023-10-09T09:42:07.120 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:07.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:07.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:07.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:07.258 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:07.284 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:07.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:07.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:07.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:07.412 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:07.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:07.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:07.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:07.547 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:07.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:07.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:07.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:07.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:07.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:07.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:07.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:07.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:07 smithi163 bash[78107]: cluster 2023-10-09T09:42:05.993962+0000 mgr.y (mgr.34107) 1081 : cluster 0 pgmap v1880: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 753 op/s; 4186/209988 objects degraded (1.993%); 497/209988 objects misplaced (0.237%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:42:07.813 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:07.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:07.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:07.861 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:42:07.918 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:07.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:07.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:07.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:08.053 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:08.062 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:07 smithi163 bash[75425]: cluster 2023-10-09T09:42:05.993962+0000 mgr.y (mgr.34107) 1081 : cluster 0 pgmap v1880: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 753 op/s; 4186/209988 objects degraded (1.993%); 497/209988 objects misplaced (0.237%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:42:08.063 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 75952 75936 75.3228 77.4375 0.010193 0.0132702 2023-10-09T09:42:08.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:07 smithi181 bash[59345]: cluster 2023-10-09T09:42:05.993962+0000 mgr.y (mgr.34107) 1081 : cluster 0 pgmap v1880: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 753 op/s; 4186/209988 objects degraded (1.993%); 497/209988 objects misplaced (0.237%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:42:08.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:08.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:08.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:08.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:08.215 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:08.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:08.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:08.318 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:08.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:08.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:08.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:08.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:08.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:08.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:08.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:08.587 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:08.624 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:08.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:08.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:08.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:08.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:08.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:08.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:08.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:08.887 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:08.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:08.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:08.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:08.999 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:09.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:09.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:09.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:09.064 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 77194 77178 75.3586 77.625 0.0107329 0.0132613 2023-10-09T09:42:09.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:09.146 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:09.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:09.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:09.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:09.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:09.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:09.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:09.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:09.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:09.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:09.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:09.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:09.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:09.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:09.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:09.668 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:09.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:09.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:09.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:09.780 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:09 smithi181 bash[59345]: cluster 2023-10-09T09:42:07.994902+0000 mgr.y (mgr.34107) 1082 : cluster 0 pgmap v1881: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 60 MiB/s wr, 960 op/s; 4186/220389 objects degraded (1.899%); 381/220389 objects misplaced (0.173%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:42:09.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:09.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:09.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:09.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:09.914 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:09 smithi163 bash[75425]: cluster 2023-10-09T09:42:07.994902+0000 mgr.y (mgr.34107) 1082 : cluster 0 pgmap v1881: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 60 MiB/s wr, 960 op/s; 4186/220389 objects degraded (1.899%); 381/220389 objects misplaced (0.173%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:42:09.915 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:09 smithi163 bash[78107]: cluster 2023-10-09T09:42:07.994902+0000 mgr.y (mgr.34107) 1082 : cluster 0 pgmap v1881: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 60 MiB/s wr, 960 op/s; 4186/220389 objects degraded (1.899%); 381/220389 objects misplaced (0.173%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:42:09.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:09.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:09.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:10.035 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:10.064 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 78171 78155 75.1386 61.0625 0.01878 0.0133032 2023-10-09T09:42:10.069 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:10.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:10.173 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:10.202 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:10.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:10.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:10.306 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:10.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:10.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:10.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:10.434 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:10.460 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:10.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:10.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:10.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:10.601 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:10.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:10.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:10.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:10.731 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:10.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:10.835 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:10.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:10.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:10.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:10.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:10.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:10.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:10.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:11.064 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 79409 79393 75.1723 77.375 0.0171533 0.0132971 2023-10-09T09:42:11.095 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:11.118 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:11.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:11.228 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:11.255 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:11.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:11.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:11.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:11.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:11.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:11.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:11.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:11.519 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:11.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:11.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:11 smithi181 bash[59345]: cluster 2023-10-09T09:42:09.995622+0000 mgr.y (mgr.34107) 1083 : cluster 0 pgmap v1882: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s; 4186/225957 objects degraded (1.853%); 381/225957 objects misplaced (0.169%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:42:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:11 smithi181 bash[59345]: audit 2023-10-09T09:42:10.392076+0000 mon.a (mon.0) 2081 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:11 smithi181 bash[59345]: audit 2023-10-09T09:42:10.396065+0000 mon.a (mon.0) 2082 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:11 smithi181 bash[59345]: cluster 2023-10-09T09:42:10.763592+0000 mon.a (mon.0) 2083 : cluster 3 Health check update: Degraded data redundancy: 4186/220389 objects degraded (1.899%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T09:42:11.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:11.631 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:11 smithi163 bash[78107]: cluster 2023-10-09T09:42:09.995622+0000 mgr.y (mgr.34107) 1083 : cluster 0 pgmap v1882: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s; 4186/225957 objects degraded (1.853%); 381/225957 objects misplaced (0.169%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:42:11.631 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:11 smithi163 bash[78107]: audit 2023-10-09T09:42:10.392076+0000 mon.a (mon.0) 2081 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:11.631 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:11 smithi163 bash[78107]: audit 2023-10-09T09:42:10.396065+0000 mon.a (mon.0) 2082 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:11.631 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:11 smithi163 bash[78107]: cluster 2023-10-09T09:42:10.763592+0000 mon.a (mon.0) 2083 : cluster 3 Health check update: Degraded data redundancy: 4186/220389 objects degraded (1.899%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T09:42:11.632 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:11 smithi163 bash[75425]: cluster 2023-10-09T09:42:09.995622+0000 mgr.y (mgr.34107) 1083 : cluster 0 pgmap v1882: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s; 4186/225957 objects degraded (1.853%); 381/225957 objects misplaced (0.169%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:42:11.632 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:11 smithi163 bash[75425]: audit 2023-10-09T09:42:10.392076+0000 mon.a (mon.0) 2081 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:42:11.632 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:11 smithi163 bash[75425]: audit 2023-10-09T09:42:10.396065+0000 mon.a (mon.0) 2082 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:11.632 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:11 smithi163 bash[75425]: cluster 2023-10-09T09:42:10.763592+0000 mon.a (mon.0) 2083 : cluster 3 Health check update: Degraded data redundancy: 4186/220389 objects degraded (1.899%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T09:42:11.646 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:11.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:11.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:11.749 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:11.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:11.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:11.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:11.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:11.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:11.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:11.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:12.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:12.034 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:12.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:12.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:12.064 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 80627 80611 75.1864 76.125 0.0087794 0.0132947 2023-10-09T09:42:12.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:12.174 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:12.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:12.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:12.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:12.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:12.322 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:12.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:12.423 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:12.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:12.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:12.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:12.563 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:12.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:12.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:12.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:12.729 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:12.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:12.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:12.832 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:12.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:12.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:12.976 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:13.011 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:42:13.012 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:42:13.015 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:13.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:13.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:13.064 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 81764 81748 75.1257 71.0625 0.015628 0.0133051 2023-10-09T09:42:13.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:13.151 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:13.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:13.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:13.256 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:13.283 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:13.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:13.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:13.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:13.396 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:42:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:42:13] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.33.4" 2023-10-09T09:42:13.399 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:13 smithi181 bash[59345]: cluster 2023-10-09T09:42:11.996423+0000 mgr.y (mgr.34107) 1084 : cluster 0 pgmap v1883: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 4186/233682 objects degraded (1.791%); 265/233682 objects misplaced (0.113%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:42:13.416 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:13.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:13.519 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:13.531 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:13 smithi163 bash[75425]: cluster 2023-10-09T09:42:11.996423+0000 mgr.y (mgr.34107) 1084 : cluster 0 pgmap v1883: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 4186/233682 objects degraded (1.791%); 265/233682 objects misplaced (0.113%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:42:13.531 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:13 smithi163 bash[78107]: cluster 2023-10-09T09:42:11.996423+0000 mgr.y (mgr.34107) 1084 : cluster 0 pgmap v1883: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 4186/233682 objects degraded (1.791%); 265/233682 objects misplaced (0.113%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:42:13.547 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:13.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:13.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:13.653 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:13.683 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:13.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:13.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:13.788 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:13.823 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:13.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:13.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:13.929 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:13.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:13.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:13.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:14.061 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:14.064 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 82973 82957 75.1319 75.5625 0.00880713 0.0133041 2023-10-09T09:42:14.088 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:14.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:14.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:14.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T09:42:14.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:14.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:14.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:14.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:14.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:14.371 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:14.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:14.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:14.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:14.501 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:14.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:14.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:14.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:14.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:14.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:14.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:14.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:14.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:14.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:14.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:14.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:14.900 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:14.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:14.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:15.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:15.029 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:15.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:15.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:15.064 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 84182 84166 75.138 75.5625 0.0127433 0.0133035 2023-10-09T09:42:15.135 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:15.161 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:15.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:15.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:15.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:15.309 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:15.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:15.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:15.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:15 smithi163 bash[75425]: cluster 2023-10-09T09:42:13.997333+0000 mgr.y (mgr.34107) 1085 : cluster 0 pgmap v1884: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s; 4186/237954 objects degraded (1.759%); 265/237954 objects misplaced (0.111%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:42:15.403 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:15 smithi163 bash[78107]: cluster 2023-10-09T09:42:13.997333+0000 mgr.y (mgr.34107) 1085 : cluster 0 pgmap v1884: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s; 4186/237954 objects degraded (1.759%); 265/237954 objects misplaced (0.111%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:42:15.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:15.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:15.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:15.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:15.543 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:15.567 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:15.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:15.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:15.671 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:15.682 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:15 smithi181 bash[59345]: cluster 2023-10-09T09:42:13.997333+0000 mgr.y (mgr.34107) 1085 : cluster 0 pgmap v1884: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s; 4186/237954 objects degraded (1.759%); 265/237954 objects misplaced (0.111%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:42:15.700 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:15.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:15.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:15.803 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:15.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:15.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:15.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:15.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:15.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:15.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:15.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:16.064 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 85147 85131 74.9291 60.3125 0.00723734 0.0133403 2023-10-09T09:42:16.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:16.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:16.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:16.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:16.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:16.237 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:16.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:16.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:16.362 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:16.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:16.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:16.466 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:16.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:16.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:16.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:16.526 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:16 smithi163 bash[75425]: cluster 2023-10-09T09:42:15.764939+0000 mon.a (mon.0) 2084 : cluster 3 Health check update: Degraded data redundancy: 4186/237954 objects degraded (1.759%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:42:16.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:16 smithi163 bash[78107]: cluster 2023-10-09T09:42:15.764939+0000 mon.a (mon.0) 2084 : cluster 3 Health check update: Degraded data redundancy: 4186/237954 objects degraded (1.759%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:42:16.595 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:16.617 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:16.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:16.720 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:16.729 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:16 smithi181 bash[59345]: cluster 2023-10-09T09:42:15.764939+0000 mon.a (mon.0) 2084 : cluster 3 Health check update: Degraded data redundancy: 4186/237954 objects degraded (1.759%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:42:16.748 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:16.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:16.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:16.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:16.880 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:16.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:16.987 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:17.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:17.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:17.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:17.064 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 86128 86112 74.7399 61.3125 0.00644208 0.013374 2023-10-09T09:42:17.123 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:17.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:17.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:17.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:17.164 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:42:17.200 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:42:17.238 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:42:17.251 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:17.276 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:17.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:17.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:17.285 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:42:17.323 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:42:17.358 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:42:17.380 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:17.396 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:42:17.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:17.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:17.426 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:42:17.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:42:17.431 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:42:17.432 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:42:17.432 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:42:17.432 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:42:17.432 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:42:17.432 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:42:17.432 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:42:17.520 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:17.546 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:17.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:17.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:17.685 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:17.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:17.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:17 smithi181 bash[59345]: cluster 2023-10-09T09:42:15.998101+0000 mgr.y (mgr.34107) 1086 : cluster 0 pgmap v1885: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s; 4186/245967 objects degraded (1.702%); 265/245967 objects misplaced (0.108%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:42:17.775 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:17 smithi181 bash[59345]: audit 2023-10-09T09:42:17.116897+0000 mon.a (mon.0) 2085 : audit 0 from='client.? 172.21.15.163:0/3785175590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:42:17.790 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:17.819 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:17.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:17.924 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:17.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:17 smithi163 bash[75425]: cluster 2023-10-09T09:42:15.998101+0000 mgr.y (mgr.34107) 1086 : cluster 0 pgmap v1885: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s; 4186/245967 objects degraded (1.702%); 265/245967 objects misplaced (0.108%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:42:17.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:17 smithi163 bash[75425]: audit 2023-10-09T09:42:17.116897+0000 mon.a (mon.0) 2085 : audit 0 from='client.? 172.21.15.163:0/3785175590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:42:17.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:17 smithi163 bash[78107]: cluster 2023-10-09T09:42:15.998101+0000 mgr.y (mgr.34107) 1086 : cluster 0 pgmap v1885: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s; 4186/245967 objects degraded (1.702%); 265/245967 objects misplaced (0.108%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:42:17.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:17 smithi163 bash[78107]: audit 2023-10-09T09:42:17.116897+0000 mon.a (mon.0) 2085 : audit 0 from='client.? 172.21.15.163:0/3785175590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:42:17.959 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:17.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:17.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:18.064 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:18.065 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 87155 87139 74.5953 64.1875 0.0727031 0.0133898 2023-10-09T09:42:18.095 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:18.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:18.201 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:18.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:18.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:18.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:18.337 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:18.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:18.366 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:18.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:18.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:18.496 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:18.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:18.601 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:18.624 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:18.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:18.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:18.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:18.751 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:18.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:18.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:18.801 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:42:18.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:18.865 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:18 smithi181 bash[59345]: audit 2023-10-09T09:42:18.035243+0000 mon.a (mon.0) 2086 : audit 1 from='client.? 172.21.15.163:0/834949307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:42:18.884 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:18.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:18.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:18.990 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:19.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:19.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:19.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:19.035 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:42:19.064 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 88310 88294 74.5627 72.1875 0.00687599 0.0134051 2023-10-09T09:42:19.135 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:19.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:19.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:19.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:18 smithi163 bash[78107]: audit 2023-10-09T09:42:18.035243+0000 mon.a (mon.0) 2086 : audit 1 from='client.? 172.21.15.163:0/834949307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:42:19.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:18 smithi163 bash[75425]: audit 2023-10-09T09:42:18.035243+0000 mon.a (mon.0) 2086 : audit 1 from='client.? 172.21.15.163:0/834949307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:42:19.274 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:19.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:19.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:19.403 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:19.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:19.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:19.535 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:19.564 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:19.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:19.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:19.668 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:19.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:19.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:19.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:19.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:19.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:19 smithi163 bash[75425]: cluster 2023-10-09T09:42:17.998954+0000 mgr.y (mgr.34107) 1087 : cluster 0 pgmap v1886: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 4186/255504 objects degraded (1.638%); 148/255504 objects misplaced (0.058%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:42:19.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:19 smithi163 bash[75425]: cluster 2023-10-09T09:42:18.773943+0000 mon.a (mon.0) 2087 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:42:19.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:19 smithi163 bash[75425]: cluster 2023-10-09T09:42:18.788857+0000 osd.5 (osd.5) 780 : cluster 3 failed to encode map e1202 with expected crc 2023-10-09T09:42:19.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:19 smithi163 bash[75425]: audit 2023-10-09T09:42:18.791221+0000 mon.a (mon.0) 2088 : audit 1 from='client.? 172.21.15.163:0/834949307' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:42:19.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:19 smithi163 bash[75425]: cluster 2023-10-09T09:42:18.791322+0000 mon.a (mon.0) 2089 : cluster 0 osdmap e1202: 8 total, 8 up, 6 in 2023-10-09T09:42:19.819 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:19 smithi163 bash[78107]: cluster 2023-10-09T09:42:17.998954+0000 mgr.y (mgr.34107) 1087 : cluster 0 pgmap v1886: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 4186/255504 objects degraded (1.638%); 148/255504 objects misplaced (0.058%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:42:19.820 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:19 smithi163 bash[78107]: cluster 2023-10-09T09:42:18.773943+0000 mon.a (mon.0) 2087 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:42:19.820 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:19 smithi163 bash[78107]: cluster 2023-10-09T09:42:18.788857+0000 osd.5 (osd.5) 780 : cluster 3 failed to encode map e1202 with expected crc 2023-10-09T09:42:19.820 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:19 smithi163 bash[78107]: audit 2023-10-09T09:42:18.791221+0000 mon.a (mon.0) 2088 : audit 1 from='client.? 172.21.15.163:0/834949307' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:42:19.820 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:19 smithi163 bash[78107]: cluster 2023-10-09T09:42:18.791322+0000 mon.a (mon.0) 2089 : cluster 0 osdmap e1202: 8 total, 8 up, 6 in 2023-10-09T09:42:19.842 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:19.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:19.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:19.946 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:19.952 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:19 smithi181 bash[59345]: cluster 2023-10-09T09:42:17.998954+0000 mgr.y (mgr.34107) 1087 : cluster 0 pgmap v1886: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 4186/255504 objects degraded (1.638%); 148/255504 objects misplaced (0.058%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T09:42:19.953 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:19 smithi181 bash[59345]: cluster 2023-10-09T09:42:18.773943+0000 mon.a (mon.0) 2087 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:42:19.953 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:19 smithi181 bash[59345]: cluster 2023-10-09T09:42:18.788857+0000 osd.5 (osd.5) 780 : cluster 3 failed to encode map e1202 with expected crc 2023-10-09T09:42:19.953 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:19 smithi181 bash[59345]: audit 2023-10-09T09:42:18.791221+0000 mon.a (mon.0) 2088 : audit 1 from='client.? 172.21.15.163:0/834949307' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:42:19.953 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:19 smithi181 bash[59345]: cluster 2023-10-09T09:42:18.791322+0000 mon.a (mon.0) 2089 : cluster 0 osdmap e1202: 8 total, 8 up, 6 in 2023-10-09T09:42:19.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:19.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:19.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:20.065 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 89387 89371 74.4659 67.3125 0.0641438 0.0134217 2023-10-09T09:42:20.079 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:20.107 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:20.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:20.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:20.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:20.245 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:20.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:20.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:20.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:20.380 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:20.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:20.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:20.484 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:20.510 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:20.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:20.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:20.622 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:20.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:20.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:20.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:20.743 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:20.767 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:20.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:20.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:20 smithi181 bash[59345]: cluster 2023-10-09T09:42:20.765987+0000 mon.a (mon.0) 2090 : cluster 3 Health check update: Degraded data redundancy: 4186/255504 objects degraded (1.638%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:42:20.876 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:20.885 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:20 smithi163 bash[78107]: cluster 2023-10-09T09:42:20.765987+0000 mon.a (mon.0) 2090 : cluster 3 Health check update: Degraded data redundancy: 4186/255504 objects degraded (1.638%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:42:20.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:20 smithi163 bash[75425]: cluster 2023-10-09T09:42:20.765987+0000 mon.a (mon.0) 2090 : cluster 3 Health check update: Degraded data redundancy: 4186/255504 objects degraded (1.638%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:42:20.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:20.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:20.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:21.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:21.037 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:21.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:21.065 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 90306 90290 74.2418 57.4375 0.010911 0.013463 2023-10-09T09:42:21.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:21.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:21.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:21.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:21.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:21.315 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:21.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:21.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:21.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:21.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:21.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:21.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:21.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:21.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:21.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:21.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:21.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:21.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:21.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:21.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:21.789 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:21 smithi181 bash[59345]: cluster 2023-10-09T09:42:19.999517+0000 mgr.y (mgr.34107) 1088 : cluster 0 pgmap v1888: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s; 4186/260265 objects degraded (1.608%); 148/260265 objects misplaced (0.057%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:42:21.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:21.843 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:21 smithi163 bash[75425]: cluster 2023-10-09T09:42:19.999517+0000 mgr.y (mgr.34107) 1088 : cluster 0 pgmap v1888: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s; 4186/260265 objects degraded (1.608%); 148/260265 objects misplaced (0.057%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:42:21.844 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:21 smithi163 bash[78107]: cluster 2023-10-09T09:42:19.999517+0000 mgr.y (mgr.34107) 1088 : cluster 0 pgmap v1888: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s; 4186/260265 objects degraded (1.608%); 148/260265 objects misplaced (0.057%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:42:21.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:21.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:21.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:21.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:21.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:21.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:21.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:22.065 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 91474 91458 74.2256 73 0.0126755 0.0134671 2023-10-09T09:42:22.096 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:22.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:22.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:22.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:22.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:22.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:22.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:22.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:22.400 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:22.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:22.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:22.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:22.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:22.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:22.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:42:22.636 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:22.668 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:22.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:22.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:22.775 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:22.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:22.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:22.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:22.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:22.945 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:22.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:22.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:23.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:23.065 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 92527 92511 74.1174 65.8125 0.00517658 0.013468 2023-10-09T09:42:23.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:23.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:23.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:23.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:23.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:23.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:23.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:23.313 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:23.339 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:23.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:23.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:23.443 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:23.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:23.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:23.578 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:23.587 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:42:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:42:23] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.33.4" 2023-10-09T09:42:23.601 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:23.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:23.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:23.638 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:42:23.639 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:42:23.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:23.732 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:23.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:23.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:23 smithi181 bash[59345]: cluster 2023-10-09T09:42:22.000296+0000 mgr.y (mgr.34107) 1089 : cluster 0 pgmap v1889: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 70 MiB/s wr, 1.11k op/s; 4186/267138 objects degraded (1.567%); 33/267138 objects misplaced (0.012%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:42:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:23 smithi181 bash[59345]: cluster 2023-10-09T09:42:22.808491+0000 osd.5 (osd.5) 781 : cluster 3 failed to encode map e1203 with expected crc 2023-10-09T09:42:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:23 smithi181 bash[59345]: cluster 2023-10-09T09:42:22.812751+0000 mon.a (mon.0) 2091 : cluster 0 osdmap e1203: 8 total, 8 up, 6 in 2023-10-09T09:42:23.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:23 smithi181 bash[59345]: cluster 2023-10-09T09:42:22.812893+0000 osd.5 (osd.5) 782 : cluster 3 failed to encode map e1203 with expected crc 2023-10-09T09:42:23.836 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:23.844 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:23 smithi163 bash[75425]: cluster 2023-10-09T09:42:22.000296+0000 mgr.y (mgr.34107) 1089 : cluster 0 pgmap v1889: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 70 MiB/s wr, 1.11k op/s; 4186/267138 objects degraded (1.567%); 33/267138 objects misplaced (0.012%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:42:23.844 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:23 smithi163 bash[75425]: cluster 2023-10-09T09:42:22.808491+0000 osd.5 (osd.5) 781 : cluster 3 failed to encode map e1203 with expected crc 2023-10-09T09:42:23.844 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:23 smithi163 bash[75425]: cluster 2023-10-09T09:42:22.812751+0000 mon.a (mon.0) 2091 : cluster 0 osdmap e1203: 8 total, 8 up, 6 in 2023-10-09T09:42:23.844 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:23 smithi163 bash[75425]: cluster 2023-10-09T09:42:22.812893+0000 osd.5 (osd.5) 782 : cluster 3 failed to encode map e1203 with expected crc 2023-10-09T09:42:23.844 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:23 smithi163 bash[78107]: cluster 2023-10-09T09:42:22.000296+0000 mgr.y (mgr.34107) 1089 : cluster 0 pgmap v1889: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 70 MiB/s wr, 1.11k op/s; 4186/267138 objects degraded (1.567%); 33/267138 objects misplaced (0.012%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:42:23.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:23 smithi163 bash[78107]: cluster 2023-10-09T09:42:22.808491+0000 osd.5 (osd.5) 781 : cluster 3 failed to encode map e1203 with expected crc 2023-10-09T09:42:23.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:23 smithi163 bash[78107]: cluster 2023-10-09T09:42:22.812751+0000 mon.a (mon.0) 2091 : cluster 0 osdmap e1203: 8 total, 8 up, 6 in 2023-10-09T09:42:23.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:23 smithi163 bash[78107]: cluster 2023-10-09T09:42:22.812893+0000 osd.5 (osd.5) 782 : cluster 3 failed to encode map e1203 with expected crc 2023-10-09T09:42:23.859 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:23.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:23.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:23.963 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:23.987 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:23.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:23.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:24.065 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 92847 92831 73.4324 20 0.00728237 0.0136118 2023-10-09T09:42:24.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:24.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:24.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:24.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:24.219 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:24.235 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:42:24.235 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:42:24.235 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:42:24.235 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:42:24.235 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:42:24.253 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:24.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:24.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:24.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:24.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:24.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:24.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:24.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:24.514 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:24.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:24.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:24.616 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:24.641 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:24.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:24.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:24.768 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:24.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:24.838 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:42:24.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:24.896 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:24.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:24.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:24.903 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:24 smithi181 bash[59345]: audit 2023-10-09T09:42:23.616774+0000 mgr.y (mgr.34107) 1090 : audit 0 from='client.45404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:42:24.903 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:24 smithi181 bash[59345]: cluster 2023-10-09T09:42:23.810066+0000 osd.5 (osd.5) 783 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:24.903 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:24 smithi181 bash[59345]: cluster 2023-10-09T09:42:23.812555+0000 osd.5 (osd.5) 784 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:24.903 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:24 smithi181 bash[59345]: cluster 2023-10-09T09:42:23.813890+0000 mon.a (mon.0) 2092 : cluster 0 osdmap e1204: 8 total, 8 up, 6 in 2023-10-09T09:42:24.903 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:24 smithi181 bash[59345]: cluster 2023-10-09T09:42:23.814955+0000 osd.5 (osd.5) 785 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:24.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:24 smithi163 bash[78107]: audit 2023-10-09T09:42:23.616774+0000 mgr.y (mgr.34107) 1090 : audit 0 from='client.45404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:42:24.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:24 smithi163 bash[78107]: cluster 2023-10-09T09:42:23.810066+0000 osd.5 (osd.5) 783 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:24 smithi163 bash[78107]: cluster 2023-10-09T09:42:23.812555+0000 osd.5 (osd.5) 784 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:24 smithi163 bash[78107]: cluster 2023-10-09T09:42:23.813890+0000 mon.a (mon.0) 2092 : cluster 0 osdmap e1204: 8 total, 8 up, 6 in 2023-10-09T09:42:24.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:24 smithi163 bash[78107]: cluster 2023-10-09T09:42:23.814955+0000 osd.5 (osd.5) 785 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:24.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:24 smithi163 bash[75425]: audit 2023-10-09T09:42:23.616774+0000 mgr.y (mgr.34107) 1090 : audit 0 from='client.45404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:42:24.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:24 smithi163 bash[75425]: cluster 2023-10-09T09:42:23.810066+0000 osd.5 (osd.5) 783 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:24.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:24 smithi163 bash[75425]: cluster 2023-10-09T09:42:23.812555+0000 osd.5 (osd.5) 784 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:24.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:24 smithi163 bash[75425]: cluster 2023-10-09T09:42:23.813890+0000 mon.a (mon.0) 2092 : cluster 0 osdmap e1204: 8 total, 8 up, 6 in 2023-10-09T09:42:24.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:24 smithi163 bash[75425]: cluster 2023-10-09T09:42:23.814955+0000 osd.5 (osd.5) 785 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:25.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:25.031 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:25.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:25.065 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:42:25.065807+0000 min lat: 0.00132859 max lat: 2.09983 avg lat: 0.0136711 2023-10-09T09:42:25.065 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:42:25.066 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 93617 93601 73.116 48.125 0.0108927 0.0136711 2023-10-09T09:42:25.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:25.167 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:25.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:25.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:25.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T09:42:25.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:25.333 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1890,"stamp":"2023-10-09T09:42:22.808483+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5924788128,"num_objects":90420,"num_object_clones":0,"num_object_copies":271260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4186,"num_objects_misplaced":33,"num_objects_unfound":0,"num_objects_dirty":90420,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":90656,"num_write_kb":5787789,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3365,"num_bytes_recovered":218635946,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129869,"ondisk_log_size":129869,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":20183516,"kb_used_data":16968812,"kb_used_omap":510,"kb_used_meta":3214145,"kb_avail":542287396,"statfs":{"total":575970213888,"available":555302293504,"internally_reserved":0,"allocated":17376063488,"data_stored":17371259472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":522608,"internal_metadata":3291285136},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,3,10,2,6,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":727580672,"num_objects":11102,"num_object_clones":0,"num_object_copies":33306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-232,"num_objects_unfound":0,"num_objects_dirty":11102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11102,"num_write_kb":710528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":15204352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.811849"},"pg_stats":[{"pgid":"9.10","version":"1202'4796","reported_seq":4837,"reported_epoch":1202,"state":"active+recovery_wait","last_fresh":"2023-10-09T09:42:19.783389+0000","last_change":"2023-10-09T09:41:58.856840+0000","last_active":"2023-10-09T09:42:19.783389+0000","last_peered":"2023-10-09T09:42:19.783389+0000","last_clean":"2023-10-09T09:41:56.844673+0000","last_became_active":"2023-10-09T09:41:57.943176+0000","last_became_peered":"2023-10-09T09:41:57.943176+0000","last_unstale":"2023-10-09T09:42:19.783389+0000","last_undegraded":"2023-10-09T09:42:19.783389+0000","last_fullsized":"2023-10-09T09:42:19.783389+0000","mapping_epoch":1199,"log_start":"0'0","ondisk_log_start":"0'0","created":1199,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4796,"log_dups_size":0,"ondisk_log_size":4796,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183369728,"num_objects":2798,"num_object_clones":0,"num_object_copies":8394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2798,"num_write_kb":179072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2798}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"61'1","reported_seq":1187,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869702+0000","last_change":"2023-10-09T09:36:43.418759+0000","last_active":"2023-10-09T09:41:58.869702+0000","last_peered":"2023-10-09T09:41:58.869702+0000","last_clean":"2023-10-09T09:41:58.869702+0000","last_became_active":"2023-10-09T09:36:43.418371+0000","last_became_peered":"2023-10-09T09:36:43.418371+0000","last_unstale":"2023-10-09T09:41:58.869702+0000","last_undegraded":"2023-10-09T09:41:58.869702+0000","last_fullsized":"2023-10-09T09:41:58.869702+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.861137+0000","last_change":"2023-10-09T09:41:11.811997+0000","last_active":"2023-10-09T09:41:58.861137+0000","last_peered":"2023-10-09T09:41:58.861137+0000","last_clean":"2023-10-09T09:41:58.861137+0000","last_became_active":"2023-10-09T09:41:11.811300+0000","last_became_peered":"2023-10-09T09:41:11.811300+0000","last_unstale":"2023-10-09T09:41:58.861137+0000","last_undegraded":"2023-10-09T09:41:58.861137+0000","last_fullsized":"2023-10-09T09:41:58.861137+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.346300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.13","version":"1202'4831","reported_seq":4876,"reported_epoch":1202,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T09:42:19.773031+0000","last_change":"2023-10-09T09:41:57.944334+0000","last_active":"2023-10-09T09:42:19.773031+0000","last_peered":"2023-10-09T09:42:19.773031+0000","last_clean":"2023-10-09T09:41:10.771882+0000","last_became_active":"2023-10-09T09:41:57.942268+0000","last_became_peered":"2023-10-09T09:41:57.942268+0000","last_unstale":"2023-10-09T09:42:19.773031+0000","last_undegraded":"2023-10-09T09:41:57.918142+0000","last_fullsized":"2023-10-09T09:42:19.773031+0000","mapping_epoch":1199,"log_start":"0'0","ondisk_log_start":"0'0","created":1199,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4831,"log_dups_size":0,"ondisk_log_size":4831,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184221696,"num_objects":2811,"num_object_clones":0,"num_object_copies":8433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":353,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2811,"num_write_kb":179904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":353},{"shards":"1,5,6","objects":2458}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1152,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.856953+0000","last_change":"2023-10-09T09:41:11.811870+0000","last_active":"2023-10-09T09:41:58.856953+0000","last_peered":"2023-10-09T09:41:58.856953+0000","last_clean":"2023-10-09T09:41:58.856953+0000","last_became_active":"2023-10-09T09:41:11.811570+0000","last_became_peered":"2023-10-09T09:41:11.811570+0000","last_unstale":"2023-10-09T09:41:58.856953+0000","last_undegraded":"2023-10-09T09:41:58.856953+0000","last_fullsized":"2023-10-09T09:41:58.856953+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:57.923729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1142,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.864246+0000","last_change":"2023-10-09T09:41:11.800661+0000","last_active":"2023-10-09T09:41:58.864246+0000","last_peered":"2023-10-09T09:41:58.864246+0000","last_clean":"2023-10-09T09:41:58.864246+0000","last_became_active":"2023-10-09T09:41:11.800279+0000","last_became_peered":"2023-10-09T09:41:11.800279+0000","last_unstale":"2023-10-09T09:41:58.864246+0000","last_undegraded":"2023-10-09T09:41:58.864246+0000","last_fullsized":"2023-10-09T09:41:58.864246+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:14:19.131995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.12","version":"1202'4772","reported_seq":4822,"reported_epoch":1202,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T09:42:19.801119+0000","last_change":"2023-10-09T09:41:57.934249+0000","last_active":"2023-10-09T09:42:19.801119+0000","last_peered":"2023-10-09T09:42:19.801119+0000","last_clean":"2023-10-09T09:41:10.776706+0000","last_became_active":"2023-10-09T09:41:57.931805+0000","last_became_peered":"2023-10-09T09:41:57.931805+0000","last_unstale":"2023-10-09T09:42:19.801119+0000","last_undegraded":"2023-10-09T09:41:57.902016+0000","last_fullsized":"2023-10-09T09:42:19.801119+0000","mapping_epoch":1199,"log_start":"0'0","ondisk_log_start":"0'0","created":1199,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4772,"log_dups_size":0,"ondisk_log_size":4772,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181927936,"num_objects":2776,"num_object_clones":0,"num_object_copies":8328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":395,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2776,"num_write_kb":177664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2,3","objects":395},{"shards":"1,2,6","objects":2381}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":676,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.864899+0000","last_change":"2023-10-09T09:41:11.801649+0000","last_active":"2023-10-09T09:41:58.864899+0000","last_peered":"2023-10-09T09:41:58.864899+0000","last_clean":"2023-10-09T09:41:58.864899+0000","last_became_active":"2023-10-09T09:41:11.798585+0000","last_became_peered":"2023-10-09T09:41:11.798585+0000","last_unstale":"2023-10-09T09:41:58.864899+0000","last_undegraded":"2023-10-09T09:41:58.864899+0000","last_fullsized":"2023-10-09T09:41:58.864899+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:11.450331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1151,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.865930+0000","last_change":"2023-10-09T09:40:25.799705+0000","last_active":"2023-10-09T09:41:58.865930+0000","last_peered":"2023-10-09T09:41:58.865930+0000","last_clean":"2023-10-09T09:41:58.865930+0000","last_became_active":"2023-10-09T09:40:25.799341+0000","last_became_peered":"2023-10-09T09:40:25.799341+0000","last_unstale":"2023-10-09T09:41:58.865930+0000","last_undegraded":"2023-10-09T09:41:58.865930+0000","last_fullsized":"2023-10-09T09:41:58.865930+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1150,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.863873+0000","last_change":"2023-10-09T09:41:11.801830+0000","last_active":"2023-10-09T09:41:58.863873+0000","last_peered":"2023-10-09T09:41:58.863873+0000","last_clean":"2023-10-09T09:41:58.863873+0000","last_became_active":"2023-10-09T09:41:11.799923+0000","last_became_peered":"2023-10-09T09:41:11.799923+0000","last_unstale":"2023-10-09T09:41:58.863873+0000","last_undegraded":"2023-10-09T09:41:58.863873+0000","last_fullsized":"2023-10-09T09:41:58.863873+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:55.450110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":594,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.857804+0000","last_change":"2023-10-09T09:41:11.814521+0000","last_active":"2023-10-09T09:41:58.857804+0000","last_peered":"2023-10-09T09:41:58.857804+0000","last_clean":"2023-10-09T09:41:58.857804+0000","last_became_active":"2023-10-09T09:41:11.814148+0000","last_became_peered":"2023-10-09T09:41:11.814148+0000","last_unstale":"2023-10-09T09:41:58.857804+0000","last_undegraded":"2023-10-09T09:41:58.857804+0000","last_fullsized":"2023-10-09T09:41:58.857804+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:45.497593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":53,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.859708+0000","last_change":"2023-10-09T09:41:11.833334+0000","last_active":"2023-10-09T09:41:58.859708+0000","last_peered":"2023-10-09T09:41:58.859708+0000","last_clean":"2023-10-09T09:41:58.859708+0000","last_became_active":"2023-10-09T09:41:11.832509+0000","last_became_peered":"2023-10-09T09:41:11.832509+0000","last_unstale":"2023-10-09T09:41:58.859708+0000","last_undegraded":"2023-10-09T09:41:58.859708+0000","last_fullsized":"2023-10-09T09:41:58.859708+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":617,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.865636+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:41:58.865636+0000","last_peered":"2023-10-09T09:41:58.865636+0000","last_clean":"2023-10-09T09:41:58.865636+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:41:58.865636+0000","last_undegraded":"2023-10-09T09:41:58.865636+0000","last_fullsized":"2023-10-09T09:41:58.865636+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":41,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.870694+0000","last_change":"2023-10-09T09:41:11.929105+0000","last_active":"2023-10-09T09:41:58.870694+0000","last_peered":"2023-10-09T09:41:58.870694+0000","last_clean":"2023-10-09T09:41:58.870694+0000","last_became_active":"2023-10-09T09:41:11.799443+0000","last_became_peered":"2023-10-09T09:41:11.799443+0000","last_unstale":"2023-10-09T09:41:58.870694+0000","last_undegraded":"2023-10-09T09:41:58.870694+0000","last_fullsized":"2023-10-09T09:41:58.870694+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":34,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.860129+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:41:58.860129+0000","last_peered":"2023-10-09T09:41:58.860129+0000","last_clean":"2023-10-09T09:41:58.860129+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:41:58.860129+0000","last_undegraded":"2023-10-09T09:41:58.860129+0000","last_fullsized":"2023-10-09T09:41:58.860129+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1179,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.870313+0000","last_change":"2023-10-09T09:40:50.110970+0000","last_active":"2023-10-09T09:41:58.870313+0000","last_peered":"2023-10-09T09:41:58.870313+0000","last_clean":"2023-10-09T09:41:58.870313+0000","last_became_active":"2023-10-09T09:40:49.792764+0000","last_became_peered":"2023-10-09T09:40:49.792764+0000","last_unstale":"2023-10-09T09:41:58.870313+0000","last_undegraded":"2023-10-09T09:41:58.870313+0000","last_fullsized":"2023-10-09T09:41:58.870313+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1152,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869722+0000","last_change":"2023-10-09T09:14:40.489200+0000","last_active":"2023-10-09T09:41:58.869722+0000","last_peered":"2023-10-09T09:41:58.869722+0000","last_clean":"2023-10-09T09:41:58.869722+0000","last_became_active":"2023-10-09T09:14:40.488771+0000","last_became_peered":"2023-10-09T09:14:40.488771+0000","last_unstale":"2023-10-09T09:41:58.869722+0000","last_undegraded":"2023-10-09T09:41:58.869722+0000","last_fullsized":"2023-10-09T09:41:58.869722+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.19","version":"1202'4905","reported_seq":6536,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:22.331007+0000","last_change":"2023-10-09T09:41:57.890326+0000","last_active":"2023-10-09T09:42:22.331007+0000","last_peered":"2023-10-09T09:42:22.331007+0000","last_clean":"2023-10-09T09:42:22.331007+0000","last_became_active":"2023-10-09T09:41:57.888576+0000","last_became_peered":"2023-10-09T09:41:57.888576+0000","last_unstale":"2023-10-09T09:42:22.331007+0000","last_undegraded":"2023-10-09T09:42:22.331007+0000","last_fullsized":"2023-10-09T09:42:22.331007+0000","mapping_epoch":1199,"log_start":"0'0","ondisk_log_start":"0'0","created":1199,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4905,"log_dups_size":0,"ondisk_log_size":4905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188645376,"num_objects":2879,"num_object_clones":0,"num_object_copies":8637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2879,"num_write_kb":184224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":391,"num_bytes_recovered":25591808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":48,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869883+0000","last_change":"2023-10-09T09:40:49.937490+0000","last_active":"2023-10-09T09:41:58.869883+0000","last_peered":"2023-10-09T09:41:58.869883+0000","last_clean":"2023-10-09T09:41:58.869883+0000","last_became_active":"2023-10-09T09:40:49.792044+0000","last_became_peered":"2023-10-09T09:40:49.792044+0000","last_unstale":"2023-10-09T09:41:58.869883+0000","last_undegraded":"2023-10-09T09:41:58.869883+0000","last_fullsized":"2023-10-09T09:41:58.869883+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":679,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.859219+0000","last_change":"2023-10-09T09:41:11.826293+0000","last_active":"2023-10-09T09:41:58.859219+0000","last_peered":"2023-10-09T09:41:58.859219+0000","last_clean":"2023-10-09T09:41:58.859219+0000","last_became_active":"2023-10-09T09:41:11.825808+0000","last_became_peered":"2023-10-09T09:41:11.825808+0000","last_unstale":"2023-10-09T09:41:58.859219+0000","last_undegraded":"2023-10-09T09:41:58.859219+0000","last_fullsized":"2023-10-09T09:41:58.859219+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:33:14.204593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.18","version":"1202'4880","reported_seq":4929,"reported_epoch":1202,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T09:42:22.331541+0000","last_change":"2023-10-09T09:41:57.921440+0000","last_active":"2023-10-09T09:42:22.331541+0000","last_peered":"2023-10-09T09:42:22.331541+0000","last_clean":"2023-10-09T09:41:10.762192+0000","last_became_active":"2023-10-09T09:41:57.919367+0000","last_became_peered":"2023-10-09T09:41:57.919367+0000","last_unstale":"2023-10-09T09:42:22.331541+0000","last_undegraded":"2023-10-09T09:41:57.906045+0000","last_fullsized":"2023-10-09T09:42:22.331541+0000","mapping_epoch":1199,"log_start":"0'0","ondisk_log_start":"0'0","created":1199,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4880,"log_dups_size":0,"ondisk_log_size":4880,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189136896,"num_objects":2886,"num_object_clones":0,"num_object_copies":8658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":383,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2886,"num_write_kb":184704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,1,5","objects":2503},{"shards":"0,3,5","objects":383}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1142,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:42:17.349782+0000","last_change":"2023-10-09T09:41:11.795047+0000","last_active":"2023-10-09T09:42:17.349782+0000","last_peered":"2023-10-09T09:42:17.349782+0000","last_clean":"2023-10-09T09:42:17.349782+0000","last_became_active":"2023-10-09T09:41:11.794662+0000","last_became_peered":"2023-10-09T09:41:11.794662+0000","last_unstale":"2023-10-09T09:42:17.349782+0000","last_undegraded":"2023-10-09T09:42:17.349782+0000","last_fullsized":"2023-10-09T09:42:17.349782+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:12.266267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":38,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.865422+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:41:58.865422+0000","last_peered":"2023-10-09T09:41:58.865422+0000","last_clean":"2023-10-09T09:41:58.865422+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:41:58.865422+0000","last_undegraded":"2023-10-09T09:41:58.865422+0000","last_fullsized":"2023-10-09T09:41:58.865422+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":31,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869736+0000","last_change":"2023-10-09T09:40:49.797133+0000","last_active":"2023-10-09T09:41:58.869736+0000","last_peered":"2023-10-09T09:41:58.869736+0000","last_clean":"2023-10-09T09:41:58.869736+0000","last_became_active":"2023-10-09T09:40:49.796773+0000","last_became_peered":"2023-10-09T09:40:49.796773+0000","last_unstale":"2023-10-09T09:41:58.869736+0000","last_undegraded":"2023-10-09T09:41:58.869736+0000","last_fullsized":"2023-10-09T09:41:58.869736+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1140,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.864058+0000","last_change":"2023-10-09T09:41:11.791237+0000","last_active":"2023-10-09T09:41:58.864058+0000","last_peered":"2023-10-09T09:41:58.864058+0000","last_clean":"2023-10-09T09:41:58.864058+0000","last_became_active":"2023-10-09T09:41:11.790857+0000","last_became_peered":"2023-10-09T09:41:11.790857+0000","last_unstale":"2023-10-09T09:41:58.864058+0000","last_undegraded":"2023-10-09T09:41:58.864058+0000","last_fullsized":"2023-10-09T09:41:58.864058+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:17.466387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1196,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.864769+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:41:58.864769+0000","last_peered":"2023-10-09T09:41:58.864769+0000","last_clean":"2023-10-09T09:41:58.864769+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:41:58.864769+0000","last_undegraded":"2023-10-09T09:41:58.864769+0000","last_fullsized":"2023-10-09T09:41:58.864769+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1162,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.859292+0000","last_change":"2023-10-09T09:41:11.805335+0000","last_active":"2023-10-09T09:41:58.859292+0000","last_peered":"2023-10-09T09:41:58.859292+0000","last_clean":"2023-10-09T09:41:58.859292+0000","last_became_active":"2023-10-09T09:41:11.804963+0000","last_became_peered":"2023-10-09T09:41:11.804963+0000","last_unstale":"2023-10-09T09:41:58.859292+0000","last_undegraded":"2023-10-09T09:41:58.859292+0000","last_fullsized":"2023-10-09T09:41:58.859292+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:09.265362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"1202'4954","reported_seq":5005,"reported_epoch":1202,"state":"active+recovery_wait","last_fresh":"2023-10-09T09:42:22.352433+0000","last_change":"2023-10-09T09:41:57.939985+0000","last_active":"2023-10-09T09:42:22.352433+0000","last_peered":"2023-10-09T09:42:22.352433+0000","last_clean":"2023-10-09T09:41:56.832868+0000","last_became_active":"2023-10-09T09:41:57.887216+0000","last_became_peered":"2023-10-09T09:41:57.887216+0000","last_unstale":"2023-10-09T09:42:22.352433+0000","last_undegraded":"2023-10-09T09:42:22.352433+0000","last_fullsized":"2023-10-09T09:42:22.352433+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4954,"log_dups_size":0,"ondisk_log_size":4954,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191266816,"num_objects":2918,"num_object_clones":0,"num_object_copies":8754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2918,"num_write_kb":186784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":2918}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":46,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.871080+0000","last_change":"2023-10-09T09:40:49.847626+0000","last_active":"2023-10-09T09:41:58.871080+0000","last_peered":"2023-10-09T09:41:58.871080+0000","last_clean":"2023-10-09T09:41:58.871080+0000","last_became_active":"2023-10-09T09:40:49.790030+0000","last_became_peered":"2023-10-09T09:40:49.790030+0000","last_unstale":"2023-10-09T09:41:58.871080+0000","last_undegraded":"2023-10-09T09:41:58.871080+0000","last_fullsized":"2023-10-09T09:41:58.871080+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":44,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.859156+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:41:58.859156+0000","last_peered":"2023-10-09T09:41:58.859156+0000","last_clean":"2023-10-09T09:41:58.859156+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:41:58.859156+0000","last_undegraded":"2023-10-09T09:41:58.859156+0000","last_fullsized":"2023-10-09T09:41:58.859156+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1202'4845","reported_seq":4901,"reported_epoch":1202,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T09:42:19.769649+0000","last_change":"2023-10-09T09:41:58.880899+0000","last_active":"2023-10-09T09:42:19.769649+0000","last_peered":"2023-10-09T09:42:19.769649+0000","last_clean":"2023-10-09T09:41:10.777223+0000","last_became_active":"2023-10-09T09:41:58.875735+0000","last_became_peered":"2023-10-09T09:41:58.875735+0000","last_unstale":"2023-10-09T09:42:19.769649+0000","last_undegraded":"2023-10-09T09:41:58.861392+0000","last_fullsized":"2023-10-09T09:42:19.769649+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4845,"log_dups_size":0,"ondisk_log_size":4845,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182910976,"num_objects":2791,"num_object_clones":0,"num_object_copies":8373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":399,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2791,"num_write_kb":178624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":98304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"0,1,4","objects":2392},{"shards":"1,3,4","objects":399}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":46,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.870362+0000","last_change":"2023-10-09T09:40:50.158191+0000","last_active":"2023-10-09T09:41:58.870362+0000","last_peered":"2023-10-09T09:41:58.870362+0000","last_clean":"2023-10-09T09:41:58.870362+0000","last_became_active":"2023-10-09T09:40:49.797431+0000","last_became_peered":"2023-10-09T09:40:49.797431+0000","last_unstale":"2023-10-09T09:41:58.870362+0000","last_undegraded":"2023-10-09T09:41:58.870362+0000","last_fullsized":"2023-10-09T09:41:58.870362+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1176,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869514+0000","last_change":"2023-10-09T09:41:11.797835+0000","last_active":"2023-10-09T09:41:58.869514+0000","last_peered":"2023-10-09T09:41:58.869514+0000","last_clean":"2023-10-09T09:41:58.869514+0000","last_became_active":"2023-10-09T09:41:11.797437+0000","last_became_peered":"2023-10-09T09:41:11.797437+0000","last_unstale":"2023-10-09T09:41:58.869514+0000","last_undegraded":"2023-10-09T09:41:58.869514+0000","last_fullsized":"2023-10-09T09:41:58.869514+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1202'4798","reported_seq":4843,"reported_epoch":1202,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T09:42:20.878129+0000","last_change":"2023-10-09T09:41:58.871565+0000","last_active":"2023-10-09T09:42:20.878129+0000","last_peered":"2023-10-09T09:42:20.878129+0000","last_clean":"2023-10-09T09:41:10.748257+0000","last_became_active":"2023-10-09T09:41:58.867758+0000","last_became_peered":"2023-10-09T09:41:58.867758+0000","last_unstale":"2023-10-09T09:42:20.878129+0000","last_undegraded":"2023-10-09T09:41:58.857649+0000","last_fullsized":"2023-10-09T09:42:20.878129+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4798,"log_dups_size":0,"ondisk_log_size":4798,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182091776,"num_objects":2778,"num_object_clones":0,"num_object_copies":8334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":397,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2778,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"1,2,5","objects":2381},{"shards":"2,3,5","objects":397}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1152,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.859866+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:41:58.859866+0000","last_peered":"2023-10-09T09:41:58.859866+0000","last_clean":"2023-10-09T09:41:58.859866+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:41:58.859866+0000","last_undegraded":"2023-10-09T09:41:58.859866+0000","last_fullsized":"2023-10-09T09:41:58.859866+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1143,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.865255+0000","last_change":"2023-10-09T09:40:49.793789+0000","last_active":"2023-10-09T09:41:58.865255+0000","last_peered":"2023-10-09T09:41:58.865255+0000","last_clean":"2023-10-09T09:41:58.865255+0000","last_became_active":"2023-10-09T09:40:49.793374+0000","last_became_peered":"2023-10-09T09:40:49.793374+0000","last_unstale":"2023-10-09T09:41:58.865255+0000","last_undegraded":"2023-10-09T09:41:58.865255+0000","last_fullsized":"2023-10-09T09:41:58.865255+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1202'4907","reported_seq":4944,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:20.853352+0000","last_change":"2023-10-09T09:41:57.881607+0000","last_active":"2023-10-09T09:42:20.853352+0000","last_peered":"2023-10-09T09:42:20.853352+0000","last_clean":"2023-10-09T09:42:20.853352+0000","last_became_active":"2023-10-09T09:41:57.879184+0000","last_became_peered":"2023-10-09T09:41:57.879184+0000","last_unstale":"2023-10-09T09:42:20.853352+0000","last_undegraded":"2023-10-09T09:42:20.853352+0000","last_fullsized":"2023-10-09T09:42:20.853352+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4907,"log_dups_size":0,"ondisk_log_size":4907,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188350464,"num_objects":2874,"num_object_clones":0,"num_object_copies":8622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2874,"num_write_kb":183936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":689,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.865230+0000","last_change":"2023-10-09T09:41:11.815662+0000","last_active":"2023-10-09T09:41:58.865230+0000","last_peered":"2023-10-09T09:41:58.865230+0000","last_clean":"2023-10-09T09:41:58.865230+0000","last_became_active":"2023-10-09T09:41:11.815203+0000","last_became_peered":"2023-10-09T09:41:11.815203+0000","last_unstale":"2023-10-09T09:41:58.865230+0000","last_undegraded":"2023-10-09T09:41:58.865230+0000","last_fullsized":"2023-10-09T09:41:58.865230+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1108,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:42:02.339065+0000","last_change":"2023-10-09T09:35:18.681438+0000","last_active":"2023-10-09T09:42:02.339065+0000","last_peered":"2023-10-09T09:42:02.339065+0000","last_clean":"2023-10-09T09:42:02.339065+0000","last_became_active":"2023-10-09T09:35:18.680806+0000","last_became_peered":"2023-10-09T09:35:18.680806+0000","last_unstale":"2023-10-09T09:42:02.339065+0000","last_undegraded":"2023-10-09T09:42:02.339065+0000","last_fullsized":"2023-10-09T09:42:02.339065+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:30.669832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"1202'4829","reported_seq":6354,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:21.263902+0000","last_change":"2023-10-09T09:41:57.856022+0000","last_active":"2023-10-09T09:42:21.263902+0000","last_peered":"2023-10-09T09:42:21.263902+0000","last_clean":"2023-10-09T09:42:21.263902+0000","last_became_active":"2023-10-09T09:41:57.853631+0000","last_became_peered":"2023-10-09T09:41:57.853631+0000","last_unstale":"2023-10-09T09:42:21.263902+0000","last_undegraded":"2023-10-09T09:42:21.263902+0000","last_fullsized":"2023-10-09T09:42:21.263902+0000","mapping_epoch":1192,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4829,"log_dups_size":0,"ondisk_log_size":4829,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184549376,"num_objects":2816,"num_object_clones":0,"num_object_copies":8448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2816,"num_write_kb":180224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":125,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:42:07.339665+0000","last_change":"2023-10-09T09:41:11.847764+0000","last_active":"2023-10-09T09:42:07.339665+0000","last_peered":"2023-10-09T09:42:07.339665+0000","last_clean":"2023-10-09T09:42:07.339665+0000","last_became_active":"2023-10-09T09:41:11.800352+0000","last_became_peered":"2023-10-09T09:41:11.800352+0000","last_unstale":"2023-10-09T09:42:07.339665+0000","last_undegraded":"2023-10-09T09:42:07.339665+0000","last_fullsized":"2023-10-09T09:42:07.339665+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:56.392044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1161,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.870238+0000","last_change":"2023-10-09T09:41:11.798393+0000","last_active":"2023-10-09T09:41:58.870238+0000","last_peered":"2023-10-09T09:41:58.870238+0000","last_clean":"2023-10-09T09:41:58.870238+0000","last_became_active":"2023-10-09T09:41:11.798022+0000","last_became_peered":"2023-10-09T09:41:11.798022+0000","last_unstale":"2023-10-09T09:41:58.870238+0000","last_undegraded":"2023-10-09T09:41:58.870238+0000","last_fullsized":"2023-10-09T09:41:58.870238+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1202'4712","reported_seq":4753,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:19.795639+0000","last_change":"2023-10-09T09:41:57.918151+0000","last_active":"2023-10-09T09:42:19.795639+0000","last_peered":"2023-10-09T09:42:19.795639+0000","last_clean":"2023-10-09T09:42:19.795639+0000","last_became_active":"2023-10-09T09:41:57.915494+0000","last_became_peered":"2023-10-09T09:41:57.915494+0000","last_unstale":"2023-10-09T09:42:19.795639+0000","last_undegraded":"2023-10-09T09:42:19.795639+0000","last_fullsized":"2023-10-09T09:42:19.795639+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4712,"log_dups_size":0,"ondisk_log_size":4712,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177864704,"num_objects":2714,"num_object_clones":0,"num_object_copies":8142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2714,"num_write_kb":173696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1135,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.865108+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:41:58.865108+0000","last_peered":"2023-10-09T09:41:58.865108+0000","last_clean":"2023-10-09T09:41:58.865108+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:41:58.865108+0000","last_undegraded":"2023-10-09T09:41:58.865108+0000","last_fullsized":"2023-10-09T09:41:58.865108+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1185,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.870454+0000","last_change":"2023-10-09T09:36:43.419348+0000","last_active":"2023-10-09T09:41:58.870454+0000","last_peered":"2023-10-09T09:41:58.870454+0000","last_clean":"2023-10-09T09:41:58.870454+0000","last_became_active":"2023-10-09T09:36:43.418525+0000","last_became_peered":"2023-10-09T09:36:43.418525+0000","last_unstale":"2023-10-09T09:41:58.870454+0000","last_undegraded":"2023-10-09T09:41:58.870454+0000","last_fullsized":"2023-10-09T09:41:58.870454+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1202'4772","reported_seq":4819,"reported_epoch":1202,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T09:42:19.665431+0000","last_change":"2023-10-09T09:41:58.879429+0000","last_active":"2023-10-09T09:42:19.665431+0000","last_peered":"2023-10-09T09:42:19.665431+0000","last_clean":"2023-10-09T09:41:10.771882+0000","last_became_active":"2023-10-09T09:41:58.874263+0000","last_became_peered":"2023-10-09T09:41:58.874263+0000","last_unstale":"2023-10-09T09:42:19.665431+0000","last_undegraded":"2023-10-09T09:41:58.859190+0000","last_fullsized":"2023-10-09T09:42:19.665431+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4772,"log_dups_size":0,"ondisk_log_size":4772,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180355072,"num_objects":2752,"num_object_clones":0,"num_object_copies":8256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":395,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2752,"num_write_kb":176128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":395},{"shards":"1,5,6","objects":2357}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1180,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869283+0000","last_change":"2023-10-09T09:40:49.785820+0000","last_active":"2023-10-09T09:41:58.869283+0000","last_peered":"2023-10-09T09:41:58.869283+0000","last_clean":"2023-10-09T09:41:58.869283+0000","last_became_active":"2023-10-09T09:40:49.785373+0000","last_became_peered":"2023-10-09T09:40:49.785373+0000","last_unstale":"2023-10-09T09:41:58.869283+0000","last_undegraded":"2023-10-09T09:41:58.869283+0000","last_fullsized":"2023-10-09T09:41:58.869283+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1152,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.870536+0000","last_change":"2023-10-09T09:14:40.488718+0000","last_active":"2023-10-09T09:41:58.870536+0000","last_peered":"2023-10-09T09:41:58.870536+0000","last_clean":"2023-10-09T09:41:58.870536+0000","last_became_active":"2023-10-09T09:14:40.488288+0000","last_became_peered":"2023-10-09T09:14:40.488288+0000","last_unstale":"2023-10-09T09:41:58.870536+0000","last_undegraded":"2023-10-09T09:41:58.870536+0000","last_fullsized":"2023-10-09T09:41:58.870536+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1202'4759","reported_seq":4810,"reported_epoch":1202,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T09:42:19.763634+0000","last_change":"2023-10-09T09:41:58.880905+0000","last_active":"2023-10-09T09:42:19.763634+0000","last_peered":"2023-10-09T09:42:19.763634+0000","last_clean":"2023-10-09T09:41:10.776706+0000","last_became_active":"2023-10-09T09:41:58.868908+0000","last_became_peered":"2023-10-09T09:41:58.868908+0000","last_unstale":"2023-10-09T09:42:19.763634+0000","last_undegraded":"2023-10-09T09:41:58.857859+0000","last_fullsized":"2023-10-09T09:42:19.763634+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4759,"log_dups_size":0,"ondisk_log_size":4759,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181075968,"num_objects":2763,"num_object_clones":0,"num_object_copies":8289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":368,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2763,"num_write_kb":176832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2,3","objects":368},{"shards":"1,2,6","objects":2395}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1111,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.858034+0000","last_change":"2023-10-09T09:41:11.814910+0000","last_active":"2023-10-09T09:41:58.858034+0000","last_peered":"2023-10-09T09:41:58.858034+0000","last_clean":"2023-10-09T09:41:58.858034+0000","last_became_active":"2023-10-09T09:41:11.814561+0000","last_became_peered":"2023-10-09T09:41:11.814561+0000","last_unstale":"2023-10-09T09:41:58.858034+0000","last_undegraded":"2023-10-09T09:41:58.858034+0000","last_fullsized":"2023-10-09T09:41:58.858034+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:25:51.140694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.855133+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:41:58.855133+0000","last_peered":"2023-10-09T09:41:58.855133+0000","last_clean":"2023-10-09T09:41:58.855133+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:41:58.855133+0000","last_undegraded":"2023-10-09T09:41:58.855133+0000","last_fullsized":"2023-10-09T09:41:58.855133+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"1202'5659","reported_seq":5696,"reported_epoch":1202,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T09:42:21.273796+0000","last_change":"2023-10-09T09:41:12.863984+0000","last_active":"2023-10-09T09:42:21.273796+0000","last_peered":"2023-10-09T09:42:21.273796+0000","last_clean":"2023-10-09T09:41:10.776940+0000","last_became_active":"2023-10-09T09:41:12.862392+0000","last_became_peered":"2023-10-09T09:41:12.862392+0000","last_unstale":"2023-10-09T09:42:21.273796+0000","last_undegraded":"2023-10-09T09:41:12.801069+0000","last_fullsized":"2023-10-09T09:41:12.794241+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":5659,"log_dups_size":0,"ondisk_log_size":5659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370868224,"num_objects":5659,"num_object_clones":0,"num_object_copies":16977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":726,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5659,"num_write_kb":362176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":726},{"shards":"1,5","objects":4933}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1164,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.859646+0000","last_change":"2023-10-09T09:40:49.781045+0000","last_active":"2023-10-09T09:41:58.859646+0000","last_peered":"2023-10-09T09:41:58.859646+0000","last_clean":"2023-10-09T09:41:58.859646+0000","last_became_active":"2023-10-09T09:40:49.780425+0000","last_became_peered":"2023-10-09T09:40:49.780425+0000","last_unstale":"2023-10-09T09:41:58.859646+0000","last_undegraded":"2023-10-09T09:41:58.859646+0000","last_fullsized":"2023-10-09T09:41:58.859646+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:56.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":44,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.859640+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:41:58.859640+0000","last_peered":"2023-10-09T09:41:58.859640+0000","last_clean":"2023-10-09T09:41:58.859640+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:41:58.859640+0000","last_undegraded":"2023-10-09T09:41:58.859640+0000","last_fullsized":"2023-10-09T09:41:58.859640+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1148,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.862602+0000","last_change":"2023-10-09T09:40:25.799081+0000","last_active":"2023-10-09T09:41:58.862602+0000","last_peered":"2023-10-09T09:41:58.862602+0000","last_clean":"2023-10-09T09:41:58.862602+0000","last_became_active":"2023-10-09T09:40:25.798693+0000","last_became_peered":"2023-10-09T09:40:25.798693+0000","last_unstale":"2023-10-09T09:41:58.862602+0000","last_undegraded":"2023-10-09T09:41:58.862602+0000","last_fullsized":"2023-10-09T09:41:58.862602+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:35.527067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"1202'4902","reported_seq":6533,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:22.240280+0000","last_change":"2023-10-09T09:41:57.864491+0000","last_active":"2023-10-09T09:42:22.240280+0000","last_peered":"2023-10-09T09:42:22.240280+0000","last_clean":"2023-10-09T09:42:22.240280+0000","last_became_active":"2023-10-09T09:41:57.862275+0000","last_became_peered":"2023-10-09T09:41:57.862275+0000","last_unstale":"2023-10-09T09:42:22.240280+0000","last_undegraded":"2023-10-09T09:42:22.240280+0000","last_fullsized":"2023-10-09T09:42:22.240280+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4902,"log_dups_size":0,"ondisk_log_size":4902,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188448768,"num_objects":2875,"num_object_clones":0,"num_object_copies":8625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2875,"num_write_kb":184032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":390,"num_bytes_recovered":25591808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":630,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.862168+0000","last_change":"2023-10-09T09:40:25.795910+0000","last_active":"2023-10-09T09:41:58.862168+0000","last_peered":"2023-10-09T09:41:58.862168+0000","last_clean":"2023-10-09T09:41:58.862168+0000","last_became_active":"2023-10-09T09:40:25.795327+0000","last_became_peered":"2023-10-09T09:40:25.795327+0000","last_unstale":"2023-10-09T09:41:58.862168+0000","last_undegraded":"2023-10-09T09:41:58.862168+0000","last_fullsized":"2023-10-09T09:41:58.862168+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:58.343181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1168,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869589+0000","last_change":"2023-10-09T09:35:18.685554+0000","last_active":"2023-10-09T09:41:58.869589+0000","last_peered":"2023-10-09T09:41:58.869589+0000","last_clean":"2023-10-09T09:41:58.869589+0000","last_became_active":"2023-10-09T09:35:18.685119+0000","last_became_peered":"2023-10-09T09:35:18.685119+0000","last_unstale":"2023-10-09T09:41:58.869589+0000","last_undegraded":"2023-10-09T09:41:58.869589+0000","last_fullsized":"2023-10-09T09:41:58.869589+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1187,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.859087+0000","last_change":"2023-10-09T09:41:11.811782+0000","last_active":"2023-10-09T09:41:58.859087+0000","last_peered":"2023-10-09T09:41:58.859087+0000","last_clean":"2023-10-09T09:41:58.859087+0000","last_became_active":"2023-10-09T09:41:11.811431+0000","last_became_peered":"2023-10-09T09:41:11.811431+0000","last_unstale":"2023-10-09T09:41:58.859087+0000","last_undegraded":"2023-10-09T09:41:58.859087+0000","last_fullsized":"2023-10-09T09:41:58.859087+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:05.197258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"1202'5648","reported_seq":7076,"reported_epoch":1202,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T09:42:21.289880+0000","last_change":"2023-10-09T09:41:51.582884+0000","last_active":"2023-10-09T09:42:21.289880+0000","last_peered":"2023-10-09T09:42:21.289880+0000","last_clean":"2023-10-09T09:41:10.774573+0000","last_became_active":"2023-10-09T09:41:12.853578+0000","last_became_peered":"2023-10-09T09:41:12.853578+0000","last_unstale":"2023-10-09T09:42:21.289880+0000","last_undegraded":"2023-10-09T09:42:21.289880+0000","last_fullsized":"2023-10-09T09:41:12.793003+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":5648,"log_dups_size":0,"ondisk_log_size":5648,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370147328,"num_objects":5648,"num_object_clones":0,"num_object_copies":16944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":33,"num_objects_unfound":0,"num_objects_dirty":5648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5648,"num_write_kb":361472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":690,"num_bytes_recovered":45219840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":33},{"shards":"1,5","objects":5615}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":73,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.852212+0000","last_change":"2023-10-09T09:41:11.801228+0000","last_active":"2023-10-09T09:41:58.852212+0000","last_peered":"2023-10-09T09:41:58.852212+0000","last_clean":"2023-10-09T09:41:58.852212+0000","last_became_active":"2023-10-09T09:41:11.800068+0000","last_became_peered":"2023-10-09T09:41:11.800068+0000","last_unstale":"2023-10-09T09:41:58.852212+0000","last_undegraded":"2023-10-09T09:41:58.852212+0000","last_fullsized":"2023-10-09T09:41:58.852212+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:33.425580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1163,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.857921+0000","last_change":"2023-10-09T09:36:43.421868+0000","last_active":"2023-10-09T09:41:58.857921+0000","last_peered":"2023-10-09T09:41:58.857921+0000","last_clean":"2023-10-09T09:41:58.857921+0000","last_became_active":"2023-10-09T09:36:43.421013+0000","last_became_peered":"2023-10-09T09:36:43.421013+0000","last_unstale":"2023-10-09T09:41:58.857921+0000","last_undegraded":"2023-10-09T09:41:58.857921+0000","last_fullsized":"2023-10-09T09:41:58.857921+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:43.353667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1167,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.859525+0000","last_change":"2023-10-09T09:40:25.799440+0000","last_active":"2023-10-09T09:41:58.859525+0000","last_peered":"2023-10-09T09:41:58.859525+0000","last_clean":"2023-10-09T09:41:58.859525+0000","last_became_active":"2023-10-09T09:40:25.797967+0000","last_became_peered":"2023-10-09T09:40:25.797967+0000","last_unstale":"2023-10-09T09:41:58.859525+0000","last_undegraded":"2023-10-09T09:41:58.859525+0000","last_fullsized":"2023-10-09T09:41:58.859525+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:51.038572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1202'4819","reported_seq":4867,"reported_epoch":1202,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T09:42:22.315906+0000","last_change":"2023-10-09T09:41:58.880134+0000","last_active":"2023-10-09T09:42:22.315906+0000","last_peered":"2023-10-09T09:42:22.315906+0000","last_clean":"2023-10-09T09:41:10.762192+0000","last_became_active":"2023-10-09T09:41:58.875643+0000","last_became_peered":"2023-10-09T09:41:58.875643+0000","last_unstale":"2023-10-09T09:42:22.315906+0000","last_undegraded":"2023-10-09T09:41:58.866803+0000","last_fullsized":"2023-10-09T09:42:22.315906+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4819,"log_dups_size":0,"ondisk_log_size":4819,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185139200,"num_objects":2825,"num_object_clones":0,"num_object_copies":8475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":406,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2825,"num_write_kb":180800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,1,5","objects":2419},{"shards":"0,3,5","objects":406}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1206,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.853227+0000","last_change":"2023-10-09T09:41:11.795802+0000","last_active":"2023-10-09T09:41:58.853227+0000","last_peered":"2023-10-09T09:41:58.853227+0000","last_clean":"2023-10-09T09:41:58.853227+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:41:58.853227+0000","last_undegraded":"2023-10-09T09:41:58.853227+0000","last_fullsized":"2023-10-09T09:41:58.853227+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":524,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.867274+0000","last_change":"2023-10-09T09:40:52.800168+0000","last_active":"2023-10-09T09:41:58.867274+0000","last_peered":"2023-10-09T09:41:58.867274+0000","last_clean":"2023-10-09T09:41:58.867274+0000","last_became_active":"2023-10-09T09:40:52.799543+0000","last_became_peered":"2023-10-09T09:40:52.799543+0000","last_unstale":"2023-10-09T09:41:58.867274+0000","last_undegraded":"2023-10-09T09:41:58.867274+0000","last_fullsized":"2023-10-09T09:41:58.867274+0000","mapping_epoch":1185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:44.481003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":44,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.860176+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:41:58.860176+0000","last_peered":"2023-10-09T09:41:58.860176+0000","last_clean":"2023-10-09T09:41:58.860176+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:41:58.860176+0000","last_undegraded":"2023-10-09T09:41:58.860176+0000","last_fullsized":"2023-10-09T09:41:58.860176+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1128,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.865467+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:41:58.865467+0000","last_peered":"2023-10-09T09:41:58.865467+0000","last_clean":"2023-10-09T09:41:58.865467+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:41:58.865467+0000","last_undegraded":"2023-10-09T09:41:58.865467+0000","last_fullsized":"2023-10-09T09:41:58.865467+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1202'5695","reported_seq":5727,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:22.331001+0000","last_change":"2023-10-09T09:40:44.315824+0000","last_active":"2023-10-09T09:42:22.331001+0000","last_peered":"2023-10-09T09:42:22.331001+0000","last_clean":"2023-10-09T09:42:22.331001+0000","last_became_active":"2023-10-09T09:40:44.315401+0000","last_became_peered":"2023-10-09T09:40:44.315401+0000","last_unstale":"2023-10-09T09:42:22.331001+0000","last_undegraded":"2023-10-09T09:42:22.331001+0000","last_fullsized":"2023-10-09T09:42:22.331001+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":5695,"log_dups_size":0,"ondisk_log_size":5695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:51.462075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373227520,"num_objects":5695,"num_object_clones":0,"num_object_copies":17085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5695,"num_write_kb":364480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1130,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.857599+0000","last_change":"2023-10-09T09:41:11.815645+0000","last_active":"2023-10-09T09:41:58.857599+0000","last_peered":"2023-10-09T09:41:58.857599+0000","last_clean":"2023-10-09T09:41:58.857599+0000","last_became_active":"2023-10-09T09:41:11.815086+0000","last_became_peered":"2023-10-09T09:41:11.815086+0000","last_unstale":"2023-10-09T09:41:58.857599+0000","last_undegraded":"2023-10-09T09:41:58.857599+0000","last_fullsized":"2023-10-09T09:41:58.857599+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:29.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1134,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.863229+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:41:58.863229+0000","last_peered":"2023-10-09T09:41:58.863229+0000","last_clean":"2023-10-09T09:41:58.863229+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:41:58.863229+0000","last_undegraded":"2023-10-09T09:41:58.863229+0000","last_fullsized":"2023-10-09T09:41:58.863229+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1131,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.864156+0000","last_change":"2023-10-09T09:39:12.949268+0000","last_active":"2023-10-09T09:41:58.864156+0000","last_peered":"2023-10-09T09:41:58.864156+0000","last_clean":"2023-10-09T09:41:58.864156+0000","last_became_active":"2023-10-09T09:39:12.948924+0000","last_became_peered":"2023-10-09T09:39:12.948924+0000","last_unstale":"2023-10-09T09:41:58.864156+0000","last_undegraded":"2023-10-09T09:41:58.864156+0000","last_fullsized":"2023-10-09T09:41:58.864156+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:31.368540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"1202'5674","reported_seq":5705,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:21.250437+0000","last_change":"2023-10-09T09:40:49.803104+0000","last_active":"2023-10-09T09:42:21.250437+0000","last_peered":"2023-10-09T09:42:21.250437+0000","last_clean":"2023-10-09T09:42:21.250437+0000","last_became_active":"2023-10-09T09:40:49.802743+0000","last_became_peered":"2023-10-09T09:40:49.802743+0000","last_unstale":"2023-10-09T09:42:21.250437+0000","last_undegraded":"2023-10-09T09:42:21.250437+0000","last_fullsized":"2023-10-09T09:42:21.250437+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371851264,"num_objects":5674,"num_object_clones":0,"num_object_copies":17022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5674,"num_write_kb":363136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1106,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.864114+0000","last_change":"2023-10-09T09:40:49.787387+0000","last_active":"2023-10-09T09:41:58.864114+0000","last_peered":"2023-10-09T09:41:58.864114+0000","last_clean":"2023-10-09T09:41:58.864114+0000","last_became_active":"2023-10-09T09:40:49.786954+0000","last_became_peered":"2023-10-09T09:40:49.786954+0000","last_unstale":"2023-10-09T09:41:58.864114+0000","last_undegraded":"2023-10-09T09:41:58.864114+0000","last_fullsized":"2023-10-09T09:41:58.864114+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:51.055735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1182,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.859435+0000","last_change":"2023-10-09T09:40:25.797470+0000","last_active":"2023-10-09T09:41:58.859435+0000","last_peered":"2023-10-09T09:41:58.859435+0000","last_clean":"2023-10-09T09:41:58.859435+0000","last_became_active":"2023-10-09T09:40:25.796966+0000","last_became_peered":"2023-10-09T09:40:25.796966+0000","last_unstale":"2023-10-09T09:41:58.859435+0000","last_undegraded":"2023-10-09T09:41:58.859435+0000","last_fullsized":"2023-10-09T09:41:58.859435+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:24.526350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1136,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.863535+0000","last_change":"2023-10-09T09:40:49.792931+0000","last_active":"2023-10-09T09:41:58.863535+0000","last_peered":"2023-10-09T09:41:58.863535+0000","last_clean":"2023-10-09T09:41:58.863535+0000","last_became_active":"2023-10-09T09:40:49.792543+0000","last_became_peered":"2023-10-09T09:40:49.792543+0000","last_unstale":"2023-10-09T09:41:58.863535+0000","last_undegraded":"2023-10-09T09:41:58.863535+0000","last_fullsized":"2023-10-09T09:41:58.863535+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:00.778249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"1202'5607","reported_seq":5638,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:19.764416+0000","last_change":"2023-10-09T09:40:49.794927+0000","last_active":"2023-10-09T09:42:19.764416+0000","last_peered":"2023-10-09T09:42:19.764416+0000","last_clean":"2023-10-09T09:42:19.764416+0000","last_became_active":"2023-10-09T09:40:49.794550+0000","last_became_peered":"2023-10-09T09:40:49.794550+0000","last_unstale":"2023-10-09T09:42:19.764416+0000","last_undegraded":"2023-10-09T09:42:19.764416+0000","last_fullsized":"2023-10-09T09:42:19.764416+0000","mapping_epoch":1182,"log_start":"1191'1100","ondisk_log_start":"1191'1100","created":1179,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4507,"log_dups_size":0,"ondisk_log_size":4507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:34.821565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367460352,"num_objects":5607,"num_object_clones":0,"num_object_copies":16821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5607,"num_write_kb":358848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":59,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.864185+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:41:58.864185+0000","last_peered":"2023-10-09T09:41:58.864185+0000","last_clean":"2023-10-09T09:41:58.864185+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:41:58.864185+0000","last_undegraded":"2023-10-09T09:41:58.864185+0000","last_fullsized":"2023-10-09T09:41:58.864185+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1128,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.860670+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:41:58.860670+0000","last_peered":"2023-10-09T09:41:58.860670+0000","last_clean":"2023-10-09T09:41:58.860670+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:41:58.860670+0000","last_undegraded":"2023-10-09T09:41:58.860670+0000","last_fullsized":"2023-10-09T09:41:58.860670+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1097,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.867620+0000","last_change":"2023-10-09T09:41:11.801414+0000","last_active":"2023-10-09T09:41:58.867620+0000","last_peered":"2023-10-09T09:41:58.867620+0000","last_clean":"2023-10-09T09:41:58.867620+0000","last_became_active":"2023-10-09T09:41:11.800211+0000","last_became_peered":"2023-10-09T09:41:11.800211+0000","last_unstale":"2023-10-09T09:41:58.867620+0000","last_undegraded":"2023-10-09T09:41:58.867620+0000","last_fullsized":"2023-10-09T09:41:58.867620+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:44.155906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"1202'5689","reported_seq":7245,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:21.230886+0000","last_change":"2023-10-09T09:41:52.844827+0000","last_active":"2023-10-09T09:42:21.230886+0000","last_peered":"2023-10-09T09:42:21.230886+0000","last_clean":"2023-10-09T09:42:21.230886+0000","last_became_active":"2023-10-09T09:41:52.810942+0000","last_became_peered":"2023-10-09T09:41:52.810942+0000","last_unstale":"2023-10-09T09:42:21.230886+0000","last_undegraded":"2023-10-09T09:42:21.230886+0000","last_fullsized":"2023-10-09T09:42:21.230886+0000","mapping_epoch":1196,"log_start":"0'0","ondisk_log_start":"0'0","created":1179,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372834304,"num_objects":5689,"num_object_clones":0,"num_object_copies":17067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5689,"num_write_kb":364096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":749,"num_bytes_recovered":49086464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1174,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.865845+0000","last_change":"2023-10-09T09:41:11.815721+0000","last_active":"2023-10-09T09:41:58.865845+0000","last_peered":"2023-10-09T09:41:58.865845+0000","last_clean":"2023-10-09T09:41:58.865845+0000","last_became_active":"2023-10-09T09:41:11.815237+0000","last_became_peered":"2023-10-09T09:41:11.815237+0000","last_unstale":"2023-10-09T09:41:58.865845+0000","last_undegraded":"2023-10-09T09:41:58.865845+0000","last_fullsized":"2023-10-09T09:41:58.865845+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1112,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.864288+0000","last_change":"2023-10-09T09:35:18.685030+0000","last_active":"2023-10-09T09:41:58.864288+0000","last_peered":"2023-10-09T09:41:58.864288+0000","last_clean":"2023-10-09T09:41:58.864288+0000","last_became_active":"2023-10-09T09:35:18.675810+0000","last_became_peered":"2023-10-09T09:35:18.675810+0000","last_unstale":"2023-10-09T09:41:58.864288+0000","last_undegraded":"2023-10-09T09:41:58.864288+0000","last_fullsized":"2023-10-09T09:41:58.864288+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:52.793405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1155,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869350+0000","last_change":"2023-10-09T09:40:25.804139+0000","last_active":"2023-10-09T09:41:58.869350+0000","last_peered":"2023-10-09T09:41:58.869350+0000","last_clean":"2023-10-09T09:41:58.869350+0000","last_became_active":"2023-10-09T09:40:25.803753+0000","last_became_peered":"2023-10-09T09:40:25.803753+0000","last_unstale":"2023-10-09T09:41:58.869350+0000","last_undegraded":"2023-10-09T09:41:58.869350+0000","last_fullsized":"2023-10-09T09:41:58.869350+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.11","version":"1202'4866","reported_seq":6390,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:21.265859+0000","last_change":"2023-10-09T09:41:57.866215+0000","last_active":"2023-10-09T09:42:21.265859+0000","last_peered":"2023-10-09T09:42:21.265859+0000","last_clean":"2023-10-09T09:42:21.265859+0000","last_became_active":"2023-10-09T09:41:57.863357+0000","last_became_peered":"2023-10-09T09:41:57.863357+0000","last_unstale":"2023-10-09T09:42:21.265859+0000","last_undegraded":"2023-10-09T09:42:21.265859+0000","last_fullsized":"2023-10-09T09:42:21.265859+0000","mapping_epoch":1199,"log_start":"0'0","ondisk_log_start":"0'0","created":1199,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4866,"log_dups_size":0,"ondisk_log_size":4866,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186974208,"num_objects":2853,"num_object_clones":0,"num_object_copies":8559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2853,"num_write_kb":182592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1120,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.863195+0000","last_change":"2023-10-09T09:40:49.788328+0000","last_active":"2023-10-09T09:41:58.863195+0000","last_peered":"2023-10-09T09:41:58.863195+0000","last_clean":"2023-10-09T09:41:58.863195+0000","last_became_active":"2023-10-09T09:40:49.787927+0000","last_became_peered":"2023-10-09T09:40:49.787927+0000","last_unstale":"2023-10-09T09:41:58.863195+0000","last_undegraded":"2023-10-09T09:41:58.863195+0000","last_fullsized":"2023-10-09T09:41:58.863195+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:32.616104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":73,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.852965+0000","last_change":"2023-10-09T09:41:11.800443+0000","last_active":"2023-10-09T09:41:58.852965+0000","last_peered":"2023-10-09T09:41:58.852965+0000","last_clean":"2023-10-09T09:41:58.852965+0000","last_became_active":"2023-10-09T09:41:11.799864+0000","last_became_peered":"2023-10-09T09:41:11.799864+0000","last_unstale":"2023-10-09T09:41:58.852965+0000","last_undegraded":"2023-10-09T09:41:58.852965+0000","last_fullsized":"2023-10-09T09:41:58.852965+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:48:22.481162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"1202'4854","reported_seq":4891,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:20.821451+0000","last_change":"2023-10-09T09:41:57.908216+0000","last_active":"2023-10-09T09:42:20.821451+0000","last_peered":"2023-10-09T09:42:20.821451+0000","last_clean":"2023-10-09T09:42:20.821451+0000","last_became_active":"2023-10-09T09:41:57.905789+0000","last_became_peered":"2023-10-09T09:41:57.905789+0000","last_unstale":"2023-10-09T09:42:20.821451+0000","last_undegraded":"2023-10-09T09:42:20.821451+0000","last_fullsized":"2023-10-09T09:42:20.821451+0000","mapping_epoch":1199,"log_start":"0'0","ondisk_log_start":"0'0","created":1199,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4854,"log_dups_size":0,"ondisk_log_size":4854,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184877056,"num_objects":2821,"num_object_clones":0,"num_object_copies":8463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2821,"num_write_kb":180544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":92,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869989+0000","last_change":"2023-10-09T09:40:25.856640+0000","last_active":"2023-10-09T09:41:58.869989+0000","last_peered":"2023-10-09T09:41:58.869989+0000","last_clean":"2023-10-09T09:41:58.869989+0000","last_became_active":"2023-10-09T09:40:25.802924+0000","last_became_peered":"2023-10-09T09:40:25.802924+0000","last_unstale":"2023-10-09T09:41:58.869989+0000","last_undegraded":"2023-10-09T09:41:58.869989+0000","last_fullsized":"2023-10-09T09:41:58.869989+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1176,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869835+0000","last_change":"2023-10-09T09:41:11.801626+0000","last_active":"2023-10-09T09:41:58.869835+0000","last_peered":"2023-10-09T09:41:58.869835+0000","last_clean":"2023-10-09T09:41:58.869835+0000","last_became_active":"2023-10-09T09:41:11.801169+0000","last_became_peered":"2023-10-09T09:41:11.801169+0000","last_unstale":"2023-10-09T09:41:58.869835+0000","last_undegraded":"2023-10-09T09:41:58.869835+0000","last_fullsized":"2023-10-09T09:41:58.869835+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.17","version":"1202'4799","reported_seq":4845,"reported_epoch":1202,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T09:42:20.928522+0000","last_change":"2023-10-09T09:41:57.931717+0000","last_active":"2023-10-09T09:42:20.928522+0000","last_peered":"2023-10-09T09:42:20.928522+0000","last_clean":"2023-10-09T09:41:10.748257+0000","last_became_active":"2023-10-09T09:41:57.927299+0000","last_became_peered":"2023-10-09T09:41:57.927299+0000","last_unstale":"2023-10-09T09:42:20.928522+0000","last_undegraded":"2023-10-09T09:41:57.914527+0000","last_fullsized":"2023-10-09T09:42:20.928522+0000","mapping_epoch":1199,"log_start":"0'0","ondisk_log_start":"0'0","created":1199,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4799,"log_dups_size":0,"ondisk_log_size":4799,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182157312,"num_objects":2780,"num_object_clones":0,"num_object_copies":8340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":364,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2780,"num_write_kb":177888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"1,2,5","objects":2416},{"shards":"2,3,5","objects":364}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":92,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.870566+0000","last_change":"2023-10-09T09:40:25.901197+0000","last_active":"2023-10-09T09:41:58.870566+0000","last_peered":"2023-10-09T09:41:58.870566+0000","last_clean":"2023-10-09T09:41:58.870566+0000","last_became_active":"2023-10-09T09:40:25.804595+0000","last_became_peered":"2023-10-09T09:40:25.804595+0000","last_unstale":"2023-10-09T09:41:58.870566+0000","last_undegraded":"2023-10-09T09:41:58.870566+0000","last_fullsized":"2023-10-09T09:41:58.870566+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1191,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869588+0000","last_change":"2023-10-09T09:41:11.801341+0000","last_active":"2023-10-09T09:41:58.869588+0000","last_peered":"2023-10-09T09:41:58.869588+0000","last_clean":"2023-10-09T09:41:58.869588+0000","last_became_active":"2023-10-09T09:41:11.800954+0000","last_became_peered":"2023-10-09T09:41:11.800954+0000","last_unstale":"2023-10-09T09:41:58.869588+0000","last_undegraded":"2023-10-09T09:41:58.869588+0000","last_fullsized":"2023-10-09T09:41:58.869588+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"1202'4910","reported_seq":5709,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:19.756511+0000","last_change":"2023-10-09T09:41:59.537318+0000","last_active":"2023-10-09T09:42:19.756511+0000","last_peered":"2023-10-09T09:42:19.756511+0000","last_clean":"2023-10-09T09:42:19.756511+0000","last_became_active":"2023-10-09T09:41:57.909456+0000","last_became_peered":"2023-10-09T09:41:57.909456+0000","last_unstale":"2023-10-09T09:42:19.756511+0000","last_undegraded":"2023-10-09T09:42:19.756511+0000","last_fullsized":"2023-10-09T09:42:19.756511+0000","mapping_epoch":1199,"log_start":"0'0","ondisk_log_start":"0'0","created":1199,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4910,"log_dups_size":0,"ondisk_log_size":4910,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187170816,"num_objects":2856,"num_object_clones":0,"num_object_copies":8568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2856,"num_write_kb":182784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":24346624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":135,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.865667+0000","last_change":"2023-10-09T09:41:11.816169+0000","last_active":"2023-10-09T09:41:58.865667+0000","last_peered":"2023-10-09T09:41:58.865667+0000","last_clean":"2023-10-09T09:41:58.865667+0000","last_became_active":"2023-10-09T09:41:11.815426+0000","last_became_peered":"2023-10-09T09:41:11.815426+0000","last_unstale":"2023-10-09T09:41:58.865667+0000","last_undegraded":"2023-10-09T09:41:58.865667+0000","last_fullsized":"2023-10-09T09:41:58.865667+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":73,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.851930+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:41:58.851930+0000","last_peered":"2023-10-09T09:41:58.851930+0000","last_clean":"2023-10-09T09:41:58.851930+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:41:58.851930+0000","last_undegraded":"2023-10-09T09:41:58.851930+0000","last_fullsized":"2023-10-09T09:41:58.851930+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"1202'4893","reported_seq":4942,"reported_epoch":1202,"state":"active+clean","last_fresh":"2023-10-09T09:42:22.340142+0000","last_change":"2023-10-09T09:41:57.916208+0000","last_active":"2023-10-09T09:42:22.340142+0000","last_peered":"2023-10-09T09:42:22.340142+0000","last_clean":"2023-10-09T09:42:22.340142+0000","last_became_active":"2023-10-09T09:41:57.911165+0000","last_became_peered":"2023-10-09T09:41:57.911165+0000","last_unstale":"2023-10-09T09:42:22.340142+0000","last_undegraded":"2023-10-09T09:42:22.340142+0000","last_fullsized":"2023-10-09T09:42:22.340142+0000","mapping_epoch":1199,"log_start":"0'0","ondisk_log_start":"0'0","created":1199,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:40:43.292673+0000","last_clean_scrub_stamp":"2023-10-09T09:40:43.292673+0000","objects_scrubbed":0,"log_size":4893,"log_dups_size":0,"ondisk_log_size":4893,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187269120,"num_objects":2858,"num_object_clones":0,"num_object_copies":8574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2858,"num_write_kb":182880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1160,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.864007+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:41:58.864007+0000","last_peered":"2023-10-09T09:41:58.864007+0000","last_clean":"2023-10-09T09:41:58.864007+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:41:58.864007+0000","last_undegraded":"2023-10-09T09:41:58.864007+0000","last_fullsized":"2023-10-09T09:41:58.864007+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1152,"reported_epoch":1201,"state":"active+clean","last_fresh":"2023-10-09T09:41:58.869303+0000","last_change":"2023-10-09T09:14:40.488294+0000","last_active":"2023-10-09T09:41:58.869303+0000","last_peered":"2023-10-09T09:41:58.869303+0000","last_clean":"2023-10-09T09:41:58.869303+0000","last_became_active":"2023-10-09T09:14:40.487864+0000","last_became_peered":"2023-10-09T09:14:40.487864+0000","last_unstale":"2023-10-09T09:41:58.869303+0000","last_undegraded":"2023-10-09T09:41:58.869303+0000","last_fullsized":"2023-10-09T09:41:58.869303+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":7306,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":5924192256,"num_objects":90396,"num_object_clones":0,"num_object_copies":271188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4186,"num_objects_misplaced":33,"num_objects_unfound":0,"num_objects_dirty":90396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90396,"num_write_kb":5785344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3327,"num_bytes_recovered":218038272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17806589952,"data_stored":17806589952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129675,"ondisk_log_size":129675,"up":78,"acting":76,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453398007,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594240,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3373732,"kb_used_data":2771740,"kb_used_omap":0,"kb_used_meta":601984,"kb_avail":90371420,"statfs":{"total":95995035648,"available":92540334080,"internally_reserved":0,"allocated":2838261760,"data_stored":2838005188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616431616},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:41:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7029999999999998,"5min":1.77,"15min":1.359},"min":{"1min":0.46700000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":22.742000000000001,"5min":22.742000000000001,"15min":22.742000000000001},"last":2.7320000000000002},{"interface":"front","average":{"1min":2.7639999999999998,"5min":2.0459999999999998,"15min":1.506},"min":{"1min":0.503,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":22.766999999999999,"5min":22.766999999999999,"15min":22.766999999999999},"last":4.1639999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:41:28 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":2.2309999999999999,"15min":2.5249999999999999},"min":{"1min":0.42099999999999999,"5min":0.27200000000000002,"15min":0.22900000000000001},"max":{"1min":2.4889999999999999,"5min":38.709000000000003,"15min":103.259},"last":2.7949999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":2.218,"15min":2.2930000000000001},"min":{"1min":0.40400000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.899,"5min":35.820999999999998,"15min":102.005},"last":0.26900000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":1.9790000000000001,"15min":1.869},"min":{"1min":0.32200000000000001,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":4.0579999999999998,"5min":32.661999999999999,"15min":33.210000000000001},"last":4.2240000000000002},{"interface":"front","average":{"1min":1.4530000000000001,"5min":2.206,"15min":1.887},"min":{"1min":0.40699999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.0289999999999999,"5min":34.030999999999999,"15min":37.246000000000002},"last":4.3490000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:42:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.671,"15min":2.423},"min":{"1min":0.36299999999999999,"5min":0.309,"15min":0.309},"max":{"1min":6.0880000000000001,"5min":11.547000000000001,"15min":123.97199999999999},"last":6.0880000000000001},{"interface":"front","average":{"1min":1.391,"5min":1.5509999999999999,"15min":2.2189999999999999},"min":{"1min":0.372,"5min":0.32900000000000001,"15min":0.29499999999999998},"max":{"1min":5.883,"5min":9.5679999999999996,"15min":124.059},"last":0.372}]},{"osd":4,"last update":"Mon Oct 9 09:42:20 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":1.2909999999999999,"15min":1.1399999999999999},"min":{"1min":0.39300000000000002,"5min":0.28399999999999997,"15min":0.28199999999999997},"max":{"1min":4.4139999999999997,"5min":8.0800000000000001,"15min":9.1099999999999994},"last":4.2690000000000001},{"interface":"front","average":{"1min":1.534,"5min":1.389,"15min":1.331},"min":{"1min":0.35699999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.9050000000000002,"5min":5.9470000000000001,"15min":9.7919999999999998},"last":1.5329999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:42:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.304,"15min":1.298},"min":{"1min":0.46000000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.8319999999999999,"5min":8.1400000000000006,"15min":9.0559999999999992},"last":4.3070000000000004},{"interface":"front","average":{"1min":1.7190000000000001,"5min":1.4510000000000001,"15min":1.464},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.9720000000000004,"5min":6.9779999999999998,"15min":9.7889999999999997},"last":0.34000000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:41:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.2789999999999999,"15min":1.393},"min":{"1min":0.40400000000000003,"5min":0.39600000000000002,"15min":0.35599999999999998},"max":{"1min":3.3599999999999999,"5min":5.1390000000000002,"15min":7.6769999999999996},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.167,"15min":1.395},"min":{"1min":0.40300000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.7230000000000001,"5min":6.931,"15min":7.6879999999999997},"last":2.762}]}]},{"osd":2,"up_from":634,"seq":2723009265993,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2654264,"kb_used_data":2112368,"kb_used_omap":122,"kb_used_meta":541765,"kb_avail":91090888,"statfs":{"total":95995035648,"available":93277069312,"internally_reserved":0,"allocated":2163064832,"data_stored":2161689340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":554767947},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:41:29 2023","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":2.6400000000000001,"15min":2.6240000000000001},"min":{"1min":0.37,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.415,"5min":86.221999999999994,"15min":86.221999999999994},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":2.6349999999999998,"15min":2.899},"min":{"1min":0.39000000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.0910000000000002,"5min":82.183000000000007,"15min":82.183000000000007},"last":0.39200000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:42:17 2023","interfaces":[{"interface":"back","average":{"1min":5.3609999999999998,"5min":3.8290000000000002,"15min":2.6880000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.184},"max":{"1min":55.295000000000002,"5min":80.655000000000001,"15min":81.662999999999997},"last":0.71399999999999997},{"interface":"front","average":{"1min":6.3090000000000002,"5min":3.9020000000000001,"15min":2.851},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.216},"max":{"1min":56.201000000000001,"5min":80.381,"15min":81.712999999999994},"last":1.046}]},{"osd":3,"last update":"Mon Oct 9 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":14.214,"5min":4.415,"15min":2.7480000000000002},"min":{"1min":0.41099999999999998,"5min":0.29699999999999999,"15min":0.252},"max":{"1min":230.476,"5min":230.476,"15min":230.476},"last":0.51900000000000002},{"interface":"front","average":{"1min":14.17,"5min":4.3200000000000003,"15min":2.7959999999999998},"min":{"1min":0.55500000000000005,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":228.893,"5min":228.893,"15min":228.893},"last":0.36599999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:42:17 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.6559999999999999,"15min":1.5409999999999999},"min":{"1min":0.34399999999999997,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.8620000000000001,"5min":11.622999999999999,"15min":11.622999999999999},"last":1.2589999999999999},{"interface":"front","average":{"1min":1.621,"5min":1.5880000000000001,"15min":1.569},"min":{"1min":0.38800000000000001,"5min":0.35399999999999998,"15min":0.313},"max":{"1min":7.6369999999999996,"5min":10.394,"15min":13.917999999999999},"last":0.26700000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.635,"15min":1.4610000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.9329999999999998,"5min":9.7379999999999995,"15min":9.7599999999999998},"last":1.714},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.7210000000000001,"15min":1.629},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.36499999999999999},"max":{"1min":5.7709999999999999,"5min":11.718999999999999,"15min":11.718999999999999},"last":0.41299999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.0499999999999998,"15min":1.8160000000000001},"min":{"1min":0.435,"5min":0.40300000000000002,"15min":0.33100000000000002},"max":{"1min":5.8259999999999996,"5min":11.648,"15min":11.648},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.9730000000000001,"5min":1.6839999999999999,"15min":1.6970000000000001},"min":{"1min":0.41999999999999998,"5min":0.35599999999999998,"15min":0.33300000000000002},"max":{"1min":7.6029999999999998,"5min":10.436999999999999,"15min":13.893000000000001},"last":0.432}]},{"osd":7,"last update":"Mon Oct 9 09:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.661,"15min":1.7849999999999999},"min":{"1min":0.42499999999999999,"5min":0.39400000000000002,"15min":0.377},"max":{"1min":7.8550000000000004,"5min":10.462999999999999,"15min":13.848000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.581,"15min":1.6200000000000001},"min":{"1min":0.46200000000000002,"5min":0.44800000000000001,"15min":0.34499999999999997},"max":{"1min":3.2090000000000001,"5min":9.7759999999999998,"15min":10.199},"last":0.55100000000000005}]}]},{"osd":1,"up_from":607,"seq":2607045149006,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4734436,"kb_used_data":4131100,"kb_used_omap":107,"kb_used_meta":603220,"kb_avail":89010716,"statfs":{"total":95995035648,"available":91146973184,"internally_reserved":0,"allocated":4230246400,"data_stored":4228884049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":617697963},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,5,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:41:28 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":1.49,"15min":1.4299999999999999},"min":{"1min":0.32500000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.611,"5min":11.458,"15min":11.458},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.9259999999999999,"15min":2.3180000000000001},"min":{"1min":0.46100000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.2349999999999999,"5min":15.603,"15min":15.603},"last":0.72499999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:41:54 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.9219999999999999,"15min":2.601},"min":{"1min":0.44500000000000001,"5min":0.23300000000000001,"15min":0.20100000000000001},"max":{"1min":4.4870000000000001,"5min":14.225,"15min":84.578999999999994},"last":0.76200000000000001},{"interface":"front","average":{"1min":1.2989999999999999,"5min":1.6399999999999999,"15min":2.1110000000000002},"min":{"1min":0.36799999999999999,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":6.3630000000000004,"5min":8.3539999999999992,"15min":84.697000000000003},"last":0.77300000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:41:28 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.6839999999999999,"15min":3.073},"min":{"1min":0.44500000000000001,"5min":0.26000000000000001,"15min":0.23599999999999999},"max":{"1min":2.964,"5min":12.589,"15min":107.426},"last":0.74299999999999999},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.9470000000000001,"15min":3.5419999999999998},"min":{"1min":0.58899999999999997,"5min":0.29099999999999998,"15min":0.23200000000000001},"max":{"1min":2.706,"5min":14.27,"15min":106.15900000000001},"last":0.84199999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.4319999999999999,"15min":1.5649999999999999},"min":{"1min":0.52400000000000002,"5min":0.33200000000000002,"15min":0.26500000000000001},"max":{"1min":4.1500000000000004,"5min":6.5339999999999998,"15min":84.875},"last":1.6759999999999999},{"interface":"front","average":{"1min":1.036,"5min":1.4730000000000001,"15min":1.75},"min":{"1min":0.52200000000000002,"5min":0.33200000000000002,"15min":0.29499999999999998},"max":{"1min":2.5539999999999998,"5min":7.7560000000000002,"15min":84.992999999999995},"last":1.097}]},{"osd":5,"last update":"Mon Oct 9 09:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.522,"15min":1.6379999999999999},"min":{"1min":0.58599999999999997,"5min":0.34699999999999998,"15min":0.32500000000000001},"max":{"1min":2.9140000000000001,"5min":8.6869999999999994,"15min":84.805000000000007},"last":1.796},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.4419999999999999,"15min":1.722},"min":{"1min":0.48799999999999999,"5min":0.35899999999999999,"15min":0.30399999999999999},"max":{"1min":3.0449999999999999,"5min":6.2569999999999997,"15min":84.951999999999998},"last":1.139}]},{"osd":6,"last update":"Mon Oct 9 09:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":1.8999999999999999,"15min":2.1400000000000001},"min":{"1min":0.59299999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.9740000000000002,"5min":14.335000000000001,"15min":85.016000000000005},"last":0.79300000000000004},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.6819999999999999,"15min":1.7829999999999999},"min":{"1min":0.40200000000000002,"5min":0.371,"15min":0.36799999999999999},"max":{"1min":1.647,"5min":6.593,"15min":84.831999999999994},"last":1.2809999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.9590000000000001,"15min":2.3109999999999999},"min":{"1min":0.68200000000000005,"5min":0.36199999999999999,"15min":0.33100000000000002},"max":{"1min":2.5659999999999998,"5min":14.364000000000001,"15min":84.888000000000005},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.7470000000000001,"15min":1.9730000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.36699999999999999},"max":{"1min":1.671,"5min":8.9540000000000006,"15min":84.742000000000004},"last":1.0469999999999999}]}]},{"osd":0,"up_from":1136,"seq":4879082848342,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2813288,"kb_used_data":2529312,"kb_used_omap":280,"kb_used_meta":283687,"kb_avail":90931864,"statfs":{"total":95995035648,"available":93114228736,"internally_reserved":0,"allocated":2590015488,"data_stored":2588703921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":290495898},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":2.3149999999999999,"15min":1.3009999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":8.3569999999999993,"5min":33.634,"15min":33.634},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.3260000000000001,"15min":1.468},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.4030000000000005,"5min":33.661000000000001,"15min":33.661000000000001},"last":0.58199999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.5700000000000001,"15min":1.1080000000000001},"min":{"1min":0.29099999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.573,"5min":8.9640000000000004,"15min":8.9640000000000004},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.5920000000000001,"15min":1.3640000000000001},"min":{"1min":0.317,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.9569999999999999,"5min":14.382999999999999,"15min":14.382999999999999},"last":0.69599999999999995}]},{"osd":3,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.4690000000000001,"15min":1.3049999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":3.6579999999999999,"5min":7.9219999999999997,"15min":7.9219999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.5940000000000001,"15min":1.103},"min":{"1min":0.25800000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.0129999999999999,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.52300000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.4079999999999999,"15min":1.2390000000000001},"min":{"1min":0.46300000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.8160000000000001,"5min":6.8390000000000004,"15min":6.8390000000000004},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.383,"5min":1.667,"15min":1.383},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.8079999999999998,"5min":6.9009999999999998,"15min":6.9009999999999998},"last":0.34300000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.482,"15min":1.3360000000000001},"min":{"1min":0.41999999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.7000000000000002,"5min":6.915,"15min":6.915},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.2729999999999999,"5min":1.5409999999999999,"15min":1.593},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0499999999999998,"5min":6.242,"15min":7.7149999999999999},"last":0.78100000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.5069999999999999,"15min":1.2170000000000001},"min":{"1min":0.44700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.762,"5min":7.6909999999999998,"15min":7.6909999999999998},"last":2.0169999999999999},{"interface":"front","average":{"1min":1.3089999999999999,"5min":1.794,"15min":1.8069999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.718,"5min":8.625,"15min":8.625},"last":0.76600000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:41:27 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.7410000000000001,"15min":1.3660000000000001},"min":{"1min":0.53500000000000003,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.7549999999999999,"5min":9.2159999999999993,"15min":9.2159999999999993},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.724,"15min":1.6439999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":3.7360000000000002,"5min":14.361000000000001,"15min":14.361000000000001},"last":0.68700000000000006}]}]},{"osd":3,"up_from":0,"seq":2847563317571,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986705,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2408036,"kb_used_data":1814236,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91337116,"statfs":{"total":95995035648,"available":93529206784,"internally_reserved":0,"allocated":1857777664,"data_stored":1857520570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:41:26 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.5169999999999999,"15min":1.143},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":1.216,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.75,"15min":1.3520000000000001},"min":{"1min":0.37,"5min":0.30099999999999999,"15min":0.29899999999999999},"max":{"1min":4.0069999999999997,"5min":16.387,"15min":16.387},"last":0.40300000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:41:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":2.4260000000000002,"15min":2.1749999999999998},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":3.9129999999999998,"5min":64.671999999999997,"15min":85.186999999999998},"last":25.422000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":2.8919999999999999,"15min":2.4300000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.4260000000000002,"5min":64.576999999999998,"15min":85.159000000000006},"last":25.956}]},{"osd":2,"last update":"Mon Oct 9 09:41:49 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.627,"15min":1.7949999999999999},"min":{"1min":0.37,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.403,"5min":33.743000000000002,"15min":99.736000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.206,"5min":1.849,"15min":1.833},"min":{"1min":0.432,"5min":0.32100000000000001,"15min":0.30099999999999999},"max":{"1min":6.9580000000000002,"5min":33.805999999999997,"15min":99.212999999999994},"last":0.52500000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.2989999999999999,"15min":2.0190000000000001},"min":{"1min":0.38600000000000001,"5min":0.30199999999999999,"15min":0.26500000000000001},"max":{"1min":3.98,"5min":5.5789999999999997,"15min":47.32},"last":0.622},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.5629999999999999,"15min":2.4769999999999999},"min":{"1min":0.47299999999999998,"5min":0.36599999999999999,"15min":0.314},"max":{"1min":2.577,"5min":16.507000000000001,"15min":93.843000000000004},"last":0.60799999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:41:37 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.4830000000000001,"15min":1.3879999999999999},"min":{"1min":0.38200000000000001,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":2.9660000000000002,"5min":17.212,"15min":17.212},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.764,"15min":1.5},"min":{"1min":0.42599999999999999,"5min":0.379,"15min":0.30599999999999999},"max":{"1min":4.0339999999999998,"5min":17.268999999999998,"15min":17.268999999999998},"last":0.55300000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.4490000000000001,"15min":1.3420000000000001},"min":{"1min":0.35699999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":2.6040000000000001,"5min":13.654,"15min":13.654},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.452,"15min":1.361},"min":{"1min":0.42299999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.2400000000000002,"5min":7.9829999999999997,"15min":9.3369999999999997},"last":0.51600000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.1200000000000001,"15min":1.264},"min":{"1min":0.45600000000000002,"5min":0.442,"15min":0.33400000000000002},"max":{"1min":3.077,"5min":5.6059999999999999,"15min":9.2889999999999997},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.194,"15min":1.3959999999999999},"min":{"1min":0.46800000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.742,"5min":5.6219999999999999,"15min":12.34},"last":0.495}]}]},{"osd":5,"up_from":37,"seq":158913790471,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4199760,"kb_used_data":3610056,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":89545392,"statfs":{"total":95995035648,"available":91694481408,"internally_reserved":0,"allocated":3696697344,"data_stored":3696456404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:41:29 2023","interfaces":[{"interface":"back","average":{"1min":6.9279999999999999,"5min":3.6989999999999998,"15min":5.7590000000000003},"min":{"1min":0.49099999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":119.724,"5min":119.724,"15min":119.724},"last":1.2609999999999999},{"interface":"front","average":{"1min":6.9889999999999999,"5min":3.5529999999999999,"15min":5.1230000000000002},"min":{"1min":0.40999999999999998,"5min":0.245,"15min":0.245},"max":{"1min":119.872,"5min":119.872,"15min":119.872},"last":3.1749999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":5.9420000000000002,"15min":3.6269999999999998},"min":{"1min":0.31900000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.4640000000000004,"5min":142.67500000000001,"15min":142.67500000000001},"last":3.323},{"interface":"front","average":{"1min":1.476,"5min":6.008,"15min":3.7080000000000002},"min":{"1min":0.43099999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.4889999999999999,"5min":142.56399999999999,"15min":142.56399999999999},"last":3.2949999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":1.373,"15min":1.5349999999999999},"min":{"1min":0.35199999999999998,"5min":0.312,"15min":0.31},"max":{"1min":2.641,"5min":7.7110000000000003,"15min":14.019},"last":3.2589999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.5069999999999999,"15min":1.829},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.28000000000000003},"max":{"1min":2.1429999999999998,"5min":13.356,"15min":14.037000000000001},"last":0.495}]},{"osd":3,"last update":"Mon Oct 9 09:42:19 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.425,"15min":1.877},"min":{"1min":0.33600000000000002,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":3.266,"5min":10.125999999999999,"15min":59.652000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.2290000000000001,"15min":1.962},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.29099999999999998},"max":{"1min":1.9099999999999999,"5min":8.3000000000000007,"15min":61.771999999999998},"last":1.8420000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.179,"15min":1.1240000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.28799999999999998},"max":{"1min":3.298,"5min":8.4019999999999992,"15min":10.747},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.5609999999999999,"15min":1.409},"min":{"1min":0.52600000000000002,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":6.9020000000000001,"5min":10.324,"15min":12.151},"last":0.45800000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.3109999999999999,"15min":1.347},"min":{"1min":0.36399999999999999,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":3.4089999999999998,"5min":6.2969999999999997,"15min":14.518000000000001},"last":0.83099999999999996},{"interface":"front","average":{"1min":1.454,"5min":1.417,"15min":1.726},"min":{"1min":0.38,"5min":0.36299999999999999,"15min":0.32900000000000001},"max":{"1min":6.8659999999999997,"5min":13.33,"15min":18.638999999999999},"last":0.80400000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":1.5569999999999999,"15min":1.6180000000000001},"min":{"1min":0.55500000000000005,"5min":0.39600000000000002,"15min":0.33400000000000002},"max":{"1min":3.7170000000000001,"5min":13.220000000000001,"15min":18.683},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.4750000000000001,"15min":1.482},"min":{"1min":0.433,"5min":0.36299999999999999,"15min":0.32100000000000001},"max":{"1min":2.6040000000000001,"5min":10.192,"15min":10.192},"last":3.2189999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2613313536,"data_stored":2613313536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4278190080,"data_stored":4278190080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2161246208,"data_stored":2161246208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":321912832,"data_stored":321912832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1863581696,"data_stored":1863581696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3707568128,"data_stored":3707568128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2860777472,"data_stored":2860777472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:42:25.340 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T09:42:25.341 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 31 2023-10-09T09:42:25.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:25.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:25.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:25.478 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:25.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:25.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:25.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:25.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:25.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:25.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:25.711 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:25.737 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:25.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:25.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:25.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:25.873 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:25.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:25.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:25.978 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:26.005 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:26.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:26.065 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 94656 94640 73.015 64.9375 0.0132725 0.0136901 2023-10-09T09:42:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:25 smithi181 bash[59345]: cluster 2023-10-09T09:42:24.000895+0000 mgr.y (mgr.34107) 1091 : cluster 0 pgmap v1892: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.5 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 55 MiB/s wr, 875 op/s; 4186/271260 objects degraded (1.543%); 33/271260 objects misplaced (0.012%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:42:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:25 smithi181 bash[59345]: cluster 2023-10-09T09:42:24.392500+0000 osd.7 (osd.7) 579 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:25 smithi181 bash[59345]: audit 2023-10-09T09:42:25.387279+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:25 smithi181 bash[59345]: cluster 2023-10-09T09:42:25.766937+0000 mon.a (mon.0) 2094 : cluster 3 Health check update: Degraded data redundancy: 4186/271260 objects degraded (1.543%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:42:26.110 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:26.118 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:25 smithi163 bash[78107]: cluster 2023-10-09T09:42:24.000895+0000 mgr.y (mgr.34107) 1091 : cluster 0 pgmap v1892: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.5 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 55 MiB/s wr, 875 op/s; 4186/271260 objects degraded (1.543%); 33/271260 objects misplaced (0.012%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:42:26.118 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:25 smithi163 bash[78107]: cluster 2023-10-09T09:42:24.392500+0000 osd.7 (osd.7) 579 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:26.118 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:25 smithi163 bash[78107]: audit 2023-10-09T09:42:25.387279+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:26.118 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:25 smithi163 bash[78107]: cluster 2023-10-09T09:42:25.766937+0000 mon.a (mon.0) 2094 : cluster 3 Health check update: Degraded data redundancy: 4186/271260 objects degraded (1.543%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:42:26.118 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:25 smithi163 bash[75425]: cluster 2023-10-09T09:42:24.000895+0000 mgr.y (mgr.34107) 1091 : cluster 0 pgmap v1892: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 86 active+clean; 5.5 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 55 MiB/s wr, 875 op/s; 4186/271260 objects degraded (1.543%); 33/271260 objects misplaced (0.012%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T09:42:26.118 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:25 smithi163 bash[75425]: cluster 2023-10-09T09:42:24.392500+0000 osd.7 (osd.7) 579 : cluster 3 failed to encode map e1204 with expected crc 2023-10-09T09:42:26.118 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:25 smithi163 bash[75425]: audit 2023-10-09T09:42:25.387279+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:26.118 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:25 smithi163 bash[75425]: cluster 2023-10-09T09:42:25.766937+0000 mon.a (mon.0) 2094 : cluster 3 Health check update: Degraded data redundancy: 4186/271260 objects degraded (1.543%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:42:26.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:26.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:26.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:26.284 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:26.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:26.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:26.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:26.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:26.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:26.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:26.549 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:26.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:26.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:26.653 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:26.692 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:26.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:26.796 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:26.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:26.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:26.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:26.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:26.971 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:26.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:26.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:27.066 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 95738 95722 72.9492 67.625 0.00944696 0.0137025 2023-10-09T09:42:27.074 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:27.103 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:27.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:27.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:27.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:27.241 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:27.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:27.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:27.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:27.375 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:27.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:27.478 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:27.509 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:27.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:27.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:27.613 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:27.643 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:27.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:27.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:27.748 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:27.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:27.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:27.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:27.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:27.919 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:27.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:27.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:28.022 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:28.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:27 smithi181 bash[59345]: cluster 2023-10-09T09:42:26.001637+0000 mgr.y (mgr.34107) 1092 : cluster 0 pgmap v1893: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 87 active+clean; 5.6 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 61 MiB/s wr, 983 op/s; 3427/276792 objects degraded (1.238%); 53/276792 objects misplaced (0.019%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T09:42:28.049 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:28.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:28.065 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 96788 96772 72.8608 65.625 0.00775816 0.0137191 2023-10-09T09:42:28.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:28.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:28.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:28.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:28.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:27 smithi163 bash[78107]: cluster 2023-10-09T09:42:26.001637+0000 mgr.y (mgr.34107) 1092 : cluster 0 pgmap v1893: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 87 active+clean; 5.6 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 61 MiB/s wr, 983 op/s; 3427/276792 objects degraded (1.238%); 53/276792 objects misplaced (0.019%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T09:42:28.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:27 smithi163 bash[75425]: cluster 2023-10-09T09:42:26.001637+0000 mgr.y (mgr.34107) 1092 : cluster 0 pgmap v1893: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 87 active+clean; 5.6 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 61 MiB/s wr, 983 op/s; 3427/276792 objects degraded (1.238%); 53/276792 objects misplaced (0.019%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T09:42:28.298 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:28.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:28.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:28.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:28.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:28.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:28.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:28.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:28.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:28.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:28.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:28.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:28.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:28.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:28.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:28.843 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:28.869 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:28.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:28.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:28.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:28.992 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:29.032 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:29.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:29.066 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 97873 97857 72.8007 67.8125 0.0152183 0.0137306 2023-10-09T09:42:29.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:29.162 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:29.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:29.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:29.299 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:29.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:29.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:29.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:29.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:29.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:29.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:29.538 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:29.567 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:29.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:29.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:29.700 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:29.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:29.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:29.804 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:29.831 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:29.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:29.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:29.888 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:29.888 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:29.935 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:29.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:29 smithi163 bash[78107]: cluster 2023-10-09T09:42:28.003115+0000 mgr.y (mgr.34107) 1093 : cluster 0 pgmap v1894: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 89 active+clean; 5.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 62 MiB/s wr, 992 op/s; 2701/284106 objects degraded (0.951%); 713/284106 objects misplaced (0.251%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T09:42:29.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:29 smithi163 bash[75425]: cluster 2023-10-09T09:42:28.003115+0000 mgr.y (mgr.34107) 1093 : cluster 0 pgmap v1894: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 89 active+clean; 5.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 62 MiB/s wr, 992 op/s; 2701/284106 objects degraded (0.951%); 713/284106 objects misplaced (0.251%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T09:42:29.971 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:29.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:29.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:30.066 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 98960 98944 72.7434 67.9375 0.0121405 0.0137418 2023-10-09T09:42:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:29 smithi181 bash[59345]: cluster 2023-10-09T09:42:28.003115+0000 mgr.y (mgr.34107) 1093 : cluster 0 pgmap v1894: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 89 active+clean; 5.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 62 MiB/s wr, 992 op/s; 2701/284106 objects degraded (0.951%); 713/284106 objects misplaced (0.251%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T09:42:30.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:30.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:30.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:30.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:30.141 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:42:30.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:30.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:30.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:30.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:30.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:30.377 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:30.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:30.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:30.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:30.513 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:30.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:30.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:30.619 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:30.653 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:30.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:30.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:30.760 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:30.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:30.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:30.904 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:30.913 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:30 smithi181 bash[59345]: cluster 2023-10-09T09:42:30.768069+0000 mon.a (mon.0) 2095 : cluster 3 Health check update: Degraded data redundancy: 2701/284106 objects degraded (0.951%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:30.938 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:30.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:30.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:31.045 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:31.066 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 99898 99882 72.5792 58.625 0.00947589 0.0137724 2023-10-09T09:42:31.071 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:31.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:31.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:31.177 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:31.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:31.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:30 smithi163 bash[78107]: cluster 2023-10-09T09:42:30.768069+0000 mon.a (mon.0) 2095 : cluster 3 Health check update: Degraded data redundancy: 2701/284106 objects degraded (0.951%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:31.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:30 smithi163 bash[75425]: cluster 2023-10-09T09:42:30.768069+0000 mon.a (mon.0) 2095 : cluster 3 Health check update: Degraded data redundancy: 2701/284106 objects degraded (0.951%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:31.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:31.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:31.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:31.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:31.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:31.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:31.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:31.478 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:31.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:31.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:31.585 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:31.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:31.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:31.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:31.715 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:31.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:31.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:31.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:31.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:31.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:31.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:31.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:31.979 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:32.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:32.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:32.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:32.066 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 101002 100986 72.5379 69 0.0198805 0.01378 2023-10-09T09:42:32.114 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:32.123 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:31 smithi181 bash[59345]: cluster 2023-10-09T09:42:30.003722+0000 mgr.y (mgr.34107) 1094 : cluster 0 pgmap v1895: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 91 active+clean; 5.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 57 MiB/s wr, 913 op/s; 2348/289080 objects degraded (0.812%); 693/289080 objects misplaced (0.240%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T09:42:32.143 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:32.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:32.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:32.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:31 smithi163 bash[78107]: cluster 2023-10-09T09:42:30.003722+0000 mgr.y (mgr.34107) 1094 : cluster 0 pgmap v1895: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 91 active+clean; 5.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 57 MiB/s wr, 913 op/s; 2348/289080 objects degraded (0.812%); 693/289080 objects misplaced (0.240%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T09:42:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:31 smithi163 bash[75425]: cluster 2023-10-09T09:42:30.003722+0000 mgr.y (mgr.34107) 1094 : cluster 0 pgmap v1895: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 91 active+clean; 5.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 57 MiB/s wr, 913 op/s; 2348/289080 objects degraded (0.812%); 693/289080 objects misplaced (0.240%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T09:42:32.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:32.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:32.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:32.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:32.415 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:32.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:32.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:32.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:32.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:32.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:32.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:32.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:32.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:32.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:32.807 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:32.840 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:32.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:32.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:32.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:32.977 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:32.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:33.066 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 102009 101993 72.4288 62.9375 0.0060879 0.0138014 2023-10-09T09:42:33.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:33.108 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:33.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:33.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:33.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:33.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:33.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:33.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:33.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:33.384 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:33.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:33.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:33.487 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:33.496 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:42:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:42:33] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.33.4" 2023-10-09T09:42:33.514 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:33.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:33.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:33.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:33.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:33.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:33.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:33.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:33.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:33.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:33.915 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:33.928 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:33 smithi181 bash[59345]: cluster 2023-10-09T09:42:32.004572+0000 mgr.y (mgr.34107) 1095 : cluster 0 pgmap v1896: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 91 active+clean; 6.0 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 56 MiB/s wr, 899 op/s; 2348/296070 objects degraded (0.793%); 576/296070 objects misplaced (0.195%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T09:42:33.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:33.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:33.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:33.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:33 smithi163 bash[75425]: cluster 2023-10-09T09:42:32.004572+0000 mgr.y (mgr.34107) 1095 : cluster 0 pgmap v1896: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 91 active+clean; 6.0 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 56 MiB/s wr, 899 op/s; 2348/296070 objects degraded (0.793%); 576/296070 objects misplaced (0.195%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T09:42:33.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:33 smithi163 bash[78107]: cluster 2023-10-09T09:42:32.004572+0000 mgr.y (mgr.34107) 1095 : cluster 0 pgmap v1896: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 91 active+clean; 6.0 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 56 MiB/s wr, 899 op/s; 2348/296070 objects degraded (0.793%); 576/296070 objects misplaced (0.195%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T09:42:34.045 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:34.065 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:34.066 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 102969 102953 72.2891 60 0.0186357 0.013825 2023-10-09T09:42:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:34.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:34.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:34.194 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:34.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:34.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:34.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:34.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:34.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:34.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:34.432 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:34.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:34.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:34.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:34.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:34.608 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:34.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:34.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:34.714 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:34.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:34.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:34.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:34.856 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:34.881 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:34.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:34.984 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:35.018 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:35.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:35.066 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.0089 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 103893 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 72.1408 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 26.7771 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 155 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 0 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1154 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 428.447 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 2480 2023-10-09T09:42:35.067 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 0 2023-10-09T09:42:35.068 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.0138569 2023-10-09T09:42:35.068 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0408663 2023-10-09T09:42:35.068 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 2.09983 2023-10-09T09:42:35.068 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.00132859 2023-10-09T09:42:35.072 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T09:42:35.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:35.155 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:35.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:35.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:35.259 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:35.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:35.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:35.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:35.365 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:42:35.365 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T09:42:35.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T09:42:35.365 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:42:35.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.365 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:155882 7.5 7:a70806e8:::benchmark_data_smithi163_130508_object47219:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1119)", 2023-10-09T09:42:35.365 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:08.392415+0000", 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 566.96953704800001, 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.161041665, 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 155882 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.392415+0000", 2023-10-09T09:42:35.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.392415+0000", 2023-10-09T09:42:35.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999971 2023-10-09T09:42:35.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.392413+0000", 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5690000000000005e-06 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.392421+0000", 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.524e-05 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.392436+0000", 2023-10-09T09:42:35.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6710000000000002e-06 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.392443+0000", 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9138999999999997e-05 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.392502+0000", 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011881500000000001 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.369 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.392621+0000", 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000119097 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.392740+0000", 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030024919999999998 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.395743+0000", 2023-10-09T09:42:35.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.153684294 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.549427+0000", 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0038544540000000002 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.553281+0000", 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8213999999999993e-05 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.553360+0000", 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7319999999999997e-05 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:08.553457+0000", 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.161041665 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.373 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:197141 7.16 7:6b255505:::benchmark_data_smithi163_130508_object88478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:42:35.373 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:55.502252+0000", 2023-10-09T09:42:35.373 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 519.85969990599995, 2023-10-09T09:42:35.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18750082900000001, 2023-10-09T09:42:35.374 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.374 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.374 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.374 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:42:35.374 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:42:35.374 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 197141 2023-10-09T09:42:35.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.374 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.502252+0000", 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.502252+0000", 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.502251+0000", 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0789999999999997e-06 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.502257+0000", 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8849999999999994e-06 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.502267+0000", 2023-10-09T09:42:35.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9620000000000001e-06 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.502272+0000", 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3488000000000001e-05 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.502316+0000", 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118029 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.502434+0000", 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00051339599999999995 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3", 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.502947+0000", 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0061200129999999997 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.509067+0000", 2023-10-09T09:42:35.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18010934000000001 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689177+0000", 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9085000000000004e-05 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689276+0000", 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00047752999999999999 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689753+0000", 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18750082900000001 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:197146 7.6 7:66725293:::benchmark_data_smithi163_130508_object88483:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:55.505060+0000", 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 519.85689188100002, 2023-10-09T09:42:35.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18442408699999999, 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 197146 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.505060+0000", 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.505060+0000", 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.505059+0000", 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7860000000000002e-06 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.505065+0000", 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1467999999999999e-05 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.505077+0000", 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2610000000000001e-06 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.505082+0000", 2023-10-09T09:42:35.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5479000000000003e-05 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.505127+0000", 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000128193 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.505256+0000", 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00055893000000000002 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3", 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.505815+0000", 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003465367 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.509280+0000", 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17992469999999999 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689205+0000", 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1336999999999997e-05 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689286+0000", 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019849099999999999 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689484+0000", 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18442408699999999 2023-10-09T09:42:35.386 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:197152 7.16 7:6bfe0d4b:::benchmark_data_smithi163_130508_object88489:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:55.511827+0000", 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 519.85012519400004, 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17844727799999999, 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.387 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 197152 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.511827+0000", 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.511827+0000", 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.511825+0000", 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4319999999999999e-06 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.511834+0000", 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5395000000000001e-05 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.511849+0000", 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9929999999999998e-06 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.511856+0000", 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010641600000000001 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.511962+0000", 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014179899999999999 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.512104+0000", 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00055665399999999996 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3", 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.512661+0000", 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0026990590000000002 2023-10-09T09:42:35.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.515360+0000", 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17451344699999999 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689873+0000", 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121589 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689995+0000", 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00027936900000000001 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.690274+0000", 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17844727799999999 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.393 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:197153 7.11 7:8f2f2253:::benchmark_data_smithi163_130508_object88490:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:55.512523+0000", 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 519.84942972600004, 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17678476000000001, 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 197153 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.512523+0000", 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.512523+0000", 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.512521+0000", 2023-10-09T09:42:35.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5589999999999994e-06 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.512531+0000", 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3154e-05 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.512544+0000", 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4200000000000004e-06 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.512550+0000", 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5338000000000004e-05 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.512616+0000", 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014709799999999999 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.512763+0000", 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00055005600000000005 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.513313+0000", 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068705299999999997 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.520184+0000", 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111736 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.520295+0000", 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.168789511 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689085+0000", 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012625399999999999 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689211+0000", 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6259000000000001e-05 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.689307+0000", 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17678476000000001 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44921.0:197162 7.16 7:6a724440:::benchmark_data_smithi163_130508_object88499:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1124)", 2023-10-09T09:42:35.400 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:33:55.530928+0000", 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 519.83102409699995, 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.163854366, 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44921", 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/519057884", 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 197162 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.530928+0000", 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.530928+0000", 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.530926+0000", 2023-10-09T09:42:35.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2347000000000001e-05 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.530938+0000", 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9298000000000001e-05 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.530957+0000", 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0080000000000008e-06 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.530966+0000", 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4521e-05 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.531041+0000", 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014114499999999999 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.531182+0000", 2023-10-09T09:42:35.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00058009999999999995 2023-10-09T09:42:35.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3", 2023-10-09T09:42:35.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.531762+0000", 2023-10-09T09:42:35.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002868476 2023-10-09T09:42:35.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.534631+0000", 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15980270499999999 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.694433+0000", 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5080000000000005e-05 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.694518+0000", 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026414500000000003 2023-10-09T09:42:35.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:33:55.694783+0000", 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.163854366 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181717 8.d 8:bb1e4fb6:::benchmark_data_smithi163_151167_object86484:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1161)", 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.915604+0000", 2023-10-09T09:42:35.407 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 203.446347946, 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5257026279999999, 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181717 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.915604+0000", 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.915604+0000", 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.915602+0000", 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5440000000000004e-06 2023-10-09T09:42:35.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.915612+0000", 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4168999999999999e-05 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.915626+0000", 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1429999999999998e-06 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.915634+0000", 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4275000000000004e-05 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.915698+0000", 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014087 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.915839+0000", 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017068000000000001 2023-10-09T09:42:35.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.916010+0000", 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067441589999999996 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.922754+0000", 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023540409999999999 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.925108+0000", 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026911101 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.952019+0000", 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3154e-05 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.952032+0000", 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0031236729999999 2023-10-09T09:42:35.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.955156+0000", 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2466000000000001e-05 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.955178+0000", 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.441091691 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.396270+0000", 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000164795 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for missing object", 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.396435+0000", 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.044580329000000002 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.441015+0000", 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029168800000000002 2023-10-09T09:42:35.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.441307+0000", 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5257026279999999 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181721 8.8 8:1cce1d9c:::benchmark_data_smithi163_151167_object86488:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1161)", 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.927636+0000", 2023-10-09T09:42:35.416 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 203.43431577300001, 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2979840440000001, 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181721 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927636+0000", 2023-10-09T09:42:35.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927636+0000", 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927635+0000", 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.559e-06 2023-10-09T09:42:35.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927642+0000", 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4263999999999999e-05 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927657+0000", 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2554e-05 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927669+0000", 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2709e-05 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927722+0000", 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103828 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927826+0000", 2023-10-09T09:42:35.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011979199999999999 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927946+0000", 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066754350000000004 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.934621+0000", 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085044769999999999 2023-10-09T09:42:35.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.943125+0000", 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2396e-05 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.943138+0000", 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.013922201 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.957060+0000", 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1285e-05 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.957071+0000", 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2684530869999999 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.225524+0000", 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6058999999999996e-05 2023-10-09T09:42:35.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.225621+0000", 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2979840440000001 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181722 8.d 8:b8caa87c:::benchmark_data_smithi163_151167_object86489:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1161)", 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.928250+0000", 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 203.433702673, 2023-10-09T09:42:35.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.547349386, 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181722 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.928250+0000", 2023-10-09T09:42:35.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.928250+0000", 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.928249+0000", 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4670000000000001e-06 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.928255+0000", 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1284000000000001e-05 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.928266+0000", 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9820000000000002e-06 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.928273+0000", 2023-10-09T09:42:35.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2772000000000001e-05 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.928346+0000", 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000174506 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.928521+0000", 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019000899999999999 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.928711+0000", 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.023377057999999999 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.952088+0000", 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2875e-05 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.952101+0000", 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.007766296 2023-10-09T09:42:35.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.959867+0000", 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1558e-05 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.959879+0000", 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.437384427 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.397263+0000", 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015602500000000001 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for missing object", 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.397419+0000", 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.078034260999999994 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.475453+0000", 2023-10-09T09:42:35.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000145697 2023-10-09T09:42:35.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.475599+0000", 2023-10-09T09:42:35.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.547349386 2023-10-09T09:42:35.432 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.432 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.432 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181729 8.d 8:b8d637bc:::benchmark_data_smithi163_151167_object86496:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1162)", 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.969593+0000", 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 203.39235910900001, 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.504450308, 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181729 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.433 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.969593+0000", 2023-10-09T09:42:35.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.969593+0000", 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999971 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.969590+0000", 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2284e-05 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.969603+0000", 2023-10-09T09:42:35.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4340000000000001e-05 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.969627+0000", 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2687000000000001e-05 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.969640+0000", 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013667599999999999 2023-10-09T09:42:35.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.969776+0000", 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7093999999999998e-05 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.969803+0000", 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99015897500000005 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.959962+0000", 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.969e-06 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.959971+0000", 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4374799730000001 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.397451+0000", 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000168219 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.397619+0000", 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086089599999999999 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.398480+0000", 2023-10-09T09:42:35.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.02397958 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.422460+0000", 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7164999999999999e-05 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.422477+0000", 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050425096000000003 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.472902+0000", 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0557999999999997e-05 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.472983+0000", 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010606979999999999 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.474043+0000", 2023-10-09T09:42:35.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.504450308 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181730 8.8 8:18d1b2f3:::benchmark_data_smithi163_151167_object86497:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1162)", 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.971189+0000", 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 203.390762927, 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.324248715, 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.442 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181730 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.971189+0000", 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.971189+0000", 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.971188+0000", 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6880000000000005e-06 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.971196+0000", 2023-10-09T09:42:35.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2722e-05 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.971219+0000", 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0725e-05 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.971230+0000", 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4518000000000006e-05 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.971314+0000", 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6582000000000001e-05 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.971331+0000", 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98576755299999996 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.957098+0000", 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5930000000000006e-06 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.957107+0000", 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.268846892 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.225954+0000", 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014737999999999999 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.226101+0000", 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00081320899999999996 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.226914+0000", 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012180913999999999 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.239095+0000", 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00051170000000000002 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.239607+0000", 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.039888126000000003 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.279495+0000", 2023-10-09T09:42:35.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015746461999999999 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.295242+0000", 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6773000000000006e-05 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.295308+0000", 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000129546 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.295438+0000", 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.324248715 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181736 8.d 8:b9f0a500:::benchmark_data_smithi163_151167_object86503:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1162)", 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.984865+0000", 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 203.37708741899999, 2023-10-09T09:42:35.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.499229787, 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181736 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984865+0000", 2023-10-09T09:42:35.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984865+0000", 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984863+0000", 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.199e-06 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984870+0000", 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3226e-05 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984883+0000", 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3690000000000003e-06 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984890+0000", 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4676999999999999e-05 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984934+0000", 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0023999999999999e-05 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984944+0000", 2023-10-09T09:42:35.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97505459100000003 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.959999+0000", 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1759999999999993e-06 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.960007+0000", 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.446133922 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.406141+0000", 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00039982499999999997 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.406541+0000", 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00051989199999999999 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.407061+0000", 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.024132746 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.431194+0000", 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010214744 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.441408+0000", 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.041528149 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.482937+0000", 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1236999999999995e-05 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.483018+0000", 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001076825 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.484095+0000", 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.499229787 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181780 8.d 8:bbc7a9c0:::benchmark_data_smithi163_151167_object86547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1164)", 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:14.334834+0000", 2023-10-09T09:42:35.460 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 201.027118328, 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.165915808, 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181780 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.334834+0000", 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.334834+0000", 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.334832+0000", 2023-10-09T09:42:35.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1089999999999998e-06 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.334840+0000", 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6276000000000001e-05 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.334857+0000", 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6370000000000002e-06 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.334863+0000", 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4938000000000002e-05 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.334908+0000", 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2959999999999999e-06 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.334917+0000", 2023-10-09T09:42:35.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.077404030999999998 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.412321+0000", 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00034287400000000001 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.412663+0000", 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023557800000000001 2023-10-09T09:42:35.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.412899+0000", 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018324271999999999 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.431223+0000", 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010259446 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.441483+0000", 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.058211054999999998 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.499694+0000", 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8217000000000001e-05 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.499762+0000", 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00098767100000000008 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.500750+0000", 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.165915808 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.468 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181783 8.d 8:b717c73d:::benchmark_data_smithi163_151167_object86550:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1164)", 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:14.340628+0000", 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 201.02132416399999, 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.160822201, 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181783 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.469 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.470 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.470 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.340628+0000", 2023-10-09T09:42:35.470 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.470 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.470 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.340628+0000", 2023-10-09T09:42:35.470 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:42:35.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.471 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.471 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.340626+0000", 2023-10-09T09:42:35.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9059999999999997e-06 2023-10-09T09:42:35.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.471 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.471 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.340636+0000", 2023-10-09T09:42:35.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5068999999999999e-05 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.340651+0000", 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4309999999999997e-06 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.340658+0000", 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.076704597999999999 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.472 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.417363+0000", 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015442799999999999 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.417517+0000", 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000197853 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:42:35.473 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.417715+0000", 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013777367 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.431493+0000", 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010051387 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.441544+0000", 2023-10-09T09:42:35.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.059252315 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.500796+0000", 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2450999999999997e-05 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.500839+0000", 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00061143299999999999 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.475 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.501450+0000", 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.160822201 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189439 8.d 8:bd48eb33:::benchmark_data_smithi163_151167_object94206:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.989276+0000", 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.37267664399999, 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26184209400000003, 2023-10-09T09:42:35.476 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189439 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989276+0000", 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.477 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989276+0000", 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989274+0000", 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7629999999999995e-06 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.478 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989283+0000", 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5821999999999999e-05 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989299+0000", 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3839999999999997e-06 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.479 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989308+0000", 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0151000000000001e-05 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989368+0000", 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018989699999999999 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989558+0000", 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024738599999999999 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989805+0000", 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053018229999999998 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.995107+0000", 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043423330000000003 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.481 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.999449+0000", 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25065389900000001 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.250103+0000", 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2815e-05 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.250176+0000", 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00094180499999999999 2023-10-09T09:42:35.482 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.251118+0000", 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26184209400000003 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189445 8.d 8:bdd58a93:::benchmark_data_smithi163_151167_object94212:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:27.004403+0000", 2023-10-09T09:42:35.483 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.35754947199999, 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25528980800000001, 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189445 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.004403+0000", 2023-10-09T09:42:35.484 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.004403+0000", 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.004401+0000", 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7279999999999998e-06 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.485 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.004408+0000", 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2411e-05 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.004420+0000", 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.485e-06 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:42:35.486 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.004426+0000", 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8804999999999997e-05 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.004484+0000", 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017276999999999999 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.004657+0000", 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017876400000000001 2023-10-09T09:42:35.487 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.004836+0000", 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040674129999999998 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.008903+0000", 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036621750000000002 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.488 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.012566+0000", 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24618774800000001 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.258753+0000", 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1229999999999994e-05 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.258825+0000", 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086803900000000003 2023-10-09T09:42:35.489 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:42:35.490 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:42:35.490 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:42:35.490 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.259693+0000", 2023-10-09T09:42:35.490 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25528980800000001 2023-10-09T09:42:35.490 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.490 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.490 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.490 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:42:35.490 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:42:35.490 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:42:35.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:35.525 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:35.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:35.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:35.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:35.662 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:35.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:35.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:35.766 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:35.800 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:35.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:35.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:35.849 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:35 smithi163 bash[75425]: cluster 2023-10-09T09:42:34.005430+0000 mgr.y (mgr.34107) 1096 : cluster 0 pgmap v1897: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 91 active+clean; 6.1 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 58 MiB/s wr, 934 op/s; 2348/299850 objects degraded (0.783%); 576/299850 objects misplaced (0.192%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T09:42:35.849 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:35 smithi163 bash[75425]: cluster 2023-10-09T09:42:35.771243+0000 mon.a (mon.0) 2096 : cluster 3 Health check update: Degraded data redundancy: 2348/299850 objects degraded (0.783%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:35.906 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:35.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:35.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:36.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:36.057 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:35 smithi181 bash[59345]: cluster 2023-10-09T09:42:34.005430+0000 mgr.y (mgr.34107) 1096 : cluster 0 pgmap v1897: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 91 active+clean; 6.1 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 58 MiB/s wr, 934 op/s; 2348/299850 objects degraded (0.783%); 576/299850 objects misplaced (0.192%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T09:42:36.057 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:35 smithi181 bash[59345]: cluster 2023-10-09T09:42:35.771243+0000 mon.a (mon.0) 2096 : cluster 3 Health check update: Degraded data redundancy: 2348/299850 objects degraded (0.783%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:36.074 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:36.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:36.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:36.183 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:36.207 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:36.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:36.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:36.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:35 smithi163 bash[78107]: cluster 2023-10-09T09:42:34.005430+0000 mgr.y (mgr.34107) 1096 : cluster 0 pgmap v1897: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 91 active+clean; 6.1 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 58 MiB/s wr, 934 op/s; 2348/299850 objects degraded (0.783%); 576/299850 objects misplaced (0.192%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T09:42:36.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:35 smithi163 bash[78107]: cluster 2023-10-09T09:42:35.771243+0000 mon.a (mon.0) 2096 : cluster 3 Health check update: Degraded data redundancy: 2348/299850 objects degraded (0.783%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:36.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:42:36.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:36.343 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:36.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:36.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:36.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:36.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:36.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:36.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:36.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:36.610 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:36.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:36.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:36.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:36.744 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:36.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:36.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:36.853 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:36.883 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:36.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:36.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:36.986 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:37.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:37.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:37.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:37.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:37.144 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:37.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:37.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:37.251 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:37.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:37.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:37.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:37.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:37.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:37.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:37.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:37.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:37.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:37.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:37.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:37.650 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:37.672 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:37.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:37.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:37.775 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:37.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:37.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:37.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:37.856 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:37 smithi163 bash[75425]: cluster 2023-10-09T09:42:36.006179+0000 mgr.y (mgr.34107) 1097 : cluster 0 pgmap v1898: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 91 active+clean; 6.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 60 MiB/s wr, 972 op/s; 2320/305682 objects degraded (0.759%); 576/305682 objects misplaced (0.188%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T09:42:37.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:37.951 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:37.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:37.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:37.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:37 smithi163 bash[78107]: cluster 2023-10-09T09:42:36.006179+0000 mgr.y (mgr.34107) 1097 : cluster 0 pgmap v1898: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 91 active+clean; 6.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 60 MiB/s wr, 972 op/s; 2320/305682 objects degraded (0.759%); 576/305682 objects misplaced (0.188%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T09:42:38.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:38.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:37 smithi181 bash[59345]: cluster 2023-10-09T09:42:36.006179+0000 mgr.y (mgr.34107) 1097 : cluster 0 pgmap v1898: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 91 active+clean; 6.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 60 MiB/s wr, 972 op/s; 2320/305682 objects degraded (0.759%); 576/305682 objects misplaced (0.188%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T09:42:38.078 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:38.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:38.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:38.182 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:38.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:38.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:38.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:38.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:38.346 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:38.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:38.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:38.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:38.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:38.485 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:38.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:38.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:38.608 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:38.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:38.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:38.710 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:38.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:38.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:38.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:38.839 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:38.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:38.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:38.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:38.987 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:39.021 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:39.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:39.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:39.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:39.165 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:39.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:39.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:39.270 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:39.306 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:39.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:39.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:39.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:39.438 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:39.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:39.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:39.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:39.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:39.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:39.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:39.679 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:39.720 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:39.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:39.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:39.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:39.850 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:39.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:39.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:39.858 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:39 smithi163 bash[75425]: cluster 2023-10-09T09:42:38.007054+0000 mgr.y (mgr.34107) 1098 : cluster 0 pgmap v1899: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 6.3 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 60 MiB/s wr, 1.05k op/s; 2169/308256 objects degraded (0.704%); 462/308256 objects misplaced (0.150%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T09:42:39.954 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:39.982 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:39.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:39.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:40.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:40.121 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:40.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:40.122 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:39 smithi163 bash[78107]: cluster 2023-10-09T09:42:38.007054+0000 mgr.y (mgr.34107) 1098 : cluster 0 pgmap v1899: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 6.3 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 60 MiB/s wr, 1.05k op/s; 2169/308256 objects degraded (0.704%); 462/308256 objects misplaced (0.150%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T09:42:40.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:40.224 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:40.233 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:39 smithi181 bash[59345]: cluster 2023-10-09T09:42:38.007054+0000 mgr.y (mgr.34107) 1098 : cluster 0 pgmap v1899: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 6.3 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 60 MiB/s wr, 1.05k op/s; 2169/308256 objects degraded (0.704%); 462/308256 objects misplaced (0.150%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T09:42:40.254 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:40.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:40.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:40.363 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:40.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:40.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:40.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:40.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:40.534 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:40.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:40.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:40.577 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:42:40.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:40.661 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:42:40.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:40.671 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:40.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:40.701 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:42:40.742 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:42:40.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:40.784 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:42:40.819 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:40.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:40.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:40.839 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:42:40.886 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:42:40.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:40.934 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:42:40.940 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:42:40.940 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:42:40.940 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:42:40.940 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:42:40.940 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:42:40.940 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:42:40.940 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:42:40.940 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:42:40.948 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:40.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:40 smithi163 bash[78107]: audit 2023-10-09T09:42:40.387859+0000 mon.a (mon.0) 2097 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:40.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:40 smithi163 bash[78107]: audit 2023-10-09T09:42:40.536791+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.163:0/1719727081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:42:40.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:40 smithi163 bash[78107]: cluster 2023-10-09T09:42:40.772355+0000 mon.a (mon.0) 2098 : cluster 3 Health check update: Degraded data redundancy: 2169/308256 objects degraded (0.704%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:40 smithi163 bash[75425]: audit 2023-10-09T09:42:40.387859+0000 mon.a (mon.0) 2097 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:40 smithi163 bash[75425]: audit 2023-10-09T09:42:40.536791+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.163:0/1719727081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:42:40.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:40 smithi163 bash[75425]: cluster 2023-10-09T09:42:40.772355+0000 mon.a (mon.0) 2098 : cluster 3 Health check update: Degraded data redundancy: 2169/308256 objects degraded (0.704%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:41.005 INFO:teuthology.orchestra.run.smithi163.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T09:42:41.053 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:41.078 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:41.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:41.183 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:41.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:41.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:41.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:41.310 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:41.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:40 smithi181 bash[59345]: audit 2023-10-09T09:42:40.387859+0000 mon.a (mon.0) 2097 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:41.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:40 smithi181 bash[59345]: audit 2023-10-09T09:42:40.536791+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.163:0/1719727081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:42:41.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:40 smithi181 bash[59345]: cluster 2023-10-09T09:42:40.772355+0000 mon.a (mon.0) 2098 : cluster 3 Health check update: Degraded data redundancy: 2169/308256 objects degraded (0.704%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:41.341 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:41.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:41.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:41.444 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:41.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:41.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:41.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:41.612 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:41.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:41.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:41.716 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:41.746 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:41.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:41.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:41.869 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:41 smithi163 bash[75425]: cluster 2023-10-09T09:42:40.007710+0000 mgr.y (mgr.34107) 1099 : cluster 0 pgmap v1900: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 91 active+clean; 6.2 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 85 B/s rd, 48 MiB/s wr, 969 op/s; 1792/304503 objects degraded (0.588%); 462/304503 objects misplaced (0.152%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T09:42:41.869 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:41 smithi163 bash[75425]: audit 2023-10-09T09:42:40.998107+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.163:0/3336609026' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T09:42:41.869 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:41 smithi163 bash[75425]: audit 2023-10-09T09:42:40.998992+0000 mon.a (mon.0) 2099 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T09:42:41.870 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:41 smithi163 bash[75425]: audit 2023-10-09T09:42:40.999367+0000 mon.a (mon.0) 2100 : audit 1 from='client.? 172.21.15.163:0/1292403298' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:42:41.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:41.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:41.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:41.895 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:42:41.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:42.005 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:42.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:42.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:42.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:41 smithi181 bash[59345]: cluster 2023-10-09T09:42:40.007710+0000 mgr.y (mgr.34107) 1099 : cluster 0 pgmap v1900: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 91 active+clean; 6.2 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 85 B/s rd, 48 MiB/s wr, 969 op/s; 1792/304503 objects degraded (0.588%); 462/304503 objects misplaced (0.152%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T09:42:42.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:41 smithi181 bash[59345]: audit 2023-10-09T09:42:40.998107+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.163:0/3336609026' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T09:42:42.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:41 smithi181 bash[59345]: audit 2023-10-09T09:42:40.998992+0000 mon.a (mon.0) 2099 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T09:42:42.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:41 smithi181 bash[59345]: audit 2023-10-09T09:42:40.999367+0000 mon.a (mon.0) 2100 : audit 1 from='client.? 172.21.15.163:0/1292403298' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:42:42.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:42.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:42.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:42.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:42.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:41 smithi163 bash[78107]: cluster 2023-10-09T09:42:40.007710+0000 mgr.y (mgr.34107) 1099 : cluster 0 pgmap v1900: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 91 active+clean; 6.2 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 85 B/s rd, 48 MiB/s wr, 969 op/s; 1792/304503 objects degraded (0.588%); 462/304503 objects misplaced (0.152%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T09:42:42.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:41 smithi163 bash[78107]: audit 2023-10-09T09:42:40.998107+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.163:0/3336609026' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T09:42:42.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:41 smithi163 bash[78107]: audit 2023-10-09T09:42:40.998992+0000 mon.a (mon.0) 2099 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T09:42:42.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:41 smithi163 bash[78107]: audit 2023-10-09T09:42:40.999367+0000 mon.a (mon.0) 2100 : audit 1 from='client.? 172.21.15.163:0/1292403298' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:42:42.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:42.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:42.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:42.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:42.383 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:42.412 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:42.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:42.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:42.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:42.543 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:42.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:42.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:42.646 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:42.673 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:42.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:42.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:42.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:42.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:42.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:42.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:42 smithi163 bash[75425]: cluster 2023-10-09T09:42:41.867243+0000 mon.a (mon.0) 2101 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:42:42.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:42 smithi163 bash[75425]: cluster 2023-10-09T09:42:41.880854+0000 osd.5 (osd.5) 786 : cluster 3 failed to encode map e1205 with expected crc 2023-10-09T09:42:42.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:42 smithi163 bash[75425]: cluster 2023-10-09T09:42:41.883215+0000 osd.6 (osd.6) 487 : cluster 3 failed to encode map e1205 with expected crc 2023-10-09T09:42:42.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:42 smithi163 bash[75425]: audit 2023-10-09T09:42:41.883583+0000 mon.a (mon.0) 2102 : audit 1 from='client.? 172.21.15.163:0/1292403298' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:42:42.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:42 smithi163 bash[75425]: cluster 2023-10-09T09:42:41.883730+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e1205: 8 total, 8 up, 6 in 2023-10-09T09:42:42.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:42.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:42.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:42.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:43.038 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:43.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:42 smithi181 bash[59345]: cluster 2023-10-09T09:42:41.867243+0000 mon.a (mon.0) 2101 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:42:43.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:42 smithi181 bash[59345]: cluster 2023-10-09T09:42:41.880854+0000 osd.5 (osd.5) 786 : cluster 3 failed to encode map e1205 with expected crc 2023-10-09T09:42:43.047 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:42 smithi181 bash[59345]: cluster 2023-10-09T09:42:41.883215+0000 osd.6 (osd.6) 487 : cluster 3 failed to encode map e1205 with expected crc 2023-10-09T09:42:43.047 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:42 smithi181 bash[59345]: audit 2023-10-09T09:42:41.883583+0000 mon.a (mon.0) 2102 : audit 1 from='client.? 172.21.15.163:0/1292403298' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:42:43.047 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:42 smithi181 bash[59345]: cluster 2023-10-09T09:42:41.883730+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e1205: 8 total, 8 up, 6 in 2023-10-09T09:42:43.067 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:43.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:43.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:43.174 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:43.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:42 smithi163 bash[78107]: cluster 2023-10-09T09:42:41.867243+0000 mon.a (mon.0) 2101 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:42:43.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:42 smithi163 bash[78107]: cluster 2023-10-09T09:42:41.880854+0000 osd.5 (osd.5) 786 : cluster 3 failed to encode map e1205 with expected crc 2023-10-09T09:42:43.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:42 smithi163 bash[78107]: cluster 2023-10-09T09:42:41.883215+0000 osd.6 (osd.6) 487 : cluster 3 failed to encode map e1205 with expected crc 2023-10-09T09:42:43.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:42 smithi163 bash[78107]: audit 2023-10-09T09:42:41.883583+0000 mon.a (mon.0) 2102 : audit 1 from='client.? 172.21.15.163:0/1292403298' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:42:43.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:42 smithi163 bash[78107]: cluster 2023-10-09T09:42:41.883730+0000 mon.a (mon.0) 2103 : cluster 0 osdmap e1205: 8 total, 8 up, 6 in 2023-10-09T09:42:43.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:43.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:43.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:43.305 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:43.334 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:43.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:43.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:43.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:43.449 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:42:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:42:43] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.33.4" 2023-10-09T09:42:43.473 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:43.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:43.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:43.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:43.606 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:43.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:43.709 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:43.732 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:43.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:43.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:43.834 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:43.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:43.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:43.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:43.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:43.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:43 smithi163 bash[78107]: cluster 2023-10-09T09:42:42.008252+0000 mgr.y (mgr.34107) 1100 : cluster 0 pgmap v1902: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 6.1 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 33 MiB/s wr, 948 op/s; 1632/298989 objects degraded (0.546%); 348/298989 objects misplaced (0.116%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:42:43.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:43 smithi163 bash[75425]: cluster 2023-10-09T09:42:42.008252+0000 mgr.y (mgr.34107) 1100 : cluster 0 pgmap v1902: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 6.1 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 33 MiB/s wr, 948 op/s; 1632/298989 objects degraded (0.546%); 348/298989 objects misplaced (0.116%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:42:43.998 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:44.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:44.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:44.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:44.111 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:43 smithi181 bash[59345]: cluster 2023-10-09T09:42:42.008252+0000 mgr.y (mgr.34107) 1100 : cluster 0 pgmap v1902: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 6.1 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 33 MiB/s wr, 948 op/s; 1632/298989 objects degraded (0.546%); 348/298989 objects misplaced (0.116%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T09:42:44.145 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:44.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:44.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:44.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:44.277 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:44.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:44.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:44.380 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:44.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:44.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:44.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:44.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:44.546 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:44.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:44.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:44.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:44.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:44.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:44.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:44.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:44.803 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:44.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:44.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:44.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:44.908 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T09:42:44.971 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:44.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:44.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:45.079 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:45.108 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:45.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:45.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:45.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:45.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:45.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:45.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:45.256 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:45.343 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:45.382 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:45.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:45.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:45.490 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:45.515 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:45.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:45.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:45.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:45.649 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:45.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:45.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:45.755 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:45.785 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:45.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:45.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:45.891 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:45.926 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:45.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:46.033 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:46.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:46.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:46.164 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:46.173 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:45 smithi163 bash[78107]: cluster 2023-10-09T09:42:44.009031+0000 mgr.y (mgr.34107) 1101 : cluster 0 pgmap v1903: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 6.0 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 25 MiB/s wr, 928 op/s; 1376/295788 objects degraded (0.465%); 348/295788 objects misplaced (0.118%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:42:46.173 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:45 smithi163 bash[78107]: cluster 2023-10-09T09:42:45.773659+0000 mon.a (mon.0) 2104 : cluster 3 Health check update: Degraded data redundancy: 1376/295788 objects degraded (0.465%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:46.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:45 smithi163 bash[75425]: cluster 2023-10-09T09:42:44.009031+0000 mgr.y (mgr.34107) 1101 : cluster 0 pgmap v1903: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 6.0 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 25 MiB/s wr, 928 op/s; 1376/295788 objects degraded (0.465%); 348/295788 objects misplaced (0.118%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:42:46.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:45 smithi163 bash[75425]: cluster 2023-10-09T09:42:45.773659+0000 mon.a (mon.0) 2104 : cluster 3 Health check update: Degraded data redundancy: 1376/295788 objects degraded (0.465%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:46.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:46.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:46.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:46.288 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:46.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:45 smithi181 bash[59345]: cluster 2023-10-09T09:42:44.009031+0000 mgr.y (mgr.34107) 1101 : cluster 0 pgmap v1903: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 6.0 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 25 MiB/s wr, 928 op/s; 1376/295788 objects degraded (0.465%); 348/295788 objects misplaced (0.118%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T09:42:46.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:45 smithi181 bash[59345]: cluster 2023-10-09T09:42:45.773659+0000 mon.a (mon.0) 2104 : cluster 3 Health check update: Degraded data redundancy: 1376/295788 objects degraded (0.465%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:46.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:46.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:46.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:46.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:46.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:46.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:46.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:46.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:46.579 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:46.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:46.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:46.683 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:46.713 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:46.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:46.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:46.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:46.847 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:46.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:46.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:46.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:42:46.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:42:46.951 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:47.025 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:47.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:47.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:46 smithi181 bash[59345]: cluster 2023-10-09T09:42:45.358781+0000 osd.6 (osd.6) 488 : cluster 0 9.1 scrub starts 2023-10-09T09:42:47.132 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:47.147 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:46 smithi163 bash[75425]: cluster 2023-10-09T09:42:45.358781+0000 osd.6 (osd.6) 488 : cluster 0 9.1 scrub starts 2023-10-09T09:42:47.147 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:46 smithi163 bash[78107]: cluster 2023-10-09T09:42:45.358781+0000 osd.6 (osd.6) 488 : cluster 0 9.1 scrub starts 2023-10-09T09:42:47.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:47.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:47.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:47.312 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:47.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:47.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:47.441 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:47.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:47.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:47.576 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:47.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:47.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:47.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:47.708 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:47.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:47.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:47.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:47.841 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:47.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:47.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:47.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:47.955 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:47 smithi163 bash[78107]: cluster 2023-10-09T09:42:46.009721+0000 mgr.y (mgr.34107) 1102 : cluster 0 pgmap v1904: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 91 active+clean; 5.9 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 12 MiB/s wr, 914 op/s; 781/289752 objects degraded (0.270%); 348/289752 objects misplaced (0.120%); 11 MiB/s, 176 objects/s recovering 2023-10-09T09:42:47.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:47 smithi163 bash[75425]: cluster 2023-10-09T09:42:46.009721+0000 mgr.y (mgr.34107) 1102 : cluster 0 pgmap v1904: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 91 active+clean; 5.9 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 12 MiB/s wr, 914 op/s; 781/289752 objects degraded (0.270%); 348/289752 objects misplaced (0.120%); 11 MiB/s, 176 objects/s recovering 2023-10-09T09:42:47.969 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:47.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:47.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:48.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:48.107 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:48.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:48.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:48.210 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:48.245 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:48.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:48.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:47 smithi181 bash[59345]: cluster 2023-10-09T09:42:46.009721+0000 mgr.y (mgr.34107) 1102 : cluster 0 pgmap v1904: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 91 active+clean; 5.9 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 12 MiB/s wr, 914 op/s; 781/289752 objects degraded (0.270%); 348/289752 objects misplaced (0.120%); 11 MiB/s, 176 objects/s recovering 2023-10-09T09:42:48.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:48.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:48.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:48.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:48.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:48.512 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:48.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:48.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:48.615 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:48.642 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:48.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:48.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:48.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:48.780 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:48.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:48.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:48.886 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:48.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:48.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:48.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:49.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:49.047 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:49.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:49.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:49.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:49.176 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:49.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:49.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:49.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:49.305 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:49.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:49.411 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:49.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:49.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:49.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:49.548 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:49.579 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:49.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:49.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:49.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:49.711 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:49.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:49.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:49.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:49.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:49.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:49.910 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 6, 2, 5] out_osds: [7, 3] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:42:49.910 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:42:49.910 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T09:42:49.910 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-10-09T09:42:49.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0 2023-10-09T09:42:49.911 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:49 smithi163 bash[75425]: cluster 2023-10-09T09:42:48.011057+0000 mgr.y (mgr.34107) 1103 : cluster 0 pgmap v1905: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 512 KiB/s wr, 907 op/s; 581/281514 objects degraded (0.206%); 102/281514 objects misplaced (0.036%); 12 MiB/s, 194 objects/s recovering 2023-10-09T09:42:49.956 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:49.980 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:49.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:49.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:50.083 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:50.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:49 smithi181 bash[59345]: cluster 2023-10-09T09:42:48.011057+0000 mgr.y (mgr.34107) 1103 : cluster 0 pgmap v1905: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 512 KiB/s wr, 907 op/s; 581/281514 objects degraded (0.206%); 102/281514 objects misplaced (0.036%); 12 MiB/s, 194 objects/s recovering 2023-10-09T09:42:50.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:50.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:50.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:49 smithi163 bash[78107]: cluster 2023-10-09T09:42:48.011057+0000 mgr.y (mgr.34107) 1103 : cluster 0 pgmap v1905: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 512 KiB/s wr, 907 op/s; 581/281514 objects degraded (0.206%); 102/281514 objects misplaced (0.036%); 12 MiB/s, 194 objects/s recovering 2023-10-09T09:42:50.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:50.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:50.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:50.363 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:50.399 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:50.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:50.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:50.530 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:50.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:50.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:50.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:50.669 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:50.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:50.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:50.776 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:50.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:50.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:50.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:50.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:50.941 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:50.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:50.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:50.985 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:42:50.985 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:42:51.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:50 smithi181 bash[59345]: cluster 2023-10-09T09:42:49.245066+0000 osd.6 (osd.6) 489 : cluster 0 9.1 scrub starts 2023-10-09T09:42:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:50 smithi181 bash[59345]: cluster 2023-10-09T09:42:50.777116+0000 mon.a (mon.0) 2105 : cluster 3 Health check update: Degraded data redundancy: 581/281514 objects degraded (0.206%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:51.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:51.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:51.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:51.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:51.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:50 smithi163 bash[78107]: cluster 2023-10-09T09:42:49.245066+0000 osd.6 (osd.6) 489 : cluster 0 9.1 scrub starts 2023-10-09T09:42:51.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:50 smithi163 bash[78107]: cluster 2023-10-09T09:42:50.777116+0000 mon.a (mon.0) 2105 : cluster 3 Health check update: Degraded data redundancy: 581/281514 objects degraded (0.206%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:51.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:50 smithi163 bash[75425]: cluster 2023-10-09T09:42:49.245066+0000 osd.6 (osd.6) 489 : cluster 0 9.1 scrub starts 2023-10-09T09:42:51.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:50 smithi163 bash[75425]: cluster 2023-10-09T09:42:50.777116+0000 mon.a (mon.0) 2105 : cluster 3 Health check update: Degraded data redundancy: 581/281514 objects degraded (0.206%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:51.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:51.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:51.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:51.306 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:51.332 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:51.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:51.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:51.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:51.458 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:51.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:51.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:51.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:51.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:51.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:51.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:51.696 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:51.732 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:51.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:51.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:51.836 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:51.868 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:51.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:51.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:51.984 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:51 smithi181 bash[59345]: cluster 2023-10-09T09:42:50.011636+0000 mgr.y (mgr.34107) 1104 : cluster 0 pgmap v1906: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait, 3 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 915 op/s; 247/277017 objects degraded (0.089%); 102/277017 objects misplaced (0.037%); 12 MiB/s, 190 objects/s recovering 2023-10-09T09:42:51.984 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:51 smithi181 bash[59345]: cluster 2023-10-09T09:42:50.222392+0000 osd.6 (osd.6) 490 : cluster 0 9.1 scrub starts 2023-10-09T09:42:52.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:52.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:52.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:52.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:52.130 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:52.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:52.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:52.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:51 smithi163 bash[78107]: cluster 2023-10-09T09:42:50.011636+0000 mgr.y (mgr.34107) 1104 : cluster 0 pgmap v1906: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait, 3 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 915 op/s; 247/277017 objects degraded (0.089%); 102/277017 objects misplaced (0.037%); 12 MiB/s, 190 objects/s recovering 2023-10-09T09:42:52.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:51 smithi163 bash[78107]: cluster 2023-10-09T09:42:50.222392+0000 osd.6 (osd.6) 490 : cluster 0 9.1 scrub starts 2023-10-09T09:42:52.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:51 smithi163 bash[75425]: cluster 2023-10-09T09:42:50.011636+0000 mgr.y (mgr.34107) 1104 : cluster 0 pgmap v1906: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait, 3 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 915 op/s; 247/277017 objects degraded (0.089%); 102/277017 objects misplaced (0.037%); 12 MiB/s, 190 objects/s recovering 2023-10-09T09:42:52.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:51 smithi163 bash[75425]: cluster 2023-10-09T09:42:50.222392+0000 osd.6 (osd.6) 490 : cluster 0 9.1 scrub starts 2023-10-09T09:42:52.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:52.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:52.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:52.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:52.368 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:52.404 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:52.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:52.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:52.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T09:42:52.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:52.581 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:52.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:52.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:52.684 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:52.708 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:52.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:52.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:52.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:52.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:52.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:52.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:52.958 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:52 smithi181 bash[59345]: cluster 2023-10-09T09:42:51.213389+0000 osd.6 (osd.6) 491 : cluster 0 9.1 scrub starts 2023-10-09T09:42:52.987 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:52.987 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:52.988 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:52.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:53.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:53.125 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:53.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:53.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:53.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:52 smithi163 bash[78107]: cluster 2023-10-09T09:42:51.213389+0000 osd.6 (osd.6) 491 : cluster 0 9.1 scrub starts 2023-10-09T09:42:53.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:52 smithi163 bash[75425]: cluster 2023-10-09T09:42:51.213389+0000 osd.6 (osd.6) 491 : cluster 0 9.1 scrub starts 2023-10-09T09:42:53.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:53.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:53.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:53.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:53.360 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:42:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:42:53] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.33.4" 2023-10-09T09:42:53.365 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:53.397 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:53.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:53.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:53.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:53.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:53.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:53.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:53.628 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:53.661 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:53.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:53.767 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:53.797 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:53.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:53.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:53.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:53.937 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:53.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:53.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:54.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:54.069 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:54.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:54.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:54.174 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:54.207 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:54.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:53 smithi181 bash[59345]: cluster 2023-10-09T09:42:52.012388+0000 mgr.y (mgr.34107) 1105 : cluster 0 pgmap v1907: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait, 2 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 930 op/s; 168/270711 objects degraded (0.062%); 11 MiB/s, 178 objects/s recovering 2023-10-09T09:42:54.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:54.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:54.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:53 smithi163 bash[75425]: cluster 2023-10-09T09:42:52.012388+0000 mgr.y (mgr.34107) 1105 : cluster 0 pgmap v1907: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait, 2 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 930 op/s; 168/270711 objects degraded (0.062%); 11 MiB/s, 178 objects/s recovering 2023-10-09T09:42:54.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:53 smithi163 bash[78107]: cluster 2023-10-09T09:42:52.012388+0000 mgr.y (mgr.34107) 1105 : cluster 0 pgmap v1907: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait, 2 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 930 op/s; 168/270711 objects degraded (0.062%); 11 MiB/s, 178 objects/s recovering 2023-10-09T09:42:54.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:54.333 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:54.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:54.436 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:54.463 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:54.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:54.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:54.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:54.591 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:54.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:54.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:54.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:54.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:54.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:54.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:54.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:54.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:54.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:54.963 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:54.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:54.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:54.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:55.026 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:55.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:55.115 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:55.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:55.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:55.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:55.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:55.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:55.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:55.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:55.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:55.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:55.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:55.474 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:55.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:55.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:55.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:55.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:55.627 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:55.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:55.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:55.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:55.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:55.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:55.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:55.862 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:55.894 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:55.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:55.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:55.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:55 smithi163 bash[75425]: cluster 2023-10-09T09:42:54.013000+0000 mgr.y (mgr.34107) 1106 : cluster 0 pgmap v1908: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait, 91 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 884 op/s; 115/267132 objects degraded (0.043%); 9.4 MiB/s, 155 objects/s recovering 2023-10-09T09:42:55.953 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:55 smithi163 bash[75425]: audit 2023-10-09T09:42:55.387906+0000 mon.a (mon.0) 2106 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:55.953 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:55 smithi163 bash[75425]: cluster 2023-10-09T09:42:55.779180+0000 mon.a (mon.0) 2107 : cluster 3 Health check update: Degraded data redundancy: 115/267132 objects degraded (0.043%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:56.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:56.030 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:56.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:56.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:56.133 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:56.142 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:55 smithi181 bash[59345]: cluster 2023-10-09T09:42:54.013000+0000 mgr.y (mgr.34107) 1106 : cluster 0 pgmap v1908: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait, 91 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 884 op/s; 115/267132 objects degraded (0.043%); 9.4 MiB/s, 155 objects/s recovering 2023-10-09T09:42:56.142 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:55 smithi181 bash[59345]: audit 2023-10-09T09:42:55.387906+0000 mon.a (mon.0) 2106 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:56.143 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:55 smithi181 bash[59345]: cluster 2023-10-09T09:42:55.779180+0000 mon.a (mon.0) 2107 : cluster 3 Health check update: Degraded data redundancy: 115/267132 objects degraded (0.043%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:56.166 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:56.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:56.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:56.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:55 smithi163 bash[78107]: cluster 2023-10-09T09:42:54.013000+0000 mgr.y (mgr.34107) 1106 : cluster 0 pgmap v1908: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait, 91 active+clean; 5.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 884 op/s; 115/267132 objects degraded (0.043%); 9.4 MiB/s, 155 objects/s recovering 2023-10-09T09:42:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:55 smithi163 bash[78107]: audit 2023-10-09T09:42:55.387906+0000 mon.a (mon.0) 2106 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:42:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:55 smithi163 bash[78107]: cluster 2023-10-09T09:42:55.779180+0000 mon.a (mon.0) 2107 : cluster 3 Health check update: Degraded data redundancy: 115/267132 objects degraded (0.043%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:42:56.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:56.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:56.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:56.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:56.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:56.428 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:56.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:56.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:56.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:56.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:56.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:56.705 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:42:56.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:56.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:42:56.811 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:56.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:56.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:56.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:56.946 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:56.969 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:56.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:57.045 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:42:57.075 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:57.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:57.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:57.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:57.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:57.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:57.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:57.334 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:57.352 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:57.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:57.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:57.455 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:57.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:42:57.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:57.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:42:57.580 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:57.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:57.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:57.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:57.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:57.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:57.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:57.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:57.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:57.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:57.848 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:57.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:57.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:57.980 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:57.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:58.084 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:58.106 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:58.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:58.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:58.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[78107]: cluster 2023-10-09T09:42:56.013580+0000 mgr.y (mgr.34107) 1107 : cluster 0 pgmap v1909: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait, 91 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 982 op/s; 115/260388 objects degraded (0.044%); 8.0 MiB/s, 134 objects/s recovering 2023-10-09T09:42:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[78107]: cluster 2023-10-09T09:42:56.189801+0000 osd.6 (osd.6) 492 : cluster 0 9.1 scrub starts 2023-10-09T09:42:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[78107]: cluster 2023-10-09T09:42:56.953370+0000 osd.5 (osd.5) 787 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[78107]: cluster 2023-10-09T09:42:56.956383+0000 mon.a (mon.0) 2108 : cluster 0 osdmap e1206: 8 total, 8 up, 6 in 2023-10-09T09:42:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[78107]: cluster 2023-10-09T09:42:56.956499+0000 osd.5 (osd.5) 788 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[78107]: cluster 2023-10-09T09:42:56.964879+0000 osd.4 (osd.4) 622 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[78107]: cluster 2023-10-09T09:42:56.966573+0000 osd.6 (osd.6) 493 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[78107]: cluster 2023-10-09T09:42:56.969698+0000 osd.6 (osd.6) 494 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[78107]: cluster 2023-10-09T09:42:57.312505+0000 osd.7 (osd.7) 580 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[75425]: cluster 2023-10-09T09:42:56.013580+0000 mgr.y (mgr.34107) 1107 : cluster 0 pgmap v1909: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait, 91 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 982 op/s; 115/260388 objects degraded (0.044%); 8.0 MiB/s, 134 objects/s recovering 2023-10-09T09:42:58.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[75425]: cluster 2023-10-09T09:42:56.189801+0000 osd.6 (osd.6) 492 : cluster 0 9.1 scrub starts 2023-10-09T09:42:58.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[75425]: cluster 2023-10-09T09:42:56.953370+0000 osd.5 (osd.5) 787 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[75425]: cluster 2023-10-09T09:42:56.956383+0000 mon.a (mon.0) 2108 : cluster 0 osdmap e1206: 8 total, 8 up, 6 in 2023-10-09T09:42:58.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[75425]: cluster 2023-10-09T09:42:56.956499+0000 osd.5 (osd.5) 788 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[75425]: cluster 2023-10-09T09:42:56.964879+0000 osd.4 (osd.4) 622 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[75425]: cluster 2023-10-09T09:42:56.966573+0000 osd.6 (osd.6) 493 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[75425]: cluster 2023-10-09T09:42:56.969698+0000 osd.6 (osd.6) 494 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:57 smithi163 bash[75425]: cluster 2023-10-09T09:42:57.312505+0000 osd.7 (osd.7) 580 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.237 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:58.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:57 smithi181 bash[59345]: cluster 2023-10-09T09:42:56.013580+0000 mgr.y (mgr.34107) 1107 : cluster 0 pgmap v1909: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait, 91 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 982 op/s; 115/260388 objects degraded (0.044%); 8.0 MiB/s, 134 objects/s recovering 2023-10-09T09:42:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:57 smithi181 bash[59345]: cluster 2023-10-09T09:42:56.189801+0000 osd.6 (osd.6) 492 : cluster 0 9.1 scrub starts 2023-10-09T09:42:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:57 smithi181 bash[59345]: cluster 2023-10-09T09:42:56.953370+0000 osd.5 (osd.5) 787 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:57 smithi181 bash[59345]: cluster 2023-10-09T09:42:56.956383+0000 mon.a (mon.0) 2108 : cluster 0 osdmap e1206: 8 total, 8 up, 6 in 2023-10-09T09:42:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:57 smithi181 bash[59345]: cluster 2023-10-09T09:42:56.956499+0000 osd.5 (osd.5) 788 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:57 smithi181 bash[59345]: cluster 2023-10-09T09:42:56.964879+0000 osd.4 (osd.4) 622 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:57 smithi181 bash[59345]: cluster 2023-10-09T09:42:56.966573+0000 osd.6 (osd.6) 493 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:57 smithi181 bash[59345]: cluster 2023-10-09T09:42:56.969698+0000 osd.6 (osd.6) 494 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:57 smithi181 bash[59345]: cluster 2023-10-09T09:42:57.312505+0000 osd.7 (osd.7) 580 : cluster 3 failed to encode map e1206 with expected crc 2023-10-09T09:42:58.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:58.381 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:58.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:58.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:58.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:58.507 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:42:58.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:58.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:42:58.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:58.638 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:58.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:58.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:58.745 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:58.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:58.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:58.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:58.884 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:58.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:58.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:58.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:59.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:59.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:59.041 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:59.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:42:59.141 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:59.161 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:42:59.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:59.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:42:59.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[78107]: cluster 2023-10-09T09:42:57.030486+0000 osd.0 (osd.0) 2 : cluster 0 9.9 scrub starts 2023-10-09T09:42:59.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[78107]: cluster 2023-10-09T09:42:57.597081+0000 osd.0 (osd.0) 3 : cluster 0 9.9 scrub ok 2023-10-09T09:42:59.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[78107]: cluster 2023-10-09T09:42:57.952828+0000 osd.5 (osd.5) 789 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[78107]: cluster 2023-10-09T09:42:57.953207+0000 osd.6 (osd.6) 495 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[78107]: cluster 2023-10-09T09:42:57.954888+0000 mon.a (mon.0) 2109 : cluster 0 osdmap e1207: 8 total, 8 up, 6 in 2023-10-09T09:42:59.228 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[78107]: cluster 2023-10-09T09:42:57.955069+0000 osd.5 (osd.5) 790 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.228 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[78107]: cluster 2023-10-09T09:42:57.985915+0000 osd.4 (osd.4) 623 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.228 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[78107]: audit 2023-10-09T09:42:58.014977+0000 mon.a (mon.0) 2110 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T09:42:59.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[75425]: cluster 2023-10-09T09:42:57.030486+0000 osd.0 (osd.0) 2 : cluster 0 9.9 scrub starts 2023-10-09T09:42:59.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[75425]: cluster 2023-10-09T09:42:57.597081+0000 osd.0 (osd.0) 3 : cluster 0 9.9 scrub ok 2023-10-09T09:42:59.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[75425]: cluster 2023-10-09T09:42:57.952828+0000 osd.5 (osd.5) 789 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[75425]: cluster 2023-10-09T09:42:57.953207+0000 osd.6 (osd.6) 495 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[75425]: cluster 2023-10-09T09:42:57.954888+0000 mon.a (mon.0) 2109 : cluster 0 osdmap e1207: 8 total, 8 up, 6 in 2023-10-09T09:42:59.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[75425]: cluster 2023-10-09T09:42:57.955069+0000 osd.5 (osd.5) 790 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[75425]: cluster 2023-10-09T09:42:57.985915+0000 osd.4 (osd.4) 623 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:58 smithi163 bash[75425]: audit 2023-10-09T09:42:58.014977+0000 mon.a (mon.0) 2110 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T09:42:59.265 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:59.296 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:59.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:59.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:58 smithi181 bash[59345]: cluster 2023-10-09T09:42:57.030486+0000 osd.0 (osd.0) 2 : cluster 0 9.9 scrub starts 2023-10-09T09:42:59.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:58 smithi181 bash[59345]: cluster 2023-10-09T09:42:57.597081+0000 osd.0 (osd.0) 3 : cluster 0 9.9 scrub ok 2023-10-09T09:42:59.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:58 smithi181 bash[59345]: cluster 2023-10-09T09:42:57.952828+0000 osd.5 (osd.5) 789 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:58 smithi181 bash[59345]: cluster 2023-10-09T09:42:57.953207+0000 osd.6 (osd.6) 495 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:58 smithi181 bash[59345]: cluster 2023-10-09T09:42:57.954888+0000 mon.a (mon.0) 2109 : cluster 0 osdmap e1207: 8 total, 8 up, 6 in 2023-10-09T09:42:59.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:58 smithi181 bash[59345]: cluster 2023-10-09T09:42:57.955069+0000 osd.5 (osd.5) 790 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:58 smithi181 bash[59345]: cluster 2023-10-09T09:42:57.985915+0000 osd.4 (osd.4) 623 : cluster 3 failed to encode map e1207 with expected crc 2023-10-09T09:42:59.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:58 smithi181 bash[59345]: audit 2023-10-09T09:42:58.014977+0000 mon.a (mon.0) 2110 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T09:42:59.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:59.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:59.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:59.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:59.535 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:59.557 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:42:59.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:59.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:42:59.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:59.688 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:42:59.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:59.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:42:59.793 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:59.823 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:42:59.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:59.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:42:59.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:59.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:42:59.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:42:59.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:00.057 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:00.083 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:00.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:00.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:00.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:00.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:59 smithi181 bash[59345]: cluster 2023-10-09T09:42:58.013945+0000 mgr.y (mgr.34107) 1108 : cluster 0 pgmap v1912: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 5 active+recovery_wait, 92 active+clean; 5.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.05k op/s; 1.3 MiB/s, 43 objects/s recovering 2023-10-09T09:43:00.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:59 smithi181 bash[59345]: cluster 2023-10-09T09:42:58.976481+0000 osd.5 (osd.5) 791 : cluster 3 failed to encode map e1208 with expected crc 2023-10-09T09:43:00.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:59 smithi181 bash[59345]: cluster 2023-10-09T09:42:58.980328+0000 osd.6 (osd.6) 496 : cluster 3 failed to encode map e1208 with expected crc 2023-10-09T09:43:00.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:59 smithi181 bash[59345]: audit 2023-10-09T09:42:58.982583+0000 mon.a (mon.0) 2111 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T09:43:00.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:42:59 smithi181 bash[59345]: cluster 2023-10-09T09:42:58.982866+0000 mon.a (mon.0) 2112 : cluster 0 osdmap e1208: 8 total, 8 up, 6 in 2023-10-09T09:43:00.218 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:00.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:00.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:00.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:00.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:00.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:00.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:42:58.013945+0000 mgr.y (mgr.34107) 1108 : cluster 0 pgmap v1912: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 5 active+recovery_wait, 92 active+clean; 5.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.05k op/s; 1.3 MiB/s, 43 objects/s recovering 2023-10-09T09:43:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:42:58.976481+0000 osd.5 (osd.5) 791 : cluster 3 failed to encode map e1208 with expected crc 2023-10-09T09:43:00.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:42:58.980328+0000 osd.6 (osd.6) 496 : cluster 3 failed to encode map e1208 with expected crc 2023-10-09T09:43:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: audit 2023-10-09T09:42:58.982583+0000 mon.a (mon.0) 2111 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T09:43:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:42:58.982866+0000 mon.a (mon.0) 2112 : cluster 0 osdmap e1208: 8 total, 8 up, 6 in 2023-10-09T09:43:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:59 smithi163 bash[75425]: cluster 2023-10-09T09:42:58.013945+0000 mgr.y (mgr.34107) 1108 : cluster 0 pgmap v1912: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 5 active+recovery_wait, 92 active+clean; 5.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.05k op/s; 1.3 MiB/s, 43 objects/s recovering 2023-10-09T09:43:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:59 smithi163 bash[75425]: cluster 2023-10-09T09:42:58.976481+0000 osd.5 (osd.5) 791 : cluster 3 failed to encode map e1208 with expected crc 2023-10-09T09:43:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:42:59 smithi163 bash[75425]: cluster 2023-10-09T09:42:58.980328+0000 osd.6 (osd.6) 496 : cluster 3 failed to encode map e1208 with expected crc 2023-10-09T09:43:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[75425]: audit 2023-10-09T09:42:58.982583+0000 mon.a (mon.0) 2111 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T09:43:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[75425]: cluster 2023-10-09T09:42:58.982866+0000 mon.a (mon.0) 2112 : cluster 0 osdmap e1208: 8 total, 8 up, 6 in 2023-10-09T09:43:00.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:00.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:00.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:00.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:00.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:00.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:00.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:00.740 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:00.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:00.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:00.843 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:00.863 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:00.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:00.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:00.966 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:00.987 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:00.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:00.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:01.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:01.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:01.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:01.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:01.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:42:59.891883+0000 osd.2 (osd.2) 6 : cluster 0 9.6 scrub starts 2023-10-09T09:43:01.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:42:59.978123+0000 osd.5 (osd.5) 792 : cluster 3 failed to encode map e1209 with expected crc 2023-10-09T09:43:01.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:42:59.979901+0000 mon.a (mon.0) 2113 : cluster 0 osdmap e1209: 8 total, 8 up, 6 in 2023-10-09T09:43:01.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:42:59.981795+0000 osd.4 (osd.4) 624 : cluster 3 failed to encode map e1209 with expected crc 2023-10-09T09:43:01.126 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:43:00.014336+0000 mgr.y (mgr.34107) 1109 : cluster 0 pgmap v1915: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 2 active+recovery_wait, 95 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1k op/s; 0 B/s, 28 objects/s recovering 2023-10-09T09:43:01.126 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: audit 2023-10-09T09:43:00.015071+0000 mon.a (mon.0) 2114 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T09:43:01.126 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:43:00.247866+0000 osd.2 (osd.2) 7 : cluster 0 9.6 scrub ok 2023-10-09T09:43:01.126 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[78107]: cluster 2023-10-09T09:43:00.780292+0000 mon.a (mon.0) 2115 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:43:01.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[75425]: cluster 2023-10-09T09:42:59.891883+0000 osd.2 (osd.2) 6 : cluster 0 9.6 scrub starts 2023-10-09T09:43:01.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[75425]: cluster 2023-10-09T09:42:59.978123+0000 osd.5 (osd.5) 792 : cluster 3 failed to encode map e1209 with expected crc 2023-10-09T09:43:01.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[75425]: cluster 2023-10-09T09:42:59.979901+0000 mon.a (mon.0) 2113 : cluster 0 osdmap e1209: 8 total, 8 up, 6 in 2023-10-09T09:43:01.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[75425]: cluster 2023-10-09T09:42:59.981795+0000 osd.4 (osd.4) 624 : cluster 3 failed to encode map e1209 with expected crc 2023-10-09T09:43:01.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[75425]: cluster 2023-10-09T09:43:00.014336+0000 mgr.y (mgr.34107) 1109 : cluster 0 pgmap v1915: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 2 active+recovery_wait, 95 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1k op/s; 0 B/s, 28 objects/s recovering 2023-10-09T09:43:01.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[75425]: audit 2023-10-09T09:43:00.015071+0000 mon.a (mon.0) 2114 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T09:43:01.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[75425]: cluster 2023-10-09T09:43:00.247866+0000 osd.2 (osd.2) 7 : cluster 0 9.6 scrub ok 2023-10-09T09:43:01.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:00 smithi163 bash[75425]: cluster 2023-10-09T09:43:00.780292+0000 mon.a (mon.0) 2115 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:43:01.212 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:01.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:01.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:00 smithi181 bash[59345]: cluster 2023-10-09T09:42:59.891883+0000 osd.2 (osd.2) 6 : cluster 0 9.6 scrub starts 2023-10-09T09:43:01.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:00 smithi181 bash[59345]: cluster 2023-10-09T09:42:59.978123+0000 osd.5 (osd.5) 792 : cluster 3 failed to encode map e1209 with expected crc 2023-10-09T09:43:01.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:00 smithi181 bash[59345]: cluster 2023-10-09T09:42:59.979901+0000 mon.a (mon.0) 2113 : cluster 0 osdmap e1209: 8 total, 8 up, 6 in 2023-10-09T09:43:01.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:00 smithi181 bash[59345]: cluster 2023-10-09T09:42:59.981795+0000 osd.4 (osd.4) 624 : cluster 3 failed to encode map e1209 with expected crc 2023-10-09T09:43:01.246 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:00 smithi181 bash[59345]: cluster 2023-10-09T09:43:00.014336+0000 mgr.y (mgr.34107) 1109 : cluster 0 pgmap v1915: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 2 active+recovery_wait, 95 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1k op/s; 0 B/s, 28 objects/s recovering 2023-10-09T09:43:01.246 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:00 smithi181 bash[59345]: audit 2023-10-09T09:43:00.015071+0000 mon.a (mon.0) 2114 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T09:43:01.246 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:00 smithi181 bash[59345]: cluster 2023-10-09T09:43:00.247866+0000 osd.2 (osd.2) 7 : cluster 0 9.6 scrub ok 2023-10-09T09:43:01.246 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:00 smithi181 bash[59345]: cluster 2023-10-09T09:43:00.780292+0000 mon.a (mon.0) 2115 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:43:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:01.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:01.348 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:01.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:01.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:01.483 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:01.518 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:01.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:01.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:01.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:01.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:01.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:01.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:01.758 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:01.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:01.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:01.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:01.900 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:01.933 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:01.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:01.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:02.036 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:02.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:02 smithi181 bash[59345]: cluster 2023-10-09T09:43:00.038041+0000 osd.6 (osd.6) 497 : cluster 3 failed to encode map e1209 with expected crc 2023-10-09T09:43:02.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:02 smithi181 bash[59345]: cluster 2023-10-09T09:43:00.253751+0000 osd.6 (osd.6) 498 : cluster 0 9.1 scrub starts 2023-10-09T09:43:02.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:02 smithi181 bash[59345]: cluster 2023-10-09T09:43:00.675468+0000 osd.6 (osd.6) 499 : cluster 0 9.1 scrub ok 2023-10-09T09:43:02.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:02 smithi181 bash[59345]: cluster 2023-10-09T09:43:00.991740+0000 osd.5 (osd.5) 793 : cluster 3 failed to encode map e1210 with expected crc 2023-10-09T09:43:02.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:02 smithi181 bash[59345]: cluster 2023-10-09T09:43:00.992165+0000 osd.4 (osd.4) 625 : cluster 3 failed to encode map e1210 with expected crc 2023-10-09T09:43:02.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:02 smithi181 bash[59345]: audit 2023-10-09T09:43:00.996284+0000 mon.a (mon.0) 2116 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T09:43:02.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:02 smithi181 bash[59345]: cluster 2023-10-09T09:43:00.996448+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e1210: 8 total, 8 up, 6 in 2023-10-09T09:43:02.046 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:02 smithi181 bash[59345]: cluster 2023-10-09T09:43:01.096483+0000 osd.1 (osd.1) 9 : cluster 0 9.0 scrub starts 2023-10-09T09:43:02.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:02.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:02.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:02.164 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:02.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:02.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:02.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:02.288 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:02.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[78107]: cluster 2023-10-09T09:43:00.038041+0000 osd.6 (osd.6) 497 : cluster 3 failed to encode map e1209 with expected crc 2023-10-09T09:43:02.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[78107]: cluster 2023-10-09T09:43:00.253751+0000 osd.6 (osd.6) 498 : cluster 0 9.1 scrub starts 2023-10-09T09:43:02.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[78107]: cluster 2023-10-09T09:43:00.675468+0000 osd.6 (osd.6) 499 : cluster 0 9.1 scrub ok 2023-10-09T09:43:02.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[78107]: cluster 2023-10-09T09:43:00.991740+0000 osd.5 (osd.5) 793 : cluster 3 failed to encode map e1210 with expected crc 2023-10-09T09:43:02.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[78107]: cluster 2023-10-09T09:43:00.992165+0000 osd.4 (osd.4) 625 : cluster 3 failed to encode map e1210 with expected crc 2023-10-09T09:43:02.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[78107]: audit 2023-10-09T09:43:00.996284+0000 mon.a (mon.0) 2116 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T09:43:02.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[78107]: cluster 2023-10-09T09:43:00.996448+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e1210: 8 total, 8 up, 6 in 2023-10-09T09:43:02.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[78107]: cluster 2023-10-09T09:43:01.096483+0000 osd.1 (osd.1) 9 : cluster 0 9.0 scrub starts 2023-10-09T09:43:02.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[75425]: cluster 2023-10-09T09:43:00.038041+0000 osd.6 (osd.6) 497 : cluster 3 failed to encode map e1209 with expected crc 2023-10-09T09:43:02.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[75425]: cluster 2023-10-09T09:43:00.253751+0000 osd.6 (osd.6) 498 : cluster 0 9.1 scrub starts 2023-10-09T09:43:02.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[75425]: cluster 2023-10-09T09:43:00.675468+0000 osd.6 (osd.6) 499 : cluster 0 9.1 scrub ok 2023-10-09T09:43:02.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[75425]: cluster 2023-10-09T09:43:00.991740+0000 osd.5 (osd.5) 793 : cluster 3 failed to encode map e1210 with expected crc 2023-10-09T09:43:02.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[75425]: cluster 2023-10-09T09:43:00.992165+0000 osd.4 (osd.4) 625 : cluster 3 failed to encode map e1210 with expected crc 2023-10-09T09:43:02.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[75425]: audit 2023-10-09T09:43:00.996284+0000 mon.a (mon.0) 2116 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T09:43:02.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[75425]: cluster 2023-10-09T09:43:00.996448+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e1210: 8 total, 8 up, 6 in 2023-10-09T09:43:02.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:02 smithi163 bash[75425]: cluster 2023-10-09T09:43:01.096483+0000 osd.1 (osd.1) 9 : cluster 0 9.0 scrub starts 2023-10-09T09:43:02.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:02.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:02.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:02.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:02.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:02.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:02.547 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:02.580 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:02.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:02.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:02.684 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:02.713 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:02.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:02.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:02.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:02.836 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:02.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:02.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:02.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:02.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:02.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:02.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:03.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:03.075 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[78107]: cluster 2023-10-09T09:43:01.440910+0000 osd.1 (osd.1) 10 : cluster 0 9.0 scrub ok 2023-10-09T09:43:03.075 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[78107]: cluster 2023-10-09T09:43:01.995425+0000 osd.5 (osd.5) 794 : cluster 3 failed to encode map e1211 with expected crc 2023-10-09T09:43:03.075 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[78107]: cluster 2023-10-09T09:43:01.995466+0000 mon.a (mon.0) 2118 : cluster 0 osdmap e1211: 8 total, 8 up, 6 in 2023-10-09T09:43:03.075 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[78107]: cluster 2023-10-09T09:43:02.000114+0000 osd.6 (osd.6) 500 : cluster 3 failed to encode map e1211 with expected crc 2023-10-09T09:43:03.075 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[78107]: cluster 2023-10-09T09:43:02.014803+0000 mgr.y (mgr.34107) 1110 : cluster 0 pgmap v1918: 99 pgs: 1 peering, 1 active+clean+scrubbing, 1 active+recovery_wait, 96 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail 2023-10-09T09:43:03.076 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[78107]: cluster 2023-10-09T09:43:02.060456+0000 osd.7 (osd.7) 581 : cluster 3 failed to encode map e1211 with expected crc 2023-10-09T09:43:03.076 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[78107]: audit 2023-10-09T09:43:02.889316+0000 mon.a (mon.0) 2119 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:43:03.076 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[75425]: cluster 2023-10-09T09:43:01.440910+0000 osd.1 (osd.1) 10 : cluster 0 9.0 scrub ok 2023-10-09T09:43:03.076 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[75425]: cluster 2023-10-09T09:43:01.995425+0000 osd.5 (osd.5) 794 : cluster 3 failed to encode map e1211 with expected crc 2023-10-09T09:43:03.076 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[75425]: cluster 2023-10-09T09:43:01.995466+0000 mon.a (mon.0) 2118 : cluster 0 osdmap e1211: 8 total, 8 up, 6 in 2023-10-09T09:43:03.076 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[75425]: cluster 2023-10-09T09:43:02.000114+0000 osd.6 (osd.6) 500 : cluster 3 failed to encode map e1211 with expected crc 2023-10-09T09:43:03.076 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[75425]: cluster 2023-10-09T09:43:02.014803+0000 mgr.y (mgr.34107) 1110 : cluster 0 pgmap v1918: 99 pgs: 1 peering, 1 active+clean+scrubbing, 1 active+recovery_wait, 96 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail 2023-10-09T09:43:03.076 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[75425]: cluster 2023-10-09T09:43:02.060456+0000 osd.7 (osd.7) 581 : cluster 3 failed to encode map e1211 with expected crc 2023-10-09T09:43:03.076 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[75425]: audit 2023-10-09T09:43:02.889316+0000 mon.a (mon.0) 2119 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:43:03.092 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:03.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:03.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:03.195 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:03.218 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:03.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:03.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:03.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:03 smithi181 bash[59345]: cluster 2023-10-09T09:43:01.440910+0000 osd.1 (osd.1) 10 : cluster 0 9.0 scrub ok 2023-10-09T09:43:03.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:03 smithi181 bash[59345]: cluster 2023-10-09T09:43:01.995425+0000 osd.5 (osd.5) 794 : cluster 3 failed to encode map e1211 with expected crc 2023-10-09T09:43:03.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:03 smithi181 bash[59345]: cluster 2023-10-09T09:43:01.995466+0000 mon.a (mon.0) 2118 : cluster 0 osdmap e1211: 8 total, 8 up, 6 in 2023-10-09T09:43:03.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:03 smithi181 bash[59345]: cluster 2023-10-09T09:43:02.000114+0000 osd.6 (osd.6) 500 : cluster 3 failed to encode map e1211 with expected crc 2023-10-09T09:43:03.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:03 smithi181 bash[59345]: cluster 2023-10-09T09:43:02.014803+0000 mgr.y (mgr.34107) 1110 : cluster 0 pgmap v1918: 99 pgs: 1 peering, 1 active+clean+scrubbing, 1 active+recovery_wait, 96 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail 2023-10-09T09:43:03.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:03 smithi181 bash[59345]: cluster 2023-10-09T09:43:02.060456+0000 osd.7 (osd.7) 581 : cluster 3 failed to encode map e1211 with expected crc 2023-10-09T09:43:03.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:03 smithi181 bash[59345]: audit 2023-10-09T09:43:02.889316+0000 mon.a (mon.0) 2119 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:43:03.322 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:03.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:03.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:03.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:03.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:03.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:03.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:03.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:03.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:03.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:03.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:03.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:43:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:43:03] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.33.4" 2023-10-09T09:43:03.742 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:03.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:03.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:03.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:03.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:04.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:04.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:04.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:04.169 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:04.202 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:04.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:04.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:04.306 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:04.315 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:04 smithi163 bash[75425]: cluster 2023-10-09T09:43:03.001669+0000 mon.a (mon.0) 2120 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:43:04.315 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:04 smithi163 bash[75425]: cluster 2023-10-09T09:43:03.011031+0000 osd.5 (osd.5) 795 : cluster 3 failed to encode map e1212 with expected crc 2023-10-09T09:43:04.315 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:04 smithi163 bash[75425]: cluster 2023-10-09T09:43:03.011035+0000 osd.6 (osd.6) 501 : cluster 3 failed to encode map e1212 with expected crc 2023-10-09T09:43:04.315 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:04 smithi163 bash[75425]: cluster 2023-10-09T09:43:03.014816+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e1212: 8 total, 8 up, 6 in 2023-10-09T09:43:04.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:04 smithi181 bash[59345]: cluster 2023-10-09T09:43:03.001669+0000 mon.a (mon.0) 2120 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:43:04.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:04 smithi181 bash[59345]: cluster 2023-10-09T09:43:03.011031+0000 osd.5 (osd.5) 795 : cluster 3 failed to encode map e1212 with expected crc 2023-10-09T09:43:04.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:04 smithi181 bash[59345]: cluster 2023-10-09T09:43:03.011035+0000 osd.6 (osd.6) 501 : cluster 3 failed to encode map e1212 with expected crc 2023-10-09T09:43:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:04 smithi181 bash[59345]: cluster 2023-10-09T09:43:03.014816+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e1212: 8 total, 8 up, 6 in 2023-10-09T09:43:04.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:04 smithi163 bash[78107]: cluster 2023-10-09T09:43:03.001669+0000 mon.a (mon.0) 2120 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:43:04.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:04 smithi163 bash[78107]: cluster 2023-10-09T09:43:03.011031+0000 osd.5 (osd.5) 795 : cluster 3 failed to encode map e1212 with expected crc 2023-10-09T09:43:04.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:04 smithi163 bash[78107]: cluster 2023-10-09T09:43:03.011035+0000 osd.6 (osd.6) 501 : cluster 3 failed to encode map e1212 with expected crc 2023-10-09T09:43:04.318 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:04 smithi163 bash[78107]: cluster 2023-10-09T09:43:03.014816+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e1212: 8 total, 8 up, 6 in 2023-10-09T09:43:04.330 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:04.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:04.433 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:04.467 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:04.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:04.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:04.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:04.603 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:04.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:04.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:04.709 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:04.732 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:04.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:04.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:04.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:04.874 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:04.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:04.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:04.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:05.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:05.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:05.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:05.103 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:05.131 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:05.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:05.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:05.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:05.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:05 smithi163 bash[75425]: cluster 2023-10-09T09:43:04.015218+0000 mgr.y (mgr.34107) 1111 : cluster 0 pgmap v1920: 99 pgs: 1 peering, 1 active+recovery_wait, 97 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail 2023-10-09T09:43:05.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:05 smithi163 bash[78107]: cluster 2023-10-09T09:43:04.015218+0000 mgr.y (mgr.34107) 1111 : cluster 0 pgmap v1920: 99 pgs: 1 peering, 1 active+recovery_wait, 97 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail 2023-10-09T09:43:05.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:05.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:05.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:05 smithi181 bash[59345]: cluster 2023-10-09T09:43:04.015218+0000 mgr.y (mgr.34107) 1111 : cluster 0 pgmap v1920: 99 pgs: 1 peering, 1 active+recovery_wait, 97 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail 2023-10-09T09:43:05.368 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:05.393 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:05.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:05.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:05.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:05.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:05.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:05.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:05.632 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:05.671 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:05.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:05.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:05.776 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:05.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:05.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:05.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:05.914 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:05.941 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:05.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:05.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:06.045 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:06.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:06.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:06.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:06.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:06.209 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:06.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:06.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:06.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:06.341 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:06.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:06.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:06.445 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:06.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:06.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:06.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:06.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:06.608 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:06.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:06.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:06.714 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:06.747 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:06.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:06.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:06.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:06 smithi163 bash[75425]: cluster 2023-10-09T09:43:06.783295+0000 mon.a (mon.0) 2122 : cluster 3 Health check failed: Degraded data redundancy: 18/239661 objects degraded (0.008%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:43:06.853 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:06.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:06.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:06.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:06.985 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:07.003 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:07.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:07.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:06 smithi181 bash[59345]: cluster 2023-10-09T09:43:06.783295+0000 mon.a (mon.0) 2122 : cluster 3 Health check failed: Degraded data redundancy: 18/239661 objects degraded (0.008%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:43:07.106 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:07.129 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:07.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:07.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:07.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:06 smithi163 bash[78107]: cluster 2023-10-09T09:43:06.783295+0000 mon.a (mon.0) 2122 : cluster 3 Health check failed: Degraded data redundancy: 18/239661 objects degraded (0.008%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:43:07.232 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:07.261 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:07.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:07.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:07.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:07.388 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:07.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:07.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:07.493 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:07.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:07.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:07.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:07.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:07.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:07.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:07.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:07.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:07.816 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:07.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:07.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:07.921 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:07.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:07.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:07.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:08.059 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:08.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:07 smithi163 bash[75425]: cluster 2023-10-09T09:43:06.015806+0000 mgr.y (mgr.34107) 1112 : cluster 0 pgmap v1921: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 97 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 18/239661 objects degraded (0.008%); 4742/239661 objects misplaced (1.979%) 2023-10-09T09:43:08.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:07 smithi163 bash[78107]: cluster 2023-10-09T09:43:06.015806+0000 mgr.y (mgr.34107) 1112 : cluster 0 pgmap v1921: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 97 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 18/239661 objects degraded (0.008%); 4742/239661 objects misplaced (1.979%) 2023-10-09T09:43:08.104 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:08.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:08.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:08.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:08.217 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:07 smithi181 bash[59345]: cluster 2023-10-09T09:43:06.015806+0000 mgr.y (mgr.34107) 1112 : cluster 0 pgmap v1921: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 97 active+clean; 4.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 18/239661 objects degraded (0.008%); 4742/239661 objects misplaced (1.979%) 2023-10-09T09:43:08.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:08.261 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:08.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:08.362 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:08.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:08.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:08.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:08.494 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:08.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:08.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:08.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:08.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:08.663 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:08.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:08.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:08.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:08.818 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:08.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:08.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:08.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:08.985 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:08.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:08.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:09.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:09.114 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:09.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:09.217 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:09.255 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:09.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:09.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:09.359 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:09.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:09.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:09.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:09.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:09.549 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:09.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:09.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:09.654 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:09.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:09.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:09.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:09.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:09.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:09.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:09.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:09.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:09.953 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:09.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:10.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:10.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:10.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:10.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:10.119 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:09 smithi163 bash[78107]: cluster 2023-10-09T09:43:08.016418+0000 mgr.y (mgr.34107) 1113 : cluster 0 pgmap v1922: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 3294/233157 objects degraded (1.413%); 4742/233157 objects misplaced (2.034%) 2023-10-09T09:43:10.120 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:09 smithi163 bash[78107]: audit 2023-10-09T09:43:09.582573+0000 mon.a (mon.0) 2123 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:10.120 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:09 smithi163 bash[78107]: audit 2023-10-09T09:43:09.591544+0000 mon.a (mon.0) 2124 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:10.120 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:09 smithi163 bash[75425]: cluster 2023-10-09T09:43:08.016418+0000 mgr.y (mgr.34107) 1113 : cluster 0 pgmap v1922: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 3294/233157 objects degraded (1.413%); 4742/233157 objects misplaced (2.034%) 2023-10-09T09:43:10.120 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:09 smithi163 bash[75425]: audit 2023-10-09T09:43:09.582573+0000 mon.a (mon.0) 2123 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:10.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:09 smithi163 bash[75425]: audit 2023-10-09T09:43:09.591544+0000 mon.a (mon.0) 2124 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:10.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:10.191 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:09 smithi181 bash[59345]: cluster 2023-10-09T09:43:08.016418+0000 mgr.y (mgr.34107) 1113 : cluster 0 pgmap v1922: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 3294/233157 objects degraded (1.413%); 4742/233157 objects misplaced (2.034%) 2023-10-09T09:43:10.191 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:09 smithi181 bash[59345]: audit 2023-10-09T09:43:09.582573+0000 mon.a (mon.0) 2123 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:10.191 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:09 smithi181 bash[59345]: audit 2023-10-09T09:43:09.591544+0000 mon.a (mon.0) 2124 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:10.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:10.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:10.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:10.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:10.355 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:10.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:10.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:10.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:10.488 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:10.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:10.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:10.594 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:10.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:10.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:10.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:10.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:10.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:10.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:10.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:10.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:10.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:10.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:10.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:11.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:11.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:11.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:11.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:11.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:11.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:10 smithi181 bash[59345]: audit 2023-10-09T09:43:10.388175+0000 mon.a (mon.0) 2125 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:11.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:10 smithi181 bash[59345]: audit 2023-10-09T09:43:10.536121+0000 mon.a (mon.0) 2126 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:11.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:10 smithi181 bash[59345]: audit 2023-10-09T09:43:10.549941+0000 mon.a (mon.0) 2127 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:11.170 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:11.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:11.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:10 smithi163 bash[75425]: audit 2023-10-09T09:43:10.388175+0000 mon.a (mon.0) 2125 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:10 smithi163 bash[75425]: audit 2023-10-09T09:43:10.536121+0000 mon.a (mon.0) 2126 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:10 smithi163 bash[75425]: audit 2023-10-09T09:43:10.549941+0000 mon.a (mon.0) 2127 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:10 smithi163 bash[78107]: audit 2023-10-09T09:43:10.388175+0000 mon.a (mon.0) 2125 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:10 smithi163 bash[78107]: audit 2023-10-09T09:43:10.536121+0000 mon.a (mon.0) 2126 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:10 smithi163 bash[78107]: audit 2023-10-09T09:43:10.549941+0000 mon.a (mon.0) 2127 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:11.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:11.305 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:11.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:11.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:11.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:11.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:11.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:11.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:11.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:11.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:11.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:11.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:11.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:11.715 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:11.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:11.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:11.822 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:11.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:11.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:11.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:11.954 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:11.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:11 smithi163 bash[75425]: cluster 2023-10-09T09:43:10.017049+0000 mgr.y (mgr.34107) 1114 : cluster 0 pgmap v1923: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 3294/228741 objects degraded (1.440%); 4476/228741 objects misplaced (1.957%) 2023-10-09T09:43:11.963 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:11 smithi163 bash[78107]: cluster 2023-10-09T09:43:10.017049+0000 mgr.y (mgr.34107) 1114 : cluster 0 pgmap v1923: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 3294/228741 objects degraded (1.440%); 4476/228741 objects misplaced (1.957%) 2023-10-09T09:43:11.997 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:12.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:12.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:12.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:12.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:11 smithi181 bash[59345]: cluster 2023-10-09T09:43:10.017049+0000 mgr.y (mgr.34107) 1114 : cluster 0 pgmap v1923: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 3294/228741 objects degraded (1.440%); 4476/228741 objects misplaced (1.957%) 2023-10-09T09:43:12.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:12.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:12.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:12.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:12.253 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:12.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:12.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:12.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:12.378 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:12.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:12.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:12.506 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:12.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:12.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:12.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:12.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:12.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:12.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:12.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:12.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:12.885 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:12.924 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:13.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:13.368 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:12 smithi163 bash[78107]: cluster 2023-10-09T09:43:12.856333+0000 mon.a (mon.0) 2128 : cluster 3 Health check update: Degraded data redundancy: 3294/222597 objects degraded (1.480%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:13.368 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:12 smithi163 bash[75425]: cluster 2023-10-09T09:43:12.856333+0000 mon.a (mon.0) 2128 : cluster 3 Health check update: Degraded data redundancy: 3294/222597 objects degraded (1.480%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:13.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:13.369 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:12 smithi181 bash[59345]: cluster 2023-10-09T09:43:12.856333+0000 mon.a (mon.0) 2128 : cluster 3 Health check update: Degraded data redundancy: 3294/222597 objects degraded (1.480%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:13.470 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:13.489 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:13.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:13.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:13.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:13.620 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:13.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:13.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:43:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:43:13] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.33.4" 2023-10-09T09:43:13.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:13.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:13.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:13.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:13.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:13.884 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:13.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:13.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:13.917 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:13 smithi181 bash[59345]: cluster 2023-10-09T09:43:12.017639+0000 mgr.y (mgr.34107) 1115 : cluster 0 pgmap v1924: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 3294/222597 objects degraded (1.480%); 4476/222597 objects misplaced (2.011%) 2023-10-09T09:43:13.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:13.997 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:13 smithi163 bash[78107]: cluster 2023-10-09T09:43:12.017639+0000 mgr.y (mgr.34107) 1115 : cluster 0 pgmap v1924: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 3294/222597 objects degraded (1.480%); 4476/222597 objects misplaced (2.011%) 2023-10-09T09:43:13.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:13 smithi163 bash[75425]: cluster 2023-10-09T09:43:12.017639+0000 mgr.y (mgr.34107) 1115 : cluster 0 pgmap v1924: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 3294/222597 objects degraded (1.480%); 4476/222597 objects misplaced (2.011%) 2023-10-09T09:43:14.013 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:14.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:14.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:14.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:14.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:14.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:14.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:14.276 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:14.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:14.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:14.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:14.396 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:14.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:14.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:14.499 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:14.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:14.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:14.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:14.654 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:14.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:14.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:14.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:14.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:14.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:14.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:14.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:14.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:14.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:14.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:15.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:15.064 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:15.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:15.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:15.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:15.208 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:15.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:15.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:15.312 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:15.343 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:15.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:15.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:15.447 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:15.481 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:15.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:15.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:15.589 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:15.620 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:15.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:15.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:15.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:15.749 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:15.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:15.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:15.854 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:15.871 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:15.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:15.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:15.973 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:15.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:15 smithi181 bash[59345]: cluster 2023-10-09T09:43:14.018308+0000 mgr.y (mgr.34107) 1116 : cluster 0 pgmap v1925: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 381 op/s; 3294/219075 objects degraded (1.504%); 4476/219075 objects misplaced (2.043%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:43:15.999 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:16.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:16.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:16.108 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:16.117 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:15 smithi163 bash[78107]: cluster 2023-10-09T09:43:14.018308+0000 mgr.y (mgr.34107) 1116 : cluster 0 pgmap v1925: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 381 op/s; 3294/219075 objects degraded (1.504%); 4476/219075 objects misplaced (2.043%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:43:16.117 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:15 smithi163 bash[75425]: cluster 2023-10-09T09:43:14.018308+0000 mgr.y (mgr.34107) 1116 : cluster 0 pgmap v1925: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 381 op/s; 3294/219075 objects degraded (1.504%); 4476/219075 objects misplaced (2.043%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T09:43:16.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:16.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:16.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:16.239 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:16.265 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:16.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:16.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:16.370 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:16.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:16.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:16.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:16.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:16.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:16.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:16.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:16.625 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:16.651 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:16.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:16.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:16.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:16.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:16.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:16.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:16.795 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:43:16.857 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:43:16.885 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:16.904 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:43:16.908 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:16.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:16.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:16.942 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:43:16.993 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:43:17.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:17.026 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:43:17.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:17.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:17.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:17.062 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:43:17.094 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:43:17.097 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:43:17.097 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:43:17.097 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:43:17.097 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:43:17.097 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:43:17.098 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:43:17.098 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:43:17.098 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:43:17.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:16 smithi163 bash[75425]: audit 2023-10-09T09:43:16.725424+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.163:0/342384796' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:43:17.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:16 smithi163 bash[78107]: audit 2023-10-09T09:43:16.725424+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.163:0/342384796' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:43:17.137 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:17.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:17.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:17.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:17.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:16 smithi181 bash[59345]: audit 2023-10-09T09:43:16.725424+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.163:0/342384796' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:43:17.362 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:17.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:17.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:17.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:17.488 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:17.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:17.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:17.591 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:17.629 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:17.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:17.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:17.731 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:17.776 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:17.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:17.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:17.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:17.906 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:17.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:17.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:17.958 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:43:18.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:17 smithi181 bash[59345]: cluster 2023-10-09T09:43:16.018940+0000 mgr.y (mgr.34107) 1117 : cluster 0 pgmap v1926: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 3294/213636 objects degraded (1.542%); 4268/213636 objects misplaced (1.998%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T09:43:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:17 smithi181 bash[59345]: audit 2023-10-09T09:43:17.147498+0000 mon.a (mon.0) 2129 : audit 1 from='client.? 172.21.15.163:0/1425395479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:43:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:17 smithi181 bash[59345]: audit 2023-10-09T09:43:17.877129+0000 mon.a (mon.0) 2130 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:17 smithi181 bash[59345]: audit 2023-10-09T09:43:17.883527+0000 mon.a (mon.0) 2131 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:18.041 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:18.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:18.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:18.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:18.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:18.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:18.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:18.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:17 smithi163 bash[75425]: cluster 2023-10-09T09:43:16.018940+0000 mgr.y (mgr.34107) 1117 : cluster 0 pgmap v1926: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 3294/213636 objects degraded (1.542%); 4268/213636 objects misplaced (1.998%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T09:43:18.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:17 smithi163 bash[75425]: audit 2023-10-09T09:43:17.147498+0000 mon.a (mon.0) 2129 : audit 1 from='client.? 172.21.15.163:0/1425395479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:43:18.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:17 smithi163 bash[75425]: audit 2023-10-09T09:43:17.877129+0000 mon.a (mon.0) 2130 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:18.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:17 smithi163 bash[75425]: audit 2023-10-09T09:43:17.883527+0000 mon.a (mon.0) 2131 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:18.246 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:17 smithi163 bash[78107]: cluster 2023-10-09T09:43:16.018940+0000 mgr.y (mgr.34107) 1117 : cluster 0 pgmap v1926: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 3294/213636 objects degraded (1.542%); 4268/213636 objects misplaced (1.998%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T09:43:18.246 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:17 smithi163 bash[78107]: audit 2023-10-09T09:43:17.147498+0000 mon.a (mon.0) 2129 : audit 1 from='client.? 172.21.15.163:0/1425395479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:43:18.246 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:17 smithi163 bash[78107]: audit 2023-10-09T09:43:17.877129+0000 mon.a (mon.0) 2130 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:18.246 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:17 smithi163 bash[78107]: audit 2023-10-09T09:43:17.883527+0000 mon.a (mon.0) 2131 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:18.277 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:18.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:18.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:18.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:18.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:18.425 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:18.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:18.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:18.527 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:18.555 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:18.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:18.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:18.658 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:18.686 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:18.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:18.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:18.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:18.813 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:18.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:18.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:18.916 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:18.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:18.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:18 smithi181 bash[59345]: cluster 2023-10-09T09:43:17.941363+0000 mon.a (mon.0) 2132 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:43:18.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:18 smithi181 bash[59345]: cluster 2023-10-09T09:43:17.949590+0000 osd.5 (osd.5) 796 : cluster 3 failed to encode map e1213 with expected crc 2023-10-09T09:43:18.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:18 smithi181 bash[59345]: audit 2023-10-09T09:43:17.952582+0000 mon.a (mon.0) 2133 : audit 1 from='client.? 172.21.15.163:0/1425395479' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:43:18.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:18 smithi181 bash[59345]: cluster 2023-10-09T09:43:17.953025+0000 mon.a (mon.0) 2134 : cluster 0 osdmap e1213: 8 total, 8 up, 6 in 2023-10-09T09:43:18.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:18.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:19.053 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:19.062 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:18 smithi163 bash[75425]: cluster 2023-10-09T09:43:17.941363+0000 mon.a (mon.0) 2132 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:43:19.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:18 smithi163 bash[75425]: cluster 2023-10-09T09:43:17.949590+0000 osd.5 (osd.5) 796 : cluster 3 failed to encode map e1213 with expected crc 2023-10-09T09:43:19.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:18 smithi163 bash[75425]: audit 2023-10-09T09:43:17.952582+0000 mon.a (mon.0) 2133 : audit 1 from='client.? 172.21.15.163:0/1425395479' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:43:19.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:18 smithi163 bash[75425]: cluster 2023-10-09T09:43:17.953025+0000 mon.a (mon.0) 2134 : cluster 0 osdmap e1213: 8 total, 8 up, 6 in 2023-10-09T09:43:19.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:18 smithi163 bash[78107]: cluster 2023-10-09T09:43:17.941363+0000 mon.a (mon.0) 2132 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:43:19.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:18 smithi163 bash[78107]: cluster 2023-10-09T09:43:17.949590+0000 osd.5 (osd.5) 796 : cluster 3 failed to encode map e1213 with expected crc 2023-10-09T09:43:19.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:18 smithi163 bash[78107]: audit 2023-10-09T09:43:17.952582+0000 mon.a (mon.0) 2133 : audit 1 from='client.? 172.21.15.163:0/1425395479' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:43:19.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:18 smithi163 bash[78107]: cluster 2023-10-09T09:43:17.953025+0000 mon.a (mon.0) 2134 : cluster 0 osdmap e1213: 8 total, 8 up, 6 in 2023-10-09T09:43:19.087 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:19.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:19.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:19.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:19.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:19.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:19.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:19.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:19.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:19.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:19.470 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:19.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:19.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:19.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:19.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:19.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:19.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:19.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:19.735 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:19.773 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:19.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:19.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:19.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:19.910 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:19.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:19.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:20.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:20.023 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:19 smithi163 bash[78107]: cluster 2023-10-09T09:43:18.019399+0000 mgr.y (mgr.34107) 1118 : cluster 0 pgmap v1928: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 877 op/s; 3294/206820 objects degraded (1.593%); 4268/206820 objects misplaced (2.064%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:43:20.023 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:19 smithi163 bash[78107]: cluster 2023-10-09T09:43:18.951297+0000 mon.a (mon.0) 2135 : cluster 3 Health check update: Degraded data redundancy: 3294/206820 objects degraded (1.593%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:20.024 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:19 smithi163 bash[75425]: cluster 2023-10-09T09:43:18.019399+0000 mgr.y (mgr.34107) 1118 : cluster 0 pgmap v1928: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 877 op/s; 3294/206820 objects degraded (1.593%); 4268/206820 objects misplaced (2.064%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:43:20.024 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:19 smithi163 bash[75425]: cluster 2023-10-09T09:43:18.951297+0000 mon.a (mon.0) 2135 : cluster 3 Health check update: Degraded data redundancy: 3294/206820 objects degraded (1.593%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:20.052 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:20.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:20.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:20.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:20.182 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:20.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:20.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:20.284 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:20.294 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:19 smithi181 bash[59345]: cluster 2023-10-09T09:43:18.019399+0000 mgr.y (mgr.34107) 1118 : cluster 0 pgmap v1928: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 877 op/s; 3294/206820 objects degraded (1.593%); 4268/206820 objects misplaced (2.064%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:43:20.294 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:19 smithi181 bash[59345]: cluster 2023-10-09T09:43:18.951297+0000 mon.a (mon.0) 2135 : cluster 3 Health check update: Degraded data redundancy: 3294/206820 objects degraded (1.593%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:20.314 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:20.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:20.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:20.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:20.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:20.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:20.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:20.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:20.585 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:20.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:20.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:20.688 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:20.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:20.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:20.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:20.835 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:20.860 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:20.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:20.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:20.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:20.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:20.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:20.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:21.099 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:21.125 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:21.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:21.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:21.232 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:21.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:21.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:21.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:21.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:21.398 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:21.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:21.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:21.529 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:21.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:21.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:21.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:21.671 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:21.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:21.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:21.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:21.814 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:21.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:21.919 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:21.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:21.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:21.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:22.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:21 smithi181 bash[59345]: cluster 2023-10-09T09:43:20.019939+0000 mgr.y (mgr.34107) 1119 : cluster 0 pgmap v1929: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 861 op/s; 3294/202902 objects degraded (1.623%); 4058/202902 objects misplaced (2.000%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:43:22.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:22.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:22.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:22.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:22.208 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:22.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:21 smithi163 bash[78107]: cluster 2023-10-09T09:43:20.019939+0000 mgr.y (mgr.34107) 1119 : cluster 0 pgmap v1929: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 861 op/s; 3294/202902 objects degraded (1.623%); 4058/202902 objects misplaced (2.000%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:43:22.211 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:21 smithi163 bash[75425]: cluster 2023-10-09T09:43:20.019939+0000 mgr.y (mgr.34107) 1119 : cluster 0 pgmap v1929: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 861 op/s; 3294/202902 objects degraded (1.623%); 4058/202902 objects misplaced (2.000%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:43:22.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:22.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:22.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:22.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:22.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:22.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:22.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:22.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:22.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:22.545 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:43:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:43:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T09:43:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T09:43:22.546 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:43:22.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:22.606 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:22.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:22.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:22.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:22.736 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:22.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:22.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:22.840 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:22.900 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:22.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:22.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:22.976 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.4 primary-affinity to 0 (802) 2023-10-09T09:43:23.003 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:23.033 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:23.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:23.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:23.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:23.148 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:22 smithi163 bash[78107]: audit 2023-10-09T09:43:22.908725+0000 mon.a (mon.0) 2136 : audit 1 from='client.? 172.21.15.163:0/2169905524' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T09:43:23.149 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:22 smithi163 bash[75425]: audit 2023-10-09T09:43:22.908725+0000 mon.a (mon.0) 2136 : audit 1 from='client.? 172.21.15.163:0/2169905524' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T09:43:23.160 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:23.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:23.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:23.264 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:23.286 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:23.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:23.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:23.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:22 smithi181 bash[59345]: audit 2023-10-09T09:43:22.908725+0000 mon.a (mon.0) 2136 : audit 1 from='client.? 172.21.15.163:0/2169905524' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T09:43:23.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:23.420 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:23.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:23.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:23.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:43:23] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.33.4" 2023-10-09T09:43:23.524 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:23.554 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:23.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:23.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:23.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:23.683 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:23.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:23.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:23.787 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:23.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:23.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:23.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:23.919 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:23.947 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:23.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:23.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:24.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:24.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:24.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:24.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:24.169 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:43:24.183 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:24.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:23 smithi181 bash[59345]: cluster 2023-10-09T09:43:22.020717+0000 mgr.y (mgr.34107) 1120 : cluster 0 pgmap v1930: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 846 op/s; 3294/197208 objects degraded (1.670%); 4058/197208 objects misplaced (2.058%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:43:24.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:23 smithi181 bash[59345]: cluster 2023-10-09T09:43:22.964856+0000 osd.5 (osd.5) 797 : cluster 3 failed to encode map e1214 with expected crc 2023-10-09T09:43:24.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:23 smithi181 bash[59345]: cluster 2023-10-09T09:43:22.965760+0000 osd.4 (osd.4) 626 : cluster 3 failed to encode map e1214 with expected crc 2023-10-09T09:43:24.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:23 smithi181 bash[59345]: audit 2023-10-09T09:43:22.968745+0000 mon.a (mon.0) 2137 : audit 1 from='client.? 172.21.15.163:0/2169905524' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T09:43:24.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:23 smithi181 bash[59345]: cluster 2023-10-09T09:43:22.968852+0000 mon.a (mon.0) 2138 : cluster 0 osdmap e1214: 8 total, 8 up, 6 in 2023-10-09T09:43:24.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:23 smithi181 bash[59345]: audit 2023-10-09T09:43:23.035327+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:24.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:23 smithi181 bash[59345]: audit 2023-10-09T09:43:23.045343+0000 mon.a (mon.0) 2140 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:24.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:23 smithi181 bash[59345]: audit 2023-10-09T09:43:23.046471+0000 mon.a (mon.0) 2141 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:43:24.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:23 smithi181 bash[59345]: audit 2023-10-09T09:43:23.047393+0000 mon.a (mon.0) 2142 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:43:24.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:23 smithi181 bash[59345]: audit 2023-10-09T09:43:23.052974+0000 mon.a (mon.0) 2143 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:24.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:24.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[78107]: cluster 2023-10-09T09:43:22.020717+0000 mgr.y (mgr.34107) 1120 : cluster 0 pgmap v1930: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 846 op/s; 3294/197208 objects degraded (1.670%); 4058/197208 objects misplaced (2.058%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:43:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[78107]: cluster 2023-10-09T09:43:22.964856+0000 osd.5 (osd.5) 797 : cluster 3 failed to encode map e1214 with expected crc 2023-10-09T09:43:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[78107]: cluster 2023-10-09T09:43:22.965760+0000 osd.4 (osd.4) 626 : cluster 3 failed to encode map e1214 with expected crc 2023-10-09T09:43:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[78107]: audit 2023-10-09T09:43:22.968745+0000 mon.a (mon.0) 2137 : audit 1 from='client.? 172.21.15.163:0/2169905524' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T09:43:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[78107]: cluster 2023-10-09T09:43:22.968852+0000 mon.a (mon.0) 2138 : cluster 0 osdmap e1214: 8 total, 8 up, 6 in 2023-10-09T09:43:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[78107]: audit 2023-10-09T09:43:23.035327+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[78107]: audit 2023-10-09T09:43:23.045343+0000 mon.a (mon.0) 2140 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[78107]: audit 2023-10-09T09:43:23.046471+0000 mon.a (mon.0) 2141 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:43:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[78107]: audit 2023-10-09T09:43:23.047393+0000 mon.a (mon.0) 2142 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:43:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[78107]: audit 2023-10-09T09:43:23.052974+0000 mon.a (mon.0) 2143 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:24.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[75425]: cluster 2023-10-09T09:43:22.020717+0000 mgr.y (mgr.34107) 1120 : cluster 0 pgmap v1930: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 846 op/s; 3294/197208 objects degraded (1.670%); 4058/197208 objects misplaced (2.058%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T09:43:24.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[75425]: cluster 2023-10-09T09:43:22.964856+0000 osd.5 (osd.5) 797 : cluster 3 failed to encode map e1214 with expected crc 2023-10-09T09:43:24.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[75425]: cluster 2023-10-09T09:43:22.965760+0000 osd.4 (osd.4) 626 : cluster 3 failed to encode map e1214 with expected crc 2023-10-09T09:43:24.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[75425]: audit 2023-10-09T09:43:22.968745+0000 mon.a (mon.0) 2137 : audit 1 from='client.? 172.21.15.163:0/2169905524' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T09:43:24.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[75425]: cluster 2023-10-09T09:43:22.968852+0000 mon.a (mon.0) 2138 : cluster 0 osdmap e1214: 8 total, 8 up, 6 in 2023-10-09T09:43:24.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[75425]: audit 2023-10-09T09:43:23.035327+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:24.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[75425]: audit 2023-10-09T09:43:23.045343+0000 mon.a (mon.0) 2140 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:24.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[75425]: audit 2023-10-09T09:43:23.046471+0000 mon.a (mon.0) 2141 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:43:24.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[75425]: audit 2023-10-09T09:43:23.047393+0000 mon.a (mon.0) 2142 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:43:24.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:23 smithi163 bash[75425]: audit 2023-10-09T09:43:23.052974+0000 mon.a (mon.0) 2143 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:24.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:43:24.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:24.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:24.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:24.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:24.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:24.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:24.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:24.488 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:24.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:24.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:24.595 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:24.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:24.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:24.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:24.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:24.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:24.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:24.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:24.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:24.893 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:24.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:24.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:24.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:25.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:25.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:25.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:25.145 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:25.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:25.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:25.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:25.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[75425]: cluster 2023-10-09T09:43:23.079683+0000 osd.0 (osd.0) 4 : cluster 0 9.5 scrub starts 2023-10-09T09:43:25.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[75425]: cluster 2023-10-09T09:43:23.974372+0000 osd.5 (osd.5) 798 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[75425]: cluster 2023-10-09T09:43:23.975552+0000 osd.7 (osd.7) 582 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[75425]: cluster 2023-10-09T09:43:23.976425+0000 mon.a (mon.0) 2144 : cluster 0 osdmap e1215: 8 total, 8 up, 6 in 2023-10-09T09:43:25.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[75425]: cluster 2023-10-09T09:43:23.976659+0000 osd.5 (osd.5) 799 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[75425]: cluster 2023-10-09T09:43:23.978758+0000 osd.6 (osd.6) 502 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[75425]: cluster 2023-10-09T09:43:23.980739+0000 osd.4 (osd.4) 627 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[75425]: cluster 2023-10-09T09:43:23.981556+0000 osd.6 (osd.6) 503 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[75425]: cluster 2023-10-09T09:43:23.982903+0000 osd.4 (osd.4) 628 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[75425]: cluster 2023-10-09T09:43:24.021225+0000 mgr.y (mgr.34107) 1121 : cluster 0 pgmap v1933: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 710 op/s; 3294/193884 objects degraded (1.699%); 4058/193884 objects misplaced (2.093%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:43:25.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[78107]: cluster 2023-10-09T09:43:23.079683+0000 osd.0 (osd.0) 4 : cluster 0 9.5 scrub starts 2023-10-09T09:43:25.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[78107]: cluster 2023-10-09T09:43:23.974372+0000 osd.5 (osd.5) 798 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[78107]: cluster 2023-10-09T09:43:23.975552+0000 osd.7 (osd.7) 582 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[78107]: cluster 2023-10-09T09:43:23.976425+0000 mon.a (mon.0) 2144 : cluster 0 osdmap e1215: 8 total, 8 up, 6 in 2023-10-09T09:43:25.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[78107]: cluster 2023-10-09T09:43:23.976659+0000 osd.5 (osd.5) 799 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[78107]: cluster 2023-10-09T09:43:23.978758+0000 osd.6 (osd.6) 502 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[78107]: cluster 2023-10-09T09:43:23.980739+0000 osd.4 (osd.4) 627 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[78107]: cluster 2023-10-09T09:43:23.981556+0000 osd.6 (osd.6) 503 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[78107]: cluster 2023-10-09T09:43:23.982903+0000 osd.4 (osd.4) 628 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:24 smithi163 bash[78107]: cluster 2023-10-09T09:43:24.021225+0000 mgr.y (mgr.34107) 1121 : cluster 0 pgmap v1933: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 710 op/s; 3294/193884 objects degraded (1.699%); 4058/193884 objects misplaced (2.093%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:43:25.277 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:24 smithi181 bash[59345]: cluster 2023-10-09T09:43:23.079683+0000 osd.0 (osd.0) 4 : cluster 0 9.5 scrub starts 2023-10-09T09:43:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:24 smithi181 bash[59345]: cluster 2023-10-09T09:43:23.974372+0000 osd.5 (osd.5) 798 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:24 smithi181 bash[59345]: cluster 2023-10-09T09:43:23.975552+0000 osd.7 (osd.7) 582 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:24 smithi181 bash[59345]: cluster 2023-10-09T09:43:23.976425+0000 mon.a (mon.0) 2144 : cluster 0 osdmap e1215: 8 total, 8 up, 6 in 2023-10-09T09:43:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:24 smithi181 bash[59345]: cluster 2023-10-09T09:43:23.976659+0000 osd.5 (osd.5) 799 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:24 smithi181 bash[59345]: cluster 2023-10-09T09:43:23.978758+0000 osd.6 (osd.6) 502 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:24 smithi181 bash[59345]: cluster 2023-10-09T09:43:23.980739+0000 osd.4 (osd.4) 627 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:24 smithi181 bash[59345]: cluster 2023-10-09T09:43:23.981556+0000 osd.6 (osd.6) 503 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:24 smithi181 bash[59345]: cluster 2023-10-09T09:43:23.982903+0000 osd.4 (osd.4) 628 : cluster 3 failed to encode map e1215 with expected crc 2023-10-09T09:43:25.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:24 smithi181 bash[59345]: cluster 2023-10-09T09:43:24.021225+0000 mgr.y (mgr.34107) 1121 : cluster 0 pgmap v1933: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 710 op/s; 3294/193884 objects degraded (1.699%); 4058/193884 objects misplaced (2.093%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T09:43:25.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:25.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:25.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:25.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T09:43:25.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:25.450 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:25.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:25.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:25.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:25.583 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:25.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:25.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:25.687 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:25.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:25.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:25.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:25.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:25.860 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:25.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:25.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:25.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:25.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:25.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:25.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:26.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:26.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:25 smithi181 bash[59345]: cluster 2023-10-09T09:43:24.973893+0000 mon.a (mon.0) 2145 : cluster 3 Health check update: Degraded data redundancy: 3294/193884 objects degraded (1.699%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:26.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:25 smithi181 bash[59345]: audit 2023-10-09T09:43:25.394223+0000 mon.a (mon.0) 2146 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:26.107 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:25 smithi181 bash[59345]: audit 2023-10-09T09:43:25.395956+0000 mon.a (mon.0) 2147 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:26.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:26.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:26.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:26.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:26.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:26.360 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:26.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:25 smithi163 bash[78107]: cluster 2023-10-09T09:43:24.973893+0000 mon.a (mon.0) 2145 : cluster 3 Health check update: Degraded data redundancy: 3294/193884 objects degraded (1.699%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:26.370 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:25 smithi163 bash[78107]: audit 2023-10-09T09:43:25.394223+0000 mon.a (mon.0) 2146 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:26.370 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:25 smithi163 bash[78107]: audit 2023-10-09T09:43:25.395956+0000 mon.a (mon.0) 2147 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:26.370 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:25 smithi163 bash[75425]: cluster 2023-10-09T09:43:24.973893+0000 mon.a (mon.0) 2145 : cluster 3 Health check update: Degraded data redundancy: 3294/193884 objects degraded (1.699%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:26.370 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:25 smithi163 bash[75425]: audit 2023-10-09T09:43:25.394223+0000 mon.a (mon.0) 2146 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:26.370 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:25 smithi163 bash[75425]: audit 2023-10-09T09:43:25.395956+0000 mon.a (mon.0) 2147 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:26.388 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:26.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:26.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:26.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:26.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:26.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:26.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:26.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:26.662 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:26.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:26.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:26.766 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:26.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:26.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:26.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:26.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:26.933 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:26.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:27.042 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:27 smithi181 bash[59345]: cluster 2023-10-09T09:43:26.021951+0000 mgr.y (mgr.34107) 1122 : cluster 0 pgmap v1934: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 5947/188541 objects degraded (3.154%); 2672/188541 objects misplaced (1.417%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:43:27.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:27.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:27.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:27.196 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:27.206 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:27 smithi163 bash[75425]: cluster 2023-10-09T09:43:26.021951+0000 mgr.y (mgr.34107) 1122 : cluster 0 pgmap v1934: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 5947/188541 objects degraded (3.154%); 2672/188541 objects misplaced (1.417%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:43:27.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:27 smithi163 bash[78107]: cluster 2023-10-09T09:43:26.021951+0000 mgr.y (mgr.34107) 1122 : cluster 0 pgmap v1934: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 5947/188541 objects degraded (3.154%); 2672/188541 objects misplaced (1.417%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T09:43:27.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:27.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:27.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:27.330 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:27.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:27.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:27.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:27.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:27.517 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:27.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:27.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:27.619 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:27.645 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:27.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:27.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:27.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:27.778 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:27.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:27.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:27.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:28.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:28.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:28.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:28.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:28.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:28.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:28.494 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:28.531 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:28.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:28.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:28.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:28.661 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:28.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:28.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:28.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:29.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:29.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:29.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:29.047 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:43:29.047 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:43:29.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:29.129 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:29.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:29.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:29.235 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:29.270 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:29.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:29.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:29.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:29 smithi181 bash[59345]: cluster 2023-10-09T09:43:28.022739+0000 mgr.y (mgr.34107) 1123 : cluster 0 pgmap v1935: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 840 op/s; 3273/182730 objects degraded (1.791%); 4469/182730 objects misplaced (2.446%); 2.2 MiB/s, 33 objects/s recovering 2023-10-09T09:43:29.373 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:29.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:29.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:29.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:29.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:29 smithi163 bash[78107]: cluster 2023-10-09T09:43:28.022739+0000 mgr.y (mgr.34107) 1123 : cluster 0 pgmap v1935: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 840 op/s; 3273/182730 objects degraded (1.791%); 4469/182730 objects misplaced (2.446%); 2.2 MiB/s, 33 objects/s recovering 2023-10-09T09:43:29.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:29 smithi163 bash[75425]: cluster 2023-10-09T09:43:28.022739+0000 mgr.y (mgr.34107) 1123 : cluster 0 pgmap v1935: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 840 op/s; 3273/182730 objects degraded (1.791%); 4469/182730 objects misplaced (2.446%); 2.2 MiB/s, 33 objects/s recovering 2023-10-09T09:43:29.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:29.535 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:29.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:29.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:29.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:29.668 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:29.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:29.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:29.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:29.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:29.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:29.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:29.901 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:29.926 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:29.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:29.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:30.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:30.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:30.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:30.062 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:43:30.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:30.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:30.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:30.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:30.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:30.305 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:30 smithi163 bash[75425]: cluster 2023-10-09T09:43:30.083403+0000 mon.a (mon.0) 2148 : cluster 3 Health check update: Degraded data redundancy: 3273/177192 objects degraded (1.847%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:30.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:30 smithi163 bash[78107]: cluster 2023-10-09T09:43:30.083403+0000 mon.a (mon.0) 2148 : cluster 3 Health check update: Degraded data redundancy: 3273/177192 objects degraded (1.847%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:30.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:30.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:30.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:30.354 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 6, 2, 5] out_osds: [7, 3] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:43:30.355 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:43:30.355 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T09:43:30.355 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.637582 2023-10-09T09:43:30.355 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0.637581602767268 2023-10-09T09:43:30.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:30.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:30.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:30.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:30.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:30.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:30 smithi181 bash[59345]: cluster 2023-10-09T09:43:30.083403+0000 mon.a (mon.0) 2148 : cluster 3 Health check update: Degraded data redundancy: 3273/177192 objects degraded (1.847%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:30.602 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:30.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:30.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:30.705 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:30.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:30.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:30.848 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:30.873 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:30.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:30.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:30.980 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:31.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:31.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:31.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:31.114 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:31.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:31.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:31.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:31.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:31.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:31.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:31.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:31.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:31.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:31.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:31.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:31.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:31.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:31.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:31.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:31.651 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:31.694 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:31.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:31.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:31.794 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:31 smithi181 bash[59345]: cluster 2023-10-09T09:43:30.023344+0000 mgr.y (mgr.34107) 1124 : cluster 0 pgmap v1936: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 833 op/s; 3273/177192 objects degraded (1.847%); 4469/177192 objects misplaced (2.522%); 2.2 MiB/s, 33 objects/s recovering 2023-10-09T09:43:31.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:31.823 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:31.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:31.932 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:31.960 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:31.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:31.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:32.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:32.093 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:32.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:32.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:32.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:32.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:31 smithi163 bash[78107]: cluster 2023-10-09T09:43:30.023344+0000 mgr.y (mgr.34107) 1124 : cluster 0 pgmap v1936: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 833 op/s; 3273/177192 objects degraded (1.847%); 4469/177192 objects misplaced (2.522%); 2.2 MiB/s, 33 objects/s recovering 2023-10-09T09:43:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:31 smithi163 bash[75425]: cluster 2023-10-09T09:43:30.023344+0000 mgr.y (mgr.34107) 1124 : cluster 0 pgmap v1936: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 833 op/s; 3273/177192 objects degraded (1.847%); 4469/177192 objects misplaced (2.522%); 2.2 MiB/s, 33 objects/s recovering 2023-10-09T09:43:32.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:32.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:32.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:32.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:32.367 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:32.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:32.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:32.473 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:32.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:32.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:32.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:32.614 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:32.646 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:32.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:32.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:32.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:32.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:32.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:32.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:32.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:32.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:32.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:32.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:33.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:33.035 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:33.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:33.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:33.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:33.173 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:33.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:33.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:33.277 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:33.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:33.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:33.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:33.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:33.433 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:43:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:43:33] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.33.4" 2023-10-09T09:43:33.454 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:33.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:33.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:33.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:33.625 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:33.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:33.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:33.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:33.752 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:33.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:33.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:33.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:33.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:33.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:33 smithi163 bash[78107]: cluster 2023-10-09T09:43:32.024042+0000 mgr.y (mgr.34107) 1125 : cluster 0 pgmap v1937: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 3273/172455 objects degraded (1.898%); 4239/172455 objects misplaced (2.458%); 3.5 MiB/s, 54 objects/s recovering 2023-10-09T09:43:33.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:33 smithi163 bash[75425]: cluster 2023-10-09T09:43:32.024042+0000 mgr.y (mgr.34107) 1125 : cluster 0 pgmap v1937: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 3273/172455 objects degraded (1.898%); 4239/172455 objects misplaced (2.458%); 3.5 MiB/s, 54 objects/s recovering 2023-10-09T09:43:33.991 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:34.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:34.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:34.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:34.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:33 smithi181 bash[59345]: cluster 2023-10-09T09:43:32.024042+0000 mgr.y (mgr.34107) 1125 : cluster 0 pgmap v1937: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 3273/172455 objects degraded (1.898%); 4239/172455 objects misplaced (2.458%); 3.5 MiB/s, 54 objects/s recovering 2023-10-09T09:43:34.132 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:34.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:34.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:34.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:34.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:34.299 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:34.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:34.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:34.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:34.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:34.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:34.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:34.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:34.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:34.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:34.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:34.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:34.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:34.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:34.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:34.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:34.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:34.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:34.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:34.949 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:34.976 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:34.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:34.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:35.080 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:35.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:35.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:35.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:35.212 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:35.241 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:35.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:35.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:35.344 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:35.367 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:35.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:35.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:35.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:35.503 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:35.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:35.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:35.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:35.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:35.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:35.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:35.739 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:35.779 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:35.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:35.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:35.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:35.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:35.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:35.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:35.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:35 smithi163 bash[78107]: cluster 2023-10-09T09:43:34.024680+0000 mgr.y (mgr.34107) 1126 : cluster 0 pgmap v1938: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 854 op/s; 3273/168108 objects degraded (1.947%); 4239/168108 objects misplaced (2.522%); 3.2 MiB/s, 49 objects/s recovering 2023-10-09T09:43:35.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:35 smithi163 bash[78107]: cluster 2023-10-09T09:43:35.787426+0000 mon.a (mon.0) 2149 : cluster 3 Health check update: Degraded data redundancy: 3273/168108 objects degraded (1.947%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:35.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:35 smithi163 bash[75425]: cluster 2023-10-09T09:43:34.024680+0000 mgr.y (mgr.34107) 1126 : cluster 0 pgmap v1938: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 854 op/s; 3273/168108 objects degraded (1.947%); 4239/168108 objects misplaced (2.522%); 3.2 MiB/s, 49 objects/s recovering 2023-10-09T09:43:35.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:35 smithi163 bash[75425]: cluster 2023-10-09T09:43:35.787426+0000 mon.a (mon.0) 2149 : cluster 3 Health check update: Degraded data redundancy: 3273/168108 objects degraded (1.947%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:36.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:36.044 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:36.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:36.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:35 smithi181 bash[59345]: cluster 2023-10-09T09:43:34.024680+0000 mgr.y (mgr.34107) 1126 : cluster 0 pgmap v1938: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 854 op/s; 3273/168108 objects degraded (1.947%); 4239/168108 objects misplaced (2.522%); 3.2 MiB/s, 49 objects/s recovering 2023-10-09T09:43:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:35 smithi181 bash[59345]: cluster 2023-10-09T09:43:35.787426+0000 mon.a (mon.0) 2149 : cluster 3 Health check update: Degraded data redundancy: 3273/168108 objects degraded (1.947%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:36.148 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:36.178 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:36.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:36.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:36.284 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:36.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:36.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:36.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:36.416 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:36.444 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:36.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:36.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:36.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:36.574 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:36.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:36.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:36.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:36.705 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:36.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:36.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:36.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:36.841 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:36.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:36.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:36.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:36.979 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:36.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:36.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:37.085 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:37.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:37.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:37.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:37.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:37.254 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:37.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:37.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:37.357 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:37.393 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:37.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:37.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:37.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:37.523 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:37.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:37.628 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:37.654 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:37.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:37.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:37.758 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:37.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:37.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:37.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:37.855 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:43:37.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:37.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:37 smithi163 bash[75425]: cluster 2023-10-09T09:43:36.025385+0000 mgr.y (mgr.34107) 1127 : cluster 0 pgmap v1939: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 900 op/s; 3273/161466 objects degraded (2.027%); 4239/161466 objects misplaced (2.625%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T09:43:37.903 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:37 smithi163 bash[78107]: cluster 2023-10-09T09:43:36.025385+0000 mgr.y (mgr.34107) 1127 : cluster 0 pgmap v1939: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 900 op/s; 3273/161466 objects degraded (2.027%); 4239/161466 objects misplaced (2.625%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T09:43:37.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:37.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:37.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:38.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:38.059 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:38.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:37 smithi181 bash[59345]: cluster 2023-10-09T09:43:36.025385+0000 mgr.y (mgr.34107) 1127 : cluster 0 pgmap v1939: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 900 op/s; 3273/161466 objects degraded (2.027%); 4239/161466 objects misplaced (2.625%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T09:43:38.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:38.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:38.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:38.206 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:38.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:38.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:38.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:38.335 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:38.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:38.438 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:38.465 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:38.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:38.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:38.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:38.594 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:38.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:38.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:38.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:38.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:38.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:38.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:38.823 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:38.869 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:38.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:38.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:38.974 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:39.003 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:39.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:39.107 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:39.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:39.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:39.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:39.281 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:39.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:39.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:39.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:39.421 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:39.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:39.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:39.527 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:39.560 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:39.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:39.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:39.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:39.694 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:39.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:39.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:39.799 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:39.837 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:39.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:39.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:39.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:39.983 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:39.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:39 smithi181 bash[59345]: cluster 2023-10-09T09:43:38.026190+0000 mgr.y (mgr.34107) 1128 : cluster 0 pgmap v1940: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 939 op/s; 3273/154695 objects degraded (2.116%); 4008/154695 objects misplaced (2.591%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:43:40.089 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:40.104 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:39 smithi163 bash[75425]: cluster 2023-10-09T09:43:38.026190+0000 mgr.y (mgr.34107) 1128 : cluster 0 pgmap v1940: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 939 op/s; 3273/154695 objects degraded (2.116%); 4008/154695 objects misplaced (2.591%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:43:40.104 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:39 smithi163 bash[78107]: cluster 2023-10-09T09:43:38.026190+0000 mgr.y (mgr.34107) 1128 : cluster 0 pgmap v1940: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 939 op/s; 3273/154695 objects degraded (2.116%); 4008/154695 objects misplaced (2.591%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:43:40.124 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:40.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:40.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:40.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:40.269 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:40.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:40.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:40.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:40.398 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:40.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:40.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:40.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:40.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:40.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:40.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:40.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:40.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:40.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:40.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:40.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:40.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:40.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:40.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:40.920 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:40.932 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:40 smithi181 bash[59345]: audit 2023-10-09T09:43:40.388885+0000 mon.a (mon.0) 2150 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:40.932 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:40 smithi181 bash[59345]: cluster 2023-10-09T09:43:40.788458+0000 mon.a (mon.0) 2151 : cluster 3 Health check update: Degraded data redundancy: 3273/154695 objects degraded (2.116%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:40.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:40.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:40.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:41.056 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:41.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:40 smithi163 bash[75425]: audit 2023-10-09T09:43:40.388885+0000 mon.a (mon.0) 2150 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:41.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:40 smithi163 bash[75425]: cluster 2023-10-09T09:43:40.788458+0000 mon.a (mon.0) 2151 : cluster 3 Health check update: Degraded data redundancy: 3273/154695 objects degraded (2.116%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:41.065 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:40 smithi163 bash[78107]: audit 2023-10-09T09:43:40.388885+0000 mon.a (mon.0) 2150 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:41.065 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:40 smithi163 bash[78107]: cluster 2023-10-09T09:43:40.788458+0000 mon.a (mon.0) 2151 : cluster 3 Health check update: Degraded data redundancy: 3273/154695 objects degraded (2.116%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:41.090 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:41.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:41.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:41.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:41.225 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:41.228 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:43:41.228 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:43:41.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:43:41.228 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:43:41.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.228 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38767 8.e 8:79fdfb4c:::benchmark_data_smithi163_151167_object38766:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:43:41.228 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.690092+0000", 2023-10-09T09:43:41.228 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 473.531101476, 2023-10-09T09:43:41.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1191520370000001, 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38767 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.690092+0000", 2023-10-09T09:43:41.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.690092+0000", 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.690090+0000", 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.690118+0000", 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7509999999999999e-05 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.690129+0000", 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1464000000000001e-05 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.690138+0000", 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7469999999999994e-06 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.693966+0000", 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038280530000000001 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.693978+0000", 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1599e-05 2023-10-09T09:43:41.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.688409+0000", 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99443152899999998 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.688431+0000", 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2328e-05 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.800198+0000", 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.111766539 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.800345+0000", 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014727699999999999 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.800531+0000", 2023-10-09T09:43:41.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018603599999999999 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.807556+0000", 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0070249600000000002 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.807599+0000", 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3081000000000003e-05 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.807642+0000", 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2533000000000001e-05 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.807700+0000", 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8136e-05 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.809244+0000", 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001543792 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38782 8.e 8:73dcdc82:::benchmark_data_smithi163_151167_object38781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.708246+0000", 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 473.51294673699999, 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1074889300000001, 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.237 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38782 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708246+0000", 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708246+0000", 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708245+0000", 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708275+0000", 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0460999999999999e-05 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708306+0000", 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0639999999999998e-05 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708314+0000", 2023-10-09T09:43:41.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3440000000000001e-06 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708368+0000", 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3953000000000003e-05 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.708379+0000", 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1129e-05 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.688477+0000", 2023-10-09T09:43:41.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98009800400000002 2023-10-09T09:43:41.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.688489+0000", 2023-10-09T09:43:41.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1695000000000001e-05 2023-10-09T09:43:41.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.804274+0000", 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1157848539999999 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.804445+0000", 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000171495 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:43:41.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.804652+0000", 2023-10-09T09:43:41.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020640099999999999 2023-10-09T09:43:41.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.811135+0000", 2023-10-09T09:43:41.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064828749999999999 2023-10-09T09:43:41.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.811268+0000", 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013374200000000001 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.811310+0000", 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1684000000000003e-05 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.811480+0000", 2023-10-09T09:43:41.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016988100000000001 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.815735+0000", 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042555780000000003 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.246 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:38791 8.e 8:7ebcdd3e:::benchmark_data_smithi163_151167_object38790:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1140)", 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:35:47.712941+0000", 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 473.50825235600001, 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.108528626, 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38791 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.247 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712941+0000", 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712941+0000", 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712939+0000", 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712967+0000", 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7712000000000002e-05 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712985+0000", 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8488999999999999e-05 2023-10-09T09:43:41.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.712994+0000", 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6309999999999992e-06 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.713066+0000", 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2076999999999994e-05 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:47.713075+0000", 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7690000000000002e-06 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.688523+0000", 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.975447866 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:48.688533+0000", 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0733e-05 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.809270+0000", 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.120736366 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.809435+0000", 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016523800000000001 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.809565+0000", 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012998400000000001 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.819681+0000", 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010115733 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.819758+0000", 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7715999999999999e-05 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.819810+0000", 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.215e-05 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.819851+0000", 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0898e-05 2023-10-09T09:43:41.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:35:49.821469+0000", 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001618097 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85447 8.e 8:7df815c7:::benchmark_data_smithi163_151167_object85446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1150)", 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:43.581142+0000", 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 417.64005091500002, 2023-10-09T09:43:41.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2748772800000001, 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85447 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.581142+0000", 2023-10-09T09:43:41.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.581142+0000", 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.581140+0000", 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.581170+0000", 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0060999999999999e-05 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.581192+0000", 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1246e-05 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.581201+0000", 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0149999999999992e-06 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.581259+0000", 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7846000000000001e-05 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.581268+0000", 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9550000000000004e-06 2023-10-09T09:43:41.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.837310+0000", 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2560411199999999 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.837461+0000", 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015137500000000001 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.837593+0000", 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013155799999999999 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.848507+0000", 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010914193000000001 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.855728+0000", 2023-10-09T09:43:41.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072210160000000002 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.855796+0000", 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8580999999999999e-05 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.855841+0000", 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4217e-05 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.855901+0000", 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0427999999999998e-05 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856020+0000", 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000118596 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85472 8.e 8:70431e47:::benchmark_data_smithi163_151167_object85471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1150)", 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:43.629988+0000", 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 417.59120503600002, 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.22628845, 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85472 2023-10-09T09:43:41.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.629988+0000", 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.629988+0000", 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.629986+0000", 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.630021+0000", 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4598999999999999e-05 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.630041+0000", 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0010999999999999e-05 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.630051+0000", 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6039999999999999e-06 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.630123+0000", 2023-10-09T09:43:41.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2860000000000004e-05 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.630133+0000", 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6169999999999996e-06 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.841542+0000", 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.21140859 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.841661+0000", 2023-10-09T09:43:41.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011900099999999999 2023-10-09T09:43:41.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T09:43:41.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.841785+0000", 2023-10-09T09:43:41.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012386499999999999 2023-10-09T09:43:41.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.855736+0000", 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013951498999999999 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856061+0000", 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00032489800000000001 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856093+0000", 2023-10-09T09:43:41.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1875999999999998e-05 2023-10-09T09:43:41.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856124+0000", 2023-10-09T09:43:41.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1211000000000003e-05 2023-10-09T09:43:41.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856172+0000", 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7920999999999997e-05 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856277+0000", 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010471600000000001 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85515 8.e 8:7d98ff2a:::benchmark_data_smithi163_151167_object85514:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1150)", 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:43.687531+0000", 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 417.53366187500001, 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.177582865, 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:43:41.273 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85515 2023-10-09T09:43:41.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.274 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.687531+0000", 2023-10-09T09:43:41.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.687531+0000", 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.687530+0000", 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.687552+0000", 2023-10-09T09:43:41.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2608000000000001e-05 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.687565+0000", 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2597e-05 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.687572+0000", 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7499999999999997e-06 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.687793+0000", 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00022117799999999999 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.687800+0000", 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2830000000000004e-06 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.844919+0000", 2023-10-09T09:43:41.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1571188720000001 2023-10-09T09:43:41.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.845054+0000", 2023-10-09T09:43:41.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013533800000000001 2023-10-09T09:43:41.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T09:43:41.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.845224+0000", 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017001900000000001 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856318+0000", 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011093946 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856355+0000", 2023-10-09T09:43:41.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6681999999999999e-05 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856467+0000", 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000112026 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.864934+0000", 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0084668520000000004 2023-10-09T09:43:41.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.864991+0000", 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7333999999999997e-05 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.865114+0000", 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000123134 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.281 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.282 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85517 8.e 8:7a744eb9:::benchmark_data_smithi163_151167_object85516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1150)", 2023-10-09T09:43:41.282 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:43.691680+0000", 2023-10-09T09:43:41.282 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 417.52951315000001, 2023-10-09T09:43:41.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1736198, 2023-10-09T09:43:41.282 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.282 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.282 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.282 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85517 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.691680+0000", 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.691680+0000", 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.691678+0000", 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.691718+0000", 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9740999999999998e-05 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.691728+0000", 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.961e-06 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.691737+0000", 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2839999999999995e-06 2023-10-09T09:43:41.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.691785+0000", 2023-10-09T09:43:41.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7454999999999999e-05 2023-10-09T09:43:41.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.691798+0000", 2023-10-09T09:43:41.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3509000000000001e-05 2023-10-09T09:43:41.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.848535+0000", 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1567365780000001 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.848705+0000", 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016997200000000001 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.848879+0000", 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017420200000000001 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856389+0000", 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0075094890000000003 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856457+0000", 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.834e-05 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856474+0000", 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6612999999999998e-05 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.865153+0000", 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.008679338 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.865202+0000", 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9055999999999999e-05 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.865300+0000", 2023-10-09T09:43:41.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7874000000000005e-05 2023-10-09T09:43:41.291 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.291 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:85536 8.e 8:7db0a872:::benchmark_data_smithi163_151167_object85535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1150)", 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:36:43.757116+0000", 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 417.464077589, 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.108375143, 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.292 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 85536 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.757116+0000", 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.757116+0000", 2023-10-09T09:43:41.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.757113+0000", 2023-10-09T09:43:41.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999971 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.757160+0000", 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7014000000000001e-05 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.757192+0000", 2023-10-09T09:43:41.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2147999999999998e-05 2023-10-09T09:43:41.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.757208+0000", 2023-10-09T09:43:41.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5628999999999999e-05 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.757287+0000", 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9164000000000003e-05 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:43.757302+0000", 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5308e-05 2023-10-09T09:43:41.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.852446+0000", 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0951438149999999 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.852595+0000", 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014915800000000001 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T09:43:41.300 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.852712+0000", 2023-10-09T09:43:41.300 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000117124 2023-10-09T09:43:41.300 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.300 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.856482+0000", 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037700250000000002 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.860107+0000", 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036246680000000002 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.301 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.862118+0000", 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002011072 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.865337+0000", 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032186390000000001 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.865392+0000", 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5047000000000003e-05 2023-10-09T09:43:41.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.303 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:36:44.865491+0000", 2023-10-09T09:43:41.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9152000000000002e-05 2023-10-09T09:43:41.303 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.303 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.303 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.303 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:41132 9.5 9:a506c85a:::benchmark_data_smithi163_175771_object41131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1195)", 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:41:34.509165+0000", 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 126.712027859, 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.34989529899999999, 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.304 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 41132 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509165+0000", 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509165+0000", 2023-10-09T09:43:41.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509163+0000", 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509195+0000", 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1612000000000002e-05 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509216+0000", 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0812999999999999e-05 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509227+0000", 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0817e-05 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509381+0000", 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015448600000000001 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509567+0000", 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018614 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509701+0000", 2023-10-09T09:43:41.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013367199999999999 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.513431+0000", 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037295240000000001 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.519236+0000", 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058054020000000003 2023-10-09T09:43:41.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.858786+0000", 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33954963199999999 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.858945+0000", 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015949599999999999 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.859061+0000", 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115609 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:41133 9.8 9:18e4377f:::benchmark_data_smithi163_175771_object41132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1195)", 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:41:34.509282+0000", 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 126.71191119300001, 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.35058322400000003, 2023-10-09T09:43:41.311 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 41133 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509282+0000", 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509282+0000", 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509281+0000", 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509307+0000", 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5908e-05 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509314+0000", 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7730000000000002e-06 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509321+0000", 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8950000000000001e-06 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509380+0000", 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9247999999999999e-05 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509521+0000", 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014051299999999999 2023-10-09T09:43:41.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.509650+0000", 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012944700000000001 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.513415+0000", 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037652129999999999 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.522305+0000", 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0088894569999999999 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.858739+0000", 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33643446399999999 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.858845+0000", 2023-10-09T09:43:41.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000105862 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.859865+0000", 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001020293 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:41136 9.9 9:9df390fc:::benchmark_data_smithi163_175771_object41135:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1195)", 2023-10-09T09:43:41.318 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:41:34.512876+0000", 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 126.70831676, 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.34928475599999997, 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 41136 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.319 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.512876+0000", 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.512876+0000", 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.512875+0000", 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:43:41.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.512900+0000", 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5619999999999999e-05 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.512918+0000", 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7329e-05 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.512926+0000", 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8269999999999998e-06 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.512971+0000", 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5325999999999999e-05 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.513092+0000", 2023-10-09T09:43:41.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121513 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.513205+0000", 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011237 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.514930+0000", 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017254320000000001 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.515757+0000", 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082681599999999997 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.861899+0000", 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.34614201300000003 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.862053+0000", 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015378700000000001 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.862161+0000", 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000108434 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:41140 9.8 9:1168e0f3:::benchmark_data_smithi163_175771_object41139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1195)", 2023-10-09T09:43:41.325 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:41:34.518026+0000", 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 126.703167034, 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.35100831700000001, 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 41140 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.518026+0000", 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.518026+0000", 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.518024+0000", 2023-10-09T09:43:41.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.518050+0000", 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5477e-05 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.518066+0000", 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6433999999999999e-05 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.518077+0000", 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0314e-05 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.518220+0000", 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014312400000000001 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.518351+0000", 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013081900000000001 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.518463+0000", 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000112367 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.520771+0000", 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002308484 2023-10-09T09:43:41.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528541+0000", 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0077697010000000004 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.867679+0000", 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33913792700000001 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.867911+0000", 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000232303 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.869035+0000", 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011230840000000001 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:41148 9.9 9:9079ab32:::benchmark_data_smithi163_175771_object41147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1195)", 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:41:34.528328+0000", 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 126.69286474499999, 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33413530499999999, 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 41148 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.333 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528328+0000", 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528328+0000", 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528326+0000", 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528360+0000", 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4261000000000003e-05 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528385+0000", 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4885999999999999e-05 2023-10-09T09:43:41.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528398+0000", 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2753e-05 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528452+0000", 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3526000000000002e-05 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528596+0000", 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014387100000000001 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528724+0000", 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012808900000000001 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.531300+0000", 2023-10-09T09:43:41.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0025760190000000001 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.531996+0000", 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00069655200000000002 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.862200+0000", 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33020355200000001 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.862347+0000", 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014748499999999999 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.862464+0000", 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000116562 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:41151 9.5 9:a668900e:::benchmark_data_smithi163_175771_object41150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1195)", 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:41:34.530502+0000", 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 126.690691489, 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33578134199999998, 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 41151 2023-10-09T09:43:41.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530502+0000", 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530502+0000", 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530500+0000", 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530530+0000", 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0182e-05 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530547+0000", 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7482999999999998e-05 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530555+0000", 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1280000000000002e-06 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530607+0000", 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2574999999999997e-05 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530749+0000", 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000141445 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530941+0000", 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000192012 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.532540+0000", 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015998329999999999 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.532719+0000", 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000178188 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.866083+0000", 2023-10-09T09:43:41.345 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33336463 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.866213+0000", 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000129794 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.866283+0000", 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0049000000000006e-05 2023-10-09T09:43:41.346 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:106689 9.18 9:18837861:::benchmark_data_smithi163_175771_object2793:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1204)", 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:42:37.944880+0000", 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 63.276312867000001, 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32398901400000002, 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.347 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 106689 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:37.944880+0000", 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.348 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:37.944880+0000", 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:37.944879+0000", 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:37.944885+0000", 2023-10-09T09:43:41.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3419999999999998e-06 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:37.944896+0000", 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0812000000000001e-05 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:37.944907+0000", 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1738e-05 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.350 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:37.947693+0000", 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002785807 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:37.950352+0000", 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026584080000000001 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:38.234785+0000", 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28443370200000001 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:38.234919+0000", 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013414500000000001 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:38.235052+0000", 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013276 2023-10-09T09:43:41.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:38.267624+0000", 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.032571846000000002 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:38.267674+0000", 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0010999999999999e-05 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:38.268436+0000", 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00076152900000000005 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:38.268484+0000", 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8579999999999999e-05 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:42:38.268869+0000", 2023-10-09T09:43:41.354 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00038518599999999998 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:144805 9.c 9:3a6de65a:::benchmark_data_smithi163_175771_object40909:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e1214)", 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:43:22.987391+0000", 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 18.233802296, 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.029655931, 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.355 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 144805 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:22.987391+0000", 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.356 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:22.987391+0000", 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:22.987389+0000", 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:22.987397+0000", 2023-10-09T09:43:41.357 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8820000000000001e-06 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:22.987410+0000", 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3397999999999999e-05 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:22.987419+0000", 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2210000000000003e-06 2023-10-09T09:43:41.358 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:22.997364+0000", 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0099441530000000007 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:22.997375+0000", 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1102e-05 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.359 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.015180+0000", 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.017805641 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.017047+0000", 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0018664930000000001 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.360 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:144832 9.c 9:34bd91e0:::benchmark_data_smithi163_175771_object40936:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1214)", 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:43:23.010239+0000", 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 18.210953986, 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0263818899999999, 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 144832 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.361 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.010239+0000", 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.010239+0000", 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.362 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.010238+0000", 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.010244+0000", 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2360000000000004e-06 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.010255+0000", 2023-10-09T09:43:41.363 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1735e-05 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.010263+0000", 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4120000000000004e-06 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.010314+0000", 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1715999999999997e-05 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.364 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.010324+0000", 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.628e-06 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.017549+0000", 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0072250599999999 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.018158+0000", 2023-10-09T09:43:41.365 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060890399999999996 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,6", 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.018296+0000", 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013767799999999999 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.033761+0000", 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015465051 2023-10-09T09:43:41.366 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.033929+0000", 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016863300000000001 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.036529+0000", 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0025994049999999999 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.367 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.036581+0000", 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1918e-05 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.036621+0000", 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0357e-05 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.368 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:144834 9.c 9:3ec3c845:::benchmark_data_smithi163_175771_object40938:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1214)", 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:43:23.017452+0000", 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 18.203741072, 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.019351517, 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 144834 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.369 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.017452+0000", 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.017452+0000", 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.370 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.017451+0000", 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.017459+0000", 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1159999999999999e-06 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.017474+0000", 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4888e-05 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.371 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.017483+0000", 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8009999999999998e-06 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.017555+0000", 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1396999999999994e-05 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.372 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.017568+0000", 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2979999999999999e-05 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.021125+0000", 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.003557244 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.021363+0000", 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023786400000000001 2023-10-09T09:43:41.373 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,6", 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.021572+0000", 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000209488 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.033884+0000", 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012311282 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.374 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.033939+0000", 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5535999999999998e-05 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.036676+0000", 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002736605 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.375 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.036763+0000", 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7251999999999997e-05 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.036804+0000", 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0601e-05 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.376 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:144849 9.c 9:371c05e2:::benchmark_data_smithi163_175771_object40953:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1214)", 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:43:23.044979+0000", 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 18.176213804, 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99505090699999998, 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 144849 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.377 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.044979+0000", 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.044979+0000", 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.378 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.044978+0000", 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.044985+0000", 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4279999999999999e-06 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.045004+0000", 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8505e-05 2023-10-09T09:43:41.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.045013+0000", 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.003e-06 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.045730+0000", 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00071757699999999995 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.380 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.045743+0000", 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2353e-05 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.025788+0000", 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.980045112 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.025961+0000", 2023-10-09T09:43:41.381 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017350500000000001 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,6", 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.026117+0000", 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015594099999999999 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.033944+0000", 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0078271360000000002 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.382 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.036850+0000", 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029058500000000002 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.039883+0000", 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030326089999999999 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.383 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.039987+0000", 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000104597 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.040030+0000", 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2929000000000002e-05 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.384 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:144861 9.c 9:36dae906:::benchmark_data_smithi163_175771_object40965:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1214)", 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:43:23.070959+0000", 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 18.150233790000001, 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96975672300000004, 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 144861 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:43:41.385 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.070959+0000", 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.070959+0000", 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:43:41.386 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.070958+0000", 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.070966+0000", 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2570000000000007e-06 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.070981+0000", 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4943000000000001e-05 2023-10-09T09:43:41.387 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.070987+0000", 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2729999999999999e-06 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.071033+0000", 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5976000000000002e-05 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:23.071041+0000", 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9149999999999996e-06 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.029992+0000", 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95895054999999996 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.030119+0000", 2023-10-09T09:43:41.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012687199999999999 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,6", 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.030243+0000", 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000124654 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.033950+0000", 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003706998 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.390 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.040312+0000", 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006361706 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.040587+0000", 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00027458499999999999 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:43:41.391 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.040677+0000", 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0791000000000005e-05 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:43:24.040716+0000", 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8753e-05 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:43:41.392 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:43:41.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:41.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:41.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:41.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:41.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:41.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:41.660 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:41.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:41.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:41.769 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:41.781 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:43:41.796 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:41.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:41.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:41.831 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:43:41.847 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:41 smithi163 bash[75425]: cluster 2023-10-09T09:43:40.026662+0000 mgr.y (mgr.34107) 1129 : cluster 0 pgmap v1941: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 778 op/s; 3273/154695 objects degraded (2.116%); 4008/154695 objects misplaced (2.591%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:41 smithi163 bash[75425]: audit 2023-10-09T09:43:41.611707+0000 mon.b (mon.1) 24 : audit 1 from='client.? 172.21.15.163:0/1165071204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:43:41.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:41 smithi163 bash[75425]: audit 2023-10-09T09:43:41.612407+0000 mon.a (mon.0) 2152 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:43:41.890 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:41 smithi163 bash[75425]: audit 2023-10-09T09:43:41.724651+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.163:0/1597482092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:43:41.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:41 smithi163 bash[78107]: cluster 2023-10-09T09:43:40.026662+0000 mgr.y (mgr.34107) 1129 : cluster 0 pgmap v1941: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 778 op/s; 3273/154695 objects degraded (2.116%); 4008/154695 objects misplaced (2.591%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T09:43:41.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:41 smithi163 bash[78107]: audit 2023-10-09T09:43:41.611707+0000 mon.b (mon.1) 24 : audit 1 from='client.? 172.21.15.163:0/1165071204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:43:41.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:41 smithi163 bash[78107]: audit 2023-10-09T09:43:41.612407+0000 mon.a (mon.0) 2152 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:43:41.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:41 smithi163 bash[78107]: audit 2023-10-09T09:43:41.724651+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.163:0/1597482092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:43:41.897 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:43:41.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:41.935 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:43:41.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:41.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:41.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:41.970 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:43:42.013 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:43:42.042 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:43:42.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:42.075 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:43:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:43:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:43:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:43:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:43:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:43:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:43:42.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:43:42.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:43:42.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:42.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:42.182 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:42.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:42.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:42.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:42.311 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:42.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:41 smithi181 bash[59345]: cluster 2023-10-09T09:43:40.026662+0000 mgr.y (mgr.34107) 1129 : cluster 0 pgmap v1941: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 778 op/s; 3273/154695 objects degraded (2.116%); 4008/154695 objects misplaced (2.591%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T09:43:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:41 smithi181 bash[59345]: audit 2023-10-09T09:43:41.611707+0000 mon.b (mon.1) 24 : audit 1 from='client.? 172.21.15.163:0/1165071204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:43:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:41 smithi181 bash[59345]: audit 2023-10-09T09:43:41.612407+0000 mon.a (mon.0) 2152 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:43:42.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:41 smithi181 bash[59345]: audit 2023-10-09T09:43:41.724651+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.163:0/1597482092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:43:42.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:42.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:42.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:42.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:42.473 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:42.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:42.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:42.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:42.599 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:42.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:42.704 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:42.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:42.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:42.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:42.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:42.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:42.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:42.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:42.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:42 smithi163 bash[78107]: cluster 2023-10-09T09:43:41.830526+0000 mon.a (mon.0) 2153 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:43:42.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:42 smithi163 bash[78107]: cluster 2023-10-09T09:43:41.838692+0000 osd.5 (osd.5) 800 : cluster 3 failed to encode map e1216 with expected crc 2023-10-09T09:43:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:42 smithi163 bash[78107]: cluster 2023-10-09T09:43:41.839908+0000 osd.6 (osd.6) 504 : cluster 3 failed to encode map e1216 with expected crc 2023-10-09T09:43:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:42 smithi163 bash[78107]: audit 2023-10-09T09:43:41.840020+0000 mon.a (mon.0) 2154 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:43:42.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:42 smithi163 bash[78107]: cluster 2023-10-09T09:43:41.840399+0000 mon.a (mon.0) 2155 : cluster 0 osdmap e1216: 8 total, 8 up, 6 in 2023-10-09T09:43:42.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:42 smithi163 bash[75425]: cluster 2023-10-09T09:43:41.830526+0000 mon.a (mon.0) 2153 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:43:42.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:42 smithi163 bash[75425]: cluster 2023-10-09T09:43:41.838692+0000 osd.5 (osd.5) 800 : cluster 3 failed to encode map e1216 with expected crc 2023-10-09T09:43:42.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:42 smithi163 bash[75425]: cluster 2023-10-09T09:43:41.839908+0000 osd.6 (osd.6) 504 : cluster 3 failed to encode map e1216 with expected crc 2023-10-09T09:43:42.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:42 smithi163 bash[75425]: audit 2023-10-09T09:43:41.840020+0000 mon.a (mon.0) 2154 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:43:42.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:42 smithi163 bash[75425]: cluster 2023-10-09T09:43:41.840399+0000 mon.a (mon.0) 2155 : cluster 0 osdmap e1216: 8 total, 8 up, 6 in 2023-10-09T09:43:42.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:43.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:43.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:43.106 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:43.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:42 smithi181 bash[59345]: cluster 2023-10-09T09:43:41.830526+0000 mon.a (mon.0) 2153 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:43:43.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:42 smithi181 bash[59345]: cluster 2023-10-09T09:43:41.838692+0000 osd.5 (osd.5) 800 : cluster 3 failed to encode map e1216 with expected crc 2023-10-09T09:43:43.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:42 smithi181 bash[59345]: cluster 2023-10-09T09:43:41.839908+0000 osd.6 (osd.6) 504 : cluster 3 failed to encode map e1216 with expected crc 2023-10-09T09:43:43.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:42 smithi181 bash[59345]: audit 2023-10-09T09:43:41.840020+0000 mon.a (mon.0) 2154 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:43:43.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:42 smithi181 bash[59345]: cluster 2023-10-09T09:43:41.840399+0000 mon.a (mon.0) 2155 : cluster 0 osdmap e1216: 8 total, 8 up, 6 in 2023-10-09T09:43:43.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:43.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:43.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:43.277 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:43.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:43.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:43.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:43.423 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:43.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:43.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:43.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:43:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:43:43] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.33.4" 2023-10-09T09:43:43.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:43.556 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:43.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:43.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:43.664 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:43.700 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:43.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:43.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:43.839 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:43.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:43.943 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:43.951 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:43 smithi181 bash[59345]: cluster 2023-10-09T09:43:42.027146+0000 mgr.y (mgr.34107) 1130 : cluster 0 pgmap v1943: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 931 op/s; 3273/144510 objects degraded (2.265%); 3740/144510 objects misplaced (2.588%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:43:43.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:43 smithi163 bash[78107]: cluster 2023-10-09T09:43:42.027146+0000 mgr.y (mgr.34107) 1130 : cluster 0 pgmap v1943: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 931 op/s; 3273/144510 objects degraded (2.265%); 3740/144510 objects misplaced (2.588%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:43:43.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:43 smithi163 bash[75425]: cluster 2023-10-09T09:43:42.027146+0000 mgr.y (mgr.34107) 1130 : cluster 0 pgmap v1943: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 931 op/s; 3273/144510 objects degraded (2.265%); 3740/144510 objects misplaced (2.588%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:43:43.971 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:43.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:43.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:44.074 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:44.102 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:44.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:44.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:44.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:44.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:44.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:44.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:44.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:44.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:44.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:44.470 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:44.496 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:44.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:44.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:44.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:44.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:44.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:44.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:44.745 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:44.775 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:44.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:44.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:44.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:44.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:44.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:44.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:45.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:45.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:45.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:45.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:45.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:43:45.161 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:45.198 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:45.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:45.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:45.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:45.326 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:45.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:45.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:45.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:45.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:45.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:45.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:45.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:45.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:45.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:45.701 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:45.728 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:45.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:45.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:45.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:45.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:45.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:45.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:45.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:45.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:45.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:46.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:45 smithi181 bash[59345]: cluster 2023-10-09T09:43:44.027747+0000 mgr.y (mgr.34107) 1131 : cluster 0 pgmap v1944: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 3273/140202 objects degraded (2.334%); 3740/140202 objects misplaced (2.668%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:43:46.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:45 smithi181 bash[59345]: cluster 2023-10-09T09:43:45.790059+0000 mon.a (mon.0) 2156 : cluster 3 Health check update: Degraded data redundancy: 3273/140202 objects degraded (2.334%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:46.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:46.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:46.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:46.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:46.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:45 smithi163 bash[78107]: cluster 2023-10-09T09:43:44.027747+0000 mgr.y (mgr.34107) 1131 : cluster 0 pgmap v1944: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 3273/140202 objects degraded (2.334%); 3740/140202 objects misplaced (2.668%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:43:46.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:45 smithi163 bash[78107]: cluster 2023-10-09T09:43:45.790059+0000 mon.a (mon.0) 2156 : cluster 3 Health check update: Degraded data redundancy: 3273/140202 objects degraded (2.334%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:46.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:45 smithi163 bash[75425]: cluster 2023-10-09T09:43:44.027747+0000 mgr.y (mgr.34107) 1131 : cluster 0 pgmap v1944: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 3273/140202 objects degraded (2.334%); 3740/140202 objects misplaced (2.668%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:43:46.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:45 smithi163 bash[75425]: cluster 2023-10-09T09:43:45.790059+0000 mon.a (mon.0) 2156 : cluster 3 Health check update: Degraded data redundancy: 3273/140202 objects degraded (2.334%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:46.235 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:46.266 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:46.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:46.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:46.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:46.399 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:46.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:46.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:46.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:46.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:46.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:46.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:46.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:46.662 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:46.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:46.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:46.766 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:46.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:46.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:46.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:46.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:46.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:46.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:47.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:47.066 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:47.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:47.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:47.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:47.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:43:47.196 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:43:47.207 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:47.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:47.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:47.311 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:47.345 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:47.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:47.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:47.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:47.477 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:47.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:47.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:47.582 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:47.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:47.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:47.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:47.717 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:47.747 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:47.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:47.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:47.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:47.884 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.0 primary-affinity to 0.637582 (8417842) 2023-10-09T09:43:47.885 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:47.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:47.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:47.990 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:47.999 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:47 smithi163 bash[78107]: cluster 2023-10-09T09:43:46.028445+0000 mgr.y (mgr.34107) 1132 : cluster 0 pgmap v1945: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 938 op/s; 3273/133293 objects degraded (2.455%); 3740/133293 objects misplaced (2.806%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:43:47.999 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:47 smithi163 bash[78107]: audit 2023-10-09T09:43:47.283053+0000 mon.a (mon.0) 2157 : audit 1 from='client.? 172.21.15.163:0/2064120897' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.637581602767268}]: dispatch 2023-10-09T09:43:47.999 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:47 smithi163 bash[75425]: cluster 2023-10-09T09:43:46.028445+0000 mgr.y (mgr.34107) 1132 : cluster 0 pgmap v1945: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 938 op/s; 3273/133293 objects degraded (2.455%); 3740/133293 objects misplaced (2.806%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:43:48.000 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:47 smithi163 bash[75425]: audit 2023-10-09T09:43:47.283053+0000 mon.a (mon.0) 2157 : audit 1 from='client.? 172.21.15.163:0/2064120897' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.637581602767268}]: dispatch 2023-10-09T09:43:48.015 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:48.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:48.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:48.134 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:47 smithi181 bash[59345]: cluster 2023-10-09T09:43:46.028445+0000 mgr.y (mgr.34107) 1132 : cluster 0 pgmap v1945: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 938 op/s; 3273/133293 objects degraded (2.455%); 3740/133293 objects misplaced (2.806%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T09:43:48.134 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:47 smithi181 bash[59345]: audit 2023-10-09T09:43:47.283053+0000 mon.a (mon.0) 2157 : audit 1 from='client.? 172.21.15.163:0/2064120897' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.637581602767268}]: dispatch 2023-10-09T09:43:48.151 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:48.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:48.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:48.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:48.281 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:48.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:48.385 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:48.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:48.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:48.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:48.517 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:48.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:48.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:48.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:48.684 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:48.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:48.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:48.788 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:48.816 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:48.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:48.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:48.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:48.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:48.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:48.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:49.053 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:48 smithi181 bash[59345]: cluster 2023-10-09T09:43:47.873262+0000 osd.5 (osd.5) 801 : cluster 3 failed to encode map e1217 with expected crc 2023-10-09T09:43:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:48 smithi181 bash[59345]: cluster 2023-10-09T09:43:47.874413+0000 osd.6 (osd.6) 505 : cluster 3 failed to encode map e1217 with expected crc 2023-10-09T09:43:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:48 smithi181 bash[59345]: audit 2023-10-09T09:43:47.876796+0000 mon.a (mon.0) 2158 : audit 1 from='client.? 172.21.15.163:0/2064120897' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.637581602767268}]': finished 2023-10-09T09:43:49.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:48 smithi181 bash[59345]: cluster 2023-10-09T09:43:47.876996+0000 mon.a (mon.0) 2159 : cluster 0 osdmap e1217: 8 total, 8 up, 6 in 2023-10-09T09:43:49.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:49.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:49.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:49.186 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:49.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:48 smithi163 bash[78107]: cluster 2023-10-09T09:43:47.873262+0000 osd.5 (osd.5) 801 : cluster 3 failed to encode map e1217 with expected crc 2023-10-09T09:43:49.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:48 smithi163 bash[78107]: cluster 2023-10-09T09:43:47.874413+0000 osd.6 (osd.6) 505 : cluster 3 failed to encode map e1217 with expected crc 2023-10-09T09:43:49.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:48 smithi163 bash[78107]: audit 2023-10-09T09:43:47.876796+0000 mon.a (mon.0) 2158 : audit 1 from='client.? 172.21.15.163:0/2064120897' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.637581602767268}]': finished 2023-10-09T09:43:49.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:48 smithi163 bash[78107]: cluster 2023-10-09T09:43:47.876996+0000 mon.a (mon.0) 2159 : cluster 0 osdmap e1217: 8 total, 8 up, 6 in 2023-10-09T09:43:49.195 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:48 smithi163 bash[75425]: cluster 2023-10-09T09:43:47.873262+0000 osd.5 (osd.5) 801 : cluster 3 failed to encode map e1217 with expected crc 2023-10-09T09:43:49.195 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:48 smithi163 bash[75425]: cluster 2023-10-09T09:43:47.874413+0000 osd.6 (osd.6) 505 : cluster 3 failed to encode map e1217 with expected crc 2023-10-09T09:43:49.195 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:48 smithi163 bash[75425]: audit 2023-10-09T09:43:47.876796+0000 mon.a (mon.0) 2158 : audit 1 from='client.? 172.21.15.163:0/2064120897' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.637581602767268}]': finished 2023-10-09T09:43:49.195 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:48 smithi163 bash[75425]: cluster 2023-10-09T09:43:47.876996+0000 mon.a (mon.0) 2159 : cluster 0 osdmap e1217: 8 total, 8 up, 6 in 2023-10-09T09:43:49.211 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:49.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:49.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:49.350 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:49.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:49.455 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:49.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:49.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:49.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:49.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:49.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:49.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:49.727 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:49.771 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:49.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:49.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:49.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:49.910 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:49.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:49.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:50.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:50.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[75425]: cluster 2023-10-09T09:43:48.028978+0000 mgr.y (mgr.34107) 1133 : cluster 0 pgmap v1947: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.19k op/s; 3273/126243 objects degraded (2.593%); 3473/126243 objects misplaced (2.751%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T09:43:50.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[75425]: cluster 2023-10-09T09:43:48.889429+0000 osd.5 (osd.5) 802 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[75425]: cluster 2023-10-09T09:43:48.892799+0000 mon.a (mon.0) 2160 : cluster 0 osdmap e1218: 8 total, 8 up, 6 in 2023-10-09T09:43:50.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[75425]: cluster 2023-10-09T09:43:48.892953+0000 osd.5 (osd.5) 803 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[75425]: cluster 2023-10-09T09:43:48.894004+0000 osd.6 (osd.6) 506 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[75425]: cluster 2023-10-09T09:43:48.897060+0000 osd.4 (osd.4) 629 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[75425]: cluster 2023-10-09T09:43:49.866568+0000 osd.7 (osd.7) 583 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[75425]: cluster 2023-10-09T09:43:49.868689+0000 osd.7 (osd.7) 584 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[78107]: cluster 2023-10-09T09:43:48.028978+0000 mgr.y (mgr.34107) 1133 : cluster 0 pgmap v1947: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.19k op/s; 3273/126243 objects degraded (2.593%); 3473/126243 objects misplaced (2.751%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T09:43:50.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[78107]: cluster 2023-10-09T09:43:48.889429+0000 osd.5 (osd.5) 802 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[78107]: cluster 2023-10-09T09:43:48.892799+0000 mon.a (mon.0) 2160 : cluster 0 osdmap e1218: 8 total, 8 up, 6 in 2023-10-09T09:43:50.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[78107]: cluster 2023-10-09T09:43:48.892953+0000 osd.5 (osd.5) 803 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[78107]: cluster 2023-10-09T09:43:48.894004+0000 osd.6 (osd.6) 506 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[78107]: cluster 2023-10-09T09:43:48.897060+0000 osd.4 (osd.4) 629 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[78107]: cluster 2023-10-09T09:43:49.866568+0000 osd.7 (osd.7) 583 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:49 smithi163 bash[78107]: cluster 2023-10-09T09:43:49.868689+0000 osd.7 (osd.7) 584 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.044 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:50.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:50.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:50.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:50.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:50.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:50.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:50.298 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:49 smithi181 bash[59345]: cluster 2023-10-09T09:43:48.028978+0000 mgr.y (mgr.34107) 1133 : cluster 0 pgmap v1947: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.19k op/s; 3273/126243 objects degraded (2.593%); 3473/126243 objects misplaced (2.751%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T09:43:50.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:49 smithi181 bash[59345]: cluster 2023-10-09T09:43:48.889429+0000 osd.5 (osd.5) 802 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:49 smithi181 bash[59345]: cluster 2023-10-09T09:43:48.892799+0000 mon.a (mon.0) 2160 : cluster 0 osdmap e1218: 8 total, 8 up, 6 in 2023-10-09T09:43:50.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:49 smithi181 bash[59345]: cluster 2023-10-09T09:43:48.892953+0000 osd.5 (osd.5) 803 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:49 smithi181 bash[59345]: cluster 2023-10-09T09:43:48.894004+0000 osd.6 (osd.6) 506 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:49 smithi181 bash[59345]: cluster 2023-10-09T09:43:48.897060+0000 osd.4 (osd.4) 629 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:49 smithi181 bash[59345]: cluster 2023-10-09T09:43:49.866568+0000 osd.7 (osd.7) 583 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:49 smithi181 bash[59345]: cluster 2023-10-09T09:43:49.868689+0000 osd.7 (osd.7) 584 : cluster 3 failed to encode map e1218 with expected crc 2023-10-09T09:43:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:50.314 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:50.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:50.415 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:50.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:50.450 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:50.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:50.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:50.577 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:50.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:50.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:50.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:50.705 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:50.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:50.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:50.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:50.839 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:50.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:50.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:50.892 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:50 smithi163 bash[75425]: cluster 2023-10-09T09:43:50.791660+0000 mon.a (mon.0) 2161 : cluster 3 Health check update: Degraded data redundancy: 3273/126243 objects degraded (2.593%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:50.943 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:50.951 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:50 smithi181 bash[59345]: cluster 2023-10-09T09:43:50.791660+0000 mon.a (mon.0) 2161 : cluster 3 Health check update: Degraded data redundancy: 3273/126243 objects degraded (2.593%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:50.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:50.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:50.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:51.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:51.110 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:51.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:51.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:51.213 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:51.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:50 smithi163 bash[78107]: cluster 2023-10-09T09:43:50.791660+0000 mon.a (mon.0) 2161 : cluster 3 Health check update: Degraded data redundancy: 3273/126243 objects degraded (2.593%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:51.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:51.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:51.365 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:51.397 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:51.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:51.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:51.502 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:51.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:51.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:51.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:51.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:51.663 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:51.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:51.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:51.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:51.794 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:51.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:51.854 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:43:51.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:51 smithi163 bash[75425]: cluster 2023-10-09T09:43:50.029462+0000 mgr.y (mgr.34107) 1134 : cluster 0 pgmap v1949: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 760 op/s; 3273/126243 objects degraded (2.593%); 3473/126243 objects misplaced (2.751%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:43:51.900 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:51.926 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:51.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:51.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:52.031 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:52.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:52.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:52.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:52.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:51 smithi163 bash[78107]: cluster 2023-10-09T09:43:50.029462+0000 mgr.y (mgr.34107) 1134 : cluster 0 pgmap v1949: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 760 op/s; 3273/126243 objects degraded (2.593%); 3473/126243 objects misplaced (2.751%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:43:52.197 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:52.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:52.300 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:51 smithi181 bash[59345]: cluster 2023-10-09T09:43:50.029462+0000 mgr.y (mgr.34107) 1134 : cluster 0 pgmap v1949: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 760 op/s; 3273/126243 objects degraded (2.593%); 3473/126243 objects misplaced (2.751%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T09:43:52.336 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:52.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:52.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:52.473 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:52.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:52.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:52.577 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:52.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:52.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:52.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:52.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:52.738 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:52.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:52.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:52.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:52.878 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:43:52.879 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:43:52.883 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:52.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:52.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:53.014 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:53.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:53.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:53.121 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:53.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:53.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:53.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:53.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:53.283 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:53.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:53.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:53.387 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:53.418 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:53.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:53.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:53.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:43:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:43:53] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.33.4" 2023-10-09T09:43:53.522 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:53.550 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:53.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:53.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:53.656 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:53.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:53.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:53.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:53.822 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:53.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:53.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:53.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:53.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:53.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:53.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:54.060 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:54.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:53 smithi181 bash[59345]: cluster 2023-10-09T09:43:52.030461+0000 mgr.y (mgr.34107) 1135 : cluster 0 pgmap v1950: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 936 op/s; 3273/117714 objects degraded (2.780%); 3147/117714 objects misplaced (2.673%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:43:54.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:54.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:54.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:54.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:54.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:54.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:54.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:54.271 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:53 smithi163 bash[75425]: cluster 2023-10-09T09:43:52.030461+0000 mgr.y (mgr.34107) 1135 : cluster 0 pgmap v1950: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 936 op/s; 3273/117714 objects degraded (2.780%); 3147/117714 objects misplaced (2.673%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:43:54.271 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:53 smithi163 bash[78107]: cluster 2023-10-09T09:43:52.030461+0000 mgr.y (mgr.34107) 1135 : cluster 0 pgmap v1950: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 936 op/s; 3273/117714 objects degraded (2.780%); 3147/117714 objects misplaced (2.673%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:43:54.321 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:54.334 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 6, 2, 5] out_osds: [7, 3] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:43:54.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T09:43:54.344 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:54.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:54.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:54.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:54.480 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:54.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:54.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:54.625 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:54.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:54.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:54.730 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:54.757 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:54.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:54.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:54.891 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:54.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:54.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:54.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:55.024 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:55.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:55.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:55.132 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:55.163 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:55.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:55.268 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:55.300 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:55.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:55.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:55.403 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:55.433 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:55.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:55.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:55.558 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:55.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:55.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:55.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:55.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:55.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:55.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:55.827 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:55.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:55.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:55.934 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:55.943 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:55 smithi181 bash[59345]: cluster 2023-10-09T09:43:54.031077+0000 mgr.y (mgr.34107) 1136 : cluster 0 pgmap v1951: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 737 op/s; 3273/115593 objects degraded (2.831%); 3147/115593 objects misplaced (2.722%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:43:55.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:55 smithi181 bash[59345]: audit 2023-10-09T09:43:55.392595+0000 mon.a (mon.0) 2162 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:55.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:55 smithi181 bash[59345]: audit 2023-10-09T09:43:55.395069+0000 mon.a (mon.0) 2163 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:55.944 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:55 smithi181 bash[59345]: cluster 2023-10-09T09:43:55.793239+0000 mon.a (mon.0) 2164 : cluster 3 Health check update: Degraded data redundancy: 3273/115593 objects degraded (2.831%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:55.959 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:55.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:55.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:56.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:56.072 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:55 smithi163 bash[78107]: cluster 2023-10-09T09:43:54.031077+0000 mgr.y (mgr.34107) 1136 : cluster 0 pgmap v1951: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 737 op/s; 3273/115593 objects degraded (2.831%); 3147/115593 objects misplaced (2.722%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:43:56.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:55 smithi163 bash[78107]: audit 2023-10-09T09:43:55.392595+0000 mon.a (mon.0) 2162 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:56.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:55 smithi163 bash[78107]: audit 2023-10-09T09:43:55.395069+0000 mon.a (mon.0) 2163 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:56.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:55 smithi163 bash[78107]: cluster 2023-10-09T09:43:55.793239+0000 mon.a (mon.0) 2164 : cluster 3 Health check update: Degraded data redundancy: 3273/115593 objects degraded (2.831%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:56.073 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:55 smithi163 bash[75425]: cluster 2023-10-09T09:43:54.031077+0000 mgr.y (mgr.34107) 1136 : cluster 0 pgmap v1951: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 737 op/s; 3273/115593 objects degraded (2.831%); 3147/115593 objects misplaced (2.722%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T09:43:56.073 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:55 smithi163 bash[75425]: audit 2023-10-09T09:43:55.392595+0000 mon.a (mon.0) 2162 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:43:56.074 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:55 smithi163 bash[75425]: audit 2023-10-09T09:43:55.395069+0000 mon.a (mon.0) 2163 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:43:56.074 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:55 smithi163 bash[75425]: cluster 2023-10-09T09:43:55.793239+0000 mon.a (mon.0) 2164 : cluster 3 Health check update: Degraded data redundancy: 3273/115593 objects degraded (2.831%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:43:56.102 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:43:56.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:56.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:43:56.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:56.240 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:56.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:56.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:56.343 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:56.372 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:56.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:56.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:56.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:56.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:56.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:56.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:56.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:56.636 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:56.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:56.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:56.744 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:56.777 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:56.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:56.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:56.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:56.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:56.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:56.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:57.023 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:57.054 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:43:57.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:57.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:43:57.161 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:57.191 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:57.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:57.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:57.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:57.325 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:57.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:57.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:57.432 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:57.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:57.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:57.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:57.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:57.603 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:57.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:57.709 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:57.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:57.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:57.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:57.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:57.880 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:57.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:57.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:57.983 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:58.007 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:58.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:58.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:58.111 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:58.137 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:58.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:58.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:57 smithi163 bash[78107]: cluster 2023-10-09T09:43:56.031947+0000 mgr.y (mgr.34107) 1137 : cluster 0 pgmap v1952: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 683 op/s; 3273/109512 objects degraded (2.989%); 3147/109512 objects misplaced (2.874%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T09:43:58.199 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:57 smithi163 bash[75425]: cluster 2023-10-09T09:43:56.031947+0000 mgr.y (mgr.34107) 1137 : cluster 0 pgmap v1952: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 683 op/s; 3273/109512 objects degraded (2.989%); 3147/109512 objects misplaced (2.874%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T09:43:58.242 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:58.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:58.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:58.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:57 smithi181 bash[59345]: cluster 2023-10-09T09:43:56.031947+0000 mgr.y (mgr.34107) 1137 : cluster 0 pgmap v1952: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 683 op/s; 3273/109512 objects degraded (2.989%); 3147/109512 objects misplaced (2.874%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T09:43:58.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:58.403 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:58.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:58.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:58.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:58.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:58.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:58.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:58.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:58.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:58.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:58.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:58.788 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:58.822 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:43:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:58.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:43:58.929 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:58.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:58.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:58.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:59.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:59.107 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:43:59.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:59.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:43:59.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:59.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:59.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:59.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:43:59.352 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:59.382 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:59.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:59.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:59.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:59.519 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:43:59.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:59.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:43:59.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:59.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:43:59.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:59.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:43:59.764 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:59.792 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:43:59.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:59.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:43:59.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:59.920 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:43:59.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:43:59.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:00.023 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:00.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:00.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:00.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:00.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:43:59 smithi163 bash[78107]: cluster 2023-10-09T09:43:58.032752+0000 mgr.y (mgr.34107) 1138 : cluster 0 pgmap v1953: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 3273/101871 objects degraded (3.213%); 2870/101871 objects misplaced (2.817%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:44:00.160 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:43:59 smithi163 bash[75425]: cluster 2023-10-09T09:43:58.032752+0000 mgr.y (mgr.34107) 1138 : cluster 0 pgmap v1953: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 3273/101871 objects degraded (3.213%); 2870/101871 objects misplaced (2.817%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:44:00.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:00.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:00.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:00.290 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:43:59 smithi181 bash[59345]: cluster 2023-10-09T09:43:58.032752+0000 mgr.y (mgr.34107) 1138 : cluster 0 pgmap v1953: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 3273/101871 objects degraded (3.213%); 2870/101871 objects misplaced (2.817%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T09:44:00.313 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:00.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:00.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:00.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:00.446 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:00.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:00.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:00.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:00.576 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:00.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:00.679 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:00.713 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:00.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:00.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:00.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:00.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:00.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:00.952 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:00.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:00 smithi181 bash[59345]: cluster 2023-10-09T09:44:00.794249+0000 mon.a (mon.0) 2165 : cluster 3 Health check update: Degraded data redundancy: 3273/101871 objects degraded (3.213%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:44:00.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:00.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:01.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:01.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:01.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:01.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:01.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:00 smithi163 bash[75425]: cluster 2023-10-09T09:44:00.794249+0000 mon.a (mon.0) 2165 : cluster 3 Health check update: Degraded data redundancy: 3273/101871 objects degraded (3.213%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:44:01.182 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:00 smithi163 bash[78107]: cluster 2023-10-09T09:44:00.794249+0000 mon.a (mon.0) 2165 : cluster 3 Health check update: Degraded data redundancy: 3273/101871 objects degraded (3.213%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:44:01.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:01.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:01.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:01.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:01.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:01.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:01.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:01.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:01.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:01.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:01.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:01.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:01.634 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:01.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:01.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:01.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:01.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:01.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:01.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:01.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:01.894 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:01.928 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:01 smithi181 bash[59345]: cluster 2023-10-09T09:44:00.033305+0000 mgr.y (mgr.34107) 1139 : cluster 0 pgmap v1954: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 3273/101871 objects degraded (3.213%); 2870/101871 objects misplaced (2.817%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:44:01.937 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:01.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:01.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:02.042 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:02.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:02.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:02.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:02.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:01 smithi163 bash[75425]: cluster 2023-10-09T09:44:00.033305+0000 mgr.y (mgr.34107) 1139 : cluster 0 pgmap v1954: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 3273/101871 objects degraded (3.213%); 2870/101871 objects misplaced (2.817%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:44:02.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:01 smithi163 bash[78107]: cluster 2023-10-09T09:44:00.033305+0000 mgr.y (mgr.34107) 1139 : cluster 0 pgmap v1954: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 3273/101871 objects degraded (3.213%); 2870/101871 objects misplaced (2.817%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T09:44:02.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:02.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:02.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:02.306 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:02.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:02.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:02.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:02.479 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:02.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:02.580 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:02.605 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:02.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:02.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:02.712 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:02.741 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:02.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:02.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:02.872 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:02.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:02.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:02.974 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:03.009 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:03.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:03.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:03.113 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:03.137 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:03.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:03.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:03.244 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:03.272 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:03.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:03.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:03.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:03.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:03.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:03.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:03.491 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:44:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:44:03] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.33.4" 2023-10-09T09:44:03.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:03.546 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:03.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:03.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:03.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:03.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:03.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:03.783 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:03.814 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:03.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:03.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:03.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:03.944 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:03.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:04.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:04.083 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:04.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:04.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:04.192 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:04.201 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:03 smithi163 bash[75425]: cluster 2023-10-09T09:44:02.034016+0000 mgr.y (mgr.34107) 1140 : cluster 0 pgmap v1955: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 985 op/s; 3273/90744 objects degraded (3.607%); 2672/90744 objects misplaced (2.945%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T09:44:04.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:03 smithi163 bash[78107]: cluster 2023-10-09T09:44:02.034016+0000 mgr.y (mgr.34107) 1140 : cluster 0 pgmap v1955: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 985 op/s; 3273/90744 objects degraded (3.607%); 2672/90744 objects misplaced (2.945%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T09:44:04.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:04.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:04.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:04.281 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:44:04.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:03 smithi181 bash[59345]: cluster 2023-10-09T09:44:02.034016+0000 mgr.y (mgr.34107) 1140 : cluster 0 pgmap v1955: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 985 op/s; 3273/90744 objects degraded (3.607%); 2672/90744 objects misplaced (2.945%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T09:44:04.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:04.357 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:04.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:04.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:04.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:04.490 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:04.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:04.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:04.595 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:04.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:04.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:04.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:04.756 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:04.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:04.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:04.927 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:04.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:04.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:04.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:05.066 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:05.096 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:05.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:05.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:05.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:05.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:05.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:05.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:05.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:05.361 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:05.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:05.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:05.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:05.481 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:44:05.481 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:44:05.500 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:05.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:05.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:05.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:05.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:05.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:05.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:05.735 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:05.771 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:05.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:05.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:05.875 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:05.903 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:05.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:05.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:05.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:05 smithi163 bash[78107]: cluster 2023-10-09T09:44:04.034710+0000 mgr.y (mgr.34107) 1141 : cluster 0 pgmap v1956: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 820 op/s; 3273/88170 objects degraded (3.712%); 2672/88170 objects misplaced (3.031%); 2.5 MiB/s, 51 objects/s recovering 2023-10-09T09:44:05.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:05 smithi163 bash[78107]: cluster 2023-10-09T09:44:05.795326+0000 mon.a (mon.0) 2166 : cluster 3 Health check update: Degraded data redundancy: 3273/88170 objects degraded (3.712%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:05.938 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:05 smithi163 bash[75425]: cluster 2023-10-09T09:44:04.034710+0000 mgr.y (mgr.34107) 1141 : cluster 0 pgmap v1956: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 820 op/s; 3273/88170 objects degraded (3.712%); 2672/88170 objects misplaced (3.031%); 2.5 MiB/s, 51 objects/s recovering 2023-10-09T09:44:05.938 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:05 smithi163 bash[75425]: cluster 2023-10-09T09:44:05.795326+0000 mon.a (mon.0) 2166 : cluster 3 Health check update: Degraded data redundancy: 3273/88170 objects degraded (3.712%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:06.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:06.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:06.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:06.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:06.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:06.156 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:06.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:06.259 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:06.295 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:06.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:06.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:05 smithi181 bash[59345]: cluster 2023-10-09T09:44:04.034710+0000 mgr.y (mgr.34107) 1141 : cluster 0 pgmap v1956: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 820 op/s; 3273/88170 objects degraded (3.712%); 2672/88170 objects misplaced (3.031%); 2.5 MiB/s, 51 objects/s recovering 2023-10-09T09:44:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:05 smithi181 bash[59345]: cluster 2023-10-09T09:44:05.795326+0000 mon.a (mon.0) 2166 : cluster 3 Health check update: Degraded data redundancy: 3273/88170 objects degraded (3.712%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:06.401 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:06.427 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:06.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:06.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:06.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:06.580 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:06.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:06.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:06.688 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:06.714 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:06.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:06.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:06.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:06.843 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:06.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:06.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:06.949 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:06.973 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:06.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:06.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:07.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:07.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:07.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:07.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:07.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:07.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:07.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:07.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:07.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:07.352 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:44:07.373 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:07.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:07.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:07.390 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:44:07.423 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:44:07.452 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:44:07.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:07.495 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:44:07.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:07.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:07.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:07.538 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:44:07.565 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:44:07.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:44:07.592 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:44:07.593 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:44:07.593 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:44:07.593 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:44:07.593 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:44:07.593 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:44:07.593 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:44:07.593 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:44:07.601 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:07.627 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:07.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:07.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:07.731 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:07.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:07.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:07.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:07.860 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:07.886 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:07.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:07.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:07.990 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:08.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:08.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:08.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:08.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:08.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:07 smithi163 bash[75425]: cluster 2023-10-09T09:44:06.035247+0000 mgr.y (mgr.34107) 1142 : cluster 0 pgmap v1957: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 938 op/s; 3273/81807 objects degraded (4.001%); 2672/81807 objects misplaced (3.266%); 2.5 MiB/s, 51 objects/s recovering 2023-10-09T09:44:08.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:07 smithi163 bash[75425]: audit 2023-10-09T09:44:07.325680+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.163:0/143436043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:44:08.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:07 smithi163 bash[78107]: cluster 2023-10-09T09:44:06.035247+0000 mgr.y (mgr.34107) 1142 : cluster 0 pgmap v1957: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 938 op/s; 3273/81807 objects degraded (4.001%); 2672/81807 objects misplaced (3.266%); 2.5 MiB/s, 51 objects/s recovering 2023-10-09T09:44:08.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:07 smithi163 bash[78107]: audit 2023-10-09T09:44:07.325680+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.163:0/143436043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:44:08.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:08.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:08.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:08.258 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:08.313 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:08.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:08.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:08.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:07 smithi181 bash[59345]: cluster 2023-10-09T09:44:06.035247+0000 mgr.y (mgr.34107) 1142 : cluster 0 pgmap v1957: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 938 op/s; 3273/81807 objects degraded (4.001%); 2672/81807 objects misplaced (3.266%); 2.5 MiB/s, 51 objects/s recovering 2023-10-09T09:44:08.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:07 smithi181 bash[59345]: audit 2023-10-09T09:44:07.325680+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.163:0/143436043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:44:08.417 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:08.446 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:08.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:08.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:08.584 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:08.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:08.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:08.688 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:08.711 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:08.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:08.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:08.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:08.840 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:08.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:08.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:08.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:08.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:08.975 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:08.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:09.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:09.104 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:09.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:09.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:09.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:09.237 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:09.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:09.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:09.341 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:09.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:09.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:09.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:09.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:09.496 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:09.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:09.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:09.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:09.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:09.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:09.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:09.739 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:09.771 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:09.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:09.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:09.875 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:09.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:09.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:09.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:09.956 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:44:10.003 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:09 smithi163 bash[78107]: cluster 2023-10-09T09:44:08.035956+0000 mgr.y (mgr.34107) 1143 : cluster 0 pgmap v1958: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 980 op/s; 3273/74205 objects degraded (4.411%); 2672/74205 objects misplaced (3.601%); 2.5 MiB/s, 75 objects/s recovering 2023-10-09T09:44:10.003 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:09 smithi163 bash[78107]: audit 2023-10-09T09:44:09.598045+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.163:0/2542761248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:44:10.003 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:09 smithi163 bash[78107]: audit 2023-10-09T09:44:09.598608+0000 mon.a (mon.0) 2167 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:44:10.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:09 smithi163 bash[75425]: cluster 2023-10-09T09:44:08.035956+0000 mgr.y (mgr.34107) 1143 : cluster 0 pgmap v1958: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 980 op/s; 3273/74205 objects degraded (4.411%); 2672/74205 objects misplaced (3.601%); 2.5 MiB/s, 75 objects/s recovering 2023-10-09T09:44:10.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:09 smithi163 bash[75425]: audit 2023-10-09T09:44:09.598045+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.163:0/2542761248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:44:10.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:09 smithi163 bash[75425]: audit 2023-10-09T09:44:09.598608+0000 mon.a (mon.0) 2167 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:44:10.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:10.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:10.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:10.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:10.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:10.149 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:09 smithi181 bash[59345]: cluster 2023-10-09T09:44:08.035956+0000 mgr.y (mgr.34107) 1143 : cluster 0 pgmap v1958: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 980 op/s; 3273/74205 objects degraded (4.411%); 2672/74205 objects misplaced (3.601%); 2.5 MiB/s, 75 objects/s recovering 2023-10-09T09:44:10.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:09 smithi181 bash[59345]: audit 2023-10-09T09:44:09.598045+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.163:0/2542761248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:44:10.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:09 smithi181 bash[59345]: audit 2023-10-09T09:44:09.598608+0000 mon.a (mon.0) 2167 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:44:10.170 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:10.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:10.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:10.274 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:10.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:10.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:10.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:10.399 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:10.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:10.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:10.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:10.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:10.564 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:10.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:10.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:10.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:10.694 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:10.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:10.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:10.731 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T09:44:10.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:10.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:10.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:10.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:10.951 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:10.987 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:10.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:10.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:11.091 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:11.101 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[75425]: cluster 2023-10-09T09:44:09.938609+0000 mon.a (mon.0) 2168 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:44:11.101 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[75425]: cluster 2023-10-09T09:44:09.941427+0000 osd.5 (osd.5) 804 : cluster 3 failed to encode map e1219 with expected crc 2023-10-09T09:44:11.102 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[75425]: cluster 2023-10-09T09:44:09.945249+0000 osd.7 (osd.7) 585 : cluster 3 failed to encode map e1219 with expected crc 2023-10-09T09:44:11.102 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[75425]: audit 2023-10-09T09:44:09.949903+0000 mon.a (mon.0) 2169 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:44:11.102 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[75425]: cluster 2023-10-09T09:44:09.950141+0000 mon.a (mon.0) 2170 : cluster 0 osdmap e1219: 8 total, 8 up, 6 in 2023-10-09T09:44:11.102 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[75425]: audit 2023-10-09T09:44:10.389397+0000 mon.a (mon.0) 2171 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:11.102 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[75425]: cluster 2023-10-09T09:44:10.796111+0000 mon.a (mon.0) 2172 : cluster 3 Health check update: Degraded data redundancy: 3273/74205 objects degraded (4.411%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:11.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[78107]: cluster 2023-10-09T09:44:09.938609+0000 mon.a (mon.0) 2168 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:44:11.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[78107]: cluster 2023-10-09T09:44:09.941427+0000 osd.5 (osd.5) 804 : cluster 3 failed to encode map e1219 with expected crc 2023-10-09T09:44:11.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[78107]: cluster 2023-10-09T09:44:09.945249+0000 osd.7 (osd.7) 585 : cluster 3 failed to encode map e1219 with expected crc 2023-10-09T09:44:11.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[78107]: audit 2023-10-09T09:44:09.949903+0000 mon.a (mon.0) 2169 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:44:11.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[78107]: cluster 2023-10-09T09:44:09.950141+0000 mon.a (mon.0) 2170 : cluster 0 osdmap e1219: 8 total, 8 up, 6 in 2023-10-09T09:44:11.103 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[78107]: audit 2023-10-09T09:44:10.389397+0000 mon.a (mon.0) 2171 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:11.103 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:10 smithi163 bash[78107]: cluster 2023-10-09T09:44:10.796111+0000 mon.a (mon.0) 2172 : cluster 3 Health check update: Degraded data redundancy: 3273/74205 objects degraded (4.411%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:11.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:11.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:11.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:11.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:11.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:11.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:11.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:10 smithi181 bash[59345]: cluster 2023-10-09T09:44:09.938609+0000 mon.a (mon.0) 2168 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:44:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:10 smithi181 bash[59345]: cluster 2023-10-09T09:44:09.941427+0000 osd.5 (osd.5) 804 : cluster 3 failed to encode map e1219 with expected crc 2023-10-09T09:44:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:10 smithi181 bash[59345]: cluster 2023-10-09T09:44:09.945249+0000 osd.7 (osd.7) 585 : cluster 3 failed to encode map e1219 with expected crc 2023-10-09T09:44:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:10 smithi181 bash[59345]: audit 2023-10-09T09:44:09.949903+0000 mon.a (mon.0) 2169 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:44:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:10 smithi181 bash[59345]: cluster 2023-10-09T09:44:09.950141+0000 mon.a (mon.0) 2170 : cluster 0 osdmap e1219: 8 total, 8 up, 6 in 2023-10-09T09:44:11.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:10 smithi181 bash[59345]: audit 2023-10-09T09:44:10.389397+0000 mon.a (mon.0) 2171 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:11.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:10 smithi181 bash[59345]: cluster 2023-10-09T09:44:10.796111+0000 mon.a (mon.0) 2172 : cluster 3 Health check update: Degraded data redundancy: 3273/74205 objects degraded (4.411%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:11.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:11.404 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:11.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:11.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:11.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:11.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:11.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:11.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:11.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:11.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:11.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:11.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:11.797 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:11.842 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:11.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:11.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:11.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:11.973 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:11.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:11.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:12.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:11 smithi181 bash[59345]: cluster 2023-10-09T09:44:10.036453+0000 mgr.y (mgr.34107) 1144 : cluster 0 pgmap v1960: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 921 op/s; 3273/74205 objects degraded (4.411%); 2672/74205 objects misplaced (3.601%); 1.2 MiB/s, 63 objects/s recovering 2023-10-09T09:44:12.076 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:12.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:12.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:12.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:12.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:12.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:12 smithi163 bash[75425]: cluster 2023-10-09T09:44:10.036453+0000 mgr.y (mgr.34107) 1144 : cluster 0 pgmap v1960: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 921 op/s; 3273/74205 objects degraded (4.411%); 2672/74205 objects misplaced (3.601%); 1.2 MiB/s, 63 objects/s recovering 2023-10-09T09:44:12.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:12 smithi163 bash[78107]: cluster 2023-10-09T09:44:10.036453+0000 mgr.y (mgr.34107) 1144 : cluster 0 pgmap v1960: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 921 op/s; 3273/74205 objects degraded (4.411%); 2672/74205 objects misplaced (3.601%); 1.2 MiB/s, 63 objects/s recovering 2023-10-09T09:44:12.230 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:12.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:12.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:12.334 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:12.365 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:12.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:12.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:12.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:12.487 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:12.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:12.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:12.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:12.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:12.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:12.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:12.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:12.751 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:12.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:12.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:12.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:44:12.858 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:12.884 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:12.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:12.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:13.032 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:13.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:13.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:13.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:13.163 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:13.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:13.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:13.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:13.287 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:13.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:13.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:13.390 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:13.424 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:13.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:13.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:13.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:13.559 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:13.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:13.665 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:13.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:13.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:13.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:44:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:44:13] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.33.4" 2023-10-09T09:44:13.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:13.823 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:13.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:13.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:13.927 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:13.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:13.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:13.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:14.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:14.098 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:14.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:14.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:14.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:14.231 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:14.233 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:13 smithi181 bash[59345]: cluster 2023-10-09T09:44:12.037207+0000 mgr.y (mgr.34107) 1145 : cluster 0 pgmap v1961: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 912 op/s; 3183/63372 objects degraded (5.023%); 2672/63372 objects misplaced (4.216%); 0 B/s, 62 objects/s recovering 2023-10-09T09:44:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:14.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:14.334 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:14.361 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:14.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:14.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:14.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:14 smithi163 bash[78107]: cluster 2023-10-09T09:44:12.037207+0000 mgr.y (mgr.34107) 1145 : cluster 0 pgmap v1961: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 912 op/s; 3183/63372 objects degraded (5.023%); 2672/63372 objects misplaced (4.216%); 0 B/s, 62 objects/s recovering 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:14 smithi163 bash[75425]: cluster 2023-10-09T09:44:12.037207+0000 mgr.y (mgr.34107) 1145 : cluster 0 pgmap v1961: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 912 op/s; 3183/63372 objects degraded (5.023%); 2672/63372 objects misplaced (4.216%); 0 B/s, 62 objects/s recovering 2023-10-09T09:44:14.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:14.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:14.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:14.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:14.625 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:14.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:14.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:14.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:14.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:44:14.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:14.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:14.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:14.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:14.888 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:14.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:14.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:14.992 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:15.021 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:15.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:15.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:15.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:15.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:15.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:15.262 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:15.286 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:15.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:15.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:15.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:15.421 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:15.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:15.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:15.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:15.559 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:15.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:15.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:15.694 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:15.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:15.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:15.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:15.832 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:15.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:15.940 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:15.973 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:15.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:16.077 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:16.102 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:16.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:16.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:16.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:16.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:15 smithi163 bash[78107]: cluster 2023-10-09T09:44:14.037767+0000 mgr.y (mgr.34107) 1146 : cluster 0 pgmap v1962: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 914 op/s; 3183/60723 objects degraded (5.242%); 2672/60723 objects misplaced (4.400%); 0 B/s, 62 objects/s recovering 2023-10-09T09:44:16.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:15 smithi163 bash[78107]: cluster 2023-10-09T09:44:15.800421+0000 mon.a (mon.0) 2173 : cluster 3 Health check update: Degraded data redundancy: 3183/60723 objects degraded (5.242%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:16.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:15 smithi163 bash[75425]: cluster 2023-10-09T09:44:14.037767+0000 mgr.y (mgr.34107) 1146 : cluster 0 pgmap v1962: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 914 op/s; 3183/60723 objects degraded (5.242%); 2672/60723 objects misplaced (4.400%); 0 B/s, 62 objects/s recovering 2023-10-09T09:44:16.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:15 smithi163 bash[75425]: cluster 2023-10-09T09:44:15.800421+0000 mon.a (mon.0) 2173 : cluster 3 Health check update: Degraded data redundancy: 3183/60723 objects degraded (5.242%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:16.236 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:16.237 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:15 smithi181 bash[59345]: cluster 2023-10-09T09:44:14.037767+0000 mgr.y (mgr.34107) 1146 : cluster 0 pgmap v1962: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 914 op/s; 3183/60723 objects degraded (5.242%); 2672/60723 objects misplaced (4.400%); 0 B/s, 62 objects/s recovering 2023-10-09T09:44:16.237 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:15 smithi181 bash[59345]: cluster 2023-10-09T09:44:15.800421+0000 mon.a (mon.0) 2173 : cluster 3 Health check update: Degraded data redundancy: 3183/60723 objects degraded (5.242%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:16.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:16.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:16.363 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:16.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:16.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:16.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:16.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:16.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:16.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:16.607 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:16.639 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:16.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:16.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:16.746 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:16.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:16.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:16.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:16.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:16.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:16.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:16.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:17.000 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T09:44:17.016 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:17.024 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:16 smithi163 bash[78107]: audit 2023-10-09T09:44:16.161682+0000 mon.a (mon.0) 2174 : audit 1 from='client.? 172.21.15.163:0/3877180870' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:44:17.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:16 smithi163 bash[75425]: audit 2023-10-09T09:44:16.161682+0000 mon.a (mon.0) 2174 : audit 1 from='client.? 172.21.15.163:0/3877180870' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:44:17.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:17.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:17.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:17.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:17.170 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:17.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:17.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:17.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:17.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:17.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:17.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:17.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:16 smithi181 bash[59345]: audit 2023-10-09T09:44:16.161682+0000 mon.a (mon.0) 2174 : audit 1 from='client.? 172.21.15.163:0/3877180870' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:44:17.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:17.436 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:17.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:17.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:17.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:17.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:17.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:17.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:17.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:17.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:17.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:17.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:17.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:17.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:17.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:17.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:17.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:17.966 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:17.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:17.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:18.071 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:18.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:17 smithi181 bash[59345]: cluster 2023-10-09T09:44:16.038387+0000 mgr.y (mgr.34107) 1147 : cluster 0 pgmap v1963: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 896 op/s; 3183/54894 objects degraded (5.798%); 2672/54894 objects misplaced (4.868%); 0 B/s, 62 objects/s recovering 2023-10-09T09:44:18.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:17 smithi181 bash[59345]: cluster 2023-10-09T09:44:16.984351+0000 osd.6 (osd.6) 507 : cluster 3 failed to encode map e1220 with expected crc 2023-10-09T09:44:18.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:17 smithi181 bash[59345]: cluster 2023-10-09T09:44:16.984901+0000 osd.5 (osd.5) 805 : cluster 3 failed to encode map e1220 with expected crc 2023-10-09T09:44:18.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:17 smithi181 bash[59345]: audit 2023-10-09T09:44:16.989164+0000 mon.a (mon.0) 2175 : audit 1 from='client.? 172.21.15.163:0/3877180870' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:44:18.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:17 smithi181 bash[59345]: cluster 2023-10-09T09:44:16.989409+0000 mon.a (mon.0) 2176 : cluster 0 osdmap e1220: 8 total, 8 up, 6 in 2023-10-09T09:44:18.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:18.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:18.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:18.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:18.236 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:18.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:18.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:18.345 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:18.353 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:17 smithi163 bash[75425]: cluster 2023-10-09T09:44:16.038387+0000 mgr.y (mgr.34107) 1147 : cluster 0 pgmap v1963: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 896 op/s; 3183/54894 objects degraded (5.798%); 2672/54894 objects misplaced (4.868%); 0 B/s, 62 objects/s recovering 2023-10-09T09:44:18.353 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:17 smithi163 bash[75425]: cluster 2023-10-09T09:44:16.984351+0000 osd.6 (osd.6) 507 : cluster 3 failed to encode map e1220 with expected crc 2023-10-09T09:44:18.353 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:17 smithi163 bash[75425]: cluster 2023-10-09T09:44:16.984901+0000 osd.5 (osd.5) 805 : cluster 3 failed to encode map e1220 with expected crc 2023-10-09T09:44:18.353 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:17 smithi163 bash[75425]: audit 2023-10-09T09:44:16.989164+0000 mon.a (mon.0) 2175 : audit 1 from='client.? 172.21.15.163:0/3877180870' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:44:18.354 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:17 smithi163 bash[75425]: cluster 2023-10-09T09:44:16.989409+0000 mon.a (mon.0) 2176 : cluster 0 osdmap e1220: 8 total, 8 up, 6 in 2023-10-09T09:44:18.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:17 smithi163 bash[78107]: cluster 2023-10-09T09:44:16.038387+0000 mgr.y (mgr.34107) 1147 : cluster 0 pgmap v1963: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 896 op/s; 3183/54894 objects degraded (5.798%); 2672/54894 objects misplaced (4.868%); 0 B/s, 62 objects/s recovering 2023-10-09T09:44:18.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:17 smithi163 bash[78107]: cluster 2023-10-09T09:44:16.984351+0000 osd.6 (osd.6) 507 : cluster 3 failed to encode map e1220 with expected crc 2023-10-09T09:44:18.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:17 smithi163 bash[78107]: cluster 2023-10-09T09:44:16.984901+0000 osd.5 (osd.5) 805 : cluster 3 failed to encode map e1220 with expected crc 2023-10-09T09:44:18.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:17 smithi163 bash[78107]: audit 2023-10-09T09:44:16.989164+0000 mon.a (mon.0) 2175 : audit 1 from='client.? 172.21.15.163:0/3877180870' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:44:18.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:17 smithi163 bash[78107]: cluster 2023-10-09T09:44:16.989409+0000 mon.a (mon.0) 2176 : cluster 0 osdmap e1220: 8 total, 8 up, 6 in 2023-10-09T09:44:18.378 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:18.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:18.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:18.489 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:18.522 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:18.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:18.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:18.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:18.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:18.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:18.763 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:18.799 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:18.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:18.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:18.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T09:44:18.903 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:18.937 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:18.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:18.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:19.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:19.073 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:19.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:19.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:19.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:19.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:18 smithi163 bash[75425]: cluster 2023-10-09T09:44:18.039105+0000 mgr.y (mgr.34107) 1148 : cluster 0 pgmap v1965: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 980 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.11k op/s; 2829/47112 objects degraded (6.005%); 2672/47112 objects misplaced (5.672%); 0 B/s, 84 objects/s recovering 2023-10-09T09:44:19.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:18 smithi163 bash[78107]: cluster 2023-10-09T09:44:18.039105+0000 mgr.y (mgr.34107) 1148 : cluster 0 pgmap v1965: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 980 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.11k op/s; 2829/47112 objects degraded (6.005%); 2672/47112 objects misplaced (5.672%); 0 B/s, 84 objects/s recovering 2023-10-09T09:44:19.208 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:19.208 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:44:19.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:19.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:19.212 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:44:19.313 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:19.315 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:18 smithi181 bash[59345]: cluster 2023-10-09T09:44:18.039105+0000 mgr.y (mgr.34107) 1148 : cluster 0 pgmap v1965: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 980 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.11k op/s; 2829/47112 objects degraded (6.005%); 2672/47112 objects misplaced (5.672%); 0 B/s, 84 objects/s recovering 2023-10-09T09:44:19.341 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:19.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:19.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:19.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:19.473 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:19.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:19.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:19.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:19.600 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:19.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:19.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:19.706 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:19.730 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:19.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:19.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:19.835 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:19.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:19.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:19.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:19.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:19.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:19.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:20.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:19 smithi181 bash[59345]: cluster 2023-10-09T09:44:18.998298+0000 osd.5 (osd.5) 806 : cluster 3 failed to encode map e1221 with expected crc 2023-10-09T09:44:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:19 smithi181 bash[59345]: cluster 2023-10-09T09:44:19.001755+0000 mon.a (mon.0) 2177 : cluster 0 osdmap e1221: 8 total, 8 up, 6 in 2023-10-09T09:44:20.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:19 smithi181 bash[59345]: cluster 2023-10-09T09:44:19.004927+0000 osd.6 (osd.6) 508 : cluster 3 failed to encode map e1221 with expected crc 2023-10-09T09:44:20.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:19 smithi181 bash[59345]: cluster 2023-10-09T09:44:19.010863+0000 osd.4 (osd.4) 630 : cluster 3 failed to encode map e1221 with expected crc 2023-10-09T09:44:20.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:20.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:20.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:20.255 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:20.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:20.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:19 smithi163 bash[75425]: cluster 2023-10-09T09:44:18.998298+0000 osd.5 (osd.5) 806 : cluster 3 failed to encode map e1221 with expected crc 2023-10-09T09:44:20.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:19 smithi163 bash[75425]: cluster 2023-10-09T09:44:19.001755+0000 mon.a (mon.0) 2177 : cluster 0 osdmap e1221: 8 total, 8 up, 6 in 2023-10-09T09:44:20.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:19 smithi163 bash[75425]: cluster 2023-10-09T09:44:19.004927+0000 osd.6 (osd.6) 508 : cluster 3 failed to encode map e1221 with expected crc 2023-10-09T09:44:20.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:19 smithi163 bash[75425]: cluster 2023-10-09T09:44:19.010863+0000 osd.4 (osd.4) 630 : cluster 3 failed to encode map e1221 with expected crc 2023-10-09T09:44:20.258 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:19 smithi163 bash[78107]: cluster 2023-10-09T09:44:18.998298+0000 osd.5 (osd.5) 806 : cluster 3 failed to encode map e1221 with expected crc 2023-10-09T09:44:20.258 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:19 smithi163 bash[78107]: cluster 2023-10-09T09:44:19.001755+0000 mon.a (mon.0) 2177 : cluster 0 osdmap e1221: 8 total, 8 up, 6 in 2023-10-09T09:44:20.259 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:19 smithi163 bash[78107]: cluster 2023-10-09T09:44:19.004927+0000 osd.6 (osd.6) 508 : cluster 3 failed to encode map e1221 with expected crc 2023-10-09T09:44:20.259 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:19 smithi163 bash[78107]: cluster 2023-10-09T09:44:19.010863+0000 osd.4 (osd.4) 630 : cluster 3 failed to encode map e1221 with expected crc 2023-10-09T09:44:20.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:20.360 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:20.394 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:20.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:20.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:20.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:20.537 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:20.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:20.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:20.641 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:20.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:20.672 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:20.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:20.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:20.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:20.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:20.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:20.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:20.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:20.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:20.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:21.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:21 smithi181 bash[59345]: cluster 2023-10-09T09:44:20.004375+0000 osd.5 (osd.5) 807 : cluster 3 failed to encode map e1222 with expected crc 2023-10-09T09:44:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:21 smithi181 bash[59345]: cluster 2023-10-09T09:44:20.005212+0000 osd.7 (osd.7) 586 : cluster 3 failed to encode map e1222 with expected crc 2023-10-09T09:44:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:21 smithi181 bash[59345]: cluster 2023-10-09T09:44:20.007394+0000 mon.a (mon.0) 2178 : cluster 0 osdmap e1222: 8 total, 8 up, 6 in 2023-10-09T09:44:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:21 smithi181 bash[59345]: cluster 2023-10-09T09:44:20.008395+0000 osd.6 (osd.6) 509 : cluster 3 failed to encode map e1222 with expected crc 2023-10-09T09:44:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:21 smithi181 bash[59345]: cluster 2023-10-09T09:44:20.039537+0000 mgr.y (mgr.34107) 1149 : cluster 0 pgmap v1968: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 980 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 755 op/s; 2829/47112 objects degraded (6.005%); 2672/47112 objects misplaced (5.672%); 0 B/s, 58 objects/s recovering 2023-10-09T09:44:21.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:21 smithi181 bash[59345]: cluster 2023-10-09T09:44:20.801717+0000 mon.a (mon.0) 2179 : cluster 3 Health check update: Degraded data redundancy: 2829/47112 objects degraded (6.005%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:21.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:21.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:21.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:21.181 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:21.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:21.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:21.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:21.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[75425]: cluster 2023-10-09T09:44:20.004375+0000 osd.5 (osd.5) 807 : cluster 3 failed to encode map e1222 with expected crc 2023-10-09T09:44:21.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[75425]: cluster 2023-10-09T09:44:20.005212+0000 osd.7 (osd.7) 586 : cluster 3 failed to encode map e1222 with expected crc 2023-10-09T09:44:21.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[75425]: cluster 2023-10-09T09:44:20.007394+0000 mon.a (mon.0) 2178 : cluster 0 osdmap e1222: 8 total, 8 up, 6 in 2023-10-09T09:44:21.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[75425]: cluster 2023-10-09T09:44:20.008395+0000 osd.6 (osd.6) 509 : cluster 3 failed to encode map e1222 with expected crc 2023-10-09T09:44:21.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[75425]: cluster 2023-10-09T09:44:20.039537+0000 mgr.y (mgr.34107) 1149 : cluster 0 pgmap v1968: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 980 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 755 op/s; 2829/47112 objects degraded (6.005%); 2672/47112 objects misplaced (5.672%); 0 B/s, 58 objects/s recovering 2023-10-09T09:44:21.239 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[75425]: cluster 2023-10-09T09:44:20.801717+0000 mon.a (mon.0) 2179 : cluster 3 Health check update: Degraded data redundancy: 2829/47112 objects degraded (6.005%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:21.239 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[78107]: cluster 2023-10-09T09:44:20.004375+0000 osd.5 (osd.5) 807 : cluster 3 failed to encode map e1222 with expected crc 2023-10-09T09:44:21.239 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[78107]: cluster 2023-10-09T09:44:20.005212+0000 osd.7 (osd.7) 586 : cluster 3 failed to encode map e1222 with expected crc 2023-10-09T09:44:21.239 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[78107]: cluster 2023-10-09T09:44:20.007394+0000 mon.a (mon.0) 2178 : cluster 0 osdmap e1222: 8 total, 8 up, 6 in 2023-10-09T09:44:21.239 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[78107]: cluster 2023-10-09T09:44:20.008395+0000 osd.6 (osd.6) 509 : cluster 3 failed to encode map e1222 with expected crc 2023-10-09T09:44:21.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[78107]: cluster 2023-10-09T09:44:20.039537+0000 mgr.y (mgr.34107) 1149 : cluster 0 pgmap v1968: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 980 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 755 op/s; 2829/47112 objects degraded (6.005%); 2672/47112 objects misplaced (5.672%); 0 B/s, 58 objects/s recovering 2023-10-09T09:44:21.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:21 smithi163 bash[78107]: cluster 2023-10-09T09:44:20.801717+0000 mon.a (mon.0) 2179 : cluster 3 Health check update: Degraded data redundancy: 2829/47112 objects degraded (6.005%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:21.294 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:44:21.316 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:21.353 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:21.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:21.457 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:21.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:21.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:21.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:21.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:21.611 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:21.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:21.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:21.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:21.744 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:21.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:21.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:21.846 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:21.874 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:21.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:21.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:22.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:22.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:22.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:22.131 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:22.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:22.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:22.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:22.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:22.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:22.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:22.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:22.394 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:22.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:22.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:22.497 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:22.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:22.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:22.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:22.658 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:22.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:22.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:22.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:22.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:22.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:22.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:22.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:22.927 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:22.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:23.032 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:23.053 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:23.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:23.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:23.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:23.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:23 smithi181 bash[59345]: cluster 2023-10-09T09:44:22.040301+0000 mgr.y (mgr.34107) 1150 : cluster 0 pgmap v1969: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 779 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 968 op/s; 1525/37458 objects degraded (4.071%); 2189/37458 objects misplaced (5.844%); 896 KiB/s, 99 objects/s recovering 2023-10-09T09:44:23.183 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:23.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:23.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:23.286 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:23.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:23.329 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:44:23.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:23.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:23.359 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:23 smithi163 bash[75425]: cluster 2023-10-09T09:44:22.040301+0000 mgr.y (mgr.34107) 1150 : cluster 0 pgmap v1969: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 779 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 968 op/s; 1525/37458 objects degraded (4.071%); 2189/37458 objects misplaced (5.844%); 896 KiB/s, 99 objects/s recovering 2023-10-09T09:44:23.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:23 smithi163 bash[78107]: cluster 2023-10-09T09:44:22.040301+0000 mgr.y (mgr.34107) 1150 : cluster 0 pgmap v1969: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 779 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 968 op/s; 1525/37458 objects degraded (4.071%); 2189/37458 objects misplaced (5.844%); 896 KiB/s, 99 objects/s recovering 2023-10-09T09:44:23.359 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:44:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:44:23] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.33.4" 2023-10-09T09:44:23.432 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:23.458 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:23.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:23.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:23.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:23.600 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:23.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:23.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:23.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:23.732 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:23.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:23.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:23.836 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:23.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:23.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:23.970 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:24.002 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:24.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:24.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:24.105 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:24.129 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:24.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:24.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:24.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:24.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:24 smithi163 bash[78107]: audit 2023-10-09T09:44:23.136973+0000 mon.a (mon.0) 2180 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:24.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:24 smithi163 bash[75425]: audit 2023-10-09T09:44:23.136973+0000 mon.a (mon.0) 2180 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:24.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:24.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:24.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:24.370 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:24 smithi181 bash[59345]: audit 2023-10-09T09:44:23.136973+0000 mon.a (mon.0) 2180 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:24.387 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:24.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:24.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:24.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:24.517 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:24.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:24.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:24.621 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:24.648 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:24.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:24.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:24.751 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:24.768 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:24.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:24.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:24.870 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:24.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:24.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:24.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:25.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:25.027 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:25.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:25.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:25.147 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:25 smithi181 bash[59345]: cluster 2023-10-09T09:44:24.040895+0000 mgr.y (mgr.34107) 1151 : cluster 0 pgmap v1970: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 731 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 689 op/s; 1525/35133 objects degraded (4.341%); 2189/35133 objects misplaced (6.231%); 762 KiB/s, 34 objects/s recovering 2023-10-09T09:44:25.168 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:25.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:25.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:25.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:25 smithi163 bash[78107]: cluster 2023-10-09T09:44:24.040895+0000 mgr.y (mgr.34107) 1151 : cluster 0 pgmap v1970: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 731 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 689 op/s; 1525/35133 objects degraded (4.341%); 2189/35133 objects misplaced (6.231%); 762 KiB/s, 34 objects/s recovering 2023-10-09T09:44:25.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:25 smithi163 bash[75425]: cluster 2023-10-09T09:44:24.040895+0000 mgr.y (mgr.34107) 1151 : cluster 0 pgmap v1970: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 731 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 689 op/s; 1525/35133 objects degraded (4.341%); 2189/35133 objects misplaced (6.231%); 762 KiB/s, 34 objects/s recovering 2023-10-09T09:44:25.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:25.294 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:25.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:25.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:25.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:25.420 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:25.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:25.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:25.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:25.564 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:25.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:25.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:25.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:25.706 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:25.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:25.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:25.812 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:25.842 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:25.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:25.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:25.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:25.973 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:25.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:25.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:26.079 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:26.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:26.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:26.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:26 smithi163 bash[78107]: audit 2023-10-09T09:44:25.389437+0000 mon.a (mon.0) 2181 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:26.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:26 smithi163 bash[78107]: cluster 2023-10-09T09:44:25.802823+0000 mon.a (mon.0) 2182 : cluster 3 Health check update: Degraded data redundancy: 1525/35133 objects degraded (4.341%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:26 smithi163 bash[75425]: audit 2023-10-09T09:44:25.389437+0000 mon.a (mon.0) 2181 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:26 smithi163 bash[75425]: cluster 2023-10-09T09:44:25.802823+0000 mon.a (mon.0) 2182 : cluster 3 Health check update: Degraded data redundancy: 1525/35133 objects degraded (4.341%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:26.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:26.241 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:26.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:26.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:26.345 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:26.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:26.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:26.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:26.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:26.512 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:26.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:26.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:26 smithi181 bash[59345]: audit 2023-10-09T09:44:25.389437+0000 mon.a (mon.0) 2181 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:26 smithi181 bash[59345]: cluster 2023-10-09T09:44:25.802823+0000 mon.a (mon.0) 2182 : cluster 3 Health check update: Degraded data redundancy: 1525/35133 objects degraded (4.341%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:26.619 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:26.643 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:26.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:26.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:26.747 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:26.784 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:26.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:26.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:26.914 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:26.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:26.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:27.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:27.053 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:27.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:27.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:27.157 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:27.188 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:27.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:27.291 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:27.318 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:27.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:27.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:27.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:27.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:27.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:27.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:27.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:27.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:27.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:27.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:27.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:27.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:27.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:27.808 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:27 smithi163 bash[75425]: cluster 2023-10-09T09:44:26.041555+0000 mgr.y (mgr.34107) 1152 : cluster 0 pgmap v1971: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 609 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 741 op/s; 1525/29301 objects degraded (5.205%); 1913/29301 objects misplaced (6.529%); 1.7 MiB/s, 47 objects/s recovering 2023-10-09T09:44:27.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:27.836 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:27.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:27.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:27.945 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:27.955 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:27 smithi181 bash[59345]: cluster 2023-10-09T09:44:26.041555+0000 mgr.y (mgr.34107) 1152 : cluster 0 pgmap v1971: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 609 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 741 op/s; 1525/29301 objects degraded (5.205%); 1913/29301 objects misplaced (6.529%); 1.7 MiB/s, 47 objects/s recovering 2023-10-09T09:44:27.979 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:27.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:27.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:28.082 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:28.108 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:28.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:28.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:28.184 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:27 smithi163 bash[78107]: cluster 2023-10-09T09:44:26.041555+0000 mgr.y (mgr.34107) 1152 : cluster 0 pgmap v1971: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 609 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 741 op/s; 1525/29301 objects degraded (5.205%); 1913/29301 objects misplaced (6.529%); 1.7 MiB/s, 47 objects/s recovering 2023-10-09T09:44:28.214 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:28.260 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:28.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:28.368 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:28.399 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:28.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:28.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:28.534 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:28.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:28.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:28.638 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:28.665 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:28.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:28.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:28.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:28.800 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:28.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:28.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:28.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:28.930 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:28.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:29.033 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:29.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:29.066 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:29.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:29.167 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:29.202 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:29.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:29.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:29.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:29.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:29.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:29.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:29.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:29.488 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:29.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:29.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:29.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:29.629 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:29.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:29.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:29.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:29.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:29.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:29.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:29.876 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:29.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:29.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:29.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:30.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:30.031 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:30.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:30.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:30.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:30.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:29 smithi181 bash[59345]: cluster 2023-10-09T09:44:28.042221+0000 mgr.y (mgr.34107) 1153 : cluster 0 pgmap v1972: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 459 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 921 op/s; 1525/22101 objects degraded (6.900%); 1913/22101 objects misplaced (8.656%); 1.5 MiB/s, 42 objects/s recovering 2023-10-09T09:44:30.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:29 smithi181 bash[59345]: audit 2023-10-09T09:44:29.489346+0000 mon.a (mon.0) 2183 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:29 smithi181 bash[59345]: audit 2023-10-09T09:44:29.498140+0000 mon.a (mon.0) 2184 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:29 smithi181 bash[59345]: audit 2023-10-09T09:44:29.695277+0000 mon.a (mon.0) 2185 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:29 smithi181 bash[59345]: audit 2023-10-09T09:44:29.705876+0000 mon.a (mon.0) 2186 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.168 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:30.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:30.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:29 smithi163 bash[75425]: cluster 2023-10-09T09:44:28.042221+0000 mgr.y (mgr.34107) 1153 : cluster 0 pgmap v1972: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 459 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 921 op/s; 1525/22101 objects degraded (6.900%); 1913/22101 objects misplaced (8.656%); 1.5 MiB/s, 42 objects/s recovering 2023-10-09T09:44:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:29 smithi163 bash[75425]: audit 2023-10-09T09:44:29.489346+0000 mon.a (mon.0) 2183 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:29 smithi163 bash[75425]: audit 2023-10-09T09:44:29.498140+0000 mon.a (mon.0) 2184 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:29 smithi163 bash[75425]: audit 2023-10-09T09:44:29.695277+0000 mon.a (mon.0) 2185 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:29 smithi163 bash[75425]: audit 2023-10-09T09:44:29.705876+0000 mon.a (mon.0) 2186 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:29 smithi163 bash[78107]: cluster 2023-10-09T09:44:28.042221+0000 mgr.y (mgr.34107) 1153 : cluster 0 pgmap v1972: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 459 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 921 op/s; 1525/22101 objects degraded (6.900%); 1913/22101 objects misplaced (8.656%); 1.5 MiB/s, 42 objects/s recovering 2023-10-09T09:44:30.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:29 smithi163 bash[78107]: audit 2023-10-09T09:44:29.489346+0000 mon.a (mon.0) 2183 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:29 smithi163 bash[78107]: audit 2023-10-09T09:44:29.498140+0000 mon.a (mon.0) 2184 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:29 smithi163 bash[78107]: audit 2023-10-09T09:44:29.695277+0000 mon.a (mon.0) 2185 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:29 smithi163 bash[78107]: audit 2023-10-09T09:44:29.705876+0000 mon.a (mon.0) 2186 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.274 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:30.313 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:30.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:30.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:30.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:30.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:30.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:30.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:30.553 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:30.579 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:30.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:30.685 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:30.708 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:30.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:30.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:30.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:30.846 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:30.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:30.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:30.950 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:30.962 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:30 smithi163 bash[78107]: audit 2023-10-09T09:44:30.292271+0000 mon.a (mon.0) 2187 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:30.962 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:30 smithi163 bash[78107]: audit 2023-10-09T09:44:30.293884+0000 mon.a (mon.0) 2188 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:30.962 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:30 smithi163 bash[78107]: audit 2023-10-09T09:44:30.302619+0000 mon.a (mon.0) 2189 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.963 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:30 smithi163 bash[78107]: cluster 2023-10-09T09:44:30.803882+0000 mon.a (mon.0) 2190 : cluster 3 Health check update: Degraded data redundancy: 1525/22101 objects degraded (6.900%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:30.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:30 smithi163 bash[75425]: audit 2023-10-09T09:44:30.292271+0000 mon.a (mon.0) 2187 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:30.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:30 smithi163 bash[75425]: audit 2023-10-09T09:44:30.293884+0000 mon.a (mon.0) 2188 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:30.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:30 smithi163 bash[75425]: audit 2023-10-09T09:44:30.302619+0000 mon.a (mon.0) 2189 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:30.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:30 smithi163 bash[75425]: cluster 2023-10-09T09:44:30.803882+0000 mon.a (mon.0) 2190 : cluster 3 Health check update: Degraded data redundancy: 1525/22101 objects degraded (6.900%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:30.978 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:30.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:31.082 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:31.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:30 smithi181 bash[59345]: audit 2023-10-09T09:44:30.292271+0000 mon.a (mon.0) 2187 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:31.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:30 smithi181 bash[59345]: audit 2023-10-09T09:44:30.293884+0000 mon.a (mon.0) 2188 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:31.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:30 smithi181 bash[59345]: audit 2023-10-09T09:44:30.302619+0000 mon.a (mon.0) 2189 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:31.096 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:30 smithi181 bash[59345]: cluster 2023-10-09T09:44:30.803882+0000 mon.a (mon.0) 2190 : cluster 3 Health check update: Degraded data redundancy: 1525/22101 objects degraded (6.900%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:31.113 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:31.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:31.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:31.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:31.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:31.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:31.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:31.381 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:31.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:31.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:31.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:31.513 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:31.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:31.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:31.621 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:31.649 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:31.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:31.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:31.752 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:31.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:31.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:31.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:31.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:31 smithi163 bash[78107]: cluster 2023-10-09T09:44:30.042696+0000 mgr.y (mgr.34107) 1154 : cluster 0 pgmap v1973: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 459 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 830 op/s; 1525/22101 objects degraded (6.900%); 1913/22101 objects misplaced (8.656%); 1.4 MiB/s, 37 objects/s recovering 2023-10-09T09:44:31.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:31 smithi163 bash[75425]: cluster 2023-10-09T09:44:30.042696+0000 mgr.y (mgr.34107) 1154 : cluster 0 pgmap v1973: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 459 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 830 op/s; 1525/22101 objects degraded (6.900%); 1913/22101 objects misplaced (8.656%); 1.4 MiB/s, 37 objects/s recovering 2023-10-09T09:44:31.885 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:31.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:31.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:31.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:32.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:32.052 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:32.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:32.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:31 smithi181 bash[59345]: cluster 2023-10-09T09:44:30.042696+0000 mgr.y (mgr.34107) 1154 : cluster 0 pgmap v1973: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 459 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 830 op/s; 1525/22101 objects degraded (6.900%); 1913/22101 objects misplaced (8.656%); 1.4 MiB/s, 37 objects/s recovering 2023-10-09T09:44:32.158 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:32.187 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:32.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:32.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:32.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:32.327 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:32.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:32.429 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:32.462 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:32.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:32.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:32.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:32.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:32.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:32.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:32.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:32.719 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:32.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:32.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:32.822 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:32.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:32.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:32.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:32.959 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:32.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:32.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:33.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:33.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:33.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:33.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:33.255 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:33.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:33.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:33.359 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:33.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:33.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:33.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:33.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:33.502 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:44:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:44:33] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.33.4" 2023-10-09T09:44:33.524 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:33.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:33.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:33.628 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:33.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:33.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:33.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:33.759 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:33.789 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:33.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:33.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:33.892 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:33.920 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:33.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:33.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:34.025 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:34.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:33 smithi163 bash[75425]: cluster 2023-10-09T09:44:32.043435+0000 mgr.y (mgr.34107) 1155 : cluster 0 pgmap v1974: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 248 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 975 op/s; 1525/11976 objects degraded (12.734%); 1625/11976 objects misplaced (13.569%); 1.9 MiB/s, 43 objects/s recovering 2023-10-09T09:44:34.034 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:33 smithi163 bash[78107]: cluster 2023-10-09T09:44:32.043435+0000 mgr.y (mgr.34107) 1155 : cluster 0 pgmap v1974: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 248 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 975 op/s; 1525/11976 objects degraded (12.734%); 1625/11976 objects misplaced (13.569%); 1.9 MiB/s, 43 objects/s recovering 2023-10-09T09:44:34.052 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:34.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:34.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:34.161 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:34.182 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:34.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:34.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:34.286 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:34.293 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:33 smithi181 bash[59345]: cluster 2023-10-09T09:44:32.043435+0000 mgr.y (mgr.34107) 1155 : cluster 0 pgmap v1974: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 248 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 975 op/s; 1525/11976 objects degraded (12.734%); 1625/11976 objects misplaced (13.569%); 1.9 MiB/s, 43 objects/s recovering 2023-10-09T09:44:34.307 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:34.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:34.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:34.410 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:34.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:34.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:34.543 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:34.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:34.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:34.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:34.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:34.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:34.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:34.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:34.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:34.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:34.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:34.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:34.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 103893 objects 2023-10-09T09:44:34.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :119.791 2023-10-09T09:44:34.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:34.963 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:34.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:34.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:35.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:35.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:35.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:35.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:35.241 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:35.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:35.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:35.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:35.374 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:35.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:35.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:35.478 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:35.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:35.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:35.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:35.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:35.638 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:35.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:35.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:35.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:35.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:35.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:35.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:35.868 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:35.881 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:35 smithi181 bash[59345]: cluster 2023-10-09T09:44:34.044007+0000 mgr.y (mgr.34107) 1156 : cluster 0 pgmap v1975: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 203 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 1525/9816 objects degraded (15.536%); 1625/9816 objects misplaced (16.555%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:44:35.881 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:35 smithi181 bash[59345]: cluster 2023-10-09T09:44:35.804962+0000 mon.a (mon.0) 2191 : cluster 3 Health check update: Degraded data redundancy: 1525/9816 objects degraded (15.536%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:35.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:35.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:35.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:36.003 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:36.111 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:36.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:36.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:36.186 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:35 smithi163 bash[78107]: cluster 2023-10-09T09:44:34.044007+0000 mgr.y (mgr.34107) 1156 : cluster 0 pgmap v1975: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 203 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 1525/9816 objects degraded (15.536%); 1625/9816 objects misplaced (16.555%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:44:36.186 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:35 smithi163 bash[78107]: cluster 2023-10-09T09:44:35.804962+0000 mon.a (mon.0) 2191 : cluster 3 Health check update: Degraded data redundancy: 1525/9816 objects degraded (15.536%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:36.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:35 smithi163 bash[75425]: cluster 2023-10-09T09:44:34.044007+0000 mgr.y (mgr.34107) 1156 : cluster 0 pgmap v1975: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 203 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 1525/9816 objects degraded (15.536%); 1625/9816 objects misplaced (16.555%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T09:44:36.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:35 smithi163 bash[75425]: cluster 2023-10-09T09:44:35.804962+0000 mon.a (mon.0) 2191 : cluster 3 Health check update: Degraded data redundancy: 1525/9816 objects degraded (15.536%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:36.215 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:36.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:36.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:36.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:36.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:36.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:36.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:36.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:36.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:36.501 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:36.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:36.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:36.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:36.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:36.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:36.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:36.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:36.757 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:36.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:36.861 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:36.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:36.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:36.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:36.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:37.031 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:37.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:37.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:37.135 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:37.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:37.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:37.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:37.271 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:37.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:37.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:37.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:37.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:37.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:37.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:37.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:37.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:37.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:37.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:37.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:37.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:37.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:37.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:37.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:37.826 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:37.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:37.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:37.929 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:37.955 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:37.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:37.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:38.059 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:38.065 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:37 smithi163 bash[78107]: cluster 2023-10-09T09:44:36.044650+0000 mgr.y (mgr.34107) 1157 : cluster 0 pgmap v1976: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 109 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 842 op/s; 1525/5262 objects degraded (28.981%); 1525/5262 objects misplaced (28.981%); 1.7 MiB/s, 40 objects/s recovering 2023-10-09T09:44:38.066 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:37 smithi163 bash[75425]: cluster 2023-10-09T09:44:36.044650+0000 mgr.y (mgr.34107) 1157 : cluster 0 pgmap v1976: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 109 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 842 op/s; 1525/5262 objects degraded (28.981%); 1525/5262 objects misplaced (28.981%); 1.7 MiB/s, 40 objects/s recovering 2023-10-09T09:44:38.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:38.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:38.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:38.184 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:38.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:37 smithi181 bash[59345]: cluster 2023-10-09T09:44:36.044650+0000 mgr.y (mgr.34107) 1157 : cluster 0 pgmap v1976: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 109 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 842 op/s; 1525/5262 objects degraded (28.981%); 1525/5262 objects misplaced (28.981%); 1.7 MiB/s, 40 objects/s recovering 2023-10-09T09:44:38.215 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:38.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:38.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:38.318 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:38.341 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:38.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:38.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:38.444 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:38.472 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:38.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:38.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:38.575 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:38.606 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:38.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:38.709 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:38.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:38.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:38.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:38.785 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T09:44:38.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T09:44:38.848 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:38.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:38.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:38.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:38.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:38.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:38.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:39.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:39.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:39.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:39.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:39.232 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:39.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:39.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:39.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:39.366 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:39.399 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:39.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:39.502 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:39.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:39.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:39.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:39.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:39.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:39.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:39.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:39.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:39.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:39.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:39.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:39.908 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:39.922 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:39 smithi181 bash[59345]: cluster 2023-10-09T09:44:38.045626+0000 mgr.y (mgr.34107) 1158 : cluster 0 pgmap v1977: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 1525/972 objects degraded (156.893%); 1525/972 objects misplaced (156.893%); 1.0 MiB/s, 29 objects/s recovering 2023-10-09T09:44:39.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:39.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:39.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:40.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:40.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:39 smithi163 bash[78107]: cluster 2023-10-09T09:44:38.045626+0000 mgr.y (mgr.34107) 1158 : cluster 0 pgmap v1977: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 1525/972 objects degraded (156.893%); 1525/972 objects misplaced (156.893%); 1.0 MiB/s, 29 objects/s recovering 2023-10-09T09:44:40.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:39 smithi163 bash[75425]: cluster 2023-10-09T09:44:38.045626+0000 mgr.y (mgr.34107) 1158 : cluster 0 pgmap v1977: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 1525/972 objects degraded (156.893%); 1525/972 objects misplaced (156.893%); 1.0 MiB/s, 29 objects/s recovering 2023-10-09T09:44:40.069 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:40.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:40.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:40.172 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:40.192 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:40.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:40.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:40.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:40.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:40.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:40.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:40.420 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:40.444 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:40.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:40.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:40.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:40.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:40.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:40.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:40.676 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:40.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:40.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:40.806 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:40.837 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:40.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:40.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:40.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:40.952 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:40 smithi181 bash[59345]: audit 2023-10-09T09:44:40.389955+0000 mon.a (mon.0) 2192 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:40.952 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:40 smithi181 bash[59345]: cluster 2023-10-09T09:44:40.806104+0000 mon.a (mon.0) 2193 : cluster 3 Health check update: Degraded data redundancy: 1525/972 objects degraded (156.893%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:40.969 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:40.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:41.072 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:41.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:41.108 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:41.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:41.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:41.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:40 smithi163 bash[75425]: audit 2023-10-09T09:44:40.389955+0000 mon.a (mon.0) 2192 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:41.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:40 smithi163 bash[75425]: cluster 2023-10-09T09:44:40.806104+0000 mon.a (mon.0) 2193 : cluster 3 Health check update: Degraded data redundancy: 1525/972 objects degraded (156.893%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:40 smithi163 bash[78107]: audit 2023-10-09T09:44:40.389955+0000 mon.a (mon.0) 2192 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:40 smithi163 bash[78107]: cluster 2023-10-09T09:44:40.806104+0000 mon.a (mon.0) 2193 : cluster 3 Health check update: Degraded data redundancy: 1525/972 objects degraded (156.893%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:41.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:41.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:41.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:41.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:41.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:41.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:41.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:41.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:41.495 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:41.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:41.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:41.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:41.621 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:41.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:41.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:41.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:41.747 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:41.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:41.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:41.856 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:41 smithi163 bash[75425]: cluster 2023-10-09T09:44:40.046247+0000 mgr.y (mgr.34107) 1159 : cluster 0 pgmap v1978: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 1525/972 objects degraded (156.893%); 1525/972 objects misplaced (156.893%); 1.0 MiB/s, 59 objects/s recovering 2023-10-09T09:44:41.883 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:41.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:41.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:41.986 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:42.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:42.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:42.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:42.119 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:42.132 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:41 smithi181 bash[59345]: cluster 2023-10-09T09:44:40.046247+0000 mgr.y (mgr.34107) 1159 : cluster 0 pgmap v1978: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 1525/972 objects degraded (156.893%); 1525/972 objects misplaced (156.893%); 1.0 MiB/s, 59 objects/s recovering 2023-10-09T09:44:42.152 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:42.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:42.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:42.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:41 smithi163 bash[78107]: cluster 2023-10-09T09:44:40.046247+0000 mgr.y (mgr.34107) 1159 : cluster 0 pgmap v1978: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 1525/972 objects degraded (156.893%); 1525/972 objects misplaced (156.893%); 1.0 MiB/s, 59 objects/s recovering 2023-10-09T09:44:42.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:42.290 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:42.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:42.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:42.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:42.420 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:42.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:42.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:42.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:42.544 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:42.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:42.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:42.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:42.680 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:42.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:42.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:42.784 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:42.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:42.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:42.914 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:42.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:42.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:42.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:43.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:43.077 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:43.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:43.181 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:43.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:43.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:43.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:43.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:43.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:43.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:43.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:43.482 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:43.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:43.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:43.529 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:44:43.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:43.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:43.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:43.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:43.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:44:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:44:43] "GET /metrics HTTP/1.1" 200 36369 "" "Prometheus/2.33.4" 2023-10-09T09:44:43.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:43.750 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:43.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:43.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:43.852 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:43.884 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:43.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:43.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:43.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:44.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:44.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:44.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:44.122 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:44.135 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:43 smithi181 bash[59345]: cluster 2023-10-09T09:44:42.047239+0000 mgr.y (mgr.34107) 1160 : cluster 0 pgmap v1979: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 1525/972 objects degraded (156.893%); 1525/972 objects misplaced (156.893%); 1.0 MiB/s, 59 objects/s recovering 2023-10-09T09:44:44.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:44.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:44.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:44.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:43 smithi163 bash[78107]: cluster 2023-10-09T09:44:42.047239+0000 mgr.y (mgr.34107) 1160 : cluster 0 pgmap v1979: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 1525/972 objects degraded (156.893%); 1525/972 objects misplaced (156.893%); 1.0 MiB/s, 59 objects/s recovering 2023-10-09T09:44:44.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:43 smithi163 bash[75425]: cluster 2023-10-09T09:44:42.047239+0000 mgr.y (mgr.34107) 1160 : cluster 0 pgmap v1979: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 1525/972 objects degraded (156.893%); 1525/972 objects misplaced (156.893%); 1.0 MiB/s, 59 objects/s recovering 2023-10-09T09:44:44.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:44.283 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:44.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:44.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:44.418 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:44.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:44.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:44.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:44.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:44.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:44.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:44.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:44.677 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:44.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:44.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:44.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:44.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:44.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:44.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:44.901 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:44.930 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:44.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:44.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:45.034 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:45.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:45.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:45.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:45.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:45.199 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:45.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:45.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:45.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:45.322 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:45.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:45.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:45.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:45.452 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:45.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:45.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:45.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:45.586 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:45.587 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:44:45.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:45.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:45.607 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:44:45.632 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:44:45.656 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:44:45.679 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:44:45.689 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:45.710 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:44:45.714 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:45.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:45.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:45.732 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:44:45.760 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:44:45.762 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:44:45.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:44:45.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:44:45.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:44:45.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:44:45.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:44:45.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:44:45.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:44:45.816 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:45.847 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:45.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:45.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:45.951 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:45.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:45 smithi163 bash[78107]: cluster 2023-10-09T09:44:44.047852+0000 mgr.y (mgr.34107) 1161 : cluster 0 pgmap v1980: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 331 op/s; 1037/972 objects degraded (106.687%); 1037/972 objects misplaced (106.687%); 267 KiB/s, 105 objects/s recovering 2023-10-09T09:44:45.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:45 smithi163 bash[78107]: audit 2023-10-09T09:44:45.565332+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.163:0/2024294411' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:44:45.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:45 smithi163 bash[78107]: cluster 2023-10-09T09:44:45.807289+0000 mon.a (mon.0) 2194 : cluster 3 Health check update: Degraded data redundancy: 1037/972 objects degraded (106.687%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:45.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:45 smithi163 bash[75425]: cluster 2023-10-09T09:44:44.047852+0000 mgr.y (mgr.34107) 1161 : cluster 0 pgmap v1980: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 331 op/s; 1037/972 objects degraded (106.687%); 1037/972 objects misplaced (106.687%); 267 KiB/s, 105 objects/s recovering 2023-10-09T09:44:45.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:45 smithi163 bash[75425]: audit 2023-10-09T09:44:45.565332+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.163:0/2024294411' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:44:45.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:45 smithi163 bash[75425]: cluster 2023-10-09T09:44:45.807289+0000 mon.a (mon.0) 2194 : cluster 3 Health check update: Degraded data redundancy: 1037/972 objects degraded (106.687%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:45.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:45.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:45.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:46.017 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:44:46.017 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:44:46.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:46.103 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:46.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:46.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:46.206 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:46.217 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:45 smithi181 bash[59345]: cluster 2023-10-09T09:44:44.047852+0000 mgr.y (mgr.34107) 1161 : cluster 0 pgmap v1980: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 331 op/s; 1037/972 objects degraded (106.687%); 1037/972 objects misplaced (106.687%); 267 KiB/s, 105 objects/s recovering 2023-10-09T09:44:46.217 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:45 smithi181 bash[59345]: audit 2023-10-09T09:44:45.565332+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.163:0/2024294411' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:44:46.217 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:45 smithi181 bash[59345]: cluster 2023-10-09T09:44:45.807289+0000 mon.a (mon.0) 2194 : cluster 3 Health check update: Degraded data redundancy: 1037/972 objects degraded (106.687%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:46.238 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:46.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:46.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:46.341 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:46.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:46.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:46.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:46.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:46.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:46.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:46.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:46.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:46.638 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:46.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:46.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:46.771 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:46.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:46.874 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:46.900 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:46.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:46.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:47.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:47.034 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:47.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:47.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:47.138 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:47.170 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:47.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:47.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:47.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:47.302 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:47.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:47.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:47.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:47.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:47.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:47.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:47.534 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:47.567 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:47.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:47.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:47.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:47.692 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:47.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:47.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:47.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:47.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:47.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:47.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:47.931 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:47.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:47.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:47.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:47 smithi163 bash[75425]: cluster 2023-10-09T09:44:46.048286+0000 mgr.y (mgr.34107) 1162 : cluster 0 pgmap v1981: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 271 op/s; 1037/972 objects degraded (106.687%); 1037/972 objects misplaced (106.687%); 267 KiB/s, 105 objects/s recovering 2023-10-09T09:44:47.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:47 smithi163 bash[78107]: cluster 2023-10-09T09:44:46.048286+0000 mgr.y (mgr.34107) 1162 : cluster 0 pgmap v1981: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 271 op/s; 1037/972 objects degraded (106.687%); 1037/972 objects misplaced (106.687%); 267 KiB/s, 105 objects/s recovering 2023-10-09T09:44:48.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:47 smithi181 bash[59345]: cluster 2023-10-09T09:44:46.048286+0000 mgr.y (mgr.34107) 1162 : cluster 0 pgmap v1981: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 271 op/s; 1037/972 objects degraded (106.687%); 1037/972 objects misplaced (106.687%); 267 KiB/s, 105 objects/s recovering 2023-10-09T09:44:48.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:48.093 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:48.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:48.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:48.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:48.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:48.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:48.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:48.361 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:48.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:48.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:48.466 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:48.490 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:48.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:48.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:48.593 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:48.616 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:48.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:48.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:48.720 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:48.751 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:48.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:48.856 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:48.883 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:48.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:48.989 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:49.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:49.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:49.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:49.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:49.154 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:49.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:49.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:49.258 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:49.292 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:49.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:49.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:49.396 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:49.429 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:49.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:49.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:49.533 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:49.562 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:49.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:49.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:49.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:49.691 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:49.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:49.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:49.794 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:49.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:49.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:49.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:49.886 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:44:49.931 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:49.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:49.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:49.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:50.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:49 smithi181 bash[59345]: cluster 2023-10-09T09:44:48.049168+0000 mgr.y (mgr.34107) 1163 : cluster 0 pgmap v1982: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 1037/972 objects degraded (106.687%); 1037/972 objects misplaced (106.687%); 0 B/s, 88 objects/s recovering 2023-10-09T09:44:50.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:49 smithi181 bash[59345]: cluster 2023-10-09T09:44:48.877637+0000 osd.5 (osd.5) 808 : cluster 3 failed to encode map e1223 with expected crc 2023-10-09T09:44:50.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:49 smithi181 bash[59345]: cluster 2023-10-09T09:44:48.879813+0000 mon.a (mon.0) 2195 : cluster 0 osdmap e1223: 8 total, 8 up, 6 in 2023-10-09T09:44:50.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:49 smithi181 bash[59345]: cluster 2023-10-09T09:44:48.888575+0000 osd.6 (osd.6) 510 : cluster 3 failed to encode map e1223 with expected crc 2023-10-09T09:44:50.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:49 smithi181 bash[59345]: cluster 2023-10-09T09:44:48.888722+0000 osd.4 (osd.4) 631 : cluster 3 failed to encode map e1223 with expected crc 2023-10-09T09:44:50.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:49 smithi181 bash[59345]: audit 2023-10-09T09:44:49.084504+0000 mon.a (mon.0) 2196 : audit 1 from='client.? 172.21.15.163:0/333017830' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:44:50.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:50.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:50.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:50.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T09:44:50.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:50.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[78107]: cluster 2023-10-09T09:44:48.049168+0000 mgr.y (mgr.34107) 1163 : cluster 0 pgmap v1982: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 1037/972 objects degraded (106.687%); 1037/972 objects misplaced (106.687%); 0 B/s, 88 objects/s recovering 2023-10-09T09:44:50.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[78107]: cluster 2023-10-09T09:44:48.877637+0000 osd.5 (osd.5) 808 : cluster 3 failed to encode map e1223 with expected crc 2023-10-09T09:44:50.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[78107]: cluster 2023-10-09T09:44:48.879813+0000 mon.a (mon.0) 2195 : cluster 0 osdmap e1223: 8 total, 8 up, 6 in 2023-10-09T09:44:50.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[78107]: cluster 2023-10-09T09:44:48.888575+0000 osd.6 (osd.6) 510 : cluster 3 failed to encode map e1223 with expected crc 2023-10-09T09:44:50.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[78107]: cluster 2023-10-09T09:44:48.888722+0000 osd.4 (osd.4) 631 : cluster 3 failed to encode map e1223 with expected crc 2023-10-09T09:44:50.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[78107]: audit 2023-10-09T09:44:49.084504+0000 mon.a (mon.0) 2196 : audit 1 from='client.? 172.21.15.163:0/333017830' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:44:50.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[75425]: cluster 2023-10-09T09:44:48.049168+0000 mgr.y (mgr.34107) 1163 : cluster 0 pgmap v1982: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 1037/972 objects degraded (106.687%); 1037/972 objects misplaced (106.687%); 0 B/s, 88 objects/s recovering 2023-10-09T09:44:50.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[75425]: cluster 2023-10-09T09:44:48.877637+0000 osd.5 (osd.5) 808 : cluster 3 failed to encode map e1223 with expected crc 2023-10-09T09:44:50.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[75425]: cluster 2023-10-09T09:44:48.879813+0000 mon.a (mon.0) 2195 : cluster 0 osdmap e1223: 8 total, 8 up, 6 in 2023-10-09T09:44:50.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[75425]: cluster 2023-10-09T09:44:48.888575+0000 osd.6 (osd.6) 510 : cluster 3 failed to encode map e1223 with expected crc 2023-10-09T09:44:50.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[75425]: cluster 2023-10-09T09:44:48.888722+0000 osd.4 (osd.4) 631 : cluster 3 failed to encode map e1223 with expected crc 2023-10-09T09:44:50.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:49 smithi163 bash[75425]: audit 2023-10-09T09:44:49.084504+0000 mon.a (mon.0) 2196 : audit 1 from='client.? 172.21.15.163:0/333017830' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:44:50.199 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:50.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:50.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:50.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:50.329 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:50.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:50.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:50.432 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:50.464 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:50.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:50.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:50.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:50.595 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:50.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:50.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:50.698 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:50.725 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:50.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:50.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:50.828 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:50.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:50.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:50.906 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T09:44:50.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:51.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:51.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:51.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:51.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:51.135 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:51.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:51.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:51.145 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[78107]: cluster 2023-10-09T09:44:49.871818+0000 mon.a (mon.0) 2197 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:44:51.145 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[78107]: cluster 2023-10-09T09:44:49.876915+0000 osd.5 (osd.5) 809 : cluster 3 failed to encode map e1224 with expected crc 2023-10-09T09:44:51.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[78107]: audit 2023-10-09T09:44:49.879854+0000 mon.a (mon.0) 2198 : audit 1 from='client.? 172.21.15.163:0/333017830' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:44:51.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[78107]: cluster 2023-10-09T09:44:49.879937+0000 mon.a (mon.0) 2199 : cluster 0 osdmap e1224: 8 total, 8 up, 6 in 2023-10-09T09:44:51.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[78107]: cluster 2023-10-09T09:44:49.882621+0000 osd.4 (osd.4) 632 : cluster 3 failed to encode map e1224 with expected crc 2023-10-09T09:44:51.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[78107]: audit 2023-10-09T09:44:50.050316+0000 mon.a (mon.0) 2200 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T09:44:51.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[78107]: audit 2023-10-09T09:44:50.059786+0000 mon.a (mon.0) 2201 : audit 1 from='client.? 172.21.15.163:0/3514945109' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:44:51.146 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[75425]: cluster 2023-10-09T09:44:49.871818+0000 mon.a (mon.0) 2197 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:44:51.146 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[75425]: cluster 2023-10-09T09:44:49.876915+0000 osd.5 (osd.5) 809 : cluster 3 failed to encode map e1224 with expected crc 2023-10-09T09:44:51.146 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[75425]: audit 2023-10-09T09:44:49.879854+0000 mon.a (mon.0) 2198 : audit 1 from='client.? 172.21.15.163:0/333017830' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:44:51.146 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[75425]: cluster 2023-10-09T09:44:49.879937+0000 mon.a (mon.0) 2199 : cluster 0 osdmap e1224: 8 total, 8 up, 6 in 2023-10-09T09:44:51.146 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[75425]: cluster 2023-10-09T09:44:49.882621+0000 osd.4 (osd.4) 632 : cluster 3 failed to encode map e1224 with expected crc 2023-10-09T09:44:51.147 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[75425]: audit 2023-10-09T09:44:50.050316+0000 mon.a (mon.0) 2200 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T09:44:51.147 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:50 smithi163 bash[75425]: audit 2023-10-09T09:44:50.059786+0000 mon.a (mon.0) 2201 : audit 1 from='client.? 172.21.15.163:0/3514945109' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:44:51.239 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:51.252 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:50 smithi181 bash[59345]: cluster 2023-10-09T09:44:49.871818+0000 mon.a (mon.0) 2197 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:44:51.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:50 smithi181 bash[59345]: cluster 2023-10-09T09:44:49.876915+0000 osd.5 (osd.5) 809 : cluster 3 failed to encode map e1224 with expected crc 2023-10-09T09:44:51.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:50 smithi181 bash[59345]: audit 2023-10-09T09:44:49.879854+0000 mon.a (mon.0) 2198 : audit 1 from='client.? 172.21.15.163:0/333017830' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:44:51.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:50 smithi181 bash[59345]: cluster 2023-10-09T09:44:49.879937+0000 mon.a (mon.0) 2199 : cluster 0 osdmap e1224: 8 total, 8 up, 6 in 2023-10-09T09:44:51.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:50 smithi181 bash[59345]: cluster 2023-10-09T09:44:49.882621+0000 osd.4 (osd.4) 632 : cluster 3 failed to encode map e1224 with expected crc 2023-10-09T09:44:51.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:50 smithi181 bash[59345]: audit 2023-10-09T09:44:50.050316+0000 mon.a (mon.0) 2200 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T09:44:51.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:50 smithi181 bash[59345]: audit 2023-10-09T09:44:50.059786+0000 mon.a (mon.0) 2201 : audit 1 from='client.? 172.21.15.163:0/3514945109' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:44:51.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:51.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:51.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:51.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:51.403 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:51.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:51.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:51.507 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:51.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:51.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:51.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:51.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:51.672 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:51.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:51.775 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:51.806 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:51.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:51.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:51.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:51.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:51.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:52.039 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:52.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:44:52.053 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:51 smithi181 bash[59345]: cluster 2023-10-09T09:44:50.049565+0000 mgr.y (mgr.34107) 1164 : cluster 0 pgmap v1985: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 216 objects/s recovering 2023-10-09T09:44:52.053 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:51 smithi181 bash[59345]: cluster 2023-10-09T09:44:50.883600+0000 mon.a (mon.0) 2202 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:52.053 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:51 smithi181 bash[59345]: cluster 2023-10-09T09:44:50.895180+0000 osd.5 (osd.5) 810 : cluster 3 failed to encode map e1225 with expected crc 2023-10-09T09:44:52.053 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:51 smithi181 bash[59345]: cluster 2023-10-09T09:44:50.895945+0000 osd.6 (osd.6) 511 : cluster 3 failed to encode map e1225 with expected crc 2023-10-09T09:44:52.054 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:51 smithi181 bash[59345]: audit 2023-10-09T09:44:50.899542+0000 mon.a (mon.0) 2203 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T09:44:52.054 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:51 smithi181 bash[59345]: audit 2023-10-09T09:44:50.899658+0000 mon.a (mon.0) 2204 : audit 1 from='client.? 172.21.15.163:0/3514945109' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:44:52.054 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:51 smithi181 bash[59345]: cluster 2023-10-09T09:44:50.899740+0000 mon.a (mon.0) 2205 : cluster 0 osdmap e1225: 8 total, 8 up, 6 in 2023-10-09T09:44:52.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:52.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:52.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:52.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:52.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[75425]: cluster 2023-10-09T09:44:50.049565+0000 mgr.y (mgr.34107) 1164 : cluster 0 pgmap v1985: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 216 objects/s recovering 2023-10-09T09:44:52.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[75425]: cluster 2023-10-09T09:44:50.883600+0000 mon.a (mon.0) 2202 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:52.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[75425]: cluster 2023-10-09T09:44:50.895180+0000 osd.5 (osd.5) 810 : cluster 3 failed to encode map e1225 with expected crc 2023-10-09T09:44:52.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[75425]: cluster 2023-10-09T09:44:50.895945+0000 osd.6 (osd.6) 511 : cluster 3 failed to encode map e1225 with expected crc 2023-10-09T09:44:52.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[75425]: audit 2023-10-09T09:44:50.899542+0000 mon.a (mon.0) 2203 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T09:44:52.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[75425]: audit 2023-10-09T09:44:50.899658+0000 mon.a (mon.0) 2204 : audit 1 from='client.? 172.21.15.163:0/3514945109' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:44:52.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[75425]: cluster 2023-10-09T09:44:50.899740+0000 mon.a (mon.0) 2205 : cluster 0 osdmap e1225: 8 total, 8 up, 6 in 2023-10-09T09:44:52.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[78107]: cluster 2023-10-09T09:44:50.049565+0000 mgr.y (mgr.34107) 1164 : cluster 0 pgmap v1985: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 216 objects/s recovering 2023-10-09T09:44:52.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[78107]: cluster 2023-10-09T09:44:50.883600+0000 mon.a (mon.0) 2202 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:44:52.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[78107]: cluster 2023-10-09T09:44:50.895180+0000 osd.5 (osd.5) 810 : cluster 3 failed to encode map e1225 with expected crc 2023-10-09T09:44:52.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[78107]: cluster 2023-10-09T09:44:50.895945+0000 osd.6 (osd.6) 511 : cluster 3 failed to encode map e1225 with expected crc 2023-10-09T09:44:52.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[78107]: audit 2023-10-09T09:44:50.899542+0000 mon.a (mon.0) 2203 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T09:44:52.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[78107]: audit 2023-10-09T09:44:50.899658+0000 mon.a (mon.0) 2204 : audit 1 from='client.? 172.21.15.163:0/3514945109' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:44:52.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:51 smithi163 bash[78107]: cluster 2023-10-09T09:44:50.899740+0000 mon.a (mon.0) 2205 : cluster 0 osdmap e1225: 8 total, 8 up, 6 in 2023-10-09T09:44:52.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:52.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:52.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:52.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:52.332 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:52.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:52.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:52.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:52.458 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:52.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:52.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:52.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:52.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:52.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:52.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:52.696 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:52.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:52.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:52.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:52.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T09:44:52.828 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:52.854 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:52.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:52.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:52.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[75425]: cluster 2023-10-09T09:44:51.650342+0000 osd.2 (osd.2) 8 : cluster 0 9.16 scrub starts 2023-10-09T09:44:52.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[75425]: cluster 2023-10-09T09:44:51.664082+0000 osd.2 (osd.2) 9 : cluster 0 9.16 scrub ok 2023-10-09T09:44:52.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[75425]: cluster 2023-10-09T09:44:51.906899+0000 osd.4 (osd.4) 633 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:52.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[75425]: cluster 2023-10-09T09:44:51.906930+0000 osd.5 (osd.5) 811 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:52.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[75425]: cluster 2023-10-09T09:44:51.908937+0000 mon.a (mon.0) 2206 : cluster 0 osdmap e1226: 8 total, 8 up, 6 in 2023-10-09T09:44:52.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[75425]: cluster 2023-10-09T09:44:51.912614+0000 osd.6 (osd.6) 512 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:52.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[75425]: audit 2023-10-09T09:44:52.051493+0000 mon.a (mon.0) 2207 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T09:44:52.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[75425]: cluster 2023-10-09T09:44:52.179230+0000 osd.7 (osd.7) 587 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:52.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[75425]: cluster 2023-10-09T09:44:52.182355+0000 osd.7 (osd.7) 588 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[78107]: cluster 2023-10-09T09:44:51.650342+0000 osd.2 (osd.2) 8 : cluster 0 9.16 scrub starts 2023-10-09T09:44:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[78107]: cluster 2023-10-09T09:44:51.664082+0000 osd.2 (osd.2) 9 : cluster 0 9.16 scrub ok 2023-10-09T09:44:52.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[78107]: cluster 2023-10-09T09:44:51.906899+0000 osd.4 (osd.4) 633 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:52.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[78107]: cluster 2023-10-09T09:44:51.906930+0000 osd.5 (osd.5) 811 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:52.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[78107]: cluster 2023-10-09T09:44:51.908937+0000 mon.a (mon.0) 2206 : cluster 0 osdmap e1226: 8 total, 8 up, 6 in 2023-10-09T09:44:52.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[78107]: cluster 2023-10-09T09:44:51.912614+0000 osd.6 (osd.6) 512 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:52.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[78107]: audit 2023-10-09T09:44:52.051493+0000 mon.a (mon.0) 2207 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T09:44:52.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[78107]: cluster 2023-10-09T09:44:52.179230+0000 osd.7 (osd.7) 587 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:52.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:52 smithi163 bash[78107]: cluster 2023-10-09T09:44:52.182355+0000 osd.7 (osd.7) 588 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:52.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:52.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:52.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:53.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:53.126 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:53.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:53.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:53.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:53.260 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:53.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:53.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:52 smithi181 bash[59345]: cluster 2023-10-09T09:44:51.650342+0000 osd.2 (osd.2) 8 : cluster 0 9.16 scrub starts 2023-10-09T09:44:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:52 smithi181 bash[59345]: cluster 2023-10-09T09:44:51.664082+0000 osd.2 (osd.2) 9 : cluster 0 9.16 scrub ok 2023-10-09T09:44:53.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:52 smithi181 bash[59345]: cluster 2023-10-09T09:44:51.906899+0000 osd.4 (osd.4) 633 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:53.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:52 smithi181 bash[59345]: cluster 2023-10-09T09:44:51.906930+0000 osd.5 (osd.5) 811 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:53.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:52 smithi181 bash[59345]: cluster 2023-10-09T09:44:51.908937+0000 mon.a (mon.0) 2206 : cluster 0 osdmap e1226: 8 total, 8 up, 6 in 2023-10-09T09:44:53.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:52 smithi181 bash[59345]: cluster 2023-10-09T09:44:51.912614+0000 osd.6 (osd.6) 512 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:53.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:52 smithi181 bash[59345]: audit 2023-10-09T09:44:52.051493+0000 mon.a (mon.0) 2207 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T09:44:53.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:52 smithi181 bash[59345]: cluster 2023-10-09T09:44:52.179230+0000 osd.7 (osd.7) 587 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:53.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:52 smithi181 bash[59345]: cluster 2023-10-09T09:44:52.182355+0000 osd.7 (osd.7) 588 : cluster 3 failed to encode map e1226 with expected crc 2023-10-09T09:44:53.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:44:53.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:53.398 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:53.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:53.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:53.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:53.511 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:44:53] "GET /metrics HTTP/1.1" 200 36369 "" "Prometheus/2.33.4" 2023-10-09T09:44:53.523 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:53.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:53.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:53.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:53.662 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:53.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:53.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:53.766 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:53.797 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:53.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:53.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:53.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:53.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:53.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:54.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:54.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: cluster 2023-10-09T09:44:52.050329+0000 mgr.y (mgr.34107) 1165 : cluster 0 pgmap v1988: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 259 objects/s recovering 2023-10-09T09:44:54.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: cluster 2023-10-09T09:44:52.229699+0000 osd.6 (osd.6) 513 : cluster 0 9.19 scrub starts 2023-10-09T09:44:54.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: cluster 2023-10-09T09:44:52.244263+0000 osd.6 (osd.6) 514 : cluster 0 9.19 scrub ok 2023-10-09T09:44:54.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: cluster 2023-10-09T09:44:52.914390+0000 osd.5 (osd.5) 812 : cluster 3 failed to encode map e1227 with expected crc 2023-10-09T09:44:54.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: cluster 2023-10-09T09:44:52.914515+0000 osd.6 (osd.6) 515 : cluster 3 failed to encode map e1227 with expected crc 2023-10-09T09:44:54.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: audit 2023-10-09T09:44:52.918187+0000 mon.a (mon.0) 2208 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T09:44:54.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: cluster 2023-10-09T09:44:52.918262+0000 mon.a (mon.0) 2209 : cluster 0 osdmap e1227: 8 total, 8 up, 6 in 2023-10-09T09:44:54.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: cluster 2023-10-09T09:44:52.920058+0000 osd.1 (osd.1) 11 : cluster 0 9.12 starting backfill to osd.0 from (0'0,0'0] MAX to 1222'8534 2023-10-09T09:44:54.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: cluster 2023-10-09T09:44:52.934782+0000 osd.1 (osd.1) 12 : cluster 0 9.12 starting backfill to osd.5 from (0'0,0'0] MAX to 1222'8534 2023-10-09T09:44:54.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: cluster 2023-10-09T09:44:53.209292+0000 osd.1 (osd.1) 13 : cluster 0 9.14 scrub starts 2023-10-09T09:44:54.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[78107]: cluster 2023-10-09T09:44:53.229451+0000 osd.1 (osd.1) 14 : cluster 0 9.14 scrub ok 2023-10-09T09:44:54.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: cluster 2023-10-09T09:44:52.050329+0000 mgr.y (mgr.34107) 1165 : cluster 0 pgmap v1988: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 259 objects/s recovering 2023-10-09T09:44:54.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: cluster 2023-10-09T09:44:52.229699+0000 osd.6 (osd.6) 513 : cluster 0 9.19 scrub starts 2023-10-09T09:44:54.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: cluster 2023-10-09T09:44:52.244263+0000 osd.6 (osd.6) 514 : cluster 0 9.19 scrub ok 2023-10-09T09:44:54.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: cluster 2023-10-09T09:44:52.914390+0000 osd.5 (osd.5) 812 : cluster 3 failed to encode map e1227 with expected crc 2023-10-09T09:44:54.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: cluster 2023-10-09T09:44:52.914515+0000 osd.6 (osd.6) 515 : cluster 3 failed to encode map e1227 with expected crc 2023-10-09T09:44:54.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: audit 2023-10-09T09:44:52.918187+0000 mon.a (mon.0) 2208 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T09:44:54.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: cluster 2023-10-09T09:44:52.918262+0000 mon.a (mon.0) 2209 : cluster 0 osdmap e1227: 8 total, 8 up, 6 in 2023-10-09T09:44:54.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: cluster 2023-10-09T09:44:52.920058+0000 osd.1 (osd.1) 11 : cluster 0 9.12 starting backfill to osd.0 from (0'0,0'0] MAX to 1222'8534 2023-10-09T09:44:54.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: cluster 2023-10-09T09:44:52.934782+0000 osd.1 (osd.1) 12 : cluster 0 9.12 starting backfill to osd.5 from (0'0,0'0] MAX to 1222'8534 2023-10-09T09:44:54.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: cluster 2023-10-09T09:44:53.209292+0000 osd.1 (osd.1) 13 : cluster 0 9.14 scrub starts 2023-10-09T09:44:54.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:53 smithi163 bash[75425]: cluster 2023-10-09T09:44:53.229451+0000 osd.1 (osd.1) 14 : cluster 0 9.14 scrub ok 2023-10-09T09:44:54.062 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:54.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:54.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:54.165 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:54.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: cluster 2023-10-09T09:44:52.050329+0000 mgr.y (mgr.34107) 1165 : cluster 0 pgmap v1988: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 19 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 259 objects/s recovering 2023-10-09T09:44:54.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: cluster 2023-10-09T09:44:52.229699+0000 osd.6 (osd.6) 513 : cluster 0 9.19 scrub starts 2023-10-09T09:44:54.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: cluster 2023-10-09T09:44:52.244263+0000 osd.6 (osd.6) 514 : cluster 0 9.19 scrub ok 2023-10-09T09:44:54.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: cluster 2023-10-09T09:44:52.914390+0000 osd.5 (osd.5) 812 : cluster 3 failed to encode map e1227 with expected crc 2023-10-09T09:44:54.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: cluster 2023-10-09T09:44:52.914515+0000 osd.6 (osd.6) 515 : cluster 3 failed to encode map e1227 with expected crc 2023-10-09T09:44:54.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: audit 2023-10-09T09:44:52.918187+0000 mon.a (mon.0) 2208 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T09:44:54.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: cluster 2023-10-09T09:44:52.918262+0000 mon.a (mon.0) 2209 : cluster 0 osdmap e1227: 8 total, 8 up, 6 in 2023-10-09T09:44:54.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: cluster 2023-10-09T09:44:52.920058+0000 osd.1 (osd.1) 11 : cluster 0 9.12 starting backfill to osd.0 from (0'0,0'0] MAX to 1222'8534 2023-10-09T09:44:54.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: cluster 2023-10-09T09:44:52.934782+0000 osd.1 (osd.1) 12 : cluster 0 9.12 starting backfill to osd.5 from (0'0,0'0] MAX to 1222'8534 2023-10-09T09:44:54.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: cluster 2023-10-09T09:44:53.209292+0000 osd.1 (osd.1) 13 : cluster 0 9.14 scrub starts 2023-10-09T09:44:54.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:53 smithi181 bash[59345]: cluster 2023-10-09T09:44:53.229451+0000 osd.1 (osd.1) 14 : cluster 0 9.14 scrub ok 2023-10-09T09:44:54.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:54.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:54.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:54.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:54.325 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:54.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:54.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:54.428 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:54.460 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:54.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:54.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:54.563 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:54.594 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:54.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:54.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:54.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:54.727 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:54.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:54.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:54.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:54.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:54.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:54.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:54.938 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: cluster 2023-10-09T09:44:53.249761+0000 osd.6 (osd.6) 516 : cluster 0 9.11 scrub starts 2023-10-09T09:44:54.938 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: cluster 2023-10-09T09:44:53.265704+0000 osd.6 (osd.6) 517 : cluster 0 9.11 scrub ok 2023-10-09T09:44:54.938 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: cluster 2023-10-09T09:44:53.927064+0000 osd.5 (osd.5) 813 : cluster 3 failed to encode map e1228 with expected crc 2023-10-09T09:44:54.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: cluster 2023-10-09T09:44:53.929255+0000 mon.a (mon.0) 2210 : cluster 0 osdmap e1228: 8 total, 8 up, 6 in 2023-10-09T09:44:54.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: audit 2023-10-09T09:44:54.052848+0000 mon.a (mon.0) 2211 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T09:44:54.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.210378+0000 osd.0 (osd.0) 5 : cluster 0 9.18 scrub starts 2023-10-09T09:44:54.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.232834+0000 osd.0 (osd.0) 6 : cluster 0 9.18 scrub ok 2023-10-09T09:44:54.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.236541+0000 osd.7 (osd.7) 589 : cluster 3 failed to encode map e1228 with expected crc 2023-10-09T09:44:54.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.245418+0000 osd.1 (osd.1) 15 : cluster 0 9.4 scrub starts 2023-10-09T09:44:54.962 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:54.965 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_4' removedpool 'unique_pool_4' removed 2023-10-09T09:44:54.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:54.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:54.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: cluster 2023-10-09T09:44:53.249761+0000 osd.6 (osd.6) 516 : cluster 0 9.11 scrub starts 2023-10-09T09:44:55.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: cluster 2023-10-09T09:44:53.265704+0000 osd.6 (osd.6) 517 : cluster 0 9.11 scrub ok 2023-10-09T09:44:55.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: cluster 2023-10-09T09:44:53.927064+0000 osd.5 (osd.5) 813 : cluster 3 failed to encode map e1228 with expected crc 2023-10-09T09:44:55.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: cluster 2023-10-09T09:44:53.929255+0000 mon.a (mon.0) 2210 : cluster 0 osdmap e1228: 8 total, 8 up, 6 in 2023-10-09T09:44:55.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: audit 2023-10-09T09:44:54.052848+0000 mon.a (mon.0) 2211 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T09:44:55.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.210378+0000 osd.0 (osd.0) 5 : cluster 0 9.18 scrub starts 2023-10-09T09:44:55.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.232834+0000 osd.0 (osd.0) 6 : cluster 0 9.18 scrub ok 2023-10-09T09:44:55.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.236541+0000 osd.7 (osd.7) 589 : cluster 3 failed to encode map e1228 with expected crc 2023-10-09T09:44:55.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.245418+0000 osd.1 (osd.1) 15 : cluster 0 9.4 scrub starts 2023-10-09T09:44:55.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.266548+0000 osd.1 (osd.1) 16 : cluster 0 9.4 scrub ok 2023-10-09T09:44:55.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: audit 2023-10-09T09:44:54.455448+0000 mon.c (mon.2) 102 : audit 1 from='client.? 172.21.15.163:0/4197626968' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:44:55.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:54 smithi181 bash[59345]: audit 2023-10-09T09:44:54.456176+0000 mon.a (mon.0) 2212 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:44:55.093 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:55.115 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:55.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:55.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:55.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: cluster 2023-10-09T09:44:53.249761+0000 osd.6 (osd.6) 516 : cluster 0 9.11 scrub starts 2023-10-09T09:44:55.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: cluster 2023-10-09T09:44:53.265704+0000 osd.6 (osd.6) 517 : cluster 0 9.11 scrub ok 2023-10-09T09:44:55.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: cluster 2023-10-09T09:44:53.927064+0000 osd.5 (osd.5) 813 : cluster 3 failed to encode map e1228 with expected crc 2023-10-09T09:44:55.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: cluster 2023-10-09T09:44:53.929255+0000 mon.a (mon.0) 2210 : cluster 0 osdmap e1228: 8 total, 8 up, 6 in 2023-10-09T09:44:55.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: audit 2023-10-09T09:44:54.052848+0000 mon.a (mon.0) 2211 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T09:44:55.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.210378+0000 osd.0 (osd.0) 5 : cluster 0 9.18 scrub starts 2023-10-09T09:44:55.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.232834+0000 osd.0 (osd.0) 6 : cluster 0 9.18 scrub ok 2023-10-09T09:44:55.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.236541+0000 osd.7 (osd.7) 589 : cluster 3 failed to encode map e1228 with expected crc 2023-10-09T09:44:55.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.245418+0000 osd.1 (osd.1) 15 : cluster 0 9.4 scrub starts 2023-10-09T09:44:55.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.266548+0000 osd.1 (osd.1) 16 : cluster 0 9.4 scrub ok 2023-10-09T09:44:55.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: audit 2023-10-09T09:44:54.455448+0000 mon.c (mon.2) 102 : audit 1 from='client.? 172.21.15.163:0/4197626968' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:44:55.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[78107]: audit 2023-10-09T09:44:54.456176+0000 mon.a (mon.0) 2212 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:44:55.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.266548+0000 osd.1 (osd.1) 16 : cluster 0 9.4 scrub ok 2023-10-09T09:44:55.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: audit 2023-10-09T09:44:54.455448+0000 mon.c (mon.2) 102 : audit 1 from='client.? 172.21.15.163:0/4197626968' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:44:55.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:54 smithi163 bash[75425]: audit 2023-10-09T09:44:54.456176+0000 mon.a (mon.0) 2212 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:44:55.219 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:55.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:55.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:55.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:55.378 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:55.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:55.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:55.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:55.507 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:55.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:55.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:55.610 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:55.642 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:55.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:55.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:55.745 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:55.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:55.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:55.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:55.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:55.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:55.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:55.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:56.015 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:56.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:56.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:56.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:56.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:56.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:56.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:56.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:56.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.002040+0000 osd.4 (osd.4) 634 : cluster 3 failed to encode map e1228 with expected crc 2023-10-09T09:44:56.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.051255+0000 mgr.y (mgr.34107) 1166 : cluster 0 pgmap v1991: 99 pgs: 1 active+clean+remapped, 98 active+clean; 16 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:44:56.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.924025+0000 mon.a (mon.0) 2213 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:44:56.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.936614+0000 osd.5 (osd.5) 814 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:56.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[75425]: audit 2023-10-09T09:44:54.941106+0000 mon.a (mon.0) 2214 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T09:44:56.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[75425]: audit 2023-10-09T09:44:54.941222+0000 mon.a (mon.0) 2215 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:44:56.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.941330+0000 mon.a (mon.0) 2216 : cluster 0 osdmap e1229: 8 total, 8 up, 6 in 2023-10-09T09:44:56.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[75425]: cluster 2023-10-09T09:44:54.942938+0000 osd.6 (osd.6) 518 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:56.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[75425]: audit 2023-10-09T09:44:55.394789+0000 mon.a (mon.0) 2217 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:56.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[75425]: audit 2023-10-09T09:44:55.396550+0000 mon.a (mon.0) 2218 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:56.186 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.002040+0000 osd.4 (osd.4) 634 : cluster 3 failed to encode map e1228 with expected crc 2023-10-09T09:44:56.186 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.051255+0000 mgr.y (mgr.34107) 1166 : cluster 0 pgmap v1991: 99 pgs: 1 active+clean+remapped, 98 active+clean; 16 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:44:56.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.924025+0000 mon.a (mon.0) 2213 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:44:56.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.936614+0000 osd.5 (osd.5) 814 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:56.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[78107]: audit 2023-10-09T09:44:54.941106+0000 mon.a (mon.0) 2214 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T09:44:56.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[78107]: audit 2023-10-09T09:44:54.941222+0000 mon.a (mon.0) 2215 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:44:56.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.941330+0000 mon.a (mon.0) 2216 : cluster 0 osdmap e1229: 8 total, 8 up, 6 in 2023-10-09T09:44:56.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[78107]: cluster 2023-10-09T09:44:54.942938+0000 osd.6 (osd.6) 518 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:56.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[78107]: audit 2023-10-09T09:44:55.394789+0000 mon.a (mon.0) 2217 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:56.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:55 smithi163 bash[78107]: audit 2023-10-09T09:44:55.396550+0000 mon.a (mon.0) 2218 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:56.189 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:44:56.190 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:44:56.191 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T09:44:56.191 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T09:44:56.284 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:56.298 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:55 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.002040+0000 osd.4 (osd.4) 634 : cluster 3 failed to encode map e1228 with expected crc 2023-10-09T09:44:56.298 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:55 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.051255+0000 mgr.y (mgr.34107) 1166 : cluster 0 pgmap v1991: 99 pgs: 1 active+clean+remapped, 98 active+clean; 16 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:44:56.298 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:55 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.924025+0000 mon.a (mon.0) 2213 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:44:56.298 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:55 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.936614+0000 osd.5 (osd.5) 814 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:56.298 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:55 smithi181 bash[59345]: audit 2023-10-09T09:44:54.941106+0000 mon.a (mon.0) 2214 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T09:44:56.298 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:55 smithi181 bash[59345]: audit 2023-10-09T09:44:54.941222+0000 mon.a (mon.0) 2215 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:44:56.298 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:55 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.941330+0000 mon.a (mon.0) 2216 : cluster 0 osdmap e1229: 8 total, 8 up, 6 in 2023-10-09T09:44:56.298 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:55 smithi181 bash[59345]: cluster 2023-10-09T09:44:54.942938+0000 osd.6 (osd.6) 518 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:56.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:55 smithi181 bash[59345]: audit 2023-10-09T09:44:55.394789+0000 mon.a (mon.0) 2217 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:44:56.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:55 smithi181 bash[59345]: audit 2023-10-09T09:44:55.396550+0000 mon.a (mon.0) 2218 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:44:56.321 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:56.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:56.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:56.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:56.458 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:56.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:56.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:56.562 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:56.594 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:56.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:56.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:56.663 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:44:56.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:56.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:56.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:56.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:56.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:56.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:56.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:56.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:56.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:56.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:56 smithi163 bash[75425]: cluster 2023-10-09T09:44:55.170283+0000 osd.4 (osd.4) 635 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:56.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:56 smithi163 bash[75425]: cluster 2023-10-09T09:44:56.607579+0000 osd.7 (osd.7) 590 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:56.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:56 smithi163 bash[78107]: cluster 2023-10-09T09:44:55.170283+0000 osd.4 (osd.4) 635 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:56.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:56 smithi163 bash[78107]: cluster 2023-10-09T09:44:56.607579+0000 osd.7 (osd.7) 590 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:56.997 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:56.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:57.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:44:57.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:57.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:56 smithi181 bash[59345]: cluster 2023-10-09T09:44:55.170283+0000 osd.4 (osd.4) 635 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:57.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:56 smithi181 bash[59345]: cluster 2023-10-09T09:44:56.607579+0000 osd.7 (osd.7) 590 : cluster 3 failed to encode map e1229 with expected crc 2023-10-09T09:44:57.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:57.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:57.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:57.239 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:57.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:57.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:57.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:57.380 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:57.409 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:57.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:57.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:57.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:57.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:57.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:57.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:57.650 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:57.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:44:57.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:57.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:44:57.781 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:57.818 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:57.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:57.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:57.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:57.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:57.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:58.055 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:58.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:57 smithi181 bash[59345]: cluster 2023-10-09T09:44:56.052051+0000 mgr.y (mgr.34107) 1167 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:44:58.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:58.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:58.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:58.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:57 smithi163 bash[78107]: cluster 2023-10-09T09:44:56.052051+0000 mgr.y (mgr.34107) 1167 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:44:58.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:57 smithi163 bash[75425]: cluster 2023-10-09T09:44:56.052051+0000 mgr.y (mgr.34107) 1167 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:44:58.225 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:58.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:58.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:58.329 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:58.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:58.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:58.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:58.494 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:58.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:58.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:58.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:44:58.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:58.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:44:58.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:58.768 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:58.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:58.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:58.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:58.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:44:58.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:58.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:44:59.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:59.038 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:59.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:59.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:59.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:59.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:59.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:59.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:59.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:59.307 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:44:59.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:59.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:44:59.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:59.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:59.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:59.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:59.547 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:59.567 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:44:59.567 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:44:59.568 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:44:59.582 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:44:59.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:59.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:44:59.685 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:59.716 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:44:59.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:59.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:44:59.820 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:59.845 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:44:59.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:59.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:44:59.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:59.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:44:59.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:44:59.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:00.075 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:00.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:00.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:00.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:00.210 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:00.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:44:59 smithi163 bash[75425]: cluster 2023-10-09T09:44:58.053137+0000 mgr.y (mgr.34107) 1168 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:44:59 smithi163 bash[78107]: cluster 2023-10-09T09:44:58.053137+0000 mgr.y (mgr.34107) 1168 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:00.223 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:44:59 smithi181 bash[59345]: cluster 2023-10-09T09:44:58.053137+0000 mgr.y (mgr.34107) 1168 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:00.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:00.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:00.347 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:00.382 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:00.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:00.486 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:00.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:00.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:00.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:00.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:00.633 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:00.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:00.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:00.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:00.760 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:00.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:00.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:00.863 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:00.895 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:00.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:00.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:00.999 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:01.030 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:01.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:01.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:01.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:01.156 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:01.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:01.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:01.260 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:01.291 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:01.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:01.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:01.395 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:01.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:01.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:01.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:01.532 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:01.568 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:01.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:01.671 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:01.698 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:01.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:01.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:01.823 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:01.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:01.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:01.957 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:01.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:01.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:02.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:02.089 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:02.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:02.132 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:02.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:02.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:01 smithi163 bash[78107]: cluster 2023-10-09T09:45:00.054078+0000 mgr.y (mgr.34107) 1169 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:02.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:01 smithi163 bash[78107]: audit 2023-10-09T09:45:01.064807+0000 mon.a (mon.0) 2219 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:02.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:01 smithi163 bash[75425]: cluster 2023-10-09T09:45:00.054078+0000 mgr.y (mgr.34107) 1169 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:02.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:01 smithi163 bash[75425]: audit 2023-10-09T09:45:01.064807+0000 mon.a (mon.0) 2219 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:02.221 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:02.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:02.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:01 smithi181 bash[59345]: cluster 2023-10-09T09:45:00.054078+0000 mgr.y (mgr.34107) 1169 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:02.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:01 smithi181 bash[59345]: audit 2023-10-09T09:45:01.064807+0000 mon.a (mon.0) 2219 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:02.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:02.337 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:45:02.337 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:45:02.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:02.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:02.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:02.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:02.489 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:02.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:02.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:02.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:02.627 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:02.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:02.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:02.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:02.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:02.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:02.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:02.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:02.903 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:02.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:02.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:03.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:03.033 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:03.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:03.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:03.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:03.170 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:03.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:03.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:03.274 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:03.305 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:03.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:03.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:03.409 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:03.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:03.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:03.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:03.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:03.569 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:03.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:03.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:03.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:03.698 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:03.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:03.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:03.775 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:45:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:45:03] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.33.4" 2023-10-09T09:45:03.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:03.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:03.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:03.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:03.932 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:03.955 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:03.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:03.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:04.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:04.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:04.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:04.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:04.202 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:04.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:04.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:04.305 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:04.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:03 smithi181 bash[59345]: cluster 2023-10-09T09:45:02.054767+0000 mgr.y (mgr.34107) 1170 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:04.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:04 smithi163 bash[75425]: cluster 2023-10-09T09:45:02.054767+0000 mgr.y (mgr.34107) 1170 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:04.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:04 smithi163 bash[78107]: cluster 2023-10-09T09:45:02.054767+0000 mgr.y (mgr.34107) 1170 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:04.327 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:04.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:04.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:04.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:04.454 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:04.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:04.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:04.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:04.580 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:04.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:04.682 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:04.716 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:04.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:04.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:04.820 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:04.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:04.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:04.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:04.950 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:04.981 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:04.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:04.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:05.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:05.114 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:05.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:05.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:05.217 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:05.251 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:05.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:05.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:05.355 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:05.387 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:05.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:05.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:05.492 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:05.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:05.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:05.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:05.619 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:05.647 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:05.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:05.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:05.750 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:05.775 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:05.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:05.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:05.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:05.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:05.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:05.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:06.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:06.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:06.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:06.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:06.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:06.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:06.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:06.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:05 smithi163 bash[75425]: cluster 2023-10-09T09:45:04.055403+0000 mgr.y (mgr.34107) 1171 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:05 smithi163 bash[78107]: cluster 2023-10-09T09:45:04.055403+0000 mgr.y (mgr.34107) 1171 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:06.285 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:06.292 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:05 smithi181 bash[59345]: cluster 2023-10-09T09:45:04.055403+0000 mgr.y (mgr.34107) 1171 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:06.313 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:06.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:06.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:06.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:06.449 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:06.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:06.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:06.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:06.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:06.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:06.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:06.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:06.717 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:06.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:06.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T09:45:06.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:06.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:06.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:06.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:06.954 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:06.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:06.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:07.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:07.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:07.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:07.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:07.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:07.252 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:07.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:07.355 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:07.382 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:07.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:07.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:07.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:07.546 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:07.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:07.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:07.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:07.678 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:07.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:07.782 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:07.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:07.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:07.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:07.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:07.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:07.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:07.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:08.031 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:08.063 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:08.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:08.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:08.159 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:07 smithi163 bash[75425]: cluster 2023-10-09T09:45:06.055917+0000 mgr.y (mgr.34107) 1172 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:08.159 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:07 smithi163 bash[78107]: cluster 2023-10-09T09:45:06.055917+0000 mgr.y (mgr.34107) 1172 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:08.167 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:08.192 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:08.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:08.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:08.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:08.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:07 smithi181 bash[59345]: cluster 2023-10-09T09:45:06.055917+0000 mgr.y (mgr.34107) 1172 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:08.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:08.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:08.428 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:08.459 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:08.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:08.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:08.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:08.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:08.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:08.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:08.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:08.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:08.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:08.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:08.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:08.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:08.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:08.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:09.002 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:09.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:09.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:09.106 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:09.137 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:09.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:09.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:09.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:09.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:09.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:09.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:09.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:09.399 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:09.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:09.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:09.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:09.536 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:09.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:09.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:09.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:09.668 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:09.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:09.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:09.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:09.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:09.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:09.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:09.901 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:09.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:09.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:09.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:10.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:10.059 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:10.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:10.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:10.162 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:10.185 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:10.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:10.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:10.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:10.302 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:09 smithi181 bash[59345]: cluster 2023-10-09T09:45:08.056787+0000 mgr.y (mgr.34107) 1173 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:10.321 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:10.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:10.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:10.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:10.433 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:09 smithi163 bash[75425]: cluster 2023-10-09T09:45:08.056787+0000 mgr.y (mgr.34107) 1173 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:10.434 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:09 smithi163 bash[78107]: cluster 2023-10-09T09:45:08.056787+0000 mgr.y (mgr.34107) 1173 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:10.451 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:10.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:10.554 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:10.583 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:10.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:10.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:10.687 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:10.718 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:10.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:10.822 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:10.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:10.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:10.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:10.950 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:10.983 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:10.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:10.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:11.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:11.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:11.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:11.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:11.225 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:11.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:11.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:11.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:11.270 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:11 smithi163 bash[78107]: audit 2023-10-09T09:45:10.391125+0000 mon.a (mon.0) 2220 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:11.270 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:10 smithi163 bash[75425]: audit 2023-10-09T09:45:10.391125+0000 mon.a (mon.0) 2220 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:10 smithi181 bash[59345]: audit 2023-10-09T09:45:10.391125+0000 mon.a (mon.0) 2220 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:11.349 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:11.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:11.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:11.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:11.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:11.509 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:11.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:11.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:11.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:11.651 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:11.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:11.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:11.755 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:11.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:11.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:11.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:11.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:11.905 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:11.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:11.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:12.007 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:12.041 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:12.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:12.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:12.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:12.176 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:12.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:12.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:11 smithi163 bash[75425]: cluster 2023-10-09T09:45:10.057377+0000 mgr.y (mgr.34107) 1174 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:12.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:11 smithi163 bash[78107]: cluster 2023-10-09T09:45:10.057377+0000 mgr.y (mgr.34107) 1174 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:12.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:12.303 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:12.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:12.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:11 smithi181 bash[59345]: cluster 2023-10-09T09:45:10.057377+0000 mgr.y (mgr.34107) 1174 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:12.407 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:12.432 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:12.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:12.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:12.535 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:12.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:12.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:12.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:12.671 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:12.699 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:12.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:12.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:12.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:12.824 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:12.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:12.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:12.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:12.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:12.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:12.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:13.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:13.074 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:13.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:13.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:13.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:13.209 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:13.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:13.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:13.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:13 smithi163 bash[78107]: cluster 2023-10-09T09:45:12.058181+0000 mgr.y (mgr.34107) 1175 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:13.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:12 smithi163 bash[75425]: cluster 2023-10-09T09:45:12.058181+0000 mgr.y (mgr.34107) 1175 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:13.312 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:13.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:12 smithi181 bash[59345]: cluster 2023-10-09T09:45:12.058181+0000 mgr.y (mgr.34107) 1175 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:13.344 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:13.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:13.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:13.448 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:13.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:13.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:13.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:13.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:13.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:13.611 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:45:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:45:13] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.33.4" 2023-10-09T09:45:13.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:13.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:13.714 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:13.741 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:13.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:13.844 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:13.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:13.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:13.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:13.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:14.000 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:14.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:14.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:14.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:14.126 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:14.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:14.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:14.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:14.247 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:14.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:14.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:14.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:14.382 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:14.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:14.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:14.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:14.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:14.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:14.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:14.643 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:14.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:14.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:14.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:14.779 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:14.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:14.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:14.882 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:14.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:14.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:14.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:15.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:15.051 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:15.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:15.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:15.133 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:45:15.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:15.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:15 smithi181 bash[59345]: cluster 2023-10-09T09:45:14.059115+0000 mgr.y (mgr.34107) 1176 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:15.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:15 smithi181 bash[59345]: audit 2023-10-09T09:45:14.302159+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.163:0/1949149232' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:45:15.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:15 smithi181 bash[59345]: audit 2023-10-09T09:45:14.302673+0000 mon.a (mon.0) 2221 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:45:15.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:15.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:15.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:15.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:15.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:15.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:15.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:15.418 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:15 smithi163 bash[78107]: cluster 2023-10-09T09:45:14.059115+0000 mgr.y (mgr.34107) 1176 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:15.419 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:15 smithi163 bash[78107]: audit 2023-10-09T09:45:14.302159+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.163:0/1949149232' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:45:15.419 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:15 smithi163 bash[78107]: audit 2023-10-09T09:45:14.302673+0000 mon.a (mon.0) 2221 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:45:15.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:15 smithi163 bash[75425]: cluster 2023-10-09T09:45:14.059115+0000 mgr.y (mgr.34107) 1176 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:15.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:15 smithi163 bash[75425]: audit 2023-10-09T09:45:14.302159+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.163:0/1949149232' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:45:15.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:15 smithi163 bash[75425]: audit 2023-10-09T09:45:14.302673+0000 mon.a (mon.0) 2221 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:45:15.423 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:15.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:15.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:15.658 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:45:15.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:15.674 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:45:15.695 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:45:15.723 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:45:15.750 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:45:15.759 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:15.777 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:45:15.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:15.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:15.802 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:45:15.829 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:45:15.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:45:15.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:45:15.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:45:15.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:45:15.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:45:15.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:45:15.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:45:15.832 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:45:15.893 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:15.917 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:15.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:15.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:16.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:16.048 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:16.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:16.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:16.151 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:16.183 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:16.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:16.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:16.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[75425]: cluster 2023-10-09T09:45:15.112428+0000 mon.a (mon.0) 2222 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:45:16.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[75425]: cluster 2023-10-09T09:45:15.112455+0000 mon.a (mon.0) 2223 : cluster 1 Cluster is now healthy 2023-10-09T09:45:16.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[75425]: cluster 2023-10-09T09:45:15.124324+0000 osd.5 (osd.5) 815 : cluster 3 failed to encode map e1230 with expected crc 2023-10-09T09:45:16.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[75425]: cluster 2023-10-09T09:45:15.124429+0000 osd.4 (osd.4) 636 : cluster 3 failed to encode map e1230 with expected crc 2023-10-09T09:45:16.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[75425]: audit 2023-10-09T09:45:15.126246+0000 mon.a (mon.0) 2224 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:45:16.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[75425]: cluster 2023-10-09T09:45:15.126321+0000 mon.a (mon.0) 2225 : cluster 0 osdmap e1230: 8 total, 8 up, 6 in 2023-10-09T09:45:16.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[75425]: audit 2023-10-09T09:45:15.624224+0000 mon.a (mon.0) 2226 : audit 0 from='client.? 172.21.15.163:0/3551525598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:45:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[78107]: cluster 2023-10-09T09:45:15.112428+0000 mon.a (mon.0) 2222 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:45:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[78107]: cluster 2023-10-09T09:45:15.112455+0000 mon.a (mon.0) 2223 : cluster 1 Cluster is now healthy 2023-10-09T09:45:16.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[78107]: cluster 2023-10-09T09:45:15.124324+0000 osd.5 (osd.5) 815 : cluster 3 failed to encode map e1230 with expected crc 2023-10-09T09:45:16.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[78107]: cluster 2023-10-09T09:45:15.124429+0000 osd.4 (osd.4) 636 : cluster 3 failed to encode map e1230 with expected crc 2023-10-09T09:45:16.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[78107]: audit 2023-10-09T09:45:15.126246+0000 mon.a (mon.0) 2224 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:45:16.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[78107]: cluster 2023-10-09T09:45:15.126321+0000 mon.a (mon.0) 2225 : cluster 0 osdmap e1230: 8 total, 8 up, 6 in 2023-10-09T09:45:16.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:16 smithi163 bash[78107]: audit 2023-10-09T09:45:15.624224+0000 mon.a (mon.0) 2226 : audit 0 from='client.? 172.21.15.163:0/3551525598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:45:16.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:16.311 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:16.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:16.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:16.415 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:16.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:16.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:16.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:16.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:16 smithi181 bash[59345]: cluster 2023-10-09T09:45:15.112428+0000 mon.a (mon.0) 2222 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:45:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:16 smithi181 bash[59345]: cluster 2023-10-09T09:45:15.112455+0000 mon.a (mon.0) 2223 : cluster 1 Cluster is now healthy 2023-10-09T09:45:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:16 smithi181 bash[59345]: cluster 2023-10-09T09:45:15.124324+0000 osd.5 (osd.5) 815 : cluster 3 failed to encode map e1230 with expected crc 2023-10-09T09:45:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:16 smithi181 bash[59345]: cluster 2023-10-09T09:45:15.124429+0000 osd.4 (osd.4) 636 : cluster 3 failed to encode map e1230 with expected crc 2023-10-09T09:45:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:16 smithi181 bash[59345]: audit 2023-10-09T09:45:15.126246+0000 mon.a (mon.0) 2224 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:45:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:16 smithi181 bash[59345]: cluster 2023-10-09T09:45:15.126321+0000 mon.a (mon.0) 2225 : cluster 0 osdmap e1230: 8 total, 8 up, 6 in 2023-10-09T09:45:16.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:16 smithi181 bash[59345]: audit 2023-10-09T09:45:15.624224+0000 mon.a (mon.0) 2226 : audit 0 from='client.? 172.21.15.163:0/3551525598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:45:16.567 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:16.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:16.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:16.670 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:16.703 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:16.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:16.807 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:16.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:16.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:16.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:16.933 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:16.967 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:16.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:16.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:17.071 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:17.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:17.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:17.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:17.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:17.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:17.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:17.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:17.343 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:17.361 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:17.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:17.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:17.486 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:17.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:17.589 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:17.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:17.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:17.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:17.714 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:17.738 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:17.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:17.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:17.840 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T09:45:17.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:17.863 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:17.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:17.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:17.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:17 smithi163 bash[78107]: cluster 2023-10-09T09:45:16.059744+0000 mgr.y (mgr.34107) 1177 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:17.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:17 smithi163 bash[78107]: cluster 2023-10-09T09:45:16.340627+0000 osd.6 (osd.6) 519 : cluster 3 failed to encode map e1230 with expected crc 2023-10-09T09:45:17.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:17 smithi163 bash[78107]: audit 2023-10-09T09:45:17.606756+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.163:0/4049661636' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:45:17.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:17 smithi163 bash[78107]: audit 2023-10-09T09:45:17.607294+0000 mon.a (mon.0) 2227 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:45:17.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:17 smithi163 bash[75425]: cluster 2023-10-09T09:45:16.059744+0000 mgr.y (mgr.34107) 1177 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:17.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:17 smithi163 bash[75425]: cluster 2023-10-09T09:45:16.340627+0000 osd.6 (osd.6) 519 : cluster 3 failed to encode map e1230 with expected crc 2023-10-09T09:45:17.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:17 smithi163 bash[75425]: audit 2023-10-09T09:45:17.606756+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.163:0/4049661636' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:45:17.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:17 smithi163 bash[75425]: audit 2023-10-09T09:45:17.607294+0000 mon.a (mon.0) 2227 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:45:17.996 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:17.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:18.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:18.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:17 smithi181 bash[59345]: cluster 2023-10-09T09:45:16.059744+0000 mgr.y (mgr.34107) 1177 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:18.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:17 smithi181 bash[59345]: cluster 2023-10-09T09:45:16.340627+0000 osd.6 (osd.6) 519 : cluster 3 failed to encode map e1230 with expected crc 2023-10-09T09:45:18.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:17 smithi181 bash[59345]: audit 2023-10-09T09:45:17.606756+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.163:0/4049661636' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:45:18.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:17 smithi181 bash[59345]: audit 2023-10-09T09:45:17.607294+0000 mon.a (mon.0) 2227 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:45:18.134 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:18.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:18.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:18.238 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:18.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:18.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:18.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:18.363 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:18.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:18.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:18.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:18.501 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:18.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:18.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:18.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:18.636 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:18.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:18.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:18.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:18.767 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:18.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:18.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:18.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:18.868 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_5' created 2023-10-09T09:45:18.896 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:18 smithi163 bash[75425]: cluster 2023-10-09T09:45:17.830602+0000 osd.5 (osd.5) 816 : cluster 3 failed to encode map e1231 with expected crc 2023-10-09T09:45:18.896 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:18 smithi163 bash[75425]: audit 2023-10-09T09:45:17.832772+0000 mon.a (mon.0) 2228 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:45:18.896 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:18 smithi163 bash[75425]: cluster 2023-10-09T09:45:17.832855+0000 mon.a (mon.0) 2229 : cluster 0 osdmap e1231: 8 total, 8 up, 6 in 2023-10-09T09:45:18.896 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:18 smithi163 bash[75425]: audit 2023-10-09T09:45:17.888473+0000 mon.a (mon.0) 2230 : audit 1 from='client.? 172.21.15.163:0/94414986' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T09:45:18.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:18 smithi163 bash[78107]: cluster 2023-10-09T09:45:17.830602+0000 osd.5 (osd.5) 816 : cluster 3 failed to encode map e1231 with expected crc 2023-10-09T09:45:18.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:18 smithi163 bash[78107]: audit 2023-10-09T09:45:17.832772+0000 mon.a (mon.0) 2228 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:45:18.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:18 smithi163 bash[78107]: cluster 2023-10-09T09:45:17.832855+0000 mon.a (mon.0) 2229 : cluster 0 osdmap e1231: 8 total, 8 up, 6 in 2023-10-09T09:45:18.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:18 smithi163 bash[78107]: audit 2023-10-09T09:45:17.888473+0000 mon.a (mon.0) 2230 : audit 1 from='client.? 172.21.15.163:0/94414986' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T09:45:18.906 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:18.940 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:18.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:19.043 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:19.054 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:18 smithi181 bash[59345]: cluster 2023-10-09T09:45:17.830602+0000 osd.5 (osd.5) 816 : cluster 3 failed to encode map e1231 with expected crc 2023-10-09T09:45:19.054 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:18 smithi181 bash[59345]: audit 2023-10-09T09:45:17.832772+0000 mon.a (mon.0) 2228 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:45:19.055 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:18 smithi181 bash[59345]: cluster 2023-10-09T09:45:17.832855+0000 mon.a (mon.0) 2229 : cluster 0 osdmap e1231: 8 total, 8 up, 6 in 2023-10-09T09:45:19.055 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:18 smithi181 bash[59345]: audit 2023-10-09T09:45:17.888473+0000 mon.a (mon.0) 2230 : audit 1 from='client.? 172.21.15.163:0/94414986' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T09:45:19.075 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:19.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:19.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:19.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T09:45:19.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:19.227 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:19.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:19.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:19.330 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:19.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:19.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:19.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:19.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:19.499 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:19.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:19.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:19.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:19.632 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:19.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:19.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:19.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:19.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:19.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:19.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:19.834 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:45:19.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:19.907 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:19.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:19.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T09:45:20.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:20.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:19 smithi181 bash[59345]: cluster 2023-10-09T09:45:18.060330+0000 mgr.y (mgr.34107) 1178 : cluster 0 pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:20.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:19 smithi181 bash[59345]: cluster 2023-10-09T09:45:18.855264+0000 osd.5 (osd.5) 817 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:19 smithi181 bash[59345]: cluster 2023-10-09T09:45:18.858602+0000 osd.5 (osd.5) 818 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:19 smithi181 bash[59345]: cluster 2023-10-09T09:45:18.858621+0000 osd.4 (osd.4) 637 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:19 smithi181 bash[59345]: audit 2023-10-09T09:45:18.860217+0000 mon.a (mon.0) 2231 : audit 1 from='client.? 172.21.15.163:0/94414986' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T09:45:20.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:19 smithi181 bash[59345]: cluster 2023-10-09T09:45:18.860432+0000 mon.a (mon.0) 2232 : cluster 0 osdmap e1232: 8 total, 8 up, 6 in 2023-10-09T09:45:20.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:19 smithi181 bash[59345]: cluster 2023-10-09T09:45:18.861337+0000 osd.4 (osd.4) 638 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:19 smithi181 bash[59345]: cluster 2023-10-09T09:45:18.861430+0000 osd.5 (osd.5) 819 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:20.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:20.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:20.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:20.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:20.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:20.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:20.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:45:20.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[78107]: cluster 2023-10-09T09:45:18.060330+0000 mgr.y (mgr.34107) 1178 : cluster 0 pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:20.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[78107]: cluster 2023-10-09T09:45:18.855264+0000 osd.5 (osd.5) 817 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[78107]: cluster 2023-10-09T09:45:18.858602+0000 osd.5 (osd.5) 818 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[78107]: cluster 2023-10-09T09:45:18.858621+0000 osd.4 (osd.4) 637 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[78107]: audit 2023-10-09T09:45:18.860217+0000 mon.a (mon.0) 2231 : audit 1 from='client.? 172.21.15.163:0/94414986' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T09:45:20.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[78107]: cluster 2023-10-09T09:45:18.860432+0000 mon.a (mon.0) 2232 : cluster 0 osdmap e1232: 8 total, 8 up, 6 in 2023-10-09T09:45:20.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[78107]: cluster 2023-10-09T09:45:18.861337+0000 osd.4 (osd.4) 638 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[78107]: cluster 2023-10-09T09:45:18.861430+0000 osd.5 (osd.5) 819 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[75425]: cluster 2023-10-09T09:45:18.060330+0000 mgr.y (mgr.34107) 1178 : cluster 0 pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:20.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[75425]: cluster 2023-10-09T09:45:18.855264+0000 osd.5 (osd.5) 817 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[75425]: cluster 2023-10-09T09:45:18.858602+0000 osd.5 (osd.5) 818 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[75425]: cluster 2023-10-09T09:45:18.858621+0000 osd.4 (osd.4) 637 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[75425]: audit 2023-10-09T09:45:18.860217+0000 mon.a (mon.0) 2231 : audit 1 from='client.? 172.21.15.163:0/94414986' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T09:45:20.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[75425]: cluster 2023-10-09T09:45:18.860432+0000 mon.a (mon.0) 2232 : cluster 0 osdmap e1232: 8 total, 8 up, 6 in 2023-10-09T09:45:20.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[75425]: cluster 2023-10-09T09:45:18.861337+0000 osd.4 (osd.4) 638 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:19 smithi163 bash[75425]: cluster 2023-10-09T09:45:18.861430+0000 osd.5 (osd.5) 819 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:20.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:20.312 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:20.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:20.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:20.360 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:20.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:20.442 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:20.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:20.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:20.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:20.576 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:20.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:20.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:20.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:20.710 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:20.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:20.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:20.815 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:20.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:20.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:20.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:20.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:20.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:20.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:21.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[75425]: cluster 2023-10-09T09:45:18.985640+0000 osd.7 (osd.7) 591 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[75425]: cluster 2023-10-09T09:45:18.988601+0000 osd.7 (osd.7) 592 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[75425]: cluster 2023-10-09T09:45:18.990935+0000 osd.7 (osd.7) 593 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[75425]: cluster 2023-10-09T09:45:19.355405+0000 osd.6 (osd.6) 520 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[75425]: cluster 2023-10-09T09:45:19.857546+0000 osd.6 (osd.6) 521 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[75425]: cluster 2023-10-09T09:45:19.857633+0000 osd.5 (osd.5) 820 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[75425]: cluster 2023-10-09T09:45:19.861358+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e1233: 8 total, 8 up, 6 in 2023-10-09T09:45:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[75425]: cluster 2023-10-09T09:45:19.861888+0000 osd.6 (osd.6) 522 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[78107]: cluster 2023-10-09T09:45:18.985640+0000 osd.7 (osd.7) 591 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[78107]: cluster 2023-10-09T09:45:18.988601+0000 osd.7 (osd.7) 592 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[78107]: cluster 2023-10-09T09:45:18.990935+0000 osd.7 (osd.7) 593 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[78107]: cluster 2023-10-09T09:45:19.355405+0000 osd.6 (osd.6) 520 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[78107]: cluster 2023-10-09T09:45:19.857546+0000 osd.6 (osd.6) 521 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[78107]: cluster 2023-10-09T09:45:19.857633+0000 osd.5 (osd.5) 820 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[78107]: cluster 2023-10-09T09:45:19.861358+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e1233: 8 total, 8 up, 6 in 2023-10-09T09:45:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:20 smithi163 bash[78107]: cluster 2023-10-09T09:45:19.861888+0000 osd.6 (osd.6) 522 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.103 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:21.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:21.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:21.206 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:21.219 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:20 smithi181 bash[59345]: cluster 2023-10-09T09:45:18.985640+0000 osd.7 (osd.7) 591 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.219 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:20 smithi181 bash[59345]: cluster 2023-10-09T09:45:18.988601+0000 osd.7 (osd.7) 592 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.219 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:20 smithi181 bash[59345]: cluster 2023-10-09T09:45:18.990935+0000 osd.7 (osd.7) 593 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.219 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:20 smithi181 bash[59345]: cluster 2023-10-09T09:45:19.355405+0000 osd.6 (osd.6) 520 : cluster 3 failed to encode map e1232 with expected crc 2023-10-09T09:45:21.219 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:20 smithi181 bash[59345]: cluster 2023-10-09T09:45:19.857546+0000 osd.6 (osd.6) 521 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:20 smithi181 bash[59345]: cluster 2023-10-09T09:45:19.857633+0000 osd.5 (osd.5) 820 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:20 smithi181 bash[59345]: cluster 2023-10-09T09:45:19.861358+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e1233: 8 total, 8 up, 6 in 2023-10-09T09:45:21.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:20 smithi181 bash[59345]: cluster 2023-10-09T09:45:19.861888+0000 osd.6 (osd.6) 522 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.238 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:21.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:21.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:21.342 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:21.377 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:21.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:21.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:21.480 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:21.514 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:21.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:21.617 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:21.652 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:21.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:21.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:21.756 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:21.787 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:21.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:21.891 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:21.903 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:21 smithi181 bash[59345]: cluster 2023-10-09T09:45:20.060884+0000 mgr.y (mgr.34107) 1179 : cluster 0 pgmap v2009: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:21.903 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:21 smithi181 bash[59345]: cluster 2023-10-09T09:45:20.086652+0000 osd.7 (osd.7) 594 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.903 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:21 smithi181 bash[59345]: cluster 2023-10-09T09:45:20.089439+0000 osd.7 (osd.7) 595 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.903 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:21 smithi181 bash[59345]: cluster 2023-10-09T09:45:20.765965+0000 osd.4 (osd.4) 639 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:21.903 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:21 smithi181 bash[59345]: cluster 2023-10-09T09:45:20.877780+0000 osd.5 (osd.5) 821 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:21.904 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:21 smithi181 bash[59345]: cluster 2023-10-09T09:45:20.882104+0000 mon.a (mon.0) 2234 : cluster 0 osdmap e1234: 8 total, 8 up, 6 in 2023-10-09T09:45:21.919 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:21.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:22.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:22.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[75425]: cluster 2023-10-09T09:45:20.060884+0000 mgr.y (mgr.34107) 1179 : cluster 0 pgmap v2009: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:22.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[75425]: cluster 2023-10-09T09:45:20.086652+0000 osd.7 (osd.7) 594 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:22.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[75425]: cluster 2023-10-09T09:45:20.089439+0000 osd.7 (osd.7) 595 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:22.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[75425]: cluster 2023-10-09T09:45:20.765965+0000 osd.4 (osd.4) 639 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:22.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[75425]: cluster 2023-10-09T09:45:20.877780+0000 osd.5 (osd.5) 821 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:22.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[75425]: cluster 2023-10-09T09:45:20.882104+0000 mon.a (mon.0) 2234 : cluster 0 osdmap e1234: 8 total, 8 up, 6 in 2023-10-09T09:45:22.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[78107]: cluster 2023-10-09T09:45:20.060884+0000 mgr.y (mgr.34107) 1179 : cluster 0 pgmap v2009: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:22.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[78107]: cluster 2023-10-09T09:45:20.086652+0000 osd.7 (osd.7) 594 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:22.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[78107]: cluster 2023-10-09T09:45:20.089439+0000 osd.7 (osd.7) 595 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:22.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[78107]: cluster 2023-10-09T09:45:20.765965+0000 osd.4 (osd.4) 639 : cluster 3 failed to encode map e1233 with expected crc 2023-10-09T09:45:22.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[78107]: cluster 2023-10-09T09:45:20.877780+0000 osd.5 (osd.5) 821 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:22.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:21 smithi163 bash[78107]: cluster 2023-10-09T09:45:20.882104+0000 mon.a (mon.0) 2234 : cluster 0 osdmap e1234: 8 total, 8 up, 6 in 2023-10-09T09:45:22.047 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:22.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:22.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:22.151 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:22.181 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:22.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:22.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:22.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:22.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:22.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:22.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:22.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:22.444 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:22.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:22.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:22.579 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:22.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:22.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:22.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:22.712 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:22.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:22.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:22.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:22.845 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:22.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:22.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:22.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:22.981 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:22.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:22.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:23.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:23.110 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:23.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:23.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:23.213 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:23.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:23.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:23.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:23.382 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:23.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:23.486 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:23.514 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:23.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:23.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:23.617 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:23.651 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:23.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:45:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:45:23] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.33.4" 2023-10-09T09:45:23.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:23.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:23.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:23.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:23.890 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:23.919 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:23.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:23.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:24.018 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:23 smithi163 bash[78107]: cluster 2023-10-09T09:45:22.061780+0000 mgr.y (mgr.34107) 1180 : cluster 0 pgmap v2011: 89 pgs: 2 creating+activating, 4 unknown, 83 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:24.018 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:23 smithi163 bash[78107]: cluster 2023-10-09T09:45:22.350268+0000 osd.6 (osd.6) 523 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:24.018 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:23 smithi163 bash[78107]: cluster 2023-10-09T09:45:22.725840+0000 osd.4 (osd.4) 640 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:24.019 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:23 smithi163 bash[75425]: cluster 2023-10-09T09:45:22.061780+0000 mgr.y (mgr.34107) 1180 : cluster 0 pgmap v2011: 89 pgs: 2 creating+activating, 4 unknown, 83 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:24.019 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:23 smithi163 bash[75425]: cluster 2023-10-09T09:45:22.350268+0000 osd.6 (osd.6) 523 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:24.019 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:23 smithi163 bash[75425]: cluster 2023-10-09T09:45:22.725840+0000 osd.4 (osd.4) 640 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:24.023 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:24.054 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:24.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:24.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:24.158 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:24.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:24.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:24.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:24.239 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:45:24.239 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:45:24.239 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T09:45:24.240 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:45:24.291 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:24.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:23 smithi181 bash[59345]: cluster 2023-10-09T09:45:22.061780+0000 mgr.y (mgr.34107) 1180 : cluster 0 pgmap v2011: 89 pgs: 2 creating+activating, 4 unknown, 83 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:24.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:23 smithi181 bash[59345]: cluster 2023-10-09T09:45:22.350268+0000 osd.6 (osd.6) 523 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:24.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:23 smithi181 bash[59345]: cluster 2023-10-09T09:45:22.725840+0000 osd.4 (osd.4) 640 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:24.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:24.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:24.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:24.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:24.453 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:24.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:24.474 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:24.475 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:24.475 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:24.556 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:24.581 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:24.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:24.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:24.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:24.711 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:24.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:24.814 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:24.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:24.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:24.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:24.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:24.971 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:24.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:24.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:25.075 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:25.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:25.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:25.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:25.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:25.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:24 smithi163 bash[78107]: cluster 2023-10-09T09:45:23.018377+0000 osd.7 (osd.7) 596 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:25.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:24 smithi163 bash[75425]: cluster 2023-10-09T09:45:23.018377+0000 osd.7 (osd.7) 596 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:25.240 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:25.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:24 smithi181 bash[59345]: cluster 2023-10-09T09:45:23.018377+0000 osd.7 (osd.7) 596 : cluster 3 failed to encode map e1234 with expected crc 2023-10-09T09:45:25.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:25.381 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:25.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:25.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:25.485 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:25.515 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:25.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:25.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:25.619 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:25.650 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:25.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:25.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:25.754 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:25.788 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:25.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:25.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:25.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:25.914 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:25.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:25.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:26.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T09:45:26.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:26.051 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:26.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:26.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:26.154 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:26.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:25 smithi181 bash[59345]: cluster 2023-10-09T09:45:24.062585+0000 mgr.y (mgr.34107) 1181 : cluster 0 pgmap v2012: 89 pgs: 2 creating+activating, 87 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:26.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:25 smithi181 bash[59345]: audit 2023-10-09T09:45:25.397700+0000 mon.a (mon.0) 2235 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:26.167 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:25 smithi181 bash[59345]: audit 2023-10-09T09:45:25.399638+0000 mon.a (mon.0) 2236 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:26.186 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:26.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:26.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:26.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:25 smithi163 bash[75425]: cluster 2023-10-09T09:45:24.062585+0000 mgr.y (mgr.34107) 1181 : cluster 0 pgmap v2012: 89 pgs: 2 creating+activating, 87 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:25 smithi163 bash[75425]: audit 2023-10-09T09:45:25.397700+0000 mon.a (mon.0) 2235 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:25 smithi163 bash[75425]: audit 2023-10-09T09:45:25.399638+0000 mon.a (mon.0) 2236 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:25 smithi163 bash[78107]: cluster 2023-10-09T09:45:24.062585+0000 mgr.y (mgr.34107) 1181 : cluster 0 pgmap v2012: 89 pgs: 2 creating+activating, 87 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:25 smithi163 bash[78107]: audit 2023-10-09T09:45:25.397700+0000 mon.a (mon.0) 2235 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:25 smithi163 bash[78107]: audit 2023-10-09T09:45:25.399638+0000 mon.a (mon.0) 2236 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:26.290 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:26.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:26.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:26.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:26.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:26.462 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:26.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:26.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:26.492 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:26.567 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:26.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:26.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:26.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:26.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:26.729 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:26.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:26.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:26.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:26.855 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:26.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:26.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:26.985 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:26.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:26.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:27.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:27.113 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:27.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:27.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:27.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:27.244 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:27.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:27.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:27.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:27.374 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:27.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:27.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:27.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:27.514 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:27.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:27.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:27.617 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:27.650 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:27.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:27.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:27.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:27.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:27.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:27.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:27.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:27.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:27.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:28.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:28.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:27 smithi163 bash[75425]: cluster 2023-10-09T09:45:26.063385+0000 mgr.y (mgr.34107) 1182 : cluster 0 pgmap v2013: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:28.017 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:27 smithi163 bash[78107]: cluster 2023-10-09T09:45:26.063385+0000 mgr.y (mgr.34107) 1182 : cluster 0 pgmap v2013: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:28.030 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:28.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:28.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:28.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:28.155 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:28.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:28.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:28.258 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:28.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:27 smithi181 bash[59345]: cluster 2023-10-09T09:45:26.063385+0000 mgr.y (mgr.34107) 1182 : cluster 0 pgmap v2013: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:28.293 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:28.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:28.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:28.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:28.421 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:28.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:28.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:28.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:28.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:28.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:28.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:28.654 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:28.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:28.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:28.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:28.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:28.829 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:28.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:28.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:28.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:28.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:28.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:28.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:29.066 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:29.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:29.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:29.194 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:29.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:29.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:29.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:29.330 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:29.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:29.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:29.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:29.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:29.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:29.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:29.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:29.598 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:29.629 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:29.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:29.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:29.733 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:29.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:29.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:29.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:29.859 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:29.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:29.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:30.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:30.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:30.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:30.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:30.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:30.147 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:29 smithi181 bash[59345]: cluster 2023-10-09T09:45:28.064382+0000 mgr.y (mgr.34107) 1183 : cluster 0 pgmap v2014: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:30.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:30.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:30.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:30.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:29 smithi163 bash[78107]: cluster 2023-10-09T09:45:28.064382+0000 mgr.y (mgr.34107) 1183 : cluster 0 pgmap v2014: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:29 smithi163 bash[75425]: cluster 2023-10-09T09:45:28.064382+0000 mgr.y (mgr.34107) 1183 : cluster 0 pgmap v2014: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:30.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:30.292 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:30.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:30.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:30.419 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:30.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:30.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:30.522 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:30.544 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:30.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:30.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:30.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:30.671 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:30.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:30.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:30.774 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:30.795 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:30.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:30.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:30.898 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:30.930 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:30.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:30.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:31.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:31.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:31.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:31.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:31.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:31.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:30 smithi181 bash[59345]: audit 2023-10-09T09:45:30.367085+0000 mon.a (mon.0) 2237 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:31.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:31.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:31.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:31.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:30 smithi163 bash[78107]: audit 2023-10-09T09:45:30.367085+0000 mon.a (mon.0) 2237 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:31.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:30 smithi163 bash[75425]: audit 2023-10-09T09:45:30.367085+0000 mon.a (mon.0) 2237 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:31.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:31.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:31.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:31.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:31.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:31.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:31.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:31.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:31.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:31.583 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:31.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:31.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:31.686 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:31.717 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:31.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:31.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:31.821 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:31.848 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:31.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:31.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:31.952 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:31.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:31 smithi163 bash[75425]: cluster 2023-10-09T09:45:30.065325+0000 mgr.y (mgr.34107) 1184 : cluster 0 pgmap v2015: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:31.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:31 smithi163 bash[75425]: audit 2023-10-09T09:45:31.084895+0000 mon.a (mon.0) 2238 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:31.963 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:31 smithi163 bash[78107]: cluster 2023-10-09T09:45:30.065325+0000 mgr.y (mgr.34107) 1184 : cluster 0 pgmap v2015: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:31.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:31 smithi163 bash[78107]: audit 2023-10-09T09:45:31.084895+0000 mon.a (mon.0) 2238 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:31.976 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:31.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:31.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:31 smithi181 bash[59345]: cluster 2023-10-09T09:45:30.065325+0000 mgr.y (mgr.34107) 1184 : cluster 0 pgmap v2015: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:32.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:31 smithi181 bash[59345]: audit 2023-10-09T09:45:31.084895+0000 mon.a (mon.0) 2238 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:32.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:32.112 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:32.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:32.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:32.215 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:32.326 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:32.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:32.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:32.459 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:32.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:32.562 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:32.591 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:32.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:32.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:32.695 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:32.725 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:32.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:32.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:32.983 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:33.012 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:33.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:33.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:33.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:33.145 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:33.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:33.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:33.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:33.279 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:33.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:33.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:33.382 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:33.496 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:45:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:45:33] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.33.4" 2023-10-09T09:45:33.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:33.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:33.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:33.651 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:33.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:33.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:33.750 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:33.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:33.786 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:33.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:33.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:33.890 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:33.923 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:33.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:34.026 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:34.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:34.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:34.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:34.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:34.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:33 smithi181 bash[59345]: cluster 2023-10-09T09:45:32.066081+0000 mgr.y (mgr.34107) 1185 : cluster 0 pgmap v2016: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:34.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:34.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:34.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:34.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:33 smithi163 bash[78107]: cluster 2023-10-09T09:45:32.066081+0000 mgr.y (mgr.34107) 1185 : cluster 0 pgmap v2016: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:33 smithi163 bash[75425]: cluster 2023-10-09T09:45:32.066081+0000 mgr.y (mgr.34107) 1185 : cluster 0 pgmap v2016: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:34.302 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:34.333 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:34.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:34.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:34.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:34.692 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:34.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:34.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:34.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:34.825 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:34.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:34.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:34.927 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:34.948 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:34.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:34.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:35.051 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:35.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:35.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:35.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:35.185 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:35.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:35.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:35.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:35.313 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:35.339 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:35.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:35.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:35.443 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:35.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:35.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:35.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:35.608 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:35.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:35.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:35.712 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:35.741 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:35.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:35.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:35.844 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:35.869 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:35.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:35.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:35.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:36.004 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:36.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:36.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:35 smithi181 bash[59345]: cluster 2023-10-09T09:45:34.066685+0000 mgr.y (mgr.34107) 1186 : cluster 0 pgmap v2017: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:36.108 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:36.131 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:36.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:36.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:36.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:35 smithi163 bash[78107]: cluster 2023-10-09T09:45:34.066685+0000 mgr.y (mgr.34107) 1186 : cluster 0 pgmap v2017: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:36.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:35 smithi163 bash[75425]: cluster 2023-10-09T09:45:34.066685+0000 mgr.y (mgr.34107) 1186 : cluster 0 pgmap v2017: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:36.235 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:36.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:36.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:36.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:36.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:36.385 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:36.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:36.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:36.488 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:36.514 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:36.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:36.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:36.648 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:36.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:36.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:36.751 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:36.784 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:36.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:36.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:36.887 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:36.918 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:36.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:36.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:37.023 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:37.053 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:37.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:37.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:37.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:37.189 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:45:37.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:37.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:37.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:37.229 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:45:37.267 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:45:37.296 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:45:37.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:37.322 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:37.323 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:45:37.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:37.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:37.350 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:45:37.385 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:45:37.420 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:45:37.422 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:45:37.422 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:45:37.422 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:45:37.422 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:45:37.422 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:45:37.422 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:45:37.422 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:45:37.422 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:45:37.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:37.454 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:37.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:37.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:37.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:37.580 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:37.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:37.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:37.684 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:37.719 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:37.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:37.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:37.823 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:37.852 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:37.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:37.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:37.934 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:45:37.934 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T09:45:37.936 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T09:45:37.955 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:37.963 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[78107]: cluster 2023-10-09T09:45:36.067461+0000 mgr.y (mgr.34107) 1187 : cluster 0 pgmap v2018: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:37.963 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[78107]: audit 2023-10-09T09:45:36.605780+0000 mon.a (mon.0) 2239 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.963 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[78107]: audit 2023-10-09T09:45:36.614537+0000 mon.a (mon.0) 2240 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[78107]: audit 2023-10-09T09:45:37.065123+0000 mon.c (mon.2) 105 : audit 1 from='client.? 172.21.15.163:0/2397258006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:45:37.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[78107]: audit 2023-10-09T09:45:37.065587+0000 mon.a (mon.0) 2241 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:45:37.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[78107]: audit 2023-10-09T09:45:37.147007+0000 mon.c (mon.2) 106 : audit 0 from='client.? 172.21.15.163:0/1794787401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:45:37.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[78107]: audit 2023-10-09T09:45:37.362049+0000 mon.a (mon.0) 2242 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[78107]: audit 2023-10-09T09:45:37.371624+0000 mon.a (mon.0) 2243 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[78107]: audit 2023-10-09T09:45:37.587078+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.163:0/1434858875' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:45:37.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[78107]: audit 2023-10-09T09:45:37.587647+0000 mon.a (mon.0) 2244 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:45:37.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[75425]: cluster 2023-10-09T09:45:36.067461+0000 mgr.y (mgr.34107) 1187 : cluster 0 pgmap v2018: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:37.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[75425]: audit 2023-10-09T09:45:36.605780+0000 mon.a (mon.0) 2239 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[75425]: audit 2023-10-09T09:45:36.614537+0000 mon.a (mon.0) 2240 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[75425]: audit 2023-10-09T09:45:37.065123+0000 mon.c (mon.2) 105 : audit 1 from='client.? 172.21.15.163:0/2397258006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:45:37.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[75425]: audit 2023-10-09T09:45:37.065587+0000 mon.a (mon.0) 2241 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:45:37.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[75425]: audit 2023-10-09T09:45:37.147007+0000 mon.c (mon.2) 106 : audit 0 from='client.? 172.21.15.163:0/1794787401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:45:37.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[75425]: audit 2023-10-09T09:45:37.362049+0000 mon.a (mon.0) 2242 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[75425]: audit 2023-10-09T09:45:37.371624+0000 mon.a (mon.0) 2243 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[75425]: audit 2023-10-09T09:45:37.587078+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.163:0/1434858875' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:45:37.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:37 smithi163 bash[75425]: audit 2023-10-09T09:45:37.587647+0000 mon.a (mon.0) 2244 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:45:37.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:37 smithi181 bash[59345]: cluster 2023-10-09T09:45:36.067461+0000 mgr.y (mgr.34107) 1187 : cluster 0 pgmap v2018: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:37.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:37 smithi181 bash[59345]: audit 2023-10-09T09:45:36.605780+0000 mon.a (mon.0) 2239 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:37 smithi181 bash[59345]: audit 2023-10-09T09:45:36.614537+0000 mon.a (mon.0) 2240 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:37 smithi181 bash[59345]: audit 2023-10-09T09:45:37.065123+0000 mon.c (mon.2) 105 : audit 1 from='client.? 172.21.15.163:0/2397258006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:45:37.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:37 smithi181 bash[59345]: audit 2023-10-09T09:45:37.065587+0000 mon.a (mon.0) 2241 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:45:37.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:37 smithi181 bash[59345]: audit 2023-10-09T09:45:37.147007+0000 mon.c (mon.2) 106 : audit 0 from='client.? 172.21.15.163:0/1794787401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:45:37.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:37 smithi181 bash[59345]: audit 2023-10-09T09:45:37.362049+0000 mon.a (mon.0) 2242 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:37 smithi181 bash[59345]: audit 2023-10-09T09:45:37.371624+0000 mon.a (mon.0) 2243 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:37.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:37 smithi181 bash[59345]: audit 2023-10-09T09:45:37.587078+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.163:0/1434858875' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:45:37.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:37 smithi181 bash[59345]: audit 2023-10-09T09:45:37.587647+0000 mon.a (mon.0) 2244 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:45:37.990 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:37.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:37.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:38.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:38.371 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:38.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:38.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:38.474 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:38.509 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:38.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:38.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:38.636 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:38.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:38.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:38.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:38.776 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:38.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:38.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:38.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: audit 2023-10-09T09:45:37.853225+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.163:0/3786854897' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:45:38.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: audit 2023-10-09T09:45:37.853794+0000 mon.a (mon.0) 2245 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:45:38.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: cluster 2023-10-09T09:45:37.922609+0000 mon.a (mon.0) 2246 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:45:38.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: cluster 2023-10-09T09:45:37.927369+0000 osd.5 (osd.5) 822 : cluster 3 failed to encode map e1235 with expected crc 2023-10-09T09:45:38.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: cluster 2023-10-09T09:45:37.927939+0000 osd.6 (osd.6) 524 : cluster 3 failed to encode map e1235 with expected crc 2023-10-09T09:45:38.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: audit 2023-10-09T09:45:37.929487+0000 mon.a (mon.0) 2247 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:45:38.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: audit 2023-10-09T09:45:37.929542+0000 mon.a (mon.0) 2248 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:45:38.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: audit 2023-10-09T09:45:37.929593+0000 mon.a (mon.0) 2249 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:45:38.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: cluster 2023-10-09T09:45:37.929639+0000 mon.a (mon.0) 2250 : cluster 0 osdmap e1235: 8 total, 8 up, 6 in 2023-10-09T09:45:38.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: audit 2023-10-09T09:45:38.563147+0000 mon.a (mon.0) 2251 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:38.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: audit 2023-10-09T09:45:38.570441+0000 mon.a (mon.0) 2252 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:38.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: audit 2023-10-09T09:45:38.571866+0000 mon.a (mon.0) 2253 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:38.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: audit 2023-10-09T09:45:38.573043+0000 mon.a (mon.0) 2254 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:38.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:38 smithi181 bash[59345]: audit 2023-10-09T09:45:38.580483+0000 mon.a (mon.0) 2255 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:38.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:38.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:39.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:39.035 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:39.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:39.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: audit 2023-10-09T09:45:37.853225+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.163:0/3786854897' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:45:39.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: audit 2023-10-09T09:45:37.853794+0000 mon.a (mon.0) 2245 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:45:39.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: cluster 2023-10-09T09:45:37.922609+0000 mon.a (mon.0) 2246 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:45:39.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: cluster 2023-10-09T09:45:37.927369+0000 osd.5 (osd.5) 822 : cluster 3 failed to encode map e1235 with expected crc 2023-10-09T09:45:39.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: cluster 2023-10-09T09:45:37.927939+0000 osd.6 (osd.6) 524 : cluster 3 failed to encode map e1235 with expected crc 2023-10-09T09:45:39.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: audit 2023-10-09T09:45:37.929487+0000 mon.a (mon.0) 2247 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:45:39.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: audit 2023-10-09T09:45:37.929542+0000 mon.a (mon.0) 2248 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:45:39.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: audit 2023-10-09T09:45:37.929593+0000 mon.a (mon.0) 2249 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:45:39.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: cluster 2023-10-09T09:45:37.929639+0000 mon.a (mon.0) 2250 : cluster 0 osdmap e1235: 8 total, 8 up, 6 in 2023-10-09T09:45:39.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: audit 2023-10-09T09:45:38.563147+0000 mon.a (mon.0) 2251 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:39.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: audit 2023-10-09T09:45:38.570441+0000 mon.a (mon.0) 2252 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:39.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: audit 2023-10-09T09:45:38.571866+0000 mon.a (mon.0) 2253 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:39.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: audit 2023-10-09T09:45:38.573043+0000 mon.a (mon.0) 2254 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:39.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[75425]: audit 2023-10-09T09:45:38.580483+0000 mon.a (mon.0) 2255 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:39.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: audit 2023-10-09T09:45:37.853225+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.163:0/3786854897' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:45:39.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: audit 2023-10-09T09:45:37.853794+0000 mon.a (mon.0) 2245 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:45:39.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: cluster 2023-10-09T09:45:37.922609+0000 mon.a (mon.0) 2246 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: cluster 2023-10-09T09:45:37.927369+0000 osd.5 (osd.5) 822 : cluster 3 failed to encode map e1235 with expected crc 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: cluster 2023-10-09T09:45:37.927939+0000 osd.6 (osd.6) 524 : cluster 3 failed to encode map e1235 with expected crc 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: audit 2023-10-09T09:45:37.929487+0000 mon.a (mon.0) 2247 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: audit 2023-10-09T09:45:37.929542+0000 mon.a (mon.0) 2248 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: audit 2023-10-09T09:45:37.929593+0000 mon.a (mon.0) 2249 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: cluster 2023-10-09T09:45:37.929639+0000 mon.a (mon.0) 2250 : cluster 0 osdmap e1235: 8 total, 8 up, 6 in 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: audit 2023-10-09T09:45:38.563147+0000 mon.a (mon.0) 2251 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: audit 2023-10-09T09:45:38.570441+0000 mon.a (mon.0) 2252 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: audit 2023-10-09T09:45:38.571866+0000 mon.a (mon.0) 2253 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: audit 2023-10-09T09:45:38.573043+0000 mon.a (mon.0) 2254 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:38 smithi163 bash[78107]: audit 2023-10-09T09:45:38.580483+0000 mon.a (mon.0) 2255 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:39.139 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:39.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:39.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:39.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:39.308 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:39.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:39.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:39.411 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:39.444 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:39.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:39.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:39.548 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:39.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:39.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:39.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:39.682 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:39.714 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:39.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:39.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:39.818 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:39.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:39.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:39.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:39.953 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:39.976 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:39.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:39 smithi181 bash[59345]: cluster 2023-10-09T09:45:38.067987+0000 mgr.y (mgr.34107) 1188 : cluster 0 pgmap v2020: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:40.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:40.103 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:40.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:40.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:40.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:40.236 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:40.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:40.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:40.375 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:40.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:40.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:40.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:40 smithi163 bash[78107]: cluster 2023-10-09T09:45:38.067987+0000 mgr.y (mgr.34107) 1188 : cluster 0 pgmap v2020: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:40.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:40 smithi163 bash[75425]: cluster 2023-10-09T09:45:38.067987+0000 mgr.y (mgr.34107) 1188 : cluster 0 pgmap v2020: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:40.478 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:40.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:40.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:40.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:40.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:40.638 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:40.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:40.743 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:40.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:40.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:40.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:40.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:40.896 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:40.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:40.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:40.999 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:41.030 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:41.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:41.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:41.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:41.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:41.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:41.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:41.265 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:41.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:41.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:41.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:41.431 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:41.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:41.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:41.535 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:41.547 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:41 smithi181 bash[59345]: cluster 2023-10-09T09:45:40.068893+0000 mgr.y (mgr.34107) 1189 : cluster 0 pgmap v2021: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:41.547 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:41 smithi181 bash[59345]: audit 2023-10-09T09:45:40.399387+0000 mon.a (mon.0) 2256 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:41.548 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:41 smithi181 bash[59345]: audit 2023-10-09T09:45:40.408842+0000 mon.a (mon.0) 2257 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:41.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:41.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:41.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:41.687 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:41 smithi163 bash[78107]: cluster 2023-10-09T09:45:40.068893+0000 mgr.y (mgr.34107) 1189 : cluster 0 pgmap v2021: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:41.687 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:41 smithi163 bash[78107]: audit 2023-10-09T09:45:40.399387+0000 mon.a (mon.0) 2256 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:41.687 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:41 smithi163 bash[78107]: audit 2023-10-09T09:45:40.408842+0000 mon.a (mon.0) 2257 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:41.687 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:41 smithi163 bash[75425]: cluster 2023-10-09T09:45:40.068893+0000 mgr.y (mgr.34107) 1189 : cluster 0 pgmap v2021: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:41.687 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:41 smithi163 bash[75425]: audit 2023-10-09T09:45:40.399387+0000 mon.a (mon.0) 2256 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:45:41.687 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:41 smithi163 bash[75425]: audit 2023-10-09T09:45:40.408842+0000 mon.a (mon.0) 2257 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:41.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:41.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:41.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:41.807 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:41.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:41.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:41.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:41.972 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:41.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:41.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:42.077 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:42.109 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:42.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:42.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:42.214 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:42.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:42.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:42.348 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:42.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:42.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:42.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:42.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:42.514 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:42.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:42.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:42.531 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:45:42.618 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:42.641 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:42.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:42.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:42.744 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:42.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:42.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:42.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:42.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:42.910 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:42.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:42.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:43.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:43.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T09:45:43.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:43.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:43.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:43.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:43.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:43.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:43.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:43.283 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:43.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:43.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:43.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:43.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:43.430 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:43 smithi163 bash[78107]: cluster 2023-10-09T09:45:42.069954+0000 mgr.y (mgr.34107) 1190 : cluster 0 pgmap v2022: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:43.430 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:43 smithi163 bash[75425]: cluster 2023-10-09T09:45:42.069954+0000 mgr.y (mgr.34107) 1190 : cluster 0 pgmap v2022: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:43.430 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:45:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:45:43] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.33.4" 2023-10-09T09:45:43.447 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:43.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:43.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:43.519 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:45:43.519 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:45:43.551 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:43.582 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:43.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:43.689 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:43.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:43 smithi181 bash[59345]: cluster 2023-10-09T09:45:42.069954+0000 mgr.y (mgr.34107) 1190 : cluster 0 pgmap v2022: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:43.723 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:43.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:43.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:43.819 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T09:45:43.823 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:45:43.823 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:45:43.827 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:43.860 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:43.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:43.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:43.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:44.002 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:44.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:44.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:44.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:45:44.105 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:44.132 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:44.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:44.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:44.259 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:44.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:44.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:44.362 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:44.394 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:44.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:44.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:44.498 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:44.530 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:44.530 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T09:45:44.530 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:45:44.531 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_194945 2023-10-09T09:45:44.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:44.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:44.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:45:44.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:45:44.635 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:44.641 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T09:45:44.664 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:44.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:44.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:44.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:44.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:44.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:44.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:44.897 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:44.932 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:44.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:44.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:45.038 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:45.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:45.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:45.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:45.172 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:45.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:45.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:45.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:45.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:45.330 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:45.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:45.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:45.434 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:45.461 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:45.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:45.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:45.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:45 smithi163 bash[78107]: cluster 2023-10-09T09:45:44.070847+0000 mgr.y (mgr.34107) 1191 : cluster 0 pgmap v2023: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:45.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:45 smithi163 bash[75425]: cluster 2023-10-09T09:45:44.070847+0000 mgr.y (mgr.34107) 1191 : cluster 0 pgmap v2023: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:45.532 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 3076 3060 191.24 191.25 0.0034313 0.00520483 2023-10-09T09:45:45.567 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:45.592 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:45.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:45.696 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:45.727 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:45.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:45.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:45 smithi181 bash[59345]: cluster 2023-10-09T09:45:44.070847+0000 mgr.y (mgr.34107) 1191 : cluster 0 pgmap v2023: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:45:45.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:45.859 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:45.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:45.963 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:45.994 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:45.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:46.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:46.125 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:46.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:46.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:46.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:46.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:46.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:46.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:46.390 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:46.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:46.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:46.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:46.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:46.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:46.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:46.532 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 5942 5926 185.174 179.125 0.00384145 0.00538691 2023-10-09T09:45:46.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:46.651 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:46.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:46.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:46.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:46.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:46.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:46.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:46.918 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:46.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:46.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:47.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:47.044 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:47.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:47.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:47.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:47.176 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:47.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:47.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:47.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:47.306 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:47.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:47.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:47.409 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:47.437 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:47.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:47.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:47.532 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 8659 8643 180.048 169.812 0.00548991 0.00554107 2023-10-09T09:45:47.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:47.571 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:47.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:47.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:47.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:47.707 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:47.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:47.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:47.811 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:47.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:47 smithi163 bash[75425]: cluster 2023-10-09T09:45:46.071367+0000 mgr.y (mgr.34107) 1192 : cluster 0 pgmap v2024: 89 pgs: 89 active+clean; 34 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 3.3 MiB/s wr, 52 op/s 2023-10-09T09:45:47.837 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:47.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:47.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:47.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:47.973 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:47.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:47.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:48.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:48.088 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:47 smithi163 bash[78107]: cluster 2023-10-09T09:45:46.071367+0000 mgr.y (mgr.34107) 1192 : cluster 0 pgmap v2024: 89 pgs: 89 active+clean; 34 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 3.3 MiB/s wr, 52 op/s 2023-10-09T09:45:48.105 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:48.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:48.177 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:48.183 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:48.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:48.220 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:47 smithi181 bash[59345]: cluster 2023-10-09T09:45:46.071367+0000 mgr.y (mgr.34107) 1192 : cluster 0 pgmap v2024: 89 pgs: 89 active+clean; 34 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 3.3 MiB/s wr, 52 op/s 2023-10-09T09:45:48.237 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:48.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:48.341 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:48.369 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:48.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:48.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:48.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:48.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:48.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:48.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:48.532 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 11228 11212 175.172 160.562 0.00532077 0.00570028 2023-10-09T09:45:48.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:48.627 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:48.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:48.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:48.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:48.759 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:48.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:48.863 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:48.891 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:48.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:48.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:48.996 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:49.023 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:49.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:49.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:49.128 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:49.154 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:49.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:49.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:49.198 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:49.198 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:45:49.258 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:49.283 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:49.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:49.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:49.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:49.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:49.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:49.532 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 15 13690 13675 170.922 153.938 0.00298164 0.00584101 2023-10-09T09:45:49.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:49.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:49.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:49.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:49.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:49.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:49.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:49.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:49.819 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:49.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:49.842 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:49 smithi163 bash[75425]: cluster 2023-10-09T09:45:48.072330+0000 mgr.y (mgr.34107) 1193 : cluster 0 pgmap v2025: 89 pgs: 89 active+clean; 280 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 28 MiB/s wr, 440 op/s 2023-10-09T09:45:49.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:49.951 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:49.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:49.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:50.055 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:49 smithi181 bash[59345]: cluster 2023-10-09T09:45:48.072330+0000 mgr.y (mgr.34107) 1193 : cluster 0 pgmap v2025: 89 pgs: 89 active+clean; 280 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 28 MiB/s wr, 440 op/s 2023-10-09T09:45:50.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:50.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:50.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:50.189 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:49 smithi163 bash[78107]: cluster 2023-10-09T09:45:48.072330+0000 mgr.y (mgr.34107) 1193 : cluster 0 pgmap v2025: 89 pgs: 89 active+clean; 280 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 28 MiB/s wr, 440 op/s 2023-10-09T09:45:50.230 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:50.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:50.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:50.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:50.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:50.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:50.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:50.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:50.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:50.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:50.532 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 15925 15909 165.704 139.625 0.00596973 0.00602746 2023-10-09T09:45:50.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:50.628 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:50.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:50.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:50.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:50.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:50.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:50.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:50.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:50.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:50.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:50.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:50.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:51.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:51.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:51.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:51.130 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:51.158 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:51.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:51.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:51.263 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:51.292 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:51.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:51.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:51.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:51.428 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:51.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:51.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:51.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:51.532 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 16 18204 18188 162.378 142.438 0.00418794 0.00615148 2023-10-09T09:45:51.564 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:51.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:51.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:51.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:51.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:51.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:51.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:51.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:51.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:51.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:51.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:51.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:51.968 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:51.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:51.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:52.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:52.102 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:52.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:52.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:52.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:51 smithi181 bash[59345]: cluster 2023-10-09T09:45:50.072905+0000 mgr.y (mgr.34107) 1194 : cluster 0 pgmap v2026: 89 pgs: 89 active+clean; 369 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 31 MiB/s wr, 490 op/s 2023-10-09T09:45:52.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:52.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:51 smithi163 bash[78107]: cluster 2023-10-09T09:45:50.072905+0000 mgr.y (mgr.34107) 1194 : cluster 0 pgmap v2026: 89 pgs: 89 active+clean; 369 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 31 MiB/s wr, 490 op/s 2023-10-09T09:45:52.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:51 smithi163 bash[75425]: cluster 2023-10-09T09:45:50.072905+0000 mgr.y (mgr.34107) 1194 : cluster 0 pgmap v2026: 89 pgs: 89 active+clean; 369 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 31 MiB/s wr, 490 op/s 2023-10-09T09:45:52.235 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:52.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:52.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:52.339 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:52.368 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:52.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:52.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:52.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:52.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:52.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:52.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:52.532 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 20372 20356 159.017 135.5 0.00697891 0.00628337 2023-10-09T09:45:52.601 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:52.627 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:52.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:52.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:52.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:52.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:52.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:52.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:52.862 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:52.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:52.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:52.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:53.015 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:53.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:53.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:53.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:53.146 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:53.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:53.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:53.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:53.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:53.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:53.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:53.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:53.391 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:45:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:45:53] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.33.4" 2023-10-09T09:45:53.411 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:53.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:53.516 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:53.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 22429 22413 155.632 128.562 0.0081913 0.00641815 2023-10-09T09:45:53.542 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:53.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:53.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:53.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:53.673 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:53.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:53.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:53.777 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:53.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:53.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:53.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:53.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:53.930 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:53.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:53.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:53 smithi163 bash[75425]: cluster 2023-10-09T09:45:52.073528+0000 mgr.y (mgr.34107) 1195 : cluster 0 pgmap v2027: 89 pgs: 89 active+clean; 959 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T09:45:53.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:53 smithi163 bash[78107]: cluster 2023-10-09T09:45:52.073528+0000 mgr.y (mgr.34107) 1195 : cluster 0 pgmap v2027: 89 pgs: 89 active+clean; 959 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T09:45:54.035 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:54.044 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:53 smithi181 bash[59345]: cluster 2023-10-09T09:45:52.073528+0000 mgr.y (mgr.34107) 1195 : cluster 0 pgmap v2027: 89 pgs: 89 active+clean; 959 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T09:45:54.063 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:54.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:54.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:54.166 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:54.191 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:54.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:54.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:54.294 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:54.327 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:54.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:54.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:54.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:54.458 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:54.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:54.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:54.532 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 24402 24386 152.399 123.312 0.00449773 0.00655527 2023-10-09T09:45:54.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:54.585 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:54.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:54.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:54.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:54.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:54.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:54.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:54.844 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:54.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:54.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:54.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:54.979 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:54.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:54.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:55.083 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:55.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:55.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:55.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:55.211 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:55.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:55.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:55.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:55.344 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:55.378 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:55.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:55.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:55.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:55.507 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:55.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:55.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:55.532 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 25861 25845 146.834 91.1875 0.00261719 0.00680182 2023-10-09T09:45:55.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:55.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:55.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:55.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:55.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:55.769 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:55.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:55.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:55.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:55.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:55.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:55.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:56.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:56.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:55 smithi163 bash[75425]: cluster 2023-10-09T09:45:54.074084+0000 mgr.y (mgr.34107) 1196 : cluster 0 pgmap v2028: 89 pgs: 89 active+clean; 1.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T09:45:56.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:55 smithi163 bash[75425]: audit 2023-10-09T09:45:55.391975+0000 mon.a (mon.0) 2258 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:56.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:55 smithi163 bash[78107]: cluster 2023-10-09T09:45:54.074084+0000 mgr.y (mgr.34107) 1196 : cluster 0 pgmap v2028: 89 pgs: 89 active+clean; 1.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T09:45:56.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:55 smithi163 bash[78107]: audit 2023-10-09T09:45:55.391975+0000 mon.a (mon.0) 2258 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:56.031 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:56.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:56.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:56.135 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:56.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:55 smithi181 bash[59345]: cluster 2023-10-09T09:45:54.074084+0000 mgr.y (mgr.34107) 1196 : cluster 0 pgmap v2028: 89 pgs: 89 active+clean; 1.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T09:45:56.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:55 smithi181 bash[59345]: audit 2023-10-09T09:45:55.391975+0000 mon.a (mon.0) 2258 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:45:56.160 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:56.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:56.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:56.263 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:56.292 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:56.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:56.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:56.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:56.425 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:56.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:56.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:56.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 27748 27732 144.425 117.938 0.00648757 0.00691729 2023-10-09T09:45:56.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:56.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:56.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:56.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:56.690 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:56.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:56.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:56.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:56.827 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:56.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:56.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:56.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:56.961 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:45:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:56.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:45:57.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:57.092 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:57.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:57.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:57.196 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:57.223 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:57.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:57.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:57.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:57.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:57.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:57.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:57.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:57.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:57.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:57.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 29452 29436 141.506 106.5 0.006763 0.00706143 2023-10-09T09:45:57.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:57.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:57.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:57.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:57.738 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:57.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:57.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:57.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:57.869 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:57.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:57.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:57.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:57.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:57 smithi163 bash[78107]: cluster 2023-10-09T09:45:56.074782+0000 mgr.y (mgr.34107) 1197 : cluster 0 pgmap v2029: 89 pgs: 89 active+clean; 1.3 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T09:45:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:57 smithi163 bash[75425]: cluster 2023-10-09T09:45:56.074782+0000 mgr.y (mgr.34107) 1197 : cluster 0 pgmap v2029: 89 pgs: 89 active+clean; 1.3 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T09:45:58.002 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:58.031 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:58.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:58.135 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:58.145 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:57 smithi181 bash[59345]: cluster 2023-10-09T09:45:56.074782+0000 mgr.y (mgr.34107) 1197 : cluster 0 pgmap v2029: 89 pgs: 89 active+clean; 1.3 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T09:45:58.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:58.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:58.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:58.265 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:58.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:58.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:58.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:58.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:45:58.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:58.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:45:58.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:58.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 31144 31128 138.952 105.75 0.0111904 0.00718996 2023-10-09T09:45:58.561 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:58.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:58.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:58.666 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:58.698 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:45:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:58.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:45:58.803 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:58.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:58.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:58.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:58.938 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:58.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:45:58.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:58.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:45:59.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:59.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:59.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:59.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:59.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:59.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:59.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:59.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:59.349 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:59.382 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:45:59.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:59.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:45:59.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:59.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:45:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:59.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:45:59.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 32657 32641 135.992 94.5625 0.00529264 0.00734693 2023-10-09T09:45:59.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:59.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:59.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:59.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:45:59.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:59.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:45:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:59.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:45:59.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:59.906 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:45:59 smithi181 bash[59345]: cluster 2023-10-09T09:45:58.075442+0000 mgr.y (mgr.34107) 1198 : cluster 0 pgmap v2030: 89 pgs: 89 active+clean; 1.6 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T09:45:59.926 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:45:59.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:45:59.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:00.029 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:00.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:00.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:00.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:00.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:00.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:45:59 smithi163 bash[78107]: cluster 2023-10-09T09:45:58.075442+0000 mgr.y (mgr.34107) 1198 : cluster 0 pgmap v2030: 89 pgs: 89 active+clean; 1.6 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T09:46:00.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:45:59 smithi163 bash[75425]: cluster 2023-10-09T09:45:58.075442+0000 mgr.y (mgr.34107) 1198 : cluster 0 pgmap v2030: 89 pgs: 89 active+clean; 1.6 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T09:46:00.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:00.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:00.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:00.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:00.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:00.428 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:00.460 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:00.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:00.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:00.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 33973 33957 132.633 82.25 0.00830285 0.00753403 2023-10-09T09:46:00.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:00.597 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:00.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:00.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:00.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:00.725 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:00.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:00.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:00.829 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:00.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:00.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:00.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:00.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:00.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:00.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:01.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:01.130 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:01.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:01.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:01.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:01.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:01.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:01.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:01.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:01.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:01.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:01.497 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:01.526 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:01.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:01.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:01.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 35605 35589 130.83 102 0.0128256 0.00763786 2023-10-09T09:46:01.630 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:01.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:01.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:01.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:01.764 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:01.789 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:01.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:01.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:01.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:01.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:01.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:01.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:02.015 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:02.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:01 smithi181 bash[59345]: cluster 2023-10-09T09:46:00.075895+0000 mgr.y (mgr.34107) 1199 : cluster 0 pgmap v2031: 89 pgs: 89 active+clean; 1.7 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T09:46:02.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:02.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:02.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:02.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:01 smithi163 bash[75425]: cluster 2023-10-09T09:46:00.075895+0000 mgr.y (mgr.34107) 1199 : cluster 0 pgmap v2031: 89 pgs: 89 active+clean; 1.7 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T09:46:02.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:01 smithi163 bash[78107]: cluster 2023-10-09T09:46:00.075895+0000 mgr.y (mgr.34107) 1199 : cluster 0 pgmap v2031: 89 pgs: 89 active+clean; 1.7 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T09:46:02.167 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:02.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:02.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:02.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:02.296 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:02.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:02.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:02.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:02.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:02.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:02.531 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:02.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 15 37023 37008 128.488 88.6875 0.00798168 0.00777696 2023-10-09T09:46:02.557 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:02.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:02.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:02.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:02.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:02.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:02.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:02.819 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:02.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:02.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:02.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:02.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:02.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:02.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:03.063 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:03.093 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:03.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:03.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:03.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:03.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:03.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:03.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:03.328 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:03.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:03.359 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:46:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:46:03] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-09T09:46:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:03.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:03.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:03.485 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:03.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:03.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:03.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 38617 38601 126.965 99.5625 0.0100549 0.00787056 2023-10-09T09:46:03.590 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:03.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:03.628 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:03.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:03.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:03.766 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:03.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:03.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:03.869 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:03.875 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:03 smithi181 bash[59345]: cluster 2023-10-09T09:46:02.076409+0000 mgr.y (mgr.34107) 1200 : cluster 0 pgmap v2032: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-10-09T09:46:03.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:03.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:03.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:04.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:04.039 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:04.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:04.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:04.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:04.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:04.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:04.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:03 smithi163 bash[75425]: cluster 2023-10-09T09:46:02.076409+0000 mgr.y (mgr.34107) 1200 : cluster 0 pgmap v2032: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-10-09T09:46:04.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:03 smithi163 bash[78107]: cluster 2023-10-09T09:46:02.076409+0000 mgr.y (mgr.34107) 1200 : cluster 0 pgmap v2032: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-10-09T09:46:04.280 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:04.311 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:04.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:04.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:04.416 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:04.441 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:04.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:04.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:04.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:46:04.533756+0000 min lat: 0.00129664 max lat: 0.198757 avg lat: 0.00796308 2023-10-09T09:46:04.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:46:04.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 40178 40162 125.495 97.5625 0.0034476 0.00796308 2023-10-09T09:46:04.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:04.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:04.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:04.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:04.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:04.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:04.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:04.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:04.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:04.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:04.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:04.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:04.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:04.969 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:04.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:04.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:05.075 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:05.106 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:05.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:05.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:05.210 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:05.236 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:05.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:05.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:05.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:05.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:05.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:05.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:05.474 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:05.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:05.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:05.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:05.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 41558 41542 123.626 86.25 0.00798923 0.00808327 2023-10-09T09:46:05.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:05.626 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:05.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:05.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:05.733 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:05.767 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:05.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:05.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:05.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:05.880 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:05 smithi163 bash[75425]: cluster 2023-10-09T09:46:04.076979+0000 mgr.y (mgr.34107) 1201 : cluster 0 pgmap v2033: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 105 MiB/s wr, 1.69k op/s 2023-10-09T09:46:05.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:05.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:05.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:06.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:06.049 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:06.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:06.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:06.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:05 smithi181 bash[59345]: cluster 2023-10-09T09:46:04.076979+0000 mgr.y (mgr.34107) 1201 : cluster 0 pgmap v2033: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 105 MiB/s wr, 1.69k op/s 2023-10-09T09:46:06.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:06.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:05 smithi163 bash[78107]: cluster 2023-10-09T09:46:04.076979+0000 mgr.y (mgr.34107) 1201 : cluster 0 pgmap v2033: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 105 MiB/s wr, 1.69k op/s 2023-10-09T09:46:06.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:06.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:06.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:06.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:06.305 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:06.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:06.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:06.411 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:06.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:06.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:06.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:06.533 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 42979 42963 122.043 88.8125 0.00703138 0.00818711 2023-10-09T09:46:06.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:06.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:06.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:06.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:06.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:06.726 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:06.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:06.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:06.833 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:06.860 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:06.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:06.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:07.002 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:07.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:07.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:07.127 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:46:07.133 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:07.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:07.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:07.173 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:46:07.220 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:46:07.239 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:07.257 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:46:07.279 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:07.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:07.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:07.287 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:46:07.330 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:46:07.359 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:46:07.381 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:46:07.382 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:46:07.383 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:46:07.383 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:46:07.383 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:46:07.383 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:46:07.383 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:46:07.383 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:46:07.383 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:46:07.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:07.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:07.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:07.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:07.505 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:07.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:07.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:07.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 44290 44274 120.298 81.9375 0.00724574 0.00830722 2023-10-09T09:46:07.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:07.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:07.674 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:07.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:07.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:07.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:07.806 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:07.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:07.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:07.908 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:07.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:07.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:07.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:08.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:08.054 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:07 smithi181 bash[59345]: cluster 2023-10-09T09:46:06.077620+0000 mgr.y (mgr.34107) 1202 : cluster 0 pgmap v2034: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T09:46:08.054 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:07 smithi181 bash[59345]: audit 2023-10-09T09:46:07.064628+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.163:0/455171787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:46:08.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:08.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:08.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:08.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:08.186 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:07 smithi163 bash[78107]: cluster 2023-10-09T09:46:06.077620+0000 mgr.y (mgr.34107) 1202 : cluster 0 pgmap v2034: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T09:46:08.186 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:07 smithi163 bash[78107]: audit 2023-10-09T09:46:07.064628+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.163:0/455171787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:46:08.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:07 smithi163 bash[75425]: cluster 2023-10-09T09:46:06.077620+0000 mgr.y (mgr.34107) 1202 : cluster 0 pgmap v2034: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T09:46:08.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:07 smithi163 bash[75425]: audit 2023-10-09T09:46:07.064628+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.163:0/455171787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:46:08.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:08.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:08.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:08.310 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:08.346 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:08.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:08.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:08.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:08.474 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:08.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:08.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 45488 45472 118.405 74.875 0.0130172 0.00843914 2023-10-09T09:46:08.577 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:08.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:08.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:08.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:08.718 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:08.742 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:08.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:08.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:08.845 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:08.873 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:08.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:08.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:08.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:09.009 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:09.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:09.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:09.113 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:09.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:09.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:09.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:09.251 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:09.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:09.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:09.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:09.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:09.421 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:09.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:09.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:09.535 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 46959 46943 117.346 91.9375 0.0127461 0.00851732 2023-10-09T09:46:09.591 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:09.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:09.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:09.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:09.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:09.757 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:09.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:09.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:09.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:09.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:09.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:09.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:10.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:10.017 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:09 smithi181 bash[59345]: cluster 2023-10-09T09:46:08.078242+0000 mgr.y (mgr.34107) 1203 : cluster 0 pgmap v2035: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-10-09T09:46:10.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:10.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:10.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:10.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:09 smithi163 bash[75425]: cluster 2023-10-09T09:46:08.078242+0000 mgr.y (mgr.34107) 1203 : cluster 0 pgmap v2035: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-10-09T09:46:10.168 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:09 smithi163 bash[78107]: cluster 2023-10-09T09:46:08.078242+0000 mgr.y (mgr.34107) 1203 : cluster 0 pgmap v2035: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-10-09T09:46:10.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:10.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:10.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:10.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:10.323 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:10.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:10.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:10.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:10.459 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:10.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:10.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:10.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 16 48256 48240 115.95 81.0625 0.00627574 0.00861651 2023-10-09T09:46:10.566 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:10.606 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:10.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:10.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:10.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:10.742 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:10.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:10.848 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:10.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:10.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:10.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:10.966 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:46:10.967 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:46:10.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:46:10.967 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:46:10.967 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.967 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149568 8.f 8:f5b59462:::benchmark_data_smithi163_151167_object54335:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e1156)", 2023-10-09T09:46:10.967 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:17.862920+0000", 2023-10-09T09:46:10.967 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 473.09573037500002, 2023-10-09T09:46:10.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0313238870000001, 2023-10-09T09:46:10.967 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149568 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862920+0000", 2023-10-09T09:46:10.968 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862920+0000", 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862917+0000", 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:10.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862928+0000", 2023-10-09T09:46:10.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0443e-05 2023-10-09T09:46:10.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:10.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862943+0000", 2023-10-09T09:46:10.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5537e-05 2023-10-09T09:46:10.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:10.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862954+0000", 2023-10-09T09:46:10.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0417999999999999e-05 2023-10-09T09:46:10.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:10.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.864772+0000", 2023-10-09T09:46:10.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0018178650000000001 2023-10-09T09:46:10.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op must wait for map", 2023-10-09T09:46:10.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.864797+0000", 2023-10-09T09:46:10.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5316e-05 2023-10-09T09:46:10.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:10.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.869967+0000", 2023-10-09T09:46:10.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051704309999999996 2023-10-09T09:46:10.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:10.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.869978+0000", 2023-10-09T09:46:10.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0801e-05 2023-10-09T09:46:10.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:10.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.894180+0000", 2023-10-09T09:46:10.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0242017859999999 2023-10-09T09:46:10.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:10.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.894243+0000", 2023-10-09T09:46:10.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3491000000000006e-05 2023-10-09T09:46:10.974 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:10.974 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:10.975 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:10.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.975 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149574 8.f 8:f702f824:::benchmark_data_smithi163_151167_object54341:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e1156)", 2023-10-09T09:46:10.975 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:17.862969+0000", 2023-10-09T09:46:10.975 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 473.095680743, 2023-10-09T09:46:10.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0628312559999999, 2023-10-09T09:46:10.975 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:10.975 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149574 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862969+0000", 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:10.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862969+0000", 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862969+0000", 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:10.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862972+0000", 2023-10-09T09:46:10.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.303e-06 2023-10-09T09:46:10.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:10.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862979+0000", 2023-10-09T09:46:10.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6980000000000001e-06 2023-10-09T09:46:10.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:10.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.862984+0000", 2023-10-09T09:46:10.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9919999999999998e-06 2023-10-09T09:46:10.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:10.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.865070+0000", 2023-10-09T09:46:10.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020865319999999999 2023-10-09T09:46:10.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T09:46:10.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.865091+0000", 2023-10-09T09:46:10.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1090999999999998e-05 2023-10-09T09:46:10.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:10.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.870002+0000", 2023-10-09T09:46:10.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049112829999999998 2023-10-09T09:46:10.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:10.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.870011+0000", 2023-10-09T09:46:10.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4940000000000003e-06 2023-10-09T09:46:10.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:10.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.894829+0000", 2023-10-09T09:46:10.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0248184119999999 2023-10-09T09:46:10.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:10.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.895067+0000", 2023-10-09T09:46:10.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023758199999999999 2023-10-09T09:46:10.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:10.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.895164+0000", 2023-10-09T09:46:10.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7553999999999995e-05 2023-10-09T09:46:10.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:10.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911292+0000", 2023-10-09T09:46:10.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016127418000000001 2023-10-09T09:46:10.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:10.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911332+0000", 2023-10-09T09:46:10.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9999000000000001e-05 2023-10-09T09:46:10.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:10.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925713+0000", 2023-10-09T09:46:10.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014380828 2023-10-09T09:46:10.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:10.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925761+0000", 2023-10-09T09:46:10.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8671999999999997e-05 2023-10-09T09:46:10.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:10.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925800+0000", 2023-10-09T09:46:10.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9050000000000001e-05 2023-10-09T09:46:10.986 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:10.986 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:10.986 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:10.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.986 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149602 8.f 8:f9b9e31f:::benchmark_data_smithi163_151167_object54369:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:10.986 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:17.909905+0000", 2023-10-09T09:46:10.986 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 473.04874469800001, 2023-10-09T09:46:10.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.015980871, 2023-10-09T09:46:10.987 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:10.987 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:10.987 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:10.987 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:10.987 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:10.987 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149602 2023-10-09T09:46:10.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.987 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:10.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:10.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909905+0000", 2023-10-09T09:46:10.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:10.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:10.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909905+0000", 2023-10-09T09:46:10.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:10.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909904+0000", 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909912+0000", 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8910000000000001e-06 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909927+0000", 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4151000000000001e-05 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.909936+0000", 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4939999999999992e-06 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:10.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.910014+0000", 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7976e-05 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.910044+0000", 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9462999999999999e-05 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.898508+0000", 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98846395499999995 2023-10-09T09:46:10.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.898613+0000", 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000105225 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.898705+0000", 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2535999999999996e-05 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911362+0000", 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012656970999999999 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911394+0000", 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1652999999999997e-05 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925810+0000", 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014416528 2023-10-09T09:46:10.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925851+0000", 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0191999999999997e-05 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925886+0000", 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5478999999999997e-05 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:10.994 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149645 8.f 8:f2fa5649:::benchmark_data_smithi163_151167_object54412:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:17.985066+0000", 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.973583953, 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95494577199999997, 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:10.995 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149645 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985066+0000", 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:10.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985066+0000", 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985064+0000", 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985075+0000", 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0907000000000001e-05 2023-10-09T09:46:10.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985091+0000", 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6419999999999999e-05 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985103+0000", 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1884e-05 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985505+0000", 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00040182999999999999 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:17.985521+0000", 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6116e-05 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:10.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.906317+0000", 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.92079660500000005 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.906486+0000", 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000168765 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.906654+0000", 2023-10-09T09:46:11.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016745799999999999 2023-10-09T09:46:11.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925894+0000", 2023-10-09T09:46:11.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019240489999999999 2023-10-09T09:46:11.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931545+0000", 2023-10-09T09:46:11.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056503940000000004 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.939918+0000", 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0083738700000000003 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.939967+0000", 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9047e-05 2023-10-09T09:46:11.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940012+0000", 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4277999999999999e-05 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149674 8.f 8:f3b05e3f:::benchmark_data_smithi163_151167_object54441:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.032157+0000", 2023-10-09T09:46:11.003 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.92649288699999, 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.90795880600000001, 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149674 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032157+0000", 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032157+0000", 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032155+0000", 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:46:11.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032164+0000", 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4780000000000008e-06 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032180+0000", 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6359e-05 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032188+0000", 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2539999999999992e-06 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032235+0000", 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6653000000000002e-05 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.032244+0000", 2023-10-09T09:46:11.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0669999999999998e-06 2023-10-09T09:46:11.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.008 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911421+0000", 2023-10-09T09:46:11.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.87917706900000003 2023-10-09T09:46:11.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.008 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911522+0000", 2023-10-09T09:46:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000100942 2023-10-09T09:46:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.911614+0000", 2023-10-09T09:46:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1802e-05 2023-10-09T09:46:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925900+0000", 2023-10-09T09:46:11.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014285613000000001 2023-10-09T09:46:11.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931626+0000", 2023-10-09T09:46:11.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0057267189999999999 2023-10-09T09:46:11.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940048+0000", 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0084219800000000008 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940084+0000", 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6093999999999997e-05 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940116+0000", 2023-10-09T09:46:11.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1548999999999999e-05 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149676 8.f 8:f59131ee:::benchmark_data_smithi163_151167_object54443:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.035104+0000", 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.92354561500002, 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.90513551400000003, 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.012 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149676 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035104+0000", 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035104+0000", 2023-10-09T09:46:11.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035103+0000", 2023-10-09T09:46:11.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:46:11.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035110+0000", 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0140000000000001e-06 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035125+0000", 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5302e-05 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035133+0000", 2023-10-09T09:46:11.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8150000000000006e-06 2023-10-09T09:46:11.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035180+0000", 2023-10-09T09:46:11.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6975999999999999e-05 2023-10-09T09:46:11.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.035189+0000", 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0259999999999996e-06 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.915840+0000", 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.88065113800000006 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.916025+0000", 2023-10-09T09:46:11.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018522600000000001 2023-10-09T09:46:11.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.916203+0000", 2023-10-09T09:46:11.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000177276 2023-10-09T09:46:11.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931018+0000", 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014815399 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940147+0000", 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091291749999999998 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940174+0000", 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6531999999999999e-05 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940209+0000", 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5293999999999998e-05 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940240+0000", 2023-10-09T09:46:11.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0859999999999999e-05 2023-10-09T09:46:11.021 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.021 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.021 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.021 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149684 8.f 8:f37ab3bd:::benchmark_data_smithi163_151167_object54451:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.021 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.051053+0000", 2023-10-09T09:46:11.021 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.90759685500001, 2023-10-09T09:46:11.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.91798173000000005, 2023-10-09T09:46:11.021 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.022 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.022 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.022 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.022 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.022 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149684 2023-10-09T09:46:11.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.022 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051053+0000", 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051053+0000", 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051051+0000", 2023-10-09T09:46:11.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051062+0000", 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1054e-05 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051080+0000", 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7592e-05 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051090+0000", 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0839000000000001e-05 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051167+0000", 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6117000000000003e-05 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.051183+0000", 2023-10-09T09:46:11.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6262000000000001e-05 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.921010+0000", 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.86982745500000003 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.921180+0000", 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000169316 2023-10-09T09:46:11.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.921331+0000", 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015102 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931026+0000", 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0096958809999999999 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940271+0000", 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0092446690000000005 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.968932+0000", 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028660354999999998 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.968989+0000", 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7973000000000002e-05 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969035+0000", 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5339000000000001e-05 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149724 8.f 8:f2f8c3fa:::benchmark_data_smithi163_151167_object54491:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.114549+0000", 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.84410068300002, 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.85463660900000005, 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149724 2023-10-09T09:46:11.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114549+0000", 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114549+0000", 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114547+0000", 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114556+0000", 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0010000000000007e-06 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114574+0000", 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7626999999999999e-05 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114584+0000", 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.713e-06 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114664+0000", 2023-10-09T09:46:11.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0255999999999999e-05 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.114679+0000", 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4728e-05 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.925931+0000", 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.81125247599999994 2023-10-09T09:46:11.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.926033+0000", 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000102045 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.926151+0000", 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000117413 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.944896+0000", 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018745458 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969082+0000", 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.02418582 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969113+0000", 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0744000000000002e-05 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969152+0000", 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9651999999999999e-05 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969186+0000", 2023-10-09T09:46:11.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3639000000000001e-05 2023-10-09T09:46:11.038 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.038 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.038 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.038 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149746 8.f 8:ff8f4271:::benchmark_data_smithi163_151167_object54513:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.038 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.159624+0000", 2023-10-09T09:46:11.038 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.79902566599998, 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.80969376900000001, 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149746 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159624+0000", 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159624+0000", 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159623+0000", 2023-10-09T09:46:11.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159632+0000", 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0610000000000002e-06 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159647+0000", 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5207e-05 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159655+0000", 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0669999999999992e-06 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159703+0000", 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8127999999999997e-05 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.159712+0000", 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8100000000000004e-06 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931662+0000", 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.77195008099999995 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931797+0000", 2023-10-09T09:46:11.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013457599999999999 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.931901+0000", 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010447599999999999 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.944903+0000", 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013002231 2023-10-09T09:46:11.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969220+0000", 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024316873999999999 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969249+0000", 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8609e-05 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969287+0000", 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8393999999999999e-05 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969318+0000", 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0932000000000001e-05 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149777 8.f 8:f1a53e60:::benchmark_data_smithi163_151167_object54544:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.216787+0000", 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.74186251700002, 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.75266294300000003, 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149777 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.047 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216787+0000", 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216787+0000", 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216785+0000", 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999971 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216796+0000", 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0947e-05 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216818+0000", 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2759e-05 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216830+0000", 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1379000000000001e-05 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216904+0000", 2023-10-09T09:46:11.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4221999999999995e-05 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.216918+0000", 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3568e-05 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.936247+0000", 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.719328996 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.936397+0000", 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000150385 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.936526+0000", 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012890500000000001 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.964436+0000", 2023-10-09T09:46:11.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.027910213999999999 2023-10-09T09:46:11.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969351+0000", 2023-10-09T09:46:11.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049151129999999996 2023-10-09T09:46:11.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969380+0000", 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8578999999999999e-05 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969418+0000", 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8563999999999999e-05 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969450+0000", 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2038000000000001e-05 2023-10-09T09:46:11.054 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149811 8.f 8:fd4d8c61:::benchmark_data_smithi163_151167_object54578:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.278487+0000", 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.68016297000003, 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.691092178, 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.055 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149811 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278487+0000", 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278487+0000", 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278485+0000", 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278495+0000", 2023-10-09T09:46:11.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0244000000000001e-05 2023-10-09T09:46:11.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278515+0000", 2023-10-09T09:46:11.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9905e-05 2023-10-09T09:46:11.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278527+0000", 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2481e-05 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278600+0000", 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2135e-05 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.278613+0000", 2023-10-09T09:46:11.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3648e-05 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940291+0000", 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.661677403 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940420+0000", 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000129636 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.940539+0000", 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000118744 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.964447+0000", 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023907726000000001 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969485+0000", 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050383040000000004 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969514+0000", 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8507999999999999e-05 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969549+0000", 2023-10-09T09:46:11.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5074999999999999e-05 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969579+0000", 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0528999999999999e-05 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.063 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149812 8.f 8:f90f3522:::benchmark_data_smithi163_151167_object54579:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.282166+0000", 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.67648383400001, 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.70568355299999996, 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149812 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282166+0000", 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282166+0000", 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282164+0000", 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282175+0000", 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0552e-05 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282194+0000", 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9919e-05 2023-10-09T09:46:11.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282213+0000", 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8349999999999999e-05 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282279+0000", 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6113999999999998e-05 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.282300+0000", 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1330000000000001e-05 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.944929+0000", 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.66262830900000003 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.945067+0000", 2023-10-09T09:46:11.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000138114 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.945189+0000", 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012192399999999999 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969612+0000", 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024423112 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969640+0000", 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8708e-05 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987752+0000", 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018111795 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987804+0000", 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1712999999999998e-05 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987850+0000", 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5710000000000001e-05 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.071 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149816 8.f 8:fc3bc3c1:::benchmark_data_smithi163_151167_object54583:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.288122+0000", 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.67052830900002, 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.69983973899999996, 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149816 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.072 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288122+0000", 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288122+0000", 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288120+0000", 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288129+0000", 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9279999999999999e-06 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288146+0000", 2023-10-09T09:46:11.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7521999999999999e-05 2023-10-09T09:46:11.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288157+0000", 2023-10-09T09:46:11.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0482999999999999e-05 2023-10-09T09:46:11.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288221+0000", 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4632000000000001e-05 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.288232+0000", 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0434999999999999e-05 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.949547+0000", 2023-10-09T09:46:11.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.66131544900000006 2023-10-09T09:46:11.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.949764+0000", 2023-10-09T09:46:11.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000216572 2023-10-09T09:46:11.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.949895+0000", 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000131421 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969670+0000", 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019774131 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969700+0000", 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0645000000000002e-05 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987896+0000", 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018195847000000001 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987930+0000", 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3992999999999998e-05 2023-10-09T09:46:11.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987961+0000", 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1390999999999998e-05 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149861 8.f 8:f3f94ebf:::benchmark_data_smithi163_151167_object54628:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.080 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.407055+0000", 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.55159531999999, 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.60205851899999996, 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149861 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407055+0000", 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407055+0000", 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407053+0000", 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407064+0000", 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1618e-05 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407091+0000", 2023-10-09T09:46:11.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6477000000000001e-05 2023-10-09T09:46:11.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407104+0000", 2023-10-09T09:46:11.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3875e-05 2023-10-09T09:46:11.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407229+0000", 2023-10-09T09:46:11.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012451699999999999 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.407257+0000", 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7826000000000001e-05 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.964488+0000", 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.55723115599999995 2023-10-09T09:46:11.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.964665+0000", 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017701699999999999 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.964792+0000", 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000126797 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987889+0000", 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023097166999999998 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987967+0000", 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8273000000000001e-05 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009012+0000", 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021045257000000001 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009071+0000", 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9008000000000002e-05 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009113+0000", 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1649000000000001e-05 2023-10-09T09:46:11.088 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.089 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.089 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.089 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149888 8.f 8:f6339202:::benchmark_data_smithi163_151167_object54655:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.089 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.507613+0000", 2023-10-09T09:46:11.089 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.45103658699998, 2023-10-09T09:46:11.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.50201627400000004, 2023-10-09T09:46:11.089 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149888 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507613+0000", 2023-10-09T09:46:11.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507613+0000", 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507611+0000", 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:46:11.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507621+0000", 2023-10-09T09:46:11.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.363e-06 2023-10-09T09:46:11.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507638+0000", 2023-10-09T09:46:11.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7513e-05 2023-10-09T09:46:11.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.093 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507654+0000", 2023-10-09T09:46:11.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5265e-05 2023-10-09T09:46:11.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.093 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507713+0000", 2023-10-09T09:46:11.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9049999999999999e-05 2023-10-09T09:46:11.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.507723+0000", 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0159999999999999e-05 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969722+0000", 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.46199886200000001 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969849+0000", 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012760500000000001 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.969972+0000", 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012303800000000001 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987972+0000", 2023-10-09T09:46:11.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017999505999999998 2023-10-09T09:46:11.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.988007+0000", 2023-10-09T09:46:11.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5256999999999998e-05 2023-10-09T09:46:11.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009540+0000", 2023-10-09T09:46:11.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021533251999999999 2023-10-09T09:46:11.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009588+0000", 2023-10-09T09:46:11.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7685999999999997e-05 2023-10-09T09:46:11.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.098 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.098 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009630+0000", 2023-10-09T09:46:11.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1603000000000001e-05 2023-10-09T09:46:11.098 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.098 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.099 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.099 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45122.0:149908 8.f 8:fcc39bf8:::benchmark_data_smithi163_151167_object54675:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1156)", 2023-10-09T09:46:11.099 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:38:18.649138+0000", 2023-10-09T09:46:11.099 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 472.30951223599999, 2023-10-09T09:46:11.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.36085224100000002, 2023-10-09T09:46:11.099 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.099 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.100 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.100 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45122", 2023-10-09T09:46:11.100 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:46:11.100 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 149908 2023-10-09T09:46:11.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.100 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649138+0000", 2023-10-09T09:46:11.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649138+0000", 2023-10-09T09:46:11.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.102 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649135+0000", 2023-10-09T09:46:11.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:46:11.102 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.102 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.102 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649148+0000", 2023-10-09T09:46:11.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2284e-05 2023-10-09T09:46:11.102 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649172+0000", 2023-10-09T09:46:11.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4083e-05 2023-10-09T09:46:11.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649188+0000", 2023-10-09T09:46:11.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6019999999999999e-05 2023-10-09T09:46:11.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649270+0000", 2023-10-09T09:46:11.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2602999999999995e-05 2023-10-09T09:46:11.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:46:11.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.649285+0000", 2023-10-09T09:46:11.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4171999999999999e-05 2023-10-09T09:46:11.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.973652+0000", 2023-10-09T09:46:11.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32436722299999998 2023-10-09T09:46:11.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.973763+0000", 2023-10-09T09:46:11.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011120600000000001 2023-10-09T09:46:11.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T09:46:11.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.973867+0000", 2023-10-09T09:46:11.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0001041 2023-10-09T09:46:11.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.107 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.987977+0000", 2023-10-09T09:46:11.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01411041 2023-10-09T09:46:11.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.107 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:18.988035+0000", 2023-10-09T09:46:11.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7345000000000001e-05 2023-10-09T09:46:11.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009683+0000", 2023-10-09T09:46:11.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021648356000000001 2023-10-09T09:46:11.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009934+0000", 2023-10-09T09:46:11.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000250726 2023-10-09T09:46:11.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:38:19.009990+0000", 2023-10-09T09:46:11.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6064999999999999e-05 2023-10-09T09:46:11.109 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.109 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.109 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.110 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45122.0:189420 8.4 e1166/1163 8:23866f45:::benchmark_data_smithi163_151167_object94187:head v 1166'11857, mlcod=1151'5395)", 2023-10-09T09:46:11.110 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:39:26.962808+0000", 2023-10-09T09:46:11.110 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 403.99584227499997, 2023-10-09T09:46:11.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.30114892100000001, 2023-10-09T09:46:11.110 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.110 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.110 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962808+0000", 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962808+0000", 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962807+0000", 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962816+0000", 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3499999999999997e-06 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962822+0000", 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6170000000000003e-06 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.962827+0000", 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8930000000000004e-06 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.968913+0000", 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060862099999999999 2023-10-09T09:46:11.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.968943+0000", 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9428999999999999e-05 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.263876+0000", 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.294933628 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.263957+0000", 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0199999999999998e-05 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45122.0:189421 8.e e1166/1150 8:7b02ee27:::benchmark_data_smithi163_151167_object94188:head v 1166'12071, mlcod=1166'12066)", 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:39:26.970736+0000", 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 403.98791443499999, 2023-10-09T09:46:11.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28797570300000003, 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970736+0000", 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970736+0000", 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970735+0000", 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970740+0000", 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7469999999999996e-06 2023-10-09T09:46:11.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970747+0000", 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2110000000000001e-06 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970752+0000", 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4890000000000003e-06 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970804+0000", 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1493999999999999e-05 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:26.970823+0000", 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8776000000000001e-05 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.258673+0000", 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28785032500000002 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:39:27.258711+0000", 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8142000000000002e-05 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.120 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:41147 9.6 9:61b14219:::benchmark_data_smithi163_175771_object41146:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1195)", 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:41:34.527825+0000", 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 276.43082506000002, 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33321299700000001, 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 41147 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.121 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.527825+0000", 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.527825+0000", 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.527823+0000", 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.527852+0000", 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8861999999999999e-05 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.527865+0000", 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3397000000000001e-05 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.527875+0000", 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1009999999999998e-06 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.527940+0000", 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5092999999999995e-05 2023-10-09T09:46:11.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528120+0000", 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017991099999999999 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.528279+0000", 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015993999999999999 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530222+0000", 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001942551 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.530803+0000", 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00058073200000000004 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.860851+0000", 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33004807899999999 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.860952+0000", 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010081000000000001 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.861038+0000", 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6236000000000006e-05 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35412.0:41156 9.7 9:e07be117:::benchmark_data_smithi163_175771_object41155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1195)", 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:41:34.534848+0000", 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 276.42380149899998, 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33431555800000001, 2023-10-09T09:46:11.128 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:46:11.129 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:46:11.129 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:46:11.129 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35412", 2023-10-09T09:46:11.129 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:46:11.129 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 41156 2023-10-09T09:46:11.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.129 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:46:11.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:46:11.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.534848+0000", 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.534848+0000", 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.534847+0000", 2023-10-09T09:46:11.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.534867+0000", 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9854999999999999e-05 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.534894+0000", 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7126000000000001e-05 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.534901+0000", 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3050000000000004e-06 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.534952+0000", 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0633000000000001e-05 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:46:11.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.535073+0000", 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121308 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.535190+0000", 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011663999999999999 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.536663+0000", 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014733229999999999 2023-10-09T09:46:11.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.539773+0000", 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031105239999999999 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.868073+0000", 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32829952099999998 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.868218+0000", 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000145012 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:41:34.869164+0000", 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00094602 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:46:11.135 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:46:11.136 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:46:11.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:11.137 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:10 smithi163 bash[78107]: audit 2023-10-09T09:46:10.392127+0000 mon.a (mon.0) 2259 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:11.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:10 smithi163 bash[75425]: audit 2023-10-09T09:46:10.392127+0000 mon.a (mon.0) 2259 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:11.165 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:11.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:10 smithi181 bash[59345]: audit 2023-10-09T09:46:10.392127+0000 mon.a (mon.0) 2259 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:11.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:11.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:11.272 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:11.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:11.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:11.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:11.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:11.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:11.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:11.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:11.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 49502 49486 114.54 77.875 0.00999804 0.00872561 2023-10-09T09:46:11.536 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:11.568 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:11.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:11.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:11.708 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:11.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:11.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:11.814 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:11.846 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:11.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:11.923 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T09:46:11.923 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:46:11.950 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:11.984 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:11.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:11.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:12.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:11 smithi181 bash[59345]: cluster 2023-10-09T09:46:10.078891+0000 mgr.y (mgr.34107) 1204 : cluster 0 pgmap v2036: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T09:46:12.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:11 smithi181 bash[59345]: audit 2023-10-09T09:46:11.046695+0000 mon.c (mon.2) 110 : audit 1 from='client.? 172.21.15.163:0/1543944673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:46:12.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:11 smithi181 bash[59345]: audit 2023-10-09T09:46:11.047783+0000 mon.a (mon.0) 2260 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:46:12.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:11 smithi181 bash[59345]: audit 2023-10-09T09:46:11.156506+0000 mon.a (mon.0) 2261 : audit 1 from='client.? 172.21.15.163:0/1248065443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:46:12.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:12.097 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:11 smithi163 bash[75425]: cluster 2023-10-09T09:46:10.078891+0000 mgr.y (mgr.34107) 1204 : cluster 0 pgmap v2036: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T09:46:12.097 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:11 smithi163 bash[75425]: audit 2023-10-09T09:46:11.046695+0000 mon.c (mon.2) 110 : audit 1 from='client.? 172.21.15.163:0/1543944673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:46:12.098 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:11 smithi163 bash[75425]: audit 2023-10-09T09:46:11.047783+0000 mon.a (mon.0) 2260 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:46:12.098 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:11 smithi163 bash[75425]: audit 2023-10-09T09:46:11.156506+0000 mon.a (mon.0) 2261 : audit 1 from='client.? 172.21.15.163:0/1248065443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:46:12.098 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:11 smithi163 bash[78107]: cluster 2023-10-09T09:46:10.078891+0000 mgr.y (mgr.34107) 1204 : cluster 0 pgmap v2036: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T09:46:12.098 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:11 smithi163 bash[78107]: audit 2023-10-09T09:46:11.046695+0000 mon.c (mon.2) 110 : audit 1 from='client.? 172.21.15.163:0/1543944673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:46:12.099 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:11 smithi163 bash[78107]: audit 2023-10-09T09:46:11.047783+0000 mon.a (mon.0) 2260 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:46:12.099 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:11 smithi163 bash[78107]: audit 2023-10-09T09:46:11.156506+0000 mon.a (mon.0) 2261 : audit 1 from='client.? 172.21.15.163:0/1248065443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:46:12.115 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:12.116 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:46:12.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:12.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:12.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:12.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:12.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:12.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:12.350 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:12.384 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:12.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:12.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:12.487 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:12.510 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:12.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:12.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:12.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 50767 50751 113.272 79.0625 0.0180266 0.00882229 2023-10-09T09:46:12.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T09:46:12.619 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:12.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:12.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:12.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:12.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:12.790 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:12.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:12.893 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:12.923 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:12.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:12.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:13.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:13.053 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:13.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:13.155 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:13.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:12 smithi181 bash[59345]: cluster 2023-10-09T09:46:11.907970+0000 mon.a (mon.0) 2262 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:46:13.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:12 smithi181 bash[59345]: cluster 2023-10-09T09:46:11.909744+0000 osd.5 (osd.5) 823 : cluster 3 failed to encode map e1236 with expected crc 2023-10-09T09:46:13.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:12 smithi181 bash[59345]: audit 2023-10-09T09:46:11.912928+0000 mon.a (mon.0) 2263 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:46:13.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:12 smithi181 bash[59345]: audit 2023-10-09T09:46:11.913095+0000 mon.a (mon.0) 2264 : audit 1 from='client.? 172.21.15.163:0/1248065443' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:46:13.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:12 smithi181 bash[59345]: cluster 2023-10-09T09:46:11.913171+0000 mon.a (mon.0) 2265 : cluster 0 osdmap e1236: 8 total, 8 up, 6 in 2023-10-09T09:46:13.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:12 smithi181 bash[59345]: cluster 2023-10-09T09:46:11.915623+0000 osd.4 (osd.4) 641 : cluster 3 failed to encode map e1236 with expected crc 2023-10-09T09:46:13.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:13.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:13.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:13.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[78107]: cluster 2023-10-09T09:46:11.907970+0000 mon.a (mon.0) 2262 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:46:13.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[78107]: cluster 2023-10-09T09:46:11.909744+0000 osd.5 (osd.5) 823 : cluster 3 failed to encode map e1236 with expected crc 2023-10-09T09:46:13.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[78107]: audit 2023-10-09T09:46:11.912928+0000 mon.a (mon.0) 2263 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:46:13.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[78107]: audit 2023-10-09T09:46:11.913095+0000 mon.a (mon.0) 2264 : audit 1 from='client.? 172.21.15.163:0/1248065443' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:46:13.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[78107]: cluster 2023-10-09T09:46:11.913171+0000 mon.a (mon.0) 2265 : cluster 0 osdmap e1236: 8 total, 8 up, 6 in 2023-10-09T09:46:13.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[78107]: cluster 2023-10-09T09:46:11.915623+0000 osd.4 (osd.4) 641 : cluster 3 failed to encode map e1236 with expected crc 2023-10-09T09:46:13.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[75425]: cluster 2023-10-09T09:46:11.907970+0000 mon.a (mon.0) 2262 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:46:13.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[75425]: cluster 2023-10-09T09:46:11.909744+0000 osd.5 (osd.5) 823 : cluster 3 failed to encode map e1236 with expected crc 2023-10-09T09:46:13.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[75425]: audit 2023-10-09T09:46:11.912928+0000 mon.a (mon.0) 2263 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:46:13.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[75425]: audit 2023-10-09T09:46:11.913095+0000 mon.a (mon.0) 2264 : audit 1 from='client.? 172.21.15.163:0/1248065443' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:46:13.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[75425]: cluster 2023-10-09T09:46:11.913171+0000 mon.a (mon.0) 2265 : cluster 0 osdmap e1236: 8 total, 8 up, 6 in 2023-10-09T09:46:13.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:12 smithi163 bash[75425]: cluster 2023-10-09T09:46:11.915623+0000 osd.4 (osd.4) 641 : cluster 3 failed to encode map e1236 with expected crc 2023-10-09T09:46:13.290 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:13.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:13.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:13.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:13.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:13.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:13.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:13.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:13.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 52056 52040 112.144 80.5625 0.00842734 0.0089107 2023-10-09T09:46:13.559 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:13.602 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:13.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:13.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:13.706 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:46:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:46:13] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.33.4" 2023-10-09T09:46:13.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:13.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:13.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:13.850 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:13.885 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:13.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:13.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:13.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:13.997 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:13 smithi181 bash[59345]: cluster 2023-10-09T09:46:12.079341+0000 mgr.y (mgr.34107) 1205 : cluster 0 pgmap v2038: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T09:46:14.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:14.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:14.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:14.122 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:14.153 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:14.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:14.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:14.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:13 smithi163 bash[78107]: cluster 2023-10-09T09:46:12.079341+0000 mgr.y (mgr.34107) 1205 : cluster 0 pgmap v2038: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T09:46:14.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:13 smithi163 bash[75425]: cluster 2023-10-09T09:46:12.079341+0000 mgr.y (mgr.34107) 1205 : cluster 0 pgmap v2038: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T09:46:14.256 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:14.290 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:14.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:14.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:14.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:14.425 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:14.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:14.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:14.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:14.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 53376 53360 111.156 82.5 0.0122439 0.00899128 2023-10-09T09:46:14.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:14.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:14.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:14.610 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T09:46:14.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:14.734 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:14.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:14.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:14.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:14.873 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:14.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:14.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:14.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:15.012 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:15.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:15.124 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:15.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:15.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:15.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:15.251 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:15.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:15.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:15.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:15.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:15.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:15.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:15.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:15.525 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:15.534 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 54609 54593 110.056 77.0625 0.0104316 0.00908102 2023-10-09T09:46:15.565 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:15.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:15.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:15.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:15.699 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:15.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:15.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:15.803 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:15.841 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:15.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:15.938 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:15 smithi181 bash[59345]: cluster 2023-10-09T09:46:14.080019+0000 mgr.y (mgr.34107) 1206 : cluster 0 pgmap v2039: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T09:46:15.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:15 smithi163 bash[75425]: cluster 2023-10-09T09:46:14.080019+0000 mgr.y (mgr.34107) 1206 : cluster 0 pgmap v2039: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T09:46:15.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:15.971 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:15.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:15.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:16.074 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:16.106 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:16.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:16.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:16.211 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:16.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:15 smithi163 bash[78107]: cluster 2023-10-09T09:46:14.080019+0000 mgr.y (mgr.34107) 1206 : cluster 0 pgmap v2039: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T09:46:16.241 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:16.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:16.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:16.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:16.374 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:16.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:16.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:16.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:16.512 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:16.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:16.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:16.535 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 55871 55855 109.081 78.875 0.0134678 0.00916184 2023-10-09T09:46:16.593 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:46:16.616 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:16.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:16.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:16.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:16.792 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:16.829 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:16.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:16.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:16.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:16.965 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:16.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:16.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:17.022 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:46:17.023 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:46:17.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:17.100 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:17.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:17.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:17.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:17.233 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:17.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:17.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:17.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:17.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:17.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:17.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:17.475 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:17.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:17.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:17.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:17.535 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 57119 57103 108.139 78 0.0125838 0.00924141 2023-10-09T09:46:17.611 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:17.643 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:17.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:17.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:17.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:17.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:17.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:17.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:17.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:17.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:17.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:18.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:18.035 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:17 smithi181 bash[59345]: cluster 2023-10-09T09:46:16.080589+0000 mgr.y (mgr.34107) 1207 : cluster 0 pgmap v2040: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T09:46:18.053 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:18.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:18.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:18.157 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:18.166 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:17 smithi163 bash[78107]: cluster 2023-10-09T09:46:16.080589+0000 mgr.y (mgr.34107) 1207 : cluster 0 pgmap v2040: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T09:46:18.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:17 smithi163 bash[75425]: cluster 2023-10-09T09:46:16.080589+0000 mgr.y (mgr.34107) 1207 : cluster 0 pgmap v2040: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T09:46:18.180 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:18.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:18.283 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:18.307 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:18.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:18.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:18.411 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:18.441 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:18.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:18.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:18.535 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 58373 58357 107.263 78.375 0.0163581 0.009318 2023-10-09T09:46:18.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:18.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:18.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:18.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:18.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:18.711 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:18.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:18.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:18.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:18.846 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:18.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:18.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:18.953 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:18.980 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:18.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:18.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:19.063 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:46:19.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:19.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:19.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:19.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:19.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:19.255 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:19.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:19.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:19.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:19.389 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:19.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:19.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:19.492 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:19.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:19.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:19.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:19.535 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 59596 59580 106.382 76.4375 0.020158 0.00939311 2023-10-09T09:46:19.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:19.651 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:19.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:19.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:19.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:19.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:19.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:19.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:19.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:19.921 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:19.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:20.030 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:20.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:19 smithi163 bash[75425]: cluster 2023-10-09T09:46:18.081336+0000 mgr.y (mgr.34107) 1208 : cluster 0 pgmap v2041: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T09:46:20.040 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:19 smithi163 bash[78107]: cluster 2023-10-09T09:46:18.081336+0000 mgr.y (mgr.34107) 1208 : cluster 0 pgmap v2041: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T09:46:20.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:20.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:20.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:19 smithi181 bash[59345]: cluster 2023-10-09T09:46:18.081336+0000 mgr.y (mgr.34107) 1208 : cluster 0 pgmap v2041: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T09:46:20.164 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:20.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:20.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:20.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:20.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:20.319 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:20.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:20.426 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:20.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:20.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:20.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:20.535 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 60743 60727 105.419 71.6875 0.0103478 0.00947974 2023-10-09T09:46:20.556 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:20.584 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:20.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:20.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:20.690 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:20.719 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:20.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:20.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:20.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:20.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:20.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:20.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:20.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:20.983 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:20.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:20.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:21.086 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:21.097 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:46:21.105 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:21.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:21.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:21.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:21.229 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:21.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:21.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:21.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:21.356 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:21.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:21.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:21.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:21.484 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:21.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:21.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:21.535 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 61845 61829 104.431 68.875 0.0136418 0.00957049 2023-10-09T09:46:21.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:21.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:21.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:21.725 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:21.756 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:21.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:21.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:21.859 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:21.886 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:21.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:21.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:21.991 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:22.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:22.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:22.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:22.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:22.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:22.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:22.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:22.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:21 smithi163 bash[78107]: cluster 2023-10-09T09:46:20.081931+0000 mgr.y (mgr.34107) 1209 : cluster 0 pgmap v2042: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T09:46:22.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:21 smithi163 bash[75425]: cluster 2023-10-09T09:46:20.081931+0000 mgr.y (mgr.34107) 1209 : cluster 0 pgmap v2042: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T09:46:22.263 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:22.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:22.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:22.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:21 smithi181 bash[59345]: cluster 2023-10-09T09:46:20.081931+0000 mgr.y (mgr.34107) 1209 : cluster 0 pgmap v2042: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T09:46:22.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:22.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:22.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:22.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:22.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:22.535 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 62967 62951 103.528 70.125 0.00878794 0.00965403 2023-10-09T09:46:22.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:22.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:22.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:22.651 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:22.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:22.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:22.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:22.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:22.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:22.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:22.935 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:22.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:22.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:23.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:23.070 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:23.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:23.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:23.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:23.204 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:23.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:23.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:23.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:23.334 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:23.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:23.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:23.440 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:23.474 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:23.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:23.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:23.535 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 64082 64066 102.66 69.6875 0.0140652 0.00973502 2023-10-09T09:46:23.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:23.611 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:23.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:23.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:23.705 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:46:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:46:23] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.33.4" 2023-10-09T09:46:23.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:23.746 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:23.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:23.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:23.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:23.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:23.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:23.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:23.989 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:24.010 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:24.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:24.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:24.116 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:24.143 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:24.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:24.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:24.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:23 smithi163 bash[75425]: cluster 2023-10-09T09:46:22.082578+0000 mgr.y (mgr.34107) 1210 : cluster 0 pgmap v2043: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-10-09T09:46:24.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:23 smithi163 bash[78107]: cluster 2023-10-09T09:46:22.082578+0000 mgr.y (mgr.34107) 1210 : cluster 0 pgmap v2043: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-10-09T09:46:24.248 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:24.276 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:24.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:24.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:24.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:23 smithi181 bash[59345]: cluster 2023-10-09T09:46:22.082578+0000 mgr.y (mgr.34107) 1210 : cluster 0 pgmap v2043: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-10-09T09:46:24.380 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:24.400 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:24.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:24.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:24.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:24.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:24.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:24.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:24.537 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:46:24.537446+0000 min lat: 0.00129664 max lat: 0.198757 avg lat: 0.00981815 2023-10-09T09:46:24.537 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:46:24.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 65174 65158 101.795 68.25 0.0158666 0.00981815 2023-10-09T09:46:24.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:24.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:24.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:24.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:24.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:24.806 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:24.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:24.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:24.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:24.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:24.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:24.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:25.057 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:25.087 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:25.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:25.190 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:25.220 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:25.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:25.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:25.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:25.350 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:25.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:25.458 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:25.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:25.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:25.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:25.537 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 66315 66299 101.052 71.3125 0.0260928 0.00989007 2023-10-09T09:46:25.597 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:25.628 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:25.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:25.734 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:25.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:25.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:25.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:25.869 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:25.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:25.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:25.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:25.998 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:26.019 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:26.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:25 smithi181 bash[59345]: cluster 2023-10-09T09:46:24.083486+0000 mgr.y (mgr.34107) 1211 : cluster 0 pgmap v2044: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T09:46:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:25 smithi181 bash[59345]: audit 2023-10-09T09:46:25.392539+0000 mon.a (mon.0) 2266 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:26.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:25 smithi181 bash[59345]: audit 2023-10-09T09:46:25.816798+0000 mon.a (mon.0) 2267 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-10-09T09:46:26.123 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:26.152 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:26.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:26.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:26.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:25 smithi163 bash[78107]: cluster 2023-10-09T09:46:24.083486+0000 mgr.y (mgr.34107) 1211 : cluster 0 pgmap v2044: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T09:46:26.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:25 smithi163 bash[78107]: audit 2023-10-09T09:46:25.392539+0000 mon.a (mon.0) 2266 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:26.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:25 smithi163 bash[78107]: audit 2023-10-09T09:46:25.816798+0000 mon.a (mon.0) 2267 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-10-09T09:46:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:25 smithi163 bash[75425]: cluster 2023-10-09T09:46:24.083486+0000 mgr.y (mgr.34107) 1211 : cluster 0 pgmap v2044: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T09:46:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:25 smithi163 bash[75425]: audit 2023-10-09T09:46:25.392539+0000 mon.a (mon.0) 2266 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:26.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:25 smithi163 bash[75425]: audit 2023-10-09T09:46:25.816798+0000 mon.a (mon.0) 2267 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-10-09T09:46:26.259 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:26.281 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:26.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:26.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:26.385 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:26.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:26.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:26.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:26.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:26.537 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 67403 67387 100.265 68 0.0224752 0.00996843 2023-10-09T09:46:26.546 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:26.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:26.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:26.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:26.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:26.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:26.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:26.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:26.821 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:26.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:26.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:26.925 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:26.950 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:26.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:26.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:27.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:26 smithi181 bash[59345]: cluster 2023-10-09T09:46:25.969396+0000 osd.5 (osd.5) 824 : cluster 3 failed to encode map e1237 with expected crc 2023-10-09T09:46:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:26 smithi181 bash[59345]: cluster 2023-10-09T09:46:25.969612+0000 osd.4 (osd.4) 642 : cluster 3 failed to encode map e1237 with expected crc 2023-10-09T09:46:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:26 smithi181 bash[59345]: audit 2023-10-09T09:46:25.973035+0000 mon.a (mon.0) 2268 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-10-09T09:46:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:26 smithi181 bash[59345]: cluster 2023-10-09T09:46:25.973224+0000 mon.a (mon.0) 2269 : cluster 0 osdmap e1237: 8 total, 8 up, 6 in 2023-10-09T09:46:27.082 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:27.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:27.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:27.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:27.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:27.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:26 smithi163 bash[78107]: cluster 2023-10-09T09:46:25.969396+0000 osd.5 (osd.5) 824 : cluster 3 failed to encode map e1237 with expected crc 2023-10-09T09:46:27.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:26 smithi163 bash[78107]: cluster 2023-10-09T09:46:25.969612+0000 osd.4 (osd.4) 642 : cluster 3 failed to encode map e1237 with expected crc 2023-10-09T09:46:27.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:26 smithi163 bash[78107]: audit 2023-10-09T09:46:25.973035+0000 mon.a (mon.0) 2268 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-10-09T09:46:27.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:26 smithi163 bash[78107]: cluster 2023-10-09T09:46:25.973224+0000 mon.a (mon.0) 2269 : cluster 0 osdmap e1237: 8 total, 8 up, 6 in 2023-10-09T09:46:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:26 smithi163 bash[75425]: cluster 2023-10-09T09:46:25.969396+0000 osd.5 (osd.5) 824 : cluster 3 failed to encode map e1237 with expected crc 2023-10-09T09:46:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:26 smithi163 bash[75425]: cluster 2023-10-09T09:46:25.969612+0000 osd.4 (osd.4) 642 : cluster 3 failed to encode map e1237 with expected crc 2023-10-09T09:46:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:26 smithi163 bash[75425]: audit 2023-10-09T09:46:25.973035+0000 mon.a (mon.0) 2268 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-10-09T09:46:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:26 smithi163 bash[75425]: cluster 2023-10-09T09:46:25.973224+0000 mon.a (mon.0) 2269 : cluster 0 osdmap e1237: 8 total, 8 up, 6 in 2023-10-09T09:46:27.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:27.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:27.322 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:27.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:27.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:27.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:27.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:27.473 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:27.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:27.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:27.537 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 68543 68527 99.5898 71.25 0.0133308 0.0100348 2023-10-09T09:46:27.578 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:27.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:27.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:27.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:27.754 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:27.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:27.857 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:27.883 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:27.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:27.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:27.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:28.013 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:28.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:28.118 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:28.149 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:28.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:28.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:28.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:28.261 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:27 smithi163 bash[78107]: cluster 2023-10-09T09:46:26.083959+0000 mgr.y (mgr.34107) 1212 : cluster 0 pgmap v2046: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T09:46:28.261 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:27 smithi163 bash[78107]: cluster 2023-10-09T09:46:26.985177+0000 osd.5 (osd.5) 825 : cluster 3 failed to encode map e1238 with expected crc 2023-10-09T09:46:28.261 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:27 smithi163 bash[78107]: cluster 2023-10-09T09:46:26.989273+0000 mon.a (mon.0) 2270 : cluster 0 osdmap e1238: 8 total, 8 up, 6 in 2023-10-09T09:46:28.261 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:27 smithi163 bash[78107]: cluster 2023-10-09T09:46:27.020742+0000 osd.7 (osd.7) 597 : cluster 3 failed to encode map e1238 with expected crc 2023-10-09T09:46:28.262 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:27 smithi163 bash[75425]: cluster 2023-10-09T09:46:26.083959+0000 mgr.y (mgr.34107) 1212 : cluster 0 pgmap v2046: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T09:46:28.262 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:27 smithi163 bash[75425]: cluster 2023-10-09T09:46:26.985177+0000 osd.5 (osd.5) 825 : cluster 3 failed to encode map e1238 with expected crc 2023-10-09T09:46:28.262 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:27 smithi163 bash[75425]: cluster 2023-10-09T09:46:26.989273+0000 mon.a (mon.0) 2270 : cluster 0 osdmap e1238: 8 total, 8 up, 6 in 2023-10-09T09:46:28.262 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:27 smithi163 bash[75425]: cluster 2023-10-09T09:46:27.020742+0000 osd.7 (osd.7) 597 : cluster 3 failed to encode map e1238 with expected crc 2023-10-09T09:46:28.279 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:28.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:28.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:28.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:27 smithi181 bash[59345]: cluster 2023-10-09T09:46:26.083959+0000 mgr.y (mgr.34107) 1212 : cluster 0 pgmap v2046: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T09:46:28.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:27 smithi181 bash[59345]: cluster 2023-10-09T09:46:26.985177+0000 osd.5 (osd.5) 825 : cluster 3 failed to encode map e1238 with expected crc 2023-10-09T09:46:28.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:27 smithi181 bash[59345]: cluster 2023-10-09T09:46:26.989273+0000 mon.a (mon.0) 2270 : cluster 0 osdmap e1238: 8 total, 8 up, 6 in 2023-10-09T09:46:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:27 smithi181 bash[59345]: cluster 2023-10-09T09:46:27.020742+0000 osd.7 (osd.7) 597 : cluster 3 failed to encode map e1238 with expected crc 2023-10-09T09:46:28.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:28.413 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:28.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:28.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:28.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:28.537 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 69609 69593 98.8404 66.625 0.0037763 0.0100994 2023-10-09T09:46:28.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:28.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:28.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:28.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:28.683 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:28.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:28.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:28.788 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:28.823 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:28.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:28.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:28.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:28.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:28.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:29.060 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:29.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:29.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:29.189 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:29.224 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:29.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:29.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:29.329 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:29.361 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:29.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:29.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:29.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:29.494 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:29.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:29.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:29.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 70400 70384 97.7425 49.4375 0.0171077 0.0102252 2023-10-09T09:46:29.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:29.627 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:29.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:29.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:29.733 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:29.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:29.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:29.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:29.869 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:29.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:29.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:29.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:30.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:30.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:30.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:30.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:30.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:29 smithi181 bash[59345]: cluster 2023-10-09T09:46:28.084448+0000 mgr.y (mgr.34107) 1213 : cluster 0 pgmap v2048: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T09:46:30.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:30.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:30.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:30.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:30.258 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:29 smithi163 bash[78107]: cluster 2023-10-09T09:46:28.084448+0000 mgr.y (mgr.34107) 1213 : cluster 0 pgmap v2048: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T09:46:30.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:29 smithi163 bash[75425]: cluster 2023-10-09T09:46:28.084448+0000 mgr.y (mgr.34107) 1213 : cluster 0 pgmap v2048: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T09:46:30.265 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:30.296 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:30.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:30.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:30.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:30.430 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:30.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:30.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:30.535 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:30.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 71488 71472 97.0958 68 0.0107249 0.010294 2023-10-09T09:46:30.567 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:30.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:30.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:30.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:30.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:30.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:30.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:30.824 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:30.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:30.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:30.929 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:30.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:30.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:30.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:31.053 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:31.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:31.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:31.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:31.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:31.244 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:31.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:31.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:31.351 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:31.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:31.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:31.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:31.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:31.499 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:31.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:31.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:31.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 72496 72480 96.3702 63 0.103326 0.0103722 2023-10-09T09:46:31.603 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:31.627 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:31.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:31.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:31.731 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:31.758 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:31.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:31.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:31.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:31.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:31.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:31.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:32.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:32.033 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:32.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:32.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:32.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:32.146 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:31 smithi181 bash[59345]: cluster 2023-10-09T09:46:30.085016+0000 mgr.y (mgr.34107) 1214 : cluster 0 pgmap v2049: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-09T09:46:32.171 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:32.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:32.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:32.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:32.284 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:31 smithi163 bash[78107]: cluster 2023-10-09T09:46:30.085016+0000 mgr.y (mgr.34107) 1214 : cluster 0 pgmap v2049: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-09T09:46:32.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:31 smithi163 bash[75425]: cluster 2023-10-09T09:46:30.085016+0000 mgr.y (mgr.34107) 1214 : cluster 0 pgmap v2049: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-09T09:46:32.302 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:32.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:32.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:32.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:32.440 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:32.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:32.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:32.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 73508 73492 95.6801 63.25 0.00953034 0.0104459 2023-10-09T09:46:32.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:32.573 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:32.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:32.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:32.680 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:32.705 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:32.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:32.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:32.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:32.839 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:32.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:32.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:32.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:32.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:32.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:32.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:33.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:33.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:33.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:33.213 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:33.251 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:33.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:33.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:33.356 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:33.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:33.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:33.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:33.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:33.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:33.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:33.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:33.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 74360 74344 94.8141 53.25 0.0313741 0.0105414 2023-10-09T09:46:33.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:33.662 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:33.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:33.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:33.665 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:46:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:46:33] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.33.4" 2023-10-09T09:46:33.766 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:33.795 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:33.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:33.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:33.901 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:33.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:33.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:34.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:34.063 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:34.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:34.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:34.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:33 smithi181 bash[59345]: cluster 2023-10-09T09:46:32.086663+0000 mgr.y (mgr.34107) 1215 : cluster 0 pgmap v2050: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:46:34.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:34.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:34.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:34.295 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:34.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:33 smithi163 bash[78107]: cluster 2023-10-09T09:46:32.086663+0000 mgr.y (mgr.34107) 1215 : cluster 0 pgmap v2050: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:46:34.305 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:33 smithi163 bash[75425]: cluster 2023-10-09T09:46:32.086663+0000 mgr.y (mgr.34107) 1215 : cluster 0 pgmap v2050: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:46:34.318 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:34.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:34.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:34.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:34.458 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:34.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:34.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:34.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 75420 75404 94.2428 66.25 0.00924328 0.0106054 2023-10-09T09:46:34.561 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:34.588 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:34.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:34.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:34.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:34.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:34.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:34.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:34.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:34.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:34.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:34.968 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:35.002 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:35.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:35.099 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:46:35.109 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:35.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:35 smithi181 bash[59345]: cluster 2023-10-09T09:46:34.087342+0000 mgr.y (mgr.34107) 1216 : cluster 0 pgmap v2051: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T09:46:35.134 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:46:35.134 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:35.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:35.164 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:46:35.201 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:46:35.239 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:46:35.239 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:35.270 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:46:35.273 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:35.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:35.307 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:46:35.338 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:46:35.342 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:46:35.342 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:46:35.342 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:46:35.342 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:46:35.342 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:46:35.342 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:46:35.343 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:46:35.343 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:46:35.374 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:35 smithi163 bash[75425]: cluster 2023-10-09T09:46:34.087342+0000 mgr.y (mgr.34107) 1216 : cluster 0 pgmap v2051: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T09:46:35.374 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:35 smithi163 bash[78107]: cluster 2023-10-09T09:46:34.087342+0000 mgr.y (mgr.34107) 1216 : cluster 0 pgmap v2051: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T09:46:35.377 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:35.403 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:35.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:35.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:35.506 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:35.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 76310 76294 93.4855 55.625 0.0149313 0.0106913 2023-10-09T09:46:35.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:35.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:35.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:35.644 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:35.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:35.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:35.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:35.777 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:35.807 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:35.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:35.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:35.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:35.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:35.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:35.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:36.052 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:36.078 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:36.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:36.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:36.183 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:36.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:36 smithi163 bash[78107]: audit 2023-10-09T09:46:35.057529+0000 mon.a (mon.0) 2271 : audit 0 from='client.? 172.21.15.163:0/290499904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:46:36.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:36 smithi163 bash[75425]: audit 2023-10-09T09:46:35.057529+0000 mon.a (mon.0) 2271 : audit 0 from='client.? 172.21.15.163:0/290499904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:46:36.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:36.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:36.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:36.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:36.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:36 smithi181 bash[59345]: audit 2023-10-09T09:46:35.057529+0000 mon.a (mon.0) 2271 : audit 0 from='client.? 172.21.15.163:0/290499904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:46:36.344 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:36.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:36.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:36.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:36.474 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:36.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:36.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:36.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 77328 77312 92.9111 63.625 0.00919762 0.0107493 2023-10-09T09:46:36.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:36.606 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:36.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:36.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:36.710 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:36.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:36.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:36.844 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:36.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:36.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:36.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:36.982 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:37.003 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:37.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:37.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:37.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:37.133 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:37.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:37.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:37.237 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:37.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:37 smithi181 bash[59345]: cluster 2023-10-09T09:46:36.087894+0000 mgr.y (mgr.34107) 1217 : cluster 0 pgmap v2052: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T09:46:37.272 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:37.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:37.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:37.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:37.404 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:37.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:37.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:37.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:37 smithi163 bash[78107]: cluster 2023-10-09T09:46:36.087894+0000 mgr.y (mgr.34107) 1217 : cluster 0 pgmap v2052: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T09:46:37.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:37 smithi163 bash[75425]: cluster 2023-10-09T09:46:36.087894+0000 mgr.y (mgr.34107) 1217 : cluster 0 pgmap v2052: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T09:46:37.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:37.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 78186 78170 92.1698 53.625 0.022319 0.0108432 2023-10-09T09:46:37.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:37.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:37.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:37.646 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:37.674 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:37.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:37.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:37.779 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:37.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:37.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:37.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:37.918 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:37.955 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:37.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:38.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:38.080 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:38.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:38.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:38.145 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:46:38.145 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:46:38.182 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:38.215 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:38.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:38.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:38.324 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:38.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:38.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:38.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:38.484 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:38.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:38.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:38.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 79081 79065 91.4988 55.9375 0.0135057 0.010923 2023-10-09T09:46:38.589 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:38.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:38.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:38.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:38.725 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:38.755 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:38.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:38.862 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:38.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:38.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:38.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:39.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:39.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:39.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:39.154 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:39.159 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:46:39.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:39 smithi163 bash[78107]: cluster 2023-10-09T09:46:38.088566+0000 mgr.y (mgr.34107) 1218 : cluster 0 pgmap v2053: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T09:46:39.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:39 smithi163 bash[78107]: audit 2023-10-09T09:46:38.285382+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.163:0/1748148405' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:46:39.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:39 smithi163 bash[78107]: audit 2023-10-09T09:46:38.287180+0000 mon.a (mon.0) 2272 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:46:39.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:39 smithi163 bash[78107]: audit 2023-10-09T09:46:38.552784+0000 mon.a (mon.0) 2273 : audit 1 from='client.? 172.21.15.163:0/2022217565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:46:39.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:39 smithi163 bash[78107]: audit 2023-10-09T09:46:38.665210+0000 mon.a (mon.0) 2274 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:39.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:39 smithi163 bash[75425]: cluster 2023-10-09T09:46:38.088566+0000 mgr.y (mgr.34107) 1218 : cluster 0 pgmap v2053: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T09:46:39.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:39 smithi163 bash[75425]: audit 2023-10-09T09:46:38.285382+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.163:0/1748148405' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:46:39.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:39 smithi163 bash[75425]: audit 2023-10-09T09:46:38.287180+0000 mon.a (mon.0) 2272 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:46:39.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:39 smithi163 bash[75425]: audit 2023-10-09T09:46:38.552784+0000 mon.a (mon.0) 2273 : audit 1 from='client.? 172.21.15.163:0/2022217565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:46:39.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:39 smithi163 bash[75425]: audit 2023-10-09T09:46:38.665210+0000 mon.a (mon.0) 2274 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:39.161 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T09:46:39.181 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:39.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:39.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:39.285 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:39.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:39 smithi181 bash[59345]: cluster 2023-10-09T09:46:38.088566+0000 mgr.y (mgr.34107) 1218 : cluster 0 pgmap v2053: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T09:46:39.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:39 smithi181 bash[59345]: audit 2023-10-09T09:46:38.285382+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.163:0/1748148405' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:46:39.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:39 smithi181 bash[59345]: audit 2023-10-09T09:46:38.287180+0000 mon.a (mon.0) 2272 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:46:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:39 smithi181 bash[59345]: audit 2023-10-09T09:46:38.552784+0000 mon.a (mon.0) 2273 : audit 1 from='client.? 172.21.15.163:0/2022217565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:46:39.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:39 smithi181 bash[59345]: audit 2023-10-09T09:46:38.665210+0000 mon.a (mon.0) 2274 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:39.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:39.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:39.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:39.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:39.454 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:39.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:39.538 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 79985 79969 90.8624 56.5 0.0223071 0.011 2023-10-09T09:46:39.559 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:39.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:39.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:39.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:39.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:46:39.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T09:46:39.704 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:39.732 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:39.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:39.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:39.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:39.867 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:39.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:39.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:39.970 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:40.012 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:40.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:40.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:40.116 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:40.145 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:40.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:40.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:40.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:40.261 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[78107]: cluster 2023-10-09T09:46:39.141803+0000 mon.a (mon.0) 2275 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:46:40.261 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[78107]: cluster 2023-10-09T09:46:39.146325+0000 osd.5 (osd.5) 826 : cluster 3 failed to encode map e1239 with expected crc 2023-10-09T09:46:40.262 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[78107]: audit 2023-10-09T09:46:39.149328+0000 mon.a (mon.0) 2276 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:46:40.262 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[78107]: audit 2023-10-09T09:46:39.149832+0000 mon.a (mon.0) 2277 : audit 1 from='client.? 172.21.15.163:0/2022217565' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:46:40.262 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[78107]: cluster 2023-10-09T09:46:39.149939+0000 mon.a (mon.0) 2278 : cluster 0 osdmap e1239: 8 total, 8 up, 6 in 2023-10-09T09:46:40.262 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[78107]: audit 2023-10-09T09:46:39.170282+0000 mon.a (mon.0) 2279 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.262 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[78107]: audit 2023-10-09T09:46:39.183116+0000 mon.a (mon.0) 2280 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.262 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[78107]: audit 2023-10-09T09:46:39.197619+0000 mon.a (mon.0) 2281 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.262 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[78107]: audit 2023-10-09T09:46:39.209557+0000 mon.a (mon.0) 2282 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.262 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[75425]: cluster 2023-10-09T09:46:39.141803+0000 mon.a (mon.0) 2275 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:46:40.262 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[75425]: cluster 2023-10-09T09:46:39.146325+0000 osd.5 (osd.5) 826 : cluster 3 failed to encode map e1239 with expected crc 2023-10-09T09:46:40.262 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[75425]: audit 2023-10-09T09:46:39.149328+0000 mon.a (mon.0) 2276 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:46:40.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[75425]: audit 2023-10-09T09:46:39.149832+0000 mon.a (mon.0) 2277 : audit 1 from='client.? 172.21.15.163:0/2022217565' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:46:40.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[75425]: cluster 2023-10-09T09:46:39.149939+0000 mon.a (mon.0) 2278 : cluster 0 osdmap e1239: 8 total, 8 up, 6 in 2023-10-09T09:46:40.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[75425]: audit 2023-10-09T09:46:39.170282+0000 mon.a (mon.0) 2279 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[75425]: audit 2023-10-09T09:46:39.183116+0000 mon.a (mon.0) 2280 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[75425]: audit 2023-10-09T09:46:39.197619+0000 mon.a (mon.0) 2281 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:40 smithi163 bash[75425]: audit 2023-10-09T09:46:39.209557+0000 mon.a (mon.0) 2282 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.274 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:40.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:40.379 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:40.408 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:40.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:40.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:40.539 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 80912 80896 90.2744 57.9375 0.0174372 0.0110716 2023-10-09T09:46:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:40 smithi181 bash[59345]: cluster 2023-10-09T09:46:39.141803+0000 mon.a (mon.0) 2275 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:46:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:40 smithi181 bash[59345]: cluster 2023-10-09T09:46:39.146325+0000 osd.5 (osd.5) 826 : cluster 3 failed to encode map e1239 with expected crc 2023-10-09T09:46:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:40 smithi181 bash[59345]: audit 2023-10-09T09:46:39.149328+0000 mon.a (mon.0) 2276 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:46:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:40 smithi181 bash[59345]: audit 2023-10-09T09:46:39.149832+0000 mon.a (mon.0) 2277 : audit 1 from='client.? 172.21.15.163:0/2022217565' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:46:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:40 smithi181 bash[59345]: cluster 2023-10-09T09:46:39.149939+0000 mon.a (mon.0) 2278 : cluster 0 osdmap e1239: 8 total, 8 up, 6 in 2023-10-09T09:46:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:40 smithi181 bash[59345]: audit 2023-10-09T09:46:39.170282+0000 mon.a (mon.0) 2279 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:40 smithi181 bash[59345]: audit 2023-10-09T09:46:39.183116+0000 mon.a (mon.0) 2280 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:40 smithi181 bash[59345]: audit 2023-10-09T09:46:39.197619+0000 mon.a (mon.0) 2281 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:40 smithi181 bash[59345]: audit 2023-10-09T09:46:39.209557+0000 mon.a (mon.0) 2282 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:40.593 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:40.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:40.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:40.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:40.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:40.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:40.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:40.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:40.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:40.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:40.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:40.989 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:41.016 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:41.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:41.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:41.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:41.148 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:41.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:41.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:41.204 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T09:46:41.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:46:41.254 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:41.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:41.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:41.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:41.427 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:41.435 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:41 smithi181 bash[59345]: cluster 2023-10-09T09:46:40.089010+0000 mgr.y (mgr.34107) 1219 : cluster 0 pgmap v2055: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T09:46:41.435 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:41 smithi181 bash[59345]: audit 2023-10-09T09:46:40.392987+0000 mon.a (mon.0) 2283 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:41.456 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:41.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:41.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:41 smithi163 bash[75425]: cluster 2023-10-09T09:46:40.089010+0000 mgr.y (mgr.34107) 1219 : cluster 0 pgmap v2055: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T09:46:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:41 smithi163 bash[75425]: audit 2023-10-09T09:46:40.392987+0000 mon.a (mon.0) 2283 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:41.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:41 smithi163 bash[78107]: cluster 2023-10-09T09:46:40.089010+0000 mgr.y (mgr.34107) 1219 : cluster 0 pgmap v2055: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T09:46:41.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:41 smithi163 bash[78107]: audit 2023-10-09T09:46:40.392987+0000 mon.a (mon.0) 2283 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:41.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 81894 81878 89.7649 61.375 0.0117947 0.0111342 2023-10-09T09:46:41.560 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:41.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:41.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:41.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:41.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:41.728 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:41.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:41.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:41.832 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:41.869 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:41.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:41.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:41.975 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:42.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:42.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:42.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:42.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:42.133 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:42.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:42.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:42.236 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:42.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:42.270 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:42.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:42.371 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:42.402 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:42.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:42.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:42.505 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:42.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 82875 82859 89.2743 61.3125 0.0246112 0.0111965 2023-10-09T09:46:42.541 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:42.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:42.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:42.646 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:42.673 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:42.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:42.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:42.779 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:42.808 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:42.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:42.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:42.915 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:42.940 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:42.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:42.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:43.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:43.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:43.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:43.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:43.186 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:43.191 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:46:43.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:43.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:43.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:43.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:43.354 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:43.358 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:43 smithi163 bash[78107]: cluster 2023-10-09T09:46:42.089563+0000 mgr.y (mgr.34107) 1220 : cluster 0 pgmap v2056: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 59 MiB/s wr, 942 op/s 2023-10-09T09:46:43.359 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:46:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:46:43] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-09T09:46:43.359 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:43 smithi163 bash[75425]: cluster 2023-10-09T09:46:42.089563+0000 mgr.y (mgr.34107) 1220 : cluster 0 pgmap v2056: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 59 MiB/s wr, 942 op/s 2023-10-09T09:46:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:43.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:43.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:43.469 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:43 smithi181 bash[59345]: cluster 2023-10-09T09:46:42.089563+0000 mgr.y (mgr.34107) 1220 : cluster 0 pgmap v2056: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 59 MiB/s wr, 942 op/s 2023-10-09T09:46:43.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:43.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:43.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:43.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 83845 83829 88.7886 60.625 0.0280443 0.011257 2023-10-09T09:46:43.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:43.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:43.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:43.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:43.745 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:43.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:43.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:43.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:43.884 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:43.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:43.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:43.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:44.014 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:44.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:44.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:44.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:44.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:44.186 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:44.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:44.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:44.202 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:46:44.206 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:46:44.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:44.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:44.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:44.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:44.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:44.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:44.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:44.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:44.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:46:44.540874+0000 min lat: 0.00129664 max lat: 0.299706 avg lat: 0.0113225 2023-10-09T09:46:44.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:46:44.541 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 84775 84759 88.2775 58.125 0.0176253 0.0113225 2023-10-09T09:46:44.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:44.606 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:44.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:44.712 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:44.759 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:44.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:44.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:44.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:44.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:44.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:44.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:45.002 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:45.023 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:45.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:45.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:45.127 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:45.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:45.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:45.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:45.263 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:46:45.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:45.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:45.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:45.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:45.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:45.415 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:45 smithi181 bash[59345]: cluster 2023-10-09T09:46:44.090183+0000 mgr.y (mgr.34107) 1221 : cluster 0 pgmap v2057: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 59 MiB/s wr, 939 op/s 2023-10-09T09:46:45.441 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:45.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:45.540 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 85411 85395 87.4819 39.75 0.0119991 0.0114251 2023-10-09T09:46:45.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:45.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:45.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:45.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:45.705 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:45 smithi163 bash[78107]: cluster 2023-10-09T09:46:44.090183+0000 mgr.y (mgr.34107) 1221 : cluster 0 pgmap v2057: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 59 MiB/s wr, 939 op/s 2023-10-09T09:46:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:45 smithi163 bash[75425]: cluster 2023-10-09T09:46:44.090183+0000 mgr.y (mgr.34107) 1221 : cluster 0 pgmap v2057: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 59 MiB/s wr, 939 op/s 2023-10-09T09:46:45.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:45.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:45.862 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:45.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:45.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:45.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:46.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:46.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:46.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:46.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:46.145 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:46.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:46.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:46.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:46.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:46.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:46.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:46.417 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:46.450 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:46.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:46.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:46.542 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 86278 86262 86.9423 54.1875 0.0112083 0.0114951 2023-10-09T09:46:46.553 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:46.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:46.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:46.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:46.684 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:46.712 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:46.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:46.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:46.815 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:46.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:46.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:46.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:46.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:46.996 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:47.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:47.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:47.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:46 smithi181 bash[59345]: audit 2023-10-09T09:46:45.818348+0000 mon.a (mon.0) 2284 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:46 smithi181 bash[59345]: audit 2023-10-09T09:46:45.833683+0000 mon.a (mon.0) 2285 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:46 smithi181 bash[59345]: audit 2023-10-09T09:46:46.160262+0000 mon.a (mon.0) 2286 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:46 smithi181 bash[59345]: audit 2023-10-09T09:46:46.167246+0000 mon.a (mon.0) 2287 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.102 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:47.132 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:47.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:47.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:47.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:46 smithi163 bash[75425]: audit 2023-10-09T09:46:45.818348+0000 mon.a (mon.0) 2284 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:46 smithi163 bash[75425]: audit 2023-10-09T09:46:45.833683+0000 mon.a (mon.0) 2285 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:46 smithi163 bash[75425]: audit 2023-10-09T09:46:46.160262+0000 mon.a (mon.0) 2286 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:46 smithi163 bash[75425]: audit 2023-10-09T09:46:46.167246+0000 mon.a (mon.0) 2287 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:46 smithi163 bash[78107]: audit 2023-10-09T09:46:45.818348+0000 mon.a (mon.0) 2284 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:46 smithi163 bash[78107]: audit 2023-10-09T09:46:45.833683+0000 mon.a (mon.0) 2285 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:46 smithi163 bash[78107]: audit 2023-10-09T09:46:46.160262+0000 mon.a (mon.0) 2286 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:46 smithi163 bash[78107]: audit 2023-10-09T09:46:46.167246+0000 mon.a (mon.0) 2287 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:46:47.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:47.255 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:46:47.287 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:47.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:47.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:47.390 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:47.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:47.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:47.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:47.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:47.545 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 87065 87049 86.3392 49.1875 0.0209171 0.0115764 2023-10-09T09:46:47.555 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:47.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:47.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:47.660 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:47.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:47.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:47.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:47.804 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:47.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:47.849 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:47 smithi163 bash[75425]: cluster 2023-10-09T09:46:46.090719+0000 mgr.y (mgr.34107) 1222 : cluster 0 pgmap v2058: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T09:46:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:47.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:47.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:47.962 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:47 smithi181 bash[59345]: cluster 2023-10-09T09:46:46.090719+0000 mgr.y (mgr.34107) 1222 : cluster 0 pgmap v2058: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T09:46:47.997 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:48.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:48.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:48.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:48.129 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:48.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:48.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:47 smithi163 bash[78107]: cluster 2023-10-09T09:46:46.090719+0000 mgr.y (mgr.34107) 1222 : cluster 0 pgmap v2058: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T09:46:48.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:48.269 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:48.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:48.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:48.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:48.408 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:48.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:48.512 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:48.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:48.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:48.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:48.545 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 88025 88009 85.9276 60 0.0238843 0.0116315 2023-10-09T09:46:48.645 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:48.674 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:48.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:48.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:48.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:48.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:48.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:48.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:48.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:48.950 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:48.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:48.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:49.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:49.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:49.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:49.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:49.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:49.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:49.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:49.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:49.322 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:49.354 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:49.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:49.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:49.458 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:49.492 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:49.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:49.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:49.545 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 88969 88953 85.5133 59 0.0105043 0.0116881 2023-10-09T09:46:49.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:49.620 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:49.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:49.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:49.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:49.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:49.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:49.866 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:49.894 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:49.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:49.997 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:50.006 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:49 smithi163 bash[78107]: cluster 2023-10-09T09:46:48.091390+0000 mgr.y (mgr.34107) 1223 : cluster 0 pgmap v2059: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s 2023-10-09T09:46:50.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:49 smithi163 bash[75425]: cluster 2023-10-09T09:46:48.091390+0000 mgr.y (mgr.34107) 1223 : cluster 0 pgmap v2059: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s 2023-10-09T09:46:50.036 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:50.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:50.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:50.172 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:50.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:50.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:50.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:50.284 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:49 smithi181 bash[59345]: cluster 2023-10-09T09:46:48.091390+0000 mgr.y (mgr.34107) 1223 : cluster 0 pgmap v2059: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s 2023-10-09T09:46:50.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:50.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:50.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:50.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:50.435 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:50.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:50.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:50.539 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:50.545 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 89860 89844 85.0614 55.6875 0.0279312 0.0117511 2023-10-09T09:46:50.569 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:50.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:50.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:50.712 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:50.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:50.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:50.815 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:50.846 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:50.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:50.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:50.950 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:50.977 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:50.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:51.081 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:51.113 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:51.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:51.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:51.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:51.243 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:51.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:51.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:51.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:51.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:51.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:51.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:51.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:51.514 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:51.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:51.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:51.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 90695 90679 84.5707 52.1875 0.0128375 0.0118193 2023-10-09T09:46:51.619 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:51.655 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:51.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:51.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:51.760 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:51.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:51.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:51.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:51.897 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:51.906 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:51 smithi181 bash[59345]: cluster 2023-10-09T09:46:50.091914+0000 mgr.y (mgr.34107) 1224 : cluster 0 pgmap v2060: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 820 op/s 2023-10-09T09:46:51.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:51.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:51.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:52.035 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:52.063 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:52.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:52.166 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:52.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:51 smithi163 bash[75425]: cluster 2023-10-09T09:46:50.091914+0000 mgr.y (mgr.34107) 1224 : cluster 0 pgmap v2060: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 820 op/s 2023-10-09T09:46:52.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:51 smithi163 bash[78107]: cluster 2023-10-09T09:46:50.091914+0000 mgr.y (mgr.34107) 1224 : cluster 0 pgmap v2060: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 820 op/s 2023-10-09T09:46:52.197 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:52.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:52.300 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:52.325 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:52.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:52.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:52.432 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:52.463 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:52.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:52.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:52.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 91624 91608 84.1809 58.0625 0.0175939 0.0118737 2023-10-09T09:46:52.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:52.595 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:52.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:52.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:52.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:52.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:52.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:52.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:52.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:52.850 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:52.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:52.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:52.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:52.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:52.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:53.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:53.122 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:53.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:53.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:53.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:53.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:53.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:53.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:53.359 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:46:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:46:53] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-09T09:46:53.365 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:53.394 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:53.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:53.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:53.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:53.528 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:53.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:53.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:53.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 92382 92366 83.6475 47.375 0.017108 0.0119497 2023-10-09T09:46:53.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:53.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:53.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:53.771 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:53.793 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:53.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:53.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:53.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:53.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:53.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:53.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:54.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:54.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:53 smithi163 bash[75425]: cluster 2023-10-09T09:46:52.092454+0000 mgr.y (mgr.34107) 1225 : cluster 0 pgmap v2061: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:46:54.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:53 smithi163 bash[78107]: cluster 2023-10-09T09:46:52.092454+0000 mgr.y (mgr.34107) 1225 : cluster 0 pgmap v2061: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:46:54.080 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:54.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:54.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:54.183 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:54.191 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:53 smithi181 bash[59345]: cluster 2023-10-09T09:46:52.092454+0000 mgr.y (mgr.34107) 1225 : cluster 0 pgmap v2061: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:46:54.208 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:54.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:54.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:54.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:54.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:54.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:54.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:54.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:54.484 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:54.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:54.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:54.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 93213 93197 83.1944 51.9375 0.0159239 0.0120147 2023-10-09T09:46:54.592 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:54.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:54.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:54.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:54.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:54.756 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:54.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:54.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:54.861 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:54.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:54.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:54.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:55.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:55.019 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:55.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:55.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:55.122 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:55.152 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:55.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:55.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:55.257 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:55.286 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:55.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:55.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:55.390 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:55.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:55.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:55.536 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:55.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 94118 94102 82.8193 56.5625 0.0177333 0.0120686 2023-10-09T09:46:55.559 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:55.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:55.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:55.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:55.691 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:55.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:55.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:55.797 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:55.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:55.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:55.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:55.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:55.963 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:55.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:55 smithi181 bash[59345]: cluster 2023-10-09T09:46:54.093133+0000 mgr.y (mgr.34107) 1226 : cluster 0 pgmap v2062: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 52 MiB/s wr, 831 op/s 2023-10-09T09:46:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:55 smithi181 bash[59345]: audit 2023-10-09T09:46:55.393743+0000 mon.a (mon.0) 2288 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:56.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:56.099 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:56.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:56.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:56.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:56.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:55 smithi163 bash[78107]: cluster 2023-10-09T09:46:54.093133+0000 mgr.y (mgr.34107) 1226 : cluster 0 pgmap v2062: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 52 MiB/s wr, 831 op/s 2023-10-09T09:46:56.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:55 smithi163 bash[78107]: audit 2023-10-09T09:46:55.393743+0000 mon.a (mon.0) 2288 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:56.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:55 smithi163 bash[75425]: cluster 2023-10-09T09:46:54.093133+0000 mgr.y (mgr.34107) 1226 : cluster 0 pgmap v2062: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 52 MiB/s wr, 831 op/s 2023-10-09T09:46:56.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:55 smithi163 bash[75425]: audit 2023-10-09T09:46:55.393743+0000 mon.a (mon.0) 2288 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:46:56.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:56.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:56.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:56.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:56.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:56.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:56.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:56.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:56.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:56.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:56.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 95039 95023 82.4685 57.5625 0.0115351 0.0121201 2023-10-09T09:46:56.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:56.634 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:56.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:56.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:56.773 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:56.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:56.876 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:56.913 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:56.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:56.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:57.018 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:57.036 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:57.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:57.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:57.139 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:57.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:57.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:57.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:57.296 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:57.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:57.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:57.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:57.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:46:57.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:57.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:46:57.541 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:57.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 95858 95842 82.0399 51.1875 0.0101888 0.0121837 2023-10-09T09:46:57.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:57.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:57.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:57.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:57.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:57.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:57.798 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:57.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:57.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:57.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:57.934 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:57.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:57.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:57.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:58.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:57 smithi181 bash[59345]: cluster 2023-10-09T09:46:56.093571+0000 mgr.y (mgr.34107) 1227 : cluster 0 pgmap v2063: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T09:46:58.074 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:58.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:58.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:58.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:58.117 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:46:58.117 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:46:58.217 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:57 smithi163 bash[78107]: cluster 2023-10-09T09:46:56.093571+0000 mgr.y (mgr.34107) 1227 : cluster 0 pgmap v2063: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T09:46:58.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:57 smithi163 bash[75425]: cluster 2023-10-09T09:46:56.093571+0000 mgr.y (mgr.34107) 1227 : cluster 0 pgmap v2063: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T09:46:58.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:58.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:58.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:58.377 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:58.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:58.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:46:58.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:58.510 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:58.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:58.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:58.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 96744 96728 81.6796 55.375 0.0214549 0.012237 2023-10-09T09:46:58.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:58.653 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:58.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:58.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:58.757 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:58.785 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:58.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:58.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:58.890 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:58.921 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:58.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:58.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:59.025 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:59.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:59.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:59.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:59.155 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:59.183 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:46:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:59.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:46:59.291 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:59.315 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:46:59.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:59.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:46:59.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:59.451 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:46:59.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:59.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:46:59.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 97536 97520 81.2505 49.5 0.00839808 0.0123021 2023-10-09T09:46:59.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:59.583 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:46:59.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:59.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:46:59.690 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:59.725 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:46:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:59.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:46:59.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:59.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:46:59.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:59.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:46:59.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:59.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:46:59.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:46:59.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:46:59 smithi181 bash[59345]: cluster 2023-10-09T09:46:58.094246+0000 mgr.y (mgr.34107) 1228 : cluster 0 pgmap v2064: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T09:47:00.099 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:00.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:46:59 smithi163 bash[78107]: cluster 2023-10-09T09:46:58.094246+0000 mgr.y (mgr.34107) 1228 : cluster 0 pgmap v2064: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T09:47:00.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:46:59 smithi163 bash[75425]: cluster 2023-10-09T09:46:58.094246+0000 mgr.y (mgr.34107) 1228 : cluster 0 pgmap v2064: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T09:47:00.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:00.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:00.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:00.242 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:00.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:00.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:00.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:00.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:00.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:00.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:00.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:00.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T09:47:00.511 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:00.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:00.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:00.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:00.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 98438 98422 80.9231 56.375 0.0114973 0.0123523 2023-10-09T09:47:00.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:00.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:00.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:00.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:00.813 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:00.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:00.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:00.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:00.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:00.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:01.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:01.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:01.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:01.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:01.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:01.216 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:01.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:01.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:01.340 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:01.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:01.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:01.443 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:01.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:01.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:01.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:01.546 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 99168 99152 80.4647 45.625 0.0119607 0.0124221 2023-10-09T09:47:01.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:01.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:01.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:01.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:01.715 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:01.747 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:01.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:01.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:01.850 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:01.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:01.884 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:01 smithi163 bash[75425]: cluster 2023-10-09T09:47:00.094798+0000 mgr.y (mgr.34107) 1229 : cluster 0 pgmap v2065: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 48 MiB/s wr, 769 op/s 2023-10-09T09:47:01.884 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:01 smithi163 bash[75425]: audit 2023-10-09T09:47:01.345848+0000 mon.a (mon.0) 2289 : audit 1 from='client.? 172.21.15.163:0/1079472392' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:47:01.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:01.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:01.890 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:47:01.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:02.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:02.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:02.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:01 smithi181 bash[59345]: cluster 2023-10-09T09:47:00.094798+0000 mgr.y (mgr.34107) 1229 : cluster 0 pgmap v2065: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 48 MiB/s wr, 769 op/s 2023-10-09T09:47:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:01 smithi181 bash[59345]: audit 2023-10-09T09:47:01.345848+0000 mon.a (mon.0) 2289 : audit 1 from='client.? 172.21.15.163:0/1079472392' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:47:02.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:02.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:01 smithi163 bash[78107]: cluster 2023-10-09T09:47:00.094798+0000 mgr.y (mgr.34107) 1229 : cluster 0 pgmap v2065: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 48 MiB/s wr, 769 op/s 2023-10-09T09:47:02.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:01 smithi163 bash[78107]: audit 2023-10-09T09:47:01.345848+0000 mon.a (mon.0) 2289 : audit 1 from='client.? 172.21.15.163:0/1079472392' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:47:02.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:02.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:02.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:02.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:02.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:02.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:02.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:02.440 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:02.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:02.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:02.478 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:47:02.519 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:47:02.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:02.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 99970 99954 80.0757 50.125 0.0160343 0.012483 2023-10-09T09:47:02.559 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:47:02.579 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:02.585 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:47:02.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:02.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:02.591 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:47:02.611 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:47:02.654 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:47:02.687 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:02.697 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:47:02.725 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:02.726 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:47:02.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:02.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:02.766 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:47:02.773 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:47:02.773 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:47:02.773 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:47:02.773 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:47:02.774 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:47:02.774 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:47:02.774 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:47:02.774 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:47:02.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:02.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:02.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:02.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:02.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[78107]: cluster 2023-10-09T09:47:01.872215+0000 mon.a (mon.0) 2290 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:47:02.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[78107]: cluster 2023-10-09T09:47:01.872262+0000 mon.a (mon.0) 2291 : cluster 1 Cluster is now healthy 2023-10-09T09:47:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[78107]: cluster 2023-10-09T09:47:01.875483+0000 osd.5 (osd.5) 827 : cluster 3 failed to encode map e1240 with expected crc 2023-10-09T09:47:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[78107]: cluster 2023-10-09T09:47:01.879515+0000 osd.6 (osd.6) 525 : cluster 3 failed to encode map e1240 with expected crc 2023-10-09T09:47:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[78107]: audit 2023-10-09T09:47:01.880225+0000 mon.a (mon.0) 2292 : audit 1 from='client.? 172.21.15.163:0/1079472392' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:47:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[78107]: cluster 2023-10-09T09:47:01.880380+0000 mon.a (mon.0) 2293 : cluster 0 osdmap e1240: 8 total, 8 up, 6 in 2023-10-09T09:47:02.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[78107]: audit 2023-10-09T09:47:02.421169+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.163:0/2049984248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:47:02.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[75425]: cluster 2023-10-09T09:47:01.872215+0000 mon.a (mon.0) 2290 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:47:02.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[75425]: cluster 2023-10-09T09:47:01.872262+0000 mon.a (mon.0) 2291 : cluster 1 Cluster is now healthy 2023-10-09T09:47:02.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[75425]: cluster 2023-10-09T09:47:01.875483+0000 osd.5 (osd.5) 827 : cluster 3 failed to encode map e1240 with expected crc 2023-10-09T09:47:02.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[75425]: cluster 2023-10-09T09:47:01.879515+0000 osd.6 (osd.6) 525 : cluster 3 failed to encode map e1240 with expected crc 2023-10-09T09:47:02.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[75425]: audit 2023-10-09T09:47:01.880225+0000 mon.a (mon.0) 2292 : audit 1 from='client.? 172.21.15.163:0/1079472392' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:47:02.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[75425]: cluster 2023-10-09T09:47:01.880380+0000 mon.a (mon.0) 2293 : cluster 0 osdmap e1240: 8 total, 8 up, 6 in 2023-10-09T09:47:02.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:02 smithi163 bash[75425]: audit 2023-10-09T09:47:02.421169+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.163:0/2049984248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:47:02.990 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:02.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:02.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:02 smithi181 bash[59345]: cluster 2023-10-09T09:47:01.872215+0000 mon.a (mon.0) 2290 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:47:03.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:02 smithi181 bash[59345]: cluster 2023-10-09T09:47:01.872262+0000 mon.a (mon.0) 2291 : cluster 1 Cluster is now healthy 2023-10-09T09:47:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:02 smithi181 bash[59345]: cluster 2023-10-09T09:47:01.875483+0000 osd.5 (osd.5) 827 : cluster 3 failed to encode map e1240 with expected crc 2023-10-09T09:47:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:02 smithi181 bash[59345]: cluster 2023-10-09T09:47:01.879515+0000 osd.6 (osd.6) 525 : cluster 3 failed to encode map e1240 with expected crc 2023-10-09T09:47:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:02 smithi181 bash[59345]: audit 2023-10-09T09:47:01.880225+0000 mon.a (mon.0) 2292 : audit 1 from='client.? 172.21.15.163:0/1079472392' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:47:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:02 smithi181 bash[59345]: cluster 2023-10-09T09:47:01.880380+0000 mon.a (mon.0) 2293 : cluster 0 osdmap e1240: 8 total, 8 up, 6 in 2023-10-09T09:47:03.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:02 smithi181 bash[59345]: audit 2023-10-09T09:47:02.421169+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.163:0/2049984248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:47:03.093 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:03.125 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:03.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:03.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:03.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:03.261 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:03.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:03.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:03.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:03.393 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:03.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:03.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:03.523 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:03.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:03.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:03.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 100661 100645 79.6088 43.1875 0.0351184 0.0125554 2023-10-09T09:47:03.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:03.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:03.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:03.721 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:47:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:47:03] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.33.4" 2023-10-09T09:47:03.761 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:03.788 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:03.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:03.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:03.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:03.917 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:03.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:04.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:04.039 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:04.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:04.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:04.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:04 smithi163 bash[78107]: cluster 2023-10-09T09:47:02.095297+0000 mgr.y (mgr.34107) 1230 : cluster 0 pgmap v2067: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 54 MiB/s wr, 856 op/s 2023-10-09T09:47:04.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:04 smithi163 bash[78107]: audit 2023-10-09T09:47:02.560795+0000 mgr.y (mgr.34107) 1231 : audit 0 from='client.35640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:47:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:03 smithi163 bash[75425]: cluster 2023-10-09T09:47:02.095297+0000 mgr.y (mgr.34107) 1230 : cluster 0 pgmap v2067: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 54 MiB/s wr, 856 op/s 2023-10-09T09:47:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:03 smithi163 bash[75425]: audit 2023-10-09T09:47:02.560795+0000 mgr.y (mgr.34107) 1231 : audit 0 from='client.35640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:47:04.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:04.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:03 smithi181 bash[59345]: cluster 2023-10-09T09:47:02.095297+0000 mgr.y (mgr.34107) 1230 : cluster 0 pgmap v2067: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 54 MiB/s wr, 856 op/s 2023-10-09T09:47:04.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:03 smithi181 bash[59345]: audit 2023-10-09T09:47:02.560795+0000 mgr.y (mgr.34107) 1231 : audit 0 from='client.35640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:47:04.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:04.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:04.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:04.314 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:04.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:04.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:04.420 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:04.446 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:04.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:04.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:04.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:47:04.547340+0000 min lat: 0.00129664 max lat: 0.299706 avg lat: 0.0126301 2023-10-09T09:47:04.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:47:04.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 101332 101316 79.1379 41.9375 0.010453 0.0126301 2023-10-09T09:47:04.549 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:04.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:04.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:04.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:04.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:04.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:04.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:04.810 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:04.837 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:04.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:04.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:04.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:04.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:04.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:05.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:05.102 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:05.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:05.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:05.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:05.237 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:05.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:05.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:05.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:05.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:05.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:05.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:05.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:05.512 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:05.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:05.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:05.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 102133 102117 78.7789 50.0625 0.021966 0.0126886 2023-10-09T09:47:05.617 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:05.646 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:05.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:05.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:05.753 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:05.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:05.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:05.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:05.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:05.917 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:05.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:05.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:06.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:06.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:06.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:06.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:06.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:05 smithi181 bash[59345]: cluster 2023-10-09T09:47:04.095878+0000 mgr.y (mgr.34107) 1232 : cluster 0 pgmap v2068: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T09:47:06.152 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:06.189 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:06.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:06.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:06.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:05 smithi163 bash[78107]: cluster 2023-10-09T09:47:04.095878+0000 mgr.y (mgr.34107) 1232 : cluster 0 pgmap v2068: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T09:47:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:05 smithi163 bash[75425]: cluster 2023-10-09T09:47:04.095878+0000 mgr.y (mgr.34107) 1232 : cluster 0 pgmap v2068: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T09:47:06.292 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:06.325 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:06.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:06.429 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:06.458 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:06.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:06.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:06.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 102997 102981 78.4767 54 0.0104325 0.012737 2023-10-09T09:47:06.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:06.593 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:06.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:06.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:06.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:06.728 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:06.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:06.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:06.867 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:06.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:06.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:06.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:07.002 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:07.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:07.109 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:07.141 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:07.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:07.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:47:07.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:07.270 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:07.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:07.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:07.373 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:07.388 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2067,"stamp":"2023-10-09T09:47:02.095058+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6438066080,"num_objects":98252,"num_object_clones":0,"num_object_copies":294756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98252,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":98488,"num_write_kb":6289037,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":592828,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97922,"ondisk_log_size":97922,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":23472044,"kb_used_data":18801532,"kb_used_omap":510,"kb_used_meta":4669953,"kb_avail":538998868,"statfs":{"total":575970213888,"available":551934840832,"internally_reserved":0,"allocated":19252768768,"data_stored":19247667564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":522608,"internal_metadata":4782032528},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":561643520,"num_objects":8570,"num_object_clones":0,"num_object_copies":25710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8570,"num_write_kb":548480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002966"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":37,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000907+0000","last_change":"2023-10-09T09:43:23.993342+0000","last_active":"2023-10-09T09:46:27.000907+0000","last_peered":"2023-10-09T09:46:27.000907+0000","last_clean":"2023-10-09T09:46:27.000907+0000","last_became_active":"2023-10-09T09:43:23.990938+0000","last_became_peered":"2023-10-09T09:43:23.990938+0000","last_unstale":"2023-10-09T09:46:27.000907+0000","last_undegraded":"2023-10-09T09:46:27.000907+0000","last_fullsized":"2023-10-09T09:46:27.000907+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:23.141315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":122,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000367+0000","last_change":"2023-10-09T09:41:11.811997+0000","last_active":"2023-10-09T09:46:27.000367+0000","last_peered":"2023-10-09T09:46:27.000367+0000","last_clean":"2023-10-09T09:46:27.000367+0000","last_became_active":"2023-10-09T09:41:11.811300+0000","last_became_peered":"2023-10-09T09:41:11.811300+0000","last_unstale":"2023-10-09T09:46:27.000367+0000","last_undegraded":"2023-10-09T09:46:27.000367+0000","last_fullsized":"2023-10-09T09:46:27.000367+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.346300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1191,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000316+0000","last_change":"2023-10-09T09:41:11.811870+0000","last_active":"2023-10-09T09:46:27.000316+0000","last_peered":"2023-10-09T09:46:27.000316+0000","last_clean":"2023-10-09T09:46:27.000316+0000","last_became_active":"2023-10-09T09:41:11.811570+0000","last_became_peered":"2023-10-09T09:41:11.811570+0000","last_unstale":"2023-10-09T09:46:27.000316+0000","last_undegraded":"2023-10-09T09:46:27.000316+0000","last_fullsized":"2023-10-09T09:46:27.000316+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:57.923729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1181,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.996983+0000","last_change":"2023-10-09T09:41:11.800661+0000","last_active":"2023-10-09T09:46:26.996983+0000","last_peered":"2023-10-09T09:46:26.996983+0000","last_clean":"2023-10-09T09:46:26.996983+0000","last_became_active":"2023-10-09T09:41:11.800279+0000","last_became_peered":"2023-10-09T09:41:11.800279+0000","last_unstale":"2023-10-09T09:46:26.996983+0000","last_undegraded":"2023-10-09T09:46:26.996983+0000","last_fullsized":"2023-10-09T09:46:26.996983+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:14:19.131995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":715,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.996319+0000","last_change":"2023-10-09T09:41:11.801649+0000","last_active":"2023-10-09T09:46:26.996319+0000","last_peered":"2023-10-09T09:46:26.996319+0000","last_clean":"2023-10-09T09:46:26.996319+0000","last_became_active":"2023-10-09T09:41:11.798585+0000","last_became_peered":"2023-10-09T09:41:11.798585+0000","last_unstale":"2023-10-09T09:46:26.996319+0000","last_undegraded":"2023-10-09T09:46:26.996319+0000","last_fullsized":"2023-10-09T09:46:26.996319+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:11.450331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1188,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.001491+0000","last_change":"2023-10-09T09:40:25.799705+0000","last_active":"2023-10-09T09:46:27.001491+0000","last_peered":"2023-10-09T09:46:27.001491+0000","last_clean":"2023-10-09T09:46:27.001491+0000","last_became_active":"2023-10-09T09:40:25.799341+0000","last_became_peered":"2023-10-09T09:40:25.799341+0000","last_unstale":"2023-10-09T09:46:27.001491+0000","last_undegraded":"2023-10-09T09:46:27.001491+0000","last_fullsized":"2023-10-09T09:46:27.001491+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1162,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.995670+0000","last_change":"2023-10-09T09:43:48.902066+0000","last_active":"2023-10-09T09:46:26.995670+0000","last_peered":"2023-10-09T09:46:26.995670+0000","last_clean":"2023-10-09T09:46:26.995670+0000","last_became_active":"2023-10-09T09:43:48.901639+0000","last_became_peered":"2023-10-09T09:43:48.901639+0000","last_unstale":"2023-10-09T09:46:26.995670+0000","last_undegraded":"2023-10-09T09:46:26.995670+0000","last_fullsized":"2023-10-09T09:46:26.995670+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":633,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.001821+0000","last_change":"2023-10-09T09:41:11.814521+0000","last_active":"2023-10-09T09:46:27.001821+0000","last_peered":"2023-10-09T09:46:27.001821+0000","last_clean":"2023-10-09T09:46:27.001821+0000","last_became_active":"2023-10-09T09:41:11.814148+0000","last_became_peered":"2023-10-09T09:41:11.814148+0000","last_unstale":"2023-10-09T09:46:27.001821+0000","last_undegraded":"2023-10-09T09:46:27.001821+0000","last_fullsized":"2023-10-09T09:46:27.001821+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:45.497593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":90,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.995264+0000","last_change":"2023-10-09T09:41:11.833334+0000","last_active":"2023-10-09T09:46:26.995264+0000","last_peered":"2023-10-09T09:46:26.995264+0000","last_clean":"2023-10-09T09:46:26.995264+0000","last_became_active":"2023-10-09T09:41:11.832509+0000","last_became_peered":"2023-10-09T09:41:11.832509+0000","last_unstale":"2023-10-09T09:46:26.995264+0000","last_undegraded":"2023-10-09T09:46:26.995264+0000","last_fullsized":"2023-10-09T09:46:26.995264+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":654,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000156+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:46:27.000156+0000","last_peered":"2023-10-09T09:46:27.000156+0000","last_clean":"2023-10-09T09:46:27.000156+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:46:27.000156+0000","last_undegraded":"2023-10-09T09:46:27.000156+0000","last_fullsized":"2023-10-09T09:46:27.000156+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":47,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.002783+0000","last_change":"2023-10-09T09:43:48.906785+0000","last_active":"2023-10-09T09:46:27.002783+0000","last_peered":"2023-10-09T09:46:27.002783+0000","last_clean":"2023-10-09T09:46:27.002783+0000","last_became_active":"2023-10-09T09:43:48.906276+0000","last_became_peered":"2023-10-09T09:43:48.906276+0000","last_unstale":"2023-10-09T09:46:27.002783+0000","last_undegraded":"2023-10-09T09:46:27.002783+0000","last_fullsized":"2023-10-09T09:46:27.002783+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":71,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.996209+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:46:26.996209+0000","last_peered":"2023-10-09T09:46:26.996209+0000","last_clean":"2023-10-09T09:46:26.996209+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:46:26.996209+0000","last_undegraded":"2023-10-09T09:46:26.996209+0000","last_fullsized":"2023-10-09T09:46:26.996209+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":395,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000478+0000","last_change":"2023-10-09T09:43:23.995526+0000","last_active":"2023-10-09T09:46:27.000478+0000","last_peered":"2023-10-09T09:46:27.000478+0000","last_clean":"2023-10-09T09:46:27.000478+0000","last_became_active":"2023-10-09T09:43:23.994816+0000","last_became_peered":"2023-10-09T09:43:23.994816+0000","last_unstale":"2023-10-09T09:46:27.000478+0000","last_undegraded":"2023-10-09T09:46:27.000478+0000","last_fullsized":"2023-10-09T09:46:27.000478+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":597,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000728+0000","last_change":"2023-10-09T09:43:23.995814+0000","last_active":"2023-10-09T09:46:27.000728+0000","last_peered":"2023-10-09T09:46:27.000728+0000","last_clean":"2023-10-09T09:46:27.000728+0000","last_became_active":"2023-10-09T09:43:23.995492+0000","last_became_peered":"2023-10-09T09:43:23.995492+0000","last_unstale":"2023-10-09T09:46:27.000728+0000","last_undegraded":"2023-10-09T09:46:27.000728+0000","last_fullsized":"2023-10-09T09:46:27.000728+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1132,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.995588+0000","last_change":"2023-10-09T09:43:23.989356+0000","last_active":"2023-10-09T09:46:26.995588+0000","last_peered":"2023-10-09T09:46:26.995588+0000","last_clean":"2023-10-09T09:46:26.995588+0000","last_became_active":"2023-10-09T09:43:23.988986+0000","last_became_peered":"2023-10-09T09:43:23.988986+0000","last_unstale":"2023-10-09T09:46:26.995588+0000","last_undegraded":"2023-10-09T09:46:26.995588+0000","last_fullsized":"2023-10-09T09:46:26.995588+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.570448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":718,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.002348+0000","last_change":"2023-10-09T09:41:11.826293+0000","last_active":"2023-10-09T09:46:27.002348+0000","last_peered":"2023-10-09T09:46:27.002348+0000","last_clean":"2023-10-09T09:46:27.002348+0000","last_became_active":"2023-10-09T09:41:11.825808+0000","last_became_peered":"2023-10-09T09:41:11.825808+0000","last_unstale":"2023-10-09T09:46:27.002348+0000","last_undegraded":"2023-10-09T09:46:27.002348+0000","last_fullsized":"2023-10-09T09:46:27.002348+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:33:14.204593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1154,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.001032+0000","last_change":"2023-10-09T09:43:48.904973+0000","last_active":"2023-10-09T09:46:27.001032+0000","last_peered":"2023-10-09T09:46:27.001032+0000","last_clean":"2023-10-09T09:46:27.001032+0000","last_became_active":"2023-10-09T09:43:48.904557+0000","last_became_peered":"2023-10-09T09:43:48.904557+0000","last_unstale":"2023-10-09T09:46:27.001032+0000","last_undegraded":"2023-10-09T09:46:27.001032+0000","last_fullsized":"2023-10-09T09:46:27.001032+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:36.528234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":75,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.002867+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:46:27.002867+0000","last_peered":"2023-10-09T09:46:27.002867+0000","last_clean":"2023-10-09T09:46:27.002867+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:46:27.002867+0000","last_undegraded":"2023-10-09T09:46:27.002867+0000","last_fullsized":"2023-10-09T09:46:27.002867+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":39,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000038+0000","last_change":"2023-10-09T09:43:23.994784+0000","last_active":"2023-10-09T09:46:27.000038+0000","last_peered":"2023-10-09T09:46:27.000038+0000","last_clean":"2023-10-09T09:46:27.000038+0000","last_became_active":"2023-10-09T09:43:23.994196+0000","last_became_peered":"2023-10-09T09:43:23.994196+0000","last_unstale":"2023-10-09T09:46:27.000038+0000","last_undegraded":"2023-10-09T09:46:27.000038+0000","last_fullsized":"2023-10-09T09:46:27.000038+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1153,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.998056+0000","last_change":"2023-10-09T09:43:48.901303+0000","last_active":"2023-10-09T09:46:26.998056+0000","last_peered":"2023-10-09T09:46:26.998056+0000","last_clean":"2023-10-09T09:46:26.998056+0000","last_became_active":"2023-10-09T09:43:48.900779+0000","last_became_peered":"2023-10-09T09:43:48.900779+0000","last_unstale":"2023-10-09T09:46:26.998056+0000","last_undegraded":"2023-10-09T09:46:26.998056+0000","last_fullsized":"2023-10-09T09:46:26.998056+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:51.853531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1233,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000029+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:46:27.000029+0000","last_peered":"2023-10-09T09:46:27.000029+0000","last_clean":"2023-10-09T09:46:27.000029+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:46:27.000029+0000","last_undegraded":"2023-10-09T09:46:27.000029+0000","last_fullsized":"2023-10-09T09:46:27.000029+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1201,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000318+0000","last_change":"2023-10-09T09:41:11.805335+0000","last_active":"2023-10-09T09:46:27.000318+0000","last_peered":"2023-10-09T09:46:27.000318+0000","last_clean":"2023-10-09T09:46:27.000318+0000","last_became_active":"2023-10-09T09:41:11.804963+0000","last_became_peered":"2023-10-09T09:41:11.804963+0000","last_unstale":"2023-10-09T09:46:27.000318+0000","last_undegraded":"2023-10-09T09:46:27.000318+0000","last_fullsized":"2023-10-09T09:46:27.000318+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:09.265362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.6","version":"1239'5847","reported_seq":5863,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:57.819791+0000","last_change":"2023-10-09T09:45:20.779324+0000","last_active":"2023-10-09T09:46:57.819791+0000","last_peered":"2023-10-09T09:46:57.819791+0000","last_clean":"2023-10-09T09:46:57.819791+0000","last_became_active":"2023-10-09T09:45:20.778629+0000","last_became_peered":"2023-10-09T09:45:20.778629+0000","last_unstale":"2023-10-09T09:46:57.819791+0000","last_undegraded":"2023-10-09T09:46:57.819791+0000","last_fullsized":"2023-10-09T09:46:57.819791+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":5847,"log_dups_size":0,"ondisk_log_size":5847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:50.507235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383188992,"num_objects":5847,"num_object_clones":0,"num_object_copies":17541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5847,"num_write_kb":374208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":48,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.997021+0000","last_change":"2023-10-09T09:43:23.993830+0000","last_active":"2023-10-09T09:46:26.997021+0000","last_peered":"2023-10-09T09:46:26.997021+0000","last_clean":"2023-10-09T09:46:26.997021+0000","last_became_active":"2023-10-09T09:43:23.993431+0000","last_became_peered":"2023-10-09T09:43:23.993431+0000","last_unstale":"2023-10-09T09:46:26.997021+0000","last_undegraded":"2023-10-09T09:46:26.997021+0000","last_fullsized":"2023-10-09T09:46:26.997021+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:27.775152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.993978+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:46:26.993978+0000","last_peered":"2023-10-09T09:46:26.993978+0000","last_clean":"2023-10-09T09:46:26.993978+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:46:26.993978+0000","last_undegraded":"2023-10-09T09:46:26.993978+0000","last_fullsized":"2023-10-09T09:46:26.993978+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"1239'6178","reported_seq":6193,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.501630+0000","last_change":"2023-10-09T09:45:19.872001+0000","last_active":"2023-10-09T09:47:01.501630+0000","last_peered":"2023-10-09T09:47:01.501630+0000","last_clean":"2023-10-09T09:47:01.501630+0000","last_became_active":"2023-10-09T09:45:19.871533+0000","last_became_peered":"2023-10-09T09:45:19.871533+0000","last_unstale":"2023-10-09T09:47:01.501630+0000","last_undegraded":"2023-10-09T09:47:01.501630+0000","last_fullsized":"2023-10-09T09:47:01.501630+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6178,"log_dups_size":0,"ondisk_log_size":6178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404881408,"num_objects":6178,"num_object_clones":0,"num_object_copies":18534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6178,"num_write_kb":395392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":48,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000591+0000","last_change":"2023-10-09T09:43:23.992053+0000","last_active":"2023-10-09T09:46:27.000591+0000","last_peered":"2023-10-09T09:46:27.000591+0000","last_clean":"2023-10-09T09:46:27.000591+0000","last_became_active":"2023-10-09T09:43:23.991695+0000","last_became_peered":"2023-10-09T09:43:23.991695+0000","last_unstale":"2023-10-09T09:46:27.000591+0000","last_undegraded":"2023-10-09T09:46:27.000591+0000","last_fullsized":"2023-10-09T09:46:27.000591+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:18.704618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1192,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000391+0000","last_change":"2023-10-09T09:43:23.995477+0000","last_active":"2023-10-09T09:46:27.000391+0000","last_peered":"2023-10-09T09:46:27.000391+0000","last_clean":"2023-10-09T09:46:27.000391+0000","last_became_active":"2023-10-09T09:43:23.995178+0000","last_became_peered":"2023-10-09T09:43:23.995178+0000","last_unstale":"2023-10-09T09:46:27.000391+0000","last_undegraded":"2023-10-09T09:46:27.000391+0000","last_fullsized":"2023-10-09T09:46:27.000391+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:22:20.575488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"1239'5962","reported_seq":5978,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:57.815045+0000","last_change":"2023-10-09T09:45:20.778840+0000","last_active":"2023-10-09T09:46:57.815045+0000","last_peered":"2023-10-09T09:46:57.815045+0000","last_clean":"2023-10-09T09:46:57.815045+0000","last_became_active":"2023-10-09T09:45:20.778306+0000","last_became_peered":"2023-10-09T09:45:20.778306+0000","last_unstale":"2023-10-09T09:46:57.815045+0000","last_undegraded":"2023-10-09T09:46:57.815045+0000","last_fullsized":"2023-10-09T09:46:57.815045+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":5962,"log_dups_size":0,"ondisk_log_size":5962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:32.098494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390725632,"num_objects":5962,"num_object_clones":0,"num_object_copies":17886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5962,"num_write_kb":381568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1189,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.995041+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:46:26.995041+0000","last_peered":"2023-10-09T09:46:26.995041+0000","last_clean":"2023-10-09T09:46:26.995041+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:46:26.995041+0000","last_undegraded":"2023-10-09T09:46:26.995041+0000","last_fullsized":"2023-10-09T09:46:26.995041+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1180,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000843+0000","last_change":"2023-10-09T09:40:49.793789+0000","last_active":"2023-10-09T09:46:27.000843+0000","last_peered":"2023-10-09T09:46:27.000843+0000","last_clean":"2023-10-09T09:46:27.000843+0000","last_became_active":"2023-10-09T09:40:49.793374+0000","last_became_peered":"2023-10-09T09:40:49.793374+0000","last_unstale":"2023-10-09T09:46:27.000843+0000","last_undegraded":"2023-10-09T09:46:27.000843+0000","last_fullsized":"2023-10-09T09:46:27.000843+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"1239'6161","reported_seq":6177,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:00.230216+0000","last_change":"2023-10-09T09:45:19.877007+0000","last_active":"2023-10-09T09:47:00.230216+0000","last_peered":"2023-10-09T09:47:00.230216+0000","last_clean":"2023-10-09T09:47:00.230216+0000","last_became_active":"2023-10-09T09:45:19.876605+0000","last_became_peered":"2023-10-09T09:45:19.876605+0000","last_unstale":"2023-10-09T09:47:00.230216+0000","last_undegraded":"2023-10-09T09:47:00.230216+0000","last_fullsized":"2023-10-09T09:47:00.230216+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:49.148260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403767296,"num_objects":6161,"num_object_clones":0,"num_object_copies":18483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6161,"num_write_kb":394304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":726,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000611+0000","last_change":"2023-10-09T09:41:11.815662+0000","last_active":"2023-10-09T09:46:27.000611+0000","last_peered":"2023-10-09T09:46:27.000611+0000","last_clean":"2023-10-09T09:46:27.000611+0000","last_became_active":"2023-10-09T09:41:11.815203+0000","last_became_peered":"2023-10-09T09:41:11.815203+0000","last_unstale":"2023-10-09T09:46:27.000611+0000","last_undegraded":"2023-10-09T09:46:27.000611+0000","last_fullsized":"2023-10-09T09:46:27.000611+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1156,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:32.807917+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:46:32.807917+0000","last_peered":"2023-10-09T09:46:32.807917+0000","last_clean":"2023-10-09T09:46:32.807917+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:46:32.807917+0000","last_undegraded":"2023-10-09T09:46:32.807917+0000","last_fullsized":"2023-10-09T09:46:32.807917+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"1239'5984","reported_seq":5999,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:58.525736+0000","last_change":"2023-10-09T09:45:20.775729+0000","last_active":"2023-10-09T09:46:58.525736+0000","last_peered":"2023-10-09T09:46:58.525736+0000","last_clean":"2023-10-09T09:46:58.525736+0000","last_became_active":"2023-10-09T09:45:20.775410+0000","last_became_peered":"2023-10-09T09:45:20.775410+0000","last_unstale":"2023-10-09T09:46:58.525736+0000","last_undegraded":"2023-10-09T09:46:58.525736+0000","last_fullsized":"2023-10-09T09:46:58.525736+0000","mapping_epoch":1232,"log_start":"1235'200","ondisk_log_start":"1235'200","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":5784,"log_dups_size":0,"ondisk_log_size":5784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:31.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392167424,"num_objects":5984,"num_object_clones":0,"num_object_copies":17952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5984,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":133,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.003363+0000","last_change":"2023-10-09T09:43:48.904982+0000","last_active":"2023-10-09T09:46:27.003363+0000","last_peered":"2023-10-09T09:46:27.003363+0000","last_clean":"2023-10-09T09:46:27.003363+0000","last_became_active":"2023-10-09T09:43:48.904605+0000","last_became_peered":"2023-10-09T09:43:48.904605+0000","last_unstale":"2023-10-09T09:46:27.003363+0000","last_undegraded":"2023-10-09T09:46:27.003363+0000","last_fullsized":"2023-10-09T09:46:27.003363+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:14.912690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1176,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000677+0000","last_change":"2023-10-09T09:43:23.996674+0000","last_active":"2023-10-09T09:46:27.000677+0000","last_peered":"2023-10-09T09:46:27.000677+0000","last_clean":"2023-10-09T09:46:27.000677+0000","last_became_active":"2023-10-09T09:43:23.995573+0000","last_became_peered":"2023-10-09T09:43:23.995573+0000","last_unstale":"2023-10-09T09:46:27.000677+0000","last_undegraded":"2023-10-09T09:46:27.000677+0000","last_fullsized":"2023-10-09T09:46:27.000677+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"1239'6292","reported_seq":6308,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:00.255125+0000","last_change":"2023-10-09T09:45:20.774813+0000","last_active":"2023-10-09T09:47:00.255125+0000","last_peered":"2023-10-09T09:47:00.255125+0000","last_clean":"2023-10-09T09:47:00.255125+0000","last_became_active":"2023-10-09T09:45:20.774338+0000","last_became_peered":"2023-10-09T09:45:20.774338+0000","last_unstale":"2023-10-09T09:47:00.255125+0000","last_undegraded":"2023-10-09T09:47:00.255125+0000","last_fullsized":"2023-10-09T09:47:00.255125+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6292,"log_dups_size":0,"ondisk_log_size":6292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:17.139271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412352512,"num_objects":6292,"num_object_clones":0,"num_object_copies":18876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6292,"num_write_kb":402688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1172,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000606+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:46:27.000606+0000","last_peered":"2023-10-09T09:46:27.000606+0000","last_clean":"2023-10-09T09:46:27.000606+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:46:27.000606+0000","last_undegraded":"2023-10-09T09:46:27.000606+0000","last_fullsized":"2023-10-09T09:46:27.000606+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1161,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.995820+0000","last_change":"2023-10-09T09:43:23.991689+0000","last_active":"2023-10-09T09:46:26.995820+0000","last_peered":"2023-10-09T09:46:26.995820+0000","last_clean":"2023-10-09T09:46:26.995820+0000","last_became_active":"2023-10-09T09:43:23.991355+0000","last_became_peered":"2023-10-09T09:43:23.991355+0000","last_unstale":"2023-10-09T09:46:26.995820+0000","last_undegraded":"2023-10-09T09:46:26.995820+0000","last_fullsized":"2023-10-09T09:46:26.995820+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:57.061947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"1239'6187","reported_seq":6201,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.502782+0000","last_change":"2023-10-09T09:45:20.775861+0000","last_active":"2023-10-09T09:47:01.502782+0000","last_peered":"2023-10-09T09:47:01.502782+0000","last_clean":"2023-10-09T09:47:01.502782+0000","last_became_active":"2023-10-09T09:45:20.775171+0000","last_became_peered":"2023-10-09T09:45:20.775171+0000","last_unstale":"2023-10-09T09:47:01.502782+0000","last_undegraded":"2023-10-09T09:47:01.502782+0000","last_fullsized":"2023-10-09T09:47:01.502782+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6187,"log_dups_size":0,"ondisk_log_size":6187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405471232,"num_objects":6187,"num_object_clones":0,"num_object_copies":18561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6187,"num_write_kb":395968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1189,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.998205+0000","last_change":"2023-10-09T09:43:23.992509+0000","last_active":"2023-10-09T09:46:26.998205+0000","last_peered":"2023-10-09T09:46:26.998205+0000","last_clean":"2023-10-09T09:46:26.998205+0000","last_became_active":"2023-10-09T09:43:23.992076+0000","last_became_peered":"2023-10-09T09:43:23.992076+0000","last_unstale":"2023-10-09T09:46:26.998205+0000","last_undegraded":"2023-10-09T09:46:26.998205+0000","last_fullsized":"2023-10-09T09:46:26.998205+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:12.832688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":597,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000336+0000","last_change":"2023-10-09T09:43:24.052308+0000","last_active":"2023-10-09T09:46:27.000336+0000","last_peered":"2023-10-09T09:46:27.000336+0000","last_clean":"2023-10-09T09:46:27.000336+0000","last_became_active":"2023-10-09T09:43:24.051972+0000","last_became_peered":"2023-10-09T09:43:24.051972+0000","last_unstale":"2023-10-09T09:46:27.000336+0000","last_undegraded":"2023-10-09T09:46:27.000336+0000","last_fullsized":"2023-10-09T09:46:27.000336+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"1239'6156","reported_seq":6172,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:58.496560+0000","last_change":"2023-10-09T09:45:20.778819+0000","last_active":"2023-10-09T09:46:58.496560+0000","last_peered":"2023-10-09T09:46:58.496560+0000","last_clean":"2023-10-09T09:46:58.496560+0000","last_became_active":"2023-10-09T09:45:20.778497+0000","last_became_peered":"2023-10-09T09:45:20.778497+0000","last_unstale":"2023-10-09T09:46:58.496560+0000","last_undegraded":"2023-10-09T09:46:58.496560+0000","last_fullsized":"2023-10-09T09:46:58.496560+0000","mapping_epoch":1232,"log_start":"1235'300","ondisk_log_start":"1235'300","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":5856,"log_dups_size":0,"ondisk_log_size":5856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:51.173183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403439616,"num_objects":6156,"num_object_clones":0,"num_object_copies":18468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6156,"num_write_kb":393984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1150,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.002132+0000","last_change":"2023-10-09T09:41:11.814910+0000","last_active":"2023-10-09T09:46:27.002132+0000","last_peered":"2023-10-09T09:46:27.002132+0000","last_clean":"2023-10-09T09:46:27.002132+0000","last_became_active":"2023-10-09T09:41:11.814561+0000","last_became_peered":"2023-10-09T09:41:11.814561+0000","last_unstale":"2023-10-09T09:46:27.002132+0000","last_undegraded":"2023-10-09T09:46:27.002132+0000","last_fullsized":"2023-10-09T09:46:27.002132+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:25:51.140694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":82,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.998915+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:46:26.998915+0000","last_peered":"2023-10-09T09:46:26.998915+0000","last_clean":"2023-10-09T09:46:26.998915+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:46:26.998915+0000","last_undegraded":"2023-10-09T09:46:26.998915+0000","last_fullsized":"2023-10-09T09:46:26.998915+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"1239'5992","reported_seq":6007,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.470643+0000","last_change":"2023-10-09T09:45:20.775453+0000","last_active":"2023-10-09T09:47:01.470643+0000","last_peered":"2023-10-09T09:47:01.470643+0000","last_clean":"2023-10-09T09:47:01.470643+0000","last_became_active":"2023-10-09T09:45:20.774585+0000","last_became_peered":"2023-10-09T09:45:20.774585+0000","last_unstale":"2023-10-09T09:47:01.470643+0000","last_undegraded":"2023-10-09T09:47:01.470643+0000","last_fullsized":"2023-10-09T09:47:01.470643+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":5992,"log_dups_size":0,"ondisk_log_size":5992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392691712,"num_objects":5992,"num_object_clones":0,"num_object_copies":17976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5992,"num_write_kb":383488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1203,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.002786+0000","last_change":"2023-10-09T09:40:49.781045+0000","last_active":"2023-10-09T09:46:27.002786+0000","last_peered":"2023-10-09T09:46:27.002786+0000","last_clean":"2023-10-09T09:46:27.002786+0000","last_became_active":"2023-10-09T09:40:49.780425+0000","last_became_peered":"2023-10-09T09:40:49.780425+0000","last_unstale":"2023-10-09T09:46:27.002786+0000","last_undegraded":"2023-10-09T09:46:27.002786+0000","last_fullsized":"2023-10-09T09:46:27.002786+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:56.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":81,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.995377+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:46:26.995377+0000","last_peered":"2023-10-09T09:46:26.995377+0000","last_clean":"2023-10-09T09:46:26.995377+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:46:26.995377+0000","last_undegraded":"2023-10-09T09:46:26.995377+0000","last_fullsized":"2023-10-09T09:46:26.995377+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1187,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.001218+0000","last_change":"2023-10-09T09:40:25.799081+0000","last_active":"2023-10-09T09:46:27.001218+0000","last_peered":"2023-10-09T09:46:27.001218+0000","last_clean":"2023-10-09T09:46:27.001218+0000","last_became_active":"2023-10-09T09:40:25.798693+0000","last_became_peered":"2023-10-09T09:40:25.798693+0000","last_unstale":"2023-10-09T09:46:27.001218+0000","last_undegraded":"2023-10-09T09:46:27.001218+0000","last_fullsized":"2023-10-09T09:46:27.001218+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:35.527067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"1239'6130","reported_seq":6145,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.515122+0000","last_change":"2023-10-09T09:45:19.874686+0000","last_active":"2023-10-09T09:47:01.515122+0000","last_peered":"2023-10-09T09:47:01.515122+0000","last_clean":"2023-10-09T09:47:01.515122+0000","last_became_active":"2023-10-09T09:45:19.874293+0000","last_became_peered":"2023-10-09T09:45:19.874293+0000","last_unstale":"2023-10-09T09:47:01.515122+0000","last_undegraded":"2023-10-09T09:47:01.515122+0000","last_fullsized":"2023-10-09T09:47:01.515122+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6130,"log_dups_size":0,"ondisk_log_size":6130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401735680,"num_objects":6130,"num_object_clones":0,"num_object_copies":18390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6130,"num_write_kb":392320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":669,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.001099+0000","last_change":"2023-10-09T09:40:25.795910+0000","last_active":"2023-10-09T09:46:27.001099+0000","last_peered":"2023-10-09T09:46:27.001099+0000","last_clean":"2023-10-09T09:46:27.001099+0000","last_became_active":"2023-10-09T09:40:25.795327+0000","last_became_peered":"2023-10-09T09:40:25.795327+0000","last_unstale":"2023-10-09T09:46:27.001099+0000","last_undegraded":"2023-10-09T09:46:27.001099+0000","last_fullsized":"2023-10-09T09:46:27.001099+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:58.343181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1129,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000518+0000","last_change":"2023-10-09T09:43:48.908190+0000","last_active":"2023-10-09T09:46:27.000518+0000","last_peered":"2023-10-09T09:46:27.000518+0000","last_clean":"2023-10-09T09:46:27.000518+0000","last_became_active":"2023-10-09T09:43:48.907799+0000","last_became_peered":"2023-10-09T09:43:48.907799+0000","last_unstale":"2023-10-09T09:46:27.000518+0000","last_undegraded":"2023-10-09T09:46:27.000518+0000","last_fullsized":"2023-10-09T09:46:27.000518+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1226,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.002281+0000","last_change":"2023-10-09T09:41:11.811782+0000","last_active":"2023-10-09T09:46:27.002281+0000","last_peered":"2023-10-09T09:46:27.002281+0000","last_clean":"2023-10-09T09:46:27.002281+0000","last_became_active":"2023-10-09T09:41:11.811431+0000","last_became_peered":"2023-10-09T09:41:11.811431+0000","last_unstale":"2023-10-09T09:46:27.002281+0000","last_undegraded":"2023-10-09T09:46:27.002281+0000","last_fullsized":"2023-10-09T09:46:27.002281+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:05.197258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"1239'6128","reported_seq":6144,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:00.259979+0000","last_change":"2023-10-09T09:45:20.779143+0000","last_active":"2023-10-09T09:47:00.259979+0000","last_peered":"2023-10-09T09:47:00.259979+0000","last_clean":"2023-10-09T09:47:00.259979+0000","last_became_active":"2023-10-09T09:45:20.778454+0000","last_became_peered":"2023-10-09T09:45:20.778454+0000","last_unstale":"2023-10-09T09:47:00.259979+0000","last_undegraded":"2023-10-09T09:47:00.259979+0000","last_fullsized":"2023-10-09T09:47:00.259979+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6128,"log_dups_size":0,"ondisk_log_size":6128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:31.777588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401604608,"num_objects":6128,"num_object_clones":0,"num_object_copies":18384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6128,"num_write_kb":392192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":112,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.998289+0000","last_change":"2023-10-09T09:41:11.801228+0000","last_active":"2023-10-09T09:46:26.998289+0000","last_peered":"2023-10-09T09:46:26.998289+0000","last_clean":"2023-10-09T09:46:26.998289+0000","last_became_active":"2023-10-09T09:41:11.800068+0000","last_became_peered":"2023-10-09T09:41:11.800068+0000","last_unstale":"2023-10-09T09:46:26.998289+0000","last_undegraded":"2023-10-09T09:46:26.998289+0000","last_fullsized":"2023-10-09T09:46:26.998289+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:33.425580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1203,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155315+0000","last_change":"2023-10-09T09:36:43.421868+0000","last_active":"2023-10-09T09:46:39.155315+0000","last_peered":"2023-10-09T09:46:39.155315+0000","last_clean":"2023-10-09T09:46:39.155315+0000","last_became_active":"2023-10-09T09:36:43.421013+0000","last_became_peered":"2023-10-09T09:36:43.421013+0000","last_unstale":"2023-10-09T09:46:39.155315+0000","last_undegraded":"2023-10-09T09:46:39.155315+0000","last_fullsized":"2023-10-09T09:46:39.155315+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:43.353667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1206,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.001717+0000","last_change":"2023-10-09T09:40:25.799440+0000","last_active":"2023-10-09T09:46:27.001717+0000","last_peered":"2023-10-09T09:46:27.001717+0000","last_clean":"2023-10-09T09:46:27.001717+0000","last_became_active":"2023-10-09T09:40:25.797967+0000","last_became_peered":"2023-10-09T09:40:25.797967+0000","last_unstale":"2023-10-09T09:46:27.001717+0000","last_undegraded":"2023-10-09T09:46:27.001717+0000","last_fullsized":"2023-10-09T09:46:27.001717+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:51.038572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"1239'6222","reported_seq":6237,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.460990+0000","last_change":"2023-10-09T09:45:19.874884+0000","last_active":"2023-10-09T09:47:01.460990+0000","last_peered":"2023-10-09T09:47:01.460990+0000","last_clean":"2023-10-09T09:47:01.460990+0000","last_became_active":"2023-10-09T09:45:19.874339+0000","last_became_peered":"2023-10-09T09:45:19.874339+0000","last_unstale":"2023-10-09T09:47:01.460990+0000","last_undegraded":"2023-10-09T09:47:01.460990+0000","last_fullsized":"2023-10-09T09:47:01.460990+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6222,"log_dups_size":0,"ondisk_log_size":6222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407764992,"num_objects":6222,"num_object_clones":0,"num_object_copies":18666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6222,"num_write_kb":398208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1246,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.160904+0000","last_change":"2023-10-09T09:41:11.795802+0000","last_active":"2023-10-09T09:46:39.160904+0000","last_peered":"2023-10-09T09:46:39.160904+0000","last_clean":"2023-10-09T09:46:39.160904+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:46:39.160904+0000","last_undegraded":"2023-10-09T09:46:39.160904+0000","last_fullsized":"2023-10-09T09:46:39.160904+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":564,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.164345+0000","last_change":"2023-10-09T09:40:52.800168+0000","last_active":"2023-10-09T09:46:39.164345+0000","last_peered":"2023-10-09T09:46:39.164345+0000","last_clean":"2023-10-09T09:46:39.164345+0000","last_became_active":"2023-10-09T09:40:52.799543+0000","last_became_peered":"2023-10-09T09:40:52.799543+0000","last_unstale":"2023-10-09T09:46:39.164345+0000","last_undegraded":"2023-10-09T09:46:39.164345+0000","last_fullsized":"2023-10-09T09:46:39.164345+0000","mapping_epoch":1185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:44.481003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":81,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.996136+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:46:26.996136+0000","last_peered":"2023-10-09T09:46:26.996136+0000","last_clean":"2023-10-09T09:46:26.996136+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:46:26.996136+0000","last_undegraded":"2023-10-09T09:46:26.996136+0000","last_fullsized":"2023-10-09T09:46:26.996136+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1165,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.002911+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:46:27.002911+0000","last_peered":"2023-10-09T09:46:27.002911+0000","last_clean":"2023-10-09T09:46:27.002911+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:46:27.002911+0000","last_undegraded":"2023-10-09T09:46:27.002911+0000","last_fullsized":"2023-10-09T09:46:27.002911+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"1239'6242","reported_seq":6258,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:00.282246+0000","last_change":"2023-10-09T09:45:19.876941+0000","last_active":"2023-10-09T09:47:00.282246+0000","last_peered":"2023-10-09T09:47:00.282246+0000","last_clean":"2023-10-09T09:47:00.282246+0000","last_became_active":"2023-10-09T09:45:19.876658+0000","last_became_peered":"2023-10-09T09:45:19.876658+0000","last_unstale":"2023-10-09T09:47:00.282246+0000","last_undegraded":"2023-10-09T09:47:00.282246+0000","last_fullsized":"2023-10-09T09:47:00.282246+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6242,"log_dups_size":0,"ondisk_log_size":6242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:44:33.624841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409075712,"num_objects":6242,"num_object_clones":0,"num_object_copies":18726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6242,"num_write_kb":399488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1170,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155032+0000","last_change":"2023-10-09T09:41:11.815645+0000","last_active":"2023-10-09T09:46:39.155032+0000","last_peered":"2023-10-09T09:46:39.155032+0000","last_clean":"2023-10-09T09:46:39.155032+0000","last_became_active":"2023-10-09T09:41:11.815086+0000","last_became_peered":"2023-10-09T09:41:11.815086+0000","last_unstale":"2023-10-09T09:46:39.155032+0000","last_undegraded":"2023-10-09T09:46:39.155032+0000","last_fullsized":"2023-10-09T09:46:39.155032+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:29.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1173,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.996628+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:46:26.996628+0000","last_peered":"2023-10-09T09:46:26.996628+0000","last_clean":"2023-10-09T09:46:26.996628+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:46:26.996628+0000","last_undegraded":"2023-10-09T09:46:26.996628+0000","last_fullsized":"2023-10-09T09:46:26.996628+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1170,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.996900+0000","last_change":"2023-10-09T09:39:12.949268+0000","last_active":"2023-10-09T09:46:26.996900+0000","last_peered":"2023-10-09T09:46:26.996900+0000","last_clean":"2023-10-09T09:46:26.996900+0000","last_became_active":"2023-10-09T09:39:12.948924+0000","last_became_peered":"2023-10-09T09:39:12.948924+0000","last_unstale":"2023-10-09T09:46:26.996900+0000","last_undegraded":"2023-10-09T09:46:26.996900+0000","last_fullsized":"2023-10-09T09:46:26.996900+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:31.368540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"1239'6237","reported_seq":6252,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.484390+0000","last_change":"2023-10-09T09:45:20.775001+0000","last_active":"2023-10-09T09:47:01.484390+0000","last_peered":"2023-10-09T09:47:01.484390+0000","last_clean":"2023-10-09T09:47:01.484390+0000","last_became_active":"2023-10-09T09:45:20.774247+0000","last_became_peered":"2023-10-09T09:45:20.774247+0000","last_unstale":"2023-10-09T09:47:01.484390+0000","last_undegraded":"2023-10-09T09:47:01.484390+0000","last_fullsized":"2023-10-09T09:47:01.484390+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6237,"log_dups_size":0,"ondisk_log_size":6237,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408748032,"num_objects":6237,"num_object_clones":0,"num_object_copies":18711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6237,"num_write_kb":399168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1145,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.997482+0000","last_change":"2023-10-09T09:40:49.787387+0000","last_active":"2023-10-09T09:46:26.997482+0000","last_peered":"2023-10-09T09:46:26.997482+0000","last_clean":"2023-10-09T09:46:26.997482+0000","last_became_active":"2023-10-09T09:40:49.786954+0000","last_became_peered":"2023-10-09T09:40:49.786954+0000","last_unstale":"2023-10-09T09:46:26.997482+0000","last_undegraded":"2023-10-09T09:46:26.997482+0000","last_fullsized":"2023-10-09T09:46:26.997482+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:51.055735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1221,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.002667+0000","last_change":"2023-10-09T09:40:25.797470+0000","last_active":"2023-10-09T09:46:27.002667+0000","last_peered":"2023-10-09T09:46:27.002667+0000","last_clean":"2023-10-09T09:46:27.002667+0000","last_became_active":"2023-10-09T09:40:25.796966+0000","last_became_peered":"2023-10-09T09:40:25.796966+0000","last_unstale":"2023-10-09T09:46:27.002667+0000","last_undegraded":"2023-10-09T09:46:27.002667+0000","last_fullsized":"2023-10-09T09:46:27.002667+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:24.526350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1140,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.997405+0000","last_change":"2023-10-09T09:43:48.907449+0000","last_active":"2023-10-09T09:46:26.997405+0000","last_peered":"2023-10-09T09:46:26.997405+0000","last_clean":"2023-10-09T09:46:26.997405+0000","last_became_active":"2023-10-09T09:43:48.905674+0000","last_became_peered":"2023-10-09T09:43:48.905674+0000","last_unstale":"2023-10-09T09:46:26.997405+0000","last_undegraded":"2023-10-09T09:46:26.997405+0000","last_fullsized":"2023-10-09T09:46:26.997405+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"1239'6270","reported_seq":6285,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.480858+0000","last_change":"2023-10-09T09:45:20.778840+0000","last_active":"2023-10-09T09:47:01.480858+0000","last_peered":"2023-10-09T09:47:01.480858+0000","last_clean":"2023-10-09T09:47:01.480858+0000","last_became_active":"2023-10-09T09:45:20.778207+0000","last_became_peered":"2023-10-09T09:45:20.778207+0000","last_unstale":"2023-10-09T09:47:01.480858+0000","last_undegraded":"2023-10-09T09:47:01.480858+0000","last_fullsized":"2023-10-09T09:47:01.480858+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6270,"log_dups_size":0,"ondisk_log_size":6270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410910720,"num_objects":6270,"num_object_clones":0,"num_object_copies":18810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6270,"num_write_kb":401280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":98,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.997099+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:46:26.997099+0000","last_peered":"2023-10-09T09:46:26.997099+0000","last_clean":"2023-10-09T09:46:26.997099+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:46:26.997099+0000","last_undegraded":"2023-10-09T09:46:26.997099+0000","last_fullsized":"2023-10-09T09:46:26.997099+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1167,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000812+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:46:27.000812+0000","last_peered":"2023-10-09T09:46:27.000812+0000","last_clean":"2023-10-09T09:46:27.000812+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:46:27.000812+0000","last_undegraded":"2023-10-09T09:46:27.000812+0000","last_fullsized":"2023-10-09T09:46:27.000812+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1110,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.001294+0000","last_change":"2023-10-09T09:43:48.906737+0000","last_active":"2023-10-09T09:46:27.001294+0000","last_peered":"2023-10-09T09:46:27.001294+0000","last_clean":"2023-10-09T09:46:27.001294+0000","last_became_active":"2023-10-09T09:43:48.906410+0000","last_became_peered":"2023-10-09T09:43:48.906410+0000","last_unstale":"2023-10-09T09:46:27.001294+0000","last_undegraded":"2023-10-09T09:46:27.001294+0000","last_fullsized":"2023-10-09T09:46:27.001294+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:24:10.024977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.f","version":"1239'6240","reported_seq":6255,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.493877+0000","last_change":"2023-10-09T09:45:20.774871+0000","last_active":"2023-10-09T09:47:01.493877+0000","last_peered":"2023-10-09T09:47:01.493877+0000","last_clean":"2023-10-09T09:47:01.493877+0000","last_became_active":"2023-10-09T09:45:20.774349+0000","last_became_peered":"2023-10-09T09:45:20.774349+0000","last_unstale":"2023-10-09T09:47:01.493877+0000","last_undegraded":"2023-10-09T09:47:01.493877+0000","last_fullsized":"2023-10-09T09:47:01.493877+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6240,"log_dups_size":0,"ondisk_log_size":6240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408944640,"num_objects":6240,"num_object_clones":0,"num_object_copies":18720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6240,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1211,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.001247+0000","last_change":"2023-10-09T09:41:11.815721+0000","last_active":"2023-10-09T09:46:27.001247+0000","last_peered":"2023-10-09T09:46:27.001247+0000","last_clean":"2023-10-09T09:46:27.001247+0000","last_became_active":"2023-10-09T09:41:11.815237+0000","last_became_peered":"2023-10-09T09:41:11.815237+0000","last_unstale":"2023-10-09T09:46:27.001247+0000","last_undegraded":"2023-10-09T09:46:27.001247+0000","last_fullsized":"2023-10-09T09:46:27.001247+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1151,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.997073+0000","last_change":"2023-10-09T09:35:18.685030+0000","last_active":"2023-10-09T09:46:26.997073+0000","last_peered":"2023-10-09T09:46:26.997073+0000","last_clean":"2023-10-09T09:46:26.997073+0000","last_became_active":"2023-10-09T09:35:18.675810+0000","last_became_peered":"2023-10-09T09:35:18.675810+0000","last_unstale":"2023-10-09T09:46:26.997073+0000","last_undegraded":"2023-10-09T09:46:26.997073+0000","last_fullsized":"2023-10-09T09:46:26.997073+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:52.793405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1157,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.994324+0000","last_change":"2023-10-09T09:43:23.992213+0000","last_active":"2023-10-09T09:46:26.994324+0000","last_peered":"2023-10-09T09:46:26.994324+0000","last_clean":"2023-10-09T09:46:26.994324+0000","last_became_active":"2023-10-09T09:43:23.991887+0000","last_became_peered":"2023-10-09T09:43:23.991887+0000","last_unstale":"2023-10-09T09:46:26.994324+0000","last_undegraded":"2023-10-09T09:46:26.994324+0000","last_fullsized":"2023-10-09T09:46:26.994324+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1159,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.997745+0000","last_change":"2023-10-09T09:40:49.788328+0000","last_active":"2023-10-09T09:46:26.997745+0000","last_peered":"2023-10-09T09:46:26.997745+0000","last_clean":"2023-10-09T09:46:26.997745+0000","last_became_active":"2023-10-09T09:40:49.787927+0000","last_became_peered":"2023-10-09T09:40:49.787927+0000","last_unstale":"2023-10-09T09:46:26.997745+0000","last_undegraded":"2023-10-09T09:46:26.997745+0000","last_fullsized":"2023-10-09T09:46:26.997745+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:32.616104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":112,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.160578+0000","last_change":"2023-10-09T09:41:11.800443+0000","last_active":"2023-10-09T09:46:39.160578+0000","last_peered":"2023-10-09T09:46:39.160578+0000","last_clean":"2023-10-09T09:46:39.160578+0000","last_became_active":"2023-10-09T09:41:11.799864+0000","last_became_peered":"2023-10-09T09:41:11.799864+0000","last_unstale":"2023-10-09T09:46:39.160578+0000","last_undegraded":"2023-10-09T09:46:39.160578+0000","last_fullsized":"2023-10-09T09:46:39.160578+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:48:22.481162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":48,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.002505+0000","last_change":"2023-10-09T09:43:23.992016+0000","last_active":"2023-10-09T09:46:27.002505+0000","last_peered":"2023-10-09T09:46:27.002505+0000","last_clean":"2023-10-09T09:46:27.002505+0000","last_became_active":"2023-10-09T09:43:23.991400+0000","last_became_peered":"2023-10-09T09:43:23.991400+0000","last_unstale":"2023-10-09T09:46:27.002505+0000","last_undegraded":"2023-10-09T09:46:27.002505+0000","last_fullsized":"2023-10-09T09:46:27.002505+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.209599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1192,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.003136+0000","last_change":"2023-10-09T09:43:23.994636+0000","last_active":"2023-10-09T09:46:27.003136+0000","last_peered":"2023-10-09T09:46:27.003136+0000","last_clean":"2023-10-09T09:46:27.003136+0000","last_became_active":"2023-10-09T09:43:23.993694+0000","last_became_peered":"2023-10-09T09:43:23.993694+0000","last_unstale":"2023-10-09T09:46:27.003136+0000","last_undegraded":"2023-10-09T09:46:27.003136+0000","last_fullsized":"2023-10-09T09:46:27.003136+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:45.895845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":47,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.003001+0000","last_change":"2023-10-09T09:43:23.995576+0000","last_active":"2023-10-09T09:46:27.003001+0000","last_peered":"2023-10-09T09:46:27.003001+0000","last_clean":"2023-10-09T09:46:27.003001+0000","last_became_active":"2023-10-09T09:43:23.995076+0000","last_became_peered":"2023-10-09T09:43:23.995076+0000","last_unstale":"2023-10-09T09:46:27.003001+0000","last_undegraded":"2023-10-09T09:46:27.003001+0000","last_fullsized":"2023-10-09T09:46:27.003001+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1206,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.995453+0000","last_change":"2023-10-09T09:43:48.904021+0000","last_active":"2023-10-09T09:46:26.995453+0000","last_peered":"2023-10-09T09:46:26.995453+0000","last_clean":"2023-10-09T09:46:26.995453+0000","last_became_active":"2023-10-09T09:43:48.903719+0000","last_became_peered":"2023-10-09T09:43:48.903719+0000","last_unstale":"2023-10-09T09:46:26.995453+0000","last_undegraded":"2023-10-09T09:46:26.995453+0000","last_fullsized":"2023-10-09T09:46:26.995453+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:16.284027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":172,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000401+0000","last_change":"2023-10-09T09:41:11.816169+0000","last_active":"2023-10-09T09:46:27.000401+0000","last_peered":"2023-10-09T09:46:27.000401+0000","last_clean":"2023-10-09T09:46:27.000401+0000","last_became_active":"2023-10-09T09:41:11.815426+0000","last_became_peered":"2023-10-09T09:41:11.815426+0000","last_unstale":"2023-10-09T09:46:27.000401+0000","last_undegraded":"2023-10-09T09:46:27.000401+0000","last_fullsized":"2023-10-09T09:46:27.000401+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":112,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:30.263816+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:46:30.263816+0000","last_peered":"2023-10-09T09:46:30.263816+0000","last_clean":"2023-10-09T09:46:30.263816+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:46:30.263816+0000","last_undegraded":"2023-10-09T09:46:30.263816+0000","last_fullsized":"2023-10-09T09:46:30.263816+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1199,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:26.996996+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:46:26.996996+0000","last_peered":"2023-10-09T09:46:26.996996+0000","last_clean":"2023-10-09T09:46:26.996996+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:46:26.996996+0000","last_undegraded":"2023-10-09T09:46:26.996996+0000","last_fullsized":"2023-10-09T09:46:26.996996+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":597,"reported_epoch":1238,"state":"active+clean","last_fresh":"2023-10-09T09:46:27.000262+0000","last_change":"2023-10-09T09:43:24.086244+0000","last_active":"2023-10-09T09:46:27.000262+0000","last_peered":"2023-10-09T09:46:27.000262+0000","last_clean":"2023-10-09T09:46:27.000262+0000","last_became_active":"2023-10-09T09:43:24.085797+0000","last_became_peered":"2023-10-09T09:43:24.085797+0000","last_unstale":"2023-10-09T09:46:27.000262+0000","last_undegraded":"2023-10-09T09:46:27.000262+0000","last_fullsized":"2023-10-09T09:46:27.000262+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6437470208,"num_objects":98228,"num_object_clones":0,"num_object_copies":294684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":98228,"num_write_kb":6286592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19287310336,"data_stored":19287310336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97728,"ondisk_log_size":97728,"up":48,"acting":48,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":2460,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453398063,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594296,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4067984,"kb_used_data":3167304,"kb_used_omap":0,"kb_used_meta":900672,"kb_avail":89677168,"statfs":{"total":95995035648,"available":91829420032,"internally_reserved":0,"allocated":3243319296,"data_stored":3243043678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922288128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:46:38 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":1.2709999999999999,"15min":1.3979999999999999},"min":{"1min":0.245,"5min":0.224,"15min":0.224},"max":{"1min":24.265999999999998,"5min":24.265999999999998,"15min":24.265999999999998},"last":0.26500000000000001},{"interface":"front","average":{"1min":2.3900000000000001,"5min":1.637,"15min":1.639},"min":{"1min":0.34899999999999998,"5min":0.32300000000000001,"15min":0.28699999999999998},"max":{"1min":24.376999999999999,"5min":24.376999999999999,"15min":24.376999999999999},"last":1.208}]},{"osd":1,"last update":"Mon Oct 9 09:46:38 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":4.4770000000000003,"15min":3.3599999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.22900000000000001},"max":{"1min":3.8039999999999998,"5min":116.953,"15min":116.953},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.794,"5min":4.6369999999999996,"15min":3.2149999999999999},"min":{"1min":0.313,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":12.032999999999999,"5min":117.459,"15min":117.459},"last":0.55900000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:46:18 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":2.0569999999999999,"15min":1.8999999999999999},"min":{"1min":0.32800000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":46.466000000000001,"5min":46.466000000000001,"15min":46.466000000000001},"last":1.0880000000000001},{"interface":"front","average":{"1min":3.427,"5min":2.2410000000000001,"15min":2.0409999999999999},"min":{"1min":0.35699999999999998,"5min":0.318,"15min":0.29499999999999998},"max":{"1min":47.151000000000003,"5min":47.151000000000003,"15min":47.151000000000003},"last":1.254}]},{"osd":3,"last update":"Mon Oct 9 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.1120000000000001,"15min":1.7010000000000001},"min":{"1min":0.33400000000000002,"5min":0.33200000000000002,"15min":0.309},"max":{"1min":3.8959999999999999,"5min":6.0880000000000001,"15min":68.686999999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.26,"15min":1.621},"min":{"1min":0.39000000000000001,"5min":0.372,"15min":0.29499999999999998},"max":{"1min":11.968999999999999,"5min":11.968999999999999,"15min":69.108999999999995},"last":0.67000000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.4330000000000001,"15min":1.2190000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28199999999999997},"max":{"1min":7.0270000000000001,"5min":7.0270000000000001,"15min":8.6890000000000001},"last":1.6000000000000001},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.4299999999999999,"15min":1.373},"min":{"1min":0.313,"5min":0.313,"15min":0.29499999999999998},"max":{"1min":12.064,"5min":12.064,"15min":12.064},"last":0.624}]},{"osd":5,"last update":"Mon Oct 9 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":1.4890000000000001,"15min":1.345},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.056,"5min":7.056,"15min":9.0559999999999992},"last":2.786},{"interface":"front","average":{"1min":2.0059999999999998,"5min":1.49,"15min":1.458},"min":{"1min":0.44800000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.000999999999999,"5min":12.000999999999999,"15min":12.000999999999999},"last":0.59899999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:46:57 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":1.5089999999999999,"15min":1.3440000000000001},"min":{"1min":0.45200000000000001,"5min":0.41099999999999998,"15min":0.35599999999999998},"max":{"1min":4.1280000000000001,"5min":11.922000000000001,"15min":11.922000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.163,"15min":1.2470000000000001},"min":{"1min":0.40699999999999997,"5min":0.371,"15min":0.33000000000000002},"max":{"1min":4.8399999999999999,"5min":5.1920000000000002,"15min":7.6879999999999997},"last":0.55000000000000004}]}]},{"osd":2,"up_from":634,"seq":2723009266049,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3461212,"kb_used_data":2757012,"kb_used_omap":122,"kb_used_meta":704069,"kb_avail":90283940,"statfs":{"total":95995035648,"available":92450754560,"internally_reserved":0,"allocated":2823180288,"data_stored":2821734733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":720967243},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:46:40 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":2.7200000000000002,"15min":2.6589999999999998},"min":{"1min":0.248,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.0230000000000001,"5min":105.29600000000001,"15min":105.29600000000001},"last":2.1030000000000002},{"interface":"front","average":{"1min":1.365,"5min":3.1299999999999999,"15min":2.9319999999999999},"min":{"1min":0.253,"5min":0.22900000000000001,"15min":0.20399999999999999},"max":{"1min":5.3159999999999998,"5min":105.33799999999999,"15min":105.33799999999999},"last":0.98999999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":3.0499999999999998,"15min":2.9180000000000001},"min":{"1min":0.27200000000000002,"5min":0.224,"15min":0.20200000000000001},"max":{"1min":9.9390000000000001,"5min":64.700999999999993,"15min":81.662999999999997},"last":0.247},{"interface":"front","average":{"1min":0.872,"5min":3.2290000000000001,"15min":2.9620000000000002},"min":{"1min":0.26400000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.0059999999999998,"5min":64.334000000000003,"15min":81.712999999999994},"last":0.64300000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:46:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":0.83299999999999996,"15min":2.3479999999999999},"min":{"1min":0.27700000000000002,"5min":0.26700000000000002,"15min":0.252},"max":{"1min":4.1429999999999998,"5min":5.181,"15min":230.476},"last":0.377},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.83199999999999996,"15min":2.302},"min":{"1min":0.34399999999999997,"5min":0.27000000000000002,"15min":0.26200000000000001},"max":{"1min":1.962,"5min":6.8540000000000001,"15min":228.893},"last":0.56599999999999995}]},{"osd":4,"last update":"Mon Oct 9 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0289999999999999,"15min":1.327},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.30099999999999999},"max":{"1min":2.077,"5min":6.8920000000000003,"15min":11.622999999999999},"last":3.0990000000000002},{"interface":"front","average":{"1min":1.3420000000000001,"5min":1.3069999999999999,"15min":1.429},"min":{"1min":0.38300000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.7169999999999996,"5min":7.6369999999999996,"15min":10.394},"last":0.77400000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.155,"15min":1.405},"min":{"1min":0.48899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.1890000000000001,"5min":4.1890000000000001,"15min":9.7599999999999998},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.2170000000000001,"15min":1.478},"min":{"1min":0.499,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.9160000000000004,"5min":6.9450000000000003,"15min":11.718999999999999},"last":0.88700000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":1.397,"15min":1.7030000000000001},"min":{"1min":0.46000000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.7370000000000001,"5min":8.4299999999999997,"15min":11.648},"last":2.7850000000000001},{"interface":"front","average":{"1min":1.4490000000000001,"5min":1.423,"15min":1.522},"min":{"1min":0.54900000000000004,"5min":0.37,"15min":0.33300000000000002},"max":{"1min":4.8849999999999998,"5min":7.6029999999999998,"15min":10.436999999999999},"last":1.097}]},{"osd":7,"last update":"Mon Oct 9 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":1.2969999999999999,"15min":1.546},"min":{"1min":0.48999999999999999,"5min":0.42499999999999999,"15min":0.39400000000000002},"max":{"1min":4.758,"5min":7.8550000000000004,"15min":10.462999999999999},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.0700000000000001,"15min":1.4850000000000001},"min":{"1min":0.52200000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":4.2199999999999998,"5min":4.2199999999999998,"15min":9.7759999999999998},"last":0.46100000000000002}]}]},{"osd":1,"up_from":607,"seq":2607045149062,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2706704,"kb_used_data":1933448,"kb_used_omap":107,"kb_used_meta":773140,"kb_avail":91038448,"statfs":{"total":95995035648,"available":93223370752,"internally_reserved":0,"allocated":1979850752,"data_stored":1978411259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":791696043},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:46:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":4.0830000000000002,"15min":2.3239999999999998},"min":{"1min":0.28000000000000003,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":21.923999999999999,"5min":134.49100000000001,"15min":134.49100000000001},"last":0.23999999999999999},{"interface":"front","average":{"1min":2.3740000000000001,"5min":4.1459999999999999,"15min":2.8620000000000001},"min":{"1min":0.29599999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":21.966000000000001,"5min":134.226,"15min":134.226},"last":0.33100000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":1.5169999999999999,"15min":2.0739999999999998},"min":{"1min":0.315,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":2.3460000000000001,"5min":28.170999999999999,"15min":69.421999999999997},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.651,"15min":1.7849999999999999},"min":{"1min":0.36799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":2.3149999999999999,"5min":29.548999999999999,"15min":71.879000000000005},"last":2.7839999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:46:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":1.159,"15min":1.724},"min":{"1min":0.28100000000000003,"5min":0.253,"15min":0.23599999999999999},"max":{"1min":4.4880000000000004,"5min":8.3729999999999993,"15min":73.564999999999998},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.1399999999999999,"15min":2.0070000000000001},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.27900000000000003},"max":{"1min":1.8999999999999999,"5min":8.0099999999999998,"15min":72.932000000000002},"last":0.52800000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.4259999999999999,"15min":1.2929999999999999},"min":{"1min":0.52900000000000003,"5min":0.30599999999999999,"15min":0.28299999999999997},"max":{"1min":6.4400000000000004,"5min":7.6900000000000004,"15min":7.6900000000000004},"last":2.5609999999999999},{"interface":"front","average":{"1min":1.099,"5min":1.4339999999999999,"15min":1.4530000000000001},"min":{"1min":0.36699999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.226,"5min":8.3249999999999993,"15min":32.151000000000003},"last":0.34799999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.5840000000000001,"15min":1.4079999999999999},"min":{"1min":0.40799999999999997,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.8660000000000001,"5min":7.7800000000000002,"15min":8.6869999999999994},"last":2.6259999999999999},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.6000000000000001,"15min":1.4019999999999999},"min":{"1min":0.434,"5min":0.34100000000000003,"15min":0.311},"max":{"1min":4.2699999999999996,"5min":9.0190000000000001,"15min":9.0190000000000001},"last":0.45200000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.504,"15min":1.716},"min":{"1min":0.44600000000000001,"5min":0.38800000000000001,"15min":0.33600000000000002},"max":{"1min":2.2050000000000001,"5min":7.9400000000000004,"15min":14.335000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.165,"5min":1.619,"15min":1.5189999999999999},"min":{"1min":0.61799999999999999,"5min":0.372,"15min":0.36799999999999999},"max":{"1min":2.891,"5min":12.128,"15min":12.128},"last":2.73}]},{"osd":7,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.3520000000000001,"15min":1.6759999999999999},"min":{"1min":0.47799999999999998,"5min":0.44400000000000001,"15min":0.36199999999999999},"max":{"1min":2.2410000000000001,"5min":7.9770000000000003,"15min":14.364000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.371,"15min":1.6040000000000001},"min":{"1min":0.57699999999999996,"5min":0.41699999999999998,"15min":0.39600000000000002},"max":{"1min":2.9260000000000002,"5min":7.8899999999999997,"15min":32.078000000000003},"last":2.6789999999999998}]}]},{"osd":0,"up_from":1136,"seq":4879082848397,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4308964,"kb_used_data":3826460,"kb_used_omap":280,"kb_used_meta":482215,"kb_avail":89436188,"statfs":{"total":95995035648,"available":91582656512,"internally_reserved":0,"allocated":3918295040,"data_stored":3916890628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":493788570},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.671,"15min":1.593},"min":{"1min":0.374,"5min":0.247,"15min":0.184},"max":{"1min":6.3170000000000002,"5min":10.851000000000001,"15min":33.634},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.7989999999999999,"15min":1.722},"min":{"1min":0.33400000000000002,"5min":0.25,"15min":0.19700000000000001},"max":{"1min":5.7439999999999998,"5min":11.114000000000001,"15min":33.661000000000001},"last":4.7720000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":5.4640000000000004,"5min":2.121,"15min":1.5229999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":48.795000000000002,"5min":48.795000000000002,"15min":48.795000000000002},"last":1.8999999999999999},{"interface":"front","average":{"1min":6.4690000000000003,"5min":2.516,"15min":1.786},"min":{"1min":0.34999999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":48.856000000000002,"5min":48.856000000000002,"15min":48.856000000000002},"last":0.73899999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.1759999999999999,"15min":1.2889999999999999},"min":{"1min":0.34699999999999998,"5min":0.27500000000000002,"15min":0.247},"max":{"1min":5.3979999999999997,"5min":10.475,"15min":10.475},"last":3.1059999999999999},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.2909999999999999,"15min":1.248},"min":{"1min":0.374,"5min":0.26500000000000001,"15min":0.248},"max":{"1min":6.5220000000000002,"5min":6.6550000000000002,"15min":9.2400000000000002},"last":0.59299999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":1.292,"15min":1.2849999999999999},"min":{"1min":0.38,"5min":0.314,"15min":0.27900000000000003},"max":{"1min":5.7030000000000003,"5min":7.7599999999999998,"15min":7.7599999999999998},"last":8.3219999999999992},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.3520000000000001,"15min":1.4199999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30199999999999999},"max":{"1min":12.044,"5min":12.044,"15min":12.044},"last":0.46000000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":1.4770000000000001,"15min":1.407},"min":{"1min":0.47699999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.827,"5min":6.8600000000000003,"15min":6.915},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.0289999999999999,"5min":1.665,"15min":1.6080000000000001},"min":{"1min":0.41099999999999998,"5min":0.38200000000000001,"15min":0.29699999999999999},"max":{"1min":6.5979999999999999,"5min":8.3510000000000009,"15min":8.3510000000000009},"last":0.56299999999999994}]},{"osd":6,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.603,"15min":1.3939999999999999},"min":{"1min":0.432,"5min":0.35199999999999998,"15min":0.315},"max":{"1min":6.0419999999999998,"5min":10.413,"15min":10.413},"last":4.875},{"interface":"front","average":{"1min":1.8,"5min":1.6659999999999999,"15min":1.758},"min":{"1min":0.51600000000000001,"5min":0.443,"15min":0.35599999999999998},"max":{"1min":5.9939999999999998,"5min":6.9790000000000001,"15min":8.625},"last":0.42599999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.367,"15min":1.429},"min":{"1min":0.50600000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.9740000000000002,"5min":6.9909999999999997,"15min":9.2159999999999993},"last":4.9690000000000003},{"interface":"front","average":{"1min":1.7749999999999999,"5min":1.454,"15min":1.5940000000000001},"min":{"1min":0.71899999999999997,"5min":0.443,"15min":0.39200000000000002},"max":{"1min":5.8860000000000001,"5min":6.7679999999999998,"15min":14.361000000000001},"last":0.52000000000000002}]}]},{"osd":3,"up_from":0,"seq":2847563317627,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986761,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5239312,"kb_used_data":4340040,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":88505840,"statfs":{"total":95995035648,"available":90629980160,"internally_reserved":0,"allocated":4444200960,"data_stored":4443924820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:46:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0600000000000001,"5min":1.415,"15min":1.296},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.253},"max":{"1min":7.8239999999999998,"5min":9.5109999999999992,"15min":16.408999999999999},"last":11.147},{"interface":"front","average":{"1min":1.351,"5min":1.1639999999999999,"15min":1.3560000000000001},"min":{"1min":0.308,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.7869999999999999,"5min":9.5399999999999991,"15min":16.387},"last":7.7699999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.353,"15min":2.1389999999999998},"min":{"1min":0.32300000000000001,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":3.2280000000000002,"5min":25.422000000000001,"15min":85.186999999999998},"last":8.1470000000000002},{"interface":"front","average":{"1min":2.0249999999999999,"5min":1.8999999999999999,"15min":2.5990000000000002},"min":{"1min":0.33000000000000002,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":5.923,"5min":25.956,"15min":85.159000000000006},"last":4.2329999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.2689999999999999,"15min":1.651},"min":{"1min":0.54600000000000004,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.988,"5min":7.5119999999999996,"15min":99.736000000000004},"last":12.000999999999999},{"interface":"front","average":{"1min":1.389,"5min":1.1859999999999999,"15min":1.7649999999999999},"min":{"1min":0.66500000000000004,"5min":0.309,"15min":0.30099999999999999},"max":{"1min":4.1310000000000002,"5min":6.9580000000000002,"15min":99.212999999999994},"last":4.2670000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":0.97199999999999998,"15min":1.6100000000000001},"min":{"1min":0.34899999999999998,"5min":0.34200000000000003,"15min":0.27600000000000002},"max":{"1min":4.7469999999999999,"5min":5.734,"15min":47.32},"last":8.1120000000000001},{"interface":"front","average":{"1min":2.032,"5min":1.365,"15min":2.1890000000000001},"min":{"1min":0.35999999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.3879999999999999,"5min":6.9950000000000001,"15min":93.843000000000004},"last":4.7720000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":1.6220000000000001,"15min":1.4259999999999999},"min":{"1min":0.41699999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.7670000000000003,"5min":7.5910000000000002,"15min":17.212},"last":11.763999999999999},{"interface":"front","average":{"1min":1.208,"5min":1.375,"15min":1.4470000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.8319999999999999,"5min":6.0330000000000004,"15min":17.268999999999998},"last":8.1760000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":2.7959999999999998,"5min":1.5780000000000001,"15min":1.401},"min":{"1min":0.45500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.1070000000000002,"5min":9.6859999999999999,"15min":13.654},"last":4.3209999999999997},{"interface":"front","average":{"1min":2.6899999999999999,"5min":1.6299999999999999,"15min":1.427},"min":{"1min":0.64700000000000002,"5min":0.39200000000000002,"15min":0.35699999999999998},"max":{"1min":8.6010000000000009,"5min":8.6010000000000009,"15min":9.3369999999999997},"last":11.972}]},{"osd":7,"last update":"Mon Oct 9 09:46:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.177,"15min":1.206},"min":{"1min":0.69299999999999995,"5min":0.376,"15min":0.376},"max":{"1min":2.0550000000000002,"5min":7.5720000000000001,"15min":9.2889999999999997},"last":11.840999999999999},{"interface":"front","average":{"1min":1.087,"5min":1.026,"15min":1.194},"min":{"1min":0.46800000000000003,"5min":0.40699999999999997,"15min":0.34999999999999998},"max":{"1min":2.0550000000000002,"5min":5.7009999999999996,"15min":12.34},"last":8.0760000000000005}]}]},{"osd":5,"up_from":37,"seq":158913790527,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3687868,"kb_used_data":2777268,"kb_used_omap":0,"kb_used_meta":910592,"kb_avail":90057284,"statfs":{"total":95995035648,"available":92218658816,"internally_reserved":0,"allocated":2843922432,"data_stored":2843662446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932446208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:46:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":5.6449999999999996,"15min":5.3780000000000001},"min":{"1min":0.27700000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.992,"5min":274.71899999999999,"15min":274.71899999999999},"last":1.248},{"interface":"front","average":{"1min":0.94499999999999995,"5min":5.6360000000000001,"15min":5.032},"min":{"1min":0.29099999999999998,"5min":0.28199999999999997,"15min":0.245},"max":{"1min":2.5739999999999998,"5min":274.73500000000001,"15min":274.73500000000001},"last":1.258}]},{"osd":1,"last update":"Mon Oct 9 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":1.8100000000000001,"15min":3.7629999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.28299999999999997},"max":{"1min":7.7679999999999998,"5min":40.503,"15min":142.67500000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.448,"5min":1.877,"15min":3.8330000000000002},"min":{"1min":0.42799999999999999,"5min":0.39500000000000002,"15min":0.33500000000000002},"max":{"1min":7.7990000000000004,"5min":40.415999999999997,"15min":142.56399999999999},"last":0.56799999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.887,"15min":1.5369999999999999},"min":{"1min":0.40899999999999997,"5min":0.35199999999999998,"15min":0.31},"max":{"1min":2.0840000000000001,"5min":50.847000000000001,"15min":50.847000000000001},"last":0.498},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.857,"15min":1.736},"min":{"1min":0.437,"5min":0.30099999999999999,"15min":0.28000000000000003},"max":{"1min":2.5760000000000001,"5min":50.656999999999996,"15min":50.656999999999996},"last":1.2170000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.2490000000000001,"15min":1.7190000000000001},"min":{"1min":0.374,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":1.6819999999999999,"5min":6.3470000000000004,"15min":59.652000000000001},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.413,"5min":1.046,"15min":1.8009999999999999},"min":{"1min":0.48199999999999998,"5min":0.33200000000000002,"15min":0.29099999999999998},"max":{"1min":4.7839999999999998,"5min":6.4930000000000003,"15min":61.771999999999998},"last":1.097}]},{"osd":4,"last update":"Mon Oct 9 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.3999999999999999,"15min":1.1599999999999999},"min":{"1min":0.33700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.2669999999999995,"5min":9.2669999999999995,"15min":9.2669999999999995},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.4199999999999999,"15min":1.446},"min":{"1min":0.40799999999999997,"5min":0.34399999999999997,"15min":0.33300000000000002},"max":{"1min":5.891,"5min":7.718,"15min":12.151},"last":1.1919999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":2.6840000000000002,"15min":1.7809999999999999},"min":{"1min":0.45800000000000002,"5min":0.34399999999999997,"15min":0.30399999999999999},"max":{"1min":6.1239999999999997,"5min":140.179,"15min":140.179},"last":0.88900000000000001},{"interface":"front","average":{"1min":1.8440000000000001,"5min":2.6019999999999999,"15min":2.04},"min":{"1min":0.45000000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.1909999999999998,"5min":137.928,"15min":137.928},"last":1.1639999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.234,"15min":1.472},"min":{"1min":0.432,"5min":0.373,"15min":0.33400000000000002},"max":{"1min":5.6790000000000003,"5min":6.5279999999999996,"15min":18.683},"last":1.133},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.96399999999999997,"15min":1.24},"min":{"1min":0.46300000000000002,"5min":0.38800000000000001,"15min":0.32100000000000001},"max":{"1min":2.609,"5min":6.5460000000000003,"15min":10.192},"last":0.63}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3920429056,"data_stored":3920429056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1983250432,"data_stored":1983250432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2841051136,"data_stored":2841051136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4458479616,"data_stored":4458479616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2842755072,"data_stored":2842755072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3241345024,"data_stored":3241345024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:47:07.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:47:07.409 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:07.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:07.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:07.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:07.540 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:07.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:07.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:07.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 103815 103799 78.1471 51.125 0.0246523 0.0127904 2023-10-09T09:47:07.644 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:07.681 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:07.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:07.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:07.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:07.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:07.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:07.938 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:07.966 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:07.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:07.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:08.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:07 smithi181 bash[59345]: cluster 2023-10-09T09:47:06.096475+0000 mgr.y (mgr.34107) 1233 : cluster 0 pgmap v2069: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 53 MiB/s wr, 840 op/s 2023-10-09T09:47:08.077 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:08.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:08.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:08.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:08.213 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:07 smithi163 bash[75425]: cluster 2023-10-09T09:47:06.096475+0000 mgr.y (mgr.34107) 1233 : cluster 0 pgmap v2069: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 53 MiB/s wr, 840 op/s 2023-10-09T09:47:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:07 smithi163 bash[78107]: cluster 2023-10-09T09:47:06.096475+0000 mgr.y (mgr.34107) 1233 : cluster 0 pgmap v2069: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 53 MiB/s wr, 840 op/s 2023-10-09T09:47:08.237 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:08.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:08.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:08.342 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:08.377 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:08.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:08.481 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:08.513 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:08.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:08.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:08.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 104648 104632 77.8366 52.0625 0.0219585 0.0128422 2023-10-09T09:47:08.617 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:08.643 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:08.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:08.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:08.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:08.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:08.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:08.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:08.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:08.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:09.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:09.044 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:09.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:09.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:09.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:09.181 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:09.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:09.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:09.285 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:09.309 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:09.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:09.412 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:09.437 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:09.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:09.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:09.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:09.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 105508 105492 77.5532 53.75 0.016203 0.0128895 2023-10-09T09:47:09.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:47:09.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:09.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:09.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:09.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:09.716 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:09.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:09.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:09.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:09.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:09.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:09.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:09.959 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:09.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:09 smithi163 bash[78107]: cluster 2023-10-09T09:47:08.097180+0000 mgr.y (mgr.34107) 1234 : cluster 0 pgmap v2070: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T09:47:09.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:09 smithi163 bash[75425]: cluster 2023-10-09T09:47:08.097180+0000 mgr.y (mgr.34107) 1234 : cluster 0 pgmap v2070: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T09:47:09.979 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:09.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:09.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:10.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:10.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:10.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:10.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:10.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:10.244 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:10.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:10.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:10.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:09 smithi181 bash[59345]: cluster 2023-10-09T09:47:08.097180+0000 mgr.y (mgr.34107) 1234 : cluster 0 pgmap v2070: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T09:47:10.349 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:10.385 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:10.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:10.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:10.523 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:10.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:10.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:10.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 106256 106240 77.195 46.75 0.0143721 0.0129485 2023-10-09T09:47:10.625 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:10.656 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:10.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:10.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:10.762 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:10.798 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:10.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:10.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:10.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:10.937 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:10.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:10.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:11.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:11.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:11.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:11.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:11.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:11.197 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:11.198 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:11.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:11.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:11.305 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:11.337 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:11.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:11.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:11.443 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:11.474 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:11.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[78107]: cluster 2023-10-09T09:47:10.097772+0000 mgr.y (mgr.34107) 1235 : cluster 0 pgmap v2071: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 50 MiB/s wr, 801 op/s 2023-10-09T09:47:11.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[78107]: audit 2023-10-09T09:47:10.399011+0000 mon.a (mon.0) 2294 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[78107]: audit 2023-10-09T09:47:10.400571+0000 mon.a (mon.0) 2295 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:11.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[78107]: audit 2023-10-09T09:47:10.742121+0000 mon.a (mon.0) 2296 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[78107]: audit 2023-10-09T09:47:10.751195+0000 mon.a (mon.0) 2297 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[78107]: audit 2023-10-09T09:47:10.755436+0000 mon.a (mon.0) 2298 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:11.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[78107]: audit 2023-10-09T09:47:10.756767+0000 mon.a (mon.0) 2299 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:11.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[78107]: audit 2023-10-09T09:47:10.763516+0000 mon.a (mon.0) 2300 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.479 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[75425]: cluster 2023-10-09T09:47:10.097772+0000 mgr.y (mgr.34107) 1235 : cluster 0 pgmap v2071: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 50 MiB/s wr, 801 op/s 2023-10-09T09:47:11.479 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[75425]: audit 2023-10-09T09:47:10.399011+0000 mon.a (mon.0) 2294 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.479 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[75425]: audit 2023-10-09T09:47:10.400571+0000 mon.a (mon.0) 2295 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:11.479 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[75425]: audit 2023-10-09T09:47:10.742121+0000 mon.a (mon.0) 2296 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.479 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[75425]: audit 2023-10-09T09:47:10.751195+0000 mon.a (mon.0) 2297 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.479 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[75425]: audit 2023-10-09T09:47:10.755436+0000 mon.a (mon.0) 2298 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:11.479 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[75425]: audit 2023-10-09T09:47:10.756767+0000 mon.a (mon.0) 2299 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:11.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:11 smithi163 bash[75425]: audit 2023-10-09T09:47:10.763516+0000 mon.a (mon.0) 2300 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:11.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:11.547 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 107104 107088 76.9168 53 0.0262027 0.0129958 2023-10-09T09:47:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:11 smithi181 bash[59345]: cluster 2023-10-09T09:47:10.097772+0000 mgr.y (mgr.34107) 1235 : cluster 0 pgmap v2071: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 50 MiB/s wr, 801 op/s 2023-10-09T09:47:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:11 smithi181 bash[59345]: audit 2023-10-09T09:47:10.399011+0000 mon.a (mon.0) 2294 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:11 smithi181 bash[59345]: audit 2023-10-09T09:47:10.400571+0000 mon.a (mon.0) 2295 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:11 smithi181 bash[59345]: audit 2023-10-09T09:47:10.742121+0000 mon.a (mon.0) 2296 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:11 smithi181 bash[59345]: audit 2023-10-09T09:47:10.751195+0000 mon.a (mon.0) 2297 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:11 smithi181 bash[59345]: audit 2023-10-09T09:47:10.755436+0000 mon.a (mon.0) 2298 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:11 smithi181 bash[59345]: audit 2023-10-09T09:47:10.756767+0000 mon.a (mon.0) 2299 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:11.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:11 smithi181 bash[59345]: audit 2023-10-09T09:47:10.763516+0000 mon.a (mon.0) 2300 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:11.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:11.626 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:11.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:11.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:11.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:11.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:11.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:11.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:11.897 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:11.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:11.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:12.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:12.034 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:12.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:12.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:12.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:12.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:12.185 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:12.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:12.286 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:12.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:12.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:12.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:12.423 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:12.451 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:12.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:12.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:12.548 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 107959 107943 76.65 53.4375 0.00862429 0.013041 2023-10-09T09:47:12.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:12.591 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:12.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:12.696 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:12.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:12.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:12.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:12.855 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:12.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:12.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:12.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:12.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:12.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:12.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:13.096 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:13.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:13.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:13.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:13.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:13.224 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:13.230 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:13.254 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:13.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:13.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:13.360 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:13.388 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:13.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:13.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:13.401 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:47:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:47:13] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.33.4" 2023-10-09T09:47:13.492 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:13.525 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:13.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:13.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:13.548 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 108777 108761 76.3632 51.125 0.0207653 0.0130901 2023-10-09T09:47:13.628 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:13.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:13.664 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:13 smithi163 bash[75425]: cluster 2023-10-09T09:47:12.098426+0000 mgr.y (mgr.34107) 1236 : cluster 0 pgmap v2072: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 48 MiB/s wr, 774 op/s 2023-10-09T09:47:13.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:13 smithi163 bash[78107]: cluster 2023-10-09T09:47:12.098426+0000 mgr.y (mgr.34107) 1236 : cluster 0 pgmap v2072: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 48 MiB/s wr, 774 op/s 2023-10-09T09:47:13.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:13.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:13.766 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:13.793 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:13.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:13.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:13 smithi181 bash[59345]: cluster 2023-10-09T09:47:12.098426+0000 mgr.y (mgr.34107) 1236 : cluster 0 pgmap v2072: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 48 MiB/s wr, 774 op/s 2023-10-09T09:47:13.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:13.924 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:13.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:13.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:14.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:14.070 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:14.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:14.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:14.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:14.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:14.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:14.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:14.317 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:14.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:14.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:14.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:14.456 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:14.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:14.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:14.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:14.548 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 90 6 109450 109444 75.989 42.6875 0.0225365 0.0131532 2023-10-09T09:47:14.590 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:14.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:14.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:14.712 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:14.738 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:14.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:14.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:14.842 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:14.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:14.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:14.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:14.974 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:14.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:14.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:14.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:15.098 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:15.130 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:15.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:15.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:15.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:15.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:15.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:15.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:15.366 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:15.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:15.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:15.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:15.506 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:15.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:15 smithi163 bash[75425]: cluster 2023-10-09T09:47:14.099070+0000 mgr.y (mgr.34107) 1237 : cluster 0 pgmap v2073: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 47 MiB/s wr, 745 op/s 2023-10-09T09:47:15.519 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:15 smithi163 bash[78107]: cluster 2023-10-09T09:47:14.099070+0000 mgr.y (mgr.34107) 1237 : cluster 0 pgmap v2073: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 47 MiB/s wr, 745 op/s 2023-10-09T09:47:15.535 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:15.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:15.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:15.548 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.0537 2023-10-09T09:47:15.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 109450 2023-10-09T09:47:15.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T09:47:15.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T09:47:15.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 75.9616 2023-10-09T09:47:15.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 32.4721 2023-10-09T09:47:15.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 191.25 2023-10-09T09:47:15.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 39.75 2023-10-09T09:47:15.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1215 2023-10-09T09:47:15.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 519.553 2023-10-09T09:47:15.549 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 3060 2023-10-09T09:47:15.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 636 2023-10-09T09:47:15.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.0131558 2023-10-09T09:47:15.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0121624 2023-10-09T09:47:15.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 0.299706 2023-10-09T09:47:15.550 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.00129664 2023-10-09T09:47:15.564 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T09:47:15.639 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:15.647 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:15 smithi181 bash[59345]: cluster 2023-10-09T09:47:14.099070+0000 mgr.y (mgr.34107) 1237 : cluster 0 pgmap v2073: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 47 MiB/s wr, 745 op/s 2023-10-09T09:47:15.667 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:15.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:15.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:15.770 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:15.805 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:15.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:15.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:15.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:15.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:15.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:15.971 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:16.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:16.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:16.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:16.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:16.193 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:16.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:16.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:16.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:16.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:16.377 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:16.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:16.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:16.481 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:16.505 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:16.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:16.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:16.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:16.640 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:16.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:16.745 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:16.775 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:16.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:16.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:16.905 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:16.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:16.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:17.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:17.050 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:17.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:17.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:17.155 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:17.194 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:17.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:17.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:17.301 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:17.331 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:17.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:17.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:17.435 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:17.467 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:17.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:17.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:17.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:17.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:17.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:17.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:17.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:17.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:17.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:17.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:17.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:17.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:17.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:17.976 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:17.985 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:17 smithi181 bash[59345]: cluster 2023-10-09T09:47:16.099577+0000 mgr.y (mgr.34107) 1238 : cluster 0 pgmap v2074: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T09:47:18.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:18.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:18.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:18.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:18.148 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:18.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:18.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:18.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:18.273 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:18.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:18.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:17 smithi163 bash[75425]: cluster 2023-10-09T09:47:16.099577+0000 mgr.y (mgr.34107) 1238 : cluster 0 pgmap v2074: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T09:47:18.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:17 smithi163 bash[78107]: cluster 2023-10-09T09:47:16.099577+0000 mgr.y (mgr.34107) 1238 : cluster 0 pgmap v2074: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T09:47:18.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:18.407 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:18.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:18.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:18.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:18.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:18.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:18.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:18.654 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:18.676 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:18.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:18.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:18.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:18.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:18.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:18.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:18.945 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:18.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:18.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:19.053 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:19.083 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:19.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:19.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:19.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:19.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:19.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:19.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:19.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:19.344 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:19.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:19.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:19.419 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:47:19.419 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:47:19.419 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:47:19.419 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [] 2023-10-09T09:47:19.419 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:47:19.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:19.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:19.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:19.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:19.587 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:19.620 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:19.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:19.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:19.725 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:19.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:19.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:19.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:19.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:19.887 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:19.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:19.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:19.992 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:20.017 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:20.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:20.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:20.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:19 smithi181 bash[59345]: cluster 2023-10-09T09:47:18.100235+0000 mgr.y (mgr.34107) 1239 : cluster 0 pgmap v2075: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 85 B/s rd, 48 MiB/s wr, 818 op/s 2023-10-09T09:47:20.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:20.131 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:19 smithi163 bash[78107]: cluster 2023-10-09T09:47:18.100235+0000 mgr.y (mgr.34107) 1239 : cluster 0 pgmap v2075: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 85 B/s rd, 48 MiB/s wr, 818 op/s 2023-10-09T09:47:20.132 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:19 smithi163 bash[75425]: cluster 2023-10-09T09:47:18.100235+0000 mgr.y (mgr.34107) 1239 : cluster 0 pgmap v2075: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 85 B/s rd, 48 MiB/s wr, 818 op/s 2023-10-09T09:47:20.151 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:20.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:20.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:20.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:20.289 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:20.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:20.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:20.394 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:20.418 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:20.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:20.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:20.525 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:20.550 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:20.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:20.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:20.657 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:20.690 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:20.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:20.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:20.792 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:20.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:20.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:20.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:20.933 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:20.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:20.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:21.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:21.096 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:21.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:21.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:21.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:21.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:21.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:21.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:21.334 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:21.358 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:21.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:21.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:21.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:21.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:21.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:21.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:21.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:21.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:21.649 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:21.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:21.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:21.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:21.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:21.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:21.869 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:21 smithi163 bash[75425]: cluster 2023-10-09T09:47:20.100759+0000 mgr.y (mgr.34107) 1240 : cluster 0 pgmap v2076: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 85 B/s rd, 36 MiB/s wr, 649 op/s 2023-10-09T09:47:21.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:21.919 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:21.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:22.024 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:22.051 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:22.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:22.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:22.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:21 smithi181 bash[59345]: cluster 2023-10-09T09:47:20.100759+0000 mgr.y (mgr.34107) 1240 : cluster 0 pgmap v2076: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 85 B/s rd, 36 MiB/s wr, 649 op/s 2023-10-09T09:47:22.159 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:22.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:22.186 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:22.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:22.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:21 smithi163 bash[78107]: cluster 2023-10-09T09:47:20.100759+0000 mgr.y (mgr.34107) 1240 : cluster 0 pgmap v2076: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 85 B/s rd, 36 MiB/s wr, 649 op/s 2023-10-09T09:47:22.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:22.312 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:22.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:22.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:22.416 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:22.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:22.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:22.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:22.549 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:22.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:22.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:22.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:22.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:22.711 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:22.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:22.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:22.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:22.842 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:22.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:22.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:22.945 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:22.976 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:22.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:22.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:23.079 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:23.112 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:23.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:23.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:23.218 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:23.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:23.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:23.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:23.394 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:23.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:23.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:23.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:47:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:47:23] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.33.4" 2023-10-09T09:47:23.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:23.523 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:23.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:23.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:23.626 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:23.654 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:23.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:23.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:23.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:23.798 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:23.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:23.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:23.902 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:23.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:23.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:23 smithi163 bash[75425]: cluster 2023-10-09T09:47:22.101432+0000 mgr.y (mgr.34107) 1241 : cluster 0 pgmap v2077: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 33 MiB/s wr, 865 op/s 2023-10-09T09:47:23.931 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:23 smithi163 bash[78107]: cluster 2023-10-09T09:47:22.101432+0000 mgr.y (mgr.34107) 1241 : cluster 0 pgmap v2077: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 33 MiB/s wr, 865 op/s 2023-10-09T09:47:23.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:23.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:24.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:24.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:24.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:24.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:23 smithi181 bash[59345]: cluster 2023-10-09T09:47:22.101432+0000 mgr.y (mgr.34107) 1241 : cluster 0 pgmap v2077: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 33 MiB/s wr, 865 op/s 2023-10-09T09:47:24.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:24.189 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:24.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:24.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:24.292 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:24.338 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:24.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:24.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:24.402 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T09:47:24.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:24.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:24.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:24.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:24.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:24.647 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:24.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:24.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:24.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:24.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:24.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:24.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:24.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:24.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:24.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:25.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:25.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:25.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:25.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:25.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:25.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:25.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:25.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:25.283 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:25.306 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:25.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:25.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:25.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:25.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:25.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:25.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:25.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:25.577 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:25.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:25.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:25.683 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:25.723 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:25.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:25.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:25.855 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:25.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:25.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:25.962 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:25.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:25.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:25.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:26.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:26.101 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:25 smithi163 bash[75425]: cluster 2023-10-09T09:47:24.102019+0000 mgr.y (mgr.34107) 1242 : cluster 0 pgmap v2078: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 17 MiB/s wr, 699 op/s 2023-10-09T09:47:26.102 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:25 smithi163 bash[75425]: audit 2023-10-09T09:47:25.394143+0000 mon.a (mon.0) 2301 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:26.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:25 smithi163 bash[78107]: cluster 2023-10-09T09:47:24.102019+0000 mgr.y (mgr.34107) 1242 : cluster 0 pgmap v2078: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 17 MiB/s wr, 699 op/s 2023-10-09T09:47:26.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:25 smithi163 bash[78107]: audit 2023-10-09T09:47:25.394143+0000 mon.a (mon.0) 2301 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:26.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:26.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:26.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:26.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:26.231 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:25 smithi181 bash[59345]: cluster 2023-10-09T09:47:24.102019+0000 mgr.y (mgr.34107) 1242 : cluster 0 pgmap v2078: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 17 MiB/s wr, 699 op/s 2023-10-09T09:47:26.231 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:25 smithi181 bash[59345]: audit 2023-10-09T09:47:25.394143+0000 mon.a (mon.0) 2301 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:26.250 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:26.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:26.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:26.385 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:26.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:26.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:26.490 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:26.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:26.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:26.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:26.622 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:26.644 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:26.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:26.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:26.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:26.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:26.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:26.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:26.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:26.905 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:26.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:26.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:27.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:27.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:27.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:27.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:27.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:27.175 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:27.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:27.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:27.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:27.312 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:27.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:27.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:27.416 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:27.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:27.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:27.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:27.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:27.576 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:27.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:27.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:27.679 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:27.708 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:27.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:27.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:27.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:27.848 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:27.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:27.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:27.952 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:27.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:27.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:27.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:28.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:27 smithi181 bash[59345]: cluster 2023-10-09T09:47:26.102517+0000 mgr.y (mgr.34107) 1243 : cluster 0 pgmap v2079: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 12 MiB/s wr, 701 op/s 2023-10-09T09:47:28.092 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:28.126 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:28.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:28.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:28.208 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:47:28.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:27 smithi163 bash[75425]: cluster 2023-10-09T09:47:26.102517+0000 mgr.y (mgr.34107) 1243 : cluster 0 pgmap v2079: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 12 MiB/s wr, 701 op/s 2023-10-09T09:47:28.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:27 smithi163 bash[78107]: cluster 2023-10-09T09:47:26.102517+0000 mgr.y (mgr.34107) 1243 : cluster 0 pgmap v2079: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 12 MiB/s wr, 701 op/s 2023-10-09T09:47:28.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:28.256 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:47:28.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:28.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:28.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:28.304 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:47:28.347 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:47:28.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:28.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:28.412 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:47:28.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:28.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:28.453 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:47:28.489 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:47:28.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:28.534 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:47:28.537 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:47:28.537 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:47:28.537 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:47:28.537 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:47:28.537 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:47:28.538 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:47:28.538 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:47:28.538 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:47:28.549 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:28.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:28.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:28.557 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:47:28.559 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:47:28.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:28.681 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:28.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:28.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:28.809 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:28.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:28.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:28.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:28 smithi163 bash[75425]: audit 2023-10-09T09:47:28.170423+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.163:0/3805058200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:47:28.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:28 smithi163 bash[75425]: audit 2023-10-09T09:47:28.576328+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.163:0/1371743060' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:47:28.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:28 smithi163 bash[75425]: audit 2023-10-09T09:47:28.578053+0000 mon.a (mon.0) 2302 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:47:28.907 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:47:28.912 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:28.941 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:28.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:28.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:29.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:29.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:28 smithi181 bash[59345]: audit 2023-10-09T09:47:28.170423+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.163:0/3805058200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:47:29.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:28 smithi181 bash[59345]: audit 2023-10-09T09:47:28.576328+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.163:0/1371743060' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:47:29.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:28 smithi181 bash[59345]: audit 2023-10-09T09:47:28.578053+0000 mon.a (mon.0) 2302 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:47:29.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:29.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:28 smithi163 bash[78107]: audit 2023-10-09T09:47:28.170423+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.163:0/3805058200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:47:29.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:28 smithi163 bash[78107]: audit 2023-10-09T09:47:28.576328+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.163:0/1371743060' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:47:29.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:28 smithi163 bash[78107]: audit 2023-10-09T09:47:28.578053+0000 mon.a (mon.0) 2302 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:47:29.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:29.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:29.185 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:29.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:47:29.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:29.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:47:29.234 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:47:29.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:29.349 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:29.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:29.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:29.456 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:29.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:29.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:29.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:29.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:29.623 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:29.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:29.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:29.725 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:29.759 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:29.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:29.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:29.866 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:29.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:29.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:29.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:30.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:30.035 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:30.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:30.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:30.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:30.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:29 smithi181 bash[59345]: cluster 2023-10-09T09:47:28.103159+0000 mgr.y (mgr.34107) 1244 : cluster 0 pgmap v2080: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 7.5 MiB/s wr, 849 op/s 2023-10-09T09:47:30.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:29 smithi181 bash[59345]: audit 2023-10-09T09:47:28.534730+0000 mgr.y (mgr.34107) 1245 : audit 0 from='client.35646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:47:30.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:29 smithi181 bash[59345]: cluster 2023-10-09T09:47:28.874616+0000 mon.a (mon.0) 2303 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:47:30.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:29 smithi181 bash[59345]: cluster 2023-10-09T09:47:28.885853+0000 osd.5 (osd.5) 828 : cluster 3 failed to encode map e1241 with expected crc 2023-10-09T09:47:30.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:29 smithi181 bash[59345]: audit 2023-10-09T09:47:28.890159+0000 mon.a (mon.0) 2304 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:47:30.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:29 smithi181 bash[59345]: cluster 2023-10-09T09:47:28.890399+0000 mon.a (mon.0) 2305 : cluster 0 osdmap e1241: 8 total, 8 up, 6 in 2023-10-09T09:47:30.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:29 smithi181 bash[59345]: cluster 2023-10-09T09:47:28.895696+0000 osd.4 (osd.4) 643 : cluster 3 failed to encode map e1241 with expected crc 2023-10-09T09:47:30.148 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:29 smithi181 bash[59345]: cluster 2023-10-09T09:47:28.899949+0000 osd.4 (osd.4) 644 : cluster 3 failed to encode map e1241 with expected crc 2023-10-09T09:47:30.174 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:30.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:30.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:30.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[78107]: cluster 2023-10-09T09:47:28.103159+0000 mgr.y (mgr.34107) 1244 : cluster 0 pgmap v2080: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 7.5 MiB/s wr, 849 op/s 2023-10-09T09:47:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[78107]: audit 2023-10-09T09:47:28.534730+0000 mgr.y (mgr.34107) 1245 : audit 0 from='client.35646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:47:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[78107]: cluster 2023-10-09T09:47:28.874616+0000 mon.a (mon.0) 2303 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:47:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[78107]: cluster 2023-10-09T09:47:28.885853+0000 osd.5 (osd.5) 828 : cluster 3 failed to encode map e1241 with expected crc 2023-10-09T09:47:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[78107]: audit 2023-10-09T09:47:28.890159+0000 mon.a (mon.0) 2304 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:47:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[78107]: cluster 2023-10-09T09:47:28.890399+0000 mon.a (mon.0) 2305 : cluster 0 osdmap e1241: 8 total, 8 up, 6 in 2023-10-09T09:47:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[78107]: cluster 2023-10-09T09:47:28.895696+0000 osd.4 (osd.4) 643 : cluster 3 failed to encode map e1241 with expected crc 2023-10-09T09:47:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[78107]: cluster 2023-10-09T09:47:28.899949+0000 osd.4 (osd.4) 644 : cluster 3 failed to encode map e1241 with expected crc 2023-10-09T09:47:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[75425]: cluster 2023-10-09T09:47:28.103159+0000 mgr.y (mgr.34107) 1244 : cluster 0 pgmap v2080: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 7.5 MiB/s wr, 849 op/s 2023-10-09T09:47:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[75425]: audit 2023-10-09T09:47:28.534730+0000 mgr.y (mgr.34107) 1245 : audit 0 from='client.35646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:47:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[75425]: cluster 2023-10-09T09:47:28.874616+0000 mon.a (mon.0) 2303 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:47:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[75425]: cluster 2023-10-09T09:47:28.885853+0000 osd.5 (osd.5) 828 : cluster 3 failed to encode map e1241 with expected crc 2023-10-09T09:47:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[75425]: audit 2023-10-09T09:47:28.890159+0000 mon.a (mon.0) 2304 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:47:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[75425]: cluster 2023-10-09T09:47:28.890399+0000 mon.a (mon.0) 2305 : cluster 0 osdmap e1241: 8 total, 8 up, 6 in 2023-10-09T09:47:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[75425]: cluster 2023-10-09T09:47:28.895696+0000 osd.4 (osd.4) 643 : cluster 3 failed to encode map e1241 with expected crc 2023-10-09T09:47:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:29 smithi163 bash[75425]: cluster 2023-10-09T09:47:28.899949+0000 osd.4 (osd.4) 644 : cluster 3 failed to encode map e1241 with expected crc 2023-10-09T09:47:30.280 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:30.305 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:30.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:30.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:30.408 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:30.435 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:30.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:30.540 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:30.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:30.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:30.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:30.668 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:30.692 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:30.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:30.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:30.799 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:30.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:30.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:30.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:30.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:30.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:30.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:31.016 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2080,"stamp":"2023-10-09T09:47:28.102701+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6599481272,"num_objects":100716,"num_object_clones":0,"num_object_copies":302148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100716,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":118470,"num_write_kb":7007246,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":592828,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106804,"ondisk_log_size":106804,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":24086184,"kb_used_data":19403896,"kb_used_omap":510,"kb_used_meta":4681537,"kb_avail":538384728,"statfs":{"total":575970213888,"available":551305961472,"internally_reserved":0,"allocated":19869589504,"data_stored":19864473798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":522608,"internal_metadata":4793894544},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,4,4,6,15,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-479920104,"num_objects":-7322,"num_object_clones":0,"num_object_copies":-21966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7322,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10196,"num_write_kb":91905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003448"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":38,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155209+0000","last_change":"2023-10-09T09:43:23.993342+0000","last_active":"2023-10-09T09:46:39.155209+0000","last_peered":"2023-10-09T09:46:39.155209+0000","last_clean":"2023-10-09T09:46:39.155209+0000","last_became_active":"2023-10-09T09:43:23.990938+0000","last_became_peered":"2023-10-09T09:43:23.990938+0000","last_unstale":"2023-10-09T09:46:39.155209+0000","last_undegraded":"2023-10-09T09:46:39.155209+0000","last_fullsized":"2023-10-09T09:46:39.155209+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:23.141315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":123,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155355+0000","last_change":"2023-10-09T09:41:11.811997+0000","last_active":"2023-10-09T09:46:39.155355+0000","last_peered":"2023-10-09T09:46:39.155355+0000","last_clean":"2023-10-09T09:46:39.155355+0000","last_became_active":"2023-10-09T09:41:11.811300+0000","last_became_peered":"2023-10-09T09:41:11.811300+0000","last_unstale":"2023-10-09T09:46:39.155355+0000","last_undegraded":"2023-10-09T09:46:39.155355+0000","last_fullsized":"2023-10-09T09:46:39.155355+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.346300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1192,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.157003+0000","last_change":"2023-10-09T09:41:11.811870+0000","last_active":"2023-10-09T09:46:39.157003+0000","last_peered":"2023-10-09T09:46:39.157003+0000","last_clean":"2023-10-09T09:46:39.157003+0000","last_became_active":"2023-10-09T09:41:11.811570+0000","last_became_peered":"2023-10-09T09:41:11.811570+0000","last_unstale":"2023-10-09T09:46:39.157003+0000","last_undegraded":"2023-10-09T09:46:39.157003+0000","last_fullsized":"2023-10-09T09:46:39.157003+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:57.923729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1183,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.911166+0000","last_change":"2023-10-09T09:41:11.800661+0000","last_active":"2023-10-09T09:47:01.911166+0000","last_peered":"2023-10-09T09:47:01.911166+0000","last_clean":"2023-10-09T09:47:01.911166+0000","last_became_active":"2023-10-09T09:41:11.800279+0000","last_became_peered":"2023-10-09T09:41:11.800279+0000","last_unstale":"2023-10-09T09:47:01.911166+0000","last_undegraded":"2023-10-09T09:47:01.911166+0000","last_fullsized":"2023-10-09T09:47:01.911166+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:14:19.131995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":717,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.891167+0000","last_change":"2023-10-09T09:41:11.801649+0000","last_active":"2023-10-09T09:47:01.891167+0000","last_peered":"2023-10-09T09:47:01.891167+0000","last_clean":"2023-10-09T09:47:01.891167+0000","last_became_active":"2023-10-09T09:41:11.798585+0000","last_became_peered":"2023-10-09T09:41:11.798585+0000","last_unstale":"2023-10-09T09:47:01.891167+0000","last_undegraded":"2023-10-09T09:47:01.891167+0000","last_fullsized":"2023-10-09T09:47:01.891167+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:11.450331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1189,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.162302+0000","last_change":"2023-10-09T09:40:25.799705+0000","last_active":"2023-10-09T09:46:39.162302+0000","last_peered":"2023-10-09T09:46:39.162302+0000","last_clean":"2023-10-09T09:46:39.162302+0000","last_became_active":"2023-10-09T09:40:25.799341+0000","last_became_peered":"2023-10-09T09:40:25.799341+0000","last_unstale":"2023-10-09T09:46:39.162302+0000","last_undegraded":"2023-10-09T09:46:39.162302+0000","last_fullsized":"2023-10-09T09:46:39.162302+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1163,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.158026+0000","last_change":"2023-10-09T09:43:48.902066+0000","last_active":"2023-10-09T09:46:39.158026+0000","last_peered":"2023-10-09T09:46:39.158026+0000","last_clean":"2023-10-09T09:46:39.158026+0000","last_became_active":"2023-10-09T09:43:48.901639+0000","last_became_peered":"2023-10-09T09:43:48.901639+0000","last_unstale":"2023-10-09T09:46:39.158026+0000","last_undegraded":"2023-10-09T09:46:39.158026+0000","last_fullsized":"2023-10-09T09:46:39.158026+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":634,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155235+0000","last_change":"2023-10-09T09:41:11.814521+0000","last_active":"2023-10-09T09:46:39.155235+0000","last_peered":"2023-10-09T09:46:39.155235+0000","last_clean":"2023-10-09T09:46:39.155235+0000","last_became_active":"2023-10-09T09:41:11.814148+0000","last_became_peered":"2023-10-09T09:41:11.814148+0000","last_unstale":"2023-10-09T09:46:39.155235+0000","last_undegraded":"2023-10-09T09:46:39.155235+0000","last_fullsized":"2023-10-09T09:46:39.155235+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:45.497593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":91,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.157555+0000","last_change":"2023-10-09T09:41:11.833334+0000","last_active":"2023-10-09T09:46:39.157555+0000","last_peered":"2023-10-09T09:46:39.157555+0000","last_clean":"2023-10-09T09:46:39.157555+0000","last_became_active":"2023-10-09T09:41:11.832509+0000","last_became_peered":"2023-10-09T09:41:11.832509+0000","last_unstale":"2023-10-09T09:46:39.157555+0000","last_undegraded":"2023-10-09T09:46:39.157555+0000","last_fullsized":"2023-10-09T09:46:39.157555+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":655,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.162043+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:46:39.162043+0000","last_peered":"2023-10-09T09:46:39.162043+0000","last_clean":"2023-10-09T09:46:39.162043+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:46:39.162043+0000","last_undegraded":"2023-10-09T09:46:39.162043+0000","last_fullsized":"2023-10-09T09:46:39.162043+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":48,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.162345+0000","last_change":"2023-10-09T09:43:48.906785+0000","last_active":"2023-10-09T09:46:39.162345+0000","last_peered":"2023-10-09T09:46:39.162345+0000","last_clean":"2023-10-09T09:46:39.162345+0000","last_became_active":"2023-10-09T09:43:48.906276+0000","last_became_peered":"2023-10-09T09:43:48.906276+0000","last_unstale":"2023-10-09T09:46:39.162345+0000","last_undegraded":"2023-10-09T09:46:39.162345+0000","last_fullsized":"2023-10-09T09:46:39.162345+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":72,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.160126+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:46:39.160126+0000","last_peered":"2023-10-09T09:46:39.160126+0000","last_clean":"2023-10-09T09:46:39.160126+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:46:39.160126+0000","last_undegraded":"2023-10-09T09:46:39.160126+0000","last_fullsized":"2023-10-09T09:46:39.160126+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":396,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.164606+0000","last_change":"2023-10-09T09:43:23.995526+0000","last_active":"2023-10-09T09:46:39.164606+0000","last_peered":"2023-10-09T09:46:39.164606+0000","last_clean":"2023-10-09T09:46:39.164606+0000","last_became_active":"2023-10-09T09:43:23.994816+0000","last_became_peered":"2023-10-09T09:43:23.994816+0000","last_unstale":"2023-10-09T09:46:39.164606+0000","last_undegraded":"2023-10-09T09:46:39.164606+0000","last_fullsized":"2023-10-09T09:46:39.164606+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":598,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.162071+0000","last_change":"2023-10-09T09:43:23.995814+0000","last_active":"2023-10-09T09:46:39.162071+0000","last_peered":"2023-10-09T09:46:39.162071+0000","last_clean":"2023-10-09T09:46:39.162071+0000","last_became_active":"2023-10-09T09:43:23.995492+0000","last_became_peered":"2023-10-09T09:43:23.995492+0000","last_unstale":"2023-10-09T09:46:39.162071+0000","last_undegraded":"2023-10-09T09:46:39.162071+0000","last_fullsized":"2023-10-09T09:46:39.162071+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1133,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.160758+0000","last_change":"2023-10-09T09:43:23.989356+0000","last_active":"2023-10-09T09:46:39.160758+0000","last_peered":"2023-10-09T09:46:39.160758+0000","last_clean":"2023-10-09T09:46:39.160758+0000","last_became_active":"2023-10-09T09:43:23.988986+0000","last_became_peered":"2023-10-09T09:43:23.988986+0000","last_unstale":"2023-10-09T09:46:39.160758+0000","last_undegraded":"2023-10-09T09:46:39.160758+0000","last_fullsized":"2023-10-09T09:46:39.160758+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.570448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":719,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155617+0000","last_change":"2023-10-09T09:41:11.826293+0000","last_active":"2023-10-09T09:46:39.155617+0000","last_peered":"2023-10-09T09:46:39.155617+0000","last_clean":"2023-10-09T09:46:39.155617+0000","last_became_active":"2023-10-09T09:41:11.825808+0000","last_became_peered":"2023-10-09T09:41:11.825808+0000","last_unstale":"2023-10-09T09:46:39.155617+0000","last_undegraded":"2023-10-09T09:46:39.155617+0000","last_fullsized":"2023-10-09T09:46:39.155617+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:33:14.204593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1155,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155352+0000","last_change":"2023-10-09T09:43:48.904973+0000","last_active":"2023-10-09T09:46:39.155352+0000","last_peered":"2023-10-09T09:46:39.155352+0000","last_clean":"2023-10-09T09:46:39.155352+0000","last_became_active":"2023-10-09T09:43:48.904557+0000","last_became_peered":"2023-10-09T09:43:48.904557+0000","last_unstale":"2023-10-09T09:46:39.155352+0000","last_undegraded":"2023-10-09T09:46:39.155352+0000","last_fullsized":"2023-10-09T09:46:39.155352+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:36.528234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":76,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.162486+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:46:39.162486+0000","last_peered":"2023-10-09T09:46:39.162486+0000","last_clean":"2023-10-09T09:46:39.162486+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:46:39.162486+0000","last_undegraded":"2023-10-09T09:46:39.162486+0000","last_fullsized":"2023-10-09T09:46:39.162486+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":40,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.161906+0000","last_change":"2023-10-09T09:43:23.994784+0000","last_active":"2023-10-09T09:46:39.161906+0000","last_peered":"2023-10-09T09:46:39.161906+0000","last_clean":"2023-10-09T09:46:39.161906+0000","last_became_active":"2023-10-09T09:43:23.994196+0000","last_became_peered":"2023-10-09T09:43:23.994196+0000","last_unstale":"2023-10-09T09:46:39.161906+0000","last_undegraded":"2023-10-09T09:46:39.161906+0000","last_fullsized":"2023-10-09T09:46:39.161906+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1154,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.167951+0000","last_change":"2023-10-09T09:43:48.901303+0000","last_active":"2023-10-09T09:46:39.167951+0000","last_peered":"2023-10-09T09:46:39.167951+0000","last_clean":"2023-10-09T09:46:39.167951+0000","last_became_active":"2023-10-09T09:43:48.900779+0000","last_became_peered":"2023-10-09T09:43:48.900779+0000","last_unstale":"2023-10-09T09:46:39.167951+0000","last_undegraded":"2023-10-09T09:46:39.167951+0000","last_fullsized":"2023-10-09T09:46:39.167951+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:51.853531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1234,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.161400+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:46:39.161400+0000","last_peered":"2023-10-09T09:46:39.161400+0000","last_clean":"2023-10-09T09:46:39.161400+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:46:39.161400+0000","last_undegraded":"2023-10-09T09:46:39.161400+0000","last_fullsized":"2023-10-09T09:46:39.161400+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1202,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.156928+0000","last_change":"2023-10-09T09:41:11.805335+0000","last_active":"2023-10-09T09:46:39.156928+0000","last_peered":"2023-10-09T09:46:39.156928+0000","last_clean":"2023-10-09T09:46:39.156928+0000","last_became_active":"2023-10-09T09:41:11.804963+0000","last_became_peered":"2023-10-09T09:41:11.804963+0000","last_unstale":"2023-10-09T09:46:39.156928+0000","last_undegraded":"2023-10-09T09:46:39.156928+0000","last_fullsized":"2023-10-09T09:46:39.156928+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:09.265362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.6","version":"1240'7287","reported_seq":7304,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:27.889424+0000","last_change":"2023-10-09T09:45:20.779324+0000","last_active":"2023-10-09T09:47:27.889424+0000","last_peered":"2023-10-09T09:47:27.889424+0000","last_clean":"2023-10-09T09:47:27.889424+0000","last_became_active":"2023-10-09T09:45:20.778629+0000","last_became_peered":"2023-10-09T09:45:20.778629+0000","last_unstale":"2023-10-09T09:47:27.889424+0000","last_undegraded":"2023-10-09T09:47:27.889424+0000","last_fullsized":"2023-10-09T09:47:27.889424+0000","mapping_epoch":1232,"log_start":"1235'300","ondisk_log_start":"1235'300","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:50.507235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397475840,"num_objects":6065,"num_object_clones":0,"num_object_copies":18195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7287,"num_write_kb":427264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":49,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.166332+0000","last_change":"2023-10-09T09:43:23.993830+0000","last_active":"2023-10-09T09:46:39.166332+0000","last_peered":"2023-10-09T09:46:39.166332+0000","last_clean":"2023-10-09T09:46:39.166332+0000","last_became_active":"2023-10-09T09:43:23.993431+0000","last_became_peered":"2023-10-09T09:43:23.993431+0000","last_unstale":"2023-10-09T09:46:39.166332+0000","last_undegraded":"2023-10-09T09:46:39.166332+0000","last_fullsized":"2023-10-09T09:46:39.166332+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:27.775152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.154133+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:46:39.154133+0000","last_peered":"2023-10-09T09:46:39.154133+0000","last_clean":"2023-10-09T09:46:39.154133+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:46:39.154133+0000","last_undegraded":"2023-10-09T09:46:39.154133+0000","last_fullsized":"2023-10-09T09:46:39.154133+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"1240'7408","reported_seq":7424,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:26.583628+0000","last_change":"2023-10-09T09:45:19.872001+0000","last_active":"2023-10-09T09:47:26.583628+0000","last_peered":"2023-10-09T09:47:26.583628+0000","last_clean":"2023-10-09T09:47:26.583628+0000","last_became_active":"2023-10-09T09:45:19.871533+0000","last_became_peered":"2023-10-09T09:45:19.871533+0000","last_unstale":"2023-10-09T09:47:26.583628+0000","last_undegraded":"2023-10-09T09:47:26.583628+0000","last_fullsized":"2023-10-09T09:47:26.583628+0000","mapping_epoch":1232,"log_start":"1235'1000","ondisk_log_start":"1235'1000","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413138944,"num_objects":6304,"num_object_clones":0,"num_object_copies":18912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7408,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":49,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.156919+0000","last_change":"2023-10-09T09:43:23.992053+0000","last_active":"2023-10-09T09:46:39.156919+0000","last_peered":"2023-10-09T09:46:39.156919+0000","last_clean":"2023-10-09T09:46:39.156919+0000","last_became_active":"2023-10-09T09:43:23.991695+0000","last_became_peered":"2023-10-09T09:43:23.991695+0000","last_unstale":"2023-10-09T09:46:39.156919+0000","last_undegraded":"2023-10-09T09:46:39.156919+0000","last_fullsized":"2023-10-09T09:46:39.156919+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:18.704618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1193,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.157097+0000","last_change":"2023-10-09T09:43:23.995477+0000","last_active":"2023-10-09T09:46:39.157097+0000","last_peered":"2023-10-09T09:46:39.157097+0000","last_clean":"2023-10-09T09:46:39.157097+0000","last_became_active":"2023-10-09T09:43:23.995178+0000","last_became_peered":"2023-10-09T09:43:23.995178+0000","last_unstale":"2023-10-09T09:46:39.157097+0000","last_undegraded":"2023-10-09T09:46:39.157097+0000","last_fullsized":"2023-10-09T09:46:39.157097+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:22:20.575488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"1240'7394","reported_seq":7412,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:27.918785+0000","last_change":"2023-10-09T09:45:20.778840+0000","last_active":"2023-10-09T09:47:27.918785+0000","last_peered":"2023-10-09T09:47:27.918785+0000","last_clean":"2023-10-09T09:47:27.918785+0000","last_became_active":"2023-10-09T09:45:20.778306+0000","last_became_peered":"2023-10-09T09:45:20.778306+0000","last_unstale":"2023-10-09T09:47:27.918785+0000","last_undegraded":"2023-10-09T09:47:27.918785+0000","last_fullsized":"2023-10-09T09:47:27.918785+0000","mapping_epoch":1232,"log_start":"1235'400","ondisk_log_start":"1235'400","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6994,"log_dups_size":0,"ondisk_log_size":6994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:32.098494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403308544,"num_objects":6154,"num_object_clones":0,"num_object_copies":18462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7394,"num_write_kb":433536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1190,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.157527+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:46:39.157527+0000","last_peered":"2023-10-09T09:46:39.157527+0000","last_clean":"2023-10-09T09:46:39.157527+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:46:39.157527+0000","last_undegraded":"2023-10-09T09:46:39.157527+0000","last_fullsized":"2023-10-09T09:46:39.157527+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1181,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.162055+0000","last_change":"2023-10-09T09:40:49.793789+0000","last_active":"2023-10-09T09:46:39.162055+0000","last_peered":"2023-10-09T09:46:39.162055+0000","last_clean":"2023-10-09T09:46:39.162055+0000","last_became_active":"2023-10-09T09:40:49.793374+0000","last_became_peered":"2023-10-09T09:40:49.793374+0000","last_unstale":"2023-10-09T09:46:39.162055+0000","last_undegraded":"2023-10-09T09:46:39.162055+0000","last_fullsized":"2023-10-09T09:46:39.162055+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"1240'7353","reported_seq":7370,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:25.340747+0000","last_change":"2023-10-09T09:45:19.877007+0000","last_active":"2023-10-09T09:47:25.340747+0000","last_peered":"2023-10-09T09:47:25.340747+0000","last_clean":"2023-10-09T09:47:25.340747+0000","last_became_active":"2023-10-09T09:45:19.876605+0000","last_became_peered":"2023-10-09T09:45:19.876605+0000","last_unstale":"2023-10-09T09:47:25.340747+0000","last_undegraded":"2023-10-09T09:47:25.340747+0000","last_fullsized":"2023-10-09T09:47:25.340747+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":7353,"log_dups_size":0,"ondisk_log_size":7353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:49.148260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413990912,"num_objects":6317,"num_object_clones":0,"num_object_copies":18951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7353,"num_write_kb":437440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":727,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.164641+0000","last_change":"2023-10-09T09:41:11.815662+0000","last_active":"2023-10-09T09:46:39.164641+0000","last_peered":"2023-10-09T09:46:39.164641+0000","last_clean":"2023-10-09T09:46:39.164641+0000","last_became_active":"2023-10-09T09:41:11.815203+0000","last_became_peered":"2023-10-09T09:41:11.815203+0000","last_unstale":"2023-10-09T09:46:39.164641+0000","last_undegraded":"2023-10-09T09:46:39.164641+0000","last_fullsized":"2023-10-09T09:46:39.164641+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1157,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.166820+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:46:39.166820+0000","last_peered":"2023-10-09T09:46:39.166820+0000","last_clean":"2023-10-09T09:46:39.166820+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:46:39.166820+0000","last_undegraded":"2023-10-09T09:46:39.166820+0000","last_fullsized":"2023-10-09T09:46:39.166820+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"1240'7191","reported_seq":7208,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:23.397110+0000","last_change":"2023-10-09T09:45:20.775729+0000","last_active":"2023-10-09T09:47:23.397110+0000","last_peered":"2023-10-09T09:47:23.397110+0000","last_clean":"2023-10-09T09:47:23.397110+0000","last_became_active":"2023-10-09T09:45:20.775410+0000","last_became_peered":"2023-10-09T09:45:20.775410+0000","last_unstale":"2023-10-09T09:47:23.397110+0000","last_undegraded":"2023-10-09T09:47:23.397110+0000","last_fullsized":"2023-10-09T09:47:23.397110+0000","mapping_epoch":1232,"log_start":"1235'1400","ondisk_log_start":"1235'1400","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":5791,"log_dups_size":0,"ondisk_log_size":5791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:31.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414121984,"num_objects":6319,"num_object_clones":0,"num_object_copies":18957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7191,"num_write_kb":432320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":134,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155644+0000","last_change":"2023-10-09T09:43:48.904982+0000","last_active":"2023-10-09T09:46:39.155644+0000","last_peered":"2023-10-09T09:46:39.155644+0000","last_clean":"2023-10-09T09:46:39.155644+0000","last_became_active":"2023-10-09T09:43:48.904605+0000","last_became_peered":"2023-10-09T09:43:48.904605+0000","last_unstale":"2023-10-09T09:46:39.155644+0000","last_undegraded":"2023-10-09T09:46:39.155644+0000","last_fullsized":"2023-10-09T09:46:39.155644+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:14.912690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1177,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.164588+0000","last_change":"2023-10-09T09:43:23.996674+0000","last_active":"2023-10-09T09:46:39.164588+0000","last_peered":"2023-10-09T09:46:39.164588+0000","last_clean":"2023-10-09T09:46:39.164588+0000","last_became_active":"2023-10-09T09:43:23.995573+0000","last_became_peered":"2023-10-09T09:43:23.995573+0000","last_unstale":"2023-10-09T09:46:39.164588+0000","last_undegraded":"2023-10-09T09:46:39.164588+0000","last_fullsized":"2023-10-09T09:46:39.164588+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"1240'7565","reported_seq":7582,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:25.334898+0000","last_change":"2023-10-09T09:45:20.774813+0000","last_active":"2023-10-09T09:47:25.334898+0000","last_peered":"2023-10-09T09:47:25.334898+0000","last_clean":"2023-10-09T09:47:25.334898+0000","last_became_active":"2023-10-09T09:45:20.774338+0000","last_became_peered":"2023-10-09T09:45:20.774338+0000","last_unstale":"2023-10-09T09:47:25.334898+0000","last_undegraded":"2023-10-09T09:47:25.334898+0000","last_fullsized":"2023-10-09T09:47:25.334898+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":7565,"log_dups_size":0,"ondisk_log_size":7565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:17.139271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426704896,"num_objects":6511,"num_object_clones":0,"num_object_copies":19533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7565,"num_write_kb":450432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1173,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.161870+0000","last_change":"2023-10-09T09:14:40.494701+0000","last_active":"2023-10-09T09:46:39.161870+0000","last_peered":"2023-10-09T09:46:39.161870+0000","last_clean":"2023-10-09T09:46:39.161870+0000","last_became_active":"2023-10-09T09:14:40.494316+0000","last_became_peered":"2023-10-09T09:14:40.494316+0000","last_unstale":"2023-10-09T09:46:39.161870+0000","last_undegraded":"2023-10-09T09:46:39.161870+0000","last_fullsized":"2023-10-09T09:46:39.161870+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1162,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.160667+0000","last_change":"2023-10-09T09:43:23.991689+0000","last_active":"2023-10-09T09:46:39.160667+0000","last_peered":"2023-10-09T09:46:39.160667+0000","last_clean":"2023-10-09T09:46:39.160667+0000","last_became_active":"2023-10-09T09:43:23.991355+0000","last_became_peered":"2023-10-09T09:43:23.991355+0000","last_unstale":"2023-10-09T09:46:39.160667+0000","last_undegraded":"2023-10-09T09:46:39.160667+0000","last_fullsized":"2023-10-09T09:46:39.160667+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:57.061947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"1240'7390","reported_seq":7406,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:26.554613+0000","last_change":"2023-10-09T09:45:20.775861+0000","last_active":"2023-10-09T09:47:26.554613+0000","last_peered":"2023-10-09T09:47:26.554613+0000","last_clean":"2023-10-09T09:47:26.554613+0000","last_became_active":"2023-10-09T09:45:20.775171+0000","last_became_peered":"2023-10-09T09:45:20.775171+0000","last_unstale":"2023-10-09T09:47:26.554613+0000","last_undegraded":"2023-10-09T09:47:26.554613+0000","last_fullsized":"2023-10-09T09:47:26.554613+0000","mapping_epoch":1232,"log_start":"1235'1000","ondisk_log_start":"1235'1000","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6390,"log_dups_size":0,"ondisk_log_size":6390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406585344,"num_objects":6204,"num_object_clones":0,"num_object_copies":18612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7390,"num_write_kb":435008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1190,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.160739+0000","last_change":"2023-10-09T09:43:23.992509+0000","last_active":"2023-10-09T09:46:39.160739+0000","last_peered":"2023-10-09T09:46:39.160739+0000","last_clean":"2023-10-09T09:46:39.160739+0000","last_became_active":"2023-10-09T09:43:23.992076+0000","last_became_peered":"2023-10-09T09:43:23.992076+0000","last_unstale":"2023-10-09T09:46:39.160739+0000","last_undegraded":"2023-10-09T09:46:39.160739+0000","last_fullsized":"2023-10-09T09:46:39.160739+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:12.832688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":598,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.163060+0000","last_change":"2023-10-09T09:43:24.052308+0000","last_active":"2023-10-09T09:46:39.163060+0000","last_peered":"2023-10-09T09:46:39.163060+0000","last_clean":"2023-10-09T09:46:39.163060+0000","last_became_active":"2023-10-09T09:43:24.051972+0000","last_became_peered":"2023-10-09T09:43:24.051972+0000","last_unstale":"2023-10-09T09:46:39.163060+0000","last_undegraded":"2023-10-09T09:46:39.163060+0000","last_fullsized":"2023-10-09T09:46:39.163060+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"1240'7373","reported_seq":7390,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:23.386183+0000","last_change":"2023-10-09T09:45:20.778819+0000","last_active":"2023-10-09T09:47:23.386183+0000","last_peered":"2023-10-09T09:47:23.386183+0000","last_clean":"2023-10-09T09:47:23.386183+0000","last_became_active":"2023-10-09T09:45:20.778497+0000","last_became_peered":"2023-10-09T09:45:20.778497+0000","last_unstale":"2023-10-09T09:47:23.386183+0000","last_undegraded":"2023-10-09T09:47:23.386183+0000","last_fullsized":"2023-10-09T09:47:23.386183+0000","mapping_epoch":1232,"log_start":"1235'1600","ondisk_log_start":"1235'1600","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":5773,"log_dups_size":0,"ondisk_log_size":5773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:51.173183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425394176,"num_objects":6491,"num_object_clones":0,"num_object_copies":19473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7373,"num_write_kb":443648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1151,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155400+0000","last_change":"2023-10-09T09:41:11.814910+0000","last_active":"2023-10-09T09:46:39.155400+0000","last_peered":"2023-10-09T09:46:39.155400+0000","last_clean":"2023-10-09T09:46:39.155400+0000","last_became_active":"2023-10-09T09:41:11.814561+0000","last_became_peered":"2023-10-09T09:41:11.814561+0000","last_unstale":"2023-10-09T09:46:39.155400+0000","last_undegraded":"2023-10-09T09:46:39.155400+0000","last_fullsized":"2023-10-09T09:46:39.155400+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:25:51.140694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.905067+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:47:01.905067+0000","last_peered":"2023-10-09T09:47:01.905067+0000","last_clean":"2023-10-09T09:47:01.905067+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:47:01.905067+0000","last_undegraded":"2023-10-09T09:47:01.905067+0000","last_fullsized":"2023-10-09T09:47:01.905067+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"1240'7225","reported_seq":7241,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:26.570072+0000","last_change":"2023-10-09T09:45:20.775453+0000","last_active":"2023-10-09T09:47:26.570072+0000","last_peered":"2023-10-09T09:47:26.570072+0000","last_clean":"2023-10-09T09:47:26.570072+0000","last_became_active":"2023-10-09T09:45:20.774585+0000","last_became_peered":"2023-10-09T09:45:20.774585+0000","last_unstale":"2023-10-09T09:47:26.570072+0000","last_undegraded":"2023-10-09T09:47:26.570072+0000","last_fullsized":"2023-10-09T09:47:26.570072+0000","mapping_epoch":1232,"log_start":"1235'800","ondisk_log_start":"1235'800","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6425,"log_dups_size":0,"ondisk_log_size":6425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398262272,"num_objects":6077,"num_object_clones":0,"num_object_copies":18231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7225,"num_write_kb":425664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1204,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155954+0000","last_change":"2023-10-09T09:40:49.781045+0000","last_active":"2023-10-09T09:46:39.155954+0000","last_peered":"2023-10-09T09:46:39.155954+0000","last_clean":"2023-10-09T09:46:39.155954+0000","last_became_active":"2023-10-09T09:40:49.780425+0000","last_became_peered":"2023-10-09T09:40:49.780425+0000","last_unstale":"2023-10-09T09:46:39.155954+0000","last_undegraded":"2023-10-09T09:46:39.155954+0000","last_fullsized":"2023-10-09T09:46:39.155954+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:56.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":82,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.157408+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:46:39.157408+0000","last_peered":"2023-10-09T09:46:39.157408+0000","last_clean":"2023-10-09T09:46:39.157408+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:46:39.157408+0000","last_undegraded":"2023-10-09T09:46:39.157408+0000","last_fullsized":"2023-10-09T09:46:39.157408+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1188,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.156145+0000","last_change":"2023-10-09T09:40:25.799081+0000","last_active":"2023-10-09T09:46:39.156145+0000","last_peered":"2023-10-09T09:46:39.156145+0000","last_clean":"2023-10-09T09:46:39.156145+0000","last_became_active":"2023-10-09T09:40:25.798693+0000","last_became_peered":"2023-10-09T09:40:25.798693+0000","last_unstale":"2023-10-09T09:46:39.156145+0000","last_undegraded":"2023-10-09T09:46:39.156145+0000","last_fullsized":"2023-10-09T09:46:39.156145+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:35.527067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"1240'7323","reported_seq":7339,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:26.544093+0000","last_change":"2023-10-09T09:45:19.874686+0000","last_active":"2023-10-09T09:47:26.544093+0000","last_peered":"2023-10-09T09:47:26.544093+0000","last_clean":"2023-10-09T09:47:26.544093+0000","last_became_active":"2023-10-09T09:45:19.874293+0000","last_became_peered":"2023-10-09T09:45:19.874293+0000","last_unstale":"2023-10-09T09:47:26.544093+0000","last_undegraded":"2023-10-09T09:47:26.544093+0000","last_fullsized":"2023-10-09T09:47:26.544093+0000","mapping_epoch":1232,"log_start":"1235'900","ondisk_log_start":"1235'900","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6423,"log_dups_size":0,"ondisk_log_size":6423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407044096,"num_objects":6211,"num_object_clones":0,"num_object_copies":18633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7323,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":671,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.898322+0000","last_change":"2023-10-09T09:40:25.795910+0000","last_active":"2023-10-09T09:47:01.898322+0000","last_peered":"2023-10-09T09:47:01.898322+0000","last_clean":"2023-10-09T09:47:01.898322+0000","last_became_active":"2023-10-09T09:40:25.795327+0000","last_became_peered":"2023-10-09T09:40:25.795327+0000","last_unstale":"2023-10-09T09:47:01.898322+0000","last_undegraded":"2023-10-09T09:47:01.898322+0000","last_fullsized":"2023-10-09T09:47:01.898322+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:58.343181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1130,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.162242+0000","last_change":"2023-10-09T09:43:48.908190+0000","last_active":"2023-10-09T09:46:39.162242+0000","last_peered":"2023-10-09T09:46:39.162242+0000","last_clean":"2023-10-09T09:46:39.162242+0000","last_became_active":"2023-10-09T09:43:48.907799+0000","last_became_peered":"2023-10-09T09:43:48.907799+0000","last_unstale":"2023-10-09T09:46:39.162242+0000","last_undegraded":"2023-10-09T09:46:39.162242+0000","last_fullsized":"2023-10-09T09:46:39.162242+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1227,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155544+0000","last_change":"2023-10-09T09:41:11.811782+0000","last_active":"2023-10-09T09:46:39.155544+0000","last_peered":"2023-10-09T09:46:39.155544+0000","last_clean":"2023-10-09T09:46:39.155544+0000","last_became_active":"2023-10-09T09:41:11.811431+0000","last_became_peered":"2023-10-09T09:41:11.811431+0000","last_unstale":"2023-10-09T09:46:39.155544+0000","last_undegraded":"2023-10-09T09:46:39.155544+0000","last_fullsized":"2023-10-09T09:46:39.155544+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:05.197258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"1240'7349","reported_seq":7366,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:25.316220+0000","last_change":"2023-10-09T09:45:20.779143+0000","last_active":"2023-10-09T09:47:25.316220+0000","last_peered":"2023-10-09T09:47:25.316220+0000","last_clean":"2023-10-09T09:47:25.316220+0000","last_became_active":"2023-10-09T09:45:20.778454+0000","last_became_peered":"2023-10-09T09:45:20.778454+0000","last_unstale":"2023-10-09T09:47:25.316220+0000","last_undegraded":"2023-10-09T09:47:25.316220+0000","last_fullsized":"2023-10-09T09:47:25.316220+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":7349,"log_dups_size":0,"ondisk_log_size":7349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:31.777588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415694848,"num_objects":6343,"num_object_clones":0,"num_object_copies":19029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7349,"num_write_kb":438144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":114,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.899958+0000","last_change":"2023-10-09T09:41:11.801228+0000","last_active":"2023-10-09T09:47:01.899958+0000","last_peered":"2023-10-09T09:47:01.899958+0000","last_clean":"2023-10-09T09:47:01.899958+0000","last_became_active":"2023-10-09T09:41:11.800068+0000","last_became_peered":"2023-10-09T09:41:11.800068+0000","last_unstale":"2023-10-09T09:47:01.899958+0000","last_undegraded":"2023-10-09T09:47:01.899958+0000","last_fullsized":"2023-10-09T09:47:01.899958+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:33.425580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1204,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:48.522304+0000","last_change":"2023-10-09T09:36:43.421868+0000","last_active":"2023-10-09T09:46:48.522304+0000","last_peered":"2023-10-09T09:46:48.522304+0000","last_clean":"2023-10-09T09:46:48.522304+0000","last_became_active":"2023-10-09T09:36:43.421013+0000","last_became_peered":"2023-10-09T09:36:43.421013+0000","last_unstale":"2023-10-09T09:46:48.522304+0000","last_undegraded":"2023-10-09T09:46:48.522304+0000","last_fullsized":"2023-10-09T09:46:48.522304+0000","mapping_epoch":1149,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:43.353667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1208,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.898030+0000","last_change":"2023-10-09T09:40:25.799440+0000","last_active":"2023-10-09T09:47:01.898030+0000","last_peered":"2023-10-09T09:47:01.898030+0000","last_clean":"2023-10-09T09:47:01.898030+0000","last_became_active":"2023-10-09T09:40:25.797967+0000","last_became_peered":"2023-10-09T09:40:25.797967+0000","last_unstale":"2023-10-09T09:47:01.898030+0000","last_undegraded":"2023-10-09T09:47:01.898030+0000","last_fullsized":"2023-10-09T09:47:01.898030+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:51.038572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"1240'7436","reported_seq":7452,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:26.587233+0000","last_change":"2023-10-09T09:45:19.874884+0000","last_active":"2023-10-09T09:47:26.587233+0000","last_peered":"2023-10-09T09:47:26.587233+0000","last_clean":"2023-10-09T09:47:26.587233+0000","last_became_active":"2023-10-09T09:45:19.874339+0000","last_became_peered":"2023-10-09T09:45:19.874339+0000","last_unstale":"2023-10-09T09:47:26.587233+0000","last_undegraded":"2023-10-09T09:47:26.587233+0000","last_fullsized":"2023-10-09T09:47:26.587233+0000","mapping_epoch":1232,"log_start":"1235'1000","ondisk_log_start":"1235'1000","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6436,"log_dups_size":0,"ondisk_log_size":6436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411697152,"num_objects":6282,"num_object_clones":0,"num_object_copies":18846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7436,"num_write_kb":438976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1247,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:47:00.313417+0000","last_change":"2023-10-09T09:41:11.795802+0000","last_active":"2023-10-09T09:47:00.313417+0000","last_peered":"2023-10-09T09:47:00.313417+0000","last_clean":"2023-10-09T09:47:00.313417+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:47:00.313417+0000","last_undegraded":"2023-10-09T09:47:00.313417+0000","last_fullsized":"2023-10-09T09:47:00.313417+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":565,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:52.854136+0000","last_change":"2023-10-09T09:40:52.800168+0000","last_active":"2023-10-09T09:46:52.854136+0000","last_peered":"2023-10-09T09:46:52.854136+0000","last_clean":"2023-10-09T09:46:52.854136+0000","last_became_active":"2023-10-09T09:40:52.799543+0000","last_became_peered":"2023-10-09T09:40:52.799543+0000","last_unstale":"2023-10-09T09:46:52.854136+0000","last_undegraded":"2023-10-09T09:46:52.854136+0000","last_fullsized":"2023-10-09T09:46:52.854136+0000","mapping_epoch":1185,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:44.481003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":82,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.160092+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:46:39.160092+0000","last_peered":"2023-10-09T09:46:39.160092+0000","last_clean":"2023-10-09T09:46:39.160092+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:46:39.160092+0000","last_undegraded":"2023-10-09T09:46:39.160092+0000","last_fullsized":"2023-10-09T09:46:39.160092+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1166,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.162545+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:46:39.162545+0000","last_peered":"2023-10-09T09:46:39.162545+0000","last_clean":"2023-10-09T09:46:39.162545+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:46:39.162545+0000","last_undegraded":"2023-10-09T09:46:39.162545+0000","last_fullsized":"2023-10-09T09:46:39.162545+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"1240'7452","reported_seq":7469,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:25.285864+0000","last_change":"2023-10-09T09:45:19.876941+0000","last_active":"2023-10-09T09:47:25.285864+0000","last_peered":"2023-10-09T09:47:25.285864+0000","last_clean":"2023-10-09T09:47:25.285864+0000","last_became_active":"2023-10-09T09:45:19.876658+0000","last_became_peered":"2023-10-09T09:45:19.876658+0000","last_unstale":"2023-10-09T09:47:25.285864+0000","last_undegraded":"2023-10-09T09:47:25.285864+0000","last_fullsized":"2023-10-09T09:47:25.285864+0000","mapping_epoch":1232,"log_start":"0'0","ondisk_log_start":"0'0","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":7452,"log_dups_size":0,"ondisk_log_size":7452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:44:33.624841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419692544,"num_objects":6404,"num_object_clones":0,"num_object_copies":19212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7452,"num_write_kb":443392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1171,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:53.528661+0000","last_change":"2023-10-09T09:41:11.815645+0000","last_active":"2023-10-09T09:46:53.528661+0000","last_peered":"2023-10-09T09:46:53.528661+0000","last_clean":"2023-10-09T09:46:53.528661+0000","last_became_active":"2023-10-09T09:41:11.815086+0000","last_became_peered":"2023-10-09T09:41:11.815086+0000","last_unstale":"2023-10-09T09:46:53.528661+0000","last_undegraded":"2023-10-09T09:46:53.528661+0000","last_fullsized":"2023-10-09T09:46:53.528661+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:29.870212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1174,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.164745+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:46:39.164745+0000","last_peered":"2023-10-09T09:46:39.164745+0000","last_clean":"2023-10-09T09:46:39.164745+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:46:39.164745+0000","last_undegraded":"2023-10-09T09:46:39.164745+0000","last_fullsized":"2023-10-09T09:46:39.164745+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1171,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.166049+0000","last_change":"2023-10-09T09:39:12.949268+0000","last_active":"2023-10-09T09:46:39.166049+0000","last_peered":"2023-10-09T09:46:39.166049+0000","last_clean":"2023-10-09T09:46:39.166049+0000","last_became_active":"2023-10-09T09:39:12.948924+0000","last_became_peered":"2023-10-09T09:39:12.948924+0000","last_unstale":"2023-10-09T09:46:39.166049+0000","last_undegraded":"2023-10-09T09:46:39.166049+0000","last_fullsized":"2023-10-09T09:46:39.166049+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:31.368540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"1240'7443","reported_seq":7460,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:26.582054+0000","last_change":"2023-10-09T09:45:20.775001+0000","last_active":"2023-10-09T09:47:26.582054+0000","last_peered":"2023-10-09T09:47:26.582054+0000","last_clean":"2023-10-09T09:47:26.582054+0000","last_became_active":"2023-10-09T09:45:20.774247+0000","last_became_peered":"2023-10-09T09:45:20.774247+0000","last_unstale":"2023-10-09T09:47:26.582054+0000","last_undegraded":"2023-10-09T09:47:26.582054+0000","last_fullsized":"2023-10-09T09:47:26.582054+0000","mapping_epoch":1232,"log_start":"1235'1000","ondisk_log_start":"1235'1000","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6443,"log_dups_size":0,"ondisk_log_size":6443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414318616,"num_objects":6323,"num_object_clones":0,"num_object_copies":18969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6323,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7443,"num_write_kb":440449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1147,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.908699+0000","last_change":"2023-10-09T09:40:49.787387+0000","last_active":"2023-10-09T09:47:01.908699+0000","last_peered":"2023-10-09T09:47:01.908699+0000","last_clean":"2023-10-09T09:47:01.908699+0000","last_became_active":"2023-10-09T09:40:49.786954+0000","last_became_peered":"2023-10-09T09:40:49.786954+0000","last_unstale":"2023-10-09T09:47:01.908699+0000","last_undegraded":"2023-10-09T09:47:01.908699+0000","last_fullsized":"2023-10-09T09:47:01.908699+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:51.055735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1222,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155829+0000","last_change":"2023-10-09T09:40:25.797470+0000","last_active":"2023-10-09T09:46:39.155829+0000","last_peered":"2023-10-09T09:46:39.155829+0000","last_clean":"2023-10-09T09:46:39.155829+0000","last_became_active":"2023-10-09T09:40:25.796966+0000","last_became_peered":"2023-10-09T09:40:25.796966+0000","last_unstale":"2023-10-09T09:46:39.155829+0000","last_undegraded":"2023-10-09T09:46:39.155829+0000","last_fullsized":"2023-10-09T09:46:39.155829+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:24.526350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1141,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.154369+0000","last_change":"2023-10-09T09:43:48.907449+0000","last_active":"2023-10-09T09:46:39.154369+0000","last_peered":"2023-10-09T09:46:39.154369+0000","last_clean":"2023-10-09T09:46:39.154369+0000","last_became_active":"2023-10-09T09:43:48.905674+0000","last_became_peered":"2023-10-09T09:43:48.905674+0000","last_unstale":"2023-10-09T09:46:39.154369+0000","last_undegraded":"2023-10-09T09:46:39.154369+0000","last_fullsized":"2023-10-09T09:46:39.154369+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"1240'7534","reported_seq":7550,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:26.561865+0000","last_change":"2023-10-09T09:45:20.778840+0000","last_active":"2023-10-09T09:47:26.561865+0000","last_peered":"2023-10-09T09:47:26.561865+0000","last_clean":"2023-10-09T09:47:26.561865+0000","last_became_active":"2023-10-09T09:45:20.778207+0000","last_became_peered":"2023-10-09T09:45:20.778207+0000","last_unstale":"2023-10-09T09:47:26.561865+0000","last_undegraded":"2023-10-09T09:47:26.561865+0000","last_fullsized":"2023-10-09T09:47:26.561865+0000","mapping_epoch":1232,"log_start":"1235'1100","ondisk_log_start":"1235'1100","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6434,"log_dups_size":0,"ondisk_log_size":6434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417857536,"num_objects":6376,"num_object_clones":0,"num_object_copies":19128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7534,"num_write_kb":445120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":100,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.917219+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:47:01.917219+0000","last_peered":"2023-10-09T09:47:01.917219+0000","last_clean":"2023-10-09T09:47:01.917219+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:47:01.917219+0000","last_undegraded":"2023-10-09T09:47:01.917219+0000","last_fullsized":"2023-10-09T09:47:01.917219+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1169,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:01.898091+0000","last_change":"2023-10-09T09:14:40.501275+0000","last_active":"2023-10-09T09:47:01.898091+0000","last_peered":"2023-10-09T09:47:01.898091+0000","last_clean":"2023-10-09T09:47:01.898091+0000","last_became_active":"2023-10-09T09:14:40.500972+0000","last_became_peered":"2023-10-09T09:14:40.500972+0000","last_unstale":"2023-10-09T09:47:01.898091+0000","last_undegraded":"2023-10-09T09:47:01.898091+0000","last_fullsized":"2023-10-09T09:47:01.898091+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:11.595564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1111,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.154806+0000","last_change":"2023-10-09T09:43:48.906737+0000","last_active":"2023-10-09T09:46:39.154806+0000","last_peered":"2023-10-09T09:46:39.154806+0000","last_clean":"2023-10-09T09:46:39.154806+0000","last_became_active":"2023-10-09T09:43:48.906410+0000","last_became_peered":"2023-10-09T09:43:48.906410+0000","last_unstale":"2023-10-09T09:46:39.154806+0000","last_undegraded":"2023-10-09T09:46:39.154806+0000","last_fullsized":"2023-10-09T09:46:39.154806+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:24:10.024977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.f","version":"1240'7487","reported_seq":7503,"reported_epoch":1240,"state":"active+clean","last_fresh":"2023-10-09T09:47:26.524741+0000","last_change":"2023-10-09T09:45:20.774871+0000","last_active":"2023-10-09T09:47:26.524741+0000","last_peered":"2023-10-09T09:47:26.524741+0000","last_clean":"2023-10-09T09:47:26.524741+0000","last_became_active":"2023-10-09T09:45:20.774349+0000","last_became_peered":"2023-10-09T09:45:20.774349+0000","last_unstale":"2023-10-09T09:47:26.524741+0000","last_undegraded":"2023-10-09T09:47:26.524741+0000","last_fullsized":"2023-10-09T09:47:26.524741+0000","mapping_epoch":1232,"log_start":"1235'1100","ondisk_log_start":"1235'1100","created":1232,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:18.836642+0000","last_clean_scrub_stamp":"2023-10-09T09:45:18.836642+0000","objects_scrubbed":0,"log_size":6387,"log_dups_size":0,"ondisk_log_size":6387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413597696,"num_objects":6311,"num_object_clones":0,"num_object_copies":18933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7487,"num_write_kb":441536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1212,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.162150+0000","last_change":"2023-10-09T09:41:11.815721+0000","last_active":"2023-10-09T09:46:39.162150+0000","last_peered":"2023-10-09T09:46:39.162150+0000","last_clean":"2023-10-09T09:46:39.162150+0000","last_became_active":"2023-10-09T09:41:11.815237+0000","last_became_peered":"2023-10-09T09:41:11.815237+0000","last_unstale":"2023-10-09T09:46:39.162150+0000","last_undegraded":"2023-10-09T09:46:39.162150+0000","last_fullsized":"2023-10-09T09:46:39.162150+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1152,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.166223+0000","last_change":"2023-10-09T09:35:18.685030+0000","last_active":"2023-10-09T09:46:39.166223+0000","last_peered":"2023-10-09T09:46:39.166223+0000","last_clean":"2023-10-09T09:46:39.166223+0000","last_became_active":"2023-10-09T09:35:18.675810+0000","last_became_peered":"2023-10-09T09:35:18.675810+0000","last_unstale":"2023-10-09T09:46:39.166223+0000","last_undegraded":"2023-10-09T09:46:39.166223+0000","last_fullsized":"2023-10-09T09:46:39.166223+0000","mapping_epoch":1136,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:52.793405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1158,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.154596+0000","last_change":"2023-10-09T09:43:23.992213+0000","last_active":"2023-10-09T09:46:39.154596+0000","last_peered":"2023-10-09T09:46:39.154596+0000","last_clean":"2023-10-09T09:46:39.154596+0000","last_became_active":"2023-10-09T09:43:23.991887+0000","last_became_peered":"2023-10-09T09:43:23.991887+0000","last_unstale":"2023-10-09T09:46:39.154596+0000","last_undegraded":"2023-10-09T09:46:39.154596+0000","last_fullsized":"2023-10-09T09:46:39.154596+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1160,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.167008+0000","last_change":"2023-10-09T09:40:49.788328+0000","last_active":"2023-10-09T09:46:39.167008+0000","last_peered":"2023-10-09T09:46:39.167008+0000","last_clean":"2023-10-09T09:46:39.167008+0000","last_became_active":"2023-10-09T09:40:49.787927+0000","last_became_peered":"2023-10-09T09:40:49.787927+0000","last_unstale":"2023-10-09T09:46:39.167008+0000","last_undegraded":"2023-10-09T09:46:39.167008+0000","last_fullsized":"2023-10-09T09:46:39.167008+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:32.616104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":113,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:45.295203+0000","last_change":"2023-10-09T09:41:11.800443+0000","last_active":"2023-10-09T09:46:45.295203+0000","last_peered":"2023-10-09T09:46:45.295203+0000","last_clean":"2023-10-09T09:46:45.295203+0000","last_became_active":"2023-10-09T09:41:11.799864+0000","last_became_peered":"2023-10-09T09:41:11.799864+0000","last_unstale":"2023-10-09T09:46:45.295203+0000","last_undegraded":"2023-10-09T09:46:45.295203+0000","last_fullsized":"2023-10-09T09:46:45.295203+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:48:22.481162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":49,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155753+0000","last_change":"2023-10-09T09:43:23.992016+0000","last_active":"2023-10-09T09:46:39.155753+0000","last_peered":"2023-10-09T09:46:39.155753+0000","last_clean":"2023-10-09T09:46:39.155753+0000","last_became_active":"2023-10-09T09:43:23.991400+0000","last_became_peered":"2023-10-09T09:43:23.991400+0000","last_unstale":"2023-10-09T09:46:39.155753+0000","last_undegraded":"2023-10-09T09:46:39.155753+0000","last_fullsized":"2023-10-09T09:46:39.155753+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.209599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1193,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.155834+0000","last_change":"2023-10-09T09:43:23.994636+0000","last_active":"2023-10-09T09:46:39.155834+0000","last_peered":"2023-10-09T09:46:39.155834+0000","last_clean":"2023-10-09T09:46:39.155834+0000","last_became_active":"2023-10-09T09:43:23.993694+0000","last_became_peered":"2023-10-09T09:43:23.993694+0000","last_unstale":"2023-10-09T09:46:39.155834+0000","last_undegraded":"2023-10-09T09:46:39.155834+0000","last_fullsized":"2023-10-09T09:46:39.155834+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:45.895845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":48,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.162649+0000","last_change":"2023-10-09T09:43:23.995576+0000","last_active":"2023-10-09T09:46:39.162649+0000","last_peered":"2023-10-09T09:46:39.162649+0000","last_clean":"2023-10-09T09:46:39.162649+0000","last_became_active":"2023-10-09T09:43:23.995076+0000","last_became_peered":"2023-10-09T09:43:23.995076+0000","last_unstale":"2023-10-09T09:46:39.162649+0000","last_undegraded":"2023-10-09T09:46:39.162649+0000","last_fullsized":"2023-10-09T09:46:39.162649+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1207,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.160458+0000","last_change":"2023-10-09T09:43:48.904021+0000","last_active":"2023-10-09T09:46:39.160458+0000","last_peered":"2023-10-09T09:46:39.160458+0000","last_clean":"2023-10-09T09:46:39.160458+0000","last_became_active":"2023-10-09T09:43:48.903719+0000","last_became_peered":"2023-10-09T09:43:48.903719+0000","last_unstale":"2023-10-09T09:46:39.160458+0000","last_undegraded":"2023-10-09T09:46:39.160458+0000","last_fullsized":"2023-10-09T09:46:39.160458+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:16.284027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":173,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.163157+0000","last_change":"2023-10-09T09:41:11.816169+0000","last_active":"2023-10-09T09:46:39.163157+0000","last_peered":"2023-10-09T09:46:39.163157+0000","last_clean":"2023-10-09T09:46:39.163157+0000","last_became_active":"2023-10-09T09:41:11.815426+0000","last_became_peered":"2023-10-09T09:41:11.815426+0000","last_unstale":"2023-10-09T09:46:39.163157+0000","last_undegraded":"2023-10-09T09:46:39.163157+0000","last_fullsized":"2023-10-09T09:46:39.163157+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.160558+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:46:39.160558+0000","last_peered":"2023-10-09T09:46:39.160558+0000","last_clean":"2023-10-09T09:46:39.160558+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:46:39.160558+0000","last_undegraded":"2023-10-09T09:46:39.160558+0000","last_fullsized":"2023-10-09T09:46:39.160558+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1200,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.166221+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:46:39.166221+0000","last_peered":"2023-10-09T09:46:39.166221+0000","last_clean":"2023-10-09T09:46:39.166221+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:46:39.166221+0000","last_undegraded":"2023-10-09T09:46:39.166221+0000","last_fullsized":"2023-10-09T09:46:39.166221+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":598,"reported_epoch":1239,"state":"active+clean","last_fresh":"2023-10-09T09:46:39.161556+0000","last_change":"2023-10-09T09:43:24.086244+0000","last_active":"2023-10-09T09:46:39.161556+0000","last_peered":"2023-10-09T09:46:39.161556+0000","last_clean":"2023-10-09T09:46:39.161556+0000","last_became_active":"2023-10-09T09:43:24.085797+0000","last_became_peered":"2023-10-09T09:43:24.085797+0000","last_unstale":"2023-10-09T09:46:39.161556+0000","last_undegraded":"2023-10-09T09:46:39.161556+0000","last_fullsized":"2023-10-09T09:46:39.161556+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6598885400,"num_objects":100692,"num_object_clones":0,"num_object_copies":302076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100692,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":118210,"num_write_kb":7004801,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19751907328,"data_stored":19751895112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106610,"ondisk_log_size":106610,"up":48,"acting":48,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":2460,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28685,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453398068,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594301,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4117660,"kb_used_data":3216916,"kb_used_omap":0,"kb_used_meta":900672,"kb_avail":89627492,"statfs":{"total":95995035648,"available":91778551808,"internally_reserved":0,"allocated":3294121984,"data_stored":3293841913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922288128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2,0,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:46:38 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":1.2709999999999999,"15min":1.3979999999999999},"min":{"1min":0.245,"5min":0.224,"15min":0.224},"max":{"1min":24.265999999999998,"5min":24.265999999999998,"15min":24.265999999999998},"last":4.2549999999999999},{"interface":"front","average":{"1min":2.3900000000000001,"5min":1.637,"15min":1.639},"min":{"1min":0.34899999999999998,"5min":0.32300000000000001,"15min":0.28699999999999998},"max":{"1min":24.376999999999999,"5min":24.376999999999999,"15min":24.376999999999999},"last":0.32200000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:46:38 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":4.4770000000000003,"15min":3.3599999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.22900000000000001},"max":{"1min":3.8039999999999998,"5min":116.953,"15min":116.953},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.794,"5min":4.6369999999999996,"15min":3.2149999999999999},"min":{"1min":0.313,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":12.032999999999999,"5min":117.459,"15min":117.459},"last":5.0289999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3239999999999998,"5min":2.2599999999999998,"15min":1.976},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":7.6079999999999997,"5min":46.466000000000001,"15min":46.466000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.8580000000000001,"5min":2.3220000000000001,"15min":2.0950000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.29499999999999998},"max":{"1min":7.6369999999999996,"5min":47.151000000000003,"15min":47.151000000000003},"last":0.57899999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.1120000000000001,"15min":1.7010000000000001},"min":{"1min":0.33400000000000002,"5min":0.33200000000000002,"15min":0.309},"max":{"1min":3.8959999999999999,"5min":6.0880000000000001,"15min":68.686999999999998},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.26,"15min":1.621},"min":{"1min":0.39000000000000001,"5min":0.372,"15min":0.29499999999999998},"max":{"1min":11.968999999999999,"5min":11.968999999999999,"15min":69.108999999999995},"last":4.9109999999999996}]},{"osd":4,"last update":"Mon Oct 9 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.4330000000000001,"15min":1.2190000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28199999999999997},"max":{"1min":7.0270000000000001,"5min":7.0270000000000001,"15min":8.6890000000000001},"last":1.7849999999999999},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.4299999999999999,"15min":1.373},"min":{"1min":0.313,"5min":0.313,"15min":0.29499999999999998},"max":{"1min":12.064,"5min":12.064,"15min":12.064},"last":4.9809999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":1.4890000000000001,"15min":1.345},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.056,"5min":7.056,"15min":9.0559999999999992},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.0059999999999998,"5min":1.49,"15min":1.458},"min":{"1min":0.44800000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.000999999999999,"5min":12.000999999999999,"15min":12.000999999999999},"last":4.9530000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:46:57 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":1.5089999999999999,"15min":1.3440000000000001},"min":{"1min":0.45200000000000001,"5min":0.41099999999999998,"15min":0.35599999999999998},"max":{"1min":4.1280000000000001,"5min":11.922000000000001,"15min":11.922000000000001},"last":5.0049999999999999},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.163,"15min":1.2470000000000001},"min":{"1min":0.40699999999999997,"5min":0.371,"15min":0.33000000000000002},"max":{"1min":4.8399999999999999,"5min":5.1920000000000002,"15min":7.6879999999999997},"last":0.59599999999999997}]}]},{"osd":2,"up_from":634,"seq":2723009266054,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3597092,"kb_used_data":2892892,"kb_used_omap":122,"kb_used_meta":704069,"kb_avail":90148060,"statfs":{"total":95995035648,"available":92311613440,"internally_reserved":0,"allocated":2962321408,"data_stored":2960875472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":720967243},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:46:40 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":2.7200000000000002,"15min":2.6589999999999998},"min":{"1min":0.248,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.0230000000000001,"5min":105.29600000000001,"15min":105.29600000000001},"last":140.63300000000001},{"interface":"front","average":{"1min":1.365,"5min":3.1299999999999999,"15min":2.9319999999999999},"min":{"1min":0.253,"5min":0.22900000000000001,"15min":0.20399999999999999},"max":{"1min":5.3159999999999998,"5min":105.33799999999999,"15min":105.33799999999999},"last":139.887}]},{"osd":1,"last update":"Mon Oct 9 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":3.0499999999999998,"15min":2.9180000000000001},"min":{"1min":0.27200000000000002,"5min":0.224,"15min":0.20200000000000001},"max":{"1min":9.9390000000000001,"5min":64.700999999999993,"15min":81.662999999999997},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.872,"5min":3.2290000000000001,"15min":2.9620000000000002},"min":{"1min":0.26400000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.0059999999999998,"5min":64.334000000000003,"15min":81.712999999999994},"last":0.34000000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:46:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":0.83299999999999996,"15min":2.3479999999999999},"min":{"1min":0.27700000000000002,"5min":0.26700000000000002,"15min":0.252},"max":{"1min":4.1429999999999998,"5min":5.181,"15min":230.476},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.83199999999999996,"15min":2.302},"min":{"1min":0.34399999999999997,"5min":0.27000000000000002,"15min":0.26200000000000001},"max":{"1min":1.962,"5min":6.8540000000000001,"15min":228.893},"last":0.51100000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0289999999999999,"15min":1.327},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.30099999999999999},"max":{"1min":2.077,"5min":6.8920000000000003,"15min":11.622999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.3420000000000001,"5min":1.3069999999999999,"15min":1.429},"min":{"1min":0.38300000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.7169999999999996,"5min":7.6369999999999996,"15min":10.394},"last":0.45800000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.155,"15min":1.405},"min":{"1min":0.48899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.1890000000000001,"5min":4.1890000000000001,"15min":9.7599999999999998},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.2170000000000001,"15min":1.478},"min":{"1min":0.499,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.9160000000000004,"5min":6.9450000000000003,"15min":11.718999999999999},"last":0.441}]},{"osd":6,"last update":"Mon Oct 9 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":1.397,"15min":1.7030000000000001},"min":{"1min":0.46000000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.7370000000000001,"5min":8.4299999999999997,"15min":11.648},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.4490000000000001,"5min":1.423,"15min":1.522},"min":{"1min":0.54900000000000004,"5min":0.37,"15min":0.33300000000000002},"max":{"1min":4.8849999999999998,"5min":7.6029999999999998,"15min":10.436999999999999},"last":0.66100000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:46:26 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":1.2969999999999999,"15min":1.546},"min":{"1min":0.48999999999999999,"5min":0.42499999999999999,"15min":0.39400000000000002},"max":{"1min":4.758,"5min":7.8550000000000004,"15min":10.462999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.0700000000000001,"15min":1.4850000000000001},"min":{"1min":0.52200000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":4.2199999999999998,"5min":4.2199999999999998,"15min":9.7759999999999998},"last":0.502}]}]},{"osd":1,"up_from":607,"seq":2607045149067,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2842264,"kb_used_data":2059152,"kb_used_omap":107,"kb_used_meta":782996,"kb_avail":90902888,"statfs":{"total":95995035648,"available":93084557312,"internally_reserved":0,"allocated":2108571648,"data_stored":2107131774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":109909,"internal_metadata":801788587},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:46:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":4.0830000000000002,"15min":2.3239999999999998},"min":{"1min":0.28000000000000003,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":21.923999999999999,"5min":134.49100000000001,"15min":134.49100000000001},"last":2.1619999999999999},{"interface":"front","average":{"1min":2.3740000000000001,"5min":4.1459999999999999,"15min":2.8620000000000001},"min":{"1min":0.29599999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":21.966000000000001,"5min":134.226,"15min":134.226},"last":6.0110000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.47,"15min":2.036},"min":{"1min":0.26000000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":3.6539999999999999,"5min":28.170999999999999,"15min":69.421999999999997},"last":5.9409999999999998},{"interface":"front","average":{"1min":2.3610000000000002,"5min":1.863,"15min":1.855},"min":{"1min":0.314,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":5.8280000000000003,"5min":29.548999999999999,"15min":71.879000000000005},"last":1.0009999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:46:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":1.159,"15min":1.724},"min":{"1min":0.28100000000000003,"5min":0.253,"15min":0.23599999999999999},"max":{"1min":4.4880000000000004,"5min":8.3729999999999993,"15min":73.564999999999998},"last":2.198},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.1399999999999999,"15min":2.0070000000000001},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.27900000000000003},"max":{"1min":1.8999999999999999,"5min":8.0099999999999998,"15min":72.932000000000002},"last":5.9740000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.4259999999999999,"15min":1.2929999999999999},"min":{"1min":0.52900000000000003,"5min":0.30599999999999999,"15min":0.28299999999999997},"max":{"1min":6.4400000000000004,"5min":7.6900000000000004,"15min":7.6900000000000004},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.099,"5min":1.4339999999999999,"15min":1.4530000000000001},"min":{"1min":0.36699999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.226,"5min":8.3249999999999993,"15min":32.151000000000003},"last":3.5310000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.5840000000000001,"15min":1.4079999999999999},"min":{"1min":0.40799999999999997,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.8660000000000001,"5min":7.7800000000000002,"15min":8.6869999999999994},"last":2.9119999999999999},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.6000000000000001,"15min":1.4019999999999999},"min":{"1min":0.434,"5min":0.34100000000000003,"15min":0.311},"max":{"1min":4.2699999999999996,"5min":9.0190000000000001,"15min":9.0190000000000001},"last":2.3769999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.504,"15min":1.716},"min":{"1min":0.44600000000000001,"5min":0.38800000000000001,"15min":0.33600000000000002},"max":{"1min":2.2050000000000001,"5min":7.9400000000000004,"15min":14.335000000000001},"last":6.3579999999999997},{"interface":"front","average":{"1min":1.165,"5min":1.619,"15min":1.5189999999999999},"min":{"1min":0.61799999999999999,"5min":0.372,"15min":0.36799999999999999},"max":{"1min":2.891,"5min":12.128,"15min":12.128},"last":0.40000000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.3520000000000001,"15min":1.6759999999999999},"min":{"1min":0.47799999999999998,"5min":0.44400000000000001,"15min":0.36199999999999999},"max":{"1min":2.2410000000000001,"5min":7.9770000000000003,"15min":14.364000000000001},"last":6.4240000000000004},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.371,"15min":1.6040000000000001},"min":{"1min":0.57699999999999996,"5min":0.41699999999999998,"15min":0.39600000000000002},"max":{"1min":2.9260000000000002,"5min":7.8899999999999997,"15min":32.078000000000003},"last":0.95999999999999996}]}]},{"osd":0,"up_from":1136,"seq":4879082848403,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4456432,"kb_used_data":3963944,"kb_used_omap":280,"kb_used_meta":492135,"kb_avail":89288720,"statfs":{"total":95995035648,"available":91431649280,"internally_reserved":0,"allocated":4059078656,"data_stored":4057669791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":503946650},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.671,"15min":1.593},"min":{"1min":0.374,"5min":0.247,"15min":0.184},"max":{"1min":6.3170000000000002,"5min":10.851000000000001,"15min":33.634},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.7989999999999999,"15min":1.722},"min":{"1min":0.33400000000000002,"5min":0.25,"15min":0.19700000000000001},"max":{"1min":5.7439999999999998,"5min":11.114000000000001,"15min":33.661000000000001},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":5.4640000000000004,"5min":2.121,"15min":1.5229999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":48.795000000000002,"5min":48.795000000000002,"15min":48.795000000000002},"last":4.7389999999999999},{"interface":"front","average":{"1min":6.4690000000000003,"5min":2.516,"15min":1.786},"min":{"1min":0.34999999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":48.856000000000002,"5min":48.856000000000002,"15min":48.856000000000002},"last":13.840999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.1759999999999999,"15min":1.2889999999999999},"min":{"1min":0.34699999999999998,"5min":0.27500000000000002,"15min":0.247},"max":{"1min":5.3979999999999997,"5min":10.475,"15min":10.475},"last":0.72299999999999998},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.2909999999999999,"15min":1.248},"min":{"1min":0.374,"5min":0.26500000000000001,"15min":0.248},"max":{"1min":6.5220000000000002,"5min":6.6550000000000002,"15min":9.2400000000000002},"last":0.878}]},{"osd":4,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":1.292,"15min":1.2849999999999999},"min":{"1min":0.38,"5min":0.314,"15min":0.27900000000000003},"max":{"1min":5.7030000000000003,"5min":7.7599999999999998,"15min":7.7599999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.3520000000000001,"15min":1.4199999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30199999999999999},"max":{"1min":12.044,"5min":12.044,"15min":12.044},"last":0.92000000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":1.4770000000000001,"15min":1.407},"min":{"1min":0.47699999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.827,"5min":6.8600000000000003,"15min":6.915},"last":4.6859999999999999},{"interface":"front","average":{"1min":2.0289999999999999,"5min":1.665,"15min":1.6080000000000001},"min":{"1min":0.41099999999999998,"5min":0.38200000000000001,"15min":0.29699999999999999},"max":{"1min":6.5979999999999999,"5min":8.3510000000000009,"15min":8.3510000000000009},"last":4.7140000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.603,"15min":1.3939999999999999},"min":{"1min":0.432,"5min":0.35199999999999998,"15min":0.315},"max":{"1min":6.0419999999999998,"5min":10.413,"15min":10.413},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.8,"5min":1.6659999999999999,"15min":1.758},"min":{"1min":0.51600000000000001,"5min":0.443,"15min":0.35599999999999998},"max":{"1min":5.9939999999999998,"5min":6.9790000000000001,"15min":8.625},"last":0.80700000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.367,"15min":1.429},"min":{"1min":0.50600000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.9740000000000002,"5min":6.9909999999999997,"15min":9.2159999999999993},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.7749999999999999,"5min":1.454,"15min":1.5940000000000001},"min":{"1min":0.71899999999999997,"5min":0.443,"15min":0.39200000000000002},"max":{"1min":5.8860000000000001,"5min":6.7679999999999998,"15min":14.361000000000001},"last":0.76400000000000001}]}]},{"osd":3,"up_from":0,"seq":2847563317632,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986766,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5346012,"kb_used_data":4450772,"kb_used_omap":0,"kb_used_meta":895168,"kb_avail":88399140,"statfs":{"total":95995035648,"available":90520719360,"internally_reserved":0,"allocated":4557590528,"data_stored":4557309935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916652032},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,1,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:46:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0600000000000001,"5min":1.415,"15min":1.296},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.253},"max":{"1min":7.8239999999999998,"5min":9.5109999999999992,"15min":16.408999999999999},"last":1.8},{"interface":"front","average":{"1min":1.351,"5min":1.1639999999999999,"15min":1.3560000000000001},"min":{"1min":0.308,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.7869999999999999,"5min":9.5399999999999991,"15min":16.387},"last":1.7869999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.353,"15min":2.1389999999999998},"min":{"1min":0.32300000000000001,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":3.2280000000000002,"5min":25.422000000000001,"15min":85.186999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.0249999999999999,"5min":1.8999999999999999,"15min":2.5990000000000002},"min":{"1min":0.33000000000000002,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":5.923,"5min":25.956,"15min":85.159000000000006},"last":0.314}]},{"osd":2,"last update":"Mon Oct 9 09:47:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":1.54,"15min":1.45},"min":{"1min":0.34000000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":12.000999999999999,"5min":12.000999999999999,"15min":33.743000000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.4510000000000001,"5min":1.4350000000000001,"15min":1.5609999999999999},"min":{"1min":0.32400000000000001,"5min":0.309,"15min":0.30099999999999999},"max":{"1min":6.7809999999999997,"5min":6.7809999999999997,"15min":33.805999999999997},"last":0.35199999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":0.97199999999999998,"15min":1.6100000000000001},"min":{"1min":0.34899999999999998,"5min":0.34200000000000003,"15min":0.27600000000000002},"max":{"1min":4.7469999999999999,"5min":5.734,"15min":47.32},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.032,"5min":1.365,"15min":2.1890000000000001},"min":{"1min":0.35999999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.3879999999999999,"5min":6.9950000000000001,"15min":93.843000000000004},"last":0.38900000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":1.6220000000000001,"15min":1.4259999999999999},"min":{"1min":0.41699999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.7670000000000003,"5min":7.5910000000000002,"15min":17.212},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.208,"5min":1.375,"15min":1.4470000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.8319999999999999,"5min":6.0330000000000004,"15min":17.268999999999998},"last":0.54200000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":2.7959999999999998,"5min":1.5780000000000001,"15min":1.401},"min":{"1min":0.45500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.1070000000000002,"5min":9.6859999999999999,"15min":13.654},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.6899999999999999,"5min":1.6299999999999999,"15min":1.427},"min":{"1min":0.64700000000000002,"5min":0.39200000000000002,"15min":0.35699999999999998},"max":{"1min":8.6010000000000009,"5min":8.6010000000000009,"15min":9.3369999999999997},"last":0.49399999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":1.4690000000000001,"15min":1.3089999999999999},"min":{"1min":0.438,"5min":0.376,"15min":0.376},"max":{"1min":11.840999999999999,"5min":11.840999999999999,"15min":11.840999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.417,"5min":1.0820000000000001,"15min":1.238},"min":{"1min":0.41399999999999998,"5min":0.40699999999999997,"15min":0.34999999999999998},"max":{"1min":8.0760000000000005,"5min":8.0760000000000005,"15min":12.34},"last":0.47099999999999997}]}]},{"osd":5,"up_from":37,"seq":158913790532,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3726724,"kb_used_data":2820220,"kb_used_omap":0,"kb_used_meta":906496,"kb_avail":90018428,"statfs":{"total":95995035648,"available":92178870272,"internally_reserved":0,"allocated":2887905280,"data_stored":2887644913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928251904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:46:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":5.6449999999999996,"15min":5.3780000000000001},"min":{"1min":0.27700000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.992,"5min":274.71899999999999,"15min":274.71899999999999},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":5.6360000000000001,"15min":5.032},"min":{"1min":0.29099999999999998,"5min":0.28199999999999997,"15min":0.245},"max":{"1min":2.5739999999999998,"5min":274.73500000000001,"15min":274.73500000000001},"last":0.34300000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":1.8100000000000001,"15min":3.7629999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.28299999999999997},"max":{"1min":7.7679999999999998,"5min":40.503,"15min":142.67500000000001},"last":2.2000000000000002},{"interface":"front","average":{"1min":1.448,"5min":1.877,"15min":3.8330000000000002},"min":{"1min":0.42799999999999999,"5min":0.39500000000000002,"15min":0.33500000000000002},"max":{"1min":7.7990000000000004,"5min":40.415999999999997,"15min":142.56399999999999},"last":2.2360000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":2.3860000000000001,"15min":1.6970000000000001},"min":{"1min":0.38600000000000001,"5min":0.36299999999999999,"15min":0.31},"max":{"1min":48.170999999999999,"5min":50.847000000000001,"15min":50.847000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":4.1619999999999999,"5min":2.5019999999999998,"15min":1.921},"min":{"1min":0.33400000000000002,"5min":0.30099999999999999,"15min":0.28000000000000003},"max":{"1min":48.128,"5min":50.656999999999996,"15min":50.656999999999996},"last":0.66000000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.2490000000000001,"15min":1.7190000000000001},"min":{"1min":0.374,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":1.6819999999999999,"5min":6.3470000000000004,"15min":59.652000000000001},"last":2.298},{"interface":"front","average":{"1min":1.413,"5min":1.046,"15min":1.8009999999999999},"min":{"1min":0.48199999999999998,"5min":0.33200000000000002,"15min":0.29099999999999998},"max":{"1min":4.7839999999999998,"5min":6.4930000000000003,"15min":61.771999999999998},"last":0.621}]},{"osd":4,"last update":"Mon Oct 9 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.3999999999999999,"15min":1.1599999999999999},"min":{"1min":0.33700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.2669999999999995,"5min":9.2669999999999995,"15min":9.2669999999999995},"last":2.3370000000000002},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.4199999999999999,"15min":1.446},"min":{"1min":0.40799999999999997,"5min":0.34399999999999997,"15min":0.33300000000000002},"max":{"1min":5.891,"5min":7.718,"15min":12.151},"last":4.593}]},{"osd":6,"last update":"Mon Oct 9 09:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":2.6840000000000002,"15min":1.7809999999999999},"min":{"1min":0.45800000000000002,"5min":0.34399999999999997,"15min":0.30399999999999999},"max":{"1min":6.1239999999999997,"5min":140.179,"15min":140.179},"last":2.367},{"interface":"front","average":{"1min":1.8440000000000001,"5min":2.6019999999999999,"15min":2.04},"min":{"1min":0.45000000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.1909999999999998,"5min":137.928,"15min":137.928},"last":0.70499999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:46:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.234,"15min":1.472},"min":{"1min":0.432,"5min":0.373,"15min":0.33400000000000002},"max":{"1min":5.6790000000000003,"5min":6.5279999999999996,"15min":18.683},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.96399999999999997,"15min":1.24},"min":{"1min":0.46300000000000002,"5min":0.38800000000000001,"15min":0.32100000000000001},"max":{"1min":2.609,"5min":6.5460000000000003,"15min":10.192},"last":0.41799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5464,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4040101888,"data_stored":4040097816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2096824320,"data_stored":2096824320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2933129216,"data_stored":2933129216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4517531648,"data_stored":4517527576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2882011136,"data_stored":2882011136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3282309120,"data_stored":3282305048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:47:31.023 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T09:47:31.023 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:47:31.023 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:47:31.023 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T09:47:31.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T09:47:31.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:31.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:31.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:31.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:31.218 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:31 smithi163 systemd[1]: Stopping Ceph osd.1 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T09:47:31.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:31.259 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:31.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:31.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:31.362 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:31.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:31.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:31.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:31.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:31.510 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:31 smithi163 bash[86800]: debug 2023-10-09T09:47:31.230+0000 7f655db8c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:47:31.510 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:31 smithi163 bash[86800]: debug 2023-10-09T09:47:31.230+0000 7f655db8c700 -1 osd.1 1241 *** Got signal Terminated *** 2023-10-09T09:47:31.510 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:31 smithi163 bash[86800]: debug 2023-10-09T09:47:31.230+0000 7f655db8c700 -1 osd.1 1241 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:47:31.530 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:31.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:31.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:31.636 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:31.664 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:31.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:31.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:31.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:31.805 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:31.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:31.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:31.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:31.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:31.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:31.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:32.039 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:32.052 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:31 smithi163 bash[78107]: cluster 2023-10-09T09:47:30.107211+0000 mgr.y (mgr.34107) 1246 : cluster 0 pgmap v2082: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 833 op/s 2023-10-09T09:47:32.052 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:31 smithi163 bash[78107]: cluster 2023-10-09T09:47:31.233543+0000 mon.a (mon.0) 2306 : cluster 1 osd.1 marked itself down and dead 2023-10-09T09:47:32.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:31 smithi163 bash[75425]: cluster 2023-10-09T09:47:30.107211+0000 mgr.y (mgr.34107) 1246 : cluster 0 pgmap v2082: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 833 op/s 2023-10-09T09:47:32.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:31 smithi163 bash[75425]: cluster 2023-10-09T09:47:31.233543+0000 mon.a (mon.0) 2306 : cluster 1 osd.1 marked itself down and dead 2023-10-09T09:47:32.069 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:32.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:32.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:32.173 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:32.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:32.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:32.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:32.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:32.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:31 smithi181 bash[59345]: cluster 2023-10-09T09:47:30.107211+0000 mgr.y (mgr.34107) 1246 : cluster 0 pgmap v2082: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 833 op/s 2023-10-09T09:47:32.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:31 smithi181 bash[59345]: cluster 2023-10-09T09:47:31.233543+0000 mon.a (mon.0) 2306 : cluster 1 osd.1 marked itself down and dead 2023-10-09T09:47:32.341 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:32.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:32.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:32.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:32.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:32.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:32.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:32.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:32.614 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:32.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:32.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:32.717 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:32.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:32.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:32.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:32.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:32.878 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:32.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:32.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:32.982 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:32.990 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:32 smithi181 bash[59345]: cluster 2023-10-09T09:47:31.886678+0000 mon.a (mon.0) 2307 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:47:32.990 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:32 smithi181 bash[59345]: cluster 2023-10-09T09:47:31.894846+0000 osd.5 (osd.5) 829 : cluster 3 failed to encode map e1242 with expected crc 2023-10-09T09:47:32.990 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:32 smithi181 bash[59345]: cluster 2023-10-09T09:47:31.897997+0000 mon.a (mon.0) 2308 : cluster 0 osdmap e1242: 8 total, 7 up, 6 in 2023-10-09T09:47:32.990 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:32 smithi181 bash[59345]: cluster 2023-10-09T09:47:31.900824+0000 osd.4 (osd.4) 645 : cluster 3 failed to encode map e1242 with expected crc 2023-10-09T09:47:33.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:33.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:33.041 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:47:33.046 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:47:33.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:32 smithi163 bash[78107]: cluster 2023-10-09T09:47:31.886678+0000 mon.a (mon.0) 2307 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:47:33.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:32 smithi163 bash[78107]: cluster 2023-10-09T09:47:31.894846+0000 osd.5 (osd.5) 829 : cluster 3 failed to encode map e1242 with expected crc 2023-10-09T09:47:33.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:32 smithi163 bash[78107]: cluster 2023-10-09T09:47:31.897997+0000 mon.a (mon.0) 2308 : cluster 0 osdmap e1242: 8 total, 7 up, 6 in 2023-10-09T09:47:33.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:32 smithi163 bash[78107]: cluster 2023-10-09T09:47:31.900824+0000 osd.4 (osd.4) 645 : cluster 3 failed to encode map e1242 with expected crc 2023-10-09T09:47:33.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:32 smithi163 bash[75425]: cluster 2023-10-09T09:47:31.886678+0000 mon.a (mon.0) 2307 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:47:33.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:32 smithi163 bash[75425]: cluster 2023-10-09T09:47:31.894846+0000 osd.5 (osd.5) 829 : cluster 3 failed to encode map e1242 with expected crc 2023-10-09T09:47:33.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:32 smithi163 bash[75425]: cluster 2023-10-09T09:47:31.897997+0000 mon.a (mon.0) 2308 : cluster 0 osdmap e1242: 8 total, 7 up, 6 in 2023-10-09T09:47:33.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:32 smithi163 bash[75425]: cluster 2023-10-09T09:47:31.900824+0000 osd.4 (osd.4) 645 : cluster 3 failed to encode map e1242 with expected crc 2023-10-09T09:47:33.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:33.168 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:33.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:33.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:33.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:33.305 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:33.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:33.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:33.411 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:33.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:33.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:33.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:33.547 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:33.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:33.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:33.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:33.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:33.709 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:47:33] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.33.4" 2023-10-09T09:47:33.710 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:33.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:33.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:33.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:33.840 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:33.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:33.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:33.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[75425]: cluster 2023-10-09T09:47:32.107819+0000 mgr.y (mgr.34107) 1247 : cluster 0 pgmap v2084: 89 pgs: 25 stale+active+clean, 64 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T09:47:33.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[75425]: cluster 2023-10-09T09:47:32.910619+0000 osd.5 (osd.5) 830 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[75425]: cluster 2023-10-09T09:47:32.914352+0000 mon.a (mon.0) 2309 : cluster 0 osdmap e1243: 8 total, 7 up, 6 in 2023-10-09T09:47:33.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[75425]: cluster 2023-10-09T09:47:32.914598+0000 osd.5 (osd.5) 831 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[75425]: cluster 2023-10-09T09:47:32.915182+0000 osd.4 (osd.4) 646 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[75425]: cluster 2023-10-09T09:47:32.915539+0000 osd.6 (osd.6) 526 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[75425]: cluster 2023-10-09T09:47:32.920724+0000 osd.4 (osd.4) 647 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.969 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[202716]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T09:47:33.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[78107]: cluster 2023-10-09T09:47:32.107819+0000 mgr.y (mgr.34107) 1247 : cluster 0 pgmap v2084: 89 pgs: 25 stale+active+clean, 64 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T09:47:33.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[78107]: cluster 2023-10-09T09:47:32.910619+0000 osd.5 (osd.5) 830 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[78107]: cluster 2023-10-09T09:47:32.914352+0000 mon.a (mon.0) 2309 : cluster 0 osdmap e1243: 8 total, 7 up, 6 in 2023-10-09T09:47:33.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[78107]: cluster 2023-10-09T09:47:32.914598+0000 osd.5 (osd.5) 831 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[78107]: cluster 2023-10-09T09:47:32.915182+0000 osd.4 (osd.4) 646 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[78107]: cluster 2023-10-09T09:47:32.915539+0000 osd.6 (osd.6) 526 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[78107]: cluster 2023-10-09T09:47:32.920724+0000 osd.4 (osd.4) 647 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.971 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:33.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:33.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:33 smithi181 bash[59345]: cluster 2023-10-09T09:47:32.107819+0000 mgr.y (mgr.34107) 1247 : cluster 0 pgmap v2084: 89 pgs: 25 stale+active+clean, 64 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T09:47:33.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:33 smithi181 bash[59345]: cluster 2023-10-09T09:47:32.910619+0000 osd.5 (osd.5) 830 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:33 smithi181 bash[59345]: cluster 2023-10-09T09:47:32.914352+0000 mon.a (mon.0) 2309 : cluster 0 osdmap e1243: 8 total, 7 up, 6 in 2023-10-09T09:47:33.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:33 smithi181 bash[59345]: cluster 2023-10-09T09:47:32.914598+0000 osd.5 (osd.5) 831 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:33 smithi181 bash[59345]: cluster 2023-10-09T09:47:32.915182+0000 osd.4 (osd.4) 646 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:33 smithi181 bash[59345]: cluster 2023-10-09T09:47:32.915539+0000 osd.6 (osd.6) 526 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:33.982 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:33 smithi181 bash[59345]: cluster 2023-10-09T09:47:32.920724+0000 osd.4 (osd.4) 647 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:34.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:34.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:34.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:34.217 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:34.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:34.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:34.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:34.293 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:47:34.293 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:47:34.351 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:34.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:34.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:34.468 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:33 smithi163 bash[202896]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T09:47:34.484 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:34.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:34.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:34.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:34.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:34.637 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:34.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:34.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:34.744 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:34.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:34.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:34.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:34.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:34.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:34.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:34.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:35.014 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:35.051 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:35.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:35.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:35.060 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:34 smithi181 bash[59345]: cluster 2023-10-09T09:47:33.338136+0000 osd.7 (osd.7) 598 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:35.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:35.202 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:35.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:35.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:34 smithi163 bash[78107]: cluster 2023-10-09T09:47:33.338136+0000 osd.7 (osd.7) 598 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:35.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:34 smithi163 bash[75425]: cluster 2023-10-09T09:47:33.338136+0000 osd.7 (osd.7) 598 : cluster 3 failed to encode map e1243 with expected crc 2023-10-09T09:47:35.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:35.338 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:35.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:35.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:35.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:35.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:35.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:35.517 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T09:47:35.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:35.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:35.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:35.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:35.709 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:35.734 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:35.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:35.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:35.838 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:35.871 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:35.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:35.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:35.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:35 smithi163 bash[75425]: cluster 2023-10-09T09:47:34.108512+0000 mgr.y (mgr.34107) 1248 : cluster 0 pgmap v2086: 89 pgs: 1 active+recovery_wait+undersized+degraded, 7 active+undersized, 5 active+undersized+degraded, 21 stale+active+clean, 55 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 5959/290277 objects degraded (2.053%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:35.925 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:35 smithi163 bash[75425]: cluster 2023-10-09T09:47:34.910919+0000 mon.a (mon.0) 2310 : cluster 3 Health check failed: Degraded data redundancy: 5959/290277 objects degraded (2.053%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:35.975 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:36.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:36.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:36.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:36.114 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:36.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:35 smithi181 bash[59345]: cluster 2023-10-09T09:47:34.108512+0000 mgr.y (mgr.34107) 1248 : cluster 0 pgmap v2086: 89 pgs: 1 active+recovery_wait+undersized+degraded, 7 active+undersized, 5 active+undersized+degraded, 21 stale+active+clean, 55 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 5959/290277 objects degraded (2.053%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:36.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:35 smithi181 bash[59345]: cluster 2023-10-09T09:47:34.910919+0000 mon.a (mon.0) 2310 : cluster 3 Health check failed: Degraded data redundancy: 5959/290277 objects degraded (2.053%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:36.152 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:36.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:36.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:35 smithi163 bash[78107]: cluster 2023-10-09T09:47:34.108512+0000 mgr.y (mgr.34107) 1248 : cluster 0 pgmap v2086: 89 pgs: 1 active+recovery_wait+undersized+degraded, 7 active+undersized, 5 active+undersized+degraded, 21 stale+active+clean, 55 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 5959/290277 objects degraded (2.053%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:36.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:35 smithi163 bash[78107]: cluster 2023-10-09T09:47:34.910919+0000 mon.a (mon.0) 2310 : cluster 3 Health check failed: Degraded data redundancy: 5959/290277 objects degraded (2.053%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:36.258 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:36.287 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:36.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:36.393 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:36.420 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:36.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:36.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:36.523 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:36.544 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:36.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:36.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:36.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:36.673 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:36.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:36.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:36.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:36.802 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:36.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:36.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:36.913 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:36.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:36.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:36.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:37.042 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:37.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:37.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:37.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:37.184 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:37.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:37.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:37.311 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T09:47:37.328 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:37.339 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T09:47:37.339 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T09:47:37.357 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:37.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:37.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:37.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:37.486 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:37.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:37.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:37.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:37.621 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:37.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:37.727 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:37.756 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:37.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:37.862 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:37.894 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:37.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:37.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:38.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:38.029 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:38.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:38.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:38.143 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:37 smithi181 bash[59345]: cluster 2023-10-09T09:47:36.109162+0000 mgr.y (mgr.34107) 1249 : cluster 0 pgmap v2087: 89 pgs: 1 active+recovery_wait+undersized+degraded, 20 active+undersized, 5 active+undersized+degraded, 12 stale+active+clean, 51 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 599 op/s; 5959/287694 objects degraded (2.071%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:38.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:38.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:38.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:38.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:37 smithi163 bash[78107]: cluster 2023-10-09T09:47:36.109162+0000 mgr.y (mgr.34107) 1249 : cluster 0 pgmap v2087: 89 pgs: 1 active+recovery_wait+undersized+degraded, 20 active+undersized, 5 active+undersized+degraded, 12 stale+active+clean, 51 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 599 op/s; 5959/287694 objects degraded (2.071%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:37 smithi163 bash[75425]: cluster 2023-10-09T09:47:36.109162+0000 mgr.y (mgr.34107) 1249 : cluster 0 pgmap v2087: 89 pgs: 1 active+recovery_wait+undersized+degraded, 20 active+undersized, 5 active+undersized+degraded, 12 stale+active+clean, 51 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 599 op/s; 5959/287694 objects degraded (2.071%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:38.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:38.299 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:38.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:38.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:38.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:38.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:38.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:38.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:38.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:38.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:38.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:38.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:38.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:38.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:38.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:38.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:38.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:38.826 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:38.827 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:38.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:38.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:38.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:38.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:38.982 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:38.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:39.083 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:39.110 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:39.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:39.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:39.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:39.238 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:39.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:39.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:39.341 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:39.377 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:39.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:39.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:39.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:39.517 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:39.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:39.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:39.622 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:39.646 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:39.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:39.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:39.754 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:39.794 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:39.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:39.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:39.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:39.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:39.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:40.036 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:40.044 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:39 smithi181 bash[59345]: cluster 2023-10-09T09:47:38.113275+0000 mgr.y (mgr.34107) 1250 : cluster 0 pgmap v2088: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.7 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 913 op/s; 28797/278742 objects degraded (10.331%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:40.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:40.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:40.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:40.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:40.207 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:40.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:40.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:40.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:39 smithi163 bash[78107]: cluster 2023-10-09T09:47:38.113275+0000 mgr.y (mgr.34107) 1250 : cluster 0 pgmap v2088: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.7 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 913 op/s; 28797/278742 objects degraded (10.331%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:40.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:39 smithi163 bash[75425]: cluster 2023-10-09T09:47:38.113275+0000 mgr.y (mgr.34107) 1250 : cluster 0 pgmap v2088: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.7 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 913 op/s; 28797/278742 objects degraded (10.331%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:40.312 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:40.354 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:40.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:40.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:40.412 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:47:40.459 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:40.479 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:40.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:40.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:40.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:40.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:40.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:40.724 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:40.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:40.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:40.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:40.858 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:40.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:40.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:40.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:40.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:41.010 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:41.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:41.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:41.116 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:41.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:41.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:41.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:41.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:41.286 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:41.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:41.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:41.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:41.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:41.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:41.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:41.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:41.550 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:41.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:41.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:41.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:41 smithi163 bash[78107]: cluster 2023-10-09T09:47:40.113714+0000 mgr.y (mgr.34107) 1251 : cluster 0 pgmap v2089: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.7 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 28797/278742 objects degraded (10.331%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:41.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:41 smithi163 bash[78107]: audit 2023-10-09T09:47:40.401320+0000 mon.a (mon.0) 2311 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:41.625 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:41 smithi163 bash[78107]: audit 2023-10-09T09:47:40.402452+0000 mon.a (mon.0) 2312 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:41.625 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:41 smithi163 bash[78107]: cluster 2023-10-09T09:47:40.855876+0000 mon.a (mon.0) 2313 : cluster 3 Health check update: Degraded data redundancy: 28797/278742 objects degraded (10.331%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:41.625 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:41 smithi163 bash[75425]: cluster 2023-10-09T09:47:40.113714+0000 mgr.y (mgr.34107) 1251 : cluster 0 pgmap v2089: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.7 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 28797/278742 objects degraded (10.331%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:41.625 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:41 smithi163 bash[75425]: audit 2023-10-09T09:47:40.401320+0000 mon.a (mon.0) 2311 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:41.625 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:41 smithi163 bash[75425]: audit 2023-10-09T09:47:40.402452+0000 mon.a (mon.0) 2312 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:41.625 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:41 smithi163 bash[75425]: cluster 2023-10-09T09:47:40.855876+0000 mon.a (mon.0) 2313 : cluster 3 Health check update: Degraded data redundancy: 28797/278742 objects degraded (10.331%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:41.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:41.666 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:41 smithi181 bash[59345]: cluster 2023-10-09T09:47:40.113714+0000 mgr.y (mgr.34107) 1251 : cluster 0 pgmap v2089: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.7 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 28797/278742 objects degraded (10.331%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:41.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:41 smithi181 bash[59345]: audit 2023-10-09T09:47:40.401320+0000 mon.a (mon.0) 2311 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:47:41.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:41 smithi181 bash[59345]: audit 2023-10-09T09:47:40.402452+0000 mon.a (mon.0) 2312 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:41.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:41 smithi181 bash[59345]: cluster 2023-10-09T09:47:40.855876+0000 mon.a (mon.0) 2313 : cluster 3 Health check update: Degraded data redundancy: 28797/278742 objects degraded (10.331%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:41.684 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:41.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:41.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:41.790 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:41.827 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:41.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:41.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:41.933 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:41.957 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:41.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:41.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:42.062 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:42.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:42.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:42.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:42.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:42.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:42.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:42.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:42.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:42.340 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 6, 2, 5] out_osds: [7, 3] dead_osds: [1] live_osds: [2, 3, 4, 5, 6, 7, 0] 2023-10-09T09:47:42.340 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-10-09T09:47:42.341 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-10-09T09:47:42.341 DEBUG:teuthology.orchestra.run.smithi163:osd.1> sudo journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T09:47:42.343 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl start ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T09:47:42.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:42.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:42.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:42.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:47:42.476 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:42.511 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:42.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:42.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:42.617 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:42.635 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:42.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:42.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:42.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:42.764 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:42.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:42.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:42.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:42.890 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:42.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:42.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:42.996 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:43.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:43.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:43.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:43.125 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:43.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:43.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:43.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:43.261 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:43.285 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:43.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:43.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:43.387 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:43.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:43.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:43.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:43.426 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:47:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:47:43] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.33.4" 2023-10-09T09:47:43.426 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:43 smithi163 bash[75425]: cluster 2023-10-09T09:47:42.114468+0000 mgr.y (mgr.34107) 1252 : cluster 0 pgmap v2090: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 810 op/s; 27730/267156 objects degraded (10.380%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:43.440 INFO:journalctl@ceph.osd.1.smithi163.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T09:47:43.520 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:43.546 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:43.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:43.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:43.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:43 smithi181 bash[59345]: cluster 2023-10-09T09:47:42.114468+0000 mgr.y (mgr.34107) 1252 : cluster 0 pgmap v2090: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 810 op/s; 27730/267156 objects degraded (10.380%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:43.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:43.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:43.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:43.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:43 smithi163 bash[78107]: cluster 2023-10-09T09:47:42.114468+0000 mgr.y (mgr.34107) 1252 : cluster 0 pgmap v2090: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 810 op/s; 27730/267156 objects degraded (10.380%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:43.801 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:43.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:43.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:43.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:43.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:43.958 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:43.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:43.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:44.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:44.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:44.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:44.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:44.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:44.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:44.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:44.351 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:44.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:44.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:44.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:44.492 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:44.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:44.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:44.599 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:44.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:44.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:44.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:44.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:44.790 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:44.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:44.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:44.898 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:44.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:44.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:45.026 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:45.053 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:45.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:45.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:45.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:45.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:45.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:45.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:45.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:45.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:45.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:45.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:45.422 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:45.445 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:45.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:45.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:45.580 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:45.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:45.687 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:45.696 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:45 smithi163 bash[78107]: cluster 2023-10-09T09:47:44.115108+0000 mgr.y (mgr.34107) 1253 : cluster 0 pgmap v2091: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 768 op/s; 27483/265671 objects degraded (10.345%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:45.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:45 smithi163 bash[75425]: cluster 2023-10-09T09:47:44.115108+0000 mgr.y (mgr.34107) 1253 : cluster 0 pgmap v2091: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 768 op/s; 27483/265671 objects degraded (10.345%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:45.710 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:45.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:45.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:45 smithi181 bash[59345]: cluster 2023-10-09T09:47:44.115108+0000 mgr.y (mgr.34107) 1253 : cluster 0 pgmap v2091: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 768 op/s; 27483/265671 objects degraded (10.345%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:45.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:45.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:45.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:45.892 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:47:45.892 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:47:45.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:45.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:45.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:45.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:46.077 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:46.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:46.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:46.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:46.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:46.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:46.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:46.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:46.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:46.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:46.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:46.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:46.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:46.511 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:46.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:46.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:46.615 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:46.633 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:46.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:46.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:46.652 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:46 smithi163 bash[78107]: cluster 2023-10-09T09:47:45.857190+0000 mon.a (mon.0) 2314 : cluster 3 Health check update: Degraded data redundancy: 27483/265671 objects degraded (10.345%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:46.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:46 smithi163 bash[75425]: cluster 2023-10-09T09:47:45.857190+0000 mon.a (mon.0) 2314 : cluster 3 Health check update: Degraded data redundancy: 27483/265671 objects degraded (10.345%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:46.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:46.746 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:46 smithi181 bash[59345]: cluster 2023-10-09T09:47:45.857190+0000 mon.a (mon.0) 2314 : cluster 3 Health check update: Degraded data redundancy: 27483/265671 objects degraded (10.345%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:46.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:46.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:46.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:46.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:46.910 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:46.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:46.966 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T09:47:47.043 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:47.075 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:47.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:47.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:47.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:47.193 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:47.219 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:47.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:47.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:47.325 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:47.358 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:47.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:47.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:47.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:47.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:47.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:47.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:47.643 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:47.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:47.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:47.748 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:47.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:47.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:47.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:47.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:47 smithi181 bash[59345]: cluster 2023-10-09T09:47:46.116132+0000 mgr.y (mgr.34107) 1254 : cluster 0 pgmap v2092: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.3 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 770 op/s; 27483/262515 objects degraded (10.469%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:47.882 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:47.910 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:47.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:47.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:48.018 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:48.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:48.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:48.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:48.161 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:48.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:47 smithi163 bash[75425]: cluster 2023-10-09T09:47:46.116132+0000 mgr.y (mgr.34107) 1254 : cluster 0 pgmap v2092: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.3 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 770 op/s; 27483/262515 objects degraded (10.469%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:48.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:47 smithi163 bash[78107]: cluster 2023-10-09T09:47:46.116132+0000 mgr.y (mgr.34107) 1254 : cluster 0 pgmap v2092: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.3 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 770 op/s; 27483/262515 objects degraded (10.469%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:48.185 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:48.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:48.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:48.291 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:48.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:48.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:48.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:48.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:48.451 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:48.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:48.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:48.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:48.584 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:48.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:48.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:48.687 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:48.710 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:48.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:48.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:48.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:48.860 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:48.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:48.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:48.969 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:49.002 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:49.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:49.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:49.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:49.133 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:49.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:49.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:49.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:49.268 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:49.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:49.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:49.375 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:49.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:49.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:49.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:49.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:49.517 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:49 smithi163 bash[203509]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T09:47:49.517 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:49 smithi163 bash[203509]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T09:47:49.517 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:49 smithi163 bash[203509]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T09:47:49.518 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:49 smithi163 bash[203509]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T09:47:49.518 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:49 smithi163 bash[203509]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T09:47:49.518 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:49 smithi163 bash[203509]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T09:47:49.518 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:49 smithi163 bash[203509]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T09:47:49.518 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:49 smithi163 bash[203509]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T09:47:49.539 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:49.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:49.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:49.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:49.671 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:49.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:49.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:49.773 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:49.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:49.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:49.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:49.901 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:49.913 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:49 smithi181 bash[59345]: cluster 2023-10-09T09:47:48.116968+0000 mgr.y (mgr.34107) 1255 : cluster 0 pgmap v2093: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 953 op/s; 26195/253338 objects degraded (10.340%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:49.938 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:49.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:49.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:49.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:49 smithi163 bash[75425]: cluster 2023-10-09T09:47:48.116968+0000 mgr.y (mgr.34107) 1255 : cluster 0 pgmap v2093: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 953 op/s; 26195/253338 objects degraded (10.340%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:49.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:49 smithi163 bash[78107]: cluster 2023-10-09T09:47:48.116968+0000 mgr.y (mgr.34107) 1255 : cluster 0 pgmap v2093: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 953 op/s; 26195/253338 objects degraded (10.340%); 0 B/s, 0 objects/s recovering 2023-10-09T09:47:50.042 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:50.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:50.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:50.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:50.177 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:50.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:50.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:50.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:50.309 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:50.338 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:50.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:50.438 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:47:50.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:50.468 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:50.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:50.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:50.498 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:47:50.529 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:47:50.560 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:47:50.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:50.599 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:50.603 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:47:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:50.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:50.634 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:47:50.662 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:47:50.663 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:47:50.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T09:47:50.663 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:47:50.663 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:47:50.663 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:47:50.663 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:47:50.664 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:47:50.664 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:47:50.707 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:50.728 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:50.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:50.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:50.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:50.855 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:50.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:50.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:50.895 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:47:50.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:50.989 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:50.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:50.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:51.006 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:50 smithi163 bash[78107]: audit 2023-10-09T09:47:50.203940+0000 mon.a (mon.0) 2315 : audit 1 from='client.? 172.21.15.163:0/3325045855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:47:51.006 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:50 smithi163 bash[78107]: audit 2023-10-09T09:47:50.406747+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.163:0/71377726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:47:51.006 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:50 smithi163 bash[78107]: cluster 2023-10-09T09:47:50.858379+0000 mon.a (mon.0) 2316 : cluster 3 Health check update: Degraded data redundancy: 26195/253338 objects degraded (10.340%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:51.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:50 smithi163 bash[75425]: audit 2023-10-09T09:47:50.203940+0000 mon.a (mon.0) 2315 : audit 1 from='client.? 172.21.15.163:0/3325045855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:47:51.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:50 smithi163 bash[75425]: audit 2023-10-09T09:47:50.406747+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.163:0/71377726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:47:51.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:50 smithi163 bash[75425]: cluster 2023-10-09T09:47:50.858379+0000 mon.a (mon.0) 2316 : cluster 3 Health check update: Degraded data redundancy: 26195/253338 objects degraded (10.340%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:51.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:51.122 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:51.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:51.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:51.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:51.234 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:50 smithi181 bash[59345]: audit 2023-10-09T09:47:50.203940+0000 mon.a (mon.0) 2315 : audit 1 from='client.? 172.21.15.163:0/3325045855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:47:51.235 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:50 smithi181 bash[59345]: audit 2023-10-09T09:47:50.406747+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.163:0/71377726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:47:51.235 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:50 smithi181 bash[59345]: cluster 2023-10-09T09:47:50.858379+0000 mon.a (mon.0) 2316 : cluster 3 Health check update: Degraded data redundancy: 26195/253338 objects degraded (10.340%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:51.253 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:51.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:51.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:51.359 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:51.389 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:51.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:51.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:51.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:51.521 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:51.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:51.625 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:51.666 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:51.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:51.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:51.772 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:51.800 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:51.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:51.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:51.873 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:47:51.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:51.925 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:51 smithi181 bash[59345]: cluster 2023-10-09T09:47:50.117487+0000 mgr.y (mgr.34107) 1256 : cluster 0 pgmap v2094: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 705 op/s; 26195/253338 objects degraded (10.340%) 2023-10-09T09:47:51.925 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:51 smithi181 bash[59345]: cluster 2023-10-09T09:47:50.870017+0000 mon.a (mon.0) 2317 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:47:51.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:51 smithi181 bash[59345]: cluster 2023-10-09T09:47:50.880568+0000 osd.5 (osd.5) 832 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:51 smithi181 bash[59345]: audit 2023-10-09T09:47:50.884374+0000 mon.a (mon.0) 2318 : audit 1 from='client.? 172.21.15.163:0/3325045855' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:47:51.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:51 smithi181 bash[59345]: cluster 2023-10-09T09:47:50.884512+0000 mon.a (mon.0) 2319 : cluster 0 osdmap e1244: 8 total, 7 up, 6 in 2023-10-09T09:47:51.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:51 smithi181 bash[59345]: cluster 2023-10-09T09:47:50.892065+0000 osd.4 (osd.4) 648 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:51 smithi181 bash[59345]: cluster 2023-10-09T09:47:50.893201+0000 osd.6 (osd.6) 527 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:51 smithi181 bash[59345]: cluster 2023-10-09T09:47:50.893881+0000 osd.4 (osd.4) 649 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:51 smithi181 bash[59345]: cluster 2023-10-09T09:47:50.895289+0000 osd.6 (osd.6) 528 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:51 smithi181 bash[59345]: cluster 2023-10-09T09:47:50.896498+0000 osd.4 (osd.4) 650 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.950 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:51.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:51.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[75425]: cluster 2023-10-09T09:47:50.117487+0000 mgr.y (mgr.34107) 1256 : cluster 0 pgmap v2094: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 705 op/s; 26195/253338 objects degraded (10.340%) 2023-10-09T09:47:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[75425]: cluster 2023-10-09T09:47:50.870017+0000 mon.a (mon.0) 2317 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:47:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[75425]: cluster 2023-10-09T09:47:50.880568+0000 osd.5 (osd.5) 832 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[75425]: audit 2023-10-09T09:47:50.884374+0000 mon.a (mon.0) 2318 : audit 1 from='client.? 172.21.15.163:0/3325045855' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:47:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[75425]: cluster 2023-10-09T09:47:50.884512+0000 mon.a (mon.0) 2319 : cluster 0 osdmap e1244: 8 total, 7 up, 6 in 2023-10-09T09:47:51.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[75425]: cluster 2023-10-09T09:47:50.892065+0000 osd.4 (osd.4) 648 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[75425]: cluster 2023-10-09T09:47:50.893201+0000 osd.6 (osd.6) 527 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[75425]: cluster 2023-10-09T09:47:50.893881+0000 osd.4 (osd.4) 649 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[75425]: cluster 2023-10-09T09:47:50.895289+0000 osd.6 (osd.6) 528 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[75425]: cluster 2023-10-09T09:47:50.896498+0000 osd.4 (osd.4) 650 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[78107]: cluster 2023-10-09T09:47:50.117487+0000 mgr.y (mgr.34107) 1256 : cluster 0 pgmap v2094: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 705 op/s; 26195/253338 objects degraded (10.340%) 2023-10-09T09:47:51.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[78107]: cluster 2023-10-09T09:47:50.870017+0000 mon.a (mon.0) 2317 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:47:51.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[78107]: cluster 2023-10-09T09:47:50.880568+0000 osd.5 (osd.5) 832 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[78107]: audit 2023-10-09T09:47:50.884374+0000 mon.a (mon.0) 2318 : audit 1 from='client.? 172.21.15.163:0/3325045855' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:47:51.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[78107]: cluster 2023-10-09T09:47:50.884512+0000 mon.a (mon.0) 2319 : cluster 0 osdmap e1244: 8 total, 7 up, 6 in 2023-10-09T09:47:51.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[78107]: cluster 2023-10-09T09:47:50.892065+0000 osd.4 (osd.4) 648 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[78107]: cluster 2023-10-09T09:47:50.893201+0000 osd.6 (osd.6) 527 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[78107]: cluster 2023-10-09T09:47:50.893881+0000 osd.4 (osd.4) 649 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[78107]: cluster 2023-10-09T09:47:50.895289+0000 osd.6 (osd.6) 528 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:51.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:51 smithi163 bash[78107]: cluster 2023-10-09T09:47:50.896498+0000 osd.4 (osd.4) 650 : cluster 3 failed to encode map e1244 with expected crc 2023-10-09T09:47:52.053 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:52.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:52.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:52.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:52.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:52.221 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:52.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:52.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:52.328 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:52.355 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:52.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:52.459 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:52.482 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:52.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:52.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:52.585 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:52.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:52.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:52.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:52.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:52.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:52.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:52.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:52.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:52.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:52.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:52.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:52.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:53.010 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:53.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:53.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:53.115 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:53.157 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:53.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:53.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:53.265 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:53.303 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:53.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:53.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:53.409 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:53.439 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:53.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:53.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:53.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:53.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:53.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:53.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:53.709 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:53.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:53.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:47:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:47:53] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.33.4" 2023-10-09T09:47:53.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:53.840 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:53.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:53.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:53.943 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:53.977 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:53.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:53.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:54.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:54.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:53 smithi181 bash[59345]: cluster 2023-10-09T09:47:52.118243+0000 mgr.y (mgr.34107) 1257 : cluster 0 pgmap v2096: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 814 op/s; 25237/242715 objects degraded (10.398%) 2023-10-09T09:47:54.106 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:54.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:54.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:54.212 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:54.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:53 smithi163 bash[75425]: cluster 2023-10-09T09:47:52.118243+0000 mgr.y (mgr.34107) 1257 : cluster 0 pgmap v2096: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 814 op/s; 25237/242715 objects degraded (10.398%) 2023-10-09T09:47:54.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:53 smithi163 bash[78107]: cluster 2023-10-09T09:47:52.118243+0000 mgr.y (mgr.34107) 1257 : cluster 0 pgmap v2096: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 814 op/s; 25237/242715 objects degraded (10.398%) 2023-10-09T09:47:54.249 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:54.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:54.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:54.273 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T09:47:54.274 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T09:47:54.353 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:54.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:54.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:54.483 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:54.510 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:54.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:54.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:54.613 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:54.638 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:54.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:54.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:54.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:54.769 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:54.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:54.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:54.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:54.909 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:54.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:54.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:55.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:55.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:55.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:55.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:55.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:55.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:55.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:55.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:55.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:55.295 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:55.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:55.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:55.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:55.427 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:55.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:55.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:55.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:55.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:55.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:55.656 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:55.680 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:55.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:55.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:55.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:55.806 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:55.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:55.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:55.912 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:55.945 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:55.949 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:55 smithi163 bash[204298]: debug 2023-10-09T09:47:55.813+0000 7f908d427700 1 -- 172.21.15.163:0/2785727075 <== mon.2 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561cc098e4e0 con 0x561cc09e8800 2023-10-09T09:47:55.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:55 smithi163 bash[78107]: cluster 2023-10-09T09:47:54.118815+0000 mgr.y (mgr.34107) 1258 : cluster 0 pgmap v2097: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 24974/241185 objects degraded (10.355%) 2023-10-09T09:47:55.950 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:55 smithi163 bash[78107]: audit 2023-10-09T09:47:55.394690+0000 mon.a (mon.0) 2320 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:55.950 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:55 smithi163 bash[78107]: cluster 2023-10-09T09:47:55.859576+0000 mon.a (mon.0) 2321 : cluster 3 Health check update: Degraded data redundancy: 24974/241185 objects degraded (10.355%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:55.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:55.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:55.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:55 smithi163 bash[75425]: cluster 2023-10-09T09:47:54.118815+0000 mgr.y (mgr.34107) 1258 : cluster 0 pgmap v2097: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 24974/241185 objects degraded (10.355%) 2023-10-09T09:47:55.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:55 smithi163 bash[75425]: audit 2023-10-09T09:47:55.394690+0000 mon.a (mon.0) 2320 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:55.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:55 smithi163 bash[75425]: cluster 2023-10-09T09:47:55.859576+0000 mon.a (mon.0) 2321 : cluster 3 Health check update: Degraded data redundancy: 24974/241185 objects degraded (10.355%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:56.044 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:56 smithi181 bash[59345]: cluster 2023-10-09T09:47:54.118815+0000 mgr.y (mgr.34107) 1258 : cluster 0 pgmap v2097: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 24974/241185 objects degraded (10.355%) 2023-10-09T09:47:56.044 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:56 smithi181 bash[59345]: audit 2023-10-09T09:47:55.394690+0000 mon.a (mon.0) 2320 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:47:56.044 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:56 smithi181 bash[59345]: cluster 2023-10-09T09:47:55.859576+0000 mon.a (mon.0) 2321 : cluster 3 Health check update: Degraded data redundancy: 24974/241185 objects degraded (10.355%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:47:56.051 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:56.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:56.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:56.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:56.210 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:56.233 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:56.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:56.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:56.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:47:56.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:47:56.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:56.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:56.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:56.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:56.475 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:56.492 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:47:56 smithi163 bash[204298]: debug 2023-10-09T09:47:56.365+0000 7f90965ad640 -1 Falling back to public interface 2023-10-09T09:47:56.509 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:56.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:56.615 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:56.644 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:56.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:56.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:56.718 INFO:teuthology.orchestra.run.smithi163.stderr:no valid command found; 10 closest matches: 2023-10-09T09:47:56.718 INFO:teuthology.orchestra.run.smithi163.stderr:0 2023-10-09T09:47:56.718 INFO:teuthology.orchestra.run.smithi163.stderr:1 2023-10-09T09:47:56.718 INFO:teuthology.orchestra.run.smithi163.stderr:2 2023-10-09T09:47:56.718 INFO:teuthology.orchestra.run.smithi163.stderr:abort 2023-10-09T09:47:56.719 INFO:teuthology.orchestra.run.smithi163.stderr:assert 2023-10-09T09:47:56.719 INFO:teuthology.orchestra.run.smithi163.stderr:config diff 2023-10-09T09:47:56.719 INFO:teuthology.orchestra.run.smithi163.stderr:config diff get 2023-10-09T09:47:56.719 INFO:teuthology.orchestra.run.smithi163.stderr:config get 2023-10-09T09:47:56.719 INFO:teuthology.orchestra.run.smithi163.stderr:config help [] 2023-10-09T09:47:56.719 INFO:teuthology.orchestra.run.smithi163.stderr:config set ... 2023-10-09T09:47:56.719 INFO:teuthology.orchestra.run.smithi163.stderr:admin_socket: invalid command 2023-10-09T09:47:56.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:56.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:47:56.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:56.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:47:56.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:56.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:56.928 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:56.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:57.029 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:57.050 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:57.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:57.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:57.153 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:57.188 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:57.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:57.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:57.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:57.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:57.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:57.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:57.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:57.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:57.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:57.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:57.546 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:57.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:57.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:57.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:57.698 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:57.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:57.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:57.800 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:47:57.800 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T09:47:57.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T09:47:57.800 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:47:57.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.800 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181706 8.4 8:2ecb8db1:::benchmark_data_smithi163_151167_object86473:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1161)", 2023-10-09T09:47:57.800 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.909264+0000", 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 525.88570399399998, 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5310189379999999, 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181706 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.801 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909264+0000", 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909264+0000", 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909264+0000", 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0439999999999997e-06 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909268+0000", 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2739999999999996e-06 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909277+0000", 2023-10-09T09:47:57.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4819999999999996e-06 2023-10-09T09:47:57.804 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.804 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.804 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.804 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909281+0000", 2023-10-09T09:47:57.804 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6717e-05 2023-10-09T09:47:57.804 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.804 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.804 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.804 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909328+0000", 2023-10-09T09:47:57.804 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000100179 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909428+0000", 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012467000000000001 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.909553+0000", 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062400060000000002 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.915793+0000", 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003638287 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.919431+0000", 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022810609999999999 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.942242+0000", 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2697000000000001e-05 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.942254+0000", 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.006853617 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.949108+0000", 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3545e-05 2023-10-09T09:47:57.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.949122+0000", 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4350656939999999 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.384187+0000", 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016446599999999999 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for missing object", 2023-10-09T09:47:57.809 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.384352+0000", 2023-10-09T09:47:57.809 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.055816106999999997 2023-10-09T09:47:57.809 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.809 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.809 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.809 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.440168+0000", 2023-10-09T09:47:57.809 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115098 2023-10-09T09:47:57.809 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.809 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.809 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.440283+0000", 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5310189379999999 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181726 8.2 8:49011ff1:::benchmark_data_smithi163_151167_object86493:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1162)", 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.961439+0000", 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 525.83352870800002, 2023-10-09T09:47:57.810 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2811808660000001, 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181726 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961439+0000", 2023-10-09T09:47:57.811 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961439+0000", 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961438+0000", 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5080000000000005e-06 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.812 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961446+0000", 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3454e-05 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961460+0000", 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.241e-06 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961466+0000", 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2775999999999997e-05 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961529+0000", 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3730000000000001e-05 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.961542+0000", 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98586582300000003 2023-10-09T09:47:57.814 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.947408+0000", 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0344999999999999e-05 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.947428+0000", 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2693667559999999 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.815 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.216795+0000", 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2419999999999999e-05 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.216888+0000", 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00064060199999999999 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T09:47:57.816 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.217528+0000", 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.024105868999999999 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.241634+0000", 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029292499999999998 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.241927+0000", 2023-10-09T09:47:57.817 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9008000000000003e-05 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.241966+0000", 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00044008900000000001 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.242406+0000", 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6975000000000002e-05 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.818 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.242473+0000", 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014713700000000001 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.242620+0000", 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2811808660000001 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.819 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181735 8.2 8:495c0466:::benchmark_data_smithi163_151167_object86502:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1162)", 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.984259+0000", 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 525.81070902299996, 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2625273049999999, 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181735 2023-10-09T09:47:57.820 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984259+0000", 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984259+0000", 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.821 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984257+0000", 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1745e-05 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984269+0000", 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4559e-05 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.822 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984293+0000", 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7567e-05 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984311+0000", 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1023000000000001e-05 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984382+0000", 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4260999999999999e-05 2023-10-09T09:47:57.823 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.984396+0000", 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96307082399999999 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.947467+0000", 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0461e-05 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.824 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:47:57.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:57.827 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.947477+0000", 2023-10-09T09:47:57.827 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2808528299999999 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.228330+0000", 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013772900000000001 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.228468+0000", 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000164411 2023-10-09T09:47:57.828 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.228632+0000", 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013010261 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.241642+0000", 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00036442499999999998 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.829 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.242007+0000", 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000356187 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.242363+0000", 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041758339999999998 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.830 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.246539+0000", 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010223 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.246641+0000", 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014521 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.246786+0000", 2023-10-09T09:47:57.831 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2625273049999999 2023-10-09T09:47:57.832 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.832 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.832 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.832 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.832 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.832 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189411 8.4 8:2d2046c1:::benchmark_data_smithi163_151167_object94178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:47:57.832 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.945426+0000", 2023-10-09T09:47:57.832 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 510.84954227999998, 2023-10-09T09:47:57.832 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29429408800000001, 2023-10-09T09:47:57.832 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.833 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.833 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.833 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:47:57.833 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:47:57.833 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189411 2023-10-09T09:47:57.833 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.833 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.833 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.833 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.833 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945426+0000", 2023-10-09T09:47:57.834 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.834 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.834 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.834 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.834 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945426+0000", 2023-10-09T09:47:57.834 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:47:57.834 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.834 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.834 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.834 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945425+0000", 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5380000000000002e-06 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945430+0000", 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9080000000000007e-06 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945440+0000", 2023-10-09T09:47:57.835 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3459999999999999e-06 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945446+0000", 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0451000000000001e-05 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945507+0000", 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021344100000000001 2023-10-09T09:47:57.836 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.837 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.837 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.837 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945720+0000", 2023-10-09T09:47:57.837 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022876 2023-10-09T09:47:57.837 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.837 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.837 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:47:57.837 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.945949+0000", 2023-10-09T09:47:57.837 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034423269999999998 2023-10-09T09:47:57.837 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.838 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.838 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.838 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.949391+0000", 2023-10-09T09:47:57.838 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015354016999999999 2023-10-09T09:47:57.838 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.838 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.838 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.838 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.964745+0000", 2023-10-09T09:47:57.838 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27384020599999997 2023-10-09T09:47:57.838 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.839 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.839 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.839 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.238586+0000", 2023-10-09T09:47:57.839 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5902000000000005e-05 2023-10-09T09:47:57.839 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.839 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.839 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.839 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.238671+0000", 2023-10-09T09:47:57.839 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010484439999999999 2023-10-09T09:47:57.839 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.239720+0000", 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29429408800000001 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189414 8.4 8:2f4d7c9e:::benchmark_data_smithi163_151167_object94181:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.951055+0000", 2023-10-09T09:47:57.840 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 510.84391354799999, 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.28962749399999999, 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189414 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.841 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951055+0000", 2023-10-09T09:47:57.842 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.842 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.842 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.842 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.842 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951055+0000", 2023-10-09T09:47:57.842 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:47:57.842 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.842 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.842 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.842 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951053+0000", 2023-10-09T09:47:57.843 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6830000000000008e-06 2023-10-09T09:47:57.843 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.843 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.843 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.843 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951061+0000", 2023-10-09T09:47:57.843 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4021e-05 2023-10-09T09:47:57.843 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.843 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.843 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.843 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951075+0000", 2023-10-09T09:47:57.844 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4969999999999998e-06 2023-10-09T09:47:57.844 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.844 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.844 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.844 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951081+0000", 2023-10-09T09:47:57.844 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019693900000000001 2023-10-09T09:47:57.844 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.844 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.844 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.845 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951278+0000", 2023-10-09T09:47:57.845 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000190495 2023-10-09T09:47:57.845 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.845 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.845 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.845 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951469+0000", 2023-10-09T09:47:57.845 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000215503 2023-10-09T09:47:57.845 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.845 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.845 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.951684+0000", 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0084840320000000007 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.960168+0000", 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0047018959999999997 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.846 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.964870+0000", 2023-10-09T09:47:57.847 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.274971242 2023-10-09T09:47:57.847 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.847 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.847 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.847 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.239841+0000", 2023-10-09T09:47:57.847 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6891000000000005e-05 2023-10-09T09:47:57.847 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.847 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.847 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.847 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.239938+0000", 2023-10-09T09:47:57.848 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00074391400000000003 2023-10-09T09:47:57.848 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.848 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.848 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.848 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.240682+0000", 2023-10-09T09:47:57.848 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.28962749399999999 2023-10-09T09:47:57.848 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.848 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.848 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.848 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189420 8.4 8:23866f45:::benchmark_data_smithi163_151167_object94187:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.955287+0000", 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 510.839681063, 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31199607499999998, 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:47:57.849 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189420 2023-10-09T09:47:57.850 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.850 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.850 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.850 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.850 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955287+0000", 2023-10-09T09:47:57.850 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.850 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.850 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.850 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.850 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955287+0000", 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955285+0000", 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9729999999999997e-06 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955293+0000", 2023-10-09T09:47:57.851 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3506000000000001e-05 2023-10-09T09:47:57.852 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.852 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.852 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.852 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955307+0000", 2023-10-09T09:47:57.852 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3180000000000001e-06 2023-10-09T09:47:57.852 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.852 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.852 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.852 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.955314+0000", 2023-10-09T09:47:57.852 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050446090000000002 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.960359+0000", 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014191900000000001 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.960501+0000", 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000132738 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.853 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.960633+0000", 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015849736 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.976483+0000", 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026229599999999999 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.854 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.002713+0000", 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26364709800000002 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.266360+0000", 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3532000000000003e-05 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.266403+0000", 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00087977300000000004 2023-10-09T09:47:57.855 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.267283+0000", 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31199607499999998 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189425 8.4 8:24365966:::benchmark_data_smithi163_151167_object94192:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.967562+0000", 2023-10-09T09:47:57.856 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 510.82740600300002, 2023-10-09T09:47:57.857 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31396539400000001, 2023-10-09T09:47:57.857 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.857 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.857 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.857 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:47:57.857 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:47:57.857 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189425 2023-10-09T09:47:57.857 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.857 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967562+0000", 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967562+0000", 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.858 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967560+0000", 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7849999999999997e-06 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967569+0000", 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6466000000000001e-05 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.859 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967586+0000", 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4059999999999999e-06 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.967593+0000", 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002803027 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.970396+0000", 2023-10-09T09:47:57.860 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020983500000000001 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.970606+0000", 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00075611799999999996 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.971362+0000", 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0063786060000000002 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.861 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.977741+0000", 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.024996424999999999 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.002737+0000", 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27793337200000001 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.863 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.280670+0000", 2023-10-09T09:47:57.863 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9372e-05 2023-10-09T09:47:57.863 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.863 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.863 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.863 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.280730+0000", 2023-10-09T09:47:57.863 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00079766199999999998 2023-10-09T09:47:57.863 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.863 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.863 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.281527+0000", 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31396539400000001 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189435 8.4 8:29678ca8:::benchmark_data_smithi163_151167_object94202:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.979108+0000", 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 510.81586051800002, 2023-10-09T09:47:57.864 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30305321699999999, 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189435 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979108+0000", 2023-10-09T09:47:57.865 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979108+0000", 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979107+0000", 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0119999999999999e-06 2023-10-09T09:47:57.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979111+0000", 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1892000000000001e-05 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979123+0000", 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5139999999999998e-06 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979129+0000", 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1113999999999999e-05 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979180+0000", 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014542900000000001 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979326+0000", 2023-10-09T09:47:57.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017861299999999999 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.979504+0000", 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016815219999999999 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.996320+0000", 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064391500000000003 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.002759+0000", 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27884090299999997 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.281600+0000", 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4296999999999998e-05 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.870 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.281664+0000", 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00049690200000000002 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.282161+0000", 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30305321699999999 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.871 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:189436 8.4 8:29159f79:::benchmark_data_smithi163_151167_object94203:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1166)", 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:26.984927+0000", 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 510.81004101399998, 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.297717699, 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 189436 2023-10-09T09:47:57.872 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.873 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.873 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.873 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.873 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984927+0000", 2023-10-09T09:47:57.873 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.873 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.873 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.873 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.873 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984927+0000", 2023-10-09T09:47:57.873 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984926+0000", 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7389999999999999e-06 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984933+0000", 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3361e-05 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.874 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984947+0000", 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1429999999999998e-06 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.984955+0000", 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043666160000000002 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989322+0000", 2023-10-09T09:47:57.875 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000151569 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989473+0000", 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000155713 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.989629+0000", 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096470640000000003 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.876 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:26.999276+0000", 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0035051119999999999 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.002781+0000", 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27945128699999999 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.877 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.282232+0000", 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6711000000000001e-05 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.282279+0000", 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00036571900000000003 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:27.282645+0000", 2023-10-09T09:47:57.878 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.297717699 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:93410 9.c 9:3dbf5f69:::benchmark_data_smithi163_175771_object93409:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1204)", 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:42:24.586043+0000", 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.20892495499999, 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.315621239, 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.879 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93410 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.586043+0000", 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.880 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.586043+0000", 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.586042+0000", 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8545000000000003e-05 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.881 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.586100+0000", 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1198e-05 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.586111+0000", 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.878e-06 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.586118+0000", 2023-10-09T09:47:57.882 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015614660000000001 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.587680+0000", 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014298300000000001 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.587823+0000", 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016608200000000001 2023-10-09T09:47:57.883 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.587989+0000", 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003744402 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.591733+0000", 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000178686 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.884 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.591912+0000", 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30959297499999999 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901505+0000", 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3024999999999998e-05 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901598+0000", 2023-10-09T09:47:57.885 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6476999999999994e-05 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901664+0000", 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.315621239 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.886 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:93421 9.c 9:32fbb122:::benchmark_data_smithi163_175771_object93420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1204)", 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:42:24.594091+0000", 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.20087663599998, 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30804981799999998, 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93421 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.887 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.594091+0000", 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.594091+0000", 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.594090+0000", 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8739000000000002e-05 2023-10-09T09:47:57.888 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.594138+0000", 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1649e-05 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.594150+0000", 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1279999999999996e-06 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.889 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.594156+0000", 2023-10-09T09:47:57.890 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00036638699999999997 2023-10-09T09:47:57.890 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.891 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.891 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.891 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.594523+0000", 2023-10-09T09:47:57.891 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000127 2023-10-09T09:47:57.891 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.891 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.891 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.891 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.594650+0000", 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017091 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.594821+0000", 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039216140000000003 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.598742+0000", 2023-10-09T09:47:57.892 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30211430900000003 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900856+0000", 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00096693300000000001 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901823+0000", 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7560999999999999e-05 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.893 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.894 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901891+0000", 2023-10-09T09:47:57.894 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00025025999999999998 2023-10-09T09:47:57.894 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.894 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.894 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.894 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.902141+0000", 2023-10-09T09:47:57.894 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30804981799999998 2023-10-09T09:47:57.894 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.895 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.895 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.895 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.895 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.895 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:93423 9.e 9:745f2b96:::benchmark_data_smithi163_175771_object93422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1204)", 2023-10-09T09:47:57.895 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:42:24.598287+0000", 2023-10-09T09:47:57.895 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.196680786, 2023-10-09T09:47:57.895 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30642129600000001, 2023-10-09T09:47:57.895 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.895 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.896 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.896 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:47:57.896 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:47:57.896 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93423 2023-10-09T09:47:57.896 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.896 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.896 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.896 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.598287+0000", 2023-10-09T09:47:57.896 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.598287+0000", 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.598285+0000", 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3837999999999994e-05 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.897 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.598349+0000", 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4007e-05 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.598363+0000", 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.374e-06 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.598371+0000", 2023-10-09T09:47:57.899 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9234999999999999e-05 2023-10-09T09:47:57.899 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.598420+0000", 2023-10-09T09:47:57.899 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115968 2023-10-09T09:47:57.899 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.598536+0000", 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010830100000000001 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.598644+0000", 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028768940000000001 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601521+0000", 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001879969 2023-10-09T09:47:57.900 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.603401+0000", 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30114374199999999 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.904545+0000", 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000122648 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.901 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.904667+0000", 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1168999999999998e-05 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.904709+0000", 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30642129600000001 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.902 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:93426 9.c 9:33432db0:::benchmark_data_smithi163_175771_object93425:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1204)", 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:42:24.601163+0000", 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.19380466000001, 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30184298300000001, 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93426 2023-10-09T09:47:57.903 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601163+0000", 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601163+0000", 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:47:57.904 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601162+0000", 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6740999999999999e-05 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601208+0000", 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3508e-05 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601222+0000", 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.725e-06 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601229+0000", 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1748000000000001e-05 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601270+0000", 2023-10-09T09:47:57.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9863000000000003e-05 2023-10-09T09:47:57.907 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.907 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.907 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601370+0000", 2023-10-09T09:47:57.907 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121699 2023-10-09T09:47:57.907 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.907 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:47:57.907 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601492+0000", 2023-10-09T09:47:57.907 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29953664499999999 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901029+0000", 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000738128 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901767+0000", 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010793739999999999 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.908 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.902846+0000", 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111735 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.902958+0000", 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8639e-05 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.909 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.903006+0000", 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30184298300000001 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop_reply(client.35412.0:93421 9.c e1204/1196 ondisk, result = 0)", 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:42:24.601372+0000", 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.19359580000003, 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29949365, 2023-10-09T09:47:57.910 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601372+0000", 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601372+0000", 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:47:57.911 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601372+0000", 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0900000000000004e-06 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601377+0000", 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4080000000000001e-06 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.912 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601384+0000", 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3970000000000004e-06 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.601388+0000", 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29902559400000001 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900414+0000", 2023-10-09T09:47:57.913 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8391000000000001e-05 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900432+0000", 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00043350099999999998 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900866+0000", 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29949365 2023-10-09T09:47:57.914 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:93431 9.e 9:775c2da9:::benchmark_data_smithi163_175771_object93430:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1204)", 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:42:24.604102+0000", 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.190865895, 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30977222599999998, 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:47:57.915 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93431 2023-10-09T09:47:57.916 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.916 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.916 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.916 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.916 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604102+0000", 2023-10-09T09:47:57.916 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.916 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.916 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.916 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.916 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604102+0000", 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604101+0000", 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9194000000000003e-05 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604160+0000", 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1542999999999999e-05 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.917 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604171+0000", 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9689999999999999e-06 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604177+0000", 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7258999999999997e-05 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604215+0000", 2023-10-09T09:47:57.918 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014666899999999999 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604361+0000", 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29617737500000002 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900539+0000", 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011651514999999999 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.919 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.912190+0000", 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9650000000000002e-06 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.912197+0000", 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015515489999999999 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.920 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.913749+0000", 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0795000000000002e-05 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.913819+0000", 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5018000000000001e-05 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.913874+0000", 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30977222599999998 2023-10-09T09:47:57.921 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop_reply(client.35412.0:93426 9.c e1204/1196 ondisk, result = 0)", 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:42:24.604744+0000", 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.19022396700001, 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29628689000000002, 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T09:47:57.922 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604744+0000", 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604744+0000", 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.923 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604743+0000", 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.438e-06 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.604750+0000", 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29562906500000002 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900379+0000", 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7105999999999999e-05 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900396+0000", 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00061688800000000001 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901013+0000", 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7859999999999997e-06 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901021+0000", 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0417999999999999e-05 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901031+0000", 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29628689000000002 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop_reply(client.35412.0:93410 9.c e1204/1196 ondisk, result = 0)", 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:42:24.606815+0000", 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.188153371, 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29495763800000002, 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.606815+0000", 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.927 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.606815+0000", 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.606814+0000", 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8810000000000001e-06 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.606820+0000", 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29359949800000001 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900419+0000", 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2969999999999998e-06 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900427+0000", 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001058408 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901485+0000", 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0890000000000006e-06 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901494+0000", 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00027830599999999997 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.901772+0000", 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29495763800000002 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:93440 9.e 9:7c572481:::benchmark_data_smithi163_175771_object93439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1204)", 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:42:24.613897+0000", 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.18107152900001, 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30259003299999998, 2023-10-09T09:47:57.931 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93440 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.613897+0000", 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.932 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.613897+0000", 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.613895+0000", 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4418000000000001e-05 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.934 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.613939+0000", 2023-10-09T09:47:57.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.286475759 2023-10-09T09:47:57.934 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.934 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900415+0000", 2023-10-09T09:47:57.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5021000000000001e-05 2023-10-09T09:47:57.934 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.900430+0000", 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043781749999999998 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.904808+0000", 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011995200000000001 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.904928+0000", 2023-10-09T09:47:57.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012588000000000001 2023-10-09T09:47:57.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.936 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:47:57.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.905054+0000", 2023-10-09T09:47:57.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.007148504 2023-10-09T09:47:57.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.936 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.912202+0000", 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017162709999999999 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.913919+0000", 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024638609999999999 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.916382+0000", 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9905000000000003e-05 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.916442+0000", 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4255000000000002e-05 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:24.916487+0000", 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30259003299999998 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:106685 9.e 9:73181117:::benchmark_data_smithi163_175771_object2789:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1204)", 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:42:37.941730+0000", 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 319.85323789300003, 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27805152300000002, 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.939 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 106685 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.941730+0000", 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.941730+0000", 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.941728+0000", 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6060000000000003e-06 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.941737+0000", 2023-10-09T09:47:57.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9184000000000001e-05 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.941756+0000", 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.636e-06 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.941765+0000", 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1226e-05 2023-10-09T09:47:57.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.941826+0000", 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015941999999999999 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.941986+0000", 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018642100000000001 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:47:57.943 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.942172+0000", 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0037984410000000001 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.945970+0000", 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00099648899999999993 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:37.946967+0000", 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27259845500000002 2023-10-09T09:47:57.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:38.219565+0000", 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120557 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:38.219686+0000", 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5773000000000006e-05 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:42:38.219782+0000", 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27805152300000002 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45545.0:138218 10.1 10:84bcbc85:::benchmark_data_smithi163_194945_object28765:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1244)", 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:47:51.046887+0000", 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 6.7480814630000001, 2023-10-09T09:47:57.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.107712603, 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45545", 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 138218 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.046887+0000", 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:47:57.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.046887+0000", 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.046885+0000", 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6429999999999995e-06 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.046893+0000", 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7359000000000001e-05 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.046910+0000", 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7530000000000001e-06 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.046918+0000", 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015417499999999999 2023-10-09T09:47:57.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.047072+0000", 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013685699999999999 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.047209+0000", 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00025199599999999999 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0", 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.047461+0000", 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039971360000000001 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.051458+0000", 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10280159799999999 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.154259+0000", 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000166766 2023-10-09T09:47:57.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.154426+0000", 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017299200000000001 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:47:51.154599+0000", 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.107712603 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.952 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.953 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.953 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:47:57.953 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:47:57.953 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:47:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:57.955 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:57.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:57.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:57 smithi163 bash[78107]: cluster 2023-10-09T09:47:56.119493+0000 mgr.y (mgr.34107) 1259 : cluster 0 pgmap v2098: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 24974/238029 objects degraded (10.492%) 2023-10-09T09:47:57.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:57 smithi163 bash[75425]: cluster 2023-10-09T09:47:56.119493+0000 mgr.y (mgr.34107) 1259 : cluster 0 pgmap v2098: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 24974/238029 objects degraded (10.492%) 2023-10-09T09:47:57.988 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T09:47:57.989 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2023-10-09T09:47:58.055 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:58.083 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:58.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:58.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:58.089 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:57 smithi181 bash[59345]: cluster 2023-10-09T09:47:56.119493+0000 mgr.y (mgr.34107) 1259 : cluster 0 pgmap v2098: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 24974/238029 objects degraded (10.492%) 2023-10-09T09:47:58.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:58.228 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:47:58.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:58.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:47:58.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:58.356 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:58.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:58.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:58.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:58.495 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:58.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:58.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:58.600 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:58.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:47:58.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:58.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:47:58.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:58.772 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:58.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:58.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:58.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:58.922 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:47:58.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:58.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:47:58.950 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T09:47:59.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:59.080 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:59.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:59.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:59.184 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:59.220 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:47:59.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:59.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:47:59.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:59.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:59.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:59.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:59.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:59.488 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:59.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:59.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:59.590 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:59.614 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:59.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:59.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:59.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:59.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:47:59.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:59.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:47:59.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:59.875 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:47:59.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:47:59.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:47:59.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:00.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:00.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:00.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:47:59 smithi181 bash[59345]: cluster 2023-10-09T09:47:58.120379+0000 mgr.y (mgr.34107) 1260 : cluster 0 pgmap v2099: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 834 op/s; 23626/228291 objects degraded (10.349%) 2023-10-09T09:48:00.128 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:00.159 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:00.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:00.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:47:59 smithi163 bash[75425]: cluster 2023-10-09T09:47:58.120379+0000 mgr.y (mgr.34107) 1260 : cluster 0 pgmap v2099: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 834 op/s; 23626/228291 objects degraded (10.349%) 2023-10-09T09:48:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:47:59 smithi163 bash[78107]: cluster 2023-10-09T09:47:58.120379+0000 mgr.y (mgr.34107) 1260 : cluster 0 pgmap v2099: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 834 op/s; 23626/228291 objects degraded (10.349%) 2023-10-09T09:48:00.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:00.299 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:00.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:00.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:00.402 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:00.426 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:00.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:00.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:00.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:00.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:00.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:00.663 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:00.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:00.694 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:00.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:00.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:00.820 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:00.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:00.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:00.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:00.932 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:00 smithi181 bash[59345]: cluster 2023-10-09T09:48:00.860759+0000 mon.a (mon.0) 2322 : cluster 3 Health check update: Degraded data redundancy: 23626/228291 objects degraded (10.349%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:00.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:00.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:00.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:01.063 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:01.099 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:01.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:01.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:01.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:01.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:00 smithi163 bash[78107]: cluster 2023-10-09T09:48:00.860759+0000 mon.a (mon.0) 2322 : cluster 3 Health check update: Degraded data redundancy: 23626/228291 objects degraded (10.349%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:01.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:00 smithi163 bash[75425]: cluster 2023-10-09T09:48:00.860759+0000 mon.a (mon.0) 2322 : cluster 3 Health check update: Degraded data redundancy: 23626/228291 objects degraded (10.349%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:01.235 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:01.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:01.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:01.341 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:01.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:01.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:01.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:01.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:01.511 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:01.514 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:01.515 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:01.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:01.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:01.617 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:01.669 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:01.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:01.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:01.774 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:01.800 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:01.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:01.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:01.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:01.910 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:01 smithi163 bash[78107]: cluster 2023-10-09T09:48:00.120907+0000 mgr.y (mgr.34107) 1261 : cluster 0 pgmap v2100: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 834 op/s; 23626/228291 objects degraded (10.349%) 2023-10-09T09:48:01.911 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:01 smithi163 bash[75425]: cluster 2023-10-09T09:48:00.120907+0000 mgr.y (mgr.34107) 1261 : cluster 0 pgmap v2100: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 834 op/s; 23626/228291 objects degraded (10.349%) 2023-10-09T09:48:01.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:01.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:01.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:02.056 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:01 smithi181 bash[59345]: cluster 2023-10-09T09:48:00.120907+0000 mgr.y (mgr.34107) 1261 : cluster 0 pgmap v2100: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 834 op/s; 23626/228291 objects degraded (10.349%) 2023-10-09T09:48:02.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:02.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:02.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:02.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:02.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:02.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:02.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:02.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:02.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:02.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:02.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:02.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:02.540 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:02.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:02.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:02.693 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:02.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:02.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:02.796 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:02.838 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:02.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:02.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:02.943 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:02.973 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:02.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:02.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:02.989 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:48:03.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:03.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:03.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:03.213 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:03.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:03.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:03.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:03.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:03.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:03.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:03.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:03.530 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:03.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:03.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:03.634 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:03.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:03.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:03.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:48:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:48:03] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.33.4" 2023-10-09T09:48:03.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:03.799 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:03.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:03.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:03.828 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:48:03.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:03.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:03.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:03.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:04.039 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:04.064 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:04.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:03 smithi181 bash[59345]: cluster 2023-10-09T09:48:02.121564+0000 mgr.y (mgr.34107) 1262 : cluster 0 pgmap v2101: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 982 op/s; 22521/217038 objects degraded (10.377%) 2023-10-09T09:48:04.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:04.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:04.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:04.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:03 smithi163 bash[75425]: cluster 2023-10-09T09:48:02.121564+0000 mgr.y (mgr.34107) 1262 : cluster 0 pgmap v2101: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 982 op/s; 22521/217038 objects degraded (10.377%) 2023-10-09T09:48:04.178 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:03 smithi163 bash[78107]: cluster 2023-10-09T09:48:02.121564+0000 mgr.y (mgr.34107) 1262 : cluster 0 pgmap v2101: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 982 op/s; 22521/217038 objects degraded (10.377%) 2023-10-09T09:48:04.196 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:04.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:04.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:04.329 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:04.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:04.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:04.432 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:04.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:04.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:04.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:04.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:04.608 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:04.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:04.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:04.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:04.736 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:04.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:04.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:04.839 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:04.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:04.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:04.972 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:05.006 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:05.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:05.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:05.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:05.147 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:05.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:05.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:05.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:05.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:05.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:05.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:05.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:05.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:05.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:05.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:05.534 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:05.554 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:05.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:05.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:05.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:05.684 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:05.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:05.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:05.718 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:48:05 smithi163 bash[204298]: debug 2023-10-09T09:48:05.609+0000 7f90965ad640 -1 osd.1 1241 log_to_monitors true 2023-10-09T09:48:05.788 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:05.815 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:05.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:05.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:05.924 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:05.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:05 smithi181 bash[59345]: cluster 2023-10-09T09:48:04.122294+0000 mgr.y (mgr.34107) 1263 : cluster 0 pgmap v2102: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 22252/215439 objects degraded (10.329%) 2023-10-09T09:48:05.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:05 smithi181 bash[59345]: audit 2023-10-09T09:48:05.617021+0000 mon.c (mon.2) 114 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3739078161,v1:172.21.15.163:6811/3739078161]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:48:05.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:05 smithi181 bash[59345]: audit 2023-10-09T09:48:05.617329+0000 mon.a (mon.0) 2323 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:48:05.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:05 smithi181 bash[59345]: cluster 2023-10-09T09:48:05.861900+0000 mon.a (mon.0) 2324 : cluster 3 Health check update: Degraded data redundancy: 22252/215439 objects degraded (10.329%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:05.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:05.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:05.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:06.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:06.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:06.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:06.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:06.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:06.218 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 09:48:05 smithi163 bash[204298]: debug 2023-10-09T09:48:05.989+0000 7f9086a17700 -1 osd.1 1241 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:48:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:05 smithi163 bash[78107]: cluster 2023-10-09T09:48:04.122294+0000 mgr.y (mgr.34107) 1263 : cluster 0 pgmap v2102: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 22252/215439 objects degraded (10.329%) 2023-10-09T09:48:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:05 smithi163 bash[78107]: audit 2023-10-09T09:48:05.617021+0000 mon.c (mon.2) 114 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3739078161,v1:172.21.15.163:6811/3739078161]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:48:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:05 smithi163 bash[78107]: audit 2023-10-09T09:48:05.617329+0000 mon.a (mon.0) 2323 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:48:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:05 smithi163 bash[78107]: cluster 2023-10-09T09:48:05.861900+0000 mon.a (mon.0) 2324 : cluster 3 Health check update: Degraded data redundancy: 22252/215439 objects degraded (10.329%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:05 smithi163 bash[75425]: cluster 2023-10-09T09:48:04.122294+0000 mgr.y (mgr.34107) 1263 : cluster 0 pgmap v2102: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 22252/215439 objects degraded (10.329%) 2023-10-09T09:48:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:05 smithi163 bash[75425]: audit 2023-10-09T09:48:05.617021+0000 mon.c (mon.2) 114 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3739078161,v1:172.21.15.163:6811/3739078161]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:48:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:05 smithi163 bash[75425]: audit 2023-10-09T09:48:05.617329+0000 mon.a (mon.0) 2323 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:48:06.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:05 smithi163 bash[75425]: cluster 2023-10-09T09:48:05.861900+0000 mon.a (mon.0) 2324 : cluster 3 Health check update: Degraded data redundancy: 22252/215439 objects degraded (10.329%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:06.233 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:06.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:06.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:06.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:06.372 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:06.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:06.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:06.480 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:06.517 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:06.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:06.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:06.621 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:06.662 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:06.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:06.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:06.770 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:06.808 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:06.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:06.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:06.913 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:06.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:06.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:06.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:07.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:07.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:07.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:07.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:07.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:07.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:07.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:07.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:07.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[78107]: cluster 2023-10-09T09:48:05.917322+0000 osd.5 (osd.5) 833 : cluster 3 failed to encode map e1245 with expected crc 2023-10-09T09:48:07.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[78107]: audit 2023-10-09T09:48:05.918431+0000 mon.a (mon.0) 2325 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:48:07.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[78107]: cluster 2023-10-09T09:48:05.918535+0000 mon.a (mon.0) 2326 : cluster 0 osdmap e1245: 8 total, 7 up, 6 in 2023-10-09T09:48:07.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[78107]: audit 2023-10-09T09:48:05.922010+0000 mon.c (mon.2) 115 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3739078161,v1:172.21.15.163:6811/3739078161]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:48:07.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[78107]: cluster 2023-10-09T09:48:05.925058+0000 osd.4 (osd.4) 651 : cluster 3 failed to encode map e1245 with expected crc 2023-10-09T09:48:07.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[78107]: cluster 2023-10-09T09:48:05.925299+0000 osd.6 (osd.6) 529 : cluster 3 failed to encode map e1245 with expected crc 2023-10-09T09:48:07.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[78107]: audit 2023-10-09T09:48:05.934120+0000 mon.a (mon.0) 2327 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:48:07.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[75425]: cluster 2023-10-09T09:48:05.917322+0000 osd.5 (osd.5) 833 : cluster 3 failed to encode map e1245 with expected crc 2023-10-09T09:48:07.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[75425]: audit 2023-10-09T09:48:05.918431+0000 mon.a (mon.0) 2325 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:48:07.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[75425]: cluster 2023-10-09T09:48:05.918535+0000 mon.a (mon.0) 2326 : cluster 0 osdmap e1245: 8 total, 7 up, 6 in 2023-10-09T09:48:07.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[75425]: audit 2023-10-09T09:48:05.922010+0000 mon.c (mon.2) 115 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3739078161,v1:172.21.15.163:6811/3739078161]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:48:07.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[75425]: cluster 2023-10-09T09:48:05.925058+0000 osd.4 (osd.4) 651 : cluster 3 failed to encode map e1245 with expected crc 2023-10-09T09:48:07.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[75425]: cluster 2023-10-09T09:48:05.925299+0000 osd.6 (osd.6) 529 : cluster 3 failed to encode map e1245 with expected crc 2023-10-09T09:48:07.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:06 smithi163 bash[75425]: audit 2023-10-09T09:48:05.934120+0000 mon.a (mon.0) 2327 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:48:07.307 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:07.391 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: cluster 2023-10-09T09:48:05.917322+0000 osd.5 (osd.5) 833 : cluster 3 failed to encode map e1245 with expected crc 2023-10-09T09:48:07.391 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: audit 2023-10-09T09:48:05.918431+0000 mon.a (mon.0) 2325 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:48:07.391 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: cluster 2023-10-09T09:48:05.918535+0000 mon.a (mon.0) 2326 : cluster 0 osdmap e1245: 8 total, 7 up, 6 in 2023-10-09T09:48:07.392 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: audit 2023-10-09T09:48:05.922010+0000 mon.c (mon.2) 115 : audit 1 from='osd.1 [v2:172.21.15.163:6810/3739078161,v1:172.21.15.163:6811/3739078161]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:48:07.392 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: cluster 2023-10-09T09:48:05.925058+0000 osd.4 (osd.4) 651 : cluster 3 failed to encode map e1245 with expected crc 2023-10-09T09:48:07.392 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: cluster 2023-10-09T09:48:05.925299+0000 osd.6 (osd.6) 529 : cluster 3 failed to encode map e1245 with expected crc 2023-10-09T09:48:07.392 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: audit 2023-10-09T09:48:05.934120+0000 mon.a (mon.0) 2327 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T09:48:07.427 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:07.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:07.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:07.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:07.561 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:07.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:07.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:07.665 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:07.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:07.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:07.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:07.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:07.832 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:07.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:07.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:07.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:07.962 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: cluster 2023-10-09T09:48:06.122753+0000 mgr.y (mgr.34107) 1264 : cluster 0 pgmap v2104: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 861 op/s; 22252/212184 objects degraded (10.487%) 2023-10-09T09:48:07.962 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: cluster 2023-10-09T09:48:06.924405+0000 mon.a (mon.0) 2328 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:48:07.962 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: cluster 2023-10-09T09:48:06.927195+0000 osd.5 (osd.5) 834 : cluster 3 failed to encode map e1246 with expected crc 2023-10-09T09:48:07.962 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: cluster 2023-10-09T09:48:06.930062+0000 mon.a (mon.0) 2329 : cluster 1 osd.1 [v2:172.21.15.163:6810/3739078161,v1:172.21.15.163:6811/3739078161] boot 2023-10-09T09:48:07.962 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: cluster 2023-10-09T09:48:06.930437+0000 mon.a (mon.0) 2330 : cluster 0 osdmap e1246: 8 total, 8 up, 6 in 2023-10-09T09:48:07.962 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:07 smithi181 bash[59345]: audit 2023-10-09T09:48:06.936890+0000 mon.a (mon.0) 2331 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:48:07.971 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:07.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:08.075 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:08.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[78107]: cluster 2023-10-09T09:48:06.122753+0000 mgr.y (mgr.34107) 1264 : cluster 0 pgmap v2104: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 861 op/s; 22252/212184 objects degraded (10.487%) 2023-10-09T09:48:08.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[78107]: cluster 2023-10-09T09:48:06.924405+0000 mon.a (mon.0) 2328 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:48:08.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[78107]: cluster 2023-10-09T09:48:06.927195+0000 osd.5 (osd.5) 834 : cluster 3 failed to encode map e1246 with expected crc 2023-10-09T09:48:08.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[78107]: cluster 2023-10-09T09:48:06.930062+0000 mon.a (mon.0) 2329 : cluster 1 osd.1 [v2:172.21.15.163:6810/3739078161,v1:172.21.15.163:6811/3739078161] boot 2023-10-09T09:48:08.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[78107]: cluster 2023-10-09T09:48:06.930437+0000 mon.a (mon.0) 2330 : cluster 0 osdmap e1246: 8 total, 8 up, 6 in 2023-10-09T09:48:08.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[78107]: audit 2023-10-09T09:48:06.936890+0000 mon.a (mon.0) 2331 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:48:08.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[75425]: cluster 2023-10-09T09:48:06.122753+0000 mgr.y (mgr.34107) 1264 : cluster 0 pgmap v2104: 89 pgs: 41 active+undersized, 11 active+undersized+degraded, 37 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 861 op/s; 22252/212184 objects degraded (10.487%) 2023-10-09T09:48:08.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[75425]: cluster 2023-10-09T09:48:06.924405+0000 mon.a (mon.0) 2328 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:48:08.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[75425]: cluster 2023-10-09T09:48:06.927195+0000 osd.5 (osd.5) 834 : cluster 3 failed to encode map e1246 with expected crc 2023-10-09T09:48:08.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[75425]: cluster 2023-10-09T09:48:06.930062+0000 mon.a (mon.0) 2329 : cluster 1 osd.1 [v2:172.21.15.163:6810/3739078161,v1:172.21.15.163:6811/3739078161] boot 2023-10-09T09:48:08.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[75425]: cluster 2023-10-09T09:48:06.930437+0000 mon.a (mon.0) 2330 : cluster 0 osdmap e1246: 8 total, 8 up, 6 in 2023-10-09T09:48:08.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:07 smithi163 bash[75425]: audit 2023-10-09T09:48:06.936890+0000 mon.a (mon.0) 2331 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:48:08.098 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:08.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:08.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:08.230 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:08.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:08.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:08.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:08.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:08.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:08.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:08.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:08.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:08.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:08.601 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:08.625 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:08.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:08.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:08.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:08.760 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:08.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:08.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:08.796 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:48:08.821 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:48:08.846 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:48:08.863 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:08.869 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:48:08.889 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:08.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:08.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:08.892 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:48:08.916 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:48:08.943 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:48:08.943 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:48:08.944 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:48:08.975 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:48:08.980 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:48:08.980 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:48:08.981 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:48:08.981 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:48:08.981 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:48:08.981 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:48:08.981 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:48:08.981 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:48:08.992 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:09.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:09.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:09.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:09.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:09.152 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:09.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[75425]: cluster 2023-10-09T09:48:07.946322+0000 osd.5 (osd.5) 835 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[75425]: cluster 2023-10-09T09:48:07.948615+0000 mon.a (mon.0) 2332 : cluster 0 osdmap e1247: 8 total, 8 up, 6 in 2023-10-09T09:48:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[75425]: cluster 2023-10-09T09:48:07.949220+0000 osd.5 (osd.5) 836 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[75425]: cluster 2023-10-09T09:48:07.953608+0000 osd.4 (osd.4) 652 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[75425]: cluster 2023-10-09T09:48:07.954798+0000 osd.6 (osd.6) 530 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[75425]: cluster 2023-10-09T09:48:07.956204+0000 osd.6 (osd.6) 531 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[75425]: audit 2023-10-09T09:48:08.770185+0000 mon.c (mon.2) 116 : audit 0 from='client.? 172.21.15.163:0/2351745007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:48:09.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[78107]: cluster 2023-10-09T09:48:07.946322+0000 osd.5 (osd.5) 835 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[78107]: cluster 2023-10-09T09:48:07.948615+0000 mon.a (mon.0) 2332 : cluster 0 osdmap e1247: 8 total, 8 up, 6 in 2023-10-09T09:48:09.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[78107]: cluster 2023-10-09T09:48:07.949220+0000 osd.5 (osd.5) 836 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[78107]: cluster 2023-10-09T09:48:07.953608+0000 osd.4 (osd.4) 652 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[78107]: cluster 2023-10-09T09:48:07.954798+0000 osd.6 (osd.6) 530 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[78107]: cluster 2023-10-09T09:48:07.956204+0000 osd.6 (osd.6) 531 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:08 smithi163 bash[78107]: audit 2023-10-09T09:48:08.770185+0000 mon.c (mon.2) 116 : audit 0 from='client.? 172.21.15.163:0/2351745007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:48:09.256 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:09.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:09.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:09.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:09.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:08 smithi181 bash[59345]: cluster 2023-10-09T09:48:07.946322+0000 osd.5 (osd.5) 835 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:08 smithi181 bash[59345]: cluster 2023-10-09T09:48:07.948615+0000 mon.a (mon.0) 2332 : cluster 0 osdmap e1247: 8 total, 8 up, 6 in 2023-10-09T09:48:09.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:08 smithi181 bash[59345]: cluster 2023-10-09T09:48:07.949220+0000 osd.5 (osd.5) 836 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:08 smithi181 bash[59345]: cluster 2023-10-09T09:48:07.953608+0000 osd.4 (osd.4) 652 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:08 smithi181 bash[59345]: cluster 2023-10-09T09:48:07.954798+0000 osd.6 (osd.6) 530 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:08 smithi181 bash[59345]: cluster 2023-10-09T09:48:07.956204+0000 osd.6 (osd.6) 531 : cluster 3 failed to encode map e1247 with expected crc 2023-10-09T09:48:09.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:08 smithi181 bash[59345]: audit 2023-10-09T09:48:08.770185+0000 mon.c (mon.2) 116 : audit 0 from='client.? 172.21.15.163:0/2351745007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:48:09.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:09.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:09.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:09.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:09.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:09.551 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:09.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:09.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:09.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:09.686 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:09.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:09.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:09.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:09.819 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T09:48:09.829 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:09.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:09.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:09.933 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:09.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:09.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:09.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:09.994 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:48:10.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:10.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[78107]: cluster 2023-10-09T09:48:08.123615+0000 mgr.y (mgr.34107) 1265 : cluster 0 pgmap v2107: 89 pgs: 3 remapped+peering, 36 active+undersized, 5 active+undersized+degraded, 45 active+clean; 4.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 758 op/s; 8346/203373 objects degraded (4.104%) 2023-10-09T09:48:10.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[78107]: cluster 2023-10-09T09:48:08.967553+0000 osd.5 (osd.5) 837 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[78107]: cluster 2023-10-09T09:48:08.968986+0000 osd.6 (osd.6) 532 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[78107]: cluster 2023-10-09T09:48:08.970467+0000 mon.a (mon.0) 2333 : cluster 0 osdmap e1248: 8 total, 8 up, 6 in 2023-10-09T09:48:10.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[78107]: cluster 2023-10-09T09:48:08.973984+0000 osd.6 (osd.6) 533 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[78107]: cluster 2023-10-09T09:48:08.976457+0000 osd.6 (osd.6) 534 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[78107]: audit 2023-10-09T09:48:09.024778+0000 mon.a (mon.0) 2334 : audit 1 from='client.? 172.21.15.163:0/2103707815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:48:10.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[75425]: cluster 2023-10-09T09:48:08.123615+0000 mgr.y (mgr.34107) 1265 : cluster 0 pgmap v2107: 89 pgs: 3 remapped+peering, 36 active+undersized, 5 active+undersized+degraded, 45 active+clean; 4.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 758 op/s; 8346/203373 objects degraded (4.104%) 2023-10-09T09:48:10.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[75425]: cluster 2023-10-09T09:48:08.967553+0000 osd.5 (osd.5) 837 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[75425]: cluster 2023-10-09T09:48:08.968986+0000 osd.6 (osd.6) 532 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[75425]: cluster 2023-10-09T09:48:08.970467+0000 mon.a (mon.0) 2333 : cluster 0 osdmap e1248: 8 total, 8 up, 6 in 2023-10-09T09:48:10.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[75425]: cluster 2023-10-09T09:48:08.973984+0000 osd.6 (osd.6) 533 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[75425]: cluster 2023-10-09T09:48:08.976457+0000 osd.6 (osd.6) 534 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:09 smithi163 bash[75425]: audit 2023-10-09T09:48:09.024778+0000 mon.a (mon.0) 2334 : audit 1 from='client.? 172.21.15.163:0/2103707815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:48:10.111 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:10.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:10.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:10.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:10.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:09 smithi181 bash[59345]: cluster 2023-10-09T09:48:08.123615+0000 mgr.y (mgr.34107) 1265 : cluster 0 pgmap v2107: 89 pgs: 3 remapped+peering, 36 active+undersized, 5 active+undersized+degraded, 45 active+clean; 4.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 758 op/s; 8346/203373 objects degraded (4.104%) 2023-10-09T09:48:10.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:09 smithi181 bash[59345]: cluster 2023-10-09T09:48:08.967553+0000 osd.5 (osd.5) 837 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:09 smithi181 bash[59345]: cluster 2023-10-09T09:48:08.968986+0000 osd.6 (osd.6) 532 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:09 smithi181 bash[59345]: cluster 2023-10-09T09:48:08.970467+0000 mon.a (mon.0) 2333 : cluster 0 osdmap e1248: 8 total, 8 up, 6 in 2023-10-09T09:48:10.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:09 smithi181 bash[59345]: cluster 2023-10-09T09:48:08.973984+0000 osd.6 (osd.6) 533 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:09 smithi181 bash[59345]: cluster 2023-10-09T09:48:08.976457+0000 osd.6 (osd.6) 534 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:10.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:09 smithi181 bash[59345]: audit 2023-10-09T09:48:09.024778+0000 mon.a (mon.0) 2334 : audit 1 from='client.? 172.21.15.163:0/2103707815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:48:10.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:10.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:10.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:10.353 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:10.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:10.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:10.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:10.485 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:10.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:10.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:10.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:10.625 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:10.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:10.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:10.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:10.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:10.791 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:10.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:10.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:10.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:10.921 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:10.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:10.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:11.025 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:11.056 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:11.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:11.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:11.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:11.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:10 smithi181 bash[59345]: cluster 2023-10-09T09:48:09.967032+0000 mon.a (mon.0) 2335 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:48:11.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:10 smithi181 bash[59345]: cluster 2023-10-09T09:48:09.984198+0000 osd.5 (osd.5) 838 : cluster 3 failed to encode map e1249 with expected crc 2023-10-09T09:48:11.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:10 smithi181 bash[59345]: cluster 2023-10-09T09:48:09.985470+0000 osd.6 (osd.6) 535 : cluster 3 failed to encode map e1249 with expected crc 2023-10-09T09:48:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:10 smithi181 bash[59345]: audit 2023-10-09T09:48:09.987192+0000 mon.a (mon.0) 2336 : audit 1 from='client.? 172.21.15.163:0/2103707815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:48:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:10 smithi181 bash[59345]: cluster 2023-10-09T09:48:09.987442+0000 mon.a (mon.0) 2337 : cluster 0 osdmap e1249: 8 total, 8 up, 6 in 2023-10-09T09:48:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:10 smithi181 bash[59345]: cluster 2023-10-09T09:48:10.048228+0000 osd.7 (osd.7) 599 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:10 smithi181 bash[59345]: audit 2023-10-09T09:48:10.401858+0000 mon.a (mon.0) 2338 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:10 smithi181 bash[59345]: audit 2023-10-09T09:48:10.403106+0000 mon.a (mon.0) 2339 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:10 smithi181 bash[59345]: audit 2023-10-09T09:48:10.845928+0000 mon.a (mon.0) 2340 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:10 smithi181 bash[59345]: cluster 2023-10-09T09:48:10.863030+0000 mon.a (mon.0) 2341 : cluster 3 Health check update: Degraded data redundancy: 8346/203373 objects degraded (4.104%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:11.199 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:11.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:11.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:11.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[75425]: cluster 2023-10-09T09:48:09.967032+0000 mon.a (mon.0) 2335 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:48:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[75425]: cluster 2023-10-09T09:48:09.984198+0000 osd.5 (osd.5) 838 : cluster 3 failed to encode map e1249 with expected crc 2023-10-09T09:48:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[75425]: cluster 2023-10-09T09:48:09.985470+0000 osd.6 (osd.6) 535 : cluster 3 failed to encode map e1249 with expected crc 2023-10-09T09:48:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[75425]: audit 2023-10-09T09:48:09.987192+0000 mon.a (mon.0) 2336 : audit 1 from='client.? 172.21.15.163:0/2103707815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:48:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[75425]: cluster 2023-10-09T09:48:09.987442+0000 mon.a (mon.0) 2337 : cluster 0 osdmap e1249: 8 total, 8 up, 6 in 2023-10-09T09:48:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[75425]: cluster 2023-10-09T09:48:10.048228+0000 osd.7 (osd.7) 599 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[75425]: audit 2023-10-09T09:48:10.401858+0000 mon.a (mon.0) 2338 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[75425]: audit 2023-10-09T09:48:10.403106+0000 mon.a (mon.0) 2339 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[75425]: audit 2023-10-09T09:48:10.845928+0000 mon.a (mon.0) 2340 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[75425]: cluster 2023-10-09T09:48:10.863030+0000 mon.a (mon.0) 2341 : cluster 3 Health check update: Degraded data redundancy: 8346/203373 objects degraded (4.104%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[78107]: cluster 2023-10-09T09:48:09.967032+0000 mon.a (mon.0) 2335 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:48:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[78107]: cluster 2023-10-09T09:48:09.984198+0000 osd.5 (osd.5) 838 : cluster 3 failed to encode map e1249 with expected crc 2023-10-09T09:48:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[78107]: cluster 2023-10-09T09:48:09.985470+0000 osd.6 (osd.6) 535 : cluster 3 failed to encode map e1249 with expected crc 2023-10-09T09:48:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[78107]: audit 2023-10-09T09:48:09.987192+0000 mon.a (mon.0) 2336 : audit 1 from='client.? 172.21.15.163:0/2103707815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:48:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[78107]: cluster 2023-10-09T09:48:09.987442+0000 mon.a (mon.0) 2337 : cluster 0 osdmap e1249: 8 total, 8 up, 6 in 2023-10-09T09:48:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[78107]: cluster 2023-10-09T09:48:10.048228+0000 osd.7 (osd.7) 599 : cluster 3 failed to encode map e1248 with expected crc 2023-10-09T09:48:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[78107]: audit 2023-10-09T09:48:10.401858+0000 mon.a (mon.0) 2338 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[78107]: audit 2023-10-09T09:48:10.403106+0000 mon.a (mon.0) 2339 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[78107]: audit 2023-10-09T09:48:10.845928+0000 mon.a (mon.0) 2340 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:11.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:10 smithi163 bash[78107]: cluster 2023-10-09T09:48:10.863030+0000 mon.a (mon.0) 2341 : cluster 3 Health check update: Degraded data redundancy: 8346/203373 objects degraded (4.104%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:11.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:11.334 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:11.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:11.439 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:11.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:11.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:11.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:11.572 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:11.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:11.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:11.714 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:11.734 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:11.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:11.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:11.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:11.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:11.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:11.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:11.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:12.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:12.032 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:12.036 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:11 smithi163 bash[78107]: cluster 2023-10-09T09:48:10.124175+0000 mgr.y (mgr.34107) 1266 : cluster 0 pgmap v2110: 89 pgs: 3 remapped+peering, 36 active+undersized, 5 active+undersized+degraded, 45 active+clean; 4.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 733 op/s; 8346/203373 objects degraded (4.104%) 2023-10-09T09:48:12.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:12.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:12.037 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:11 smithi163 bash[75425]: cluster 2023-10-09T09:48:10.124175+0000 mgr.y (mgr.34107) 1266 : cluster 0 pgmap v2110: 89 pgs: 3 remapped+peering, 36 active+undersized, 5 active+undersized+degraded, 45 active+clean; 4.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 733 op/s; 8346/203373 objects degraded (4.104%) 2023-10-09T09:48:12.125 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:48:12.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:12.201 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:12.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:12.304 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:11 smithi181 bash[59345]: cluster 2023-10-09T09:48:10.124175+0000 mgr.y (mgr.34107) 1266 : cluster 0 pgmap v2110: 89 pgs: 3 remapped+peering, 36 active+undersized, 5 active+undersized+degraded, 45 active+clean; 4.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 733 op/s; 8346/203373 objects degraded (4.104%) 2023-10-09T09:48:12.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:12.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:12.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:12.478 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:12.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:12.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:12.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:12.617 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:12.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:12.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:12.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:12.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:12.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:12.862 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:12.884 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:12.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:12.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:13.029 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:13.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:13.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:13.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:13.160 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:13.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:13.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:13.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:48:13.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:13.296 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:13.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:13.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:13.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:13.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:13.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:13.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:13.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:13.556 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:48:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:48:13] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.33.4" 2023-10-09T09:48:13.563 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:13.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:13.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:13.671 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:13.698 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:13.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:13.801 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:13.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:13.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:13.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:13.938 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:13.976 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:13.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:13.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:14.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:13 smithi181 bash[59345]: cluster 2023-10-09T09:48:12.124996+0000 mgr.y (mgr.34107) 1267 : cluster 0 pgmap v2111: 89 pgs: 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 3616/199569 objects degraded (1.812%); 0 B/s, 0 objects/s recovering 2023-10-09T09:48:14.084 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:14.112 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:14.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:14.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:14.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:14.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:13 smithi163 bash[75425]: cluster 2023-10-09T09:48:12.124996+0000 mgr.y (mgr.34107) 1267 : cluster 0 pgmap v2111: 89 pgs: 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 3616/199569 objects degraded (1.812%); 0 B/s, 0 objects/s recovering 2023-10-09T09:48:14.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:13 smithi163 bash[78107]: cluster 2023-10-09T09:48:12.124996+0000 mgr.y (mgr.34107) 1267 : cluster 0 pgmap v2111: 89 pgs: 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 3616/199569 objects degraded (1.812%); 0 B/s, 0 objects/s recovering 2023-10-09T09:48:14.249 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:14.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:14.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:14.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:14.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:14.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:14.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:14.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:14.521 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:14.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:14.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:14.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:14.663 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:48:14.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:14.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:14.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:14.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:14.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:14.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:14.897 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:14.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:14.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:14.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:15.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:15.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:15.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:15.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:15.172 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:15.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:15.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:15.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:15.306 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:15.329 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:15.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:15.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:15.433 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:15.465 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:15.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:15.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:15.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:15.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:15.599 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:15.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:15.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:15.730 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:15.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:15.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:15.833 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:15.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:15.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:15.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:15.973 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:16.014 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:16.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:16.117 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:16.154 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:16.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:16.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:16 smithi163 bash[75425]: cluster 2023-10-09T09:48:14.125601+0000 mgr.y (mgr.34107) 1268 : cluster 0 pgmap v2112: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 380 op/s; 7124/197586 objects degraded (3.606%); 0 B/s, 27 objects/s recovering 2023-10-09T09:48:16.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:16 smithi163 bash[75425]: cluster 2023-10-09T09:48:15.868704+0000 mon.a (mon.0) 2342 : cluster 3 Health check update: Degraded data redundancy: 7124/197586 objects degraded (3.606%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:16 smithi163 bash[78107]: cluster 2023-10-09T09:48:14.125601+0000 mgr.y (mgr.34107) 1268 : cluster 0 pgmap v2112: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 380 op/s; 7124/197586 objects degraded (3.606%); 0 B/s, 27 objects/s recovering 2023-10-09T09:48:16.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:16 smithi163 bash[78107]: cluster 2023-10-09T09:48:15.868704+0000 mon.a (mon.0) 2342 : cluster 3 Health check update: Degraded data redundancy: 7124/197586 objects degraded (3.606%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:16.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:16.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:16 smithi181 bash[59345]: cluster 2023-10-09T09:48:14.125601+0000 mgr.y (mgr.34107) 1268 : cluster 0 pgmap v2112: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 380 op/s; 7124/197586 objects degraded (3.606%); 0 B/s, 27 objects/s recovering 2023-10-09T09:48:16.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:16 smithi181 bash[59345]: cluster 2023-10-09T09:48:15.868704+0000 mon.a (mon.0) 2342 : cluster 3 Health check update: Degraded data redundancy: 7124/197586 objects degraded (3.606%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:16.330 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:16.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:16.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:16.434 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:16.476 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:16.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:16.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:16.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:16.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:16.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:16.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:16.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:16.773 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:16.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:16.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:16.826 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:48:16.826 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:48:16.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:16.914 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:16.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:16.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:17.019 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:17.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:17.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:17.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:17.145 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:17.173 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:17.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:17.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:17.282 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:17.313 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:17.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:17.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:17.418 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:17.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:17.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:17.560 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:17.592 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:17.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:17.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:17.699 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:17.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:17.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:17.884 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:17.919 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:17.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:17.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:18.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:18.053 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:18.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:18.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:18.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:18.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:18 smithi181 bash[59345]: cluster 2023-10-09T09:48:16.126325+0000 mgr.y (mgr.34107) 1269 : cluster 0 pgmap v2113: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 362 op/s; 7124/194658 objects degraded (3.660%); 0 B/s, 21 objects/s recovering 2023-10-09T09:48:18.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:18 smithi181 bash[59345]: audit 2023-10-09T09:48:17.203680+0000 mon.a (mon.0) 2343 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:18 smithi181 bash[59345]: audit 2023-10-09T09:48:17.213676+0000 mon.a (mon.0) 2344 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:18 smithi181 bash[59345]: audit 2023-10-09T09:48:17.451792+0000 mon.a (mon.0) 2345 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:18 smithi181 bash[59345]: audit 2023-10-09T09:48:17.460893+0000 mon.a (mon.0) 2346 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.201 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:18.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:18.227 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T09:48:18.305 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:18.331 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:18.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:18.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:18.437 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:18.464 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:18.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:18.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:18.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:18 smithi163 bash[75425]: cluster 2023-10-09T09:48:16.126325+0000 mgr.y (mgr.34107) 1269 : cluster 0 pgmap v2113: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 362 op/s; 7124/194658 objects degraded (3.660%); 0 B/s, 21 objects/s recovering 2023-10-09T09:48:18.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:18 smithi163 bash[75425]: audit 2023-10-09T09:48:17.203680+0000 mon.a (mon.0) 2343 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:18 smithi163 bash[75425]: audit 2023-10-09T09:48:17.213676+0000 mon.a (mon.0) 2344 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:18 smithi163 bash[75425]: audit 2023-10-09T09:48:17.451792+0000 mon.a (mon.0) 2345 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:18 smithi163 bash[75425]: audit 2023-10-09T09:48:17.460893+0000 mon.a (mon.0) 2346 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:18 smithi163 bash[78107]: cluster 2023-10-09T09:48:16.126325+0000 mgr.y (mgr.34107) 1269 : cluster 0 pgmap v2113: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 362 op/s; 7124/194658 objects degraded (3.660%); 0 B/s, 21 objects/s recovering 2023-10-09T09:48:18.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:18 smithi163 bash[78107]: audit 2023-10-09T09:48:17.203680+0000 mon.a (mon.0) 2343 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:18 smithi163 bash[78107]: audit 2023-10-09T09:48:17.213676+0000 mon.a (mon.0) 2344 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:18 smithi163 bash[78107]: audit 2023-10-09T09:48:17.451792+0000 mon.a (mon.0) 2345 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:18 smithi163 bash[78107]: audit 2023-10-09T09:48:17.460893+0000 mon.a (mon.0) 2346 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:18.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:18.604 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:18.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:18.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:18.655 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:18.655 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:18.707 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:18.738 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:18.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:18.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:18.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:18.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:18.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:18.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:18.990 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:19.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:19.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:19.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:19.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:19.128 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:19 smithi181 bash[59345]: audit 2023-10-09T09:48:18.109920+0000 mon.a (mon.0) 2347 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:19.128 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:19 smithi181 bash[59345]: audit 2023-10-09T09:48:18.111174+0000 mon.a (mon.0) 2348 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:19.128 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:19 smithi181 bash[59345]: audit 2023-10-09T09:48:18.120716+0000 mon.a (mon.0) 2349 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:19.160 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:19.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:19.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:19.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:19.291 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:19.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:19.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:19.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:19.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:19.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:19.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:19 smithi163 bash[78107]: audit 2023-10-09T09:48:18.109920+0000 mon.a (mon.0) 2347 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:19.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:19 smithi163 bash[78107]: audit 2023-10-09T09:48:18.111174+0000 mon.a (mon.0) 2348 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:19.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:19 smithi163 bash[78107]: audit 2023-10-09T09:48:18.120716+0000 mon.a (mon.0) 2349 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:19.441 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:19 smithi163 bash[75425]: audit 2023-10-09T09:48:18.109920+0000 mon.a (mon.0) 2347 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:19.441 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:19 smithi163 bash[75425]: audit 2023-10-09T09:48:18.111174+0000 mon.a (mon.0) 2348 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:19.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:19 smithi163 bash[75425]: audit 2023-10-09T09:48:18.120716+0000 mon.a (mon.0) 2349 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:19.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:19.557 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:19.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:19.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:19.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:19.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:19.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:19.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:19.826 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:19.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:19.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:19.933 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:19.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:19.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:20.058 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:20.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:20.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:20.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:20.192 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:20.199 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:20 smithi163 bash[78107]: cluster 2023-10-09T09:48:18.127129+0000 mgr.y (mgr.34107) 1270 : cluster 0 pgmap v2114: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 7124/186111 objects degraded (3.828%); 0 B/s, 48 objects/s recovering 2023-10-09T09:48:20.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:20 smithi163 bash[75425]: cluster 2023-10-09T09:48:18.127129+0000 mgr.y (mgr.34107) 1270 : cluster 0 pgmap v2114: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 7124/186111 objects degraded (3.828%); 0 B/s, 48 objects/s recovering 2023-10-09T09:48:20.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:20.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:20.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:20.243 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:48:20.243 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:48:20.289 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:20 smithi181 bash[59345]: cluster 2023-10-09T09:48:18.127129+0000 mgr.y (mgr.34107) 1270 : cluster 0 pgmap v2114: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 7124/186111 objects degraded (3.828%); 0 B/s, 48 objects/s recovering 2023-10-09T09:48:20.323 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:20.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:20.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:20.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:20.455 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:20.482 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:20.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:20.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:20.585 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:20.614 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:20.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:20.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:20.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:20.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:20.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:20.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:20.855 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:20.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:20.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:20.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:21.029 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:21.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:21.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:21.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:21.168 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:21.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:21.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:21.274 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:21.283 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:21 smithi181 bash[59345]: cluster 2023-10-09T09:48:20.870563+0000 mon.a (mon.0) 2350 : cluster 3 Health check update: Degraded data redundancy: 7124/186111 objects degraded (3.828%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:21.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:21.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:21.402 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T09:48:21.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:21.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:21 smithi163 bash[75425]: cluster 2023-10-09T09:48:20.870563+0000 mon.a (mon.0) 2350 : cluster 3 Health check update: Degraded data redundancy: 7124/186111 objects degraded (3.828%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:21.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:21 smithi163 bash[78107]: cluster 2023-10-09T09:48:20.870563+0000 mon.a (mon.0) 2350 : cluster 3 Health check update: Degraded data redundancy: 7124/186111 objects degraded (3.828%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:21.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:21.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:21.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:21.583 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:21.613 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:21.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:21.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:21.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:21.749 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:21.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:21.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:21.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:21.888 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:21.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:21.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:21.995 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:22.021 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:22.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:22.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:22.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:22.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:22.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:22.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:22.260 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:22 smithi181 bash[59345]: cluster 2023-10-09T09:48:20.127626+0000 mgr.y (mgr.34107) 1271 : cluster 0 pgmap v2115: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 7124/186111 objects degraded (3.828%); 0 B/s, 44 objects/s recovering 2023-10-09T09:48:22.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:22.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:22.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:22.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:22.412 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:22 smithi163 bash[75425]: cluster 2023-10-09T09:48:20.127626+0000 mgr.y (mgr.34107) 1271 : cluster 0 pgmap v2115: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 7124/186111 objects degraded (3.828%); 0 B/s, 44 objects/s recovering 2023-10-09T09:48:22.412 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:22 smithi163 bash[78107]: cluster 2023-10-09T09:48:20.127626+0000 mgr.y (mgr.34107) 1271 : cluster 0 pgmap v2115: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 7124/186111 objects degraded (3.828%); 0 B/s, 44 objects/s recovering 2023-10-09T09:48:22.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:22.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:22.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:22.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:22.569 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:22.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:22.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:22.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:22.712 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:22.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:22.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:22.816 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:22.848 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:22.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:22.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:22.953 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:22.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:22.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:22.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:23.090 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:23.122 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:23.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:23.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:23.227 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:23.264 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:23.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:23.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:23.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:23.393 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:23.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:23.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:23.497 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:23.521 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:23.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:23.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:23.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:23.638 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:48:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:48:23] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.33.4" 2023-10-09T09:48:23.654 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:23.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:23.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:23.758 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:23.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:23.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:23.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:23.888 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:23.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:23.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:23.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:24.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:24 smithi181 bash[59345]: cluster 2023-10-09T09:48:22.128394+0000 mgr.y (mgr.34107) 1272 : cluster 0 pgmap v2116: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 713 op/s; 7124/177678 objects degraded (4.010%); 0 B/s, 37 objects/s recovering 2023-10-09T09:48:24.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:24.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:24.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:24.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:24.211 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:24.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:24.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:24.316 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:24.325 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:24 smithi163 bash[78107]: cluster 2023-10-09T09:48:22.128394+0000 mgr.y (mgr.34107) 1272 : cluster 0 pgmap v2116: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 713 op/s; 7124/177678 objects degraded (4.010%); 0 B/s, 37 objects/s recovering 2023-10-09T09:48:24.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:24 smithi163 bash[75425]: cluster 2023-10-09T09:48:22.128394+0000 mgr.y (mgr.34107) 1272 : cluster 0 pgmap v2116: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 713 op/s; 7124/177678 objects degraded (4.010%); 0 B/s, 37 objects/s recovering 2023-10-09T09:48:24.349 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:24.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:24.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:24.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:24.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:24.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:24.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:24.489 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:24.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:24.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:24.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:24.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:24.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:24.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:24.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:24.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:24.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:24.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:24.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:24.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:25.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:25.035 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:25.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:25.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:25.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:25.173 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:25.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:25.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:25.277 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:25.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:25.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:25.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:25.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:25.449 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:25.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:25.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:25.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:25.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:25.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:25.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:25.729 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:25.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:25.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:25.838 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:25.864 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:25.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:25.968 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:26.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:26.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:26.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:26.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:26.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:26 smithi163 bash[78107]: cluster 2023-10-09T09:48:24.129039+0000 mgr.y (mgr.34107) 1273 : cluster 0 pgmap v2117: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 7124/174570 objects degraded (4.081%); 0 B/s, 64 objects/s recovering 2023-10-09T09:48:26.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:26 smithi163 bash[78107]: audit 2023-10-09T09:48:25.395642+0000 mon.a (mon.0) 2351 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:26.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:26 smithi163 bash[78107]: cluster 2023-10-09T09:48:25.871775+0000 mon.a (mon.0) 2352 : cluster 3 Health check update: Degraded data redundancy: 7124/174570 objects degraded (4.081%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:26.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:26.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:26.246 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:26.247 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:48:26.275 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:26 smithi181 bash[59345]: cluster 2023-10-09T09:48:24.129039+0000 mgr.y (mgr.34107) 1273 : cluster 0 pgmap v2117: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 7124/174570 objects degraded (4.081%); 0 B/s, 64 objects/s recovering 2023-10-09T09:48:26.275 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:26 smithi181 bash[59345]: audit 2023-10-09T09:48:25.395642+0000 mon.a (mon.0) 2351 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:26.275 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:26 smithi181 bash[59345]: cluster 2023-10-09T09:48:25.871775+0000 mon.a (mon.0) 2352 : cluster 3 Health check update: Degraded data redundancy: 7124/174570 objects degraded (4.081%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:26.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:26.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:26.390 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:26.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:26.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:26.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:26.433 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:26 smithi163 bash[75425]: cluster 2023-10-09T09:48:24.129039+0000 mgr.y (mgr.34107) 1273 : cluster 0 pgmap v2117: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 7124/174570 objects degraded (4.081%); 0 B/s, 64 objects/s recovering 2023-10-09T09:48:26.433 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:26 smithi163 bash[75425]: audit 2023-10-09T09:48:25.395642+0000 mon.a (mon.0) 2351 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:26.433 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:26 smithi163 bash[75425]: cluster 2023-10-09T09:48:25.871775+0000 mon.a (mon.0) 2352 : cluster 3 Health check update: Degraded data redundancy: 7124/174570 objects degraded (4.081%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:26.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:26.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:26.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:26.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:26.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:26.685 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:26.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:26.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:26.789 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:26.815 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:26.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:26.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:26.920 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:26.950 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:26.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:26.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:27.053 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:27.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:27.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:27.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:27.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:27.227 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:27.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:27.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:27 smithi181 bash[59345]: cluster 2023-10-09T09:48:26.129726+0000 mgr.y (mgr.34107) 1274 : cluster 0 pgmap v2118: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 719 op/s; 7124/171675 objects degraded (4.150%); 0 B/s, 50 objects/s recovering 2023-10-09T09:48:27.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:27.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:27.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:27.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:27.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:27.477 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:27 smithi163 bash[75425]: cluster 2023-10-09T09:48:26.129726+0000 mgr.y (mgr.34107) 1274 : cluster 0 pgmap v2118: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 719 op/s; 7124/171675 objects degraded (4.150%); 0 B/s, 50 objects/s recovering 2023-10-09T09:48:27.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:27 smithi163 bash[78107]: cluster 2023-10-09T09:48:26.129726+0000 mgr.y (mgr.34107) 1274 : cluster 0 pgmap v2118: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 719 op/s; 7124/171675 objects degraded (4.150%); 0 B/s, 50 objects/s recovering 2023-10-09T09:48:27.525 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:27.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:27.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:27.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:27.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:27.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:27.762 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:27.799 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:27.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:27.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:27.926 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:27.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:27.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:28.029 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:28.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:28.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:28.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:28.167 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:28.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:28.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:28.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:28.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:28.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:28.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:28.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:28.438 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:28.469 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:28.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:28.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:28.572 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:28.601 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:28.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:28.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:28.731 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:28.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:28.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:28.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:28.867 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:28.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:28.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:28.974 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:29.004 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:29.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:29.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:29.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:29.137 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:29.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:29.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:29.188 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:29 smithi181 bash[59345]: cluster 2023-10-09T09:48:28.130466+0000 mgr.y (mgr.34107) 1275 : cluster 0 pgmap v2119: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 7124/165678 objects degraded (4.300%); 0 B/s, 50 objects/s recovering 2023-10-09T09:48:29.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:29 smithi163 bash[75425]: cluster 2023-10-09T09:48:28.130466+0000 mgr.y (mgr.34107) 1275 : cluster 0 pgmap v2119: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 7124/165678 objects degraded (4.300%); 0 B/s, 50 objects/s recovering 2023-10-09T09:48:29.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:29 smithi163 bash[78107]: cluster 2023-10-09T09:48:28.130466+0000 mgr.y (mgr.34107) 1275 : cluster 0 pgmap v2119: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 7124/165678 objects degraded (4.300%); 0 B/s, 50 objects/s recovering 2023-10-09T09:48:29.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:29.282 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:48:29.283 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:29.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:29.336 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:48:29.383 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:48:29.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:29.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:29.426 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:48:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:29.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:29.459 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:48:29.495 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:48:29.524 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:48:29.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:29.556 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:29.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:29.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:29.566 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:48:29.569 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:48:29.569 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:48:29.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:48:29.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:48:29.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:48:29.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:48:29.570 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:48:29.571 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:48:29.659 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:29.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:29.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:29.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:29.821 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:29.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:29.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:29.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:29.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:29.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:30.059 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:30.096 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:30.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:30.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:30.202 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:30.208 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:48:30.230 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:30.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:30.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:30.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:30.341 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:30 smithi181 bash[59345]: audit 2023-10-09T09:48:29.236644+0000 mon.a (mon.0) 2353 : audit 0 from='client.? 172.21.15.163:0/3377625084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:48:30.341 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:30 smithi181 bash[59345]: audit 2023-10-09T09:48:29.519248+0000 mon.a (mon.0) 2354 : audit 1 from='client.? 172.21.15.163:0/1957149789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:48:30.361 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:30.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:30.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:30.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:30.496 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:30 smithi163 bash[75425]: audit 2023-10-09T09:48:29.236644+0000 mon.a (mon.0) 2353 : audit 0 from='client.? 172.21.15.163:0/3377625084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:48:30.497 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:30 smithi163 bash[75425]: audit 2023-10-09T09:48:29.519248+0000 mon.a (mon.0) 2354 : audit 1 from='client.? 172.21.15.163:0/1957149789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:48:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:30 smithi163 bash[78107]: audit 2023-10-09T09:48:29.236644+0000 mon.a (mon.0) 2353 : audit 0 from='client.? 172.21.15.163:0/3377625084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:48:30.497 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:30 smithi163 bash[78107]: audit 2023-10-09T09:48:29.519248+0000 mon.a (mon.0) 2354 : audit 1 from='client.? 172.21.15.163:0/1957149789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:48:30.504 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:30.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:30.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:30.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:30.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:30.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:30.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:30.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:30.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:30.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:30.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:30.860 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:30.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:30.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:30.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:30.997 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:31.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:31.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:31.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:31.076 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:48:31.076 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T09:48:31.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T09:48:31.076 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:48:31.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.076 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181714 8.9 8:9fbf3c52:::benchmark_data_smithi163_151167_object86481:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1161)", 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.913200+0000", 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 559.15409745099998, 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3823221800000001, 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181714 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913200+0000", 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913200+0000", 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913198+0000", 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6140000000000003e-06 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913206+0000", 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5335000000000001e-05 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913221+0000", 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9950000000000005e-06 2023-10-09T09:48:31.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913229+0000", 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000755384 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913985+0000", 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000126945 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.914112+0000", 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000156441 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.914268+0000", 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010549005 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.924817+0000", 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002454814 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927272+0000", 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010801431 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.938073+0000", 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3888e-05 2023-10-09T09:48:31.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.938087+0000", 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0116360419999999 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.949723+0000", 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3546999999999999e-05 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.949737+0000", 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3218906910000001 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.271627+0000", 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.023894807000000001 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.295522+0000", 2023-10-09T09:48:31.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3823221800000001 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:181715 8.9 8:9c787169:::benchmark_data_smithi163_151167_object86482:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1161)", 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:11.913908+0000", 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 559.15338967100001, 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3629415580000002, 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T09:48:31.085 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 181715 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913908+0000", 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913908+0000", 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913907+0000", 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2729999999999999e-06 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913913+0000", 2023-10-09T09:48:31.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.806e-05 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913931+0000", 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7849999999999992e-06 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.913939+0000", 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0058540930000000003 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.919793+0000", 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00044839899999999999 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.920241+0000", 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000160922 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.920402+0000", 2023-10-09T09:48:31.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004489463 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.924892+0000", 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025955370000000002 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.927487+0000", 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010709781999999999 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.938197+0000", 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5780000000000005e-06 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:11.938207+0000", 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0115679959999999 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.949775+0000", 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5710000000000004e-06 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:12.949784+0000", 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3269953270000001 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.276779+0000", 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9950000000000006e-05 2023-10-09T09:48:31.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:14.276849+0000", 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3629415580000002 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185159 8.9 8:9c02d527:::benchmark_data_smithi163_151167_object89926:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1164)", 2023-10-09T09:48:31.093 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.732470+0000", 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.33482715499997, 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0585163849999999, 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185159 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.732470+0000", 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.732470+0000", 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.732469+0000", 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4339999999999998e-06 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.732478+0000", 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5950000000000001e-05 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.732494+0000", 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3359999999999995e-06 2023-10-09T09:48:31.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.732502+0000", 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2404999999999996e-05 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.732565+0000", 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013471199999999999 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.732699+0000", 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015232099999999999 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.732852+0000", 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012308684 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.745160+0000", 2023-10-09T09:48:31.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014758830000000001 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.746636+0000", 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00592525 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.752562+0000", 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0665999999999999e-05 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.752572+0000", 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.038296447 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.790869+0000", 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118046 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.790987+0000", 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0585163849999999 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185164 8.9 8:9a65cff3:::benchmark_data_smithi163_151167_object89931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1164)", 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.746679+0000", 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.32061886400004, 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0448527620000001, 2023-10-09T09:48:31.101 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185164 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.746679+0000", 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.746679+0000", 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.746677+0000", 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2840000000000006e-06 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.746685+0000", 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3249e-05 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.746699+0000", 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0079999999999996e-06 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.746706+0000", 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0188999999999997e-05 2023-10-09T09:48:31.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.746756+0000", 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014581500000000001 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.746902+0000", 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016767 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T09:48:31.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.747069+0000", 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057516959999999997 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.752821+0000", 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5980000000000002e-06 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.752828+0000", 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0385768339999999 2023-10-09T09:48:31.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.791404+0000", 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000127083 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.791531+0000", 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0448527620000001 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.107 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185181 8.9 8:9417f256:::benchmark_data_smithi163_151167_object89948:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.774554+0000", 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.29274372600003, 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0377413289999999, 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.108 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185181 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.774554+0000", 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.774554+0000", 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.774552+0000", 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0178e-05 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.774562+0000", 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5829e-05 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.774578+0000", 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6470000000000001e-06 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.774585+0000", 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3288999999999998e-05 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.774638+0000", 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4190000000000002e-06 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.774646+0000", 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0170051870000001 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.791652+0000", 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019953700000000001 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.791851+0000", 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031169100000000001 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.792163+0000", 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011965277999999999 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.804128+0000", 2023-10-09T09:48:31.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5645000000000002e-05 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.804164+0000", 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0080200219999999999 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812184+0000", 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1539999999999997e-05 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812245+0000", 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9917999999999999e-05 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812295+0000", 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0377413289999999 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185194 8.9 8:96b38b8e:::benchmark_data_smithi163_151167_object89961:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.793704+0000", 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.27359396600002, 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0188482860000001, 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185194 2023-10-09T09:48:31.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.793704+0000", 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.793704+0000", 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.793702+0000", 2023-10-09T09:48:31.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3689999999999996e-06 2023-10-09T09:48:31.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.793711+0000", 2023-10-09T09:48:31.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3033999999999999e-05 2023-10-09T09:48:31.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.793724+0000", 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8329999999999996e-06 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.793731+0000", 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7658000000000003e-05 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.793789+0000", 2023-10-09T09:48:31.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5200000000000003e-06 2023-10-09T09:48:31.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.793795+0000", 2023-10-09T09:48:31.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.005910189 2023-10-09T09:48:31.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.799705+0000", 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000167345 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.799873+0000", 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020814100000000001 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.800081+0000", 2023-10-09T09:48:31.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042456179999999996 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.804326+0000", 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081077430000000006 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812434+0000", 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2072999999999999e-05 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812446+0000", 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4759999999999997e-05 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812501+0000", 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0831000000000002e-05 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812552+0000", 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0188482860000001 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185199 8.9 8:97bf1287:::benchmark_data_smithi163_151167_object89966:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.802033+0000", 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.265264914, 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.01075921, 2023-10-09T09:48:31.124 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185199 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.802033+0000", 2023-10-09T09:48:31.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.802033+0000", 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.802031+0000", 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0881e-05 2023-10-09T09:48:31.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.802042+0000", 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7836999999999999e-05 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.802059+0000", 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3209999999999994e-06 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.802069+0000", 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8956999999999999e-05 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.802118+0000", 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0077e-05 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.802128+0000", 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.00223335 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.804361+0000", 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3954000000000003e-05 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.804445+0000", 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4707e-05 2023-10-09T09:48:31.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.804540+0000", 2023-10-09T09:48:31.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0078262269999999998 2023-10-09T09:48:31.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812366+0000", 2023-10-09T09:48:31.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019688299999999999 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812563+0000", 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013333299999999999 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812696+0000", 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6845000000000003e-05 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812743+0000", 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8767000000000003e-05 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812792+0000", 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.01075921 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.132 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185221 8.9 8:9dff8266:::benchmark_data_smithi163_151167_object89988:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.834012+0000", 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.23328585800004, 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98355745900000002, 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.133 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185221 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.834012+0000", 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.834012+0000", 2023-10-09T09:48:31.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.834010+0000", 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1939999999999998e-06 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.834018+0000", 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2723e-05 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.834031+0000", 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4859999999999997e-06 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.834037+0000", 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7428000000000001e-05 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.834084+0000", 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7290000000000006e-06 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.834093+0000", 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97410440399999998 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.808197+0000", 2023-10-09T09:48:31.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0106999999999997e-05 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.808277+0000", 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7388999999999999e-05 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.808375+0000", 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004242493 2023-10-09T09:48:31.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812617+0000", 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000189557 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812807+0000", 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046598940000000004 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.817467+0000", 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9282e-05 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.817516+0000", 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3180999999999998e-05 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.817569+0000", 2023-10-09T09:48:31.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98355745900000002 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185223 8.9 8:9674a36a:::benchmark_data_smithi163_151167_object89990:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.837433+0000", 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.22986407400003, 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99097697799999995, 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.141 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185223 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837433+0000", 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837433+0000", 2023-10-09T09:48:31.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:48:31.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837432+0000", 2023-10-09T09:48:31.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6170000000000007e-06 2023-10-09T09:48:31.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837440+0000", 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4955e-05 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837455+0000", 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.421e-06 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837462+0000", 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1378000000000003e-05 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837503+0000", 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1300000000000003e-06 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837510+0000", 2023-10-09T09:48:31.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.975328627 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812839+0000", 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9538999999999998e-05 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.812928+0000", 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010196699999999999 2023-10-09T09:48:31.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.813030+0000", 2023-10-09T09:48:31.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015181451 2023-10-09T09:48:31.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828212+0000", 2023-10-09T09:48:31.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5646000000000002e-05 2023-10-09T09:48:31.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828268+0000", 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9167999999999999e-05 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828297+0000", 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9176999999999999e-05 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828356+0000", 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4546999999999998e-05 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828410+0000", 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99097697799999995 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.149 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185224 8.9 8:95a722a6:::benchmark_data_smithi163_151167_object89991:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.837976+0000", 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.22932106500002, 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99068787899999999, 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185224 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.150 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837976+0000", 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837976+0000", 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837976+0000", 2023-10-09T09:48:31.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6810000000000003e-06 2023-10-09T09:48:31.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837981+0000", 2023-10-09T09:48:31.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1685000000000001e-05 2023-10-09T09:48:31.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837993+0000", 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1909999999999999e-06 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.837998+0000", 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9987000000000002e-05 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.838038+0000", 2023-10-09T09:48:31.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.028e-06 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.838046+0000", 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97957358999999999 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.817620+0000", 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023918 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.817859+0000", 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019645400000000001 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.818055+0000", 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010205844 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828261+0000", 2023-10-09T09:48:31.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000170586 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828432+0000", 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012911800000000001 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828561+0000", 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5268e-05 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828616+0000", 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8180999999999998e-05 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828664+0000", 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99068787899999999 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185243 8.9 8:9c12dba8:::benchmark_data_smithi163_151167_object90010:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.866168+0000", 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.20112928000003, 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97223059300000003, 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185243 2023-10-09T09:48:31.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.866168+0000", 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.866168+0000", 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.866166+0000", 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0420000000000008e-06 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.866175+0000", 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6847999999999999e-05 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.866192+0000", 2023-10-09T09:48:31.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0620000000000001e-06 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.866200+0000", 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1065e-05 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.866251+0000", 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2251e-05 2023-10-09T09:48:31.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.866263+0000", 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.95717016300000002 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.823434+0000", 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000141182 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.823575+0000", 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017849599999999999 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.823753+0000", 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0047478889999999999 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828501+0000", 2023-10-09T09:48:31.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000176424 2023-10-09T09:48:31.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828678+0000", 2023-10-09T09:48:31.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0095801830000000008 2023-10-09T09:48:31.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.838258+0000", 2023-10-09T09:48:31.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3363999999999998e-05 2023-10-09T09:48:31.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.838331+0000", 2023-10-09T09:48:31.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7712999999999994e-05 2023-10-09T09:48:31.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.838399+0000", 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97223059300000003 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185282 8.9 8:972764c0:::benchmark_data_smithi163_151167_object90049:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.935608+0000", 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.13168906199996, 2023-10-09T09:48:31.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93085631800000002, 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185282 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.935608+0000", 2023-10-09T09:48:31.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.935608+0000", 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.935607+0000", 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7399999999999993e-06 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.935616+0000", 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5401e-05 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.935631+0000", 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3990000000000004e-06 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.935640+0000", 2023-10-09T09:48:31.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1598000000000003e-05 2023-10-09T09:48:31.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.935691+0000", 2023-10-09T09:48:31.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8900000000000007e-06 2023-10-09T09:48:31.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.935699+0000", 2023-10-09T09:48:31.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89301874800000003 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828718+0000", 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125695 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828844+0000", 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015445500000000001 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.828998+0000", 2023-10-09T09:48:31.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.019023175 2023-10-09T09:48:31.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.848021+0000", 2023-10-09T09:48:31.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3183999999999999e-05 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.848055+0000", 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018125171999999998 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.866180+0000", 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000230549 2023-10-09T09:48:31.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.866410+0000", 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4545e-05 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.866465+0000", 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93085631800000002 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.174 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.175 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185300 8.9 8:970073c9:::benchmark_data_smithi163_151167_object90067:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.175 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.974100+0000", 2023-10-09T09:48:31.175 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.09319715499998, 2023-10-09T09:48:31.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89579286000000002, 2023-10-09T09:48:31.175 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.175 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.175 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.175 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185300 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.974100+0000", 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.974100+0000", 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.974099+0000", 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8389999999999996e-06 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.974108+0000", 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7880000000000002e-05 2023-10-09T09:48:31.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.974125+0000", 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5070000000000003e-06 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.974133+0000", 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5143999999999999e-05 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.974188+0000", 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3650000000000004e-06 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.974196+0000", 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.85925995399999999 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.833456+0000", 2023-10-09T09:48:31.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010259699999999999 2023-10-09T09:48:31.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.833559+0000", 2023-10-09T09:48:31.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019356199999999999 2023-10-09T09:48:31.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.833753+0000", 2023-10-09T09:48:31.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014334876999999999 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.848087+0000", 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018075320999999998 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.866163+0000", 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031932599999999998 2023-10-09T09:48:31.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.866482+0000", 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033796360000000001 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.869862+0000", 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1476999999999997e-05 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.869893+0000", 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89579286000000002 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185310 8.9 8:9274e13b:::benchmark_data_smithi163_151167_object90077:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:19.992342+0000", 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.0749558, 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87771823400000004, 2023-10-09T09:48:31.184 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185310 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.992342+0000", 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.992342+0000", 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.992340+0000", 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3680000000000002e-06 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.992348+0000", 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5675000000000002e-05 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.992364+0000", 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9630000000000001e-06 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.992371+0000", 2023-10-09T09:48:31.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7168999999999993e-05 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.992438+0000", 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4749999999999999e-06 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:19.992447+0000", 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84600726699999995 2023-10-09T09:48:31.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.838455+0000", 2023-10-09T09:48:31.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012524500000000001 2023-10-09T09:48:31.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.838580+0000", 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3046000000000003e-05 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.838673+0000", 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.021320735 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.859994+0000", 2023-10-09T09:48:31.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0099540949999999996 2023-10-09T09:48:31.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.869948+0000", 2023-10-09T09:48:31.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8873999999999999e-05 2023-10-09T09:48:31.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.870006+0000", 2023-10-09T09:48:31.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9865e-05 2023-10-09T09:48:31.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.870036+0000", 2023-10-09T09:48:31.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3600000000000001e-05 2023-10-09T09:48:31.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.870060+0000", 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87771823400000004 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.45122.0:185340 8.9 8:9435a830:::benchmark_data_smithi163_151167_object90107:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1165)", 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:39:20.055519+0000", 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 551.01177819300005, 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.81837634500000001, 2023-10-09T09:48:31.193 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.194 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.194 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.194 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.45122", 2023-10-09T09:48:31.194 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2278554988", 2023-10-09T09:48:31.194 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185340 2023-10-09T09:48:31.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.194 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.055519+0000", 2023-10-09T09:48:31.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.055519+0000", 2023-10-09T09:48:31.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:48:31.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.055518+0000", 2023-10-09T09:48:31.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7679999999999997e-06 2023-10-09T09:48:31.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.055524+0000", 2023-10-09T09:48:31.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1705999999999999e-05 2023-10-09T09:48:31.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.055536+0000", 2023-10-09T09:48:31.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0110000000000003e-06 2023-10-09T09:48:31.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.055543+0000", 2023-10-09T09:48:31.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1480999999999998e-05 2023-10-09T09:48:31.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.055584+0000", 2023-10-09T09:48:31.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.429e-06 2023-10-09T09:48:31.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.055590+0000", 2023-10-09T09:48:31.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79251854799999999 2023-10-09T09:48:31.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.848108+0000", 2023-10-09T09:48:31.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011680600000000001 2023-10-09T09:48:31.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.848225+0000", 2023-10-09T09:48:31.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014078100000000001 2023-10-09T09:48:31.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,3", 2023-10-09T09:48:31.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.848366+0000", 2023-10-09T09:48:31.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.021761154000000001 2023-10-09T09:48:31.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.870127+0000", 2023-10-09T09:48:31.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1433000000000006e-05 2023-10-09T09:48:31.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.870189+0000", 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00361553 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.873804+0000", 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2976999999999999e-05 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.873847+0000", 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8615000000000001e-05 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:39:20.873896+0000", 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.81837634500000001 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.203 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:194500 9.11 9:8b6d6b71:::benchmark_data_smithi163_175771_object90604:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1221)", 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:44:19.050832+0000", 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 252.01646573299999, 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99498853799999998, 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 194500 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.050832+0000", 2023-10-09T09:48:31.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.050832+0000", 2023-10-09T09:48:31.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:48:31.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.050830+0000", 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1470000000000004e-06 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.050837+0000", 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1265e-05 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.050849+0000", 2023-10-09T09:48:31.206 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6450000000000002e-06 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.050854+0000", 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0587000000000002e-05 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.050905+0000", 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0516999999999999e-05 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.050915+0000", 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98206752200000003 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.032983+0000", 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3262000000000003e-05 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.033076+0000", 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050541300000000002 2023-10-09T09:48:31.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4", 2023-10-09T09:48:31.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.033582+0000", 2023-10-09T09:48:31.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011719521 2023-10-09T09:48:31.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045301+0000", 2023-10-09T09:48:31.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017197800000000001 2023-10-09T09:48:31.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045473+0000", 2023-10-09T09:48:31.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012860700000000001 2023-10-09T09:48:31.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045602+0000", 2023-10-09T09:48:31.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124144 2023-10-09T09:48:31.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045726+0000", 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4420999999999996e-05 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045820+0000", 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99498853799999998 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.211 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:194585 9.11 9:8d9ec71f:::benchmark_data_smithi163_175771_object90689:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1221)", 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:44:19.149603+0000", 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 251.91769490900001, 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.896551239, 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:48:31.212 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 194585 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.149603+0000", 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.149603+0000", 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:48:31.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.149601+0000", 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1669999999999998e-06 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.149607+0000", 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8130000000000007e-06 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.149616+0000", 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1029999999999996e-06 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.149624+0000", 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3929000000000003e-05 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.149697+0000", 2023-10-09T09:48:31.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6339999999999996e-06 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.149707+0000", 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88687253200000005 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.036580+0000", 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3311000000000002e-05 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.036663+0000", 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029204699999999998 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4", 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.036955+0000", 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085617920000000004 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045517+0000", 2023-10-09T09:48:31.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2169999999999997e-06 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045524+0000", 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00044648300000000001 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045970+0000", 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6955999999999998e-05 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046067+0000", 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6452000000000005e-05 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046154+0000", 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.896551239 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:194598 9.11 9:8ad3b71a:::benchmark_data_smithi163_175771_object90702:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1221)", 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:44:19.159125+0000", 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 251.90817230600001, 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88720100199999996, 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 194598 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.220 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.159125+0000", 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.159125+0000", 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.159124+0000", 2023-10-09T09:48:31.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8340000000000001e-06 2023-10-09T09:48:31.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.159131+0000", 2023-10-09T09:48:31.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1348999999999999e-05 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.159142+0000", 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0209999999999996e-06 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.159148+0000", 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5892e-05 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.159194+0000", 2023-10-09T09:48:31.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4759999999999998e-06 2023-10-09T09:48:31.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.159202+0000", 2023-10-09T09:48:31.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87993136500000002 2023-10-09T09:48:31.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.039134+0000", 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000169519 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.039303+0000", 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00054176699999999996 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.226 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4", 2023-10-09T09:48:31.226 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.039845+0000", 2023-10-09T09:48:31.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056821340000000001 2023-10-09T09:48:31.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.226 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.226 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045527+0000", 2023-10-09T09:48:31.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000355065 2023-10-09T09:48:31.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045882+0000", 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00036886800000000001 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046251+0000", 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4827000000000003e-05 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046296+0000", 2023-10-09T09:48:31.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0416000000000002e-05 2023-10-09T09:48:31.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046326+0000", 2023-10-09T09:48:31.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88720100199999996 2023-10-09T09:48:31.228 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.228 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.229 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.229 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:194648 9.11 9:89c7e2a7:::benchmark_data_smithi163_175771_object90752:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1221)", 2023-10-09T09:48:31.229 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:44:19.221172+0000", 2023-10-09T09:48:31.229 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 251.846125124, 2023-10-09T09:48:31.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82533234, 2023-10-09T09:48:31.229 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.229 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.229 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.230 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:48:31.230 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:48:31.230 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 194648 2023-10-09T09:48:31.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.230 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.221172+0000", 2023-10-09T09:48:31.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.221172+0000", 2023-10-09T09:48:31.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:48:31.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.221172+0000", 2023-10-09T09:48:31.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2220000000000004e-06 2023-10-09T09:48:31.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.221176+0000", 2023-10-09T09:48:31.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5249999999999994e-06 2023-10-09T09:48:31.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.221186+0000", 2023-10-09T09:48:31.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5609999999999999e-06 2023-10-09T09:48:31.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.221189+0000", 2023-10-09T09:48:31.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3046000000000005e-05 2023-10-09T09:48:31.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.221262+0000", 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2329999999999995e-06 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.221271+0000", 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82183577200000002 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.043107+0000", 2023-10-09T09:48:31.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8387000000000006e-05 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.043196+0000", 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000372417 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4", 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.043568+0000", 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023203899999999999 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.045888+0000", 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000527481 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046416+0000", 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0474999999999999e-05 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046446+0000", 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2889000000000003e-05 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046479+0000", 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5534e-05 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046505+0000", 2023-10-09T09:48:31.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82533234 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35412.0:194651 9.11 9:88df3236:::benchmark_data_smithi163_175771_object90755:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1221)", 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:44:19.226620+0000", 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 251.84067792299999, 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82972668400000005, 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:48:31.238 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:48:31.239 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:48:31.239 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35412", 2023-10-09T09:48:31.239 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1935580533", 2023-10-09T09:48:31.239 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 194651 2023-10-09T09:48:31.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.239 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:48:31.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:48:31.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.226620+0000", 2023-10-09T09:48:31.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.226620+0000", 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.226618+0000", 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7909999999999999e-06 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.226625+0000", 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3623e-05 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.226639+0000", 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1460000000000001e-06 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.226645+0000", 2023-10-09T09:48:31.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9408999999999996e-05 2023-10-09T09:48:31.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.226744+0000", 2023-10-09T09:48:31.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2299e-05 2023-10-09T09:48:31.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T09:48:31.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:19.226756+0000", 2023-10-09T09:48:31.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.81980502300000002 2023-10-09T09:48:31.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:48:31.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046561+0000", 2023-10-09T09:48:31.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023038400000000001 2023-10-09T09:48:31.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:48:31.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.046792+0000", 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00044119199999999997 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4", 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.047233+0000", 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089469130000000008 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.056180+0000", 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5920999999999994e-05 2023-10-09T09:48:31.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:48:31.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.056246+0000", 2023-10-09T09:48:31.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2309e-05 2023-10-09T09:48:31.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:48:31.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.056258+0000", 2023-10-09T09:48:31.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7373000000000001e-05 2023-10-09T09:48:31.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:48:31.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.056315+0000", 2023-10-09T09:48:31.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0790999999999997e-05 2023-10-09T09:48:31.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:48:31.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:48:31.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:48:31.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:44:20.056346+0000", 2023-10-09T09:48:31.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82972668400000005 2023-10-09T09:48:31.246 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.247 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.247 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.247 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:48:31.247 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:48:31.247 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:48:31.247 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:31.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[75425]: cluster 2023-10-09T09:48:30.131133+0000 mgr.y (mgr.34107) 1276 : cluster 0 pgmap v2120: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 648 op/s; 7124/162774 objects degraded (4.377%); 0 B/s, 60 objects/s recovering 2023-10-09T09:48:31.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[75425]: cluster 2023-10-09T09:48:30.187446+0000 mon.a (mon.0) 2355 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:48:31.248 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[75425]: cluster 2023-10-09T09:48:30.197602+0000 osd.5 (osd.5) 839 : cluster 3 failed to encode map e1250 with expected crc 2023-10-09T09:48:31.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[75425]: cluster 2023-10-09T09:48:30.198996+0000 osd.7 (osd.7) 600 : cluster 3 failed to encode map e1250 with expected crc 2023-10-09T09:48:31.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[75425]: audit 2023-10-09T09:48:30.202462+0000 mon.a (mon.0) 2356 : audit 1 from='client.? 172.21.15.163:0/1957149789' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:48:31.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[75425]: cluster 2023-10-09T09:48:30.202672+0000 mon.a (mon.0) 2357 : cluster 0 osdmap e1250: 8 total, 8 up, 6 in 2023-10-09T09:48:31.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[75425]: cluster 2023-10-09T09:48:30.872862+0000 mon.a (mon.0) 2358 : cluster 3 Health check update: Degraded data redundancy: 7124/162774 objects degraded (4.377%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:31.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:31.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:31.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:31.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:31.384 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[78107]: cluster 2023-10-09T09:48:30.131133+0000 mgr.y (mgr.34107) 1276 : cluster 0 pgmap v2120: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 648 op/s; 7124/162774 objects degraded (4.377%); 0 B/s, 60 objects/s recovering 2023-10-09T09:48:31.384 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[78107]: cluster 2023-10-09T09:48:30.187446+0000 mon.a (mon.0) 2355 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:48:31.384 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[78107]: cluster 2023-10-09T09:48:30.197602+0000 osd.5 (osd.5) 839 : cluster 3 failed to encode map e1250 with expected crc 2023-10-09T09:48:31.384 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[78107]: cluster 2023-10-09T09:48:30.198996+0000 osd.7 (osd.7) 600 : cluster 3 failed to encode map e1250 with expected crc 2023-10-09T09:48:31.384 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[78107]: audit 2023-10-09T09:48:30.202462+0000 mon.a (mon.0) 2356 : audit 1 from='client.? 172.21.15.163:0/1957149789' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:48:31.384 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[78107]: cluster 2023-10-09T09:48:30.202672+0000 mon.a (mon.0) 2357 : cluster 0 osdmap e1250: 8 total, 8 up, 6 in 2023-10-09T09:48:31.384 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:31 smithi163 bash[78107]: cluster 2023-10-09T09:48:30.872862+0000 mon.a (mon.0) 2358 : cluster 3 Health check update: Degraded data redundancy: 7124/162774 objects degraded (4.377%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:31.400 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:31.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:31.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:31.505 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:31.538 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:31.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:31.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:31.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:31 smithi181 bash[59345]: cluster 2023-10-09T09:48:30.131133+0000 mgr.y (mgr.34107) 1276 : cluster 0 pgmap v2120: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 648 op/s; 7124/162774 objects degraded (4.377%); 0 B/s, 60 objects/s recovering 2023-10-09T09:48:31.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:31 smithi181 bash[59345]: cluster 2023-10-09T09:48:30.187446+0000 mon.a (mon.0) 2355 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:48:31.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:31 smithi181 bash[59345]: cluster 2023-10-09T09:48:30.197602+0000 osd.5 (osd.5) 839 : cluster 3 failed to encode map e1250 with expected crc 2023-10-09T09:48:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:31 smithi181 bash[59345]: cluster 2023-10-09T09:48:30.198996+0000 osd.7 (osd.7) 600 : cluster 3 failed to encode map e1250 with expected crc 2023-10-09T09:48:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:31 smithi181 bash[59345]: audit 2023-10-09T09:48:30.202462+0000 mon.a (mon.0) 2356 : audit 1 from='client.? 172.21.15.163:0/1957149789' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:48:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:31 smithi181 bash[59345]: cluster 2023-10-09T09:48:30.202672+0000 mon.a (mon.0) 2357 : cluster 0 osdmap e1250: 8 total, 8 up, 6 in 2023-10-09T09:48:31.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:31 smithi181 bash[59345]: cluster 2023-10-09T09:48:30.872862+0000 mon.a (mon.0) 2358 : cluster 3 Health check update: Degraded data redundancy: 7124/162774 objects degraded (4.377%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:31.644 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:31.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:31.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:31.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:31.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:31.811 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:31.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:31.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:31.916 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:31.943 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:31.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:32.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:32.082 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:32.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:32.127 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T09:48:32.185 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:32.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:32.223 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:32.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:32.324 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:32.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:32.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:32.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:32.483 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:32.512 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:32.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:32.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:32.617 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:32.642 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:32.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:32.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:32.745 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:32.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:32.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:32.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:32.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:32.909 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:32.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:32.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:33.014 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:33.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:33.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:33.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:33.204 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:33 smithi163 bash[78107]: cluster 2023-10-09T09:48:32.131829+0000 mgr.y (mgr.34107) 1277 : cluster 0 pgmap v2122: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 7124/154050 objects degraded (4.624%); 0 B/s, 72 objects/s recovering 2023-10-09T09:48:33.253 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:33.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:33.290 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:33.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:33.391 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:33.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:33.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:33.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:33.517 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:33.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:33 smithi181 bash[59345]: cluster 2023-10-09T09:48:32.131829+0000 mgr.y (mgr.34107) 1277 : cluster 0 pgmap v2122: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 7124/154050 objects degraded (4.624%); 0 B/s, 72 objects/s recovering 2023-10-09T09:48:33.561 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:33.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:33.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:33.668 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:33.700 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:33.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:33.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:33.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:33 smithi163 bash[75425]: cluster 2023-10-09T09:48:32.131829+0000 mgr.y (mgr.34107) 1277 : cluster 0 pgmap v2122: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 7124/154050 objects degraded (4.624%); 0 B/s, 72 objects/s recovering 2023-10-09T09:48:33.711 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:48:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:48:33] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.33.4" 2023-10-09T09:48:33.804 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:33.836 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:33.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:33.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:33.941 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:33.971 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:33.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:33.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:34.074 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:34.114 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:34.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:34.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:34.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:34.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:34.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:34.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:34.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:34.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:34.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:34.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:34.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:34.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:34.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:34.636 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:34.673 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:34.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:34.775 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:34.810 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:34.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:34.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:34.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:34.962 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:34.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:34.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:35.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:35.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:35.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:35.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:35.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:48:35.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:35.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:35 smithi163 bash[75425]: cluster 2023-10-09T09:48:34.135411+0000 mgr.y (mgr.34107) 1278 : cluster 0 pgmap v2123: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 785 op/s; 7124/150981 objects degraded (4.718%); 0 B/s, 83 objects/s recovering 2023-10-09T09:48:35.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:35 smithi163 bash[78107]: cluster 2023-10-09T09:48:34.135411+0000 mgr.y (mgr.34107) 1278 : cluster 0 pgmap v2123: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 785 op/s; 7124/150981 objects degraded (4.718%); 0 B/s, 83 objects/s recovering 2023-10-09T09:48:35.231 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:35.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:35.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:35.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:48:35.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:35.368 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:35.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:35.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:35.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:35.510 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:35.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:35.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:35.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:35 smithi181 bash[59345]: cluster 2023-10-09T09:48:34.135411+0000 mgr.y (mgr.34107) 1278 : cluster 0 pgmap v2123: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 785 op/s; 7124/150981 objects degraded (4.718%); 0 B/s, 83 objects/s recovering 2023-10-09T09:48:35.615 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:35.640 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:35.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:35.743 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:35.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:35.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:35.876 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:35.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:35.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:35.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:36.010 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:36.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:36.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:36.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:36.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:36.179 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:36.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:36.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:36.232 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T09:48:36.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[78107]: cluster 2023-10-09T09:48:35.213995+0000 osd.5 (osd.5) 840 : cluster 3 failed to encode map e1251 with expected crc 2023-10-09T09:48:36.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[78107]: cluster 2023-10-09T09:48:35.221966+0000 mon.a (mon.0) 2359 : cluster 0 osdmap e1251: 8 total, 8 up, 6 in 2023-10-09T09:48:36.280 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[78107]: cluster 2023-10-09T09:48:35.232150+0000 osd.6 (osd.6) 536 : cluster 3 failed to encode map e1251 with expected crc 2023-10-09T09:48:36.280 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[78107]: cluster 2023-10-09T09:48:35.232906+0000 osd.4 (osd.4) 653 : cluster 3 failed to encode map e1251 with expected crc 2023-10-09T09:48:36.281 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[78107]: audit 2023-10-09T09:48:35.577668+0000 mon.a (mon.0) 2360 : audit 1 from='client.? 172.21.15.163:0/103412719' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:48:36.281 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[78107]: cluster 2023-10-09T09:48:35.874151+0000 mon.a (mon.0) 2361 : cluster 3 Health check update: Degraded data redundancy: 7124/150981 objects degraded (4.718%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:36.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[75425]: cluster 2023-10-09T09:48:35.213995+0000 osd.5 (osd.5) 840 : cluster 3 failed to encode map e1251 with expected crc 2023-10-09T09:48:36.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[75425]: cluster 2023-10-09T09:48:35.221966+0000 mon.a (mon.0) 2359 : cluster 0 osdmap e1251: 8 total, 8 up, 6 in 2023-10-09T09:48:36.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[75425]: cluster 2023-10-09T09:48:35.232150+0000 osd.6 (osd.6) 536 : cluster 3 failed to encode map e1251 with expected crc 2023-10-09T09:48:36.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[75425]: cluster 2023-10-09T09:48:35.232906+0000 osd.4 (osd.4) 653 : cluster 3 failed to encode map e1251 with expected crc 2023-10-09T09:48:36.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[75425]: audit 2023-10-09T09:48:35.577668+0000 mon.a (mon.0) 2360 : audit 1 from='client.? 172.21.15.163:0/103412719' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:48:36.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:36 smithi163 bash[75425]: cluster 2023-10-09T09:48:35.874151+0000 mon.a (mon.0) 2361 : cluster 3 Health check update: Degraded data redundancy: 7124/150981 objects degraded (4.718%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:36.283 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:36.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:36.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:36.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:36.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:36.456 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:36.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:36.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:36.560 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:36 smithi181 bash[59345]: cluster 2023-10-09T09:48:35.213995+0000 osd.5 (osd.5) 840 : cluster 3 failed to encode map e1251 with expected crc 2023-10-09T09:48:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:36 smithi181 bash[59345]: cluster 2023-10-09T09:48:35.221966+0000 mon.a (mon.0) 2359 : cluster 0 osdmap e1251: 8 total, 8 up, 6 in 2023-10-09T09:48:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:36 smithi181 bash[59345]: cluster 2023-10-09T09:48:35.232150+0000 osd.6 (osd.6) 536 : cluster 3 failed to encode map e1251 with expected crc 2023-10-09T09:48:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:36 smithi181 bash[59345]: cluster 2023-10-09T09:48:35.232906+0000 osd.4 (osd.4) 653 : cluster 3 failed to encode map e1251 with expected crc 2023-10-09T09:48:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:36 smithi181 bash[59345]: audit 2023-10-09T09:48:35.577668+0000 mon.a (mon.0) 2360 : audit 1 from='client.? 172.21.15.163:0/103412719' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:48:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:36 smithi181 bash[59345]: cluster 2023-10-09T09:48:35.874151+0000 mon.a (mon.0) 2361 : cluster 3 Health check update: Degraded data redundancy: 7124/150981 objects degraded (4.718%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:36.585 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:36.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:36.691 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:36.720 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:36.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:36.828 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:36.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:36.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:36.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:36.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:36.992 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:48:37.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:37.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:37.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:37.117 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:37.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:37.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:37.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:37.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:37 smithi181 bash[59345]: cluster 2023-10-09T09:48:36.136326+0000 mgr.y (mgr.34107) 1279 : cluster 0 pgmap v2125: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 7124/148074 objects degraded (4.811%); 0 B/s, 104 objects/s recovering 2023-10-09T09:48:37.230 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:37 smithi181 bash[59345]: cluster 2023-10-09T09:48:36.223410+0000 osd.5 (osd.5) 841 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:37.230 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:37 smithi181 bash[59345]: cluster 2023-10-09T09:48:36.224040+0000 osd.6 (osd.6) 537 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:37.230 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:37 smithi181 bash[59345]: audit 2023-10-09T09:48:36.225967+0000 mon.a (mon.0) 2362 : audit 1 from='client.? 172.21.15.163:0/103412719' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:48:37.230 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:37 smithi181 bash[59345]: cluster 2023-10-09T09:48:36.226052+0000 mon.a (mon.0) 2363 : cluster 0 osdmap e1252: 8 total, 8 up, 6 in 2023-10-09T09:48:37.230 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:37 smithi181 bash[59345]: cluster 2023-10-09T09:48:36.232322+0000 osd.4 (osd.4) 654 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:37.255 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:37.278 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:37.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:37.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:37.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:37.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:37.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:37.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:37.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:37.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:37.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:37.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:37.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:37.656 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[78107]: cluster 2023-10-09T09:48:36.136326+0000 mgr.y (mgr.34107) 1279 : cluster 0 pgmap v2125: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 7124/148074 objects degraded (4.811%); 0 B/s, 104 objects/s recovering 2023-10-09T09:48:37.656 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[78107]: cluster 2023-10-09T09:48:36.223410+0000 osd.5 (osd.5) 841 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:37.656 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[78107]: cluster 2023-10-09T09:48:36.224040+0000 osd.6 (osd.6) 537 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:37.656 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[78107]: audit 2023-10-09T09:48:36.225967+0000 mon.a (mon.0) 2362 : audit 1 from='client.? 172.21.15.163:0/103412719' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:48:37.656 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[78107]: cluster 2023-10-09T09:48:36.226052+0000 mon.a (mon.0) 2363 : cluster 0 osdmap e1252: 8 total, 8 up, 6 in 2023-10-09T09:48:37.656 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[78107]: cluster 2023-10-09T09:48:36.232322+0000 osd.4 (osd.4) 654 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:37.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[75425]: cluster 2023-10-09T09:48:36.136326+0000 mgr.y (mgr.34107) 1279 : cluster 0 pgmap v2125: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 7124/148074 objects degraded (4.811%); 0 B/s, 104 objects/s recovering 2023-10-09T09:48:37.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[75425]: cluster 2023-10-09T09:48:36.223410+0000 osd.5 (osd.5) 841 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:37.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[75425]: cluster 2023-10-09T09:48:36.224040+0000 osd.6 (osd.6) 537 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:37.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[75425]: audit 2023-10-09T09:48:36.225967+0000 mon.a (mon.0) 2362 : audit 1 from='client.? 172.21.15.163:0/103412719' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:48:37.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[75425]: cluster 2023-10-09T09:48:36.226052+0000 mon.a (mon.0) 2363 : cluster 0 osdmap e1252: 8 total, 8 up, 6 in 2023-10-09T09:48:37.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:37 smithi163 bash[75425]: cluster 2023-10-09T09:48:36.232322+0000 osd.4 (osd.4) 654 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:37.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:37.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:37.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:37.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T09:48:37.784 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:37.831 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:37.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:37.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:37.937 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:37.969 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:37.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:37.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:38.075 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:38.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:38.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:38.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:38.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:38.252 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:38.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:38.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:38.358 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:38.370 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:38 smithi181 bash[59345]: cluster 2023-10-09T09:48:37.014484+0000 osd.7 (osd.7) 601 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:38.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:38.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:38.497 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:38.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:38 smithi163 bash[75425]: cluster 2023-10-09T09:48:37.014484+0000 osd.7 (osd.7) 601 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:38.508 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:38 smithi163 bash[78107]: cluster 2023-10-09T09:48:37.014484+0000 osd.7 (osd.7) 601 : cluster 3 failed to encode map e1252 with expected crc 2023-10-09T09:48:38.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:38.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:38.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:38.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:38.687 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:38.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:38.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:38.793 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:38.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:38.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:38.933 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:38.967 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:38.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:38.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:39.075 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:39.112 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:39.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:39.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:39.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:39.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:39.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:39.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:39.342 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:39.353 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:39 smithi163 bash[75425]: cluster 2023-10-09T09:48:38.137064+0000 mgr.y (mgr.34107) 1280 : cluster 0 pgmap v2127: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 824 op/s; 5354/143133 objects degraded (3.741%); 0 B/s, 77 objects/s recovering 2023-10-09T09:48:39.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:39 smithi163 bash[78107]: cluster 2023-10-09T09:48:38.137064+0000 mgr.y (mgr.34107) 1280 : cluster 0 pgmap v2127: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 824 op/s; 5354/143133 objects degraded (3.741%); 0 B/s, 77 objects/s recovering 2023-10-09T09:48:39.371 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:39.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:39.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:39.474 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:39.501 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:39.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:39.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:39.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:39 smithi181 bash[59345]: cluster 2023-10-09T09:48:38.137064+0000 mgr.y (mgr.34107) 1280 : cluster 0 pgmap v2127: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 824 op/s; 5354/143133 objects degraded (3.741%); 0 B/s, 77 objects/s recovering 2023-10-09T09:48:39.608 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:39.632 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:39.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:39.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:39.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:39.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:39.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:39.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:39.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:39.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:39.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:39.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:40.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:40.029 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:40.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:40.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:40.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:40.161 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:40.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:40.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:40.265 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:40.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:40.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:40.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:40.427 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:40.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:40.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:40.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:40.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:40.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:40.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:40.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:40.730 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:40.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:40.834 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:40.860 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:40.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:40.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:40.963 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:40.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:41.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:41.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:41.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:41.131 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:41.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:41.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:41.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:41.260 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:41.260 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:41.274 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:41.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:41.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:41.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:41.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:41.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:41.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:41.548 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:41.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:41.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:41.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:41.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:41 smithi163 bash[78107]: cluster 2023-10-09T09:48:40.137727+0000 mgr.y (mgr.34107) 1281 : cluster 0 pgmap v2128: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 5354/140610 objects degraded (3.808%); 0 B/s, 91 objects/s recovering 2023-10-09T09:48:41.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:41 smithi163 bash[78107]: audit 2023-10-09T09:48:40.405362+0000 mon.a (mon.0) 2364 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:41.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:41 smithi163 bash[78107]: audit 2023-10-09T09:48:40.408109+0000 mon.a (mon.0) 2365 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:41.661 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:41 smithi163 bash[78107]: cluster 2023-10-09T09:48:40.875747+0000 mon.a (mon.0) 2366 : cluster 3 Health check update: Degraded data redundancy: 5354/140610 objects degraded (3.808%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:41.661 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:41 smithi163 bash[75425]: cluster 2023-10-09T09:48:40.137727+0000 mgr.y (mgr.34107) 1281 : cluster 0 pgmap v2128: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 5354/140610 objects degraded (3.808%); 0 B/s, 91 objects/s recovering 2023-10-09T09:48:41.661 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:41 smithi163 bash[75425]: audit 2023-10-09T09:48:40.405362+0000 mon.a (mon.0) 2364 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:41.661 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:41 smithi163 bash[75425]: audit 2023-10-09T09:48:40.408109+0000 mon.a (mon.0) 2365 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:41.661 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:41 smithi163 bash[75425]: cluster 2023-10-09T09:48:40.875747+0000 mon.a (mon.0) 2366 : cluster 3 Health check update: Degraded data redundancy: 5354/140610 objects degraded (3.808%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:41.683 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:41.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:41.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:41.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:41 smithi181 bash[59345]: cluster 2023-10-09T09:48:40.137727+0000 mgr.y (mgr.34107) 1281 : cluster 0 pgmap v2128: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 5354/140610 objects degraded (3.808%); 0 B/s, 91 objects/s recovering 2023-10-09T09:48:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:41 smithi181 bash[59345]: audit 2023-10-09T09:48:40.405362+0000 mon.a (mon.0) 2364 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:41 smithi181 bash[59345]: audit 2023-10-09T09:48:40.408109+0000 mon.a (mon.0) 2365 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:41 smithi181 bash[59345]: cluster 2023-10-09T09:48:40.875747+0000 mon.a (mon.0) 2366 : cluster 3 Health check update: Degraded data redundancy: 5354/140610 objects degraded (3.808%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:41.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:41.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:41.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:41.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:41.955 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:41.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:41.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:42.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:42.083 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:42.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:42.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:42.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:42.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:42.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:42.280 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:42.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:42.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:42.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:42.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:42.456 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:42.493 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:42.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:42.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:42.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:42.629 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:42.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:42.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:42.731 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:42.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:42.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:42.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:42.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:42.888 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:48:42.889 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:48:42.889 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T09:48:42.889 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:48:42.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:42.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:42.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:42.996 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:43.018 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:43.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:43.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:43.122 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:43.159 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:43.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:43.265 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:43.300 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:43.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:43.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:43.619 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:43 smithi181 bash[59345]: cluster 2023-10-09T09:48:42.138484+0000 mgr.y (mgr.34107) 1282 : cluster 0 pgmap v2129: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 5354/132141 objects degraded (4.052%); 0 B/s, 73 objects/s recovering 2023-10-09T09:48:43.995 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:43 smithi163 bash[75425]: cluster 2023-10-09T09:48:42.138484+0000 mgr.y (mgr.34107) 1282 : cluster 0 pgmap v2129: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 5354/132141 objects degraded (4.052%); 0 B/s, 73 objects/s recovering 2023-10-09T09:48:43.995 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:43 smithi163 bash[78107]: cluster 2023-10-09T09:48:42.138484+0000 mgr.y (mgr.34107) 1282 : cluster 0 pgmap v2129: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 5354/132141 objects degraded (4.052%); 0 B/s, 73 objects/s recovering 2023-10-09T09:48:44.001 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:48:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:48:43] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.33.4" 2023-10-09T09:48:44.021 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:44.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:44.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T09:48:44.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:44.265 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:44.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:44.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:44.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:44.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:44.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:44.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:44.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:44.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:44.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:44.691 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:44.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:44.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:44.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:44.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:44.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:44.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:44.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:44.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:44.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:44.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:44.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:45.095 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:45.115 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:45.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:45.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:45.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:45.248 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:45.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:45.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:45.383 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:45.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:45.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:45.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:45.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:45.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:45.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:45.619 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:45.628 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:45 smithi181 bash[59345]: cluster 2023-10-09T09:48:44.139046+0000 mgr.y (mgr.34107) 1283 : cluster 0 pgmap v2130: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 805 op/s; 5354/129411 objects degraded (4.137%); 0 B/s, 66 objects/s recovering 2023-10-09T09:48:45.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:45.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:45.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:45 smithi163 bash[75425]: cluster 2023-10-09T09:48:44.139046+0000 mgr.y (mgr.34107) 1283 : cluster 0 pgmap v2130: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 805 op/s; 5354/129411 objects degraded (4.137%); 0 B/s, 66 objects/s recovering 2023-10-09T09:48:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:45 smithi163 bash[78107]: cluster 2023-10-09T09:48:44.139046+0000 mgr.y (mgr.34107) 1283 : cluster 0 pgmap v2130: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 805 op/s; 5354/129411 objects degraded (4.137%); 0 B/s, 66 objects/s recovering 2023-10-09T09:48:45.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:45.800 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:45.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:45.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:45.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:45.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:45.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:45.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:46.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:46.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:46.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:46.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:46.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:46.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:46.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:46.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:46.349 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:46.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:46.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:46 smithi163 bash[75425]: cluster 2023-10-09T09:48:45.876866+0000 mon.a (mon.0) 2367 : cluster 3 Health check update: Degraded data redundancy: 5354/129411 objects degraded (4.137%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:46 smithi163 bash[78107]: cluster 2023-10-09T09:48:45.876866+0000 mon.a (mon.0) 2367 : cluster 3 Health check update: Degraded data redundancy: 5354/129411 objects degraded (4.137%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:46.489 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:46.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:46.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:46.592 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:46.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:46.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:46.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:46.725 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:46.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:46.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:46.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:46 smithi181 bash[59345]: cluster 2023-10-09T09:48:45.876866+0000 mon.a (mon.0) 2367 : cluster 3 Health check update: Degraded data redundancy: 5354/129411 objects degraded (4.137%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:46.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:46.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:46.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:46.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:47.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:47.044 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:47.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:47.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:47.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:47.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:47.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:47.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:47.287 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:47.322 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:47.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:47.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:47.428 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:47.459 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:47.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:47.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:47.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:47.591 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:47.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:47.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:47.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:47.719 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:47.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:47.823 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:47.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:47.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:47.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:47.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:47.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:47 smithi163 bash[75425]: cluster 2023-10-09T09:48:46.139939+0000 mgr.y (mgr.34107) 1284 : cluster 0 pgmap v2131: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 706 op/s; 5354/126879 objects degraded (4.220%); 0 B/s, 58 objects/s recovering 2023-10-09T09:48:47.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:47 smithi163 bash[78107]: cluster 2023-10-09T09:48:46.139939+0000 mgr.y (mgr.34107) 1284 : cluster 0 pgmap v2131: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 706 op/s; 5354/126879 objects degraded (4.220%); 0 B/s, 58 objects/s recovering 2023-10-09T09:48:47.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:47.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:48.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:48.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:48.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:47 smithi181 bash[59345]: cluster 2023-10-09T09:48:46.139939+0000 mgr.y (mgr.34107) 1284 : cluster 0 pgmap v2131: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 706 op/s; 5354/126879 objects degraded (4.220%); 0 B/s, 58 objects/s recovering 2023-10-09T09:48:48.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:48.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:48.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:48.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:48.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:48.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:48.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:48.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:48.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:48.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:48.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:48.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:48.551 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:48.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:48.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:48.681 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:48.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:48.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:48.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:48.812 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:48.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:48.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:48.945 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:48.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:48.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:49.009 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:48:49.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:49.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:49.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:49.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:49.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:49.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:49.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:49.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:49.349 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:49.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:49.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:49.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:49.490 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:49.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:49.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:49.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:49.623 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:49.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:49.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:49.730 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:49.762 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:49.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:49.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:49.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:49.907 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:49.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:49.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:50.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:50.040 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:50.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:50.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:49 smithi181 bash[59345]: cluster 2023-10-09T09:48:48.140680+0000 mgr.y (mgr.34107) 1285 : cluster 0 pgmap v2132: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 5354/121971 objects degraded (4.390%); 0 B/s, 80 objects/s recovering 2023-10-09T09:48:50.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:50.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:50.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:49 smithi163 bash[78107]: cluster 2023-10-09T09:48:48.140680+0000 mgr.y (mgr.34107) 1285 : cluster 0 pgmap v2132: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 5354/121971 objects degraded (4.390%); 0 B/s, 80 objects/s recovering 2023-10-09T09:48:50.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:49 smithi163 bash[75425]: cluster 2023-10-09T09:48:48.140680+0000 mgr.y (mgr.34107) 1285 : cluster 0 pgmap v2132: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 5354/121971 objects degraded (4.390%); 0 B/s, 80 objects/s recovering 2023-10-09T09:48:50.271 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:50.299 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:50.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:50.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:50.402 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:50.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:50.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:50.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:50.534 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:50.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:50.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:50.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:50.667 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:50.694 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:50.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:50.803 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:50.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:50.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:50.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:50.938 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:50.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:50.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:50.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:51.071 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:51.100 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:51.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:51.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:51.203 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:51.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:50 smithi163 bash[75425]: cluster 2023-10-09T09:48:50.878078+0000 mon.a (mon.0) 2368 : cluster 3 Health check update: Degraded data redundancy: 5354/121971 objects degraded (4.390%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:51.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:50 smithi163 bash[78107]: cluster 2023-10-09T09:48:50.878078+0000 mon.a (mon.0) 2368 : cluster 3 Health check update: Degraded data redundancy: 5354/121971 objects degraded (4.390%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:51.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:51.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:51.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:51.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:50 smithi181 bash[59345]: cluster 2023-10-09T09:48:50.878078+0000 mon.a (mon.0) 2368 : cluster 3 Health check update: Degraded data redundancy: 5354/121971 objects degraded (4.390%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:51.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:51.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:51.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:51.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:51.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:51.502 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:51.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:51.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:51.608 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:51.633 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:51.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:51.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:51.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:51.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:51.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:51.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:51.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:51.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:51.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:52.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:52.048 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:52.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:52.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:52.151 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:52.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:51 smithi163 bash[75425]: cluster 2023-10-09T09:48:50.141277+0000 mgr.y (mgr.34107) 1286 : cluster 0 pgmap v2133: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 662 op/s; 5354/119292 objects degraded (4.488%); 0 B/s, 64 objects/s recovering 2023-10-09T09:48:52.161 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:51 smithi163 bash[78107]: cluster 2023-10-09T09:48:50.141277+0000 mgr.y (mgr.34107) 1286 : cluster 0 pgmap v2133: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 662 op/s; 5354/119292 objects degraded (4.488%); 0 B/s, 64 objects/s recovering 2023-10-09T09:48:52.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:52.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:52.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:52.214 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:51 smithi181 bash[59345]: cluster 2023-10-09T09:48:50.141277+0000 mgr.y (mgr.34107) 1286 : cluster 0 pgmap v2133: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 662 op/s; 5354/119292 objects degraded (4.488%); 0 B/s, 64 objects/s recovering 2023-10-09T09:48:52.290 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:52.321 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:52.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:52.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:52.331 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:48:52.399 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:48:52.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:52.452 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:48:52.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:52.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:52.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:52.488 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:48:52.535 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:48:52.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:52.568 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:48:52.583 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:52.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:52.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:52.611 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:48:52.647 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:48:52.654 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:48:52.654 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:48:52.654 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:48:52.654 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:48:52.654 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:48:52.654 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:48:52.654 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:48:52.655 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:48:52.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:52.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:52.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:52.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:52.820 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:52.866 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:52.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:52.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:52.926 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:48:52.933 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T09:48:52.959 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:52 smithi163 bash[75425]: audit 2023-10-09T09:48:52.281844+0000 mon.c (mon.2) 117 : audit 0 from='client.? 172.21.15.163:0/3273579613' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:48:52.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:52 smithi163 bash[75425]: audit 2023-10-09T09:48:52.346429+0000 mon.a (mon.0) 2369 : audit 1 from='client.? 172.21.15.163:0/597499941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:48:52.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:52 smithi163 bash[75425]: audit 2023-10-09T09:48:52.741878+0000 mon.a (mon.0) 2370 : audit 1 from='client.? 172.21.15.163:0/3223960250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:48:52.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:52 smithi163 bash[78107]: audit 2023-10-09T09:48:52.281844+0000 mon.c (mon.2) 117 : audit 0 from='client.? 172.21.15.163:0/3273579613' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:48:52.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:52 smithi163 bash[78107]: audit 2023-10-09T09:48:52.346429+0000 mon.a (mon.0) 2369 : audit 1 from='client.? 172.21.15.163:0/597499941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:48:52.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:52 smithi163 bash[78107]: audit 2023-10-09T09:48:52.741878+0000 mon.a (mon.0) 2370 : audit 1 from='client.? 172.21.15.163:0/3223960250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:48:52.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:53.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:53.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:53.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:53.106 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:53.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:52 smithi181 bash[59345]: audit 2023-10-09T09:48:52.281844+0000 mon.c (mon.2) 117 : audit 0 from='client.? 172.21.15.163:0/3273579613' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:48:53.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:52 smithi181 bash[59345]: audit 2023-10-09T09:48:52.346429+0000 mon.a (mon.0) 2369 : audit 1 from='client.? 172.21.15.163:0/597499941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:48:53.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:52 smithi181 bash[59345]: audit 2023-10-09T09:48:52.741878+0000 mon.a (mon.0) 2370 : audit 1 from='client.? 172.21.15.163:0/3223960250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:48:53.138 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:53.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:53.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:53.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:53.289 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:53.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:53.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:53.394 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:53.421 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:53.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:53.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:53.525 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:53.533 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:48:53] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.33.4" 2023-10-09T09:48:53.547 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:53.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:53.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:53.653 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:53.681 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:53.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:53.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:53.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:53.821 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:53.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:53.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:53.925 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:53.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:53.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:53.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:54.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:54.090 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:54.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:54.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:54.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:54.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[78107]: cluster 2023-10-09T09:48:52.142011+0000 mgr.y (mgr.34107) 1287 : cluster 0 pgmap v2134: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 841 op/s; 5354/110319 objects degraded (4.853%); 0 B/s, 86 objects/s recovering 2023-10-09T09:48:54.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[78107]: cluster 2023-10-09T09:48:52.903786+0000 mon.a (mon.0) 2371 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:48:54.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[78107]: audit 2023-10-09T09:48:52.918440+0000 mon.a (mon.0) 2372 : audit 1 from='client.? 172.21.15.163:0/597499941' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:48:54.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[78107]: audit 2023-10-09T09:48:52.919068+0000 mon.a (mon.0) 2373 : audit 1 from='client.? 172.21.15.163:0/3223960250' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:48:54.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[78107]: cluster 2023-10-09T09:48:52.919115+0000 mon.a (mon.0) 2374 : cluster 0 osdmap e1253: 8 total, 8 up, 6 in 2023-10-09T09:48:54.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[78107]: cluster 2023-10-09T09:48:52.920325+0000 osd.6 (osd.6) 538 : cluster 3 failed to encode map e1253 with expected crc 2023-10-09T09:48:54.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[78107]: cluster 2023-10-09T09:48:52.921970+0000 osd.5 (osd.5) 842 : cluster 3 failed to encode map e1253 with expected crc 2023-10-09T09:48:54.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[75425]: cluster 2023-10-09T09:48:52.142011+0000 mgr.y (mgr.34107) 1287 : cluster 0 pgmap v2134: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 841 op/s; 5354/110319 objects degraded (4.853%); 0 B/s, 86 objects/s recovering 2023-10-09T09:48:54.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[75425]: cluster 2023-10-09T09:48:52.903786+0000 mon.a (mon.0) 2371 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:48:54.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[75425]: audit 2023-10-09T09:48:52.918440+0000 mon.a (mon.0) 2372 : audit 1 from='client.? 172.21.15.163:0/597499941' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:48:54.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[75425]: audit 2023-10-09T09:48:52.919068+0000 mon.a (mon.0) 2373 : audit 1 from='client.? 172.21.15.163:0/3223960250' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:48:54.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[75425]: cluster 2023-10-09T09:48:52.919115+0000 mon.a (mon.0) 2374 : cluster 0 osdmap e1253: 8 total, 8 up, 6 in 2023-10-09T09:48:54.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[75425]: cluster 2023-10-09T09:48:52.920325+0000 osd.6 (osd.6) 538 : cluster 3 failed to encode map e1253 with expected crc 2023-10-09T09:48:54.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:53 smithi163 bash[75425]: cluster 2023-10-09T09:48:52.921970+0000 osd.5 (osd.5) 842 : cluster 3 failed to encode map e1253 with expected crc 2023-10-09T09:48:54.224 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:54.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:54.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:53 smithi181 bash[59345]: cluster 2023-10-09T09:48:52.142011+0000 mgr.y (mgr.34107) 1287 : cluster 0 pgmap v2134: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 841 op/s; 5354/110319 objects degraded (4.853%); 0 B/s, 86 objects/s recovering 2023-10-09T09:48:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:53 smithi181 bash[59345]: cluster 2023-10-09T09:48:52.903786+0000 mon.a (mon.0) 2371 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:48:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:53 smithi181 bash[59345]: audit 2023-10-09T09:48:52.918440+0000 mon.a (mon.0) 2372 : audit 1 from='client.? 172.21.15.163:0/597499941' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:48:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:53 smithi181 bash[59345]: audit 2023-10-09T09:48:52.919068+0000 mon.a (mon.0) 2373 : audit 1 from='client.? 172.21.15.163:0/3223960250' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:48:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:53 smithi181 bash[59345]: cluster 2023-10-09T09:48:52.919115+0000 mon.a (mon.0) 2374 : cluster 0 osdmap e1253: 8 total, 8 up, 6 in 2023-10-09T09:48:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:53 smithi181 bash[59345]: cluster 2023-10-09T09:48:52.920325+0000 osd.6 (osd.6) 538 : cluster 3 failed to encode map e1253 with expected crc 2023-10-09T09:48:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:53 smithi181 bash[59345]: cluster 2023-10-09T09:48:52.921970+0000 osd.5 (osd.5) 842 : cluster 3 failed to encode map e1253 with expected crc 2023-10-09T09:48:54.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:54.353 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:54.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:54.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:54.456 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:54.478 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:54.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:54.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:54.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:54.604 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:54.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:54.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:54.710 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:54.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:54.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:54.781 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:48:54.781 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:48:54.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:54.877 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:54.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:54.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:55.010 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:55.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:55.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:55.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:55.156 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:55.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:55.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:55.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:55.309 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:55.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:55.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:55.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:55.444 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:55.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:55.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:55.549 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:55.576 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:55.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:55.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:55.685 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:55.767 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T09:48:55.793 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:55.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:55.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:55.889 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T09:48:55.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:56.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:55 smithi163 bash[75425]: cluster 2023-10-09T09:48:54.142635+0000 mgr.y (mgr.34107) 1288 : cluster 0 pgmap v2136: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 738 op/s; 5354/107262 objects degraded (4.992%); 0 B/s, 74 objects/s recovering 2023-10-09T09:48:56.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:55 smithi163 bash[75425]: audit 2023-10-09T09:48:55.400810+0000 mon.a (mon.0) 2375 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:56.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:55 smithi163 bash[75425]: audit 2023-10-09T09:48:55.402139+0000 mon.a (mon.0) 2376 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:56.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:55 smithi163 bash[75425]: cluster 2023-10-09T09:48:55.879186+0000 mon.a (mon.0) 2377 : cluster 3 Health check update: Degraded data redundancy: 5354/107262 objects degraded (4.992%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:55 smithi181 bash[59345]: cluster 2023-10-09T09:48:54.142635+0000 mgr.y (mgr.34107) 1288 : cluster 0 pgmap v2136: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 738 op/s; 5354/107262 objects degraded (4.992%); 0 B/s, 74 objects/s recovering 2023-10-09T09:48:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:55 smithi181 bash[59345]: audit 2023-10-09T09:48:55.400810+0000 mon.a (mon.0) 2375 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:55 smithi181 bash[59345]: audit 2023-10-09T09:48:55.402139+0000 mon.a (mon.0) 2376 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:55 smithi181 bash[59345]: cluster 2023-10-09T09:48:55.879186+0000 mon.a (mon.0) 2377 : cluster 3 Health check update: Degraded data redundancy: 5354/107262 objects degraded (4.992%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:56.318 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:55 smithi163 bash[78107]: cluster 2023-10-09T09:48:54.142635+0000 mgr.y (mgr.34107) 1288 : cluster 0 pgmap v2136: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 738 op/s; 5354/107262 objects degraded (4.992%); 0 B/s, 74 objects/s recovering 2023-10-09T09:48:56.318 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:55 smithi163 bash[78107]: audit 2023-10-09T09:48:55.400810+0000 mon.a (mon.0) 2375 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:48:56.318 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:55 smithi163 bash[78107]: audit 2023-10-09T09:48:55.402139+0000 mon.a (mon.0) 2376 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:48:56.318 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:55 smithi163 bash[78107]: cluster 2023-10-09T09:48:55.879186+0000 mon.a (mon.0) 2377 : cluster 3 Health check update: Degraded data redundancy: 5354/107262 objects degraded (4.992%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:48:56.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:56.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:56.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:56.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:56.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:56.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:56.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:56.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:56.706 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:56.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:56.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:56.812 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:57.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:57.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:57.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:57.293 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:57.329 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:57.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:57.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:57.435 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:57.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:48:57.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:57.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:48:57.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:57.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:57.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:57.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:57.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:48:57.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:57.787 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:48:57.797 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:57.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:57.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:57.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:57.920 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:57 smithi163 bash[75425]: cluster 2023-10-09T09:48:56.143293+0000 mgr.y (mgr.34107) 1289 : cluster 0 pgmap v2137: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 754 op/s; 5354/104241 objects degraded (5.136%); 0 B/s, 74 objects/s recovering 2023-10-09T09:48:57.921 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:57 smithi163 bash[78107]: cluster 2023-10-09T09:48:56.143293+0000 mgr.y (mgr.34107) 1289 : cluster 0 pgmap v2137: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 754 op/s; 5354/104241 objects degraded (5.136%); 0 B/s, 74 objects/s recovering 2023-10-09T09:48:57.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:48:57.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:57.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:48:58.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:58.082 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:58.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:58.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:58.195 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:58.292 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:57 smithi181 bash[59345]: cluster 2023-10-09T09:48:56.143293+0000 mgr.y (mgr.34107) 1289 : cluster 0 pgmap v2137: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 754 op/s; 5354/104241 objects degraded (5.136%); 0 B/s, 74 objects/s recovering 2023-10-09T09:48:58.308 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:48:58.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:58.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:48:58.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:58.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:58.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:58.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:58.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:58.571 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:58.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:58.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:58.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:58.707 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:58.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:58.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:58.810 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:58.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:48:58.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:58.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:48:58.940 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:58.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:58.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:58.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:59.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:59.100 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:59.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:59.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:59.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:59.229 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:59.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:59.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:59.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:59.360 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:59.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:59.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:59.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:59.490 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:48:59.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:59.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:48:59.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:59.625 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:48:59.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:59.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:48:59.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:59.757 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:48:59.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:59.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:48:59.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:59.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:48:59.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:48:59.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:48:59.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:00.021 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:00.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:00.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:59 smithi181 bash[59345]: cluster 2023-10-09T09:48:58.143950+0000 mgr.y (mgr.34107) 1290 : cluster 0 pgmap v2138: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 778 op/s; 5354/98598 objects degraded (5.430%); 0 B/s, 79 objects/s recovering 2023-10-09T09:49:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:59 smithi181 bash[59345]: cluster 2023-10-09T09:48:58.925949+0000 osd.6 (osd.6) 539 : cluster 3 failed to encode map e1254 with expected crc 2023-10-09T09:49:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:59 smithi181 bash[59345]: cluster 2023-10-09T09:48:58.927892+0000 osd.5 (osd.5) 843 : cluster 3 failed to encode map e1254 with expected crc 2023-10-09T09:49:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:48:59 smithi181 bash[59345]: cluster 2023-10-09T09:48:58.931175+0000 mon.a (mon.0) 2378 : cluster 0 osdmap e1254: 8 total, 8 up, 6 in 2023-10-09T09:49:00.133 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:00.181 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:00.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:00.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:00.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:59 smithi163 bash[78107]: cluster 2023-10-09T09:48:58.143950+0000 mgr.y (mgr.34107) 1290 : cluster 0 pgmap v2138: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 778 op/s; 5354/98598 objects degraded (5.430%); 0 B/s, 79 objects/s recovering 2023-10-09T09:49:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:59 smithi163 bash[78107]: cluster 2023-10-09T09:48:58.925949+0000 osd.6 (osd.6) 539 : cluster 3 failed to encode map e1254 with expected crc 2023-10-09T09:49:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:59 smithi163 bash[78107]: cluster 2023-10-09T09:48:58.927892+0000 osd.5 (osd.5) 843 : cluster 3 failed to encode map e1254 with expected crc 2023-10-09T09:49:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:48:59 smithi163 bash[78107]: cluster 2023-10-09T09:48:58.931175+0000 mon.a (mon.0) 2378 : cluster 0 osdmap e1254: 8 total, 8 up, 6 in 2023-10-09T09:49:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:59 smithi163 bash[75425]: cluster 2023-10-09T09:48:58.143950+0000 mgr.y (mgr.34107) 1290 : cluster 0 pgmap v2138: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 778 op/s; 5354/98598 objects degraded (5.430%); 0 B/s, 79 objects/s recovering 2023-10-09T09:49:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:59 smithi163 bash[75425]: cluster 2023-10-09T09:48:58.925949+0000 osd.6 (osd.6) 539 : cluster 3 failed to encode map e1254 with expected crc 2023-10-09T09:49:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:59 smithi163 bash[75425]: cluster 2023-10-09T09:48:58.927892+0000 osd.5 (osd.5) 843 : cluster 3 failed to encode map e1254 with expected crc 2023-10-09T09:49:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:48:59 smithi163 bash[75425]: cluster 2023-10-09T09:48:58.931175+0000 mon.a (mon.0) 2378 : cluster 0 osdmap e1254: 8 total, 8 up, 6 in 2023-10-09T09:49:00.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:00.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:00.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:00.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:00.410 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:00.450 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:00.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:00.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:00.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:00.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:00.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:00.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:00.642 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:00.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:00.734 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:49:00.752 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:00.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:00.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:00.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:00.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:00.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:00.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:00.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:00 smithi163 bash[75425]: cluster 2023-10-09T09:48:59.939192+0000 osd.5 (osd.5) 844 : cluster 3 failed to encode map e1255 with expected crc 2023-10-09T09:49:00.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:00 smithi163 bash[75425]: cluster 2023-10-09T09:48:59.942163+0000 mon.a (mon.0) 2379 : cluster 0 osdmap e1255: 8 total, 8 up, 6 in 2023-10-09T09:49:00.984 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:01.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:01.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:01.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:01.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:01.152 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:01.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:01.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:00 smithi163 bash[75425]: cluster 2023-10-09T09:48:59.943894+0000 osd.6 (osd.6) 540 : cluster 3 failed to encode map e1255 with expected crc 2023-10-09T09:49:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:00 smithi163 bash[75425]: cluster 2023-10-09T09:49:00.880584+0000 mon.a (mon.0) 2380 : cluster 3 Health check update: Degraded data redundancy: 5354/98598 objects degraded (5.430%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:49:01.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:00 smithi163 bash[78107]: cluster 2023-10-09T09:48:59.939192+0000 osd.5 (osd.5) 844 : cluster 3 failed to encode map e1255 with expected crc 2023-10-09T09:49:01.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:00 smithi163 bash[78107]: cluster 2023-10-09T09:48:59.942163+0000 mon.a (mon.0) 2379 : cluster 0 osdmap e1255: 8 total, 8 up, 6 in 2023-10-09T09:49:01.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:00 smithi163 bash[78107]: cluster 2023-10-09T09:48:59.943894+0000 osd.6 (osd.6) 540 : cluster 3 failed to encode map e1255 with expected crc 2023-10-09T09:49:01.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:00 smithi163 bash[78107]: cluster 2023-10-09T09:49:00.880584+0000 mon.a (mon.0) 2380 : cluster 3 Health check update: Degraded data redundancy: 5354/98598 objects degraded (5.430%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:49:01.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:01.265 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:00 smithi181 bash[59345]: cluster 2023-10-09T09:48:59.939192+0000 osd.5 (osd.5) 844 : cluster 3 failed to encode map e1255 with expected crc 2023-10-09T09:49:01.265 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:00 smithi181 bash[59345]: cluster 2023-10-09T09:48:59.942163+0000 mon.a (mon.0) 2379 : cluster 0 osdmap e1255: 8 total, 8 up, 6 in 2023-10-09T09:49:01.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:00 smithi181 bash[59345]: cluster 2023-10-09T09:48:59.943894+0000 osd.6 (osd.6) 540 : cluster 3 failed to encode map e1255 with expected crc 2023-10-09T09:49:01.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:00 smithi181 bash[59345]: cluster 2023-10-09T09:49:00.880584+0000 mon.a (mon.0) 2380 : cluster 3 Health check update: Degraded data redundancy: 5354/98598 objects degraded (5.430%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T09:49:01.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:01.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:01.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:01.390 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:01.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:01.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:01.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:01.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:01.556 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:01.675 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:01.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:01.776 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:01.809 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:01.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:01.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:01.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:01.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:01.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:01.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:02.055 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:02.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:01 smithi181 bash[59345]: cluster 2023-10-09T09:49:00.144414+0000 mgr.y (mgr.34107) 1291 : cluster 0 pgmap v2141: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 684 op/s; 5354/95487 objects degraded (5.607%); 0 B/s, 56 objects/s recovering 2023-10-09T09:49:02.102 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:02.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:02.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:02.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:02.217 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:01 smithi163 bash[78107]: cluster 2023-10-09T09:49:00.144414+0000 mgr.y (mgr.34107) 1291 : cluster 0 pgmap v2141: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 684 op/s; 5354/95487 objects degraded (5.607%); 0 B/s, 56 objects/s recovering 2023-10-09T09:49:02.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:01 smithi163 bash[75425]: cluster 2023-10-09T09:49:00.144414+0000 mgr.y (mgr.34107) 1291 : cluster 0 pgmap v2141: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 684 op/s; 5354/95487 objects degraded (5.607%); 0 B/s, 56 objects/s recovering 2023-10-09T09:49:02.240 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:02.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:02.344 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:02.372 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:02.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:02.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:02.475 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:02.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:02.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:02.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:02.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:02.640 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:02.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:02.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:02.676 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:02.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:02.777 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:02.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:02.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:02.881 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:02.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:02.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:02.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:03.014 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:03.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:03.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:03.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:03.153 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:03.183 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:03.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:03.290 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:03.320 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:03.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:03.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:03.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:03.447 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:03.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:03.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:03.550 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:03.564 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:49:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:49:03] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.33.4" 2023-10-09T09:49:03.577 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:03.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:03.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:03.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:03.708 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:03.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:03.811 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:03.836 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:03.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:03.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:03.942 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:03.974 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:03.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:03.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:04.076 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:04.105 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:04.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:04.213 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:04.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:03 smithi163 bash[75425]: cluster 2023-10-09T09:49:02.145147+0000 mgr.y (mgr.34107) 1292 : cluster 0 pgmap v2142: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 811 op/s; 5354/87786 objects degraded (6.099%); 0 B/s, 68 objects/s recovering 2023-10-09T09:49:04.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:03 smithi163 bash[78107]: cluster 2023-10-09T09:49:02.145147+0000 mgr.y (mgr.34107) 1292 : cluster 0 pgmap v2142: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 811 op/s; 5354/87786 objects degraded (6.099%); 0 B/s, 68 objects/s recovering 2023-10-09T09:49:04.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:03 smithi181 bash[59345]: cluster 2023-10-09T09:49:02.145147+0000 mgr.y (mgr.34107) 1292 : cluster 0 pgmap v2142: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 811 op/s; 5354/87786 objects degraded (6.099%); 0 B/s, 68 objects/s recovering 2023-10-09T09:49:04.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:04.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:04.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:04.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:04.394 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:04.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:04.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:04.498 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:04.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:04.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:04.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:04.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:04.681 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:04.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:04.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:04.784 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:04.831 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:04.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:04.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:04.989 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:04.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:04.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:05.096 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:05.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:05.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:05.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:05.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:05.256 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:05.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:05.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:05.366 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:05.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:05.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:05.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:05.475 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:49:05.475 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:49:05.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:05.531 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:05.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:05.638 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:05.680 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:05.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:05.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:05.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:05.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:05.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:05.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:05.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:05.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:05.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:06.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:06.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:05 smithi163 bash[75425]: cluster 2023-10-09T09:49:04.145739+0000 mgr.y (mgr.34107) 1293 : cluster 0 pgmap v2143: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 792 op/s; 3566/85221 objects degraded (4.184%); 0 B/s, 112 objects/s recovering 2023-10-09T09:49:06.066 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:05 smithi163 bash[75425]: cluster 2023-10-09T09:49:05.881977+0000 mon.a (mon.0) 2381 : cluster 3 Health check update: Degraded data redundancy: 3566/85221 objects degraded (4.184%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:49:06.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:05 smithi163 bash[78107]: cluster 2023-10-09T09:49:04.145739+0000 mgr.y (mgr.34107) 1293 : cluster 0 pgmap v2143: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 792 op/s; 3566/85221 objects degraded (4.184%); 0 B/s, 112 objects/s recovering 2023-10-09T09:49:06.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:05 smithi163 bash[78107]: cluster 2023-10-09T09:49:05.881977+0000 mon.a (mon.0) 2381 : cluster 3 Health check update: Degraded data redundancy: 3566/85221 objects degraded (4.184%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:49:06.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:06.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:06.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:06.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:06.216 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:06.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:05 smithi181 bash[59345]: cluster 2023-10-09T09:49:04.145739+0000 mgr.y (mgr.34107) 1293 : cluster 0 pgmap v2143: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 792 op/s; 3566/85221 objects degraded (4.184%); 0 B/s, 112 objects/s recovering 2023-10-09T09:49:06.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:05 smithi181 bash[59345]: cluster 2023-10-09T09:49:05.881977+0000 mon.a (mon.0) 2381 : cluster 3 Health check update: Degraded data redundancy: 3566/85221 objects degraded (4.184%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:49:06.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:06.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:06.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:06.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:06.447 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:06.486 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:06.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:06.534 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:49:06.595 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:06.623 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:06.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:06.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:06.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:06.757 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:06.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:06.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:06.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:06.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:07.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:07.035 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:07.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:07.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:07.137 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:07.166 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:07.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:07.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:07.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:07.299 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:07.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:07.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:07.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:07.440 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:07.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:07.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:07.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:07.576 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:07.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:07.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:07.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:07.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:07.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:07.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:07.849 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:07.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:07.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:07.956 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:07.988 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:07.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:07.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:08.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:08.103 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:08 smithi163 bash[78107]: cluster 2023-10-09T09:49:06.146378+0000 mgr.y (mgr.34107) 1294 : cluster 0 pgmap v2144: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 676 op/s; 3566/82353 objects degraded (4.330%); 0 B/s, 61 objects/s recovering 2023-10-09T09:49:08.103 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:08 smithi163 bash[75425]: cluster 2023-10-09T09:49:06.146378+0000 mgr.y (mgr.34107) 1294 : cluster 0 pgmap v2144: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 676 op/s; 3566/82353 objects degraded (4.330%); 0 B/s, 61 objects/s recovering 2023-10-09T09:49:08.120 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:08.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:08.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:08.225 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:08.234 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:07 smithi181 bash[59345]: cluster 2023-10-09T09:49:06.146378+0000 mgr.y (mgr.34107) 1294 : cluster 0 pgmap v2144: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 676 op/s; 3566/82353 objects degraded (4.330%); 0 B/s, 61 objects/s recovering 2023-10-09T09:49:08.250 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:08.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:08.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:08.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:08.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:08.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:08.510 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:08.539 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:08.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:08.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:08.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:08.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:08.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:08.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:08.777 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:08.806 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:08.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:08.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:08.913 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:08.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:08.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:08.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:09.050 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:09.087 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:09.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:09.194 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:09.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:09.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:09.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:09.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:09.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:09.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:09.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:09.484 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:09.512 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:09.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:09.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:09.616 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:09.650 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:09.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:09.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:09.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:09.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:09.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:09.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:09.913 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:09.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:09.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:09 smithi181 bash[59345]: cluster 2023-10-09T09:49:08.147037+0000 mgr.y (mgr.34107) 1295 : cluster 0 pgmap v2145: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 3566/76884 objects degraded (4.638%); 0 B/s, 53 objects/s recovering 2023-10-09T09:49:10.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:10.049 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:10.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:10.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:10.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:10.165 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:09 smithi163 bash[78107]: cluster 2023-10-09T09:49:08.147037+0000 mgr.y (mgr.34107) 1295 : cluster 0 pgmap v2145: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 3566/76884 objects degraded (4.638%); 0 B/s, 53 objects/s recovering 2023-10-09T09:49:10.165 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:09 smithi163 bash[75425]: cluster 2023-10-09T09:49:08.147037+0000 mgr.y (mgr.34107) 1295 : cluster 0 pgmap v2145: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 3566/76884 objects degraded (4.638%); 0 B/s, 53 objects/s recovering 2023-10-09T09:49:10.185 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:10.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:10.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:10.290 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:10.318 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:10.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:10.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:10.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:10.457 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:10.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:10.560 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:10.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:10.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:10.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:10.726 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:10.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:10.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:10.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:10.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:10.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:10.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:10.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:10.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:10.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:11.085 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:11.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:11.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:11.222 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:11.231 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:10 smithi181 bash[59345]: audit 2023-10-09T09:49:10.396679+0000 mon.a (mon.0) 2382 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:11.231 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:10 smithi181 bash[59345]: cluster 2023-10-09T09:49:10.885959+0000 mon.a (mon.0) 2383 : cluster 3 Health check update: Degraded data redundancy: 3566/76884 objects degraded (4.638%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:49:11.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:11 smithi163 bash[78107]: audit 2023-10-09T09:49:10.396679+0000 mon.a (mon.0) 2382 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:11.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:11 smithi163 bash[78107]: cluster 2023-10-09T09:49:10.885959+0000 mon.a (mon.0) 2383 : cluster 3 Health check update: Degraded data redundancy: 3566/76884 objects degraded (4.638%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:49:11.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:10 smithi163 bash[75425]: audit 2023-10-09T09:49:10.396679+0000 mon.a (mon.0) 2382 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:11.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:10 smithi163 bash[75425]: cluster 2023-10-09T09:49:10.885959+0000 mon.a (mon.0) 2383 : cluster 3 Health check update: Degraded data redundancy: 3566/76884 objects degraded (4.638%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:49:11.258 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:11.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:11.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:11.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:11.390 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:11.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:11.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:11.521 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:11.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:11.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:11.625 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:11.667 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:11.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:11.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:11.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:11.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:11.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:11.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:11.951 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:11.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:12.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:12.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:11 smithi181 bash[59345]: cluster 2023-10-09T09:49:10.147738+0000 mgr.y (mgr.34107) 1296 : cluster 0 pgmap v2146: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 696 op/s; 3566/74151 objects degraded (4.809%); 0 B/s, 77 objects/s recovering 2023-10-09T09:49:12.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:12.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:12.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:12.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:12.218 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:12.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:12.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:12.328 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:12.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:12.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:12.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:12.473 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:12.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:11 smithi163 bash[75425]: cluster 2023-10-09T09:49:10.147738+0000 mgr.y (mgr.34107) 1296 : cluster 0 pgmap v2146: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 696 op/s; 3566/74151 objects degraded (4.809%); 0 B/s, 77 objects/s recovering 2023-10-09T09:49:12.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:11 smithi163 bash[78107]: cluster 2023-10-09T09:49:10.147738+0000 mgr.y (mgr.34107) 1296 : cluster 0 pgmap v2146: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 696 op/s; 3566/74151 objects degraded (4.809%); 0 B/s, 77 objects/s recovering 2023-10-09T09:49:12.514 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:12.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:12.617 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:12.646 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:12.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:12.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:12.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:12.779 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:12.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:12.885 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:12.915 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:12.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:12.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:13.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:13.054 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:13.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:13.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:13.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:13.186 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:13.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:13.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:13.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:13.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:13.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:13.420 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:13.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:13.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:13.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:13.556 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:13.577 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:13.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:13.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:13.679 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:13.688 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:49:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:49:13] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.33.4" 2023-10-09T09:49:13.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:13.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:13.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:13.816 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:13.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:13.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:13.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:13.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:13.980 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:13.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:13.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:14.082 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:14.091 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:13 smithi181 bash[59345]: cluster 2023-10-09T09:49:12.148534+0000 mgr.y (mgr.34107) 1297 : cluster 0 pgmap v2147: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 831 op/s; 3566/65547 objects degraded (5.440%); 0 B/s, 65 objects/s recovering 2023-10-09T09:49:14.114 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:14.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:14.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:14.216 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:14.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:13 smithi163 bash[75425]: cluster 2023-10-09T09:49:12.148534+0000 mgr.y (mgr.34107) 1297 : cluster 0 pgmap v2147: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 831 op/s; 3566/65547 objects degraded (5.440%); 0 B/s, 65 objects/s recovering 2023-10-09T09:49:14.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:13 smithi163 bash[78107]: cluster 2023-10-09T09:49:12.148534+0000 mgr.y (mgr.34107) 1297 : cluster 0 pgmap v2147: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 831 op/s; 3566/65547 objects degraded (5.440%); 0 B/s, 65 objects/s recovering 2023-10-09T09:49:14.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:14.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:14.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:14.350 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:14.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:14.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:14.481 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:14.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:14.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:14.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:14.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:14.647 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:14.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:14.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:14.757 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:14.780 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:14.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:14.883 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:14.922 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:14.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:14.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:15.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:15.054 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:15.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:15.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:15.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:15.182 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:15.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:15.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:15.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:15.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:15.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:15.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:15.429 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:15.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:15.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:15.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:15.574 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:15.610 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:15.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:15.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:15.717 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:15.747 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:15.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:15.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:15.854 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:15.885 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:15.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:15.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:15.967 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:15.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:16.007 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:16.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:16.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:16.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:16 smithi163 bash[75425]: cluster 2023-10-09T09:49:14.149062+0000 mgr.y (mgr.34107) 1298 : cluster 0 pgmap v2148: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 3566/62589 objects degraded (5.697%); 0 B/s, 82 objects/s recovering 2023-10-09T09:49:16.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:16 smithi163 bash[75425]: cluster 2023-10-09T09:49:15.886960+0000 mon.a (mon.0) 2384 : cluster 3 Health check update: Degraded data redundancy: 3566/62589 objects degraded (5.697%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:16.018 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:16 smithi163 bash[78107]: cluster 2023-10-09T09:49:14.149062+0000 mgr.y (mgr.34107) 1298 : cluster 0 pgmap v2148: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 3566/62589 objects degraded (5.697%); 0 B/s, 82 objects/s recovering 2023-10-09T09:49:16.018 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:16 smithi163 bash[78107]: cluster 2023-10-09T09:49:15.886960+0000 mon.a (mon.0) 2384 : cluster 3 Health check update: Degraded data redundancy: 3566/62589 objects degraded (5.697%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:16.109 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:16.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:15 smithi181 bash[59345]: cluster 2023-10-09T09:49:14.149062+0000 mgr.y (mgr.34107) 1298 : cluster 0 pgmap v2148: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 3566/62589 objects degraded (5.697%); 0 B/s, 82 objects/s recovering 2023-10-09T09:49:16.118 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:15 smithi181 bash[59345]: cluster 2023-10-09T09:49:15.886960+0000 mon.a (mon.0) 2384 : cluster 3 Health check update: Degraded data redundancy: 3566/62589 objects degraded (5.697%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:16.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:16.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:16.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:16.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:16.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:16.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:16.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:16.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:16.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:16.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:16.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:16.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:16.557 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:16.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:16.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:16.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:16.697 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:16.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:16.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:16.803 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:16.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:16.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:16.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:16.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:16.978 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:16.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:16.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:17.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:17.108 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:17.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:17.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:17.214 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:17.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:17.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:17.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:17.345 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:17.378 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:17.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:17.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:17.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:17.506 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:17.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:17.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:17.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:17.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:17.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:17.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:17.774 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:17.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:17.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:17.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:17.909 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:17.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:17.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:18.006 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T09:49:18.014 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:18.045 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:18.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:18.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:18.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:17 smithi163 bash[75425]: cluster 2023-10-09T09:49:16.149616+0000 mgr.y (mgr.34107) 1299 : cluster 0 pgmap v2149: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 705 op/s; 3566/59811 objects degraded (5.962%); 0 B/s, 53 objects/s recovering 2023-10-09T09:49:18.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:17 smithi163 bash[75425]: audit 2023-10-09T09:49:17.002771+0000 mon.c (mon.2) 118 : audit 1 from='client.? 172.21.15.163:0/1979486916' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:49:18.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:17 smithi163 bash[75425]: audit 2023-10-09T09:49:17.003274+0000 mon.a (mon.0) 2385 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:49:18.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:17 smithi163 bash[78107]: cluster 2023-10-09T09:49:16.149616+0000 mgr.y (mgr.34107) 1299 : cluster 0 pgmap v2149: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 705 op/s; 3566/59811 objects degraded (5.962%); 0 B/s, 53 objects/s recovering 2023-10-09T09:49:18.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:17 smithi163 bash[78107]: audit 2023-10-09T09:49:17.002771+0000 mon.c (mon.2) 118 : audit 1 from='client.? 172.21.15.163:0/1979486916' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:49:18.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:17 smithi163 bash[78107]: audit 2023-10-09T09:49:17.003274+0000 mon.a (mon.0) 2385 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:49:18.151 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:18.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:17 smithi181 bash[59345]: cluster 2023-10-09T09:49:16.149616+0000 mgr.y (mgr.34107) 1299 : cluster 0 pgmap v2149: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 705 op/s; 3566/59811 objects degraded (5.962%); 0 B/s, 53 objects/s recovering 2023-10-09T09:49:18.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:17 smithi181 bash[59345]: audit 2023-10-09T09:49:17.002771+0000 mon.c (mon.2) 118 : audit 1 from='client.? 172.21.15.163:0/1979486916' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:49:18.161 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:17 smithi181 bash[59345]: audit 2023-10-09T09:49:17.003274+0000 mon.a (mon.0) 2385 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:49:18.184 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:18.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:18.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:18.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:18.319 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:18.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:18.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:18.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:18.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:18.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:18.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:18.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:18.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:18.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:18.693 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:18.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:18.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:18.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:18.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:18.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:18.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:18.969 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:19.006 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:19.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:19.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:19.109 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:19.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:19.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:19.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:19.244 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:19.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:19 smithi163 bash[75425]: cluster 2023-10-09T09:49:17.994317+0000 osd.5 (osd.5) 845 : cluster 3 failed to encode map e1256 with expected crc 2023-10-09T09:49:19.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:19 smithi163 bash[75425]: audit 2023-10-09T09:49:17.996991+0000 mon.a (mon.0) 2386 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:49:19.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:19 smithi163 bash[75425]: cluster 2023-10-09T09:49:17.997058+0000 mon.a (mon.0) 2387 : cluster 0 osdmap e1256: 8 total, 8 up, 6 in 2023-10-09T09:49:19.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:19 smithi163 bash[75425]: audit 2023-10-09T09:49:18.204405+0000 mon.a (mon.0) 2388 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:19.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:19 smithi163 bash[78107]: cluster 2023-10-09T09:49:17.994317+0000 osd.5 (osd.5) 845 : cluster 3 failed to encode map e1256 with expected crc 2023-10-09T09:49:19.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:19 smithi163 bash[78107]: audit 2023-10-09T09:49:17.996991+0000 mon.a (mon.0) 2386 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:49:19.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:19 smithi163 bash[78107]: cluster 2023-10-09T09:49:17.997058+0000 mon.a (mon.0) 2387 : cluster 0 osdmap e1256: 8 total, 8 up, 6 in 2023-10-09T09:49:19.255 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:19 smithi163 bash[78107]: audit 2023-10-09T09:49:18.204405+0000 mon.a (mon.0) 2388 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:19.276 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:19.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:19.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:19 smithi181 bash[59345]: cluster 2023-10-09T09:49:17.994317+0000 osd.5 (osd.5) 845 : cluster 3 failed to encode map e1256 with expected crc 2023-10-09T09:49:19.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:19 smithi181 bash[59345]: audit 2023-10-09T09:49:17.996991+0000 mon.a (mon.0) 2386 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:49:19.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:19 smithi181 bash[59345]: cluster 2023-10-09T09:49:17.997058+0000 mon.a (mon.0) 2387 : cluster 0 osdmap e1256: 8 total, 8 up, 6 in 2023-10-09T09:49:19.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:19 smithi181 bash[59345]: audit 2023-10-09T09:49:18.204405+0000 mon.a (mon.0) 2388 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:19.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:19.410 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:19.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:19.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:19.516 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:19.540 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:19.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:19.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:19.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:19.681 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:19.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:19.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:19.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:19.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:19.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:19.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:19.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:19.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:19.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:20.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:20 smithi163 bash[75425]: cluster 2023-10-09T09:49:18.150082+0000 mgr.y (mgr.34107) 1300 : cluster 0 pgmap v2151: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 753 op/s; 3566/54276 objects degraded (6.570%); 0 B/s, 63 objects/s recovering 2023-10-09T09:49:20.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:20 smithi163 bash[78107]: cluster 2023-10-09T09:49:18.150082+0000 mgr.y (mgr.34107) 1300 : cluster 0 pgmap v2151: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 753 op/s; 3566/54276 objects degraded (6.570%); 0 B/s, 63 objects/s recovering 2023-10-09T09:49:20.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:20.087 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:20.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:20.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:20.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:20.218 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:20.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:20.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:20 smithi181 bash[59345]: cluster 2023-10-09T09:49:18.150082+0000 mgr.y (mgr.34107) 1300 : cluster 0 pgmap v2151: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 753 op/s; 3566/54276 objects degraded (6.570%); 0 B/s, 63 objects/s recovering 2023-10-09T09:49:20.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:20.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:20.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:20.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:20.453 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:20.482 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:20.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:20.587 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:20.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:20.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:20.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:20.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:20.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:20.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:20.776 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:49:20.820 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:49:20.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:20.866 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:49:20.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:20.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:20.909 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:49:20.948 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:49:20.995 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:49:21.002 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:21.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:21 smithi181 bash[59345]: audit 2023-10-09T09:49:20.353551+0000 mon.a (mon.0) 2389 : audit 1 from='client.? 172.21.15.163:0/2758166746' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:49:21.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:21 smithi181 bash[59345]: audit 2023-10-09T09:49:20.710761+0000 mon.a (mon.0) 2390 : audit 0 from='client.? 172.21.15.163:0/3744442706' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:49:21.006 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:21 smithi181 bash[59345]: cluster 2023-10-09T09:49:20.890436+0000 mon.a (mon.0) 2391 : cluster 3 Health check update: Degraded data redundancy: 3566/54276 objects degraded (6.570%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:21.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:21 smithi163 bash[75425]: audit 2023-10-09T09:49:20.353551+0000 mon.a (mon.0) 2389 : audit 1 from='client.? 172.21.15.163:0/2758166746' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:49:21.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:21 smithi163 bash[75425]: audit 2023-10-09T09:49:20.710761+0000 mon.a (mon.0) 2390 : audit 0 from='client.? 172.21.15.163:0/3744442706' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:49:21.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:21 smithi163 bash[75425]: cluster 2023-10-09T09:49:20.890436+0000 mon.a (mon.0) 2391 : cluster 3 Health check update: Degraded data redundancy: 3566/54276 objects degraded (6.570%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:21.027 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:49:21.037 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:21.039 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:49:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:21.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:21.078 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:49:21.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:49:21.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:49:21.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:49:21.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:49:21.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:49:21.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:49:21.082 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:49:21.082 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:49:21.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:21.193 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:21.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:21.294 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:21.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:21.327 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:21.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:21.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T09:49:21.409 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:21 smithi163 bash[78107]: audit 2023-10-09T09:49:20.353551+0000 mon.a (mon.0) 2389 : audit 1 from='client.? 172.21.15.163:0/2758166746' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:49:21.409 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:21 smithi163 bash[78107]: audit 2023-10-09T09:49:20.710761+0000 mon.a (mon.0) 2390 : audit 0 from='client.? 172.21.15.163:0/3744442706' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:49:21.409 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:21 smithi163 bash[78107]: cluster 2023-10-09T09:49:20.890436+0000 mon.a (mon.0) 2391 : cluster 3 Health check update: Degraded data redundancy: 3566/54276 objects degraded (6.570%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:21.428 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:21.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:21.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:21.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:21.580 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:21.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:21.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:21.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:21.714 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:21.741 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:21.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:21.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:21.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:21.871 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:21.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:21.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:21.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:22.004 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:22.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:22.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:22.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:22.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:22.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[78107]: cluster 2023-10-09T09:49:20.150670+0000 mgr.y (mgr.34107) 1301 : cluster 0 pgmap v2152: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 754 op/s; 3566/51498 objects degraded (6.925%); 0 B/s, 72 objects/s recovering 2023-10-09T09:49:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[78107]: cluster 2023-10-09T09:49:21.003953+0000 mon.a (mon.0) 2392 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:49:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[78107]: cluster 2023-10-09T09:49:21.018721+0000 osd.5 (osd.5) 846 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[78107]: audit 2023-10-09T09:49:21.020063+0000 mon.a (mon.0) 2393 : audit 1 from='client.? 172.21.15.163:0/2758166746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:49:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[78107]: cluster 2023-10-09T09:49:21.020303+0000 mon.a (mon.0) 2394 : cluster 0 osdmap e1257: 8 total, 8 up, 6 in 2023-10-09T09:49:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[78107]: cluster 2023-10-09T09:49:21.026584+0000 osd.6 (osd.6) 541 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[78107]: cluster 2023-10-09T09:49:21.032466+0000 osd.4 (osd.4) 655 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[78107]: cluster 2023-10-09T09:49:21.034902+0000 osd.4 (osd.4) 656 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[78107]: cluster 2023-10-09T09:49:21.622060+0000 osd.7 (osd.7) 602 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[75425]: cluster 2023-10-09T09:49:20.150670+0000 mgr.y (mgr.34107) 1301 : cluster 0 pgmap v2152: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 754 op/s; 3566/51498 objects degraded (6.925%); 0 B/s, 72 objects/s recovering 2023-10-09T09:49:22.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[75425]: cluster 2023-10-09T09:49:21.003953+0000 mon.a (mon.0) 2392 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:49:22.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[75425]: cluster 2023-10-09T09:49:21.018721+0000 osd.5 (osd.5) 846 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[75425]: audit 2023-10-09T09:49:21.020063+0000 mon.a (mon.0) 2393 : audit 1 from='client.? 172.21.15.163:0/2758166746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:49:22.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[75425]: cluster 2023-10-09T09:49:21.020303+0000 mon.a (mon.0) 2394 : cluster 0 osdmap e1257: 8 total, 8 up, 6 in 2023-10-09T09:49:22.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[75425]: cluster 2023-10-09T09:49:21.026584+0000 osd.6 (osd.6) 541 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[75425]: cluster 2023-10-09T09:49:21.032466+0000 osd.4 (osd.4) 655 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[75425]: cluster 2023-10-09T09:49:21.034902+0000 osd.4 (osd.4) 656 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:22 smithi163 bash[75425]: cluster 2023-10-09T09:49:21.622060+0000 osd.7 (osd.7) 602 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:22.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:22.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:22.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:22 smithi181 bash[59345]: cluster 2023-10-09T09:49:20.150670+0000 mgr.y (mgr.34107) 1301 : cluster 0 pgmap v2152: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 754 op/s; 3566/51498 objects degraded (6.925%); 0 B/s, 72 objects/s recovering 2023-10-09T09:49:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:22 smithi181 bash[59345]: cluster 2023-10-09T09:49:21.003953+0000 mon.a (mon.0) 2392 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:49:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:22 smithi181 bash[59345]: cluster 2023-10-09T09:49:21.018721+0000 osd.5 (osd.5) 846 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:22 smithi181 bash[59345]: audit 2023-10-09T09:49:21.020063+0000 mon.a (mon.0) 2393 : audit 1 from='client.? 172.21.15.163:0/2758166746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:49:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:22 smithi181 bash[59345]: cluster 2023-10-09T09:49:21.020303+0000 mon.a (mon.0) 2394 : cluster 0 osdmap e1257: 8 total, 8 up, 6 in 2023-10-09T09:49:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:22 smithi181 bash[59345]: cluster 2023-10-09T09:49:21.026584+0000 osd.6 (osd.6) 541 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:22 smithi181 bash[59345]: cluster 2023-10-09T09:49:21.032466+0000 osd.4 (osd.4) 655 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:22 smithi181 bash[59345]: cluster 2023-10-09T09:49:21.034902+0000 osd.4 (osd.4) 656 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:22 smithi181 bash[59345]: cluster 2023-10-09T09:49:21.622060+0000 osd.7 (osd.7) 602 : cluster 3 failed to encode map e1257 with expected crc 2023-10-09T09:49:22.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:22.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:22.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:22.503 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:22.533 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:22.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:22.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:22.638 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:22.662 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:22.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:22.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:22.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:22.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:22.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:22.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:22.908 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:22.943 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:22.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:22.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:23.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:23.071 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:23.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:23.173 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:23.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:23.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:23.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:23.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:23.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:23.453 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:23.479 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:23.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:23.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:23.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:23.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:23.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:23.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:23.612 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:49:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:49:23] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.33.4" 2023-10-09T09:49:23.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:23.759 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:23.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:23.866 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:23.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:23.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:23.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:24.013 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:24.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:24 smithi181 bash[59345]: cluster 2023-10-09T09:49:22.151384+0000 mgr.y (mgr.34107) 1302 : cluster 0 pgmap v2154: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 903 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 800 op/s; 3566/43380 objects degraded (8.220%); 0 B/s, 47 objects/s recovering 2023-10-09T09:49:24.041 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:24.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:24.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:24.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:24.171 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:24.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:24.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:24.290 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:24 smithi163 bash[78107]: cluster 2023-10-09T09:49:22.151384+0000 mgr.y (mgr.34107) 1302 : cluster 0 pgmap v2154: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 903 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 800 op/s; 3566/43380 objects degraded (8.220%); 0 B/s, 47 objects/s recovering 2023-10-09T09:49:24.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:24 smithi163 bash[75425]: cluster 2023-10-09T09:49:22.151384+0000 mgr.y (mgr.34107) 1302 : cluster 0 pgmap v2154: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 903 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 800 op/s; 3566/43380 objects degraded (8.220%); 0 B/s, 47 objects/s recovering 2023-10-09T09:49:24.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:24.323 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:24.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:24.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:24.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:24.450 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:24.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:24.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:24.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:24.586 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:24.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:24.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:24.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:24.717 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:24.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:24.820 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:24.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:24.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:24.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:24.950 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:24.977 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:24.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:24.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:25.081 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:25.107 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:25.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:25.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:25.214 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:25.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:25 smithi181 bash[59345]: cluster 2023-10-09T09:49:24.012500+0000 osd.5 (osd.5) 847 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:25 smithi181 bash[59345]: cluster 2023-10-09T09:49:24.012824+0000 mon.a (mon.0) 2395 : cluster 0 osdmap e1258: 8 total, 8 up, 6 in 2023-10-09T09:49:25.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:25 smithi181 bash[59345]: cluster 2023-10-09T09:49:24.016509+0000 osd.7 (osd.7) 603 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:25 smithi181 bash[59345]: cluster 2023-10-09T09:49:24.029165+0000 osd.4 (osd.4) 657 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:25 smithi181 bash[59345]: cluster 2023-10-09T09:49:24.030979+0000 osd.6 (osd.6) 542 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:25 smithi181 bash[59345]: cluster 2023-10-09T09:49:24.036300+0000 osd.6 (osd.6) 543 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:25 smithi181 bash[59345]: audit 2023-10-09T09:49:24.758149+0000 mon.a (mon.0) 2396 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:25.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:25 smithi181 bash[59345]: audit 2023-10-09T09:49:24.764850+0000 mon.a (mon.0) 2397 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:25.259 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:25.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:25.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:25.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:25.375 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[78107]: cluster 2023-10-09T09:49:24.012500+0000 osd.5 (osd.5) 847 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.375 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[78107]: cluster 2023-10-09T09:49:24.012824+0000 mon.a (mon.0) 2395 : cluster 0 osdmap e1258: 8 total, 8 up, 6 in 2023-10-09T09:49:25.376 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[78107]: cluster 2023-10-09T09:49:24.016509+0000 osd.7 (osd.7) 603 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.376 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[78107]: cluster 2023-10-09T09:49:24.029165+0000 osd.4 (osd.4) 657 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.376 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[78107]: cluster 2023-10-09T09:49:24.030979+0000 osd.6 (osd.6) 542 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.376 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[78107]: cluster 2023-10-09T09:49:24.036300+0000 osd.6 (osd.6) 543 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.376 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[78107]: audit 2023-10-09T09:49:24.758149+0000 mon.a (mon.0) 2396 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:25.376 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[78107]: audit 2023-10-09T09:49:24.764850+0000 mon.a (mon.0) 2397 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:25.376 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[75425]: cluster 2023-10-09T09:49:24.012500+0000 osd.5 (osd.5) 847 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.376 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[75425]: cluster 2023-10-09T09:49:24.012824+0000 mon.a (mon.0) 2395 : cluster 0 osdmap e1258: 8 total, 8 up, 6 in 2023-10-09T09:49:25.376 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[75425]: cluster 2023-10-09T09:49:24.016509+0000 osd.7 (osd.7) 603 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[75425]: cluster 2023-10-09T09:49:24.029165+0000 osd.4 (osd.4) 657 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[75425]: cluster 2023-10-09T09:49:24.030979+0000 osd.6 (osd.6) 542 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[75425]: cluster 2023-10-09T09:49:24.036300+0000 osd.6 (osd.6) 543 : cluster 3 failed to encode map e1258 with expected crc 2023-10-09T09:49:25.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[75425]: audit 2023-10-09T09:49:24.758149+0000 mon.a (mon.0) 2396 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:25.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:25 smithi163 bash[75425]: audit 2023-10-09T09:49:24.764850+0000 mon.a (mon.0) 2397 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:25.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:25.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:25.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:25.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:25.542 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:25.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:25.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:25.645 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:25.672 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:25.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:25.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:25.775 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:25.805 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:25.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:25.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:25.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:25.937 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:25.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:25.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:26.007 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45545.0:207909 10.a e1259/1232 10:50ef0470:::benchmark_data_smithi163_194945_object98456:head v 1259'12856, mlcod=1259'12854)", 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:25.983973+0000", 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.020514972999999999, 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020549082999999999, 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.983973+0000", 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.983973+0000", 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:49:26.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.983964+0000", 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999905 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.983979+0000", 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5026e-05 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.983992+0000", 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2490000000000001e-05 2023-10-09T09:49:26.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.983997+0000", 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5910000000000003e-06 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.986067+0000", 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020695069999999999 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.986135+0000", 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8752999999999997e-05 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207918 10.6 10:60b1ff0b:::benchmark_data_smithi163_194945_object98465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1259)", 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:25.994382+0000", 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.010105935, 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010211435, 2023-10-09T09:49:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207918 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.994382+0000", 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.994382+0000", 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.994381+0000", 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:49:26.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.994386+0000", 2023-10-09T09:49:26.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3630000000000002e-06 2023-10-09T09:49:26.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:49:26.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.994398+0000", 2023-10-09T09:49:26.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2075999999999999e-05 2023-10-09T09:49:26.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:49:26.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.994406+0000", 2023-10-09T09:49:26.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8569999999999995e-06 2023-10-09T09:49:26.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:49:26.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.994451+0000", 2023-10-09T09:49:26.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4484000000000003e-05 2023-10-09T09:49:26.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:49:26.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.994589+0000", 2023-10-09T09:49:26.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013862499999999999 2023-10-09T09:49:26.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,5", 2023-10-09T09:49:26.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.994715+0000", 2023-10-09T09:49:26.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012568599999999999 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.999229+0000", 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045137880000000003 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.999236+0000", 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8530000000000003e-06 2023-10-09T09:49:26.018 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.019 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:49:26.019 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.019 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207922 10.6 10:6e3b847a:::benchmark_data_smithi163_194945_object98469:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1259)", 2023-10-09T09:49:26.019 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:26.002310+0000", 2023-10-09T09:49:26.019 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0021784339999999999, 2023-10-09T09:49:26.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002371906, 2023-10-09T09:49:26.019 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:49:26.020 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:49:26.020 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:49:26.020 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:49:26.020 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:49:26.020 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207922 2023-10-09T09:49:26.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.020 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:49:26.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:49:26.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002310+0000", 2023-10-09T09:49:26.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:49:26.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002310+0000", 2023-10-09T09:49:26.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002308+0000", 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002315+0000", 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.404e-06 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:49:26.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002329+0000", 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.396e-05 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002369+0000", 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0541999999999999e-05 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002443+0000", 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3139000000000006e-05 2023-10-09T09:49:26.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002607+0000", 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016478900000000001 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,5", 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002748+0000", 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014103799999999999 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:49:26.024 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207923 10.4 10:287a02d3:::benchmark_data_smithi163_194945_object98470:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1259)", 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:26.002384+0000", 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0021039370000000002, 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002371345, 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:49:26.025 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:49:26.026 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207923 2023-10-09T09:49:26.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.026 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:49:26.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:49:26.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002384+0000", 2023-10-09T09:49:26.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002384+0000", 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002384+0000", 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:49:26.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002388+0000", 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7890000000000002e-06 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002407+0000", 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8397999999999999e-05 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002415+0000", 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7950000000000008e-06 2023-10-09T09:49:26.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:49:26.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002503+0000", 2023-10-09T09:49:26.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8861000000000002e-05 2023-10-09T09:49:26.030 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.031 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:49:26.031 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.031 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45545.0:207915 10.5 e1259/1232)", 2023-10-09T09:49:26.031 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:26.002461+0000", 2023-10-09T09:49:26.031 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.002027647, 2023-10-09T09:49:26.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023530880000000001, 2023-10-09T09:49:26.032 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:49:26.033 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:49:26.033 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:49:26.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:49:26.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002461+0000", 2023-10-09T09:49:26.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:49:26.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002461+0000", 2023-10-09T09:49:26.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:49:26.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002460+0000", 2023-10-09T09:49:26.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:49:26.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:49:26.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002465+0000", 2023-10-09T09:49:26.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6640000000000003e-06 2023-10-09T09:49:26.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:49:26.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002479+0000", 2023-10-09T09:49:26.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3444999999999999e-05 2023-10-09T09:49:26.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:49:26.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002484+0000", 2023-10-09T09:49:26.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5770000000000001e-06 2023-10-09T09:49:26.038 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.038 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:49:26.038 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.038 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45545.0:207921 10.8 e1259/1232 10:1f5eb0e0:::benchmark_data_smithi163_194945_object98468:head v 1259'13185, mlcod=1259'13183)", 2023-10-09T09:49:26.038 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:26.002497+0000", 2023-10-09T09:49:26.038 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.001991666, 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023666260000000001, 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002497+0000", 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:49:26.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002497+0000", 2023-10-09T09:49:26.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:49:26.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002496+0000", 2023-10-09T09:49:26.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:49:26.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002501+0000", 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9760000000000003e-06 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002507+0000", 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8370000000000002e-06 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002512+0000", 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.758e-06 2023-10-09T09:49:26.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002546+0000", 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3923e-05 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002566+0000", 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0341000000000001e-05 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.042 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207927 10.4 10.527fc7f4 (undecoded) ondisk+write+known_if_redirected e1259)", 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:26.002717+0000", 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.001771402, 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022041420000000001, 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:49:26.043 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207927 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002717+0000", 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002717+0000", 2023-10-09T09:49:26.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002716+0000", 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002721+0000", 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1059999999999999e-06 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002731+0000", 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0216e-05 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:26.002738+0000", 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8410000000000002e-06 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.046 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:49:26.047 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T09:49:26.047 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 7 2023-10-09T09:49:26.047 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:49:26.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:26.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:26.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:26.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:26.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:26.201 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:26.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:26.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:26.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:49:26.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[75425]: cluster 2023-10-09T09:49:24.151861+0000 mgr.y (mgr.34107) 1303 : cluster 0 pgmap v2156: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 846 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 738 op/s; 3566/40632 objects degraded (8.776%); 0 B/s, 129 objects/s recovering 2023-10-09T09:49:26.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[75425]: cluster 2023-10-09T09:49:25.030601+0000 osd.5 (osd.5) 848 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:26.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[75425]: cluster 2023-10-09T09:49:25.030715+0000 osd.4 (osd.4) 658 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:26.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[75425]: cluster 2023-10-09T09:49:25.034253+0000 mon.a (mon.0) 2398 : cluster 0 osdmap e1259: 8 total, 8 up, 6 in 2023-10-09T09:49:26.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[75425]: cluster 2023-10-09T09:49:25.041537+0000 osd.6 (osd.6) 544 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:26.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[75425]: audit 2023-10-09T09:49:25.399115+0000 mon.a (mon.0) 2399 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:26.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[75425]: cluster 2023-10-09T09:49:25.891538+0000 mon.a (mon.0) 2400 : cluster 3 Health check update: Degraded data redundancy: 3566/40632 objects degraded (8.776%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:26.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[78107]: cluster 2023-10-09T09:49:24.151861+0000 mgr.y (mgr.34107) 1303 : cluster 0 pgmap v2156: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 846 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 738 op/s; 3566/40632 objects degraded (8.776%); 0 B/s, 129 objects/s recovering 2023-10-09T09:49:26.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[78107]: cluster 2023-10-09T09:49:25.030601+0000 osd.5 (osd.5) 848 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:26.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[78107]: cluster 2023-10-09T09:49:25.030715+0000 osd.4 (osd.4) 658 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:26.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[78107]: cluster 2023-10-09T09:49:25.034253+0000 mon.a (mon.0) 2398 : cluster 0 osdmap e1259: 8 total, 8 up, 6 in 2023-10-09T09:49:26.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[78107]: cluster 2023-10-09T09:49:25.041537+0000 osd.6 (osd.6) 544 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:26.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[78107]: audit 2023-10-09T09:49:25.399115+0000 mon.a (mon.0) 2399 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:26.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:26 smithi163 bash[78107]: cluster 2023-10-09T09:49:25.891538+0000 mon.a (mon.0) 2400 : cluster 3 Health check update: Degraded data redundancy: 3566/40632 objects degraded (8.776%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:26.306 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:26.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:26 smithi181 bash[59345]: cluster 2023-10-09T09:49:24.151861+0000 mgr.y (mgr.34107) 1303 : cluster 0 pgmap v2156: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 846 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 738 op/s; 3566/40632 objects degraded (8.776%); 0 B/s, 129 objects/s recovering 2023-10-09T09:49:26.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:26 smithi181 bash[59345]: cluster 2023-10-09T09:49:25.030601+0000 osd.5 (osd.5) 848 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:26.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:26 smithi181 bash[59345]: cluster 2023-10-09T09:49:25.030715+0000 osd.4 (osd.4) 658 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:26.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:26 smithi181 bash[59345]: cluster 2023-10-09T09:49:25.034253+0000 mon.a (mon.0) 2398 : cluster 0 osdmap e1259: 8 total, 8 up, 6 in 2023-10-09T09:49:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:26 smithi181 bash[59345]: cluster 2023-10-09T09:49:25.041537+0000 osd.6 (osd.6) 544 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:26 smithi181 bash[59345]: audit 2023-10-09T09:49:25.399115+0000 mon.a (mon.0) 2399 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:26 smithi181 bash[59345]: cluster 2023-10-09T09:49:25.891538+0000 mon.a (mon.0) 2400 : cluster 3 Health check update: Degraded data redundancy: 3566/40632 objects degraded (8.776%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:26.333 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:26.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:26.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:26.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:49:26.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:26.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:26.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:26.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:26.577 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:26.603 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:26.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:26.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:26.709 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:26.737 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:26.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:26.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:26.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:26.896 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:26.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:26.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:27.001 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:27.030 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:27.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:27.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:27.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:27.142 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:27 smithi181 bash[59345]: cluster 2023-10-09T09:49:26.046310+0000 osd.7 (osd.7) 604 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:27.178 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:27.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:27.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:27.282 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:27.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T09:49:27.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:27 smithi163 bash[75425]: cluster 2023-10-09T09:49:26.046310+0000 osd.7 (osd.7) 604 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:27.291 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:27 smithi163 bash[78107]: cluster 2023-10-09T09:49:26.046310+0000 osd.7 (osd.7) 604 : cluster 3 failed to encode map e1259 with expected crc 2023-10-09T09:49:27.315 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:27.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:27.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:27.418 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:27.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:27.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:27.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:27.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:27.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:27.644 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:27.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:27.745 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:27.775 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:27.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:27.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:27.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:27.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:27.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:28.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:28.074 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:28.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:28.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:28.177 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:28.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:28.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:28.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:28.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:28 smithi181 bash[59345]: cluster 2023-10-09T09:49:26.152516+0000 mgr.y (mgr.34107) 1304 : cluster 0 pgmap v2158: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 790 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 751 op/s; 3566/37971 objects degraded (9.391%); 0 B/s, 69 objects/s recovering 2023-10-09T09:49:28.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:28.361 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:28.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:28.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:28.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:28 smithi163 bash[78107]: cluster 2023-10-09T09:49:26.152516+0000 mgr.y (mgr.34107) 1304 : cluster 0 pgmap v2158: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 790 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 751 op/s; 3566/37971 objects degraded (9.391%); 0 B/s, 69 objects/s recovering 2023-10-09T09:49:28.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:28 smithi163 bash[75425]: cluster 2023-10-09T09:49:26.152516+0000 mgr.y (mgr.34107) 1304 : cluster 0 pgmap v2158: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 790 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 751 op/s; 3566/37971 objects degraded (9.391%); 0 B/s, 69 objects/s recovering 2023-10-09T09:49:28.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:28.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:28.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:28.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:28.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:28.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:28.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:28.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:28.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:28.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:28.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:28.857 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:28.878 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:28.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:28.983 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:29.014 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:29.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:29.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:29.156 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:29.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:29.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:29.262 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:29.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:29 smithi181 bash[59345]: cluster 2023-10-09T09:49:28.153056+0000 mgr.y (mgr.34107) 1305 : cluster 0 pgmap v2159: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 689 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 1720/33105 objects degraded (5.196%); 0 B/s, 92 objects/s recovering 2023-10-09T09:49:29.287 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:29.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:29.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:29.391 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:29.400 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:29 smithi163 bash[78107]: cluster 2023-10-09T09:49:28.153056+0000 mgr.y (mgr.34107) 1305 : cluster 0 pgmap v2159: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 689 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 1720/33105 objects degraded (5.196%); 0 B/s, 92 objects/s recovering 2023-10-09T09:49:29.401 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:29 smithi163 bash[75425]: cluster 2023-10-09T09:49:28.153056+0000 mgr.y (mgr.34107) 1305 : cluster 0 pgmap v2159: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 689 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 1720/33105 objects degraded (5.196%); 0 B/s, 92 objects/s recovering 2023-10-09T09:49:29.425 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:29.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:29.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:29.532 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:29.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:29.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:29.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:29.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:29.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:29.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:29.803 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:29.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:29.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:29.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:29.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:29.962 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:29.990 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:29.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:29.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:30.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:30.120 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:30.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:30.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:30.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:30.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:30 smithi181 bash[59345]: audit 2023-10-09T09:49:29.227520+0000 mon.a (mon.0) 2401 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:30.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:30 smithi181 bash[59345]: audit 2023-10-09T09:49:29.237568+0000 mon.a (mon.0) 2402 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:30.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:30 smithi181 bash[59345]: audit 2023-10-09T09:49:29.834339+0000 mon.a (mon.0) 2403 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:30.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:30 smithi181 bash[59345]: audit 2023-10-09T09:49:29.835777+0000 mon.a (mon.0) 2404 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:30.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:30 smithi181 bash[59345]: audit 2023-10-09T09:49:29.850510+0000 mon.a (mon.0) 2405 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:30.255 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:30.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:30.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:30.360 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:30.368 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:30 smithi163 bash[78107]: audit 2023-10-09T09:49:29.227520+0000 mon.a (mon.0) 2401 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:30.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:30 smithi163 bash[78107]: audit 2023-10-09T09:49:29.237568+0000 mon.a (mon.0) 2402 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:30.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:30 smithi163 bash[78107]: audit 2023-10-09T09:49:29.834339+0000 mon.a (mon.0) 2403 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:30.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:30 smithi163 bash[78107]: audit 2023-10-09T09:49:29.835777+0000 mon.a (mon.0) 2404 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:30.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:30 smithi163 bash[78107]: audit 2023-10-09T09:49:29.850510+0000 mon.a (mon.0) 2405 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:30.369 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:30 smithi163 bash[75425]: audit 2023-10-09T09:49:29.227520+0000 mon.a (mon.0) 2401 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:30.369 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:30 smithi163 bash[75425]: audit 2023-10-09T09:49:29.237568+0000 mon.a (mon.0) 2402 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:30.369 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:30 smithi163 bash[75425]: audit 2023-10-09T09:49:29.834339+0000 mon.a (mon.0) 2403 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:30.370 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:30 smithi163 bash[75425]: audit 2023-10-09T09:49:29.835777+0000 mon.a (mon.0) 2404 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:30.370 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:30 smithi163 bash[75425]: audit 2023-10-09T09:49:29.850510+0000 mon.a (mon.0) 2405 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:30.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:30.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:30.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:30.487 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:30.534 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:30.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:30.640 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:30.671 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:30.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:30.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:30.777 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:30.822 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:30.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:30.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:30.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:30.998 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:31.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:31.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:31.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:31.130 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:31.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:31.234 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:31.261 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:31.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:31.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:31.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:31.394 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:31.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:31.500 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:31.528 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:31.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:31.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:31.666 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:31.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:31.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:31.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:31.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:31.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:31.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:31.836 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:31.839 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:31.839 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:31.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:31.941 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:31.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:31.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:32.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:32.062 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:31 smithi163 bash[75425]: cluster 2023-10-09T09:49:30.153625+0000 mgr.y (mgr.34107) 1306 : cluster 0 pgmap v2160: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 652 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 1720/31338 objects degraded (5.489%); 0 B/s, 82 objects/s recovering 2023-10-09T09:49:32.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:31 smithi163 bash[75425]: cluster 2023-10-09T09:49:30.892843+0000 mon.a (mon.0) 2406 : cluster 3 Health check update: Degraded data redundancy: 1720/31338 objects degraded (5.489%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:32.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:31 smithi163 bash[78107]: cluster 2023-10-09T09:49:30.153625+0000 mgr.y (mgr.34107) 1306 : cluster 0 pgmap v2160: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 652 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 1720/31338 objects degraded (5.489%); 0 B/s, 82 objects/s recovering 2023-10-09T09:49:32.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:31 smithi163 bash[78107]: cluster 2023-10-09T09:49:30.892843+0000 mon.a (mon.0) 2406 : cluster 3 Health check update: Degraded data redundancy: 1720/31338 objects degraded (5.489%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:32.079 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:32.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:32.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:32.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:32.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:31 smithi181 bash[59345]: cluster 2023-10-09T09:49:30.153625+0000 mgr.y (mgr.34107) 1306 : cluster 0 pgmap v2160: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 652 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 1720/31338 objects degraded (5.489%); 0 B/s, 82 objects/s recovering 2023-10-09T09:49:32.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:31 smithi181 bash[59345]: cluster 2023-10-09T09:49:30.892843+0000 mon.a (mon.0) 2406 : cluster 3 Health check update: Degraded data redundancy: 1720/31338 objects degraded (5.489%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:32.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:32.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:32.313 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:32.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:32.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:32.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:32.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:32.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:32.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:32.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:32.590 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:32.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:32.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:32.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:32.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:32.759 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:32.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:32.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:32.897 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:32.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:33.002 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:33.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:33.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:33.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:33.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:33.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:33.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:33.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:33.260 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:33.301 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:33.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:33.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:33.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:33.437 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:33.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:33.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:33.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:33.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:33.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:33.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:33.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:33.708 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:33.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:33.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:33.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:49:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:49:33] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-09T09:49:33.811 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:33.836 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:33.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:33.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:33.975 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:33.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:34.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:33 smithi181 bash[59345]: cluster 2023-10-09T09:49:32.155557+0000 mgr.y (mgr.34107) 1307 : cluster 0 pgmap v2161: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 449 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 779 op/s; 1720/21597 objects degraded (7.964%); 0 B/s, 64 objects/s recovering 2023-10-09T09:49:34.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:34.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:34.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:34.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:34.213 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:34.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:33 smithi163 bash[78107]: cluster 2023-10-09T09:49:32.155557+0000 mgr.y (mgr.34107) 1307 : cluster 0 pgmap v2161: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 449 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 779 op/s; 1720/21597 objects degraded (7.964%); 0 B/s, 64 objects/s recovering 2023-10-09T09:49:34.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:33 smithi163 bash[75425]: cluster 2023-10-09T09:49:32.155557+0000 mgr.y (mgr.34107) 1307 : cluster 0 pgmap v2161: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 449 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 779 op/s; 1720/21597 objects degraded (7.964%); 0 B/s, 64 objects/s recovering 2023-10-09T09:49:34.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:34.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:34.343 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:34.366 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:34.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:34.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:34.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:34.523 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:34.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:34.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:34.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:34.680 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:34.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:34.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:34.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:34.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:34.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:34.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:34.906 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:34.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:34.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:34.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:35.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:35.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:35.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:35.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:35.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:35.209 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:35.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:35.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:35.312 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:35.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:35.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:35.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:35.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:35.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:35.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:35.587 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:35.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:35.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:35.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:35.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:35.754 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:35.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:35.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:35.855 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:35 smithi163 bash[75425]: cluster 2023-10-09T09:49:34.156153+0000 mgr.y (mgr.34107) 1308 : cluster 0 pgmap v2162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 406 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 1720/19548 objects degraded (8.799%); 0 B/s, 52 objects/s recovering 2023-10-09T09:49:35.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:35.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:35.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:35.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:35.992 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:36.018 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:36.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:36.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:36.124 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:36.133 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:35 smithi181 bash[59345]: cluster 2023-10-09T09:49:34.156153+0000 mgr.y (mgr.34107) 1308 : cluster 0 pgmap v2162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 406 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 1720/19548 objects degraded (8.799%); 0 B/s, 52 objects/s recovering 2023-10-09T09:49:36.160 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:36.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:36.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:36.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:35 smithi163 bash[78107]: cluster 2023-10-09T09:49:34.156153+0000 mgr.y (mgr.34107) 1308 : cluster 0 pgmap v2162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 406 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 1720/19548 objects degraded (8.799%); 0 B/s, 52 objects/s recovering 2023-10-09T09:49:36.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:36.294 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:36.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:36.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:36.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:36.438 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:36.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:36.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:36.584 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:36.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:36.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:36.689 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:36.721 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:36.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:36.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:36.824 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:36.852 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:36.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:36.961 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:36.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:36 smithi163 bash[78107]: cluster 2023-10-09T09:49:35.894044+0000 mon.a (mon.0) 2407 : cluster 3 Health check update: Degraded data redundancy: 1720/19548 objects degraded (8.799%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:36.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:36 smithi163 bash[75425]: cluster 2023-10-09T09:49:35.894044+0000 mon.a (mon.0) 2407 : cluster 3 Health check update: Degraded data redundancy: 1720/19548 objects degraded (8.799%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:36.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:36.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:36.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:37.098 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:37.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:37.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:37.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:37.236 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:37.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:36 smithi181 bash[59345]: cluster 2023-10-09T09:49:35.894044+0000 mon.a (mon.0) 2407 : cluster 3 Health check update: Degraded data redundancy: 1720/19548 objects degraded (8.799%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:37.265 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:37.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:37.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:37.409 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:37.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:37.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:37.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:37.540 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:37.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:37.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:37.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:37.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:37.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:37.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:37.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:37.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:37.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:37.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:37.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:37.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:38.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:37 smithi163 bash[75425]: cluster 2023-10-09T09:49:36.156882+0000 mgr.y (mgr.34107) 1309 : cluster 0 pgmap v2163: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 333 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 737 op/s; 1720/16014 objects degraded (10.741%); 0 B/s, 47 objects/s recovering 2023-10-09T09:49:38.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:38 smithi163 bash[78107]: cluster 2023-10-09T09:49:36.156882+0000 mgr.y (mgr.34107) 1309 : cluster 0 pgmap v2163: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 333 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 737 op/s; 1720/16014 objects degraded (10.741%); 0 B/s, 47 objects/s recovering 2023-10-09T09:49:38.038 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:38.065 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:38.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:38.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:38.197 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:38.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:38.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:38.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:38.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:37 smithi181 bash[59345]: cluster 2023-10-09T09:49:36.156882+0000 mgr.y (mgr.34107) 1309 : cluster 0 pgmap v2163: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 333 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 737 op/s; 1720/16014 objects degraded (10.741%); 0 B/s, 47 objects/s recovering 2023-10-09T09:49:38.330 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:38.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:38.437 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:38.474 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:38.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:38.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:38.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:38.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:38.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:38.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:38.715 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:38.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:38.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:38.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:38.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:38.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:38.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:38.980 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:39.018 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:39.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:39.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:39.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:39.144 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:39.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:39.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:39.248 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:39.273 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:39.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:39.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:39.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:39.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:39.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:39.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:39.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:39.538 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:39.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:39.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:39.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:39.670 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:39.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:39.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:39.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:39.803 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:39.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:39.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:39.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:39.947 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:39.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:40.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:40.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:39 smithi163 bash[78107]: cluster 2023-10-09T09:49:38.157562+0000 mgr.y (mgr.34107) 1310 : cluster 0 pgmap v2164: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 212 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 770 op/s; 1720/10239 objects degraded (16.799%); 0 B/s, 73 objects/s recovering 2023-10-09T09:49:40.058 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:39 smithi163 bash[75425]: cluster 2023-10-09T09:49:38.157562+0000 mgr.y (mgr.34107) 1310 : cluster 0 pgmap v2164: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 212 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 770 op/s; 1720/10239 objects degraded (16.799%); 0 B/s, 73 objects/s recovering 2023-10-09T09:49:40.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:40.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:40.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:40.182 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:40.192 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:39 smithi181 bash[59345]: cluster 2023-10-09T09:49:38.157562+0000 mgr.y (mgr.34107) 1310 : cluster 0 pgmap v2164: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 212 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 770 op/s; 1720/10239 objects degraded (16.799%); 0 B/s, 73 objects/s recovering 2023-10-09T09:49:40.215 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:40.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:40.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:40.274 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 109450 objects 2023-10-09T09:49:40.274 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :144.71 2023-10-09T09:49:40.320 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:40.341 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:40.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:40.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:40.444 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:40.473 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:40.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:40.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:40.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:40.608 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:40.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:40.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:40.712 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:40.737 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:40.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:40.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:40.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:40.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:40.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:40.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:40.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:40.990 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:40.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:40.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:41.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:40 smithi163 bash[78107]: audit 2023-10-09T09:49:40.399235+0000 mon.a (mon.0) 2408 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:41.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:40 smithi163 bash[78107]: cluster 2023-10-09T09:49:40.895397+0000 mon.a (mon.0) 2409 : cluster 3 Health check update: Degraded data redundancy: 1720/10239 objects degraded (16.799%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:41.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:40 smithi163 bash[75425]: audit 2023-10-09T09:49:40.399235+0000 mon.a (mon.0) 2408 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:41.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:40 smithi163 bash[75425]: cluster 2023-10-09T09:49:40.895397+0000 mon.a (mon.0) 2409 : cluster 3 Health check update: Degraded data redundancy: 1720/10239 objects degraded (16.799%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:41.093 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:41.122 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:41.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:41.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:41.225 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:41.236 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:40 smithi181 bash[59345]: audit 2023-10-09T09:49:40.399235+0000 mon.a (mon.0) 2408 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:41.236 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:40 smithi181 bash[59345]: cluster 2023-10-09T09:49:40.895397+0000 mon.a (mon.0) 2409 : cluster 3 Health check update: Degraded data redundancy: 1720/10239 objects degraded (16.799%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:41.254 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:41.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:41.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:41.360 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:41.381 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:41.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:41.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:41.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:41.509 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:41.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:41.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:41.613 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:41.634 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:41.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:41.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:41.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:41.763 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:41.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:41.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:41.779 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T09:49:41.779 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T09:49:41.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:41.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:41.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:41.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:41.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:42.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:42.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:42.127 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:42.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:41 smithi163 bash[78107]: cluster 2023-10-09T09:49:40.158167+0000 mgr.y (mgr.34107) 1311 : cluster 0 pgmap v2165: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 167 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 695 op/s; 1720/8049 objects degraded (21.369%); 0 B/s, 53 objects/s recovering 2023-10-09T09:49:42.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:41 smithi163 bash[75425]: cluster 2023-10-09T09:49:40.158167+0000 mgr.y (mgr.34107) 1311 : cluster 0 pgmap v2165: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 167 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 695 op/s; 1720/8049 objects degraded (21.369%); 0 B/s, 53 objects/s recovering 2023-10-09T09:49:42.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:42.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:42.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:42.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:42.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:41 smithi181 bash[59345]: cluster 2023-10-09T09:49:40.158167+0000 mgr.y (mgr.34107) 1311 : cluster 0 pgmap v2165: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 167 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 695 op/s; 1720/8049 objects degraded (21.369%); 0 B/s, 53 objects/s recovering 2023-10-09T09:49:42.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:42.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:42.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:42.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:42.412 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:42.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:42.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:42.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:42.543 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:42.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:42.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:42.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:42.676 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:42.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:42.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:42.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:42.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:42.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:42.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:42.903 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:42.925 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:42.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:42.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:43.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:43.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:43.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:43.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:43.162 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:43.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:43.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:43.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:43.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:43.314 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:43.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:43.417 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:43.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:43.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:43.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:43.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:43.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:43.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:43.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:43.585 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:49:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:49:43] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.33.4" 2023-10-09T09:49:43.674 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:43.699 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:43.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:43.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:43.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:43.823 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:43.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:43.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:43.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:43.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:43.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:44.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:44.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:44.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:44.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:44.203 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:43 smithi181 bash[59345]: cluster 2023-10-09T09:49:42.159379+0000 mgr.y (mgr.34107) 1312 : cluster 0 pgmap v2166: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 16 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 848 op/s; 1720/792 objects degraded (217.172%); 0 B/s, 107 objects/s recovering 2023-10-09T09:49:44.203 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:43 smithi181 bash[59345]: cluster 2023-10-09T09:49:42.918471+0000 osd.5 (osd.5) 849 : cluster 3 failed to encode map e1260 with expected crc 2023-10-09T09:49:44.203 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:43 smithi181 bash[59345]: cluster 2023-10-09T09:49:42.921188+0000 mon.a (mon.0) 2410 : cluster 0 osdmap e1260: 8 total, 8 up, 6 in 2023-10-09T09:49:44.203 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:43 smithi181 bash[59345]: cluster 2023-10-09T09:49:42.922055+0000 osd.6 (osd.6) 545 : cluster 3 failed to encode map e1260 with expected crc 2023-10-09T09:49:44.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:43 smithi163 bash[78107]: cluster 2023-10-09T09:49:42.159379+0000 mgr.y (mgr.34107) 1312 : cluster 0 pgmap v2166: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 16 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 848 op/s; 1720/792 objects degraded (217.172%); 0 B/s, 107 objects/s recovering 2023-10-09T09:49:44.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:43 smithi163 bash[78107]: cluster 2023-10-09T09:49:42.918471+0000 osd.5 (osd.5) 849 : cluster 3 failed to encode map e1260 with expected crc 2023-10-09T09:49:44.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:43 smithi163 bash[78107]: cluster 2023-10-09T09:49:42.921188+0000 mon.a (mon.0) 2410 : cluster 0 osdmap e1260: 8 total, 8 up, 6 in 2023-10-09T09:49:44.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:43 smithi163 bash[78107]: cluster 2023-10-09T09:49:42.922055+0000 osd.6 (osd.6) 545 : cluster 3 failed to encode map e1260 with expected crc 2023-10-09T09:49:44.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:43 smithi163 bash[75425]: cluster 2023-10-09T09:49:42.159379+0000 mgr.y (mgr.34107) 1312 : cluster 0 pgmap v2166: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 16 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 848 op/s; 1720/792 objects degraded (217.172%); 0 B/s, 107 objects/s recovering 2023-10-09T09:49:44.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:43 smithi163 bash[75425]: cluster 2023-10-09T09:49:42.918471+0000 osd.5 (osd.5) 849 : cluster 3 failed to encode map e1260 with expected crc 2023-10-09T09:49:44.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:43 smithi163 bash[75425]: cluster 2023-10-09T09:49:42.921188+0000 mon.a (mon.0) 2410 : cluster 0 osdmap e1260: 8 total, 8 up, 6 in 2023-10-09T09:49:44.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:43 smithi163 bash[75425]: cluster 2023-10-09T09:49:42.922055+0000 osd.6 (osd.6) 545 : cluster 3 failed to encode map e1260 with expected crc 2023-10-09T09:49:44.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:44.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:44.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:44.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:44.362 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:44.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:44.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:44.465 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:44.488 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:44.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:44.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:44.592 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:44.628 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:44.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:44.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:44.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:44.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:44.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:44.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:44.868 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:44.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:44.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:44.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:44.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:44 smithi163 bash[75425]: cluster 2023-10-09T09:49:43.931309+0000 osd.5 (osd.5) 850 : cluster 3 failed to encode map e1261 with expected crc 2023-10-09T09:49:44.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:44 smithi163 bash[75425]: cluster 2023-10-09T09:49:43.932640+0000 mon.a (mon.0) 2411 : cluster 0 osdmap e1261: 8 total, 8 up, 6 in 2023-10-09T09:49:44.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:44 smithi163 bash[75425]: cluster 2023-10-09T09:49:43.933273+0000 osd.6 (osd.6) 546 : cluster 3 failed to encode map e1261 with expected crc 2023-10-09T09:49:44.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:44 smithi163 bash[75425]: audit 2023-10-09T09:49:44.094460+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.163:0/3939752790' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:49:44.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:44 smithi163 bash[75425]: audit 2023-10-09T09:49:44.094973+0000 mon.a (mon.0) 2412 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:49:44.946 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T09:49:45.003 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:45.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:45.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:45.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:45.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:45.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:45.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:45.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:44 smithi163 bash[78107]: cluster 2023-10-09T09:49:43.931309+0000 osd.5 (osd.5) 850 : cluster 3 failed to encode map e1261 with expected crc 2023-10-09T09:49:45.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:44 smithi163 bash[78107]: cluster 2023-10-09T09:49:43.932640+0000 mon.a (mon.0) 2411 : cluster 0 osdmap e1261: 8 total, 8 up, 6 in 2023-10-09T09:49:45.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:44 smithi163 bash[78107]: cluster 2023-10-09T09:49:43.933273+0000 osd.6 (osd.6) 546 : cluster 3 failed to encode map e1261 with expected crc 2023-10-09T09:49:45.241 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:44 smithi163 bash[78107]: audit 2023-10-09T09:49:44.094460+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.163:0/3939752790' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:49:45.241 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:44 smithi163 bash[78107]: audit 2023-10-09T09:49:44.094973+0000 mon.a (mon.0) 2412 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:49:45.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:45.283 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:45.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:45.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:44 smithi181 bash[59345]: cluster 2023-10-09T09:49:43.931309+0000 osd.5 (osd.5) 850 : cluster 3 failed to encode map e1261 with expected crc 2023-10-09T09:49:45.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:44 smithi181 bash[59345]: cluster 2023-10-09T09:49:43.932640+0000 mon.a (mon.0) 2411 : cluster 0 osdmap e1261: 8 total, 8 up, 6 in 2023-10-09T09:49:45.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:44 smithi181 bash[59345]: cluster 2023-10-09T09:49:43.933273+0000 osd.6 (osd.6) 546 : cluster 3 failed to encode map e1261 with expected crc 2023-10-09T09:49:45.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:44 smithi181 bash[59345]: audit 2023-10-09T09:49:44.094460+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.163:0/3939752790' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:49:45.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:44 smithi181 bash[59345]: audit 2023-10-09T09:49:44.094973+0000 mon.a (mon.0) 2412 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:49:45.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:45.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:45.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:45.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:45.511 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:45.547 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:45.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:45.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:45.651 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:45.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:45.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:45.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:45.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:45.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:45.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:45.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:45.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:45.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:45.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:46.032 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:46.066 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:46.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:46.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:46.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:46.206 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:46.207 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:45 smithi181 bash[59345]: cluster 2023-10-09T09:49:44.061086+0000 osd.7 (osd.7) 605 : cluster 3 failed to encode map e1261 with expected crc 2023-10-09T09:49:46.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:45 smithi181 bash[59345]: cluster 2023-10-09T09:49:44.160072+0000 mgr.y (mgr.34107) 1313 : cluster 0 pgmap v2169: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 0 B/s, 158 objects/s recovering 2023-10-09T09:49:46.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:45 smithi181 bash[59345]: cluster 2023-10-09T09:49:44.932334+0000 osd.5 (osd.5) 851 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:46.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:45 smithi181 bash[59345]: audit 2023-10-09T09:49:44.934643+0000 mon.a (mon.0) 2413 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:49:46.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:45 smithi181 bash[59345]: cluster 2023-10-09T09:49:44.934740+0000 mon.a (mon.0) 2414 : cluster 0 osdmap e1262: 8 total, 8 up, 6 in 2023-10-09T09:49:46.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:45 smithi181 bash[59345]: cluster 2023-10-09T09:49:45.050718+0000 osd.6 (osd.6) 547 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:46.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:45 smithi181 bash[59345]: cluster 2023-10-09T09:49:45.050809+0000 osd.4 (osd.4) 659 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:46.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:45 smithi181 bash[59345]: cluster 2023-10-09T09:49:45.896439+0000 mon.a (mon.0) 2415 : cluster 3 Health check update: Degraded data redundancy: 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:46.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:46.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[78107]: cluster 2023-10-09T09:49:44.061086+0000 osd.7 (osd.7) 605 : cluster 3 failed to encode map e1261 with expected crc 2023-10-09T09:49:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[78107]: cluster 2023-10-09T09:49:44.160072+0000 mgr.y (mgr.34107) 1313 : cluster 0 pgmap v2169: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 0 B/s, 158 objects/s recovering 2023-10-09T09:49:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[78107]: cluster 2023-10-09T09:49:44.932334+0000 osd.5 (osd.5) 851 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[78107]: audit 2023-10-09T09:49:44.934643+0000 mon.a (mon.0) 2413 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:49:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[78107]: cluster 2023-10-09T09:49:44.934740+0000 mon.a (mon.0) 2414 : cluster 0 osdmap e1262: 8 total, 8 up, 6 in 2023-10-09T09:49:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[78107]: cluster 2023-10-09T09:49:45.050718+0000 osd.6 (osd.6) 547 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[78107]: cluster 2023-10-09T09:49:45.050809+0000 osd.4 (osd.4) 659 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:46.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[78107]: cluster 2023-10-09T09:49:45.896439+0000 mon.a (mon.0) 2415 : cluster 3 Health check update: Degraded data redundancy: 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:46.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[75425]: cluster 2023-10-09T09:49:44.061086+0000 osd.7 (osd.7) 605 : cluster 3 failed to encode map e1261 with expected crc 2023-10-09T09:49:46.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[75425]: cluster 2023-10-09T09:49:44.160072+0000 mgr.y (mgr.34107) 1313 : cluster 0 pgmap v2169: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 0 B/s, 158 objects/s recovering 2023-10-09T09:49:46.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[75425]: cluster 2023-10-09T09:49:44.932334+0000 osd.5 (osd.5) 851 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:46.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[75425]: audit 2023-10-09T09:49:44.934643+0000 mon.a (mon.0) 2413 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:49:46.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[75425]: cluster 2023-10-09T09:49:44.934740+0000 mon.a (mon.0) 2414 : cluster 0 osdmap e1262: 8 total, 8 up, 6 in 2023-10-09T09:49:46.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[75425]: cluster 2023-10-09T09:49:45.050718+0000 osd.6 (osd.6) 547 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:46.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[75425]: cluster 2023-10-09T09:49:45.050809+0000 osd.4 (osd.4) 659 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:46.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:45 smithi163 bash[75425]: cluster 2023-10-09T09:49:45.896439+0000 mon.a (mon.0) 2415 : cluster 3 Health check update: Degraded data redundancy: 2 pgs undersized (PG_DEGRADED) 2023-10-09T09:49:46.309 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:46.333 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:46.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:46.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:46.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:46.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:46.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:46.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:46.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:46.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:46.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:46.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:46.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:46.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:46.833 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:46.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:46.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:46.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:46.961 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:46.988 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:46.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:46.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:47.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:46 smithi181 bash[59345]: cluster 2023-10-09T09:49:45.050903+0000 osd.7 (osd.7) 606 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:47.091 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:47.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:47.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:47.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:47.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:46 smithi163 bash[75425]: cluster 2023-10-09T09:49:45.050903+0000 osd.7 (osd.7) 606 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:47.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:46 smithi163 bash[78107]: cluster 2023-10-09T09:49:45.050903+0000 osd.7 (osd.7) 606 : cluster 3 failed to encode map e1262 with expected crc 2023-10-09T09:49:47.227 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:47.258 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:47.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:47.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:47.362 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:47.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:47.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:47.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:47.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:47.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:47.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:47.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:47.655 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:47.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:47.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:47.759 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:47.784 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:47.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:47.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:47.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:47.920 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:47.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:47.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:47.963 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:47 smithi181 bash[59345]: cluster 2023-10-09T09:49:46.160748+0000 mgr.y (mgr.34107) 1314 : cluster 0 pgmap v2171: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 0 B/s, 151 objects/s recovering 2023-10-09T09:49:48.024 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:48.055 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:48.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:48.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:48.167 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:47 smithi163 bash[78107]: cluster 2023-10-09T09:49:46.160748+0000 mgr.y (mgr.34107) 1314 : cluster 0 pgmap v2171: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 0 B/s, 151 objects/s recovering 2023-10-09T09:49:48.167 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:47 smithi163 bash[75425]: cluster 2023-10-09T09:49:46.160748+0000 mgr.y (mgr.34107) 1314 : cluster 0 pgmap v2171: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 0 B/s, 151 objects/s recovering 2023-10-09T09:49:48.185 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:48.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:48.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:48.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:48.324 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:48.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:48.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:48.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:48.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:48.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:48.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:48.556 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:48.586 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:48.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:48.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:48.690 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:48.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:48.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:48.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:48.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:48.855 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:48.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:48.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:48.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:48.990 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:48.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:48.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:49.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:49.133 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:48 smithi163 bash[75425]: cluster 2023-10-09T09:49:47.954889+0000 osd.4 (osd.4) 660 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:49.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:48 smithi163 bash[75425]: cluster 2023-10-09T09:49:47.955117+0000 osd.5 (osd.5) 852 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:49.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:48 smithi163 bash[75425]: cluster 2023-10-09T09:49:47.959010+0000 mon.a (mon.0) 2416 : cluster 0 osdmap e1263: 8 total, 8 up, 6 in 2023-10-09T09:49:49.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:48 smithi163 bash[75425]: cluster 2023-10-09T09:49:48.560410+0000 osd.7 (osd.7) 607 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:49.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:48 smithi163 bash[78107]: cluster 2023-10-09T09:49:47.954889+0000 osd.4 (osd.4) 660 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:49.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:48 smithi163 bash[78107]: cluster 2023-10-09T09:49:47.955117+0000 osd.5 (osd.5) 852 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:49.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:48 smithi163 bash[78107]: cluster 2023-10-09T09:49:47.959010+0000 mon.a (mon.0) 2416 : cluster 0 osdmap e1263: 8 total, 8 up, 6 in 2023-10-09T09:49:49.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:48 smithi163 bash[78107]: cluster 2023-10-09T09:49:48.560410+0000 osd.7 (osd.7) 607 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:49.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:49.181 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:49.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:49.282 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:49.295 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:48 smithi181 bash[59345]: cluster 2023-10-09T09:49:47.954889+0000 osd.4 (osd.4) 660 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:49.295 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:48 smithi181 bash[59345]: cluster 2023-10-09T09:49:47.955117+0000 osd.5 (osd.5) 852 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:49.295 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:48 smithi181 bash[59345]: cluster 2023-10-09T09:49:47.959010+0000 mon.a (mon.0) 2416 : cluster 0 osdmap e1263: 8 total, 8 up, 6 in 2023-10-09T09:49:49.295 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:48 smithi181 bash[59345]: cluster 2023-10-09T09:49:48.560410+0000 osd.7 (osd.7) 607 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:49.315 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:49.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:49.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:49.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:49.443 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:49.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:49.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:49.546 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:49.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:49.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:49.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:49.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:49.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:49.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:49.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:49.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T09:49:49.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:49.846 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:49.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:49.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:49.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:49.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:49.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:50.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:50.112 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:50.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:50.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:50.215 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:50.226 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:49 smithi181 bash[59345]: cluster 2023-10-09T09:49:48.161462+0000 mgr.y (mgr.34107) 1315 : cluster 0 pgmap v2173: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 0 B/s, 110 objects/s recovering 2023-10-09T09:49:50.226 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:49 smithi181 bash[59345]: cluster 2023-10-09T09:49:48.560518+0000 osd.6 (osd.6) 548 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:50.226 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:49 smithi181 bash[59345]: cluster 2023-10-09T09:49:48.969072+0000 osd.5 (osd.5) 853 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:50.227 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:49 smithi181 bash[59345]: cluster 2023-10-09T09:49:48.972914+0000 mon.a (mon.0) 2417 : cluster 0 osdmap e1264: 8 total, 8 up, 6 in 2023-10-09T09:49:50.227 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:49 smithi181 bash[59345]: cluster 2023-10-09T09:49:48.977095+0000 osd.4 (osd.4) 661 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:50.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:50.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:50.349 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:50.350 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:49:50.350 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:49:50.350 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T09:49:50.350 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T09:49:50.350 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:49:50.385 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:50.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:50.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:50.393 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:49 smithi163 bash[78107]: cluster 2023-10-09T09:49:48.161462+0000 mgr.y (mgr.34107) 1315 : cluster 0 pgmap v2173: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 0 B/s, 110 objects/s recovering 2023-10-09T09:49:50.393 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:49 smithi163 bash[78107]: cluster 2023-10-09T09:49:48.560518+0000 osd.6 (osd.6) 548 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:50.394 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:49 smithi163 bash[78107]: cluster 2023-10-09T09:49:48.969072+0000 osd.5 (osd.5) 853 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:50.394 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:49 smithi163 bash[78107]: cluster 2023-10-09T09:49:48.972914+0000 mon.a (mon.0) 2417 : cluster 0 osdmap e1264: 8 total, 8 up, 6 in 2023-10-09T09:49:50.394 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:49 smithi163 bash[78107]: cluster 2023-10-09T09:49:48.977095+0000 osd.4 (osd.4) 661 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:50.394 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:49 smithi163 bash[75425]: cluster 2023-10-09T09:49:48.161462+0000 mgr.y (mgr.34107) 1315 : cluster 0 pgmap v2173: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 0 B/s, 110 objects/s recovering 2023-10-09T09:49:50.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:49 smithi163 bash[75425]: cluster 2023-10-09T09:49:48.560518+0000 osd.6 (osd.6) 548 : cluster 3 failed to encode map e1263 with expected crc 2023-10-09T09:49:50.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:49 smithi163 bash[75425]: cluster 2023-10-09T09:49:48.969072+0000 osd.5 (osd.5) 853 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:50.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:49 smithi163 bash[75425]: cluster 2023-10-09T09:49:48.972914+0000 mon.a (mon.0) 2417 : cluster 0 osdmap e1264: 8 total, 8 up, 6 in 2023-10-09T09:49:50.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:49 smithi163 bash[75425]: cluster 2023-10-09T09:49:48.977095+0000 osd.4 (osd.4) 661 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:50.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:50.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:50.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:50.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:50.618 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:50.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:50.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:50.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:50.648 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:49:50.669 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:49:50.688 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:49:50.717 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:49:50.739 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:49:50.743 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:50.761 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:49:50.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:50.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:50.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:50.782 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:49:50.805 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:49:50.806 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:49:50.806 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:49:50.806 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:49:50.807 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:49:50.807 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:49:50.807 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:49:50.807 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:49:50.807 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:49:50.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:50.905 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:50.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:50.987 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:49:51.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:51.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:51.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:51.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:50 smithi181 bash[59345]: cluster 2023-10-09T09:49:49.750980+0000 osd.6 (osd.6) 549 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:50 smithi181 bash[59345]: cluster 2023-10-09T09:49:50.251596+0000 osd.7 (osd.7) 608 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:50 smithi181 bash[59345]: audit 2023-10-09T09:49:50.625109+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.163:0/3957286871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:49:51.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:50 smithi181 bash[59345]: audit 2023-10-09T09:49:50.850074+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.163:0/2756242795' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:49:51.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:50 smithi181 bash[59345]: audit 2023-10-09T09:49:50.850632+0000 mon.a (mon.0) 2418 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:49:51.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:50 smithi181 bash[59345]: cluster 2023-10-09T09:49:50.897676+0000 mon.a (mon.0) 2419 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:49:51.129 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:51.163 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:51.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:51.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:51.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:51.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[75425]: cluster 2023-10-09T09:49:49.750980+0000 osd.6 (osd.6) 549 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:51.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[75425]: cluster 2023-10-09T09:49:50.251596+0000 osd.7 (osd.7) 608 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:51.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[75425]: audit 2023-10-09T09:49:50.625109+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.163:0/3957286871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:49:51.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[75425]: audit 2023-10-09T09:49:50.850074+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.163:0/2756242795' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:49:51.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[75425]: audit 2023-10-09T09:49:50.850632+0000 mon.a (mon.0) 2418 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:49:51.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[75425]: cluster 2023-10-09T09:49:50.897676+0000 mon.a (mon.0) 2419 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:49:51.281 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[78107]: cluster 2023-10-09T09:49:49.750980+0000 osd.6 (osd.6) 549 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:51.281 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[78107]: cluster 2023-10-09T09:49:50.251596+0000 osd.7 (osd.7) 608 : cluster 3 failed to encode map e1264 with expected crc 2023-10-09T09:49:51.281 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[78107]: audit 2023-10-09T09:49:50.625109+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.163:0/3957286871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:49:51.281 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[78107]: audit 2023-10-09T09:49:50.850074+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.163:0/2756242795' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:49:51.281 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[78107]: audit 2023-10-09T09:49:50.850632+0000 mon.a (mon.0) 2418 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:49:51.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:50 smithi163 bash[78107]: cluster 2023-10-09T09:49:50.897676+0000 mon.a (mon.0) 2419 : cluster 3 Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T09:49:51.293 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:51.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:51.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:51.424 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:51.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:51.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:51.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:51.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:51.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:51.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:51.666 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:51.690 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:51.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:51.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:51.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:51.825 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:51.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:51.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:51.957 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:51.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:51.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:51.963 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T09:49:52.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:52.094 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:52.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:52.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:52.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:52.211 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:51 smithi181 bash[59345]: cluster 2023-10-09T09:49:50.162232+0000 mgr.y (mgr.34107) 1316 : cluster 0 pgmap v2175: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 52 objects/s recovering 2023-10-09T09:49:52.211 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:51 smithi181 bash[59345]: cluster 2023-10-09T09:49:50.969895+0000 mon.a (mon.0) 2420 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:49:52.211 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:51 smithi181 bash[59345]: cluster 2023-10-09T09:49:50.979126+0000 osd.5 (osd.5) 854 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:52.211 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:51 smithi181 bash[59345]: audit 2023-10-09T09:49:50.982686+0000 mon.a (mon.0) 2421 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:49:52.211 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:51 smithi181 bash[59345]: cluster 2023-10-09T09:49:50.982777+0000 mon.a (mon.0) 2422 : cluster 0 osdmap e1265: 8 total, 8 up, 6 in 2023-10-09T09:49:52.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[75425]: cluster 2023-10-09T09:49:50.162232+0000 mgr.y (mgr.34107) 1316 : cluster 0 pgmap v2175: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 52 objects/s recovering 2023-10-09T09:49:52.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[75425]: cluster 2023-10-09T09:49:50.969895+0000 mon.a (mon.0) 2420 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:49:52.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[75425]: cluster 2023-10-09T09:49:50.979126+0000 osd.5 (osd.5) 854 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:52.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[75425]: audit 2023-10-09T09:49:50.982686+0000 mon.a (mon.0) 2421 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:49:52.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[75425]: cluster 2023-10-09T09:49:50.982777+0000 mon.a (mon.0) 2422 : cluster 0 osdmap e1265: 8 total, 8 up, 6 in 2023-10-09T09:49:52.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[78107]: cluster 2023-10-09T09:49:50.162232+0000 mgr.y (mgr.34107) 1316 : cluster 0 pgmap v2175: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 52 objects/s recovering 2023-10-09T09:49:52.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[78107]: cluster 2023-10-09T09:49:50.969895+0000 mon.a (mon.0) 2420 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:49:52.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[78107]: cluster 2023-10-09T09:49:50.979126+0000 osd.5 (osd.5) 854 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:52.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[78107]: audit 2023-10-09T09:49:50.982686+0000 mon.a (mon.0) 2421 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:49:52.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[78107]: cluster 2023-10-09T09:49:50.982777+0000 mon.a (mon.0) 2422 : cluster 0 osdmap e1265: 8 total, 8 up, 6 in 2023-10-09T09:49:52.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:52.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:52.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:52.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:52.371 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:52.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:52.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:52.502 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:52.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:52.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:52.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:52.630 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:52.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:52.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:52.733 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:52.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:52.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:52.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:52.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:52.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:52.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:52.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:52.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:53.030 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:53.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:53.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:53.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:53.144 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[78107]: cluster 2023-10-09T09:49:51.572741+0000 osd.6 (osd.6) 550 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:53.144 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[78107]: cluster 2023-10-09T09:49:52.427160+0000 osd.7 (osd.7) 609 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:53.145 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[78107]: cluster 2023-10-09T09:49:52.429628+0000 osd.7 (osd.7) 610 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:53.145 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[75425]: cluster 2023-10-09T09:49:51.572741+0000 osd.6 (osd.6) 550 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:53.145 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[75425]: cluster 2023-10-09T09:49:52.427160+0000 osd.7 (osd.7) 609 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:53.145 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:52 smithi163 bash[75425]: cluster 2023-10-09T09:49:52.429628+0000 osd.7 (osd.7) 610 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:53.164 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:53.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:53.267 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:53.281 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:52 smithi181 bash[59345]: cluster 2023-10-09T09:49:51.572741+0000 osd.6 (osd.6) 550 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:53.281 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:52 smithi181 bash[59345]: cluster 2023-10-09T09:49:52.427160+0000 osd.7 (osd.7) 609 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:53.282 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:52 smithi181 bash[59345]: cluster 2023-10-09T09:49:52.429628+0000 osd.7 (osd.7) 610 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:53.300 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:53.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:53.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:53.397 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:49:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:49:53] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.33.4" 2023-10-09T09:49:53.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:53.471 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:53.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:53.574 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:53.601 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:53.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:53.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:53.735 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:53.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:53.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:53.840 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:53.870 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:53.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:53.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:53.974 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:54.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:54.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:54.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:54.109 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:54.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:54.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:54.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:54.311 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:53 smithi181 bash[59345]: cluster 2023-10-09T09:49:52.163007+0000 mgr.y (mgr.34107) 1317 : cluster 0 pgmap v2177: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 295 objects/s recovering 2023-10-09T09:49:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:53 smithi181 bash[59345]: cluster 2023-10-09T09:49:52.561196+0000 osd.4 (osd.4) 662 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:53 smithi181 bash[59345]: cluster 2023-10-09T09:49:52.978685+0000 mon.a (mon.0) 2423 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:49:54.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:53 smithi163 bash[75425]: cluster 2023-10-09T09:49:52.163007+0000 mgr.y (mgr.34107) 1317 : cluster 0 pgmap v2177: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 295 objects/s recovering 2023-10-09T09:49:54.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:53 smithi163 bash[75425]: cluster 2023-10-09T09:49:52.561196+0000 osd.4 (osd.4) 662 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:54.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:53 smithi163 bash[75425]: cluster 2023-10-09T09:49:52.978685+0000 mon.a (mon.0) 2423 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:49:54.324 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:53 smithi163 bash[78107]: cluster 2023-10-09T09:49:52.163007+0000 mgr.y (mgr.34107) 1317 : cluster 0 pgmap v2177: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 295 objects/s recovering 2023-10-09T09:49:54.324 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:53 smithi163 bash[78107]: cluster 2023-10-09T09:49:52.561196+0000 osd.4 (osd.4) 662 : cluster 3 failed to encode map e1265 with expected crc 2023-10-09T09:49:54.324 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:53 smithi163 bash[78107]: cluster 2023-10-09T09:49:52.978685+0000 mon.a (mon.0) 2423 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T09:49:54.343 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:54.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:54.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:54.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:54.480 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:54.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:54.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:54.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:54.617 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:54.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:54.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:54.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:54.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:54.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:54.856 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:54.887 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:54.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:54.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:54.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:54.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:55.027 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:49:55.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:55.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:49:55.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:55.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:55.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:55.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:55.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:55.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:55.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:55.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:49:55.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:55.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:49:55.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:55.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:55.536 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:55.564 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:55.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:55.667 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:55.696 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:55.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:55.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:55.800 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:55.835 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:55.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:55.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:55.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:55.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:55.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:55 smithi181 bash[59345]: cluster 2023-10-09T09:49:54.163753+0000 mgr.y (mgr.34107) 1318 : cluster 0 pgmap v2178: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 234 objects/s recovering 2023-10-09T09:49:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:55 smithi181 bash[59345]: audit 2023-10-09T09:49:55.404461+0000 mon.a (mon.0) 2424 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:56.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:55 smithi181 bash[59345]: audit 2023-10-09T09:49:55.405654+0000 mon.a (mon.0) 2425 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:56.074 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:56.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:56.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:56.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:56.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:56.238 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:56.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:56.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:56.342 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:56.353 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:55 smithi163 bash[75425]: cluster 2023-10-09T09:49:54.163753+0000 mgr.y (mgr.34107) 1318 : cluster 0 pgmap v2178: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 234 objects/s recovering 2023-10-09T09:49:56.353 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:55 smithi163 bash[75425]: audit 2023-10-09T09:49:55.404461+0000 mon.a (mon.0) 2424 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:56.353 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:55 smithi163 bash[75425]: audit 2023-10-09T09:49:55.405654+0000 mon.a (mon.0) 2425 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:56.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:55 smithi163 bash[78107]: cluster 2023-10-09T09:49:54.163753+0000 mgr.y (mgr.34107) 1318 : cluster 0 pgmap v2178: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 234 objects/s recovering 2023-10-09T09:49:56.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:55 smithi163 bash[78107]: audit 2023-10-09T09:49:55.404461+0000 mon.a (mon.0) 2424 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:49:56.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:55 smithi163 bash[78107]: audit 2023-10-09T09:49:55.405654+0000 mon.a (mon.0) 2425 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:49:56.368 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:56.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:56.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:56.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:56.505 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:56.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:56.610 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:56.642 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:56.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:56.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:56.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:56.780 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:56.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:56.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:56.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:56.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:56.915 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:57.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:57.035 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:57.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:57.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:57.138 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:57.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:57.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:57.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:57.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:57.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:57.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:57.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:57.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:57.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:57.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:57.543 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:57.569 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:57.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:57.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:57.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:57.709 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:57.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:57.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:57.814 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:57.843 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:57.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:57.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:57.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:57.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:57.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:57.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:58.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:58.105 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:49:58.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:58.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:49:58.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:58.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:49:57 smithi163 bash[78107]: cluster 2023-10-09T09:49:56.164695+0000 mgr.y (mgr.34107) 1319 : cluster 0 pgmap v2179: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 182 objects/s recovering 2023-10-09T09:49:58.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:49:57 smithi163 bash[75425]: cluster 2023-10-09T09:49:56.164695+0000 mgr.y (mgr.34107) 1319 : cluster 0 pgmap v2179: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 182 objects/s recovering 2023-10-09T09:49:58.240 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:49:58.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:58.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:49:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:49:57 smithi181 bash[59345]: cluster 2023-10-09T09:49:56.164695+0000 mgr.y (mgr.34107) 1319 : cluster 0 pgmap v2179: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 182 objects/s recovering 2023-10-09T09:49:58.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:58.377 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:58.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:58.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:58.481 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:58.511 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:58.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:58.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:58.614 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:58.652 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:49:58.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:58.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:49:58.755 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:58.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:58.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:58.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:58.890 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:58.924 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:58.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:58.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:58.975 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:59.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:59.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:59.058 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:49:59.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:49:59.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:59.192 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:59.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:59.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:59.294 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:59.327 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:49:59.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:59.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:49:59.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:59.463 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:59.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:59.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:59.566 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:59.596 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:59.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:59.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:59.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:59.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:49:59.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:59.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:49:59.827 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:59.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:49:59.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:59.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:49:59.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:00.003 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:00.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:00.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:00.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:00.137 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:00.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:00.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:00.240 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:00.256 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:00 smithi181 bash[59345]: cluster 2023-10-09T09:49:58.165632+0000 mgr.y (mgr.34107) 1320 : cluster 0 pgmap v2180: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 158 objects/s recovering 2023-10-09T09:50:00.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:00.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:00.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:00.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:00.395 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:00 smithi163 bash[78107]: cluster 2023-10-09T09:49:58.165632+0000 mgr.y (mgr.34107) 1320 : cluster 0 pgmap v2180: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 158 objects/s recovering 2023-10-09T09:50:00.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:00 smithi163 bash[75425]: cluster 2023-10-09T09:49:58.165632+0000 mgr.y (mgr.34107) 1320 : cluster 0 pgmap v2180: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 158 objects/s recovering 2023-10-09T09:50:00.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:00.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:00.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:00.511 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:00.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:00.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:00.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:00.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:00.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:00.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:00.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:00.784 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:00.817 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:00.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:00.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:00.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:00.953 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:00.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:00.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:01.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:01 smithi181 bash[59345]: cluster 2023-10-09T09:50:00.003329+0000 mon.a (mon.0) 2426 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T09:50:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:01 smithi181 bash[59345]: cluster 2023-10-09T09:50:00.003361+0000 mon.a (mon.0) 2427 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T09:50:01.090 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:01.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:01.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:01.194 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:01.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:01.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:01.329 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:01.358 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:01 smithi163 bash[75425]: cluster 2023-10-09T09:50:00.003329+0000 mon.a (mon.0) 2426 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T09:50:01.358 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:01 smithi163 bash[75425]: cluster 2023-10-09T09:50:00.003361+0000 mon.a (mon.0) 2427 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T09:50:01.358 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:01 smithi163 bash[78107]: cluster 2023-10-09T09:50:00.003329+0000 mon.a (mon.0) 2426 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T09:50:01.358 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:01 smithi163 bash[78107]: cluster 2023-10-09T09:50:00.003361+0000 mon.a (mon.0) 2427 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T09:50:01.360 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:01.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:01.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:01.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:01.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:01.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:01.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:01.537 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:01.538 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:01.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:01.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:01.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:01.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:01.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:01.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:01.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:01.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:01.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:01.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:01.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:01.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:01.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:02.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:02.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:02.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:02.125 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:02.150 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:02.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:02.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:02.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:02 smithi181 bash[59345]: cluster 2023-10-09T09:50:00.166192+0000 mgr.y (mgr.34107) 1321 : cluster 0 pgmap v2181: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 145 objects/s recovering 2023-10-09T09:50:02.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:02 smithi181 bash[59345]: audit 2023-10-09T09:50:01.148526+0000 mon.a (mon.0) 2428 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:02.283 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:02.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:02.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:02.387 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:02.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:02.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:02.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:02.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:02 smithi163 bash[75425]: cluster 2023-10-09T09:50:00.166192+0000 mgr.y (mgr.34107) 1321 : cluster 0 pgmap v2181: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 145 objects/s recovering 2023-10-09T09:50:02.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:02 smithi163 bash[75425]: audit 2023-10-09T09:50:01.148526+0000 mon.a (mon.0) 2428 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:02.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:02 smithi163 bash[78107]: cluster 2023-10-09T09:50:00.166192+0000 mgr.y (mgr.34107) 1321 : cluster 0 pgmap v2181: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 145 objects/s recovering 2023-10-09T09:50:02.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:02 smithi163 bash[78107]: audit 2023-10-09T09:50:01.148526+0000 mon.a (mon.0) 2428 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:02.512 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:02.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:02.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:02.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:02.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:02.677 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:02.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:02.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:02.781 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:02.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:02.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:02.914 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:02.939 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:02.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:02.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:03.042 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:03.073 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:03.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:03.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:03.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:03.207 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:03.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:03.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:03.311 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:03.346 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:03.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:03.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:03.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:03.483 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:03.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:03.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:03.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:03.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:03.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:03.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:03.619 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:50:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:50:03] "GET /metrics HTTP/1.1" 200 36339 "" "Prometheus/2.33.4" 2023-10-09T09:50:03.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:03.748 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:03.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:03.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:03.852 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:03.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:03.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:03.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:03.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:04.006 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:04.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:04.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:04.109 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:04.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:04.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:04.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:04.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:04.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:04.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:04.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:04.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:04 smithi181 bash[59345]: cluster 2023-10-09T09:50:02.166843+0000 mgr.y (mgr.34107) 1322 : cluster 0 pgmap v2182: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 130 objects/s recovering 2023-10-09T09:50:04.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:04.390 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:04.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:04.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:04 smithi163 bash[78107]: cluster 2023-10-09T09:50:02.166843+0000 mgr.y (mgr.34107) 1322 : cluster 0 pgmap v2182: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 130 objects/s recovering 2023-10-09T09:50:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:04 smithi163 bash[75425]: cluster 2023-10-09T09:50:02.166843+0000 mgr.y (mgr.34107) 1322 : cluster 0 pgmap v2182: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 130 objects/s recovering 2023-10-09T09:50:04.492 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:04.523 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:04.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:04.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:04.625 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:04.646 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:04.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:04.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:04.749 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:04.782 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:04.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:04.886 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:04.913 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:04.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:04.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:05.016 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:05.035 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:05.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:05.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:05.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:05.165 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:05.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:05.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:05.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:05.287 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:05.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:05.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:05.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:05.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:05.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:05.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:05.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:05.540 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:05.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:05.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:05.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:05.669 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:05.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:05.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:05.772 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:05.806 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:05.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:05.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:05.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:05.933 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:05.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:05.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:06.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:06.056 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_5' removed 2023-10-09T09:50:06.066 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:06.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:06 smithi181 bash[59345]: cluster 2023-10-09T09:50:04.167437+0000 mgr.y (mgr.34107) 1323 : cluster 0 pgmap v2183: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:06.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:06 smithi181 bash[59345]: audit 2023-10-09T09:50:05.320071+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.163:0/2914923266' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:50:06.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:06 smithi181 bash[59345]: audit 2023-10-09T09:50:05.320733+0000 mon.a (mon.0) 2429 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:50:06.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:06.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:06.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:06.201 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:06.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:06.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:06 smithi163 bash[78107]: cluster 2023-10-09T09:50:04.167437+0000 mgr.y (mgr.34107) 1323 : cluster 0 pgmap v2183: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:06 smithi163 bash[78107]: audit 2023-10-09T09:50:05.320071+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.163:0/2914923266' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:50:06.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:06 smithi163 bash[78107]: audit 2023-10-09T09:50:05.320733+0000 mon.a (mon.0) 2429 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:50:06.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:06 smithi163 bash[75425]: cluster 2023-10-09T09:50:04.167437+0000 mgr.y (mgr.34107) 1323 : cluster 0 pgmap v2183: 89 pgs: 89 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:06.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:06 smithi163 bash[75425]: audit 2023-10-09T09:50:05.320071+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.163:0/2914923266' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:50:06.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:06 smithi163 bash[75425]: audit 2023-10-09T09:50:05.320733+0000 mon.a (mon.0) 2429 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:50:06.304 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:06.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:06.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:06.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:06.471 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:06.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:06.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:06.574 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:06.606 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:06.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:06.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:06.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:06.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:06.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:06.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:06.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:06.873 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:06.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:06.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:06.976 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:07.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:07.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:07.108 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:07.124 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:07 smithi163 bash[78107]: cluster 2023-10-09T09:50:06.026435+0000 osd.5 (osd.5) 855 : cluster 3 failed to encode map e1266 with expected crc 2023-10-09T09:50:07.124 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:07 smithi163 bash[78107]: audit 2023-10-09T09:50:06.028186+0000 mon.a (mon.0) 2430 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:50:07.124 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:07 smithi163 bash[78107]: cluster 2023-10-09T09:50:06.028265+0000 mon.a (mon.0) 2431 : cluster 0 osdmap e1266: 8 total, 8 up, 6 in 2023-10-09T09:50:07.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:07 smithi163 bash[75425]: cluster 2023-10-09T09:50:06.026435+0000 osd.5 (osd.5) 855 : cluster 3 failed to encode map e1266 with expected crc 2023-10-09T09:50:07.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:07 smithi163 bash[75425]: audit 2023-10-09T09:50:06.028186+0000 mon.a (mon.0) 2430 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:50:07.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:07 smithi163 bash[75425]: cluster 2023-10-09T09:50:06.028265+0000 mon.a (mon.0) 2431 : cluster 0 osdmap e1266: 8 total, 8 up, 6 in 2023-10-09T09:50:07.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:07.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:07.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:07.239 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:07.251 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:07 smithi181 bash[59345]: cluster 2023-10-09T09:50:06.026435+0000 osd.5 (osd.5) 855 : cluster 3 failed to encode map e1266 with expected crc 2023-10-09T09:50:07.251 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:07 smithi181 bash[59345]: audit 2023-10-09T09:50:06.028186+0000 mon.a (mon.0) 2430 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:50:07.252 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:07 smithi181 bash[59345]: cluster 2023-10-09T09:50:06.028265+0000 mon.a (mon.0) 2431 : cluster 0 osdmap e1266: 8 total, 8 up, 6 in 2023-10-09T09:50:07.269 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:07.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:07.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:07.373 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:07.404 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:07.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:07.507 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:07.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:07.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:07.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:07.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:07.668 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:07.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:07.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:07.771 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:07.796 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:07.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:07.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:07.899 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:07.925 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:07.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:08.029 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:08.051 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:08.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:08.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:08.154 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:08.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:08.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:08.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:08 smithi163 bash[78107]: cluster 2023-10-09T09:50:06.167903+0000 mgr.y (mgr.34107) 1324 : cluster 0 pgmap v2185: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:08.260 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:08 smithi163 bash[75425]: cluster 2023-10-09T09:50:06.167903+0000 mgr.y (mgr.34107) 1324 : cluster 0 pgmap v2185: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:08.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:08.302 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:08.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:08.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:08.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:08 smithi181 bash[59345]: cluster 2023-10-09T09:50:06.167903+0000 mgr.y (mgr.34107) 1324 : cluster 0 pgmap v2185: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:08.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:08.425 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:08.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:08.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:08.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:08.557 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:08.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:08.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:08.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:08.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:08.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:08.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:08.783 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:08.817 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:08.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:08.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:08.921 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:08.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:08.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:08.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:09.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:09.065 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:09.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:09.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:09.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:09.202 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:09.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:09.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:09.306 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:09.340 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:09.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:09.444 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:09.477 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:09.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:09.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:09.580 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:09.611 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:09.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:09.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:09.715 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:09.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:09.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:09.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:09.843 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:09.874 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:09.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:09.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:09.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:10.001 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:10.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:10.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:10.104 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:10.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:10.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:10.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:10.240 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:10.272 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:10.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:10.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:10 smithi181 bash[59345]: cluster 2023-10-09T09:50:08.168573+0000 mgr.y (mgr.34107) 1325 : cluster 0 pgmap v2186: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:10.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:10.388 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:10 smithi163 bash[75425]: cluster 2023-10-09T09:50:08.168573+0000 mgr.y (mgr.34107) 1325 : cluster 0 pgmap v2186: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:10.389 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:10 smithi163 bash[78107]: cluster 2023-10-09T09:50:08.168573+0000 mgr.y (mgr.34107) 1325 : cluster 0 pgmap v2186: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:10.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:10.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:10.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:10.506 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:10.538 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:10.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:10.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:10.613 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:50:10.614 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:50:10.614 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2023-10-09T09:50:10.614 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_6 16 2023-10-09T09:50:10.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:10.683 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:10.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:10.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:10.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:10.809 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:10.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:10.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:10.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:10.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:10.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:10.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:11.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:11 smithi163 bash[75425]: audit 2023-10-09T09:50:10.399934+0000 mon.a (mon.0) 2432 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:11.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:11.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:11.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:11.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:11.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:11.183 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:11 smithi181 bash[59345]: audit 2023-10-09T09:50:10.399934+0000 mon.a (mon.0) 2432 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:11.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:11.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:11.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:11.307 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:11.338 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:11.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:11.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:11.455 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:11 smithi163 bash[78107]: audit 2023-10-09T09:50:10.399934+0000 mon.a (mon.0) 2432 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:11.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:11.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:11.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:11.573 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:11.601 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:11.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:11.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:11.704 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:11.740 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:11.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:11.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:11.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:11.875 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:11.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:11.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:11.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:12.012 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:12.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:12.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:12.065 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T09:50:12.114 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:12 smithi163 bash[78107]: cluster 2023-10-09T09:50:10.169137+0000 mgr.y (mgr.34107) 1326 : cluster 0 pgmap v2187: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:12.114 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:12 smithi163 bash[78107]: audit 2023-10-09T09:50:11.067916+0000 mon.a (mon.0) 2433 : audit 1 from='client.? 172.21.15.163:0/902170469' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:50:12.114 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:12 smithi163 bash[75425]: cluster 2023-10-09T09:50:10.169137+0000 mgr.y (mgr.34107) 1326 : cluster 0 pgmap v2187: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:12.114 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:12 smithi163 bash[75425]: audit 2023-10-09T09:50:11.067916+0000 mon.a (mon.0) 2433 : audit 1 from='client.? 172.21.15.163:0/902170469' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:50:12.116 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:12.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:12.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:12.239 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:12.260 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:12.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:12.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:12 smithi181 bash[59345]: cluster 2023-10-09T09:50:10.169137+0000 mgr.y (mgr.34107) 1326 : cluster 0 pgmap v2187: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:12 smithi181 bash[59345]: audit 2023-10-09T09:50:11.067916+0000 mon.a (mon.0) 2433 : audit 1 from='client.? 172.21.15.163:0/902170469' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:50:12.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:12.395 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:12.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:12.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:12.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:12.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:12.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:12.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:12.630 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:12.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:12.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:12.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:12.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:12.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:12.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:12.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:12.906 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:12.940 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:12.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:12.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:13.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:13.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:13.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:13.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[75425]: cluster 2023-10-09T09:50:12.052727+0000 osd.5 (osd.5) 856 : cluster 3 failed to encode map e1267 with expected crc 2023-10-09T09:50:13.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[75425]: cluster 2023-10-09T09:50:12.052827+0000 osd.6 (osd.6) 551 : cluster 3 failed to encode map e1267 with expected crc 2023-10-09T09:50:13.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[75425]: audit 2023-10-09T09:50:12.056295+0000 mon.a (mon.0) 2434 : audit 1 from='client.? 172.21.15.163:0/902170469' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:50:13.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[75425]: cluster 2023-10-09T09:50:12.056454+0000 mon.a (mon.0) 2435 : cluster 0 osdmap e1267: 8 total, 8 up, 6 in 2023-10-09T09:50:13.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[75425]: cluster 2023-10-09T09:50:12.169750+0000 mgr.y (mgr.34107) 1327 : cluster 0 pgmap v2189: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:13.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[78107]: cluster 2023-10-09T09:50:12.052727+0000 osd.5 (osd.5) 856 : cluster 3 failed to encode map e1267 with expected crc 2023-10-09T09:50:13.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[78107]: cluster 2023-10-09T09:50:12.052827+0000 osd.6 (osd.6) 551 : cluster 3 failed to encode map e1267 with expected crc 2023-10-09T09:50:13.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[78107]: audit 2023-10-09T09:50:12.056295+0000 mon.a (mon.0) 2434 : audit 1 from='client.? 172.21.15.163:0/902170469' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:50:13.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[78107]: cluster 2023-10-09T09:50:12.056454+0000 mon.a (mon.0) 2435 : cluster 0 osdmap e1267: 8 total, 8 up, 6 in 2023-10-09T09:50:13.140 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[78107]: cluster 2023-10-09T09:50:12.169750+0000 mgr.y (mgr.34107) 1327 : cluster 0 pgmap v2189: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:13.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:13.207 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:13.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:13.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:13.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:13.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:13 smithi181 bash[59345]: cluster 2023-10-09T09:50:12.052727+0000 osd.5 (osd.5) 856 : cluster 3 failed to encode map e1267 with expected crc 2023-10-09T09:50:13.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:13 smithi181 bash[59345]: cluster 2023-10-09T09:50:12.052827+0000 osd.6 (osd.6) 551 : cluster 3 failed to encode map e1267 with expected crc 2023-10-09T09:50:13.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:13 smithi181 bash[59345]: audit 2023-10-09T09:50:12.056295+0000 mon.a (mon.0) 2434 : audit 1 from='client.? 172.21.15.163:0/902170469' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:50:13.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:13 smithi181 bash[59345]: cluster 2023-10-09T09:50:12.056454+0000 mon.a (mon.0) 2435 : cluster 0 osdmap e1267: 8 total, 8 up, 6 in 2023-10-09T09:50:13.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:13 smithi181 bash[59345]: cluster 2023-10-09T09:50:12.169750+0000 mgr.y (mgr.34107) 1327 : cluster 0 pgmap v2189: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:13.338 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:13.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:13.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:13.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:13.453 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:50:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:50:13] "GET /metrics HTTP/1.1" 200 33983 "" "Prometheus/2.33.4" 2023-10-09T09:50:13.471 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:13.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:13.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:13.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:13.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:13.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:13.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:13.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:13.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:13.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:13.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:13.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:13.878 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:13.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:13.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:13.982 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:14.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:14.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:14.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:14.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:14.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:14.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:14.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:14.278 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:14.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:14.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:14.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:14.412 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:14.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:14.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:14.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:14.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:14.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:14.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:14.683 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:14.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:14.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:14.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:14.818 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:14.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:14.864 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:50:14.864 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:50:14.922 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:14.948 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:14.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:14.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:15.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:15.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:15.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:15.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:15.181 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:15.217 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:15.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:15.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:15.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:15.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:15.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:15.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:15.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T09:50:15.457 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:15.470 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:15 smithi181 bash[59345]: cluster 2023-10-09T09:50:14.170293+0000 mgr.y (mgr.34107) 1328 : cluster 0 pgmap v2190: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:15.489 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:15.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:15.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:15.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:15.626 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:15.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:15.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:15.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:15 smithi163 bash[78107]: cluster 2023-10-09T09:50:14.170293+0000 mgr.y (mgr.34107) 1328 : cluster 0 pgmap v2190: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:15.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:15 smithi163 bash[75425]: cluster 2023-10-09T09:50:14.170293+0000 mgr.y (mgr.34107) 1328 : cluster 0 pgmap v2190: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:15.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:15.765 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:15.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:15.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:15.869 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:15.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:15.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:15.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:16.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:16.034 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:16.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:16.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:16.138 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:16.165 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:16.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:16.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:16.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:16.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:16.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:16.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:16.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:16.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:16.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:16.538 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:16.572 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:16.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:16.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:16.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:16.705 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:16.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:16.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:16.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:50:16.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:16.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:16.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:16.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:16.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:16.957 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:50:16.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:16.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:16.996 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:50:17.035 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:50:17.067 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:50:17.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:17.094 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:50:17.101 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:17.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:17.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:17.120 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:50:17.144 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:50:17.168 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:50:17.170 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:50:17.170 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:50:17.171 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:50:17.171 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:50:17.171 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:50:17.171 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:50:17.171 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:50:17.171 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:50:17.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:17.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:17.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:17.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:17.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:17.366 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:17.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:17.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:17.470 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:17.506 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:17.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:17.610 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:17.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:17.647 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:17.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:17.748 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:17.773 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:17.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:17.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:17.910 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:17.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:17.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:17.923 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:50:18.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:18.049 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:18.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:18.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:18.155 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:18.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:17 smithi181 bash[59345]: cluster 2023-10-09T09:50:16.170858+0000 mgr.y (mgr.34107) 1329 : cluster 0 pgmap v2191: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:18.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:17 smithi181 bash[59345]: audit 2023-10-09T09:50:16.917346+0000 mon.a (mon.0) 2436 : audit 0 from='client.? 172.21.15.163:0/2705559064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:50:18.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:17 smithi181 bash[59345]: audit 2023-10-09T09:50:17.187303+0000 mon.a (mon.0) 2437 : audit 1 from='client.? 172.21.15.163:0/179834439' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:50:18.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:18.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:18.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:18.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:17 smithi163 bash[75425]: cluster 2023-10-09T09:50:16.170858+0000 mgr.y (mgr.34107) 1329 : cluster 0 pgmap v2191: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:18.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:17 smithi163 bash[75425]: audit 2023-10-09T09:50:16.917346+0000 mon.a (mon.0) 2436 : audit 0 from='client.? 172.21.15.163:0/2705559064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:50:18.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:17 smithi163 bash[75425]: audit 2023-10-09T09:50:17.187303+0000 mon.a (mon.0) 2437 : audit 1 from='client.? 172.21.15.163:0/179834439' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:50:18.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:17 smithi163 bash[78107]: cluster 2023-10-09T09:50:16.170858+0000 mgr.y (mgr.34107) 1329 : cluster 0 pgmap v2191: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:18.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:17 smithi163 bash[78107]: audit 2023-10-09T09:50:16.917346+0000 mon.a (mon.0) 2436 : audit 0 from='client.? 172.21.15.163:0/2705559064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:50:18.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:17 smithi163 bash[78107]: audit 2023-10-09T09:50:17.187303+0000 mon.a (mon.0) 2437 : audit 1 from='client.? 172.21.15.163:0/179834439' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:50:18.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:18.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:18.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:18.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:18.422 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:18.447 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:18.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:18.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:18.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:18.581 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:18.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:18.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:18.685 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:18.716 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:18.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:18.819 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:18.853 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:18.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:18.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:18.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:18 smithi163 bash[75425]: cluster 2023-10-09T09:50:17.909776+0000 mon.a (mon.0) 2438 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:50:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:18 smithi163 bash[75425]: cluster 2023-10-09T09:50:17.909828+0000 mon.a (mon.0) 2439 : cluster 1 Cluster is now healthy 2023-10-09T09:50:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:18 smithi163 bash[75425]: cluster 2023-10-09T09:50:17.913033+0000 osd.5 (osd.5) 857 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:18 smithi163 bash[75425]: audit 2023-10-09T09:50:17.917202+0000 mon.a (mon.0) 2440 : audit 1 from='client.? 172.21.15.163:0/179834439' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:50:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:18 smithi163 bash[75425]: cluster 2023-10-09T09:50:17.917292+0000 mon.a (mon.0) 2441 : cluster 0 osdmap e1268: 8 total, 8 up, 6 in 2023-10-09T09:50:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:18 smithi163 bash[78107]: cluster 2023-10-09T09:50:17.909776+0000 mon.a (mon.0) 2438 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:50:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:18 smithi163 bash[78107]: cluster 2023-10-09T09:50:17.909828+0000 mon.a (mon.0) 2439 : cluster 1 Cluster is now healthy 2023-10-09T09:50:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:18 smithi163 bash[78107]: cluster 2023-10-09T09:50:17.913033+0000 osd.5 (osd.5) 857 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:18 smithi163 bash[78107]: audit 2023-10-09T09:50:17.917202+0000 mon.a (mon.0) 2440 : audit 1 from='client.? 172.21.15.163:0/179834439' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:50:18.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:18 smithi163 bash[78107]: cluster 2023-10-09T09:50:17.917292+0000 mon.a (mon.0) 2441 : cluster 0 osdmap e1268: 8 total, 8 up, 6 in 2023-10-09T09:50:18.988 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:18.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:18.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:19.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:19.129 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:19.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:19.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:19.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:19.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:18 smithi181 bash[59345]: cluster 2023-10-09T09:50:17.909776+0000 mon.a (mon.0) 2438 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:50:19.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:18 smithi181 bash[59345]: cluster 2023-10-09T09:50:17.909828+0000 mon.a (mon.0) 2439 : cluster 1 Cluster is now healthy 2023-10-09T09:50:19.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:18 smithi181 bash[59345]: cluster 2023-10-09T09:50:17.913033+0000 osd.5 (osd.5) 857 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:19.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:18 smithi181 bash[59345]: audit 2023-10-09T09:50:17.917202+0000 mon.a (mon.0) 2440 : audit 1 from='client.? 172.21.15.163:0/179834439' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:50:19.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:18 smithi181 bash[59345]: cluster 2023-10-09T09:50:17.917292+0000 mon.a (mon.0) 2441 : cluster 0 osdmap e1268: 8 total, 8 up, 6 in 2023-10-09T09:50:19.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:19.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:19.371 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:19.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:19.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:19.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:19.506 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:19.535 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:19.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:19.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:19.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:19.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:19.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:19.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:19.720 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:19.779 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:19.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:19.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:19.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:19.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:19.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:19.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:19.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:20.050 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:20.062 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:19 smithi181 bash[59345]: cluster 2023-10-09T09:50:18.171240+0000 mgr.y (mgr.34107) 1330 : cluster 0 pgmap v2193: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:20.083 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:20.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:20.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:20.188 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:20.200 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:19 smithi163 bash[78107]: cluster 2023-10-09T09:50:18.171240+0000 mgr.y (mgr.34107) 1330 : cluster 0 pgmap v2193: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:20.201 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:19 smithi163 bash[75425]: cluster 2023-10-09T09:50:18.171240+0000 mgr.y (mgr.34107) 1330 : cluster 0 pgmap v2193: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:20.215 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:20.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:20.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:20.223 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:50:20.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:20.349 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:20.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:20.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:20.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:20.489 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:20.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:20.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:20.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:20.626 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:20.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:20.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:20.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:20.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:20.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:20.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:20.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:20.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:20.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:20.923 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:50:20.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:21.018 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:21.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:21.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:21.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:21.129 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:20 smithi163 bash[78107]: cluster 2023-10-09T09:50:19.134870+0000 osd.7 (osd.7) 611 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:21.129 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:20 smithi163 bash[78107]: cluster 2023-10-09T09:50:19.607714+0000 osd.6 (osd.6) 552 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:21.130 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:20 smithi163 bash[78107]: cluster 2023-10-09T09:50:19.669306+0000 osd.4 (osd.4) 663 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:21.130 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:20 smithi163 bash[75425]: cluster 2023-10-09T09:50:19.134870+0000 osd.7 (osd.7) 611 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:21.130 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:20 smithi163 bash[75425]: cluster 2023-10-09T09:50:19.607714+0000 osd.6 (osd.6) 552 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:21.130 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:20 smithi163 bash[75425]: cluster 2023-10-09T09:50:19.669306+0000 osd.4 (osd.4) 663 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:21.144 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:21.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:21.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:21.274 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:21.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:21.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:21.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:20 smithi181 bash[59345]: cluster 2023-10-09T09:50:19.134870+0000 osd.7 (osd.7) 611 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:21.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:20 smithi181 bash[59345]: cluster 2023-10-09T09:50:19.607714+0000 osd.6 (osd.6) 552 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:21.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:20 smithi181 bash[59345]: cluster 2023-10-09T09:50:19.669306+0000 osd.4 (osd.4) 663 : cluster 3 failed to encode map e1268 with expected crc 2023-10-09T09:50:21.378 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:21.412 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:21.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:21.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:21.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:21.549 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:21.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:21.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:21.685 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:21.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:21.752 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:21.857 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:21.858 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:21.889 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:21.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:21.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:21.992 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:22.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:22.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:22.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:22.126 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:22.140 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:21 smithi163 bash[78107]: cluster 2023-10-09T09:50:20.171959+0000 mgr.y (mgr.34107) 1331 : cluster 0 pgmap v2194: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:22.141 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:21 smithi163 bash[75425]: cluster 2023-10-09T09:50:20.171959+0000 mgr.y (mgr.34107) 1331 : cluster 0 pgmap v2194: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:22.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:22.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:22.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:22.263 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:22.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:22.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:22.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:21 smithi181 bash[59345]: cluster 2023-10-09T09:50:20.171959+0000 mgr.y (mgr.34107) 1331 : cluster 0 pgmap v2194: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:22.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:22.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:22.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:22.527 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:22.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:22.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:22.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:22.664 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:22.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:22.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:22.800 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:22.829 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:22.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:22.933 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:22.954 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:22.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:22.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:23.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:23.082 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:23.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:23.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:23.185 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:23.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:23.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:23.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:23.313 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:23.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:23.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:23.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:23.485 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:23.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:23.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:23.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:23.618 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:23.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:23.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:23.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:50:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:50:23] "GET /metrics HTTP/1.1" 200 33983 "" "Prometheus/2.33.4" 2023-10-09T09:50:23.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:23.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:23.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:23.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:23.858 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:23.888 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:23.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:23.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:23.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:24.018 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:24.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:24.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:24.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:24.133 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:23 smithi181 bash[59345]: cluster 2023-10-09T09:50:22.172932+0000 mgr.y (mgr.34107) 1332 : cluster 0 pgmap v2195: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:24.152 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:24.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:24.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:24.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:23 smithi163 bash[78107]: cluster 2023-10-09T09:50:22.172932+0000 mgr.y (mgr.34107) 1332 : cluster 0 pgmap v2195: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:24.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:23 smithi163 bash[75425]: cluster 2023-10-09T09:50:22.172932+0000 mgr.y (mgr.34107) 1332 : cluster 0 pgmap v2195: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:24.256 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:24.292 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:24.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:24.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:24.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:24.422 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:24.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:24.525 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:24.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:24.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:24.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:24.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:24.684 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:24.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:24.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:24.786 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:24.821 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:24.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:24.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:24.925 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:24.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:24.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:24.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:25.060 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:25.083 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:25.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:25.187 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:25.207 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:25.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:25.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:25.310 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:25.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:25.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:25.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:25.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:25.484 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:25.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:25.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:25.588 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:25.625 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:25.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:25.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:25.731 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:25.767 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:25.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:25.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:25.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:25.896 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:25.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:25.969 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_6' created 2023-10-09T09:50:25.990 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:25 smithi163 bash[78107]: cluster 2023-10-09T09:50:24.173689+0000 mgr.y (mgr.34107) 1333 : cluster 0 pgmap v2196: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:25.990 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:25 smithi163 bash[78107]: audit 2023-10-09T09:50:25.389730+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.163:0/3651421143' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T09:50:25.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:25 smithi163 bash[78107]: audit 2023-10-09T09:50:25.390075+0000 mon.a (mon.0) 2442 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T09:50:25.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:25 smithi163 bash[78107]: audit 2023-10-09T09:50:25.843347+0000 mon.a (mon.0) 2443 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:25.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:25 smithi163 bash[78107]: audit 2023-10-09T09:50:25.844278+0000 mon.a (mon.0) 2444 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:25.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:25 smithi163 bash[75425]: cluster 2023-10-09T09:50:24.173689+0000 mgr.y (mgr.34107) 1333 : cluster 0 pgmap v2196: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:25.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:25 smithi163 bash[75425]: audit 2023-10-09T09:50:25.389730+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.163:0/3651421143' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T09:50:25.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:25 smithi163 bash[75425]: audit 2023-10-09T09:50:25.390075+0000 mon.a (mon.0) 2442 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T09:50:25.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:25 smithi163 bash[75425]: audit 2023-10-09T09:50:25.843347+0000 mon.a (mon.0) 2443 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:25.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:25 smithi163 bash[75425]: audit 2023-10-09T09:50:25.844278+0000 mon.a (mon.0) 2444 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:26.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:26.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:26.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:26.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:26.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:26.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:26.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:26.299 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:26.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:25 smithi181 bash[59345]: cluster 2023-10-09T09:50:24.173689+0000 mgr.y (mgr.34107) 1333 : cluster 0 pgmap v2196: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:26.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:25 smithi181 bash[59345]: audit 2023-10-09T09:50:25.389730+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.163:0/3651421143' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T09:50:26.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:25 smithi181 bash[59345]: audit 2023-10-09T09:50:25.390075+0000 mon.a (mon.0) 2442 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T09:50:26.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:25 smithi181 bash[59345]: audit 2023-10-09T09:50:25.843347+0000 mon.a (mon.0) 2443 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:26.320 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:25 smithi181 bash[59345]: audit 2023-10-09T09:50:25.844278+0000 mon.a (mon.0) 2444 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:26.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:26.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:26.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:26.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:26.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:26.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:26.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:26.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:26.607 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:26.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:26.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:26.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:26.751 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:26.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:26.854 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:26.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:26.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:26.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:26.995 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:27.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:26 smithi163 bash[78107]: cluster 2023-10-09T09:50:25.951564+0000 osd.5 (osd.5) 858 : cluster 3 failed to encode map e1269 with expected crc 2023-10-09T09:50:27.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:26 smithi163 bash[78107]: cluster 2023-10-09T09:50:25.951714+0000 osd.6 (osd.6) 553 : cluster 3 failed to encode map e1269 with expected crc 2023-10-09T09:50:27.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:26 smithi163 bash[78107]: audit 2023-10-09T09:50:25.955971+0000 mon.a (mon.0) 2445 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T09:50:27.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:26 smithi163 bash[78107]: cluster 2023-10-09T09:50:25.956074+0000 mon.a (mon.0) 2446 : cluster 0 osdmap e1269: 8 total, 8 up, 6 in 2023-10-09T09:50:27.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:26 smithi163 bash[75425]: cluster 2023-10-09T09:50:25.951564+0000 osd.5 (osd.5) 858 : cluster 3 failed to encode map e1269 with expected crc 2023-10-09T09:50:27.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:26 smithi163 bash[75425]: cluster 2023-10-09T09:50:25.951714+0000 osd.6 (osd.6) 553 : cluster 3 failed to encode map e1269 with expected crc 2023-10-09T09:50:27.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:26 smithi163 bash[75425]: audit 2023-10-09T09:50:25.955971+0000 mon.a (mon.0) 2445 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T09:50:27.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:26 smithi163 bash[75425]: cluster 2023-10-09T09:50:25.956074+0000 mon.a (mon.0) 2446 : cluster 0 osdmap e1269: 8 total, 8 up, 6 in 2023-10-09T09:50:27.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:27.033 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:27.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:27.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:27.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:26 smithi181 bash[59345]: cluster 2023-10-09T09:50:25.951564+0000 osd.5 (osd.5) 858 : cluster 3 failed to encode map e1269 with expected crc 2023-10-09T09:50:27.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:26 smithi181 bash[59345]: cluster 2023-10-09T09:50:25.951714+0000 osd.6 (osd.6) 553 : cluster 3 failed to encode map e1269 with expected crc 2023-10-09T09:50:27.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:26 smithi181 bash[59345]: audit 2023-10-09T09:50:25.955971+0000 mon.a (mon.0) 2445 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T09:50:27.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:26 smithi181 bash[59345]: cluster 2023-10-09T09:50:25.956074+0000 mon.a (mon.0) 2446 : cluster 0 osdmap e1269: 8 total, 8 up, 6 in 2023-10-09T09:50:27.170 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:27.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:27.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:27.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:27.305 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:27.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:27.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:27.410 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:27.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:27.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:27.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:27.550 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:27.579 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:27.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:27.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:27.683 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:27.718 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:27.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:27.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:27.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:27.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:27.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:27.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:27.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:27.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:27.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:27.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:28.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:28.123 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:28.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:28.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:28.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:28.243 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:27 smithi181 bash[59345]: cluster 2023-10-09T09:50:26.174633+0000 mgr.y (mgr.34107) 1334 : cluster 0 pgmap v2198: 89 pgs: 6 creating+peering, 10 unknown, 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:28.244 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:27 smithi181 bash[59345]: cluster 2023-10-09T09:50:26.968845+0000 osd.5 (osd.5) 859 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.244 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:27 smithi181 bash[59345]: cluster 2023-10-09T09:50:26.971558+0000 osd.7 (osd.7) 612 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.244 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:27 smithi181 bash[59345]: cluster 2023-10-09T09:50:26.973621+0000 mon.a (mon.0) 2447 : cluster 0 osdmap e1270: 8 total, 8 up, 6 in 2023-10-09T09:50:28.244 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:27 smithi181 bash[59345]: cluster 2023-10-09T09:50:26.975121+0000 osd.6 (osd.6) 554 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.244 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:27 smithi181 bash[59345]: cluster 2023-10-09T09:50:26.975504+0000 osd.5 (osd.5) 860 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.264 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:28.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:28.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:28.368 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:28.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:28.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:28.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[78107]: cluster 2023-10-09T09:50:26.174633+0000 mgr.y (mgr.34107) 1334 : cluster 0 pgmap v2198: 89 pgs: 6 creating+peering, 10 unknown, 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[78107]: cluster 2023-10-09T09:50:26.968845+0000 osd.5 (osd.5) 859 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[78107]: cluster 2023-10-09T09:50:26.971558+0000 osd.7 (osd.7) 612 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[78107]: cluster 2023-10-09T09:50:26.973621+0000 mon.a (mon.0) 2447 : cluster 0 osdmap e1270: 8 total, 8 up, 6 in 2023-10-09T09:50:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[78107]: cluster 2023-10-09T09:50:26.975121+0000 osd.6 (osd.6) 554 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[78107]: cluster 2023-10-09T09:50:26.975504+0000 osd.5 (osd.5) 860 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[75425]: cluster 2023-10-09T09:50:26.174633+0000 mgr.y (mgr.34107) 1334 : cluster 0 pgmap v2198: 89 pgs: 6 creating+peering, 10 unknown, 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[75425]: cluster 2023-10-09T09:50:26.968845+0000 osd.5 (osd.5) 859 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[75425]: cluster 2023-10-09T09:50:26.971558+0000 osd.7 (osd.7) 612 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[75425]: cluster 2023-10-09T09:50:26.973621+0000 mon.a (mon.0) 2447 : cluster 0 osdmap e1270: 8 total, 8 up, 6 in 2023-10-09T09:50:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[75425]: cluster 2023-10-09T09:50:26.975121+0000 osd.6 (osd.6) 554 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:27 smithi163 bash[75425]: cluster 2023-10-09T09:50:26.975504+0000 osd.5 (osd.5) 860 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:28.505 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:28.536 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:28.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:28.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:28.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:28.670 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:28.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:28.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:28.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:28.803 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:28.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:28.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:28.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:28.939 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:28.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:28.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:29.044 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:29.044 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:29.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:29.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:28 smithi181 bash[59345]: cluster 2023-10-09T09:50:27.369121+0000 osd.4 (osd.4) 664 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:29.060 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:28 smithi181 bash[59345]: cluster 2023-10-09T09:50:27.980791+0000 osd.5 (osd.5) 861 : cluster 3 failed to encode map e1271 with expected crc 2023-10-09T09:50:29.060 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:28 smithi181 bash[59345]: cluster 2023-10-09T09:50:27.981420+0000 osd.6 (osd.6) 555 : cluster 3 failed to encode map e1271 with expected crc 2023-10-09T09:50:29.060 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:28 smithi181 bash[59345]: cluster 2023-10-09T09:50:27.983532+0000 mon.a (mon.0) 2448 : cluster 0 osdmap e1271: 8 total, 8 up, 6 in 2023-10-09T09:50:29.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2023-10-09T09:50:29.079 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:29.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:29.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:29.183 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:29.218 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:29.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:29.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:29.321 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:29.354 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:29.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:29.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:29.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:29.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:28 smithi163 bash[78107]: cluster 2023-10-09T09:50:27.369121+0000 osd.4 (osd.4) 664 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:29.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:28 smithi163 bash[78107]: cluster 2023-10-09T09:50:27.980791+0000 osd.5 (osd.5) 861 : cluster 3 failed to encode map e1271 with expected crc 2023-10-09T09:50:29.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:28 smithi163 bash[78107]: cluster 2023-10-09T09:50:27.981420+0000 osd.6 (osd.6) 555 : cluster 3 failed to encode map e1271 with expected crc 2023-10-09T09:50:29.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:28 smithi163 bash[78107]: cluster 2023-10-09T09:50:27.983532+0000 mon.a (mon.0) 2448 : cluster 0 osdmap e1271: 8 total, 8 up, 6 in 2023-10-09T09:50:29.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:28 smithi163 bash[75425]: cluster 2023-10-09T09:50:27.369121+0000 osd.4 (osd.4) 664 : cluster 3 failed to encode map e1270 with expected crc 2023-10-09T09:50:29.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:28 smithi163 bash[75425]: cluster 2023-10-09T09:50:27.980791+0000 osd.5 (osd.5) 861 : cluster 3 failed to encode map e1271 with expected crc 2023-10-09T09:50:29.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:28 smithi163 bash[75425]: cluster 2023-10-09T09:50:27.981420+0000 osd.6 (osd.6) 555 : cluster 3 failed to encode map e1271 with expected crc 2023-10-09T09:50:29.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:28 smithi163 bash[75425]: cluster 2023-10-09T09:50:27.983532+0000 mon.a (mon.0) 2448 : cluster 0 osdmap e1271: 8 total, 8 up, 6 in 2023-10-09T09:50:29.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:29.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:29.597 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:29.622 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:29.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:29.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:29.726 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:29.757 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:29.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:29.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:29.859 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:29.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:29.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:29.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:29.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:30.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:30.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:30.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:30.125 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:30.150 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:30.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:30.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:30.265 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:29 smithi163 bash[75425]: cluster 2023-10-09T09:50:28.175396+0000 mgr.y (mgr.34107) 1335 : cluster 0 pgmap v2201: 89 pgs: 13 creating+peering, 3 unknown, 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:30.265 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:29 smithi163 bash[75425]: cluster 2023-10-09T09:50:28.636146+0000 osd.4 (osd.4) 665 : cluster 3 failed to encode map e1271 with expected crc 2023-10-09T09:50:30.265 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:29 smithi163 bash[75425]: audit 2023-10-09T09:50:29.947306+0000 mon.a (mon.0) 2449 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:30.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:29 smithi163 bash[78107]: cluster 2023-10-09T09:50:28.175396+0000 mgr.y (mgr.34107) 1335 : cluster 0 pgmap v2201: 89 pgs: 13 creating+peering, 3 unknown, 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:30.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:29 smithi163 bash[78107]: cluster 2023-10-09T09:50:28.636146+0000 osd.4 (osd.4) 665 : cluster 3 failed to encode map e1271 with expected crc 2023-10-09T09:50:30.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:29 smithi163 bash[78107]: audit 2023-10-09T09:50:29.947306+0000 mon.a (mon.0) 2449 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:30.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:30.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:30.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:29 smithi181 bash[59345]: cluster 2023-10-09T09:50:28.175396+0000 mgr.y (mgr.34107) 1335 : cluster 0 pgmap v2201: 89 pgs: 13 creating+peering, 3 unknown, 73 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:29 smithi181 bash[59345]: cluster 2023-10-09T09:50:28.636146+0000 osd.4 (osd.4) 665 : cluster 3 failed to encode map e1271 with expected crc 2023-10-09T09:50:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:29 smithi181 bash[59345]: audit 2023-10-09T09:50:29.947306+0000 mon.a (mon.0) 2449 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:30.382 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:30.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:30.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:30.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:30.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:30.534 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:30.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:30.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:30.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:30.666 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:30.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:30.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:30.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:30.794 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:30.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:30.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:30.898 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:30.923 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:30.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:30.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:31.026 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:31.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:31.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:31.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:31.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:31.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:31.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:31.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:31.311 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:31.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:31.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:31.415 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:31.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:31.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:31.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:31.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:31.585 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:31.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:31.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:31.688 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:31.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:31.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:31.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:31.820 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:31.852 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:31.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:31.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:31.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:31.970 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:50:31.970 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:50:31.970 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T09:50:31.970 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:50:31.980 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:31.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:31.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:32.082 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:32.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:31 smithi181 bash[59345]: cluster 2023-10-09T09:50:30.175870+0000 mgr.y (mgr.34107) 1336 : cluster 0 pgmap v2202: 89 pgs: 13 creating+peering, 2 unknown, 74 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:32.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:32.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:32.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:32.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:31 smithi163 bash[78107]: cluster 2023-10-09T09:50:30.175870+0000 mgr.y (mgr.34107) 1336 : cluster 0 pgmap v2202: 89 pgs: 13 creating+peering, 2 unknown, 74 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:31 smithi163 bash[75425]: cluster 2023-10-09T09:50:30.175870+0000 mgr.y (mgr.34107) 1336 : cluster 0 pgmap v2202: 89 pgs: 13 creating+peering, 2 unknown, 74 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:32.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:32.253 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:32.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:32.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:32.358 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:32.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:32.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:32.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:32.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:32.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:32.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:32.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:32.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:32.656 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:32.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:32.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:32.760 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:32.792 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:32.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:32.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:32.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:32.916 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:32.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:32.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:33.013 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T09:50:33.018 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:33.041 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:33.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:33.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:33.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:33.175 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:33.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:33.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:33.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:32 smithi163 bash[75425]: audit 2023-10-09T09:50:32.289802+0000 mon.a (mon.0) 2450 : audit 1 from='client.? 172.21.15.163:0/932251915' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:50:33.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:32 smithi163 bash[78107]: audit 2023-10-09T09:50:32.289802+0000 mon.a (mon.0) 2450 : audit 1 from='client.? 172.21.15.163:0/932251915' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:50:33.278 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:33.292 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:32 smithi181 bash[59345]: audit 2023-10-09T09:50:32.289802+0000 mon.a (mon.0) 2450 : audit 1 from='client.? 172.21.15.163:0/932251915' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:50:33.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:33.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:33.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:33.414 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:33.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:33.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:33.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:33.547 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:33.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:33.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:33.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:33.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:33.690 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:50:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:50:33] "GET /metrics HTTP/1.1" 200 33980 "" "Prometheus/2.33.4" 2023-10-09T09:50:33.706 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:33.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:33.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:33.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:33.839 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:33.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:33.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:33.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:33.969 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:33.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:33.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:34.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:34.101 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:34.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:34.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:34.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:34.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:34 smithi163 bash[75425]: cluster 2023-10-09T09:50:32.176972+0000 mgr.y (mgr.34107) 1337 : cluster 0 pgmap v2203: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:34.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:34 smithi163 bash[75425]: cluster 2023-10-09T09:50:33.006169+0000 osd.5 (osd.5) 862 : cluster 3 failed to encode map e1272 with expected crc 2023-10-09T09:50:34.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:34 smithi163 bash[75425]: audit 2023-10-09T09:50:33.007728+0000 mon.a (mon.0) 2451 : audit 1 from='client.? 172.21.15.163:0/932251915' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:50:34.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:34 smithi163 bash[75425]: cluster 2023-10-09T09:50:33.007777+0000 mon.a (mon.0) 2452 : cluster 0 osdmap e1272: 8 total, 8 up, 6 in 2023-10-09T09:50:34.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:34 smithi163 bash[75425]: cluster 2023-10-09T09:50:33.008627+0000 osd.6 (osd.6) 556 : cluster 3 failed to encode map e1272 with expected crc 2023-10-09T09:50:34.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:34 smithi163 bash[78107]: cluster 2023-10-09T09:50:32.176972+0000 mgr.y (mgr.34107) 1337 : cluster 0 pgmap v2203: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:34.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:34 smithi163 bash[78107]: cluster 2023-10-09T09:50:33.006169+0000 osd.5 (osd.5) 862 : cluster 3 failed to encode map e1272 with expected crc 2023-10-09T09:50:34.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:34 smithi163 bash[78107]: audit 2023-10-09T09:50:33.007728+0000 mon.a (mon.0) 2451 : audit 1 from='client.? 172.21.15.163:0/932251915' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:50:34.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:34 smithi163 bash[78107]: cluster 2023-10-09T09:50:33.007777+0000 mon.a (mon.0) 2452 : cluster 0 osdmap e1272: 8 total, 8 up, 6 in 2023-10-09T09:50:34.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:34 smithi163 bash[78107]: cluster 2023-10-09T09:50:33.008627+0000 osd.6 (osd.6) 556 : cluster 3 failed to encode map e1272 with expected crc 2023-10-09T09:50:34.233 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:34.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:34.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:34 smithi181 bash[59345]: cluster 2023-10-09T09:50:32.176972+0000 mgr.y (mgr.34107) 1337 : cluster 0 pgmap v2203: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:34 smithi181 bash[59345]: cluster 2023-10-09T09:50:33.006169+0000 osd.5 (osd.5) 862 : cluster 3 failed to encode map e1272 with expected crc 2023-10-09T09:50:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:34 smithi181 bash[59345]: audit 2023-10-09T09:50:33.007728+0000 mon.a (mon.0) 2451 : audit 1 from='client.? 172.21.15.163:0/932251915' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:50:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:34 smithi181 bash[59345]: cluster 2023-10-09T09:50:33.007777+0000 mon.a (mon.0) 2452 : cluster 0 osdmap e1272: 8 total, 8 up, 6 in 2023-10-09T09:50:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:34 smithi181 bash[59345]: cluster 2023-10-09T09:50:33.008627+0000 osd.6 (osd.6) 556 : cluster 3 failed to encode map e1272 with expected crc 2023-10-09T09:50:34.336 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:34.362 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:34.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:34.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:34.465 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:34.489 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:34.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:34.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:34.592 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:34.613 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:34.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:34.716 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:34.749 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:34.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:34.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:34.853 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:34.874 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:34.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:34.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:34.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T09:50:34.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:35.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:35.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:35.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:35.109 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:35.129 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:35.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:35.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:35.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:35.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:35.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:35.365 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:35.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:35.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:35.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:35.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:35.523 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:35.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:35.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:35.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:35.650 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:35.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:35.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:35.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:35.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:35.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:35.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:35.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:35.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:35.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:35.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:36.018 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:36.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:36.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:36.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:36.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:36.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:36 smithi181 bash[59345]: cluster 2023-10-09T09:50:34.177581+0000 mgr.y (mgr.34107) 1338 : cluster 0 pgmap v2205: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:36.194 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:36.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:36.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:36.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:36 smithi163 bash[75425]: cluster 2023-10-09T09:50:34.177581+0000 mgr.y (mgr.34107) 1338 : cluster 0 pgmap v2205: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:36.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:36 smithi163 bash[78107]: cluster 2023-10-09T09:50:34.177581+0000 mgr.y (mgr.34107) 1338 : cluster 0 pgmap v2205: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:36.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:36.329 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:36.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:36.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:36.466 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:36.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:36.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:36.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:36.602 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:36.628 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:36.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:36.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:36.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:36.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:36.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:36.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:37.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:37.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:37.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:37.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:37.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:37.235 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:37.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:37.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:37.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:37 smithi163 bash[75425]: audit 2023-10-09T09:50:36.171920+0000 mon.a (mon.0) 2453 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:37.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:37 smithi163 bash[75425]: cluster 2023-10-09T09:50:36.178163+0000 mgr.y (mgr.34107) 1339 : cluster 0 pgmap v2206: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:37.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:37 smithi163 bash[75425]: audit 2023-10-09T09:50:36.285792+0000 mon.a (mon.0) 2454 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:37.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:37 smithi163 bash[75425]: audit 2023-10-09T09:50:36.307534+0000 mon.a (mon.0) 2455 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:37.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:37 smithi163 bash[78107]: audit 2023-10-09T09:50:36.171920+0000 mon.a (mon.0) 2453 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:37.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:37 smithi163 bash[78107]: cluster 2023-10-09T09:50:36.178163+0000 mgr.y (mgr.34107) 1339 : cluster 0 pgmap v2206: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:37.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:37 smithi163 bash[78107]: audit 2023-10-09T09:50:36.285792+0000 mon.a (mon.0) 2454 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:37.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:37 smithi163 bash[78107]: audit 2023-10-09T09:50:36.307534+0000 mon.a (mon.0) 2455 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:37.338 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T09:50:37.338 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:50:37.339 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:37.366 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:37.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:37.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:37.421 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:37.475 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:37.499 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:37.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:37.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:37 smithi181 bash[59345]: audit 2023-10-09T09:50:36.171920+0000 mon.a (mon.0) 2453 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:37.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:37 smithi181 bash[59345]: cluster 2023-10-09T09:50:36.178163+0000 mgr.y (mgr.34107) 1339 : cluster 0 pgmap v2206: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:37.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:37 smithi181 bash[59345]: audit 2023-10-09T09:50:36.285792+0000 mon.a (mon.0) 2454 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:37.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:37 smithi181 bash[59345]: audit 2023-10-09T09:50:36.307534+0000 mon.a (mon.0) 2455 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:37.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:37.696 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:37.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:37.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:37.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:37.828 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:37.849 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:37.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:37.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:37.952 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:37.977 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:37.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:37.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:38.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:38.112 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:38.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:38.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:38.215 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:38.381 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:38.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:38.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:38.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:38.512 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:38.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:38.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:38.616 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:38.649 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:38.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:38.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:38.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:38.779 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:38.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:38.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:38.882 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:38.917 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:38.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:38.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:39.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:39.051 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:39.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:39.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:39.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:39.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:39.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:39.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:39.286 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:39.318 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:39.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:39.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:39.422 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:39.425 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:39 smithi163 bash[78107]: cluster 2023-10-09T09:50:38.178987+0000 mgr.y (mgr.34107) 1340 : cluster 0 pgmap v2207: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:39.425 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:39 smithi163 bash[75425]: cluster 2023-10-09T09:50:38.178987+0000 mgr.y (mgr.34107) 1340 : cluster 0 pgmap v2207: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:39.449 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:39.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:39.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:39.553 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:39.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:39 smithi181 bash[59345]: cluster 2023-10-09T09:50:38.178987+0000 mgr.y (mgr.34107) 1340 : cluster 0 pgmap v2207: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:39.585 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:39.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:39.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:39.689 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:39.698 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:50:39.720 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:39.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:39.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:39.724 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:50:39.744 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:50:39.774 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:50:39.802 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:50:39.824 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:39.830 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:50:39.854 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:39.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:39.858 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:50:39.898 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:50:39.903 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:50:39.903 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:50:39.903 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:50:39.903 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:50:39.904 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:50:39.904 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:50:39.904 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:50:39.904 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:50:39.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:39.990 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:39.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:39.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:40.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:40.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:40.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:40.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:40.219 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:40.251 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:40.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:40.355 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:40.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:40 smithi163 bash[78107]: audit 2023-10-09T09:50:39.674006+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.163:0/3992056421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:50:40.365 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:40 smithi163 bash[75425]: audit 2023-10-09T09:50:39.674006+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.163:0/3992056421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:50:40.378 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:40.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:40.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:40.494 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:40 smithi181 bash[59345]: audit 2023-10-09T09:50:39.674006+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.163:0/3992056421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:50:40.516 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:40.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:40.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:40.621 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:40.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:40.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:40.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:40.760 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:40.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:40.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:40.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:40.897 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:40.928 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:40.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:40.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:41.032 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:41.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:41.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:41.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:41.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:41.185 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:41.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:41.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:41.258 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:50:41.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:41.321 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:41.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:41.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:41.436 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[78107]: cluster 2023-10-09T09:50:40.179501+0000 mgr.y (mgr.34107) 1341 : cluster 0 pgmap v2208: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:41.436 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[78107]: audit 2023-10-09T09:50:40.254276+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.163:0/2004587924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:50:41.436 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[78107]: audit 2023-10-09T09:50:40.254771+0000 mon.a (mon.0) 2456 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:50:41.436 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[78107]: audit 2023-10-09T09:50:40.400012+0000 mon.a (mon.0) 2457 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:41.436 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[78107]: audit 2023-10-09T09:50:41.003733+0000 mon.a (mon.0) 2458 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:41.436 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[78107]: audit 2023-10-09T09:50:41.012267+0000 mon.a (mon.0) 2459 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:41.436 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[75425]: cluster 2023-10-09T09:50:40.179501+0000 mgr.y (mgr.34107) 1341 : cluster 0 pgmap v2208: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:41.437 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[75425]: audit 2023-10-09T09:50:40.254276+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.163:0/2004587924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:50:41.437 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[75425]: audit 2023-10-09T09:50:40.254771+0000 mon.a (mon.0) 2456 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:50:41.437 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[75425]: audit 2023-10-09T09:50:40.400012+0000 mon.a (mon.0) 2457 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:41.437 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[75425]: audit 2023-10-09T09:50:41.003733+0000 mon.a (mon.0) 2458 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:41.437 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:41 smithi163 bash[75425]: audit 2023-10-09T09:50:41.012267+0000 mon.a (mon.0) 2459 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:41.447 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:41.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:41.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:41.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:41 smithi181 bash[59345]: cluster 2023-10-09T09:50:40.179501+0000 mgr.y (mgr.34107) 1341 : cluster 0 pgmap v2208: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:41 smithi181 bash[59345]: audit 2023-10-09T09:50:40.254276+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.163:0/2004587924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:50:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:41 smithi181 bash[59345]: audit 2023-10-09T09:50:40.254771+0000 mon.a (mon.0) 2456 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:50:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:41 smithi181 bash[59345]: audit 2023-10-09T09:50:40.400012+0000 mon.a (mon.0) 2457 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:41 smithi181 bash[59345]: audit 2023-10-09T09:50:41.003733+0000 mon.a (mon.0) 2458 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:41 smithi181 bash[59345]: audit 2023-10-09T09:50:41.012267+0000 mon.a (mon.0) 2459 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:41.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:41.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:41.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:41.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:41.712 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:41.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:41.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:41.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:41.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:41.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:41.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:41.951 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:41.983 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:41.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:41.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:42.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:42.118 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:42.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:42.222 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:42.252 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:42.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:42.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:42.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:42.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:42.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:42.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:42.488 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:42.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:42 smithi181 bash[59345]: cluster 2023-10-09T09:50:41.242931+0000 mon.a (mon.0) 2460 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:50:42.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:42 smithi181 bash[59345]: cluster 2023-10-09T09:50:41.250633+0000 osd.5 (osd.5) 863 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:42 smithi181 bash[59345]: audit 2023-10-09T09:50:41.252406+0000 mon.a (mon.0) 2461 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:50:42.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:42 smithi181 bash[59345]: cluster 2023-10-09T09:50:41.252479+0000 mon.a (mon.0) 2462 : cluster 0 osdmap e1273: 8 total, 8 up, 6 in 2023-10-09T09:50:42.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:42 smithi181 bash[59345]: audit 2023-10-09T09:50:41.483889+0000 mon.a (mon.0) 2463 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:42.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:42 smithi181 bash[59345]: audit 2023-10-09T09:50:41.484967+0000 mon.a (mon.0) 2464 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:50:42.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:42 smithi181 bash[59345]: audit 2023-10-09T09:50:41.492493+0000 mon.a (mon.0) 2465 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:42.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:42 smithi181 bash[59345]: cluster 2023-10-09T09:50:41.574741+0000 osd.7 (osd.7) 613 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:42 smithi181 bash[59345]: cluster 2023-10-09T09:50:41.574802+0000 osd.6 (osd.6) 557 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.501 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:42 smithi181 bash[59345]: cluster 2023-10-09T09:50:41.574901+0000 osd.4 (osd.4) 666 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.517 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:42.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:42.622 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:42.635 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[75425]: cluster 2023-10-09T09:50:41.242931+0000 mon.a (mon.0) 2460 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:50:42.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[75425]: cluster 2023-10-09T09:50:41.250633+0000 osd.5 (osd.5) 863 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[75425]: audit 2023-10-09T09:50:41.252406+0000 mon.a (mon.0) 2461 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:50:42.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[75425]: cluster 2023-10-09T09:50:41.252479+0000 mon.a (mon.0) 2462 : cluster 0 osdmap e1273: 8 total, 8 up, 6 in 2023-10-09T09:50:42.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[75425]: audit 2023-10-09T09:50:41.483889+0000 mon.a (mon.0) 2463 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:42.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[75425]: audit 2023-10-09T09:50:41.484967+0000 mon.a (mon.0) 2464 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:50:42.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[75425]: audit 2023-10-09T09:50:41.492493+0000 mon.a (mon.0) 2465 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:42.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[75425]: cluster 2023-10-09T09:50:41.574741+0000 osd.7 (osd.7) 613 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[75425]: cluster 2023-10-09T09:50:41.574802+0000 osd.6 (osd.6) 557 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[75425]: cluster 2023-10-09T09:50:41.574901+0000 osd.4 (osd.4) 666 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[78107]: cluster 2023-10-09T09:50:41.242931+0000 mon.a (mon.0) 2460 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:50:42.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[78107]: cluster 2023-10-09T09:50:41.250633+0000 osd.5 (osd.5) 863 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[78107]: audit 2023-10-09T09:50:41.252406+0000 mon.a (mon.0) 2461 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:50:42.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[78107]: cluster 2023-10-09T09:50:41.252479+0000 mon.a (mon.0) 2462 : cluster 0 osdmap e1273: 8 total, 8 up, 6 in 2023-10-09T09:50:42.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[78107]: audit 2023-10-09T09:50:41.483889+0000 mon.a (mon.0) 2463 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:42.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[78107]: audit 2023-10-09T09:50:41.484967+0000 mon.a (mon.0) 2464 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:50:42.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[78107]: audit 2023-10-09T09:50:41.492493+0000 mon.a (mon.0) 2465 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:42.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[78107]: cluster 2023-10-09T09:50:41.574741+0000 osd.7 (osd.7) 613 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[78107]: cluster 2023-10-09T09:50:41.574802+0000 osd.6 (osd.6) 557 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:42 smithi163 bash[78107]: cluster 2023-10-09T09:50:41.574901+0000 osd.4 (osd.4) 666 : cluster 3 failed to encode map e1273 with expected crc 2023-10-09T09:50:42.651 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:42.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:42.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:42.754 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:42.787 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:42.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:42.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:42.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:42.914 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:42.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:42.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:43.017 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:43.047 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:43.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:43.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:43.151 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:43.177 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:43.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:43.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:43.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:43.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:43.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:43.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:43.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:43.432 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:50:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:50:43] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.33.4" 2023-10-09T09:50:43.447 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:43.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:43.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:43.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:43.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:43 smithi181 bash[59345]: cluster 2023-10-09T09:50:42.180395+0000 mgr.y (mgr.34107) 1342 : cluster 0 pgmap v2210: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:43.582 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:43.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:43.671 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:50:43.689 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:43.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:43 smithi163 bash[78107]: cluster 2023-10-09T09:50:42.180395+0000 mgr.y (mgr.34107) 1342 : cluster 0 pgmap v2210: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:43 smithi163 bash[75425]: cluster 2023-10-09T09:50:42.180395+0000 mgr.y (mgr.34107) 1342 : cluster 0 pgmap v2210: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:43.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:43.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:43.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:43.835 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:43.846 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:43.879 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:43.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:43.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:43.983 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:44.010 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:44.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:44.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:44.075 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:44.113 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:44.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:44.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:44.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:44.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:44.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:44.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:44.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:44.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:44.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:44.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:44.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:44.514 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_6' 2023-10-09T09:50:44.533 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:44.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:44.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:44.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:44 smithi181 bash[59345]: audit 2023-10-09T09:50:44.226972+0000 mon.a (mon.0) 2466 : audit 1 from='client.? 172.21.15.163:0/643297211' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:44.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:44.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:44 smithi163 bash[75425]: audit 2023-10-09T09:50:44.226972+0000 mon.a (mon.0) 2466 : audit 1 from='client.? 172.21.15.163:0/643297211' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:44.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:44 smithi163 bash[78107]: audit 2023-10-09T09:50:44.226972+0000 mon.a (mon.0) 2466 : audit 1 from='client.? 172.21.15.163:0/643297211' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:44.673 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:44.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:44.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:44.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:44.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:44.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:44.915 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:44.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:44.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:44.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:45.050 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:45.081 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:45.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:45.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:45.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:45.218 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:45.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:45.322 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:45.349 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:45.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:45.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:45.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:45.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:45.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:45.587 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:45.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:50:45.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:45 smithi163 bash[75425]: cluster 2023-10-09T09:50:44.181145+0000 mgr.y (mgr.34107) 1343 : cluster 0 pgmap v2211: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:45.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:45 smithi163 bash[75425]: cluster 2023-10-09T09:50:44.505980+0000 osd.5 (osd.5) 864 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:45.606 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:45 smithi163 bash[75425]: audit 2023-10-09T09:50:44.509418+0000 mon.a (mon.0) 2467 : audit 1 from='client.? 172.21.15.163:0/643297211' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:50:45.606 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:45 smithi163 bash[75425]: cluster 2023-10-09T09:50:44.509537+0000 mon.a (mon.0) 2468 : cluster 0 osdmap e1274: 8 total, 8 up, 6 in 2023-10-09T09:50:45.606 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:45 smithi163 bash[75425]: cluster 2023-10-09T09:50:45.161703+0000 osd.6 (osd.6) 558 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:45.606 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:45 smithi163 bash[78107]: cluster 2023-10-09T09:50:44.181145+0000 mgr.y (mgr.34107) 1343 : cluster 0 pgmap v2211: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:45.606 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:45 smithi163 bash[78107]: cluster 2023-10-09T09:50:44.505980+0000 osd.5 (osd.5) 864 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:45.606 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:45 smithi163 bash[78107]: audit 2023-10-09T09:50:44.509418+0000 mon.a (mon.0) 2467 : audit 1 from='client.? 172.21.15.163:0/643297211' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:50:45.606 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:45 smithi163 bash[78107]: cluster 2023-10-09T09:50:44.509537+0000 mon.a (mon.0) 2468 : cluster 0 osdmap e1274: 8 total, 8 up, 6 in 2023-10-09T09:50:45.606 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:45 smithi163 bash[78107]: cluster 2023-10-09T09:50:45.161703+0000 osd.6 (osd.6) 558 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:45.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:45.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:45.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:45.726 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:45.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:45.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:45.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:45 smithi181 bash[59345]: cluster 2023-10-09T09:50:44.181145+0000 mgr.y (mgr.34107) 1343 : cluster 0 pgmap v2211: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:45 smithi181 bash[59345]: cluster 2023-10-09T09:50:44.505980+0000 osd.5 (osd.5) 864 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:45 smithi181 bash[59345]: audit 2023-10-09T09:50:44.509418+0000 mon.a (mon.0) 2467 : audit 1 from='client.? 172.21.15.163:0/643297211' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:50:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:45 smithi181 bash[59345]: cluster 2023-10-09T09:50:44.509537+0000 mon.a (mon.0) 2468 : cluster 0 osdmap e1274: 8 total, 8 up, 6 in 2023-10-09T09:50:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:45 smithi181 bash[59345]: cluster 2023-10-09T09:50:45.161703+0000 osd.6 (osd.6) 558 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:45.848 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:45.871 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:45.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:45.974 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:46.007 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:46.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:46.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:46.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:46.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:46.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:46.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:46.242 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:46.278 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:46.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:46.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:46.409 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:46.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:46.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:46.512 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:46.539 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:46.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:46.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:46.643 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:46.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:46.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:46.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:46.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:46.802 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:46.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:46.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:46.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:46.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:46.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:46.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:47.032 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:47.065 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:47.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:47.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:47.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:47.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:46 smithi163 bash[78107]: cluster 2023-10-09T09:50:45.161627+0000 osd.4 (osd.4) 667 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:47.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:46 smithi163 bash[78107]: cluster 2023-10-09T09:50:45.161884+0000 osd.7 (osd.7) 614 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:47.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:46 smithi163 bash[75425]: cluster 2023-10-09T09:50:45.161627+0000 osd.4 (osd.4) 667 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:47.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:46 smithi163 bash[75425]: cluster 2023-10-09T09:50:45.161884+0000 osd.7 (osd.7) 614 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:47.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:47.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:47.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:47.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:47.310 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:46 smithi181 bash[59345]: cluster 2023-10-09T09:50:45.161627+0000 osd.4 (osd.4) 667 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:47.310 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:46 smithi181 bash[59345]: cluster 2023-10-09T09:50:45.161884+0000 osd.7 (osd.7) 614 : cluster 3 failed to encode map e1274 with expected crc 2023-10-09T09:50:47.332 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:47.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:47.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:47.436 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:47.463 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:47.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:47.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:47.566 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:47.601 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:47.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:47.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:47.706 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:47.741 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:47.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:47.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:47.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:47.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:47.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:47.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:47.980 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:48.002 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:47 smithi163 bash[78107]: cluster 2023-10-09T09:50:46.181897+0000 mgr.y (mgr.34107) 1344 : cluster 0 pgmap v2213: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:48.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:47 smithi163 bash[75425]: cluster 2023-10-09T09:50:46.181897+0000 mgr.y (mgr.34107) 1344 : cluster 0 pgmap v2213: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:48.010 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:48.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:48.114 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:48.125 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:47 smithi181 bash[59345]: cluster 2023-10-09T09:50:46.181897+0000 mgr.y (mgr.34107) 1344 : cluster 0 pgmap v2213: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:48.145 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:48.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:48.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:48.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:48.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:48.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:48.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:48.357 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2023-10-09T09:50:48.359 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:50:48.359 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:50:48.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:48.402 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T09:50:48.402 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:50:48.402 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_218041 2023-10-09T09:50:48.403 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:50:48.403 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:50:48.424 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:48.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:48.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:48.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:48.557 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:48.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:48.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:48.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:48.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:48.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:48.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:48.794 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:48.822 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:48.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:48.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:48.930 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:48.955 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:48.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:48.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:49.058 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:49.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:49.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:49.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:49.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:49.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:49.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:49.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:49.317 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:49.351 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:49.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:49.403 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 3030 3014 188.365 188.375 0.00465728 0.00529343 2023-10-09T09:50:49.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:49.483 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:49.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:49.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:49.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:49.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:49.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:49.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:49.715 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:49.739 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:49.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:49.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:49.843 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:49.867 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:49.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:49.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:49.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:49.980 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:49 smithi163 bash[78107]: cluster 2023-10-09T09:50:48.182786+0000 mgr.y (mgr.34107) 1345 : cluster 0 pgmap v2214: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:49.980 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:49 smithi163 bash[75425]: cluster 2023-10-09T09:50:48.182786+0000 mgr.y (mgr.34107) 1345 : cluster 0 pgmap v2214: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:49.997 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:49.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:49.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:50.100 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:50.129 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:50.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:50.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:50.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:50.260 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:50.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:50.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:49 smithi181 bash[59345]: cluster 2023-10-09T09:50:48.182786+0000 mgr.y (mgr.34107) 1345 : cluster 0 pgmap v2214: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:50:50.363 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:50.390 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:50.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:50.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:50.403 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 5758 5742 179.424 170.5 0.00167827 0.00550908 2023-10-09T09:50:50.471 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:50.473 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:50:50.493 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:50.522 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:50.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:50.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:50.626 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:50.656 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:50.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:50.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:50.762 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:50.789 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:50.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:50.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:50.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:50.918 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:50.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:50.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:51.022 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:51.050 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:51.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:51.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:51.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:51.180 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:51.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:51.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:51.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:51.305 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:51.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:51.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:51.403 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 8287 8271 172.298 158.062 0.0107327 0.00579665 2023-10-09T09:50:51.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:51.434 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:51.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:51.537 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:51.560 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:51.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:51.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:51.663 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:51.694 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:51.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:51.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:51.798 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:51.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:51.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:51.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:51.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:51.988 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:51.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:51.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:52.092 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:52.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:52.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:52.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:52.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:51 smithi163 bash[78107]: cluster 2023-10-09T09:50:50.183398+0000 mgr.y (mgr.34107) 1346 : cluster 0 pgmap v2215: 89 pgs: 89 active+clean; 4.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 502 KiB/s wr, 7 op/s 2023-10-09T09:50:52.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:51 smithi163 bash[75425]: cluster 2023-10-09T09:50:50.183398+0000 mgr.y (mgr.34107) 1346 : cluster 0 pgmap v2215: 89 pgs: 89 active+clean; 4.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 502 KiB/s wr, 7 op/s 2023-10-09T09:50:52.222 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:52.232 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:51 smithi181 bash[59345]: cluster 2023-10-09T09:50:50.183398+0000 mgr.y (mgr.34107) 1346 : cluster 0 pgmap v2215: 89 pgs: 89 active+clean; 4.9 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 502 KiB/s wr, 7 op/s 2023-10-09T09:50:52.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:52.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:52.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:52.352 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:52.378 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:52.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:52.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:52.403 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 10694 10678 166.829 150.438 0.00283473 0.00598471 2023-10-09T09:50:52.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:52.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:52.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:52.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:52.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:52.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:52.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:52.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:52.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:52.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:52.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:52.881 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:52.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:52.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:52.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:53.008 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:53.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:53.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:53.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:53.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:53.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:53.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:53.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:53.292 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:53.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:53.396 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:53.403 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 13040 13024 162.786 146.625 0.0084715 0.00613492 2023-10-09T09:50:53.427 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:53.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:53.530 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:53.538 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:50:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:50:53] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.33.4" 2023-10-09T09:50:53.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:53.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:53.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:53.656 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:53.679 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:53.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:53.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:53.782 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:53.805 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:53.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:53.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:53.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:53.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:53.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:53.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:54.039 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:54.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:54.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:54.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:54.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:54.191 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:54.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:54.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:54.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:53 smithi163 bash[75425]: cluster 2023-10-09T09:50:52.184120+0000 mgr.y (mgr.34107) 1347 : cluster 0 pgmap v2216: 89 pgs: 89 active+clean; 481 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 48 MiB/s wr, 767 op/s 2023-10-09T09:50:54.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:53 smithi163 bash[78107]: cluster 2023-10-09T09:50:52.184120+0000 mgr.y (mgr.34107) 1347 : cluster 0 pgmap v2216: 89 pgs: 89 active+clean; 481 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 48 MiB/s wr, 767 op/s 2023-10-09T09:50:54.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:53 smithi181 bash[59345]: cluster 2023-10-09T09:50:52.184120+0000 mgr.y (mgr.34107) 1347 : cluster 0 pgmap v2216: 89 pgs: 89 active+clean; 481 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 48 MiB/s wr, 767 op/s 2023-10-09T09:50:54.327 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:54.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:54.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:54.403 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 15317 15301 159.371 142.312 0.00489862 0.00626697 2023-10-09T09:50:54.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:54.457 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:54.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:54.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:54.562 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:54.586 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:54.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:54.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:54.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:54.718 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:54.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:54.822 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:54.849 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:54.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:54.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:54.953 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:54.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:54.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:54.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:55.085 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:55.115 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:55.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:55.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:55.218 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:55.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:55.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:55.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:55.350 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:55.376 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:55.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:55.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:55.403 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 16 17535 17519 156.406 138.625 0.00845065 0.00638567 2023-10-09T09:50:55.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:55.506 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:55.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:55.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:55.610 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:55.638 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:55.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:55.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:55.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:55.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:55.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:55.876 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:55.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:55.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:55.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:56.008 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:56.035 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:56.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:56.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:56.139 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:56.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:55 smithi181 bash[59345]: cluster 2023-10-09T09:50:54.184561+0000 mgr.y (mgr.34107) 1348 : cluster 0 pgmap v2217: 89 pgs: 89 active+clean; 532 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T09:50:56.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:55 smithi181 bash[59345]: audit 2023-10-09T09:50:55.409132+0000 mon.a (mon.0) 2469 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:56.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:55 smithi181 bash[59345]: audit 2023-10-09T09:50:55.411625+0000 mon.a (mon.0) 2470 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:56.170 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:56.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:56.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:56.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:55 smithi163 bash[75425]: cluster 2023-10-09T09:50:54.184561+0000 mgr.y (mgr.34107) 1348 : cluster 0 pgmap v2217: 89 pgs: 89 active+clean; 532 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T09:50:56.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:55 smithi163 bash[75425]: audit 2023-10-09T09:50:55.409132+0000 mon.a (mon.0) 2469 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:56.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:55 smithi163 bash[75425]: audit 2023-10-09T09:50:55.411625+0000 mon.a (mon.0) 2470 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:55 smithi163 bash[78107]: cluster 2023-10-09T09:50:54.184561+0000 mgr.y (mgr.34107) 1348 : cluster 0 pgmap v2217: 89 pgs: 89 active+clean; 532 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T09:50:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:55 smithi163 bash[78107]: audit 2023-10-09T09:50:55.409132+0000 mon.a (mon.0) 2469 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:50:56.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:55 smithi163 bash[78107]: audit 2023-10-09T09:50:55.411625+0000 mon.a (mon.0) 2470 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:56.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:56.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:56.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:56.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:56.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 19052 19036 148.706 94.8125 0.00911112 0.00671868 2023-10-09T09:50:56.408 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:56.440 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:56.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:56.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:56.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:56.569 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:56.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:56.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:56.701 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:56.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:56.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:56.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:56.839 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:56.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:56.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:56.943 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:56.976 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:56.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:56.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:57.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:57.111 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:57.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:57.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:57.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:57.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:57.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:57.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:57.350 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:57.377 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:57.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:57.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:57.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 21097 21081 146.383 127.812 0.0151143 0.00682454 2023-10-09T09:50:57.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:57.513 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:57.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:57.616 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:57.641 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:50:57.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:57.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:50:57.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:57.776 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:57.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:57.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:57.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:57.909 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:57.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:58.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:58.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:57 smithi181 bash[59345]: cluster 2023-10-09T09:50:56.185438+0000 mgr.y (mgr.34107) 1349 : cluster 0 pgmap v2218: 89 pgs: 89 active+clean; 893 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T09:50:58.041 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:50:58.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:58.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:50:58.146 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:58.174 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:58.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:57 smithi163 bash[75425]: cluster 2023-10-09T09:50:56.185438+0000 mgr.y (mgr.34107) 1349 : cluster 0 pgmap v2218: 89 pgs: 89 active+clean; 893 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T09:50:58.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:57 smithi163 bash[78107]: cluster 2023-10-09T09:50:56.185438+0000 mgr.y (mgr.34107) 1349 : cluster 0 pgmap v2218: 89 pgs: 89 active+clean; 893 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T09:50:58.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:58.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:58.178 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:50:58.178 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:50:58.278 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:58.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:58.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:58.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:58.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 23035 23019 143.856 121.125 0.00708288 0.00694595 2023-10-09T09:50:58.407 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:58.437 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:58.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:58.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:58.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:50:58.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:58.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:50:58.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:58.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:58.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:58.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:58.801 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:58.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:58.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:58.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:58.933 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:58.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:58.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:58.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:59.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:59.090 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:59.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:59.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:59.224 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:50:59.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:59.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:50:59.328 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:59.354 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:59.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:59.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:59.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 24743 24727 140.482 106.75 0.0113368 0.0071124 2023-10-09T09:50:59.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:59.501 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:50:59.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:59.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:50:59.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:59.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:50:59.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:59.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:50:59.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:59.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:50:59.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:59.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:50:59.811 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:50:59.813 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:50:59.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:59.891 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:50:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:50:59.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:50:59.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:00.025 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:00.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:00.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:00.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:00.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:00.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:00.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:50:59 smithi163 bash[75425]: cluster 2023-10-09T09:50:58.187422+0000 mgr.y (mgr.34107) 1350 : cluster 0 pgmap v2219: 89 pgs: 89 active+clean; 1.2 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T09:51:00.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:50:59 smithi163 bash[78107]: cluster 2023-10-09T09:50:58.187422+0000 mgr.y (mgr.34107) 1350 : cluster 0 pgmap v2219: 89 pgs: 89 active+clean; 1.2 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T09:51:00.261 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:00.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:50:59 smithi181 bash[59345]: cluster 2023-10-09T09:50:58.187422+0000 mgr.y (mgr.34107) 1350 : cluster 0 pgmap v2219: 89 pgs: 89 active+clean; 1.2 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T09:51:00.292 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:00.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:00.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:00.396 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:00.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 26522 26506 138.04 111.188 0.0073017 0.00723797 2023-10-09T09:51:00.423 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:00.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:00.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:00.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:00.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:00.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:00.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:00.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:00.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:00.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:00.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:00.797 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:00.829 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:00.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:00.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:00.933 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:00.959 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:00.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:00.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:01.062 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:01.087 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:01.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:01.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:01.191 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:01.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:00 smithi163 bash[75425]: audit 2023-10-09T09:50:59.788900+0000 mgr.y (mgr.34107) 1351 : audit 0 from='client.45761 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:51:01.201 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:00 smithi163 bash[78107]: audit 2023-10-09T09:50:59.788900+0000 mgr.y (mgr.34107) 1351 : audit 0 from='client.45761 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:51:01.216 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:01.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:01.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:00 smithi181 bash[59345]: audit 2023-10-09T09:50:59.788900+0000 mgr.y (mgr.34107) 1351 : audit 0 from='client.45761 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:51:01.320 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:01.345 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:01.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:01.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:01.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 28334 28318 136.132 113.25 0.00422927 0.00733837 2023-10-09T09:51:01.448 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:01.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:01.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:01.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:01.587 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:01.617 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:01.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:01.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:01.720 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:01.747 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:01.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:01.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:01.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:01.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:01.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:01.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:01.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:02.012 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:02.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:02.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:02.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:02.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:01 smithi163 bash[75425]: cluster 2023-10-09T09:51:00.188805+0000 mgr.y (mgr.34107) 1352 : cluster 0 pgmap v2220: 89 pgs: 89 active+clean; 1.2 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T09:51:02.127 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:01 smithi163 bash[78107]: cluster 2023-10-09T09:51:00.188805+0000 mgr.y (mgr.34107) 1352 : cluster 0 pgmap v2220: 89 pgs: 89 active+clean; 1.2 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T09:51:02.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:02.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:02.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:02.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:01 smithi181 bash[59345]: cluster 2023-10-09T09:51:00.188805+0000 mgr.y (mgr.34107) 1352 : cluster 0 pgmap v2220: 89 pgs: 89 active+clean; 1.2 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T09:51:02.268 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:02.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:02.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:02.372 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:02.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 29886 29870 133.336 97 0.0055517 0.00749321 2023-10-09T09:51:02.412 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:02.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:02.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:02.518 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:02.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:02.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:02.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:02.658 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:02.688 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:02.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:02.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:02.793 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:02.823 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:02.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:02.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:02.927 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:02.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:02.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:03.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:03.088 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:03.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:03.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:03.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:03.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:03.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:03.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:03.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:03.358 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:03.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:03.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:03.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 31082 31066 129.43 74.75 0.0405376 0.00770647 2023-10-09T09:51:03.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:03.485 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:03.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:03.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:03.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:03.617 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:03.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:03.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:51:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:51:03] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.33.4" 2023-10-09T09:51:03.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:03.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:03.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:03.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:03.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:03.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:03.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:03.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:03.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:04.019 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:04.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:04.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:04.125 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:04.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:03 smithi163 bash[75425]: cluster 2023-10-09T09:51:02.189570+0000 mgr.y (mgr.34107) 1353 : cluster 0 pgmap v2221: 89 pgs: 89 active+clean; 1.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 145 MiB/s wr, 2.33k op/s 2023-10-09T09:51:04.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:03 smithi163 bash[78107]: cluster 2023-10-09T09:51:02.189570+0000 mgr.y (mgr.34107) 1353 : cluster 0 pgmap v2221: 89 pgs: 89 active+clean; 1.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 145 MiB/s wr, 2.33k op/s 2023-10-09T09:51:04.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:04.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:04.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:04.200 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T09:51:04.258 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:04.286 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:04.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:04.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:03 smithi181 bash[59345]: cluster 2023-10-09T09:51:02.189570+0000 mgr.y (mgr.34107) 1353 : cluster 0 pgmap v2221: 89 pgs: 89 active+clean; 1.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 145 MiB/s wr, 2.33k op/s 2023-10-09T09:51:04.390 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:04.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 32585 32569 127.211 93.9375 0.00900534 0.00785306 2023-10-09T09:51:04.417 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:04.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:04.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:04.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:04.554 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:04.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:04.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:04.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:04.685 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:04.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:04.789 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:04.819 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:04.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:04.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:04.922 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:04.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:04.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:04.990 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2219,"stamp":"2023-10-09T09:50:58.186988+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1240930208,"num_objects":18950,"num_object_clones":0,"num_object_copies":56850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18950,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":19186,"num_write_kb":1213709,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":592828,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":19120,"ondisk_log_size":19120,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7641032,"kb_used_data":3363736,"kb_used_omap":607,"kb_used_meta":4276640,"kb_avail":554829880,"statfs":{"total":575970213888,"available":568145797120,"internally_reserved":0,"allocated":3444465664,"data_stored":3439044618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":621657,"internal_metadata":4379280295},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1240334336,"num_objects":18926,"num_object_clones":0,"num_object_copies":56778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18926,"num_write_kb":1211264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005536"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":52,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576430+0000","last_change":"2023-10-09T09:48:07.991289+0000","last_active":"2023-10-09T09:50:41.576430+0000","last_peered":"2023-10-09T09:50:41.576430+0000","last_clean":"2023-10-09T09:50:41.576430+0000","last_became_active":"2023-10-09T09:48:07.990686+0000","last_became_peered":"2023-10-09T09:48:07.990686+0000","last_unstale":"2023-10-09T09:50:41.576430+0000","last_undegraded":"2023-10-09T09:50:41.576430+0000","last_fullsized":"2023-10-09T09:50:41.576430+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:24.421374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":155,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.577045+0000","last_change":"2023-10-09T09:48:07.999947+0000","last_active":"2023-10-09T09:50:41.577045+0000","last_peered":"2023-10-09T09:50:41.577045+0000","last_clean":"2023-10-09T09:50:41.577045+0000","last_became_active":"2023-10-09T09:48:07.999704+0000","last_became_peered":"2023-10-09T09:48:07.999704+0000","last_unstale":"2023-10-09T09:50:41.577045+0000","last_undegraded":"2023-10-09T09:50:41.577045+0000","last_fullsized":"2023-10-09T09:50:41.577045+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:14.832042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1224,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575695+0000","last_change":"2023-10-09T09:48:07.998214+0000","last_active":"2023-10-09T09:50:41.575695+0000","last_peered":"2023-10-09T09:50:41.575695+0000","last_clean":"2023-10-09T09:50:41.575695+0000","last_became_active":"2023-10-09T09:48:07.997544+0000","last_became_peered":"2023-10-09T09:48:07.997544+0000","last_unstale":"2023-10-09T09:50:41.575695+0000","last_undegraded":"2023-10-09T09:50:41.575695+0000","last_fullsized":"2023-10-09T09:50:41.575695+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.304000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1233,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.536599+0000","last_change":"2023-10-09T09:48:07.980358+0000","last_active":"2023-10-09T09:50:41.536599+0000","last_peered":"2023-10-09T09:50:41.536599+0000","last_clean":"2023-10-09T09:50:41.536599+0000","last_became_active":"2023-10-09T09:48:07.979229+0000","last_became_peered":"2023-10-09T09:48:07.979229+0000","last_unstale":"2023-10-09T09:50:41.536599+0000","last_undegraded":"2023-10-09T09:50:41.536599+0000","last_fullsized":"2023-10-09T09:50:41.536599+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:31.456347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":767,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.535895+0000","last_change":"2023-10-09T09:48:07.969728+0000","last_active":"2023-10-09T09:50:41.535895+0000","last_peered":"2023-10-09T09:50:41.535895+0000","last_clean":"2023-10-09T09:50:41.535895+0000","last_became_active":"2023-10-09T09:48:07.968773+0000","last_became_peered":"2023-10-09T09:48:07.968773+0000","last_unstale":"2023-10-09T09:50:41.535895+0000","last_undegraded":"2023-10-09T09:50:41.535895+0000","last_fullsized":"2023-10-09T09:50:41.535895+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:39.023317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1238,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.581074+0000","last_change":"2023-10-09T09:48:07.988928+0000","last_active":"2023-10-09T09:50:41.581074+0000","last_peered":"2023-10-09T09:50:41.581074+0000","last_clean":"2023-10-09T09:50:41.581074+0000","last_became_active":"2023-10-09T09:48:07.988051+0000","last_became_peered":"2023-10-09T09:48:07.988051+0000","last_unstale":"2023-10-09T09:50:41.581074+0000","last_undegraded":"2023-10-09T09:50:41.581074+0000","last_fullsized":"2023-10-09T09:50:41.581074+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1197,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.256175+0000","last_change":"2023-10-09T09:43:48.902066+0000","last_active":"2023-10-09T09:50:41.256175+0000","last_peered":"2023-10-09T09:50:41.256175+0000","last_clean":"2023-10-09T09:50:41.256175+0000","last_became_active":"2023-10-09T09:43:48.901639+0000","last_became_peered":"2023-10-09T09:43:48.901639+0000","last_unstale":"2023-10-09T09:50:41.256175+0000","last_undegraded":"2023-10-09T09:50:41.256175+0000","last_fullsized":"2023-10-09T09:50:41.256175+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":666,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575615+0000","last_change":"2023-10-09T09:48:07.997787+0000","last_active":"2023-10-09T09:50:41.575615+0000","last_peered":"2023-10-09T09:50:41.575615+0000","last_clean":"2023-10-09T09:50:41.575615+0000","last_became_active":"2023-10-09T09:48:07.997356+0000","last_became_peered":"2023-10-09T09:48:07.997356+0000","last_unstale":"2023-10-09T09:50:41.575615+0000","last_undegraded":"2023-10-09T09:50:41.575615+0000","last_fullsized":"2023-10-09T09:50:41.575615+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:03.024096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":140,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.255945+0000","last_change":"2023-10-09T09:48:07.975547+0000","last_active":"2023-10-09T09:50:41.255945+0000","last_peered":"2023-10-09T09:50:41.255945+0000","last_clean":"2023-10-09T09:50:41.255945+0000","last_became_active":"2023-10-09T09:48:07.975150+0000","last_became_peered":"2023-10-09T09:48:07.975150+0000","last_unstale":"2023-10-09T09:50:41.255945+0000","last_undegraded":"2023-10-09T09:50:41.255945+0000","last_fullsized":"2023-10-09T09:50:41.255945+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":689,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.579775+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:50:41.579775+0000","last_peered":"2023-10-09T09:50:41.579775+0000","last_clean":"2023-10-09T09:50:41.579775+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:50:41.579775+0000","last_undegraded":"2023-10-09T09:50:41.579775+0000","last_fullsized":"2023-10-09T09:50:41.579775+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":82,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.581120+0000","last_change":"2023-10-09T09:43:48.906785+0000","last_active":"2023-10-09T09:50:41.581120+0000","last_peered":"2023-10-09T09:50:41.581120+0000","last_clean":"2023-10-09T09:50:41.581120+0000","last_became_active":"2023-10-09T09:43:48.906276+0000","last_became_peered":"2023-10-09T09:43:48.906276+0000","last_unstale":"2023-10-09T09:50:41.581120+0000","last_undegraded":"2023-10-09T09:50:41.581120+0000","last_fullsized":"2023-10-09T09:50:41.581120+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":106,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.255638+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:50:41.255638+0000","last_peered":"2023-10-09T09:50:41.255638+0000","last_clean":"2023-10-09T09:50:41.255638+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:50:41.255638+0000","last_undegraded":"2023-10-09T09:50:41.255638+0000","last_fullsized":"2023-10-09T09:50:41.255638+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":430,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.581794+0000","last_change":"2023-10-09T09:43:23.995526+0000","last_active":"2023-10-09T09:50:41.581794+0000","last_peered":"2023-10-09T09:50:41.581794+0000","last_clean":"2023-10-09T09:50:41.581794+0000","last_became_active":"2023-10-09T09:43:23.994816+0000","last_became_peered":"2023-10-09T09:43:23.994816+0000","last_unstale":"2023-10-09T09:50:41.581794+0000","last_undegraded":"2023-10-09T09:50:41.581794+0000","last_fullsized":"2023-10-09T09:50:41.581794+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":647,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.580295+0000","last_change":"2023-10-09T09:48:07.987637+0000","last_active":"2023-10-09T09:50:41.580295+0000","last_peered":"2023-10-09T09:50:41.580295+0000","last_clean":"2023-10-09T09:50:41.580295+0000","last_became_active":"2023-10-09T09:48:07.987085+0000","last_became_peered":"2023-10-09T09:48:07.987085+0000","last_unstale":"2023-10-09T09:50:41.580295+0000","last_undegraded":"2023-10-09T09:50:41.580295+0000","last_fullsized":"2023-10-09T09:50:41.580295+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1169,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575705+0000","last_change":"2023-10-09T09:43:23.989356+0000","last_active":"2023-10-09T09:50:41.575705+0000","last_peered":"2023-10-09T09:50:41.575705+0000","last_clean":"2023-10-09T09:50:41.575705+0000","last_became_active":"2023-10-09T09:43:23.988986+0000","last_became_peered":"2023-10-09T09:43:23.988986+0000","last_unstale":"2023-10-09T09:50:41.575705+0000","last_undegraded":"2023-10-09T09:50:41.575705+0000","last_fullsized":"2023-10-09T09:50:41.575705+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.570448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":751,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.577040+0000","last_change":"2023-10-09T09:48:07.984006+0000","last_active":"2023-10-09T09:50:41.577040+0000","last_peered":"2023-10-09T09:50:41.577040+0000","last_clean":"2023-10-09T09:50:41.577040+0000","last_became_active":"2023-10-09T09:48:07.978179+0000","last_became_peered":"2023-10-09T09:48:07.978179+0000","last_unstale":"2023-10-09T09:50:41.577040+0000","last_undegraded":"2023-10-09T09:50:41.577040+0000","last_fullsized":"2023-10-09T09:50:41.577040+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:34.164004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1172,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576290+0000","last_change":"2023-10-09T09:48:07.975335+0000","last_active":"2023-10-09T09:50:41.576290+0000","last_peered":"2023-10-09T09:50:41.576290+0000","last_clean":"2023-10-09T09:50:41.576290+0000","last_became_active":"2023-10-09T09:48:07.974437+0000","last_became_peered":"2023-10-09T09:48:07.974437+0000","last_unstale":"2023-10-09T09:50:41.576290+0000","last_undegraded":"2023-10-09T09:50:41.576290+0000","last_fullsized":"2023-10-09T09:50:41.576290+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:32.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":110,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.581260+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:50:41.581260+0000","last_peered":"2023-10-09T09:50:41.581260+0000","last_clean":"2023-10-09T09:50:41.581260+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:50:41.581260+0000","last_undegraded":"2023-10-09T09:50:41.581260+0000","last_fullsized":"2023-10-09T09:50:41.581260+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":74,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.579726+0000","last_change":"2023-10-09T09:43:23.994784+0000","last_active":"2023-10-09T09:50:41.579726+0000","last_peered":"2023-10-09T09:50:41.579726+0000","last_clean":"2023-10-09T09:50:41.579726+0000","last_became_active":"2023-10-09T09:43:23.994196+0000","last_became_peered":"2023-10-09T09:43:23.994196+0000","last_unstale":"2023-10-09T09:50:41.579726+0000","last_undegraded":"2023-10-09T09:50:41.579726+0000","last_fullsized":"2023-10-09T09:50:41.579726+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1189,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576695+0000","last_change":"2023-10-09T09:43:48.901303+0000","last_active":"2023-10-09T09:50:41.576695+0000","last_peered":"2023-10-09T09:50:41.576695+0000","last_clean":"2023-10-09T09:50:41.576695+0000","last_became_active":"2023-10-09T09:43:48.900779+0000","last_became_peered":"2023-10-09T09:43:48.900779+0000","last_unstale":"2023-10-09T09:50:41.576695+0000","last_undegraded":"2023-10-09T09:50:41.576695+0000","last_fullsized":"2023-10-09T09:50:41.576695+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:51.853531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1268,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.579165+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:50:41.579165+0000","last_peered":"2023-10-09T09:50:41.579165+0000","last_clean":"2023-10-09T09:50:41.579165+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:50:41.579165+0000","last_undegraded":"2023-10-09T09:50:41.579165+0000","last_fullsized":"2023-10-09T09:50:41.579165+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1234,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576943+0000","last_change":"2023-10-09T09:48:08.000694+0000","last_active":"2023-10-09T09:50:41.576943+0000","last_peered":"2023-10-09T09:50:41.576943+0000","last_clean":"2023-10-09T09:50:41.576943+0000","last_became_active":"2023-10-09T09:48:07.999476+0000","last_became_peered":"2023-10-09T09:48:07.999476+0000","last_unstale":"2023-10-09T09:50:41.576943+0000","last_undegraded":"2023-10-09T09:50:41.576943+0000","last_fullsized":"2023-10-09T09:50:41.576943+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:37.211076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.7","version":"1274'1098","reported_seq":1110,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.055306+0000","last_change":"2023-10-09T09:50:27.379119+0000","last_active":"2023-10-09T09:50:56.055306+0000","last_peered":"2023-10-09T09:50:56.055306+0000","last_clean":"2023-10-09T09:50:56.055306+0000","last_became_active":"2023-10-09T09:50:27.378562+0000","last_became_peered":"2023-10-09T09:50:27.378562+0000","last_unstale":"2023-10-09T09:50:56.055306+0000","last_undegraded":"2023-10-09T09:50:56.055306+0000","last_fullsized":"2023-10-09T09:50:56.055306+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1098,"log_dups_size":0,"ondisk_log_size":1098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:01.665220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71958528,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1098,"num_write_kb":70272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":100,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.536503+0000","last_change":"2023-10-09T09:48:07.976558+0000","last_active":"2023-10-09T09:50:41.536503+0000","last_peered":"2023-10-09T09:50:41.536503+0000","last_clean":"2023-10-09T09:50:41.536503+0000","last_became_active":"2023-10-09T09:48:07.975961+0000","last_became_peered":"2023-10-09T09:48:07.975961+0000","last_unstale":"2023-10-09T09:50:41.536503+0000","last_undegraded":"2023-10-09T09:50:41.536503+0000","last_fullsized":"2023-10-09T09:50:41.536503+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:40.925248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":116,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.255275+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:50:41.255275+0000","last_peered":"2023-10-09T09:50:41.255275+0000","last_clean":"2023-10-09T09:50:41.255275+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:50:41.255275+0000","last_undegraded":"2023-10-09T09:50:41.255275+0000","last_fullsized":"2023-10-09T09:50:41.255275+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"1274'901","reported_seq":915,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:53.712666+0000","last_change":"2023-10-09T09:50:26.983846+0000","last_active":"2023-10-09T09:50:53.712666+0000","last_peered":"2023-10-09T09:50:53.712666+0000","last_clean":"2023-10-09T09:50:53.712666+0000","last_became_active":"2023-10-09T09:50:26.983409+0000","last_became_peered":"2023-10-09T09:50:26.983409+0000","last_unstale":"2023-10-09T09:50:53.712666+0000","last_undegraded":"2023-10-09T09:50:53.712666+0000","last_fullsized":"2023-10-09T09:50:53.712666+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":901,"log_dups_size":0,"ondisk_log_size":901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:48.191471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59047936,"num_objects":901,"num_object_clones":0,"num_object_copies":2703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":901,"num_write_kb":57664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":63,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576686+0000","last_change":"2023-10-09T09:48:07.990711+0000","last_active":"2023-10-09T09:50:41.576686+0000","last_peered":"2023-10-09T09:50:41.576686+0000","last_clean":"2023-10-09T09:50:41.576686+0000","last_became_active":"2023-10-09T09:48:07.990267+0000","last_became_peered":"2023-10-09T09:48:07.990267+0000","last_unstale":"2023-10-09T09:50:41.576686+0000","last_undegraded":"2023-10-09T09:50:41.576686+0000","last_fullsized":"2023-10-09T09:50:41.576686+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:51.244840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1207,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575795+0000","last_change":"2023-10-09T09:48:07.991440+0000","last_active":"2023-10-09T09:50:41.575795+0000","last_peered":"2023-10-09T09:50:41.575795+0000","last_clean":"2023-10-09T09:50:41.575795+0000","last_became_active":"2023-10-09T09:48:07.991072+0000","last_became_peered":"2023-10-09T09:48:07.991072+0000","last_unstale":"2023-10-09T09:50:41.575795+0000","last_undegraded":"2023-10-09T09:50:41.575795+0000","last_fullsized":"2023-10-09T09:50:41.575795+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:28.577159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"1274'1243","reported_seq":1256,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.607677+0000","last_change":"2023-10-09T09:50:26.985990+0000","last_active":"2023-10-09T09:50:56.607677+0000","last_peered":"2023-10-09T09:50:56.607677+0000","last_clean":"2023-10-09T09:50:56.607677+0000","last_became_active":"2023-10-09T09:50:26.985591+0000","last_became_peered":"2023-10-09T09:50:26.985591+0000","last_unstale":"2023-10-09T09:50:56.607677+0000","last_undegraded":"2023-10-09T09:50:56.607677+0000","last_fullsized":"2023-10-09T09:50:56.607677+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1243,"log_dups_size":0,"ondisk_log_size":1243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81461248,"num_objects":1243,"num_object_clones":0,"num_object_copies":3729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1243,"num_write_kb":79552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1224,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.256054+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:50:41.256054+0000","last_peered":"2023-10-09T09:50:41.256054+0000","last_clean":"2023-10-09T09:50:41.256054+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:50:41.256054+0000","last_undegraded":"2023-10-09T09:50:41.256054+0000","last_fullsized":"2023-10-09T09:50:41.256054+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1230,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.580847+0000","last_change":"2023-10-09T09:48:07.987819+0000","last_active":"2023-10-09T09:50:41.580847+0000","last_peered":"2023-10-09T09:50:41.580847+0000","last_clean":"2023-10-09T09:50:41.580847+0000","last_became_active":"2023-10-09T09:48:07.986354+0000","last_became_peered":"2023-10-09T09:48:07.986354+0000","last_unstale":"2023-10-09T09:50:41.580847+0000","last_undegraded":"2023-10-09T09:50:41.580847+0000","last_fullsized":"2023-10-09T09:50:41.580847+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"1274'1278","reported_seq":1291,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.933956+0000","last_change":"2023-10-09T09:50:26.987956+0000","last_active":"2023-10-09T09:50:56.933956+0000","last_peered":"2023-10-09T09:50:56.933956+0000","last_clean":"2023-10-09T09:50:56.933956+0000","last_became_active":"2023-10-09T09:50:26.987609+0000","last_became_peered":"2023-10-09T09:50:26.987609+0000","last_unstale":"2023-10-09T09:50:56.933956+0000","last_undegraded":"2023-10-09T09:50:56.933956+0000","last_fullsized":"2023-10-09T09:50:56.933956+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1278,"log_dups_size":0,"ondisk_log_size":1278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83755008,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1278,"num_write_kb":81792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":776,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.581847+0000","last_change":"2023-10-09T09:48:07.987506+0000","last_active":"2023-10-09T09:50:41.581847+0000","last_peered":"2023-10-09T09:50:41.581847+0000","last_clean":"2023-10-09T09:50:41.581847+0000","last_became_active":"2023-10-09T09:48:07.986988+0000","last_became_peered":"2023-10-09T09:48:07.986988+0000","last_unstale":"2023-10-09T09:50:41.581847+0000","last_undegraded":"2023-10-09T09:50:41.581847+0000","last_fullsized":"2023-10-09T09:50:41.581847+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1192,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.536626+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:50:41.536626+0000","last_peered":"2023-10-09T09:50:41.536626+0000","last_clean":"2023-10-09T09:50:41.536626+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:50:41.536626+0000","last_undegraded":"2023-10-09T09:50:41.536626+0000","last_fullsized":"2023-10-09T09:50:41.536626+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"1274'1243","reported_seq":1256,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.915752+0000","last_change":"2023-10-09T09:50:26.988587+0000","last_active":"2023-10-09T09:50:56.915752+0000","last_peered":"2023-10-09T09:50:56.915752+0000","last_clean":"2023-10-09T09:50:56.915752+0000","last_became_active":"2023-10-09T09:50:26.987497+0000","last_became_peered":"2023-10-09T09:50:26.987497+0000","last_unstale":"2023-10-09T09:50:56.915752+0000","last_undegraded":"2023-10-09T09:50:56.915752+0000","last_fullsized":"2023-10-09T09:50:56.915752+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1243,"log_dups_size":0,"ondisk_log_size":1243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81461248,"num_objects":1243,"num_object_clones":0,"num_object_copies":3729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1243,"num_write_kb":79552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":151,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576481+0000","last_change":"2023-10-09T09:48:07.977143+0000","last_active":"2023-10-09T09:50:41.576481+0000","last_peered":"2023-10-09T09:50:41.576481+0000","last_clean":"2023-10-09T09:50:41.576481+0000","last_became_active":"2023-10-09T09:48:07.976338+0000","last_became_peered":"2023-10-09T09:48:07.976338+0000","last_unstale":"2023-10-09T09:50:41.576481+0000","last_undegraded":"2023-10-09T09:50:41.576481+0000","last_fullsized":"2023-10-09T09:50:41.576481+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:12.015053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1226,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.581742+0000","last_change":"2023-10-09T09:48:08.001503+0000","last_active":"2023-10-09T09:50:41.581742+0000","last_peered":"2023-10-09T09:50:41.581742+0000","last_clean":"2023-10-09T09:50:41.581742+0000","last_became_active":"2023-10-09T09:48:08.000921+0000","last_became_peered":"2023-10-09T09:48:08.000921+0000","last_unstale":"2023-10-09T09:50:41.581742+0000","last_undegraded":"2023-10-09T09:50:41.581742+0000","last_fullsized":"2023-10-09T09:50:41.581742+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.2","version":"1274'1097","reported_seq":1110,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:55.639650+0000","last_change":"2023-10-09T09:50:26.980085+0000","last_active":"2023-10-09T09:50:55.639650+0000","last_peered":"2023-10-09T09:50:55.639650+0000","last_clean":"2023-10-09T09:50:55.639650+0000","last_became_active":"2023-10-09T09:50:26.979773+0000","last_became_peered":"2023-10-09T09:50:26.979773+0000","last_unstale":"2023-10-09T09:50:55.639650+0000","last_undegraded":"2023-10-09T09:50:55.639650+0000","last_fullsized":"2023-10-09T09:50:55.639650+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1097,"log_dups_size":0,"ondisk_log_size":1097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:37.729645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71892992,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1097,"num_write_kb":70208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1222,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.579480+0000","last_change":"2023-10-09T09:48:07.988586+0000","last_active":"2023-10-09T09:50:41.579480+0000","last_peered":"2023-10-09T09:50:41.579480+0000","last_clean":"2023-10-09T09:50:41.579480+0000","last_became_active":"2023-10-09T09:48:07.988226+0000","last_became_peered":"2023-10-09T09:48:07.988226+0000","last_unstale":"2023-10-09T09:50:41.579480+0000","last_undegraded":"2023-10-09T09:50:41.579480+0000","last_fullsized":"2023-10-09T09:50:41.579480+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1213,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576177+0000","last_change":"2023-10-09T09:48:07.987993+0000","last_active":"2023-10-09T09:50:41.576177+0000","last_peered":"2023-10-09T09:50:41.576177+0000","last_clean":"2023-10-09T09:50:41.576177+0000","last_became_active":"2023-10-09T09:48:07.987664+0000","last_became_peered":"2023-10-09T09:48:07.987664+0000","last_unstale":"2023-10-09T09:50:41.576177+0000","last_undegraded":"2023-10-09T09:50:41.576177+0000","last_fullsized":"2023-10-09T09:50:41.576177+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:32.253321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"1274'1265","reported_seq":1278,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.928597+0000","last_change":"2023-10-09T09:50:26.987126+0000","last_active":"2023-10-09T09:50:56.928597+0000","last_peered":"2023-10-09T09:50:56.928597+0000","last_clean":"2023-10-09T09:50:56.928597+0000","last_became_active":"2023-10-09T09:50:26.986696+0000","last_became_peered":"2023-10-09T09:50:26.986696+0000","last_unstale":"2023-10-09T09:50:56.928597+0000","last_undegraded":"2023-10-09T09:50:56.928597+0000","last_fullsized":"2023-10-09T09:50:56.928597+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1265,"log_dups_size":0,"ondisk_log_size":1265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82903040,"num_objects":1265,"num_object_clones":0,"num_object_copies":3795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1265,"num_write_kb":80960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1241,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575741+0000","last_change":"2023-10-09T09:48:07.984079+0000","last_active":"2023-10-09T09:50:41.575741+0000","last_peered":"2023-10-09T09:50:41.575741+0000","last_clean":"2023-10-09T09:50:41.575741+0000","last_became_active":"2023-10-09T09:48:07.983792+0000","last_became_peered":"2023-10-09T09:48:07.983792+0000","last_unstale":"2023-10-09T09:50:41.575741+0000","last_undegraded":"2023-10-09T09:50:41.575741+0000","last_fullsized":"2023-10-09T09:50:41.575741+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:28:42.289832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":647,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.580482+0000","last_change":"2023-10-09T09:48:07.989047+0000","last_active":"2023-10-09T09:50:41.580482+0000","last_peered":"2023-10-09T09:50:41.580482+0000","last_clean":"2023-10-09T09:50:41.580482+0000","last_became_active":"2023-10-09T09:48:07.988131+0000","last_became_peered":"2023-10-09T09:48:07.988131+0000","last_unstale":"2023-10-09T09:50:41.580482+0000","last_undegraded":"2023-10-09T09:50:41.580482+0000","last_fullsized":"2023-10-09T09:50:41.580482+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"1274'1193","reported_seq":1207,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.047268+0000","last_change":"2023-10-09T09:50:26.985963+0000","last_active":"2023-10-09T09:50:56.047268+0000","last_peered":"2023-10-09T09:50:56.047268+0000","last_clean":"2023-10-09T09:50:56.047268+0000","last_became_active":"2023-10-09T09:50:26.985572+0000","last_became_peered":"2023-10-09T09:50:26.985572+0000","last_unstale":"2023-10-09T09:50:56.047268+0000","last_undegraded":"2023-10-09T09:50:56.047268+0000","last_fullsized":"2023-10-09T09:50:56.047268+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1193,"log_dups_size":0,"ondisk_log_size":1193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:48.639926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78184448,"num_objects":1193,"num_object_clones":0,"num_object_copies":3579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1193,"num_write_kb":76352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1183,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575982+0000","last_change":"2023-10-09T09:48:07.976884+0000","last_active":"2023-10-09T09:50:41.575982+0000","last_peered":"2023-10-09T09:50:41.575982+0000","last_clean":"2023-10-09T09:50:41.575982+0000","last_became_active":"2023-10-09T09:48:07.975932+0000","last_became_peered":"2023-10-09T09:48:07.975932+0000","last_unstale":"2023-10-09T09:50:41.575982+0000","last_undegraded":"2023-10-09T09:50:41.575982+0000","last_fullsized":"2023-10-09T09:50:41.575982+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:22.785728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":119,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.577542+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:50:41.577542+0000","last_peered":"2023-10-09T09:50:41.577542+0000","last_clean":"2023-10-09T09:50:41.577542+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:50:41.577542+0000","last_undegraded":"2023-10-09T09:50:41.577542+0000","last_fullsized":"2023-10-09T09:50:41.577542+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"1274'1170","reported_seq":1184,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:55.629308+0000","last_change":"2023-10-09T09:50:27.376496+0000","last_active":"2023-10-09T09:50:55.629308+0000","last_peered":"2023-10-09T09:50:55.629308+0000","last_clean":"2023-10-09T09:50:55.629308+0000","last_became_active":"2023-10-09T09:50:27.376004+0000","last_became_peered":"2023-10-09T09:50:27.376004+0000","last_unstale":"2023-10-09T09:50:55.629308+0000","last_undegraded":"2023-10-09T09:50:55.629308+0000","last_fullsized":"2023-10-09T09:50:55.629308+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1170,"log_dups_size":0,"ondisk_log_size":1170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:22.820238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76677120,"num_objects":1170,"num_object_clones":0,"num_object_copies":3510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1170,"num_write_kb":74880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1229,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576969+0000","last_change":"2023-10-09T09:48:07.984553+0000","last_active":"2023-10-09T09:50:41.576969+0000","last_peered":"2023-10-09T09:50:41.576969+0000","last_clean":"2023-10-09T09:50:41.576969+0000","last_became_active":"2023-10-09T09:48:07.983302+0000","last_became_peered":"2023-10-09T09:48:07.983302+0000","last_unstale":"2023-10-09T09:50:41.576969+0000","last_undegraded":"2023-10-09T09:50:41.576969+0000","last_fullsized":"2023-10-09T09:50:41.576969+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:50.714295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.255738+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:50:41.255738+0000","last_peered":"2023-10-09T09:50:41.255738+0000","last_clean":"2023-10-09T09:50:41.255738+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:50:41.255738+0000","last_undegraded":"2023-10-09T09:50:41.255738+0000","last_fullsized":"2023-10-09T09:50:41.255738+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1206,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576413+0000","last_change":"2023-10-09T09:48:07.977683+0000","last_active":"2023-10-09T09:50:41.576413+0000","last_peered":"2023-10-09T09:50:41.576413+0000","last_clean":"2023-10-09T09:50:41.576413+0000","last_became_active":"2023-10-09T09:48:07.977024+0000","last_became_peered":"2023-10-09T09:48:07.977024+0000","last_unstale":"2023-10-09T09:50:41.576413+0000","last_undegraded":"2023-10-09T09:50:41.576413+0000","last_fullsized":"2023-10-09T09:50:41.576413+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:13.438431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"1274'1200","reported_seq":1214,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.044475+0000","last_change":"2023-10-09T09:50:27.379238+0000","last_active":"2023-10-09T09:50:56.044475+0000","last_peered":"2023-10-09T09:50:56.044475+0000","last_clean":"2023-10-09T09:50:56.044475+0000","last_became_active":"2023-10-09T09:50:27.378551+0000","last_became_peered":"2023-10-09T09:50:27.378551+0000","last_unstale":"2023-10-09T09:50:56.044475+0000","last_undegraded":"2023-10-09T09:50:56.044475+0000","last_fullsized":"2023-10-09T09:50:56.044475+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1200,"log_dups_size":0,"ondisk_log_size":1200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:19.610902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78643200,"num_objects":1200,"num_object_clones":0,"num_object_copies":3600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1200,"num_write_kb":76800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":689,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576201+0000","last_change":"2023-10-09T09:48:07.997811+0000","last_active":"2023-10-09T09:50:41.576201+0000","last_peered":"2023-10-09T09:50:41.576201+0000","last_clean":"2023-10-09T09:50:41.576201+0000","last_became_active":"2023-10-09T09:48:07.997389+0000","last_became_peered":"2023-10-09T09:48:07.997389+0000","last_unstale":"2023-10-09T09:50:41.576201+0000","last_undegraded":"2023-10-09T09:50:41.576201+0000","last_fullsized":"2023-10-09T09:50:41.576201+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:21.978398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1164,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.580530+0000","last_change":"2023-10-09T09:43:48.908190+0000","last_active":"2023-10-09T09:50:41.580530+0000","last_peered":"2023-10-09T09:50:41.580530+0000","last_clean":"2023-10-09T09:50:41.580530+0000","last_became_active":"2023-10-09T09:43:48.907799+0000","last_became_peered":"2023-10-09T09:43:48.907799+0000","last_unstale":"2023-10-09T09:50:41.580530+0000","last_undegraded":"2023-10-09T09:50:41.580530+0000","last_fullsized":"2023-10-09T09:50:41.580530+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1259,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576587+0000","last_change":"2023-10-09T09:48:08.000154+0000","last_active":"2023-10-09T09:50:41.576587+0000","last_peered":"2023-10-09T09:50:41.576587+0000","last_clean":"2023-10-09T09:50:41.576587+0000","last_became_active":"2023-10-09T09:48:07.999714+0000","last_became_peered":"2023-10-09T09:48:07.999714+0000","last_unstale":"2023-10-09T09:50:41.576587+0000","last_undegraded":"2023-10-09T09:50:41.576587+0000","last_fullsized":"2023-10-09T09:50:41.576587+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:12.921693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"1274'1193","reported_seq":1206,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.944383+0000","last_change":"2023-10-09T09:50:26.987164+0000","last_active":"2023-10-09T09:50:56.944383+0000","last_peered":"2023-10-09T09:50:56.944383+0000","last_clean":"2023-10-09T09:50:56.944383+0000","last_became_active":"2023-10-09T09:50:26.986703+0000","last_became_peered":"2023-10-09T09:50:26.986703+0000","last_unstale":"2023-10-09T09:50:56.944383+0000","last_undegraded":"2023-10-09T09:50:56.944383+0000","last_fullsized":"2023-10-09T09:50:56.944383+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1193,"log_dups_size":0,"ondisk_log_size":1193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78184448,"num_objects":1193,"num_object_clones":0,"num_object_copies":3579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1193,"num_write_kb":76352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":164,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575890+0000","last_change":"2023-10-09T09:48:07.983969+0000","last_active":"2023-10-09T09:50:41.575890+0000","last_peered":"2023-10-09T09:50:41.575890+0000","last_clean":"2023-10-09T09:50:41.575890+0000","last_became_active":"2023-10-09T09:48:07.983490+0000","last_became_peered":"2023-10-09T09:48:07.983490+0000","last_unstale":"2023-10-09T09:50:41.575890+0000","last_undegraded":"2023-10-09T09:50:41.575890+0000","last_fullsized":"2023-10-09T09:50:41.575890+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:21.122987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1237,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575784+0000","last_change":"2023-10-09T09:48:07.998234+0000","last_active":"2023-10-09T09:50:41.575784+0000","last_peered":"2023-10-09T09:50:41.575784+0000","last_clean":"2023-10-09T09:50:41.575784+0000","last_became_active":"2023-10-09T09:48:07.997909+0000","last_became_peered":"2023-10-09T09:48:07.997909+0000","last_unstale":"2023-10-09T09:50:41.575784+0000","last_undegraded":"2023-10-09T09:50:41.575784+0000","last_fullsized":"2023-10-09T09:50:41.575784+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:21.750617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1226,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576803+0000","last_change":"2023-10-09T09:48:07.973284+0000","last_active":"2023-10-09T09:50:41.576803+0000","last_peered":"2023-10-09T09:50:41.576803+0000","last_clean":"2023-10-09T09:50:41.576803+0000","last_became_active":"2023-10-09T09:48:07.971235+0000","last_became_peered":"2023-10-09T09:48:07.971235+0000","last_unstale":"2023-10-09T09:50:41.576803+0000","last_undegraded":"2023-10-09T09:50:41.576803+0000","last_fullsized":"2023-10-09T09:50:41.576803+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:40.951420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"1274'1262","reported_seq":1275,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.942494+0000","last_change":"2023-10-09T09:50:26.987803+0000","last_active":"2023-10-09T09:50:56.942494+0000","last_peered":"2023-10-09T09:50:56.942494+0000","last_clean":"2023-10-09T09:50:56.942494+0000","last_became_active":"2023-10-09T09:50:26.987440+0000","last_became_peered":"2023-10-09T09:50:26.987440+0000","last_unstale":"2023-10-09T09:50:56.942494+0000","last_undegraded":"2023-10-09T09:50:56.942494+0000","last_fullsized":"2023-10-09T09:50:56.942494+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1262,"log_dups_size":0,"ondisk_log_size":1262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82706432,"num_objects":1262,"num_object_clones":0,"num_object_copies":3786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1262,"num_write_kb":80768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1282,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576518+0000","last_change":"2023-10-09T09:41:11.795802+0000","last_active":"2023-10-09T09:50:41.576518+0000","last_peered":"2023-10-09T09:50:41.576518+0000","last_clean":"2023-10-09T09:50:41.576518+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:50:41.576518+0000","last_undegraded":"2023-10-09T09:50:41.576518+0000","last_fullsized":"2023-10-09T09:50:41.576518+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":616,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.036761+0000","last_change":"2023-10-09T09:48:07.978435+0000","last_active":"2023-10-09T09:50:45.036761+0000","last_peered":"2023-10-09T09:50:45.036761+0000","last_clean":"2023-10-09T09:50:45.036761+0000","last_became_active":"2023-10-09T09:48:07.977417+0000","last_became_peered":"2023-10-09T09:48:07.977417+0000","last_unstale":"2023-10-09T09:50:45.036761+0000","last_undegraded":"2023-10-09T09:50:45.036761+0000","last_fullsized":"2023-10-09T09:50:45.036761+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:19.288202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":116,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.255522+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:50:41.255522+0000","last_peered":"2023-10-09T09:50:41.255522+0000","last_clean":"2023-10-09T09:50:41.255522+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:50:41.255522+0000","last_undegraded":"2023-10-09T09:50:41.255522+0000","last_fullsized":"2023-10-09T09:50:41.255522+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1200,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.581332+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:50:41.581332+0000","last_peered":"2023-10-09T09:50:41.581332+0000","last_clean":"2023-10-09T09:50:41.581332+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:50:41.581332+0000","last_undegraded":"2023-10-09T09:50:41.581332+0000","last_fullsized":"2023-10-09T09:50:41.581332+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"1274'1217","reported_seq":1229,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.944182+0000","last_change":"2023-10-09T09:50:26.988221+0000","last_active":"2023-10-09T09:50:56.944182+0000","last_peered":"2023-10-09T09:50:56.944182+0000","last_clean":"2023-10-09T09:50:56.944182+0000","last_became_active":"2023-10-09T09:50:26.987465+0000","last_became_peered":"2023-10-09T09:50:26.987465+0000","last_unstale":"2023-10-09T09:50:56.944182+0000","last_undegraded":"2023-10-09T09:50:56.944182+0000","last_fullsized":"2023-10-09T09:50:56.944182+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1217,"log_dups_size":0,"ondisk_log_size":1217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79757312,"num_objects":1217,"num_object_clones":0,"num_object_copies":3651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1217,"num_write_kb":77888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1203,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575723+0000","last_change":"2023-10-09T09:48:07.973246+0000","last_active":"2023-10-09T09:50:41.575723+0000","last_peered":"2023-10-09T09:50:41.575723+0000","last_clean":"2023-10-09T09:50:41.575723+0000","last_became_active":"2023-10-09T09:48:07.971221+0000","last_became_peered":"2023-10-09T09:48:07.971221+0000","last_unstale":"2023-10-09T09:50:41.575723+0000","last_undegraded":"2023-10-09T09:50:41.575723+0000","last_fullsized":"2023-10-09T09:50:41.575723+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:16.759514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1210,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.536072+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:50:41.536072+0000","last_peered":"2023-10-09T09:50:41.536072+0000","last_clean":"2023-10-09T09:50:41.536072+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:50:41.536072+0000","last_undegraded":"2023-10-09T09:50:41.536072+0000","last_fullsized":"2023-10-09T09:50:41.536072+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1222,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.536499+0000","last_change":"2023-10-09T09:48:07.981410+0000","last_active":"2023-10-09T09:50:41.536499+0000","last_peered":"2023-10-09T09:50:41.536499+0000","last_clean":"2023-10-09T09:50:41.536499+0000","last_became_active":"2023-10-09T09:48:07.979850+0000","last_became_peered":"2023-10-09T09:48:07.979850+0000","last_unstale":"2023-10-09T09:50:41.536499+0000","last_undegraded":"2023-10-09T09:50:41.536499+0000","last_fullsized":"2023-10-09T09:50:41.536499+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:31.201104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"1274'1166","reported_seq":1180,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.037707+0000","last_change":"2023-10-09T09:50:27.379333+0000","last_active":"2023-10-09T09:50:56.037707+0000","last_peered":"2023-10-09T09:50:56.037707+0000","last_clean":"2023-10-09T09:50:56.037707+0000","last_became_active":"2023-10-09T09:50:27.378836+0000","last_became_peered":"2023-10-09T09:50:27.378836+0000","last_unstale":"2023-10-09T09:50:56.037707+0000","last_undegraded":"2023-10-09T09:50:56.037707+0000","last_fullsized":"2023-10-09T09:50:56.037707+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1166,"log_dups_size":0,"ondisk_log_size":1166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:59.239495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76414976,"num_objects":1166,"num_object_clones":0,"num_object_copies":3498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1166,"num_write_kb":74624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1197,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.536552+0000","last_change":"2023-10-09T09:48:07.975786+0000","last_active":"2023-10-09T09:50:41.536552+0000","last_peered":"2023-10-09T09:50:41.536552+0000","last_clean":"2023-10-09T09:50:41.536552+0000","last_became_active":"2023-10-09T09:48:07.975338+0000","last_became_peered":"2023-10-09T09:48:07.975338+0000","last_unstale":"2023-10-09T09:50:41.536552+0000","last_undegraded":"2023-10-09T09:50:41.536552+0000","last_fullsized":"2023-10-09T09:50:41.536552+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:05.175956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1240,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576836+0000","last_change":"2023-10-09T09:48:07.983773+0000","last_active":"2023-10-09T09:50:41.576836+0000","last_peered":"2023-10-09T09:50:41.576836+0000","last_clean":"2023-10-09T09:50:41.576836+0000","last_became_active":"2023-10-09T09:48:07.976869+0000","last_became_peered":"2023-10-09T09:48:07.976869+0000","last_unstale":"2023-10-09T09:50:41.576836+0000","last_undegraded":"2023-10-09T09:50:41.576836+0000","last_fullsized":"2023-10-09T09:50:41.576836+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:58.173236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1190,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.255550+0000","last_change":"2023-10-09T09:48:07.962302+0000","last_active":"2023-10-09T09:50:41.255550+0000","last_peered":"2023-10-09T09:50:41.255550+0000","last_clean":"2023-10-09T09:50:41.255550+0000","last_became_active":"2023-10-09T09:48:07.961888+0000","last_became_peered":"2023-10-09T09:48:07.961888+0000","last_unstale":"2023-10-09T09:50:41.255550+0000","last_undegraded":"2023-10-09T09:50:41.255550+0000","last_fullsized":"2023-10-09T09:50:41.255550+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"1274'1194","reported_seq":1208,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.052765+0000","last_change":"2023-10-09T09:50:26.989029+0000","last_active":"2023-10-09T09:50:56.052765+0000","last_peered":"2023-10-09T09:50:56.052765+0000","last_clean":"2023-10-09T09:50:56.052765+0000","last_became_active":"2023-10-09T09:50:26.988265+0000","last_became_peered":"2023-10-09T09:50:26.988265+0000","last_unstale":"2023-10-09T09:50:56.052765+0000","last_undegraded":"2023-10-09T09:50:56.052765+0000","last_fullsized":"2023-10-09T09:50:56.052765+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1194,"log_dups_size":0,"ondisk_log_size":1194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.248282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78249984,"num_objects":1194,"num_object_clones":0,"num_object_copies":3582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1194,"num_write_kb":76416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":135,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.536616+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:50:41.536616+0000","last_peered":"2023-10-09T09:50:41.536616+0000","last_clean":"2023-10-09T09:50:41.536616+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:50:41.536616+0000","last_undegraded":"2023-10-09T09:50:41.536616+0000","last_fullsized":"2023-10-09T09:50:41.536616+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1192,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.577188+0000","last_change":"2023-10-09T09:48:07.999034+0000","last_active":"2023-10-09T09:50:41.577188+0000","last_peered":"2023-10-09T09:50:41.577188+0000","last_clean":"2023-10-09T09:50:41.577188+0000","last_became_active":"2023-10-09T09:48:07.998606+0000","last_became_peered":"2023-10-09T09:48:07.998606+0000","last_unstale":"2023-10-09T09:50:41.577188+0000","last_undegraded":"2023-10-09T09:50:41.577188+0000","last_fullsized":"2023-10-09T09:50:41.577188+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.469476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1128,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575588+0000","last_change":"2023-10-09T09:48:07.976350+0000","last_active":"2023-10-09T09:50:41.575588+0000","last_peered":"2023-10-09T09:50:41.575588+0000","last_clean":"2023-10-09T09:50:41.575588+0000","last_became_active":"2023-10-09T09:48:07.973099+0000","last_became_peered":"2023-10-09T09:48:07.973099+0000","last_unstale":"2023-10-09T09:50:41.575588+0000","last_undegraded":"2023-10-09T09:50:41.575588+0000","last_fullsized":"2023-10-09T09:50:41.575588+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:48.812722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1274'1206","reported_seq":1219,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:56.043353+0000","last_change":"2023-10-09T09:50:27.380948+0000","last_active":"2023-10-09T09:50:56.043353+0000","last_peered":"2023-10-09T09:50:56.043353+0000","last_clean":"2023-10-09T09:50:56.043353+0000","last_became_active":"2023-10-09T09:50:27.380423+0000","last_became_peered":"2023-10-09T09:50:27.380423+0000","last_unstale":"2023-10-09T09:50:56.043353+0000","last_undegraded":"2023-10-09T09:50:56.043353+0000","last_fullsized":"2023-10-09T09:50:56.043353+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":1206,"log_dups_size":0,"ondisk_log_size":1206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:09.358786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79036416,"num_objects":1206,"num_object_clones":0,"num_object_copies":3618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1206,"num_write_kb":77184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1261,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.580853+0000","last_change":"2023-10-09T09:48:07.990005+0000","last_active":"2023-10-09T09:50:41.580853+0000","last_peered":"2023-10-09T09:50:41.580853+0000","last_clean":"2023-10-09T09:50:41.580853+0000","last_became_active":"2023-10-09T09:48:07.987124+0000","last_became_peered":"2023-10-09T09:48:07.987124+0000","last_unstale":"2023-10-09T09:50:41.580853+0000","last_undegraded":"2023-10-09T09:50:41.580853+0000","last_fullsized":"2023-10-09T09:50:41.580853+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1203,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.536671+0000","last_change":"2023-10-09T09:48:07.980600+0000","last_active":"2023-10-09T09:50:41.536671+0000","last_peered":"2023-10-09T09:50:41.536671+0000","last_clean":"2023-10-09T09:50:41.536671+0000","last_became_active":"2023-10-09T09:48:07.979544+0000","last_became_peered":"2023-10-09T09:48:07.979544+0000","last_unstale":"2023-10-09T09:50:41.536671+0000","last_undegraded":"2023-10-09T09:50:41.536671+0000","last_fullsized":"2023-10-09T09:50:41.536671+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:47.320689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1192,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.255506+0000","last_change":"2023-10-09T09:43:23.992213+0000","last_active":"2023-10-09T09:50:41.255506+0000","last_peered":"2023-10-09T09:50:41.255506+0000","last_clean":"2023-10-09T09:50:41.255506+0000","last_became_active":"2023-10-09T09:43:23.991887+0000","last_became_peered":"2023-10-09T09:43:23.991887+0000","last_unstale":"2023-10-09T09:50:41.255506+0000","last_undegraded":"2023-10-09T09:50:41.255506+0000","last_fullsized":"2023-10-09T09:50:41.255506+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1211,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.536823+0000","last_change":"2023-10-09T09:48:07.969929+0000","last_active":"2023-10-09T09:50:41.536823+0000","last_peered":"2023-10-09T09:50:41.536823+0000","last_clean":"2023-10-09T09:50:41.536823+0000","last_became_active":"2023-10-09T09:48:07.969537+0000","last_became_peered":"2023-10-09T09:48:07.969537+0000","last_unstale":"2023-10-09T09:50:41.536823+0000","last_undegraded":"2023-10-09T09:50:41.536823+0000","last_fullsized":"2023-10-09T09:50:41.536823+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:42.344608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":163,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576324+0000","last_change":"2023-10-09T09:48:07.987618+0000","last_active":"2023-10-09T09:50:41.576324+0000","last_peered":"2023-10-09T09:50:41.576324+0000","last_clean":"2023-10-09T09:50:41.576324+0000","last_became_active":"2023-10-09T09:48:07.987341+0000","last_became_peered":"2023-10-09T09:48:07.987341+0000","last_unstale":"2023-10-09T09:50:41.576324+0000","last_undegraded":"2023-10-09T09:50:41.576324+0000","last_fullsized":"2023-10-09T09:50:41.576324+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:19.972891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":63,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576632+0000","last_change":"2023-10-09T09:48:07.984247+0000","last_active":"2023-10-09T09:50:41.576632+0000","last_peered":"2023-10-09T09:50:41.576632+0000","last_clean":"2023-10-09T09:50:41.576632+0000","last_became_active":"2023-10-09T09:48:07.978613+0000","last_became_peered":"2023-10-09T09:48:07.978613+0000","last_unstale":"2023-10-09T09:50:41.576632+0000","last_undegraded":"2023-10-09T09:50:41.576632+0000","last_fullsized":"2023-10-09T09:50:41.576632+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:34.663108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1207,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.576134+0000","last_change":"2023-10-09T09:48:07.991053+0000","last_active":"2023-10-09T09:50:41.576134+0000","last_peered":"2023-10-09T09:50:41.576134+0000","last_clean":"2023-10-09T09:50:41.576134+0000","last_became_active":"2023-10-09T09:48:07.990656+0000","last_became_peered":"2023-10-09T09:48:07.990656+0000","last_unstale":"2023-10-09T09:50:41.576134+0000","last_undegraded":"2023-10-09T09:50:41.576134+0000","last_fullsized":"2023-10-09T09:50:41.576134+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:06.223305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":82,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.581523+0000","last_change":"2023-10-09T09:43:23.995576+0000","last_active":"2023-10-09T09:50:41.581523+0000","last_peered":"2023-10-09T09:50:41.581523+0000","last_clean":"2023-10-09T09:50:41.581523+0000","last_became_active":"2023-10-09T09:43:23.995076+0000","last_became_peered":"2023-10-09T09:43:23.995076+0000","last_unstale":"2023-10-09T09:50:41.581523+0000","last_undegraded":"2023-10-09T09:50:41.581523+0000","last_fullsized":"2023-10-09T09:50:41.581523+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1242,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575945+0000","last_change":"2023-10-09T09:43:48.904021+0000","last_active":"2023-10-09T09:50:41.575945+0000","last_peered":"2023-10-09T09:50:41.575945+0000","last_clean":"2023-10-09T09:50:41.575945+0000","last_became_active":"2023-10-09T09:43:48.903719+0000","last_became_peered":"2023-10-09T09:43:48.903719+0000","last_unstale":"2023-10-09T09:50:41.575945+0000","last_undegraded":"2023-10-09T09:50:41.575945+0000","last_fullsized":"2023-10-09T09:50:41.575945+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:16.284027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":222,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.580636+0000","last_change":"2023-10-09T09:48:08.001392+0000","last_active":"2023-10-09T09:50:41.580636+0000","last_peered":"2023-10-09T09:50:41.580636+0000","last_clean":"2023-10-09T09:50:41.580636+0000","last_became_active":"2023-10-09T09:48:08.000798+0000","last_became_peered":"2023-10-09T09:48:08.000798+0000","last_unstale":"2023-10-09T09:50:41.580636+0000","last_undegraded":"2023-10-09T09:50:41.580636+0000","last_fullsized":"2023-10-09T09:50:41.580636+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":148,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.575563+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:50:41.575563+0000","last_peered":"2023-10-09T09:50:41.575563+0000","last_clean":"2023-10-09T09:50:41.575563+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:50:41.575563+0000","last_undegraded":"2023-10-09T09:50:41.575563+0000","last_fullsized":"2023-10-09T09:50:41.575563+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1236,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.536418+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:50:41.536418+0000","last_peered":"2023-10-09T09:50:41.536418+0000","last_clean":"2023-10-09T09:50:41.536418+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:50:41.536418+0000","last_undegraded":"2023-10-09T09:50:41.536418+0000","last_fullsized":"2023-10-09T09:50:41.536418+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":647,"reported_epoch":1273,"state":"active+clean","last_fresh":"2023-10-09T09:50:41.579505+0000","last_change":"2023-10-09T09:48:07.990205+0000","last_active":"2023-10-09T09:50:41.579505+0000","last_peered":"2023-10-09T09:50:41.579505+0000","last_clean":"2023-10-09T09:50:41.579505+0000","last_became_active":"2023-10-09T09:48:07.989619+0000","last_became_peered":"2023-10-09T09:48:07.989619+0000","last_unstale":"2023-10-09T09:50:41.579505+0000","last_undegraded":"2023-10-09T09:50:41.579505+0000","last_fullsized":"2023-10-09T09:50:41.579505+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1240334336,"num_objects":18926,"num_object_clones":0,"num_object_copies":56778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18926,"num_write_kb":1211264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3554607104,"data_stored":3554607104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":18926,"ondisk_log_size":18926,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":2460,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30942,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453398110,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594343,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1602156,"kb_used_data":705572,"kb_used_omap":0,"kb_used_meta":896576,"kb_avail":92142996,"statfs":{"total":95995035648,"available":94354427904,"internally_reserved":0,"allocated":722505728,"data_stored":722213383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918093824},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":5.2720000000000002,"15min":2.706},"min":{"1min":0.44800000000000001,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":1.2629999999999999,"5min":149.095,"15min":149.095},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":5.4640000000000004,"15min":2.972},"min":{"1min":0.59299999999999997,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":1.3440000000000001,"5min":149.17500000000001,"15min":149.17500000000001},"last":0.48599999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:50:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":1.962,"15min":2.145},"min":{"1min":0.34799999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.1960000000000002,"5min":9.391,"15min":9.391},"last":0.37},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.8720000000000001,"15min":1.9850000000000001},"min":{"1min":0.49299999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.9039999999999999,"5min":7.9240000000000004,"15min":7.9240000000000004},"last":1.0489999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":5.0780000000000003,"15min":2.871},"min":{"1min":0.34000000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2430000000000001,"5min":145.41399999999999,"15min":145.41399999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.89700000000000002,"5min":5.0039999999999996,"15min":2.9790000000000001},"min":{"1min":0.41199999999999998,"5min":0.309,"15min":0.29499999999999998},"max":{"1min":1.218,"5min":151.29300000000001,"15min":151.29300000000001},"last":0.995}]},{"osd":3,"last update":"Mon Oct 9 09:50:39 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.3109999999999999,"15min":1.3919999999999999},"min":{"1min":0.57499999999999996,"5min":0.33400000000000002,"15min":0.309},"max":{"1min":1.103,"5min":7.9139999999999997,"15min":11.547000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.8720000000000001,"15min":1.5},"min":{"1min":0.58999999999999997,"5min":0.39000000000000001,"15min":0.29499999999999998},"max":{"1min":1.3480000000000001,"5min":11.968999999999999,"15min":11.968999999999999},"last":0.97199999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:50:39 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.7589999999999999,"15min":1.4390000000000001},"min":{"1min":0.373,"5min":0.28399999999999997,"15min":0.28199999999999997},"max":{"1min":1.3420000000000001,"5min":12.148999999999999,"15min":12.148999999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.998,"5min":2.4140000000000001,"15min":1.7090000000000001},"min":{"1min":0.60099999999999998,"5min":0.313,"15min":0.29499999999999998},"max":{"1min":1.3660000000000001,"5min":12.064,"15min":12.064},"last":0.92400000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:50:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.786,"15min":1.577},"min":{"1min":0.39300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3819999999999999,"5min":10.529,"15min":10.529},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.0800000000000001,"5min":2.2280000000000002,"15min":1.736},"min":{"1min":0.73399999999999999,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":1.4339999999999999,"5min":12.000999999999999,"15min":12.000999999999999},"last":1.0129999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:50:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":2.1760000000000002,"15min":1.583},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.9930000000000003,"5min":11.922000000000001,"15min":11.922000000000001},"last":1.0940000000000001},{"interface":"front","average":{"1min":1.05,"5min":1.373,"15min":1.256},"min":{"1min":0.42199999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.1840000000000002,"5min":7.9359999999999999,"15min":7.9359999999999999},"last":0.54700000000000004}]}]},{"osd":2,"up_from":634,"seq":2723009266096,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1256268,"kb_used_data":502020,"kb_used_omap":122,"kb_used_meta":754117,"kb_avail":92488884,"statfs":{"total":95995035648,"available":94708617216,"internally_reserved":0,"allocated":514068480,"data_stored":512529726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":772216395},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":5.4980000000000002,"15min":3.7360000000000002},"min":{"1min":0.35199999999999998,"5min":0.22900000000000001,"15min":0.20200000000000001},"max":{"1min":0.84899999999999998,"5min":163.84,"15min":163.84},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":5.6369999999999996,"15min":3.9119999999999999},"min":{"1min":0.35799999999999998,"5min":0.20799999999999999,"15min":0.20399999999999999},"max":{"1min":0.96599999999999997,"5min":163.965,"15min":163.965},"last":0.82199999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:50:09 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.99399999999999999,"15min":1.034},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.4700000000000002,"5min":4.6740000000000004,"15min":4.6740000000000004},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.113,"15min":1.1479999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.827,"5min":3.827,"15min":3.827},"last":1.0629999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.92500000000000004,"15min":2.0409999999999999},"min":{"1min":0.40400000000000003,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":1.01,"5min":4.1429999999999998,"15min":230.476},"last":0.439},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.71499999999999997,"15min":1.9710000000000001},"min":{"1min":0.433,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":0.999,"5min":3.7879999999999998,"15min":228.893},"last":0.69599999999999995}]},{"osd":4,"last update":"Mon Oct 9 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.9550000000000001,"15min":1.575},"min":{"1min":0.495,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.351,"5min":71.876000000000005,"15min":71.876000000000005},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.321,"15min":1.4079999999999999},"min":{"1min":0.64400000000000002,"5min":0.29899999999999999,"15min":0.26700000000000002},"max":{"1min":2.5739999999999998,"5min":9.4640000000000004,"15min":10.394},"last":0.71499999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.466,"15min":1.395},"min":{"1min":0.37,"5min":0.34699999999999998,"15min":0.29799999999999999},"max":{"1min":5.5140000000000002,"5min":8.5830000000000002,"15min":9.7599999999999998},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.2150000000000001,"15min":1.3819999999999999},"min":{"1min":0.55700000000000005,"5min":0.41499999999999998,"15min":0.32200000000000001},"max":{"1min":1.3899999999999999,"5min":5.806,"15min":11.718999999999999},"last":0.64000000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":2.4399999999999999,"15min":1.9319999999999999},"min":{"1min":0.64100000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.6919999999999999,"5min":71.795000000000002,"15min":71.795000000000002},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.0069999999999999,"5min":2.2959999999999998,"15min":1.724},"min":{"1min":0.72199999999999998,"5min":0.38700000000000001,"15min":0.35599999999999998},"max":{"1min":1.512,"5min":72.322000000000003,"15min":72.322000000000003},"last":0.84699999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.2370000000000001,"15min":1.363},"min":{"1min":0.57799999999999996,"5min":0.374,"15min":0.374},"max":{"1min":1.587,"5min":8.4659999999999993,"15min":10.462999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.044,"5min":1.25,"15min":1.2929999999999999},"min":{"1min":0.44400000000000001,"5min":0.441,"15min":0.40400000000000003},"max":{"1min":1.5640000000000001,"5min":5.1029999999999998,"15min":9.7759999999999998},"last":0.98499999999999999}]}]},{"osd":1,"up_from":1246,"seq":5351529250851,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1071676,"kb_used_data":865652,"kb_used_omap":204,"kb_used_meta":205811,"kb_avail":92673476,"statfs":{"total":95995035648,"available":94897639424,"internally_reserved":0,"allocated":886427648,"data_stored":884901127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208958,"internal_metadata":210751426},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.5409999999999999,"15min":1.6830000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.343,"5min":4.343,"15min":4.343},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.349,"5min":1.9870000000000001,"15min":2.093},"min":{"1min":0.309,"5min":0.214,"15min":0.214},"max":{"1min":8.5579999999999998,"5min":8.5579999999999998,"15min":8.5579999999999998},"last":4.0309999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":1.401,"15min":1.5009999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.2429999999999999,"5min":5.8570000000000002,"15min":5.8570000000000002},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.4430000000000001,"15min":1.571},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.3410000000000002,"5min":5.5990000000000002,"15min":5.5990000000000002},"last":0.64000000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:50:12 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.5169999999999999,"15min":1.504},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.7140000000000004,"5min":8.7140000000000004,"15min":8.7140000000000004},"last":3.3290000000000002},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.2509999999999999,"15min":1.337},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":3.4620000000000002,"5min":4.4279999999999999,"15min":4.4279999999999999},"last":0.58699999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:50:12 2023","interfaces":[{"interface":"back","average":{"1min":1.724,"5min":2.528,"15min":2.6619999999999999},"min":{"1min":0.38,"5min":0.379,"15min":0.379},"max":{"1min":8.8070000000000004,"5min":8.8070000000000004,"15min":8.8070000000000004},"last":3.476},{"interface":"front","average":{"1min":1.4990000000000001,"5min":2.2989999999999999,"15min":2.4329999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":7.0259999999999998,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.55300000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:50:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.7170000000000001,"15min":1.8049999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.9000000000000004,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.276,"5min":1.774,"15min":1.857},"min":{"1min":0.61199999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":4.29,"5min":5.952,"15min":5.952},"last":0.69399999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:50:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.772,"15min":1.8360000000000001},"min":{"1min":0.53600000000000003,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":7.4119999999999999,"5min":7.4119999999999999,"15min":7.4119999999999999},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.853,"5min":2.968,"15min":3.1539999999999999},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":9.827,"5min":9.827,"15min":9.827},"last":3.5059999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:50:12 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.5149999999999999,"15min":1.5980000000000001},"min":{"1min":0.54700000000000004,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":4.3159999999999998,"5min":5.6970000000000001,"15min":5.6970000000000001},"last":0.748},{"interface":"front","average":{"1min":1.7070000000000001,"5min":2.2629999999999999,"15min":2.355},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":8.7690000000000001,"5min":8.7690000000000001,"15min":8.7690000000000001},"last":3.4369999999999998}]}]},{"osd":0,"up_from":1136,"seq":4879082848444,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1163600,"kb_used_data":544840,"kb_used_omap":280,"kb_used_meta":618471,"kb_avail":92581552,"statfs":{"total":95995035648,"available":94803509248,"internally_reserved":0,"allocated":557916160,"data_stored":556422314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":633314714},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:50:12 2023","interfaces":[{"interface":"back","average":{"1min":4.5750000000000002,"5min":2.0310000000000001,"15min":1.607},"min":{"1min":0.27600000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":52.793999999999997,"5min":52.793999999999997,"15min":52.793999999999997},"last":0.79400000000000004},{"interface":"front","average":{"1min":4.1820000000000004,"5min":2.25,"15min":1.9279999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":52.563000000000002,"5min":52.563000000000002,"15min":52.563000000000002},"last":2.2400000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.262,"15min":1.679},"min":{"1min":0.38400000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.1920000000000002,"5min":48.795000000000002,"15min":48.795000000000002},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.68700000000000006,"5min":2.9510000000000001,"15min":2.0049999999999999},"min":{"1min":0.39800000000000002,"5min":0.23699999999999999,"15min":0.23300000000000001},"max":{"1min":1.1100000000000001,"5min":48.856000000000002,"15min":48.856000000000002},"last":2.0339999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.0169999999999999,"15min":1.22},"min":{"1min":0.443,"5min":0.27500000000000002,"15min":0.247},"max":{"1min":4.2240000000000002,"5min":8.6750000000000007,"15min":10.475},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.69599999999999995,"5min":1.028,"15min":1.25},"min":{"1min":0.47899999999999998,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":0.89100000000000001,"5min":6.5220000000000002,"15min":9.2400000000000002},"last":1.5269999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.6799999999999999,"15min":1.448},"min":{"1min":0.60499999999999998,"5min":0.32400000000000001,"15min":0.27900000000000003},"max":{"1min":1.3,"5min":20.079000000000001,"15min":20.079000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.556,"15min":1.494},"min":{"1min":0.59199999999999997,"5min":0.32700000000000001,"15min":0.30199999999999999},"max":{"1min":4.5199999999999996,"5min":12.044,"15min":12.044},"last":1.04}]},{"osd":5,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.593,"15min":1.4910000000000001},"min":{"1min":0.61099999999999999,"5min":0.313,"15min":0.28399999999999997},"max":{"1min":4.3760000000000003,"5min":9.4280000000000008,"15min":9.4280000000000008},"last":1.2110000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.667,"15min":1.597},"min":{"1min":0.65200000000000002,"5min":0.33700000000000002,"15min":0.29699999999999999},"max":{"1min":1.2270000000000001,"5min":9.6289999999999996,"15min":9.6289999999999996},"last":2.0739999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":1.373,"15min":1.46},"min":{"1min":0.64000000000000001,"5min":0.372,"15min":0.315},"max":{"1min":1.202,"5min":6.2110000000000003,"15min":10.413},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.8169999999999999,"15min":1.76},"min":{"1min":0.71899999999999997,"5min":0.42599999999999999,"15min":0.35599999999999998},"max":{"1min":4.4900000000000002,"5min":9.3260000000000005,"15min":9.3260000000000005},"last":1.5529999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.2969999999999999,"15min":1.4419999999999999},"min":{"1min":0.55400000000000005,"5min":0.38,"15min":0.38},"max":{"1min":1.171,"5min":6.9669999999999996,"15min":9.2159999999999993},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.288,"5min":1.393,"15min":1.512},"min":{"1min":0.77600000000000002,"5min":0.438,"15min":0.39200000000000002},"max":{"1min":4.4420000000000002,"5min":10.087999999999999,"15min":14.361000000000001},"last":1.1040000000000001}]}]},{"osd":3,"up_from":0,"seq":2847563317674,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986808,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266028,"kb_used_data":370852,"kb_used_omap":0,"kb_used_meta":895168,"kb_avail":92479124,"statfs":{"total":95995035648,"available":94698622976,"internally_reserved":0,"allocated":379752448,"data_stored":379459581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916652032},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:50:44 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":2.4460000000000002,"15min":1.7190000000000001},"min":{"1min":0.54700000000000004,"5min":0.26800000000000002,"15min":0.253},"max":{"1min":1.327,"5min":64.372,"15min":64.372},"last":3.2959999999999998},{"interface":"front","average":{"1min":0.91800000000000004,"5min":2.1859999999999999,"15min":1.6859999999999999},"min":{"1min":0.59599999999999997,"5min":0.30099999999999999,"15min":0.27700000000000002},"max":{"1min":1.3160000000000001,"5min":64.356999999999999,"15min":64.356999999999999},"last":3.3450000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:50:13 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.583,"15min":1.595},"min":{"1min":0.42899999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.4080000000000004,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.006,"15min":0.92100000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.2249999999999996,"5min":5.2249999999999996,"15min":5.2249999999999996},"last":1.5549999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:50:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":1.851,"15min":1.577},"min":{"1min":0.55100000000000005,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":10.471,"5min":12.000999999999999,"15min":33.743000000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.661,"15min":1.5409999999999999},"min":{"1min":0.32000000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.2629999999999999,"5min":7.3730000000000002,"15min":33.805999999999997},"last":0.50800000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.079,"15min":1.133},"min":{"1min":0.64000000000000001,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":1.4059999999999999,"5min":8.1120000000000001,"15min":8.1120000000000001},"last":1.6080000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.4119999999999999,"15min":1.4099999999999999},"min":{"1min":0.53800000000000003,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":2.2269999999999999,"5min":7.4180000000000001,"15min":16.507000000000001},"last":0.46300000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.8540000000000001,"15min":1.645},"min":{"1min":0.60099999999999998,"5min":0.35599999999999998,"15min":0.26900000000000002},"max":{"1min":1.407,"5min":11.763999999999999,"15min":17.212},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.4990000000000001,"15min":1.6000000000000001},"min":{"1min":0.56499999999999995,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.5800000000000001,"5min":12.446999999999999,"15min":17.268999999999998},"last":1.7809999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":2.1440000000000001,"15min":1.6319999999999999},"min":{"1min":0.498,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":1.3140000000000001,"5min":11.701000000000001,"15min":13.654},"last":1.1379999999999999},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.3170000000000002,"15min":1.716},"min":{"1min":0.84099999999999997,"5min":0.41899999999999998,"15min":0.35699999999999998},"max":{"1min":1.518,"5min":11.972,"15min":11.972},"last":1.157}]},{"osd":7,"last update":"Mon Oct 9 09:50:13 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.6599999999999999,"15min":1.3819999999999999},"min":{"1min":0.60299999999999998,"5min":0.373,"15min":0.373},"max":{"1min":5.2709999999999999,"5min":11.840999999999999,"15min":11.840999999999999},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.115,"5min":1.1990000000000001,"15min":1.1850000000000001},"min":{"1min":0.46700000000000003,"5min":0.41299999999999998,"15min":0.34999999999999998},"max":{"1min":2.5720000000000001,"5min":9.6210000000000004,"15min":9.6210000000000004},"last":1.6579999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790574,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1281304,"kb_used_data":374800,"kb_used_omap":0,"kb_used_meta":906496,"kb_avail":92463848,"statfs":{"total":95995035648,"available":94682980352,"internally_reserved":0,"allocated":383795200,"data_stored":383518487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928251904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:50:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":2.4790000000000001,"15min":4.2990000000000004},"min":{"1min":0.60999999999999999,"5min":0.253,"15min":0.23599999999999999},"max":{"1min":1.617,"5min":49.954999999999998,"15min":274.71899999999999},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.93100000000000005,"5min":2.0449999999999999,"15min":4.0750000000000002},"min":{"1min":0.68799999999999994,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":1.375,"5min":50.042000000000002,"15min":274.73500000000001},"last":0.81799999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:50:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":1.853,"15min":1.869},"min":{"1min":0.57599999999999996,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":10.146000000000001,"5min":16.172000000000001,"15min":16.172000000000001},"last":2.157},{"interface":"front","average":{"1min":1.5880000000000001,"5min":1.248,"15min":1.1910000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":13.728,"5min":13.728,"15min":13.728},"last":2.1880000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:50:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.619,"15min":1.6359999999999999},"min":{"1min":0.316,"5min":0.312,"15min":0.312},"max":{"1min":3.5920000000000001,"5min":48.170999999999999,"15min":50.847000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.615,"5min":2.1480000000000001,"15min":1.99},"min":{"1min":0.46400000000000002,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":8.7759999999999998,"5min":48.128,"15min":50.656999999999996},"last":0.52900000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:50:34 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.085,"15min":1.29},"min":{"1min":0.44700000000000001,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":1.298,"5min":7.2270000000000003,"15min":10.125999999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.452,"15min":1.3819999999999999},"min":{"1min":0.66600000000000004,"5min":0.38,"15min":0.29099999999999998},"max":{"1min":2.79,"5min":16.201000000000001,"15min":16.201000000000001},"last":0.73599999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:50:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":1.5600000000000001,"15min":1.3420000000000001},"min":{"1min":0.47899999999999998,"5min":0.309,"15min":0.27000000000000002},"max":{"1min":6.4470000000000001,"5min":9.4209999999999994,"15min":9.4209999999999994},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.478,"5min":1.9730000000000001,"15min":1.724},"min":{"1min":0.39600000000000002,"5min":0.374,"15min":0.33300000000000002},"max":{"1min":3.5449999999999999,"5min":16.053000000000001,"15min":16.053000000000001},"last":0.78000000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:50:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.74,"15min":1.946},"min":{"1min":0.41699999999999998,"5min":0.35399999999999998,"15min":0.30399999999999999},"max":{"1min":1.6779999999999999,"5min":9.6270000000000007,"15min":140.179},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.3149999999999999,"5min":2.0870000000000002,"15min":2.23},"min":{"1min":0.66300000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.9470000000000001,"5min":16.145,"15min":137.928},"last":0.94099999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:50:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":1.647,"15min":1.6259999999999999},"min":{"1min":0.501,"5min":0.432,"15min":0.33400000000000002},"max":{"1min":4.3040000000000003,"5min":16.119,"15min":18.683},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.1080000000000001,"15min":1.2230000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32100000000000001},"max":{"1min":1.734,"5min":4.6779999999999999,"15min":10.192},"last":0.54700000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7721,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":564461568,"data_stored":564461568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":927531008,"data_stored":927531008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":523829248,"data_stored":523829248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":404684800,"data_stored":404684800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":393478144,"data_stored":393478144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":740622336,"data_stored":740622336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:51:04.997 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:51:05.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:05.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:05.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:05.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:05.193 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:05.223 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:05.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:05.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:05.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:05.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:05.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:05.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:05.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 34129 34113 125.404 96.5 0.0121188 0.00796781 2023-10-09T09:51:05.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:05.486 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:05.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:05.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:05.588 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:05.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:05.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:05.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:05.744 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:05.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:05.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:05.847 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:05.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:05.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:05.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:05.987 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:05.988 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:05 smithi181 bash[59345]: cluster 2023-10-09T09:51:04.190069+0000 mgr.y (mgr.34107) 1354 : cluster 0 pgmap v2222: 89 pgs: 89 active+clean; 1.7 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T09:51:06.011 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:06.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:06.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:06.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:06 smithi163 bash[78107]: cluster 2023-10-09T09:51:04.190069+0000 mgr.y (mgr.34107) 1354 : cluster 0 pgmap v2222: 89 pgs: 89 active+clean; 1.7 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T09:51:06.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:06 smithi163 bash[75425]: cluster 2023-10-09T09:51:04.190069+0000 mgr.y (mgr.34107) 1354 : cluster 0 pgmap v2222: 89 pgs: 89 active+clean; 1.7 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T09:51:06.115 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:06.140 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:06.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:06.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:06.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:06.270 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:06.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:06.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:06.375 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:06.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:06.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:06.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:06.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 15 35728 35713 123.988 100 0.0165825 0.00805732 2023-10-09T09:51:06.506 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:06.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:06.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:06.638 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:06.662 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:06.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:06.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:06.766 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:06.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:06.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:06.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:06.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:06.926 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:06.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:06.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:07.030 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:07.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:07.062 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:07.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:07.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:07.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:07.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:07.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:07.296 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:07.323 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:07.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:07.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:07.405 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 37288 37272 122.59 97.4375 0.00374249 0.00815044 2023-10-09T09:51:07.428 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:07.453 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:07.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:07.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:07.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:07.584 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:07.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:07.690 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:07.725 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:07.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:07.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:07.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:07.859 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:07.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:07.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:07.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:07.991 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:07.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:07.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:08.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:08.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:08.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:08.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:08.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:07 smithi163 bash[75425]: cluster 2023-10-09T09:51:06.190676+0000 mgr.y (mgr.34107) 1355 : cluster 0 pgmap v2223: 89 pgs: 89 active+clean; 2.0 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T09:51:08.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:07 smithi163 bash[78107]: cluster 2023-10-09T09:51:06.190676+0000 mgr.y (mgr.34107) 1355 : cluster 0 pgmap v2223: 89 pgs: 89 active+clean; 2.0 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T09:51:08.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:08.256 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:08.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:08.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:08.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:07 smithi181 bash[59345]: cluster 2023-10-09T09:51:06.190676+0000 mgr.y (mgr.34107) 1355 : cluster 0 pgmap v2223: 89 pgs: 89 active+clean; 2.0 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T09:51:08.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:08.393 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:08.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:08.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:08.405 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:51:08.405674+0000 min lat: 0.00138896 max lat: 0.291612 avg lat: 0.00825245 2023-10-09T09:51:08.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:51:08.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 38770 38754 121.091 92.625 0.00641989 0.00825245 2023-10-09T09:51:08.497 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:08.526 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:08.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:08.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:08.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:08.666 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:08.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:08.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:08.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:08.797 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:08.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:08.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:08.827 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:51:08.876 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:51:08.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:08.917 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:51:08.949 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:51:08.961 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:08.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:08.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:08.985 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:51:09.019 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:51:09.055 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:51:09.056 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:51:09.061 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:51:09.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:09 smithi181 bash[59345]: audit 2023-10-09T09:51:08.781956+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.163:0/3269366158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:51:09.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:09.093 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:09.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:09.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:09.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:51:09.109 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:51:09.109 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:51:09.110 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:51:09.110 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:51:09.110 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:51:09.110 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:51:09.110 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:51:09.110 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:51:09.198 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:09.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:09 smithi163 bash[78107]: audit 2023-10-09T09:51:08.781956+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.163:0/3269366158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:51:09.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:09 smithi163 bash[75425]: audit 2023-10-09T09:51:08.781956+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.163:0/3269366158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:51:09.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:09.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:09.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:09.334 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:09.367 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:09.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:09.405 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 39913 39897 118.727 71.4375 0.0103631 0.00841712 2023-10-09T09:51:09.475 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:09.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:09.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:09.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:09.608 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:09.639 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:09.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:09.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:09.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:09.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:09.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:09.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:09.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:09.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:09.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:09.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:10.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:10.013 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:10 smithi181 bash[59345]: cluster 2023-10-09T09:51:08.191299+0000 mgr.y (mgr.34107) 1356 : cluster 0 pgmap v2224: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T09:51:10.013 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:10 smithi181 bash[59345]: audit 2023-10-09T09:51:09.195513+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.163:0/2767364382' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:51:10.013 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:10 smithi181 bash[59345]: audit 2023-10-09T09:51:09.196706+0000 mon.a (mon.0) 2471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:51:10.015 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:51:10.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:10.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:10.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:10.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:10.175 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:10.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:10.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:10.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:10 smithi163 bash[78107]: cluster 2023-10-09T09:51:08.191299+0000 mgr.y (mgr.34107) 1356 : cluster 0 pgmap v2224: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T09:51:10.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:10 smithi163 bash[78107]: audit 2023-10-09T09:51:09.195513+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.163:0/2767364382' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:51:10.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:10 smithi163 bash[78107]: audit 2023-10-09T09:51:09.196706+0000 mon.a (mon.0) 2471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:51:10.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:10 smithi163 bash[75425]: cluster 2023-10-09T09:51:08.191299+0000 mgr.y (mgr.34107) 1356 : cluster 0 pgmap v2224: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T09:51:10.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:10 smithi163 bash[75425]: audit 2023-10-09T09:51:09.195513+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.163:0/2767364382' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:51:10.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:10 smithi163 bash[75425]: audit 2023-10-09T09:51:09.196706+0000 mon.a (mon.0) 2471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:51:10.279 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:10.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:10.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:10.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:10.405 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 41050 41034 116.56 71.0625 0.0160651 0.00857242 2023-10-09T09:51:10.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:10.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:10.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:10.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:10.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:10.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:10.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:10.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:10.672 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:10.698 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:10.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:10.804 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:10.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:10.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:10.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:10.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:10.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:10.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:10.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:11.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:11.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:11 smithi163 bash[78107]: cluster 2023-10-09T09:51:10.001065+0000 mon.a (mon.0) 2472 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:51:11.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:11 smithi163 bash[78107]: cluster 2023-10-09T09:51:10.005015+0000 osd.5 (osd.5) 865 : cluster 3 failed to encode map e1275 with expected crc 2023-10-09T09:51:11.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:11 smithi163 bash[78107]: audit 2023-10-09T09:51:10.007959+0000 mon.a (mon.0) 2473 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:51:11.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:11 smithi163 bash[78107]: cluster 2023-10-09T09:51:10.008248+0000 mon.a (mon.0) 2474 : cluster 0 osdmap e1275: 8 total, 8 up, 6 in 2023-10-09T09:51:11.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:11 smithi163 bash[78107]: audit 2023-10-09T09:51:10.400886+0000 mon.a (mon.0) 2475 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:11.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:11 smithi163 bash[75425]: cluster 2023-10-09T09:51:10.001065+0000 mon.a (mon.0) 2472 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:51:11.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:11 smithi163 bash[75425]: cluster 2023-10-09T09:51:10.005015+0000 osd.5 (osd.5) 865 : cluster 3 failed to encode map e1275 with expected crc 2023-10-09T09:51:11.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:11 smithi163 bash[75425]: audit 2023-10-09T09:51:10.007959+0000 mon.a (mon.0) 2473 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:51:11.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:11 smithi163 bash[75425]: cluster 2023-10-09T09:51:10.008248+0000 mon.a (mon.0) 2474 : cluster 0 osdmap e1275: 8 total, 8 up, 6 in 2023-10-09T09:51:11.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:11 smithi163 bash[75425]: audit 2023-10-09T09:51:10.400886+0000 mon.a (mon.0) 2475 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:11.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:11.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:11.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:11.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:11.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:11.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:11.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:11.309 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:11 smithi181 bash[59345]: cluster 2023-10-09T09:51:10.001065+0000 mon.a (mon.0) 2472 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:51:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:11 smithi181 bash[59345]: cluster 2023-10-09T09:51:10.005015+0000 osd.5 (osd.5) 865 : cluster 3 failed to encode map e1275 with expected crc 2023-10-09T09:51:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:11 smithi181 bash[59345]: audit 2023-10-09T09:51:10.007959+0000 mon.a (mon.0) 2473 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:51:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:11 smithi181 bash[59345]: cluster 2023-10-09T09:51:10.008248+0000 mon.a (mon.0) 2474 : cluster 0 osdmap e1275: 8 total, 8 up, 6 in 2023-10-09T09:51:11.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:11 smithi181 bash[59345]: audit 2023-10-09T09:51:10.400886+0000 mon.a (mon.0) 2475 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:11.338 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:11.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:11.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 42178 42162 114.557 70.5 0.00975367 0.00872378 2023-10-09T09:51:11.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:11.465 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:11.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:11.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:11.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:11.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:11.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:11.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:11.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:11.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:11.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:11.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:11.833 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:11.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:11.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:11.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:11.994 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:11.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:11.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:12.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:12.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:12.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:12 smithi163 bash[75425]: cluster 2023-10-09T09:51:10.191768+0000 mgr.y (mgr.34107) 1357 : cluster 0 pgmap v2226: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-09T09:51:12.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:12.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:12.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:12 smithi163 bash[78107]: cluster 2023-10-09T09:51:10.191768+0000 mgr.y (mgr.34107) 1357 : cluster 0 pgmap v2226: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-09T09:51:12.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:12.235 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:12 smithi181 bash[59345]: cluster 2023-10-09T09:51:10.191768+0000 mgr.y (mgr.34107) 1357 : cluster 0 pgmap v2226: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-09T09:51:12.255 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:12.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:12.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:12.360 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:12.388 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:12.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:12.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 43450 43434 113.096 79.5 0.00266632 0.0087897 2023-10-09T09:51:12.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:12.519 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:12.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:12.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:12.622 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:12.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:12.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:12.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:12.762 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:12.786 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:12.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:12.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:12.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:12.916 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:12.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:12.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:13.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:13.050 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:13.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:13.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:13.182 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:13.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:13.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:13.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:13.313 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:13.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:13.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:13.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 44641 44625 111.549 74.4375 0.00734228 0.00895942 2023-10-09T09:51:13.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:13.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:13.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:13.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:51:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:51:13] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.33.4" 2023-10-09T09:51:13.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:13.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:13.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:13.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:13.673 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:13.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:13.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:13.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:13.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:13.825 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:13.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:13.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:13.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:13.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:13.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:14.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:14 smithi181 bash[59345]: cluster 2023-10-09T09:51:12.192393+0000 mgr.y (mgr.34107) 1358 : cluster 0 pgmap v2227: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:51:14.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:14.085 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:14.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:14.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:14.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:14.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:14.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:14.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:14.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:14 smithi163 bash[75425]: cluster 2023-10-09T09:51:12.192393+0000 mgr.y (mgr.34107) 1358 : cluster 0 pgmap v2227: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:51:14.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:14 smithi163 bash[78107]: cluster 2023-10-09T09:51:12.192393+0000 mgr.y (mgr.34107) 1358 : cluster 0 pgmap v2227: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:51:14.318 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:14.349 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:14.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:14.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:14.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 15 45812 45797 110.076 73.25 0.00756316 0.00907845 2023-10-09T09:51:14.451 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:14.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:14.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:14.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:14.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:14.617 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:14.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:14.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:14.760 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:14.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:14.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:14.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:14.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:14.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:14.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:15.023 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:15.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:15.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:15.127 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:15.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:15.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:15.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:15.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:15.293 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:15.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:15.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:15.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:15.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 47049 47033 108.86 77.25 0.0145155 0.0091785 2023-10-09T09:51:15.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:15.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:15.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:15.528 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:15.556 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:15.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:15.664 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:15.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:15.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:15.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:15.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:15.820 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:15.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:15.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:15.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:15.957 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:15.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:16.064 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:16.073 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:16 smithi163 bash[75425]: cluster 2023-10-09T09:51:14.192876+0000 mgr.y (mgr.34107) 1359 : cluster 0 pgmap v2228: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T09:51:16.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:16 smithi163 bash[78107]: cluster 2023-10-09T09:51:14.192876+0000 mgr.y (mgr.34107) 1359 : cluster 0 pgmap v2228: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T09:51:16.090 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:16.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:16.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:16.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:16.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:16.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:16.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:16.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:16 smithi181 bash[59345]: cluster 2023-10-09T09:51:14.192876+0000 mgr.y (mgr.34107) 1359 : cluster 0 pgmap v2228: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T09:51:16.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:16.365 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:16.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:16.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:16.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 48349 48333 107.874 81.25 0.00900423 0.00926471 2023-10-09T09:51:16.470 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:16.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:16.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:16.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:16.603 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:16.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:16.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:16.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:16.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:51:16.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:16.766 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:16.769 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:51:16.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:16.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:16.871 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:16.907 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:16.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:16.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:17.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:17.035 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:17.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:17.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:17.172 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:17.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:17.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:17.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:17.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:17.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:17.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:17.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 49432 49416 106.488 67.6875 0.0214044 0.00938457 2023-10-09T09:51:17.431 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:17.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:17.535 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:17.561 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:17.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:17.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:17.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:17.693 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:17.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:17.796 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:17.832 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:17.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:17.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:17.935 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:17.959 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:17.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:17.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:18.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:18.071 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:18 smithi163 bash[75425]: cluster 2023-10-09T09:51:16.193491+0000 mgr.y (mgr.34107) 1360 : cluster 0 pgmap v2229: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:51:18.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:18 smithi163 bash[78107]: cluster 2023-10-09T09:51:16.193491+0000 mgr.y (mgr.34107) 1360 : cluster 0 pgmap v2229: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:51:18.093 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:18.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:18.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:18.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:18.208 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:18 smithi181 bash[59345]: cluster 2023-10-09T09:51:16.193491+0000 mgr.y (mgr.34107) 1360 : cluster 0 pgmap v2229: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:51:18.223 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:18.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:18.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:18.327 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:18.351 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:18.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:18.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:18.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 50702 50686 105.584 79.375 0.0083127 0.00946585 2023-10-09T09:51:18.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:18.478 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:18.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:18.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:18.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:18.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:18.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:18.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:18.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:18.748 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:18.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:18.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:18.853 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:18.877 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:18.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:18.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:18.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:19.009 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:19.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:19.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:19.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:19.144 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:19.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:19.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:19.277 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:19.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:19.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:19.385 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:19.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 52028 52012 104.848 82.875 0.0198108 0.00953116 2023-10-09T09:51:19.416 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:19.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:19.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:19.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:19.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:19.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:19.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:19.664 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:19.692 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:19.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:19.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:19.796 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:19.824 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:19.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:19.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:19.951 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:19.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:19.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:20.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:20.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:20.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:20.196 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:20.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:20.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:20 smithi181 bash[59345]: cluster 2023-10-09T09:51:18.194351+0000 mgr.y (mgr.34107) 1361 : cluster 0 pgmap v2230: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T09:51:20.331 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:20.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:20 smithi163 bash[78107]: cluster 2023-10-09T09:51:18.194351+0000 mgr.y (mgr.34107) 1361 : cluster 0 pgmap v2230: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T09:51:20.340 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:20 smithi163 bash[75425]: cluster 2023-10-09T09:51:18.194351+0000 mgr.y (mgr.34107) 1361 : cluster 0 pgmap v2230: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T09:51:20.364 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:20.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:20.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:20.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 53245 53229 103.948 76.0625 0.0128804 0.0096144 2023-10-09T09:51:20.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:20.494 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:20.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:20.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:20.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:20.598 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:51:20.598 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:51:20.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:51:20.598 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:51:20.598 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.598 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:151552 10.2 10.cd5d1cc2 (undecoded) ondisk+retry+write+known_if_redirected e1246)", 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:06.973052+0000", 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 193.62109920399999, 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99834798800000002, 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151552 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.599 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973052+0000", 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973052+0000", 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.600 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973050+0000", 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973057+0000", 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2040000000000002e-06 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973079+0000", 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1994999999999999e-05 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.601 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973089+0000", 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0094e-05 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973138+0000", 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8884999999999997e-05 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.602 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973154+0000", 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6475999999999999e-05 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.971386+0000", 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99823186699999999 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.971400+0000", 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4035e-05 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.603 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:151559 10.2 10.db1cafd2 (undecoded) ondisk+retry+write+known_if_redirected e1246)", 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:06.973100+0000", 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 193.62105102300001, 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99833410499999997, 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.604 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151559 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973100+0000", 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973100+0000", 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973100+0000", 2023-10-09T09:51:20.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:51:20.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.608 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973103+0000", 2023-10-09T09:51:20.608 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8070000000000001e-06 2023-10-09T09:51:20.608 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.608 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.608 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.608 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973111+0000", 2023-10-09T09:51:20.608 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4669999999999998e-06 2023-10-09T09:51:20.608 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.608 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973117+0000", 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3539999999999996e-06 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973173+0000", 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5609000000000002e-05 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973178+0000", 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1279999999999999e-06 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.971425+0000", 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.998246945 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.971434+0000", 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3519999999999996e-06 2023-10-09T09:51:20.610 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:151560 10.1 10.ed0c51c1 (undecoded) ondisk+retry+write+known_if_redirected e1246)", 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:06.973122+0000", 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 193.621029693, 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98378926899999997, 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.611 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151560 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973122+0000", 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973122+0000", 2023-10-09T09:51:20.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973121+0000", 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973124+0000", 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0319999999999999e-06 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.613 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973130+0000", 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1040000000000003e-06 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973135+0000", 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2190000000000003e-06 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.614 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.975549+0000", 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024139769999999999 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.975560+0000", 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0217000000000001e-05 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.956894+0000", 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.981334657 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.615 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.956911+0000", 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6497000000000001e-05 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:151568 10.2 10.d976b002 (undecoded) ondisk+write+known_if_redirected e1246)", 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:06.973639+0000", 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 193.62051241, 2023-10-09T09:51:20.616 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99783578699999997, 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151568 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973639+0000", 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.617 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973639+0000", 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973638+0000", 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.973643+0000", 2023-10-09T09:51:20.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8940000000000001e-06 2023-10-09T09:51:20.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.975562+0000", 2023-10-09T09:51:20.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019185179999999999 2023-10-09T09:51:20.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.975570+0000", 2023-10-09T09:51:20.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5159999999999994e-06 2023-10-09T09:51:20.622 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.622 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.622 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.975770+0000", 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000199228 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.975780+0000", 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0942e-05 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.971465+0000", 2023-10-09T09:51:20.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99568438599999998 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.971475+0000", 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8099999999999992e-06 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:151573 10.1 10.3b21c2a1 (undecoded) ondisk+write+known_if_redirected e1246)", 2023-10-09T09:51:20.624 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:06.979575+0000", 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 193.61457612800001, 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97759422200000001, 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151573 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.625 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.979575+0000", 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.979575+0000", 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.979573+0000", 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:51:20.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.979580+0000", 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4330000000000001e-06 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.979590+0000", 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.027e-05 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.979599+0000", 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9539999999999993e-06 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.981089+0000", 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001489619 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.981101+0000", 2023-10-09T09:51:20.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2714000000000001e-05 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.957157+0000", 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97605524899999996 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.957169+0000", 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2765999999999999e-05 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.629 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:151580 10.2 10.b9d41032 (undecoded) ondisk+write+known_if_redirected e1246)", 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:06.987881+0000", 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 193.606269881, 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98512333299999999, 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.630 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151580 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987881+0000", 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987881+0000", 2023-10-09T09:51:20.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987880+0000", 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987887+0000", 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5529999999999998e-06 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987901+0000", 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4654000000000001e-05 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987908+0000", 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0910000000000004e-06 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987950+0000", 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2168e-05 2023-10-09T09:51:20.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987958+0000", 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5689999999999999e-06 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.972989+0000", 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98503079100000002 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.973005+0000", 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5894999999999999e-05 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:151581 10.2 10.4e8700a2 (undecoded) ondisk+write+known_if_redirected e1246)", 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:06.987969+0000", 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 193.606182166, 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98506986500000004, 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.635 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151581 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987969+0000", 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987969+0000", 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987968+0000", 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987975+0000", 2023-10-09T09:51:20.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7349999999999996e-06 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987988+0000", 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2860000000000001e-05 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.987998+0000", 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7669999999999998e-06 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.988027+0000", 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9323e-05 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.988032+0000", 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6210000000000003e-06 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.973029+0000", 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98499684899999995 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.973039+0000", 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0124e-05 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:151583 10.1 10.a6971241 (undecoded) ondisk+write+known_if_redirected e1246)", 2023-10-09T09:51:20.640 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:06.990137+0000", 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 193.60401377900001, 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96706904999999999, 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151583 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.990137+0000", 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.990137+0000", 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.990137+0000", 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:51:20.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.990143+0000", 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2899999999999999e-06 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.990154+0000", 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1538e-05 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.990162+0000", 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1369999999999996e-06 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.990201+0000", 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9826000000000001e-05 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:06.990208+0000", 2023-10-09T09:51:20.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2979999999999997e-06 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.957198+0000", 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96698991099999998 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.957207+0000", 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8890000000000008e-06 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.645 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:151594 10.2 10.cebd8282 (undecoded) ondisk+write+known_if_redirected e1246)", 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:07.016658+0000", 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 193.57749372500001, 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95641912299999998, 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151594 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.016658+0000", 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.016658+0000", 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.016655+0000", 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999971 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.016668+0000", 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2917e-05 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.016701+0000", 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3456e-05 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.016717+0000", 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5668e-05 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.016795+0000", 2023-10-09T09:51:20.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7643000000000002e-05 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.016804+0000", 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5659999999999996e-06 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.973066+0000", 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95626194799999997 2023-10-09T09:51:20.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:07.973077+0000", 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0638999999999999e-05 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:206965 10.1 10:8a31bece:::benchmark_data_smithi163_194945_object97512:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e1258)", 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.043478+0000", 2023-10-09T09:51:20.651 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.55067308300001, 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.03414775, 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 206965 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.043478+0000", 2023-10-09T09:51:20.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.043478+0000", 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.043476+0000", 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.043487+0000", 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1004e-05 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.043505+0000", 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7728999999999999e-05 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.043515+0000", 2023-10-09T09:51:20.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0383999999999999e-05 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.043593+0000", 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7766e-05 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.043607+0000", 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4212e-05 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.077571+0000", 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0339644269999999 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.077626+0000", 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4447999999999998e-05 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:206986 10.1 10:8f50cb9e:::benchmark_data_smithi163_194945_object97533:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1258)", 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.062214+0000", 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.53193721, 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0336791679999999, 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 206986 2023-10-09T09:51:20.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.062214+0000", 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.062214+0000", 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.062212+0000", 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.062219+0000", 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4030000000000004e-06 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.062231+0000", 2023-10-09T09:51:20.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1983e-05 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.062238+0000", 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4159999999999998e-06 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.062278+0000", 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9879000000000002e-05 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.062295+0000", 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.698e-05 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.078112+0000", 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0158173690000001 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.078234+0000", 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121342 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.078761+0000", 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00052723500000000003 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.095702+0000", 2023-10-09T09:51:20.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016940526000000001 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.095783+0000", 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1032000000000006e-05 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.095791+0000", 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4549999999999995e-06 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.095845+0000", 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3470999999999997e-05 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.095893+0000", 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8664e-05 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:206990 10.1 10:8a9c52ce:::benchmark_data_smithi163_194945_object97537:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1258)", 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.066502+0000", 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.527649544, 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.042679087, 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 206990 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.665 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.066502+0000", 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.066502+0000", 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.066500+0000", 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.066508+0000", 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0809999999999994e-06 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.066526+0000", 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7680999999999999e-05 2023-10-09T09:51:20.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.066536+0000", 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0353000000000001e-05 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.068485+0000", 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019485629999999999 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.068493+0000", 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9519999999999994e-06 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.093366+0000", 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0248735680000001 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.093459+0000", 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2826999999999997e-05 2023-10-09T09:51:20.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.093540+0000", 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1156999999999996e-05 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.106309+0000", 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012768893999999999 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109091+0000", 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027823639999999998 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109122+0000", 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1046000000000001e-05 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109154+0000", 2023-10-09T09:51:20.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1384000000000003e-05 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109181+0000", 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6974e-05 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207022 10.1 10:8bc95e91:::benchmark_data_smithi163_194945_object97569:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1258)", 2023-10-09T09:51:20.672 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.113108+0000", 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.481043392, 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99632531999999996, 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207022 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.113108+0000", 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.113108+0000", 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.113105+0000", 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999967 2023-10-09T09:51:20.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.113116+0000", 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1589e-05 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.113139+0000", 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3125999999999998e-05 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.113152+0000", 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2478e-05 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.113223+0000", 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1098000000000003e-05 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.113245+0000", 2023-10-09T09:51:20.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2084000000000001e-05 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.095944+0000", 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98269875399999995 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.096086+0000", 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014195700000000001 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.096195+0000", 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000109143 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109038+0000", 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012843538 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109257+0000", 2023-10-09T09:51:20.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000218505 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109327+0000", 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9711999999999999e-05 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109388+0000", 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1099000000000006e-05 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109433+0000", 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5426000000000002e-05 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207027 10.1 10:81ca0ddc:::benchmark_data_smithi163_194945_object97574:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1258)", 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.115827+0000", 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.47832413099999, 2023-10-09T09:51:20.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99380943399999999, 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207027 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.115827+0000", 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.115827+0000", 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.115826+0000", 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.115835+0000", 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0620000000000007e-06 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.115865+0000", 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0406e-05 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.115879+0000", 2023-10-09T09:51:20.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4161e-05 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.115963+0000", 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3489e-05 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.115975+0000", 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2354000000000001e-05 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.098428+0000", 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98245279900000004 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.098539+0000", 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000110881 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.098664+0000", 2023-10-09T09:51:20.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000125382 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109044+0000", 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010379849 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109494+0000", 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00044991200000000001 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109539+0000", 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5420999999999998e-05 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109593+0000", 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3749999999999999e-05 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109637+0000", 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3457000000000001e-05 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207049 10.1 10:87baf418:::benchmark_data_smithi163_194945_object97596:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1258)", 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.147644+0000", 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.446507647, 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97566123599999999, 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:51:20.688 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207049 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.147644+0000", 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.147644+0000", 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.147641+0000", 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.147653+0000", 2023-10-09T09:51:20.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1831000000000001e-05 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.147682+0000", 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8779e-05 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.147692+0000", 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0331e-05 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.147759+0000", 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6637000000000006e-05 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.147772+0000", 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2594e-05 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.101105+0000", 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.953333235 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.101224+0000", 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011875399999999999 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.101354+0000", 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000130541 2023-10-09T09:51:20.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109051+0000", 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.007697181 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.123139+0000", 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014088101 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.123212+0000", 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3063000000000002e-05 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.123264+0000", 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1517999999999997e-05 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.123305+0000", 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0871999999999998e-05 2023-10-09T09:51:20.695 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207078 10.1 10:8d5524eb:::benchmark_data_smithi163_194945_object97625:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1258)", 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.198900+0000", 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.395251096, 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.92456187499999998, 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.696 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207078 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.198900+0000", 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.198900+0000", 2023-10-09T09:51:20.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.198898+0000", 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.198907+0000", 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3410000000000003e-06 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.198924+0000", 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7416999999999999e-05 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.198935+0000", 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.045e-05 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.199017+0000", 2023-10-09T09:51:20.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2363999999999999e-05 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.199031+0000", 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4025999999999999e-05 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.103465+0000", 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.90443428299999995 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.103618+0000", 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015249600000000001 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.103764+0000", 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000146052 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:51:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109199+0000", 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005435601 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.123179+0000", 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013979877999999999 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.123383+0000", 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020326599999999999 2023-10-09T09:51:20.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.123425+0000", 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2171e-05 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.123462+0000", 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7324999999999999e-05 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207082 10.1 10:875638dc:::benchmark_data_smithi163_194945_object97629:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1258)", 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.202577+0000", 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.39157455900001, 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.92716097600000003, 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207082 2023-10-09T09:51:20.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.202577+0000", 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.202577+0000", 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.202575+0000", 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.202586+0000", 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0985000000000001e-05 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.202601+0000", 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4939e-05 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.202611+0000", 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0203999999999999e-05 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.202678+0000", 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6196000000000001e-05 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.202689+0000", 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1304000000000001e-05 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.106336+0000", 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.90364744100000005 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:51:20.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.106464+0000", 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012762099999999999 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.106600+0000", 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000135649 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109450+0000", 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028506959999999998 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.123347+0000", 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013896550000000001 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.129642+0000", 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062950799999999998 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:51:20.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.129695+0000", 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2536e-05 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.129738+0000", 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3171999999999999e-05 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.711 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207119 10.1 10:89d03f63:::benchmark_data_smithi163_194945_object97666:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1258)", 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.269877+0000", 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.324274429, 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.860134544, 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207119 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.269877+0000", 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.269877+0000", 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.269875+0000", 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:51:20.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.269882+0000", 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8870000000000003e-06 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.269896+0000", 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3701e-05 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.269904+0000", 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4370000000000007e-06 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.269951+0000", 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6536000000000003e-05 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.269960+0000", 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7690000000000002e-06 2023-10-09T09:51:20.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109675+0000", 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.83971546200000002 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109809+0000", 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013421 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.109956+0000", 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014658 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.123504+0000", 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013548183 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.129600+0000", 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060963479999999997 2023-10-09T09:51:20.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.129883+0000", 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00028280699999999999 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.129950+0000", 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6694999999999998e-05 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.130011+0000", 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1585000000000001e-05 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207125 10.1 10:897fb76a:::benchmark_data_smithi163_194945_object97672:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1258)", 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.277741+0000", 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.31641010200001, 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.85253909299999997, 2023-10-09T09:51:20.719 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207125 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.277741+0000", 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.277741+0000", 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.277739+0000", 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.277759+0000", 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9290999999999999e-05 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.277773+0000", 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4902000000000001e-05 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.277781+0000", 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.96e-06 2023-10-09T09:51:20.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.277850+0000", 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8825999999999995e-05 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.277864+0000", 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3890000000000001e-05 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.112688+0000", 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.83482412100000003 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.112808+0000", 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000119239 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.112923+0000", 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011548 2023-10-09T09:51:20.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.129589+0000", 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016666311 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.129607+0000", 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8179e-05 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.130106+0000", 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000498939 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.130166+0000", 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9598000000000001e-05 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.130280+0000", 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011423 2023-10-09T09:51:20.726 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:207149 10.1 10:894ec768:::benchmark_data_smithi163_194945_object97696:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1258)", 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:49:24.318408+0000", 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 116.27574322700001, 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.82132828300000005, 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:51:20.727 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207149 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.318408+0000", 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:51:20.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.318408+0000", 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.318406+0000", 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.318416+0000", 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3519999999999996e-06 2023-10-09T09:51:20.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.318432+0000", 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6691999999999999e-05 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.318441+0000", 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1400000000000006e-06 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.318490+0000", 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8136999999999997e-05 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:24.318498+0000", 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9230000000000008e-06 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.115396+0000", 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.79689774099999999 2023-10-09T09:51:20.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.115476+0000", 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9987000000000005e-05 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.115542+0000", 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6082999999999997e-05 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.130034+0000", 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014491736 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.139615+0000", 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0095807289999999996 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.139657+0000", 2023-10-09T09:51:20.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1959000000000003e-05 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.139702+0000", 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5388e-05 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:49:25.139736+0000", 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4246999999999999e-05 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.734 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.735 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:51:20.735 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:51:20.735 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:51:20.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:20.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:20.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:20.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:20.866 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:20.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:20.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:20.949 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:51:20.951 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:51:20.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:20.998 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:21.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:21.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:21.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:21.123 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:21.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:21.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:21.228 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:21.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:21.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:21.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:21.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:21.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:21.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:21.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:21.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 54408 54392 103.001 72.6875 0.00927425 0.00970271 2023-10-09T09:51:21.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:21.528 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:21.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:21.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:21.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:21.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:21.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:21.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:21.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:21.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:21.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:21.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:21.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:21.919 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:21.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:22.023 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:22.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:22.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:22.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:22.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:22.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:22.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:22.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:22.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:22 smithi163 bash[75425]: cluster 2023-10-09T09:51:20.194758+0000 mgr.y (mgr.34107) 1362 : cluster 0 pgmap v2231: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:51:22.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:22 smithi163 bash[75425]: audit 2023-10-09T09:51:20.924370+0000 mgr.y (mgr.34107) 1363 : audit 0 from='client.45779 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:51:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:22 smithi163 bash[78107]: cluster 2023-10-09T09:51:20.194758+0000 mgr.y (mgr.34107) 1362 : cluster 0 pgmap v2231: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:51:22.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:22 smithi163 bash[78107]: audit 2023-10-09T09:51:20.924370+0000 mgr.y (mgr.34107) 1363 : audit 0 from='client.45779 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:51:22.291 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:22 smithi181 bash[59345]: cluster 2023-10-09T09:51:20.194758+0000 mgr.y (mgr.34107) 1362 : cluster 0 pgmap v2231: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:51:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:22 smithi181 bash[59345]: audit 2023-10-09T09:51:20.924370+0000 mgr.y (mgr.34107) 1363 : audit 0 from='client.45779 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:51:22.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:22.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:22.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:22.408 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 55374 55358 101.747 60.375 0.0094883 0.00982197 2023-10-09T09:51:22.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:22.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:22.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:22.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:22.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:22.587 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:22.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:22.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:22.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:22.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:22.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:22.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:22.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:22.856 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:22.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:22.961 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:22.990 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:22.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:23.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:23.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:23.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:23.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:23.294 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:23.322 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:23.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:23.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:23.361 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:51:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:51:23] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.33.4" 2023-10-09T09:51:23.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T09:51:23.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 56596 56580 101.022 76.375 0.0118515 0.00989334 2023-10-09T09:51:23.435 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:23.456 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2231,"stamp":"2023-10-09T09:51:20.194527+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3178895264,"num_objects":48521,"num_object_clones":0,"num_object_copies":145563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":48521,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":48757,"num_write_kb":3106253,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":592828,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":48691,"ondisk_log_size":48691,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13714748,"kb_used_data":9334924,"kb_used_omap":607,"kb_used_meta":4379168,"kb_avail":548756164,"statfs":{"total":575970213888,"available":561926311936,"internally_reserved":0,"allocated":9558962176,"data_stored":9553535108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":621657,"internal_metadata":4484268967},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,8,4,6,8],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":823263232,"num_objects":12562,"num_object_clones":0,"num_object_copies":37686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12562,"num_write_kb":803968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.189755"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":53,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.508099+0000","last_change":"2023-10-09T09:48:07.991289+0000","last_active":"2023-10-09T09:50:44.508099+0000","last_peered":"2023-10-09T09:50:44.508099+0000","last_clean":"2023-10-09T09:50:44.508099+0000","last_became_active":"2023-10-09T09:48:07.990686+0000","last_became_peered":"2023-10-09T09:48:07.990686+0000","last_unstale":"2023-10-09T09:50:44.508099+0000","last_undegraded":"2023-10-09T09:50:44.508099+0000","last_fullsized":"2023-10-09T09:50:44.508099+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:24.421374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":156,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507962+0000","last_change":"2023-10-09T09:48:07.999947+0000","last_active":"2023-10-09T09:50:44.507962+0000","last_peered":"2023-10-09T09:50:44.507962+0000","last_clean":"2023-10-09T09:50:44.507962+0000","last_became_active":"2023-10-09T09:48:07.999704+0000","last_became_peered":"2023-10-09T09:48:07.999704+0000","last_unstale":"2023-10-09T09:50:44.507962+0000","last_undegraded":"2023-10-09T09:50:44.507962+0000","last_fullsized":"2023-10-09T09:50:44.507962+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:14.832042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1225,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.508360+0000","last_change":"2023-10-09T09:48:07.998214+0000","last_active":"2023-10-09T09:50:44.508360+0000","last_peered":"2023-10-09T09:50:44.508360+0000","last_clean":"2023-10-09T09:50:44.508360+0000","last_became_active":"2023-10-09T09:48:07.997544+0000","last_became_peered":"2023-10-09T09:48:07.997544+0000","last_unstale":"2023-10-09T09:50:44.508360+0000","last_undegraded":"2023-10-09T09:50:44.508360+0000","last_fullsized":"2023-10-09T09:50:44.508360+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.304000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1235,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.249662+0000","last_change":"2023-10-09T09:48:07.980358+0000","last_active":"2023-10-09T09:51:10.249662+0000","last_peered":"2023-10-09T09:51:10.249662+0000","last_clean":"2023-10-09T09:51:10.249662+0000","last_became_active":"2023-10-09T09:48:07.979229+0000","last_became_peered":"2023-10-09T09:48:07.979229+0000","last_unstale":"2023-10-09T09:51:10.249662+0000","last_undegraded":"2023-10-09T09:51:10.249662+0000","last_fullsized":"2023-10-09T09:51:10.249662+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:31.456347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":768,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.036576+0000","last_change":"2023-10-09T09:48:07.969728+0000","last_active":"2023-10-09T09:50:45.036576+0000","last_peered":"2023-10-09T09:50:45.036576+0000","last_clean":"2023-10-09T09:50:45.036576+0000","last_became_active":"2023-10-09T09:48:07.968773+0000","last_became_peered":"2023-10-09T09:48:07.968773+0000","last_unstale":"2023-10-09T09:50:45.036576+0000","last_undegraded":"2023-10-09T09:50:45.036576+0000","last_fullsized":"2023-10-09T09:50:45.036576+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:39.023317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1239,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.169463+0000","last_change":"2023-10-09T09:48:07.988928+0000","last_active":"2023-10-09T09:50:45.169463+0000","last_peered":"2023-10-09T09:50:45.169463+0000","last_clean":"2023-10-09T09:50:45.169463+0000","last_became_active":"2023-10-09T09:48:07.988051+0000","last_became_peered":"2023-10-09T09:48:07.988051+0000","last_unstale":"2023-10-09T09:50:45.169463+0000","last_undegraded":"2023-10-09T09:50:45.169463+0000","last_fullsized":"2023-10-09T09:50:45.169463+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1198,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.512866+0000","last_change":"2023-10-09T09:43:48.902066+0000","last_active":"2023-10-09T09:50:44.512866+0000","last_peered":"2023-10-09T09:50:44.512866+0000","last_clean":"2023-10-09T09:50:44.512866+0000","last_became_active":"2023-10-09T09:43:48.901639+0000","last_became_peered":"2023-10-09T09:43:48.901639+0000","last_unstale":"2023-10-09T09:50:44.512866+0000","last_undegraded":"2023-10-09T09:50:44.512866+0000","last_fullsized":"2023-10-09T09:50:44.512866+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":667,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507343+0000","last_change":"2023-10-09T09:48:07.997787+0000","last_active":"2023-10-09T09:50:44.507343+0000","last_peered":"2023-10-09T09:50:44.507343+0000","last_clean":"2023-10-09T09:50:44.507343+0000","last_became_active":"2023-10-09T09:48:07.997356+0000","last_became_peered":"2023-10-09T09:48:07.997356+0000","last_unstale":"2023-10-09T09:50:44.507343+0000","last_undegraded":"2023-10-09T09:50:44.507343+0000","last_fullsized":"2023-10-09T09:50:44.507343+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:03.024096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":141,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.512580+0000","last_change":"2023-10-09T09:48:07.975547+0000","last_active":"2023-10-09T09:50:44.512580+0000","last_peered":"2023-10-09T09:50:44.512580+0000","last_clean":"2023-10-09T09:50:44.512580+0000","last_became_active":"2023-10-09T09:48:07.975150+0000","last_became_peered":"2023-10-09T09:48:07.975150+0000","last_unstale":"2023-10-09T09:50:44.512580+0000","last_undegraded":"2023-10-09T09:50:44.512580+0000","last_fullsized":"2023-10-09T09:50:44.512580+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":690,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.168337+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:50:45.168337+0000","last_peered":"2023-10-09T09:50:45.168337+0000","last_clean":"2023-10-09T09:50:45.168337+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:50:45.168337+0000","last_undegraded":"2023-10-09T09:50:45.168337+0000","last_fullsized":"2023-10-09T09:50:45.168337+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":83,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.169186+0000","last_change":"2023-10-09T09:43:48.906785+0000","last_active":"2023-10-09T09:50:45.169186+0000","last_peered":"2023-10-09T09:50:45.169186+0000","last_clean":"2023-10-09T09:50:45.169186+0000","last_became_active":"2023-10-09T09:43:48.906276+0000","last_became_peered":"2023-10-09T09:43:48.906276+0000","last_unstale":"2023-10-09T09:50:45.169186+0000","last_undegraded":"2023-10-09T09:50:45.169186+0000","last_fullsized":"2023-10-09T09:50:45.169186+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":107,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.512247+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:50:44.512247+0000","last_peered":"2023-10-09T09:50:44.512247+0000","last_clean":"2023-10-09T09:50:44.512247+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:50:44.512247+0000","last_undegraded":"2023-10-09T09:50:44.512247+0000","last_fullsized":"2023-10-09T09:50:44.512247+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":431,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.168557+0000","last_change":"2023-10-09T09:43:23.995526+0000","last_active":"2023-10-09T09:50:45.168557+0000","last_peered":"2023-10-09T09:50:45.168557+0000","last_clean":"2023-10-09T09:50:45.168557+0000","last_became_active":"2023-10-09T09:43:23.994816+0000","last_became_peered":"2023-10-09T09:43:23.994816+0000","last_unstale":"2023-10-09T09:50:45.168557+0000","last_undegraded":"2023-10-09T09:50:45.168557+0000","last_fullsized":"2023-10-09T09:50:45.168557+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":648,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.168436+0000","last_change":"2023-10-09T09:48:07.987637+0000","last_active":"2023-10-09T09:50:45.168436+0000","last_peered":"2023-10-09T09:50:45.168436+0000","last_clean":"2023-10-09T09:50:45.168436+0000","last_became_active":"2023-10-09T09:48:07.987085+0000","last_became_peered":"2023-10-09T09:48:07.987085+0000","last_unstale":"2023-10-09T09:50:45.168436+0000","last_undegraded":"2023-10-09T09:50:45.168436+0000","last_fullsized":"2023-10-09T09:50:45.168436+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1170,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.162411+0000","last_change":"2023-10-09T09:43:23.989356+0000","last_active":"2023-10-09T09:50:45.162411+0000","last_peered":"2023-10-09T09:50:45.162411+0000","last_clean":"2023-10-09T09:50:45.162411+0000","last_became_active":"2023-10-09T09:43:23.988986+0000","last_became_peered":"2023-10-09T09:43:23.988986+0000","last_unstale":"2023-10-09T09:50:45.162411+0000","last_undegraded":"2023-10-09T09:50:45.162411+0000","last_fullsized":"2023-10-09T09:50:45.162411+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.570448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":752,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.509088+0000","last_change":"2023-10-09T09:48:07.984006+0000","last_active":"2023-10-09T09:50:44.509088+0000","last_peered":"2023-10-09T09:50:44.509088+0000","last_clean":"2023-10-09T09:50:44.509088+0000","last_became_active":"2023-10-09T09:48:07.978179+0000","last_became_peered":"2023-10-09T09:48:07.978179+0000","last_unstale":"2023-10-09T09:50:44.509088+0000","last_undegraded":"2023-10-09T09:50:44.509088+0000","last_fullsized":"2023-10-09T09:50:44.509088+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:34.164004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1173,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507961+0000","last_change":"2023-10-09T09:48:07.975335+0000","last_active":"2023-10-09T09:50:44.507961+0000","last_peered":"2023-10-09T09:50:44.507961+0000","last_clean":"2023-10-09T09:50:44.507961+0000","last_became_active":"2023-10-09T09:48:07.974437+0000","last_became_peered":"2023-10-09T09:48:07.974437+0000","last_unstale":"2023-10-09T09:50:44.507961+0000","last_undegraded":"2023-10-09T09:50:44.507961+0000","last_fullsized":"2023-10-09T09:50:44.507961+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:32.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":111,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.169282+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:50:45.169282+0000","last_peered":"2023-10-09T09:50:45.169282+0000","last_clean":"2023-10-09T09:50:45.169282+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:50:45.169282+0000","last_undegraded":"2023-10-09T09:50:45.169282+0000","last_fullsized":"2023-10-09T09:50:45.169282+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":75,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.168189+0000","last_change":"2023-10-09T09:43:23.994784+0000","last_active":"2023-10-09T09:50:45.168189+0000","last_peered":"2023-10-09T09:50:45.168189+0000","last_clean":"2023-10-09T09:50:45.168189+0000","last_became_active":"2023-10-09T09:43:23.994196+0000","last_became_peered":"2023-10-09T09:43:23.994196+0000","last_unstale":"2023-10-09T09:50:45.168189+0000","last_undegraded":"2023-10-09T09:50:45.168189+0000","last_fullsized":"2023-10-09T09:50:45.168189+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1191,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:51:05.645415+0000","last_change":"2023-10-09T09:43:48.901303+0000","last_active":"2023-10-09T09:51:05.645415+0000","last_peered":"2023-10-09T09:51:05.645415+0000","last_clean":"2023-10-09T09:51:05.645415+0000","last_became_active":"2023-10-09T09:43:48.900779+0000","last_became_peered":"2023-10-09T09:43:48.900779+0000","last_unstale":"2023-10-09T09:51:05.645415+0000","last_undegraded":"2023-10-09T09:51:05.645415+0000","last_fullsized":"2023-10-09T09:51:05.645415+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:51.853531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1269,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.166612+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:50:45.166612+0000","last_peered":"2023-10-09T09:50:45.166612+0000","last_clean":"2023-10-09T09:50:45.166612+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:50:45.166612+0000","last_undegraded":"2023-10-09T09:50:45.166612+0000","last_fullsized":"2023-10-09T09:50:45.166612+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1235,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.508876+0000","last_change":"2023-10-09T09:48:08.000694+0000","last_active":"2023-10-09T09:50:44.508876+0000","last_peered":"2023-10-09T09:50:44.508876+0000","last_clean":"2023-10-09T09:50:44.508876+0000","last_became_active":"2023-10-09T09:48:07.999476+0000","last_became_peered":"2023-10-09T09:48:07.999476+0000","last_unstale":"2023-10-09T09:50:44.508876+0000","last_undegraded":"2023-10-09T09:50:44.508876+0000","last_fullsized":"2023-10-09T09:50:44.508876+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:37.211076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.7","version":"1275'2988","reported_seq":3003,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.060045+0000","last_change":"2023-10-09T09:50:27.379119+0000","last_active":"2023-10-09T09:51:16.060045+0000","last_peered":"2023-10-09T09:51:16.060045+0000","last_clean":"2023-10-09T09:51:16.060045+0000","last_became_active":"2023-10-09T09:50:27.378562+0000","last_became_peered":"2023-10-09T09:50:27.378562+0000","last_unstale":"2023-10-09T09:51:16.060045+0000","last_undegraded":"2023-10-09T09:51:16.060045+0000","last_fullsized":"2023-10-09T09:51:16.060045+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":2988,"log_dups_size":0,"ondisk_log_size":2988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:01.665220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195821568,"num_objects":2988,"num_object_clones":0,"num_object_copies":8964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2988,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":101,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.037092+0000","last_change":"2023-10-09T09:48:07.976558+0000","last_active":"2023-10-09T09:50:45.037092+0000","last_peered":"2023-10-09T09:50:45.037092+0000","last_clean":"2023-10-09T09:50:45.037092+0000","last_became_active":"2023-10-09T09:48:07.975961+0000","last_became_peered":"2023-10-09T09:48:07.975961+0000","last_unstale":"2023-10-09T09:50:45.037092+0000","last_undegraded":"2023-10-09T09:50:45.037092+0000","last_fullsized":"2023-10-09T09:50:45.037092+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:40.925248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":117,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.511680+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:50:44.511680+0000","last_peered":"2023-10-09T09:50:44.511680+0000","last_clean":"2023-10-09T09:50:44.511680+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:50:44.511680+0000","last_undegraded":"2023-10-09T09:50:44.511680+0000","last_fullsized":"2023-10-09T09:50:44.511680+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"1275'3283","reported_seq":3298,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:18.741548+0000","last_change":"2023-10-09T09:50:26.983846+0000","last_active":"2023-10-09T09:51:18.741548+0000","last_peered":"2023-10-09T09:51:18.741548+0000","last_clean":"2023-10-09T09:51:18.741548+0000","last_became_active":"2023-10-09T09:50:26.983409+0000","last_became_peered":"2023-10-09T09:50:26.983409+0000","last_unstale":"2023-10-09T09:51:18.741548+0000","last_undegraded":"2023-10-09T09:51:18.741548+0000","last_fullsized":"2023-10-09T09:51:18.741548+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":3283,"log_dups_size":0,"ondisk_log_size":3283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:48.191471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215154688,"num_objects":3283,"num_object_clones":0,"num_object_copies":9849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3283,"num_write_kb":210112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":64,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.508260+0000","last_change":"2023-10-09T09:48:07.990711+0000","last_active":"2023-10-09T09:50:44.508260+0000","last_peered":"2023-10-09T09:50:44.508260+0000","last_clean":"2023-10-09T09:50:44.508260+0000","last_became_active":"2023-10-09T09:48:07.990267+0000","last_became_peered":"2023-10-09T09:48:07.990267+0000","last_unstale":"2023-10-09T09:50:44.508260+0000","last_undegraded":"2023-10-09T09:50:44.508260+0000","last_fullsized":"2023-10-09T09:50:44.508260+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:51.244840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1208,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507530+0000","last_change":"2023-10-09T09:48:07.991440+0000","last_active":"2023-10-09T09:50:44.507530+0000","last_peered":"2023-10-09T09:50:44.507530+0000","last_clean":"2023-10-09T09:50:44.507530+0000","last_became_active":"2023-10-09T09:48:07.991072+0000","last_became_peered":"2023-10-09T09:48:07.991072+0000","last_unstale":"2023-10-09T09:50:44.507530+0000","last_undegraded":"2023-10-09T09:50:44.507530+0000","last_fullsized":"2023-10-09T09:50:44.507530+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:28.577159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"1275'3034","reported_seq":3048,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.646440+0000","last_change":"2023-10-09T09:50:26.985990+0000","last_active":"2023-10-09T09:51:16.646440+0000","last_peered":"2023-10-09T09:51:16.646440+0000","last_clean":"2023-10-09T09:51:16.646440+0000","last_became_active":"2023-10-09T09:50:26.985591+0000","last_became_peered":"2023-10-09T09:50:26.985591+0000","last_unstale":"2023-10-09T09:51:16.646440+0000","last_undegraded":"2023-10-09T09:51:16.646440+0000","last_fullsized":"2023-10-09T09:51:16.646440+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":3034,"log_dups_size":0,"ondisk_log_size":3034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198836224,"num_objects":3034,"num_object_clones":0,"num_object_copies":9102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3034,"num_write_kb":194176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1225,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.512443+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:50:44.512443+0000","last_peered":"2023-10-09T09:50:44.512443+0000","last_clean":"2023-10-09T09:50:44.512443+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:50:44.512443+0000","last_undegraded":"2023-10-09T09:50:44.512443+0000","last_fullsized":"2023-10-09T09:50:44.512443+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1231,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.168451+0000","last_change":"2023-10-09T09:48:07.987819+0000","last_active":"2023-10-09T09:50:45.168451+0000","last_peered":"2023-10-09T09:50:45.168451+0000","last_clean":"2023-10-09T09:50:45.168451+0000","last_became_active":"2023-10-09T09:48:07.986354+0000","last_became_peered":"2023-10-09T09:48:07.986354+0000","last_unstale":"2023-10-09T09:50:45.168451+0000","last_undegraded":"2023-10-09T09:50:45.168451+0000","last_fullsized":"2023-10-09T09:50:45.168451+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"1275'3056","reported_seq":3070,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.903622+0000","last_change":"2023-10-09T09:50:26.987956+0000","last_active":"2023-10-09T09:51:16.903622+0000","last_peered":"2023-10-09T09:51:16.903622+0000","last_clean":"2023-10-09T09:51:16.903622+0000","last_became_active":"2023-10-09T09:50:26.987609+0000","last_became_peered":"2023-10-09T09:50:26.987609+0000","last_unstale":"2023-10-09T09:51:16.903622+0000","last_undegraded":"2023-10-09T09:51:16.903622+0000","last_fullsized":"2023-10-09T09:51:16.903622+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":3056,"log_dups_size":0,"ondisk_log_size":3056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200278016,"num_objects":3056,"num_object_clones":0,"num_object_copies":9168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3056,"num_write_kb":195584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":777,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.168902+0000","last_change":"2023-10-09T09:48:07.987506+0000","last_active":"2023-10-09T09:50:45.168902+0000","last_peered":"2023-10-09T09:50:45.168902+0000","last_clean":"2023-10-09T09:50:45.168902+0000","last_became_active":"2023-10-09T09:48:07.986988+0000","last_became_peered":"2023-10-09T09:48:07.986988+0000","last_unstale":"2023-10-09T09:50:45.168902+0000","last_undegraded":"2023-10-09T09:50:45.168902+0000","last_fullsized":"2023-10-09T09:50:45.168902+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1194,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:51:03.731331+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:51:03.731331+0000","last_peered":"2023-10-09T09:51:03.731331+0000","last_clean":"2023-10-09T09:51:03.731331+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:51:03.731331+0000","last_undegraded":"2023-10-09T09:51:03.731331+0000","last_fullsized":"2023-10-09T09:51:03.731331+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"1275'2991","reported_seq":3005,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.953619+0000","last_change":"2023-10-09T09:50:26.988587+0000","last_active":"2023-10-09T09:51:16.953619+0000","last_peered":"2023-10-09T09:51:16.953619+0000","last_clean":"2023-10-09T09:51:16.953619+0000","last_became_active":"2023-10-09T09:50:26.987497+0000","last_became_peered":"2023-10-09T09:50:26.987497+0000","last_unstale":"2023-10-09T09:51:16.953619+0000","last_undegraded":"2023-10-09T09:51:16.953619+0000","last_fullsized":"2023-10-09T09:51:16.953619+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":2991,"log_dups_size":0,"ondisk_log_size":2991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196018176,"num_objects":2991,"num_object_clones":0,"num_object_copies":8973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2991,"num_write_kb":191424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":152,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.508332+0000","last_change":"2023-10-09T09:48:07.977143+0000","last_active":"2023-10-09T09:50:44.508332+0000","last_peered":"2023-10-09T09:50:44.508332+0000","last_clean":"2023-10-09T09:50:44.508332+0000","last_became_active":"2023-10-09T09:48:07.976338+0000","last_became_peered":"2023-10-09T09:48:07.976338+0000","last_unstale":"2023-10-09T09:50:44.508332+0000","last_undegraded":"2023-10-09T09:50:44.508332+0000","last_fullsized":"2023-10-09T09:50:44.508332+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:12.015053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1227,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.168819+0000","last_change":"2023-10-09T09:48:08.001503+0000","last_active":"2023-10-09T09:50:45.168819+0000","last_peered":"2023-10-09T09:50:45.168819+0000","last_clean":"2023-10-09T09:50:45.168819+0000","last_became_active":"2023-10-09T09:48:08.000921+0000","last_became_peered":"2023-10-09T09:48:08.000921+0000","last_unstale":"2023-10-09T09:50:45.168819+0000","last_undegraded":"2023-10-09T09:50:45.168819+0000","last_fullsized":"2023-10-09T09:50:45.168819+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.2","version":"1275'2949","reported_seq":2964,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:15.643162+0000","last_change":"2023-10-09T09:50:26.980085+0000","last_active":"2023-10-09T09:51:15.643162+0000","last_peered":"2023-10-09T09:51:15.643162+0000","last_clean":"2023-10-09T09:51:15.643162+0000","last_became_active":"2023-10-09T09:50:26.979773+0000","last_became_peered":"2023-10-09T09:50:26.979773+0000","last_unstale":"2023-10-09T09:51:15.643162+0000","last_undegraded":"2023-10-09T09:51:15.643162+0000","last_fullsized":"2023-10-09T09:51:15.643162+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":2949,"log_dups_size":0,"ondisk_log_size":2949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:37.729645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193265664,"num_objects":2949,"num_object_clones":0,"num_object_copies":8847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2949,"num_write_kb":188736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1223,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.168088+0000","last_change":"2023-10-09T09:48:07.988586+0000","last_active":"2023-10-09T09:50:45.168088+0000","last_peered":"2023-10-09T09:50:45.168088+0000","last_clean":"2023-10-09T09:50:45.168088+0000","last_became_active":"2023-10-09T09:48:07.988226+0000","last_became_peered":"2023-10-09T09:48:07.988226+0000","last_unstale":"2023-10-09T09:50:45.168088+0000","last_undegraded":"2023-10-09T09:50:45.168088+0000","last_fullsized":"2023-10-09T09:50:45.168088+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1214,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.162608+0000","last_change":"2023-10-09T09:48:07.987993+0000","last_active":"2023-10-09T09:50:45.162608+0000","last_peered":"2023-10-09T09:50:45.162608+0000","last_clean":"2023-10-09T09:50:45.162608+0000","last_became_active":"2023-10-09T09:48:07.987664+0000","last_became_peered":"2023-10-09T09:48:07.987664+0000","last_unstale":"2023-10-09T09:50:45.162608+0000","last_undegraded":"2023-10-09T09:50:45.162608+0000","last_fullsized":"2023-10-09T09:50:45.162608+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:32.253321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"1275'3039","reported_seq":3053,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.951391+0000","last_change":"2023-10-09T09:50:26.987126+0000","last_active":"2023-10-09T09:51:16.951391+0000","last_peered":"2023-10-09T09:51:16.951391+0000","last_clean":"2023-10-09T09:51:16.951391+0000","last_became_active":"2023-10-09T09:50:26.986696+0000","last_became_peered":"2023-10-09T09:50:26.986696+0000","last_unstale":"2023-10-09T09:51:16.951391+0000","last_undegraded":"2023-10-09T09:51:16.951391+0000","last_fullsized":"2023-10-09T09:51:16.951391+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":3039,"log_dups_size":0,"ondisk_log_size":3039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199163904,"num_objects":3039,"num_object_clones":0,"num_object_copies":9117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3039,"num_write_kb":194496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1242,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.162083+0000","last_change":"2023-10-09T09:48:07.984079+0000","last_active":"2023-10-09T09:50:45.162083+0000","last_peered":"2023-10-09T09:50:45.162083+0000","last_clean":"2023-10-09T09:50:45.162083+0000","last_became_active":"2023-10-09T09:48:07.983792+0000","last_became_peered":"2023-10-09T09:48:07.983792+0000","last_unstale":"2023-10-09T09:50:45.162083+0000","last_undegraded":"2023-10-09T09:50:45.162083+0000","last_fullsized":"2023-10-09T09:50:45.162083+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:28:42.289832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":648,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.169226+0000","last_change":"2023-10-09T09:48:07.989047+0000","last_active":"2023-10-09T09:50:45.169226+0000","last_peered":"2023-10-09T09:50:45.169226+0000","last_clean":"2023-10-09T09:50:45.169226+0000","last_became_active":"2023-10-09T09:48:07.988131+0000","last_became_peered":"2023-10-09T09:48:07.988131+0000","last_unstale":"2023-10-09T09:50:45.169226+0000","last_undegraded":"2023-10-09T09:50:45.169226+0000","last_fullsized":"2023-10-09T09:50:45.169226+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"1275'2986","reported_seq":3000,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.059989+0000","last_change":"2023-10-09T09:50:26.985963+0000","last_active":"2023-10-09T09:51:16.059989+0000","last_peered":"2023-10-09T09:51:16.059989+0000","last_clean":"2023-10-09T09:51:16.059989+0000","last_became_active":"2023-10-09T09:50:26.985572+0000","last_became_peered":"2023-10-09T09:50:26.985572+0000","last_unstale":"2023-10-09T09:51:16.059989+0000","last_undegraded":"2023-10-09T09:51:16.059989+0000","last_fullsized":"2023-10-09T09:51:16.059989+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":2986,"log_dups_size":0,"ondisk_log_size":2986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:48.639926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195690496,"num_objects":2986,"num_object_clones":0,"num_object_copies":8958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2986,"num_write_kb":191104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1184,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507684+0000","last_change":"2023-10-09T09:48:07.976884+0000","last_active":"2023-10-09T09:50:44.507684+0000","last_peered":"2023-10-09T09:50:44.507684+0000","last_clean":"2023-10-09T09:50:44.507684+0000","last_became_active":"2023-10-09T09:48:07.975932+0000","last_became_peered":"2023-10-09T09:48:07.975932+0000","last_unstale":"2023-10-09T09:50:44.507684+0000","last_undegraded":"2023-10-09T09:50:44.507684+0000","last_fullsized":"2023-10-09T09:50:44.507684+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:22.785728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":120,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.162925+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:50:45.162925+0000","last_peered":"2023-10-09T09:50:45.162925+0000","last_clean":"2023-10-09T09:50:45.162925+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:50:45.162925+0000","last_undegraded":"2023-10-09T09:50:45.162925+0000","last_fullsized":"2023-10-09T09:50:45.162925+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"1275'2992","reported_seq":3007,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:15.638007+0000","last_change":"2023-10-09T09:50:27.376496+0000","last_active":"2023-10-09T09:51:15.638007+0000","last_peered":"2023-10-09T09:51:15.638007+0000","last_clean":"2023-10-09T09:51:15.638007+0000","last_became_active":"2023-10-09T09:50:27.376004+0000","last_became_peered":"2023-10-09T09:50:27.376004+0000","last_unstale":"2023-10-09T09:51:15.638007+0000","last_undegraded":"2023-10-09T09:51:15.638007+0000","last_fullsized":"2023-10-09T09:51:15.638007+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":2992,"log_dups_size":0,"ondisk_log_size":2992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:22.820238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196083712,"num_objects":2992,"num_object_clones":0,"num_object_copies":8976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2992,"num_write_kb":191488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1230,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.509038+0000","last_change":"2023-10-09T09:48:07.984553+0000","last_active":"2023-10-09T09:50:44.509038+0000","last_peered":"2023-10-09T09:50:44.509038+0000","last_clean":"2023-10-09T09:50:44.509038+0000","last_became_active":"2023-10-09T09:48:07.983302+0000","last_became_peered":"2023-10-09T09:48:07.983302+0000","last_unstale":"2023-10-09T09:50:44.509038+0000","last_undegraded":"2023-10-09T09:50:44.509038+0000","last_fullsized":"2023-10-09T09:50:44.509038+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:50.714295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.512286+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:50:44.512286+0000","last_peered":"2023-10-09T09:50:44.512286+0000","last_clean":"2023-10-09T09:50:44.512286+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:50:44.512286+0000","last_undegraded":"2023-10-09T09:50:44.512286+0000","last_fullsized":"2023-10-09T09:50:44.512286+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1207,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507559+0000","last_change":"2023-10-09T09:48:07.977683+0000","last_active":"2023-10-09T09:50:44.507559+0000","last_peered":"2023-10-09T09:50:44.507559+0000","last_clean":"2023-10-09T09:50:44.507559+0000","last_became_active":"2023-10-09T09:48:07.977024+0000","last_became_peered":"2023-10-09T09:48:07.977024+0000","last_unstale":"2023-10-09T09:50:44.507559+0000","last_undegraded":"2023-10-09T09:50:44.507559+0000","last_fullsized":"2023-10-09T09:50:44.507559+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:13.438431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"1275'3034","reported_seq":3049,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.078283+0000","last_change":"2023-10-09T09:50:27.379238+0000","last_active":"2023-10-09T09:51:16.078283+0000","last_peered":"2023-10-09T09:51:16.078283+0000","last_clean":"2023-10-09T09:51:16.078283+0000","last_became_active":"2023-10-09T09:50:27.378551+0000","last_became_peered":"2023-10-09T09:50:27.378551+0000","last_unstale":"2023-10-09T09:51:16.078283+0000","last_undegraded":"2023-10-09T09:51:16.078283+0000","last_fullsized":"2023-10-09T09:51:16.078283+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":3034,"log_dups_size":0,"ondisk_log_size":3034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:19.610902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198836224,"num_objects":3034,"num_object_clones":0,"num_object_copies":9102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3034,"num_write_kb":194176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":690,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507924+0000","last_change":"2023-10-09T09:48:07.997811+0000","last_active":"2023-10-09T09:50:44.507924+0000","last_peered":"2023-10-09T09:50:44.507924+0000","last_clean":"2023-10-09T09:50:44.507924+0000","last_became_active":"2023-10-09T09:48:07.997389+0000","last_became_peered":"2023-10-09T09:48:07.997389+0000","last_unstale":"2023-10-09T09:50:44.507924+0000","last_undegraded":"2023-10-09T09:50:44.507924+0000","last_fullsized":"2023-10-09T09:50:44.507924+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:21.978398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1165,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.168603+0000","last_change":"2023-10-09T09:43:48.908190+0000","last_active":"2023-10-09T09:50:45.168603+0000","last_peered":"2023-10-09T09:50:45.168603+0000","last_clean":"2023-10-09T09:50:45.168603+0000","last_became_active":"2023-10-09T09:43:48.907799+0000","last_became_peered":"2023-10-09T09:43:48.907799+0000","last_unstale":"2023-10-09T09:50:45.168603+0000","last_undegraded":"2023-10-09T09:50:45.168603+0000","last_fullsized":"2023-10-09T09:50:45.168603+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1260,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.508477+0000","last_change":"2023-10-09T09:48:08.000154+0000","last_active":"2023-10-09T09:50:44.508477+0000","last_peered":"2023-10-09T09:50:44.508477+0000","last_clean":"2023-10-09T09:50:44.508477+0000","last_became_active":"2023-10-09T09:48:07.999714+0000","last_became_peered":"2023-10-09T09:48:07.999714+0000","last_unstale":"2023-10-09T09:50:44.508477+0000","last_undegraded":"2023-10-09T09:50:44.508477+0000","last_fullsized":"2023-10-09T09:50:44.508477+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:12.921693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"1275'3003","reported_seq":3017,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.956123+0000","last_change":"2023-10-09T09:50:26.987164+0000","last_active":"2023-10-09T09:51:16.956123+0000","last_peered":"2023-10-09T09:51:16.956123+0000","last_clean":"2023-10-09T09:51:16.956123+0000","last_became_active":"2023-10-09T09:50:26.986703+0000","last_became_peered":"2023-10-09T09:50:26.986703+0000","last_unstale":"2023-10-09T09:51:16.956123+0000","last_undegraded":"2023-10-09T09:51:16.956123+0000","last_fullsized":"2023-10-09T09:51:16.956123+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":3003,"log_dups_size":0,"ondisk_log_size":3003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196804608,"num_objects":3003,"num_object_clones":0,"num_object_copies":9009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3003,"num_write_kb":192192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":165,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.162210+0000","last_change":"2023-10-09T09:48:07.983969+0000","last_active":"2023-10-09T09:50:45.162210+0000","last_peered":"2023-10-09T09:50:45.162210+0000","last_clean":"2023-10-09T09:50:45.162210+0000","last_became_active":"2023-10-09T09:48:07.983490+0000","last_became_peered":"2023-10-09T09:48:07.983490+0000","last_unstale":"2023-10-09T09:50:45.162210+0000","last_undegraded":"2023-10-09T09:50:45.162210+0000","last_fullsized":"2023-10-09T09:50:45.162210+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:21.122987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1238,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507464+0000","last_change":"2023-10-09T09:48:07.998234+0000","last_active":"2023-10-09T09:50:44.507464+0000","last_peered":"2023-10-09T09:50:44.507464+0000","last_clean":"2023-10-09T09:50:44.507464+0000","last_became_active":"2023-10-09T09:48:07.997909+0000","last_became_peered":"2023-10-09T09:48:07.997909+0000","last_unstale":"2023-10-09T09:50:44.507464+0000","last_undegraded":"2023-10-09T09:50:44.507464+0000","last_fullsized":"2023-10-09T09:50:44.507464+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:21.750617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1227,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.508763+0000","last_change":"2023-10-09T09:48:07.973284+0000","last_active":"2023-10-09T09:50:44.508763+0000","last_peered":"2023-10-09T09:50:44.508763+0000","last_clean":"2023-10-09T09:50:44.508763+0000","last_became_active":"2023-10-09T09:48:07.971235+0000","last_became_peered":"2023-10-09T09:48:07.971235+0000","last_unstale":"2023-10-09T09:50:44.508763+0000","last_undegraded":"2023-10-09T09:50:44.508763+0000","last_fullsized":"2023-10-09T09:50:44.508763+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:40.951420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"1275'3021","reported_seq":3035,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.931295+0000","last_change":"2023-10-09T09:50:26.987803+0000","last_active":"2023-10-09T09:51:16.931295+0000","last_peered":"2023-10-09T09:51:16.931295+0000","last_clean":"2023-10-09T09:51:16.931295+0000","last_became_active":"2023-10-09T09:50:26.987440+0000","last_became_peered":"2023-10-09T09:50:26.987440+0000","last_unstale":"2023-10-09T09:51:16.931295+0000","last_undegraded":"2023-10-09T09:51:16.931295+0000","last_fullsized":"2023-10-09T09:51:16.931295+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":3021,"log_dups_size":0,"ondisk_log_size":3021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197984256,"num_objects":3021,"num_object_clones":0,"num_object_copies":9063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3021,"num_write_kb":193344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1284,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.024193+0000","last_change":"2023-10-09T09:41:11.795802+0000","last_active":"2023-10-09T09:51:10.024193+0000","last_peered":"2023-10-09T09:51:10.024193+0000","last_clean":"2023-10-09T09:51:10.024193+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:51:10.024193+0000","last_undegraded":"2023-10-09T09:51:10.024193+0000","last_fullsized":"2023-10-09T09:51:10.024193+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":617,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:53.728744+0000","last_change":"2023-10-09T09:48:07.978435+0000","last_active":"2023-10-09T09:50:53.728744+0000","last_peered":"2023-10-09T09:50:53.728744+0000","last_clean":"2023-10-09T09:50:53.728744+0000","last_became_active":"2023-10-09T09:48:07.977417+0000","last_became_peered":"2023-10-09T09:48:07.977417+0000","last_unstale":"2023-10-09T09:50:53.728744+0000","last_undegraded":"2023-10-09T09:50:53.728744+0000","last_fullsized":"2023-10-09T09:50:53.728744+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:19.288202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.512132+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:50:44.512132+0000","last_peered":"2023-10-09T09:50:44.512132+0000","last_clean":"2023-10-09T09:50:44.512132+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:50:44.512132+0000","last_undegraded":"2023-10-09T09:50:44.512132+0000","last_fullsized":"2023-10-09T09:50:44.512132+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1201,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.169381+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:50:45.169381+0000","last_peered":"2023-10-09T09:50:45.169381+0000","last_clean":"2023-10-09T09:50:45.169381+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:50:45.169381+0000","last_undegraded":"2023-10-09T09:50:45.169381+0000","last_fullsized":"2023-10-09T09:50:45.169381+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"1275'3031","reported_seq":3045,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.952148+0000","last_change":"2023-10-09T09:50:26.988221+0000","last_active":"2023-10-09T09:51:16.952148+0000","last_peered":"2023-10-09T09:51:16.952148+0000","last_clean":"2023-10-09T09:51:16.952148+0000","last_became_active":"2023-10-09T09:50:26.987465+0000","last_became_peered":"2023-10-09T09:50:26.987465+0000","last_unstale":"2023-10-09T09:51:16.952148+0000","last_undegraded":"2023-10-09T09:51:16.952148+0000","last_fullsized":"2023-10-09T09:51:16.952148+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":3031,"log_dups_size":0,"ondisk_log_size":3031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198639616,"num_objects":3031,"num_object_clones":0,"num_object_copies":9093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3031,"num_write_kb":193984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1204,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507443+0000","last_change":"2023-10-09T09:48:07.973246+0000","last_active":"2023-10-09T09:50:44.507443+0000","last_peered":"2023-10-09T09:50:44.507443+0000","last_clean":"2023-10-09T09:50:44.507443+0000","last_became_active":"2023-10-09T09:48:07.971221+0000","last_became_peered":"2023-10-09T09:48:07.971221+0000","last_unstale":"2023-10-09T09:50:44.507443+0000","last_undegraded":"2023-10-09T09:50:44.507443+0000","last_fullsized":"2023-10-09T09:50:44.507443+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:16.759514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1211,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.036449+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:50:45.036449+0000","last_peered":"2023-10-09T09:50:45.036449+0000","last_clean":"2023-10-09T09:50:45.036449+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:50:45.036449+0000","last_undegraded":"2023-10-09T09:50:45.036449+0000","last_fullsized":"2023-10-09T09:50:45.036449+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1223,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.036769+0000","last_change":"2023-10-09T09:48:07.981410+0000","last_active":"2023-10-09T09:50:45.036769+0000","last_peered":"2023-10-09T09:50:45.036769+0000","last_clean":"2023-10-09T09:50:45.036769+0000","last_became_active":"2023-10-09T09:48:07.979850+0000","last_became_peered":"2023-10-09T09:48:07.979850+0000","last_unstale":"2023-10-09T09:50:45.036769+0000","last_undegraded":"2023-10-09T09:50:45.036769+0000","last_fullsized":"2023-10-09T09:50:45.036769+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:31.201104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"1275'3017","reported_seq":3032,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.063275+0000","last_change":"2023-10-09T09:50:27.379333+0000","last_active":"2023-10-09T09:51:16.063275+0000","last_peered":"2023-10-09T09:51:16.063275+0000","last_clean":"2023-10-09T09:51:16.063275+0000","last_became_active":"2023-10-09T09:50:27.378836+0000","last_became_peered":"2023-10-09T09:50:27.378836+0000","last_unstale":"2023-10-09T09:51:16.063275+0000","last_undegraded":"2023-10-09T09:51:16.063275+0000","last_fullsized":"2023-10-09T09:51:16.063275+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":3017,"log_dups_size":0,"ondisk_log_size":3017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:59.239495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197722112,"num_objects":3017,"num_object_clones":0,"num_object_copies":9051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3017,"num_write_kb":193088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1199,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.252736+0000","last_change":"2023-10-09T09:48:07.975786+0000","last_active":"2023-10-09T09:51:10.252736+0000","last_peered":"2023-10-09T09:51:10.252736+0000","last_clean":"2023-10-09T09:51:10.252736+0000","last_became_active":"2023-10-09T09:48:07.975338+0000","last_became_peered":"2023-10-09T09:48:07.975338+0000","last_unstale":"2023-10-09T09:51:10.252736+0000","last_undegraded":"2023-10-09T09:51:10.252736+0000","last_fullsized":"2023-10-09T09:51:10.252736+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:05.175956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1241,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.508935+0000","last_change":"2023-10-09T09:48:07.983773+0000","last_active":"2023-10-09T09:50:44.508935+0000","last_peered":"2023-10-09T09:50:44.508935+0000","last_clean":"2023-10-09T09:50:44.508935+0000","last_became_active":"2023-10-09T09:48:07.976869+0000","last_became_peered":"2023-10-09T09:48:07.976869+0000","last_unstale":"2023-10-09T09:50:44.508935+0000","last_undegraded":"2023-10-09T09:50:44.508935+0000","last_fullsized":"2023-10-09T09:50:44.508935+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:58.173236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1191,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.512068+0000","last_change":"2023-10-09T09:48:07.962302+0000","last_active":"2023-10-09T09:50:44.512068+0000","last_peered":"2023-10-09T09:50:44.512068+0000","last_clean":"2023-10-09T09:50:44.512068+0000","last_became_active":"2023-10-09T09:48:07.961888+0000","last_became_peered":"2023-10-09T09:48:07.961888+0000","last_unstale":"2023-10-09T09:50:44.512068+0000","last_undegraded":"2023-10-09T09:50:44.512068+0000","last_fullsized":"2023-10-09T09:50:44.512068+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"1275'3079","reported_seq":3094,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.063823+0000","last_change":"2023-10-09T09:50:26.989029+0000","last_active":"2023-10-09T09:51:16.063823+0000","last_peered":"2023-10-09T09:51:16.063823+0000","last_clean":"2023-10-09T09:51:16.063823+0000","last_became_active":"2023-10-09T09:50:26.988265+0000","last_became_peered":"2023-10-09T09:50:26.988265+0000","last_unstale":"2023-10-09T09:51:16.063823+0000","last_undegraded":"2023-10-09T09:51:16.063823+0000","last_fullsized":"2023-10-09T09:51:16.063823+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":3079,"log_dups_size":0,"ondisk_log_size":3079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.248282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201785344,"num_objects":3079,"num_object_clones":0,"num_object_copies":9237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3079,"num_write_kb":197056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":136,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.037176+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:50:45.037176+0000","last_peered":"2023-10-09T09:50:45.037176+0000","last_clean":"2023-10-09T09:50:45.037176+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:50:45.037176+0000","last_undegraded":"2023-10-09T09:50:45.037176+0000","last_fullsized":"2023-10-09T09:50:45.037176+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1193,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.508797+0000","last_change":"2023-10-09T09:48:07.999034+0000","last_active":"2023-10-09T09:50:44.508797+0000","last_peered":"2023-10-09T09:50:44.508797+0000","last_clean":"2023-10-09T09:50:44.508797+0000","last_became_active":"2023-10-09T09:48:07.998606+0000","last_became_peered":"2023-10-09T09:48:07.998606+0000","last_unstale":"2023-10-09T09:50:44.508797+0000","last_undegraded":"2023-10-09T09:50:44.508797+0000","last_fullsized":"2023-10-09T09:50:44.508797+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.469476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1129,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507269+0000","last_change":"2023-10-09T09:48:07.976350+0000","last_active":"2023-10-09T09:50:44.507269+0000","last_peered":"2023-10-09T09:50:44.507269+0000","last_clean":"2023-10-09T09:50:44.507269+0000","last_became_active":"2023-10-09T09:48:07.973099+0000","last_became_peered":"2023-10-09T09:48:07.973099+0000","last_unstale":"2023-10-09T09:50:44.507269+0000","last_undegraded":"2023-10-09T09:50:44.507269+0000","last_fullsized":"2023-10-09T09:50:44.507269+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:48.812722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1275'2994","reported_seq":3009,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:16.048883+0000","last_change":"2023-10-09T09:50:27.380948+0000","last_active":"2023-10-09T09:51:16.048883+0000","last_peered":"2023-10-09T09:51:16.048883+0000","last_clean":"2023-10-09T09:51:16.048883+0000","last_became_active":"2023-10-09T09:50:27.380423+0000","last_became_peered":"2023-10-09T09:50:27.380423+0000","last_unstale":"2023-10-09T09:51:16.048883+0000","last_undegraded":"2023-10-09T09:51:16.048883+0000","last_fullsized":"2023-10-09T09:51:16.048883+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":2994,"log_dups_size":0,"ondisk_log_size":2994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:09.358786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196214784,"num_objects":2994,"num_object_clones":0,"num_object_copies":8982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2994,"num_write_kb":191616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1262,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.169317+0000","last_change":"2023-10-09T09:48:07.990005+0000","last_active":"2023-10-09T09:50:45.169317+0000","last_peered":"2023-10-09T09:50:45.169317+0000","last_clean":"2023-10-09T09:50:45.169317+0000","last_became_active":"2023-10-09T09:48:07.987124+0000","last_became_peered":"2023-10-09T09:48:07.987124+0000","last_unstale":"2023-10-09T09:50:45.169317+0000","last_undegraded":"2023-10-09T09:50:45.169317+0000","last_fullsized":"2023-10-09T09:50:45.169317+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1204,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.036972+0000","last_change":"2023-10-09T09:48:07.980600+0000","last_active":"2023-10-09T09:50:45.036972+0000","last_peered":"2023-10-09T09:50:45.036972+0000","last_clean":"2023-10-09T09:50:45.036972+0000","last_became_active":"2023-10-09T09:48:07.979544+0000","last_became_peered":"2023-10-09T09:48:07.979544+0000","last_unstale":"2023-10-09T09:50:45.036972+0000","last_undegraded":"2023-10-09T09:50:45.036972+0000","last_fullsized":"2023-10-09T09:50:45.036972+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:47.320689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1193,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.512035+0000","last_change":"2023-10-09T09:43:23.992213+0000","last_active":"2023-10-09T09:50:44.512035+0000","last_peered":"2023-10-09T09:50:44.512035+0000","last_clean":"2023-10-09T09:50:44.512035+0000","last_became_active":"2023-10-09T09:43:23.991887+0000","last_became_peered":"2023-10-09T09:43:23.991887+0000","last_unstale":"2023-10-09T09:50:44.512035+0000","last_undegraded":"2023-10-09T09:50:44.512035+0000","last_fullsized":"2023-10-09T09:50:44.512035+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1212,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.036998+0000","last_change":"2023-10-09T09:48:07.969929+0000","last_active":"2023-10-09T09:50:45.036998+0000","last_peered":"2023-10-09T09:50:45.036998+0000","last_clean":"2023-10-09T09:50:45.036998+0000","last_became_active":"2023-10-09T09:48:07.969537+0000","last_became_peered":"2023-10-09T09:48:07.969537+0000","last_unstale":"2023-10-09T09:50:45.036998+0000","last_undegraded":"2023-10-09T09:50:45.036998+0000","last_fullsized":"2023-10-09T09:50:45.036998+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:42.344608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":164,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.162662+0000","last_change":"2023-10-09T09:48:07.987618+0000","last_active":"2023-10-09T09:50:45.162662+0000","last_peered":"2023-10-09T09:50:45.162662+0000","last_clean":"2023-10-09T09:50:45.162662+0000","last_became_active":"2023-10-09T09:48:07.987341+0000","last_became_peered":"2023-10-09T09:48:07.987341+0000","last_unstale":"2023-10-09T09:50:45.162662+0000","last_undegraded":"2023-10-09T09:50:45.162662+0000","last_fullsized":"2023-10-09T09:50:45.162662+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:19.972891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":64,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.508472+0000","last_change":"2023-10-09T09:48:07.984247+0000","last_active":"2023-10-09T09:50:44.508472+0000","last_peered":"2023-10-09T09:50:44.508472+0000","last_clean":"2023-10-09T09:50:44.508472+0000","last_became_active":"2023-10-09T09:48:07.978613+0000","last_became_peered":"2023-10-09T09:48:07.978613+0000","last_unstale":"2023-10-09T09:50:44.508472+0000","last_undegraded":"2023-10-09T09:50:44.508472+0000","last_fullsized":"2023-10-09T09:50:44.508472+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:34.663108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1208,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:44.507871+0000","last_change":"2023-10-09T09:48:07.991053+0000","last_active":"2023-10-09T09:50:44.507871+0000","last_peered":"2023-10-09T09:50:44.507871+0000","last_clean":"2023-10-09T09:50:44.507871+0000","last_became_active":"2023-10-09T09:48:07.990656+0000","last_became_peered":"2023-10-09T09:48:07.990656+0000","last_unstale":"2023-10-09T09:50:44.507871+0000","last_undegraded":"2023-10-09T09:50:44.507871+0000","last_fullsized":"2023-10-09T09:50:44.507871+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:06.223305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":83,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.169523+0000","last_change":"2023-10-09T09:43:23.995576+0000","last_active":"2023-10-09T09:50:45.169523+0000","last_peered":"2023-10-09T09:50:45.169523+0000","last_clean":"2023-10-09T09:50:45.169523+0000","last_became_active":"2023-10-09T09:43:23.995076+0000","last_became_peered":"2023-10-09T09:43:23.995076+0000","last_unstale":"2023-10-09T09:50:45.169523+0000","last_undegraded":"2023-10-09T09:50:45.169523+0000","last_fullsized":"2023-10-09T09:50:45.169523+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1243,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.162317+0000","last_change":"2023-10-09T09:43:48.904021+0000","last_active":"2023-10-09T09:50:45.162317+0000","last_peered":"2023-10-09T09:50:45.162317+0000","last_clean":"2023-10-09T09:50:45.162317+0000","last_became_active":"2023-10-09T09:43:48.903719+0000","last_became_peered":"2023-10-09T09:43:48.903719+0000","last_unstale":"2023-10-09T09:50:45.162317+0000","last_undegraded":"2023-10-09T09:50:45.162317+0000","last_fullsized":"2023-10-09T09:50:45.162317+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:16.284027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":223,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.169365+0000","last_change":"2023-10-09T09:48:08.001392+0000","last_active":"2023-10-09T09:50:45.169365+0000","last_peered":"2023-10-09T09:50:45.169365+0000","last_clean":"2023-10-09T09:50:45.169365+0000","last_became_active":"2023-10-09T09:48:08.000798+0000","last_became_peered":"2023-10-09T09:48:08.000798+0000","last_unstale":"2023-10-09T09:50:45.169365+0000","last_undegraded":"2023-10-09T09:50:45.169365+0000","last_fullsized":"2023-10-09T09:50:45.169365+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":149,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.162010+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:50:45.162010+0000","last_peered":"2023-10-09T09:50:45.162010+0000","last_clean":"2023-10-09T09:50:45.162010+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:50:45.162010+0000","last_undegraded":"2023-10-09T09:50:45.162010+0000","last_fullsized":"2023-10-09T09:50:45.162010+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1237,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.037074+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:50:45.037074+0000","last_peered":"2023-10-09T09:50:45.037074+0000","last_clean":"2023-10-09T09:50:45.037074+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:50:45.037074+0000","last_undegraded":"2023-10-09T09:50:45.037074+0000","last_fullsized":"2023-10-09T09:50:45.037074+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":648,"reported_epoch":1274,"state":"active+clean","last_fresh":"2023-10-09T09:50:45.167293+0000","last_change":"2023-10-09T09:48:07.990205+0000","last_active":"2023-10-09T09:50:45.167293+0000","last_peered":"2023-10-09T09:50:45.167293+0000","last_clean":"2023-10-09T09:50:45.167293+0000","last_became_active":"2023-10-09T09:48:07.989619+0000","last_became_peered":"2023-10-09T09:48:07.989619+0000","last_unstale":"2023-10-09T09:50:45.167293+0000","last_undegraded":"2023-10-09T09:50:45.167293+0000","last_fullsized":"2023-10-09T09:50:45.167293+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3178299392,"num_objects":48497,"num_object_clones":0,"num_object_copies":145491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":48497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48497,"num_write_kb":3103808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9607970816,"data_stored":9607970816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":48497,"ondisk_log_size":48497,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":2460,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30942,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453398114,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594347,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2649460,"kb_used_data":1744684,"kb_used_omap":0,"kb_used_meta":904768,"kb_avail":91095692,"statfs":{"total":95995035648,"available":93281988608,"internally_reserved":0,"allocated":1786556416,"data_stored":1786263690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926482432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":5.2720000000000002,"15min":2.706},"min":{"1min":0.44800000000000001,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":1.2629999999999999,"5min":149.095,"15min":149.095},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":5.4640000000000004,"15min":2.972},"min":{"1min":0.59299999999999997,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":1.3440000000000001,"5min":149.17500000000001,"15min":149.17500000000001},"last":0.505}]},{"osd":1,"last update":"Mon Oct 9 09:51:12 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.744,"15min":2.0720000000000001},"min":{"1min":0.37,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.0640000000000001,"5min":9.391,"15min":9.391},"last":4.7370000000000001},{"interface":"front","average":{"1min":1.024,"5min":1.6679999999999999,"15min":1.917},"min":{"1min":0.38800000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.3820000000000001,"5min":7.9240000000000004,"15min":7.9240000000000004},"last":2.8599999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":5.0780000000000003,"15min":2.871},"min":{"1min":0.34000000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2430000000000001,"5min":145.41399999999999,"15min":145.41399999999999},"last":2.843},{"interface":"front","average":{"1min":0.89700000000000002,"5min":5.0039999999999996,"15min":2.9790000000000001},"min":{"1min":0.41199999999999998,"5min":0.309,"15min":0.29499999999999998},"max":{"1min":1.218,"5min":151.29300000000001,"15min":151.29300000000001},"last":0.44900000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:50:39 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.3109999999999999,"15min":1.3919999999999999},"min":{"1min":0.57499999999999996,"5min":0.33400000000000002,"15min":0.309},"max":{"1min":1.103,"5min":7.9139999999999997,"15min":11.547000000000001},"last":2.75},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.8720000000000001,"15min":1.5},"min":{"1min":0.58999999999999997,"5min":0.39000000000000001,"15min":0.29499999999999998},"max":{"1min":1.3480000000000001,"5min":11.968999999999999,"15min":11.968999999999999},"last":0.40999999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:50:39 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.7589999999999999,"15min":1.4390000000000001},"min":{"1min":0.373,"5min":0.28399999999999997,"15min":0.28199999999999997},"max":{"1min":1.3420000000000001,"5min":12.148999999999999,"15min":12.148999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.998,"5min":2.4140000000000001,"15min":1.7090000000000001},"min":{"1min":0.60099999999999998,"5min":0.313,"15min":0.29499999999999998},"max":{"1min":1.3660000000000001,"5min":12.064,"15min":12.064},"last":0.47999999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:50:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.786,"15min":1.577},"min":{"1min":0.39300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3819999999999999,"5min":10.529,"15min":10.529},"last":2.7759999999999998},{"interface":"front","average":{"1min":1.0800000000000001,"5min":2.2280000000000002,"15min":1.736},"min":{"1min":0.73399999999999999,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":1.4339999999999999,"5min":12.000999999999999,"15min":12.000999999999999},"last":2.8069999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:51:04 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":2.0379999999999998,"15min":1.597},"min":{"1min":0.55300000000000005,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":2.3210000000000002,"5min":8.2769999999999992,"15min":11.922000000000001},"last":0.626},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.3540000000000001,"15min":1.2729999999999999},"min":{"1min":0.53600000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.8210000000000002,"5min":7.9359999999999999,"15min":7.9359999999999999},"last":2.8290000000000002}]}]},{"osd":2,"up_from":634,"seq":2723009266100,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2111956,"kb_used_data":1343756,"kb_used_omap":122,"kb_used_meta":768069,"kb_avail":91633196,"statfs":{"total":95995035648,"available":93832392704,"internally_reserved":0,"allocated":1376006144,"data_stored":1374467009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":786503243},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":5.4980000000000002,"15min":3.7360000000000002},"min":{"1min":0.35199999999999998,"5min":0.22900000000000001,"15min":0.20200000000000001},"max":{"1min":0.84899999999999998,"5min":163.84,"15min":163.84},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":5.6369999999999996,"15min":3.9119999999999999},"min":{"1min":0.35799999999999998,"5min":0.20799999999999999,"15min":0.20399999999999999},"max":{"1min":0.96599999999999997,"5min":163.965,"15min":163.965},"last":1.9139999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:51:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":1.117,"15min":1.075},"min":{"1min":0.41099999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":12.819000000000001,"5min":12.819000000000001,"15min":12.819000000000001},"last":4.1970000000000001},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.179,"15min":1.1699999999999999},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.724,"5min":12.724,"15min":12.724},"last":2.4079999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.92500000000000004,"15min":2.0409999999999999},"min":{"1min":0.40400000000000003,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":1.01,"5min":4.1429999999999998,"15min":230.476},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.71499999999999997,"15min":1.9710000000000001},"min":{"1min":0.433,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":0.999,"5min":3.7879999999999998,"15min":228.893},"last":0.52400000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.9550000000000001,"15min":1.575},"min":{"1min":0.495,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.351,"5min":71.876000000000005,"15min":71.876000000000005},"last":2.4489999999999998},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.321,"15min":1.4079999999999999},"min":{"1min":0.64400000000000002,"5min":0.29899999999999999,"15min":0.26700000000000002},"max":{"1min":2.5739999999999998,"5min":9.4640000000000004,"15min":10.394},"last":4.1349999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.466,"15min":1.395},"min":{"1min":0.37,"5min":0.34699999999999998,"15min":0.29799999999999999},"max":{"1min":5.5140000000000002,"5min":8.5830000000000002,"15min":9.7599999999999998},"last":2.3679999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.2150000000000001,"15min":1.3819999999999999},"min":{"1min":0.55700000000000005,"5min":0.41499999999999998,"15min":0.32200000000000001},"max":{"1min":1.3899999999999999,"5min":5.806,"15min":11.718999999999999},"last":0.72999999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":2.4399999999999999,"15min":1.9319999999999999},"min":{"1min":0.64100000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.6919999999999999,"5min":71.795000000000002,"15min":71.795000000000002},"last":3.6150000000000002},{"interface":"front","average":{"1min":1.0069999999999999,"5min":2.2959999999999998,"15min":1.724},"min":{"1min":0.72199999999999998,"5min":0.38700000000000001,"15min":0.35599999999999998},"max":{"1min":1.512,"5min":72.322000000000003,"15min":72.322000000000003},"last":4.0570000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:50:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.2370000000000001,"15min":1.363},"min":{"1min":0.57799999999999996,"5min":0.374,"15min":0.374},"max":{"1min":1.587,"5min":8.4659999999999993,"15min":10.462999999999999},"last":2.2730000000000001},{"interface":"front","average":{"1min":1.044,"5min":1.25,"15min":1.2929999999999999},"min":{"1min":0.44400000000000001,"5min":0.441,"15min":0.40400000000000003},"max":{"1min":1.5640000000000001,"5min":5.1029999999999998,"15min":9.7759999999999998},"last":2.4380000000000002}]}]},{"osd":1,"up_from":1246,"seq":5351529250855,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2515012,"kb_used_data":2272892,"kb_used_omap":204,"kb_used_meta":241907,"kb_avail":91230140,"statfs":{"total":95995035648,"available":93419663360,"internally_reserved":0,"allocated":2327441408,"data_stored":2325914506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208958,"internal_metadata":247713730},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":1.633,"15min":1.714},"min":{"1min":0.46600000000000003,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.294,"5min":10.294,"15min":10.294},"last":10.294},{"interface":"front","average":{"1min":1.2689999999999999,"5min":1.8109999999999999,"15min":2.0350000000000001},"min":{"1min":0.33000000000000002,"5min":0.214,"15min":0.214},"max":{"1min":4.0309999999999997,"5min":8.5579999999999998,"15min":8.5579999999999998},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.3919999999999999,"15min":1.498},"min":{"1min":0.49399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.3010000000000002,"5min":5.8570000000000002,"15min":5.8570000000000002},"last":5.3010000000000002},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.365,"15min":1.5449999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.75,"5min":5.75,"15min":5.75},"last":0.252}]},{"osd":3,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":1.462,"15min":1.486},"min":{"1min":0.38300000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.3290000000000002,"5min":8.7140000000000004,"15min":8.7140000000000004},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.556,"5min":1.286,"15min":1.349},"min":{"1min":0.498,"5min":0.311,"15min":0.311},"max":{"1min":5.2519999999999998,"5min":5.2519999999999998,"15min":5.2519999999999998},"last":5.2519999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":2.2850000000000001,"15min":2.581},"min":{"1min":0.71299999999999997,"5min":0.379,"15min":0.379},"max":{"1min":3.476,"5min":8.8070000000000004,"15min":8.8070000000000004},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.901,"5min":2.1800000000000002,"15min":2.3929999999999998},"min":{"1min":0.55300000000000005,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":10.263,"5min":10.263,"15min":10.263},"last":10.263}]},{"osd":5,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9830000000000001,"5min":1.744,"15min":1.8140000000000001},"min":{"1min":0.59099999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.214,"5min":10.214,"15min":10.214},"last":10.214},{"interface":"front","average":{"1min":1.373,"5min":1.669,"15min":1.8220000000000001},"min":{"1min":0.45600000000000002,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":5.8239999999999998,"5min":5.952,"15min":5.952},"last":0.502}]},{"osd":6,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.704,"15min":1.8140000000000001},"min":{"1min":0.42999999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":5.7919999999999998,"5min":7.4119999999999999,"15min":7.4119999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.7210000000000001,"5min":2.6629999999999998,"15min":3.052},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":4.0129999999999999,"5min":9.827,"15min":9.827},"last":1.0860000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.492,"15min":1.5900000000000001},"min":{"1min":0.495,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":5.851,"5min":5.851,"15min":5.851},"last":0.754},{"interface":"front","average":{"1min":1.627,"5min":2.1080000000000001,"15min":2.3039999999999998},"min":{"1min":0.63900000000000001,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":3.4369999999999998,"5min":8.7690000000000001,"15min":8.7690000000000001},"last":0.63900000000000001}]}]},{"osd":0,"up_from":1136,"seq":4879082848449,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2699100,"kb_used_data":2040148,"kb_used_omap":280,"kb_used_meta":658663,"kb_avail":91046052,"statfs":{"total":95995035648,"available":93231157248,"internally_reserved":0,"allocated":2089111552,"data_stored":2087613589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":674471322},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,3,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:51:18 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5499999999999998,"15min":1.78},"min":{"1min":0.34300000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":69.680000000000007,"5min":69.680000000000007,"15min":69.680000000000007},"last":0.55100000000000005},{"interface":"front","average":{"1min":4.1509999999999998,"5min":2.7269999999999999,"15min":2.0870000000000002},"min":{"1min":0.48199999999999998,"5min":0.307,"15min":0.307},"max":{"1min":70.668999999999997,"5min":70.668999999999997,"15min":70.668999999999997},"last":2.3540000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.262,"15min":1.679},"min":{"1min":0.38400000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.1920000000000002,"5min":48.795000000000002,"15min":48.795000000000002},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.68700000000000006,"5min":2.9510000000000001,"15min":2.0049999999999999},"min":{"1min":0.39800000000000002,"5min":0.23699999999999999,"15min":0.23300000000000001},"max":{"1min":1.1100000000000001,"5min":48.856000000000002,"15min":48.856000000000002},"last":0.29999999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.0169999999999999,"15min":1.22},"min":{"1min":0.443,"5min":0.27500000000000002,"15min":0.247},"max":{"1min":4.2240000000000002,"5min":8.6750000000000007,"15min":10.475},"last":2.097},{"interface":"front","average":{"1min":0.69599999999999995,"5min":1.028,"15min":1.25},"min":{"1min":0.47899999999999998,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":0.89100000000000001,"5min":6.5220000000000002,"15min":9.2400000000000002},"last":2.1349999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.6799999999999999,"15min":1.448},"min":{"1min":0.60499999999999998,"5min":0.32400000000000001,"15min":0.27900000000000003},"max":{"1min":1.3,"5min":20.079000000000001,"15min":20.079000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.556,"15min":1.494},"min":{"1min":0.59199999999999997,"5min":0.32700000000000001,"15min":0.30199999999999999},"max":{"1min":4.5199999999999996,"5min":12.044,"15min":12.044},"last":0.39300000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.593,"15min":1.4910000000000001},"min":{"1min":0.61099999999999999,"5min":0.313,"15min":0.28399999999999997},"max":{"1min":4.3760000000000003,"5min":9.4280000000000008,"15min":9.4280000000000008},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.667,"15min":1.597},"min":{"1min":0.65200000000000002,"5min":0.33700000000000002,"15min":0.29699999999999999},"max":{"1min":1.2270000000000001,"5min":9.6289999999999996,"15min":9.6289999999999996},"last":0.71399999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":1.373,"15min":1.46},"min":{"1min":0.64000000000000001,"5min":0.372,"15min":0.315},"max":{"1min":1.202,"5min":6.2110000000000003,"15min":10.413},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.8169999999999999,"15min":1.76},"min":{"1min":0.71899999999999997,"5min":0.42599999999999999,"15min":0.35599999999999998},"max":{"1min":4.4900000000000002,"5min":9.3260000000000005,"15min":9.3260000000000005},"last":1.0720000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.2969999999999999,"15min":1.4419999999999999},"min":{"1min":0.55400000000000005,"5min":0.38,"15min":0.38},"max":{"1min":1.171,"5min":6.9669999999999996,"15min":9.2159999999999993},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.288,"5min":1.393,"15min":1.512},"min":{"1min":0.77600000000000002,"5min":0.438,"15min":0.39200000000000002},"max":{"1min":4.4420000000000002,"5min":10.087999999999999,"15min":14.361000000000001},"last":0.435}]}]},{"osd":3,"up_from":0,"seq":2847563317679,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986812,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1884724,"kb_used_data":985452,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":91860428,"statfs":{"total":95995035648,"available":94065078272,"internally_reserved":0,"allocated":1009102848,"data_stored":1008809600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:50:44 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":2.4460000000000002,"15min":1.7190000000000001},"min":{"1min":0.54700000000000004,"5min":0.26800000000000002,"15min":0.253},"max":{"1min":1.327,"5min":64.372,"15min":64.372},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.91800000000000004,"5min":2.1859999999999999,"15min":1.6859999999999999},"min":{"1min":0.59599999999999997,"5min":0.30099999999999999,"15min":0.27700000000000002},"max":{"1min":1.3160000000000001,"5min":64.356999999999999,"15min":64.356999999999999},"last":0.38600000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.4890000000000001,"15min":1.5629999999999999},"min":{"1min":0.48499999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.867,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.0880000000000001,"15min":0.94799999999999995},"min":{"1min":0.38700000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.0010000000000003,"5min":5.2249999999999996,"15min":5.2249999999999996},"last":0.55500000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":1.8560000000000001,"15min":1.583},"min":{"1min":0.53300000000000003,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":1.79,"5min":12.000999999999999,"15min":33.743000000000002},"last":0.626},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.6020000000000001,"15min":1.5580000000000001},"min":{"1min":0.42399999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.649,"5min":7.3730000000000002,"15min":33.805999999999997},"last":2.0299999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.079,"15min":1.133},"min":{"1min":0.64000000000000001,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":1.4059999999999999,"5min":8.1120000000000001,"15min":8.1120000000000001},"last":0.443},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.4119999999999999,"15min":1.4099999999999999},"min":{"1min":0.53800000000000003,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":2.2269999999999999,"5min":7.4180000000000001,"15min":16.507000000000001},"last":0.46899999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.8540000000000001,"15min":1.645},"min":{"1min":0.60099999999999998,"5min":0.35599999999999998,"15min":0.26900000000000002},"max":{"1min":1.407,"5min":11.763999999999999,"15min":17.212},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.4990000000000001,"15min":1.6000000000000001},"min":{"1min":0.56499999999999995,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.5800000000000001,"5min":12.446999999999999,"15min":17.268999999999998},"last":0.50700000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":2.1440000000000001,"15min":1.6319999999999999},"min":{"1min":0.498,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":1.3140000000000001,"5min":11.701000000000001,"15min":13.654},"last":0.93500000000000005},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.3170000000000002,"15min":1.716},"min":{"1min":0.84099999999999997,"5min":0.41899999999999998,"15min":0.35699999999999998},"max":{"1min":1.518,"5min":11.972,"15min":11.972},"last":0.95399999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.637,"15min":1.379},"min":{"1min":0.57899999999999996,"5min":0.373,"15min":0.373},"max":{"1min":1.2849999999999999,"5min":11.840999999999999,"15min":11.840999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.141,"5min":1.21,"15min":1.1899999999999999},"min":{"1min":0.54600000000000004,"5min":0.41299999999999998,"15min":0.34999999999999998},"max":{"1min":1.7669999999999999,"5min":9.6210000000000004,"15min":9.6210000000000004},"last":0.58799999999999997}]}]},{"osd":5,"up_from":37,"seq":158913790578,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1854496,"kb_used_data":947992,"kb_used_omap":0,"kb_used_meta":906496,"kb_avail":91890656,"statfs":{"total":95995035648,"available":94096031744,"internally_reserved":0,"allocated":970743808,"data_stored":970466714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928251904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:50:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":2.4790000000000001,"15min":4.2990000000000004},"min":{"1min":0.60999999999999999,"5min":0.253,"15min":0.23599999999999999},"max":{"1min":1.617,"5min":49.954999999999998,"15min":274.71899999999999},"last":2.0379999999999998},{"interface":"front","average":{"1min":0.93100000000000005,"5min":2.0449999999999999,"15min":4.0750000000000002},"min":{"1min":0.68799999999999994,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":1.375,"5min":50.042000000000002,"15min":274.73500000000001},"last":2.0529999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:51:14 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":1.6519999999999999,"15min":1.802},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.157,"5min":16.172000000000001,"15min":16.172000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.1779999999999999,"15min":1.1679999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.1880000000000002,"5min":13.728,"15min":13.728},"last":0.314}]},{"osd":2,"last update":"Mon Oct 9 09:50:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.619,"15min":1.6359999999999999},"min":{"1min":0.316,"5min":0.312,"15min":0.312},"max":{"1min":3.5920000000000001,"5min":48.170999999999999,"15min":50.847000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.615,"5min":2.1480000000000001,"15min":1.99},"min":{"1min":0.46400000000000002,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":8.7759999999999998,"5min":48.128,"15min":50.656999999999996},"last":0.88600000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:50:34 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.085,"15min":1.29},"min":{"1min":0.44700000000000001,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":1.298,"5min":7.2270000000000003,"15min":10.125999999999999},"last":2.2290000000000001},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.452,"15min":1.3819999999999999},"min":{"1min":0.66600000000000004,"5min":0.38,"15min":0.29099999999999998},"max":{"1min":2.79,"5min":16.201000000000001,"15min":16.201000000000001},"last":1.0189999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:51:08 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.5229999999999999,"15min":1.3560000000000001},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.27000000000000002},"max":{"1min":2.798,"5min":9.4209999999999994,"15min":9.4209999999999994},"last":2.1699999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.9410000000000001,"15min":1.722},"min":{"1min":0.53900000000000003,"5min":0.374,"15min":0.33300000000000002},"max":{"1min":1.526,"5min":16.053000000000001,"15min":16.053000000000001},"last":1.052}]},{"osd":6,"last update":"Mon Oct 9 09:50:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.74,"15min":1.946},"min":{"1min":0.41699999999999998,"5min":0.35399999999999998,"15min":0.30399999999999999},"max":{"1min":1.6779999999999999,"5min":9.6270000000000007,"15min":140.179},"last":2.2719999999999998},{"interface":"front","average":{"1min":1.3149999999999999,"5min":2.0870000000000002,"15min":2.23},"min":{"1min":0.66300000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.9470000000000001,"5min":16.145,"15min":137.928},"last":1.96}]},{"osd":7,"last update":"Mon Oct 9 09:51:08 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.593,"15min":1.629},"min":{"1min":0.47799999999999998,"5min":0.432,"15min":0.33400000000000002},"max":{"1min":1.367,"5min":16.119,"15min":18.683},"last":1.282},{"interface":"front","average":{"1min":0.91400000000000003,"5min":1.0589999999999999,"15min":1.2130000000000001},"min":{"1min":0.5,"5min":0.34200000000000003,"15min":0.32100000000000001},"max":{"1min":1.395,"5min":4.6779999999999999,"15min":10.192},"last":0.91200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7721,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2098790400,"data_stored":2098790400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2359361536,"data_stored":2359361536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1369571328,"data_stored":1369571328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1004666880,"data_stored":1004666880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":984088576,"data_stored":984088576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1791492096,"data_stored":1791492096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:51:23.462 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T09:51:23.462 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:51:23.463 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T09:51:23.463 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:51:23.463 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:51:23.463 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T09:51:23.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:23.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:23.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:23.574 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:23.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:23.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:23.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:23.706 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:23.733 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:23.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:23.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:23.838 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:23.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:23.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:23.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:23.926 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:51:23.933 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:51:23.969 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:23.997 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:24.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:24.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:24.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:24.134 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:24.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:24.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:24.237 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:24.246 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:24 smithi181 bash[59345]: cluster 2023-10-09T09:51:22.195532+0000 mgr.y (mgr.34107) 1364 : cluster 0 pgmap v2232: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:51:24.268 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:24.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:24.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:24.373 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:24.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:24.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:24.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:24.408 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 57720 57704 100.167 70.25 0.00800345 0.00997788 2023-10-09T09:51:24.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:24 smithi163 bash[78107]: cluster 2023-10-09T09:51:22.195532+0000 mgr.y (mgr.34107) 1364 : cluster 0 pgmap v2232: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:51:24.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:24 smithi163 bash[75425]: cluster 2023-10-09T09:51:22.195532+0000 mgr.y (mgr.34107) 1364 : cluster 0 pgmap v2232: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:51:24.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:24.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:24.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:24.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:24.636 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:24.660 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:24.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:24.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:24.767 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:24.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:24.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:24.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:24.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:24.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:24.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:24.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:25.033 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:25.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:25.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:25.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:25.165 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:25.194 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:25.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:25.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:25.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:25.325 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:25.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:25.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:25.408 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 58913 58897 99.4747 74.5625 0.0154606 0.0100468 2023-10-09T09:51:25.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:25.455 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:25.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:25.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:25.559 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:25.594 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:25.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:25.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:25.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:25.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:25.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:25.832 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:25.865 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:25.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:25.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:26.001 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:26.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:26.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:26.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:26.114 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:26 smithi181 bash[59345]: cluster 2023-10-09T09:51:24.196047+0000 mgr.y (mgr.34107) 1365 : cluster 0 pgmap v2233: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T09:51:26.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:26 smithi181 bash[59345]: audit 2023-10-09T09:51:25.401098+0000 mon.a (mon.0) 2476 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:26.132 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:26.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:26.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:26.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:26.244 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:26 smithi163 bash[78107]: cluster 2023-10-09T09:51:24.196047+0000 mgr.y (mgr.34107) 1365 : cluster 0 pgmap v2233: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T09:51:26.244 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:26 smithi163 bash[78107]: audit 2023-10-09T09:51:25.401098+0000 mon.a (mon.0) 2476 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:26.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:26 smithi163 bash[75425]: cluster 2023-10-09T09:51:24.196047+0000 mgr.y (mgr.34107) 1365 : cluster 0 pgmap v2233: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T09:51:26.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:26 smithi163 bash[75425]: audit 2023-10-09T09:51:25.401098+0000 mon.a (mon.0) 2476 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:26.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:26.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:26.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:26.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:26.402 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:26.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:26.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:26.408 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 59803 59787 98.3207 55.625 0.00595201 0.0101655 2023-10-09T09:51:26.506 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:26.539 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:26.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:26.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:26.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:26.673 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:26.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:26.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:26.777 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:26.806 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:26.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:26.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:26.908 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:26.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:26.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:26.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:27.040 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:27.065 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:27.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:27.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:27.172 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:27.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:27.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:27.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:27.298 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:27.321 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:27.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:27.408 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 60964 60948 97.66 72.5625 0.0135367 0.010234 2023-10-09T09:51:27.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:27.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:27.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:27.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:27.560 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:27.590 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:27.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:27.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:27.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:27.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:27.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:27.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:27.829 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:27.859 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:27.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:27.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:27.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:27.989 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:27.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:27.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:28.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:28.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:28 smithi181 bash[59345]: cluster 2023-10-09T09:51:26.196535+0000 mgr.y (mgr.34107) 1366 : cluster 0 pgmap v2234: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:51:28.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:28.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:28.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:28.224 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:28.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:28.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:28.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:28.285 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:51:28.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:28.383 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:28.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:28.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:28.408 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:51:28.408495+0000 min lat: 0.00138896 max lat: 0.341984 avg lat: 0.0103062 2023-10-09T09:51:28.408 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:51:28.409 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 62090 62074 96.9778 70.375 0.0108839 0.0103062 2023-10-09T09:51:28.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:28 smithi163 bash[75425]: cluster 2023-10-09T09:51:26.196535+0000 mgr.y (mgr.34107) 1366 : cluster 0 pgmap v2234: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:51:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:28 smithi163 bash[78107]: cluster 2023-10-09T09:51:26.196535+0000 mgr.y (mgr.34107) 1366 : cluster 0 pgmap v2234: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T09:51:28.488 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:28.518 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:28.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:28.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:28.621 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:28.650 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:28.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:28.760 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:28.790 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:28.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:28.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:28.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:28.922 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:28.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:28.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:29.029 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:29.058 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:29.058 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:51:29.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:29.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:29.160 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:29.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:29.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:29.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:29.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:29.323 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:29.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:29.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:29.408 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 63232 63216 96.3532 71.375 0.00886502 0.0103728 2023-10-09T09:51:29.428 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:29.453 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:29.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:29.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:29.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:29.587 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:29.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:29.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:29.693 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:29.718 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:29.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:29.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:29.824 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:29.848 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:29.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:29.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:29.951 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:29.976 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:29.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:29.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:30.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:30.105 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:30.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:30.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:30.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:30.234 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:30.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:30.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:30 smithi181 bash[59345]: cluster 2023-10-09T09:51:28.197152+0000 mgr.y (mgr.34107) 1367 : cluster 0 pgmap v2235: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:51:30.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:30.362 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:30.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:30.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:30.408 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 64365 64349 95.7449 70.8125 0.0123607 0.0104388 2023-10-09T09:51:30.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:30 smithi163 bash[75425]: cluster 2023-10-09T09:51:28.197152+0000 mgr.y (mgr.34107) 1367 : cluster 0 pgmap v2235: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:51:30.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:30 smithi163 bash[78107]: cluster 2023-10-09T09:51:28.197152+0000 mgr.y (mgr.34107) 1367 : cluster 0 pgmap v2235: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T09:51:30.496 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:30.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:30.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:30.601 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:30.642 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:31.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:31.734 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 65480 65464 95.1388 69.6875 0.00944736 0.0105053 2023-10-09T09:51:31.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:31.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:31.883 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:31.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:31.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:31.986 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:32.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:32.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:32.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:32.111 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:32.120 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:31 smithi163 bash[78107]: cluster 2023-10-09T09:51:30.197695+0000 mgr.y (mgr.34107) 1368 : cluster 0 pgmap v2236: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T09:51:32.120 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:31 smithi163 bash[75425]: cluster 2023-10-09T09:51:30.197695+0000 mgr.y (mgr.34107) 1368 : cluster 0 pgmap v2236: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T09:51:32.145 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:32.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:32.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:31 smithi181 bash[59345]: cluster 2023-10-09T09:51:30.197695+0000 mgr.y (mgr.34107) 1368 : cluster 0 pgmap v2236: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T09:51:32.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:32.577 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 66575 66559 94.5318 68.4375 0.00630397 0.0105731 2023-10-09T09:51:32.659 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:51:32.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:32.867 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:51:32.869 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:51:32.869 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:51:32.869 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:51:32.869 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:51:32.879 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:51:32.895 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:32.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:32.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:32.919 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:51:32.923 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:51:32.923 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:51:32.923 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:51:32.923 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:51:32.923 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:51:32.923 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:51:32.923 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:51:32.923 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:51:32.971 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:51:32.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:33.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:33.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:32 smithi181 bash[59345]: audit 2023-10-09T09:51:32.608810+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.163:0/1756694465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:51:33.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:32 smithi181 bash[59345]: audit 2023-10-09T09:51:32.910361+0000 mon.c (mon.2) 128 : audit 1 from='client.? 172.21.15.163:0/1577505383' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:51:33.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:32 smithi181 bash[59345]: audit 2023-10-09T09:51:32.910837+0000 mon.a (mon.0) 2477 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:51:33.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:33.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:33.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:32 smithi163 bash[78107]: audit 2023-10-09T09:51:32.608810+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.163:0/1756694465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:51:33.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:32 smithi163 bash[78107]: audit 2023-10-09T09:51:32.910361+0000 mon.c (mon.2) 128 : audit 1 from='client.? 172.21.15.163:0/1577505383' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:51:33.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:32 smithi163 bash[78107]: audit 2023-10-09T09:51:32.910837+0000 mon.a (mon.0) 2477 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:51:33.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:32 smithi163 bash[75425]: audit 2023-10-09T09:51:32.608810+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.163:0/1756694465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:51:33.405 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:32 smithi163 bash[75425]: audit 2023-10-09T09:51:32.910361+0000 mon.c (mon.2) 128 : audit 1 from='client.? 172.21.15.163:0/1577505383' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:51:33.405 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:32 smithi163 bash[75425]: audit 2023-10-09T09:51:32.910837+0000 mon.a (mon.0) 2477 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:51:33.409 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 67541 67525 93.7727 60.375 0.00817937 0.0106586 2023-10-09T09:51:33.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:33.515 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:33.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:33.618 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:33.628 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:51:33] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.33.4" 2023-10-09T09:51:33.643 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:33.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:33.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:33.663 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:51:33.664 INFO:teuthology.orchestra.run.smithi181.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T09:51:33.664 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:51:33.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:33.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:33.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:33.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:33.893 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:33.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:33.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:33.945 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:33 smithi181 bash[59345]: cluster 2023-10-09T09:51:32.198290+0000 mgr.y (mgr.34107) 1369 : cluster 0 pgmap v2237: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:51:33.945 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:33 smithi181 bash[59345]: cluster 2023-10-09T09:51:32.936857+0000 mon.a (mon.0) 2478 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:51:33.945 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:33 smithi181 bash[59345]: cluster 2023-10-09T09:51:32.943694+0000 osd.5 (osd.5) 866 : cluster 3 failed to encode map e1276 with expected crc 2023-10-09T09:51:33.946 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:33 smithi181 bash[59345]: audit 2023-10-09T09:51:32.947154+0000 mon.a (mon.0) 2479 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:51:33.946 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:33 smithi181 bash[59345]: cluster 2023-10-09T09:51:32.947293+0000 mon.a (mon.0) 2480 : cluster 0 osdmap e1276: 8 total, 8 up, 6 in 2023-10-09T09:51:34.042 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:34.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:34.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:34.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:34.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:34.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[75425]: cluster 2023-10-09T09:51:32.198290+0000 mgr.y (mgr.34107) 1369 : cluster 0 pgmap v2237: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:51:34.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[75425]: cluster 2023-10-09T09:51:32.936857+0000 mon.a (mon.0) 2478 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:51:34.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[75425]: cluster 2023-10-09T09:51:32.943694+0000 osd.5 (osd.5) 866 : cluster 3 failed to encode map e1276 with expected crc 2023-10-09T09:51:34.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[75425]: audit 2023-10-09T09:51:32.947154+0000 mon.a (mon.0) 2479 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:51:34.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[75425]: cluster 2023-10-09T09:51:32.947293+0000 mon.a (mon.0) 2480 : cluster 0 osdmap e1276: 8 total, 8 up, 6 in 2023-10-09T09:51:34.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[78107]: cluster 2023-10-09T09:51:32.198290+0000 mgr.y (mgr.34107) 1369 : cluster 0 pgmap v2237: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T09:51:34.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[78107]: cluster 2023-10-09T09:51:32.936857+0000 mon.a (mon.0) 2478 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:51:34.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[78107]: cluster 2023-10-09T09:51:32.943694+0000 osd.5 (osd.5) 866 : cluster 3 failed to encode map e1276 with expected crc 2023-10-09T09:51:34.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[78107]: audit 2023-10-09T09:51:32.947154+0000 mon.a (mon.0) 2479 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:51:34.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:33 smithi163 bash[78107]: cluster 2023-10-09T09:51:32.947293+0000 mon.a (mon.0) 2480 : cluster 0 osdmap e1276: 8 total, 8 up, 6 in 2023-10-09T09:51:34.202 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:34.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:34.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:34.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:34.327 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:34.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:34.409 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 15 68470 68455 92.9977 58.125 0.0165917 0.0107474 2023-10-09T09:51:34.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:34.456 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:34.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:34.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:34.562 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:34.596 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:34.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:34.700 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:34.751 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:34.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:34.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:34.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:34.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:34.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:34.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:35.022 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:35.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:35.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:35.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:35.151 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:35.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:35.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:35.257 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:35.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:35.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:35.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:35.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:35.409 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 69395 69379 92.2476 57.75 0.00408849 0.0108078 2023-10-09T09:51:35.428 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:35.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:35.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:35.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:35.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:35.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:35.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:35.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:35.696 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:35.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:35.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:35.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:35.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:35.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:35.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:35.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:35.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:35.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:36.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:36.093 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:36.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:36.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:36.197 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:36.205 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:35 smithi181 bash[59345]: cluster 2023-10-09T09:51:34.198770+0000 mgr.y (mgr.34107) 1370 : cluster 0 pgmap v2239: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T09:51:36.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:35 smithi163 bash[78107]: cluster 2023-10-09T09:51:34.198770+0000 mgr.y (mgr.34107) 1370 : cluster 0 pgmap v2239: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T09:51:36.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:35 smithi163 bash[75425]: cluster 2023-10-09T09:51:34.198770+0000 mgr.y (mgr.34107) 1370 : cluster 0 pgmap v2239: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T09:51:36.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:36.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:36.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:36.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:36.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:36.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:36.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:36.409 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 70403 70387 91.6381 63 0.0232132 0.0109059 2023-10-09T09:51:36.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:36.490 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:36.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:36.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:36.595 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:36.618 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:36.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:36.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:36.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:36.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:36.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:36.858 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:36.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:36.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:36.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:37.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:37.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:37.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:37.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:37.130 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:37.157 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:37.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:37.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:37.260 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:37.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:37.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:37.390 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:37.411 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 71481 71465 91.139 67.375 0.0171525 0.0109663 2023-10-09T09:51:37.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:37.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:37.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:37.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:37.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:37.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:37.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:37.718 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:37.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:37.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:37.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:37.863 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:37.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:37.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:37.968 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:37.997 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:38.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:38.102 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:38.110 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:37 smithi181 bash[59345]: cluster 2023-10-09T09:51:36.199789+0000 mgr.y (mgr.34107) 1371 : cluster 0 pgmap v2240: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:51:38.135 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:38.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:38.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:38.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:37 smithi163 bash[75425]: cluster 2023-10-09T09:51:36.199789+0000 mgr.y (mgr.34107) 1371 : cluster 0 pgmap v2240: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:51:38.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:37 smithi163 bash[78107]: cluster 2023-10-09T09:51:36.199789+0000 mgr.y (mgr.34107) 1371 : cluster 0 pgmap v2240: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T09:51:38.239 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:38.264 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:38.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:38.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:38.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:38.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:38.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:38.411 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 72497 72481 90.5862 63.5 0.00818875 0.011033 2023-10-09T09:51:38.465 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:51:38.465 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:51:38.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:38.526 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:38.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:38.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:38.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:38.667 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:38.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:38.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:38.774 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:38.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:38.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:38.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:38.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:38.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:38.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:38.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:39.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:39.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:39.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:39.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:39.175 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:39.200 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:39.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:39.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:39.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:39.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:39.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:39.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:39.411 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 73532 73516 90.0783 64.6875 0.0155713 0.0110944 2023-10-09T09:51:39.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:39.462 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:39.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:39.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:39.567 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:39.595 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:39.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:39.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:39.699 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:39.730 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:39.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:39.760 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 6, 2, 5] out_osds: [7, 3] dead_osds: [] live_osds: [2, 3, 4, 5, 6, 7, 0, 1] 2023-10-09T09:51:39.760 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:51:39.760 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T09:51:39.760 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:51:39.788 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T09:51:39.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:39.910 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:39.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:39.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:39.925 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:51:39.925 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:51:40.014 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:40.041 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:40.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:40.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:40.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:40.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:39 smithi163 bash[75425]: cluster 2023-10-09T09:51:38.200635+0000 mgr.y (mgr.34107) 1372 : cluster 0 pgmap v2241: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T09:51:40.161 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:39 smithi163 bash[78107]: cluster 2023-10-09T09:51:38.200635+0000 mgr.y (mgr.34107) 1372 : cluster 0 pgmap v2241: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T09:51:40.182 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:40.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:40.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:40.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:40.297 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:39 smithi181 bash[59345]: cluster 2023-10-09T09:51:38.200635+0000 mgr.y (mgr.34107) 1372 : cluster 0 pgmap v2241: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T09:51:40.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:40.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:40.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:40.411 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 74392 74376 89.3796 53.75 0.00422868 0.011182 2023-10-09T09:51:40.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:40.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:40.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:40.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:40.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:40.581 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:40.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:40.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:40.689 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:40.714 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:40.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:40.819 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:40.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:40.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:40.951 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:40.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:40.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:40.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:41.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:41.113 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:41.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:41.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:41.218 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:40 smithi163 bash[78107]: audit 2023-10-09T09:51:40.403712+0000 mon.a (mon.0) 2481 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:41.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:40 smithi163 bash[75425]: audit 2023-10-09T09:51:40.403712+0000 mon.a (mon.0) 2481 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:41.226 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:40 smithi181 bash[59345]: audit 2023-10-09T09:51:40.403712+0000 mon.a (mon.0) 2481 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:41.251 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:41.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:41.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:41.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:41.387 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:41.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:41.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:41.411 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 75323 75307 88.791 58.1875 0.0232577 0.0112562 2023-10-09T09:51:41.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:41.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:41.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:41.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:41.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:41.669 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:41.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:41.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:41.775 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:41.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:41.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:41.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:41.908 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:41.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:41.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:41.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:42.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:42.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:41 smithi181 bash[59345]: cluster 2023-10-09T09:51:40.201191+0000 mgr.y (mgr.34107) 1373 : cluster 0 pgmap v2242: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T09:51:42.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:41 smithi181 bash[59345]: audit 2023-10-09T09:51:41.573689+0000 mon.a (mon.0) 2482 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:42.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:42.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:42.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:42.182 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:42.209 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:42.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:42.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:41 smithi163 bash[75425]: cluster 2023-10-09T09:51:40.201191+0000 mgr.y (mgr.34107) 1373 : cluster 0 pgmap v2242: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T09:51:42.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:41 smithi163 bash[75425]: audit 2023-10-09T09:51:41.573689+0000 mon.a (mon.0) 2482 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:42.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:41 smithi163 bash[78107]: cluster 2023-10-09T09:51:40.201191+0000 mgr.y (mgr.34107) 1373 : cluster 0 pgmap v2242: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T09:51:42.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:41 smithi163 bash[78107]: audit 2023-10-09T09:51:41.573689+0000 mon.a (mon.0) 2482 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:42.313 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:42.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:42.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:42.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:42.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 76241 76225 88.2092 57.375 0.0173981 0.01133 2023-10-09T09:51:42.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:42.476 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:42.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:42.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:42.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:42.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:42.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:42.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:42.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:42.746 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:42.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:42.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:42.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:42.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:42.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:42.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:42.980 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:43.006 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:43.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:43.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:43.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:43.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:43.242 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:43.266 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:43.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:43.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:43.370 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:43.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:43.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:43.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:43.411 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 77182 77166 87.6746 58.8125 0.00710847 0.0114003 2023-10-09T09:51:43.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:51:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:51:43] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-09T09:51:43.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:43.535 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:43.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:43.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:43.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:43.681 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:43.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:43.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:43.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:43.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:43.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:43.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:43.947 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:43.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:44.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:44.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:43 smithi181 bash[59345]: cluster 2023-10-09T09:51:42.201954+0000 mgr.y (mgr.34107) 1374 : cluster 0 pgmap v2243: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T09:51:44.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:44.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:44.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:44.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:44.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:43 smithi163 bash[78107]: cluster 2023-10-09T09:51:42.201954+0000 mgr.y (mgr.34107) 1374 : cluster 0 pgmap v2243: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T09:51:44.199 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:43 smithi163 bash[75425]: cluster 2023-10-09T09:51:42.201954+0000 mgr.y (mgr.34107) 1374 : cluster 0 pgmap v2243: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T09:51:44.220 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:44.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:44.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:44.323 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:44.362 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:44.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:44.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:44.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 78124 78108 87.1603 58.875 0.0120683 0.0114662 2023-10-09T09:51:44.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:44.498 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:44.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:44.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:44.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:44.636 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:44.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:44.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:44.670 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:51:44.670 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:51:44.671 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:51:44.672 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:51:44.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:44.773 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:44.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:44.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:44.876 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:44.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:44.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:44.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:45.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:45.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:45.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:45.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:45.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:45.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:45.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:45.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:45.282 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:45.313 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:45.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:45.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:45.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 79041 79025 86.6365 57.3125 0.0311036 0.0115376 2023-10-09T09:51:45.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:45.449 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:45.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:45.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:45.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:45.577 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:45.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:45.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:45.682 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:45.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:45.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:45.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:45.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:45.852 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:45.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:45.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:45.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:45.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:45.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:45.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:46.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:46.099 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:45 smithi163 bash[78107]: cluster 2023-10-09T09:51:44.202518+0000 mgr.y (mgr.34107) 1375 : cluster 0 pgmap v2244: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T09:51:46.099 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:45 smithi163 bash[75425]: cluster 2023-10-09T09:51:44.202518+0000 mgr.y (mgr.34107) 1375 : cluster 0 pgmap v2244: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T09:51:46.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:46.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:46.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:46.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:46.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:46.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:46.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:46.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:45 smithi181 bash[59345]: cluster 2023-10-09T09:51:44.202518+0000 mgr.y (mgr.34107) 1375 : cluster 0 pgmap v2244: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T09:51:46.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:46.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:46.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:46.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:46.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 79942 79926 86.1137 56.3125 0.0225757 0.011606 2023-10-09T09:51:46.510 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:46.538 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:46.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:46.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:46.641 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:46.676 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:46.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:46.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:46.782 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:46.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:46.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:46.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:46.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:46.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:46.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:47.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:47.096 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:47.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:47.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:47.201 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:47.241 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:47.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:47.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:47.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:47.373 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:47.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:47.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:47.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 80786 80770 85.5481 52.75 0.0151506 0.0116828 2023-10-09T09:51:47.480 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:47.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:47.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:47.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:47.607 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:47.644 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:47.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:47.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:47.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:47.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:47.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:47.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:47.885 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:47.908 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:47.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:47.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:48.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:48.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:47 smithi163 bash[78107]: cluster 2023-10-09T09:51:46.203170+0000 mgr.y (mgr.34107) 1376 : cluster 0 pgmap v2245: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:51:48.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:47 smithi163 bash[78107]: audit 2023-10-09T09:51:47.493367+0000 mon.a (mon.0) 2483 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:48.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:47 smithi163 bash[78107]: audit 2023-10-09T09:51:47.511793+0000 mon.a (mon.0) 2484 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:48.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:47 smithi163 bash[75425]: cluster 2023-10-09T09:51:46.203170+0000 mgr.y (mgr.34107) 1376 : cluster 0 pgmap v2245: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:51:48.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:47 smithi163 bash[75425]: audit 2023-10-09T09:51:47.493367+0000 mon.a (mon.0) 2483 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:48.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:47 smithi163 bash[75425]: audit 2023-10-09T09:51:47.511793+0000 mon.a (mon.0) 2484 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:48.063 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:48.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:47 smithi181 bash[59345]: cluster 2023-10-09T09:51:46.203170+0000 mgr.y (mgr.34107) 1376 : cluster 0 pgmap v2245: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T09:51:48.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:47 smithi181 bash[59345]: audit 2023-10-09T09:51:47.493367+0000 mon.a (mon.0) 2483 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:48.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:47 smithi181 bash[59345]: audit 2023-10-09T09:51:47.511793+0000 mon.a (mon.0) 2484 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:48.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:48.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:48.172 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:48.202 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:48.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:48.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:48.305 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:48.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:48.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:48.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:51:48.412476+0000 min lat: 0.00138896 max lat: 0.341984 avg lat: 0.0117531 2023-10-09T09:51:48.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:51:48.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 81662 81646 85.0348 54.75 0.0145533 0.0117531 2023-10-09T09:51:48.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:48.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:48.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:48.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:48.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:48.608 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:48.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:48.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:48.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:48.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:48.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:48.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:48.847 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:48.875 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:48.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:48.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:48.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:48.998 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:48.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:48.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:49.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:49 smithi181 bash[59345]: audit 2023-10-09T09:51:48.043322+0000 mon.a (mon.0) 2485 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:49.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:49 smithi181 bash[59345]: audit 2023-10-09T09:51:48.055472+0000 mon.a (mon.0) 2486 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:49.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:49 smithi181 bash[59345]: audit 2023-10-09T09:51:48.760666+0000 mon.a (mon.0) 2487 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:49.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:49 smithi181 bash[59345]: audit 2023-10-09T09:51:48.763826+0000 mon.a (mon.0) 2488 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:49.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:49 smithi181 bash[59345]: audit 2023-10-09T09:51:48.774124+0000 mon.a (mon.0) 2489 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:49.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:49.125 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:49.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:49.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:49.228 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:49.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:49.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:49.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:49.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:49.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:49.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:49.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 82615 82599 84.6171 59.5625 0.00972647 0.0118126 2023-10-09T09:51:49.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:49 smithi163 bash[78107]: audit 2023-10-09T09:51:48.043322+0000 mon.a (mon.0) 2485 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:49.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:49 smithi163 bash[78107]: audit 2023-10-09T09:51:48.055472+0000 mon.a (mon.0) 2486 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:49.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:49 smithi163 bash[78107]: audit 2023-10-09T09:51:48.760666+0000 mon.a (mon.0) 2487 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:49.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:49 smithi163 bash[78107]: audit 2023-10-09T09:51:48.763826+0000 mon.a (mon.0) 2488 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:49.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:49 smithi163 bash[78107]: audit 2023-10-09T09:51:48.774124+0000 mon.a (mon.0) 2489 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:49 smithi163 bash[75425]: audit 2023-10-09T09:51:48.043322+0000 mon.a (mon.0) 2485 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:49 smithi163 bash[75425]: audit 2023-10-09T09:51:48.055472+0000 mon.a (mon.0) 2486 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:49 smithi163 bash[75425]: audit 2023-10-09T09:51:48.760666+0000 mon.a (mon.0) 2487 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:49 smithi163 bash[75425]: audit 2023-10-09T09:51:48.763826+0000 mon.a (mon.0) 2488 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:49 smithi163 bash[75425]: audit 2023-10-09T09:51:48.774124+0000 mon.a (mon.0) 2489 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:51:49.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:49.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:49.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:49.625 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:49.649 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:49.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:49.759 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:49.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:49.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:49.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:49.889 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:49.922 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:49.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:49.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:50.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:50.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:50.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:50.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:50.155 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:50.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:50.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:50.292 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:50.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:50 smithi181 bash[59345]: cluster 2023-10-09T09:51:48.203905+0000 mgr.y (mgr.34107) 1377 : cluster 0 pgmap v2246: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-09T09:51:50.328 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:50.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:50 smithi163 bash[78107]: cluster 2023-10-09T09:51:48.203905+0000 mgr.y (mgr.34107) 1377 : cluster 0 pgmap v2246: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-09T09:51:50.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:50.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:50 smithi163 bash[75425]: cluster 2023-10-09T09:51:48.203905+0000 mgr.y (mgr.34107) 1377 : cluster 0 pgmap v2246: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-09T09:51:50.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 83531 83515 84.1756 57.25 0.0238852 0.0118745 2023-10-09T09:51:50.434 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:50.462 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:50.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:50.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:50.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:50.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:50.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:50.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:50.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:50.731 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:50.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:50.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:50.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:50.873 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:50.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:50.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:50.975 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:51.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:51.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:51.114 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:51.141 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:51.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:51.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:51.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:51.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:51.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:51.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:51.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:51.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 84451 84435 83.7522 57.5 0.0103092 0.0119342 2023-10-09T09:51:51.420 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:51.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:51.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:51.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:51.556 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:51.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:51.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:51.660 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:51.698 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:51.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:51.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:51.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:51.832 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:51.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:51.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:51.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:51.954 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:51.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:51.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:52.058 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:52.089 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:52.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:52.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:52.193 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:52.221 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:52.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:52.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:52 smithi181 bash[59345]: cluster 2023-10-09T09:51:50.204421+0000 mgr.y (mgr.34107) 1378 : cluster 0 pgmap v2247: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T09:51:52.325 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:52.351 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:52.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:52.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:52.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 85230 85214 83.2042 48.6875 0.0126428 0.0120124 2023-10-09T09:51:52.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:52.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:52 smithi163 bash[78107]: cluster 2023-10-09T09:51:50.204421+0000 mgr.y (mgr.34107) 1378 : cluster 0 pgmap v2247: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T09:51:52.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:52 smithi163 bash[75425]: cluster 2023-10-09T09:51:50.204421+0000 mgr.y (mgr.34107) 1378 : cluster 0 pgmap v2247: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T09:51:52.487 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:52.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:52.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:52.590 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:52.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:52.622 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:52.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:52.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:52.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:52.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:52.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:52.851 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:52.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:52.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:52.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:52.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:53.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:53.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:53.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:53.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:53.141 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:53.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:53.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:53.271 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:53.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:53.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:53.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:53.403 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:53.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:53.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:53.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 86069 86053 82.7309 52.4375 0.0117198 0.012082 2023-10-09T09:51:53.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:53.540 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:53.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:53.645 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:53.672 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:53.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:53.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:51:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:51:53] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-09T09:51:53.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:53.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:53.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:53.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:53.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:53.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:53.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:54.039 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:54.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:54 smithi163 bash[75425]: cluster 2023-10-09T09:51:52.205137+0000 mgr.y (mgr.34107) 1379 : cluster 0 pgmap v2248: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:51:54.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:54.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:54.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:54.172 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:54.180 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:54 smithi181 bash[59345]: cluster 2023-10-09T09:51:52.205137+0000 mgr.y (mgr.34107) 1379 : cluster 0 pgmap v2248: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:51:54.204 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:54.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:54.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:54.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:51:54.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:54.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:51:54.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 87010 86994 82.3684 58.8125 0.00771006 0.012134 2023-10-09T09:51:54.439 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:54 smithi163 bash[78107]: cluster 2023-10-09T09:51:52.205137+0000 mgr.y (mgr.34107) 1379 : cluster 0 pgmap v2248: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:51:54.469 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:54.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:54.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:54.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:54.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:54.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:54.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:54.717 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:54.741 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:54.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:54.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:54.845 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:54.871 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:54.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:54.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:54.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:55.011 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:55.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:55.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:55.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:55.194 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:55.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:55.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:55.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:55.333 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:55.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:55.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:55.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 87925 87909 81.9925 57.1875 0.00799956 0.0121906 2023-10-09T09:51:55.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:55.460 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:55.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:55.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:55.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:55.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:55.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:55.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:55.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:55.727 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:55.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:55.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:55.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:55.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:55.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:55.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:55.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:55.998 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:56.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:56.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:56.102 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:56.129 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:56.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:56.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:56.235 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:56.244 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:56 smithi181 bash[59345]: cluster 2023-10-09T09:51:54.205610+0000 mgr.y (mgr.34107) 1380 : cluster 0 pgmap v2249: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-09T09:51:56.245 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:56 smithi181 bash[59345]: audit 2023-10-09T09:51:55.403991+0000 mon.a (mon.0) 2490 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:56.261 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:56.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:56.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:56.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:56.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:56 smithi163 bash[75425]: cluster 2023-10-09T09:51:54.205610+0000 mgr.y (mgr.34107) 1380 : cluster 0 pgmap v2249: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-09T09:51:56.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:56 smithi163 bash[75425]: audit 2023-10-09T09:51:55.403991+0000 mon.a (mon.0) 2490 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:56.377 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:56 smithi163 bash[78107]: cluster 2023-10-09T09:51:54.205610+0000 mgr.y (mgr.34107) 1380 : cluster 0 pgmap v2249: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-09T09:51:56.377 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:56 smithi163 bash[78107]: audit 2023-10-09T09:51:55.403991+0000 mon.a (mon.0) 2490 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:56.390 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:56.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:56.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 88831 88815 81.6194 56.625 0.0194065 0.0122466 2023-10-09T09:51:56.494 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:56.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:56.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:56.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:56.624 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:56.660 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:56.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:56.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:56.761 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:56.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:56.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:56.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:56.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:56.918 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:56.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:57.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:57.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:57.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:57.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:57.150 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:57.173 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:57.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:57.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:57.283 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:57.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:57.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:57.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 89743 89727 81.2625 57 0.0144761 0.0122998 2023-10-09T09:51:57.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:57.451 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:57.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:57.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:57.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:57.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:57.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:57.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:57.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:57.728 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:57.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:57.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:57.855 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:57.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:57.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:57.959 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:57.985 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:57.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:57.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:58.089 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:58.098 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:51:58 smithi163 bash[78107]: cluster 2023-10-09T09:51:56.206108+0000 mgr.y (mgr.34107) 1381 : cluster 0 pgmap v2250: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T09:51:58.099 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:51:58 smithi163 bash[75425]: cluster 2023-10-09T09:51:56.206108+0000 mgr.y (mgr.34107) 1381 : cluster 0 pgmap v2250: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T09:51:58.124 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:58.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:58.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:58.227 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:58.251 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:58.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:58.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:58.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:58.390 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:58.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:58.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:58.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 90685 90669 80.9427 58.875 0.0136138 0.0123485 2023-10-09T09:51:58.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:58.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:58.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:58.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:58.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:51:58 smithi181 bash[59345]: cluster 2023-10-09T09:51:56.206108+0000 mgr.y (mgr.34107) 1381 : cluster 0 pgmap v2250: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T09:51:58.628 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:58.658 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:58.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:58.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:58.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:58.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:58.790 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:51:58.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:51:58.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:58.921 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:51:58.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:58.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:51:59.025 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:59.051 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:51:59.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:59.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:51:59.154 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:59.180 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:51:59.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:59.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:51:59.286 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:59.311 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:59.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:59.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:59.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 91600 91584 80.608 57.1875 0.00834182 0.0123989 2023-10-09T09:51:59.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:59.441 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:59.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:59.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:59.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:59.594 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:51:59.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:59.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:51:59.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:59.714 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:51:59.714 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:51:59.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:51:59.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:59.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:51:59.842 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:59.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:51:59.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:51:59.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:51:59.978 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:00.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:00.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:00.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:00.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:52:00.075 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:00 smithi163 bash[78107]: cluster 2023-10-09T09:51:58.206840+0000 mgr.y (mgr.34107) 1382 : cluster 0 pgmap v2251: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 57 MiB/s wr, 907 op/s 2023-10-09T09:52:00.076 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:00 smithi163 bash[78107]: audit 2023-10-09T09:51:59.991175+0000 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.163:0/1819204274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:52:00.076 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:00 smithi163 bash[75425]: cluster 2023-10-09T09:51:58.206840+0000 mgr.y (mgr.34107) 1382 : cluster 0 pgmap v2251: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 57 MiB/s wr, 907 op/s 2023-10-09T09:52:00.076 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:00 smithi163 bash[75425]: audit 2023-10-09T09:51:59.991175+0000 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.163:0/1819204274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:52:00.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:00.113 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:52:00.141 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:00.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:00.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:00.174 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:52:00.219 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:52:00.244 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:00.264 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:52:00.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:00.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:00.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:00.296 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:52:00.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:00 smithi181 bash[59345]: cluster 2023-10-09T09:51:58.206840+0000 mgr.y (mgr.34107) 1382 : cluster 0 pgmap v2251: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 57 MiB/s wr, 907 op/s 2023-10-09T09:52:00.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:00 smithi181 bash[59345]: audit 2023-10-09T09:51:59.991175+0000 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.163:0/1819204274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:52:00.319 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:52:00.349 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:52:00.351 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:52:00.351 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:52:00.351 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:52:00.351 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:52:00.351 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:52:00.351 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:52:00.351 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:52:00.351 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:52:00.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:00.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 92334 92318 80.1256 45.875 0.0263336 0.0124742 2023-10-09T09:52:00.415 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:00.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:00.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:00.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:00.543 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:00.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:00.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:00.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:00.673 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:00.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:00.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:00.745 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:52:00.754 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:52:00.776 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:00.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:00.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:00.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:00.904 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:00.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:00.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:00.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:01.033 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:01.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:01.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:01.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:01.104 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:52:01.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:01.173 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:01 smithi181 bash[59345]: audit 2023-10-09T09:52:00.758707+0000 mon.a (mon.0) 2491 : audit 1 from='client.? 172.21.15.163:0/2729685824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:52:01.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:01.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:01.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:01.299 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:01.327 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:01.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:01.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:01.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 93159 93143 79.7343 51.5625 0.00864703 0.0125357 2023-10-09T09:52:01.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:01.458 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:01.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:01.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:01.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:01 smithi163 bash[78107]: audit 2023-10-09T09:52:00.758707+0000 mon.a (mon.0) 2491 : audit 1 from='client.? 172.21.15.163:0/2729685824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:52:01.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:01 smithi163 bash[75425]: audit 2023-10-09T09:52:00.758707+0000 mon.a (mon.0) 2491 : audit 1 from='client.? 172.21.15.163:0/2729685824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:52:01.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:01.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:01.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:01.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:01.699 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:01.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:01.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:01.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:01.839 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:01.865 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:01.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:01.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:01.970 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:02.003 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:02.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:02.108 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:02.131 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:02.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:02.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:02.267 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:02.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:02.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:02.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:02 smithi181 bash[59345]: cluster 2023-10-09T09:52:00.207359+0000 mgr.y (mgr.34107) 1383 : cluster 0 pgmap v2252: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 48 MiB/s wr, 765 op/s 2023-10-09T09:52:02.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:02 smithi181 bash[59345]: audit 2023-10-09T09:52:00.727021+0000 mgr.y (mgr.34107) 1384 : audit 0 from='client.45803 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:52:02.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:02 smithi181 bash[59345]: cluster 2023-10-09T09:52:01.074659+0000 mon.a (mon.0) 2492 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:52:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:02 smithi181 bash[59345]: cluster 2023-10-09T09:52:01.074691+0000 mon.a (mon.0) 2493 : cluster 1 Cluster is now healthy 2023-10-09T09:52:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:02 smithi181 bash[59345]: cluster 2023-10-09T09:52:01.090093+0000 osd.5 (osd.5) 867 : cluster 3 failed to encode map e1277 with expected crc 2023-10-09T09:52:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:02 smithi181 bash[59345]: audit 2023-10-09T09:52:01.090397+0000 mon.a (mon.0) 2494 : audit 1 from='client.? 172.21.15.163:0/2729685824' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:52:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:02 smithi181 bash[59345]: cluster 2023-10-09T09:52:01.090453+0000 mon.a (mon.0) 2495 : cluster 0 osdmap e1277: 8 total, 8 up, 6 in 2023-10-09T09:52:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:02 smithi181 bash[59345]: cluster 2023-10-09T09:52:01.090465+0000 osd.4 (osd.4) 668 : cluster 3 failed to encode map e1277 with expected crc 2023-10-09T09:52:02.373 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:02.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:02.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:02.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 94010 93994 79.3754 53.1875 0.00656949 0.0125869 2023-10-09T09:52:02.445 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[75425]: cluster 2023-10-09T09:52:00.207359+0000 mgr.y (mgr.34107) 1383 : cluster 0 pgmap v2252: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 48 MiB/s wr, 765 op/s 2023-10-09T09:52:02.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[75425]: audit 2023-10-09T09:52:00.727021+0000 mgr.y (mgr.34107) 1384 : audit 0 from='client.45803 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:52:02.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[75425]: cluster 2023-10-09T09:52:01.074659+0000 mon.a (mon.0) 2492 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:52:02.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[75425]: cluster 2023-10-09T09:52:01.074691+0000 mon.a (mon.0) 2493 : cluster 1 Cluster is now healthy 2023-10-09T09:52:02.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[75425]: cluster 2023-10-09T09:52:01.090093+0000 osd.5 (osd.5) 867 : cluster 3 failed to encode map e1277 with expected crc 2023-10-09T09:52:02.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[75425]: audit 2023-10-09T09:52:01.090397+0000 mon.a (mon.0) 2494 : audit 1 from='client.? 172.21.15.163:0/2729685824' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:52:02.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[75425]: cluster 2023-10-09T09:52:01.090453+0000 mon.a (mon.0) 2495 : cluster 0 osdmap e1277: 8 total, 8 up, 6 in 2023-10-09T09:52:02.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[75425]: cluster 2023-10-09T09:52:01.090465+0000 osd.4 (osd.4) 668 : cluster 3 failed to encode map e1277 with expected crc 2023-10-09T09:52:02.446 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[78107]: cluster 2023-10-09T09:52:00.207359+0000 mgr.y (mgr.34107) 1383 : cluster 0 pgmap v2252: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 48 MiB/s wr, 765 op/s 2023-10-09T09:52:02.446 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[78107]: audit 2023-10-09T09:52:00.727021+0000 mgr.y (mgr.34107) 1384 : audit 0 from='client.45803 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:52:02.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[78107]: cluster 2023-10-09T09:52:01.074659+0000 mon.a (mon.0) 2492 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:52:02.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[78107]: cluster 2023-10-09T09:52:01.074691+0000 mon.a (mon.0) 2493 : cluster 1 Cluster is now healthy 2023-10-09T09:52:02.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[78107]: cluster 2023-10-09T09:52:01.090093+0000 osd.5 (osd.5) 867 : cluster 3 failed to encode map e1277 with expected crc 2023-10-09T09:52:02.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[78107]: audit 2023-10-09T09:52:01.090397+0000 mon.a (mon.0) 2494 : audit 1 from='client.? 172.21.15.163:0/2729685824' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:52:02.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[78107]: cluster 2023-10-09T09:52:01.090453+0000 mon.a (mon.0) 2495 : cluster 0 osdmap e1277: 8 total, 8 up, 6 in 2023-10-09T09:52:02.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:02 smithi163 bash[78107]: cluster 2023-10-09T09:52:01.090465+0000 osd.4 (osd.4) 668 : cluster 3 failed to encode map e1277 with expected crc 2023-10-09T09:52:02.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:02.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:02.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:02.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:02.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:02.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:02.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:02.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:02.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:02.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:02.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:02.924 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:02.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:02.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:03.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:03.050 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:03.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:03.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:03 smithi163 bash[78107]: cluster 2023-10-09T09:52:02.208089+0000 mgr.y (mgr.34107) 1385 : cluster 0 pgmap v2254: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 872 op/s 2023-10-09T09:52:03.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:03 smithi163 bash[75425]: cluster 2023-10-09T09:52:02.208089+0000 mgr.y (mgr.34107) 1385 : cluster 0 pgmap v2254: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 872 op/s 2023-10-09T09:52:03.154 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:03.182 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:03.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:03.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:03.285 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:03.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:03.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:03.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:03.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:03 smithi181 bash[59345]: cluster 2023-10-09T09:52:02.208089+0000 mgr.y (mgr.34107) 1385 : cluster 0 pgmap v2254: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 872 op/s 2023-10-09T09:52:03.414 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 94888 94872 79.0487 54.875 0.0125367 0.012645 2023-10-09T09:52:03.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:03.422 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:52:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:52:03] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.33.4" 2023-10-09T09:52:03.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:03.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:03.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:03.558 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:03.581 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:03.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:03.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:03.686 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:03.716 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:03.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:03.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:03.852 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:03.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:03.956 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:03.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:03.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:03.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:04.096 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:04.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:04.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:04.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:04.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:04.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:04.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:04.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:04.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T09:52:04.363 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2252,"stamp":"2023-10-09T09:52:00.207030+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5836117920,"num_objects":89067,"num_object_clones":0,"num_object_copies":267201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89067,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":89303,"num_write_kb":5701197,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":592828,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86737,"ondisk_log_size":86737,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":21610732,"kb_used_data":17108668,"kb_used_omap":607,"kb_used_meta":4501408,"kb_avail":540860180,"statfs":{"total":575970213888,"available":553840824320,"internally_reserved":0,"allocated":17519276032,"data_stored":17513846678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":621657,"internal_metadata":4609442727},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":602472448,"num_objects":9193,"num_object_clones":0,"num_object_copies":27579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9193,"num_write_kb":588352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003576"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":54,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019109+0000","last_change":"2023-10-09T09:48:07.991289+0000","last_active":"2023-10-09T09:51:10.019109+0000","last_peered":"2023-10-09T09:51:10.019109+0000","last_clean":"2023-10-09T09:51:10.019109+0000","last_became_active":"2023-10-09T09:48:07.990686+0000","last_became_peered":"2023-10-09T09:48:07.990686+0000","last_unstale":"2023-10-09T09:51:10.019109+0000","last_undegraded":"2023-10-09T09:51:10.019109+0000","last_fullsized":"2023-10-09T09:51:10.019109+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:24.421374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":157,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019704+0000","last_change":"2023-10-09T09:48:07.999947+0000","last_active":"2023-10-09T09:51:10.019704+0000","last_peered":"2023-10-09T09:51:10.019704+0000","last_clean":"2023-10-09T09:51:10.019704+0000","last_became_active":"2023-10-09T09:48:07.999704+0000","last_became_peered":"2023-10-09T09:48:07.999704+0000","last_unstale":"2023-10-09T09:51:10.019704+0000","last_undegraded":"2023-10-09T09:51:10.019704+0000","last_fullsized":"2023-10-09T09:51:10.019704+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:14.832042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1226,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.018658+0000","last_change":"2023-10-09T09:48:07.998214+0000","last_active":"2023-10-09T09:51:10.018658+0000","last_peered":"2023-10-09T09:51:10.018658+0000","last_clean":"2023-10-09T09:51:10.018658+0000","last_became_active":"2023-10-09T09:48:07.997544+0000","last_became_peered":"2023-10-09T09:48:07.997544+0000","last_unstale":"2023-10-09T09:51:10.018658+0000","last_undegraded":"2023-10-09T09:51:10.018658+0000","last_fullsized":"2023-10-09T09:51:10.018658+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.304000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1236,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:18.750175+0000","last_change":"2023-10-09T09:48:07.980358+0000","last_active":"2023-10-09T09:51:18.750175+0000","last_peered":"2023-10-09T09:51:18.750175+0000","last_clean":"2023-10-09T09:51:18.750175+0000","last_became_active":"2023-10-09T09:48:07.979229+0000","last_became_peered":"2023-10-09T09:48:07.979229+0000","last_unstale":"2023-10-09T09:51:18.750175+0000","last_undegraded":"2023-10-09T09:51:18.750175+0000","last_fullsized":"2023-10-09T09:51:18.750175+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:31.456347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":769,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.252944+0000","last_change":"2023-10-09T09:48:07.969728+0000","last_active":"2023-10-09T09:51:10.252944+0000","last_peered":"2023-10-09T09:51:10.252944+0000","last_clean":"2023-10-09T09:51:10.252944+0000","last_became_active":"2023-10-09T09:48:07.968773+0000","last_became_peered":"2023-10-09T09:48:07.968773+0000","last_unstale":"2023-10-09T09:51:10.252944+0000","last_undegraded":"2023-10-09T09:51:10.252944+0000","last_fullsized":"2023-10-09T09:51:10.252944+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:39.023317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1240,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.016297+0000","last_change":"2023-10-09T09:48:07.988928+0000","last_active":"2023-10-09T09:51:10.016297+0000","last_peered":"2023-10-09T09:51:10.016297+0000","last_clean":"2023-10-09T09:51:10.016297+0000","last_became_active":"2023-10-09T09:48:07.988051+0000","last_became_peered":"2023-10-09T09:48:07.988051+0000","last_unstale":"2023-10-09T09:51:10.016297+0000","last_undegraded":"2023-10-09T09:51:10.016297+0000","last_fullsized":"2023-10-09T09:51:10.016297+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1199,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.011850+0000","last_change":"2023-10-09T09:43:48.902066+0000","last_active":"2023-10-09T09:51:10.011850+0000","last_peered":"2023-10-09T09:51:10.011850+0000","last_clean":"2023-10-09T09:51:10.011850+0000","last_became_active":"2023-10-09T09:43:48.901639+0000","last_became_peered":"2023-10-09T09:43:48.901639+0000","last_unstale":"2023-10-09T09:51:10.011850+0000","last_undegraded":"2023-10-09T09:51:10.011850+0000","last_fullsized":"2023-10-09T09:51:10.011850+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":668,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.018909+0000","last_change":"2023-10-09T09:48:07.997787+0000","last_active":"2023-10-09T09:51:10.018909+0000","last_peered":"2023-10-09T09:51:10.018909+0000","last_clean":"2023-10-09T09:51:10.018909+0000","last_became_active":"2023-10-09T09:48:07.997356+0000","last_became_peered":"2023-10-09T09:48:07.997356+0000","last_unstale":"2023-10-09T09:51:10.018909+0000","last_undegraded":"2023-10-09T09:51:10.018909+0000","last_fullsized":"2023-10-09T09:51:10.018909+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:03.024096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":142,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.011619+0000","last_change":"2023-10-09T09:48:07.975547+0000","last_active":"2023-10-09T09:51:10.011619+0000","last_peered":"2023-10-09T09:51:10.011619+0000","last_clean":"2023-10-09T09:51:10.011619+0000","last_became_active":"2023-10-09T09:48:07.975150+0000","last_became_peered":"2023-10-09T09:48:07.975150+0000","last_unstale":"2023-10-09T09:51:10.011619+0000","last_undegraded":"2023-10-09T09:51:10.011619+0000","last_fullsized":"2023-10-09T09:51:10.011619+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":691,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.016127+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:51:10.016127+0000","last_peered":"2023-10-09T09:51:10.016127+0000","last_clean":"2023-10-09T09:51:10.016127+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:51:10.016127+0000","last_undegraded":"2023-10-09T09:51:10.016127+0000","last_fullsized":"2023-10-09T09:51:10.016127+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":84,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.015480+0000","last_change":"2023-10-09T09:43:48.906785+0000","last_active":"2023-10-09T09:51:10.015480+0000","last_peered":"2023-10-09T09:51:10.015480+0000","last_clean":"2023-10-09T09:51:10.015480+0000","last_became_active":"2023-10-09T09:43:48.906276+0000","last_became_peered":"2023-10-09T09:43:48.906276+0000","last_unstale":"2023-10-09T09:51:10.015480+0000","last_undegraded":"2023-10-09T09:51:10.015480+0000","last_fullsized":"2023-10-09T09:51:10.015480+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":108,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.010665+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:51:10.010665+0000","last_peered":"2023-10-09T09:51:10.010665+0000","last_clean":"2023-10-09T09:51:10.010665+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:51:10.010665+0000","last_undegraded":"2023-10-09T09:51:10.010665+0000","last_fullsized":"2023-10-09T09:51:10.010665+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":432,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.014980+0000","last_change":"2023-10-09T09:43:23.995526+0000","last_active":"2023-10-09T09:51:10.014980+0000","last_peered":"2023-10-09T09:51:10.014980+0000","last_clean":"2023-10-09T09:51:10.014980+0000","last_became_active":"2023-10-09T09:43:23.994816+0000","last_became_peered":"2023-10-09T09:43:23.994816+0000","last_unstale":"2023-10-09T09:51:10.014980+0000","last_undegraded":"2023-10-09T09:51:10.014980+0000","last_fullsized":"2023-10-09T09:51:10.014980+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":649,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.016205+0000","last_change":"2023-10-09T09:48:07.987637+0000","last_active":"2023-10-09T09:51:10.016205+0000","last_peered":"2023-10-09T09:51:10.016205+0000","last_clean":"2023-10-09T09:51:10.016205+0000","last_became_active":"2023-10-09T09:48:07.987085+0000","last_became_peered":"2023-10-09T09:48:07.987085+0000","last_unstale":"2023-10-09T09:51:10.016205+0000","last_undegraded":"2023-10-09T09:51:10.016205+0000","last_fullsized":"2023-10-09T09:51:10.016205+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1171,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.024125+0000","last_change":"2023-10-09T09:43:23.989356+0000","last_active":"2023-10-09T09:51:10.024125+0000","last_peered":"2023-10-09T09:51:10.024125+0000","last_clean":"2023-10-09T09:51:10.024125+0000","last_became_active":"2023-10-09T09:43:23.988986+0000","last_became_peered":"2023-10-09T09:43:23.988986+0000","last_unstale":"2023-10-09T09:51:10.024125+0000","last_undegraded":"2023-10-09T09:51:10.024125+0000","last_fullsized":"2023-10-09T09:51:10.024125+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.570448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":753,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.020245+0000","last_change":"2023-10-09T09:48:07.984006+0000","last_active":"2023-10-09T09:51:10.020245+0000","last_peered":"2023-10-09T09:51:10.020245+0000","last_clean":"2023-10-09T09:51:10.020245+0000","last_became_active":"2023-10-09T09:48:07.978179+0000","last_became_peered":"2023-10-09T09:48:07.978179+0000","last_unstale":"2023-10-09T09:51:10.020245+0000","last_undegraded":"2023-10-09T09:51:10.020245+0000","last_fullsized":"2023-10-09T09:51:10.020245+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:34.164004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1174,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019097+0000","last_change":"2023-10-09T09:48:07.975335+0000","last_active":"2023-10-09T09:51:10.019097+0000","last_peered":"2023-10-09T09:51:10.019097+0000","last_clean":"2023-10-09T09:51:10.019097+0000","last_became_active":"2023-10-09T09:48:07.974437+0000","last_became_peered":"2023-10-09T09:48:07.974437+0000","last_unstale":"2023-10-09T09:51:10.019097+0000","last_undegraded":"2023-10-09T09:51:10.019097+0000","last_fullsized":"2023-10-09T09:51:10.019097+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:32.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":112,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.015659+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:51:10.015659+0000","last_peered":"2023-10-09T09:51:10.015659+0000","last_clean":"2023-10-09T09:51:10.015659+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:51:10.015659+0000","last_undegraded":"2023-10-09T09:51:10.015659+0000","last_fullsized":"2023-10-09T09:51:10.015659+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":76,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.015904+0000","last_change":"2023-10-09T09:43:23.994784+0000","last_active":"2023-10-09T09:51:10.015904+0000","last_peered":"2023-10-09T09:51:10.015904+0000","last_clean":"2023-10-09T09:51:10.015904+0000","last_became_active":"2023-10-09T09:43:23.994196+0000","last_became_peered":"2023-10-09T09:43:23.994196+0000","last_unstale":"2023-10-09T09:51:10.015904+0000","last_undegraded":"2023-10-09T09:51:10.015904+0000","last_fullsized":"2023-10-09T09:51:10.015904+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1192,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.024768+0000","last_change":"2023-10-09T09:43:48.901303+0000","last_active":"2023-10-09T09:51:10.024768+0000","last_peered":"2023-10-09T09:51:10.024768+0000","last_clean":"2023-10-09T09:51:10.024768+0000","last_became_active":"2023-10-09T09:43:48.900779+0000","last_became_peered":"2023-10-09T09:43:48.900779+0000","last_unstale":"2023-10-09T09:51:10.024768+0000","last_undegraded":"2023-10-09T09:51:10.024768+0000","last_fullsized":"2023-10-09T09:51:10.024768+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:51.853531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1270,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.014702+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:51:10.014702+0000","last_peered":"2023-10-09T09:51:10.014702+0000","last_clean":"2023-10-09T09:51:10.014702+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:51:10.014702+0000","last_undegraded":"2023-10-09T09:51:10.014702+0000","last_fullsized":"2023-10-09T09:51:10.014702+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1236,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.020134+0000","last_change":"2023-10-09T09:48:08.000694+0000","last_active":"2023-10-09T09:51:10.020134+0000","last_peered":"2023-10-09T09:51:10.020134+0000","last_clean":"2023-10-09T09:51:10.020134+0000","last_became_active":"2023-10-09T09:48:07.999476+0000","last_became_peered":"2023-10-09T09:48:07.999476+0000","last_unstale":"2023-10-09T09:51:10.020134+0000","last_undegraded":"2023-10-09T09:51:10.020134+0000","last_fullsized":"2023-10-09T09:51:10.020134+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:37.211076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.7","version":"1276'5452","reported_seq":5468,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.182444+0000","last_change":"2023-10-09T09:50:27.379119+0000","last_active":"2023-10-09T09:51:56.182444+0000","last_peered":"2023-10-09T09:51:56.182444+0000","last_clean":"2023-10-09T09:51:56.182444+0000","last_became_active":"2023-10-09T09:50:27.378562+0000","last_became_peered":"2023-10-09T09:50:27.378562+0000","last_unstale":"2023-10-09T09:51:56.182444+0000","last_undegraded":"2023-10-09T09:51:56.182444+0000","last_fullsized":"2023-10-09T09:51:56.182444+0000","mapping_epoch":1269,"log_start":"1274'300","ondisk_log_start":"1274'300","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5152,"log_dups_size":0,"ondisk_log_size":5152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:01.665220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357302272,"num_objects":5452,"num_object_clones":0,"num_object_copies":16356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5452,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":102,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.250802+0000","last_change":"2023-10-09T09:48:07.976558+0000","last_active":"2023-10-09T09:51:10.250802+0000","last_peered":"2023-10-09T09:51:10.250802+0000","last_clean":"2023-10-09T09:51:10.250802+0000","last_became_active":"2023-10-09T09:48:07.975961+0000","last_became_peered":"2023-10-09T09:48:07.975961+0000","last_unstale":"2023-10-09T09:51:10.250802+0000","last_undegraded":"2023-10-09T09:51:10.250802+0000","last_fullsized":"2023-10-09T09:51:10.250802+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:40.925248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":118,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.010122+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:51:10.010122+0000","last_peered":"2023-10-09T09:51:10.010122+0000","last_clean":"2023-10-09T09:51:10.010122+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:51:10.010122+0000","last_undegraded":"2023-10-09T09:51:10.010122+0000","last_fullsized":"2023-10-09T09:51:10.010122+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"1276'5756","reported_seq":5772,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:58.753239+0000","last_change":"2023-10-09T09:50:26.983846+0000","last_active":"2023-10-09T09:51:58.753239+0000","last_peered":"2023-10-09T09:51:58.753239+0000","last_clean":"2023-10-09T09:51:58.753239+0000","last_became_active":"2023-10-09T09:50:26.983409+0000","last_became_peered":"2023-10-09T09:50:26.983409+0000","last_unstale":"2023-10-09T09:51:58.753239+0000","last_undegraded":"2023-10-09T09:51:58.753239+0000","last_fullsized":"2023-10-09T09:51:58.753239+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:48.191471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377225216,"num_objects":5756,"num_object_clones":0,"num_object_copies":17268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5756,"num_write_kb":368384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":65,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019810+0000","last_change":"2023-10-09T09:48:07.990711+0000","last_active":"2023-10-09T09:51:10.019810+0000","last_peered":"2023-10-09T09:51:10.019810+0000","last_clean":"2023-10-09T09:51:10.019810+0000","last_became_active":"2023-10-09T09:48:07.990267+0000","last_became_peered":"2023-10-09T09:48:07.990267+0000","last_unstale":"2023-10-09T09:51:10.019810+0000","last_undegraded":"2023-10-09T09:51:10.019810+0000","last_fullsized":"2023-10-09T09:51:10.019810+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:51.244840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1209,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.018690+0000","last_change":"2023-10-09T09:48:07.991440+0000","last_active":"2023-10-09T09:51:10.018690+0000","last_peered":"2023-10-09T09:51:10.018690+0000","last_clean":"2023-10-09T09:51:10.018690+0000","last_became_active":"2023-10-09T09:48:07.991072+0000","last_became_peered":"2023-10-09T09:48:07.991072+0000","last_unstale":"2023-10-09T09:51:10.018690+0000","last_undegraded":"2023-10-09T09:51:10.018690+0000","last_fullsized":"2023-10-09T09:51:10.018690+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:28.577159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"1276'5543","reported_seq":5558,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.597964+0000","last_change":"2023-10-09T09:50:26.985990+0000","last_active":"2023-10-09T09:51:56.597964+0000","last_peered":"2023-10-09T09:51:56.597964+0000","last_clean":"2023-10-09T09:51:56.597964+0000","last_became_active":"2023-10-09T09:50:26.985591+0000","last_became_peered":"2023-10-09T09:50:26.985591+0000","last_unstale":"2023-10-09T09:51:56.597964+0000","last_undegraded":"2023-10-09T09:51:56.597964+0000","last_fullsized":"2023-10-09T09:51:56.597964+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5543,"log_dups_size":0,"ondisk_log_size":5543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363266048,"num_objects":5543,"num_object_clones":0,"num_object_copies":16629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5543,"num_write_kb":354752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1226,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.013866+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:51:10.013866+0000","last_peered":"2023-10-09T09:51:10.013866+0000","last_clean":"2023-10-09T09:51:10.013866+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:51:10.013866+0000","last_undegraded":"2023-10-09T09:51:10.013866+0000","last_fullsized":"2023-10-09T09:51:10.013866+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1232,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.014898+0000","last_change":"2023-10-09T09:48:07.987819+0000","last_active":"2023-10-09T09:51:10.014898+0000","last_peered":"2023-10-09T09:51:10.014898+0000","last_clean":"2023-10-09T09:51:10.014898+0000","last_became_active":"2023-10-09T09:48:07.986354+0000","last_became_peered":"2023-10-09T09:48:07.986354+0000","last_unstale":"2023-10-09T09:51:10.014898+0000","last_undegraded":"2023-10-09T09:51:10.014898+0000","last_fullsized":"2023-10-09T09:51:10.014898+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"1276'5591","reported_seq":5606,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.972182+0000","last_change":"2023-10-09T09:50:26.987956+0000","last_active":"2023-10-09T09:51:56.972182+0000","last_peered":"2023-10-09T09:51:56.972182+0000","last_clean":"2023-10-09T09:51:56.972182+0000","last_became_active":"2023-10-09T09:50:26.987609+0000","last_became_peered":"2023-10-09T09:50:26.987609+0000","last_unstale":"2023-10-09T09:51:56.972182+0000","last_undegraded":"2023-10-09T09:51:56.972182+0000","last_fullsized":"2023-10-09T09:51:56.972182+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5591,"log_dups_size":0,"ondisk_log_size":5591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366411776,"num_objects":5591,"num_object_clones":0,"num_object_copies":16773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5591,"num_write_kb":357824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":778,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.015235+0000","last_change":"2023-10-09T09:48:07.987506+0000","last_active":"2023-10-09T09:51:10.015235+0000","last_peered":"2023-10-09T09:51:10.015235+0000","last_clean":"2023-10-09T09:51:10.015235+0000","last_became_active":"2023-10-09T09:48:07.986988+0000","last_became_peered":"2023-10-09T09:48:07.986988+0000","last_unstale":"2023-10-09T09:51:10.015235+0000","last_undegraded":"2023-10-09T09:51:10.015235+0000","last_fullsized":"2023-10-09T09:51:10.015235+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1195,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.250588+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:51:10.250588+0000","last_peered":"2023-10-09T09:51:10.250588+0000","last_clean":"2023-10-09T09:51:10.250588+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:51:10.250588+0000","last_undegraded":"2023-10-09T09:51:10.250588+0000","last_fullsized":"2023-10-09T09:51:10.250588+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"1276'5545","reported_seq":5560,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.960428+0000","last_change":"2023-10-09T09:50:26.988587+0000","last_active":"2023-10-09T09:51:56.960428+0000","last_peered":"2023-10-09T09:51:56.960428+0000","last_clean":"2023-10-09T09:51:56.960428+0000","last_became_active":"2023-10-09T09:50:26.987497+0000","last_became_peered":"2023-10-09T09:50:26.987497+0000","last_unstale":"2023-10-09T09:51:56.960428+0000","last_undegraded":"2023-10-09T09:51:56.960428+0000","last_fullsized":"2023-10-09T09:51:56.960428+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5545,"log_dups_size":0,"ondisk_log_size":5545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363397120,"num_objects":5545,"num_object_clones":0,"num_object_copies":16635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5545,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":153,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019830+0000","last_change":"2023-10-09T09:48:07.977143+0000","last_active":"2023-10-09T09:51:10.019830+0000","last_peered":"2023-10-09T09:51:10.019830+0000","last_clean":"2023-10-09T09:51:10.019830+0000","last_became_active":"2023-10-09T09:48:07.976338+0000","last_became_peered":"2023-10-09T09:48:07.976338+0000","last_unstale":"2023-10-09T09:51:10.019830+0000","last_undegraded":"2023-10-09T09:51:10.019830+0000","last_fullsized":"2023-10-09T09:51:10.019830+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:12.015053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1228,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.015176+0000","last_change":"2023-10-09T09:48:08.001503+0000","last_active":"2023-10-09T09:51:10.015176+0000","last_peered":"2023-10-09T09:51:10.015176+0000","last_clean":"2023-10-09T09:51:10.015176+0000","last_became_active":"2023-10-09T09:48:08.000921+0000","last_became_peered":"2023-10-09T09:48:08.000921+0000","last_unstale":"2023-10-09T09:51:10.015176+0000","last_undegraded":"2023-10-09T09:51:10.015176+0000","last_fullsized":"2023-10-09T09:51:10.015176+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.2","version":"1276'5496","reported_seq":5512,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:55.674171+0000","last_change":"2023-10-09T09:50:26.980085+0000","last_active":"2023-10-09T09:51:55.674171+0000","last_peered":"2023-10-09T09:51:55.674171+0000","last_clean":"2023-10-09T09:51:55.674171+0000","last_became_active":"2023-10-09T09:50:26.979773+0000","last_became_peered":"2023-10-09T09:50:26.979773+0000","last_unstale":"2023-10-09T09:51:55.674171+0000","last_undegraded":"2023-10-09T09:51:55.674171+0000","last_fullsized":"2023-10-09T09:51:55.674171+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5496,"log_dups_size":0,"ondisk_log_size":5496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:37.729645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360185856,"num_objects":5496,"num_object_clones":0,"num_object_copies":16488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5496,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1224,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.014787+0000","last_change":"2023-10-09T09:48:07.988586+0000","last_active":"2023-10-09T09:51:10.014787+0000","last_peered":"2023-10-09T09:51:10.014787+0000","last_clean":"2023-10-09T09:51:10.014787+0000","last_became_active":"2023-10-09T09:48:07.988226+0000","last_became_peered":"2023-10-09T09:48:07.988226+0000","last_unstale":"2023-10-09T09:51:10.014787+0000","last_undegraded":"2023-10-09T09:51:10.014787+0000","last_fullsized":"2023-10-09T09:51:10.014787+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1215,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.024355+0000","last_change":"2023-10-09T09:48:07.987993+0000","last_active":"2023-10-09T09:51:10.024355+0000","last_peered":"2023-10-09T09:51:10.024355+0000","last_clean":"2023-10-09T09:51:10.024355+0000","last_became_active":"2023-10-09T09:48:07.987664+0000","last_became_peered":"2023-10-09T09:48:07.987664+0000","last_unstale":"2023-10-09T09:51:10.024355+0000","last_undegraded":"2023-10-09T09:51:10.024355+0000","last_fullsized":"2023-10-09T09:51:10.024355+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:32.253321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"1276'5580","reported_seq":5595,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.989027+0000","last_change":"2023-10-09T09:50:26.987126+0000","last_active":"2023-10-09T09:51:56.989027+0000","last_peered":"2023-10-09T09:51:56.989027+0000","last_clean":"2023-10-09T09:51:56.989027+0000","last_became_active":"2023-10-09T09:50:26.986696+0000","last_became_peered":"2023-10-09T09:50:26.986696+0000","last_unstale":"2023-10-09T09:51:56.989027+0000","last_undegraded":"2023-10-09T09:51:56.989027+0000","last_fullsized":"2023-10-09T09:51:56.989027+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5580,"log_dups_size":0,"ondisk_log_size":5580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365690880,"num_objects":5580,"num_object_clones":0,"num_object_copies":16740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5580,"num_write_kb":357120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1243,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.024599+0000","last_change":"2023-10-09T09:48:07.984079+0000","last_active":"2023-10-09T09:51:10.024599+0000","last_peered":"2023-10-09T09:51:10.024599+0000","last_clean":"2023-10-09T09:51:10.024599+0000","last_became_active":"2023-10-09T09:48:07.983792+0000","last_became_peered":"2023-10-09T09:48:07.983792+0000","last_unstale":"2023-10-09T09:51:10.024599+0000","last_undegraded":"2023-10-09T09:51:10.024599+0000","last_fullsized":"2023-10-09T09:51:10.024599+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:28:42.289832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":649,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.015636+0000","last_change":"2023-10-09T09:48:07.989047+0000","last_active":"2023-10-09T09:51:10.015636+0000","last_peered":"2023-10-09T09:51:10.015636+0000","last_clean":"2023-10-09T09:51:10.015636+0000","last_became_active":"2023-10-09T09:48:07.988131+0000","last_became_peered":"2023-10-09T09:48:07.988131+0000","last_unstale":"2023-10-09T09:51:10.015636+0000","last_undegraded":"2023-10-09T09:51:10.015636+0000","last_fullsized":"2023-10-09T09:51:10.015636+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"1276'5542","reported_seq":5558,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.189743+0000","last_change":"2023-10-09T09:50:26.985963+0000","last_active":"2023-10-09T09:51:56.189743+0000","last_peered":"2023-10-09T09:51:56.189743+0000","last_clean":"2023-10-09T09:51:56.189743+0000","last_became_active":"2023-10-09T09:50:26.985572+0000","last_became_peered":"2023-10-09T09:50:26.985572+0000","last_unstale":"2023-10-09T09:51:56.189743+0000","last_undegraded":"2023-10-09T09:51:56.189743+0000","last_fullsized":"2023-10-09T09:51:56.189743+0000","mapping_epoch":1269,"log_start":"1274'400","ondisk_log_start":"1274'400","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5142,"log_dups_size":0,"ondisk_log_size":5142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:48.639926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363200512,"num_objects":5542,"num_object_clones":0,"num_object_copies":16626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5542,"num_write_kb":354688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1185,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019327+0000","last_change":"2023-10-09T09:48:07.976884+0000","last_active":"2023-10-09T09:51:10.019327+0000","last_peered":"2023-10-09T09:51:10.019327+0000","last_clean":"2023-10-09T09:51:10.019327+0000","last_became_active":"2023-10-09T09:48:07.975932+0000","last_became_peered":"2023-10-09T09:48:07.975932+0000","last_unstale":"2023-10-09T09:51:10.019327+0000","last_undegraded":"2023-10-09T09:51:10.019327+0000","last_fullsized":"2023-10-09T09:51:10.019327+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:22.785728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":122,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:32.961467+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:51:32.961467+0000","last_peered":"2023-10-09T09:51:32.961467+0000","last_clean":"2023-10-09T09:51:32.961467+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:51:32.961467+0000","last_undegraded":"2023-10-09T09:51:32.961467+0000","last_fullsized":"2023-10-09T09:51:32.961467+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"1276'5548","reported_seq":5564,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:55.662617+0000","last_change":"2023-10-09T09:50:27.376496+0000","last_active":"2023-10-09T09:51:55.662617+0000","last_peered":"2023-10-09T09:51:55.662617+0000","last_clean":"2023-10-09T09:51:55.662617+0000","last_became_active":"2023-10-09T09:50:27.376004+0000","last_became_peered":"2023-10-09T09:50:27.376004+0000","last_unstale":"2023-10-09T09:51:55.662617+0000","last_undegraded":"2023-10-09T09:51:55.662617+0000","last_fullsized":"2023-10-09T09:51:55.662617+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:22.820238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363593728,"num_objects":5548,"num_object_clones":0,"num_object_copies":16644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5548,"num_write_kb":355072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1231,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.020170+0000","last_change":"2023-10-09T09:48:07.984553+0000","last_active":"2023-10-09T09:51:10.020170+0000","last_peered":"2023-10-09T09:51:10.020170+0000","last_clean":"2023-10-09T09:51:10.020170+0000","last_became_active":"2023-10-09T09:48:07.983302+0000","last_became_peered":"2023-10-09T09:48:07.983302+0000","last_unstale":"2023-10-09T09:51:10.020170+0000","last_undegraded":"2023-10-09T09:51:10.020170+0000","last_fullsized":"2023-10-09T09:51:10.020170+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:50.714295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":118,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.011666+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:51:10.011666+0000","last_peered":"2023-10-09T09:51:10.011666+0000","last_clean":"2023-10-09T09:51:10.011666+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:51:10.011666+0000","last_undegraded":"2023-10-09T09:51:10.011666+0000","last_fullsized":"2023-10-09T09:51:10.011666+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1208,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.018802+0000","last_change":"2023-10-09T09:48:07.977683+0000","last_active":"2023-10-09T09:51:10.018802+0000","last_peered":"2023-10-09T09:51:10.018802+0000","last_clean":"2023-10-09T09:51:10.018802+0000","last_became_active":"2023-10-09T09:48:07.977024+0000","last_became_peered":"2023-10-09T09:48:07.977024+0000","last_unstale":"2023-10-09T09:51:10.018802+0000","last_undegraded":"2023-10-09T09:51:10.018802+0000","last_fullsized":"2023-10-09T09:51:10.018802+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:13.438431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"1276'5579","reported_seq":5595,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.138717+0000","last_change":"2023-10-09T09:50:27.379238+0000","last_active":"2023-10-09T09:51:56.138717+0000","last_peered":"2023-10-09T09:51:56.138717+0000","last_clean":"2023-10-09T09:51:56.138717+0000","last_became_active":"2023-10-09T09:50:27.378551+0000","last_became_peered":"2023-10-09T09:50:27.378551+0000","last_unstale":"2023-10-09T09:51:56.138717+0000","last_undegraded":"2023-10-09T09:51:56.138717+0000","last_fullsized":"2023-10-09T09:51:56.138717+0000","mapping_epoch":1269,"log_start":"1274'400","ondisk_log_start":"1274'400","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5179,"log_dups_size":0,"ondisk_log_size":5179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:19.610902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365625344,"num_objects":5579,"num_object_clones":0,"num_object_copies":16737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5579,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":691,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019018+0000","last_change":"2023-10-09T09:48:07.997811+0000","last_active":"2023-10-09T09:51:10.019018+0000","last_peered":"2023-10-09T09:51:10.019018+0000","last_clean":"2023-10-09T09:51:10.019018+0000","last_became_active":"2023-10-09T09:48:07.997389+0000","last_became_peered":"2023-10-09T09:48:07.997389+0000","last_unstale":"2023-10-09T09:51:10.019018+0000","last_undegraded":"2023-10-09T09:51:10.019018+0000","last_fullsized":"2023-10-09T09:51:10.019018+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:21.978398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1166,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.016363+0000","last_change":"2023-10-09T09:43:48.908190+0000","last_active":"2023-10-09T09:51:10.016363+0000","last_peered":"2023-10-09T09:51:10.016363+0000","last_clean":"2023-10-09T09:51:10.016363+0000","last_became_active":"2023-10-09T09:43:48.907799+0000","last_became_peered":"2023-10-09T09:43:48.907799+0000","last_unstale":"2023-10-09T09:51:10.016363+0000","last_undegraded":"2023-10-09T09:51:10.016363+0000","last_fullsized":"2023-10-09T09:51:10.016363+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1261,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019688+0000","last_change":"2023-10-09T09:48:08.000154+0000","last_active":"2023-10-09T09:51:10.019688+0000","last_peered":"2023-10-09T09:51:10.019688+0000","last_clean":"2023-10-09T09:51:10.019688+0000","last_became_active":"2023-10-09T09:48:07.999714+0000","last_became_peered":"2023-10-09T09:48:07.999714+0000","last_unstale":"2023-10-09T09:51:10.019688+0000","last_undegraded":"2023-10-09T09:51:10.019688+0000","last_fullsized":"2023-10-09T09:51:10.019688+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:12.921693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"1276'5548","reported_seq":5563,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.976764+0000","last_change":"2023-10-09T09:50:26.987164+0000","last_active":"2023-10-09T09:51:56.976764+0000","last_peered":"2023-10-09T09:51:56.976764+0000","last_clean":"2023-10-09T09:51:56.976764+0000","last_became_active":"2023-10-09T09:50:26.986703+0000","last_became_peered":"2023-10-09T09:50:26.986703+0000","last_unstale":"2023-10-09T09:51:56.976764+0000","last_undegraded":"2023-10-09T09:51:56.976764+0000","last_fullsized":"2023-10-09T09:51:56.976764+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363593728,"num_objects":5548,"num_object_clones":0,"num_object_copies":16644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5548,"num_write_kb":355072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":167,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:20.662090+0000","last_change":"2023-10-09T09:48:07.983969+0000","last_active":"2023-10-09T09:51:20.662090+0000","last_peered":"2023-10-09T09:51:20.662090+0000","last_clean":"2023-10-09T09:51:20.662090+0000","last_became_active":"2023-10-09T09:48:07.983490+0000","last_became_peered":"2023-10-09T09:48:07.983490+0000","last_unstale":"2023-10-09T09:51:20.662090+0000","last_undegraded":"2023-10-09T09:51:20.662090+0000","last_fullsized":"2023-10-09T09:51:20.662090+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:21.122987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1239,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019002+0000","last_change":"2023-10-09T09:48:07.998234+0000","last_active":"2023-10-09T09:51:10.019002+0000","last_peered":"2023-10-09T09:51:10.019002+0000","last_clean":"2023-10-09T09:51:10.019002+0000","last_became_active":"2023-10-09T09:48:07.997909+0000","last_became_peered":"2023-10-09T09:48:07.997909+0000","last_unstale":"2023-10-09T09:51:10.019002+0000","last_undegraded":"2023-10-09T09:51:10.019002+0000","last_fullsized":"2023-10-09T09:51:10.019002+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:21.750617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1228,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019979+0000","last_change":"2023-10-09T09:48:07.973284+0000","last_active":"2023-10-09T09:51:10.019979+0000","last_peered":"2023-10-09T09:51:10.019979+0000","last_clean":"2023-10-09T09:51:10.019979+0000","last_became_active":"2023-10-09T09:48:07.971235+0000","last_became_peered":"2023-10-09T09:48:07.971235+0000","last_unstale":"2023-10-09T09:51:10.019979+0000","last_undegraded":"2023-10-09T09:51:10.019979+0000","last_fullsized":"2023-10-09T09:51:10.019979+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:40.951420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"1276'5521","reported_seq":5536,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.977273+0000","last_change":"2023-10-09T09:50:26.987803+0000","last_active":"2023-10-09T09:51:56.977273+0000","last_peered":"2023-10-09T09:51:56.977273+0000","last_clean":"2023-10-09T09:51:56.977273+0000","last_became_active":"2023-10-09T09:50:26.987440+0000","last_became_peered":"2023-10-09T09:50:26.987440+0000","last_unstale":"2023-10-09T09:51:56.977273+0000","last_undegraded":"2023-10-09T09:51:56.977273+0000","last_fullsized":"2023-10-09T09:51:56.977273+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361824256,"num_objects":5521,"num_object_clones":0,"num_object_copies":16563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5521,"num_write_kb":353344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1285,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.647073+0000","last_change":"2023-10-09T09:41:11.795802+0000","last_active":"2023-10-09T09:51:10.647073+0000","last_peered":"2023-10-09T09:51:10.647073+0000","last_clean":"2023-10-09T09:51:10.647073+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:51:10.647073+0000","last_undegraded":"2023-10-09T09:51:10.647073+0000","last_fullsized":"2023-10-09T09:51:10.647073+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":618,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.253041+0000","last_change":"2023-10-09T09:48:07.978435+0000","last_active":"2023-10-09T09:51:10.253041+0000","last_peered":"2023-10-09T09:51:10.253041+0000","last_clean":"2023-10-09T09:51:10.253041+0000","last_became_active":"2023-10-09T09:48:07.977417+0000","last_became_peered":"2023-10-09T09:48:07.977417+0000","last_unstale":"2023-10-09T09:51:10.253041+0000","last_undegraded":"2023-10-09T09:51:10.253041+0000","last_fullsized":"2023-10-09T09:51:10.253041+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:19.288202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":118,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.010564+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:51:10.010564+0000","last_peered":"2023-10-09T09:51:10.010564+0000","last_clean":"2023-10-09T09:51:10.010564+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:51:10.010564+0000","last_undegraded":"2023-10-09T09:51:10.010564+0000","last_fullsized":"2023-10-09T09:51:10.010564+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1202,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.015747+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:51:10.015747+0000","last_peered":"2023-10-09T09:51:10.015747+0000","last_clean":"2023-10-09T09:51:10.015747+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:51:10.015747+0000","last_undegraded":"2023-10-09T09:51:10.015747+0000","last_fullsized":"2023-10-09T09:51:10.015747+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"1276'5544","reported_seq":5558,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.989281+0000","last_change":"2023-10-09T09:50:26.988221+0000","last_active":"2023-10-09T09:51:56.989281+0000","last_peered":"2023-10-09T09:51:56.989281+0000","last_clean":"2023-10-09T09:51:56.989281+0000","last_became_active":"2023-10-09T09:50:26.987465+0000","last_became_peered":"2023-10-09T09:50:26.987465+0000","last_unstale":"2023-10-09T09:51:56.989281+0000","last_undegraded":"2023-10-09T09:51:56.989281+0000","last_fullsized":"2023-10-09T09:51:56.989281+0000","mapping_epoch":1269,"log_start":"0'0","ondisk_log_start":"0'0","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5544,"log_dups_size":0,"ondisk_log_size":5544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363331584,"num_objects":5544,"num_object_clones":0,"num_object_copies":16632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5544,"num_write_kb":354816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1205,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019090+0000","last_change":"2023-10-09T09:48:07.973246+0000","last_active":"2023-10-09T09:51:10.019090+0000","last_peered":"2023-10-09T09:51:10.019090+0000","last_clean":"2023-10-09T09:51:10.019090+0000","last_became_active":"2023-10-09T09:48:07.971221+0000","last_became_peered":"2023-10-09T09:48:07.971221+0000","last_unstale":"2023-10-09T09:51:10.019090+0000","last_undegraded":"2023-10-09T09:51:10.019090+0000","last_fullsized":"2023-10-09T09:51:10.019090+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:16.759514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1213,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:32.955618+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:51:32.955618+0000","last_peered":"2023-10-09T09:51:32.955618+0000","last_clean":"2023-10-09T09:51:32.955618+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:51:32.955618+0000","last_undegraded":"2023-10-09T09:51:32.955618+0000","last_fullsized":"2023-10-09T09:51:32.955618+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1224,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.249393+0000","last_change":"2023-10-09T09:48:07.981410+0000","last_active":"2023-10-09T09:51:10.249393+0000","last_peered":"2023-10-09T09:51:10.249393+0000","last_clean":"2023-10-09T09:51:10.249393+0000","last_became_active":"2023-10-09T09:48:07.979850+0000","last_became_peered":"2023-10-09T09:48:07.979850+0000","last_unstale":"2023-10-09T09:51:10.249393+0000","last_undegraded":"2023-10-09T09:51:10.249393+0000","last_fullsized":"2023-10-09T09:51:10.249393+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:31.201104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"1276'5622","reported_seq":5637,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.140670+0000","last_change":"2023-10-09T09:50:27.379333+0000","last_active":"2023-10-09T09:51:56.140670+0000","last_peered":"2023-10-09T09:51:56.140670+0000","last_clean":"2023-10-09T09:51:56.140670+0000","last_became_active":"2023-10-09T09:50:27.378836+0000","last_became_peered":"2023-10-09T09:50:27.378836+0000","last_unstale":"2023-10-09T09:51:56.140670+0000","last_undegraded":"2023-10-09T09:51:56.140670+0000","last_fullsized":"2023-10-09T09:51:56.140670+0000","mapping_epoch":1269,"log_start":"1274'500","ondisk_log_start":"1274'500","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5122,"log_dups_size":0,"ondisk_log_size":5122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:59.239495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368443392,"num_objects":5622,"num_object_clones":0,"num_object_copies":16866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5622,"num_write_kb":359808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1200,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:18.750138+0000","last_change":"2023-10-09T09:48:07.975786+0000","last_active":"2023-10-09T09:51:18.750138+0000","last_peered":"2023-10-09T09:51:18.750138+0000","last_clean":"2023-10-09T09:51:18.750138+0000","last_became_active":"2023-10-09T09:48:07.975338+0000","last_became_peered":"2023-10-09T09:48:07.975338+0000","last_unstale":"2023-10-09T09:51:18.750138+0000","last_undegraded":"2023-10-09T09:51:18.750138+0000","last_fullsized":"2023-10-09T09:51:18.750138+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:05.175956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1242,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.020037+0000","last_change":"2023-10-09T09:48:07.983773+0000","last_active":"2023-10-09T09:51:10.020037+0000","last_peered":"2023-10-09T09:51:10.020037+0000","last_clean":"2023-10-09T09:51:10.020037+0000","last_became_active":"2023-10-09T09:48:07.976869+0000","last_became_peered":"2023-10-09T09:48:07.976869+0000","last_unstale":"2023-10-09T09:51:10.020037+0000","last_undegraded":"2023-10-09T09:51:10.020037+0000","last_fullsized":"2023-10-09T09:51:10.020037+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:58.173236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1192,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.010219+0000","last_change":"2023-10-09T09:48:07.962302+0000","last_active":"2023-10-09T09:51:10.010219+0000","last_peered":"2023-10-09T09:51:10.010219+0000","last_clean":"2023-10-09T09:51:10.010219+0000","last_became_active":"2023-10-09T09:48:07.961888+0000","last_became_peered":"2023-10-09T09:48:07.961888+0000","last_unstale":"2023-10-09T09:51:10.010219+0000","last_undegraded":"2023-10-09T09:51:10.010219+0000","last_fullsized":"2023-10-09T09:51:10.010219+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"1276'5603","reported_seq":5619,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.131212+0000","last_change":"2023-10-09T09:50:26.989029+0000","last_active":"2023-10-09T09:51:56.131212+0000","last_peered":"2023-10-09T09:51:56.131212+0000","last_clean":"2023-10-09T09:51:56.131212+0000","last_became_active":"2023-10-09T09:50:26.988265+0000","last_became_peered":"2023-10-09T09:50:26.988265+0000","last_unstale":"2023-10-09T09:51:56.131212+0000","last_undegraded":"2023-10-09T09:51:56.131212+0000","last_fullsized":"2023-10-09T09:51:56.131212+0000","mapping_epoch":1269,"log_start":"1274'500","ondisk_log_start":"1274'500","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5103,"log_dups_size":0,"ondisk_log_size":5103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:28.248282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367198208,"num_objects":5603,"num_object_clones":0,"num_object_copies":16809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5603,"num_write_kb":358592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":138,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:32.955027+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:51:32.955027+0000","last_peered":"2023-10-09T09:51:32.955027+0000","last_clean":"2023-10-09T09:51:32.955027+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:51:32.955027+0000","last_undegraded":"2023-10-09T09:51:32.955027+0000","last_fullsized":"2023-10-09T09:51:32.955027+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1194,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.020050+0000","last_change":"2023-10-09T09:48:07.999034+0000","last_active":"2023-10-09T09:51:10.020050+0000","last_peered":"2023-10-09T09:51:10.020050+0000","last_clean":"2023-10-09T09:51:10.020050+0000","last_became_active":"2023-10-09T09:48:07.998606+0000","last_became_peered":"2023-10-09T09:48:07.998606+0000","last_unstale":"2023-10-09T09:51:10.020050+0000","last_undegraded":"2023-10-09T09:51:10.020050+0000","last_fullsized":"2023-10-09T09:51:10.020050+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.469476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1130,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.018746+0000","last_change":"2023-10-09T09:48:07.976350+0000","last_active":"2023-10-09T09:51:10.018746+0000","last_peered":"2023-10-09T09:51:10.018746+0000","last_clean":"2023-10-09T09:51:10.018746+0000","last_became_active":"2023-10-09T09:48:07.973099+0000","last_became_peered":"2023-10-09T09:48:07.973099+0000","last_unstale":"2023-10-09T09:51:10.018746+0000","last_undegraded":"2023-10-09T09:51:10.018746+0000","last_fullsized":"2023-10-09T09:51:10.018746+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:48.812722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1276'5573","reported_seq":5589,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:56.068112+0000","last_change":"2023-10-09T09:50:27.380948+0000","last_active":"2023-10-09T09:51:56.068112+0000","last_peered":"2023-10-09T09:51:56.068112+0000","last_clean":"2023-10-09T09:51:56.068112+0000","last_became_active":"2023-10-09T09:50:27.380423+0000","last_became_peered":"2023-10-09T09:50:27.380423+0000","last_unstale":"2023-10-09T09:51:56.068112+0000","last_undegraded":"2023-10-09T09:51:56.068112+0000","last_fullsized":"2023-10-09T09:51:56.068112+0000","mapping_epoch":1269,"log_start":"1274'400","ondisk_log_start":"1274'400","created":1269,"last_epoch_clean":1270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:50:25.945192+0000","last_clean_scrub_stamp":"2023-10-09T09:50:25.945192+0000","objects_scrubbed":0,"log_size":5173,"log_dups_size":0,"ondisk_log_size":5173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:09.358786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365232128,"num_objects":5573,"num_object_clones":0,"num_object_copies":16719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5573,"num_write_kb":356672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1263,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.016219+0000","last_change":"2023-10-09T09:48:07.990005+0000","last_active":"2023-10-09T09:51:10.016219+0000","last_peered":"2023-10-09T09:51:10.016219+0000","last_clean":"2023-10-09T09:51:10.016219+0000","last_became_active":"2023-10-09T09:48:07.987124+0000","last_became_peered":"2023-10-09T09:48:07.987124+0000","last_unstale":"2023-10-09T09:51:10.016219+0000","last_undegraded":"2023-10-09T09:51:10.016219+0000","last_fullsized":"2023-10-09T09:51:10.016219+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1205,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.250090+0000","last_change":"2023-10-09T09:48:07.980600+0000","last_active":"2023-10-09T09:51:10.250090+0000","last_peered":"2023-10-09T09:51:10.250090+0000","last_clean":"2023-10-09T09:51:10.250090+0000","last_became_active":"2023-10-09T09:48:07.979544+0000","last_became_peered":"2023-10-09T09:48:07.979544+0000","last_unstale":"2023-10-09T09:51:10.250090+0000","last_undegraded":"2023-10-09T09:51:10.250090+0000","last_fullsized":"2023-10-09T09:51:10.250090+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:47.320689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1194,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.010468+0000","last_change":"2023-10-09T09:43:23.992213+0000","last_active":"2023-10-09T09:51:10.010468+0000","last_peered":"2023-10-09T09:51:10.010468+0000","last_clean":"2023-10-09T09:51:10.010468+0000","last_became_active":"2023-10-09T09:43:23.991887+0000","last_became_peered":"2023-10-09T09:43:23.991887+0000","last_unstale":"2023-10-09T09:51:10.010468+0000","last_undegraded":"2023-10-09T09:51:10.010468+0000","last_fullsized":"2023-10-09T09:51:10.010468+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1213,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.250805+0000","last_change":"2023-10-09T09:48:07.969929+0000","last_active":"2023-10-09T09:51:10.250805+0000","last_peered":"2023-10-09T09:51:10.250805+0000","last_clean":"2023-10-09T09:51:10.250805+0000","last_became_active":"2023-10-09T09:48:07.969537+0000","last_became_peered":"2023-10-09T09:48:07.969537+0000","last_unstale":"2023-10-09T09:51:10.250805+0000","last_undegraded":"2023-10-09T09:51:10.250805+0000","last_fullsized":"2023-10-09T09:51:10.250805+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:42.344608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":166,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:32.961447+0000","last_change":"2023-10-09T09:48:07.987618+0000","last_active":"2023-10-09T09:51:32.961447+0000","last_peered":"2023-10-09T09:51:32.961447+0000","last_clean":"2023-10-09T09:51:32.961447+0000","last_became_active":"2023-10-09T09:48:07.987341+0000","last_became_peered":"2023-10-09T09:48:07.987341+0000","last_unstale":"2023-10-09T09:51:32.961447+0000","last_undegraded":"2023-10-09T09:51:32.961447+0000","last_fullsized":"2023-10-09T09:51:32.961447+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:19.972891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":65,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.019787+0000","last_change":"2023-10-09T09:48:07.984247+0000","last_active":"2023-10-09T09:51:10.019787+0000","last_peered":"2023-10-09T09:51:10.019787+0000","last_clean":"2023-10-09T09:51:10.019787+0000","last_became_active":"2023-10-09T09:48:07.978613+0000","last_became_peered":"2023-10-09T09:48:07.978613+0000","last_unstale":"2023-10-09T09:51:10.019787+0000","last_undegraded":"2023-10-09T09:51:10.019787+0000","last_fullsized":"2023-10-09T09:51:10.019787+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:34.663108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1209,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.020014+0000","last_change":"2023-10-09T09:48:07.991053+0000","last_active":"2023-10-09T09:51:10.020014+0000","last_peered":"2023-10-09T09:51:10.020014+0000","last_clean":"2023-10-09T09:51:10.020014+0000","last_became_active":"2023-10-09T09:48:07.990656+0000","last_became_peered":"2023-10-09T09:48:07.990656+0000","last_unstale":"2023-10-09T09:51:10.020014+0000","last_undegraded":"2023-10-09T09:51:10.020014+0000","last_fullsized":"2023-10-09T09:51:10.020014+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:06.223305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":84,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.016295+0000","last_change":"2023-10-09T09:43:23.995576+0000","last_active":"2023-10-09T09:51:10.016295+0000","last_peered":"2023-10-09T09:51:10.016295+0000","last_clean":"2023-10-09T09:51:10.016295+0000","last_became_active":"2023-10-09T09:43:23.995076+0000","last_became_peered":"2023-10-09T09:43:23.995076+0000","last_unstale":"2023-10-09T09:51:10.016295+0000","last_undegraded":"2023-10-09T09:51:10.016295+0000","last_fullsized":"2023-10-09T09:51:10.016295+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1245,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:20.662131+0000","last_change":"2023-10-09T09:43:48.904021+0000","last_active":"2023-10-09T09:51:20.662131+0000","last_peered":"2023-10-09T09:51:20.662131+0000","last_clean":"2023-10-09T09:51:20.662131+0000","last_became_active":"2023-10-09T09:43:48.903719+0000","last_became_peered":"2023-10-09T09:43:48.903719+0000","last_unstale":"2023-10-09T09:51:20.662131+0000","last_undegraded":"2023-10-09T09:51:20.662131+0000","last_fullsized":"2023-10-09T09:51:20.662131+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:16.284027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":224,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.015747+0000","last_change":"2023-10-09T09:48:08.001392+0000","last_active":"2023-10-09T09:51:10.015747+0000","last_peered":"2023-10-09T09:51:10.015747+0000","last_clean":"2023-10-09T09:51:10.015747+0000","last_became_active":"2023-10-09T09:48:08.000798+0000","last_became_peered":"2023-10-09T09:48:08.000798+0000","last_unstale":"2023-10-09T09:51:10.015747+0000","last_undegraded":"2023-10-09T09:51:10.015747+0000","last_fullsized":"2023-10-09T09:51:10.015747+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":151,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:20.662057+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:51:20.662057+0000","last_peered":"2023-10-09T09:51:20.662057+0000","last_clean":"2023-10-09T09:51:20.662057+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:51:20.662057+0000","last_undegraded":"2023-10-09T09:51:20.662057+0000","last_fullsized":"2023-10-09T09:51:20.662057+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1239,"reported_epoch":1276,"state":"active+clean","last_fresh":"2023-10-09T09:51:32.954819+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:51:32.954819+0000","last_peered":"2023-10-09T09:51:32.954819+0000","last_clean":"2023-10-09T09:51:32.954819+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:51:32.954819+0000","last_undegraded":"2023-10-09T09:51:32.954819+0000","last_fullsized":"2023-10-09T09:51:32.954819+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":649,"reported_epoch":1275,"state":"active+clean","last_fresh":"2023-10-09T09:51:10.015431+0000","last_change":"2023-10-09T09:48:07.990205+0000","last_active":"2023-10-09T09:51:10.015431+0000","last_peered":"2023-10-09T09:51:10.015431+0000","last_clean":"2023-10-09T09:51:10.015431+0000","last_became_active":"2023-10-09T09:48:07.989619+0000","last_became_peered":"2023-10-09T09:48:07.989619+0000","last_unstale":"2023-10-09T09:51:10.015431+0000","last_undegraded":"2023-10-09T09:51:10.015431+0000","last_fullsized":"2023-10-09T09:51:10.015431+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5835522048,"num_objects":89043,"num_object_clones":0,"num_object_copies":267129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":89043,"num_write_kb":5698752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17557749760,"data_stored":17557749760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86543,"ondisk_log_size":86543,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":2460,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30942,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453398122,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594355,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4103484,"kb_used_data":3198708,"kb_used_omap":0,"kb_used_meta":904768,"kb_avail":89641668,"statfs":{"total":95995035648,"available":91793068032,"internally_reserved":0,"allocated":3275476992,"data_stored":3275183885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926482432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:51:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":5.1070000000000002,"15min":2.7160000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":5.6269999999999998,"5min":149.095,"15min":149.095},"last":1.0189999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":5.1920000000000002,"15min":2.9580000000000002},"min":{"1min":0.35499999999999998,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":4.3079999999999998,"5min":149.17500000000001,"15min":149.17500000000001},"last":0.49099999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:51:12 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.744,"15min":2.0720000000000001},"min":{"1min":0.37,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.0640000000000001,"5min":9.391,"15min":9.391},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.024,"5min":1.6679999999999999,"15min":1.917},"min":{"1min":0.38800000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.3820000000000001,"5min":7.9240000000000004,"15min":7.9240000000000004},"last":1.796}]},{"osd":2,"last update":"Mon Oct 9 09:51:31 2023","interfaces":[{"interface":"back","average":{"1min":11.436,"5min":6.7140000000000004,"15min":3.573},"min":{"1min":0.375,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":206.97200000000001,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":11.722,"5min":6.6630000000000003,"15min":3.7010000000000001},"min":{"1min":0.44900000000000001,"5min":0.309,"15min":0.29499999999999998},"max":{"1min":208.90899999999999,"5min":208.90899999999999,"15min":208.90899999999999},"last":1.1539999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:51:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":1.329,"15min":1.3999999999999999},"min":{"1min":0.34599999999999997,"5min":0.33900000000000002,"15min":0.309},"max":{"1min":5.5910000000000002,"5min":7.9139999999999997,"15min":11.547000000000001},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.181,"5min":1.736,"15min":1.522},"min":{"1min":0.40999999999999998,"5min":0.39300000000000002,"15min":0.29499999999999998},"max":{"1min":3.137,"5min":8.2490000000000006,"15min":11.968999999999999},"last":1.8580000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:51:39 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.649,"15min":1.4339999999999999},"min":{"1min":0.41699999999999998,"5min":0.373,"15min":0.28199999999999997},"max":{"1min":4.335,"5min":12.148999999999999,"15min":12.148999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.94099999999999995,"5min":2.226,"15min":1.6859999999999999},"min":{"1min":0.40799999999999997,"5min":0.36899999999999999,"15min":0.29499999999999998},"max":{"1min":2.8420000000000001,"5min":8.2360000000000007,"15min":12.064},"last":1.948}]},{"osd":5,"last update":"Mon Oct 9 09:51:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.746,"15min":1.5169999999999999},"min":{"1min":0.46700000000000003,"5min":0.36499999999999999,"15min":0.33100000000000002},"max":{"1min":4.3639999999999999,"5min":10.529,"15min":10.529},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.2430000000000001,"5min":2.0750000000000002,"15min":1.7170000000000001},"min":{"1min":0.39100000000000001,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":2.8839999999999999,"5min":8.6110000000000007,"15min":12.000999999999999},"last":1.907}]},{"osd":7,"last update":"Mon Oct 9 09:51:04 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":2.0379999999999998,"15min":1.597},"min":{"1min":0.55300000000000005,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":2.3210000000000002,"5min":8.2769999999999992,"15min":11.922000000000001},"last":1.988},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.3540000000000001,"15min":1.2729999999999999},"min":{"1min":0.53600000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.8210000000000002,"5min":7.9359999999999999,"15min":7.9359999999999999},"last":1.347}]}]},{"osd":2,"up_from":634,"seq":2723009266108,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3319900,"kb_used_data":2483604,"kb_used_omap":122,"kb_used_meta":836165,"kb_avail":90425252,"statfs":{"total":95995035648,"available":92595458048,"internally_reserved":0,"allocated":2543210496,"data_stored":2541670980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":856233547},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:51:53 2023","interfaces":[{"interface":"back","average":{"1min":12.154,"5min":7.7560000000000002,"15min":4.3719999999999999},"min":{"1min":0.27700000000000002,"5min":0.22900000000000001,"15min":0.20200000000000001},"max":{"1min":218.30799999999999,"5min":218.30799999999999,"15min":218.30799999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":12.214,"5min":7.8070000000000004,"15min":4.524},"min":{"1min":0.27200000000000002,"5min":0.20799999999999999,"15min":0.20399999999999999},"max":{"1min":217.67500000000001,"5min":217.67500000000001,"15min":217.67500000000001},"last":0.35899999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:51:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":1.117,"15min":1.075},"min":{"1min":0.41099999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":12.819000000000001,"5min":12.819000000000001,"15min":12.819000000000001},"last":1.2010000000000001},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.179,"15min":1.1699999999999999},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.724,"5min":12.724,"15min":12.724},"last":1.0009999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:51:53 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":0.93200000000000005,"15min":2.0600000000000001},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26700000000000002},"max":{"1min":4.194,"5min":4.194,"15min":230.476},"last":0.96399999999999997},{"interface":"front","average":{"1min":1.3080000000000001,"5min":0.81999999999999995,"15min":1.9910000000000001},"min":{"1min":0.36099999999999999,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":7.3609999999999998,"5min":7.3609999999999998,"15min":228.893},"last":0.41099999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:51:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":2.0499999999999998,"15min":1.6040000000000001},"min":{"1min":0.41699999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.2910000000000004,"5min":71.876000000000005,"15min":71.876000000000005},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.2869999999999999,"15min":1.3999999999999999},"min":{"1min":0.32100000000000001,"5min":0.29899999999999999,"15min":0.26700000000000002},"max":{"1min":4.1349999999999998,"5min":9.4640000000000004,"15min":10.394},"last":0.60799999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:51:37 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":1.4650000000000001,"15min":1.393},"min":{"1min":0.58799999999999997,"5min":0.34699999999999998,"15min":0.29799999999999999},"max":{"1min":4.3319999999999999,"5min":8.5830000000000002,"15min":9.7379999999999995},"last":1.1080000000000001},{"interface":"front","average":{"1min":1.357,"5min":1.177,"15min":1.3939999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.32200000000000001},"max":{"1min":7.2350000000000003,"5min":7.2350000000000003,"15min":11.718999999999999},"last":0.78300000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:51:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":2.3359999999999999,"15min":1.9550000000000001},"min":{"1min":0.439,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.3289999999999997,"5min":71.795000000000002,"15min":71.795000000000002},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.2709999999999999,"5min":2.2610000000000001,"15min":1.7350000000000001},"min":{"1min":0.56100000000000005,"5min":0.38700000000000001,"15min":0.35599999999999998},"max":{"1min":4.0570000000000004,"5min":72.322000000000003,"15min":72.322000000000003},"last":0.52400000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:51:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.179,"15min":1.363},"min":{"1min":0.43099999999999999,"5min":0.374,"15min":0.374},"max":{"1min":2.8639999999999999,"5min":8.4659999999999993,"15min":10.462999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.4410000000000001,"5min":1.2509999999999999,"15min":1.3240000000000001},"min":{"1min":0.624,"5min":0.441,"15min":0.40400000000000003},"max":{"1min":4.2969999999999997,"5min":5.1029999999999998,"15min":9.7759999999999998},"last":1.1379999999999999}]}]},{"osd":1,"up_from":1246,"seq":5351529250863,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4516556,"kb_used_data":4238340,"kb_used_omap":204,"kb_used_meta":278003,"kb_avail":89228596,"statfs":{"total":95995035648,"available":91370082304,"internally_reserved":0,"allocated":4340060160,"data_stored":4338532877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208958,"internal_metadata":284676034},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":1.633,"15min":1.714},"min":{"1min":0.46600000000000003,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.294,"5min":10.294,"15min":10.294},"last":1.6990000000000001},{"interface":"front","average":{"1min":1.2689999999999999,"5min":1.8109999999999999,"15min":2.0350000000000001},"min":{"1min":0.33000000000000002,"5min":0.214,"15min":0.214},"max":{"1min":4.0309999999999997,"5min":8.5579999999999998,"15min":8.5579999999999998},"last":4.0529999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.3919999999999999,"15min":1.498},"min":{"1min":0.49399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.3010000000000002,"5min":5.8570000000000002,"15min":5.8570000000000002},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.365,"15min":1.5449999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.75,"5min":5.75,"15min":5.75},"last":7.7699999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":1.462,"15min":1.486},"min":{"1min":0.38300000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.3290000000000002,"5min":8.7140000000000004,"15min":8.7140000000000004},"last":4.0910000000000002},{"interface":"front","average":{"1min":1.556,"5min":1.286,"15min":1.349},"min":{"1min":0.498,"5min":0.311,"15min":0.311},"max":{"1min":5.2519999999999998,"5min":5.2519999999999998,"15min":5.2519999999999998},"last":0.36799999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":2.2850000000000001,"15min":2.581},"min":{"1min":0.71299999999999997,"5min":0.379,"15min":0.379},"max":{"1min":3.476,"5min":8.8070000000000004,"15min":8.8070000000000004},"last":4.1230000000000002},{"interface":"front","average":{"1min":1.901,"5min":2.1800000000000002,"15min":2.3929999999999998},"min":{"1min":0.55300000000000005,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":10.263,"5min":10.263,"15min":10.263},"last":0.64300000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9830000000000001,"5min":1.744,"15min":1.8140000000000001},"min":{"1min":0.59099999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.214,"5min":10.214,"15min":10.214},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.373,"5min":1.669,"15min":1.8220000000000001},"min":{"1min":0.45600000000000002,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":5.8239999999999998,"5min":5.952,"15min":5.952},"last":3.6440000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.704,"15min":1.8140000000000001},"min":{"1min":0.42999999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":5.7919999999999998,"5min":7.4119999999999999,"15min":7.4119999999999999},"last":3.6030000000000002},{"interface":"front","average":{"1min":1.7210000000000001,"5min":2.6629999999999998,"15min":3.052},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":4.0129999999999999,"5min":9.827,"15min":9.827},"last":4.1710000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.492,"15min":1.5900000000000001},"min":{"1min":0.495,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":5.851,"5min":5.851,"15min":5.851},"last":3.6840000000000002},{"interface":"front","average":{"1min":1.627,"5min":2.1080000000000001,"15min":2.3039999999999998},"min":{"1min":0.63900000000000001,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":3.4369999999999998,"5min":8.7690000000000001,"15min":8.7690000000000001},"last":4.1500000000000004}]}]},{"osd":0,"up_from":1136,"seq":4879082848457,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4329956,"kb_used_data":3652956,"kb_used_omap":280,"kb_used_meta":676711,"kb_avail":89415196,"statfs":{"total":95995035648,"available":91561160704,"internally_reserved":0,"allocated":3740626944,"data_stored":3739128600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":692952474},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:51:18 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5499999999999998,"15min":1.78},"min":{"1min":0.34300000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":69.680000000000007,"5min":69.680000000000007,"15min":69.680000000000007},"last":1.27},{"interface":"front","average":{"1min":4.1509999999999998,"5min":2.7269999999999999,"15min":2.0870000000000002},"min":{"1min":0.48199999999999998,"5min":0.307,"15min":0.307},"max":{"1min":70.668999999999997,"5min":70.668999999999997,"15min":70.668999999999997},"last":3.1859999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:51:42 2023","interfaces":[{"interface":"back","average":{"1min":13.673999999999999,"5min":3.9039999999999999,"15min":2.532},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":254.89599999999999,"5min":254.89599999999999,"15min":254.89599999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":13.813000000000001,"5min":4.4199999999999999,"15min":2.843},"min":{"1min":0.29999999999999999,"5min":0.23699999999999999,"15min":0.23300000000000001},"max":{"1min":256.85399999999998,"5min":256.85399999999998,"15min":256.85399999999998},"last":0.63700000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.98099999999999998,"15min":1.208},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":6.6790000000000003,"5min":8.6750000000000007,"15min":10.475},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.0980000000000001,"5min":0.91300000000000003,"15min":1.266},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.248},"max":{"1min":4.5469999999999997,"5min":5.056,"15min":9.2400000000000002},"last":0.60199999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.615,"15min":1.4379999999999999},"min":{"1min":0.34999999999999998,"5min":0.32400000000000001,"15min":0.27900000000000003},"max":{"1min":3.6739999999999999,"5min":20.079000000000001,"15min":20.079000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.4039999999999999,"5min":1.4950000000000001,"15min":1.5049999999999999},"min":{"1min":0.39300000000000002,"5min":0.33700000000000002,"15min":0.30199999999999999},"max":{"1min":6.8769999999999998,"5min":10.282999999999999,"15min":12.044},"last":1.222}]},{"osd":5,"last update":"Mon Oct 9 09:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":1.5469999999999999,"15min":1.502},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.28399999999999997},"max":{"1min":6.8479999999999999,"5min":9.4280000000000008,"15min":9.4280000000000008},"last":1.179},{"interface":"front","average":{"1min":1.373,"5min":1.536,"15min":1.581},"min":{"1min":0.438,"5min":0.33700000000000002,"15min":0.29699999999999999},"max":{"1min":4.6020000000000003,"5min":9.6289999999999996,"15min":9.6289999999999996},"last":0.82199999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":1.3009999999999999,"15min":1.4710000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.315},"max":{"1min":3.2850000000000001,"5min":6.2110000000000003,"15min":10.413},"last":3.52},{"interface":"front","average":{"1min":1.819,"5min":1.8200000000000001,"15min":1.76},"min":{"1min":0.5,"5min":0.42599999999999999,"15min":0.35599999999999998},"max":{"1min":6.9400000000000004,"5min":9.3260000000000005,"15min":9.3260000000000005},"last":5.782}]},{"osd":7,"last update":"Mon Oct 9 09:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.2270000000000001,"15min":1.4450000000000001},"min":{"1min":0.41399999999999998,"5min":0.38,"15min":0.38},"max":{"1min":3.2120000000000002,"5min":6.9669999999999996,"15min":9.2159999999999993},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.4610000000000001,"5min":1.3300000000000001,"15min":1.502},"min":{"1min":0.435,"5min":0.435,"15min":0.39200000000000002},"max":{"1min":6.9089999999999998,"5min":10.087999999999999,"15min":14.361000000000001},"last":1.2509999999999999}]}]},{"osd":3,"up_from":0,"seq":2847563317687,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986820,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2678460,"kb_used_data":1779188,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":91066692,"statfs":{"total":95995035648,"available":93252292608,"internally_reserved":0,"allocated":1821888512,"data_stored":1821594883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:51:45 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":2.258,"15min":1.73},"min":{"1min":0.30299999999999999,"5min":0.28299999999999997,"15min":0.253},"max":{"1min":3.2959999999999998,"5min":64.372,"15min":64.372},"last":2.7759999999999998},{"interface":"front","average":{"1min":1.4119999999999999,"5min":2.198,"15min":1.704},"min":{"1min":0.38600000000000001,"5min":0.30099999999999999,"15min":0.27700000000000002},"max":{"1min":5.0650000000000004,"5min":64.356999999999999,"15min":64.356999999999999},"last":3.7480000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.4890000000000001,"15min":1.5629999999999999},"min":{"1min":0.48499999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.867,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.0880000000000001,"15min":0.94799999999999995},"min":{"1min":0.38700000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.0010000000000003,"5min":5.2249999999999996,"15min":5.2249999999999996},"last":2.4710000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":1.8560000000000001,"15min":1.583},"min":{"1min":0.53300000000000003,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":1.79,"5min":12.000999999999999,"15min":33.743000000000002},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.6020000000000001,"15min":1.5580000000000001},"min":{"1min":0.42399999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.649,"5min":7.3730000000000002,"15min":33.805999999999997},"last":0.38900000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:51:54 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.119,"15min":1.1299999999999999},"min":{"1min":0.39800000000000002,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":5.5129999999999999,"5min":8.1120000000000001,"15min":8.1120000000000001},"last":2.3090000000000002},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.1779999999999999,"15min":1.3680000000000001},"min":{"1min":0.38700000000000001,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":5.3109999999999999,"5min":7.4180000000000001,"15min":16.507000000000001},"last":0.48199999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:51:54 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.671,"15min":1.5920000000000001},"min":{"1min":0.38700000000000001,"5min":0.35599999999999998,"15min":0.26900000000000002},"max":{"1min":2.843,"5min":11.763999999999999,"15min":17.212},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.343,"5min":1.526,"15min":1.5549999999999999},"min":{"1min":0.439,"5min":0.29599999999999999,"15min":0.28499999999999998},"max":{"1min":5.1289999999999996,"5min":12.446999999999999,"15min":17.268999999999998},"last":2.3700000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:51:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.7909999999999999,"15min":1.6060000000000001},"min":{"1min":0.38800000000000001,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":5.2690000000000001,"5min":11.701000000000001,"15min":13.654},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.876,"5min":1.954,"15min":1.6779999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.35699999999999998},"max":{"1min":1.4990000000000001,"5min":11.972,"15min":11.972},"last":0.875}]},{"osd":7,"last update":"Mon Oct 9 09:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.637,"15min":1.379},"min":{"1min":0.57899999999999996,"5min":0.373,"15min":0.373},"max":{"1min":1.2849999999999999,"5min":11.840999999999999,"15min":11.840999999999999},"last":0.82599999999999996},{"interface":"front","average":{"1min":1.141,"5min":1.21,"15min":1.1899999999999999},"min":{"1min":0.54600000000000004,"5min":0.41299999999999998,"15min":0.34999999999999998},"max":{"1min":1.7669999999999999,"5min":9.6210000000000004,"15min":9.6210000000000004},"last":2.3450000000000002}]}]},{"osd":5,"up_from":37,"seq":158913790586,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2662376,"kb_used_data":1755872,"kb_used_omap":0,"kb_used_meta":906496,"kb_avail":91082776,"statfs":{"total":95995035648,"available":93268762624,"internally_reserved":0,"allocated":1798012928,"data_stored":1797735453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928251904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:51:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":2.4830000000000001,"15min":3.9420000000000002},"min":{"1min":0.33100000000000002,"5min":0.253,"15min":0.23599999999999999},"max":{"1min":9.8960000000000008,"5min":49.954999999999998,"15min":274.71899999999999},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.95399999999999996,"5min":2.0459999999999998,"15min":3.7450000000000001},"min":{"1min":0.38600000000000001,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":3.4929999999999999,"5min":50.042000000000002,"15min":274.73500000000001},"last":0.76000000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:51:14 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":1.6519999999999999,"15min":1.802},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.157,"5min":16.172000000000001,"15min":16.172000000000001},"last":24.957000000000001},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.1779999999999999,"15min":1.1679999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.1880000000000002,"5min":13.728,"15min":13.728},"last":24.920000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.607,"15min":1.6140000000000001},"min":{"1min":0.40200000000000002,"5min":0.312,"15min":0.312},"max":{"1min":1.6739999999999999,"5min":48.170999999999999,"15min":50.847000000000001},"last":0.94999999999999996},{"interface":"front","average":{"1min":0.98099999999999998,"5min":2.1480000000000001,"15min":1.98},"min":{"1min":0.41599999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"max":{"1min":1.655,"5min":48.128,"15min":50.656999999999996},"last":1.1319999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:51:36 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.1080000000000001,"15min":1.27},"min":{"1min":0.41199999999999998,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":2.8410000000000002,"5min":7.2270000000000003,"15min":10.125999999999999},"last":0.80000000000000004},{"interface":"front","average":{"1min":1.02,"5min":1.373,"15min":1.3480000000000001},"min":{"1min":0.44600000000000001,"5min":0.38,"15min":0.29099999999999998},"max":{"1min":2.4750000000000001,"5min":16.201000000000001,"15min":16.201000000000001},"last":1.083}]},{"osd":4,"last update":"Mon Oct 9 09:51:08 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.5229999999999999,"15min":1.3560000000000001},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.27000000000000002},"max":{"1min":2.798,"5min":9.4209999999999994,"15min":9.4209999999999994},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.9410000000000001,"15min":1.722},"min":{"1min":0.53900000000000003,"5min":0.374,"15min":0.33300000000000002},"max":{"1min":1.526,"5min":16.053000000000001,"15min":16.053000000000001},"last":1.1830000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:51:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.6739999999999999,"15min":1.931},"min":{"1min":0.36699999999999999,"5min":0.35399999999999998,"15min":0.33600000000000002},"max":{"1min":2.9079999999999999,"5min":9.6270000000000007,"15min":140.179},"last":1.042},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.9770000000000001,"15min":2.1869999999999998},"min":{"1min":0.498,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.1680000000000001,"5min":16.145,"15min":137.928},"last":1.2569999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:51:08 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.593,"15min":1.629},"min":{"1min":0.47799999999999998,"5min":0.432,"15min":0.33400000000000002},"max":{"1min":1.367,"5min":16.119,"15min":18.683},"last":1.22},{"interface":"front","average":{"1min":0.91400000000000003,"5min":1.0589999999999999,"15min":1.2130000000000001},"min":{"1min":0.5,"5min":0.34200000000000003,"15min":0.32100000000000001},"max":{"1min":1.395,"5min":4.6779999999999999,"15min":10.192},"last":1.1950000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7721,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3733258240,"data_stored":3733258240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4358799360,"data_stored":4358799360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2542600192,"data_stored":2542600192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1831731200,"data_stored":1831731200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1808007168,"data_stored":1808007168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3283353600,"data_stored":3283353600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:52:04.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 31 2023-10-09T09:52:04.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:04.420 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 95702 95686 78.6733 50.875 0.038647 0.012704 2023-10-09T09:52:04.422 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:04.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:04.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:04.527 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:04.563 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:04.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:04.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:04.667 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:04.693 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:04.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:04.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:04.796 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:04.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:04.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:04.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:04.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:04.965 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:04.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:05.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:05.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:05.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:05.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:05.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:05.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:05.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:05.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:05.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:05 smithi181 bash[59345]: cluster 2023-10-09T09:52:04.208603+0000 mgr.y (mgr.34107) 1386 : cluster 0 pgmap v2255: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 873 op/s 2023-10-09T09:52:05.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:05.359 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:05.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:05.418 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 96557 96541 78.3456 53.4375 0.0100132 0.0127588 2023-10-09T09:52:05.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:05.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:05 smithi163 bash[75425]: cluster 2023-10-09T09:52:04.208603+0000 mgr.y (mgr.34107) 1386 : cluster 0 pgmap v2255: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 873 op/s 2023-10-09T09:52:05.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:05 smithi163 bash[78107]: cluster 2023-10-09T09:52:04.208603+0000 mgr.y (mgr.34107) 1386 : cluster 0 pgmap v2255: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 873 op/s 2023-10-09T09:52:05.488 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:05.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:05.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:05.594 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:05.616 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:05.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:05.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:05.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:05.760 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:05.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:05.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:05.863 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:05.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:05.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:05.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:05.992 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:06.022 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:06.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:06.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:06.153 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:06.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:06.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:06.256 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:06.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:06.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:06.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:06.373 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:52:06.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:06.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:52:06.419 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 97427 97411 78.0382 54.375 0.0332172 0.0128081 2023-10-09T09:52:06.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:06.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:06.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:06.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:06.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:06.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:06.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:06.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:06.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:06.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:06.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:06.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:06.910 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:06.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:06.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:07.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:07.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:07.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:07.151 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:07.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:07.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:07.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:07.293 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:07.318 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:07.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:07.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:07.418 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 98194 98178 77.6572 47.9375 0.0157703 0.0128713 2023-10-09T09:52:07.424 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:07.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:07.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:07.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:07.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:07.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:07.685 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:07.716 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:07.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:07.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:07.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:07.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:07.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:07.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:07.978 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:07.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:07.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:08.084 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:08.093 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:07 smithi181 bash[59345]: cluster 2023-10-09T09:52:06.209035+0000 mgr.y (mgr.34107) 1387 : cluster 0 pgmap v2256: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-09T09:52:08.112 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:08.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:08.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:08.215 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:08.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:07 smithi163 bash[75425]: cluster 2023-10-09T09:52:06.209035+0000 mgr.y (mgr.34107) 1387 : cluster 0 pgmap v2256: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-09T09:52:08.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:07 smithi163 bash[78107]: cluster 2023-10-09T09:52:06.209035+0000 mgr.y (mgr.34107) 1387 : cluster 0 pgmap v2256: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-09T09:52:08.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:08.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:08.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:08.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:08.380 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:08.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:08.419 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T09:52:08.418849+0000 min lat: 0.00138896 max lat: 0.341984 avg lat: 0.0129227 2023-10-09T09:52:08.419 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:52:08.419 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 99046 99030 77.3521 53.25 0.0174551 0.0129227 2023-10-09T09:52:08.487 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:08.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:08.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:08.622 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:08.657 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:08.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:08.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:08.766 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:08.791 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:08.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:08.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:08.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:08.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:08.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:08.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:09.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:09.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:09.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:09.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:09.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:09.182 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:52:09.183 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:52:09.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:09.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:09.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:09.353 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:09.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:09.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:09.419 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 99874 99858 77.0359 51.75 0.03403 0.0129754 2023-10-09T09:52:09.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:09.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:09.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:09.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:09.587 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:09.620 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:09.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:09.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:09.726 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:09.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:09.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:09.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:09.887 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:09.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:09.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:09.989 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:09.999 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:09 smithi181 bash[59345]: cluster 2023-10-09T09:52:08.209730+0000 mgr.y (mgr.34107) 1388 : cluster 0 pgmap v2257: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T09:52:10.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:10.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:10.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:10.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:10.153 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:10.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:10.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:10.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:09 smithi163 bash[75425]: cluster 2023-10-09T09:52:08.209730+0000 mgr.y (mgr.34107) 1388 : cluster 0 pgmap v2257: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T09:52:10.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:09 smithi163 bash[78107]: cluster 2023-10-09T09:52:08.209730+0000 mgr.y (mgr.34107) 1388 : cluster 0 pgmap v2257: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T09:52:10.256 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:10.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:10.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:10.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:10.394 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:10.419 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 100720 100704 76.741 52.875 0.00934242 0.0130252 2023-10-09T09:52:10.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:10.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:10.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:10.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:10.574 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:10.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:10.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:10.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:10.710 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:10.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:10.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:10.815 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:10.848 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:10.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:10.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:10.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:10.991 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:10.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:10.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:11.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:11.127 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:11.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:11.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:11.230 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:52:11.231 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:52:11.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:11.295 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:11.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:11.399 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:11.412 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:11 smithi181 bash[59345]: cluster 2023-10-09T09:52:10.210181+0000 mgr.y (mgr.34107) 1389 : cluster 0 pgmap v2258: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s 2023-10-09T09:52:11.412 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:11 smithi181 bash[59345]: audit 2023-10-09T09:52:10.410898+0000 mon.a (mon.0) 2496 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:11.412 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:11 smithi181 bash[59345]: audit 2023-10-09T09:52:10.411928+0000 mon.a (mon.0) 2497 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:11.423 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 101558 101542 76.4438 52.375 0.0237031 0.0130752 2023-10-09T09:52:11.426 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:11.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:11.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:11.564 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:11.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:11.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:11.672 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:11.703 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:11.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:11.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:11 smithi163 bash[75425]: cluster 2023-10-09T09:52:10.210181+0000 mgr.y (mgr.34107) 1389 : cluster 0 pgmap v2258: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s 2023-10-09T09:52:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:11 smithi163 bash[75425]: audit 2023-10-09T09:52:10.410898+0000 mon.a (mon.0) 2496 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:11 smithi163 bash[75425]: audit 2023-10-09T09:52:10.411928+0000 mon.a (mon.0) 2497 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:11 smithi163 bash[78107]: cluster 2023-10-09T09:52:10.210181+0000 mgr.y (mgr.34107) 1389 : cluster 0 pgmap v2258: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s 2023-10-09T09:52:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:11 smithi163 bash[78107]: audit 2023-10-09T09:52:10.410898+0000 mon.a (mon.0) 2496 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:11.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:11 smithi163 bash[78107]: audit 2023-10-09T09:52:10.411928+0000 mon.a (mon.0) 2497 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:11.807 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:11.843 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:11.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:11.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:11.948 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:11.969 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:11.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:11.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:12.073 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:12.101 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:12.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:12.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:12.206 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:12.241 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:12.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:12.344 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:12.369 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:12.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:12.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:12.423 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 102468 102452 76.2108 56.875 0.0100018 0.0131147 2023-10-09T09:52:12.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:12.496 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:12.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:12.601 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:12.634 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:12.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:12.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:12.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:12.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:12.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:12.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:12.881 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:12.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:12.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:12.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:13.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:13.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:13.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:13.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:13.145 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:13.176 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:13.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:13.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:13.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:13.320 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:13.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:13.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:13.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:13.423 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 103338 103322 75.9539 54.375 0.0251346 0.01316 2023-10-09T09:52:13.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:13.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:13.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:13.556 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:13.567 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:52:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:52:13] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.33.4" 2023-10-09T09:52:13.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:13 smithi163 bash[75425]: cluster 2023-10-09T09:52:12.211036+0000 mgr.y (mgr.34107) 1390 : cluster 0 pgmap v2259: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T09:52:13.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:13 smithi163 bash[78107]: cluster 2023-10-09T09:52:12.211036+0000 mgr.y (mgr.34107) 1390 : cluster 0 pgmap v2259: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T09:52:13.593 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:13.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:13.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:13.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:13.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:13.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:13.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:13 smithi181 bash[59345]: cluster 2023-10-09T09:52:12.211036+0000 mgr.y (mgr.34107) 1390 : cluster 0 pgmap v2259: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T09:52:13.828 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:13.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:13.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:13.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:13.962 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:13.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:13.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:13.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:14.096 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:14.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:14.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:14.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:14.227 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:14.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:14.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:14.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:14.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:14.382 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:14.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:14.423 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 104076 104060 75.607 46.125 0.0222969 0.0132203 2023-10-09T09:52:14.485 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:14.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:14.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:14.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:14.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:14.644 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:14.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:14.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:14.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:14.768 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:14.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:14.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:14.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:14.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:14.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:14.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:15.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:15.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:15.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:15.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:15.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:15.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:15.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:15.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:15.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:15.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:15.293 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:15.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:15.394 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:15.421 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:15.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:15.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:15.425 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 104810 104794 75.2653 45.875 0.0196248 0.0132805 2023-10-09T09:52:15.525 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:15.549 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:15.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:15.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:15.663 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:15 smithi181 bash[59345]: cluster 2023-10-09T09:52:14.211696+0000 mgr.y (mgr.34107) 1391 : cluster 0 pgmap v2260: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 45 MiB/s wr, 724 op/s 2023-10-09T09:52:15.679 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:15.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:15.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:15.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:15 smithi163 bash[78107]: cluster 2023-10-09T09:52:14.211696+0000 mgr.y (mgr.34107) 1391 : cluster 0 pgmap v2260: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 45 MiB/s wr, 724 op/s 2023-10-09T09:52:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:15 smithi163 bash[75425]: cluster 2023-10-09T09:52:14.211696+0000 mgr.y (mgr.34107) 1391 : cluster 0 pgmap v2260: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 45 MiB/s wr, 724 op/s 2023-10-09T09:52:15.782 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:15.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:15.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:15.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:15.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:15.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:15.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:15.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:16.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:16.073 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:16.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:16.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:16.177 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:16.208 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:16.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:16.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:16.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:16.343 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:16.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:16.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:16.424 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 105637 105621 74.9973 51.6875 0.0166627 0.0133285 2023-10-09T09:52:16.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:16.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:16.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:16.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:16.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:16.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:16.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:16.724 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:16.750 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:16.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:16.855 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:16.883 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:16.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:16.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:16.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:17.019 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:17.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:17.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:17.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:17.154 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:17.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:17.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:17.257 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:17.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:17.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:17.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:17.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:17.413 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:17.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:17.423 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 106411 106395 74.6982 48.375 0.040997 0.0133815 2023-10-09T09:52:17.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:17.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:17.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:17.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:17.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:17.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:17.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:17.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:17.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:17.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:17.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:17.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:17.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:17.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:17.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:17.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:17 smithi163 bash[75425]: cluster 2023-10-09T09:52:16.212333+0000 mgr.y (mgr.34107) 1392 : cluster 0 pgmap v2261: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 46 MiB/s wr, 743 op/s 2023-10-09T09:52:17.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:17 smithi163 bash[78107]: cluster 2023-10-09T09:52:16.212333+0000 mgr.y (mgr.34107) 1392 : cluster 0 pgmap v2261: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 46 MiB/s wr, 743 op/s 2023-10-09T09:52:18.052 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:18.075 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:18.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:18.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:18.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:18.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:18.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:18.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:18.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:17 smithi181 bash[59345]: cluster 2023-10-09T09:52:16.212333+0000 mgr.y (mgr.34107) 1392 : cluster 0 pgmap v2261: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 46 MiB/s wr, 743 op/s 2023-10-09T09:52:18.316 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:18.343 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:18.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:18.424 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.0082 2023-10-09T09:52:18.424 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 107273 2023-10-09T09:52:18.424 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T09:52:18.424 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T09:52:18.424 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 74.4884 2023-10-09T09:52:18.425 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 30.0639 2023-10-09T09:52:18.425 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 188.375 2023-10-09T09:52:18.425 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 45.875 2023-10-09T09:52:18.425 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1191 2023-10-09T09:52:18.425 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 481.022 2023-10-09T09:52:18.425 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 3014 2023-10-09T09:52:18.425 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 734 2023-10-09T09:52:18.425 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.0134203 2023-10-09T09:52:18.425 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0126984 2023-10-09T09:52:18.426 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 0.341984 2023-10-09T09:52:18.426 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.00138896 2023-10-09T09:52:18.432 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T09:52:18.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:18.475 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:18.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:18.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:18.582 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:18.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:18.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:18.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:18.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:18.740 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:18.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:18.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:18.848 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:18.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:18.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:18.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:18.975 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:19.012 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:19.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:19.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:19.116 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:19.144 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:19.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:19.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:19.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:19.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:19.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:19.377 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:19.403 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:19.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:19.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:19.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:19.536 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:19.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:19.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:19.639 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:19.667 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:19.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:19.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:19.892 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:19.926 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:19.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:19.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:19.953 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:19 smithi163 bash[75425]: cluster 2023-10-09T09:52:18.212968+0000 mgr.y (mgr.34107) 1393 : cluster 0 pgmap v2262: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 61 MiB/s wr, 973 op/s 2023-10-09T09:52:20.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:20.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:20.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:20.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:20.189 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:20.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:20.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:19 smithi163 bash[78107]: cluster 2023-10-09T09:52:18.212968+0000 mgr.y (mgr.34107) 1393 : cluster 0 pgmap v2262: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 61 MiB/s wr, 973 op/s 2023-10-09T09:52:20.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:19 smithi181 bash[59345]: cluster 2023-10-09T09:52:18.212968+0000 mgr.y (mgr.34107) 1393 : cluster 0 pgmap v2262: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 61 MiB/s wr, 973 op/s 2023-10-09T09:52:20.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:20.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:20.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:20.428 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:20.457 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:20.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:20.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:20.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:20.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:20.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:20.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:20.727 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:20.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:20.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:20.834 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:20.862 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:20.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:20.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:20.996 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:21.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:21.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:21.102 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:21.126 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:21.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:21.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:21.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:21.255 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:21.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:21.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:21.363 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:21.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:21.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:21.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:21.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:21.534 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:21.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:21.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:21.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:21.666 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:21.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:21.774 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:21.809 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:21.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:21.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:21.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:21.940 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:21.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:21.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:21.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:21 smithi163 bash[75425]: cluster 2023-10-09T09:52:20.213459+0000 mgr.y (mgr.34107) 1394 : cluster 0 pgmap v2263: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-09T09:52:22.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:22.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:22.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:22.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:22.172 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:22.182 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:21 smithi181 bash[59345]: cluster 2023-10-09T09:52:20.213459+0000 mgr.y (mgr.34107) 1394 : cluster 0 pgmap v2263: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-09T09:52:22.199 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:22.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:22.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:21 smithi163 bash[78107]: cluster 2023-10-09T09:52:20.213459+0000 mgr.y (mgr.34107) 1394 : cluster 0 pgmap v2263: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-09T09:52:22.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:22.338 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:22.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:22.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:22.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:22.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:22.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:22.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:22.580 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:22.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:22.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:22.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:22.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:22.736 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:22.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:22.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:22.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:22.865 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:22.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:22.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:22.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:22.999 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:23.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:23.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:23.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:23.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:23.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:23.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:23.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:23.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:23.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:23.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:23.362 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:23.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:23.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:23.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:23.488 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:23.535 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:23.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:23.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:23.641 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:23.653 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:52:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:52:23] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.33.4" 2023-10-09T09:52:23.670 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:23.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:23.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:23.774 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:23.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:23.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:23.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:23.935 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:23.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:23.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:23 smithi163 bash[78107]: cluster 2023-10-09T09:52:22.214207+0000 mgr.y (mgr.34107) 1395 : cluster 0 pgmap v2264: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 50 MiB/s wr, 1.02k op/s 2023-10-09T09:52:23.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:23 smithi163 bash[75425]: cluster 2023-10-09T09:52:22.214207+0000 mgr.y (mgr.34107) 1395 : cluster 0 pgmap v2264: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 50 MiB/s wr, 1.02k op/s 2023-10-09T09:52:24.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:23 smithi181 bash[59345]: cluster 2023-10-09T09:52:22.214207+0000 mgr.y (mgr.34107) 1395 : cluster 0 pgmap v2264: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 50 MiB/s wr, 1.02k op/s 2023-10-09T09:52:24.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:24.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:24.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:24.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:24.202 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:24.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:24.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:24.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:24.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:24.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:24.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:24.434 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:24.478 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:24.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:24.582 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:24.602 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:24.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:24.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:24.705 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:24.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:24.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:24.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:24.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:24.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:24.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:24.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:24.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:25.005 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:25.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:25.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:25.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:25.141 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:25.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:25.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:25.247 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:25.271 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:25.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:25.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:25.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:25.405 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:25.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:25.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:25.409 INFO:teuthology.orchestra.run.smithi163.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T09:52:25.452 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:52:25.452 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T09:52:25.452 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T09:52:25.452 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T09:52:25.452 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.452 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:69945 10.5 10:a0af1f1a:::benchmark_data_smithi163_194945_object69944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1238)", 2023-10-09T09:52:25.452 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:46:28.850536+0000", 2023-10-09T09:52:25.452 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 356.58856782300001, 2023-10-09T09:52:25.452 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29601533699999999, 2023-10-09T09:52:25.452 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.453 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.453 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.453 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:52:25.453 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:52:25.453 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 69945 2023-10-09T09:52:25.453 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.453 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.453 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.454 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.454 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:28.850536+0000", 2023-10-09T09:52:25.454 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.454 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.454 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.454 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.454 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:28.850536+0000", 2023-10-09T09:52:25.454 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.454 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.454 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.455 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.455 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:28.850535+0000", 2023-10-09T09:52:25.455 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:52:25.455 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.455 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.455 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.455 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:28.850560+0000", 2023-10-09T09:52:25.455 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5168999999999999e-05 2023-10-09T09:52:25.455 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:28.850569+0000", 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3699999999999995e-06 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:28.850575+0000", 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8229999999999997e-06 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.456 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:29.134195+0000", 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.283619181 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:29.134375+0000", 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000180015 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:29.134571+0000", 2023-10-09T09:52:25.457 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000196572 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:29.141111+0000", 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065395200000000001 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:29.142235+0000", 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001123878 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.458 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:29.146465+0000", 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042305329999999999 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:29.146511+0000", 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5938e-05 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.459 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:46:29.146551+0000", 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0098999999999997e-05 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:176119 10.5 10:a585fa91:::benchmark_data_smithi163_194945_object66666:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1252)", 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:42.107319+0000", 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 223.331784261, 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32273748200000002, 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.460 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 176119 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.107319+0000", 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.461 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.107319+0000", 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.107318+0000", 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.462 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.107324+0000", 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0260000000000004e-06 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.107335+0000", 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0263e-05 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.107343+0000", 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0979999999999994e-06 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.463 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.112999+0000", 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056560079999999997 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.113128+0000", 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000129634 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.113249+0000", 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012114 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.121682+0000", 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0084323690000000003 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.122008+0000", 2023-10-09T09:52:25.465 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000325748 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.429946+0000", 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.30793820199999999 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.429995+0000", 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8761999999999998e-05 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.466 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.430057+0000", 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2273000000000005e-05 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:176140 10.5 10:a47fefca:::benchmark_data_smithi163_194945_object66687:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1252)", 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:42.143155+0000", 2023-10-09T09:52:25.467 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 223.295948544, 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28713754800000002, 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 176140 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.468 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.143155+0000", 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.143155+0000", 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.143154+0000", 2023-10-09T09:52:25.469 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.143158+0000", 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1119999999999998e-06 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.143168+0000", 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9709999999999999e-06 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.470 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.143174+0000", 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.246e-06 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.143215+0000", 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0741999999999997e-05 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.471 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.143328+0000", 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000112459 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.143423+0000", 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5408000000000005e-05 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.148177+0000", 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0047537250000000003 2023-10-09T09:52:25.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.148412+0000", 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023531 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.430102+0000", 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28168955000000001 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.430161+0000", 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9039000000000002e-05 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:42.430293+0000", 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000132046 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.474 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:177833 10.8 10:1b5220e8:::benchmark_data_smithi163_194945_object68380:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1252)", 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:44.879620+0000", 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 220.55948366800001, 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25698281000000001, 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 177833 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.475 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.879620+0000", 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.879620+0000", 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.476 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.879618+0000", 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.879625+0000", 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4420000000000003e-06 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.879639+0000", 2023-10-09T09:52:25.477 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.419e-05 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.879649+0000", 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7510000000000003e-06 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.882623+0000", 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029745119999999999 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.478 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.882741+0000", 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000117518 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.882856+0000", 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115264 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.892956+0000", 2023-10-09T09:52:25.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010100336999999999 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.901147+0000", 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0081904179999999997 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136374+0000", 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23522723100000001 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.480 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136488+0000", 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000113894 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136603+0000", 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000114763 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.481 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:177839 10.8 10:16c2c202:::benchmark_data_smithi163_194945_object68386:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1252)", 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:44.884455+0000", 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 220.55464887299999, 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25293237200000002, 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 177839 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.482 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.884455+0000", 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.884455+0000", 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.483 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.884453+0000", 2023-10-09T09:52:25.484 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:52:25.484 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.484 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.484 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.484 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.884459+0000", 2023-10-09T09:52:25.484 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0290000000000002e-06 2023-10-09T09:52:25.484 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.484 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.884469+0000", 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2469999999999997e-06 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.884475+0000", 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1680000000000001e-06 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.485 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.887505+0000", 2023-10-09T09:52:25.486 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003029833 2023-10-09T09:52:25.486 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.486 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.486 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.486 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.887617+0000", 2023-10-09T09:52:25.486 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011260999999999999 2023-10-09T09:52:25.486 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.486 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.887730+0000", 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011249200000000001 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.892962+0000", 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052324600000000004 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.487 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.901184+0000", 2023-10-09T09:52:25.488 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082214679999999991 2023-10-09T09:52:25.488 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.488 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.488 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.488 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136692+0000", 2023-10-09T09:52:25.488 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23550886700000001 2023-10-09T09:52:25.488 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.488 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.488 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.488 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136768+0000", 2023-10-09T09:52:25.489 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5365999999999996e-05 2023-10-09T09:52:25.489 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.489 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.489 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.489 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.137387+0000", 2023-10-09T09:52:25.489 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00061925100000000002 2023-10-09T09:52:25.489 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.489 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.489 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.489 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.490 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.490 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45545.0:177845 10.e e1252/1232 10:78afbca9:::benchmark_data_smithi163_194945_object68392:head v 1252'11189, mlcod=1252'11187)", 2023-10-09T09:52:25.490 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:44.895208+0000", 2023-10-09T09:52:25.490 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 220.543895091, 2023-10-09T09:52:25.490 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24099290700000001, 2023-10-09T09:52:25.490 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.490 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.490 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.490 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.491 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.491 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.895208+0000", 2023-10-09T09:52:25.491 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.491 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.491 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.491 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.491 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.895208+0000", 2023-10-09T09:52:25.491 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.491 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.492 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.492 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.492 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.895208+0000", 2023-10-09T09:52:25.492 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:52:25.492 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.492 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.492 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.492 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.895222+0000", 2023-10-09T09:52:25.493 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3991999999999999e-05 2023-10-09T09:52:25.493 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.493 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.493 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.493 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.895233+0000", 2023-10-09T09:52:25.493 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1395e-05 2023-10-09T09:52:25.493 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.493 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.493 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.493 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.895242+0000", 2023-10-09T09:52:25.494 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5490000000000006e-06 2023-10-09T09:52:25.494 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.494 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.494 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.494 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.895290+0000", 2023-10-09T09:52:25.494 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.782e-05 2023-10-09T09:52:25.494 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.494 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.494 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.494 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.895308+0000", 2023-10-09T09:52:25.495 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8196999999999999e-05 2023-10-09T09:52:25.495 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.495 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.495 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.495 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136148+0000", 2023-10-09T09:52:25.496 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.240840357 2023-10-09T09:52:25.496 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.496 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.496 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.496 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136201+0000", 2023-10-09T09:52:25.496 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3313999999999998e-05 2023-10-09T09:52:25.496 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.496 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.496 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.496 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.497 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.497 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.45545.0:177851 10.8 10:1572de10:::benchmark_data_smithi163_194945_object68398:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1252)", 2023-10-09T09:52:25.497 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:44.896936+0000", 2023-10-09T09:52:25.497 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 220.542167729, 2023-10-09T09:52:25.497 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.240692139, 2023-10-09T09:52:25.497 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.497 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.497 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.498 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.45545", 2023-10-09T09:52:25.498 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3273018963", 2023-10-09T09:52:25.498 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 177851 2023-10-09T09:52:25.498 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.498 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.498 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.498 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.498 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.896936+0000", 2023-10-09T09:52:25.499 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.499 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.499 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.499 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.499 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.896936+0000", 2023-10-09T09:52:25.499 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.499 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.499 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.501 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.501 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.896934+0000", 2023-10-09T09:52:25.501 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:52:25.501 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.501 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.501 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.501 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.896941+0000", 2023-10-09T09:52:25.501 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1210000000000003e-06 2023-10-09T09:52:25.501 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.501 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.896952+0000", 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1289e-05 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.896959+0000", 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4390000000000003e-06 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.502 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.503 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.897010+0000", 2023-10-09T09:52:25.503 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0355999999999997e-05 2023-10-09T09:52:25.503 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.503 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.504 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.504 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.897151+0000", 2023-10-09T09:52:25.504 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014123799999999999 2023-10-09T09:52:25.504 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.897295+0000", 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000144015 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136120+0000", 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23882482099999999 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.505 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.506 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136210+0000", 2023-10-09T09:52:25.506 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0694999999999998e-05 2023-10-09T09:52:25.506 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.506 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.137478+0000", 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001267904 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.137560+0000", 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1502000000000007e-05 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.137628+0000", 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8220999999999996e-05 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.45545.0:177853 10.f e1252/1232 10:fbe98c70:::benchmark_data_smithi163_194945_object68400:head v 1252'11197, mlcod=1252'11194)", 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:44.902161+0000", 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 220.53694245299999, 2023-10-09T09:52:25.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24843013, 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.902161+0000", 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.902161+0000", 2023-10-09T09:52:25.510 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.511 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.511 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.511 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.511 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.902160+0000", 2023-10-09T09:52:25.511 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:52:25.511 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.511 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.512 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.512 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.902168+0000", 2023-10-09T09:52:25.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7810000000000005e-06 2023-10-09T09:52:25.512 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.512 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.512 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.512 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136052+0000", 2023-10-09T09:52:25.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23388434799999999 2023-10-09T09:52:25.512 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136067+0000", 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5148e-05 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136167+0000", 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010020499999999999 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.513 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136250+0000", 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2534999999999995e-05 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.150559+0000", 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014308598 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.150591+0000", 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2702e-05 2023-10-09T09:52:25.514 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.45545.0:177851 10.8 e1252/1232 ondisk, result = 0)", 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:48:44.905826+0000", 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 220.53327744200001, 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.230391402, 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.515 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.905826+0000", 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.905826+0000", 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.905825+0000", 2023-10-09T09:52:25.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:52:25.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.517 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:44.905831+0000", 2023-10-09T09:52:25.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0179999999999998e-06 2023-10-09T09:52:25.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.517 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136081+0000", 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.230250023 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136107+0000", 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5661999999999999e-05 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136177+0000", 2023-10-09T09:52:25.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9843000000000002e-05 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136196+0000", 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9188e-05 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:48:45.136218+0000", 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1424999999999999e-05 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.519 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.520 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.520 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.520 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35814.0:40795 11.2 11:4a4c366c:::benchmark_data_smithi163_218041_object40794:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1274)", 2023-10-09T09:52:25.520 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:51:10.003807+0000", 2023-10-09T09:52:25.520 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 75.435296610999998, 2023-10-09T09:52:25.520 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24401931299999999, 2023-10-09T09:52:25.520 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.520 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35814", 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 40795 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.003807+0000", 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.522 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.522 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.522 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.003807+0000", 2023-10-09T09:52:25.522 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.522 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.522 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.523 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.523 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.003805+0000", 2023-10-09T09:52:25.523 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T09:52:25.523 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.523 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.523 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.003829+0000", 2023-10-09T09:52:25.523 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4258000000000001e-05 2023-10-09T09:52:25.523 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.003844+0000", 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4525000000000001e-05 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.003854+0000", 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0172e-05 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.003918+0000", 2023-10-09T09:52:25.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3689999999999995e-05 2023-10-09T09:52:25.525 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.004129+0000", 2023-10-09T09:52:25.526 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000211664 2023-10-09T09:52:25.526 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.526 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.526 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:52:25.526 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.004318+0000", 2023-10-09T09:52:25.526 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018861999999999999 2023-10-09T09:52:25.526 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.526 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.526 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.527 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.007399+0000", 2023-10-09T09:52:25.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00308085 2023-10-09T09:52:25.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.527 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.527 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.008228+0000", 2023-10-09T09:52:25.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082886400000000001 2023-10-09T09:52:25.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.247502+0000", 2023-10-09T09:52:25.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23927373699999999 2023-10-09T09:52:25.528 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.528 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.247636+0000", 2023-10-09T09:52:25.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000134614 2023-10-09T09:52:25.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.529 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.529 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.247826+0000", 2023-10-09T09:52:25.529 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019014499999999999 2023-10-09T09:52:25.529 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.529 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.529 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.530 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35814.0:41730 11.2 11:41535d2b:::benchmark_data_smithi163_218041_object41729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:52:25.530 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:51:10.868680+0000", 2023-10-09T09:52:25.530 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 74.570423621000003, 2023-10-09T09:52:25.530 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.18965774499999999, 2023-10-09T09:52:25.530 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.530 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.530 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.531 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35814", 2023-10-09T09:52:25.531 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:52:25.531 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 41730 2023-10-09T09:52:25.531 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.531 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.531 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.531 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.531 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.868680+0000", 2023-10-09T09:52:25.531 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.532 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.532 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.868680+0000", 2023-10-09T09:52:25.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.868679+0000", 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.868703+0000", 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4626000000000001e-05 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.534 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.534 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.868716+0000", 2023-10-09T09:52:25.534 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2622999999999999e-05 2023-10-09T09:52:25.534 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.534 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.534 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.534 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.868725+0000", 2023-10-09T09:52:25.535 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8060000000000001e-06 2023-10-09T09:52:25.535 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.868781+0000", 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6511000000000002e-05 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.868931+0000", 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000150283 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:52:25.537 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:10.869079+0000", 2023-10-09T09:52:25.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014774800000000001 2023-10-09T09:52:25.537 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.537 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.537 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.537 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:11.049104+0000", 2023-10-09T09:52:25.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.180024516 2023-10-09T09:52:25.537 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.538 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.538 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.538 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:11.058009+0000", 2023-10-09T09:52:25.538 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.008905296 2023-10-09T09:52:25.538 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.538 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.538 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.538 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:11.058129+0000", 2023-10-09T09:52:25.538 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000119726 2023-10-09T09:52:25.538 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:11.058198+0000", 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8831999999999994e-05 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:11.058338+0000", 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000140152 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.539 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35814.0:43417 11.4 e1275/1269 11:29e96952:::benchmark_data_smithi163_218041_object43416:head v 1275'2698, mlcod=1275'2695)", 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:51:12.262266+0000", 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.176837206000002, 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33743848799999998, 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.540 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.262266+0000", 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.262266+0000", 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.262266+0000", 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:52:25.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.542 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.542 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.542 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.262310+0000", 2023-10-09T09:52:25.542 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4245000000000001e-05 2023-10-09T09:52:25.542 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.542 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.542 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.542 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.262321+0000", 2023-10-09T09:52:25.542 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0933e-05 2023-10-09T09:52:25.542 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.543 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.543 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.543 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.262400+0000", 2023-10-09T09:52:25.543 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9709000000000005e-05 2023-10-09T09:52:25.543 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.543 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.543 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.543 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.262446+0000", 2023-10-09T09:52:25.543 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5834000000000002e-05 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.262467+0000", 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1160000000000001e-05 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.599639+0000", 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33717140899999998 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.599705+0000", 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5983999999999997e-05 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35814.0:43425 11.d 11:bb06e52f:::benchmark_data_smithi163_218041_object43424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:51:12.266377+0000", 2023-10-09T09:52:25.545 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.172726913000005, 2023-10-09T09:52:25.546 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32958547500000002, 2023-10-09T09:52:25.546 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.546 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.546 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.546 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35814", 2023-10-09T09:52:25.546 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:52:25.546 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 43425 2023-10-09T09:52:25.546 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.546 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.266377+0000", 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.266377+0000", 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.547 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.266375+0000", 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.266400+0000", 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5063999999999999e-05 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.266413+0000", 2023-10-09T09:52:25.548 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2805e-05 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.266421+0000", 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3450000000000006e-06 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.266464+0000", 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3106999999999999e-05 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.549 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.550 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.550 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.266616+0000", 2023-10-09T09:52:25.550 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015175599999999999 2023-10-09T09:52:25.550 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.550 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.550 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,6", 2023-10-09T09:52:25.550 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.266744+0000", 2023-10-09T09:52:25.550 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127632 2023-10-09T09:52:25.550 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.269417+0000", 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002673473 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.269777+0000", 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00035959499999999999 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.552 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595866+0000", 2023-10-09T09:52:25.552 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32608881499999998 2023-10-09T09:52:25.552 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.552 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.552 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.552 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595919+0000", 2023-10-09T09:52:25.552 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3424000000000002e-05 2023-10-09T09:52:25.552 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.552 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.552 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.553 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595962+0000", 2023-10-09T09:52:25.553 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3037000000000001e-05 2023-10-09T09:52:25.553 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.553 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.553 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.554 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.554 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.554 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35814.0:43426 11.c e1275/1269 11:3cb3015d:::benchmark_data_smithi163_218041_object43425:head v 1275'2726, mlcod=1275'2724)", 2023-10-09T09:52:25.554 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:51:12.268402+0000", 2023-10-09T09:52:25.554 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.170701425000004, 2023-10-09T09:52:25.554 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33749076900000002, 2023-10-09T09:52:25.554 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.554 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268402+0000", 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268402+0000", 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.556 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.556 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.556 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268401+0000", 2023-10-09T09:52:25.556 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:52:25.556 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.556 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.556 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.556 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268427+0000", 2023-10-09T09:52:25.556 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5833000000000002e-05 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268438+0000", 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.041e-05 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268444+0000", 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0129999999999999e-06 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.557 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.558 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.558 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268484+0000", 2023-10-09T09:52:25.558 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0828000000000003e-05 2023-10-09T09:52:25.558 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.558 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.558 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268508+0000", 2023-10-09T09:52:25.558 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3453000000000001e-05 2023-10-09T09:52:25.558 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.559 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.559 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.603550+0000", 2023-10-09T09:52:25.559 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33504206199999997 2023-10-09T09:52:25.559 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.559 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.559 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.559 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.605893+0000", 2023-10-09T09:52:25.559 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023429259999999999 2023-10-09T09:52:25.559 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.559 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35814.0:43427 11.4 e1275/1269 11:2311dd09:::benchmark_data_smithi163_218041_object43426:head v 1275'2699, mlcod=1275'2696)", 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:51:12.268781+0000", 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.170322773999999, 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33880807499999999, 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.560 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268781+0000", 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268781+0000", 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.561 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.562 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268780+0000", 2023-10-09T09:52:25.562 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T09:52:25.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.562 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268831+0000", 2023-10-09T09:52:25.562 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0862999999999997e-05 2023-10-09T09:52:25.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268842+0000", 2023-10-09T09:52:25.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0920999999999999e-05 2023-10-09T09:52:25.563 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.563 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.563 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268849+0000", 2023-10-09T09:52:25.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5020000000000003e-06 2023-10-09T09:52:25.564 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.564 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.564 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.564 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268892+0000", 2023-10-09T09:52:25.564 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2512999999999998e-05 2023-10-09T09:52:25.564 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.564 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.565 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.565 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.268915+0000", 2023-10-09T09:52:25.565 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2840000000000002e-05 2023-10-09T09:52:25.565 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.565 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.565 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.565 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.607553+0000", 2023-10-09T09:52:25.565 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33863857600000002 2023-10-09T09:52:25.565 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.566 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.566 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.566 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.607589+0000", 2023-10-09T09:52:25.566 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5648000000000002e-05 2023-10-09T09:52:25.566 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.566 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.566 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.566 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.566 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.566 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35814.0:43428 11.c e1275/1269 11:37d7f575:::benchmark_data_smithi163_218041_object43427:head v 1275'2727, mlcod=1275'2724)", 2023-10-09T09:52:25.567 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:51:12.270447+0000", 2023-10-09T09:52:25.567 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.168656440999996, 2023-10-09T09:52:25.567 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33716750600000001, 2023-10-09T09:52:25.567 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.567 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.567 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.567 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.567 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.567 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270447+0000", 2023-10-09T09:52:25.567 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.568 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.568 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.568 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.568 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270447+0000", 2023-10-09T09:52:25.568 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.568 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.568 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.568 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.568 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270446+0000", 2023-10-09T09:52:25.569 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:52:25.569 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.569 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.569 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.569 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270469+0000", 2023-10-09T09:52:25.569 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2926e-05 2023-10-09T09:52:25.569 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.569 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.569 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270479+0000", 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7769999999999997e-06 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270486+0000", 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4899999999999997e-06 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.570 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595778+0000", 2023-10-09T09:52:25.571 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32529199800000003 2023-10-09T09:52:25.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.571 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.571 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.571 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595800+0000", 2023-10-09T09:52:25.571 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2535999999999999e-05 2023-10-09T09:52:25.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.571 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.571 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.571 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.607592+0000", 2023-10-09T09:52:25.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011791797 2023-10-09T09:52:25.572 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.572 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.572 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.607615+0000", 2023-10-09T09:52:25.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2640999999999999e-05 2023-10-09T09:52:25.572 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.572 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.572 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.572 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35814.0:43431 11.2 11:49009a50:::benchmark_data_smithi163_218041_object43430:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:51:12.270554+0000", 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.168549408000004, 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33244010400000001, 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35814", 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 43431 2023-10-09T09:52:25.573 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.574 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.574 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.574 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.574 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270554+0000", 2023-10-09T09:52:25.575 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.575 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.575 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.575 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270554+0000", 2023-10-09T09:52:25.575 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.575 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.575 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.575 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270553+0000", 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270573+0000", 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0513000000000002e-05 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595747+0000", 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32517418199999998 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.576 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.577 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.577 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595784+0000", 2023-10-09T09:52:25.577 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6942e-05 2023-10-09T09:52:25.578 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.578 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.578 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.578 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595911+0000", 2023-10-09T09:52:25.578 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000126618 2023-10-09T09:52:25.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.579 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.579 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.579 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.596291+0000", 2023-10-09T09:52:25.579 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00038000499999999999 2023-10-09T09:52:25.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.579 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.579 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:52:25.580 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.596619+0000", 2023-10-09T09:52:25.580 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00032812899999999997 2023-10-09T09:52:25.580 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.580 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.580 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.580 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.601223+0000", 2023-10-09T09:52:25.580 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046036039999999999 2023-10-09T09:52:25.580 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.580 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.601901+0000", 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00067832299999999999 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.602819+0000", 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00091838200000000003 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.581 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.602946+0000", 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012698000000000001 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.602994+0000", 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7923000000000002e-05 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.582 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35814.0:43425 11.d e1275/1269 ondisk, result = 0)", 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:51:12.270628+0000", 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.168475920000006, 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32534252200000002, 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270628+0000", 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.583 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270628+0000", 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270627+0000", 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.584 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.270630+0000", 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.14e-06 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595764+0000", 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.32513374900000003 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595789+0000", 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5236e-05 2023-10-09T09:52:25.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595849+0000", 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9744999999999998e-05 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595858+0000", 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2059999999999996e-06 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.587 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.587 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:51:12.595970+0000", 2023-10-09T09:52:25.587 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111975 2023-10-09T09:52:25.587 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.587 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.587 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.587 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.587 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.587 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35814.0:103920 11.2 11:4cb122d2:::benchmark_data_smithi163_218041_object103919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:52:25.587 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:52:14.112235+0000", 2023-10-09T09:52:25.588 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 11.326868377, 2023-10-09T09:52:25.588 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.151050566, 2023-10-09T09:52:25.588 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.588 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.588 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.588 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35814", 2023-10-09T09:52:25.588 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:52:25.588 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 103920 2023-10-09T09:52:25.588 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.588 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.589 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.589 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.589 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.112235+0000", 2023-10-09T09:52:25.589 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.589 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.589 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.589 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.589 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.112235+0000", 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.112234+0000", 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.112258+0000", 2023-10-09T09:52:25.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4077999999999999e-05 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.112267+0000", 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8009999999999998e-06 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.112277+0000", 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9499999999999996e-06 2023-10-09T09:52:25.591 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.592 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.592 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.592 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.112455+0000", 2023-10-09T09:52:25.592 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017734700000000001 2023-10-09T09:52:25.592 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.592 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.593 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.593 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.112604+0000", 2023-10-09T09:52:25.593 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014924 2023-10-09T09:52:25.593 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.593 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.593 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:52:25.593 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.112962+0000", 2023-10-09T09:52:25.593 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00035848599999999998 2023-10-09T09:52:25.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.594 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.594 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.594 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.118420+0000", 2023-10-09T09:52:25.594 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054577239999999997 2023-10-09T09:52:25.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.594 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.595 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.122990+0000", 2023-10-09T09:52:25.595 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045701209999999999 2023-10-09T09:52:25.595 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.595 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.263077+0000", 2023-10-09T09:52:25.596 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14008689599999999 2023-10-09T09:52:25.596 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.596 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.596 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.596 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.263218+0000", 2023-10-09T09:52:25.596 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000140836 2023-10-09T09:52:25.596 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.596 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.597 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.597 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.263286+0000", 2023-10-09T09:52:25.597 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7751999999999998e-05 2023-10-09T09:52:25.597 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.597 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.597 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.597 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.598 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.598 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35814.0:104373 11.2 11:4635643e:::benchmark_data_smithi163_218041_object104372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:52:25.598 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T09:52:14.754080+0000", 2023-10-09T09:52:25.598 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 10.685023561, 2023-10-09T09:52:25.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14005571999999999, 2023-10-09T09:52:25.599 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T09:52:25.599 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:52:25.599 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T09:52:25.599 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35814", 2023-10-09T09:52:25.599 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:52:25.599 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 104373 2023-10-09T09:52:25.599 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.600 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T09:52:25.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T09:52:25.602 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.754080+0000", 2023-10-09T09:52:25.602 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.602 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T09:52:25.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.754080+0000", 2023-10-09T09:52:25.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T09:52:25.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T09:52:25.604 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.754079+0000", 2023-10-09T09:52:25.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T09:52:25.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.604 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T09:52:25.604 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.754104+0000", 2023-10-09T09:52:25.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4995e-05 2023-10-09T09:52:25.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T09:52:25.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.754117+0000", 2023-10-09T09:52:25.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3043e-05 2023-10-09T09:52:25.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T09:52:25.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.754125+0000", 2023-10-09T09:52:25.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1820000000000006e-06 2023-10-09T09:52:25.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.607 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T09:52:25.607 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.754178+0000", 2023-10-09T09:52:25.608 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3409999999999999e-05 2023-10-09T09:52:25.608 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.608 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.608 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T09:52:25.608 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.754363+0000", 2023-10-09T09:52:25.608 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018484400000000001 2023-10-09T09:52:25.608 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.608 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.608 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.754521+0000", 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015847800000000001 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.757593+0000", 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003071243 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.758489+0000", 2023-10-09T09:52:25.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00089634999999999997 2023-10-09T09:52:25.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:52:25.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.893915+0000", 2023-10-09T09:52:25.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.135425563 2023-10-09T09:52:25.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T09:52:25.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.894092+0000", 2023-10-09T09:52:25.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000177147 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T09:52:14.894136+0000", 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3942e-05 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T09:52:25.611 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:52:25.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:25.613 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:52:25.646 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:25.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:25.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:25.679 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:52:25.720 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:52:25.751 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:25.761 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:52:25.778 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:25.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:25.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:25.799 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:52:25.833 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:52:25.873 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:52:25.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:25.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:25.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:25.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:25.915 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:52:25.917 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:52:25.917 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:52:25.917 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:52:25.918 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:52:25.918 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:52:25.918 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:52:25.918 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:52:25.918 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:52:26.008 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:26.075 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:26.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:26.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:26.182 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:26.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:26.234 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:25 smithi181 bash[59345]: cluster 2023-10-09T09:52:24.214610+0000 mgr.y (mgr.34107) 1396 : cluster 0 pgmap v2265: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 30 MiB/s wr, 720 op/s 2023-10-09T09:52:26.234 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:25 smithi181 bash[59345]: audit 2023-10-09T09:52:25.405289+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.163:0/1932338788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T09:52:26.234 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:25 smithi181 bash[59345]: audit 2023-10-09T09:52:25.405878+0000 mon.a (mon.0) 2498 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T09:52:26.234 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:25 smithi181 bash[59345]: audit 2023-10-09T09:52:25.407520+0000 mon.a (mon.0) 2499 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:26.234 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:25 smithi181 bash[59345]: audit 2023-10-09T09:52:25.557379+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.163:0/593062783' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:52:26.234 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:25 smithi181 bash[59345]: audit 2023-10-09T09:52:25.943385+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.163:0/1895309057' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:52:26.234 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:25 smithi181 bash[59345]: audit 2023-10-09T09:52:25.952243+0000 mon.a (mon.0) 2500 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:52:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:26.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:26.336 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:26.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[75425]: cluster 2023-10-09T09:52:24.214610+0000 mgr.y (mgr.34107) 1396 : cluster 0 pgmap v2265: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 30 MiB/s wr, 720 op/s 2023-10-09T09:52:26.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[75425]: audit 2023-10-09T09:52:25.405289+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.163:0/1932338788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T09:52:26.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[75425]: audit 2023-10-09T09:52:25.405878+0000 mon.a (mon.0) 2498 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T09:52:26.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[75425]: audit 2023-10-09T09:52:25.407520+0000 mon.a (mon.0) 2499 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:26.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[75425]: audit 2023-10-09T09:52:25.557379+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.163:0/593062783' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:52:26.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[75425]: audit 2023-10-09T09:52:25.943385+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.163:0/1895309057' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:52:26.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[75425]: audit 2023-10-09T09:52:25.952243+0000 mon.a (mon.0) 2500 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:52:26.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[78107]: cluster 2023-10-09T09:52:24.214610+0000 mgr.y (mgr.34107) 1396 : cluster 0 pgmap v2265: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 30 MiB/s wr, 720 op/s 2023-10-09T09:52:26.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[78107]: audit 2023-10-09T09:52:25.405289+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.163:0/1932338788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T09:52:26.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[78107]: audit 2023-10-09T09:52:25.405878+0000 mon.a (mon.0) 2498 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T09:52:26.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[78107]: audit 2023-10-09T09:52:25.407520+0000 mon.a (mon.0) 2499 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:26.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[78107]: audit 2023-10-09T09:52:25.557379+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.163:0/593062783' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:52:26.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[78107]: audit 2023-10-09T09:52:25.943385+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.163:0/1895309057' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:52:26.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[78107]: audit 2023-10-09T09:52:25.952243+0000 mon.a (mon.0) 2500 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:52:26.366 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:26.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:26.470 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:26.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:26.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:26.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:26.602 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:26.630 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:26.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:26.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:26.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:26.769 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:26.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:26.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:26.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:26.907 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:26.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:26.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:26.965 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:52:27.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:27.045 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:27.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:27.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:26 smithi181 bash[59345]: cluster 2023-10-09T09:52:26.944475+0000 mon.a (mon.0) 2501 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:52:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:26 smithi181 bash[59345]: audit 2023-10-09T09:52:26.950099+0000 mon.a (mon.0) 2502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:52:27.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:26 smithi181 bash[59345]: cluster 2023-10-09T09:52:26.950438+0000 mon.a (mon.0) 2503 : cluster 0 osdmap e1278: 8 total, 8 up, 6 in 2023-10-09T09:52:27.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:27.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:27.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:27.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[75425]: cluster 2023-10-09T09:52:26.944475+0000 mon.a (mon.0) 2501 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:52:27.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[75425]: audit 2023-10-09T09:52:26.950099+0000 mon.a (mon.0) 2502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:52:27.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[75425]: cluster 2023-10-09T09:52:26.950438+0000 mon.a (mon.0) 2503 : cluster 0 osdmap e1278: 8 total, 8 up, 6 in 2023-10-09T09:52:27.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[78107]: cluster 2023-10-09T09:52:26.944475+0000 mon.a (mon.0) 2501 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:52:27.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[78107]: audit 2023-10-09T09:52:26.950099+0000 mon.a (mon.0) 2502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:52:27.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:26 smithi163 bash[78107]: cluster 2023-10-09T09:52:26.950438+0000 mon.a (mon.0) 2503 : cluster 0 osdmap e1278: 8 total, 8 up, 6 in 2023-10-09T09:52:27.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:27.309 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:27.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:27.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:27.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:27.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:27.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:27.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:27.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:27.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:27.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:27.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:27.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:27.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:27.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:27.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:27.830 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:27.856 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:27.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:27.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:27.963 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:27.980 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:27 smithi181 bash[59345]: cluster 2023-10-09T09:52:26.218609+0000 mgr.y (mgr.34107) 1397 : cluster 0 pgmap v2266: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 29 MiB/s wr, 745 op/s 2023-10-09T09:52:27.980 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:27 smithi181 bash[59345]: cluster 2023-10-09T09:52:26.946427+0000 osd.5 (osd.5) 868 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:27.980 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:27 smithi181 bash[59345]: cluster 2023-10-09T09:52:26.955168+0000 osd.4 (osd.4) 669 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:27.980 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:27 smithi181 bash[59345]: cluster 2023-10-09T09:52:26.955299+0000 osd.6 (osd.6) 559 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:27.980 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:27 smithi181 bash[59345]: cluster 2023-10-09T09:52:27.906436+0000 osd.7 (osd.7) 615 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:27.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:27.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:27.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:28.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:27 smithi163 bash[78107]: cluster 2023-10-09T09:52:26.218609+0000 mgr.y (mgr.34107) 1397 : cluster 0 pgmap v2266: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 29 MiB/s wr, 745 op/s 2023-10-09T09:52:28.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:27 smithi163 bash[78107]: cluster 2023-10-09T09:52:26.946427+0000 osd.5 (osd.5) 868 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:28.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:27 smithi163 bash[78107]: cluster 2023-10-09T09:52:26.955168+0000 osd.4 (osd.4) 669 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:28.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:27 smithi163 bash[78107]: cluster 2023-10-09T09:52:26.955299+0000 osd.6 (osd.6) 559 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:28.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:27 smithi163 bash[78107]: cluster 2023-10-09T09:52:27.906436+0000 osd.7 (osd.7) 615 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:28.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:27 smithi163 bash[75425]: cluster 2023-10-09T09:52:26.218609+0000 mgr.y (mgr.34107) 1397 : cluster 0 pgmap v2266: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 85 B/s rd, 29 MiB/s wr, 745 op/s 2023-10-09T09:52:28.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:27 smithi163 bash[75425]: cluster 2023-10-09T09:52:26.946427+0000 osd.5 (osd.5) 868 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:28.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:27 smithi163 bash[75425]: cluster 2023-10-09T09:52:26.955168+0000 osd.4 (osd.4) 669 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:28.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:27 smithi163 bash[75425]: cluster 2023-10-09T09:52:26.955299+0000 osd.6 (osd.6) 559 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:28.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:27 smithi163 bash[75425]: cluster 2023-10-09T09:52:27.906436+0000 osd.7 (osd.7) 615 : cluster 3 failed to encode map e1278 with expected crc 2023-10-09T09:52:28.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:28.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:28.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:28.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:28.232 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:28.261 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:28.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:28.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:28.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:28.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:28.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:28.494 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:28.524 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:28.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:28.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:28.627 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:28.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:28.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:28.762 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:28.782 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:28.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:28.886 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:28.910 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:28.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:28.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:29.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:29.041 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:29.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:29.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:29.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:29.170 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:29.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:29.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:29.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:29.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:29.306 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:29.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:29.407 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:29.436 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:29.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:29.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:29.444 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T09:52:29.540 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:29.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:29.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:29.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:29.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:29.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:29.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:29.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:29.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:29.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:29.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:29.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:29.945 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:29.977 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:29.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:29.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:30.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:30.111 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:30.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:30.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:30.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:29 smithi163 bash[75425]: cluster 2023-10-09T09:52:28.219129+0000 mgr.y (mgr.34107) 1398 : cluster 0 pgmap v2268: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 10 MiB/s wr, 841 op/s 2023-10-09T09:52:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:29 smithi163 bash[78107]: cluster 2023-10-09T09:52:28.219129+0000 mgr.y (mgr.34107) 1398 : cluster 0 pgmap v2268: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 10 MiB/s wr, 841 op/s 2023-10-09T09:52:30.255 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:30.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:30.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:29 smithi181 bash[59345]: cluster 2023-10-09T09:52:28.219129+0000 mgr.y (mgr.34107) 1398 : cluster 0 pgmap v2268: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 10 MiB/s wr, 841 op/s 2023-10-09T09:52:30.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:30.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T09:52:30.385 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:30.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:30.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:30.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:30.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:30.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:30.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:30.630 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:30.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:30.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:30.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:30.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:30.797 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:30.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:30.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:30.901 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:30.939 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:30.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:30.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:31.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:31.069 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:31.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:31.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:31.174 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:31.202 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:31.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:31.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:31.309 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:31.343 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:31.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:31.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:31.448 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:31.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:31.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:31.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:31.577 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:31.599 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:31.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:31.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:31.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:31.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:31.843 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:31.867 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:31.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:31.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:31.973 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:32.007 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:32.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:32.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:32.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:32.120 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:31 smithi163 bash[75425]: cluster 2023-10-09T09:52:30.219611+0000 mgr.y (mgr.34107) 1399 : cluster 0 pgmap v2269: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 9.0 MiB/s wr, 841 op/s 2023-10-09T09:52:32.121 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:31 smithi163 bash[78107]: cluster 2023-10-09T09:52:30.219611+0000 mgr.y (mgr.34107) 1399 : cluster 0 pgmap v2269: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 9.0 MiB/s wr, 841 op/s 2023-10-09T09:52:32.141 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:32.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:32.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:32.246 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:32.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:32.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:32.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:31 smithi181 bash[59345]: cluster 2023-10-09T09:52:30.219611+0000 mgr.y (mgr.34107) 1399 : cluster 0 pgmap v2269: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 102 B/s rd, 9.0 MiB/s wr, 841 op/s 2023-10-09T09:52:32.385 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:52:32.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:32.390 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:52:32.409 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:32.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:32.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:32.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:32.550 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:32.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:32.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:32.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:32.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:32.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:32.798 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:32.836 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:32.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:32.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:32.941 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:32.965 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:32.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:32.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:33.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:33.100 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:33.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:33.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:33.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:33.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:33.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:33.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:33.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:33.360 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:52:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:52:33] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-09T09:52:33.372 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:33.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:33.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:33.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:33.506 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:33.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:33.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:33.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:33.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:33.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:33.738 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:33.760 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:33.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:33.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:33.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:33.898 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:33.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:33.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:33.989 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:33 smithi181 bash[59345]: cluster 2023-10-09T09:52:32.220329+0000 mgr.y (mgr.34107) 1400 : cluster 0 pgmap v2270: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 832 op/s 2023-10-09T09:52:34.002 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:34.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:34.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:34.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:34.133 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:34.159 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:34.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:34.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:33 smithi163 bash[75425]: cluster 2023-10-09T09:52:32.220329+0000 mgr.y (mgr.34107) 1400 : cluster 0 pgmap v2270: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 832 op/s 2023-10-09T09:52:34.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:33 smithi163 bash[78107]: cluster 2023-10-09T09:52:32.220329+0000 mgr.y (mgr.34107) 1400 : cluster 0 pgmap v2270: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 832 op/s 2023-10-09T09:52:34.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:34.305 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:34.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:34.407 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:34.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:34.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:34.445 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 6, 2, 5] out_osds: [7, 3] dead_osds: [] live_osds: [2, 3, 4, 5, 6, 7, 0, 1] 2023-10-09T09:52:34.445 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T09:52:34.540 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:34.584 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:34.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:34.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:34.693 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:34.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:34.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:34.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:34.833 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:34.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:34.875 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:34.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:34.976 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:35.004 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:35.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:35.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:35.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:35.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:35.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:35.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:35.237 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:52:35.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:35.269 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:35.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:35.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:35.373 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:35.417 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:35.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:35.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:35.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:35.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:35.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:35.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:35.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:35.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:35.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:35.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:35.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:35.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:35.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:35.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:35.943 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:35.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:36.048 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:36.083 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:36.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:36.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:36.192 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:36.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:35 smithi163 bash[75425]: cluster 2023-10-09T09:52:34.220844+0000 mgr.y (mgr.34107) 1401 : cluster 0 pgmap v2271: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 826 op/s 2023-10-09T09:52:36.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:35 smithi163 bash[78107]: cluster 2023-10-09T09:52:34.220844+0000 mgr.y (mgr.34107) 1401 : cluster 0 pgmap v2271: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 826 op/s 2023-10-09T09:52:36.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:36.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:36.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:36.261 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:52:36.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:35 smithi181 bash[59345]: cluster 2023-10-09T09:52:34.220844+0000 mgr.y (mgr.34107) 1401 : cluster 0 pgmap v2271: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 826 op/s 2023-10-09T09:52:36.328 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:36.361 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:36.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:36.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:36.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:36.509 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:36.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:36.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:36.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:36.645 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:36.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:36.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:36.748 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:36.791 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:36.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:36.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:36.894 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:36.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:36.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:37.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:37.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:37.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:37.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:37.174 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:37.205 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:37.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:37.279 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:52:37.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:37.350 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:37.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:37.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:37.488 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:37.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:37.595 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:37.621 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:37.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:37.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:37.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:37.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:37.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:37.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:37.858 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:37.886 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:37.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:37.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:37.992 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:38.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:38.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:38.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:38.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:38.152 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:38.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:38.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:38.257 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:38.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:37 smithi163 bash[75425]: cluster 2023-10-09T09:52:36.221312+0000 mgr.y (mgr.34107) 1402 : cluster 0 pgmap v2272: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T09:52:38.268 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:37 smithi163 bash[78107]: cluster 2023-10-09T09:52:36.221312+0000 mgr.y (mgr.34107) 1402 : cluster 0 pgmap v2272: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T09:52:38.281 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:38.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:38.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:38.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:37 smithi181 bash[59345]: cluster 2023-10-09T09:52:36.221312+0000 mgr.y (mgr.34107) 1402 : cluster 0 pgmap v2272: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T09:52:38.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:38.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:38.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:38.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:38.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:38.544 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:38.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:38.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:38.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:38.675 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:38.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:38.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:38.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:38.805 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:38.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:38.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:38.911 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:38.937 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:38.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:38.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:39.041 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:39.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:39.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:39.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:39.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:39.214 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:39.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:39.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:39.307 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:52:39.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:39.358 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:39.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:39.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:39.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:39.487 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:39.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:39.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:39.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:39.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:39.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:39.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:39.731 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:39.756 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:39.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:39.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:39.859 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:39.886 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:39.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:39.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:39.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:39.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:39 smithi163 bash[75425]: cluster 2023-10-09T09:52:38.221963+0000 mgr.y (mgr.34107) 1403 : cluster 0 pgmap v2273: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 873 op/s 2023-10-09T09:52:40.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:40.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:40.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:40.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:40.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:40.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:40.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:40.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:40.270 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:39 smithi163 bash[78107]: cluster 2023-10-09T09:52:38.221963+0000 mgr.y (mgr.34107) 1403 : cluster 0 pgmap v2273: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 873 op/s 2023-10-09T09:52:40.282 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:40.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:40.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:40.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:39 smithi181 bash[59345]: cluster 2023-10-09T09:52:38.221963+0000 mgr.y (mgr.34107) 1403 : cluster 0 pgmap v2273: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 873 op/s 2023-10-09T09:52:40.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:40.409 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:40.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:40.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:40.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:40.542 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:40.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:40.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:40.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:40.669 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:40.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:40.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:40.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:40.797 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:40.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:40.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:40.902 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:40.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:40.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:40.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:41.042 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:41.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:41.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:41.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:41.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:41.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:41.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:41.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:41.332 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:41.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:41.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:41.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:41.460 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:41.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:41.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:41.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:41 smithi163 bash[75425]: cluster 2023-10-09T09:52:40.222441+0000 mgr.y (mgr.34107) 1404 : cluster 0 pgmap v2274: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 717 op/s 2023-10-09T09:52:41.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:41 smithi163 bash[75425]: audit 2023-10-09T09:52:40.412906+0000 mon.a (mon.0) 2504 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:41.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:41 smithi163 bash[75425]: audit 2023-10-09T09:52:40.416435+0000 mon.a (mon.0) 2505 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:41.563 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:41.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:41 smithi181 bash[59345]: cluster 2023-10-09T09:52:40.222441+0000 mgr.y (mgr.34107) 1404 : cluster 0 pgmap v2274: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 717 op/s 2023-10-09T09:52:41.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:41 smithi181 bash[59345]: audit 2023-10-09T09:52:40.412906+0000 mon.a (mon.0) 2504 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:41.571 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:41 smithi181 bash[59345]: audit 2023-10-09T09:52:40.416435+0000 mon.a (mon.0) 2505 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:41.594 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:41.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:41.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:41.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:41.720 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:41.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:41.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:41.779 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:41 smithi163 bash[78107]: cluster 2023-10-09T09:52:40.222441+0000 mgr.y (mgr.34107) 1404 : cluster 0 pgmap v2274: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 717 op/s 2023-10-09T09:52:41.779 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:41 smithi163 bash[78107]: audit 2023-10-09T09:52:40.412906+0000 mon.a (mon.0) 2504 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:41.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:41 smithi163 bash[78107]: audit 2023-10-09T09:52:40.416435+0000 mon.a (mon.0) 2505 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:41.823 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:41.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:41.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:41.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:41.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:41.987 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:41.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:41.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:42.093 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:42.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:42.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:42.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:42.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:42.256 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:42.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:42.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:42.359 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:42.397 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:42.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:42.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:42.502 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:42.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:42.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:42.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:42.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:42.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:42.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:42.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:42.770 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:42.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:42.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:42.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:42.899 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:42.928 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:42.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:42.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:43.034 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:43.070 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:43.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:43.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:43.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:43.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:43.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:43.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:43.306 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:43.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:43.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:43.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:43.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:43.462 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:43.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:43.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:43.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:43.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:43 smithi181 bash[59345]: cluster 2023-10-09T09:52:42.223249+0000 mgr.y (mgr.34107) 1405 : cluster 0 pgmap v2275: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T09:52:43.573 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:52:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:52:43] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.33.4" 2023-10-09T09:52:43.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:43 smithi163 bash[78107]: cluster 2023-10-09T09:52:42.223249+0000 mgr.y (mgr.34107) 1405 : cluster 0 pgmap v2275: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T09:52:43.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:43 smithi163 bash[75425]: cluster 2023-10-09T09:52:42.223249+0000 mgr.y (mgr.34107) 1405 : cluster 0 pgmap v2275: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T09:52:43.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:43.597 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:43.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:43.698 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:43.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:43.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:43.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:43.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:43.856 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:43.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:43.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:43.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:43.992 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:43.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:43.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:44.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:44.126 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:44.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:44.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:44.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:44.276 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:44.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:44.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:44.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:44.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:44.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:44.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:44.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:44.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:44.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:44.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:44.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:44.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:44.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:44.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:44.779 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:44.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:44.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:44.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:44.918 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:44.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:44.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:44.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:45.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:45.083 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:45.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:45.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:45.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:45.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:45.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:45.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:45.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:45.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:45.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:45.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:45.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:45.469 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:45 smithi181 bash[59345]: cluster 2023-10-09T09:52:44.223787+0000 mgr.y (mgr.34107) 1406 : cluster 0 pgmap v2276: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 711 op/s 2023-10-09T09:52:45.489 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:45.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:45.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:45.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:45.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:45.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:45.718 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:45 smithi163 bash[78107]: cluster 2023-10-09T09:52:44.223787+0000 mgr.y (mgr.34107) 1406 : cluster 0 pgmap v2276: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 711 op/s 2023-10-09T09:52:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:45 smithi163 bash[75425]: cluster 2023-10-09T09:52:44.223787+0000 mgr.y (mgr.34107) 1406 : cluster 0 pgmap v2276: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 711 op/s 2023-10-09T09:52:45.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:45.772 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:45.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:45.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:45.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:45.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:45.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:45.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:46.010 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:46.038 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:46.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:46.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:46.170 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:46.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:46.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:46.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:46.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:46.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:46.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:46.402 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:46.432 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:46.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:46.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:46.535 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:46.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:46.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:46.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:46.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:46.695 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:46.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:46.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:46.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:46.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:46.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:46.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:46.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:46.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:46.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:47.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:47.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:47.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:47.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:47.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:47.260 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:47.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:47.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:47.363 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:47.393 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:47.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:47.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:47.498 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:47.535 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:47.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:47.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:47.638 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:47.666 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:47.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:47.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:47.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:47.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:47.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:47.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:47.903 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:47.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:47.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:48.037 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:48.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:48.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:48.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:48.169 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:48.196 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:48.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:48.229 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:48 smithi163 bash[78107]: cluster 2023-10-09T09:52:46.224409+0000 mgr.y (mgr.34107) 1407 : cluster 0 pgmap v2277: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T09:52:48.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:48 smithi163 bash[75425]: cluster 2023-10-09T09:52:46.224409+0000 mgr.y (mgr.34107) 1407 : cluster 0 pgmap v2277: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T09:52:48.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:48.312 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:47 smithi181 bash[59345]: cluster 2023-10-09T09:52:46.224409+0000 mgr.y (mgr.34107) 1407 : cluster 0 pgmap v2277: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T09:52:48.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:48.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:48.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:48.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:48.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:48.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:48.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:48.569 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:48.595 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:48.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:48.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:48.727 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:48.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:48.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:48.837 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:48.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:48.863 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:48.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:48.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:48 smithi163 bash[75425]: audit 2023-10-09T09:52:48.847104+0000 mon.a (mon.0) 2506 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:52:48.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:49.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:49.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:49.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:49.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:49.129 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:48 smithi181 bash[59345]: audit 2023-10-09T09:52:48.847104+0000 mon.a (mon.0) 2506 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:52:49.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:49.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:49.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:49.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:48 smithi163 bash[78107]: audit 2023-10-09T09:52:48.847104+0000 mon.a (mon.0) 2506 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:52:49.250 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:49.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:49.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:49.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:49.395 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:49.423 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:49.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:49.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:49.527 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:49.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:49.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:49.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:49.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:49.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:49.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:49.777 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:49.805 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:49.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:49.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:49.910 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:49.939 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:49.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:49.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:50.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:50.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:49 smithi181 bash[59345]: cluster 2023-10-09T09:52:48.224970+0000 mgr.y (mgr.34107) 1408 : cluster 0 pgmap v2278: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 978 op/s 2023-10-09T09:52:50.090 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:50.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:50.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:50.195 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:50.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:49 smithi163 bash[78107]: cluster 2023-10-09T09:52:48.224970+0000 mgr.y (mgr.34107) 1408 : cluster 0 pgmap v2278: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 978 op/s 2023-10-09T09:52:50.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:49 smithi163 bash[75425]: cluster 2023-10-09T09:52:48.224970+0000 mgr.y (mgr.34107) 1408 : cluster 0 pgmap v2278: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 978 op/s 2023-10-09T09:52:50.223 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:50.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:50.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:50.330 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:50.362 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:50.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:50.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:50.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:50.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:50.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:50.609 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:50.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:50.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:50.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:50.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:50.759 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:50.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:50.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:50.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:50.888 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:50.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:50.990 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:51.017 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:51.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:51.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:51.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:51.155 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:51.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:51.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:51.258 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:51.285 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:51.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:51.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:51.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:51.427 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:51.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:51.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:51.536 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:51.563 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:51.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:51.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:51.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:51.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:51.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:51.806 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:51.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:51.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:51.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:51.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:51.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:51.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:51.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:51 smithi163 bash[75425]: cluster 2023-10-09T09:52:50.227265+0000 mgr.y (mgr.34107) 1409 : cluster 0 pgmap v2279: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 711 op/s 2023-10-09T09:52:51.995 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:52:51.995 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:52:52.066 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:52.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:51 smithi181 bash[59345]: cluster 2023-10-09T09:52:50.227265+0000 mgr.y (mgr.34107) 1409 : cluster 0 pgmap v2279: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 711 op/s 2023-10-09T09:52:52.093 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:52.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:52.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:52.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:52.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:52.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:52.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:52.236 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:51 smithi163 bash[78107]: cluster 2023-10-09T09:52:50.227265+0000 mgr.y (mgr.34107) 1409 : cluster 0 pgmap v2279: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 711 op/s 2023-10-09T09:52:52.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:52.354 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:52.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:52.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:52.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:52.510 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:52.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:52.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:52.615 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:52.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:52.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:52.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:52.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:52.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:52.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:52.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:52.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:52.918 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:52.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:52.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:52.995 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T09:52:53.026 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:53.034 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:52 smithi181 bash[59345]: audit 2023-10-09T09:52:52.697869+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.163:0/2034313549' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:52:53.034 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:52 smithi181 bash[59345]: audit 2023-10-09T09:52:52.698364+0000 mon.a (mon.0) 2507 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:52:53.054 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:53.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:53.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:53.129 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:52 smithi163 bash[75425]: audit 2023-10-09T09:52:52.697869+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.163:0/2034313549' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:52:53.129 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:52 smithi163 bash[75425]: audit 2023-10-09T09:52:52.698364+0000 mon.a (mon.0) 2507 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:52:53.130 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:52 smithi163 bash[78107]: audit 2023-10-09T09:52:52.697869+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.163:0/2034313549' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:52:53.130 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:52 smithi163 bash[78107]: audit 2023-10-09T09:52:52.698364+0000 mon.a (mon.0) 2507 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:52:53.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:53.168 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:52:53.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:53.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:53.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:53.206 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:52:53.250 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:52:53.283 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:52:53.287 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:53.314 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:53.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:53.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:53.323 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:52:53.357 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:52:53.389 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:52:53.412 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:52:53.418 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:52:53] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.33.4" 2023-10-09T09:52:53.422 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:53.426 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:52:53.426 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:52:53.426 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:52:53.426 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:52:53.426 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:52:53.426 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:52:53.426 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:52:53.426 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:52:53.443 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:53.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:53.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:53.581 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:53.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:53.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:53.725 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:53.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:53.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:53.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:53.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:53.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:53.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:53.997 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:54.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:54.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:54.126 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:54.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:54.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:54.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[78107]: cluster 2023-10-09T09:52:52.227963+0000 mgr.y (mgr.34107) 1410 : cluster 0 pgmap v2280: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T09:52:54.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[78107]: cluster 2023-10-09T09:52:52.972609+0000 osd.5 (osd.5) 869 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[78107]: audit 2023-10-09T09:52:52.979962+0000 mon.a (mon.0) 2508 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:52:54.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[78107]: cluster 2023-10-09T09:52:52.980140+0000 mon.a (mon.0) 2509 : cluster 0 osdmap e1279: 8 total, 8 up, 6 in 2023-10-09T09:52:54.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[78107]: cluster 2023-10-09T09:52:52.986900+0000 osd.6 (osd.6) 560 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[78107]: cluster 2023-10-09T09:52:52.992275+0000 osd.6 (osd.6) 561 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[78107]: cluster 2023-10-09T09:52:53.007450+0000 osd.4 (osd.4) 670 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[78107]: audit 2023-10-09T09:52:53.127446+0000 mon.a (mon.0) 2510 : audit 0 from='client.? 172.21.15.163:0/2168635011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:52:54.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[75425]: cluster 2023-10-09T09:52:52.227963+0000 mgr.y (mgr.34107) 1410 : cluster 0 pgmap v2280: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T09:52:54.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[75425]: cluster 2023-10-09T09:52:52.972609+0000 osd.5 (osd.5) 869 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[75425]: audit 2023-10-09T09:52:52.979962+0000 mon.a (mon.0) 2508 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:52:54.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[75425]: cluster 2023-10-09T09:52:52.980140+0000 mon.a (mon.0) 2509 : cluster 0 osdmap e1279: 8 total, 8 up, 6 in 2023-10-09T09:52:54.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[75425]: cluster 2023-10-09T09:52:52.986900+0000 osd.6 (osd.6) 560 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[75425]: cluster 2023-10-09T09:52:52.992275+0000 osd.6 (osd.6) 561 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[75425]: cluster 2023-10-09T09:52:53.007450+0000 osd.4 (osd.4) 670 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:53 smithi163 bash[75425]: audit 2023-10-09T09:52:53.127446+0000 mon.a (mon.0) 2510 : audit 0 from='client.? 172.21.15.163:0/2168635011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:52:54.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:54.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:54.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:53 smithi181 bash[59345]: cluster 2023-10-09T09:52:52.227963+0000 mgr.y (mgr.34107) 1410 : cluster 0 pgmap v2280: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T09:52:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:53 smithi181 bash[59345]: cluster 2023-10-09T09:52:52.972609+0000 osd.5 (osd.5) 869 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:53 smithi181 bash[59345]: audit 2023-10-09T09:52:52.979962+0000 mon.a (mon.0) 2508 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:52:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:53 smithi181 bash[59345]: cluster 2023-10-09T09:52:52.980140+0000 mon.a (mon.0) 2509 : cluster 0 osdmap e1279: 8 total, 8 up, 6 in 2023-10-09T09:52:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:53 smithi181 bash[59345]: cluster 2023-10-09T09:52:52.986900+0000 osd.6 (osd.6) 560 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:53 smithi181 bash[59345]: cluster 2023-10-09T09:52:52.992275+0000 osd.6 (osd.6) 561 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:53 smithi181 bash[59345]: cluster 2023-10-09T09:52:53.007450+0000 osd.4 (osd.4) 670 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:54.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:53 smithi181 bash[59345]: audit 2023-10-09T09:52:53.127446+0000 mon.a (mon.0) 2510 : audit 0 from='client.? 172.21.15.163:0/2168635011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:52:54.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:54.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:54.409 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:54.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:54.510 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:54.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:54.539 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:54.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:54.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:54.673 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:54.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:54.777 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:54.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:52:54.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:54.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:52:54.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:54.975 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:54.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:54.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:55.007 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:52:55.080 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:55.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: cluster 2023-10-09T09:52:53.516340+0000 osd.7 (osd.7) 616 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:55.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: audit 2023-10-09T09:52:53.993193+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.163:0/4223755574' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:52:55.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: audit 2023-10-09T09:52:53.993731+0000 mon.a (mon.0) 2511 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:52:55.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:55.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:55.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:55.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:55.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:55.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:55.373 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:54 smithi181 bash[59345]: cluster 2023-10-09T09:52:53.516340+0000 osd.7 (osd.7) 616 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:55.373 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:54 smithi181 bash[59345]: audit 2023-10-09T09:52:53.993193+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.163:0/4223755574' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:52:55.373 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:54 smithi181 bash[59345]: audit 2023-10-09T09:52:53.993731+0000 mon.a (mon.0) 2511 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:52:55.377 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:55.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:55.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:55.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: cluster 2023-10-09T09:52:53.516340+0000 osd.7 (osd.7) 616 : cluster 3 failed to encode map e1279 with expected crc 2023-10-09T09:52:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: audit 2023-10-09T09:52:53.993193+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.163:0/4223755574' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:52:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: audit 2023-10-09T09:52:53.993731+0000 mon.a (mon.0) 2511 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:52:55.523 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:55.550 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:55.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:55.653 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:55.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:55.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:55.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:55.793 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:55.831 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:55.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:55.935 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:55.972 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:55.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:55.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:56.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:56.111 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:56.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:56.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:56.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:56.250 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:56.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:56.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:56.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: cluster 2023-10-09T09:52:54.228443+0000 mgr.y (mgr.34107) 1411 : cluster 0 pgmap v2282: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 834 op/s 2023-10-09T09:52:56.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: cluster 2023-10-09T09:52:54.973413+0000 mon.a (mon.0) 2512 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:52:56.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: cluster 2023-10-09T09:52:54.990529+0000 osd.5 (osd.5) 870 : cluster 3 failed to encode map e1280 with expected crc 2023-10-09T09:52:56.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: audit 2023-10-09T09:52:54.994639+0000 mon.a (mon.0) 2513 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:52:56.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: cluster 2023-10-09T09:52:54.994862+0000 mon.a (mon.0) 2514 : cluster 0 osdmap e1280: 8 total, 8 up, 6 in 2023-10-09T09:52:56.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: cluster 2023-10-09T09:52:55.002208+0000 osd.4 (osd.4) 671 : cluster 3 failed to encode map e1280 with expected crc 2023-10-09T09:52:56.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: audit 2023-10-09T09:52:55.408312+0000 mon.a (mon.0) 2515 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:56.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: audit 2023-10-09T09:52:55.554676+0000 mon.a (mon.0) 2516 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:56.267 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[75425]: audit 2023-10-09T09:52:55.565096+0000 mon.a (mon.0) 2517 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:56.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: cluster 2023-10-09T09:52:54.228443+0000 mgr.y (mgr.34107) 1411 : cluster 0 pgmap v2282: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 834 op/s 2023-10-09T09:52:56.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: cluster 2023-10-09T09:52:54.973413+0000 mon.a (mon.0) 2512 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:52:56.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: cluster 2023-10-09T09:52:54.990529+0000 osd.5 (osd.5) 870 : cluster 3 failed to encode map e1280 with expected crc 2023-10-09T09:52:56.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: audit 2023-10-09T09:52:54.994639+0000 mon.a (mon.0) 2513 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:52:56.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: cluster 2023-10-09T09:52:54.994862+0000 mon.a (mon.0) 2514 : cluster 0 osdmap e1280: 8 total, 8 up, 6 in 2023-10-09T09:52:56.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: cluster 2023-10-09T09:52:55.002208+0000 osd.4 (osd.4) 671 : cluster 3 failed to encode map e1280 with expected crc 2023-10-09T09:52:56.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: audit 2023-10-09T09:52:55.408312+0000 mon.a (mon.0) 2515 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:56.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: audit 2023-10-09T09:52:55.554676+0000 mon.a (mon.0) 2516 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:56.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:55 smithi163 bash[78107]: audit 2023-10-09T09:52:55.565096+0000 mon.a (mon.0) 2517 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:55 smithi181 bash[59345]: cluster 2023-10-09T09:52:54.228443+0000 mgr.y (mgr.34107) 1411 : cluster 0 pgmap v2282: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 834 op/s 2023-10-09T09:52:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:55 smithi181 bash[59345]: cluster 2023-10-09T09:52:54.973413+0000 mon.a (mon.0) 2512 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:52:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:55 smithi181 bash[59345]: cluster 2023-10-09T09:52:54.990529+0000 osd.5 (osd.5) 870 : cluster 3 failed to encode map e1280 with expected crc 2023-10-09T09:52:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:55 smithi181 bash[59345]: audit 2023-10-09T09:52:54.994639+0000 mon.a (mon.0) 2513 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:52:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:55 smithi181 bash[59345]: cluster 2023-10-09T09:52:54.994862+0000 mon.a (mon.0) 2514 : cluster 0 osdmap e1280: 8 total, 8 up, 6 in 2023-10-09T09:52:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:55 smithi181 bash[59345]: cluster 2023-10-09T09:52:55.002208+0000 osd.4 (osd.4) 671 : cluster 3 failed to encode map e1280 with expected crc 2023-10-09T09:52:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:55 smithi181 bash[59345]: audit 2023-10-09T09:52:55.408312+0000 mon.a (mon.0) 2515 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:55 smithi181 bash[59345]: audit 2023-10-09T09:52:55.554676+0000 mon.a (mon.0) 2516 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:56.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:55 smithi181 bash[59345]: audit 2023-10-09T09:52:55.565096+0000 mon.a (mon.0) 2517 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:52:56.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:56.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:56.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:56.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:56.492 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T09:52:56.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:56.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:56.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:56.628 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:56.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:56.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:56.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:56.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:56.790 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:52:56.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:56.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:52:56.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T09:52:56.898 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:56.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:56.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:56.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:57.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:57.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:57.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:57.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:57.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:57.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:57.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:57.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:57.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:57.322 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:57.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:57.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:57.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:57.467 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:57.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:57.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:57.570 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:57.600 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:57.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:57.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:57.703 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:57.737 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:57.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:57.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:57.845 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:57.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:57.880 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:57.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:57.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:58.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:58.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:58.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:58.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:58.157 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:58.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:58.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:58.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:58.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:57 smithi163 bash[75425]: cluster 2023-10-09T09:52:56.229001+0000 mgr.y (mgr.34107) 1412 : cluster 0 pgmap v2284: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 605 op/s 2023-10-09T09:52:58.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:57 smithi163 bash[78107]: cluster 2023-10-09T09:52:56.229001+0000 mgr.y (mgr.34107) 1412 : cluster 0 pgmap v2284: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 605 op/s 2023-10-09T09:52:58.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:57 smithi181 bash[59345]: cluster 2023-10-09T09:52:56.229001+0000 mgr.y (mgr.34107) 1412 : cluster 0 pgmap v2284: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 605 op/s 2023-10-09T09:52:58.318 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:58.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:58.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:58.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:58.462 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:52:58.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:58.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:52:58.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:58.588 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:52:58.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:58.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:52:58.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:52:58.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:58.720 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:58.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:58.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:58.824 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:58.871 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:58.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:58.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:58.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:59.007 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:59.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:59.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:59.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:59.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:59.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:59.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:59.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:59.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:59.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:59.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:59.378 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:59.404 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:52:59.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:59.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:52:59.507 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:59.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:52:59.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:59.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:52:59.640 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:59.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:59.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:59.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:52:59.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:52:59.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:59.799 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:52:59.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:59.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:52:59.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:59.941 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:52:59.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:52:59.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:00.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:00.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:00.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:00.172 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:00.216 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:00.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:00.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:59 smithi181 bash[59345]: cluster 2023-10-09T09:52:58.229698+0000 mgr.y (mgr.34107) 1413 : cluster 0 pgmap v2285: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T09:53:00.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:59 smithi181 bash[59345]: audit 2023-10-09T09:52:59.687609+0000 mon.a (mon.0) 2518 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:00.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:52:59 smithi181 bash[59345]: audit 2023-10-09T09:52:59.697424+0000 mon.a (mon.0) 2519 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:00.321 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:00.351 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:00.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:00.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:00.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:59 smithi163 bash[75425]: cluster 2023-10-09T09:52:58.229698+0000 mgr.y (mgr.34107) 1413 : cluster 0 pgmap v2285: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T09:53:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:59 smithi163 bash[75425]: audit 2023-10-09T09:52:59.687609+0000 mon.a (mon.0) 2518 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:52:59 smithi163 bash[75425]: audit 2023-10-09T09:52:59.697424+0000 mon.a (mon.0) 2519 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:59 smithi163 bash[78107]: cluster 2023-10-09T09:52:58.229698+0000 mgr.y (mgr.34107) 1413 : cluster 0 pgmap v2285: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T09:53:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:52:59 smithi163 bash[78107]: audit 2023-10-09T09:52:59.687609+0000 mon.a (mon.0) 2518 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:00 smithi163 bash[78107]: audit 2023-10-09T09:52:59.697424+0000 mon.a (mon.0) 2519 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:00.478 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:00.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:00.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:00.585 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:00.608 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:00.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:00.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:00.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:00.737 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:00.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:00.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:00.842 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:00.867 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:00.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:00.973 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:01.006 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:01.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:01.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:01.111 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:01.120 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:01 smithi163 bash[78107]: audit 2023-10-09T09:53:00.281927+0000 mon.a (mon.0) 2520 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:53:01.120 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:01 smithi163 bash[78107]: audit 2023-10-09T09:53:00.283236+0000 mon.a (mon.0) 2521 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:53:01.121 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:01 smithi163 bash[78107]: audit 2023-10-09T09:53:00.296625+0000 mon.a (mon.0) 2522 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:01.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:01 smithi163 bash[75425]: audit 2023-10-09T09:53:00.281927+0000 mon.a (mon.0) 2520 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:53:01.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:01 smithi163 bash[75425]: audit 2023-10-09T09:53:00.283236+0000 mon.a (mon.0) 2521 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:53:01.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:01 smithi163 bash[75425]: audit 2023-10-09T09:53:00.296625+0000 mon.a (mon.0) 2522 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:01.156 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:01.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:01.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:01.245 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:01.247 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:01.262 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:01.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:01.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:01.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:01 smithi181 bash[59345]: audit 2023-10-09T09:53:00.281927+0000 mon.a (mon.0) 2520 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:53:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:01 smithi181 bash[59345]: audit 2023-10-09T09:53:00.283236+0000 mon.a (mon.0) 2521 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:53:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:01 smithi181 bash[59345]: audit 2023-10-09T09:53:00.296625+0000 mon.a (mon.0) 2522 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:01.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:01.436 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:01.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:01.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:01.540 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:01.576 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:01.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:01.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:01.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:01.715 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:01.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:01.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:01.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:01.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:01.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:01.943 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:01.974 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:01.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:01.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:02.081 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:02.090 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:02 smithi163 bash[75425]: cluster 2023-10-09T09:53:00.230189+0000 mgr.y (mgr.34107) 1414 : cluster 0 pgmap v2286: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 497 op/s 2023-10-09T09:53:02.090 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:02 smithi163 bash[78107]: cluster 2023-10-09T09:53:00.230189+0000 mgr.y (mgr.34107) 1414 : cluster 0 pgmap v2286: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 497 op/s 2023-10-09T09:53:02.106 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:02.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:02.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:02.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:02.216 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:02 smithi181 bash[59345]: cluster 2023-10-09T09:53:00.230189+0000 mgr.y (mgr.34107) 1414 : cluster 0 pgmap v2286: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 497 op/s 2023-10-09T09:53:02.235 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:02.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:02.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:02.339 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:02.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:02.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:02.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:02.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:02.529 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:02.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:02.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:02.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:02.662 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:02.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:02.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:02.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:02.799 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:02.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:02.904 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:02.932 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:02.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:02.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:03.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:03.073 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:03.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:03.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:03.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:03.211 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:03.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:03.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:03.296 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:03.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:03.341 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:03.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:03.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:03.446 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:03.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:03.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:03.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:03.585 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:03.613 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:03.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:03.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:53:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:53:03] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.33.4" 2023-10-09T09:53:03.720 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:03.754 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:03.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:03.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:03.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:03.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:03.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:03.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:03.999 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:04.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:04.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:04.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:04.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:04.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:04.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:04.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:04.262 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:04.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:04 smithi181 bash[59345]: cluster 2023-10-09T09:53:02.230959+0000 mgr.y (mgr.34107) 1415 : cluster 0 pgmap v2287: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T09:53:04.294 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:04.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:04.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:04.313 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:04.351 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:04 smithi163 bash[75425]: cluster 2023-10-09T09:53:02.230959+0000 mgr.y (mgr.34107) 1415 : cluster 0 pgmap v2287: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T09:53:04.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:04 smithi163 bash[78107]: cluster 2023-10-09T09:53:02.230959+0000 mgr.y (mgr.34107) 1415 : cluster 0 pgmap v2287: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T09:53:04.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:04.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:04.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:04.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:04.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:04.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:04.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:04.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:04.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:04.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:04.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:04.800 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:04.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:04.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:04.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:04.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:04.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:04.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:04.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:05.072 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:05.099 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:05.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:05.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:05.206 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:05.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:05.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:05.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:05.343 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:05.378 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:05.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:05.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:05.482 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:05.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:05.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:05.609 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:05.636 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:05.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:05.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:05.742 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:05.769 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:05.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:05.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:05.898 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:05.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:05.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:06.002 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:06.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:06.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:06.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:06.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:06.167 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:06.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:06.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:06.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:06.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:06 smithi163 bash[75425]: cluster 2023-10-09T09:53:04.231532+0000 mgr.y (mgr.34107) 1416 : cluster 0 pgmap v2288: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 761 op/s 2023-10-09T09:53:06.285 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:06 smithi163 bash[78107]: cluster 2023-10-09T09:53:04.231532+0000 mgr.y (mgr.34107) 1416 : cluster 0 pgmap v2288: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 761 op/s 2023-10-09T09:53:06.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:06.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:06.404 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:06 smithi181 bash[59345]: cluster 2023-10-09T09:53:04.231532+0000 mgr.y (mgr.34107) 1416 : cluster 0 pgmap v2288: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 761 op/s 2023-10-09T09:53:06.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:06.505 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:06.535 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:06.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:06.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:06.639 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:06.669 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:06.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:06.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:06.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:06.797 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:06.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:06.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:06.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:06.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:06.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:06.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:07.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:07.065 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:07.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:07.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:07.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:07.202 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:07.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:07.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:07.306 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:07.342 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:07.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:07.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:07.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:07.474 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:07.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:07.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:07.772 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:07.803 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:07.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:07.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:07.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:07.937 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:07.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:07.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:08.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:08.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:08 smithi181 bash[59345]: cluster 2023-10-09T09:53:06.232040+0000 mgr.y (mgr.34107) 1417 : cluster 0 pgmap v2289: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T09:53:08.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:08.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:08.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:08.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:08.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:08 smithi163 bash[75425]: cluster 2023-10-09T09:53:06.232040+0000 mgr.y (mgr.34107) 1417 : cluster 0 pgmap v2289: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T09:53:08.184 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:08 smithi163 bash[78107]: cluster 2023-10-09T09:53:06.232040+0000 mgr.y (mgr.34107) 1417 : cluster 0 pgmap v2289: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T09:53:08.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:08.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:08.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:08.306 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:08.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:08.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:08.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:08.473 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:08.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:08.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:08.578 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:08.607 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:08.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:08.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:08.711 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:08.743 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:08.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:08.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:08.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:08.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:08.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:08.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:08.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:09.019 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:09.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:09.126 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:09.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:09.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:09.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:09.262 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:09.289 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:09.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:09.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:09.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:09.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:09.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:09.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:09.543 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:09.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:09.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:09.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:09.664 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:09.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:09.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:09.766 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:09.794 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:09.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:09.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:09.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:09.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:09.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:10.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:10.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:10 smithi163 bash[75425]: cluster 2023-10-09T09:53:08.232914+0000 mgr.y (mgr.34107) 1418 : cluster 0 pgmap v2290: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 916 op/s 2023-10-09T09:53:10.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:10 smithi163 bash[78107]: cluster 2023-10-09T09:53:08.232914+0000 mgr.y (mgr.34107) 1418 : cluster 0 pgmap v2290: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 916 op/s 2023-10-09T09:53:10.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:10.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:10.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:10.169 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:10.197 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:10.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:10.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:10.302 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:10.310 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:10 smithi181 bash[59345]: cluster 2023-10-09T09:53:08.232914+0000 mgr.y (mgr.34107) 1418 : cluster 0 pgmap v2290: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 916 op/s 2023-10-09T09:53:10.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:10.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:10.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:10.445 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:10.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:10.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:10.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:10.575 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:10.599 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:10.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:10.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:10.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:10.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:10.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:10.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:10.946 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:10.976 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:10.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:11.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:11 smithi163 bash[75425]: audit 2023-10-09T09:53:10.408648+0000 mon.a (mon.0) 2523 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:11.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:11 smithi163 bash[78107]: audit 2023-10-09T09:53:10.408648+0000 mon.a (mon.0) 2523 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:11.083 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:11.105 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:11.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:11.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:11.218 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:11 smithi181 bash[59345]: audit 2023-10-09T09:53:10.408648+0000 mon.a (mon.0) 2523 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:11.237 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:11.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:11.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:11.343 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:11.377 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:11.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:11.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:11.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:11.510 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:11.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:11.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:11.615 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:11.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:11.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:11.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:11.756 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:11.784 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:11.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:11.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:11.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:11.911 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:11.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:11.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:12.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:12.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:12 smithi163 bash[75425]: cluster 2023-10-09T09:53:10.233452+0000 mgr.y (mgr.34107) 1419 : cluster 0 pgmap v2291: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T09:53:12.051 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:12.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:12.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:12.159 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:12.196 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:12.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:12.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:12.301 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:12.309 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:12 smithi163 bash[78107]: cluster 2023-10-09T09:53:10.233452+0000 mgr.y (mgr.34107) 1419 : cluster 0 pgmap v2291: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T09:53:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:12 smithi181 bash[59345]: cluster 2023-10-09T09:53:10.233452+0000 mgr.y (mgr.34107) 1419 : cluster 0 pgmap v2291: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T09:53:12.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:12.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:12.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:12.433 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:12.456 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:12.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:12.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:12.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:12.603 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:12.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:12.709 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:12.740 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:12.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:12.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:12.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:12.885 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:12.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:12.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:12.990 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:13.022 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:13.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:13.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:13.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:13.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:13.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:13.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:13.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:13.289 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:13.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:13.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:13.396 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:13.423 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:13.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:13.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:13.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:13.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:13.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:13.656 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:13.680 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:13.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:13.741 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:53:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:53:13] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-09T09:53:13.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:13.812 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:13.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:13.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:13.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:13.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:13.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:13.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:14.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:14 smithi163 bash[75425]: cluster 2023-10-09T09:53:12.234261+0000 mgr.y (mgr.34107) 1420 : cluster 0 pgmap v2292: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 994 op/s 2023-10-09T09:53:14.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:14 smithi163 bash[78107]: cluster 2023-10-09T09:53:12.234261+0000 mgr.y (mgr.34107) 1420 : cluster 0 pgmap v2292: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 994 op/s 2023-10-09T09:53:14.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:14.077 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:14.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:14.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:14.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:14.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:14 smithi181 bash[59345]: cluster 2023-10-09T09:53:12.234261+0000 mgr.y (mgr.34107) 1420 : cluster 0 pgmap v2292: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 994 op/s 2023-10-09T09:53:14.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:14.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:14.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:14.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:14.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:14.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:14.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:14.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:14.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:14.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:14.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:14.589 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:14.617 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:14.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:14.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:14.720 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:14.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:14.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:14.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:14.860 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:14.894 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:14.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:14.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:14.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:15.023 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:15.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:15.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:15.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:15.176 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:15.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:15.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:15.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:15.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:15.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:15.416 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:15.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:15.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:15.549 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:15.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:15.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:15.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:15.698 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:15.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:15.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:15.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:15.852 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:15.883 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:15.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:15.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:16.020 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:16.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:16.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:16.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:16.133 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:16 smithi163 bash[78107]: cluster 2023-10-09T09:53:14.234770+0000 mgr.y (mgr.34107) 1421 : cluster 0 pgmap v2293: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 717 op/s 2023-10-09T09:53:16.133 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:16 smithi163 bash[75425]: cluster 2023-10-09T09:53:14.234770+0000 mgr.y (mgr.34107) 1421 : cluster 0 pgmap v2293: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 717 op/s 2023-10-09T09:53:16.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:16.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:16.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:16.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:16.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:16.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:16.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:16.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:16 smithi181 bash[59345]: cluster 2023-10-09T09:53:14.234770+0000 mgr.y (mgr.34107) 1421 : cluster 0 pgmap v2293: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 717 op/s 2023-10-09T09:53:16.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:16.428 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:16.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:16.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:16.562 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:16.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:16.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:16.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:16.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:16.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:16.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:16.852 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:16.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:16.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:16.956 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:16.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:16.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:16.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:17.091 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:17.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:17.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:17.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:17.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:17.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:17.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:17.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:17.400 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:17.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:17.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:17.505 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:17.529 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:17.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:17.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:17.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:17.670 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:17.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:17.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:17.801 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:17.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:17.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:17.937 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:17.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:17.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:18.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:18.090 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:18.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:18.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:18.196 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:18.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:18 smithi163 bash[75425]: cluster 2023-10-09T09:53:16.235238+0000 mgr.y (mgr.34107) 1422 : cluster 0 pgmap v2294: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 741 op/s 2023-10-09T09:53:18.205 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:18 smithi163 bash[78107]: cluster 2023-10-09T09:53:16.235238+0000 mgr.y (mgr.34107) 1422 : cluster 0 pgmap v2294: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 741 op/s 2023-10-09T09:53:18.228 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:18.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:18.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:18.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:18 smithi181 bash[59345]: cluster 2023-10-09T09:53:16.235238+0000 mgr.y (mgr.34107) 1422 : cluster 0 pgmap v2294: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 741 op/s 2023-10-09T09:53:18.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:18.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:18.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:18.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:18.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:18.486 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:18.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:18.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:18.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:18.616 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:18.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:18.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:18.630 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:53:18.630 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:53:18.631 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T09:53:18.631 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T09:53:18.631 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:53:18.725 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:18.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:18.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:18.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:18.782 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:53:18.834 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:53:18.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:18.880 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:53:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:18.907 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:18.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:18.920 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:53:18.957 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:53:18.999 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:53:19.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:19.047 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:19.047 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:53:19.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:19.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:19.056 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:19 smithi181 bash[59345]: audit 2023-10-09T09:53:18.379283+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.163:0/413686918' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:53:19.056 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:19 smithi181 bash[59345]: audit 2023-10-09T09:53:18.380415+0000 mon.a (mon.0) 2524 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:53:19.057 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:19 smithi181 bash[59345]: audit 2023-10-09T09:53:18.691238+0000 mon.a (mon.0) 2525 : audit 0 from='client.? 172.21.15.163:0/107983433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:53:19.057 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:19 smithi181 bash[59345]: audit 2023-10-09T09:53:18.993525+0000 mon.a (mon.0) 2526 : audit 1 from='client.? 172.21.15.163:0/3127007731' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:53:19.081 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:53:19.086 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:53:19.087 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:53:19.087 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:53:19.087 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:53:19.088 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:53:19.088 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:53:19.088 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:53:19.088 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:53:19.088 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:53:19.090 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T09:53:19.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:19 smithi163 bash[75425]: audit 2023-10-09T09:53:18.379283+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.163:0/413686918' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:53:19.125 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:19 smithi163 bash[75425]: audit 2023-10-09T09:53:18.380415+0000 mon.a (mon.0) 2524 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:53:19.125 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:19 smithi163 bash[75425]: audit 2023-10-09T09:53:18.691238+0000 mon.a (mon.0) 2525 : audit 0 from='client.? 172.21.15.163:0/107983433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:53:19.125 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:19 smithi163 bash[75425]: audit 2023-10-09T09:53:18.993525+0000 mon.a (mon.0) 2526 : audit 1 from='client.? 172.21.15.163:0/3127007731' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:53:19.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:19 smithi163 bash[78107]: audit 2023-10-09T09:53:18.379283+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.163:0/413686918' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:53:19.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:19 smithi163 bash[78107]: audit 2023-10-09T09:53:18.380415+0000 mon.a (mon.0) 2524 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:53:19.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:19 smithi163 bash[78107]: audit 2023-10-09T09:53:18.691238+0000 mon.a (mon.0) 2525 : audit 0 from='client.? 172.21.15.163:0/107983433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:53:19.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:19 smithi163 bash[78107]: audit 2023-10-09T09:53:18.993525+0000 mon.a (mon.0) 2526 : audit 1 from='client.? 172.21.15.163:0/3127007731' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:53:19.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:19.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:19.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:19.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:19.291 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:19.320 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:19.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:19.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:19.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:19.467 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:19.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:19.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:19.596 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:19.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:19.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:19.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:19.732 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:19.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:19.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:19.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:19.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:19.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:19.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:19.966 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:20.002 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:20.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:20.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:20.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:20.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:20.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:20.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:20.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[78107]: cluster 2023-10-09T09:53:18.235923+0000 mgr.y (mgr.34107) 1423 : cluster 0 pgmap v2295: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 976 op/s 2023-10-09T09:53:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[78107]: cluster 2023-10-09T09:53:19.053951+0000 mon.a (mon.0) 2527 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:53:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[78107]: cluster 2023-10-09T09:53:19.072779+0000 osd.7 (osd.7) 617 : cluster 3 failed to encode map e1281 with expected crc 2023-10-09T09:53:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[78107]: cluster 2023-10-09T09:53:19.075718+0000 osd.5 (osd.5) 871 : cluster 3 failed to encode map e1281 with expected crc 2023-10-09T09:53:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[78107]: audit 2023-10-09T09:53:19.078126+0000 mon.a (mon.0) 2528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:53:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[78107]: audit 2023-10-09T09:53:19.078379+0000 mon.a (mon.0) 2529 : audit 1 from='client.? 172.21.15.163:0/3127007731' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:53:20.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[78107]: cluster 2023-10-09T09:53:19.078540+0000 mon.a (mon.0) 2530 : cluster 0 osdmap e1281: 8 total, 8 up, 6 in 2023-10-09T09:53:20.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[75425]: cluster 2023-10-09T09:53:18.235923+0000 mgr.y (mgr.34107) 1423 : cluster 0 pgmap v2295: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 976 op/s 2023-10-09T09:53:20.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[75425]: cluster 2023-10-09T09:53:19.053951+0000 mon.a (mon.0) 2527 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:53:20.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[75425]: cluster 2023-10-09T09:53:19.072779+0000 osd.7 (osd.7) 617 : cluster 3 failed to encode map e1281 with expected crc 2023-10-09T09:53:20.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[75425]: cluster 2023-10-09T09:53:19.075718+0000 osd.5 (osd.5) 871 : cluster 3 failed to encode map e1281 with expected crc 2023-10-09T09:53:20.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[75425]: audit 2023-10-09T09:53:19.078126+0000 mon.a (mon.0) 2528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:53:20.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[75425]: audit 2023-10-09T09:53:19.078379+0000 mon.a (mon.0) 2529 : audit 1 from='client.? 172.21.15.163:0/3127007731' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:53:20.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:20 smithi163 bash[75425]: cluster 2023-10-09T09:53:19.078540+0000 mon.a (mon.0) 2530 : cluster 0 osdmap e1281: 8 total, 8 up, 6 in 2023-10-09T09:53:20.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:20.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:20.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:20.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:20 smithi181 bash[59345]: cluster 2023-10-09T09:53:18.235923+0000 mgr.y (mgr.34107) 1423 : cluster 0 pgmap v2295: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 976 op/s 2023-10-09T09:53:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:20 smithi181 bash[59345]: cluster 2023-10-09T09:53:19.053951+0000 mon.a (mon.0) 2527 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:53:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:20 smithi181 bash[59345]: cluster 2023-10-09T09:53:19.072779+0000 osd.7 (osd.7) 617 : cluster 3 failed to encode map e1281 with expected crc 2023-10-09T09:53:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:20 smithi181 bash[59345]: cluster 2023-10-09T09:53:19.075718+0000 osd.5 (osd.5) 871 : cluster 3 failed to encode map e1281 with expected crc 2023-10-09T09:53:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:20 smithi181 bash[59345]: audit 2023-10-09T09:53:19.078126+0000 mon.a (mon.0) 2528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:53:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:20 smithi181 bash[59345]: audit 2023-10-09T09:53:19.078379+0000 mon.a (mon.0) 2529 : audit 1 from='client.? 172.21.15.163:0/3127007731' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:53:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:20 smithi181 bash[59345]: cluster 2023-10-09T09:53:19.078540+0000 mon.a (mon.0) 2530 : cluster 0 osdmap e1281: 8 total, 8 up, 6 in 2023-10-09T09:53:20.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:20.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:20.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:20.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:20.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:20.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:20.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:20.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:20.630 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:20.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:20.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:20.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:20.763 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:20.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:20.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:20.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:20.884 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:20.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T09:53:20.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:20.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:20.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:21.019 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:21.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:21.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:21.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:21.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:21.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:21.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:21.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:21.287 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:21.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:21.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:21.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:21.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:21.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:21.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:21.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:21.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:21.580 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:21.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:21.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:21.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:21.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:21.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:21.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:21.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:21.855 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:21.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:21.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:21.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:21.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:21.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:21.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:22.089 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:22.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:22 smithi181 bash[59345]: cluster 2023-10-09T09:53:20.236344+0000 mgr.y (mgr.34107) 1424 : cluster 0 pgmap v2297: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 833 op/s 2023-10-09T09:53:22.116 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:22.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:22.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:22.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:22 smithi163 bash[75425]: cluster 2023-10-09T09:53:20.236344+0000 mgr.y (mgr.34107) 1424 : cluster 0 pgmap v2297: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 833 op/s 2023-10-09T09:53:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:22 smithi163 bash[78107]: cluster 2023-10-09T09:53:20.236344+0000 mgr.y (mgr.34107) 1424 : cluster 0 pgmap v2297: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 833 op/s 2023-10-09T09:53:22.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:22.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:22.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:22.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:22.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:22.383 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:22.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:22.487 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:22.529 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:22.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:22.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:22.634 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:22.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:22.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:22.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:22.764 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:22.796 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:22.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:22.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:22.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:22.926 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:22.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:23.030 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:23.058 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:23.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:23.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:23.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:23.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:23.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:23.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:23.295 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T09:53:23.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:23.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:23.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:23.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:23.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:23.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:53:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:53:23] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-09T09:53:23.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:23.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:23.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:23.615 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:23.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:23.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:23.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:23.752 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:23.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:23.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:23.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:23.886 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:23.944 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:23.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:23.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:24.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:24.079 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:24.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:24.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:24.183 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:24.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:24 smithi163 bash[75425]: cluster 2023-10-09T09:53:22.237065+0000 mgr.y (mgr.34107) 1425 : cluster 0 pgmap v2298: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 806 op/s 2023-10-09T09:53:24.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:24 smithi163 bash[78107]: cluster 2023-10-09T09:53:22.237065+0000 mgr.y (mgr.34107) 1425 : cluster 0 pgmap v2298: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 806 op/s 2023-10-09T09:53:24.224 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:24.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:24.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:24.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:24 smithi181 bash[59345]: cluster 2023-10-09T09:53:22.237065+0000 mgr.y (mgr.34107) 1425 : cluster 0 pgmap v2298: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 806 op/s 2023-10-09T09:53:24.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:24.358 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:24.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:24.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:24.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:24.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:24.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:24.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:24.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:24.630 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:24.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:24.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:24.734 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:24.758 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:24.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:24.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:24.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:24.895 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:24.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:24.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:24.999 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:25.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:25.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:25.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:25.130 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:25.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:25.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:25.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:25.263 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:25.287 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:25.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:25.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:25.390 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:25.422 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:25.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:25.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:25.525 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:25.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:25.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:25.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:25.696 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:25.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:25.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:25.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:25.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:25.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:25.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:25.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:53:25.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:53:25.941 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:25.965 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:25.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:26.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:26.108 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:26.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:26.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:26.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:26.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:26.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:26 smithi181 bash[59345]: cluster 2023-10-09T09:53:24.237573+0000 mgr.y (mgr.34107) 1426 : cluster 0 pgmap v2299: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 800 op/s 2023-10-09T09:53:26.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:26 smithi181 bash[59345]: audit 2023-10-09T09:53:25.411307+0000 mon.a (mon.0) 2531 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:26.356 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:26.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:26.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:26 smithi163 bash[78107]: cluster 2023-10-09T09:53:24.237573+0000 mgr.y (mgr.34107) 1426 : cluster 0 pgmap v2299: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 800 op/s 2023-10-09T09:53:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:26 smithi163 bash[78107]: audit 2023-10-09T09:53:25.411307+0000 mon.a (mon.0) 2531 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:26 smithi163 bash[75425]: cluster 2023-10-09T09:53:24.237573+0000 mgr.y (mgr.34107) 1426 : cluster 0 pgmap v2299: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 800 op/s 2023-10-09T09:53:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:26 smithi163 bash[75425]: audit 2023-10-09T09:53:25.411307+0000 mon.a (mon.0) 2531 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:26.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:26.519 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:26.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:26.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:26.628 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:26.658 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:26.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:26.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:26.761 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:26.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:26.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:26.808 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:26.892 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:26.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:26.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:27.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:27.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:27.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:27.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:27.153 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:27.184 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:27.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:27.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:27.287 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:27.318 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:27.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:27.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:27.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:27.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:27.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:27.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:27.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:27.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:27.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:27.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:27.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:27.705 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:27.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:27.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:27.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:27.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:27.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:27.836 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:27.935 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:27.968 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:27.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:27.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:28.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:28.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:28.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:28.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:28.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:28.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:28.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:28.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:28.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:28.356 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:28 smithi181 bash[59345]: cluster 2023-10-09T09:53:26.238079+0000 mgr.y (mgr.34107) 1427 : cluster 0 pgmap v2300: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 797 op/s 2023-10-09T09:53:28.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:28.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:28.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:28 smithi163 bash[78107]: cluster 2023-10-09T09:53:26.238079+0000 mgr.y (mgr.34107) 1427 : cluster 0 pgmap v2300: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 797 op/s 2023-10-09T09:53:28.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:28 smithi163 bash[75425]: cluster 2023-10-09T09:53:26.238079+0000 mgr.y (mgr.34107) 1427 : cluster 0 pgmap v2300: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 797 op/s 2023-10-09T09:53:28.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:28.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:28.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:28.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:28.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:28.647 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:28.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:28.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:28.753 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:28.777 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:28.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:28.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:28.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:28.915 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:28.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:28.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:29.019 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:29.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:29.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:29.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:29.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:29.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:29.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:29.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:29.280 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:29.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:29.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:29.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:29.417 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:29.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:29.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:29.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:29.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:29.582 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:29.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:29.689 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:29.713 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:29.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:29.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:29.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:29.852 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:29.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:29.866 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:29.867 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:29.956 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:29.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:29.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:29.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:30.077 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:30 smithi163 bash[78107]: cluster 2023-10-09T09:53:28.238828+0000 mgr.y (mgr.34107) 1428 : cluster 0 pgmap v2301: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 805 op/s 2023-10-09T09:53:30.077 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:30 smithi163 bash[75425]: cluster 2023-10-09T09:53:28.238828+0000 mgr.y (mgr.34107) 1428 : cluster 0 pgmap v2301: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 805 op/s 2023-10-09T09:53:30.091 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:30.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:30.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:30.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:30.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:30.254 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:30.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:30.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:30 smithi181 bash[59345]: cluster 2023-10-09T09:53:28.238828+0000 mgr.y (mgr.34107) 1428 : cluster 0 pgmap v2301: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 805 op/s 2023-10-09T09:53:30.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:30.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:30.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:30.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:30.497 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:30.533 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:30.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:30.640 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:30.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:30.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:30.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:30.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:30.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:30.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:30.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:30.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:30.945 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:30.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:31.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:31.079 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:31.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:31.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:31.212 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:31.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:31.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:31.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:31.342 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:31.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:31.446 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:31.477 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:31.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:31.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:31.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:31.617 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:31.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:31.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:31.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:31.738 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:31.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:31.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:31.840 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:31.873 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:31.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:31.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:31.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:32.002 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:32.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:32.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:32.111 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:32.119 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:32 smithi163 bash[78107]: cluster 2023-10-09T09:53:30.239329+0000 mgr.y (mgr.34107) 1429 : cluster 0 pgmap v2302: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T09:53:32.119 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:32 smithi163 bash[75425]: cluster 2023-10-09T09:53:30.239329+0000 mgr.y (mgr.34107) 1429 : cluster 0 pgmap v2302: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T09:53:32.132 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:32.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:32.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:32.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:32.267 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:32.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:32.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:32.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:32 smithi181 bash[59345]: cluster 2023-10-09T09:53:30.239329+0000 mgr.y (mgr.34107) 1429 : cluster 0 pgmap v2302: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T09:53:32.374 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:32.398 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:32.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:32.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:32.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:32.534 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:32.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:32.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:32.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:32.666 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:32.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:32.772 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:32.803 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:32.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:32.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:32.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:32.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:32.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:32.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:33.036 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:33.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:33.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:33.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:33.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:33.200 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:33.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:33.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:33.304 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:33.326 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:33.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:33.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:33.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:33.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:33.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:33.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:33.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:33.576 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:53:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:53:33] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-09T09:53:33.594 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:33.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:33.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:33.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:33.733 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:33.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:33.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:33.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:33.864 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:33.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:33.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:33.971 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:34.004 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:34.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:34.108 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:34.115 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:34 smithi163 bash[75425]: cluster 2023-10-09T09:53:32.242449+0000 mgr.y (mgr.34107) 1430 : cluster 0 pgmap v2303: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1k op/s 2023-10-09T09:53:34.115 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:34 smithi163 bash[78107]: cluster 2023-10-09T09:53:32.242449+0000 mgr.y (mgr.34107) 1430 : cluster 0 pgmap v2303: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1k op/s 2023-10-09T09:53:34.134 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:34.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:34.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:34.238 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:34.263 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:34.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:34.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:34.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:34.398 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:34.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:34.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:34.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:34.531 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:34.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:34.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:34 smithi181 bash[59345]: cluster 2023-10-09T09:53:32.242449+0000 mgr.y (mgr.34107) 1430 : cluster 0 pgmap v2303: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1k op/s 2023-10-09T09:53:34.636 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:34.662 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:34.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:34.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:34.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:34.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:34.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:34.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:34.893 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:34.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:34.923 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:34.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:35.024 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:35.055 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:35.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:35.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:35.160 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:35.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:35.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:35.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:35.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:35.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:35.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:35.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:35.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:35.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:35.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:35.568 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:35.603 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:35.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:35.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:35.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:35.734 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:35.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:35.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:35.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:35.869 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:35.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:35.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:35.974 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:35.999 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:36.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:36.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:36.105 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:36.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:36.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:36.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:36.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:36.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:36.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:36.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:36.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:36 smithi181 bash[59345]: cluster 2023-10-09T09:53:34.243152+0000 mgr.y (mgr.34107) 1431 : cluster 0 pgmap v2304: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T09:53:36.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:36.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:36.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:36.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:36.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:36 smithi163 bash[78107]: cluster 2023-10-09T09:53:34.243152+0000 mgr.y (mgr.34107) 1431 : cluster 0 pgmap v2304: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T09:53:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:36 smithi163 bash[75425]: cluster 2023-10-09T09:53:34.243152+0000 mgr.y (mgr.34107) 1431 : cluster 0 pgmap v2304: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T09:53:36.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:36.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:36.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:36.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:36.644 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:36.665 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:36.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:36.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:36.767 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:36.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:36.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:36.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:36.911 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:36.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:36.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:36.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:37.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:37.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:37.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:37.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:37.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:37.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:37.219 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:37.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:37.320 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:37.353 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:37.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:37.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:37.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:37.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:37.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:37.579 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:37.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:37.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:37.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:37.718 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:37.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:37.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:37.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:37.879 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:37.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:37.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:37.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:38.010 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:38.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:38.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:38.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:38 smithi163 bash[75425]: cluster 2023-10-09T09:53:36.243636+0000 mgr.y (mgr.34107) 1432 : cluster 0 pgmap v2305: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T09:53:38.113 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:38.149 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:38.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:38.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:38.253 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:38.263 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:38 smithi181 bash[59345]: cluster 2023-10-09T09:53:36.243636+0000 mgr.y (mgr.34107) 1432 : cluster 0 pgmap v2305: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T09:53:38.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:38.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:38.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:38.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:38.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:38.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:38.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:38 smithi163 bash[78107]: cluster 2023-10-09T09:53:36.243636+0000 mgr.y (mgr.34107) 1432 : cluster 0 pgmap v2305: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T09:53:38.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:38.550 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:38.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:38.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:38.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:38.691 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:38.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:38.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:38.796 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:38.826 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:38.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:38.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:38.932 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:38.964 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:38.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:38.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:39.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:39.099 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:39.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:39.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:39.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:39.233 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:39.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:39.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:39.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:39.367 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:39.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:39.470 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:39.500 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:39.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:39.603 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:39.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:39.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:39.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:39.739 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:39.772 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:39.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:39.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:39.909 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:39.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:39.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:40.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:40.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:40.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:40.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:40.146 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:40.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:40.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:40.291 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:40.301 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:40 smithi181 bash[59345]: cluster 2023-10-09T09:53:38.244195+0000 mgr.y (mgr.34107) 1433 : cluster 0 pgmap v2306: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 958 op/s 2023-10-09T09:53:40.321 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:40.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:40.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:40.424 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:40.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:40.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:40.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:40.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:40 smithi163 bash[75425]: cluster 2023-10-09T09:53:38.244195+0000 mgr.y (mgr.34107) 1433 : cluster 0 pgmap v2306: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 958 op/s 2023-10-09T09:53:40.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:40 smithi163 bash[78107]: cluster 2023-10-09T09:53:38.244195+0000 mgr.y (mgr.34107) 1433 : cluster 0 pgmap v2306: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 958 op/s 2023-10-09T09:53:40.551 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:40.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:40.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:40.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:40.679 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:40.703 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:40.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:40.806 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:40.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:40.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:40.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:40.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:40.966 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:40.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:40.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:41.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:41.096 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:41.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:41.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:41.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:41.235 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:41.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:41.344 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:41.366 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:41 smithi163 bash[75425]: audit 2023-10-09T09:53:40.415553+0000 mon.a (mon.0) 2532 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:41.368 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:41 smithi163 bash[78107]: audit 2023-10-09T09:53:40.415553+0000 mon.a (mon.0) 2532 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:41.374 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:41.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:41.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:41.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:41.488 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:41 smithi181 bash[59345]: audit 2023-10-09T09:53:40.415553+0000 mon.a (mon.0) 2532 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:41.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:41.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:41.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:41.611 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:41.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:41.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:41.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:41.745 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:41.769 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:41.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:41.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:41.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:41.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:41.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:42.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:42.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:42.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:42.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:42.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:42.145 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:42 smithi163 bash[75425]: cluster 2023-10-09T09:53:40.244810+0000 mgr.y (mgr.34107) 1434 : cluster 0 pgmap v2307: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T09:53:42.145 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:42 smithi163 bash[78107]: cluster 2023-10-09T09:53:40.244810+0000 mgr.y (mgr.34107) 1434 : cluster 0 pgmap v2307: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T09:53:42.176 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:42.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:42.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:42.278 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:42.289 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:42 smithi181 bash[59345]: cluster 2023-10-09T09:53:40.244810+0000 mgr.y (mgr.34107) 1434 : cluster 0 pgmap v2307: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T09:53:42.303 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:42.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:42.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:42.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:42.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:42.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:42.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:42.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:42.571 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:42.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:42.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:42.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:42.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:42.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:42.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:42.799 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:42.824 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:42.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:42.929 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:42.956 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:42.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:43.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:43.102 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:43.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:43.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:43.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:43.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:43.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:43.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:43.342 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:43.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:43.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:43.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:43.473 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:43.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:43.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:43.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:43.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:43.647 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:43.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:43.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:43.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:53:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:53:43] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-09T09:53:43.751 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:43.784 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:43.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:43.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:43.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:43.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:43.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:43.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:43.985 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:53:43.985 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:53:44.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:44.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:44.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:44.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:44.173 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:44.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:44.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:44.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:44.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:44 smithi163 bash[75425]: cluster 2023-10-09T09:53:42.245538+0000 mgr.y (mgr.34107) 1435 : cluster 0 pgmap v2308: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 990 op/s 2023-10-09T09:53:44.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:44 smithi163 bash[78107]: cluster 2023-10-09T09:53:42.245538+0000 mgr.y (mgr.34107) 1435 : cluster 0 pgmap v2308: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 990 op/s 2023-10-09T09:53:44.447 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:44.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:44.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:44.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:44.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:44 smithi181 bash[59345]: cluster 2023-10-09T09:53:42.245538+0000 mgr.y (mgr.34107) 1435 : cluster 0 pgmap v2308: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 990 op/s 2023-10-09T09:53:44.587 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:44.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:44.691 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:44.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:44.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:44.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:44.827 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:44.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:44.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:44.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:44.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:44.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:44.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:44.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:45.099 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:45.138 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T09:53:45.139 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:45.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:45.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:45.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:45.268 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:45.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:45.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:45.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:45.380 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:45 smithi163 bash[75425]: audit 2023-10-09T09:53:44.693102+0000 mon.c (mon.2) 134 : audit 1 from='client.? 172.21.15.163:0/462021171' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:53:45.380 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:45 smithi163 bash[75425]: audit 2023-10-09T09:53:44.693731+0000 mon.a (mon.0) 2533 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:53:45.381 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:45 smithi163 bash[78107]: audit 2023-10-09T09:53:44.693102+0000 mon.c (mon.2) 134 : audit 1 from='client.? 172.21.15.163:0/462021171' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:53:45.381 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:45 smithi163 bash[78107]: audit 2023-10-09T09:53:44.693731+0000 mon.a (mon.0) 2533 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:53:45.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:45.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:45.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:45.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:45.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:45 smithi181 bash[59345]: audit 2023-10-09T09:53:44.693102+0000 mon.c (mon.2) 134 : audit 1 from='client.? 172.21.15.163:0/462021171' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:53:45.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:45 smithi181 bash[59345]: audit 2023-10-09T09:53:44.693731+0000 mon.a (mon.0) 2533 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:53:45.547 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:45.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:45.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:45.654 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:45.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:45.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:45.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:45.814 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:45.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:45.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:45.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:45.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:45.959 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:45.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:46.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:46.092 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:46.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:46.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:46.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:46.219 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:46.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:46.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:46.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:46.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:46.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:46.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:46 smithi163 bash[75425]: cluster 2023-10-09T09:53:44.246432+0000 mgr.y (mgr.34107) 1436 : cluster 0 pgmap v2309: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 683 op/s 2023-10-09T09:53:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:46 smithi163 bash[75425]: cluster 2023-10-09T09:53:45.117983+0000 osd.5 (osd.5) 872 : cluster 3 failed to encode map e1282 with expected crc 2023-10-09T09:53:46.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:46 smithi163 bash[75425]: cluster 2023-10-09T09:53:45.120064+0000 osd.6 (osd.6) 562 : cluster 3 failed to encode map e1282 with expected crc 2023-10-09T09:53:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:46 smithi163 bash[75425]: audit 2023-10-09T09:53:45.126828+0000 mon.a (mon.0) 2534 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:53:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:46 smithi163 bash[75425]: cluster 2023-10-09T09:53:45.127049+0000 mon.a (mon.0) 2535 : cluster 0 osdmap e1282: 8 total, 8 up, 6 in 2023-10-09T09:53:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:46 smithi163 bash[78107]: cluster 2023-10-09T09:53:44.246432+0000 mgr.y (mgr.34107) 1436 : cluster 0 pgmap v2309: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 683 op/s 2023-10-09T09:53:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:46 smithi163 bash[78107]: cluster 2023-10-09T09:53:45.117983+0000 osd.5 (osd.5) 872 : cluster 3 failed to encode map e1282 with expected crc 2023-10-09T09:53:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:46 smithi163 bash[78107]: cluster 2023-10-09T09:53:45.120064+0000 osd.6 (osd.6) 562 : cluster 3 failed to encode map e1282 with expected crc 2023-10-09T09:53:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:46 smithi163 bash[78107]: audit 2023-10-09T09:53:45.126828+0000 mon.a (mon.0) 2534 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:53:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:46 smithi163 bash[78107]: cluster 2023-10-09T09:53:45.127049+0000 mon.a (mon.0) 2535 : cluster 0 osdmap e1282: 8 total, 8 up, 6 in 2023-10-09T09:53:46.480 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:46.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:46.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:46 smithi181 bash[59345]: cluster 2023-10-09T09:53:44.246432+0000 mgr.y (mgr.34107) 1436 : cluster 0 pgmap v2309: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 683 op/s 2023-10-09T09:53:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:46 smithi181 bash[59345]: cluster 2023-10-09T09:53:45.117983+0000 osd.5 (osd.5) 872 : cluster 3 failed to encode map e1282 with expected crc 2023-10-09T09:53:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:46 smithi181 bash[59345]: cluster 2023-10-09T09:53:45.120064+0000 osd.6 (osd.6) 562 : cluster 3 failed to encode map e1282 with expected crc 2023-10-09T09:53:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:46 smithi181 bash[59345]: audit 2023-10-09T09:53:45.126828+0000 mon.a (mon.0) 2534 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:53:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:46 smithi181 bash[59345]: cluster 2023-10-09T09:53:45.127049+0000 mon.a (mon.0) 2535 : cluster 0 osdmap e1282: 8 total, 8 up, 6 in 2023-10-09T09:53:46.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:46.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:46.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:46.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:46.725 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:46.749 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:46.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:46.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:46.855 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:46.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:46.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:46.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:47.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:47.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:47.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:47.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:47.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:47.174 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:47.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:47.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:47.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:47 smithi163 bash[75425]: cluster 2023-10-09T09:53:46.246877+0000 mgr.y (mgr.34107) 1437 : cluster 0 pgmap v2311: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T09:53:47.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:47 smithi163 bash[78107]: cluster 2023-10-09T09:53:46.246877+0000 mgr.y (mgr.34107) 1437 : cluster 0 pgmap v2311: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T09:53:47.277 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:47.285 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:47 smithi181 bash[59345]: cluster 2023-10-09T09:53:46.246877+0000 mgr.y (mgr.34107) 1437 : cluster 0 pgmap v2311: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 786 op/s 2023-10-09T09:53:47.308 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:47.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:47.415 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:47.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:47.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:47.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:47.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:47.579 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:47.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:47.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:47.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:47.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:47.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:47.825 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:47.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:47.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:47.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:47.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:47.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:47.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:47.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:48.086 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:48.116 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:48.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:48.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:48.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:48.263 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:48.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:48.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:48.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:48.407 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:48.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:48.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:48.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:48.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:48.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:48.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:48.679 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:48.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:48.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:48.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:48.806 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:48.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:48.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:48.904 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T09:53:48.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:48.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T09:53:48.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:48.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:48.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:49.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:49.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:49.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:49.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:49.225 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:49.256 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:49.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:49.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:49.308 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:49 smithi163 bash[78107]: cluster 2023-10-09T09:53:48.247509+0000 mgr.y (mgr.34107) 1438 : cluster 0 pgmap v2312: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 770 op/s 2023-10-09T09:53:49.308 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:49 smithi163 bash[78107]: audit 2023-10-09T09:53:49.251096+0000 mon.a (mon.0) 2536 : audit 0 from='client.? 172.21.15.163:0/401097707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:53:49.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:49 smithi163 bash[75425]: cluster 2023-10-09T09:53:48.247509+0000 mgr.y (mgr.34107) 1438 : cluster 0 pgmap v2312: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 770 op/s 2023-10-09T09:53:49.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:49 smithi163 bash[75425]: audit 2023-10-09T09:53:49.251096+0000 mon.a (mon.0) 2536 : audit 0 from='client.? 172.21.15.163:0/401097707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:53:49.310 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:53:49.359 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:53:49.360 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:49.385 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:49.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:49.409 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:53:49.451 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:53:49.488 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:49.490 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:53:49.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:49 smithi181 bash[59345]: cluster 2023-10-09T09:53:48.247509+0000 mgr.y (mgr.34107) 1438 : cluster 0 pgmap v2312: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 770 op/s 2023-10-09T09:53:49.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:49 smithi181 bash[59345]: audit 2023-10-09T09:53:49.251096+0000 mon.a (mon.0) 2536 : audit 0 from='client.? 172.21.15.163:0/401097707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:53:49.513 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:49.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:49.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:49.522 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:53:49.553 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:53:49.585 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:53:49.587 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:53:49.587 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:53:49.587 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:53:49.588 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:53:49.588 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:53:49.588 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:53:49.588 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:53:49.588 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:53:49.616 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:49.646 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:49.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:49.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:49.749 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:49.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:49.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:49.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:49.889 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:49.918 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:49.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:49.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:50.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:50.055 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:50.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:50.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:50.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:50.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:50.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:50.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:50.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:50.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:50.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:50.338 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:53:50.427 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:50.452 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:50.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:50.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:50.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:50 smithi181 bash[59345]: audit 2023-10-09T09:53:49.469990+0000 mon.b (mon.1) 31 : audit 1 from='client.? 172.21.15.163:0/4140699796' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:53:50.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:50 smithi181 bash[59345]: audit 2023-10-09T09:53:49.470751+0000 mon.a (mon.0) 2537 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:53:50.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:50.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:50.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:50.607 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:50 smithi163 bash[78107]: audit 2023-10-09T09:53:49.469990+0000 mon.b (mon.1) 31 : audit 1 from='client.? 172.21.15.163:0/4140699796' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:53:50.607 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:50 smithi163 bash[78107]: audit 2023-10-09T09:53:49.470751+0000 mon.a (mon.0) 2537 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:53:50.607 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:50 smithi163 bash[75425]: audit 2023-10-09T09:53:49.469990+0000 mon.b (mon.1) 31 : audit 1 from='client.? 172.21.15.163:0/4140699796' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:53:50.608 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:50 smithi163 bash[75425]: audit 2023-10-09T09:53:49.470751+0000 mon.a (mon.0) 2537 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:53:50.701 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:50.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:50.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:50.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:50.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:50.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:50.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:50.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:50.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:50.997 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:50.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:50.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:51.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:51.141 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:51.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:51.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:51.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:51.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:51.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:51.383 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:51.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[75425]: cluster 2023-10-09T09:53:50.247990+0000 mgr.y (mgr.34107) 1439 : cluster 0 pgmap v2313: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T09:53:51.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[75425]: cluster 2023-10-09T09:53:50.307035+0000 mon.a (mon.0) 2538 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:53:51.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[75425]: cluster 2023-10-09T09:53:50.307071+0000 mon.a (mon.0) 2539 : cluster 1 Cluster is now healthy 2023-10-09T09:53:51.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[75425]: cluster 2023-10-09T09:53:50.324628+0000 osd.5 (osd.5) 873 : cluster 3 failed to encode map e1283 with expected crc 2023-10-09T09:53:51.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[75425]: audit 2023-10-09T09:53:50.329533+0000 mon.a (mon.0) 2540 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:53:51.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[75425]: cluster 2023-10-09T09:53:50.333053+0000 mon.a (mon.0) 2541 : cluster 0 osdmap e1283: 8 total, 8 up, 6 in 2023-10-09T09:53:51.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[75425]: cluster 2023-10-09T09:53:50.349432+0000 osd.4 (osd.4) 672 : cluster 3 failed to encode map e1283 with expected crc 2023-10-09T09:53:51.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[75425]: cluster 2023-10-09T09:53:50.539586+0000 osd.7 (osd.7) 618 : cluster 3 failed to encode map e1283 with expected crc 2023-10-09T09:53:51.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[78107]: cluster 2023-10-09T09:53:50.247990+0000 mgr.y (mgr.34107) 1439 : cluster 0 pgmap v2313: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T09:53:51.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[78107]: cluster 2023-10-09T09:53:50.307035+0000 mon.a (mon.0) 2538 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:53:51.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[78107]: cluster 2023-10-09T09:53:50.307071+0000 mon.a (mon.0) 2539 : cluster 1 Cluster is now healthy 2023-10-09T09:53:51.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[78107]: cluster 2023-10-09T09:53:50.324628+0000 osd.5 (osd.5) 873 : cluster 3 failed to encode map e1283 with expected crc 2023-10-09T09:53:51.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[78107]: audit 2023-10-09T09:53:50.329533+0000 mon.a (mon.0) 2540 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:53:51.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[78107]: cluster 2023-10-09T09:53:50.333053+0000 mon.a (mon.0) 2541 : cluster 0 osdmap e1283: 8 total, 8 up, 6 in 2023-10-09T09:53:51.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[78107]: cluster 2023-10-09T09:53:50.349432+0000 osd.4 (osd.4) 672 : cluster 3 failed to encode map e1283 with expected crc 2023-10-09T09:53:51.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:51 smithi163 bash[78107]: cluster 2023-10-09T09:53:50.539586+0000 osd.7 (osd.7) 618 : cluster 3 failed to encode map e1283 with expected crc 2023-10-09T09:53:51.415 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:51.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:51.533 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:51.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:51 smithi181 bash[59345]: cluster 2023-10-09T09:53:50.247990+0000 mgr.y (mgr.34107) 1439 : cluster 0 pgmap v2313: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T09:53:51.577 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:51 smithi181 bash[59345]: cluster 2023-10-09T09:53:50.307035+0000 mon.a (mon.0) 2538 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:53:51.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:51 smithi181 bash[59345]: cluster 2023-10-09T09:53:50.307071+0000 mon.a (mon.0) 2539 : cluster 1 Cluster is now healthy 2023-10-09T09:53:51.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:51 smithi181 bash[59345]: cluster 2023-10-09T09:53:50.324628+0000 osd.5 (osd.5) 873 : cluster 3 failed to encode map e1283 with expected crc 2023-10-09T09:53:51.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:51 smithi181 bash[59345]: audit 2023-10-09T09:53:50.329533+0000 mon.a (mon.0) 2540 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:53:51.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:51 smithi181 bash[59345]: cluster 2023-10-09T09:53:50.333053+0000 mon.a (mon.0) 2541 : cluster 0 osdmap e1283: 8 total, 8 up, 6 in 2023-10-09T09:53:51.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:51 smithi181 bash[59345]: cluster 2023-10-09T09:53:50.349432+0000 osd.4 (osd.4) 672 : cluster 3 failed to encode map e1283 with expected crc 2023-10-09T09:53:51.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:51 smithi181 bash[59345]: cluster 2023-10-09T09:53:50.539586+0000 osd.7 (osd.7) 618 : cluster 3 failed to encode map e1283 with expected crc 2023-10-09T09:53:51.587 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:51.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:51.689 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:51.714 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:51.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:51.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:51.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:51.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:51.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:51.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:51.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:51.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:51.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:52.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:52.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:52.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:52.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:52.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:52.265 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:52.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:52.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:52.373 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:52.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:52.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:52.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:52.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:52.544 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:52.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:52.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:52.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:52.672 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:52.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:52.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:52.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:52.807 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:52.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:52.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:52.913 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:52.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:52.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:52.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:53.048 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:53.073 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:53.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:53.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:53.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:53.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:53.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:53.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:53.311 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:53.320 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:53:53.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:53.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:53.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:53.439 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:53.449 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:53:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:53:53] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-09T09:53:53.449 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:53 smithi163 bash[75425]: cluster 2023-10-09T09:53:52.248701+0000 mgr.y (mgr.34107) 1440 : cluster 0 pgmap v2315: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 923 op/s 2023-10-09T09:53:53.450 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:53 smithi163 bash[78107]: cluster 2023-10-09T09:53:52.248701+0000 mgr.y (mgr.34107) 1440 : cluster 0 pgmap v2315: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 923 op/s 2023-10-09T09:53:53.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:53.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:53.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:53.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:53.620 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:53.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:53.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:53.700 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:53:53.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:53.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:53 smithi181 bash[59345]: cluster 2023-10-09T09:53:52.248701+0000 mgr.y (mgr.34107) 1440 : cluster 0 pgmap v2315: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 923 op/s 2023-10-09T09:53:53.759 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:53.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:53.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:53.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:53.900 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:53.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:53.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:54.008 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:54.039 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:54.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:54.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:54.145 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:54.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:54.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:54.287 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:54.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:54.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:54.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:53:54.419 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:54.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:54.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:54.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:54.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:54.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:54.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:54.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:54.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:54.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:54.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:54.811 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:54.835 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:54.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:54.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:54.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:54.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:54.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:55.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:55.094 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:55.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:55.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:55.198 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:55.221 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:55.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:55.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:55.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:55.355 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:55.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:55.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:55.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:55.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:55 smithi163 bash[78107]: cluster 2023-10-09T09:53:54.249213+0000 mgr.y (mgr.34107) 1441 : cluster 0 pgmap v2316: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 835 op/s 2023-10-09T09:53:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:55 smithi163 bash[75425]: cluster 2023-10-09T09:53:54.249213+0000 mgr.y (mgr.34107) 1441 : cluster 0 pgmap v2316: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 835 op/s 2023-10-09T09:53:55.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:55.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:55.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:55.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:55 smithi181 bash[59345]: cluster 2023-10-09T09:53:54.249213+0000 mgr.y (mgr.34107) 1441 : cluster 0 pgmap v2316: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 835 op/s 2023-10-09T09:53:55.598 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:55.624 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:55.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:55.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:55.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:55.762 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:55.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:55.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:55.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:55.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:55.907 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:55.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:55.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:56.014 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:56.040 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:56.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:56.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:56.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:56.184 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:56.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:56.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:56.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:56.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:56.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:56.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:56.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:56.426 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:56 smithi163 bash[78107]: audit 2023-10-09T09:53:55.420420+0000 mon.a (mon.0) 2542 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:56.426 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:56 smithi163 bash[78107]: audit 2023-10-09T09:53:55.421932+0000 mon.a (mon.0) 2543 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:56.446 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:56.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:56.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:56.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:56.557 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:56 smithi181 bash[59345]: audit 2023-10-09T09:53:55.420420+0000 mon.a (mon.0) 2542 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:56.557 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:56 smithi181 bash[59345]: audit 2023-10-09T09:53:55.421932+0000 mon.a (mon.0) 2543 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:56.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:56.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:56.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:56.679 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:56.687 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:56 smithi163 bash[75425]: audit 2023-10-09T09:53:55.420420+0000 mon.a (mon.0) 2542 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:53:56.687 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:56 smithi163 bash[75425]: audit 2023-10-09T09:53:55.421932+0000 mon.a (mon.0) 2543 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:56.705 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:53:56.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:56.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:53:56.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:56.846 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:56.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:56.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:56.948 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:56.973 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:56.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:56.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:57.083 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:57.112 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:57.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:57.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:57.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:57.305 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:57.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:57.409 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:57.431 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:57.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:57.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:57.789 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:58.000 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:58.031 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:58.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:58.135 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:58.158 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:58.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:58.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:58.262 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:58.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:57 smithi181 bash[59345]: cluster 2023-10-09T09:53:56.249777+0000 mgr.y (mgr.34107) 1442 : cluster 0 pgmap v2317: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T09:53:58.285 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:58.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:58.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:58.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:58.400 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:53:57 smithi163 bash[75425]: cluster 2023-10-09T09:53:56.249777+0000 mgr.y (mgr.34107) 1442 : cluster 0 pgmap v2317: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T09:53:58.400 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:53:57 smithi163 bash[78107]: cluster 2023-10-09T09:53:56.249777+0000 mgr.y (mgr.34107) 1442 : cluster 0 pgmap v2317: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T09:53:58.416 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:53:58.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:58.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:53:58.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:58.550 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:53:58.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:58.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:53:58.570 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:53:58.570 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:53:58.570 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T09:53:58.570 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:53:58.654 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:58.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:58.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:58.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:58.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:58.818 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:58.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:58.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:58.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:58.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:58.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:58.973 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:53:59.053 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:59.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:53:59.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:59.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:53:59.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:59.227 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:59.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:59.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:59.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:59.366 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:59.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:59.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:53:59.469 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:59.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:53:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:59.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:53:59.532 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T09:53:59.602 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:59.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:53:59.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:59.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:53:59.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:59.772 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:53:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:59.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:53:59.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:59.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:53:59.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:53:59.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:00.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:00.035 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:00.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:00.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:53:59 smithi181 bash[59345]: cluster 2023-10-09T09:53:58.250356+0000 mgr.y (mgr.34107) 1443 : cluster 0 pgmap v2318: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 810 op/s 2023-10-09T09:54:00.143 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:00.156 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:00 smithi163 bash[75425]: cluster 2023-10-09T09:53:58.250356+0000 mgr.y (mgr.34107) 1443 : cluster 0 pgmap v2318: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 810 op/s 2023-10-09T09:54:00.157 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:00 smithi163 bash[78107]: cluster 2023-10-09T09:53:58.250356+0000 mgr.y (mgr.34107) 1443 : cluster 0 pgmap v2318: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 810 op/s 2023-10-09T09:54:00.180 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:00.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:00.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:00.286 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:00.314 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:00.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:00.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:00.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:00.449 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:00.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:00.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:00.553 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:00.591 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:00.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:00.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:00.700 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:00.723 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:00.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:00.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:00.827 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:00.854 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:00.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:00.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:00.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:00.983 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:00.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:01.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:01.112 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:01.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:01.216 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:00 smithi163 bash[75425]: audit 2023-10-09T09:54:00.378162+0000 mon.a (mon.0) 2544 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:01.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:00 smithi163 bash[78107]: audit 2023-10-09T09:54:00.378162+0000 mon.a (mon.0) 2544 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:01.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:01.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:01.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:00 smithi181 bash[59345]: audit 2023-10-09T09:54:00.378162+0000 mon.a (mon.0) 2544 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:01.368 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:01.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:01.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:01.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:01.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:01.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:01.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:01.632 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:01.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:01.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:01.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:01.762 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:01.790 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:01.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:01.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:01.894 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:01.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:01.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:02.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:02.037 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:02 smithi163 bash[78107]: cluster 2023-10-09T09:54:00.250911+0000 mgr.y (mgr.34107) 1444 : cluster 0 pgmap v2319: 89 pgs: 89 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 809 op/s 2023-10-09T09:54:02.037 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:02 smithi163 bash[75425]: cluster 2023-10-09T09:54:00.250911+0000 mgr.y (mgr.34107) 1444 : cluster 0 pgmap v2319: 89 pgs: 89 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 809 op/s 2023-10-09T09:54:02.062 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:02.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:02.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:02.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:02.196 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:02.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:02.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:02.300 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:02.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:01 smithi181 bash[59345]: cluster 2023-10-09T09:54:00.250911+0000 mgr.y (mgr.34107) 1444 : cluster 0 pgmap v2319: 89 pgs: 89 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 809 op/s 2023-10-09T09:54:02.326 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:02.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:02.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:02.429 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:02.454 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:02.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:02.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:02.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:02.585 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:02.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:02.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:02.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:02.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:02.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:02.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:02.826 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:02.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:02.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:02.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:02.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:02.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:02.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:02.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:03.089 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:03.113 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:03.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:03.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:03.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:03.235 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:03.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:03.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:03.338 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:03.362 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:03.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:03.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:03.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:03.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:03.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:03.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:03.598 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:03.627 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:03.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:03.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:03.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:54:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:54:03] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.33.4" 2023-10-09T09:54:03.731 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:03.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:03.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:03.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:03.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:03.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:03.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:03.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:04.001 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:04 smithi181 bash[59345]: cluster 2023-10-09T09:54:02.252940+0000 mgr.y (mgr.34107) 1445 : cluster 0 pgmap v2320: 89 pgs: 89 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 970 op/s 2023-10-09T09:54:04.002 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:04 smithi163 bash[78107]: cluster 2023-10-09T09:54:02.252940+0000 mgr.y (mgr.34107) 1445 : cluster 0 pgmap v2320: 89 pgs: 89 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 970 op/s 2023-10-09T09:54:04.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:04 smithi163 bash[75425]: cluster 2023-10-09T09:54:02.252940+0000 mgr.y (mgr.34107) 1445 : cluster 0 pgmap v2320: 89 pgs: 89 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 970 op/s 2023-10-09T09:54:04.015 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:04.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:04.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:04.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:04.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:04.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:04.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:04.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:04.295 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:04.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:04.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:04.397 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:04.403 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:54:04.423 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:04.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:04.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:04.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:04.579 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:04.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:04.683 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:04.714 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:04.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:04.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:04.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:04.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:04.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:04.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:04.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:04.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:04.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:05.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:05.110 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:05.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:05.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:05.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:05.247 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:05.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:05.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:05.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:05.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:05.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:05.490 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:05.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:05.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:05.625 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:05.656 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:05.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:05.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:05.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:05.790 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:05.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:05.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:05.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:05.925 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:05.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:05.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:06.029 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:06.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:06 smithi163 bash[75425]: cluster 2023-10-09T09:54:04.253630+0000 mgr.y (mgr.34107) 1446 : cluster 0 pgmap v2321: 89 pgs: 89 active+clean; 1.5 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T09:54:06.040 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:06 smithi163 bash[78107]: cluster 2023-10-09T09:54:04.253630+0000 mgr.y (mgr.34107) 1446 : cluster 0 pgmap v2321: 89 pgs: 89 active+clean; 1.5 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T09:54:06.064 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:06.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:06.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:06.206 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:06.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:06.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:06.307 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:06 smithi181 bash[59345]: cluster 2023-10-09T09:54:04.253630+0000 mgr.y (mgr.34107) 1446 : cluster 0 pgmap v2321: 89 pgs: 89 active+clean; 1.5 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T09:54:06.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:06.344 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:06.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:06.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:06.453 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:06.478 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:06.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:06.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:06.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:06.604 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:06.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:06.710 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:06.733 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:06.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:06.836 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:06.865 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:06.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:06.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:06.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:06.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:07.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:07.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:07.131 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:07.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:07.238 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:07.260 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:07.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:07.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:07 smithi181 bash[59345]: audit 2023-10-09T09:54:06.025634+0000 mon.a (mon.0) 2545 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:07.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:07 smithi181 bash[59345]: audit 2023-10-09T09:54:06.037502+0000 mon.a (mon.0) 2546 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:07.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:07.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:07.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:07 smithi163 bash[75425]: audit 2023-10-09T09:54:06.025634+0000 mon.a (mon.0) 2545 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:07 smithi163 bash[75425]: audit 2023-10-09T09:54:06.037502+0000 mon.a (mon.0) 2546 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:07 smithi163 bash[78107]: audit 2023-10-09T09:54:06.025634+0000 mon.a (mon.0) 2545 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:07.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:07 smithi163 bash[78107]: audit 2023-10-09T09:54:06.037502+0000 mon.a (mon.0) 2546 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:07.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:07.538 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:07.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:07.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:07.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:07.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:07.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:07.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:07.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:07.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:07.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:07.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:07.897 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:07.925 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:07.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:07.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:08.030 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:08.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:08 smithi181 bash[59345]: cluster 2023-10-09T09:54:06.254126+0000 mgr.y (mgr.34107) 1447 : cluster 0 pgmap v2322: 89 pgs: 89 active+clean; 1.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T09:54:08.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:08.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:08.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:08.160 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:08.189 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:08.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:08.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:08 smithi163 bash[78107]: cluster 2023-10-09T09:54:06.254126+0000 mgr.y (mgr.34107) 1447 : cluster 0 pgmap v2322: 89 pgs: 89 active+clean; 1.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T09:54:08.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:08 smithi163 bash[75425]: cluster 2023-10-09T09:54:06.254126+0000 mgr.y (mgr.34107) 1447 : cluster 0 pgmap v2322: 89 pgs: 89 active+clean; 1.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T09:54:08.292 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:08.329 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:08.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:08.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:08.438 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:08.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:08.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:08.577 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:08.600 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:08.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:08.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:08.708 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:08.725 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:08.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:08.828 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:08.854 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:08.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:08.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:08.958 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:08.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:08.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:08.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:09.002 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:54:09.002 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:54:09.002 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:54:09.002 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:54:09.002 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:54:09.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:09.126 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:09.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:09.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:09.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:09.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:09.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:09.397 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:09.426 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:09.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:09.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:09.529 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:09.549 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:09.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:09.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:09.654 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:09.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:09.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:09.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:09.803 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:09.871 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:09.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:09.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:09.973 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T09:54:09.982 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:10.021 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:54:10.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:10.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:10.054 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T09:54:10.085 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:54:10.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:10.165 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:54:10.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:10.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:10.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:10.204 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:54:10.243 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:54:10.275 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:10.287 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:10 smithi181 bash[59345]: cluster 2023-10-09T09:54:08.254654+0000 mgr.y (mgr.34107) 1448 : cluster 0 pgmap v2323: 89 pgs: 89 active+clean; 1.2 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 892 op/s 2023-10-09T09:54:10.287 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:10 smithi181 bash[59345]: audit 2023-10-09T09:54:09.427087+0000 mon.a (mon.0) 2547 : audit 1 from='client.? 172.21.15.163:0/2072186209' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:54:10.287 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:10 smithi181 bash[59345]: audit 2023-10-09T09:54:09.968306+0000 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.163:0/1918113897' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:54:10.288 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:54:10.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:10.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:10.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:10.318 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:54:10.354 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:54:10.391 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:54:10.391 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:54:10.391 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:54:10.391 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:54:10.391 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:54:10.391 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:54:10.391 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:54:10.392 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:54:10.408 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:10.421 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:10 smithi163 bash[75425]: cluster 2023-10-09T09:54:08.254654+0000 mgr.y (mgr.34107) 1448 : cluster 0 pgmap v2323: 89 pgs: 89 active+clean; 1.2 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 892 op/s 2023-10-09T09:54:10.421 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:10 smithi163 bash[75425]: audit 2023-10-09T09:54:09.427087+0000 mon.a (mon.0) 2547 : audit 1 from='client.? 172.21.15.163:0/2072186209' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:54:10.421 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:10 smithi163 bash[75425]: audit 2023-10-09T09:54:09.968306+0000 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.163:0/1918113897' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:54:10.422 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:10 smithi163 bash[78107]: cluster 2023-10-09T09:54:08.254654+0000 mgr.y (mgr.34107) 1448 : cluster 0 pgmap v2323: 89 pgs: 89 active+clean; 1.2 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 892 op/s 2023-10-09T09:54:10.422 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:10 smithi163 bash[78107]: audit 2023-10-09T09:54:09.427087+0000 mon.a (mon.0) 2547 : audit 1 from='client.? 172.21.15.163:0/2072186209' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:54:10.422 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:10 smithi163 bash[78107]: audit 2023-10-09T09:54:09.968306+0000 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.163:0/1918113897' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:54:10.448 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:10.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:10.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:10.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:10.582 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:10.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:10.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:10.689 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:10.713 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:10.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:10.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:10.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:10.841 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:10.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:10.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:10.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:10.979 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:10.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:11.082 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:54:11.085 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:11.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:11.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:11.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[78107]: cluster 2023-10-09T09:54:10.042574+0000 osd.5 (osd.5) 874 : cluster 3 failed to encode map e1284 with expected crc 2023-10-09T09:54:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[78107]: cluster 2023-10-09T09:54:10.042832+0000 osd.6 (osd.6) 563 : cluster 3 failed to encode map e1284 with expected crc 2023-10-09T09:54:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[78107]: audit 2023-10-09T09:54:10.045307+0000 mon.a (mon.0) 2548 : audit 1 from='client.? 172.21.15.163:0/2072186209' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:54:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[78107]: cluster 2023-10-09T09:54:10.045378+0000 mon.a (mon.0) 2549 : cluster 0 osdmap e1284: 8 total, 8 up, 6 in 2023-10-09T09:54:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[78107]: audit 2023-10-09T09:54:10.416712+0000 mon.a (mon.0) 2550 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[78107]: audit 2023-10-09T09:54:10.607321+0000 mon.a (mon.0) 2551 : audit 1 from='client.? 172.21.15.163:0/3405680545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:54:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[78107]: audit 2023-10-09T09:54:10.679080+0000 mon.a (mon.0) 2552 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[78107]: audit 2023-10-09T09:54:10.687052+0000 mon.a (mon.0) 2553 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[75425]: cluster 2023-10-09T09:54:10.042574+0000 osd.5 (osd.5) 874 : cluster 3 failed to encode map e1284 with expected crc 2023-10-09T09:54:11.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[75425]: cluster 2023-10-09T09:54:10.042832+0000 osd.6 (osd.6) 563 : cluster 3 failed to encode map e1284 with expected crc 2023-10-09T09:54:11.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[75425]: audit 2023-10-09T09:54:10.045307+0000 mon.a (mon.0) 2548 : audit 1 from='client.? 172.21.15.163:0/2072186209' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:54:11.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[75425]: cluster 2023-10-09T09:54:10.045378+0000 mon.a (mon.0) 2549 : cluster 0 osdmap e1284: 8 total, 8 up, 6 in 2023-10-09T09:54:11.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[75425]: audit 2023-10-09T09:54:10.416712+0000 mon.a (mon.0) 2550 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:11.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[75425]: audit 2023-10-09T09:54:10.607321+0000 mon.a (mon.0) 2551 : audit 1 from='client.? 172.21.15.163:0/3405680545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:54:11.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[75425]: audit 2023-10-09T09:54:10.679080+0000 mon.a (mon.0) 2552 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:11.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:11 smithi163 bash[75425]: audit 2023-10-09T09:54:10.687052+0000 mon.a (mon.0) 2553 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:11.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:11.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:11 smithi181 bash[59345]: cluster 2023-10-09T09:54:10.042574+0000 osd.5 (osd.5) 874 : cluster 3 failed to encode map e1284 with expected crc 2023-10-09T09:54:11.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:11 smithi181 bash[59345]: cluster 2023-10-09T09:54:10.042832+0000 osd.6 (osd.6) 563 : cluster 3 failed to encode map e1284 with expected crc 2023-10-09T09:54:11.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:11 smithi181 bash[59345]: audit 2023-10-09T09:54:10.045307+0000 mon.a (mon.0) 2548 : audit 1 from='client.? 172.21.15.163:0/2072186209' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:54:11.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:11 smithi181 bash[59345]: cluster 2023-10-09T09:54:10.045378+0000 mon.a (mon.0) 2549 : cluster 0 osdmap e1284: 8 total, 8 up, 6 in 2023-10-09T09:54:11.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:11 smithi181 bash[59345]: audit 2023-10-09T09:54:10.416712+0000 mon.a (mon.0) 2550 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:11.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:11 smithi181 bash[59345]: audit 2023-10-09T09:54:10.607321+0000 mon.a (mon.0) 2551 : audit 1 from='client.? 172.21.15.163:0/3405680545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:54:11.253 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:11 smithi181 bash[59345]: audit 2023-10-09T09:54:10.679080+0000 mon.a (mon.0) 2552 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:11.254 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:11 smithi181 bash[59345]: audit 2023-10-09T09:54:10.687052+0000 mon.a (mon.0) 2553 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:11.270 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:11.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:11.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:11.374 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:11.403 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:11.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:11.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:11.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:11.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:11.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:11.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:11.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:11.669 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:11.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:11.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:11.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:11.797 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:11.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:11.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:11.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:11.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:11.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:12.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:12.071 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:12.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:12.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:12.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T09:54:12.174 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:12.200 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:12.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:12.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:12.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:12 smithi181 bash[59345]: cluster 2023-10-09T09:54:10.255243+0000 mgr.y (mgr.34107) 1449 : cluster 0 pgmap v2325: 89 pgs: 89 active+clean; 1.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 734 op/s 2023-10-09T09:54:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:12 smithi181 bash[59345]: cluster 2023-10-09T09:54:11.053112+0000 mon.a (mon.0) 2554 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:54:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:12 smithi181 bash[59345]: cluster 2023-10-09T09:54:11.071386+0000 osd.5 (osd.5) 875 : cluster 3 failed to encode map e1285 with expected crc 2023-10-09T09:54:12.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:12 smithi181 bash[59345]: audit 2023-10-09T09:54:11.075096+0000 mon.a (mon.0) 2555 : audit 1 from='client.? 172.21.15.163:0/3405680545' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:54:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:12 smithi181 bash[59345]: cluster 2023-10-09T09:54:11.075335+0000 mon.a (mon.0) 2556 : cluster 0 osdmap e1285: 8 total, 8 up, 6 in 2023-10-09T09:54:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:12 smithi181 bash[59345]: cluster 2023-10-09T09:54:11.089955+0000 osd.4 (osd.4) 673 : cluster 3 failed to encode map e1285 with expected crc 2023-10-09T09:54:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:12 smithi181 bash[59345]: audit 2023-10-09T09:54:11.273910+0000 mon.a (mon.0) 2557 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:12 smithi181 bash[59345]: audit 2023-10-09T09:54:11.275795+0000 mon.a (mon.0) 2558 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:12 smithi181 bash[59345]: audit 2023-10-09T09:54:11.287269+0000 mon.a (mon.0) 2559 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:12.335 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:12.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:12.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:12.438 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:12.462 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:12.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:12.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:12.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[75425]: cluster 2023-10-09T09:54:10.255243+0000 mgr.y (mgr.34107) 1449 : cluster 0 pgmap v2325: 89 pgs: 89 active+clean; 1.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 734 op/s 2023-10-09T09:54:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[75425]: cluster 2023-10-09T09:54:11.053112+0000 mon.a (mon.0) 2554 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:54:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[75425]: cluster 2023-10-09T09:54:11.071386+0000 osd.5 (osd.5) 875 : cluster 3 failed to encode map e1285 with expected crc 2023-10-09T09:54:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[75425]: audit 2023-10-09T09:54:11.075096+0000 mon.a (mon.0) 2555 : audit 1 from='client.? 172.21.15.163:0/3405680545' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:54:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[75425]: cluster 2023-10-09T09:54:11.075335+0000 mon.a (mon.0) 2556 : cluster 0 osdmap e1285: 8 total, 8 up, 6 in 2023-10-09T09:54:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[75425]: cluster 2023-10-09T09:54:11.089955+0000 osd.4 (osd.4) 673 : cluster 3 failed to encode map e1285 with expected crc 2023-10-09T09:54:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[75425]: audit 2023-10-09T09:54:11.273910+0000 mon.a (mon.0) 2557 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[75425]: audit 2023-10-09T09:54:11.275795+0000 mon.a (mon.0) 2558 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[75425]: audit 2023-10-09T09:54:11.287269+0000 mon.a (mon.0) 2559 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:12.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[78107]: cluster 2023-10-09T09:54:10.255243+0000 mgr.y (mgr.34107) 1449 : cluster 0 pgmap v2325: 89 pgs: 89 active+clean; 1.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 734 op/s 2023-10-09T09:54:12.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[78107]: cluster 2023-10-09T09:54:11.053112+0000 mon.a (mon.0) 2554 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:54:12.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[78107]: cluster 2023-10-09T09:54:11.071386+0000 osd.5 (osd.5) 875 : cluster 3 failed to encode map e1285 with expected crc 2023-10-09T09:54:12.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[78107]: audit 2023-10-09T09:54:11.075096+0000 mon.a (mon.0) 2555 : audit 1 from='client.? 172.21.15.163:0/3405680545' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:54:12.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[78107]: cluster 2023-10-09T09:54:11.075335+0000 mon.a (mon.0) 2556 : cluster 0 osdmap e1285: 8 total, 8 up, 6 in 2023-10-09T09:54:12.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[78107]: cluster 2023-10-09T09:54:11.089955+0000 osd.4 (osd.4) 673 : cluster 3 failed to encode map e1285 with expected crc 2023-10-09T09:54:12.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[78107]: audit 2023-10-09T09:54:11.273910+0000 mon.a (mon.0) 2557 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:12.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[78107]: audit 2023-10-09T09:54:11.275795+0000 mon.a (mon.0) 2558 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:12.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:12 smithi163 bash[78107]: audit 2023-10-09T09:54:11.287269+0000 mon.a (mon.0) 2559 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:12.566 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:12.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:12.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:12.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:12.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:12.716 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:12.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:12.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:12.824 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:12.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:12.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:12.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:12.954 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:12.985 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:12.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:12.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:13.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:13.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:13.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:13.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:13.221 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:13.250 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:13.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:13.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:13.386 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:13.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:13.571 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:13.608 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:13.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:13.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:13.611 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:54:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:54:13] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.33.4" 2023-10-09T09:54:13.712 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:13.760 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:13.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:13.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:13.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:13.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:13.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:13.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:13.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:14.029 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:14.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:14.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:14.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:14.165 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:14.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:14.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:14.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:14 smithi163 bash[75425]: cluster 2023-10-09T09:54:12.255904+0000 mgr.y (mgr.34107) 1450 : cluster 0 pgmap v2327: 89 pgs: 89 active+clean; 1.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 883 op/s 2023-10-09T09:54:14.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:14 smithi163 bash[78107]: cluster 2023-10-09T09:54:12.255904+0000 mgr.y (mgr.34107) 1450 : cluster 0 pgmap v2327: 89 pgs: 89 active+clean; 1.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 883 op/s 2023-10-09T09:54:14.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:14.306 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:14.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:14.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:14.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:14.437 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:14.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:14.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:14.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:14.551 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:14 smithi181 bash[59345]: cluster 2023-10-09T09:54:12.255904+0000 mgr.y (mgr.34107) 1450 : cluster 0 pgmap v2327: 89 pgs: 89 active+clean; 1.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 883 op/s 2023-10-09T09:54:14.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:14.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:14.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:14.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:14.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:14.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:14.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:14.785 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:54:14.811 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:14.848 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:14.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:14.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:14.952 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:14.977 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:14.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:14.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:15.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:15.105 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:15.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:15.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:15.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:15.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:15.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:15.374 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:15.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:15.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:15.480 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:15.511 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:15.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:15.614 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:15.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:15.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:15.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:15.769 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:15.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:15.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:15.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:15.901 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:15.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:15.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:15.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:54:16.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:54:16.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:16.029 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:16.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:16.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:16.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:16.172 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:16.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:16.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:16.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:16.302 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:16 smithi181 bash[59345]: cluster 2023-10-09T09:54:14.256389+0000 mgr.y (mgr.34107) 1451 : cluster 0 pgmap v2328: 89 pgs: 89 active+clean; 1.0 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 859 op/s 2023-10-09T09:54:16.308 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:16.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:16.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:16.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:16.423 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:16 smithi163 bash[75425]: cluster 2023-10-09T09:54:14.256389+0000 mgr.y (mgr.34107) 1451 : cluster 0 pgmap v2328: 89 pgs: 89 active+clean; 1.0 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 859 op/s 2023-10-09T09:54:16.423 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:16 smithi163 bash[78107]: cluster 2023-10-09T09:54:14.256389+0000 mgr.y (mgr.34107) 1451 : cluster 0 pgmap v2328: 89 pgs: 89 active+clean; 1.0 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 859 op/s 2023-10-09T09:54:16.444 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:16.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:16.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:16.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:16.581 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:16.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:16.688 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:16.716 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:16.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:16.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:16.821 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:16.851 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:16.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:16.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:16.956 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:16.975 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:54:16.984 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:16.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:16.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:17.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:17.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:17.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:17.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:17.231 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:17.255 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:17.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:17.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:17.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:17.390 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:17.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:17.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:17.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:17.525 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:17.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:17.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:17.632 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:17.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:17.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:17.763 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:17.957 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:17.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:17.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:18.062 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:18.104 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:18.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:18.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:18.210 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:18.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:18 smithi163 bash[75425]: cluster 2023-10-09T09:54:16.256870+0000 mgr.y (mgr.34107) 1452 : cluster 0 pgmap v2329: 89 pgs: 89 active+clean; 993 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 554 op/s 2023-10-09T09:54:18.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:18 smithi163 bash[78107]: cluster 2023-10-09T09:54:16.256870+0000 mgr.y (mgr.34107) 1452 : cluster 0 pgmap v2329: 89 pgs: 89 active+clean; 993 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 554 op/s 2023-10-09T09:54:18.255 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:18.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:18.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:18.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:18.397 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:18.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:18.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:18.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:18.533 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:18.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:18.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:18 smithi181 bash[59345]: cluster 2023-10-09T09:54:16.256870+0000 mgr.y (mgr.34107) 1452 : cluster 0 pgmap v2329: 89 pgs: 89 active+clean; 993 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 554 op/s 2023-10-09T09:54:18.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:18.667 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:18.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:18.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:18.775 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:18.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:18.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:18.922 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:18.979 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:18.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:18.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:19.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:19.123 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:19.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:19.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:19.230 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:19.259 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:19.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:19.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:19.366 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:19.395 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:19.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:19.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:19.417 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:54:19.417 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:54:19.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:19.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:19.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:19.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:19.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:19.684 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:19.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:19.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:19.788 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:19.829 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:19.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:19.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:19.971 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:19.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:19.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:20.076 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:20.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:20 smithi163 bash[78107]: cluster 2023-10-09T09:54:18.257534+0000 mgr.y (mgr.34107) 1453 : cluster 0 pgmap v2330: 89 pgs: 89 active+clean; 782 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 922 op/s 2023-10-09T09:54:20.091 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:20 smithi163 bash[75425]: cluster 2023-10-09T09:54:18.257534+0000 mgr.y (mgr.34107) 1453 : cluster 0 pgmap v2330: 89 pgs: 89 active+clean; 782 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 922 op/s 2023-10-09T09:54:20.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:20.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:20.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:20.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:20.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:20.240 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:20 smithi181 bash[59345]: cluster 2023-10-09T09:54:18.257534+0000 mgr.y (mgr.34107) 1453 : cluster 0 pgmap v2330: 89 pgs: 89 active+clean; 782 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 922 op/s 2023-10-09T09:54:20.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T09:54:20.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:20.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:20.344 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:20.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:20.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:20.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:20.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:20.505 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:20.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:20.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:20.608 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:20.636 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:20.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:20.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:20.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:20.776 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:20.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:20.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:20.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:20.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:20.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:20.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:21.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:21.039 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:21.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:21.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:21.173 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:21.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:21.243 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:54:21.243 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:54:21.278 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:21.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:21.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:21.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:21.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:21.436 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:21.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:21.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:21.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:21.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:21.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:21.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:21.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:21.698 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:21.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:21.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:21.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:21.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:21.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:21.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:21.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:21.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:21.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:21.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:22.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:22.096 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:22.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:22.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:22.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:22.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:22 smithi163 bash[78107]: cluster 2023-10-09T09:54:20.258187+0000 mgr.y (mgr.34107) 1454 : cluster 0 pgmap v2331: 89 pgs: 89 active+clean; 766 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 783 op/s 2023-10-09T09:54:22.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:22 smithi163 bash[75425]: cluster 2023-10-09T09:54:20.258187+0000 mgr.y (mgr.34107) 1454 : cluster 0 pgmap v2331: 89 pgs: 89 active+clean; 766 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 783 op/s 2023-10-09T09:54:22.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:22.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:22.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:22.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:22.361 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:22.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:22.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:22.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:22.477 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:22 smithi181 bash[59345]: cluster 2023-10-09T09:54:20.258187+0000 mgr.y (mgr.34107) 1454 : cluster 0 pgmap v2331: 89 pgs: 89 active+clean; 766 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 783 op/s 2023-10-09T09:54:22.502 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:22.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:22.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:22.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:22.629 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:22.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:22.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:22.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:22.766 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:22.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:22.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:22.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:22.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:22.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:22.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:23.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:23.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:23.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:23.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:23.141 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:23.176 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:23.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:23.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:23.277 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:23.313 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:23.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:23.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:23.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:23.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:23.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:23.468 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:54:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:54:23] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.33.4" 2023-10-09T09:54:23.549 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:23.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:23.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:23.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:23.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:23.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:23.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:23.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:23.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:23.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:23.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:23.994 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:24.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:24.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:24.124 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:24.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:24.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:24.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:24.265 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:24.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:25.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:25.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:24 smithi163 bash[78107]: cluster 2023-10-09T09:54:22.259082+0000 mgr.y (mgr.34107) 1455 : cluster 0 pgmap v2332: 89 pgs: 89 active+clean; 521 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T09:54:25.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:24 smithi163 bash[75425]: cluster 2023-10-09T09:54:22.259082+0000 mgr.y (mgr.34107) 1455 : cluster 0 pgmap v2332: 89 pgs: 89 active+clean; 521 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T09:54:25.489 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:24 smithi181 bash[59345]: cluster 2023-10-09T09:54:22.259082+0000 mgr.y (mgr.34107) 1455 : cluster 0 pgmap v2332: 89 pgs: 89 active+clean; 521 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T09:54:25.489 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:54:25.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:25.887 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:25.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:25.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:25.992 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:26.018 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:26.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:26.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:26.118 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T09:54:26.122 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:26.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:26.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:26.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:26.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:26.381 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:26 smithi181 bash[59345]: cluster 2023-10-09T09:54:24.259608+0000 mgr.y (mgr.34107) 1456 : cluster 0 pgmap v2333: 89 pgs: 89 active+clean; 505 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T09:54:26.381 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:26 smithi181 bash[59345]: audit 2023-10-09T09:54:25.389351+0000 mon.a (mon.0) 2560 : audit 1 from='client.? 172.21.15.163:0/3023467248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:54:26.382 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:26 smithi181 bash[59345]: audit 2023-10-09T09:54:25.422150+0000 mon.a (mon.0) 2561 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:26.382 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:26 smithi181 bash[59345]: audit 2023-10-09T09:54:25.424310+0000 mon.a (mon.0) 2562 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:26.395 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:26.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:26.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:26 smithi163 bash[78107]: cluster 2023-10-09T09:54:24.259608+0000 mgr.y (mgr.34107) 1456 : cluster 0 pgmap v2333: 89 pgs: 89 active+clean; 505 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T09:54:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:26 smithi163 bash[78107]: audit 2023-10-09T09:54:25.389351+0000 mon.a (mon.0) 2560 : audit 1 from='client.? 172.21.15.163:0/3023467248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:54:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:26 smithi163 bash[78107]: audit 2023-10-09T09:54:25.422150+0000 mon.a (mon.0) 2561 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:26 smithi163 bash[78107]: audit 2023-10-09T09:54:25.424310+0000 mon.a (mon.0) 2562 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:26 smithi163 bash[75425]: cluster 2023-10-09T09:54:24.259608+0000 mgr.y (mgr.34107) 1456 : cluster 0 pgmap v2333: 89 pgs: 89 active+clean; 505 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T09:54:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:26 smithi163 bash[75425]: audit 2023-10-09T09:54:25.389351+0000 mon.a (mon.0) 2560 : audit 1 from='client.? 172.21.15.163:0/3023467248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:54:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:26 smithi163 bash[75425]: audit 2023-10-09T09:54:25.422150+0000 mon.a (mon.0) 2561 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:54:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:26 smithi163 bash[75425]: audit 2023-10-09T09:54:25.424310+0000 mon.a (mon.0) 2562 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:26.500 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:26.529 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:26.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:26.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:26.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:26.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:26.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:26.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:26.760 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:26.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:26.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:26.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:27.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:27.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:27.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:27.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:27 smithi181 bash[59345]: cluster 2023-10-09T09:54:26.110337+0000 osd.5 (osd.5) 876 : cluster 3 failed to encode map e1286 with expected crc 2023-10-09T09:54:27.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:27 smithi181 bash[59345]: audit 2023-10-09T09:54:26.113253+0000 mon.a (mon.0) 2563 : audit 1 from='client.? 172.21.15.163:0/3023467248' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:54:27.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:27 smithi181 bash[59345]: cluster 2023-10-09T09:54:26.113438+0000 mon.a (mon.0) 2564 : cluster 0 osdmap e1286: 8 total, 8 up, 6 in 2023-10-09T09:54:27.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:27.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:27 smithi163 bash[75425]: cluster 2023-10-09T09:54:26.110337+0000 osd.5 (osd.5) 876 : cluster 3 failed to encode map e1286 with expected crc 2023-10-09T09:54:27.383 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:27 smithi163 bash[75425]: audit 2023-10-09T09:54:26.113253+0000 mon.a (mon.0) 2563 : audit 1 from='client.? 172.21.15.163:0/3023467248' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:54:27.383 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:27 smithi163 bash[75425]: cluster 2023-10-09T09:54:26.113438+0000 mon.a (mon.0) 2564 : cluster 0 osdmap e1286: 8 total, 8 up, 6 in 2023-10-09T09:54:27.383 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:27 smithi163 bash[78107]: cluster 2023-10-09T09:54:26.110337+0000 osd.5 (osd.5) 876 : cluster 3 failed to encode map e1286 with expected crc 2023-10-09T09:54:27.383 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:27 smithi163 bash[78107]: audit 2023-10-09T09:54:26.113253+0000 mon.a (mon.0) 2563 : audit 1 from='client.? 172.21.15.163:0/3023467248' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:54:27.384 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:27 smithi163 bash[78107]: cluster 2023-10-09T09:54:26.113438+0000 mon.a (mon.0) 2564 : cluster 0 osdmap e1286: 8 total, 8 up, 6 in 2023-10-09T09:54:27.398 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:27.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:27.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:27.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:27.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:27.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:27.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:27.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:27.654 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:27.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:27.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:27.758 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:27.790 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:27.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:27.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:27.898 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:27.921 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:27.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:27.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:28.024 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:28.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:28.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:28.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:28.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:28.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:28 smithi181 bash[59345]: cluster 2023-10-09T09:54:26.260355+0000 mgr.y (mgr.34107) 1457 : cluster 0 pgmap v2335: 89 pgs: 89 active+clean; 457 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 858 op/s 2023-10-09T09:54:28.193 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:28.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:28.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:28.298 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:28.327 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:28.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:28.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:28.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:28.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:28 smithi163 bash[75425]: cluster 2023-10-09T09:54:26.260355+0000 mgr.y (mgr.34107) 1457 : cluster 0 pgmap v2335: 89 pgs: 89 active+clean; 457 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 858 op/s 2023-10-09T09:54:28.446 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:28 smithi163 bash[78107]: cluster 2023-10-09T09:54:26.260355+0000 mgr.y (mgr.34107) 1457 : cluster 0 pgmap v2335: 89 pgs: 89 active+clean; 457 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 858 op/s 2023-10-09T09:54:28.463 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:28.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:28.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:28.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:28.591 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:28.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:28.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:28.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:28.725 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:28.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:28.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:28.829 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:28.860 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:28.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:28.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:28.963 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:28.994 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:28.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:29.100 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:29.125 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:29.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:29.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:29.133 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:29 smithi181 bash[59345]: cluster 2023-10-09T09:54:28.261077+0000 mgr.y (mgr.34107) 1458 : cluster 0 pgmap v2336: 89 pgs: 89 active+clean; 272 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T09:54:29.231 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:29.256 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:29.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:29.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:29 smithi163 bash[75425]: cluster 2023-10-09T09:54:28.261077+0000 mgr.y (mgr.34107) 1458 : cluster 0 pgmap v2336: 89 pgs: 89 active+clean; 272 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T09:54:29.308 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:29 smithi163 bash[78107]: cluster 2023-10-09T09:54:28.261077+0000 mgr.y (mgr.34107) 1458 : cluster 0 pgmap v2336: 89 pgs: 89 active+clean; 272 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T09:54:29.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:29.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:29.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:29.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:29.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T09:54:29.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:29.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:29.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:29.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:29.627 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:29.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:29.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:29.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:29.726 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:54:29.726 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:54:29.761 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:29.795 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:29.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:29.900 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:29.930 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:29.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:29.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:30.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:30.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:30.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:30.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:30.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:30.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:30.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:30.296 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:30.328 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:30.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:30.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:30.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:30.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:30.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:30.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:30.574 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:30.601 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:30.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:30.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:30.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:30.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:30.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:30.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:30.845 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:30.870 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:30.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:30.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:30.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T09:54:30.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:31.003 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:31.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:31.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:31.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:31.149 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:31.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:31.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:31.264 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:31.295 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:31.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:31.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:31.397 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:31.423 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:31.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:31.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:31.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:31.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:31.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:31.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:31.640 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:54:31.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:31.684 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:31.687 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:54:31.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:31.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:31.739 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:54:31.778 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:54:31.792 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:31.814 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:54:31.826 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:31.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:31.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:54:31.894 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:54:31.932 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:54:31.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:31.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:54:31.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:54:31.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:54:31.936 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:54:31.936 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:54:31.936 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:54:31.936 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:54:31.936 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:54:31.968 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:31.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:32.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:32.096 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:32.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:32.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:32.157 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 107273 objects 2023-10-09T09:54:32.157 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :133.725 2023-10-09T09:54:32.201 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:32.209 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:31 smithi181 bash[59345]: cluster 2023-10-09T09:54:30.261592+0000 mgr.y (mgr.34107) 1459 : cluster 0 pgmap v2337: 89 pgs: 89 active+clean; 258 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 811 op/s 2023-10-09T09:54:32.210 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:31 smithi181 bash[59345]: audit 2023-10-09T09:54:31.589358+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.163:0/79440397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:54:32.227 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:32.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:32.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:32.331 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:32.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:31 smithi163 bash[78107]: cluster 2023-10-09T09:54:30.261592+0000 mgr.y (mgr.34107) 1459 : cluster 0 pgmap v2337: 89 pgs: 89 active+clean; 258 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 811 op/s 2023-10-09T09:54:32.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:31 smithi163 bash[78107]: audit 2023-10-09T09:54:31.589358+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.163:0/79440397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:54:32.345 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:31 smithi163 bash[75425]: cluster 2023-10-09T09:54:30.261592+0000 mgr.y (mgr.34107) 1459 : cluster 0 pgmap v2337: 89 pgs: 89 active+clean; 258 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 811 op/s 2023-10-09T09:54:32.345 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:31 smithi163 bash[75425]: audit 2023-10-09T09:54:31.589358+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.163:0/79440397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:54:32.358 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:32.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:32.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:32.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:32.490 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:32.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:32.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:32.595 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:32.626 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:32.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:32.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:32.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:32.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:32.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:32.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:32.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:32.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:32.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:32.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:32.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:33.005 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:54:33.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:33.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:33.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:33.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:32 smithi163 bash[75425]: audit 2023-10-09T09:54:32.014196+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.163:0/854321784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:54:33.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:32 smithi163 bash[75425]: audit 2023-10-09T09:54:32.014597+0000 mon.a (mon.0) 2565 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:54:33.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:32 smithi163 bash[78107]: audit 2023-10-09T09:54:32.014196+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.163:0/854321784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:54:33.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:32 smithi163 bash[78107]: audit 2023-10-09T09:54:32.014597+0000 mon.a (mon.0) 2565 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:54:33.129 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:33.162 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:33.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:33.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:33.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:33.279 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:32 smithi181 bash[59345]: audit 2023-10-09T09:54:32.014196+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.163:0/854321784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:54:33.279 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:32 smithi181 bash[59345]: audit 2023-10-09T09:54:32.014597+0000 mon.a (mon.0) 2565 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:54:33.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:33.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:33.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:33.401 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:33.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:33.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:33.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:33.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:33.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:33.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:33.670 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:33.701 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:33.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:33.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:33.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:54:33] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.33.4" 2023-10-09T09:54:33.805 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:33.835 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:33.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:33.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:33.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:33.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:33.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:33.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:34.075 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:34.112 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:34.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:34.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:34.215 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[75425]: cluster 2023-10-09T09:54:32.262756+0000 mgr.y (mgr.34107) 1460 : cluster 0 pgmap v2338: 89 pgs: 89 active+clean; 23 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 797 op/s 2023-10-09T09:54:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[75425]: cluster 2023-10-09T09:54:32.986001+0000 mon.a (mon.0) 2566 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:54:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[75425]: cluster 2023-10-09T09:54:32.994087+0000 osd.5 (osd.5) 877 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[75425]: audit 2023-10-09T09:54:32.995682+0000 mon.a (mon.0) 2567 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:54:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[75425]: cluster 2023-10-09T09:54:32.995777+0000 mon.a (mon.0) 2568 : cluster 0 osdmap e1287: 8 total, 8 up, 6 in 2023-10-09T09:54:34.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[78107]: cluster 2023-10-09T09:54:32.262756+0000 mgr.y (mgr.34107) 1460 : cluster 0 pgmap v2338: 89 pgs: 89 active+clean; 23 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 797 op/s 2023-10-09T09:54:34.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[78107]: cluster 2023-10-09T09:54:32.986001+0000 mon.a (mon.0) 2566 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:54:34.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[78107]: cluster 2023-10-09T09:54:32.994087+0000 osd.5 (osd.5) 877 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:34.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[78107]: audit 2023-10-09T09:54:32.995682+0000 mon.a (mon.0) 2567 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:54:34.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:33 smithi163 bash[78107]: cluster 2023-10-09T09:54:32.995777+0000 mon.a (mon.0) 2568 : cluster 0 osdmap e1287: 8 total, 8 up, 6 in 2023-10-09T09:54:34.245 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:34.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:34.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:33 smithi181 bash[59345]: cluster 2023-10-09T09:54:32.262756+0000 mgr.y (mgr.34107) 1460 : cluster 0 pgmap v2338: 89 pgs: 89 active+clean; 23 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 797 op/s 2023-10-09T09:54:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:33 smithi181 bash[59345]: cluster 2023-10-09T09:54:32.986001+0000 mon.a (mon.0) 2566 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:54:34.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:33 smithi181 bash[59345]: cluster 2023-10-09T09:54:32.994087+0000 osd.5 (osd.5) 877 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:34.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:33 smithi181 bash[59345]: audit 2023-10-09T09:54:32.995682+0000 mon.a (mon.0) 2567 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:54:34.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:33 smithi181 bash[59345]: cluster 2023-10-09T09:54:32.995777+0000 mon.a (mon.0) 2568 : cluster 0 osdmap e1287: 8 total, 8 up, 6 in 2023-10-09T09:54:34.349 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:34.383 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:34.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:34.487 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:34.519 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:34.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:34.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:34.624 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:34.653 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:34.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:34.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:34.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:34.772 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2023-10-09T09:54:34.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2023-10-09T09:54:34.787 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:34.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:34.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:34.891 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:34.922 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:34.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:34.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:35.026 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:35.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:35.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:35.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:35.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:34 smithi181 bash[59345]: cluster 2023-10-09T09:54:33.896194+0000 osd.6 (osd.6) 564 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:35.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:34 smithi181 bash[59345]: cluster 2023-10-09T09:54:33.896228+0000 osd.7 (osd.7) 619 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:35.175 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:34 smithi181 bash[59345]: cluster 2023-10-09T09:54:33.896276+0000 osd.4 (osd.4) 674 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:35.196 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:35.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:35.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:35 smithi163 bash[78107]: cluster 2023-10-09T09:54:33.896194+0000 osd.6 (osd.6) 564 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:35.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:35 smithi163 bash[78107]: cluster 2023-10-09T09:54:33.896228+0000 osd.7 (osd.7) 619 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:35.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:35 smithi163 bash[78107]: cluster 2023-10-09T09:54:33.896276+0000 osd.4 (osd.4) 674 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:35.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:34 smithi163 bash[75425]: cluster 2023-10-09T09:54:33.896194+0000 osd.6 (osd.6) 564 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:35.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:34 smithi163 bash[75425]: cluster 2023-10-09T09:54:33.896228+0000 osd.7 (osd.7) 619 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:35.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:34 smithi163 bash[75425]: cluster 2023-10-09T09:54:33.896276+0000 osd.4 (osd.4) 674 : cluster 3 failed to encode map e1287 with expected crc 2023-10-09T09:54:35.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:54:35.299 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:35.333 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:35.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:35.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:35.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:35.460 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:35.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:35.564 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:35.593 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:35.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:35.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:35.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:35.730 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:35.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:35.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:35.758 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:54:35.834 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:35.859 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:35.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:35.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:35.962 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:35.990 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:54:35.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:35.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:35.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:36.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:36 smithi181 bash[59345]: cluster 2023-10-09T09:54:34.263475+0000 mgr.y (mgr.34107) 1461 : cluster 0 pgmap v2340: 89 pgs: 89 active+clean; 12 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 966 op/s 2023-10-09T09:54:36.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:36.125 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:36.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:36.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:36.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:36.243 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:36 smithi163 bash[78107]: cluster 2023-10-09T09:54:34.263475+0000 mgr.y (mgr.34107) 1461 : cluster 0 pgmap v2340: 89 pgs: 89 active+clean; 12 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 966 op/s 2023-10-09T09:54:36.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:36 smithi163 bash[75425]: cluster 2023-10-09T09:54:34.263475+0000 mgr.y (mgr.34107) 1461 : cluster 0 pgmap v2340: 89 pgs: 89 active+clean; 12 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 966 op/s 2023-10-09T09:54:36.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:36.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:36.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:36.366 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:36.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:36.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:36.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:36.491 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:54:36.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:36.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:36.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:36.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:36.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:36.655 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:36.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:36.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:36.758 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:36.780 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:36.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:36.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:36.883 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:36.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:36.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:37.018 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:37.048 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:37.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:37.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:37.153 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:37.182 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:37.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:37.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:37.285 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:37.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:37.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:37.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:37.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:37.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:37.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:37.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:37.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:37.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:37.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:37.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:37.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:37.719 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:37.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:37.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:37.823 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:37.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:37.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:37.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:37.955 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:37.981 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:37.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:37.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:38.084 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:38.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:38.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:38.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:38.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:38 smithi163 bash[78107]: cluster 2023-10-09T09:54:36.264326+0000 mgr.y (mgr.34107) 1462 : cluster 0 pgmap v2341: 89 pgs: 89 active+clean; 3.1 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T09:54:38.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:38 smithi163 bash[75425]: cluster 2023-10-09T09:54:36.264326+0000 mgr.y (mgr.34107) 1462 : cluster 0 pgmap v2341: 89 pgs: 89 active+clean; 3.1 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T09:54:38.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:38.252 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:38.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:38.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:38.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:38 smithi181 bash[59345]: cluster 2023-10-09T09:54:36.264326+0000 mgr.y (mgr.34107) 1462 : cluster 0 pgmap v2341: 89 pgs: 89 active+clean; 3.1 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 725 op/s 2023-10-09T09:54:38.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:38.384 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:38.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:38.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:38.517 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:38.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:38.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:38.648 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:38.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:38.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:38.751 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:38.777 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:38.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:38.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:38.908 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:38.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:39.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:39.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:39.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:39.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:39.146 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:39.173 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:39.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:39.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:39.307 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:39.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:39.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:39.410 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:39.440 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:39.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:39.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:39.572 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:39.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:39.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:39.600 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:54:39.601 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:54:39.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:39.711 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:39.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:39.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:39.763 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:54:39.911 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:54:39.911 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:54:39.912 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:54:39.912 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:54:39.912 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:39.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:39.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:39.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:40.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:40.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:40 smithi181 bash[59345]: cluster 2023-10-09T09:54:38.265184+0000 mgr.y (mgr.34107) 1463 : cluster 0 pgmap v2342: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 434 op/s 2023-10-09T09:54:40.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:40.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:40.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:40.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:40.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:40.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:40.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:40.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:40 smithi163 bash[75425]: cluster 2023-10-09T09:54:38.265184+0000 mgr.y (mgr.34107) 1463 : cluster 0 pgmap v2342: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 434 op/s 2023-10-09T09:54:40.217 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:40 smithi163 bash[78107]: cluster 2023-10-09T09:54:38.265184+0000 mgr.y (mgr.34107) 1463 : cluster 0 pgmap v2342: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 434 op/s 2023-10-09T09:54:40.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:40.328 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:40.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:40.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:40.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:40.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:40.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:40.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:40.557 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:40.558 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:54:40.587 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:40.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:40.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:40.623 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T09:54:40.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:40.716 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:40.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:40.819 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:40.847 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:40.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:40.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:40.951 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:40.975 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:40.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:41.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:41.099 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:41.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:41.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:41.202 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:41.215 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:41 smithi181 bash[59345]: audit 2023-10-09T09:54:40.417831+0000 mon.a (mon.0) 2569 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:41.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:41 smithi163 bash[78107]: audit 2023-10-09T09:54:40.417831+0000 mon.a (mon.0) 2569 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:41.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:41 smithi163 bash[75425]: audit 2023-10-09T09:54:40.417831+0000 mon.a (mon.0) 2569 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:41.234 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:41.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:41.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:41.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:41.372 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:41.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:41.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:41.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:41.501 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:41.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:41.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:41.625 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:41.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:41.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:41.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:41.750 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:41.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:41.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:41.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:41.883 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:41.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:41.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:41.987 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:42.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:42.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:42.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:42.121 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:42.142 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:42.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:42.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:42.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:42.271 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:42.272 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:42 smithi163 bash[78107]: cluster 2023-10-09T09:54:40.265988+0000 mgr.y (mgr.34107) 1464 : cluster 0 pgmap v2343: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 412 op/s 2023-10-09T09:54:42.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:42 smithi163 bash[75425]: cluster 2023-10-09T09:54:40.265988+0000 mgr.y (mgr.34107) 1464 : cluster 0 pgmap v2343: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 412 op/s 2023-10-09T09:54:42.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:42.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:42.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:42 smithi181 bash[59345]: cluster 2023-10-09T09:54:40.265988+0000 mgr.y (mgr.34107) 1464 : cluster 0 pgmap v2343: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 412 op/s 2023-10-09T09:54:42.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:42.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:42.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:42.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:42.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:42.529 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:42.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:42.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:42.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:42.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:42.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:42.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:42.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:42.789 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:42.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:42.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:42.892 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:42.921 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:42.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:43.025 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:43.045 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:43.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:43.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:43.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:43.171 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:43.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:43.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:43.274 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:43.306 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:43.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:43.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:43.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:43.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:43.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:43.539 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:43.569 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:43.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:43.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:43.672 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:43.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:43.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:43.736 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:54:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:54:43] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.33.4" 2023-10-09T09:54:43.836 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:43.868 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:43.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:43.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:43.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:43.998 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:44.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:44.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:44.037 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T09:54:44.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:44 smithi163 bash[78107]: cluster 2023-10-09T09:54:42.266713+0000 mgr.y (mgr.34107) 1465 : cluster 0 pgmap v2344: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 35 op/s 2023-10-09T09:54:44.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:44 smithi163 bash[78107]: audit 2023-10-09T09:54:43.364522+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.163:0/3585100010' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:54:44.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:44 smithi163 bash[78107]: audit 2023-10-09T09:54:43.365035+0000 mon.a (mon.0) 2570 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:54:44.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:44 smithi163 bash[75425]: cluster 2023-10-09T09:54:42.266713+0000 mgr.y (mgr.34107) 1465 : cluster 0 pgmap v2344: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 35 op/s 2023-10-09T09:54:44.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:44 smithi163 bash[75425]: audit 2023-10-09T09:54:43.364522+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.163:0/3585100010' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:54:44.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:44 smithi163 bash[75425]: audit 2023-10-09T09:54:43.365035+0000 mon.a (mon.0) 2570 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:54:44.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:44.130 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:44.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:44.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:44.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:44.260 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:44.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:44.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:44.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:44 smithi181 bash[59345]: cluster 2023-10-09T09:54:42.266713+0000 mgr.y (mgr.34107) 1465 : cluster 0 pgmap v2344: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 35 op/s 2023-10-09T09:54:44.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:44 smithi181 bash[59345]: audit 2023-10-09T09:54:43.364522+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.163:0/3585100010' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:54:44.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:44 smithi181 bash[59345]: audit 2023-10-09T09:54:43.365035+0000 mon.a (mon.0) 2570 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:54:44.364 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:44.399 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:44.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:44.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:44.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:44.535 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:44.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:44.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:44.639 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:44.673 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:44.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:44.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:44.777 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:44.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:44.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:44.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:44.908 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:44.939 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:44.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:44.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:45.042 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:45.069 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:45.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:45.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:45.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:45.184 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:45 smithi181 bash[59345]: cluster 2023-10-09T09:54:44.028924+0000 osd.5 (osd.5) 878 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.185 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:45 smithi181 bash[59345]: cluster 2023-10-09T09:54:44.030135+0000 osd.6 (osd.6) 565 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.185 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:45 smithi181 bash[59345]: audit 2023-10-09T09:54:44.030969+0000 mon.a (mon.0) 2571 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:54:45.185 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:45 smithi181 bash[59345]: cluster 2023-10-09T09:54:44.031097+0000 mon.a (mon.0) 2572 : cluster 0 osdmap e1288: 8 total, 8 up, 6 in 2023-10-09T09:54:45.185 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:45 smithi181 bash[59345]: cluster 2023-10-09T09:54:44.085925+0000 osd.4 (osd.4) 675 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.185 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:45 smithi181 bash[59345]: cluster 2023-10-09T09:54:44.397893+0000 osd.7 (osd.7) 620 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.207 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:45.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:45.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:45.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[75425]: cluster 2023-10-09T09:54:44.028924+0000 osd.5 (osd.5) 878 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[75425]: cluster 2023-10-09T09:54:44.030135+0000 osd.6 (osd.6) 565 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[75425]: audit 2023-10-09T09:54:44.030969+0000 mon.a (mon.0) 2571 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:54:45.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[75425]: cluster 2023-10-09T09:54:44.031097+0000 mon.a (mon.0) 2572 : cluster 0 osdmap e1288: 8 total, 8 up, 6 in 2023-10-09T09:54:45.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[75425]: cluster 2023-10-09T09:54:44.085925+0000 osd.4 (osd.4) 675 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[75425]: cluster 2023-10-09T09:54:44.397893+0000 osd.7 (osd.7) 620 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[78107]: cluster 2023-10-09T09:54:44.028924+0000 osd.5 (osd.5) 878 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[78107]: cluster 2023-10-09T09:54:44.030135+0000 osd.6 (osd.6) 565 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[78107]: audit 2023-10-09T09:54:44.030969+0000 mon.a (mon.0) 2571 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:54:45.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[78107]: cluster 2023-10-09T09:54:44.031097+0000 mon.a (mon.0) 2572 : cluster 0 osdmap e1288: 8 total, 8 up, 6 in 2023-10-09T09:54:45.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[78107]: cluster 2023-10-09T09:54:44.085925+0000 osd.4 (osd.4) 675 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:45 smithi163 bash[78107]: cluster 2023-10-09T09:54:44.397893+0000 osd.7 (osd.7) 620 : cluster 3 failed to encode map e1288 with expected crc 2023-10-09T09:54:45.311 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:45.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:45.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:45.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:45.442 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:54:45.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:45.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:45.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:45.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:45.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:45.606 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:45.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:45.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:45.709 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:45.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:45.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:45.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:45.839 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:45.867 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:45.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:45.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:45.972 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:45.999 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:46.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:46.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:46.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:46.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:46.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:46 smithi163 bash[75425]: cluster 2023-10-09T09:54:44.267216+0000 mgr.y (mgr.34107) 1466 : cluster 0 pgmap v2346: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 18 op/s 2023-10-09T09:54:46.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:46 smithi163 bash[78107]: cluster 2023-10-09T09:54:44.267216+0000 mgr.y (mgr.34107) 1466 : cluster 0 pgmap v2346: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 18 op/s 2023-10-09T09:54:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:46.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:46.240 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:46.252 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:46 smithi181 bash[59345]: cluster 2023-10-09T09:54:44.267216+0000 mgr.y (mgr.34107) 1466 : cluster 0 pgmap v2346: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 18 op/s 2023-10-09T09:54:46.266 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:46.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:46.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:46.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:46.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:46.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:46.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:46.495 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:46.522 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:46.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:46.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:46.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:46.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:46.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:46.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:46.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:46.785 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:46.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:46.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:46.889 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:46.920 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:46.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:47.020 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T09:54:47.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:54:47.023 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:47.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:47.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:47.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:47.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:47.182 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:47.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:47.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:47.285 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:47.307 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:47.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:47.411 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:47.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:47.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:47.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:47.533 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:47.557 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:47.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:47.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:47.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:47.691 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:47.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:47.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:47.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:47.822 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:47.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:47.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:47.925 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:47.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:47.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:47.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:48.050 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:48.079 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:48.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:48.183 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:48.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:48 smithi163 bash[75425]: cluster 2023-10-09T09:54:46.267670+0000 mgr.y (mgr.34107) 1467 : cluster 0 pgmap v2347: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 4 op/s 2023-10-09T09:54:48.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:48 smithi163 bash[78107]: cluster 2023-10-09T09:54:46.267670+0000 mgr.y (mgr.34107) 1467 : cluster 0 pgmap v2347: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 4 op/s 2023-10-09T09:54:48.211 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:48.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:48.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:48.314 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:48.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:48 smithi181 bash[59345]: cluster 2023-10-09T09:54:46.267670+0000 mgr.y (mgr.34107) 1467 : cluster 0 pgmap v2347: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 4 op/s 2023-10-09T09:54:48.341 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:48.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:48.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:48.444 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:48.482 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:48.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:48.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:48.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:48.607 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:48.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:48.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:48.710 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:48.733 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:48.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:48.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:48.837 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:48.869 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:48.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:48.972 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:49.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:49.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:49.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:49.095 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:54:49.096 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T09:54:49.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T09:54:49.096 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:54:49.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.097 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:40797 11.5 11:a5f92202:::benchmark_data_smithi163_218041_object40796:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1274)", 2023-10-09T09:54:49.097 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:10.006342+0000", 2023-10-09T09:54:49.097 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 219.081257685, 2023-10-09T09:54:49.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.246998945, 2023-10-09T09:54:49.097 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.097 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.097 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.097 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.098 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.098 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 40797 2023-10-09T09:54:49.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.098 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006342+0000", 2023-10-09T09:54:49.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006342+0000", 2023-10-09T09:54:49.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:54:49.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006339+0000", 2023-10-09T09:54:49.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5295000000000002e-05 2023-10-09T09:54:49.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006435+0000", 2023-10-09T09:54:49.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9142000000000001e-05 2023-10-09T09:54:49.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006454+0000", 2023-10-09T09:54:49.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8059999999999995e-06 2023-10-09T09:54:49.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006462+0000", 2023-10-09T09:54:49.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00072703700000000004 2023-10-09T09:54:49.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.007189+0000", 2023-10-09T09:54:49.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020330399999999999 2023-10-09T09:54:49.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.007392+0000", 2023-10-09T09:54:49.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024304799999999999 2023-10-09T09:54:49.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.007635+0000", 2023-10-09T09:54:49.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0065956900000000004 2023-10-09T09:54:49.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.014231+0000", 2023-10-09T09:54:49.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018349110000000001 2023-10-09T09:54:49.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.016066+0000", 2023-10-09T09:54:49.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23688267099999999 2023-10-09T09:54:49.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.252948+0000", 2023-10-09T09:54:49.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029218799999999997 2023-10-09T09:54:49.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.253241+0000", 2023-10-09T09:54:49.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000100225 2023-10-09T09:54:49.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.253341+0000", 2023-10-09T09:54:49.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.246998945 2023-10-09T09:54:49.108 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.108 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.108 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.108 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:40799 11.5 11:abe3691e:::benchmark_data_smithi163_218041_object40798:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1274)", 2023-10-09T09:54:49.109 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:10.006715+0000", 2023-10-09T09:54:49.109 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 219.080884446, 2023-10-09T09:54:49.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24714496999999999, 2023-10-09T09:54:49.109 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.109 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.110 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.110 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.110 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 40799 2023-10-09T09:54:49.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.111 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006715+0000", 2023-10-09T09:54:49.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006715+0000", 2023-10-09T09:54:49.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:54:49.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006714+0000", 2023-10-09T09:54:49.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7466000000000006e-05 2023-10-09T09:54:49.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006801+0000", 2023-10-09T09:54:49.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0137e-05 2023-10-09T09:54:49.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006812+0000", 2023-10-09T09:54:49.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3849999999999999e-06 2023-10-09T09:54:49.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.006818+0000", 2023-10-09T09:54:49.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002382192 2023-10-09T09:54:49.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.009200+0000", 2023-10-09T09:54:49.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015967499999999999 2023-10-09T09:54:49.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.009360+0000", 2023-10-09T09:54:49.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018038599999999999 2023-10-09T09:54:49.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.009540+0000", 2023-10-09T09:54:49.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064834469999999998 2023-10-09T09:54:49.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.016024+0000", 2023-10-09T09:54:49.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2147000000000001e-05 2023-10-09T09:54:49.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.016076+0000", 2023-10-09T09:54:49.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23739463799999999 2023-10-09T09:54:49.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.253470+0000", 2023-10-09T09:54:49.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020123299999999999 2023-10-09T09:54:49.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.253672+0000", 2023-10-09T09:54:49.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018839 2023-10-09T09:54:49.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.253860+0000", 2023-10-09T09:54:49.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24714496999999999 2023-10-09T09:54:49.119 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.119 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.119 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.119 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:59037 11.5 11:a577e751:::benchmark_data_smithi163_218041_object59036:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:54:49.120 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:25.524681+0000", 2023-10-09T09:54:49.120 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 203.56291878600001, 2023-10-09T09:54:49.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25213682500000001, 2023-10-09T09:54:49.120 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.120 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.120 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.120 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.121 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.121 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 59037 2023-10-09T09:54:49.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.121 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.524681+0000", 2023-10-09T09:54:49.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.524681+0000", 2023-10-09T09:54:49.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999895 2023-10-09T09:54:49.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.524670+0000", 2023-10-09T09:54:49.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7240000000000001e-05 2023-10-09T09:54:49.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.524728+0000", 2023-10-09T09:54:49.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6356e-05 2023-10-09T09:54:49.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.524754+0000", 2023-10-09T09:54:49.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7610000000000002e-06 2023-10-09T09:54:49.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.524764+0000", 2023-10-09T09:54:49.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5581999999999997e-05 2023-10-09T09:54:49.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.524829+0000", 2023-10-09T09:54:49.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000174518 2023-10-09T09:54:49.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.525004+0000", 2023-10-09T09:54:49.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015243799999999999 2023-10-09T09:54:49.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.525156+0000", 2023-10-09T09:54:49.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052175829999999996 2023-10-09T09:54:49.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.530374+0000", 2023-10-09T09:54:49.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9131999999999999e-05 2023-10-09T09:54:49.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.530393+0000", 2023-10-09T09:54:49.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24623398499999999 2023-10-09T09:54:49.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.776627+0000", 2023-10-09T09:54:49.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107198 2023-10-09T09:54:49.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.776734+0000", 2023-10-09T09:54:49.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3325999999999995e-05 2023-10-09T09:54:49.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.776818+0000", 2023-10-09T09:54:49.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25213682500000001 2023-10-09T09:54:49.133 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.133 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.133 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.133 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:59039 11.5 11:af4791a0:::benchmark_data_smithi163_218041_object59038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:54:49.134 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:25.525003+0000", 2023-10-09T09:54:49.135 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 203.562596103, 2023-10-09T09:54:49.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25207463200000002, 2023-10-09T09:54:49.135 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.136 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.136 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.136 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.136 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.136 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 59039 2023-10-09T09:54:49.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.137 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.525003+0000", 2023-10-09T09:54:49.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.525003+0000", 2023-10-09T09:54:49.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999948 2023-10-09T09:54:49.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.524998+0000", 2023-10-09T09:54:49.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1437000000000002e-05 2023-10-09T09:54:49.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.525050+0000", 2023-10-09T09:54:49.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1362999999999999e-05 2023-10-09T09:54:49.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.525071+0000", 2023-10-09T09:54:49.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2879999999999998e-06 2023-10-09T09:54:49.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.525077+0000", 2023-10-09T09:54:49.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025270190000000001 2023-10-09T09:54:49.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.527604+0000", 2023-10-09T09:54:49.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017297100000000001 2023-10-09T09:54:49.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.527777+0000", 2023-10-09T09:54:49.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022028299999999999 2023-10-09T09:54:49.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.527998+0000", 2023-10-09T09:54:49.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002555229 2023-10-09T09:54:49.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.530553+0000", 2023-10-09T09:54:49.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00061827400000000004 2023-10-09T09:54:49.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.531171+0000", 2023-10-09T09:54:49.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24574896199999999 2023-10-09T09:54:49.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.776920+0000", 2023-10-09T09:54:49.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8912000000000005e-05 2023-10-09T09:54:49.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.777009+0000", 2023-10-09T09:54:49.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9119e-05 2023-10-09T09:54:49.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.777078+0000", 2023-10-09T09:54:49.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25207463200000002 2023-10-09T09:54:49.149 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.149 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.149 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.149 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:59057 11.5 11:adc1bc68:::benchmark_data_smithi163_218041_object59056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:54:49.149 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:25.539959+0000", 2023-10-09T09:54:49.150 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 203.547640766, 2023-10-09T09:54:49.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25003228500000002, 2023-10-09T09:54:49.150 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.150 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.150 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.150 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.150 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.150 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 59057 2023-10-09T09:54:49.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.151 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.539959+0000", 2023-10-09T09:54:49.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.539959+0000", 2023-10-09T09:54:49.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:54:49.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.539957+0000", 2023-10-09T09:54:49.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8813000000000002e-05 2023-10-09T09:54:49.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.540006+0000", 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3747000000000001e-05 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.540020+0000", 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4400000000000002e-06 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.540026+0000", 2023-10-09T09:54:49.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7910000000000005e-05 2023-10-09T09:54:49.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.540094+0000", 2023-10-09T09:54:49.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000180736 2023-10-09T09:54:49.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.540275+0000", 2023-10-09T09:54:49.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020809499999999999 2023-10-09T09:54:49.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.540483+0000", 2023-10-09T09:54:49.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022293629999999998 2023-10-09T09:54:49.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.542712+0000", 2023-10-09T09:54:49.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001283668 2023-10-09T09:54:49.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.543996+0000", 2023-10-09T09:54:49.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.245833891 2023-10-09T09:54:49.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.789830+0000", 2023-10-09T09:54:49.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115748 2023-10-09T09:54:49.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.789945+0000", 2023-10-09T09:54:49.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5583e-05 2023-10-09T09:54:49.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.789991+0000", 2023-10-09T09:54:49.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25003228500000002 2023-10-09T09:54:49.166 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.166 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.166 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.166 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:103901 11.5 11:afb59a13:::benchmark_data_smithi163_218041_object103900:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.167 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:14.088516+0000", 2023-10-09T09:54:49.167 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 154.99908381500001, 2023-10-09T09:54:49.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16949808799999999, 2023-10-09T09:54:49.167 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.167 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.167 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.168 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.168 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.168 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 103901 2023-10-09T09:54:49.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.168 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.088516+0000", 2023-10-09T09:54:49.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.088516+0000", 2023-10-09T09:54:49.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:54:49.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.088515+0000", 2023-10-09T09:54:49.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012259099999999999 2023-10-09T09:54:49.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.088637+0000", 2023-10-09T09:54:49.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7285000000000001e-05 2023-10-09T09:54:49.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.088654+0000", 2023-10-09T09:54:49.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3610000000000003e-06 2023-10-09T09:54:49.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.088662+0000", 2023-10-09T09:54:49.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028804339999999999 2023-10-09T09:54:49.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.091542+0000", 2023-10-09T09:54:49.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7425999999999999e-05 2023-10-09T09:54:49.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.091640+0000", 2023-10-09T09:54:49.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000147539 2023-10-09T09:54:49.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.091787+0000", 2023-10-09T09:54:49.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010616126 2023-10-09T09:54:49.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.102403+0000", 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056415450000000004 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.108045+0000", 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14975734700000001 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.257802+0000", 2023-10-09T09:54:49.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118978 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.257921+0000", 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2608000000000005e-05 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.258014+0000", 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16949808799999999 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.180 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:103905 11.5 11:add26b22:::benchmark_data_smithi163_218041_object103904:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:14.091386+0000", 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 154.99621390900001, 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16692992200000001, 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.181 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 103905 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.091386+0000", 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.091386+0000", 2023-10-09T09:54:49.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.091384+0000", 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1229999999999995e-05 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.091445+0000", 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4127e-05 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.091459+0000", 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7849999999999992e-06 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.091467+0000", 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003772732 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.184 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.095240+0000", 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014722300000000001 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.095387+0000", 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000167076 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.095554+0000", 2023-10-09T09:54:49.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012427499 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.107982+0000", 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116672 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.108098+0000", 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.150023345 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.258122+0000", 2023-10-09T09:54:49.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107716 2023-10-09T09:54:49.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.258229+0000", 2023-10-09T09:54:49.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6296000000000002e-05 2023-10-09T09:54:49.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.258316+0000", 2023-10-09T09:54:49.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16692992200000001 2023-10-09T09:54:49.188 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.188 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.188 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:103906 11.5 11:a498aab2:::benchmark_data_smithi163_218041_object103905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:14.093666+0000", 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 154.99393357599999, 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16504775699999999, 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 103906 2023-10-09T09:54:49.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.093666+0000", 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.093666+0000", 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:54:49.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.093664+0000", 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4419999999999996e-05 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.093729+0000", 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5892999999999998e-05 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.093745+0000", 2023-10-09T09:54:49.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6920000000000002e-06 2023-10-09T09:54:49.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.093752+0000", 2023-10-09T09:54:49.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052940590000000003 2023-10-09T09:54:49.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.099046+0000", 2023-10-09T09:54:49.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016723700000000001 2023-10-09T09:54:49.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.193 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.099213+0000", 2023-10-09T09:54:49.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021493599999999999 2023-10-09T09:54:49.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.099428+0000", 2023-10-09T09:54:49.194 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0086841239999999997 2023-10-09T09:54:49.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.108113+0000", 2023-10-09T09:54:49.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2734000000000002e-05 2023-10-09T09:54:49.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.108145+0000", 2023-10-09T09:54:49.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.150259857 2023-10-09T09:54:49.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.258405+0000", 2023-10-09T09:54:49.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020533899999999999 2023-10-09T09:54:49.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.258610+0000", 2023-10-09T09:54:49.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103238 2023-10-09T09:54:49.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.258714+0000", 2023-10-09T09:54:49.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16504775699999999 2023-10-09T09:54:49.197 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:103909 11.5 11:aa59ae99:::benchmark_data_smithi163_218041_object103908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:14.094668+0000", 2023-10-09T09:54:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 154.99293106799999, 2023-10-09T09:54:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16444483200000001, 2023-10-09T09:54:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 103909 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.094668+0000", 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.094668+0000", 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.094667+0000", 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8667999999999999e-05 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.094755+0000", 2023-10-09T09:54:49.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.906e-05 2023-10-09T09:54:49.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.094774+0000", 2023-10-09T09:54:49.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0674e-05 2023-10-09T09:54:49.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.094785+0000", 2023-10-09T09:54:49.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0077308189999999999 2023-10-09T09:54:49.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.102516+0000", 2023-10-09T09:54:49.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019464499999999999 2023-10-09T09:54:49.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.203 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.203 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.102711+0000", 2023-10-09T09:54:49.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020824299999999999 2023-10-09T09:54:49.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.102919+0000", 2023-10-09T09:54:49.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052464319999999997 2023-10-09T09:54:49.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:49.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.206 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.206 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.108165+0000", 2023-10-09T09:54:49.206 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4125999999999998e-05 2023-10-09T09:54:49.206 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.206 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.108199+0000", 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15062584600000001 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.258825+0000", 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020875200000000001 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.259034+0000", 2023-10-09T09:54:49.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9351000000000006e-05 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:14.259113+0000", 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16444483200000001 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.208 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:111628 11.5 11:a8d4536c:::benchmark_data_smithi163_218041_object4352:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.209 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:23.345352+0000", 2023-10-09T09:54:49.209 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.74224804299999, 2023-10-09T09:54:49.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.124023657, 2023-10-09T09:54:49.209 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.209 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.209 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.209 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.209 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.210 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111628 2023-10-09T09:54:49.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.210 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.345352+0000", 2023-10-09T09:54:49.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.345352+0000", 2023-10-09T09:54:49.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T09:54:49.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.212 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.345349+0000", 2023-10-09T09:54:49.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4860000000000003e-06 2023-10-09T09:54:49.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.345359+0000", 2023-10-09T09:54:49.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.662e-05 2023-10-09T09:54:49.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.345375+0000", 2023-10-09T09:54:49.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6730000000000003e-06 2023-10-09T09:54:49.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.345382+0000", 2023-10-09T09:54:49.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6183000000000001e-05 2023-10-09T09:54:49.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.345438+0000", 2023-10-09T09:54:49.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108453 2023-10-09T09:54:49.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.345547+0000", 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6506000000000001e-05 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.345643+0000", 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041044699999999998 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.349748+0000", 2023-10-09T09:54:49.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003743354 2023-10-09T09:54:49.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.217 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.353491+0000", 2023-10-09T09:54:49.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.115509654 2023-10-09T09:54:49.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.217 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.469001+0000", 2023-10-09T09:54:49.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000219349 2023-10-09T09:54:49.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.469220+0000", 2023-10-09T09:54:49.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000155064 2023-10-09T09:54:49.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.469375+0000", 2023-10-09T09:54:49.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.124023657 2023-10-09T09:54:49.219 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.219 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.220 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.220 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:111641 11.5 11:a42e465b:::benchmark_data_smithi163_218041_object4365:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.220 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:23.356664+0000", 2023-10-09T09:54:49.220 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.73093518300001, 2023-10-09T09:54:49.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.123121208, 2023-10-09T09:54:49.221 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.221 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.221 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.221 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.221 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.221 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111641 2023-10-09T09:54:49.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.222 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.356664+0000", 2023-10-09T09:54:49.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.356664+0000", 2023-10-09T09:54:49.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:54:49.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.356662+0000", 2023-10-09T09:54:49.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7869999999999996e-06 2023-10-09T09:54:49.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.356672+0000", 2023-10-09T09:54:49.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4149e-05 2023-10-09T09:54:49.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.356686+0000", 2023-10-09T09:54:49.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3270000000000003e-06 2023-10-09T09:54:49.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.226 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.226 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.356694+0000", 2023-10-09T09:54:49.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3731000000000003e-05 2023-10-09T09:54:49.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.356767+0000", 2023-10-09T09:54:49.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015353 2023-10-09T09:54:49.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.356921+0000", 2023-10-09T09:54:49.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016910499999999999 2023-10-09T09:54:49.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.357090+0000", 2023-10-09T09:54:49.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040033270000000001 2023-10-09T09:54:49.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.361093+0000", 2023-10-09T09:54:49.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041727270000000002 2023-10-09T09:54:49.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.365266+0000", 2023-10-09T09:54:49.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.114394991 2023-10-09T09:54:49.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.479661+0000", 2023-10-09T09:54:49.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0952999999999996e-05 2023-10-09T09:54:49.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.479742+0000", 2023-10-09T09:54:49.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3699000000000003e-05 2023-10-09T09:54:49.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.479786+0000", 2023-10-09T09:54:49.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.123121208 2023-10-09T09:54:49.232 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.232 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.232 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.232 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:111663 11.5 11:a0c890d7:::benchmark_data_smithi163_218041_object4387:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.233 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:23.492648+0000", 2023-10-09T09:54:49.233 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.59495171200001, 2023-10-09T09:54:49.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050195333000000002, 2023-10-09T09:54:49.233 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.233 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.233 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.233 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.234 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.234 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111663 2023-10-09T09:54:49.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.234 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492648+0000", 2023-10-09T09:54:49.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492648+0000", 2023-10-09T09:54:49.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999933 2023-10-09T09:54:49.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492641+0000", 2023-10-09T09:54:49.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5914999999999999e-05 2023-10-09T09:54:49.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492657+0000", 2023-10-09T09:54:49.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9040999999999999e-05 2023-10-09T09:54:49.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492676+0000", 2023-10-09T09:54:49.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0899999999999994e-06 2023-10-09T09:54:49.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492685+0000", 2023-10-09T09:54:49.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026424100000000002 2023-10-09T09:54:49.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492950+0000", 2023-10-09T09:54:49.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014527600000000001 2023-10-09T09:54:49.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.493095+0000", 2023-10-09T09:54:49.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000155193 2023-10-09T09:54:49.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.493250+0000", 2023-10-09T09:54:49.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0092890309999999997 2023-10-09T09:54:49.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.502539+0000", 2023-10-09T09:54:49.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01023951 2023-10-09T09:54:49.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.512779+0000", 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029924968999999999 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.542704+0000", 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4763000000000001e-05 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.542788+0000", 2023-10-09T09:54:49.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4882000000000001e-05 2023-10-09T09:54:49.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.542843+0000", 2023-10-09T09:54:49.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050195333000000002 2023-10-09T09:54:49.243 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.243 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.243 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.244 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:111664 11.5 11:a1462f37:::benchmark_data_smithi163_218041_object4388:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.244 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:23.492710+0000", 2023-10-09T09:54:49.244 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.59488920800001, 2023-10-09T09:54:49.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050264449000000003, 2023-10-09T09:54:49.244 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.244 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.244 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.245 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.245 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.245 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111664 2023-10-09T09:54:49.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.245 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492710+0000", 2023-10-09T09:54:49.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492710+0000", 2023-10-09T09:54:49.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:54:49.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492710+0000", 2023-10-09T09:54:49.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3470000000000001e-06 2023-10-09T09:54:49.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492714+0000", 2023-10-09T09:54:49.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0698e-05 2023-10-09T09:54:49.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492725+0000", 2023-10-09T09:54:49.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4249999999999997e-06 2023-10-09T09:54:49.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492730+0000", 2023-10-09T09:54:49.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0051587359999999997 2023-10-09T09:54:49.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.497889+0000", 2023-10-09T09:54:49.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000137556 2023-10-09T09:54:49.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.498026+0000", 2023-10-09T09:54:49.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014564499999999999 2023-10-09T09:54:49.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.252 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.252 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.498172+0000", 2023-10-09T09:54:49.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044235919999999996 2023-10-09T09:54:49.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.502596+0000", 2023-10-09T09:54:49.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010189164000000001 2023-10-09T09:54:49.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.512785+0000", 2023-10-09T09:54:49.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.030100265000000001 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.542885+0000", 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8245999999999999e-05 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.542933+0000", 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1511999999999998e-05 2023-10-09T09:54:49.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.542975+0000", 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050264449000000003 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:111667 11.5 11:a286aebd:::benchmark_data_smithi163_218041_object4391:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:23.492736+0000", 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.59486349400001, 2023-10-09T09:54:49.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050385739999999998, 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111667 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492736+0000", 2023-10-09T09:54:49.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492736+0000", 2023-10-09T09:54:49.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:54:49.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492735+0000", 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.061e-06 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492739+0000", 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4826e-05 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492753+0000", 2023-10-09T09:54:49.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4839999999999998e-06 2023-10-09T09:54:49.259 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.259 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.259 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.259 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492758+0000", 2023-10-09T09:54:49.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0098733810000000005 2023-10-09T09:54:49.259 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.259 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.259 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.502631+0000", 2023-10-09T09:54:49.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015114600000000001 2023-10-09T09:54:49.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.502782+0000", 2023-10-09T09:54:49.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016885700000000001 2023-10-09T09:54:49.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.502951+0000", 2023-10-09T09:54:49.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.009817971 2023-10-09T09:54:49.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.512769+0000", 2023-10-09T09:54:49.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1815e-05 2023-10-09T09:54:49.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.512791+0000", 2023-10-09T09:54:49.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.030240514999999999 2023-10-09T09:54:49.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.543032+0000", 2023-10-09T09:54:49.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8210000000000001e-05 2023-10-09T09:54:49.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.543080+0000", 2023-10-09T09:54:49.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2030000000000002e-05 2023-10-09T09:54:49.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.543122+0000", 2023-10-09T09:54:49.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050385739999999998 2023-10-09T09:54:49.265 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.265 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.265 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.265 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:111671 11.5 11:a5766c6d:::benchmark_data_smithi163_218041_object4395:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.265 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:23.492793+0000", 2023-10-09T09:54:49.265 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.59480627299999, 2023-10-09T09:54:49.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050452559000000001, 2023-10-09T09:54:49.266 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.266 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.266 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.266 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.266 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.266 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111671 2023-10-09T09:54:49.267 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.267 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.267 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.267 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492793+0000", 2023-10-09T09:54:49.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.267 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.268 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.268 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492793+0000", 2023-10-09T09:54:49.268 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:54:49.268 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.268 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.268 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492793+0000", 2023-10-09T09:54:49.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.861e-06 2023-10-09T09:54:49.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.269 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.269 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492796+0000", 2023-10-09T09:54:49.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9090000000000003e-06 2023-10-09T09:54:49.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492803+0000", 2023-10-09T09:54:49.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6090000000000003e-06 2023-10-09T09:54:49.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.492808+0000", 2023-10-09T09:54:49.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015157299000000001 2023-10-09T09:54:49.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.507965+0000", 2023-10-09T09:54:49.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014654199999999999 2023-10-09T09:54:49.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.508112+0000", 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016809400000000001 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.508280+0000", 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0045390889999999996 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.512819+0000", 2023-10-09T09:54:49.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005540133 2023-10-09T09:54:49.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.518359+0000", 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.024799776999999999 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.543159+0000", 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6131000000000003e-05 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.543205+0000", 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0802e-05 2023-10-09T09:54:49.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:23.543246+0000", 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050452559000000001 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:112901 11.5 11:aac21548:::benchmark_data_smithi163_218041_object5625:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1277)", 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:24.896054+0000", 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 144.19154573399999, 2023-10-09T09:54:49.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050385034000000002, 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 112901 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.896054+0000", 2023-10-09T09:54:49.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.896054+0000", 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.896052+0000", 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1849999999999993e-06 2023-10-09T09:54:49.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.896061+0000", 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4239e-05 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.896075+0000", 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4259999999999997e-06 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.896083+0000", 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012439327 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.908522+0000", 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015678900000000001 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.908679+0000", 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000140216 2023-10-09T09:54:49.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.908819+0000", 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010612633 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.919432+0000", 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013535942 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.932968+0000", 2023-10-09T09:54:49.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013340331 2023-10-09T09:54:49.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.946308+0000", 2023-10-09T09:54:49.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4808000000000006e-05 2023-10-09T09:54:49.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.946383+0000", 2023-10-09T09:54:49.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5921999999999998e-05 2023-10-09T09:54:49.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:24.946439+0000", 2023-10-09T09:54:49.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.050385034000000002 2023-10-09T09:54:49.284 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.285 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.285 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.285 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:137581 11.5 11:aed9be28:::benchmark_data_smithi163_218041_object30305:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1280)", 2023-10-09T09:54:49.285 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:52:55.184940+0000", 2023-10-09T09:54:49.286 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 113.902659246, 2023-10-09T09:54:49.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.051002763, 2023-10-09T09:54:49.286 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.286 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.286 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.287 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.287 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.287 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 137581 2023-10-09T09:54:49.287 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.287 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.288 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.184940+0000", 2023-10-09T09:54:49.288 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.184940+0000", 2023-10-09T09:54:49.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:54:49.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.184939+0000", 2023-10-09T09:54:49.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5370000000000003e-06 2023-10-09T09:54:49.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.184946+0000", 2023-10-09T09:54:49.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4674e-05 2023-10-09T09:54:49.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.184960+0000", 2023-10-09T09:54:49.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4700000000000001e-06 2023-10-09T09:54:49.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.184966+0000", 2023-10-09T09:54:49.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9496999999999998e-05 2023-10-09T09:54:49.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.185005+0000", 2023-10-09T09:54:49.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2200999999999994e-05 2023-10-09T09:54:49.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.185097+0000", 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125134 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.185223+0000", 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0045308800000000002 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.189753+0000", 2023-10-09T09:54:49.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0538999999999998e-05 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.189804+0000", 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.045874232000000001 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.235678+0000", 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000131703 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.235810+0000", 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000133172 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:52:55.235943+0000", 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.051002763 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.297 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:190491 11.5 11:a895c280:::benchmark_data_smithi163_218041_object83215:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1283)", 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:01.521082+0000", 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 47.566517216000001, 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.047652604000000001, 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.298 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 190491 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.521082+0000", 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.521082+0000", 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.521081+0000", 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8189999999999997e-06 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.521087+0000", 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1843e-05 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.521099+0000", 2023-10-09T09:54:49.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.328e-06 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.521106+0000", 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.006026801 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.527133+0000", 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015189700000000001 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.527285+0000", 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118421 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.527403+0000", 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0095040060000000006 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.536907+0000", 2023-10-09T09:54:49.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031008400000000002 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.537217+0000", 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.031408884999999998 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.568626+0000", 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9160999999999998e-05 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.568685+0000", 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9573000000000001e-05 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:01.568735+0000", 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.047652604000000001 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:195432 11.5 11:a800facb:::benchmark_data_smithi163_218041_object88156:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1283)", 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:08.111050+0000", 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 40.976549802999997, 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.115212893, 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 195432 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.111050+0000", 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.111050+0000", 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.111048+0000", 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5170000000000002e-06 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.111055+0000", 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2999999999999999e-05 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.111068+0000", 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5180000000000004e-06 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.111077+0000", 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010320200000000001 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.111180+0000", 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023735800000000001 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.111417+0000", 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00030130400000000001 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.111719+0000", 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0069941270000000002 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.118713+0000", 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00070313099999999996 2023-10-09T09:54:49.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.119416+0000", 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.106673288 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.226089+0000", 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1515999999999996e-05 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.226181+0000", 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1771e-05 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.226263+0000", 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.115212893 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.311 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:195442 11.5 11:a6073f9e:::benchmark_data_smithi163_218041_object88166:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1283)", 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:08.126184+0000", 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 40.961415481000003, 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.100400528, 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 195442 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:54:49.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.126184+0000", 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.126184+0000", 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.126183+0000", 2023-10-09T09:54:49.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8070000000000006e-06 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.126191+0000", 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4123e-05 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.126206+0000", 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9030000000000001e-06 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.314 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.126211+0000", 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6842000000000003e-05 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.126258+0000", 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5171000000000001e-05 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.126353+0000", 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1977000000000004e-05 2023-10-09T09:54:49.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.126445+0000", 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036311310000000001 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.130077+0000", 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000135592 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.130212+0000", 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.096174038000000003 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.226386+0000", 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109551 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.226496+0000", 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8874000000000003e-05 2023-10-09T09:54:49.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:54:49.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:54:49.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:54:49.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:08.226585+0000", 2023-10-09T09:54:49.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.100400528 2023-10-09T09:54:49.318 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.318 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.318 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.318 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:54:49.318 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:54:49.318 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:54:49.343 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:49.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:49.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:49.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:49.473 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:49.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:49.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:49.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:49.604 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:49.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:49.709 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:49.734 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:49.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:49.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:49.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:49.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:49.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:49.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T09:54:49.969 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:49.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:49.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:49.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:50.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:50.129 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:50.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:50.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:50 smithi163 bash[78107]: cluster 2023-10-09T09:54:48.268647+0000 mgr.y (mgr.34107) 1468 : cluster 0 pgmap v2348: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:50.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:50 smithi163 bash[75425]: cluster 2023-10-09T09:54:48.268647+0000 mgr.y (mgr.34107) 1468 : cluster 0 pgmap v2348: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:50.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:50.247 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:50 smithi181 bash[59345]: cluster 2023-10-09T09:54:48.268647+0000 mgr.y (mgr.34107) 1468 : cluster 0 pgmap v2348: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:50.266 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:50.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:50.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:50.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:50.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:50.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:50.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:50.505 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:50.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:50.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:50.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:50.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:50.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:50.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:50.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:50.798 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:50.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:50.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:50.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:50.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:50.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:50.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:51.031 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:51.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:51.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:51.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:51.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:51.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:51.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:51.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:51.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:51.327 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:51.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:51.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:51.432 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:51.464 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:51.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:51.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:51.569 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:51.603 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:51.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:51.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:51.707 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:51.737 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:51.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:51.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:51.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:51.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:51.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:51.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:52.006 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:52.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:52.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:52.109 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:52.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:52 smithi163 bash[78107]: cluster 2023-10-09T09:54:50.269467+0000 mgr.y (mgr.34107) 1469 : cluster 0 pgmap v2349: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:52.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:52 smithi163 bash[75425]: cluster 2023-10-09T09:54:50.269467+0000 mgr.y (mgr.34107) 1469 : cluster 0 pgmap v2349: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:52.140 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:52.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:52.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:52.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:52.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:52.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:52.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:52 smithi181 bash[59345]: cluster 2023-10-09T09:54:50.269467+0000 mgr.y (mgr.34107) 1469 : cluster 0 pgmap v2349: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:52.369 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:54:52.378 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:52.393 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:54:52.413 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:52.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:52.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:52.423 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:54:52.452 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:54:52.479 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:54:52.508 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:54:52.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:52.534 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:54:52.549 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:52.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:52.561 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:54:52.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:54:52.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:54:52.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:54:52.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:54:52.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:54:52.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:54:52.564 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:54:52.564 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:54:52.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:52.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:52.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:52.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:52.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:52.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:52.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:52.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:52.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:52.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:52.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:53.056 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:53.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:53.086 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_6' removed 2023-10-09T09:54:53.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:53.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:53.095 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:54:53.191 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:53.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:53.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:53.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:53.313 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:53 smithi181 bash[59345]: audit 2023-10-09T09:54:52.277477+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.163:0/1458495921' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:54:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:53 smithi181 bash[59345]: audit 2023-10-09T09:54:52.277999+0000 mon.a (mon.0) 2573 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:54:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:53 smithi181 bash[59345]: audit 2023-10-09T09:54:52.340933+0000 mon.a (mon.0) 2574 : audit 0 from='client.? 172.21.15.163:0/1452457163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:54:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:53 smithi181 bash[59345]: audit 2023-10-09T09:54:52.669539+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.163:0/3600141135' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:54:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:53 smithi181 bash[59345]: audit 2023-10-09T09:54:52.669931+0000 mon.a (mon.0) 2575 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:54:53.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:53.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:53.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:53.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:53.458 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[78107]: audit 2023-10-09T09:54:52.277477+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.163:0/1458495921' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:54:53.458 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[78107]: audit 2023-10-09T09:54:52.277999+0000 mon.a (mon.0) 2573 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:54:53.458 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[78107]: audit 2023-10-09T09:54:52.340933+0000 mon.a (mon.0) 2574 : audit 0 from='client.? 172.21.15.163:0/1452457163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:54:53.458 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[78107]: audit 2023-10-09T09:54:52.669539+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.163:0/3600141135' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:54:53.458 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[78107]: audit 2023-10-09T09:54:52.669931+0000 mon.a (mon.0) 2575 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:54:53.458 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[75425]: audit 2023-10-09T09:54:52.277477+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.163:0/1458495921' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:54:53.458 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[75425]: audit 2023-10-09T09:54:52.277999+0000 mon.a (mon.0) 2573 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T09:54:53.459 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[75425]: audit 2023-10-09T09:54:52.340933+0000 mon.a (mon.0) 2574 : audit 0 from='client.? 172.21.15.163:0/1452457163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:54:53.459 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[75425]: audit 2023-10-09T09:54:52.669539+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.163:0/3600141135' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:54:53.459 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[75425]: audit 2023-10-09T09:54:52.669931+0000 mon.a (mon.0) 2575 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:54:53.474 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:53.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:53.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:53.577 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:53.621 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:53.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:53.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:54:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:54:53] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.33.4" 2023-10-09T09:54:53.726 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:53.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:53.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:53.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:53.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:53.884 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:53.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:53.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:53.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:54.012 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:54.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:54.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:54.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:54.137 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:54.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:54.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:54.240 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:54.254 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:54 smithi181 bash[59345]: cluster 2023-10-09T09:54:52.270358+0000 mgr.y (mgr.34107) 1470 : cluster 0 pgmap v2350: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:54.255 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:54 smithi181 bash[59345]: cluster 2023-10-09T09:54:53.051057+0000 mon.a (mon.0) 2576 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:54:54.255 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:54 smithi181 bash[59345]: cluster 2023-10-09T09:54:53.061238+0000 osd.5 (osd.5) 879 : cluster 3 failed to encode map e1289 with expected crc 2023-10-09T09:54:54.255 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:54 smithi181 bash[59345]: audit 2023-10-09T09:54:53.068047+0000 mon.a (mon.0) 2577 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:54:54.255 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:54 smithi181 bash[59345]: audit 2023-10-09T09:54:53.068176+0000 mon.a (mon.0) 2578 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:54:54.255 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:54 smithi181 bash[59345]: cluster 2023-10-09T09:54:53.068250+0000 mon.a (mon.0) 2579 : cluster 0 osdmap e1289: 8 total, 8 up, 6 in 2023-10-09T09:54:54.255 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:54 smithi181 bash[59345]: cluster 2023-10-09T09:54:53.177848+0000 osd.7 (osd.7) 621 : cluster 3 failed to encode map e1289 with expected crc 2023-10-09T09:54:54.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:54.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:54.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:54.378 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:54.409 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:54.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:54.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[75425]: cluster 2023-10-09T09:54:52.270358+0000 mgr.y (mgr.34107) 1470 : cluster 0 pgmap v2350: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[75425]: cluster 2023-10-09T09:54:53.051057+0000 mon.a (mon.0) 2576 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:54:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[75425]: cluster 2023-10-09T09:54:53.061238+0000 osd.5 (osd.5) 879 : cluster 3 failed to encode map e1289 with expected crc 2023-10-09T09:54:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[75425]: audit 2023-10-09T09:54:53.068047+0000 mon.a (mon.0) 2577 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:54:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[75425]: audit 2023-10-09T09:54:53.068176+0000 mon.a (mon.0) 2578 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:54:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[75425]: cluster 2023-10-09T09:54:53.068250+0000 mon.a (mon.0) 2579 : cluster 0 osdmap e1289: 8 total, 8 up, 6 in 2023-10-09T09:54:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[75425]: cluster 2023-10-09T09:54:53.177848+0000 osd.7 (osd.7) 621 : cluster 3 failed to encode map e1289 with expected crc 2023-10-09T09:54:54.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[78107]: cluster 2023-10-09T09:54:52.270358+0000 mgr.y (mgr.34107) 1470 : cluster 0 pgmap v2350: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:54.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[78107]: cluster 2023-10-09T09:54:53.051057+0000 mon.a (mon.0) 2576 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:54:54.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[78107]: cluster 2023-10-09T09:54:53.061238+0000 osd.5 (osd.5) 879 : cluster 3 failed to encode map e1289 with expected crc 2023-10-09T09:54:54.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[78107]: audit 2023-10-09T09:54:53.068047+0000 mon.a (mon.0) 2577 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T09:54:54.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[78107]: audit 2023-10-09T09:54:53.068176+0000 mon.a (mon.0) 2578 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:54:54.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[78107]: cluster 2023-10-09T09:54:53.068250+0000 mon.a (mon.0) 2579 : cluster 0 osdmap e1289: 8 total, 8 up, 6 in 2023-10-09T09:54:54.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:54 smithi163 bash[78107]: cluster 2023-10-09T09:54:53.177848+0000 osd.7 (osd.7) 621 : cluster 3 failed to encode map e1289 with expected crc 2023-10-09T09:54:54.513 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:54.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:54.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:54.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:54.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:54.682 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:54.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:54.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:54.727 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:54:54.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:54.817 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:54.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:54.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:54.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:54.951 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:54.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:54.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:55.053 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:55.080 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:55.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:55.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:55.183 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:55.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:55.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:55.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:55.246 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T09:54:55.247 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T09:54:55.247 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T09:54:55.251 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T09:54:55.251 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T09:54:55.251 DEBUG:teuthology.orchestra.run.smithi163:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:54:55.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:55.324 INFO:teuthology.orchestra.run.smithi163.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T09:54:55.324 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:54:55.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:55.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:55.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:55.380 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T09:54:55.380 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T09:54:55.433 INFO:tasks.workunit:timeout=3h 2023-10-09T09:54:55.434 INFO:tasks.workunit:cleanup=True 2023-10-09T09:54:55.434 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T09:54:55.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:55.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:55 smithi163 bash[75425]: cluster 2023-10-09T09:54:53.436052+0000 osd.6 (osd.6) 566 : cluster 3 failed to encode map e1289 with expected crc 2023-10-09T09:54:55.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:55 smithi163 bash[78107]: cluster 2023-10-09T09:54:53.436052+0000 osd.6 (osd.6) 566 : cluster 3 failed to encode map e1289 with expected crc 2023-10-09T09:54:55.496 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:55.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:55.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:55.535 INFO:tasks.workunit.client.0.smithi163.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T09:54:55.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:55 smithi181 bash[59345]: cluster 2023-10-09T09:54:53.436052+0000 osd.6 (osd.6) 566 : cluster 3 failed to encode map e1289 with expected crc 2023-10-09T09:54:55.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:55.631 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:55.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:55.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:55.656 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:54:55.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:55.767 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:55.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:55.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:55.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:54:55.870 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:55.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:55.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:55.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:55.995 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:56.016 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:56.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:56.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:56.119 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:56.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:56.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:56.253 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:56.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:56.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:56.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:56.337 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:56 smithi181 bash[59345]: cluster 2023-10-09T09:54:54.271154+0000 mgr.y (mgr.34107) 1471 : cluster 0 pgmap v2352: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:56.338 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:56 smithi181 bash[59345]: audit 2023-10-09T09:54:55.418230+0000 mon.a (mon.0) 2580 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:56.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:56.412 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:56.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:56.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:56 smithi163 bash[78107]: cluster 2023-10-09T09:54:54.271154+0000 mgr.y (mgr.34107) 1471 : cluster 0 pgmap v2352: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:56 smithi163 bash[78107]: audit 2023-10-09T09:54:55.418230+0000 mon.a (mon.0) 2580 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:56 smithi163 bash[75425]: cluster 2023-10-09T09:54:54.271154+0000 mgr.y (mgr.34107) 1471 : cluster 0 pgmap v2352: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:56 smithi163 bash[75425]: audit 2023-10-09T09:54:55.418230+0000 mon.a (mon.0) 2580 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:56.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:56.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:56.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:56.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:56.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:56.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:56.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:56.777 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:56.805 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:56.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:56.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:56.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:56.943 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:56.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:57.046 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:57.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:57.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:57.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:57.182 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:57.205 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:57.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:57.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:57.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:54:57.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:57.337 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:57.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:57.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:57.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:57.458 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:57.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:57.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:57.561 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:57.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:57.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:57.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:57.693 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:57.723 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:54:57.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:57.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:54:57.826 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:57.856 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:57.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:57.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:57.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:57.990 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:57.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:57.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:58.095 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:58.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:54:58 smithi163 bash[75425]: cluster 2023-10-09T09:54:56.271772+0000 mgr.y (mgr.34107) 1472 : cluster 0 pgmap v2353: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:58.108 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:54:58 smithi163 bash[78107]: cluster 2023-10-09T09:54:56.271772+0000 mgr.y (mgr.34107) 1472 : cluster 0 pgmap v2353: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:58.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:54:58.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:58.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:54:58.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:58.255 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:54:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:58.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:54:58.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:58.361 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:54:58.361 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:54:58.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:58.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:54:58 smithi181 bash[59345]: cluster 2023-10-09T09:54:56.271772+0000 mgr.y (mgr.34107) 1472 : cluster 0 pgmap v2353: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:54:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:58.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:58.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:58.515 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:58.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:58.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:58.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:58.651 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:58.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:58.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:58.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:58.782 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:58.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:58.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:58.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:58.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:58.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:59.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:59.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:59.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:59.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:59.145 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:59.152 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T09:54:59.169 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:59.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:59.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:59.272 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:59.302 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:54:59.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:59.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:54:59.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:59.436 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:54:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:59.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:54:59.540 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:59.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:54:59.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:59.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:54:59.673 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:59.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:59.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:59.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:54:59.805 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:59.838 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:54:59.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:59.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:54:59.920 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:54:59.923 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:54:59.941 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:59.975 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:54:59.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:54:59.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:00.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:00.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:00.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:00.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:00.211 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:00.223 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:00 smithi181 bash[59345]: cluster 2023-10-09T09:54:58.272437+0000 mgr.y (mgr.34107) 1473 : cluster 0 pgmap v2354: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:00.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:00.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:00.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:00.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:00.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:00 smithi163 bash[78107]: cluster 2023-10-09T09:54:58.272437+0000 mgr.y (mgr.34107) 1473 : cluster 0 pgmap v2354: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:00.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:00 smithi163 bash[75425]: cluster 2023-10-09T09:54:58.272437+0000 mgr.y (mgr.34107) 1473 : cluster 0 pgmap v2354: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:00.369 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:00.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:00.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:00.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:00.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:00.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:00.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:00.599 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:00.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:00.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:00.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:00.726 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:00.760 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:00.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:00.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:00.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:00.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:00.992 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:01.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:01.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:01.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:01.127 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:01.160 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:01.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:01.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:01 smithi163 bash[75425]: audit 2023-10-09T09:54:59.902439+0000 mgr.y (mgr.34107) 1474 : audit 0 from='client.45914 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:55:01.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:01 smithi163 bash[78107]: audit 2023-10-09T09:54:59.902439+0000 mgr.y (mgr.34107) 1474 : audit 0 from='client.45914 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:55:01.263 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:01.276 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:01 smithi181 bash[59345]: audit 2023-10-09T09:54:59.902439+0000 mgr.y (mgr.34107) 1474 : audit 0 from='client.45914 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:55:01.292 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:01.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:01.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:01.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:01.426 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:01.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:01.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:01.495 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2354,"stamp":"2023-10-09T09:54:58.271958+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":592828,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":229,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4818628,"kb_used_data":39636,"kb_used_omap":607,"kb_used_meta":4778336,"kb_avail":557652284,"statfs":{"total":575970213888,"available":571035938816,"internally_reserved":0,"allocated":40587264,"data_stored":35028952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":621657,"internal_metadata":4893016999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":113,"apply_latency_ms":113,"commit_latency_ns":113000000,"apply_latency_ns":113000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-214548,"num_write_kb":-6865473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004032"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":68,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400972+0000","last_change":"2023-10-09T09:48:07.991289+0000","last_active":"2023-10-09T09:54:44.400972+0000","last_peered":"2023-10-09T09:54:44.400972+0000","last_clean":"2023-10-09T09:54:44.400972+0000","last_became_active":"2023-10-09T09:48:07.990686+0000","last_became_peered":"2023-10-09T09:48:07.990686+0000","last_unstale":"2023-10-09T09:54:44.400972+0000","last_undegraded":"2023-10-09T09:54:44.400972+0000","last_fullsized":"2023-10-09T09:54:44.400972+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:24.421374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":171,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400334+0000","last_change":"2023-10-09T09:48:07.999947+0000","last_active":"2023-10-09T09:54:44.400334+0000","last_peered":"2023-10-09T09:54:44.400334+0000","last_clean":"2023-10-09T09:54:44.400334+0000","last_became_active":"2023-10-09T09:48:07.999704+0000","last_became_peered":"2023-10-09T09:48:07.999704+0000","last_unstale":"2023-10-09T09:54:44.400334+0000","last_undegraded":"2023-10-09T09:54:44.400334+0000","last_fullsized":"2023-10-09T09:54:44.400334+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:14.832042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1240,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399677+0000","last_change":"2023-10-09T09:48:07.998214+0000","last_active":"2023-10-09T09:54:44.399677+0000","last_peered":"2023-10-09T09:54:44.399677+0000","last_clean":"2023-10-09T09:54:44.399677+0000","last_became_active":"2023-10-09T09:48:07.997544+0000","last_became_peered":"2023-10-09T09:48:07.997544+0000","last_unstale":"2023-10-09T09:54:44.399677+0000","last_undegraded":"2023-10-09T09:54:44.399677+0000","last_fullsized":"2023-10-09T09:54:44.399677+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.304000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1250,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217455+0000","last_change":"2023-10-09T09:48:07.980358+0000","last_active":"2023-10-09T09:54:44.217455+0000","last_peered":"2023-10-09T09:54:44.217455+0000","last_clean":"2023-10-09T09:54:44.217455+0000","last_became_active":"2023-10-09T09:48:07.979229+0000","last_became_peered":"2023-10-09T09:48:07.979229+0000","last_unstale":"2023-10-09T09:54:44.217455+0000","last_undegraded":"2023-10-09T09:54:44.217455+0000","last_fullsized":"2023-10-09T09:54:44.217455+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:31.456347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":784,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.216413+0000","last_change":"2023-10-09T09:48:07.969728+0000","last_active":"2023-10-09T09:54:44.216413+0000","last_peered":"2023-10-09T09:54:44.216413+0000","last_clean":"2023-10-09T09:54:44.216413+0000","last_became_active":"2023-10-09T09:48:07.968773+0000","last_became_peered":"2023-10-09T09:48:07.968773+0000","last_unstale":"2023-10-09T09:54:44.216413+0000","last_undegraded":"2023-10-09T09:54:44.216413+0000","last_fullsized":"2023-10-09T09:54:44.216413+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:39.023317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1253,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036471+0000","last_change":"2023-10-09T09:48:07.988928+0000","last_active":"2023-10-09T09:54:44.036471+0000","last_peered":"2023-10-09T09:54:44.036471+0000","last_clean":"2023-10-09T09:54:44.036471+0000","last_became_active":"2023-10-09T09:48:07.988051+0000","last_became_peered":"2023-10-09T09:48:07.988051+0000","last_unstale":"2023-10-09T09:54:44.036471+0000","last_undegraded":"2023-10-09T09:54:44.036471+0000","last_fullsized":"2023-10-09T09:54:44.036471+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1212,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034547+0000","last_change":"2023-10-09T09:43:48.902066+0000","last_active":"2023-10-09T09:54:44.034547+0000","last_peered":"2023-10-09T09:54:44.034547+0000","last_clean":"2023-10-09T09:54:44.034547+0000","last_became_active":"2023-10-09T09:43:48.901639+0000","last_became_peered":"2023-10-09T09:43:48.901639+0000","last_unstale":"2023-10-09T09:54:44.034547+0000","last_undegraded":"2023-10-09T09:54:44.034547+0000","last_fullsized":"2023-10-09T09:54:44.034547+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":682,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399462+0000","last_change":"2023-10-09T09:48:07.997787+0000","last_active":"2023-10-09T09:54:44.399462+0000","last_peered":"2023-10-09T09:54:44.399462+0000","last_clean":"2023-10-09T09:54:44.399462+0000","last_became_active":"2023-10-09T09:48:07.997356+0000","last_became_peered":"2023-10-09T09:48:07.997356+0000","last_unstale":"2023-10-09T09:54:44.399462+0000","last_undegraded":"2023-10-09T09:54:44.399462+0000","last_fullsized":"2023-10-09T09:54:44.399462+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:03.024096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":155,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034155+0000","last_change":"2023-10-09T09:48:07.975547+0000","last_active":"2023-10-09T09:54:44.034155+0000","last_peered":"2023-10-09T09:54:44.034155+0000","last_clean":"2023-10-09T09:54:44.034155+0000","last_became_active":"2023-10-09T09:48:07.975150+0000","last_became_peered":"2023-10-09T09:48:07.975150+0000","last_unstale":"2023-10-09T09:54:44.034155+0000","last_undegraded":"2023-10-09T09:54:44.034155+0000","last_fullsized":"2023-10-09T09:54:44.034155+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":704,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036405+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:54:44.036405+0000","last_peered":"2023-10-09T09:54:44.036405+0000","last_clean":"2023-10-09T09:54:44.036405+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:54:44.036405+0000","last_undegraded":"2023-10-09T09:54:44.036405+0000","last_fullsized":"2023-10-09T09:54:44.036405+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":97,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035611+0000","last_change":"2023-10-09T09:43:48.906785+0000","last_active":"2023-10-09T09:54:44.035611+0000","last_peered":"2023-10-09T09:54:44.035611+0000","last_clean":"2023-10-09T09:54:44.035611+0000","last_became_active":"2023-10-09T09:43:48.906276+0000","last_became_peered":"2023-10-09T09:43:48.906276+0000","last_unstale":"2023-10-09T09:54:44.035611+0000","last_undegraded":"2023-10-09T09:54:44.035611+0000","last_fullsized":"2023-10-09T09:54:44.035611+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":121,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.033330+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:54:44.033330+0000","last_peered":"2023-10-09T09:54:44.033330+0000","last_clean":"2023-10-09T09:54:44.033330+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:54:44.033330+0000","last_undegraded":"2023-10-09T09:54:44.033330+0000","last_fullsized":"2023-10-09T09:54:44.033330+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":445,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035159+0000","last_change":"2023-10-09T09:43:23.995526+0000","last_active":"2023-10-09T09:54:44.035159+0000","last_peered":"2023-10-09T09:54:44.035159+0000","last_clean":"2023-10-09T09:54:44.035159+0000","last_became_active":"2023-10-09T09:43:23.994816+0000","last_became_peered":"2023-10-09T09:43:23.994816+0000","last_unstale":"2023-10-09T09:54:44.035159+0000","last_undegraded":"2023-10-09T09:54:44.035159+0000","last_fullsized":"2023-10-09T09:54:44.035159+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":662,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035975+0000","last_change":"2023-10-09T09:48:07.987637+0000","last_active":"2023-10-09T09:54:44.035975+0000","last_peered":"2023-10-09T09:54:44.035975+0000","last_clean":"2023-10-09T09:54:44.035975+0000","last_became_active":"2023-10-09T09:48:07.987085+0000","last_became_peered":"2023-10-09T09:48:07.987085+0000","last_unstale":"2023-10-09T09:54:44.035975+0000","last_undegraded":"2023-10-09T09:54:44.035975+0000","last_fullsized":"2023-10-09T09:54:44.035975+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1185,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.398534+0000","last_change":"2023-10-09T09:43:23.989356+0000","last_active":"2023-10-09T09:54:44.398534+0000","last_peered":"2023-10-09T09:54:44.398534+0000","last_clean":"2023-10-09T09:54:44.398534+0000","last_became_active":"2023-10-09T09:43:23.988986+0000","last_became_peered":"2023-10-09T09:43:23.988986+0000","last_unstale":"2023-10-09T09:54:44.398534+0000","last_undegraded":"2023-10-09T09:54:44.398534+0000","last_fullsized":"2023-10-09T09:54:44.398534+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.570448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":767,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.401585+0000","last_change":"2023-10-09T09:48:07.984006+0000","last_active":"2023-10-09T09:54:44.401585+0000","last_peered":"2023-10-09T09:54:44.401585+0000","last_clean":"2023-10-09T09:54:44.401585+0000","last_became_active":"2023-10-09T09:48:07.978179+0000","last_became_peered":"2023-10-09T09:48:07.978179+0000","last_unstale":"2023-10-09T09:54:44.401585+0000","last_undegraded":"2023-10-09T09:54:44.401585+0000","last_fullsized":"2023-10-09T09:54:44.401585+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:34.164004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1188,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.401113+0000","last_change":"2023-10-09T09:48:07.975335+0000","last_active":"2023-10-09T09:54:44.401113+0000","last_peered":"2023-10-09T09:54:44.401113+0000","last_clean":"2023-10-09T09:54:44.401113+0000","last_became_active":"2023-10-09T09:48:07.974437+0000","last_became_peered":"2023-10-09T09:48:07.974437+0000","last_unstale":"2023-10-09T09:54:44.401113+0000","last_undegraded":"2023-10-09T09:54:44.401113+0000","last_fullsized":"2023-10-09T09:54:44.401113+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:32.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":125,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036835+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:54:44.036835+0000","last_peered":"2023-10-09T09:54:44.036835+0000","last_clean":"2023-10-09T09:54:44.036835+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:54:44.036835+0000","last_undegraded":"2023-10-09T09:54:44.036835+0000","last_fullsized":"2023-10-09T09:54:44.036835+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":89,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035683+0000","last_change":"2023-10-09T09:43:23.994784+0000","last_active":"2023-10-09T09:54:44.035683+0000","last_peered":"2023-10-09T09:54:44.035683+0000","last_clean":"2023-10-09T09:54:44.035683+0000","last_became_active":"2023-10-09T09:43:23.994196+0000","last_became_peered":"2023-10-09T09:43:23.994196+0000","last_unstale":"2023-10-09T09:54:44.035683+0000","last_undegraded":"2023-10-09T09:54:44.035683+0000","last_fullsized":"2023-10-09T09:54:44.035683+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1206,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399686+0000","last_change":"2023-10-09T09:43:48.901303+0000","last_active":"2023-10-09T09:54:44.399686+0000","last_peered":"2023-10-09T09:54:44.399686+0000","last_clean":"2023-10-09T09:54:44.399686+0000","last_became_active":"2023-10-09T09:43:48.900779+0000","last_became_peered":"2023-10-09T09:43:48.900779+0000","last_unstale":"2023-10-09T09:54:44.399686+0000","last_undegraded":"2023-10-09T09:54:44.399686+0000","last_fullsized":"2023-10-09T09:54:44.399686+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:51.853531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1283,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034638+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:54:44.034638+0000","last_peered":"2023-10-09T09:54:44.034638+0000","last_clean":"2023-10-09T09:54:44.034638+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:54:44.034638+0000","last_undegraded":"2023-10-09T09:54:44.034638+0000","last_fullsized":"2023-10-09T09:54:44.034638+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1250,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.401076+0000","last_change":"2023-10-09T09:48:08.000694+0000","last_active":"2023-10-09T09:54:44.401076+0000","last_peered":"2023-10-09T09:54:44.401076+0000","last_clean":"2023-10-09T09:54:44.401076+0000","last_became_active":"2023-10-09T09:48:07.999476+0000","last_became_peered":"2023-10-09T09:48:07.999476+0000","last_unstale":"2023-10-09T09:54:44.401076+0000","last_undegraded":"2023-10-09T09:54:44.401076+0000","last_fullsized":"2023-10-09T09:54:44.401076+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:37.211076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":117,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217596+0000","last_change":"2023-10-09T09:48:07.976558+0000","last_active":"2023-10-09T09:54:44.217596+0000","last_peered":"2023-10-09T09:54:44.217596+0000","last_clean":"2023-10-09T09:54:44.217596+0000","last_became_active":"2023-10-09T09:48:07.975961+0000","last_became_peered":"2023-10-09T09:48:07.975961+0000","last_unstale":"2023-10-09T09:54:44.217596+0000","last_undegraded":"2023-10-09T09:54:44.217596+0000","last_fullsized":"2023-10-09T09:54:44.217596+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:40.925248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":131,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.032754+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:54:44.032754+0000","last_peered":"2023-10-09T09:54:44.032754+0000","last_clean":"2023-10-09T09:54:44.032754+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:54:44.032754+0000","last_undegraded":"2023-10-09T09:54:44.032754+0000","last_fullsized":"2023-10-09T09:54:44.032754+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":80,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:52.023668+0000","last_change":"2023-10-09T09:48:07.990711+0000","last_active":"2023-10-09T09:54:52.023668+0000","last_peered":"2023-10-09T09:54:52.023668+0000","last_clean":"2023-10-09T09:54:52.023668+0000","last_became_active":"2023-10-09T09:48:07.990267+0000","last_became_peered":"2023-10-09T09:48:07.990267+0000","last_unstale":"2023-10-09T09:54:52.023668+0000","last_undegraded":"2023-10-09T09:54:52.023668+0000","last_fullsized":"2023-10-09T09:54:52.023668+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:51.244840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1224,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399845+0000","last_change":"2023-10-09T09:48:07.991440+0000","last_active":"2023-10-09T09:54:44.399845+0000","last_peered":"2023-10-09T09:54:44.399845+0000","last_clean":"2023-10-09T09:54:44.399845+0000","last_became_active":"2023-10-09T09:48:07.991072+0000","last_became_peered":"2023-10-09T09:48:07.991072+0000","last_unstale":"2023-10-09T09:54:44.399845+0000","last_undegraded":"2023-10-09T09:54:44.399845+0000","last_fullsized":"2023-10-09T09:54:44.399845+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:28.577159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1239,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034132+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:54:44.034132+0000","last_peered":"2023-10-09T09:54:44.034132+0000","last_clean":"2023-10-09T09:54:44.034132+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:54:44.034132+0000","last_undegraded":"2023-10-09T09:54:44.034132+0000","last_fullsized":"2023-10-09T09:54:44.034132+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1245,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034994+0000","last_change":"2023-10-09T09:48:07.987819+0000","last_active":"2023-10-09T09:54:44.034994+0000","last_peered":"2023-10-09T09:54:44.034994+0000","last_clean":"2023-10-09T09:54:44.034994+0000","last_became_active":"2023-10-09T09:48:07.986354+0000","last_became_peered":"2023-10-09T09:48:07.986354+0000","last_unstale":"2023-10-09T09:54:44.034994+0000","last_undegraded":"2023-10-09T09:54:44.034994+0000","last_fullsized":"2023-10-09T09:54:44.034994+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":791,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035547+0000","last_change":"2023-10-09T09:48:07.987506+0000","last_active":"2023-10-09T09:54:44.035547+0000","last_peered":"2023-10-09T09:54:44.035547+0000","last_clean":"2023-10-09T09:54:44.035547+0000","last_became_active":"2023-10-09T09:48:07.986988+0000","last_became_peered":"2023-10-09T09:48:07.986988+0000","last_unstale":"2023-10-09T09:54:44.035547+0000","last_undegraded":"2023-10-09T09:54:44.035547+0000","last_fullsized":"2023-10-09T09:54:44.035547+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1209,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217182+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:54:44.217182+0000","last_peered":"2023-10-09T09:54:44.217182+0000","last_clean":"2023-10-09T09:54:44.217182+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:54:44.217182+0000","last_undegraded":"2023-10-09T09:54:44.217182+0000","last_fullsized":"2023-10-09T09:54:44.217182+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":168,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:52.023551+0000","last_change":"2023-10-09T09:48:07.977143+0000","last_active":"2023-10-09T09:54:52.023551+0000","last_peered":"2023-10-09T09:54:52.023551+0000","last_clean":"2023-10-09T09:54:52.023551+0000","last_became_active":"2023-10-09T09:48:07.976338+0000","last_became_peered":"2023-10-09T09:48:07.976338+0000","last_unstale":"2023-10-09T09:54:52.023551+0000","last_undegraded":"2023-10-09T09:54:52.023551+0000","last_fullsized":"2023-10-09T09:54:52.023551+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:12.015053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1241,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035385+0000","last_change":"2023-10-09T09:48:08.001503+0000","last_active":"2023-10-09T09:54:44.035385+0000","last_peered":"2023-10-09T09:54:44.035385+0000","last_clean":"2023-10-09T09:54:44.035385+0000","last_became_active":"2023-10-09T09:48:08.000921+0000","last_became_peered":"2023-10-09T09:48:08.000921+0000","last_unstale":"2023-10-09T09:54:44.035385+0000","last_undegraded":"2023-10-09T09:54:44.035385+0000","last_fullsized":"2023-10-09T09:54:44.035385+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1237,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036275+0000","last_change":"2023-10-09T09:48:07.988586+0000","last_active":"2023-10-09T09:54:44.036275+0000","last_peered":"2023-10-09T09:54:44.036275+0000","last_clean":"2023-10-09T09:54:44.036275+0000","last_became_active":"2023-10-09T09:48:07.988226+0000","last_became_peered":"2023-10-09T09:48:07.988226+0000","last_unstale":"2023-10-09T09:54:44.036275+0000","last_undegraded":"2023-10-09T09:54:44.036275+0000","last_fullsized":"2023-10-09T09:54:44.036275+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1230,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.398948+0000","last_change":"2023-10-09T09:48:07.987993+0000","last_active":"2023-10-09T09:54:44.398948+0000","last_peered":"2023-10-09T09:54:44.398948+0000","last_clean":"2023-10-09T09:54:44.398948+0000","last_became_active":"2023-10-09T09:48:07.987664+0000","last_became_peered":"2023-10-09T09:48:07.987664+0000","last_unstale":"2023-10-09T09:54:44.398948+0000","last_undegraded":"2023-10-09T09:54:44.398948+0000","last_fullsized":"2023-10-09T09:54:44.398948+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:32.253321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1258,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399116+0000","last_change":"2023-10-09T09:48:07.984079+0000","last_active":"2023-10-09T09:54:44.399116+0000","last_peered":"2023-10-09T09:54:44.399116+0000","last_clean":"2023-10-09T09:54:44.399116+0000","last_became_active":"2023-10-09T09:48:07.983792+0000","last_became_peered":"2023-10-09T09:48:07.983792+0000","last_unstale":"2023-10-09T09:54:44.399116+0000","last_undegraded":"2023-10-09T09:54:44.399116+0000","last_fullsized":"2023-10-09T09:54:44.399116+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:28:42.289832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":662,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035945+0000","last_change":"2023-10-09T09:48:07.989047+0000","last_active":"2023-10-09T09:54:44.035945+0000","last_peered":"2023-10-09T09:54:44.035945+0000","last_clean":"2023-10-09T09:54:44.035945+0000","last_became_active":"2023-10-09T09:48:07.988131+0000","last_became_peered":"2023-10-09T09:48:07.988131+0000","last_unstale":"2023-10-09T09:54:44.035945+0000","last_undegraded":"2023-10-09T09:54:44.035945+0000","last_fullsized":"2023-10-09T09:54:44.035945+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1200,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400105+0000","last_change":"2023-10-09T09:48:07.976884+0000","last_active":"2023-10-09T09:54:44.400105+0000","last_peered":"2023-10-09T09:54:44.400105+0000","last_clean":"2023-10-09T09:54:44.400105+0000","last_became_active":"2023-10-09T09:48:07.975932+0000","last_became_peered":"2023-10-09T09:48:07.975932+0000","last_unstale":"2023-10-09T09:54:44.400105+0000","last_undegraded":"2023-10-09T09:54:44.400105+0000","last_fullsized":"2023-10-09T09:54:44.400105+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:22.785728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":136,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399422+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:54:44.399422+0000","last_peered":"2023-10-09T09:54:44.399422+0000","last_clean":"2023-10-09T09:54:44.399422+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:54:44.399422+0000","last_undegraded":"2023-10-09T09:54:44.399422+0000","last_fullsized":"2023-10-09T09:54:44.399422+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1246,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.401507+0000","last_change":"2023-10-09T09:48:07.984553+0000","last_active":"2023-10-09T09:54:44.401507+0000","last_peered":"2023-10-09T09:54:44.401507+0000","last_clean":"2023-10-09T09:54:44.401507+0000","last_became_active":"2023-10-09T09:48:07.983302+0000","last_became_peered":"2023-10-09T09:48:07.983302+0000","last_unstale":"2023-10-09T09:54:44.401507+0000","last_undegraded":"2023-10-09T09:54:44.401507+0000","last_fullsized":"2023-10-09T09:54:44.401507+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:50.714295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":131,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.033909+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:54:44.033909+0000","last_peered":"2023-10-09T09:54:44.033909+0000","last_clean":"2023-10-09T09:54:44.033909+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:54:44.033909+0000","last_undegraded":"2023-10-09T09:54:44.033909+0000","last_fullsized":"2023-10-09T09:54:44.033909+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1223,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399707+0000","last_change":"2023-10-09T09:48:07.977683+0000","last_active":"2023-10-09T09:54:44.399707+0000","last_peered":"2023-10-09T09:54:44.399707+0000","last_clean":"2023-10-09T09:54:44.399707+0000","last_became_active":"2023-10-09T09:48:07.977024+0000","last_became_peered":"2023-10-09T09:48:07.977024+0000","last_unstale":"2023-10-09T09:54:44.399707+0000","last_undegraded":"2023-10-09T09:54:44.399707+0000","last_fullsized":"2023-10-09T09:54:44.399707+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:13.438431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":706,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400727+0000","last_change":"2023-10-09T09:48:07.997811+0000","last_active":"2023-10-09T09:54:44.400727+0000","last_peered":"2023-10-09T09:54:44.400727+0000","last_clean":"2023-10-09T09:54:44.400727+0000","last_became_active":"2023-10-09T09:48:07.997389+0000","last_became_peered":"2023-10-09T09:48:07.997389+0000","last_unstale":"2023-10-09T09:54:44.400727+0000","last_undegraded":"2023-10-09T09:54:44.400727+0000","last_fullsized":"2023-10-09T09:54:44.400727+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:21.978398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1179,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036322+0000","last_change":"2023-10-09T09:43:48.908190+0000","last_active":"2023-10-09T09:54:44.036322+0000","last_peered":"2023-10-09T09:54:44.036322+0000","last_clean":"2023-10-09T09:54:44.036322+0000","last_became_active":"2023-10-09T09:43:48.907799+0000","last_became_peered":"2023-10-09T09:43:48.907799+0000","last_unstale":"2023-10-09T09:54:44.036322+0000","last_undegraded":"2023-10-09T09:54:44.036322+0000","last_fullsized":"2023-10-09T09:54:44.036322+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1276,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400635+0000","last_change":"2023-10-09T09:48:08.000154+0000","last_active":"2023-10-09T09:54:44.400635+0000","last_peered":"2023-10-09T09:54:44.400635+0000","last_clean":"2023-10-09T09:54:44.400635+0000","last_became_active":"2023-10-09T09:48:07.999714+0000","last_became_peered":"2023-10-09T09:48:07.999714+0000","last_unstale":"2023-10-09T09:54:44.400635+0000","last_undegraded":"2023-10-09T09:54:44.400635+0000","last_fullsized":"2023-10-09T09:54:44.400635+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:12.921693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":181,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399290+0000","last_change":"2023-10-09T09:48:07.983969+0000","last_active":"2023-10-09T09:54:44.399290+0000","last_peered":"2023-10-09T09:54:44.399290+0000","last_clean":"2023-10-09T09:54:44.399290+0000","last_became_active":"2023-10-09T09:48:07.983490+0000","last_became_peered":"2023-10-09T09:48:07.983490+0000","last_unstale":"2023-10-09T09:54:44.399290+0000","last_undegraded":"2023-10-09T09:54:44.399290+0000","last_fullsized":"2023-10-09T09:54:44.399290+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:21.122987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1254,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399596+0000","last_change":"2023-10-09T09:48:07.998234+0000","last_active":"2023-10-09T09:54:44.399596+0000","last_peered":"2023-10-09T09:54:44.399596+0000","last_clean":"2023-10-09T09:54:44.399596+0000","last_became_active":"2023-10-09T09:48:07.997909+0000","last_became_peered":"2023-10-09T09:48:07.997909+0000","last_unstale":"2023-10-09T09:54:44.399596+0000","last_undegraded":"2023-10-09T09:54:44.399596+0000","last_fullsized":"2023-10-09T09:54:44.399596+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:21.750617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1243,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400587+0000","last_change":"2023-10-09T09:48:07.973284+0000","last_active":"2023-10-09T09:54:44.400587+0000","last_peered":"2023-10-09T09:54:44.400587+0000","last_clean":"2023-10-09T09:54:44.400587+0000","last_became_active":"2023-10-09T09:48:07.971235+0000","last_became_peered":"2023-10-09T09:48:07.971235+0000","last_unstale":"2023-10-09T09:54:44.400587+0000","last_undegraded":"2023-10-09T09:54:44.400587+0000","last_fullsized":"2023-10-09T09:54:44.400587+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:40.951420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1299,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.398831+0000","last_change":"2023-10-09T09:41:11.795802+0000","last_active":"2023-10-09T09:54:44.398831+0000","last_peered":"2023-10-09T09:54:44.398831+0000","last_clean":"2023-10-09T09:54:44.398831+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:54:44.398831+0000","last_undegraded":"2023-10-09T09:54:44.398831+0000","last_fullsized":"2023-10-09T09:54:44.398831+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":633,"reported_epoch":1289,"state":"active+clean","last_fresh":"2023-10-09T09:54:53.526821+0000","last_change":"2023-10-09T09:48:07.978435+0000","last_active":"2023-10-09T09:54:53.526821+0000","last_peered":"2023-10-09T09:54:53.526821+0000","last_clean":"2023-10-09T09:54:53.526821+0000","last_became_active":"2023-10-09T09:48:07.977417+0000","last_became_peered":"2023-10-09T09:48:07.977417+0000","last_unstale":"2023-10-09T09:54:53.526821+0000","last_undegraded":"2023-10-09T09:54:53.526821+0000","last_fullsized":"2023-10-09T09:54:53.526821+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:19.288202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":131,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.033240+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:54:44.033240+0000","last_peered":"2023-10-09T09:54:44.033240+0000","last_clean":"2023-10-09T09:54:44.033240+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:54:44.033240+0000","last_undegraded":"2023-10-09T09:54:44.033240+0000","last_fullsized":"2023-10-09T09:54:44.033240+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1215,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036469+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:54:44.036469+0000","last_peered":"2023-10-09T09:54:44.036469+0000","last_clean":"2023-10-09T09:54:44.036469+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:54:44.036469+0000","last_undegraded":"2023-10-09T09:54:44.036469+0000","last_fullsized":"2023-10-09T09:54:44.036469+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1220,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399734+0000","last_change":"2023-10-09T09:48:07.973246+0000","last_active":"2023-10-09T09:54:44.399734+0000","last_peered":"2023-10-09T09:54:44.399734+0000","last_clean":"2023-10-09T09:54:44.399734+0000","last_became_active":"2023-10-09T09:48:07.971221+0000","last_became_peered":"2023-10-09T09:48:07.971221+0000","last_unstale":"2023-10-09T09:54:44.399734+0000","last_undegraded":"2023-10-09T09:54:44.399734+0000","last_fullsized":"2023-10-09T09:54:44.399734+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:16.759514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1227,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.216793+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:54:44.216793+0000","last_peered":"2023-10-09T09:54:44.216793+0000","last_clean":"2023-10-09T09:54:44.216793+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:54:44.216793+0000","last_undegraded":"2023-10-09T09:54:44.216793+0000","last_fullsized":"2023-10-09T09:54:44.216793+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1239,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:49.148600+0000","last_change":"2023-10-09T09:48:07.981410+0000","last_active":"2023-10-09T09:54:49.148600+0000","last_peered":"2023-10-09T09:54:49.148600+0000","last_clean":"2023-10-09T09:54:49.148600+0000","last_became_active":"2023-10-09T09:48:07.979850+0000","last_became_peered":"2023-10-09T09:48:07.979850+0000","last_unstale":"2023-10-09T09:54:49.148600+0000","last_undegraded":"2023-10-09T09:54:49.148600+0000","last_fullsized":"2023-10-09T09:54:49.148600+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:31.201104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1214,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217239+0000","last_change":"2023-10-09T09:48:07.975786+0000","last_active":"2023-10-09T09:54:44.217239+0000","last_peered":"2023-10-09T09:54:44.217239+0000","last_clean":"2023-10-09T09:54:44.217239+0000","last_became_active":"2023-10-09T09:48:07.975338+0000","last_became_peered":"2023-10-09T09:48:07.975338+0000","last_unstale":"2023-10-09T09:54:44.217239+0000","last_undegraded":"2023-10-09T09:54:44.217239+0000","last_fullsized":"2023-10-09T09:54:44.217239+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:05.175956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1257,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.401287+0000","last_change":"2023-10-09T09:48:07.983773+0000","last_active":"2023-10-09T09:54:44.401287+0000","last_peered":"2023-10-09T09:54:44.401287+0000","last_clean":"2023-10-09T09:54:44.401287+0000","last_became_active":"2023-10-09T09:48:07.976869+0000","last_became_peered":"2023-10-09T09:48:07.976869+0000","last_unstale":"2023-10-09T09:54:44.401287+0000","last_undegraded":"2023-10-09T09:54:44.401287+0000","last_fullsized":"2023-10-09T09:54:44.401287+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:58.173236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1205,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.033269+0000","last_change":"2023-10-09T09:48:07.962302+0000","last_active":"2023-10-09T09:54:44.033269+0000","last_peered":"2023-10-09T09:54:44.033269+0000","last_clean":"2023-10-09T09:54:44.033269+0000","last_became_active":"2023-10-09T09:48:07.961888+0000","last_became_peered":"2023-10-09T09:48:07.961888+0000","last_unstale":"2023-10-09T09:54:44.033269+0000","last_undegraded":"2023-10-09T09:54:44.033269+0000","last_fullsized":"2023-10-09T09:54:44.033269+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":152,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217747+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:54:44.217747+0000","last_peered":"2023-10-09T09:54:44.217747+0000","last_clean":"2023-10-09T09:54:44.217747+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:54:44.217747+0000","last_undegraded":"2023-10-09T09:54:44.217747+0000","last_fullsized":"2023-10-09T09:54:44.217747+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1209,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.401008+0000","last_change":"2023-10-09T09:48:07.999034+0000","last_active":"2023-10-09T09:54:44.401008+0000","last_peered":"2023-10-09T09:54:44.401008+0000","last_clean":"2023-10-09T09:54:44.401008+0000","last_became_active":"2023-10-09T09:48:07.998606+0000","last_became_peered":"2023-10-09T09:48:07.998606+0000","last_unstale":"2023-10-09T09:54:44.401008+0000","last_undegraded":"2023-10-09T09:54:44.401008+0000","last_fullsized":"2023-10-09T09:54:44.401008+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.469476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1145,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399283+0000","last_change":"2023-10-09T09:48:07.976350+0000","last_active":"2023-10-09T09:54:44.399283+0000","last_peered":"2023-10-09T09:54:44.399283+0000","last_clean":"2023-10-09T09:54:44.399283+0000","last_became_active":"2023-10-09T09:48:07.973099+0000","last_became_peered":"2023-10-09T09:48:07.973099+0000","last_unstale":"2023-10-09T09:54:44.399283+0000","last_undegraded":"2023-10-09T09:54:44.399283+0000","last_fullsized":"2023-10-09T09:54:44.399283+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:48.812722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1276,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036611+0000","last_change":"2023-10-09T09:48:07.990005+0000","last_active":"2023-10-09T09:54:44.036611+0000","last_peered":"2023-10-09T09:54:44.036611+0000","last_clean":"2023-10-09T09:54:44.036611+0000","last_became_active":"2023-10-09T09:48:07.987124+0000","last_became_peered":"2023-10-09T09:48:07.987124+0000","last_unstale":"2023-10-09T09:54:44.036611+0000","last_undegraded":"2023-10-09T09:54:44.036611+0000","last_fullsized":"2023-10-09T09:54:44.036611+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1220,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:49.148639+0000","last_change":"2023-10-09T09:48:07.980600+0000","last_active":"2023-10-09T09:54:49.148639+0000","last_peered":"2023-10-09T09:54:49.148639+0000","last_clean":"2023-10-09T09:54:49.148639+0000","last_became_active":"2023-10-09T09:48:07.979544+0000","last_became_peered":"2023-10-09T09:48:07.979544+0000","last_unstale":"2023-10-09T09:54:49.148639+0000","last_undegraded":"2023-10-09T09:54:49.148639+0000","last_fullsized":"2023-10-09T09:54:49.148639+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:47.320689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1207,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.033139+0000","last_change":"2023-10-09T09:43:23.992213+0000","last_active":"2023-10-09T09:54:44.033139+0000","last_peered":"2023-10-09T09:54:44.033139+0000","last_clean":"2023-10-09T09:54:44.033139+0000","last_became_active":"2023-10-09T09:43:23.991887+0000","last_became_peered":"2023-10-09T09:43:23.991887+0000","last_unstale":"2023-10-09T09:54:44.033139+0000","last_undegraded":"2023-10-09T09:54:44.033139+0000","last_fullsized":"2023-10-09T09:54:44.033139+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1227,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217318+0000","last_change":"2023-10-09T09:48:07.969929+0000","last_active":"2023-10-09T09:54:44.217318+0000","last_peered":"2023-10-09T09:54:44.217318+0000","last_clean":"2023-10-09T09:54:44.217318+0000","last_became_active":"2023-10-09T09:48:07.969537+0000","last_became_peered":"2023-10-09T09:48:07.969537+0000","last_unstale":"2023-10-09T09:54:44.217318+0000","last_undegraded":"2023-10-09T09:54:44.217318+0000","last_fullsized":"2023-10-09T09:54:44.217318+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:42.344608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":180,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399951+0000","last_change":"2023-10-09T09:48:07.987618+0000","last_active":"2023-10-09T09:54:44.399951+0000","last_peered":"2023-10-09T09:54:44.399951+0000","last_clean":"2023-10-09T09:54:44.399951+0000","last_became_active":"2023-10-09T09:48:07.987341+0000","last_became_peered":"2023-10-09T09:48:07.987341+0000","last_unstale":"2023-10-09T09:54:44.399951+0000","last_undegraded":"2023-10-09T09:54:44.399951+0000","last_fullsized":"2023-10-09T09:54:44.399951+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:19.972891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":79,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400799+0000","last_change":"2023-10-09T09:48:07.984247+0000","last_active":"2023-10-09T09:54:44.400799+0000","last_peered":"2023-10-09T09:54:44.400799+0000","last_clean":"2023-10-09T09:54:44.400799+0000","last_became_active":"2023-10-09T09:48:07.978613+0000","last_became_peered":"2023-10-09T09:48:07.978613+0000","last_unstale":"2023-10-09T09:54:44.400799+0000","last_undegraded":"2023-10-09T09:54:44.400799+0000","last_fullsized":"2023-10-09T09:54:44.400799+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:34.663108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1223,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400299+0000","last_change":"2023-10-09T09:48:07.991053+0000","last_active":"2023-10-09T09:54:44.400299+0000","last_peered":"2023-10-09T09:54:44.400299+0000","last_clean":"2023-10-09T09:54:44.400299+0000","last_became_active":"2023-10-09T09:48:07.990656+0000","last_became_peered":"2023-10-09T09:48:07.990656+0000","last_unstale":"2023-10-09T09:54:44.400299+0000","last_undegraded":"2023-10-09T09:54:44.400299+0000","last_fullsized":"2023-10-09T09:54:44.400299+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:06.223305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":97,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036904+0000","last_change":"2023-10-09T09:43:23.995576+0000","last_active":"2023-10-09T09:54:44.036904+0000","last_peered":"2023-10-09T09:54:44.036904+0000","last_clean":"2023-10-09T09:54:44.036904+0000","last_became_active":"2023-10-09T09:43:23.995076+0000","last_became_peered":"2023-10-09T09:43:23.995076+0000","last_unstale":"2023-10-09T09:54:44.036904+0000","last_undegraded":"2023-10-09T09:54:44.036904+0000","last_fullsized":"2023-10-09T09:54:44.036904+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1259,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399113+0000","last_change":"2023-10-09T09:43:48.904021+0000","last_active":"2023-10-09T09:54:44.399113+0000","last_peered":"2023-10-09T09:54:44.399113+0000","last_clean":"2023-10-09T09:54:44.399113+0000","last_became_active":"2023-10-09T09:43:48.903719+0000","last_became_peered":"2023-10-09T09:43:48.903719+0000","last_unstale":"2023-10-09T09:54:44.399113+0000","last_undegraded":"2023-10-09T09:54:44.399113+0000","last_fullsized":"2023-10-09T09:54:44.399113+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:16.284027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":237,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035896+0000","last_change":"2023-10-09T09:48:08.001392+0000","last_active":"2023-10-09T09:54:44.035896+0000","last_peered":"2023-10-09T09:54:44.035896+0000","last_clean":"2023-10-09T09:54:44.035896+0000","last_became_active":"2023-10-09T09:48:08.000798+0000","last_became_peered":"2023-10-09T09:48:08.000798+0000","last_unstale":"2023-10-09T09:54:44.035896+0000","last_undegraded":"2023-10-09T09:54:44.035896+0000","last_fullsized":"2023-10-09T09:54:44.035896+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":165,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.398605+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:54:44.398605+0000","last_peered":"2023-10-09T09:54:44.398605+0000","last_clean":"2023-10-09T09:54:44.398605+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:54:44.398605+0000","last_undegraded":"2023-10-09T09:54:44.398605+0000","last_fullsized":"2023-10-09T09:54:44.398605+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1253,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217483+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:54:44.217483+0000","last_peered":"2023-10-09T09:54:44.217483+0000","last_clean":"2023-10-09T09:54:44.217483+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:54:44.217483+0000","last_undegraded":"2023-10-09T09:54:44.217483+0000","last_fullsized":"2023-10-09T09:54:44.217483+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":662,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034823+0000","last_change":"2023-10-09T09:48:07.990205+0000","last_active":"2023-10-09T09:54:44.034823+0000","last_peered":"2023-10-09T09:54:44.034823+0000","last_clean":"2023-10-09T09:54:44.034823+0000","last_became_active":"2023-10-09T09:48:07.989619+0000","last_became_peered":"2023-10-09T09:48:07.989619+0000","last_unstale":"2023-10-09T09:54:44.034823+0000","last_undegraded":"2023-10-09T09:54:44.034823+0000","last_fullsized":"2023-10-09T09:54:44.034823+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":2460,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30942,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453398158,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594391,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910952,"kb_used_data":6176,"kb_used_omap":0,"kb_used_meta":904768,"kb_avail":92834200,"statfs":{"total":95995035648,"available":95062220800,"internally_reserved":0,"allocated":6324224,"data_stored":6022068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926482432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:53:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":4.1779999999999999,"15min":2.5840000000000001},"min":{"1min":0.33300000000000002,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":5.4880000000000004,"5min":149.095,"15min":149.095},"last":0.877},{"interface":"front","average":{"1min":1.75,"5min":4.3460000000000001,"15min":2.899},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.29499999999999998},"max":{"1min":7.0860000000000003,"5min":149.17500000000001,"15min":149.17500000000001},"last":0.65700000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:54:18 2023","interfaces":[{"interface":"back","average":{"1min":5.452,"5min":2.3340000000000001,"15min":2.2690000000000001},"min":{"1min":0.40699999999999997,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":55.368000000000002,"5min":55.368000000000002,"15min":55.368000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":5.2629999999999999,"5min":2.3399999999999999,"15min":2.141},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":55.404000000000003,"5min":55.404000000000003,"15min":55.404000000000003},"last":1.075}]},{"osd":2,"last update":"Mon Oct 9 09:54:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8370000000000002,"5min":3.6179999999999999,"15min":3.5790000000000002},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.29799999999999999},"max":{"1min":23.547000000000001,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.68799999999999994},{"interface":"front","average":{"1min":3.23,"5min":4.032,"15min":3.7509999999999999},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.309},"max":{"1min":23.600999999999999,"5min":208.90899999999999,"15min":208.90899999999999},"last":0.85299999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:54:44 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":0.94199999999999995,"15min":1.1379999999999999},"min":{"1min":0.42999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.6559999999999997,"5min":5.5910000000000002,"15min":7.9139999999999997},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.96899999999999997,"15min":1.3520000000000001},"min":{"1min":0.41099999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":3.3500000000000001,"5min":4.1669999999999998,"15min":11.968999999999999},"last":0.91400000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:54:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.1359999999999999,"15min":1.3779999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.28399999999999997},"max":{"1min":7.0010000000000003,"5min":7.0010000000000003,"15min":12.148999999999999},"last":1.2869999999999999},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.98099999999999998,"15min":1.552},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.313},"max":{"1min":3.3170000000000002,"5min":6.7750000000000004,"15min":12.064},"last":1.0109999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:54:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":1.2150000000000001,"15min":1.446},"min":{"1min":0.433,"5min":0.318,"15min":0.318},"max":{"1min":7.3639999999999999,"5min":7.3639999999999999,"15min":10.529},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.02,"5min":1.02,"15min":1.5},"min":{"1min":0.48999999999999999,"5min":0.34300000000000003,"15min":0.34000000000000002},"max":{"1min":3.282,"5min":4.0629999999999997,"15min":12.000999999999999},"last":0.753}]},{"osd":7,"last update":"Mon Oct 9 09:54:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.1579999999999999,"15min":1.526},"min":{"1min":0.438,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.9740000000000002,"5min":9.6389999999999993,"15min":11.922000000000001},"last":1.143},{"interface":"front","average":{"1min":1.163,"5min":1.097,"15min":1.194},"min":{"1min":0.44900000000000001,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":4.6239999999999997,"5min":4.9459999999999997,"15min":7.9359999999999999},"last":1.262}]}]},{"osd":2,"up_from":634,"seq":2723009266144,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899492,"kb_used_data":7068,"kb_used_omap":122,"kb_used_meta":892293,"kb_avail":92845660,"statfs":{"total":95995035648,"available":95073955840,"internally_reserved":0,"allocated":7237632,"data_stored":5662939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":913708619},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:53:56 2023","interfaces":[{"interface":"back","average":{"1min":2.4249999999999998,"5min":4.6109999999999998,"15min":4.0270000000000001},"min":{"1min":0.223,"5min":0.221,"15min":0.20200000000000001},"max":{"1min":9.3339999999999996,"5min":218.30799999999999,"15min":218.30799999999999},"last":0.83199999999999996},{"interface":"front","average":{"1min":2.8570000000000002,"5min":4.7439999999999998,"15min":4.2199999999999998},"min":{"1min":0.23499999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":11.132,"5min":217.67500000000001,"15min":217.67500000000001},"last":0.64500000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:54:21 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":3.766,"15min":1.958},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":8.5709999999999997,"5min":106.089,"15min":106.089},"last":0.69999999999999996},{"interface":"front","average":{"1min":3.1989999999999998,"5min":3.6059999999999999,"15min":1.9790000000000001},"min":{"1min":0.311,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.2249999999999996,"5min":105.571,"15min":105.571},"last":0.86699999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:53:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1859999999999999,"5min":1.1799999999999999,"15min":2.0600000000000001},"min":{"1min":0.312,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":8.1289999999999996,"5min":8.1289999999999996,"15min":230.476},"last":0.751},{"interface":"front","average":{"1min":1.637,"5min":1.145,"15min":1.9530000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.2810000000000006,"5min":9.2810000000000006,"15min":228.893},"last":0.78200000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.323,"5min":1.276,"15min":1.415},"min":{"1min":0.51400000000000001,"5min":0.33100000000000002,"15min":0.27100000000000002},"max":{"1min":4.4660000000000002,"5min":7.9690000000000003,"15min":71.876000000000005},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.8520000000000001,"5min":1.6539999999999999,"15min":1.3839999999999999},"min":{"1min":0.35499999999999998,"5min":0.29499999999999998,"15min":0.26700000000000002},"max":{"1min":5.0919999999999996,"5min":8.3989999999999991,"15min":9.4640000000000004},"last":1.0009999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.5860000000000001,"15min":1.373},"min":{"1min":0.47799999999999998,"5min":0.34100000000000003,"15min":0.29799999999999999},"max":{"1min":4.524,"5min":8.9350000000000005,"15min":8.9350000000000005},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.361,"5min":1.3280000000000001,"15min":1.2150000000000001},"min":{"1min":0.48899999999999999,"5min":0.35199999999999998,"15min":0.32200000000000001},"max":{"1min":4.4329999999999998,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":1.0349999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.4790000000000001,"15min":1.75},"min":{"1min":0.41699999999999998,"5min":0.32900000000000001,"15min":0.30199999999999999},"max":{"1min":4.4960000000000004,"5min":8.0719999999999992,"15min":71.795000000000002},"last":0.91300000000000003},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.77,"15min":1.7969999999999999},"min":{"1min":0.46800000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":5.1639999999999997,"5min":8.4580000000000002,"15min":72.322000000000003},"last":1.0980000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":1.704,"15min":1.3959999999999999},"min":{"1min":0.496,"5min":0.39400000000000002,"15min":0.374},"max":{"1min":5.1299999999999999,"5min":8.4290000000000003,"15min":8.4659999999999993},"last":1.1819999999999999},{"interface":"front","average":{"1min":1.597,"5min":1.6020000000000001,"15min":1.292},"min":{"1min":0.51000000000000001,"5min":0.374,"15min":0.374},"max":{"1min":4.5499999999999998,"5min":8.8849999999999998,"15min":8.8849999999999998},"last":1.135}]}]},{"osd":1,"up_from":1246,"seq":5351529250899,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":413332,"kb_used_data":7628,"kb_used_omap":204,"kb_used_meta":405491,"kb_avail":93331820,"statfs":{"total":95995035648,"available":95571783680,"internally_reserved":0,"allocated":7811072,"data_stored":6248604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208958,"internal_metadata":415223746},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":35,"apply_latency_ms":35,"commit_latency_ns":35000000,"apply_latency_ns":35000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.6030000000000002,"15min":2.0369999999999999},"min":{"1min":0.251,"5min":0.191,"15min":0.191},"max":{"1min":8.2850000000000001,"5min":58.033000000000001,"15min":58.033000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.1549999999999998,"5min":2.5329999999999999,"15min":2.2749999999999999},"min":{"1min":0.21199999999999999,"5min":0.19,"15min":0.19},"max":{"1min":6.7080000000000002,"5min":65.921999999999997,"15min":65.921999999999997},"last":0.59799999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":2.6099999999999999,"15min":1.9039999999999999},"min":{"1min":0.33800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":10.647,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.67400000000000004},{"interface":"front","average":{"1min":2.2999999999999998,"5min":2.4199999999999999,"15min":1.8959999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.297000000000001,"5min":58.682000000000002,"15min":58.682000000000002},"last":0.78100000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.552,"5min":1.514,"15min":1.5029999999999999},"min":{"1min":0.217,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.1040000000000001,"5min":8.7140000000000004,"15min":8.7140000000000004},"last":0.73799999999999999},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.3899999999999999,"15min":1.3839999999999999},"min":{"1min":0.29099999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1509999999999998,"5min":7.5780000000000003,"15min":7.5780000000000003},"last":0.88600000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9830000000000001,"5min":1.8029999999999999,"15min":2.4209999999999998},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.242,"5min":8.8070000000000004,"15min":8.8070000000000004},"last":0.80900000000000005},{"interface":"front","average":{"1min":2.2120000000000002,"5min":1.7450000000000001,"15min":2.2480000000000002},"min":{"1min":0.36599999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.0860000000000003,"5min":10.263,"15min":10.263},"last":0.752}]},{"osd":5,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":1.7010000000000001,"15min":1.8},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.1200000000000001,"5min":10.214,"15min":10.214},"last":0.82799999999999996},{"interface":"front","average":{"1min":2.0059999999999998,"5min":1.675,"15min":1.8240000000000001},"min":{"1min":0.32800000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.6849999999999996,"5min":8.5139999999999993,"15min":8.5139999999999993},"last":0.91100000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":2.1230000000000002,"15min":1.9530000000000001},"min":{"1min":0.35499999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.798,"5min":9.3309999999999995,"15min":9.3309999999999995},"last":0.85299999999999998},{"interface":"front","average":{"1min":2.1970000000000001,"5min":2.129,"15min":2.8740000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.2939999999999996,"5min":9.8659999999999997,"15min":9.8659999999999997},"last":0.872}]},{"osd":7,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3439999999999999,"5min":1.8460000000000001,"15min":1.708},"min":{"1min":0.40300000000000002,"5min":0.374,"15min":0.374},"max":{"1min":5.7309999999999999,"5min":9.2929999999999993,"15min":9.2929999999999993},"last":0.98699999999999999},{"interface":"front","average":{"1min":2.125,"5min":1.946,"15min":2.25},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":5.2720000000000002,"5min":9.4529999999999994,"15min":9.4529999999999994},"last":0.96799999999999997}]}]},{"osd":0,"up_from":1136,"seq":4879082848492,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":761512,"kb_used_data":7584,"kb_used_omap":280,"kb_used_meta":753639,"kb_avail":92983640,"statfs":{"total":95995035648,"available":95215247360,"internally_reserved":0,"allocated":7766016,"data_stored":6236223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":771726746},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":30,"apply_latency_ms":30,"commit_latency_ns":30000000,"apply_latency_ns":30000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:54:24 2023","interfaces":[{"interface":"back","average":{"1min":3.3039999999999998,"5min":3.004,"15min":1.931},"min":{"1min":0.34399999999999997,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":9.0129999999999999,"5min":69.680000000000007,"15min":69.680000000000007},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.1680000000000001,"5min":2.9750000000000001,"15min":2.1699999999999999},"min":{"1min":0.30199999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.404999999999999,"5min":70.668999999999997,"15min":70.668999999999997},"last":0.60899999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":4.2770000000000001,"15min":2.6040000000000001},"min":{"1min":0.33900000000000002,"5min":0.22,"15min":0.22},"max":{"1min":9.0419999999999998,"5min":254.89599999999999,"15min":254.89599999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":2.4969999999999999,"5min":4.633,"15min":3.008},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":12.881,"5min":256.85399999999998,"15min":256.85399999999998},"last":0.52100000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.139,"15min":1.095},"min":{"1min":0.36699999999999999,"5min":0.247,"15min":0.247},"max":{"1min":8.8309999999999995,"5min":8.8309999999999995,"15min":10.475},"last":0.504},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.0169999999999999,"15min":1.069},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.25800000000000001},"max":{"1min":5.5860000000000003,"5min":6.4160000000000004,"15min":6.6550000000000002},"last":0.56100000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":1.504,"15min":1.462},"min":{"1min":0.40200000000000002,"5min":0.318,"15min":0.314},"max":{"1min":7.3129999999999997,"5min":7.3129999999999997,"15min":20.079000000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.891,"5min":1.379,"15min":1.4079999999999999},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":8.9480000000000004,"5min":8.9480000000000004,"15min":12.044},"last":0.65300000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.4530000000000001,"15min":1.4750000000000001},"min":{"1min":0.45800000000000002,"5min":0.33000000000000002,"15min":0.28399999999999997},"max":{"1min":8.8800000000000008,"5min":8.8800000000000008,"15min":9.4280000000000008},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.923,"5min":1.391,"15min":1.522},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.29699999999999999},"max":{"1min":8.7010000000000005,"5min":8.7010000000000005,"15min":9.6289999999999996},"last":0.95999999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.74,"15min":1.5369999999999999},"min":{"1min":0.40799999999999997,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7809999999999997,"5min":9.5950000000000006,"15min":10.413},"last":0.76600000000000001},{"interface":"front","average":{"1min":2.2069999999999999,"5min":1.72,"15min":1.702},"min":{"1min":0.503,"5min":0.439,"15min":0.35599999999999998},"max":{"1min":8.9809999999999999,"5min":8.9809999999999999,"15min":9.3260000000000005},"last":0.88900000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2389999999999999,"5min":1.595,"15min":1.379},"min":{"1min":0.45800000000000002,"5min":0.38500000000000001,"15min":0.38},"max":{"1min":6.7389999999999999,"5min":7.0149999999999997,"15min":7.0149999999999997},"last":0.91100000000000003},{"interface":"front","average":{"1min":2.1259999999999999,"5min":1.492,"15min":1.4319999999999999},"min":{"1min":0.53200000000000003,"5min":0.435,"15min":0.39200000000000002},"max":{"1min":8.9160000000000004,"5min":8.9160000000000004,"15min":14.361000000000001},"last":0.94099999999999995}]}]},{"osd":3,"up_from":0,"seq":2847563317722,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986856,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":913064,"kb_used_data":5600,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":92832088,"statfs":{"total":95995035648,"available":95060058112,"internally_reserved":0,"allocated":5734400,"data_stored":5431722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:54:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":1.222,"15min":1.5920000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":5.7670000000000003,"5min":5.7670000000000003,"15min":64.372},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.214,"5min":1.3720000000000001,"15min":1.5389999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.27700000000000002},"max":{"1min":5.7889999999999997,"5min":5.7889999999999997,"15min":64.356999999999999},"last":1.085}]},{"osd":1,"last update":"Mon Oct 9 09:54:18 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":4.5330000000000004,"15min":2.5779999999999998},"min":{"1min":0.35299999999999998,"5min":0.32000000000000001,"15min":0.27700000000000002},"max":{"1min":2.7410000000000001,"5min":216.16900000000001,"15min":216.16900000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.96499999999999997,"5min":4.8129999999999997,"15min":2.1899999999999999},"min":{"1min":0.47599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.6930000000000001,"5min":216.262,"15min":216.262},"last":0.60299999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:54:18 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":1.3839999999999999,"15min":1.5549999999999999},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.29199999999999998},"max":{"1min":15.853999999999999,"5min":15.853999999999999,"15min":33.743000000000002},"last":0.77000000000000002},{"interface":"front","average":{"1min":1.665,"5min":1.262,"15min":1.5009999999999999},"min":{"1min":0.40200000000000002,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":15.881,"5min":15.881,"15min":33.805999999999997},"last":0.91000000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.0700000000000001,"15min":1.0449999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":2.3690000000000002,"5min":5.5129999999999999,"15min":8.1120000000000001},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.89800000000000002,"15min":1.196},"min":{"1min":0.35799999999999998,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":3.2719999999999998,"5min":6.3659999999999997,"15min":16.507000000000001},"last":1.0069999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.2030000000000001,"15min":1.4710000000000001},"min":{"1min":0.433,"5min":0.373,"15min":0.26900000000000002},"max":{"1min":5.4660000000000002,"5min":11.055,"15min":11.763999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.3260000000000001,"15min":1.454},"min":{"1min":0.48799999999999999,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":5.4790000000000001,"5min":12.446999999999999,"15min":12.446999999999999},"last":0.93799999999999994}]},{"osd":6,"last update":"Mon Oct 9 09:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.139,"15min":1.488},"min":{"1min":0.33700000000000002,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":3.7469999999999999,"5min":8.1620000000000008,"15min":13.654},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.024,"5min":1.222,"15min":1.5469999999999999},"min":{"1min":0.45300000000000001,"5min":0.40799999999999997,"15min":0.39200000000000002},"max":{"1min":3.7240000000000002,"5min":11.013,"15min":11.972},"last":0.86299999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:54:18 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.96099999999999997,"15min":1.236},"min":{"1min":0.48699999999999999,"5min":0.45600000000000002,"15min":0.373},"max":{"1min":2.4180000000000001,"5min":5.2709999999999999,"15min":11.840999999999999},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.071,"15min":1.1200000000000001},"min":{"1min":0.52900000000000003,"5min":0.46700000000000003,"15min":0.34999999999999998},"max":{"1min":2.4199999999999999,"5min":5.1980000000000004,"15min":9.6210000000000004},"last":1.032}]}]},{"osd":5,"up_from":37,"seq":158913790622,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920276,"kb_used_data":5580,"kb_used_omap":0,"kb_used_meta":914688,"kb_avail":92824876,"statfs":{"total":95995035648,"available":95052673024,"internally_reserved":0,"allocated":5713920,"data_stored":5427396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":936640512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":3.4820000000000002,"15min":4.218},"min":{"1min":0.28100000000000003,"5min":0.26600000000000001,"15min":0.23599999999999999},"max":{"1min":4.3719999999999999,"5min":121.977,"15min":274.71899999999999},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.125,"5min":3.3610000000000002,"15min":4.0739999999999998},"min":{"1min":0.35299999999999998,"5min":0.33400000000000002,"15min":0.26800000000000002},"max":{"1min":4.4240000000000004,"5min":122.13200000000001,"15min":274.73500000000001},"last":0.95999999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:54:19 2023","interfaces":[{"interface":"back","average":{"1min":12.821,"5min":4.1050000000000004,"15min":2.6190000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":192.72200000000001,"5min":192.72200000000001,"15min":192.72200000000001},"last":0.71599999999999997},{"interface":"front","average":{"1min":12.872,"5min":4.056,"15min":2.1269999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":192.69300000000001,"5min":192.69300000000001,"15min":192.69300000000001},"last":1.0129999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:54:30 2023","interfaces":[{"interface":"back","average":{"1min":7.2539999999999996,"5min":2.2679999999999998,"15min":1.929},"min":{"1min":0.374,"5min":0.316,"15min":0.312},"max":{"1min":122.02800000000001,"5min":122.02800000000001,"15min":122.02800000000001},"last":1.0649999999999999},{"interface":"front","average":{"1min":7.3550000000000004,"5min":2.54,"15min":2.1659999999999999},"min":{"1min":0.38300000000000001,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":122.05800000000001,"5min":122.05800000000001,"15min":122.05800000000001},"last":0.97699999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:54:39 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.79100000000000004,"15min":1.0700000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.33100000000000002},"max":{"1min":3.601,"5min":3.601,"15min":10.125999999999999},"last":1.079},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.93000000000000005,"15min":1.089},"min":{"1min":0.41099999999999998,"5min":0.38,"15min":0.33200000000000002},"max":{"1min":1.0429999999999999,"5min":6.7750000000000004,"15min":16.201000000000001},"last":1.038}]},{"osd":4,"last update":"Mon Oct 9 09:54:14 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.90300000000000002,"15min":1.258},"min":{"1min":0.35799999999999998,"5min":0.35699999999999998,"15min":0.27000000000000002},"max":{"1min":3.6899999999999999,"5min":6.4470000000000001,"15min":9.4209999999999994},"last":1.0249999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":1.0409999999999999,"15min":1.4370000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.95199999999999996,"5min":7.601,"15min":16.053000000000001},"last":1.1040000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:54:39 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.099,"15min":1.802},"min":{"1min":0.438,"5min":0.308,"15min":0.308},"max":{"1min":5.4640000000000004,"5min":7.798,"15min":140.179},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.095,"5min":1.2330000000000001,"15min":1.901},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.34399999999999997},"max":{"1min":5.673,"5min":7.8070000000000004,"15min":137.928},"last":1.159}]},{"osd":7,"last update":"Mon Oct 9 09:54:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.0309999999999999,"15min":1.302},"min":{"1min":0.44,"5min":0.42399999999999999,"15min":0.373},"max":{"1min":1.077,"5min":4.3040000000000003,"15min":16.119},"last":1.1319999999999999},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.82799999999999996,"15min":0.98099999999999998},"min":{"1min":0.45000000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.0600000000000001,"5min":3.3700000000000001,"15min":6.5460000000000003},"last":0.874}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7721,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:55:01.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:55:01.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:01.564 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:01.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:01.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:01.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:01.696 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:01.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:01.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:01.800 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:01.826 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:01.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:01.929 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:01.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:01.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:01.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:01.979 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:55:02.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:02.114 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:02.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:02.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:02.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:02.252 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:02.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:02.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:02.299 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T09:55:02.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:02 smithi181 bash[59345]: cluster 2023-10-09T09:55:00.272948+0000 mgr.y (mgr.34107) 1475 : cluster 0 pgmap v2355: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:02.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:02.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:02 smithi163 bash[78107]: cluster 2023-10-09T09:55:00.272948+0000 mgr.y (mgr.34107) 1475 : cluster 0 pgmap v2355: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:02.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:02 smithi163 bash[75425]: cluster 2023-10-09T09:55:00.272948+0000 mgr.y (mgr.34107) 1475 : cluster 0 pgmap v2355: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:02.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:02.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:02.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:02.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:02.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:02.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:02.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:02.552 INFO:tasks.workunit.client.0.smithi163.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T09:55:02.588 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T09:55:02.588 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T09:55:02.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:02.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:02.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:02.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:02.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:02.809 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:02.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:02.912 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:02.937 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:02.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:02.994 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:55:03.041 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:03.074 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:03.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:03.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:03.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:03.211 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:03.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:03.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:03.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:03.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:03.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:03.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:03.451 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:03.464 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:55:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:55:03] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.33.4" 2023-10-09T09:55:03.482 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:03.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:03.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:03.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:03.621 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:03.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:03.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:03.725 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:03.749 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:03.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:03.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:03.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:03.885 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:03.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:03.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:03.934 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:55:03.990 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:04.015 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:04.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:04.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:04.118 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:04.129 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:04 smithi181 bash[59345]: cluster 2023-10-09T09:55:02.273582+0000 mgr.y (mgr.34107) 1476 : cluster 0 pgmap v2356: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:04.143 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:04.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:04.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:04.247 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:04.260 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:04 smithi163 bash[75425]: cluster 2023-10-09T09:55:02.273582+0000 mgr.y (mgr.34107) 1476 : cluster 0 pgmap v2356: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:04.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:04 smithi163 bash[78107]: cluster 2023-10-09T09:55:02.273582+0000 mgr.y (mgr.34107) 1476 : cluster 0 pgmap v2356: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:04.275 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:04.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:04.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:04.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:04.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:04.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:04.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:04.510 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:04.544 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:04.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:04.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:04.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:04.671 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:04.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:04.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:04.774 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:04.821 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:04.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:04.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:04.924 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:04.968 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:04.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:04.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:05.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:05.096 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:05.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:05.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:05.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:05.236 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:05.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:05.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:05.339 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:05.367 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:05.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:05.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:05.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:05.502 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:05.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:05.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:05.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:05.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:05.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:05.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:05.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:05.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:05.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:05.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:05.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:05.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:05.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:05.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:05.997 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:06.021 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:06.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:06.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:06.037 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:55:06.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:06.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:06.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:06.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:06.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:06.283 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:06.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:06.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:06.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:06.397 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:06 smithi181 bash[59345]: cluster 2023-10-09T09:55:04.274086+0000 mgr.y (mgr.34107) 1477 : cluster 0 pgmap v2357: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:06.412 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:06.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:06.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:06 smithi163 bash[78107]: cluster 2023-10-09T09:55:04.274086+0000 mgr.y (mgr.34107) 1477 : cluster 0 pgmap v2357: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:06 smithi163 bash[75425]: cluster 2023-10-09T09:55:04.274086+0000 mgr.y (mgr.34107) 1477 : cluster 0 pgmap v2357: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:06.517 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:06.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:06.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:06.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:06.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:06.679 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:06.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:06.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:06.783 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:06.806 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:06.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:06.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:06.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:06.937 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:06.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:06.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:07.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:07.069 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:07.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:07.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:07.173 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:07.198 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:07.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:07.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:07.301 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:07.329 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:07.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:07.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:07.456 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:07.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:07.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:07.560 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:07.588 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:07.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:07.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:07.607 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:55:07.607 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:55:07.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:07.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:07.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:07.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:07.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:07.848 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:07.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:07.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:07.951 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:07.973 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:07.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:07.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:08.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:08.104 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:08.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:08.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:08.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:08.244 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:08.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:08.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:08.347 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:08.358 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:08 smithi163 bash[75425]: cluster 2023-10-09T09:55:06.274913+0000 mgr.y (mgr.34107) 1478 : cluster 0 pgmap v2358: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:08.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:08 smithi163 bash[78107]: cluster 2023-10-09T09:55:06.274913+0000 mgr.y (mgr.34107) 1478 : cluster 0 pgmap v2358: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:08.376 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:08.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:08.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:08.423 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:08 smithi181 bash[59345]: cluster 2023-10-09T09:55:06.274913+0000 mgr.y (mgr.34107) 1478 : cluster 0 pgmap v2358: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:08.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T09:55:08.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:08.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:08.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:08.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:08.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:08.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:08.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:08.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:08.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:08.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:08.869 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:08.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:08.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:08.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:08.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:09.016 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:09.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:09.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:09.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:09.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:09.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:09.254 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:09.286 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:09.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:09.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:09.390 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:09.422 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:09.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:09.527 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:09.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:09.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:09.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:09.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:09.691 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:09.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:09.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:09.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:09.826 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:09.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:09.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:09.929 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:09.955 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:09.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:10.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:10.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:10.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:10.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:10.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:10.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:10 smithi163 bash[75425]: cluster 2023-10-09T09:55:08.275889+0000 mgr.y (mgr.34107) 1479 : cluster 0 pgmap v2359: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:10.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:10 smithi163 bash[78107]: cluster 2023-10-09T09:55:08.275889+0000 mgr.y (mgr.34107) 1479 : cluster 0 pgmap v2359: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:10.222 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:10.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:10.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:10.350 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:10.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:10.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:10.453 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:10.476 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:10.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:10.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:10.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:10 smithi181 bash[59345]: cluster 2023-10-09T09:55:08.275889+0000 mgr.y (mgr.34107) 1479 : cluster 0 pgmap v2359: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:10.579 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:10.604 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:10.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:10.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:10.708 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:10.739 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:10.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:10.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:10.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:10.864 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:10.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:10.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:10.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:10.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:10.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:11.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:11.125 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:11.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:11 smithi163 bash[78107]: audit 2023-10-09T09:55:10.417811+0000 mon.a (mon.0) 2581 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:11 smithi163 bash[75425]: audit 2023-10-09T09:55:10.417811+0000 mon.a (mon.0) 2581 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:11.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:11.254 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:11.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:11.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:11.357 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:11.382 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:11.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:11.486 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:11.536 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:11.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:11.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:11 smithi181 bash[59345]: audit 2023-10-09T09:55:10.417811+0000 mon.a (mon.0) 2581 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:11.639 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:11.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:11.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:11.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:11.775 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:11.808 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:11.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:11.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:11.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:11.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:11.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:11.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:12.045 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:12.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:12.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:12.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:12.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:12.192 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:12 smithi181 bash[59345]: cluster 2023-10-09T09:55:10.276468+0000 mgr.y (mgr.34107) 1480 : cluster 0 pgmap v2360: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:12.192 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:12 smithi181 bash[59345]: audit 2023-10-09T09:55:11.372141+0000 mon.a (mon.0) 2582 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:12.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:12.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:12.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:12.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:12.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:12.440 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:12 smithi163 bash[75425]: cluster 2023-10-09T09:55:10.276468+0000 mgr.y (mgr.34107) 1480 : cluster 0 pgmap v2360: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:12 smithi163 bash[75425]: audit 2023-10-09T09:55:11.372141+0000 mon.a (mon.0) 2582 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:12.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:12 smithi163 bash[78107]: cluster 2023-10-09T09:55:10.276468+0000 mgr.y (mgr.34107) 1480 : cluster 0 pgmap v2360: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:12.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:12 smithi163 bash[78107]: audit 2023-10-09T09:55:11.372141+0000 mon.a (mon.0) 2582 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:12.477 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:12.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:12.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:12.582 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:12.606 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:12.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:12.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:12.709 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:12.732 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:12.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:12.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:12.835 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:12.871 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:12.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:12.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:12.975 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:13.006 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:13.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:13.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:13.111 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:13.139 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:13.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:13.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:13.201 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:55:13.242 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:13.271 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:13.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:13.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:13.375 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:13.396 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:13.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:13.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:13.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:13.520 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:13.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:13.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:13.623 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:13.651 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:13.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:13.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:55:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:55:13] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.33.4" 2023-10-09T09:55:13.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:13.784 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:13.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:13.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:13.887 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:13.917 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:13.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:13.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:14.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:14.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:14.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:14.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:14.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:14.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:14 smithi181 bash[59345]: cluster 2023-10-09T09:55:12.277447+0000 mgr.y (mgr.34107) 1481 : cluster 0 pgmap v2361: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:14.182 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:14.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:14.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:14.248 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:55:14.275 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:55:14.285 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:14.299 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:55:14.318 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:14.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:14.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:14.325 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:55:14.347 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:55:14.371 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:55:14.399 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:55:14.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:14.428 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:55:14.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:55:14.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:55:14.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:55:14.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:55:14.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:55:14.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:55:14.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:55:14.430 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:55:14.447 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:14.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:14.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:14.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:14 smithi163 bash[78107]: cluster 2023-10-09T09:55:12.277447+0000 mgr.y (mgr.34107) 1481 : cluster 0 pgmap v2361: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:14 smithi163 bash[75425]: cluster 2023-10-09T09:55:12.277447+0000 mgr.y (mgr.34107) 1481 : cluster 0 pgmap v2361: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:14.550 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:14.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:14.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:14.665 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T09:55:14.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:14.708 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:14.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:14.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:14.811 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:14.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:14.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:14.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:14.941 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:14.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:14.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:14.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:15.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:15.093 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:15.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:15.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:15.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:15.209 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:15 smithi181 bash[59345]: audit 2023-10-09T09:55:14.222943+0000 mon.a (mon.0) 2583 : audit 0 from='client.? 172.21.15.163:0/817033955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:55:15.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:15.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:15.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:15.347 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:15.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:15.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:15.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:15.509 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:15.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:15.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:15.577 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:15 smithi163 bash[78107]: audit 2023-10-09T09:55:14.222943+0000 mon.a (mon.0) 2583 : audit 0 from='client.? 172.21.15.163:0/817033955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:55:15.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:15 smithi163 bash[75425]: audit 2023-10-09T09:55:14.222943+0000 mon.a (mon.0) 2583 : audit 0 from='client.? 172.21.15.163:0/817033955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:55:15.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:15.633 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:15.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:15.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:15.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:15.762 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:15.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:15.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:15.845 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T09:55:15.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:15.885 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:15.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:15.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:15.935 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T09:55:15.937 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T09:55:15.937 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T09:55:15.988 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T09:55:15.989 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:15.991 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T09:55:15.992 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T09:55:16.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:16.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:16.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:16.103 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T09:55:16.105 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T09:55:16.105 DEBUG:teuthology.orchestra.run.smithi163:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T09:55:16.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:16.171 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:16.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:16.264 INFO:tasks.workunit.client.0.smithi163.stdout:Running main() from gmock_main.cc 2023-10-09T09:55:16.264 INFO:tasks.workunit.client.0.smithi163.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T09:55:16.264 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] Global test environment set-up. 2023-10-09T09:55:16.265 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T09:55:16.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:16.287 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:16 smithi163 bash[78107]: cluster 2023-10-09T09:55:14.277903+0000 mgr.y (mgr.34107) 1482 : cluster 0 pgmap v2362: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:16.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:16 smithi163 bash[75425]: cluster 2023-10-09T09:55:14.277903+0000 mgr.y (mgr.34107) 1482 : cluster 0 pgmap v2362: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:16.304 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:16.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:16.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:16.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:16 smithi181 bash[59345]: cluster 2023-10-09T09:55:14.277903+0000 mgr.y (mgr.34107) 1482 : cluster 0 pgmap v2362: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:16.408 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:16.437 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:16.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:16.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:16.563 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:16.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:16.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:16.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:16.692 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:16.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:16.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:16.713 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:55:16.713 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:40791 11.8 11:1d917cbe:::benchmark_data_smithi163_218041_object40790:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1274)", 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:09.997581+0000", 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 246.709602566, 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25407731500000003, 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.714 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 40791 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:09.997581+0000", 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:09.997581+0000", 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:09.997580+0000", 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9875e-05 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:09.997620+0000", 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2525e-05 2023-10-09T09:55:16.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:09.997632+0000", 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7619999999999995e-06 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:09.997640+0000", 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003326748 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.000967+0000", 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000149281 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.001116+0000", 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000179192 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.001295+0000", 2023-10-09T09:55:16.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002629358 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.003925+0000", 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011095839999999999 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.015021+0000", 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.236338827 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.251359+0000", 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017109699999999999 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.251530+0000", 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012800100000000001 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:10.251658+0000", 2023-10-09T09:55:16.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25407731500000003 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:43417 11.4 11:29e96952:::benchmark_data_smithi163_218041_object43416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:12.260797+0000", 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 244.44638680099999, 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.3397077, 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.721 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.722 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.722 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.722 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43417 2023-10-09T09:55:16.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.722 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.722 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.260797+0000", 2023-10-09T09:55:16.722 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.260797+0000", 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999852 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.260782+0000", 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0463000000000006e-05 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.723 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.260852+0000", 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3523000000000001e-05 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.260866+0000", 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7219999999999994e-06 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.724 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.260876+0000", 2023-10-09T09:55:16.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00072312299999999999 2023-10-09T09:55:16.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.725 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.261599+0000", 2023-10-09T09:55:16.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018924799999999999 2023-10-09T09:55:16.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.726 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.726 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.261788+0000", 2023-10-09T09:55:16.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020796500000000001 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.261996+0000", 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019416940000000001 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.263938+0000", 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014374399999999999 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.265375+0000", 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.33496113399999999 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.600336+0000", 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010218200000000001 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.600439+0000", 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6047e-05 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.600505+0000", 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.3397077 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.729 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:43427 11.4 11:2311dd09:::benchmark_data_smithi163_218041_object43426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:12.267471+0000", 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 244.43971245899999, 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.34078518800000002, 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43427 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.730 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.267471+0000", 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.267471+0000", 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.731 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.267469+0000", 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4129000000000003e-05 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.267524+0000", 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9385000000000002e-05 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.267543+0000", 2023-10-09T09:55:16.732 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0319999999999998e-06 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.267550+0000", 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7046000000000003e-05 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.267597+0000", 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014530500000000001 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.733 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.267742+0000", 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021104300000000001 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.267953+0000", 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017200480000000001 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.734 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.269673+0000", 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002446124 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.272120+0000", 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.335819643 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.607939+0000", 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022086799999999999 2023-10-09T09:55:16.735 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.608160+0000", 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6342999999999996e-05 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.608256+0000", 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.34078518800000002 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.736 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:43429 11.1 11:8ad6d55a:::benchmark_data_smithi163_218041_object43428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:12.270062+0000", 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 244.43712143400001, 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.33294033000000001, 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43429 2023-10-09T09:55:16.737 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.270062+0000", 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.270062+0000", 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.270061+0000", 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2546000000000006e-05 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.270144+0000", 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1314e-05 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.270155+0000", 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1439e-05 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.270166+0000", 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00034947599999999999 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.270516+0000", 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010913600000000001 2023-10-09T09:55:16.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.270625+0000", 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013413100000000001 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.270759+0000", 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022884870000000001 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.741 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.273048+0000", 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00160531 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.274653+0000", 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.32813150600000002 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.602784+0000", 2023-10-09T09:55:16.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1254000000000004e-05 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.602876+0000", 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012682200000000001 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.603003+0000", 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.33294033000000001 2023-10-09T09:55:16.743 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:43437 11.4 11:23bd14ef:::benchmark_data_smithi163_218041_object43436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:12.273489+0000", 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 244.43369485900001, 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.335028722, 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.744 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43437 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.273489+0000", 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.745 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.273489+0000", 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.273487+0000", 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1789000000000002e-05 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.273539+0000", 2023-10-09T09:55:16.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5743000000000002e-05 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.273555+0000", 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6349999999999995e-06 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.273563+0000", 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7497000000000003e-05 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.273611+0000", 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016135499999999999 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.273772+0000", 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000166916 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:55:16.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.273939+0000", 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015801890000000001 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.275519+0000", 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0026234600000000002 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.278143+0000", 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.33024000799999997 2023-10-09T09:55:16.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.608383+0000", 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5108e-05 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.608458+0000", 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9701000000000003e-05 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.750 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.608518+0000", 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.335028722 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:43445 11.1 11:8ca7eae5:::benchmark_data_smithi163_218041_object43444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:12.282098+0000", 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 244.42508575400001, 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.32133028899999999, 2023-10-09T09:55:16.751 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43445 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.282098+0000", 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.752 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.282098+0000", 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.282097+0000", 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1240000000000003e-05 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.282158+0000", 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2373e-05 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.282170+0000", 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4179999999999996e-06 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.282176+0000", 2023-10-09T09:55:16.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4045999999999999e-05 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.282210+0000", 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016978300000000001 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.282380+0000", 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024014199999999999 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.282620+0000", 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001872833 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.284493+0000", 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000356611 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.284849+0000", 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31828926800000001 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.603139+0000", 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000159591 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.603298+0000", 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000130061 2023-10-09T09:55:16.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.603428+0000", 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.32133028899999999 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:43447 11.4 11:29263538:::benchmark_data_smithi163_218041_object43446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:55:16.758 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:12.285094+0000", 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 244.42209014400001, 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.327380587, 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 43447 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.759 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.285094+0000", 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.285094+0000", 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.285092+0000", 2023-10-09T09:55:16.760 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2386999999999997e-05 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.285164+0000", 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9037000000000001e-05 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.285183+0000", 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0621e-05 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.761 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.285194+0000", 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1576999999999998e-05 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.285245+0000", 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014500299999999999 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.285390+0000", 2023-10-09T09:55:16.763 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013349299999999999 2023-10-09T09:55:16.763 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.763 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.763 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:55:16.763 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.285524+0000", 2023-10-09T09:55:16.763 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001686964 2023-10-09T09:55:16.763 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.763 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.763 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.287211+0000", 2023-10-09T09:55:16.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031784700000000001 2023-10-09T09:55:16.765 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.765 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.287529+0000", 2023-10-09T09:55:16.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.32478352700000002 2023-10-09T09:55:16.766 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.766 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.766 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.612312+0000", 2023-10-09T09:55:16.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0031000000000006e-05 2023-10-09T09:55:16.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.767 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.767 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.767 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.612402+0000", 2023-10-09T09:55:16.768 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1923999999999998e-05 2023-10-09T09:55:16.768 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.768 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.768 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.768 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:12.612474+0000", 2023-10-09T09:55:16.769 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.327380587 2023-10-09T09:55:16.769 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.769 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.769 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.769 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.769 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.770 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:59044 11.1 11:8bd5a6ae:::benchmark_data_smithi163_218041_object59043:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e1275)", 2023-10-09T09:55:16.770 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:51:25.528774+0000", 2023-10-09T09:55:16.770 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 231.17840978500001, 2023-10-09T09:55:16.770 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25419493399999998, 2023-10-09T09:55:16.770 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.770 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.770 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.771 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.771 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.771 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 59044 2023-10-09T09:55:16.771 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.771 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.771 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.771 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.771 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.528774+0000", 2023-10-09T09:55:16.772 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.772 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.772 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.772 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.772 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.528774+0000", 2023-10-09T09:55:16.772 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:55:16.772 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.773 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.773 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.773 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.528772+0000", 2023-10-09T09:55:16.773 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1847999999999995e-05 2023-10-09T09:55:16.773 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.773 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.773 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.773 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.528834+0000", 2023-10-09T09:55:16.774 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3237999999999999e-05 2023-10-09T09:55:16.774 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.774 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.774 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.528847+0000", 2023-10-09T09:55:16.774 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0115000000000001e-05 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.528858+0000", 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0336e-05 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.528918+0000", 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017024600000000001 2023-10-09T09:55:16.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.529088+0000", 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000144456 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.529233+0000", 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021650530000000001 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.776 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.531398+0000", 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015102010000000001 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.532908+0000", 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.249766301 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.782674+0000", 2023-10-09T09:55:16.777 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021874700000000001 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.782893+0000", 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5956000000000003e-05 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:51:25.782969+0000", 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25419493399999998 2023-10-09T09:55:16.778 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:146008 11.a 11:5200a16b:::benchmark_data_smithi163_218041_object38732:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1280)", 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:53:05.678749+0000", 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 131.028434578, 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26827695200000001, 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.779 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 146008 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678749+0000", 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678749+0000", 2023-10-09T09:55:16.780 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678748+0000", 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0479999999999999e-06 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678756+0000", 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6016999999999999e-05 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678772+0000", 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1380000000000007e-06 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678780+0000", 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011285000000000001 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678893+0000", 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042840450000000002 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.683177+0000", 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000164549 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.683341+0000", 2023-10-09T09:55:16.783 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044551840000000001 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.687796+0000", 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018429699999999999 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.687981+0000", 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25885754399999999 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.784 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.946838+0000", 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000128646 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.946967+0000", 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9114999999999999e-05 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.785 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.947026+0000", 2023-10-09T09:55:16.786 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26827695200000001 2023-10-09T09:55:16.786 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.786 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.786 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.786 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.786 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.786 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:146010 11.3 11:c18fba99:::benchmark_data_smithi163_218041_object38734:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1280)", 2023-10-09T09:55:16.786 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:53:05.678795+0000", 2023-10-09T09:55:16.786 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 131.02838850099999, 2023-10-09T09:55:16.787 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.272233752, 2023-10-09T09:55:16.787 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.787 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.787 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.787 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.787 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.787 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 146010 2023-10-09T09:55:16.787 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.788 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.788 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.788 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.788 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678795+0000", 2023-10-09T09:55:16.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.788 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.788 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.789 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678795+0000", 2023-10-09T09:55:16.789 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T09:55:16.789 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.789 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.789 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.789 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678795+0000", 2023-10-09T09:55:16.789 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2219999999999998e-06 2023-10-09T09:55:16.790 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.790 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.790 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.790 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678798+0000", 2023-10-09T09:55:16.790 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9811999999999999e-05 2023-10-09T09:55:16.791 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.791 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.791 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.791 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678818+0000", 2023-10-09T09:55:16.791 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3215e-05 2023-10-09T09:55:16.791 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.791 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.792 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.792 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678831+0000", 2023-10-09T09:55:16.792 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9354000000000001e-05 2023-10-09T09:55:16.792 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.792 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.792 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.792 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.678890+0000", 2023-10-09T09:55:16.793 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012938399999999999 2023-10-09T09:55:16.793 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.793 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.793 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.793 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.679020+0000", 2023-10-09T09:55:16.793 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000110277 2023-10-09T09:55:16.793 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.794 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.794 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:55:16.794 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.679130+0000", 2023-10-09T09:55:16.794 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081961559999999996 2023-10-09T09:55:16.794 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.794 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.794 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.794 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.687326+0000", 2023-10-09T09:55:16.794 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00046228799999999999 2023-10-09T09:55:16.794 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.687788+0000", 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.263072315 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.950861+0000", 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011488400000000001 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.795 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.950975+0000", 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3461000000000002e-05 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:53:05.951029+0000", 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.272233752 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.796 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:192561 11.a 11:576d3ed8:::benchmark_data_smithi163_218041_object85285:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1283)", 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:04.215102+0000", 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 72.492081960999997, 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.36055632900000001, 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 192561 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.215102+0000", 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.215102+0000", 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.215100+0000", 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2439999999999996e-06 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.215108+0000", 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1015e-05 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.215119+0000", 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6059999999999999e-06 2023-10-09T09:55:16.799 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.215125+0000", 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6582000000000002e-05 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.215172+0000", 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00048875700000000004 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.215661+0000", 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015480299999999999 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.215815+0000", 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042309460000000002 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220046+0000", 2023-10-09T09:55:16.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9802999999999997e-05 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220096+0000", 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.35545426400000002 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.575550+0000", 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5569999999999997e-05 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.575616+0000", 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2046999999999999e-05 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.575658+0000", 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.36055632900000001 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.803 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:192566 11.4 11:25ae75dc:::benchmark_data_smithi163_218041_object85290:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1283)", 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:04.220307+0000", 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 72.486876283000001, 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.343187566, 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 192566 2023-10-09T09:55:16.804 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220307+0000", 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220307+0000", 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220306+0000", 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8679999999999994e-06 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220315+0000", 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7031e-05 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220332+0000", 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2690000000000005e-06 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220342+0000", 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7825999999999999e-05 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220390+0000", 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015110300000000001 2023-10-09T09:55:16.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220541+0000", 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017775500000000001 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.220718+0000", 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042424960000000001 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.808 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.224961+0000", 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0075470889999999999 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.232508+0000", 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.33083112199999998 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.563339+0000", 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0896000000000005e-05 2023-10-09T09:55:16.809 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.563430+0000", 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4982999999999998e-05 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.563495+0000", 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.343187566 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.810 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:192571 11.a 11:517b3f8d:::benchmark_data_smithi163_218041_object85295:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1283)", 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:04.221203+0000", 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 72.485980677000001, 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.35459151500000002, 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.811 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 192571 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.221203+0000", 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.221203+0000", 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:55:16.812 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.221202+0000", 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2469999999999999e-06 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.221207+0000", 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0101e-05 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.221217+0000", 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6770000000000003e-06 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.221222+0000", 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5825000000000003e-05 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.221268+0000", 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019581199999999999 2023-10-09T09:55:16.814 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.221464+0000", 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000142213 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.221606+0000", 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040046589999999998 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.815 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.225611+0000", 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0077242739999999997 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.233335+0000", 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.34236916099999998 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.575704+0000", 2023-10-09T09:55:16.816 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9793000000000003e-05 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.575754+0000", 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0630999999999998e-05 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.575795+0000", 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.35459151500000002 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.817 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:192576 11.4 11:253814fa:::benchmark_data_smithi163_218041_object85300:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1283)", 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:04.229369+0000", 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 72.477814961999997, 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.34003924499999999, 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.818 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 192576 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.229369+0000", 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.229369+0000", 2023-10-09T09:55:16.819 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.229367+0000", 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.422e-06 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.229375+0000", 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9434999999999999e-05 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.820 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.229395+0000", 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2255e-05 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.229407+0000", 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2740000000000001e-05 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.229450+0000", 2023-10-09T09:55:16.821 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9409999999999999e-05 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.229539+0000", 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4883999999999995e-05 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.229634+0000", 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00297773 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.822 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.232612+0000", 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00039428899999999998 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.233006+0000", 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.33627094200000002 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.823 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.569277+0000", 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1556000000000004e-05 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.569349+0000", 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9255000000000001e-05 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.569408+0000", 2023-10-09T09:55:16.824 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.34003924499999999 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:192580 11.9 11:95da3ce7:::benchmark_data_smithi163_218041_object85304:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1283)", 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:04.233810+0000", 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 72.473374054999994, 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.337526302, 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.825 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 192580 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.233810+0000", 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.826 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.233810+0000", 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.233808+0000", 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2380000000000002e-06 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.233814+0000", 2023-10-09T09:55:16.827 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1599e-05 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.233826+0000", 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5899999999999998e-06 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.233832+0000", 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0396000000000003e-05 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.828 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.233882+0000", 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000147699 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.234030+0000", 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000168727 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.234198+0000", 2023-10-09T09:55:16.829 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028715590000000001 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.237070+0000", 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118483 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.237188+0000", 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.334046811 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.830 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.571235+0000", 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6708e-05 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.571292+0000", 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3949999999999998e-05 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:04.571336+0000", 2023-10-09T09:55:16.831 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.337526302 2023-10-09T09:55:16.832 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.832 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.832 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.832 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.832 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.832 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:214355 11.4 11:23380857:::benchmark_data_smithi163_218041_object107079:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1286)", 2023-10-09T09:55:16.832 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:31.911039+0000", 2023-10-09T09:55:16.832 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 44.796144394000002, 2023-10-09T09:55:16.833 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.034206482000000003, 2023-10-09T09:55:16.833 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.833 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.833 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.833 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.833 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.833 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 214355 2023-10-09T09:55:16.834 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.834 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.834 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.834 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.834 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.911039+0000", 2023-10-09T09:55:16.834 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.835 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.835 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.835 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.835 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.911039+0000", 2023-10-09T09:55:16.835 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T09:55:16.835 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.836 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.836 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.836 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.911037+0000", 2023-10-09T09:55:16.836 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9039999999999998e-06 2023-10-09T09:55:16.836 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.836 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.837 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.837 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.911046+0000", 2023-10-09T09:55:16.837 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5540999999999999e-05 2023-10-09T09:55:16.837 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.837 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.837 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.837 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.911062+0000", 2023-10-09T09:55:16.838 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.323e-06 2023-10-09T09:55:16.838 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.838 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.838 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.838 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.911069+0000", 2023-10-09T09:55:16.838 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.52e-05 2023-10-09T09:55:16.838 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.839 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.839 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.839 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.911124+0000", 2023-10-09T09:55:16.839 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017508899999999999 2023-10-09T09:55:16.839 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.839 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.840 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.840 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.911299+0000", 2023-10-09T09:55:16.840 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021121000000000001 2023-10-09T09:55:16.840 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.840 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.840 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:55:16.841 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.911511+0000", 2023-10-09T09:55:16.841 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041829390000000001 2023-10-09T09:55:16.841 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.841 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.841 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.841 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.915693+0000", 2023-10-09T09:55:16.842 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0074515650000000003 2023-10-09T09:55:16.842 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.842 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.842 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.842 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.923145+0000", 2023-10-09T09:55:16.842 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022025855 2023-10-09T09:55:16.842 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.842 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.842 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.842 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.945171+0000", 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9960999999999999e-05 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.945211+0000", 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4904999999999999e-05 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.945246+0000", 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.034206482000000003 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.843 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:214360 11.8 11:19e55d02:::benchmark_data_smithi163_218041_object107084:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1286)", 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:31.912958+0000", 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 44.794225865999998, 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027448800999999998, 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.844 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 214360 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.912958+0000", 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.912958+0000", 2023-10-09T09:55:16.845 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.912957+0000", 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7060000000000001e-06 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.912962+0000", 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0465000000000001e-05 2023-10-09T09:55:16.846 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.912973+0000", 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7480000000000001e-06 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.912978+0000", 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8642e-05 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.847 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.913016+0000", 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0482999999999994e-05 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.913107+0000", 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1013999999999994e-05 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.913188+0000", 2023-10-09T09:55:16.848 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0038394620000000001 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.917027+0000", 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011155290000000001 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.918143+0000", 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022135202999999999 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.849 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.940278+0000", 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5819000000000001e-05 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.940344+0000", 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2781999999999996e-05 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.850 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.940407+0000", 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027448800999999998 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:214377 11.8 11:13ffd580:::benchmark_data_smithi163_218041_object107101:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1286)", 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:31.935049+0000", 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 44.772135167999998, 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029077446, 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.851 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 214377 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.935049+0000", 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.852 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.935049+0000", 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.935047+0000", 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9063e-05 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.935066+0000", 2023-10-09T09:55:16.853 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4338999999999999e-05 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.935080+0000", 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.773e-05 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.935108+0000", 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000104002 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.854 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.935212+0000", 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016755900000000001 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.935380+0000", 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00036845600000000002 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.855 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,5", 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.935748+0000", 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067976210000000002 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.942546+0000", 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00032932799999999999 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.942875+0000", 2023-10-09T09:55:16.856 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.021167423000000001 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.964043+0000", 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4200999999999998e-05 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.964097+0000", 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9176999999999999e-05 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.857 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.964126+0000", 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029077446 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:214420 11.3 11:c266599f:::benchmark_data_smithi163_218041_object107144:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1286)", 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:31.998496+0000", 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 44.708687761999997, 2023-10-09T09:55:16.858 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.025905564999999998, 2023-10-09T09:55:16.859 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.859 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.859 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.859 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.859 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.859 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 214420 2023-10-09T09:55:16.859 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.859 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.859 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.860 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.860 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.998496+0000", 2023-10-09T09:55:16.860 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.860 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.860 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.860 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.860 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.998496+0000", 2023-10-09T09:55:16.860 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T09:55:16.860 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.860 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.998494+0000", 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9369999999999993e-06 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.998503+0000", 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6622000000000001e-05 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.998520+0000", 2023-10-09T09:55:16.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4059999999999999e-06 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.998527+0000", 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9787999999999997e-05 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.998587+0000", 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011685599999999999 2023-10-09T09:55:16.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.998704+0000", 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012574799999999999 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,5", 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:31.998830+0000", 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0045167929999999998 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.863 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.003346+0000", 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004770664 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.008117+0000", 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016179710999999999 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.024297+0000", 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6666000000000003e-05 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.864 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.024353+0000", 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8040000000000001e-05 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.024401+0000", 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.025905564999999998 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.865 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35814.0:214478 11.4 11:2ca926ea:::benchmark_data_smithi163_218041_object107202:head [delete] snapc 0=[] ondisk+write+known_if_redirected e1286)", 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T09:54:32.059975+0000", 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 44.647208788999997, 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.031066716000000001, 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35814", 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1374136351", 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 214478 2023-10-09T09:55:16.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.059975+0000", 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.059975+0000", 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.059974+0000", 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3530000000000002e-06 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.059979+0000", 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0054999999999999e-05 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.059989+0000", 2023-10-09T09:55:16.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3300000000000004e-06 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.059996+0000", 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3554999999999999e-05 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.060039+0000", 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000104755 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.869 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.060144+0000", 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013235699999999999 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.060276+0000", 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0038248900000000001 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T09:55:16.870 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.064101+0000", 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00160505 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.065706+0000", 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.025265850999999999 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.090972+0000", 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8068000000000002e-05 2023-10-09T09:55:16.871 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.091010+0000", 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1371999999999997e-05 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T09:54:32.091042+0000", 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.031066716000000001 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.872 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T09:55:16.873 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T09:55:16.873 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:55:16.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:16.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:16.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:16.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:17.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:17.036 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:17.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:17.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:17.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:17.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:17.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:17.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:17.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:17.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:17.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:17.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:17.431 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:17.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:17.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:17.535 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:17.558 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T09:55:17.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:17.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:17.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:17.701 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:17.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:17.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:17.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:17.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:17.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:17.938 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:18.001 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:18.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:18.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:18.064 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:55:18.066 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:55:18.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:18.133 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:18.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:18.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:18.148 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T09:55:18.155 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:55:18.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[78107]: cluster 2023-10-09T09:55:16.278348+0000 mgr.y (mgr.34107) 1483 : cluster 0 pgmap v2363: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:18.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[78107]: cluster 2023-10-09T09:55:17.136140+0000 osd.5 (osd.5) 880 : cluster 3 failed to encode map e1290 with expected crc 2023-10-09T09:55:18.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[78107]: cluster 2023-10-09T09:55:17.136453+0000 osd.6 (osd.6) 567 : cluster 3 failed to encode map e1290 with expected crc 2023-10-09T09:55:18.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[78107]: cluster 2023-10-09T09:55:17.138552+0000 mon.a (mon.0) 2584 : cluster 0 osdmap e1290: 8 total, 8 up, 6 in 2023-10-09T09:55:18.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[78107]: audit 2023-10-09T09:55:17.140346+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.163:0/2386238030' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-235913-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:55:18.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[78107]: audit 2023-10-09T09:55:17.140812+0000 mon.a (mon.0) 2585 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-235913-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:55:18.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[78107]: audit 2023-10-09T09:55:17.561987+0000 mon.a (mon.0) 2586 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:18.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[78107]: audit 2023-10-09T09:55:17.568655+0000 mon.a (mon.0) 2587 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:18.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[78107]: audit 2023-10-09T09:55:18.090298+0000 mon.a (mon.0) 2588 : audit 1 from='client.? 172.21.15.163:0/2583723883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[75425]: cluster 2023-10-09T09:55:16.278348+0000 mgr.y (mgr.34107) 1483 : cluster 0 pgmap v2363: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[75425]: cluster 2023-10-09T09:55:17.136140+0000 osd.5 (osd.5) 880 : cluster 3 failed to encode map e1290 with expected crc 2023-10-09T09:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[75425]: cluster 2023-10-09T09:55:17.136453+0000 osd.6 (osd.6) 567 : cluster 3 failed to encode map e1290 with expected crc 2023-10-09T09:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[75425]: cluster 2023-10-09T09:55:17.138552+0000 mon.a (mon.0) 2584 : cluster 0 osdmap e1290: 8 total, 8 up, 6 in 2023-10-09T09:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[75425]: audit 2023-10-09T09:55:17.140346+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.163:0/2386238030' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-235913-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[75425]: audit 2023-10-09T09:55:17.140812+0000 mon.a (mon.0) 2585 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-235913-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[75425]: audit 2023-10-09T09:55:17.561987+0000 mon.a (mon.0) 2586 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[75425]: audit 2023-10-09T09:55:17.568655+0000 mon.a (mon.0) 2587 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:18.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:18 smithi163 bash[75425]: audit 2023-10-09T09:55:18.090298+0000 mon.a (mon.0) 2588 : audit 1 from='client.? 172.21.15.163:0/2583723883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:55:18.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:18.239 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_all_features (91 ms) 2023-10-09T09:55:18.239 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T09:55:18.257 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:18.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:18.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:18.360 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:18.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:18.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:18.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:18.461 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.copyup (222 ms) 2023-10-09T09:55:18.461 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T09:55:18.490 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:18.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:18.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:18.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:18.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:18 smithi181 bash[59345]: cluster 2023-10-09T09:55:16.278348+0000 mgr.y (mgr.34107) 1483 : cluster 0 pgmap v2363: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:18 smithi181 bash[59345]: cluster 2023-10-09T09:55:17.136140+0000 osd.5 (osd.5) 880 : cluster 3 failed to encode map e1290 with expected crc 2023-10-09T09:55:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:18 smithi181 bash[59345]: cluster 2023-10-09T09:55:17.136453+0000 osd.6 (osd.6) 567 : cluster 3 failed to encode map e1290 with expected crc 2023-10-09T09:55:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:18 smithi181 bash[59345]: cluster 2023-10-09T09:55:17.138552+0000 mon.a (mon.0) 2584 : cluster 0 osdmap e1290: 8 total, 8 up, 6 in 2023-10-09T09:55:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:18 smithi181 bash[59345]: audit 2023-10-09T09:55:17.140346+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.163:0/2386238030' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-235913-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:55:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:18 smithi181 bash[59345]: audit 2023-10-09T09:55:17.140812+0000 mon.a (mon.0) 2585 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-235913-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:55:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:18 smithi181 bash[59345]: audit 2023-10-09T09:55:17.561987+0000 mon.a (mon.0) 2586 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:18 smithi181 bash[59345]: audit 2023-10-09T09:55:17.568655+0000 mon.a (mon.0) 2587 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:18.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:18 smithi181 bash[59345]: audit 2023-10-09T09:55:18.090298+0000 mon.a (mon.0) 2588 : audit 1 from='client.? 172.21.15.163:0/2583723883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:55:18.586 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.sparse_copyup (125 ms) 2023-10-09T09:55:18.586 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T09:55:18.601 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_and_set_id (14 ms) 2023-10-09T09:55:18.601 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T09:55:18.617 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.add_remove_child (17 ms) 2023-10-09T09:55:18.618 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T09:55:18.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:18.649 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:18.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:18.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:18.661 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.directory_methods (43 ms) 2023-10-09T09:55:18.661 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.create 2023-10-09T09:55:18.684 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.create (24 ms) 2023-10-09T09:55:18.684 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T09:55:18.688 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_features (3 ms) 2023-10-09T09:55:18.688 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T09:55:18.690 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_object_prefix (3 ms) 2023-10-09T09:55:18.690 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T09:55:18.692 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-09T09:55:18.692 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T09:55:18.695 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_access_timestamp (3 ms) 2023-10-09T09:55:18.695 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T09:55:18.699 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_modify_timestamp (4 ms) 2023-10-09T09:55:18.699 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T09:55:18.706 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_data_pool (7 ms) 2023-10-09T09:55:18.706 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T09:55:18.711 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_size (5 ms) 2023-10-09T09:55:18.711 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T09:55:18.720 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.set_size (9 ms) 2023-10-09T09:55:18.720 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T09:55:18.746 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.protection_status (26 ms) 2023-10-09T09:55:18.746 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T09:55:18.752 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:18.759 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.snapshot_limits (13 ms) 2023-10-09T09:55:18.760 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T09:55:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:18.787 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:18.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:18.820 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.parents_v1 (61 ms) 2023-10-09T09:55:18.820 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T09:55:18.865 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.parents_v2 (45 ms) 2023-10-09T09:55:18.865 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T09:55:18.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:18.902 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.snapshots (37 ms) 2023-10-09T09:55:18.902 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T09:55:18.911 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:18.911 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.snapid_race (9 ms) 2023-10-09T09:55:18.912 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T09:55:18.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:18.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:18.928 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.stripingv2 (17 ms) 2023-10-09T09:55:18.928 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T09:55:18.932 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2023-10-09T09:55:18.934 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T09:55:18.941 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2023-10-09T09:55:18.941 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T09:55:18.951 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_update (11 ms) 2023-10-09T09:55:18.952 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T09:55:18.952 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T09:55:18.952 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T09:55:18.960 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_snap_add (9 ms) 2023-10-09T09:55:18.960 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T09:55:18.970 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_snap_remove (10 ms) 2023-10-09T09:55:18.971 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T09:55:18.978 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.flags (7 ms) 2023-10-09T09:55:18.978 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T09:55:19.012 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.metadata (34 ms) 2023-10-09T09:55:19.013 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T09:55:19.014 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:19.027 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.set_features (15 ms) 2023-10-09T09:55:19.027 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T09:55:19.039 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:19.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:19.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:19.078 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror (50 ms) 2023-10-09T09:55:19.078 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T09:55:19.104 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_image (26 ms) 2023-10-09T09:55:19.104 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T09:55:19.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:19.176 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:19.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:19.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: audit 2023-10-09T09:55:18.041909+0000 mgr.y (mgr.34107) 1484 : audit 0 from='client.35988 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:55:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: cluster 2023-10-09T09:55:18.135739+0000 mon.a (mon.0) 2589 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:55:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: cluster 2023-10-09T09:55:18.135777+0000 mon.a (mon.0) 2590 : cluster 1 Cluster is now healthy 2023-10-09T09:55:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: cluster 2023-10-09T09:55:18.138163+0000 osd.5 (osd.5) 881 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: cluster 2023-10-09T09:55:18.138783+0000 osd.7 (osd.7) 622 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: audit 2023-10-09T09:55:18.140732+0000 mon.a (mon.0) 2591 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-235913-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T09:55:19.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: audit 2023-10-09T09:55:18.140937+0000 mon.a (mon.0) 2592 : audit 1 from='client.? 172.21.15.163:0/2583723883' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:55:19.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: cluster 2023-10-09T09:55:18.140993+0000 mon.a (mon.0) 2593 : cluster 0 osdmap e1291: 8 total, 8 up, 6 in 2023-10-09T09:55:19.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: cluster 2023-10-09T09:55:18.141367+0000 osd.5 (osd.5) 882 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: cluster 2023-10-09T09:55:18.141569+0000 osd.6 (osd.6) 568 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: audit 2023-10-09T09:55:18.968910+0000 mon.a (mon.0) 2594 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:19.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[78107]: audit 2023-10-09T09:55:18.976177+0000 mon.a (mon.0) 2595 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:19.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: audit 2023-10-09T09:55:18.041909+0000 mgr.y (mgr.34107) 1484 : audit 0 from='client.35988 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:55:19.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: cluster 2023-10-09T09:55:18.135739+0000 mon.a (mon.0) 2589 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:55:19.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: cluster 2023-10-09T09:55:18.135777+0000 mon.a (mon.0) 2590 : cluster 1 Cluster is now healthy 2023-10-09T09:55:19.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: cluster 2023-10-09T09:55:18.138163+0000 osd.5 (osd.5) 881 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: cluster 2023-10-09T09:55:18.138783+0000 osd.7 (osd.7) 622 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: audit 2023-10-09T09:55:18.140732+0000 mon.a (mon.0) 2591 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-235913-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T09:55:19.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: audit 2023-10-09T09:55:18.140937+0000 mon.a (mon.0) 2592 : audit 1 from='client.? 172.21.15.163:0/2583723883' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:55:19.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: cluster 2023-10-09T09:55:18.140993+0000 mon.a (mon.0) 2593 : cluster 0 osdmap e1291: 8 total, 8 up, 6 in 2023-10-09T09:55:19.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: cluster 2023-10-09T09:55:18.141367+0000 osd.5 (osd.5) 882 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: cluster 2023-10-09T09:55:18.141569+0000 osd.6 (osd.6) 568 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: audit 2023-10-09T09:55:18.968910+0000 mon.a (mon.0) 2594 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:19.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:19 smithi163 bash[75425]: audit 2023-10-09T09:55:18.976177+0000 mon.a (mon.0) 2595 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:19.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:19.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:19.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:19.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:19.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:55:19.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:19.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: audit 2023-10-09T09:55:18.041909+0000 mgr.y (mgr.34107) 1484 : audit 0 from='client.35988 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:55:19.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: cluster 2023-10-09T09:55:18.135739+0000 mon.a (mon.0) 2589 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:55:19.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: cluster 2023-10-09T09:55:18.135777+0000 mon.a (mon.0) 2590 : cluster 1 Cluster is now healthy 2023-10-09T09:55:19.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: cluster 2023-10-09T09:55:18.138163+0000 osd.5 (osd.5) 881 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: cluster 2023-10-09T09:55:18.138783+0000 osd.7 (osd.7) 622 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: audit 2023-10-09T09:55:18.140732+0000 mon.a (mon.0) 2591 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-235913-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T09:55:19.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: audit 2023-10-09T09:55:18.140937+0000 mon.a (mon.0) 2592 : audit 1 from='client.? 172.21.15.163:0/2583723883' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:55:19.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: cluster 2023-10-09T09:55:18.140993+0000 mon.a (mon.0) 2593 : cluster 0 osdmap e1291: 8 total, 8 up, 6 in 2023-10-09T09:55:19.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: cluster 2023-10-09T09:55:18.141367+0000 osd.5 (osd.5) 882 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: cluster 2023-10-09T09:55:18.141569+0000 osd.6 (osd.6) 568 : cluster 3 failed to encode map e1291 with expected crc 2023-10-09T09:55:19.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: audit 2023-10-09T09:55:18.968910+0000 mon.a (mon.0) 2594 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:19.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:19 smithi181 bash[59345]: audit 2023-10-09T09:55:18.976177+0000 mon.a (mon.0) 2595 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:19.438 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:19.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:19.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:19.541 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:19.565 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:19.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:19.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:19.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:19.693 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:19.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:19.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:19.796 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:19.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:19.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:19.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2364,"stamp":"2023-10-09T09:55:17.135858+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":592828,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4818628,"kb_used_data":39636,"kb_used_omap":607,"kb_used_meta":4778336,"kb_avail":557652284,"statfs":{"total":575970213888,"available":571035938816,"internally_reserved":0,"allocated":40587264,"data_stored":35028952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":621657,"internal_metadata":4893016999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.861489"},"pg_stats":[{"pgid":"12.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1a","version":"61'1","reported_seq":68,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400972+0000","last_change":"2023-10-09T09:48:07.991289+0000","last_active":"2023-10-09T09:54:44.400972+0000","last_peered":"2023-10-09T09:54:44.400972+0000","last_clean":"2023-10-09T09:54:44.400972+0000","last_became_active":"2023-10-09T09:48:07.990686+0000","last_became_peered":"2023-10-09T09:48:07.990686+0000","last_unstale":"2023-10-09T09:54:44.400972+0000","last_undegraded":"2023-10-09T09:54:44.400972+0000","last_fullsized":"2023-10-09T09:54:44.400972+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:24.421374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":171,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400334+0000","last_change":"2023-10-09T09:48:07.999947+0000","last_active":"2023-10-09T09:54:44.400334+0000","last_peered":"2023-10-09T09:54:44.400334+0000","last_clean":"2023-10-09T09:54:44.400334+0000","last_became_active":"2023-10-09T09:48:07.999704+0000","last_became_peered":"2023-10-09T09:48:07.999704+0000","last_unstale":"2023-10-09T09:54:44.400334+0000","last_undegraded":"2023-10-09T09:54:44.400334+0000","last_fullsized":"2023-10-09T09:54:44.400334+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:14.832042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1240,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399677+0000","last_change":"2023-10-09T09:48:07.998214+0000","last_active":"2023-10-09T09:54:44.399677+0000","last_peered":"2023-10-09T09:54:44.399677+0000","last_clean":"2023-10-09T09:54:44.399677+0000","last_became_active":"2023-10-09T09:48:07.997544+0000","last_became_peered":"2023-10-09T09:48:07.997544+0000","last_unstale":"2023-10-09T09:54:44.399677+0000","last_undegraded":"2023-10-09T09:54:44.399677+0000","last_fullsized":"2023-10-09T09:54:44.399677+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.304000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1250,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217455+0000","last_change":"2023-10-09T09:48:07.980358+0000","last_active":"2023-10-09T09:54:44.217455+0000","last_peered":"2023-10-09T09:54:44.217455+0000","last_clean":"2023-10-09T09:54:44.217455+0000","last_became_active":"2023-10-09T09:48:07.979229+0000","last_became_peered":"2023-10-09T09:48:07.979229+0000","last_unstale":"2023-10-09T09:54:44.217455+0000","last_undegraded":"2023-10-09T09:54:44.217455+0000","last_fullsized":"2023-10-09T09:54:44.217455+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:31.456347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":784,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.216413+0000","last_change":"2023-10-09T09:48:07.969728+0000","last_active":"2023-10-09T09:54:44.216413+0000","last_peered":"2023-10-09T09:54:44.216413+0000","last_clean":"2023-10-09T09:54:44.216413+0000","last_became_active":"2023-10-09T09:48:07.968773+0000","last_became_peered":"2023-10-09T09:48:07.968773+0000","last_unstale":"2023-10-09T09:54:44.216413+0000","last_undegraded":"2023-10-09T09:54:44.216413+0000","last_fullsized":"2023-10-09T09:54:44.216413+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:39.023317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1253,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036471+0000","last_change":"2023-10-09T09:48:07.988928+0000","last_active":"2023-10-09T09:54:44.036471+0000","last_peered":"2023-10-09T09:54:44.036471+0000","last_clean":"2023-10-09T09:54:44.036471+0000","last_became_active":"2023-10-09T09:48:07.988051+0000","last_became_peered":"2023-10-09T09:48:07.988051+0000","last_unstale":"2023-10-09T09:54:44.036471+0000","last_undegraded":"2023-10-09T09:54:44.036471+0000","last_fullsized":"2023-10-09T09:54:44.036471+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1212,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034547+0000","last_change":"2023-10-09T09:43:48.902066+0000","last_active":"2023-10-09T09:54:44.034547+0000","last_peered":"2023-10-09T09:54:44.034547+0000","last_clean":"2023-10-09T09:54:44.034547+0000","last_became_active":"2023-10-09T09:43:48.901639+0000","last_became_peered":"2023-10-09T09:43:48.901639+0000","last_unstale":"2023-10-09T09:54:44.034547+0000","last_undegraded":"2023-10-09T09:54:44.034547+0000","last_fullsized":"2023-10-09T09:54:44.034547+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":683,"reported_epoch":1289,"state":"active+clean","last_fresh":"2023-10-09T09:54:53.520090+0000","last_change":"2023-10-09T09:48:07.997787+0000","last_active":"2023-10-09T09:54:53.520090+0000","last_peered":"2023-10-09T09:54:53.520090+0000","last_clean":"2023-10-09T09:54:53.520090+0000","last_became_active":"2023-10-09T09:48:07.997356+0000","last_became_peered":"2023-10-09T09:48:07.997356+0000","last_unstale":"2023-10-09T09:54:53.520090+0000","last_undegraded":"2023-10-09T09:54:53.520090+0000","last_fullsized":"2023-10-09T09:54:53.520090+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:03.024096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":155,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034155+0000","last_change":"2023-10-09T09:48:07.975547+0000","last_active":"2023-10-09T09:54:44.034155+0000","last_peered":"2023-10-09T09:54:44.034155+0000","last_clean":"2023-10-09T09:54:44.034155+0000","last_became_active":"2023-10-09T09:48:07.975150+0000","last_became_peered":"2023-10-09T09:48:07.975150+0000","last_unstale":"2023-10-09T09:54:44.034155+0000","last_undegraded":"2023-10-09T09:54:44.034155+0000","last_fullsized":"2023-10-09T09:54:44.034155+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":704,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036405+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:54:44.036405+0000","last_peered":"2023-10-09T09:54:44.036405+0000","last_clean":"2023-10-09T09:54:44.036405+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:54:44.036405+0000","last_undegraded":"2023-10-09T09:54:44.036405+0000","last_fullsized":"2023-10-09T09:54:44.036405+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":97,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035611+0000","last_change":"2023-10-09T09:43:48.906785+0000","last_active":"2023-10-09T09:54:44.035611+0000","last_peered":"2023-10-09T09:54:44.035611+0000","last_clean":"2023-10-09T09:54:44.035611+0000","last_became_active":"2023-10-09T09:43:48.906276+0000","last_became_peered":"2023-10-09T09:43:48.906276+0000","last_unstale":"2023-10-09T09:54:44.035611+0000","last_undegraded":"2023-10-09T09:54:44.035611+0000","last_fullsized":"2023-10-09T09:54:44.035611+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":121,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.033330+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:54:44.033330+0000","last_peered":"2023-10-09T09:54:44.033330+0000","last_clean":"2023-10-09T09:54:44.033330+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:54:44.033330+0000","last_undegraded":"2023-10-09T09:54:44.033330+0000","last_fullsized":"2023-10-09T09:54:44.033330+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":445,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035159+0000","last_change":"2023-10-09T09:43:23.995526+0000","last_active":"2023-10-09T09:54:44.035159+0000","last_peered":"2023-10-09T09:54:44.035159+0000","last_clean":"2023-10-09T09:54:44.035159+0000","last_became_active":"2023-10-09T09:43:23.994816+0000","last_became_peered":"2023-10-09T09:43:23.994816+0000","last_unstale":"2023-10-09T09:54:44.035159+0000","last_undegraded":"2023-10-09T09:54:44.035159+0000","last_fullsized":"2023-10-09T09:54:44.035159+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":662,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035975+0000","last_change":"2023-10-09T09:48:07.987637+0000","last_active":"2023-10-09T09:54:44.035975+0000","last_peered":"2023-10-09T09:54:44.035975+0000","last_clean":"2023-10-09T09:54:44.035975+0000","last_became_active":"2023-10-09T09:48:07.987085+0000","last_became_peered":"2023-10-09T09:48:07.987085+0000","last_unstale":"2023-10-09T09:54:44.035975+0000","last_undegraded":"2023-10-09T09:54:44.035975+0000","last_fullsized":"2023-10-09T09:54:44.035975+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1186,"reported_epoch":1289,"state":"active+clean","last_fresh":"2023-10-09T09:54:53.065784+0000","last_change":"2023-10-09T09:43:23.989356+0000","last_active":"2023-10-09T09:54:53.065784+0000","last_peered":"2023-10-09T09:54:53.065784+0000","last_clean":"2023-10-09T09:54:53.065784+0000","last_became_active":"2023-10-09T09:43:23.988986+0000","last_became_peered":"2023-10-09T09:43:23.988986+0000","last_unstale":"2023-10-09T09:54:53.065784+0000","last_undegraded":"2023-10-09T09:54:53.065784+0000","last_fullsized":"2023-10-09T09:54:53.065784+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.570448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":768,"reported_epoch":1289,"state":"active+clean","last_fresh":"2023-10-09T09:54:53.537233+0000","last_change":"2023-10-09T09:48:07.984006+0000","last_active":"2023-10-09T09:54:53.537233+0000","last_peered":"2023-10-09T09:54:53.537233+0000","last_clean":"2023-10-09T09:54:53.537233+0000","last_became_active":"2023-10-09T09:48:07.978179+0000","last_became_peered":"2023-10-09T09:48:07.978179+0000","last_unstale":"2023-10-09T09:54:53.537233+0000","last_undegraded":"2023-10-09T09:54:53.537233+0000","last_fullsized":"2023-10-09T09:54:53.537233+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:34.164004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1189,"reported_epoch":1289,"state":"active+clean","last_fresh":"2023-10-09T09:54:53.523088+0000","last_change":"2023-10-09T09:48:07.975335+0000","last_active":"2023-10-09T09:54:53.523088+0000","last_peered":"2023-10-09T09:54:53.523088+0000","last_clean":"2023-10-09T09:54:53.523088+0000","last_became_active":"2023-10-09T09:48:07.974437+0000","last_became_peered":"2023-10-09T09:48:07.974437+0000","last_unstale":"2023-10-09T09:54:53.523088+0000","last_undegraded":"2023-10-09T09:54:53.523088+0000","last_fullsized":"2023-10-09T09:54:53.523088+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:32.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":125,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036835+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:54:44.036835+0000","last_peered":"2023-10-09T09:54:44.036835+0000","last_clean":"2023-10-09T09:54:44.036835+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:54:44.036835+0000","last_undegraded":"2023-10-09T09:54:44.036835+0000","last_fullsized":"2023-10-09T09:54:44.036835+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":89,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035683+0000","last_change":"2023-10-09T09:43:23.994784+0000","last_active":"2023-10-09T09:54:44.035683+0000","last_peered":"2023-10-09T09:54:44.035683+0000","last_clean":"2023-10-09T09:54:44.035683+0000","last_became_active":"2023-10-09T09:43:23.994196+0000","last_became_peered":"2023-10-09T09:43:23.994196+0000","last_unstale":"2023-10-09T09:54:44.035683+0000","last_undegraded":"2023-10-09T09:54:44.035683+0000","last_fullsized":"2023-10-09T09:54:44.035683+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1206,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399686+0000","last_change":"2023-10-09T09:43:48.901303+0000","last_active":"2023-10-09T09:54:44.399686+0000","last_peered":"2023-10-09T09:54:44.399686+0000","last_clean":"2023-10-09T09:54:44.399686+0000","last_became_active":"2023-10-09T09:43:48.900779+0000","last_became_peered":"2023-10-09T09:43:48.900779+0000","last_unstale":"2023-10-09T09:54:44.399686+0000","last_undegraded":"2023-10-09T09:54:44.399686+0000","last_fullsized":"2023-10-09T09:54:44.399686+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:51.853531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1283,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034638+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:54:44.034638+0000","last_peered":"2023-10-09T09:54:44.034638+0000","last_clean":"2023-10-09T09:54:44.034638+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:54:44.034638+0000","last_undegraded":"2023-10-09T09:54:44.034638+0000","last_fullsized":"2023-10-09T09:54:44.034638+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1250,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.401076+0000","last_change":"2023-10-09T09:48:08.000694+0000","last_active":"2023-10-09T09:54:44.401076+0000","last_peered":"2023-10-09T09:54:44.401076+0000","last_clean":"2023-10-09T09:54:44.401076+0000","last_became_active":"2023-10-09T09:48:07.999476+0000","last_became_peered":"2023-10-09T09:48:07.999476+0000","last_unstale":"2023-10-09T09:54:44.401076+0000","last_undegraded":"2023-10-09T09:54:44.401076+0000","last_fullsized":"2023-10-09T09:54:44.401076+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:37.211076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":117,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217596+0000","last_change":"2023-10-09T09:48:07.976558+0000","last_active":"2023-10-09T09:54:44.217596+0000","last_peered":"2023-10-09T09:54:44.217596+0000","last_clean":"2023-10-09T09:54:44.217596+0000","last_became_active":"2023-10-09T09:48:07.975961+0000","last_became_peered":"2023-10-09T09:48:07.975961+0000","last_unstale":"2023-10-09T09:54:44.217596+0000","last_undegraded":"2023-10-09T09:54:44.217596+0000","last_fullsized":"2023-10-09T09:54:44.217596+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:40.925248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":131,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.032754+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:54:44.032754+0000","last_peered":"2023-10-09T09:54:44.032754+0000","last_clean":"2023-10-09T09:54:44.032754+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:54:44.032754+0000","last_undegraded":"2023-10-09T09:54:44.032754+0000","last_fullsized":"2023-10-09T09:54:44.032754+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":80,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:52.023668+0000","last_change":"2023-10-09T09:48:07.990711+0000","last_active":"2023-10-09T09:54:52.023668+0000","last_peered":"2023-10-09T09:54:52.023668+0000","last_clean":"2023-10-09T09:54:52.023668+0000","last_became_active":"2023-10-09T09:48:07.990267+0000","last_became_peered":"2023-10-09T09:48:07.990267+0000","last_unstale":"2023-10-09T09:54:52.023668+0000","last_undegraded":"2023-10-09T09:54:52.023668+0000","last_fullsized":"2023-10-09T09:54:52.023668+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:51.244840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1224,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399845+0000","last_change":"2023-10-09T09:48:07.991440+0000","last_active":"2023-10-09T09:54:44.399845+0000","last_peered":"2023-10-09T09:54:44.399845+0000","last_clean":"2023-10-09T09:54:44.399845+0000","last_became_active":"2023-10-09T09:48:07.991072+0000","last_became_peered":"2023-10-09T09:48:07.991072+0000","last_unstale":"2023-10-09T09:54:44.399845+0000","last_undegraded":"2023-10-09T09:54:44.399845+0000","last_fullsized":"2023-10-09T09:54:44.399845+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:28.577159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1239,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034132+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:54:44.034132+0000","last_peered":"2023-10-09T09:54:44.034132+0000","last_clean":"2023-10-09T09:54:44.034132+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:54:44.034132+0000","last_undegraded":"2023-10-09T09:54:44.034132+0000","last_fullsized":"2023-10-09T09:54:44.034132+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1245,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034994+0000","last_change":"2023-10-09T09:48:07.987819+0000","last_active":"2023-10-09T09:54:44.034994+0000","last_peered":"2023-10-09T09:54:44.034994+0000","last_clean":"2023-10-09T09:54:44.034994+0000","last_became_active":"2023-10-09T09:48:07.986354+0000","last_became_peered":"2023-10-09T09:48:07.986354+0000","last_unstale":"2023-10-09T09:54:44.034994+0000","last_undegraded":"2023-10-09T09:54:44.034994+0000","last_fullsized":"2023-10-09T09:54:44.034994+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":791,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035547+0000","last_change":"2023-10-09T09:48:07.987506+0000","last_active":"2023-10-09T09:54:44.035547+0000","last_peered":"2023-10-09T09:54:44.035547+0000","last_clean":"2023-10-09T09:54:44.035547+0000","last_became_active":"2023-10-09T09:48:07.986988+0000","last_became_peered":"2023-10-09T09:48:07.986988+0000","last_unstale":"2023-10-09T09:54:44.035547+0000","last_undegraded":"2023-10-09T09:54:44.035547+0000","last_fullsized":"2023-10-09T09:54:44.035547+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1209,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217182+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:54:44.217182+0000","last_peered":"2023-10-09T09:54:44.217182+0000","last_clean":"2023-10-09T09:54:44.217182+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:54:44.217182+0000","last_undegraded":"2023-10-09T09:54:44.217182+0000","last_fullsized":"2023-10-09T09:54:44.217182+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":168,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:52.023551+0000","last_change":"2023-10-09T09:48:07.977143+0000","last_active":"2023-10-09T09:54:52.023551+0000","last_peered":"2023-10-09T09:54:52.023551+0000","last_clean":"2023-10-09T09:54:52.023551+0000","last_became_active":"2023-10-09T09:48:07.976338+0000","last_became_peered":"2023-10-09T09:48:07.976338+0000","last_unstale":"2023-10-09T09:54:52.023551+0000","last_undegraded":"2023-10-09T09:54:52.023551+0000","last_fullsized":"2023-10-09T09:54:52.023551+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:12.015053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1241,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035385+0000","last_change":"2023-10-09T09:48:08.001503+0000","last_active":"2023-10-09T09:54:44.035385+0000","last_peered":"2023-10-09T09:54:44.035385+0000","last_clean":"2023-10-09T09:54:44.035385+0000","last_became_active":"2023-10-09T09:48:08.000921+0000","last_became_peered":"2023-10-09T09:48:08.000921+0000","last_unstale":"2023-10-09T09:54:44.035385+0000","last_undegraded":"2023-10-09T09:54:44.035385+0000","last_fullsized":"2023-10-09T09:54:44.035385+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1237,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036275+0000","last_change":"2023-10-09T09:48:07.988586+0000","last_active":"2023-10-09T09:54:44.036275+0000","last_peered":"2023-10-09T09:54:44.036275+0000","last_clean":"2023-10-09T09:54:44.036275+0000","last_became_active":"2023-10-09T09:48:07.988226+0000","last_became_peered":"2023-10-09T09:48:07.988226+0000","last_unstale":"2023-10-09T09:54:44.036275+0000","last_undegraded":"2023-10-09T09:54:44.036275+0000","last_fullsized":"2023-10-09T09:54:44.036275+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1230,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.398948+0000","last_change":"2023-10-09T09:48:07.987993+0000","last_active":"2023-10-09T09:54:44.398948+0000","last_peered":"2023-10-09T09:54:44.398948+0000","last_clean":"2023-10-09T09:54:44.398948+0000","last_became_active":"2023-10-09T09:48:07.987664+0000","last_became_peered":"2023-10-09T09:48:07.987664+0000","last_unstale":"2023-10-09T09:54:44.398948+0000","last_undegraded":"2023-10-09T09:54:44.398948+0000","last_fullsized":"2023-10-09T09:54:44.398948+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:32.253321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1258,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399116+0000","last_change":"2023-10-09T09:48:07.984079+0000","last_active":"2023-10-09T09:54:44.399116+0000","last_peered":"2023-10-09T09:54:44.399116+0000","last_clean":"2023-10-09T09:54:44.399116+0000","last_became_active":"2023-10-09T09:48:07.983792+0000","last_became_peered":"2023-10-09T09:48:07.983792+0000","last_unstale":"2023-10-09T09:54:44.399116+0000","last_undegraded":"2023-10-09T09:54:44.399116+0000","last_fullsized":"2023-10-09T09:54:44.399116+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:28:42.289832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":662,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035945+0000","last_change":"2023-10-09T09:48:07.989047+0000","last_active":"2023-10-09T09:54:44.035945+0000","last_peered":"2023-10-09T09:54:44.035945+0000","last_clean":"2023-10-09T09:54:44.035945+0000","last_became_active":"2023-10-09T09:48:07.988131+0000","last_became_peered":"2023-10-09T09:48:07.988131+0000","last_unstale":"2023-10-09T09:54:44.035945+0000","last_undegraded":"2023-10-09T09:54:44.035945+0000","last_fullsized":"2023-10-09T09:54:44.035945+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1200,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400105+0000","last_change":"2023-10-09T09:48:07.976884+0000","last_active":"2023-10-09T09:54:44.400105+0000","last_peered":"2023-10-09T09:54:44.400105+0000","last_clean":"2023-10-09T09:54:44.400105+0000","last_became_active":"2023-10-09T09:48:07.975932+0000","last_became_peered":"2023-10-09T09:48:07.975932+0000","last_unstale":"2023-10-09T09:54:44.400105+0000","last_undegraded":"2023-10-09T09:54:44.400105+0000","last_fullsized":"2023-10-09T09:54:44.400105+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:22.785728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":136,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399422+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:54:44.399422+0000","last_peered":"2023-10-09T09:54:44.399422+0000","last_clean":"2023-10-09T09:54:44.399422+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:54:44.399422+0000","last_undegraded":"2023-10-09T09:54:44.399422+0000","last_fullsized":"2023-10-09T09:54:44.399422+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1246,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.401507+0000","last_change":"2023-10-09T09:48:07.984553+0000","last_active":"2023-10-09T09:54:44.401507+0000","last_peered":"2023-10-09T09:54:44.401507+0000","last_clean":"2023-10-09T09:54:44.401507+0000","last_became_active":"2023-10-09T09:48:07.983302+0000","last_became_peered":"2023-10-09T09:48:07.983302+0000","last_unstale":"2023-10-09T09:54:44.401507+0000","last_undegraded":"2023-10-09T09:54:44.401507+0000","last_fullsized":"2023-10-09T09:54:44.401507+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:50.714295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":131,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.033909+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:54:44.033909+0000","last_peered":"2023-10-09T09:54:44.033909+0000","last_clean":"2023-10-09T09:54:44.033909+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:54:44.033909+0000","last_undegraded":"2023-10-09T09:54:44.033909+0000","last_fullsized":"2023-10-09T09:54:44.033909+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1223,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399707+0000","last_change":"2023-10-09T09:48:07.977683+0000","last_active":"2023-10-09T09:54:44.399707+0000","last_peered":"2023-10-09T09:54:44.399707+0000","last_clean":"2023-10-09T09:54:44.399707+0000","last_became_active":"2023-10-09T09:48:07.977024+0000","last_became_peered":"2023-10-09T09:48:07.977024+0000","last_unstale":"2023-10-09T09:54:44.399707+0000","last_undegraded":"2023-10-09T09:54:44.399707+0000","last_fullsized":"2023-10-09T09:54:44.399707+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:13.438431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":706,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400727+0000","last_change":"2023-10-09T09:48:07.997811+0000","last_active":"2023-10-09T09:54:44.400727+0000","last_peered":"2023-10-09T09:54:44.400727+0000","last_clean":"2023-10-09T09:54:44.400727+0000","last_became_active":"2023-10-09T09:48:07.997389+0000","last_became_peered":"2023-10-09T09:48:07.997389+0000","last_unstale":"2023-10-09T09:54:44.400727+0000","last_undegraded":"2023-10-09T09:54:44.400727+0000","last_fullsized":"2023-10-09T09:54:44.400727+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:21.978398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1179,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036322+0000","last_change":"2023-10-09T09:43:48.908190+0000","last_active":"2023-10-09T09:54:44.036322+0000","last_peered":"2023-10-09T09:54:44.036322+0000","last_clean":"2023-10-09T09:54:44.036322+0000","last_became_active":"2023-10-09T09:43:48.907799+0000","last_became_peered":"2023-10-09T09:43:48.907799+0000","last_unstale":"2023-10-09T09:54:44.036322+0000","last_undegraded":"2023-10-09T09:54:44.036322+0000","last_fullsized":"2023-10-09T09:54:44.036322+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1276,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400635+0000","last_change":"2023-10-09T09:48:08.000154+0000","last_active":"2023-10-09T09:54:44.400635+0000","last_peered":"2023-10-09T09:54:44.400635+0000","last_clean":"2023-10-09T09:54:44.400635+0000","last_became_active":"2023-10-09T09:48:07.999714+0000","last_became_peered":"2023-10-09T09:48:07.999714+0000","last_unstale":"2023-10-09T09:54:44.400635+0000","last_undegraded":"2023-10-09T09:54:44.400635+0000","last_fullsized":"2023-10-09T09:54:44.400635+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:12.921693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":181,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399290+0000","last_change":"2023-10-09T09:48:07.983969+0000","last_active":"2023-10-09T09:54:44.399290+0000","last_peered":"2023-10-09T09:54:44.399290+0000","last_clean":"2023-10-09T09:54:44.399290+0000","last_became_active":"2023-10-09T09:48:07.983490+0000","last_became_peered":"2023-10-09T09:48:07.983490+0000","last_unstale":"2023-10-09T09:54:44.399290+0000","last_undegraded":"2023-10-09T09:54:44.399290+0000","last_fullsized":"2023-10-09T09:54:44.399290+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:21.122987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1254,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399596+0000","last_change":"2023-10-09T09:48:07.998234+0000","last_active":"2023-10-09T09:54:44.399596+0000","last_peered":"2023-10-09T09:54:44.399596+0000","last_clean":"2023-10-09T09:54:44.399596+0000","last_became_active":"2023-10-09T09:48:07.997909+0000","last_became_peered":"2023-10-09T09:48:07.997909+0000","last_unstale":"2023-10-09T09:54:44.399596+0000","last_undegraded":"2023-10-09T09:54:44.399596+0000","last_fullsized":"2023-10-09T09:54:44.399596+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:21.750617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1243,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400587+0000","last_change":"2023-10-09T09:48:07.973284+0000","last_active":"2023-10-09T09:54:44.400587+0000","last_peered":"2023-10-09T09:54:44.400587+0000","last_clean":"2023-10-09T09:54:44.400587+0000","last_became_active":"2023-10-09T09:48:07.971235+0000","last_became_peered":"2023-10-09T09:48:07.971235+0000","last_unstale":"2023-10-09T09:54:44.400587+0000","last_undegraded":"2023-10-09T09:54:44.400587+0000","last_fullsized":"2023-10-09T09:54:44.400587+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:40.951420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1299,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.398831+0000","last_change":"2023-10-09T09:41:11.795802+0000","last_active":"2023-10-09T09:54:44.398831+0000","last_peered":"2023-10-09T09:54:44.398831+0000","last_clean":"2023-10-09T09:54:44.398831+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:54:44.398831+0000","last_undegraded":"2023-10-09T09:54:44.398831+0000","last_fullsized":"2023-10-09T09:54:44.398831+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":633,"reported_epoch":1289,"state":"active+clean","last_fresh":"2023-10-09T09:54:53.526821+0000","last_change":"2023-10-09T09:48:07.978435+0000","last_active":"2023-10-09T09:54:53.526821+0000","last_peered":"2023-10-09T09:54:53.526821+0000","last_clean":"2023-10-09T09:54:53.526821+0000","last_became_active":"2023-10-09T09:48:07.977417+0000","last_became_peered":"2023-10-09T09:48:07.977417+0000","last_unstale":"2023-10-09T09:54:53.526821+0000","last_undegraded":"2023-10-09T09:54:53.526821+0000","last_fullsized":"2023-10-09T09:54:53.526821+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:19.288202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":131,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.033240+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:54:44.033240+0000","last_peered":"2023-10-09T09:54:44.033240+0000","last_clean":"2023-10-09T09:54:44.033240+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:54:44.033240+0000","last_undegraded":"2023-10-09T09:54:44.033240+0000","last_fullsized":"2023-10-09T09:54:44.033240+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1215,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036469+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:54:44.036469+0000","last_peered":"2023-10-09T09:54:44.036469+0000","last_clean":"2023-10-09T09:54:44.036469+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:54:44.036469+0000","last_undegraded":"2023-10-09T09:54:44.036469+0000","last_fullsized":"2023-10-09T09:54:44.036469+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1220,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399734+0000","last_change":"2023-10-09T09:48:07.973246+0000","last_active":"2023-10-09T09:54:44.399734+0000","last_peered":"2023-10-09T09:54:44.399734+0000","last_clean":"2023-10-09T09:54:44.399734+0000","last_became_active":"2023-10-09T09:48:07.971221+0000","last_became_peered":"2023-10-09T09:48:07.971221+0000","last_unstale":"2023-10-09T09:54:44.399734+0000","last_undegraded":"2023-10-09T09:54:44.399734+0000","last_fullsized":"2023-10-09T09:54:44.399734+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:16.759514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1227,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.216793+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:54:44.216793+0000","last_peered":"2023-10-09T09:54:44.216793+0000","last_clean":"2023-10-09T09:54:44.216793+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:54:44.216793+0000","last_undegraded":"2023-10-09T09:54:44.216793+0000","last_fullsized":"2023-10-09T09:54:44.216793+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1239,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:49.148600+0000","last_change":"2023-10-09T09:48:07.981410+0000","last_active":"2023-10-09T09:54:49.148600+0000","last_peered":"2023-10-09T09:54:49.148600+0000","last_clean":"2023-10-09T09:54:49.148600+0000","last_became_active":"2023-10-09T09:48:07.979850+0000","last_became_peered":"2023-10-09T09:48:07.979850+0000","last_unstale":"2023-10-09T09:54:49.148600+0000","last_undegraded":"2023-10-09T09:54:49.148600+0000","last_fullsized":"2023-10-09T09:54:49.148600+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:31.201104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1214,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217239+0000","last_change":"2023-10-09T09:48:07.975786+0000","last_active":"2023-10-09T09:54:44.217239+0000","last_peered":"2023-10-09T09:54:44.217239+0000","last_clean":"2023-10-09T09:54:44.217239+0000","last_became_active":"2023-10-09T09:48:07.975338+0000","last_became_peered":"2023-10-09T09:48:07.975338+0000","last_unstale":"2023-10-09T09:54:44.217239+0000","last_undegraded":"2023-10-09T09:54:44.217239+0000","last_fullsized":"2023-10-09T09:54:44.217239+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:05.175956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1257,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.401287+0000","last_change":"2023-10-09T09:48:07.983773+0000","last_active":"2023-10-09T09:54:44.401287+0000","last_peered":"2023-10-09T09:54:44.401287+0000","last_clean":"2023-10-09T09:54:44.401287+0000","last_became_active":"2023-10-09T09:48:07.976869+0000","last_became_peered":"2023-10-09T09:48:07.976869+0000","last_unstale":"2023-10-09T09:54:44.401287+0000","last_undegraded":"2023-10-09T09:54:44.401287+0000","last_fullsized":"2023-10-09T09:54:44.401287+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:58.173236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1205,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.033269+0000","last_change":"2023-10-09T09:48:07.962302+0000","last_active":"2023-10-09T09:54:44.033269+0000","last_peered":"2023-10-09T09:54:44.033269+0000","last_clean":"2023-10-09T09:54:44.033269+0000","last_became_active":"2023-10-09T09:48:07.961888+0000","last_became_peered":"2023-10-09T09:48:07.961888+0000","last_unstale":"2023-10-09T09:54:44.033269+0000","last_undegraded":"2023-10-09T09:54:44.033269+0000","last_fullsized":"2023-10-09T09:54:44.033269+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":152,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217747+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:54:44.217747+0000","last_peered":"2023-10-09T09:54:44.217747+0000","last_clean":"2023-10-09T09:54:44.217747+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:54:44.217747+0000","last_undegraded":"2023-10-09T09:54:44.217747+0000","last_fullsized":"2023-10-09T09:54:44.217747+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1209,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.401008+0000","last_change":"2023-10-09T09:48:07.999034+0000","last_active":"2023-10-09T09:54:44.401008+0000","last_peered":"2023-10-09T09:54:44.401008+0000","last_clean":"2023-10-09T09:54:44.401008+0000","last_became_active":"2023-10-09T09:48:07.998606+0000","last_became_peered":"2023-10-09T09:48:07.998606+0000","last_unstale":"2023-10-09T09:54:44.401008+0000","last_undegraded":"2023-10-09T09:54:44.401008+0000","last_fullsized":"2023-10-09T09:54:44.401008+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.469476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1145,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399283+0000","last_change":"2023-10-09T09:48:07.976350+0000","last_active":"2023-10-09T09:54:44.399283+0000","last_peered":"2023-10-09T09:54:44.399283+0000","last_clean":"2023-10-09T09:54:44.399283+0000","last_became_active":"2023-10-09T09:48:07.973099+0000","last_became_peered":"2023-10-09T09:48:07.973099+0000","last_unstale":"2023-10-09T09:54:44.399283+0000","last_undegraded":"2023-10-09T09:54:44.399283+0000","last_fullsized":"2023-10-09T09:54:44.399283+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:48.812722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1276,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036611+0000","last_change":"2023-10-09T09:48:07.990005+0000","last_active":"2023-10-09T09:54:44.036611+0000","last_peered":"2023-10-09T09:54:44.036611+0000","last_clean":"2023-10-09T09:54:44.036611+0000","last_became_active":"2023-10-09T09:48:07.987124+0000","last_became_peered":"2023-10-09T09:48:07.987124+0000","last_unstale":"2023-10-09T09:54:44.036611+0000","last_undegraded":"2023-10-09T09:54:44.036611+0000","last_fullsized":"2023-10-09T09:54:44.036611+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1220,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:49.148639+0000","last_change":"2023-10-09T09:48:07.980600+0000","last_active":"2023-10-09T09:54:49.148639+0000","last_peered":"2023-10-09T09:54:49.148639+0000","last_clean":"2023-10-09T09:54:49.148639+0000","last_became_active":"2023-10-09T09:48:07.979544+0000","last_became_peered":"2023-10-09T09:48:07.979544+0000","last_unstale":"2023-10-09T09:54:49.148639+0000","last_undegraded":"2023-10-09T09:54:49.148639+0000","last_fullsized":"2023-10-09T09:54:49.148639+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:47.320689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1207,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.033139+0000","last_change":"2023-10-09T09:43:23.992213+0000","last_active":"2023-10-09T09:54:44.033139+0000","last_peered":"2023-10-09T09:54:44.033139+0000","last_clean":"2023-10-09T09:54:44.033139+0000","last_became_active":"2023-10-09T09:43:23.991887+0000","last_became_peered":"2023-10-09T09:43:23.991887+0000","last_unstale":"2023-10-09T09:54:44.033139+0000","last_undegraded":"2023-10-09T09:54:44.033139+0000","last_fullsized":"2023-10-09T09:54:44.033139+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1227,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217318+0000","last_change":"2023-10-09T09:48:07.969929+0000","last_active":"2023-10-09T09:54:44.217318+0000","last_peered":"2023-10-09T09:54:44.217318+0000","last_clean":"2023-10-09T09:54:44.217318+0000","last_became_active":"2023-10-09T09:48:07.969537+0000","last_became_peered":"2023-10-09T09:48:07.969537+0000","last_unstale":"2023-10-09T09:54:44.217318+0000","last_undegraded":"2023-10-09T09:54:44.217318+0000","last_fullsized":"2023-10-09T09:54:44.217318+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:42.344608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":180,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399951+0000","last_change":"2023-10-09T09:48:07.987618+0000","last_active":"2023-10-09T09:54:44.399951+0000","last_peered":"2023-10-09T09:54:44.399951+0000","last_clean":"2023-10-09T09:54:44.399951+0000","last_became_active":"2023-10-09T09:48:07.987341+0000","last_became_peered":"2023-10-09T09:48:07.987341+0000","last_unstale":"2023-10-09T09:54:44.399951+0000","last_undegraded":"2023-10-09T09:54:44.399951+0000","last_fullsized":"2023-10-09T09:54:44.399951+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:19.972891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":79,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400799+0000","last_change":"2023-10-09T09:48:07.984247+0000","last_active":"2023-10-09T09:54:44.400799+0000","last_peered":"2023-10-09T09:54:44.400799+0000","last_clean":"2023-10-09T09:54:44.400799+0000","last_became_active":"2023-10-09T09:48:07.978613+0000","last_became_peered":"2023-10-09T09:48:07.978613+0000","last_unstale":"2023-10-09T09:54:44.400799+0000","last_undegraded":"2023-10-09T09:54:44.400799+0000","last_fullsized":"2023-10-09T09:54:44.400799+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:34.663108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1223,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.400299+0000","last_change":"2023-10-09T09:48:07.991053+0000","last_active":"2023-10-09T09:54:44.400299+0000","last_peered":"2023-10-09T09:54:44.400299+0000","last_clean":"2023-10-09T09:54:44.400299+0000","last_became_active":"2023-10-09T09:48:07.990656+0000","last_became_peered":"2023-10-09T09:48:07.990656+0000","last_unstale":"2023-10-09T09:54:44.400299+0000","last_undegraded":"2023-10-09T09:54:44.400299+0000","last_fullsized":"2023-10-09T09:54:44.400299+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:06.223305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":97,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.036904+0000","last_change":"2023-10-09T09:43:23.995576+0000","last_active":"2023-10-09T09:54:44.036904+0000","last_peered":"2023-10-09T09:54:44.036904+0000","last_clean":"2023-10-09T09:54:44.036904+0000","last_became_active":"2023-10-09T09:43:23.995076+0000","last_became_peered":"2023-10-09T09:43:23.995076+0000","last_unstale":"2023-10-09T09:54:44.036904+0000","last_undegraded":"2023-10-09T09:54:44.036904+0000","last_fullsized":"2023-10-09T09:54:44.036904+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1259,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.399113+0000","last_change":"2023-10-09T09:43:48.904021+0000","last_active":"2023-10-09T09:54:44.399113+0000","last_peered":"2023-10-09T09:54:44.399113+0000","last_clean":"2023-10-09T09:54:44.399113+0000","last_became_active":"2023-10-09T09:43:48.903719+0000","last_became_peered":"2023-10-09T09:43:48.903719+0000","last_unstale":"2023-10-09T09:54:44.399113+0000","last_undegraded":"2023-10-09T09:54:44.399113+0000","last_fullsized":"2023-10-09T09:54:44.399113+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:16.284027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":237,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.035896+0000","last_change":"2023-10-09T09:48:08.001392+0000","last_active":"2023-10-09T09:54:44.035896+0000","last_peered":"2023-10-09T09:54:44.035896+0000","last_clean":"2023-10-09T09:54:44.035896+0000","last_became_active":"2023-10-09T09:48:08.000798+0000","last_became_peered":"2023-10-09T09:48:08.000798+0000","last_unstale":"2023-10-09T09:54:44.035896+0000","last_undegraded":"2023-10-09T09:54:44.035896+0000","last_fullsized":"2023-10-09T09:54:44.035896+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":165,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.398605+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:54:44.398605+0000","last_peered":"2023-10-09T09:54:44.398605+0000","last_clean":"2023-10-09T09:54:44.398605+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:54:44.398605+0000","last_undegraded":"2023-10-09T09:54:44.398605+0000","last_fullsized":"2023-10-09T09:54:44.398605+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T09:55:17.123854+0000","last_change":"2023-10-09T09:55:17.123854+0000","last_active":"2023-10-09T09:55:17.123854+0000","last_peered":"2023-10-09T09:55:17.123854+0000","last_clean":"2023-10-09T09:55:17.123854+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T09:55:17.123854+0000","last_undegraded":"2023-10-09T09:55:17.123854+0000","last_fullsized":"2023-10-09T09:55:17.123854+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:55:17.123854+0000","last_clean_scrub_stamp":"2023-10-09T09:55:17.123854+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1253,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.217483+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:54:44.217483+0000","last_peered":"2023-10-09T09:54:44.217483+0000","last_clean":"2023-10-09T09:54:44.217483+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:54:44.217483+0000","last_undegraded":"2023-10-09T09:54:44.217483+0000","last_fullsized":"2023-10-09T09:54:44.217483+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":662,"reported_epoch":1288,"state":"active+clean","last_fresh":"2023-10-09T09:54:44.034823+0000","last_change":"2023-10-09T09:48:07.990205+0000","last_active":"2023-10-09T09:54:44.034823+0000","last_peered":"2023-10-09T09:54:44.034823+0000","last_clean":"2023-10-09T09:54:44.034823+0000","last_became_active":"2023-10-09T09:48:07.989619+0000","last_became_peered":"2023-10-09T09:48:07.989619+0000","last_unstale":"2023-10-09T09:54:44.034823+0000","last_undegraded":"2023-10-09T09:54:44.034823+0000","last_fullsized":"2023-10-09T09:54:44.034823+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":2460,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30942,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453398162,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594394,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910952,"kb_used_data":6176,"kb_used_omap":0,"kb_used_meta":904768,"kb_avail":92834200,"statfs":{"total":95995035648,"available":95062220800,"internally_reserved":0,"allocated":6324224,"data_stored":6022068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926482432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:54:57 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.1399999999999999,"15min":2.5939999999999999},"min":{"1min":0.35099999999999998,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":5.7279999999999998,"5min":5.7279999999999998,"15min":149.095},"last":1.077},{"interface":"front","average":{"1min":1.3859999999999999,"5min":1.3049999999999999,"15min":2.8109999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.3360000000000003,"5min":7.3360000000000003,"15min":149.17500000000001},"last":1.054}]},{"osd":1,"last update":"Mon Oct 9 09:54:18 2023","interfaces":[{"interface":"back","average":{"1min":5.452,"5min":2.3340000000000001,"15min":2.2690000000000001},"min":{"1min":0.40699999999999997,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":55.368000000000002,"5min":55.368000000000002,"15min":55.368000000000002},"last":1.143},{"interface":"front","average":{"1min":5.2629999999999999,"5min":2.3399999999999999,"15min":2.141},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":55.404000000000003,"5min":55.404000000000003,"15min":55.404000000000003},"last":1.379}]},{"osd":2,"last update":"Mon Oct 9 09:54:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8370000000000002,"5min":3.6179999999999999,"15min":3.5790000000000002},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.29799999999999999},"max":{"1min":23.547000000000001,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.79300000000000004},{"interface":"front","average":{"1min":3.23,"5min":4.032,"15min":3.7509999999999999},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.309},"max":{"1min":23.600999999999999,"5min":208.90899999999999,"15min":208.90899999999999},"last":1.3280000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:54:44 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":0.94199999999999995,"15min":1.1379999999999999},"min":{"1min":0.42999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.6559999999999997,"5min":5.5910000000000002,"15min":7.9139999999999997},"last":1.2490000000000001},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.96899999999999997,"15min":1.3520000000000001},"min":{"1min":0.41099999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":3.3500000000000001,"5min":4.1669999999999998,"15min":11.968999999999999},"last":1.482}]},{"osd":4,"last update":"Mon Oct 9 09:54:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.1359999999999999,"15min":1.3779999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.28399999999999997},"max":{"1min":7.0010000000000003,"5min":7.0010000000000003,"15min":12.148999999999999},"last":1.27},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.98099999999999998,"15min":1.552},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.313},"max":{"1min":3.3170000000000002,"5min":6.7750000000000004,"15min":12.064},"last":1.3080000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:54:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":1.2150000000000001,"15min":1.446},"min":{"1min":0.433,"5min":0.318,"15min":0.318},"max":{"1min":7.3639999999999999,"5min":7.3639999999999999,"15min":10.529},"last":1.2130000000000001},{"interface":"front","average":{"1min":1.02,"5min":1.02,"15min":1.5},"min":{"1min":0.48999999999999999,"5min":0.34300000000000003,"15min":0.34000000000000002},"max":{"1min":3.282,"5min":4.0629999999999997,"15min":12.000999999999999},"last":1.522}]},{"osd":7,"last update":"Mon Oct 9 09:54:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.1579999999999999,"15min":1.526},"min":{"1min":0.438,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.9740000000000002,"5min":9.6389999999999993,"15min":11.922000000000001},"last":1.4339999999999999},{"interface":"front","average":{"1min":1.163,"5min":1.097,"15min":1.194},"min":{"1min":0.44900000000000001,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":4.6239999999999997,"5min":4.9459999999999997,"15min":7.9359999999999999},"last":1.1870000000000001}]}]},{"osd":2,"up_from":634,"seq":2723009266148,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899492,"kb_used_data":7068,"kb_used_omap":122,"kb_used_meta":892293,"kb_avail":92845660,"statfs":{"total":95995035648,"available":95073955840,"internally_reserved":0,"allocated":7237632,"data_stored":5662939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":913708619},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:54:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":3.6720000000000002,"15min":3.96},"min":{"1min":0.26500000000000001,"5min":0.221,"15min":0.20200000000000001},"max":{"1min":9.048,"5min":218.30799999999999,"15min":218.30799999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.893,"5min":3.871,"15min":4.1820000000000004},"min":{"1min":0.27900000000000003,"5min":0.221,"15min":0.20799999999999999},"max":{"1min":6.9699999999999998,"5min":217.67500000000001,"15min":217.67500000000001},"last":0.48399999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:54:21 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":3.766,"15min":1.958},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":8.5709999999999997,"5min":106.089,"15min":106.089},"last":0.60399999999999998},{"interface":"front","average":{"1min":3.1989999999999998,"5min":3.6059999999999999,"15min":1.9790000000000001},"min":{"1min":0.311,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.2249999999999996,"5min":105.571,"15min":105.571},"last":0.45900000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:54:57 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.2989999999999999,"15min":1.905},"min":{"1min":0.376,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.4459999999999997,"5min":8.1289999999999996,"15min":230.476},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.3089999999999999,"5min":1.2529999999999999,"15min":1.8260000000000001},"min":{"1min":0.40799999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.226,"5min":9.2810000000000006,"15min":228.893},"last":0.57499999999999996}]},{"osd":4,"last update":"Mon Oct 9 09:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.323,"5min":1.276,"15min":1.415},"min":{"1min":0.51400000000000001,"5min":0.33100000000000002,"15min":0.27100000000000002},"max":{"1min":4.4660000000000002,"5min":7.9690000000000003,"15min":71.876000000000005},"last":0.97699999999999998},{"interface":"front","average":{"1min":1.8520000000000001,"5min":1.6539999999999999,"15min":1.3839999999999999},"min":{"1min":0.35499999999999998,"5min":0.29499999999999998,"15min":0.26700000000000002},"max":{"1min":5.0919999999999996,"5min":8.3989999999999991,"15min":9.4640000000000004},"last":0.86499999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.5860000000000001,"15min":1.373},"min":{"1min":0.47799999999999998,"5min":0.34100000000000003,"15min":0.29799999999999999},"max":{"1min":4.524,"5min":8.9350000000000005,"15min":8.9350000000000005},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.361,"5min":1.3280000000000001,"15min":1.2150000000000001},"min":{"1min":0.48899999999999999,"5min":0.35199999999999998,"15min":0.32200000000000001},"max":{"1min":4.4329999999999998,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":1.002}]},{"osd":6,"last update":"Mon Oct 9 09:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.4790000000000001,"15min":1.75},"min":{"1min":0.41699999999999998,"5min":0.32900000000000001,"15min":0.30199999999999999},"max":{"1min":4.4960000000000004,"5min":8.0719999999999992,"15min":71.795000000000002},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.77,"15min":1.7969999999999999},"min":{"1min":0.46800000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":5.1639999999999997,"5min":8.4580000000000002,"15min":72.322000000000003},"last":0.79300000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":1.704,"15min":1.3959999999999999},"min":{"1min":0.496,"5min":0.39400000000000002,"15min":0.374},"max":{"1min":5.1299999999999999,"5min":8.4290000000000003,"15min":8.4659999999999993},"last":0.748},{"interface":"front","average":{"1min":1.597,"5min":1.6020000000000001,"15min":1.292},"min":{"1min":0.51000000000000001,"5min":0.374,"15min":0.374},"max":{"1min":4.5499999999999998,"5min":8.8849999999999998,"15min":8.8849999999999998},"last":0.88800000000000001}]}]},{"osd":1,"up_from":1246,"seq":5351529250903,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":413332,"kb_used_data":7628,"kb_used_omap":204,"kb_used_meta":405491,"kb_avail":93331820,"statfs":{"total":95995035648,"available":95571783680,"internally_reserved":0,"allocated":7811072,"data_stored":6248604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208958,"internal_metadata":415223746},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.6030000000000002,"15min":2.0369999999999999},"min":{"1min":0.251,"5min":0.191,"15min":0.191},"max":{"1min":8.2850000000000001,"5min":58.033000000000001,"15min":58.033000000000001},"last":0.67500000000000004},{"interface":"front","average":{"1min":2.1549999999999998,"5min":2.5329999999999999,"15min":2.2749999999999999},"min":{"1min":0.21199999999999999,"5min":0.19,"15min":0.19},"max":{"1min":6.7080000000000002,"5min":65.921999999999997,"15min":65.921999999999997},"last":0.65100000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":2.6099999999999999,"15min":1.9039999999999999},"min":{"1min":0.33800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":10.647,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.2999999999999998,"5min":2.4199999999999999,"15min":1.8959999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.297000000000001,"5min":58.682000000000002,"15min":58.682000000000002},"last":0.499}]},{"osd":3,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.552,"5min":1.514,"15min":1.5029999999999999},"min":{"1min":0.217,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.1040000000000001,"5min":8.7140000000000004,"15min":8.7140000000000004},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.3899999999999999,"15min":1.3839999999999999},"min":{"1min":0.29099999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1509999999999998,"5min":7.5780000000000003,"15min":7.5780000000000003},"last":0.98199999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9830000000000001,"5min":1.8029999999999999,"15min":2.4209999999999998},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.242,"5min":8.8070000000000004,"15min":8.8070000000000004},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.2120000000000002,"5min":1.7450000000000001,"15min":2.2480000000000002},"min":{"1min":0.36599999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.0860000000000003,"5min":10.263,"15min":10.263},"last":1.101}]},{"osd":5,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":1.7010000000000001,"15min":1.8},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.1200000000000001,"5min":10.214,"15min":10.214},"last":1.0489999999999999},{"interface":"front","average":{"1min":2.0059999999999998,"5min":1.675,"15min":1.8240000000000001},"min":{"1min":0.32800000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.6849999999999996,"5min":8.5139999999999993,"15min":8.5139999999999993},"last":0.62}]},{"osd":6,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":2.1230000000000002,"15min":1.9530000000000001},"min":{"1min":0.35499999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.798,"5min":9.3309999999999995,"15min":9.3309999999999995},"last":0.754},{"interface":"front","average":{"1min":2.1970000000000001,"5min":2.129,"15min":2.8740000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.2939999999999996,"5min":9.8659999999999997,"15min":9.8659999999999997},"last":0.77600000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:54:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3439999999999999,"5min":1.8460000000000001,"15min":1.708},"min":{"1min":0.40300000000000002,"5min":0.374,"15min":0.374},"max":{"1min":5.7309999999999999,"5min":9.2929999999999993,"15min":9.2929999999999993},"last":0.70599999999999996},{"interface":"front","average":{"1min":2.125,"5min":1.946,"15min":2.25},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":5.2720000000000002,"5min":9.4529999999999994,"15min":9.4529999999999994},"last":0.81599999999999995}]}]},{"osd":0,"up_from":1136,"seq":4879082848496,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":761512,"kb_used_data":7584,"kb_used_omap":280,"kb_used_meta":753639,"kb_avail":92983640,"statfs":{"total":95995035648,"available":95215247360,"internally_reserved":0,"allocated":7766016,"data_stored":6236223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":771726746},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:54:24 2023","interfaces":[{"interface":"back","average":{"1min":3.3039999999999998,"5min":3.004,"15min":1.931},"min":{"1min":0.34399999999999997,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":9.0129999999999999,"5min":69.680000000000007,"15min":69.680000000000007},"last":0.438},{"interface":"front","average":{"1min":3.1680000000000001,"5min":2.9750000000000001,"15min":2.1699999999999999},"min":{"1min":0.30199999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.404999999999999,"5min":70.668999999999997,"15min":70.668999999999997},"last":0.36299999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":4.2770000000000001,"15min":2.6040000000000001},"min":{"1min":0.33900000000000002,"5min":0.22,"15min":0.22},"max":{"1min":9.0419999999999998,"5min":254.89599999999999,"15min":254.89599999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.4969999999999999,"5min":4.633,"15min":3.008},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":12.881,"5min":256.85399999999998,"15min":256.85399999999998},"last":0.41399999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.139,"15min":1.095},"min":{"1min":0.36699999999999999,"5min":0.247,"15min":0.247},"max":{"1min":8.8309999999999995,"5min":8.8309999999999995,"15min":10.475},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.0169999999999999,"15min":1.069},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.25800000000000001},"max":{"1min":5.5860000000000003,"5min":6.4160000000000004,"15min":6.6550000000000002},"last":0.57899999999999996}]},{"osd":4,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":1.504,"15min":1.462},"min":{"1min":0.40200000000000002,"5min":0.318,"15min":0.314},"max":{"1min":7.3129999999999997,"5min":7.3129999999999997,"15min":20.079000000000001},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.891,"5min":1.379,"15min":1.4079999999999999},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":8.9480000000000004,"5min":8.9480000000000004,"15min":12.044},"last":0.71399999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.4530000000000001,"15min":1.4750000000000001},"min":{"1min":0.45800000000000002,"5min":0.33000000000000002,"15min":0.28399999999999997},"max":{"1min":8.8800000000000008,"5min":8.8800000000000008,"15min":9.4280000000000008},"last":0.76500000000000001},{"interface":"front","average":{"1min":1.923,"5min":1.391,"15min":1.522},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.29699999999999999},"max":{"1min":8.7010000000000005,"5min":8.7010000000000005,"15min":9.6289999999999996},"last":0.78500000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.74,"15min":1.5369999999999999},"min":{"1min":0.40799999999999997,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7809999999999997,"5min":9.5950000000000006,"15min":10.413},"last":0.67100000000000004},{"interface":"front","average":{"1min":2.2069999999999999,"5min":1.72,"15min":1.702},"min":{"1min":0.503,"5min":0.439,"15min":0.35599999999999998},"max":{"1min":8.9809999999999999,"5min":8.9809999999999999,"15min":9.3260000000000005},"last":0.83699999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2389999999999999,"5min":1.595,"15min":1.379},"min":{"1min":0.45800000000000002,"5min":0.38500000000000001,"15min":0.38},"max":{"1min":6.7389999999999999,"5min":7.0149999999999997,"15min":7.0149999999999997},"last":0.877},{"interface":"front","average":{"1min":2.1259999999999999,"5min":1.492,"15min":1.4319999999999999},"min":{"1min":0.53200000000000003,"5min":0.435,"15min":0.39200000000000002},"max":{"1min":8.9160000000000004,"5min":8.9160000000000004,"15min":14.361000000000001},"last":0.90200000000000002}]}]},{"osd":3,"up_from":0,"seq":2847563317726,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986860,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":913064,"kb_used_data":5600,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":92832088,"statfs":{"total":95995035648,"available":95060058112,"internally_reserved":0,"allocated":5734400,"data_stored":5431722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:54:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":1.222,"15min":1.5920000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":5.7670000000000003,"5min":5.7670000000000003,"15min":64.372},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.214,"5min":1.3720000000000001,"15min":1.5389999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.27700000000000002},"max":{"1min":5.7889999999999997,"5min":5.7889999999999997,"15min":64.356999999999999},"last":0.73299999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:54:18 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":4.5330000000000004,"15min":2.5779999999999998},"min":{"1min":0.35299999999999998,"5min":0.32000000000000001,"15min":0.27700000000000002},"max":{"1min":2.7410000000000001,"5min":216.16900000000001,"15min":216.16900000000001},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.96499999999999997,"5min":4.8129999999999997,"15min":2.1899999999999999},"min":{"1min":0.47599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.6930000000000001,"5min":216.262,"15min":216.262},"last":0.872}]},{"osd":2,"last update":"Mon Oct 9 09:54:18 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":1.3839999999999999,"15min":1.5549999999999999},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.29199999999999998},"max":{"1min":15.853999999999999,"5min":15.853999999999999,"15min":33.743000000000002},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.665,"5min":1.262,"15min":1.5009999999999999},"min":{"1min":0.40200000000000002,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":15.881,"5min":15.881,"15min":33.805999999999997},"last":0.99399999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:55:02 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.004,"15min":1.006},"min":{"1min":0.443,"5min":0.36299999999999999,"15min":0.32200000000000001},"max":{"1min":1.8620000000000001,"5min":5.5129999999999999,"15min":8.1120000000000001},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.81299999999999994,"15min":1.1120000000000001},"min":{"1min":0.39700000000000002,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":1.25,"5min":5.3109999999999999,"15min":7.4180000000000001},"last":0.878}]},{"osd":5,"last update":"Mon Oct 9 09:55:02 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.90600000000000003,"15min":1.3919999999999999},"min":{"1min":0.435,"5min":0.373,"15min":0.26900000000000002},"max":{"1min":2.3820000000000001,"5min":5.4660000000000002,"15min":11.763999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.111,"15min":1.3180000000000001},"min":{"1min":0.42799999999999999,"5min":0.41399999999999998,"15min":0.28499999999999998},"max":{"1min":1.284,"5min":5.4790000000000001,"15min":12.446999999999999},"last":0.58799999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:55:02 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.97899999999999998,"15min":1.4350000000000001},"min":{"1min":0.46999999999999997,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":2.9550000000000001,"5min":5.2690000000000001,"15min":11.701000000000001},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.0089999999999999,"15min":1.5249999999999999},"min":{"1min":0.46100000000000002,"5min":0.40799999999999997,"15min":0.39200000000000002},"max":{"1min":2.9009999999999998,"5min":3.7240000000000002,"15min":11.972},"last":1.014}]},{"osd":7,"last update":"Mon Oct 9 09:54:18 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.96099999999999997,"15min":1.236},"min":{"1min":0.48699999999999999,"5min":0.45600000000000002,"15min":0.373},"max":{"1min":2.4180000000000001,"5min":5.2709999999999999,"15min":11.840999999999999},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.071,"15min":1.1200000000000001},"min":{"1min":0.52900000000000003,"5min":0.46700000000000003,"15min":0.34999999999999998},"max":{"1min":2.4199999999999999,"5min":5.1980000000000004,"15min":9.6210000000000004},"last":0.95199999999999996}]}]},{"osd":5,"up_from":37,"seq":158913790626,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920276,"kb_used_data":5580,"kb_used_omap":0,"kb_used_meta":914688,"kb_avail":92824876,"statfs":{"total":95995035648,"available":95052673024,"internally_reserved":0,"allocated":5713920,"data_stored":5427396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":936640512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":3.4820000000000002,"15min":4.218},"min":{"1min":0.28100000000000003,"5min":0.26600000000000001,"15min":0.23599999999999999},"max":{"1min":4.3719999999999999,"5min":121.977,"15min":274.71899999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":1.125,"5min":3.3610000000000002,"15min":4.0739999999999998},"min":{"1min":0.35299999999999998,"5min":0.33400000000000002,"15min":0.26800000000000002},"max":{"1min":4.4240000000000004,"5min":122.13200000000001,"15min":274.73500000000001},"last":0.79100000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:54:19 2023","interfaces":[{"interface":"back","average":{"1min":12.821,"5min":4.1050000000000004,"15min":2.6190000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":192.72200000000001,"5min":192.72200000000001,"15min":192.72200000000001},"last":1.105},{"interface":"front","average":{"1min":12.872,"5min":4.056,"15min":2.1269999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":192.69300000000001,"5min":192.69300000000001,"15min":192.69300000000001},"last":1.022}]},{"osd":2,"last update":"Mon Oct 9 09:54:30 2023","interfaces":[{"interface":"back","average":{"1min":7.2539999999999996,"5min":2.2679999999999998,"15min":1.929},"min":{"1min":0.374,"5min":0.316,"15min":0.312},"max":{"1min":122.02800000000001,"5min":122.02800000000001,"15min":122.02800000000001},"last":0.95699999999999996},{"interface":"front","average":{"1min":7.3550000000000004,"5min":2.54,"15min":2.1659999999999999},"min":{"1min":0.38300000000000001,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":122.05800000000001,"5min":122.05800000000001,"15min":122.05800000000001},"last":0.99199999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:54:39 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.79100000000000004,"15min":1.0700000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.33100000000000002},"max":{"1min":3.601,"5min":3.601,"15min":10.125999999999999},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.93000000000000005,"15min":1.089},"min":{"1min":0.41099999999999998,"5min":0.38,"15min":0.33200000000000002},"max":{"1min":1.0429999999999999,"5min":6.7750000000000004,"15min":16.201000000000001},"last":1.153}]},{"osd":4,"last update":"Mon Oct 9 09:54:14 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.90300000000000002,"15min":1.258},"min":{"1min":0.35799999999999998,"5min":0.35699999999999998,"15min":0.27000000000000002},"max":{"1min":3.6899999999999999,"5min":6.4470000000000001,"15min":9.4209999999999994},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":1.0409999999999999,"15min":1.4370000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.95199999999999996,"5min":7.601,"15min":16.053000000000001},"last":0.85699999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:54:39 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.099,"15min":1.802},"min":{"1min":0.438,"5min":0.308,"15min":0.308},"max":{"1min":5.4640000000000004,"5min":7.798,"15min":140.179},"last":0.97299999999999998},{"interface":"front","average":{"1min":1.095,"5min":1.2330000000000001,"15min":1.901},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.34399999999999997},"max":{"1min":5.673,"5min":7.8070000000000004,"15min":137.928},"last":1.1990000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:54:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.0309999999999999,"15min":1.302},"min":{"1min":0.44,"5min":0.42399999999999999,"15min":0.373},"max":{"1min":1.077,"5min":4.3040000000000003,"15min":16.119},"last":1.0429999999999999},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.82799999999999996,"15min":0.98099999999999998},"min":{"1min":0.45000000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.0600000000000001,"5min":3.3700000000000001,"15min":6.5460000000000003},"last":0.88400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7721,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:55:19.854 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T09:55:19.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:19.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:19.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:19.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:20.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:20.098 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:20.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:20.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:20.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[75425]: cluster 2023-10-09T09:55:18.278781+0000 mgr.y (mgr.34107) 1485 : cluster 0 pgmap v2366: 105 pgs: 11 creating+peering, 21 unknown, 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:20.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[75425]: cluster 2023-10-09T09:55:19.148112+0000 osd.6 (osd.6) 569 : cluster 3 failed to encode map e1292 with expected crc 2023-10-09T09:55:20.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[75425]: cluster 2023-10-09T09:55:19.150857+0000 osd.5 (osd.5) 883 : cluster 3 failed to encode map e1292 with expected crc 2023-10-09T09:55:20.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[75425]: cluster 2023-10-09T09:55:19.153476+0000 mon.a (mon.0) 2596 : cluster 0 osdmap e1292: 8 total, 8 up, 6 in 2023-10-09T09:55:20.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[75425]: audit 2023-10-09T09:55:19.495033+0000 mon.a (mon.0) 2597 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:20.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[75425]: audit 2023-10-09T09:55:19.496146+0000 mon.a (mon.0) 2598 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:55:20.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[75425]: audit 2023-10-09T09:55:19.502771+0000 mon.a (mon.0) 2599 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:20.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:20.233 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:20.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:20.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:20 smithi181 bash[59345]: cluster 2023-10-09T09:55:18.278781+0000 mgr.y (mgr.34107) 1485 : cluster 0 pgmap v2366: 105 pgs: 11 creating+peering, 21 unknown, 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:20 smithi181 bash[59345]: cluster 2023-10-09T09:55:19.148112+0000 osd.6 (osd.6) 569 : cluster 3 failed to encode map e1292 with expected crc 2023-10-09T09:55:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:20 smithi181 bash[59345]: cluster 2023-10-09T09:55:19.150857+0000 osd.5 (osd.5) 883 : cluster 3 failed to encode map e1292 with expected crc 2023-10-09T09:55:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:20 smithi181 bash[59345]: cluster 2023-10-09T09:55:19.153476+0000 mon.a (mon.0) 2596 : cluster 0 osdmap e1292: 8 total, 8 up, 6 in 2023-10-09T09:55:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:20 smithi181 bash[59345]: audit 2023-10-09T09:55:19.495033+0000 mon.a (mon.0) 2597 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:20 smithi181 bash[59345]: audit 2023-10-09T09:55:19.496146+0000 mon.a (mon.0) 2598 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:55:20.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:20 smithi181 bash[59345]: audit 2023-10-09T09:55:19.502771+0000 mon.a (mon.0) 2599 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:20.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:20.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:20.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[78107]: cluster 2023-10-09T09:55:18.278781+0000 mgr.y (mgr.34107) 1485 : cluster 0 pgmap v2366: 105 pgs: 11 creating+peering, 21 unknown, 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[78107]: cluster 2023-10-09T09:55:19.148112+0000 osd.6 (osd.6) 569 : cluster 3 failed to encode map e1292 with expected crc 2023-10-09T09:55:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[78107]: cluster 2023-10-09T09:55:19.150857+0000 osd.5 (osd.5) 883 : cluster 3 failed to encode map e1292 with expected crc 2023-10-09T09:55:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[78107]: cluster 2023-10-09T09:55:19.153476+0000 mon.a (mon.0) 2596 : cluster 0 osdmap e1292: 8 total, 8 up, 6 in 2023-10-09T09:55:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[78107]: audit 2023-10-09T09:55:19.495033+0000 mon.a (mon.0) 2597 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[78107]: audit 2023-10-09T09:55:19.496146+0000 mon.a (mon.0) 2598 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:55:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:20 smithi163 bash[78107]: audit 2023-10-09T09:55:19.502771+0000 mon.a (mon.0) 2599 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:20.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:20.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:20.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:20.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:20.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:20.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:20.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:20.740 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:20.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:20.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:20.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:20.906 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:20.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:20.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:21.010 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:21.038 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:21.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:21.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:21.142 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:21.169 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:21.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:21.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:21.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:21.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:21.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:21.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:21.408 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:21.437 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:21.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:21.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:21.540 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:21.569 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:21.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:21.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:21.673 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:21.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:21.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:21.810 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:21.839 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:21.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:21.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:21.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:55:21.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:21.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:21.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:21.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:22.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:22.095 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:22.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:22.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:22.210 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:22 smithi181 bash[59345]: cluster 2023-10-09T09:55:20.279441+0000 mgr.y (mgr.34107) 1486 : cluster 0 pgmap v2368: 105 pgs: 11 creating+peering, 16 unknown, 78 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 9.5 KiB/s rd, 6.8 KiB/s wr, 7 op/s 2023-10-09T09:55:22.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:22 smithi163 bash[75425]: cluster 2023-10-09T09:55:20.279441+0000 mgr.y (mgr.34107) 1486 : cluster 0 pgmap v2368: 105 pgs: 11 creating+peering, 16 unknown, 78 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 9.5 KiB/s rd, 6.8 KiB/s wr, 7 op/s 2023-10-09T09:55:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:22 smithi163 bash[78107]: cluster 2023-10-09T09:55:20.279441+0000 mgr.y (mgr.34107) 1486 : cluster 0 pgmap v2368: 105 pgs: 11 creating+peering, 16 unknown, 78 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 9.5 KiB/s rd, 6.8 KiB/s wr, 7 op/s 2023-10-09T09:55:22.228 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:22.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:22.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:22.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:22.338 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:55:22.362 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:22.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:22.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:22.466 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:22.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:22.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:22.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:22.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:22.628 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:22.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:22.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:22.731 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:22.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:22.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:22.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:22.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:22.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:22.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:22.996 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:23.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:23.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:23.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:23.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:23.152 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:23.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:23.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:23.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:23.287 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:23.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:23.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:23.346 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_image_status (4242 ms) 2023-10-09T09:55:23.346 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T09:55:23.391 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:23.402 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:55:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:55:23] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.33.4" 2023-10-09T09:55:23.419 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:23.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:23.486 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_image_map (140 ms) 2023-10-09T09:55:23.486 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T09:55:23.502 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_instances (15 ms) 2023-10-09T09:55:23.502 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T09:55:23.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:23.524 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_snapshot (22 ms) 2023-10-09T09:55:23.524 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T09:55:23.528 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_list (5 ms) 2023-10-09T09:55:23.528 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T09:55:23.532 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_add (4 ms) 2023-10-09T09:55:23.532 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T09:55:23.537 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.dir_add_already_existing (4 ms) 2023-10-09T09:55:23.537 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T09:55:23.547 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_rename (11 ms) 2023-10-09T09:55:23.547 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T09:55:23.554 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_remove (7 ms) 2023-10-09T09:55:23.554 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T09:55:23.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:23.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:23.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:23.563 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (9 ms) 2023-10-09T09:55:23.563 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T09:55:23.567 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-10-09T09:55:23.567 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T09:55:23.571 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_remove (5 ms) 2023-10-09T09:55:23.571 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T09:55:23.575 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-10-09T09:55:23.575 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T09:55:23.582 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_clean (6 ms) 2023-10-09T09:55:23.582 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T09:55:23.585 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.image_group_add (4 ms) 2023-10-09T09:55:23.586 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T09:55:23.592 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.image_group_remove (6 ms) 2023-10-09T09:55:23.592 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T09:55:23.595 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.image_group_get (4 ms) 2023-10-09T09:55:23.595 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T09:55:23.598 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (3 ms) 2023-10-09T09:55:23.598 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T09:55:23.601 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (3 ms) 2023-10-09T09:55:23.601 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T09:55:23.606 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (5 ms) 2023-10-09T09:55:23.606 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T09:55:23.610 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (4 ms) 2023-10-09T09:55:23.611 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T09:55:23.614 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set (4 ms) 2023-10-09T09:55:23.614 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T09:55:23.620 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_list (6 ms) 2023-10-09T09:55:23.620 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T09:55:23.649 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (29 ms) 2023-10-09T09:55:23.649 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T09:55:23.659 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:23.685 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:23.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:23.790 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:23.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:23.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:23.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:23.945 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:23.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:23.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:23.998 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (348 ms) 2023-10-09T09:55:23.998 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T09:55:24.006 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_remove (9 ms) 2023-10-09T09:55:24.006 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T09:55:24.010 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (4 ms) 2023-10-09T09:55:24.010 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T09:55:24.021 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.trash_methods (11 ms) 2023-10-09T09:55:24.021 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T09:55:24.033 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.op_features (12 ms) 2023-10-09T09:55:24.033 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T09:55:24.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:24.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:24 smithi181 bash[59345]: cluster 2023-10-09T09:55:22.280550+0000 mgr.y (mgr.34107) 1487 : cluster 0 pgmap v2369: 105 pgs: 105 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 1.08k op/s 2023-10-09T09:55:24.069 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.clone_parent (36 ms) 2023-10-09T09:55:24.069 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T09:55:24.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:24.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:24.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:24.091 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.clone_parent_ns (21 ms) 2023-10-09T09:55:24.091 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T09:55:24.112 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.clone_child (21 ms) 2023-10-09T09:55:24.112 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T09:55:24.125 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.namespace_methods (14 ms) 2023-10-09T09:55:24.126 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T09:55:24.143 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.migration (18 ms) 2023-10-09T09:55:24.144 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T09:55:24.160 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.migration_v1 (17 ms) 2023-10-09T09:55:24.161 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T09:55:24.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:24.181 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:55:24.182 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:55:24.213 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:24.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:24.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:24.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:24.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:24.354 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:24.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:24.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:24.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:24 smithi163 bash[75425]: cluster 2023-10-09T09:55:22.280550+0000 mgr.y (mgr.34107) 1487 : cluster 0 pgmap v2369: 105 pgs: 105 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 1.08k op/s 2023-10-09T09:55:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:24 smithi163 bash[78107]: cluster 2023-10-09T09:55:22.280550+0000 mgr.y (mgr.34107) 1487 : cluster 0 pgmap v2369: 105 pgs: 105 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 1.08k op/s 2023-10-09T09:55:24.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:24.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:24.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:24.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:24.613 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:24.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:24.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:24.718 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:24.749 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:24.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:24.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:24.853 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:24.881 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:24.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:24.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:24.985 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:25.011 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:25.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:25.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:25.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:25.139 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:25.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:25.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:25.242 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:25.271 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:25.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:25.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:25.374 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:25.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:25.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:25.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:25.505 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:25.549 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:25.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:25.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:25.653 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:25.684 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:25.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:25.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:25.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:25.819 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:25.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:25.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:25.923 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:25.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:25.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:25.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:25.961 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:55:25.961 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:55:25.961 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T09:55:25.961 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:55:26.031 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1871 ms) 2023-10-09T09:55:26.031 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T09:55:26.061 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:26.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:26.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:26.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:26.099 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.sparsify (67 ms) 2023-10-09T09:55:26.189 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:26.209 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:55:26.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:26.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:26.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[75425]: cluster 2023-10-09T09:55:24.281366+0000 mgr.y (mgr.34107) 1488 : cluster 0 pgmap v2370: 105 pgs: 105 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 1.7 MiB/s rd, 925 KiB/s wr, 923 op/s 2023-10-09T09:55:26.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[75425]: cluster 2023-10-09T09:55:25.014902+0000 osd.6 (osd.6) 570 : cluster 3 failed to encode map e1293 with expected crc 2023-10-09T09:55:26.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[75425]: cluster 2023-10-09T09:55:25.015061+0000 osd.5 (osd.5) 884 : cluster 3 failed to encode map e1293 with expected crc 2023-10-09T09:55:26.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[75425]: cluster 2023-10-09T09:55:25.017422+0000 mon.a (mon.0) 2600 : cluster 0 osdmap e1293: 8 total, 8 up, 6 in 2023-10-09T09:55:26.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[75425]: cluster 2023-10-09T09:55:25.022110+0000 osd.4 (osd.4) 676 : cluster 3 failed to encode map e1293 with expected crc 2023-10-09T09:55:26.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[75425]: audit 2023-10-09T09:55:25.423242+0000 mon.a (mon.0) 2601 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:26.225 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[75425]: audit 2023-10-09T09:55:25.424555+0000 mon.a (mon.0) 2602 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:26.225 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[78107]: cluster 2023-10-09T09:55:24.281366+0000 mgr.y (mgr.34107) 1488 : cluster 0 pgmap v2370: 105 pgs: 105 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 1.7 MiB/s rd, 925 KiB/s wr, 923 op/s 2023-10-09T09:55:26.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[78107]: cluster 2023-10-09T09:55:25.014902+0000 osd.6 (osd.6) 570 : cluster 3 failed to encode map e1293 with expected crc 2023-10-09T09:55:26.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[78107]: cluster 2023-10-09T09:55:25.015061+0000 osd.5 (osd.5) 884 : cluster 3 failed to encode map e1293 with expected crc 2023-10-09T09:55:26.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[78107]: cluster 2023-10-09T09:55:25.017422+0000 mon.a (mon.0) 2600 : cluster 0 osdmap e1293: 8 total, 8 up, 6 in 2023-10-09T09:55:26.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[78107]: cluster 2023-10-09T09:55:25.022110+0000 osd.4 (osd.4) 676 : cluster 3 failed to encode map e1293 with expected crc 2023-10-09T09:55:26.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[78107]: audit 2023-10-09T09:55:25.423242+0000 mon.a (mon.0) 2601 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:26.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:26 smithi163 bash[78107]: audit 2023-10-09T09:55:25.424555+0000 mon.a (mon.0) 2602 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:26.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:26 smithi181 bash[59345]: cluster 2023-10-09T09:55:24.281366+0000 mgr.y (mgr.34107) 1488 : cluster 0 pgmap v2370: 105 pgs: 105 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 1.7 MiB/s rd, 925 KiB/s wr, 923 op/s 2023-10-09T09:55:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:26 smithi181 bash[59345]: cluster 2023-10-09T09:55:25.014902+0000 osd.6 (osd.6) 570 : cluster 3 failed to encode map e1293 with expected crc 2023-10-09T09:55:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:26 smithi181 bash[59345]: cluster 2023-10-09T09:55:25.015061+0000 osd.5 (osd.5) 884 : cluster 3 failed to encode map e1293 with expected crc 2023-10-09T09:55:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:26 smithi181 bash[59345]: cluster 2023-10-09T09:55:25.017422+0000 mon.a (mon.0) 2600 : cluster 0 osdmap e1293: 8 total, 8 up, 6 in 2023-10-09T09:55:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:26 smithi181 bash[59345]: cluster 2023-10-09T09:55:25.022110+0000 osd.4 (osd.4) 676 : cluster 3 failed to encode map e1293 with expected crc 2023-10-09T09:55:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:26 smithi181 bash[59345]: audit 2023-10-09T09:55:25.423242+0000 mon.a (mon.0) 2601 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:26 smithi181 bash[59345]: audit 2023-10-09T09:55:25.424555+0000 mon.a (mon.0) 2602 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:26.318 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:26.346 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:26.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:26.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:26.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:26.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:26.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:26.583 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:26.613 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:26.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:26.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:26.717 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:26.746 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:26.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:26.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:26.846 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T09:55:26.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:26.873 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:26.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:26.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:26.976 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:27.009 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:27.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:27.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:27.044 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] 70 tests from TestClsRbd (7950 ms total) 2023-10-09T09:55:27.045 INFO:tasks.workunit.client.0.smithi163.stdout: 2023-10-09T09:55:27.045 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] Global test environment tear-down 2023-10-09T09:55:27.045 INFO:tasks.workunit.client.0.smithi163.stdout:[==========] 70 tests from 1 test suite ran. (10780 ms total) 2023-10-09T09:55:27.045 INFO:tasks.workunit.client.0.smithi163.stdout:[ PASSED ] 70 tests. 2023-10-09T09:55:27.047 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T09:55:27.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T09:55:27.057 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T09:55:27.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T09:55:27.113 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:27.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:27 smithi181 bash[59345]: cluster 2023-10-09T09:55:26.027881+0000 osd.5 (osd.5) 885 : cluster 3 failed to encode map e1294 with expected crc 2023-10-09T09:55:27.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:27 smithi181 bash[59345]: cluster 2023-10-09T09:55:26.030093+0000 mon.a (mon.0) 2603 : cluster 0 osdmap e1294: 8 total, 8 up, 6 in 2023-10-09T09:55:27.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:27 smithi181 bash[59345]: cluster 2023-10-09T09:55:26.032282+0000 osd.6 (osd.6) 571 : cluster 3 failed to encode map e1294 with expected crc 2023-10-09T09:55:27.147 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:27.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:27.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:27.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:27 smithi163 bash[78107]: cluster 2023-10-09T09:55:26.027881+0000 osd.5 (osd.5) 885 : cluster 3 failed to encode map e1294 with expected crc 2023-10-09T09:55:27.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:27 smithi163 bash[78107]: cluster 2023-10-09T09:55:26.030093+0000 mon.a (mon.0) 2603 : cluster 0 osdmap e1294: 8 total, 8 up, 6 in 2023-10-09T09:55:27.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:27 smithi163 bash[78107]: cluster 2023-10-09T09:55:26.032282+0000 osd.6 (osd.6) 571 : cluster 3 failed to encode map e1294 with expected crc 2023-10-09T09:55:27.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:27 smithi163 bash[75425]: cluster 2023-10-09T09:55:26.027881+0000 osd.5 (osd.5) 885 : cluster 3 failed to encode map e1294 with expected crc 2023-10-09T09:55:27.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:27 smithi163 bash[75425]: cluster 2023-10-09T09:55:26.030093+0000 mon.a (mon.0) 2603 : cluster 0 osdmap e1294: 8 total, 8 up, 6 in 2023-10-09T09:55:27.181 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:27 smithi163 bash[75425]: cluster 2023-10-09T09:55:26.032282+0000 osd.6 (osd.6) 571 : cluster 3 failed to encode map e1294 with expected crc 2023-10-09T09:55:27.251 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:27.274 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:27.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:27.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:27.378 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:27.394 DEBUG:teuthology.parallel:result is None 2023-10-09T09:55:27.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:55:27.409 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:55:27.410 DEBUG:teuthology.orchestra.run.smithi163:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:55:27.415 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:27.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:27.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:27.520 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:27.549 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:27.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:27.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:27.625 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:55:27.625 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T09:55:27.625 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T09:55:27.625 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T09:55:27.626 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T09:55:27.626 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T09:55:27.626 DEBUG:teuthology.orchestra.run.smithi163:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:55:27.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:27.634 INFO:teuthology.orchestra.run.smithi163.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T09:55:27.635 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:55:27.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:27.683 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:27.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:27.703 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T09:55:27.703 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T09:55:27.753 INFO:tasks.workunit:timeout=3h 2023-10-09T09:55:27.753 INFO:tasks.workunit:cleanup=True 2023-10-09T09:55:27.754 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T09:55:27.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:27.802 INFO:tasks.workunit.client.0.smithi163.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T09:55:27.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:27.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:27.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:27.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:27.955 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:27.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:27.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:28.059 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:28.088 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:28.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:28.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:28.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:28.203 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:28 smithi181 bash[59345]: cluster 2023-10-09T09:55:26.281944+0000 mgr.y (mgr.34107) 1489 : cluster 0 pgmap v2373: 105 pgs: 105 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 1.8 MiB/s rd, 949 KiB/s wr, 1.01k op/s 2023-10-09T09:55:28.204 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:28 smithi181 bash[59345]: cluster 2023-10-09T09:55:27.039483+0000 osd.7 (osd.7) 623 : cluster 3 failed to encode map e1295 with expected crc 2023-10-09T09:55:28.204 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:28 smithi181 bash[59345]: cluster 2023-10-09T09:55:27.039488+0000 osd.5 (osd.5) 886 : cluster 3 failed to encode map e1295 with expected crc 2023-10-09T09:55:28.204 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:28 smithi181 bash[59345]: cluster 2023-10-09T09:55:27.042426+0000 mon.a (mon.0) 2604 : cluster 0 osdmap e1295: 8 total, 8 up, 6 in 2023-10-09T09:55:28.221 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:28.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:28.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:28.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:28.360 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:28.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:28.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:28.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:28.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:28 smithi163 bash[75425]: cluster 2023-10-09T09:55:26.281944+0000 mgr.y (mgr.34107) 1489 : cluster 0 pgmap v2373: 105 pgs: 105 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 1.8 MiB/s rd, 949 KiB/s wr, 1.01k op/s 2023-10-09T09:55:28.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:28 smithi163 bash[75425]: cluster 2023-10-09T09:55:27.039483+0000 osd.7 (osd.7) 623 : cluster 3 failed to encode map e1295 with expected crc 2023-10-09T09:55:28.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:28 smithi163 bash[75425]: cluster 2023-10-09T09:55:27.039488+0000 osd.5 (osd.5) 886 : cluster 3 failed to encode map e1295 with expected crc 2023-10-09T09:55:28.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:28 smithi163 bash[75425]: cluster 2023-10-09T09:55:27.042426+0000 mon.a (mon.0) 2604 : cluster 0 osdmap e1295: 8 total, 8 up, 6 in 2023-10-09T09:55:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:28 smithi163 bash[78107]: cluster 2023-10-09T09:55:26.281944+0000 mgr.y (mgr.34107) 1489 : cluster 0 pgmap v2373: 105 pgs: 105 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 1.8 MiB/s rd, 949 KiB/s wr, 1.01k op/s 2023-10-09T09:55:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:28 smithi163 bash[78107]: cluster 2023-10-09T09:55:27.039483+0000 osd.7 (osd.7) 623 : cluster 3 failed to encode map e1295 with expected crc 2023-10-09T09:55:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:28 smithi163 bash[78107]: cluster 2023-10-09T09:55:27.039488+0000 osd.5 (osd.5) 886 : cluster 3 failed to encode map e1295 with expected crc 2023-10-09T09:55:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:28 smithi163 bash[78107]: cluster 2023-10-09T09:55:27.042426+0000 mon.a (mon.0) 2604 : cluster 0 osdmap e1295: 8 total, 8 up, 6 in 2023-10-09T09:55:28.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:28.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:28.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:28.602 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:28.627 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:28.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:28.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:28.731 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:28.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:28.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:28.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:28.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:28.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:28.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:29.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:29.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:29.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:29.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:29.156 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:29.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:29.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:29.259 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:29.291 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:29.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:29.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:29.395 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:29.425 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:29.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:29.529 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:29.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:29.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:29.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:29.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:29.696 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:29.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:29.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:29.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:29.825 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:29.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:29.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:29.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:29.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:29.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:30.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:30.094 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:30.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:30.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:30.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:30.223 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:30.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:30.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:30.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:30 smithi181 bash[59345]: cluster 2023-10-09T09:55:28.282667+0000 mgr.y (mgr.34107) 1490 : cluster 0 pgmap v2375: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:30.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:30.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:30 smithi163 bash[78107]: cluster 2023-10-09T09:55:28.282667+0000 mgr.y (mgr.34107) 1490 : cluster 0 pgmap v2375: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:30.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:30 smithi163 bash[75425]: cluster 2023-10-09T09:55:28.282667+0000 mgr.y (mgr.34107) 1490 : cluster 0 pgmap v2375: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:30.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:30.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:30.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:30.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:30.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:30.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:30.573 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:30.594 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:30.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:30.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:30.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:30.729 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:30.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:30.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:30.856 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:30.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:30.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:30.959 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:30.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:30.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:30.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:31.098 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:31.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:31.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:31.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:31.248 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:31.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:31.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:31.351 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:31.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:31.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:31.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:31.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:31.511 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:31.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:31.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:31.615 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:31.622 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:55:31.646 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:31.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:31.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:31.784 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:31.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:31.887 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:31.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:31.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:31.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:32.018 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:32.040 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:32.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:32.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:32.143 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:32.163 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:32.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:32.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:32.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:32.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:32 smithi163 bash[78107]: cluster 2023-10-09T09:55:30.283300+0000 mgr.y (mgr.34107) 1491 : cluster 0 pgmap v2376: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:32.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:32 smithi163 bash[78107]: audit 2023-10-09T09:55:31.236255+0000 mon.a (mon.0) 2605 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:32.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:32 smithi163 bash[75425]: cluster 2023-10-09T09:55:30.283300+0000 mgr.y (mgr.34107) 1491 : cluster 0 pgmap v2376: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:32.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:32 smithi163 bash[75425]: audit 2023-10-09T09:55:31.236255+0000 mon.a (mon.0) 2605 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:32.296 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:32.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:32.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:32.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:32.415 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:32 smithi181 bash[59345]: cluster 2023-10-09T09:55:30.283300+0000 mgr.y (mgr.34107) 1491 : cluster 0 pgmap v2376: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:32.415 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:32 smithi181 bash[59345]: audit 2023-10-09T09:55:31.236255+0000 mon.a (mon.0) 2605 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:32.430 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:32.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:32.534 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:32.562 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:32.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:32.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:32.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:32.692 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:32.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:32.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:32.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:32.816 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:32.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:32.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:32.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:32.943 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:32.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:32.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:33.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:33.199 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T09:55:33.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:33.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:33.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:33.203 INFO:tasks.workunit.client.0.smithi163.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T09:55:33.205 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T09:55:33.205 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T09:55:33.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:33.326 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:33.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:33.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:33.429 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:33.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:33.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:33.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:33.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:33.566 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:55:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:55:33] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-09T09:55:33.580 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:33.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:33.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:33.683 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:33.712 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:33.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:33.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:33.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:33.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:33.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:33.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:33.911 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T09:55:33.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:33.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:33.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:33.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:34.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:34.095 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:34.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:34.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:34.111 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T09:55:34.189 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T09:55:34.191 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T09:55:34.192 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T09:55:34.198 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:34.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:34 smithi163 bash[78107]: cluster 2023-10-09T09:55:32.283944+0000 mgr.y (mgr.34107) 1492 : cluster 0 pgmap v2377: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:34.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:34 smithi163 bash[75425]: cluster 2023-10-09T09:55:32.283944+0000 mgr.y (mgr.34107) 1492 : cluster 0 pgmap v2377: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:34.229 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:34.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:34.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:34.255 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T09:55:34.261 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T09:55:34.261 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T09:55:34.310 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T09:55:34.312 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T09:55:34.312 DEBUG:teuthology.orchestra.run.smithi163:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T09:55:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:34 smithi181 bash[59345]: cluster 2023-10-09T09:55:32.283944+0000 mgr.y (mgr.34107) 1492 : cluster 0 pgmap v2377: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:34.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:34.359 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:34.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:34.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:34.363 INFO:tasks.workunit.client.0.smithi163.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T09:55:34.363 INFO:tasks.workunit.client.0.smithi163.stderr:+ TMPDIR=/tmp/rbd_import_export_237542 2023-10-09T09:55:34.363 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm -rf /tmp/rbd_import_export_237542 2023-10-09T09:55:34.364 INFO:tasks.workunit.client.0.smithi163.stderr:+ mkdir /tmp/rbd_import_export_237542 2023-10-09T09:55:34.365 INFO:tasks.workunit.client.0.smithi163.stderr:+ trap rm -rf /tmp/rbd_import_export_237542 INT TERM EXIT 2023-10-09T09:55:34.365 INFO:tasks.workunit.client.0.smithi163.stderr:+ mkdir foo.237542 2023-10-09T09:55:34.366 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import foo.237542 foo.dir 2023-10-09T09:55:34.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:34.486 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:34.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:34.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:34.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:34.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:34.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:34.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:34.718 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:34.749 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:34.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:34.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:34.884 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:34.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:34.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:34.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:35.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:35.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:35.123 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:35.153 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:35.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:35.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:35.257 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:35.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:35.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:35.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:35.325 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T09:55:35.325 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T09:55:35.325 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T09:55:35.325 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T09:55:35.325 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T09:55:35.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:35.417 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:35.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:35.520 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:35.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:35.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:35.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:35.673 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:35.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:35.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:35.775 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:35.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:35.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:35.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:35.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:35.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:35.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:36.037 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:36.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:36.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:36.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:36.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T09:55:36.172 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:36.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:36 smithi163 bash[75425]: cluster 2023-10-09T09:55:34.284669+0000 mgr.y (mgr.34107) 1493 : cluster 0 pgmap v2378: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:36.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:36 smithi163 bash[78107]: cluster 2023-10-09T09:55:34.284669+0000 mgr.y (mgr.34107) 1493 : cluster 0 pgmap v2378: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:36.197 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:36.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:36.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:36.310 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:36 smithi181 bash[59345]: cluster 2023-10-09T09:55:34.284669+0000 mgr.y (mgr.34107) 1493 : cluster 0 pgmap v2378: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:36.326 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:36.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:36.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:36.455 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:36.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:36.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:36.492 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: cannot import a directory 2023-10-09T09:55:36.492 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 0% complete...failed. 2023-10-09T09:55:36.493 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: import failed: (21) Is a directory 2023-10-09T09:55:36.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:36.579 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:36.581 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T09:55:36.581 INFO:tasks.workunit.client.0.smithi163.stderr:+ rmdir foo.237542 2023-10-09T09:55:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:36.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:36.582 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_237542/img bs=1k count=1 seek=10 2023-10-09T09:55:36.583 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records in 2023-10-09T09:55:36.583 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records out 2023-10-09T09:55:36.583 INFO:tasks.workunit.client.0.smithi163.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000151419 s, 6.8 MB/s 2023-10-09T09:55:36.583 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_237542/img bs=1k count=10 seek=100 2023-10-09T09:55:36.583 INFO:tasks.workunit.client.0.smithi163.stderr:10+0 records in 2023-10-09T09:55:36.584 INFO:tasks.workunit.client.0.smithi163.stderr:10+0 records out 2023-10-09T09:55:36.584 INFO:tasks.workunit.client.0.smithi163.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000157049 s, 65.2 MB/s 2023-10-09T09:55:36.584 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_237542/img bs=1k count=100 seek=1000 2023-10-09T09:55:36.584 INFO:tasks.workunit.client.0.smithi163.stderr:70+1 records in 2023-10-09T09:55:36.584 INFO:tasks.workunit.client.0.smithi163.stderr:70+1 records out 2023-10-09T09:55:36.585 INFO:tasks.workunit.client.0.smithi163.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000256548 s, 281 MB/s 2023-10-09T09:55:36.585 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_237542/img bs=1k seek=10000 2023-10-09T09:55:36.585 INFO:tasks.workunit.client.0.smithi163.stderr:138+1 records in 2023-10-09T09:55:36.586 INFO:tasks.workunit.client.0.smithi163.stderr:138+1 records out 2023-10-09T09:55:36.586 INFO:tasks.workunit.client.0.smithi163.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.000435419 s, 326 MB/s 2023-10-09T09:55:36.586 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_237542/img bs=1k seek=100000 2023-10-09T09:55:36.632 INFO:tasks.workunit.client.0.smithi163.stderr:74+1 records in 2023-10-09T09:55:36.632 INFO:tasks.workunit.client.0.smithi163.stderr:74+1 records out 2023-10-09T09:55:36.632 INFO:tasks.workunit.client.0.smithi163.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.0455535 s, 1.7 MB/s 2023-10-09T09:55:36.632 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_237542/img bs=1k seek=1000000 2023-10-09T09:55:36.662 INFO:tasks.workunit.client.0.smithi163.stderr:194+1 records in 2023-10-09T09:55:36.663 INFO:tasks.workunit.client.0.smithi163.stderr:194+1 records out 2023-10-09T09:55:36.663 INFO:tasks.workunit.client.0.smithi163.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.0291828 s, 6.8 MB/s 2023-10-09T09:55:36.663 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm testimg 2023-10-09T09:55:36.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:36.710 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:36.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:36.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:36.803 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 0% complete...failed. 2023-10-09T09:55:36.803 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T09:55:36.811 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T09:55:36.811 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format /tmp/rbd_import_export_237542/img testimg 2023-10-09T09:55:36.814 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:36.839 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:36.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:36.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:36.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:36.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:36.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:36.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:37.041 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:55:37.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:37.205 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:55:37.205 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:55:37.206 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:55:37.206 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:55:37.206 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:55:37.206 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:55:37.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:37 smithi163 bash[75425]: audit 2023-10-09T09:55:37.000538+0000 mon.a (mon.0) 2606 : audit 0 from='client.? 172.21.15.163:0/880735108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:55:37.217 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:37 smithi163 bash[78107]: audit 2023-10-09T09:55:37.000538+0000 mon.a (mon.0) 2606 : audit 0 from='client.? 172.21.15.163:0/880735108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:55:37.222 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:55:37.224 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:55:37.224 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:55:37.224 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:55:37.224 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:55:37.225 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:55:37.225 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:55:37.225 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:55:37.225 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:55:37.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:37.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:37.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:37.347 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:37.374 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:37.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:37.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:37.424 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:55:37.424 INFO:teuthology.orchestra.run.smithi163.stdout:{"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2588,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":1296,"num_osds":8,"num_up_osds":8,"osd_up_since":1696844886,"num_in_osds":6,"osd_in_since":1696844470,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73}],"num_pgs":73,"num_pools":4,"num_objects":24,"data_bytes":595872,"bytes_used":4888530944,"bytes_avail":571081682944,"bytes_total":575970213888},"fsmap":{"epoch":6,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi163.vnodzj","status":"up:active","gid":14640}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"}},"servicemap":{"epoch":40,"modified":"2023-10-09T09:55:26.281769+0000","services":{}},"progress_events":{"c48e0150-28d3-400f-b3b6-6d71876589e4":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T09:55:37.478 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:37.491 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:37 smithi181 bash[59345]: audit 2023-10-09T09:55:37.000538+0000 mon.a (mon.0) 2606 : audit 0 from='client.? 172.21.15.163:0/880735108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:55:37.510 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:37.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:37.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:37.614 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:37.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:37.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:37.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:37.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:37.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:37.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:37.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:38.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:38.104 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T09:55:38.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:38.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:38.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:38.221 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:38.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:38.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:38.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:38.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:38.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:38 smithi181 bash[59345]: cluster 2023-10-09T09:55:36.285214+0000 mgr.y (mgr.34107) 1494 : cluster 0 pgmap v2379: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:38.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:38 smithi181 bash[59345]: cluster 2023-10-09T09:55:37.083731+0000 osd.5 (osd.5) 887 : cluster 3 failed to encode map e1296 with expected crc 2023-10-09T09:55:38.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:38 smithi181 bash[59345]: cluster 2023-10-09T09:55:37.087159+0000 mon.a (mon.0) 2607 : cluster 0 osdmap e1296: 8 total, 8 up, 6 in 2023-10-09T09:55:38.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:38 smithi181 bash[59345]: cluster 2023-10-09T09:55:37.092198+0000 osd.6 (osd.6) 572 : cluster 3 failed to encode map e1296 with expected crc 2023-10-09T09:55:38.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:38 smithi181 bash[59345]: audit 2023-10-09T09:55:37.355078+0000 mon.a (mon.0) 2608 : audit 1 from='client.? 172.21.15.163:0/4067515742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:55:38.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:38 smithi181 bash[59345]: audit 2023-10-09T09:55:37.423096+0000 mon.a (mon.0) 2609 : audit 0 from='client.? 172.21.15.163:0/772275581' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T09:55:38.373 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:38.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:38.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:38.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[78107]: cluster 2023-10-09T09:55:36.285214+0000 mgr.y (mgr.34107) 1494 : cluster 0 pgmap v2379: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:38.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[78107]: cluster 2023-10-09T09:55:37.083731+0000 osd.5 (osd.5) 887 : cluster 3 failed to encode map e1296 with expected crc 2023-10-09T09:55:38.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[78107]: cluster 2023-10-09T09:55:37.087159+0000 mon.a (mon.0) 2607 : cluster 0 osdmap e1296: 8 total, 8 up, 6 in 2023-10-09T09:55:38.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[78107]: cluster 2023-10-09T09:55:37.092198+0000 osd.6 (osd.6) 572 : cluster 3 failed to encode map e1296 with expected crc 2023-10-09T09:55:38.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[78107]: audit 2023-10-09T09:55:37.355078+0000 mon.a (mon.0) 2608 : audit 1 from='client.? 172.21.15.163:0/4067515742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:55:38.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[78107]: audit 2023-10-09T09:55:37.423096+0000 mon.a (mon.0) 2609 : audit 0 from='client.? 172.21.15.163:0/772275581' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T09:55:38.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[75425]: cluster 2023-10-09T09:55:36.285214+0000 mgr.y (mgr.34107) 1494 : cluster 0 pgmap v2379: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:38.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[75425]: cluster 2023-10-09T09:55:37.083731+0000 osd.5 (osd.5) 887 : cluster 3 failed to encode map e1296 with expected crc 2023-10-09T09:55:38.421 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[75425]: cluster 2023-10-09T09:55:37.087159+0000 mon.a (mon.0) 2607 : cluster 0 osdmap e1296: 8 total, 8 up, 6 in 2023-10-09T09:55:38.421 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[75425]: cluster 2023-10-09T09:55:37.092198+0000 osd.6 (osd.6) 572 : cluster 3 failed to encode map e1296 with expected crc 2023-10-09T09:55:38.421 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[75425]: audit 2023-10-09T09:55:37.355078+0000 mon.a (mon.0) 2608 : audit 1 from='client.? 172.21.15.163:0/4067515742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:55:38.421 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:38 smithi163 bash[75425]: audit 2023-10-09T09:55:37.423096+0000 mon.a (mon.0) 2609 : audit 0 from='client.? 172.21.15.163:0/772275581' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T09:55:38.478 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:38.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:38.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:38.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:38.783 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:38.995 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T09:55:39.178 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:55:39.178 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg /tmp/rbd_import_export_237542/img2 2023-10-09T09:55:39.191 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:39 smithi163 bash[78107]: cluster 2023-10-09T09:55:38.083978+0000 mon.a (mon.0) 2610 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:55:39.249 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:39 smithi163 bash[78107]: cluster 2023-10-09T09:55:38.095029+0000 osd.5 (osd.5) 888 : cluster 3 failed to encode map e1297 with expected crc 2023-10-09T09:55:39.249 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:39 smithi163 bash[78107]: cluster 2023-10-09T09:55:38.095294+0000 osd.6 (osd.6) 573 : cluster 3 failed to encode map e1297 with expected crc 2023-10-09T09:55:39.249 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:39 smithi163 bash[78107]: audit 2023-10-09T09:55:38.097294+0000 mon.a (mon.0) 2611 : audit 1 from='client.? 172.21.15.163:0/4067515742' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:55:39.249 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:39 smithi163 bash[78107]: cluster 2023-10-09T09:55:38.097381+0000 mon.a (mon.0) 2612 : cluster 0 osdmap e1297: 8 total, 8 up, 6 in 2023-10-09T09:55:39.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:39 smithi163 bash[75425]: cluster 2023-10-09T09:55:38.083978+0000 mon.a (mon.0) 2610 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:55:39.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:39 smithi163 bash[75425]: cluster 2023-10-09T09:55:38.095029+0000 osd.5 (osd.5) 888 : cluster 3 failed to encode map e1297 with expected crc 2023-10-09T09:55:39.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:39 smithi163 bash[75425]: cluster 2023-10-09T09:55:38.095294+0000 osd.6 (osd.6) 573 : cluster 3 failed to encode map e1297 with expected crc 2023-10-09T09:55:39.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:39 smithi163 bash[75425]: audit 2023-10-09T09:55:38.097294+0000 mon.a (mon.0) 2611 : audit 1 from='client.? 172.21.15.163:0/4067515742' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:55:39.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:39 smithi163 bash[75425]: cluster 2023-10-09T09:55:38.097381+0000 mon.a (mon.0) 2612 : cluster 0 osdmap e1297: 8 total, 8 up, 6 in 2023-10-09T09:55:39.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:39.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:39.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:39.351 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:39.364 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:39 smithi181 bash[59345]: cluster 2023-10-09T09:55:38.083978+0000 mon.a (mon.0) 2610 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:55:39.364 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:39 smithi181 bash[59345]: cluster 2023-10-09T09:55:38.095029+0000 osd.5 (osd.5) 888 : cluster 3 failed to encode map e1297 with expected crc 2023-10-09T09:55:39.364 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:39 smithi181 bash[59345]: cluster 2023-10-09T09:55:38.095294+0000 osd.6 (osd.6) 573 : cluster 3 failed to encode map e1297 with expected crc 2023-10-09T09:55:39.364 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:39 smithi181 bash[59345]: audit 2023-10-09T09:55:38.097294+0000 mon.a (mon.0) 2611 : audit 1 from='client.? 172.21.15.163:0/4067515742' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:55:39.364 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:39 smithi181 bash[59345]: cluster 2023-10-09T09:55:38.097381+0000 mon.a (mon.0) 2612 : cluster 0 osdmap e1297: 8 total, 8 up, 6 in 2023-10-09T09:55:39.383 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:39.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:39.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:39.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:39.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:39.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:39.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:39.557 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T09:55:39.562 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg - 2023-10-09T09:55:39.611 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:39.642 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:39.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:39.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:39.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:39.773 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:39.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:39.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:39.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:39.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:39.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:40.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:40.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:40.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:40.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:40.097 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:40 smithi163 bash[75425]: cluster 2023-10-09T09:55:38.285589+0000 mgr.y (mgr.34107) 1495 : cluster 0 pgmap v2382: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:40.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:40.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:40.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:40.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:40.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:40.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:40.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:40.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:40.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:40 smithi181 bash[59345]: cluster 2023-10-09T09:55:38.285589+0000 mgr.y (mgr.34107) 1495 : cluster 0 pgmap v2382: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:40.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:40 smithi163 bash[78107]: cluster 2023-10-09T09:55:38.285589+0000 mgr.y (mgr.34107) 1495 : cluster 0 pgmap v2382: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:55:40.380 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T09:55:40.380 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:55:40.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:40.434 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:40.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:40.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:40.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:40.541 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T09:55:40.548 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm testimg 2023-10-09T09:55:40.569 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:40.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:40.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:40.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:40.682 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T09:55:40.688 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp /tmp/rbd_import_export_237542/img /tmp/rbd_import_export_237542/img2 2023-10-09T09:55:40.699 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:40.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:55:40.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:40.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:40.805 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:40.837 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:40.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:40.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:40.922 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:55:40.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:40.969 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:40.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:41.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:41.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:41.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:41.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:41.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:41.236 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:41.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:41.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:41.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:41.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:41.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:41.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:41.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:41.492 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp /tmp/rbd_import_export_237542/img /tmp/rbd_import_export_237542/img3 2023-10-09T09:55:41.500 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:41.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:41.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:41.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:41.622 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:41 smithi181 bash[59345]: cluster 2023-10-09T09:55:40.286133+0000 mgr.y (mgr.34107) 1496 : cluster 0 pgmap v2383: 73 pgs: 73 active+clean; 2.5 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 17 KiB/s wr, 0 op/s 2023-10-09T09:55:41.622 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:41 smithi181 bash[59345]: audit 2023-10-09T09:55:40.424842+0000 mon.a (mon.0) 2613 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:41.622 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:41 smithi181 bash[59345]: audit 2023-10-09T09:55:40.425947+0000 mon.a (mon.0) 2614 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:41.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:41.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:41.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:41 smithi163 bash[78107]: cluster 2023-10-09T09:55:40.286133+0000 mgr.y (mgr.34107) 1496 : cluster 0 pgmap v2383: 73 pgs: 73 active+clean; 2.5 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 17 KiB/s wr, 0 op/s 2023-10-09T09:55:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:41 smithi163 bash[78107]: audit 2023-10-09T09:55:40.424842+0000 mon.a (mon.0) 2613 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:41 smithi163 bash[78107]: audit 2023-10-09T09:55:40.425947+0000 mon.a (mon.0) 2614 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:41 smithi163 bash[75425]: cluster 2023-10-09T09:55:40.286133+0000 mgr.y (mgr.34107) 1496 : cluster 0 pgmap v2383: 73 pgs: 73 active+clean; 2.5 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 17 KiB/s wr, 0 op/s 2023-10-09T09:55:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:41 smithi163 bash[75425]: audit 2023-10-09T09:55:40.424842+0000 mon.a (mon.0) 2613 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:55:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:41 smithi163 bash[75425]: audit 2023-10-09T09:55:40.425947+0000 mon.a (mon.0) 2614 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:41.739 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:41.774 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:41.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:41.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:41.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:41.919 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:41.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:41.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:42.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:42.054 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/img2 /tmp/rbd_import_export_237542/img3 2023-10-09T09:55:42.055 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:42.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:42.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:42.158 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:42.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:42.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:42.295 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format - testimg 2023-10-09T09:55:42.295 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:42.326 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:42.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:42.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:42.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:55:42.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:42.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:42.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:42.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:42.614 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:42.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:42.722 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:42.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:42.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:42.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:42.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:42.875 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T09:55:42.876 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:55:42.880 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg /tmp/rbd_import_export_237542/img2 2023-10-09T09:55:42.888 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:42.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:42.991 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:43.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:43.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:43.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:43.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:43.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:43.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:43.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:43.255 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T09:55:43.259 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:43.262 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg - 2023-10-09T09:55:43.289 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:43.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:43.393 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:43.403 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:55:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:55:43] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-10-09T09:55:43.425 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:43.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:43.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:43.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:43.564 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:43.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:43.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:43.667 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:43.679 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:43 smithi163 bash[78107]: cluster 2023-10-09T09:55:42.287145+0000 mgr.y (mgr.34107) 1497 : cluster 0 pgmap v2384: 73 pgs: 73 active+clean; 2.5 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 116 KiB/s rd, 65 KiB/s wr, 37 op/s 2023-10-09T09:55:43.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:43 smithi163 bash[75425]: cluster 2023-10-09T09:55:42.287145+0000 mgr.y (mgr.34107) 1497 : cluster 0 pgmap v2384: 73 pgs: 73 active+clean; 2.5 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 116 KiB/s rd, 65 KiB/s wr, 37 op/s 2023-10-09T09:55:43.695 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:43.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:43.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:43.799 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:43.810 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:43 smithi181 bash[59345]: cluster 2023-10-09T09:55:42.287145+0000 mgr.y (mgr.34107) 1497 : cluster 0 pgmap v2384: 73 pgs: 73 active+clean; 2.5 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 116 KiB/s rd, 65 KiB/s wr, 37 op/s 2023-10-09T09:55:43.827 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:43.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:43.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:43.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:43.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:43.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:43.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:44.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:44.099 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:44.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:44.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:44.202 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:44.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:44.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:44.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:44.240 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T09:55:44.245 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm testimg 2023-10-09T09:55:44.326 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T09:55:44.327 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:55:44.330 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:44.356 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:44.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:44.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:44.382 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T09:55:44.387 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp /tmp/rbd_import_export_237542/img /tmp/rbd_import_export_237542/img2 2023-10-09T09:55:44.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:44.486 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:44.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:44.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:44.590 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:44.616 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:44.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:44.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:44.720 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:44.747 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:44.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:44.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:44.850 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:44.879 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:44.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:44.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:44.983 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:45.015 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:45.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:45.118 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:45.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:55:45.147 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:45.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:45.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:45.152 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp /tmp/rbd_import_export_237542/img /tmp/rbd_import_export_237542/img3 2023-10-09T09:55:45.213 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:55:45.214 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:55:45.250 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:45.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:45.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:45.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:45.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:45.418 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:45.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:45.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:45.522 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:45.550 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:45.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:45.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:45.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:45.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:45 smithi163 bash[75425]: cluster 2023-10-09T09:55:44.287942+0000 mgr.y (mgr.34107) 1498 : cluster 0 pgmap v2385: 73 pgs: 73 active+clean; 1.3 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 175 KiB/s rd, 89 KiB/s wr, 38 op/s 2023-10-09T09:55:45.666 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:45 smithi163 bash[78107]: cluster 2023-10-09T09:55:44.287942+0000 mgr.y (mgr.34107) 1498 : cluster 0 pgmap v2385: 73 pgs: 73 active+clean; 1.3 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 175 KiB/s rd, 89 KiB/s wr, 38 op/s 2023-10-09T09:55:45.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:45.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:45.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:45.689 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/img /tmp/rbd_import_export_237542/img2 /tmp/rbd_import_export_237542/img3 2023-10-09T09:55:45.782 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:45.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:45.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:45.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:45 smithi181 bash[59345]: cluster 2023-10-09T09:55:44.287942+0000 mgr.y (mgr.34107) 1498 : cluster 0 pgmap v2385: 73 pgs: 73 active+clean; 1.3 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 175 KiB/s rd, 89 KiB/s wr, 38 op/s 2023-10-09T09:55:45.913 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:45.947 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:45.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:45.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:46.052 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:46.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:46.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:46.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:46.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:46.218 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:46.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:46.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:46.225 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:55:46.322 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:46.352 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:46.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:46.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:46.456 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:46.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:46.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:46.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:46.593 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:46.625 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:46.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:46.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:46.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:46.853 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:46.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:46.956 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:46.989 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:46.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:46.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:47.092 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:47.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:47.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:47.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:47.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:47 smithi163 bash[75425]: cluster 2023-10-09T09:55:46.010096+0000 osd.5 (osd.5) 889 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:47.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:47 smithi163 bash[75425]: cluster 2023-10-09T09:55:46.010407+0000 osd.6 (osd.6) 574 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:47.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:47 smithi163 bash[75425]: cluster 2023-10-09T09:55:46.012979+0000 mon.a (mon.0) 2615 : cluster 0 osdmap e1298: 8 total, 8 up, 6 in 2023-10-09T09:55:47.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:47 smithi163 bash[75425]: cluster 2023-10-09T09:55:46.130416+0000 osd.4 (osd.4) 677 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:47.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:47 smithi163 bash[78107]: cluster 2023-10-09T09:55:46.010096+0000 osd.5 (osd.5) 889 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:47.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:47 smithi163 bash[78107]: cluster 2023-10-09T09:55:46.010407+0000 osd.6 (osd.6) 574 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:47.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:47 smithi163 bash[78107]: cluster 2023-10-09T09:55:46.012979+0000 mon.a (mon.0) 2615 : cluster 0 osdmap e1298: 8 total, 8 up, 6 in 2023-10-09T09:55:47.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:47 smithi163 bash[78107]: cluster 2023-10-09T09:55:46.130416+0000 osd.4 (osd.4) 677 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:47.230 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:47.266 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:47.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:47 smithi181 bash[59345]: cluster 2023-10-09T09:55:46.010096+0000 osd.5 (osd.5) 889 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:47.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:47 smithi181 bash[59345]: cluster 2023-10-09T09:55:46.010407+0000 osd.6 (osd.6) 574 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:47.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:47 smithi181 bash[59345]: cluster 2023-10-09T09:55:46.012979+0000 mon.a (mon.0) 2615 : cluster 0 osdmap e1298: 8 total, 8 up, 6 in 2023-10-09T09:55:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:47 smithi181 bash[59345]: cluster 2023-10-09T09:55:46.130416+0000 osd.4 (osd.4) 677 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:47.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:47.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:47.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:47.398 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:47.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:47.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:47.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:47.534 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:47.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:47.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:47.639 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:47.671 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:47.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:47.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:47.775 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:47.810 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:47.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:47.913 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:47.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:47.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:47.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:48.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:48.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:48 smithi181 bash[59345]: cluster 2023-10-09T09:55:46.288719+0000 mgr.y (mgr.34107) 1499 : cluster 0 pgmap v2387: 73 pgs: 73 active+clean; 1.3 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 205 KiB/s rd, 89 KiB/s wr, 79 op/s 2023-10-09T09:55:48.060 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:48 smithi181 bash[59345]: cluster 2023-10-09T09:55:47.012615+0000 osd.7 (osd.7) 624 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:48.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:48.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:48.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:48.186 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:48.215 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:48.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:48.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:48.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:48 smithi163 bash[78107]: cluster 2023-10-09T09:55:46.288719+0000 mgr.y (mgr.34107) 1499 : cluster 0 pgmap v2387: 73 pgs: 73 active+clean; 1.3 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 205 KiB/s rd, 89 KiB/s wr, 79 op/s 2023-10-09T09:55:48.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:48 smithi163 bash[78107]: cluster 2023-10-09T09:55:47.012615+0000 osd.7 (osd.7) 624 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:48.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:48 smithi163 bash[75425]: cluster 2023-10-09T09:55:46.288719+0000 mgr.y (mgr.34107) 1499 : cluster 0 pgmap v2387: 73 pgs: 73 active+clean; 1.3 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 205 KiB/s rd, 89 KiB/s wr, 79 op/s 2023-10-09T09:55:48.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:48 smithi163 bash[75425]: cluster 2023-10-09T09:55:47.012615+0000 osd.7 (osd.7) 624 : cluster 3 failed to encode map e1298 with expected crc 2023-10-09T09:55:48.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:48.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:48.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:48.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:48.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:48.482 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:48.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:48.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:48.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:48.618 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:48.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:48.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:48.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:48.754 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:48.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:48.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:48.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:48.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:48.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:48.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:48.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:49.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:49.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:49.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:49.158 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:49.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:49.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:49.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:49.271 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:55:49.289 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:49.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:49.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:49.392 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:49.422 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:49.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:49.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:49.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:49.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:49.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:49.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:49.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:49.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:49.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:49.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:49.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:49.827 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:49.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:49.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:49.931 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:49.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:49.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:49.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:50.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:50.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:50.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:50.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:50.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:50 smithi163 bash[75425]: cluster 2023-10-09T09:55:48.289446+0000 mgr.y (mgr.34107) 1500 : cluster 0 pgmap v2388: 73 pgs: 73 active+clean; 1.3 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 237 KiB/s rd, 106 KiB/s wr, 80 op/s 2023-10-09T09:55:50.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:50 smithi163 bash[78107]: cluster 2023-10-09T09:55:48.289446+0000 mgr.y (mgr.34107) 1500 : cluster 0 pgmap v2388: 73 pgs: 73 active+clean; 1.3 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 237 KiB/s rd, 106 KiB/s wr, 80 op/s 2023-10-09T09:55:50.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:50.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:50.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:50.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:50 smithi181 bash[59345]: cluster 2023-10-09T09:55:48.289446+0000 mgr.y (mgr.34107) 1500 : cluster 0 pgmap v2388: 73 pgs: 73 active+clean; 1.3 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 237 KiB/s rd, 106 KiB/s wr, 80 op/s 2023-10-09T09:55:50.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:50.366 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:50.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:50.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:50.469 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:50.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:50.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:50.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:50.608 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:50.630 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:50.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:50.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:50.733 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:50.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:50.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:50.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:50.858 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:50.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:50.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:50.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:50.998 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:51.025 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:51.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:51.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:51.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:51.165 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:51.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:51.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:51.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:51.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:51.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:51.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:51.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:51.440 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:51.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:51.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:51.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:51.580 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:51.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:51.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:51.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:51.714 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:51.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:51.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:51.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:51.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:51.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:51.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:51.895 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd help export 2023-10-09T09:55:51.895 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep -q export-format 2023-10-09T09:55:51.913 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_237542/img bs=1k count=10 seek=100 2023-10-09T09:55:51.913 INFO:tasks.workunit.client.0.smithi163.stderr:10+0 records in 2023-10-09T09:55:51.913 INFO:tasks.workunit.client.0.smithi163.stderr:10+0 records out 2023-10-09T09:55:51.914 INFO:tasks.workunit.client.0.smithi163.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000139184 s, 73.6 MB/s 2023-10-09T09:55:51.914 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format /tmp/rbd_import_export_237542/img testimg 2023-10-09T09:55:51.952 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:51.978 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T09:55:51.979 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:55:51.982 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap create testimg@snap 2023-10-09T09:55:51.987 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:51.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:51.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:52.047 INFO:tasks.workunit.client.0.smithi163.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T09:55:52.054 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T09:55:52.091 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:52.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:52 smithi181 bash[59345]: cluster 2023-10-09T09:55:50.289926+0000 mgr.y (mgr.34107) 1501 : cluster 0 pgmap v2389: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 256 KiB/s rd, 92 KiB/s wr, 80 op/s 2023-10-09T09:55:52.104 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd image-meta list testimg 2023-10-09T09:55:52.122 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:52.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:52.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:52.149 INFO:tasks.workunit.client.0.smithi163.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-10-09T09:55:52.150 INFO:tasks.workunit.client.0.smithi163.stderr: 2023-10-09T09:55:52.150 INFO:tasks.workunit.client.0.smithi163.stderr:Key Value 2023-10-09T09:55:52.150 INFO:tasks.workunit.client.0.smithi163.stderr:key1 value1 2023-10-09T09:55:52.150 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_237542/img_v2 2023-10-09T09:55:52.193 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T09:55:52.196 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_237542/img_v2 testimg_import 2023-10-09T09:55:52.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:52.259 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:52.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:52.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:52.363 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:52.395 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:52.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:52.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:52.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:52 smithi163 bash[75425]: cluster 2023-10-09T09:55:50.289926+0000 mgr.y (mgr.34107) 1501 : cluster 0 pgmap v2389: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 256 KiB/s rd, 92 KiB/s wr, 80 op/s 2023-10-09T09:55:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:52 smithi163 bash[78107]: cluster 2023-10-09T09:55:50.289926+0000 mgr.y (mgr.34107) 1501 : cluster 0 pgmap v2389: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 256 KiB/s rd, 92 KiB/s wr, 80 op/s 2023-10-09T09:55:52.499 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:52.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:52.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:52.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:52.634 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:52.667 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:52.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:52.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:52.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:52.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:52.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:52.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:52.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:52.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:52.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:52.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:53.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:53.066 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:53.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:53.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:53.070 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T09:55:53.075 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import 2023-10-09T09:55:53.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:53.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[78107]: cluster 2023-10-09T09:55:52.030876+0000 osd.5 (osd.5) 890 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[78107]: cluster 2023-10-09T09:55:52.032887+0000 mon.a (mon.0) 2616 : cluster 0 osdmap e1299: 8 total, 8 up, 6 in 2023-10-09T09:55:53.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[78107]: cluster 2023-10-09T09:55:52.044113+0000 osd.6 (osd.6) 575 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[78107]: cluster 2023-10-09T09:55:52.078253+0000 osd.4 (osd.4) 678 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[78107]: cluster 2023-10-09T09:55:52.312934+0000 osd.7 (osd.7) 625 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[75425]: cluster 2023-10-09T09:55:52.030876+0000 osd.5 (osd.5) 890 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[75425]: cluster 2023-10-09T09:55:52.032887+0000 mon.a (mon.0) 2616 : cluster 0 osdmap e1299: 8 total, 8 up, 6 in 2023-10-09T09:55:53.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[75425]: cluster 2023-10-09T09:55:52.044113+0000 osd.6 (osd.6) 575 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[75425]: cluster 2023-10-09T09:55:52.078253+0000 osd.4 (osd.4) 678 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[75425]: cluster 2023-10-09T09:55:52.312934+0000 osd.7 (osd.7) 625 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.195 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:53.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:53.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:53.206 INFO:tasks.workunit.client.0.smithi163.stdout:rbd image 'testimg_import': 2023-10-09T09:55:53.206 INFO:tasks.workunit.client.0.smithi163.stdout: size 110 KiB in 1 objects 2023-10-09T09:55:53.206 INFO:tasks.workunit.client.0.smithi163.stdout: order 22 (4 MiB objects) 2023-10-09T09:55:53.206 INFO:tasks.workunit.client.0.smithi163.stdout: snapshot_count: 1 2023-10-09T09:55:53.206 INFO:tasks.workunit.client.0.smithi163.stdout: id: b3e42edc78f6 2023-10-09T09:55:53.206 INFO:tasks.workunit.client.0.smithi163.stdout: block_name_prefix: rbd_data.b3e42edc78f6 2023-10-09T09:55:53.207 INFO:tasks.workunit.client.0.smithi163.stdout: format: 2 2023-10-09T09:55:53.207 INFO:tasks.workunit.client.0.smithi163.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T09:55:53.207 INFO:tasks.workunit.client.0.smithi163.stdout: op_features: 2023-10-09T09:55:53.207 INFO:tasks.workunit.client.0.smithi163.stdout: flags: 2023-10-09T09:55:53.207 INFO:tasks.workunit.client.0.smithi163.stdout: create_timestamp: Mon Oct 9 09:55:52 2023 2023-10-09T09:55:53.207 INFO:tasks.workunit.client.0.smithi163.stdout: access_timestamp: Mon Oct 9 09:55:52 2023 2023-10-09T09:55:53.207 INFO:tasks.workunit.client.0.smithi163.stdout: modify_timestamp: Mon Oct 9 09:55:52 2023 2023-10-09T09:55:53.210 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import@snap 2023-10-09T09:55:53.254 INFO:tasks.workunit.client.0.smithi163.stdout:rbd image 'testimg_import': 2023-10-09T09:55:53.255 INFO:tasks.workunit.client.0.smithi163.stdout: size 110 KiB in 1 objects 2023-10-09T09:55:53.255 INFO:tasks.workunit.client.0.smithi163.stdout: order 22 (4 MiB objects) 2023-10-09T09:55:53.255 INFO:tasks.workunit.client.0.smithi163.stdout: snapshot_count: 1 2023-10-09T09:55:53.255 INFO:tasks.workunit.client.0.smithi163.stdout: id: b3e42edc78f6 2023-10-09T09:55:53.255 INFO:tasks.workunit.client.0.smithi163.stdout: block_name_prefix: rbd_data.b3e42edc78f6 2023-10-09T09:55:53.255 INFO:tasks.workunit.client.0.smithi163.stdout: format: 2 2023-10-09T09:55:53.255 INFO:tasks.workunit.client.0.smithi163.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T09:55:53.255 INFO:tasks.workunit.client.0.smithi163.stdout: op_features: 2023-10-09T09:55:53.255 INFO:tasks.workunit.client.0.smithi163.stdout: flags: 2023-10-09T09:55:53.255 INFO:tasks.workunit.client.0.smithi163.stdout: create_timestamp: Mon Oct 9 09:55:52 2023 2023-10-09T09:55:53.256 INFO:tasks.workunit.client.0.smithi163.stdout: access_timestamp: Mon Oct 9 09:55:52 2023 2023-10-09T09:55:53.256 INFO:tasks.workunit.client.0.smithi163.stdout: modify_timestamp: Mon Oct 9 09:55:52 2023 2023-10-09T09:55:53.256 INFO:tasks.workunit.client.0.smithi163.stdout: protected: False 2023-10-09T09:55:53.259 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd image-meta list testimg_import 2023-10-09T09:55:53.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:53.305 INFO:tasks.workunit.client.0.smithi163.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-10-09T09:55:53.305 INFO:tasks.workunit.client.0.smithi163.stderr: 2023-10-09T09:55:53.305 INFO:tasks.workunit.client.0.smithi163.stderr:Key Value 2023-10-09T09:55:53.305 INFO:tasks.workunit.client.0.smithi163.stderr:key1 value1 2023-10-09T09:55:53.306 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ There is 1 metadatum on this image: 2023-10-09T09:55:53.306 INFO:tasks.workunit.client.0.smithi163.stderr: 2023-10-09T09:55:53.306 INFO:tasks.workunit.client.0.smithi163.stderr:Key Value 2023-10-09T09:55:53.306 INFO:tasks.workunit.client.0.smithi163.stderr:key1 value1 = There is 1 metadatum on this image: 2023-10-09T09:55:53.306 INFO:tasks.workunit.client.0.smithi163.stderr: 2023-10-09T09:55:53.306 INFO:tasks.workunit.client.0.smithi163.stderr:Key Value 2023-10-09T09:55:53.306 INFO:tasks.workunit.client.0.smithi163.stderr:key1 value1 ] 2023-10-09T09:55:53.306 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg_import /tmp/rbd_import_export_237542/img_import 2023-10-09T09:55:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:53 smithi181 bash[59345]: cluster 2023-10-09T09:55:52.030876+0000 osd.5 (osd.5) 890 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:53 smithi181 bash[59345]: cluster 2023-10-09T09:55:52.032887+0000 mon.a (mon.0) 2616 : cluster 0 osdmap e1299: 8 total, 8 up, 6 in 2023-10-09T09:55:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:53 smithi181 bash[59345]: cluster 2023-10-09T09:55:52.044113+0000 osd.6 (osd.6) 575 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:53 smithi181 bash[59345]: cluster 2023-10-09T09:55:52.078253+0000 osd.4 (osd.4) 678 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:53 smithi181 bash[59345]: cluster 2023-10-09T09:55:52.312934+0000 osd.7 (osd.7) 625 : cluster 3 failed to encode map e1299 with expected crc 2023-10-09T09:55:53.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:53.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:53.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:53.348 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete...done. 2023-10-09T09:55:53.353 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_237542/img /tmp/rbd_import_export_237542/img_import 2023-10-09T09:55:53.353 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_237542/img /tmp/rbd_import_export_237542/img_import 2023-10-09T09:55:53.354 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/img --format %b 2023-10-09T09:55:53.356 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=24 2023-10-09T09:55:53.356 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/img_import --format %b 2023-10-09T09:55:53.357 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=224 2023-10-09T09:55:53.357 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=200 2023-10-09T09:55:53.357 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=200 2023-10-09T09:55:53.357 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T09:55:53.357 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_237542/img_snap 2023-10-09T09:55:53.400 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete...done. 2023-10-09T09:55:53.404 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_237542/img_snap_import 2023-10-09T09:55:53.427 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:53.451 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete...done. 2023-10-09T09:55:53.456 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_237542/img_snap /tmp/rbd_import_export_237542/img_snap_import 2023-10-09T09:55:53.456 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_237542/img_snap /tmp/rbd_import_export_237542/img_snap_import 2023-10-09T09:55:53.457 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/img_snap --format %b 2023-10-09T09:55:53.459 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=224 2023-10-09T09:55:53.459 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/img_snap_import --format %b 2023-10-09T09:55:53.460 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=224 2023-10-09T09:55:53.460 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T09:55:53.460 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T09:55:53.460 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T09:55:53.460 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/img_v2 2023-10-09T09:55:53.460 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/img_import 2023-10-09T09:55:53.461 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/img_snap 2023-10-09T09:55:53.461 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:53.462 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/img_snap_import 2023-10-09T09:55:53.462 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T09:55:53.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:53.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:53.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:55:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:55:53] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-10-09T09:55:53.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:53.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:53.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:53.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:53.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:53.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:53.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:53.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:53.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:53.860 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:53.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:53.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:53.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:53.988 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:53.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:53.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:54.069 INFO:tasks.workunit.client.0.smithi163.stderr: Removing snap: 100% complete...done. 2023-10-09T09:55:54.080 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg_import 2023-10-09T09:55:54.092 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:54.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:54 smithi181 bash[59345]: cluster 2023-10-09T09:55:52.291042+0000 mgr.y (mgr.34107) 1502 : cluster 0 pgmap v2391: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 153 KiB/s rd, 44 KiB/s wr, 72 op/s 2023-10-09T09:55:54.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:54 smithi181 bash[59345]: cluster 2023-10-09T09:55:53.045923+0000 osd.5 (osd.5) 891 : cluster 3 failed to encode map e1300 with expected crc 2023-10-09T09:55:54.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:54 smithi181 bash[59345]: cluster 2023-10-09T09:55:53.046511+0000 osd.4 (osd.4) 679 : cluster 3 failed to encode map e1300 with expected crc 2023-10-09T09:55:54.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:54 smithi181 bash[59345]: cluster 2023-10-09T09:55:53.049227+0000 mon.a (mon.0) 2617 : cluster 0 osdmap e1300: 8 total, 8 up, 6 in 2023-10-09T09:55:54.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:54 smithi181 bash[59345]: cluster 2023-10-09T09:55:53.060630+0000 osd.6 (osd.6) 576 : cluster 3 failed to encode map e1300 with expected crc 2023-10-09T09:55:54.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:54.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:54.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:54.204 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T09:55:54.209 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap rm testimg@snap 2023-10-09T09:55:54.227 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:54.260 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:54.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:54.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:54 smithi163 bash[75425]: cluster 2023-10-09T09:55:52.291042+0000 mgr.y (mgr.34107) 1502 : cluster 0 pgmap v2391: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 153 KiB/s rd, 44 KiB/s wr, 72 op/s 2023-10-09T09:55:54.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:54 smithi163 bash[75425]: cluster 2023-10-09T09:55:53.045923+0000 osd.5 (osd.5) 891 : cluster 3 failed to encode map e1300 with expected crc 2023-10-09T09:55:54.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:54 smithi163 bash[75425]: cluster 2023-10-09T09:55:53.046511+0000 osd.4 (osd.4) 679 : cluster 3 failed to encode map e1300 with expected crc 2023-10-09T09:55:54.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:54 smithi163 bash[75425]: cluster 2023-10-09T09:55:53.049227+0000 mon.a (mon.0) 2617 : cluster 0 osdmap e1300: 8 total, 8 up, 6 in 2023-10-09T09:55:54.327 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:54 smithi163 bash[75425]: cluster 2023-10-09T09:55:53.060630+0000 osd.6 (osd.6) 576 : cluster 3 failed to encode map e1300 with expected crc 2023-10-09T09:55:54.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:54 smithi163 bash[78107]: cluster 2023-10-09T09:55:52.291042+0000 mgr.y (mgr.34107) 1502 : cluster 0 pgmap v2391: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 153 KiB/s rd, 44 KiB/s wr, 72 op/s 2023-10-09T09:55:54.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:54 smithi163 bash[78107]: cluster 2023-10-09T09:55:53.045923+0000 osd.5 (osd.5) 891 : cluster 3 failed to encode map e1300 with expected crc 2023-10-09T09:55:54.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:54 smithi163 bash[78107]: cluster 2023-10-09T09:55:53.046511+0000 osd.4 (osd.4) 679 : cluster 3 failed to encode map e1300 with expected crc 2023-10-09T09:55:54.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:54 smithi163 bash[78107]: cluster 2023-10-09T09:55:53.049227+0000 mon.a (mon.0) 2617 : cluster 0 osdmap e1300: 8 total, 8 up, 6 in 2023-10-09T09:55:54.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:54 smithi163 bash[78107]: cluster 2023-10-09T09:55:53.060630+0000 osd.6 (osd.6) 576 : cluster 3 failed to encode map e1300 with expected crc 2023-10-09T09:55:54.364 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:54.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:54.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:54.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:54.500 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:54.524 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:54.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:54.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:54.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:54.654 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:54.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:54.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:54.757 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:54.780 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:54.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:54.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:54.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:54.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:54.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:55.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:55.038 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:55.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:55.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:55.082 INFO:tasks.workunit.client.0.smithi163.stderr: Removing snap: 100% complete...done. 2023-10-09T09:55:55.091 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm testimg 2023-10-09T09:55:55.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:55.149 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:55 smithi163 bash[75425]: cluster 2023-10-09T09:55:54.056376+0000 osd.5 (osd.5) 892 : cluster 3 failed to encode map e1301 with expected crc 2023-10-09T09:55:55.149 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:55 smithi163 bash[75425]: cluster 2023-10-09T09:55:54.056611+0000 osd.4 (osd.4) 680 : cluster 3 failed to encode map e1301 with expected crc 2023-10-09T09:55:55.149 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:55 smithi163 bash[75425]: cluster 2023-10-09T09:55:54.058438+0000 mon.a (mon.0) 2618 : cluster 0 osdmap e1301: 8 total, 8 up, 6 in 2023-10-09T09:55:55.149 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:55 smithi163 bash[75425]: cluster 2023-10-09T09:55:54.071742+0000 osd.6 (osd.6) 577 : cluster 3 failed to encode map e1301 with expected crc 2023-10-09T09:55:55.149 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:55 smithi163 bash[78107]: cluster 2023-10-09T09:55:54.056376+0000 osd.5 (osd.5) 892 : cluster 3 failed to encode map e1301 with expected crc 2023-10-09T09:55:55.149 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:55 smithi163 bash[78107]: cluster 2023-10-09T09:55:54.056611+0000 osd.4 (osd.4) 680 : cluster 3 failed to encode map e1301 with expected crc 2023-10-09T09:55:55.150 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:55 smithi163 bash[78107]: cluster 2023-10-09T09:55:54.058438+0000 mon.a (mon.0) 2618 : cluster 0 osdmap e1301: 8 total, 8 up, 6 in 2023-10-09T09:55:55.150 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:55 smithi163 bash[78107]: cluster 2023-10-09T09:55:54.071742+0000 osd.6 (osd.6) 577 : cluster 3 failed to encode map e1301 with expected crc 2023-10-09T09:55:55.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:55.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:55.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:55.189 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T09:55:55.192 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --order 20 /tmp/rbd_import_export_237542/img testimg 2023-10-09T09:55:55.256 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T09:55:55.257 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:55:55.260 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_237542/img_v2 2023-10-09T09:55:55.265 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:55.292 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:55.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:55.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:55.352 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T09:55:55.356 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_237542/img_v2 testimg_import 2023-10-09T09:55:55.397 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:55.409 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:55 smithi181 bash[59345]: cluster 2023-10-09T09:55:54.056376+0000 osd.5 (osd.5) 892 : cluster 3 failed to encode map e1301 with expected crc 2023-10-09T09:55:55.409 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:55 smithi181 bash[59345]: cluster 2023-10-09T09:55:54.056611+0000 osd.4 (osd.4) 680 : cluster 3 failed to encode map e1301 with expected crc 2023-10-09T09:55:55.409 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:55 smithi181 bash[59345]: cluster 2023-10-09T09:55:54.058438+0000 mon.a (mon.0) 2618 : cluster 0 osdmap e1301: 8 total, 8 up, 6 in 2023-10-09T09:55:55.409 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:55 smithi181 bash[59345]: cluster 2023-10-09T09:55:54.071742+0000 osd.6 (osd.6) 577 : cluster 3 failed to encode map e1301 with expected crc 2023-10-09T09:55:55.427 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:55.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:55.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:55.431 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T09:55:55.435 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import 2023-10-09T09:55:55.435 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep order 2023-10-09T09:55:55.435 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk {print $2} 2023-10-09T09:55:55.436 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep 20 2023-10-09T09:55:55.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:55.563 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:55.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:55.667 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:55.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:55.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:55.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:55.827 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:55.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:55.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:55.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:55.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:55.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:56.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:56.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:56.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:56.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:56.106 INFO:tasks.workunit.client.0.smithi163.stdout:20 2023-10-09T09:55:56.106 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/img_v2 2023-10-09T09:55:56.171 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg_import 2023-10-09T09:55:56.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:56.220 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:56.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:56.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:56.321 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T09:55:56.323 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:56.326 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg 2023-10-09T09:55:56.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[78107]: cluster 2023-10-09T09:55:54.291797+0000 mgr.y (mgr.34107) 1503 : cluster 0 pgmap v2394: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 43 KiB/s rd, 2.5 KiB/s wr, 14 op/s 2023-10-09T09:55:56.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[78107]: cluster 2023-10-09T09:55:55.066046+0000 osd.5 (osd.5) 893 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[78107]: cluster 2023-10-09T09:55:55.069319+0000 osd.6 (osd.6) 578 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[78107]: cluster 2023-10-09T09:55:55.071209+0000 mon.a (mon.0) 2619 : cluster 0 osdmap e1302: 8 total, 8 up, 6 in 2023-10-09T09:55:56.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[78107]: cluster 2023-10-09T09:55:55.090164+0000 osd.4 (osd.4) 681 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[78107]: cluster 2023-10-09T09:55:55.130145+0000 osd.7 (osd.7) 626 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[78107]: audit 2023-10-09T09:55:55.419259+0000 mon.a (mon.0) 2620 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:56.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[75425]: cluster 2023-10-09T09:55:54.291797+0000 mgr.y (mgr.34107) 1503 : cluster 0 pgmap v2394: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 43 KiB/s rd, 2.5 KiB/s wr, 14 op/s 2023-10-09T09:55:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[75425]: cluster 2023-10-09T09:55:55.066046+0000 osd.5 (osd.5) 893 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[75425]: cluster 2023-10-09T09:55:55.069319+0000 osd.6 (osd.6) 578 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[75425]: cluster 2023-10-09T09:55:55.071209+0000 mon.a (mon.0) 2619 : cluster 0 osdmap e1302: 8 total, 8 up, 6 in 2023-10-09T09:55:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[75425]: cluster 2023-10-09T09:55:55.090164+0000 osd.4 (osd.4) 681 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[75425]: cluster 2023-10-09T09:55:55.130145+0000 osd.7 (osd.7) 626 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:56 smithi163 bash[75425]: audit 2023-10-09T09:55:55.419259+0000 mon.a (mon.0) 2620 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:56.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:56.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:56.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:56.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:56.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:56 smithi181 bash[59345]: cluster 2023-10-09T09:55:54.291797+0000 mgr.y (mgr.34107) 1503 : cluster 0 pgmap v2394: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 43 KiB/s rd, 2.5 KiB/s wr, 14 op/s 2023-10-09T09:55:56.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:56 smithi181 bash[59345]: cluster 2023-10-09T09:55:55.066046+0000 osd.5 (osd.5) 893 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:56 smithi181 bash[59345]: cluster 2023-10-09T09:55:55.069319+0000 osd.6 (osd.6) 578 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:56 smithi181 bash[59345]: cluster 2023-10-09T09:55:55.071209+0000 mon.a (mon.0) 2619 : cluster 0 osdmap e1302: 8 total, 8 up, 6 in 2023-10-09T09:55:56.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:56 smithi181 bash[59345]: cluster 2023-10-09T09:55:55.090164+0000 osd.4 (osd.4) 681 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:56 smithi181 bash[59345]: cluster 2023-10-09T09:55:55.130145+0000 osd.7 (osd.7) 626 : cluster 3 failed to encode map e1302 with expected crc 2023-10-09T09:55:56.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:56 smithi181 bash[59345]: audit 2023-10-09T09:55:55.419259+0000 mon.a (mon.0) 2620 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:56.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:56.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:56.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:56.500 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T09:55:56.503 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_237542/img testimg 2023-10-09T09:55:56.556 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T09:55:56.560 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg 2023-10-09T09:55:56.560 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep features 2023-10-09T09:55:56.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:56.601 INFO:tasks.workunit.client.0.smithi163.stderr:+ FEATURES_BEFORE= features: layering 2023-10-09T09:55:56.602 INFO:tasks.workunit.client.0.smithi163.stderr: op_features: 2023-10-09T09:55:56.602 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_237542/img_v2 2023-10-09T09:55:56.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:56.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:56.639 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T09:55:56.643 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_237542/img_v2 testimg_import 2023-10-09T09:55:56.705 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T09:55:56.709 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import 2023-10-09T09:55:56.709 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep features 2023-10-09T09:55:56.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:56.736 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:56.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:56.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:56.754 INFO:tasks.workunit.client.0.smithi163.stderr:+ FEATURES_AFTER= features: layering 2023-10-09T09:55:56.755 INFO:tasks.workunit.client.0.smithi163.stderr: op_features: 2023-10-09T09:55:56.755 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ features: layering 2023-10-09T09:55:56.755 INFO:tasks.workunit.client.0.smithi163.stderr: op_features: != features: layering 2023-10-09T09:55:56.755 INFO:tasks.workunit.client.0.smithi163.stderr: op_features: ] 2023-10-09T09:55:56.755 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/img_v2 2023-10-09T09:55:56.755 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg_import 2023-10-09T09:55:56.836 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T09:55:56.839 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:56.840 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg 2023-10-09T09:55:56.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:56.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:56.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:56.944 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T09:55:56.947 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_237542/img testimg 2023-10-09T09:55:56.961 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:56.985 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:56.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:56.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:57.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:57.110 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T09:55:57.114 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_237542/img_v2 2023-10-09T09:55:57.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:57.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:57.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:57.250 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:57.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:57.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:55:57.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:57.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:57.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:57.394 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T09:55:57.399 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_237542/img_v2 testimg_import 2023-10-09T09:55:57.475 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T09:55:57.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:57.480 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import 2023-10-09T09:55:57.480 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep stripe unit 2023-10-09T09:55:57.480 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep -Ei (4 KiB|4096) 2023-10-09T09:55:57.499 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:57.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:57.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:57.521 INFO:tasks.workunit.client.0.smithi163.stdout: stripe unit: 4 KiB 2023-10-09T09:55:57.521 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import 2023-10-09T09:55:57.522 INFO:tasks.workunit.client.0.smithi163.stderr:+ + grep stripe count 2023-10-09T09:55:57.522 INFO:tasks.workunit.client.0.smithi163.stderr:awk {print $3} 2023-10-09T09:55:57.522 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep 1000 2023-10-09T09:55:57.562 INFO:tasks.workunit.client.0.smithi163.stdout:1000 2023-10-09T09:55:57.563 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/img_v2 2023-10-09T09:55:57.563 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg_import 2023-10-09T09:55:57.603 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:57.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:57.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:57.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:57.679 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T09:55:57.684 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg 2023-10-09T09:55:57.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:57.769 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:57.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:57.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:57.795 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T09:55:57.800 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_237542/img testimg 2023-10-09T09:55:57.868 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T09:55:57.873 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap create testimg@snap1 2023-10-09T09:55:57.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:57.909 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:57.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:57.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:58.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:58.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:58.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:58.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:58.096 INFO:tasks.workunit.client.0.smithi163.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T09:55:58.105 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap create testimg@snap2 2023-10-09T09:55:58.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:58.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:58 smithi181 bash[59345]: cluster 2023-10-09T09:55:56.292375+0000 mgr.y (mgr.34107) 1504 : cluster 0 pgmap v2396: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 29 KiB/s rd, 8.0 KiB/s wr, 51 op/s 2023-10-09T09:55:58.183 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:58.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:58.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:58.286 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:58.320 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:58.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:58.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:58.424 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:58.457 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:58.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:58.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:58 smithi163 bash[78107]: cluster 2023-10-09T09:55:56.292375+0000 mgr.y (mgr.34107) 1504 : cluster 0 pgmap v2396: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 29 KiB/s rd, 8.0 KiB/s wr, 51 op/s 2023-10-09T09:55:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:58 smithi163 bash[75425]: cluster 2023-10-09T09:55:56.292375+0000 mgr.y (mgr.34107) 1504 : cluster 0 pgmap v2396: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 29 KiB/s rd, 8.0 KiB/s wr, 51 op/s 2023-10-09T09:55:58.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:58.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:58.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:58.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:58.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:58.727 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:55:58.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:58.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:55:58.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:58.863 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:58.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:58.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:58.966 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:58.998 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:55:59.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:59.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:55:59.102 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:59.104 INFO:tasks.workunit.client.0.smithi163.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T09:55:59.114 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap protect testimg@snap2 2023-10-09T09:55:59.129 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:55:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:59.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:55:59.173 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_237542/snap_protect 2023-10-09T09:55:59.225 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T09:55:59.230 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_237542/snap_protect testimg_import 2023-10-09T09:55:59.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:59.246 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:59 smithi181 bash[59345]: cluster 2023-10-09T09:55:58.079891+0000 osd.5 (osd.5) 894 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:55:59.246 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:59 smithi181 bash[59345]: cluster 2023-10-09T09:55:58.081696+0000 mon.a (mon.0) 2621 : cluster 0 osdmap e1303: 8 total, 8 up, 6 in 2023-10-09T09:55:59.246 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:59 smithi181 bash[59345]: cluster 2023-10-09T09:55:58.086782+0000 osd.6 (osd.6) 579 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:55:59.246 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:55:59 smithi181 bash[59345]: cluster 2023-10-09T09:55:58.091757+0000 osd.4 (osd.4) 682 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:55:59.264 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:55:59.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:59.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:55:59.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:59.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:59 smithi163 bash[75425]: cluster 2023-10-09T09:55:58.079891+0000 osd.5 (osd.5) 894 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:55:59.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:59 smithi163 bash[75425]: cluster 2023-10-09T09:55:58.081696+0000 mon.a (mon.0) 2621 : cluster 0 osdmap e1303: 8 total, 8 up, 6 in 2023-10-09T09:55:59.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:59 smithi163 bash[75425]: cluster 2023-10-09T09:55:58.086782+0000 osd.6 (osd.6) 579 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:55:59.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:55:59 smithi163 bash[75425]: cluster 2023-10-09T09:55:58.091757+0000 osd.4 (osd.4) 682 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:55:59.377 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:59 smithi163 bash[78107]: cluster 2023-10-09T09:55:58.079891+0000 osd.5 (osd.5) 894 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:55:59.377 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:59 smithi163 bash[78107]: cluster 2023-10-09T09:55:58.081696+0000 mon.a (mon.0) 2621 : cluster 0 osdmap e1303: 8 total, 8 up, 6 in 2023-10-09T09:55:59.377 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:59 smithi163 bash[78107]: cluster 2023-10-09T09:55:58.086782+0000 osd.6 (osd.6) 579 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:55:59.377 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:55:59 smithi163 bash[78107]: cluster 2023-10-09T09:55:58.091757+0000 osd.4 (osd.4) 682 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:55:59.390 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:55:59.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:59.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:55:59.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:59.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:55:59.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:59.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:55:59.632 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:59.666 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:59.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:59.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:59.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:59.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:55:59.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:59.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:55:59.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:59.926 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:55:59.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:55:59.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:56:00.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:00.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:00.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:56:00.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:00.191 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:00.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:00.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:56:00.199 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:00 smithi163 bash[78107]: cluster 2023-10-09T09:55:58.293129+0000 mgr.y (mgr.34107) 1505 : cluster 0 pgmap v2398: 73 pgs: 73 active+clean; 588 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 213 KiB/s rd, 18 KiB/s wr, 273 op/s 2023-10-09T09:56:00.200 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:00 smithi163 bash[78107]: cluster 2023-10-09T09:55:59.088120+0000 osd.7 (osd.7) 627 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:56:00.200 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:00 smithi163 bash[78107]: cluster 2023-10-09T09:55:59.088143+0000 osd.5 (osd.5) 895 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:00.200 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:00 smithi163 bash[78107]: cluster 2023-10-09T09:55:59.092780+0000 mon.a (mon.0) 2622 : cluster 0 osdmap e1304: 8 total, 8 up, 6 in 2023-10-09T09:56:00.200 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:00 smithi163 bash[78107]: cluster 2023-10-09T09:55:59.098228+0000 osd.6 (osd.6) 580 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:00.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:00 smithi163 bash[75425]: cluster 2023-10-09T09:55:58.293129+0000 mgr.y (mgr.34107) 1505 : cluster 0 pgmap v2398: 73 pgs: 73 active+clean; 588 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 213 KiB/s rd, 18 KiB/s wr, 273 op/s 2023-10-09T09:56:00.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:00 smithi163 bash[75425]: cluster 2023-10-09T09:55:59.088120+0000 osd.7 (osd.7) 627 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:56:00.201 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:00 smithi163 bash[75425]: cluster 2023-10-09T09:55:59.088143+0000 osd.5 (osd.5) 895 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:00.201 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:00 smithi163 bash[75425]: cluster 2023-10-09T09:55:59.092780+0000 mon.a (mon.0) 2622 : cluster 0 osdmap e1304: 8 total, 8 up, 6 in 2023-10-09T09:56:00.201 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:00 smithi163 bash[75425]: cluster 2023-10-09T09:55:59.098228+0000 osd.6 (osd.6) 580 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:00.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:00 smithi181 bash[59345]: cluster 2023-10-09T09:55:58.293129+0000 mgr.y (mgr.34107) 1505 : cluster 0 pgmap v2398: 73 pgs: 73 active+clean; 588 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 213 KiB/s rd, 18 KiB/s wr, 273 op/s 2023-10-09T09:56:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:00 smithi181 bash[59345]: cluster 2023-10-09T09:55:59.088120+0000 osd.7 (osd.7) 627 : cluster 3 failed to encode map e1303 with expected crc 2023-10-09T09:56:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:00 smithi181 bash[59345]: cluster 2023-10-09T09:55:59.088143+0000 osd.5 (osd.5) 895 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:00 smithi181 bash[59345]: cluster 2023-10-09T09:55:59.092780+0000 mon.a (mon.0) 2622 : cluster 0 osdmap e1304: 8 total, 8 up, 6 in 2023-10-09T09:56:00.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:00 smithi181 bash[59345]: cluster 2023-10-09T09:55:59.098228+0000 osd.6 (osd.6) 580 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:00.323 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:00.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:00.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:00.426 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:00.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:00.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:00.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:00.559 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:00.591 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:00.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:00.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:00.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:00.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:00.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:56:00.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:56:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:00.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:56:00.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:00.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:00.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:00.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:56:01.045 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T09:56:01.050 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import@snap1 2023-10-09T09:56:01.050 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep protected: False 2023-10-09T09:56:01.094 INFO:tasks.workunit.client.0.smithi163.stdout: protected: False 2023-10-09T09:56:01.094 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import@snap2 2023-10-09T09:56:01.095 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep protected: True 2023-10-09T09:56:01.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:01.133 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:01.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:01.136 INFO:tasks.workunit.client.0.smithi163.stdout: protected: True 2023-10-09T09:56:01.136 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/snap_protect 2023-10-09T09:56:01.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:56:01.137 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T09:56:01.193 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T09:56:01.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:01.246 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap purge testimg 2023-10-09T09:56:01.263 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:01.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:01 smithi181 bash[59345]: cluster 2023-10-09T09:55:59.265892+0000 osd.4 (osd.4) 683 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:01 smithi181 bash[59345]: cluster 2023-10-09T09:55:59.268046+0000 osd.4 (osd.4) 684 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:01.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:01 smithi181 bash[59345]: cluster 2023-10-09T09:56:00.098097+0000 osd.5 (osd.5) 896 : cluster 3 failed to encode map e1305 with expected crc 2023-10-09T09:56:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:01 smithi181 bash[59345]: cluster 2023-10-09T09:56:00.099762+0000 mon.a (mon.0) 2623 : cluster 0 osdmap e1305: 8 total, 8 up, 6 in 2023-10-09T09:56:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:01 smithi181 bash[59345]: cluster 2023-10-09T09:56:00.153492+0000 osd.4 (osd.4) 685 : cluster 3 failed to encode map e1305 with expected crc 2023-10-09T09:56:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:01 smithi181 bash[59345]: cluster 2023-10-09T09:56:00.157260+0000 osd.6 (osd.6) 581 : cluster 3 failed to encode map e1305 with expected crc 2023-10-09T09:56:01.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:01 smithi181 bash[59345]: cluster 2023-10-09T09:56:01.016403+0000 mon.a (mon.0) 2624 : cluster 0 osdmap e1306: 8 total, 8 up, 6 in 2023-10-09T09:56:01.366 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:01.374 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[78107]: cluster 2023-10-09T09:55:59.265892+0000 osd.4 (osd.4) 683 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:01.374 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[78107]: cluster 2023-10-09T09:55:59.268046+0000 osd.4 (osd.4) 684 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:01.374 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[78107]: cluster 2023-10-09T09:56:00.098097+0000 osd.5 (osd.5) 896 : cluster 3 failed to encode map e1305 with expected crc 2023-10-09T09:56:01.374 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[78107]: cluster 2023-10-09T09:56:00.099762+0000 mon.a (mon.0) 2623 : cluster 0 osdmap e1305: 8 total, 8 up, 6 in 2023-10-09T09:56:01.374 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[78107]: cluster 2023-10-09T09:56:00.153492+0000 osd.4 (osd.4) 685 : cluster 3 failed to encode map e1305 with expected crc 2023-10-09T09:56:01.374 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[78107]: cluster 2023-10-09T09:56:00.157260+0000 osd.6 (osd.6) 581 : cluster 3 failed to encode map e1305 with expected crc 2023-10-09T09:56:01.374 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[78107]: cluster 2023-10-09T09:56:01.016403+0000 mon.a (mon.0) 2624 : cluster 0 osdmap e1306: 8 total, 8 up, 6 in 2023-10-09T09:56:01.374 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[75425]: cluster 2023-10-09T09:55:59.265892+0000 osd.4 (osd.4) 683 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:01.375 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[75425]: cluster 2023-10-09T09:55:59.268046+0000 osd.4 (osd.4) 684 : cluster 3 failed to encode map e1304 with expected crc 2023-10-09T09:56:01.375 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[75425]: cluster 2023-10-09T09:56:00.098097+0000 osd.5 (osd.5) 896 : cluster 3 failed to encode map e1305 with expected crc 2023-10-09T09:56:01.375 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[75425]: cluster 2023-10-09T09:56:00.099762+0000 mon.a (mon.0) 2623 : cluster 0 osdmap e1305: 8 total, 8 up, 6 in 2023-10-09T09:56:01.375 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[75425]: cluster 2023-10-09T09:56:00.153492+0000 osd.4 (osd.4) 685 : cluster 3 failed to encode map e1305 with expected crc 2023-10-09T09:56:01.375 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[75425]: cluster 2023-10-09T09:56:00.157260+0000 osd.6 (osd.6) 581 : cluster 3 failed to encode map e1305 with expected crc 2023-10-09T09:56:01.375 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:01 smithi163 bash[75425]: cluster 2023-10-09T09:56:01.016403+0000 mon.a (mon.0) 2624 : cluster 0 osdmap e1306: 8 total, 8 up, 6 in 2023-10-09T09:56:01.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:01.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:01.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:56:01.489 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:01.521 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:01.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:01.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:56:01.624 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:01.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:01.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:01.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:01.751 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:01.786 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:01.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:01.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:01.889 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:01.919 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:01.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:01.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:56:02.024 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:02.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:02.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:02.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:02.162 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:02.193 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:02.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:02.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:56:02.298 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:02.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:02 smithi181 bash[59345]: cluster 2023-10-09T09:56:00.293599+0000 mgr.y (mgr.34107) 1506 : cluster 0 pgmap v2401: 73 pgs: 73 active+clean; 588 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 277 KiB/s rd, 24 KiB/s wr, 356 op/s 2023-10-09T09:56:02.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:02 smithi181 bash[59345]: cluster 2023-10-09T09:56:01.014374+0000 osd.5 (osd.5) 897 : cluster 3 failed to encode map e1306 with expected crc 2023-10-09T09:56:02.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:02 smithi181 bash[59345]: cluster 2023-10-09T09:56:01.024021+0000 osd.4 (osd.4) 686 : cluster 3 failed to encode map e1306 with expected crc 2023-10-09T09:56:02.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:02 smithi181 bash[59345]: cluster 2023-10-09T09:56:01.030142+0000 osd.6 (osd.6) 582 : cluster 3 failed to encode map e1306 with expected crc 2023-10-09T09:56:02.330 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:02.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:02.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:56:02.434 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:02.444 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:02 smithi163 bash[78107]: cluster 2023-10-09T09:56:00.293599+0000 mgr.y (mgr.34107) 1506 : cluster 0 pgmap v2401: 73 pgs: 73 active+clean; 588 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 277 KiB/s rd, 24 KiB/s wr, 356 op/s 2023-10-09T09:56:02.445 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:02 smithi163 bash[78107]: cluster 2023-10-09T09:56:01.014374+0000 osd.5 (osd.5) 897 : cluster 3 failed to encode map e1306 with expected crc 2023-10-09T09:56:02.445 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:02 smithi163 bash[78107]: cluster 2023-10-09T09:56:01.024021+0000 osd.4 (osd.4) 686 : cluster 3 failed to encode map e1306 with expected crc 2023-10-09T09:56:02.445 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:02 smithi163 bash[78107]: cluster 2023-10-09T09:56:01.030142+0000 osd.6 (osd.6) 582 : cluster 3 failed to encode map e1306 with expected crc 2023-10-09T09:56:02.445 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:02 smithi163 bash[75425]: cluster 2023-10-09T09:56:00.293599+0000 mgr.y (mgr.34107) 1506 : cluster 0 pgmap v2401: 73 pgs: 73 active+clean; 588 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 277 KiB/s rd, 24 KiB/s wr, 356 op/s 2023-10-09T09:56:02.445 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:02 smithi163 bash[75425]: cluster 2023-10-09T09:56:01.014374+0000 osd.5 (osd.5) 897 : cluster 3 failed to encode map e1306 with expected crc 2023-10-09T09:56:02.445 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:02 smithi163 bash[75425]: cluster 2023-10-09T09:56:01.024021+0000 osd.4 (osd.4) 686 : cluster 3 failed to encode map e1306 with expected crc 2023-10-09T09:56:02.445 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:02 smithi163 bash[75425]: cluster 2023-10-09T09:56:01.030142+0000 osd.6 (osd.6) 582 : cluster 3 failed to encode map e1306 with expected crc 2023-10-09T09:56:02.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:02.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:02.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:02.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:02.598 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:02.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:02.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:02.702 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:02.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:02.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:02.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:56:02.840 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:02.866 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:02.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:02.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:56:02.969 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:56:02.999 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:56:03.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:03.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:56:03.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:03.131 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:03.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:03.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:56:03.138 INFO:tasks.workunit.client.0.smithi163.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T09:56:03.148 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap purge testimg_import 2023-10-09T09:56:03.234 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:03.269 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:03.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:03.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:56:03.373 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:03.387 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:56:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:56:03] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.33.4" 2023-10-09T09:56:03.387 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:03 smithi163 bash[78107]: cluster 2023-10-09T09:56:02.117768+0000 osd.5 (osd.5) 898 : cluster 3 failed to encode map e1307 with expected crc 2023-10-09T09:56:03.387 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:03 smithi163 bash[78107]: cluster 2023-10-09T09:56:02.119361+0000 mon.a (mon.0) 2625 : cluster 0 osdmap e1307: 8 total, 8 up, 6 in 2023-10-09T09:56:03.387 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:03 smithi163 bash[78107]: cluster 2023-10-09T09:56:02.125827+0000 osd.6 (osd.6) 583 : cluster 3 failed to encode map e1307 with expected crc 2023-10-09T09:56:03.387 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:03 smithi163 bash[78107]: cluster 2023-10-09T09:56:02.133991+0000 osd.4 (osd.4) 687 : cluster 3 failed to encode map e1307 with expected crc 2023-10-09T09:56:03.388 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:03 smithi163 bash[75425]: cluster 2023-10-09T09:56:02.117768+0000 osd.5 (osd.5) 898 : cluster 3 failed to encode map e1307 with expected crc 2023-10-09T09:56:03.388 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:03 smithi163 bash[75425]: cluster 2023-10-09T09:56:02.119361+0000 mon.a (mon.0) 2625 : cluster 0 osdmap e1307: 8 total, 8 up, 6 in 2023-10-09T09:56:03.388 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:03 smithi163 bash[75425]: cluster 2023-10-09T09:56:02.125827+0000 osd.6 (osd.6) 583 : cluster 3 failed to encode map e1307 with expected crc 2023-10-09T09:56:03.388 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:03 smithi163 bash[75425]: cluster 2023-10-09T09:56:02.133991+0000 osd.4 (osd.4) 687 : cluster 3 failed to encode map e1307 with expected crc 2023-10-09T09:56:03.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:03.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:03.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:56:03.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:03.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:03 smithi181 bash[59345]: cluster 2023-10-09T09:56:02.117768+0000 osd.5 (osd.5) 898 : cluster 3 failed to encode map e1307 with expected crc 2023-10-09T09:56:03.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:03 smithi181 bash[59345]: cluster 2023-10-09T09:56:02.119361+0000 mon.a (mon.0) 2625 : cluster 0 osdmap e1307: 8 total, 8 up, 6 in 2023-10-09T09:56:03.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:03 smithi181 bash[59345]: cluster 2023-10-09T09:56:02.125827+0000 osd.6 (osd.6) 583 : cluster 3 failed to encode map e1307 with expected crc 2023-10-09T09:56:03.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:03 smithi181 bash[59345]: cluster 2023-10-09T09:56:02.133991+0000 osd.4 (osd.4) 687 : cluster 3 failed to encode map e1307 with expected crc 2023-10-09T09:56:03.536 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:03.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:03.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:03.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:03.668 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:03.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:03.773 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:03.796 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:03.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:03.899 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:03.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:03.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:03.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:03.936 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:56:03.936 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:56:03.956 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:56:03.977 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:56:04.003 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:56:04.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:04.043 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:56:04.053 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:04.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:04.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:04.082 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:56:04.109 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:56:04.137 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:56:04.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:04.167 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:56:04.169 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:56:04.169 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:56:04.169 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:56:04.170 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:56:04.170 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:56:04.170 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:56:04.170 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:56:04.170 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:56:04.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:04.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:04.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:04.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[78107]: cluster 2023-10-09T09:56:02.294155+0000 mgr.y (mgr.34107) 1507 : cluster 0 pgmap v2404: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 270 KiB/s rd, 34 KiB/s wr, 366 op/s 2023-10-09T09:56:04.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[78107]: cluster 2023-10-09T09:56:03.129015+0000 osd.5 (osd.5) 899 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[78107]: cluster 2023-10-09T09:56:03.129270+0000 osd.4 (osd.4) 688 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[78107]: cluster 2023-10-09T09:56:03.131846+0000 mon.a (mon.0) 2626 : cluster 0 osdmap e1308: 8 total, 8 up, 6 in 2023-10-09T09:56:04.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[78107]: cluster 2023-10-09T09:56:03.136519+0000 osd.6 (osd.6) 584 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[78107]: cluster 2023-10-09T09:56:03.892265+0000 osd.7 (osd.7) 628 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[78107]: cluster 2023-10-09T09:56:03.895119+0000 osd.7 (osd.7) 629 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[78107]: audit 2023-10-09T09:56:03.917814+0000 mon.a (mon.0) 2627 : audit 0 from='client.? 172.21.15.163:0/2208492047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:56:04.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[75425]: cluster 2023-10-09T09:56:02.294155+0000 mgr.y (mgr.34107) 1507 : cluster 0 pgmap v2404: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 270 KiB/s rd, 34 KiB/s wr, 366 op/s 2023-10-09T09:56:04.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[75425]: cluster 2023-10-09T09:56:03.129015+0000 osd.5 (osd.5) 899 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[75425]: cluster 2023-10-09T09:56:03.129270+0000 osd.4 (osd.4) 688 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[75425]: cluster 2023-10-09T09:56:03.131846+0000 mon.a (mon.0) 2626 : cluster 0 osdmap e1308: 8 total, 8 up, 6 in 2023-10-09T09:56:04.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[75425]: cluster 2023-10-09T09:56:03.136519+0000 osd.6 (osd.6) 584 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[75425]: cluster 2023-10-09T09:56:03.892265+0000 osd.7 (osd.7) 628 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[75425]: cluster 2023-10-09T09:56:03.895119+0000 osd.7 (osd.7) 629 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:04 smithi163 bash[75425]: audit 2023-10-09T09:56:03.917814+0000 mon.a (mon.0) 2627 : audit 0 from='client.? 172.21.15.163:0/2208492047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:56:04.282 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:04.305 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:04.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:56:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:04 smithi181 bash[59345]: cluster 2023-10-09T09:56:02.294155+0000 mgr.y (mgr.34107) 1507 : cluster 0 pgmap v2404: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 270 KiB/s rd, 34 KiB/s wr, 366 op/s 2023-10-09T09:56:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:04 smithi181 bash[59345]: cluster 2023-10-09T09:56:03.129015+0000 osd.5 (osd.5) 899 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:04 smithi181 bash[59345]: cluster 2023-10-09T09:56:03.129270+0000 osd.4 (osd.4) 688 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:04 smithi181 bash[59345]: cluster 2023-10-09T09:56:03.131846+0000 mon.a (mon.0) 2626 : cluster 0 osdmap e1308: 8 total, 8 up, 6 in 2023-10-09T09:56:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:04 smithi181 bash[59345]: cluster 2023-10-09T09:56:03.136519+0000 osd.6 (osd.6) 584 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:04 smithi181 bash[59345]: cluster 2023-10-09T09:56:03.892265+0000 osd.7 (osd.7) 628 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:04 smithi181 bash[59345]: cluster 2023-10-09T09:56:03.895119+0000 osd.7 (osd.7) 629 : cluster 3 failed to encode map e1308 with expected crc 2023-10-09T09:56:04.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:04 smithi181 bash[59345]: audit 2023-10-09T09:56:03.917814+0000 mon.a (mon.0) 2627 : audit 0 from='client.? 172.21.15.163:0/2208492047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:56:04.361 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:56:04.362 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:56:04.408 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:04.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:04.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:04.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:04.547 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:04.574 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:04.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:04.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:04.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:04.705 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:04.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:04.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:04.841 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:04.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:04.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:04.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:56:04.974 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:56:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:04.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:56:05.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:05.101 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:05.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:05.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:05.145 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T09:56:05.151 INFO:tasks.workunit.client.0.smithi163.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T09:56:05.163 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg 2023-10-09T09:56:05.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:05.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[78107]: cluster 2023-10-09T09:56:04.130056+0000 osd.5 (osd.5) 900 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[78107]: cluster 2023-10-09T09:56:04.132664+0000 mon.a (mon.0) 2628 : cluster 0 osdmap e1309: 8 total, 8 up, 6 in 2023-10-09T09:56:05.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[78107]: cluster 2023-10-09T09:56:04.140783+0000 osd.4 (osd.4) 689 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[78107]: cluster 2023-10-09T09:56:04.156135+0000 osd.6 (osd.6) 585 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[78107]: audit 2023-10-09T09:56:04.156703+0000 mon.a (mon.0) 2629 : audit 1 from='client.? 172.21.15.163:0/1149922058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:56:05.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[78107]: cluster 2023-10-09T09:56:04.392942+0000 osd.7 (osd.7) 630 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[78107]: cluster 2023-10-09T09:56:04.396050+0000 osd.7 (osd.7) 631 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[78107]: cluster 2023-10-09T09:56:04.398615+0000 osd.7 (osd.7) 632 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[78107]: cluster 2023-10-09T09:56:04.400020+0000 osd.7 (osd.7) 633 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[78107]: cluster 2023-10-09T09:56:05.134944+0000 mon.a (mon.0) 2630 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:56:05.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[75425]: cluster 2023-10-09T09:56:04.130056+0000 osd.5 (osd.5) 900 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[75425]: cluster 2023-10-09T09:56:04.132664+0000 mon.a (mon.0) 2628 : cluster 0 osdmap e1309: 8 total, 8 up, 6 in 2023-10-09T09:56:05.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[75425]: cluster 2023-10-09T09:56:04.140783+0000 osd.4 (osd.4) 689 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[75425]: cluster 2023-10-09T09:56:04.156135+0000 osd.6 (osd.6) 585 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[75425]: audit 2023-10-09T09:56:04.156703+0000 mon.a (mon.0) 2629 : audit 1 from='client.? 172.21.15.163:0/1149922058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:56:05.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[75425]: cluster 2023-10-09T09:56:04.392942+0000 osd.7 (osd.7) 630 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[75425]: cluster 2023-10-09T09:56:04.396050+0000 osd.7 (osd.7) 631 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[75425]: cluster 2023-10-09T09:56:04.398615+0000 osd.7 (osd.7) 632 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[75425]: cluster 2023-10-09T09:56:04.400020+0000 osd.7 (osd.7) 633 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:05 smithi163 bash[75425]: cluster 2023-10-09T09:56:05.134944+0000 mon.a (mon.0) 2630 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:56:05.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:05.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:05.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:05.276 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T09:56:05.280 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg_import 2023-10-09T09:56:05.328 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:05.358 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:05.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:05.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:05.388 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T09:56:05.392 INFO:tasks.workunit.client.0.smithi163.stderr:+ tiered=0 2023-10-09T09:56:05.392 INFO:tasks.workunit.client.0.smithi163.stderr:+ ceph osd dump 2023-10-09T09:56:05.392 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep ^pool 2023-10-09T09:56:05.392 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep 'rbd' 2023-10-09T09:56:05.392 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep tier 2023-10-09T09:56:05.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:05.472 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:05 smithi181 bash[59345]: cluster 2023-10-09T09:56:04.130056+0000 osd.5 (osd.5) 900 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.472 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:05 smithi181 bash[59345]: cluster 2023-10-09T09:56:04.132664+0000 mon.a (mon.0) 2628 : cluster 0 osdmap e1309: 8 total, 8 up, 6 in 2023-10-09T09:56:05.472 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:05 smithi181 bash[59345]: cluster 2023-10-09T09:56:04.140783+0000 osd.4 (osd.4) 689 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.472 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:05 smithi181 bash[59345]: cluster 2023-10-09T09:56:04.156135+0000 osd.6 (osd.6) 585 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.472 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:05 smithi181 bash[59345]: audit 2023-10-09T09:56:04.156703+0000 mon.a (mon.0) 2629 : audit 1 from='client.? 172.21.15.163:0/1149922058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:56:05.472 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:05 smithi181 bash[59345]: cluster 2023-10-09T09:56:04.392942+0000 osd.7 (osd.7) 630 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.472 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:05 smithi181 bash[59345]: cluster 2023-10-09T09:56:04.396050+0000 osd.7 (osd.7) 631 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.472 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:05 smithi181 bash[59345]: cluster 2023-10-09T09:56:04.398615+0000 osd.7 (osd.7) 632 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.472 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:05 smithi181 bash[59345]: cluster 2023-10-09T09:56:04.400020+0000 osd.7 (osd.7) 633 : cluster 3 failed to encode map e1309 with expected crc 2023-10-09T09:56:05.473 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:05 smithi181 bash[59345]: cluster 2023-10-09T09:56:05.134944+0000 mon.a (mon.0) 2630 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:56:05.486 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:05.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:05.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:05.590 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:05.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:05.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:05.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:05.722 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:05.742 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:05.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:05.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:05.846 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:05.867 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:05.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:05.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:05.970 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:05.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:05.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:05.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:56:06.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:06.113 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T09:56:06.130 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:06.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:06.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:06.234 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:06.270 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:06.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:06.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:56:06.286 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_237542/sparse1 2023-10-09T09:56:06.290 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records in 2023-10-09T09:56:06.290 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records out 2023-10-09T09:56:06.291 INFO:tasks.workunit.client.0.smithi163.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00359106 s, 292 MB/s 2023-10-09T09:56:06.291 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_237542/sparse2 2023-10-09T09:56:06.295 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records in 2023-10-09T09:56:06.295 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records out 2023-10-09T09:56:06.295 INFO:tasks.workunit.client.0.smithi163.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00418173 s, 251 MB/s 2023-10-09T09:56:06.295 INFO:tasks.workunit.client.0.smithi163.stderr:+ truncate /tmp/rbd_import_export_237542/sparse2 -s 2M 2023-10-09T09:56:06.374 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:06.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:06 smithi181 bash[59345]: cluster 2023-10-09T09:56:04.294592+0000 mgr.y (mgr.34107) 1508 : cluster 0 pgmap v2407: 73 pgs: 73 active+clean; 802 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 260 KiB/s rd, 33 KiB/s wr, 352 op/s 2023-10-09T09:56:06.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:06 smithi181 bash[59345]: audit 2023-10-09T09:56:04.349517+0000 mgr.y (mgr.34107) 1509 : audit 0 from='client.46322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:56:06.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:06 smithi181 bash[59345]: cluster 2023-10-09T09:56:05.138024+0000 osd.5 (osd.5) 901 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:06 smithi181 bash[59345]: cluster 2023-10-09T09:56:05.138973+0000 osd.6 (osd.6) 586 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:06 smithi181 bash[59345]: audit 2023-10-09T09:56:05.139899+0000 mon.a (mon.0) 2631 : audit 1 from='client.? 172.21.15.163:0/1149922058' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:56:06.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:06 smithi181 bash[59345]: cluster 2023-10-09T09:56:05.140005+0000 mon.a (mon.0) 2632 : cluster 0 osdmap e1310: 8 total, 8 up, 6 in 2023-10-09T09:56:06.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:06 smithi181 bash[59345]: cluster 2023-10-09T09:56:05.146281+0000 osd.4 (osd.4) 690 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.388 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:06 smithi181 bash[59345]: cluster 2023-10-09T09:56:05.275136+0000 osd.7 (osd.7) 634 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.388 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:06 smithi181 bash[59345]: cluster 2023-10-09T09:56:06.021010+0000 mon.a (mon.0) 2633 : cluster 0 osdmap e1311: 8 total, 8 up, 6 in 2023-10-09T09:56:06.392 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse1 2023-10-09T09:56:06.409 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:56:06.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:06.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:56:06.443 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 0% complete...failed. 2023-10-09T09:56:06.443 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T09:56:06.446 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T09:56:06.446 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_237542/sparse1 2023-10-09T09:56:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[75425]: cluster 2023-10-09T09:56:04.294592+0000 mgr.y (mgr.34107) 1508 : cluster 0 pgmap v2407: 73 pgs: 73 active+clean; 802 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 260 KiB/s rd, 33 KiB/s wr, 352 op/s 2023-10-09T09:56:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[75425]: audit 2023-10-09T09:56:04.349517+0000 mgr.y (mgr.34107) 1509 : audit 0 from='client.46322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:56:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[75425]: cluster 2023-10-09T09:56:05.138024+0000 osd.5 (osd.5) 901 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[75425]: cluster 2023-10-09T09:56:05.138973+0000 osd.6 (osd.6) 586 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[75425]: audit 2023-10-09T09:56:05.139899+0000 mon.a (mon.0) 2631 : audit 1 from='client.? 172.21.15.163:0/1149922058' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:56:06.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[75425]: cluster 2023-10-09T09:56:05.140005+0000 mon.a (mon.0) 2632 : cluster 0 osdmap e1310: 8 total, 8 up, 6 in 2023-10-09T09:56:06.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[75425]: cluster 2023-10-09T09:56:05.146281+0000 osd.4 (osd.4) 690 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[75425]: cluster 2023-10-09T09:56:05.275136+0000 osd.7 (osd.7) 634 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[75425]: cluster 2023-10-09T09:56:06.021010+0000 mon.a (mon.0) 2633 : cluster 0 osdmap e1311: 8 total, 8 up, 6 in 2023-10-09T09:56:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[78107]: cluster 2023-10-09T09:56:04.294592+0000 mgr.y (mgr.34107) 1508 : cluster 0 pgmap v2407: 73 pgs: 73 active+clean; 802 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 260 KiB/s rd, 33 KiB/s wr, 352 op/s 2023-10-09T09:56:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[78107]: audit 2023-10-09T09:56:04.349517+0000 mgr.y (mgr.34107) 1509 : audit 0 from='client.46322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:56:06.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[78107]: cluster 2023-10-09T09:56:05.138024+0000 osd.5 (osd.5) 901 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[78107]: cluster 2023-10-09T09:56:05.138973+0000 osd.6 (osd.6) 586 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[78107]: audit 2023-10-09T09:56:05.139899+0000 mon.a (mon.0) 2631 : audit 1 from='client.? 172.21.15.163:0/1149922058' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:56:06.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[78107]: cluster 2023-10-09T09:56:05.140005+0000 mon.a (mon.0) 2632 : cluster 0 osdmap e1310: 8 total, 8 up, 6 in 2023-10-09T09:56:06.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[78107]: cluster 2023-10-09T09:56:05.146281+0000 osd.4 (osd.4) 690 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[78107]: cluster 2023-10-09T09:56:05.275136+0000 osd.7 (osd.7) 634 : cluster 3 failed to encode map e1310 with expected crc 2023-10-09T09:56:06.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:06 smithi163 bash[78107]: cluster 2023-10-09T09:56:06.021010+0000 mon.a (mon.0) 2633 : cluster 0 osdmap e1311: 8 total, 8 up, 6 in 2023-10-09T09:56:06.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:06.521 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T09:56:06.522 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:56:06.522 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:56:06.526 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd ls -l 2023-10-09T09:56:06.526 INFO:tasks.workunit.client.0.smithi163.stderr:+ + grep -Ei (2 MiB|2048k) 2023-10-09T09:56:06.526 INFO:tasks.workunit.client.0.smithi163.stderr:grep sparse1 2023-10-09T09:56:06.548 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:06.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:06.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:06.570 INFO:tasks.workunit.client.0.smithi163.stdout:sparse1 2 MiB 2 2023-10-09T09:56:06.571 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects sparse1 2023-10-09T09:56:06.571 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse1 2023-10-09T09:56:06.571 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse1 2023-10-09T09:56:06.571 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep block_name_prefix 2023-10-09T09:56:06.571 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk {print $NF;} 2023-10-09T09:56:06.613 INFO:tasks.workunit.client.0.smithi163.stderr:+ prefix=rbd_data.b5101b262d99 2023-10-09T09:56:06.613 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep rbd_data.b5101b262d99 2023-10-09T09:56:06.613 INFO:tasks.workunit.client.0.smithi163.stderr:+ + get_image_data_pool sparse1 2023-10-09T09:56:06.613 INFO:tasks.workunit.client.0.smithi163.stderr:sed -e s/rbd_data.b5101b262d99\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:56:06.613 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse1 2023-10-09T09:56:06.613 INFO:tasks.workunit.client.0.smithi163.stderr:+ sort -u 2023-10-09T09:56:06.613 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse1 2023-10-09T09:56:06.613 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep data_pool: 2023-10-09T09:56:06.614 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk -F: { print $NF } 2023-10-09T09:56:06.649 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool= 2023-10-09T09:56:06.649 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ -z ] 2023-10-09T09:56:06.649 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool=rbd 2023-10-09T09:56:06.649 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo rbd 2023-10-09T09:56:06.650 INFO:tasks.workunit.client.0.smithi163.stderr:+ rados ls -p rbd 2023-10-09T09:56:06.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:06.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:06.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:06.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:06.689 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects=1 2023-10-09T09:56:06.689 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo 1 2023-10-09T09:56:06.689 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T09:56:06.689 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export sparse1 /tmp/rbd_import_export_237542/sparse1.out 2023-10-09T09:56:06.726 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T09:56:06.730 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_237542/sparse1 /tmp/rbd_import_export_237542/sparse1.out 2023-10-09T09:56:06.730 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_237542/sparse1 /tmp/rbd_import_export_237542/sparse1.out 2023-10-09T09:56:06.732 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/sparse1 --format %b 2023-10-09T09:56:06.733 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=2048 2023-10-09T09:56:06.733 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/sparse1.out --format %b 2023-10-09T09:56:06.733 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=2048 2023-10-09T09:56:06.733 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T09:56:06.733 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T09:56:06.734 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T09:56:06.734 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/sparse1.out 2023-10-09T09:56:06.734 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse1 2023-10-09T09:56:06.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:06.808 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:06.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:06.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:06.838 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T09:56:06.842 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse2 2023-10-09T09:56:06.880 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 0% complete...failed. 2023-10-09T09:56:06.880 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T09:56:06.883 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T09:56:06.883 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_237542/sparse2 2023-10-09T09:56:06.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:06.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:06.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:06.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:06.979 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T09:56:06.979 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:56:06.979 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:56:06.982 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd ls -l 2023-10-09T09:56:06.982 INFO:tasks.workunit.client.0.smithi163.stderr:+ + grepgrep -Ei (2 MiB|2048k) 2023-10-09T09:56:06.983 INFO:tasks.workunit.client.0.smithi163.stderr: sparse2 2023-10-09T09:56:07.028 INFO:tasks.workunit.client.0.smithi163.stdout:sparse2 2 MiB 2 2023-10-09T09:56:07.029 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects sparse2 2023-10-09T09:56:07.029 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse2 2023-10-09T09:56:07.029 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse2 2023-10-09T09:56:07.029 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep block_name_prefix 2023-10-09T09:56:07.030 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk {print $NF;} 2023-10-09T09:56:07.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:07.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:07.075 INFO:tasks.workunit.client.0.smithi163.stderr:+ prefix=rbd_data.b543344f54ec 2023-10-09T09:56:07.078 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep rbd_data.b543344f54ec 2023-10-09T09:56:07.078 INFO:tasks.workunit.client.0.smithi163.stderr:+ get_image_data_pool sparse2 2023-10-09T09:56:07.078 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse2 2023-10-09T09:56:07.078 INFO:tasks.workunit.client.0.smithi163.stderr:+ sed -e s/rbd_data.b543344f54ec\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:56:07.078 INFO:tasks.workunit.client.0.smithi163.stderr:+ sort -u 2023-10-09T09:56:07.078 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse2 2023-10-09T09:56:07.079 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep data_pool: 2023-10-09T09:56:07.079 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk -F: { print $NF } 2023-10-09T09:56:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:07.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:56:07.116 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool= 2023-10-09T09:56:07.117 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ -z ] 2023-10-09T09:56:07.117 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool=rbd 2023-10-09T09:56:07.117 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo rbd 2023-10-09T09:56:07.117 INFO:tasks.workunit.client.0.smithi163.stderr:+ rados ls -p rbd 2023-10-09T09:56:07.151 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects=0 2023-10-09T09:56:07.152 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo 0 2023-10-09T09:56:07.152 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T09:56:07.152 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export sparse2 /tmp/rbd_import_export_237542/sparse2.out 2023-10-09T09:56:07.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:07.189 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T09:56:07.195 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_237542/sparse2 /tmp/rbd_import_export_237542/sparse2.out 2023-10-09T09:56:07.195 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_237542/sparse2 /tmp/rbd_import_export_237542/sparse2.out 2023-10-09T09:56:07.197 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/sparse2 --format %b 2023-10-09T09:56:07.198 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=2048 2023-10-09T09:56:07.198 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/sparse2.out --format %b 2023-10-09T09:56:07.199 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=2048 2023-10-09T09:56:07.199 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T09:56:07.199 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T09:56:07.199 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T09:56:07.199 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/sparse2.out 2023-10-09T09:56:07.200 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse2 2023-10-09T09:56:07.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:07.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:07.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:56:07.308 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T09:56:07.313 INFO:tasks.workunit.client.0.smithi163.stderr:+ truncate /tmp/rbd_import_export_237542/sparse1 -s 10M 2023-10-09T09:56:07.313 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T09:56:07.318 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:07.356 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:07.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:07.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:07.434 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T09:56:07.435 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:56:07.435 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:56:07.439 INFO:tasks.workunit.client.0.smithi163.stderr:+ + rbd ls -l 2023-10-09T09:56:07.440 INFO:tasks.workunit.client.0.smithi163.stderr:grep sparse1 2023-10-09T09:56:07.440 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep -Ei (10 MiB|10240k) 2023-10-09T09:56:07.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:07.475 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:07 smithi181 bash[59345]: cluster 2023-10-09T09:56:06.017031+0000 osd.7 (osd.7) 635 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:07.476 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:07 smithi181 bash[59345]: cluster 2023-10-09T09:56:06.017039+0000 osd.5 (osd.5) 902 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:07.476 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:07 smithi181 bash[59345]: audit 2023-10-09T09:56:06.163418+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.163:0/2617820550' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:56:07.476 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:07 smithi181 bash[59345]: audit 2023-10-09T09:56:06.273054+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.163:0/2617820550' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:56:07.476 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:07 smithi181 bash[59345]: cluster 2023-10-09T09:56:06.295337+0000 mgr.y (mgr.34107) 1510 : cluster 0 pgmap v2410: 73 pgs: 73 active+clean; 802 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 76 KiB/s rd, 8.2 KiB/s wr, 100 op/s 2023-10-09T09:56:07.491 INFO:tasks.workunit.client.0.smithi163.stdout:sparse1 10 MiB 2 2023-10-09T09:56:07.493 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects sparse1 2023-10-09T09:56:07.493 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse1 2023-10-09T09:56:07.494 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse1 2023-10-09T09:56:07.494 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep block_name_prefix 2023-10-09T09:56:07.494 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk {print $NF;} 2023-10-09T09:56:07.496 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:07.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:07.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:07.544 INFO:tasks.workunit.client.0.smithi163.stderr:+ prefix=rbd_data.b570c89b7221 2023-10-09T09:56:07.547 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep rbd_data.b570c89b7221 2023-10-09T09:56:07.547 INFO:tasks.workunit.client.0.smithi163.stderr:+ sed -e s/rbd_data.b570c89b7221\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:56:07.547 INFO:tasks.workunit.client.0.smithi163.stderr:+ get_image_data_pool sparse1 2023-10-09T09:56:07.547 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse1 2023-10-09T09:56:07.547 INFO:tasks.workunit.client.0.smithi163.stderr:+ sort -u 2023-10-09T09:56:07.547 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse1 2023-10-09T09:56:07.547 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep data_pool: 2023-10-09T09:56:07.548 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk -F: { print $NF } 2023-10-09T09:56:07.596 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool= 2023-10-09T09:56:07.597 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ -z ] 2023-10-09T09:56:07.597 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool=rbd 2023-10-09T09:56:07.597 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo rbd 2023-10-09T09:56:07.598 INFO:tasks.workunit.client.0.smithi163.stderr:+ rados ls -p rbd 2023-10-09T09:56:07.601 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:07.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:07.632 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects=1 2023-10-09T09:56:07.632 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo 1 2023-10-09T09:56:07.632 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T09:56:07.633 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export sparse1 /tmp/rbd_import_export_237542/sparse1.out 2023-10-09T09:56:07.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:07.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:07.683 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T09:56:07.687 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_237542/sparse1 /tmp/rbd_import_export_237542/sparse1.out 2023-10-09T09:56:07.687 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_237542/sparse1 /tmp/rbd_import_export_237542/sparse1.out 2023-10-09T09:56:07.700 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/sparse1 --format %b 2023-10-09T09:56:07.701 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=2048 2023-10-09T09:56:07.701 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/sparse1.out --format %b 2023-10-09T09:56:07.702 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=2048 2023-10-09T09:56:07.702 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T09:56:07.702 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T09:56:07.702 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T09:56:07.702 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/sparse1.out 2023-10-09T09:56:07.703 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse1 2023-10-09T09:56:07.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:07 smithi163 bash[75425]: cluster 2023-10-09T09:56:06.017031+0000 osd.7 (osd.7) 635 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:07.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:07 smithi163 bash[75425]: cluster 2023-10-09T09:56:06.017039+0000 osd.5 (osd.5) 902 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:07.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:07 smithi163 bash[75425]: audit 2023-10-09T09:56:06.163418+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.163:0/2617820550' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:56:07.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:07 smithi163 bash[75425]: audit 2023-10-09T09:56:06.273054+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.163:0/2617820550' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:56:07.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:07 smithi163 bash[75425]: cluster 2023-10-09T09:56:06.295337+0000 mgr.y (mgr.34107) 1510 : cluster 0 pgmap v2410: 73 pgs: 73 active+clean; 802 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 76 KiB/s rd, 8.2 KiB/s wr, 100 op/s 2023-10-09T09:56:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:07 smithi163 bash[78107]: cluster 2023-10-09T09:56:06.017031+0000 osd.7 (osd.7) 635 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:07 smithi163 bash[78107]: cluster 2023-10-09T09:56:06.017039+0000 osd.5 (osd.5) 902 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:07 smithi163 bash[78107]: audit 2023-10-09T09:56:06.163418+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.163:0/2617820550' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:56:07.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:07 smithi163 bash[78107]: audit 2023-10-09T09:56:06.273054+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.163:0/2617820550' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:56:07.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:07 smithi163 bash[78107]: cluster 2023-10-09T09:56:06.295337+0000 mgr.y (mgr.34107) 1510 : cluster 0 pgmap v2410: 73 pgs: 73 active+clean; 802 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 76 KiB/s rd, 8.2 KiB/s wr, 100 op/s 2023-10-09T09:56:07.735 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:07.757 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:07.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:07.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:07.846 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T09:56:07.850 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_237542/sparse2 oflag=append conv=notrunc 2023-10-09T09:56:07.860 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:07.862 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records in 2023-10-09T09:56:07.862 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records out 2023-10-09T09:56:07.862 INFO:tasks.workunit.client.0.smithi163.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.011055 s, 190 MB/s 2023-10-09T09:56:07.862 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T09:56:07.890 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:07.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:07.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:07.978 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T09:56:07.980 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:56:07.980 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:56:07.984 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd ls -l 2023-10-09T09:56:07.985 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep sparse2 2023-10-09T09:56:07.985 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep -Ei (4 MiB|4096k) 2023-10-09T09:56:07.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:08.029 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:08.031 INFO:tasks.workunit.client.0.smithi163.stdout:sparse2 4 MiB 2 2023-10-09T09:56:08.032 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects sparse2 2023-10-09T09:56:08.033 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse2 2023-10-09T09:56:08.033 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse2 2023-10-09T09:56:08.033 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep block_name_prefix 2023-10-09T09:56:08.033 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk {print $NF;} 2023-10-09T09:56:08.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:08.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:08.075 INFO:tasks.workunit.client.0.smithi163.stderr:+ prefix=rbd_data.8ec58ab9e3f 2023-10-09T09:56:08.075 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep rbd_data.8ec58ab9e3f 2023-10-09T09:56:08.075 INFO:tasks.workunit.client.0.smithi163.stderr:+ get_image_data_pool sparse2 2023-10-09T09:56:08.076 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse2 2023-10-09T09:56:08.076 INFO:tasks.workunit.client.0.smithi163.stderr:+ sed -e s/rbd_data.8ec58ab9e3f\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:56:08.076 INFO:tasks.workunit.client.0.smithi163.stderr:+ sort -u 2023-10-09T09:56:08.076 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse2 2023-10-09T09:56:08.076 INFO:tasks.workunit.client.0.smithi163.stderr:+ + awk -F: { print $NF } 2023-10-09T09:56:08.076 INFO:tasks.workunit.client.0.smithi163.stderr:grep data_pool: 2023-10-09T09:56:08.116 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool= 2023-10-09T09:56:08.116 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ -z ] 2023-10-09T09:56:08.116 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool=rbd 2023-10-09T09:56:08.116 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo rbd 2023-10-09T09:56:08.117 INFO:tasks.workunit.client.0.smithi163.stderr:+ rados ls -p rbd 2023-10-09T09:56:08.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:08.156 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects=0 2023-10-09T09:56:08.156 INFO:tasks.workunit.client.0.smithi163.stderr:2 2023-10-09T09:56:08.156 INFO:tasks.workunit.client.0.smithi163.stderr:3 2023-10-09T09:56:08.156 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo 0 2 3 2023-10-09T09:56:08.156 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-10-09T09:56:08.156 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export sparse2 /tmp/rbd_import_export_237542/sparse2.out 2023-10-09T09:56:08.159 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:08 smithi163 bash[75425]: cluster 2023-10-09T09:56:06.435963+0000 osd.6 (osd.6) 587 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.159 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:08 smithi163 bash[75425]: cluster 2023-10-09T09:56:06.439079+0000 osd.6 (osd.6) 588 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.159 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:08 smithi163 bash[75425]: cluster 2023-10-09T09:56:06.439284+0000 osd.4 (osd.4) 691 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.159 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:08 smithi163 bash[75425]: cluster 2023-10-09T09:56:06.441887+0000 osd.4 (osd.4) 692 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:08.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:08.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:08.202 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T09:56:08.205 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_237542/sparse2 /tmp/rbd_import_export_237542/sparse2.out 2023-10-09T09:56:08.206 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_237542/sparse2 /tmp/rbd_import_export_237542/sparse2.out 2023-10-09T09:56:08.209 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/sparse2 --format %b 2023-10-09T09:56:08.210 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=6144 2023-10-09T09:56:08.210 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/sparse2.out --format %b 2023-10-09T09:56:08.211 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=6144 2023-10-09T09:56:08.211 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T09:56:08.211 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T09:56:08.212 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T09:56:08.212 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/sparse2.out 2023-10-09T09:56:08.212 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse2 2023-10-09T09:56:08.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:08.300 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:08.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:08.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:56:08.333 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T09:56:08.337 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-10-09T09:56:08.338 INFO:tasks.workunit.client.0.smithi163.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T09:56:08.338 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_237542/sparse1 sparse 2023-10-09T09:56:08.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:08.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:08 smithi163 bash[78107]: cluster 2023-10-09T09:56:06.435963+0000 osd.6 (osd.6) 587 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:08 smithi163 bash[78107]: cluster 2023-10-09T09:56:06.439079+0000 osd.6 (osd.6) 588 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:08 smithi163 bash[78107]: cluster 2023-10-09T09:56:06.439284+0000 osd.4 (osd.4) 691 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:08 smithi163 bash[78107]: cluster 2023-10-09T09:56:06.441887+0000 osd.4 (osd.4) 692 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.417 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T09:56:08.418 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:56:08.418 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T09:56:08.424 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects sparse 2023-10-09T09:56:08.424 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse 2023-10-09T09:56:08.424 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse 2023-10-09T09:56:08.424 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep block_name_prefix 2023-10-09T09:56:08.424 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk {print $NF;} 2023-10-09T09:56:08.431 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2407,"stamp":"2023-10-09T09:56:04.294354+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":821378,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":3947,"num_read_kb":5612,"num_write":769,"num_write_kb":3775,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":592828,"num_keys_recovered":10,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":773,"ondisk_log_size":773,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4778816,"kb_used_data":40784,"kb_used_omap":607,"kb_used_meta":4737376,"kb_avail":557692096,"statfs":{"total":575970213888,"available":571076706304,"internally_reserved":0,"allocated":41762816,"data_stored":35887064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":621657,"internal_metadata":4851073959},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":219293,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":1241,"num_read_kb":1039,"num_write":169,"num_write_kb":133,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000967"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":86,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155498+0000","last_change":"2023-10-09T09:48:07.991289+0000","last_active":"2023-10-09T09:56:00.155498+0000","last_peered":"2023-10-09T09:56:00.155498+0000","last_clean":"2023-10-09T09:56:00.155498+0000","last_became_active":"2023-10-09T09:48:07.990686+0000","last_became_peered":"2023-10-09T09:48:07.990686+0000","last_unstale":"2023-10-09T09:56:00.155498+0000","last_undegraded":"2023-10-09T09:56:00.155498+0000","last_fullsized":"2023-10-09T09:56:00.155498+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:24.421374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":189,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155299+0000","last_change":"2023-10-09T09:48:07.999947+0000","last_active":"2023-10-09T09:56:00.155299+0000","last_peered":"2023-10-09T09:56:00.155299+0000","last_clean":"2023-10-09T09:56:00.155299+0000","last_became_active":"2023-10-09T09:48:07.999704+0000","last_became_peered":"2023-10-09T09:48:07.999704+0000","last_unstale":"2023-10-09T09:56:00.155299+0000","last_undegraded":"2023-10-09T09:56:00.155299+0000","last_fullsized":"2023-10-09T09:56:00.155299+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:14.832042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1258,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.154612+0000","last_change":"2023-10-09T09:48:07.998214+0000","last_active":"2023-10-09T09:56:00.154612+0000","last_peered":"2023-10-09T09:56:00.154612+0000","last_clean":"2023-10-09T09:56:00.154612+0000","last_became_active":"2023-10-09T09:48:07.997544+0000","last_became_peered":"2023-10-09T09:48:07.997544+0000","last_unstale":"2023-10-09T09:56:00.154612+0000","last_undegraded":"2023-10-09T09:56:00.154612+0000","last_fullsized":"2023-10-09T09:56:00.154612+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.304000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1271,"reported_epoch":1308,"state":"active+clean","last_fresh":"2023-10-09T09:56:03.183593+0000","last_change":"2023-10-09T09:48:07.980358+0000","last_active":"2023-10-09T09:56:03.183593+0000","last_peered":"2023-10-09T09:56:03.183593+0000","last_clean":"2023-10-09T09:56:03.183593+0000","last_became_active":"2023-10-09T09:48:07.979229+0000","last_became_peered":"2023-10-09T09:48:07.979229+0000","last_unstale":"2023-10-09T09:56:03.183593+0000","last_undegraded":"2023-10-09T09:56:03.183593+0000","last_fullsized":"2023-10-09T09:56:03.183593+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:31.456347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":804,"reported_epoch":1308,"state":"active+clean","last_fresh":"2023-10-09T09:56:03.182956+0000","last_change":"2023-10-09T09:48:07.969728+0000","last_active":"2023-10-09T09:56:03.182956+0000","last_peered":"2023-10-09T09:56:03.182956+0000","last_clean":"2023-10-09T09:56:03.182956+0000","last_became_active":"2023-10-09T09:48:07.968773+0000","last_became_peered":"2023-10-09T09:48:07.968773+0000","last_unstale":"2023-10-09T09:56:03.182956+0000","last_undegraded":"2023-10-09T09:56:03.182956+0000","last_fullsized":"2023-10-09T09:56:03.182956+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:39.023317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1271,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.034439+0000","last_change":"2023-10-09T09:48:07.988928+0000","last_active":"2023-10-09T09:56:01.034439+0000","last_peered":"2023-10-09T09:56:01.034439+0000","last_clean":"2023-10-09T09:56:01.034439+0000","last_became_active":"2023-10-09T09:48:07.988051+0000","last_became_peered":"2023-10-09T09:48:07.988051+0000","last_unstale":"2023-10-09T09:56:01.034439+0000","last_undegraded":"2023-10-09T09:56:01.034439+0000","last_fullsized":"2023-10-09T09:56:01.034439+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1229,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.102831+0000","last_change":"2023-10-09T09:43:48.902066+0000","last_active":"2023-10-09T09:56:00.102831+0000","last_peered":"2023-10-09T09:56:00.102831+0000","last_clean":"2023-10-09T09:56:00.102831+0000","last_became_active":"2023-10-09T09:43:48.901639+0000","last_became_peered":"2023-10-09T09:43:48.901639+0000","last_unstale":"2023-10-09T09:56:00.102831+0000","last_undegraded":"2023-10-09T09:56:00.102831+0000","last_fullsized":"2023-10-09T09:56:00.102831+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":700,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155060+0000","last_change":"2023-10-09T09:48:07.997787+0000","last_active":"2023-10-09T09:56:00.155060+0000","last_peered":"2023-10-09T09:56:00.155060+0000","last_clean":"2023-10-09T09:56:00.155060+0000","last_became_active":"2023-10-09T09:48:07.997356+0000","last_became_peered":"2023-10-09T09:48:07.997356+0000","last_unstale":"2023-10-09T09:56:00.155060+0000","last_undegraded":"2023-10-09T09:56:00.155060+0000","last_fullsized":"2023-10-09T09:56:00.155060+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:03.024096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":172,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.102549+0000","last_change":"2023-10-09T09:48:07.975547+0000","last_active":"2023-10-09T09:56:00.102549+0000","last_peered":"2023-10-09T09:56:00.102549+0000","last_clean":"2023-10-09T09:56:00.102549+0000","last_became_active":"2023-10-09T09:48:07.975150+0000","last_became_peered":"2023-10-09T09:48:07.975150+0000","last_unstale":"2023-10-09T09:56:00.102549+0000","last_undegraded":"2023-10-09T09:56:00.102549+0000","last_fullsized":"2023-10-09T09:56:00.102549+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":722,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.033825+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:56:01.033825+0000","last_peered":"2023-10-09T09:56:01.033825+0000","last_clean":"2023-10-09T09:56:01.033825+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:56:01.033825+0000","last_undegraded":"2023-10-09T09:56:01.033825+0000","last_fullsized":"2023-10-09T09:56:01.033825+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":115,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.034215+0000","last_change":"2023-10-09T09:43:48.906785+0000","last_active":"2023-10-09T09:56:01.034215+0000","last_peered":"2023-10-09T09:56:01.034215+0000","last_clean":"2023-10-09T09:56:01.034215+0000","last_became_active":"2023-10-09T09:43:48.906276+0000","last_became_peered":"2023-10-09T09:43:48.906276+0000","last_unstale":"2023-10-09T09:56:01.034215+0000","last_undegraded":"2023-10-09T09:56:01.034215+0000","last_fullsized":"2023-10-09T09:56:01.034215+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":138,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.102741+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:56:00.102741+0000","last_peered":"2023-10-09T09:56:00.102741+0000","last_clean":"2023-10-09T09:56:00.102741+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:56:00.102741+0000","last_undegraded":"2023-10-09T09:56:00.102741+0000","last_fullsized":"2023-10-09T09:56:00.102741+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":463,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.033941+0000","last_change":"2023-10-09T09:43:23.995526+0000","last_active":"2023-10-09T09:56:01.033941+0000","last_peered":"2023-10-09T09:56:01.033941+0000","last_clean":"2023-10-09T09:56:01.033941+0000","last_became_active":"2023-10-09T09:43:23.994816+0000","last_became_peered":"2023-10-09T09:43:23.994816+0000","last_unstale":"2023-10-09T09:56:01.033941+0000","last_undegraded":"2023-10-09T09:56:01.033941+0000","last_fullsized":"2023-10-09T09:56:01.033941+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":680,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.033864+0000","last_change":"2023-10-09T09:48:07.987637+0000","last_active":"2023-10-09T09:56:01.033864+0000","last_peered":"2023-10-09T09:56:01.033864+0000","last_clean":"2023-10-09T09:56:01.033864+0000","last_became_active":"2023-10-09T09:48:07.987085+0000","last_became_peered":"2023-10-09T09:48:07.987085+0000","last_unstale":"2023-10-09T09:56:01.033864+0000","last_undegraded":"2023-10-09T09:56:01.033864+0000","last_fullsized":"2023-10-09T09:56:01.033864+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1203,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.710922+0000","last_change":"2023-10-09T09:43:23.989356+0000","last_active":"2023-10-09T09:56:00.710922+0000","last_peered":"2023-10-09T09:56:00.710922+0000","last_clean":"2023-10-09T09:56:00.710922+0000","last_became_active":"2023-10-09T09:43:23.988986+0000","last_became_peered":"2023-10-09T09:43:23.988986+0000","last_unstale":"2023-10-09T09:56:00.710922+0000","last_undegraded":"2023-10-09T09:56:00.710922+0000","last_fullsized":"2023-10-09T09:56:00.710922+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.570448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":785,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155517+0000","last_change":"2023-10-09T09:48:07.984006+0000","last_active":"2023-10-09T09:56:00.155517+0000","last_peered":"2023-10-09T09:56:00.155517+0000","last_clean":"2023-10-09T09:56:00.155517+0000","last_became_active":"2023-10-09T09:48:07.978179+0000","last_became_peered":"2023-10-09T09:48:07.978179+0000","last_unstale":"2023-10-09T09:56:00.155517+0000","last_undegraded":"2023-10-09T09:56:00.155517+0000","last_fullsized":"2023-10-09T09:56:00.155517+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:34.164004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1206,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155523+0000","last_change":"2023-10-09T09:48:07.975335+0000","last_active":"2023-10-09T09:56:00.155523+0000","last_peered":"2023-10-09T09:56:00.155523+0000","last_clean":"2023-10-09T09:56:00.155523+0000","last_became_active":"2023-10-09T09:48:07.974437+0000","last_became_peered":"2023-10-09T09:48:07.974437+0000","last_unstale":"2023-10-09T09:56:00.155523+0000","last_undegraded":"2023-10-09T09:56:00.155523+0000","last_fullsized":"2023-10-09T09:56:00.155523+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:32.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":143,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.034358+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:56:01.034358+0000","last_peered":"2023-10-09T09:56:01.034358+0000","last_clean":"2023-10-09T09:56:01.034358+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:56:01.034358+0000","last_undegraded":"2023-10-09T09:56:01.034358+0000","last_fullsized":"2023-10-09T09:56:01.034358+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.033615+0000","last_change":"2023-10-09T09:43:23.994784+0000","last_active":"2023-10-09T09:56:01.033615+0000","last_peered":"2023-10-09T09:56:01.033615+0000","last_clean":"2023-10-09T09:56:01.033615+0000","last_became_active":"2023-10-09T09:43:23.994196+0000","last_became_peered":"2023-10-09T09:43:23.994196+0000","last_unstale":"2023-10-09T09:56:01.033615+0000","last_undegraded":"2023-10-09T09:56:01.033615+0000","last_fullsized":"2023-10-09T09:56:01.033615+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1224,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.711549+0000","last_change":"2023-10-09T09:43:48.901303+0000","last_active":"2023-10-09T09:56:00.711549+0000","last_peered":"2023-10-09T09:56:00.711549+0000","last_clean":"2023-10-09T09:56:00.711549+0000","last_became_active":"2023-10-09T09:43:48.900779+0000","last_became_peered":"2023-10-09T09:43:48.900779+0000","last_unstale":"2023-10-09T09:56:00.711549+0000","last_undegraded":"2023-10-09T09:56:00.711549+0000","last_fullsized":"2023-10-09T09:56:00.711549+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:51.853531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1301,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.033396+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:56:01.033396+0000","last_peered":"2023-10-09T09:56:01.033396+0000","last_clean":"2023-10-09T09:56:01.033396+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:56:01.033396+0000","last_undegraded":"2023-10-09T09:56:01.033396+0000","last_fullsized":"2023-10-09T09:56:01.033396+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1268,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155775+0000","last_change":"2023-10-09T09:48:08.000694+0000","last_active":"2023-10-09T09:56:00.155775+0000","last_peered":"2023-10-09T09:56:00.155775+0000","last_clean":"2023-10-09T09:56:00.155775+0000","last_became_active":"2023-10-09T09:48:07.999476+0000","last_became_peered":"2023-10-09T09:48:07.999476+0000","last_unstale":"2023-10-09T09:56:00.155775+0000","last_undegraded":"2023-10-09T09:56:00.155775+0000","last_fullsized":"2023-10-09T09:56:00.155775+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:37.211076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":137,"reported_epoch":1308,"state":"active+clean","last_fresh":"2023-10-09T09:56:03.183769+0000","last_change":"2023-10-09T09:48:07.976558+0000","last_active":"2023-10-09T09:56:03.183769+0000","last_peered":"2023-10-09T09:56:03.183769+0000","last_clean":"2023-10-09T09:56:03.183769+0000","last_became_active":"2023-10-09T09:48:07.975961+0000","last_became_peered":"2023-10-09T09:48:07.975961+0000","last_unstale":"2023-10-09T09:56:03.183769+0000","last_undegraded":"2023-10-09T09:56:03.183769+0000","last_fullsized":"2023-10-09T09:56:03.183769+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:40.925248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":148,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.101942+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:56:00.101942+0000","last_peered":"2023-10-09T09:56:00.101942+0000","last_clean":"2023-10-09T09:56:00.101942+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:56:00.101942+0000","last_undegraded":"2023-10-09T09:56:00.101942+0000","last_fullsized":"2023-10-09T09:56:00.101942+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":97,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155368+0000","last_change":"2023-10-09T09:48:07.990711+0000","last_active":"2023-10-09T09:56:00.155368+0000","last_peered":"2023-10-09T09:56:00.155368+0000","last_clean":"2023-10-09T09:56:00.155368+0000","last_became_active":"2023-10-09T09:48:07.990267+0000","last_became_peered":"2023-10-09T09:48:07.990267+0000","last_unstale":"2023-10-09T09:56:00.155368+0000","last_undegraded":"2023-10-09T09:56:00.155368+0000","last_fullsized":"2023-10-09T09:56:00.155368+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:51.244840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1241,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.154731+0000","last_change":"2023-10-09T09:48:07.991440+0000","last_active":"2023-10-09T09:56:00.154731+0000","last_peered":"2023-10-09T09:56:00.154731+0000","last_clean":"2023-10-09T09:56:00.154731+0000","last_became_active":"2023-10-09T09:48:07.991072+0000","last_became_peered":"2023-10-09T09:48:07.991072+0000","last_unstale":"2023-10-09T09:56:00.154731+0000","last_undegraded":"2023-10-09T09:56:00.154731+0000","last_fullsized":"2023-10-09T09:56:00.154731+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:28.577159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1256,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.102579+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:56:00.102579+0000","last_peered":"2023-10-09T09:56:00.102579+0000","last_clean":"2023-10-09T09:56:00.102579+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:56:00.102579+0000","last_undegraded":"2023-10-09T09:56:00.102579+0000","last_fullsized":"2023-10-09T09:56:00.102579+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1263,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.033757+0000","last_change":"2023-10-09T09:48:07.987819+0000","last_active":"2023-10-09T09:56:01.033757+0000","last_peered":"2023-10-09T09:56:01.033757+0000","last_clean":"2023-10-09T09:56:01.033757+0000","last_became_active":"2023-10-09T09:48:07.986354+0000","last_became_peered":"2023-10-09T09:48:07.986354+0000","last_unstale":"2023-10-09T09:56:01.033757+0000","last_undegraded":"2023-10-09T09:56:01.033757+0000","last_fullsized":"2023-10-09T09:56:01.033757+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":809,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.034153+0000","last_change":"2023-10-09T09:48:07.987506+0000","last_active":"2023-10-09T09:56:01.034153+0000","last_peered":"2023-10-09T09:56:01.034153+0000","last_clean":"2023-10-09T09:56:01.034153+0000","last_became_active":"2023-10-09T09:48:07.986988+0000","last_became_peered":"2023-10-09T09:48:07.986988+0000","last_unstale":"2023-10-09T09:56:01.034153+0000","last_undegraded":"2023-10-09T09:56:01.034153+0000","last_fullsized":"2023-10-09T09:56:01.034153+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1230,"reported_epoch":1308,"state":"active+clean","last_fresh":"2023-10-09T09:56:03.183323+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:56:03.183323+0000","last_peered":"2023-10-09T09:56:03.183323+0000","last_clean":"2023-10-09T09:56:03.183323+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:56:03.183323+0000","last_undegraded":"2023-10-09T09:56:03.183323+0000","last_fullsized":"2023-10-09T09:56:03.183323+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":185,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155041+0000","last_change":"2023-10-09T09:48:07.977143+0000","last_active":"2023-10-09T09:56:00.155041+0000","last_peered":"2023-10-09T09:56:00.155041+0000","last_clean":"2023-10-09T09:56:00.155041+0000","last_became_active":"2023-10-09T09:48:07.976338+0000","last_became_peered":"2023-10-09T09:48:07.976338+0000","last_unstale":"2023-10-09T09:56:00.155041+0000","last_undegraded":"2023-10-09T09:56:00.155041+0000","last_fullsized":"2023-10-09T09:56:00.155041+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:12.015053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1259,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.034103+0000","last_change":"2023-10-09T09:48:08.001503+0000","last_active":"2023-10-09T09:56:01.034103+0000","last_peered":"2023-10-09T09:56:01.034103+0000","last_clean":"2023-10-09T09:56:01.034103+0000","last_became_active":"2023-10-09T09:48:08.000921+0000","last_became_peered":"2023-10-09T09:48:08.000921+0000","last_unstale":"2023-10-09T09:56:01.034103+0000","last_undegraded":"2023-10-09T09:56:01.034103+0000","last_fullsized":"2023-10-09T09:56:01.034103+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1255,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.033564+0000","last_change":"2023-10-09T09:48:07.988586+0000","last_active":"2023-10-09T09:56:01.033564+0000","last_peered":"2023-10-09T09:56:01.033564+0000","last_clean":"2023-10-09T09:56:01.033564+0000","last_became_active":"2023-10-09T09:48:07.988226+0000","last_became_peered":"2023-10-09T09:48:07.988226+0000","last_unstale":"2023-10-09T09:56:01.033564+0000","last_undegraded":"2023-10-09T09:56:01.033564+0000","last_fullsized":"2023-10-09T09:56:01.033564+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1247,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.711271+0000","last_change":"2023-10-09T09:48:07.987993+0000","last_active":"2023-10-09T09:56:00.711271+0000","last_peered":"2023-10-09T09:56:00.711271+0000","last_clean":"2023-10-09T09:56:00.711271+0000","last_became_active":"2023-10-09T09:48:07.987664+0000","last_became_peered":"2023-10-09T09:48:07.987664+0000","last_unstale":"2023-10-09T09:56:00.711271+0000","last_undegraded":"2023-10-09T09:56:00.711271+0000","last_fullsized":"2023-10-09T09:56:00.711271+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:32.253321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1275,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.710861+0000","last_change":"2023-10-09T09:48:07.984079+0000","last_active":"2023-10-09T09:56:00.710861+0000","last_peered":"2023-10-09T09:56:00.710861+0000","last_clean":"2023-10-09T09:56:00.710861+0000","last_became_active":"2023-10-09T09:48:07.983792+0000","last_became_peered":"2023-10-09T09:48:07.983792+0000","last_unstale":"2023-10-09T09:56:00.710861+0000","last_undegraded":"2023-10-09T09:56:00.710861+0000","last_fullsized":"2023-10-09T09:56:00.710861+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:28:42.289832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":680,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.034284+0000","last_change":"2023-10-09T09:48:07.989047+0000","last_active":"2023-10-09T09:56:01.034284+0000","last_peered":"2023-10-09T09:56:01.034284+0000","last_clean":"2023-10-09T09:56:01.034284+0000","last_became_active":"2023-10-09T09:48:07.988131+0000","last_became_peered":"2023-10-09T09:48:07.988131+0000","last_unstale":"2023-10-09T09:56:01.034284+0000","last_undegraded":"2023-10-09T09:56:01.034284+0000","last_fullsized":"2023-10-09T09:56:01.034284+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1217,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.154373+0000","last_change":"2023-10-09T09:48:07.976884+0000","last_active":"2023-10-09T09:56:00.154373+0000","last_peered":"2023-10-09T09:56:00.154373+0000","last_clean":"2023-10-09T09:56:00.154373+0000","last_became_active":"2023-10-09T09:48:07.975932+0000","last_became_peered":"2023-10-09T09:48:07.975932+0000","last_unstale":"2023-10-09T09:56:00.154373+0000","last_undegraded":"2023-10-09T09:56:00.154373+0000","last_fullsized":"2023-10-09T09:56:00.154373+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:22.785728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":153,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.711628+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:56:00.711628+0000","last_peered":"2023-10-09T09:56:00.711628+0000","last_clean":"2023-10-09T09:56:00.711628+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:56:00.711628+0000","last_undegraded":"2023-10-09T09:56:00.711628+0000","last_fullsized":"2023-10-09T09:56:00.711628+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"1304'127","reported_seq":1735,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.026346+0000","last_change":"2023-10-09T09:55:55.079046+0000","last_active":"2023-10-09T09:56:01.026346+0000","last_peered":"2023-10-09T09:56:01.026346+0000","last_clean":"2023-10-09T09:56:01.026346+0000","last_became_active":"2023-10-09T09:48:07.983302+0000","last_became_peered":"2023-10-09T09:48:07.983302+0000","last_unstale":"2023-10-09T09:56:01.026346+0000","last_undegraded":"2023-10-09T09:56:01.026346+0000","last_fullsized":"2023-10-09T09:56:01.026346+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:50.714295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0041257339999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":860,"num_read_kb":907,"num_write":121,"num_write_kb":121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"1"}]},{"pgid":"3.5","version":"0'0","reported_seq":148,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.102649+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:56:00.102649+0000","last_peered":"2023-10-09T09:56:00.102649+0000","last_clean":"2023-10-09T09:56:00.102649+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:56:00.102649+0000","last_undegraded":"2023-10-09T09:56:00.102649+0000","last_fullsized":"2023-10-09T09:56:00.102649+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1240,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155381+0000","last_change":"2023-10-09T09:48:07.977683+0000","last_active":"2023-10-09T09:56:00.155381+0000","last_peered":"2023-10-09T09:56:00.155381+0000","last_clean":"2023-10-09T09:56:00.155381+0000","last_became_active":"2023-10-09T09:48:07.977024+0000","last_became_peered":"2023-10-09T09:48:07.977024+0000","last_unstale":"2023-10-09T09:56:00.155381+0000","last_undegraded":"2023-10-09T09:56:00.155381+0000","last_fullsized":"2023-10-09T09:56:00.155381+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:13.438431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"1306'100","reported_seq":1215,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.290430+0000","last_change":"2023-10-09T09:55:55.076480+0000","last_active":"2023-10-09T09:56:01.290430+0000","last_peered":"2023-10-09T09:56:01.290430+0000","last_clean":"2023-10-09T09:56:01.290430+0000","last_became_active":"2023-10-09T09:48:07.997389+0000","last_became_peered":"2023-10-09T09:48:07.997389+0000","last_unstale":"2023-10-09T09:56:01.290430+0000","last_undegraded":"2023-10-09T09:56:01.290430+0000","last_fullsized":"2023-10-09T09:56:01.290430+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:21.978398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001873291,"stat_sum":{"num_bytes":159,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":758,"num_read_kb":1008,"num_write":74,"num_write_kb":213,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"1"}]},{"pgid":"3.3","version":"0'0","reported_seq":1197,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.034109+0000","last_change":"2023-10-09T09:43:48.908190+0000","last_active":"2023-10-09T09:56:01.034109+0000","last_peered":"2023-10-09T09:56:01.034109+0000","last_clean":"2023-10-09T09:56:01.034109+0000","last_became_active":"2023-10-09T09:43:48.907799+0000","last_became_peered":"2023-10-09T09:43:48.907799+0000","last_unstale":"2023-10-09T09:56:01.034109+0000","last_undegraded":"2023-10-09T09:56:01.034109+0000","last_fullsized":"2023-10-09T09:56:01.034109+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1293,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.154758+0000","last_change":"2023-10-09T09:48:08.000154+0000","last_active":"2023-10-09T09:56:00.154758+0000","last_peered":"2023-10-09T09:56:00.154758+0000","last_clean":"2023-10-09T09:56:00.154758+0000","last_became_active":"2023-10-09T09:48:07.999714+0000","last_became_peered":"2023-10-09T09:48:07.999714+0000","last_unstale":"2023-10-09T09:56:00.154758+0000","last_undegraded":"2023-10-09T09:56:00.154758+0000","last_fullsized":"2023-10-09T09:56:00.154758+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:12.921693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"1306'95","reported_seq":618,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.028905+0000","last_change":"2023-10-09T09:55:55.081323+0000","last_active":"2023-10-09T09:56:01.028905+0000","last_peered":"2023-10-09T09:56:01.028905+0000","last_clean":"2023-10-09T09:56:01.028905+0000","last_became_active":"2023-10-09T09:48:07.983490+0000","last_became_peered":"2023-10-09T09:48:07.983490+0000","last_unstale":"2023-10-09T09:56:01.028905+0000","last_undegraded":"2023-10-09T09:56:01.028905+0000","last_fullsized":"2023-10-09T09:56:01.028905+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":95,"log_dups_size":0,"ondisk_log_size":95,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:21.122987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.003859506,"stat_sum":{"num_bytes":51,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":759,"num_read_kb":1006,"num_write":71,"num_write_kb":237,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"1"}]},{"pgid":"3.0","version":"0'0","reported_seq":1271,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155181+0000","last_change":"2023-10-09T09:48:07.998234+0000","last_active":"2023-10-09T09:56:00.155181+0000","last_peered":"2023-10-09T09:56:00.155181+0000","last_clean":"2023-10-09T09:56:00.155181+0000","last_became_active":"2023-10-09T09:48:07.997909+0000","last_became_peered":"2023-10-09T09:48:07.997909+0000","last_unstale":"2023-10-09T09:56:00.155181+0000","last_undegraded":"2023-10-09T09:56:00.155181+0000","last_fullsized":"2023-10-09T09:56:00.155181+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:21.750617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1260,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.154732+0000","last_change":"2023-10-09T09:48:07.973284+0000","last_active":"2023-10-09T09:56:00.154732+0000","last_peered":"2023-10-09T09:56:00.154732+0000","last_clean":"2023-10-09T09:56:00.154732+0000","last_became_active":"2023-10-09T09:48:07.971235+0000","last_became_peered":"2023-10-09T09:48:07.971235+0000","last_unstale":"2023-10-09T09:56:00.154732+0000","last_undegraded":"2023-10-09T09:56:00.154732+0000","last_fullsized":"2023-10-09T09:56:00.154732+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:40.951420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"1302'79","reported_seq":1526,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.017840+0000","last_change":"2023-10-09T09:55:55.082299+0000","last_active":"2023-10-09T09:56:01.017840+0000","last_peered":"2023-10-09T09:56:01.017840+0000","last_clean":"2023-10-09T09:56:01.017840+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:56:01.017840+0000","last_undegraded":"2023-10-09T09:56:01.017840+0000","last_fullsized":"2023-10-09T09:56:01.017840+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0044504389999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":270,"num_read_kb":236,"num_write":73,"num_write_kb":43,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"1"}]},{"pgid":"1.0","version":"304'139","reported_seq":653,"reported_epoch":1308,"state":"active+clean","last_fresh":"2023-10-09T09:56:03.183003+0000","last_change":"2023-10-09T09:48:07.978435+0000","last_active":"2023-10-09T09:56:03.183003+0000","last_peered":"2023-10-09T09:56:03.183003+0000","last_clean":"2023-10-09T09:56:03.183003+0000","last_became_active":"2023-10-09T09:48:07.977417+0000","last_became_peered":"2023-10-09T09:48:07.977417+0000","last_unstale":"2023-10-09T09:56:03.183003+0000","last_undegraded":"2023-10-09T09:56:03.183003+0000","last_fullsized":"2023-10-09T09:56:03.183003+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:19.288202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":148,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.102683+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:56:00.102683+0000","last_peered":"2023-10-09T09:56:00.102683+0000","last_clean":"2023-10-09T09:56:00.102683+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:56:00.102683+0000","last_undegraded":"2023-10-09T09:56:00.102683+0000","last_fullsized":"2023-10-09T09:56:00.102683+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1233,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.034359+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:56:01.034359+0000","last_peered":"2023-10-09T09:56:01.034359+0000","last_clean":"2023-10-09T09:56:01.034359+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:56:01.034359+0000","last_undegraded":"2023-10-09T09:56:01.034359+0000","last_fullsized":"2023-10-09T09:56:01.034359+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"1302'23","reported_seq":1284,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.025211+0000","last_change":"2023-10-09T09:55:55.076073+0000","last_active":"2023-10-09T09:56:01.025211+0000","last_peered":"2023-10-09T09:56:01.025211+0000","last_clean":"2023-10-09T09:56:01.025211+0000","last_became_active":"2023-10-09T09:48:07.971221+0000","last_became_peered":"2023-10-09T09:48:07.971221+0000","last_unstale":"2023-10-09T09:56:01.025211+0000","last_undegraded":"2023-10-09T09:56:01.025211+0000","last_fullsized":"2023-10-09T09:56:01.025211+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:16.759514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018864699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":56,"num_read_kb":338,"num_write":27,"num_write_kb":166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"1"}]},{"pgid":"3.1","version":"0'0","reported_seq":1247,"reported_epoch":1308,"state":"active+clean","last_fresh":"2023-10-09T09:56:03.183189+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:56:03.183189+0000","last_peered":"2023-10-09T09:56:03.183189+0000","last_clean":"2023-10-09T09:56:03.183189+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:56:03.183189+0000","last_undegraded":"2023-10-09T09:56:03.183189+0000","last_fullsized":"2023-10-09T09:56:03.183189+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1259,"reported_epoch":1308,"state":"active+clean","last_fresh":"2023-10-09T09:56:03.183592+0000","last_change":"2023-10-09T09:48:07.981410+0000","last_active":"2023-10-09T09:56:03.183592+0000","last_peered":"2023-10-09T09:56:03.183592+0000","last_clean":"2023-10-09T09:56:03.183592+0000","last_became_active":"2023-10-09T09:48:07.979850+0000","last_became_peered":"2023-10-09T09:48:07.979850+0000","last_unstale":"2023-10-09T09:56:03.183592+0000","last_undegraded":"2023-10-09T09:56:03.183592+0000","last_fullsized":"2023-10-09T09:56:03.183592+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:31.201104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"1309'79","reported_seq":1587,"reported_epoch":1309,"state":"active+clean","last_fresh":"2023-10-09T09:56:04.151067+0000","last_change":"2023-10-09T09:56:04.143332+0000","last_active":"2023-10-09T09:56:04.151067+0000","last_peered":"2023-10-09T09:56:04.151067+0000","last_clean":"2023-10-09T09:56:04.151067+0000","last_became_active":"2023-10-09T09:48:07.975338+0000","last_became_peered":"2023-10-09T09:48:07.975338+0000","last_unstale":"2023-10-09T09:56:04.151067+0000","last_undegraded":"2023-10-09T09:56:04.151067+0000","last_fullsized":"2023-10-09T09:56:04.151067+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:05.175956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0048194639999999999,"stat_sum":{"num_bytes":112640,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":739,"num_read_kb":1293,"num_write":66,"num_write_kb":390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"4","length":"1"},{"start":"8","length":"3"}]},{"pgid":"3.4","version":"0'0","reported_seq":1274,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.154550+0000","last_change":"2023-10-09T09:48:07.983773+0000","last_active":"2023-10-09T09:56:00.154550+0000","last_peered":"2023-10-09T09:56:00.154550+0000","last_clean":"2023-10-09T09:56:00.154550+0000","last_became_active":"2023-10-09T09:48:07.976869+0000","last_became_peered":"2023-10-09T09:48:07.976869+0000","last_unstale":"2023-10-09T09:56:00.154550+0000","last_undegraded":"2023-10-09T09:56:00.154550+0000","last_fullsized":"2023-10-09T09:56:00.154550+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:58.173236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1222,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.102544+0000","last_change":"2023-10-09T09:48:07.962302+0000","last_active":"2023-10-09T09:56:00.102544+0000","last_peered":"2023-10-09T09:56:00.102544+0000","last_clean":"2023-10-09T09:56:00.102544+0000","last_became_active":"2023-10-09T09:48:07.961888+0000","last_became_peered":"2023-10-09T09:48:07.961888+0000","last_unstale":"2023-10-09T09:56:00.102544+0000","last_undegraded":"2023-10-09T09:56:00.102544+0000","last_fullsized":"2023-10-09T09:56:00.102544+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":173,"reported_epoch":1309,"state":"active+clean","last_fresh":"2023-10-09T09:56:04.137896+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:56:04.137896+0000","last_peered":"2023-10-09T09:56:04.137896+0000","last_clean":"2023-10-09T09:56:04.137896+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:56:04.137896+0000","last_undegraded":"2023-10-09T09:56:04.137896+0000","last_fullsized":"2023-10-09T09:56:04.137896+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"1304'26","reported_seq":1306,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.225144+0000","last_change":"2023-10-09T09:55:55.078537+0000","last_active":"2023-10-09T09:56:01.225144+0000","last_peered":"2023-10-09T09:56:01.225144+0000","last_clean":"2023-10-09T09:56:01.225144+0000","last_became_active":"2023-10-09T09:48:07.998606+0000","last_became_peered":"2023-10-09T09:48:07.998606+0000","last_unstale":"2023-10-09T09:56:01.225144+0000","last_undegraded":"2023-10-09T09:56:01.225144+0000","last_fullsized":"2023-10-09T09:56:01.225144+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.469476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0033396329999999998,"stat_sum":{"num_bytes":16,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":68,"num_read_kb":32,"num_write":15,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"1"}]},{"pgid":"4.0","version":"0'0","reported_seq":1162,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.154724+0000","last_change":"2023-10-09T09:48:07.976350+0000","last_active":"2023-10-09T09:56:00.154724+0000","last_peered":"2023-10-09T09:56:00.154724+0000","last_clean":"2023-10-09T09:56:00.154724+0000","last_became_active":"2023-10-09T09:48:07.973099+0000","last_became_peered":"2023-10-09T09:48:07.973099+0000","last_unstale":"2023-10-09T09:56:00.154724+0000","last_undegraded":"2023-10-09T09:56:00.154724+0000","last_fullsized":"2023-10-09T09:56:00.154724+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:48.812722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"1302'50","reported_seq":1380,"reported_epoch":1307,"state":"active+clean","last_fresh":"2023-10-09T09:56:02.130052+0000","last_change":"2023-10-09T09:56:02.131124+0000","last_active":"2023-10-09T09:56:02.130052+0000","last_peered":"2023-10-09T09:56:02.130052+0000","last_clean":"2023-10-09T09:56:02.130052+0000","last_became_active":"2023-10-09T09:48:07.987124+0000","last_became_peered":"2023-10-09T09:48:07.987124+0000","last_unstale":"2023-10-09T09:56:02.130052+0000","last_undegraded":"2023-10-09T09:56:02.130052+0000","last_fullsized":"2023-10-09T09:56:02.130052+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112640,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":143,"num_read_kb":292,"num_write":62,"num_write_kb":139,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"4","length":"1"},{"start":"8","length":"1"}]},{"pgid":"3.6","version":"61'1","reported_seq":1240,"reported_epoch":1308,"state":"active+clean","last_fresh":"2023-10-09T09:56:03.183726+0000","last_change":"2023-10-09T09:48:07.980600+0000","last_active":"2023-10-09T09:56:03.183726+0000","last_peered":"2023-10-09T09:56:03.183726+0000","last_clean":"2023-10-09T09:56:03.183726+0000","last_became_active":"2023-10-09T09:48:07.979544+0000","last_became_peered":"2023-10-09T09:48:07.979544+0000","last_unstale":"2023-10-09T09:56:03.183726+0000","last_undegraded":"2023-10-09T09:56:03.183726+0000","last_fullsized":"2023-10-09T09:56:03.183726+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:47.320689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1224,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.102096+0000","last_change":"2023-10-09T09:43:23.992213+0000","last_active":"2023-10-09T09:56:00.102096+0000","last_peered":"2023-10-09T09:56:00.102096+0000","last_clean":"2023-10-09T09:56:00.102096+0000","last_became_active":"2023-10-09T09:43:23.991887+0000","last_became_peered":"2023-10-09T09:43:23.991887+0000","last_unstale":"2023-10-09T09:56:00.102096+0000","last_undegraded":"2023-10-09T09:56:00.102096+0000","last_fullsized":"2023-10-09T09:56:00.102096+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1248,"reported_epoch":1308,"state":"active+clean","last_fresh":"2023-10-09T09:56:03.183425+0000","last_change":"2023-10-09T09:48:07.969929+0000","last_active":"2023-10-09T09:56:03.183425+0000","last_peered":"2023-10-09T09:56:03.183425+0000","last_clean":"2023-10-09T09:56:03.183425+0000","last_became_active":"2023-10-09T09:48:07.969537+0000","last_became_peered":"2023-10-09T09:48:07.969537+0000","last_unstale":"2023-10-09T09:56:03.183425+0000","last_undegraded":"2023-10-09T09:56:03.183425+0000","last_fullsized":"2023-10-09T09:56:03.183425+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:42.344608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":197,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.710965+0000","last_change":"2023-10-09T09:48:07.987618+0000","last_active":"2023-10-09T09:56:00.710965+0000","last_peered":"2023-10-09T09:56:00.710965+0000","last_clean":"2023-10-09T09:56:00.710965+0000","last_became_active":"2023-10-09T09:48:07.987341+0000","last_became_peered":"2023-10-09T09:48:07.987341+0000","last_unstale":"2023-10-09T09:56:00.710965+0000","last_undegraded":"2023-10-09T09:56:00.710965+0000","last_fullsized":"2023-10-09T09:56:00.710965+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:19.972891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":97,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.155384+0000","last_change":"2023-10-09T09:48:07.984247+0000","last_active":"2023-10-09T09:56:00.155384+0000","last_peered":"2023-10-09T09:56:00.155384+0000","last_clean":"2023-10-09T09:56:00.155384+0000","last_became_active":"2023-10-09T09:48:07.978613+0000","last_became_peered":"2023-10-09T09:48:07.978613+0000","last_unstale":"2023-10-09T09:56:00.155384+0000","last_undegraded":"2023-10-09T09:56:00.155384+0000","last_fullsized":"2023-10-09T09:56:00.155384+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:34.663108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1241,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.154687+0000","last_change":"2023-10-09T09:48:07.991053+0000","last_active":"2023-10-09T09:56:00.154687+0000","last_peered":"2023-10-09T09:56:00.154687+0000","last_clean":"2023-10-09T09:56:00.154687+0000","last_became_active":"2023-10-09T09:48:07.990656+0000","last_became_peered":"2023-10-09T09:48:07.990656+0000","last_unstale":"2023-10-09T09:56:00.154687+0000","last_undegraded":"2023-10-09T09:56:00.154687+0000","last_fullsized":"2023-10-09T09:56:00.154687+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:06.223305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":115,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.034556+0000","last_change":"2023-10-09T09:43:23.995576+0000","last_active":"2023-10-09T09:56:01.034556+0000","last_peered":"2023-10-09T09:56:01.034556+0000","last_clean":"2023-10-09T09:56:01.034556+0000","last_became_active":"2023-10-09T09:43:23.995076+0000","last_became_peered":"2023-10-09T09:43:23.995076+0000","last_unstale":"2023-10-09T09:56:01.034556+0000","last_undegraded":"2023-10-09T09:56:01.034556+0000","last_fullsized":"2023-10-09T09:56:01.034556+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1276,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.710656+0000","last_change":"2023-10-09T09:43:48.904021+0000","last_active":"2023-10-09T09:56:00.710656+0000","last_peered":"2023-10-09T09:56:00.710656+0000","last_clean":"2023-10-09T09:56:00.710656+0000","last_became_active":"2023-10-09T09:43:48.903719+0000","last_became_peered":"2023-10-09T09:43:48.903719+0000","last_unstale":"2023-10-09T09:56:00.710656+0000","last_undegraded":"2023-10-09T09:56:00.710656+0000","last_fullsized":"2023-10-09T09:56:00.710656+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:16.284027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":255,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.034349+0000","last_change":"2023-10-09T09:48:08.001392+0000","last_active":"2023-10-09T09:56:01.034349+0000","last_peered":"2023-10-09T09:56:01.034349+0000","last_clean":"2023-10-09T09:56:01.034349+0000","last_became_active":"2023-10-09T09:48:08.000798+0000","last_became_peered":"2023-10-09T09:48:08.000798+0000","last_unstale":"2023-10-09T09:56:01.034349+0000","last_undegraded":"2023-10-09T09:56:01.034349+0000","last_fullsized":"2023-10-09T09:56:01.034349+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":182,"reported_epoch":1305,"state":"active+clean","last_fresh":"2023-10-09T09:56:00.710729+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:56:00.710729+0000","last_peered":"2023-10-09T09:56:00.710729+0000","last_clean":"2023-10-09T09:56:00.710729+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:56:00.710729+0000","last_undegraded":"2023-10-09T09:56:00.710729+0000","last_fullsized":"2023-10-09T09:56:00.710729+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1273,"reported_epoch":1308,"state":"active+clean","last_fresh":"2023-10-09T09:56:03.183697+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:56:03.183697+0000","last_peered":"2023-10-09T09:56:03.183697+0000","last_clean":"2023-10-09T09:56:03.183697+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:56:03.183697+0000","last_undegraded":"2023-10-09T09:56:03.183697+0000","last_fullsized":"2023-10-09T09:56:03.183697+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":680,"reported_epoch":1306,"state":"active+clean","last_fresh":"2023-10-09T09:56:01.033607+0000","last_change":"2023-10-09T09:48:07.990205+0000","last_active":"2023-10-09T09:56:01.033607+0000","last_peered":"2023-10-09T09:56:01.033607+0000","last_clean":"2023-10-09T09:56:01.033607+0000","last_became_active":"2023-10-09T09:48:07.989619+0000","last_became_peered":"2023-10-09T09:48:07.989619+0000","last_unstale":"2023-10-09T09:56:01.033607+0000","last_undegraded":"2023-10-09T09:56:01.033607+0000","last_fullsized":"2023-10-09T09:56:01.033607+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":2460,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30942,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":225506,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":3653,"num_read_kb":5112,"num_write":509,"num_write_kb":1330,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":62083,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":579,"ondisk_log_size":579,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453398171,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594404,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907032,"kb_used_data":6352,"kb_used_omap":0,"kb_used_meta":900672,"kb_avail":92838120,"statfs":{"total":95995035648,"available":95066234880,"internally_reserved":0,"allocated":6504448,"data_stored":6162110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922288128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.1479999999999999,"15min":2.6019999999999999},"min":{"1min":0.5,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":1.216,"5min":5.7279999999999998,"15min":149.095},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.276,"15min":2.8170000000000002},"min":{"1min":0.59899999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.171,"5min":7.3360000000000003,"15min":149.17500000000001},"last":0.871}]},{"osd":1,"last update":"Mon Oct 9 09:55:19 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":2.472,"15min":2.2240000000000002},"min":{"1min":0.33300000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":13.117000000000001,"5min":55.368000000000002,"15min":55.368000000000002},"last":0.436},{"interface":"front","average":{"1min":1.6919999999999999,"5min":2.4399999999999999,"15min":2.1179999999999999},"min":{"1min":0.57799999999999996,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.25,"5min":55.404000000000003,"15min":55.404000000000003},"last":0.83799999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:55:38 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":3.6930000000000001,"15min":3.4399999999999999},"min":{"1min":0.52600000000000002,"5min":0.34799999999999998,"15min":0.29799999999999999},"max":{"1min":7.3840000000000003,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.319,"5min":4.1159999999999997,"15min":3.6120000000000001},"min":{"1min":0.497,"5min":0.32100000000000001,"15min":0.309},"max":{"1min":7.4100000000000001,"5min":208.90899999999999,"15min":208.90899999999999},"last":0.55300000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.0089999999999999,"15min":1.1140000000000001},"min":{"1min":0.51900000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.774,"5min":5.774,"15min":7.9139999999999997},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.96099999999999997,"15min":1.294},"min":{"1min":0.65200000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.482,"5min":4.1669999999999998,"15min":11.968999999999999},"last":0.64300000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.23,"15min":1.419},"min":{"1min":0.60399999999999998,"5min":0.34399999999999997,"15min":0.28399999999999997},"max":{"1min":7.2960000000000003,"5min":7.2960000000000003,"15min":12.148999999999999},"last":1.022},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.97699999999999998,"15min":1.5329999999999999},"min":{"1min":0.41499999999999998,"5min":0.33700000000000002,"15min":0.313},"max":{"1min":1.3540000000000001,"5min":6.7750000000000004,"15min":12.064},"last":0.91600000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.278,"15min":1.482},"min":{"1min":0.56699999999999995,"5min":0.318,"15min":0.318},"max":{"1min":7.3600000000000003,"5min":7.3639999999999999,"15min":10.529},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.042,"5min":1.0129999999999999,"15min":1.4990000000000001},"min":{"1min":0.47999999999999998,"5min":0.34300000000000003,"15min":0.34000000000000002},"max":{"1min":1.522,"5min":4.0629999999999997,"15min":12.000999999999999},"last":0.995}]},{"osd":7,"last update":"Mon Oct 9 09:55:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.133,"15min":1.5009999999999999},"min":{"1min":0.53700000000000003,"5min":0.42999999999999999,"15min":0.38500000000000001},"max":{"1min":3.5179999999999998,"5min":9.6389999999999993,"15min":11.922000000000001},"last":1.234},{"interface":"front","average":{"1min":1.2490000000000001,"5min":1.137,"15min":1.1779999999999999},"min":{"1min":0.58599999999999997,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":3.8610000000000002,"5min":4.9459999999999997,"15min":7.9359999999999999},"last":1.319}]}]},{"osd":2,"up_from":634,"seq":2723009266157,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903772,"kb_used_data":7252,"kb_used_omap":122,"kb_used_meta":896389,"kb_avail":92841380,"statfs":{"total":95995035648,"available":95069573120,"internally_reserved":0,"allocated":7426048,"data_stored":5795286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":917902923},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:54:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":3.6720000000000002,"15min":3.96},"min":{"1min":0.26500000000000001,"5min":0.221,"15min":0.20200000000000001},"max":{"1min":9.048,"5min":218.30799999999999,"15min":218.30799999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.893,"5min":3.871,"15min":4.1820000000000004},"min":{"1min":0.27900000000000003,"5min":0.221,"15min":0.20799999999999999},"max":{"1min":6.9699999999999998,"5min":217.67500000000001,"15min":217.67500000000001},"last":0.41999999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":3.7759999999999998,"15min":1.9419999999999999},"min":{"1min":0.439,"5min":0.22,"15min":0.22},"max":{"1min":1.6850000000000001,"5min":106.089,"15min":106.089},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.84999999999999998,"5min":3.5950000000000002,"15min":1.958},"min":{"1min":0.41899999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.9700000000000002,"5min":105.571,"15min":105.571},"last":0.57699999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:54:57 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.2989999999999999,"15min":1.905},"min":{"1min":0.376,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.4459999999999997,"5min":8.1289999999999996,"15min":230.476},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.3089999999999999,"5min":1.2529999999999999,"15min":1.8260000000000001},"min":{"1min":0.40799999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.226,"5min":9.2810000000000006,"15min":228.893},"last":0.46899999999999997}]},{"osd":4,"last update":"Mon Oct 9 09:55:52 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.278,"15min":1.4139999999999999},"min":{"1min":0.58199999999999996,"5min":0.33100000000000002,"15min":0.27100000000000002},"max":{"1min":1.625,"5min":7.9690000000000003,"15min":71.876000000000005},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.649,"15min":1.3919999999999999},"min":{"1min":0.66600000000000004,"5min":0.29499999999999998,"15min":0.26700000000000002},"max":{"1min":1.7370000000000001,"5min":8.3989999999999991,"15min":9.4640000000000004},"last":0.64400000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:55:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.5549999999999999,"15min":1.3580000000000001},"min":{"1min":0.63400000000000001,"5min":0.34100000000000003,"15min":0.29799999999999999},"max":{"1min":1.4910000000000001,"5min":8.9350000000000005,"15min":8.9350000000000005},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.327,"15min":1.212},"min":{"1min":0.54100000000000004,"5min":0.35199999999999998,"15min":0.32200000000000001},"max":{"1min":1.4850000000000001,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":0.98399999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:55:52 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.466,"15min":1.6919999999999999},"min":{"1min":0.42799999999999999,"5min":0.32900000000000001,"15min":0.30199999999999999},"max":{"1min":1.3740000000000001,"5min":8.0719999999999992,"15min":71.795000000000002},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.996,"5min":1.768,"15min":1.7929999999999999},"min":{"1min":0.621,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.534,"5min":8.4580000000000002,"15min":72.322000000000003},"last":0.70299999999999996}]},{"osd":7,"last update":"Mon Oct 9 09:55:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.712,"15min":1.399},"min":{"1min":0.71099999999999997,"5min":0.39400000000000002,"15min":0.374},"max":{"1min":1.639,"5min":8.4290000000000003,"15min":8.4659999999999993},"last":0.87},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.6100000000000001,"15min":1.284},"min":{"1min":0.626,"5min":0.374,"15min":0.374},"max":{"1min":1.5629999999999999,"5min":8.8849999999999998,"15min":8.8849999999999998},"last":0.82999999999999996}]}]},{"osd":1,"up_from":1246,"seq":5351529250912,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":409476,"kb_used_data":7868,"kb_used_omap":204,"kb_used_meta":401395,"kb_avail":93335676,"statfs":{"total":95995035648,"available":95575732224,"internally_reserved":0,"allocated":8056832,"data_stored":6406769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208958,"internal_metadata":411029442},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":2.6269999999999998,"15min":1.974},"min":{"1min":0.42299999999999999,"5min":0.191,"15min":0.191},"max":{"1min":3.593,"5min":58.033000000000001,"15min":58.033000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.0189999999999999,"5min":2.4670000000000001,"15min":2.2000000000000002},"min":{"1min":0.496,"5min":0.19,"15min":0.19},"max":{"1min":6.4950000000000001,"5min":65.921999999999997,"15min":65.921999999999997},"last":0.64000000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":2.6160000000000001,"15min":1.8560000000000001},"min":{"1min":0.497,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.2490000000000001,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.61599999999999999,"5min":2.4079999999999999,"15min":1.8280000000000001},"min":{"1min":0.36499999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.78500000000000003,"5min":58.682000000000002,"15min":58.682000000000002},"last":0.68100000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.3180000000000001,"15min":1.444},"min":{"1min":0.41199999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.875,"5min":7.6459999999999999,"15min":8.7140000000000004},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.4350000000000001,"15min":1.3560000000000001},"min":{"1min":0.497,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.419,"5min":7.5780000000000003,"15min":7.5780000000000003},"last":1.157}]},{"osd":4,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.6339999999999999,"15min":2.2970000000000002},"min":{"1min":0.54400000000000004,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.3200000000000001,"5min":8.1120000000000001,"15min":8.8070000000000004},"last":0.80900000000000005},{"interface":"front","average":{"1min":1.099,"5min":1.665,"15min":2.1539999999999999},"min":{"1min":0.66900000000000004,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.3540000000000001,"5min":10.263,"15min":10.263},"last":1.0629999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.6919999999999999,"15min":1.7529999999999999},"min":{"1min":0.60499999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.399,"5min":10.214,"15min":10.214},"last":1.131},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.599,"15min":1.7569999999999999},"min":{"1min":0.62,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.304,"5min":8.5139999999999993,"15min":8.5139999999999993},"last":1.111}]},{"osd":6,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":2.0640000000000001,"15min":1.901},"min":{"1min":0.67500000000000004,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.5569999999999999,"5min":9.3309999999999995,"15min":9.3309999999999995},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.9530000000000001,"15min":2.722},"min":{"1min":0.624,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.353,"5min":9.8659999999999997,"15min":9.8659999999999997},"last":1.085}]},{"osd":7,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.839,"15min":1.665},"min":{"1min":0.70599999999999996,"5min":0.374,"15min":0.374},"max":{"1min":1.385,"5min":9.2929999999999993,"15min":9.2929999999999993},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.016,"5min":1.8069999999999999,"15min":2.157},"min":{"1min":0.61099999999999999,"5min":0.371,"15min":0.371},"max":{"1min":1.385,"5min":9.4529999999999994,"15min":9.4529999999999994},"last":1.167}]}]},{"osd":0,"up_from":1136,"seq":4879082848506,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":757652,"kb_used_data":7820,"kb_used_omap":280,"kb_used_meta":749543,"kb_avail":92987500,"statfs":{"total":95995035648,"available":95219200000,"internally_reserved":0,"allocated":8007680,"data_stored":6401607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":767532442},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":2.302,"15min":1.909},"min":{"1min":0.30599999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":7.617,"5min":69.680000000000007,"15min":69.680000000000007},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.96299999999999997,"5min":2.331,"15min":2.1160000000000001},"min":{"1min":0.36299999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.6120000000000001,"5min":70.668999999999997,"15min":70.668999999999997},"last":0.76900000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":4.2460000000000004,"15min":2.5859999999999999},"min":{"1min":0.40400000000000003,"5min":0.22,"15min":0.22},"max":{"1min":1.847,"5min":254.89599999999999,"15min":254.89599999999999},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.63500000000000001,"5min":4.6219999999999999,"15min":2.9940000000000002},"min":{"1min":0.41399999999999998,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":1.3100000000000001,"5min":256.85399999999998,"15min":256.85399999999998},"last":1.018}]},{"osd":3,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.087,"15min":1.079},"min":{"1min":0.435,"5min":0.247,"15min":0.247},"max":{"1min":1.0209999999999999,"5min":8.8309999999999995,"15min":10.475},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.032,"15min":1.071},"min":{"1min":0.53500000000000003,"5min":0.28299999999999997,"15min":0.25800000000000001},"max":{"1min":1.996,"5min":6.4160000000000004,"15min":6.6550000000000002},"last":0.88400000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":1.5109999999999999,"15min":1.4610000000000001},"min":{"1min":0.56499999999999995,"5min":0.318,"15min":0.314},"max":{"1min":1.2430000000000001,"5min":7.3129999999999997,"15min":20.079000000000001},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.345,"15min":1.3959999999999999},"min":{"1min":0.65300000000000002,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":1.383,"5min":8.9480000000000004,"15min":12.044},"last":1.2010000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.4179999999999999,"15min":1.464},"min":{"1min":0.64800000000000002,"5min":0.33000000000000002,"15min":0.28399999999999997},"max":{"1min":1.5089999999999999,"5min":8.8800000000000008,"15min":9.4280000000000008},"last":1.2569999999999999},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.399,"15min":1.526},"min":{"1min":0.71399999999999997,"5min":0.34399999999999997,"15min":0.29699999999999999},"max":{"1min":1.641,"5min":8.7010000000000005,"15min":9.6289999999999996},"last":1.151}]},{"osd":6,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.7430000000000001,"15min":1.5309999999999999},"min":{"1min":0.59699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.419,"5min":9.5950000000000006,"15min":10.413},"last":1.0860000000000001},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.6870000000000001,"15min":1.6910000000000001},"min":{"1min":0.83699999999999997,"5min":0.439,"15min":0.35599999999999998},"max":{"1min":1.595,"5min":8.9809999999999999,"15min":9.3260000000000005},"last":1.4490000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.5920000000000001,"15min":1.373},"min":{"1min":0.59599999999999997,"5min":0.38500000000000001,"15min":0.38},"max":{"1min":1.3320000000000001,"5min":7.0149999999999997,"15min":7.0149999999999997},"last":0.98499999999999999},{"interface":"front","average":{"1min":1.077,"5min":1.45,"15min":1.391},"min":{"1min":0.69899999999999995,"5min":0.435,"15min":0.39200000000000002},"max":{"1min":1.9510000000000001,"5min":8.9160000000000004,"15min":10.087999999999999},"last":1.5089999999999999}]}]},{"osd":3,"up_from":0,"seq":2847563317736,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986869,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896840,"kb_used_data":5760,"kb_used_omap":0,"kb_used_meta":891072,"kb_avail":92848312,"statfs":{"total":95995035648,"available":95076671488,"internally_reserved":0,"allocated":5898240,"data_stored":5571640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912457728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.2010000000000001,"15min":1.6000000000000001},"min":{"1min":0.51500000000000001,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":1.2410000000000001,"5min":5.7670000000000003,"15min":64.372},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.369,"15min":1.5429999999999999},"min":{"1min":0.56100000000000005,"5min":0.32400000000000001,"15min":0.27700000000000002},"max":{"1min":1.2829999999999999,"5min":5.7889999999999997,"15min":64.356999999999999},"last":0.73199999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":4.3860000000000001,"15min":2.5230000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.27700000000000002},"max":{"1min":1.165,"5min":216.16900000000001,"15min":216.16900000000001},"last":1.2130000000000001},{"interface":"front","average":{"1min":0.82399999999999995,"5min":4.6740000000000004,"15min":2.1859999999999999},"min":{"1min":0.38200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.1830000000000001,"5min":216.262,"15min":216.262},"last":1.0800000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.1040000000000001,"15min":1.3979999999999999},"min":{"1min":0.38600000000000001,"5min":0.31,"15min":0.29199999999999998},"max":{"1min":1.1519999999999999,"5min":15.853999999999999,"15min":15.853999999999999},"last":0.873},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.234,"15min":1.321},"min":{"1min":0.40799999999999997,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":1.4039999999999999,"5min":15.881,"15min":15.881},"last":0.77400000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:55:02 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.004,"15min":1.006},"min":{"1min":0.443,"5min":0.36299999999999999,"15min":0.32200000000000001},"max":{"1min":1.8620000000000001,"5min":5.5129999999999999,"15min":8.1120000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.81299999999999994,"15min":1.1120000000000001},"min":{"1min":0.39700000000000002,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":1.25,"5min":5.3109999999999999,"15min":7.4180000000000001},"last":0.91800000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:55:02 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.90600000000000003,"15min":1.3919999999999999},"min":{"1min":0.435,"5min":0.373,"15min":0.26900000000000002},"max":{"1min":2.3820000000000001,"5min":5.4660000000000002,"15min":11.763999999999999},"last":1.329},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.111,"15min":1.3180000000000001},"min":{"1min":0.42799999999999999,"5min":0.41399999999999998,"15min":0.28499999999999998},"max":{"1min":1.284,"5min":5.4790000000000001,"15min":12.446999999999999},"last":1.167}]},{"osd":6,"last update":"Mon Oct 9 09:55:02 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.97899999999999998,"15min":1.4350000000000001},"min":{"1min":0.46999999999999997,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":2.9550000000000001,"5min":5.2690000000000001,"15min":11.701000000000001},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.0089999999999999,"15min":1.5249999999999999},"min":{"1min":0.46100000000000002,"5min":0.40799999999999997,"15min":0.39200000000000002},"max":{"1min":2.9009999999999998,"5min":3.7240000000000002,"15min":11.972},"last":1.284}]},{"osd":7,"last update":"Mon Oct 9 09:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.84999999999999998,"15min":1.2170000000000001},"min":{"1min":0.54000000000000004,"5min":0.45600000000000002,"15min":0.373},"max":{"1min":1.425,"5min":2.4180000000000001,"15min":11.840999999999999},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.04,"15min":1.091},"min":{"1min":0.49299999999999999,"5min":0.48699999999999999,"15min":0.40699999999999997},"max":{"1min":1.2090000000000001,"5min":5.1980000000000004,"15min":9.6210000000000004},"last":1.242}]}]},{"osd":5,"up_from":37,"seq":158913790635,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":904044,"kb_used_data":5732,"kb_used_omap":0,"kb_used_meta":898304,"kb_avail":92841108,"statfs":{"total":95995035648,"available":95069294592,"internally_reserved":0,"allocated":5869568,"data_stored":5549652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919863296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:55:43 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":3.4660000000000002,"15min":4.2309999999999999},"min":{"1min":0.49099999999999999,"5min":0.26600000000000001,"15min":0.23599999999999999},"max":{"1min":1.254,"5min":121.977,"15min":274.71899999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.88900000000000001,"5min":3.3530000000000002,"15min":4.0810000000000004},"min":{"1min":0.55900000000000005,"5min":0.33400000000000002,"15min":0.26800000000000002},"max":{"1min":1.1859999999999999,"5min":122.13200000000001,"15min":274.73500000000001},"last":0.84699999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:55:23 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":3.9249999999999998,"15min":2.5510000000000002},"min":{"1min":0.34699999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.2609999999999999,"5min":192.72200000000001,"15min":192.72200000000001},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.999,"5min":3.9380000000000002,"15min":2.1160000000000001},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.3050000000000002,"5min":192.69300000000001,"15min":192.69300000000001},"last":0.69299999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":2.2469999999999999,"15min":1.911},"min":{"1min":0.72899999999999998,"5min":0.34699999999999998,"15min":0.312},"max":{"1min":1.2170000000000001,"5min":122.02800000000001,"15min":122.02800000000001},"last":0.93200000000000005},{"interface":"front","average":{"1min":1.0389999999999999,"5min":2.4239999999999999,"15min":2.1320000000000001},"min":{"1min":0.71699999999999997,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":1.284,"5min":122.05800000000001,"15min":122.05800000000001},"last":0.995}]},{"osd":3,"last update":"Mon Oct 9 09:55:39 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.78600000000000003,"15min":1.0389999999999999},"min":{"1min":0.60199999999999998,"5min":0.374,"15min":0.33100000000000002},"max":{"1min":1.079,"5min":3.601,"15min":7.2270000000000003},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.91100000000000003,"15min":1.103},"min":{"1min":0.78100000000000003,"5min":0.38,"15min":0.33200000000000002},"max":{"1min":1.2829999999999999,"5min":6.7750000000000004,"15min":16.201000000000001},"last":1.071}]},{"osd":4,"last update":"Mon Oct 9 09:55:17 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.81699999999999995,"15min":1.2190000000000001},"min":{"1min":0.44900000000000001,"5min":0.35699999999999998,"15min":0.27000000000000002},"max":{"1min":3.4140000000000001,"5min":3.6899999999999999,"15min":9.4209999999999994},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.92900000000000005,"15min":1.3979999999999999},"min":{"1min":0.40799999999999997,"5min":0.318,"15min":0.318},"max":{"1min":1.1970000000000001,"5min":7.601,"15min":16.053000000000001},"last":1.1299999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:55:39 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.097,"15min":1.8},"min":{"1min":0.67500000000000004,"5min":0.308,"15min":0.308},"max":{"1min":1.242,"5min":7.798,"15min":140.179},"last":0.88900000000000001},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.2,"15min":1.9059999999999999},"min":{"1min":0.71499999999999997,"5min":0.42799999999999999,"15min":0.34399999999999997},"max":{"1min":1.8240000000000001,"5min":7.8070000000000004,"15min":137.928},"last":1.194}]},{"osd":7,"last update":"Mon Oct 9 09:55:17 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":0.89700000000000002,"15min":1.2230000000000001},"min":{"1min":0.60099999999999998,"5min":0.42399999999999999,"15min":0.373},"max":{"1min":1.397,"5min":3.3180000000000001,"15min":16.119},"last":1.2629999999999999},{"interface":"front","average":{"1min":1.234,"5min":0.88600000000000001,"15min":0.97299999999999998},"min":{"1min":0.48299999999999998,"5min":0.42799999999999999,"15min":0.34200000000000003},"max":{"1min":3.6619999999999999,"5min":3.6619999999999999,"15min":6.5460000000000003},"last":0.97199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":10256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":20706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":10291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":10291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":10380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7721,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:56:08.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:08.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:56:08.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:56:08.467 INFO:tasks.workunit.client.0.smithi163.stderr:+ prefix=rbd_data.b5caebe03033 2023-10-09T09:56:08.469 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep+ rbd_data.b5caebe03033 2023-10-09T09:56:08.469 INFO:tasks.workunit.client.0.smithi163.stderr:get_image_data_pool sparse 2023-10-09T09:56:08.469 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse 2023-10-09T09:56:08.469 INFO:tasks.workunit.client.0.smithi163.stderr:+ sed -e s/rbd_data.b5caebe03033\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:56:08.469 INFO:tasks.workunit.client.0.smithi163.stderr:+ sort -u 2023-10-09T09:56:08.469 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse 2023-10-09T09:56:08.469 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep data_pool: 2023-10-09T09:56:08.469 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk -F: { print $NF } 2023-10-09T09:56:08.514 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool= 2023-10-09T09:56:08.514 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ -z ] 2023-10-09T09:56:08.514 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool=rbd 2023-10-09T09:56:08.514 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo rbd 2023-10-09T09:56:08.514 INFO:tasks.workunit.client.0.smithi163.stderr:+ rados ls -p rbd 2023-10-09T09:56:08.539 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:08.558 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects=1 2023-10-09T09:56:08.558 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo 1 2023-10-09T09:56:08.558 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T09:56:08.558 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse 2023-10-09T09:56:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:08 smithi181 bash[59345]: cluster 2023-10-09T09:56:06.435963+0000 osd.6 (osd.6) 587 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:08 smithi181 bash[59345]: cluster 2023-10-09T09:56:06.439079+0000 osd.6 (osd.6) 588 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:08 smithi181 bash[59345]: cluster 2023-10-09T09:56:06.439284+0000 osd.4 (osd.4) 691 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:08 smithi181 bash[59345]: cluster 2023-10-09T09:56:06.441887+0000 osd.4 (osd.4) 692 : cluster 3 failed to encode map e1311 with expected crc 2023-10-09T09:56:08.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:08.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:08.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:56:08.698 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T09:56:08.700 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:56:08.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:08.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:56:08.704 INFO:tasks.workunit.client.0.smithi163.stdout:zeros import through stdin to sparse image 2023-10-09T09:56:08.705 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo zeros import through stdin to sparse image 2023-10-09T09:56:08.705 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T09:56:08.705 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format - sparse 2023-10-09T09:56:08.770 INFO:tasks.workunit.client.0.smithi163.stderr:4+0 records in 2023-10-09T09:56:08.770 INFO:tasks.workunit.client.0.smithi163.stderr:4+0 records out 2023-10-09T09:56:08.770 INFO:tasks.workunit.client.0.smithi163.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0646788 s, 64.8 MB/s 2023-10-09T09:56:08.789 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T09:56:08.790 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:56:08.794 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects sparse 2023-10-09T09:56:08.794 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse 2023-10-09T09:56:08.795 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse 2023-10-09T09:56:08.795 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep block_name_prefix 2023-10-09T09:56:08.795 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk {print $NF;} 2023-10-09T09:56:08.804 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:08.836 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:08.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:08.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:08.843 INFO:tasks.workunit.client.0.smithi163.stderr:+ prefix=rbd_data.8f1328a7af8f 2023-10-09T09:56:08.845 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep rbd_data.8f1328a7af8f 2023-10-09T09:56:08.845 INFO:tasks.workunit.client.0.smithi163.stderr:+ get_image_data_pool sparse 2023-10-09T09:56:08.845 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse 2023-10-09T09:56:08.845 INFO:tasks.workunit.client.0.smithi163.stderr:+ sed -e s/rbd_data.8f1328a7af8f\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:56:08.845 INFO:tasks.workunit.client.0.smithi163.stderr:+ sort -u 2023-10-09T09:56:08.846 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse 2023-10-09T09:56:08.846 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep data_pool: 2023-10-09T09:56:08.846 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk -F: { print $NF } 2023-10-09T09:56:08.894 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool= 2023-10-09T09:56:08.894 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ -z ] 2023-10-09T09:56:08.894 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool=rbd 2023-10-09T09:56:08.895 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo rbd 2023-10-09T09:56:08.895 INFO:tasks.workunit.client.0.smithi163.stderr:+ rados ls -p rbd 2023-10-09T09:56:08.939 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects= 2023-10-09T09:56:08.939 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo 2023-10-09T09:56:08.939 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ 0 -eq 1 -o = ] 2023-10-09T09:56:08.939 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse 2023-10-09T09:56:08.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:08.972 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:08.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:08.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:09.055 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:56:09.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:09.080 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T09:56:09.085 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo zeros export to sparse file 2023-10-09T09:56:09.085 INFO:tasks.workunit.client.0.smithi163.stdout:zeros export to sparse file 2023-10-09T09:56:09.086 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T09:56:09.104 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:09.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:09.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:09.140 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T09:56:09.144 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse 2023-10-09T09:56:09.145 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep block_name_prefix 2023-10-09T09:56:09.145 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk {print $NF;} 2023-10-09T09:56:09.191 INFO:tasks.workunit.client.0.smithi163.stderr:+ prefix=rbd_data.8f34d61458a6 2023-10-09T09:56:09.191 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T09:56:09.191 INFO:tasks.workunit.client.0.smithi163.stderr:+ get_image_data_pool sparse 2023-10-09T09:56:09.191 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse 2023-10-09T09:56:09.191 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse 2023-10-09T09:56:09.192 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep data_pool: 2023-10-09T09:56:09.192 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk -F: { print $NF } 2023-10-09T09:56:09.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:09.233 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:09.236 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool= 2023-10-09T09:56:09.236 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ -z ] 2023-10-09T09:56:09.237 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool=rbd 2023-10-09T09:56:09.237 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo rbd 2023-10-09T09:56:09.237 INFO:tasks.workunit.client.0.smithi163.stderr:+ rados -p rbd put rbd_data.8f34d61458a6.000000000000 - 2023-10-09T09:56:09.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:09.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:09.276 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records in 2023-10-09T09:56:09.276 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records out 2023-10-09T09:56:09.276 INFO:tasks.workunit.client.0.smithi163.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0846578 s, 49.5 MB/s 2023-10-09T09:56:09.298 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects sparse 2023-10-09T09:56:09.298 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse 2023-10-09T09:56:09.299 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse 2023-10-09T09:56:09.299 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep block_name_prefix 2023-10-09T09:56:09.299 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk {print $NF;} 2023-10-09T09:56:09.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:09 smithi181 bash[59345]: cluster 2023-10-09T09:56:08.295876+0000 mgr.y (mgr.34107) 1511 : cluster 0 pgmap v2411: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 349 KiB/s rd, 210 KiB/s wr, 201 op/s 2023-10-09T09:56:09.339 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:09.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:09 smithi163 bash[78107]: cluster 2023-10-09T09:56:08.295876+0000 mgr.y (mgr.34107) 1511 : cluster 0 pgmap v2411: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 349 KiB/s rd, 210 KiB/s wr, 201 op/s 2023-10-09T09:56:09.349 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:09 smithi163 bash[75425]: cluster 2023-10-09T09:56:08.295876+0000 mgr.y (mgr.34107) 1511 : cluster 0 pgmap v2411: 73 pgs: 73 active+clean; 692 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 349 KiB/s rd, 210 KiB/s wr, 201 op/s 2023-10-09T09:56:09.351 INFO:tasks.workunit.client.0.smithi163.stderr:+ prefix=rbd_data.8f34d61458a6 2023-10-09T09:56:09.352 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep rbd_data.8f34d61458a6 2023-10-09T09:56:09.352 INFO:tasks.workunit.client.0.smithi163.stderr:+ get_image_data_pool sparse 2023-10-09T09:56:09.352 INFO:tasks.workunit.client.0.smithi163.stderr:+ image=sparse 2023-10-09T09:56:09.353 INFO:tasks.workunit.client.0.smithi163.stderr:+ sed -e s/rbd_data.8f34d61458a6\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T09:56:09.353 INFO:tasks.workunit.client.0.smithi163.stderr:+ sort -u 2023-10-09T09:56:09.353 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info sparse 2023-10-09T09:56:09.353 INFO:tasks.workunit.client.0.smithi163.stderr:+ + awk -F: { print $NF } 2023-10-09T09:56:09.353 INFO:tasks.workunit.client.0.smithi163.stderr:grep data_pool: 2023-10-09T09:56:09.365 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:56:09.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:09.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:56:09.401 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool= 2023-10-09T09:56:09.402 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ -z ] 2023-10-09T09:56:09.402 INFO:tasks.workunit.client.0.smithi163.stderr:+ data_pool=rbd 2023-10-09T09:56:09.402 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo rbd 2023-10-09T09:56:09.402 INFO:tasks.workunit.client.0.smithi163.stderr:+ rados ls -p rbd 2023-10-09T09:56:09.449 INFO:tasks.workunit.client.0.smithi163.stderr:+ objects=0 2023-10-09T09:56:09.449 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo 0 2023-10-09T09:56:09.449 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T09:56:09.449 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/sparse 2023-10-09T09:56:09.450 INFO:tasks.workunit.client.0.smithi163.stderr:rm: cannot remove '/tmp/rbd_import_export_237542/sparse': No such file or directory 2023-10-09T09:56:09.450 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T09:56:09.450 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export sparse /tmp/rbd_import_export_237542/sparse 2023-10-09T09:56:09.469 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:09.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:09.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:09.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:09.504 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete...done. 2023-10-09T09:56:09.511 INFO:tasks.workunit.client.0.smithi163.stderr:+ stat /tmp/rbd_import_export_237542/sparse --format=%b 2023-10-09T09:56:09.512 INFO:tasks.workunit.client.0.smithi163.stderr:+ [ 0 = 0 ] 2023-10-09T09:56:09.512 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse 2023-10-09T09:56:09.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:09.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:09.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:09.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:56:09.654 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T09:56:09.660 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_237542/sparse /tmp/rbd_import_export_237542/sparse1 /tmp/rbd_import_export_237542/sparse2 /tmp/rbd_import_export_237542/sparse3 2023-10-09T09:56:09.662 INFO:tasks.workunit.client.0.smithi163.stderr:rm: cannot remove '/tmp/rbd_import_export_237542/sparse3': No such file or directory 2023-10-09T09:56:09.663 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T09:56:09.663 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo OK 2023-10-09T09:56:09.663 INFO:tasks.workunit.client.0.smithi163.stdout:OK 2023-10-09T09:56:09.663 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm -rf /tmp/rbd_import_export_237542 2023-10-09T09:56:09.664 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T09:56:09.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T09:56:09.723 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T09:56:09.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T09:56:09.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:09.769 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:56:09.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:09.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:56:09.873 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:09.905 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:56:09.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:09.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:56:10.006 DEBUG:teuthology.parallel:result is None 2023-10-09T09:56:10.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:56:10.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:10.021 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:56:10.021 DEBUG:teuthology.orchestra.run.smithi163:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:56:10.045 INFO:teuthology.orchestra.run.smithi181.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:56:10.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:10.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:56:10.069 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:56:10.069 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T09:56:10.069 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T09:56:10.069 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T09:56:10.070 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T09:56:10.071 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T09:56:10.071 DEBUG:teuthology.orchestra.run.smithi163:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:56:10.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:10.118 INFO:teuthology.orchestra.run.smithi163.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T09:56:10.118 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:56:10.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:10.165 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T09:56:10.166 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T09:56:10.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error response from daemon: Cannot kill container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:56:10.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:56:10.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 1: 'sudo docker kill -s 1 ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:56:10.193 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T09:56:10.446 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:56:10.873 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:11.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:56:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:11 smithi163 bash[75425]: cluster 2023-10-09T09:56:10.296731+0000 mgr.y (mgr.34107) 1512 : cluster 0 pgmap v2412: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 476 KiB/s rd, 509 KiB/s wr, 197 op/s 2023-10-09T09:56:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:11 smithi163 bash[75425]: audit 2023-10-09T09:56:10.420008+0000 mon.a (mon.0) 2634 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:11 smithi163 bash[75425]: cluster 2023-10-09T09:56:11.017943+0000 osd.5 (osd.5) 903 : cluster 3 failed to encode map e1312 with expected crc 2023-10-09T09:56:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:11 smithi163 bash[75425]: cluster 2023-10-09T09:56:11.019472+0000 mon.a (mon.0) 2635 : cluster 0 osdmap e1312: 8 total, 8 up, 6 in 2023-10-09T09:56:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:11 smithi163 bash[75425]: cluster 2023-10-09T09:56:11.253776+0000 osd.6 (osd.6) 589 : cluster 3 failed to encode map e1312 with expected crc 2023-10-09T09:56:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:11 smithi163 bash[78107]: cluster 2023-10-09T09:56:10.296731+0000 mgr.y (mgr.34107) 1512 : cluster 0 pgmap v2412: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 476 KiB/s rd, 509 KiB/s wr, 197 op/s 2023-10-09T09:56:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:11 smithi163 bash[78107]: audit 2023-10-09T09:56:10.420008+0000 mon.a (mon.0) 2634 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:11.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:11 smithi163 bash[78107]: cluster 2023-10-09T09:56:11.017943+0000 osd.5 (osd.5) 903 : cluster 3 failed to encode map e1312 with expected crc 2023-10-09T09:56:11.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:11 smithi163 bash[78107]: cluster 2023-10-09T09:56:11.019472+0000 mon.a (mon.0) 2635 : cluster 0 osdmap e1312: 8 total, 8 up, 6 in 2023-10-09T09:56:11.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:11 smithi163 bash[78107]: cluster 2023-10-09T09:56:11.253776+0000 osd.6 (osd.6) 589 : cluster 3 failed to encode map e1312 with expected crc 2023-10-09T09:56:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:11 smithi181 bash[59345]: cluster 2023-10-09T09:56:10.296731+0000 mgr.y (mgr.34107) 1512 : cluster 0 pgmap v2412: 73 pgs: 73 active+clean; 582 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 476 KiB/s rd, 509 KiB/s wr, 197 op/s 2023-10-09T09:56:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:11 smithi181 bash[59345]: audit 2023-10-09T09:56:10.420008+0000 mon.a (mon.0) 2634 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:11 smithi181 bash[59345]: cluster 2023-10-09T09:56:11.017943+0000 osd.5 (osd.5) 903 : cluster 3 failed to encode map e1312 with expected crc 2023-10-09T09:56:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:11 smithi181 bash[59345]: cluster 2023-10-09T09:56:11.019472+0000 mon.a (mon.0) 2635 : cluster 0 osdmap e1312: 8 total, 8 up, 6 in 2023-10-09T09:56:11.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:11 smithi181 bash[59345]: cluster 2023-10-09T09:56:11.253776+0000 osd.6 (osd.6) 589 : cluster 3 failed to encode map e1312 with expected crc 2023-10-09T09:56:12.650 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:12 smithi163 bash[75425]: cluster 2023-10-09T09:56:11.197857+0000 osd.4 (osd.4) 693 : cluster 3 failed to encode map e1312 with expected crc 2023-10-09T09:56:12.650 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:12 smithi163 bash[78107]: cluster 2023-10-09T09:56:11.197857+0000 osd.4 (osd.4) 693 : cluster 3 failed to encode map e1312 with expected crc 2023-10-09T09:56:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:12 smithi181 bash[59345]: cluster 2023-10-09T09:56:11.197857+0000 osd.4 (osd.4) 693 : cluster 3 failed to encode map e1312 with expected crc 2023-10-09T09:56:12.904 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:13.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:56:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:56:13] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.33.4" 2023-10-09T09:56:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:13 smithi163 bash[75425]: cluster 2023-10-09T09:56:12.297861+0000 mgr.y (mgr.34107) 1513 : cluster 0 pgmap v2414: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 344 op/s 2023-10-09T09:56:13.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:13 smithi163 bash[78107]: cluster 2023-10-09T09:56:12.297861+0000 mgr.y (mgr.34107) 1513 : cluster 0 pgmap v2414: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 344 op/s 2023-10-09T09:56:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:13 smithi181 bash[59345]: cluster 2023-10-09T09:56:12.297861+0000 mgr.y (mgr.34107) 1513 : cluster 0 pgmap v2414: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 344 op/s 2023-10-09T09:56:14.932 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:15.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:15 smithi163 bash[78107]: cluster 2023-10-09T09:56:14.298581+0000 mgr.y (mgr.34107) 1514 : cluster 0 pgmap v2415: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 982 KiB/s rd, 1.3 MiB/s wr, 323 op/s 2023-10-09T09:56:15.667 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:15 smithi163 bash[75425]: cluster 2023-10-09T09:56:14.298581+0000 mgr.y (mgr.34107) 1514 : cluster 0 pgmap v2415: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 982 KiB/s rd, 1.3 MiB/s wr, 323 op/s 2023-10-09T09:56:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:15 smithi181 bash[59345]: cluster 2023-10-09T09:56:14.298581+0000 mgr.y (mgr.34107) 1514 : cluster 0 pgmap v2415: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 982 KiB/s rd, 1.3 MiB/s wr, 323 op/s 2023-10-09T09:56:16.436 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T09:56:16.436 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T09:56:16.436 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T09:56:16.436 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T09:56:16.436 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T09:56:18.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:18 smithi181 bash[59345]: cluster 2023-10-09T09:56:16.299548+0000 mgr.y (mgr.34107) 1515 : cluster 0 pgmap v2416: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 264 op/s 2023-10-09T09:56:18.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:18 smithi163 bash[78107]: cluster 2023-10-09T09:56:16.299548+0000 mgr.y (mgr.34107) 1515 : cluster 0 pgmap v2416: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 264 op/s 2023-10-09T09:56:18.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:18 smithi163 bash[75425]: cluster 2023-10-09T09:56:16.299548+0000 mgr.y (mgr.34107) 1515 : cluster 0 pgmap v2416: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 264 op/s 2023-10-09T09:56:18.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T09:56:20.133 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:20 smithi163 bash[78107]: cluster 2023-10-09T09:56:18.300527+0000 mgr.y (mgr.34107) 1516 : cluster 0 pgmap v2417: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 662 KiB/s rd, 1.0 MiB/s wr, 200 op/s 2023-10-09T09:56:20.133 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:20 smithi163 bash[78107]: audit 2023-10-09T09:56:19.579375+0000 mon.a (mon.0) 2636 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:20.133 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:20 smithi163 bash[75425]: cluster 2023-10-09T09:56:18.300527+0000 mgr.y (mgr.34107) 1516 : cluster 0 pgmap v2417: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 662 KiB/s rd, 1.0 MiB/s wr, 200 op/s 2023-10-09T09:56:20.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:20 smithi163 bash[75425]: audit 2023-10-09T09:56:19.579375+0000 mon.a (mon.0) 2636 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:20.293 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:20 smithi181 bash[59345]: cluster 2023-10-09T09:56:18.300527+0000 mgr.y (mgr.34107) 1516 : cluster 0 pgmap v2417: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 662 KiB/s rd, 1.0 MiB/s wr, 200 op/s 2023-10-09T09:56:20.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:20 smithi181 bash[59345]: audit 2023-10-09T09:56:19.579375+0000 mon.a (mon.0) 2636 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:20.370 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T09:56:20.393 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T09:56:20.412 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T09:56:20.433 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T09:56:20.455 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T09:56:20.477 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T09:56:20.498 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T09:56:20.526 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T09:56:20.528 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:56:20.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:56:20.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:56:20.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:56:20.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:56:20.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:56:20.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:56:20.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:56:21.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:21 smithi181 bash[59345]: audit 2023-10-09T09:56:20.349202+0000 mon.a (mon.0) 2637 : audit 0 from='client.? 172.21.15.163:0/2086856644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:56:21.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:21 smithi163 bash[75425]: audit 2023-10-09T09:56:20.349202+0000 mon.a (mon.0) 2637 : audit 0 from='client.? 172.21.15.163:0/2086856644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:56:21.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:21 smithi163 bash[78107]: audit 2023-10-09T09:56:20.349202+0000 mon.a (mon.0) 2637 : audit 0 from='client.? 172.21.15.163:0/2086856644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:56:22.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:22 smithi163 bash[75425]: cluster 2023-10-09T09:56:20.301228+0000 mgr.y (mgr.34107) 1517 : cluster 0 pgmap v2418: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 549 KiB/s rd, 827 KiB/s wr, 183 op/s 2023-10-09T09:56:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:22 smithi163 bash[78107]: cluster 2023-10-09T09:56:20.301228+0000 mgr.y (mgr.34107) 1517 : cluster 0 pgmap v2418: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 549 KiB/s rd, 827 KiB/s wr, 183 op/s 2023-10-09T09:56:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:22 smithi181 bash[59345]: cluster 2023-10-09T09:56:20.301228+0000 mgr.y (mgr.34107) 1517 : cluster 0 pgmap v2418: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 549 KiB/s rd, 827 KiB/s wr, 183 op/s 2023-10-09T09:56:23.675 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:56:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:56:23] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.33.4" 2023-10-09T09:56:24.059 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T09:56:24.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:24 smithi181 bash[59345]: cluster 2023-10-09T09:56:22.302019+0000 mgr.y (mgr.34107) 1518 : cluster 0 pgmap v2419: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 487 KiB/s rd, 733 KiB/s wr, 162 op/s 2023-10-09T09:56:24.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:24 smithi181 bash[59345]: audit 2023-10-09T09:56:23.049286+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.163:0/1031529609' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:56:24.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:24 smithi181 bash[59345]: audit 2023-10-09T09:56:23.049711+0000 mon.a (mon.0) 2638 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:56:24.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:24 smithi163 bash[75425]: cluster 2023-10-09T09:56:22.302019+0000 mgr.y (mgr.34107) 1518 : cluster 0 pgmap v2419: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 487 KiB/s rd, 733 KiB/s wr, 162 op/s 2023-10-09T09:56:24.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:24 smithi163 bash[75425]: audit 2023-10-09T09:56:23.049286+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.163:0/1031529609' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:56:24.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:24 smithi163 bash[75425]: audit 2023-10-09T09:56:23.049711+0000 mon.a (mon.0) 2638 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:56:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:24 smithi163 bash[78107]: cluster 2023-10-09T09:56:22.302019+0000 mgr.y (mgr.34107) 1518 : cluster 0 pgmap v2419: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 487 KiB/s rd, 733 KiB/s wr, 162 op/s 2023-10-09T09:56:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:24 smithi163 bash[78107]: audit 2023-10-09T09:56:23.049286+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.163:0/1031529609' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:56:24.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:24 smithi163 bash[78107]: audit 2023-10-09T09:56:23.049711+0000 mon.a (mon.0) 2638 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:56:25.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:56:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:25 smithi181 bash[59345]: cluster 2023-10-09T09:56:24.042329+0000 mon.a (mon.0) 2639 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:56:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:25 smithi181 bash[59345]: cluster 2023-10-09T09:56:24.050587+0000 osd.5 (osd.5) 904 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:25 smithi181 bash[59345]: audit 2023-10-09T09:56:24.053270+0000 mon.a (mon.0) 2640 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:56:25.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:25 smithi181 bash[59345]: cluster 2023-10-09T09:56:24.053393+0000 mon.a (mon.0) 2641 : cluster 0 osdmap e1313: 8 total, 8 up, 6 in 2023-10-09T09:56:25.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:25 smithi181 bash[59345]: cluster 2023-10-09T09:56:24.197513+0000 osd.7 (osd.7) 636 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:25.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:25 smithi181 bash[59345]: cluster 2023-10-09T09:56:24.199602+0000 osd.7 (osd.7) 637 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:25.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[78107]: cluster 2023-10-09T09:56:24.042329+0000 mon.a (mon.0) 2639 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:56:25.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[78107]: cluster 2023-10-09T09:56:24.050587+0000 osd.5 (osd.5) 904 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:25.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[78107]: audit 2023-10-09T09:56:24.053270+0000 mon.a (mon.0) 2640 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:56:25.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[78107]: cluster 2023-10-09T09:56:24.053393+0000 mon.a (mon.0) 2641 : cluster 0 osdmap e1313: 8 total, 8 up, 6 in 2023-10-09T09:56:25.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[78107]: cluster 2023-10-09T09:56:24.197513+0000 osd.7 (osd.7) 636 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:25.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[78107]: cluster 2023-10-09T09:56:24.199602+0000 osd.7 (osd.7) 637 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:25.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[75425]: cluster 2023-10-09T09:56:24.042329+0000 mon.a (mon.0) 2639 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:56:25.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[75425]: cluster 2023-10-09T09:56:24.050587+0000 osd.5 (osd.5) 904 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:25.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[75425]: audit 2023-10-09T09:56:24.053270+0000 mon.a (mon.0) 2640 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:56:25.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[75425]: cluster 2023-10-09T09:56:24.053393+0000 mon.a (mon.0) 2641 : cluster 0 osdmap e1313: 8 total, 8 up, 6 in 2023-10-09T09:56:25.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[75425]: cluster 2023-10-09T09:56:24.197513+0000 osd.7 (osd.7) 636 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:25.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:25 smithi163 bash[75425]: cluster 2023-10-09T09:56:24.199602+0000 osd.7 (osd.7) 637 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:25.748 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:25.790 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:56:25.791 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T09:56:26.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[75425]: cluster 2023-10-09T09:56:24.302750+0000 mgr.y (mgr.34107) 1519 : cluster 0 pgmap v2421: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:26.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[75425]: cluster 2023-10-09T09:56:25.056525+0000 osd.4 (osd.4) 694 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:26.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[75425]: cluster 2023-10-09T09:56:25.058154+0000 osd.4 (osd.4) 695 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:26.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[75425]: audit 2023-10-09T09:56:25.420205+0000 mon.a (mon.0) 2642 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:26.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[75425]: audit 2023-10-09T09:56:25.867311+0000 mon.a (mon.0) 2643 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:26.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[75425]: audit 2023-10-09T09:56:25.873721+0000 mon.a (mon.0) 2644 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:26.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:26 smithi181 bash[59345]: cluster 2023-10-09T09:56:24.302750+0000 mgr.y (mgr.34107) 1519 : cluster 0 pgmap v2421: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:26.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:26 smithi181 bash[59345]: cluster 2023-10-09T09:56:25.056525+0000 osd.4 (osd.4) 694 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:26 smithi181 bash[59345]: cluster 2023-10-09T09:56:25.058154+0000 osd.4 (osd.4) 695 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:26 smithi181 bash[59345]: audit 2023-10-09T09:56:25.420205+0000 mon.a (mon.0) 2642 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:26 smithi181 bash[59345]: audit 2023-10-09T09:56:25.867311+0000 mon.a (mon.0) 2643 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:26 smithi181 bash[59345]: audit 2023-10-09T09:56:25.873721+0000 mon.a (mon.0) 2644 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:26.483 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[78107]: cluster 2023-10-09T09:56:24.302750+0000 mgr.y (mgr.34107) 1519 : cluster 0 pgmap v2421: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:26.483 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[78107]: cluster 2023-10-09T09:56:25.056525+0000 osd.4 (osd.4) 694 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:26.483 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[78107]: cluster 2023-10-09T09:56:25.058154+0000 osd.4 (osd.4) 695 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:26.484 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[78107]: audit 2023-10-09T09:56:25.420205+0000 mon.a (mon.0) 2642 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:26.484 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[78107]: audit 2023-10-09T09:56:25.867311+0000 mon.a (mon.0) 2643 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:26.484 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:26 smithi163 bash[78107]: audit 2023-10-09T09:56:25.873721+0000 mon.a (mon.0) 2644 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:27 smithi181 bash[59345]: cluster 2023-10-09T09:56:25.453352+0000 osd.6 (osd.6) 590 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:27 smithi181 bash[59345]: cluster 2023-10-09T09:56:25.455575+0000 osd.6 (osd.6) 591 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:27.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:27 smithi181 bash[59345]: audit 2023-10-09T09:56:25.775924+0000 mgr.y (mgr.34107) 1520 : audit 0 from='client.46676 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:56:27.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:27 smithi181 bash[59345]: audit 2023-10-09T09:56:26.887879+0000 mon.a (mon.0) 2645 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:27.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:27 smithi181 bash[59345]: audit 2023-10-09T09:56:26.896568+0000 mon.a (mon.0) 2646 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:27.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:27 smithi163 bash[75425]: cluster 2023-10-09T09:56:25.453352+0000 osd.6 (osd.6) 590 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:27.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:27 smithi163 bash[75425]: cluster 2023-10-09T09:56:25.455575+0000 osd.6 (osd.6) 591 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:27.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:27 smithi163 bash[75425]: audit 2023-10-09T09:56:25.775924+0000 mgr.y (mgr.34107) 1520 : audit 0 from='client.46676 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:56:27.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:27 smithi163 bash[75425]: audit 2023-10-09T09:56:26.887879+0000 mon.a (mon.0) 2645 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:27.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:27 smithi163 bash[75425]: audit 2023-10-09T09:56:26.896568+0000 mon.a (mon.0) 2646 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:27.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:27 smithi163 bash[78107]: cluster 2023-10-09T09:56:25.453352+0000 osd.6 (osd.6) 590 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:27.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:27 smithi163 bash[78107]: cluster 2023-10-09T09:56:25.455575+0000 osd.6 (osd.6) 591 : cluster 3 failed to encode map e1313 with expected crc 2023-10-09T09:56:27.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:27 smithi163 bash[78107]: audit 2023-10-09T09:56:25.775924+0000 mgr.y (mgr.34107) 1520 : audit 0 from='client.46676 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:56:27.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:27 smithi163 bash[78107]: audit 2023-10-09T09:56:26.887879+0000 mon.a (mon.0) 2645 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:27.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:27 smithi163 bash[78107]: audit 2023-10-09T09:56:26.896568+0000 mon.a (mon.0) 2646 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:27.444 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2421,"stamp":"2023-10-09T09:56:24.302291+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":592828,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4795412,"kb_used_data":53284,"kb_used_omap":607,"kb_used_meta":4741472,"kb_avail":557675500,"statfs":{"total":575970213888,"available":571059712000,"internally_reserved":0,"allocated":54562816,"data_stored":48697693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":621657,"internal_metadata":4855268263},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004157"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":92,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423222+0000","last_change":"2023-10-09T09:48:07.991289+0000","last_active":"2023-10-09T09:56:06.423222+0000","last_peered":"2023-10-09T09:56:06.423222+0000","last_clean":"2023-10-09T09:56:06.423222+0000","last_became_active":"2023-10-09T09:48:07.990686+0000","last_became_peered":"2023-10-09T09:48:07.990686+0000","last_unstale":"2023-10-09T09:56:06.423222+0000","last_undegraded":"2023-10-09T09:56:06.423222+0000","last_fullsized":"2023-10-09T09:56:06.423222+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:24.421374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":195,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423154+0000","last_change":"2023-10-09T09:48:07.999947+0000","last_active":"2023-10-09T09:56:06.423154+0000","last_peered":"2023-10-09T09:56:06.423154+0000","last_clean":"2023-10-09T09:56:06.423154+0000","last_became_active":"2023-10-09T09:48:07.999704+0000","last_became_peered":"2023-10-09T09:48:07.999704+0000","last_unstale":"2023-10-09T09:56:06.423154+0000","last_undegraded":"2023-10-09T09:56:06.423154+0000","last_fullsized":"2023-10-09T09:56:06.423154+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:14.832042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1264,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.422656+0000","last_change":"2023-10-09T09:48:07.998214+0000","last_active":"2023-10-09T09:56:06.422656+0000","last_peered":"2023-10-09T09:56:06.422656+0000","last_clean":"2023-10-09T09:56:06.422656+0000","last_became_active":"2023-10-09T09:48:07.997544+0000","last_became_peered":"2023-10-09T09:48:07.997544+0000","last_unstale":"2023-10-09T09:56:06.422656+0000","last_undegraded":"2023-10-09T09:56:06.422656+0000","last_fullsized":"2023-10-09T09:56:06.422656+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.304000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1275,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.140602+0000","last_change":"2023-10-09T09:48:07.980358+0000","last_active":"2023-10-09T09:56:11.140602+0000","last_peered":"2023-10-09T09:56:11.140602+0000","last_clean":"2023-10-09T09:56:11.140602+0000","last_became_active":"2023-10-09T09:48:07.979229+0000","last_became_peered":"2023-10-09T09:48:07.979229+0000","last_unstale":"2023-10-09T09:56:11.140602+0000","last_undegraded":"2023-10-09T09:56:11.140602+0000","last_fullsized":"2023-10-09T09:56:11.140602+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:31.456347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":808,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.139981+0000","last_change":"2023-10-09T09:48:07.969728+0000","last_active":"2023-10-09T09:56:11.139981+0000","last_peered":"2023-10-09T09:56:11.139981+0000","last_clean":"2023-10-09T09:56:11.139981+0000","last_became_active":"2023-10-09T09:48:07.968773+0000","last_became_peered":"2023-10-09T09:48:07.968773+0000","last_unstale":"2023-10-09T09:56:11.139981+0000","last_undegraded":"2023-10-09T09:56:11.139981+0000","last_fullsized":"2023-10-09T09:56:11.139981+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:39.023317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1276,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.444208+0000","last_change":"2023-10-09T09:48:07.988928+0000","last_active":"2023-10-09T09:56:06.444208+0000","last_peered":"2023-10-09T09:56:06.444208+0000","last_clean":"2023-10-09T09:56:06.444208+0000","last_became_active":"2023-10-09T09:48:07.988051+0000","last_became_peered":"2023-10-09T09:48:07.988051+0000","last_unstale":"2023-10-09T09:56:06.444208+0000","last_undegraded":"2023-10-09T09:56:06.444208+0000","last_fullsized":"2023-10-09T09:56:06.444208+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1235,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.024777+0000","last_change":"2023-10-09T09:43:48.902066+0000","last_active":"2023-10-09T09:56:06.024777+0000","last_peered":"2023-10-09T09:56:06.024777+0000","last_clean":"2023-10-09T09:56:06.024777+0000","last_became_active":"2023-10-09T09:43:48.901639+0000","last_became_peered":"2023-10-09T09:43:48.901639+0000","last_unstale":"2023-10-09T09:56:06.024777+0000","last_undegraded":"2023-10-09T09:56:06.024777+0000","last_fullsized":"2023-10-09T09:56:06.024777+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":706,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.422647+0000","last_change":"2023-10-09T09:48:07.997787+0000","last_active":"2023-10-09T09:56:06.422647+0000","last_peered":"2023-10-09T09:56:06.422647+0000","last_clean":"2023-10-09T09:56:06.422647+0000","last_became_active":"2023-10-09T09:48:07.997356+0000","last_became_peered":"2023-10-09T09:48:07.997356+0000","last_unstale":"2023-10-09T09:56:06.422647+0000","last_undegraded":"2023-10-09T09:56:06.422647+0000","last_fullsized":"2023-10-09T09:56:06.422647+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:03.024096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":178,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.024406+0000","last_change":"2023-10-09T09:48:07.975547+0000","last_active":"2023-10-09T09:56:06.024406+0000","last_peered":"2023-10-09T09:56:06.024406+0000","last_clean":"2023-10-09T09:56:06.024406+0000","last_became_active":"2023-10-09T09:48:07.975150+0000","last_became_peered":"2023-10-09T09:48:07.975150+0000","last_unstale":"2023-10-09T09:56:06.024406+0000","last_undegraded":"2023-10-09T09:56:06.024406+0000","last_fullsized":"2023-10-09T09:56:06.024406+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":727,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443445+0000","last_change":"2023-10-09T09:41:11.800341+0000","last_active":"2023-10-09T09:56:06.443445+0000","last_peered":"2023-10-09T09:56:06.443445+0000","last_clean":"2023-10-09T09:56:06.443445+0000","last_became_active":"2023-10-09T09:41:11.799985+0000","last_became_peered":"2023-10-09T09:41:11.799985+0000","last_unstale":"2023-10-09T09:56:06.443445+0000","last_undegraded":"2023-10-09T09:56:06.443445+0000","last_fullsized":"2023-10-09T09:56:06.443445+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":120,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443834+0000","last_change":"2023-10-09T09:43:48.906785+0000","last_active":"2023-10-09T09:56:06.443834+0000","last_peered":"2023-10-09T09:56:06.443834+0000","last_clean":"2023-10-09T09:56:06.443834+0000","last_became_active":"2023-10-09T09:43:48.906276+0000","last_became_peered":"2023-10-09T09:43:48.906276+0000","last_unstale":"2023-10-09T09:56:06.443834+0000","last_undegraded":"2023-10-09T09:56:06.443834+0000","last_fullsized":"2023-10-09T09:56:06.443834+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":144,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.023979+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:56:06.023979+0000","last_peered":"2023-10-09T09:56:06.023979+0000","last_clean":"2023-10-09T09:56:06.023979+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:56:06.023979+0000","last_undegraded":"2023-10-09T09:56:06.023979+0000","last_fullsized":"2023-10-09T09:56:06.023979+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":468,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443722+0000","last_change":"2023-10-09T09:43:23.995526+0000","last_active":"2023-10-09T09:56:06.443722+0000","last_peered":"2023-10-09T09:56:06.443722+0000","last_clean":"2023-10-09T09:56:06.443722+0000","last_became_active":"2023-10-09T09:43:23.994816+0000","last_became_peered":"2023-10-09T09:43:23.994816+0000","last_unstale":"2023-10-09T09:56:06.443722+0000","last_undegraded":"2023-10-09T09:56:06.443722+0000","last_fullsized":"2023-10-09T09:56:06.443722+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":685,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443505+0000","last_change":"2023-10-09T09:48:07.987637+0000","last_active":"2023-10-09T09:56:06.443505+0000","last_peered":"2023-10-09T09:56:06.443505+0000","last_clean":"2023-10-09T09:56:06.443505+0000","last_became_active":"2023-10-09T09:48:07.987085+0000","last_became_peered":"2023-10-09T09:48:07.987085+0000","last_unstale":"2023-10-09T09:56:06.443505+0000","last_undegraded":"2023-10-09T09:56:06.443505+0000","last_fullsized":"2023-10-09T09:56:06.443505+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":1209,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.426069+0000","last_change":"2023-10-09T09:43:23.989356+0000","last_active":"2023-10-09T09:56:06.426069+0000","last_peered":"2023-10-09T09:56:06.426069+0000","last_clean":"2023-10-09T09:56:06.426069+0000","last_became_active":"2023-10-09T09:43:23.988986+0000","last_became_peered":"2023-10-09T09:43:23.988986+0000","last_unstale":"2023-10-09T09:56:06.426069+0000","last_undegraded":"2023-10-09T09:56:06.426069+0000","last_fullsized":"2023-10-09T09:56:06.426069+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.570448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":791,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423332+0000","last_change":"2023-10-09T09:48:07.984006+0000","last_active":"2023-10-09T09:56:06.423332+0000","last_peered":"2023-10-09T09:56:06.423332+0000","last_clean":"2023-10-09T09:56:06.423332+0000","last_became_active":"2023-10-09T09:48:07.978179+0000","last_became_peered":"2023-10-09T09:48:07.978179+0000","last_unstale":"2023-10-09T09:56:06.423332+0000","last_undegraded":"2023-10-09T09:56:06.423332+0000","last_fullsized":"2023-10-09T09:56:06.423332+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:34.164004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1212,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423061+0000","last_change":"2023-10-09T09:48:07.975335+0000","last_active":"2023-10-09T09:56:06.423061+0000","last_peered":"2023-10-09T09:56:06.423061+0000","last_clean":"2023-10-09T09:56:06.423061+0000","last_became_active":"2023-10-09T09:48:07.974437+0000","last_became_peered":"2023-10-09T09:48:07.974437+0000","last_unstale":"2023-10-09T09:56:06.423061+0000","last_undegraded":"2023-10-09T09:56:06.423061+0000","last_fullsized":"2023-10-09T09:56:06.423061+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:32.588071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":148,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443909+0000","last_change":"2023-10-09T09:41:11.800342+0000","last_active":"2023-10-09T09:56:06.443909+0000","last_peered":"2023-10-09T09:56:06.443909+0000","last_clean":"2023-10-09T09:56:06.443909+0000","last_became_active":"2023-10-09T09:41:11.799950+0000","last_became_peered":"2023-10-09T09:41:11.799950+0000","last_unstale":"2023-10-09T09:56:06.443909+0000","last_undegraded":"2023-10-09T09:56:06.443909+0000","last_fullsized":"2023-10-09T09:56:06.443909+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":112,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443428+0000","last_change":"2023-10-09T09:43:23.994784+0000","last_active":"2023-10-09T09:56:06.443428+0000","last_peered":"2023-10-09T09:56:06.443428+0000","last_clean":"2023-10-09T09:56:06.443428+0000","last_became_active":"2023-10-09T09:43:23.994196+0000","last_became_peered":"2023-10-09T09:43:23.994196+0000","last_unstale":"2023-10-09T09:56:06.443428+0000","last_undegraded":"2023-10-09T09:56:06.443428+0000","last_fullsized":"2023-10-09T09:56:06.443428+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1230,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.426311+0000","last_change":"2023-10-09T09:43:48.901303+0000","last_active":"2023-10-09T09:56:06.426311+0000","last_peered":"2023-10-09T09:56:06.426311+0000","last_clean":"2023-10-09T09:56:06.426311+0000","last_became_active":"2023-10-09T09:43:48.900779+0000","last_became_peered":"2023-10-09T09:43:48.900779+0000","last_unstale":"2023-10-09T09:56:06.426311+0000","last_undegraded":"2023-10-09T09:56:06.426311+0000","last_fullsized":"2023-10-09T09:56:06.426311+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:51.853531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1306,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443108+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:56:06.443108+0000","last_peered":"2023-10-09T09:56:06.443108+0000","last_clean":"2023-10-09T09:56:06.443108+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:56:06.443108+0000","last_undegraded":"2023-10-09T09:56:06.443108+0000","last_fullsized":"2023-10-09T09:56:06.443108+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1274,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423835+0000","last_change":"2023-10-09T09:48:08.000694+0000","last_active":"2023-10-09T09:56:06.423835+0000","last_peered":"2023-10-09T09:56:06.423835+0000","last_clean":"2023-10-09T09:56:06.423835+0000","last_became_active":"2023-10-09T09:48:07.999476+0000","last_became_peered":"2023-10-09T09:48:07.999476+0000","last_unstale":"2023-10-09T09:56:06.423835+0000","last_undegraded":"2023-10-09T09:56:06.423835+0000","last_fullsized":"2023-10-09T09:56:06.423835+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:37.211076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":141,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.140207+0000","last_change":"2023-10-09T09:48:07.976558+0000","last_active":"2023-10-09T09:56:11.140207+0000","last_peered":"2023-10-09T09:56:11.140207+0000","last_clean":"2023-10-09T09:56:11.140207+0000","last_became_active":"2023-10-09T09:48:07.975961+0000","last_became_peered":"2023-10-09T09:48:07.975961+0000","last_unstale":"2023-10-09T09:56:11.140207+0000","last_undegraded":"2023-10-09T09:56:11.140207+0000","last_fullsized":"2023-10-09T09:56:11.140207+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:40.925248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":154,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.023504+0000","last_change":"2023-10-09T09:40:49.792888+0000","last_active":"2023-10-09T09:56:06.023504+0000","last_peered":"2023-10-09T09:56:06.023504+0000","last_clean":"2023-10-09T09:56:06.023504+0000","last_became_active":"2023-10-09T09:40:49.792516+0000","last_became_peered":"2023-10-09T09:40:49.792516+0000","last_unstale":"2023-10-09T09:56:06.023504+0000","last_undegraded":"2023-10-09T09:56:06.023504+0000","last_fullsized":"2023-10-09T09:56:06.023504+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":103,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423216+0000","last_change":"2023-10-09T09:48:07.990711+0000","last_active":"2023-10-09T09:56:06.423216+0000","last_peered":"2023-10-09T09:56:06.423216+0000","last_clean":"2023-10-09T09:56:06.423216+0000","last_became_active":"2023-10-09T09:48:07.990267+0000","last_became_peered":"2023-10-09T09:48:07.990267+0000","last_unstale":"2023-10-09T09:56:06.423216+0000","last_undegraded":"2023-10-09T09:56:06.423216+0000","last_fullsized":"2023-10-09T09:56:06.423216+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:51.244840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1247,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.422694+0000","last_change":"2023-10-09T09:48:07.991440+0000","last_active":"2023-10-09T09:56:06.422694+0000","last_peered":"2023-10-09T09:56:06.422694+0000","last_clean":"2023-10-09T09:56:06.422694+0000","last_became_active":"2023-10-09T09:48:07.991072+0000","last_became_peered":"2023-10-09T09:48:07.991072+0000","last_unstale":"2023-10-09T09:56:06.422694+0000","last_undegraded":"2023-10-09T09:56:06.422694+0000","last_fullsized":"2023-10-09T09:56:06.422694+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:28.577159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":1262,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.024488+0000","last_change":"2023-10-09T09:41:11.842716+0000","last_active":"2023-10-09T09:56:06.024488+0000","last_peered":"2023-10-09T09:56:06.024488+0000","last_clean":"2023-10-09T09:56:06.024488+0000","last_became_active":"2023-10-09T09:41:11.795210+0000","last_became_peered":"2023-10-09T09:41:11.795210+0000","last_unstale":"2023-10-09T09:56:06.024488+0000","last_undegraded":"2023-10-09T09:56:06.024488+0000","last_fullsized":"2023-10-09T09:56:06.024488+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1268,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.444057+0000","last_change":"2023-10-09T09:48:07.987819+0000","last_active":"2023-10-09T09:56:06.444057+0000","last_peered":"2023-10-09T09:56:06.444057+0000","last_clean":"2023-10-09T09:56:06.444057+0000","last_became_active":"2023-10-09T09:48:07.986354+0000","last_became_peered":"2023-10-09T09:48:07.986354+0000","last_unstale":"2023-10-09T09:56:06.444057+0000","last_undegraded":"2023-10-09T09:56:06.444057+0000","last_fullsized":"2023-10-09T09:56:06.444057+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":814,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443917+0000","last_change":"2023-10-09T09:48:07.987506+0000","last_active":"2023-10-09T09:56:06.443917+0000","last_peered":"2023-10-09T09:56:06.443917+0000","last_clean":"2023-10-09T09:56:06.443917+0000","last_became_active":"2023-10-09T09:48:07.986988+0000","last_became_peered":"2023-10-09T09:48:07.986988+0000","last_unstale":"2023-10-09T09:56:06.443917+0000","last_undegraded":"2023-10-09T09:56:06.443917+0000","last_fullsized":"2023-10-09T09:56:06.443917+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1234,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.139362+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:56:11.139362+0000","last_peered":"2023-10-09T09:56:11.139362+0000","last_clean":"2023-10-09T09:56:11.139362+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:56:11.139362+0000","last_undegraded":"2023-10-09T09:56:11.139362+0000","last_fullsized":"2023-10-09T09:56:11.139362+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":191,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.422912+0000","last_change":"2023-10-09T09:48:07.977143+0000","last_active":"2023-10-09T09:56:06.422912+0000","last_peered":"2023-10-09T09:56:06.422912+0000","last_clean":"2023-10-09T09:56:06.422912+0000","last_became_active":"2023-10-09T09:48:07.976338+0000","last_became_peered":"2023-10-09T09:48:07.976338+0000","last_unstale":"2023-10-09T09:56:06.422912+0000","last_undegraded":"2023-10-09T09:56:06.422912+0000","last_fullsized":"2023-10-09T09:56:06.422912+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:12.015053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1264,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443835+0000","last_change":"2023-10-09T09:48:08.001503+0000","last_active":"2023-10-09T09:56:06.443835+0000","last_peered":"2023-10-09T09:56:06.443835+0000","last_clean":"2023-10-09T09:56:06.443835+0000","last_became_active":"2023-10-09T09:48:08.000921+0000","last_became_peered":"2023-10-09T09:48:08.000921+0000","last_unstale":"2023-10-09T09:56:06.443835+0000","last_undegraded":"2023-10-09T09:56:06.443835+0000","last_fullsized":"2023-10-09T09:56:06.443835+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1260,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443660+0000","last_change":"2023-10-09T09:48:07.988586+0000","last_active":"2023-10-09T09:56:06.443660+0000","last_peered":"2023-10-09T09:56:06.443660+0000","last_clean":"2023-10-09T09:56:06.443660+0000","last_became_active":"2023-10-09T09:48:07.988226+0000","last_became_peered":"2023-10-09T09:48:07.988226+0000","last_unstale":"2023-10-09T09:56:06.443660+0000","last_undegraded":"2023-10-09T09:56:06.443660+0000","last_fullsized":"2023-10-09T09:56:06.443660+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1253,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.426190+0000","last_change":"2023-10-09T09:48:07.987993+0000","last_active":"2023-10-09T09:56:06.426190+0000","last_peered":"2023-10-09T09:56:06.426190+0000","last_clean":"2023-10-09T09:56:06.426190+0000","last_became_active":"2023-10-09T09:48:07.987664+0000","last_became_peered":"2023-10-09T09:48:07.987664+0000","last_unstale":"2023-10-09T09:56:06.426190+0000","last_undegraded":"2023-10-09T09:56:06.426190+0000","last_fullsized":"2023-10-09T09:56:06.426190+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:32.253321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1281,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.425831+0000","last_change":"2023-10-09T09:48:07.984079+0000","last_active":"2023-10-09T09:56:06.425831+0000","last_peered":"2023-10-09T09:56:06.425831+0000","last_clean":"2023-10-09T09:56:06.425831+0000","last_became_active":"2023-10-09T09:48:07.983792+0000","last_became_peered":"2023-10-09T09:48:07.983792+0000","last_unstale":"2023-10-09T09:56:06.425831+0000","last_undegraded":"2023-10-09T09:56:06.425831+0000","last_fullsized":"2023-10-09T09:56:06.425831+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:28:42.289832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":685,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.444123+0000","last_change":"2023-10-09T09:48:07.989047+0000","last_active":"2023-10-09T09:56:06.444123+0000","last_peered":"2023-10-09T09:56:06.444123+0000","last_clean":"2023-10-09T09:56:06.444123+0000","last_became_active":"2023-10-09T09:48:07.988131+0000","last_became_peered":"2023-10-09T09:48:07.988131+0000","last_unstale":"2023-10-09T09:56:06.444123+0000","last_undegraded":"2023-10-09T09:56:06.444123+0000","last_fullsized":"2023-10-09T09:56:06.444123+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1223,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423000+0000","last_change":"2023-10-09T09:48:07.976884+0000","last_active":"2023-10-09T09:56:06.423000+0000","last_peered":"2023-10-09T09:56:06.423000+0000","last_clean":"2023-10-09T09:56:06.423000+0000","last_became_active":"2023-10-09T09:48:07.975932+0000","last_became_peered":"2023-10-09T09:48:07.975932+0000","last_unstale":"2023-10-09T09:56:06.423000+0000","last_undegraded":"2023-10-09T09:56:06.423000+0000","last_fullsized":"2023-10-09T09:56:06.423000+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:22.785728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":160,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.023066+0000","last_change":"2023-10-09T09:41:11.797580+0000","last_active":"2023-10-09T09:56:11.023066+0000","last_peered":"2023-10-09T09:56:11.023066+0000","last_clean":"2023-10-09T09:56:11.023066+0000","last_became_active":"2023-10-09T09:41:11.797156+0000","last_became_peered":"2023-10-09T09:41:11.797156+0000","last_unstale":"2023-10-09T09:56:11.023066+0000","last_undegraded":"2023-10-09T09:56:11.023066+0000","last_fullsized":"2023-10-09T09:56:11.023066+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.913746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"1311'169","reported_seq":1870,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.700838+0000","last_change":"2023-10-09T09:56:05.152894+0000","last_active":"2023-10-09T09:56:11.700838+0000","last_peered":"2023-10-09T09:56:11.700838+0000","last_clean":"2023-10-09T09:56:11.700838+0000","last_became_active":"2023-10-09T09:48:07.983302+0000","last_became_peered":"2023-10-09T09:48:07.983302+0000","last_unstale":"2023-10-09T09:56:11.700838+0000","last_undegraded":"2023-10-09T09:56:11.700838+0000","last_fullsized":"2023-10-09T09:56:11.700838+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:50.714295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001668896,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1002,"num_read_kb":2031,"num_write":171,"num_write_kb":5258,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":154,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.024470+0000","last_change":"2023-10-09T09:41:11.796606+0000","last_active":"2023-10-09T09:56:06.024470+0000","last_peered":"2023-10-09T09:56:06.024470+0000","last_clean":"2023-10-09T09:56:06.024470+0000","last_became_active":"2023-10-09T09:41:11.795757+0000","last_became_peered":"2023-10-09T09:41:11.795757+0000","last_unstale":"2023-10-09T09:56:06.024470+0000","last_undegraded":"2023-10-09T09:56:06.024470+0000","last_fullsized":"2023-10-09T09:56:06.024470+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1246,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.422676+0000","last_change":"2023-10-09T09:48:07.977683+0000","last_active":"2023-10-09T09:56:06.422676+0000","last_peered":"2023-10-09T09:56:06.422676+0000","last_clean":"2023-10-09T09:56:06.422676+0000","last_became_active":"2023-10-09T09:48:07.977024+0000","last_became_peered":"2023-10-09T09:48:07.977024+0000","last_unstale":"2023-10-09T09:56:06.422676+0000","last_undegraded":"2023-10-09T09:56:06.422676+0000","last_fullsized":"2023-10-09T09:56:06.422676+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:13.438431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"1311'247","reported_seq":1806,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.701621+0000","last_change":"2023-10-09T09:56:05.154553+0000","last_active":"2023-10-09T09:56:11.701621+0000","last_peered":"2023-10-09T09:56:11.701621+0000","last_clean":"2023-10-09T09:56:11.701621+0000","last_became_active":"2023-10-09T09:48:07.997389+0000","last_became_peered":"2023-10-09T09:48:07.997389+0000","last_unstale":"2023-10-09T09:56:11.701621+0000","last_undegraded":"2023-10-09T09:56:11.701621+0000","last_fullsized":"2023-10-09T09:56:11.701621+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":247,"log_dups_size":0,"ondisk_log_size":247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:21.978398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002482213,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1725,"num_read_kb":1857,"num_write":205,"num_write_kb":259,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1202,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443681+0000","last_change":"2023-10-09T09:43:48.908190+0000","last_active":"2023-10-09T09:56:06.443681+0000","last_peered":"2023-10-09T09:56:06.443681+0000","last_clean":"2023-10-09T09:56:06.443681+0000","last_became_active":"2023-10-09T09:43:48.907799+0000","last_became_peered":"2023-10-09T09:43:48.907799+0000","last_unstale":"2023-10-09T09:56:06.443681+0000","last_undegraded":"2023-10-09T09:56:06.443681+0000","last_fullsized":"2023-10-09T09:56:06.443681+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1299,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423062+0000","last_change":"2023-10-09T09:48:08.000154+0000","last_active":"2023-10-09T09:56:06.423062+0000","last_peered":"2023-10-09T09:56:06.423062+0000","last_clean":"2023-10-09T09:56:06.423062+0000","last_became_active":"2023-10-09T09:48:07.999714+0000","last_became_peered":"2023-10-09T09:48:07.999714+0000","last_unstale":"2023-10-09T09:56:06.423062+0000","last_undegraded":"2023-10-09T09:56:06.423062+0000","last_fullsized":"2023-10-09T09:56:06.423062+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:12.921693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"1311'103","reported_seq":675,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.022479+0000","last_change":"2023-10-09T09:56:05.235461+0000","last_active":"2023-10-09T09:56:11.022479+0000","last_peered":"2023-10-09T09:56:11.022479+0000","last_clean":"2023-10-09T09:56:11.022479+0000","last_became_active":"2023-10-09T09:48:07.983490+0000","last_became_peered":"2023-10-09T09:48:07.983490+0000","last_unstale":"2023-10-09T09:56:11.022479+0000","last_undegraded":"2023-10-09T09:56:11.022479+0000","last_fullsized":"2023-10-09T09:56:11.022479+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:21.122987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019660929999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":1020,"num_write":78,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1278,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:07.039598+0000","last_change":"2023-10-09T09:48:07.998234+0000","last_active":"2023-10-09T09:56:07.039598+0000","last_peered":"2023-10-09T09:56:07.039598+0000","last_clean":"2023-10-09T09:56:07.039598+0000","last_became_active":"2023-10-09T09:48:07.997909+0000","last_became_peered":"2023-10-09T09:48:07.997909+0000","last_unstale":"2023-10-09T09:56:07.039598+0000","last_undegraded":"2023-10-09T09:56:07.039598+0000","last_fullsized":"2023-10-09T09:56:07.039598+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:21.750617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1266,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423698+0000","last_change":"2023-10-09T09:48:07.973284+0000","last_active":"2023-10-09T09:56:06.423698+0000","last_peered":"2023-10-09T09:56:06.423698+0000","last_clean":"2023-10-09T09:56:06.423698+0000","last_became_active":"2023-10-09T09:48:07.971235+0000","last_became_peered":"2023-10-09T09:48:07.971235+0000","last_unstale":"2023-10-09T09:56:06.423698+0000","last_undegraded":"2023-10-09T09:56:06.423698+0000","last_fullsized":"2023-10-09T09:56:06.423698+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:40.951420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"1311'146","reported_seq":1668,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.022541+0000","last_change":"2023-10-09T09:56:05.235865+0000","last_active":"2023-10-09T09:56:11.022541+0000","last_peered":"2023-10-09T09:56:11.022541+0000","last_clean":"2023-10-09T09:56:11.022541+0000","last_became_active":"2023-10-09T09:41:11.795440+0000","last_became_peered":"2023-10-09T09:41:11.795440+0000","last_unstale":"2023-10-09T09:56:11.022541+0000","last_undegraded":"2023-10-09T09:56:11.022541+0000","last_fullsized":"2023-10-09T09:56:11.022541+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":146,"log_dups_size":0,"ondisk_log_size":146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:46.703876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020527309999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":398,"num_read_kb":2392,"num_write":143,"num_write_kb":2132,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":657,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.139452+0000","last_change":"2023-10-09T09:48:07.978435+0000","last_active":"2023-10-09T09:56:11.139452+0000","last_peered":"2023-10-09T09:56:11.139452+0000","last_clean":"2023-10-09T09:56:11.139452+0000","last_became_active":"2023-10-09T09:48:07.977417+0000","last_became_peered":"2023-10-09T09:48:07.977417+0000","last_unstale":"2023-10-09T09:56:11.139452+0000","last_undegraded":"2023-10-09T09:56:11.139452+0000","last_fullsized":"2023-10-09T09:56:11.139452+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:19.288202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":154,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.023982+0000","last_change":"2023-10-09T09:41:11.794549+0000","last_active":"2023-10-09T09:56:06.023982+0000","last_peered":"2023-10-09T09:56:06.023982+0000","last_clean":"2023-10-09T09:56:06.023982+0000","last_became_active":"2023-10-09T09:41:11.794226+0000","last_became_peered":"2023-10-09T09:41:11.794226+0000","last_unstale":"2023-10-09T09:56:06.023982+0000","last_undegraded":"2023-10-09T09:56:06.023982+0000","last_fullsized":"2023-10-09T09:56:06.023982+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1238,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443947+0000","last_change":"2023-10-09T09:40:49.795765+0000","last_active":"2023-10-09T09:56:06.443947+0000","last_peered":"2023-10-09T09:56:06.443947+0000","last_clean":"2023-10-09T09:56:06.443947+0000","last_became_active":"2023-10-09T09:40:49.795134+0000","last_became_peered":"2023-10-09T09:40:49.795134+0000","last_unstale":"2023-10-09T09:56:06.443947+0000","last_undegraded":"2023-10-09T09:56:06.443947+0000","last_fullsized":"2023-10-09T09:56:06.443947+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"1311'48","reported_seq":1426,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.700609+0000","last_change":"2023-10-09T09:56:05.153257+0000","last_active":"2023-10-09T09:56:11.700609+0000","last_peered":"2023-10-09T09:56:11.700609+0000","last_clean":"2023-10-09T09:56:11.700609+0000","last_became_active":"2023-10-09T09:48:07.971221+0000","last_became_peered":"2023-10-09T09:48:07.971221+0000","last_unstale":"2023-10-09T09:56:11.700609+0000","last_undegraded":"2023-10-09T09:56:11.700609+0000","last_fullsized":"2023-10-09T09:56:11.700609+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":48,"log_dups_size":0,"ondisk_log_size":48,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:16.759514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0029095229999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":269,"num_read_kb":2576,"num_write":41,"num_write_kb":2215,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1252,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.140064+0000","last_change":"2023-10-09T09:41:11.803369+0000","last_active":"2023-10-09T09:56:11.140064+0000","last_peered":"2023-10-09T09:56:11.140064+0000","last_clean":"2023-10-09T09:56:11.140064+0000","last_became_active":"2023-10-09T09:41:11.803021+0000","last_became_peered":"2023-10-09T09:41:11.803021+0000","last_unstale":"2023-10-09T09:56:11.140064+0000","last_undegraded":"2023-10-09T09:56:11.140064+0000","last_fullsized":"2023-10-09T09:56:11.140064+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:40.591499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1263,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.140360+0000","last_change":"2023-10-09T09:48:07.981410+0000","last_active":"2023-10-09T09:56:11.140360+0000","last_peered":"2023-10-09T09:56:11.140360+0000","last_clean":"2023-10-09T09:56:11.140360+0000","last_became_active":"2023-10-09T09:48:07.979850+0000","last_became_peered":"2023-10-09T09:48:07.979850+0000","last_unstale":"2023-10-09T09:56:11.140360+0000","last_undegraded":"2023-10-09T09:56:11.140360+0000","last_fullsized":"2023-10-09T09:56:11.140360+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:31.201104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"1311'131","reported_seq":1808,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.140781+0000","last_change":"2023-10-09T09:56:05.148213+0000","last_active":"2023-10-09T09:56:11.140781+0000","last_peered":"2023-10-09T09:56:11.140781+0000","last_clean":"2023-10-09T09:56:11.140781+0000","last_became_active":"2023-10-09T09:48:07.975338+0000","last_became_peered":"2023-10-09T09:48:07.975338+0000","last_unstale":"2023-10-09T09:56:11.140781+0000","last_undegraded":"2023-10-09T09:56:11.140781+0000","last_fullsized":"2023-10-09T09:56:11.140781+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:05.175956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0033466429999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1092,"num_read_kb":2605,"num_write":104,"num_write_kb":2443,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1280,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423257+0000","last_change":"2023-10-09T09:48:07.983773+0000","last_active":"2023-10-09T09:56:06.423257+0000","last_peered":"2023-10-09T09:56:06.423257+0000","last_clean":"2023-10-09T09:56:06.423257+0000","last_became_active":"2023-10-09T09:48:07.976869+0000","last_became_peered":"2023-10-09T09:48:07.976869+0000","last_unstale":"2023-10-09T09:56:06.423257+0000","last_undegraded":"2023-10-09T09:56:06.423257+0000","last_fullsized":"2023-10-09T09:56:06.423257+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:58.173236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1228,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.024097+0000","last_change":"2023-10-09T09:48:07.962302+0000","last_active":"2023-10-09T09:56:06.024097+0000","last_peered":"2023-10-09T09:56:06.024097+0000","last_clean":"2023-10-09T09:56:06.024097+0000","last_became_active":"2023-10-09T09:48:07.961888+0000","last_became_peered":"2023-10-09T09:48:07.961888+0000","last_unstale":"2023-10-09T09:56:06.024097+0000","last_undegraded":"2023-10-09T09:56:06.024097+0000","last_fullsized":"2023-10-09T09:56:06.024097+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":177,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.140319+0000","last_change":"2023-10-09T09:41:11.853334+0000","last_active":"2023-10-09T09:56:11.140319+0000","last_peered":"2023-10-09T09:56:11.140319+0000","last_clean":"2023-10-09T09:56:11.140319+0000","last_became_active":"2023-10-09T09:41:11.803918+0000","last_became_peered":"2023-10-09T09:41:11.803918+0000","last_unstale":"2023-10-09T09:56:11.140319+0000","last_undegraded":"2023-10-09T09:56:11.140319+0000","last_fullsized":"2023-10-09T09:56:11.140319+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:40.442441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"1311'57","reported_seq":1453,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.701270+0000","last_change":"2023-10-09T09:56:05.154277+0000","last_active":"2023-10-09T09:56:11.701270+0000","last_peered":"2023-10-09T09:56:11.701270+0000","last_clean":"2023-10-09T09:56:11.701270+0000","last_became_active":"2023-10-09T09:48:07.998606+0000","last_became_peered":"2023-10-09T09:48:07.998606+0000","last_unstale":"2023-10-09T09:56:11.701270+0000","last_undegraded":"2023-10-09T09:56:11.701270+0000","last_fullsized":"2023-10-09T09:56:11.701270+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":57,"log_dups_size":0,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.469476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0028883569999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":282,"num_read_kb":223,"num_write":26,"num_write_kb":22,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1169,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:07.039560+0000","last_change":"2023-10-09T09:48:07.976350+0000","last_active":"2023-10-09T09:56:07.039560+0000","last_peered":"2023-10-09T09:56:07.039560+0000","last_clean":"2023-10-09T09:56:07.039560+0000","last_became_active":"2023-10-09T09:48:07.973099+0000","last_became_peered":"2023-10-09T09:48:07.973099+0000","last_unstale":"2023-10-09T09:56:07.039560+0000","last_undegraded":"2023-10-09T09:56:07.039560+0000","last_fullsized":"2023-10-09T09:56:07.039560+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:48.812722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"1311'75","reported_seq":1540,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.259270+0000","last_change":"2023-10-09T09:56:05.158280+0000","last_active":"2023-10-09T09:56:11.259270+0000","last_peered":"2023-10-09T09:56:11.259270+0000","last_clean":"2023-10-09T09:56:11.259270+0000","last_became_active":"2023-10-09T09:48:07.987124+0000","last_became_peered":"2023-10-09T09:48:07.987124+0000","last_unstale":"2023-10-09T09:56:11.259270+0000","last_undegraded":"2023-10-09T09:56:11.259270+0000","last_fullsized":"2023-10-09T09:56:11.259270+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":75,"log_dups_size":0,"ondisk_log_size":75,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":424,"num_read_kb":544,"num_write":78,"num_write_kb":145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1244,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.140642+0000","last_change":"2023-10-09T09:48:07.980600+0000","last_active":"2023-10-09T09:56:11.140642+0000","last_peered":"2023-10-09T09:56:11.140642+0000","last_clean":"2023-10-09T09:56:11.140642+0000","last_became_active":"2023-10-09T09:48:07.979544+0000","last_became_peered":"2023-10-09T09:48:07.979544+0000","last_unstale":"2023-10-09T09:56:11.140642+0000","last_undegraded":"2023-10-09T09:56:11.140642+0000","last_fullsized":"2023-10-09T09:56:11.140642+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:47.320689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1230,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.023666+0000","last_change":"2023-10-09T09:43:23.992213+0000","last_active":"2023-10-09T09:56:06.023666+0000","last_peered":"2023-10-09T09:56:06.023666+0000","last_clean":"2023-10-09T09:56:06.023666+0000","last_became_active":"2023-10-09T09:43:23.991887+0000","last_became_peered":"2023-10-09T09:43:23.991887+0000","last_unstale":"2023-10-09T09:56:06.023666+0000","last_undegraded":"2023-10-09T09:56:06.023666+0000","last_fullsized":"2023-10-09T09:56:06.023666+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1252,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.140207+0000","last_change":"2023-10-09T09:48:07.969929+0000","last_active":"2023-10-09T09:56:11.140207+0000","last_peered":"2023-10-09T09:56:11.140207+0000","last_clean":"2023-10-09T09:56:11.140207+0000","last_became_active":"2023-10-09T09:48:07.969537+0000","last_became_peered":"2023-10-09T09:48:07.969537+0000","last_unstale":"2023-10-09T09:56:11.140207+0000","last_undegraded":"2023-10-09T09:56:11.140207+0000","last_fullsized":"2023-10-09T09:56:11.140207+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:42.344608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":203,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.425975+0000","last_change":"2023-10-09T09:48:07.987618+0000","last_active":"2023-10-09T09:56:06.425975+0000","last_peered":"2023-10-09T09:56:06.425975+0000","last_clean":"2023-10-09T09:56:06.425975+0000","last_became_active":"2023-10-09T09:48:07.987341+0000","last_became_peered":"2023-10-09T09:48:07.987341+0000","last_unstale":"2023-10-09T09:56:06.425975+0000","last_undegraded":"2023-10-09T09:56:06.425975+0000","last_fullsized":"2023-10-09T09:56:06.425975+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:19.972891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":103,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.423196+0000","last_change":"2023-10-09T09:48:07.984247+0000","last_active":"2023-10-09T09:56:06.423196+0000","last_peered":"2023-10-09T09:56:06.423196+0000","last_clean":"2023-10-09T09:56:06.423196+0000","last_became_active":"2023-10-09T09:48:07.978613+0000","last_became_peered":"2023-10-09T09:48:07.978613+0000","last_unstale":"2023-10-09T09:56:06.423196+0000","last_undegraded":"2023-10-09T09:56:06.423196+0000","last_fullsized":"2023-10-09T09:56:06.423196+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:34.663108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1247,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.422766+0000","last_change":"2023-10-09T09:48:07.991053+0000","last_active":"2023-10-09T09:56:06.422766+0000","last_peered":"2023-10-09T09:56:06.422766+0000","last_clean":"2023-10-09T09:56:06.422766+0000","last_became_active":"2023-10-09T09:48:07.990656+0000","last_became_peered":"2023-10-09T09:48:07.990656+0000","last_unstale":"2023-10-09T09:56:06.422766+0000","last_undegraded":"2023-10-09T09:56:06.422766+0000","last_fullsized":"2023-10-09T09:56:06.422766+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:06.223305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":120,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.444172+0000","last_change":"2023-10-09T09:43:23.995576+0000","last_active":"2023-10-09T09:56:06.444172+0000","last_peered":"2023-10-09T09:56:06.444172+0000","last_clean":"2023-10-09T09:56:06.444172+0000","last_became_active":"2023-10-09T09:43:23.995076+0000","last_became_peered":"2023-10-09T09:43:23.995076+0000","last_unstale":"2023-10-09T09:56:06.444172+0000","last_undegraded":"2023-10-09T09:56:06.444172+0000","last_fullsized":"2023-10-09T09:56:06.444172+0000","mapping_epoch":1214,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1283,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.022241+0000","last_change":"2023-10-09T09:43:48.904021+0000","last_active":"2023-10-09T09:56:11.022241+0000","last_peered":"2023-10-09T09:56:11.022241+0000","last_clean":"2023-10-09T09:56:11.022241+0000","last_became_active":"2023-10-09T09:43:48.903719+0000","last_became_peered":"2023-10-09T09:43:48.903719+0000","last_unstale":"2023-10-09T09:56:11.022241+0000","last_undegraded":"2023-10-09T09:56:11.022241+0000","last_fullsized":"2023-10-09T09:56:11.022241+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:16.284027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":260,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.444255+0000","last_change":"2023-10-09T09:48:08.001392+0000","last_active":"2023-10-09T09:56:06.444255+0000","last_peered":"2023-10-09T09:56:06.444255+0000","last_clean":"2023-10-09T09:56:06.444255+0000","last_became_active":"2023-10-09T09:48:08.000798+0000","last_became_peered":"2023-10-09T09:48:08.000798+0000","last_unstale":"2023-10-09T09:56:06.444255+0000","last_undegraded":"2023-10-09T09:56:06.444255+0000","last_fullsized":"2023-10-09T09:56:06.444255+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":189,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.022208+0000","last_change":"2023-10-09T09:41:11.800991+0000","last_active":"2023-10-09T09:56:11.022208+0000","last_peered":"2023-10-09T09:56:11.022208+0000","last_clean":"2023-10-09T09:56:11.022208+0000","last_became_active":"2023-10-09T09:41:11.800636+0000","last_became_peered":"2023-10-09T09:41:11.800636+0000","last_unstale":"2023-10-09T09:56:11.022208+0000","last_undegraded":"2023-10-09T09:56:11.022208+0000","last_fullsized":"2023-10-09T09:56:11.022208+0000","mapping_epoch":1189,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:14.014573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2023-10-09T09:56:11.140299+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:56:11.140299+0000","last_peered":"2023-10-09T09:56:11.140299+0000","last_clean":"2023-10-09T09:56:11.140299+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:56:11.140299+0000","last_undegraded":"2023-10-09T09:56:11.140299+0000","last_fullsized":"2023-10-09T09:56:11.140299+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":685,"reported_epoch":1311,"state":"active+clean","last_fresh":"2023-10-09T09:56:06.443233+0000","last_change":"2023-10-09T09:48:07.990205+0000","last_active":"2023-10-09T09:56:06.443233+0000","last_peered":"2023-10-09T09:56:06.443233+0000","last_clean":"2023-10-09T09:56:06.443233+0000","last_became_active":"2023-10-09T09:48:07.989619+0000","last_became_peered":"2023-10-09T09:48:07.989619+0000","last_unstale":"2023-10-09T09:56:06.443233+0000","last_undegraded":"2023-10-09T09:56:06.443233+0000","last_fullsized":"2023-10-09T09:56:06.443233+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":2460,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":21622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30942,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453398175,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":43,"seq":184683594408,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907048,"kb_used_data":6368,"kb_used_omap":0,"kb_used_meta":900672,"kb_avail":92838104,"statfs":{"total":95995035648,"available":95066218496,"internally_reserved":0,"allocated":6520832,"data_stored":6196930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922288128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.1479999999999999,"15min":2.6019999999999999},"min":{"1min":0.5,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":1.216,"5min":5.7279999999999998,"15min":149.095},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.276,"15min":2.8170000000000002},"min":{"1min":0.59899999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.171,"5min":7.3360000000000003,"15min":149.17500000000001},"last":0.66300000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:56:21 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":2.4300000000000002,"15min":2.137},"min":{"1min":0.436,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.496,"5min":55.368000000000002,"15min":55.368000000000002},"last":0.92800000000000005},{"interface":"front","average":{"1min":0.85599999999999998,"5min":2.4060000000000001,"15min":2.0390000000000001},"min":{"1min":0.58199999999999996,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.1619999999999999,"5min":55.404000000000003,"15min":55.404000000000003},"last":0.755}]},{"osd":2,"last update":"Mon Oct 9 09:55:38 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":3.6930000000000001,"15min":3.4399999999999999},"min":{"1min":0.52600000000000002,"5min":0.34799999999999998,"15min":0.29799999999999999},"max":{"1min":7.3840000000000003,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.319,"5min":4.1159999999999997,"15min":3.6120000000000001},"min":{"1min":0.497,"5min":0.32100000000000001,"15min":0.309},"max":{"1min":7.4100000000000001,"5min":208.90899999999999,"15min":208.90899999999999},"last":1.0940000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.0089999999999999,"15min":1.1140000000000001},"min":{"1min":0.51900000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.774,"5min":5.774,"15min":7.9139999999999997},"last":0.878},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.96099999999999997,"15min":1.294},"min":{"1min":0.65200000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.482,"5min":4.1669999999999998,"15min":11.968999999999999},"last":1.0089999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.23,"15min":1.419},"min":{"1min":0.60399999999999998,"5min":0.34399999999999997,"15min":0.28399999999999997},"max":{"1min":7.2960000000000003,"5min":7.2960000000000003,"15min":12.148999999999999},"last":1.474},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.97699999999999998,"15min":1.5329999999999999},"min":{"1min":0.41499999999999998,"5min":0.33700000000000002,"15min":0.313},"max":{"1min":1.3540000000000001,"5min":6.7750000000000004,"15min":12.064},"last":1.262}]},{"osd":5,"last update":"Mon Oct 9 09:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.278,"15min":1.482},"min":{"1min":0.56699999999999995,"5min":0.318,"15min":0.318},"max":{"1min":7.3600000000000003,"5min":7.3639999999999999,"15min":10.529},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.042,"5min":1.0129999999999999,"15min":1.4990000000000001},"min":{"1min":0.47999999999999998,"5min":0.34300000000000003,"15min":0.34000000000000002},"max":{"1min":1.522,"5min":4.0629999999999997,"15min":12.000999999999999},"last":1.349}]},{"osd":7,"last update":"Mon Oct 9 09:56:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":1.133,"15min":1.5209999999999999},"min":{"1min":0.64700000000000002,"5min":0.42999999999999999,"15min":0.38500000000000001},"max":{"1min":1.5429999999999999,"5min":9.6389999999999993,"15min":11.922000000000001},"last":1.4239999999999999},{"interface":"front","average":{"1min":1.179,"5min":1.133,"15min":1.1930000000000001},"min":{"1min":0.66200000000000003,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":1.4530000000000001,"5min":4.9459999999999997,"15min":7.9359999999999999},"last":1.1950000000000001}]}]},{"osd":2,"up_from":634,"seq":2723009266161,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907948,"kb_used_data":11428,"kb_used_omap":122,"kb_used_meta":896389,"kb_avail":92837204,"statfs":{"total":95995035648,"available":95065296896,"internally_reserved":0,"allocated":11702272,"data_stored":10044686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":917902923},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":3.6749999999999998,"15min":3.9609999999999999},"min":{"1min":0.27900000000000003,"5min":0.221,"15min":0.20200000000000001},"max":{"1min":0.94399999999999995,"5min":218.30799999999999,"15min":218.30799999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.60599999999999998,"5min":3.871,"15min":4.1870000000000003},"min":{"1min":0.33500000000000002,"5min":0.221,"15min":0.20799999999999999},"max":{"1min":0.97899999999999998,"5min":217.67500000000001,"15min":217.67500000000001},"last":0.59099999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":3.7759999999999998,"15min":1.9419999999999999},"min":{"1min":0.439,"5min":0.22,"15min":0.22},"max":{"1min":1.6850000000000001,"5min":106.089,"15min":106.089},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.84999999999999998,"5min":3.5950000000000002,"15min":1.958},"min":{"1min":0.41899999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.9700000000000002,"5min":105.571,"15min":105.571},"last":0.80100000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.2969999999999999,"15min":1.891},"min":{"1min":0.39700000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.0600000000000001,"5min":8.1289999999999996,"15min":230.476},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.2509999999999999,"15min":1.825},"min":{"1min":0.33800000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.129,"5min":9.2810000000000006,"15min":228.893},"last":0.56000000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:55:52 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.278,"15min":1.4139999999999999},"min":{"1min":0.58199999999999996,"5min":0.33100000000000002,"15min":0.27100000000000002},"max":{"1min":1.625,"5min":7.9690000000000003,"15min":71.876000000000005},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.649,"15min":1.3919999999999999},"min":{"1min":0.66600000000000004,"5min":0.29499999999999998,"15min":0.26700000000000002},"max":{"1min":1.7370000000000001,"5min":8.3989999999999991,"15min":9.4640000000000004},"last":0.77500000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:55:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.5549999999999999,"15min":1.3580000000000001},"min":{"1min":0.63400000000000001,"5min":0.34100000000000003,"15min":0.29799999999999999},"max":{"1min":1.4910000000000001,"5min":8.9350000000000005,"15min":8.9350000000000005},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.327,"15min":1.212},"min":{"1min":0.54100000000000004,"5min":0.35199999999999998,"15min":0.32200000000000001},"max":{"1min":1.4850000000000001,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":0.96299999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:55:52 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.466,"15min":1.6919999999999999},"min":{"1min":0.42799999999999999,"5min":0.32900000000000001,"15min":0.30199999999999999},"max":{"1min":1.3740000000000001,"5min":8.0719999999999992,"15min":71.795000000000002},"last":1.167},{"interface":"front","average":{"1min":0.996,"5min":1.768,"15min":1.7929999999999999},"min":{"1min":0.621,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.534,"5min":8.4580000000000002,"15min":72.322000000000003},"last":1.0489999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:55:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.712,"15min":1.399},"min":{"1min":0.71099999999999997,"5min":0.39400000000000002,"15min":0.374},"max":{"1min":1.639,"5min":8.4290000000000003,"15min":8.4659999999999993},"last":1.226},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.6100000000000001,"15min":1.284},"min":{"1min":0.626,"5min":0.374,"15min":0.374},"max":{"1min":1.5629999999999999,"5min":8.8849999999999998,"15min":8.8849999999999998},"last":1.202}]}]},{"osd":1,"up_from":1246,"seq":5351529250916,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":417688,"kb_used_data":11984,"kb_used_omap":204,"kb_used_meta":405491,"kb_avail":93327464,"statfs":{"total":95995035648,"available":95567323136,"internally_reserved":0,"allocated":12271616,"data_stored":10629282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208958,"internal_metadata":415223746},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":2.6269999999999998,"15min":1.974},"min":{"1min":0.42299999999999999,"5min":0.191,"15min":0.191},"max":{"1min":3.593,"5min":58.033000000000001,"15min":58.033000000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.0189999999999999,"5min":2.4670000000000001,"15min":2.2000000000000002},"min":{"1min":0.496,"5min":0.19,"15min":0.19},"max":{"1min":6.4950000000000001,"5min":65.921999999999997,"15min":65.921999999999997},"last":0.72799999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":2.6160000000000001,"15min":1.8560000000000001},"min":{"1min":0.497,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.2490000000000001,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.61599999999999999,"5min":2.4079999999999999,"15min":1.8280000000000001},"min":{"1min":0.36499999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.78500000000000003,"5min":58.682000000000002,"15min":58.682000000000002},"last":0.75800000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.3180000000000001,"15min":1.444},"min":{"1min":0.41199999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.875,"5min":7.6459999999999999,"15min":8.7140000000000004},"last":0.874},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.4350000000000001,"15min":1.3560000000000001},"min":{"1min":0.497,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.419,"5min":7.5780000000000003,"15min":7.5780000000000003},"last":0.79500000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.6339999999999999,"15min":2.2970000000000002},"min":{"1min":0.54400000000000004,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.3200000000000001,"5min":8.1120000000000001,"15min":8.8070000000000004},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.099,"5min":1.665,"15min":2.1539999999999999},"min":{"1min":0.66900000000000004,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.3540000000000001,"5min":10.263,"15min":10.263},"last":1.1020000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.6919999999999999,"15min":1.7529999999999999},"min":{"1min":0.60499999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.399,"5min":10.214,"15min":10.214},"last":1.161},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.599,"15min":1.7569999999999999},"min":{"1min":0.62,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.304,"5min":8.5139999999999993,"15min":8.5139999999999993},"last":0.84299999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":2.0640000000000001,"15min":1.901},"min":{"1min":0.67500000000000004,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.5569999999999999,"5min":9.3309999999999995,"15min":9.3309999999999995},"last":0.94599999999999995},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.9530000000000001,"15min":2.722},"min":{"1min":0.624,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.353,"5min":9.8659999999999997,"15min":9.8659999999999997},"last":1.0669999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.839,"15min":1.665},"min":{"1min":0.70599999999999996,"5min":0.374,"15min":0.374},"max":{"1min":1.385,"5min":9.2929999999999993,"15min":9.2929999999999993},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.016,"5min":1.8069999999999999,"15min":2.157},"min":{"1min":0.61099999999999999,"5min":0.371,"15min":0.371},"max":{"1min":1.385,"5min":9.4529999999999994,"15min":9.4529999999999994},"last":1.2949999999999999}]}]},{"osd":0,"up_from":1136,"seq":4879082848510,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":761776,"kb_used_data":11944,"kb_used_omap":280,"kb_used_meta":749543,"kb_avail":92983376,"statfs":{"total":95995035648,"available":95214977024,"internally_reserved":0,"allocated":12230656,"data_stored":10617972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":767532442},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:55:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":2.302,"15min":1.909},"min":{"1min":0.30599999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":7.617,"5min":69.680000000000007,"15min":69.680000000000007},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":2.331,"15min":2.1160000000000001},"min":{"1min":0.36299999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.6120000000000001,"5min":70.668999999999997,"15min":70.668999999999997},"last":0.89600000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":4.2460000000000004,"15min":2.5859999999999999},"min":{"1min":0.40400000000000003,"5min":0.22,"15min":0.22},"max":{"1min":1.847,"5min":254.89599999999999,"15min":254.89599999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.63500000000000001,"5min":4.6219999999999999,"15min":2.9940000000000002},"min":{"1min":0.41399999999999998,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":1.3100000000000001,"5min":256.85399999999998,"15min":256.85399999999998},"last":0.61799999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.087,"15min":1.079},"min":{"1min":0.435,"5min":0.247,"15min":0.247},"max":{"1min":1.0209999999999999,"5min":8.8309999999999995,"15min":10.475},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.032,"15min":1.071},"min":{"1min":0.53500000000000003,"5min":0.28299999999999997,"15min":0.25800000000000001},"max":{"1min":1.996,"5min":6.4160000000000004,"15min":6.6550000000000002},"last":0.80600000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":1.5109999999999999,"15min":1.4610000000000001},"min":{"1min":0.56499999999999995,"5min":0.318,"15min":0.314},"max":{"1min":1.2430000000000001,"5min":7.3129999999999997,"15min":20.079000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.345,"15min":1.3959999999999999},"min":{"1min":0.65300000000000002,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":1.383,"5min":8.9480000000000004,"15min":12.044},"last":0.79400000000000004}]},{"osd":5,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.4179999999999999,"15min":1.464},"min":{"1min":0.64800000000000002,"5min":0.33000000000000002,"15min":0.28399999999999997},"max":{"1min":1.5089999999999999,"5min":8.8800000000000008,"15min":9.4280000000000008},"last":0.755},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.399,"15min":1.526},"min":{"1min":0.71399999999999997,"5min":0.34399999999999997,"15min":0.29699999999999999},"max":{"1min":1.641,"5min":8.7010000000000005,"15min":9.6289999999999996},"last":0.97599999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.7430000000000001,"15min":1.5309999999999999},"min":{"1min":0.59699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.419,"5min":9.5950000000000006,"15min":10.413},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.6870000000000001,"15min":1.6910000000000001},"min":{"1min":0.83699999999999997,"5min":0.439,"15min":0.35599999999999998},"max":{"1min":1.595,"5min":8.9809999999999999,"15min":9.3260000000000005},"last":0.71999999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.5920000000000001,"15min":1.373},"min":{"1min":0.59599999999999997,"5min":0.38500000000000001,"15min":0.38},"max":{"1min":1.3320000000000001,"5min":7.0149999999999997,"15min":7.0149999999999997},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.077,"5min":1.45,"15min":1.391},"min":{"1min":0.69899999999999995,"5min":0.435,"15min":0.39200000000000002},"max":{"1min":1.9510000000000001,"5min":8.9160000000000004,"15min":10.087999999999999},"last":0.69699999999999995}]}]},{"osd":3,"up_from":0,"seq":2847563317740,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986873,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896868,"kb_used_data":5788,"kb_used_omap":0,"kb_used_meta":891072,"kb_avail":92848284,"statfs":{"total":95995035648,"available":95076642816,"internally_reserved":0,"allocated":5926912,"data_stored":5606565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912457728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.2010000000000001,"15min":1.6000000000000001},"min":{"1min":0.51500000000000001,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":1.2410000000000001,"5min":5.7670000000000003,"15min":64.372},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.369,"15min":1.5429999999999999},"min":{"1min":0.56100000000000005,"5min":0.32400000000000001,"15min":0.27700000000000002},"max":{"1min":1.2829999999999999,"5min":5.7889999999999997,"15min":64.356999999999999},"last":0.96799999999999997}]},{"osd":1,"last update":"Mon Oct 9 09:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":4.3860000000000001,"15min":2.5230000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.27700000000000002},"max":{"1min":1.165,"5min":216.16900000000001,"15min":216.16900000000001},"last":0.627},{"interface":"front","average":{"1min":0.82399999999999995,"5min":4.6740000000000004,"15min":2.1859999999999999},"min":{"1min":0.38200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.1830000000000001,"5min":216.262,"15min":216.262},"last":0.81899999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.1040000000000001,"15min":1.3979999999999999},"min":{"1min":0.38600000000000001,"5min":0.31,"15min":0.29199999999999998},"max":{"1min":1.1519999999999999,"5min":15.853999999999999,"15min":15.853999999999999},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.234,"15min":1.321},"min":{"1min":0.40799999999999997,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":1.4039999999999999,"5min":15.881,"15min":15.881},"last":0.77000000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.018,"15min":1.018},"min":{"1min":0.57099999999999995,"5min":0.36299999999999999,"15min":0.32200000000000001},"max":{"1min":2.7759999999999998,"5min":5.5129999999999999,"15min":8.1120000000000001},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.79900000000000004,"15min":1.119},"min":{"1min":0.69899999999999995,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":1.645,"5min":5.3109999999999999,"15min":7.4180000000000001},"last":0.92600000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.91400000000000003,"15min":1.4079999999999999},"min":{"1min":0.57799999999999996,"5min":0.373,"15min":0.26900000000000002},"max":{"1min":1.329,"5min":5.4660000000000002,"15min":11.763999999999999},"last":0.997},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.113,"15min":1.331},"min":{"1min":0.58799999999999997,"5min":0.41399999999999998,"15min":0.28499999999999998},"max":{"1min":1.528,"5min":5.4790000000000001,"15min":12.446999999999999},"last":1.0169999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:56:06 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.96899999999999997,"15min":1.4399999999999999},"min":{"1min":0.69299999999999995,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":1.2709999999999999,"5min":5.2690000000000001,"15min":11.701000000000001},"last":1.004},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.0009999999999999,"15min":1.5369999999999999},"min":{"1min":0.85399999999999998,"5min":0.40799999999999997,"15min":0.39200000000000002},"max":{"1min":1.3879999999999999,"5min":3.7240000000000002,"15min":11.972},"last":1.038}]},{"osd":7,"last update":"Mon Oct 9 09:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.84999999999999998,"15min":1.2170000000000001},"min":{"1min":0.54000000000000004,"5min":0.45600000000000002,"15min":0.373},"max":{"1min":1.425,"5min":2.4180000000000001,"15min":11.840999999999999},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.04,"15min":1.091},"min":{"1min":0.49299999999999999,"5min":0.48699999999999999,"15min":0.40699999999999997},"max":{"1min":1.2090000000000001,"5min":5.1980000000000004,"15min":9.6210000000000004},"last":0.88600000000000001}]}]},{"osd":5,"up_from":37,"seq":158913790639,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":904084,"kb_used_data":5772,"kb_used_omap":0,"kb_used_meta":898304,"kb_avail":92841068,"statfs":{"total":95995035648,"available":95069253632,"internally_reserved":0,"allocated":5910528,"data_stored":5602258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919863296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:55:43 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":3.4660000000000002,"15min":4.2309999999999999},"min":{"1min":0.49099999999999999,"5min":0.26600000000000001,"15min":0.23599999999999999},"max":{"1min":1.254,"5min":121.977,"15min":274.71899999999999},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.88900000000000001,"5min":3.3530000000000002,"15min":4.0810000000000004},"min":{"1min":0.55900000000000005,"5min":0.33400000000000002,"15min":0.26800000000000002},"max":{"1min":1.1859999999999999,"5min":122.13200000000001,"15min":274.73500000000001},"last":1.125}]},{"osd":1,"last update":"Mon Oct 9 09:55:23 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":3.9249999999999998,"15min":2.5510000000000002},"min":{"1min":0.34699999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.2609999999999999,"5min":192.72200000000001,"15min":192.72200000000001},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.999,"5min":3.9380000000000002,"15min":2.1160000000000001},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.3050000000000002,"5min":192.69300000000001,"15min":192.69300000000001},"last":0.90900000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":2.2469999999999999,"15min":1.911},"min":{"1min":0.72899999999999998,"5min":0.34699999999999998,"15min":0.312},"max":{"1min":1.2170000000000001,"5min":122.02800000000001,"15min":122.02800000000001},"last":0.752},{"interface":"front","average":{"1min":1.0389999999999999,"5min":2.4239999999999999,"15min":2.1320000000000001},"min":{"1min":0.71699999999999997,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":1.284,"5min":122.05800000000001,"15min":122.05800000000001},"last":1.0649999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:55:39 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.78600000000000003,"15min":1.0389999999999999},"min":{"1min":0.60199999999999998,"5min":0.374,"15min":0.33100000000000002},"max":{"1min":1.079,"5min":3.601,"15min":7.2270000000000003},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.91100000000000003,"15min":1.103},"min":{"1min":0.78100000000000003,"5min":0.38,"15min":0.33200000000000002},"max":{"1min":1.2829999999999999,"5min":6.7750000000000004,"15min":16.201000000000001},"last":1.161}]},{"osd":4,"last update":"Mon Oct 9 09:56:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.80800000000000005,"15min":1.2310000000000001},"min":{"1min":0.69099999999999995,"5min":0.35699999999999998,"15min":0.27000000000000002},"max":{"1min":1.3180000000000001,"5min":3.6899999999999999,"15min":9.4209999999999994},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.0960000000000001,"5min":0.95399999999999996,"15min":1.417},"min":{"1min":0.626,"5min":0.318,"15min":0.318},"max":{"1min":1.6499999999999999,"5min":7.601,"15min":16.053000000000001},"last":1.306}]},{"osd":6,"last update":"Mon Oct 9 09:55:39 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.097,"15min":1.8},"min":{"1min":0.67500000000000004,"5min":0.308,"15min":0.308},"max":{"1min":1.242,"5min":7.798,"15min":140.179},"last":0.99199999999999999},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.2,"15min":1.9059999999999999},"min":{"1min":0.71499999999999997,"5min":0.42799999999999999,"15min":0.34399999999999997},"max":{"1min":1.8240000000000001,"5min":7.8070000000000004,"15min":137.928},"last":1.2390000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:56:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":0.93899999999999995,"15min":1.2410000000000001},"min":{"1min":0.57899999999999996,"5min":0.42399999999999999,"15min":0.373},"max":{"1min":1.8879999999999999,"5min":3.3180000000000001,"15min":16.119},"last":1.3759999999999999},{"interface":"front","average":{"1min":1.1080000000000001,"5min":0.92400000000000004,"15min":0.98399999999999999},"min":{"1min":0.59999999999999998,"5min":0.42799999999999999,"15min":0.34200000000000003},"max":{"1min":1.494,"5min":3.6619999999999999,"15min":6.5460000000000003},"last":1.3899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7721,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:56:27.449 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T09:56:27.449 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:56:27.449 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-10-09T09:56:27.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd in 3 2023-10-09T09:56:28.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:28 smithi163 bash[78107]: cluster 2023-10-09T09:56:26.303393+0000 mgr.y (mgr.34107) 1521 : cluster 0 pgmap v2422: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:28.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:28 smithi163 bash[78107]: audit 2023-10-09T09:56:27.373878+0000 mon.a (mon.0) 2647 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:28.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:28 smithi163 bash[78107]: audit 2023-10-09T09:56:27.375240+0000 mon.a (mon.0) 2648 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:56:28.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:28 smithi163 bash[78107]: audit 2023-10-09T09:56:27.381297+0000 mon.a (mon.0) 2649 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:28.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:28 smithi163 bash[75425]: cluster 2023-10-09T09:56:26.303393+0000 mgr.y (mgr.34107) 1521 : cluster 0 pgmap v2422: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:28.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:28 smithi163 bash[75425]: audit 2023-10-09T09:56:27.373878+0000 mon.a (mon.0) 2647 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:28.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:28 smithi163 bash[75425]: audit 2023-10-09T09:56:27.375240+0000 mon.a (mon.0) 2648 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:56:28.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:28 smithi163 bash[75425]: audit 2023-10-09T09:56:27.381297+0000 mon.a (mon.0) 2649 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:28.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:28 smithi181 bash[59345]: cluster 2023-10-09T09:56:26.303393+0000 mgr.y (mgr.34107) 1521 : cluster 0 pgmap v2422: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:28.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:28 smithi181 bash[59345]: audit 2023-10-09T09:56:27.373878+0000 mon.a (mon.0) 2647 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:28.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:28 smithi181 bash[59345]: audit 2023-10-09T09:56:27.375240+0000 mon.a (mon.0) 2648 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:56:28.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:28 smithi181 bash[59345]: audit 2023-10-09T09:56:27.381297+0000 mon.a (mon.0) 2649 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:29.865 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:56:29.866 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:56:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:30 smithi163 bash[78107]: cluster 2023-10-09T09:56:28.304175+0000 mgr.y (mgr.34107) 1522 : cluster 0 pgmap v2423: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:30.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:30 smithi163 bash[75425]: cluster 2023-10-09T09:56:28.304175+0000 mgr.y (mgr.34107) 1522 : cluster 0 pgmap v2423: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:30.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:30 smithi181 bash[59345]: cluster 2023-10-09T09:56:28.304175+0000 mgr.y (mgr.34107) 1522 : cluster 0 pgmap v2423: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:30.771 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:56:31.253 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:32.267 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:32.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:32 smithi163 bash[78107]: cluster 2023-10-09T09:56:30.304839+0000 mgr.y (mgr.34107) 1523 : cluster 0 pgmap v2424: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:32 smithi163 bash[75425]: cluster 2023-10-09T09:56:30.304839+0000 mgr.y (mgr.34107) 1523 : cluster 0 pgmap v2424: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:32.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:32 smithi181 bash[59345]: cluster 2023-10-09T09:56:30.304839+0000 mgr.y (mgr.34107) 1523 : cluster 0 pgmap v2424: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:33.453 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:56:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:56:33] "GET /metrics HTTP/1.1" 200 34031 "" "Prometheus/2.33.4" 2023-10-09T09:56:34.376 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:34 smithi163 bash[75425]: cluster 2023-10-09T09:56:32.305489+0000 mgr.y (mgr.34107) 1524 : cluster 0 pgmap v2425: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:34.376 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:34 smithi163 bash[78107]: cluster 2023-10-09T09:56:32.305489+0000 mgr.y (mgr.34107) 1524 : cluster 0 pgmap v2425: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:34.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:34 smithi181 bash[59345]: cluster 2023-10-09T09:56:32.305489+0000 mgr.y (mgr.34107) 1524 : cluster 0 pgmap v2425: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:35.316 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:36 smithi163 bash[78107]: cluster 2023-10-09T09:56:34.305905+0000 mgr.y (mgr.34107) 1525 : cluster 0 pgmap v2426: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:36 smithi163 bash[75425]: cluster 2023-10-09T09:56:34.305905+0000 mgr.y (mgr.34107) 1525 : cluster 0 pgmap v2426: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:36 smithi181 bash[59345]: cluster 2023-10-09T09:56:34.305905+0000 mgr.y (mgr.34107) 1525 : cluster 0 pgmap v2426: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:38.095 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T09:56:38.368 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:38 smithi163 bash[78107]: cluster 2023-10-09T09:56:36.306440+0000 mgr.y (mgr.34107) 1526 : cluster 0 pgmap v2427: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:38.368 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:38 smithi163 bash[78107]: audit 2023-10-09T09:56:37.464256+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.163:0/3938370406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:56:38.368 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:38 smithi163 bash[78107]: audit 2023-10-09T09:56:37.464720+0000 mon.a (mon.0) 2650 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:56:38.369 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:38 smithi163 bash[75425]: cluster 2023-10-09T09:56:36.306440+0000 mgr.y (mgr.34107) 1526 : cluster 0 pgmap v2427: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:38.369 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:38 smithi163 bash[75425]: audit 2023-10-09T09:56:37.464256+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.163:0/3938370406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:56:38.369 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:38 smithi163 bash[75425]: audit 2023-10-09T09:56:37.464720+0000 mon.a (mon.0) 2650 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:56:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:38 smithi181 bash[59345]: cluster 2023-10-09T09:56:36.306440+0000 mgr.y (mgr.34107) 1526 : cluster 0 pgmap v2427: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:38 smithi181 bash[59345]: audit 2023-10-09T09:56:37.464256+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.163:0/3938370406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:56:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:38 smithi181 bash[59345]: audit 2023-10-09T09:56:37.464720+0000 mon.a (mon.0) 2650 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:56:39.105 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:38 smithi163 bash[75425]: debug 2023-10-09T09:56:38.813+0000 7f22f321a700 -1 mon.a@0(leader).osd e1314 definitely_dead 0 2023-10-09T09:56:39.133 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.3. 2023-10-09T09:56:39.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[78107]: cluster 2023-10-09T09:56:38.080474+0000 mon.a (mon.0) 2651 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:56:39.534 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[78107]: cluster 2023-10-09T09:56:38.080505+0000 mon.a (mon.0) 2652 : cluster 1 Cluster is now healthy 2023-10-09T09:56:39.534 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[78107]: cluster 2023-10-09T09:56:38.084547+0000 osd.5 (osd.5) 905 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:39.534 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[78107]: audit 2023-10-09T09:56:38.090181+0000 mon.a (mon.0) 2653 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:56:39.534 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[78107]: cluster 2023-10-09T09:56:38.090278+0000 mon.a (mon.0) 2654 : cluster 0 osdmap e1314: 8 total, 8 up, 6 in 2023-10-09T09:56:39.534 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[78107]: audit 2023-10-09T09:56:38.816544+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.163:0/3987026693' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T09:56:39.535 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[78107]: audit 2023-10-09T09:56:38.816970+0000 mon.a (mon.0) 2655 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T09:56:39.535 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[75425]: cluster 2023-10-09T09:56:38.080474+0000 mon.a (mon.0) 2651 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:56:39.535 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[75425]: cluster 2023-10-09T09:56:38.080505+0000 mon.a (mon.0) 2652 : cluster 1 Cluster is now healthy 2023-10-09T09:56:39.536 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[75425]: cluster 2023-10-09T09:56:38.084547+0000 osd.5 (osd.5) 905 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:39.536 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[75425]: audit 2023-10-09T09:56:38.090181+0000 mon.a (mon.0) 2653 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:56:39.536 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[75425]: cluster 2023-10-09T09:56:38.090278+0000 mon.a (mon.0) 2654 : cluster 0 osdmap e1314: 8 total, 8 up, 6 in 2023-10-09T09:56:39.536 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[75425]: audit 2023-10-09T09:56:38.816544+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.163:0/3987026693' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T09:56:39.536 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:39 smithi163 bash[75425]: audit 2023-10-09T09:56:38.816970+0000 mon.a (mon.0) 2655 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T09:56:39.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:39 smithi181 bash[59345]: cluster 2023-10-09T09:56:38.080474+0000 mon.a (mon.0) 2651 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:56:39.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:39 smithi181 bash[59345]: cluster 2023-10-09T09:56:38.080505+0000 mon.a (mon.0) 2652 : cluster 1 Cluster is now healthy 2023-10-09T09:56:39.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:39 smithi181 bash[59345]: cluster 2023-10-09T09:56:38.084547+0000 osd.5 (osd.5) 905 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:39.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:39 smithi181 bash[59345]: audit 2023-10-09T09:56:38.090181+0000 mon.a (mon.0) 2653 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:56:39.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:39 smithi181 bash[59345]: cluster 2023-10-09T09:56:38.090278+0000 mon.a (mon.0) 2654 : cluster 0 osdmap e1314: 8 total, 8 up, 6 in 2023-10-09T09:56:39.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:39 smithi181 bash[59345]: audit 2023-10-09T09:56:38.816544+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.163:0/3987026693' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T09:56:39.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:39 smithi181 bash[59345]: audit 2023-10-09T09:56:38.816970+0000 mon.a (mon.0) 2655 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T09:56:40.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[78107]: cluster 2023-10-09T09:56:38.306931+0000 mgr.y (mgr.34107) 1527 : cluster 0 pgmap v2429: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:40.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[78107]: cluster 2023-10-09T09:56:39.059677+0000 osd.4 (osd.4) 696 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.541 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[78107]: cluster 2023-10-09T09:56:39.105337+0000 osd.5 (osd.5) 906 : cluster 3 failed to encode map e1315 with expected crc 2023-10-09T09:56:40.541 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[78107]: cluster 2023-10-09T09:56:39.110732+0000 osd.5 (osd.5) 907 : cluster 3 failed to encode map e1315 with expected crc 2023-10-09T09:56:40.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[78107]: cluster 2023-10-09T09:56:39.110742+0000 osd.4 (osd.4) 697 : cluster 3 failed to encode map e1315 with expected crc 2023-10-09T09:56:40.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[78107]: cluster 2023-10-09T09:56:39.111494+0000 osd.6 (osd.6) 592 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[78107]: audit 2023-10-09T09:56:39.112127+0000 mon.a (mon.0) 2656 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T09:56:40.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[78107]: cluster 2023-10-09T09:56:39.112236+0000 mon.a (mon.0) 2657 : cluster 0 osdmap e1315: 8 total, 8 up, 7 in 2023-10-09T09:56:40.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[78107]: cluster 2023-10-09T09:56:39.115841+0000 osd.6 (osd.6) 593 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[78107]: cluster 2023-10-09T09:56:39.839116+0000 osd.7 (osd.7) 638 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[75425]: cluster 2023-10-09T09:56:38.306931+0000 mgr.y (mgr.34107) 1527 : cluster 0 pgmap v2429: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[75425]: cluster 2023-10-09T09:56:39.059677+0000 osd.4 (osd.4) 696 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[75425]: cluster 2023-10-09T09:56:39.105337+0000 osd.5 (osd.5) 906 : cluster 3 failed to encode map e1315 with expected crc 2023-10-09T09:56:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[75425]: cluster 2023-10-09T09:56:39.110732+0000 osd.5 (osd.5) 907 : cluster 3 failed to encode map e1315 with expected crc 2023-10-09T09:56:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[75425]: cluster 2023-10-09T09:56:39.110742+0000 osd.4 (osd.4) 697 : cluster 3 failed to encode map e1315 with expected crc 2023-10-09T09:56:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[75425]: cluster 2023-10-09T09:56:39.111494+0000 osd.6 (osd.6) 592 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[75425]: audit 2023-10-09T09:56:39.112127+0000 mon.a (mon.0) 2656 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T09:56:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[75425]: cluster 2023-10-09T09:56:39.112236+0000 mon.a (mon.0) 2657 : cluster 0 osdmap e1315: 8 total, 8 up, 7 in 2023-10-09T09:56:40.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[75425]: cluster 2023-10-09T09:56:39.115841+0000 osd.6 (osd.6) 593 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:40 smithi163 bash[75425]: cluster 2023-10-09T09:56:39.839116+0000 osd.7 (osd.7) 638 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:40 smithi181 bash[59345]: cluster 2023-10-09T09:56:38.306931+0000 mgr.y (mgr.34107) 1527 : cluster 0 pgmap v2429: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-10-09T09:56:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:40 smithi181 bash[59345]: cluster 2023-10-09T09:56:39.059677+0000 osd.4 (osd.4) 696 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:40 smithi181 bash[59345]: cluster 2023-10-09T09:56:39.105337+0000 osd.5 (osd.5) 906 : cluster 3 failed to encode map e1315 with expected crc 2023-10-09T09:56:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:40 smithi181 bash[59345]: cluster 2023-10-09T09:56:39.110732+0000 osd.5 (osd.5) 907 : cluster 3 failed to encode map e1315 with expected crc 2023-10-09T09:56:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:40 smithi181 bash[59345]: cluster 2023-10-09T09:56:39.110742+0000 osd.4 (osd.4) 697 : cluster 3 failed to encode map e1315 with expected crc 2023-10-09T09:56:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:40 smithi181 bash[59345]: cluster 2023-10-09T09:56:39.111494+0000 osd.6 (osd.6) 592 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:40 smithi181 bash[59345]: audit 2023-10-09T09:56:39.112127+0000 mon.a (mon.0) 2656 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T09:56:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:40 smithi181 bash[59345]: cluster 2023-10-09T09:56:39.112236+0000 mon.a (mon.0) 2657 : cluster 0 osdmap e1315: 8 total, 8 up, 7 in 2023-10-09T09:56:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:40 smithi181 bash[59345]: cluster 2023-10-09T09:56:39.115841+0000 osd.6 (osd.6) 593 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:40.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:40 smithi181 bash[59345]: cluster 2023-10-09T09:56:39.839116+0000 osd.7 (osd.7) 638 : cluster 3 failed to encode map e1314 with expected crc 2023-10-09T09:56:41.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[75425]: cluster 2023-10-09T09:56:40.104954+0000 osd.5 (osd.5) 908 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[75425]: cluster 2023-10-09T09:56:40.105367+0000 osd.6 (osd.6) 594 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[75425]: cluster 2023-10-09T09:56:40.108585+0000 mon.a (mon.0) 2658 : cluster 0 osdmap e1316: 8 total, 8 up, 7 in 2023-10-09T09:56:41.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[75425]: cluster 2023-10-09T09:56:40.109216+0000 osd.5 (osd.5) 909 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[75425]: cluster 2023-10-09T09:56:40.109650+0000 osd.4 (osd.4) 698 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[75425]: cluster 2023-10-09T09:56:40.114393+0000 osd.4 (osd.4) 699 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[75425]: cluster 2023-10-09T09:56:40.117817+0000 osd.4 (osd.4) 700 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[75425]: audit 2023-10-09T09:56:40.425851+0000 mon.a (mon.0) 2659 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:41.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[75425]: audit 2023-10-09T09:56:40.426972+0000 mon.a (mon.0) 2660 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:41.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[78107]: cluster 2023-10-09T09:56:40.104954+0000 osd.5 (osd.5) 908 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[78107]: cluster 2023-10-09T09:56:40.105367+0000 osd.6 (osd.6) 594 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[78107]: cluster 2023-10-09T09:56:40.108585+0000 mon.a (mon.0) 2658 : cluster 0 osdmap e1316: 8 total, 8 up, 7 in 2023-10-09T09:56:41.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[78107]: cluster 2023-10-09T09:56:40.109216+0000 osd.5 (osd.5) 909 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[78107]: cluster 2023-10-09T09:56:40.109650+0000 osd.4 (osd.4) 698 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[78107]: cluster 2023-10-09T09:56:40.114393+0000 osd.4 (osd.4) 699 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[78107]: cluster 2023-10-09T09:56:40.117817+0000 osd.4 (osd.4) 700 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[78107]: audit 2023-10-09T09:56:40.425851+0000 mon.a (mon.0) 2659 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:41.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:41 smithi163 bash[78107]: audit 2023-10-09T09:56:40.426972+0000 mon.a (mon.0) 2660 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:41 smithi181 bash[59345]: cluster 2023-10-09T09:56:40.104954+0000 osd.5 (osd.5) 908 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:41 smithi181 bash[59345]: cluster 2023-10-09T09:56:40.105367+0000 osd.6 (osd.6) 594 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:41 smithi181 bash[59345]: cluster 2023-10-09T09:56:40.108585+0000 mon.a (mon.0) 2658 : cluster 0 osdmap e1316: 8 total, 8 up, 7 in 2023-10-09T09:56:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:41 smithi181 bash[59345]: cluster 2023-10-09T09:56:40.109216+0000 osd.5 (osd.5) 909 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:41 smithi181 bash[59345]: cluster 2023-10-09T09:56:40.109650+0000 osd.4 (osd.4) 698 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:41 smithi181 bash[59345]: cluster 2023-10-09T09:56:40.114393+0000 osd.4 (osd.4) 699 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:41 smithi181 bash[59345]: cluster 2023-10-09T09:56:40.117817+0000 osd.4 (osd.4) 700 : cluster 3 failed to encode map e1316 with expected crc 2023-10-09T09:56:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:41 smithi181 bash[59345]: audit 2023-10-09T09:56:40.425851+0000 mon.a (mon.0) 2659 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:41.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:41 smithi181 bash[59345]: audit 2023-10-09T09:56:40.426972+0000 mon.a (mon.0) 2660 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:41.744 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-10-09T09:56:42.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[75425]: cluster 2023-10-09T09:56:40.307573+0000 mgr.y (mgr.34107) 1528 : cluster 0 pgmap v2432: 73 pgs: 9 peering, 64 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:56:42.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[75425]: cluster 2023-10-09T09:56:41.112039+0000 mon.a (mon.0) 2661 : cluster 3 Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T09:56:42.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[75425]: cluster 2023-10-09T09:56:41.124461+0000 osd.5 (osd.5) 910 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:42.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[75425]: cluster 2023-10-09T09:56:41.126861+0000 mon.a (mon.0) 2662 : cluster 0 osdmap e1317: 8 total, 8 up, 7 in 2023-10-09T09:56:42.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[75425]: cluster 2023-10-09T09:56:41.131232+0000 osd.6 (osd.6) 595 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:42.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[75425]: cluster 2023-10-09T09:56:41.420810+0000 osd.7 (osd.7) 639 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:42.395 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[78107]: cluster 2023-10-09T09:56:40.307573+0000 mgr.y (mgr.34107) 1528 : cluster 0 pgmap v2432: 73 pgs: 9 peering, 64 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:56:42.395 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[78107]: cluster 2023-10-09T09:56:41.112039+0000 mon.a (mon.0) 2661 : cluster 3 Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T09:56:42.395 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[78107]: cluster 2023-10-09T09:56:41.124461+0000 osd.5 (osd.5) 910 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:42.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[78107]: cluster 2023-10-09T09:56:41.126861+0000 mon.a (mon.0) 2662 : cluster 0 osdmap e1317: 8 total, 8 up, 7 in 2023-10-09T09:56:42.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[78107]: cluster 2023-10-09T09:56:41.131232+0000 osd.6 (osd.6) 595 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:42.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:42 smithi163 bash[78107]: cluster 2023-10-09T09:56:41.420810+0000 osd.7 (osd.7) 639 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:42 smithi181 bash[59345]: cluster 2023-10-09T09:56:40.307573+0000 mgr.y (mgr.34107) 1528 : cluster 0 pgmap v2432: 73 pgs: 9 peering, 64 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:56:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:42 smithi181 bash[59345]: cluster 2023-10-09T09:56:41.112039+0000 mon.a (mon.0) 2661 : cluster 3 Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T09:56:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:42 smithi181 bash[59345]: cluster 2023-10-09T09:56:41.124461+0000 osd.5 (osd.5) 910 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:42 smithi181 bash[59345]: cluster 2023-10-09T09:56:41.126861+0000 mon.a (mon.0) 2662 : cluster 0 osdmap e1317: 8 total, 8 up, 7 in 2023-10-09T09:56:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:42 smithi181 bash[59345]: cluster 2023-10-09T09:56:41.131232+0000 osd.6 (osd.6) 595 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:42.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:42 smithi181 bash[59345]: cluster 2023-10-09T09:56:41.420810+0000 osd.7 (osd.7) 639 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:42.579 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:56:42.579 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:56:43.467 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:56:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:56:43] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.33.4" 2023-10-09T09:56:43.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:43 smithi163 bash[78107]: cluster 2023-10-09T09:56:41.420788+0000 osd.4 (osd.4) 701 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:43.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:43 smithi163 bash[78107]: cluster 2023-10-09T09:56:42.130121+0000 osd.5 (osd.5) 911 : cluster 3 failed to encode map e1318 with expected crc 2023-10-09T09:56:43.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:43 smithi163 bash[78107]: cluster 2023-10-09T09:56:42.131766+0000 mon.a (mon.0) 2663 : cluster 0 osdmap e1318: 8 total, 8 up, 7 in 2023-10-09T09:56:43.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:43 smithi163 bash[75425]: cluster 2023-10-09T09:56:41.420788+0000 osd.4 (osd.4) 701 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:43.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:43 smithi163 bash[75425]: cluster 2023-10-09T09:56:42.130121+0000 osd.5 (osd.5) 911 : cluster 3 failed to encode map e1318 with expected crc 2023-10-09T09:56:43.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:43 smithi163 bash[75425]: cluster 2023-10-09T09:56:42.131766+0000 mon.a (mon.0) 2663 : cluster 0 osdmap e1318: 8 total, 8 up, 7 in 2023-10-09T09:56:43.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T09:56:43.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:43 smithi181 bash[59345]: cluster 2023-10-09T09:56:41.420788+0000 osd.4 (osd.4) 701 : cluster 3 failed to encode map e1317 with expected crc 2023-10-09T09:56:43.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:43 smithi181 bash[59345]: cluster 2023-10-09T09:56:42.130121+0000 osd.5 (osd.5) 911 : cluster 3 failed to encode map e1318 with expected crc 2023-10-09T09:56:43.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:43 smithi181 bash[59345]: cluster 2023-10-09T09:56:42.131766+0000 mon.a (mon.0) 2663 : cluster 0 osdmap e1318: 8 total, 8 up, 7 in 2023-10-09T09:56:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:44 smithi163 bash[78107]: cluster 2023-10-09T09:56:42.308640+0000 mgr.y (mgr.34107) 1529 : cluster 0 pgmap v2435: 73 pgs: 2 active+undersized+remapped, 9 peering, 62 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 17 B/s, 7 objects/s recovering 2023-10-09T09:56:44.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:44 smithi163 bash[78107]: cluster 2023-10-09T09:56:43.135974+0000 osd.5 (osd.5) 912 : cluster 3 failed to encode map e1319 with expected crc 2023-10-09T09:56:44.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:44 smithi163 bash[78107]: cluster 2023-10-09T09:56:43.139531+0000 osd.5 (osd.5) 913 : cluster 3 failed to encode map e1319 with expected crc 2023-10-09T09:56:44.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:44 smithi163 bash[78107]: cluster 2023-10-09T09:56:43.140133+0000 mon.a (mon.0) 2664 : cluster 0 osdmap e1319: 8 total, 8 up, 7 in 2023-10-09T09:56:44.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:44 smithi163 bash[75425]: cluster 2023-10-09T09:56:42.308640+0000 mgr.y (mgr.34107) 1529 : cluster 0 pgmap v2435: 73 pgs: 2 active+undersized+remapped, 9 peering, 62 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 17 B/s, 7 objects/s recovering 2023-10-09T09:56:44.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:44 smithi163 bash[75425]: cluster 2023-10-09T09:56:43.135974+0000 osd.5 (osd.5) 912 : cluster 3 failed to encode map e1319 with expected crc 2023-10-09T09:56:44.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:44 smithi163 bash[75425]: cluster 2023-10-09T09:56:43.139531+0000 osd.5 (osd.5) 913 : cluster 3 failed to encode map e1319 with expected crc 2023-10-09T09:56:44.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:44 smithi163 bash[75425]: cluster 2023-10-09T09:56:43.140133+0000 mon.a (mon.0) 2664 : cluster 0 osdmap e1319: 8 total, 8 up, 7 in 2023-10-09T09:56:44.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:44 smithi181 bash[59345]: cluster 2023-10-09T09:56:42.308640+0000 mgr.y (mgr.34107) 1529 : cluster 0 pgmap v2435: 73 pgs: 2 active+undersized+remapped, 9 peering, 62 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 17 B/s, 7 objects/s recovering 2023-10-09T09:56:44.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:44 smithi181 bash[59345]: cluster 2023-10-09T09:56:43.135974+0000 osd.5 (osd.5) 912 : cluster 3 failed to encode map e1319 with expected crc 2023-10-09T09:56:44.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:44 smithi181 bash[59345]: cluster 2023-10-09T09:56:43.139531+0000 osd.5 (osd.5) 913 : cluster 3 failed to encode map e1319 with expected crc 2023-10-09T09:56:44.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:44 smithi181 bash[59345]: cluster 2023-10-09T09:56:43.140133+0000 mon.a (mon.0) 2664 : cluster 0 osdmap e1319: 8 total, 8 up, 7 in 2023-10-09T09:56:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:45 smithi163 bash[78107]: cluster 2023-10-09T09:56:44.309461+0000 mgr.y (mgr.34107) 1530 : cluster 0 pgmap v2437: 73 pgs: 2 active+undersized+remapped, 71 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 38 B/s, 8 objects/s recovering 2023-10-09T09:56:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:45 smithi163 bash[78107]: cluster 2023-10-09T09:56:45.136977+0000 mon.a (mon.0) 2665 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T09:56:45.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:45 smithi163 bash[78107]: cluster 2023-10-09T09:56:45.137031+0000 mon.a (mon.0) 2666 : cluster 1 Cluster is now healthy 2023-10-09T09:56:45.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:45 smithi163 bash[75425]: cluster 2023-10-09T09:56:44.309461+0000 mgr.y (mgr.34107) 1530 : cluster 0 pgmap v2437: 73 pgs: 2 active+undersized+remapped, 71 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 38 B/s, 8 objects/s recovering 2023-10-09T09:56:45.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:45 smithi163 bash[75425]: cluster 2023-10-09T09:56:45.136977+0000 mon.a (mon.0) 2665 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T09:56:45.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:45 smithi163 bash[75425]: cluster 2023-10-09T09:56:45.137031+0000 mon.a (mon.0) 2666 : cluster 1 Cluster is now healthy 2023-10-09T09:56:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:45 smithi181 bash[59345]: cluster 2023-10-09T09:56:44.309461+0000 mgr.y (mgr.34107) 1530 : cluster 0 pgmap v2437: 73 pgs: 2 active+undersized+remapped, 71 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 38 B/s, 8 objects/s recovering 2023-10-09T09:56:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:45 smithi181 bash[59345]: cluster 2023-10-09T09:56:45.136977+0000 mon.a (mon.0) 2665 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T09:56:45.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:45 smithi181 bash[59345]: cluster 2023-10-09T09:56:45.137031+0000 mon.a (mon.0) 2666 : cluster 1 Cluster is now healthy 2023-10-09T09:56:46.745 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T09:56:46.746 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T09:56:46.746 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T09:56:48.257 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:48.288 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:48 smithi163 bash[78107]: cluster 2023-10-09T09:56:46.310344+0000 mgr.y (mgr.34107) 1531 : cluster 0 pgmap v2438: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 26 B/s, 5 objects/s recovering 2023-10-09T09:56:48.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:48 smithi163 bash[75425]: cluster 2023-10-09T09:56:46.310344+0000 mgr.y (mgr.34107) 1531 : cluster 0 pgmap v2438: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 26 B/s, 5 objects/s recovering 2023-10-09T09:56:48.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:48 smithi181 bash[59345]: cluster 2023-10-09T09:56:46.310344+0000 mgr.y (mgr.34107) 1531 : cluster 0 pgmap v2438: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 26 B/s, 5 objects/s recovering 2023-10-09T09:56:50.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:50 smithi163 bash[78107]: cluster 2023-10-09T09:56:48.311443+0000 mgr.y (mgr.34107) 1532 : cluster 0 pgmap v2439: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 22 B/s, 5 objects/s recovering 2023-10-09T09:56:50.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:50 smithi163 bash[75425]: cluster 2023-10-09T09:56:48.311443+0000 mgr.y (mgr.34107) 1532 : cluster 0 pgmap v2439: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 22 B/s, 5 objects/s recovering 2023-10-09T09:56:50.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:50 smithi181 bash[59345]: cluster 2023-10-09T09:56:48.311443+0000 mgr.y (mgr.34107) 1532 : cluster 0 pgmap v2439: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 22 B/s, 5 objects/s recovering 2023-10-09T09:56:51.306 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:52.115 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:56:52.116 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:56:52.203 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:52 smithi163 bash[75425]: cluster 2023-10-09T09:56:50.312310+0000 mgr.y (mgr.34107) 1533 : cluster 0 pgmap v2440: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 10 B/s, 1 objects/s recovering 2023-10-09T09:56:52.203 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:52 smithi163 bash[75425]: audit 2023-10-09T09:56:51.261136+0000 mon.a (mon.0) 2667 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:52.203 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:52 smithi163 bash[78107]: cluster 2023-10-09T09:56:50.312310+0000 mgr.y (mgr.34107) 1533 : cluster 0 pgmap v2440: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 10 B/s, 1 objects/s recovering 2023-10-09T09:56:52.203 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:52 smithi163 bash[78107]: audit 2023-10-09T09:56:51.261136+0000 mon.a (mon.0) 2667 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:52 smithi181 bash[59345]: cluster 2023-10-09T09:56:50.312310+0000 mgr.y (mgr.34107) 1533 : cluster 0 pgmap v2440: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 10 B/s, 1 objects/s recovering 2023-10-09T09:56:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:52 smithi181 bash[59345]: audit 2023-10-09T09:56:51.261136+0000 mon.a (mon.0) 2667 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:56:53.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T09:56:53.718 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:56:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:56:53] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.33.4" 2023-10-09T09:56:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:54 smithi181 bash[59345]: cluster 2023-10-09T09:56:52.313371+0000 mgr.y (mgr.34107) 1534 : cluster 0 pgmap v2441: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T09:56:54.384 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:54 smithi163 bash[78107]: cluster 2023-10-09T09:56:52.313371+0000 mgr.y (mgr.34107) 1534 : cluster 0 pgmap v2441: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T09:56:54.384 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:54 smithi163 bash[75425]: cluster 2023-10-09T09:56:52.313371+0000 mgr.y (mgr.34107) 1534 : cluster 0 pgmap v2441: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T09:56:56.071 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.7. 2023-10-09T09:56:56.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:55 smithi163 bash[75425]: debug 2023-10-09T09:56:55.897+0000 7f22f321a700 -1 mon.a@0(leader).osd e1319 definitely_dead 0 2023-10-09T09:56:56.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:56 smithi163 bash[75425]: cluster 2023-10-09T09:56:54.313966+0000 mgr.y (mgr.34107) 1535 : cluster 0 pgmap v2442: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T09:56:56.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:56 smithi163 bash[75425]: audit 2023-10-09T09:56:55.420552+0000 mon.a (mon.0) 2668 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:56.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:56 smithi163 bash[75425]: audit 2023-10-09T09:56:55.900212+0000 mon.a (mon.0) 2669 : audit 0 from='client.? 172.21.15.163:0/1751849406' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T09:56:56.155 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:56 smithi163 bash[78107]: cluster 2023-10-09T09:56:54.313966+0000 mgr.y (mgr.34107) 1535 : cluster 0 pgmap v2442: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T09:56:56.155 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:56 smithi163 bash[78107]: audit 2023-10-09T09:56:55.420552+0000 mon.a (mon.0) 2668 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:56.155 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:56 smithi163 bash[78107]: audit 2023-10-09T09:56:55.900212+0000 mon.a (mon.0) 2669 : audit 0 from='client.? 172.21.15.163:0/1751849406' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T09:56:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:56 smithi181 bash[59345]: cluster 2023-10-09T09:56:54.313966+0000 mgr.y (mgr.34107) 1535 : cluster 0 pgmap v2442: 73 pgs: 73 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T09:56:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:56 smithi181 bash[59345]: audit 2023-10-09T09:56:55.420552+0000 mon.a (mon.0) 2668 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:56.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:56 smithi181 bash[59345]: audit 2023-10-09T09:56:55.900212+0000 mon.a (mon.0) 2669 : audit 0 from='client.? 172.21.15.163:0/1751849406' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T09:56:56.965 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T09:56:56.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T09:56:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:57 smithi163 bash[75425]: cluster 2023-10-09T09:56:56.052742+0000 osd.5 (osd.5) 914 : cluster 3 failed to encode map e1320 with expected crc 2023-10-09T09:56:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:57 smithi163 bash[75425]: cluster 2023-10-09T09:56:56.053088+0000 osd.6 (osd.6) 596 : cluster 3 failed to encode map e1320 with expected crc 2023-10-09T09:56:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:57 smithi163 bash[75425]: audit 2023-10-09T09:56:56.054721+0000 mon.a (mon.0) 2670 : audit 1 from='client.? 172.21.15.163:0/1751849406' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T09:56:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:57 smithi163 bash[75425]: cluster 2023-10-09T09:56:56.054789+0000 mon.a (mon.0) 2671 : cluster 0 osdmap e1320: 8 total, 8 up, 8 in 2023-10-09T09:56:57.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:57 smithi163 bash[78107]: cluster 2023-10-09T09:56:56.052742+0000 osd.5 (osd.5) 914 : cluster 3 failed to encode map e1320 with expected crc 2023-10-09T09:56:57.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:57 smithi163 bash[78107]: cluster 2023-10-09T09:56:56.053088+0000 osd.6 (osd.6) 596 : cluster 3 failed to encode map e1320 with expected crc 2023-10-09T09:56:57.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:57 smithi163 bash[78107]: audit 2023-10-09T09:56:56.054721+0000 mon.a (mon.0) 2670 : audit 1 from='client.? 172.21.15.163:0/1751849406' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T09:56:57.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:57 smithi163 bash[78107]: cluster 2023-10-09T09:56:56.054789+0000 mon.a (mon.0) 2671 : cluster 0 osdmap e1320: 8 total, 8 up, 8 in 2023-10-09T09:56:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:57 smithi181 bash[59345]: cluster 2023-10-09T09:56:56.052742+0000 osd.5 (osd.5) 914 : cluster 3 failed to encode map e1320 with expected crc 2023-10-09T09:56:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:57 smithi181 bash[59345]: cluster 2023-10-09T09:56:56.053088+0000 osd.6 (osd.6) 596 : cluster 3 failed to encode map e1320 with expected crc 2023-10-09T09:56:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:57 smithi181 bash[59345]: audit 2023-10-09T09:56:56.054721+0000 mon.a (mon.0) 2670 : audit 1 from='client.? 172.21.15.163:0/1751849406' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T09:56:57.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:57 smithi181 bash[59345]: cluster 2023-10-09T09:56:56.054789+0000 mon.a (mon.0) 2671 : cluster 0 osdmap e1320: 8 total, 8 up, 8 in 2023-10-09T09:56:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[75425]: cluster 2023-10-09T09:56:56.314636+0000 mgr.y (mgr.34107) 1536 : cluster 0 pgmap v2444: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:56:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[75425]: cluster 2023-10-09T09:56:57.060737+0000 osd.5 (osd.5) 915 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[75425]: cluster 2023-10-09T09:56:57.061183+0000 osd.6 (osd.6) 597 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[75425]: cluster 2023-10-09T09:56:57.063601+0000 mon.a (mon.0) 2672 : cluster 0 osdmap e1321: 8 total, 8 up, 8 in 2023-10-09T09:56:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[75425]: cluster 2023-10-09T09:56:57.064029+0000 osd.5 (osd.5) 916 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[75425]: cluster 2023-10-09T09:56:57.064561+0000 osd.7 (osd.7) 640 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[75425]: cluster 2023-10-09T09:56:57.064781+0000 osd.6 (osd.6) 598 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[75425]: cluster 2023-10-09T09:56:57.066865+0000 osd.6 (osd.6) 599 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[75425]: cluster 2023-10-09T09:56:57.072981+0000 osd.4 (osd.4) 702 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[78107]: cluster 2023-10-09T09:56:56.314636+0000 mgr.y (mgr.34107) 1536 : cluster 0 pgmap v2444: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:56:58.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[78107]: cluster 2023-10-09T09:56:57.060737+0000 osd.5 (osd.5) 915 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[78107]: cluster 2023-10-09T09:56:57.061183+0000 osd.6 (osd.6) 597 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[78107]: cluster 2023-10-09T09:56:57.063601+0000 mon.a (mon.0) 2672 : cluster 0 osdmap e1321: 8 total, 8 up, 8 in 2023-10-09T09:56:58.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[78107]: cluster 2023-10-09T09:56:57.064029+0000 osd.5 (osd.5) 916 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[78107]: cluster 2023-10-09T09:56:57.064561+0000 osd.7 (osd.7) 640 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[78107]: cluster 2023-10-09T09:56:57.064781+0000 osd.6 (osd.6) 598 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[78107]: cluster 2023-10-09T09:56:57.066865+0000 osd.6 (osd.6) 599 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:58 smithi163 bash[78107]: cluster 2023-10-09T09:56:57.072981+0000 osd.4 (osd.4) 702 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:58 smithi181 bash[59345]: cluster 2023-10-09T09:56:56.314636+0000 mgr.y (mgr.34107) 1536 : cluster 0 pgmap v2444: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2023-10-09T09:56:58.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:58 smithi181 bash[59345]: cluster 2023-10-09T09:56:57.060737+0000 osd.5 (osd.5) 915 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:58 smithi181 bash[59345]: cluster 2023-10-09T09:56:57.061183+0000 osd.6 (osd.6) 597 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:58 smithi181 bash[59345]: cluster 2023-10-09T09:56:57.063601+0000 mon.a (mon.0) 2672 : cluster 0 osdmap e1321: 8 total, 8 up, 8 in 2023-10-09T09:56:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:58 smithi181 bash[59345]: cluster 2023-10-09T09:56:57.064029+0000 osd.5 (osd.5) 916 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:58 smithi181 bash[59345]: cluster 2023-10-09T09:56:57.064561+0000 osd.7 (osd.7) 640 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:58 smithi181 bash[59345]: cluster 2023-10-09T09:56:57.064781+0000 osd.6 (osd.6) 598 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:58 smithi181 bash[59345]: cluster 2023-10-09T09:56:57.066865+0000 osd.6 (osd.6) 599 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:58.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:58 smithi181 bash[59345]: cluster 2023-10-09T09:56:57.072981+0000 osd.4 (osd.4) 702 : cluster 3 failed to encode map e1321 with expected crc 2023-10-09T09:56:59.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:59 smithi163 bash[78107]: cluster 2023-10-09T09:56:58.063230+0000 osd.5 (osd.5) 917 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:59 smithi163 bash[78107]: cluster 2023-10-09T09:56:58.067088+0000 mon.a (mon.0) 2673 : cluster 0 osdmap e1322: 8 total, 8 up, 8 in 2023-10-09T09:56:59.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:59 smithi163 bash[78107]: cluster 2023-10-09T09:56:58.074502+0000 osd.4 (osd.4) 703 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:59 smithi163 bash[78107]: cluster 2023-10-09T09:56:58.074553+0000 osd.6 (osd.6) 600 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:56:59 smithi163 bash[78107]: cluster 2023-10-09T09:56:58.321390+0000 osd.7 (osd.7) 641 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:59 smithi163 bash[75425]: cluster 2023-10-09T09:56:58.063230+0000 osd.5 (osd.5) 917 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:59 smithi163 bash[75425]: cluster 2023-10-09T09:56:58.067088+0000 mon.a (mon.0) 2673 : cluster 0 osdmap e1322: 8 total, 8 up, 8 in 2023-10-09T09:56:59.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:59 smithi163 bash[75425]: cluster 2023-10-09T09:56:58.074502+0000 osd.4 (osd.4) 703 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:59 smithi163 bash[75425]: cluster 2023-10-09T09:56:58.074553+0000 osd.6 (osd.6) 600 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:56:59 smithi163 bash[75425]: cluster 2023-10-09T09:56:58.321390+0000 osd.7 (osd.7) 641 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.473 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:56:59.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:59 smithi181 bash[59345]: cluster 2023-10-09T09:56:58.063230+0000 osd.5 (osd.5) 917 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:59 smithi181 bash[59345]: cluster 2023-10-09T09:56:58.067088+0000 mon.a (mon.0) 2673 : cluster 0 osdmap e1322: 8 total, 8 up, 8 in 2023-10-09T09:56:59.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:59 smithi181 bash[59345]: cluster 2023-10-09T09:56:58.074502+0000 osd.4 (osd.4) 703 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:59 smithi181 bash[59345]: cluster 2023-10-09T09:56:58.074553+0000 osd.6 (osd.6) 600 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:56:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:56:59 smithi181 bash[59345]: cluster 2023-10-09T09:56:58.321390+0000 osd.7 (osd.7) 641 : cluster 3 failed to encode map e1322 with expected crc 2023-10-09T09:57:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[78107]: cluster 2023-10-09T09:56:58.315253+0000 mgr.y (mgr.34107) 1537 : cluster 0 pgmap v2447: 73 pgs: 21 peering, 52 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[78107]: cluster 2023-10-09T09:56:59.084295+0000 osd.5 (osd.5) 918 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[78107]: cluster 2023-10-09T09:56:59.087734+0000 mon.a (mon.0) 2674 : cluster 0 osdmap e1323: 8 total, 8 up, 8 in 2023-10-09T09:57:00.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[78107]: cluster 2023-10-09T09:56:59.088257+0000 osd.5 (osd.5) 919 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[78107]: cluster 2023-10-09T09:56:59.088427+0000 osd.7 (osd.7) 642 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[78107]: cluster 2023-10-09T09:56:59.090404+0000 osd.7 (osd.7) 643 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[75425]: cluster 2023-10-09T09:56:58.315253+0000 mgr.y (mgr.34107) 1537 : cluster 0 pgmap v2447: 73 pgs: 21 peering, 52 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:00.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[75425]: cluster 2023-10-09T09:56:59.084295+0000 osd.5 (osd.5) 918 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[75425]: cluster 2023-10-09T09:56:59.087734+0000 mon.a (mon.0) 2674 : cluster 0 osdmap e1323: 8 total, 8 up, 8 in 2023-10-09T09:57:00.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[75425]: cluster 2023-10-09T09:56:59.088257+0000 osd.5 (osd.5) 919 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[75425]: cluster 2023-10-09T09:56:59.088427+0000 osd.7 (osd.7) 642 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:00 smithi163 bash[75425]: cluster 2023-10-09T09:56:59.090404+0000 osd.7 (osd.7) 643 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:00 smithi181 bash[59345]: cluster 2023-10-09T09:56:58.315253+0000 mgr.y (mgr.34107) 1537 : cluster 0 pgmap v2447: 73 pgs: 21 peering, 52 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:00.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:00 smithi181 bash[59345]: cluster 2023-10-09T09:56:59.084295+0000 osd.5 (osd.5) 918 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:00 smithi181 bash[59345]: cluster 2023-10-09T09:56:59.087734+0000 mon.a (mon.0) 2674 : cluster 0 osdmap e1323: 8 total, 8 up, 8 in 2023-10-09T09:57:00.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:00 smithi181 bash[59345]: cluster 2023-10-09T09:56:59.088257+0000 osd.5 (osd.5) 919 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:00 smithi181 bash[59345]: cluster 2023-10-09T09:56:59.088427+0000 osd.7 (osd.7) 642 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:00.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:00 smithi181 bash[59345]: cluster 2023-10-09T09:56:59.090404+0000 osd.7 (osd.7) 643 : cluster 3 failed to encode map e1323 with expected crc 2023-10-09T09:57:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[75425]: cluster 2023-10-09T09:57:00.096773+0000 osd.5 (osd.5) 920 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[75425]: cluster 2023-10-09T09:57:00.099233+0000 mon.a (mon.0) 2675 : cluster 0 osdmap e1324: 8 total, 8 up, 8 in 2023-10-09T09:57:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[75425]: cluster 2023-10-09T09:57:00.099317+0000 osd.5 (osd.5) 921 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[75425]: cluster 2023-10-09T09:57:00.099618+0000 osd.7 (osd.7) 644 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[75425]: cluster 2023-10-09T09:57:00.101806+0000 osd.7 (osd.7) 645 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[75425]: cluster 2023-10-09T09:57:00.103926+0000 osd.7 (osd.7) 646 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[75425]: cluster 2023-10-09T09:57:00.105724+0000 osd.7 (osd.7) 647 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[78107]: cluster 2023-10-09T09:57:00.096773+0000 osd.5 (osd.5) 920 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[78107]: cluster 2023-10-09T09:57:00.099233+0000 mon.a (mon.0) 2675 : cluster 0 osdmap e1324: 8 total, 8 up, 8 in 2023-10-09T09:57:01.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[78107]: cluster 2023-10-09T09:57:00.099317+0000 osd.5 (osd.5) 921 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[78107]: cluster 2023-10-09T09:57:00.099618+0000 osd.7 (osd.7) 644 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[78107]: cluster 2023-10-09T09:57:00.101806+0000 osd.7 (osd.7) 645 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[78107]: cluster 2023-10-09T09:57:00.103926+0000 osd.7 (osd.7) 646 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:01 smithi163 bash[78107]: cluster 2023-10-09T09:57:00.105724+0000 osd.7 (osd.7) 647 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:01 smithi181 bash[59345]: cluster 2023-10-09T09:57:00.096773+0000 osd.5 (osd.5) 920 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:01 smithi181 bash[59345]: cluster 2023-10-09T09:57:00.099233+0000 mon.a (mon.0) 2675 : cluster 0 osdmap e1324: 8 total, 8 up, 8 in 2023-10-09T09:57:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:01 smithi181 bash[59345]: cluster 2023-10-09T09:57:00.099317+0000 osd.5 (osd.5) 921 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:01 smithi181 bash[59345]: cluster 2023-10-09T09:57:00.099618+0000 osd.7 (osd.7) 644 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:01 smithi181 bash[59345]: cluster 2023-10-09T09:57:00.101806+0000 osd.7 (osd.7) 645 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:01 smithi181 bash[59345]: cluster 2023-10-09T09:57:00.103926+0000 osd.7 (osd.7) 646 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:01.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:01 smithi181 bash[59345]: cluster 2023-10-09T09:57:00.105724+0000 osd.7 (osd.7) 647 : cluster 3 failed to encode map e1324 with expected crc 2023-10-09T09:57:02.104 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:02 smithi163 bash[75425]: cluster 2023-10-09T09:57:00.315915+0000 mgr.y (mgr.34107) 1538 : cluster 0 pgmap v2450: 73 pgs: 21 peering, 52 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:02.294 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:57:02.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:02 smithi163 bash[78107]: cluster 2023-10-09T09:57:00.315915+0000 mgr.y (mgr.34107) 1538 : cluster 0 pgmap v2450: 73 pgs: 21 peering, 52 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:02.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:02 smithi181 bash[59345]: cluster 2023-10-09T09:57:00.315915+0000 mgr.y (mgr.34107) 1538 : cluster 0 pgmap v2450: 73 pgs: 21 peering, 52 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:03.682 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:57:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:57:03] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.33.4" 2023-10-09T09:57:03.730 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T09:57:03.731 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:57:04.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:04 smithi163 bash[75425]: cluster 2023-10-09T09:57:02.316548+0000 mgr.y (mgr.34107) 1539 : cluster 0 pgmap v2451: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 295 KiB/s rd, 427 KiB/s wr, 65 op/s; 890 KiB/s, 3 keys/s, 12 objects/s recovering 2023-10-09T09:57:04.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:04 smithi163 bash[78107]: cluster 2023-10-09T09:57:02.316548+0000 mgr.y (mgr.34107) 1539 : cluster 0 pgmap v2451: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 295 KiB/s rd, 427 KiB/s wr, 65 op/s; 890 KiB/s, 3 keys/s, 12 objects/s recovering 2023-10-09T09:57:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:04 smithi181 bash[59345]: cluster 2023-10-09T09:57:02.316548+0000 mgr.y (mgr.34107) 1539 : cluster 0 pgmap v2451: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 295 KiB/s rd, 427 KiB/s wr, 65 op/s; 890 KiB/s, 3 keys/s, 12 objects/s recovering 2023-10-09T09:57:06.373 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:06 smithi163 bash[78107]: cluster 2023-10-09T09:57:04.317219+0000 mgr.y (mgr.34107) 1540 : cluster 0 pgmap v2452: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 248 KiB/s rd, 359 KiB/s wr, 55 op/s; 748 KiB/s, 3 keys/s, 9 objects/s recovering 2023-10-09T09:57:06.373 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:06 smithi163 bash[75425]: cluster 2023-10-09T09:57:04.317219+0000 mgr.y (mgr.34107) 1540 : cluster 0 pgmap v2452: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 248 KiB/s rd, 359 KiB/s wr, 55 op/s; 748 KiB/s, 3 keys/s, 9 objects/s recovering 2023-10-09T09:57:06.556 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a: {} 2023-10-09T09:57:06.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:06 smithi181 bash[59345]: cluster 2023-10-09T09:57:04.317219+0000 mgr.y (mgr.34107) 1540 : cluster 0 pgmap v2452: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 248 KiB/s rd, 359 KiB/s wr, 55 op/s; 748 KiB/s, 3 keys/s, 9 objects/s recovering 2023-10-09T09:57:06.573 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b: {} 2023-10-09T09:57:06.592 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c: {} 2023-10-09T09:57:06.594 INFO:teuthology.orchestra.run.smithi163.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T09:57:06.594 INFO:teuthology.orchestra.run.smithi163.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T09:57:06.594 INFO:teuthology.orchestra.run.smithi163.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T09:57:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:07 smithi163 bash[75425]: audit 2023-10-09T09:57:06.537607+0000 mon.c (mon.2) 147 : audit 0 from='client.? 172.21.15.163:0/3936457751' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:57:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:07 smithi163 bash[78107]: audit 2023-10-09T09:57:06.537607+0000 mon.c (mon.2) 147 : audit 0 from='client.? 172.21.15.163:0/3936457751' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:57:07.548 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T09:57:07.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:07 smithi181 bash[59345]: audit 2023-10-09T09:57:06.537607+0000 mon.c (mon.2) 147 : audit 0 from='client.? 172.21.15.163:0/3936457751' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:57:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:08 smithi163 bash[75425]: cluster 2023-10-09T09:57:06.317986+0000 mgr.y (mgr.34107) 1541 : cluster 0 pgmap v2453: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 194 KiB/s rd, 280 KiB/s wr, 43 op/s; 585 KiB/s, 2 keys/s, 7 objects/s recovering 2023-10-09T09:57:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:08 smithi163 bash[78107]: cluster 2023-10-09T09:57:06.317986+0000 mgr.y (mgr.34107) 1541 : cluster 0 pgmap v2453: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 194 KiB/s rd, 280 KiB/s wr, 43 op/s; 585 KiB/s, 2 keys/s, 7 objects/s recovering 2023-10-09T09:57:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:08 smithi181 bash[59345]: cluster 2023-10-09T09:57:06.317986+0000 mgr.y (mgr.34107) 1541 : cluster 0 pgmap v2453: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 194 KiB/s rd, 280 KiB/s wr, 43 op/s; 585 KiB/s, 2 keys/s, 7 objects/s recovering 2023-10-09T09:57:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:10 smithi163 bash[75425]: cluster 2023-10-09T09:57:08.319060+0000 mgr.y (mgr.34107) 1542 : cluster 0 pgmap v2454: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 168 KiB/s rd, 243 KiB/s wr, 37 op/s; 507 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T09:57:10.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:10 smithi163 bash[78107]: cluster 2023-10-09T09:57:08.319060+0000 mgr.y (mgr.34107) 1542 : cluster 0 pgmap v2454: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 168 KiB/s rd, 243 KiB/s wr, 37 op/s; 507 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T09:57:10.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:10 smithi181 bash[59345]: cluster 2023-10-09T09:57:08.319060+0000 mgr.y (mgr.34107) 1542 : cluster 0 pgmap v2454: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 168 KiB/s rd, 243 KiB/s wr, 37 op/s; 507 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T09:57:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:11 smithi163 bash[75425]: audit 2023-10-09T09:57:10.421279+0000 mon.a (mon.0) 2676 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:11 smithi163 bash[78107]: audit 2023-10-09T09:57:10.421279+0000 mon.a (mon.0) 2676 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:11 smithi181 bash[59345]: audit 2023-10-09T09:57:10.421279+0000 mon.a (mon.0) 2676 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:12.327 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:57:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:12 smithi163 bash[75425]: cluster 2023-10-09T09:57:10.319634+0000 mgr.y (mgr.34107) 1543 : cluster 0 pgmap v2455: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 152 KiB/s rd, 219 KiB/s wr, 33 op/s; 458 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T09:57:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:12 smithi163 bash[75425]: audit 2023-10-09T09:57:11.283201+0000 mon.a (mon.0) 2677 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:12.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:12 smithi163 bash[78107]: cluster 2023-10-09T09:57:10.319634+0000 mgr.y (mgr.34107) 1543 : cluster 0 pgmap v2455: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 152 KiB/s rd, 219 KiB/s wr, 33 op/s; 458 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T09:57:12.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:12 smithi163 bash[78107]: audit 2023-10-09T09:57:11.283201+0000 mon.a (mon.0) 2677 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:12.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:12 smithi181 bash[59345]: cluster 2023-10-09T09:57:10.319634+0000 mgr.y (mgr.34107) 1543 : cluster 0 pgmap v2455: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 152 KiB/s rd, 219 KiB/s wr, 33 op/s; 458 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T09:57:12.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:12 smithi181 bash[59345]: audit 2023-10-09T09:57:11.283201+0000 mon.a (mon.0) 2677 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:13.713 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:57:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:57:13] "GET /metrics HTTP/1.1" 200 34062 "" "Prometheus/2.33.4" 2023-10-09T09:57:14.435 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:14 smithi163 bash[78107]: cluster 2023-10-09T09:57:12.320618+0000 mgr.y (mgr.34107) 1544 : cluster 0 pgmap v2456: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 129 KiB/s rd, 187 KiB/s wr, 28 op/s; 390 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T09:57:14.435 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:14 smithi163 bash[75425]: cluster 2023-10-09T09:57:12.320618+0000 mgr.y (mgr.34107) 1544 : cluster 0 pgmap v2456: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 129 KiB/s rd, 187 KiB/s wr, 28 op/s; 390 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T09:57:14.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:14 smithi181 bash[59345]: cluster 2023-10-09T09:57:12.320618+0000 mgr.y (mgr.34107) 1544 : cluster 0 pgmap v2456: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 129 KiB/s rd, 187 KiB/s wr, 28 op/s; 390 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T09:57:16.157 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T09:57:16.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:16 smithi163 bash[75425]: cluster 2023-10-09T09:57:14.321346+0000 mgr.y (mgr.34107) 1545 : cluster 0 pgmap v2457: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:16.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:16 smithi163 bash[75425]: audit 2023-10-09T09:57:15.852526+0000 mon.a (mon.0) 2678 : audit 1 from='client.? 172.21.15.163:0/1522474578' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:57:16.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:16 smithi163 bash[78107]: cluster 2023-10-09T09:57:14.321346+0000 mgr.y (mgr.34107) 1545 : cluster 0 pgmap v2457: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:16.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:16 smithi163 bash[78107]: audit 2023-10-09T09:57:15.852526+0000 mon.a (mon.0) 2678 : audit 1 from='client.? 172.21.15.163:0/1522474578' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:57:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:16 smithi181 bash[59345]: cluster 2023-10-09T09:57:14.321346+0000 mgr.y (mgr.34107) 1545 : cluster 0 pgmap v2457: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:16 smithi181 bash[59345]: audit 2023-10-09T09:57:15.852526+0000 mon.a (mon.0) 2678 : audit 1 from='client.? 172.21.15.163:0/1522474578' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:57:17.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T09:57:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:17 smithi163 bash[78107]: cluster 2023-10-09T09:57:16.146077+0000 osd.5 (osd.5) 922 : cluster 3 failed to encode map e1325 with expected crc 2023-10-09T09:57:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:17 smithi163 bash[78107]: cluster 2023-10-09T09:57:16.146480+0000 osd.6 (osd.6) 601 : cluster 3 failed to encode map e1325 with expected crc 2023-10-09T09:57:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:17 smithi163 bash[78107]: audit 2023-10-09T09:57:16.150317+0000 mon.a (mon.0) 2679 : audit 1 from='client.? 172.21.15.163:0/1522474578' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:57:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:17 smithi163 bash[78107]: cluster 2023-10-09T09:57:16.150489+0000 mon.a (mon.0) 2680 : cluster 0 osdmap e1325: 8 total, 8 up, 8 in 2023-10-09T09:57:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:17 smithi163 bash[78107]: cluster 2023-10-09T09:57:16.322080+0000 mgr.y (mgr.34107) 1546 : cluster 0 pgmap v2459: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:17.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:17 smithi163 bash[75425]: cluster 2023-10-09T09:57:16.146077+0000 osd.5 (osd.5) 922 : cluster 3 failed to encode map e1325 with expected crc 2023-10-09T09:57:17.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:17 smithi163 bash[75425]: cluster 2023-10-09T09:57:16.146480+0000 osd.6 (osd.6) 601 : cluster 3 failed to encode map e1325 with expected crc 2023-10-09T09:57:17.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:17 smithi163 bash[75425]: audit 2023-10-09T09:57:16.150317+0000 mon.a (mon.0) 2679 : audit 1 from='client.? 172.21.15.163:0/1522474578' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:57:17.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:17 smithi163 bash[75425]: cluster 2023-10-09T09:57:16.150489+0000 mon.a (mon.0) 2680 : cluster 0 osdmap e1325: 8 total, 8 up, 8 in 2023-10-09T09:57:17.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:17 smithi163 bash[75425]: cluster 2023-10-09T09:57:16.322080+0000 mgr.y (mgr.34107) 1546 : cluster 0 pgmap v2459: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:17 smithi181 bash[59345]: cluster 2023-10-09T09:57:16.146077+0000 osd.5 (osd.5) 922 : cluster 3 failed to encode map e1325 with expected crc 2023-10-09T09:57:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:17 smithi181 bash[59345]: cluster 2023-10-09T09:57:16.146480+0000 osd.6 (osd.6) 601 : cluster 3 failed to encode map e1325 with expected crc 2023-10-09T09:57:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:17 smithi181 bash[59345]: audit 2023-10-09T09:57:16.150317+0000 mon.a (mon.0) 2679 : audit 1 from='client.? 172.21.15.163:0/1522474578' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:57:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:17 smithi181 bash[59345]: cluster 2023-10-09T09:57:16.150489+0000 mon.a (mon.0) 2680 : cluster 0 osdmap e1325: 8 total, 8 up, 8 in 2023-10-09T09:57:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:17 smithi181 bash[59345]: cluster 2023-10-09T09:57:16.322080+0000 mgr.y (mgr.34107) 1546 : cluster 0 pgmap v2459: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:19.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:19 smithi163 bash[78107]: cluster 2023-10-09T09:57:18.323024+0000 mgr.y (mgr.34107) 1547 : cluster 0 pgmap v2460: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:19.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:19 smithi163 bash[75425]: cluster 2023-10-09T09:57:18.323024+0000 mgr.y (mgr.34107) 1547 : cluster 0 pgmap v2460: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:19 smithi181 bash[59345]: cluster 2023-10-09T09:57:18.323024+0000 mgr.y (mgr.34107) 1547 : cluster 0 pgmap v2460: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:21.844 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:57:22.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:22 smithi163 bash[78107]: cluster 2023-10-09T09:57:20.323632+0000 mgr.y (mgr.34107) 1548 : cluster 0 pgmap v2461: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:22.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:22 smithi163 bash[75425]: cluster 2023-10-09T09:57:20.323632+0000 mgr.y (mgr.34107) 1548 : cluster 0 pgmap v2461: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:22.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:22 smithi181 bash[59345]: cluster 2023-10-09T09:57:20.323632+0000 mgr.y (mgr.34107) 1548 : cluster 0 pgmap v2461: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:23.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:57:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:57:23] "GET /metrics HTTP/1.1" 200 34062 "" "Prometheus/2.33.4" 2023-10-09T09:57:24.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:24 smithi181 bash[59345]: cluster 2023-10-09T09:57:22.324598+0000 mgr.y (mgr.34107) 1549 : cluster 0 pgmap v2462: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:24 smithi163 bash[78107]: cluster 2023-10-09T09:57:22.324598+0000 mgr.y (mgr.34107) 1549 : cluster 0 pgmap v2462: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:24.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:24 smithi163 bash[75425]: cluster 2023-10-09T09:57:22.324598+0000 mgr.y (mgr.34107) 1549 : cluster 0 pgmap v2462: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:26.062 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T09:57:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:26 smithi181 bash[59345]: cluster 2023-10-09T09:57:24.325296+0000 mgr.y (mgr.34107) 1550 : cluster 0 pgmap v2463: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:26 smithi181 bash[59345]: audit 2023-10-09T09:57:25.385042+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.163:0/3838684468' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:57:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:26 smithi181 bash[59345]: audit 2023-10-09T09:57:25.385708+0000 mon.a (mon.0) 2681 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:57:26.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:26 smithi181 bash[59345]: audit 2023-10-09T09:57:25.421530+0000 mon.a (mon.0) 2682 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:26 smithi163 bash[78107]: cluster 2023-10-09T09:57:24.325296+0000 mgr.y (mgr.34107) 1550 : cluster 0 pgmap v2463: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:26 smithi163 bash[78107]: audit 2023-10-09T09:57:25.385042+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.163:0/3838684468' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:57:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:26 smithi163 bash[78107]: audit 2023-10-09T09:57:25.385708+0000 mon.a (mon.0) 2681 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:57:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:26 smithi163 bash[78107]: audit 2023-10-09T09:57:25.421530+0000 mon.a (mon.0) 2682 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:26 smithi163 bash[75425]: cluster 2023-10-09T09:57:24.325296+0000 mgr.y (mgr.34107) 1550 : cluster 0 pgmap v2463: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:26 smithi163 bash[75425]: audit 2023-10-09T09:57:25.385042+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.163:0/3838684468' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:57:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:26 smithi163 bash[75425]: audit 2023-10-09T09:57:25.385708+0000 mon.a (mon.0) 2681 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:57:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:26 smithi163 bash[75425]: audit 2023-10-09T09:57:25.421530+0000 mon.a (mon.0) 2682 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:27.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T09:57:27.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:27 smithi163 bash[78107]: cluster 2023-10-09T09:57:26.051386+0000 osd.5 (osd.5) 923 : cluster 3 failed to encode map e1326 with expected crc 2023-10-09T09:57:27.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:27 smithi163 bash[78107]: audit 2023-10-09T09:57:26.053031+0000 mon.a (mon.0) 2683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:57:27.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:27 smithi163 bash[78107]: cluster 2023-10-09T09:57:26.053155+0000 mon.a (mon.0) 2684 : cluster 0 osdmap e1326: 8 total, 8 up, 8 in 2023-10-09T09:57:27.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:27 smithi163 bash[75425]: cluster 2023-10-09T09:57:26.051386+0000 osd.5 (osd.5) 923 : cluster 3 failed to encode map e1326 with expected crc 2023-10-09T09:57:27.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:27 smithi163 bash[75425]: audit 2023-10-09T09:57:26.053031+0000 mon.a (mon.0) 2683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:57:27.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:27 smithi163 bash[75425]: cluster 2023-10-09T09:57:26.053155+0000 mon.a (mon.0) 2684 : cluster 0 osdmap e1326: 8 total, 8 up, 8 in 2023-10-09T09:57:27.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:27 smithi181 bash[59345]: cluster 2023-10-09T09:57:26.051386+0000 osd.5 (osd.5) 923 : cluster 3 failed to encode map e1326 with expected crc 2023-10-09T09:57:28.135 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:27 smithi181 bash[59345]: audit 2023-10-09T09:57:26.053031+0000 mon.a (mon.0) 2683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:57:28.135 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:27 smithi181 bash[59345]: cluster 2023-10-09T09:57:26.053155+0000 mon.a (mon.0) 2684 : cluster 0 osdmap e1326: 8 total, 8 up, 8 in 2023-10-09T09:57:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[78107]: cluster 2023-10-09T09:57:26.326067+0000 mgr.y (mgr.34107) 1551 : cluster 0 pgmap v2465: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[78107]: audit 2023-10-09T09:57:27.460158+0000 mon.a (mon.0) 2685 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[78107]: audit 2023-10-09T09:57:27.838654+0000 mon.a (mon.0) 2686 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[78107]: audit 2023-10-09T09:57:27.844171+0000 mon.a (mon.0) 2687 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[78107]: audit 2023-10-09T09:57:27.849209+0000 mon.a (mon.0) 2688 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[78107]: audit 2023-10-09T09:57:27.853880+0000 mon.a (mon.0) 2689 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[75425]: cluster 2023-10-09T09:57:26.326067+0000 mgr.y (mgr.34107) 1551 : cluster 0 pgmap v2465: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[75425]: audit 2023-10-09T09:57:27.460158+0000 mon.a (mon.0) 2685 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[75425]: audit 2023-10-09T09:57:27.838654+0000 mon.a (mon.0) 2686 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[75425]: audit 2023-10-09T09:57:27.844171+0000 mon.a (mon.0) 2687 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[75425]: audit 2023-10-09T09:57:27.849209+0000 mon.a (mon.0) 2688 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:28 smithi163 bash[75425]: audit 2023-10-09T09:57:27.853880+0000 mon.a (mon.0) 2689 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:28 smithi181 bash[59345]: cluster 2023-10-09T09:57:26.326067+0000 mgr.y (mgr.34107) 1551 : cluster 0 pgmap v2465: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:28 smithi181 bash[59345]: audit 2023-10-09T09:57:27.460158+0000 mon.a (mon.0) 2685 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:28 smithi181 bash[59345]: audit 2023-10-09T09:57:27.838654+0000 mon.a (mon.0) 2686 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:28 smithi181 bash[59345]: audit 2023-10-09T09:57:27.844171+0000 mon.a (mon.0) 2687 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:28 smithi181 bash[59345]: audit 2023-10-09T09:57:27.849209+0000 mon.a (mon.0) 2688 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:28.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:28 smithi181 bash[59345]: audit 2023-10-09T09:57:27.853880+0000 mon.a (mon.0) 2689 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:30.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:30 smithi163 bash[78107]: cluster 2023-10-09T09:57:28.327051+0000 mgr.y (mgr.34107) 1552 : cluster 0 pgmap v2466: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:30.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:30 smithi163 bash[75425]: cluster 2023-10-09T09:57:28.327051+0000 mgr.y (mgr.34107) 1552 : cluster 0 pgmap v2466: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:30.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:30 smithi181 bash[59345]: cluster 2023-10-09T09:57:28.327051+0000 mgr.y (mgr.34107) 1552 : cluster 0 pgmap v2466: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:31.787 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:57:32.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:32 smithi163 bash[78107]: cluster 2023-10-09T09:57:30.327805+0000 mgr.y (mgr.34107) 1553 : cluster 0 pgmap v2467: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:32 smithi163 bash[75425]: cluster 2023-10-09T09:57:30.327805+0000 mgr.y (mgr.34107) 1553 : cluster 0 pgmap v2467: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:32.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:32 smithi181 bash[59345]: cluster 2023-10-09T09:57:30.327805+0000 mgr.y (mgr.34107) 1553 : cluster 0 pgmap v2467: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:33.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:57:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:57:33] "GET /metrics HTTP/1.1" 200 34038 "" "Prometheus/2.33.4" 2023-10-09T09:57:34.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:33 smithi163 bash[78107]: cluster 2023-10-09T09:57:32.328885+0000 mgr.y (mgr.34107) 1554 : cluster 0 pgmap v2468: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:34.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:33 smithi163 bash[78107]: audit 2023-10-09T09:57:32.936542+0000 mon.a (mon.0) 2690 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:34.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:33 smithi163 bash[78107]: audit 2023-10-09T09:57:32.942363+0000 mon.a (mon.0) 2691 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:34.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:33 smithi163 bash[75425]: cluster 2023-10-09T09:57:32.328885+0000 mgr.y (mgr.34107) 1554 : cluster 0 pgmap v2468: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:34.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:33 smithi163 bash[75425]: audit 2023-10-09T09:57:32.936542+0000 mon.a (mon.0) 2690 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:34.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:33 smithi163 bash[75425]: audit 2023-10-09T09:57:32.942363+0000 mon.a (mon.0) 2691 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:33 smithi181 bash[59345]: cluster 2023-10-09T09:57:32.328885+0000 mgr.y (mgr.34107) 1554 : cluster 0 pgmap v2468: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:33 smithi181 bash[59345]: audit 2023-10-09T09:57:32.936542+0000 mon.a (mon.0) 2690 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:34.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:33 smithi181 bash[59345]: audit 2023-10-09T09:57:32.942363+0000 mon.a (mon.0) 2691 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:35 smithi163 bash[78107]: audit 2023-10-09T09:57:34.148721+0000 mon.a (mon.0) 2692 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:35 smithi163 bash[78107]: audit 2023-10-09T09:57:34.155092+0000 mon.a (mon.0) 2693 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:35 smithi163 bash[78107]: audit 2023-10-09T09:57:34.621682+0000 mon.a (mon.0) 2694 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:35 smithi163 bash[78107]: audit 2023-10-09T09:57:34.623424+0000 mon.a (mon.0) 2695 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:35.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:35 smithi163 bash[78107]: audit 2023-10-09T09:57:34.632303+0000 mon.a (mon.0) 2696 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:35.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:35 smithi163 bash[75425]: audit 2023-10-09T09:57:34.148721+0000 mon.a (mon.0) 2692 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:35.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:35 smithi163 bash[75425]: audit 2023-10-09T09:57:34.155092+0000 mon.a (mon.0) 2693 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:35.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:35 smithi163 bash[75425]: audit 2023-10-09T09:57:34.621682+0000 mon.a (mon.0) 2694 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:35.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:35 smithi163 bash[75425]: audit 2023-10-09T09:57:34.623424+0000 mon.a (mon.0) 2695 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:35.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:35 smithi163 bash[75425]: audit 2023-10-09T09:57:34.632303+0000 mon.a (mon.0) 2696 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:35.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:35 smithi181 bash[59345]: audit 2023-10-09T09:57:34.148721+0000 mon.a (mon.0) 2692 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:35.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:35 smithi181 bash[59345]: audit 2023-10-09T09:57:34.155092+0000 mon.a (mon.0) 2693 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:35.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:35 smithi181 bash[59345]: audit 2023-10-09T09:57:34.621682+0000 mon.a (mon.0) 2694 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:35.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:35 smithi181 bash[59345]: audit 2023-10-09T09:57:34.623424+0000 mon.a (mon.0) 2695 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:35.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:35 smithi181 bash[59345]: audit 2023-10-09T09:57:34.632303+0000 mon.a (mon.0) 2696 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:57:36.168 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T09:57:36.408 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:36 smithi163 bash[78107]: cluster 2023-10-09T09:57:34.329623+0000 mgr.y (mgr.34107) 1555 : cluster 0 pgmap v2469: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:36.408 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:36 smithi163 bash[78107]: audit 2023-10-09T09:57:35.563415+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.163:0/534027650' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:36.408 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:36 smithi163 bash[78107]: audit 2023-10-09T09:57:35.564104+0000 mon.a (mon.0) 2697 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:36.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:36 smithi163 bash[75425]: cluster 2023-10-09T09:57:34.329623+0000 mgr.y (mgr.34107) 1555 : cluster 0 pgmap v2469: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:36.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:36 smithi163 bash[75425]: audit 2023-10-09T09:57:35.563415+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.163:0/534027650' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:36.408 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:36 smithi163 bash[75425]: audit 2023-10-09T09:57:35.564104+0000 mon.a (mon.0) 2697 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:36 smithi181 bash[59345]: cluster 2023-10-09T09:57:34.329623+0000 mgr.y (mgr.34107) 1555 : cluster 0 pgmap v2469: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:36 smithi181 bash[59345]: audit 2023-10-09T09:57:35.563415+0000 mon.c (mon.2) 149 : audit 1 from='client.? 172.21.15.163:0/534027650' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:36 smithi181 bash[59345]: audit 2023-10-09T09:57:35.564104+0000 mon.a (mon.0) 2697 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:37.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T09:57:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:37 smithi163 bash[75425]: cluster 2023-10-09T09:57:36.160374+0000 osd.5 (osd.5) 924 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:37 smithi163 bash[75425]: audit 2023-10-09T09:57:36.162611+0000 mon.a (mon.0) 2698 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:57:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:37 smithi163 bash[75425]: cluster 2023-10-09T09:57:36.162713+0000 mon.a (mon.0) 2699 : cluster 0 osdmap e1327: 8 total, 8 up, 8 in 2023-10-09T09:57:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:37 smithi163 bash[75425]: cluster 2023-10-09T09:57:36.330394+0000 mgr.y (mgr.34107) 1556 : cluster 0 pgmap v2471: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:37 smithi163 bash[75425]: cluster 2023-10-09T09:57:36.555162+0000 osd.7 (osd.7) 648 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:37.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:37 smithi163 bash[78107]: cluster 2023-10-09T09:57:36.160374+0000 osd.5 (osd.5) 924 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:37.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:37 smithi163 bash[78107]: audit 2023-10-09T09:57:36.162611+0000 mon.a (mon.0) 2698 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:57:37.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:37 smithi163 bash[78107]: cluster 2023-10-09T09:57:36.162713+0000 mon.a (mon.0) 2699 : cluster 0 osdmap e1327: 8 total, 8 up, 8 in 2023-10-09T09:57:37.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:37 smithi163 bash[78107]: cluster 2023-10-09T09:57:36.330394+0000 mgr.y (mgr.34107) 1556 : cluster 0 pgmap v2471: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:37.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:37 smithi163 bash[78107]: cluster 2023-10-09T09:57:36.555162+0000 osd.7 (osd.7) 648 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:37 smithi181 bash[59345]: cluster 2023-10-09T09:57:36.160374+0000 osd.5 (osd.5) 924 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:37 smithi181 bash[59345]: audit 2023-10-09T09:57:36.162611+0000 mon.a (mon.0) 2698 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:57:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:37 smithi181 bash[59345]: cluster 2023-10-09T09:57:36.162713+0000 mon.a (mon.0) 2699 : cluster 0 osdmap e1327: 8 total, 8 up, 8 in 2023-10-09T09:57:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:37 smithi181 bash[59345]: cluster 2023-10-09T09:57:36.330394+0000 mgr.y (mgr.34107) 1556 : cluster 0 pgmap v2471: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:37 smithi181 bash[59345]: cluster 2023-10-09T09:57:36.555162+0000 osd.7 (osd.7) 648 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:38.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:38 smithi163 bash[75425]: cluster 2023-10-09T09:57:36.554866+0000 osd.4 (osd.4) 704 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:38.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:38 smithi163 bash[75425]: cluster 2023-10-09T09:57:36.554990+0000 osd.6 (osd.6) 602 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:38 smithi163 bash[78107]: cluster 2023-10-09T09:57:36.554866+0000 osd.4 (osd.4) 704 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:38 smithi163 bash[78107]: cluster 2023-10-09T09:57:36.554990+0000 osd.6 (osd.6) 602 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:38 smithi181 bash[59345]: cluster 2023-10-09T09:57:36.554866+0000 osd.4 (osd.4) 704 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:38 smithi181 bash[59345]: cluster 2023-10-09T09:57:36.554990+0000 osd.6 (osd.6) 602 : cluster 3 failed to encode map e1327 with expected crc 2023-10-09T09:57:39.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:39 smithi163 bash[78107]: cluster 2023-10-09T09:57:38.331161+0000 mgr.y (mgr.34107) 1557 : cluster 0 pgmap v2472: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:39.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:39 smithi163 bash[75425]: cluster 2023-10-09T09:57:38.331161+0000 mgr.y (mgr.34107) 1557 : cluster 0 pgmap v2472: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:39.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:39 smithi181 bash[59345]: cluster 2023-10-09T09:57:38.331161+0000 mgr.y (mgr.34107) 1557 : cluster 0 pgmap v2472: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:41 smithi163 bash[78107]: cluster 2023-10-09T09:57:40.331836+0000 mgr.y (mgr.34107) 1558 : cluster 0 pgmap v2473: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:41.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:41 smithi163 bash[78107]: audit 2023-10-09T09:57:40.421879+0000 mon.a (mon.0) 2700 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:41 smithi163 bash[75425]: cluster 2023-10-09T09:57:40.331836+0000 mgr.y (mgr.34107) 1558 : cluster 0 pgmap v2473: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:41.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:41 smithi163 bash[75425]: audit 2023-10-09T09:57:40.421879+0000 mon.a (mon.0) 2700 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:41 smithi181 bash[59345]: cluster 2023-10-09T09:57:40.331836+0000 mgr.y (mgr.34107) 1558 : cluster 0 pgmap v2473: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:41.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:41 smithi181 bash[59345]: audit 2023-10-09T09:57:40.421879+0000 mon.a (mon.0) 2700 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:41.888 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:57:43.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:57:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:57:43] "GET /metrics HTTP/1.1" 200 34033 "" "Prometheus/2.33.4" 2023-10-09T09:57:43.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:43 smithi163 bash[78107]: cluster 2023-10-09T09:57:42.332833+0000 mgr.y (mgr.34107) 1559 : cluster 0 pgmap v2474: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:43.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:43 smithi163 bash[75425]: cluster 2023-10-09T09:57:42.332833+0000 mgr.y (mgr.34107) 1559 : cluster 0 pgmap v2474: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:43.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:43 smithi181 bash[59345]: cluster 2023-10-09T09:57:42.332833+0000 mgr.y (mgr.34107) 1559 : cluster 0 pgmap v2474: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:45.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:45 smithi163 bash[75425]: cluster 2023-10-09T09:57:44.333393+0000 mgr.y (mgr.34107) 1560 : cluster 0 pgmap v2475: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:45.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:45 smithi163 bash[78107]: cluster 2023-10-09T09:57:44.333393+0000 mgr.y (mgr.34107) 1560 : cluster 0 pgmap v2475: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:45 smithi181 bash[59345]: cluster 2023-10-09T09:57:44.333393+0000 mgr.y (mgr.34107) 1560 : cluster 0 pgmap v2475: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:46.425 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T09:57:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:46 smithi163 bash[75425]: audit 2023-10-09T09:57:45.403790+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.163:0/1755383186' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:46.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:46 smithi163 bash[75425]: audit 2023-10-09T09:57:45.404189+0000 mon.a (mon.0) 2701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:46 smithi163 bash[78107]: audit 2023-10-09T09:57:45.403790+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.163:0/1755383186' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:46.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:46 smithi163 bash[78107]: audit 2023-10-09T09:57:45.404189+0000 mon.a (mon.0) 2701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:46 smithi181 bash[59345]: audit 2023-10-09T09:57:45.403790+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.163:0/1755383186' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:46.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:46 smithi181 bash[59345]: audit 2023-10-09T09:57:45.404189+0000 mon.a (mon.0) 2701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:57:47.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T09:57:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[78107]: cluster 2023-10-09T09:57:46.334140+0000 mgr.y (mgr.34107) 1561 : cluster 0 pgmap v2476: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[78107]: cluster 2023-10-09T09:57:46.406635+0000 osd.5 (osd.5) 925 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[78107]: audit 2023-10-09T09:57:46.407995+0000 mon.a (mon.0) 2702 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:57:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[78107]: cluster 2023-10-09T09:57:46.408056+0000 mon.a (mon.0) 2703 : cluster 0 osdmap e1328: 8 total, 8 up, 8 in 2023-10-09T09:57:47.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[78107]: cluster 2023-10-09T09:57:46.412890+0000 osd.7 (osd.7) 649 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[78107]: cluster 2023-10-09T09:57:46.413485+0000 osd.6 (osd.6) 603 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[78107]: cluster 2023-10-09T09:57:46.416992+0000 osd.4 (osd.4) 705 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[75425]: cluster 2023-10-09T09:57:46.334140+0000 mgr.y (mgr.34107) 1561 : cluster 0 pgmap v2476: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:47.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[75425]: cluster 2023-10-09T09:57:46.406635+0000 osd.5 (osd.5) 925 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[75425]: audit 2023-10-09T09:57:46.407995+0000 mon.a (mon.0) 2702 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:57:47.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[75425]: cluster 2023-10-09T09:57:46.408056+0000 mon.a (mon.0) 2703 : cluster 0 osdmap e1328: 8 total, 8 up, 8 in 2023-10-09T09:57:47.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[75425]: cluster 2023-10-09T09:57:46.412890+0000 osd.7 (osd.7) 649 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[75425]: cluster 2023-10-09T09:57:46.413485+0000 osd.6 (osd.6) 603 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:47 smithi163 bash[75425]: cluster 2023-10-09T09:57:46.416992+0000 osd.4 (osd.4) 705 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:47 smithi181 bash[59345]: cluster 2023-10-09T09:57:46.334140+0000 mgr.y (mgr.34107) 1561 : cluster 0 pgmap v2476: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:47 smithi181 bash[59345]: cluster 2023-10-09T09:57:46.406635+0000 osd.5 (osd.5) 925 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:47 smithi181 bash[59345]: audit 2023-10-09T09:57:46.407995+0000 mon.a (mon.0) 2702 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:57:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:47 smithi181 bash[59345]: cluster 2023-10-09T09:57:46.408056+0000 mon.a (mon.0) 2703 : cluster 0 osdmap e1328: 8 total, 8 up, 8 in 2023-10-09T09:57:47.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:47 smithi181 bash[59345]: cluster 2023-10-09T09:57:46.412890+0000 osd.7 (osd.7) 649 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:47 smithi181 bash[59345]: cluster 2023-10-09T09:57:46.413485+0000 osd.6 (osd.6) 603 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:47.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:47 smithi181 bash[59345]: cluster 2023-10-09T09:57:46.416992+0000 osd.4 (osd.4) 705 : cluster 3 failed to encode map e1328 with expected crc 2023-10-09T09:57:48.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:48 smithi163 bash[78107]: cluster 2023-10-09T09:57:47.418334+0000 osd.5 (osd.5) 926 : cluster 3 failed to encode map e1329 with expected crc 2023-10-09T09:57:48.921 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:48 smithi163 bash[78107]: cluster 2023-10-09T09:57:47.418741+0000 osd.7 (osd.7) 650 : cluster 3 failed to encode map e1329 with expected crc 2023-10-09T09:57:48.927 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:48 smithi163 bash[78107]: cluster 2023-10-09T09:57:47.420406+0000 mon.a (mon.0) 2704 : cluster 0 osdmap e1329: 8 total, 8 up, 8 in 2023-10-09T09:57:48.927 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:48 smithi163 bash[78107]: cluster 2023-10-09T09:57:47.426508+0000 osd.6 (osd.6) 604 : cluster 3 failed to encode map e1329 with expected crc 2023-10-09T09:57:48.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:48 smithi163 bash[75425]: cluster 2023-10-09T09:57:47.418334+0000 osd.5 (osd.5) 926 : cluster 3 failed to encode map e1329 with expected crc 2023-10-09T09:57:48.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:48 smithi163 bash[75425]: cluster 2023-10-09T09:57:47.418741+0000 osd.7 (osd.7) 650 : cluster 3 failed to encode map e1329 with expected crc 2023-10-09T09:57:48.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:48 smithi163 bash[75425]: cluster 2023-10-09T09:57:47.420406+0000 mon.a (mon.0) 2704 : cluster 0 osdmap e1329: 8 total, 8 up, 8 in 2023-10-09T09:57:48.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:48 smithi163 bash[75425]: cluster 2023-10-09T09:57:47.426508+0000 osd.6 (osd.6) 604 : cluster 3 failed to encode map e1329 with expected crc 2023-10-09T09:57:48.929 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:48 smithi181 bash[59345]: cluster 2023-10-09T09:57:47.418334+0000 osd.5 (osd.5) 926 : cluster 3 failed to encode map e1329 with expected crc 2023-10-09T09:57:48.930 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:48 smithi181 bash[59345]: cluster 2023-10-09T09:57:47.418741+0000 osd.7 (osd.7) 650 : cluster 3 failed to encode map e1329 with expected crc 2023-10-09T09:57:48.930 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:48 smithi181 bash[59345]: cluster 2023-10-09T09:57:47.420406+0000 mon.a (mon.0) 2704 : cluster 0 osdmap e1329: 8 total, 8 up, 8 in 2023-10-09T09:57:48.930 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:48 smithi181 bash[59345]: cluster 2023-10-09T09:57:47.426508+0000 osd.6 (osd.6) 604 : cluster 3 failed to encode map e1329 with expected crc 2023-10-09T09:57:49.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:49 smithi163 bash[75425]: cluster 2023-10-09T09:57:48.334757+0000 mgr.y (mgr.34107) 1562 : cluster 0 pgmap v2479: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:49.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:49 smithi163 bash[78107]: cluster 2023-10-09T09:57:48.334757+0000 mgr.y (mgr.34107) 1562 : cluster 0 pgmap v2479: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:49.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:49 smithi181 bash[59345]: cluster 2023-10-09T09:57:48.334757+0000 mgr.y (mgr.34107) 1562 : cluster 0 pgmap v2479: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:52.155 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:57:52.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:52 smithi181 bash[59345]: cluster 2023-10-09T09:57:50.335586+0000 mgr.y (mgr.34107) 1563 : cluster 0 pgmap v2480: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:52 smithi163 bash[78107]: cluster 2023-10-09T09:57:50.335586+0000 mgr.y (mgr.34107) 1563 : cluster 0 pgmap v2480: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:52.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:52 smithi163 bash[75425]: cluster 2023-10-09T09:57:50.335586+0000 mgr.y (mgr.34107) 1563 : cluster 0 pgmap v2480: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:53.572 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:57:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:57:53] "GET /metrics HTTP/1.1" 200 34033 "" "Prometheus/2.33.4" 2023-10-09T09:57:54.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:54 smithi181 bash[59345]: cluster 2023-10-09T09:57:52.336510+0000 mgr.y (mgr.34107) 1564 : cluster 0 pgmap v2481: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:54 smithi163 bash[75425]: cluster 2023-10-09T09:57:52.336510+0000 mgr.y (mgr.34107) 1564 : cluster 0 pgmap v2481: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:54 smithi163 bash[78107]: cluster 2023-10-09T09:57:52.336510+0000 mgr.y (mgr.34107) 1564 : cluster 0 pgmap v2481: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:56.070 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T09:57:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:56 smithi163 bash[78107]: cluster 2023-10-09T09:57:54.337301+0000 mgr.y (mgr.34107) 1565 : cluster 0 pgmap v2482: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:56 smithi163 bash[78107]: audit 2023-10-09T09:57:55.422188+0000 mon.a (mon.0) 2705 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:56 smithi163 bash[78107]: audit 2023-10-09T09:57:55.724680+0000 mon.a (mon.0) 2706 : audit 1 from='client.? 172.21.15.163:0/545457371' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:57:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:56 smithi163 bash[75425]: cluster 2023-10-09T09:57:54.337301+0000 mgr.y (mgr.34107) 1565 : cluster 0 pgmap v2482: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:56 smithi163 bash[75425]: audit 2023-10-09T09:57:55.422188+0000 mon.a (mon.0) 2705 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:56.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:56 smithi163 bash[75425]: audit 2023-10-09T09:57:55.724680+0000 mon.a (mon.0) 2706 : audit 1 from='client.? 172.21.15.163:0/545457371' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:57:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:56 smithi181 bash[59345]: cluster 2023-10-09T09:57:54.337301+0000 mgr.y (mgr.34107) 1565 : cluster 0 pgmap v2482: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:56 smithi181 bash[59345]: audit 2023-10-09T09:57:55.422188+0000 mon.a (mon.0) 2705 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:56 smithi181 bash[59345]: audit 2023-10-09T09:57:55.724680+0000 mon.a (mon.0) 2706 : audit 1 from='client.? 172.21.15.163:0/545457371' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:57:56.958 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T09:57:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:57 smithi163 bash[75425]: cluster 2023-10-09T09:57:56.061614+0000 osd.5 (osd.5) 927 : cluster 3 failed to encode map e1330 with expected crc 2023-10-09T09:57:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:57 smithi163 bash[75425]: audit 2023-10-09T09:57:56.063369+0000 mon.a (mon.0) 2707 : audit 1 from='client.? 172.21.15.163:0/545457371' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:57:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:57 smithi163 bash[75425]: cluster 2023-10-09T09:57:56.063472+0000 mon.a (mon.0) 2708 : cluster 0 osdmap e1330: 8 total, 8 up, 8 in 2023-10-09T09:57:57.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:57 smithi163 bash[75425]: cluster 2023-10-09T09:57:56.269306+0000 osd.6 (osd.6) 605 : cluster 3 failed to encode map e1330 with expected crc 2023-10-09T09:57:57.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:57 smithi163 bash[78107]: cluster 2023-10-09T09:57:56.061614+0000 osd.5 (osd.5) 927 : cluster 3 failed to encode map e1330 with expected crc 2023-10-09T09:57:57.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:57 smithi163 bash[78107]: audit 2023-10-09T09:57:56.063369+0000 mon.a (mon.0) 2707 : audit 1 from='client.? 172.21.15.163:0/545457371' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:57:57.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:57 smithi163 bash[78107]: cluster 2023-10-09T09:57:56.063472+0000 mon.a (mon.0) 2708 : cluster 0 osdmap e1330: 8 total, 8 up, 8 in 2023-10-09T09:57:57.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:57 smithi163 bash[78107]: cluster 2023-10-09T09:57:56.269306+0000 osd.6 (osd.6) 605 : cluster 3 failed to encode map e1330 with expected crc 2023-10-09T09:57:57.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:57 smithi181 bash[59345]: cluster 2023-10-09T09:57:56.061614+0000 osd.5 (osd.5) 927 : cluster 3 failed to encode map e1330 with expected crc 2023-10-09T09:57:57.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:57 smithi181 bash[59345]: audit 2023-10-09T09:57:56.063369+0000 mon.a (mon.0) 2707 : audit 1 from='client.? 172.21.15.163:0/545457371' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:57:57.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:57 smithi181 bash[59345]: cluster 2023-10-09T09:57:56.063472+0000 mon.a (mon.0) 2708 : cluster 0 osdmap e1330: 8 total, 8 up, 8 in 2023-10-09T09:57:57.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:57 smithi181 bash[59345]: cluster 2023-10-09T09:57:56.269306+0000 osd.6 (osd.6) 605 : cluster 3 failed to encode map e1330 with expected crc 2023-10-09T09:57:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:58 smithi163 bash[78107]: cluster 2023-10-09T09:57:56.337750+0000 mgr.y (mgr.34107) 1566 : cluster 0 pgmap v2484: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:57:58 smithi163 bash[78107]: cluster 2023-10-09T09:57:57.060768+0000 osd.4 (osd.4) 706 : cluster 3 failed to encode map e1330 with expected crc 2023-10-09T09:57:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:58 smithi163 bash[75425]: cluster 2023-10-09T09:57:56.337750+0000 mgr.y (mgr.34107) 1566 : cluster 0 pgmap v2484: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:57:58 smithi163 bash[75425]: cluster 2023-10-09T09:57:57.060768+0000 osd.4 (osd.4) 706 : cluster 3 failed to encode map e1330 with expected crc 2023-10-09T09:57:58.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:58 smithi181 bash[59345]: cluster 2023-10-09T09:57:56.337750+0000 mgr.y (mgr.34107) 1566 : cluster 0 pgmap v2484: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:57:58.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:57:58 smithi181 bash[59345]: cluster 2023-10-09T09:57:57.060768+0000 osd.4 (osd.4) 706 : cluster 3 failed to encode map e1330 with expected crc 2023-10-09T09:58:00.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:00 smithi163 bash[75425]: cluster 2023-10-09T09:57:58.338763+0000 mgr.y (mgr.34107) 1567 : cluster 0 pgmap v2485: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:00.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:00 smithi163 bash[78107]: cluster 2023-10-09T09:57:58.338763+0000 mgr.y (mgr.34107) 1567 : cluster 0 pgmap v2485: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:00.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:00 smithi181 bash[59345]: cluster 2023-10-09T09:57:58.338763+0000 mgr.y (mgr.34107) 1567 : cluster 0 pgmap v2485: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:01.742 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:02 smithi163 bash[75425]: cluster 2023-10-09T09:58:00.339655+0000 mgr.y (mgr.34107) 1568 : cluster 0 pgmap v2486: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:02.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:02 smithi163 bash[78107]: cluster 2023-10-09T09:58:00.339655+0000 mgr.y (mgr.34107) 1568 : cluster 0 pgmap v2486: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:02.570 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:02 smithi181 bash[59345]: cluster 2023-10-09T09:58:00.339655+0000 mgr.y (mgr.34107) 1568 : cluster 0 pgmap v2486: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:03.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:58:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:58:03] "GET /metrics HTTP/1.1" 200 34038 "" "Prometheus/2.33.4" 2023-10-09T09:58:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:04 smithi163 bash[75425]: cluster 2023-10-09T09:58:02.340279+0000 mgr.y (mgr.34107) 1569 : cluster 0 pgmap v2487: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:04 smithi163 bash[78107]: cluster 2023-10-09T09:58:02.340279+0000 mgr.y (mgr.34107) 1569 : cluster 0 pgmap v2487: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:04 smithi181 bash[59345]: cluster 2023-10-09T09:58:02.340279+0000 mgr.y (mgr.34107) 1569 : cluster 0 pgmap v2487: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:06.101 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T09:58:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:06 smithi163 bash[78107]: cluster 2023-10-09T09:58:04.340878+0000 mgr.y (mgr.34107) 1570 : cluster 0 pgmap v2488: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:06 smithi163 bash[78107]: audit 2023-10-09T09:58:05.646049+0000 mon.a (mon.0) 2709 : audit 1 from='client.? 172.21.15.163:0/1779278542' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:58:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:06 smithi163 bash[75425]: cluster 2023-10-09T09:58:04.340878+0000 mgr.y (mgr.34107) 1570 : cluster 0 pgmap v2488: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:06 smithi163 bash[75425]: audit 2023-10-09T09:58:05.646049+0000 mon.a (mon.0) 2709 : audit 1 from='client.? 172.21.15.163:0/1779278542' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:58:06.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:06 smithi181 bash[59345]: cluster 2023-10-09T09:58:04.340878+0000 mgr.y (mgr.34107) 1570 : cluster 0 pgmap v2488: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:06.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:06 smithi181 bash[59345]: audit 2023-10-09T09:58:05.646049+0000 mon.a (mon.0) 2709 : audit 1 from='client.? 172.21.15.163:0/1779278542' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:58:07.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T09:58:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:07 smithi163 bash[75425]: cluster 2023-10-09T09:58:06.094258+0000 osd.5 (osd.5) 928 : cluster 3 failed to encode map e1331 with expected crc 2023-10-09T09:58:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:07 smithi163 bash[75425]: cluster 2023-10-09T09:58:06.094953+0000 osd.7 (osd.7) 651 : cluster 3 failed to encode map e1331 with expected crc 2023-10-09T09:58:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:07 smithi163 bash[75425]: audit 2023-10-09T09:58:06.095784+0000 mon.a (mon.0) 2710 : audit 1 from='client.? 172.21.15.163:0/1779278542' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:58:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:07 smithi163 bash[75425]: cluster 2023-10-09T09:58:06.095860+0000 mon.a (mon.0) 2711 : cluster 0 osdmap e1331: 8 total, 8 up, 8 in 2023-10-09T09:58:07.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:07 smithi163 bash[78107]: cluster 2023-10-09T09:58:06.094258+0000 osd.5 (osd.5) 928 : cluster 3 failed to encode map e1331 with expected crc 2023-10-09T09:58:07.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:07 smithi163 bash[78107]: cluster 2023-10-09T09:58:06.094953+0000 osd.7 (osd.7) 651 : cluster 3 failed to encode map e1331 with expected crc 2023-10-09T09:58:07.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:07 smithi163 bash[78107]: audit 2023-10-09T09:58:06.095784+0000 mon.a (mon.0) 2710 : audit 1 from='client.? 172.21.15.163:0/1779278542' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:58:07.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:07 smithi163 bash[78107]: cluster 2023-10-09T09:58:06.095860+0000 mon.a (mon.0) 2711 : cluster 0 osdmap e1331: 8 total, 8 up, 8 in 2023-10-09T09:58:07.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:07 smithi181 bash[59345]: cluster 2023-10-09T09:58:06.094258+0000 osd.5 (osd.5) 928 : cluster 3 failed to encode map e1331 with expected crc 2023-10-09T09:58:07.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:07 smithi181 bash[59345]: cluster 2023-10-09T09:58:06.094953+0000 osd.7 (osd.7) 651 : cluster 3 failed to encode map e1331 with expected crc 2023-10-09T09:58:07.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:07 smithi181 bash[59345]: audit 2023-10-09T09:58:06.095784+0000 mon.a (mon.0) 2710 : audit 1 from='client.? 172.21.15.163:0/1779278542' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:58:07.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:07 smithi181 bash[59345]: cluster 2023-10-09T09:58:06.095860+0000 mon.a (mon.0) 2711 : cluster 0 osdmap e1331: 8 total, 8 up, 8 in 2023-10-09T09:58:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[75425]: cluster 2023-10-09T09:58:06.341938+0000 mgr.y (mgr.34107) 1571 : cluster 0 pgmap v2490: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[75425]: cluster 2023-10-09T09:58:07.100312+0000 mon.a (mon.0) 2712 : cluster 3 Health check failed: Reduced data availability: 11 pgs inactive, 15 pgs peering (PG_AVAILABILITY) 2023-10-09T09:58:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[75425]: cluster 2023-10-09T09:58:07.110274+0000 osd.5 (osd.5) 929 : cluster 3 failed to encode map e1332 with expected crc 2023-10-09T09:58:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[75425]: cluster 2023-10-09T09:58:07.113300+0000 osd.4 (osd.4) 707 : cluster 3 failed to encode map e1332 with expected crc 2023-10-09T09:58:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[75425]: cluster 2023-10-09T09:58:07.115031+0000 mon.a (mon.0) 2713 : cluster 0 osdmap e1332: 8 total, 8 up, 8 in 2023-10-09T09:58:08.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[75425]: cluster 2023-10-09T09:58:07.116570+0000 osd.4 (osd.4) 708 : cluster 3 failed to encode map e1332 with expected crc 2023-10-09T09:58:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[78107]: cluster 2023-10-09T09:58:06.341938+0000 mgr.y (mgr.34107) 1571 : cluster 0 pgmap v2490: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[78107]: cluster 2023-10-09T09:58:07.100312+0000 mon.a (mon.0) 2712 : cluster 3 Health check failed: Reduced data availability: 11 pgs inactive, 15 pgs peering (PG_AVAILABILITY) 2023-10-09T09:58:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[78107]: cluster 2023-10-09T09:58:07.110274+0000 osd.5 (osd.5) 929 : cluster 3 failed to encode map e1332 with expected crc 2023-10-09T09:58:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[78107]: cluster 2023-10-09T09:58:07.113300+0000 osd.4 (osd.4) 707 : cluster 3 failed to encode map e1332 with expected crc 2023-10-09T09:58:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[78107]: cluster 2023-10-09T09:58:07.115031+0000 mon.a (mon.0) 2713 : cluster 0 osdmap e1332: 8 total, 8 up, 8 in 2023-10-09T09:58:08.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:08 smithi163 bash[78107]: cluster 2023-10-09T09:58:07.116570+0000 osd.4 (osd.4) 708 : cluster 3 failed to encode map e1332 with expected crc 2023-10-09T09:58:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:08 smithi181 bash[59345]: cluster 2023-10-09T09:58:06.341938+0000 mgr.y (mgr.34107) 1571 : cluster 0 pgmap v2490: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:08 smithi181 bash[59345]: cluster 2023-10-09T09:58:07.100312+0000 mon.a (mon.0) 2712 : cluster 3 Health check failed: Reduced data availability: 11 pgs inactive, 15 pgs peering (PG_AVAILABILITY) 2023-10-09T09:58:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:08 smithi181 bash[59345]: cluster 2023-10-09T09:58:07.110274+0000 osd.5 (osd.5) 929 : cluster 3 failed to encode map e1332 with expected crc 2023-10-09T09:58:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:08 smithi181 bash[59345]: cluster 2023-10-09T09:58:07.113300+0000 osd.4 (osd.4) 707 : cluster 3 failed to encode map e1332 with expected crc 2023-10-09T09:58:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:08 smithi181 bash[59345]: cluster 2023-10-09T09:58:07.115031+0000 mon.a (mon.0) 2713 : cluster 0 osdmap e1332: 8 total, 8 up, 8 in 2023-10-09T09:58:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:08 smithi181 bash[59345]: cluster 2023-10-09T09:58:07.116570+0000 osd.4 (osd.4) 708 : cluster 3 failed to encode map e1332 with expected crc 2023-10-09T09:58:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:10 smithi163 bash[75425]: cluster 2023-10-09T09:58:08.342851+0000 mgr.y (mgr.34107) 1572 : cluster 0 pgmap v2492: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:10.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:10 smithi163 bash[78107]: cluster 2023-10-09T09:58:08.342851+0000 mgr.y (mgr.34107) 1572 : cluster 0 pgmap v2492: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:10.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:10 smithi181 bash[59345]: cluster 2023-10-09T09:58:08.342851+0000 mgr.y (mgr.34107) 1572 : cluster 0 pgmap v2492: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:11 smithi163 bash[75425]: cluster 2023-10-09T09:58:10.343945+0000 mgr.y (mgr.34107) 1573 : cluster 0 pgmap v2493: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:11 smithi163 bash[75425]: audit 2023-10-09T09:58:10.430270+0000 mon.a (mon.0) 2714 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:11.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:11 smithi163 bash[75425]: audit 2023-10-09T09:58:10.431741+0000 mon.a (mon.0) 2715 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:11.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:11 smithi163 bash[78107]: cluster 2023-10-09T09:58:10.343945+0000 mgr.y (mgr.34107) 1573 : cluster 0 pgmap v2493: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:11.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:11 smithi163 bash[78107]: audit 2023-10-09T09:58:10.430270+0000 mon.a (mon.0) 2714 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:11.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:11 smithi163 bash[78107]: audit 2023-10-09T09:58:10.431741+0000 mon.a (mon.0) 2715 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:11 smithi181 bash[59345]: cluster 2023-10-09T09:58:10.343945+0000 mgr.y (mgr.34107) 1573 : cluster 0 pgmap v2493: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:11 smithi181 bash[59345]: audit 2023-10-09T09:58:10.430270+0000 mon.a (mon.0) 2714 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:11.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:11 smithi181 bash[59345]: audit 2023-10-09T09:58:10.431741+0000 mon.a (mon.0) 2715 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:11.850 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:12 smithi181 bash[59345]: cluster 2023-10-09T09:58:12.426176+0000 mon.a (mon.0) 2716 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 15 pgs peering) 2023-10-09T09:58:12.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:12 smithi181 bash[59345]: cluster 2023-10-09T09:58:12.426239+0000 mon.a (mon.0) 2717 : cluster 1 Cluster is now healthy 2023-10-09T09:58:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:12 smithi163 bash[75425]: cluster 2023-10-09T09:58:12.426176+0000 mon.a (mon.0) 2716 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 15 pgs peering) 2023-10-09T09:58:12.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:12 smithi163 bash[75425]: cluster 2023-10-09T09:58:12.426239+0000 mon.a (mon.0) 2717 : cluster 1 Cluster is now healthy 2023-10-09T09:58:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:12 smithi163 bash[78107]: cluster 2023-10-09T09:58:12.426176+0000 mon.a (mon.0) 2716 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 15 pgs peering) 2023-10-09T09:58:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:12 smithi163 bash[78107]: cluster 2023-10-09T09:58:12.426239+0000 mon.a (mon.0) 2717 : cluster 1 Cluster is now healthy 2023-10-09T09:58:13.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:13 smithi163 bash[75425]: cluster 2023-10-09T09:58:12.344760+0000 mgr.y (mgr.34107) 1574 : cluster 0 pgmap v2494: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:13.743 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:58:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:58:13] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.33.4" 2023-10-09T09:58:13.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:13 smithi163 bash[78107]: cluster 2023-10-09T09:58:12.344760+0000 mgr.y (mgr.34107) 1574 : cluster 0 pgmap v2494: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:13 smithi181 bash[59345]: cluster 2023-10-09T09:58:12.344760+0000 mgr.y (mgr.34107) 1574 : cluster 0 pgmap v2494: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:15.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:15 smithi163 bash[75425]: cluster 2023-10-09T09:58:14.345455+0000 mgr.y (mgr.34107) 1575 : cluster 0 pgmap v2495: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:15.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:15 smithi163 bash[78107]: cluster 2023-10-09T09:58:14.345455+0000 mgr.y (mgr.34107) 1575 : cluster 0 pgmap v2495: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:15.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:15 smithi181 bash[59345]: cluster 2023-10-09T09:58:14.345455+0000 mgr.y (mgr.34107) 1575 : cluster 0 pgmap v2495: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T09:58:16.507 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T09:58:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:16 smithi181 bash[59345]: audit 2023-10-09T09:58:15.708394+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.163:0/1737173347' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:16.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:16 smithi181 bash[59345]: audit 2023-10-09T09:58:15.708897+0000 mon.a (mon.0) 2718 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:16 smithi163 bash[75425]: audit 2023-10-09T09:58:15.708394+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.163:0/1737173347' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:16.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:16 smithi163 bash[75425]: audit 2023-10-09T09:58:15.708897+0000 mon.a (mon.0) 2718 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:16 smithi163 bash[78107]: audit 2023-10-09T09:58:15.708394+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.163:0/1737173347' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:16.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:16 smithi163 bash[78107]: audit 2023-10-09T09:58:15.708897+0000 mon.a (mon.0) 2718 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:17.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T09:58:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:17 smithi181 bash[59345]: cluster 2023-10-09T09:58:16.346203+0000 mgr.y (mgr.34107) 1576 : cluster 0 pgmap v2496: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:17 smithi181 bash[59345]: cluster 2023-10-09T09:58:16.498336+0000 osd.5 (osd.5) 930 : cluster 3 failed to encode map e1333 with expected crc 2023-10-09T09:58:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:17 smithi181 bash[59345]: audit 2023-10-09T09:58:16.500394+0000 mon.a (mon.0) 2719 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:58:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:17 smithi181 bash[59345]: cluster 2023-10-09T09:58:16.500503+0000 mon.a (mon.0) 2720 : cluster 0 osdmap e1333: 8 total, 8 up, 8 in 2023-10-09T09:58:17.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:17 smithi181 bash[59345]: cluster 2023-10-09T09:58:17.174393+0000 osd.6 (osd.6) 606 : cluster 3 failed to encode map e1333 with expected crc 2023-10-09T09:58:17.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:17 smithi163 bash[78107]: cluster 2023-10-09T09:58:16.346203+0000 mgr.y (mgr.34107) 1576 : cluster 0 pgmap v2496: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:17.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:17 smithi163 bash[78107]: cluster 2023-10-09T09:58:16.498336+0000 osd.5 (osd.5) 930 : cluster 3 failed to encode map e1333 with expected crc 2023-10-09T09:58:17.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:17 smithi163 bash[78107]: audit 2023-10-09T09:58:16.500394+0000 mon.a (mon.0) 2719 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:58:17.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:17 smithi163 bash[78107]: cluster 2023-10-09T09:58:16.500503+0000 mon.a (mon.0) 2720 : cluster 0 osdmap e1333: 8 total, 8 up, 8 in 2023-10-09T09:58:17.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:17 smithi163 bash[78107]: cluster 2023-10-09T09:58:17.174393+0000 osd.6 (osd.6) 606 : cluster 3 failed to encode map e1333 with expected crc 2023-10-09T09:58:17.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:17 smithi163 bash[75425]: cluster 2023-10-09T09:58:16.346203+0000 mgr.y (mgr.34107) 1576 : cluster 0 pgmap v2496: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:17.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:17 smithi163 bash[75425]: cluster 2023-10-09T09:58:16.498336+0000 osd.5 (osd.5) 930 : cluster 3 failed to encode map e1333 with expected crc 2023-10-09T09:58:17.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:17 smithi163 bash[75425]: audit 2023-10-09T09:58:16.500394+0000 mon.a (mon.0) 2719 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:58:17.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:17 smithi163 bash[75425]: cluster 2023-10-09T09:58:16.500503+0000 mon.a (mon.0) 2720 : cluster 0 osdmap e1333: 8 total, 8 up, 8 in 2023-10-09T09:58:17.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:17 smithi163 bash[75425]: cluster 2023-10-09T09:58:17.174393+0000 osd.6 (osd.6) 606 : cluster 3 failed to encode map e1333 with expected crc 2023-10-09T09:58:18.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:18 smithi181 bash[59345]: cluster 2023-10-09T09:58:17.578578+0000 osd.7 (osd.7) 652 : cluster 3 failed to encode map e1333 with expected crc 2023-10-09T09:58:18.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:18 smithi163 bash[78107]: cluster 2023-10-09T09:58:17.578578+0000 osd.7 (osd.7) 652 : cluster 3 failed to encode map e1333 with expected crc 2023-10-09T09:58:18.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:18 smithi163 bash[75425]: cluster 2023-10-09T09:58:17.578578+0000 osd.7 (osd.7) 652 : cluster 3 failed to encode map e1333 with expected crc 2023-10-09T09:58:19.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:19 smithi181 bash[59345]: cluster 2023-10-09T09:58:18.347051+0000 mgr.y (mgr.34107) 1577 : cluster 0 pgmap v2498: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:19.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:19 smithi163 bash[78107]: cluster 2023-10-09T09:58:18.347051+0000 mgr.y (mgr.34107) 1577 : cluster 0 pgmap v2498: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:19.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:19 smithi163 bash[75425]: cluster 2023-10-09T09:58:18.347051+0000 mgr.y (mgr.34107) 1577 : cluster 0 pgmap v2498: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:22.265 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:22 smithi181 bash[59345]: cluster 2023-10-09T09:58:20.347851+0000 mgr.y (mgr.34107) 1578 : cluster 0 pgmap v2499: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:22.316 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:22 smithi181 bash[59345]: audit 2023-10-09T09:58:21.314477+0000 mon.a (mon.0) 2721 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:22 smithi163 bash[78107]: cluster 2023-10-09T09:58:20.347851+0000 mgr.y (mgr.34107) 1578 : cluster 0 pgmap v2499: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:22 smithi163 bash[78107]: audit 2023-10-09T09:58:21.314477+0000 mon.a (mon.0) 2721 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:22 smithi163 bash[75425]: cluster 2023-10-09T09:58:20.347851+0000 mgr.y (mgr.34107) 1578 : cluster 0 pgmap v2499: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:22 smithi163 bash[75425]: audit 2023-10-09T09:58:21.314477+0000 mon.a (mon.0) 2721 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:23.676 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:58:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:58:23] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.33.4" 2023-10-09T09:58:24.390 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:24 smithi163 bash[78107]: cluster 2023-10-09T09:58:22.348804+0000 mgr.y (mgr.34107) 1579 : cluster 0 pgmap v2500: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:24.390 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:24 smithi163 bash[75425]: cluster 2023-10-09T09:58:22.348804+0000 mgr.y (mgr.34107) 1579 : cluster 0 pgmap v2500: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:24.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:24 smithi181 bash[59345]: cluster 2023-10-09T09:58:22.348804+0000 mgr.y (mgr.34107) 1579 : cluster 0 pgmap v2500: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:26.440 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T09:58:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:26 smithi163 bash[75425]: cluster 2023-10-09T09:58:24.349823+0000 mgr.y (mgr.34107) 1580 : cluster 0 pgmap v2501: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:26 smithi163 bash[75425]: audit 2023-10-09T09:58:25.427425+0000 mon.a (mon.0) 2722 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:26 smithi163 bash[75425]: audit 2023-10-09T09:58:25.429270+0000 mon.a (mon.0) 2723 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:26 smithi163 bash[75425]: audit 2023-10-09T09:58:25.857847+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.163:0/3244507154' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:26.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:26 smithi163 bash[75425]: audit 2023-10-09T09:58:25.858558+0000 mon.a (mon.0) 2724 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:26.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:26 smithi163 bash[78107]: cluster 2023-10-09T09:58:24.349823+0000 mgr.y (mgr.34107) 1580 : cluster 0 pgmap v2501: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:26.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:26 smithi163 bash[78107]: audit 2023-10-09T09:58:25.427425+0000 mon.a (mon.0) 2722 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:26.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:26 smithi163 bash[78107]: audit 2023-10-09T09:58:25.429270+0000 mon.a (mon.0) 2723 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:26.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:26 smithi163 bash[78107]: audit 2023-10-09T09:58:25.857847+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.163:0/3244507154' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:26.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:26 smithi163 bash[78107]: audit 2023-10-09T09:58:25.858558+0000 mon.a (mon.0) 2724 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:26 smithi181 bash[59345]: cluster 2023-10-09T09:58:24.349823+0000 mgr.y (mgr.34107) 1580 : cluster 0 pgmap v2501: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:26 smithi181 bash[59345]: audit 2023-10-09T09:58:25.427425+0000 mon.a (mon.0) 2722 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:26 smithi181 bash[59345]: audit 2023-10-09T09:58:25.429270+0000 mon.a (mon.0) 2723 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:26 smithi181 bash[59345]: audit 2023-10-09T09:58:25.857847+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.163:0/3244507154' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:26 smithi181 bash[59345]: audit 2023-10-09T09:58:25.858558+0000 mon.a (mon.0) 2724 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:58:27.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T09:58:27.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:27 smithi163 bash[75425]: cluster 2023-10-09T09:58:26.350636+0000 mgr.y (mgr.34107) 1581 : cluster 0 pgmap v2502: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:27.754 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:27 smithi163 bash[75425]: cluster 2023-10-09T09:58:26.428868+0000 osd.5 (osd.5) 931 : cluster 3 failed to encode map e1334 with expected crc 2023-10-09T09:58:27.754 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:27 smithi163 bash[75425]: cluster 2023-10-09T09:58:26.433656+0000 osd.5 (osd.5) 932 : cluster 3 failed to encode map e1334 with expected crc 2023-10-09T09:58:27.754 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:27 smithi163 bash[75425]: audit 2023-10-09T09:58:26.434050+0000 mon.a (mon.0) 2725 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:58:27.754 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:27 smithi163 bash[75425]: cluster 2023-10-09T09:58:26.434313+0000 mon.a (mon.0) 2726 : cluster 0 osdmap e1334: 8 total, 8 up, 8 in 2023-10-09T09:58:27.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:27 smithi163 bash[78107]: cluster 2023-10-09T09:58:26.350636+0000 mgr.y (mgr.34107) 1581 : cluster 0 pgmap v2502: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:27.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:27 smithi163 bash[78107]: cluster 2023-10-09T09:58:26.428868+0000 osd.5 (osd.5) 931 : cluster 3 failed to encode map e1334 with expected crc 2023-10-09T09:58:27.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:27 smithi163 bash[78107]: cluster 2023-10-09T09:58:26.433656+0000 osd.5 (osd.5) 932 : cluster 3 failed to encode map e1334 with expected crc 2023-10-09T09:58:27.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:27 smithi163 bash[78107]: audit 2023-10-09T09:58:26.434050+0000 mon.a (mon.0) 2725 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:58:27.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:27 smithi163 bash[78107]: cluster 2023-10-09T09:58:26.434313+0000 mon.a (mon.0) 2726 : cluster 0 osdmap e1334: 8 total, 8 up, 8 in 2023-10-09T09:58:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:27 smithi181 bash[59345]: cluster 2023-10-09T09:58:26.350636+0000 mgr.y (mgr.34107) 1581 : cluster 0 pgmap v2502: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:27 smithi181 bash[59345]: cluster 2023-10-09T09:58:26.428868+0000 osd.5 (osd.5) 931 : cluster 3 failed to encode map e1334 with expected crc 2023-10-09T09:58:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:27 smithi181 bash[59345]: cluster 2023-10-09T09:58:26.433656+0000 osd.5 (osd.5) 932 : cluster 3 failed to encode map e1334 with expected crc 2023-10-09T09:58:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:27 smithi181 bash[59345]: audit 2023-10-09T09:58:26.434050+0000 mon.a (mon.0) 2725 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:58:27.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:27 smithi181 bash[59345]: cluster 2023-10-09T09:58:26.434313+0000 mon.a (mon.0) 2726 : cluster 0 osdmap e1334: 8 total, 8 up, 8 in 2023-10-09T09:58:28.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:28 smithi163 bash[78107]: cluster 2023-10-09T09:58:27.439176+0000 osd.5 (osd.5) 933 : cluster 3 failed to encode map e1335 with expected crc 2023-10-09T09:58:28.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:28 smithi163 bash[78107]: cluster 2023-10-09T09:58:27.440990+0000 mon.a (mon.0) 2727 : cluster 0 osdmap e1335: 8 total, 8 up, 8 in 2023-10-09T09:58:28.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:28 smithi163 bash[78107]: cluster 2023-10-09T09:58:27.442208+0000 osd.5 (osd.5) 934 : cluster 3 failed to encode map e1335 with expected crc 2023-10-09T09:58:28.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:28 smithi163 bash[78107]: cluster 2023-10-09T09:58:27.444187+0000 osd.5 (osd.5) 935 : cluster 3 failed to encode map e1335 with expected crc 2023-10-09T09:58:28.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:28 smithi163 bash[75425]: cluster 2023-10-09T09:58:27.439176+0000 osd.5 (osd.5) 933 : cluster 3 failed to encode map e1335 with expected crc 2023-10-09T09:58:28.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:28 smithi163 bash[75425]: cluster 2023-10-09T09:58:27.440990+0000 mon.a (mon.0) 2727 : cluster 0 osdmap e1335: 8 total, 8 up, 8 in 2023-10-09T09:58:28.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:28 smithi163 bash[75425]: cluster 2023-10-09T09:58:27.442208+0000 osd.5 (osd.5) 934 : cluster 3 failed to encode map e1335 with expected crc 2023-10-09T09:58:28.720 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:28 smithi163 bash[75425]: cluster 2023-10-09T09:58:27.444187+0000 osd.5 (osd.5) 935 : cluster 3 failed to encode map e1335 with expected crc 2023-10-09T09:58:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:28 smithi181 bash[59345]: cluster 2023-10-09T09:58:27.439176+0000 osd.5 (osd.5) 933 : cluster 3 failed to encode map e1335 with expected crc 2023-10-09T09:58:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:28 smithi181 bash[59345]: cluster 2023-10-09T09:58:27.440990+0000 mon.a (mon.0) 2727 : cluster 0 osdmap e1335: 8 total, 8 up, 8 in 2023-10-09T09:58:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:28 smithi181 bash[59345]: cluster 2023-10-09T09:58:27.442208+0000 osd.5 (osd.5) 934 : cluster 3 failed to encode map e1335 with expected crc 2023-10-09T09:58:28.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:28 smithi181 bash[59345]: cluster 2023-10-09T09:58:27.444187+0000 osd.5 (osd.5) 935 : cluster 3 failed to encode map e1335 with expected crc 2023-10-09T09:58:29.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:29 smithi163 bash[78107]: cluster 2023-10-09T09:58:28.351303+0000 mgr.y (mgr.34107) 1582 : cluster 0 pgmap v2505: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:29.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:29 smithi163 bash[75425]: cluster 2023-10-09T09:58:28.351303+0000 mgr.y (mgr.34107) 1582 : cluster 0 pgmap v2505: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:29.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:29 smithi181 bash[59345]: cluster 2023-10-09T09:58:28.351303+0000 mgr.y (mgr.34107) 1582 : cluster 0 pgmap v2505: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:32.162 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:32 smithi163 bash[75425]: cluster 2023-10-09T09:58:30.352058+0000 mgr.y (mgr.34107) 1583 : cluster 0 pgmap v2506: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:32.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:32 smithi163 bash[78107]: cluster 2023-10-09T09:58:30.352058+0000 mgr.y (mgr.34107) 1583 : cluster 0 pgmap v2506: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:32.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:32 smithi181 bash[59345]: cluster 2023-10-09T09:58:30.352058+0000 mgr.y (mgr.34107) 1583 : cluster 0 pgmap v2506: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:33.615 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:58:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:58:33] "GET /metrics HTTP/1.1" 200 34039 "" "Prometheus/2.33.4" 2023-10-09T09:58:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:34 smithi163 bash[75425]: cluster 2023-10-09T09:58:32.352997+0000 mgr.y (mgr.34107) 1584 : cluster 0 pgmap v2507: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:34 smithi163 bash[78107]: cluster 2023-10-09T09:58:32.352997+0000 mgr.y (mgr.34107) 1584 : cluster 0 pgmap v2507: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:34.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:34 smithi181 bash[59345]: cluster 2023-10-09T09:58:32.352997+0000 mgr.y (mgr.34107) 1584 : cluster 0 pgmap v2507: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:35.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:35 smithi163 bash[78107]: audit 2023-10-09T09:58:34.715071+0000 mon.a (mon.0) 2728 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:58:35.369 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:35 smithi163 bash[75425]: audit 2023-10-09T09:58:34.715071+0000 mon.a (mon.0) 2728 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:58:35.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:35 smithi181 bash[59345]: audit 2023-10-09T09:58:34.715071+0000 mon.a (mon.0) 2728 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:58:36.101 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T09:58:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:36 smithi163 bash[75425]: cluster 2023-10-09T09:58:34.353632+0000 mgr.y (mgr.34107) 1585 : cluster 0 pgmap v2508: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:36 smithi163 bash[75425]: audit 2023-10-09T09:58:35.765366+0000 mon.a (mon.0) 2729 : audit 1 from='client.? 172.21.15.163:0/2879309832' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:58:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:36 smithi163 bash[78107]: cluster 2023-10-09T09:58:34.353632+0000 mgr.y (mgr.34107) 1585 : cluster 0 pgmap v2508: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:36 smithi163 bash[78107]: audit 2023-10-09T09:58:35.765366+0000 mon.a (mon.0) 2729 : audit 1 from='client.? 172.21.15.163:0/2879309832' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:58:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:36 smithi181 bash[59345]: cluster 2023-10-09T09:58:34.353632+0000 mgr.y (mgr.34107) 1585 : cluster 0 pgmap v2508: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:36 smithi181 bash[59345]: audit 2023-10-09T09:58:35.765366+0000 mon.a (mon.0) 2729 : audit 1 from='client.? 172.21.15.163:0/2879309832' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:58:37.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T09:58:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:37 smithi163 bash[78107]: cluster 2023-10-09T09:58:36.092599+0000 osd.5 (osd.5) 936 : cluster 3 failed to encode map e1336 with expected crc 2023-10-09T09:58:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:37 smithi163 bash[78107]: audit 2023-10-09T09:58:36.094263+0000 mon.a (mon.0) 2730 : audit 1 from='client.? 172.21.15.163:0/2879309832' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:58:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:37 smithi163 bash[78107]: cluster 2023-10-09T09:58:36.094335+0000 mon.a (mon.0) 2731 : cluster 0 osdmap e1336: 8 total, 8 up, 8 in 2023-10-09T09:58:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:37 smithi163 bash[78107]: cluster 2023-10-09T09:58:36.186060+0000 osd.4 (osd.4) 709 : cluster 3 failed to encode map e1336 with expected crc 2023-10-09T09:58:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:37 smithi163 bash[75425]: cluster 2023-10-09T09:58:36.092599+0000 osd.5 (osd.5) 936 : cluster 3 failed to encode map e1336 with expected crc 2023-10-09T09:58:37.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:37 smithi163 bash[75425]: audit 2023-10-09T09:58:36.094263+0000 mon.a (mon.0) 2730 : audit 1 from='client.? 172.21.15.163:0/2879309832' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:58:37.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:37 smithi163 bash[75425]: cluster 2023-10-09T09:58:36.094335+0000 mon.a (mon.0) 2731 : cluster 0 osdmap e1336: 8 total, 8 up, 8 in 2023-10-09T09:58:37.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:37 smithi163 bash[75425]: cluster 2023-10-09T09:58:36.186060+0000 osd.4 (osd.4) 709 : cluster 3 failed to encode map e1336 with expected crc 2023-10-09T09:58:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:37 smithi181 bash[59345]: cluster 2023-10-09T09:58:36.092599+0000 osd.5 (osd.5) 936 : cluster 3 failed to encode map e1336 with expected crc 2023-10-09T09:58:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:37 smithi181 bash[59345]: audit 2023-10-09T09:58:36.094263+0000 mon.a (mon.0) 2730 : audit 1 from='client.? 172.21.15.163:0/2879309832' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:58:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:37 smithi181 bash[59345]: cluster 2023-10-09T09:58:36.094335+0000 mon.a (mon.0) 2731 : cluster 0 osdmap e1336: 8 total, 8 up, 8 in 2023-10-09T09:58:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:37 smithi181 bash[59345]: cluster 2023-10-09T09:58:36.186060+0000 osd.4 (osd.4) 709 : cluster 3 failed to encode map e1336 with expected crc 2023-10-09T09:58:38.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:38 smithi163 bash[75425]: cluster 2023-10-09T09:58:36.354202+0000 mgr.y (mgr.34107) 1586 : cluster 0 pgmap v2510: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:38 smithi163 bash[78107]: cluster 2023-10-09T09:58:36.354202+0000 mgr.y (mgr.34107) 1586 : cluster 0 pgmap v2510: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:38 smithi181 bash[59345]: cluster 2023-10-09T09:58:36.354202+0000 mgr.y (mgr.34107) 1586 : cluster 0 pgmap v2510: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:40.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:40 smithi163 bash[78107]: cluster 2023-10-09T09:58:38.355088+0000 mgr.y (mgr.34107) 1587 : cluster 0 pgmap v2511: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:40.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:40 smithi163 bash[75425]: cluster 2023-10-09T09:58:38.355088+0000 mgr.y (mgr.34107) 1587 : cluster 0 pgmap v2511: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:40 smithi181 bash[59345]: cluster 2023-10-09T09:58:38.355088+0000 mgr.y (mgr.34107) 1587 : cluster 0 pgmap v2511: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:41 smithi163 bash[75425]: audit 2023-10-09T09:58:40.423701+0000 mon.a (mon.0) 2732 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:41 smithi163 bash[75425]: audit 2023-10-09T09:58:40.817967+0000 mon.a (mon.0) 2733 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:41 smithi163 bash[75425]: audit 2023-10-09T09:58:40.824695+0000 mon.a (mon.0) 2734 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:41 smithi163 bash[75425]: audit 2023-10-09T09:58:41.019553+0000 mon.a (mon.0) 2735 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:41 smithi163 bash[75425]: audit 2023-10-09T09:58:41.029761+0000 mon.a (mon.0) 2736 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:41 smithi163 bash[78107]: audit 2023-10-09T09:58:40.423701+0000 mon.a (mon.0) 2732 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:41.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:41 smithi163 bash[78107]: audit 2023-10-09T09:58:40.817967+0000 mon.a (mon.0) 2733 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:41 smithi163 bash[78107]: audit 2023-10-09T09:58:40.824695+0000 mon.a (mon.0) 2734 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:41 smithi163 bash[78107]: audit 2023-10-09T09:58:41.019553+0000 mon.a (mon.0) 2735 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:41 smithi163 bash[78107]: audit 2023-10-09T09:58:41.029761+0000 mon.a (mon.0) 2736 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:41 smithi181 bash[59345]: audit 2023-10-09T09:58:40.423701+0000 mon.a (mon.0) 2732 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:41 smithi181 bash[59345]: audit 2023-10-09T09:58:40.817967+0000 mon.a (mon.0) 2733 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:41 smithi181 bash[59345]: audit 2023-10-09T09:58:40.824695+0000 mon.a (mon.0) 2734 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:41 smithi181 bash[59345]: audit 2023-10-09T09:58:41.019553+0000 mon.a (mon.0) 2735 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:41 smithi181 bash[59345]: audit 2023-10-09T09:58:41.029761+0000 mon.a (mon.0) 2736 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:41.626 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:42 smithi163 bash[75425]: cluster 2023-10-09T09:58:40.355830+0000 mgr.y (mgr.34107) 1588 : cluster 0 pgmap v2512: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:42 smithi163 bash[75425]: audit 2023-10-09T09:58:41.500059+0000 mon.a (mon.0) 2737 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:58:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:42 smithi163 bash[75425]: audit 2023-10-09T09:58:41.501858+0000 mon.a (mon.0) 2738 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:58:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:42 smithi163 bash[75425]: audit 2023-10-09T09:58:41.511467+0000 mon.a (mon.0) 2739 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:42 smithi163 bash[78107]: cluster 2023-10-09T09:58:40.355830+0000 mgr.y (mgr.34107) 1588 : cluster 0 pgmap v2512: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:42 smithi163 bash[78107]: audit 2023-10-09T09:58:41.500059+0000 mon.a (mon.0) 2737 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:58:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:42 smithi163 bash[78107]: audit 2023-10-09T09:58:41.501858+0000 mon.a (mon.0) 2738 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:58:42.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:42 smithi163 bash[78107]: audit 2023-10-09T09:58:41.511467+0000 mon.a (mon.0) 2739 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:42 smithi181 bash[59345]: cluster 2023-10-09T09:58:40.355830+0000 mgr.y (mgr.34107) 1588 : cluster 0 pgmap v2512: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:42 smithi181 bash[59345]: audit 2023-10-09T09:58:41.500059+0000 mon.a (mon.0) 2737 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:58:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:42 smithi181 bash[59345]: audit 2023-10-09T09:58:41.501858+0000 mon.a (mon.0) 2738 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:58:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:42 smithi181 bash[59345]: audit 2023-10-09T09:58:41.511467+0000 mon.a (mon.0) 2739 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:58:43.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:58:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:58:43] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.33.4" 2023-10-09T09:58:44.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:44 smithi163 bash[75425]: cluster 2023-10-09T09:58:42.356941+0000 mgr.y (mgr.34107) 1589 : cluster 0 pgmap v2513: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:44 smithi163 bash[78107]: cluster 2023-10-09T09:58:42.356941+0000 mgr.y (mgr.34107) 1589 : cluster 0 pgmap v2513: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:44.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:44 smithi181 bash[59345]: cluster 2023-10-09T09:58:42.356941+0000 mgr.y (mgr.34107) 1589 : cluster 0 pgmap v2513: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:46.138 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T09:58:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:46 smithi163 bash[75425]: cluster 2023-10-09T09:58:44.357740+0000 mgr.y (mgr.34107) 1590 : cluster 0 pgmap v2514: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:46 smithi163 bash[75425]: audit 2023-10-09T09:58:45.468729+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.163:0/1247846688' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:58:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:46 smithi163 bash[75425]: audit 2023-10-09T09:58:45.469577+0000 mon.a (mon.0) 2740 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:58:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:46 smithi163 bash[78107]: cluster 2023-10-09T09:58:44.357740+0000 mgr.y (mgr.34107) 1590 : cluster 0 pgmap v2514: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:46 smithi163 bash[78107]: audit 2023-10-09T09:58:45.468729+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.163:0/1247846688' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:58:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:46 smithi163 bash[78107]: audit 2023-10-09T09:58:45.469577+0000 mon.a (mon.0) 2740 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:58:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:46 smithi181 bash[59345]: cluster 2023-10-09T09:58:44.357740+0000 mgr.y (mgr.34107) 1590 : cluster 0 pgmap v2514: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:46 smithi181 bash[59345]: audit 2023-10-09T09:58:45.468729+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.163:0/1247846688' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:58:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:46 smithi181 bash[59345]: audit 2023-10-09T09:58:45.469577+0000 mon.a (mon.0) 2740 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:58:47.074 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T09:58:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:47 smithi163 bash[78107]: cluster 2023-10-09T09:58:46.127117+0000 osd.5 (osd.5) 937 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:47 smithi163 bash[78107]: audit 2023-10-09T09:58:46.131397+0000 mon.a (mon.0) 2741 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:58:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:47 smithi163 bash[78107]: cluster 2023-10-09T09:58:46.131495+0000 mon.a (mon.0) 2742 : cluster 0 osdmap e1337: 8 total, 8 up, 8 in 2023-10-09T09:58:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:47 smithi163 bash[78107]: cluster 2023-10-09T09:58:46.533864+0000 osd.7 (osd.7) 653 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:47.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:47 smithi163 bash[75425]: cluster 2023-10-09T09:58:46.127117+0000 osd.5 (osd.5) 937 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:47.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:47 smithi163 bash[75425]: audit 2023-10-09T09:58:46.131397+0000 mon.a (mon.0) 2741 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:58:47.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:47 smithi163 bash[75425]: cluster 2023-10-09T09:58:46.131495+0000 mon.a (mon.0) 2742 : cluster 0 osdmap e1337: 8 total, 8 up, 8 in 2023-10-09T09:58:47.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:47 smithi163 bash[75425]: cluster 2023-10-09T09:58:46.533864+0000 osd.7 (osd.7) 653 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:47.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:47 smithi181 bash[59345]: cluster 2023-10-09T09:58:46.127117+0000 osd.5 (osd.5) 937 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:47.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:47 smithi181 bash[59345]: audit 2023-10-09T09:58:46.131397+0000 mon.a (mon.0) 2741 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:58:47.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:47 smithi181 bash[59345]: cluster 2023-10-09T09:58:46.131495+0000 mon.a (mon.0) 2742 : cluster 0 osdmap e1337: 8 total, 8 up, 8 in 2023-10-09T09:58:47.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:47 smithi181 bash[59345]: cluster 2023-10-09T09:58:46.533864+0000 osd.7 (osd.7) 653 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:48 smithi163 bash[78107]: cluster 2023-10-09T09:58:46.358553+0000 mgr.y (mgr.34107) 1591 : cluster 0 pgmap v2516: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:48 smithi163 bash[78107]: cluster 2023-10-09T09:58:47.071161+0000 osd.4 (osd.4) 710 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:48 smithi163 bash[78107]: cluster 2023-10-09T09:58:47.071371+0000 osd.6 (osd.6) 607 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:48.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:48 smithi163 bash[75425]: cluster 2023-10-09T09:58:46.358553+0000 mgr.y (mgr.34107) 1591 : cluster 0 pgmap v2516: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:48.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:48 smithi163 bash[75425]: cluster 2023-10-09T09:58:47.071161+0000 osd.4 (osd.4) 710 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:48.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:48 smithi163 bash[75425]: cluster 2023-10-09T09:58:47.071371+0000 osd.6 (osd.6) 607 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:48 smithi181 bash[59345]: cluster 2023-10-09T09:58:46.358553+0000 mgr.y (mgr.34107) 1591 : cluster 0 pgmap v2516: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:48 smithi181 bash[59345]: cluster 2023-10-09T09:58:47.071161+0000 osd.4 (osd.4) 710 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:48 smithi181 bash[59345]: cluster 2023-10-09T09:58:47.071371+0000 osd.6 (osd.6) 607 : cluster 3 failed to encode map e1337 with expected crc 2023-10-09T09:58:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:50 smithi163 bash[78107]: cluster 2023-10-09T09:58:48.359443+0000 mgr.y (mgr.34107) 1592 : cluster 0 pgmap v2517: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:50 smithi163 bash[75425]: cluster 2023-10-09T09:58:48.359443+0000 mgr.y (mgr.34107) 1592 : cluster 0 pgmap v2517: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:50.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:50 smithi181 bash[59345]: cluster 2023-10-09T09:58:48.359443+0000 mgr.y (mgr.34107) 1592 : cluster 0 pgmap v2517: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:51.851 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:52.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:52 smithi163 bash[75425]: cluster 2023-10-09T09:58:50.360289+0000 mgr.y (mgr.34107) 1593 : cluster 0 pgmap v2518: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:52.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:52 smithi163 bash[78107]: cluster 2023-10-09T09:58:50.360289+0000 mgr.y (mgr.34107) 1593 : cluster 0 pgmap v2518: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:52.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:52 smithi181 bash[59345]: cluster 2023-10-09T09:58:50.360289+0000 mgr.y (mgr.34107) 1593 : cluster 0 pgmap v2518: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:53.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:58:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:58:53] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.33.4" 2023-10-09T09:58:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:54 smithi163 bash[78107]: cluster 2023-10-09T09:58:52.361220+0000 mgr.y (mgr.34107) 1594 : cluster 0 pgmap v2519: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:54 smithi163 bash[75425]: cluster 2023-10-09T09:58:52.361220+0000 mgr.y (mgr.34107) 1594 : cluster 0 pgmap v2519: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:54.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:54 smithi181 bash[59345]: cluster 2023-10-09T09:58:52.361220+0000 mgr.y (mgr.34107) 1594 : cluster 0 pgmap v2519: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:56.174 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T09:58:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:56 smithi163 bash[78107]: cluster 2023-10-09T09:58:54.361883+0000 mgr.y (mgr.34107) 1595 : cluster 0 pgmap v2520: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:56 smithi163 bash[78107]: audit 2023-10-09T09:58:55.423637+0000 mon.a (mon.0) 2743 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:56 smithi163 bash[78107]: audit 2023-10-09T09:58:55.640226+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.163:0/1237125644' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:58:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:56 smithi163 bash[78107]: audit 2023-10-09T09:58:55.641002+0000 mon.a (mon.0) 2744 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:58:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:56 smithi163 bash[75425]: cluster 2023-10-09T09:58:54.361883+0000 mgr.y (mgr.34107) 1595 : cluster 0 pgmap v2520: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:56 smithi163 bash[75425]: audit 2023-10-09T09:58:55.423637+0000 mon.a (mon.0) 2743 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:56 smithi163 bash[75425]: audit 2023-10-09T09:58:55.640226+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.163:0/1237125644' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:58:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:56 smithi163 bash[75425]: audit 2023-10-09T09:58:55.641002+0000 mon.a (mon.0) 2744 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:58:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:56 smithi181 bash[59345]: cluster 2023-10-09T09:58:54.361883+0000 mgr.y (mgr.34107) 1595 : cluster 0 pgmap v2520: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:56 smithi181 bash[59345]: audit 2023-10-09T09:58:55.423637+0000 mon.a (mon.0) 2743 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:56 smithi181 bash[59345]: audit 2023-10-09T09:58:55.640226+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.163:0/1237125644' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:58:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:56 smithi181 bash[59345]: audit 2023-10-09T09:58:55.641002+0000 mon.a (mon.0) 2744 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:58:57.060 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T09:58:57.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:57 smithi163 bash[75425]: cluster 2023-10-09T09:58:56.167156+0000 osd.5 (osd.5) 938 : cluster 3 failed to encode map e1338 with expected crc 2023-10-09T09:58:57.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:57 smithi163 bash[75425]: cluster 2023-10-09T09:58:56.167238+0000 osd.6 (osd.6) 608 : cluster 3 failed to encode map e1338 with expected crc 2023-10-09T09:58:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:57 smithi163 bash[75425]: audit 2023-10-09T09:58:56.169020+0000 mon.a (mon.0) 2745 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:58:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:57 smithi163 bash[75425]: cluster 2023-10-09T09:58:56.169142+0000 mon.a (mon.0) 2746 : cluster 0 osdmap e1338: 8 total, 8 up, 8 in 2023-10-09T09:58:57.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:57 smithi163 bash[78107]: cluster 2023-10-09T09:58:56.167156+0000 osd.5 (osd.5) 938 : cluster 3 failed to encode map e1338 with expected crc 2023-10-09T09:58:57.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:57 smithi163 bash[78107]: cluster 2023-10-09T09:58:56.167238+0000 osd.6 (osd.6) 608 : cluster 3 failed to encode map e1338 with expected crc 2023-10-09T09:58:57.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:57 smithi163 bash[78107]: audit 2023-10-09T09:58:56.169020+0000 mon.a (mon.0) 2745 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:58:57.471 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:57 smithi163 bash[78107]: cluster 2023-10-09T09:58:56.169142+0000 mon.a (mon.0) 2746 : cluster 0 osdmap e1338: 8 total, 8 up, 8 in 2023-10-09T09:58:57.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:57 smithi181 bash[59345]: cluster 2023-10-09T09:58:56.167156+0000 osd.5 (osd.5) 938 : cluster 3 failed to encode map e1338 with expected crc 2023-10-09T09:58:57.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:57 smithi181 bash[59345]: cluster 2023-10-09T09:58:56.167238+0000 osd.6 (osd.6) 608 : cluster 3 failed to encode map e1338 with expected crc 2023-10-09T09:58:57.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:57 smithi181 bash[59345]: audit 2023-10-09T09:58:56.169020+0000 mon.a (mon.0) 2745 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:58:57.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:57 smithi181 bash[59345]: cluster 2023-10-09T09:58:56.169142+0000 mon.a (mon.0) 2746 : cluster 0 osdmap e1338: 8 total, 8 up, 8 in 2023-10-09T09:58:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:58 smithi163 bash[75425]: cluster 2023-10-09T09:58:56.362620+0000 mgr.y (mgr.34107) 1596 : cluster 0 pgmap v2522: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:58 smithi163 bash[78107]: cluster 2023-10-09T09:58:56.362620+0000 mgr.y (mgr.34107) 1596 : cluster 0 pgmap v2522: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:58.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:58 smithi181 bash[59345]: cluster 2023-10-09T09:58:56.362620+0000 mgr.y (mgr.34107) 1596 : cluster 0 pgmap v2522: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:59.567 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:58:59 smithi181 bash[59345]: cluster 2023-10-09T09:58:58.363496+0000 mgr.y (mgr.34107) 1597 : cluster 0 pgmap v2523: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:58:59 smithi163 bash[78107]: cluster 2023-10-09T09:58:58.363496+0000 mgr.y (mgr.34107) 1597 : cluster 0 pgmap v2523: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:58:59.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:58:59 smithi163 bash[75425]: cluster 2023-10-09T09:58:58.363496+0000 mgr.y (mgr.34107) 1597 : cluster 0 pgmap v2523: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:01.845 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:02.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:02 smithi163 bash[75425]: cluster 2023-10-09T09:59:00.364250+0000 mgr.y (mgr.34107) 1598 : cluster 0 pgmap v2524: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:02.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:02 smithi163 bash[78107]: cluster 2023-10-09T09:59:00.364250+0000 mgr.y (mgr.34107) 1598 : cluster 0 pgmap v2524: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:02.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:02 smithi181 bash[59345]: cluster 2023-10-09T09:59:00.364250+0000 mgr.y (mgr.34107) 1598 : cluster 0 pgmap v2524: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:03.418 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:59:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:59:03] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.33.4" 2023-10-09T09:59:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:04 smithi163 bash[75425]: cluster 2023-10-09T09:59:02.365278+0000 mgr.y (mgr.34107) 1599 : cluster 0 pgmap v2525: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:04 smithi163 bash[78107]: cluster 2023-10-09T09:59:02.365278+0000 mgr.y (mgr.34107) 1599 : cluster 0 pgmap v2525: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:04 smithi181 bash[59345]: cluster 2023-10-09T09:59:02.365278+0000 mgr.y (mgr.34107) 1599 : cluster 0 pgmap v2525: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:06.095 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T09:59:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:06 smithi163 bash[75425]: cluster 2023-10-09T09:59:04.366078+0000 mgr.y (mgr.34107) 1600 : cluster 0 pgmap v2526: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:06 smithi163 bash[75425]: audit 2023-10-09T09:59:05.768075+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.163:0/4059156977' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:06 smithi163 bash[75425]: audit 2023-10-09T09:59:05.768839+0000 mon.a (mon.0) 2747 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:06 smithi163 bash[78107]: cluster 2023-10-09T09:59:04.366078+0000 mgr.y (mgr.34107) 1600 : cluster 0 pgmap v2526: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:06 smithi163 bash[78107]: audit 2023-10-09T09:59:05.768075+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.163:0/4059156977' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:06.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:06 smithi163 bash[78107]: audit 2023-10-09T09:59:05.768839+0000 mon.a (mon.0) 2747 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:06.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:06 smithi181 bash[59345]: cluster 2023-10-09T09:59:04.366078+0000 mgr.y (mgr.34107) 1600 : cluster 0 pgmap v2526: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:06.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:06 smithi181 bash[59345]: audit 2023-10-09T09:59:05.768075+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.163:0/4059156977' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:06.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:06 smithi181 bash[59345]: audit 2023-10-09T09:59:05.768839+0000 mon.a (mon.0) 2747 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:07.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T09:59:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:07 smithi163 bash[78107]: cluster 2023-10-09T09:59:06.085268+0000 osd.5 (osd.5) 939 : cluster 3 failed to encode map e1339 with expected crc 2023-10-09T09:59:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:07 smithi163 bash[78107]: cluster 2023-10-09T09:59:06.086123+0000 osd.6 (osd.6) 609 : cluster 3 failed to encode map e1339 with expected crc 2023-10-09T09:59:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:07 smithi163 bash[78107]: audit 2023-10-09T09:59:06.086924+0000 mon.a (mon.0) 2748 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:59:07.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:07 smithi163 bash[78107]: cluster 2023-10-09T09:59:06.087000+0000 mon.a (mon.0) 2749 : cluster 0 osdmap e1339: 8 total, 8 up, 8 in 2023-10-09T09:59:07.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:07 smithi163 bash[75425]: cluster 2023-10-09T09:59:06.085268+0000 osd.5 (osd.5) 939 : cluster 3 failed to encode map e1339 with expected crc 2023-10-09T09:59:07.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:07 smithi163 bash[75425]: cluster 2023-10-09T09:59:06.086123+0000 osd.6 (osd.6) 609 : cluster 3 failed to encode map e1339 with expected crc 2023-10-09T09:59:07.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:07 smithi163 bash[75425]: audit 2023-10-09T09:59:06.086924+0000 mon.a (mon.0) 2748 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:59:07.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:07 smithi163 bash[75425]: cluster 2023-10-09T09:59:06.087000+0000 mon.a (mon.0) 2749 : cluster 0 osdmap e1339: 8 total, 8 up, 8 in 2023-10-09T09:59:07.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:07 smithi181 bash[59345]: cluster 2023-10-09T09:59:06.085268+0000 osd.5 (osd.5) 939 : cluster 3 failed to encode map e1339 with expected crc 2023-10-09T09:59:07.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:07 smithi181 bash[59345]: cluster 2023-10-09T09:59:06.086123+0000 osd.6 (osd.6) 609 : cluster 3 failed to encode map e1339 with expected crc 2023-10-09T09:59:07.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:07 smithi181 bash[59345]: audit 2023-10-09T09:59:06.086924+0000 mon.a (mon.0) 2748 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:59:07.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:07 smithi181 bash[59345]: cluster 2023-10-09T09:59:06.087000+0000 mon.a (mon.0) 2749 : cluster 0 osdmap e1339: 8 total, 8 up, 8 in 2023-10-09T09:59:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:08 smithi163 bash[78107]: cluster 2023-10-09T09:59:06.366940+0000 mgr.y (mgr.34107) 1601 : cluster 0 pgmap v2528: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:08 smithi163 bash[75425]: cluster 2023-10-09T09:59:06.366940+0000 mgr.y (mgr.34107) 1601 : cluster 0 pgmap v2528: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:08 smithi181 bash[59345]: cluster 2023-10-09T09:59:06.366940+0000 mgr.y (mgr.34107) 1601 : cluster 0 pgmap v2528: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:10.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:10 smithi163 bash[78107]: cluster 2023-10-09T09:59:08.367694+0000 mgr.y (mgr.34107) 1602 : cluster 0 pgmap v2529: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:10 smithi163 bash[75425]: cluster 2023-10-09T09:59:08.367694+0000 mgr.y (mgr.34107) 1602 : cluster 0 pgmap v2529: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:10.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:10 smithi181 bash[59345]: cluster 2023-10-09T09:59:08.367694+0000 mgr.y (mgr.34107) 1602 : cluster 0 pgmap v2529: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:11 smithi163 bash[78107]: audit 2023-10-09T09:59:10.424152+0000 mon.a (mon.0) 2750 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:11 smithi163 bash[75425]: audit 2023-10-09T09:59:10.424152+0000 mon.a (mon.0) 2750 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:11.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:11 smithi181 bash[59345]: audit 2023-10-09T09:59:10.424152+0000 mon.a (mon.0) 2750 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:11.789 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:12 smithi163 bash[75425]: cluster 2023-10-09T09:59:10.368494+0000 mgr.y (mgr.34107) 1603 : cluster 0 pgmap v2530: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:12.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:12 smithi163 bash[78107]: cluster 2023-10-09T09:59:10.368494+0000 mgr.y (mgr.34107) 1603 : cluster 0 pgmap v2530: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:12.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:12 smithi181 bash[59345]: cluster 2023-10-09T09:59:10.368494+0000 mgr.y (mgr.34107) 1603 : cluster 0 pgmap v2530: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:13.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:59:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:59:13] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.33.4" 2023-10-09T09:59:14.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:14 smithi163 bash[78107]: cluster 2023-10-09T09:59:12.369460+0000 mgr.y (mgr.34107) 1604 : cluster 0 pgmap v2531: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:14 smithi163 bash[75425]: cluster 2023-10-09T09:59:12.369460+0000 mgr.y (mgr.34107) 1604 : cluster 0 pgmap v2531: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:14.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:14 smithi181 bash[59345]: cluster 2023-10-09T09:59:12.369460+0000 mgr.y (mgr.34107) 1604 : cluster 0 pgmap v2531: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:16.137 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T09:59:16.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:16 smithi163 bash[75425]: cluster 2023-10-09T09:59:14.370244+0000 mgr.y (mgr.34107) 1605 : cluster 0 pgmap v2532: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:16.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:16 smithi163 bash[75425]: audit 2023-10-09T09:59:15.391513+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.163:0/831740541' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:59:16.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:16 smithi163 bash[75425]: audit 2023-10-09T09:59:15.392433+0000 mon.a (mon.0) 2751 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:59:16.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:16 smithi163 bash[78107]: cluster 2023-10-09T09:59:14.370244+0000 mgr.y (mgr.34107) 1605 : cluster 0 pgmap v2532: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:16.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:16 smithi163 bash[78107]: audit 2023-10-09T09:59:15.391513+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.163:0/831740541' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:59:16.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:16 smithi163 bash[78107]: audit 2023-10-09T09:59:15.392433+0000 mon.a (mon.0) 2751 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:59:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:16 smithi181 bash[59345]: cluster 2023-10-09T09:59:14.370244+0000 mgr.y (mgr.34107) 1605 : cluster 0 pgmap v2532: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:16 smithi181 bash[59345]: audit 2023-10-09T09:59:15.391513+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.163:0/831740541' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:59:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:16 smithi181 bash[59345]: audit 2023-10-09T09:59:15.392433+0000 mon.a (mon.0) 2751 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:59:17.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T09:59:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:17 smithi163 bash[78107]: cluster 2023-10-09T09:59:16.127899+0000 osd.5 (osd.5) 940 : cluster 3 failed to encode map e1340 with expected crc 2023-10-09T09:59:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:17 smithi163 bash[78107]: cluster 2023-10-09T09:59:16.128603+0000 osd.6 (osd.6) 610 : cluster 3 failed to encode map e1340 with expected crc 2023-10-09T09:59:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:17 smithi163 bash[78107]: audit 2023-10-09T09:59:16.130324+0000 mon.a (mon.0) 2752 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:59:17.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:17 smithi163 bash[78107]: cluster 2023-10-09T09:59:16.130473+0000 mon.a (mon.0) 2753 : cluster 0 osdmap e1340: 8 total, 8 up, 8 in 2023-10-09T09:59:17.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:17 smithi163 bash[75425]: cluster 2023-10-09T09:59:16.127899+0000 osd.5 (osd.5) 940 : cluster 3 failed to encode map e1340 with expected crc 2023-10-09T09:59:17.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:17 smithi163 bash[75425]: cluster 2023-10-09T09:59:16.128603+0000 osd.6 (osd.6) 610 : cluster 3 failed to encode map e1340 with expected crc 2023-10-09T09:59:17.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:17 smithi163 bash[75425]: audit 2023-10-09T09:59:16.130324+0000 mon.a (mon.0) 2752 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:59:17.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:17 smithi163 bash[75425]: cluster 2023-10-09T09:59:16.130473+0000 mon.a (mon.0) 2753 : cluster 0 osdmap e1340: 8 total, 8 up, 8 in 2023-10-09T09:59:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:17 smithi181 bash[59345]: cluster 2023-10-09T09:59:16.127899+0000 osd.5 (osd.5) 940 : cluster 3 failed to encode map e1340 with expected crc 2023-10-09T09:59:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:17 smithi181 bash[59345]: cluster 2023-10-09T09:59:16.128603+0000 osd.6 (osd.6) 610 : cluster 3 failed to encode map e1340 with expected crc 2023-10-09T09:59:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:17 smithi181 bash[59345]: audit 2023-10-09T09:59:16.130324+0000 mon.a (mon.0) 2752 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:59:17.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:17 smithi181 bash[59345]: cluster 2023-10-09T09:59:16.130473+0000 mon.a (mon.0) 2753 : cluster 0 osdmap e1340: 8 total, 8 up, 8 in 2023-10-09T09:59:18.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:18 smithi163 bash[75425]: cluster 2023-10-09T09:59:16.371074+0000 mgr.y (mgr.34107) 1606 : cluster 0 pgmap v2534: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:18.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:18 smithi163 bash[78107]: cluster 2023-10-09T09:59:16.371074+0000 mgr.y (mgr.34107) 1606 : cluster 0 pgmap v2534: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:18.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:18 smithi181 bash[59345]: cluster 2023-10-09T09:59:16.371074+0000 mgr.y (mgr.34107) 1606 : cluster 0 pgmap v2534: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:20.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:20 smithi163 bash[75425]: cluster 2023-10-09T09:59:18.371847+0000 mgr.y (mgr.34107) 1607 : cluster 0 pgmap v2535: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:20 smithi163 bash[78107]: cluster 2023-10-09T09:59:18.371847+0000 mgr.y (mgr.34107) 1607 : cluster 0 pgmap v2535: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:20.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:20 smithi181 bash[59345]: cluster 2023-10-09T09:59:18.371847+0000 mgr.y (mgr.34107) 1607 : cluster 0 pgmap v2535: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:21.828 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:22 smithi163 bash[78107]: cluster 2023-10-09T09:59:20.372693+0000 mgr.y (mgr.34107) 1608 : cluster 0 pgmap v2536: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:22 smithi163 bash[75425]: cluster 2023-10-09T09:59:20.372693+0000 mgr.y (mgr.34107) 1608 : cluster 0 pgmap v2536: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:22.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:22 smithi181 bash[59345]: cluster 2023-10-09T09:59:20.372693+0000 mgr.y (mgr.34107) 1608 : cluster 0 pgmap v2536: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:23.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:59:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:59:23] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.33.4" 2023-10-09T09:59:24.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:24 smithi163 bash[78107]: cluster 2023-10-09T09:59:22.373706+0000 mgr.y (mgr.34107) 1609 : cluster 0 pgmap v2537: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:24.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:24 smithi163 bash[75425]: cluster 2023-10-09T09:59:22.373706+0000 mgr.y (mgr.34107) 1609 : cluster 0 pgmap v2537: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:24.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:24 smithi181 bash[59345]: cluster 2023-10-09T09:59:22.373706+0000 mgr.y (mgr.34107) 1609 : cluster 0 pgmap v2537: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:26.175 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T09:59:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:26 smithi163 bash[75425]: cluster 2023-10-09T09:59:24.374497+0000 mgr.y (mgr.34107) 1610 : cluster 0 pgmap v2538: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:26 smithi163 bash[75425]: audit 2023-10-09T09:59:25.424552+0000 mon.a (mon.0) 2754 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:26 smithi163 bash[75425]: audit 2023-10-09T09:59:25.566899+0000 mon.a (mon.0) 2755 : audit 1 from='client.? 172.21.15.163:0/3685646587' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:59:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:26 smithi163 bash[78107]: cluster 2023-10-09T09:59:24.374497+0000 mgr.y (mgr.34107) 1610 : cluster 0 pgmap v2538: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:26 smithi163 bash[78107]: audit 2023-10-09T09:59:25.424552+0000 mon.a (mon.0) 2754 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:26 smithi163 bash[78107]: audit 2023-10-09T09:59:25.566899+0000 mon.a (mon.0) 2755 : audit 1 from='client.? 172.21.15.163:0/3685646587' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:59:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:26 smithi181 bash[59345]: cluster 2023-10-09T09:59:24.374497+0000 mgr.y (mgr.34107) 1610 : cluster 0 pgmap v2538: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:26 smithi181 bash[59345]: audit 2023-10-09T09:59:25.424552+0000 mon.a (mon.0) 2754 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:26 smithi181 bash[59345]: audit 2023-10-09T09:59:25.566899+0000 mon.a (mon.0) 2755 : audit 1 from='client.? 172.21.15.163:0/3685646587' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:59:27.081 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T09:59:27.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:27 smithi163 bash[78107]: cluster 2023-10-09T09:59:26.162222+0000 osd.5 (osd.5) 941 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:27.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:27 smithi163 bash[78107]: audit 2023-10-09T09:59:26.165936+0000 mon.a (mon.0) 2756 : audit 1 from='client.? 172.21.15.163:0/3685646587' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:59:27.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:27 smithi163 bash[78107]: cluster 2023-10-09T09:59:26.166114+0000 mon.a (mon.0) 2757 : cluster 0 osdmap e1341: 8 total, 8 up, 8 in 2023-10-09T09:59:27.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:27 smithi163 bash[78107]: cluster 2023-10-09T09:59:26.172267+0000 osd.7 (osd.7) 654 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:27.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:27 smithi163 bash[75425]: cluster 2023-10-09T09:59:26.162222+0000 osd.5 (osd.5) 941 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:27.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:27 smithi163 bash[75425]: audit 2023-10-09T09:59:26.165936+0000 mon.a (mon.0) 2756 : audit 1 from='client.? 172.21.15.163:0/3685646587' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:59:27.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:27 smithi163 bash[75425]: cluster 2023-10-09T09:59:26.166114+0000 mon.a (mon.0) 2757 : cluster 0 osdmap e1341: 8 total, 8 up, 8 in 2023-10-09T09:59:27.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:27 smithi163 bash[75425]: cluster 2023-10-09T09:59:26.172267+0000 osd.7 (osd.7) 654 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:27.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:27 smithi181 bash[59345]: cluster 2023-10-09T09:59:26.162222+0000 osd.5 (osd.5) 941 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:27.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:27 smithi181 bash[59345]: audit 2023-10-09T09:59:26.165936+0000 mon.a (mon.0) 2756 : audit 1 from='client.? 172.21.15.163:0/3685646587' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:59:27.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:27 smithi181 bash[59345]: cluster 2023-10-09T09:59:26.166114+0000 mon.a (mon.0) 2757 : cluster 0 osdmap e1341: 8 total, 8 up, 8 in 2023-10-09T09:59:27.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:27 smithi181 bash[59345]: cluster 2023-10-09T09:59:26.172267+0000 osd.7 (osd.7) 654 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[78107]: cluster 2023-10-09T09:59:26.375026+0000 mgr.y (mgr.34107) 1611 : cluster 0 pgmap v2540: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[78107]: cluster 2023-10-09T09:59:26.662630+0000 osd.4 (osd.4) 711 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[78107]: cluster 2023-10-09T09:59:26.662797+0000 osd.6 (osd.6) 611 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[78107]: cluster 2023-10-09T09:59:27.186404+0000 osd.6 (osd.6) 612 : cluster 3 failed to encode map e1342 with expected crc 2023-10-09T09:59:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[78107]: cluster 2023-10-09T09:59:27.186481+0000 osd.5 (osd.5) 942 : cluster 3 failed to encode map e1342 with expected crc 2023-10-09T09:59:28.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[78107]: cluster 2023-10-09T09:59:27.189693+0000 mon.a (mon.0) 2758 : cluster 0 osdmap e1342: 8 total, 8 up, 8 in 2023-10-09T09:59:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[75425]: cluster 2023-10-09T09:59:26.375026+0000 mgr.y (mgr.34107) 1611 : cluster 0 pgmap v2540: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[75425]: cluster 2023-10-09T09:59:26.662630+0000 osd.4 (osd.4) 711 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[75425]: cluster 2023-10-09T09:59:26.662797+0000 osd.6 (osd.6) 611 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[75425]: cluster 2023-10-09T09:59:27.186404+0000 osd.6 (osd.6) 612 : cluster 3 failed to encode map e1342 with expected crc 2023-10-09T09:59:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[75425]: cluster 2023-10-09T09:59:27.186481+0000 osd.5 (osd.5) 942 : cluster 3 failed to encode map e1342 with expected crc 2023-10-09T09:59:28.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:28 smithi163 bash[75425]: cluster 2023-10-09T09:59:27.189693+0000 mon.a (mon.0) 2758 : cluster 0 osdmap e1342: 8 total, 8 up, 8 in 2023-10-09T09:59:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:28 smithi181 bash[59345]: cluster 2023-10-09T09:59:26.375026+0000 mgr.y (mgr.34107) 1611 : cluster 0 pgmap v2540: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:28 smithi181 bash[59345]: cluster 2023-10-09T09:59:26.662630+0000 osd.4 (osd.4) 711 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:28 smithi181 bash[59345]: cluster 2023-10-09T09:59:26.662797+0000 osd.6 (osd.6) 611 : cluster 3 failed to encode map e1341 with expected crc 2023-10-09T09:59:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:28 smithi181 bash[59345]: cluster 2023-10-09T09:59:27.186404+0000 osd.6 (osd.6) 612 : cluster 3 failed to encode map e1342 with expected crc 2023-10-09T09:59:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:28 smithi181 bash[59345]: cluster 2023-10-09T09:59:27.186481+0000 osd.5 (osd.5) 942 : cluster 3 failed to encode map e1342 with expected crc 2023-10-09T09:59:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:28 smithi181 bash[59345]: cluster 2023-10-09T09:59:27.189693+0000 mon.a (mon.0) 2758 : cluster 0 osdmap e1342: 8 total, 8 up, 8 in 2023-10-09T09:59:29.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:29 smithi163 bash[78107]: cluster 2023-10-09T09:59:28.375663+0000 mgr.y (mgr.34107) 1612 : cluster 0 pgmap v2542: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:29.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:29 smithi163 bash[75425]: cluster 2023-10-09T09:59:28.375663+0000 mgr.y (mgr.34107) 1612 : cluster 0 pgmap v2542: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:29.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:29 smithi181 bash[59345]: cluster 2023-10-09T09:59:28.375663+0000 mgr.y (mgr.34107) 1612 : cluster 0 pgmap v2542: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:31.860 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:32.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:32 smithi163 bash[78107]: cluster 2023-10-09T09:59:30.376523+0000 mgr.y (mgr.34107) 1613 : cluster 0 pgmap v2543: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:32.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:32 smithi163 bash[75425]: cluster 2023-10-09T09:59:30.376523+0000 mgr.y (mgr.34107) 1613 : cluster 0 pgmap v2543: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:32.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:32 smithi181 bash[59345]: cluster 2023-10-09T09:59:30.376523+0000 mgr.y (mgr.34107) 1613 : cluster 0 pgmap v2543: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:33.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:59:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:59:33] "GET /metrics HTTP/1.1" 200 34034 "" "Prometheus/2.33.4" 2023-10-09T09:59:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:34 smithi163 bash[78107]: cluster 2023-10-09T09:59:32.377265+0000 mgr.y (mgr.34107) 1614 : cluster 0 pgmap v2544: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:34 smithi163 bash[75425]: cluster 2023-10-09T09:59:32.377265+0000 mgr.y (mgr.34107) 1614 : cluster 0 pgmap v2544: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:34.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:34 smithi181 bash[59345]: cluster 2023-10-09T09:59:32.377265+0000 mgr.y (mgr.34107) 1614 : cluster 0 pgmap v2544: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:36.107 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T09:59:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:36 smithi163 bash[78107]: cluster 2023-10-09T09:59:34.377947+0000 mgr.y (mgr.34107) 1615 : cluster 0 pgmap v2545: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:36 smithi163 bash[78107]: audit 2023-10-09T09:59:35.615419+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.163:0/735155876' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:36 smithi163 bash[78107]: audit 2023-10-09T09:59:35.615994+0000 mon.a (mon.0) 2759 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:36 smithi163 bash[75425]: cluster 2023-10-09T09:59:34.377947+0000 mgr.y (mgr.34107) 1615 : cluster 0 pgmap v2545: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:36.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:36 smithi163 bash[75425]: audit 2023-10-09T09:59:35.615419+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.163:0/735155876' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:36.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:36 smithi163 bash[75425]: audit 2023-10-09T09:59:35.615994+0000 mon.a (mon.0) 2759 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:36 smithi181 bash[59345]: cluster 2023-10-09T09:59:34.377947+0000 mgr.y (mgr.34107) 1615 : cluster 0 pgmap v2545: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:36 smithi181 bash[59345]: audit 2023-10-09T09:59:35.615419+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.163:0/735155876' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:36.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:36 smithi181 bash[59345]: audit 2023-10-09T09:59:35.615994+0000 mon.a (mon.0) 2759 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:37.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T09:59:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:37 smithi163 bash[78107]: cluster 2023-10-09T09:59:36.097209+0000 osd.5 (osd.5) 943 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:37 smithi163 bash[78107]: audit 2023-10-09T09:59:36.099286+0000 mon.a (mon.0) 2760 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:59:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:37 smithi163 bash[78107]: cluster 2023-10-09T09:59:36.099405+0000 mon.a (mon.0) 2761 : cluster 0 osdmap e1343: 8 total, 8 up, 8 in 2023-10-09T09:59:37.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:37 smithi163 bash[78107]: cluster 2023-10-09T09:59:36.926804+0000 osd.7 (osd.7) 655 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:37.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:37 smithi163 bash[75425]: cluster 2023-10-09T09:59:36.097209+0000 osd.5 (osd.5) 943 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:37.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:37 smithi163 bash[75425]: audit 2023-10-09T09:59:36.099286+0000 mon.a (mon.0) 2760 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:59:37.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:37 smithi163 bash[75425]: cluster 2023-10-09T09:59:36.099405+0000 mon.a (mon.0) 2761 : cluster 0 osdmap e1343: 8 total, 8 up, 8 in 2023-10-09T09:59:37.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:37 smithi163 bash[75425]: cluster 2023-10-09T09:59:36.926804+0000 osd.7 (osd.7) 655 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:37 smithi181 bash[59345]: cluster 2023-10-09T09:59:36.097209+0000 osd.5 (osd.5) 943 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:37 smithi181 bash[59345]: audit 2023-10-09T09:59:36.099286+0000 mon.a (mon.0) 2760 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:59:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:37 smithi181 bash[59345]: cluster 2023-10-09T09:59:36.099405+0000 mon.a (mon.0) 2761 : cluster 0 osdmap e1343: 8 total, 8 up, 8 in 2023-10-09T09:59:37.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:37 smithi181 bash[59345]: cluster 2023-10-09T09:59:36.926804+0000 osd.7 (osd.7) 655 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:38.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:38 smithi163 bash[75425]: cluster 2023-10-09T09:59:36.378627+0000 mgr.y (mgr.34107) 1616 : cluster 0 pgmap v2547: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:38.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:38 smithi163 bash[75425]: cluster 2023-10-09T09:59:36.927112+0000 osd.6 (osd.6) 613 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:38.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:38 smithi163 bash[75425]: cluster 2023-10-09T09:59:36.927388+0000 osd.4 (osd.4) 712 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:38 smithi163 bash[78107]: cluster 2023-10-09T09:59:36.378627+0000 mgr.y (mgr.34107) 1616 : cluster 0 pgmap v2547: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:38.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:38 smithi163 bash[78107]: cluster 2023-10-09T09:59:36.927112+0000 osd.6 (osd.6) 613 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:38.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:38 smithi163 bash[78107]: cluster 2023-10-09T09:59:36.927388+0000 osd.4 (osd.4) 712 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:38 smithi181 bash[59345]: cluster 2023-10-09T09:59:36.378627+0000 mgr.y (mgr.34107) 1616 : cluster 0 pgmap v2547: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:38 smithi181 bash[59345]: cluster 2023-10-09T09:59:36.927112+0000 osd.6 (osd.6) 613 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:38.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:38 smithi181 bash[59345]: cluster 2023-10-09T09:59:36.927388+0000 osd.4 (osd.4) 712 : cluster 3 failed to encode map e1343 with expected crc 2023-10-09T09:59:40.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:40 smithi163 bash[75425]: cluster 2023-10-09T09:59:38.379431+0000 mgr.y (mgr.34107) 1617 : cluster 0 pgmap v2548: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:40.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:40 smithi163 bash[78107]: cluster 2023-10-09T09:59:38.379431+0000 mgr.y (mgr.34107) 1617 : cluster 0 pgmap v2548: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:40.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:40 smithi181 bash[59345]: cluster 2023-10-09T09:59:38.379431+0000 mgr.y (mgr.34107) 1617 : cluster 0 pgmap v2548: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:41.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:41 smithi163 bash[78107]: audit 2023-10-09T09:59:40.424881+0000 mon.a (mon.0) 2762 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:41 smithi163 bash[75425]: audit 2023-10-09T09:59:40.424881+0000 mon.a (mon.0) 2762 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:41.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:41 smithi181 bash[59345]: audit 2023-10-09T09:59:40.424881+0000 mon.a (mon.0) 2762 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:41.855 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:42.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:42 smithi163 bash[78107]: cluster 2023-10-09T09:59:40.380250+0000 mgr.y (mgr.34107) 1618 : cluster 0 pgmap v2549: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:42.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:42 smithi163 bash[78107]: audit 2023-10-09T09:59:41.594768+0000 mon.a (mon.0) 2763 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:59:42.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:42 smithi163 bash[75425]: cluster 2023-10-09T09:59:40.380250+0000 mgr.y (mgr.34107) 1618 : cluster 0 pgmap v2549: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:42.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:42 smithi163 bash[75425]: audit 2023-10-09T09:59:41.594768+0000 mon.a (mon.0) 2763 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:59:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:42 smithi181 bash[59345]: cluster 2023-10-09T09:59:40.380250+0000 mgr.y (mgr.34107) 1618 : cluster 0 pgmap v2549: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:42 smithi181 bash[59345]: audit 2023-10-09T09:59:41.594768+0000 mon.a (mon.0) 2763 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:59:43.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:59:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:59:43] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.33.4" 2023-10-09T09:59:44.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:44 smithi163 bash[75425]: cluster 2023-10-09T09:59:42.381222+0000 mgr.y (mgr.34107) 1619 : cluster 0 pgmap v2550: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:44.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:44 smithi163 bash[78107]: cluster 2023-10-09T09:59:42.381222+0000 mgr.y (mgr.34107) 1619 : cluster 0 pgmap v2550: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:44.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:44 smithi181 bash[59345]: cluster 2023-10-09T09:59:42.381222+0000 mgr.y (mgr.34107) 1619 : cluster 0 pgmap v2550: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:46.154 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T09:59:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:46 smithi163 bash[78107]: cluster 2023-10-09T09:59:44.381994+0000 mgr.y (mgr.34107) 1620 : cluster 0 pgmap v2551: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:46 smithi163 bash[78107]: audit 2023-10-09T09:59:45.449079+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.163:0/2157791199' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:46.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:46 smithi163 bash[78107]: audit 2023-10-09T09:59:45.449864+0000 mon.a (mon.0) 2764 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:46 smithi163 bash[75425]: cluster 2023-10-09T09:59:44.381994+0000 mgr.y (mgr.34107) 1620 : cluster 0 pgmap v2551: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:46 smithi163 bash[75425]: audit 2023-10-09T09:59:45.449079+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.163:0/2157791199' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:46.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:46 smithi163 bash[75425]: audit 2023-10-09T09:59:45.449864+0000 mon.a (mon.0) 2764 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:46.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:46 smithi181 bash[59345]: cluster 2023-10-09T09:59:44.381994+0000 mgr.y (mgr.34107) 1620 : cluster 0 pgmap v2551: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:46 smithi181 bash[59345]: audit 2023-10-09T09:59:45.449079+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.163:0/2157791199' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:46.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:46 smithi181 bash[59345]: audit 2023-10-09T09:59:45.449864+0000 mon.a (mon.0) 2764 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:59:47.085 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T09:59:47.086 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T09:59:47.086 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T09:59:47.086 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T09:59:47.086 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T09:59:47.086 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:59:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:47 smithi163 bash[78107]: cluster 2023-10-09T09:59:46.143435+0000 osd.5 (osd.5) 944 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:47 smithi163 bash[78107]: cluster 2023-10-09T09:59:46.144617+0000 osd.6 (osd.6) 614 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:47 smithi163 bash[78107]: audit 2023-10-09T09:59:46.146308+0000 mon.a (mon.0) 2765 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:59:47.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:47 smithi163 bash[78107]: cluster 2023-10-09T09:59:46.146516+0000 mon.a (mon.0) 2766 : cluster 0 osdmap e1344: 8 total, 8 up, 8 in 2023-10-09T09:59:47.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:47 smithi163 bash[75425]: cluster 2023-10-09T09:59:46.143435+0000 osd.5 (osd.5) 944 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:47.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:47 smithi163 bash[75425]: cluster 2023-10-09T09:59:46.144617+0000 osd.6 (osd.6) 614 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:47.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:47 smithi163 bash[75425]: audit 2023-10-09T09:59:46.146308+0000 mon.a (mon.0) 2765 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:59:47.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:47 smithi163 bash[75425]: cluster 2023-10-09T09:59:46.146516+0000 mon.a (mon.0) 2766 : cluster 0 osdmap e1344: 8 total, 8 up, 8 in 2023-10-09T09:59:47.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:47 smithi181 bash[59345]: cluster 2023-10-09T09:59:46.143435+0000 osd.5 (osd.5) 944 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:47.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:47 smithi181 bash[59345]: cluster 2023-10-09T09:59:46.144617+0000 osd.6 (osd.6) 614 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:47.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:47 smithi181 bash[59345]: audit 2023-10-09T09:59:46.146308+0000 mon.a (mon.0) 2765 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:59:47.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:47 smithi181 bash[59345]: cluster 2023-10-09T09:59:46.146516+0000 mon.a (mon.0) 2766 : cluster 0 osdmap e1344: 8 total, 8 up, 8 in 2023-10-09T09:59:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:48 smithi163 bash[78107]: cluster 2023-10-09T09:59:46.382808+0000 mgr.y (mgr.34107) 1621 : cluster 0 pgmap v2553: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:48 smithi163 bash[78107]: audit 2023-10-09T09:59:47.892102+0000 mon.a (mon.0) 2767 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:48 smithi163 bash[78107]: audit 2023-10-09T09:59:47.902398+0000 mon.a (mon.0) 2768 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:48.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:48 smithi163 bash[78107]: cluster 2023-10-09T09:59:47.974612+0000 osd.7 (osd.7) 656 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:48.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:48 smithi163 bash[75425]: cluster 2023-10-09T09:59:46.382808+0000 mgr.y (mgr.34107) 1621 : cluster 0 pgmap v2553: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:48.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:48 smithi163 bash[75425]: audit 2023-10-09T09:59:47.892102+0000 mon.a (mon.0) 2767 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:48.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:48 smithi163 bash[75425]: audit 2023-10-09T09:59:47.902398+0000 mon.a (mon.0) 2768 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:48.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:48 smithi163 bash[75425]: cluster 2023-10-09T09:59:47.974612+0000 osd.7 (osd.7) 656 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:48 smithi181 bash[59345]: cluster 2023-10-09T09:59:46.382808+0000 mgr.y (mgr.34107) 1621 : cluster 0 pgmap v2553: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:48 smithi181 bash[59345]: audit 2023-10-09T09:59:47.892102+0000 mon.a (mon.0) 2767 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:48 smithi181 bash[59345]: audit 2023-10-09T09:59:47.902398+0000 mon.a (mon.0) 2768 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:48 smithi181 bash[59345]: cluster 2023-10-09T09:59:47.974612+0000 osd.7 (osd.7) 656 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:49.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:49 smithi163 bash[78107]: cluster 2023-10-09T09:59:48.167154+0000 osd.4 (osd.4) 713 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:49.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:49 smithi163 bash[78107]: audit 2023-10-09T09:59:48.768642+0000 mon.a (mon.0) 2769 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:49.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:49 smithi163 bash[78107]: audit 2023-10-09T09:59:48.776498+0000 mon.a (mon.0) 2770 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:49 smithi163 bash[75425]: cluster 2023-10-09T09:59:48.167154+0000 osd.4 (osd.4) 713 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:49 smithi163 bash[75425]: audit 2023-10-09T09:59:48.768642+0000 mon.a (mon.0) 2769 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:49.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:49 smithi163 bash[75425]: audit 2023-10-09T09:59:48.776498+0000 mon.a (mon.0) 2770 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:49.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:49 smithi181 bash[59345]: cluster 2023-10-09T09:59:48.167154+0000 osd.4 (osd.4) 713 : cluster 3 failed to encode map e1344 with expected crc 2023-10-09T09:59:49.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:49 smithi181 bash[59345]: audit 2023-10-09T09:59:48.768642+0000 mon.a (mon.0) 2769 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:49.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:49 smithi181 bash[59345]: audit 2023-10-09T09:59:48.776498+0000 mon.a (mon.0) 2770 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:50 smithi163 bash[75425]: cluster 2023-10-09T09:59:48.384420+0000 mgr.y (mgr.34107) 1622 : cluster 0 pgmap v2554: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:50 smithi163 bash[75425]: audit 2023-10-09T09:59:49.249011+0000 mon.a (mon.0) 2771 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:59:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:50 smithi163 bash[75425]: audit 2023-10-09T09:59:49.250767+0000 mon.a (mon.0) 2772 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:59:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:50 smithi163 bash[75425]: audit 2023-10-09T09:59:49.262586+0000 mon.a (mon.0) 2773 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:50 smithi163 bash[78107]: cluster 2023-10-09T09:59:48.384420+0000 mgr.y (mgr.34107) 1622 : cluster 0 pgmap v2554: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:50 smithi163 bash[78107]: audit 2023-10-09T09:59:49.249011+0000 mon.a (mon.0) 2771 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:59:50.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:50 smithi163 bash[78107]: audit 2023-10-09T09:59:49.250767+0000 mon.a (mon.0) 2772 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:59:50.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:50 smithi163 bash[78107]: audit 2023-10-09T09:59:49.262586+0000 mon.a (mon.0) 2773 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:50.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:50 smithi181 bash[59345]: cluster 2023-10-09T09:59:48.384420+0000 mgr.y (mgr.34107) 1622 : cluster 0 pgmap v2554: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:50.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:50 smithi181 bash[59345]: audit 2023-10-09T09:59:49.249011+0000 mon.a (mon.0) 2771 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:59:50.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:50 smithi181 bash[59345]: audit 2023-10-09T09:59:49.250767+0000 mon.a (mon.0) 2772 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:59:50.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:50 smithi181 bash[59345]: audit 2023-10-09T09:59:49.262586+0000 mon.a (mon.0) 2773 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T09:59:51.622 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:52.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:52 smithi163 bash[75425]: cluster 2023-10-09T09:59:50.385225+0000 mgr.y (mgr.34107) 1623 : cluster 0 pgmap v2555: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:52.470 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:52 smithi163 bash[78107]: cluster 2023-10-09T09:59:50.385225+0000 mgr.y (mgr.34107) 1623 : cluster 0 pgmap v2555: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:52.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:52 smithi181 bash[59345]: cluster 2023-10-09T09:59:50.385225+0000 mgr.y (mgr.34107) 1623 : cluster 0 pgmap v2555: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:53.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 09:59:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:09:59:53] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.33.4" 2023-10-09T09:59:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:54 smithi163 bash[78107]: cluster 2023-10-09T09:59:52.386184+0000 mgr.y (mgr.34107) 1624 : cluster 0 pgmap v2556: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:54 smithi163 bash[75425]: cluster 2023-10-09T09:59:52.386184+0000 mgr.y (mgr.34107) 1624 : cluster 0 pgmap v2556: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:54.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:54 smithi181 bash[59345]: cluster 2023-10-09T09:59:52.386184+0000 mgr.y (mgr.34107) 1624 : cluster 0 pgmap v2556: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:55.223 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T09:59:55.223 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":1344,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:59:46.138890+0000","last_up_change":"2023-10-09T09:48:06.923694+0000","last_in_change":"2023-10-09T09:56:56.044985+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"152","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:01:33.304980+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1310","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":1310,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:03:23.129371+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:03:23.743990+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":581,"last_clean_end":1120,"up_from":1136,"up_thru":1341,"down_at":1121,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":3998822132},{"type":"v1","addr":"172.21.15.163:6803","nonce":3998822132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":3998822132},{"type":"v1","addr":"172.21.15.163:6805","nonce":3998822132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":3998822132},{"type":"v1","addr":"172.21.15.163:6809","nonce":3998822132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":3998822132},{"type":"v1","addr":"172.21.15.163:6807","nonce":3998822132}]},"public_addr":"172.21.15.163:6803/3998822132","cluster_addr":"172.21.15.163:6805/3998822132","heartbeat_back_addr":"172.21.15.163:6809/3998822132","heartbeat_front_addr":"172.21.15.163:6807/3998822132","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":607,"last_clean_end":1241,"up_from":1246,"up_thru":1323,"down_at":1242,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":3739078161},{"type":"v1","addr":"172.21.15.163:6811","nonce":3739078161}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":3739078161},{"type":"v1","addr":"172.21.15.163:6813","nonce":3739078161}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":3739078161},{"type":"v1","addr":"172.21.15.163:6817","nonce":3739078161}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":3739078161},{"type":"v1","addr":"172.21.15.163:6815","nonce":3739078161}]},"public_addr":"172.21.15.163:6811/3739078161","cluster_addr":"172.21.15.163:6813/3739078161","heartbeat_back_addr":"172.21.15.163:6817/3739078161","heartbeat_front_addr":"172.21.15.163:6815/3739078161","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":620,"up_from":634,"up_thru":1323,"down_at":621,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6819","nonce":2749314903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6821","nonce":2749314903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6825","nonce":2749314903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6823","nonce":2749314903}]},"public_addr":"172.21.15.163:6819/2749314903","cluster_addr":"172.21.15.163:6821/2749314903","heartbeat_back_addr":"172.21.15.163:6825/2749314903","heartbeat_front_addr":"172.21.15.163:6823/2749314903","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":648,"up_from":663,"up_thru":1328,"down_at":649,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6827","nonce":198665121}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6829","nonce":198665121}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6833","nonce":198665121}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6831","nonce":198665121}]},"public_addr":"172.21.15.163:6827/198665121","cluster_addr":"172.21.15.163:6829/198665121","heartbeat_back_addr":"172.21.15.163:6833/198665121","heartbeat_front_addr":"172.21.15.163:6831/198665121","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":1331,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":1334,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":1320,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":1323,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":1120},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":1241},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":620},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":648},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/3642753484":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6801/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/1436605859":"2023-10-10T09:09:25.185363+0000","172.21.15.181:0/394087329":"2023-10-10T09:08:23.434950+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/623143106":"2023-10-10T09:06:46.871855+0000","172.21.15.181:0/1788638593":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/689482878":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/3090084368":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.181:0/2794784192":"2023-10-10T09:08:23.434950+0000","172.21.15.181:0/769962302":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/2563190797":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/2199287559":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6800/1737049372":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6801/1737049372":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/536711455":"2023-10-10T09:06:46.871855+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6800/3919298633":"2023-10-10T09:06:46.871855+0000","172.21.15.163:6801/3919298633":"2023-10-10T09:06:46.871855+0000","172.21.15.181:6835/3017734555":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/566729834":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/1618377325":"2023-10-10T09:09:25.185363+0000","172.21.15.181:6834/3017734555":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:59:55.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:55 smithi181 bash[59345]: cluster 2023-10-09T09:59:54.387069+0000 mgr.y (mgr.34107) 1625 : cluster 0 pgmap v2557: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:55.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:55 smithi181 bash[59345]: audit 2023-10-09T09:59:55.219208+0000 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.163:0/680037869' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:59:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:55 smithi163 bash[78107]: cluster 2023-10-09T09:59:54.387069+0000 mgr.y (mgr.34107) 1625 : cluster 0 pgmap v2557: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:55.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:55 smithi163 bash[78107]: audit 2023-10-09T09:59:55.219208+0000 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.163:0/680037869' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:59:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:55 smithi163 bash[75425]: cluster 2023-10-09T09:59:54.387069+0000 mgr.y (mgr.34107) 1625 : cluster 0 pgmap v2557: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:55.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:55 smithi163 bash[75425]: audit 2023-10-09T09:59:55.219208+0000 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.163:0/680037869' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:59:56.143 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T09:59:56.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:59:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:56 smithi163 bash[75425]: audit 2023-10-09T09:59:55.424690+0000 mon.a (mon.0) 2774 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:56 smithi163 bash[78107]: audit 2023-10-09T09:59:55.424690+0000 mon.a (mon.0) 2774 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:56.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:56 smithi181 bash[59345]: audit 2023-10-09T09:59:55.424690+0000 mon.a (mon.0) 2774 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:57.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:57 smithi181 bash[59345]: cluster 2023-10-09T09:59:56.387842+0000 mgr.y (mgr.34107) 1626 : cluster 0 pgmap v2558: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:57.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:57 smithi163 bash[78107]: cluster 2023-10-09T09:59:56.387842+0000 mgr.y (mgr.34107) 1626 : cluster 0 pgmap v2558: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:57.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:57 smithi163 bash[75425]: cluster 2023-10-09T09:59:56.387842+0000 mgr.y (mgr.34107) 1626 : cluster 0 pgmap v2558: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 09:59:59 smithi163 bash[78107]: cluster 2023-10-09T09:59:58.388575+0000 mgr.y (mgr.34107) 1627 : cluster 0 pgmap v2559: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:59.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 09:59:59 smithi163 bash[75425]: cluster 2023-10-09T09:59:58.388575+0000 mgr.y (mgr.34107) 1627 : cluster 0 pgmap v2559: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T09:59:59.815 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 09:59:59 smithi181 bash[59345]: cluster 2023-10-09T09:59:58.388575+0000 mgr.y (mgr.34107) 1627 : cluster 0 pgmap v2559: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:00.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:00 smithi163 bash[78107]: cluster 2023-10-09T10:00:00.000109+0000 mon.a (mon.0) 2775 : cluster 1 overall HEALTH_OK 2023-10-09T10:00:00.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:00 smithi163 bash[75425]: cluster 2023-10-09T10:00:00.000109+0000 mon.a (mon.0) 2775 : cluster 1 overall HEALTH_OK 2023-10-09T10:00:00.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:00 smithi181 bash[59345]: cluster 2023-10-09T10:00:00.000109+0000 mon.a (mon.0) 2775 : cluster 1 overall HEALTH_OK 2023-10-09T10:00:00.919 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:02.356 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:02 smithi163 bash[78107]: cluster 2023-10-09T10:00:00.389350+0000 mgr.y (mgr.34107) 1628 : cluster 0 pgmap v2560: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:02.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:02 smithi163 bash[75425]: cluster 2023-10-09T10:00:00.389350+0000 mgr.y (mgr.34107) 1628 : cluster 0 pgmap v2560: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:02.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:02 smithi181 bash[59345]: cluster 2023-10-09T10:00:00.389350+0000 mgr.y (mgr.34107) 1628 : cluster 0 pgmap v2560: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:03.719 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:00:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:10:00:03] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.33.4" 2023-10-09T10:00:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:04 smithi163 bash[78107]: cluster 2023-10-09T10:00:02.390335+0000 mgr.y (mgr.34107) 1629 : cluster 0 pgmap v2561: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:04 smithi163 bash[75425]: cluster 2023-10-09T10:00:02.390335+0000 mgr.y (mgr.34107) 1629 : cluster 0 pgmap v2561: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:04 smithi181 bash[59345]: cluster 2023-10-09T10:00:02.390335+0000 mgr.y (mgr.34107) 1629 : cluster 0 pgmap v2561: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:04.592 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T10:00:04.593 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":1344,"fsid":"097c0480-6681-11ee-8db5-212e2dc638e7","created":"2023-10-09T08:53:11.193188+0000","modified":"2023-10-09T09:59:46.138890+0000","last_up_change":"2023-10-09T09:48:06.923694+0000","last_in_change":"2023-10-09T09:56:56.044985+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T08:57:13.290916+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"152","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:01:33.304980+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1310","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":1310,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:03:23.129371+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:03:23.743990+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"61","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"09899392-dd0c-4eaf-b933-aafe194ff2b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":581,"last_clean_end":1120,"up_from":1136,"up_thru":1341,"down_at":1121,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":3998822132},{"type":"v1","addr":"172.21.15.163:6803","nonce":3998822132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":3998822132},{"type":"v1","addr":"172.21.15.163:6805","nonce":3998822132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":3998822132},{"type":"v1","addr":"172.21.15.163:6809","nonce":3998822132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":3998822132},{"type":"v1","addr":"172.21.15.163:6807","nonce":3998822132}]},"public_addr":"172.21.15.163:6803/3998822132","cluster_addr":"172.21.15.163:6805/3998822132","heartbeat_back_addr":"172.21.15.163:6809/3998822132","heartbeat_front_addr":"172.21.15.163:6807/3998822132","state":["exists","up"]},{"osd":1,"uuid":"fb74bcac-37d3-472b-abae-6c72df5b264b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":607,"last_clean_end":1241,"up_from":1246,"up_thru":1323,"down_at":1242,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":3739078161},{"type":"v1","addr":"172.21.15.163:6811","nonce":3739078161}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":3739078161},{"type":"v1","addr":"172.21.15.163:6813","nonce":3739078161}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":3739078161},{"type":"v1","addr":"172.21.15.163:6817","nonce":3739078161}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":3739078161},{"type":"v1","addr":"172.21.15.163:6815","nonce":3739078161}]},"public_addr":"172.21.15.163:6811/3739078161","cluster_addr":"172.21.15.163:6813/3739078161","heartbeat_back_addr":"172.21.15.163:6817/3739078161","heartbeat_front_addr":"172.21.15.163:6815/3739078161","state":["exists","up"]},{"osd":2,"uuid":"3f368e12-b9e7-46dc-93eb-5feee9e4ea2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":620,"up_from":634,"up_thru":1323,"down_at":621,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6819","nonce":2749314903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6821","nonce":2749314903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6825","nonce":2749314903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2749314903},{"type":"v1","addr":"172.21.15.163:6823","nonce":2749314903}]},"public_addr":"172.21.15.163:6819/2749314903","cluster_addr":"172.21.15.163:6821/2749314903","heartbeat_back_addr":"172.21.15.163:6825/2749314903","heartbeat_front_addr":"172.21.15.163:6823/2749314903","state":["exists","up"]},{"osd":3,"uuid":"228a3c3f-20e9-4d74-a360-64c80ceadaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":648,"up_from":663,"up_thru":1328,"down_at":649,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6827","nonce":198665121}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6829","nonce":198665121}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6833","nonce":198665121}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":198665121},{"type":"v1","addr":"172.21.15.163:6831","nonce":198665121}]},"public_addr":"172.21.15.163:6827/198665121","cluster_addr":"172.21.15.163:6829/198665121","heartbeat_back_addr":"172.21.15.163:6833/198665121","heartbeat_front_addr":"172.21.15.163:6831/198665121","state":["exists","up"]},{"osd":4,"uuid":"70b76f58-2db5-421c-a558-db7c00067177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":1331,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6801","nonce":2112712913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6803","nonce":2112712913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6807","nonce":2112712913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2112712913},{"type":"v1","addr":"172.21.15.181:6805","nonce":2112712913}]},"public_addr":"172.21.15.181:6801/2112712913","cluster_addr":"172.21.15.181:6803/2112712913","heartbeat_back_addr":"172.21.15.181:6807/2112712913","heartbeat_front_addr":"172.21.15.181:6805/2112712913","state":["exists","up"]},{"osd":5,"uuid":"ea6afd8e-b32e-4bf6-a460-0fd5f45de7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":1334,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6809","nonce":1025108733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6811","nonce":1025108733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6815","nonce":1025108733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1025108733},{"type":"v1","addr":"172.21.15.181:6813","nonce":1025108733}]},"public_addr":"172.21.15.181:6809/1025108733","cluster_addr":"172.21.15.181:6811/1025108733","heartbeat_back_addr":"172.21.15.181:6815/1025108733","heartbeat_front_addr":"172.21.15.181:6813/1025108733","state":["exists","up"]},{"osd":6,"uuid":"5de7d354-0821-4d89-ad3e-25c5aaed463a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":1320,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6817","nonce":1623825176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623825176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623825176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623825176},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623825176}]},"public_addr":"172.21.15.181:6817/1623825176","cluster_addr":"172.21.15.181:6819/1623825176","heartbeat_back_addr":"172.21.15.181:6823/1623825176","heartbeat_front_addr":"172.21.15.181:6821/1623825176","state":["exists","up"]},{"osd":7,"uuid":"fbb1831a-b34a-4d3a-b19b-507f65324930","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":1323,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6825","nonce":800314963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6827","nonce":800314963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6831","nonce":800314963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":800314963},{"type":"v1","addr":"172.21.15.181:6829","nonce":800314963}]},"public_addr":"172.21.15.181:6825/800314963","cluster_addr":"172.21.15.181:6827/800314963","heartbeat_back_addr":"172.21.15.181:6831/800314963","heartbeat_front_addr":"172.21.15.181:6829/800314963","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:55:48.509473+0000","dead_epoch":1120},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:56:29.770329+0000","dead_epoch":1241},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:11.170522+0000","dead_epoch":620},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:57:52.328658+0000","dead_epoch":648},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:58:31.119895+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:11.498389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T08:59:52.424534+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:00:34.236510+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/3642753484":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/1436605859":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/689482878":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6800/1737049372":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/3090084368":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/2563190797":"2023-10-10T09:09:25.185363+0000","172.21.15.163:6800/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6801/1080938935":"2023-10-10T09:01:07.146560+0000","172.21.15.163:6801/1737049372":"2023-10-10T09:09:25.185363+0000","172.21.15.163:0/536711455":"2023-10-10T09:06:46.871855+0000","172.21.15.163:6801/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.181:0/2794784192":"2023-10-10T09:08:23.434950+0000","172.21.15.181:0/769962302":"2023-10-10T09:08:23.434950+0000","172.21.15.181:0/394087329":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/2199287559":"2023-10-10T09:01:07.146560+0000","172.21.15.163:0/2356626184":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/1916578112":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6800/1232823824":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6800/3919298633":"2023-10-10T09:06:46.871855+0000","172.21.15.163:6801/3919298633":"2023-10-10T09:06:46.871855+0000","172.21.15.163:0/1618377325":"2023-10-10T09:09:25.185363+0000","172.21.15.181:6835/3017734555":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/566729834":"2023-10-10T09:01:07.146560+0000","172.21.15.181:6834/3017734555":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/3487497332":"2023-10-10T08:53:37.161178+0000","172.21.15.163:6801/829587412":"2023-10-10T08:54:05.000912+0000","172.21.15.163:0/623143106":"2023-10-10T09:06:46.871855+0000","172.21.15.181:0/1788638593":"2023-10-10T09:08:23.434950+0000","172.21.15.163:0/625604585":"2023-10-10T08:54:05.000912+0000","172.21.15.163:6800/1080938935":"2023-10-10T09:01:07.146560+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:00:05.111 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:05 smithi163 bash[75425]: audit 2023-10-09T10:00:04.588451+0000 mon.a (mon.0) 2776 : audit 0 from='client.? 172.21.15.163:0/3272358841' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:00:05.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:05 smithi163 bash[78107]: audit 2023-10-09T10:00:04.588451+0000 mon.a (mon.0) 2776 : audit 0 from='client.? 172.21.15.163:0/3272358841' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:00:05.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T10:00:05.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T10:00:05.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T10:00:05.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T10:00:05.510 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T10:00:05.510 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T10:00:05.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T10:00:05.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T10:00:05.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:05 smithi181 bash[59345]: audit 2023-10-09T10:00:04.588451+0000 mon.a (mon.0) 2776 : audit 0 from='client.? 172.21.15.163:0/3272358841' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:00:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:06 smithi163 bash[78107]: cluster 2023-10-09T10:00:04.391091+0000 mgr.y (mgr.34107) 1630 : cluster 0 pgmap v2562: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:06 smithi163 bash[75425]: cluster 2023-10-09T10:00:04.391091+0000 mgr.y (mgr.34107) 1630 : cluster 0 pgmap v2562: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:06.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:06 smithi181 bash[59345]: cluster 2023-10-09T10:00:04.391091+0000 mgr.y (mgr.34107) 1630 : cluster 0 pgmap v2562: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:08 smithi163 bash[75425]: cluster 2023-10-09T10:00:06.391968+0000 mgr.y (mgr.34107) 1631 : cluster 0 pgmap v2563: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:08 smithi163 bash[78107]: cluster 2023-10-09T10:00:06.391968+0000 mgr.y (mgr.34107) 1631 : cluster 0 pgmap v2563: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:08.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:08 smithi181 bash[59345]: cluster 2023-10-09T10:00:06.391968+0000 mgr.y (mgr.34107) 1631 : cluster 0 pgmap v2563: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:10.391 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:10.393 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:10.393 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:10.394 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:10.395 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:10.396 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:10.401 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:10.409 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:10.427 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:10 smithi163 bash[78107]: cluster 2023-10-09T10:00:08.392545+0000 mgr.y (mgr.34107) 1632 : cluster 0 pgmap v2564: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:10.427 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:10 smithi163 bash[75425]: cluster 2023-10-09T10:00:08.392545+0000 mgr.y (mgr.34107) 1632 : cluster 0 pgmap v2564: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:10.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:10 smithi181 bash[59345]: cluster 2023-10-09T10:00:08.392545+0000 mgr.y (mgr.34107) 1632 : cluster 0 pgmap v2564: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:11.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:11 smithi163 bash[78107]: audit 2023-10-09T10:00:10.428739+0000 mon.a (mon.0) 2777 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:11.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:11 smithi163 bash[75425]: audit 2023-10-09T10:00:10.428739+0000 mon.a (mon.0) 2777 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:11.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:11 smithi181 bash[59345]: audit 2023-10-09T10:00:10.428739+0000 mon.a (mon.0) 2777 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:12.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:12 smithi163 bash[75425]: cluster 2023-10-09T10:00:10.393495+0000 mgr.y (mgr.34107) 1633 : cluster 0 pgmap v2565: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:12.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:12 smithi163 bash[78107]: cluster 2023-10-09T10:00:10.393495+0000 mgr.y (mgr.34107) 1633 : cluster 0 pgmap v2565: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:12.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:12 smithi181 bash[59345]: cluster 2023-10-09T10:00:10.393495+0000 mgr.y (mgr.34107) 1633 : cluster 0 pgmap v2565: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:13.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:00:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:10:00:13] "GET /metrics HTTP/1.1" 200 34034 "" "Prometheus/2.33.4" 2023-10-09T10:00:14.189 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:14 smithi163 bash[78107]: cluster 2023-10-09T10:00:12.394488+0000 mgr.y (mgr.34107) 1634 : cluster 0 pgmap v2566: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:14.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:14 smithi163 bash[75425]: cluster 2023-10-09T10:00:12.394488+0000 mgr.y (mgr.34107) 1634 : cluster 0 pgmap v2566: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:14.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:14 smithi181 bash[59345]: cluster 2023-10-09T10:00:12.394488+0000 mgr.y (mgr.34107) 1634 : cluster 0 pgmap v2566: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:16.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:16 smithi163 bash[78107]: cluster 2023-10-09T10:00:14.395362+0000 mgr.y (mgr.34107) 1635 : cluster 0 pgmap v2567: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:16.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:16 smithi163 bash[75425]: cluster 2023-10-09T10:00:14.395362+0000 mgr.y (mgr.34107) 1635 : cluster 0 pgmap v2567: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:16.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:16 smithi181 bash[59345]: cluster 2023-10-09T10:00:14.395362+0000 mgr.y (mgr.34107) 1635 : cluster 0 pgmap v2567: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:18.444 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:18 smithi163 bash[78107]: cluster 2023-10-09T10:00:16.396120+0000 mgr.y (mgr.34107) 1636 : cluster 0 pgmap v2568: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:18.444 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:18 smithi163 bash[75425]: cluster 2023-10-09T10:00:16.396120+0000 mgr.y (mgr.34107) 1636 : cluster 0 pgmap v2568: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:18.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:18 smithi181 bash[59345]: cluster 2023-10-09T10:00:16.396120+0000 mgr.y (mgr.34107) 1636 : cluster 0 pgmap v2568: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:20.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:20 smithi163 bash[78107]: cluster 2023-10-09T10:00:18.396606+0000 mgr.y (mgr.34107) 1637 : cluster 0 pgmap v2569: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:20.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:20 smithi163 bash[75425]: cluster 2023-10-09T10:00:18.396606+0000 mgr.y (mgr.34107) 1637 : cluster 0 pgmap v2569: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:20.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:20 smithi181 bash[59345]: cluster 2023-10-09T10:00:18.396606+0000 mgr.y (mgr.34107) 1637 : cluster 0 pgmap v2569: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:22.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:22 smithi163 bash[75425]: cluster 2023-10-09T10:00:20.397264+0000 mgr.y (mgr.34107) 1638 : cluster 0 pgmap v2570: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:22.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:22 smithi163 bash[78107]: cluster 2023-10-09T10:00:20.397264+0000 mgr.y (mgr.34107) 1638 : cluster 0 pgmap v2570: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:22.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:22 smithi181 bash[59345]: cluster 2023-10-09T10:00:20.397264+0000 mgr.y (mgr.34107) 1638 : cluster 0 pgmap v2570: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:23.610 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:00:23 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:10:00:23] "GET /metrics HTTP/1.1" 200 34034 "" "Prometheus/2.33.4" 2023-10-09T10:00:24.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:24 smithi163 bash[78107]: cluster 2023-10-09T10:00:22.398150+0000 mgr.y (mgr.34107) 1639 : cluster 0 pgmap v2571: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:24.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:24 smithi163 bash[75425]: cluster 2023-10-09T10:00:22.398150+0000 mgr.y (mgr.34107) 1639 : cluster 0 pgmap v2571: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:24.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:24 smithi181 bash[59345]: cluster 2023-10-09T10:00:22.398150+0000 mgr.y (mgr.34107) 1639 : cluster 0 pgmap v2571: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:25.780 INFO:teuthology.orchestra.run.smithi163.stdout:184683594457 2023-10-09T10:00:25.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T10:00:26.347 INFO:teuthology.orchestra.run.smithi163.stdout:158913790688 2023-10-09T10:00:26.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T10:00:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:26 smithi163 bash[75425]: cluster 2023-10-09T10:00:24.398962+0000 mgr.y (mgr.34107) 1640 : cluster 0 pgmap v2572: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:26.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:26 smithi163 bash[75425]: audit 2023-10-09T10:00:25.428770+0000 mon.a (mon.0) 2778 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:26 smithi163 bash[78107]: cluster 2023-10-09T10:00:24.398962+0000 mgr.y (mgr.34107) 1640 : cluster 0 pgmap v2572: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:26.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:26 smithi163 bash[78107]: audit 2023-10-09T10:00:25.428770+0000 mon.a (mon.0) 2778 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:26 smithi181 bash[59345]: cluster 2023-10-09T10:00:24.398962+0000 mgr.y (mgr.34107) 1640 : cluster 0 pgmap v2572: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:26.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:26 smithi181 bash[59345]: audit 2023-10-09T10:00:25.428770+0000 mon.a (mon.0) 2778 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:27.940 INFO:teuthology.orchestra.run.smithi163.stdout:133143986922 2023-10-09T10:00:27.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T10:00:28.403 INFO:teuthology.orchestra.run.smithi163.stdout:4879082848559 2023-10-09T10:00:28.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T10:00:28.411 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:28 smithi163 bash[78107]: cluster 2023-10-09T10:00:26.399431+0000 mgr.y (mgr.34107) 1641 : cluster 0 pgmap v2573: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:28.411 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:28 smithi163 bash[75425]: cluster 2023-10-09T10:00:26.399431+0000 mgr.y (mgr.34107) 1641 : cluster 0 pgmap v2573: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:28.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:28 smithi181 bash[59345]: cluster 2023-10-09T10:00:26.399431+0000 mgr.y (mgr.34107) 1641 : cluster 0 pgmap v2573: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:30.248 INFO:teuthology.orchestra.run.smithi163.stdout:2847563317789 2023-10-09T10:00:30.248 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T10:00:30.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:30 smithi163 bash[78107]: cluster 2023-10-09T10:00:28.400038+0000 mgr.y (mgr.34107) 1642 : cluster 0 pgmap v2574: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:30.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:30 smithi163 bash[75425]: cluster 2023-10-09T10:00:28.400038+0000 mgr.y (mgr.34107) 1642 : cluster 0 pgmap v2574: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:30.421 INFO:teuthology.orchestra.run.smithi163.stdout:210453398225 2023-10-09T10:00:30.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T10:00:30.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:30 smithi181 bash[59345]: cluster 2023-10-09T10:00:28.400038+0000 mgr.y (mgr.34107) 1642 : cluster 0 pgmap v2574: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:31.127 INFO:teuthology.orchestra.run.smithi163.stdout:2723009266211 2023-10-09T10:00:31.127 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T10:00:31.377 INFO:teuthology.orchestra.run.smithi163.stdout:5351529250966 2023-10-09T10:00:31.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T10:00:32.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:32 smithi163 bash[78107]: cluster 2023-10-09T10:00:30.400642+0000 mgr.y (mgr.34107) 1643 : cluster 0 pgmap v2575: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:32.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:32 smithi163 bash[75425]: cluster 2023-10-09T10:00:30.400642+0000 mgr.y (mgr.34107) 1643 : cluster 0 pgmap v2575: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:32.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:32 smithi181 bash[59345]: cluster 2023-10-09T10:00:30.400642+0000 mgr.y (mgr.34107) 1643 : cluster 0 pgmap v2575: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:33.573 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:33.724 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:33.725 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:33.725 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:33.726 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:00:33 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:10:00:33] "GET /metrics HTTP/1.1" 200 34031 "" "Prometheus/2.33.4" 2023-10-09T10:00:34.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:34 smithi163 bash[75425]: cluster 2023-10-09T10:00:32.401563+0000 mgr.y (mgr.34107) 1644 : cluster 0 pgmap v2576: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:34.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:34 smithi163 bash[78107]: cluster 2023-10-09T10:00:32.401563+0000 mgr.y (mgr.34107) 1644 : cluster 0 pgmap v2576: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:34.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:34 smithi181 bash[59345]: cluster 2023-10-09T10:00:32.401563+0000 mgr.y (mgr.34107) 1644 : cluster 0 pgmap v2576: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:34.587 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:34.587 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:35.601 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:35.602 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:36.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:36 smithi163 bash[75425]: cluster 2023-10-09T10:00:34.402358+0000 mgr.y (mgr.34107) 1645 : cluster 0 pgmap v2577: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:36.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:36 smithi163 bash[78107]: cluster 2023-10-09T10:00:34.402358+0000 mgr.y (mgr.34107) 1645 : cluster 0 pgmap v2577: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:36.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:36 smithi181 bash[59345]: cluster 2023-10-09T10:00:34.402358+0000 mgr.y (mgr.34107) 1645 : cluster 0 pgmap v2577: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:38.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:38 smithi163 bash[75425]: cluster 2023-10-09T10:00:36.403211+0000 mgr.y (mgr.34107) 1646 : cluster 0 pgmap v2578: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:38.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:38 smithi163 bash[78107]: cluster 2023-10-09T10:00:36.403211+0000 mgr.y (mgr.34107) 1646 : cluster 0 pgmap v2578: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:38.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:38 smithi181 bash[59345]: cluster 2023-10-09T10:00:36.403211+0000 mgr.y (mgr.34107) 1646 : cluster 0 pgmap v2578: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:40.353 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:40 smithi163 bash[75425]: cluster 2023-10-09T10:00:38.403892+0000 mgr.y (mgr.34107) 1647 : cluster 0 pgmap v2579: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:40.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:40 smithi163 bash[78107]: cluster 2023-10-09T10:00:38.403892+0000 mgr.y (mgr.34107) 1647 : cluster 0 pgmap v2579: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:40.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:40 smithi181 bash[59345]: cluster 2023-10-09T10:00:38.403892+0000 mgr.y (mgr.34107) 1647 : cluster 0 pgmap v2579: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:41.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:41 smithi163 bash[75425]: audit 2023-10-09T10:00:40.429051+0000 mon.a (mon.0) 2779 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:41.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:41 smithi163 bash[78107]: audit 2023-10-09T10:00:40.429051+0000 mon.a (mon.0) 2779 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:41.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:41 smithi181 bash[59345]: audit 2023-10-09T10:00:40.429051+0000 mon.a (mon.0) 2779 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:42.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:42 smithi163 bash[78107]: cluster 2023-10-09T10:00:40.404572+0000 mgr.y (mgr.34107) 1648 : cluster 0 pgmap v2580: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:42.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:42 smithi163 bash[75425]: cluster 2023-10-09T10:00:40.404572+0000 mgr.y (mgr.34107) 1648 : cluster 0 pgmap v2580: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:42.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:42 smithi181 bash[59345]: cluster 2023-10-09T10:00:40.404572+0000 mgr.y (mgr.34107) 1648 : cluster 0 pgmap v2580: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:43.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:43 smithi163 bash[75425]: cluster 2023-10-09T10:00:42.405124+0000 mgr.y (mgr.34107) 1649 : cluster 0 pgmap v2581: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:43.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:43 smithi163 bash[78107]: cluster 2023-10-09T10:00:42.405124+0000 mgr.y (mgr.34107) 1649 : cluster 0 pgmap v2581: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:43.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:00:43 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:10:00:43] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.33.4" 2023-10-09T10:00:43.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:43 smithi181 bash[59345]: cluster 2023-10-09T10:00:42.405124+0000 mgr.y (mgr.34107) 1649 : cluster 0 pgmap v2581: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:45.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:45 smithi163 bash[75425]: cluster 2023-10-09T10:00:44.405725+0000 mgr.y (mgr.34107) 1650 : cluster 0 pgmap v2582: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:45.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:45 smithi163 bash[78107]: cluster 2023-10-09T10:00:44.405725+0000 mgr.y (mgr.34107) 1650 : cluster 0 pgmap v2582: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:45.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:45 smithi181 bash[59345]: cluster 2023-10-09T10:00:44.405725+0000 mgr.y (mgr.34107) 1650 : cluster 0 pgmap v2582: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:47.216 INFO:teuthology.orchestra.run.smithi163.stdout:4879082848563 2023-10-09T10:00:47.511 INFO:teuthology.orchestra.run.smithi163.stdout:133143986927 2023-10-09T10:00:47.535 INFO:teuthology.orchestra.run.smithi163.stdout:158913790693 2023-10-09T10:00:47.971 INFO:teuthology.orchestra.run.smithi163.stdout:184683594461 2023-10-09T10:00:48.045 INFO:teuthology.orchestra.run.smithi163.stdout:2847563317793 2023-10-09T10:00:48.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[75425]: cluster 2023-10-09T10:00:46.406568+0000 mgr.y (mgr.34107) 1651 : cluster 0 pgmap v2583: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:48.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[75425]: audit 2023-10-09T10:00:47.215283+0000 mon.c (mon.2) 160 : audit 0 from='client.? 172.21.15.163:0/4062352644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:00:48.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[75425]: audit 2023-10-09T10:00:47.511198+0000 mon.a (mon.0) 2780 : audit 0 from='client.? 172.21.15.163:0/811331913' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:00:48.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[75425]: audit 2023-10-09T10:00:47.534707+0000 mon.a (mon.0) 2781 : audit 0 from='client.? 172.21.15.163:0/341271494' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:00:48.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[75425]: audit 2023-10-09T10:00:47.970490+0000 mon.c (mon.2) 161 : audit 0 from='client.? 172.21.15.163:0/2466905699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:00:48.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[75425]: audit 2023-10-09T10:00:48.044190+0000 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.163:0/2184733734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:00:48.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[78107]: cluster 2023-10-09T10:00:46.406568+0000 mgr.y (mgr.34107) 1651 : cluster 0 pgmap v2583: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:48.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[78107]: audit 2023-10-09T10:00:47.215283+0000 mon.c (mon.2) 160 : audit 0 from='client.? 172.21.15.163:0/4062352644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:00:48.380 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[78107]: audit 2023-10-09T10:00:47.511198+0000 mon.a (mon.0) 2780 : audit 0 from='client.? 172.21.15.163:0/811331913' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:00:48.380 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[78107]: audit 2023-10-09T10:00:47.534707+0000 mon.a (mon.0) 2781 : audit 0 from='client.? 172.21.15.163:0/341271494' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:00:48.380 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[78107]: audit 2023-10-09T10:00:47.970490+0000 mon.c (mon.2) 161 : audit 0 from='client.? 172.21.15.163:0/2466905699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:00:48.380 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:48 smithi163 bash[78107]: audit 2023-10-09T10:00:48.044190+0000 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.163:0/2184733734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:00:48.383 INFO:teuthology.orchestra.run.smithi163.stdout:210453398229 2023-10-09T10:00:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:48 smithi181 bash[59345]: cluster 2023-10-09T10:00:46.406568+0000 mgr.y (mgr.34107) 1651 : cluster 0 pgmap v2583: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:48 smithi181 bash[59345]: audit 2023-10-09T10:00:47.215283+0000 mon.c (mon.2) 160 : audit 0 from='client.? 172.21.15.163:0/4062352644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:00:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:48 smithi181 bash[59345]: audit 2023-10-09T10:00:47.511198+0000 mon.a (mon.0) 2780 : audit 0 from='client.? 172.21.15.163:0/811331913' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:00:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:48 smithi181 bash[59345]: audit 2023-10-09T10:00:47.534707+0000 mon.a (mon.0) 2781 : audit 0 from='client.? 172.21.15.163:0/341271494' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:00:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:48 smithi181 bash[59345]: audit 2023-10-09T10:00:47.970490+0000 mon.c (mon.2) 161 : audit 0 from='client.? 172.21.15.163:0/2466905699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:00:48.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:48 smithi181 bash[59345]: audit 2023-10-09T10:00:48.044190+0000 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.163:0/2184733734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:00:48.757 INFO:teuthology.orchestra.run.smithi163.stdout:5351529250969 2023-10-09T10:00:49.409 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:49 smithi163 bash[78107]: audit 2023-10-09T10:00:48.383222+0000 mon.a (mon.0) 2782 : audit 0 from='client.? 172.21.15.163:0/2964584603' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:00:49.409 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:49 smithi163 bash[78107]: audit 2023-10-09T10:00:48.756445+0000 mon.a (mon.0) 2783 : audit 0 from='client.? 172.21.15.163:0/4204210400' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:00:49.410 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:49 smithi163 bash[75425]: audit 2023-10-09T10:00:48.383222+0000 mon.a (mon.0) 2782 : audit 0 from='client.? 172.21.15.163:0/2964584603' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:00:49.410 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:49 smithi163 bash[75425]: audit 2023-10-09T10:00:48.756445+0000 mon.a (mon.0) 2783 : audit 0 from='client.? 172.21.15.163:0/4204210400' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:00:49.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:49 smithi181 bash[59345]: audit 2023-10-09T10:00:48.383222+0000 mon.a (mon.0) 2782 : audit 0 from='client.? 172.21.15.163:0/2964584603' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:00:49.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:49 smithi181 bash[59345]: audit 2023-10-09T10:00:48.756445+0000 mon.a (mon.0) 2783 : audit 0 from='client.? 172.21.15.163:0/4204210400' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:00:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:50 smithi163 bash[78107]: cluster 2023-10-09T10:00:48.407087+0000 mgr.y (mgr.34107) 1652 : cluster 0 pgmap v2584: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:50.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:50 smithi163 bash[78107]: audit 2023-10-09T10:00:49.344557+0000 mon.a (mon.0) 2784 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:00:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:50 smithi163 bash[75425]: cluster 2023-10-09T10:00:48.407087+0000 mgr.y (mgr.34107) 1652 : cluster 0 pgmap v2584: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:50.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:50 smithi163 bash[75425]: audit 2023-10-09T10:00:49.344557+0000 mon.a (mon.0) 2784 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:00:50.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:50 smithi181 bash[59345]: cluster 2023-10-09T10:00:48.407087+0000 mgr.y (mgr.34107) 1652 : cluster 0 pgmap v2584: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:50.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:50 smithi181 bash[59345]: audit 2023-10-09T10:00:49.344557+0000 mon.a (mon.0) 2784 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:00:50.785 INFO:tasks.cephadm.ceph_manager.ceph:need seq 4879082848559 got 4879082848563 for osd.0 2023-10-09T10:00:50.785 DEBUG:teuthology.parallel:result is None 2023-10-09T10:00:51.371 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683594457 got 184683594461 for osd.6 2023-10-09T10:00:51.371 DEBUG:teuthology.parallel:result is None 2023-10-09T10:00:51.687 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986922 got 133143986927 for osd.4 2023-10-09T10:00:51.687 DEBUG:teuthology.parallel:result is None 2023-10-09T10:00:52.186 INFO:teuthology.orchestra.run.smithi163.stdout:2723009266216 2023-10-09T10:00:52.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:52 smithi163 bash[75425]: cluster 2023-10-09T10:00:50.407576+0000 mgr.y (mgr.34107) 1653 : cluster 0 pgmap v2585: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:52.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:52 smithi163 bash[78107]: cluster 2023-10-09T10:00:50.407576+0000 mgr.y (mgr.34107) 1653 : cluster 0 pgmap v2585: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:52.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:52 smithi181 bash[59345]: cluster 2023-10-09T10:00:50.407576+0000 mgr.y (mgr.34107) 1653 : cluster 0 pgmap v2585: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:52.684 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2847563317789 got 2847563317793 for osd.3 2023-10-09T10:00:52.685 DEBUG:teuthology.parallel:result is None 2023-10-09T10:00:53.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:00:53 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:10:00:53] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.33.4" 2023-10-09T10:00:53.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:53 smithi163 bash[75425]: audit 2023-10-09T10:00:52.185540+0000 mon.c (mon.2) 163 : audit 0 from='client.? 172.21.15.163:0/4141114237' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:00:53.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:53 smithi163 bash[78107]: audit 2023-10-09T10:00:52.185540+0000 mon.c (mon.2) 163 : audit 0 from='client.? 172.21.15.163:0/4141114237' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:00:53.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:53 smithi181 bash[59345]: audit 2023-10-09T10:00:52.185540+0000 mon.c (mon.2) 163 : audit 0 from='client.? 172.21.15.163:0/4141114237' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:00:53.706 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790688 got 158913790693 for osd.5 2023-10-09T10:00:53.706 DEBUG:teuthology.parallel:result is None 2023-10-09T10:00:54.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:54 smithi163 bash[78107]: cluster 2023-10-09T10:00:52.408126+0000 mgr.y (mgr.34107) 1654 : cluster 0 pgmap v2586: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:54.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:54 smithi163 bash[75425]: cluster 2023-10-09T10:00:52.408126+0000 mgr.y (mgr.34107) 1654 : cluster 0 pgmap v2586: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:54.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:54 smithi181 bash[59345]: cluster 2023-10-09T10:00:52.408126+0000 mgr.y (mgr.34107) 1654 : cluster 0 pgmap v2586: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:54.670 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453398225 got 210453398229 for osd.7 2023-10-09T10:00:54.670 DEBUG:teuthology.parallel:result is None 2023-10-09T10:00:54.678 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5351529250966 got 5351529250969 for osd.1 2023-10-09T10:00:54.678 DEBUG:teuthology.parallel:result is None 2023-10-09T10:00:54.727 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2723009266211 got 2723009266216 for osd.2 2023-10-09T10:00:54.727 DEBUG:teuthology.parallel:result is None 2023-10-09T10:00:54.727 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T10:00:54.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:00:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:56 smithi163 bash[78107]: cluster 2023-10-09T10:00:54.408636+0000 mgr.y (mgr.34107) 1655 : cluster 0 pgmap v2587: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:56 smithi163 bash[78107]: audit 2023-10-09T10:00:55.430235+0000 mon.a (mon.0) 2785 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:56 smithi163 bash[78107]: audit 2023-10-09T10:00:55.589418+0000 mon.a (mon.0) 2786 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:56.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:56 smithi163 bash[78107]: audit 2023-10-09T10:00:55.601945+0000 mon.a (mon.0) 2787 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:56.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:56 smithi163 bash[75425]: cluster 2023-10-09T10:00:54.408636+0000 mgr.y (mgr.34107) 1655 : cluster 0 pgmap v2587: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:56.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:56 smithi163 bash[75425]: audit 2023-10-09T10:00:55.430235+0000 mon.a (mon.0) 2785 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:56.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:56 smithi163 bash[75425]: audit 2023-10-09T10:00:55.589418+0000 mon.a (mon.0) 2786 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:56.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:56 smithi163 bash[75425]: audit 2023-10-09T10:00:55.601945+0000 mon.a (mon.0) 2787 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:56 smithi181 bash[59345]: cluster 2023-10-09T10:00:54.408636+0000 mgr.y (mgr.34107) 1655 : cluster 0 pgmap v2587: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:56 smithi181 bash[59345]: audit 2023-10-09T10:00:55.430235+0000 mon.a (mon.0) 2785 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:56 smithi181 bash[59345]: audit 2023-10-09T10:00:55.589418+0000 mon.a (mon.0) 2786 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:56.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:56 smithi181 bash[59345]: audit 2023-10-09T10:00:55.601945+0000 mon.a (mon.0) 2787 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:58.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:58 smithi163 bash[75425]: cluster 2023-10-09T10:00:56.409463+0000 mgr.y (mgr.34107) 1656 : cluster 0 pgmap v2588: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:58.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:58 smithi163 bash[78107]: cluster 2023-10-09T10:00:56.409463+0000 mgr.y (mgr.34107) 1656 : cluster 0 pgmap v2588: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:58.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:58 smithi181 bash[59345]: cluster 2023-10-09T10:00:56.409463+0000 mgr.y (mgr.34107) 1656 : cluster 0 pgmap v2588: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:59.507 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[75425]: cluster 2023-10-09T10:00:58.410163+0000 mgr.y (mgr.34107) 1657 : cluster 0 pgmap v2589: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[75425]: audit 2023-10-09T10:00:58.692379+0000 mon.a (mon.0) 2788 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[75425]: audit 2023-10-09T10:00:58.699651+0000 mon.a (mon.0) 2789 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[75425]: audit 2023-10-09T10:00:59.219045+0000 mon.a (mon.0) 2790 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:00:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[75425]: audit 2023-10-09T10:00:59.220911+0000 mon.a (mon.0) 2791 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:00:59.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[75425]: audit 2023-10-09T10:00:59.229484+0000 mon.a (mon.0) 2792 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:59.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[78107]: cluster 2023-10-09T10:00:58.410163+0000 mgr.y (mgr.34107) 1657 : cluster 0 pgmap v2589: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:00:59.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[78107]: audit 2023-10-09T10:00:58.692379+0000 mon.a (mon.0) 2788 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:59.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[78107]: audit 2023-10-09T10:00:58.699651+0000 mon.a (mon.0) 2789 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:00:59.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[78107]: audit 2023-10-09T10:00:59.219045+0000 mon.a (mon.0) 2790 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:00:59.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[78107]: audit 2023-10-09T10:00:59.220911+0000 mon.a (mon.0) 2791 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:00:59.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:00:59 smithi163 bash[78107]: audit 2023-10-09T10:00:59.229484+0000 mon.a (mon.0) 2792 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:01:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:59 smithi181 bash[59345]: cluster 2023-10-09T10:00:58.410163+0000 mgr.y (mgr.34107) 1657 : cluster 0 pgmap v2589: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:59 smithi181 bash[59345]: audit 2023-10-09T10:00:58.692379+0000 mon.a (mon.0) 2788 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:01:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:59 smithi181 bash[59345]: audit 2023-10-09T10:00:58.699651+0000 mon.a (mon.0) 2789 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:01:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:59 smithi181 bash[59345]: audit 2023-10-09T10:00:59.219045+0000 mon.a (mon.0) 2790 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:01:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:59 smithi181 bash[59345]: audit 2023-10-09T10:00:59.220911+0000 mon.a (mon.0) 2791 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:01:00.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:00:59 smithi181 bash[59345]: audit 2023-10-09T10:00:59.229484+0000 mon.a (mon.0) 2792 : audit 1 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' 2023-10-09T10:01:02.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:02 smithi163 bash[75425]: cluster 2023-10-09T10:01:00.411061+0000 mgr.y (mgr.34107) 1658 : cluster 0 pgmap v2590: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:02.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:02 smithi163 bash[78107]: cluster 2023-10-09T10:01:00.411061+0000 mgr.y (mgr.34107) 1658 : cluster 0 pgmap v2590: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:02.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:02 smithi181 bash[59345]: cluster 2023-10-09T10:01:00.411061+0000 mgr.y (mgr.34107) 1658 : cluster 0 pgmap v2590: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:03.108 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T10:01:03.110 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T10:01:03.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:03 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:10:01:03] "GET /metrics HTTP/1.1" 200 34033 "" "Prometheus/2.33.4" 2023-10-09T10:01:04.022 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2591,"stamp":"2023-10-09T10:01:02.411309+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6277,"num_read_kb":12713,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":5382450,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5704208,"kb_used_data":68496,"kb_used_omap":662,"kb_used_meta":5634985,"kb_avail":744257008,"statfs":{"total":767960285184,"available":762119176192,"internally_reserved":0,"allocated":70139904,"data_stored":62057704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":678391,"internal_metadata":5770225161},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004037"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1221,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935386+0000","last_change":"2023-10-09T09:58:07.126507+0000","last_active":"2023-10-09T09:59:36.935386+0000","last_peered":"2023-10-09T09:59:36.935386+0000","last_clean":"2023-10-09T09:59:36.935386+0000","last_became_active":"2023-10-09T09:58:07.125959+0000","last_became_peered":"2023-10-09T09:58:07.125959+0000","last_unstale":"2023-10-09T09:59:36.935386+0000","last_undegraded":"2023-10-09T09:59:36.935386+0000","last_fullsized":"2023-10-09T09:59:36.935386+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":226,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285221+0000","last_change":"2023-10-09T09:57:47.472431+0000","last_active":"2023-10-09T09:59:47.285221+0000","last_peered":"2023-10-09T09:59:47.285221+0000","last_clean":"2023-10-09T09:59:47.285221+0000","last_became_active":"2023-10-09T09:57:47.472035+0000","last_became_peered":"2023-10-09T09:57:47.472035+0000","last_unstale":"2023-10-09T09:59:47.285221+0000","last_undegraded":"2023-10-09T09:59:47.285221+0000","last_fullsized":"2023-10-09T09:59:47.285221+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:36.597330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1306,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.023676+0000","last_change":"2023-10-09T09:56:40.141288+0000","last_active":"2023-10-09T09:59:48.023676+0000","last_peered":"2023-10-09T09:59:48.023676+0000","last_clean":"2023-10-09T09:59:48.023676+0000","last_became_active":"2023-10-09T09:56:40.140682+0000","last_became_peered":"2023-10-09T09:56:40.140682+0000","last_unstale":"2023-10-09T09:59:48.023676+0000","last_undegraded":"2023-10-09T09:59:48.023676+0000","last_fullsized":"2023-10-09T09:59:48.023676+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:19.241067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1326,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.592234+0000","last_change":"2023-10-09T09:56:57.078210+0000","last_active":"2023-10-09T09:59:36.592234+0000","last_peered":"2023-10-09T09:59:36.592234+0000","last_clean":"2023-10-09T09:59:36.592234+0000","last_became_active":"2023-10-09T09:56:57.077775+0000","last_became_peered":"2023-10-09T09:56:57.077775+0000","last_unstale":"2023-10-09T09:59:36.592234+0000","last_undegraded":"2023-10-09T09:59:36.592234+0000","last_fullsized":"2023-10-09T09:59:36.592234+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:27.531666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":839,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.284899+0000","last_change":"2023-10-09T09:57:47.472986+0000","last_active":"2023-10-09T09:59:47.284899+0000","last_peered":"2023-10-09T09:59:47.284899+0000","last_clean":"2023-10-09T09:59:47.284899+0000","last_became_active":"2023-10-09T09:57:47.472634+0000","last_became_peered":"2023-10-09T09:57:47.472634+0000","last_unstale":"2023-10-09T09:59:47.284899+0000","last_undegraded":"2023-10-09T09:59:47.284899+0000","last_fullsized":"2023-10-09T09:59:47.284899+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:33.049868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1307,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934945+0000","last_change":"2023-10-09T09:48:07.988928+0000","last_active":"2023-10-09T09:59:36.934945+0000","last_peered":"2023-10-09T09:59:36.934945+0000","last_clean":"2023-10-09T09:59:36.934945+0000","last_became_active":"2023-10-09T09:48:07.988051+0000","last_became_peered":"2023-10-09T09:48:07.988051+0000","last_unstale":"2023-10-09T09:59:36.934945+0000","last_undegraded":"2023-10-09T09:59:36.934945+0000","last_fullsized":"2023-10-09T09:59:36.934945+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1253,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.592251+0000","last_change":"2023-10-09T09:59:27.198059+0000","last_active":"2023-10-09T09:59:36.592251+0000","last_peered":"2023-10-09T09:59:36.592251+0000","last_clean":"2023-10-09T09:59:36.592251+0000","last_became_active":"2023-10-09T09:59:27.197563+0000","last_became_peered":"2023-10-09T09:59:27.197563+0000","last_unstale":"2023-10-09T09:59:36.592251+0000","last_undegraded":"2023-10-09T09:59:36.592251+0000","last_fullsized":"2023-10-09T09:59:36.592251+0000","mapping_epoch":1341,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:47.921652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":40,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.181678+0000","last_change":"2023-10-09T09:56:40.139613+0000","last_active":"2023-10-09T09:59:36.181678+0000","last_peered":"2023-10-09T09:59:36.181678+0000","last_clean":"2023-10-09T09:59:36.181678+0000","last_became_active":"2023-10-09T09:56:40.139072+0000","last_became_peered":"2023-10-09T09:56:40.139072+0000","last_unstale":"2023-10-09T09:59:36.181678+0000","last_undegraded":"2023-10-09T09:59:36.181678+0000","last_fullsized":"2023-10-09T09:59:36.181678+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:31.377762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":228,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102536+0000","last_change":"2023-10-09T09:56:57.078220+0000","last_active":"2023-10-09T09:59:36.102536+0000","last_peered":"2023-10-09T09:59:36.102536+0000","last_clean":"2023-10-09T09:59:36.102536+0000","last_became_active":"2023-10-09T09:56:57.077758+0000","last_became_peered":"2023-10-09T09:56:57.077758+0000","last_unstale":"2023-10-09T09:59:36.102536+0000","last_undegraded":"2023-10-09T09:59:36.102536+0000","last_fullsized":"2023-10-09T09:59:36.102536+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":758,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285483+0000","last_change":"2023-10-09T09:57:47.431308+0000","last_active":"2023-10-09T09:59:47.285483+0000","last_peered":"2023-10-09T09:59:47.285483+0000","last_clean":"2023-10-09T09:59:47.285483+0000","last_became_active":"2023-10-09T09:57:47.430812+0000","last_became_peered":"2023-10-09T09:57:47.430812+0000","last_unstale":"2023-10-09T09:59:47.285483+0000","last_undegraded":"2023-10-09T09:59:47.285483+0000","last_fullsized":"2023-10-09T09:59:47.285483+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:05.105709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":39,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932454+0000","last_change":"2023-10-09T09:56:57.282883+0000","last_active":"2023-10-09T09:59:36.932454+0000","last_peered":"2023-10-09T09:59:36.932454+0000","last_clean":"2023-10-09T09:59:36.932454+0000","last_became_active":"2023-10-09T09:56:57.086116+0000","last_became_peered":"2023-10-09T09:56:57.086116+0000","last_unstale":"2023-10-09T09:59:36.932454+0000","last_undegraded":"2023-10-09T09:59:36.932454+0000","last_fullsized":"2023-10-09T09:59:36.932454+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":176,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102642+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:59:36.102642+0000","last_peered":"2023-10-09T09:59:36.102642+0000","last_clean":"2023-10-09T09:59:36.102642+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:59:36.102642+0000","last_undegraded":"2023-10-09T09:59:36.102642+0000","last_fullsized":"2023-10-09T09:59:36.102642+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1213,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935509+0000","last_change":"2023-10-09T09:58:07.131535+0000","last_active":"2023-10-09T09:59:36.935509+0000","last_peered":"2023-10-09T09:59:36.935509+0000","last_clean":"2023-10-09T09:59:36.935509+0000","last_became_active":"2023-10-09T09:58:07.131120+0000","last_became_peered":"2023-10-09T09:58:07.131120+0000","last_unstale":"2023-10-09T09:59:36.935509+0000","last_undegraded":"2023-10-09T09:59:36.935509+0000","last_fullsized":"2023-10-09T09:59:36.935509+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1186,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935443+0000","last_change":"2023-10-09T09:58:07.130871+0000","last_active":"2023-10-09T09:59:36.935443+0000","last_peered":"2023-10-09T09:59:36.935443+0000","last_clean":"2023-10-09T09:59:36.935443+0000","last_became_active":"2023-10-09T09:58:07.130302+0000","last_became_peered":"2023-10-09T09:58:07.130302+0000","last_unstale":"2023-10-09T09:59:36.935443+0000","last_undegraded":"2023-10-09T09:59:36.935443+0000","last_fullsized":"2023-10-09T09:59:36.935443+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":39,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932100+0000","last_change":"2023-10-09T09:56:57.271889+0000","last_active":"2023-10-09T09:59:36.932100+0000","last_peered":"2023-10-09T09:59:36.932100+0000","last_clean":"2023-10-09T09:59:36.932100+0000","last_became_active":"2023-10-09T09:56:57.085853+0000","last_became_peered":"2023-10-09T09:56:57.085853+0000","last_unstale":"2023-10-09T09:59:36.932100+0000","last_undegraded":"2023-10-09T09:59:36.932100+0000","last_fullsized":"2023-10-09T09:59:36.932100+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":836,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285276+0000","last_change":"2023-10-09T09:57:47.473231+0000","last_active":"2023-10-09T09:59:47.285276+0000","last_peered":"2023-10-09T09:59:47.285276+0000","last_clean":"2023-10-09T09:59:47.285276+0000","last_became_active":"2023-10-09T09:57:47.472433+0000","last_became_peered":"2023-10-09T09:57:47.472433+0000","last_unstale":"2023-10-09T09:59:47.285276+0000","last_undegraded":"2023-10-09T09:59:47.285276+0000","last_fullsized":"2023-10-09T09:59:47.285276+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:42.280204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1229,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.592004+0000","last_change":"2023-10-09T09:59:27.201093+0000","last_active":"2023-10-09T09:59:36.592004+0000","last_peered":"2023-10-09T09:59:36.592004+0000","last_clean":"2023-10-09T09:59:36.592004+0000","last_became_active":"2023-10-09T09:59:27.200780+0000","last_became_peered":"2023-10-09T09:59:27.200780+0000","last_unstale":"2023-10-09T09:59:36.592004+0000","last_undegraded":"2023-10-09T09:59:36.592004+0000","last_fullsized":"2023-10-09T09:59:36.592004+0000","mapping_epoch":1341,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:03:31.399625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":666,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102642+0000","last_change":"2023-10-09T09:58:27.454413+0000","last_active":"2023-10-09T09:59:36.102642+0000","last_peered":"2023-10-09T09:59:36.102642+0000","last_clean":"2023-10-09T09:59:36.102642+0000","last_became_active":"2023-10-09T09:58:27.453999+0000","last_became_peered":"2023-10-09T09:58:27.453999+0000","last_unstale":"2023-10-09T09:59:36.102642+0000","last_undegraded":"2023-10-09T09:59:36.102642+0000","last_fullsized":"2023-10-09T09:59:36.102642+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":33,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.931710+0000","last_change":"2023-10-09T09:56:57.084062+0000","last_active":"2023-10-09T09:59:36.931710+0000","last_peered":"2023-10-09T09:59:36.931710+0000","last_clean":"2023-10-09T09:59:36.931710+0000","last_became_active":"2023-10-09T09:56:57.083675+0000","last_became_peered":"2023-10-09T09:56:57.083675+0000","last_unstale":"2023-10-09T09:59:36.931710+0000","last_undegraded":"2023-10-09T09:59:36.931710+0000","last_fullsized":"2023-10-09T09:59:36.931710+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1249,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.592149+0000","last_change":"2023-10-09T09:59:27.201410+0000","last_active":"2023-10-09T09:59:36.592149+0000","last_peered":"2023-10-09T09:59:36.592149+0000","last_clean":"2023-10-09T09:59:36.592149+0000","last_became_active":"2023-10-09T09:59:27.201029+0000","last_became_peered":"2023-10-09T09:59:27.201029+0000","last_unstale":"2023-10-09T09:59:36.592149+0000","last_undegraded":"2023-10-09T09:59:36.592149+0000","last_fullsized":"2023-10-09T09:59:36.592149+0000","mapping_epoch":1341,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:31:50.223832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1337,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934302+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:59:36.934302+0000","last_peered":"2023-10-09T09:59:36.934302+0000","last_clean":"2023-10-09T09:59:36.934302+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:59:36.934302+0000","last_undegraded":"2023-10-09T09:59:36.934302+0000","last_fullsized":"2023-10-09T09:59:36.934302+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":49,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.182799+0000","last_change":"2023-10-09T09:56:57.087457+0000","last_active":"2023-10-09T09:59:36.182799+0000","last_peered":"2023-10-09T09:59:36.182799+0000","last_clean":"2023-10-09T09:59:36.182799+0000","last_became_active":"2023-10-09T09:56:57.087090+0000","last_became_peered":"2023-10-09T09:56:57.087090+0000","last_unstale":"2023-10-09T09:59:36.182799+0000","last_undegraded":"2023-10-09T09:59:36.182799+0000","last_fullsized":"2023-10-09T09:59:36.182799+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:12:57.268609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":37,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932864+0000","last_change":"2023-10-09T09:56:57.278364+0000","last_active":"2023-10-09T09:59:36.932864+0000","last_peered":"2023-10-09T09:59:36.932864+0000","last_clean":"2023-10-09T09:59:36.932864+0000","last_became_active":"2023-10-09T09:56:57.085732+0000","last_became_peered":"2023-10-09T09:56:57.085732+0000","last_unstale":"2023-10-09T09:59:36.932864+0000","last_undegraded":"2023-10-09T09:59:36.932864+0000","last_fullsized":"2023-10-09T09:59:36.932864+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":195,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102323+0000","last_change":"2023-10-09T09:56:57.078240+0000","last_active":"2023-10-09T09:59:36.102323+0000","last_peered":"2023-10-09T09:59:36.102323+0000","last_clean":"2023-10-09T09:59:36.102323+0000","last_became_active":"2023-10-09T09:56:57.077813+0000","last_became_peered":"2023-10-09T09:56:57.077813+0000","last_unstale":"2023-10-09T09:59:36.102323+0000","last_undegraded":"2023-10-09T09:59:36.102323+0000","last_fullsized":"2023-10-09T09:59:36.102323+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":37,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932515+0000","last_change":"2023-10-09T09:56:57.285622+0000","last_active":"2023-10-09T09:59:36.932515+0000","last_peered":"2023-10-09T09:59:36.932515+0000","last_clean":"2023-10-09T09:59:36.932515+0000","last_became_active":"2023-10-09T09:56:57.087847+0000","last_became_peered":"2023-10-09T09:56:57.087847+0000","last_unstale":"2023-10-09T09:59:36.932515+0000","last_undegraded":"2023-10-09T09:59:36.932515+0000","last_fullsized":"2023-10-09T09:59:36.932515+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1274,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935802+0000","last_change":"2023-10-09T09:58:07.778049+0000","last_active":"2023-10-09T09:59:36.935802+0000","last_peered":"2023-10-09T09:59:36.935802+0000","last_clean":"2023-10-09T09:59:36.935802+0000","last_became_active":"2023-10-09T09:58:07.777129+0000","last_became_peered":"2023-10-09T09:58:07.777129+0000","last_unstale":"2023-10-09T09:59:36.935802+0000","last_undegraded":"2023-10-09T09:59:36.935802+0000","last_fullsized":"2023-10-09T09:59:36.935802+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":37,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932837+0000","last_change":"2023-10-09T09:56:57.092620+0000","last_active":"2023-10-09T09:59:36.932837+0000","last_peered":"2023-10-09T09:59:36.932837+0000","last_clean":"2023-10-09T09:59:36.932837+0000","last_became_active":"2023-10-09T09:56:57.083126+0000","last_became_peered":"2023-10-09T09:56:57.083126+0000","last_unstale":"2023-10-09T09:59:36.932837+0000","last_undegraded":"2023-10-09T09:59:36.932837+0000","last_fullsized":"2023-10-09T09:59:36.932837+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1308,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934518+0000","last_change":"2023-10-09T09:56:57.081279+0000","last_active":"2023-10-09T09:59:36.934518+0000","last_peered":"2023-10-09T09:59:36.934518+0000","last_clean":"2023-10-09T09:59:36.934518+0000","last_became_active":"2023-10-09T09:56:57.080887+0000","last_became_peered":"2023-10-09T09:56:57.080887+0000","last_unstale":"2023-10-09T09:59:36.934518+0000","last_undegraded":"2023-10-09T09:59:36.934518+0000","last_fullsized":"2023-10-09T09:59:36.934518+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":840,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102596+0000","last_change":"2023-10-09T09:58:27.455524+0000","last_active":"2023-10-09T09:59:36.102596+0000","last_peered":"2023-10-09T09:59:36.102596+0000","last_clean":"2023-10-09T09:59:36.102596+0000","last_became_active":"2023-10-09T09:58:27.455105+0000","last_became_peered":"2023-10-09T09:58:27.455105+0000","last_unstale":"2023-10-09T09:59:36.102596+0000","last_undegraded":"2023-10-09T09:59:36.102596+0000","last_fullsized":"2023-10-09T09:59:36.102596+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1267,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.571749+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:59:47.571749+0000","last_peered":"2023-10-09T09:59:47.571749+0000","last_clean":"2023-10-09T09:59:47.571749+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:59:47.571749+0000","last_undegraded":"2023-10-09T09:59:47.571749+0000","last_fullsized":"2023-10-09T09:59:47.571749+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":44,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.182194+0000","last_change":"2023-10-09T09:56:40.145464+0000","last_active":"2023-10-09T09:59:36.182194+0000","last_peered":"2023-10-09T09:59:36.182194+0000","last_clean":"2023-10-09T09:59:36.182194+0000","last_became_active":"2023-10-09T09:56:40.128235+0000","last_became_peered":"2023-10-09T09:56:40.128235+0000","last_unstale":"2023-10-09T09:59:36.182194+0000","last_undegraded":"2023-10-09T09:59:36.182194+0000","last_fullsized":"2023-10-09T09:59:36.182194+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:03.254704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1293,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935532+0000","last_change":"2023-10-09T09:58:07.778084+0000","last_active":"2023-10-09T09:59:36.935532+0000","last_peered":"2023-10-09T09:59:36.935532+0000","last_clean":"2023-10-09T09:59:36.935532+0000","last_became_active":"2023-10-09T09:58:07.777094+0000","last_became_peered":"2023-10-09T09:58:07.777094+0000","last_unstale":"2023-10-09T09:59:36.935532+0000","last_undegraded":"2023-10-09T09:59:36.935532+0000","last_fullsized":"2023-10-09T09:59:36.935532+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1291,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934164+0000","last_change":"2023-10-09T09:48:07.988586+0000","last_active":"2023-10-09T09:59:36.934164+0000","last_peered":"2023-10-09T09:59:36.934164+0000","last_clean":"2023-10-09T09:59:36.934164+0000","last_became_active":"2023-10-09T09:48:07.988226+0000","last_became_peered":"2023-10-09T09:48:07.988226+0000","last_unstale":"2023-10-09T09:59:36.934164+0000","last_undegraded":"2023-10-09T09:59:36.934164+0000","last_fullsized":"2023-10-09T09:59:36.934164+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1219,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935623+0000","last_change":"2023-10-09T09:58:07.126784+0000","last_active":"2023-10-09T09:59:36.935623+0000","last_peered":"2023-10-09T09:59:36.935623+0000","last_clean":"2023-10-09T09:59:36.935623+0000","last_became_active":"2023-10-09T09:58:07.126143+0000","last_became_peered":"2023-10-09T09:58:07.126143+0000","last_unstale":"2023-10-09T09:59:36.935623+0000","last_undegraded":"2023-10-09T09:59:36.935623+0000","last_fullsized":"2023-10-09T09:59:36.935623+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1314,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935228+0000","last_change":"2023-10-09T09:58:07.126735+0000","last_active":"2023-10-09T09:59:36.935228+0000","last_peered":"2023-10-09T09:59:36.935228+0000","last_clean":"2023-10-09T09:59:36.935228+0000","last_became_active":"2023-10-09T09:58:07.126401+0000","last_became_peered":"2023-10-09T09:58:07.126401+0000","last_unstale":"2023-10-09T09:59:36.935228+0000","last_undegraded":"2023-10-09T09:59:36.935228+0000","last_fullsized":"2023-10-09T09:59:36.935228+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1186,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.936139+0000","last_change":"2023-10-09T09:58:07.130343+0000","last_active":"2023-10-09T09:59:36.936139+0000","last_peered":"2023-10-09T09:59:36.936139+0000","last_clean":"2023-10-09T09:59:36.936139+0000","last_became_active":"2023-10-09T09:58:07.129931+0000","last_became_peered":"2023-10-09T09:58:07.129931+0000","last_unstale":"2023-10-09T09:59:36.936139+0000","last_undegraded":"2023-10-09T09:59:36.936139+0000","last_fullsized":"2023-10-09T09:59:36.936139+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":50,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.284922+0000","last_change":"2023-10-09T09:56:57.087804+0000","last_active":"2023-10-09T09:59:47.284922+0000","last_peered":"2023-10-09T09:59:47.284922+0000","last_clean":"2023-10-09T09:59:47.284922+0000","last_became_active":"2023-10-09T09:56:57.087423+0000","last_became_peered":"2023-10-09T09:56:57.087423+0000","last_unstale":"2023-10-09T09:59:47.284922+0000","last_undegraded":"2023-10-09T09:59:47.284922+0000","last_fullsized":"2023-10-09T09:59:47.284922+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:17.120194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":202,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.166514+0000","last_change":"2023-10-09T09:56:40.127642+0000","last_active":"2023-10-09T09:59:47.166514+0000","last_peered":"2023-10-09T09:59:47.166514+0000","last_clean":"2023-10-09T09:59:47.166514+0000","last_became_active":"2023-10-09T09:56:40.127292+0000","last_became_peered":"2023-10-09T09:56:40.127292+0000","last_unstale":"2023-10-09T09:59:47.166514+0000","last_undegraded":"2023-10-09T09:59:47.166514+0000","last_fullsized":"2023-10-09T09:59:47.166514+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:44.501837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"1311'169","reported_seq":1943,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.024707+0000","last_change":"2023-10-09T09:57:00.111698+0000","last_active":"2023-10-09T09:59:48.024707+0000","last_peered":"2023-10-09T09:59:48.024707+0000","last_clean":"2023-10-09T09:59:48.024707+0000","last_became_active":"2023-10-09T09:57:00.110919+0000","last_became_peered":"2023-10-09T09:57:00.110919+0000","last_unstale":"2023-10-09T09:59:48.024707+0000","last_undegraded":"2023-10-09T09:59:48.024707+0000","last_fullsized":"2023-10-09T09:59:48.024707+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:55.189795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001668896,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1002,"num_read_kb":2031,"num_write":171,"num_write_kb":5258,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":195,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102474+0000","last_change":"2023-10-09T09:56:40.128833+0000","last_active":"2023-10-09T09:59:36.102474+0000","last_peered":"2023-10-09T09:59:36.102474+0000","last_clean":"2023-10-09T09:59:36.102474+0000","last_became_active":"2023-10-09T09:56:40.128373+0000","last_became_peered":"2023-10-09T09:56:40.128373+0000","last_unstale":"2023-10-09T09:59:36.102474+0000","last_undegraded":"2023-10-09T09:59:36.102474+0000","last_fullsized":"2023-10-09T09:59:36.102474+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1280,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.023632+0000","last_change":"2023-10-09T09:48:07.977683+0000","last_active":"2023-10-09T09:59:48.023632+0000","last_peered":"2023-10-09T09:59:48.023632+0000","last_clean":"2023-10-09T09:59:48.023632+0000","last_became_active":"2023-10-09T09:48:07.977024+0000","last_became_peered":"2023-10-09T09:48:07.977024+0000","last_unstale":"2023-10-09T09:59:48.023632+0000","last_undegraded":"2023-10-09T09:59:48.023632+0000","last_fullsized":"2023-10-09T09:59:48.023632+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:13.438431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"1311'247","reported_seq":1811,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102793+0000","last_change":"2023-10-09T09:58:27.455629+0000","last_active":"2023-10-09T09:59:36.102793+0000","last_peered":"2023-10-09T09:59:36.102793+0000","last_clean":"2023-10-09T09:59:36.102793+0000","last_became_active":"2023-10-09T09:58:27.455003+0000","last_became_peered":"2023-10-09T09:58:27.455003+0000","last_unstale":"2023-10-09T09:59:36.102793+0000","last_undegraded":"2023-10-09T09:59:36.102793+0000","last_fullsized":"2023-10-09T09:59:36.102793+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":247,"log_dups_size":0,"ondisk_log_size":247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1714,"num_read_kb":1846,"num_write":205,"num_write_kb":259,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1202,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935415+0000","last_change":"2023-10-09T09:58:07.605971+0000","last_active":"2023-10-09T09:59:36.935415+0000","last_peered":"2023-10-09T09:59:36.935415+0000","last_clean":"2023-10-09T09:59:36.935415+0000","last_became_active":"2023-10-09T09:58:07.605139+0000","last_became_peered":"2023-10-09T09:58:07.605139+0000","last_unstale":"2023-10-09T09:59:36.935415+0000","last_undegraded":"2023-10-09T09:59:36.935415+0000","last_fullsized":"2023-10-09T09:59:36.935415+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1342,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.024423+0000","last_change":"2023-10-09T09:56:40.142238+0000","last_active":"2023-10-09T09:59:48.024423+0000","last_peered":"2023-10-09T09:59:48.024423+0000","last_clean":"2023-10-09T09:59:48.024423+0000","last_became_active":"2023-10-09T09:56:40.141699+0000","last_became_peered":"2023-10-09T09:56:40.141699+0000","last_unstale":"2023-10-09T09:59:48.024423+0000","last_undegraded":"2023-10-09T09:59:48.024423+0000","last_fullsized":"2023-10-09T09:59:48.024423+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:27.595630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"1311'103","reported_seq":747,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:41.182731+0000","last_change":"2023-10-09T09:56:43.143931+0000","last_active":"2023-10-09T09:59:41.182731+0000","last_peered":"2023-10-09T09:59:41.182731+0000","last_clean":"2023-10-09T09:59:41.182731+0000","last_became_active":"2023-10-09T09:56:43.143196+0000","last_became_peered":"2023-10-09T09:56:43.143196+0000","last_unstale":"2023-10-09T09:59:41.182731+0000","last_undegraded":"2023-10-09T09:59:41.182731+0000","last_fullsized":"2023-10-09T09:59:41.182731+0000","mapping_epoch":1318,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:18.533297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019660929999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":1020,"num_write":78,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1311,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.023805+0000","last_change":"2023-10-09T09:48:07.998234+0000","last_active":"2023-10-09T09:59:48.023805+0000","last_peered":"2023-10-09T09:59:48.023805+0000","last_clean":"2023-10-09T09:59:48.023805+0000","last_became_active":"2023-10-09T09:48:07.997909+0000","last_became_peered":"2023-10-09T09:48:07.997909+0000","last_unstale":"2023-10-09T09:59:48.023805+0000","last_undegraded":"2023-10-09T09:59:48.023805+0000","last_fullsized":"2023-10-09T09:59:48.023805+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:21.750617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1299,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.099336+0000","last_change":"2023-10-09T09:48:07.973284+0000","last_active":"2023-10-09T09:59:36.099336+0000","last_peered":"2023-10-09T09:59:36.099336+0000","last_clean":"2023-10-09T09:59:36.099336+0000","last_became_active":"2023-10-09T09:48:07.971235+0000","last_became_peered":"2023-10-09T09:48:07.971235+0000","last_unstale":"2023-10-09T09:59:36.099336+0000","last_undegraded":"2023-10-09T09:59:36.099336+0000","last_fullsized":"2023-10-09T09:59:36.099336+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:40.951420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"1311'146","reported_seq":1684,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102458+0000","last_change":"2023-10-09T09:58:27.454543+0000","last_active":"2023-10-09T09:59:36.102458+0000","last_peered":"2023-10-09T09:59:36.102458+0000","last_clean":"2023-10-09T09:59:36.102458+0000","last_became_active":"2023-10-09T09:58:27.454003+0000","last_became_peered":"2023-10-09T09:58:27.454003+0000","last_unstale":"2023-10-09T09:59:36.102458+0000","last_undegraded":"2023-10-09T09:59:36.102458+0000","last_fullsized":"2023-10-09T09:59:36.102458+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":146,"log_dups_size":0,"ondisk_log_size":146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":398,"num_read_kb":2392,"num_write":143,"num_write_kb":2132,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":698,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.931691+0000","last_change":"2023-10-09T09:57:00.113166+0000","last_active":"2023-10-09T09:59:36.931691+0000","last_peered":"2023-10-09T09:59:36.931691+0000","last_clean":"2023-10-09T09:59:36.931691+0000","last_became_active":"2023-10-09T09:57:00.112533+0000","last_became_peered":"2023-10-09T09:57:00.112533+0000","last_unstale":"2023-10-09T09:59:36.931691+0000","last_undegraded":"2023-10-09T09:59:36.931691+0000","last_fullsized":"2023-10-09T09:59:36.931691+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":186,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285246+0000","last_change":"2023-10-09T09:57:47.431484+0000","last_active":"2023-10-09T09:59:47.285246+0000","last_peered":"2023-10-09T09:59:47.285246+0000","last_clean":"2023-10-09T09:59:47.285246+0000","last_became_active":"2023-10-09T09:57:47.431063+0000","last_became_peered":"2023-10-09T09:57:47.431063+0000","last_unstale":"2023-10-09T09:59:47.285246+0000","last_undegraded":"2023-10-09T09:59:47.285246+0000","last_fullsized":"2023-10-09T09:59:47.285246+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:01:28.903833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1278,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934641+0000","last_change":"2023-10-09T09:56:57.087442+0000","last_active":"2023-10-09T09:59:36.934641+0000","last_peered":"2023-10-09T09:59:36.934641+0000","last_clean":"2023-10-09T09:59:36.934641+0000","last_became_active":"2023-10-09T09:56:57.087063+0000","last_became_peered":"2023-10-09T09:56:57.087063+0000","last_unstale":"2023-10-09T09:59:36.934641+0000","last_undegraded":"2023-10-09T09:59:36.934641+0000","last_fullsized":"2023-10-09T09:59:36.934641+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"1311'48","reported_seq":52,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.931663+0000","last_change":"2023-10-09T09:56:57.291785+0000","last_active":"2023-10-09T09:59:36.931663+0000","last_peered":"2023-10-09T09:59:36.931663+0000","last_clean":"2023-10-09T09:59:36.931663+0000","last_became_active":"2023-10-09T09:56:57.078576+0000","last_became_peered":"2023-10-09T09:56:57.078576+0000","last_unstale":"2023-10-09T09:59:36.931663+0000","last_undegraded":"2023-10-09T09:59:36.931663+0000","last_fullsized":"2023-10-09T09:59:36.931663+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":48,"log_dups_size":0,"ondisk_log_size":48,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":268,"num_read_kb":1552,"num_write":41,"num_write_kb":2215,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1294,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.570903+0000","last_change":"2023-10-09T09:56:40.130793+0000","last_active":"2023-10-09T09:59:47.570903+0000","last_peered":"2023-10-09T09:59:47.570903+0000","last_clean":"2023-10-09T09:59:47.570903+0000","last_became_active":"2023-10-09T09:56:40.130419+0000","last_became_peered":"2023-10-09T09:56:40.130419+0000","last_unstale":"2023-10-09T09:59:47.570903+0000","last_undegraded":"2023-10-09T09:59:47.570903+0000","last_fullsized":"2023-10-09T09:59:47.570903+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:50.436130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1296,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.591539+0000","last_change":"2023-10-09T09:48:07.981410+0000","last_active":"2023-10-09T09:59:36.591539+0000","last_peered":"2023-10-09T09:59:36.591539+0000","last_clean":"2023-10-09T09:59:36.591539+0000","last_became_active":"2023-10-09T09:48:07.979850+0000","last_became_peered":"2023-10-09T09:48:07.979850+0000","last_unstale":"2023-10-09T09:59:36.591539+0000","last_undegraded":"2023-10-09T09:59:36.591539+0000","last_fullsized":"2023-10-09T09:59:36.591539+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:31.201104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"1311'131","reported_seq":1848,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932450+0000","last_change":"2023-10-09T09:57:00.115713+0000","last_active":"2023-10-09T09:59:36.932450+0000","last_peered":"2023-10-09T09:59:36.932450+0000","last_clean":"2023-10-09T09:59:36.932450+0000","last_became_active":"2023-10-09T09:57:00.114879+0000","last_became_peered":"2023-10-09T09:57:00.114879+0000","last_unstale":"2023-10-09T09:59:36.932450+0000","last_undegraded":"2023-10-09T09:59:36.932450+0000","last_fullsized":"2023-10-09T09:59:36.932450+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1092,"num_read_kb":2605,"num_write":104,"num_write_kb":2443,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1314,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.024623+0000","last_change":"2023-10-09T09:48:07.983773+0000","last_active":"2023-10-09T09:59:48.024623+0000","last_peered":"2023-10-09T09:59:48.024623+0000","last_clean":"2023-10-09T09:59:48.024623+0000","last_became_active":"2023-10-09T09:48:07.976869+0000","last_became_peered":"2023-10-09T09:48:07.976869+0000","last_unstale":"2023-10-09T09:59:48.024623+0000","last_undegraded":"2023-10-09T09:59:48.024623+0000","last_fullsized":"2023-10-09T09:59:48.024623+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:58.173236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1251,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.592089+0000","last_change":"2023-10-09T09:59:27.201228+0000","last_active":"2023-10-09T09:59:36.592089+0000","last_peered":"2023-10-09T09:59:36.592089+0000","last_clean":"2023-10-09T09:59:36.592089+0000","last_became_active":"2023-10-09T09:59:27.200884+0000","last_became_peered":"2023-10-09T09:59:27.200884+0000","last_unstale":"2023-10-09T09:59:36.592089+0000","last_undegraded":"2023-10-09T09:59:36.592089+0000","last_fullsized":"2023-10-09T09:59:36.592089+0000","mapping_epoch":1341,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:12.022663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":208,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285557+0000","last_change":"2023-10-09T09:57:47.431776+0000","last_active":"2023-10-09T09:59:47.285557+0000","last_peered":"2023-10-09T09:59:47.285557+0000","last_clean":"2023-10-09T09:59:47.285557+0000","last_became_active":"2023-10-09T09:57:47.431275+0000","last_became_peered":"2023-10-09T09:57:47.431275+0000","last_unstale":"2023-10-09T09:59:47.285557+0000","last_undegraded":"2023-10-09T09:59:47.285557+0000","last_fullsized":"2023-10-09T09:59:47.285557+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:39.717442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"1311'57","reported_seq":1486,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.099341+0000","last_change":"2023-10-09T09:56:05.154277+0000","last_active":"2023-10-09T09:59:36.099341+0000","last_peered":"2023-10-09T09:59:36.099341+0000","last_clean":"2023-10-09T09:59:36.099341+0000","last_became_active":"2023-10-09T09:48:07.998606+0000","last_became_peered":"2023-10-09T09:48:07.998606+0000","last_unstale":"2023-10-09T09:59:36.099341+0000","last_undegraded":"2023-10-09T09:59:36.099341+0000","last_fullsized":"2023-10-09T09:59:36.099341+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":57,"log_dups_size":0,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.469476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0028883569999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":282,"num_read_kb":223,"num_write":26,"num_write_kb":22,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1200,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.284819+0000","last_change":"2023-10-09T09:57:47.433836+0000","last_active":"2023-10-09T09:59:47.284819+0000","last_peered":"2023-10-09T09:59:47.284819+0000","last_clean":"2023-10-09T09:59:47.284819+0000","last_became_active":"2023-10-09T09:57:47.432833+0000","last_became_peered":"2023-10-09T09:57:47.432833+0000","last_unstale":"2023-10-09T09:59:47.284819+0000","last_undegraded":"2023-10-09T09:59:47.284819+0000","last_fullsized":"2023-10-09T09:59:47.284819+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:18.447204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"1311'75","reported_seq":1621,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934998+0000","last_change":"2023-10-09T09:56:57.097915+0000","last_active":"2023-10-09T09:59:36.934998+0000","last_peered":"2023-10-09T09:59:36.934998+0000","last_clean":"2023-10-09T09:59:36.934998+0000","last_became_active":"2023-10-09T09:56:57.081618+0000","last_became_peered":"2023-10-09T09:56:57.081618+0000","last_unstale":"2023-10-09T09:59:36.934998+0000","last_undegraded":"2023-10-09T09:59:36.934998+0000","last_fullsized":"2023-10-09T09:59:36.934998+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":75,"log_dups_size":0,"ondisk_log_size":75,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":424,"num_read_kb":544,"num_write":78,"num_write_kb":145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1277,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.591576+0000","last_change":"2023-10-09T09:48:07.980600+0000","last_active":"2023-10-09T09:59:36.591576+0000","last_peered":"2023-10-09T09:59:36.591576+0000","last_clean":"2023-10-09T09:59:36.591576+0000","last_became_active":"2023-10-09T09:48:07.979544+0000","last_became_peered":"2023-10-09T09:48:07.979544+0000","last_unstale":"2023-10-09T09:59:36.591576+0000","last_undegraded":"2023-10-09T09:59:36.591576+0000","last_fullsized":"2023-10-09T09:59:36.591576+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:47.320689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1189,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935365+0000","last_change":"2023-10-09T09:58:07.131154+0000","last_active":"2023-10-09T09:59:36.935365+0000","last_peered":"2023-10-09T09:59:36.935365+0000","last_clean":"2023-10-09T09:59:36.935365+0000","last_became_active":"2023-10-09T09:58:07.130161+0000","last_became_peered":"2023-10-09T09:58:07.130161+0000","last_unstale":"2023-10-09T09:59:36.935365+0000","last_undegraded":"2023-10-09T09:59:36.935365+0000","last_fullsized":"2023-10-09T09:59:36.935365+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1294,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.571802+0000","last_change":"2023-10-09T09:56:57.083025+0000","last_active":"2023-10-09T09:59:47.571802+0000","last_peered":"2023-10-09T09:59:47.571802+0000","last_clean":"2023-10-09T09:59:47.571802+0000","last_became_active":"2023-10-09T09:56:57.082456+0000","last_became_peered":"2023-10-09T09:56:57.082456+0000","last_unstale":"2023-10-09T09:59:47.571802+0000","last_undegraded":"2023-10-09T09:59:47.571802+0000","last_fullsized":"2023-10-09T09:59:47.571802+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:35.646438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":245,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.175876+0000","last_change":"2023-10-09T09:56:40.130207+0000","last_active":"2023-10-09T09:59:36.175876+0000","last_peered":"2023-10-09T09:59:36.175876+0000","last_clean":"2023-10-09T09:59:36.175876+0000","last_became_active":"2023-10-09T09:56:40.129857+0000","last_became_peered":"2023-10-09T09:56:40.129857+0000","last_unstale":"2023-10-09T09:59:36.175876+0000","last_undegraded":"2023-10-09T09:59:36.175876+0000","last_fullsized":"2023-10-09T09:59:36.175876+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:02.149845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":85,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102377+0000","last_change":"2023-10-09T09:58:27.455418+0000","last_active":"2023-10-09T09:59:36.102377+0000","last_peered":"2023-10-09T09:59:36.102377+0000","last_clean":"2023-10-09T09:59:36.102377+0000","last_became_active":"2023-10-09T09:58:27.454696+0000","last_became_peered":"2023-10-09T09:58:27.454696+0000","last_unstale":"2023-10-09T09:59:36.102377+0000","last_undegraded":"2023-10-09T09:59:36.102377+0000","last_fullsized":"2023-10-09T09:59:36.102377+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1276,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935605+0000","last_change":"2023-10-09T09:58:07.778693+0000","last_active":"2023-10-09T09:59:36.935605+0000","last_peered":"2023-10-09T09:59:36.935605+0000","last_clean":"2023-10-09T09:59:36.935605+0000","last_became_active":"2023-10-09T09:58:07.777319+0000","last_became_peered":"2023-10-09T09:58:07.777319+0000","last_unstale":"2023-10-09T09:59:36.935605+0000","last_undegraded":"2023-10-09T09:59:36.935605+0000","last_fullsized":"2023-10-09T09:59:36.935605+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":75,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102873+0000","last_change":"2023-10-09T09:58:27.456714+0000","last_active":"2023-10-09T09:59:36.102873+0000","last_peered":"2023-10-09T09:59:36.102873+0000","last_clean":"2023-10-09T09:59:36.102873+0000","last_became_active":"2023-10-09T09:58:27.456216+0000","last_became_peered":"2023-10-09T09:58:27.456216+0000","last_unstale":"2023-10-09T09:59:36.102873+0000","last_undegraded":"2023-10-09T09:59:36.102873+0000","last_fullsized":"2023-10-09T09:59:36.102873+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1247,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935383+0000","last_change":"2023-10-09T09:58:07.778929+0000","last_active":"2023-10-09T09:59:36.935383+0000","last_peered":"2023-10-09T09:59:36.935383+0000","last_clean":"2023-10-09T09:59:36.935383+0000","last_became_active":"2023-10-09T09:58:07.777990+0000","last_became_peered":"2023-10-09T09:58:07.777990+0000","last_unstale":"2023-10-09T09:59:36.935383+0000","last_undegraded":"2023-10-09T09:59:36.935383+0000","last_fullsized":"2023-10-09T09:59:36.935383+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":291,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285453+0000","last_change":"2023-10-09T09:57:47.433889+0000","last_active":"2023-10-09T09:59:47.285453+0000","last_peered":"2023-10-09T09:59:47.285453+0000","last_clean":"2023-10-09T09:59:47.285453+0000","last_became_active":"2023-10-09T09:57:47.433362+0000","last_became_peered":"2023-10-09T09:57:47.433362+0000","last_unstale":"2023-10-09T09:59:47.285453+0000","last_undegraded":"2023-10-09T09:59:47.285453+0000","last_fullsized":"2023-10-09T09:59:47.285453+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:28:05.820832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":220,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285427+0000","last_change":"2023-10-09T09:57:47.433721+0000","last_active":"2023-10-09T09:59:47.285427+0000","last_peered":"2023-10-09T09:59:47.285427+0000","last_clean":"2023-10-09T09:59:47.285427+0000","last_became_active":"2023-10-09T09:57:47.433352+0000","last_became_peered":"2023-10-09T09:57:47.433352+0000","last_unstale":"2023-10-09T09:59:47.285427+0000","last_undegraded":"2023-10-09T09:59:47.285427+0000","last_fullsized":"2023-10-09T09:59:47.285427+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:08.161454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1311,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.571656+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:59:47.571656+0000","last_peered":"2023-10-09T09:59:47.571656+0000","last_clean":"2023-10-09T09:59:47.571656+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:59:47.571656+0000","last_undegraded":"2023-10-09T09:59:47.571656+0000","last_fullsized":"2023-10-09T09:59:47.571656+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1186,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935229+0000","last_change":"2023-10-09T09:58:07.128856+0000","last_active":"2023-10-09T09:59:36.935229+0000","last_peered":"2023-10-09T09:59:36.935229+0000","last_clean":"2023-10-09T09:59:36.935229+0000","last_became_active":"2023-10-09T09:58:07.128405+0000","last_became_peered":"2023-10-09T09:58:07.128405+0000","last_unstale":"2023-10-09T09:59:36.935229+0000","last_undegraded":"2023-10-09T09:59:36.935229+0000","last_fullsized":"2023-10-09T09:59:36.935229+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":7410,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30942,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":12213,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398232,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312040,"kb_used_data":10720,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93433112,"statfs":{"total":95995035648,"available":95675506688,"internally_reserved":0,"allocated":10977280,"data_stored":10630753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.0029999999999999,"15min":1.22},"min":{"1min":0.58799999999999997,"5min":0.53400000000000003,"15min":0.26500000000000001},"max":{"1min":1.2669999999999999,"5min":1.325,"15min":46.389000000000003},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.98999999999999999,"15min":1.2609999999999999},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.28999999999999998},"max":{"1min":1.276,"5min":1.3169999999999999,"15min":46.558},"last":0.78800000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.99399999999999999,"15min":1.2789999999999999},"min":{"1min":0.629,"5min":0.629,"15min":0.29599999999999999},"max":{"1min":1.2629999999999999,"5min":1.5820000000000001,"15min":43.752000000000002},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.92000000000000004,"15min":1.1160000000000001},"min":{"1min":0.754,"5min":0.54800000000000004,"15min":0.307},"max":{"1min":1.1799999999999999,"5min":1.27,"15min":43.503999999999998},"last":0.95399999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:00:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.0840000000000001,"15min":1.944},"min":{"1min":0.69899999999999995,"5min":0.66800000000000004,"15min":0.29099999999999998},"max":{"1min":1.3819999999999999,"5min":1.5329999999999999,"15min":106.124},"last":0.63},{"interface":"front","average":{"1min":1.093,"5min":1.032,"15min":1.915},"min":{"1min":0.76100000000000001,"5min":0.60599999999999998,"15min":0.27700000000000002},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":106.188},"last":0.91600000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:00:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.0049999999999999,"15min":1.0740000000000001},"min":{"1min":0.84299999999999997,"5min":0.56499999999999995,"15min":0.35699999999999998},"max":{"1min":1.3129999999999999,"5min":1.397,"15min":14.98},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.1160000000000001,"15min":1.109},"min":{"1min":0.83199999999999996,"5min":0.65300000000000002,"15min":0.35399999999999998},"max":{"1min":1.5920000000000001,"5min":1.5920000000000001,"15min":15.018000000000001},"last":1.1870000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:00:33 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.98499999999999999,"15min":1.0289999999999999},"min":{"1min":0.74299999999999999,"5min":0.64100000000000001,"15min":0.27700000000000002},"max":{"1min":1.3320000000000001,"5min":1.381,"15min":5.8479999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.089,"5min":1.05,"15min":1.022},"min":{"1min":0.83299999999999996,"5min":0.67200000000000004,"15min":0.33500000000000002},"max":{"1min":1.45,"5min":1.5720000000000001,"15min":7.4379999999999997},"last":1.054}]},{"osd":5,"last update":"Mon Oct 9 10:00:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.141,"15min":1.157},"min":{"1min":0.83899999999999997,"5min":0.70499999999999996,"15min":0.377},"max":{"1min":1.496,"5min":1.637,"15min":10.378},"last":1.0940000000000001},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.139,"15min":1.042},"min":{"1min":0.84499999999999997,"5min":0.58999999999999997,"15min":0.35299999999999998},"max":{"1min":1.3839999999999999,"5min":1.591,"15min":7.1459999999999999},"last":1.03}]},{"osd":6,"last update":"Mon Oct 9 10:00:33 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.1879999999999999,"15min":1.3009999999999999},"min":{"1min":0.80200000000000005,"5min":0.68700000000000006,"15min":0.38},"max":{"1min":1.3839999999999999,"5min":1.5109999999999999,"15min":9.0589999999999993},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.169,"15min":1.3029999999999999},"min":{"1min":0.86499999999999999,"5min":0.60799999999999998,"15min":0.36499999999999999},"max":{"1min":1.518,"5min":1.6639999999999999,"15min":8.0329999999999995},"last":1.155}]}]},{"osd":6,"up_from":43,"seq":184683594465,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907296,"kb_used_data":6616,"kb_used_omap":0,"kb_used_meta":900672,"kb_avail":92837856,"statfs":{"total":95995035648,"available":95065964544,"internally_reserved":0,"allocated":6774784,"data_stored":6436322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922288128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:06 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.90300000000000002,"15min":2.4369999999999998},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.23599999999999999},"max":{"1min":1.415,"5min":1.415,"15min":149.095},"last":1.0840000000000001},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.022,"15min":2.6040000000000001},"min":{"1min":0.69699999999999995,"5min":0.59899999999999998,"15min":0.28699999999999998},"max":{"1min":1.488,"5min":1.488,"15min":149.17500000000001},"last":1.236}]},{"osd":1,"last update":"Mon Oct 9 10:00:27 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.96999999999999997,"15min":1.8009999999999999},"min":{"1min":0.51700000000000002,"5min":0.436,"15min":0.28599999999999998},"max":{"1min":1.327,"5min":1.496,"15min":55.368000000000002},"last":1.129},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.92900000000000005,"15min":1.7470000000000001},"min":{"1min":0.628,"5min":0.58199999999999996,"15min":0.30499999999999999},"max":{"1min":1.7529999999999999,"5min":1.7529999999999999,"15min":55.404000000000003},"last":0.91400000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.94999999999999996,"15min":3.2400000000000002},"min":{"1min":0.70699999999999996,"5min":0.59299999999999997,"15min":0.29799999999999999},"max":{"1min":1.369,"5min":1.369,"15min":206.97200000000001},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.0589999999999999,"15min":3.3929999999999998},"min":{"1min":0.626,"5min":0.55300000000000005,"15min":0.309},"max":{"1min":1.254,"5min":1.4139999999999999,"15min":208.90899999999999},"last":0.97899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.88800000000000001,"15min":1.069},"min":{"1min":0.60099999999999998,"5min":0.56799999999999995,"15min":0.29099999999999998},"max":{"1min":1.1850000000000001,"5min":1.5109999999999999,"15min":7.9139999999999997},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.99099999999999999,"15min":1.2749999999999999},"min":{"1min":0.748,"5min":0.63700000000000001,"15min":0.34899999999999998},"max":{"1min":1.2689999999999999,"5min":1.645,"15min":11.968999999999999},"last":1.0960000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:00:54 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0329999999999999,"15min":1.341},"min":{"1min":0.70899999999999996,"5min":0.501,"15min":0.28399999999999997},"max":{"1min":1.355,"5min":1.712,"15min":12.148999999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.996,"5min":0.98999999999999999,"15min":1.46},"min":{"1min":0.73499999999999999,"5min":0.57299999999999995,"15min":0.313},"max":{"1min":1.28,"5min":1.554,"15min":12.064},"last":0.80100000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:00:54 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.125,"15min":1.3959999999999999},"min":{"1min":0.71099999999999997,"5min":0.54300000000000004,"15min":0.318},"max":{"1min":1.337,"5min":1.8029999999999999,"15min":10.529},"last":0.83399999999999996},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.113,"15min":1.4510000000000001},"min":{"1min":0.80700000000000005,"5min":0.48299999999999998,"15min":0.34300000000000003},"max":{"1min":1.3500000000000001,"5min":1.881,"15min":12.000999999999999},"last":1.01}]},{"osd":7,"last update":"Mon Oct 9 10:00:21 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.1759999999999999,"15min":1.4950000000000001},"min":{"1min":0.94499999999999995,"5min":0.64700000000000002,"15min":0.38500000000000001},"max":{"1min":1.825,"5min":1.825,"15min":11.922000000000001},"last":1.204},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1439999999999999,"15min":1.218},"min":{"1min":0.83599999999999997,"5min":0.496,"15min":0.32700000000000001},"max":{"1min":1.784,"5min":1.784,"15min":7.9359999999999999},"last":0.77500000000000002}]}]},{"osd":2,"up_from":634,"seq":2723009266218,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":904176,"kb_used_data":7656,"kb_used_omap":122,"kb_used_meta":896389,"kb_avail":92840976,"statfs":{"total":95995035648,"available":95069159424,"internally_reserved":0,"allocated":7839744,"data_stored":6093716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":917902923},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:10 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.66700000000000004,"15min":3.2789999999999999},"min":{"1min":0.53300000000000003,"5min":0.27900000000000003,"15min":0.221},"max":{"1min":0.88800000000000001,"5min":1.149,"15min":218.30799999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.72899999999999998,"15min":3.411},"min":{"1min":0.51300000000000001,"5min":0.33500000000000002,"15min":0.20799999999999999},"max":{"1min":1.163,"5min":1.2250000000000001,"15min":217.67500000000001},"last":0.80200000000000005}]},{"osd":1,"last update":"Mon Oct 9 10:00:32 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.78000000000000003,"15min":1.8500000000000001},"min":{"1min":0.47499999999999998,"5min":0.374,"15min":0.22},"max":{"1min":1.234,"5min":1.321,"15min":106.089},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.75,"15min":1.819},"min":{"1min":0.372,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.2030000000000001,"5min":1.2809999999999999,"15min":105.571},"last":0.86199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:00:10 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.77900000000000003,"15min":0.997},"min":{"1min":0.499,"5min":0.315,"15min":0.26500000000000001},"max":{"1min":1.202,"5min":1.202,"15min":8.1289999999999996},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.78000000000000003,"15min":0.91600000000000004},"min":{"1min":0.54800000000000004,"5min":0.33800000000000002,"15min":0.26100000000000001},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":9.2810000000000006},"last":0.91100000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:59:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.98899999999999999,"15min":1.4119999999999999},"min":{"1min":0.73399999999999999,"5min":0.55600000000000005,"15min":0.27100000000000002},"max":{"1min":1.621,"5min":1.625,"15min":71.876000000000005},"last":1.0449999999999999},{"interface":"front","average":{"1min":1.04,"5min":0.98699999999999999,"15min":1.3129999999999999},"min":{"1min":0.69199999999999995,"5min":0.48899999999999999,"15min":0.29499999999999998},"max":{"1min":1.5269999999999999,"5min":1.7370000000000001,"15min":9.4640000000000004},"last":1.0760000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:59:57 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":0.998,"15min":1.3340000000000001},"min":{"1min":0.72199999999999998,"5min":0.436,"15min":0.34100000000000003},"max":{"1min":1.3260000000000001,"5min":1.4910000000000001,"15min":8.9350000000000005},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.2050000000000001},"min":{"1min":0.77900000000000003,"5min":0.50800000000000001,"15min":0.35199999999999998},"max":{"1min":1.47,"5min":1.5840000000000001,"15min":8.0109999999999992},"last":1.1419999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:59:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.075,"15min":1.663},"min":{"1min":0.63300000000000001,"5min":0.42799999999999999,"15min":0.30199999999999999},"max":{"1min":1.694,"5min":1.694,"15min":71.795000000000002},"last":1.196},{"interface":"front","average":{"1min":1.161,"5min":1.099,"15min":1.7190000000000001},"min":{"1min":0.89400000000000002,"5min":0.621,"15min":0.36299999999999999},"max":{"1min":1.573,"5min":1.758,"15min":72.322000000000003},"last":1.167}]},{"osd":7,"last update":"Mon Oct 9 09:59:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.1479999999999999,"15min":1.365},"min":{"1min":0.67300000000000004,"5min":0.57599999999999996,"15min":0.374},"max":{"1min":1.7230000000000001,"5min":1.8400000000000001,"15min":8.4659999999999993},"last":0.97799999999999998},{"interface":"front","average":{"1min":1.159,"5min":1.1479999999999999,"15min":1.3340000000000001},"min":{"1min":0.77200000000000002,"5min":0.626,"15min":0.374},"max":{"1min":1.3879999999999999,"5min":1.702,"15min":8.8849999999999998},"last":1.115}]}]},{"osd":1,"up_from":1246,"seq":5351529250973,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":417436,"kb_used_data":11732,"kb_used_omap":204,"kb_used_meta":405491,"kb_avail":93327716,"statfs":{"total":95995035648,"available":95567581184,"internally_reserved":0,"allocated":12013568,"data_stored":10286592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208958,"internal_metadata":415223746},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.69699999999999995,"15min":1.621},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.191},"max":{"1min":1.131,"5min":1.1659999999999999,"15min":58.033000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.70399999999999996,"15min":1.7190000000000001},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.19},"max":{"1min":1.1830000000000001,"5min":1.2709999999999999,"15min":65.921999999999997},"last":0.82899999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.75,"15min":1.589},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.26600000000000001},"max":{"1min":1.1060000000000001,"5min":1.3240000000000001,"15min":58.067999999999998},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.72599999999999998,"15min":1.5249999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.20200000000000001},"max":{"1min":1.042,"5min":1.278,"15min":58.682000000000002},"last":0.66400000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.80800000000000005,"15min":1.2150000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.20899999999999999},"max":{"1min":1.042,"5min":1.2030000000000001,"15min":8.7140000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.90700000000000003,"15min":1.198},"min":{"1min":0.42899999999999999,"5min":0.39600000000000002,"15min":0.27500000000000002},"max":{"1min":1.3,"5min":1.375,"15min":7.5780000000000003},"last":0.80100000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.94899999999999995,"15min":1.704},"min":{"1min":0.61599999999999999,"5min":0.51200000000000001,"15min":0.30599999999999999},"max":{"1min":1.2490000000000001,"5min":1.522,"15min":8.8070000000000004},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.026,"5min":1.0349999999999999,"15min":1.6659999999999999},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.32000000000000001},"max":{"1min":1.6579999999999999,"5min":1.6579999999999999,"15min":10.263},"last":0.93200000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.0529999999999999,"15min":1.4870000000000001},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.33500000000000002},"max":{"1min":1.506,"5min":1.5669999999999999,"15min":10.214},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0349999999999999,"15min":1.4690000000000001},"min":{"1min":0.70999999999999996,"5min":0.65600000000000003,"15min":0.32400000000000001},"max":{"1min":1.7849999999999999,"5min":1.7849999999999999,"15min":8.5139999999999993},"last":0.88900000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0609999999999999,"15min":1.6319999999999999},"min":{"1min":0.63400000000000001,"5min":0.63,"15min":0.35199999999999998},"max":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":9.3309999999999995},"last":0.96199999999999997},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0720000000000001,"15min":1.9970000000000001},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.34799999999999998},"max":{"1min":1.488,"5min":1.756,"15min":9.8659999999999997},"last":1.004}]},{"osd":7,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.1319999999999999,"15min":1.4950000000000001},"min":{"1min":0.81399999999999995,"5min":0.70299999999999996,"15min":0.374},"max":{"1min":1.6910000000000001,"5min":1.786,"15min":9.2929999999999993},"last":1.024},{"interface":"front","average":{"1min":1.131,"5min":1.1679999999999999,"15min":1.746},"min":{"1min":0.70899999999999996,"5min":0.70399999999999996,"15min":0.371},"max":{"1min":1.5720000000000001,"5min":1.8520000000000001,"15min":9.4529999999999994},"last":1.123}]}]},{"osd":0,"up_from":1136,"seq":4879082848566,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":762124,"kb_used_data":12292,"kb_used_omap":280,"kb_used_meta":749543,"kb_avail":92983028,"statfs":{"total":95995035648,"available":95214620672,"internally_reserved":0,"allocated":12587008,"data_stored":10867464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":767532442},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:00:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.68200000000000005,"15min":1.671},"min":{"1min":0.30199999999999999,"5min":0.29599999999999999,"15min":0.19400000000000001},"max":{"1min":1.2230000000000001,"5min":1.238,"15min":69.680000000000007},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.73399999999999999,"15min":1.772},"min":{"1min":0.34799999999999998,"5min":0.33100000000000002,"15min":0.20399999999999999},"max":{"1min":1.0229999999999999,"5min":1.6120000000000001,"15min":70.668999999999997},"last":1.0960000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":1.7170000000000001,"15min":2.742},"min":{"1min":0.40100000000000002,"5min":0.38200000000000001,"15min":0.22},"max":{"1min":1.2949999999999999,"5min":80.686999999999998,"15min":254.89599999999999},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.6859999999999999,"15min":3.0859999999999999},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.23699999999999999},"max":{"1min":0.94399999999999995,"5min":80.843000000000004,"15min":256.85399999999998},"last":0.76700000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.81200000000000006,"15min":0.97199999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.247},"max":{"1min":1.151,"5min":1.5229999999999999,"15min":8.8309999999999995},"last":0.97599999999999998},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.81599999999999995,"15min":0.95899999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.28299999999999997},"max":{"1min":1.1180000000000001,"5min":1.369,"15min":6.5220000000000002},"last":0.85299999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.89100000000000001,"15min":1.361},"min":{"1min":0.61399999999999999,"5min":0.57699999999999996,"15min":0.318},"max":{"1min":1.3839999999999999,"5min":1.8080000000000001,"15min":20.079000000000001},"last":0.94599999999999995},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.0820000000000001,"15min":1.3280000000000001},"min":{"1min":0.69499999999999995,"5min":0.60599999999999998,"15min":0.32700000000000001},"max":{"1min":1.476,"5min":1.7929999999999999,"15min":12.044},"last":1.0589999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.1100000000000001,"15min":1.3740000000000001},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.313},"max":{"1min":1.581,"5min":1.863,"15min":9.4280000000000008},"last":0.91300000000000003},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.0449999999999999,"15min":1.3700000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.33700000000000002},"max":{"1min":1.3480000000000001,"5min":1.6890000000000001,"15min":9.6289999999999996},"last":1.1870000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.0109999999999999,"15min":1.375},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.34300000000000003},"max":{"1min":1.45,"5min":1.5329999999999999,"15min":9.5950000000000006},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.181,"15min":1.5609999999999999},"min":{"1min":0.79600000000000004,"5min":0.71999999999999997,"15min":0.42599999999999999},"max":{"1min":1.6839999999999999,"5min":1.7350000000000001,"15min":9.3260000000000005},"last":1.1259999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.0389999999999999,"15min":1.3089999999999999},"min":{"1min":0.65600000000000003,"5min":0.61399999999999999,"15min":0.38},"max":{"1min":1.532,"5min":1.532,"15min":7.0149999999999997},"last":1.151},{"interface":"front","average":{"1min":1.127,"5min":1.222,"15min":1.355},"min":{"1min":0.76100000000000001,"5min":0.69699999999999995,"15min":0.435},"max":{"1min":1.6399999999999999,"5min":1.8520000000000001,"15min":10.087999999999999},"last":1.286}]}]},{"osd":3,"up_from":663,"seq":2847563317796,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599676,"kb_used_data":7412,"kb_used_omap":55,"kb_used_meta":592200,"kb_avail":93145476,"statfs":{"total":95995035648,"available":95380967424,"internally_reserved":0,"allocated":7589888,"data_stored":6055072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":606413410},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.70499999999999996,"15min":1.6459999999999999},"min":{"1min":0.41499999999999998,"5min":0.28799999999999998,"15min":0.182},"max":{"1min":1.2010000000000001,"5min":1.2010000000000001,"15min":120.443},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.74399999999999999,"15min":1.7689999999999999},"min":{"1min":0.439,"5min":0.36899999999999999,"15min":0.20300000000000001},"max":{"1min":1.306,"5min":1.306,"15min":120.07599999999999},"last":0.86299999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:00:34 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.71099999999999997,"15min":1.7350000000000001},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.255},"max":{"1min":1.478,"5min":1.478,"15min":90.396000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.83599999999999997,"15min":1.8540000000000001},"min":{"1min":0.40999999999999998,"5min":0.29699999999999999,"15min":0.245},"max":{"1min":1.2450000000000001,"5min":1.8819999999999999,"15min":90.260000000000005},"last":0.54600000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.86399999999999999,"15min":1.5489999999999999},"min":{"1min":0.497,"5min":0.32900000000000001,"15min":0.254},"max":{"1min":1.3380000000000001,"5min":1.659,"15min":120.256},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.80300000000000005,"15min":1.651},"min":{"1min":0.46999999999999997,"5min":0.372,"15min":0.23999999999999999},"max":{"1min":1.381,"5min":1.413,"15min":119.55800000000001},"last":0.93100000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.98199999999999998,"15min":1.0209999999999999},"min":{"1min":0.624,"5min":0.624,"15min":0.317},"max":{"1min":1.3360000000000001,"5min":1.631,"15min":7.9729999999999999},"last":1.343},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0249999999999999,"15min":1.0349999999999999},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.33300000000000002},"max":{"1min":1.4019999999999999,"5min":1.7569999999999999,"15min":8.1199999999999992},"last":0.747}]},{"osd":5,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.0840000000000001,"15min":1.018},"min":{"1min":0.747,"5min":0.63100000000000001,"15min":0.33400000000000002},"max":{"1min":1.397,"5min":2.4140000000000001,"15min":8.7430000000000003},"last":1.0449999999999999},{"interface":"front","average":{"1min":1.081,"5min":1.0609999999999999,"15min":1.0129999999999999},"min":{"1min":0.753,"5min":0.61799999999999999,"15min":0.34399999999999997},"max":{"1min":1.411,"5min":1.5740000000000001,"15min":7.9459999999999997},"last":1.6200000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.107,"15min":1.5069999999999999},"min":{"1min":0.66400000000000003,"5min":0.63700000000000001,"15min":0.36099999999999999},"max":{"1min":1.6579999999999999,"5min":1.927,"15min":98.084999999999994},"last":1.0109999999999999},{"interface":"front","average":{"1min":1.131,"5min":1.125,"15min":1.6339999999999999},"min":{"1min":0.79300000000000004,"5min":0.71099999999999997,"15min":0.40899999999999997},"max":{"1min":1.361,"5min":1.6619999999999999,"15min":99.144000000000005},"last":1.506}]},{"osd":7,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":1.143,"15min":0.98499999999999999},"min":{"1min":0.97599999999999998,"5min":0.64200000000000002,"15min":0.39000000000000001},"max":{"1min":1.427,"5min":2.3010000000000002,"15min":8.0960000000000001},"last":1.1140000000000001},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.1639999999999999,"15min":0.99399999999999999},"min":{"1min":0.88200000000000001,"5min":0.70899999999999996,"15min":0.40899999999999997},"max":{"1min":1.76,"5min":1.8460000000000001,"15min":7.8879999999999999},"last":1.0980000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986930,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":897120,"kb_used_data":6040,"kb_used_omap":0,"kb_used_meta":891072,"kb_avail":92848032,"statfs":{"total":95995035648,"available":95076384768,"internally_reserved":0,"allocated":6184960,"data_stored":5846011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912457728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.90100000000000002,"15min":1.5229999999999999},"min":{"1min":0.52000000000000002,"5min":0.51500000000000001,"15min":0.26200000000000001},"max":{"1min":1.4419999999999999,"5min":1.4419999999999999,"15min":64.372},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.052,"5min":0.95599999999999996,"15min":1.506},"min":{"1min":0.61599999999999999,"5min":0.497,"15min":0.30099999999999999},"max":{"1min":1.3069999999999999,"5min":1.3660000000000001,"15min":64.356999999999999},"last":0.83899999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.95899999999999996,"15min":2.3090000000000002},"min":{"1min":0.59199999999999997,"5min":0.54500000000000004,"15min":0.27700000000000002},"max":{"1min":1.444,"5min":3.653,"15min":216.16900000000001},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.04,"15min":2.2400000000000002},"min":{"1min":0.74099999999999999,"5min":0.68700000000000006,"15min":0.29299999999999998},"max":{"1min":1.2869999999999999,"5min":3.5870000000000002,"15min":216.262},"last":1.1559999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.90500000000000003,"15min":1.2869999999999999},"min":{"1min":0.66700000000000004,"5min":0.53000000000000003,"15min":0.31},"max":{"1min":1.075,"5min":1.341,"15min":15.853999999999999},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.92900000000000005,"5min":0.95199999999999996,"15min":1.282},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.28799999999999998},"max":{"1min":1.4890000000000001,"5min":4.1449999999999996,"15min":15.881},"last":0.81899999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:00:16 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.93600000000000005,"15min":1.01},"min":{"1min":0.66600000000000004,"5min":0.57099999999999995,"15min":0.32200000000000001},"max":{"1min":1.286,"5min":2.7759999999999998,"15min":8.1120000000000001},"last":1.006},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.95499999999999996,"15min":1.0620000000000001},"min":{"1min":0.63100000000000001,"5min":0.59599999999999997,"15min":0.34899999999999998},"max":{"1min":1.5980000000000001,"5min":1.645,"15min":7.4180000000000001},"last":1.034}]},{"osd":5,"last update":"Mon Oct 9 10:00:16 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.0069999999999999,"15min":1.2589999999999999},"min":{"1min":0.68700000000000006,"5min":0.57199999999999995,"15min":0.35599999999999998},"max":{"1min":1.345,"5min":1.423,"15min":11.763999999999999},"last":1.133},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.1120000000000001,"15min":1.2450000000000001},"min":{"1min":0.88900000000000001,"5min":0.58799999999999997,"15min":0.28499999999999998},"max":{"1min":1.4119999999999999,"5min":1.528,"15min":12.446999999999999},"last":1.081}]},{"osd":6,"last update":"Mon Oct 9 10:00:16 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0009999999999999,"15min":1.3839999999999999},"min":{"1min":0.69399999999999995,"5min":0.69299999999999995,"15min":0.32300000000000001},"max":{"1min":1.8700000000000001,"5min":1.8700000000000001,"15min":11.701000000000001},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.167,"15min":1.502},"min":{"1min":0.88900000000000001,"5min":0.79300000000000004,"15min":0.40799999999999997},"max":{"1min":2.089,"5min":2.089,"15min":11.972},"last":0.97799999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:00:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.1000000000000001,"15min":1.2030000000000001},"min":{"1min":0.88600000000000001,"5min":0.64500000000000002,"15min":0.373},"max":{"1min":1.5669999999999999,"5min":1.581,"15min":11.840999999999999},"last":0.749},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.129,"15min":1.1220000000000001},"min":{"1min":0.81899999999999995,"5min":0.68400000000000005,"15min":0.41299999999999998},"max":{"1min":1.542,"5min":1.615,"15min":9.6210000000000004},"last":0.94399999999999995}]}]},{"osd":5,"up_from":37,"seq":158913790696,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":904340,"kb_used_data":6028,"kb_used_omap":0,"kb_used_meta":898304,"kb_avail":92840812,"statfs":{"total":95995035648,"available":95068991488,"internally_reserved":0,"allocated":6172672,"data_stored":5841774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919863296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.95199999999999996,"15min":2.2989999999999999},"min":{"1min":0.53400000000000003,"5min":0.41899999999999998,"15min":0.253},"max":{"1min":1.25,"5min":1.4570000000000001,"15min":121.977},"last":1.0429999999999999},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.93999999999999995,"15min":2.1120000000000001},"min":{"1min":0.57899999999999996,"5min":0.54000000000000004,"15min":0.26800000000000002},"max":{"1min":1.1719999999999999,"5min":1.351,"15min":122.13200000000001},"last":1.0860000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:00:31 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.97299999999999998,"15min":2.25},"min":{"1min":0.65600000000000003,"5min":0.57899999999999996,"15min":0.25900000000000001},"max":{"1min":1.292,"5min":1.5329999999999999,"15min":192.72200000000001},"last":1.286},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.996,"15min":2.0609999999999999},"min":{"1min":0.67400000000000004,"5min":0.66500000000000004,"15min":0.28299999999999997},"max":{"1min":1.2989999999999999,"5min":1.4159999999999999,"15min":192.69300000000001},"last":1.2310000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:00:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.95999999999999996,"15min":1.609},"min":{"1min":0.624,"5min":0.624,"15min":0.312},"max":{"1min":1.2669999999999999,"5min":1.429,"15min":122.02800000000001},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.016,"5min":1.0680000000000001,"15min":1.8799999999999999},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.31900000000000001},"max":{"1min":1.272,"5min":1.7949999999999999,"15min":122.05800000000001},"last":0.93400000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:00:45 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.92100000000000004,"15min":0.93100000000000005},"min":{"1min":0.66800000000000004,"5min":0.59499999999999997,"15min":0.33100000000000002},"max":{"1min":1.157,"5min":1.3360000000000001,"15min":7.2270000000000003},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.04,"5min":1.052,"15min":1.1379999999999999},"min":{"1min":0.82999999999999996,"5min":0.63700000000000001,"15min":0.38},"max":{"1min":1.194,"5min":1.659,"15min":16.201000000000001},"last":0.67100000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:00:31 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.91300000000000003,"15min":1.097},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.309},"max":{"1min":1.1990000000000001,"5min":1.393,"15min":9.4209999999999994},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.044,"5min":1.042,"15min":1.3140000000000001},"min":{"1min":0.68000000000000005,"5min":0.498,"15min":0.318},"max":{"1min":1.4159999999999999,"5min":1.6499999999999999,"15min":16.053000000000001},"last":0.83599999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:00:45 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.026,"15min":1.2869999999999999},"min":{"1min":0.70299999999999996,"5min":0.47099999999999997,"15min":0.308},"max":{"1min":1.2989999999999999,"5min":1.3999999999999999,"15min":9.6270000000000007},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.2350000000000001,"5min":1.242,"15min":1.51},"min":{"1min":1.129,"5min":0.621,"15min":0.34399999999999997},"max":{"1min":1.4770000000000001,"5min":1.954,"15min":16.145},"last":1.1819999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:00:31 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.2210000000000001,"15min":1.2549999999999999},"min":{"1min":0.85499999999999998,"5min":0.57899999999999996,"15min":0.42399999999999999},"max":{"1min":1.5429999999999999,"5min":1.8879999999999999,"15min":16.119},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.131,"15min":1.0409999999999999},"min":{"1min":0.78000000000000003,"5min":0.59999999999999998,"15min":0.34200000000000003},"max":{"1min":1.423,"5min":1.633,"15min":4.6779999999999999},"last":0.98899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7721,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:01:04.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:01:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:04 smithi163 bash[78107]: cluster 2023-10-09T10:01:02.411980+0000 mgr.y (mgr.34107) 1659 : cluster 0 pgmap v2591: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:04.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:04 smithi163 bash[78107]: audit 2023-10-09T10:01:03.089295+0000 mgr.y (mgr.34107) 1660 : audit 0 from='client.36972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:01:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:04 smithi163 bash[75425]: cluster 2023-10-09T10:01:02.411980+0000 mgr.y (mgr.34107) 1659 : cluster 0 pgmap v2591: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:04.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:04 smithi163 bash[75425]: audit 2023-10-09T10:01:03.089295+0000 mgr.y (mgr.34107) 1660 : audit 0 from='client.36972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:01:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:04 smithi181 bash[59345]: cluster 2023-10-09T10:01:02.411980+0000 mgr.y (mgr.34107) 1659 : cluster 0 pgmap v2591: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:04.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:04 smithi181 bash[59345]: audit 2023-10-09T10:01:03.089295+0000 mgr.y (mgr.34107) 1660 : audit 0 from='client.36972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:01:06.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:06 smithi163 bash[78107]: cluster 2023-10-09T10:01:04.412825+0000 mgr.y (mgr.34107) 1661 : cluster 0 pgmap v2592: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:06.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:06 smithi163 bash[75425]: cluster 2023-10-09T10:01:04.412825+0000 mgr.y (mgr.34107) 1661 : cluster 0 pgmap v2592: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:06.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:06 smithi181 bash[59345]: cluster 2023-10-09T10:01:04.412825+0000 mgr.y (mgr.34107) 1661 : cluster 0 pgmap v2592: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:08.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:08 smithi163 bash[75425]: cluster 2023-10-09T10:01:06.413871+0000 mgr.y (mgr.34107) 1662 : cluster 0 pgmap v2593: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:08.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:08 smithi163 bash[78107]: cluster 2023-10-09T10:01:06.413871+0000 mgr.y (mgr.34107) 1662 : cluster 0 pgmap v2593: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:08.566 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:08 smithi181 bash[59345]: cluster 2023-10-09T10:01:06.413871+0000 mgr.y (mgr.34107) 1662 : cluster 0 pgmap v2593: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:08.812 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:10.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:10 smithi163 bash[78107]: cluster 2023-10-09T10:01:08.414689+0000 mgr.y (mgr.34107) 1663 : cluster 0 pgmap v2594: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:10.469 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:10 smithi163 bash[75425]: cluster 2023-10-09T10:01:08.414689+0000 mgr.y (mgr.34107) 1663 : cluster 0 pgmap v2594: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:10.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:10 smithi181 bash[59345]: cluster 2023-10-09T10:01:08.414689+0000 mgr.y (mgr.34107) 1663 : cluster 0 pgmap v2594: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:11.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:11 smithi163 bash[78107]: audit 2023-10-09T10:01:10.429713+0000 mon.a (mon.0) 2793 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:11.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:11 smithi163 bash[75425]: audit 2023-10-09T10:01:10.429713+0000 mon.a (mon.0) 2793 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:11.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:11 smithi181 bash[59345]: audit 2023-10-09T10:01:10.429713+0000 mon.a (mon.0) 2793 : audit 0 from='mgr.34107 172.21.15.163:0/3155936245' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:12.333 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T10:01:12.335 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T10:01:12.393 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:12 smithi163 bash[78107]: cluster 2023-10-09T10:01:10.415620+0000 mgr.y (mgr.34107) 1664 : cluster 0 pgmap v2595: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:12.394 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:12 smithi163 bash[75425]: cluster 2023-10-09T10:01:10.415620+0000 mgr.y (mgr.34107) 1664 : cluster 0 pgmap v2595: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:12.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:12 smithi181 bash[59345]: cluster 2023-10-09T10:01:10.415620+0000 mgr.y (mgr.34107) 1664 : cluster 0 pgmap v2595: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T10:01:13.313 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":2595,"stamp":"2023-10-09T10:01:10.415048+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6277,"num_read_kb":12713,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":5382450,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5704208,"kb_used_data":68496,"kb_used_omap":662,"kb_used_meta":5634985,"kb_avail":744257008,"statfs":{"total":767960285184,"available":762119176192,"internally_reserved":0,"allocated":70139904,"data_stored":62057704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":678391,"internal_metadata":5770225161},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005360"},"pg_stats":[{"pgid":"3.1a","version":"61'1","reported_seq":1221,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935386+0000","last_change":"2023-10-09T09:58:07.126507+0000","last_active":"2023-10-09T09:59:36.935386+0000","last_peered":"2023-10-09T09:59:36.935386+0000","last_clean":"2023-10-09T09:59:36.935386+0000","last_became_active":"2023-10-09T09:58:07.125959+0000","last_became_peered":"2023-10-09T09:58:07.125959+0000","last_unstale":"2023-10-09T09:59:36.935386+0000","last_undegraded":"2023-10-09T09:59:36.935386+0000","last_fullsized":"2023-10-09T09:59:36.935386+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":226,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285221+0000","last_change":"2023-10-09T09:57:47.472431+0000","last_active":"2023-10-09T09:59:47.285221+0000","last_peered":"2023-10-09T09:59:47.285221+0000","last_clean":"2023-10-09T09:59:47.285221+0000","last_became_active":"2023-10-09T09:57:47.472035+0000","last_became_peered":"2023-10-09T09:57:47.472035+0000","last_unstale":"2023-10-09T09:59:47.285221+0000","last_undegraded":"2023-10-09T09:59:47.285221+0000","last_fullsized":"2023-10-09T09:59:47.285221+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:36.597330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1306,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.023676+0000","last_change":"2023-10-09T09:56:40.141288+0000","last_active":"2023-10-09T09:59:48.023676+0000","last_peered":"2023-10-09T09:59:48.023676+0000","last_clean":"2023-10-09T09:59:48.023676+0000","last_became_active":"2023-10-09T09:56:40.140682+0000","last_became_peered":"2023-10-09T09:56:40.140682+0000","last_unstale":"2023-10-09T09:59:48.023676+0000","last_undegraded":"2023-10-09T09:59:48.023676+0000","last_fullsized":"2023-10-09T09:59:48.023676+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:19.241067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1326,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.592234+0000","last_change":"2023-10-09T09:56:57.078210+0000","last_active":"2023-10-09T09:59:36.592234+0000","last_peered":"2023-10-09T09:59:36.592234+0000","last_clean":"2023-10-09T09:59:36.592234+0000","last_became_active":"2023-10-09T09:56:57.077775+0000","last_became_peered":"2023-10-09T09:56:57.077775+0000","last_unstale":"2023-10-09T09:59:36.592234+0000","last_undegraded":"2023-10-09T09:59:36.592234+0000","last_fullsized":"2023-10-09T09:59:36.592234+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:27.531666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":839,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.284899+0000","last_change":"2023-10-09T09:57:47.472986+0000","last_active":"2023-10-09T09:59:47.284899+0000","last_peered":"2023-10-09T09:59:47.284899+0000","last_clean":"2023-10-09T09:59:47.284899+0000","last_became_active":"2023-10-09T09:57:47.472634+0000","last_became_peered":"2023-10-09T09:57:47.472634+0000","last_unstale":"2023-10-09T09:59:47.284899+0000","last_undegraded":"2023-10-09T09:59:47.284899+0000","last_fullsized":"2023-10-09T09:59:47.284899+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:33.049868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1307,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934945+0000","last_change":"2023-10-09T09:48:07.988928+0000","last_active":"2023-10-09T09:59:36.934945+0000","last_peered":"2023-10-09T09:59:36.934945+0000","last_clean":"2023-10-09T09:59:36.934945+0000","last_became_active":"2023-10-09T09:48:07.988051+0000","last_became_peered":"2023-10-09T09:48:07.988051+0000","last_unstale":"2023-10-09T09:59:36.934945+0000","last_undegraded":"2023-10-09T09:59:36.934945+0000","last_fullsized":"2023-10-09T09:59:36.934945+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1253,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.592251+0000","last_change":"2023-10-09T09:59:27.198059+0000","last_active":"2023-10-09T09:59:36.592251+0000","last_peered":"2023-10-09T09:59:36.592251+0000","last_clean":"2023-10-09T09:59:36.592251+0000","last_became_active":"2023-10-09T09:59:27.197563+0000","last_became_peered":"2023-10-09T09:59:27.197563+0000","last_unstale":"2023-10-09T09:59:36.592251+0000","last_undegraded":"2023-10-09T09:59:36.592251+0000","last_fullsized":"2023-10-09T09:59:36.592251+0000","mapping_epoch":1341,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:47.921652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":40,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.181678+0000","last_change":"2023-10-09T09:56:40.139613+0000","last_active":"2023-10-09T09:59:36.181678+0000","last_peered":"2023-10-09T09:59:36.181678+0000","last_clean":"2023-10-09T09:59:36.181678+0000","last_became_active":"2023-10-09T09:56:40.139072+0000","last_became_peered":"2023-10-09T09:56:40.139072+0000","last_unstale":"2023-10-09T09:59:36.181678+0000","last_undegraded":"2023-10-09T09:59:36.181678+0000","last_fullsized":"2023-10-09T09:59:36.181678+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:31.377762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":228,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102536+0000","last_change":"2023-10-09T09:56:57.078220+0000","last_active":"2023-10-09T09:59:36.102536+0000","last_peered":"2023-10-09T09:59:36.102536+0000","last_clean":"2023-10-09T09:59:36.102536+0000","last_became_active":"2023-10-09T09:56:57.077758+0000","last_became_peered":"2023-10-09T09:56:57.077758+0000","last_unstale":"2023-10-09T09:59:36.102536+0000","last_undegraded":"2023-10-09T09:59:36.102536+0000","last_fullsized":"2023-10-09T09:59:36.102536+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":758,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285483+0000","last_change":"2023-10-09T09:57:47.431308+0000","last_active":"2023-10-09T09:59:47.285483+0000","last_peered":"2023-10-09T09:59:47.285483+0000","last_clean":"2023-10-09T09:59:47.285483+0000","last_became_active":"2023-10-09T09:57:47.430812+0000","last_became_peered":"2023-10-09T09:57:47.430812+0000","last_unstale":"2023-10-09T09:59:47.285483+0000","last_undegraded":"2023-10-09T09:59:47.285483+0000","last_fullsized":"2023-10-09T09:59:47.285483+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:05.105709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"61'3","reported_seq":39,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932454+0000","last_change":"2023-10-09T09:56:57.282883+0000","last_active":"2023-10-09T09:59:36.932454+0000","last_peered":"2023-10-09T09:59:36.932454+0000","last_clean":"2023-10-09T09:59:36.932454+0000","last_became_active":"2023-10-09T09:56:57.086116+0000","last_became_peered":"2023-10-09T09:56:57.086116+0000","last_unstale":"2023-10-09T09:59:36.932454+0000","last_undegraded":"2023-10-09T09:59:36.932454+0000","last_fullsized":"2023-10-09T09:59:36.932454+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":176,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102642+0000","last_change":"2023-10-09T09:40:27.026887+0000","last_active":"2023-10-09T09:59:36.102642+0000","last_peered":"2023-10-09T09:59:36.102642+0000","last_clean":"2023-10-09T09:59:36.102642+0000","last_became_active":"2023-10-09T09:40:27.026491+0000","last_became_peered":"2023-10-09T09:40:27.026491+0000","last_unstale":"2023-10-09T09:59:36.102642+0000","last_undegraded":"2023-10-09T09:59:36.102642+0000","last_fullsized":"2023-10-09T09:59:36.102642+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"388'22","reported_seq":1213,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935509+0000","last_change":"2023-10-09T09:58:07.131535+0000","last_active":"2023-10-09T09:59:36.935509+0000","last_peered":"2023-10-09T09:59:36.935509+0000","last_clean":"2023-10-09T09:59:36.935509+0000","last_became_active":"2023-10-09T09:58:07.131120+0000","last_became_peered":"2023-10-09T09:58:07.131120+0000","last_unstale":"2023-10-09T09:59:36.935509+0000","last_undegraded":"2023-10-09T09:59:36.935509+0000","last_fullsized":"2023-10-09T09:59:36.935509+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1186,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935443+0000","last_change":"2023-10-09T09:58:07.130871+0000","last_active":"2023-10-09T09:59:36.935443+0000","last_peered":"2023-10-09T09:59:36.935443+0000","last_clean":"2023-10-09T09:59:36.935443+0000","last_became_active":"2023-10-09T09:58:07.130302+0000","last_became_peered":"2023-10-09T09:58:07.130302+0000","last_unstale":"2023-10-09T09:59:36.935443+0000","last_undegraded":"2023-10-09T09:59:36.935443+0000","last_fullsized":"2023-10-09T09:59:36.935443+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"61'2","reported_seq":39,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932100+0000","last_change":"2023-10-09T09:56:57.271889+0000","last_active":"2023-10-09T09:59:36.932100+0000","last_peered":"2023-10-09T09:59:36.932100+0000","last_clean":"2023-10-09T09:59:36.932100+0000","last_became_active":"2023-10-09T09:56:57.085853+0000","last_became_peered":"2023-10-09T09:56:57.085853+0000","last_unstale":"2023-10-09T09:59:36.932100+0000","last_undegraded":"2023-10-09T09:59:36.932100+0000","last_fullsized":"2023-10-09T09:59:36.932100+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":836,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285276+0000","last_change":"2023-10-09T09:57:47.473231+0000","last_active":"2023-10-09T09:59:47.285276+0000","last_peered":"2023-10-09T09:59:47.285276+0000","last_clean":"2023-10-09T09:59:47.285276+0000","last_became_active":"2023-10-09T09:57:47.472433+0000","last_became_peered":"2023-10-09T09:57:47.472433+0000","last_unstale":"2023-10-09T09:59:47.285276+0000","last_undegraded":"2023-10-09T09:59:47.285276+0000","last_fullsized":"2023-10-09T09:59:47.285276+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:42.280204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1229,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.592004+0000","last_change":"2023-10-09T09:59:27.201093+0000","last_active":"2023-10-09T09:59:36.592004+0000","last_peered":"2023-10-09T09:59:36.592004+0000","last_clean":"2023-10-09T09:59:36.592004+0000","last_became_active":"2023-10-09T09:59:27.200780+0000","last_became_peered":"2023-10-09T09:59:27.200780+0000","last_unstale":"2023-10-09T09:59:36.592004+0000","last_undegraded":"2023-10-09T09:59:36.592004+0000","last_fullsized":"2023-10-09T09:59:36.592004+0000","mapping_epoch":1341,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:03:31.399625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":666,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102642+0000","last_change":"2023-10-09T09:58:27.454413+0000","last_active":"2023-10-09T09:59:36.102642+0000","last_peered":"2023-10-09T09:59:36.102642+0000","last_clean":"2023-10-09T09:59:36.102642+0000","last_became_active":"2023-10-09T09:58:27.453999+0000","last_became_peered":"2023-10-09T09:58:27.453999+0000","last_unstale":"2023-10-09T09:59:36.102642+0000","last_undegraded":"2023-10-09T09:59:36.102642+0000","last_fullsized":"2023-10-09T09:59:36.102642+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":33,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.931710+0000","last_change":"2023-10-09T09:56:57.084062+0000","last_active":"2023-10-09T09:59:36.931710+0000","last_peered":"2023-10-09T09:59:36.931710+0000","last_clean":"2023-10-09T09:59:36.931710+0000","last_became_active":"2023-10-09T09:56:57.083675+0000","last_became_peered":"2023-10-09T09:56:57.083675+0000","last_unstale":"2023-10-09T09:59:36.931710+0000","last_undegraded":"2023-10-09T09:59:36.931710+0000","last_fullsized":"2023-10-09T09:59:36.931710+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1249,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.592149+0000","last_change":"2023-10-09T09:59:27.201410+0000","last_active":"2023-10-09T09:59:36.592149+0000","last_peered":"2023-10-09T09:59:36.592149+0000","last_clean":"2023-10-09T09:59:36.592149+0000","last_became_active":"2023-10-09T09:59:27.201029+0000","last_became_peered":"2023-10-09T09:59:27.201029+0000","last_unstale":"2023-10-09T09:59:36.592149+0000","last_undegraded":"2023-10-09T09:59:36.592149+0000","last_fullsized":"2023-10-09T09:59:36.592149+0000","mapping_epoch":1341,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:31:50.223832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"61'8","reported_seq":1337,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934302+0000","last_change":"2023-10-09T09:40:25.810748+0000","last_active":"2023-10-09T09:59:36.934302+0000","last_peered":"2023-10-09T09:59:36.934302+0000","last_clean":"2023-10-09T09:59:36.934302+0000","last_became_active":"2023-10-09T09:40:25.798904+0000","last_became_peered":"2023-10-09T09:40:25.798904+0000","last_unstale":"2023-10-09T09:59:36.934302+0000","last_undegraded":"2023-10-09T09:59:36.934302+0000","last_fullsized":"2023-10-09T09:59:36.934302+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":49,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.182799+0000","last_change":"2023-10-09T09:56:57.087457+0000","last_active":"2023-10-09T09:59:36.182799+0000","last_peered":"2023-10-09T09:59:36.182799+0000","last_clean":"2023-10-09T09:59:36.182799+0000","last_became_active":"2023-10-09T09:56:57.087090+0000","last_became_peered":"2023-10-09T09:56:57.087090+0000","last_unstale":"2023-10-09T09:59:36.182799+0000","last_undegraded":"2023-10-09T09:59:36.182799+0000","last_fullsized":"2023-10-09T09:59:36.182799+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:12:57.268609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"61'1","reported_seq":37,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932864+0000","last_change":"2023-10-09T09:56:57.278364+0000","last_active":"2023-10-09T09:59:36.932864+0000","last_peered":"2023-10-09T09:59:36.932864+0000","last_clean":"2023-10-09T09:59:36.932864+0000","last_became_active":"2023-10-09T09:56:57.085732+0000","last_became_peered":"2023-10-09T09:56:57.085732+0000","last_unstale":"2023-10-09T09:59:36.932864+0000","last_undegraded":"2023-10-09T09:59:36.932864+0000","last_fullsized":"2023-10-09T09:59:36.932864+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":195,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102323+0000","last_change":"2023-10-09T09:56:57.078240+0000","last_active":"2023-10-09T09:59:36.102323+0000","last_peered":"2023-10-09T09:59:36.102323+0000","last_clean":"2023-10-09T09:59:36.102323+0000","last_became_active":"2023-10-09T09:56:57.077813+0000","last_became_peered":"2023-10-09T09:56:57.077813+0000","last_unstale":"2023-10-09T09:59:36.102323+0000","last_undegraded":"2023-10-09T09:59:36.102323+0000","last_fullsized":"2023-10-09T09:59:36.102323+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"61'2","reported_seq":37,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932515+0000","last_change":"2023-10-09T09:56:57.285622+0000","last_active":"2023-10-09T09:59:36.932515+0000","last_peered":"2023-10-09T09:59:36.932515+0000","last_clean":"2023-10-09T09:59:36.932515+0000","last_became_active":"2023-10-09T09:56:57.087847+0000","last_became_peered":"2023-10-09T09:56:57.087847+0000","last_unstale":"2023-10-09T09:59:36.932515+0000","last_undegraded":"2023-10-09T09:59:36.932515+0000","last_fullsized":"2023-10-09T09:59:36.932515+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1274,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935802+0000","last_change":"2023-10-09T09:58:07.778049+0000","last_active":"2023-10-09T09:59:36.935802+0000","last_peered":"2023-10-09T09:59:36.935802+0000","last_clean":"2023-10-09T09:59:36.935802+0000","last_became_active":"2023-10-09T09:58:07.777129+0000","last_became_peered":"2023-10-09T09:58:07.777129+0000","last_unstale":"2023-10-09T09:59:36.935802+0000","last_undegraded":"2023-10-09T09:59:36.935802+0000","last_fullsized":"2023-10-09T09:59:36.935802+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"61'1","reported_seq":37,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932837+0000","last_change":"2023-10-09T09:56:57.092620+0000","last_active":"2023-10-09T09:59:36.932837+0000","last_peered":"2023-10-09T09:59:36.932837+0000","last_clean":"2023-10-09T09:59:36.932837+0000","last_became_active":"2023-10-09T09:56:57.083126+0000","last_became_peered":"2023-10-09T09:56:57.083126+0000","last_unstale":"2023-10-09T09:59:36.932837+0000","last_undegraded":"2023-10-09T09:59:36.932837+0000","last_fullsized":"2023-10-09T09:59:36.932837+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1308,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934518+0000","last_change":"2023-10-09T09:56:57.081279+0000","last_active":"2023-10-09T09:59:36.934518+0000","last_peered":"2023-10-09T09:59:36.934518+0000","last_clean":"2023-10-09T09:59:36.934518+0000","last_became_active":"2023-10-09T09:56:57.080887+0000","last_became_peered":"2023-10-09T09:56:57.080887+0000","last_unstale":"2023-10-09T09:59:36.934518+0000","last_undegraded":"2023-10-09T09:59:36.934518+0000","last_fullsized":"2023-10-09T09:59:36.934518+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":840,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102596+0000","last_change":"2023-10-09T09:58:27.455524+0000","last_active":"2023-10-09T09:59:36.102596+0000","last_peered":"2023-10-09T09:59:36.102596+0000","last_clean":"2023-10-09T09:59:36.102596+0000","last_became_active":"2023-10-09T09:58:27.455105+0000","last_became_peered":"2023-10-09T09:58:27.455105+0000","last_unstale":"2023-10-09T09:59:36.102596+0000","last_undegraded":"2023-10-09T09:59:36.102596+0000","last_fullsized":"2023-10-09T09:59:36.102596+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1267,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.571749+0000","last_change":"2023-10-09T09:46:27.004258+0000","last_active":"2023-10-09T09:59:47.571749+0000","last_peered":"2023-10-09T09:59:47.571749+0000","last_clean":"2023-10-09T09:59:47.571749+0000","last_became_active":"2023-10-09T09:46:27.003708+0000","last_became_peered":"2023-10-09T09:46:27.003708+0000","last_unstale":"2023-10-09T09:59:47.571749+0000","last_undegraded":"2023-10-09T09:59:47.571749+0000","last_fullsized":"2023-10-09T09:59:47.571749+0000","mapping_epoch":1237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:22.221059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"61'2","reported_seq":45,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285101+0000","last_change":"2023-10-09T09:56:40.145464+0000","last_active":"2023-10-09T09:59:47.285101+0000","last_peered":"2023-10-09T09:59:47.285101+0000","last_clean":"2023-10-09T09:59:47.285101+0000","last_became_active":"2023-10-09T09:56:40.128235+0000","last_became_peered":"2023-10-09T09:56:40.128235+0000","last_unstale":"2023-10-09T09:59:47.285101+0000","last_undegraded":"2023-10-09T09:59:47.285101+0000","last_fullsized":"2023-10-09T09:59:47.285101+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:03.254704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1293,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935532+0000","last_change":"2023-10-09T09:58:07.778084+0000","last_active":"2023-10-09T09:59:36.935532+0000","last_peered":"2023-10-09T09:59:36.935532+0000","last_clean":"2023-10-09T09:59:36.935532+0000","last_became_active":"2023-10-09T09:58:07.777094+0000","last_became_peered":"2023-10-09T09:58:07.777094+0000","last_unstale":"2023-10-09T09:59:36.935532+0000","last_undegraded":"2023-10-09T09:59:36.935532+0000","last_fullsized":"2023-10-09T09:59:36.935532+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1291,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934164+0000","last_change":"2023-10-09T09:48:07.988586+0000","last_active":"2023-10-09T09:59:36.934164+0000","last_peered":"2023-10-09T09:59:36.934164+0000","last_clean":"2023-10-09T09:59:36.934164+0000","last_became_active":"2023-10-09T09:48:07.988226+0000","last_became_peered":"2023-10-09T09:48:07.988226+0000","last_unstale":"2023-10-09T09:59:36.934164+0000","last_undegraded":"2023-10-09T09:59:36.934164+0000","last_fullsized":"2023-10-09T09:59:36.934164+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1219,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935623+0000","last_change":"2023-10-09T09:58:07.126784+0000","last_active":"2023-10-09T09:59:36.935623+0000","last_peered":"2023-10-09T09:59:36.935623+0000","last_clean":"2023-10-09T09:59:36.935623+0000","last_became_active":"2023-10-09T09:58:07.126143+0000","last_became_peered":"2023-10-09T09:58:07.126143+0000","last_unstale":"2023-10-09T09:59:36.935623+0000","last_undegraded":"2023-10-09T09:59:36.935623+0000","last_fullsized":"2023-10-09T09:59:36.935623+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1314,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935228+0000","last_change":"2023-10-09T09:58:07.126735+0000","last_active":"2023-10-09T09:59:36.935228+0000","last_peered":"2023-10-09T09:59:36.935228+0000","last_clean":"2023-10-09T09:59:36.935228+0000","last_became_active":"2023-10-09T09:58:07.126401+0000","last_became_peered":"2023-10-09T09:58:07.126401+0000","last_unstale":"2023-10-09T09:59:36.935228+0000","last_undegraded":"2023-10-09T09:59:36.935228+0000","last_fullsized":"2023-10-09T09:59:36.935228+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1186,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.936139+0000","last_change":"2023-10-09T09:58:07.130343+0000","last_active":"2023-10-09T09:59:36.936139+0000","last_peered":"2023-10-09T09:59:36.936139+0000","last_clean":"2023-10-09T09:59:36.936139+0000","last_became_active":"2023-10-09T09:58:07.129931+0000","last_became_peered":"2023-10-09T09:58:07.129931+0000","last_unstale":"2023-10-09T09:59:36.936139+0000","last_undegraded":"2023-10-09T09:59:36.936139+0000","last_fullsized":"2023-10-09T09:59:36.936139+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":50,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.284922+0000","last_change":"2023-10-09T09:56:57.087804+0000","last_active":"2023-10-09T09:59:47.284922+0000","last_peered":"2023-10-09T09:59:47.284922+0000","last_clean":"2023-10-09T09:59:47.284922+0000","last_became_active":"2023-10-09T09:56:57.087423+0000","last_became_peered":"2023-10-09T09:56:57.087423+0000","last_unstale":"2023-10-09T09:59:47.284922+0000","last_undegraded":"2023-10-09T09:59:47.284922+0000","last_fullsized":"2023-10-09T09:59:47.284922+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:17.120194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":202,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.166514+0000","last_change":"2023-10-09T09:56:40.127642+0000","last_active":"2023-10-09T09:59:47.166514+0000","last_peered":"2023-10-09T09:59:47.166514+0000","last_clean":"2023-10-09T09:59:47.166514+0000","last_became_active":"2023-10-09T09:56:40.127292+0000","last_became_peered":"2023-10-09T09:56:40.127292+0000","last_unstale":"2023-10-09T09:59:47.166514+0000","last_undegraded":"2023-10-09T09:59:47.166514+0000","last_fullsized":"2023-10-09T09:59:47.166514+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:44.501837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"1311'169","reported_seq":1943,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.024707+0000","last_change":"2023-10-09T09:57:00.111698+0000","last_active":"2023-10-09T09:59:48.024707+0000","last_peered":"2023-10-09T09:59:48.024707+0000","last_clean":"2023-10-09T09:59:48.024707+0000","last_became_active":"2023-10-09T09:57:00.110919+0000","last_became_peered":"2023-10-09T09:57:00.110919+0000","last_unstale":"2023-10-09T09:59:48.024707+0000","last_undegraded":"2023-10-09T09:59:48.024707+0000","last_fullsized":"2023-10-09T09:59:48.024707+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":169,"log_dups_size":0,"ondisk_log_size":169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:55.189795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001668896,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1002,"num_read_kb":2031,"num_write":171,"num_write_kb":5258,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":195,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102474+0000","last_change":"2023-10-09T09:56:40.128833+0000","last_active":"2023-10-09T09:59:36.102474+0000","last_peered":"2023-10-09T09:59:36.102474+0000","last_clean":"2023-10-09T09:59:36.102474+0000","last_became_active":"2023-10-09T09:56:40.128373+0000","last_became_peered":"2023-10-09T09:56:40.128373+0000","last_unstale":"2023-10-09T09:59:36.102474+0000","last_undegraded":"2023-10-09T09:59:36.102474+0000","last_fullsized":"2023-10-09T09:59:36.102474+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1280,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.023632+0000","last_change":"2023-10-09T09:48:07.977683+0000","last_active":"2023-10-09T09:59:48.023632+0000","last_peered":"2023-10-09T09:59:48.023632+0000","last_clean":"2023-10-09T09:59:48.023632+0000","last_became_active":"2023-10-09T09:48:07.977024+0000","last_became_peered":"2023-10-09T09:48:07.977024+0000","last_unstale":"2023-10-09T09:59:48.023632+0000","last_undegraded":"2023-10-09T09:59:48.023632+0000","last_fullsized":"2023-10-09T09:59:48.023632+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:13.438431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"1311'247","reported_seq":1811,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102793+0000","last_change":"2023-10-09T09:58:27.455629+0000","last_active":"2023-10-09T09:59:36.102793+0000","last_peered":"2023-10-09T09:59:36.102793+0000","last_clean":"2023-10-09T09:59:36.102793+0000","last_became_active":"2023-10-09T09:58:27.455003+0000","last_became_peered":"2023-10-09T09:58:27.455003+0000","last_unstale":"2023-10-09T09:59:36.102793+0000","last_undegraded":"2023-10-09T09:59:36.102793+0000","last_fullsized":"2023-10-09T09:59:36.102793+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":247,"log_dups_size":0,"ondisk_log_size":247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1714,"num_read_kb":1846,"num_write":205,"num_write_kb":259,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1202,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935415+0000","last_change":"2023-10-09T09:58:07.605971+0000","last_active":"2023-10-09T09:59:36.935415+0000","last_peered":"2023-10-09T09:59:36.935415+0000","last_clean":"2023-10-09T09:59:36.935415+0000","last_became_active":"2023-10-09T09:58:07.605139+0000","last_became_peered":"2023-10-09T09:58:07.605139+0000","last_unstale":"2023-10-09T09:59:36.935415+0000","last_undegraded":"2023-10-09T09:59:36.935415+0000","last_fullsized":"2023-10-09T09:59:36.935415+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1342,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.024423+0000","last_change":"2023-10-09T09:56:40.142238+0000","last_active":"2023-10-09T09:59:48.024423+0000","last_peered":"2023-10-09T09:59:48.024423+0000","last_clean":"2023-10-09T09:59:48.024423+0000","last_became_active":"2023-10-09T09:56:40.141699+0000","last_became_peered":"2023-10-09T09:56:40.141699+0000","last_unstale":"2023-10-09T09:59:48.024423+0000","last_undegraded":"2023-10-09T09:59:48.024423+0000","last_fullsized":"2023-10-09T09:59:48.024423+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:27.595630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"1311'103","reported_seq":747,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:41.182731+0000","last_change":"2023-10-09T09:56:43.143931+0000","last_active":"2023-10-09T09:59:41.182731+0000","last_peered":"2023-10-09T09:59:41.182731+0000","last_clean":"2023-10-09T09:59:41.182731+0000","last_became_active":"2023-10-09T09:56:43.143196+0000","last_became_peered":"2023-10-09T09:56:43.143196+0000","last_unstale":"2023-10-09T09:59:41.182731+0000","last_undegraded":"2023-10-09T09:59:41.182731+0000","last_fullsized":"2023-10-09T09:59:41.182731+0000","mapping_epoch":1318,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:18.533297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019660929999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":1020,"num_write":78,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1311,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.023805+0000","last_change":"2023-10-09T09:48:07.998234+0000","last_active":"2023-10-09T09:59:48.023805+0000","last_peered":"2023-10-09T09:59:48.023805+0000","last_clean":"2023-10-09T09:59:48.023805+0000","last_became_active":"2023-10-09T09:48:07.997909+0000","last_became_peered":"2023-10-09T09:48:07.997909+0000","last_unstale":"2023-10-09T09:59:48.023805+0000","last_undegraded":"2023-10-09T09:59:48.023805+0000","last_fullsized":"2023-10-09T09:59:48.023805+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:21.750617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1300,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.024811+0000","last_change":"2023-10-09T09:48:07.973284+0000","last_active":"2023-10-09T09:59:48.024811+0000","last_peered":"2023-10-09T09:59:48.024811+0000","last_clean":"2023-10-09T09:59:48.024811+0000","last_became_active":"2023-10-09T09:48:07.971235+0000","last_became_peered":"2023-10-09T09:48:07.971235+0000","last_unstale":"2023-10-09T09:59:48.024811+0000","last_undegraded":"2023-10-09T09:59:48.024811+0000","last_fullsized":"2023-10-09T09:59:48.024811+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:40.951420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"1311'146","reported_seq":1684,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102458+0000","last_change":"2023-10-09T09:58:27.454543+0000","last_active":"2023-10-09T09:59:36.102458+0000","last_peered":"2023-10-09T09:59:36.102458+0000","last_clean":"2023-10-09T09:59:36.102458+0000","last_became_active":"2023-10-09T09:58:27.454003+0000","last_became_peered":"2023-10-09T09:58:27.454003+0000","last_unstale":"2023-10-09T09:59:36.102458+0000","last_undegraded":"2023-10-09T09:59:36.102458+0000","last_fullsized":"2023-10-09T09:59:36.102458+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":146,"log_dups_size":0,"ondisk_log_size":146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":398,"num_read_kb":2392,"num_write":143,"num_write_kb":2132,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"304'139","reported_seq":698,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.931691+0000","last_change":"2023-10-09T09:57:00.113166+0000","last_active":"2023-10-09T09:59:36.931691+0000","last_peered":"2023-10-09T09:59:36.931691+0000","last_clean":"2023-10-09T09:59:36.931691+0000","last_became_active":"2023-10-09T09:57:00.112533+0000","last_became_peered":"2023-10-09T09:57:00.112533+0000","last_unstale":"2023-10-09T09:59:36.931691+0000","last_undegraded":"2023-10-09T09:59:36.931691+0000","last_fullsized":"2023-10-09T09:59:36.931691+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T08:57:13.839388+0000","last_clean_scrub_stamp":"2023-10-09T08:57:13.839388+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":186,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285246+0000","last_change":"2023-10-09T09:57:47.431484+0000","last_active":"2023-10-09T09:59:47.285246+0000","last_peered":"2023-10-09T09:59:47.285246+0000","last_clean":"2023-10-09T09:59:47.285246+0000","last_became_active":"2023-10-09T09:57:47.431063+0000","last_became_peered":"2023-10-09T09:57:47.431063+0000","last_unstale":"2023-10-09T09:59:47.285246+0000","last_undegraded":"2023-10-09T09:59:47.285246+0000","last_fullsized":"2023-10-09T09:59:47.285246+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:01:28.903833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1278,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934641+0000","last_change":"2023-10-09T09:56:57.087442+0000","last_active":"2023-10-09T09:59:36.934641+0000","last_peered":"2023-10-09T09:59:36.934641+0000","last_clean":"2023-10-09T09:59:36.934641+0000","last_became_active":"2023-10-09T09:56:57.087063+0000","last_became_peered":"2023-10-09T09:56:57.087063+0000","last_unstale":"2023-10-09T09:59:36.934641+0000","last_undegraded":"2023-10-09T09:59:36.934641+0000","last_fullsized":"2023-10-09T09:59:36.934641+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"1311'48","reported_seq":52,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.931663+0000","last_change":"2023-10-09T09:56:57.291785+0000","last_active":"2023-10-09T09:59:36.931663+0000","last_peered":"2023-10-09T09:59:36.931663+0000","last_clean":"2023-10-09T09:59:36.931663+0000","last_became_active":"2023-10-09T09:56:57.078576+0000","last_became_peered":"2023-10-09T09:56:57.078576+0000","last_unstale":"2023-10-09T09:59:36.931663+0000","last_undegraded":"2023-10-09T09:59:36.931663+0000","last_fullsized":"2023-10-09T09:59:36.931663+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":48,"log_dups_size":0,"ondisk_log_size":48,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":268,"num_read_kb":1552,"num_write":41,"num_write_kb":2215,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1294,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.570903+0000","last_change":"2023-10-09T09:56:40.130793+0000","last_active":"2023-10-09T09:59:47.570903+0000","last_peered":"2023-10-09T09:59:47.570903+0000","last_clean":"2023-10-09T09:59:47.570903+0000","last_became_active":"2023-10-09T09:56:40.130419+0000","last_became_peered":"2023-10-09T09:56:40.130419+0000","last_unstale":"2023-10-09T09:59:47.570903+0000","last_undegraded":"2023-10-09T09:59:47.570903+0000","last_fullsized":"2023-10-09T09:59:47.570903+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:50.436130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1296,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.591539+0000","last_change":"2023-10-09T09:48:07.981410+0000","last_active":"2023-10-09T09:59:36.591539+0000","last_peered":"2023-10-09T09:59:36.591539+0000","last_clean":"2023-10-09T09:59:36.591539+0000","last_became_active":"2023-10-09T09:48:07.979850+0000","last_became_peered":"2023-10-09T09:48:07.979850+0000","last_unstale":"2023-10-09T09:59:36.591539+0000","last_undegraded":"2023-10-09T09:59:36.591539+0000","last_fullsized":"2023-10-09T09:59:36.591539+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:31.201104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"1311'131","reported_seq":1848,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.932450+0000","last_change":"2023-10-09T09:57:00.115713+0000","last_active":"2023-10-09T09:59:36.932450+0000","last_peered":"2023-10-09T09:59:36.932450+0000","last_clean":"2023-10-09T09:59:36.932450+0000","last_became_active":"2023-10-09T09:57:00.114879+0000","last_became_peered":"2023-10-09T09:57:00.114879+0000","last_unstale":"2023-10-09T09:59:36.932450+0000","last_undegraded":"2023-10-09T09:59:36.932450+0000","last_fullsized":"2023-10-09T09:59:36.932450+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1092,"num_read_kb":2605,"num_write":104,"num_write_kb":2443,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1314,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.024623+0000","last_change":"2023-10-09T09:48:07.983773+0000","last_active":"2023-10-09T09:59:48.024623+0000","last_peered":"2023-10-09T09:59:48.024623+0000","last_clean":"2023-10-09T09:59:48.024623+0000","last_became_active":"2023-10-09T09:48:07.976869+0000","last_became_peered":"2023-10-09T09:48:07.976869+0000","last_unstale":"2023-10-09T09:59:48.024623+0000","last_undegraded":"2023-10-09T09:59:48.024623+0000","last_fullsized":"2023-10-09T09:59:48.024623+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:58.173236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1251,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.592089+0000","last_change":"2023-10-09T09:59:27.201228+0000","last_active":"2023-10-09T09:59:36.592089+0000","last_peered":"2023-10-09T09:59:36.592089+0000","last_clean":"2023-10-09T09:59:36.592089+0000","last_became_active":"2023-10-09T09:59:27.200884+0000","last_became_peered":"2023-10-09T09:59:27.200884+0000","last_unstale":"2023-10-09T09:59:36.592089+0000","last_undegraded":"2023-10-09T09:59:36.592089+0000","last_fullsized":"2023-10-09T09:59:36.592089+0000","mapping_epoch":1341,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:12.022663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"61'2","reported_seq":208,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285557+0000","last_change":"2023-10-09T09:57:47.431776+0000","last_active":"2023-10-09T09:59:47.285557+0000","last_peered":"2023-10-09T09:59:47.285557+0000","last_clean":"2023-10-09T09:59:47.285557+0000","last_became_active":"2023-10-09T09:57:47.431275+0000","last_became_peered":"2023-10-09T09:57:47.431275+0000","last_unstale":"2023-10-09T09:59:47.285557+0000","last_undegraded":"2023-10-09T09:59:47.285557+0000","last_fullsized":"2023-10-09T09:59:47.285557+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:39.717442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"1311'57","reported_seq":1487,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:48.024891+0000","last_change":"2023-10-09T09:56:05.154277+0000","last_active":"2023-10-09T09:59:48.024891+0000","last_peered":"2023-10-09T09:59:48.024891+0000","last_clean":"2023-10-09T09:59:48.024891+0000","last_became_active":"2023-10-09T09:48:07.998606+0000","last_became_peered":"2023-10-09T09:48:07.998606+0000","last_unstale":"2023-10-09T09:59:48.024891+0000","last_undegraded":"2023-10-09T09:59:48.024891+0000","last_fullsized":"2023-10-09T09:59:48.024891+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":57,"log_dups_size":0,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.469476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0028883569999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":282,"num_read_kb":223,"num_write":26,"num_write_kb":22,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1200,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.284819+0000","last_change":"2023-10-09T09:57:47.433836+0000","last_active":"2023-10-09T09:59:47.284819+0000","last_peered":"2023-10-09T09:59:47.284819+0000","last_clean":"2023-10-09T09:59:47.284819+0000","last_became_active":"2023-10-09T09:57:47.432833+0000","last_became_peered":"2023-10-09T09:57:47.432833+0000","last_unstale":"2023-10-09T09:59:47.284819+0000","last_undegraded":"2023-10-09T09:59:47.284819+0000","last_fullsized":"2023-10-09T09:59:47.284819+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:18.447204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"1311'75","reported_seq":1621,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.934998+0000","last_change":"2023-10-09T09:56:57.097915+0000","last_active":"2023-10-09T09:59:36.934998+0000","last_peered":"2023-10-09T09:59:36.934998+0000","last_clean":"2023-10-09T09:59:36.934998+0000","last_became_active":"2023-10-09T09:56:57.081618+0000","last_became_peered":"2023-10-09T09:56:57.081618+0000","last_unstale":"2023-10-09T09:59:36.934998+0000","last_undegraded":"2023-10-09T09:59:36.934998+0000","last_fullsized":"2023-10-09T09:59:36.934998+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:01:33.347260+0000","last_clean_scrub_stamp":"2023-10-09T09:01:33.347260+0000","objects_scrubbed":0,"log_size":75,"log_dups_size":0,"ondisk_log_size":75,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":424,"num_read_kb":544,"num_write":78,"num_write_kb":145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"61'1","reported_seq":1277,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.591576+0000","last_change":"2023-10-09T09:48:07.980600+0000","last_active":"2023-10-09T09:59:36.591576+0000","last_peered":"2023-10-09T09:59:36.591576+0000","last_clean":"2023-10-09T09:59:36.591576+0000","last_became_active":"2023-10-09T09:48:07.979544+0000","last_became_peered":"2023-10-09T09:48:07.979544+0000","last_unstale":"2023-10-09T09:59:36.591576+0000","last_undegraded":"2023-10-09T09:59:36.591576+0000","last_fullsized":"2023-10-09T09:59:36.591576+0000","mapping_epoch":1246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:47.320689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1189,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935365+0000","last_change":"2023-10-09T09:58:07.131154+0000","last_active":"2023-10-09T09:59:36.935365+0000","last_peered":"2023-10-09T09:59:36.935365+0000","last_clean":"2023-10-09T09:59:36.935365+0000","last_became_active":"2023-10-09T09:58:07.130161+0000","last_became_peered":"2023-10-09T09:58:07.130161+0000","last_unstale":"2023-10-09T09:59:36.935365+0000","last_undegraded":"2023-10-09T09:59:36.935365+0000","last_fullsized":"2023-10-09T09:59:36.935365+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"61'1","reported_seq":1294,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.571802+0000","last_change":"2023-10-09T09:56:57.083025+0000","last_active":"2023-10-09T09:59:47.571802+0000","last_peered":"2023-10-09T09:59:47.571802+0000","last_clean":"2023-10-09T09:59:47.571802+0000","last_became_active":"2023-10-09T09:56:57.082456+0000","last_became_peered":"2023-10-09T09:56:57.082456+0000","last_unstale":"2023-10-09T09:59:47.571802+0000","last_undegraded":"2023-10-09T09:59:47.571802+0000","last_fullsized":"2023-10-09T09:59:47.571802+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:35.646438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":245,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.175876+0000","last_change":"2023-10-09T09:56:40.130207+0000","last_active":"2023-10-09T09:59:36.175876+0000","last_peered":"2023-10-09T09:59:36.175876+0000","last_clean":"2023-10-09T09:59:36.175876+0000","last_became_active":"2023-10-09T09:56:40.129857+0000","last_became_peered":"2023-10-09T09:56:40.129857+0000","last_unstale":"2023-10-09T09:59:36.175876+0000","last_undegraded":"2023-10-09T09:59:36.175876+0000","last_fullsized":"2023-10-09T09:59:36.175876+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:02.149845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"61'2","reported_seq":85,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102377+0000","last_change":"2023-10-09T09:58:27.455418+0000","last_active":"2023-10-09T09:59:36.102377+0000","last_peered":"2023-10-09T09:59:36.102377+0000","last_clean":"2023-10-09T09:59:36.102377+0000","last_became_active":"2023-10-09T09:58:27.454696+0000","last_became_peered":"2023-10-09T09:58:27.454696+0000","last_unstale":"2023-10-09T09:59:36.102377+0000","last_undegraded":"2023-10-09T09:59:36.102377+0000","last_fullsized":"2023-10-09T09:59:36.102377+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1276,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935605+0000","last_change":"2023-10-09T09:58:07.778693+0000","last_active":"2023-10-09T09:59:36.935605+0000","last_peered":"2023-10-09T09:59:36.935605+0000","last_clean":"2023-10-09T09:59:36.935605+0000","last_became_active":"2023-10-09T09:58:07.777319+0000","last_became_peered":"2023-10-09T09:58:07.777319+0000","last_unstale":"2023-10-09T09:59:36.935605+0000","last_undegraded":"2023-10-09T09:59:36.935605+0000","last_fullsized":"2023-10-09T09:59:36.935605+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"61'2","reported_seq":75,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.102873+0000","last_change":"2023-10-09T09:58:27.456714+0000","last_active":"2023-10-09T09:59:36.102873+0000","last_peered":"2023-10-09T09:59:36.102873+0000","last_clean":"2023-10-09T09:59:36.102873+0000","last_became_active":"2023-10-09T09:58:27.456216+0000","last_became_peered":"2023-10-09T09:58:27.456216+0000","last_unstale":"2023-10-09T09:59:36.102873+0000","last_undegraded":"2023-10-09T09:59:36.102873+0000","last_fullsized":"2023-10-09T09:59:36.102873+0000","mapping_epoch":1334,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1247,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935383+0000","last_change":"2023-10-09T09:58:07.778929+0000","last_active":"2023-10-09T09:59:36.935383+0000","last_peered":"2023-10-09T09:59:36.935383+0000","last_clean":"2023-10-09T09:59:36.935383+0000","last_became_active":"2023-10-09T09:58:07.777990+0000","last_became_peered":"2023-10-09T09:58:07.777990+0000","last_unstale":"2023-10-09T09:59:36.935383+0000","last_undegraded":"2023-10-09T09:59:36.935383+0000","last_fullsized":"2023-10-09T09:59:36.935383+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"61'1","reported_seq":291,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285453+0000","last_change":"2023-10-09T09:57:47.433889+0000","last_active":"2023-10-09T09:59:47.285453+0000","last_peered":"2023-10-09T09:59:47.285453+0000","last_clean":"2023-10-09T09:59:47.285453+0000","last_became_active":"2023-10-09T09:57:47.433362+0000","last_became_peered":"2023-10-09T09:57:47.433362+0000","last_unstale":"2023-10-09T09:59:47.285453+0000","last_undegraded":"2023-10-09T09:59:47.285453+0000","last_fullsized":"2023-10-09T09:59:47.285453+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:28:05.820832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":220,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.285427+0000","last_change":"2023-10-09T09:57:47.433721+0000","last_active":"2023-10-09T09:59:47.285427+0000","last_peered":"2023-10-09T09:59:47.285427+0000","last_clean":"2023-10-09T09:59:47.285427+0000","last_became_active":"2023-10-09T09:57:47.433352+0000","last_became_peered":"2023-10-09T09:57:47.433352+0000","last_unstale":"2023-10-09T09:59:47.285427+0000","last_undegraded":"2023-10-09T09:59:47.285427+0000","last_fullsized":"2023-10-09T09:59:47.285427+0000","mapping_epoch":1328,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:08.161454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"61'4","reported_seq":1311,"reported_epoch":1344,"state":"active+clean","last_fresh":"2023-10-09T09:59:47.571656+0000","last_change":"2023-10-09T09:40:25.806781+0000","last_active":"2023-10-09T09:59:47.571656+0000","last_peered":"2023-10-09T09:59:47.571656+0000","last_clean":"2023-10-09T09:59:47.571656+0000","last_became_active":"2023-10-09T09:40:25.799085+0000","last_became_peered":"2023-10-09T09:40:25.799085+0000","last_unstale":"2023-10-09T09:59:47.571656+0000","last_undegraded":"2023-10-09T09:59:47.571656+0000","last_fullsized":"2023-10-09T09:59:47.571656+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:23.262803+0000","last_clean_scrub_stamp":"2023-10-09T09:03:23.262803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:03.313485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1186,"reported_epoch":1343,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.935229+0000","last_change":"2023-10-09T09:58:07.128856+0000","last_active":"2023-10-09T09:59:36.935229+0000","last_peered":"2023-10-09T09:59:36.935229+0000","last_clean":"2023-10-09T09:59:36.935229+0000","last_became_active":"2023-10-09T09:58:07.128405+0000","last_became_peered":"2023-10-09T09:58:07.128405+0000","last_unstale":"2023-10-09T09:59:36.935229+0000","last_undegraded":"2023-10-09T09:59:36.935229+0000","last_fullsized":"2023-10-09T09:59:36.935229+0000","mapping_epoch":1331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":1332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:03:24.275886+0000","last_clean_scrub_stamp":"2023-10-09T09:03:24.275886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":7410,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30942,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":12213,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398233,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312040,"kb_used_data":10720,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93433112,"statfs":{"total":95995035648,"available":95675506688,"internally_reserved":0,"allocated":10977280,"data_stored":10630753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.0029999999999999,"15min":1.22},"min":{"1min":0.58799999999999997,"5min":0.53400000000000003,"15min":0.26500000000000001},"max":{"1min":1.2669999999999999,"5min":1.325,"15min":46.389000000000003},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.98999999999999999,"15min":1.2609999999999999},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.28999999999999998},"max":{"1min":1.276,"5min":1.3169999999999999,"15min":46.558},"last":0.95999999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.99399999999999999,"15min":1.2789999999999999},"min":{"1min":0.629,"5min":0.629,"15min":0.29599999999999999},"max":{"1min":1.2629999999999999,"5min":1.5820000000000001,"15min":43.752000000000002},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.92000000000000004,"15min":1.1160000000000001},"min":{"1min":0.754,"5min":0.54800000000000004,"15min":0.307},"max":{"1min":1.1799999999999999,"5min":1.27,"15min":43.503999999999998},"last":0.74099999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:00:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.0840000000000001,"15min":1.944},"min":{"1min":0.69899999999999995,"5min":0.66800000000000004,"15min":0.29099999999999998},"max":{"1min":1.3819999999999999,"5min":1.5329999999999999,"15min":106.124},"last":1.1299999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.032,"15min":1.915},"min":{"1min":0.76100000000000001,"5min":0.60599999999999998,"15min":0.27700000000000002},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":106.188},"last":1.054}]},{"osd":3,"last update":"Mon Oct 9 10:00:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.0049999999999999,"15min":1.0740000000000001},"min":{"1min":0.84299999999999997,"5min":0.56499999999999995,"15min":0.35699999999999998},"max":{"1min":1.3129999999999999,"5min":1.397,"15min":14.98},"last":1.077},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.1160000000000001,"15min":1.109},"min":{"1min":0.83199999999999996,"5min":0.65300000000000002,"15min":0.35399999999999998},"max":{"1min":1.5920000000000001,"5min":1.5920000000000001,"15min":15.018000000000001},"last":0.82799999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:00:33 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.98499999999999999,"15min":1.0289999999999999},"min":{"1min":0.74299999999999999,"5min":0.64100000000000001,"15min":0.27700000000000002},"max":{"1min":1.3320000000000001,"5min":1.381,"15min":5.8479999999999999},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.089,"5min":1.05,"15min":1.022},"min":{"1min":0.83299999999999996,"5min":0.67200000000000004,"15min":0.33500000000000002},"max":{"1min":1.45,"5min":1.5720000000000001,"15min":7.4379999999999997},"last":0.76600000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:00:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.141,"15min":1.157},"min":{"1min":0.83899999999999997,"5min":0.70499999999999996,"15min":0.377},"max":{"1min":1.496,"5min":1.637,"15min":10.378},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.139,"15min":1.042},"min":{"1min":0.84499999999999997,"5min":0.58999999999999997,"15min":0.35299999999999998},"max":{"1min":1.3839999999999999,"5min":1.591,"15min":7.1459999999999999},"last":0.85799999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:00:33 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.1879999999999999,"15min":1.3009999999999999},"min":{"1min":0.80200000000000005,"5min":0.68700000000000006,"15min":0.38},"max":{"1min":1.3839999999999999,"5min":1.5109999999999999,"15min":9.0589999999999993},"last":1.018},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.169,"15min":1.3029999999999999},"min":{"1min":0.86499999999999999,"5min":0.60799999999999998,"15min":0.36499999999999999},"max":{"1min":1.518,"5min":1.6639999999999999,"15min":8.0329999999999995},"last":0.88900000000000001}]}]},{"osd":6,"up_from":43,"seq":184683594466,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907296,"kb_used_data":6616,"kb_used_omap":0,"kb_used_meta":900672,"kb_avail":92837856,"statfs":{"total":95995035648,"available":95065964544,"internally_reserved":0,"allocated":6774784,"data_stored":6436322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922288128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:06 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.90300000000000002,"15min":2.4369999999999998},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.23599999999999999},"max":{"1min":1.415,"5min":1.415,"15min":149.095},"last":1.02},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.022,"15min":2.6040000000000001},"min":{"1min":0.69699999999999995,"5min":0.59899999999999998,"15min":0.28699999999999998},"max":{"1min":1.488,"5min":1.488,"15min":149.17500000000001},"last":1.2749999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:00:27 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.96999999999999997,"15min":1.8009999999999999},"min":{"1min":0.51700000000000002,"5min":0.436,"15min":0.28599999999999998},"max":{"1min":1.327,"5min":1.496,"15min":55.368000000000002},"last":1.2210000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.92900000000000005,"15min":1.7470000000000001},"min":{"1min":0.628,"5min":0.58199999999999996,"15min":0.30499999999999999},"max":{"1min":1.7529999999999999,"5min":1.7529999999999999,"15min":55.404000000000003},"last":0.93100000000000005}]},{"osd":2,"last update":"Mon Oct 9 10:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.94999999999999996,"15min":3.2400000000000002},"min":{"1min":0.70699999999999996,"5min":0.59299999999999997,"15min":0.29799999999999999},"max":{"1min":1.369,"5min":1.369,"15min":206.97200000000001},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.0589999999999999,"15min":3.3929999999999998},"min":{"1min":0.626,"5min":0.55300000000000005,"15min":0.309},"max":{"1min":1.254,"5min":1.4139999999999999,"15min":208.90899999999999},"last":1.151}]},{"osd":3,"last update":"Mon Oct 9 10:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.88800000000000001,"15min":1.069},"min":{"1min":0.60099999999999998,"5min":0.56799999999999995,"15min":0.29099999999999998},"max":{"1min":1.1850000000000001,"5min":1.5109999999999999,"15min":7.9139999999999997},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.99099999999999999,"15min":1.2749999999999999},"min":{"1min":0.748,"5min":0.63700000000000001,"15min":0.34899999999999998},"max":{"1min":1.2689999999999999,"5min":1.645,"15min":11.968999999999999},"last":1.1200000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:00:54 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0329999999999999,"15min":1.341},"min":{"1min":0.70899999999999996,"5min":0.501,"15min":0.28399999999999997},"max":{"1min":1.355,"5min":1.712,"15min":12.148999999999999},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.996,"5min":0.98999999999999999,"15min":1.46},"min":{"1min":0.73499999999999999,"5min":0.57299999999999995,"15min":0.313},"max":{"1min":1.28,"5min":1.554,"15min":12.064},"last":1.0389999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:00:54 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.125,"15min":1.3959999999999999},"min":{"1min":0.71099999999999997,"5min":0.54300000000000004,"15min":0.318},"max":{"1min":1.337,"5min":1.8029999999999999,"15min":10.529},"last":0.99099999999999999},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.113,"15min":1.4510000000000001},"min":{"1min":0.80700000000000005,"5min":0.48299999999999998,"15min":0.34300000000000003},"max":{"1min":1.3500000000000001,"5min":1.881,"15min":12.000999999999999},"last":1.1859999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:00:21 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.1759999999999999,"15min":1.4950000000000001},"min":{"1min":0.94499999999999995,"5min":0.64700000000000002,"15min":0.38500000000000001},"max":{"1min":1.825,"5min":1.825,"15min":11.922000000000001},"last":1.246},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1439999999999999,"15min":1.218},"min":{"1min":0.83599999999999997,"5min":0.496,"15min":0.32700000000000001},"max":{"1min":1.784,"5min":1.784,"15min":7.9359999999999999},"last":1.093}]}]},{"osd":2,"up_from":634,"seq":2723009266220,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":904176,"kb_used_data":7656,"kb_used_omap":122,"kb_used_meta":896389,"kb_avail":92840976,"statfs":{"total":95995035648,"available":95069159424,"internally_reserved":0,"allocated":7839744,"data_stored":6093716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125365,"internal_metadata":917902923},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:10 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.66700000000000004,"15min":3.2789999999999999},"min":{"1min":0.53300000000000003,"5min":0.27900000000000003,"15min":0.221},"max":{"1min":0.88800000000000001,"5min":1.149,"15min":218.30799999999999},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.72899999999999998,"15min":3.411},"min":{"1min":0.51300000000000001,"5min":0.33500000000000002,"15min":0.20799999999999999},"max":{"1min":1.163,"5min":1.2250000000000001,"15min":217.67500000000001},"last":0.77600000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:00:32 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.78000000000000003,"15min":1.8500000000000001},"min":{"1min":0.47499999999999998,"5min":0.374,"15min":0.22},"max":{"1min":1.234,"5min":1.321,"15min":106.089},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.75,"15min":1.819},"min":{"1min":0.372,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.2030000000000001,"5min":1.2809999999999999,"15min":105.571},"last":1.1319999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:00:10 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.77900000000000003,"15min":0.997},"min":{"1min":0.499,"5min":0.315,"15min":0.26500000000000001},"max":{"1min":1.202,"5min":1.202,"15min":8.1289999999999996},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.78000000000000003,"15min":0.91600000000000004},"min":{"1min":0.54800000000000004,"5min":0.33800000000000002,"15min":0.26100000000000001},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":9.2810000000000006},"last":0.65500000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.98999999999999999,"15min":1.4079999999999999},"min":{"1min":0.57099999999999995,"5min":0.55600000000000005,"15min":0.27100000000000002},"max":{"1min":1.244,"5min":1.621,"15min":71.876000000000005},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.98099999999999998,"15min":1.3169999999999999},"min":{"1min":0.65300000000000002,"5min":0.48899999999999999,"15min":0.29499999999999998},"max":{"1min":1.286,"5min":1.615,"15min":9.4640000000000004},"last":1.024}]},{"osd":5,"last update":"Mon Oct 9 10:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.98899999999999999,"15min":1.337},"min":{"1min":0.64800000000000002,"5min":0.436,"15min":0.34100000000000003},"max":{"1min":1.2210000000000001,"5min":1.4690000000000001,"15min":8.9350000000000005},"last":1.069},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.0629999999999999,"15min":1.202},"min":{"1min":0.64700000000000002,"5min":0.50800000000000001,"15min":0.35199999999999998},"max":{"1min":1.2310000000000001,"5min":1.5840000000000001,"15min":8.0109999999999992},"last":1.095}]},{"osd":6,"last update":"Mon Oct 9 10:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.083,"15min":1.663},"min":{"1min":0.624,"5min":0.55700000000000005,"15min":0.30199999999999999},"max":{"1min":1.3080000000000001,"5min":1.694,"15min":71.795000000000002},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.0980000000000001,"15min":1.7210000000000001},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.36299999999999999},"max":{"1min":1.3120000000000001,"5min":1.758,"15min":72.322000000000003},"last":1.369}]},{"osd":7,"last update":"Mon Oct 9 10:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.1339999999999999,"15min":1.361},"min":{"1min":0.73699999999999999,"5min":0.57599999999999996,"15min":0.374},"max":{"1min":1.276,"5min":1.8400000000000001,"15min":8.4659999999999993},"last":1.542},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.137,"15min":1.3320000000000001},"min":{"1min":0.79000000000000004,"5min":0.65500000000000003,"15min":0.374},"max":{"1min":1.284,"5min":1.702,"15min":8.8849999999999998},"last":0.95599999999999996}]}]},{"osd":1,"up_from":1246,"seq":5351529250974,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":417436,"kb_used_data":11732,"kb_used_omap":204,"kb_used_meta":405491,"kb_avail":93327716,"statfs":{"total":95995035648,"available":95567581184,"internally_reserved":0,"allocated":12013568,"data_stored":10286592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208958,"internal_metadata":415223746},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.69699999999999995,"15min":1.621},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.191},"max":{"1min":1.131,"5min":1.1659999999999999,"15min":58.033000000000001},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.70399999999999996,"15min":1.7190000000000001},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.19},"max":{"1min":1.1830000000000001,"5min":1.2709999999999999,"15min":65.921999999999997},"last":0.85199999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.75,"15min":1.589},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.26600000000000001},"max":{"1min":1.1060000000000001,"5min":1.3240000000000001,"15min":58.067999999999998},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.72599999999999998,"15min":1.5249999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.20200000000000001},"max":{"1min":1.042,"5min":1.278,"15min":58.682000000000002},"last":0.754}]},{"osd":3,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.80800000000000005,"15min":1.2150000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.20899999999999999},"max":{"1min":1.042,"5min":1.2030000000000001,"15min":8.7140000000000004},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.90700000000000003,"15min":1.198},"min":{"1min":0.42899999999999999,"5min":0.39600000000000002,"15min":0.27500000000000002},"max":{"1min":1.3,"5min":1.375,"15min":7.5780000000000003},"last":0.93400000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.94899999999999995,"15min":1.704},"min":{"1min":0.61599999999999999,"5min":0.51200000000000001,"15min":0.30599999999999999},"max":{"1min":1.2490000000000001,"5min":1.522,"15min":8.8070000000000004},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.026,"5min":1.0349999999999999,"15min":1.6659999999999999},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.32000000000000001},"max":{"1min":1.6579999999999999,"5min":1.6579999999999999,"15min":10.263},"last":1.002}]},{"osd":5,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.0529999999999999,"15min":1.4870000000000001},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.33500000000000002},"max":{"1min":1.506,"5min":1.5669999999999999,"15min":10.214},"last":1.044},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0349999999999999,"15min":1.4690000000000001},"min":{"1min":0.70999999999999996,"5min":0.65600000000000003,"15min":0.32400000000000001},"max":{"1min":1.7849999999999999,"5min":1.7849999999999999,"15min":8.5139999999999993},"last":0.88400000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0609999999999999,"15min":1.6319999999999999},"min":{"1min":0.63400000000000001,"5min":0.63,"15min":0.35199999999999998},"max":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":9.3309999999999995},"last":0.95699999999999996},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0720000000000001,"15min":1.9970000000000001},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.34799999999999998},"max":{"1min":1.488,"5min":1.756,"15min":9.8659999999999997},"last":1.085}]},{"osd":7,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.1319999999999999,"15min":1.4950000000000001},"min":{"1min":0.81399999999999995,"5min":0.70299999999999996,"15min":0.374},"max":{"1min":1.6910000000000001,"5min":1.786,"15min":9.2929999999999993},"last":1.097},{"interface":"front","average":{"1min":1.131,"5min":1.1679999999999999,"15min":1.746},"min":{"1min":0.70899999999999996,"5min":0.70399999999999996,"15min":0.371},"max":{"1min":1.5720000000000001,"5min":1.8520000000000001,"15min":9.4529999999999994},"last":1.1950000000000001}]}]},{"osd":0,"up_from":1136,"seq":4879082848568,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":762124,"kb_used_data":12292,"kb_used_omap":280,"kb_used_meta":749543,"kb_avail":92983028,"statfs":{"total":95995035648,"available":95214620672,"internally_reserved":0,"allocated":12587008,"data_stored":10867464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":287334,"internal_metadata":767532442},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:00:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.68200000000000005,"15min":1.671},"min":{"1min":0.30199999999999999,"5min":0.29599999999999999,"15min":0.19400000000000001},"max":{"1min":1.2230000000000001,"5min":1.238,"15min":69.680000000000007},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.73399999999999999,"15min":1.772},"min":{"1min":0.34799999999999998,"5min":0.33100000000000002,"15min":0.20399999999999999},"max":{"1min":1.0229999999999999,"5min":1.6120000000000001,"15min":70.668999999999997},"last":0.753}]},{"osd":2,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":1.7170000000000001,"15min":2.742},"min":{"1min":0.40100000000000002,"5min":0.38200000000000001,"15min":0.22},"max":{"1min":1.2949999999999999,"5min":80.686999999999998,"15min":254.89599999999999},"last":0.97599999999999998},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.6859999999999999,"15min":3.0859999999999999},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.23699999999999999},"max":{"1min":0.94399999999999995,"5min":80.843000000000004,"15min":256.85399999999998},"last":1.014}]},{"osd":3,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.81200000000000006,"15min":0.97199999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.247},"max":{"1min":1.151,"5min":1.5229999999999999,"15min":8.8309999999999995},"last":1.129},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.81599999999999995,"15min":0.95899999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.28299999999999997},"max":{"1min":1.1180000000000001,"5min":1.369,"15min":6.5220000000000002},"last":0.92600000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.89100000000000001,"15min":1.361},"min":{"1min":0.61399999999999999,"5min":0.57699999999999996,"15min":0.318},"max":{"1min":1.3839999999999999,"5min":1.8080000000000001,"15min":20.079000000000001},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.0820000000000001,"15min":1.3280000000000001},"min":{"1min":0.69499999999999995,"5min":0.60599999999999998,"15min":0.32700000000000001},"max":{"1min":1.476,"5min":1.7929999999999999,"15min":12.044},"last":1.389}]},{"osd":5,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.1100000000000001,"15min":1.3740000000000001},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.313},"max":{"1min":1.581,"5min":1.863,"15min":9.4280000000000008},"last":1.2050000000000001},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.0449999999999999,"15min":1.3700000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.33700000000000002},"max":{"1min":1.3480000000000001,"5min":1.6890000000000001,"15min":9.6289999999999996},"last":1.2589999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.0109999999999999,"15min":1.375},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.34300000000000003},"max":{"1min":1.45,"5min":1.5329999999999999,"15min":9.5950000000000006},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.181,"15min":1.5609999999999999},"min":{"1min":0.79600000000000004,"5min":0.71999999999999997,"15min":0.42599999999999999},"max":{"1min":1.6839999999999999,"5min":1.7350000000000001,"15min":9.3260000000000005},"last":1.319}]},{"osd":7,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.0389999999999999,"15min":1.3089999999999999},"min":{"1min":0.65600000000000003,"5min":0.61399999999999999,"15min":0.38},"max":{"1min":1.532,"5min":1.532,"15min":7.0149999999999997},"last":1.286},{"interface":"front","average":{"1min":1.127,"5min":1.222,"15min":1.355},"min":{"1min":0.76100000000000001,"5min":0.69699999999999995,"15min":0.435},"max":{"1min":1.6399999999999999,"5min":1.8520000000000001,"15min":10.087999999999999},"last":1.4379999999999999}]}]},{"osd":3,"up_from":663,"seq":2847563317798,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599676,"kb_used_data":7412,"kb_used_omap":55,"kb_used_meta":592200,"kb_avail":93145476,"statfs":{"total":95995035648,"available":95380967424,"internally_reserved":0,"allocated":7589888,"data_stored":6055072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56734,"internal_metadata":606413410},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.70499999999999996,"15min":1.6459999999999999},"min":{"1min":0.41499999999999998,"5min":0.28799999999999998,"15min":0.182},"max":{"1min":1.2010000000000001,"5min":1.2010000000000001,"15min":120.443},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.74399999999999999,"15min":1.7689999999999999},"min":{"1min":0.439,"5min":0.36899999999999999,"15min":0.20300000000000001},"max":{"1min":1.306,"5min":1.306,"15min":120.07599999999999},"last":0.64800000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:00:34 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.71099999999999997,"15min":1.7350000000000001},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.255},"max":{"1min":1.478,"5min":1.478,"15min":90.396000000000001},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.83599999999999997,"15min":1.8540000000000001},"min":{"1min":0.40999999999999998,"5min":0.29699999999999999,"15min":0.245},"max":{"1min":1.2450000000000001,"5min":1.8819999999999999,"15min":90.260000000000005},"last":0.88700000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.86399999999999999,"15min":1.5489999999999999},"min":{"1min":0.497,"5min":0.32900000000000001,"15min":0.254},"max":{"1min":1.3380000000000001,"5min":1.659,"15min":120.256},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.80300000000000005,"15min":1.651},"min":{"1min":0.46999999999999997,"5min":0.372,"15min":0.23999999999999999},"max":{"1min":1.381,"5min":1.413,"15min":119.55800000000001},"last":0.751}]},{"osd":4,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.98199999999999998,"15min":1.0209999999999999},"min":{"1min":0.624,"5min":0.624,"15min":0.317},"max":{"1min":1.3360000000000001,"5min":1.631,"15min":7.9729999999999999},"last":1.091},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0249999999999999,"15min":1.0349999999999999},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.33300000000000002},"max":{"1min":1.4019999999999999,"5min":1.7569999999999999,"15min":8.1199999999999992},"last":0.93899999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.0840000000000001,"15min":1.018},"min":{"1min":0.747,"5min":0.63100000000000001,"15min":0.33400000000000002},"max":{"1min":1.397,"5min":2.4140000000000001,"15min":8.7430000000000003},"last":1.3149999999999999},{"interface":"front","average":{"1min":1.081,"5min":1.0609999999999999,"15min":1.0129999999999999},"min":{"1min":0.753,"5min":0.61799999999999999,"15min":0.34399999999999997},"max":{"1min":1.411,"5min":1.5740000000000001,"15min":7.9459999999999997},"last":1.1519999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.107,"15min":1.5069999999999999},"min":{"1min":0.66400000000000003,"5min":0.63700000000000001,"15min":0.36099999999999999},"max":{"1min":1.6579999999999999,"5min":1.927,"15min":98.084999999999994},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.131,"5min":1.125,"15min":1.6339999999999999},"min":{"1min":0.79300000000000004,"5min":0.71099999999999997,"15min":0.40899999999999997},"max":{"1min":1.361,"5min":1.6619999999999999,"15min":99.144000000000005},"last":1.194}]},{"osd":7,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":1.143,"15min":0.98499999999999999},"min":{"1min":0.97599999999999998,"5min":0.64200000000000002,"15min":0.39000000000000001},"max":{"1min":1.427,"5min":2.3010000000000002,"15min":8.0960000000000001},"last":1.369},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.1639999999999999,"15min":0.99399999999999999},"min":{"1min":0.88200000000000001,"5min":0.70899999999999996,"15min":0.40899999999999997},"max":{"1min":1.76,"5min":1.8460000000000001,"15min":7.8879999999999999},"last":1.129}]}]},{"osd":4,"up_from":31,"seq":133143986931,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":897120,"kb_used_data":6040,"kb_used_omap":0,"kb_used_meta":891072,"kb_avail":92848032,"statfs":{"total":95995035648,"available":95076384768,"internally_reserved":0,"allocated":6184960,"data_stored":5846011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912457728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.90100000000000002,"15min":1.5229999999999999},"min":{"1min":0.52000000000000002,"5min":0.51500000000000001,"15min":0.26200000000000001},"max":{"1min":1.4419999999999999,"5min":1.4419999999999999,"15min":64.372},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.052,"5min":0.95599999999999996,"15min":1.506},"min":{"1min":0.61599999999999999,"5min":0.497,"15min":0.30099999999999999},"max":{"1min":1.3069999999999999,"5min":1.3660000000000001,"15min":64.356999999999999},"last":1.1339999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.95899999999999996,"15min":2.3090000000000002},"min":{"1min":0.59199999999999997,"5min":0.54500000000000004,"15min":0.27700000000000002},"max":{"1min":1.444,"5min":3.653,"15min":216.16900000000001},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.04,"15min":2.2400000000000002},"min":{"1min":0.74099999999999999,"5min":0.68700000000000006,"15min":0.29299999999999998},"max":{"1min":1.2869999999999999,"5min":3.5870000000000002,"15min":216.262},"last":1.2110000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.90500000000000003,"15min":1.2869999999999999},"min":{"1min":0.66700000000000004,"5min":0.53000000000000003,"15min":0.31},"max":{"1min":1.075,"5min":1.341,"15min":15.853999999999999},"last":0.995},{"interface":"front","average":{"1min":0.92900000000000005,"5min":0.95199999999999996,"15min":1.282},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.28799999999999998},"max":{"1min":1.4890000000000001,"5min":4.1449999999999996,"15min":15.881},"last":1.02}]},{"osd":3,"last update":"Mon Oct 9 10:00:16 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.93600000000000005,"15min":1.01},"min":{"1min":0.66600000000000004,"5min":0.57099999999999995,"15min":0.32200000000000001},"max":{"1min":1.286,"5min":2.7759999999999998,"15min":8.1120000000000001},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.95499999999999996,"15min":1.0620000000000001},"min":{"1min":0.63100000000000001,"5min":0.59599999999999997,"15min":0.34899999999999998},"max":{"1min":1.5980000000000001,"5min":1.645,"15min":7.4180000000000001},"last":1.149}]},{"osd":5,"last update":"Mon Oct 9 10:00:16 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.0069999999999999,"15min":1.2589999999999999},"min":{"1min":0.68700000000000006,"5min":0.57199999999999995,"15min":0.35599999999999998},"max":{"1min":1.345,"5min":1.423,"15min":11.763999999999999},"last":0.751},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.1120000000000001,"15min":1.2450000000000001},"min":{"1min":0.88900000000000001,"5min":0.58799999999999997,"15min":0.28499999999999998},"max":{"1min":1.4119999999999999,"5min":1.528,"15min":12.446999999999999},"last":1.3320000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:00:16 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0009999999999999,"15min":1.3839999999999999},"min":{"1min":0.69399999999999995,"5min":0.69299999999999995,"15min":0.32300000000000001},"max":{"1min":1.8700000000000001,"5min":1.8700000000000001,"15min":11.701000000000001},"last":1.2829999999999999},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.167,"15min":1.502},"min":{"1min":0.88900000000000001,"5min":0.79300000000000004,"15min":0.40799999999999997},"max":{"1min":2.089,"5min":2.089,"15min":11.972},"last":1.3080000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:00:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.1000000000000001,"15min":1.2030000000000001},"min":{"1min":0.88600000000000001,"5min":0.64500000000000002,"15min":0.373},"max":{"1min":1.5669999999999999,"5min":1.581,"15min":11.840999999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.129,"15min":1.1220000000000001},"min":{"1min":0.81899999999999995,"5min":0.68400000000000005,"15min":0.41299999999999998},"max":{"1min":1.542,"5min":1.615,"15min":9.6210000000000004},"last":1.0669999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790697,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":904340,"kb_used_data":6028,"kb_used_omap":0,"kb_used_meta":898304,"kb_avail":92840812,"statfs":{"total":95995035648,"available":95068991488,"internally_reserved":0,"allocated":6172672,"data_stored":5841774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919863296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.95199999999999996,"15min":2.2989999999999999},"min":{"1min":0.53400000000000003,"5min":0.41899999999999998,"15min":0.253},"max":{"1min":1.25,"5min":1.4570000000000001,"15min":121.977},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.93999999999999995,"15min":2.1120000000000001},"min":{"1min":0.57899999999999996,"5min":0.54000000000000004,"15min":0.26800000000000002},"max":{"1min":1.1719999999999999,"5min":1.351,"15min":122.13200000000001},"last":0.88500000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:00:31 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.97299999999999998,"15min":2.25},"min":{"1min":0.65600000000000003,"5min":0.57899999999999996,"15min":0.25900000000000001},"max":{"1min":1.292,"5min":1.5329999999999999,"15min":192.72200000000001},"last":1.091},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.996,"15min":2.0609999999999999},"min":{"1min":0.67400000000000004,"5min":0.66500000000000004,"15min":0.28299999999999997},"max":{"1min":1.2989999999999999,"5min":1.4159999999999999,"15min":192.69300000000001},"last":0.96099999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:00:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.95999999999999996,"15min":1.609},"min":{"1min":0.624,"5min":0.624,"15min":0.312},"max":{"1min":1.2669999999999999,"5min":1.429,"15min":122.02800000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.016,"5min":1.0680000000000001,"15min":1.8799999999999999},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.31900000000000001},"max":{"1min":1.272,"5min":1.7949999999999999,"15min":122.05800000000001},"last":1.1699999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:00:45 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.92100000000000004,"15min":0.93100000000000005},"min":{"1min":0.66800000000000004,"5min":0.59499999999999997,"15min":0.33100000000000002},"max":{"1min":1.157,"5min":1.3360000000000001,"15min":7.2270000000000003},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.04,"5min":1.052,"15min":1.1379999999999999},"min":{"1min":0.82999999999999996,"5min":0.63700000000000001,"15min":0.38},"max":{"1min":1.194,"5min":1.659,"15min":16.201000000000001},"last":1.234}]},{"osd":4,"last update":"Mon Oct 9 10:00:31 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.91300000000000003,"15min":1.097},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.309},"max":{"1min":1.1990000000000001,"5min":1.393,"15min":9.4209999999999994},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.044,"5min":1.042,"15min":1.3140000000000001},"min":{"1min":0.68000000000000005,"5min":0.498,"15min":0.318},"max":{"1min":1.4159999999999999,"5min":1.6499999999999999,"15min":16.053000000000001},"last":1.304}]},{"osd":6,"last update":"Mon Oct 9 10:00:45 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.026,"15min":1.2869999999999999},"min":{"1min":0.70299999999999996,"5min":0.47099999999999997,"15min":0.308},"max":{"1min":1.2989999999999999,"5min":1.3999999999999999,"15min":9.6270000000000007},"last":1.206},{"interface":"front","average":{"1min":1.2350000000000001,"5min":1.242,"15min":1.51},"min":{"1min":1.129,"5min":0.621,"15min":0.34399999999999997},"max":{"1min":1.4770000000000001,"5min":1.954,"15min":16.145},"last":1.3959999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:00:31 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.2210000000000001,"15min":1.2549999999999999},"min":{"1min":0.85499999999999998,"5min":0.57899999999999996,"15min":0.42399999999999999},"max":{"1min":1.5429999999999999,"5min":1.8879999999999999,"15min":16.119},"last":1.3500000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.131,"15min":1.0409999999999999},"min":{"1min":0.78000000000000003,"5min":0.59999999999999998,"15min":0.34200000000000003},"max":{"1min":1.423,"5min":1.633,"15min":4.6779999999999999},"last":1.2609999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8469,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7721,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4850,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:01:13.318 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T10:01:13.321 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T10:01:13.389 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=b6ce050b71744227ad418ac7da23b0e3 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T10:01:13.391 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T10:01:13.402 INFO:tasks.cephadm:Teardown begin 2023-10-09T10:01:13.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:01:13.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:01:13.448 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T10:01:13.449 DEBUG:teuthology.orchestra.run.smithi163:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T10:01:13.481 DEBUG:teuthology.orchestra.run.smithi181:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T10:01:13.495 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T10:01:13.496 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T10:01:13.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T10:01:13.691 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:13 smithi163 bash[71599]: ::ffff:172.21.15.181 - - [09/Oct/2023:10:01:13] "GET /metrics HTTP/1.1" 200 34034 "" "Prometheus/2.33.4" 2023-10-09T10:01:13.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:13 smithi163 systemd[1]: Stopping Ceph mon.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:01:13.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:13 smithi163 bash[75425]: debug 2023-10-09T10:01:13.683+0000 7f22f8a25700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T10:01:13.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:13 smithi163 bash[75425]: debug 2023-10-09T10:01:13.683+0000 7f22f8a25700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T10:01:13.952 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:13 smithi163 bash[71599]: [09/Oct/2023:10:01:13] ENGINE Bus STOPPING 2023-10-09T10:01:13.952 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:13 smithi163 bash[71599]: [09/Oct/2023:10:01:13] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:01:13.952 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:13 smithi163 bash[71599]: [09/Oct/2023:10:01:13] ENGINE Bus STOPPED 2023-10-09T10:01:13.952 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:13 smithi163 bash[71599]: [09/Oct/2023:10:01:13] ENGINE Bus STARTING 2023-10-09T10:01:13.952 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:13 smithi163 bash[71599]: [09/Oct/2023:10:01:13] ENGINE Serving on http://:::9283 2023-10-09T10:01:13.952 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:13 smithi163 bash[71599]: [09/Oct/2023:10:01:13] ENGINE Bus STARTED 2023-10-09T10:01:14.877 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 10:01:14 smithi163 bash[254464]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T10:01:14.884 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T10:01:14.967 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:01:14.967 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T10:01:14.967 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T10:01:14.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T10:01:15.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:14 smithi163 systemd[1]: Stopping Ceph mon.c for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:01:15.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:15 smithi163 bash[78107]: debug 2023-10-09T10:01:15.127+0000 7fe5d12c1700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T10:01:15.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:15 smithi163 bash[78107]: debug 2023-10-09T10:01:15.127+0000 7fe5d12c1700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T10:01:15.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:15 smithi163 bash[71599]: [09/Oct/2023:10:01:15] ENGINE Bus STOPPING 2023-10-09T10:01:15.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:15 smithi163 bash[71599]: [09/Oct/2023:10:01:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:01:15.469 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:15 smithi163 bash[71599]: [09/Oct/2023:10:01:15] ENGINE Bus STOPPED 2023-10-09T10:01:15.470 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:15 smithi163 bash[71599]: [09/Oct/2023:10:01:15] ENGINE Bus STARTING 2023-10-09T10:01:15.470 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:15 smithi163 bash[71599]: [09/Oct/2023:10:01:15] ENGINE Serving on http://:::9283 2023-10-09T10:01:15.470 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:15 smithi163 bash[71599]: [09/Oct/2023:10:01:15] ENGINE Bus STARTED 2023-10-09T10:01:16.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:16 smithi163 bash[254577]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T10:01:16.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:16 smithi163 bash[254651]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T10:01:16.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 10:01:16 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T10:01:16.305 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T10:01:16.342 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:01:16.342 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T10:01:16.343 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T10:01:16.343 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T10:01:16.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:16 smithi181 systemd[1]: Stopping Ceph mon.b for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:01:16.637 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:16 smithi181 bash[59345]: debug 2023-10-09T10:01:16.547+0000 7f5653fac700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T10:01:16.637 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:16 smithi181 bash[59345]: debug 2023-10-09T10:01:16.547+0000 7f5653fac700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T10:01:17.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 10:01:17 smithi181 bash[151515]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T10:01:17.627 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T10:01:17.672 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:01:17.673 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T10:01:17.673 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T10:01:17.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T10:01:17.940 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:17 smithi163 systemd[1]: Stopping Ceph mgr.y for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:01:17.941 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:17 smithi163 bash[71599]: debug 2023-10-09T10:01:17.855+0000 7f5ca5412700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:01:17.941 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 10:01:17 smithi163 bash[71599]: debug 2023-10-09T10:01:17.855+0000 7f5ca5412700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-10-09T10:01:18.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T10:01:18.913 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:01:18.913 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T10:01:18.914 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T10:01:18.914 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T10:01:19.176 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 10:01:18 smithi181 systemd[1]: Stopping Ceph mgr.x for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:01:20.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T10:01:20.039 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:01:20.040 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T10:01:20.040 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T10:01:20.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T10:01:20.481 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 10:01:20 smithi163 systemd[1]: Stopping Ceph osd.0 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:01:20.481 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 10:01:20 smithi163 bash[149053]: debug 2023-10-09T10:01:20.227+0000 7f33322d3700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:01:20.481 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 10:01:20 smithi163 bash[149053]: debug 2023-10-09T10:01:20.227+0000 7f33322d3700 -1 osd.0 1344 *** Got signal Terminated *** 2023-10-09T10:01:20.481 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 10:01:20 smithi163 bash[149053]: debug 2023-10-09T10:01:20.227+0000 7f33322d3700 -1 osd.0 1344 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:01:26.969 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 10:01:26 smithi163 bash[254792]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T10:01:26.969 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 10:01:26 smithi163 bash[254858]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T10:01:29.487 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T10:01:29.517 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:01:29.517 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T10:01:29.517 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T10:01:29.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T10:01:29.742 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 10:01:29 smithi163 systemd[1]: Stopping Ceph osd.1 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:01:30.219 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 10:01:29 smithi163 bash[204298]: debug 2023-10-09T10:01:29.734+0000 7f908cc26700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:01:30.219 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 10:01:29 smithi163 bash[204298]: debug 2023-10-09T10:01:29.734+0000 7f908cc26700 -1 osd.1 1344 *** Got signal Terminated *** 2023-10-09T10:01:30.219 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 10:01:29 smithi163 bash[204298]: debug 2023-10-09T10:01:29.734+0000 7f908cc26700 -1 osd.1 1344 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:01:36.206 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 10:01:35 smithi163 bash[254997]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T10:01:36.206 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 10:01:36 smithi163 bash[255061]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T10:01:38.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T10:01:38.229 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:01:38.229 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T10:01:38.229 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T10:01:38.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T10:01:38.452 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 10:01:38 smithi163 systemd[1]: Stopping Ceph osd.2 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:01:38.719 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 10:01:38 smithi163 bash[89638]: debug 2023-10-09T10:01:38.450+0000 7f4fac177700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:01:38.719 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 10:01:38 smithi163 bash[89638]: debug 2023-10-09T10:01:38.450+0000 7f4fac177700 -1 osd.2 1344 *** Got signal Terminated *** 2023-10-09T10:01:38.719 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 10:01:38 smithi163 bash[89638]: debug 2023-10-09T10:01:38.450+0000 7f4fac177700 -1 osd.2 1344 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:01:45.073 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 10:01:44 smithi163 bash[255202]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T10:01:45.073 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 10:01:45 smithi163 bash[255273]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T10:01:47.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:46 smithi181 bash[31288]: debug 2023-10-09T10:01:46.779+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:47.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:46 smithi181 bash[35349]: debug 2023-10-09T10:01:46.755+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:47.080 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T10:01:47.110 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:01:47.111 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T10:01:47.111 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T10:01:47.111 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T10:01:47.469 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:47 smithi163 systemd[1]: Stopping Ceph osd.3 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:01:47.469 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:47 smithi163 bash[92392]: debug 2023-10-09T10:01:47.306+0000 7fb3859c6700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:01:47.469 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:47 smithi163 bash[92392]: debug 2023-10-09T10:01:47.306+0000 7fb3859c6700 -1 osd.3 1344 *** Got signal Terminated *** 2023-10-09T10:01:47.469 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:47 smithi163 bash[92392]: debug 2023-10-09T10:01:47.306+0000 7fb3859c6700 -1 osd.3 1344 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:01:47.778 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:47 smithi181 bash[43623]: debug 2023-10-09T10:01:47.391+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:48.065 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:47 smithi181 bash[31288]: debug 2023-10-09T10:01:47.823+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:48.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:47 smithi181 bash[35349]: debug 2023-10-09T10:01:47.775+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:48.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:47 smithi181 bash[39479]: debug 2023-10-09T10:01:47.823+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:48.066 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:01:48 smithi181 bash[54907]: ts=2023-10-09T10:01:48.003Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T10:01:48.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:01:48 smithi181 bash[54907]: ts=2023-10-09T10:01:48.008Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T10:01:48.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:01:48 smithi181 bash[54907]: ts=2023-10-09T10:01:48.010Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T10:01:48.067 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:01:48 smithi181 bash[54907]: ts=2023-10-09T10:01:48.010Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T10:01:48.218 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:47 smithi163 bash[92392]: debug 2023-10-09T10:01:47.894+0000 7fb3807b9700 -1 osd.3 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.606513+0000 front 2023-10-09T10:01:21.606596+0000 (oldest deadline 2023-10-09T10:01:47.506693+0000) 2023-10-09T10:01:48.756 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:48 smithi181 bash[43623]: debug 2023-10-09T10:01:48.387+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:49.065 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:48 smithi181 bash[31288]: debug 2023-10-09T10:01:48.835+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:49.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:48 smithi181 bash[39479]: debug 2023-10-09T10:01:48.851+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:49.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:48 smithi181 bash[35349]: debug 2023-10-09T10:01:48.751+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:49.219 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:48 smithi163 bash[92392]: debug 2023-10-09T10:01:48.858+0000 7fb3807b9700 -1 osd.3 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.606513+0000 front 2023-10-09T10:01:21.606596+0000 (oldest deadline 2023-10-09T10:01:47.506693+0000) 2023-10-09T10:01:49.767 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:49 smithi181 bash[43623]: debug 2023-10-09T10:01:49.431+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:50.065 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:49 smithi181 bash[31288]: debug 2023-10-09T10:01:49.791+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:50.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:49 smithi181 bash[35349]: debug 2023-10-09T10:01:49.763+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:50.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:49 smithi181 bash[39479]: debug 2023-10-09T10:01:49.843+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:50.218 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:49 smithi163 bash[92392]: debug 2023-10-09T10:01:49.838+0000 7fb3807b9700 -1 osd.3 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.606513+0000 front 2023-10-09T10:01:21.606596+0000 (oldest deadline 2023-10-09T10:01:47.506693+0000) 2023-10-09T10:01:50.766 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:50 smithi181 bash[35349]: debug 2023-10-09T10:01:50.726+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:50.766 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:50 smithi181 bash[43623]: debug 2023-10-09T10:01:50.478+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:51.065 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:50 smithi181 bash[39479]: debug 2023-10-09T10:01:50.858+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:51.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:50 smithi181 bash[31288]: debug 2023-10-09T10:01:50.762+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:51.219 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:50 smithi163 bash[92392]: debug 2023-10-09T10:01:50.810+0000 7fb3807b9700 -1 osd.3 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.606513+0000 front 2023-10-09T10:01:21.606596+0000 (oldest deadline 2023-10-09T10:01:47.506693+0000) 2023-10-09T10:01:51.754 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:51 smithi181 bash[43623]: debug 2023-10-09T10:01:51.482+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:52.065 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:51 smithi181 bash[31288]: debug 2023-10-09T10:01:51.786+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:52.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:51 smithi181 bash[35349]: debug 2023-10-09T10:01:51.750+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:52.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:51 smithi181 bash[39479]: debug 2023-10-09T10:01:51.902+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:52.218 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:51 smithi163 bash[92392]: debug 2023-10-09T10:01:51.778+0000 7fb3807b9700 -1 osd.3 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.606513+0000 front 2023-10-09T10:01:21.606596+0000 (oldest deadline 2023-10-09T10:01:47.506693+0000) 2023-10-09T10:01:52.783 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:52 smithi181 bash[35349]: debug 2023-10-09T10:01:52.726+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:52.783 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:52 smithi181 bash[43623]: debug 2023-10-09T10:01:52.494+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:53.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:52 smithi181 bash[31288]: debug 2023-10-09T10:01:52.778+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:53.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:52 smithi181 bash[39479]: debug 2023-10-09T10:01:52.870+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:53.719 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:53 smithi181 bash[43623]: debug 2023-10-09T10:01:53.446+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:53.945 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:53 smithi163 bash[255413]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T10:01:53.945 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 10:01:53 smithi163 bash[255477]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T10:01:54.065 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:53 smithi181 bash[31288]: debug 2023-10-09T10:01:53.810+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:54.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:53 smithi181 bash[35349]: debug 2023-10-09T10:01:53.714+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:54.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:53 smithi181 bash[39479]: debug 2023-10-09T10:01:53.886+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:54.766 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:54 smithi181 bash[43623]: debug 2023-10-09T10:01:54.414+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:55.065 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:54 smithi181 bash[39479]: debug 2023-10-09T10:01:54.870+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:55.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:54 smithi181 bash[31288]: debug 2023-10-09T10:01:54.858+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:55.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:54 smithi181 bash[35349]: debug 2023-10-09T10:01:54.762+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:55.770 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:55 smithi181 bash[43623]: debug 2023-10-09T10:01:55.410+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:55.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T10:01:55.935 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:01:55.936 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T10:01:55.936 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T10:01:55.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T10:01:56.065 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:55 smithi181 bash[31288]: debug 2023-10-09T10:01:55.874+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:56.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:55 smithi181 systemd[1]: Stopping Ceph osd.4 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:01:56.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:55 smithi181 bash[35349]: debug 2023-10-09T10:01:55.766+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:56.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:55 smithi181 bash[39479]: debug 2023-10-09T10:01:55.842+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:56.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:55 smithi181 bash[39479]: debug 2023-10-09T10:01:55.842+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:01:56.413 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:56 smithi181 bash[31288]: debug 2023-10-09T10:01:56.126+0000 7ff6131f9700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:01:56.413 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:56 smithi181 bash[31288]: debug 2023-10-09T10:01:56.126+0000 7ff6131f9700 -1 osd.4 1344 *** Got signal Terminated *** 2023-10-09T10:01:56.413 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:56 smithi181 bash[31288]: debug 2023-10-09T10:01:56.126+0000 7ff6131f9700 -1 osd.4 1344 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:01:56.768 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:56 smithi181 bash[43623]: debug 2023-10-09T10:01:56.410+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:57.065 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:56 smithi181 bash[31288]: debug 2023-10-09T10:01:56.838+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:57.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:56 smithi181 bash[31288]: debug 2023-10-09T10:01:56.838+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.220304+0000 front 2023-10-09T10:01:31.220394+0000 (oldest deadline 2023-10-09T10:01:55.919862+0000) 2023-10-09T10:01:57.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:56 smithi181 bash[39479]: debug 2023-10-09T10:01:56.794+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:57.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:56 smithi181 bash[39479]: debug 2023-10-09T10:01:56.794+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:01:57.067 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:56 smithi181 bash[35349]: debug 2023-10-09T10:01:56.762+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:57.067 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:56 smithi181 bash[35349]: debug 2023-10-09T10:01:56.762+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:01:57.754 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:57 smithi181 bash[43623]: debug 2023-10-09T10:01:57.406+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:58.065 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:57 smithi181 bash[31288]: debug 2023-10-09T10:01:57.838+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:58.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:57 smithi181 bash[31288]: debug 2023-10-09T10:01:57.838+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.220304+0000 front 2023-10-09T10:01:31.220394+0000 (oldest deadline 2023-10-09T10:01:55.919862+0000) 2023-10-09T10:01:58.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:57 smithi181 bash[35349]: debug 2023-10-09T10:01:57.750+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:58.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:57 smithi181 bash[35349]: debug 2023-10-09T10:01:57.750+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:01:58.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:57 smithi181 bash[39479]: debug 2023-10-09T10:01:57.802+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:58.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:57 smithi181 bash[39479]: debug 2023-10-09T10:01:57.802+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:01:58.759 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:58 smithi181 bash[43623]: debug 2023-10-09T10:01:58.450+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:59.065 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:58 smithi181 bash[39479]: debug 2023-10-09T10:01:58.818+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:01:59.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:58 smithi181 bash[39479]: debug 2023-10-09T10:01:58.818+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:01:59.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:58 smithi181 bash[31288]: debug 2023-10-09T10:01:58.862+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:01:59.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:58 smithi181 bash[31288]: debug 2023-10-09T10:01:58.862+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.220304+0000 front 2023-10-09T10:01:31.220394+0000 (oldest deadline 2023-10-09T10:01:55.919862+0000) 2023-10-09T10:01:59.067 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:58 smithi181 bash[35349]: debug 2023-10-09T10:01:58.754+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:01:59.067 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:58 smithi181 bash[35349]: debug 2023-10-09T10:01:58.754+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:01:59.781 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:59 smithi181 bash[43623]: debug 2023-10-09T10:01:59.410+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:01:59.781 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:01:59 smithi181 bash[43623]: debug 2023-10-09T10:01:59.410+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:00.065 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:59 smithi181 bash[35349]: debug 2023-10-09T10:01:59.774+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:00.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:01:59 smithi181 bash[35349]: debug 2023-10-09T10:01:59.774+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:00.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:59 smithi181 bash[31288]: debug 2023-10-09T10:01:59.874+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:02:00.066 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:01:59 smithi181 bash[31288]: debug 2023-10-09T10:01:59.874+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.220304+0000 front 2023-10-09T10:01:31.220394+0000 (oldest deadline 2023-10-09T10:01:55.919862+0000) 2023-10-09T10:02:00.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:59 smithi181 bash[39479]: debug 2023-10-09T10:01:59.830+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:00.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:01:59 smithi181 bash[39479]: debug 2023-10-09T10:01:59.830+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:00.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:00 smithi181 bash[43623]: debug 2023-10-09T10:02:00.402+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:00.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:00 smithi181 bash[43623]: debug 2023-10-09T10:02:00.402+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:01.315 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:02:00 smithi181 bash[31288]: debug 2023-10-09T10:02:00.830+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.217874+0000 front 2023-10-09T10:01:20.217847+0000 (oldest deadline 2023-10-09T10:01:46.117539+0000) 2023-10-09T10:02:01.315 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:02:00 smithi181 bash[31288]: debug 2023-10-09T10:02:00.830+0000 7ff60c16e700 -1 osd.4 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.220304+0000 front 2023-10-09T10:01:31.220394+0000 (oldest deadline 2023-10-09T10:01:55.919862+0000) 2023-10-09T10:02:01.315 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:00 smithi181 bash[35349]: debug 2023-10-09T10:02:00.814+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:01.315 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:00 smithi181 bash[35349]: debug 2023-10-09T10:02:00.814+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:01.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:00 smithi181 bash[39479]: debug 2023-10-09T10:02:00.878+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:01.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:00 smithi181 bash[39479]: debug 2023-10-09T10:02:00.878+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:01.565 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:01 smithi181 bash[43623]: debug 2023-10-09T10:02:01.374+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:01.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:01 smithi181 bash[43623]: debug 2023-10-09T10:02:01.374+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:02.065 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:02:01 smithi181 bash[151735]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T10:02:02.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:01 smithi181 bash[35349]: debug 2023-10-09T10:02:01.778+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:02.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:01 smithi181 bash[35349]: debug 2023-10-09T10:02:01.778+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:02.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:01 smithi181 bash[39479]: debug 2023-10-09T10:02:01.866+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:02.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:01 smithi181 bash[39479]: debug 2023-10-09T10:02:01.866+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:02.415 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 10:02:02 smithi181 bash[151796]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T10:02:02.565 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:02 smithi181 bash[43623]: debug 2023-10-09T10:02:02.410+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:02.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:02 smithi181 bash[43623]: debug 2023-10-09T10:02:02.410+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:03.065 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:02 smithi181 bash[39479]: debug 2023-10-09T10:02:02.878+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:03.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:02 smithi181 bash[39479]: debug 2023-10-09T10:02:02.878+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:03.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:02 smithi181 bash[35349]: debug 2023-10-09T10:02:02.734+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:03.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:02 smithi181 bash[35349]: debug 2023-10-09T10:02:02.734+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:03.741 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:03 smithi181 bash[43623]: debug 2023-10-09T10:02:03.426+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:03.742 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:03 smithi181 bash[43623]: debug 2023-10-09T10:02:03.426+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:04.028 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:03 smithi181 bash[35349]: debug 2023-10-09T10:02:03.734+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:04.028 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:03 smithi181 bash[35349]: debug 2023-10-09T10:02:03.734+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:04.029 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:03 smithi181 bash[39479]: debug 2023-10-09T10:02:03.834+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:04.029 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:03 smithi181 bash[39479]: debug 2023-10-09T10:02:03.834+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:04.490 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T10:02:04.521 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:02:04.521 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T10:02:04.521 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T10:02:04.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T10:02:04.732 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:04 smithi181 systemd[1]: Stopping Ceph osd.5 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:02:04.733 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[43623]: debug 2023-10-09T10:02:04.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:04.733 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[43623]: debug 2023-10-09T10:02:04.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:04.733 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[43623]: debug 2023-10-09T10:02:04.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:05.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[39479]: debug 2023-10-09T10:02:04.870+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:05.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[39479]: debug 2023-10-09T10:02:04.870+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:05.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[35349]: debug 2023-10-09T10:02:04.726+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:05.067 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[35349]: debug 2023-10-09T10:02:04.726+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:05.067 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[35349]: debug 2023-10-09T10:02:04.726+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:42.413951+0000 front 2023-10-09T10:01:42.413920+0000 (oldest deadline 2023-10-09T10:02:04.713575+0000) 2023-10-09T10:02:05.067 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[35349]: debug 2023-10-09T10:02:04.742+0000 7fcd8db3c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:02:05.067 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[35349]: debug 2023-10-09T10:02:04.742+0000 7fcd8db3c700 -1 osd.5 1344 *** Got signal Terminated *** 2023-10-09T10:02:05.067 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:04 smithi181 bash[35349]: debug 2023-10-09T10:02:04.742+0000 7fcd8db3c700 -1 osd.5 1344 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:02:05.782 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:05 smithi181 bash[43623]: debug 2023-10-09T10:02:05.426+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:05.782 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:05 smithi181 bash[43623]: debug 2023-10-09T10:02:05.426+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:05.783 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:05 smithi181 bash[43623]: debug 2023-10-09T10:02:05.426+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:06.065 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:05 smithi181 bash[35349]: debug 2023-10-09T10:02:05.778+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:06.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:05 smithi181 bash[35349]: debug 2023-10-09T10:02:05.778+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:06.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:05 smithi181 bash[35349]: debug 2023-10-09T10:02:05.778+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:42.413951+0000 front 2023-10-09T10:01:42.413920+0000 (oldest deadline 2023-10-09T10:02:04.713575+0000) 2023-10-09T10:02:06.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:05 smithi181 bash[39479]: debug 2023-10-09T10:02:05.830+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:06.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:05 smithi181 bash[39479]: debug 2023-10-09T10:02:05.830+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:06.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:05 smithi181 bash[39479]: debug 2023-10-09T10:02:05.830+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:06.748 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:06 smithi181 bash[43623]: debug 2023-10-09T10:02:06.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:06.748 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:06 smithi181 bash[43623]: debug 2023-10-09T10:02:06.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:06.748 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:06 smithi181 bash[43623]: debug 2023-10-09T10:02:06.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:07.065 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:06 smithi181 bash[35349]: debug 2023-10-09T10:02:06.742+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:07.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:06 smithi181 bash[35349]: debug 2023-10-09T10:02:06.742+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:07.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:06 smithi181 bash[35349]: debug 2023-10-09T10:02:06.742+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:42.413951+0000 front 2023-10-09T10:01:42.413920+0000 (oldest deadline 2023-10-09T10:02:04.713575+0000) 2023-10-09T10:02:07.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:06 smithi181 bash[39479]: debug 2023-10-09T10:02:06.814+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:07.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:06 smithi181 bash[39479]: debug 2023-10-09T10:02:06.814+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:07.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:06 smithi181 bash[39479]: debug 2023-10-09T10:02:06.814+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:07.718 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:07 smithi181 bash[43623]: debug 2023-10-09T10:02:07.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:07.719 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:07 smithi181 bash[43623]: debug 2023-10-09T10:02:07.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:07.719 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:07 smithi181 bash[43623]: debug 2023-10-09T10:02:07.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:08.065 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:07 smithi181 bash[35349]: debug 2023-10-09T10:02:07.714+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:08.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:07 smithi181 bash[35349]: debug 2023-10-09T10:02:07.714+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:08.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:07 smithi181 bash[35349]: debug 2023-10-09T10:02:07.714+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:42.413951+0000 front 2023-10-09T10:01:42.413920+0000 (oldest deadline 2023-10-09T10:02:04.713575+0000) 2023-10-09T10:02:08.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:07 smithi181 bash[39479]: debug 2023-10-09T10:02:07.858+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:08.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:07 smithi181 bash[39479]: debug 2023-10-09T10:02:07.858+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:08.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:07 smithi181 bash[39479]: debug 2023-10-09T10:02:07.858+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:08.752 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:08 smithi181 bash[43623]: debug 2023-10-09T10:02:08.482+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:08.752 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:08 smithi181 bash[43623]: debug 2023-10-09T10:02:08.482+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:08.752 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:08 smithi181 bash[43623]: debug 2023-10-09T10:02:08.482+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:09.065 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:08 smithi181 bash[35349]: debug 2023-10-09T10:02:08.746+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:09.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:08 smithi181 bash[35349]: debug 2023-10-09T10:02:08.746+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:09.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:08 smithi181 bash[35349]: debug 2023-10-09T10:02:08.746+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:42.413951+0000 front 2023-10-09T10:01:42.413920+0000 (oldest deadline 2023-10-09T10:02:04.713575+0000) 2023-10-09T10:02:09.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:08 smithi181 bash[39479]: debug 2023-10-09T10:02:08.822+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:09.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:08 smithi181 bash[39479]: debug 2023-10-09T10:02:08.822+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:09.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:08 smithi181 bash[39479]: debug 2023-10-09T10:02:08.822+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:09.740 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:09 smithi181 bash[43623]: debug 2023-10-09T10:02:09.430+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:09.740 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:09 smithi181 bash[43623]: debug 2023-10-09T10:02:09.430+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:09.741 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:09 smithi181 bash[43623]: debug 2023-10-09T10:02:09.430+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:10.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:09 smithi181 bash[35349]: debug 2023-10-09T10:02:09.738+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.810030+0000 front 2023-10-09T10:01:20.809912+0000 (oldest deadline 2023-10-09T10:01:46.109329+0000) 2023-10-09T10:02:10.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:09 smithi181 bash[35349]: debug 2023-10-09T10:02:09.738+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:31.911790+0000 front 2023-10-09T10:01:31.911939+0000 (oldest deadline 2023-10-09T10:01:56.011546+0000) 2023-10-09T10:02:10.066 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:09 smithi181 bash[35349]: debug 2023-10-09T10:02:09.738+0000 7fcd86ab1700 -1 osd.5 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:42.413951+0000 front 2023-10-09T10:01:42.413920+0000 (oldest deadline 2023-10-09T10:02:04.713575+0000) 2023-10-09T10:02:10.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:09 smithi181 bash[39479]: debug 2023-10-09T10:02:09.814+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:10.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:09 smithi181 bash[39479]: debug 2023-10-09T10:02:09.814+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:10.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:09 smithi181 bash[39479]: debug 2023-10-09T10:02:09.814+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:10.771 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:10 smithi181 bash[151936]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T10:02:10.771 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:10 smithi181 bash[43623]: debug 2023-10-09T10:02:10.466+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:10.771 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:10 smithi181 bash[43623]: debug 2023-10-09T10:02:10.466+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:10.772 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:10 smithi181 bash[43623]: debug 2023-10-09T10:02:10.466+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:11.065 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 10:02:10 smithi181 bash[152004]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T10:02:11.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:10 smithi181 bash[39479]: debug 2023-10-09T10:02:10.774+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:11.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:10 smithi181 bash[39479]: debug 2023-10-09T10:02:10.774+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:11.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:10 smithi181 bash[39479]: debug 2023-10-09T10:02:10.774+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:11.815 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:11 smithi181 bash[39479]: debug 2023-10-09T10:02:11.750+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:11.816 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:11 smithi181 bash[39479]: debug 2023-10-09T10:02:11.750+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:11.816 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:11 smithi181 bash[39479]: debug 2023-10-09T10:02:11.750+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:11.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:11 smithi181 bash[43623]: debug 2023-10-09T10:02:11.506+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:11.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:11 smithi181 bash[43623]: debug 2023-10-09T10:02:11.506+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:11.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:11 smithi181 bash[43623]: debug 2023-10-09T10:02:11.506+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:12.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T10:02:12.687 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:02:12.688 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T10:02:12.688 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T10:02:12.688 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T10:02:12.741 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:12 smithi181 bash[43623]: debug 2023-10-09T10:02:12.470+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:12.741 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:12 smithi181 bash[43623]: debug 2023-10-09T10:02:12.470+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:12.741 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:12 smithi181 bash[43623]: debug 2023-10-09T10:02:12.470+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:13.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:12 smithi181 systemd[1]: Stopping Ceph osd.6 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:02:13.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:12 smithi181 bash[39479]: debug 2023-10-09T10:02:12.786+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:13.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:12 smithi181 bash[39479]: debug 2023-10-09T10:02:12.786+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:13.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:12 smithi181 bash[39479]: debug 2023-10-09T10:02:12.786+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:13.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:12 smithi181 bash[39479]: debug 2023-10-09T10:02:12.910+0000 7f3b37dce700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:02:13.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:12 smithi181 bash[39479]: debug 2023-10-09T10:02:12.910+0000 7f3b37dce700 -1 osd.6 1344 *** Got signal Terminated *** 2023-10-09T10:02:13.067 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:12 smithi181 bash[39479]: debug 2023-10-09T10:02:12.910+0000 7f3b37dce700 -1 osd.6 1344 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:02:13.799 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:13 smithi181 bash[43623]: debug 2023-10-09T10:02:13.478+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:13.800 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:13 smithi181 bash[43623]: debug 2023-10-09T10:02:13.478+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:13.800 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:13 smithi181 bash[43623]: debug 2023-10-09T10:02:13.478+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:14.065 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:13 smithi181 bash[39479]: debug 2023-10-09T10:02:13.798+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:14.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:13 smithi181 bash[39479]: debug 2023-10-09T10:02:13.798+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:14.066 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:13 smithi181 bash[39479]: debug 2023-10-09T10:02:13.798+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:14.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:14 smithi181 bash[43623]: debug 2023-10-09T10:02:14.450+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:14.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:14 smithi181 bash[43623]: debug 2023-10-09T10:02:14.450+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:14.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:14 smithi181 bash[43623]: debug 2023-10-09T10:02:14.450+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:15.315 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:14 smithi181 bash[39479]: debug 2023-10-09T10:02:14.818+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:15.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:14 smithi181 bash[39479]: debug 2023-10-09T10:02:14.818+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:15.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:14 smithi181 bash[39479]: debug 2023-10-09T10:02:14.818+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:15.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:14 smithi181 bash[39479]: debug 2023-10-09T10:02:14.818+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:48.927871+0000 front 2023-10-09T10:01:48.927922+0000 (oldest deadline 2023-10-09T10:02:14.227703+0000) 2023-10-09T10:02:15.815 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:15 smithi181 bash[43623]: debug 2023-10-09T10:02:15.414+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:15.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:15 smithi181 bash[43623]: debug 2023-10-09T10:02:15.414+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:15.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:15 smithi181 bash[43623]: debug 2023-10-09T10:02:15.414+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:16.315 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:15 smithi181 bash[39479]: debug 2023-10-09T10:02:15.846+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:16.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:15 smithi181 bash[39479]: debug 2023-10-09T10:02:15.846+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:16.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:15 smithi181 bash[39479]: debug 2023-10-09T10:02:15.846+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:16.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:15 smithi181 bash[39479]: debug 2023-10-09T10:02:15.846+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:48.927871+0000 front 2023-10-09T10:01:48.927922+0000 (oldest deadline 2023-10-09T10:02:14.227703+0000) 2023-10-09T10:02:16.815 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:16 smithi181 bash[43623]: debug 2023-10-09T10:02:16.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:16.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:16 smithi181 bash[43623]: debug 2023-10-09T10:02:16.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:16.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:16 smithi181 bash[43623]: debug 2023-10-09T10:02:16.458+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:17.315 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:16 smithi181 bash[39479]: debug 2023-10-09T10:02:16.846+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:17.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:16 smithi181 bash[39479]: debug 2023-10-09T10:02:16.846+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:17.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:16 smithi181 bash[39479]: debug 2023-10-09T10:02:16.846+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:17.316 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:16 smithi181 bash[39479]: debug 2023-10-09T10:02:16.846+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:48.927871+0000 front 2023-10-09T10:01:48.927922+0000 (oldest deadline 2023-10-09T10:02:14.227703+0000) 2023-10-09T10:02:17.815 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:17 smithi181 bash[43623]: debug 2023-10-09T10:02:17.478+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:17.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:17 smithi181 bash[43623]: debug 2023-10-09T10:02:17.478+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:17.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:17 smithi181 bash[43623]: debug 2023-10-09T10:02:17.478+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:17.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:17 smithi181 bash[43623]: debug 2023-10-09T10:02:17.478+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:51.979535+0000 front 2023-10-09T10:01:51.979446+0000 (oldest deadline 2023-10-09T10:02:17.279280+0000) 2023-10-09T10:02:18.180 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:17 smithi181 bash[39479]: debug 2023-10-09T10:02:17.886+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:20.422301+0000 front 2023-10-09T10:01:20.422266+0000 (oldest deadline 2023-10-09T10:01:47.422429+0000) 2023-10-09T10:02:18.180 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:17 smithi181 bash[39479]: debug 2023-10-09T10:02:17.886+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:33.824523+0000 front 2023-10-09T10:01:33.824692+0000 (oldest deadline 2023-10-09T10:01:55.524425+0000) 2023-10-09T10:02:18.180 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:17 smithi181 bash[39479]: debug 2023-10-09T10:02:17.886+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:40.126614+0000 front 2023-10-09T10:01:40.126686+0000 (oldest deadline 2023-10-09T10:02:05.426447+0000) 2023-10-09T10:02:18.180 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:17 smithi181 bash[39479]: debug 2023-10-09T10:02:17.886+0000 7f3b34d4b700 -1 osd.6 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:48.927871+0000 front 2023-10-09T10:01:48.927922+0000 (oldest deadline 2023-10-09T10:02:14.227703+0000) 2023-10-09T10:02:18.565 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:18 smithi181 bash[43623]: debug 2023-10-09T10:02:18.494+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:18.689 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:18 smithi181 bash[43623]: debug 2023-10-09T10:02:18.494+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:18.689 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:18 smithi181 bash[43623]: debug 2023-10-09T10:02:18.494+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:18.689 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:18 smithi181 bash[43623]: debug 2023-10-09T10:02:18.494+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:51.979535+0000 front 2023-10-09T10:01:51.979446+0000 (oldest deadline 2023-10-09T10:02:17.279280+0000) 2023-10-09T10:02:18.815 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:18 smithi181 bash[152145]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T10:02:19.315 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 10:02:18 smithi181 bash[152209]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T10:02:19.763 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:19 smithi181 bash[43623]: debug 2023-10-09T10:02:19.462+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:19.763 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:19 smithi181 bash[43623]: debug 2023-10-09T10:02:19.462+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:19.763 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:19 smithi181 bash[43623]: debug 2023-10-09T10:02:19.462+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:19.764 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:19 smithi181 bash[43623]: debug 2023-10-09T10:02:19.462+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:51.979535+0000 front 2023-10-09T10:01:51.979446+0000 (oldest deadline 2023-10-09T10:02:17.279280+0000) 2023-10-09T10:02:20.565 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:20 smithi181 bash[43623]: debug 2023-10-09T10:02:20.418+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:20.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:20 smithi181 bash[43623]: debug 2023-10-09T10:02:20.418+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:20.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:20 smithi181 bash[43623]: debug 2023-10-09T10:02:20.418+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:20.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:20 smithi181 bash[43623]: debug 2023-10-09T10:02:20.418+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:51.979535+0000 front 2023-10-09T10:01:51.979446+0000 (oldest deadline 2023-10-09T10:02:17.279280+0000) 2023-10-09T10:02:20.899 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T10:02:20.973 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:02:20.974 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T10:02:20.974 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T10:02:20.974 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T10:02:21.181 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:21 smithi181 systemd[1]: Stopping Ceph osd.7 for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:02:21.565 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:21 smithi181 bash[43623]: debug 2023-10-09T10:02:21.174+0000 7f9af01d7700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:02:21.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:21 smithi181 bash[43623]: debug 2023-10-09T10:02:21.174+0000 7f9af01d7700 -1 osd.7 1344 *** Got signal Terminated *** 2023-10-09T10:02:21.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:21 smithi181 bash[43623]: debug 2023-10-09T10:02:21.174+0000 7f9af01d7700 -1 osd.7 1344 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:02:21.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:21 smithi181 bash[43623]: debug 2023-10-09T10:02:21.418+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:21.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:21 smithi181 bash[43623]: debug 2023-10-09T10:02:21.418+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:21.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:21 smithi181 bash[43623]: debug 2023-10-09T10:02:21.418+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:21.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:21 smithi181 bash[43623]: debug 2023-10-09T10:02:21.418+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:51.979535+0000 front 2023-10-09T10:01:51.979446+0000 (oldest deadline 2023-10-09T10:02:17.279280+0000) 2023-10-09T10:02:22.815 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:22 smithi181 bash[43623]: debug 2023-10-09T10:02:22.390+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:22.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:22 smithi181 bash[43623]: debug 2023-10-09T10:02:22.390+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:22.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:22 smithi181 bash[43623]: debug 2023-10-09T10:02:22.390+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:22.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:22 smithi181 bash[43623]: debug 2023-10-09T10:02:22.390+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:51.979535+0000 front 2023-10-09T10:01:51.979446+0000 (oldest deadline 2023-10-09T10:02:17.279280+0000) 2023-10-09T10:02:23.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:23 smithi181 bash[43623]: debug 2023-10-09T10:02:23.394+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:23.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:23 smithi181 bash[43623]: debug 2023-10-09T10:02:23.394+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:23.817 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:23 smithi181 bash[43623]: debug 2023-10-09T10:02:23.394+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:23.817 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:23 smithi181 bash[43623]: debug 2023-10-09T10:02:23.394+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:51.979535+0000 front 2023-10-09T10:01:51.979446+0000 (oldest deadline 2023-10-09T10:02:17.279280+0000) 2023-10-09T10:02:24.815 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:24 smithi181 bash[43623]: debug 2023-10-09T10:02:24.350+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:24.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:24 smithi181 bash[43623]: debug 2023-10-09T10:02:24.350+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:24.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:24 smithi181 bash[43623]: debug 2023-10-09T10:02:24.350+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:24.816 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:24 smithi181 bash[43623]: debug 2023-10-09T10:02:24.350+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:51.979535+0000 front 2023-10-09T10:01:51.979446+0000 (oldest deadline 2023-10-09T10:02:17.279280+0000) 2023-10-09T10:02:25.565 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:25 smithi181 bash[43623]: debug 2023-10-09T10:02:25.302+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T10:01:21.772734+0000 front 2023-10-09T10:01:21.772701+0000 (oldest deadline 2023-10-09T10:01:47.072357+0000) 2023-10-09T10:02:25.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:25 smithi181 bash[43623]: debug 2023-10-09T10:02:25.302+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T10:01:32.875235+0000 front 2023-10-09T10:01:32.874770+0000 (oldest deadline 2023-10-09T10:01:58.774691+0000) 2023-10-09T10:02:25.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:25 smithi181 bash[43623]: debug 2023-10-09T10:02:25.302+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T10:01:43.277627+0000 front 2023-10-09T10:01:43.277792+0000 (oldest deadline 2023-10-09T10:02:04.377278+0000) 2023-10-09T10:02:25.566 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:25 smithi181 bash[43623]: debug 2023-10-09T10:02:25.302+0000 7f9aeb951700 -1 osd.7 1344 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T10:01:51.979535+0000 front 2023-10-09T10:01:51.979446+0000 (oldest deadline 2023-10-09T10:02:17.279280+0000) 2023-10-09T10:02:27.234 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:26 smithi181 bash[152352]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T10:02:27.234 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 10:02:27 smithi181 bash[152418]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T10:02:29.090 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T10:02:29.120 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:02:29.120 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T10:02:29.121 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T10:02:29.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-097c0480-6681-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T10:02:29.364 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 systemd[1]: Stopping Ceph prometheus.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:02:29.365 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.330Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T10:02:29.365 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.330Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T10:02:29.365 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.330Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T10:02:29.365 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.330Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T10:02:29.365 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.330Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T10:02:29.366 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.330Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T10:02:29.366 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.331Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T10:02:29.366 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.331Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T10:02:29.366 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.331Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T10:02:29.366 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.336Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T10:02:29.366 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.336Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T10:02:29.366 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 10:02:29 smithi181 bash[54907]: ts=2023-10-09T10:02:29.336Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T10:02:30.082 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-097c0480-6681-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T10:02:30.111 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:02:30.112 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T10:02:30.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T10:02:36.969 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 10:02:36 smithi163 systemd[1]: Stopping Ceph node-exporter.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:02:37.509 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 10:02:37 smithi163 bash[255840]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T10:02:37.781 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 10:02:37 smithi163 bash[255904]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T10:02:37.781 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 10:02:37 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T10:02:37.781 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 10:02:37 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T10:02:37.782 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 10:02:37 smithi163 systemd[1]: Stopped Ceph node-exporter.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T10:02:38.182 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 10:02:37 smithi163 systemd[1]: Stopping Ceph alertmanager.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:02:38.182 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 10:02:37 smithi163 bash[70722]: level=info ts=2023-10-09T10:02:37.923Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T10:02:38.969 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 10:02:38 smithi163 bash[255967]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T10:02:38.969 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 10:02:38 smithi163 bash[256030]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T10:02:38.969 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 10:02:38 smithi163 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T10:02:38.969 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 10:02:38 smithi163 systemd[1]: Stopped Ceph alertmanager.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T10:02:47.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T10:02:53.816 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 10:02:53 smithi181 systemd[1]: Stopping Ceph node-exporter.b for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:02:54.565 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 10:02:54 smithi181 bash[152886]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T10:02:54.566 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 10:02:54 smithi181 bash[152947]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T10:02:54.566 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 10:02:54 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T10:02:54.566 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 10:02:54 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T10:02:54.566 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 10:02:54 smithi181 systemd[1]: Stopped Ceph node-exporter.b for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T10:02:55.066 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 10:02:54 smithi181 systemd[1]: Stopping Ceph grafana.a for 097c0480-6681-11ee-8db5-212e2dc638e7... 2023-10-09T10:02:55.066 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 10:02:54 smithi181 bash[53532]: t=2023-10-09T10:02:54+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T10:02:55.815 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 10:02:55 smithi181 bash[153008]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T10:02:55.816 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 10:02:55 smithi181 bash[153070]: Error response from daemon: No such container: ceph-097c0480-6681-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T10:02:55.816 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 10:02:55 smithi181 systemd[1]: ceph-097c0480-6681-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T10:02:55.816 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 10:02:55 smithi181 systemd[1]: Stopped Ceph grafana.a for 097c0480-6681-11ee-8db5-212e2dc638e7. 2023-10-09T10:03:01.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:03:01.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:03:01.239 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T10:03:01.241 DEBUG:teuthology.misc:Transferring archived files from smithi163:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607/remote/smithi163/crash 2023-10-09T10:03:01.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo tar c -f - -C /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T10:03:01.290 INFO:teuthology.orchestra.run.smithi163.stderr:tar: /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T10:03:01.291 INFO:teuthology.orchestra.run.smithi163.stderr:tar: Error is not recoverable: exiting now 2023-10-09T10:03:01.292 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607/remote/smithi181/crash 2023-10-09T10:03:01.293 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T10:03:01.325 INFO:teuthology.orchestra.run.smithi181.stderr:tar: /var/lib/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T10:03:01.325 INFO:teuthology.orchestra.run.smithi181.stderr:tar: Error is not recoverable: exiting now 2023-10-09T10:03:01.326 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T10:03:01.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T10:03:01.353 INFO:teuthology.orchestra.run.smithi163.stderr:grep: /var/log/ceph/097c0480-6681-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T10:03:01.355 INFO:tasks.cephadm:Compressing logs... 2023-10-09T10:03:01.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T10:03:01.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T10:03:01.414 INFO:teuthology.orchestra.run.smithi163.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-09T10:03:01.417 INFO:teuthology.orchestra.run.smithi181.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-09T10:10:27.602 INFO:tasks.cephadm:Archiving logs... 2023-10-09T10:10:27.603 DEBUG:teuthology.misc:Transferring archived files from smithi163:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607/remote/smithi163/log 2023-10-09T10:10:27.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T10:11:20.566 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607/remote/smithi181/log 2023-10-09T10:11:20.568 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T10:12:01.215 INFO:tasks.cephadm:Removing cluster... 2023-10-09T10:12:01.216 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 --force 2023-10-09T10:12:03.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 097c0480-6681-11ee-8db5-212e2dc638e7 --force 2023-10-09T10:12:05.265 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T10:12:05.265 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T10:12:05.291 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T10:12:05.318 INFO:tasks.cephadm:Teardown complete 2023-10-09T10:12:05.318 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T10:12:05.329 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T10:12:05.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T10:12:05.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T10:12:05.427 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T10:12:05.436 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T10:12:05.437 DEBUG:teuthology.orchestra.run.smithi163:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T10:12:05.439 DEBUG:teuthology.orchestra.run.smithi181:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T10:12:05.493 INFO:teuthology.orchestra.run.smithi163.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T10:12:05.493 INFO:teuthology.orchestra.run.smithi163.stdout:============================================================================== 2023-10-09T10:12:05.494 INFO:teuthology.orchestra.run.smithi163.stdout:*hv01.front.sepi 44.4.53.2 2 u 29 128 377 0.201 0.726 0.269 2023-10-09T10:12:05.494 INFO:teuthology.orchestra.run.smithi163.stdout:+hv02.front.sepi 63.231.80.2 3 u 36 128 377 0.198 -0.241 0.273 2023-10-09T10:12:05.494 INFO:teuthology.orchestra.run.smithi163.stdout:+hv03.front.sepi 74.6.168.72 3 u 31 128 377 0.139 1.237 0.291 2023-10-09T10:12:05.494 INFO:teuthology.orchestra.run.smithi163.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-09T10:12:05.496 INFO:teuthology.orchestra.run.smithi181.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T10:12:05.496 INFO:teuthology.orchestra.run.smithi181.stdout:============================================================================== 2023-10-09T10:12:05.496 INFO:teuthology.orchestra.run.smithi181.stdout:*hv01.front.sepi 44.4.53.2 2 u 38 128 377 0.083 0.024 0.105 2023-10-09T10:12:05.496 INFO:teuthology.orchestra.run.smithi181.stdout:+hv02.front.sepi 63.231.80.2 3 u 44 64 377 0.113 -0.948 0.104 2023-10-09T10:12:05.496 INFO:teuthology.orchestra.run.smithi181.stdout:+hv03.front.sepi 74.6.168.72 3 u 27 128 377 0.109 0.200 0.064 2023-10-09T10:12:05.496 INFO:teuthology.orchestra.run.smithi181.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-09T10:12:05.497 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T10:12:05.507 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T10:12:05.508 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T10:12:05.517 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T10:12:05.526 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T10:12:05.535 INFO:teuthology.task.internal:Duration was 5315.680256 seconds 2023-10-09T10:12:05.535 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T10:12:05.544 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T10:12:05.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T10:12:05.547 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T10:12:05.677 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T10:12:05.677 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi163.front.sepia.ceph.com 2023-10-09T10:12:05.677 DEBUG:teuthology.orchestra.run.smithi163:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T10:12:05.698 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi181.front.sepia.ceph.com 2023-10-09T10:12:05.698 DEBUG:teuthology.orchestra.run.smithi181:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T10:12:05.720 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T10:12:05.720 DEBUG:teuthology.orchestra.run.smithi163:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T10:12:05.740 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T10:12:05.943 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T10:12:05.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T10:12:05.945 DEBUG:teuthology.orchestra.run.smithi181:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T10:12:06.421 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T10:12:06.433 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T10:12:06.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T10:12:06.480 DEBUG:teuthology.orchestra.run.smithi181:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T10:12:06.517 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T10:12:06.527 DEBUG:teuthology.orchestra.run.smithi163:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T10:12:06.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T10:12:06.548 INFO:teuthology.orchestra.run.smithi163.stdout:kernel.core_pattern = core 2023-10-09T10:12:06.575 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = core 2023-10-09T10:12:06.620 DEBUG:teuthology.orchestra.run.smithi163:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T10:12:06.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:12:06.626 DEBUG:teuthology.orchestra.run.smithi181:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T10:12:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:12:06.666 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T10:12:06.677 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T10:12:06.678 DEBUG:teuthology.misc:Transferring archived files from smithi163:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607/remote/smithi163 2023-10-09T10:12:06.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T10:12:06.731 DEBUG:teuthology.misc:Transferring archived files from smithi181:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607/remote/smithi181 2023-10-09T10:12:06.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T10:12:06.774 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T10:12:06.774 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T10:12:06.777 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T10:12:06.815 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T10:12:06.828 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T10:12:06.828 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T10:12:06.839 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T10:12:06.839 DEBUG:teuthology.orchestra.run.smithi163:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T10:12:06.842 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T10:12:06.848 INFO:teuthology.orchestra.run.smithi163.stdout: 1048618 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 10:12 /home/ubuntu/cephtest 2023-10-09T10:12:06.848 INFO:teuthology.orchestra.run.smithi163.stdout: 1441984 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 09:56 /home/ubuntu/cephtest/mnt.0 2023-10-09T10:12:06.848 INFO:teuthology.orchestra.run.smithi163.stdout: 1441985 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 09:56 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:12:06.849 INFO:teuthology.orchestra.run.smithi163.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T10:12:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:12:06.858 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi163 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T10:12:06.865 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T10:12:06.876 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T10:12:06.886 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi163 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T10:12:06.887 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T10:12:06.901 INFO:teuthology.orchestra.run.smithi181.stdout: 1048617 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 10:12 /home/ubuntu/cephtest 2023-10-09T10:12:06.932 DEBUG:teuthology.nuke:shortname: smithi163 2023-10-09T10:12:06.933 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T10:12:06.956 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi163.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 08:33:16.710457', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-09T10:12:06.959 DEBUG:teuthology.nuke:shortname: smithi181 2023-10-09T10:12:06.960 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T10:12:06.981 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi181.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417607', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 08:33:16.713244', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-09T10:12:07.011 INFO:teuthology.orchestra.console.smithi163:Power off 2023-10-09T10:12:07.012 DEBUG:teuthology.orchestra.console.smithi163:pexpect command: ipmitool -H smithi163.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T10:12:07.027 INFO:teuthology.orchestra.console.smithi181:Power off 2023-10-09T10:12:07.028 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T10:12:07.044 DEBUG:teuthology.orchestra.console.smithi163:power off output: Chassis Power Control: Down/Off 2023-10-09T10:12:07.045 DEBUG:teuthology.orchestra.console.smithi163:pexpect command: ipmitool -H smithi163.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:12:07.055 DEBUG:teuthology.orchestra.console.smithi181:power off output: Chassis Power Control: Down/Off 2023-10-09T10:12:07.055 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:12:07.069 DEBUG:teuthology.orchestra.console.smithi163:check power output: Chassis Power is on 2023-10-09T10:12:07.081 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is on 2023-10-09T10:12:11.070 DEBUG:teuthology.orchestra.console.smithi163:pexpect command: ipmitool -H smithi163.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:12:11.082 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:12:11.180 DEBUG:teuthology.orchestra.console.smithi163:check power output: Chassis Power is on 2023-10-09T10:12:11.191 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is on 2023-10-09T10:12:15.185 DEBUG:teuthology.orchestra.console.smithi163:pexpect command: ipmitool -H smithi163.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:12:15.194 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:12:15.296 DEBUG:teuthology.orchestra.console.smithi163:check power output: Chassis Power is off 2023-10-09T10:12:15.305 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is off 2023-10-09T10:12:15.397 INFO:teuthology.orchestra.console.smithi163:Power off completed 2023-10-09T10:12:15.406 INFO:teuthology.orchestra.console.smithi181:Power off completed 2023-10-09T10:12:15.510 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 5315.68025636673 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=b6ce050b71744227ad418ac7da23b0e3 status: fail success: false 2023-10-09T10:12:15.511 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T10:12:15.587 INFO:teuthology.run:FAIL